[llvm] [X86] check test changes due to legalising funnel shifts (PR #186673)
Gergo Stomfai via llvm-commits
llvm-commits at lists.llvm.org
Fri May 1 07:58:43 PDT 2026
https://github.com/stomfaig updated https://github.com/llvm/llvm-project/pull/186673
>From 39e90ba8546c8cb44baea91cb1311b3c6ad7baf2 Mon Sep 17 00:00:00 2001
From: stomfaig <stomfaig at gmail.com>
Date: Sun, 15 Mar 2026 14:32:03 +0000
Subject: [PATCH 01/11] check test changes due to legalizing dshuffles
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 6 +-
llvm/test/CodeGen/X86/early-ifcvt-remarks.ll | 39 +-
llvm/test/CodeGen/X86/fshl.ll | 521 +-
llvm/test/CodeGen/X86/fshr.ll | 493 +-
...lar-shift-by-byte-multiple-legalization.ll | 12838 ++++++----------
.../X86/wide-scalar-shift-legalization.ll | 8756 +++--------
...ad-of-small-alloca-with-zero-upper-half.ll | 3482 +----
.../CodeGen/X86/widen-load-of-small-alloca.ll | 2171 +--
.../X86/x86-64-double-precision-shift-left.ll | 18 +-
.../x86-64-double-precision-shift-right.ll | 19 +-
.../X86/x86-64-double-shifts-Oz-Os-O2.ll | 5 +-
.../CodeGen/X86/x86-64-double-shifts-var.ll | 50 +-
12 files changed, 8784 insertions(+), 19614 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index ddba1c7dbdf00..f886a0631033e 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -229,14 +229,12 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
// Funnel shifts.
for (auto ShiftOp : {ISD::FSHL, ISD::FSHR}) {
- // For slow shld targets we only lower for code size.
- LegalizeAction ShiftDoubleAction = Subtarget.isSHLDSlow() ? Custom : Legal;
setOperationAction(ShiftOp , MVT::i8 , Custom);
setOperationAction(ShiftOp , MVT::i16 , Custom);
- setOperationAction(ShiftOp , MVT::i32 , ShiftDoubleAction);
+ setOperationAction(ShiftOp, MVT::i32, Legal);
if (Subtarget.is64Bit())
- setOperationAction(ShiftOp , MVT::i64 , ShiftDoubleAction);
+ setOperationAction(ShiftOp, MVT::i64, Legal);
}
if (!Subtarget.useSoftFloat()) {
diff --git a/llvm/test/CodeGen/X86/early-ifcvt-remarks.ll b/llvm/test/CodeGen/X86/early-ifcvt-remarks.ll
index 054485a358066..72507de17d73c 100644
--- a/llvm/test/CodeGen/X86/early-ifcvt-remarks.ll
+++ b/llvm/test/CodeGen/X86/early-ifcvt-remarks.ll
@@ -73,42 +73,35 @@ define i64 @mm3(i1 %pred, i64 %val, i64 %e1, i128 %e2, i128 %e3, i128 %e4, i128
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: movq %rsi, %rax
; CHECK-NEXT: testb $1, %dil
-; CHECK-NEXT: movq %rsi, %r10
; CHECK-NEXT: jne .LBB2_2
; CHECK-NEXT: # %bb.1: # %if.false
+; CHECK-NEXT: movq %rcx, %r9
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rcx
; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rsi
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rdi
; CHECK-NEXT: imulq %rdx, %rdx
; CHECK-NEXT: movq %rdx, %r10
; CHECK-NEXT: sarq $63, %r10
-; CHECK-NEXT: movq %rcx, %rax
-; CHECK-NEXT: movq %rdx, %r9
+; CHECK-NEXT: movq %r9, %rax
+; CHECK-NEXT: movq %rdx, %rdi
; CHECK-NEXT: mulq %rdx
-; CHECK-NEXT: imulq %rcx, %r10
-; CHECK-NEXT: imulq %r9, %r8
+; CHECK-NEXT: imulq %r9, %r10
+; CHECK-NEXT: imulq %rdi, %r8
; CHECK-NEXT: addq %rdx, %r8
; CHECK-NEXT: addq %r10, %r8
; CHECK-NEXT: addq {{[0-9]+}}(%rsp), %rax
; CHECK-NEXT: adcq {{[0-9]+}}(%rsp), %r8
-; CHECK-NEXT: xorq {{[0-9]+}}(%rsp), %rdi
-; CHECK-NEXT: xorq %r8, %rdi
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r10
-; CHECK-NEXT: xorq %rsi, %r10
-; CHECK-NEXT: xorq %rax, %r10
+; CHECK-NEXT: xorq {{[0-9]+}}(%rsp), %rsi
+; CHECK-NEXT: xorq %r8, %rsi
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; CHECK-NEXT: xorq %rcx, %rdx
+; CHECK-NEXT: xorq %rax, %rdx
+; CHECK-NEXT: shrdq %cl, %rsi, %rdx
+; CHECK-NEXT: sarq %cl, %rsi
+; CHECK-NEXT: testb $64, %cl
+; CHECK-NEXT: cmoveq %rdx, %rsi
; CHECK-NEXT: movq %rdi, %rax
-; CHECK-NEXT: movl %esi, %ecx
-; CHECK-NEXT: sarq %cl, %rax
-; CHECK-NEXT: addq %rdi, %rdi
-; CHECK-NEXT: notb %cl
-; CHECK-NEXT: shlq %cl, %rdi
-; CHECK-NEXT: movl %esi, %ecx
-; CHECK-NEXT: shrq %cl, %r10
-; CHECK-NEXT: orq %rdi, %r10
-; CHECK-NEXT: testb $64, %sil
-; CHECK-NEXT: cmovneq %rax, %r10
-; CHECK-NEXT: movq %r9, %rax
; CHECK-NEXT: .LBB2_2: # %if.endif
-; CHECK-NEXT: addq %r10, %rax
+; CHECK-NEXT: addq %rsi, %rax
; CHECK-NEXT: retq
entry:
br i1 %pred, label %if.true, label %if.false
diff --git a/llvm/test/CodeGen/X86/fshl.ll b/llvm/test/CodeGen/X86/fshl.ll
index 9da2640ea8392..7a6e5f825b97c 100644
--- a/llvm/test/CodeGen/X86/fshl.ll
+++ b/llvm/test/CodeGen/X86/fshl.ll
@@ -93,45 +93,21 @@ define i16 @var_shift_i16(i16 %x, i16 %y, i16 %z) nounwind {
}
define i32 @var_shift_i32(i32 %x, i32 %y, i32 %z) nounwind {
-; X86-FAST-LABEL: var_shift_i32:
-; X86-FAST: # %bb.0:
-; X86-FAST-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-FAST-NEXT: shldl %cl, %edx, %eax
-; X86-FAST-NEXT: retl
-;
-; X86-SLOW-LABEL: var_shift_i32:
-; X86-SLOW: # %bb.0:
-; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-SLOW-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-SLOW-NEXT: shll %cl, %edx
-; X86-SLOW-NEXT: notb %cl
-; X86-SLOW-NEXT: shrl %eax
-; X86-SLOW-NEXT: shrl %cl, %eax
-; X86-SLOW-NEXT: orl %edx, %eax
-; X86-SLOW-NEXT: retl
-;
-; X64-FAST-LABEL: var_shift_i32:
-; X64-FAST: # %bb.0:
-; X64-FAST-NEXT: movl %edx, %ecx
-; X64-FAST-NEXT: movl %edi, %eax
-; X64-FAST-NEXT: # kill: def $cl killed $cl killed $ecx
-; X64-FAST-NEXT: shldl %cl, %esi, %eax
-; X64-FAST-NEXT: retq
+; X86-LABEL: var_shift_i32:
+; X86: # %bb.0:
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: shldl %cl, %edx, %eax
+; X86-NEXT: retl
;
-; X64-SLOW-LABEL: var_shift_i32:
-; X64-SLOW: # %bb.0:
-; X64-SLOW-NEXT: movl %edx, %ecx
-; X64-SLOW-NEXT: movl %esi, %eax
-; X64-SLOW-NEXT: shll %cl, %edi
-; X64-SLOW-NEXT: shrl %eax
-; X64-SLOW-NEXT: notb %cl
-; X64-SLOW-NEXT: # kill: def $cl killed $cl killed $ecx
-; X64-SLOW-NEXT: shrl %cl, %eax
-; X64-SLOW-NEXT: orl %edi, %eax
-; X64-SLOW-NEXT: retq
+; X64-LABEL: var_shift_i32:
+; X64: # %bb.0:
+; X64-NEXT: movl %edx, %ecx
+; X64-NEXT: movl %edi, %eax
+; X64-NEXT: # kill: def $cl killed $cl killed $ecx
+; X64-NEXT: shldl %cl, %esi, %eax
+; X64-NEXT: retq
%tmp = tail call i32 @llvm.fshl.i32(i32 %x, i32 %y, i32 %z)
ret i32 %tmp
}
@@ -177,284 +153,120 @@ define i32 @var_shift_i32_pgso(i32 %x, i32 %y, i32 %z) nounwind !prof !14 {
}
define i64 @var_shift_i64(i64 %x, i64 %y, i64 %z) nounwind {
-; X86-FAST-LABEL: var_shift_i64:
-; X86-FAST: # %bb.0:
-; X86-FAST-NEXT: pushl %edi
-; X86-FAST-NEXT: pushl %esi
-; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-FAST-NEXT: testb $32, %cl
-; X86-FAST-NEXT: jne .LBB5_1
-; X86-FAST-NEXT: # %bb.2:
-; X86-FAST-NEXT: movl %edx, %edi
-; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-FAST-NEXT: jmp .LBB5_3
-; X86-FAST-NEXT: .LBB5_1:
-; X86-FAST-NEXT: movl %esi, %edi
-; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-FAST-NEXT: .LBB5_3:
-; X86-FAST-NEXT: movl %edi, %eax
-; X86-FAST-NEXT: shldl %cl, %esi, %eax
-; X86-FAST-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-FAST-NEXT: shldl %cl, %edi, %edx
-; X86-FAST-NEXT: popl %esi
-; X86-FAST-NEXT: popl %edi
-; X86-FAST-NEXT: retl
-;
-; X86-SLOW-LABEL: var_shift_i64:
-; X86-SLOW: # %bb.0:
-; X86-SLOW-NEXT: pushl %ebx
-; X86-SLOW-NEXT: pushl %edi
-; X86-SLOW-NEXT: pushl %esi
-; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X86-SLOW-NEXT: testb $32, %bl
-; X86-SLOW-NEXT: jne .LBB5_1
-; X86-SLOW-NEXT: # %bb.2:
-; X86-SLOW-NEXT: movl %edx, %esi
-; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-SLOW-NEXT: jmp .LBB5_3
-; X86-SLOW-NEXT: .LBB5_1:
-; X86-SLOW-NEXT: movl %eax, %esi
-; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-SLOW-NEXT: .LBB5_3:
-; X86-SLOW-NEXT: movl %esi, %edi
-; X86-SLOW-NEXT: movl %ebx, %ecx
-; X86-SLOW-NEXT: shll %cl, %edi
-; X86-SLOW-NEXT: shrl %eax
-; X86-SLOW-NEXT: notb %cl
-; X86-SLOW-NEXT: shrl %cl, %eax
-; X86-SLOW-NEXT: orl %edi, %eax
-; X86-SLOW-NEXT: shrl %esi
-; X86-SLOW-NEXT: shrl %cl, %esi
-; X86-SLOW-NEXT: movl %ebx, %ecx
-; X86-SLOW-NEXT: shll %cl, %edx
-; X86-SLOW-NEXT: orl %esi, %edx
-; X86-SLOW-NEXT: popl %esi
-; X86-SLOW-NEXT: popl %edi
-; X86-SLOW-NEXT: popl %ebx
-; X86-SLOW-NEXT: retl
-;
-; X64-FAST-LABEL: var_shift_i64:
-; X64-FAST: # %bb.0:
-; X64-FAST-NEXT: movq %rdx, %rcx
-; X64-FAST-NEXT: movq %rdi, %rax
-; X64-FAST-NEXT: # kill: def $cl killed $cl killed $rcx
-; X64-FAST-NEXT: shldq %cl, %rsi, %rax
-; X64-FAST-NEXT: retq
+; X86-LABEL: var_shift_i64:
+; X86: # %bb.0:
+; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: testb $32, %cl
+; X86-NEXT: jne .LBB5_1
+; X86-NEXT: # %bb.2:
+; X86-NEXT: movl %edx, %edi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: jmp .LBB5_3
+; X86-NEXT: .LBB5_1:
+; X86-NEXT: movl %esi, %edi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT: .LBB5_3:
+; X86-NEXT: movl %edi, %eax
+; X86-NEXT: shldl %cl, %esi, %eax
+; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: shldl %cl, %edi, %edx
+; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
+; X86-NEXT: retl
;
-; X64-SLOW-LABEL: var_shift_i64:
-; X64-SLOW: # %bb.0:
-; X64-SLOW-NEXT: movq %rdx, %rcx
-; X64-SLOW-NEXT: movq %rsi, %rax
-; X64-SLOW-NEXT: shlq %cl, %rdi
-; X64-SLOW-NEXT: shrq %rax
-; X64-SLOW-NEXT: notb %cl
-; X64-SLOW-NEXT: # kill: def $cl killed $cl killed $rcx
-; X64-SLOW-NEXT: shrq %cl, %rax
-; X64-SLOW-NEXT: orq %rdi, %rax
-; X64-SLOW-NEXT: retq
+; X64-LABEL: var_shift_i64:
+; X64: # %bb.0:
+; X64-NEXT: movq %rdx, %rcx
+; X64-NEXT: movq %rdi, %rax
+; X64-NEXT: # kill: def $cl killed $cl killed $rcx
+; X64-NEXT: shldq %cl, %rsi, %rax
+; X64-NEXT: retq
%tmp = tail call i64 @llvm.fshl.i64(i64 %x, i64 %y, i64 %z)
ret i64 %tmp
}
define i128 @var_shift_i128(i128 %x, i128 %y, i128 %z) nounwind {
-; X86-FAST-LABEL: var_shift_i128:
-; X86-FAST: # %bb.0:
-; X86-FAST-NEXT: pushl %ebp
-; X86-FAST-NEXT: movl %esp, %ebp
-; X86-FAST-NEXT: pushl %ebx
-; X86-FAST-NEXT: pushl %edi
-; X86-FAST-NEXT: pushl %esi
-; X86-FAST-NEXT: andl $-16, %esp
-; X86-FAST-NEXT: subl $16, %esp
-; X86-FAST-NEXT: movl 24(%ebp), %edi
-; X86-FAST-NEXT: movl 28(%ebp), %edx
-; X86-FAST-NEXT: movl 48(%ebp), %esi
-; X86-FAST-NEXT: movl 56(%ebp), %ecx
-; X86-FAST-NEXT: testb $64, %cl
-; X86-FAST-NEXT: movl 52(%ebp), %eax
-; X86-FAST-NEXT: jne .LBB6_1
-; X86-FAST-NEXT: # %bb.2:
-; X86-FAST-NEXT: movl %esi, %ebx
-; X86-FAST-NEXT: movl %edi, %esi
-; X86-FAST-NEXT: movl 32(%ebp), %edi
-; X86-FAST-NEXT: movl %eax, (%esp) # 4-byte Spill
-; X86-FAST-NEXT: movl %edx, %eax
-; X86-FAST-NEXT: movl 36(%ebp), %edx
-; X86-FAST-NEXT: testb $32, %cl
-; X86-FAST-NEXT: je .LBB6_5
-; X86-FAST-NEXT: .LBB6_4:
-; X86-FAST-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-FAST-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-FAST-NEXT: movl %esi, %eax
-; X86-FAST-NEXT: movl (%esp), %esi # 4-byte Reload
-; X86-FAST-NEXT: jmp .LBB6_6
-; X86-FAST-NEXT: .LBB6_1:
-; X86-FAST-NEXT: movl 44(%ebp), %ebx
-; X86-FAST-NEXT: movl %ebx, (%esp) # 4-byte Spill
-; X86-FAST-NEXT: movl 40(%ebp), %ebx
-; X86-FAST-NEXT: testb $32, %cl
-; X86-FAST-NEXT: jne .LBB6_4
-; X86-FAST-NEXT: .LBB6_5:
-; X86-FAST-NEXT: movl (%esp), %ebx # 4-byte Reload
-; X86-FAST-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-FAST-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-FAST-NEXT: .LBB6_6:
-; X86-FAST-NEXT: movl %esi, %edi
-; X86-FAST-NEXT: shldl %cl, %ebx, %edi
-; X86-FAST-NEXT: movl %eax, %edx
-; X86-FAST-NEXT: movl %eax, %ebx
-; X86-FAST-NEXT: shldl %cl, %esi, %ebx
-; X86-FAST-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-FAST-NEXT: movl %eax, %esi
-; X86-FAST-NEXT: shldl %cl, %edx, %esi
-; X86-FAST-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-FAST-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-FAST-NEXT: shldl %cl, %eax, %edx
-; X86-FAST-NEXT: movl 8(%ebp), %eax
-; X86-FAST-NEXT: movl %edx, 12(%eax)
-; X86-FAST-NEXT: movl %esi, 8(%eax)
-; X86-FAST-NEXT: movl %ebx, 4(%eax)
-; X86-FAST-NEXT: movl %edi, (%eax)
-; X86-FAST-NEXT: leal -12(%ebp), %esp
-; X86-FAST-NEXT: popl %esi
-; X86-FAST-NEXT: popl %edi
-; X86-FAST-NEXT: popl %ebx
-; X86-FAST-NEXT: popl %ebp
-; X86-FAST-NEXT: retl $4
-;
-; X86-SLOW-LABEL: var_shift_i128:
-; X86-SLOW: # %bb.0:
-; X86-SLOW-NEXT: pushl %ebp
-; X86-SLOW-NEXT: movl %esp, %ebp
-; X86-SLOW-NEXT: pushl %ebx
-; X86-SLOW-NEXT: pushl %edi
-; X86-SLOW-NEXT: pushl %esi
-; X86-SLOW-NEXT: andl $-16, %esp
-; X86-SLOW-NEXT: subl $32, %esp
-; X86-SLOW-NEXT: movl 24(%ebp), %edi
-; X86-SLOW-NEXT: movl 28(%ebp), %eax
-; X86-SLOW-NEXT: movl 48(%ebp), %edx
-; X86-SLOW-NEXT: movl 56(%ebp), %ecx
-; X86-SLOW-NEXT: testb $64, %cl
-; X86-SLOW-NEXT: movl 52(%ebp), %ebx
-; X86-SLOW-NEXT: jne .LBB6_1
-; X86-SLOW-NEXT: # %bb.2:
-; X86-SLOW-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-SLOW-NEXT: movl %edi, %edx
-; X86-SLOW-NEXT: movl 32(%ebp), %edi
-; X86-SLOW-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-SLOW-NEXT: movl %eax, %ebx
-; X86-SLOW-NEXT: movl 36(%ebp), %eax
-; X86-SLOW-NEXT: jmp .LBB6_3
-; X86-SLOW-NEXT: .LBB6_1:
-; X86-SLOW-NEXT: movl 40(%ebp), %ecx
-; X86-SLOW-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-SLOW-NEXT: movl 44(%ebp), %ecx
-; X86-SLOW-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-SLOW-NEXT: .LBB6_3:
-; X86-SLOW-NEXT: movl 56(%ebp), %ecx
-; X86-SLOW-NEXT: testb $32, %cl
-; X86-SLOW-NEXT: jne .LBB6_4
-; X86-SLOW-NEXT: # %bb.5:
-; X86-SLOW-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-SLOW-NEXT: movl %edx, %edi
-; X86-SLOW-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-SLOW-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-SLOW-NEXT: jmp .LBB6_6
-; X86-SLOW-NEXT: .LBB6_4:
-; X86-SLOW-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-SLOW-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-SLOW-NEXT: movl %edx, %ebx
-; X86-SLOW-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-SLOW-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-SLOW-NEXT: .LBB6_6:
-; X86-SLOW-NEXT: movl %edi, %eax
-; X86-SLOW-NEXT: shll %cl, %eax
-; X86-SLOW-NEXT: shrl %esi
-; X86-SLOW-NEXT: movl %ecx, %edx
-; X86-SLOW-NEXT: notb %dl
-; X86-SLOW-NEXT: movl %edx, %ecx
-; X86-SLOW-NEXT: shrl %cl, %esi
-; X86-SLOW-NEXT: orl %eax, %esi
-; X86-SLOW-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-SLOW-NEXT: movl %ebx, %eax
-; X86-SLOW-NEXT: movl 56(%ebp), %ecx
-; X86-SLOW-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-SLOW-NEXT: shll %cl, %eax
-; X86-SLOW-NEXT: shrl %edi
-; X86-SLOW-NEXT: movl %edx, %ecx
-; X86-SLOW-NEXT: shrl %cl, %edi
-; X86-SLOW-NEXT: orl %eax, %edi
-; X86-SLOW-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-SLOW-NEXT: movl %esi, %eax
-; X86-SLOW-NEXT: movl 56(%ebp), %ecx
-; X86-SLOW-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-SLOW-NEXT: shll %cl, %eax
-; X86-SLOW-NEXT: shrl %ebx
-; X86-SLOW-NEXT: movl %edx, %ecx
-; X86-SLOW-NEXT: shrl %cl, %ebx
-; X86-SLOW-NEXT: orl %eax, %ebx
-; X86-SLOW-NEXT: movl 56(%ebp), %ecx
-; X86-SLOW-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-SLOW-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-SLOW-NEXT: shll %cl, %eax
-; X86-SLOW-NEXT: shrl %esi
-; X86-SLOW-NEXT: movl %edx, %ecx
-; X86-SLOW-NEXT: shrl %cl, %esi
-; X86-SLOW-NEXT: orl %eax, %esi
-; X86-SLOW-NEXT: movl 8(%ebp), %eax
-; X86-SLOW-NEXT: movl %esi, 12(%eax)
-; X86-SLOW-NEXT: movl %ebx, 8(%eax)
-; X86-SLOW-NEXT: movl %edi, 4(%eax)
-; X86-SLOW-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-SLOW-NEXT: movl %ecx, (%eax)
-; X86-SLOW-NEXT: leal -12(%ebp), %esp
-; X86-SLOW-NEXT: popl %esi
-; X86-SLOW-NEXT: popl %edi
-; X86-SLOW-NEXT: popl %ebx
-; X86-SLOW-NEXT: popl %ebp
-; X86-SLOW-NEXT: retl $4
-;
-; X64-FAST-LABEL: var_shift_i128:
-; X64-FAST: # %bb.0:
-; X64-FAST-NEXT: testb $64, %r8b
-; X64-FAST-NEXT: cmovneq %rdi, %rsi
-; X64-FAST-NEXT: cmoveq %rcx, %rdx
-; X64-FAST-NEXT: cmovneq %rcx, %rdi
-; X64-FAST-NEXT: movq %rdi, %rax
-; X64-FAST-NEXT: movl %r8d, %ecx
-; X64-FAST-NEXT: shldq %cl, %rdx, %rax
-; X64-FAST-NEXT: shldq %cl, %rdi, %rsi
-; X64-FAST-NEXT: movq %rsi, %rdx
-; X64-FAST-NEXT: retq
-;
-; X64-SLOW-LABEL: var_shift_i128:
-; X64-SLOW: # %bb.0:
-; X64-SLOW-NEXT: testb $64, %r8b
-; X64-SLOW-NEXT: cmovneq %rdi, %rsi
-; X64-SLOW-NEXT: cmoveq %rcx, %rdx
-; X64-SLOW-NEXT: cmovneq %rcx, %rdi
-; X64-SLOW-NEXT: movq %rdi, %rax
-; X64-SLOW-NEXT: movl %r8d, %ecx
-; X64-SLOW-NEXT: shlq %cl, %rax
-; X64-SLOW-NEXT: shrq %rdx
-; X64-SLOW-NEXT: movl %r8d, %r9d
-; X64-SLOW-NEXT: notb %r9b
-; X64-SLOW-NEXT: movl %r9d, %ecx
-; X64-SLOW-NEXT: shrq %cl, %rdx
-; X64-SLOW-NEXT: orq %rdx, %rax
-; X64-SLOW-NEXT: movl %r8d, %ecx
-; X64-SLOW-NEXT: shlq %cl, %rsi
-; X64-SLOW-NEXT: shrq %rdi
-; X64-SLOW-NEXT: movl %r9d, %ecx
-; X64-SLOW-NEXT: shrq %cl, %rdi
-; X64-SLOW-NEXT: orq %rsi, %rdi
-; X64-SLOW-NEXT: movq %rdi, %rdx
-; X64-SLOW-NEXT: retq
+; X86-LABEL: var_shift_i128:
+; X86: # %bb.0:
+; X86-NEXT: pushl %ebp
+; X86-NEXT: movl %esp, %ebp
+; X86-NEXT: pushl %ebx
+; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %esi
+; X86-NEXT: andl $-16, %esp
+; X86-NEXT: subl $16, %esp
+; X86-NEXT: movl 24(%ebp), %edi
+; X86-NEXT: movl 28(%ebp), %edx
+; X86-NEXT: movl 48(%ebp), %esi
+; X86-NEXT: movl 56(%ebp), %ecx
+; X86-NEXT: testb $64, %cl
+; X86-NEXT: movl 52(%ebp), %eax
+; X86-NEXT: jne .LBB6_1
+; X86-NEXT: # %bb.2:
+; X86-NEXT: movl %esi, %ebx
+; X86-NEXT: movl %edi, %esi
+; X86-NEXT: movl 32(%ebp), %edi
+; X86-NEXT: movl %eax, (%esp) # 4-byte Spill
+; X86-NEXT: movl %edx, %eax
+; X86-NEXT: movl 36(%ebp), %edx
+; X86-NEXT: testb $32, %cl
+; X86-NEXT: je .LBB6_5
+; X86-NEXT: .LBB6_4:
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %esi, %eax
+; X86-NEXT: movl (%esp), %esi # 4-byte Reload
+; X86-NEXT: jmp .LBB6_6
+; X86-NEXT: .LBB6_1:
+; X86-NEXT: movl 44(%ebp), %ebx
+; X86-NEXT: movl %ebx, (%esp) # 4-byte Spill
+; X86-NEXT: movl 40(%ebp), %ebx
+; X86-NEXT: testb $32, %cl
+; X86-NEXT: jne .LBB6_4
+; X86-NEXT: .LBB6_5:
+; X86-NEXT: movl (%esp), %ebx # 4-byte Reload
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: .LBB6_6:
+; X86-NEXT: movl %esi, %edi
+; X86-NEXT: shldl %cl, %ebx, %edi
+; X86-NEXT: movl %eax, %edx
+; X86-NEXT: movl %eax, %ebx
+; X86-NEXT: shldl %cl, %esi, %ebx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl %eax, %esi
+; X86-NEXT: shldl %cl, %edx, %esi
+; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: shldl %cl, %eax, %edx
+; X86-NEXT: movl 8(%ebp), %eax
+; X86-NEXT: movl %edx, 12(%eax)
+; X86-NEXT: movl %esi, 8(%eax)
+; X86-NEXT: movl %ebx, 4(%eax)
+; X86-NEXT: movl %edi, (%eax)
+; X86-NEXT: leal -12(%ebp), %esp
+; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
+; X86-NEXT: popl %ebx
+; X86-NEXT: popl %ebp
+; X86-NEXT: retl $4
+;
+; X64-LABEL: var_shift_i128:
+; X64: # %bb.0:
+; X64-NEXT: testb $64, %r8b
+; X64-NEXT: cmovneq %rdi, %rsi
+; X64-NEXT: cmoveq %rcx, %rdx
+; X64-NEXT: cmovneq %rcx, %rdi
+; X64-NEXT: movq %rdi, %rax
+; X64-NEXT: movl %r8d, %ecx
+; X64-NEXT: shldq %cl, %rdx, %rax
+; X64-NEXT: shldq %cl, %rdi, %rsi
+; X64-NEXT: movq %rsi, %rdx
+; X64-NEXT: retq
%tmp = tail call i128 @llvm.fshl.i128(i128 %x, i128 %y, i128 %z)
ret i128 %tmp
}
@@ -524,78 +336,37 @@ define i16 @const_shift_i16(i16 %x, i16 %y) nounwind {
}
define i32 @const_shift_i32(i32 %x, i32 %y) nounwind {
-; X86-FAST-LABEL: const_shift_i32:
-; X86-FAST: # %bb.0:
-; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-FAST-NEXT: shldl $7, %ecx, %eax
-; X86-FAST-NEXT: retl
-;
-; X86-SLOW-LABEL: const_shift_i32:
-; X86-SLOW: # %bb.0:
-; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-SLOW-NEXT: shrl $25, %ecx
-; X86-SLOW-NEXT: shll $7, %eax
-; X86-SLOW-NEXT: orl %ecx, %eax
-; X86-SLOW-NEXT: retl
-;
-; X64-FAST-LABEL: const_shift_i32:
-; X64-FAST: # %bb.0:
-; X64-FAST-NEXT: movl %edi, %eax
-; X64-FAST-NEXT: shldl $7, %esi, %eax
-; X64-FAST-NEXT: retq
+; X86-LABEL: const_shift_i32:
+; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: shldl $7, %ecx, %eax
+; X86-NEXT: retl
;
-; X64-SLOW-LABEL: const_shift_i32:
-; X64-SLOW: # %bb.0:
-; X64-SLOW-NEXT: # kill: def $esi killed $esi def $rsi
-; X64-SLOW-NEXT: # kill: def $edi killed $edi def $rdi
-; X64-SLOW-NEXT: shrl $25, %esi
-; X64-SLOW-NEXT: shll $7, %edi
-; X64-SLOW-NEXT: leal (%rdi,%rsi), %eax
-; X64-SLOW-NEXT: retq
+; X64-LABEL: const_shift_i32:
+; X64: # %bb.0:
+; X64-NEXT: movl %edi, %eax
+; X64-NEXT: shldl $7, %esi, %eax
+; X64-NEXT: retq
%tmp = tail call i32 @llvm.fshl.i32(i32 %x, i32 %y, i32 7)
ret i32 %tmp
}
define i64 @const_shift_i64(i64 %x, i64 %y) nounwind {
-; X86-FAST-LABEL: const_shift_i64:
-; X86-FAST: # %bb.0:
-; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-FAST-NEXT: shrdl $25, %ecx, %eax
-; X86-FAST-NEXT: shldl $7, %ecx, %edx
-; X86-FAST-NEXT: retl
-;
-; X86-SLOW-LABEL: const_shift_i64:
-; X86-SLOW: # %bb.0:
-; X86-SLOW-NEXT: pushl %esi
-; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-SLOW-NEXT: movl %ecx, %esi
-; X86-SLOW-NEXT: shrl $25, %esi
-; X86-SLOW-NEXT: shll $7, %edx
-; X86-SLOW-NEXT: orl %esi, %edx
-; X86-SLOW-NEXT: shll $7, %ecx
-; X86-SLOW-NEXT: shrl $25, %eax
-; X86-SLOW-NEXT: orl %ecx, %eax
-; X86-SLOW-NEXT: popl %esi
-; X86-SLOW-NEXT: retl
-;
-; X64-FAST-LABEL: const_shift_i64:
-; X64-FAST: # %bb.0:
-; X64-FAST-NEXT: movq %rdi, %rax
-; X64-FAST-NEXT: shldq $7, %rsi, %rax
-; X64-FAST-NEXT: retq
+; X86-LABEL: const_shift_i64:
+; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: shrdl $25, %ecx, %eax
+; X86-NEXT: shldl $7, %ecx, %edx
+; X86-NEXT: retl
;
-; X64-SLOW-LABEL: const_shift_i64:
-; X64-SLOW: # %bb.0:
-; X64-SLOW-NEXT: shrq $57, %rsi
-; X64-SLOW-NEXT: shlq $7, %rdi
-; X64-SLOW-NEXT: leaq (%rdi,%rsi), %rax
-; X64-SLOW-NEXT: retq
+; X64-LABEL: const_shift_i64:
+; X64: # %bb.0:
+; X64-NEXT: movq %rdi, %rax
+; X64-NEXT: shldq $7, %rsi, %rax
+; X64-NEXT: retq
%tmp = tail call i64 @llvm.fshl.i64(i64 %x, i64 %y, i64 7)
ret i64 %tmp
}
diff --git a/llvm/test/CodeGen/X86/fshr.ll b/llvm/test/CodeGen/X86/fshr.ll
index c307833e488c9..f77ffe92472a2 100644
--- a/llvm/test/CodeGen/X86/fshr.ll
+++ b/llvm/test/CodeGen/X86/fshr.ll
@@ -90,45 +90,21 @@ define i16 @var_shift_i16(i16 %x, i16 %y, i16 %z) nounwind {
}
define i32 @var_shift_i32(i32 %x, i32 %y, i32 %z) nounwind {
-; X86-FAST-LABEL: var_shift_i32:
-; X86-FAST: # %bb.0:
-; X86-FAST-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-FAST-NEXT: shrdl %cl, %edx, %eax
-; X86-FAST-NEXT: retl
-;
-; X86-SLOW-LABEL: var_shift_i32:
-; X86-SLOW: # %bb.0:
-; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-SLOW-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-SLOW-NEXT: shrl %cl, %edx
-; X86-SLOW-NEXT: notb %cl
-; X86-SLOW-NEXT: addl %eax, %eax
-; X86-SLOW-NEXT: shll %cl, %eax
-; X86-SLOW-NEXT: orl %edx, %eax
-; X86-SLOW-NEXT: retl
-;
-; X64-FAST-LABEL: var_shift_i32:
-; X64-FAST: # %bb.0:
-; X64-FAST-NEXT: movl %edx, %ecx
-; X64-FAST-NEXT: movl %esi, %eax
-; X64-FAST-NEXT: # kill: def $cl killed $cl killed $ecx
-; X64-FAST-NEXT: shrdl %cl, %edi, %eax
-; X64-FAST-NEXT: retq
+; X86-LABEL: var_shift_i32:
+; X86: # %bb.0:
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: shrdl %cl, %edx, %eax
+; X86-NEXT: retl
;
-; X64-SLOW-LABEL: var_shift_i32:
-; X64-SLOW: # %bb.0:
-; X64-SLOW-NEXT: movl %edx, %ecx
-; X64-SLOW-NEXT: # kill: def $edi killed $edi def $rdi
-; X64-SLOW-NEXT: shrl %cl, %esi
-; X64-SLOW-NEXT: leal (%rdi,%rdi), %eax
-; X64-SLOW-NEXT: notb %cl
-; X64-SLOW-NEXT: # kill: def $cl killed $cl killed $ecx
-; X64-SLOW-NEXT: shll %cl, %eax
-; X64-SLOW-NEXT: orl %esi, %eax
-; X64-SLOW-NEXT: retq
+; X64-LABEL: var_shift_i32:
+; X64: # %bb.0:
+; X64-NEXT: movl %edx, %ecx
+; X64-NEXT: movl %esi, %eax
+; X64-NEXT: # kill: def $cl killed $cl killed $ecx
+; X64-NEXT: shrdl %cl, %edi, %eax
+; X64-NEXT: retq
%tmp = tail call i32 @llvm.fshr.i32(i32 %x, i32 %y, i32 %z)
ret i32 %tmp
}
@@ -174,268 +150,108 @@ define i32 @var_shift_i32_pgso(i32 %x, i32 %y, i32 %z) nounwind !prof !14 {
}
define i64 @var_shift_i64(i64 %x, i64 %y, i64 %z) nounwind {
-; X86-FAST-LABEL: var_shift_i64:
-; X86-FAST: # %bb.0:
-; X86-FAST-NEXT: pushl %esi
-; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-FAST-NEXT: testb $32, %cl
-; X86-FAST-NEXT: je .LBB5_1
-; X86-FAST-NEXT: # %bb.2:
-; X86-FAST-NEXT: movl %esi, %edx
-; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-FAST-NEXT: jmp .LBB5_3
-; X86-FAST-NEXT: .LBB5_1:
-; X86-FAST-NEXT: movl %eax, %edx
-; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-FAST-NEXT: .LBB5_3:
-; X86-FAST-NEXT: shrdl %cl, %edx, %eax
-; X86-FAST-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-FAST-NEXT: shrdl %cl, %esi, %edx
-; X86-FAST-NEXT: popl %esi
-; X86-FAST-NEXT: retl
-;
-; X86-SLOW-LABEL: var_shift_i64:
-; X86-SLOW: # %bb.0:
-; X86-SLOW-NEXT: pushl %ebx
-; X86-SLOW-NEXT: pushl %edi
-; X86-SLOW-NEXT: pushl %esi
-; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X86-SLOW-NEXT: testb $32, %bl
-; X86-SLOW-NEXT: je .LBB5_1
-; X86-SLOW-NEXT: # %bb.2:
-; X86-SLOW-NEXT: movl %edx, %esi
-; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-SLOW-NEXT: jmp .LBB5_3
-; X86-SLOW-NEXT: .LBB5_1:
-; X86-SLOW-NEXT: movl %eax, %esi
-; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-SLOW-NEXT: .LBB5_3:
-; X86-SLOW-NEXT: leal (%esi,%esi), %edi
-; X86-SLOW-NEXT: movb %bl, %ch
-; X86-SLOW-NEXT: notb %ch
-; X86-SLOW-NEXT: movb %ch, %cl
-; X86-SLOW-NEXT: shll %cl, %edi
-; X86-SLOW-NEXT: movb %bl, %cl
-; X86-SLOW-NEXT: shrl %cl, %eax
-; X86-SLOW-NEXT: orl %edi, %eax
-; X86-SLOW-NEXT: shrl %cl, %esi
-; X86-SLOW-NEXT: addl %edx, %edx
-; X86-SLOW-NEXT: movb %ch, %cl
-; X86-SLOW-NEXT: shll %cl, %edx
-; X86-SLOW-NEXT: orl %esi, %edx
-; X86-SLOW-NEXT: popl %esi
-; X86-SLOW-NEXT: popl %edi
-; X86-SLOW-NEXT: popl %ebx
-; X86-SLOW-NEXT: retl
-;
-; X64-FAST-LABEL: var_shift_i64:
-; X64-FAST: # %bb.0:
-; X64-FAST-NEXT: movq %rdx, %rcx
-; X64-FAST-NEXT: movq %rsi, %rax
-; X64-FAST-NEXT: # kill: def $cl killed $cl killed $rcx
-; X64-FAST-NEXT: shrdq %cl, %rdi, %rax
-; X64-FAST-NEXT: retq
+; X86-LABEL: var_shift_i64:
+; X86: # %bb.0:
+; X86-NEXT: pushl %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: testb $32, %cl
+; X86-NEXT: je .LBB5_1
+; X86-NEXT: # %bb.2:
+; X86-NEXT: movl %esi, %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT: jmp .LBB5_3
+; X86-NEXT: .LBB5_1:
+; X86-NEXT: movl %eax, %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: .LBB5_3:
+; X86-NEXT: shrdl %cl, %edx, %eax
+; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: shrdl %cl, %esi, %edx
+; X86-NEXT: popl %esi
+; X86-NEXT: retl
;
-; X64-SLOW-LABEL: var_shift_i64:
-; X64-SLOW: # %bb.0:
-; X64-SLOW-NEXT: movq %rdx, %rcx
-; X64-SLOW-NEXT: shrq %cl, %rsi
-; X64-SLOW-NEXT: leaq (%rdi,%rdi), %rax
-; X64-SLOW-NEXT: notb %cl
-; X64-SLOW-NEXT: # kill: def $cl killed $cl killed $rcx
-; X64-SLOW-NEXT: shlq %cl, %rax
-; X64-SLOW-NEXT: orq %rsi, %rax
-; X64-SLOW-NEXT: retq
+; X64-LABEL: var_shift_i64:
+; X64: # %bb.0:
+; X64-NEXT: movq %rdx, %rcx
+; X64-NEXT: movq %rsi, %rax
+; X64-NEXT: # kill: def $cl killed $cl killed $rcx
+; X64-NEXT: shrdq %cl, %rdi, %rax
+; X64-NEXT: retq
%tmp = tail call i64 @llvm.fshr.i64(i64 %x, i64 %y, i64 %z)
ret i64 %tmp
}
define i128 @var_shift_i128(i128 %x, i128 %y, i128 %z) nounwind {
-; X86-FAST-LABEL: var_shift_i128:
-; X86-FAST: # %bb.0:
-; X86-FAST-NEXT: pushl %ebp
-; X86-FAST-NEXT: movl %esp, %ebp
-; X86-FAST-NEXT: pushl %ebx
-; X86-FAST-NEXT: pushl %edi
-; X86-FAST-NEXT: pushl %esi
-; X86-FAST-NEXT: andl $-16, %esp
-; X86-FAST-NEXT: subl $16, %esp
-; X86-FAST-NEXT: movl 24(%ebp), %esi
-; X86-FAST-NEXT: movl 28(%ebp), %eax
-; X86-FAST-NEXT: movl 48(%ebp), %edx
-; X86-FAST-NEXT: movl 56(%ebp), %ecx
-; X86-FAST-NEXT: testb $64, %cl
-; X86-FAST-NEXT: movl 52(%ebp), %ebx
-; X86-FAST-NEXT: je .LBB6_1
-; X86-FAST-NEXT: # %bb.2:
-; X86-FAST-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-FAST-NEXT: movl %esi, %edx
-; X86-FAST-NEXT: movl 32(%ebp), %esi
-; X86-FAST-NEXT: movl %ebx, %edi
-; X86-FAST-NEXT: movl %eax, %ebx
-; X86-FAST-NEXT: movl 36(%ebp), %eax
-; X86-FAST-NEXT: testb $32, %cl
-; X86-FAST-NEXT: je .LBB6_4
-; X86-FAST-NEXT: jmp .LBB6_5
-; X86-FAST-NEXT: .LBB6_1:
-; X86-FAST-NEXT: movl 40(%ebp), %edi
-; X86-FAST-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-FAST-NEXT: movl 44(%ebp), %edi
-; X86-FAST-NEXT: testb $32, %cl
-; X86-FAST-NEXT: jne .LBB6_5
-; X86-FAST-NEXT: .LBB6_4:
-; X86-FAST-NEXT: movl %esi, %eax
-; X86-FAST-NEXT: movl %ebx, %esi
-; X86-FAST-NEXT: movl %edx, %ebx
-; X86-FAST-NEXT: movl %edi, %edx
-; X86-FAST-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-FAST-NEXT: .LBB6_5:
-; X86-FAST-NEXT: shrdl %cl, %edx, %edi
-; X86-FAST-NEXT: shrdl %cl, %ebx, %edx
-; X86-FAST-NEXT: shrdl %cl, %esi, %ebx
-; X86-FAST-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-FAST-NEXT: shrdl %cl, %eax, %esi
-; X86-FAST-NEXT: movl 8(%ebp), %eax
-; X86-FAST-NEXT: movl %esi, 12(%eax)
-; X86-FAST-NEXT: movl %ebx, 8(%eax)
-; X86-FAST-NEXT: movl %edx, 4(%eax)
-; X86-FAST-NEXT: movl %edi, (%eax)
-; X86-FAST-NEXT: leal -12(%ebp), %esp
-; X86-FAST-NEXT: popl %esi
-; X86-FAST-NEXT: popl %edi
-; X86-FAST-NEXT: popl %ebx
-; X86-FAST-NEXT: popl %ebp
-; X86-FAST-NEXT: retl $4
-;
-; X86-SLOW-LABEL: var_shift_i128:
-; X86-SLOW: # %bb.0:
-; X86-SLOW-NEXT: pushl %ebp
-; X86-SLOW-NEXT: movl %esp, %ebp
-; X86-SLOW-NEXT: pushl %ebx
-; X86-SLOW-NEXT: pushl %edi
-; X86-SLOW-NEXT: pushl %esi
-; X86-SLOW-NEXT: andl $-16, %esp
-; X86-SLOW-NEXT: subl $16, %esp
-; X86-SLOW-NEXT: movl 24(%ebp), %edx
-; X86-SLOW-NEXT: movl 28(%ebp), %esi
-; X86-SLOW-NEXT: movl 48(%ebp), %edi
-; X86-SLOW-NEXT: movl 56(%ebp), %eax
-; X86-SLOW-NEXT: testb $64, %al
-; X86-SLOW-NEXT: movl 52(%ebp), %eax
-; X86-SLOW-NEXT: je .LBB6_1
-; X86-SLOW-NEXT: # %bb.2:
-; X86-SLOW-NEXT: movl %edi, (%esp) # 4-byte Spill
-; X86-SLOW-NEXT: movl %edx, %edi
-; X86-SLOW-NEXT: movl 32(%ebp), %edx
-; X86-SLOW-NEXT: movl %eax, %ecx
-; X86-SLOW-NEXT: movl %esi, %eax
-; X86-SLOW-NEXT: movl 36(%ebp), %esi
-; X86-SLOW-NEXT: jmp .LBB6_3
-; X86-SLOW-NEXT: .LBB6_1:
-; X86-SLOW-NEXT: movl 40(%ebp), %ecx
-; X86-SLOW-NEXT: movl %ecx, (%esp) # 4-byte Spill
-; X86-SLOW-NEXT: movl 44(%ebp), %ecx
-; X86-SLOW-NEXT: .LBB6_3:
-; X86-SLOW-NEXT: movl 56(%ebp), %ebx
-; X86-SLOW-NEXT: testb $32, %bl
-; X86-SLOW-NEXT: je .LBB6_4
-; X86-SLOW-NEXT: # %bb.5:
-; X86-SLOW-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-SLOW-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-SLOW-NEXT: movl %ecx, %ebx
-; X86-SLOW-NEXT: jmp .LBB6_6
-; X86-SLOW-NEXT: .LBB6_4:
-; X86-SLOW-NEXT: movl %edx, %esi
-; X86-SLOW-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-SLOW-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-SLOW-NEXT: movl %ecx, %edi
-; X86-SLOW-NEXT: movl (%esp), %ebx # 4-byte Reload
-; X86-SLOW-NEXT: .LBB6_6:
-; X86-SLOW-NEXT: movl 56(%ebp), %ecx
-; X86-SLOW-NEXT: shrl %cl, %ebx
-; X86-SLOW-NEXT: movl %ecx, %edx
-; X86-SLOW-NEXT: notb %dl
-; X86-SLOW-NEXT: movl %edi, %eax
-; X86-SLOW-NEXT: addl %edi, %edi
-; X86-SLOW-NEXT: movl %edx, %ecx
-; X86-SLOW-NEXT: shll %cl, %edi
-; X86-SLOW-NEXT: orl %ebx, %edi
-; X86-SLOW-NEXT: movl %edi, (%esp) # 4-byte Spill
-; X86-SLOW-NEXT: movl 56(%ebp), %ecx
-; X86-SLOW-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-SLOW-NEXT: shrl %cl, %eax
-; X86-SLOW-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-SLOW-NEXT: leal (%edi,%edi), %ebx
-; X86-SLOW-NEXT: movl %edx, %ecx
-; X86-SLOW-NEXT: shll %cl, %ebx
-; X86-SLOW-NEXT: orl %eax, %ebx
-; X86-SLOW-NEXT: movl 56(%ebp), %ecx
-; X86-SLOW-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-SLOW-NEXT: shrl %cl, %edi
-; X86-SLOW-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-SLOW-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-SLOW-NEXT: leal (%eax,%eax), %edi
-; X86-SLOW-NEXT: movl %edx, %ecx
-; X86-SLOW-NEXT: shll %cl, %edi
-; X86-SLOW-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X86-SLOW-NEXT: movl 56(%ebp), %ecx
-; X86-SLOW-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-SLOW-NEXT: shrl %cl, %eax
-; X86-SLOW-NEXT: addl %esi, %esi
-; X86-SLOW-NEXT: movl %edx, %ecx
-; X86-SLOW-NEXT: shll %cl, %esi
-; X86-SLOW-NEXT: orl %eax, %esi
-; X86-SLOW-NEXT: movl 8(%ebp), %eax
-; X86-SLOW-NEXT: movl %esi, 12(%eax)
-; X86-SLOW-NEXT: movl %edi, 8(%eax)
-; X86-SLOW-NEXT: movl %ebx, 4(%eax)
-; X86-SLOW-NEXT: movl (%esp), %ecx # 4-byte Reload
-; X86-SLOW-NEXT: movl %ecx, (%eax)
-; X86-SLOW-NEXT: leal -12(%ebp), %esp
-; X86-SLOW-NEXT: popl %esi
-; X86-SLOW-NEXT: popl %edi
-; X86-SLOW-NEXT: popl %ebx
-; X86-SLOW-NEXT: popl %ebp
-; X86-SLOW-NEXT: retl $4
-;
-; X64-FAST-LABEL: var_shift_i128:
-; X64-FAST: # %bb.0:
-; X64-FAST-NEXT: movq %rdx, %rax
-; X64-FAST-NEXT: testb $64, %r8b
-; X64-FAST-NEXT: cmoveq %rdi, %rsi
-; X64-FAST-NEXT: cmoveq %rcx, %rdi
-; X64-FAST-NEXT: cmovneq %rcx, %rax
-; X64-FAST-NEXT: movl %r8d, %ecx
-; X64-FAST-NEXT: shrdq %cl, %rdi, %rax
-; X64-FAST-NEXT: shrdq %cl, %rsi, %rdi
-; X64-FAST-NEXT: movq %rdi, %rdx
-; X64-FAST-NEXT: retq
-;
-; X64-SLOW-LABEL: var_shift_i128:
-; X64-SLOW: # %bb.0:
-; X64-SLOW-NEXT: testb $64, %r8b
-; X64-SLOW-NEXT: cmoveq %rdi, %rsi
-; X64-SLOW-NEXT: cmoveq %rcx, %rdi
-; X64-SLOW-NEXT: cmovneq %rcx, %rdx
-; X64-SLOW-NEXT: movl %r8d, %ecx
-; X64-SLOW-NEXT: shrq %cl, %rdx
-; X64-SLOW-NEXT: leaq (%rdi,%rdi), %rax
-; X64-SLOW-NEXT: movl %r8d, %r9d
-; X64-SLOW-NEXT: notb %r9b
-; X64-SLOW-NEXT: movl %r9d, %ecx
-; X64-SLOW-NEXT: shlq %cl, %rax
-; X64-SLOW-NEXT: orq %rdx, %rax
-; X64-SLOW-NEXT: movl %r8d, %ecx
-; X64-SLOW-NEXT: shrq %cl, %rdi
-; X64-SLOW-NEXT: leaq (%rsi,%rsi), %rdx
-; X64-SLOW-NEXT: movl %r9d, %ecx
-; X64-SLOW-NEXT: shlq %cl, %rdx
-; X64-SLOW-NEXT: orq %rdi, %rdx
-; X64-SLOW-NEXT: retq
+; X86-LABEL: var_shift_i128:
+; X86: # %bb.0:
+; X86-NEXT: pushl %ebp
+; X86-NEXT: movl %esp, %ebp
+; X86-NEXT: pushl %ebx
+; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %esi
+; X86-NEXT: andl $-16, %esp
+; X86-NEXT: subl $16, %esp
+; X86-NEXT: movl 24(%ebp), %esi
+; X86-NEXT: movl 28(%ebp), %eax
+; X86-NEXT: movl 48(%ebp), %edx
+; X86-NEXT: movl 56(%ebp), %ecx
+; X86-NEXT: testb $64, %cl
+; X86-NEXT: movl 52(%ebp), %ebx
+; X86-NEXT: je .LBB6_1
+; X86-NEXT: # %bb.2:
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %esi, %edx
+; X86-NEXT: movl 32(%ebp), %esi
+; X86-NEXT: movl %ebx, %edi
+; X86-NEXT: movl %eax, %ebx
+; X86-NEXT: movl 36(%ebp), %eax
+; X86-NEXT: testb $32, %cl
+; X86-NEXT: je .LBB6_4
+; X86-NEXT: jmp .LBB6_5
+; X86-NEXT: .LBB6_1:
+; X86-NEXT: movl 40(%ebp), %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 44(%ebp), %edi
+; X86-NEXT: testb $32, %cl
+; X86-NEXT: jne .LBB6_5
+; X86-NEXT: .LBB6_4:
+; X86-NEXT: movl %esi, %eax
+; X86-NEXT: movl %ebx, %esi
+; X86-NEXT: movl %edx, %ebx
+; X86-NEXT: movl %edi, %edx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: .LBB6_5:
+; X86-NEXT: shrdl %cl, %edx, %edi
+; X86-NEXT: shrdl %cl, %ebx, %edx
+; X86-NEXT: shrdl %cl, %esi, %ebx
+; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: shrdl %cl, %eax, %esi
+; X86-NEXT: movl 8(%ebp), %eax
+; X86-NEXT: movl %esi, 12(%eax)
+; X86-NEXT: movl %ebx, 8(%eax)
+; X86-NEXT: movl %edx, 4(%eax)
+; X86-NEXT: movl %edi, (%eax)
+; X86-NEXT: leal -12(%ebp), %esp
+; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
+; X86-NEXT: popl %ebx
+; X86-NEXT: popl %ebp
+; X86-NEXT: retl $4
+;
+; X64-LABEL: var_shift_i128:
+; X64: # %bb.0:
+; X64-NEXT: movq %rdx, %rax
+; X64-NEXT: testb $64, %r8b
+; X64-NEXT: cmoveq %rdi, %rsi
+; X64-NEXT: cmoveq %rcx, %rdi
+; X64-NEXT: cmovneq %rcx, %rax
+; X64-NEXT: movl %r8d, %ecx
+; X64-NEXT: shrdq %cl, %rdi, %rax
+; X64-NEXT: shrdq %cl, %rsi, %rdi
+; X64-NEXT: movq %rdi, %rdx
+; X64-NEXT: retq
%tmp = tail call i128 @llvm.fshr.i128(i128 %x, i128 %y, i128 %z)
ret i128 %tmp
}
@@ -504,78 +320,37 @@ define i16 @const_shift_i16(i16 %x, i16 %y) nounwind {
}
define i32 @const_shift_i32(i32 %x, i32 %y) nounwind {
-; X86-FAST-LABEL: const_shift_i32:
-; X86-FAST: # %bb.0:
-; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-FAST-NEXT: shrdl $7, %ecx, %eax
-; X86-FAST-NEXT: retl
-;
-; X86-SLOW-LABEL: const_shift_i32:
-; X86-SLOW: # %bb.0:
-; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-SLOW-NEXT: shrl $7, %ecx
-; X86-SLOW-NEXT: shll $25, %eax
-; X86-SLOW-NEXT: orl %ecx, %eax
-; X86-SLOW-NEXT: retl
-;
-; X64-FAST-LABEL: const_shift_i32:
-; X64-FAST: # %bb.0:
-; X64-FAST-NEXT: movl %edi, %eax
-; X64-FAST-NEXT: shldl $25, %esi, %eax
-; X64-FAST-NEXT: retq
+; X86-LABEL: const_shift_i32:
+; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: shrdl $7, %ecx, %eax
+; X86-NEXT: retl
;
-; X64-SLOW-LABEL: const_shift_i32:
-; X64-SLOW: # %bb.0:
-; X64-SLOW-NEXT: # kill: def $esi killed $esi def $rsi
-; X64-SLOW-NEXT: # kill: def $edi killed $edi def $rdi
-; X64-SLOW-NEXT: shrl $7, %esi
-; X64-SLOW-NEXT: shll $25, %edi
-; X64-SLOW-NEXT: leal (%rdi,%rsi), %eax
-; X64-SLOW-NEXT: retq
+; X64-LABEL: const_shift_i32:
+; X64: # %bb.0:
+; X64-NEXT: movl %edi, %eax
+; X64-NEXT: shldl $25, %esi, %eax
+; X64-NEXT: retq
%tmp = tail call i32 @llvm.fshr.i32(i32 %x, i32 %y, i32 7)
ret i32 %tmp
}
define i64 @const_shift_i64(i64 %x, i64 %y) nounwind {
-; X86-FAST-LABEL: const_shift_i64:
-; X86-FAST: # %bb.0:
-; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-FAST-NEXT: shldl $25, %ecx, %edx
-; X86-FAST-NEXT: shrdl $7, %ecx, %eax
-; X86-FAST-NEXT: retl
-;
-; X86-SLOW-LABEL: const_shift_i64:
-; X86-SLOW: # %bb.0:
-; X86-SLOW-NEXT: pushl %esi
-; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-SLOW-NEXT: shrl $7, %ecx
-; X86-SLOW-NEXT: movl %esi, %eax
-; X86-SLOW-NEXT: shll $25, %eax
-; X86-SLOW-NEXT: orl %ecx, %eax
-; X86-SLOW-NEXT: shrl $7, %esi
-; X86-SLOW-NEXT: shll $25, %edx
-; X86-SLOW-NEXT: orl %esi, %edx
-; X86-SLOW-NEXT: popl %esi
-; X86-SLOW-NEXT: retl
-;
-; X64-FAST-LABEL: const_shift_i64:
-; X64-FAST: # %bb.0:
-; X64-FAST-NEXT: movq %rdi, %rax
-; X64-FAST-NEXT: shldq $57, %rsi, %rax
-; X64-FAST-NEXT: retq
+; X86-LABEL: const_shift_i64:
+; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: shldl $25, %ecx, %edx
+; X86-NEXT: shrdl $7, %ecx, %eax
+; X86-NEXT: retl
;
-; X64-SLOW-LABEL: const_shift_i64:
-; X64-SLOW: # %bb.0:
-; X64-SLOW-NEXT: shrq $7, %rsi
-; X64-SLOW-NEXT: shlq $57, %rdi
-; X64-SLOW-NEXT: leaq (%rdi,%rsi), %rax
-; X64-SLOW-NEXT: retq
+; X64-LABEL: const_shift_i64:
+; X64: # %bb.0:
+; X64-NEXT: movq %rdi, %rax
+; X64-NEXT: shldq $57, %rsi, %rax
+; X64-NEXT: retq
%tmp = tail call i64 @llvm.fshr.i64(i64 %x, i64 %y, i64 7)
ret i64 %tmp
}
diff --git a/llvm/test/CodeGen/X86/wide-scalar-shift-by-byte-multiple-legalization.ll b/llvm/test/CodeGen/X86/wide-scalar-shift-by-byte-multiple-legalization.ll
index e9ddc576c6cd8..77180c6aa1b10 100644
--- a/llvm/test/CodeGen/X86/wide-scalar-shift-by-byte-multiple-legalization.ll
+++ b/llvm/test/CodeGen/X86/wide-scalar-shift-by-byte-multiple-legalization.ll
@@ -192,112 +192,52 @@ define void @lshr_8bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-HAVE-BMI2-NEXT: movq %rax, (%rdx)
; X64-HAVE-BMI2-NEXT: retq
;
-; X86-NO-SHLD-NO-BMI2-LABEL: lshr_8bytes:
-; X86-NO-SHLD-NO-BMI2: # %bb.0:
-; X86-NO-SHLD-NO-BMI2-NEXT: pushl %ebx
-; X86-NO-SHLD-NO-BMI2-NEXT: pushl %edi
-; X86-NO-SHLD-NO-BMI2-NEXT: pushl %esi
-; X86-NO-SHLD-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-SHLD-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-SHLD-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-SHLD-NO-BMI2-NEXT: movl (%ecx), %ebx
-; X86-NO-SHLD-NO-BMI2-NEXT: movl 4(%ecx), %esi
-; X86-NO-SHLD-NO-BMI2-NEXT: movzbl (%eax), %eax
-; X86-NO-SHLD-NO-BMI2-NEXT: shlb $3, %al
-; X86-NO-SHLD-NO-BMI2-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-NEXT: leal (%esi,%esi), %edi
-; X86-NO-SHLD-NO-BMI2-NEXT: notb %cl
-; X86-NO-SHLD-NO-BMI2-NEXT: shll %cl, %edi
-; X86-NO-SHLD-NO-BMI2-NEXT: orl %ebx, %edi
-; X86-NO-SHLD-NO-BMI2-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-NEXT: shrl %cl, %esi
-; X86-NO-SHLD-NO-BMI2-NEXT: xorl %ecx, %ecx
-; X86-NO-SHLD-NO-BMI2-NEXT: testb $32, %al
-; X86-NO-SHLD-NO-BMI2-NEXT: cmovnel %esi, %edi
-; X86-NO-SHLD-NO-BMI2-NEXT: cmovel %esi, %ecx
-; X86-NO-SHLD-NO-BMI2-NEXT: movl %ecx, 4(%edx)
-; X86-NO-SHLD-NO-BMI2-NEXT: movl %edi, (%edx)
-; X86-NO-SHLD-NO-BMI2-NEXT: popl %esi
-; X86-NO-SHLD-NO-BMI2-NEXT: popl %edi
-; X86-NO-SHLD-NO-BMI2-NEXT: popl %ebx
-; X86-NO-SHLD-NO-BMI2-NEXT: retl
-;
-; X86-HAVE-SHLD-NO-BMI2-LABEL: lshr_8bytes:
-; X86-HAVE-SHLD-NO-BMI2: # %bb.0:
-; X86-HAVE-SHLD-NO-BMI2-NEXT: pushl %edi
-; X86-HAVE-SHLD-NO-BMI2-NEXT: pushl %esi
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl (%esi), %edx
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl 4(%esi), %esi
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movzbl (%ecx), %ecx
-; X86-HAVE-SHLD-NO-BMI2-NEXT: shlb $3, %cl
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl %esi, %edi
-; X86-HAVE-SHLD-NO-BMI2-NEXT: shrl %cl, %edi
-; X86-HAVE-SHLD-NO-BMI2-NEXT: shrdl %cl, %esi, %edx
-; X86-HAVE-SHLD-NO-BMI2-NEXT: xorl %esi, %esi
-; X86-HAVE-SHLD-NO-BMI2-NEXT: testb $32, %cl
-; X86-HAVE-SHLD-NO-BMI2-NEXT: cmovnel %edi, %edx
-; X86-HAVE-SHLD-NO-BMI2-NEXT: cmovel %edi, %esi
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl %esi, 4(%eax)
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl %edx, (%eax)
-; X86-HAVE-SHLD-NO-BMI2-NEXT: popl %esi
-; X86-HAVE-SHLD-NO-BMI2-NEXT: popl %edi
-; X86-HAVE-SHLD-NO-BMI2-NEXT: retl
-;
-; X86-NO-SHLD-HAVE-BMI2-LABEL: lshr_8bytes:
-; X86-NO-SHLD-HAVE-BMI2: # %bb.0:
-; X86-NO-SHLD-HAVE-BMI2-NEXT: pushl %ebx
-; X86-NO-SHLD-HAVE-BMI2-NEXT: pushl %edi
-; X86-NO-SHLD-HAVE-BMI2-NEXT: pushl %esi
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movl 4(%edx), %esi
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movzbl (%ecx), %ecx
-; X86-NO-SHLD-HAVE-BMI2-NEXT: shlb $3, %cl
-; X86-NO-SHLD-HAVE-BMI2-NEXT: shrxl %ecx, (%edx), %edx
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movl %ecx, %ebx
-; X86-NO-SHLD-HAVE-BMI2-NEXT: notb %bl
-; X86-NO-SHLD-HAVE-BMI2-NEXT: leal (%esi,%esi), %edi
-; X86-NO-SHLD-HAVE-BMI2-NEXT: shlxl %ebx, %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-NEXT: orl %edx, %edi
-; X86-NO-SHLD-HAVE-BMI2-NEXT: shrxl %ecx, %esi, %edx
-; X86-NO-SHLD-HAVE-BMI2-NEXT: xorl %esi, %esi
-; X86-NO-SHLD-HAVE-BMI2-NEXT: testb $32, %cl
-; X86-NO-SHLD-HAVE-BMI2-NEXT: cmovnel %edx, %edi
-; X86-NO-SHLD-HAVE-BMI2-NEXT: cmovel %edx, %esi
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movl %esi, 4(%eax)
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movl %edi, (%eax)
-; X86-NO-SHLD-HAVE-BMI2-NEXT: popl %esi
-; X86-NO-SHLD-HAVE-BMI2-NEXT: popl %edi
-; X86-NO-SHLD-HAVE-BMI2-NEXT: popl %ebx
-; X86-NO-SHLD-HAVE-BMI2-NEXT: retl
-;
-; X86-HAVE-SHLD-HAVE-BMI2-LABEL: lshr_8bytes:
-; X86-HAVE-SHLD-HAVE-BMI2: # %bb.0:
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: pushl %edi
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: pushl %esi
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movl (%esi), %edx
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movl 4(%esi), %esi
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movzbl (%ecx), %ecx
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: shlb $3, %cl
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: shrdl %cl, %esi, %edx
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: shrxl %ecx, %esi, %esi
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: xorl %edi, %edi
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: testb $32, %cl
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: cmovnel %esi, %edx
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: cmovel %esi, %edi
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movl %edi, 4(%eax)
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movl %edx, (%eax)
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: popl %esi
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: popl %edi
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: retl
+; X86-NO-BMI2-LABEL: lshr_8bytes:
+; X86-NO-BMI2: # %bb.0:
+; X86-NO-BMI2-NEXT: pushl %edi
+; X86-NO-BMI2-NEXT: pushl %esi
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-NO-BMI2-NEXT: movl (%esi), %edx
+; X86-NO-BMI2-NEXT: movl 4(%esi), %esi
+; X86-NO-BMI2-NEXT: movzbl (%ecx), %ecx
+; X86-NO-BMI2-NEXT: shlb $3, %cl
+; X86-NO-BMI2-NEXT: movl %esi, %edi
+; X86-NO-BMI2-NEXT: shrl %cl, %edi
+; X86-NO-BMI2-NEXT: shrdl %cl, %esi, %edx
+; X86-NO-BMI2-NEXT: xorl %esi, %esi
+; X86-NO-BMI2-NEXT: testb $32, %cl
+; X86-NO-BMI2-NEXT: cmovnel %edi, %edx
+; X86-NO-BMI2-NEXT: cmovel %edi, %esi
+; X86-NO-BMI2-NEXT: movl %esi, 4(%eax)
+; X86-NO-BMI2-NEXT: movl %edx, (%eax)
+; X86-NO-BMI2-NEXT: popl %esi
+; X86-NO-BMI2-NEXT: popl %edi
+; X86-NO-BMI2-NEXT: retl
+;
+; X86-HAVE-BMI2-LABEL: lshr_8bytes:
+; X86-HAVE-BMI2: # %bb.0:
+; X86-HAVE-BMI2-NEXT: pushl %edi
+; X86-HAVE-BMI2-NEXT: pushl %esi
+; X86-HAVE-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-BMI2-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-HAVE-BMI2-NEXT: movl (%esi), %edx
+; X86-HAVE-BMI2-NEXT: movl 4(%esi), %esi
+; X86-HAVE-BMI2-NEXT: movzbl (%ecx), %ecx
+; X86-HAVE-BMI2-NEXT: shlb $3, %cl
+; X86-HAVE-BMI2-NEXT: shrdl %cl, %esi, %edx
+; X86-HAVE-BMI2-NEXT: shrxl %ecx, %esi, %esi
+; X86-HAVE-BMI2-NEXT: xorl %edi, %edi
+; X86-HAVE-BMI2-NEXT: testb $32, %cl
+; X86-HAVE-BMI2-NEXT: cmovnel %esi, %edx
+; X86-HAVE-BMI2-NEXT: cmovel %esi, %edi
+; X86-HAVE-BMI2-NEXT: movl %edi, 4(%eax)
+; X86-HAVE-BMI2-NEXT: movl %edx, (%eax)
+; X86-HAVE-BMI2-NEXT: popl %esi
+; X86-HAVE-BMI2-NEXT: popl %edi
+; X86-HAVE-BMI2-NEXT: retl
%src = load i64, ptr %src.ptr, align 1
%byteOff = load i64, ptr %byteOff.ptr, align 1
%bitOff = shl i64 %byteOff, 3
@@ -323,113 +263,52 @@ define void @shl_8bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-HAVE-BMI2-NEXT: movq %rax, (%rdx)
; X64-HAVE-BMI2-NEXT: retq
;
-; X86-NO-SHLD-NO-BMI2-LABEL: shl_8bytes:
-; X86-NO-SHLD-NO-BMI2: # %bb.0:
-; X86-NO-SHLD-NO-BMI2-NEXT: pushl %ebx
-; X86-NO-SHLD-NO-BMI2-NEXT: pushl %edi
-; X86-NO-SHLD-NO-BMI2-NEXT: pushl %esi
-; X86-NO-SHLD-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-SHLD-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-SHLD-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-SHLD-NO-BMI2-NEXT: movl (%ecx), %esi
-; X86-NO-SHLD-NO-BMI2-NEXT: movl 4(%ecx), %ebx
-; X86-NO-SHLD-NO-BMI2-NEXT: movzbl (%eax), %eax
-; X86-NO-SHLD-NO-BMI2-NEXT: shlb $3, %al
-; X86-NO-SHLD-NO-BMI2-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-NEXT: movl %esi, %edi
-; X86-NO-SHLD-NO-BMI2-NEXT: shrl %edi
-; X86-NO-SHLD-NO-BMI2-NEXT: notb %cl
-; X86-NO-SHLD-NO-BMI2-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-NEXT: orl %ebx, %edi
-; X86-NO-SHLD-NO-BMI2-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-NEXT: shll %cl, %esi
-; X86-NO-SHLD-NO-BMI2-NEXT: xorl %ecx, %ecx
-; X86-NO-SHLD-NO-BMI2-NEXT: testb $32, %al
-; X86-NO-SHLD-NO-BMI2-NEXT: cmovnel %esi, %edi
-; X86-NO-SHLD-NO-BMI2-NEXT: cmovel %esi, %ecx
-; X86-NO-SHLD-NO-BMI2-NEXT: movl %ecx, (%edx)
-; X86-NO-SHLD-NO-BMI2-NEXT: movl %edi, 4(%edx)
-; X86-NO-SHLD-NO-BMI2-NEXT: popl %esi
-; X86-NO-SHLD-NO-BMI2-NEXT: popl %edi
-; X86-NO-SHLD-NO-BMI2-NEXT: popl %ebx
-; X86-NO-SHLD-NO-BMI2-NEXT: retl
-;
-; X86-HAVE-SHLD-NO-BMI2-LABEL: shl_8bytes:
-; X86-HAVE-SHLD-NO-BMI2: # %bb.0:
-; X86-HAVE-SHLD-NO-BMI2-NEXT: pushl %edi
-; X86-HAVE-SHLD-NO-BMI2-NEXT: pushl %esi
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl (%edx), %esi
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl 4(%edx), %edx
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movzbl (%ecx), %ecx
-; X86-HAVE-SHLD-NO-BMI2-NEXT: shlb $3, %cl
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl %esi, %edi
-; X86-HAVE-SHLD-NO-BMI2-NEXT: shll %cl, %edi
-; X86-HAVE-SHLD-NO-BMI2-NEXT: shldl %cl, %esi, %edx
-; X86-HAVE-SHLD-NO-BMI2-NEXT: xorl %esi, %esi
-; X86-HAVE-SHLD-NO-BMI2-NEXT: testb $32, %cl
-; X86-HAVE-SHLD-NO-BMI2-NEXT: cmovnel %edi, %edx
-; X86-HAVE-SHLD-NO-BMI2-NEXT: cmovel %edi, %esi
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl %edx, 4(%eax)
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl %esi, (%eax)
-; X86-HAVE-SHLD-NO-BMI2-NEXT: popl %esi
-; X86-HAVE-SHLD-NO-BMI2-NEXT: popl %edi
-; X86-HAVE-SHLD-NO-BMI2-NEXT: retl
-;
-; X86-NO-SHLD-HAVE-BMI2-LABEL: shl_8bytes:
-; X86-NO-SHLD-HAVE-BMI2: # %bb.0:
-; X86-NO-SHLD-HAVE-BMI2-NEXT: pushl %ebx
-; X86-NO-SHLD-HAVE-BMI2-NEXT: pushl %edi
-; X86-NO-SHLD-HAVE-BMI2-NEXT: pushl %esi
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movl (%edx), %esi
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movzbl (%ecx), %ecx
-; X86-NO-SHLD-HAVE-BMI2-NEXT: shlb $3, %cl
-; X86-NO-SHLD-HAVE-BMI2-NEXT: shlxl %ecx, 4(%edx), %edx
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movl %ecx, %ebx
-; X86-NO-SHLD-HAVE-BMI2-NEXT: notb %bl
-; X86-NO-SHLD-HAVE-BMI2-NEXT: shlxl %ecx, %esi, %edi
-; X86-NO-SHLD-HAVE-BMI2-NEXT: shrl %esi
-; X86-NO-SHLD-HAVE-BMI2-NEXT: shrxl %ebx, %esi, %esi
-; X86-NO-SHLD-HAVE-BMI2-NEXT: orl %edx, %esi
-; X86-NO-SHLD-HAVE-BMI2-NEXT: xorl %edx, %edx
-; X86-NO-SHLD-HAVE-BMI2-NEXT: testb $32, %cl
-; X86-NO-SHLD-HAVE-BMI2-NEXT: cmovnel %edi, %esi
-; X86-NO-SHLD-HAVE-BMI2-NEXT: cmovel %edi, %edx
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movl %edx, (%eax)
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movl %esi, 4(%eax)
-; X86-NO-SHLD-HAVE-BMI2-NEXT: popl %esi
-; X86-NO-SHLD-HAVE-BMI2-NEXT: popl %edi
-; X86-NO-SHLD-HAVE-BMI2-NEXT: popl %ebx
-; X86-NO-SHLD-HAVE-BMI2-NEXT: retl
-;
-; X86-HAVE-SHLD-HAVE-BMI2-LABEL: shl_8bytes:
-; X86-HAVE-SHLD-HAVE-BMI2: # %bb.0:
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: pushl %edi
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: pushl %esi
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movl (%edx), %esi
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movl 4(%edx), %edx
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movzbl (%ecx), %ecx
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: shlb $3, %cl
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: shldl %cl, %esi, %edx
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: shlxl %ecx, %esi, %esi
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: xorl %edi, %edi
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: testb $32, %cl
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: cmovnel %esi, %edx
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: cmovel %esi, %edi
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movl %edx, 4(%eax)
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movl %edi, (%eax)
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: popl %esi
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: popl %edi
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: retl
+; X86-NO-BMI2-LABEL: shl_8bytes:
+; X86-NO-BMI2: # %bb.0:
+; X86-NO-BMI2-NEXT: pushl %edi
+; X86-NO-BMI2-NEXT: pushl %esi
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NO-BMI2-NEXT: movl (%edx), %esi
+; X86-NO-BMI2-NEXT: movl 4(%edx), %edx
+; X86-NO-BMI2-NEXT: movzbl (%ecx), %ecx
+; X86-NO-BMI2-NEXT: shlb $3, %cl
+; X86-NO-BMI2-NEXT: movl %esi, %edi
+; X86-NO-BMI2-NEXT: shll %cl, %edi
+; X86-NO-BMI2-NEXT: shldl %cl, %esi, %edx
+; X86-NO-BMI2-NEXT: xorl %esi, %esi
+; X86-NO-BMI2-NEXT: testb $32, %cl
+; X86-NO-BMI2-NEXT: cmovnel %edi, %edx
+; X86-NO-BMI2-NEXT: cmovel %edi, %esi
+; X86-NO-BMI2-NEXT: movl %edx, 4(%eax)
+; X86-NO-BMI2-NEXT: movl %esi, (%eax)
+; X86-NO-BMI2-NEXT: popl %esi
+; X86-NO-BMI2-NEXT: popl %edi
+; X86-NO-BMI2-NEXT: retl
+;
+; X86-HAVE-BMI2-LABEL: shl_8bytes:
+; X86-HAVE-BMI2: # %bb.0:
+; X86-HAVE-BMI2-NEXT: pushl %edi
+; X86-HAVE-BMI2-NEXT: pushl %esi
+; X86-HAVE-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-HAVE-BMI2-NEXT: movl (%edx), %esi
+; X86-HAVE-BMI2-NEXT: movl 4(%edx), %edx
+; X86-HAVE-BMI2-NEXT: movzbl (%ecx), %ecx
+; X86-HAVE-BMI2-NEXT: shlb $3, %cl
+; X86-HAVE-BMI2-NEXT: shldl %cl, %esi, %edx
+; X86-HAVE-BMI2-NEXT: shlxl %ecx, %esi, %esi
+; X86-HAVE-BMI2-NEXT: xorl %edi, %edi
+; X86-HAVE-BMI2-NEXT: testb $32, %cl
+; X86-HAVE-BMI2-NEXT: cmovnel %esi, %edx
+; X86-HAVE-BMI2-NEXT: cmovel %esi, %edi
+; X86-HAVE-BMI2-NEXT: movl %edx, 4(%eax)
+; X86-HAVE-BMI2-NEXT: movl %edi, (%eax)
+; X86-HAVE-BMI2-NEXT: popl %esi
+; X86-HAVE-BMI2-NEXT: popl %edi
+; X86-HAVE-BMI2-NEXT: retl
%src = load i64, ptr %src.ptr, align 1
%byteOff = load i64, ptr %byteOff.ptr, align 1
%bitOff = shl i64 %byteOff, 3
@@ -455,113 +334,52 @@ define void @ashr_8bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-HAVE-BMI2-NEXT: movq %rax, (%rdx)
; X64-HAVE-BMI2-NEXT: retq
;
-; X86-NO-SHLD-NO-BMI2-LABEL: ashr_8bytes:
-; X86-NO-SHLD-NO-BMI2: # %bb.0:
-; X86-NO-SHLD-NO-BMI2-NEXT: pushl %ebx
-; X86-NO-SHLD-NO-BMI2-NEXT: pushl %edi
-; X86-NO-SHLD-NO-BMI2-NEXT: pushl %esi
-; X86-NO-SHLD-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-SHLD-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-SHLD-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-SHLD-NO-BMI2-NEXT: movl (%ecx), %ebx
-; X86-NO-SHLD-NO-BMI2-NEXT: movl 4(%ecx), %esi
-; X86-NO-SHLD-NO-BMI2-NEXT: movzbl (%eax), %eax
-; X86-NO-SHLD-NO-BMI2-NEXT: shlb $3, %al
-; X86-NO-SHLD-NO-BMI2-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-NEXT: leal (%esi,%esi), %edi
-; X86-NO-SHLD-NO-BMI2-NEXT: notb %cl
-; X86-NO-SHLD-NO-BMI2-NEXT: shll %cl, %edi
-; X86-NO-SHLD-NO-BMI2-NEXT: orl %ebx, %edi
-; X86-NO-SHLD-NO-BMI2-NEXT: movl %esi, %ebx
-; X86-NO-SHLD-NO-BMI2-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-NEXT: sarl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-NEXT: sarl $31, %esi
-; X86-NO-SHLD-NO-BMI2-NEXT: testb $32, %al
-; X86-NO-SHLD-NO-BMI2-NEXT: cmovnel %ebx, %edi
-; X86-NO-SHLD-NO-BMI2-NEXT: cmovel %ebx, %esi
-; X86-NO-SHLD-NO-BMI2-NEXT: movl %esi, 4(%edx)
-; X86-NO-SHLD-NO-BMI2-NEXT: movl %edi, (%edx)
-; X86-NO-SHLD-NO-BMI2-NEXT: popl %esi
-; X86-NO-SHLD-NO-BMI2-NEXT: popl %edi
-; X86-NO-SHLD-NO-BMI2-NEXT: popl %ebx
-; X86-NO-SHLD-NO-BMI2-NEXT: retl
-;
-; X86-HAVE-SHLD-NO-BMI2-LABEL: ashr_8bytes:
-; X86-HAVE-SHLD-NO-BMI2: # %bb.0:
-; X86-HAVE-SHLD-NO-BMI2-NEXT: pushl %edi
-; X86-HAVE-SHLD-NO-BMI2-NEXT: pushl %esi
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl (%esi), %edx
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl 4(%esi), %esi
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movzbl (%ecx), %ecx
-; X86-HAVE-SHLD-NO-BMI2-NEXT: shlb $3, %cl
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl %esi, %edi
-; X86-HAVE-SHLD-NO-BMI2-NEXT: sarl %cl, %edi
-; X86-HAVE-SHLD-NO-BMI2-NEXT: shrdl %cl, %esi, %edx
-; X86-HAVE-SHLD-NO-BMI2-NEXT: sarl $31, %esi
-; X86-HAVE-SHLD-NO-BMI2-NEXT: testb $32, %cl
-; X86-HAVE-SHLD-NO-BMI2-NEXT: cmovnel %edi, %edx
-; X86-HAVE-SHLD-NO-BMI2-NEXT: cmovel %edi, %esi
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl %esi, 4(%eax)
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl %edx, (%eax)
-; X86-HAVE-SHLD-NO-BMI2-NEXT: popl %esi
-; X86-HAVE-SHLD-NO-BMI2-NEXT: popl %edi
-; X86-HAVE-SHLD-NO-BMI2-NEXT: retl
-;
-; X86-NO-SHLD-HAVE-BMI2-LABEL: ashr_8bytes:
-; X86-NO-SHLD-HAVE-BMI2: # %bb.0:
-; X86-NO-SHLD-HAVE-BMI2-NEXT: pushl %ebx
-; X86-NO-SHLD-HAVE-BMI2-NEXT: pushl %edi
-; X86-NO-SHLD-HAVE-BMI2-NEXT: pushl %esi
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movl 4(%esi), %ecx
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movzbl (%edx), %edx
-; X86-NO-SHLD-HAVE-BMI2-NEXT: shlb $3, %dl
-; X86-NO-SHLD-HAVE-BMI2-NEXT: shrxl %edx, (%esi), %esi
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movl %edx, %ebx
-; X86-NO-SHLD-HAVE-BMI2-NEXT: notb %bl
-; X86-NO-SHLD-HAVE-BMI2-NEXT: leal (%ecx,%ecx), %edi
-; X86-NO-SHLD-HAVE-BMI2-NEXT: shlxl %ebx, %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-NEXT: orl %esi, %edi
-; X86-NO-SHLD-HAVE-BMI2-NEXT: sarxl %edx, %ecx, %esi
-; X86-NO-SHLD-HAVE-BMI2-NEXT: sarl $31, %ecx
-; X86-NO-SHLD-HAVE-BMI2-NEXT: testb $32, %dl
-; X86-NO-SHLD-HAVE-BMI2-NEXT: cmovnel %esi, %edi
-; X86-NO-SHLD-HAVE-BMI2-NEXT: cmovel %esi, %ecx
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movl %ecx, 4(%eax)
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movl %edi, (%eax)
-; X86-NO-SHLD-HAVE-BMI2-NEXT: popl %esi
-; X86-NO-SHLD-HAVE-BMI2-NEXT: popl %edi
-; X86-NO-SHLD-HAVE-BMI2-NEXT: popl %ebx
-; X86-NO-SHLD-HAVE-BMI2-NEXT: retl
-;
-; X86-HAVE-SHLD-HAVE-BMI2-LABEL: ashr_8bytes:
-; X86-HAVE-SHLD-HAVE-BMI2: # %bb.0:
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: pushl %edi
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: pushl %esi
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movl (%esi), %edx
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movl 4(%esi), %esi
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movzbl (%ecx), %ecx
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: shlb $3, %cl
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: shrdl %cl, %esi, %edx
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: sarxl %ecx, %esi, %edi
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: sarl $31, %esi
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: testb $32, %cl
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: cmovnel %edi, %edx
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: cmovel %edi, %esi
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movl %esi, 4(%eax)
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movl %edx, (%eax)
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: popl %esi
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: popl %edi
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: retl
+; X86-NO-BMI2-LABEL: ashr_8bytes:
+; X86-NO-BMI2: # %bb.0:
+; X86-NO-BMI2-NEXT: pushl %edi
+; X86-NO-BMI2-NEXT: pushl %esi
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-NO-BMI2-NEXT: movl (%esi), %edx
+; X86-NO-BMI2-NEXT: movl 4(%esi), %esi
+; X86-NO-BMI2-NEXT: movzbl (%ecx), %ecx
+; X86-NO-BMI2-NEXT: shlb $3, %cl
+; X86-NO-BMI2-NEXT: movl %esi, %edi
+; X86-NO-BMI2-NEXT: sarl %cl, %edi
+; X86-NO-BMI2-NEXT: shrdl %cl, %esi, %edx
+; X86-NO-BMI2-NEXT: sarl $31, %esi
+; X86-NO-BMI2-NEXT: testb $32, %cl
+; X86-NO-BMI2-NEXT: cmovnel %edi, %edx
+; X86-NO-BMI2-NEXT: cmovel %edi, %esi
+; X86-NO-BMI2-NEXT: movl %esi, 4(%eax)
+; X86-NO-BMI2-NEXT: movl %edx, (%eax)
+; X86-NO-BMI2-NEXT: popl %esi
+; X86-NO-BMI2-NEXT: popl %edi
+; X86-NO-BMI2-NEXT: retl
+;
+; X86-HAVE-BMI2-LABEL: ashr_8bytes:
+; X86-HAVE-BMI2: # %bb.0:
+; X86-HAVE-BMI2-NEXT: pushl %edi
+; X86-HAVE-BMI2-NEXT: pushl %esi
+; X86-HAVE-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-HAVE-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-BMI2-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-HAVE-BMI2-NEXT: movl (%esi), %edx
+; X86-HAVE-BMI2-NEXT: movl 4(%esi), %esi
+; X86-HAVE-BMI2-NEXT: movzbl (%ecx), %ecx
+; X86-HAVE-BMI2-NEXT: shlb $3, %cl
+; X86-HAVE-BMI2-NEXT: shrdl %cl, %esi, %edx
+; X86-HAVE-BMI2-NEXT: sarxl %ecx, %esi, %edi
+; X86-HAVE-BMI2-NEXT: sarl $31, %esi
+; X86-HAVE-BMI2-NEXT: testb $32, %cl
+; X86-HAVE-BMI2-NEXT: cmovnel %edi, %edx
+; X86-HAVE-BMI2-NEXT: cmovel %edi, %esi
+; X86-HAVE-BMI2-NEXT: movl %esi, 4(%eax)
+; X86-HAVE-BMI2-NEXT: movl %edx, (%eax)
+; X86-HAVE-BMI2-NEXT: popl %esi
+; X86-HAVE-BMI2-NEXT: popl %edi
+; X86-HAVE-BMI2-NEXT: retl
%src = load i64, ptr %src.ptr, align 1
%byteOff = load i64, ptr %byteOff.ptr, align 1
%bitOff = shl i64 %byteOff, 3
@@ -571,80 +389,38 @@ define void @ashr_8bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
}
define void @lshr_16bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
-; X64-NO-SHLD-NO-BMI2-LABEL: lshr_16bytes:
-; X64-NO-SHLD-NO-BMI2: # %bb.0:
-; X64-NO-SHLD-NO-BMI2-NEXT: movq (%rdi), %r8
-; X64-NO-SHLD-NO-BMI2-NEXT: movq 8(%rdi), %rdi
-; X64-NO-SHLD-NO-BMI2-NEXT: movzbl (%rsi), %eax
-; X64-NO-SHLD-NO-BMI2-NEXT: shlb $3, %al
-; X64-NO-SHLD-NO-BMI2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-NEXT: shrq %cl, %r8
-; X64-NO-SHLD-NO-BMI2-NEXT: leaq (%rdi,%rdi), %rsi
-; X64-NO-SHLD-NO-BMI2-NEXT: notb %cl
-; X64-NO-SHLD-NO-BMI2-NEXT: shlq %cl, %rsi
-; X64-NO-SHLD-NO-BMI2-NEXT: orq %r8, %rsi
-; X64-NO-SHLD-NO-BMI2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-NEXT: shrq %cl, %rdi
-; X64-NO-SHLD-NO-BMI2-NEXT: xorl %ecx, %ecx
-; X64-NO-SHLD-NO-BMI2-NEXT: testb $64, %al
-; X64-NO-SHLD-NO-BMI2-NEXT: cmovneq %rdi, %rsi
-; X64-NO-SHLD-NO-BMI2-NEXT: cmoveq %rdi, %rcx
-; X64-NO-SHLD-NO-BMI2-NEXT: movq %rcx, 8(%rdx)
-; X64-NO-SHLD-NO-BMI2-NEXT: movq %rsi, (%rdx)
-; X64-NO-SHLD-NO-BMI2-NEXT: retq
-;
-; X64-HAVE-SHLD-NO-BMI2-LABEL: lshr_16bytes:
-; X64-HAVE-SHLD-NO-BMI2: # %bb.0:
-; X64-HAVE-SHLD-NO-BMI2-NEXT: movq (%rdi), %rax
-; X64-HAVE-SHLD-NO-BMI2-NEXT: movq 8(%rdi), %rdi
-; X64-HAVE-SHLD-NO-BMI2-NEXT: movzbl (%rsi), %ecx
-; X64-HAVE-SHLD-NO-BMI2-NEXT: shlb $3, %cl
-; X64-HAVE-SHLD-NO-BMI2-NEXT: movq %rdi, %rsi
-; X64-HAVE-SHLD-NO-BMI2-NEXT: shrq %cl, %rsi
-; X64-HAVE-SHLD-NO-BMI2-NEXT: shrdq %cl, %rdi, %rax
-; X64-HAVE-SHLD-NO-BMI2-NEXT: xorl %edi, %edi
-; X64-HAVE-SHLD-NO-BMI2-NEXT: testb $64, %cl
-; X64-HAVE-SHLD-NO-BMI2-NEXT: cmovneq %rsi, %rax
-; X64-HAVE-SHLD-NO-BMI2-NEXT: cmoveq %rsi, %rdi
-; X64-HAVE-SHLD-NO-BMI2-NEXT: movq %rdi, 8(%rdx)
-; X64-HAVE-SHLD-NO-BMI2-NEXT: movq %rax, (%rdx)
-; X64-HAVE-SHLD-NO-BMI2-NEXT: retq
-;
-; X64-NO-SHLD-HAVE-BMI2-LABEL: lshr_16bytes:
-; X64-NO-SHLD-HAVE-BMI2: # %bb.0:
-; X64-NO-SHLD-HAVE-BMI2-NEXT: movq 8(%rdi), %rax
-; X64-NO-SHLD-HAVE-BMI2-NEXT: movzbl (%rsi), %ecx
-; X64-NO-SHLD-HAVE-BMI2-NEXT: shlb $3, %cl
-; X64-NO-SHLD-HAVE-BMI2-NEXT: shrxq %rcx, (%rdi), %rsi
-; X64-NO-SHLD-HAVE-BMI2-NEXT: movl %ecx, %edi
-; X64-NO-SHLD-HAVE-BMI2-NEXT: notb %dil
-; X64-NO-SHLD-HAVE-BMI2-NEXT: leaq (%rax,%rax), %r8
-; X64-NO-SHLD-HAVE-BMI2-NEXT: shlxq %rdi, %r8, %rdi
-; X64-NO-SHLD-HAVE-BMI2-NEXT: orq %rsi, %rdi
-; X64-NO-SHLD-HAVE-BMI2-NEXT: shrxq %rcx, %rax, %rax
-; X64-NO-SHLD-HAVE-BMI2-NEXT: xorl %esi, %esi
-; X64-NO-SHLD-HAVE-BMI2-NEXT: testb $64, %cl
-; X64-NO-SHLD-HAVE-BMI2-NEXT: cmovneq %rax, %rdi
-; X64-NO-SHLD-HAVE-BMI2-NEXT: cmoveq %rax, %rsi
-; X64-NO-SHLD-HAVE-BMI2-NEXT: movq %rsi, 8(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-NEXT: movq %rdi, (%rdx)
-; X64-NO-SHLD-HAVE-BMI2-NEXT: retq
-;
-; X64-HAVE-SHLD-HAVE-BMI2-LABEL: lshr_16bytes:
-; X64-HAVE-SHLD-HAVE-BMI2: # %bb.0:
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: movq (%rdi), %rax
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: movq 8(%rdi), %rdi
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: movzbl (%rsi), %ecx
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: shlb $3, %cl
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: shrdq %cl, %rdi, %rax
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: shrxq %rcx, %rdi, %rsi
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: xorl %edi, %edi
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: testb $64, %cl
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: cmovneq %rsi, %rax
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: cmoveq %rsi, %rdi
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: movq %rdi, 8(%rdx)
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: movq %rax, (%rdx)
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: retq
+; X64-NO-BMI2-LABEL: lshr_16bytes:
+; X64-NO-BMI2: # %bb.0:
+; X64-NO-BMI2-NEXT: movq (%rdi), %rax
+; X64-NO-BMI2-NEXT: movq 8(%rdi), %rdi
+; X64-NO-BMI2-NEXT: movzbl (%rsi), %ecx
+; X64-NO-BMI2-NEXT: shlb $3, %cl
+; X64-NO-BMI2-NEXT: movq %rdi, %rsi
+; X64-NO-BMI2-NEXT: shrq %cl, %rsi
+; X64-NO-BMI2-NEXT: shrdq %cl, %rdi, %rax
+; X64-NO-BMI2-NEXT: xorl %edi, %edi
+; X64-NO-BMI2-NEXT: testb $64, %cl
+; X64-NO-BMI2-NEXT: cmovneq %rsi, %rax
+; X64-NO-BMI2-NEXT: cmoveq %rsi, %rdi
+; X64-NO-BMI2-NEXT: movq %rdi, 8(%rdx)
+; X64-NO-BMI2-NEXT: movq %rax, (%rdx)
+; X64-NO-BMI2-NEXT: retq
+;
+; X64-HAVE-BMI2-LABEL: lshr_16bytes:
+; X64-HAVE-BMI2: # %bb.0:
+; X64-HAVE-BMI2-NEXT: movq (%rdi), %rax
+; X64-HAVE-BMI2-NEXT: movq 8(%rdi), %rdi
+; X64-HAVE-BMI2-NEXT: movzbl (%rsi), %ecx
+; X64-HAVE-BMI2-NEXT: shlb $3, %cl
+; X64-HAVE-BMI2-NEXT: shrdq %cl, %rdi, %rax
+; X64-HAVE-BMI2-NEXT: shrxq %rcx, %rdi, %rsi
+; X64-HAVE-BMI2-NEXT: xorl %edi, %edi
+; X64-HAVE-BMI2-NEXT: testb $64, %cl
+; X64-HAVE-BMI2-NEXT: cmovneq %rsi, %rax
+; X64-HAVE-BMI2-NEXT: cmoveq %rsi, %rdi
+; X64-HAVE-BMI2-NEXT: movq %rdi, 8(%rdx)
+; X64-HAVE-BMI2-NEXT: movq %rax, (%rdx)
+; X64-HAVE-BMI2-NEXT: retq
;
; X86-NO-SHLD-NO-BMI2-SSE2-LABEL: lshr_16bytes:
; X86-NO-SHLD-NO-BMI2-SSE2: # %bb.0:
@@ -652,58 +428,39 @@ define void @lshr_16bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: pushl %ebx
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: pushl %edi
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: pushl %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: subl $60, %esp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: subl $44, %esp
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl (%ecx), %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 4(%ecx), %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 8(%ecx), %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 12(%ecx), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb (%eax), %ah
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ah, %al
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shlb $3, %al
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl (%edx), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 4(%edx), %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 8(%edx), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 12(%edx), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb (%ecx), %ch
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shlb $3, %cl
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: xorps %xmm0, %xmm0
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: andb $12, %ah
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movzbl %ah, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 20(%esp,%ebp), %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: notb %dl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 24(%esp,%ebp), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: leal (%ecx,%ecx), %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %ebx, %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 16(%esp,%ebp), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: addl %esi, %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %ebx, %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 28(%esp,%ebp), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: leal (%ebx,%ebx), %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, (%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: andb $12, %ch
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movzbl %ch, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 8(%esp,%ebx), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl (%esp,%ebx), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 4(%esp,%ebx), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebp, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %esi, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 12(%esp,%ebx), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %ebx, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %ebp, %edx
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, 12(%edx)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebp, 8(%edx)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, (%edx)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, 4(%edx)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: addl $60, %esp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, 8(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, 12(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, (%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, 4(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: addl $44, %esp
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: popl %esi
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: popl %edi
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: popl %ebx
@@ -762,46 +519,38 @@ define void @lshr_16bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: pushl %edi
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: pushl %esi
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: subl $44, %esp
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ebp
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl (%ecx), %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 4(%ecx), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 8(%ecx), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 12(%ecx), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movzbl (%eax), %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebx, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlb $3, %al
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl (%edx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 4(%edx), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 8(%edx), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 12(%edx), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movzbl (%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlb $3, %cl
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: xorps %xmm0, %xmm0
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebx, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, (%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: andb $12, %bl
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movzbl %bl, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 4(%esp,%edi), %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 8(%esp,%edi), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ecx, %ebx, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: notb %al
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leal (%esi,%esi), %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %eax, %edx, %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %ebp, %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ecx, (%esp,%edi), %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addl %ebx, %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %eax, %ebx, %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %ebp, %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 12(%esp,%edi), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leal (%edi,%edi), %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %eax, %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ecx, %esi, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ecx, %edi, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, 12(%esi)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 8(%esi)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebx, (%esi)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, 4(%esi)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, (%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: andb $12, %al
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movzbl %al, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 8(%esp,%eax), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl (%esp,%eax), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 4(%esp,%eax), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %ebx, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 12(%esp,%eax), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %eax, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebx, 8(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ecx, %eax, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 12(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %esi, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, (%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edi, 4(%ebp)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addl $44, %esp
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: popl %esi
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: popl %edi
@@ -861,55 +610,34 @@ define void @lshr_16bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: pushl %ebx
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: pushl %edi
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: pushl %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: subl $60, %esp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: subl $44, %esp
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movups (%ecx), %xmm0
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movzbl (%eax), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shlb $3, %al
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movups (%edx), %xmm0
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movzbl (%ecx), %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shlb $3, %cl
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: xorps %xmm1, %xmm1
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: andb $12, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movzbl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 16(%esp,%edi), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 20(%esp,%edi), %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: notb %dl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: addl %esi, %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %ebx, %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 24(%esp,%edi), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 28(%esp,%edi), %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: leal (%edi,%edi), %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %esi, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: addl %ebx, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %esi, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edi, 12(%edx)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, 4(%edx)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebp, 8(%edx)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, (%edx)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: addl $60, %esp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm0, (%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: andb $12, %dl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movzbl %dl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 12(%esp,%ebx), %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 8(%esp,%ebx), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebp, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %edx, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl (%esp,%ebx), %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 4(%esp,%ebx), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %ebp, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, 4(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edi, 8(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %eax, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, 12(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, (%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: addl $44, %esp
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: popl %esi
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: popl %edi
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: popl %ebx
@@ -962,43 +690,35 @@ define void @lshr_16bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: pushl %ebx
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: pushl %edi
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: pushl %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: subl $60, %esp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: subl $44, %esp
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movups (%ecx), %xmm0
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movzbl (%eax), %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlb $3, %al
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movups (%edx), %xmm0
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movzbl (%ecx), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlb $3, %cl
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: xorps %xmm1, %xmm1
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm0, (%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: andb $12, %dl
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movzbl %dl, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ecx, 16(%esp,%edi), %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: notb %al
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 20(%esp,%edi), %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 24(%esp,%edi), %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: addl %edx, %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %eax, %edx, %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %ebp, %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 28(%esp,%edi), %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leal (%ebp,%ebp), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %eax, %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ecx, %ebx, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %esi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: addl %ebx, %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %eax, %ebx, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ecx, %ebp, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, 12(%esi)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 4(%esi)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edi, 8(%esi)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, (%esi)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: addl $60, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movzbl %dl, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 12(%esp,%ebx), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 8(%esp,%ebx), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ebp, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %edx, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl (%esp,%ebx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 4(%esp,%ebx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %ebp, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ebx, 4(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edi, 8(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ecx, %edx, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, 12(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %eax, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %esi, (%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: addl $44, %esp
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: popl %esi
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: popl %edi
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: popl %ebx
@@ -1052,55 +772,34 @@ define void @lshr_16bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: pushl %ebx
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: pushl %edi
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: pushl %esi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: subl $60, %esp
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: subl $44, %esp
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: vmovups (%ecx), %xmm0
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movzbl (%eax), %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ecx, %eax
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shlb $3, %al
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: vmovups (%edx), %xmm0
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movzbl (%ecx), %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shlb $3, %cl
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: vxorps %xmm1, %xmm1, %xmm1
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: vmovaps %xmm1, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: vmovaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: andb $12, %cl
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movzbl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 16(%esp,%edi), %ebx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 20(%esp,%edi), %esi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %edx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: notb %dl
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: addl %esi, %esi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shll %cl, %esi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: orl %ebx, %esi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 24(%esp,%edi), %ebx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ebx, %esi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrl %cl, %esi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 28(%esp,%edi), %edi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: leal (%edi,%edi), %ebp
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shll %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: orl %esi, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrl %cl, %esi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: addl %ebx, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: orl %esi, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edi, 12(%edx)
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ebx, 4(%edx)
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ebp, 8(%edx)
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, (%edx)
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: addl $60, %esp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: vmovaps %xmm0, (%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: andb $12, %dl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movzbl %dl, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 12(%esp,%ebx), %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 8(%esp,%ebx), %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ebp, %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrdl %cl, %edx, %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl (%esp,%ebx), %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 4(%esp,%ebx), %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrdl %cl, %ebp, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ebx, 4(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edi, 8(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrdl %cl, %eax, %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrl %cl, %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, 12(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %esi, (%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: addl $44, %esp
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: popl %esi
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: popl %edi
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: popl %ebx
@@ -1153,43 +852,35 @@ define void @lshr_16bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: pushl %ebx
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: pushl %edi
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: pushl %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: subl $60, %esp
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: subl $44, %esp
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: vmovups (%ecx), %xmm0
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movzbl (%eax), %edx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edx, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlb $3, %al
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: vmovups (%edx), %xmm0
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movzbl (%ecx), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edx, %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlb $3, %cl
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: vxorps %xmm1, %xmm1, %xmm1
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: vmovaps %xmm1, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: vmovaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: vmovaps %xmm0, (%esp)
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: andb $12, %dl
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movzbl %dl, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxl %ecx, 16(%esp,%edi), %ebp
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: notb %al
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 20(%esp,%edi), %edx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 24(%esp,%edi), %ebx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: addl %edx, %edx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxl %eax, %edx, %edx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: orl %ebp, %edx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 28(%esp,%edi), %ebp
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: leal (%ebp,%ebp), %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxl %eax, %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxl %ecx, %ebx, %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: orl %esi, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: addl %ebx, %ebx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxl %eax, %ebx, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: orl %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxl %ecx, %ebp, %ecx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %ecx, 12(%esi)
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, 4(%esi)
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edi, 8(%esi)
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edx, (%esi)
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: addl $60, %esp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movzbl %dl, %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 12(%esp,%ebx), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 8(%esp,%ebx), %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %ebp, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrdl %cl, %edx, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl (%esp,%ebx), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 4(%esp,%ebx), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrdl %cl, %ebp, %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %ebx, 4(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edi, 8(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxl %ecx, %edx, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edx, 12(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrdl %cl, %eax, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %esi, (%ebp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: addl $44, %esp
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: popl %esi
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: popl %edi
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: popl %ebx
@@ -1245,80 +936,38 @@ define void @lshr_16bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
}
define void @lshr_16bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) nounwind {
-; X64-NO-SHLD-NO-BMI2-LABEL: lshr_16bytes_dwordOff:
-; X64-NO-SHLD-NO-BMI2: # %bb.0:
-; X64-NO-SHLD-NO-BMI2-NEXT: movq (%rdi), %r8
-; X64-NO-SHLD-NO-BMI2-NEXT: movq 8(%rdi), %rdi
-; X64-NO-SHLD-NO-BMI2-NEXT: movzbl (%rsi), %eax
-; X64-NO-SHLD-NO-BMI2-NEXT: shlb $5, %al
-; X64-NO-SHLD-NO-BMI2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-NEXT: shrq %cl, %r8
-; X64-NO-SHLD-NO-BMI2-NEXT: leaq (%rdi,%rdi), %rsi
-; X64-NO-SHLD-NO-BMI2-NEXT: notb %cl
-; X64-NO-SHLD-NO-BMI2-NEXT: shlq %cl, %rsi
-; X64-NO-SHLD-NO-BMI2-NEXT: orq %r8, %rsi
-; X64-NO-SHLD-NO-BMI2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-NEXT: shrq %cl, %rdi
-; X64-NO-SHLD-NO-BMI2-NEXT: xorl %ecx, %ecx
-; X64-NO-SHLD-NO-BMI2-NEXT: testb $64, %al
-; X64-NO-SHLD-NO-BMI2-NEXT: cmovneq %rdi, %rsi
-; X64-NO-SHLD-NO-BMI2-NEXT: cmoveq %rdi, %rcx
-; X64-NO-SHLD-NO-BMI2-NEXT: movq %rcx, 8(%rdx)
-; X64-NO-SHLD-NO-BMI2-NEXT: movq %rsi, (%rdx)
-; X64-NO-SHLD-NO-BMI2-NEXT: retq
-;
-; X64-HAVE-SHLD-NO-BMI2-LABEL: lshr_16bytes_dwordOff:
-; X64-HAVE-SHLD-NO-BMI2: # %bb.0:
-; X64-HAVE-SHLD-NO-BMI2-NEXT: movq (%rdi), %rax
-; X64-HAVE-SHLD-NO-BMI2-NEXT: movq 8(%rdi), %rdi
-; X64-HAVE-SHLD-NO-BMI2-NEXT: movzbl (%rsi), %ecx
-; X64-HAVE-SHLD-NO-BMI2-NEXT: shlb $5, %cl
-; X64-HAVE-SHLD-NO-BMI2-NEXT: movq %rdi, %rsi
-; X64-HAVE-SHLD-NO-BMI2-NEXT: shrq %cl, %rsi
-; X64-HAVE-SHLD-NO-BMI2-NEXT: shrdq %cl, %rdi, %rax
-; X64-HAVE-SHLD-NO-BMI2-NEXT: xorl %edi, %edi
-; X64-HAVE-SHLD-NO-BMI2-NEXT: testb $64, %cl
-; X64-HAVE-SHLD-NO-BMI2-NEXT: cmovneq %rsi, %rax
-; X64-HAVE-SHLD-NO-BMI2-NEXT: cmoveq %rsi, %rdi
-; X64-HAVE-SHLD-NO-BMI2-NEXT: movq %rdi, 8(%rdx)
-; X64-HAVE-SHLD-NO-BMI2-NEXT: movq %rax, (%rdx)
-; X64-HAVE-SHLD-NO-BMI2-NEXT: retq
-;
-; X64-NO-SHLD-HAVE-BMI2-LABEL: lshr_16bytes_dwordOff:
-; X64-NO-SHLD-HAVE-BMI2: # %bb.0:
-; X64-NO-SHLD-HAVE-BMI2-NEXT: movq 8(%rdi), %rax
-; X64-NO-SHLD-HAVE-BMI2-NEXT: movzbl (%rsi), %ecx
-; X64-NO-SHLD-HAVE-BMI2-NEXT: shlb $5, %cl
-; X64-NO-SHLD-HAVE-BMI2-NEXT: shrxq %rcx, (%rdi), %rsi
-; X64-NO-SHLD-HAVE-BMI2-NEXT: movl %ecx, %edi
-; X64-NO-SHLD-HAVE-BMI2-NEXT: notb %dil
-; X64-NO-SHLD-HAVE-BMI2-NEXT: leaq (%rax,%rax), %r8
-; X64-NO-SHLD-HAVE-BMI2-NEXT: shlxq %rdi, %r8, %rdi
-; X64-NO-SHLD-HAVE-BMI2-NEXT: orq %rsi, %rdi
-; X64-NO-SHLD-HAVE-BMI2-NEXT: shrxq %rcx, %rax, %rax
-; X64-NO-SHLD-HAVE-BMI2-NEXT: xorl %esi, %esi
-; X64-NO-SHLD-HAVE-BMI2-NEXT: testb $64, %cl
-; X64-NO-SHLD-HAVE-BMI2-NEXT: cmovneq %rax, %rdi
-; X64-NO-SHLD-HAVE-BMI2-NEXT: cmoveq %rax, %rsi
-; X64-NO-SHLD-HAVE-BMI2-NEXT: movq %rsi, 8(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-NEXT: movq %rdi, (%rdx)
-; X64-NO-SHLD-HAVE-BMI2-NEXT: retq
-;
-; X64-HAVE-SHLD-HAVE-BMI2-LABEL: lshr_16bytes_dwordOff:
-; X64-HAVE-SHLD-HAVE-BMI2: # %bb.0:
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: movq (%rdi), %rax
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: movq 8(%rdi), %rdi
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: movzbl (%rsi), %ecx
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: shlb $5, %cl
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: shrdq %cl, %rdi, %rax
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: shrxq %rcx, %rdi, %rsi
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: xorl %edi, %edi
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: testb $64, %cl
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: cmovneq %rsi, %rax
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: cmoveq %rsi, %rdi
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: movq %rdi, 8(%rdx)
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: movq %rax, (%rdx)
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: retq
+; X64-NO-BMI2-LABEL: lshr_16bytes_dwordOff:
+; X64-NO-BMI2: # %bb.0:
+; X64-NO-BMI2-NEXT: movq (%rdi), %rax
+; X64-NO-BMI2-NEXT: movq 8(%rdi), %rdi
+; X64-NO-BMI2-NEXT: movzbl (%rsi), %ecx
+; X64-NO-BMI2-NEXT: shlb $5, %cl
+; X64-NO-BMI2-NEXT: movq %rdi, %rsi
+; X64-NO-BMI2-NEXT: shrq %cl, %rsi
+; X64-NO-BMI2-NEXT: shrdq %cl, %rdi, %rax
+; X64-NO-BMI2-NEXT: xorl %edi, %edi
+; X64-NO-BMI2-NEXT: testb $64, %cl
+; X64-NO-BMI2-NEXT: cmovneq %rsi, %rax
+; X64-NO-BMI2-NEXT: cmoveq %rsi, %rdi
+; X64-NO-BMI2-NEXT: movq %rdi, 8(%rdx)
+; X64-NO-BMI2-NEXT: movq %rax, (%rdx)
+; X64-NO-BMI2-NEXT: retq
+;
+; X64-HAVE-BMI2-LABEL: lshr_16bytes_dwordOff:
+; X64-HAVE-BMI2: # %bb.0:
+; X64-HAVE-BMI2-NEXT: movq (%rdi), %rax
+; X64-HAVE-BMI2-NEXT: movq 8(%rdi), %rdi
+; X64-HAVE-BMI2-NEXT: movzbl (%rsi), %ecx
+; X64-HAVE-BMI2-NEXT: shlb $5, %cl
+; X64-HAVE-BMI2-NEXT: shrdq %cl, %rdi, %rax
+; X64-HAVE-BMI2-NEXT: shrxq %rcx, %rdi, %rsi
+; X64-HAVE-BMI2-NEXT: xorl %edi, %edi
+; X64-HAVE-BMI2-NEXT: testb $64, %cl
+; X64-HAVE-BMI2-NEXT: cmovneq %rsi, %rax
+; X64-HAVE-BMI2-NEXT: cmoveq %rsi, %rdi
+; X64-HAVE-BMI2-NEXT: movq %rdi, 8(%rdx)
+; X64-HAVE-BMI2-NEXT: movq %rax, (%rdx)
+; X64-HAVE-BMI2-NEXT: retq
;
; X86-SSE2-LABEL: lshr_16bytes_dwordOff:
; X86-SSE2: # %bb.0:
@@ -1397,147 +1046,80 @@ define void @lshr_16bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) no
}
define void @shl_16bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
-; X64-NO-SHLD-NO-BMI2-LABEL: shl_16bytes:
-; X64-NO-SHLD-NO-BMI2: # %bb.0:
-; X64-NO-SHLD-NO-BMI2-NEXT: movq (%rdi), %r8
-; X64-NO-SHLD-NO-BMI2-NEXT: movq 8(%rdi), %rdi
-; X64-NO-SHLD-NO-BMI2-NEXT: movzbl (%rsi), %eax
-; X64-NO-SHLD-NO-BMI2-NEXT: shlb $3, %al
-; X64-NO-SHLD-NO-BMI2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-NEXT: shlq %cl, %rdi
-; X64-NO-SHLD-NO-BMI2-NEXT: movq %r8, %rsi
-; X64-NO-SHLD-NO-BMI2-NEXT: shrq %rsi
-; X64-NO-SHLD-NO-BMI2-NEXT: notb %cl
-; X64-NO-SHLD-NO-BMI2-NEXT: shrq %cl, %rsi
-; X64-NO-SHLD-NO-BMI2-NEXT: orq %rdi, %rsi
-; X64-NO-SHLD-NO-BMI2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-NEXT: shlq %cl, %r8
-; X64-NO-SHLD-NO-BMI2-NEXT: xorl %ecx, %ecx
-; X64-NO-SHLD-NO-BMI2-NEXT: testb $64, %al
-; X64-NO-SHLD-NO-BMI2-NEXT: cmovneq %r8, %rsi
-; X64-NO-SHLD-NO-BMI2-NEXT: cmoveq %r8, %rcx
-; X64-NO-SHLD-NO-BMI2-NEXT: movq %rcx, (%rdx)
-; X64-NO-SHLD-NO-BMI2-NEXT: movq %rsi, 8(%rdx)
-; X64-NO-SHLD-NO-BMI2-NEXT: retq
-;
-; X64-HAVE-SHLD-NO-BMI2-LABEL: shl_16bytes:
-; X64-HAVE-SHLD-NO-BMI2: # %bb.0:
-; X64-HAVE-SHLD-NO-BMI2-NEXT: movq (%rdi), %rax
-; X64-HAVE-SHLD-NO-BMI2-NEXT: movq 8(%rdi), %rdi
-; X64-HAVE-SHLD-NO-BMI2-NEXT: movzbl (%rsi), %ecx
-; X64-HAVE-SHLD-NO-BMI2-NEXT: shlb $3, %cl
-; X64-HAVE-SHLD-NO-BMI2-NEXT: movq %rax, %rsi
-; X64-HAVE-SHLD-NO-BMI2-NEXT: shlq %cl, %rsi
-; X64-HAVE-SHLD-NO-BMI2-NEXT: shldq %cl, %rax, %rdi
-; X64-HAVE-SHLD-NO-BMI2-NEXT: xorl %eax, %eax
-; X64-HAVE-SHLD-NO-BMI2-NEXT: testb $64, %cl
-; X64-HAVE-SHLD-NO-BMI2-NEXT: cmovneq %rsi, %rdi
-; X64-HAVE-SHLD-NO-BMI2-NEXT: cmoveq %rsi, %rax
-; X64-HAVE-SHLD-NO-BMI2-NEXT: movq %rdi, 8(%rdx)
-; X64-HAVE-SHLD-NO-BMI2-NEXT: movq %rax, (%rdx)
-; X64-HAVE-SHLD-NO-BMI2-NEXT: retq
-;
-; X64-NO-SHLD-HAVE-BMI2-LABEL: shl_16bytes:
-; X64-NO-SHLD-HAVE-BMI2: # %bb.0:
-; X64-NO-SHLD-HAVE-BMI2-NEXT: movq (%rdi), %rax
-; X64-NO-SHLD-HAVE-BMI2-NEXT: movzbl (%rsi), %ecx
-; X64-NO-SHLD-HAVE-BMI2-NEXT: shlb $3, %cl
-; X64-NO-SHLD-HAVE-BMI2-NEXT: shlxq %rcx, 8(%rdi), %rsi
-; X64-NO-SHLD-HAVE-BMI2-NEXT: movl %ecx, %edi
-; X64-NO-SHLD-HAVE-BMI2-NEXT: notb %dil
-; X64-NO-SHLD-HAVE-BMI2-NEXT: shlxq %rcx, %rax, %r8
-; X64-NO-SHLD-HAVE-BMI2-NEXT: shrq %rax
-; X64-NO-SHLD-HAVE-BMI2-NEXT: shrxq %rdi, %rax, %rax
-; X64-NO-SHLD-HAVE-BMI2-NEXT: orq %rsi, %rax
-; X64-NO-SHLD-HAVE-BMI2-NEXT: xorl %esi, %esi
-; X64-NO-SHLD-HAVE-BMI2-NEXT: testb $64, %cl
-; X64-NO-SHLD-HAVE-BMI2-NEXT: cmovneq %r8, %rax
-; X64-NO-SHLD-HAVE-BMI2-NEXT: cmoveq %r8, %rsi
-; X64-NO-SHLD-HAVE-BMI2-NEXT: movq %rsi, (%rdx)
-; X64-NO-SHLD-HAVE-BMI2-NEXT: movq %rax, 8(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-NEXT: retq
-;
-; X64-HAVE-SHLD-HAVE-BMI2-LABEL: shl_16bytes:
-; X64-HAVE-SHLD-HAVE-BMI2: # %bb.0:
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: movq (%rdi), %rax
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: movq 8(%rdi), %rdi
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: movzbl (%rsi), %ecx
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: shlb $3, %cl
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: shldq %cl, %rax, %rdi
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: shlxq %rcx, %rax, %rax
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: xorl %esi, %esi
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: testb $64, %cl
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: cmovneq %rax, %rdi
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: cmoveq %rax, %rsi
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: movq %rdi, 8(%rdx)
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: movq %rsi, (%rdx)
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: retq
+; X64-NO-BMI2-LABEL: shl_16bytes:
+; X64-NO-BMI2: # %bb.0:
+; X64-NO-BMI2-NEXT: movq (%rdi), %rax
+; X64-NO-BMI2-NEXT: movq 8(%rdi), %rdi
+; X64-NO-BMI2-NEXT: movzbl (%rsi), %ecx
+; X64-NO-BMI2-NEXT: shlb $3, %cl
+; X64-NO-BMI2-NEXT: movq %rax, %rsi
+; X64-NO-BMI2-NEXT: shlq %cl, %rsi
+; X64-NO-BMI2-NEXT: shldq %cl, %rax, %rdi
+; X64-NO-BMI2-NEXT: xorl %eax, %eax
+; X64-NO-BMI2-NEXT: testb $64, %cl
+; X64-NO-BMI2-NEXT: cmovneq %rsi, %rdi
+; X64-NO-BMI2-NEXT: cmoveq %rsi, %rax
+; X64-NO-BMI2-NEXT: movq %rdi, 8(%rdx)
+; X64-NO-BMI2-NEXT: movq %rax, (%rdx)
+; X64-NO-BMI2-NEXT: retq
+;
+; X64-HAVE-BMI2-LABEL: shl_16bytes:
+; X64-HAVE-BMI2: # %bb.0:
+; X64-HAVE-BMI2-NEXT: movq (%rdi), %rax
+; X64-HAVE-BMI2-NEXT: movq 8(%rdi), %rdi
+; X64-HAVE-BMI2-NEXT: movzbl (%rsi), %ecx
+; X64-HAVE-BMI2-NEXT: shlb $3, %cl
+; X64-HAVE-BMI2-NEXT: shldq %cl, %rax, %rdi
+; X64-HAVE-BMI2-NEXT: shlxq %rcx, %rax, %rax
+; X64-HAVE-BMI2-NEXT: xorl %esi, %esi
+; X64-HAVE-BMI2-NEXT: testb $64, %cl
+; X64-HAVE-BMI2-NEXT: cmovneq %rax, %rdi
+; X64-HAVE-BMI2-NEXT: cmoveq %rax, %rsi
+; X64-HAVE-BMI2-NEXT: movq %rdi, 8(%rdx)
+; X64-HAVE-BMI2-NEXT: movq %rsi, (%rdx)
+; X64-HAVE-BMI2-NEXT: retq
;
; X86-NO-SHLD-NO-BMI2-SSE2-LABEL: shl_16bytes:
; X86-NO-SHLD-NO-BMI2-SSE2: # %bb.0:
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: pushl %ebp
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: pushl %ebx
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: pushl %edi
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: pushl %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: subl $60, %esp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: subl $32, %esp
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl (%ecx), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 4(%ecx), %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 8(%ecx), %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 12(%ecx), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb (%eax), %ah
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ah, %dh
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shlb $3, %dh
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl (%edx), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 4(%edx), %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 8(%edx), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 12(%edx), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb (%ecx), %ch
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shlb $3, %cl
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: xorps %xmm0, %xmm0
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movaps %xmm0, (%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: andb $12, %ah
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: negb %ah
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movsbl %ah, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 32(%esp,%ebp), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 36(%esp,%ebp), %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %dh, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %dh, %dl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: notb %dl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %edi, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 44(%esp,%ebp), %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %dh, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 40(%esp,%ebp), %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %eax, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %dh, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %edi, %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %dh, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, (%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: andb $12, %ch
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: negb %ch
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movsbl %ch, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 24(%esp,%edi), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 28(%esp,%edi), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shldl %cl, %esi, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 16(%esp,%edi), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 20(%esp,%edi), %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shldl %cl, %edi, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shldl %cl, %ebx, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %ebx
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, 8(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebp, 12(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, 4(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: addl $60, %esp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, 12(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, (%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, 4(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: addl $32, %esp
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: popl %esi
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: popl %edi
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: popl %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: popl %ebp
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: retl
;
; X86-HAVE-SHLD-NO-BMI2-SSE2-LABEL: shl_16bytes:
@@ -1590,47 +1172,37 @@ define void @shl_16bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: pushl %edi
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: pushl %esi
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: subl $44, %esp
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ebp
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl (%ecx), %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 4(%ecx), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 8(%ecx), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 12(%ecx), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movzbl (%eax), %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebx, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlb $3, %al
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl (%edx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 4(%edx), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 8(%edx), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 12(%edx), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movzbl (%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlb $3, %cl
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: xorps %xmm0, %xmm0
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movaps %xmm0, (%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebx, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: andb $12, %bl
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: negb %bl
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movsbl %bl, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 16(%esp,%esi), %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 20(%esp,%esi), %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %ecx, %edx, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: notb %al
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %ecx, %ebx, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrl %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %eax, %ebx, %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %edi, %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %ecx, 28(%esp,%esi), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 24(%esp,%esi), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %ecx, %esi, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrl %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %eax, %esi, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %edi, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrl %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %eax, %edx, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %ecx, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebp, (%ecx)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 8(%ecx)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, 12(%ecx)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebx, 4(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: andb $12, %al
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: negb %al
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movsbl %al, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 24(%esp,%eax), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 28(%esp,%eax), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shldl %cl, %esi, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 16(%esp,%eax), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 20(%esp,%eax), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shldl %cl, %eax, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shldl %cl, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %ecx, %edi, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, 8(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, 12(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, (%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 4(%ebp)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addl $44, %esp
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: popl %esi
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: popl %edi
@@ -1689,56 +1261,34 @@ define void @shl_16bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: pushl %ebx
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: pushl %edi
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: pushl %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: subl $60, %esp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: subl $44, %esp
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movups (%ecx), %xmm0
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movzbl (%eax), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shlb $3, %al
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movups (%edx), %xmm0
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movzbl (%ecx), %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shlb $3, %cl
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: xorps %xmm1, %xmm1
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm1, (%esp)
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: andb $12, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: negb %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movsbl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 44(%esp,%edi), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: notb %dl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 40(%esp,%edi), %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebp, %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %ebx, %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: andb $12, %dl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: negb %dl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movsbl %dl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 24(%esp,%edi), %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 28(%esp,%edi), %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shldl %cl, %esi, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 16(%esp,%edi), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 20(%esp,%edi), %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shldl %cl, %edi, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, %ebp
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 32(%esp,%edi), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 36(%esp,%edi), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %ebp, %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %ebx, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, (%edx)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebp, 4(%edx)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edi, 8(%edx)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, 12(%edx)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: addl $60, %esp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shldl %cl, %ebx, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edi, 4(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, 8(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, 12(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebp, (%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: addl $44, %esp
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: popl %esi
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: popl %edi
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: popl %ebx
@@ -1794,40 +1344,30 @@ define void @shl_16bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: subl $44, %esp
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movups (%ecx), %xmm0
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movzbl (%eax), %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlb $3, %al
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movups (%edx), %xmm0
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movzbl (%ecx), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlb $3, %cl
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: xorps %xmm1, %xmm1
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm1, (%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, %ecx
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: andb $12, %dl
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: negb %dl
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movsbl %dl, %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %ecx, 28(%esp,%edx), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: notb %al
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 24(%esp,%edx), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %ecx, %esi, %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrl %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %eax, %esi, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %edi, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 20(%esp,%edx), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrl %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %eax, %ebp, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %ebx, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %ecx, %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 16(%esp,%edx), %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %ecx, %edx, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrl %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %eax, %edx, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, (%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 4(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ebp, 8(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %esi, 12(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movsbl %dl, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 24(%esp,%edi), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 28(%esp,%edi), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shldl %cl, %esi, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 16(%esp,%edi), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 20(%esp,%edi), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shldl %cl, %edi, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %ecx, %ebx, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shldl %cl, %ebx, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edi, 4(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %esi, 8(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, 12(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ebp, (%eax)
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: addl $44, %esp
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: popl %esi
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: popl %edi
@@ -1881,56 +1421,34 @@ define void @shl_16bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: pushl %ebx
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: pushl %edi
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: pushl %esi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: subl $60, %esp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: subl $44, %esp
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: vmovups (%ecx), %xmm0
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movzbl (%eax), %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ecx, %eax
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shlb $3, %al
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: vmovups (%edx), %xmm0
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movzbl (%ecx), %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shlb $3, %cl
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: vmovaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: vmovaps %xmm1, (%esp)
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: vmovaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: andb $12, %cl
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: negb %cl
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movsbl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 44(%esp,%edi), %ebx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %edx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: notb %dl
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 40(%esp,%edi), %ebp
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ebp, %esi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrl %esi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrl %cl, %esi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: orl %ebx, %esi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: andb $12, %dl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: negb %dl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movsbl %dl, %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 24(%esp,%edi), %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 28(%esp,%edi), %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shldl %cl, %esi, %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 16(%esp,%edi), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 20(%esp,%edi), %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shldl %cl, %edi, %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ebx, %ebp
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shll %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 32(%esp,%edi), %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 36(%esp,%edi), %ebx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ebx, %edi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrl %edi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: orl %ebp, %edi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrl %ebp
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrl %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: orl %ebx, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shll %cl, %eax
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, (%edx)
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ebp, 4(%edx)
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edi, 8(%edx)
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %esi, 12(%edx)
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: addl $60, %esp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shldl %cl, %ebx, %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edi, 4(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %esi, 8(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, 12(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ebp, (%eax)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: addl $44, %esp
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: popl %esi
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: popl %edi
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: popl %ebx
@@ -1986,40 +1504,30 @@ define void @shl_16bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: subl $44, %esp
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: vmovups (%ecx), %xmm0
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movzbl (%eax), %edx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edx, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlb $3, %al
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: vmovups (%edx), %xmm0
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movzbl (%ecx), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edx, %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlb $3, %cl
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: vxorps %xmm1, %xmm1, %xmm1
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: vmovaps %xmm1, (%esp)
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: vmovaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, %ecx
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: andb $12, %dl
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: negb %dl
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movsbl %dl, %edx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxl %ecx, 28(%esp,%edx), %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: notb %al
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 24(%esp,%edx), %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxl %ecx, %esi, %ebx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrl %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxl %eax, %esi, %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: orl %edi, %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 20(%esp,%edx), %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrl %ebp
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxl %eax, %ebp, %ebp
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: orl %ebx, %ebp
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxl %ecx, %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 16(%esp,%edx), %edx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxl %ecx, %edx, %ecx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrl %edx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxl %eax, %edx, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: orl %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %ecx, (%edx)
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, 4(%edx)
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %ebp, 8(%edx)
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %esi, 12(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movsbl %dl, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 24(%esp,%edi), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 28(%esp,%edi), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shldl %cl, %esi, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 16(%esp,%edi), %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 20(%esp,%edi), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shldl %cl, %edi, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxl %ecx, %ebx, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shldl %cl, %ebx, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edi, 4(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %esi, 8(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edx, 12(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %ebp, (%eax)
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: addl $44, %esp
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: popl %esi
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: popl %edi
@@ -2075,81 +1583,38 @@ define void @shl_16bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
}
define void @shl_16bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) nounwind {
-; X64-NO-SHLD-NO-BMI2-LABEL: shl_16bytes_dwordOff:
-; X64-NO-SHLD-NO-BMI2: # %bb.0:
-; X64-NO-SHLD-NO-BMI2-NEXT: movq (%rdi), %r8
-; X64-NO-SHLD-NO-BMI2-NEXT: movq 8(%rdi), %rdi
-; X64-NO-SHLD-NO-BMI2-NEXT: movzbl (%rsi), %eax
-; X64-NO-SHLD-NO-BMI2-NEXT: shlb $5, %al
-; X64-NO-SHLD-NO-BMI2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-NEXT: shlq %cl, %rdi
-; X64-NO-SHLD-NO-BMI2-NEXT: movq %r8, %rsi
-; X64-NO-SHLD-NO-BMI2-NEXT: shrq %rsi
-; X64-NO-SHLD-NO-BMI2-NEXT: notb %cl
-; X64-NO-SHLD-NO-BMI2-NEXT: shrq %cl, %rsi
-; X64-NO-SHLD-NO-BMI2-NEXT: orq %rdi, %rsi
-; X64-NO-SHLD-NO-BMI2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-NEXT: shlq %cl, %r8
-; X64-NO-SHLD-NO-BMI2-NEXT: xorl %ecx, %ecx
-; X64-NO-SHLD-NO-BMI2-NEXT: testb $64, %al
-; X64-NO-SHLD-NO-BMI2-NEXT: cmovneq %r8, %rsi
-; X64-NO-SHLD-NO-BMI2-NEXT: cmoveq %r8, %rcx
-; X64-NO-SHLD-NO-BMI2-NEXT: movq %rcx, (%rdx)
-; X64-NO-SHLD-NO-BMI2-NEXT: movq %rsi, 8(%rdx)
-; X64-NO-SHLD-NO-BMI2-NEXT: retq
-;
-; X64-HAVE-SHLD-NO-BMI2-LABEL: shl_16bytes_dwordOff:
-; X64-HAVE-SHLD-NO-BMI2: # %bb.0:
-; X64-HAVE-SHLD-NO-BMI2-NEXT: movq (%rdi), %rax
-; X64-HAVE-SHLD-NO-BMI2-NEXT: movq 8(%rdi), %rdi
-; X64-HAVE-SHLD-NO-BMI2-NEXT: movzbl (%rsi), %ecx
-; X64-HAVE-SHLD-NO-BMI2-NEXT: shlb $5, %cl
-; X64-HAVE-SHLD-NO-BMI2-NEXT: movq %rax, %rsi
-; X64-HAVE-SHLD-NO-BMI2-NEXT: shlq %cl, %rsi
-; X64-HAVE-SHLD-NO-BMI2-NEXT: shldq %cl, %rax, %rdi
-; X64-HAVE-SHLD-NO-BMI2-NEXT: xorl %eax, %eax
-; X64-HAVE-SHLD-NO-BMI2-NEXT: testb $64, %cl
-; X64-HAVE-SHLD-NO-BMI2-NEXT: cmovneq %rsi, %rdi
-; X64-HAVE-SHLD-NO-BMI2-NEXT: cmoveq %rsi, %rax
-; X64-HAVE-SHLD-NO-BMI2-NEXT: movq %rdi, 8(%rdx)
-; X64-HAVE-SHLD-NO-BMI2-NEXT: movq %rax, (%rdx)
-; X64-HAVE-SHLD-NO-BMI2-NEXT: retq
-;
-; X64-NO-SHLD-HAVE-BMI2-LABEL: shl_16bytes_dwordOff:
-; X64-NO-SHLD-HAVE-BMI2: # %bb.0:
-; X64-NO-SHLD-HAVE-BMI2-NEXT: movq (%rdi), %rax
-; X64-NO-SHLD-HAVE-BMI2-NEXT: movzbl (%rsi), %ecx
-; X64-NO-SHLD-HAVE-BMI2-NEXT: shlb $5, %cl
-; X64-NO-SHLD-HAVE-BMI2-NEXT: shlxq %rcx, 8(%rdi), %rsi
-; X64-NO-SHLD-HAVE-BMI2-NEXT: movl %ecx, %edi
-; X64-NO-SHLD-HAVE-BMI2-NEXT: notb %dil
-; X64-NO-SHLD-HAVE-BMI2-NEXT: shlxq %rcx, %rax, %r8
-; X64-NO-SHLD-HAVE-BMI2-NEXT: shrq %rax
-; X64-NO-SHLD-HAVE-BMI2-NEXT: shrxq %rdi, %rax, %rax
-; X64-NO-SHLD-HAVE-BMI2-NEXT: orq %rsi, %rax
-; X64-NO-SHLD-HAVE-BMI2-NEXT: xorl %esi, %esi
-; X64-NO-SHLD-HAVE-BMI2-NEXT: testb $64, %cl
-; X64-NO-SHLD-HAVE-BMI2-NEXT: cmovneq %r8, %rax
-; X64-NO-SHLD-HAVE-BMI2-NEXT: cmoveq %r8, %rsi
-; X64-NO-SHLD-HAVE-BMI2-NEXT: movq %rsi, (%rdx)
-; X64-NO-SHLD-HAVE-BMI2-NEXT: movq %rax, 8(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-NEXT: retq
-;
-; X64-HAVE-SHLD-HAVE-BMI2-LABEL: shl_16bytes_dwordOff:
-; X64-HAVE-SHLD-HAVE-BMI2: # %bb.0:
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: movq (%rdi), %rax
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: movq 8(%rdi), %rdi
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: movzbl (%rsi), %ecx
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: shlb $5, %cl
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: shldq %cl, %rax, %rdi
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: shlxq %rcx, %rax, %rax
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: xorl %esi, %esi
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: testb $64, %cl
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: cmovneq %rax, %rdi
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: cmoveq %rax, %rsi
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: movq %rdi, 8(%rdx)
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: movq %rsi, (%rdx)
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: retq
+; X64-NO-BMI2-LABEL: shl_16bytes_dwordOff:
+; X64-NO-BMI2: # %bb.0:
+; X64-NO-BMI2-NEXT: movq (%rdi), %rax
+; X64-NO-BMI2-NEXT: movq 8(%rdi), %rdi
+; X64-NO-BMI2-NEXT: movzbl (%rsi), %ecx
+; X64-NO-BMI2-NEXT: shlb $5, %cl
+; X64-NO-BMI2-NEXT: movq %rax, %rsi
+; X64-NO-BMI2-NEXT: shlq %cl, %rsi
+; X64-NO-BMI2-NEXT: shldq %cl, %rax, %rdi
+; X64-NO-BMI2-NEXT: xorl %eax, %eax
+; X64-NO-BMI2-NEXT: testb $64, %cl
+; X64-NO-BMI2-NEXT: cmovneq %rsi, %rdi
+; X64-NO-BMI2-NEXT: cmoveq %rsi, %rax
+; X64-NO-BMI2-NEXT: movq %rdi, 8(%rdx)
+; X64-NO-BMI2-NEXT: movq %rax, (%rdx)
+; X64-NO-BMI2-NEXT: retq
+;
+; X64-HAVE-BMI2-LABEL: shl_16bytes_dwordOff:
+; X64-HAVE-BMI2: # %bb.0:
+; X64-HAVE-BMI2-NEXT: movq (%rdi), %rax
+; X64-HAVE-BMI2-NEXT: movq 8(%rdi), %rdi
+; X64-HAVE-BMI2-NEXT: movzbl (%rsi), %ecx
+; X64-HAVE-BMI2-NEXT: shlb $5, %cl
+; X64-HAVE-BMI2-NEXT: shldq %cl, %rax, %rdi
+; X64-HAVE-BMI2-NEXT: shlxq %rcx, %rax, %rax
+; X64-HAVE-BMI2-NEXT: xorl %esi, %esi
+; X64-HAVE-BMI2-NEXT: testb $64, %cl
+; X64-HAVE-BMI2-NEXT: cmovneq %rax, %rdi
+; X64-HAVE-BMI2-NEXT: cmoveq %rax, %rsi
+; X64-HAVE-BMI2-NEXT: movq %rdi, 8(%rdx)
+; X64-HAVE-BMI2-NEXT: movq %rsi, (%rdx)
+; X64-HAVE-BMI2-NEXT: retq
;
; X86-SSE2-LABEL: shl_16bytes_dwordOff:
; X86-SSE2: # %bb.0:
@@ -2237,302 +1702,135 @@ define void @shl_16bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) nou
}
define void @ashr_16bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
-; X64-NO-SHLD-NO-BMI2-LABEL: ashr_16bytes:
-; X64-NO-SHLD-NO-BMI2: # %bb.0:
-; X64-NO-SHLD-NO-BMI2-NEXT: movq (%rdi), %r8
-; X64-NO-SHLD-NO-BMI2-NEXT: movq 8(%rdi), %rdi
-; X64-NO-SHLD-NO-BMI2-NEXT: movzbl (%rsi), %eax
-; X64-NO-SHLD-NO-BMI2-NEXT: shlb $3, %al
-; X64-NO-SHLD-NO-BMI2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-NEXT: shrq %cl, %r8
-; X64-NO-SHLD-NO-BMI2-NEXT: leaq (%rdi,%rdi), %rsi
-; X64-NO-SHLD-NO-BMI2-NEXT: notb %cl
-; X64-NO-SHLD-NO-BMI2-NEXT: shlq %cl, %rsi
-; X64-NO-SHLD-NO-BMI2-NEXT: orq %r8, %rsi
-; X64-NO-SHLD-NO-BMI2-NEXT: movq %rdi, %r8
-; X64-NO-SHLD-NO-BMI2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-NEXT: sarq %cl, %r8
-; X64-NO-SHLD-NO-BMI2-NEXT: sarq $63, %rdi
-; X64-NO-SHLD-NO-BMI2-NEXT: testb $64, %al
-; X64-NO-SHLD-NO-BMI2-NEXT: cmovneq %r8, %rsi
-; X64-NO-SHLD-NO-BMI2-NEXT: cmoveq %r8, %rdi
-; X64-NO-SHLD-NO-BMI2-NEXT: movq %rdi, 8(%rdx)
-; X64-NO-SHLD-NO-BMI2-NEXT: movq %rsi, (%rdx)
-; X64-NO-SHLD-NO-BMI2-NEXT: retq
-;
-; X64-HAVE-SHLD-NO-BMI2-LABEL: ashr_16bytes:
-; X64-HAVE-SHLD-NO-BMI2: # %bb.0:
-; X64-HAVE-SHLD-NO-BMI2-NEXT: movq (%rdi), %rax
-; X64-HAVE-SHLD-NO-BMI2-NEXT: movq 8(%rdi), %rdi
-; X64-HAVE-SHLD-NO-BMI2-NEXT: movzbl (%rsi), %ecx
-; X64-HAVE-SHLD-NO-BMI2-NEXT: shlb $3, %cl
-; X64-HAVE-SHLD-NO-BMI2-NEXT: movq %rdi, %rsi
-; X64-HAVE-SHLD-NO-BMI2-NEXT: sarq %cl, %rsi
-; X64-HAVE-SHLD-NO-BMI2-NEXT: shrdq %cl, %rdi, %rax
-; X64-HAVE-SHLD-NO-BMI2-NEXT: sarq $63, %rdi
-; X64-HAVE-SHLD-NO-BMI2-NEXT: testb $64, %cl
-; X64-HAVE-SHLD-NO-BMI2-NEXT: cmovneq %rsi, %rax
-; X64-HAVE-SHLD-NO-BMI2-NEXT: cmoveq %rsi, %rdi
-; X64-HAVE-SHLD-NO-BMI2-NEXT: movq %rdi, 8(%rdx)
-; X64-HAVE-SHLD-NO-BMI2-NEXT: movq %rax, (%rdx)
-; X64-HAVE-SHLD-NO-BMI2-NEXT: retq
-;
-; X64-NO-SHLD-HAVE-BMI2-LABEL: ashr_16bytes:
-; X64-NO-SHLD-HAVE-BMI2: # %bb.0:
-; X64-NO-SHLD-HAVE-BMI2-NEXT: movq 8(%rdi), %rax
-; X64-NO-SHLD-HAVE-BMI2-NEXT: movzbl (%rsi), %ecx
-; X64-NO-SHLD-HAVE-BMI2-NEXT: shlb $3, %cl
-; X64-NO-SHLD-HAVE-BMI2-NEXT: shrxq %rcx, (%rdi), %rsi
-; X64-NO-SHLD-HAVE-BMI2-NEXT: movl %ecx, %edi
-; X64-NO-SHLD-HAVE-BMI2-NEXT: notb %dil
-; X64-NO-SHLD-HAVE-BMI2-NEXT: leaq (%rax,%rax), %r8
-; X64-NO-SHLD-HAVE-BMI2-NEXT: shlxq %rdi, %r8, %rdi
-; X64-NO-SHLD-HAVE-BMI2-NEXT: orq %rsi, %rdi
-; X64-NO-SHLD-HAVE-BMI2-NEXT: sarxq %rcx, %rax, %rsi
-; X64-NO-SHLD-HAVE-BMI2-NEXT: sarq $63, %rax
-; X64-NO-SHLD-HAVE-BMI2-NEXT: testb $64, %cl
-; X64-NO-SHLD-HAVE-BMI2-NEXT: cmovneq %rsi, %rdi
-; X64-NO-SHLD-HAVE-BMI2-NEXT: cmoveq %rsi, %rax
-; X64-NO-SHLD-HAVE-BMI2-NEXT: movq %rax, 8(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-NEXT: movq %rdi, (%rdx)
-; X64-NO-SHLD-HAVE-BMI2-NEXT: retq
-;
-; X64-HAVE-SHLD-HAVE-BMI2-LABEL: ashr_16bytes:
-; X64-HAVE-SHLD-HAVE-BMI2: # %bb.0:
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: movq (%rdi), %rax
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: movq 8(%rdi), %rdi
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: movzbl (%rsi), %ecx
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: shlb $3, %cl
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: shrdq %cl, %rdi, %rax
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: sarxq %rcx, %rdi, %rsi
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: sarq $63, %rdi
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: testb $64, %cl
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: cmovneq %rsi, %rax
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: cmoveq %rsi, %rdi
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: movq %rdi, 8(%rdx)
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: movq %rax, (%rdx)
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: retq
-;
-; X86-NO-SHLD-NO-BMI2-LABEL: ashr_16bytes:
-; X86-NO-SHLD-NO-BMI2: # %bb.0:
-; X86-NO-SHLD-NO-BMI2-NEXT: pushl %ebp
-; X86-NO-SHLD-NO-BMI2-NEXT: pushl %ebx
-; X86-NO-SHLD-NO-BMI2-NEXT: pushl %edi
-; X86-NO-SHLD-NO-BMI2-NEXT: pushl %esi
-; X86-NO-SHLD-NO-BMI2-NEXT: subl $60, %esp
-; X86-NO-SHLD-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-SHLD-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-SHLD-NO-BMI2-NEXT: movl (%ecx), %edx
-; X86-NO-SHLD-NO-BMI2-NEXT: movl 4(%ecx), %esi
-; X86-NO-SHLD-NO-BMI2-NEXT: movl 8(%ecx), %edi
-; X86-NO-SHLD-NO-BMI2-NEXT: movl 12(%ecx), %ecx
-; X86-NO-SHLD-NO-BMI2-NEXT: movb (%eax), %ah
-; X86-NO-SHLD-NO-BMI2-NEXT: movb %ah, %al
-; X86-NO-SHLD-NO-BMI2-NEXT: shlb $3, %al
-; X86-NO-SHLD-NO-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-NEXT: sarl $31, %ecx
-; X86-NO-SHLD-NO-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-NEXT: andb $12, %ah
-; X86-NO-SHLD-NO-BMI2-NEXT: movzbl %ah, %ebp
-; X86-NO-SHLD-NO-BMI2-NEXT: movl 20(%esp,%ebp), %esi
-; X86-NO-SHLD-NO-BMI2-NEXT: movl %esi, %ebx
-; X86-NO-SHLD-NO-BMI2-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-NEXT: movl %eax, %edx
-; X86-NO-SHLD-NO-BMI2-NEXT: notb %dl
-; X86-NO-SHLD-NO-BMI2-NEXT: movl 24(%esp,%ebp), %ecx
-; X86-NO-SHLD-NO-BMI2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-NEXT: leal (%ecx,%ecx), %edi
-; X86-NO-SHLD-NO-BMI2-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-NEXT: shll %cl, %edi
-; X86-NO-SHLD-NO-BMI2-NEXT: orl %ebx, %edi
-; X86-NO-SHLD-NO-BMI2-NEXT: movl 16(%esp,%ebp), %ebx
-; X86-NO-SHLD-NO-BMI2-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-NEXT: addl %esi, %esi
-; X86-NO-SHLD-NO-BMI2-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-NEXT: shll %cl, %esi
-; X86-NO-SHLD-NO-BMI2-NEXT: orl %ebx, %esi
-; X86-NO-SHLD-NO-BMI2-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-NEXT: shrl %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NO-SHLD-NO-BMI2-NEXT: movl 28(%esp,%ebp), %ebx
-; X86-NO-SHLD-NO-BMI2-NEXT: leal (%ebx,%ebx), %ebp
-; X86-NO-SHLD-NO-BMI2-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-NEXT: shll %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X86-NO-SHLD-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-SHLD-NO-BMI2-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-NEXT: sarl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-NEXT: movl %ebx, 12(%edx)
-; X86-NO-SHLD-NO-BMI2-NEXT: movl %ebp, 8(%edx)
-; X86-NO-SHLD-NO-BMI2-NEXT: movl %esi, (%edx)
-; X86-NO-SHLD-NO-BMI2-NEXT: movl %edi, 4(%edx)
-; X86-NO-SHLD-NO-BMI2-NEXT: addl $60, %esp
-; X86-NO-SHLD-NO-BMI2-NEXT: popl %esi
-; X86-NO-SHLD-NO-BMI2-NEXT: popl %edi
-; X86-NO-SHLD-NO-BMI2-NEXT: popl %ebx
-; X86-NO-SHLD-NO-BMI2-NEXT: popl %ebp
-; X86-NO-SHLD-NO-BMI2-NEXT: retl
-;
-; X86-HAVE-SHLD-NO-BMI2-LABEL: ashr_16bytes:
-; X86-HAVE-SHLD-NO-BMI2: # %bb.0:
-; X86-HAVE-SHLD-NO-BMI2-NEXT: pushl %ebp
-; X86-HAVE-SHLD-NO-BMI2-NEXT: pushl %ebx
-; X86-HAVE-SHLD-NO-BMI2-NEXT: pushl %edi
-; X86-HAVE-SHLD-NO-BMI2-NEXT: pushl %esi
-; X86-HAVE-SHLD-NO-BMI2-NEXT: subl $44, %esp
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl (%edx), %esi
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl 4(%edx), %edi
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl 8(%edx), %ebx
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl 12(%edx), %edx
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movb (%ecx), %ch
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movb %ch, %cl
-; X86-HAVE-SHLD-NO-BMI2-NEXT: shlb $3, %cl
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl %esi, (%esp)
-; X86-HAVE-SHLD-NO-BMI2-NEXT: sarl $31, %edx
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-HAVE-SHLD-NO-BMI2-NEXT: andb $12, %ch
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movzbl %ch, %ebx
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl 8(%esp,%ebx), %esi
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl (%esp,%ebx), %edx
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl 4(%esp,%ebx), %ebp
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl %ebp, %edi
-; X86-HAVE-SHLD-NO-BMI2-NEXT: shrdl %cl, %esi, %edi
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl 12(%esp,%ebx), %ebx
-; X86-HAVE-SHLD-NO-BMI2-NEXT: shrdl %cl, %ebx, %esi
-; X86-HAVE-SHLD-NO-BMI2-NEXT: shrdl %cl, %ebp, %edx
-; X86-HAVE-SHLD-NO-BMI2-NEXT: sarl %cl, %ebx
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl %esi, 8(%eax)
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl %ebx, 12(%eax)
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl %edx, (%eax)
-; X86-HAVE-SHLD-NO-BMI2-NEXT: movl %edi, 4(%eax)
-; X86-HAVE-SHLD-NO-BMI2-NEXT: addl $44, %esp
-; X86-HAVE-SHLD-NO-BMI2-NEXT: popl %esi
-; X86-HAVE-SHLD-NO-BMI2-NEXT: popl %edi
-; X86-HAVE-SHLD-NO-BMI2-NEXT: popl %ebx
-; X86-HAVE-SHLD-NO-BMI2-NEXT: popl %ebp
-; X86-HAVE-SHLD-NO-BMI2-NEXT: retl
-;
-; X86-NO-SHLD-HAVE-BMI2-LABEL: ashr_16bytes:
-; X86-NO-SHLD-HAVE-BMI2: # %bb.0:
-; X86-NO-SHLD-HAVE-BMI2-NEXT: pushl %ebp
-; X86-NO-SHLD-HAVE-BMI2-NEXT: pushl %ebx
-; X86-NO-SHLD-HAVE-BMI2-NEXT: pushl %edi
-; X86-NO-SHLD-HAVE-BMI2-NEXT: pushl %esi
-; X86-NO-SHLD-HAVE-BMI2-NEXT: subl $44, %esp
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movl (%ecx), %edx
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movl 4(%ecx), %esi
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movl 8(%ecx), %edi
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movl 12(%ecx), %ecx
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movzbl (%eax), %ebx
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movl %ebx, %eax
-; X86-NO-SHLD-HAVE-BMI2-NEXT: shlb $3, %al
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movl %edx, (%esp)
-; X86-NO-SHLD-HAVE-BMI2-NEXT: sarl $31, %ecx
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-HAVE-BMI2-NEXT: andb $12, %bl
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movzbl %bl, %edi
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movl 4(%esp,%edi), %ebx
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movl 8(%esp,%edi), %esi
-; X86-NO-SHLD-HAVE-BMI2-NEXT: shrxl %ecx, %ebx, %ebp
-; X86-NO-SHLD-HAVE-BMI2-NEXT: notb %al
-; X86-NO-SHLD-HAVE-BMI2-NEXT: leal (%esi,%esi), %edx
-; X86-NO-SHLD-HAVE-BMI2-NEXT: shlxl %eax, %edx, %edx
-; X86-NO-SHLD-HAVE-BMI2-NEXT: orl %ebp, %edx
-; X86-NO-SHLD-HAVE-BMI2-NEXT: shrxl %ecx, (%esp,%edi), %ebp
-; X86-NO-SHLD-HAVE-BMI2-NEXT: addl %ebx, %ebx
-; X86-NO-SHLD-HAVE-BMI2-NEXT: shlxl %eax, %ebx, %ebx
-; X86-NO-SHLD-HAVE-BMI2-NEXT: orl %ebp, %ebx
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movl 12(%esp,%edi), %edi
-; X86-NO-SHLD-HAVE-BMI2-NEXT: leal (%edi,%edi), %ebp
-; X86-NO-SHLD-HAVE-BMI2-NEXT: shlxl %eax, %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-NEXT: shrxl %ecx, %esi, %esi
-; X86-NO-SHLD-HAVE-BMI2-NEXT: orl %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-NEXT: sarxl %ecx, %edi, %ecx
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movl %ecx, 12(%esi)
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movl %eax, 8(%esi)
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movl %ebx, (%esi)
-; X86-NO-SHLD-HAVE-BMI2-NEXT: movl %edx, 4(%esi)
-; X86-NO-SHLD-HAVE-BMI2-NEXT: addl $44, %esp
-; X86-NO-SHLD-HAVE-BMI2-NEXT: popl %esi
-; X86-NO-SHLD-HAVE-BMI2-NEXT: popl %edi
-; X86-NO-SHLD-HAVE-BMI2-NEXT: popl %ebx
-; X86-NO-SHLD-HAVE-BMI2-NEXT: popl %ebp
-; X86-NO-SHLD-HAVE-BMI2-NEXT: retl
-;
-; X86-HAVE-SHLD-HAVE-BMI2-LABEL: ashr_16bytes:
-; X86-HAVE-SHLD-HAVE-BMI2: # %bb.0:
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: pushl %ebp
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: pushl %ebx
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: pushl %edi
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: pushl %esi
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: subl $44, %esp
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movl (%edx), %esi
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movl 4(%edx), %edi
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movl 8(%edx), %ebx
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movl 12(%edx), %edx
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movzbl (%ecx), %eax
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movl %eax, %ecx
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: shlb $3, %cl
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movl %esi, (%esp)
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: sarl $31, %edx
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: andb $12, %al
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movzbl %al, %eax
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movl 8(%esp,%eax), %ebx
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movl (%esp,%eax), %edx
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movl 4(%esp,%eax), %esi
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movl %esi, %edi
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: shrdl %cl, %ebx, %edi
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movl 12(%esp,%eax), %eax
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: shrdl %cl, %eax, %ebx
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movl %ebx, 8(%ebp)
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: sarxl %ecx, %eax, %eax
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movl %eax, 12(%ebp)
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: shrdl %cl, %esi, %edx
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movl %edx, (%ebp)
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: movl %edi, 4(%ebp)
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: addl $44, %esp
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: popl %esi
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: popl %edi
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: popl %ebx
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: popl %ebp
-; X86-HAVE-SHLD-HAVE-BMI2-NEXT: retl
+; X64-NO-BMI2-LABEL: ashr_16bytes:
+; X64-NO-BMI2: # %bb.0:
+; X64-NO-BMI2-NEXT: movq (%rdi), %rax
+; X64-NO-BMI2-NEXT: movq 8(%rdi), %rdi
+; X64-NO-BMI2-NEXT: movzbl (%rsi), %ecx
+; X64-NO-BMI2-NEXT: shlb $3, %cl
+; X64-NO-BMI2-NEXT: movq %rdi, %rsi
+; X64-NO-BMI2-NEXT: sarq %cl, %rsi
+; X64-NO-BMI2-NEXT: shrdq %cl, %rdi, %rax
+; X64-NO-BMI2-NEXT: sarq $63, %rdi
+; X64-NO-BMI2-NEXT: testb $64, %cl
+; X64-NO-BMI2-NEXT: cmovneq %rsi, %rax
+; X64-NO-BMI2-NEXT: cmoveq %rsi, %rdi
+; X64-NO-BMI2-NEXT: movq %rdi, 8(%rdx)
+; X64-NO-BMI2-NEXT: movq %rax, (%rdx)
+; X64-NO-BMI2-NEXT: retq
+;
+; X64-HAVE-BMI2-LABEL: ashr_16bytes:
+; X64-HAVE-BMI2: # %bb.0:
+; X64-HAVE-BMI2-NEXT: movq (%rdi), %rax
+; X64-HAVE-BMI2-NEXT: movq 8(%rdi), %rdi
+; X64-HAVE-BMI2-NEXT: movzbl (%rsi), %ecx
+; X64-HAVE-BMI2-NEXT: shlb $3, %cl
+; X64-HAVE-BMI2-NEXT: shrdq %cl, %rdi, %rax
+; X64-HAVE-BMI2-NEXT: sarxq %rcx, %rdi, %rsi
+; X64-HAVE-BMI2-NEXT: sarq $63, %rdi
+; X64-HAVE-BMI2-NEXT: testb $64, %cl
+; X64-HAVE-BMI2-NEXT: cmovneq %rsi, %rax
+; X64-HAVE-BMI2-NEXT: cmoveq %rsi, %rdi
+; X64-HAVE-BMI2-NEXT: movq %rdi, 8(%rdx)
+; X64-HAVE-BMI2-NEXT: movq %rax, (%rdx)
+; X64-HAVE-BMI2-NEXT: retq
+;
+; X86-NO-BMI2-LABEL: ashr_16bytes:
+; X86-NO-BMI2: # %bb.0:
+; X86-NO-BMI2-NEXT: pushl %ebp
+; X86-NO-BMI2-NEXT: pushl %ebx
+; X86-NO-BMI2-NEXT: pushl %edi
+; X86-NO-BMI2-NEXT: pushl %esi
+; X86-NO-BMI2-NEXT: subl $44, %esp
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NO-BMI2-NEXT: movl (%edx), %esi
+; X86-NO-BMI2-NEXT: movl 4(%edx), %edi
+; X86-NO-BMI2-NEXT: movl 8(%edx), %ebx
+; X86-NO-BMI2-NEXT: movl 12(%edx), %edx
+; X86-NO-BMI2-NEXT: movb (%ecx), %ch
+; X86-NO-BMI2-NEXT: movb %ch, %cl
+; X86-NO-BMI2-NEXT: shlb $3, %cl
+; X86-NO-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %esi, (%esp)
+; X86-NO-BMI2-NEXT: sarl $31, %edx
+; X86-NO-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: andb $12, %ch
+; X86-NO-BMI2-NEXT: movzbl %ch, %ebx
+; X86-NO-BMI2-NEXT: movl 8(%esp,%ebx), %esi
+; X86-NO-BMI2-NEXT: movl (%esp,%ebx), %edx
+; X86-NO-BMI2-NEXT: movl 4(%esp,%ebx), %ebp
+; X86-NO-BMI2-NEXT: movl %ebp, %edi
+; X86-NO-BMI2-NEXT: shrdl %cl, %esi, %edi
+; X86-NO-BMI2-NEXT: movl 12(%esp,%ebx), %ebx
+; X86-NO-BMI2-NEXT: shrdl %cl, %ebx, %esi
+; X86-NO-BMI2-NEXT: shrdl %cl, %ebp, %edx
+; X86-NO-BMI2-NEXT: sarl %cl, %ebx
+; X86-NO-BMI2-NEXT: movl %esi, 8(%eax)
+; X86-NO-BMI2-NEXT: movl %ebx, 12(%eax)
+; X86-NO-BMI2-NEXT: movl %edx, (%eax)
+; X86-NO-BMI2-NEXT: movl %edi, 4(%eax)
+; X86-NO-BMI2-NEXT: addl $44, %esp
+; X86-NO-BMI2-NEXT: popl %esi
+; X86-NO-BMI2-NEXT: popl %edi
+; X86-NO-BMI2-NEXT: popl %ebx
+; X86-NO-BMI2-NEXT: popl %ebp
+; X86-NO-BMI2-NEXT: retl
+;
+; X86-HAVE-BMI2-LABEL: ashr_16bytes:
+; X86-HAVE-BMI2: # %bb.0:
+; X86-HAVE-BMI2-NEXT: pushl %ebp
+; X86-HAVE-BMI2-NEXT: pushl %ebx
+; X86-HAVE-BMI2-NEXT: pushl %edi
+; X86-HAVE-BMI2-NEXT: pushl %esi
+; X86-HAVE-BMI2-NEXT: subl $44, %esp
+; X86-HAVE-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-HAVE-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-HAVE-BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-HAVE-BMI2-NEXT: movl (%edx), %esi
+; X86-HAVE-BMI2-NEXT: movl 4(%edx), %edi
+; X86-HAVE-BMI2-NEXT: movl 8(%edx), %ebx
+; X86-HAVE-BMI2-NEXT: movl 12(%edx), %edx
+; X86-HAVE-BMI2-NEXT: movzbl (%ecx), %eax
+; X86-HAVE-BMI2-NEXT: movl %eax, %ecx
+; X86-HAVE-BMI2-NEXT: shlb $3, %cl
+; X86-HAVE-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-BMI2-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; X86-HAVE-BMI2-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-HAVE-BMI2-NEXT: movl %esi, (%esp)
+; X86-HAVE-BMI2-NEXT: sarl $31, %edx
+; X86-HAVE-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-HAVE-BMI2-NEXT: andb $12, %al
+; X86-HAVE-BMI2-NEXT: movzbl %al, %eax
+; X86-HAVE-BMI2-NEXT: movl 8(%esp,%eax), %ebx
+; X86-HAVE-BMI2-NEXT: movl (%esp,%eax), %edx
+; X86-HAVE-BMI2-NEXT: movl 4(%esp,%eax), %esi
+; X86-HAVE-BMI2-NEXT: movl %esi, %edi
+; X86-HAVE-BMI2-NEXT: shrdl %cl, %ebx, %edi
+; X86-HAVE-BMI2-NEXT: movl 12(%esp,%eax), %eax
+; X86-HAVE-BMI2-NEXT: shrdl %cl, %eax, %ebx
+; X86-HAVE-BMI2-NEXT: movl %ebx, 8(%ebp)
+; X86-HAVE-BMI2-NEXT: sarxl %ecx, %eax, %eax
+; X86-HAVE-BMI2-NEXT: movl %eax, 12(%ebp)
+; X86-HAVE-BMI2-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-HAVE-BMI2-NEXT: shrdl %cl, %esi, %edx
+; X86-HAVE-BMI2-NEXT: movl %edx, (%ebp)
+; X86-HAVE-BMI2-NEXT: movl %edi, 4(%ebp)
+; X86-HAVE-BMI2-NEXT: addl $44, %esp
+; X86-HAVE-BMI2-NEXT: popl %esi
+; X86-HAVE-BMI2-NEXT: popl %edi
+; X86-HAVE-BMI2-NEXT: popl %ebx
+; X86-HAVE-BMI2-NEXT: popl %ebp
+; X86-HAVE-BMI2-NEXT: retl
%src = load i128, ptr %src.ptr, align 1
%byteOff = load i128, ptr %byteOff.ptr, align 1
%bitOff = shl i128 %byteOff, 3
@@ -2542,81 +1840,38 @@ define void @ashr_16bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
}
define void @ashr_16bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) nounwind {
-; X64-NO-SHLD-NO-BMI2-LABEL: ashr_16bytes_dwordOff:
-; X64-NO-SHLD-NO-BMI2: # %bb.0:
-; X64-NO-SHLD-NO-BMI2-NEXT: movq (%rdi), %r8
-; X64-NO-SHLD-NO-BMI2-NEXT: movq 8(%rdi), %rdi
-; X64-NO-SHLD-NO-BMI2-NEXT: movzbl (%rsi), %eax
-; X64-NO-SHLD-NO-BMI2-NEXT: shlb $5, %al
-; X64-NO-SHLD-NO-BMI2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-NEXT: shrq %cl, %r8
-; X64-NO-SHLD-NO-BMI2-NEXT: leaq (%rdi,%rdi), %rsi
-; X64-NO-SHLD-NO-BMI2-NEXT: notb %cl
-; X64-NO-SHLD-NO-BMI2-NEXT: shlq %cl, %rsi
-; X64-NO-SHLD-NO-BMI2-NEXT: orq %r8, %rsi
-; X64-NO-SHLD-NO-BMI2-NEXT: movq %rdi, %r8
-; X64-NO-SHLD-NO-BMI2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-NEXT: sarq %cl, %r8
-; X64-NO-SHLD-NO-BMI2-NEXT: sarq $63, %rdi
-; X64-NO-SHLD-NO-BMI2-NEXT: testb $64, %al
-; X64-NO-SHLD-NO-BMI2-NEXT: cmovneq %r8, %rsi
-; X64-NO-SHLD-NO-BMI2-NEXT: cmoveq %r8, %rdi
-; X64-NO-SHLD-NO-BMI2-NEXT: movq %rdi, 8(%rdx)
-; X64-NO-SHLD-NO-BMI2-NEXT: movq %rsi, (%rdx)
-; X64-NO-SHLD-NO-BMI2-NEXT: retq
-;
-; X64-HAVE-SHLD-NO-BMI2-LABEL: ashr_16bytes_dwordOff:
-; X64-HAVE-SHLD-NO-BMI2: # %bb.0:
-; X64-HAVE-SHLD-NO-BMI2-NEXT: movq (%rdi), %rax
-; X64-HAVE-SHLD-NO-BMI2-NEXT: movq 8(%rdi), %rdi
-; X64-HAVE-SHLD-NO-BMI2-NEXT: movzbl (%rsi), %ecx
-; X64-HAVE-SHLD-NO-BMI2-NEXT: shlb $5, %cl
-; X64-HAVE-SHLD-NO-BMI2-NEXT: movq %rdi, %rsi
-; X64-HAVE-SHLD-NO-BMI2-NEXT: sarq %cl, %rsi
-; X64-HAVE-SHLD-NO-BMI2-NEXT: shrdq %cl, %rdi, %rax
-; X64-HAVE-SHLD-NO-BMI2-NEXT: sarq $63, %rdi
-; X64-HAVE-SHLD-NO-BMI2-NEXT: testb $64, %cl
-; X64-HAVE-SHLD-NO-BMI2-NEXT: cmovneq %rsi, %rax
-; X64-HAVE-SHLD-NO-BMI2-NEXT: cmoveq %rsi, %rdi
-; X64-HAVE-SHLD-NO-BMI2-NEXT: movq %rdi, 8(%rdx)
-; X64-HAVE-SHLD-NO-BMI2-NEXT: movq %rax, (%rdx)
-; X64-HAVE-SHLD-NO-BMI2-NEXT: retq
-;
-; X64-NO-SHLD-HAVE-BMI2-LABEL: ashr_16bytes_dwordOff:
-; X64-NO-SHLD-HAVE-BMI2: # %bb.0:
-; X64-NO-SHLD-HAVE-BMI2-NEXT: movq 8(%rdi), %rax
-; X64-NO-SHLD-HAVE-BMI2-NEXT: movzbl (%rsi), %ecx
-; X64-NO-SHLD-HAVE-BMI2-NEXT: shlb $5, %cl
-; X64-NO-SHLD-HAVE-BMI2-NEXT: shrxq %rcx, (%rdi), %rsi
-; X64-NO-SHLD-HAVE-BMI2-NEXT: movl %ecx, %edi
-; X64-NO-SHLD-HAVE-BMI2-NEXT: notb %dil
-; X64-NO-SHLD-HAVE-BMI2-NEXT: leaq (%rax,%rax), %r8
-; X64-NO-SHLD-HAVE-BMI2-NEXT: shlxq %rdi, %r8, %rdi
-; X64-NO-SHLD-HAVE-BMI2-NEXT: orq %rsi, %rdi
-; X64-NO-SHLD-HAVE-BMI2-NEXT: sarxq %rcx, %rax, %rsi
-; X64-NO-SHLD-HAVE-BMI2-NEXT: sarq $63, %rax
-; X64-NO-SHLD-HAVE-BMI2-NEXT: testb $64, %cl
-; X64-NO-SHLD-HAVE-BMI2-NEXT: cmovneq %rsi, %rdi
-; X64-NO-SHLD-HAVE-BMI2-NEXT: cmoveq %rsi, %rax
-; X64-NO-SHLD-HAVE-BMI2-NEXT: movq %rax, 8(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-NEXT: movq %rdi, (%rdx)
-; X64-NO-SHLD-HAVE-BMI2-NEXT: retq
-;
-; X64-HAVE-SHLD-HAVE-BMI2-LABEL: ashr_16bytes_dwordOff:
-; X64-HAVE-SHLD-HAVE-BMI2: # %bb.0:
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: movq (%rdi), %rax
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: movq 8(%rdi), %rdi
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: movzbl (%rsi), %ecx
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: shlb $5, %cl
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: shrdq %cl, %rdi, %rax
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: sarxq %rcx, %rdi, %rsi
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: sarq $63, %rdi
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: testb $64, %cl
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: cmovneq %rsi, %rax
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: cmoveq %rsi, %rdi
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: movq %rdi, 8(%rdx)
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: movq %rax, (%rdx)
-; X64-HAVE-SHLD-HAVE-BMI2-NEXT: retq
+; X64-NO-BMI2-LABEL: ashr_16bytes_dwordOff:
+; X64-NO-BMI2: # %bb.0:
+; X64-NO-BMI2-NEXT: movq (%rdi), %rax
+; X64-NO-BMI2-NEXT: movq 8(%rdi), %rdi
+; X64-NO-BMI2-NEXT: movzbl (%rsi), %ecx
+; X64-NO-BMI2-NEXT: shlb $5, %cl
+; X64-NO-BMI2-NEXT: movq %rdi, %rsi
+; X64-NO-BMI2-NEXT: sarq %cl, %rsi
+; X64-NO-BMI2-NEXT: shrdq %cl, %rdi, %rax
+; X64-NO-BMI2-NEXT: sarq $63, %rdi
+; X64-NO-BMI2-NEXT: testb $64, %cl
+; X64-NO-BMI2-NEXT: cmovneq %rsi, %rax
+; X64-NO-BMI2-NEXT: cmoveq %rsi, %rdi
+; X64-NO-BMI2-NEXT: movq %rdi, 8(%rdx)
+; X64-NO-BMI2-NEXT: movq %rax, (%rdx)
+; X64-NO-BMI2-NEXT: retq
+;
+; X64-HAVE-BMI2-LABEL: ashr_16bytes_dwordOff:
+; X64-HAVE-BMI2: # %bb.0:
+; X64-HAVE-BMI2-NEXT: movq (%rdi), %rax
+; X64-HAVE-BMI2-NEXT: movq 8(%rdi), %rdi
+; X64-HAVE-BMI2-NEXT: movzbl (%rsi), %ecx
+; X64-HAVE-BMI2-NEXT: shlb $5, %cl
+; X64-HAVE-BMI2-NEXT: shrdq %cl, %rdi, %rax
+; X64-HAVE-BMI2-NEXT: sarxq %rcx, %rdi, %rsi
+; X64-HAVE-BMI2-NEXT: sarq $63, %rdi
+; X64-HAVE-BMI2-NEXT: testb $64, %cl
+; X64-HAVE-BMI2-NEXT: cmovneq %rsi, %rax
+; X64-HAVE-BMI2-NEXT: cmoveq %rsi, %rdi
+; X64-HAVE-BMI2-NEXT: movq %rdi, 8(%rdx)
+; X64-HAVE-BMI2-NEXT: movq %rax, (%rdx)
+; X64-HAVE-BMI2-NEXT: retq
;
; X86-SSE2-LABEL: ashr_16bytes_dwordOff:
; X86-SSE2: # %bb.0:
@@ -2730,54 +1985,35 @@ define void @ashr_16bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) no
define void @lshr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-NO-BMI2-SSE2-LABEL: lshr_32bytes:
; X64-NO-SHLD-NO-BMI2-SSE2: # %bb.0:
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: pushq %rbx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq (%rdi), %rcx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq (%rdi), %rax
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 8(%rdi), %r8
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 16(%rdi), %r9
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 24(%rdi), %rdi
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movzbl (%rsi), %esi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: leal (,%rsi,8), %eax
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: leal (,%rsi,8), %ecx
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: xorps %xmm0, %xmm0
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: andb $24, %sil
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movzbl %sil, %r9d
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -64(%rsp,%r9), %r10
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -56(%rsp,%r9), %rdi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rdi, %r11
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %cl, %r11
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %esi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: notb %sil
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -48(%rsp,%r9), %rbx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: leaq (%rbx,%rbx), %r8
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlq %cl, %r8
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: orq %r11, %r8
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: addq %rdi, %rdi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlq %cl, %rdi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: orq %r10, %rdi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %cl, %rbx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -40(%rsp,%r9), %r9
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: leaq (%r9,%r9), %r10
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: orq %rbx, %r10
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %cl, %r9
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r9, 24(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r10, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movzbl %sil, %eax
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -56(%rsp,%rax), %rsi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -72(%rsp,%rax), %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -64(%rsp,%rax), %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r8, %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdq %cl, %rsi, %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -48(%rsp,%rax), %rax
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdq %cl, %rax, %rsi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdq %cl, %r8, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: # kill: def $cl killed $cl killed $ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %cl, %rax
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rsi, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rax, 24(%rdx)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rdi, (%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r8, 8(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: popq %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r9, 8(%rdx)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: retq
;
; X64-HAVE-SHLD-NO-BMI2-SSE2-LABEL: lshr_32bytes:
@@ -2815,43 +2051,34 @@ define void @lshr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
;
; X64-NO-SHLD-HAVE-BMI2-SSE2-LABEL: lshr_32bytes:
; X64-NO-SHLD-HAVE-BMI2-SSE2: # %bb.0:
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq (%rdi), %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq (%rdi), %rax
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq 8(%rdi), %r8
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq 16(%rdi), %r9
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq 24(%rdi), %rdi
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movzbl (%rsi), %esi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leal (,%rsi,8), %eax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leal (,%rsi,8), %ecx
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: xorps %xmm0, %xmm0
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, %ecx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: andb $24, %sil
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movzbl %sil, %esi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -64(%rsp,%rsi), %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -56(%rsp,%rsi), %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxq %rcx, %rdi, %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: notb %al
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leaq (%r8,%r8), %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxq %rax, %r10, %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orq %r9, %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxq %rcx, -72(%rsp,%rsi), %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addq %rdi, %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxq %rax, %rdi, %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orq %r9, %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxq %rcx, %r8, %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -48(%rsp,%rsi), %rsi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leaq (%rsi,%rsi), %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxq %rax, %r9, %rax
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orq %r8, %rax
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxq %rcx, %rsi, %rcx
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rcx, 24(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rax, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movzbl %sil, %eax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -56(%rsp,%rax), %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -72(%rsp,%rax), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -64(%rsp,%rax), %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r8, %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdq %cl, %rsi, %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -48(%rsp,%rax), %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdq %cl, %rax, %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdq %cl, %r8, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxq %rcx, %rax, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rsi, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rax, 24(%rdx)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rdi, (%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r10, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r9, 8(%rdx)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: retq
;
; X64-HAVE-SHLD-HAVE-BMI2-SSE2-LABEL: lshr_32bytes:
@@ -2888,50 +2115,32 @@ define void @lshr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
;
; X64-NO-SHLD-NO-BMI2-SSE4-LABEL: lshr_32bytes:
; X64-NO-SHLD-NO-BMI2-SSE4: # %bb.0:
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: pushq %rbx
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movups (%rdi), %xmm0
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movups 16(%rdi), %xmm1
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movzbl (%rsi), %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: leal (,%rcx,8), %eax
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movzbl (%rsi), %eax
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: leal (,%rax,8), %ecx
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: xorps %xmm2, %xmm2
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: andb $24, %cl
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movzbl %cl, %r9d
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -64(%rsp,%r9), %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -56(%rsp,%r9), %r8
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %esi
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: notb %sil
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: leaq (%r8,%r8), %rdi
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlq %cl, %rdi
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: orq %r10, %rdi
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -48(%rsp,%r9), %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r10, %r11
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %cl, %r11
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -40(%rsp,%r9), %r9
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: leaq (%r9,%r9), %rbx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlq %cl, %rbx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: orq %r11, %rbx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %cl, %r8
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: addq %r10, %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: orq %r8, %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %cl, %r9
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r9, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: andb $24, %al
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movzbl %al, %eax
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -48(%rsp,%rax), %rsi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -56(%rsp,%rax), %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rdi, %r8
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdq %cl, %rsi, %r8
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -72(%rsp,%rax), %r9
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -64(%rsp,%rax), %rax
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rax, %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdq %cl, %rdi, %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdq %cl, %rax, %r9
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: # kill: def $cl killed $cl killed $ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %cl, %rsi
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r10, 8(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rbx, 16(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rdi, (%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: popq %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r8, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rsi, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r9, (%rdx)
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: retq
;
; X64-HAVE-SHLD-NO-BMI2-SSE4-LABEL: lshr_32bytes:
@@ -2968,37 +2177,29 @@ define void @lshr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-HAVE-BMI2-SSE4: # %bb.0:
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movups (%rdi), %xmm0
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movups 16(%rdi), %xmm1
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movzbl (%rsi), %ecx
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leal (,%rcx,8), %eax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movzbl (%rsi), %eax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leal (,%rax,8), %ecx
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: xorps %xmm2, %xmm2
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, %esi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: andb $24, %cl
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movzbl %cl, %ecx
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxq %rsi, -72(%rsp,%rcx), %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: notb %al
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -64(%rsp,%rcx), %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -56(%rsp,%rcx), %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leaq (%r8,%r8), %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxq %rax, %r10, %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orq %rdi, %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxq %rsi, %r9, %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -48(%rsp,%rcx), %rcx
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leaq (%rcx,%rcx), %r11
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxq %rax, %r11, %r11
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orq %rdi, %r11
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxq %rsi, %r8, %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: addq %r9, %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxq %rax, %r9, %rax
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orq %rdi, %rax
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxq %rsi, %rcx, %rcx
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rcx, 24(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rax, 8(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r11, 16(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r10, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: andb $24, %al
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movzbl %al, %eax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -48(%rsp,%rax), %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -56(%rsp,%rax), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rdi, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdq %cl, %rsi, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -72(%rsp,%rax), %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -64(%rsp,%rax), %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rax, %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdq %cl, %rdi, %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdq %cl, %rax, %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxq %rcx, %rsi, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r10, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r8, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rax, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r9, (%rdx)
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: retq
;
; X64-HAVE-SHLD-HAVE-BMI2-SSE4-LABEL: lshr_32bytes:
@@ -3032,47 +2233,29 @@ define void @lshr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
;
; X64-NO-SHLD-NO-BMI2-AVX-LABEL: lshr_32bytes:
; X64-NO-SHLD-NO-BMI2-AVX: # %bb.0:
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: pushq %rbx
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: vmovups (%rdi), %ymm0
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movzbl (%rsi), %ecx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: leal (,%rcx,8), %eax
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movzbl (%rsi), %eax
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: leal (,%rax,8), %ecx
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: vxorps %xmm1, %xmm1, %xmm1
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: andb $24, %cl
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movzbl %cl, %r9d
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -64(%rsp,%r9), %r10
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -56(%rsp,%r9), %r8
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shrq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %esi
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: notb %sil
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: leaq (%r8,%r8), %rdi
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shlq %cl, %rdi
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: orq %r10, %rdi
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -48(%rsp,%r9), %r10
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %r10, %r11
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shrq %cl, %r11
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -40(%rsp,%r9), %r9
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: leaq (%r9,%r9), %rbx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shlq %cl, %rbx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: orq %r11, %rbx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shrq %cl, %r8
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: addq %r10, %r10
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shlq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: orq %r8, %r10
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shrq %cl, %r9
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %r9, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: andb $24, %al
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movzbl %al, %eax
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -48(%rsp,%rax), %rsi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -56(%rsp,%rax), %rdi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %rdi, %r8
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shrdq %cl, %rsi, %r8
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -72(%rsp,%rax), %r9
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -64(%rsp,%rax), %rax
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %rax, %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shrdq %cl, %rdi, %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shrdq %cl, %rax, %r9
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: # kill: def $cl killed $cl killed $ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shrq %cl, %rsi
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %r10, 8(%rdx)
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %rbx, 16(%rdx)
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %rdi, (%rdx)
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: popq %rbx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %r8, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %rsi, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %r9, (%rdx)
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: vzeroupper
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: retq
;
@@ -3112,30 +2295,22 @@ define void @lshr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: vxorps %xmm1, %xmm1, %xmm1
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %ecx, %esi
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: andb $24, %al
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movzbl %al, %eax
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxq %rsi, -72(%rsp,%rax), %rdi
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: notb %cl
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq -64(%rsp,%rax), %r8
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq -56(%rsp,%rax), %r9
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: leaq (%r8,%r8), %r10
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxq %rcx, %r10, %r10
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: orq %rdi, %r10
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxq %rsi, %r9, %rdi
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq -48(%rsp,%rax), %rax
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: leaq (%rax,%rax), %r11
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxq %rcx, %r11, %r11
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: orq %rdi, %r11
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxq %rsi, %r8, %rdi
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: addq %r9, %r9
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxq %rcx, %r9, %rcx
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: orq %rdi, %rcx
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxq %rsi, %rax, %rax
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq -48(%rsp,%rax), %rsi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq -56(%rsp,%rax), %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %rdi, %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrdq %cl, %rsi, %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq -72(%rsp,%rax), %r9
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq -64(%rsp,%rax), %rax
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %rax, %r10
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrdq %cl, %rdi, %r10
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrdq %cl, %rax, %r9
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxq %rcx, %rsi, %rax
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %r10, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %r8, 16(%rdx)
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %rax, 24(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %rcx, 8(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %r11, 16(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %r10, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %r9, (%rdx)
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: vzeroupper
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: retq
;
@@ -3172,122 +2347,73 @@ define void @lshr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: pushl %ebx
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: pushl %edi
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: pushl %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: subl $108, %esp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: subl $92, %esp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl (%ebp), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 4(%ebp), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 8(%ebp), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl (%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 4(%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, (%esp) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 8(%ebp), %esi
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 12(%ebp), %edi
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 16(%ebp), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb (%eax), %ah
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 20(%ebp), %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 24(%ebp), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb (%ecx), %ch
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 20(%ebp), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 24(%ebp), %eax
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 28(%ebp), %ebp
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: xorps %xmm0, %xmm0
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebp, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ah, %dh
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shlb $3, %dh
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shlb $3, %cl
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: andb $28, %ah
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movzbl %ah, %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 32(%esp,%edi), %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 36(%esp,%edi), %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %dh, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %dh, %dl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: notb %dl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 40(%esp,%edi), %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: leal (%edi,%edi), %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %ebx, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %dh, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: addl %eax, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %esi, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 44(%esp,%eax), %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebp, %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %dh, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 48(%esp,%eax), %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: leal (%eax,%eax), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %esi, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %dh, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: addl %ebp, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %edi, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl (%esp), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 52(%esp,%eax), %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %dh, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 56(%esp,%eax), %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: leal (%esi,%esi), %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %ebx, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: andb $28, %ch
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movzbl %ch, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 24(%esp,%ebp), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 20(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, (%esp) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %edx, %eax
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %dh, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: addl %edi, %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %ebx, %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %dh, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 32(%esp,%ebp), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 28(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %ebx, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %eax, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 40(%esp,%ebp), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 36(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %edx, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %eax, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 16(%esp,%ebp), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 44(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %eax, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, 24(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl (%esp), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %edx, %esi
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 60(%esp,%ecx), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: leal (%ebx,%ebx), %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %eax, %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %dh, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, 28(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, 24(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, 16(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, 20(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebp, 8(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, 12(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, (%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, 4(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: addl $108, %esp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, 28(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, 16(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, 20(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, 8(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, 12(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, (%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, 4(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: addl $92, %esp
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: popl %esi
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: popl %edi
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: popl %ebx
@@ -3379,24 +2505,24 @@ define void @lshr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: pushl %ebx
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: pushl %edi
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: pushl %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: subl $108, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: subl $92, %esp
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl (%eax), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 4(%eax), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 8(%eax), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 12(%eax), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 16(%eax), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl (%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 4(%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, (%esp) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 8(%ecx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 12(%ecx), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 16(%ecx), %ebp
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movzbl (%ebx), %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 20(%eax), %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 24(%eax), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 28(%eax), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 20(%ecx), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 24(%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 28(%ecx), %ecx
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: xorps %xmm0, %xmm0
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebx, %ecx
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlb $3, %cl
@@ -3404,70 +2530,49 @@ define void @lshr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebp, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl (%esp), %eax # 4-byte Reload
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, %eax
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: andb $28, %bl
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movzbl %bl, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 36(%esp,%esi), %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 40(%esp,%esi), %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %eax, %edx, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: notb %cl
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leal (%ebp,%ebp), %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %ecx, %ebx, %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %edi, %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %eax, 32(%esp,%esi), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addl %edx, %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %ecx, %edx, %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %edi, %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 48(%esp,%esi), %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addl %edx, %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %ecx, %edx, %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 44(%esp,%esi), %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %eax, %edx, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %edi, %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %eax, %ebp, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addl %edx, %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %ecx, %edx, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movzbl %bl, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 24(%esp,%ebp), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 20(%esp,%ebp), %eax
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 56(%esp,%esi), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leal (%edi,%edi), %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %ecx, %edx, %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 52(%esp,%esi), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ebp, %eax, %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %ebx, %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ebp, {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addl %eax, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %ecx, %eax, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %ebx, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 60(%esp,%esi), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leal (%esi,%esi), %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %ecx, %ebx, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ebp, %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %edi, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ebp, %esi, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, 28(%edi)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, 24(%edi)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 16(%edi)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, 20(%edi)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 8(%edi)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, (%esp) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 32(%esp,%ebp), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 28(%esp,%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %ebx, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %eax, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 40(%esp,%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 36(%esp,%ebp), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %eax, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %edx, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 16(%esp,%ebp), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 44(%esp,%ebp), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 24(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ecx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 28(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebx, 16(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, 20(%ebp)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 12(%edi)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 8(%ebp)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, (%edi)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 12(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: # kill: def $cl killed $cl killed $ecx
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 4(%edi)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addl $108, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, (%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl (%esp), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 4(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addl $92, %esp
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: popl %esi
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: popl %edi
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: popl %ebx
@@ -3565,96 +2670,54 @@ define void @lshr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movups (%ecx), %xmm0
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movups 16(%ecx), %xmm1
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movzbl (%eax), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shlb $3, %al
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movzbl (%eax), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shlb $3, %cl
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: xorps %xmm2, %xmm2
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: andb $28, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movzbl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 32(%esp,%edi), %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 36(%esp,%edi), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: andb $28, %al
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movzbl %al, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 48(%esp,%ebp), %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 44(%esp,%ebp), %eax
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: notb %dl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: addl %ebx, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %esi, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 44(%esp,%edi), %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebp, %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 48(%esp,%edi), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: leal (%ecx,%ecx), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %esi, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 40(%esp,%edi), %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: addl %ebp, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %ebx, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 52(%esp,%edi), %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebp, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 56(%esp,%edi), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, (%esp) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: leal (%ecx,%ecx), %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %ebx, %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: addl %ebp, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %edi, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, (%esp) # 4-byte Folded Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 60(%esp,%ecx), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: leal (%ebx,%ebx), %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl (%esp), %edi # 4-byte Folded Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: addl %esi, %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, 28(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, 4(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edi, 24(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebp, 16(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, 20(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, 8(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, 12(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, (%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %esi, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 40(%esp,%ebp), %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %eax, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 56(%esp,%ebp), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 52(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %ebx, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %eax, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 60(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %eax, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 32(%esp,%ebp), %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 36(%esp,%ebp), %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edi, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %ebp, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, 4(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, 24(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %edi, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 28(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 16(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 20(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 8(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 12(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, (%ebp)
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: addl $108, %esp
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: popl %esi
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: popl %edi
@@ -3739,76 +2802,53 @@ define void @lshr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movups (%ecx), %xmm0
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movups 16(%ecx), %xmm1
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movzbl (%eax), %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movzbl (%eax), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, %ecx
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlb $3, %cl
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: xorps %xmm2, %xmm2
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: andb $28, %dl
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movzbl %dl, %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %eax, 32(%esp,%ebx), %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: notb %cl
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 36(%esp,%ebx), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leal (%eax,%eax), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %ecx, %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %edx, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 48(%esp,%ebx), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leal (%eax,%eax), %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %ecx, %edx, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 44(%esp,%ebx), %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %edi, %edx, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: addl %edx, %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %ecx, %edx, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: andb $28, %al
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movzbl %al, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 48(%esp,%ebx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 44(%esp,%ebx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %esi, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 40(%esp,%ebx), %edx
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %edi, %edx, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edi, %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 56(%esp,%ebx), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leal (%esi,%esi), %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %ecx, %ebp, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 56(%esp,%ebx), %ebp
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 52(%esp,%ebx), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %edi, %eax, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %edi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %edx, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: addl %eax, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %ecx, %eax, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %edx, %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 60(%esp,%ebx), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leal (%esi,%esi), %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %ecx, %ebx, %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %eax, %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: addl %eax, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %ecx, %eax, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %edx, {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %ecx, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %edx, %esi, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, 28(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 4(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ebx, 24(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edi, 16(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ebp, 20(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 8(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 12(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %ebp, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %eax, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 60(%esp,%ebx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %eax, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 32(%esp,%ebx), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 36(%esp,%ebx), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, (%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %eax, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ebx, 4(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ebp, 24(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ebx, 28(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %esi, 16(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edi, 20(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %esi, 8(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %esi, 12(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %esi, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, (%eax)
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: addl $108, %esp
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: popl %esi
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: popl %edi
@@ -3891,94 +2931,52 @@ define void @lshr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: vmovups (%ecx), %ymm0
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movzbl (%eax), %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ecx, %eax
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shlb $3, %al
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movzbl (%eax), %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shlb $3, %cl
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: vxorps %xmm1, %xmm1, %xmm1
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: vmovups %ymm1, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: vmovups %ymm0, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: andb $28, %cl
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movzbl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 32(%esp,%edi), %esi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 36(%esp,%edi), %ebx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrl %cl, %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: andb $28, %al
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movzbl %al, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 48(%esp,%ebp), %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 44(%esp,%ebp), %eax
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %edx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: notb %dl
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: addl %ebx, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: orl %esi, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 44(%esp,%edi), %ebp
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ebp, %esi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrl %cl, %esi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 48(%esp,%edi), %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: leal (%ecx,%ecx), %ebx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: orl %esi, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 40(%esp,%edi), %esi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %esi, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: addl %ebp, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shll %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: orl %ebx, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 52(%esp,%edi), %ebp
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ebp, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 56(%esp,%edi), %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ecx, (%esp) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: leal (%ecx,%ecx), %edi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shll %cl, %edi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: orl %ebx, %edi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: addl %ebp, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shll %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: orl %edi, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrl %cl, (%esp) # 4-byte Folded Spill
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 60(%esp,%ecx), %ebx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: leal (%ebx,%ebx), %edi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shll %cl, %edi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: orl (%esp), %edi # 4-byte Folded Reload
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrl %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: addl %esi, %esi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shll %cl, %esi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ebx, 28(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %esi, 4(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edi, 24(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ebp, 16(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ecx, 20(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ecx, 8(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ecx, 12(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ecx, (%eax)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrdl %cl, %esi, %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 40(%esp,%ebp), %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrdl %cl, %eax, %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 56(%esp,%ebp), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 52(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrdl %cl, %ebx, %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrdl %cl, %eax, %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 60(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrdl %cl, %eax, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 32(%esp,%ebp), %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 36(%esp,%ebp), %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edi, %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrdl %cl, %ebp, %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %esi, 4(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ebx, 24(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrdl %cl, %edi, %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, 28(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, 16(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, 20(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, 8(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, 12(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, (%ebp)
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: addl $108, %esp
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: popl %esi
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: popl %edi
@@ -4061,74 +3059,51 @@ define void @lshr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: vmovups (%ecx), %ymm0
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movzbl (%eax), %edx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edx, %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movzbl (%eax), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, %ecx
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlb $3, %cl
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: vxorps %xmm1, %xmm1, %xmm1
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: vmovups %ymm1, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: vmovups %ymm0, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %ecx, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: andb $28, %dl
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movzbl %dl, %ebx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxl %eax, 32(%esp,%ebx), %edx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: notb %cl
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 36(%esp,%ebx), %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: leal (%eax,%eax), %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxl %ecx, %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: orl %edx, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 48(%esp,%ebx), %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: leal (%eax,%eax), %edx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxl %ecx, %edx, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 44(%esp,%ebx), %edx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxl %edi, %edx, %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: orl %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: addl %edx, %edx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxl %ecx, %edx, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: andb $28, %al
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movzbl %al, %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 48(%esp,%ebx), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 44(%esp,%ebx), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrdl %cl, %esi, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 40(%esp,%ebx), %edx
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxl %edi, %edx, %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edi, %edx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: orl %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 56(%esp,%ebx), %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: leal (%esi,%esi), %ebp
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxl %ecx, %ebp, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrdl %cl, %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 56(%esp,%ebx), %ebp
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 52(%esp,%ebx), %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxl %edi, %eax, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: orl %edi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxl %edx, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: addl %eax, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxl %ecx, %eax, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxl %edx, %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 60(%esp,%ebx), %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: leal (%esi,%esi), %ebx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxl %ecx, %ebx, %ebx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: orl %eax, %ebx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: addl %eax, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxl %ecx, %eax, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxl %edx, {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: orl %ecx, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxl %edx, %esi, %ecx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %ecx, 28(%edx)
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, 4(%edx)
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %ebx, 24(%edx)
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edi, 16(%edx)
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %ebp, 20(%edx)
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, 8(%edx)
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, 12(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrdl %cl, %ebp, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrdl %cl, %eax, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 60(%esp,%ebx), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrdl %cl, %eax, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 32(%esp,%ebx), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 36(%esp,%ebx), %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, (%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrdl %cl, %eax, %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %ebx, 4(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %ebp, 24(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %ebx, 28(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %esi, 16(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edi, 20(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %esi, 8(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %esi, 12(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrdl %cl, %esi, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edx, (%eax)
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: addl $108, %esp
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: popl %esi
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: popl %edi
@@ -4210,55 +3185,35 @@ define void @lshr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
define void @lshr_32bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-NO-BMI2-SSE2-LABEL: lshr_32bytes_dwordOff:
; X64-NO-SHLD-NO-BMI2-SSE2: # %bb.0:
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: pushq %rbx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq (%rdi), %rcx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq (%rdi), %rax
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 8(%rdi), %r8
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 16(%rdi), %r9
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 24(%rdi), %rdi
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movzbl (%rsi), %esi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %eax
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlb $5, %al
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlb $5, %cl
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: xorps %xmm0, %xmm0
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: andb $6, %sil
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movzbl %sil, %r9d
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -64(%rsp,%r9,4), %r10
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -56(%rsp,%r9,4), %rdi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rdi, %r11
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %cl, %r11
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %esi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: notb %sil
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -48(%rsp,%r9,4), %rbx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: leaq (%rbx,%rbx), %r8
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlq %cl, %r8
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: orq %r11, %r8
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: addq %rdi, %rdi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlq %cl, %rdi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: orq %r10, %rdi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %cl, %rbx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -40(%rsp,%r9,4), %r9
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: leaq (%r9,%r9), %r10
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: orq %rbx, %r10
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %cl, %r9
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r9, 24(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r10, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movzbl %sil, %eax
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -56(%rsp,%rax,4), %rsi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -72(%rsp,%rax,4), %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -64(%rsp,%rax,4), %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r8, %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdq %cl, %rsi, %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -48(%rsp,%rax,4), %rax
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdq %cl, %rax, %rsi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdq %cl, %r8, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %cl, %rax
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rsi, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rax, 24(%rdx)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rdi, (%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r8, 8(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: popq %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r9, 8(%rdx)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: retq
;
; X64-HAVE-SHLD-NO-BMI2-SSE2-LABEL: lshr_32bytes_dwordOff:
@@ -4296,44 +3251,35 @@ define void @lshr_32bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) no
;
; X64-NO-SHLD-HAVE-BMI2-SSE2-LABEL: lshr_32bytes_dwordOff:
; X64-NO-SHLD-HAVE-BMI2-SSE2: # %bb.0:
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq (%rdi), %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq (%rdi), %rax
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq 8(%rdi), %r8
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq 16(%rdi), %r9
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq 24(%rdi), %rdi
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movzbl (%rsi), %esi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, %eax
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlb $5, %al
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, %ecx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlb $5, %cl
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: xorps %xmm0, %xmm0
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, %ecx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: andb $6, %sil
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movzbl %sil, %esi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -64(%rsp,%rsi,4), %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -56(%rsp,%rsi,4), %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxq %rcx, %rdi, %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: notb %al
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leaq (%r8,%r8), %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxq %rax, %r10, %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orq %r9, %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxq %rcx, -72(%rsp,%rsi,4), %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addq %rdi, %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxq %rax, %rdi, %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orq %r9, %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxq %rcx, %r8, %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -48(%rsp,%rsi,4), %rsi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leaq (%rsi,%rsi), %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxq %rax, %r9, %rax
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orq %r8, %rax
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxq %rcx, %rsi, %rcx
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rcx, 24(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rax, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movzbl %sil, %eax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -56(%rsp,%rax,4), %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -72(%rsp,%rax,4), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -64(%rsp,%rax,4), %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r8, %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdq %cl, %rsi, %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -48(%rsp,%rax,4), %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdq %cl, %rax, %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdq %cl, %r8, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxq %rcx, %rax, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rsi, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rax, 24(%rdx)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rdi, (%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r10, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r9, 8(%rdx)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: retq
;
; X64-HAVE-SHLD-HAVE-BMI2-SSE2-LABEL: lshr_32bytes_dwordOff:
@@ -4371,51 +3317,32 @@ define void @lshr_32bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) no
;
; X64-NO-SHLD-NO-BMI2-SSE4-LABEL: lshr_32bytes_dwordOff:
; X64-NO-SHLD-NO-BMI2-SSE4: # %bb.0:
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: pushq %rbx
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movups (%rdi), %xmm0
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movups 16(%rdi), %xmm1
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movzbl (%rsi), %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, %eax
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlb $5, %al
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movzbl (%rsi), %eax
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlb $5, %cl
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: xorps %xmm2, %xmm2
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: andb $6, %cl
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movzbl %cl, %r9d
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -64(%rsp,%r9,4), %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -56(%rsp,%r9,4), %r8
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %esi
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: notb %sil
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: leaq (%r8,%r8), %rdi
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlq %cl, %rdi
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: orq %r10, %rdi
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -48(%rsp,%r9,4), %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r10, %r11
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %cl, %r11
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -40(%rsp,%r9,4), %r9
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: leaq (%r9,%r9), %rbx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlq %cl, %rbx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: orq %r11, %rbx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %cl, %r8
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: addq %r10, %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: orq %r8, %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %cl, %r9
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r9, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: andb $6, %al
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movzbl %al, %eax
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -48(%rsp,%rax,4), %rsi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -56(%rsp,%rax,4), %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rdi, %r8
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdq %cl, %rsi, %r8
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -72(%rsp,%rax,4), %r9
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -64(%rsp,%rax,4), %rax
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rax, %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdq %cl, %rdi, %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdq %cl, %rax, %r9
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %cl, %rsi
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r10, 8(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rbx, 16(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rdi, (%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: popq %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r8, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rsi, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r9, (%rdx)
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: retq
;
; X64-HAVE-SHLD-NO-BMI2-SSE4-LABEL: lshr_32bytes_dwordOff:
@@ -4452,38 +3379,30 @@ define void @lshr_32bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) no
; X64-NO-SHLD-HAVE-BMI2-SSE4: # %bb.0:
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movups (%rdi), %xmm0
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movups 16(%rdi), %xmm1
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movzbl (%rsi), %ecx
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, %eax
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlb $5, %al
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movzbl (%rsi), %eax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, %ecx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlb $5, %cl
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: xorps %xmm2, %xmm2
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, %esi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: andb $6, %cl
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movzbl %cl, %ecx
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxq %rsi, -72(%rsp,%rcx,4), %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: notb %al
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -64(%rsp,%rcx,4), %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -56(%rsp,%rcx,4), %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leaq (%r8,%r8), %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxq %rax, %r10, %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orq %rdi, %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxq %rsi, %r9, %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -48(%rsp,%rcx,4), %rcx
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leaq (%rcx,%rcx), %r11
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxq %rax, %r11, %r11
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orq %rdi, %r11
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxq %rsi, %r8, %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: addq %r9, %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxq %rax, %r9, %rax
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orq %rdi, %rax
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxq %rsi, %rcx, %rcx
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rcx, 24(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rax, 8(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r11, 16(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r10, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: andb $6, %al
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movzbl %al, %eax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -48(%rsp,%rax,4), %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -56(%rsp,%rax,4), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rdi, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdq %cl, %rsi, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -72(%rsp,%rax,4), %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -64(%rsp,%rax,4), %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rax, %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdq %cl, %rdi, %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdq %cl, %rax, %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxq %rcx, %rsi, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r10, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r8, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rax, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r9, (%rdx)
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: retq
;
; X64-HAVE-SHLD-HAVE-BMI2-SSE4-LABEL: lshr_32bytes_dwordOff:
@@ -4518,48 +3437,29 @@ define void @lshr_32bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) no
;
; X64-NO-SHLD-NO-BMI2-AVX-LABEL: lshr_32bytes_dwordOff:
; X64-NO-SHLD-NO-BMI2-AVX: # %bb.0:
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: pushq %rbx
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: vmovups (%rdi), %ymm0
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movzbl (%rsi), %ecx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ecx, %eax
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shlb $5, %al
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movzbl (%rsi), %eax
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shlb $5, %cl
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: vxorps %xmm1, %xmm1, %xmm1
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: andb $6, %cl
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movzbl %cl, %r9d
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -64(%rsp,%r9,4), %r10
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -56(%rsp,%r9,4), %r8
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shrq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %esi
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: notb %sil
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: leaq (%r8,%r8), %rdi
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shlq %cl, %rdi
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: orq %r10, %rdi
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -48(%rsp,%r9,4), %r10
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %r10, %r11
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shrq %cl, %r11
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -40(%rsp,%r9,4), %r9
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: leaq (%r9,%r9), %rbx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shlq %cl, %rbx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: orq %r11, %rbx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shrq %cl, %r8
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: addq %r10, %r10
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shlq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: orq %r8, %r10
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shrq %cl, %r9
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %r9, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: andb $6, %al
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movzbl %al, %eax
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -48(%rsp,%rax,4), %rsi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -56(%rsp,%rax,4), %rdi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %rdi, %r8
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shrdq %cl, %rsi, %r8
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -72(%rsp,%rax,4), %r9
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -64(%rsp,%rax,4), %rax
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %rax, %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shrdq %cl, %rdi, %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shrdq %cl, %rax, %r9
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shrq %cl, %rsi
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %r10, 8(%rdx)
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %rbx, 16(%rdx)
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %rdi, (%rdx)
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: popq %rbx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %r8, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %rsi, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %r9, (%rdx)
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: vzeroupper
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: retq
;
@@ -4600,30 +3500,22 @@ define void @lshr_32bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) no
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: vxorps %xmm1, %xmm1, %xmm1
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %ecx, %esi
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: andb $6, %al
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movzbl %al, %eax
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxq %rsi, -72(%rsp,%rax,4), %rdi
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: notb %cl
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq -64(%rsp,%rax,4), %r8
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq -56(%rsp,%rax,4), %r9
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: leaq (%r8,%r8), %r10
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxq %rcx, %r10, %r10
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: orq %rdi, %r10
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxq %rsi, %r9, %rdi
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq -48(%rsp,%rax,4), %rax
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: leaq (%rax,%rax), %r11
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxq %rcx, %r11, %r11
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: orq %rdi, %r11
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxq %rsi, %r8, %rdi
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: addq %r9, %r9
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxq %rcx, %r9, %rcx
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: orq %rdi, %rcx
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxq %rsi, %rax, %rax
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq -48(%rsp,%rax,4), %rsi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq -56(%rsp,%rax,4), %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %rdi, %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrdq %cl, %rsi, %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq -72(%rsp,%rax,4), %r9
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq -64(%rsp,%rax,4), %rax
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %rax, %r10
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrdq %cl, %rdi, %r10
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrdq %cl, %rax, %r9
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxq %rcx, %rsi, %rax
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %r10, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %r8, 16(%rdx)
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %rax, 24(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %rcx, 8(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %r11, 16(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %r10, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %r9, (%rdx)
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: vzeroupper
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: retq
;
@@ -4938,57 +3830,35 @@ define void @lshr_32bytes_qwordOff(ptr %src.ptr, ptr %qwordOff.ptr, ptr %dst) no
define void @shl_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-NO-BMI2-SSE2-LABEL: shl_32bytes:
; X64-NO-SHLD-NO-BMI2-SSE2: # %bb.0:
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: pushq %rbx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq (%rdi), %rcx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq (%rdi), %rax
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 8(%rdi), %r8
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 16(%rdi), %r9
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 24(%rdi), %rdi
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movzbl (%rsi), %esi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: leal (,%rsi,8), %eax
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: leal (,%rsi,8), %ecx
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: xorps %xmm0, %xmm0
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: andb $24, %sil
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: negb %sil
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movsbq %sil, %r10
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -32(%rsp,%r10), %r8
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -24(%rsp,%r10), %rdi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rdi, %r11
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlq %cl, %r11
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %esi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: notb %sil
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r8, %r9
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %r9
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %cl, %r9
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: orq %r11, %r9
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -8(%rsp,%r10), %r11
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlq %cl, %r11
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -16(%rsp,%r10), %r10
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r10, %rbx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %rbx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %cl, %rbx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: orq %r11, %rbx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %rdi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %cl, %rdi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: orq %r10, %rdi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movsbq %sil, %rax
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -24(%rsp,%rax), %rsi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -16(%rsp,%rax), %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shldq %cl, %rsi, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -40(%rsp,%rax), %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -32(%rsp,%rax), %rax
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shldq %cl, %rax, %rsi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shldq %cl, %r8, %rax
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: # kill: def $cl killed $cl killed $ecx
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlq %cl, %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rsi, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rdi, 24(%rdx)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r8, (%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rdi, 16(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rbx, 24(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r9, 8(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: popq %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rax, 8(%rdx)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: retq
;
; X64-HAVE-SHLD-NO-BMI2-SSE2-LABEL: shl_32bytes:
@@ -5026,44 +3896,34 @@ define void @shl_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
;
; X64-NO-SHLD-HAVE-BMI2-SSE2-LABEL: shl_32bytes:
; X64-NO-SHLD-HAVE-BMI2-SSE2: # %bb.0:
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq (%rdi), %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq (%rdi), %rax
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq 8(%rdi), %r8
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq 16(%rdi), %r9
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq 24(%rdi), %rdi
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movzbl (%rsi), %esi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leal (,%rsi,8), %eax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leal (,%rsi,8), %ecx
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: xorps %xmm0, %xmm0
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, %ecx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: andb $24, %sil
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: negb %sil
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movsbq %sil, %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -40(%rsp,%rdi), %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -32(%rsp,%rdi), %rsi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxq %rcx, %rsi, %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: notb %al
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxq %rcx, %r8, %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrq %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxq %rax, %r8, %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orq %r9, %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxq %rcx, -16(%rsp,%rdi), %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -24(%rsp,%rdi), %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxq %rcx, %rdi, %rcx
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrq %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxq %rax, %rdi, %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orq %r9, %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrq %rsi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxq %rax, %rsi, %rax
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orq %rcx, %rax
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r10, (%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rax, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movsbq %sil, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -24(%rsp,%rax), %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -16(%rsp,%rax), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shldq %cl, %rsi, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -40(%rsp,%rax), %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -32(%rsp,%rax), %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shldq %cl, %rax, %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shldq %cl, %r8, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxq %rcx, %r8, %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rsi, 16(%rdx)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rdi, 24(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r8, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rcx, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rax, 8(%rdx)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: retq
;
; X64-HAVE-SHLD-HAVE-BMI2-SSE2-LABEL: shl_32bytes:
@@ -5102,49 +3962,30 @@ define void @shl_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-NO-BMI2-SSE4: # %bb.0:
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movups (%rdi), %xmm0
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movups 16(%rdi), %xmm1
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movzbl (%rsi), %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: leal (,%rcx,8), %eax
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movzbl (%rsi), %eax
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: leal (,%rax,8), %ecx
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: xorps %xmm2, %xmm2
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: andb $24, %cl
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: negb %cl
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movsbq %cl, %r8
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -16(%rsp,%r8), %r9
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlq %cl, %r9
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %esi
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: notb %sil
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -24(%rsp,%r8), %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r10, %rdi
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %rdi
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %cl, %rdi
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: orq %r9, %rdi
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -40(%rsp,%r8), %r9
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -32(%rsp,%r8), %r8
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r8, %r11
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %r11
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %cl, %r11
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: orq %r10, %r11
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlq %cl, %r8
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r9, %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: orq %r8, %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: andb $24, %al
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: negb %al
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movsbq %al, %rax
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -24(%rsp,%rax), %rsi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -16(%rsp,%rax), %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shldq %cl, %rsi, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -40(%rsp,%rax), %r8
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -32(%rsp,%rax), %rax
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shldq %cl, %rax, %rsi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r8, %r9
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlq %cl, %r9
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r9, (%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r10, 8(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r11, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: # kill: def $cl killed $cl killed $ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shldq %cl, %r8, %rax
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rax, 8(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rsi, 16(%rdx)
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rdi, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r9, (%rdx)
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: retq
;
; X64-HAVE-SHLD-NO-BMI2-SSE4-LABEL: shl_32bytes:
@@ -5181,38 +4022,29 @@ define void @shl_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-HAVE-BMI2-SSE4: # %bb.0:
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movups (%rdi), %xmm0
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movups 16(%rdi), %xmm1
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movzbl (%rsi), %esi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leal (,%rsi,8), %eax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movzbl (%rsi), %eax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leal (,%rax,8), %ecx
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: xorps %xmm2, %xmm2
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: andb $24, %sil
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: negb %sil
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movsbq %sil, %rsi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxq %rcx, -16(%rsp,%rsi), %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: notb %al
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -24(%rsp,%rsi), %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: andb $24, %al
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: negb %al
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movsbq %al, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -24(%rsp,%rax), %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -16(%rsp,%rax), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shldq %cl, %rsi, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -40(%rsp,%rax), %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -32(%rsp,%rax), %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shldq %cl, %rax, %rsi
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxq %rcx, %r8, %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrq %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxq %rax, %r8, %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orq %rdi, %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -40(%rsp,%rsi), %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -32(%rsp,%rsi), %rsi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxq %rcx, %rsi, %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrq %rsi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxq %rax, %rsi, %rsi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orq %r9, %rsi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxq %rcx, %rdi, %rcx
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrq %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxq %rax, %rdi, %rax
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orq %r10, %rax
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rcx, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: # kill: def $cl killed $cl killed $rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shldq %cl, %r8, %rax
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rax, 8(%rdx)
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rsi, 16(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r8, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rdi, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r9, (%rdx)
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: retq
;
; X64-HAVE-SHLD-HAVE-BMI2-SSE4-LABEL: shl_32bytes:
@@ -5247,47 +4079,28 @@ define void @shl_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-NO-BMI2-AVX-LABEL: shl_32bytes:
; X64-NO-SHLD-NO-BMI2-AVX: # %bb.0:
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: vmovups (%rdi), %ymm0
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movzbl (%rsi), %ecx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: leal (,%rcx,8), %eax
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movzbl (%rsi), %eax
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: leal (,%rax,8), %ecx
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: vxorps %xmm1, %xmm1, %xmm1
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: andb $24, %cl
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: negb %cl
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movsbq %cl, %r8
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -16(%rsp,%r8), %r9
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shlq %cl, %r9
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %esi
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: notb %sil
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -24(%rsp,%r8), %r10
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %r10, %rdi
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shrq %rdi
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shrq %cl, %rdi
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: orq %r9, %rdi
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shlq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -40(%rsp,%r8), %r9
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -32(%rsp,%r8), %r8
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %r8, %r11
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shrq %r11
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shrq %cl, %r11
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: orq %r10, %r11
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shlq %cl, %r8
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %r9, %r10
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shrq %r10
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shrq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: orq %r8, %r10
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: andb $24, %al
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: negb %al
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movsbq %al, %rax
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -24(%rsp,%rax), %rsi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -16(%rsp,%rax), %rdi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shldq %cl, %rsi, %rdi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -40(%rsp,%rax), %r8
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -32(%rsp,%rax), %rax
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shldq %cl, %rax, %rsi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %r8, %r9
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shlq %cl, %r9
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %r9, (%rdx)
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %r10, 8(%rdx)
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %r11, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: # kill: def $cl killed $cl killed $ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shldq %cl, %r8, %rax
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %rax, 8(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %rsi, 16(%rdx)
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %rdi, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %r9, (%rdx)
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: vzeroupper
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: retq
;
@@ -5322,36 +4135,27 @@ define void @shl_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-HAVE-BMI2-AVX-LABEL: shl_32bytes:
; X64-NO-SHLD-HAVE-BMI2-AVX: # %bb.0:
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: vmovups (%rdi), %ymm0
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movzbl (%rsi), %esi
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: leal (,%rsi,8), %eax
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movzbl (%rsi), %eax
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: leal (,%rax,8), %ecx
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: vxorps %xmm1, %xmm1, %xmm1
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: andb $24, %sil
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: negb %sil
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movsbq %sil, %rsi
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxq %rcx, -16(%rsp,%rsi), %rdi
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: notb %al
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq -24(%rsp,%rsi), %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: andb $24, %al
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: negb %al
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movsbq %al, %rax
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq -24(%rsp,%rax), %rsi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq -16(%rsp,%rax), %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shldq %cl, %rsi, %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq -40(%rsp,%rax), %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq -32(%rsp,%rax), %rax
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shldq %cl, %rax, %rsi
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxq %rcx, %r8, %r9
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrq %r8
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxq %rax, %r8, %r8
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: orq %rdi, %r8
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq -40(%rsp,%rsi), %rdi
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq -32(%rsp,%rsi), %rsi
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxq %rcx, %rsi, %r10
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrq %rsi
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxq %rax, %rsi, %rsi
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: orq %r9, %rsi
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxq %rcx, %rdi, %rcx
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrq %rdi
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxq %rax, %rdi, %rax
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: orq %r10, %rax
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %rcx, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: # kill: def $cl killed $cl killed $rcx
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shldq %cl, %r8, %rax
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %rax, 8(%rdx)
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %rsi, 16(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %r8, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %rdi, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %r9, (%rdx)
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: vzeroupper
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: retq
;
@@ -5388,24 +4192,24 @@ define void @shl_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: pushl %ebx
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: pushl %edi
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: pushl %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: subl $108, %esp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: subl $92, %esp
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl (%ecx), %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 4(%ecx), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl (%eax), %edx
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 8(%ecx), %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 12(%ecx), %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 16(%ecx), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb (%eax), %ah
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 20(%ecx), %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 24(%ecx), %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 28(%ecx), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 4(%eax), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, (%esp) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 8(%eax), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 12(%eax), %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 16(%eax), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb (%ecx), %ch
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 20(%eax), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 24(%eax), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 28(%eax), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ah, %ch
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shlb $3, %ch
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shlb $3, %cl
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: xorps %xmm0, %xmm0
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
@@ -5413,94 +4217,48 @@ define void @shl_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: andb $28, %ah
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: negb %ah
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movsbl %ah, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 64(%esp,%ebx), %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 68(%esp,%ebx), %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %dl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: notb %dl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %dl, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %esi, %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 76(%esp,%ebx), %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 72(%esp,%ebx), %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %dl, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %edi, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl (%esp), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: andb $28, %ch
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: negb %ch
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movsbl %ch, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 56(%esp,%eax), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 60(%esp,%eax), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shldl %cl, %edx, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 52(%esp,%eax), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, (%esp) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shldl %cl, %esi, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 64(%esp,%eax), %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 68(%esp,%eax), %ebp
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %dl, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %esi, %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 84(%esp,%ebx), %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 80(%esp,%ebx), %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %dl, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %eax, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shldl %cl, %edi, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shldl %cl, %ebx, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 48(%esp,%eax), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 72(%esp,%eax), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 76(%esp,%eax), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shldl %cl, %edx, %esi
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %dl, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %edi, %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 92(%esp,%ebx), %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 88(%esp,%ebx), %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %dl, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %eax, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %dl, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %edi, %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shldl %cl, %eax, %edx
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, (%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, 24(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, 28(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, 16(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, 24(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, 28(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, 16(%eax)
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebp, 20(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, 8(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, 12(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, 4(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: addl $108, %esp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, 8(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, 12(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl (%esp), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shldl %cl, %ebx, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, (%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, 4(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: addl $92, %esp
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: popl %esi
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: popl %edi
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: popl %ebx
@@ -5592,25 +4350,25 @@ define void @shl_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: pushl %ebx
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: pushl %edi
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: pushl %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: subl $108, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: subl $92, %esp
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl (%eax), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 4(%eax), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 8(%eax), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 12(%eax), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 16(%eax), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl (%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 4(%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 8(%ecx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 12(%ecx), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 16(%ecx), %ebp
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movzbl (%ebx), %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 20(%eax), %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 24(%eax), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 28(%eax), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 20(%ecx), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 24(%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 28(%ecx), %ecx
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebx, %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlb $3, %dl
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebx, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlb $3, %cl
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: xorps %xmm0, %xmm0
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
@@ -5621,76 +4379,46 @@ define void @shl_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, %ebp
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: andb $28, %bl
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: negb %bl
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movsbl %bl, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 64(%esp,%esi), %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 68(%esp,%esi), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movsbl %bl, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 56(%esp,%eax), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 60(%esp,%eax), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, (%esp) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shldl %cl, %edx, %esi
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %eax, %ecx, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: notb %dl
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrl %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %edx, %ebx, %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %edi, %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 72(%esp,%esi), %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebx, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrl %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %edx, %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 76(%esp,%esi), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebp, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %ebp, %edi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %esi, %ebx, %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrl %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %edx, %ecx, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %ebx, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 80(%esp,%ebp), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrl %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %edx, %ebx, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 84(%esp,%ebp), %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %esi, %ebx, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %esi, %ecx, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrl %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %edx, %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %ecx, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %esi, {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %esi, 92(%esp,%ecx), %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 88(%esp,%ecx), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %eax, %esi, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrl %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %edx, %esi, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %ebp, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrl %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %edx, %ebx, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %ecx, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, (%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 24(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, 28(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edi, 16(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 20(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 8(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 12(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 52(%esp,%eax), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shldl %cl, %ebx, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 64(%esp,%eax), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 68(%esp,%eax), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shldl %cl, %edi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl (%esp), %edx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shldl %cl, %edx, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 48(%esp,%eax), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, (%esp) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 72(%esp,%eax), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 76(%esp,%eax), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shldl %cl, %edx, %esi
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 4(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addl $108, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shldl %cl, %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, 24(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, 28(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edi, 16(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebp, 20(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, 8(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, 12(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl (%esp), %esi # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %ecx, %esi, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, (%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shldl %cl, %esi, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebx, 4(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addl $92, %esp
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: popl %esi
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: popl %edi
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: popl %ebx
@@ -5784,106 +4512,58 @@ define void @shl_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: pushl %ebx
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: pushl %edi
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: pushl %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: subl $108, %esp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: subl $92, %esp
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movups (%ecx), %xmm0
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movups 16(%ecx), %xmm1
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movzbl (%eax), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %cl, %dh
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shlb $3, %dh
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movzbl (%eax), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shlb $3, %cl
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: xorps %xmm2, %xmm2
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: andb $28, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: negb %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movsbl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 84(%esp,%ebx), %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %dh, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %dh, %dl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: notb %dl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 80(%esp,%ebx), %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %edi, %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %dh, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 76(%esp,%ebx), %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebp, %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %esi, %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %dh, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 72(%esp,%ebx), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %dh, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 68(%esp,%edi), %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebp, %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %ebx, %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %dh, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 64(%esp,%edi), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %ebp, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 88(%esp,%edi), %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebp, %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %dh, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %edi, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: andb $28, %al
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: negb %al
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movsbl %al, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 64(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 68(%esp,%ebp), %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shldl %cl, %eax, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 60(%esp,%ebp), %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shldl %cl, %edx, %eax
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 92(%esp,%eax), %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %dh, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %edi, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %dh, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, (%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebp, 28(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, 24(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, 4(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, 8(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, 12(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 56(%esp,%ebp), %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shldl %cl, %edi, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, (%esp) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 52(%esp,%ebp), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shldl %cl, %ebx, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 72(%esp,%ebp), %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shldl %cl, %esi, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 48(%esp,%ebp), %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 76(%esp,%ebp), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shldl %cl, %edx, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebp, 28(%edx)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 24(%edx)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shldl %cl, %esi, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, 4(%edx)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edi, 8(%edx)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl (%esp), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, 12(%edx)
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, 16(%eax)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, 16(%edx)
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, 20(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: addl $108, %esp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, 20(%edx)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, (%edx)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: addl $92, %esp
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: popl %esi
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: popl %edi
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: popl %ebx
@@ -5960,82 +4640,58 @@ define void @shl_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: pushl %ebx
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: pushl %edi
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: pushl %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: subl $108, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: subl $92, %esp
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movups (%ecx), %xmm0
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movups 16(%ecx), %xmm1
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movzbl (%eax), %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movzbl (%eax), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, %ecx
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlb $3, %cl
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: xorps %xmm2, %xmm2
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: andb $28, %dl
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: negb %dl
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movsbl %dl, %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 84(%esp,%edx), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %ebx, %eax, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: notb %cl
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 80(%esp,%edx), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %ebx, %edi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrl %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ecx, %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %esi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 76(%esp,%edx), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %esi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrl %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ecx, %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %ebp, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %ebx, %esi, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 72(%esp,%edx), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrl %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ecx, %ebp, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %esi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %ebx, %edi, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 68(%esp,%edx), %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ebp, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrl %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ecx, %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %esi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %ebx, %ebp, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 64(%esp,%edx), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrl %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ecx, %esi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %ebp, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrl %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ecx, %eax, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 88(%esp,%edx), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %ebx, %eax, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %ebp, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %ebx, 92(%esp,%edx), %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrl %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ecx, %eax, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %edx, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ebp, (%ecx)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 28(%ecx)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %esi, 24(%ecx)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edi, 4(%ecx)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 8(%ecx)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 12(%ecx)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 16(%ecx)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 20(%ecx)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: addl $108, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: andb $28, %al
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: negb %al
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movsbl %al, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 64(%esp,%ebx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 68(%esp,%ebx), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shldl %cl, %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 60(%esp,%ebx), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shldl %cl, %edx, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 56(%esp,%ebx), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shldl %cl, %edi, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, (%esp) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 52(%esp,%ebx), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shldl %cl, %ebp, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 72(%esp,%ebx), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shldl %cl, %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 48(%esp,%ebx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 76(%esp,%ebx), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shldl %cl, %edx, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ebx, 28(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 24(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %ecx, %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shldl %cl, %esi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ebp, 4(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edi, 8(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl (%esp), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, 12(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, 16(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, 20(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, (%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: addl $92, %esp
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: popl %esi
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: popl %edi
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: popl %ebx
@@ -6112,103 +4768,55 @@ define void @shl_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: pushl %ebx
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: pushl %edi
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: pushl %esi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: subl $108, %esp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: subl $92, %esp
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: vmovups (%ecx), %ymm0
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movzbl (%eax), %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movb %cl, %dh
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shlb $3, %dh
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movzbl (%eax), %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shlb $3, %cl
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: vxorps %xmm1, %xmm1, %xmm1
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: vmovups %ymm1, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: vmovups %ymm0, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: andb $28, %cl
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: negb %cl
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movsbl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 84(%esp,%ebx), %edi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movb %dh, %cl
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shll %cl, %edi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movb %dh, %dl
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: notb %dl
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 80(%esp,%ebx), %esi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %esi, %eax
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrl %eax
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrl %cl, %eax
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: orl %edi, %eax
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movb %dh, %cl
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shll %cl, %esi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ebx, %edi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 76(%esp,%ebx), %ebp
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ebp, %eax
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrl %eax
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrl %cl, %eax
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: orl %esi, %eax
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movb %dh, %cl
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shll %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 72(%esp,%ebx), %ebx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ebx, %eax
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrl %eax
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrl %cl, %eax
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movb %dh, %cl
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 68(%esp,%edi), %ebp
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ebp, %esi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrl %esi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrl %cl, %esi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: orl %ebx, %esi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movb %dh, %cl
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shll %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 64(%esp,%edi), %ebx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrl %ebx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: orl %ebp, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 88(%esp,%edi), %ebp
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ebp, %edi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movb %dh, %cl
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shll %cl, %edi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrl %eax
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrl %cl, %eax
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: orl %edi, %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: andb $28, %al
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: negb %al
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movsbl %al, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 64(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 68(%esp,%ebp), %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shldl %cl, %eax, %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 60(%esp,%ebp), %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shldl %cl, %edx, %eax
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 92(%esp,%eax), %edi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movb %dh, %cl
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shll %cl, %edi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrl %ebp
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrl %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: orl %edi, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movb %dh, %cl
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shll %cl, %edx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, (%eax)
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ebp, 28(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ecx, 24(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ebx, 4(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %esi, 8(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ecx, 12(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 56(%esp,%ebp), %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shldl %cl, %edi, %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, (%esp) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 52(%esp,%ebp), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shldl %cl, %ebx, %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 72(%esp,%ebp), %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shldl %cl, %esi, %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 48(%esp,%ebp), %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 76(%esp,%ebp), %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shldl %cl, %edx, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ebp, 28(%edx)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, 24(%edx)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %esi, %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shll %cl, %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shldl %cl, %esi, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ebx, 4(%edx)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edi, 8(%edx)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl (%esp), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ecx, 12(%edx)
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ecx, 16(%eax)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ecx, 16(%edx)
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ecx, 20(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: addl $108, %esp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ecx, 20(%edx)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, (%edx)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: addl $92, %esp
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: popl %esi
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: popl %edi
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: popl %ebx
@@ -6284,79 +4892,55 @@ define void @shl_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: pushl %ebx
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: pushl %edi
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: pushl %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: subl $108, %esp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: subl $92, %esp
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: vmovups (%ecx), %ymm0
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movzbl (%eax), %edx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edx, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlb $3, %al
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movzbl (%eax), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlb $3, %cl
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: vxorps %xmm1, %xmm1, %xmm1
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: vmovups %ymm1, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: vmovups %ymm0, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, %ebx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: andb $28, %dl
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: negb %dl
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movsbl %dl, %edx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 84(%esp,%edx), %ecx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxl %ebx, %ecx, %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: notb %al
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 80(%esp,%edx), %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxl %ebx, %edi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrl %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxl %eax, %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: orl %esi, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 76(%esp,%edx), %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %esi, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrl %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxl %eax, %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: orl %ebp, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxl %ebx, %esi, %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 72(%esp,%edx), %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrl %ebp
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxl %eax, %ebp, %ebp
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: orl %esi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxl %ebx, %edi, %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 68(%esp,%edx), %ebp
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %ebp, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrl %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxl %eax, %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: orl %esi, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxl %ebx, %ebp, %ebp
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 64(%esp,%edx), %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrl %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxl %eax, %esi, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: orl %ebp, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrl %ecx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxl %eax, %ecx, %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 88(%esp,%edx), %ecx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxl %ebx, %ecx, %ebp
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: orl %ebp, %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxl %ebx, 92(%esp,%edx), %edx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrl %ecx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxl %eax, %ecx, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: orl %edx, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %ebp, (%ecx)
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, 28(%ecx)
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %esi, 24(%ecx)
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edi, 4(%ecx)
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, 8(%ecx)
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, 12(%ecx)
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, 16(%ecx)
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, 20(%ecx)
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: addl $108, %esp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: andb $28, %al
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: negb %al
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movsbl %al, %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 64(%esp,%ebx), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 68(%esp,%ebx), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shldl %cl, %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 60(%esp,%ebx), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shldl %cl, %edx, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 56(%esp,%ebx), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shldl %cl, %edi, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edx, (%esp) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 52(%esp,%ebx), %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shldl %cl, %ebp, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 72(%esp,%ebx), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edx, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shldl %cl, %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 48(%esp,%ebx), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 76(%esp,%ebx), %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shldl %cl, %edx, %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %ebx, 28(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, 24(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxl %ecx, %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shldl %cl, %esi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %ebp, 4(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edi, 8(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl (%esp), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %ecx, 12(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %ecx, 16(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %ecx, 20(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, (%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: addl $92, %esp
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: popl %esi
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: popl %edi
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: popl %ebx
@@ -6436,59 +5020,36 @@ define void @shl_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
define void @shl_32bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-NO-BMI2-SSE2-LABEL: shl_32bytes_dwordOff:
; X64-NO-SHLD-NO-BMI2-SSE2: # %bb.0:
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: pushq %rbx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq (%rdi), %rcx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq (%rdi), %rax
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 8(%rdi), %r8
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 16(%rdi), %r9
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 24(%rdi), %rdi
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movzbl (%rsi), %esi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %eax
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlb $5, %al
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlb $5, %cl
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: xorps %xmm0, %xmm0
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlb $2, %sil
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: andb $24, %sil
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: negb %sil
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movsbq %sil, %r10
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -32(%rsp,%r10), %r8
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -24(%rsp,%r10), %rdi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rdi, %r11
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlq %cl, %r11
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %esi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: notb %sil
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r8, %r9
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %r9
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %cl, %r9
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: orq %r11, %r9
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -8(%rsp,%r10), %r11
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlq %cl, %r11
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -16(%rsp,%r10), %r10
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r10, %rbx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %rbx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %cl, %rbx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: orq %r11, %rbx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %rdi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %cl, %rdi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: orq %r10, %rdi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movsbq %sil, %rax
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -24(%rsp,%rax), %rsi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -16(%rsp,%rax), %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shldq %cl, %rsi, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -40(%rsp,%rax), %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -32(%rsp,%rax), %rax
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shldq %cl, %rax, %rsi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shldq %cl, %r8, %rax
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlq %cl, %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rsi, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rdi, 24(%rdx)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r8, (%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rdi, 16(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rbx, 24(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r9, 8(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: popq %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rax, 8(%rdx)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: retq
;
; X64-HAVE-SHLD-NO-BMI2-SSE2-LABEL: shl_32bytes_dwordOff:
@@ -6527,46 +5088,36 @@ define void @shl_32bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) nou
;
; X64-NO-SHLD-HAVE-BMI2-SSE2-LABEL: shl_32bytes_dwordOff:
; X64-NO-SHLD-HAVE-BMI2-SSE2: # %bb.0:
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq (%rdi), %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq (%rdi), %rax
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq 8(%rdi), %r8
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq 16(%rdi), %r9
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq 24(%rdi), %rdi
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movzbl (%rsi), %esi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, %eax
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlb $5, %al
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, %ecx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlb $5, %cl
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: xorps %xmm0, %xmm0
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, %ecx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlb $2, %sil
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: andb $24, %sil
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: negb %sil
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movsbq %sil, %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -40(%rsp,%rdi), %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -32(%rsp,%rdi), %rsi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxq %rcx, %rsi, %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: notb %al
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxq %rcx, %r8, %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrq %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxq %rax, %r8, %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orq %r9, %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxq %rcx, -16(%rsp,%rdi), %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -24(%rsp,%rdi), %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxq %rcx, %rdi, %rcx
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrq %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxq %rax, %rdi, %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orq %r9, %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrq %rsi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxq %rax, %rsi, %rax
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orq %rcx, %rax
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r10, (%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rax, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movsbq %sil, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -24(%rsp,%rax), %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -16(%rsp,%rax), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shldq %cl, %rsi, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -40(%rsp,%rax), %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -32(%rsp,%rax), %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shldq %cl, %rax, %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shldq %cl, %r8, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxq %rcx, %r8, %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rsi, 16(%rdx)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rdi, 24(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r8, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rcx, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rax, 8(%rdx)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: retq
;
; X64-HAVE-SHLD-HAVE-BMI2-SSE2-LABEL: shl_32bytes_dwordOff:
@@ -6607,51 +5158,31 @@ define void @shl_32bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) nou
; X64-NO-SHLD-NO-BMI2-SSE4: # %bb.0:
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movups (%rdi), %xmm0
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movups 16(%rdi), %xmm1
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movzbl (%rsi), %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, %eax
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlb $5, %al
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movzbl (%rsi), %eax
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlb $5, %cl
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: xorps %xmm2, %xmm2
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlb $2, %cl
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: andb $24, %cl
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: negb %cl
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movsbq %cl, %r8
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -16(%rsp,%r8), %r9
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlq %cl, %r9
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %esi
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: notb %sil
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -24(%rsp,%r8), %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r10, %rdi
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %rdi
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %cl, %rdi
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: orq %r9, %rdi
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -40(%rsp,%r8), %r9
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -32(%rsp,%r8), %r8
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r8, %r11
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %r11
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %cl, %r11
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: orq %r10, %r11
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlq %cl, %r8
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r9, %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: orq %r8, %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlb $2, %al
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: andb $24, %al
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: negb %al
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movsbq %al, %rax
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -24(%rsp,%rax), %rsi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -16(%rsp,%rax), %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shldq %cl, %rsi, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -40(%rsp,%rax), %r8
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -32(%rsp,%rax), %rax
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shldq %cl, %rax, %rsi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r8, %r9
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlq %cl, %r9
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r9, (%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r10, 8(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r11, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shldq %cl, %r8, %rax
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rax, 8(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rsi, 16(%rdx)
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rdi, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r9, (%rdx)
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: retq
;
; X64-HAVE-SHLD-NO-BMI2-SSE4-LABEL: shl_32bytes_dwordOff:
@@ -6689,40 +5220,31 @@ define void @shl_32bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) nou
; X64-NO-SHLD-HAVE-BMI2-SSE4: # %bb.0:
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movups (%rdi), %xmm0
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movups 16(%rdi), %xmm1
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movzbl (%rsi), %esi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %esi, %eax
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlb $5, %al
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movzbl (%rsi), %eax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, %ecx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlb $5, %cl
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: xorps %xmm2, %xmm2
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlb $2, %sil
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: andb $24, %sil
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: negb %sil
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movsbq %sil, %rsi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxq %rcx, -16(%rsp,%rsi), %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: notb %al
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -24(%rsp,%rsi), %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlb $2, %al
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: andb $24, %al
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: negb %al
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movsbq %al, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -24(%rsp,%rax), %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -16(%rsp,%rax), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shldq %cl, %rsi, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -40(%rsp,%rax), %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -32(%rsp,%rax), %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shldq %cl, %rax, %rsi
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxq %rcx, %r8, %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrq %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxq %rax, %r8, %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orq %rdi, %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -40(%rsp,%rsi), %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -32(%rsp,%rsi), %rsi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxq %rcx, %rsi, %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrq %rsi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxq %rax, %rsi, %rsi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orq %r9, %rsi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxq %rcx, %rdi, %rcx
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrq %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxq %rax, %rdi, %rax
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orq %r10, %rax
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rcx, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: # kill: def $cl killed $cl killed $rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shldq %cl, %r8, %rax
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rax, 8(%rdx)
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rsi, 16(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r8, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rdi, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r9, (%rdx)
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: retq
;
; X64-HAVE-SHLD-HAVE-BMI2-SSE4-LABEL: shl_32bytes_dwordOff:
@@ -6759,49 +5281,29 @@ define void @shl_32bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) nou
; X64-NO-SHLD-NO-BMI2-AVX-LABEL: shl_32bytes_dwordOff:
; X64-NO-SHLD-NO-BMI2-AVX: # %bb.0:
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: vmovups (%rdi), %ymm0
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movzbl (%rsi), %ecx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ecx, %eax
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shlb $5, %al
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movzbl (%rsi), %eax
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shlb $5, %cl
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: vxorps %xmm1, %xmm1, %xmm1
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shlb $2, %cl
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: andb $24, %cl
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: negb %cl
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movsbq %cl, %r8
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -16(%rsp,%r8), %r9
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shlq %cl, %r9
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %esi
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: notb %sil
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -24(%rsp,%r8), %r10
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %r10, %rdi
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shrq %rdi
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shrq %cl, %rdi
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: orq %r9, %rdi
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shlq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -40(%rsp,%r8), %r9
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -32(%rsp,%r8), %r8
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %r8, %r11
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shrq %r11
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shrq %cl, %r11
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: orq %r10, %r11
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shlq %cl, %r8
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %r9, %r10
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shrq %r10
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shrq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: orq %r8, %r10
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shlb $2, %al
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: andb $24, %al
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: negb %al
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movsbq %al, %rax
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -24(%rsp,%rax), %rsi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -16(%rsp,%rax), %rdi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shldq %cl, %rsi, %rdi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -40(%rsp,%rax), %r8
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -32(%rsp,%rax), %rax
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shldq %cl, %rax, %rsi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %r8, %r9
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shlq %cl, %r9
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %r9, (%rdx)
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %r10, 8(%rdx)
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %r11, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shldq %cl, %r8, %rax
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %rax, 8(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %rsi, 16(%rdx)
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %rdi, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %r9, (%rdx)
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: vzeroupper
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: retq
;
@@ -6837,38 +5339,29 @@ define void @shl_32bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) nou
; X64-NO-SHLD-HAVE-BMI2-AVX-LABEL: shl_32bytes_dwordOff:
; X64-NO-SHLD-HAVE-BMI2-AVX: # %bb.0:
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: vmovups (%rdi), %ymm0
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movzbl (%rsi), %esi
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %esi, %eax
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlb $5, %al
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movzbl (%rsi), %eax
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, %ecx
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlb $5, %cl
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: vxorps %xmm1, %xmm1, %xmm1
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlb $2, %sil
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: andb $24, %sil
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: negb %sil
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movsbq %sil, %rsi
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxq %rcx, -16(%rsp,%rsi), %rdi
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: notb %al
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq -24(%rsp,%rsi), %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlb $2, %al
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: andb $24, %al
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: negb %al
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movsbq %al, %rax
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq -24(%rsp,%rax), %rsi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq -16(%rsp,%rax), %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shldq %cl, %rsi, %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq -40(%rsp,%rax), %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq -32(%rsp,%rax), %rax
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shldq %cl, %rax, %rsi
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxq %rcx, %r8, %r9
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrq %r8
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxq %rax, %r8, %r8
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: orq %rdi, %r8
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq -40(%rsp,%rsi), %rdi
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq -32(%rsp,%rsi), %rsi
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxq %rcx, %rsi, %r10
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrq %rsi
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxq %rax, %rsi, %rsi
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: orq %r9, %rsi
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxq %rcx, %rdi, %rcx
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrq %rdi
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxq %rax, %rdi, %rax
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: orq %r10, %rax
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %rcx, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: # kill: def $cl killed $cl killed $rcx
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shldq %cl, %r8, %rax
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %rax, 8(%rdx)
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %rsi, 16(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %r8, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %rdi, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %r9, (%rdx)
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: vzeroupper
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: retq
;
@@ -7211,56 +5704,37 @@ define void @shl_32bytes_qwordOff(ptr %src.ptr, ptr %qwordOff.ptr, ptr %dst) nou
define void @ashr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-NO-BMI2-SSE2-LABEL: ashr_32bytes:
; X64-NO-SHLD-NO-BMI2-SSE2: # %bb.0:
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: pushq %rbx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq (%rdi), %rcx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq (%rdi), %rax
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 8(%rdi), %r8
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 16(%rdi), %r9
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 24(%rdi), %rdi
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movzbl (%rsi), %esi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: leal (,%rsi,8), %eax
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: leal (,%rsi,8), %ecx
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: sarq $63, %rdi
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: andb $24, %sil
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movzbl %sil, %r9d
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -64(%rsp,%r9), %r10
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -56(%rsp,%r9), %rdi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rdi, %r11
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %cl, %r11
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %esi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: notb %sil
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -48(%rsp,%r9), %rbx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: leaq (%rbx,%rbx), %r8
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlq %cl, %r8
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: orq %r11, %r8
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: addq %rdi, %rdi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlq %cl, %rdi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: orq %r10, %rdi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %cl, %rbx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -40(%rsp,%r9), %r9
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: leaq (%r9,%r9), %r10
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: orq %rbx, %r10
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: sarq %cl, %r9
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r9, 24(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r10, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movzbl %sil, %eax
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -56(%rsp,%rax), %rsi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -72(%rsp,%rax), %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -64(%rsp,%rax), %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r8, %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdq %cl, %rsi, %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -48(%rsp,%rax), %rax
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdq %cl, %rax, %rsi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdq %cl, %r8, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: # kill: def $cl killed $cl killed $ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: sarq %cl, %rax
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rsi, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rax, 24(%rdx)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rdi, (%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r8, 8(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: popq %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r9, 8(%rdx)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: retq
;
; X64-HAVE-SHLD-NO-BMI2-SSE2-LABEL: ashr_32bytes:
@@ -7300,45 +5774,36 @@ define void @ashr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
;
; X64-NO-SHLD-HAVE-BMI2-SSE2-LABEL: ashr_32bytes:
; X64-NO-SHLD-HAVE-BMI2-SSE2: # %bb.0:
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq (%rdi), %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq (%rdi), %rax
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq 8(%rdi), %r8
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq 16(%rdi), %r9
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq 24(%rdi), %rdi
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movzbl (%rsi), %esi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leal (,%rsi,8), %eax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leal (,%rsi,8), %ecx
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: sarq $63, %rdi
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, %ecx
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: andb $24, %sil
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movzbl %sil, %esi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -64(%rsp,%rsi), %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -56(%rsp,%rsi), %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxq %rcx, %rdi, %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: notb %al
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leaq (%r8,%r8), %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxq %rax, %r10, %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orq %r9, %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxq %rcx, -72(%rsp,%rsi), %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addq %rdi, %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxq %rax, %rdi, %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orq %r9, %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxq %rcx, %r8, %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -48(%rsp,%rsi), %rsi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leaq (%rsi,%rsi), %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxq %rax, %r9, %rax
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orq %r8, %rax
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: sarxq %rcx, %rsi, %rcx
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rcx, 24(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rax, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movzbl %sil, %eax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -56(%rsp,%rax), %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -72(%rsp,%rax), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -64(%rsp,%rax), %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r8, %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdq %cl, %rsi, %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -48(%rsp,%rax), %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdq %cl, %rax, %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdq %cl, %r8, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: sarxq %rcx, %rax, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rsi, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rax, 24(%rdx)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rdi, (%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r10, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r9, 8(%rdx)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: retq
;
; X64-HAVE-SHLD-HAVE-BMI2-SSE2-LABEL: ashr_32bytes:
@@ -7377,14 +5842,13 @@ define void @ashr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
;
; X64-NO-SHLD-NO-BMI2-SSE4-LABEL: ashr_32bytes:
; X64-NO-SHLD-NO-BMI2-SSE4: # %bb.0:
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: pushq %rbx
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movups (%rdi), %xmm0
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq 16(%rdi), %rcx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq 16(%rdi), %rax
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq 24(%rdi), %rdi
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movzbl (%rsi), %esi
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: leal (,%rsi,8), %eax
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: leal (,%rsi,8), %ecx
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: sarq $63, %rdi
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
@@ -7392,39 +5856,22 @@ define void @ashr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: andb $24, %sil
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movzbl %sil, %r9d
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -64(%rsp,%r9), %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -56(%rsp,%r9), %r8
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %esi
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: notb %sil
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: leaq (%r8,%r8), %rdi
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlq %cl, %rdi
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: orq %r10, %rdi
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -48(%rsp,%r9), %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r10, %r11
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %cl, %r11
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -40(%rsp,%r9), %r9
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: leaq (%r9,%r9), %rbx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlq %cl, %rbx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: orq %r11, %rbx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %cl, %r8
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: addq %r10, %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: orq %r8, %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: sarq %cl, %r9
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r9, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movzbl %sil, %eax
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -48(%rsp,%rax), %rsi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -56(%rsp,%rax), %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rdi, %r8
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdq %cl, %rsi, %r8
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -72(%rsp,%rax), %r9
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -64(%rsp,%rax), %rax
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rax, %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdq %cl, %rdi, %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdq %cl, %rax, %r9
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: # kill: def $cl killed $cl killed $ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: sarq %cl, %rsi
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r10, 8(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rbx, 16(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rdi, (%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: popq %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r8, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rsi, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r9, (%rdx)
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: retq
;
; X64-HAVE-SHLD-NO-BMI2-SSE4-LABEL: ashr_32bytes:
@@ -7464,42 +5911,34 @@ define void @ashr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-HAVE-BMI2-SSE4-LABEL: ashr_32bytes:
; X64-NO-SHLD-HAVE-BMI2-SSE4: # %bb.0:
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movups (%rdi), %xmm0
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq 16(%rdi), %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq 16(%rdi), %rax
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq 24(%rdi), %rdi
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movzbl (%rsi), %esi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leal (,%rsi,8), %eax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leal (,%rsi,8), %ecx
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: sarq $63, %rdi
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, %ecx
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: andb $24, %sil
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movzbl %sil, %esi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxq %rcx, -72(%rsp,%rsi), %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: notb %al
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -64(%rsp,%rsi), %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -56(%rsp,%rsi), %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leaq (%r8,%r8), %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxq %rax, %r10, %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orq %rdi, %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxq %rcx, %r9, %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -48(%rsp,%rsi), %rsi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leaq (%rsi,%rsi), %r11
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxq %rax, %r11, %r11
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orq %rdi, %r11
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxq %rcx, %r8, %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: addq %r9, %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxq %rax, %r9, %rax
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orq %rdi, %rax
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: sarxq %rcx, %rsi, %rcx
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rcx, 24(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rax, 8(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r11, 16(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r10, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movzbl %sil, %eax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -48(%rsp,%rax), %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -56(%rsp,%rax), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rdi, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdq %cl, %rsi, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -72(%rsp,%rax), %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -64(%rsp,%rax), %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rax, %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdq %cl, %rdi, %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdq %cl, %rax, %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: sarxq %rcx, %rsi, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r10, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r8, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rax, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r9, (%rdx)
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: retq
;
; X64-HAVE-SHLD-HAVE-BMI2-SSE4-LABEL: ashr_32bytes:
@@ -7537,14 +5976,13 @@ define void @ashr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
;
; X64-NO-SHLD-NO-BMI2-AVX-LABEL: ashr_32bytes:
; X64-NO-SHLD-NO-BMI2-AVX: # %bb.0:
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: pushq %rbx
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: vmovups (%rdi), %xmm0
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq 16(%rdi), %rcx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq 16(%rdi), %rax
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq 24(%rdi), %rdi
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movzbl (%rsi), %esi
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: leal (,%rsi,8), %eax
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: leal (,%rsi,8), %ecx
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: sarq $63, %rdi
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
@@ -7552,39 +5990,22 @@ define void @ashr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: andb $24, %sil
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movzbl %sil, %r9d
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -64(%rsp,%r9), %r10
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -56(%rsp,%r9), %r8
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shrq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %esi
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: notb %sil
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: leaq (%r8,%r8), %rdi
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shlq %cl, %rdi
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: orq %r10, %rdi
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -48(%rsp,%r9), %r10
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %r10, %r11
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shrq %cl, %r11
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -40(%rsp,%r9), %r9
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: leaq (%r9,%r9), %rbx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shlq %cl, %rbx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: orq %r11, %rbx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shrq %cl, %r8
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: addq %r10, %r10
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shlq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: orq %r8, %r10
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: sarq %cl, %r9
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %r9, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movzbl %sil, %eax
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -48(%rsp,%rax), %rsi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -56(%rsp,%rax), %rdi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %rdi, %r8
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shrdq %cl, %rsi, %r8
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -72(%rsp,%rax), %r9
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -64(%rsp,%rax), %rax
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %rax, %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shrdq %cl, %rdi, %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shrdq %cl, %rax, %r9
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: # kill: def $cl killed $cl killed $ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: sarq %cl, %rsi
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %r10, 8(%rdx)
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %rbx, 16(%rdx)
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %rdi, (%rdx)
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: popq %rbx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %r8, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %rsi, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %r9, (%rdx)
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: retq
;
; X64-HAVE-SHLD-NO-BMI2-AVX-LABEL: ashr_32bytes:
@@ -7624,42 +6045,34 @@ define void @ashr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-HAVE-BMI2-AVX-LABEL: ashr_32bytes:
; X64-NO-SHLD-HAVE-BMI2-AVX: # %bb.0:
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: vmovups (%rdi), %xmm0
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq 16(%rdi), %rcx
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq 16(%rdi), %rax
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq 24(%rdi), %rdi
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movzbl (%rsi), %esi
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: leal (,%rsi,8), %eax
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: leal (,%rsi,8), %ecx
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: sarq $63, %rdi
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, %ecx
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: andb $24, %sil
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movzbl %sil, %esi
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxq %rcx, -72(%rsp,%rsi), %rdi
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: notb %al
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq -64(%rsp,%rsi), %r8
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq -56(%rsp,%rsi), %r9
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: leaq (%r8,%r8), %r10
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxq %rax, %r10, %r10
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: orq %rdi, %r10
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxq %rcx, %r9, %rdi
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq -48(%rsp,%rsi), %rsi
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: leaq (%rsi,%rsi), %r11
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxq %rax, %r11, %r11
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: orq %rdi, %r11
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxq %rcx, %r8, %rdi
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: addq %r9, %r9
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxq %rax, %r9, %rax
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: orq %rdi, %rax
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: sarxq %rcx, %rsi, %rcx
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %rcx, 24(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %rax, 8(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %r11, 16(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %r10, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movzbl %sil, %eax
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq -48(%rsp,%rax), %rsi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq -56(%rsp,%rax), %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %rdi, %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrdq %cl, %rsi, %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq -72(%rsp,%rax), %r9
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq -64(%rsp,%rax), %rax
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %rax, %r10
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrdq %cl, %rdi, %r10
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrdq %cl, %rax, %r9
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: sarxq %rcx, %rsi, %rax
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %r10, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %r8, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %rax, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %r9, (%rdx)
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: retq
;
; X64-HAVE-SHLD-HAVE-BMI2-AVX-LABEL: ashr_32bytes:
@@ -7701,32 +6114,34 @@ define void @ashr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: pushl %ebx
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: pushl %edi
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: pushl %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: subl $108, %esp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: subl $92, %esp
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl (%esi), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 4(%esi), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 8(%esi), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 12(%esi), %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 16(%esi), %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movzbl (%eax), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 20(%esi), %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 24(%esi), %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 28(%esi), %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl (%ecx), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 4(%ecx), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 8(%ecx), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, (%esp) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 12(%ecx), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 16(%ecx), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movzbl (%eax), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 20(%ecx), %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 24(%ecx), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 28(%ecx), %esi
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shlb $3, %dl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebp, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shlb $3, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebp, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl (%esp), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: sarl $31, %esi
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, {{[0-9]+}}(%esp)
@@ -7736,90 +6151,44 @@ define void @ashr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: andb $28, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movzbl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 32(%esp,%edi), %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 36(%esp,%edi), %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %dl, %ch
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: notb %ch
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 40(%esp,%edi), %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: leal (%edi,%edi), %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %ebx, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %dl, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: addl %eax, %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %esi, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: andb $28, %al
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movzbl %al, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 24(%esp,%ebp), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 20(%esp,%ebp), %eax
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 44(%esp,%eax), %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebp, %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %dl, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 48(%esp,%eax), %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: leal (%edx,%edx), %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %esi, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %edx, %eax
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %bl, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: addl %ebp, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %edi, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 52(%esp,%esi), %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %bl, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 56(%esp,%esi), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: leal (%ebx,%ebx), %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %eax, %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %dl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %dl, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: addl %edi, %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %eax, %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %dl, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 60(%esp,%eax), %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: leal (%eax,%eax), %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %ebx, %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 32(%esp,%ebp), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 28(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %ebx, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, (%esp) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %eax, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 40(%esp,%ebp), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 36(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %edx, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %eax, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 16(%esp,%ebp), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 44(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %eax, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, 24(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %edx, %esi
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: sarl %cl, %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, 28(%ecx)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, 24(%ecx)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, 16(%ecx)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, 20(%ecx)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebp, 8(%ecx)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, 28(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, 16(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, 20(%ebp)
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, 12(%ecx)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, 8(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl (%esp), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, 12(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, (%ebp)
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, (%ecx)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, 4(%ecx)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: addl $108, %esp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, 4(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: addl $92, %esp
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: popl %esi
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: popl %edi
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: popl %ebx
@@ -7919,32 +6288,34 @@ define void @ashr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: pushl %ebx
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: pushl %edi
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: pushl %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: subl $108, %esp
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl (%esi), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 4(%esi), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 8(%esi), %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 12(%esi), %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 16(%esi), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movzbl (%edx), %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 20(%esi), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 24(%esi), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 28(%esi), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: subl $92, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl (%ecx), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 4(%ecx), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 8(%ecx), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, (%esp) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 12(%ecx), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 16(%ecx), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movzbl (%eax), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 20(%ecx), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 24(%ecx), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 28(%ecx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, %ecx
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlb $3, %cl
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebp, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl (%esp), %edx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: sarl $31, %esi
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, {{[0-9]+}}(%esp)
@@ -7954,66 +6325,45 @@ define void @ashr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: andb $28, %dl
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movzbl %dl, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 36(%esp,%esi), %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 40(%esp,%esi), %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %eax, %edx, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: notb %cl
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leal (%ebp,%ebp), %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %ecx, %ebx, %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %edi, %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %eax, 32(%esp,%esi), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addl %edx, %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %ecx, %edx, %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %edi, %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 48(%esp,%esi), %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addl %edx, %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %ecx, %edx, %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 44(%esp,%esi), %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %eax, %edx, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %edi, %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %eax, %ebp, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addl %edx, %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %ecx, %edx, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: andb $28, %al
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movzbl %al, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 24(%esp,%ebp), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 20(%esp,%ebp), %eax
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 56(%esp,%esi), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leal (%edi,%edi), %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %ecx, %edx, %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 52(%esp,%esi), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ebp, %eax, %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %ebx, %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ebp, {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addl %eax, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %ecx, %eax, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %ebx, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 60(%esp,%esi), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leal (%esi,%esi), %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %ecx, %ebx, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ebp, %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %edi, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: sarxl %ebp, %esi, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, 28(%edi)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, 24(%edi)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 16(%edi)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, 20(%edi)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 8(%edi)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 32(%esp,%ebp), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 28(%esp,%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %ebx, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, (%esp) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %eax, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 40(%esp,%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 36(%esp,%ebp), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %eax, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %edx, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 16(%esp,%ebp), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 44(%esp,%ebp), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 24(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: sarxl %ecx, %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 28(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebx, 16(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, 20(%ebp)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 12(%edi)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 8(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl (%esp), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 12(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: # kill: def $cl killed $cl killed $ecx
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, (%edi)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, (%ebp)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 4(%edi)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addl $108, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 4(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addl $92, %esp
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: popl %esi
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: popl %edi
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: popl %ebx
@@ -8140,87 +6490,45 @@ define void @ashr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: andb $28, %al
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movzbl %al, %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 32(%esp,%edi), %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 36(%esp,%edi), %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %cl, %dh
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: notb %dl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: addl %esi, %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %eax, %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 44(%esp,%edi), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %dh, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 48(%esp,%edi), %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: addl %esi, %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %eax, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movzbl %al, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 48(%esp,%ebp), %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 44(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %esi, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 40(%esp,%ebp), %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %eax, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 56(%esp,%ebp), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 52(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %ebx, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %eax, %esi
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 40(%esp,%edi), %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %dh, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: addl %ebx, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %eax, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 52(%esp,%edi), %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebp, %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %dh, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 56(%esp,%edi), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: leal (%ecx,%ecx), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %eax, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %dh, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: addl %ebp, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %eax, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %dh, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 60(%esp,%edi), %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: leal (%eax,%eax), %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %ebx, %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %dh, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: addl %esi, %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %ebx, %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %dh, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 60(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %eax, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 32(%esp,%ebp), %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 36(%esp,%ebp), %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edi, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %ebp, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, 4(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, 24(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %edi, %edx
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: sarl %cl, %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 28(%ecx)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, 4(%ecx)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edi, 24(%ecx)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebp, 16(%ecx)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 28(%ebp)
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 20(%ecx)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 16(%ebp)
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 8(%ecx)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 20(%ebp)
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 12(%ecx)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 8(%ebp)
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, (%ecx)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 12(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, (%ebp)
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: addl $108, %esp
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: popl %esi
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: popl %edi
@@ -8318,88 +6626,64 @@ define void @ashr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movups (%ecx), %xmm0
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 16(%ecx), %esi
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 20(%ecx), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 24(%ecx), %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 28(%ecx), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movzbl (%eax), %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlb $3, %bl
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ebp, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 24(%ecx), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 28(%ecx), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movzbl (%eax), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlb $3, %cl
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ebx, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %esi, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: sarl $31, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ebx, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: andb $28, %dl
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movzbl %dl, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %eax, 32(%esp,%ecx), %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: notb %bl
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 36(%esp,%ecx), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leal (%eax,%eax), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %ebx, %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %edx, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 48(%esp,%ecx), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leal (%eax,%eax), %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %ebx, %edx, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 44(%esp,%ecx), %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ebp, %edx, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %esi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: addl %edx, %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %ebx, %edx, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 40(%esp,%ecx), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: sarl $31, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: andb $28, %al
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movzbl %al, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 48(%esp,%ebx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 44(%esp,%ebx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %esi, %edx
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ebp, %edx, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ebp, %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %esi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 56(%esp,%ecx), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leal (%esi,%esi), %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %ebx, %ebp, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 52(%esp,%ecx), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %edx, %eax, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %edi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %edx, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: addl %eax, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %ebx, %eax, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %edx, %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 60(%esp,%ecx), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leal (%ecx,%ecx), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %ebx, %esi, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %eax, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: addl %eax, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %ebx, %eax, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %edx, {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %edx, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: sarxl %ebx, %ecx, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, 28(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 4(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %esi, 24(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edi, 16(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ebp, 20(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 8(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 12(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 40(%esp,%ebx), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 56(%esp,%ebx), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 52(%esp,%ebx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %ebp, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %eax, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 60(%esp,%ebx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %eax, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 32(%esp,%ebx), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 36(%esp,%ebx), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, (%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %eax, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ebx, 4(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ebp, 24(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: sarxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ebx, 28(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %esi, 16(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edi, 20(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %esi, 8(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %esi, 12(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %esi, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, (%eax)
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: addl $108, %esp
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: popl %esi
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: popl %edi
@@ -8516,87 +6800,45 @@ define void @ashr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: andb $28, %al
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movzbl %al, %edi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 32(%esp,%edi), %eax
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 36(%esp,%edi), %esi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrl %cl, %eax
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ecx, %edx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movb %cl, %dh
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: notb %dl
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: addl %esi, %esi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shll %cl, %esi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: orl %eax, %esi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 44(%esp,%edi), %ebx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ebx, %eax
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movb %dh, %cl
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrl %cl, %eax
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 48(%esp,%edi), %esi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: addl %esi, %esi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shll %cl, %esi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: orl %eax, %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movzbl %al, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 48(%esp,%ebp), %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 44(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrdl %cl, %esi, %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 40(%esp,%ebp), %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrdl %cl, %eax, %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 56(%esp,%ebp), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 52(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrdl %cl, %ebx, %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrdl %cl, %eax, %esi
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 40(%esp,%edi), %esi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %esi, %eax
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movb %dh, %cl
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrl %cl, %eax
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: addl %ebx, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: orl %eax, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 52(%esp,%edi), %ebp
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ebp, %eax
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movb %dh, %cl
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrl %cl, %eax
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 56(%esp,%edi), %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: leal (%ecx,%ecx), %ebx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: orl %eax, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movb %dh, %cl
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrl %cl, %eax
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: addl %ebp, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shll %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: orl %eax, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movb %dh, %cl
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 60(%esp,%edi), %eax
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: leal (%eax,%eax), %edi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shll %cl, %edi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: orl %ebx, %edi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movb %dh, %cl
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: addl %esi, %esi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shll %cl, %esi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: orl %ebx, %esi
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movb %dh, %cl
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 60(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrdl %cl, %eax, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 32(%esp,%ebp), %edx
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl 36(%esp,%ebp), %edi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edi, %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrdl %cl, %ebp, %esi
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %esi, 4(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ebx, 24(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shrdl %cl, %edi, %edx
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: sarl %cl, %eax
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, 28(%ecx)
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %esi, 4(%ecx)
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edi, 24(%ecx)
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ebp, 16(%ecx)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, 28(%ebp)
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, 20(%ecx)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, 16(%ebp)
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, 8(%ecx)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, 20(%ebp)
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, 12(%ecx)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, 8(%ebp)
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, (%ecx)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, 12(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movl %edx, (%ebp)
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: addl $108, %esp
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: popl %esi
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: popl %edi
@@ -8694,88 +6936,64 @@ define void @ashr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: vmovups (%ecx), %xmm0
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 16(%ecx), %esi
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 20(%ecx), %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 24(%ecx), %ebp
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 28(%ecx), %ecx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movzbl (%eax), %edx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edx, %ebx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlb $3, %bl
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %ebp, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 24(%ecx), %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 28(%ecx), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movzbl (%eax), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlb $3, %cl
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %ebx, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %esi, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: vmovaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: sarl $31, %ecx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %ebx, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: andb $28, %dl
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movzbl %dl, %ecx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxl %eax, 32(%esp,%ecx), %edx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, %ebp
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: notb %bl
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 36(%esp,%ecx), %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: leal (%eax,%eax), %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxl %ebx, %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: orl %edx, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 48(%esp,%ecx), %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: leal (%eax,%eax), %edx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxl %ebx, %edx, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 44(%esp,%ecx), %edx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxl %ebp, %edx, %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: orl %esi, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: addl %edx, %edx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxl %ebx, %edx, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 40(%esp,%ecx), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: sarl $31, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: andb $28, %al
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movzbl %al, %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 48(%esp,%ebx), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 44(%esp,%ebx), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrdl %cl, %esi, %edx
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxl %ebp, %edx, %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %ebp, %edx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: orl %esi, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 56(%esp,%ecx), %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: leal (%esi,%esi), %ebp
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxl %ebx, %ebp, %ebp
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 52(%esp,%ecx), %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxl %edx, %eax, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: orl %edi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxl %edx, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: addl %eax, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxl %ebx, %eax, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxl %edx, %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 60(%esp,%ecx), %ecx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: leal (%ecx,%ecx), %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxl %ebx, %esi, %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: orl %eax, %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: addl %eax, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxl %ebx, %eax, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edx, %ebx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxl %edx, {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: orl %edx, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: sarxl %ebx, %ecx, %ecx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %ecx, 28(%edx)
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, 4(%edx)
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %esi, 24(%edx)
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edi, 16(%edx)
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %ebp, 20(%edx)
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, 8(%edx)
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, 12(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 40(%esp,%ebx), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrdl %cl, %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 56(%esp,%ebx), %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 52(%esp,%ebx), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrdl %cl, %ebp, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrdl %cl, %eax, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 60(%esp,%ebx), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrdl %cl, %eax, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 32(%esp,%ebx), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl 36(%esp,%ebx), %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, (%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrdl %cl, %eax, %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %ebx, 4(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %ebp, 24(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: sarxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %ebx, 28(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %esi, 16(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edi, 20(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %esi, 8(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %esi, 12(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrdl %cl, %esi, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edx, (%eax)
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: addl $108, %esp
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: popl %esi
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: popl %edi
@@ -8870,57 +7088,37 @@ define void @ashr_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
define void @ashr_32bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-NO-BMI2-SSE2-LABEL: ashr_32bytes_dwordOff:
; X64-NO-SHLD-NO-BMI2-SSE2: # %bb.0:
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: pushq %rbx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq (%rdi), %rcx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq (%rdi), %rax
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 8(%rdi), %r8
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 16(%rdi), %r9
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 24(%rdi), %rdi
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movzbl (%rsi), %esi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %eax
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlb $5, %al
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlb $5, %cl
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: sarq $63, %rdi
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: andb $6, %sil
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movzbl %sil, %r9d
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -64(%rsp,%r9,4), %r10
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -56(%rsp,%r9,4), %rdi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rdi, %r11
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %cl, %r11
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %esi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: notb %sil
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -48(%rsp,%r9,4), %rbx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: leaq (%rbx,%rbx), %r8
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlq %cl, %r8
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: orq %r11, %r8
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: addq %rdi, %rdi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlq %cl, %rdi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: orq %r10, %rdi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %cl, %rbx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -40(%rsp,%r9,4), %r9
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: leaq (%r9,%r9), %r10
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: orq %rbx, %r10
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: sarq %cl, %r9
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r9, 24(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r10, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movzbl %sil, %eax
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -56(%rsp,%rax,4), %rsi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -72(%rsp,%rax,4), %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -64(%rsp,%rax,4), %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r8, %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdq %cl, %rsi, %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -48(%rsp,%rax,4), %rax
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdq %cl, %rax, %rsi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdq %cl, %r8, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: sarq %cl, %rax
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rsi, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rax, 24(%rdx)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rdi, (%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r8, 8(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: popq %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r9, 8(%rdx)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: retq
;
; X64-HAVE-SHLD-NO-BMI2-SSE2-LABEL: ashr_32bytes_dwordOff:
@@ -8960,46 +7158,37 @@ define void @ashr_32bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) no
;
; X64-NO-SHLD-HAVE-BMI2-SSE2-LABEL: ashr_32bytes_dwordOff:
; X64-NO-SHLD-HAVE-BMI2-SSE2: # %bb.0:
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq (%rdi), %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq (%rdi), %rax
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq 8(%rdi), %r8
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq 16(%rdi), %r9
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq 24(%rdi), %rdi
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movzbl (%rsi), %esi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, %eax
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlb $5, %al
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, %ecx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlb $5, %cl
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: sarq $63, %rdi
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, %ecx
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: andb $6, %sil
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movzbl %sil, %esi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -64(%rsp,%rsi,4), %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -56(%rsp,%rsi,4), %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxq %rcx, %rdi, %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: notb %al
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leaq (%r8,%r8), %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxq %rax, %r10, %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orq %r9, %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxq %rcx, -72(%rsp,%rsi,4), %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addq %rdi, %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxq %rax, %rdi, %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orq %r9, %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxq %rcx, %r8, %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -48(%rsp,%rsi,4), %rsi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leaq (%rsi,%rsi), %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxq %rax, %r9, %rax
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orq %r8, %rax
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: sarxq %rcx, %rsi, %rcx
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rcx, 24(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rax, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movzbl %sil, %eax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -56(%rsp,%rax,4), %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -72(%rsp,%rax,4), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -64(%rsp,%rax,4), %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r8, %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdq %cl, %rsi, %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -48(%rsp,%rax,4), %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdq %cl, %rax, %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdq %cl, %r8, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: sarxq %rcx, %rax, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rsi, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rax, 24(%rdx)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rdi, (%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r10, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r9, 8(%rdx)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: retq
;
; X64-HAVE-SHLD-HAVE-BMI2-SSE2-LABEL: ashr_32bytes_dwordOff:
@@ -9039,15 +7228,14 @@ define void @ashr_32bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) no
;
; X64-NO-SHLD-NO-BMI2-SSE4-LABEL: ashr_32bytes_dwordOff:
; X64-NO-SHLD-NO-BMI2-SSE4: # %bb.0:
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: pushq %rbx
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movups (%rdi), %xmm0
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq 16(%rdi), %rcx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq 16(%rdi), %rax
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq 24(%rdi), %rdi
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movzbl (%rsi), %esi
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %eax
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlb $5, %al
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlb $5, %cl
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: sarq $63, %rdi
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
@@ -9055,39 +7243,21 @@ define void @ashr_32bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) no
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: andb $6, %sil
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movzbl %sil, %r9d
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -64(%rsp,%r9,4), %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -56(%rsp,%r9,4), %r8
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %esi
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: notb %sil
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: leaq (%r8,%r8), %rdi
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlq %cl, %rdi
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: orq %r10, %rdi
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -48(%rsp,%r9,4), %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r10, %r11
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %cl, %r11
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -40(%rsp,%r9,4), %r9
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: leaq (%r9,%r9), %rbx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlq %cl, %rbx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: orq %r11, %rbx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %cl, %r8
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: addq %r10, %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: orq %r8, %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: sarq %cl, %r9
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r9, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movzbl %sil, %eax
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -48(%rsp,%rax,4), %rsi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -56(%rsp,%rax,4), %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rdi, %r8
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdq %cl, %rsi, %r8
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -72(%rsp,%rax,4), %r9
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -64(%rsp,%rax,4), %rax
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rax, %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdq %cl, %rdi, %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdq %cl, %rax, %r9
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: sarq %cl, %rsi
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r10, 8(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rbx, 16(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rdi, (%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: popq %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r8, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rsi, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r9, (%rdx)
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: retq
;
; X64-HAVE-SHLD-NO-BMI2-SSE4-LABEL: ashr_32bytes_dwordOff:
@@ -9127,43 +7297,35 @@ define void @ashr_32bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) no
; X64-NO-SHLD-HAVE-BMI2-SSE4-LABEL: ashr_32bytes_dwordOff:
; X64-NO-SHLD-HAVE-BMI2-SSE4: # %bb.0:
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movups (%rdi), %xmm0
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq 16(%rdi), %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq 16(%rdi), %rax
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq 24(%rdi), %rdi
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movzbl (%rsi), %esi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %esi, %eax
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlb $5, %al
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %esi, %ecx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlb $5, %cl
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: sarq $63, %rdi
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, %ecx
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: andb $6, %sil
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movzbl %sil, %esi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxq %rcx, -72(%rsp,%rsi,4), %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: notb %al
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -64(%rsp,%rsi,4), %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -56(%rsp,%rsi,4), %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leaq (%r8,%r8), %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxq %rax, %r10, %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orq %rdi, %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxq %rcx, %r9, %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -48(%rsp,%rsi,4), %rsi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leaq (%rsi,%rsi), %r11
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxq %rax, %r11, %r11
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orq %rdi, %r11
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxq %rcx, %r8, %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: addq %r9, %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxq %rax, %r9, %rax
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orq %rdi, %rax
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: sarxq %rcx, %rsi, %rcx
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rcx, 24(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rax, 8(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r11, 16(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r10, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movzbl %sil, %eax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -48(%rsp,%rax,4), %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -56(%rsp,%rax,4), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rdi, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdq %cl, %rsi, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -72(%rsp,%rax,4), %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -64(%rsp,%rax,4), %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rax, %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdq %cl, %rdi, %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdq %cl, %rax, %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: sarxq %rcx, %rsi, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r10, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r8, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rax, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r9, (%rdx)
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: retq
;
; X64-HAVE-SHLD-HAVE-BMI2-SSE4-LABEL: ashr_32bytes_dwordOff:
@@ -9202,15 +7364,14 @@ define void @ashr_32bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) no
;
; X64-NO-SHLD-NO-BMI2-AVX-LABEL: ashr_32bytes_dwordOff:
; X64-NO-SHLD-NO-BMI2-AVX: # %bb.0:
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: pushq %rbx
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: vmovups (%rdi), %xmm0
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq 16(%rdi), %rcx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq 16(%rdi), %rax
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq 24(%rdi), %rdi
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movzbl (%rsi), %esi
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %esi, %eax
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shlb $5, %al
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %esi, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shlb $5, %cl
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: sarq $63, %rdi
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
@@ -9218,39 +7379,21 @@ define void @ashr_32bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) no
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: andb $6, %sil
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movzbl %sil, %r9d
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -64(%rsp,%r9,4), %r10
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -56(%rsp,%r9,4), %r8
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shrq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %esi
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: notb %sil
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: leaq (%r8,%r8), %rdi
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shlq %cl, %rdi
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: orq %r10, %rdi
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -48(%rsp,%r9,4), %r10
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %r10, %r11
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shrq %cl, %r11
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -40(%rsp,%r9,4), %r9
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: leaq (%r9,%r9), %rbx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shlq %cl, %rbx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: orq %r11, %rbx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shrq %cl, %r8
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: addq %r10, %r10
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shlq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: orq %r8, %r10
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: sarq %cl, %r9
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %r9, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movzbl %sil, %eax
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -48(%rsp,%rax,4), %rsi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -56(%rsp,%rax,4), %rdi
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %rdi, %r8
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shrdq %cl, %rsi, %r8
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -72(%rsp,%rax,4), %r9
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq -64(%rsp,%rax,4), %rax
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %rax, %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shrdq %cl, %rdi, %r10
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shrdq %cl, %rax, %r9
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: sarq %cl, %rsi
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %r10, 8(%rdx)
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %rbx, 16(%rdx)
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %rdi, (%rdx)
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: popq %rbx
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %r8, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %rsi, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movq %r9, (%rdx)
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: retq
;
; X64-HAVE-SHLD-NO-BMI2-AVX-LABEL: ashr_32bytes_dwordOff:
@@ -9290,43 +7433,35 @@ define void @ashr_32bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) no
; X64-NO-SHLD-HAVE-BMI2-AVX-LABEL: ashr_32bytes_dwordOff:
; X64-NO-SHLD-HAVE-BMI2-AVX: # %bb.0:
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: vmovups (%rdi), %xmm0
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq 16(%rdi), %rcx
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq 16(%rdi), %rax
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq 24(%rdi), %rdi
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movzbl (%rsi), %esi
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %esi, %eax
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlb $5, %al
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %esi, %ecx
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlb $5, %cl
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: sarq $63, %rdi
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, %ecx
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: andb $6, %sil
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movzbl %sil, %esi
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxq %rcx, -72(%rsp,%rsi,4), %rdi
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: notb %al
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq -64(%rsp,%rsi,4), %r8
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq -56(%rsp,%rsi,4), %r9
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: leaq (%r8,%r8), %r10
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxq %rax, %r10, %r10
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: orq %rdi, %r10
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxq %rcx, %r9, %rdi
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq -48(%rsp,%rsi,4), %rsi
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: leaq (%rsi,%rsi), %r11
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxq %rax, %r11, %r11
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: orq %rdi, %r11
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrxq %rcx, %r8, %rdi
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: addq %r9, %r9
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlxq %rax, %r9, %rax
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: orq %rdi, %rax
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: sarxq %rcx, %rsi, %rcx
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %rcx, 24(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %rax, 8(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %r11, 16(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %r10, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movzbl %sil, %eax
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq -48(%rsp,%rax,4), %rsi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq -56(%rsp,%rax,4), %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %rdi, %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrdq %cl, %rsi, %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq -72(%rsp,%rax,4), %r9
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq -64(%rsp,%rax,4), %rax
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %rax, %r10
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrdq %cl, %rdi, %r10
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shrdq %cl, %rax, %r9
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: sarxq %rcx, %rsi, %rax
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %r10, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %r8, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %rax, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movq %r9, (%rdx)
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: retq
;
; X64-HAVE-SHLD-HAVE-BMI2-AVX-LABEL: ashr_32bytes_dwordOff:
@@ -9748,23 +7883,22 @@ define void @lshr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-NO-BMI2-SSE2: # %bb.0:
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: pushq %r15
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: pushq %r14
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: pushq %r13
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: pushq %r12
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: pushq %rbx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq (%rdi), %rax
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 8(%rdi), %rcx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 16(%rdi), %r8
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 24(%rdi), %r9
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 32(%rdi), %r10
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 40(%rdi), %r11
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 48(%rdi), %rbx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 56(%rdi), %r14
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl (%rsi), %edi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq (%rdi), %rcx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 8(%rdi), %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 16(%rdi), %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 24(%rdi), %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 32(%rdi), %r11
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 40(%rdi), %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 48(%rdi), %r14
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 56(%rdi), %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl (%rsi), %eax
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: xorps %xmm0, %xmm0
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r14, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rbx, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
@@ -9772,78 +7906,38 @@ define void @lshr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: leal (,%rdi,8), %eax
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: leal (,%rax,8), %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: andl $56, %ecx
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: andl $56, %eax
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: andl $56, %edi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -128(%rsp,%rdi), %r10
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -120(%rsp,%rdi), %r8
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r8, %r11
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %cl, %r11
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %esi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: notb %sil
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -112(%rsp,%rdi), %rbx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: leaq (%rbx,%rbx), %r9
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlq %cl, %r9
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: orq %r11, %r9
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: addq %r8, %r8
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlq %cl, %r8
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: orq %r10, %r8
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -104(%rsp,%rdi), %r10
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r10, %r15
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %cl, %r15
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -96(%rsp,%rdi), %r14
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: leaq (%r14,%r14), %r11
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlq %cl, %r11
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: orq %r15, %r11
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %cl, %rbx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: addq %r10, %r10
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: orq %rbx, %r10
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -88(%rsp,%rdi), %rbx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rbx, %r12
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %cl, %r12
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -80(%rsp,%rdi), %r13
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: leaq (%r13,%r13), %r15
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlq %cl, %r15
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: orq %r12, %r15
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %cl, %r14
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: addq %rbx, %rbx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlq %cl, %rbx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: orq %r14, %rbx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %cl, %r13
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -72(%rsp,%rdi), %rdi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: leaq (%rdi,%rdi), %r14
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlq %cl, %r14
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: orq %r13, %r14
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %cl, %rdi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rdi, 56(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r14, 48(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rbx, 32(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -112(%rsp,%rax), %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -128(%rsp,%rax), %rsi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -120(%rsp,%rax), %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r9, %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdq %cl, %rdi, %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -96(%rsp,%rax), %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -104(%rsp,%rax), %r11
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r11, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdq %cl, %r10, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdq %cl, %r11, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -80(%rsp,%rax), %r11
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -88(%rsp,%rax), %r14
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r14, %r15
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdq %cl, %r11, %r15
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdq %cl, %r14, %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -72(%rsp,%rax), %rax
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdq %cl, %rax, %r11
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdq %cl, %r9, %rsi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: # kill: def $cl killed $cl killed $ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %cl, %rax
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r11, 48(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rax, 56(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r10, 32(%rdx)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r15, 40(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r10, 16(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r11, 24(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r8, (%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r9, 8(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rdi, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rbx, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rsi, (%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r8, 8(%rdx)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: popq %rbx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: popq %r12
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: popq %r13
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: popq %r14
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: popq %r15
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: retq
@@ -9915,9 +8009,7 @@ define void @lshr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-HAVE-BMI2-SSE2: # %bb.0:
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: pushq %r15
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: pushq %r14
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: pushq %r12
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: pushq %rbx
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: pushq %rax
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq (%rdi), %rcx
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq 8(%rdi), %r8
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq 16(%rdi), %r9
@@ -9942,56 +8034,36 @@ define void @lshr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leal (,%rax,8), %ecx
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: andl $56, %ecx
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, %esi
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: andl $56, %eax
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -120(%rsp,%rax), %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -112(%rsp,%rax), %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxq %rsi, %r8, %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: notb %cl
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leaq (%r10,%r10), %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxq %rcx, %rdi, %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orq %r9, %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxq %rsi, -128(%rsp,%rax), %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addq %r8, %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxq %rcx, %r8, %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orq %r9, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -112(%rsp,%rax), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -128(%rsp,%rax), %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -120(%rsp,%rax), %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r9, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdq %cl, %rdi, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -96(%rsp,%rax), %r10
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -104(%rsp,%rax), %r11
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxq %rsi, %r11, %rbx
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -96(%rsp,%rax), %r14
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leaq (%r14,%r14), %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxq %rcx, %r9, %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orq %rbx, %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxq %rsi, %r10, %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addq %r11, %r11
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxq %rcx, %r11, %r11
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orq %r10, %r11
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -88(%rsp,%rax), %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxq %rsi, %r10, %rbx
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -80(%rsp,%rax), %r15
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leaq (%r15,%r15), %r12
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxq %rcx, %r12, %r12
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orq %rbx, %r12
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxq %rsi, %r14, %rbx
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addq %r10, %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxq %rcx, %r10, %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orq %rbx, %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxq %rsi, %r15, %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r11, %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdq %cl, %r10, %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdq %cl, %r11, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -80(%rsp,%rax), %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -88(%rsp,%rax), %r14
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r14, %r15
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdq %cl, %r11, %r15
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdq %cl, %r14, %r10
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -72(%rsp,%rax), %rax
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leaq (%rax,%rax), %r14
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxq %rcx, %r14, %rcx
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orq %rbx, %rcx
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxq %rsi, %rax, %rax
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rax, 56(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rcx, 48(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdq %cl, %rax, %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxq %rcx, %rax, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: # kill: def $cl killed $cl killed $rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdq %cl, %r9, %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r11, 48(%rdx)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r10, 32(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r12, 40(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r11, 16(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r9, 24(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r8, (%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rdi, 8(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addq $8, %rsp
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r15, 40(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rdi, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rbx, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rsi, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r8, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rax, 56(%rdx)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: popq %rbx
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: popq %r12
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: popq %r14
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: popq %r15
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: retq
@@ -10061,18 +8133,14 @@ define void @lshr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
;
; X64-NO-SHLD-NO-BMI2-SSE4-LABEL: lshr_64bytes:
; X64-NO-SHLD-NO-BMI2-SSE4: # %bb.0:
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: pushq %rbp
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: pushq %r15
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: pushq %r14
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: pushq %r13
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: pushq %r12
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: pushq %rbx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: pushq %rax
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movups (%rdi), %xmm0
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movups 16(%rdi), %xmm1
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movups 32(%rdi), %xmm2
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movups 48(%rdi), %xmm3
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl (%rsi), %r8d
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl (%rsi), %eax
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: xorps %xmm4, %xmm4
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm4, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm4, -{{[0-9]+}}(%rsp)
@@ -10082,81 +8150,41 @@ define void @lshr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: leal (,%r8,8), %eax
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: leal (,%rax,8), %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: andl $56, %ecx
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: andl $56, %eax
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: andl $56, %r8d
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -128(%rsp,%r8), %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -120(%rsp,%r8), %r9
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %esi
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: notb %sil
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: leaq (%r9,%r9), %rdi
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlq %cl, %rdi
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: orq %r10, %rdi
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -104(%rsp,%r8), %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r10, %rbx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %cl, %rbx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -96(%rsp,%r8), %r12
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: leaq (%r12,%r12), %r11
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlq %cl, %r11
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: orq %rbx, %r11
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -112(%rsp,%r8), %rbx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rbx, %r14
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %cl, %r14
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: addq %r10, %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: orq %r14, %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -88(%rsp,%r8), %r14
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r14, %r13
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %cl, %r13
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -80(%rsp,%r8), %rbp
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: leaq (%rbp,%rbp), %r15
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlq %cl, %r15
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: orq %r13, %r15
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %cl, %r12
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: addq %r14, %r14
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlq %cl, %r14
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: orq %r12, %r14
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %cl, %rbp
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -72(%rsp,%r8), %r8
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: leaq (%r8,%r8), %r12
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlq %cl, %r12
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: orq %rbp, %r12
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %cl, %r9
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: addq %rbx, %rbx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlq %cl, %rbx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: orq %r9, %rbx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %cl, %r8
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r8, 56(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rbx, 8(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r12, 48(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r14, 32(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r15, 40(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r10, 16(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r11, 24(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rdi, (%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: addq $8, %rsp
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -96(%rsp,%rax), %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -104(%rsp,%rax), %r9
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r9, %rsi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdq %cl, %rdi, %rsi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -112(%rsp,%rax), %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r10, %r8
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdq %cl, %r9, %r8
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -80(%rsp,%rax), %r9
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -88(%rsp,%rax), %r11
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r11, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdq %cl, %r9, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdq %cl, %r11, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -72(%rsp,%rax), %r11
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdq %cl, %r11, %r9
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -128(%rsp,%rax), %r14
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -120(%rsp,%rax), %rax
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rax, %r15
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdq %cl, %r10, %r15
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdq %cl, %rax, %r14
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: # kill: def $cl killed $cl killed $ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %cl, %r11
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r15, 8(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r9, 48(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r11, 56(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rdi, 32(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rbx, 40(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r8, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rsi, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r14, (%rdx)
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: popq %rbx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: popq %r12
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: popq %r13
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: popq %r14
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: popq %r15
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: popq %rbp
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: retq
;
; X64-HAVE-SHLD-NO-BMI2-SSE4-LABEL: lshr_64bytes:
@@ -10219,8 +8247,6 @@ define void @lshr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-HAVE-BMI2-SSE4: # %bb.0:
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: pushq %r15
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: pushq %r14
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: pushq %r13
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: pushq %r12
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: pushq %rbx
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movups (%rdi), %xmm0
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movups 16(%rdi), %xmm1
@@ -10238,56 +8264,37 @@ define void @lshr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leal (,%rax,8), %ecx
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: andl $56, %ecx
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, %esi
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: andl $56, %eax
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxq %rsi, -128(%rsp,%rax), %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: notb %cl
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -120(%rsp,%rax), %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -112(%rsp,%rax), %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leaq (%r10,%r10), %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxq %rcx, %rdi, %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orq %r8, %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -104(%rsp,%rax), %r11
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxq %rsi, %r11, %rbx
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -96(%rsp,%rax), %r14
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leaq (%r14,%r14), %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxq %rcx, %r8, %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orq %rbx, %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxq %rsi, %r9, %rbx
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: addq %r11, %r11
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxq %rcx, %r11, %r11
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orq %rbx, %r11
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -88(%rsp,%rax), %rbx
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxq %rsi, %rbx, %r15
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -80(%rsp,%rax), %r12
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leaq (%r12,%r12), %r13
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxq %rcx, %r13, %r13
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orq %r15, %r13
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxq %rsi, %r14, %r14
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: addq %rbx, %rbx
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxq %rcx, %rbx, %rbx
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orq %r14, %rbx
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxq %rsi, %r12, %r14
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -72(%rsp,%rax), %rax
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leaq (%rax,%rax), %r15
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxq %rcx, %r15, %r15
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orq %r14, %r15
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxq %rsi, %r10, %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: addq %r9, %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxq %rcx, %r9, %rcx
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orq %r10, %rcx
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxq %rsi, %rax, %rax
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rax, 56(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rcx, 8(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r15, 48(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rbx, 32(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r13, 40(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r11, 16(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r8, 24(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rdi, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -96(%rsp,%rax), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -104(%rsp,%rax), %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r9, %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdq %cl, %rdi, %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -112(%rsp,%rax), %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r10, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdq %cl, %r9, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -80(%rsp,%rax), %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -88(%rsp,%rax), %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r11, %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdq %cl, %r9, %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdq %cl, %r11, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -72(%rsp,%rax), %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdq %cl, %r11, %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -128(%rsp,%rax), %r14
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -120(%rsp,%rax), %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rax, %r15
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdq %cl, %r10, %r15
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxq %rcx, %r11, %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: # kill: def $cl killed $cl killed $rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdq %cl, %rax, %r14
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r15, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r9, 48(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rdi, 32(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rbx, 40(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r8, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rsi, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r14, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r10, 56(%rdx)
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: popq %rbx
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: popq %r12
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: popq %r13
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: popq %r14
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: popq %r15
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: retq
@@ -10350,96 +8357,52 @@ define void @lshr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
;
; X64-NO-SHLD-NO-BMI2-AVX1-LABEL: lshr_64bytes:
; X64-NO-SHLD-NO-BMI2-AVX1: # %bb.0:
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: pushq %rbp
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: pushq %r15
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: pushq %r14
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: pushq %r13
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: pushq %r12
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: pushq %rbx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: pushq %rax
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups (%rdi), %ymm0
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups 32(%rdi), %ymm1
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl (%rsi), %r9d
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl (%rsi), %eax
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: vxorps %xmm2, %xmm2, %xmm2
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: leal (,%r9,8), %eax
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: leal (,%rax,8), %ecx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: andl $56, %ecx
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: andl $56, %eax
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: andl $56, %r9d
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -128(%rsp,%r9), %r10
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -120(%rsp,%r9), %r8
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %esi
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: notb %sil
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: leaq (%r8,%r8), %rdi
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shlq %cl, %rdi
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: orq %r10, %rdi
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -104(%rsp,%r9), %r10
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r10, %rbx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrq %cl, %rbx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -96(%rsp,%r9), %r12
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: leaq (%r12,%r12), %r11
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shlq %cl, %r11
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: orq %rbx, %r11
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -112(%rsp,%r9), %rbx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %rbx, %r14
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrq %cl, %r14
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: addq %r10, %r10
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shlq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: orq %r14, %r10
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -88(%rsp,%r9), %r14
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r14, %r13
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrq %cl, %r13
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -80(%rsp,%r9), %rbp
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: leaq (%rbp,%rbp), %r15
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shlq %cl, %r15
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: orq %r13, %r15
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrq %cl, %r12
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: addq %r14, %r14
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shlq %cl, %r14
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: orq %r12, %r14
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrq %cl, %rbp
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -72(%rsp,%r9), %r9
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: leaq (%r9,%r9), %r12
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shlq %cl, %r12
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: orq %rbp, %r12
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrq %cl, %r8
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: addq %rbx, %rbx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shlq %cl, %rbx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: orq %r8, %rbx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrq %cl, %r9
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r9, 56(%rdx)
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %rbx, 8(%rdx)
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r12, 48(%rdx)
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r14, 32(%rdx)
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r15, 40(%rdx)
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r10, 16(%rdx)
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r11, 24(%rdx)
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %rdi, (%rdx)
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: addq $8, %rsp
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -96(%rsp,%rax), %rdi
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -104(%rsp,%rax), %r9
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r9, %rsi
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrdq %cl, %rdi, %rsi
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -112(%rsp,%rax), %r10
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r10, %r8
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrdq %cl, %r9, %r8
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -80(%rsp,%rax), %r9
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -88(%rsp,%rax), %r11
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r11, %rbx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrdq %cl, %r9, %rbx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrdq %cl, %r11, %rdi
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -72(%rsp,%rax), %r11
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrdq %cl, %r11, %r9
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -128(%rsp,%rax), %r14
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -120(%rsp,%rax), %rax
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %rax, %r15
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrdq %cl, %r10, %r15
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrdq %cl, %rax, %r14
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: # kill: def $cl killed $cl killed $ecx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrq %cl, %r11
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r15, 8(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r9, 48(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r11, 56(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %rdi, 32(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %rbx, 40(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r8, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %rsi, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r14, (%rdx)
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: popq %rbx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: popq %r12
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: popq %r13
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: popq %r14
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: popq %r15
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: popq %rbp
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: vzeroupper
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: retq
;
@@ -10498,69 +8461,48 @@ define void @lshr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-HAVE-BMI2-AVX1: # %bb.0:
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: pushq %r15
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: pushq %r14
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: pushq %r13
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: pushq %r12
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: pushq %rbx
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups (%rdi), %ymm0
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups 32(%rdi), %ymm1
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl (%rsi), %esi
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl (%rsi), %eax
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vxorps %xmm2, %xmm2, %xmm2
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: leal (,%rsi,8), %eax
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: leal (,%rax,8), %ecx
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: andl $56, %ecx
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: andl $56, %eax
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: andl $56, %esi
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxq %rcx, -128(%rsp,%rsi), %r8
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: notb %al
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -120(%rsp,%rsi), %r10
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -112(%rsp,%rsi), %r9
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: leaq (%r10,%r10), %rdi
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxq %rax, %rdi, %rdi
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orq %r8, %rdi
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -104(%rsp,%rsi), %r11
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxq %rcx, %r11, %rbx
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -96(%rsp,%rsi), %r14
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: leaq (%r14,%r14), %r8
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxq %rax, %r8, %r8
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orq %rbx, %r8
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxq %rcx, %r9, %rbx
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: addq %r11, %r11
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxq %rax, %r11, %r11
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orq %rbx, %r11
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -88(%rsp,%rsi), %rbx
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxq %rcx, %rbx, %r15
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -80(%rsp,%rsi), %r12
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: leaq (%r12,%r12), %r13
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxq %rax, %r13, %r13
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orq %r15, %r13
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxq %rcx, %r14, %r14
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: addq %rbx, %rbx
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxq %rax, %rbx, %rbx
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orq %r14, %rbx
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxq %rcx, %r12, %r14
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -72(%rsp,%rsi), %rsi
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: leaq (%rsi,%rsi), %r15
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxq %rax, %r15, %r15
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orq %r14, %r15
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxq %rcx, %r10, %r10
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: addq %r9, %r9
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxq %rax, %r9, %rax
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orq %r10, %rax
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxq %rcx, %rsi, %rcx
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %rcx, 56(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %rax, 8(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %r15, 48(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %rbx, 32(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %r13, 40(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %r11, 16(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %r8, 24(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %rdi, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -96(%rsp,%rax), %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -104(%rsp,%rax), %r9
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %r9, %rsi
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrdq %cl, %rdi, %rsi
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -112(%rsp,%rax), %r10
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %r10, %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrdq %cl, %r9, %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -80(%rsp,%rax), %r9
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -88(%rsp,%rax), %r11
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %r11, %rbx
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrdq %cl, %r9, %rbx
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrdq %cl, %r11, %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -72(%rsp,%rax), %r11
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrdq %cl, %r11, %r9
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -128(%rsp,%rax), %r14
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -120(%rsp,%rax), %rax
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %rax, %r15
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrdq %cl, %r10, %r15
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxq %rcx, %r11, %r10
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: # kill: def $cl killed $cl killed $rcx
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrdq %cl, %rax, %r14
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %r15, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %r9, 48(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %rdi, 32(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %rbx, 40(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %r8, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %rsi, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %r14, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %r10, 56(%rdx)
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: popq %rbx
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: popq %r12
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: popq %r13
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: popq %r14
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: popq %r15
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vzeroupper
@@ -10672,244 +8614,153 @@ define void @lshr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: pushl %ebx
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: pushl %edi
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: pushl %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: subl $204, %esp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl (%eax), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 4(%eax), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 8(%eax), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 12(%eax), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 16(%eax), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 20(%eax), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 24(%eax), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 28(%eax), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 32(%eax), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 36(%eax), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 40(%eax), %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 44(%eax), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 48(%eax), %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 52(%eax), %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 56(%eax), %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 60(%eax), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl (%eax), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: subl $188, %esp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl (%ecx), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 4(%ecx), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 8(%ecx), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 12(%ecx), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 16(%ecx), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 20(%ecx), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 24(%ecx), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 28(%ecx), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 32(%ecx), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 36(%ecx), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, (%esp) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 40(%ecx), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 44(%ecx), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 48(%ecx), %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 52(%ecx), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 56(%ecx), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 60(%ecx), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl (%ecx), %ecx
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: xorps %xmm0, %xmm0
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebp, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl (%esp), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: andl $60, %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 68(%esp,%esi), %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll $3, %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: andl $24, %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 72(%esp,%esi), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: leal (%ecx,%ecx), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %al, %ch
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: notb %ch
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %edi, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 64(%esp,%esi), %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: addl %edx, %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %edi, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: andl $60, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 56(%esp,%ebp), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 52(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll $3, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: andl $24, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %edx, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 64(%esp,%ebp), %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 60(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %edi, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %eax, %edx
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 76(%esp,%esi), %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 80(%esp,%esi), %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: leal (%edi,%edi), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %ebp, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: addl %edx, %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %ebx, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 72(%esp,%ebp), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 68(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %esi, %edx
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 84(%esp,%esi), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %eax, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 80(%esp,%ebp), %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 76(%esp,%ebp), %eax
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %dl, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 88(%esp,%esi), %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: addl %eax, %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %dl, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: addl %ebx, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %edi, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 92(%esp,%esi), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %dl, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 96(%esp,%esi), %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: leal (%edi,%edi), %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %edi, %edx
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: addl %ebx, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %eax, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 100(%esp,%esi), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %dl, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 104(%esp,%esi), %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: leal (%edx,%edx), %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: addl %ebx, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %edi, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 108(%esp,%esi), %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 112(%esp,%esi), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: leal (%ecx,%ecx), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %ch # 1-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %ebp, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: addl %edi, %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %edx, %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %eax, %esi
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 116(%esp,%esi), %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 120(%esp,%edx), %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: leal (%eax,%eax), %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %ebx, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %dl, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: addl %esi, %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %ebx, %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %dl, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 88(%esp,%ebp), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 84(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %esi, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %eax, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, (%esp) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 96(%esp,%ebp), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 92(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %esi, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %eax, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 104(%esp,%ebp), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 100(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %edx, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %eax, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 48(%esp,%ebp), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 108(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %eax, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, 56(%ebp)
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 124(%esp,%edx), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: leal (%ebx,%ebx), %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %eax, %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %edx, %ebx
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, 60(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, 56(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, 48(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebp, 52(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, 40(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, 44(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, 32(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, 36(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, 24(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, 28(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, 16(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, 20(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, 8(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, 12(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, (%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, 4(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: addl $204, %esp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, 60(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, 48(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, 52(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, 40(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, 44(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl (%esp), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, 32(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, 36(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, 24(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, 28(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, 16(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, 20(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, 8(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, 12(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, (%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, 4(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: addl $188, %esp
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: popl %esi
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: popl %edi
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: popl %ebx
@@ -11081,49 +8932,47 @@ define void @lshr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: pushl %ebx
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: pushl %edi
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: pushl %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: subl $204, %esp
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl (%eax), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 4(%eax), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 8(%eax), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 12(%eax), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 16(%eax), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 20(%eax), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 24(%eax), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 28(%eax), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 32(%eax), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 36(%eax), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 40(%eax), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 44(%eax), %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 48(%eax), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 52(%eax), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 56(%eax), %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 60(%eax), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl (%eax), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: subl $188, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl (%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 4(%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 8(%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 12(%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 16(%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 20(%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 24(%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 28(%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 32(%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 36(%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, (%esp) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 40(%ecx), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 44(%ecx), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 48(%ecx), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 52(%ecx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 56(%ecx), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 60(%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl (%ecx), %ecx
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: xorps %xmm0, %xmm0
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebx, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebp, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl (%esp), %eax # 4-byte Reload
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
@@ -11144,137 +8993,90 @@ define void @lshr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leal (,%ebx,8), %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: andl $24, %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: andl $60, %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 68(%esp,%ebx), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 72(%esp,%ebx), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ecx, %esi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: notb %dl
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leal (%eax,%eax), %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %edx, %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ecx, 64(%esp,%ebx), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addl %esi, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %edx, %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 80(%esp,%ebx), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leal (%esi,%esi), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %edx, %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 76(%esp,%ebx), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ecx, %edi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addl %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %edx, %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %eax, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 88(%esp,%ebx), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leal (%eax,%eax), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %edx, %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 84(%esp,%ebx), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ecx, %edi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ecx, %esi, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addl %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %edx, %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %esi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: andl $60, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 56(%esp,%ebp), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 52(%esp,%ebp), %eax
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 96(%esp,%ebx), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leal (%esi,%esi), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %edx, %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 92(%esp,%ebx), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ecx, %edi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shll $3, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: andl $24, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %edx, %eax
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addl %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %edx, %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 64(%esp,%ebp), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 60(%esp,%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %edi, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 72(%esp,%ebp), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 68(%esp,%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %esi, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %eax, %edi
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 104(%esp,%ebx), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leal (%eax,%eax), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %edx, %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 100(%esp,%ebx), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ecx, %edi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ecx, %esi, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addl %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %edx, %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 112(%esp,%ebx), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leal (%eax,%eax), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %edx, %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 108(%esp,%ebx), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ecx, %esi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addl %esi, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %edx, %esi, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %eax, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 120(%esp,%ebx), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leal (%edi,%edi), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %edx, %ecx, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 116(%esp,%ebx), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebp, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ebp, %eax, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %ebp, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 80(%esp,%ebp), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 76(%esp,%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %edi, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %eax, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 88(%esp,%ebp), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 84(%esp,%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %ebx, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edi, (%esp) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 96(%esp,%ebp), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 92(%esp,%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %esi, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %eax, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 104(%esp,%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 100(%esp,%ebp), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edi, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %edi, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 48(%esp,%ebp), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 108(%esp,%ebp), %ebp
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addl %eax, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %edx, %eax, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 124(%esp,%ebx), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leal (%eax,%eax), %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %edx, %ebx, %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ebp, %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %edi, %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ebp, %eax, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edi, 60(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, 56(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, 48(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, 52(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, 40(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, 44(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, 32(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, 36(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, 24(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, 28(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, 16(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, 20(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, 8(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, 12(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, (%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 56(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, 48(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, 52(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebx, 40(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 44(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl (%esp), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 32(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 36(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 24(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 28(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 16(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 20(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 8(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 12(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %edx, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edi, (%ebp)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, 4(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addl $204, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, 4(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 60(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addl $188, %esp
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: popl %esi
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: popl %edi
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: popl %ebx
@@ -11444,14 +9246,14 @@ define void @lshr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: pushl %ebx
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: pushl %edi
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: pushl %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: subl $204, %esp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: subl $188, %esp
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movups (%ecx), %xmm0
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movups 16(%ecx), %xmm1
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movups 32(%ecx), %xmm2
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movups 48(%ecx), %xmm3
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl (%eax), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl (%eax), %ecx
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: xorps %xmm4, %xmm4
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm4, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm4, {{[0-9]+}}(%esp)
@@ -11461,183 +9263,92 @@ define void @lshr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: andl $60, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 56(%esp,%ebp), %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 52(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll $3, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: andl $24, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %edx, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 64(%esp,%ebp), %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 60(%esp,%ebp), %eax
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: andl $60, %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 68(%esp,%esi), %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll $3, %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: andl $24, %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 72(%esp,%esi), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: leal (%ecx,%ecx), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %al, %ch
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: notb %ch
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %edi, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 64(%esp,%esi), %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: addl %edx, %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %edi, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %edi, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %eax, %edx
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 76(%esp,%esi), %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 80(%esp,%esi), %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: leal (%edi,%edi), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %ebp, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: addl %edx, %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %ebx, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 72(%esp,%ebp), %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 68(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %esi, %edx
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 84(%esp,%esi), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %eax, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 80(%esp,%ebp), %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 76(%esp,%ebp), %eax
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %dl, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 88(%esp,%esi), %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: addl %eax, %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %dl, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: addl %ebx, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %edi, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 92(%esp,%esi), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %dl, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 96(%esp,%esi), %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: leal (%edi,%edi), %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %edi, %edx
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: addl %ebx, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %eax, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 100(%esp,%esi), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %dl, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 104(%esp,%esi), %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: leal (%edx,%edx), %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: addl %ebx, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %edi, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 108(%esp,%esi), %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edi, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 112(%esp,%esi), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: leal (%ecx,%ecx), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %ch # 1-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %ebp, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: addl %edi, %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %edx, %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %eax, %esi
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 116(%esp,%esi), %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 120(%esp,%edx), %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: leal (%eax,%eax), %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %ebx, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %dl, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: addl %esi, %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %ebx, %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %dl, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 88(%esp,%ebp), %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 84(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %esi, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %eax, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 96(%esp,%ebp), %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 92(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %esi, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %eax, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, (%esp) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 104(%esp,%ebp), %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 100(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %edx, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %eax, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 48(%esp,%ebp), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 108(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %eax, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, 56(%ebp)
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 124(%esp,%edx), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: leal (%ebx,%ebx), %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %eax, %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %edx, %ebx
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, 60(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, 56(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, 48(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebp, 52(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edi, 40(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, 44(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, 32(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, 36(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, 24(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, 28(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, 16(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, 20(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, 8(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, 12(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, (%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, 4(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: addl $204, %esp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 60(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, 48(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edi, 52(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl (%esp), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 40(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 44(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 32(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 36(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 24(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 28(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 16(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 20(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 8(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 12(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, (%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 4(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: addl $188, %esp
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: popl %esi
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: popl %edi
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: popl %ebx
@@ -11761,158 +9472,111 @@ define void @lshr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
;
; X86-NO-SHLD-HAVE-BMI2-SSE4-LABEL: lshr_64bytes:
; X86-NO-SHLD-HAVE-BMI2-SSE4: # %bb.0:
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: pushl %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: pushl %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: pushl %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: pushl %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: subl $204, %esp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movups (%ecx), %xmm0
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movups 16(%ecx), %xmm1
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movups 32(%ecx), %xmm2
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movups 48(%ecx), %xmm3
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl (%eax), %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: xorps %xmm4, %xmm4
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm4, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm4, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm4, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm4, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm3, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leal (,%ebx,8), %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: andl $24, %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: andl $60, %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 68(%esp,%ebx), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 72(%esp,%ebx), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ecx, %esi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: notb %dl
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leal (%eax,%eax), %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %edx, %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ecx, 64(%esp,%ebx), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: addl %esi, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %edx, %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 80(%esp,%ebx), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leal (%esi,%esi), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %edx, %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 76(%esp,%ebx), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ecx, %edi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: addl %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %edx, %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %eax, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 88(%esp,%ebx), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leal (%eax,%eax), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %edx, %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 84(%esp,%ebx), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ecx, %edi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ecx, %esi, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: addl %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %edx, %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 96(%esp,%ebx), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leal (%esi,%esi), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %edx, %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 92(%esp,%ebx), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ecx, %edi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: addl %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %edx, %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %eax, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 104(%esp,%ebx), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leal (%eax,%eax), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %edx, %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 100(%esp,%ebx), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ecx, %edi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ecx, %esi, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: addl %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %edx, %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 112(%esp,%ebx), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leal (%eax,%eax), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %edx, %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 108(%esp,%ebx), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ecx, %esi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: addl %esi, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %edx, %esi, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %eax, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 120(%esp,%ebx), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leal (%edi,%edi), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %edx, %ecx, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 116(%esp,%ebx), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ebp, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ebp, %eax, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %ebp, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: addl %eax, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %edx, %eax, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 124(%esp,%ebx), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leal (%eax,%eax), %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %edx, %ebx, %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ebp, %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %edi, %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ebp, %eax, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edi, 60(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, 56(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, 48(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %esi, 52(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, 40(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, 44(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, 32(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, 36(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, 24(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, 28(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, 16(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, 20(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, 8(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, 12(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, (%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: pushl %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: pushl %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: pushl %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: pushl %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: subl $188, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movups (%ecx), %xmm0
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movups 16(%ecx), %xmm1
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movups 32(%ecx), %xmm2
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movups 48(%ecx), %xmm3
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl (%eax), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: xorps %xmm4, %xmm4
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm4, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm4, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm4, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm4, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm3, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: andl $60, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 56(%esp,%ebp), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 52(%esp,%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shll $3, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: andl $24, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %edx, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 64(%esp,%ebp), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 60(%esp,%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %edi, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 72(%esp,%ebp), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 68(%esp,%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %esi, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 80(%esp,%ebp), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 76(%esp,%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %edi, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %eax, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 88(%esp,%ebp), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 84(%esp,%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %ebx, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 96(%esp,%ebp), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 92(%esp,%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %esi, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %eax, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 104(%esp,%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 100(%esp,%ebp), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edi, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %edi, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 48(%esp,%ebp), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 108(%esp,%ebp), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ebp, (%esp) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 56(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %esi, 48(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, 52(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ebx, 40(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 44(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 32(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 36(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 24(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 28(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 16(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 20(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 8(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 12(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ecx, (%esp), %eax # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %edx, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edi, (%ebp)
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, 4(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: addl $204, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, 4(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 60(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: addl $188, %esp
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: popl %esi
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: popl %edi
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: popl %ebx
@@ -12038,7 +9702,7 @@ define void @lshr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: pushl %ebx
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: pushl %edi
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: pushl %esi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: subl $204, %esp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: subl $188, %esp
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups (%ecx), %ymm0
@@ -12049,184 +9713,92 @@ define void @lshr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm2, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm1, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm0, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, %esi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: andl $60, %esi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 68(%esp,%esi), %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: andl $60, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 56(%esp,%ebp), %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 52(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll $3, %ecx
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: andl $24, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edx, %edi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 72(%esp,%esi), %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrdl %cl, %edx, %eax
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: leal (%eax,%eax), %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %cl, %ch
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: notb %ch
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: orl %edi, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 64(%esp,%esi), %edi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ebp, %eax
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: addl %edx, %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: orl %edi, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 64(%esp,%ebp), %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 60(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %esi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrdl %cl, %edi, %esi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrdl %cl, %eax, %edx
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 76(%esp,%esi), %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edx, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 80(%esp,%esi), %edi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: leal (%edi,%edi), %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: orl %ebp, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: addl %edx, %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: orl %ebx, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 72(%esp,%ebp), %esi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 68(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrdl %cl, %esi, %edx
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 84(%esp,%esi), %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ebx, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrdl %cl, %eax, %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 80(%esp,%ebp), %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 76(%esp,%ebp), %eax
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %dl, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 88(%esp,%esi), %eax
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: addl %eax, %eax
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %eax
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %dl, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: addl %ebx, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: orl %edi, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 92(%esp,%esi), %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ebx, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %dl, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 96(%esp,%esi), %edi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: leal (%edi,%edi), %eax
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %eax
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrdl %cl, %edi, %edx
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %eax
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: addl %ebx, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: orl %eax, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 100(%esp,%esi), %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ebx, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %dl, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 104(%esp,%esi), %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: leal (%edx,%edx), %eax
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %eax
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: addl %ebx, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: orl %edi, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 108(%esp,%esi), %edi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edi, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 112(%esp,%esi), %ecx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: leal (%ecx,%ecx), %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %ch # 1-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: orl %ebp, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: addl %edi, %edi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %edi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: orl %edx, %edi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrdl %cl, %eax, %esi
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 116(%esp,%esi), %esi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 120(%esp,%edx), %eax
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: leal (%eax,%eax), %ebp
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: orl %ebx, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %dl, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: addl %esi, %esi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %esi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: orl %ebx, %esi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %dl, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 88(%esp,%ebp), %esi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 84(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrdl %cl, %esi, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrdl %cl, %eax, %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 96(%esp,%ebp), %esi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 92(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrdl %cl, %esi, %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrdl %cl, %eax, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edx, (%esp) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 104(%esp,%ebp), %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 100(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrdl %cl, %edx, %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrdl %cl, %eax, %esi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 48(%esp,%ebp), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 108(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrdl %cl, %eax, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edx, 56(%ebp)
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 124(%esp,%edx), %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: leal (%ebx,%ebx), %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: orl %eax, %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrdl %cl, %edx, %ebx
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ebx, 60(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edx, 56(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, 48(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ebp, 52(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edi, 40(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, 44(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, 32(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, 36(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, 24(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, 28(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, 16(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, 20(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, 8(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, 12(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, (%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, 4(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: addl $204, %esp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, 60(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, 48(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edi, 52(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl (%esp), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, 40(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, 44(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, 32(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, 36(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, 24(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, 28(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, 16(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, 20(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, 8(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, 12(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ebx, (%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, 4(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: addl $188, %esp
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: popl %esi
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: popl %edi
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: popl %ebx
@@ -12350,7 +9922,7 @@ define void @lshr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: pushl %ebx
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: pushl %edi
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: pushl %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: subl $204, %esp
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: subl $188, %esp
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups (%ecx), %ymm0
@@ -12361,132 +9933,90 @@ define void @lshr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups %ymm2, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups %ymm1, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups %ymm0, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: leal (,%ecx,8), %edx
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: andl $24, %edx
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %edx, %ebx
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: andl $60, %ecx
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 68(%esp,%ecx), %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 72(%esp,%ecx), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: andl $60, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 56(%esp,%ebp), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 52(%esp,%ebp), %eax
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ebx, %esi, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: notb %dl
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: leal (%eax,%eax), %ebp
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %edx, %ebp, %ebp
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orl %edi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ebx, 64(%esp,%ecx), %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: addl %esi, %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %edx, %esi, %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orl %edi, %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 80(%esp,%ecx), %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: leal (%esi,%esi), %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %edx, %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 76(%esp,%ecx), %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ebx, %edi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shll $3, %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: andl $24, %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrdl %cl, %edx, %eax
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: addl %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %edx, %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orl %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 64(%esp,%ebp), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 60(%esp,%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrdl %cl, %edi, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrdl %cl, %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 72(%esp,%ebp), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 68(%esp,%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrdl %cl, %esi, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrdl %cl, %eax, %edi
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 88(%esp,%ecx), %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: leal (%eax,%eax), %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %edx, %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 84(%esp,%ecx), %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ebx, %edi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ebx, %esi, %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: addl %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %edx, %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orl %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 96(%esp,%ecx), %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: leal (%esi,%esi), %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %edx, %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 92(%esp,%ecx), %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ebx, %edi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: addl %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %edx, %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orl %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 80(%esp,%ebp), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 76(%esp,%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrdl %cl, %edi, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrdl %cl, %eax, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 88(%esp,%ebp), %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 84(%esp,%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrdl %cl, %ebx, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrdl %cl, %eax, %edi
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 104(%esp,%ecx), %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: leal (%eax,%eax), %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %edx, %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 100(%esp,%ecx), %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ebx, %edi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ebx, %esi, %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: addl %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %edx, %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orl %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 112(%esp,%ecx), %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: leal (%eax,%eax), %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %edx, %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 108(%esp,%ecx), %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ebx, %esi, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orl %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: addl %esi, %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %edx, %esi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orl %eax, %ebp
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 120(%esp,%ecx), %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: addl %eax, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %edx, %eax, %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 116(%esp,%ecx), %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ebx, %eax, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orl %edi, %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: addl %eax, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %edx, %eax, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orl %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 124(%esp,%ecx), %ecx
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: leal (%ecx,%ecx), %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %edx, %edi, %edx
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orl %edi, %edx
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ebx, %ecx, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %edi, 60(%ecx)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %edx, 56(%ecx)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, 48(%ecx)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %esi, 52(%ecx)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ebp, 40(%ecx)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, 44(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 96(%esp,%ebp), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 92(%esp,%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrdl %cl, %esi, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrdl %cl, %eax, %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 104(%esp,%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 100(%esp,%ebp), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %edi, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrdl %cl, %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrdl %cl, %edi, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 48(%esp,%ebp), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 108(%esp,%ebp), %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ebp, (%esp) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrdl %cl, %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, 56(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %esi, 48(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %edx, 52(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ebx, 40(%ebp)
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, 32(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, 44(%ebp)
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, 36(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, 32(%ebp)
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, 24(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, 36(%ebp)
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, 28(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, 24(%ebp)
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, 16(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, 28(%ebp)
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, 20(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, 16(%ebp)
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, 8(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, 20(%ebp)
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, 12(%ecx)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, 8(%ebp)
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, (%ecx)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, 4(%ecx)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: addl $204, %esp
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, 12(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ecx, (%esp), %eax # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrdl %cl, %edx, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %edi, (%ebp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, 4(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, 60(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: addl $188, %esp
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: popl %esi
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: popl %edi
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: popl %ebx
@@ -13014,11 +10544,9 @@ define void @lshr_64bytes_qwordOff(ptr %src.ptr, ptr %qwordOff.ptr, ptr %dst) no
define void @shl_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-NO-BMI2-SSE2-LABEL: shl_64bytes:
; X64-NO-SHLD-NO-BMI2-SSE2: # %bb.0:
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: pushq %r15
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: pushq %r14
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: pushq %r13
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: pushq %r12
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: pushq %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: pushq %rax
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq (%rdi), %rax
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 8(%rdi), %rcx
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 16(%rdi), %r8
@@ -13041,85 +10569,41 @@ define void @shl_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: leal (,%rsi,8), %eax
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: andl $56, %eax
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: leal (,%rsi,8), %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: andl $56, %ecx
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: andl $56, %esi
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: negl %esi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movslq %esi, %rbx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -64(%rsp,%rbx), %r8
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -56(%rsp,%rbx), %rdi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rdi, %r10
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %esi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: notb %sil
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r8, %r9
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %r9
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %cl, %r9
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: orq %r10, %r9
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -40(%rsp,%rbx), %r10
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r10, %r14
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlq %cl, %r14
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -48(%rsp,%rbx), %r15
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r15, %r11
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %r11
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %cl, %r11
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: orq %r14, %r11
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlq %cl, %r15
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %rdi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %cl, %rdi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: orq %r15, %rdi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -24(%rsp,%rbx), %r14
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r14, %r12
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlq %cl, %r12
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -32(%rsp,%rbx), %r13
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r13, %r15
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %r15
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %cl, %r15
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: orq %r12, %r15
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlq %cl, %r13
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %r10
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: orq %r13, %r10
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -8(%rsp,%rbx), %r12
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlq %cl, %r12
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -16(%rsp,%rbx), %rbx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rbx, %r13
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %r13
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %cl, %r13
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: orq %r12, %r13
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlq %cl, %rbx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %r14
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %cl, %r14
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: orq %rbx, %r14
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movslq %esi, %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -48(%rsp,%r9), %rax
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -40(%rsp,%r9), %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r10, %rsi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shldq %cl, %rax, %rsi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -64(%rsp,%r9), %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -56(%rsp,%r9), %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shldq %cl, %rdi, %rax
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -32(%rsp,%r9), %r11
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -24(%rsp,%r9), %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rbx, %r14
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shldq %cl, %r11, %r14
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shldq %cl, %r10, %r11
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -16(%rsp,%r9), %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -8(%rsp,%r9), %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shldq %cl, %r10, %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shldq %cl, %rbx, %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shldq %cl, %r8, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: # kill: def $cl killed $cl killed $ecx
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlq %cl, %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r10, 48(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r9, 56(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r11, 32(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r14, 40(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rax, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rsi, 24(%rdx)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r8, (%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r14, 48(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r13, 56(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r10, 32(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r15, 40(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rdi, 16(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r11, 24(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r9, 8(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rdi, 8(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: addq $8, %rsp
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: popq %rbx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: popq %r12
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: popq %r13
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: popq %r14
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: popq %r15
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: retq
;
; X64-HAVE-SHLD-NO-BMI2-SSE2-LABEL: shl_64bytes:
@@ -13188,9 +10672,7 @@ define void @shl_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
;
; X64-NO-SHLD-HAVE-BMI2-SSE2-LABEL: shl_64bytes:
; X64-NO-SHLD-HAVE-BMI2-SSE2: # %bb.0:
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: pushq %r15
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: pushq %r14
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: pushq %r12
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: pushq %rbx
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: pushq %rax
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq (%rdi), %rax
@@ -13215,62 +10697,40 @@ define void @shl_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leal (,%rsi,8), %eax
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: andl $56, %eax
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, %ecx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leal (,%rsi,8), %ecx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: andl $56, %ecx
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: andl $56, %esi
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: negl %esi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movslq %esi, %rsi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -64(%rsp,%rsi), %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -56(%rsp,%rsi), %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxq %rcx, %rdi, %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: notb %al
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxq %rcx, %r9, %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrq %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxq %rax, %r9, %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orq %r8, %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -40(%rsp,%rsi), %r11
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxq %rcx, %r11, %rbx
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -48(%rsp,%rsi), %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxq %rcx, %r8, %r14
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrq %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxq %rax, %r8, %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orq %rbx, %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrq %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxq %rax, %rdi, %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orq %r14, %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -24(%rsp,%rsi), %rbx
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxq %rcx, %rbx, %r14
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -32(%rsp,%rsi), %r15
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxq %rcx, %r15, %r12
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrq %r15
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxq %rax, %r15, %r15
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orq %r14, %r15
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrq %r11
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxq %rax, %r11, %r11
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orq %r12, %r11
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxq %rcx, -8(%rsp,%rsi), %r14
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -16(%rsp,%rsi), %rsi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxq %rcx, %rsi, %rcx
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrq %rsi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxq %rax, %rsi, %rsi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orq %r14, %rsi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrq %rbx
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxq %rax, %rbx, %rax
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orq %rcx, %rax
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r10, (%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rax, 48(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rsi, 56(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movslq %esi, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -48(%rsp,%r8), %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -40(%rsp,%r8), %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r9, %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shldq %cl, %rax, %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -64(%rsp,%r8), %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -56(%rsp,%r8), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shldq %cl, %rdi, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -32(%rsp,%r8), %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -24(%rsp,%r8), %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rbx, %r14
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shldq %cl, %r11, %r14
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shldq %cl, %r9, %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -16(%rsp,%r8), %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -8(%rsp,%r8), %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shldq %cl, %r9, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shldq %cl, %rbx, %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shldq %cl, %r10, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxq %rcx, %r10, %rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r9, 48(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r8, 56(%rdx)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r11, 32(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r15, 40(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rdi, 16(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r8, 24(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r9, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r14, 40(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rax, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rsi, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rcx, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rdi, 8(%rdx)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addq $8, %rsp
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: popq %rbx
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: popq %r12
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: popq %r14
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: popq %r15
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: retq
;
; X64-HAVE-SHLD-HAVE-BMI2-SSE2-LABEL: shl_64bytes:
@@ -13340,14 +10800,12 @@ define void @shl_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-NO-BMI2-SSE4: # %bb.0:
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: pushq %r15
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: pushq %r14
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: pushq %r13
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: pushq %r12
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: pushq %rbx
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movups (%rdi), %xmm0
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movups 16(%rdi), %xmm1
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movups 32(%rdi), %xmm2
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movups 48(%rdi), %xmm3
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl (%rsi), %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl (%rsi), %eax
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: xorps %xmm4, %xmm4
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm4, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm4, -{{[0-9]+}}(%rsp)
@@ -13357,83 +10815,40 @@ define void @shl_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: leal (,%rcx,8), %eax
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: andl $56, %eax
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: leal (,%rax,8), %ecx
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: andl $56, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: negl %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movslq %ecx, %r9
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -24(%rsp,%r9), %rdi
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rdi, %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %esi
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: notb %sil
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -32(%rsp,%r9), %r11
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r11, %r8
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %r8
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %cl, %r8
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: orq %r10, %r8
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlq %cl, %r11
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -40(%rsp,%r9), %rbx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rbx, %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: orq %r11, %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlq %cl, %rbx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -48(%rsp,%r9), %r15
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r15, %r11
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %r11
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %cl, %r11
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: orq %rbx, %r11
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlq %cl, %r15
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -64(%rsp,%r9), %r14
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -56(%rsp,%r9), %r12
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r12, %rbx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %rbx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %cl, %rbx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: orq %r15, %rbx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlq %cl, %r12
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: andl $56, %eax
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: negl %eax
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movslq %eax, %r8
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -32(%rsp,%r8), %rax
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -24(%rsp,%r8), %r9
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r9, %rsi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shldq %cl, %rax, %rsi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -40(%rsp,%r8), %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shldq %cl, %rdi, %rax
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -48(%rsp,%r8), %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shldq %cl, %r10, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -64(%rsp,%r8), %r11
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -56(%rsp,%r8), %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shldq %cl, %rbx, %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -16(%rsp,%r8), %r14
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r14, %r15
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %r15
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %cl, %r15
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: orq %r12, %r15
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -16(%rsp,%r9), %r12
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r12, %r13
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlq %cl, %r13
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %rdi
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %cl, %rdi
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: orq %r13, %rdi
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -8(%rsp,%r9), %r9
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shldq %cl, %r9, %r15
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -8(%rsp,%r8), %r8
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shldq %cl, %r14, %r8
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r11, %r9
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlq %cl, %r9
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %r12
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %cl, %r12
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: orq %r9, %r12
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlq %cl, %r14
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r14, (%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r12, 56(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rdi, 48(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r15, 8(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rbx, 16(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r11, 24(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r10, 32(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r8, 40(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: # kill: def $cl killed $cl killed $ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shldq %cl, %r11, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r8, 56(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r15, 48(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rbx, 8(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r10, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rdi, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rax, 32(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rsi, 40(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r9, (%rdx)
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: popq %rbx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: popq %r12
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: popq %r13
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: popq %r14
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: popq %r15
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: retq
@@ -13499,14 +10914,12 @@ define void @shl_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-HAVE-BMI2-SSE4: # %bb.0:
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: pushq %r15
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: pushq %r14
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: pushq %r12
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: pushq %rbx
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: pushq %rax
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movups (%rdi), %xmm0
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movups 16(%rdi), %xmm1
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movups 32(%rdi), %xmm2
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movups 48(%rdi), %xmm3
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl (%rsi), %esi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl (%rsi), %eax
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: xorps %xmm4, %xmm4
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm4, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm4, -{{[0-9]+}}(%rsp)
@@ -13516,60 +10929,39 @@ define void @shl_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leal (,%rsi,8), %eax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leal (,%rax,8), %ecx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: andl $56, %ecx
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: andl $56, %eax
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: andl $56, %esi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: negl %esi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movslq %esi, %rsi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -24(%rsp,%rsi), %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxq %rcx, %rdi, %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: notb %al
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -32(%rsp,%rsi), %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxq %rcx, %r8, %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrq %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxq %rax, %r8, %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orq %r9, %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -40(%rsp,%rsi), %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxq %rcx, %r9, %r11
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrq %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxq %rax, %r9, %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orq %r10, %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -48(%rsp,%rsi), %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxq %rcx, %r10, %r14
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrq %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxq %rax, %r10, %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orq %r11, %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -64(%rsp,%rsi), %rbx
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -56(%rsp,%rsi), %r11
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxq %rcx, %r11, %r15
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrq %r11
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxq %rax, %r11, %r11
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orq %r14, %r11
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxq %rcx, %rbx, %r14
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrq %rbx
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxq %rax, %rbx, %rbx
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orq %r15, %rbx
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -16(%rsp,%rsi), %r15
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxq %rcx, %r15, %r12
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrq %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxq %rax, %rdi, %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orq %r12, %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxq %rcx, -8(%rsp,%rsi), %rcx
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrq %r15
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxq %rax, %r15, %rax
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orq %rcx, %rax
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r14, (%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rax, 56(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rdi, 48(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: negl %eax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movslq %eax, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -32(%rsp,%r8), %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -24(%rsp,%r8), %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r9, %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shldq %cl, %rax, %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -40(%rsp,%r8), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shldq %cl, %rdi, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -48(%rsp,%r8), %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shldq %cl, %r10, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -64(%rsp,%r8), %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -56(%rsp,%r8), %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shldq %cl, %rbx, %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -16(%rsp,%r8), %r14
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r14, %r15
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shldq %cl, %r9, %r15
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -8(%rsp,%r8), %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shldq %cl, %r14, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxq %rcx, %r11, %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: # kill: def $cl killed $cl killed $rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shldq %cl, %r11, %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r8, 56(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r15, 48(%rdx)
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rbx, 8(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r11, 16(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r10, 24(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r9, 32(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r8, 40(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: addq $8, %rsp
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r10, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rdi, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rax, 32(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rsi, 40(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r9, (%rdx)
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: popq %rbx
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: popq %r12
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: popq %r14
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: popq %r15
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: retq
@@ -13634,94 +11026,49 @@ define void @shl_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-NO-BMI2-AVX1: # %bb.0:
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: pushq %r15
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: pushq %r14
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: pushq %r13
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: pushq %r12
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: pushq %rbx
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups (%rdi), %ymm0
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups 32(%rdi), %ymm1
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl (%rsi), %ecx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl (%rsi), %eax
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: vxorps %xmm2, %xmm2, %xmm2
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: leal (,%rcx,8), %eax
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: andl $56, %eax
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: leal (,%rax,8), %ecx
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: andl $56, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: negl %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movslq %ecx, %r9
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -24(%rsp,%r9), %rdi
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %rdi, %r10
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shlq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %esi
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: notb %sil
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -32(%rsp,%r9), %r11
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r11, %r8
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrq %r8
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrq %cl, %r8
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: orq %r10, %r8
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shlq %cl, %r11
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -40(%rsp,%r9), %rbx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %rbx, %r10
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrq %r10
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: orq %r11, %r10
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shlq %cl, %rbx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -48(%rsp,%r9), %r15
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r15, %r11
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrq %r11
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrq %cl, %r11
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: orq %rbx, %r11
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shlq %cl, %r15
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -64(%rsp,%r9), %r14
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -56(%rsp,%r9), %r12
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r12, %rbx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrq %rbx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrq %cl, %rbx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: orq %r15, %rbx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shlq %cl, %r12
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: andl $56, %eax
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: negl %eax
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movslq %eax, %r8
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -32(%rsp,%r8), %rax
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -24(%rsp,%r8), %r9
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r9, %rsi
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shldq %cl, %rax, %rsi
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -40(%rsp,%r8), %rdi
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shldq %cl, %rdi, %rax
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -48(%rsp,%r8), %r10
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shldq %cl, %r10, %rdi
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -64(%rsp,%r8), %r11
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -56(%rsp,%r8), %rbx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shldq %cl, %rbx, %r10
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -16(%rsp,%r8), %r14
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r14, %r15
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrq %r15
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrq %cl, %r15
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: orq %r12, %r15
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -16(%rsp,%r9), %r12
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r12, %r13
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shlq %cl, %r13
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrq %rdi
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrq %cl, %rdi
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: orq %r13, %rdi
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -8(%rsp,%r9), %r9
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %ecx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shldq %cl, %r9, %r15
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -8(%rsp,%r8), %r8
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shldq %cl, %r14, %r8
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r11, %r9
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shlq %cl, %r9
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrq %r12
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrq %cl, %r12
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: orq %r9, %r12
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shlq %cl, %r14
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r14, (%rdx)
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r12, 56(%rdx)
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %rdi, 48(%rdx)
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r15, 8(%rdx)
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %rbx, 16(%rdx)
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r11, 24(%rdx)
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r10, 32(%rdx)
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r8, 40(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: # kill: def $cl killed $cl killed $ecx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shldq %cl, %r11, %rbx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r8, 56(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r15, 48(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %rbx, 8(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r10, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %rdi, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %rax, 32(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %rsi, 40(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r9, (%rdx)
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: popq %rbx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: popq %r12
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: popq %r13
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: popq %r14
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: popq %r15
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: vzeroupper
@@ -13783,71 +11130,48 @@ define void @shl_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-HAVE-BMI2-AVX1: # %bb.0:
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: pushq %r15
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: pushq %r14
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: pushq %r12
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: pushq %rbx
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: pushq %rax
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups (%rdi), %ymm0
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups 32(%rdi), %ymm1
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl (%rsi), %esi
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl (%rsi), %eax
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vxorps %xmm2, %xmm2, %xmm2
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: leal (,%rsi,8), %eax
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: leal (,%rax,8), %ecx
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: andl $56, %ecx
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: andl $56, %eax
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: andl $56, %esi
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: negl %esi
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movslq %esi, %rsi
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -24(%rsp,%rsi), %rdi
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxq %rcx, %rdi, %r9
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: notb %al
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -32(%rsp,%rsi), %r8
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxq %rcx, %r8, %r10
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrq %r8
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxq %rax, %r8, %r8
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orq %r9, %r8
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -40(%rsp,%rsi), %r9
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxq %rcx, %r9, %r11
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrq %r9
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxq %rax, %r9, %r9
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orq %r10, %r9
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -48(%rsp,%rsi), %r10
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxq %rcx, %r10, %r14
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrq %r10
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxq %rax, %r10, %r10
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orq %r11, %r10
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -64(%rsp,%rsi), %rbx
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -56(%rsp,%rsi), %r11
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxq %rcx, %r11, %r15
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrq %r11
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxq %rax, %r11, %r11
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orq %r14, %r11
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxq %rcx, %rbx, %r14
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrq %rbx
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxq %rax, %rbx, %rbx
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orq %r15, %rbx
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -16(%rsp,%rsi), %r15
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxq %rcx, %r15, %r12
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrq %rdi
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxq %rax, %rdi, %rdi
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orq %r12, %rdi
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxq %rcx, -8(%rsp,%rsi), %rcx
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrq %r15
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxq %rax, %r15, %rax
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orq %rcx, %rax
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %r14, (%rdx)
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %rax, 56(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %rdi, 48(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: negl %eax
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movslq %eax, %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -32(%rsp,%r8), %rax
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -24(%rsp,%r8), %r9
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %r9, %rsi
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shldq %cl, %rax, %rsi
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -40(%rsp,%r8), %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shldq %cl, %rdi, %rax
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -48(%rsp,%r8), %r10
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shldq %cl, %r10, %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -64(%rsp,%r8), %r11
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -56(%rsp,%r8), %rbx
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shldq %cl, %rbx, %r10
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -16(%rsp,%r8), %r14
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %r14, %r15
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shldq %cl, %r9, %r15
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -8(%rsp,%r8), %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shldq %cl, %r14, %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxq %rcx, %r11, %r9
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: # kill: def $cl killed $cl killed $rcx
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shldq %cl, %r11, %rbx
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %r8, 56(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %r15, 48(%rdx)
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %rbx, 8(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %r11, 16(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %r10, 24(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %r9, 32(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %r8, 40(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: addq $8, %rsp
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %r10, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %rdi, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %rax, 32(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %rsi, 40(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %r9, (%rdx)
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: popq %rbx
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: popq %r12
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: popq %r14
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: popq %r15
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vzeroupper
@@ -13957,256 +11281,153 @@ define void @shl_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: pushl %ebx
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: pushl %edi
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: pushl %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: subl $204, %esp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl (%eax), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 4(%eax), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 8(%eax), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 12(%eax), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 16(%eax), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 20(%eax), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 24(%eax), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 28(%eax), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 32(%eax), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 36(%eax), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 40(%eax), %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 44(%eax), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 48(%eax), %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 52(%eax), %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 56(%eax), %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 60(%eax), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl (%eax), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: subl $188, %esp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl (%ecx), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 4(%ecx), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 8(%ecx), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 12(%ecx), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 16(%ecx), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 20(%ecx), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 24(%ecx), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 28(%ecx), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 32(%ecx), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 36(%ecx), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, (%esp) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 40(%ecx), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 44(%ecx), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 48(%ecx), %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 52(%ecx), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 56(%ecx), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 60(%ecx), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl (%ecx), %ecx
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: xorps %xmm0, %xmm0
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebp, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl (%esp), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: andl $60, %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: leal {{[0-9]+}}(%esp), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: subl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl (%ecx), %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 4(%ecx), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll $3, %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: andl $24, %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %al, %ch
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: notb %ch
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %esi, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: andl $60, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: leal {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: subl %ebp, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 8(%eax), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 12(%eax), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll $3, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: andl $24, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shldl %cl, %esi, %edi
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 12(%ebp), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 8(%ebp), %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebp, %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %ebx, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %esi, %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 20(%edi), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 16(%edi), %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %ebx, %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %esi, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 4(%eax), %edi
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebp, %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 28(%ebp), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shldl %cl, %edi, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 16(%eax), %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 20(%eax), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shldl %cl, %edi, %ebx
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 24(%ebp), %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %ebx, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shldl %cl, %edx, %edi
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %esi, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 36(%edx), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 24(%eax), %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 28(%eax), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shldl %cl, %edi, %ebx
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 32(%edx), %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %ebx, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shldl %cl, %esi, %edi
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %esi, %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 44(%ebp), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 32(%eax), %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 36(%eax), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shldl %cl, %edi, %ebx
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 40(%ebp), %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %ebx, %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %esi, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shldl %cl, %edx, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 40(%eax), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 44(%eax), %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shldl %cl, %edx, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, (%esp) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shldl %cl, %esi, %edx
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 52(%ebp), %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 56(%eax), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 60(%eax), %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shldl %cl, %edx, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl (%eax), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 52(%eax), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shldl %cl, %esi, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: negl %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 160(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, 56(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, 60(%ebp)
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: negl %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 176(%esp,%edx), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %edi, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shldl %cl, %ebx, %edx
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %ebx, %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shldl %cl, %eax, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: # kill: def $cl killed $cl killed $ecx
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 60(%edi), %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 56(%edi), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %edx, %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %ebx, %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, (%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, 56(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, 60(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, 48(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebp, 52(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, 40(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, 44(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, 32(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, 36(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, 24(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, 28(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, 16(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, 20(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, 8(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, 12(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, 4(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: addl $204, %esp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shldl %cl, %edi, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, 48(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, 52(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, 40(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl (%esp), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, 44(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, 32(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, 36(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, 24(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, 28(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, 16(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, 20(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, 8(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, 12(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, (%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, 4(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: addl $188, %esp
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: popl %esi
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: popl %edi
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: popl %ebx
@@ -14379,33 +11600,33 @@ define void @shl_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: pushl %edi
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: pushl %esi
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: subl $204, %esp
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl (%eax), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 4(%eax), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 8(%eax), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 12(%eax), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 16(%eax), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 20(%eax), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 24(%eax), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 28(%eax), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 32(%eax), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 36(%eax), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 40(%eax), %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 44(%eax), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 48(%eax), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 52(%eax), %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 56(%eax), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 60(%eax), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl (%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 4(%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 8(%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 12(%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 16(%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 20(%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 24(%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 28(%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 32(%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 36(%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 40(%ebp), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 44(%ebp), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 48(%ebp), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 52(%ebp), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 56(%ebp), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 60(%ebp), %eax
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ebp
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl (%ebp), %ebp
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: xorps %xmm0, %xmm0
@@ -14439,149 +11660,92 @@ define void @shl_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leal (,%ebp,8), %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: andl $24, %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebx, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leal (,%ebp,8), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: andl $24, %ecx
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: andl $60, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leal {{[0-9]+}}(%esp), %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: subl %ebp, %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl (%edx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leal {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: subl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 4(%eax), %esi
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 4(%edx), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: notb %bl
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrl %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ebx, %esi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %eax, %ecx, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %esi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 8(%eax), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 12(%eax), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shldl %cl, %edi, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shldl %cl, %esi, %edi
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 8(%edx), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrl %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ebx, %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 12(%edx), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebp, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %ebp, %esi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %edi, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrl %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ebx, %ecx, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %eax, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 16(%edx), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrl %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ebx, %ecx, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 20(%edx), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %edi, %ecx, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %edi, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrl %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ebx, %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 24(%edx), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrl %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ebx, %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 28(%edx), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %edi, %esi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %edi, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrl %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ebx, %ecx, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 32(%edx), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrl %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ebx, %ecx, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 36(%edx), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %edi, %ecx, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %edi, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrl %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ebx, %esi, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %ebp, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 40(%edx), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edi, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrl %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ebx, %esi, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 44(%edx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 16(%eax), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 20(%eax), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shldl %cl, %edi, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shldl %cl, %edx, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 24(%eax), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 28(%eax), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shldl %cl, %edi, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shldl %cl, %esi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 32(%eax), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 36(%eax), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shldl %cl, %edi, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shldl %cl, %edx, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 40(%eax), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 44(%eax), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shldl %cl, %ebx, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shldl %cl, %esi, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 56(%eax), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 60(%eax), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shldl %cl, %edx, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl (%eax), %esi
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %eax, %esi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %ebp, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %eax, %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrl %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ebx, %eax, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 48(%edx), %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebp, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrl %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ebx, %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 52(%edx), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %esi, %ecx, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %esi, %ebp, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrl %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ebx, %eax, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %edi, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 56(%edx), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrl %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ebx, %ecx, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %ebp, %edi, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %ecx, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrl %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ebx, %edi, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %ebp, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: negl %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %ebp, 188(%esp,%ebx), %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %ecx, %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edi, (%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 56(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebx, 60(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, 48(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 52(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 40(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 44(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 32(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 36(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 24(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 28(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 16(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 20(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 8(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 12(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 4(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 52(%eax), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shldl %cl, %esi, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: negl %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 176(%esp,%ebp), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, 56(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edi, 60(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %ecx, %edx, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shldl %cl, %edx, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shldl %cl, %ebp, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shldl %cl, %edx, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebp, 48(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, 52(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebx, 40(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, 44(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, 32(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, 36(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, 24(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, 28(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, 16(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, 20(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, 8(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, 12(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edi, 4(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, (%eax)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addl $204, %esp
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: popl %esi
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: popl %edi
@@ -14755,14 +11919,14 @@ define void @shl_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: pushl %ebx
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: pushl %edi
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: pushl %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: subl $204, %esp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: subl $188, %esp
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movups (%ecx), %xmm0
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movups 16(%ecx), %xmm1
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movups 32(%ecx), %xmm2
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movups 48(%ecx), %xmm3
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl (%eax), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl (%eax), %ecx
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: xorps %xmm4, %xmm4
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm4, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm4, {{[0-9]+}}(%esp)
@@ -14772,195 +11936,92 @@ define void @shl_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: andl $60, %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: leal {{[0-9]+}}(%esp), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: subl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl (%ecx), %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 4(%ecx), %edx
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll $3, %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: andl $24, %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %al, %ch
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: notb %ch
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %esi, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: andl $60, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: leal {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: subl %ebp, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 8(%eax), %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 12(%eax), %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll $3, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: andl $24, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shldl %cl, %esi, %edi
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 12(%ebp), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 8(%ebp), %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebp, %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %ebx, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %esi, %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edi, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 20(%edi), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 16(%edi), %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %ebx, %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %esi, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 4(%eax), %edi
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebp, %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 28(%ebp), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shldl %cl, %edi, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 16(%eax), %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 20(%eax), %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shldl %cl, %edi, %ebx
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 24(%ebp), %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %ebx, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shldl %cl, %edx, %edi
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %esi, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 36(%edx), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 24(%eax), %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 28(%eax), %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shldl %cl, %edi, %ebx
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 32(%edx), %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %ebx, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shldl %cl, %esi, %edi
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %esi, %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 44(%ebp), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 32(%eax), %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 36(%eax), %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shldl %cl, %edi, %ebx
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 40(%ebp), %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %ebx, %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %esi, %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 52(%ebp), %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shldl %cl, %edx, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 40(%eax), %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 44(%eax), %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shldl %cl, %edx, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shldl %cl, %esi, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, (%esp) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 56(%eax), %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 60(%eax), %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shldl %cl, %edx, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl (%eax), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 52(%eax), %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shldl %cl, %esi, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: negl %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 160(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, 56(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edi, 60(%ebp)
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: negl %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 176(%esp,%edx), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %edi, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shldl %cl, %ebx, %edx
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %ebx, %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shldl %cl, %eax, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: # kill: def $cl killed $cl killed $ecx
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 60(%edi), %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 56(%edi), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %edx, %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %ebx, %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, (%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, 56(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edi, 60(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, 48(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebp, 52(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, 40(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, 44(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, 32(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, 36(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, 24(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, 28(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, 16(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, 20(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, 8(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, 12(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, 4(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: addl $204, %esp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shldl %cl, %edi, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 48(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, 52(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl (%esp), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 40(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 44(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 32(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 36(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 24(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 28(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 16(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 20(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 8(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 12(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, (%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, 4(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: addl $188, %esp
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: popl %esi
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: popl %edi
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: popl %ebx
@@ -15095,7 +12156,7 @@ define void @shl_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movups 16(%ecx), %xmm1
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movups 32(%ecx), %xmm2
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movups 48(%ecx), %xmm3
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl (%eax), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl (%eax), %ebp
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: xorps %xmm4, %xmm4
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm4, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm4, {{[0-9]+}}(%esp)
@@ -15105,150 +12166,92 @@ define void @shl_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leal (,%eax,8), %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: andl $24, %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ebx, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: andl $60, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leal {{[0-9]+}}(%esp), %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: subl %eax, %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl (%edx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leal (,%ebp,8), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: andl $24, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: andl $60, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leal {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: subl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 4(%eax), %esi
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 4(%edx), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: notb %bl
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrl %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ebx, %esi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %ecx, %eax, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %esi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 8(%eax), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 12(%eax), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shldl %cl, %edi, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shldl %cl, %esi, %edi
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 8(%edx), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrl %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ebx, %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 12(%edx), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %ecx, %esi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrl %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ebx, %ecx, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %eax, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 16(%edx), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrl %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ebx, %ecx, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 20(%edx), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %edi, %ecx, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %edi, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrl %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ebx, %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 24(%edx), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrl %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ebx, %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 28(%edx), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %edi, %esi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %edi, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrl %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ebx, %ecx, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 32(%edx), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrl %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ebx, %ecx, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 36(%edx), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %edi, %ecx, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %edi, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrl %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ebx, %esi, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %ebp, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 40(%edx), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edi, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrl %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ebx, %esi, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 44(%edx), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 16(%eax), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 20(%eax), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %esi, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shldl %cl, %edi, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shldl %cl, %edx, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 24(%eax), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 28(%eax), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shldl %cl, %edi, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shldl %cl, %esi, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 32(%eax), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 36(%eax), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %esi, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shldl %cl, %edi, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shldl %cl, %edx, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 40(%eax), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 44(%eax), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shldl %cl, %ebx, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shldl %cl, %esi, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 56(%eax), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 60(%eax), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shldl %cl, %edx, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl (%eax), %esi
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %eax, %esi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %ebp, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %eax, %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrl %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ebx, %eax, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 48(%edx), %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ebp, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrl %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ebx, %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 52(%edx), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %esi, %ecx, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %esi, %ebp, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %esi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrl %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ebx, %eax, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %edi, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 56(%edx), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrl %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ebx, %ecx, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %ebp, %edi, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %ecx, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrl %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ebx, %edi, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %ebp, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: negl %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %ebp, 188(%esp,%ebx), %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %ecx, %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edi, (%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 56(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ebx, 60(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %esi, 48(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 52(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 40(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 44(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 32(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 36(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 24(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 28(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 16(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 20(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 8(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 12(%edx)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 4(%edx)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 52(%eax), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shldl %cl, %esi, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: negl %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 176(%esp,%ebp), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, 56(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edi, 60(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %ecx, %edx, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shldl %cl, %edx, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shldl %cl, %ebp, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shldl %cl, %edx, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ebp, 48(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %esi, 52(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ebx, 40(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, 44(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, 32(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, 36(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, 24(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, 28(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, 16(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, 20(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, 8(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, 12(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edi, 4(%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, (%eax)
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: addl $204, %esp
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: popl %esi
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: popl %edi
@@ -15378,206 +12381,103 @@ define void @shl_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: pushl %ebx
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: pushl %edi
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: pushl %esi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: subl $204, %esp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: subl $188, %esp
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups (%ecx), %ymm0
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups 32(%ecx), %ymm1
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl (%eax), %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl (%eax), %ecx
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: vxorps %xmm2, %xmm2, %xmm2
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm2, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm2, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm1, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm0, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: andl $60, %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: leal {{[0-9]+}}(%esp), %ecx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: subl %edx, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl (%ecx), %edi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 4(%ecx), %edx
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll $3, %eax
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: andl $24, %eax
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edx, %esi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %esi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %edi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %al, %ch
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: notb %ch
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: orl %esi, %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: andl $60, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: leal {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: subl %ebp, %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 8(%eax), %esi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 12(%eax), %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll $3, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: andl $24, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edx, %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shldl %cl, %esi, %edi
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 12(%ebp), %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 8(%ebp), %esi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ebp, %edi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %ebp
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: orl %ebx, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %esi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: orl %esi, %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edi, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 20(%edi), %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 16(%edi), %esi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: orl %ebx, %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %esi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %edi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: orl %esi, %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 4(%eax), %edi
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ebp, %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 28(%ebp), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shldl %cl, %edi, %esi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 16(%eax), %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 20(%eax), %esi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shldl %cl, %edi, %ebx
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 24(%ebp), %esi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, %edi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %edi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: orl %ebx, %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shldl %cl, %edx, %edi
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %esi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %ebp
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: orl %esi, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 36(%edx), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 24(%eax), %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 28(%eax), %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edx, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shldl %cl, %edi, %ebx
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 32(%edx), %esi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edx, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, %edi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %edi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: orl %ebx, %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shldl %cl, %esi, %edi
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %esi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: orl %esi, %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 44(%ebp), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 32(%eax), %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 36(%eax), %esi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shldl %cl, %edi, %ebx
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 40(%ebp), %esi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: orl %ebx, %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %esi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: orl %esi, %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 52(%ebp), %esi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, %edi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shldl %cl, %edx, %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 40(%eax), %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 44(%eax), %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shldl %cl, %edx, %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shldl %cl, %esi, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edx, (%esp) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 56(%eax), %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 60(%eax), %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shldl %cl, %edx, %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl (%eax), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 52(%eax), %esi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shldl %cl, %esi, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: negl %ebp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 160(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edx, 56(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edi, 60(%ebp)
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: negl %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 176(%esp,%edx), %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ebx, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %ebp
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: orl %edi, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %al, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shldl %cl, %ebx, %edx
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: orl %ebx, %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shldl %cl, %eax, %esi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: # kill: def $cl killed $cl killed $ecx
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 60(%edi), %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 56(%edi), %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ebx, %edi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %edi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: orl %edx, %edi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %esi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %esi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: orl %ebx, %esi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edx, (%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, 56(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edi, 60(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, 48(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ebp, 52(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, 40(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, 44(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, 32(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, 36(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, 24(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, 28(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, 16(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, 20(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, 8(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, 12(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, 4(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: addl $204, %esp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shldl %cl, %edi, %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, 48(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, 52(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl (%esp), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, 40(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, 44(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, 32(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, 36(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, 24(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, 28(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, 16(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, 20(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, 8(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, 12(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ebx, (%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edx, 4(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: addl $188, %esp
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: popl %esi
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: popl %edi
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: popl %ebx
@@ -15706,156 +12606,98 @@ define void @shl_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups (%ecx), %ymm0
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups 32(%ecx), %ymm1
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl (%eax), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl (%eax), %ebx
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vxorps %xmm2, %xmm2, %xmm2
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups %ymm2, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups %ymm2, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups %ymm1, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups %ymm0, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: leal (,%eax,8), %ebx
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: andl $24, %ebx
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ebx, %ecx
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: andl $60, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: leal {{[0-9]+}}(%esp), %edx
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: subl %eax, %edx
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl (%edx), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: leal (,%ebx,8), %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: andl $24, %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: andl $60, %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: leal {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: subl %ebx, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 4(%eax), %esi
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 4(%edx), %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: notb %bl
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrl %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ebx, %esi, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %ecx, %eax, %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orl %esi, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 8(%eax), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 12(%eax), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %edx, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shldl %cl, %edi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shldl %cl, %esi, %edi
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 8(%edx), %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrl %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ebx, %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 12(%edx), %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %ecx, %esi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrl %ecx
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ebx, %ecx, %ecx
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orl %eax, %ecx
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 16(%edx), %ecx
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrl %ecx
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ebx, %ecx, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 20(%edx), %ecx
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %edi, %ecx, %ebp
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %edi, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrl %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ebx, %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 24(%edx), %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrl %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ebx, %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 28(%edx), %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %edi, %esi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %edi, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrl %ecx
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ebx, %ecx, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 32(%edx), %ecx
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrl %ecx
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ebx, %ecx, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 36(%edx), %ecx
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %edi, %ecx, %ebp
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %edi, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrl %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ebx, %esi, %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orl %ebp, %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 40(%edx), %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %edi, %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrl %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ebx, %esi, %ecx
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 44(%edx), %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %eax, %esi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orl %ebp, %ecx
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %eax, %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrl %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ebx, %eax, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orl %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 48(%edx), %ebp
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ebp, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrl %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ebx, %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 52(%edx), %ecx
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %esi, %ecx, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orl %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %esi, %ebp, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 16(%eax), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 20(%eax), %esi
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %esi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrl %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ebx, %eax, %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orl %edi, %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 56(%edx), %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrl %ecx
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ebx, %ecx, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %ebp, %edi, %ecx
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orl %ecx, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrl %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ebx, %edi, %ecx
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %ebp, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shldl %cl, %edi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shldl %cl, %edx, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 24(%eax), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 28(%eax), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %edx, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shldl %cl, %edi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shldl %cl, %esi, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 32(%eax), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 36(%eax), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %esi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shldl %cl, %edi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shldl %cl, %edx, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 40(%eax), %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 44(%eax), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shldl %cl, %ebp, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shldl %cl, %esi, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 56(%eax), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 60(%eax), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shldl %cl, %edx, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl (%eax), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 52(%eax), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shldl %cl, %esi, %edx
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: negl %ebx
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %ebp, 188(%esp,%ebx), %ebx
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orl %ecx, %ebx
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %edi, (%edx)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, 56(%edx)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ebx, 60(%edx)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %esi, 48(%edx)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, 52(%edx)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, 40(%edx)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, 44(%edx)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, 32(%edx)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, 36(%edx)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, 24(%edx)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, 28(%edx)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, 16(%edx)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, 20(%edx)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, 8(%edx)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, 12(%edx)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, 4(%edx)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 176(%esp,%ebx), %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %edx, 56(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %edi, 60(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %ecx, %edx, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shldl %cl, %edx, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shldl %cl, %ebx, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shldl %cl, %edx, %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ebx, 48(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %esi, 52(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ebp, 40(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, 44(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, 32(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, 36(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, 24(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, 28(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, 16(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, 20(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, 8(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, 12(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %edi, 4(%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, (%eax)
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: addl $204, %esp
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: popl %esi
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: popl %edi
@@ -16412,18 +13254,17 @@ define void @ashr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-NO-BMI2-SSE2: # %bb.0:
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: pushq %r15
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: pushq %r14
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: pushq %r13
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: pushq %r12
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: pushq %rbx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq (%rdi), %rax
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 8(%rdi), %rcx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 16(%rdi), %r8
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 24(%rdi), %r9
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 32(%rdi), %r10
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 40(%rdi), %r11
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 48(%rdi), %rbx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 56(%rdi), %r14
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl (%rsi), %edi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq (%rdi), %rcx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 8(%rdi), %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 16(%rdi), %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 24(%rdi), %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 32(%rdi), %r11
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 40(%rdi), %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 48(%rdi), %r14
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq 56(%rdi), %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl (%rsi), %eax
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r14, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rbx, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
@@ -16431,87 +13272,47 @@ define void @ashr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: sarq $63, %r14
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r14, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r14, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r14, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r14, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r14, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r14, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r14, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r14, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: leal (,%rdi,8), %eax
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: sarq $63, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: leal (,%rax,8), %ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: andl $56, %ecx
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: andl $56, %eax
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: andl $56, %edi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -128(%rsp,%rdi), %r10
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -120(%rsp,%rdi), %r8
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r8, %r11
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %cl, %r11
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %esi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: notb %sil
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -112(%rsp,%rdi), %rbx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: leaq (%rbx,%rbx), %r9
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlq %cl, %r9
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: orq %r11, %r9
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: addq %r8, %r8
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlq %cl, %r8
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: orq %r10, %r8
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -104(%rsp,%rdi), %r10
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r10, %r15
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %cl, %r15
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -96(%rsp,%rdi), %r14
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: leaq (%r14,%r14), %r11
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlq %cl, %r11
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: orq %r15, %r11
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %cl, %rbx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: addq %r10, %r10
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: orq %rbx, %r10
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -88(%rsp,%rdi), %rbx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rbx, %r12
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %cl, %r12
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -80(%rsp,%rdi), %r13
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: leaq (%r13,%r13), %r15
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlq %cl, %r15
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: orq %r12, %r15
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %cl, %r14
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: addq %rbx, %rbx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlq %cl, %rbx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: orq %r14, %rbx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrq %cl, %r13
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -72(%rsp,%rdi), %rdi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: leaq (%rdi,%rdi), %r14
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shlq %cl, %r14
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: orq %r13, %r14
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: sarq %cl, %rdi
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rdi, 56(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r14, 48(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rbx, 32(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -112(%rsp,%rax), %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -128(%rsp,%rax), %rsi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -120(%rsp,%rax), %r9
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r9, %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdq %cl, %rdi, %r8
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -96(%rsp,%rax), %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -104(%rsp,%rax), %r11
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r11, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdq %cl, %r10, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdq %cl, %r11, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -80(%rsp,%rax), %r11
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -88(%rsp,%rax), %r14
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r14, %r15
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdq %cl, %r11, %r15
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdq %cl, %r14, %r10
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq -72(%rsp,%rax), %rax
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdq %cl, %rax, %r11
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdq %cl, %r9, %rsi
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: # kill: def $cl killed $cl killed $ecx
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: sarq %cl, %rax
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r11, 48(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rax, 56(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r10, 32(%rdx)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r15, 40(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r10, 16(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r11, 24(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r8, (%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r9, 8(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rdi, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rbx, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %rsi, (%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: movq %r8, 8(%rdx)
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: popq %rbx
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: popq %r12
-; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: popq %r13
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: popq %r14
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: popq %r15
; X64-NO-SHLD-NO-BMI2-SSE2-NEXT: retq
@@ -16587,9 +13388,7 @@ define void @ashr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-HAVE-BMI2-SSE2: # %bb.0:
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: pushq %r15
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: pushq %r14
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: pushq %r12
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: pushq %rbx
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: pushq %rax
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq (%rdi), %rcx
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq 8(%rdi), %r8
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq 16(%rdi), %r9
@@ -16618,56 +13417,36 @@ define void @ashr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leal (,%rax,8), %ecx
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: andl $56, %ecx
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, %esi
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: andl $56, %eax
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -120(%rsp,%rax), %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -112(%rsp,%rax), %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxq %rsi, %r8, %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: notb %cl
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leaq (%r10,%r10), %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxq %rcx, %rdi, %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orq %r9, %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxq %rsi, -128(%rsp,%rax), %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addq %r8, %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxq %rcx, %r8, %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orq %r9, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -112(%rsp,%rax), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -128(%rsp,%rax), %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -120(%rsp,%rax), %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r9, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdq %cl, %rdi, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -96(%rsp,%rax), %r10
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -104(%rsp,%rax), %r11
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxq %rsi, %r11, %rbx
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -96(%rsp,%rax), %r14
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leaq (%r14,%r14), %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxq %rcx, %r9, %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orq %rbx, %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxq %rsi, %r10, %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addq %r11, %r11
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxq %rcx, %r11, %r11
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orq %r10, %r11
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -88(%rsp,%rax), %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxq %rsi, %r10, %rbx
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -80(%rsp,%rax), %r15
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leaq (%r15,%r15), %r12
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxq %rcx, %r12, %r12
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orq %rbx, %r12
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxq %rsi, %r14, %rbx
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addq %r10, %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxq %rcx, %r10, %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orq %rbx, %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxq %rsi, %r15, %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r11, %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdq %cl, %r10, %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdq %cl, %r11, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -80(%rsp,%rax), %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -88(%rsp,%rax), %r14
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r14, %r15
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdq %cl, %r11, %r15
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdq %cl, %r14, %r10
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq -72(%rsp,%rax), %rax
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leaq (%rax,%rax), %r14
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxq %rcx, %r14, %rcx
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orq %rbx, %rcx
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: sarxq %rsi, %rax, %rax
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rax, 56(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rcx, 48(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdq %cl, %rax, %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: sarxq %rcx, %rax, %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: # kill: def $cl killed $cl killed $rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdq %cl, %r9, %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r11, 48(%rdx)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r10, 32(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r12, 40(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r11, 16(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r9, 24(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r8, (%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rdi, 8(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addq $8, %rsp
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r15, 40(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rdi, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rbx, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rsi, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %r8, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movq %rax, 56(%rdx)
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: popq %rbx
-; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: popq %r12
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: popq %r14
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: popq %r15
; X64-NO-SHLD-HAVE-BMI2-SSE2-NEXT: retq
@@ -16741,108 +13520,64 @@ define void @ashr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
;
; X64-NO-SHLD-NO-BMI2-SSE4-LABEL: ashr_64bytes:
; X64-NO-SHLD-NO-BMI2-SSE4: # %bb.0:
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: pushq %rbp
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: pushq %r15
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: pushq %r14
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: pushq %r13
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: pushq %r12
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: pushq %rbx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: pushq %rax
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movups (%rdi), %xmm0
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movups 16(%rdi), %xmm1
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movups 32(%rdi), %xmm2
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq 48(%rdi), %rax
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq 56(%rdi), %rcx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl (%rsi), %edi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq 48(%rdi), %rcx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq 56(%rdi), %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl (%rsi), %eax
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: sarq $63, %rcx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: leal (,%rdi,8), %eax
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: sarq $63, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: leal (,%rax,8), %ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: andl $56, %ecx
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: andl $56, %eax
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: andl $56, %edi
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -128(%rsp,%rdi), %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -120(%rsp,%rdi), %r9
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %esi
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: notb %sil
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: leaq (%r9,%r9), %r8
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlq %cl, %r8
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: orq %r10, %r8
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -104(%rsp,%rdi), %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r10, %rbx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %cl, %rbx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -96(%rsp,%rdi), %r12
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: leaq (%r12,%r12), %r11
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlq %cl, %r11
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: orq %rbx, %r11
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -112(%rsp,%rdi), %rbx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rbx, %r14
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %cl, %r14
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: addq %r10, %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: orq %r14, %r10
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -88(%rsp,%rdi), %r14
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r14, %r13
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %cl, %r13
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -80(%rsp,%rdi), %rbp
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: leaq (%rbp,%rbp), %r15
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlq %cl, %r15
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: orq %r13, %r15
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %cl, %r12
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: addq %r14, %r14
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlq %cl, %r14
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: orq %r12, %r14
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %cl, %rbp
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -72(%rsp,%rdi), %rdi
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: leaq (%rdi,%rdi), %r12
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlq %cl, %r12
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: orq %rbp, %r12
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrq %cl, %r9
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: addq %rbx, %rbx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shlq %cl, %rbx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: orq %r9, %rbx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: sarq %cl, %rdi
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rdi, 56(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rbx, 8(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r12, 48(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r14, 32(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r15, 40(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r10, 16(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r11, 24(%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r8, (%rdx)
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: addq $8, %rsp
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -96(%rsp,%rax), %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -104(%rsp,%rax), %r9
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r9, %rsi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdq %cl, %rdi, %rsi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -112(%rsp,%rax), %r10
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r10, %r8
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdq %cl, %r9, %r8
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -80(%rsp,%rax), %r9
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -88(%rsp,%rax), %r11
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r11, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdq %cl, %r9, %rbx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdq %cl, %r11, %rdi
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -72(%rsp,%rax), %r11
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdq %cl, %r11, %r9
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -128(%rsp,%rax), %r14
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq -120(%rsp,%rax), %rax
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rax, %r15
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdq %cl, %r10, %r15
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdq %cl, %rax, %r14
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: # kill: def $cl killed $cl killed $ecx
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: sarq %cl, %r11
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r15, 8(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r9, 48(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r11, 56(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rdi, 32(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rbx, 40(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r8, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %rsi, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: movq %r14, (%rdx)
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: popq %rbx
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: popq %r12
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: popq %r13
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: popq %r14
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: popq %r15
-; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: popq %rbp
; X64-NO-SHLD-NO-BMI2-SSE4-NEXT: retq
;
; X64-HAVE-SHLD-NO-BMI2-SSE4-LABEL: ashr_64bytes:
@@ -16911,8 +13646,6 @@ define void @ashr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-HAVE-BMI2-SSE4: # %bb.0:
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: pushq %r15
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: pushq %r14
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: pushq %r13
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: pushq %r12
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: pushq %rbx
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movups (%rdi), %xmm0
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movups 16(%rdi), %xmm1
@@ -16936,56 +13669,37 @@ define void @ashr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leal (,%rax,8), %ecx
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: andl $56, %ecx
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, %esi
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: andl $56, %eax
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxq %rsi, -128(%rsp,%rax), %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: notb %cl
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -120(%rsp,%rax), %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -112(%rsp,%rax), %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leaq (%r10,%r10), %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxq %rcx, %rdi, %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orq %r8, %rdi
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -104(%rsp,%rax), %r11
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxq %rsi, %r11, %rbx
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -96(%rsp,%rax), %r14
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leaq (%r14,%r14), %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxq %rcx, %r8, %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orq %rbx, %r8
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxq %rsi, %r9, %rbx
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: addq %r11, %r11
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxq %rcx, %r11, %r11
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orq %rbx, %r11
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -88(%rsp,%rax), %rbx
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxq %rsi, %rbx, %r15
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -80(%rsp,%rax), %r12
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leaq (%r12,%r12), %r13
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxq %rcx, %r13, %r13
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orq %r15, %r13
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxq %rsi, %r14, %r14
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: addq %rbx, %rbx
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxq %rcx, %rbx, %rbx
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orq %r14, %rbx
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxq %rsi, %r12, %r14
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -72(%rsp,%rax), %rax
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leaq (%rax,%rax), %r15
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxq %rcx, %r15, %r15
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orq %r14, %r15
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxq %rsi, %r10, %r10
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: addq %r9, %r9
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxq %rcx, %r9, %rcx
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orq %r10, %rcx
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: sarxq %rsi, %rax, %rax
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rax, 56(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rcx, 8(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r15, 48(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rbx, 32(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r13, 40(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r11, 16(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r8, 24(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rdi, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -96(%rsp,%rax), %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -104(%rsp,%rax), %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r9, %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdq %cl, %rdi, %rsi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -112(%rsp,%rax), %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r10, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdq %cl, %r9, %r8
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -80(%rsp,%rax), %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -88(%rsp,%rax), %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r11, %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdq %cl, %r9, %rbx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdq %cl, %r11, %rdi
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -72(%rsp,%rax), %r11
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdq %cl, %r11, %r9
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -128(%rsp,%rax), %r14
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq -120(%rsp,%rax), %rax
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rax, %r15
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdq %cl, %r10, %r15
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: sarxq %rcx, %r11, %r10
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: # kill: def $cl killed $cl killed $rcx
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdq %cl, %rax, %r14
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r15, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r9, 48(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rdi, 32(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rbx, 40(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r8, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %rsi, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r14, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movq %r10, 56(%rdx)
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: popq %rbx
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: popq %r12
-; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: popq %r13
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: popq %r14
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: popq %r15
; X64-NO-SHLD-HAVE-BMI2-SSE4-NEXT: retq
@@ -17054,106 +13768,62 @@ define void @ashr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
;
; X64-NO-SHLD-NO-BMI2-AVX1-LABEL: ashr_64bytes:
; X64-NO-SHLD-NO-BMI2-AVX1: # %bb.0:
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: pushq %rbp
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: pushq %r15
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: pushq %r14
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: pushq %r13
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: pushq %r12
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: pushq %rbx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: pushq %rax
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups (%rdi), %ymm0
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups 32(%rdi), %xmm1
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq 48(%rdi), %rax
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq 56(%rdi), %rcx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl (%rsi), %edi
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq 48(%rdi), %rcx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq 56(%rdi), %rdi
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl (%rsi), %eax
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovaps %xmm1, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: sarq $63, %rcx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: leal (,%rdi,8), %eax
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: sarq $63, %rdi
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: leal (,%rax,8), %ecx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: andl $56, %ecx
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: andl $56, %eax
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: andl $56, %edi
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -128(%rsp,%rdi), %r10
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -120(%rsp,%rdi), %r9
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %esi
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: notb %sil
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: leaq (%r9,%r9), %r8
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shlq %cl, %r8
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: orq %r10, %r8
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -104(%rsp,%rdi), %r10
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r10, %rbx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrq %cl, %rbx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -96(%rsp,%rdi), %r12
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: leaq (%r12,%r12), %r11
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shlq %cl, %r11
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: orq %rbx, %r11
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -112(%rsp,%rdi), %rbx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %rbx, %r14
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrq %cl, %r14
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: addq %r10, %r10
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shlq %cl, %r10
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: orq %r14, %r10
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -88(%rsp,%rdi), %r14
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r14, %r13
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrq %cl, %r13
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -80(%rsp,%rdi), %rbp
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: leaq (%rbp,%rbp), %r15
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shlq %cl, %r15
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: orq %r13, %r15
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrq %cl, %r12
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: addq %r14, %r14
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shlq %cl, %r14
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: orq %r12, %r14
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrq %cl, %rbp
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -72(%rsp,%rdi), %rdi
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: leaq (%rdi,%rdi), %r12
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shlq %cl, %r12
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: orq %rbp, %r12
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrq %cl, %r9
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: addq %rbx, %rbx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shlq %cl, %rbx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: orq %r9, %rbx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: sarq %cl, %rdi
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %rdi, 56(%rdx)
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %rbx, 8(%rdx)
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r12, 48(%rdx)
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r14, 32(%rdx)
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r15, 40(%rdx)
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r10, 16(%rdx)
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r11, 24(%rdx)
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r8, (%rdx)
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: addq $8, %rsp
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -96(%rsp,%rax), %rdi
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -104(%rsp,%rax), %r9
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r9, %rsi
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrdq %cl, %rdi, %rsi
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -112(%rsp,%rax), %r10
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r10, %r8
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrdq %cl, %r9, %r8
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -80(%rsp,%rax), %r9
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -88(%rsp,%rax), %r11
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r11, %rbx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrdq %cl, %r9, %rbx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrdq %cl, %r11, %rdi
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -72(%rsp,%rax), %r11
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrdq %cl, %r11, %r9
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -128(%rsp,%rax), %r14
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -120(%rsp,%rax), %rax
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %rax, %r15
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrdq %cl, %r10, %r15
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrdq %cl, %rax, %r14
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: # kill: def $cl killed $cl killed $ecx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: sarq %cl, %r11
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r15, 8(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r9, 48(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r11, 56(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %rdi, 32(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %rbx, 40(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r8, 16(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %rsi, 24(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r14, (%rdx)
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: popq %rbx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: popq %r12
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: popq %r13
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: popq %r14
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: popq %r15
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: popq %rbp
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: vzeroupper
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: retq
;
@@ -17222,8 +13892,6 @@ define void @ashr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-HAVE-BMI2-AVX1: # %bb.0:
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: pushq %r15
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: pushq %r14
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: pushq %r13
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: pushq %r12
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: pushq %rbx
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups (%rdi), %ymm0
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups 32(%rdi), %xmm1
@@ -17245,56 +13913,37 @@ define void @ashr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: leal (,%rax,8), %ecx
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: andl $56, %ecx
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, %esi
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: andl $56, %eax
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxq %rsi, -128(%rsp,%rax), %r8
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: notb %cl
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -120(%rsp,%rax), %r10
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -112(%rsp,%rax), %r9
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: leaq (%r10,%r10), %rdi
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxq %rcx, %rdi, %rdi
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orq %r8, %rdi
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -104(%rsp,%rax), %r11
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxq %rsi, %r11, %rbx
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -96(%rsp,%rax), %r14
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: leaq (%r14,%r14), %r8
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxq %rcx, %r8, %r8
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orq %rbx, %r8
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxq %rsi, %r9, %rbx
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: addq %r11, %r11
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxq %rcx, %r11, %r11
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orq %rbx, %r11
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -88(%rsp,%rax), %rbx
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxq %rsi, %rbx, %r15
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -80(%rsp,%rax), %r12
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: leaq (%r12,%r12), %r13
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxq %rcx, %r13, %r13
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orq %r15, %r13
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxq %rsi, %r14, %r14
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: addq %rbx, %rbx
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxq %rcx, %rbx, %rbx
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orq %r14, %rbx
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxq %rsi, %r12, %r14
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -72(%rsp,%rax), %rax
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: leaq (%rax,%rax), %r15
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxq %rcx, %r15, %r15
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orq %r14, %r15
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxq %rsi, %r10, %r10
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: addq %r9, %r9
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxq %rcx, %r9, %rcx
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orq %r10, %rcx
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: sarxq %rsi, %rax, %rax
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %rax, 56(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %rcx, 8(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %r15, 48(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %rbx, 32(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %r13, 40(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %r11, 16(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %r8, 24(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %rdi, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -96(%rsp,%rax), %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -104(%rsp,%rax), %r9
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %r9, %rsi
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrdq %cl, %rdi, %rsi
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -112(%rsp,%rax), %r10
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %r10, %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrdq %cl, %r9, %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -80(%rsp,%rax), %r9
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -88(%rsp,%rax), %r11
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %r11, %rbx
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrdq %cl, %r9, %rbx
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrdq %cl, %r11, %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -72(%rsp,%rax), %r11
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrdq %cl, %r11, %r9
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -128(%rsp,%rax), %r14
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -120(%rsp,%rax), %rax
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %rax, %r15
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrdq %cl, %r10, %r15
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: sarxq %rcx, %r11, %r10
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: # kill: def $cl killed $cl killed $rcx
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrdq %cl, %rax, %r14
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %r15, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %r9, 48(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %rdi, 32(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %rbx, 40(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %r8, 16(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %rsi, 24(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %r14, (%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %r10, 56(%rdx)
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: popq %rbx
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: popq %r12
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: popq %r13
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: popq %r14
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: popq %r15
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vzeroupper
@@ -17424,264 +14073,165 @@ define void @ashr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: pushl %ebx
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: pushl %edi
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: pushl %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: subl $204, %esp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: subl $188, %esp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl (%eax), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 4(%eax), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 8(%eax), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 12(%eax), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 16(%eax), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 20(%eax), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 24(%eax), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 28(%eax), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 32(%eax), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 36(%eax), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, (%esp) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 40(%eax), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 44(%eax), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 48(%eax), %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 52(%eax), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 56(%eax), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 60(%eax), %eax
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl (%ecx), %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 4(%ecx), %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 8(%ecx), %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 12(%ecx), %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 16(%ecx), %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 20(%ecx), %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 24(%ecx), %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 28(%ecx), %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 32(%ecx), %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 36(%ecx), %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 40(%ecx), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 44(%ecx), %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 48(%ecx), %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 52(%ecx), %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 56(%ecx), %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 60(%ecx), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl (%ebp), %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl (%ecx), %ecx
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebp, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl (%esp), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: sarl $31, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: sarl $31, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebp, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebp, %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: andl $60, %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 68(%esp,%esi), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: andl $60, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 56(%esp,%ebp), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 52(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll $3, %ecx
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: andl $24, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 72(%esp,%esi), %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: addl %edi, %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %ch # 1-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: notb %ch
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %eax, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %edx, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 64(%esp,%ebp), %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 60(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %edi, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %eax, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 72(%esp,%ebp), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 68(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %esi, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %eax, %edi
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 64(%esp,%esi), %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %bl, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: addl %edx, %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %eax, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 80(%esp,%ebp), %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 76(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %edi, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %eax, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 88(%esp,%ebp), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 84(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %esi, %edx
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 76(%esp,%esi), %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebp, %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %bl, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 80(%esp,%esi), %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: leal (%edi,%edi), %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %edx, %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %bl, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: addl %ebp, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %eax, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 84(%esp,%esi), %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %bl, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 88(%esp,%esi), %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: leal (%esi,%esi), %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %eax, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %bl, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: addl %ebx, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %edi, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %eax, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, (%esp) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 96(%esp,%ebp), %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 92(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %esi, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %eax, %edx
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 92(%esp,%edx), %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebp, %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %bl, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 96(%esp,%eax), %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: leal (%edi,%edi), %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %edx, %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %bl, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: addl %ebp, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %esi, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 100(%esp,%edx), %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %bl, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 104(%esp,%edx), %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: leal (%esi,%esi), %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %eax, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %dl, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: addl %ebx, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %edi, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 108(%esp,%ebp), %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 112(%esp,%ebp), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebp, %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: leal (%ecx,%ecx), %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %ch # 1-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %eax, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %bl, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: addl %edi, %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %esi, %edi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 116(%esp,%edx), %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 120(%esp,%edx), %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: leal (%edx,%edx), %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %ch # 1-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %eax, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %bl, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: addl %esi, %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %eax, %esi
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %bl, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 104(%esp,%ebp), %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 100(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %edx, %edi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %eax, %esi
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 48(%esp,%ebp), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 108(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %eax, %edx
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, 56(%ebp)
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl 124(%esp,%edx), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: leal (%ebx,%ebx), %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shll %cl, %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: orl %eax, %edx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: shrdl %cl, %edx, %ebx
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: sarl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, 60(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edx, 56(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, 48(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebp, 52(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, 40(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, 44(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, 32(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, 36(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, 24(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, 28(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, 16(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, 20(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, 8(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, 12(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, (%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ecx, 4(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: addl $204, %esp
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: sarl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, 60(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %esi, 48(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %edi, 52(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, 40(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, 44(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl (%esp), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, 32(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, 36(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, 24(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, 28(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, 16(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, 20(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, 8(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, 12(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %ebx, (%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: movl %eax, 4(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: addl $188, %esp
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: popl %esi
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: popl %edi
; X86-NO-SHLD-NO-BMI2-SSE2-NEXT: popl %ebx
@@ -17865,7 +14415,7 @@ define void @ashr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: pushl %ebx
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: pushl %edi
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: pushl %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: subl $204, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: subl $188, %esp
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl (%eax), %ecx
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
@@ -17886,21 +14436,21 @@ define void @ashr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 32(%eax), %ecx
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 36(%eax), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, (%esp) # 4-byte Spill
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 40(%eax), %ebp
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 44(%eax), %ebx
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 48(%eax), %edi
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 52(%eax), %esi
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 56(%eax), %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 60(%eax), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl (%eax), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 60(%eax), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl (%ecx), %ecx
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebx, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebp, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl (%esp), %edx # 4-byte Reload
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, {{[0-9]+}}(%esp)
@@ -17910,7 +14460,7 @@ define void @ashr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
@@ -17921,155 +14471,107 @@ define void @ashr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: sarl $31, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leal (,%eax,8), %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: andl $24, %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: andl $60, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 68(%esp,%ecx), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 72(%esp,%ecx), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ebx, %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: notb %dl
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leal (%edi,%edi), %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %edx, %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ebx, 64(%esp,%ecx), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addl %esi, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %edx, %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 80(%esp,%ecx), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leal (%esi,%esi), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %edx, %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 76(%esp,%ecx), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ebx, %edi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addl %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %edx, %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %eax, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 88(%esp,%ecx), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leal (%eax,%eax), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %edx, %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 84(%esp,%ecx), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ebx, %edi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ebx, %esi, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addl %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %edx, %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 96(%esp,%ecx), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leal (%esi,%esi), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %edx, %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 92(%esp,%ecx), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ebx, %edi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addl %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %edx, %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %eax, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 104(%esp,%ecx), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leal (%eax,%eax), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %edx, %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 100(%esp,%ecx), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ebx, %edi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ebx, %esi, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addl %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %edx, %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: sarl $31, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 112(%esp,%ecx), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leal (%eax,%eax), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %edx, %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 108(%esp,%ecx), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ebx, %esi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: andl $60, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 56(%esp,%ebp), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 52(%esp,%ebp), %eax
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addl %esi, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %edx, %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %ecx, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shll $3, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: andl $24, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %edx, %eax
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 120(%esp,%ebp), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leal (%edi,%edi), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %edx, %ecx, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 116(%esp,%ebp), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ebx, %eax, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %ebp, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addl %eax, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %edx, %eax, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %ebp, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 64(%esp,%ebp), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 60(%esp,%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %edi, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 72(%esp,%ebp), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 68(%esp,%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %esi, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 80(%esp,%ebp), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 76(%esp,%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %edi, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %eax, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 88(%esp,%ebp), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 84(%esp,%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %ebx, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edi, (%esp) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 96(%esp,%ebp), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 92(%esp,%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %esi, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %eax, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 104(%esp,%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 100(%esp,%ebp), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edi, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %edi, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 48(%esp,%ebp), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 108(%esp,%ebp), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 56(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, 48(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, 52(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ebx, 40(%ebp)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl 124(%esp,%eax), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: leal (%eax,%eax), %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shlxl %edx, %ebp, %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrxl %ebx, %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: orl %edi, %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: sarxl %ebx, %eax, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edi, 60(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edx, 56(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, 48(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %esi, 52(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, 40(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, 44(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, 32(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, 36(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, 24(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, 28(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, 16(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, 20(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, 8(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, 12(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, (%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 44(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl (%esp), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 32(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 36(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 24(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 28(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 16(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 20(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 8(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 12(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: sarxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: shrdl %cl, %edx, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %edi, (%ebp)
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, 4(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addl $204, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %ecx, 4(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: movl %eax, 60(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: addl $188, %esp
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: popl %esi
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: popl %edi
; X86-NO-SHLD-HAVE-BMI2-SSE2-NEXT: popl %ebx
@@ -18251,218 +14753,127 @@ define void @ashr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: pushl %ebx
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: pushl %edi
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: pushl %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: subl $204, %esp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: subl $188, %esp
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movups (%ecx), %xmm0
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movups 16(%ecx), %xmm1
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movups 32(%ecx), %xmm2
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 48(%ecx), %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 52(%ecx), %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 56(%ecx), %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 60(%ecx), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl (%eax), %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movups (%eax), %xmm0
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movups 16(%eax), %xmm1
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movups 32(%eax), %xmm2
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 48(%eax), %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 52(%eax), %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 56(%eax), %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 60(%eax), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl (%ecx), %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: sarl $31, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: sarl $31, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: andl $60, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 56(%esp,%ebp), %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 52(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll $3, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: andl $24, %ecx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %edx, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 64(%esp,%ebp), %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 60(%esp,%ebp), %eax
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: andl $60, %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 68(%esp,%esi), %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll $3, %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: andl $24, %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 72(%esp,%esi), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: leal (%ecx,%ecx), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %al, %ch
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: notb %ch
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %edi, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 64(%esp,%esi), %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: addl %edx, %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %edi, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %edi, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %eax, %edx
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 76(%esp,%esi), %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 80(%esp,%esi), %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: leal (%edi,%edi), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %ebp, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: addl %edx, %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %ebx, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 72(%esp,%ebp), %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 68(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %esi, %edx
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 84(%esp,%esi), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %eax, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 80(%esp,%ebp), %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 76(%esp,%ebp), %eax
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %dl, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 88(%esp,%esi), %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: addl %eax, %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %dl, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: addl %ebx, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %edi, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 92(%esp,%esi), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %dl, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 96(%esp,%esi), %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: leal (%edi,%edi), %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %edi, %edx
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: addl %ebx, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %eax, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 100(%esp,%esi), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %dl, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 104(%esp,%esi), %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: leal (%edx,%edx), %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: addl %ebx, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %edi, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 108(%esp,%esi), %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edi, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 112(%esp,%esi), %ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: leal (%ecx,%ecx), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %ch # 1-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %ebp, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: addl %edi, %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %edx, %edi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %eax, %esi
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 116(%esp,%esi), %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 120(%esp,%edx), %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: leal (%eax,%eax), %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %ebx, %ebp
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %dl, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: addl %esi, %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %ebx, %esi
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %dl, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 88(%esp,%ebp), %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 84(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %esi, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %eax, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 96(%esp,%ebp), %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 92(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %esi, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %eax, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, (%esp) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 104(%esp,%ebp), %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 100(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %edx, %edi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %eax, %esi
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 48(%esp,%ebp), %ebx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 108(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %eax, %edx
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, 56(%ebp)
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl 124(%esp,%edx), %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: leal (%ebx,%ebx), %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shll %cl, %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: orl %eax, %edx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: shrdl %cl, %edx, %ebx
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: sarl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, 60(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edx, 56(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, 48(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebp, 52(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edi, 40(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, 44(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, 32(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, 36(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, 24(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, 28(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, 16(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, 20(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, 8(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, 12(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, (%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ecx, 4(%eax)
-; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: addl $204, %esp
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: sarl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 60(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %esi, 48(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %edi, 52(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl (%esp), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 40(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 44(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 32(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 36(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 24(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 28(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 16(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 20(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 8(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 12(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %ebx, (%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: movl %eax, 4(%ebp)
+; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: addl $188, %esp
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: popl %esi
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: popl %edi
; X86-NO-SHLD-NO-BMI2-SSE4-NEXT: popl %ebx
@@ -18608,173 +15019,125 @@ define void @ashr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: pushl %ebx
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: pushl %edi
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: pushl %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: subl $204, %esp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: subl $188, %esp
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movups (%ecx), %xmm0
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movups 16(%ecx), %xmm1
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movups 32(%ecx), %xmm2
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 48(%ecx), %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 52(%ecx), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 56(%ecx), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 60(%ecx), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl (%eax), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movups (%eax), %xmm0
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movups 16(%eax), %xmm1
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movups 32(%eax), %xmm2
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 48(%eax), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 52(%eax), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 56(%eax), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 60(%eax), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl (%ecx), %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %esi, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: sarl $31, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leal (,%eax,8), %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: andl $24, %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, %ebx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: andl $60, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 68(%esp,%ecx), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 72(%esp,%ecx), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ebx, %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: notb %dl
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leal (%edi,%edi), %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %edx, %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ebx, 64(%esp,%ecx), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: addl %esi, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %edx, %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: sarl $31, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: andl $60, %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 56(%esp,%ebp), %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 52(%esp,%ebp), %eax
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 80(%esp,%ecx), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leal (%esi,%esi), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %edx, %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 76(%esp,%ecx), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ebx, %edi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shll $3, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: andl $24, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %edx, %eax
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: addl %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %edx, %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 64(%esp,%ebp), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 60(%esp,%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %edi, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 72(%esp,%ebp), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 68(%esp,%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %esi, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %eax, %edi
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 88(%esp,%ecx), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leal (%eax,%eax), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %edx, %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 84(%esp,%ecx), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ebx, %edi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ebx, %esi, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: addl %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %edx, %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 96(%esp,%ecx), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leal (%esi,%esi), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %edx, %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 92(%esp,%ecx), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ebx, %edi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: addl %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %edx, %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 80(%esp,%ebp), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 76(%esp,%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %edi, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %eax, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 88(%esp,%ebp), %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 84(%esp,%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %ebx, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %eax, %edi
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 104(%esp,%ecx), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leal (%eax,%eax), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %edx, %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 100(%esp,%ecx), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ebx, %edi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ebx, %esi, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: addl %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %edx, %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 112(%esp,%ecx), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leal (%eax,%eax), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %edx, %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 108(%esp,%ecx), %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ebx, %esi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: addl %esi, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %edx, %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %ecx, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 120(%esp,%ebp), %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leal (%edi,%edi), %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %edx, %ecx, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 116(%esp,%ebp), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ebx, %eax, %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %ebp, %esi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: addl %eax, %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %edx, %eax, %ecx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %ebp, %ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 96(%esp,%ebp), %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 92(%esp,%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %esi, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %eax, %ebx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 104(%esp,%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 100(%esp,%ebp), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edi, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %edi, %esi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 48(%esp,%ebp), %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 108(%esp,%ebp), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ebp, (%esp) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 56(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %esi, 48(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, 52(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ebx, 40(%ebp)
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl 124(%esp,%eax), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: leal (%eax,%eax), %ebp
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shlxl %edx, %ebp, %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrxl %ebx, %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: orl %edi, %edx
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: sarxl %ebx, %eax, %edi
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edi, 60(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edx, 56(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, 48(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %esi, 52(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, 40(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, 44(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, 32(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, 36(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, 24(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, 28(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, 16(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, 20(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, 8(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, 12(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, (%eax)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 44(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 32(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 36(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 24(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 28(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 16(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 20(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 8(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 12(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: sarxl %ecx, (%esp), %eax # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: shrdl %cl, %edx, %edi
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %edi, (%ebp)
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, 4(%eax)
-; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: addl $204, %esp
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %ecx, 4(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: movl %eax, 60(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: addl $188, %esp
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: popl %esi
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: popl %edi
; X86-NO-SHLD-HAVE-BMI2-SSE4-NEXT: popl %ebx
@@ -18918,216 +15281,125 @@ define void @ashr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: pushl %ebx
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: pushl %edi
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: pushl %esi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: subl $204, %esp
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: subl $188, %esp
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups (%ecx), %ymm0
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups 32(%ecx), %xmm1
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 48(%ecx), %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 52(%ecx), %esi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 56(%ecx), %edi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 60(%ecx), %ecx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl (%eax), %eax
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups (%eax), %ymm0
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups 32(%eax), %xmm1
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 48(%eax), %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 52(%eax), %esi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 56(%eax), %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 60(%eax), %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl (%ecx), %ecx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovaps %xmm1, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm0, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: sarl $31, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: sarl $31, %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: andl $60, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 56(%esp,%ebp), %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 52(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll $3, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: andl $24, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrdl %cl, %edx, %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 64(%esp,%ebp), %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 60(%esp,%ebp), %eax
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %esi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: andl $60, %esi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 68(%esp,%esi), %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll $3, %eax
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: andl $24, %eax
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edx, %edi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 72(%esp,%esi), %ecx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: leal (%ecx,%ecx), %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %al, %ch
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: notb %ch
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: orl %edi, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 64(%esp,%esi), %edi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: addl %edx, %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: orl %edi, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrdl %cl, %edi, %esi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrdl %cl, %eax, %edx
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 76(%esp,%esi), %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edx, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 80(%esp,%esi), %edi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: leal (%edi,%edi), %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: orl %ebp, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: addl %edx, %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: orl %ebx, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 72(%esp,%ebp), %esi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 68(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrdl %cl, %esi, %edx
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 84(%esp,%esi), %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ebx, %ebp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrdl %cl, %eax, %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 80(%esp,%ebp), %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 76(%esp,%ebp), %eax
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %dl, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 88(%esp,%esi), %eax
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: addl %eax, %eax
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %eax
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %dl, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: addl %ebx, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: orl %edi, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 92(%esp,%esi), %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ebx, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %dl, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 96(%esp,%esi), %edi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: leal (%edi,%edi), %eax
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %eax
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %dl, %cl
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrdl %cl, %edi, %edx
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %eax
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: addl %ebx, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: orl %eax, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 100(%esp,%esi), %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ebx, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %dl, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 104(%esp,%esi), %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: leal (%edx,%edx), %eax
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %eax
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: addl %ebx, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: orl %edi, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 108(%esp,%esi), %edi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edi, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 112(%esp,%esi), %ecx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: leal (%ecx,%ecx), %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %ch # 1-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: orl %ebp, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: addl %edi, %edi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %edi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: orl %edx, %edi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrdl %cl, %eax, %esi
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 116(%esp,%esi), %esi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %al, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 120(%esp,%edx), %eax
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: leal (%eax,%eax), %ebp
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: orl %ebx, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %dl, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: addl %esi, %esi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %esi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: orl %ebx, %esi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %dl, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 88(%esp,%ebp), %esi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 84(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrdl %cl, %esi, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrdl %cl, %eax, %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 96(%esp,%ebp), %esi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 92(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrdl %cl, %esi, %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrdl %cl, %eax, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edx, (%esp) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 104(%esp,%ebp), %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 100(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrdl %cl, %edx, %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrdl %cl, %eax, %esi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 48(%esp,%ebp), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 108(%esp,%ebp), %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrdl %cl, %eax, %edx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edx, 56(%ebp)
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 124(%esp,%edx), %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: leal (%ebx,%ebx), %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, %cl
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: orl %eax, %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrdl %cl, %edx, %ebx
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: sarl %cl, %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ebx, 60(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edx, 56(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, 48(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ebp, 52(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edi, 40(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, 44(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, 32(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, 36(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, 24(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, 28(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, 16(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, 20(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, 8(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, 12(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, (%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, 4(%eax)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: addl $204, %esp
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: sarl %cl, %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, 60(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, 48(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edi, 52(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl (%esp), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, 40(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, 44(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, 32(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, 36(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, 24(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, 28(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, 16(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, 20(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, 8(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, 12(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ebx, (%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, 4(%ebp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: addl $188, %esp
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: popl %esi
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: popl %edi
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: popl %ebx
@@ -19273,171 +15545,123 @@ define void @ashr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: pushl %ebx
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: pushl %edi
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: pushl %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: subl $204, %esp
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: subl $188, %esp
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups (%ecx), %ymm0
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups 32(%ecx), %xmm1
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 48(%ecx), %edx
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 52(%ecx), %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 56(%ecx), %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 60(%ecx), %ecx
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl (%eax), %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups (%eax), %ymm0
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups 32(%eax), %xmm1
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 48(%eax), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 52(%eax), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 56(%eax), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 60(%eax), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl (%ecx), %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %esi, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovaps %xmm1, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups %ymm0, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: sarl $31, %ecx
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, %ecx
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: leal (,%eax,8), %edx
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: andl $24, %edx
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %edx, %ebx
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: andl $60, %ecx
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 68(%esp,%ecx), %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 72(%esp,%ecx), %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ebx, %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: notb %dl
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: leal (%edi,%edi), %ebp
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %edx, %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ebx, 64(%esp,%ecx), %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: addl %esi, %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %edx, %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orl %edi, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: sarl $31, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: andl $60, %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 56(%esp,%ebp), %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 52(%esp,%ebp), %eax
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 80(%esp,%ecx), %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: leal (%esi,%esi), %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %edx, %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 76(%esp,%ecx), %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ebx, %edi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orl %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shll $3, %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: andl $24, %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrdl %cl, %edx, %eax
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: addl %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %edx, %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orl %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 64(%esp,%ebp), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 60(%esp,%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrdl %cl, %edi, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrdl %cl, %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 72(%esp,%ebp), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 68(%esp,%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrdl %cl, %esi, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrdl %cl, %eax, %edi
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 88(%esp,%ecx), %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: leal (%eax,%eax), %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %edx, %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 84(%esp,%ecx), %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ebx, %edi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ebx, %esi, %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: addl %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %edx, %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orl %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 96(%esp,%ecx), %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: leal (%esi,%esi), %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %edx, %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 92(%esp,%ecx), %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ebx, %edi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: addl %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %edx, %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orl %eax, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 80(%esp,%ebp), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 76(%esp,%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrdl %cl, %edi, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrdl %cl, %eax, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 88(%esp,%ebp), %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 84(%esp,%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrdl %cl, %ebx, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrdl %cl, %eax, %edi
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 104(%esp,%ecx), %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: leal (%eax,%eax), %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %edx, %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 100(%esp,%ecx), %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ebx, %edi, %ebp
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orl %ebp, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ebx, %esi, %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: addl %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %edx, %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orl %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, %ebp
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 112(%esp,%ecx), %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: leal (%eax,%eax), %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %edx, %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 108(%esp,%ecx), %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ebx, %esi, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orl %edi, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: addl %esi, %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %edx, %esi, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orl %ecx, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 120(%esp,%ebp), %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: leal (%edi,%edi), %ecx
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %edx, %ecx, %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 116(%esp,%ebp), %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ebx, %eax, %ebp
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orl %ebp, %esi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ebx, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: addl %eax, %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %edx, %eax, %ecx
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orl %ebp, %ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 96(%esp,%ebp), %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 92(%esp,%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrdl %cl, %esi, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrdl %cl, %eax, %ebx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 104(%esp,%ebp), %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 100(%esp,%ebp), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %edi, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrdl %cl, %eax, %edx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrdl %cl, %edi, %esi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 48(%esp,%ebp), %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 108(%esp,%ebp), %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ebp, (%esp) # 4-byte Spill
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrdl %cl, %ebp, %eax
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, 56(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %esi, 48(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %edx, 52(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ebx, 40(%ebp)
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl 124(%esp,%eax), %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: leal (%eax,%eax), %ebp
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxl %edx, %ebp, %edx
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxl %ebx, %edi, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orl %edi, %edx
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: sarxl %ebx, %eax, %edi
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %edi, 60(%eax)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %edx, 56(%eax)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, 48(%eax)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %esi, 52(%eax)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, 40(%eax)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, 44(%eax)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, 32(%eax)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, 36(%eax)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, 24(%eax)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, 28(%eax)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, 16(%eax)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, 20(%eax)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, 8(%eax)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, 12(%eax)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, (%eax)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, 44(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, 32(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, 36(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, 24(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, 28(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, 16(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, 20(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, 8(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, 12(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: sarxl %ecx, (%esp), %eax # 4-byte Folded Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrdl %cl, %edx, %edi
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %edi, (%ebp)
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, 4(%eax)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: addl $204, %esp
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, 4(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, 60(%ebp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: addl $188, %esp
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: popl %esi
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: popl %edi
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: popl %ebx
@@ -20094,13 +16318,21 @@ define void @ashr_64bytes_qwordOff(ptr %src.ptr, ptr %qwordOff.ptr, ptr %dst) no
; ALL: {{.*}}
; X64: {{.*}}
; X64-AVX512: {{.*}}
+; X64-HAVE-SHLD-HAVE-BMI2: {{.*}}
; X64-HAVE-SHLD-HAVE-BMI2-AVX512: {{.*}}
+; X64-HAVE-SHLD-NO-BMI2: {{.*}}
; X64-HAVE-SHLD-NO-BMI2-AVX512: {{.*}}
+; X64-NO-SHLD-HAVE-BMI2: {{.*}}
; X64-NO-SHLD-HAVE-BMI2-AVX512: {{.*}}
+; X64-NO-SHLD-NO-BMI2: {{.*}}
; X64-NO-SHLD-NO-BMI2-AVX512: {{.*}}
; X86: {{.*}}
; X86-AVX512: {{.*}}
+; X86-HAVE-SHLD-HAVE-BMI2: {{.*}}
; X86-HAVE-SHLD-HAVE-BMI2-AVX512: {{.*}}
+; X86-HAVE-SHLD-NO-BMI2: {{.*}}
; X86-HAVE-SHLD-NO-BMI2-AVX512: {{.*}}
+; X86-NO-SHLD-HAVE-BMI2: {{.*}}
; X86-NO-SHLD-HAVE-BMI2-AVX512: {{.*}}
+; X86-NO-SHLD-NO-BMI2: {{.*}}
; X86-NO-SHLD-NO-BMI2-AVX512: {{.*}}
diff --git a/llvm/test/CodeGen/X86/wide-scalar-shift-legalization.ll b/llvm/test/CodeGen/X86/wide-scalar-shift-legalization.ll
index 221a51ed44696..98bc1b0b95747 100644
--- a/llvm/test/CodeGen/X86/wide-scalar-shift-legalization.ll
+++ b/llvm/test/CodeGen/X86/wide-scalar-shift-legalization.ll
@@ -151,108 +151,50 @@ define void @lshr_8bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
; X64-BMI2-NEXT: movq %rax, (%rdx)
; X64-BMI2-NEXT: retq
;
-; X86-NO-BMI2-NO-SHLD-LABEL: lshr_8bytes:
-; X86-NO-BMI2-NO-SHLD: # %bb.0:
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl (%ecx), %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 4(%ecx), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movzbl (%eax), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: notb %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%esi,%esi), %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %ebx, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: xorl %ecx, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: testb $32, %al
-; X86-NO-BMI2-NO-SHLD-NEXT: cmovnel %esi, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: cmovel %esi, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, 4(%edx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, (%edx)
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: retl
-;
-; X86-NO-BMI2-HAVE-SHLD-LABEL: lshr_8bytes:
-; X86-NO-BMI2-HAVE-SHLD: # %bb.0:
-; X86-NO-BMI2-HAVE-SHLD-NEXT: pushl %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl (%esi), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 4(%esi), %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movzbl (%ecx), %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %esi, %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrl %cl, %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %esi, %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: xorl %esi, %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: testb $32, %cl
-; X86-NO-BMI2-HAVE-SHLD-NEXT: cmovnel %edi, %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: cmovel %edi, %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %esi, 4(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, (%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: popl %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: retl
-;
-; X86-HAVE-BMI2-NO-SHLD-LABEL: lshr_8bytes:
-; X86-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 4(%edx), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movzbl (%ecx), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, (%edx), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: notb %bl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%esi,%esi), %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ebx, %edi, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %edx, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, %esi, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: xorl %esi, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: testb $32, %cl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: cmovnel %edx, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: cmovel %edx, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, 4(%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edi, (%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: retl
+; X86-NO-BMI2-LABEL: lshr_8bytes:
+; X86-NO-BMI2: # %bb.0:
+; X86-NO-BMI2-NEXT: pushl %edi
+; X86-NO-BMI2-NEXT: pushl %esi
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-NO-BMI2-NEXT: movl (%esi), %edx
+; X86-NO-BMI2-NEXT: movl 4(%esi), %esi
+; X86-NO-BMI2-NEXT: movzbl (%ecx), %ecx
+; X86-NO-BMI2-NEXT: movl %esi, %edi
+; X86-NO-BMI2-NEXT: shrl %cl, %edi
+; X86-NO-BMI2-NEXT: shrdl %cl, %esi, %edx
+; X86-NO-BMI2-NEXT: xorl %esi, %esi
+; X86-NO-BMI2-NEXT: testb $32, %cl
+; X86-NO-BMI2-NEXT: cmovnel %edi, %edx
+; X86-NO-BMI2-NEXT: cmovel %edi, %esi
+; X86-NO-BMI2-NEXT: movl %esi, 4(%eax)
+; X86-NO-BMI2-NEXT: movl %edx, (%eax)
+; X86-NO-BMI2-NEXT: popl %esi
+; X86-NO-BMI2-NEXT: popl %edi
+; X86-NO-BMI2-NEXT: retl
;
-; X86-HAVE-BMI2-HAVE-SHLD-LABEL: lshr_8bytes:
-; X86-HAVE-BMI2-HAVE-SHLD: # %bb.0:
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: pushl %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl (%esi), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 4(%esi), %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movzbl (%ecx), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %esi, %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrxl %ecx, %esi, %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: xorl %edi, %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: testb $32, %cl
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: cmovnel %esi, %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: cmovel %esi, %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edi, 4(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, (%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: popl %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: retl
+; X86-BMI2-LABEL: lshr_8bytes:
+; X86-BMI2: # %bb.0:
+; X86-BMI2-NEXT: pushl %edi
+; X86-BMI2-NEXT: pushl %esi
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-BMI2-NEXT: movl (%esi), %edx
+; X86-BMI2-NEXT: movl 4(%esi), %esi
+; X86-BMI2-NEXT: movzbl (%ecx), %ecx
+; X86-BMI2-NEXT: shrdl %cl, %esi, %edx
+; X86-BMI2-NEXT: shrxl %ecx, %esi, %esi
+; X86-BMI2-NEXT: xorl %edi, %edi
+; X86-BMI2-NEXT: testb $32, %cl
+; X86-BMI2-NEXT: cmovnel %esi, %edx
+; X86-BMI2-NEXT: cmovel %esi, %edi
+; X86-BMI2-NEXT: movl %edi, 4(%eax)
+; X86-BMI2-NEXT: movl %edx, (%eax)
+; X86-BMI2-NEXT: popl %esi
+; X86-BMI2-NEXT: popl %edi
+; X86-BMI2-NEXT: retl
%src = load i64, ptr %src.ptr, align 1
%bitOff = load i64, ptr %bitOff.ptr, align 1
%res = lshr i64 %src, %bitOff
@@ -275,109 +217,50 @@ define void @shl_8bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
; X64-BMI2-NEXT: movq %rax, (%rdx)
; X64-BMI2-NEXT: retq
;
-; X86-NO-BMI2-NO-SHLD-LABEL: shl_8bytes:
-; X86-NO-BMI2-NO-SHLD: # %bb.0:
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl (%ecx), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 4(%ecx), %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movzbl (%eax), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: notb %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %esi, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %ebx, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: xorl %ecx, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: testb $32, %al
-; X86-NO-BMI2-NO-SHLD-NEXT: cmovnel %esi, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: cmovel %esi, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, (%edx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, 4(%edx)
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: retl
-;
-; X86-NO-BMI2-HAVE-SHLD-LABEL: shl_8bytes:
-; X86-NO-BMI2-HAVE-SHLD: # %bb.0:
-; X86-NO-BMI2-HAVE-SHLD-NEXT: pushl %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl (%edx), %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 4(%edx), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movzbl (%ecx), %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %esi, %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shll %cl, %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shldl %cl, %esi, %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: xorl %esi, %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: testb $32, %cl
-; X86-NO-BMI2-HAVE-SHLD-NEXT: cmovnel %edi, %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: cmovel %edi, %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, 4(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %esi, (%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: popl %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: retl
-;
-; X86-HAVE-BMI2-NO-SHLD-LABEL: shl_8bytes:
-; X86-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl (%edx), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movzbl (%ecx), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, 4(%edx), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: notb %bl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %esi, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ebx, %esi, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %edx, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: xorl %edx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: testb $32, %cl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: cmovnel %edi, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: cmovel %edi, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edx, (%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, 4(%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: retl
+; X86-NO-BMI2-LABEL: shl_8bytes:
+; X86-NO-BMI2: # %bb.0:
+; X86-NO-BMI2-NEXT: pushl %edi
+; X86-NO-BMI2-NEXT: pushl %esi
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NO-BMI2-NEXT: movl (%edx), %esi
+; X86-NO-BMI2-NEXT: movl 4(%edx), %edx
+; X86-NO-BMI2-NEXT: movzbl (%ecx), %ecx
+; X86-NO-BMI2-NEXT: movl %esi, %edi
+; X86-NO-BMI2-NEXT: shll %cl, %edi
+; X86-NO-BMI2-NEXT: shldl %cl, %esi, %edx
+; X86-NO-BMI2-NEXT: xorl %esi, %esi
+; X86-NO-BMI2-NEXT: testb $32, %cl
+; X86-NO-BMI2-NEXT: cmovnel %edi, %edx
+; X86-NO-BMI2-NEXT: cmovel %edi, %esi
+; X86-NO-BMI2-NEXT: movl %edx, 4(%eax)
+; X86-NO-BMI2-NEXT: movl %esi, (%eax)
+; X86-NO-BMI2-NEXT: popl %esi
+; X86-NO-BMI2-NEXT: popl %edi
+; X86-NO-BMI2-NEXT: retl
;
-; X86-HAVE-BMI2-HAVE-SHLD-LABEL: shl_8bytes:
-; X86-HAVE-BMI2-HAVE-SHLD: # %bb.0:
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: pushl %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl (%edx), %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 4(%edx), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movzbl (%ecx), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shldl %cl, %esi, %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shlxl %ecx, %esi, %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: xorl %edi, %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: testb $32, %cl
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: cmovnel %esi, %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: cmovel %esi, %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, 4(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edi, (%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: popl %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: retl
+; X86-BMI2-LABEL: shl_8bytes:
+; X86-BMI2: # %bb.0:
+; X86-BMI2-NEXT: pushl %edi
+; X86-BMI2-NEXT: pushl %esi
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-BMI2-NEXT: movl (%edx), %esi
+; X86-BMI2-NEXT: movl 4(%edx), %edx
+; X86-BMI2-NEXT: movzbl (%ecx), %ecx
+; X86-BMI2-NEXT: shldl %cl, %esi, %edx
+; X86-BMI2-NEXT: shlxl %ecx, %esi, %esi
+; X86-BMI2-NEXT: xorl %edi, %edi
+; X86-BMI2-NEXT: testb $32, %cl
+; X86-BMI2-NEXT: cmovnel %esi, %edx
+; X86-BMI2-NEXT: cmovel %esi, %edi
+; X86-BMI2-NEXT: movl %edx, 4(%eax)
+; X86-BMI2-NEXT: movl %edi, (%eax)
+; X86-BMI2-NEXT: popl %esi
+; X86-BMI2-NEXT: popl %edi
+; X86-BMI2-NEXT: retl
%src = load i64, ptr %src.ptr, align 1
%bitOff = load i64, ptr %bitOff.ptr, align 1
%res = shl i64 %src, %bitOff
@@ -400,109 +283,50 @@ define void @ashr_8bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
; X64-BMI2-NEXT: movq %rax, (%rdx)
; X64-BMI2-NEXT: retq
;
-; X86-NO-BMI2-NO-SHLD-LABEL: ashr_8bytes:
-; X86-NO-BMI2-NO-SHLD: # %bb.0:
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl (%ecx), %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 4(%ecx), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movzbl (%eax), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: notb %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%esi,%esi), %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %ebx, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %esi, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: sarl %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: sarl $31, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: testb $32, %al
-; X86-NO-BMI2-NO-SHLD-NEXT: cmovnel %ebx, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: cmovel %ebx, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %esi, 4(%edx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, (%edx)
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: retl
-;
-; X86-NO-BMI2-HAVE-SHLD-LABEL: ashr_8bytes:
-; X86-NO-BMI2-HAVE-SHLD: # %bb.0:
-; X86-NO-BMI2-HAVE-SHLD-NEXT: pushl %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl (%esi), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 4(%esi), %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movzbl (%ecx), %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %esi, %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: sarl %cl, %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %esi, %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: sarl $31, %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: testb $32, %cl
-; X86-NO-BMI2-HAVE-SHLD-NEXT: cmovnel %edi, %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: cmovel %edi, %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %esi, 4(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, (%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: popl %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: retl
-;
-; X86-HAVE-BMI2-NO-SHLD-LABEL: ashr_8bytes:
-; X86-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 4(%esi), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movzbl (%edx), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %edx, (%esi), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edx, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: notb %bl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%ecx,%ecx), %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ebx, %edi, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %esi, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: sarxl %edx, %ecx, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: sarl $31, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: testb $32, %dl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: cmovnel %esi, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: cmovel %esi, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, 4(%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edi, (%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: retl
+; X86-NO-BMI2-LABEL: ashr_8bytes:
+; X86-NO-BMI2: # %bb.0:
+; X86-NO-BMI2-NEXT: pushl %edi
+; X86-NO-BMI2-NEXT: pushl %esi
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-NO-BMI2-NEXT: movl (%esi), %edx
+; X86-NO-BMI2-NEXT: movl 4(%esi), %esi
+; X86-NO-BMI2-NEXT: movzbl (%ecx), %ecx
+; X86-NO-BMI2-NEXT: movl %esi, %edi
+; X86-NO-BMI2-NEXT: sarl %cl, %edi
+; X86-NO-BMI2-NEXT: shrdl %cl, %esi, %edx
+; X86-NO-BMI2-NEXT: sarl $31, %esi
+; X86-NO-BMI2-NEXT: testb $32, %cl
+; X86-NO-BMI2-NEXT: cmovnel %edi, %edx
+; X86-NO-BMI2-NEXT: cmovel %edi, %esi
+; X86-NO-BMI2-NEXT: movl %esi, 4(%eax)
+; X86-NO-BMI2-NEXT: movl %edx, (%eax)
+; X86-NO-BMI2-NEXT: popl %esi
+; X86-NO-BMI2-NEXT: popl %edi
+; X86-NO-BMI2-NEXT: retl
;
-; X86-HAVE-BMI2-HAVE-SHLD-LABEL: ashr_8bytes:
-; X86-HAVE-BMI2-HAVE-SHLD: # %bb.0:
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: pushl %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl (%esi), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 4(%esi), %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movzbl (%ecx), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %esi, %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: sarxl %ecx, %esi, %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: sarl $31, %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: testb $32, %cl
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: cmovnel %edi, %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: cmovel %edi, %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %esi, 4(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, (%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: popl %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: retl
+; X86-BMI2-LABEL: ashr_8bytes:
+; X86-BMI2: # %bb.0:
+; X86-BMI2-NEXT: pushl %edi
+; X86-BMI2-NEXT: pushl %esi
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-BMI2-NEXT: movl (%esi), %edx
+; X86-BMI2-NEXT: movl 4(%esi), %esi
+; X86-BMI2-NEXT: movzbl (%ecx), %ecx
+; X86-BMI2-NEXT: shrdl %cl, %esi, %edx
+; X86-BMI2-NEXT: sarxl %ecx, %esi, %edi
+; X86-BMI2-NEXT: sarl $31, %esi
+; X86-BMI2-NEXT: testb $32, %cl
+; X86-BMI2-NEXT: cmovnel %edi, %edx
+; X86-BMI2-NEXT: cmovel %edi, %esi
+; X86-BMI2-NEXT: movl %esi, 4(%eax)
+; X86-BMI2-NEXT: movl %edx, (%eax)
+; X86-BMI2-NEXT: popl %esi
+; X86-BMI2-NEXT: popl %edi
+; X86-BMI2-NEXT: retl
%src = load i64, ptr %src.ptr, align 1
%bitOff = load i64, ptr %bitOff.ptr, align 1
%res = ashr i64 %src, %bitOff
@@ -511,286 +335,127 @@ define void @ashr_8bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
}
define void @lshr_16bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
-; X64-NO-BMI2-NO-SHLD-LABEL: lshr_16bytes:
-; X64-NO-BMI2-NO-SHLD: # %bb.0:
-; X64-NO-BMI2-NO-SHLD-NEXT: movq (%rdi), %r8
-; X64-NO-BMI2-NO-SHLD-NEXT: movq 8(%rdi), %rdi
-; X64-NO-BMI2-NO-SHLD-NEXT: movzbl (%rsi), %eax
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %r8
-; X64-NO-BMI2-NO-SHLD-NEXT: notb %cl
-; X64-NO-BMI2-NO-SHLD-NEXT: leaq (%rdi,%rdi), %rsi
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %rsi
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %r8, %rsi
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %rdi
-; X64-NO-BMI2-NO-SHLD-NEXT: xorl %ecx, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: testb $64, %al
-; X64-NO-BMI2-NO-SHLD-NEXT: cmovneq %rdi, %rsi
-; X64-NO-BMI2-NO-SHLD-NEXT: cmoveq %rdi, %rcx
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rcx, 8(%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rsi, (%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: retq
-;
-; X64-NO-BMI2-HAVE-SHLD-LABEL: lshr_16bytes:
-; X64-NO-BMI2-HAVE-SHLD: # %bb.0:
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq (%rdi), %rax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq 8(%rdi), %rdi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movzbl (%rsi), %ecx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rdi, %rsi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrq %cl, %rsi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %rdi, %rax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: xorl %edi, %edi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: testb $64, %cl
-; X64-NO-BMI2-HAVE-SHLD-NEXT: cmovneq %rsi, %rax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: cmoveq %rsi, %rdi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rdi, 8(%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rax, (%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: retq
-;
-; X64-HAVE-BMI2-NO-SHLD-LABEL: lshr_16bytes:
-; X64-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq 8(%rdi), %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movzbl (%rsi), %ecx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rcx, (%rdi), %rsi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, %edi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: notb %dil
-; X64-HAVE-BMI2-NO-SHLD-NEXT: leaq (%rax,%rax), %r8
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rdi, %r8, %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %rsi, %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rcx, %rax, %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: xorl %esi, %esi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: testb $64, %cl
-; X64-HAVE-BMI2-NO-SHLD-NEXT: cmovneq %rax, %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: cmoveq %rax, %rsi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rsi, 8(%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rdi, (%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: retq
-;
-; X64-HAVE-BMI2-HAVE-SHLD-LABEL: lshr_16bytes:
-; X64-HAVE-BMI2-HAVE-SHLD: # %bb.0:
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq (%rdi), %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq 8(%rdi), %rdi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movzbl (%rsi), %ecx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %rdi, %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrxq %rcx, %rdi, %rsi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: xorl %edi, %edi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: testb $64, %cl
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: cmovneq %rsi, %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: cmoveq %rsi, %rdi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rdi, 8(%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rax, (%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: retq
-;
-; X86-NO-BMI2-NO-SHLD-LABEL: lshr_16bytes:
-; X86-NO-BMI2-NO-SHLD: # %bb.0:
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: subl $44, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl (%ecx), %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 4(%ecx), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 8(%ecx), %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 12(%ecx), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movb (%eax), %dh
-; X86-NO-BMI2-NO-SHLD-NEXT: xorps %xmm0, %xmm0
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, (%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dh, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shrb $3, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: andb $12, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: movzbl %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 4(%esp,%eax), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %esi, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dh, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dh, %dl
-; X86-NO-BMI2-NO-SHLD-NEXT: andb $31, %dl
-; X86-NO-BMI2-NO-SHLD-NEXT: xorb $31, %dl
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 8(%esp,%eax), %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%ebp,%ebp), %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %ebx, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl (%esp,%eax), %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dh, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %esi, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %ebx, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dh, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 12(%esp,%eax), %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%ebx,%ebx), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %ebp, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dh, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, 12(%ebp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, 8(%ebp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %esi, (%ebp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, 4(%ebp)
-; X86-NO-BMI2-NO-SHLD-NEXT: addl $44, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: retl
+; X64-NO-BMI2-LABEL: lshr_16bytes:
+; X64-NO-BMI2: # %bb.0:
+; X64-NO-BMI2-NEXT: movq (%rdi), %rax
+; X64-NO-BMI2-NEXT: movq 8(%rdi), %rdi
+; X64-NO-BMI2-NEXT: movzbl (%rsi), %ecx
+; X64-NO-BMI2-NEXT: movq %rdi, %rsi
+; X64-NO-BMI2-NEXT: shrq %cl, %rsi
+; X64-NO-BMI2-NEXT: shrdq %cl, %rdi, %rax
+; X64-NO-BMI2-NEXT: xorl %edi, %edi
+; X64-NO-BMI2-NEXT: testb $64, %cl
+; X64-NO-BMI2-NEXT: cmovneq %rsi, %rax
+; X64-NO-BMI2-NEXT: cmoveq %rsi, %rdi
+; X64-NO-BMI2-NEXT: movq %rdi, 8(%rdx)
+; X64-NO-BMI2-NEXT: movq %rax, (%rdx)
+; X64-NO-BMI2-NEXT: retq
;
-; X86-NO-BMI2-HAVE-SHLD-LABEL: lshr_16bytes:
-; X86-NO-BMI2-HAVE-SHLD: # %bb.0:
-; X86-NO-BMI2-HAVE-SHLD-NEXT: pushl %ebp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: pushl %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: pushl %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: subl $44, %esp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl (%edx), %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 4(%edx), %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 8(%edx), %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 12(%edx), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movzbl (%ecx), %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: xorps %xmm0, %xmm0
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %esi, (%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrb $3, %dl
-; X86-NO-BMI2-HAVE-SHLD-NEXT: andb $12, %dl
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movzbl %dl, %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 8(%esp,%ebx), %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl (%esp,%ebx), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 4(%esp,%ebx), %ebp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ebp, %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %esi, %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 12(%esp,%ebx), %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %ebx, %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %ebp, %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrl %cl, %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %esi, 8(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ebx, 12(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, (%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edi, 4(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: addl $44, %esp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: popl %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: popl %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: popl %ebp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: retl
+; X64-BMI2-LABEL: lshr_16bytes:
+; X64-BMI2: # %bb.0:
+; X64-BMI2-NEXT: movq (%rdi), %rax
+; X64-BMI2-NEXT: movq 8(%rdi), %rdi
+; X64-BMI2-NEXT: movzbl (%rsi), %ecx
+; X64-BMI2-NEXT: shrdq %cl, %rdi, %rax
+; X64-BMI2-NEXT: shrxq %rcx, %rdi, %rsi
+; X64-BMI2-NEXT: xorl %edi, %edi
+; X64-BMI2-NEXT: testb $64, %cl
+; X64-BMI2-NEXT: cmovneq %rsi, %rax
+; X64-BMI2-NEXT: cmoveq %rsi, %rdi
+; X64-BMI2-NEXT: movq %rdi, 8(%rdx)
+; X64-BMI2-NEXT: movq %rax, (%rdx)
+; X64-BMI2-NEXT: retq
;
-; X86-HAVE-BMI2-NO-SHLD-LABEL: lshr_16bytes:
-; X86-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: subl $44, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl (%ecx), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 4(%ecx), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 8(%ecx), %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 12(%ecx), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movzbl (%eax), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: xorps %xmm0, %xmm0
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edx, (%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrb $3, %dl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: andb $12, %dl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movzbl %dl, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 4(%esp,%edi), %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 8(%esp,%edi), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, %ebx, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: andb $31, %al
-; X86-HAVE-BMI2-NO-SHLD-NEXT: xorb $31, %al
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%edx,%edx), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %eax, %esi, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %ebp, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, (%esp,%edi), %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %ebx, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %eax, %ebx, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %ebp, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 12(%esp,%edi), %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%edi,%edi), %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %eax, %ebp, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, %edx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %edx, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, %edi, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, 12(%edx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, 8(%edx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ebx, (%edx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, 4(%edx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl $44, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: retl
+; X86-NO-BMI2-LABEL: lshr_16bytes:
+; X86-NO-BMI2: # %bb.0:
+; X86-NO-BMI2-NEXT: pushl %ebp
+; X86-NO-BMI2-NEXT: pushl %ebx
+; X86-NO-BMI2-NEXT: pushl %edi
+; X86-NO-BMI2-NEXT: pushl %esi
+; X86-NO-BMI2-NEXT: subl $44, %esp
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NO-BMI2-NEXT: movl (%edx), %esi
+; X86-NO-BMI2-NEXT: movl 4(%edx), %edi
+; X86-NO-BMI2-NEXT: movl 8(%edx), %ebx
+; X86-NO-BMI2-NEXT: movl 12(%edx), %edx
+; X86-NO-BMI2-NEXT: movzbl (%ecx), %ecx
+; X86-NO-BMI2-NEXT: xorps %xmm0, %xmm0
+; X86-NO-BMI2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %esi, (%esp)
+; X86-NO-BMI2-NEXT: movl %ecx, %edx
+; X86-NO-BMI2-NEXT: shrb $3, %dl
+; X86-NO-BMI2-NEXT: andb $12, %dl
+; X86-NO-BMI2-NEXT: movzbl %dl, %ebx
+; X86-NO-BMI2-NEXT: movl 8(%esp,%ebx), %esi
+; X86-NO-BMI2-NEXT: movl (%esp,%ebx), %edx
+; X86-NO-BMI2-NEXT: movl 4(%esp,%ebx), %ebp
+; X86-NO-BMI2-NEXT: movl %ebp, %edi
+; X86-NO-BMI2-NEXT: shrdl %cl, %esi, %edi
+; X86-NO-BMI2-NEXT: movl 12(%esp,%ebx), %ebx
+; X86-NO-BMI2-NEXT: shrdl %cl, %ebx, %esi
+; X86-NO-BMI2-NEXT: shrdl %cl, %ebp, %edx
+; X86-NO-BMI2-NEXT: shrl %cl, %ebx
+; X86-NO-BMI2-NEXT: movl %esi, 8(%eax)
+; X86-NO-BMI2-NEXT: movl %ebx, 12(%eax)
+; X86-NO-BMI2-NEXT: movl %edx, (%eax)
+; X86-NO-BMI2-NEXT: movl %edi, 4(%eax)
+; X86-NO-BMI2-NEXT: addl $44, %esp
+; X86-NO-BMI2-NEXT: popl %esi
+; X86-NO-BMI2-NEXT: popl %edi
+; X86-NO-BMI2-NEXT: popl %ebx
+; X86-NO-BMI2-NEXT: popl %ebp
+; X86-NO-BMI2-NEXT: retl
;
-; X86-HAVE-BMI2-HAVE-SHLD-LABEL: lshr_16bytes:
-; X86-HAVE-BMI2-HAVE-SHLD: # %bb.0:
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: pushl %ebp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: pushl %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: pushl %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: subl $44, %esp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl (%edx), %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 4(%edx), %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 8(%edx), %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 12(%edx), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movzbl (%ecx), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: xorps %xmm0, %xmm0
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %esi, (%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrb $3, %dl
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: andb $12, %dl
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movzbl %dl, %ebp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 8(%esp,%ebp), %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl (%esp,%ebp), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 4(%esp,%ebp), %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %esi, %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %ebx, %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 12(%esp,%ebp), %ebp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %ebp, %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ebx, 8(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrxl %ecx, %ebp, %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ebx, 12(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %esi, %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, (%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edi, 4(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: addl $44, %esp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: popl %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: popl %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: popl %ebp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: retl
+; X86-BMI2-LABEL: lshr_16bytes:
+; X86-BMI2: # %bb.0:
+; X86-BMI2-NEXT: pushl %ebp
+; X86-BMI2-NEXT: pushl %ebx
+; X86-BMI2-NEXT: pushl %edi
+; X86-BMI2-NEXT: pushl %esi
+; X86-BMI2-NEXT: subl $44, %esp
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-BMI2-NEXT: movl (%edx), %esi
+; X86-BMI2-NEXT: movl 4(%edx), %edi
+; X86-BMI2-NEXT: movl 8(%edx), %ebx
+; X86-BMI2-NEXT: movl 12(%edx), %edx
+; X86-BMI2-NEXT: movzbl (%ecx), %ecx
+; X86-BMI2-NEXT: xorps %xmm0, %xmm0
+; X86-BMI2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %esi, (%esp)
+; X86-BMI2-NEXT: movl %ecx, %edx
+; X86-BMI2-NEXT: shrb $3, %dl
+; X86-BMI2-NEXT: andb $12, %dl
+; X86-BMI2-NEXT: movzbl %dl, %ebp
+; X86-BMI2-NEXT: movl 8(%esp,%ebp), %ebx
+; X86-BMI2-NEXT: movl (%esp,%ebp), %edx
+; X86-BMI2-NEXT: movl 4(%esp,%ebp), %esi
+; X86-BMI2-NEXT: movl %esi, %edi
+; X86-BMI2-NEXT: shrdl %cl, %ebx, %edi
+; X86-BMI2-NEXT: movl 12(%esp,%ebp), %ebp
+; X86-BMI2-NEXT: shrdl %cl, %ebp, %ebx
+; X86-BMI2-NEXT: movl %ebx, 8(%eax)
+; X86-BMI2-NEXT: shrxl %ecx, %ebp, %ebx
+; X86-BMI2-NEXT: movl %ebx, 12(%eax)
+; X86-BMI2-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-BMI2-NEXT: shrdl %cl, %esi, %edx
+; X86-BMI2-NEXT: movl %edx, (%eax)
+; X86-BMI2-NEXT: movl %edi, 4(%eax)
+; X86-BMI2-NEXT: addl $44, %esp
+; X86-BMI2-NEXT: popl %esi
+; X86-BMI2-NEXT: popl %edi
+; X86-BMI2-NEXT: popl %ebx
+; X86-BMI2-NEXT: popl %ebp
+; X86-BMI2-NEXT: retl
%src = load i128, ptr %src.ptr, align 1
%bitOff = load i128, ptr %bitOff.ptr, align 1
%res = lshr i128 %src, %bitOff
@@ -798,287 +463,122 @@ define void @lshr_16bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
ret void
}
define void @shl_16bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
-; X64-NO-BMI2-NO-SHLD-LABEL: shl_16bytes:
-; X64-NO-BMI2-NO-SHLD: # %bb.0:
-; X64-NO-BMI2-NO-SHLD-NEXT: movq (%rdi), %r8
-; X64-NO-BMI2-NO-SHLD-NEXT: movq 8(%rdi), %rdi
-; X64-NO-BMI2-NO-SHLD-NEXT: movzbl (%rsi), %eax
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %rdi
-; X64-NO-BMI2-NO-SHLD-NEXT: notb %cl
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r8, %rsi
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %rsi
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %rsi
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %rdi, %rsi
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %r8
-; X64-NO-BMI2-NO-SHLD-NEXT: xorl %ecx, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: testb $64, %al
-; X64-NO-BMI2-NO-SHLD-NEXT: cmovneq %r8, %rsi
-; X64-NO-BMI2-NO-SHLD-NEXT: cmoveq %r8, %rcx
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rcx, (%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rsi, 8(%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: retq
-;
-; X64-NO-BMI2-HAVE-SHLD-LABEL: shl_16bytes:
-; X64-NO-BMI2-HAVE-SHLD: # %bb.0:
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq (%rdi), %rax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq 8(%rdi), %rdi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movzbl (%rsi), %ecx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rax, %rsi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shlq %cl, %rsi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shldq %cl, %rax, %rdi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: xorl %eax, %eax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: testb $64, %cl
-; X64-NO-BMI2-HAVE-SHLD-NEXT: cmovneq %rsi, %rdi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: cmoveq %rsi, %rax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rdi, 8(%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rax, (%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: retq
-;
-; X64-HAVE-BMI2-NO-SHLD-LABEL: shl_16bytes:
-; X64-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq (%rdi), %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movzbl (%rsi), %ecx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rcx, 8(%rdi), %rsi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, %edi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: notb %dil
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rcx, %rax, %r8
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrq %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rdi, %rax, %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %rsi, %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: xorl %esi, %esi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: testb $64, %cl
-; X64-HAVE-BMI2-NO-SHLD-NEXT: cmovneq %r8, %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: cmoveq %r8, %rsi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rsi, (%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rax, 8(%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: retq
-;
-; X64-HAVE-BMI2-HAVE-SHLD-LABEL: shl_16bytes:
-; X64-HAVE-BMI2-HAVE-SHLD: # %bb.0:
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq (%rdi), %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq 8(%rdi), %rdi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movzbl (%rsi), %ecx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shldq %cl, %rax, %rdi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shlxq %rcx, %rax, %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: xorl %esi, %esi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: testb $64, %cl
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: cmovneq %rax, %rdi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: cmoveq %rax, %rsi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rdi, 8(%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rsi, (%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: retq
-;
-; X86-NO-BMI2-NO-SHLD-LABEL: shl_16bytes:
-; X86-NO-BMI2-NO-SHLD: # %bb.0:
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: subl $60, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl (%ecx), %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 4(%ecx), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 8(%ecx), %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 12(%ecx), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movb (%eax), %dh
-; X86-NO-BMI2-NO-SHLD-NEXT: xorps %xmm0, %xmm0
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dh, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shrb $3, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: andb $12, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: negb %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: movsbl %cl, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 32(%esp,%ebp), %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 36(%esp,%ebp), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %esi, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dh, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dh, %dl
-; X86-NO-BMI2-NO-SHLD-NEXT: andb $31, %dl
-; X86-NO-BMI2-NO-SHLD-NEXT: xorb $31, %dl
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %edi, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 44(%esp,%ebp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dh, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 40(%esp,%ebp), %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %eax, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dh, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %edi, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dh, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, (%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %esi, 8(%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebp, 12(%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, 4(%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: addl $60, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: retl
+; X64-NO-BMI2-LABEL: shl_16bytes:
+; X64-NO-BMI2: # %bb.0:
+; X64-NO-BMI2-NEXT: movq (%rdi), %rax
+; X64-NO-BMI2-NEXT: movq 8(%rdi), %rdi
+; X64-NO-BMI2-NEXT: movzbl (%rsi), %ecx
+; X64-NO-BMI2-NEXT: movq %rax, %rsi
+; X64-NO-BMI2-NEXT: shlq %cl, %rsi
+; X64-NO-BMI2-NEXT: shldq %cl, %rax, %rdi
+; X64-NO-BMI2-NEXT: xorl %eax, %eax
+; X64-NO-BMI2-NEXT: testb $64, %cl
+; X64-NO-BMI2-NEXT: cmovneq %rsi, %rdi
+; X64-NO-BMI2-NEXT: cmoveq %rsi, %rax
+; X64-NO-BMI2-NEXT: movq %rdi, 8(%rdx)
+; X64-NO-BMI2-NEXT: movq %rax, (%rdx)
+; X64-NO-BMI2-NEXT: retq
;
-; X86-NO-BMI2-HAVE-SHLD-LABEL: shl_16bytes:
-; X86-NO-BMI2-HAVE-SHLD: # %bb.0:
-; X86-NO-BMI2-HAVE-SHLD-NEXT: pushl %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: pushl %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: subl $32, %esp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl (%edx), %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 4(%edx), %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 8(%edx), %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 12(%edx), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movzbl (%ecx), %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: xorps %xmm0, %xmm0
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrb $3, %dl
-; X86-NO-BMI2-HAVE-SHLD-NEXT: andb $12, %dl
-; X86-NO-BMI2-HAVE-SHLD-NEXT: negb %dl
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movsbl %dl, %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 24(%esp,%edi), %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 28(%esp,%edi), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shldl %cl, %esi, %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 16(%esp,%edi), %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 20(%esp,%edi), %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shldl %cl, %edi, %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shldl %cl, %ebx, %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shll %cl, %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %esi, 8(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, 12(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ebx, (%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edi, 4(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: addl $32, %esp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: popl %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: popl %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: retl
+; X64-BMI2-LABEL: shl_16bytes:
+; X64-BMI2: # %bb.0:
+; X64-BMI2-NEXT: movq (%rdi), %rax
+; X64-BMI2-NEXT: movq 8(%rdi), %rdi
+; X64-BMI2-NEXT: movzbl (%rsi), %ecx
+; X64-BMI2-NEXT: shldq %cl, %rax, %rdi
+; X64-BMI2-NEXT: shlxq %rcx, %rax, %rax
+; X64-BMI2-NEXT: xorl %esi, %esi
+; X64-BMI2-NEXT: testb $64, %cl
+; X64-BMI2-NEXT: cmovneq %rax, %rdi
+; X64-BMI2-NEXT: cmoveq %rax, %rsi
+; X64-BMI2-NEXT: movq %rdi, 8(%rdx)
+; X64-BMI2-NEXT: movq %rsi, (%rdx)
+; X64-BMI2-NEXT: retq
;
-; X86-HAVE-BMI2-NO-SHLD-LABEL: shl_16bytes:
-; X86-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: subl $44, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl (%ecx), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 4(%ecx), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 8(%ecx), %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 12(%ecx), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movzbl (%eax), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: xorps %xmm0, %xmm0
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrb $3, %dl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: andb $12, %dl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: negb %dl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movsbl %dl, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 16(%esp,%esi), %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 20(%esp,%esi), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %edx, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: andb $31, %al
-; X86-HAVE-BMI2-NO-SHLD-NEXT: xorb $31, %al
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %ebx, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %eax, %ebx, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %edi, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, 28(%esp,%esi), %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 24(%esp,%esi), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %esi, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %eax, %esi, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %edi, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrl %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %eax, %edx, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %ecx, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ebp, (%ecx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, 8(%ecx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, 12(%ecx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ebx, 4(%ecx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl $44, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: retl
+; X86-NO-BMI2-LABEL: shl_16bytes:
+; X86-NO-BMI2: # %bb.0:
+; X86-NO-BMI2-NEXT: pushl %ebx
+; X86-NO-BMI2-NEXT: pushl %edi
+; X86-NO-BMI2-NEXT: pushl %esi
+; X86-NO-BMI2-NEXT: subl $32, %esp
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NO-BMI2-NEXT: movl (%edx), %esi
+; X86-NO-BMI2-NEXT: movl 4(%edx), %edi
+; X86-NO-BMI2-NEXT: movl 8(%edx), %ebx
+; X86-NO-BMI2-NEXT: movl 12(%edx), %edx
+; X86-NO-BMI2-NEXT: movzbl (%ecx), %ecx
+; X86-NO-BMI2-NEXT: xorps %xmm0, %xmm0
+; X86-NO-BMI2-NEXT: movaps %xmm0, (%esp)
+; X86-NO-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %ecx, %edx
+; X86-NO-BMI2-NEXT: shrb $3, %dl
+; X86-NO-BMI2-NEXT: andb $12, %dl
+; X86-NO-BMI2-NEXT: negb %dl
+; X86-NO-BMI2-NEXT: movsbl %dl, %edi
+; X86-NO-BMI2-NEXT: movl 24(%esp,%edi), %esi
+; X86-NO-BMI2-NEXT: movl 28(%esp,%edi), %edx
+; X86-NO-BMI2-NEXT: shldl %cl, %esi, %edx
+; X86-NO-BMI2-NEXT: movl 16(%esp,%edi), %ebx
+; X86-NO-BMI2-NEXT: movl 20(%esp,%edi), %edi
+; X86-NO-BMI2-NEXT: shldl %cl, %edi, %esi
+; X86-NO-BMI2-NEXT: shldl %cl, %ebx, %edi
+; X86-NO-BMI2-NEXT: shll %cl, %ebx
+; X86-NO-BMI2-NEXT: movl %esi, 8(%eax)
+; X86-NO-BMI2-NEXT: movl %edx, 12(%eax)
+; X86-NO-BMI2-NEXT: movl %ebx, (%eax)
+; X86-NO-BMI2-NEXT: movl %edi, 4(%eax)
+; X86-NO-BMI2-NEXT: addl $32, %esp
+; X86-NO-BMI2-NEXT: popl %esi
+; X86-NO-BMI2-NEXT: popl %edi
+; X86-NO-BMI2-NEXT: popl %ebx
+; X86-NO-BMI2-NEXT: retl
;
-; X86-HAVE-BMI2-HAVE-SHLD-LABEL: shl_16bytes:
-; X86-HAVE-BMI2-HAVE-SHLD: # %bb.0:
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: pushl %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: pushl %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: subl $32, %esp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl (%edx), %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 4(%edx), %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 8(%edx), %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 12(%edx), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movzbl (%ecx), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: xorps %xmm0, %xmm0
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrb $3, %dl
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: andb $12, %dl
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: negb %dl
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movsbl %dl, %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 24(%esp,%edi), %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 28(%esp,%edi), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shldl %cl, %esi, %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 16(%esp,%edi), %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 20(%esp,%edi), %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shldl %cl, %edi, %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shldl %cl, %ebx, %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shlxl %ecx, %ebx, %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %esi, 8(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, 12(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, (%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edi, 4(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: addl $32, %esp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: popl %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: popl %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: retl
+; X86-BMI2-LABEL: shl_16bytes:
+; X86-BMI2: # %bb.0:
+; X86-BMI2-NEXT: pushl %ebx
+; X86-BMI2-NEXT: pushl %edi
+; X86-BMI2-NEXT: pushl %esi
+; X86-BMI2-NEXT: subl $32, %esp
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-BMI2-NEXT: movl (%edx), %esi
+; X86-BMI2-NEXT: movl 4(%edx), %edi
+; X86-BMI2-NEXT: movl 8(%edx), %ebx
+; X86-BMI2-NEXT: movl 12(%edx), %edx
+; X86-BMI2-NEXT: movzbl (%ecx), %ecx
+; X86-BMI2-NEXT: xorps %xmm0, %xmm0
+; X86-BMI2-NEXT: movaps %xmm0, (%esp)
+; X86-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %ecx, %edx
+; X86-BMI2-NEXT: shrb $3, %dl
+; X86-BMI2-NEXT: andb $12, %dl
+; X86-BMI2-NEXT: negb %dl
+; X86-BMI2-NEXT: movsbl %dl, %edi
+; X86-BMI2-NEXT: movl 24(%esp,%edi), %esi
+; X86-BMI2-NEXT: movl 28(%esp,%edi), %edx
+; X86-BMI2-NEXT: shldl %cl, %esi, %edx
+; X86-BMI2-NEXT: movl 16(%esp,%edi), %ebx
+; X86-BMI2-NEXT: movl 20(%esp,%edi), %edi
+; X86-BMI2-NEXT: shldl %cl, %edi, %esi
+; X86-BMI2-NEXT: shldl %cl, %ebx, %edi
+; X86-BMI2-NEXT: shlxl %ecx, %ebx, %ecx
+; X86-BMI2-NEXT: movl %esi, 8(%eax)
+; X86-BMI2-NEXT: movl %edx, 12(%eax)
+; X86-BMI2-NEXT: movl %ecx, (%eax)
+; X86-BMI2-NEXT: movl %edi, 4(%eax)
+; X86-BMI2-NEXT: addl $32, %esp
+; X86-BMI2-NEXT: popl %esi
+; X86-BMI2-NEXT: popl %edi
+; X86-BMI2-NEXT: popl %ebx
+; X86-BMI2-NEXT: retl
%src = load i128, ptr %src.ptr, align 1
%bitOff = load i128, ptr %bitOff.ptr, align 1
%res = shl i128 %src, %bitOff
@@ -1086,299 +586,133 @@ define void @shl_16bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
ret void
}
define void @ashr_16bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
-; X64-NO-BMI2-NO-SHLD-LABEL: ashr_16bytes:
-; X64-NO-BMI2-NO-SHLD: # %bb.0:
-; X64-NO-BMI2-NO-SHLD-NEXT: movq (%rdi), %r8
-; X64-NO-BMI2-NO-SHLD-NEXT: movq 8(%rdi), %rdi
-; X64-NO-BMI2-NO-SHLD-NEXT: movzbl (%rsi), %eax
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %r8
-; X64-NO-BMI2-NO-SHLD-NEXT: notb %cl
-; X64-NO-BMI2-NO-SHLD-NEXT: leaq (%rdi,%rdi), %rsi
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %rsi
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %r8, %rsi
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rdi, %r8
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: sarq %cl, %r8
-; X64-NO-BMI2-NO-SHLD-NEXT: sarq $63, %rdi
-; X64-NO-BMI2-NO-SHLD-NEXT: testb $64, %al
-; X64-NO-BMI2-NO-SHLD-NEXT: cmovneq %r8, %rsi
-; X64-NO-BMI2-NO-SHLD-NEXT: cmoveq %r8, %rdi
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rdi, 8(%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rsi, (%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: retq
-;
-; X64-NO-BMI2-HAVE-SHLD-LABEL: ashr_16bytes:
-; X64-NO-BMI2-HAVE-SHLD: # %bb.0:
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq (%rdi), %rax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq 8(%rdi), %rdi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movzbl (%rsi), %ecx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rdi, %rsi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: sarq %cl, %rsi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %rdi, %rax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: sarq $63, %rdi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: testb $64, %cl
-; X64-NO-BMI2-HAVE-SHLD-NEXT: cmovneq %rsi, %rax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: cmoveq %rsi, %rdi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rdi, 8(%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rax, (%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: retq
-;
-; X64-HAVE-BMI2-NO-SHLD-LABEL: ashr_16bytes:
-; X64-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq 8(%rdi), %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movzbl (%rsi), %ecx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rcx, (%rdi), %rsi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, %edi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: notb %dil
-; X64-HAVE-BMI2-NO-SHLD-NEXT: leaq (%rax,%rax), %r8
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rdi, %r8, %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %rsi, %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: sarxq %rcx, %rax, %rsi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: sarq $63, %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: testb $64, %cl
-; X64-HAVE-BMI2-NO-SHLD-NEXT: cmovneq %rsi, %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: cmoveq %rsi, %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rax, 8(%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rdi, (%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: retq
-;
-; X64-HAVE-BMI2-HAVE-SHLD-LABEL: ashr_16bytes:
-; X64-HAVE-BMI2-HAVE-SHLD: # %bb.0:
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq (%rdi), %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq 8(%rdi), %rdi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movzbl (%rsi), %ecx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %rdi, %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: sarxq %rcx, %rdi, %rsi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: sarq $63, %rdi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: testb $64, %cl
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: cmovneq %rsi, %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: cmoveq %rsi, %rdi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rdi, 8(%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rax, (%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: retq
-;
-; X86-NO-BMI2-NO-SHLD-LABEL: ashr_16bytes:
-; X86-NO-BMI2-NO-SHLD: # %bb.0:
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: subl $44, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl (%ecx), %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 4(%ecx), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 8(%ecx), %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 12(%ecx), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movb (%eax), %dh
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, (%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: sarl $31, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dh, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shrb $3, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: andb $12, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: movzbl %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 4(%esp,%ebx), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %esi, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dh, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dh, %dl
-; X86-NO-BMI2-NO-SHLD-NEXT: andb $31, %dl
-; X86-NO-BMI2-NO-SHLD-NEXT: xorb $31, %dl
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 8(%esp,%ebx), %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%ebp,%ebp), %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %eax, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl (%esp,%ebx), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dh, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %esi, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %eax, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dh, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 12(%esp,%ebx), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%eax,%eax), %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %ebp, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dh, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: sarl %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, 12(%ebp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, 8(%ebp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %esi, (%ebp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, 4(%ebp)
-; X86-NO-BMI2-NO-SHLD-NEXT: addl $44, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: retl
+; X64-NO-BMI2-LABEL: ashr_16bytes:
+; X64-NO-BMI2: # %bb.0:
+; X64-NO-BMI2-NEXT: movq (%rdi), %rax
+; X64-NO-BMI2-NEXT: movq 8(%rdi), %rdi
+; X64-NO-BMI2-NEXT: movzbl (%rsi), %ecx
+; X64-NO-BMI2-NEXT: movq %rdi, %rsi
+; X64-NO-BMI2-NEXT: sarq %cl, %rsi
+; X64-NO-BMI2-NEXT: shrdq %cl, %rdi, %rax
+; X64-NO-BMI2-NEXT: sarq $63, %rdi
+; X64-NO-BMI2-NEXT: testb $64, %cl
+; X64-NO-BMI2-NEXT: cmovneq %rsi, %rax
+; X64-NO-BMI2-NEXT: cmoveq %rsi, %rdi
+; X64-NO-BMI2-NEXT: movq %rdi, 8(%rdx)
+; X64-NO-BMI2-NEXT: movq %rax, (%rdx)
+; X64-NO-BMI2-NEXT: retq
;
-; X86-NO-BMI2-HAVE-SHLD-LABEL: ashr_16bytes:
-; X86-NO-BMI2-HAVE-SHLD: # %bb.0:
-; X86-NO-BMI2-HAVE-SHLD-NEXT: pushl %ebp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: pushl %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: pushl %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: subl $44, %esp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl (%edx), %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 4(%edx), %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 8(%edx), %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 12(%edx), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movzbl (%ecx), %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %esi, (%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: sarl $31, %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrb $3, %dl
-; X86-NO-BMI2-HAVE-SHLD-NEXT: andb $12, %dl
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movzbl %dl, %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 8(%esp,%ebx), %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl (%esp,%ebx), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 4(%esp,%ebx), %ebp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ebp, %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %esi, %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 12(%esp,%ebx), %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %ebx, %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %ebp, %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: sarl %cl, %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %esi, 8(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ebx, 12(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, (%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edi, 4(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: addl $44, %esp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: popl %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: popl %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: popl %ebp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: retl
+; X64-BMI2-LABEL: ashr_16bytes:
+; X64-BMI2: # %bb.0:
+; X64-BMI2-NEXT: movq (%rdi), %rax
+; X64-BMI2-NEXT: movq 8(%rdi), %rdi
+; X64-BMI2-NEXT: movzbl (%rsi), %ecx
+; X64-BMI2-NEXT: shrdq %cl, %rdi, %rax
+; X64-BMI2-NEXT: sarxq %rcx, %rdi, %rsi
+; X64-BMI2-NEXT: sarq $63, %rdi
+; X64-BMI2-NEXT: testb $64, %cl
+; X64-BMI2-NEXT: cmovneq %rsi, %rax
+; X64-BMI2-NEXT: cmoveq %rsi, %rdi
+; X64-BMI2-NEXT: movq %rdi, 8(%rdx)
+; X64-BMI2-NEXT: movq %rax, (%rdx)
+; X64-BMI2-NEXT: retq
;
-; X86-HAVE-BMI2-NO-SHLD-LABEL: ashr_16bytes:
-; X86-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: subl $44, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl (%ecx), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 4(%ecx), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 8(%ecx), %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 12(%ecx), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movzbl (%eax), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edx, (%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: sarl $31, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrb $3, %dl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: andb $12, %dl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movzbl %dl, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 4(%esp,%edi), %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 8(%esp,%edi), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, %ebx, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: andb $31, %al
-; X86-HAVE-BMI2-NO-SHLD-NEXT: xorb $31, %al
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%edx,%edx), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %eax, %esi, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %ebp, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, (%esp,%edi), %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %ebx, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %eax, %ebx, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %ebp, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 12(%esp,%edi), %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%edi,%edi), %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %eax, %ebp, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, %edx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %edx, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: sarxl %ecx, %edi, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, 12(%edx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, 8(%edx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ebx, (%edx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, 4(%edx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl $44, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: retl
+; X86-NO-BMI2-LABEL: ashr_16bytes:
+; X86-NO-BMI2: # %bb.0:
+; X86-NO-BMI2-NEXT: pushl %ebp
+; X86-NO-BMI2-NEXT: pushl %ebx
+; X86-NO-BMI2-NEXT: pushl %edi
+; X86-NO-BMI2-NEXT: pushl %esi
+; X86-NO-BMI2-NEXT: subl $44, %esp
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NO-BMI2-NEXT: movl (%edx), %esi
+; X86-NO-BMI2-NEXT: movl 4(%edx), %edi
+; X86-NO-BMI2-NEXT: movl 8(%edx), %ebx
+; X86-NO-BMI2-NEXT: movl 12(%edx), %edx
+; X86-NO-BMI2-NEXT: movzbl (%ecx), %ecx
+; X86-NO-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %esi, (%esp)
+; X86-NO-BMI2-NEXT: sarl $31, %edx
+; X86-NO-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %ecx, %edx
+; X86-NO-BMI2-NEXT: shrb $3, %dl
+; X86-NO-BMI2-NEXT: andb $12, %dl
+; X86-NO-BMI2-NEXT: movzbl %dl, %ebx
+; X86-NO-BMI2-NEXT: movl 8(%esp,%ebx), %esi
+; X86-NO-BMI2-NEXT: movl (%esp,%ebx), %edx
+; X86-NO-BMI2-NEXT: movl 4(%esp,%ebx), %ebp
+; X86-NO-BMI2-NEXT: movl %ebp, %edi
+; X86-NO-BMI2-NEXT: shrdl %cl, %esi, %edi
+; X86-NO-BMI2-NEXT: movl 12(%esp,%ebx), %ebx
+; X86-NO-BMI2-NEXT: shrdl %cl, %ebx, %esi
+; X86-NO-BMI2-NEXT: shrdl %cl, %ebp, %edx
+; X86-NO-BMI2-NEXT: sarl %cl, %ebx
+; X86-NO-BMI2-NEXT: movl %esi, 8(%eax)
+; X86-NO-BMI2-NEXT: movl %ebx, 12(%eax)
+; X86-NO-BMI2-NEXT: movl %edx, (%eax)
+; X86-NO-BMI2-NEXT: movl %edi, 4(%eax)
+; X86-NO-BMI2-NEXT: addl $44, %esp
+; X86-NO-BMI2-NEXT: popl %esi
+; X86-NO-BMI2-NEXT: popl %edi
+; X86-NO-BMI2-NEXT: popl %ebx
+; X86-NO-BMI2-NEXT: popl %ebp
+; X86-NO-BMI2-NEXT: retl
;
-; X86-HAVE-BMI2-HAVE-SHLD-LABEL: ashr_16bytes:
-; X86-HAVE-BMI2-HAVE-SHLD: # %bb.0:
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: pushl %ebp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: pushl %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: pushl %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: subl $44, %esp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl (%edx), %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 4(%edx), %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 8(%edx), %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 12(%edx), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movzbl (%ecx), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %esi, (%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: sarl $31, %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrb $3, %dl
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: andb $12, %dl
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movzbl %dl, %ebp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 8(%esp,%ebp), %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl (%esp,%ebp), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 4(%esp,%ebp), %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %esi, %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %ebx, %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 12(%esp,%ebp), %ebp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %ebp, %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ebx, 8(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: sarxl %ecx, %ebp, %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ebx, 12(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %esi, %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, (%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edi, 4(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: addl $44, %esp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: popl %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: popl %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: popl %ebp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: retl
+; X86-BMI2-LABEL: ashr_16bytes:
+; X86-BMI2: # %bb.0:
+; X86-BMI2-NEXT: pushl %ebp
+; X86-BMI2-NEXT: pushl %ebx
+; X86-BMI2-NEXT: pushl %edi
+; X86-BMI2-NEXT: pushl %esi
+; X86-BMI2-NEXT: subl $44, %esp
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-BMI2-NEXT: movl (%edx), %esi
+; X86-BMI2-NEXT: movl 4(%edx), %edi
+; X86-BMI2-NEXT: movl 8(%edx), %ebx
+; X86-BMI2-NEXT: movl 12(%edx), %edx
+; X86-BMI2-NEXT: movzbl (%ecx), %ecx
+; X86-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %esi, (%esp)
+; X86-BMI2-NEXT: sarl $31, %edx
+; X86-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %ecx, %edx
+; X86-BMI2-NEXT: shrb $3, %dl
+; X86-BMI2-NEXT: andb $12, %dl
+; X86-BMI2-NEXT: movzbl %dl, %ebp
+; X86-BMI2-NEXT: movl 8(%esp,%ebp), %ebx
+; X86-BMI2-NEXT: movl (%esp,%ebp), %edx
+; X86-BMI2-NEXT: movl 4(%esp,%ebp), %esi
+; X86-BMI2-NEXT: movl %esi, %edi
+; X86-BMI2-NEXT: shrdl %cl, %ebx, %edi
+; X86-BMI2-NEXT: movl 12(%esp,%ebp), %ebp
+; X86-BMI2-NEXT: shrdl %cl, %ebp, %ebx
+; X86-BMI2-NEXT: movl %ebx, 8(%eax)
+; X86-BMI2-NEXT: sarxl %ecx, %ebp, %ebx
+; X86-BMI2-NEXT: movl %ebx, 12(%eax)
+; X86-BMI2-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-BMI2-NEXT: shrdl %cl, %esi, %edx
+; X86-BMI2-NEXT: movl %edx, (%eax)
+; X86-BMI2-NEXT: movl %edi, 4(%eax)
+; X86-BMI2-NEXT: addl $44, %esp
+; X86-BMI2-NEXT: popl %esi
+; X86-BMI2-NEXT: popl %edi
+; X86-BMI2-NEXT: popl %ebx
+; X86-BMI2-NEXT: popl %ebp
+; X86-BMI2-NEXT: retl
%src = load i128, ptr %src.ptr, align 1
%bitOff = load i128, ptr %bitOff.ptr, align 1
%res = ashr i128 %src, %bitOff
@@ -1387,549 +721,226 @@ define void @ashr_16bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
}
define void @lshr_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
-; X64-NO-BMI2-NO-SHLD-LABEL: lshr_32bytes:
-; X64-NO-BMI2-NO-SHLD: # %bb.0:
-; X64-NO-BMI2-NO-SHLD-NEXT: pushq %rbx
-; X64-NO-BMI2-NO-SHLD-NEXT: movq (%rdi), %rcx
-; X64-NO-BMI2-NO-SHLD-NEXT: movq 8(%rdi), %r8
-; X64-NO-BMI2-NO-SHLD-NEXT: movq 16(%rdi), %r9
-; X64-NO-BMI2-NO-SHLD-NEXT: movq 24(%rdi), %rdi
-; X64-NO-BMI2-NO-SHLD-NEXT: movzbl (%rsi), %eax
-; X64-NO-BMI2-NO-SHLD-NEXT: xorps %xmm0, %xmm0
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrb $6, %cl
-; X64-NO-BMI2-NO-SHLD-NEXT: movzbl %cl, %r8d
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -64(%rsp,%r8,8), %r10
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -56(%rsp,%r8,8), %rdi
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rdi, %r11
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %r11
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %esi
-; X64-NO-BMI2-NO-SHLD-NEXT: andb $63, %sil
-; X64-NO-BMI2-NO-SHLD-NEXT: xorb $63, %sil
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -48(%rsp,%r8,8), %rbx
-; X64-NO-BMI2-NO-SHLD-NEXT: leaq (%rbx,%rbx), %r9
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %esi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %r9
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %r11, %r9
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %r10
-; X64-NO-BMI2-NO-SHLD-NEXT: addq %rdi, %rdi
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %esi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %rdi
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %r10, %rdi
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %rbx
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -40(%rsp,%r8,8), %r8
-; X64-NO-BMI2-NO-SHLD-NEXT: leaq (%r8,%r8), %r10
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %esi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %r10
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %rbx, %r10
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %r8
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r8, 24(%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r10, 16(%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rdi, (%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r9, 8(%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: popq %rbx
-; X64-NO-BMI2-NO-SHLD-NEXT: retq
-;
-; X64-NO-BMI2-HAVE-SHLD-LABEL: lshr_32bytes:
-; X64-NO-BMI2-HAVE-SHLD: # %bb.0:
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq (%rdi), %rax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq 8(%rdi), %r8
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq 16(%rdi), %r9
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq 24(%rdi), %rdi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movzbl (%rsi), %ecx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: xorps %xmm0, %xmm0
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, %eax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrb $6, %al
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movzbl %al, %eax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -56(%rsp,%rax,8), %rsi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -72(%rsp,%rax,8), %rdi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -64(%rsp,%rax,8), %r8
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r8, %r9
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %rsi, %r9
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -48(%rsp,%rax,8), %rax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %rax, %rsi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %r8, %rdi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrq %cl, %rax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rsi, 16(%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rax, 24(%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rdi, (%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r9, 8(%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: retq
-;
-; X64-HAVE-BMI2-NO-SHLD-LABEL: lshr_32bytes:
-; X64-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq (%rdi), %rcx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq 8(%rdi), %r8
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq 16(%rdi), %r9
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq 24(%rdi), %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movzbl (%rsi), %eax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: xorps %xmm0, %xmm0
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, %esi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrb $6, %sil
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movzbl %sil, %esi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq -64(%rsp,%rsi,8), %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq -56(%rsp,%rsi,8), %r8
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rcx, %rdi, %r9
-; X64-HAVE-BMI2-NO-SHLD-NEXT: andb $63, %al
-; X64-HAVE-BMI2-NO-SHLD-NEXT: xorb $63, %al
-; X64-HAVE-BMI2-NO-SHLD-NEXT: leaq (%r8,%r8), %r10
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rax, %r10, %r10
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %r9, %r10
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rcx, -72(%rsp,%rsi,8), %r9
-; X64-HAVE-BMI2-NO-SHLD-NEXT: addq %rdi, %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rax, %rdi, %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %r9, %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rcx, %r8, %r8
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq -48(%rsp,%rsi,8), %rsi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: leaq (%rsi,%rsi), %r9
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rax, %r9, %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %r8, %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rcx, %rsi, %rcx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rcx, 24(%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rax, 16(%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rdi, (%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %r10, 8(%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: retq
-;
-; X64-HAVE-BMI2-HAVE-SHLD-LABEL: lshr_32bytes:
-; X64-HAVE-BMI2-HAVE-SHLD: # %bb.0:
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq (%rdi), %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq 8(%rdi), %r8
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq 16(%rdi), %r9
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq 24(%rdi), %rdi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movzbl (%rsi), %ecx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: xorps %xmm0, %xmm0
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, %eax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrb $6, %al
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movzbl %al, %eax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -56(%rsp,%rax,8), %rsi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -72(%rsp,%rax,8), %rdi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -64(%rsp,%rax,8), %r8
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %r8, %r9
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %rsi, %r9
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -48(%rsp,%rax,8), %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %rax, %rsi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %r8, %rdi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrxq %rcx, %rax, %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rsi, 16(%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rax, 24(%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rdi, (%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %r9, 8(%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: retq
-;
-; X86-NO-BMI2-NO-SHLD-LABEL: lshr_32bytes:
-; X86-NO-BMI2-NO-SHLD: # %bb.0:
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: subl $108, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl (%ebp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 4(%ebp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 8(%ebp), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 12(%ebp), %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 16(%ebp), %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movzbl (%ecx), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 20(%ebp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 24(%ebp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 28(%ebp), %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: xorps %xmm0, %xmm0
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebp, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: shrb $5, %al
-; X86-NO-BMI2-NO-SHLD-NEXT: movzbl %al, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 32(%esp,%edi,4), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 36(%esp,%edi,4), %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dl, %ch
-; X86-NO-BMI2-NO-SHLD-NEXT: andb $31, %ch
-; X86-NO-BMI2-NO-SHLD-NEXT: xorb $31, %ch
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 40(%esp,%edi,4), %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%edi,%edi), %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %eax, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dl, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %ebx, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 44(%esp,%esi,4), %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebp, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dl, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 48(%esp,%esi,4), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%edx,%edx), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %eax, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %bl, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %ebp, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %edi, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 52(%esp,%esi,4), %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %bl, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 56(%esp,%esi,4), %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%ebx,%ebx), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %eax, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dl, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %edi, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %eax, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dl, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 60(%esp,%eax,4), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%eax,%eax), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %ebx, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, 28(%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, 24(%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, 16(%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %esi, 20(%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebp, 8(%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, 12(%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, (%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, 4(%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: addl $108, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: retl
+; X64-NO-BMI2-LABEL: lshr_32bytes:
+; X64-NO-BMI2: # %bb.0:
+; X64-NO-BMI2-NEXT: movq (%rdi), %rax
+; X64-NO-BMI2-NEXT: movq 8(%rdi), %r8
+; X64-NO-BMI2-NEXT: movq 16(%rdi), %r9
+; X64-NO-BMI2-NEXT: movq 24(%rdi), %rdi
+; X64-NO-BMI2-NEXT: movzbl (%rsi), %ecx
+; X64-NO-BMI2-NEXT: xorps %xmm0, %xmm0
+; X64-NO-BMI2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movl %ecx, %eax
+; X64-NO-BMI2-NEXT: shrb $6, %al
+; X64-NO-BMI2-NEXT: movzbl %al, %eax
+; X64-NO-BMI2-NEXT: movq -56(%rsp,%rax,8), %rsi
+; X64-NO-BMI2-NEXT: movq -72(%rsp,%rax,8), %rdi
+; X64-NO-BMI2-NEXT: movq -64(%rsp,%rax,8), %r8
+; X64-NO-BMI2-NEXT: movq %r8, %r9
+; X64-NO-BMI2-NEXT: shrdq %cl, %rsi, %r9
+; X64-NO-BMI2-NEXT: movq -48(%rsp,%rax,8), %rax
+; X64-NO-BMI2-NEXT: shrdq %cl, %rax, %rsi
+; X64-NO-BMI2-NEXT: shrdq %cl, %r8, %rdi
+; X64-NO-BMI2-NEXT: shrq %cl, %rax
+; X64-NO-BMI2-NEXT: movq %rsi, 16(%rdx)
+; X64-NO-BMI2-NEXT: movq %rax, 24(%rdx)
+; X64-NO-BMI2-NEXT: movq %rdi, (%rdx)
+; X64-NO-BMI2-NEXT: movq %r9, 8(%rdx)
+; X64-NO-BMI2-NEXT: retq
;
-; X86-NO-BMI2-HAVE-SHLD-LABEL: lshr_32bytes:
-; X86-NO-BMI2-HAVE-SHLD: # %bb.0:
-; X86-NO-BMI2-HAVE-SHLD-NEXT: pushl %ebp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: pushl %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: pushl %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: subl $92, %esp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl (%ebp), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 4(%ebp), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, (%esp) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 8(%ebp), %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 12(%ebp), %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 16(%ebp), %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movzbl (%ecx), %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 20(%ebp), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 24(%ebp), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 28(%ebp), %ebp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: xorps %xmm0, %xmm0
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ebp, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl (%esp), %eax # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrb $5, %al
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movzbl %al, %ebp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 24(%esp,%ebp,4), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 20(%esp,%ebp,4), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, (%esp) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edx, %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 32(%esp,%ebp,4), %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 28(%esp,%ebp,4), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %ebx, %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %eax, %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 40(%esp,%ebp,4), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 36(%esp,%ebp,4), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edx, %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %eax, %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 16(%esp,%ebp,4), %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 44(%esp,%ebp,4), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %eax, %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, 24(%ebp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl (%esp), %edx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edx, %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrl %cl, %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, 28(%ebp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ebx, 16(%ebp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edi, 20(%ebp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, 8(%ebp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, 12(%ebp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %esi, (%ebp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, 4(%ebp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: addl $92, %esp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: popl %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: popl %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: popl %ebp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: retl
+; X64-BMI2-LABEL: lshr_32bytes:
+; X64-BMI2: # %bb.0:
+; X64-BMI2-NEXT: movq (%rdi), %rax
+; X64-BMI2-NEXT: movq 8(%rdi), %r8
+; X64-BMI2-NEXT: movq 16(%rdi), %r9
+; X64-BMI2-NEXT: movq 24(%rdi), %rdi
+; X64-BMI2-NEXT: movzbl (%rsi), %ecx
+; X64-BMI2-NEXT: xorps %xmm0, %xmm0
+; X64-BMI2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movl %ecx, %eax
+; X64-BMI2-NEXT: shrb $6, %al
+; X64-BMI2-NEXT: movzbl %al, %eax
+; X64-BMI2-NEXT: movq -56(%rsp,%rax,8), %rsi
+; X64-BMI2-NEXT: movq -72(%rsp,%rax,8), %rdi
+; X64-BMI2-NEXT: movq -64(%rsp,%rax,8), %r8
+; X64-BMI2-NEXT: movq %r8, %r9
+; X64-BMI2-NEXT: shrdq %cl, %rsi, %r9
+; X64-BMI2-NEXT: movq -48(%rsp,%rax,8), %rax
+; X64-BMI2-NEXT: shrdq %cl, %rax, %rsi
+; X64-BMI2-NEXT: shrdq %cl, %r8, %rdi
+; X64-BMI2-NEXT: shrxq %rcx, %rax, %rax
+; X64-BMI2-NEXT: movq %rsi, 16(%rdx)
+; X64-BMI2-NEXT: movq %rax, 24(%rdx)
+; X64-BMI2-NEXT: movq %rdi, (%rdx)
+; X64-BMI2-NEXT: movq %r9, 8(%rdx)
+; X64-BMI2-NEXT: retq
;
-; X86-HAVE-BMI2-NO-SHLD-LABEL: lshr_32bytes:
-; X86-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: subl $108, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl (%eax), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 4(%eax), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 8(%eax), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 12(%eax), %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 16(%eax), %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 20(%eax), %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 24(%eax), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 28(%eax), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movzbl (%eax), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: xorps %xmm0, %xmm0
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ebp, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrb $5, %dl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movzbl %dl, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 36(%esp,%esi,4), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 40(%esp,%esi,4), %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %eax, %edx, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: andb $31, %cl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: xorb $31, %cl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%ebp,%ebp), %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %ebx, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %edi, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %eax, 32(%esp,%esi,4), %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %edx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %edx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %edi, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 48(%esp,%esi,4), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %edx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %edx, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 44(%esp,%esi,4), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %eax, %edx, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %edi, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %eax, %ebp, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %edx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %edx, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %edi, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 56(%esp,%esi,4), %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%edi,%edi), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %edx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 52(%esp,%esi,4), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ebp, %eax, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %ebx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ebp, {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %eax, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %eax, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %ebx, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 60(%esp,%esi,4), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%esi,%esi), %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %ebx, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ebp, %edi, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %edi, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ebp, %esi, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, 28(%edi)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, 24(%edi)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, 16(%edi)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edx, 20(%edi)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, 8(%edi)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, 12(%edi)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, (%edi)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, 4(%edi)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl $108, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: retl
+; X86-NO-BMI2-LABEL: lshr_32bytes:
+; X86-NO-BMI2: # %bb.0:
+; X86-NO-BMI2-NEXT: pushl %ebp
+; X86-NO-BMI2-NEXT: pushl %ebx
+; X86-NO-BMI2-NEXT: pushl %edi
+; X86-NO-BMI2-NEXT: pushl %esi
+; X86-NO-BMI2-NEXT: subl $92, %esp
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NO-BMI2-NEXT: movl (%ebp), %eax
+; X86-NO-BMI2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 4(%ebp), %eax
+; X86-NO-BMI2-NEXT: movl %eax, (%esp) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 8(%ebp), %esi
+; X86-NO-BMI2-NEXT: movl 12(%ebp), %edi
+; X86-NO-BMI2-NEXT: movl 16(%ebp), %ebx
+; X86-NO-BMI2-NEXT: movzbl (%ecx), %ecx
+; X86-NO-BMI2-NEXT: movl 20(%ebp), %edx
+; X86-NO-BMI2-NEXT: movl 24(%ebp), %eax
+; X86-NO-BMI2-NEXT: movl 28(%ebp), %ebp
+; X86-NO-BMI2-NEXT: xorps %xmm0, %xmm0
+; X86-NO-BMI2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %ebp, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl (%esp), %eax # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %ecx, %eax
+; X86-NO-BMI2-NEXT: shrb $5, %al
+; X86-NO-BMI2-NEXT: movzbl %al, %ebp
+; X86-NO-BMI2-NEXT: movl 24(%esp,%ebp,4), %edx
+; X86-NO-BMI2-NEXT: movl 20(%esp,%ebp,4), %eax
+; X86-NO-BMI2-NEXT: movl %eax, (%esp) # 4-byte Spill
+; X86-NO-BMI2-NEXT: shrdl %cl, %edx, %eax
+; X86-NO-BMI2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 32(%esp,%ebp,4), %ebx
+; X86-NO-BMI2-NEXT: movl 28(%esp,%ebp,4), %eax
+; X86-NO-BMI2-NEXT: movl %eax, %esi
+; X86-NO-BMI2-NEXT: shrdl %cl, %ebx, %esi
+; X86-NO-BMI2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: shrdl %cl, %eax, %edx
+; X86-NO-BMI2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 40(%esp,%ebp,4), %edx
+; X86-NO-BMI2-NEXT: movl 36(%esp,%ebp,4), %eax
+; X86-NO-BMI2-NEXT: movl %eax, %edi
+; X86-NO-BMI2-NEXT: shrdl %cl, %edx, %edi
+; X86-NO-BMI2-NEXT: shrdl %cl, %eax, %ebx
+; X86-NO-BMI2-NEXT: movl 16(%esp,%ebp,4), %esi
+; X86-NO-BMI2-NEXT: movl 44(%esp,%ebp,4), %eax
+; X86-NO-BMI2-NEXT: shrdl %cl, %eax, %edx
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NO-BMI2-NEXT: movl %edx, 24(%ebp)
+; X86-NO-BMI2-NEXT: movl (%esp), %edx # 4-byte Reload
+; X86-NO-BMI2-NEXT: shrdl %cl, %edx, %esi
+; X86-NO-BMI2-NEXT: shrl %cl, %eax
+; X86-NO-BMI2-NEXT: movl %eax, 28(%ebp)
+; X86-NO-BMI2-NEXT: movl %ebx, 16(%ebp)
+; X86-NO-BMI2-NEXT: movl %edi, 20(%ebp)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %eax, 8(%ebp)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %eax, 12(%ebp)
+; X86-NO-BMI2-NEXT: movl %esi, (%ebp)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %eax, 4(%ebp)
+; X86-NO-BMI2-NEXT: addl $92, %esp
+; X86-NO-BMI2-NEXT: popl %esi
+; X86-NO-BMI2-NEXT: popl %edi
+; X86-NO-BMI2-NEXT: popl %ebx
+; X86-NO-BMI2-NEXT: popl %ebp
+; X86-NO-BMI2-NEXT: retl
;
-; X86-HAVE-BMI2-HAVE-SHLD-LABEL: lshr_32bytes:
-; X86-HAVE-BMI2-HAVE-SHLD: # %bb.0:
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: pushl %ebp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: pushl %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: pushl %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: subl $92, %esp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl (%ecx), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 4(%ecx), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, (%esp) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 8(%ecx), %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 12(%ecx), %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 16(%ecx), %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 20(%ecx), %ebp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 24(%ecx), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 28(%ecx), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movzbl (%ecx), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: xorps %xmm0, %xmm0
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ebp, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl (%esp), %eax # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrb $5, %al
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movzbl %al, %ebp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 24(%esp,%ebp,4), %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 20(%esp,%ebp,4), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %esi, %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, (%esp) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 32(%esp,%ebp,4), %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 28(%esp,%ebp,4), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %ebx, %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %eax, %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 40(%esp,%ebp,4), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 36(%esp,%ebp,4), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %eax, %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edx, %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 16(%esp,%ebp,4), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 44(%esp,%ebp,4), %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edi, %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, 24(%ebp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrxl %ecx, %edi, %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, 28(%ebp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ebx, 16(%ebp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %esi, 20(%ebp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, 8(%ebp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, 12(%ebp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %eax, %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, (%ebp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl (%esp), %eax # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, 4(%ebp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: addl $92, %esp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: popl %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: popl %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: popl %ebp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: retl
+; X86-BMI2-LABEL: lshr_32bytes:
+; X86-BMI2: # %bb.0:
+; X86-BMI2-NEXT: pushl %ebp
+; X86-BMI2-NEXT: pushl %ebx
+; X86-BMI2-NEXT: pushl %edi
+; X86-BMI2-NEXT: pushl %esi
+; X86-BMI2-NEXT: subl $92, %esp
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-BMI2-NEXT: movl (%ecx), %eax
+; X86-BMI2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 4(%ecx), %eax
+; X86-BMI2-NEXT: movl %eax, (%esp) # 4-byte Spill
+; X86-BMI2-NEXT: movl 8(%ecx), %esi
+; X86-BMI2-NEXT: movl 12(%ecx), %edi
+; X86-BMI2-NEXT: movl 16(%ecx), %ebx
+; X86-BMI2-NEXT: movl 20(%ecx), %ebp
+; X86-BMI2-NEXT: movl 24(%ecx), %edx
+; X86-BMI2-NEXT: movl 28(%ecx), %eax
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-BMI2-NEXT: movzbl (%ecx), %ecx
+; X86-BMI2-NEXT: xorps %xmm0, %xmm0
+; X86-BMI2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %ebp, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl (%esp), %eax # 4-byte Reload
+; X86-BMI2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-BMI2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %ecx, %eax
+; X86-BMI2-NEXT: shrb $5, %al
+; X86-BMI2-NEXT: movzbl %al, %ebp
+; X86-BMI2-NEXT: movl 24(%esp,%ebp,4), %esi
+; X86-BMI2-NEXT: movl 20(%esp,%ebp,4), %eax
+; X86-BMI2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: shrdl %cl, %esi, %eax
+; X86-BMI2-NEXT: movl %eax, (%esp) # 4-byte Spill
+; X86-BMI2-NEXT: movl 32(%esp,%ebp,4), %ebx
+; X86-BMI2-NEXT: movl 28(%esp,%ebp,4), %eax
+; X86-BMI2-NEXT: movl %eax, %edx
+; X86-BMI2-NEXT: shrdl %cl, %ebx, %edx
+; X86-BMI2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: shrdl %cl, %eax, %esi
+; X86-BMI2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 40(%esp,%ebp,4), %eax
+; X86-BMI2-NEXT: movl 36(%esp,%ebp,4), %edx
+; X86-BMI2-NEXT: movl %edx, %esi
+; X86-BMI2-NEXT: shrdl %cl, %eax, %esi
+; X86-BMI2-NEXT: shrdl %cl, %edx, %ebx
+; X86-BMI2-NEXT: movl 16(%esp,%ebp,4), %edx
+; X86-BMI2-NEXT: movl 44(%esp,%ebp,4), %edi
+; X86-BMI2-NEXT: shrdl %cl, %edi, %eax
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-BMI2-NEXT: movl %eax, 24(%ebp)
+; X86-BMI2-NEXT: shrxl %ecx, %edi, %eax
+; X86-BMI2-NEXT: movl %eax, 28(%ebp)
+; X86-BMI2-NEXT: movl %ebx, 16(%ebp)
+; X86-BMI2-NEXT: movl %esi, 20(%ebp)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-BMI2-NEXT: movl %eax, 8(%ebp)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-BMI2-NEXT: movl %eax, 12(%ebp)
+; X86-BMI2-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-BMI2-NEXT: shrdl %cl, %eax, %edx
+; X86-BMI2-NEXT: movl %edx, (%ebp)
+; X86-BMI2-NEXT: movl (%esp), %eax # 4-byte Reload
+; X86-BMI2-NEXT: movl %eax, 4(%ebp)
+; X86-BMI2-NEXT: addl $92, %esp
+; X86-BMI2-NEXT: popl %esi
+; X86-BMI2-NEXT: popl %edi
+; X86-BMI2-NEXT: popl %ebx
+; X86-BMI2-NEXT: popl %ebp
+; X86-BMI2-NEXT: retl
%src = load i256, ptr %src.ptr, align 1
%bitOff = load i256, ptr %bitOff.ptr, align 1
%res = lshr i256 %src, %bitOff
@@ -1937,572 +948,231 @@ define void @lshr_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
ret void
}
define void @shl_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
-; X64-NO-BMI2-NO-SHLD-LABEL: shl_32bytes:
-; X64-NO-BMI2-NO-SHLD: # %bb.0:
-; X64-NO-BMI2-NO-SHLD-NEXT: pushq %rbx
-; X64-NO-BMI2-NO-SHLD-NEXT: movq (%rdi), %rcx
-; X64-NO-BMI2-NO-SHLD-NEXT: movq 8(%rdi), %r8
-; X64-NO-BMI2-NO-SHLD-NEXT: movq 16(%rdi), %r9
-; X64-NO-BMI2-NO-SHLD-NEXT: movq 24(%rdi), %rdi
-; X64-NO-BMI2-NO-SHLD-NEXT: movzbl (%rsi), %eax
-; X64-NO-BMI2-NO-SHLD-NEXT: xorps %xmm0, %xmm0
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrb $3, %cl
-; X64-NO-BMI2-NO-SHLD-NEXT: andb $24, %cl
-; X64-NO-BMI2-NO-SHLD-NEXT: negb %cl
-; X64-NO-BMI2-NO-SHLD-NEXT: movsbq %cl, %r10
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -32(%rsp,%r10), %r8
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -24(%rsp,%r10), %rdi
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rdi, %r11
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %r11
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %esi
-; X64-NO-BMI2-NO-SHLD-NEXT: andb $63, %sil
-; X64-NO-BMI2-NO-SHLD-NEXT: xorb $63, %sil
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r8, %r9
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %r9
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %esi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %r9
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %r11, %r9
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -8(%rsp,%r10), %r11
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %r11
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -16(%rsp,%r10), %r10
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r10, %rbx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %rbx
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %esi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %rbx
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %r11, %rbx
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %r10
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %rdi
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %esi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %rdi
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %r10, %rdi
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %r8
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r8, (%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rdi, 16(%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rbx, 24(%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r9, 8(%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: popq %rbx
-; X64-NO-BMI2-NO-SHLD-NEXT: retq
-;
-; X64-NO-BMI2-HAVE-SHLD-LABEL: shl_32bytes:
-; X64-NO-BMI2-HAVE-SHLD: # %bb.0:
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq (%rdi), %rax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq 8(%rdi), %r8
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq 16(%rdi), %r9
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq 24(%rdi), %rdi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movzbl (%rsi), %ecx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: xorps %xmm0, %xmm0
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, %eax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrb $3, %al
-; X64-NO-BMI2-HAVE-SHLD-NEXT: andb $24, %al
-; X64-NO-BMI2-HAVE-SHLD-NEXT: negb %al
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movsbq %al, %rax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -24(%rsp,%rax), %rsi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -16(%rsp,%rax), %rdi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shldq %cl, %rsi, %rdi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -40(%rsp,%rax), %r8
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -32(%rsp,%rax), %rax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shldq %cl, %rax, %rsi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shldq %cl, %r8, %rax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shlq %cl, %r8
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rsi, 16(%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rdi, 24(%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r8, (%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rax, 8(%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: retq
-;
-; X64-HAVE-BMI2-NO-SHLD-LABEL: shl_32bytes:
-; X64-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq (%rdi), %rcx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq 8(%rdi), %r8
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq 16(%rdi), %r9
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq 24(%rdi), %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movzbl (%rsi), %eax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: xorps %xmm0, %xmm0
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, %esi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrb $3, %sil
-; X64-HAVE-BMI2-NO-SHLD-NEXT: andb $24, %sil
-; X64-HAVE-BMI2-NO-SHLD-NEXT: negb %sil
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movsbq %sil, %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq -40(%rsp,%rdi), %r8
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq -32(%rsp,%rdi), %rsi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rcx, %rsi, %r9
-; X64-HAVE-BMI2-NO-SHLD-NEXT: andb $63, %al
-; X64-HAVE-BMI2-NO-SHLD-NEXT: xorb $63, %al
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rcx, %r8, %r10
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrq %r8
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rax, %r8, %r8
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %r9, %r8
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rcx, -16(%rsp,%rdi), %r9
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq -24(%rsp,%rdi), %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rcx, %rdi, %rcx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrq %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rax, %rdi, %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %r9, %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrq %rsi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rax, %rsi, %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %rcx, %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %r10, (%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rax, 16(%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rdi, 24(%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %r8, 8(%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: retq
-;
-; X64-HAVE-BMI2-HAVE-SHLD-LABEL: shl_32bytes:
-; X64-HAVE-BMI2-HAVE-SHLD: # %bb.0:
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq (%rdi), %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq 8(%rdi), %r8
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq 16(%rdi), %r9
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq 24(%rdi), %rdi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movzbl (%rsi), %ecx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: xorps %xmm0, %xmm0
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, %eax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrb $3, %al
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: andb $24, %al
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: negb %al
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movsbq %al, %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -24(%rsp,%rax), %rsi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -16(%rsp,%rax), %rdi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shldq %cl, %rsi, %rdi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -40(%rsp,%rax), %r8
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -32(%rsp,%rax), %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shldq %cl, %rax, %rsi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shldq %cl, %r8, %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shlxq %rcx, %r8, %rcx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rsi, 16(%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rdi, 24(%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rcx, (%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rax, 8(%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: retq
-;
-; X86-NO-BMI2-NO-SHLD-LABEL: shl_32bytes:
-; X86-NO-BMI2-NO-SHLD: # %bb.0:
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: subl $108, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl (%ebp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 4(%ebp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 8(%ebp), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 12(%ebp), %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 16(%ebp), %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movb (%ecx), %ch
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 20(%ebp), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 24(%ebp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 28(%ebp), %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebp, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: xorps %xmm0, %xmm0
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %al
-; X86-NO-BMI2-NO-SHLD-NEXT: shrb $3, %al
-; X86-NO-BMI2-NO-SHLD-NEXT: andb $28, %al
-; X86-NO-BMI2-NO-SHLD-NEXT: negb %al
-; X86-NO-BMI2-NO-SHLD-NEXT: movsbl %al, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 64(%esp,%ebx), %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 68(%esp,%ebx), %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %dl
-; X86-NO-BMI2-NO-SHLD-NEXT: andb $31, %dl
-; X86-NO-BMI2-NO-SHLD-NEXT: xorb $31, %dl
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dl, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %eax, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 76(%esp,%ebx), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 72(%esp,%ebx), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dl, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dl, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %eax, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 84(%esp,%ebx), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %esi, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 80(%esp,%ebx), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dl, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %edi, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dl, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %eax, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 92(%esp,%ebx), %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 88(%esp,%ebx), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dl, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %edi, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dl, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %eax, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, (%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %esi, 24(%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, 28(%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, 16(%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebp, 20(%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, 8(%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, 12(%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, 4(%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: addl $108, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: retl
+; X64-NO-BMI2-LABEL: shl_32bytes:
+; X64-NO-BMI2: # %bb.0:
+; X64-NO-BMI2-NEXT: movq (%rdi), %rax
+; X64-NO-BMI2-NEXT: movq 8(%rdi), %r8
+; X64-NO-BMI2-NEXT: movq 16(%rdi), %r9
+; X64-NO-BMI2-NEXT: movq 24(%rdi), %rdi
+; X64-NO-BMI2-NEXT: movzbl (%rsi), %ecx
+; X64-NO-BMI2-NEXT: xorps %xmm0, %xmm0
+; X64-NO-BMI2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movl %ecx, %eax
+; X64-NO-BMI2-NEXT: shrb $3, %al
+; X64-NO-BMI2-NEXT: andb $24, %al
+; X64-NO-BMI2-NEXT: negb %al
+; X64-NO-BMI2-NEXT: movsbq %al, %rax
+; X64-NO-BMI2-NEXT: movq -24(%rsp,%rax), %rsi
+; X64-NO-BMI2-NEXT: movq -16(%rsp,%rax), %rdi
+; X64-NO-BMI2-NEXT: shldq %cl, %rsi, %rdi
+; X64-NO-BMI2-NEXT: movq -40(%rsp,%rax), %r8
+; X64-NO-BMI2-NEXT: movq -32(%rsp,%rax), %rax
+; X64-NO-BMI2-NEXT: shldq %cl, %rax, %rsi
+; X64-NO-BMI2-NEXT: shldq %cl, %r8, %rax
+; X64-NO-BMI2-NEXT: shlq %cl, %r8
+; X64-NO-BMI2-NEXT: movq %rsi, 16(%rdx)
+; X64-NO-BMI2-NEXT: movq %rdi, 24(%rdx)
+; X64-NO-BMI2-NEXT: movq %r8, (%rdx)
+; X64-NO-BMI2-NEXT: movq %rax, 8(%rdx)
+; X64-NO-BMI2-NEXT: retq
;
-; X86-NO-BMI2-HAVE-SHLD-LABEL: shl_32bytes:
-; X86-NO-BMI2-HAVE-SHLD: # %bb.0:
-; X86-NO-BMI2-HAVE-SHLD-NEXT: pushl %ebp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: pushl %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: pushl %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: subl $92, %esp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl (%ebp), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 4(%ebp), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, (%esp) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 8(%ebp), %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 12(%ebp), %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 16(%ebp), %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movzbl (%ecx), %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 20(%ebp), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 24(%ebp), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 28(%ebp), %ebp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ebp, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: xorps %xmm0, %xmm0
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl (%esp), %eax # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrb $3, %al
-; X86-NO-BMI2-HAVE-SHLD-NEXT: andb $28, %al
-; X86-NO-BMI2-HAVE-SHLD-NEXT: negb %al
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movsbl %al, %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 56(%esp,%eax), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 60(%esp,%eax), %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ebx, %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shldl %cl, %edx, %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 52(%esp,%eax), %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %esi, (%esp) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shldl %cl, %esi, %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 64(%esp,%eax), %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 68(%esp,%eax), %ebp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shldl %cl, %edi, %ebp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shldl %cl, %ebx, %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 48(%esp,%eax), %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 72(%esp,%eax), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 76(%esp,%eax), %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shldl %cl, %edx, %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shldl %cl, %eax, %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, 24(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %esi, 28(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edi, 16(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ebp, 20(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, 8(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, 12(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl (%esp), %edx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shldl %cl, %ebx, %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shll %cl, %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ebx, (%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, 4(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: addl $92, %esp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: popl %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: popl %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: popl %ebp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: retl
+; X64-BMI2-LABEL: shl_32bytes:
+; X64-BMI2: # %bb.0:
+; X64-BMI2-NEXT: movq (%rdi), %rax
+; X64-BMI2-NEXT: movq 8(%rdi), %r8
+; X64-BMI2-NEXT: movq 16(%rdi), %r9
+; X64-BMI2-NEXT: movq 24(%rdi), %rdi
+; X64-BMI2-NEXT: movzbl (%rsi), %ecx
+; X64-BMI2-NEXT: xorps %xmm0, %xmm0
+; X64-BMI2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movl %ecx, %eax
+; X64-BMI2-NEXT: shrb $3, %al
+; X64-BMI2-NEXT: andb $24, %al
+; X64-BMI2-NEXT: negb %al
+; X64-BMI2-NEXT: movsbq %al, %rax
+; X64-BMI2-NEXT: movq -24(%rsp,%rax), %rsi
+; X64-BMI2-NEXT: movq -16(%rsp,%rax), %rdi
+; X64-BMI2-NEXT: shldq %cl, %rsi, %rdi
+; X64-BMI2-NEXT: movq -40(%rsp,%rax), %r8
+; X64-BMI2-NEXT: movq -32(%rsp,%rax), %rax
+; X64-BMI2-NEXT: shldq %cl, %rax, %rsi
+; X64-BMI2-NEXT: shldq %cl, %r8, %rax
+; X64-BMI2-NEXT: shlxq %rcx, %r8, %rcx
+; X64-BMI2-NEXT: movq %rsi, 16(%rdx)
+; X64-BMI2-NEXT: movq %rdi, 24(%rdx)
+; X64-BMI2-NEXT: movq %rcx, (%rdx)
+; X64-BMI2-NEXT: movq %rax, 8(%rdx)
+; X64-BMI2-NEXT: retq
;
-; X86-HAVE-BMI2-NO-SHLD-LABEL: shl_32bytes:
-; X86-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: subl $108, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl (%eax), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 4(%eax), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 8(%eax), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 12(%eax), %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 16(%eax), %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 20(%eax), %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 24(%eax), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 28(%eax), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movzbl (%eax), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: xorps %xmm0, %xmm0
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ebp, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrb $3, %dl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: andb $28, %dl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: negb %dl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movsbl %dl, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 64(%esp,%esi), %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 68(%esp,%esi), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %eax, %edx, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: andb $31, %cl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: xorb $31, %cl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, %ebx, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %edi, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 72(%esp,%esi), %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ebx, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, %edi, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 76(%esp,%esi), %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ebp, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ebp, %edi, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %ebp, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %esi, %ebx, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrl %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, %edx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %ebx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 80(%esp,%ebp), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edx, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, %ebx, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 84(%esp,%ebp), %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %esi, %ebx, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %ebp, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %esi, %edx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, %edi, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %edx, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %esi, {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %esi, 92(%esp,%edx), %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 88(%esp,%edx), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %eax, %esi, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, %esi, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %ebp, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, %ebx, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %edx, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edx, (%ecx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, 24(%ecx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, 28(%ecx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edi, 16(%ecx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, 20(%ecx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, 8(%ecx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, 12(%ecx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, 4(%ecx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl $108, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: retl
+; X86-NO-BMI2-LABEL: shl_32bytes:
+; X86-NO-BMI2: # %bb.0:
+; X86-NO-BMI2-NEXT: pushl %ebp
+; X86-NO-BMI2-NEXT: pushl %ebx
+; X86-NO-BMI2-NEXT: pushl %edi
+; X86-NO-BMI2-NEXT: pushl %esi
+; X86-NO-BMI2-NEXT: subl $92, %esp
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NO-BMI2-NEXT: movl (%ebp), %eax
+; X86-NO-BMI2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 4(%ebp), %eax
+; X86-NO-BMI2-NEXT: movl %eax, (%esp) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 8(%ebp), %esi
+; X86-NO-BMI2-NEXT: movl 12(%ebp), %edi
+; X86-NO-BMI2-NEXT: movl 16(%ebp), %ebx
+; X86-NO-BMI2-NEXT: movzbl (%ecx), %ecx
+; X86-NO-BMI2-NEXT: movl 20(%ebp), %edx
+; X86-NO-BMI2-NEXT: movl 24(%ebp), %eax
+; X86-NO-BMI2-NEXT: movl 28(%ebp), %ebp
+; X86-NO-BMI2-NEXT: movl %ebp, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: xorps %xmm0, %xmm0
+; X86-NO-BMI2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl (%esp), %eax # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %ecx, %eax
+; X86-NO-BMI2-NEXT: shrb $3, %al
+; X86-NO-BMI2-NEXT: andb $28, %al
+; X86-NO-BMI2-NEXT: negb %al
+; X86-NO-BMI2-NEXT: movsbl %al, %eax
+; X86-NO-BMI2-NEXT: movl 56(%esp,%eax), %edx
+; X86-NO-BMI2-NEXT: movl 60(%esp,%eax), %ebx
+; X86-NO-BMI2-NEXT: movl %ebx, %esi
+; X86-NO-BMI2-NEXT: shldl %cl, %edx, %esi
+; X86-NO-BMI2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 52(%esp,%eax), %esi
+; X86-NO-BMI2-NEXT: movl %esi, (%esp) # 4-byte Spill
+; X86-NO-BMI2-NEXT: shldl %cl, %esi, %edx
+; X86-NO-BMI2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 64(%esp,%eax), %edi
+; X86-NO-BMI2-NEXT: movl 68(%esp,%eax), %ebp
+; X86-NO-BMI2-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: shldl %cl, %edi, %ebp
+; X86-NO-BMI2-NEXT: shldl %cl, %ebx, %edi
+; X86-NO-BMI2-NEXT: movl 48(%esp,%eax), %ebx
+; X86-NO-BMI2-NEXT: movl 72(%esp,%eax), %edx
+; X86-NO-BMI2-NEXT: movl 76(%esp,%eax), %esi
+; X86-NO-BMI2-NEXT: shldl %cl, %edx, %esi
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-BMI2-NEXT: shldl %cl, %eax, %edx
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-BMI2-NEXT: movl %edx, 24(%eax)
+; X86-NO-BMI2-NEXT: movl %esi, 28(%eax)
+; X86-NO-BMI2-NEXT: movl %edi, 16(%eax)
+; X86-NO-BMI2-NEXT: movl %ebp, 20(%eax)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %edx, 8(%eax)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %edx, 12(%eax)
+; X86-NO-BMI2-NEXT: movl (%esp), %edx # 4-byte Reload
+; X86-NO-BMI2-NEXT: shldl %cl, %ebx, %edx
+; X86-NO-BMI2-NEXT: shll %cl, %ebx
+; X86-NO-BMI2-NEXT: movl %ebx, (%eax)
+; X86-NO-BMI2-NEXT: movl %edx, 4(%eax)
+; X86-NO-BMI2-NEXT: addl $92, %esp
+; X86-NO-BMI2-NEXT: popl %esi
+; X86-NO-BMI2-NEXT: popl %edi
+; X86-NO-BMI2-NEXT: popl %ebx
+; X86-NO-BMI2-NEXT: popl %ebp
+; X86-NO-BMI2-NEXT: retl
;
-; X86-HAVE-BMI2-HAVE-SHLD-LABEL: shl_32bytes:
-; X86-HAVE-BMI2-HAVE-SHLD: # %bb.0:
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: pushl %ebp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: pushl %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: pushl %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: subl $92, %esp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl (%ecx), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 4(%ecx), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 8(%ecx), %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 12(%ecx), %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 16(%ecx), %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 20(%ecx), %ebp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 24(%ecx), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 28(%ecx), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movzbl (%ecx), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: xorps %xmm0, %xmm0
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ebp, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrb $3, %al
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: andb $28, %al
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: negb %al
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movsbl %al, %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 56(%esp,%eax), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 60(%esp,%eax), %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %esi, (%esp) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shldl %cl, %edx, %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 52(%esp,%eax), %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shldl %cl, %ebx, %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 64(%esp,%eax), %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 68(%esp,%eax), %ebp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shldl %cl, %edi, %ebp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl (%esp), %edx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shldl %cl, %edx, %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 48(%esp,%eax), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, (%esp) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 72(%esp,%eax), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 76(%esp,%eax), %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shldl %cl, %edx, %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shldl %cl, %eax, %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, 24(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %esi, 28(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edi, 16(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ebp, 20(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, 8(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, 12(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl (%esp), %esi # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shlxl %ecx, %esi, %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, (%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shldl %cl, %esi, %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ebx, 4(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: addl $92, %esp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: popl %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: popl %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: popl %ebp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: retl
+; X86-BMI2-LABEL: shl_32bytes:
+; X86-BMI2: # %bb.0:
+; X86-BMI2-NEXT: pushl %ebp
+; X86-BMI2-NEXT: pushl %ebx
+; X86-BMI2-NEXT: pushl %edi
+; X86-BMI2-NEXT: pushl %esi
+; X86-BMI2-NEXT: subl $92, %esp
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-BMI2-NEXT: movl (%ecx), %eax
+; X86-BMI2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 4(%ecx), %eax
+; X86-BMI2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 8(%ecx), %esi
+; X86-BMI2-NEXT: movl 12(%ecx), %edi
+; X86-BMI2-NEXT: movl 16(%ecx), %ebx
+; X86-BMI2-NEXT: movl 20(%ecx), %ebp
+; X86-BMI2-NEXT: movl 24(%ecx), %edx
+; X86-BMI2-NEXT: movl 28(%ecx), %eax
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-BMI2-NEXT: movzbl (%ecx), %ecx
+; X86-BMI2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: xorps %xmm0, %xmm0
+; X86-BMI2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %ebp, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-BMI2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-BMI2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %ecx, %eax
+; X86-BMI2-NEXT: shrb $3, %al
+; X86-BMI2-NEXT: andb $28, %al
+; X86-BMI2-NEXT: negb %al
+; X86-BMI2-NEXT: movsbl %al, %eax
+; X86-BMI2-NEXT: movl 56(%esp,%eax), %edx
+; X86-BMI2-NEXT: movl 60(%esp,%eax), %esi
+; X86-BMI2-NEXT: movl %esi, (%esp) # 4-byte Spill
+; X86-BMI2-NEXT: shldl %cl, %edx, %esi
+; X86-BMI2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 52(%esp,%eax), %ebx
+; X86-BMI2-NEXT: shldl %cl, %ebx, %edx
+; X86-BMI2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 64(%esp,%eax), %edi
+; X86-BMI2-NEXT: movl 68(%esp,%eax), %ebp
+; X86-BMI2-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: shldl %cl, %edi, %ebp
+; X86-BMI2-NEXT: movl (%esp), %edx # 4-byte Reload
+; X86-BMI2-NEXT: shldl %cl, %edx, %edi
+; X86-BMI2-NEXT: movl 48(%esp,%eax), %edx
+; X86-BMI2-NEXT: movl %edx, (%esp) # 4-byte Spill
+; X86-BMI2-NEXT: movl 72(%esp,%eax), %edx
+; X86-BMI2-NEXT: movl 76(%esp,%eax), %esi
+; X86-BMI2-NEXT: shldl %cl, %edx, %esi
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-BMI2-NEXT: shldl %cl, %eax, %edx
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT: movl %edx, 24(%eax)
+; X86-BMI2-NEXT: movl %esi, 28(%eax)
+; X86-BMI2-NEXT: movl %edi, 16(%eax)
+; X86-BMI2-NEXT: movl %ebp, 20(%eax)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-BMI2-NEXT: movl %edx, 8(%eax)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-BMI2-NEXT: movl %edx, 12(%eax)
+; X86-BMI2-NEXT: movl (%esp), %esi # 4-byte Reload
+; X86-BMI2-NEXT: shlxl %ecx, %esi, %edx
+; X86-BMI2-NEXT: movl %edx, (%eax)
+; X86-BMI2-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-BMI2-NEXT: shldl %cl, %esi, %ebx
+; X86-BMI2-NEXT: movl %ebx, 4(%eax)
+; X86-BMI2-NEXT: addl $92, %esp
+; X86-BMI2-NEXT: popl %esi
+; X86-BMI2-NEXT: popl %edi
+; X86-BMI2-NEXT: popl %ebx
+; X86-BMI2-NEXT: popl %ebp
+; X86-BMI2-NEXT: retl
%src = load i256, ptr %src.ptr, align 1
%bitOff = load i256, ptr %bitOff.ptr, align 1
%res = shl i256 %src, %bitOff
@@ -2510,579 +1180,242 @@ define void @shl_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
ret void
}
define void @ashr_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
-; X64-NO-BMI2-NO-SHLD-LABEL: ashr_32bytes:
-; X64-NO-BMI2-NO-SHLD: # %bb.0:
-; X64-NO-BMI2-NO-SHLD-NEXT: pushq %rbx
-; X64-NO-BMI2-NO-SHLD-NEXT: movq (%rdi), %rcx
-; X64-NO-BMI2-NO-SHLD-NEXT: movq 8(%rdi), %r8
-; X64-NO-BMI2-NO-SHLD-NEXT: movq 16(%rdi), %r9
-; X64-NO-BMI2-NO-SHLD-NEXT: movq 24(%rdi), %rdi
-; X64-NO-BMI2-NO-SHLD-NEXT: movzbl (%rsi), %eax
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: sarq $63, %rdi
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrb $6, %cl
-; X64-NO-BMI2-NO-SHLD-NEXT: movzbl %cl, %r8d
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -64(%rsp,%r8,8), %r10
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -56(%rsp,%r8,8), %rdi
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rdi, %r11
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %r11
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %esi
-; X64-NO-BMI2-NO-SHLD-NEXT: andb $63, %sil
-; X64-NO-BMI2-NO-SHLD-NEXT: xorb $63, %sil
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -48(%rsp,%r8,8), %rbx
-; X64-NO-BMI2-NO-SHLD-NEXT: leaq (%rbx,%rbx), %r9
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %esi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %r9
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %r11, %r9
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %r10
-; X64-NO-BMI2-NO-SHLD-NEXT: addq %rdi, %rdi
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %esi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %rdi
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %r10, %rdi
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %rbx
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -40(%rsp,%r8,8), %r8
-; X64-NO-BMI2-NO-SHLD-NEXT: leaq (%r8,%r8), %r10
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %esi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %r10
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %rbx, %r10
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: sarq %cl, %r8
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r8, 24(%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r10, 16(%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rdi, (%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r9, 8(%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: popq %rbx
-; X64-NO-BMI2-NO-SHLD-NEXT: retq
-;
-; X64-NO-BMI2-HAVE-SHLD-LABEL: ashr_32bytes:
-; X64-NO-BMI2-HAVE-SHLD: # %bb.0:
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq (%rdi), %rax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq 8(%rdi), %r8
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq 16(%rdi), %r9
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq 24(%rdi), %rdi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movzbl (%rsi), %ecx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: sarq $63, %rdi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, %eax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrb $6, %al
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movzbl %al, %eax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -56(%rsp,%rax,8), %rsi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -72(%rsp,%rax,8), %rdi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -64(%rsp,%rax,8), %r8
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r8, %r9
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %rsi, %r9
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -48(%rsp,%rax,8), %rax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %rax, %rsi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %r8, %rdi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: sarq %cl, %rax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rsi, 16(%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rax, 24(%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rdi, (%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r9, 8(%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: retq
-;
-; X64-HAVE-BMI2-NO-SHLD-LABEL: ashr_32bytes:
-; X64-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq (%rdi), %rcx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq 8(%rdi), %r8
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq 16(%rdi), %r9
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq 24(%rdi), %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movzbl (%rsi), %eax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: sarq $63, %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, %esi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrb $6, %sil
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movzbl %sil, %esi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq -64(%rsp,%rsi,8), %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq -56(%rsp,%rsi,8), %r8
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rcx, %rdi, %r9
-; X64-HAVE-BMI2-NO-SHLD-NEXT: andb $63, %al
-; X64-HAVE-BMI2-NO-SHLD-NEXT: xorb $63, %al
-; X64-HAVE-BMI2-NO-SHLD-NEXT: leaq (%r8,%r8), %r10
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rax, %r10, %r10
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %r9, %r10
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rcx, -72(%rsp,%rsi,8), %r9
-; X64-HAVE-BMI2-NO-SHLD-NEXT: addq %rdi, %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rax, %rdi, %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %r9, %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rcx, %r8, %r8
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq -48(%rsp,%rsi,8), %rsi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: leaq (%rsi,%rsi), %r9
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rax, %r9, %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %r8, %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: sarxq %rcx, %rsi, %rcx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rcx, 24(%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rax, 16(%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rdi, (%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %r10, 8(%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: retq
-;
-; X64-HAVE-BMI2-HAVE-SHLD-LABEL: ashr_32bytes:
-; X64-HAVE-BMI2-HAVE-SHLD: # %bb.0:
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq (%rdi), %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq 8(%rdi), %r8
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq 16(%rdi), %r9
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq 24(%rdi), %rdi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movzbl (%rsi), %ecx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: sarq $63, %rdi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, %eax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrb $6, %al
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movzbl %al, %eax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -56(%rsp,%rax,8), %rsi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -72(%rsp,%rax,8), %rdi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -64(%rsp,%rax,8), %r8
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %r8, %r9
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %rsi, %r9
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -48(%rsp,%rax,8), %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %rax, %rsi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %r8, %rdi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: sarxq %rcx, %rax, %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rsi, 16(%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rax, 24(%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rdi, (%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %r9, 8(%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: retq
-;
-; X86-NO-BMI2-NO-SHLD-LABEL: ashr_32bytes:
-; X86-NO-BMI2-NO-SHLD: # %bb.0:
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: subl $108, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl (%edx), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 4(%edx), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 8(%edx), %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 12(%edx), %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 16(%edx), %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movzbl (%ecx), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 20(%edx), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 24(%edx), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 28(%edx), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebp, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: sarl $31, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: shrb $5, %al
-; X86-NO-BMI2-NO-SHLD-NEXT: movzbl %al, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 32(%esp,%ebp,4), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 36(%esp,%ebp,4), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %cl, %ch
-; X86-NO-BMI2-NO-SHLD-NEXT: andb $31, %ch
-; X86-NO-BMI2-NO-SHLD-NEXT: xorb $31, %ch
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 40(%esp,%ebp,4), %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%edi,%edi), %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %eax, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 1-byte Folded Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %bl, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %edx, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebp, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 44(%esp,%ebp,4), %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebp, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %bl, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 48(%esp,%esi,4), %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %esi, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%edi,%edi), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %eax, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dl, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %ebp, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %eax, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 52(%esp,%ebx,4), %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dl, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 56(%esp,%ebx,4), %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%ebx,%ebx), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %eax, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dl, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %edi, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %eax, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dl, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 60(%esp,%eax,4), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%eax,%eax), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %ebx, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: sarl %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, 28(%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, 24(%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, 16(%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %esi, 20(%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebp, 8(%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, 12(%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, (%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, 4(%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: addl $108, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: retl
+; X64-NO-BMI2-LABEL: ashr_32bytes:
+; X64-NO-BMI2: # %bb.0:
+; X64-NO-BMI2-NEXT: movq (%rdi), %rax
+; X64-NO-BMI2-NEXT: movq 8(%rdi), %r8
+; X64-NO-BMI2-NEXT: movq 16(%rdi), %r9
+; X64-NO-BMI2-NEXT: movq 24(%rdi), %rdi
+; X64-NO-BMI2-NEXT: movzbl (%rsi), %ecx
+; X64-NO-BMI2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: sarq $63, %rdi
+; X64-NO-BMI2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movl %ecx, %eax
+; X64-NO-BMI2-NEXT: shrb $6, %al
+; X64-NO-BMI2-NEXT: movzbl %al, %eax
+; X64-NO-BMI2-NEXT: movq -56(%rsp,%rax,8), %rsi
+; X64-NO-BMI2-NEXT: movq -72(%rsp,%rax,8), %rdi
+; X64-NO-BMI2-NEXT: movq -64(%rsp,%rax,8), %r8
+; X64-NO-BMI2-NEXT: movq %r8, %r9
+; X64-NO-BMI2-NEXT: shrdq %cl, %rsi, %r9
+; X64-NO-BMI2-NEXT: movq -48(%rsp,%rax,8), %rax
+; X64-NO-BMI2-NEXT: shrdq %cl, %rax, %rsi
+; X64-NO-BMI2-NEXT: shrdq %cl, %r8, %rdi
+; X64-NO-BMI2-NEXT: sarq %cl, %rax
+; X64-NO-BMI2-NEXT: movq %rsi, 16(%rdx)
+; X64-NO-BMI2-NEXT: movq %rax, 24(%rdx)
+; X64-NO-BMI2-NEXT: movq %rdi, (%rdx)
+; X64-NO-BMI2-NEXT: movq %r9, 8(%rdx)
+; X64-NO-BMI2-NEXT: retq
;
-; X86-NO-BMI2-HAVE-SHLD-LABEL: ashr_32bytes:
-; X86-NO-BMI2-HAVE-SHLD: # %bb.0:
-; X86-NO-BMI2-HAVE-SHLD-NEXT: pushl %ebp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: pushl %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: pushl %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: subl $92, %esp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl (%edx), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 4(%edx), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, (%esp) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 8(%edx), %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 12(%edx), %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 16(%edx), %ebp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movzbl (%ecx), %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 20(%edx), %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 24(%edx), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 28(%edx), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ebp, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl (%esp), %eax # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: sarl $31, %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrb $5, %al
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movzbl %al, %ebp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 24(%esp,%ebp,4), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 20(%esp,%ebp,4), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, (%esp) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edx, %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 32(%esp,%ebp,4), %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 28(%esp,%ebp,4), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %ebx, %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %eax, %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 40(%esp,%ebp,4), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 36(%esp,%ebp,4), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edx, %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %eax, %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 16(%esp,%ebp,4), %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 44(%esp,%ebp,4), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %eax, %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, 24(%ebp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl (%esp), %edx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edx, %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: sarl %cl, %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, 28(%ebp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ebx, 16(%ebp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edi, 20(%ebp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, 8(%ebp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, 12(%ebp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %esi, (%ebp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, 4(%ebp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: addl $92, %esp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: popl %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: popl %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: popl %ebp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: retl
+; X64-BMI2-LABEL: ashr_32bytes:
+; X64-BMI2: # %bb.0:
+; X64-BMI2-NEXT: movq (%rdi), %rax
+; X64-BMI2-NEXT: movq 8(%rdi), %r8
+; X64-BMI2-NEXT: movq 16(%rdi), %r9
+; X64-BMI2-NEXT: movq 24(%rdi), %rdi
+; X64-BMI2-NEXT: movzbl (%rsi), %ecx
+; X64-BMI2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: sarq $63, %rdi
+; X64-BMI2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movl %ecx, %eax
+; X64-BMI2-NEXT: shrb $6, %al
+; X64-BMI2-NEXT: movzbl %al, %eax
+; X64-BMI2-NEXT: movq -56(%rsp,%rax,8), %rsi
+; X64-BMI2-NEXT: movq -72(%rsp,%rax,8), %rdi
+; X64-BMI2-NEXT: movq -64(%rsp,%rax,8), %r8
+; X64-BMI2-NEXT: movq %r8, %r9
+; X64-BMI2-NEXT: shrdq %cl, %rsi, %r9
+; X64-BMI2-NEXT: movq -48(%rsp,%rax,8), %rax
+; X64-BMI2-NEXT: shrdq %cl, %rax, %rsi
+; X64-BMI2-NEXT: shrdq %cl, %r8, %rdi
+; X64-BMI2-NEXT: sarxq %rcx, %rax, %rax
+; X64-BMI2-NEXT: movq %rsi, 16(%rdx)
+; X64-BMI2-NEXT: movq %rax, 24(%rdx)
+; X64-BMI2-NEXT: movq %rdi, (%rdx)
+; X64-BMI2-NEXT: movq %r9, 8(%rdx)
+; X64-BMI2-NEXT: retq
;
-; X86-HAVE-BMI2-NO-SHLD-LABEL: ashr_32bytes:
-; X86-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: subl $108, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl (%eax), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 4(%eax), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 8(%eax), %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 12(%eax), %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 16(%eax), %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 20(%eax), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 24(%eax), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 28(%eax), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movzbl (%eax), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ebp, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: sarl $31, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrb $5, %dl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movzbl %dl, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 36(%esp,%esi,4), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 40(%esp,%esi,4), %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, %edx, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: andb $31, %al
-; X86-HAVE-BMI2-NO-SHLD-NEXT: xorb $31, %al
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%ebp,%ebp), %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %eax, %ebx, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %edi, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, 32(%esp,%esi,4), %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %edx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %eax, %edx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %edi, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 48(%esp,%esi,4), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %edx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %eax, %edx, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 44(%esp,%esi,4), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, %edx, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %edi, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, %ebp, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %edx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %eax, %edx, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %edi, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 56(%esp,%esi,4), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %edx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %eax, %edx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 52(%esp,%esi,4), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, %eax, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %ebx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %eax, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %edi, %eax, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %ebx, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 60(%esp,%esi,4), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%esi,%esi), %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %edi, %ebx, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %edi, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: sarxl %ecx, %esi, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, 28(%esi)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ebx, 24(%esi)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, 16(%esi)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edx, 20(%esi)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ebp, 8(%esi)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, 12(%esi)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, (%esi)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, 4(%esi)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl $108, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: retl
+; X86-NO-BMI2-LABEL: ashr_32bytes:
+; X86-NO-BMI2: # %bb.0:
+; X86-NO-BMI2-NEXT: pushl %ebp
+; X86-NO-BMI2-NEXT: pushl %ebx
+; X86-NO-BMI2-NEXT: pushl %edi
+; X86-NO-BMI2-NEXT: pushl %esi
+; X86-NO-BMI2-NEXT: subl $92, %esp
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NO-BMI2-NEXT: movl (%edx), %eax
+; X86-NO-BMI2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 4(%edx), %eax
+; X86-NO-BMI2-NEXT: movl %eax, (%esp) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 8(%edx), %edi
+; X86-NO-BMI2-NEXT: movl 12(%edx), %ebx
+; X86-NO-BMI2-NEXT: movl 16(%edx), %ebp
+; X86-NO-BMI2-NEXT: movzbl (%ecx), %ecx
+; X86-NO-BMI2-NEXT: movl 20(%edx), %esi
+; X86-NO-BMI2-NEXT: movl 24(%edx), %eax
+; X86-NO-BMI2-NEXT: movl 28(%edx), %edx
+; X86-NO-BMI2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %ebp, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl (%esp), %eax # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: sarl $31, %edx
+; X86-NO-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %ecx, %eax
+; X86-NO-BMI2-NEXT: shrb $5, %al
+; X86-NO-BMI2-NEXT: movzbl %al, %ebp
+; X86-NO-BMI2-NEXT: movl 24(%esp,%ebp,4), %edx
+; X86-NO-BMI2-NEXT: movl 20(%esp,%ebp,4), %eax
+; X86-NO-BMI2-NEXT: movl %eax, (%esp) # 4-byte Spill
+; X86-NO-BMI2-NEXT: shrdl %cl, %edx, %eax
+; X86-NO-BMI2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 32(%esp,%ebp,4), %ebx
+; X86-NO-BMI2-NEXT: movl 28(%esp,%ebp,4), %eax
+; X86-NO-BMI2-NEXT: movl %eax, %esi
+; X86-NO-BMI2-NEXT: shrdl %cl, %ebx, %esi
+; X86-NO-BMI2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: shrdl %cl, %eax, %edx
+; X86-NO-BMI2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 40(%esp,%ebp,4), %edx
+; X86-NO-BMI2-NEXT: movl 36(%esp,%ebp,4), %eax
+; X86-NO-BMI2-NEXT: movl %eax, %edi
+; X86-NO-BMI2-NEXT: shrdl %cl, %edx, %edi
+; X86-NO-BMI2-NEXT: shrdl %cl, %eax, %ebx
+; X86-NO-BMI2-NEXT: movl 16(%esp,%ebp,4), %esi
+; X86-NO-BMI2-NEXT: movl 44(%esp,%ebp,4), %eax
+; X86-NO-BMI2-NEXT: shrdl %cl, %eax, %edx
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NO-BMI2-NEXT: movl %edx, 24(%ebp)
+; X86-NO-BMI2-NEXT: movl (%esp), %edx # 4-byte Reload
+; X86-NO-BMI2-NEXT: shrdl %cl, %edx, %esi
+; X86-NO-BMI2-NEXT: sarl %cl, %eax
+; X86-NO-BMI2-NEXT: movl %eax, 28(%ebp)
+; X86-NO-BMI2-NEXT: movl %ebx, 16(%ebp)
+; X86-NO-BMI2-NEXT: movl %edi, 20(%ebp)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %eax, 8(%ebp)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %eax, 12(%ebp)
+; X86-NO-BMI2-NEXT: movl %esi, (%ebp)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %eax, 4(%ebp)
+; X86-NO-BMI2-NEXT: addl $92, %esp
+; X86-NO-BMI2-NEXT: popl %esi
+; X86-NO-BMI2-NEXT: popl %edi
+; X86-NO-BMI2-NEXT: popl %ebx
+; X86-NO-BMI2-NEXT: popl %ebp
+; X86-NO-BMI2-NEXT: retl
;
-; X86-HAVE-BMI2-HAVE-SHLD-LABEL: ashr_32bytes:
-; X86-HAVE-BMI2-HAVE-SHLD: # %bb.0:
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: pushl %ebp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: pushl %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: pushl %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: subl $92, %esp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl (%eax), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 4(%eax), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, (%esp) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 8(%eax), %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 12(%eax), %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 16(%eax), %ebp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 20(%eax), %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 24(%eax), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 28(%eax), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movzbl (%ecx), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ebp, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl (%esp), %edx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: sarl $31, %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrb $5, %al
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movzbl %al, %ebp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 24(%esp,%ebp,4), %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 20(%esp,%ebp,4), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %esi, %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, (%esp) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 32(%esp,%ebp,4), %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 28(%esp,%ebp,4), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %ebx, %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %eax, %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 40(%esp,%ebp,4), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 36(%esp,%ebp,4), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %eax, %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edx, %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 16(%esp,%ebp,4), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 44(%esp,%ebp,4), %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edi, %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, 24(%ebp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: sarxl %ecx, %edi, %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, 28(%ebp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ebx, 16(%ebp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %esi, 20(%ebp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, 8(%ebp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, 12(%ebp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %eax, %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, (%ebp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl (%esp), %eax # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, 4(%ebp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: addl $92, %esp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: popl %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: popl %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: popl %ebp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: retl
+; X86-BMI2-LABEL: ashr_32bytes:
+; X86-BMI2: # %bb.0:
+; X86-BMI2-NEXT: pushl %ebp
+; X86-BMI2-NEXT: pushl %ebx
+; X86-BMI2-NEXT: pushl %edi
+; X86-BMI2-NEXT: pushl %esi
+; X86-BMI2-NEXT: subl $92, %esp
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT: movl (%eax), %ecx
+; X86-BMI2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 4(%eax), %ecx
+; X86-BMI2-NEXT: movl %ecx, (%esp) # 4-byte Spill
+; X86-BMI2-NEXT: movl 8(%eax), %edi
+; X86-BMI2-NEXT: movl 12(%eax), %ebx
+; X86-BMI2-NEXT: movl 16(%eax), %ebp
+; X86-BMI2-NEXT: movl 20(%eax), %esi
+; X86-BMI2-NEXT: movl 24(%eax), %edx
+; X86-BMI2-NEXT: movl 28(%eax), %eax
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-BMI2-NEXT: movzbl (%ecx), %ecx
+; X86-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %ebp, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl (%esp), %edx # 4-byte Reload
+; X86-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: sarl $31, %eax
+; X86-BMI2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %ecx, %eax
+; X86-BMI2-NEXT: shrb $5, %al
+; X86-BMI2-NEXT: movzbl %al, %ebp
+; X86-BMI2-NEXT: movl 24(%esp,%ebp,4), %esi
+; X86-BMI2-NEXT: movl 20(%esp,%ebp,4), %eax
+; X86-BMI2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: shrdl %cl, %esi, %eax
+; X86-BMI2-NEXT: movl %eax, (%esp) # 4-byte Spill
+; X86-BMI2-NEXT: movl 32(%esp,%ebp,4), %ebx
+; X86-BMI2-NEXT: movl 28(%esp,%ebp,4), %eax
+; X86-BMI2-NEXT: movl %eax, %edx
+; X86-BMI2-NEXT: shrdl %cl, %ebx, %edx
+; X86-BMI2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: shrdl %cl, %eax, %esi
+; X86-BMI2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 40(%esp,%ebp,4), %eax
+; X86-BMI2-NEXT: movl 36(%esp,%ebp,4), %edx
+; X86-BMI2-NEXT: movl %edx, %esi
+; X86-BMI2-NEXT: shrdl %cl, %eax, %esi
+; X86-BMI2-NEXT: shrdl %cl, %edx, %ebx
+; X86-BMI2-NEXT: movl 16(%esp,%ebp,4), %edx
+; X86-BMI2-NEXT: movl 44(%esp,%ebp,4), %edi
+; X86-BMI2-NEXT: shrdl %cl, %edi, %eax
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-BMI2-NEXT: movl %eax, 24(%ebp)
+; X86-BMI2-NEXT: sarxl %ecx, %edi, %eax
+; X86-BMI2-NEXT: movl %eax, 28(%ebp)
+; X86-BMI2-NEXT: movl %ebx, 16(%ebp)
+; X86-BMI2-NEXT: movl %esi, 20(%ebp)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-BMI2-NEXT: movl %eax, 8(%ebp)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-BMI2-NEXT: movl %eax, 12(%ebp)
+; X86-BMI2-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-BMI2-NEXT: shrdl %cl, %eax, %edx
+; X86-BMI2-NEXT: movl %edx, (%ebp)
+; X86-BMI2-NEXT: movl (%esp), %eax # 4-byte Reload
+; X86-BMI2-NEXT: movl %eax, 4(%ebp)
+; X86-BMI2-NEXT: addl $92, %esp
+; X86-BMI2-NEXT: popl %esi
+; X86-BMI2-NEXT: popl %edi
+; X86-BMI2-NEXT: popl %ebx
+; X86-BMI2-NEXT: popl %ebp
+; X86-BMI2-NEXT: retl
%src = load i256, ptr %src.ptr, align 1
%bitOff = load i256, ptr %bitOff.ptr, align 1
%res = ashr i256 %src, %bitOff
@@ -3091,1120 +1424,448 @@ define void @ashr_32bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
}
define void @lshr_64bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
-; X64-NO-BMI2-NO-SHLD-LABEL: lshr_64bytes:
-; X64-NO-BMI2-NO-SHLD: # %bb.0:
-; X64-NO-BMI2-NO-SHLD-NEXT: pushq %rbp
-; X64-NO-BMI2-NO-SHLD-NEXT: pushq %r15
-; X64-NO-BMI2-NO-SHLD-NEXT: pushq %r14
-; X64-NO-BMI2-NO-SHLD-NEXT: pushq %r13
-; X64-NO-BMI2-NO-SHLD-NEXT: pushq %r12
-; X64-NO-BMI2-NO-SHLD-NEXT: pushq %rbx
-; X64-NO-BMI2-NO-SHLD-NEXT: pushq %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: movq (%rdi), %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: movq 8(%rdi), %rcx
-; X64-NO-BMI2-NO-SHLD-NEXT: movq 16(%rdi), %r9
-; X64-NO-BMI2-NO-SHLD-NEXT: movq 24(%rdi), %r10
-; X64-NO-BMI2-NO-SHLD-NEXT: movq 32(%rdi), %r11
-; X64-NO-BMI2-NO-SHLD-NEXT: movq 40(%rdi), %rbx
-; X64-NO-BMI2-NO-SHLD-NEXT: movq 48(%rdi), %r14
-; X64-NO-BMI2-NO-SHLD-NEXT: movq 56(%rdi), %rdi
-; X64-NO-BMI2-NO-SHLD-NEXT: movl (%rsi), %r8d
-; X64-NO-BMI2-NO-SHLD-NEXT: xorps %xmm0, %xmm0
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r14, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rbx, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r10, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %r8d, %eax
-; X64-NO-BMI2-NO-SHLD-NEXT: andl $63, %eax
-; X64-NO-BMI2-NO-SHLD-NEXT: shrl $3, %r8d
-; X64-NO-BMI2-NO-SHLD-NEXT: andl $56, %r8d
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -128(%rsp,%r8), %r11
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -120(%rsp,%r8), %r9
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r9, %rsi
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %rsi
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %edi
-; X64-NO-BMI2-NO-SHLD-NEXT: notl %edi
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -112(%rsp,%r8), %r14
-; X64-NO-BMI2-NO-SHLD-NEXT: leaq (%r14,%r14), %r10
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %edi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %r10
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %rsi, %r10
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %r11
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %esi
-; X64-NO-BMI2-NO-SHLD-NEXT: xorb $63, %sil
-; X64-NO-BMI2-NO-SHLD-NEXT: addq %r9, %r9
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %esi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %r9
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %r11, %r9
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -104(%rsp,%r8), %r11
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r11, %r12
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %r12
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -96(%rsp,%r8), %r15
-; X64-NO-BMI2-NO-SHLD-NEXT: leaq (%r15,%r15), %rbx
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %edi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %rbx
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %r12, %rbx
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %r14
-; X64-NO-BMI2-NO-SHLD-NEXT: addq %r11, %r11
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %esi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %r11
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %r14, %r11
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -88(%rsp,%r8), %r14
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r14, %r13
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %r13
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -80(%rsp,%r8), %rbp
-; X64-NO-BMI2-NO-SHLD-NEXT: leaq (%rbp,%rbp), %r12
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %edi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %r12
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %r13, %r12
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %r15
-; X64-NO-BMI2-NO-SHLD-NEXT: addq %r14, %r14
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %esi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %r14
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %r15, %r14
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %rbp
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -72(%rsp,%r8), %rdi
-; X64-NO-BMI2-NO-SHLD-NEXT: leaq (%rdi,%rdi), %r8
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %esi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %r8
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %rbp, %r8
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %rdi
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rdi, 56(%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r8, 48(%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r14, 32(%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r12, 40(%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r11, 16(%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rbx, 24(%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r9, (%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r10, 8(%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: addq $8, %rsp
-; X64-NO-BMI2-NO-SHLD-NEXT: popq %rbx
-; X64-NO-BMI2-NO-SHLD-NEXT: popq %r12
-; X64-NO-BMI2-NO-SHLD-NEXT: popq %r13
-; X64-NO-BMI2-NO-SHLD-NEXT: popq %r14
-; X64-NO-BMI2-NO-SHLD-NEXT: popq %r15
-; X64-NO-BMI2-NO-SHLD-NEXT: popq %rbp
-; X64-NO-BMI2-NO-SHLD-NEXT: retq
-;
-; X64-NO-BMI2-HAVE-SHLD-LABEL: lshr_64bytes:
-; X64-NO-BMI2-HAVE-SHLD: # %bb.0:
-; X64-NO-BMI2-HAVE-SHLD-NEXT: pushq %r15
-; X64-NO-BMI2-HAVE-SHLD-NEXT: pushq %r14
-; X64-NO-BMI2-HAVE-SHLD-NEXT: pushq %rbx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq (%rdi), %rcx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq 8(%rdi), %r8
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq 16(%rdi), %r9
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq 24(%rdi), %r10
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq 32(%rdi), %r11
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq 40(%rdi), %rbx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq 48(%rdi), %r14
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq 56(%rdi), %rdi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movl (%rsi), %eax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: xorps %xmm0, %xmm0
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r14, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rbx, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r10, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: andl $63, %ecx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrl $3, %eax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: andl $56, %eax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -112(%rsp,%rax), %rdi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -128(%rsp,%rax), %rsi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -120(%rsp,%rax), %r9
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r9, %r8
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %rdi, %r8
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -96(%rsp,%rax), %r10
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -104(%rsp,%rax), %r11
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r11, %rbx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %r10, %rbx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %r11, %rdi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -80(%rsp,%rax), %r11
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -88(%rsp,%rax), %r14
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r14, %r15
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %r11, %r15
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %r14, %r10
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -72(%rsp,%rax), %rax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %rax, %r11
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %r9, %rsi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrq %cl, %rax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r11, 48(%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rax, 56(%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r10, 32(%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r15, 40(%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rdi, 16(%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rbx, 24(%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rsi, (%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r8, 8(%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: popq %rbx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: popq %r14
-; X64-NO-BMI2-HAVE-SHLD-NEXT: popq %r15
-; X64-NO-BMI2-HAVE-SHLD-NEXT: retq
-;
-; X64-HAVE-BMI2-NO-SHLD-LABEL: lshr_64bytes:
-; X64-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X64-HAVE-BMI2-NO-SHLD-NEXT: pushq %r15
-; X64-HAVE-BMI2-NO-SHLD-NEXT: pushq %r14
-; X64-HAVE-BMI2-NO-SHLD-NEXT: pushq %r13
-; X64-HAVE-BMI2-NO-SHLD-NEXT: pushq %r12
-; X64-HAVE-BMI2-NO-SHLD-NEXT: pushq %rbx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq (%rdi), %rcx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq 8(%rdi), %r8
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq 16(%rdi), %r9
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq 24(%rdi), %r10
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq 32(%rdi), %r11
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq 40(%rdi), %rbx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq 48(%rdi), %r14
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq 56(%rdi), %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movl (%rsi), %eax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: xorps %xmm0, %xmm0
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %r14, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rbx, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %r10, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, %esi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: andl $63, %esi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, %ecx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrl $3, %eax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: andl $56, %eax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq -120(%rsp,%rax), %r8
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq -112(%rsp,%rax), %r11
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rcx, %r8, %r9
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, %r10d
-; X64-HAVE-BMI2-NO-SHLD-NEXT: notl %r10d
-; X64-HAVE-BMI2-NO-SHLD-NEXT: leaq (%r11,%r11), %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %r10, %rdi, %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %r9, %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rcx, -128(%rsp,%rax), %r9
-; X64-HAVE-BMI2-NO-SHLD-NEXT: xorb $63, %sil
-; X64-HAVE-BMI2-NO-SHLD-NEXT: addq %r8, %r8
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rsi, %r8, %r8
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %r9, %r8
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq -104(%rsp,%rax), %rbx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rcx, %rbx, %r14
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq -96(%rsp,%rax), %r15
-; X64-HAVE-BMI2-NO-SHLD-NEXT: leaq (%r15,%r15), %r9
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %r10, %r9, %r9
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %r14, %r9
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rcx, %r11, %r11
-; X64-HAVE-BMI2-NO-SHLD-NEXT: addq %rbx, %rbx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rsi, %rbx, %rbx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %r11, %rbx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq -88(%rsp,%rax), %r11
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rcx, %r11, %r14
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq -80(%rsp,%rax), %r12
-; X64-HAVE-BMI2-NO-SHLD-NEXT: leaq (%r12,%r12), %r13
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %r10, %r13, %r10
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %r14, %r10
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rcx, %r15, %r14
-; X64-HAVE-BMI2-NO-SHLD-NEXT: addq %r11, %r11
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rsi, %r11, %r11
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %r14, %r11
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rcx, %r12, %r14
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq -72(%rsp,%rax), %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: leaq (%rax,%rax), %r15
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rsi, %r15, %rsi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %r14, %rsi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rcx, %rax, %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rax, 56(%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rsi, 48(%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %r11, 32(%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %r10, 40(%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rbx, 16(%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %r9, 24(%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %r8, (%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rdi, 8(%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: popq %rbx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: popq %r12
-; X64-HAVE-BMI2-NO-SHLD-NEXT: popq %r13
-; X64-HAVE-BMI2-NO-SHLD-NEXT: popq %r14
-; X64-HAVE-BMI2-NO-SHLD-NEXT: popq %r15
-; X64-HAVE-BMI2-NO-SHLD-NEXT: retq
-;
-; X64-HAVE-BMI2-HAVE-SHLD-LABEL: lshr_64bytes:
-; X64-HAVE-BMI2-HAVE-SHLD: # %bb.0:
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: pushq %r15
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: pushq %r14
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: pushq %rbx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq (%rdi), %rcx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq 8(%rdi), %r8
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq 16(%rdi), %r9
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq 24(%rdi), %r10
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq 32(%rdi), %r11
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq 40(%rdi), %rbx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq 48(%rdi), %r14
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq 56(%rdi), %rdi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movl (%rsi), %eax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: xorps %xmm0, %xmm0
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %r14, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rbx, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %r10, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, %ecx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: andl $63, %ecx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrl $3, %eax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: andl $56, %eax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -112(%rsp,%rax), %rdi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -128(%rsp,%rax), %rsi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -120(%rsp,%rax), %r9
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %r9, %r8
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %rdi, %r8
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -96(%rsp,%rax), %r10
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -104(%rsp,%rax), %r11
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %r11, %rbx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %r10, %rbx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %r11, %rdi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -80(%rsp,%rax), %r11
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -88(%rsp,%rax), %r14
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %r14, %r15
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %r11, %r15
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %r14, %r10
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -72(%rsp,%rax), %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %rax, %r11
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %r9, %rsi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrxq %rcx, %rax, %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %r11, 48(%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rax, 56(%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %r10, 32(%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %r15, 40(%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rdi, 16(%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rbx, 24(%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rsi, (%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %r8, 8(%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: popq %rbx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: popq %r14
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: popq %r15
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: retq
-;
-; X86-NO-BMI2-NO-SHLD-LABEL: lshr_64bytes:
-; X86-NO-BMI2-NO-SHLD: # %bb.0:
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: subl $204, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl (%edi), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 4(%edi), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, (%esp) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 8(%edi), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 12(%edi), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 16(%edi), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 20(%edi), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 24(%edi), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 28(%edi), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 32(%edi), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 36(%edi), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 40(%edi), %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 44(%edi), %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 48(%edi), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 52(%edi), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 56(%edi), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 60(%edi), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl (%edi), %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: xorps %xmm0, %xmm0
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebp, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl (%esp), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: andl $31, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl $3, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: andl $60, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 68(%esp,%edi), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: notl %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 72(%esp,%edi), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%edx,%edx), %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 64(%esp,%edi), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: xorb $31, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %cl, (%esp) # 1-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %eax, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 76(%esp,%edi), %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebp, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 80(%esp,%edi), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%esi,%esi), %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %ebx, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %eax, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movzbl (%esp), %ecx # 1-byte Folded Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %edx, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 84(%esp,%edi), %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebp, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 88(%esp,%edi), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %eax, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %ebx, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %ebp, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movzbl (%esp), %ecx # 1-byte Folded Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 92(%esp,%edi), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 96(%esp,%edi), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%eax,%eax), %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %edx, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movzbl (%esp), %ecx # 1-byte Folded Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %eax, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 100(%esp,%edi), %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebp, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 104(%esp,%edi), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%edx,%edx), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %ebp, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movzbl (%esp), %ecx # 1-byte Folded Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 108(%esp,%edi), %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 112(%esp,%edi), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%ecx,%ecx), %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %ebx, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movzbl (%esp), %ecx # 1-byte Folded Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %edx, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 116(%esp,%edi), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %esi, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebp, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 120(%esp,%edi), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%ecx,%ecx), %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %eax, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %esi, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movb (%esp), %ch # 1-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %edx, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %al, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 124(%esp,%edi), %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%edi,%edi), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %eax, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, 60(%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, 56(%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %esi, 48(%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebp, 52(%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, 40(%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, 44(%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, 32(%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, 36(%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, 24(%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, 28(%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, 16(%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, 20(%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, 8(%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, 12(%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, (%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, 4(%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: addl $204, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: retl
+; X64-NO-BMI2-LABEL: lshr_64bytes:
+; X64-NO-BMI2: # %bb.0:
+; X64-NO-BMI2-NEXT: pushq %r15
+; X64-NO-BMI2-NEXT: pushq %r14
+; X64-NO-BMI2-NEXT: pushq %rbx
+; X64-NO-BMI2-NEXT: movq (%rdi), %rcx
+; X64-NO-BMI2-NEXT: movq 8(%rdi), %r8
+; X64-NO-BMI2-NEXT: movq 16(%rdi), %r9
+; X64-NO-BMI2-NEXT: movq 24(%rdi), %r10
+; X64-NO-BMI2-NEXT: movq 32(%rdi), %r11
+; X64-NO-BMI2-NEXT: movq 40(%rdi), %rbx
+; X64-NO-BMI2-NEXT: movq 48(%rdi), %r14
+; X64-NO-BMI2-NEXT: movq 56(%rdi), %rdi
+; X64-NO-BMI2-NEXT: movl (%rsi), %eax
+; X64-NO-BMI2-NEXT: xorps %xmm0, %xmm0
+; X64-NO-BMI2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movq %r14, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movq %rbx, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movq %r10, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movl %eax, %ecx
+; X64-NO-BMI2-NEXT: andl $63, %ecx
+; X64-NO-BMI2-NEXT: shrl $3, %eax
+; X64-NO-BMI2-NEXT: andl $56, %eax
+; X64-NO-BMI2-NEXT: movq -112(%rsp,%rax), %rdi
+; X64-NO-BMI2-NEXT: movq -128(%rsp,%rax), %rsi
+; X64-NO-BMI2-NEXT: movq -120(%rsp,%rax), %r9
+; X64-NO-BMI2-NEXT: movq %r9, %r8
+; X64-NO-BMI2-NEXT: shrdq %cl, %rdi, %r8
+; X64-NO-BMI2-NEXT: movq -96(%rsp,%rax), %r10
+; X64-NO-BMI2-NEXT: movq -104(%rsp,%rax), %r11
+; X64-NO-BMI2-NEXT: movq %r11, %rbx
+; X64-NO-BMI2-NEXT: shrdq %cl, %r10, %rbx
+; X64-NO-BMI2-NEXT: shrdq %cl, %r11, %rdi
+; X64-NO-BMI2-NEXT: movq -80(%rsp,%rax), %r11
+; X64-NO-BMI2-NEXT: movq -88(%rsp,%rax), %r14
+; X64-NO-BMI2-NEXT: movq %r14, %r15
+; X64-NO-BMI2-NEXT: shrdq %cl, %r11, %r15
+; X64-NO-BMI2-NEXT: shrdq %cl, %r14, %r10
+; X64-NO-BMI2-NEXT: movq -72(%rsp,%rax), %rax
+; X64-NO-BMI2-NEXT: shrdq %cl, %rax, %r11
+; X64-NO-BMI2-NEXT: shrdq %cl, %r9, %rsi
+; X64-NO-BMI2-NEXT: # kill: def $cl killed $cl killed $ecx
+; X64-NO-BMI2-NEXT: shrq %cl, %rax
+; X64-NO-BMI2-NEXT: movq %r11, 48(%rdx)
+; X64-NO-BMI2-NEXT: movq %rax, 56(%rdx)
+; X64-NO-BMI2-NEXT: movq %r10, 32(%rdx)
+; X64-NO-BMI2-NEXT: movq %r15, 40(%rdx)
+; X64-NO-BMI2-NEXT: movq %rdi, 16(%rdx)
+; X64-NO-BMI2-NEXT: movq %rbx, 24(%rdx)
+; X64-NO-BMI2-NEXT: movq %rsi, (%rdx)
+; X64-NO-BMI2-NEXT: movq %r8, 8(%rdx)
+; X64-NO-BMI2-NEXT: popq %rbx
+; X64-NO-BMI2-NEXT: popq %r14
+; X64-NO-BMI2-NEXT: popq %r15
+; X64-NO-BMI2-NEXT: retq
;
-; X86-NO-BMI2-HAVE-SHLD-LABEL: lshr_64bytes:
-; X86-NO-BMI2-HAVE-SHLD: # %bb.0:
-; X86-NO-BMI2-HAVE-SHLD-NEXT: pushl %ebp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: pushl %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: pushl %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: subl $188, %esp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl (%eax), %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 4(%eax), %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 8(%eax), %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 12(%eax), %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 16(%eax), %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 20(%eax), %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 24(%eax), %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 28(%eax), %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 32(%eax), %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 36(%eax), %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, (%esp) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 40(%eax), %ebp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 44(%eax), %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 48(%eax), %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 52(%eax), %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 56(%eax), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 60(%eax), %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl (%eax), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: xorps %xmm0, %xmm0
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ebp, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl (%esp), %ecx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: andl $31, %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrl $3, %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: andl $60, %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 56(%esp,%eax), %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 52(%esp,%eax), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %esi, %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 64(%esp,%eax), %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 60(%esp,%eax), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %ebx, %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edx, %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 72(%esp,%eax), %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 68(%esp,%eax), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edi, %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edx, %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 80(%esp,%eax), %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 76(%esp,%eax), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %ebx, %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edx, %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 88(%esp,%eax), %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 84(%esp,%eax), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edi, %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edi, %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edx, %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ebx, (%esp) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 96(%esp,%eax), %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 92(%esp,%eax), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edi, %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edx, %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 104(%esp,%eax), %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 100(%esp,%eax), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %esi, %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edx, %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 48(%esp,%eax), %ebp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 108(%esp,%eax), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edx, %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %esi, 56(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %esi, %ebp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrl %cl, %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, 60(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edi, 48(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ebx, 52(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, 40(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, 44(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl (%esp), %ecx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, 32(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, 36(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, 24(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, 28(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, 16(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, 20(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, 8(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, 12(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ebp, (%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, 4(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: addl $188, %esp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: popl %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: popl %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: popl %ebp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: retl
+; X64-BMI2-LABEL: lshr_64bytes:
+; X64-BMI2: # %bb.0:
+; X64-BMI2-NEXT: pushq %r15
+; X64-BMI2-NEXT: pushq %r14
+; X64-BMI2-NEXT: pushq %rbx
+; X64-BMI2-NEXT: movq (%rdi), %rcx
+; X64-BMI2-NEXT: movq 8(%rdi), %r8
+; X64-BMI2-NEXT: movq 16(%rdi), %r9
+; X64-BMI2-NEXT: movq 24(%rdi), %r10
+; X64-BMI2-NEXT: movq 32(%rdi), %r11
+; X64-BMI2-NEXT: movq 40(%rdi), %rbx
+; X64-BMI2-NEXT: movq 48(%rdi), %r14
+; X64-BMI2-NEXT: movq 56(%rdi), %rdi
+; X64-BMI2-NEXT: movl (%rsi), %eax
+; X64-BMI2-NEXT: xorps %xmm0, %xmm0
+; X64-BMI2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movq %r14, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movq %rbx, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movq %r10, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movl %eax, %ecx
+; X64-BMI2-NEXT: andl $63, %ecx
+; X64-BMI2-NEXT: shrl $3, %eax
+; X64-BMI2-NEXT: andl $56, %eax
+; X64-BMI2-NEXT: movq -112(%rsp,%rax), %rdi
+; X64-BMI2-NEXT: movq -128(%rsp,%rax), %rsi
+; X64-BMI2-NEXT: movq -120(%rsp,%rax), %r9
+; X64-BMI2-NEXT: movq %r9, %r8
+; X64-BMI2-NEXT: shrdq %cl, %rdi, %r8
+; X64-BMI2-NEXT: movq -96(%rsp,%rax), %r10
+; X64-BMI2-NEXT: movq -104(%rsp,%rax), %r11
+; X64-BMI2-NEXT: movq %r11, %rbx
+; X64-BMI2-NEXT: shrdq %cl, %r10, %rbx
+; X64-BMI2-NEXT: shrdq %cl, %r11, %rdi
+; X64-BMI2-NEXT: movq -80(%rsp,%rax), %r11
+; X64-BMI2-NEXT: movq -88(%rsp,%rax), %r14
+; X64-BMI2-NEXT: movq %r14, %r15
+; X64-BMI2-NEXT: shrdq %cl, %r11, %r15
+; X64-BMI2-NEXT: shrdq %cl, %r14, %r10
+; X64-BMI2-NEXT: movq -72(%rsp,%rax), %rax
+; X64-BMI2-NEXT: shrdq %cl, %rax, %r11
+; X64-BMI2-NEXT: shrdq %cl, %r9, %rsi
+; X64-BMI2-NEXT: shrxq %rcx, %rax, %rax
+; X64-BMI2-NEXT: movq %r11, 48(%rdx)
+; X64-BMI2-NEXT: movq %rax, 56(%rdx)
+; X64-BMI2-NEXT: movq %r10, 32(%rdx)
+; X64-BMI2-NEXT: movq %r15, 40(%rdx)
+; X64-BMI2-NEXT: movq %rdi, 16(%rdx)
+; X64-BMI2-NEXT: movq %rbx, 24(%rdx)
+; X64-BMI2-NEXT: movq %rsi, (%rdx)
+; X64-BMI2-NEXT: movq %r8, 8(%rdx)
+; X64-BMI2-NEXT: popq %rbx
+; X64-BMI2-NEXT: popq %r14
+; X64-BMI2-NEXT: popq %r15
+; X64-BMI2-NEXT: retq
;
-; X86-HAVE-BMI2-NO-SHLD-LABEL: lshr_64bytes:
-; X86-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: subl $204, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl (%eax), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 4(%eax), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 8(%eax), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 12(%eax), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 16(%eax), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 20(%eax), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 24(%eax), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 28(%eax), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 32(%eax), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 36(%eax), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 40(%eax), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 44(%eax), %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 48(%eax), %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 52(%eax), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 56(%eax), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 60(%eax), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl (%eax), %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: xorps %xmm0, %xmm0
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ebp, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ebx, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: andl $31, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrl $3, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: andl $60, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 68(%esp,%ebx), %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 72(%esp,%ebx), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: notl %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%esi,%esi), %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %eax, %ebp, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %edx, %edi, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %ebp, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: xorb $31, %cl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %edi, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %edi, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %edx, 64(%esp,%ebx), %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %edi, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 80(%esp,%ebx), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%esi,%esi), %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %eax, %ebp, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 76(%esp,%ebx), %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %edx, %ebp, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %esi, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %edx, {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %ebp, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %ebp, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %esi, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 88(%esp,%ebx), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%eax,%eax), %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %edi, %ebp, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 84(%esp,%ebx), %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %edx, %ebp, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %esi, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %edx, {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %ebp, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %ebp, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %esi, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 96(%esp,%ebx), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%eax,%eax), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %edi, %esi, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 92(%esp,%ebx), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %edx, %esi, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %ebp, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %edx, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %esi, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %esi, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %ebp, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 104(%esp,%ebx), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%eax,%eax), %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %edi, %ebp, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 100(%esp,%ebx), %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %edx, %ebp, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %esi, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %edx, {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %ebp, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %ebp, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %esi, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 112(%esp,%ebx), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%eax,%eax), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edi, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %edi, %esi, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 108(%esp,%ebx), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %edx, %esi, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %edi, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %edx, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %esi, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %esi, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %edi, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 120(%esp,%ebx), %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%edi,%edi), %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %eax, %ebp, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 116(%esp,%ebx), %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %edx, %ebp, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %esi, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %edx, {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %ebp, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %ebp, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %esi, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 124(%esp,%ebx), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%eax,%eax), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %esi, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %edx, %edi, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %edi, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %edx, %eax, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edx, 60(%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, 56(%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ebp, 48(%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, 52(%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, 40(%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, 44(%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, 32(%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, 36(%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, 24(%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, 28(%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, 16(%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, 20(%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, 8(%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, 12(%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, (%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, 4(%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl $204, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: retl
+; X86-NO-BMI2-LABEL: lshr_64bytes:
+; X86-NO-BMI2: # %bb.0:
+; X86-NO-BMI2-NEXT: pushl %ebp
+; X86-NO-BMI2-NEXT: pushl %ebx
+; X86-NO-BMI2-NEXT: pushl %edi
+; X86-NO-BMI2-NEXT: pushl %esi
+; X86-NO-BMI2-NEXT: subl $188, %esp
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-BMI2-NEXT: movl (%eax), %ecx
+; X86-NO-BMI2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 4(%eax), %ecx
+; X86-NO-BMI2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 8(%eax), %ecx
+; X86-NO-BMI2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 12(%eax), %ecx
+; X86-NO-BMI2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 16(%eax), %ecx
+; X86-NO-BMI2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 20(%eax), %ecx
+; X86-NO-BMI2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 24(%eax), %ecx
+; X86-NO-BMI2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 28(%eax), %ecx
+; X86-NO-BMI2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 32(%eax), %ecx
+; X86-NO-BMI2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 36(%eax), %ecx
+; X86-NO-BMI2-NEXT: movl %ecx, (%esp) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 40(%eax), %ebp
+; X86-NO-BMI2-NEXT: movl 44(%eax), %ebx
+; X86-NO-BMI2-NEXT: movl 48(%eax), %edi
+; X86-NO-BMI2-NEXT: movl 52(%eax), %esi
+; X86-NO-BMI2-NEXT: movl 56(%eax), %edx
+; X86-NO-BMI2-NEXT: movl 60(%eax), %ecx
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-BMI2-NEXT: movl (%eax), %eax
+; X86-NO-BMI2-NEXT: xorps %xmm0, %xmm0
+; X86-NO-BMI2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %ebp, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl (%esp), %ecx # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %eax, %ecx
+; X86-NO-BMI2-NEXT: andl $31, %ecx
+; X86-NO-BMI2-NEXT: shrl $3, %eax
+; X86-NO-BMI2-NEXT: andl $60, %eax
+; X86-NO-BMI2-NEXT: movl 56(%esp,%eax), %esi
+; X86-NO-BMI2-NEXT: movl 52(%esp,%eax), %edx
+; X86-NO-BMI2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: shrdl %cl, %esi, %edx
+; X86-NO-BMI2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 64(%esp,%eax), %ebx
+; X86-NO-BMI2-NEXT: movl 60(%esp,%eax), %edx
+; X86-NO-BMI2-NEXT: movl %edx, %edi
+; X86-NO-BMI2-NEXT: shrdl %cl, %ebx, %edi
+; X86-NO-BMI2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: shrdl %cl, %edx, %esi
+; X86-NO-BMI2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 72(%esp,%eax), %edi
+; X86-NO-BMI2-NEXT: movl 68(%esp,%eax), %edx
+; X86-NO-BMI2-NEXT: movl %edx, %esi
+; X86-NO-BMI2-NEXT: shrdl %cl, %edi, %esi
+; X86-NO-BMI2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: shrdl %cl, %edx, %ebx
+; X86-NO-BMI2-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 80(%esp,%eax), %ebx
+; X86-NO-BMI2-NEXT: movl 76(%esp,%eax), %edx
+; X86-NO-BMI2-NEXT: movl %edx, %esi
+; X86-NO-BMI2-NEXT: shrdl %cl, %ebx, %esi
+; X86-NO-BMI2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: shrdl %cl, %edx, %edi
+; X86-NO-BMI2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 88(%esp,%eax), %edi
+; X86-NO-BMI2-NEXT: movl 84(%esp,%eax), %edx
+; X86-NO-BMI2-NEXT: movl %edx, %esi
+; X86-NO-BMI2-NEXT: shrdl %cl, %edi, %esi
+; X86-NO-BMI2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl %edi, %esi
+; X86-NO-BMI2-NEXT: shrdl %cl, %edx, %ebx
+; X86-NO-BMI2-NEXT: movl %ebx, (%esp) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 96(%esp,%eax), %edi
+; X86-NO-BMI2-NEXT: movl 92(%esp,%eax), %edx
+; X86-NO-BMI2-NEXT: movl %edx, %ebx
+; X86-NO-BMI2-NEXT: shrdl %cl, %edi, %ebx
+; X86-NO-BMI2-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: shrdl %cl, %edx, %esi
+; X86-NO-BMI2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 104(%esp,%eax), %esi
+; X86-NO-BMI2-NEXT: movl 100(%esp,%eax), %edx
+; X86-NO-BMI2-NEXT: movl %edx, %ebx
+; X86-NO-BMI2-NEXT: shrdl %cl, %esi, %ebx
+; X86-NO-BMI2-NEXT: shrdl %cl, %edx, %edi
+; X86-NO-BMI2-NEXT: movl 48(%esp,%eax), %ebp
+; X86-NO-BMI2-NEXT: movl 108(%esp,%eax), %edx
+; X86-NO-BMI2-NEXT: shrdl %cl, %edx, %esi
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-BMI2-NEXT: movl %esi, 56(%eax)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NO-BMI2-NEXT: shrdl %cl, %esi, %ebp
+; X86-NO-BMI2-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NO-BMI2-NEXT: shrl %cl, %edx
+; X86-NO-BMI2-NEXT: movl %edx, 60(%eax)
+; X86-NO-BMI2-NEXT: movl %edi, 48(%eax)
+; X86-NO-BMI2-NEXT: movl %ebx, 52(%eax)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %ecx, 40(%eax)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %ecx, 44(%eax)
+; X86-NO-BMI2-NEXT: movl (%esp), %ecx # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %ecx, 32(%eax)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %ecx, 36(%eax)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %ecx, 24(%eax)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %ecx, 28(%eax)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %ecx, 16(%eax)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %ecx, 20(%eax)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %ecx, 8(%eax)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %ecx, 12(%eax)
+; X86-NO-BMI2-NEXT: movl %ebp, (%eax)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %ecx, 4(%eax)
+; X86-NO-BMI2-NEXT: addl $188, %esp
+; X86-NO-BMI2-NEXT: popl %esi
+; X86-NO-BMI2-NEXT: popl %edi
+; X86-NO-BMI2-NEXT: popl %ebx
+; X86-NO-BMI2-NEXT: popl %ebp
+; X86-NO-BMI2-NEXT: retl
;
-; X86-HAVE-BMI2-HAVE-SHLD-LABEL: lshr_64bytes:
-; X86-HAVE-BMI2-HAVE-SHLD: # %bb.0:
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: pushl %ebp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: pushl %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: pushl %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: subl $188, %esp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl (%eax), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 4(%eax), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 8(%eax), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 12(%eax), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 16(%eax), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 20(%eax), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 24(%eax), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 28(%eax), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 32(%eax), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 36(%eax), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, (%esp) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 40(%eax), %ebp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 44(%eax), %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 48(%eax), %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 52(%eax), %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 56(%eax), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 60(%eax), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl (%eax), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: xorps %xmm0, %xmm0
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ebp, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl (%esp), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: andl $31, %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrl $3, %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: andl $60, %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 56(%esp,%eax), %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 52(%esp,%eax), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edi, %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 64(%esp,%eax), %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 60(%esp,%eax), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %ebx, %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edx, %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 72(%esp,%eax), %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 68(%esp,%eax), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edi, %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edx, %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 80(%esp,%eax), %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 76(%esp,%eax), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %ebx, %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edx, %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 88(%esp,%eax), %ebp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 84(%esp,%eax), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %ebp, %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edx, %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ebx, (%esp) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 96(%esp,%eax), %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 92(%esp,%eax), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edi, %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edx, %ebp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 104(%esp,%eax), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 100(%esp,%eax), %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ebx, %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edx, %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %ebx, %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 48(%esp,%eax), %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 108(%esp,%eax), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %eax, %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, 56(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edi, 48(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %esi, 52(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ebp, 40(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, 44(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl (%esp), %edx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, 32(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, 36(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, 24(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, 28(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, 16(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, 20(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, 8(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, 12(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %esi, %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ebx, (%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, 4(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, 60(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: addl $188, %esp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: popl %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: popl %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: popl %ebp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: retl
+; X86-BMI2-LABEL: lshr_64bytes:
+; X86-BMI2: # %bb.0:
+; X86-BMI2-NEXT: pushl %ebp
+; X86-BMI2-NEXT: pushl %ebx
+; X86-BMI2-NEXT: pushl %edi
+; X86-BMI2-NEXT: pushl %esi
+; X86-BMI2-NEXT: subl $188, %esp
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT: movl (%eax), %ecx
+; X86-BMI2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 4(%eax), %ecx
+; X86-BMI2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 8(%eax), %ecx
+; X86-BMI2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 12(%eax), %ecx
+; X86-BMI2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 16(%eax), %ecx
+; X86-BMI2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 20(%eax), %ecx
+; X86-BMI2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 24(%eax), %ecx
+; X86-BMI2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 28(%eax), %ecx
+; X86-BMI2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 32(%eax), %ecx
+; X86-BMI2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 36(%eax), %ecx
+; X86-BMI2-NEXT: movl %ecx, (%esp) # 4-byte Spill
+; X86-BMI2-NEXT: movl 40(%eax), %ebp
+; X86-BMI2-NEXT: movl 44(%eax), %ebx
+; X86-BMI2-NEXT: movl 48(%eax), %edi
+; X86-BMI2-NEXT: movl 52(%eax), %esi
+; X86-BMI2-NEXT: movl 56(%eax), %edx
+; X86-BMI2-NEXT: movl 60(%eax), %ecx
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT: movl (%eax), %eax
+; X86-BMI2-NEXT: xorps %xmm0, %xmm0
+; X86-BMI2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %ebp, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl (%esp), %ecx # 4-byte Reload
+; X86-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %eax, %ecx
+; X86-BMI2-NEXT: andl $31, %ecx
+; X86-BMI2-NEXT: shrl $3, %eax
+; X86-BMI2-NEXT: andl $60, %eax
+; X86-BMI2-NEXT: movl 56(%esp,%eax), %edi
+; X86-BMI2-NEXT: movl 52(%esp,%eax), %edx
+; X86-BMI2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: shrdl %cl, %edi, %edx
+; X86-BMI2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 64(%esp,%eax), %ebx
+; X86-BMI2-NEXT: movl 60(%esp,%eax), %edx
+; X86-BMI2-NEXT: movl %edx, %esi
+; X86-BMI2-NEXT: shrdl %cl, %ebx, %esi
+; X86-BMI2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: shrdl %cl, %edx, %edi
+; X86-BMI2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 72(%esp,%eax), %edi
+; X86-BMI2-NEXT: movl 68(%esp,%eax), %edx
+; X86-BMI2-NEXT: movl %edx, %esi
+; X86-BMI2-NEXT: shrdl %cl, %edi, %esi
+; X86-BMI2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: shrdl %cl, %edx, %ebx
+; X86-BMI2-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 80(%esp,%eax), %ebx
+; X86-BMI2-NEXT: movl 76(%esp,%eax), %edx
+; X86-BMI2-NEXT: movl %edx, %esi
+; X86-BMI2-NEXT: shrdl %cl, %ebx, %esi
+; X86-BMI2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: shrdl %cl, %edx, %edi
+; X86-BMI2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 88(%esp,%eax), %ebp
+; X86-BMI2-NEXT: movl 84(%esp,%eax), %edx
+; X86-BMI2-NEXT: movl %edx, %esi
+; X86-BMI2-NEXT: shrdl %cl, %ebp, %esi
+; X86-BMI2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: shrdl %cl, %edx, %ebx
+; X86-BMI2-NEXT: movl %ebx, (%esp) # 4-byte Spill
+; X86-BMI2-NEXT: movl 96(%esp,%eax), %edi
+; X86-BMI2-NEXT: movl 92(%esp,%eax), %edx
+; X86-BMI2-NEXT: movl %edx, %esi
+; X86-BMI2-NEXT: shrdl %cl, %edi, %esi
+; X86-BMI2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: shrdl %cl, %edx, %ebp
+; X86-BMI2-NEXT: movl 104(%esp,%eax), %edx
+; X86-BMI2-NEXT: movl 100(%esp,%eax), %ebx
+; X86-BMI2-NEXT: movl %ebx, %esi
+; X86-BMI2-NEXT: shrdl %cl, %edx, %esi
+; X86-BMI2-NEXT: shrdl %cl, %ebx, %edi
+; X86-BMI2-NEXT: movl 48(%esp,%eax), %ebx
+; X86-BMI2-NEXT: movl 108(%esp,%eax), %eax
+; X86-BMI2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: shrdl %cl, %eax, %edx
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT: movl %edx, 56(%eax)
+; X86-BMI2-NEXT: movl %edi, 48(%eax)
+; X86-BMI2-NEXT: movl %esi, 52(%eax)
+; X86-BMI2-NEXT: movl %ebp, 40(%eax)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-BMI2-NEXT: movl %edx, 44(%eax)
+; X86-BMI2-NEXT: movl (%esp), %edx # 4-byte Reload
+; X86-BMI2-NEXT: movl %edx, 32(%eax)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-BMI2-NEXT: movl %edx, 36(%eax)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-BMI2-NEXT: movl %edx, 24(%eax)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-BMI2-NEXT: movl %edx, 28(%eax)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-BMI2-NEXT: movl %edx, 16(%eax)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-BMI2-NEXT: movl %edx, 20(%eax)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-BMI2-NEXT: movl %edx, 8(%eax)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-BMI2-NEXT: movl %edx, 12(%eax)
+; X86-BMI2-NEXT: shrxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X86-BMI2-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-BMI2-NEXT: shrdl %cl, %esi, %ebx
+; X86-BMI2-NEXT: movl %ebx, (%eax)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-BMI2-NEXT: movl %ecx, 4(%eax)
+; X86-BMI2-NEXT: movl %edx, 60(%eax)
+; X86-BMI2-NEXT: addl $188, %esp
+; X86-BMI2-NEXT: popl %esi
+; X86-BMI2-NEXT: popl %edi
+; X86-BMI2-NEXT: popl %ebx
+; X86-BMI2-NEXT: popl %ebp
+; X86-BMI2-NEXT: retl
%src = load i512, ptr %src.ptr, align 1
%bitOff = load i512, ptr %bitOff.ptr, align 1
%res = lshr i512 %src, %bitOff
@@ -4212,1137 +1873,454 @@ define void @lshr_64bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
ret void
}
define void @shl_64bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
-; X64-NO-BMI2-NO-SHLD-LABEL: shl_64bytes:
-; X64-NO-BMI2-NO-SHLD: # %bb.0:
-; X64-NO-BMI2-NO-SHLD-NEXT: pushq %r15
-; X64-NO-BMI2-NO-SHLD-NEXT: pushq %r14
-; X64-NO-BMI2-NO-SHLD-NEXT: pushq %r13
-; X64-NO-BMI2-NO-SHLD-NEXT: pushq %r12
-; X64-NO-BMI2-NO-SHLD-NEXT: pushq %rbx
-; X64-NO-BMI2-NO-SHLD-NEXT: movq (%rdi), %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: movq 8(%rdi), %rcx
-; X64-NO-BMI2-NO-SHLD-NEXT: movq 16(%rdi), %r8
-; X64-NO-BMI2-NO-SHLD-NEXT: movq 24(%rdi), %r9
-; X64-NO-BMI2-NO-SHLD-NEXT: movq 32(%rdi), %r10
-; X64-NO-BMI2-NO-SHLD-NEXT: movq 40(%rdi), %r11
-; X64-NO-BMI2-NO-SHLD-NEXT: movq 48(%rdi), %rbx
-; X64-NO-BMI2-NO-SHLD-NEXT: movq 56(%rdi), %rdi
-; X64-NO-BMI2-NO-SHLD-NEXT: movl (%rsi), %esi
-; X64-NO-BMI2-NO-SHLD-NEXT: xorps %xmm0, %xmm0
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rbx, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r10, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %esi, %eax
-; X64-NO-BMI2-NO-SHLD-NEXT: andl $63, %eax
-; X64-NO-BMI2-NO-SHLD-NEXT: shrl $3, %esi
-; X64-NO-BMI2-NO-SHLD-NEXT: andl $56, %esi
-; X64-NO-BMI2-NO-SHLD-NEXT: negl %esi
-; X64-NO-BMI2-NO-SHLD-NEXT: movslq %esi, %rbx
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -64(%rsp,%rbx), %r8
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -56(%rsp,%rbx), %rdi
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rdi, %r10
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %r10
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %esi
-; X64-NO-BMI2-NO-SHLD-NEXT: xorb $63, %sil
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r8, %r9
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %r9
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %esi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %r9
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %r10, %r9
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -40(%rsp,%rbx), %r10
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r10, %r14
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %r14
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -48(%rsp,%rbx), %r15
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r15, %r11
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %r11
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %esi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %r11
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %r14, %r11
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %r15
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %rdi
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %esi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %rdi
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %r15, %rdi
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -24(%rsp,%rbx), %r14
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r14, %r12
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %r12
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -32(%rsp,%rbx), %r13
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r13, %r15
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %r15
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %esi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %r15
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %r12, %r15
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %r13
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %r10
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %esi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %r10
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %r13, %r10
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -8(%rsp,%rbx), %r12
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %r12
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -16(%rsp,%rbx), %rbx
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rbx, %r13
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %r13
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %esi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %r13
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %r12, %r13
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %rbx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %r14
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %esi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %r14
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %rbx, %r14
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %r8
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r8, (%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r14, 48(%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r13, 56(%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r10, 32(%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r15, 40(%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rdi, 16(%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r11, 24(%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r9, 8(%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: popq %rbx
-; X64-NO-BMI2-NO-SHLD-NEXT: popq %r12
-; X64-NO-BMI2-NO-SHLD-NEXT: popq %r13
-; X64-NO-BMI2-NO-SHLD-NEXT: popq %r14
-; X64-NO-BMI2-NO-SHLD-NEXT: popq %r15
-; X64-NO-BMI2-NO-SHLD-NEXT: retq
-;
-; X64-NO-BMI2-HAVE-SHLD-LABEL: shl_64bytes:
-; X64-NO-BMI2-HAVE-SHLD: # %bb.0:
-; X64-NO-BMI2-HAVE-SHLD-NEXT: pushq %r14
-; X64-NO-BMI2-HAVE-SHLD-NEXT: pushq %rbx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: pushq %rax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq (%rdi), %rax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq 8(%rdi), %rcx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq 16(%rdi), %r8
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq 24(%rdi), %r9
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq 32(%rdi), %r10
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq 40(%rdi), %r11
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq 48(%rdi), %rbx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq 56(%rdi), %rdi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movl (%rsi), %esi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: xorps %xmm0, %xmm0
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rbx, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r10, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movl %esi, %ecx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: andl $63, %ecx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrl $3, %esi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: andl $56, %esi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: negl %esi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movslq %esi, %r9
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -48(%rsp,%r9), %rax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -40(%rsp,%r9), %r10
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r10, %rsi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shldq %cl, %rax, %rsi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -64(%rsp,%r9), %r8
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -56(%rsp,%r9), %rdi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shldq %cl, %rdi, %rax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -32(%rsp,%r9), %r11
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -24(%rsp,%r9), %rbx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rbx, %r14
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shldq %cl, %r11, %r14
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shldq %cl, %r10, %r11
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -16(%rsp,%r9), %r10
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -8(%rsp,%r9), %r9
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shldq %cl, %r10, %r9
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shldq %cl, %rbx, %r10
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shldq %cl, %r8, %rdi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shlq %cl, %r8
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r10, 48(%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r9, 56(%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r11, 32(%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r14, 40(%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rax, 16(%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rsi, 24(%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r8, (%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rdi, 8(%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: addq $8, %rsp
-; X64-NO-BMI2-HAVE-SHLD-NEXT: popq %rbx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: popq %r14
-; X64-NO-BMI2-HAVE-SHLD-NEXT: retq
-;
-; X64-HAVE-BMI2-NO-SHLD-LABEL: shl_64bytes:
-; X64-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X64-HAVE-BMI2-NO-SHLD-NEXT: pushq %r15
-; X64-HAVE-BMI2-NO-SHLD-NEXT: pushq %r14
-; X64-HAVE-BMI2-NO-SHLD-NEXT: pushq %r12
-; X64-HAVE-BMI2-NO-SHLD-NEXT: pushq %rbx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: pushq %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq (%rdi), %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq 8(%rdi), %rcx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq 16(%rdi), %r8
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq 24(%rdi), %r9
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq 32(%rdi), %r10
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq 40(%rdi), %r11
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq 48(%rdi), %rbx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq 56(%rdi), %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movl (%rsi), %esi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: xorps %xmm0, %xmm0
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rbx, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %r10, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, %eax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: andl $63, %eax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrl $3, %esi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: andl $56, %esi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: negl %esi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movslq %esi, %rsi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq -64(%rsp,%rsi), %r9
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq -56(%rsp,%rsi), %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rcx, %rdi, %r8
-; X64-HAVE-BMI2-NO-SHLD-NEXT: xorb $63, %al
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rcx, %r9, %r10
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrq %r9
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rax, %r9, %r9
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %r8, %r9
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq -40(%rsp,%rsi), %r11
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rcx, %r11, %rbx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq -48(%rsp,%rsi), %r8
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rcx, %r8, %r14
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrq %r8
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rax, %r8, %r8
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %rbx, %r8
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrq %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rax, %rdi, %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %r14, %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq -24(%rsp,%rsi), %rbx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rcx, %rbx, %r14
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq -32(%rsp,%rsi), %r15
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rcx, %r15, %r12
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrq %r15
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rax, %r15, %r15
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %r14, %r15
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrq %r11
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rax, %r11, %r11
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %r12, %r11
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rcx, -8(%rsp,%rsi), %r14
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq -16(%rsp,%rsi), %rsi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rcx, %rsi, %rcx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrq %rsi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rax, %rsi, %rsi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %r14, %rsi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrq %rbx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rax, %rbx, %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %rcx, %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %r10, (%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rax, 48(%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rsi, 56(%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %r11, 32(%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %r15, 40(%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rdi, 16(%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %r8, 24(%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %r9, 8(%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: addq $8, %rsp
-; X64-HAVE-BMI2-NO-SHLD-NEXT: popq %rbx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: popq %r12
-; X64-HAVE-BMI2-NO-SHLD-NEXT: popq %r14
-; X64-HAVE-BMI2-NO-SHLD-NEXT: popq %r15
-; X64-HAVE-BMI2-NO-SHLD-NEXT: retq
-;
-; X64-HAVE-BMI2-HAVE-SHLD-LABEL: shl_64bytes:
-; X64-HAVE-BMI2-HAVE-SHLD: # %bb.0:
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: pushq %r14
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: pushq %rbx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: pushq %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq (%rdi), %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq 8(%rdi), %rcx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq 16(%rdi), %r8
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq 24(%rdi), %r9
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq 32(%rdi), %r10
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq 40(%rdi), %r11
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq 48(%rdi), %rbx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq 56(%rdi), %rdi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movl (%rsi), %esi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: xorps %xmm0, %xmm0
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rbx, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %r10, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movl %esi, %ecx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: andl $63, %ecx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrl $3, %esi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: andl $56, %esi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: negl %esi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movslq %esi, %r8
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -48(%rsp,%r8), %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -40(%rsp,%r8), %r9
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %r9, %rsi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shldq %cl, %rax, %rsi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -64(%rsp,%r8), %r10
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -56(%rsp,%r8), %rdi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shldq %cl, %rdi, %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -32(%rsp,%r8), %r11
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -24(%rsp,%r8), %rbx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rbx, %r14
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shldq %cl, %r11, %r14
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shldq %cl, %r9, %r11
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -16(%rsp,%r8), %r9
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -8(%rsp,%r8), %r8
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shldq %cl, %r9, %r8
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shldq %cl, %rbx, %r9
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shldq %cl, %r10, %rdi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shlxq %rcx, %r10, %rcx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %r9, 48(%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %r8, 56(%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %r11, 32(%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %r14, 40(%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rax, 16(%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rsi, 24(%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rcx, (%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rdi, 8(%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: addq $8, %rsp
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: popq %rbx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: popq %r14
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: retq
-;
-; X86-NO-BMI2-NO-SHLD-LABEL: shl_64bytes:
-; X86-NO-BMI2-NO-SHLD: # %bb.0:
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: subl $204, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl (%eax), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 4(%eax), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 8(%eax), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 12(%eax), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 16(%eax), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 20(%eax), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 24(%eax), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 28(%eax), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 32(%eax), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 36(%eax), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 40(%eax), %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 44(%eax), %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 48(%eax), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 52(%eax), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 56(%eax), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 60(%eax), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl (%ebp), %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: xorps %xmm0, %xmm0
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebp, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebp, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl $3, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: andl $60, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: leal {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: subl %ecx, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl (%eax), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 4(%eax), %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: andl $31, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dl, %ch
-; X86-NO-BMI2-NO-SHLD-NEXT: xorb $31, %ch
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %eax, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 12(%ebp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dl, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 8(%ebp), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %esi, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %eax, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %bl, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 20(%ebp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %bl, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 16(%ebp), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %esi, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %eax, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %bl, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 28(%ebp), %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %bl, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 24(%ebp), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %esi, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %eax, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dl, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 36(%ebp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dl, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 32(%ebp), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %esi, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %eax, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dl, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 44(%ebp), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %esi, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dl, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 40(%ebp), %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %eax, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dl, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %ebx, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 52(%eax), %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebp, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dl, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: negl %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 176(%esp,%eax), %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %edi, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dl, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %ebx, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 60(%edi), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 56(%edi), %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %ch # 1-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %edx, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dl, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %ebx, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, (%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebp, 56(%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, 60(%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %esi, 48(%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, 52(%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, 40(%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, 44(%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, 32(%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, 36(%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, 24(%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, 28(%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, 16(%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, 20(%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, 8(%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, 12(%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, 4(%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: addl $204, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: retl
+; X64-NO-BMI2-LABEL: shl_64bytes:
+; X64-NO-BMI2: # %bb.0:
+; X64-NO-BMI2-NEXT: pushq %r14
+; X64-NO-BMI2-NEXT: pushq %rbx
+; X64-NO-BMI2-NEXT: pushq %rax
+; X64-NO-BMI2-NEXT: movq (%rdi), %rax
+; X64-NO-BMI2-NEXT: movq 8(%rdi), %rcx
+; X64-NO-BMI2-NEXT: movq 16(%rdi), %r8
+; X64-NO-BMI2-NEXT: movq 24(%rdi), %r9
+; X64-NO-BMI2-NEXT: movq 32(%rdi), %r10
+; X64-NO-BMI2-NEXT: movq 40(%rdi), %r11
+; X64-NO-BMI2-NEXT: movq 48(%rdi), %rbx
+; X64-NO-BMI2-NEXT: movq 56(%rdi), %rdi
+; X64-NO-BMI2-NEXT: movl (%rsi), %esi
+; X64-NO-BMI2-NEXT: xorps %xmm0, %xmm0
+; X64-NO-BMI2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movq %rbx, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movq %r10, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movl %esi, %ecx
+; X64-NO-BMI2-NEXT: andl $63, %ecx
+; X64-NO-BMI2-NEXT: shrl $3, %esi
+; X64-NO-BMI2-NEXT: andl $56, %esi
+; X64-NO-BMI2-NEXT: negl %esi
+; X64-NO-BMI2-NEXT: movslq %esi, %r9
+; X64-NO-BMI2-NEXT: movq -48(%rsp,%r9), %rax
+; X64-NO-BMI2-NEXT: movq -40(%rsp,%r9), %r10
+; X64-NO-BMI2-NEXT: movq %r10, %rsi
+; X64-NO-BMI2-NEXT: shldq %cl, %rax, %rsi
+; X64-NO-BMI2-NEXT: movq -64(%rsp,%r9), %r8
+; X64-NO-BMI2-NEXT: movq -56(%rsp,%r9), %rdi
+; X64-NO-BMI2-NEXT: shldq %cl, %rdi, %rax
+; X64-NO-BMI2-NEXT: movq -32(%rsp,%r9), %r11
+; X64-NO-BMI2-NEXT: movq -24(%rsp,%r9), %rbx
+; X64-NO-BMI2-NEXT: movq %rbx, %r14
+; X64-NO-BMI2-NEXT: shldq %cl, %r11, %r14
+; X64-NO-BMI2-NEXT: shldq %cl, %r10, %r11
+; X64-NO-BMI2-NEXT: movq -16(%rsp,%r9), %r10
+; X64-NO-BMI2-NEXT: movq -8(%rsp,%r9), %r9
+; X64-NO-BMI2-NEXT: shldq %cl, %r10, %r9
+; X64-NO-BMI2-NEXT: shldq %cl, %rbx, %r10
+; X64-NO-BMI2-NEXT: shldq %cl, %r8, %rdi
+; X64-NO-BMI2-NEXT: # kill: def $cl killed $cl killed $ecx
+; X64-NO-BMI2-NEXT: shlq %cl, %r8
+; X64-NO-BMI2-NEXT: movq %r10, 48(%rdx)
+; X64-NO-BMI2-NEXT: movq %r9, 56(%rdx)
+; X64-NO-BMI2-NEXT: movq %r11, 32(%rdx)
+; X64-NO-BMI2-NEXT: movq %r14, 40(%rdx)
+; X64-NO-BMI2-NEXT: movq %rax, 16(%rdx)
+; X64-NO-BMI2-NEXT: movq %rsi, 24(%rdx)
+; X64-NO-BMI2-NEXT: movq %r8, (%rdx)
+; X64-NO-BMI2-NEXT: movq %rdi, 8(%rdx)
+; X64-NO-BMI2-NEXT: addq $8, %rsp
+; X64-NO-BMI2-NEXT: popq %rbx
+; X64-NO-BMI2-NEXT: popq %r14
+; X64-NO-BMI2-NEXT: retq
;
-; X86-NO-BMI2-HAVE-SHLD-LABEL: shl_64bytes:
-; X86-NO-BMI2-HAVE-SHLD: # %bb.0:
-; X86-NO-BMI2-HAVE-SHLD-NEXT: pushl %ebp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: pushl %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: pushl %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: subl $188, %esp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl (%ecx), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 4(%ecx), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 8(%ecx), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 12(%ecx), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 16(%ecx), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 20(%ecx), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 24(%ecx), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 28(%ecx), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 32(%ecx), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 36(%ecx), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, (%esp) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 40(%ecx), %ebp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 44(%ecx), %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 48(%ecx), %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 52(%ecx), %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 56(%ecx), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 60(%ecx), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl (%ecx), %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: xorps %xmm0, %xmm0
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ebp, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl (%esp), %eax # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, %ebp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrl $3, %ebp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: andl $60, %ebp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: leal {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: subl %ebp, %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 8(%eax), %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 12(%eax), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: andl $31, %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shldl %cl, %esi, %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 4(%eax), %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shldl %cl, %edi, %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 16(%eax), %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 20(%eax), %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %esi, %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shldl %cl, %edi, %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shldl %cl, %edx, %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 24(%eax), %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 28(%eax), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shldl %cl, %edi, %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shldl %cl, %esi, %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 32(%eax), %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 36(%eax), %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %esi, %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shldl %cl, %edi, %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shldl %cl, %edx, %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 40(%eax), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 44(%eax), %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shldl %cl, %edx, %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edi, (%esp) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shldl %cl, %esi, %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 56(%eax), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 60(%eax), %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shldl %cl, %edx, %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl (%eax), %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 52(%eax), %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shldl %cl, %esi, %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: negl %ebp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 160(%esp,%ebp), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, 56(%ebp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edi, 60(%ebp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shldl %cl, %ebx, %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shll %cl, %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shldl %cl, %eax, %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shldl %cl, %edi, %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, 48(%ebp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %esi, 52(%ebp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, 40(%ebp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl (%esp), %eax # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, 44(%ebp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, 32(%ebp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, 36(%ebp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, 24(%ebp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, 28(%ebp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, 16(%ebp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, 20(%ebp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, 8(%ebp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, 12(%ebp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ebx, (%ebp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, 4(%ebp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: addl $188, %esp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: popl %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: popl %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: popl %ebp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: retl
+; X64-BMI2-LABEL: shl_64bytes:
+; X64-BMI2: # %bb.0:
+; X64-BMI2-NEXT: pushq %r14
+; X64-BMI2-NEXT: pushq %rbx
+; X64-BMI2-NEXT: pushq %rax
+; X64-BMI2-NEXT: movq (%rdi), %rax
+; X64-BMI2-NEXT: movq 8(%rdi), %rcx
+; X64-BMI2-NEXT: movq 16(%rdi), %r8
+; X64-BMI2-NEXT: movq 24(%rdi), %r9
+; X64-BMI2-NEXT: movq 32(%rdi), %r10
+; X64-BMI2-NEXT: movq 40(%rdi), %r11
+; X64-BMI2-NEXT: movq 48(%rdi), %rbx
+; X64-BMI2-NEXT: movq 56(%rdi), %rdi
+; X64-BMI2-NEXT: movl (%rsi), %esi
+; X64-BMI2-NEXT: xorps %xmm0, %xmm0
+; X64-BMI2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movq %rbx, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movq %r10, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movl %esi, %ecx
+; X64-BMI2-NEXT: andl $63, %ecx
+; X64-BMI2-NEXT: shrl $3, %esi
+; X64-BMI2-NEXT: andl $56, %esi
+; X64-BMI2-NEXT: negl %esi
+; X64-BMI2-NEXT: movslq %esi, %r8
+; X64-BMI2-NEXT: movq -48(%rsp,%r8), %rax
+; X64-BMI2-NEXT: movq -40(%rsp,%r8), %r9
+; X64-BMI2-NEXT: movq %r9, %rsi
+; X64-BMI2-NEXT: shldq %cl, %rax, %rsi
+; X64-BMI2-NEXT: movq -64(%rsp,%r8), %r10
+; X64-BMI2-NEXT: movq -56(%rsp,%r8), %rdi
+; X64-BMI2-NEXT: shldq %cl, %rdi, %rax
+; X64-BMI2-NEXT: movq -32(%rsp,%r8), %r11
+; X64-BMI2-NEXT: movq -24(%rsp,%r8), %rbx
+; X64-BMI2-NEXT: movq %rbx, %r14
+; X64-BMI2-NEXT: shldq %cl, %r11, %r14
+; X64-BMI2-NEXT: shldq %cl, %r9, %r11
+; X64-BMI2-NEXT: movq -16(%rsp,%r8), %r9
+; X64-BMI2-NEXT: movq -8(%rsp,%r8), %r8
+; X64-BMI2-NEXT: shldq %cl, %r9, %r8
+; X64-BMI2-NEXT: shldq %cl, %rbx, %r9
+; X64-BMI2-NEXT: shldq %cl, %r10, %rdi
+; X64-BMI2-NEXT: shlxq %rcx, %r10, %rcx
+; X64-BMI2-NEXT: movq %r9, 48(%rdx)
+; X64-BMI2-NEXT: movq %r8, 56(%rdx)
+; X64-BMI2-NEXT: movq %r11, 32(%rdx)
+; X64-BMI2-NEXT: movq %r14, 40(%rdx)
+; X64-BMI2-NEXT: movq %rax, 16(%rdx)
+; X64-BMI2-NEXT: movq %rsi, 24(%rdx)
+; X64-BMI2-NEXT: movq %rcx, (%rdx)
+; X64-BMI2-NEXT: movq %rdi, 8(%rdx)
+; X64-BMI2-NEXT: addq $8, %rsp
+; X64-BMI2-NEXT: popq %rbx
+; X64-BMI2-NEXT: popq %r14
+; X64-BMI2-NEXT: retq
;
-; X86-HAVE-BMI2-NO-SHLD-LABEL: shl_64bytes:
-; X86-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: subl $204, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl (%eax), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 4(%eax), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 8(%eax), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 12(%eax), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 16(%eax), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 20(%eax), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 24(%eax), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 28(%eax), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 32(%eax), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 36(%eax), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 40(%eax), %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 44(%eax), %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 48(%eax), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 52(%eax), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 56(%eax), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 60(%eax), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl (%ebp), %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: xorps %xmm0, %xmm0
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ebp, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ebp, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: andl $31, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ebx, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrl $3, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: andl $60, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal {{[0-9]+}}(%esp), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: subl %eax, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl (%edx), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 4(%edx), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: xorb $31, %bl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ebx, %esi, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %eax, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %esi, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 8(%edx), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ebx, %esi, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 12(%edx), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %esi, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %ebp, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrl %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ebx, %ecx, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %eax, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 16(%edx), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrl %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ebx, %ecx, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 20(%edx), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %edi, %ecx, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %ebp, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %edi, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ebx, %esi, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %ebp, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 24(%edx), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ebx, %esi, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 28(%edx), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %edi, %esi, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %ebp, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %edi, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrl %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ebx, %ecx, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %ebp, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 32(%edx), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrl %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ebx, %ecx, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 36(%edx), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %edi, %ecx, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %ebp, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %edi, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edi, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ebx, %esi, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %ebp, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 40(%edx), %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edi, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ebx, %esi, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 44(%edx), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %eax, %esi, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %ebp, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %eax, %edi, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrl %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ebx, %eax, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %edi, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 48(%edx), %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ebp, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ebx, %edi, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 52(%edx), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %esi, %ecx, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %edi, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %esi, %ebp, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrl %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ebx, %eax, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %edi, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 56(%edx), %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrl %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ebx, %ecx, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ebp, %edi, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %ecx, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ebx, %edi, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ebp, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: negl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ebp, 188(%esp,%ebx), %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %ecx, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edi, (%edx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, 56(%edx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ebx, 60(%edx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, 48(%edx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, 52(%edx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, 40(%edx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, 44(%edx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, 32(%edx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, 36(%edx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, 24(%edx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, 28(%edx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, 16(%edx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, 20(%edx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, 8(%edx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, 12(%edx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, 4(%edx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl $204, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: retl
+; X86-NO-BMI2-LABEL: shl_64bytes:
+; X86-NO-BMI2: # %bb.0:
+; X86-NO-BMI2-NEXT: pushl %ebp
+; X86-NO-BMI2-NEXT: pushl %ebx
+; X86-NO-BMI2-NEXT: pushl %edi
+; X86-NO-BMI2-NEXT: pushl %esi
+; X86-NO-BMI2-NEXT: subl $188, %esp
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-BMI2-NEXT: movl (%ecx), %eax
+; X86-NO-BMI2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 4(%ecx), %eax
+; X86-NO-BMI2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 8(%ecx), %eax
+; X86-NO-BMI2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 12(%ecx), %eax
+; X86-NO-BMI2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 16(%ecx), %eax
+; X86-NO-BMI2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 20(%ecx), %eax
+; X86-NO-BMI2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 24(%ecx), %eax
+; X86-NO-BMI2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 28(%ecx), %eax
+; X86-NO-BMI2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 32(%ecx), %eax
+; X86-NO-BMI2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 36(%ecx), %eax
+; X86-NO-BMI2-NEXT: movl %eax, (%esp) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 40(%ecx), %ebp
+; X86-NO-BMI2-NEXT: movl 44(%ecx), %ebx
+; X86-NO-BMI2-NEXT: movl 48(%ecx), %edi
+; X86-NO-BMI2-NEXT: movl 52(%ecx), %esi
+; X86-NO-BMI2-NEXT: movl 56(%ecx), %edx
+; X86-NO-BMI2-NEXT: movl 60(%ecx), %eax
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-BMI2-NEXT: movl (%ecx), %ecx
+; X86-NO-BMI2-NEXT: xorps %xmm0, %xmm0
+; X86-NO-BMI2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %ebp, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl (%esp), %eax # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %ecx, %ebp
+; X86-NO-BMI2-NEXT: shrl $3, %ebp
+; X86-NO-BMI2-NEXT: andl $60, %ebp
+; X86-NO-BMI2-NEXT: leal {{[0-9]+}}(%esp), %eax
+; X86-NO-BMI2-NEXT: subl %ebp, %eax
+; X86-NO-BMI2-NEXT: movl 8(%eax), %esi
+; X86-NO-BMI2-NEXT: movl 12(%eax), %edx
+; X86-NO-BMI2-NEXT: andl $31, %ecx
+; X86-NO-BMI2-NEXT: movl %edx, %edi
+; X86-NO-BMI2-NEXT: shldl %cl, %esi, %edi
+; X86-NO-BMI2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 4(%eax), %edi
+; X86-NO-BMI2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: shldl %cl, %edi, %esi
+; X86-NO-BMI2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 16(%eax), %edi
+; X86-NO-BMI2-NEXT: movl 20(%eax), %esi
+; X86-NO-BMI2-NEXT: movl %esi, %ebx
+; X86-NO-BMI2-NEXT: shldl %cl, %edi, %ebx
+; X86-NO-BMI2-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: shldl %cl, %edx, %edi
+; X86-NO-BMI2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 24(%eax), %edi
+; X86-NO-BMI2-NEXT: movl 28(%eax), %edx
+; X86-NO-BMI2-NEXT: movl %edx, %ebx
+; X86-NO-BMI2-NEXT: shldl %cl, %edi, %ebx
+; X86-NO-BMI2-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: shldl %cl, %esi, %edi
+; X86-NO-BMI2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 32(%eax), %edi
+; X86-NO-BMI2-NEXT: movl 36(%eax), %esi
+; X86-NO-BMI2-NEXT: movl %esi, %ebx
+; X86-NO-BMI2-NEXT: shldl %cl, %edi, %ebx
+; X86-NO-BMI2-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: shldl %cl, %edx, %edi
+; X86-NO-BMI2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 40(%eax), %edx
+; X86-NO-BMI2-NEXT: movl 44(%eax), %edi
+; X86-NO-BMI2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: shldl %cl, %edx, %edi
+; X86-NO-BMI2-NEXT: movl %edi, (%esp) # 4-byte Spill
+; X86-NO-BMI2-NEXT: shldl %cl, %esi, %edx
+; X86-NO-BMI2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 56(%eax), %edx
+; X86-NO-BMI2-NEXT: movl 60(%eax), %edi
+; X86-NO-BMI2-NEXT: shldl %cl, %edx, %edi
+; X86-NO-BMI2-NEXT: movl (%eax), %ebx
+; X86-NO-BMI2-NEXT: movl 52(%eax), %esi
+; X86-NO-BMI2-NEXT: shldl %cl, %esi, %edx
+; X86-NO-BMI2-NEXT: negl %ebp
+; X86-NO-BMI2-NEXT: movl 160(%esp,%ebp), %eax
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NO-BMI2-NEXT: movl %edx, 56(%ebp)
+; X86-NO-BMI2-NEXT: movl %edi, 60(%ebp)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-BMI2-NEXT: shldl %cl, %ebx, %edx
+; X86-NO-BMI2-NEXT: shll %cl, %ebx
+; X86-NO-BMI2-NEXT: shldl %cl, %eax, %esi
+; X86-NO-BMI2-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NO-BMI2-NEXT: shldl %cl, %edi, %eax
+; X86-NO-BMI2-NEXT: movl %eax, 48(%ebp)
+; X86-NO-BMI2-NEXT: movl %esi, 52(%ebp)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %eax, 40(%ebp)
+; X86-NO-BMI2-NEXT: movl (%esp), %eax # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %eax, 44(%ebp)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %eax, 32(%ebp)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %eax, 36(%ebp)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %eax, 24(%ebp)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %eax, 28(%ebp)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %eax, 16(%ebp)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %eax, 20(%ebp)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %eax, 8(%ebp)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %eax, 12(%ebp)
+; X86-NO-BMI2-NEXT: movl %ebx, (%ebp)
+; X86-NO-BMI2-NEXT: movl %edx, 4(%ebp)
+; X86-NO-BMI2-NEXT: addl $188, %esp
+; X86-NO-BMI2-NEXT: popl %esi
+; X86-NO-BMI2-NEXT: popl %edi
+; X86-NO-BMI2-NEXT: popl %ebx
+; X86-NO-BMI2-NEXT: popl %ebp
+; X86-NO-BMI2-NEXT: retl
;
-; X86-HAVE-BMI2-HAVE-SHLD-LABEL: shl_64bytes:
-; X86-HAVE-BMI2-HAVE-SHLD: # %bb.0:
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: pushl %ebp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: pushl %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: pushl %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: subl $204, %esp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl (%ebx), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 4(%ebx), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 8(%ebx), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 12(%ebx), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 16(%ebx), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 20(%ebx), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 24(%ebx), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 28(%ebx), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 32(%ebx), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 36(%ebx), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 40(%ebx), %ebp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 44(%ebx), %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 48(%ebx), %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 52(%ebx), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 56(%ebx), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 60(%ebx), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl (%ebx), %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: xorps %xmm0, %xmm0
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ebp, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ebx, %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: andl $31, %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrl $3, %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: andl $60, %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: leal {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: subl %ebx, %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 4(%eax), %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 8(%eax), %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 12(%eax), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, %ebp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shldl %cl, %edi, %ebp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shldl %cl, %esi, %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 16(%eax), %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 20(%eax), %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %esi, %ebp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shldl %cl, %edi, %ebp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shldl %cl, %edx, %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 24(%eax), %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 28(%eax), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, %ebp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shldl %cl, %edi, %ebp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shldl %cl, %esi, %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 32(%eax), %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 36(%eax), %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %esi, %ebp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shldl %cl, %edi, %ebp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shldl %cl, %edx, %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 40(%eax), %ebp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 44(%eax), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shldl %cl, %ebp, %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shldl %cl, %esi, %ebp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 56(%eax), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 60(%eax), %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shldl %cl, %edx, %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl (%eax), %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 52(%eax), %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shldl %cl, %esi, %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: negl %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 176(%esp,%ebx), %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, 56(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edi, 60(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shlxl %ecx, %edx, %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shldl %cl, %edx, %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shldl %cl, %ebx, %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shldl %cl, %edx, %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ebx, 48(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %esi, 52(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ebp, 40(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, 44(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, 32(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, 36(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, 24(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, 28(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, 16(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, 20(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, 8(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, 12(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, (%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edi, 4(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: addl $204, %esp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: popl %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: popl %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: popl %ebp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: retl
+; X86-BMI2-LABEL: shl_64bytes:
+; X86-BMI2: # %bb.0:
+; X86-BMI2-NEXT: pushl %ebp
+; X86-BMI2-NEXT: pushl %ebx
+; X86-BMI2-NEXT: pushl %edi
+; X86-BMI2-NEXT: pushl %esi
+; X86-BMI2-NEXT: subl $204, %esp
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X86-BMI2-NEXT: movl (%ebx), %eax
+; X86-BMI2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 4(%ebx), %eax
+; X86-BMI2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 8(%ebx), %eax
+; X86-BMI2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 12(%ebx), %eax
+; X86-BMI2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 16(%ebx), %eax
+; X86-BMI2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 20(%ebx), %eax
+; X86-BMI2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 24(%ebx), %eax
+; X86-BMI2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 28(%ebx), %eax
+; X86-BMI2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 32(%ebx), %eax
+; X86-BMI2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 36(%ebx), %eax
+; X86-BMI2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 40(%ebx), %ebp
+; X86-BMI2-NEXT: movl 44(%ebx), %edi
+; X86-BMI2-NEXT: movl 48(%ebx), %esi
+; X86-BMI2-NEXT: movl 52(%ebx), %edx
+; X86-BMI2-NEXT: movl 56(%ebx), %ecx
+; X86-BMI2-NEXT: movl 60(%ebx), %eax
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X86-BMI2-NEXT: movl (%ebx), %ebx
+; X86-BMI2-NEXT: xorps %xmm0, %xmm0
+; X86-BMI2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %ebp, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-BMI2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-BMI2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-BMI2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-BMI2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-BMI2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-BMI2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-BMI2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-BMI2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-BMI2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-BMI2-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %ebx, %ecx
+; X86-BMI2-NEXT: andl $31, %ecx
+; X86-BMI2-NEXT: shrl $3, %ebx
+; X86-BMI2-NEXT: andl $60, %ebx
+; X86-BMI2-NEXT: leal {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT: subl %ebx, %eax
+; X86-BMI2-NEXT: movl 4(%eax), %esi
+; X86-BMI2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 8(%eax), %edi
+; X86-BMI2-NEXT: movl 12(%eax), %edx
+; X86-BMI2-NEXT: movl %edx, %ebp
+; X86-BMI2-NEXT: shldl %cl, %edi, %ebp
+; X86-BMI2-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: shldl %cl, %esi, %edi
+; X86-BMI2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 16(%eax), %edi
+; X86-BMI2-NEXT: movl 20(%eax), %esi
+; X86-BMI2-NEXT: movl %esi, %ebp
+; X86-BMI2-NEXT: shldl %cl, %edi, %ebp
+; X86-BMI2-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: shldl %cl, %edx, %edi
+; X86-BMI2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 24(%eax), %edi
+; X86-BMI2-NEXT: movl 28(%eax), %edx
+; X86-BMI2-NEXT: movl %edx, %ebp
+; X86-BMI2-NEXT: shldl %cl, %edi, %ebp
+; X86-BMI2-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: shldl %cl, %esi, %edi
+; X86-BMI2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 32(%eax), %edi
+; X86-BMI2-NEXT: movl 36(%eax), %esi
+; X86-BMI2-NEXT: movl %esi, %ebp
+; X86-BMI2-NEXT: shldl %cl, %edi, %ebp
+; X86-BMI2-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: shldl %cl, %edx, %edi
+; X86-BMI2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 40(%eax), %ebp
+; X86-BMI2-NEXT: movl 44(%eax), %edx
+; X86-BMI2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: shldl %cl, %ebp, %edx
+; X86-BMI2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: shldl %cl, %esi, %ebp
+; X86-BMI2-NEXT: movl 56(%eax), %edx
+; X86-BMI2-NEXT: movl 60(%eax), %edi
+; X86-BMI2-NEXT: shldl %cl, %edx, %edi
+; X86-BMI2-NEXT: movl (%eax), %esi
+; X86-BMI2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 52(%eax), %esi
+; X86-BMI2-NEXT: shldl %cl, %esi, %edx
+; X86-BMI2-NEXT: negl %ebx
+; X86-BMI2-NEXT: movl 176(%esp,%ebx), %ebx
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT: movl %edx, 56(%eax)
+; X86-BMI2-NEXT: movl %edi, 60(%eax)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-BMI2-NEXT: shlxl %ecx, %edx, %edi
+; X86-BMI2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-BMI2-NEXT: shldl %cl, %edx, %edi
+; X86-BMI2-NEXT: shldl %cl, %ebx, %esi
+; X86-BMI2-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-BMI2-NEXT: shldl %cl, %edx, %ebx
+; X86-BMI2-NEXT: movl %ebx, 48(%eax)
+; X86-BMI2-NEXT: movl %esi, 52(%eax)
+; X86-BMI2-NEXT: movl %ebp, 40(%eax)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-BMI2-NEXT: movl %ecx, 44(%eax)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-BMI2-NEXT: movl %ecx, 32(%eax)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-BMI2-NEXT: movl %ecx, 36(%eax)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-BMI2-NEXT: movl %ecx, 24(%eax)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-BMI2-NEXT: movl %ecx, 28(%eax)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-BMI2-NEXT: movl %ecx, 16(%eax)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-BMI2-NEXT: movl %ecx, 20(%eax)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-BMI2-NEXT: movl %ecx, 8(%eax)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-BMI2-NEXT: movl %ecx, 12(%eax)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-BMI2-NEXT: movl %ecx, (%eax)
+; X86-BMI2-NEXT: movl %edi, 4(%eax)
+; X86-BMI2-NEXT: addl $204, %esp
+; X86-BMI2-NEXT: popl %esi
+; X86-BMI2-NEXT: popl %edi
+; X86-BMI2-NEXT: popl %ebx
+; X86-BMI2-NEXT: popl %ebp
+; X86-BMI2-NEXT: retl
%src = load i512, ptr %src.ptr, align 1
%bitOff = load i512, ptr %bitOff.ptr, align 1
%res = shl i512 %src, %bitOff
@@ -5350,1184 +2328,480 @@ define void @shl_64bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
ret void
}
define void @ashr_64bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
-; X64-NO-BMI2-NO-SHLD-LABEL: ashr_64bytes:
-; X64-NO-BMI2-NO-SHLD: # %bb.0:
-; X64-NO-BMI2-NO-SHLD-NEXT: pushq %rbp
-; X64-NO-BMI2-NO-SHLD-NEXT: pushq %r15
-; X64-NO-BMI2-NO-SHLD-NEXT: pushq %r14
-; X64-NO-BMI2-NO-SHLD-NEXT: pushq %r13
-; X64-NO-BMI2-NO-SHLD-NEXT: pushq %r12
-; X64-NO-BMI2-NO-SHLD-NEXT: pushq %rbx
-; X64-NO-BMI2-NO-SHLD-NEXT: pushq %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: movq (%rdi), %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: movq 8(%rdi), %rcx
-; X64-NO-BMI2-NO-SHLD-NEXT: movq 16(%rdi), %r9
-; X64-NO-BMI2-NO-SHLD-NEXT: movq 24(%rdi), %r10
-; X64-NO-BMI2-NO-SHLD-NEXT: movq 32(%rdi), %r11
-; X64-NO-BMI2-NO-SHLD-NEXT: movq 40(%rdi), %rbx
-; X64-NO-BMI2-NO-SHLD-NEXT: movq 48(%rdi), %r14
-; X64-NO-BMI2-NO-SHLD-NEXT: movq 56(%rdi), %rdi
-; X64-NO-BMI2-NO-SHLD-NEXT: movl (%rsi), %r8d
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r14, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rbx, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r10, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: sarq $63, %rdi
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %r8d, %eax
-; X64-NO-BMI2-NO-SHLD-NEXT: andl $63, %eax
-; X64-NO-BMI2-NO-SHLD-NEXT: shrl $3, %r8d
-; X64-NO-BMI2-NO-SHLD-NEXT: andl $56, %r8d
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -128(%rsp,%r8), %r11
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -120(%rsp,%r8), %r9
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r9, %rsi
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %rsi
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %edi
-; X64-NO-BMI2-NO-SHLD-NEXT: notl %edi
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -112(%rsp,%r8), %r14
-; X64-NO-BMI2-NO-SHLD-NEXT: leaq (%r14,%r14), %r10
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %edi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %r10
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %rsi, %r10
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %r11
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %esi
-; X64-NO-BMI2-NO-SHLD-NEXT: xorb $63, %sil
-; X64-NO-BMI2-NO-SHLD-NEXT: addq %r9, %r9
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %esi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %r9
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %r11, %r9
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -104(%rsp,%r8), %r11
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r11, %r12
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %r12
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -96(%rsp,%r8), %r15
-; X64-NO-BMI2-NO-SHLD-NEXT: leaq (%r15,%r15), %rbx
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %edi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %rbx
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %r12, %rbx
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %r14
-; X64-NO-BMI2-NO-SHLD-NEXT: addq %r11, %r11
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %esi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %r11
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %r14, %r11
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -88(%rsp,%r8), %r14
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r14, %r13
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %r13
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -80(%rsp,%r8), %rbp
-; X64-NO-BMI2-NO-SHLD-NEXT: leaq (%rbp,%rbp), %r12
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %edi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %r12
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %r13, %r12
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %r15
-; X64-NO-BMI2-NO-SHLD-NEXT: addq %r14, %r14
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %esi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %r14
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %r15, %r14
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %rbp
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -72(%rsp,%r8), %rdi
-; X64-NO-BMI2-NO-SHLD-NEXT: leaq (%rdi,%rdi), %r8
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %esi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %r8
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %rbp, %r8
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: sarq %cl, %rdi
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rdi, 56(%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r8, 48(%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r14, 32(%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r12, 40(%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r11, 16(%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rbx, 24(%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r9, (%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r10, 8(%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: addq $8, %rsp
-; X64-NO-BMI2-NO-SHLD-NEXT: popq %rbx
-; X64-NO-BMI2-NO-SHLD-NEXT: popq %r12
-; X64-NO-BMI2-NO-SHLD-NEXT: popq %r13
-; X64-NO-BMI2-NO-SHLD-NEXT: popq %r14
-; X64-NO-BMI2-NO-SHLD-NEXT: popq %r15
-; X64-NO-BMI2-NO-SHLD-NEXT: popq %rbp
-; X64-NO-BMI2-NO-SHLD-NEXT: retq
-;
-; X64-NO-BMI2-HAVE-SHLD-LABEL: ashr_64bytes:
-; X64-NO-BMI2-HAVE-SHLD: # %bb.0:
-; X64-NO-BMI2-HAVE-SHLD-NEXT: pushq %r15
-; X64-NO-BMI2-HAVE-SHLD-NEXT: pushq %r14
-; X64-NO-BMI2-HAVE-SHLD-NEXT: pushq %rbx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq (%rdi), %rcx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq 8(%rdi), %r8
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq 16(%rdi), %r9
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq 24(%rdi), %r10
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq 32(%rdi), %r11
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq 40(%rdi), %rbx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq 48(%rdi), %r14
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq 56(%rdi), %rdi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movl (%rsi), %eax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r14, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rbx, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r10, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: sarq $63, %rdi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: andl $63, %ecx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrl $3, %eax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: andl $56, %eax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -112(%rsp,%rax), %rdi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -128(%rsp,%rax), %rsi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -120(%rsp,%rax), %r9
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r9, %r8
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %rdi, %r8
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -96(%rsp,%rax), %r10
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -104(%rsp,%rax), %r11
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r11, %rbx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %r10, %rbx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %r11, %rdi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -80(%rsp,%rax), %r11
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -88(%rsp,%rax), %r14
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r14, %r15
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %r11, %r15
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %r14, %r10
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -72(%rsp,%rax), %rax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %rax, %r11
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %r9, %rsi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: sarq %cl, %rax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r11, 48(%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rax, 56(%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r10, 32(%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r15, 40(%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rdi, 16(%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rbx, 24(%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rsi, (%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r8, 8(%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: popq %rbx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: popq %r14
-; X64-NO-BMI2-HAVE-SHLD-NEXT: popq %r15
-; X64-NO-BMI2-HAVE-SHLD-NEXT: retq
-;
-; X64-HAVE-BMI2-NO-SHLD-LABEL: ashr_64bytes:
-; X64-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X64-HAVE-BMI2-NO-SHLD-NEXT: pushq %r15
-; X64-HAVE-BMI2-NO-SHLD-NEXT: pushq %r14
-; X64-HAVE-BMI2-NO-SHLD-NEXT: pushq %r13
-; X64-HAVE-BMI2-NO-SHLD-NEXT: pushq %r12
-; X64-HAVE-BMI2-NO-SHLD-NEXT: pushq %rbx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq (%rdi), %rcx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq 8(%rdi), %r8
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq 16(%rdi), %r9
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq 24(%rdi), %r10
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq 32(%rdi), %r11
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq 40(%rdi), %rbx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq 48(%rdi), %r14
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq 56(%rdi), %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movl (%rsi), %eax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %r14, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rbx, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %r10, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: sarq $63, %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, %esi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: andl $63, %esi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, %ecx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrl $3, %eax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: andl $56, %eax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq -120(%rsp,%rax), %r8
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq -112(%rsp,%rax), %r11
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rcx, %r8, %r9
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, %r10d
-; X64-HAVE-BMI2-NO-SHLD-NEXT: notl %r10d
-; X64-HAVE-BMI2-NO-SHLD-NEXT: leaq (%r11,%r11), %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %r10, %rdi, %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %r9, %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rcx, -128(%rsp,%rax), %r9
-; X64-HAVE-BMI2-NO-SHLD-NEXT: xorb $63, %sil
-; X64-HAVE-BMI2-NO-SHLD-NEXT: addq %r8, %r8
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rsi, %r8, %r8
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %r9, %r8
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq -104(%rsp,%rax), %rbx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rcx, %rbx, %r14
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq -96(%rsp,%rax), %r15
-; X64-HAVE-BMI2-NO-SHLD-NEXT: leaq (%r15,%r15), %r9
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %r10, %r9, %r9
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %r14, %r9
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rcx, %r11, %r11
-; X64-HAVE-BMI2-NO-SHLD-NEXT: addq %rbx, %rbx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rsi, %rbx, %rbx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %r11, %rbx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq -88(%rsp,%rax), %r11
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rcx, %r11, %r14
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq -80(%rsp,%rax), %r12
-; X64-HAVE-BMI2-NO-SHLD-NEXT: leaq (%r12,%r12), %r13
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %r10, %r13, %r10
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %r14, %r10
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rcx, %r15, %r14
-; X64-HAVE-BMI2-NO-SHLD-NEXT: addq %r11, %r11
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rsi, %r11, %r11
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %r14, %r11
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rcx, %r12, %r14
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq -72(%rsp,%rax), %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: leaq (%rax,%rax), %r15
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rsi, %r15, %rsi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %r14, %rsi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: sarxq %rcx, %rax, %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rax, 56(%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rsi, 48(%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %r11, 32(%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %r10, 40(%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rbx, 16(%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %r9, 24(%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %r8, (%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rdi, 8(%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: popq %rbx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: popq %r12
-; X64-HAVE-BMI2-NO-SHLD-NEXT: popq %r13
-; X64-HAVE-BMI2-NO-SHLD-NEXT: popq %r14
-; X64-HAVE-BMI2-NO-SHLD-NEXT: popq %r15
-; X64-HAVE-BMI2-NO-SHLD-NEXT: retq
-;
-; X64-HAVE-BMI2-HAVE-SHLD-LABEL: ashr_64bytes:
-; X64-HAVE-BMI2-HAVE-SHLD: # %bb.0:
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: pushq %r15
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: pushq %r14
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: pushq %rbx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq (%rdi), %rcx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq 8(%rdi), %r8
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq 16(%rdi), %r9
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq 24(%rdi), %r10
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq 32(%rdi), %r11
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq 40(%rdi), %rbx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq 48(%rdi), %r14
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq 56(%rdi), %rdi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movl (%rsi), %eax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %r14, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rbx, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %r10, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: sarq $63, %rdi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, %ecx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: andl $63, %ecx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrl $3, %eax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: andl $56, %eax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -112(%rsp,%rax), %rdi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -128(%rsp,%rax), %rsi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -120(%rsp,%rax), %r9
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %r9, %r8
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %rdi, %r8
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -96(%rsp,%rax), %r10
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -104(%rsp,%rax), %r11
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %r11, %rbx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %r10, %rbx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %r11, %rdi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -80(%rsp,%rax), %r11
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -88(%rsp,%rax), %r14
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %r14, %r15
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %r11, %r15
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %r14, %r10
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -72(%rsp,%rax), %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %rax, %r11
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %r9, %rsi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: sarxq %rcx, %rax, %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %r11, 48(%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rax, 56(%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %r10, 32(%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %r15, 40(%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rdi, 16(%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rbx, 24(%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rsi, (%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %r8, 8(%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: popq %rbx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: popq %r14
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: popq %r15
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: retq
-;
-; X86-NO-BMI2-NO-SHLD-LABEL: ashr_64bytes:
-; X86-NO-BMI2-NO-SHLD: # %bb.0:
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: subl $204, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl (%eax), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 4(%eax), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, (%esp) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 8(%eax), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 12(%eax), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 16(%eax), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 20(%eax), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 24(%eax), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 28(%eax), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 32(%eax), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 36(%eax), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 40(%eax), %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 44(%eax), %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 48(%eax), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 52(%eax), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 56(%eax), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 60(%eax), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl (%ebp), %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl (%esp), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: sarl $31, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebp, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: andl $31, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl $3, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: andl $60, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 68(%esp,%ebp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: notl %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 72(%esp,%ebp), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%edx,%edx), %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 64(%esp,%ebp), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: xorb $31, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %cl, (%esp) # 1-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %eax, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 76(%esp,%ebp), %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 80(%esp,%ebp), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%esi,%esi), %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %edi, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %eax, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movzbl (%esp), %ecx # 1-byte Folded Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %edx, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 84(%esp,%ebp), %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 88(%esp,%ebp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %eax, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %edi, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %ebx, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movzbl (%esp), %ecx # 1-byte Folded Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 92(%esp,%ebp), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 96(%esp,%ebp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%eax,%eax), %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %edx, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movzbl (%esp), %ecx # 1-byte Folded Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %eax, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 100(%esp,%ebp), %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 104(%esp,%ebp), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%edx,%edx), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %edi, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movzbl (%esp), %ecx # 1-byte Folded Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 108(%esp,%ebp), %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 112(%esp,%ebp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%ecx,%ecx), %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %ebx, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movzbl (%esp), %ecx # 1-byte Folded Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %edx, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 116(%esp,%ebp), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %esi, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 120(%esp,%ebp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%ecx,%ecx), %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %eax, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %esi, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movb (%esp), %ch # 1-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %edx, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %al, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 124(%esp,%ebp), %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%ebp,%ebp), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %eax, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: sarl %cl, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebp, 60(%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, 56(%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %esi, 48(%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, 52(%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, 40(%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, 44(%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, 32(%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, 36(%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, 24(%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, 28(%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, 16(%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, 20(%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, 8(%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, 12(%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, (%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, 4(%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: addl $204, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: retl
+; X64-NO-BMI2-LABEL: ashr_64bytes:
+; X64-NO-BMI2: # %bb.0:
+; X64-NO-BMI2-NEXT: pushq %r15
+; X64-NO-BMI2-NEXT: pushq %r14
+; X64-NO-BMI2-NEXT: pushq %rbx
+; X64-NO-BMI2-NEXT: movq (%rdi), %rcx
+; X64-NO-BMI2-NEXT: movq 8(%rdi), %r8
+; X64-NO-BMI2-NEXT: movq 16(%rdi), %r9
+; X64-NO-BMI2-NEXT: movq 24(%rdi), %r10
+; X64-NO-BMI2-NEXT: movq 32(%rdi), %r11
+; X64-NO-BMI2-NEXT: movq 40(%rdi), %rbx
+; X64-NO-BMI2-NEXT: movq 48(%rdi), %r14
+; X64-NO-BMI2-NEXT: movq 56(%rdi), %rdi
+; X64-NO-BMI2-NEXT: movl (%rsi), %eax
+; X64-NO-BMI2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movq %r14, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movq %rbx, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movq %r10, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: sarq $63, %rdi
+; X64-NO-BMI2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movl %eax, %ecx
+; X64-NO-BMI2-NEXT: andl $63, %ecx
+; X64-NO-BMI2-NEXT: shrl $3, %eax
+; X64-NO-BMI2-NEXT: andl $56, %eax
+; X64-NO-BMI2-NEXT: movq -112(%rsp,%rax), %rdi
+; X64-NO-BMI2-NEXT: movq -128(%rsp,%rax), %rsi
+; X64-NO-BMI2-NEXT: movq -120(%rsp,%rax), %r9
+; X64-NO-BMI2-NEXT: movq %r9, %r8
+; X64-NO-BMI2-NEXT: shrdq %cl, %rdi, %r8
+; X64-NO-BMI2-NEXT: movq -96(%rsp,%rax), %r10
+; X64-NO-BMI2-NEXT: movq -104(%rsp,%rax), %r11
+; X64-NO-BMI2-NEXT: movq %r11, %rbx
+; X64-NO-BMI2-NEXT: shrdq %cl, %r10, %rbx
+; X64-NO-BMI2-NEXT: shrdq %cl, %r11, %rdi
+; X64-NO-BMI2-NEXT: movq -80(%rsp,%rax), %r11
+; X64-NO-BMI2-NEXT: movq -88(%rsp,%rax), %r14
+; X64-NO-BMI2-NEXT: movq %r14, %r15
+; X64-NO-BMI2-NEXT: shrdq %cl, %r11, %r15
+; X64-NO-BMI2-NEXT: shrdq %cl, %r14, %r10
+; X64-NO-BMI2-NEXT: movq -72(%rsp,%rax), %rax
+; X64-NO-BMI2-NEXT: shrdq %cl, %rax, %r11
+; X64-NO-BMI2-NEXT: shrdq %cl, %r9, %rsi
+; X64-NO-BMI2-NEXT: # kill: def $cl killed $cl killed $ecx
+; X64-NO-BMI2-NEXT: sarq %cl, %rax
+; X64-NO-BMI2-NEXT: movq %r11, 48(%rdx)
+; X64-NO-BMI2-NEXT: movq %rax, 56(%rdx)
+; X64-NO-BMI2-NEXT: movq %r10, 32(%rdx)
+; X64-NO-BMI2-NEXT: movq %r15, 40(%rdx)
+; X64-NO-BMI2-NEXT: movq %rdi, 16(%rdx)
+; X64-NO-BMI2-NEXT: movq %rbx, 24(%rdx)
+; X64-NO-BMI2-NEXT: movq %rsi, (%rdx)
+; X64-NO-BMI2-NEXT: movq %r8, 8(%rdx)
+; X64-NO-BMI2-NEXT: popq %rbx
+; X64-NO-BMI2-NEXT: popq %r14
+; X64-NO-BMI2-NEXT: popq %r15
+; X64-NO-BMI2-NEXT: retq
;
-; X86-NO-BMI2-HAVE-SHLD-LABEL: ashr_64bytes:
-; X86-NO-BMI2-HAVE-SHLD: # %bb.0:
-; X86-NO-BMI2-HAVE-SHLD-NEXT: pushl %ebp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: pushl %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: pushl %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: subl $188, %esp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl (%eax), %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 4(%eax), %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 8(%eax), %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 12(%eax), %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 16(%eax), %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 20(%eax), %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 24(%eax), %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 28(%eax), %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 32(%eax), %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 36(%eax), %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, (%esp) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 40(%eax), %ebp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 44(%eax), %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 48(%eax), %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 52(%eax), %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 56(%eax), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 60(%eax), %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl (%eax), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ebp, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl (%esp), %edx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: sarl $31, %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: andl $31, %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrl $3, %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: andl $60, %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 56(%esp,%eax), %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 52(%esp,%eax), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %esi, %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 64(%esp,%eax), %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 60(%esp,%eax), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %ebx, %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edx, %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 72(%esp,%eax), %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 68(%esp,%eax), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edi, %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edx, %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 80(%esp,%eax), %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 76(%esp,%eax), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %ebx, %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edx, %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 88(%esp,%eax), %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 84(%esp,%eax), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edi, %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edi, %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edx, %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ebx, (%esp) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 96(%esp,%eax), %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 92(%esp,%eax), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edi, %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edx, %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 104(%esp,%eax), %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 100(%esp,%eax), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %esi, %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edx, %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 48(%esp,%eax), %ebp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 108(%esp,%eax), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edx, %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %esi, 56(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %esi, %ebp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: sarl %cl, %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, 60(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edi, 48(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ebx, 52(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, 40(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, 44(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl (%esp), %ecx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, 32(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, 36(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, 24(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, 28(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, 16(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, 20(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, 8(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, 12(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ebp, (%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %ecx, 4(%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: addl $188, %esp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: popl %edi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: popl %ebx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: popl %ebp
-; X86-NO-BMI2-HAVE-SHLD-NEXT: retl
+; X64-BMI2-LABEL: ashr_64bytes:
+; X64-BMI2: # %bb.0:
+; X64-BMI2-NEXT: pushq %r15
+; X64-BMI2-NEXT: pushq %r14
+; X64-BMI2-NEXT: pushq %rbx
+; X64-BMI2-NEXT: movq (%rdi), %rcx
+; X64-BMI2-NEXT: movq 8(%rdi), %r8
+; X64-BMI2-NEXT: movq 16(%rdi), %r9
+; X64-BMI2-NEXT: movq 24(%rdi), %r10
+; X64-BMI2-NEXT: movq 32(%rdi), %r11
+; X64-BMI2-NEXT: movq 40(%rdi), %rbx
+; X64-BMI2-NEXT: movq 48(%rdi), %r14
+; X64-BMI2-NEXT: movq 56(%rdi), %rdi
+; X64-BMI2-NEXT: movl (%rsi), %eax
+; X64-BMI2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movq %r14, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movq %rbx, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movq %r10, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: sarq $63, %rdi
+; X64-BMI2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movl %eax, %ecx
+; X64-BMI2-NEXT: andl $63, %ecx
+; X64-BMI2-NEXT: shrl $3, %eax
+; X64-BMI2-NEXT: andl $56, %eax
+; X64-BMI2-NEXT: movq -112(%rsp,%rax), %rdi
+; X64-BMI2-NEXT: movq -128(%rsp,%rax), %rsi
+; X64-BMI2-NEXT: movq -120(%rsp,%rax), %r9
+; X64-BMI2-NEXT: movq %r9, %r8
+; X64-BMI2-NEXT: shrdq %cl, %rdi, %r8
+; X64-BMI2-NEXT: movq -96(%rsp,%rax), %r10
+; X64-BMI2-NEXT: movq -104(%rsp,%rax), %r11
+; X64-BMI2-NEXT: movq %r11, %rbx
+; X64-BMI2-NEXT: shrdq %cl, %r10, %rbx
+; X64-BMI2-NEXT: shrdq %cl, %r11, %rdi
+; X64-BMI2-NEXT: movq -80(%rsp,%rax), %r11
+; X64-BMI2-NEXT: movq -88(%rsp,%rax), %r14
+; X64-BMI2-NEXT: movq %r14, %r15
+; X64-BMI2-NEXT: shrdq %cl, %r11, %r15
+; X64-BMI2-NEXT: shrdq %cl, %r14, %r10
+; X64-BMI2-NEXT: movq -72(%rsp,%rax), %rax
+; X64-BMI2-NEXT: shrdq %cl, %rax, %r11
+; X64-BMI2-NEXT: shrdq %cl, %r9, %rsi
+; X64-BMI2-NEXT: sarxq %rcx, %rax, %rax
+; X64-BMI2-NEXT: movq %r11, 48(%rdx)
+; X64-BMI2-NEXT: movq %rax, 56(%rdx)
+; X64-BMI2-NEXT: movq %r10, 32(%rdx)
+; X64-BMI2-NEXT: movq %r15, 40(%rdx)
+; X64-BMI2-NEXT: movq %rdi, 16(%rdx)
+; X64-BMI2-NEXT: movq %rbx, 24(%rdx)
+; X64-BMI2-NEXT: movq %rsi, (%rdx)
+; X64-BMI2-NEXT: movq %r8, 8(%rdx)
+; X64-BMI2-NEXT: popq %rbx
+; X64-BMI2-NEXT: popq %r14
+; X64-BMI2-NEXT: popq %r15
+; X64-BMI2-NEXT: retq
;
-; X86-HAVE-BMI2-NO-SHLD-LABEL: ashr_64bytes:
-; X86-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: subl $204, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl (%eax), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 4(%eax), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 8(%eax), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 12(%eax), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 16(%eax), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 20(%eax), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 24(%eax), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 28(%eax), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 32(%eax), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 36(%eax), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 40(%eax), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 44(%eax), %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 48(%eax), %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 52(%eax), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 56(%eax), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 60(%eax), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl (%eax), %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ebp, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: sarl $31, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ebx, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: andl $31, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrl $3, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: andl $60, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 68(%esp,%ebx), %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 72(%esp,%ebx), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: notl %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%esi,%esi), %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %eax, %ebp, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %edx, %edi, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %ebp, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: xorb $31, %cl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %edi, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %edi, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %edx, 64(%esp,%ebx), %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %edi, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 80(%esp,%ebx), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%esi,%esi), %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %eax, %ebp, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 76(%esp,%ebx), %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %edx, %ebp, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %esi, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %edx, {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %ebp, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %ebp, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %esi, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 88(%esp,%ebx), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%eax,%eax), %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %edi, %ebp, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 84(%esp,%ebx), %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %edx, %ebp, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %esi, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %edx, {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %ebp, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %ebp, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %esi, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 96(%esp,%ebx), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%eax,%eax), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %edi, %esi, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 92(%esp,%ebx), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %edx, %esi, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %ebp, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %edx, {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %esi, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %esi, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %ebp, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 104(%esp,%ebx), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%eax,%eax), %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %edi, %ebp, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 100(%esp,%ebx), %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %edx, %ebp, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %esi, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %edx, {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %ebp, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %ebp, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %esi, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 112(%esp,%ebx), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%eax,%eax), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edi, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %edi, %esi, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 108(%esp,%ebx), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %edx, %esi, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %edi, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %edx, {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %esi, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %esi, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %edi, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 120(%esp,%ebx), %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%edi,%edi), %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %eax, %ebp, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 116(%esp,%ebx), %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %edx, %ebp, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %esi, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %edx, {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %ebp, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %ebp, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %esi, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 124(%esp,%ebx), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%eax,%eax), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %esi, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %edx, %edi, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %edi, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: sarxl %edx, %eax, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edx, 60(%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, 56(%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ebp, 48(%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, 52(%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, 40(%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, 44(%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, 32(%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, 36(%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, 24(%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, 28(%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, 16(%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, 20(%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, 8(%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, 12(%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, (%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, 4(%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl $204, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: retl
+; X86-NO-BMI2-LABEL: ashr_64bytes:
+; X86-NO-BMI2: # %bb.0:
+; X86-NO-BMI2-NEXT: pushl %ebp
+; X86-NO-BMI2-NEXT: pushl %ebx
+; X86-NO-BMI2-NEXT: pushl %edi
+; X86-NO-BMI2-NEXT: pushl %esi
+; X86-NO-BMI2-NEXT: subl $188, %esp
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-BMI2-NEXT: movl (%eax), %ecx
+; X86-NO-BMI2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 4(%eax), %ecx
+; X86-NO-BMI2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 8(%eax), %ecx
+; X86-NO-BMI2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 12(%eax), %ecx
+; X86-NO-BMI2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 16(%eax), %ecx
+; X86-NO-BMI2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 20(%eax), %ecx
+; X86-NO-BMI2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 24(%eax), %ecx
+; X86-NO-BMI2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 28(%eax), %ecx
+; X86-NO-BMI2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 32(%eax), %ecx
+; X86-NO-BMI2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 36(%eax), %ecx
+; X86-NO-BMI2-NEXT: movl %ecx, (%esp) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 40(%eax), %ebp
+; X86-NO-BMI2-NEXT: movl 44(%eax), %ebx
+; X86-NO-BMI2-NEXT: movl 48(%eax), %edi
+; X86-NO-BMI2-NEXT: movl 52(%eax), %esi
+; X86-NO-BMI2-NEXT: movl 56(%eax), %edx
+; X86-NO-BMI2-NEXT: movl 60(%eax), %ecx
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-BMI2-NEXT: movl (%eax), %eax
+; X86-NO-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %ebp, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl (%esp), %edx # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: sarl $31, %ecx
+; X86-NO-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NO-BMI2-NEXT: movl %eax, %ecx
+; X86-NO-BMI2-NEXT: andl $31, %ecx
+; X86-NO-BMI2-NEXT: shrl $3, %eax
+; X86-NO-BMI2-NEXT: andl $60, %eax
+; X86-NO-BMI2-NEXT: movl 56(%esp,%eax), %esi
+; X86-NO-BMI2-NEXT: movl 52(%esp,%eax), %edx
+; X86-NO-BMI2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: shrdl %cl, %esi, %edx
+; X86-NO-BMI2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 64(%esp,%eax), %ebx
+; X86-NO-BMI2-NEXT: movl 60(%esp,%eax), %edx
+; X86-NO-BMI2-NEXT: movl %edx, %edi
+; X86-NO-BMI2-NEXT: shrdl %cl, %ebx, %edi
+; X86-NO-BMI2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: shrdl %cl, %edx, %esi
+; X86-NO-BMI2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 72(%esp,%eax), %edi
+; X86-NO-BMI2-NEXT: movl 68(%esp,%eax), %edx
+; X86-NO-BMI2-NEXT: movl %edx, %esi
+; X86-NO-BMI2-NEXT: shrdl %cl, %edi, %esi
+; X86-NO-BMI2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: shrdl %cl, %edx, %ebx
+; X86-NO-BMI2-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 80(%esp,%eax), %ebx
+; X86-NO-BMI2-NEXT: movl 76(%esp,%eax), %edx
+; X86-NO-BMI2-NEXT: movl %edx, %esi
+; X86-NO-BMI2-NEXT: shrdl %cl, %ebx, %esi
+; X86-NO-BMI2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: shrdl %cl, %edx, %edi
+; X86-NO-BMI2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 88(%esp,%eax), %edi
+; X86-NO-BMI2-NEXT: movl 84(%esp,%eax), %edx
+; X86-NO-BMI2-NEXT: movl %edx, %esi
+; X86-NO-BMI2-NEXT: shrdl %cl, %edi, %esi
+; X86-NO-BMI2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl %edi, %esi
+; X86-NO-BMI2-NEXT: shrdl %cl, %edx, %ebx
+; X86-NO-BMI2-NEXT: movl %ebx, (%esp) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 96(%esp,%eax), %edi
+; X86-NO-BMI2-NEXT: movl 92(%esp,%eax), %edx
+; X86-NO-BMI2-NEXT: movl %edx, %ebx
+; X86-NO-BMI2-NEXT: shrdl %cl, %edi, %ebx
+; X86-NO-BMI2-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: shrdl %cl, %edx, %esi
+; X86-NO-BMI2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-BMI2-NEXT: movl 104(%esp,%eax), %esi
+; X86-NO-BMI2-NEXT: movl 100(%esp,%eax), %edx
+; X86-NO-BMI2-NEXT: movl %edx, %ebx
+; X86-NO-BMI2-NEXT: shrdl %cl, %esi, %ebx
+; X86-NO-BMI2-NEXT: shrdl %cl, %edx, %edi
+; X86-NO-BMI2-NEXT: movl 48(%esp,%eax), %ebp
+; X86-NO-BMI2-NEXT: movl 108(%esp,%eax), %edx
+; X86-NO-BMI2-NEXT: shrdl %cl, %edx, %esi
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-BMI2-NEXT: movl %esi, 56(%eax)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NO-BMI2-NEXT: shrdl %cl, %esi, %ebp
+; X86-NO-BMI2-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NO-BMI2-NEXT: sarl %cl, %edx
+; X86-NO-BMI2-NEXT: movl %edx, 60(%eax)
+; X86-NO-BMI2-NEXT: movl %edi, 48(%eax)
+; X86-NO-BMI2-NEXT: movl %ebx, 52(%eax)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %ecx, 40(%eax)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %ecx, 44(%eax)
+; X86-NO-BMI2-NEXT: movl (%esp), %ecx # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %ecx, 32(%eax)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %ecx, 36(%eax)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %ecx, 24(%eax)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %ecx, 28(%eax)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %ecx, 16(%eax)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %ecx, 20(%eax)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %ecx, 8(%eax)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %ecx, 12(%eax)
+; X86-NO-BMI2-NEXT: movl %ebp, (%eax)
+; X86-NO-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NO-BMI2-NEXT: movl %ecx, 4(%eax)
+; X86-NO-BMI2-NEXT: addl $188, %esp
+; X86-NO-BMI2-NEXT: popl %esi
+; X86-NO-BMI2-NEXT: popl %edi
+; X86-NO-BMI2-NEXT: popl %ebx
+; X86-NO-BMI2-NEXT: popl %ebp
+; X86-NO-BMI2-NEXT: retl
;
-; X86-HAVE-BMI2-HAVE-SHLD-LABEL: ashr_64bytes:
-; X86-HAVE-BMI2-HAVE-SHLD: # %bb.0:
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: pushl %ebp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: pushl %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: pushl %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: subl $188, %esp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl (%eax), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 4(%eax), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 8(%eax), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 12(%eax), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 16(%eax), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 20(%eax), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 24(%eax), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 28(%eax), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 32(%eax), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 36(%eax), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, (%esp) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 40(%eax), %ebp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 44(%eax), %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 48(%eax), %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 52(%eax), %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 56(%eax), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 60(%eax), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl (%eax), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ebp, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl (%esp), %edx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: sarl $31, %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: andl $31, %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrl $3, %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: andl $60, %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 56(%esp,%eax), %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 52(%esp,%eax), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edi, %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 64(%esp,%eax), %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 60(%esp,%eax), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %ebx, %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edx, %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 72(%esp,%eax), %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 68(%esp,%eax), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edi, %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edx, %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 80(%esp,%eax), %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 76(%esp,%eax), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %ebx, %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edx, %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 88(%esp,%eax), %ebp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 84(%esp,%eax), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %ebp, %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edx, %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ebx, (%esp) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 96(%esp,%eax), %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 92(%esp,%eax), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edi, %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edx, %ebp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 104(%esp,%eax), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 100(%esp,%eax), %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ebx, %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edx, %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %ebx, %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 48(%esp,%eax), %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 108(%esp,%eax), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %eax, %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, 56(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edi, 48(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %esi, 52(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ebp, 40(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, 44(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl (%esp), %edx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, 32(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, 36(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, 24(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, 28(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, 16(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, 20(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, 8(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, 12(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: sarxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %esi, %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ebx, (%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, 4(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, 60(%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: addl $188, %esp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: popl %edi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: popl %ebx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: popl %ebp
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: retl
+; X86-BMI2-LABEL: ashr_64bytes:
+; X86-BMI2: # %bb.0:
+; X86-BMI2-NEXT: pushl %ebp
+; X86-BMI2-NEXT: pushl %ebx
+; X86-BMI2-NEXT: pushl %edi
+; X86-BMI2-NEXT: pushl %esi
+; X86-BMI2-NEXT: subl $188, %esp
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT: movl (%eax), %ecx
+; X86-BMI2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 4(%eax), %ecx
+; X86-BMI2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 8(%eax), %ecx
+; X86-BMI2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 12(%eax), %ecx
+; X86-BMI2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 16(%eax), %ecx
+; X86-BMI2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 20(%eax), %ecx
+; X86-BMI2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 24(%eax), %ecx
+; X86-BMI2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 28(%eax), %ecx
+; X86-BMI2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 32(%eax), %ecx
+; X86-BMI2-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 36(%eax), %ecx
+; X86-BMI2-NEXT: movl %ecx, (%esp) # 4-byte Spill
+; X86-BMI2-NEXT: movl 40(%eax), %ebp
+; X86-BMI2-NEXT: movl 44(%eax), %ebx
+; X86-BMI2-NEXT: movl 48(%eax), %edi
+; X86-BMI2-NEXT: movl 52(%eax), %esi
+; X86-BMI2-NEXT: movl 56(%eax), %edx
+; X86-BMI2-NEXT: movl 60(%eax), %ecx
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT: movl (%eax), %eax
+; X86-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %ebp, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl (%esp), %edx # 4-byte Reload
+; X86-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-BMI2-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: sarl $31, %ecx
+; X86-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-BMI2-NEXT: movl %eax, %ecx
+; X86-BMI2-NEXT: andl $31, %ecx
+; X86-BMI2-NEXT: shrl $3, %eax
+; X86-BMI2-NEXT: andl $60, %eax
+; X86-BMI2-NEXT: movl 56(%esp,%eax), %edi
+; X86-BMI2-NEXT: movl 52(%esp,%eax), %edx
+; X86-BMI2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: shrdl %cl, %edi, %edx
+; X86-BMI2-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 64(%esp,%eax), %ebx
+; X86-BMI2-NEXT: movl 60(%esp,%eax), %edx
+; X86-BMI2-NEXT: movl %edx, %esi
+; X86-BMI2-NEXT: shrdl %cl, %ebx, %esi
+; X86-BMI2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: shrdl %cl, %edx, %edi
+; X86-BMI2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 72(%esp,%eax), %edi
+; X86-BMI2-NEXT: movl 68(%esp,%eax), %edx
+; X86-BMI2-NEXT: movl %edx, %esi
+; X86-BMI2-NEXT: shrdl %cl, %edi, %esi
+; X86-BMI2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: shrdl %cl, %edx, %ebx
+; X86-BMI2-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 80(%esp,%eax), %ebx
+; X86-BMI2-NEXT: movl 76(%esp,%eax), %edx
+; X86-BMI2-NEXT: movl %edx, %esi
+; X86-BMI2-NEXT: shrdl %cl, %ebx, %esi
+; X86-BMI2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: shrdl %cl, %edx, %edi
+; X86-BMI2-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: movl 88(%esp,%eax), %ebp
+; X86-BMI2-NEXT: movl 84(%esp,%eax), %edx
+; X86-BMI2-NEXT: movl %edx, %esi
+; X86-BMI2-NEXT: shrdl %cl, %ebp, %esi
+; X86-BMI2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: shrdl %cl, %edx, %ebx
+; X86-BMI2-NEXT: movl %ebx, (%esp) # 4-byte Spill
+; X86-BMI2-NEXT: movl 96(%esp,%eax), %edi
+; X86-BMI2-NEXT: movl 92(%esp,%eax), %edx
+; X86-BMI2-NEXT: movl %edx, %esi
+; X86-BMI2-NEXT: shrdl %cl, %edi, %esi
+; X86-BMI2-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: shrdl %cl, %edx, %ebp
+; X86-BMI2-NEXT: movl 104(%esp,%eax), %edx
+; X86-BMI2-NEXT: movl 100(%esp,%eax), %ebx
+; X86-BMI2-NEXT: movl %ebx, %esi
+; X86-BMI2-NEXT: shrdl %cl, %edx, %esi
+; X86-BMI2-NEXT: shrdl %cl, %ebx, %edi
+; X86-BMI2-NEXT: movl 48(%esp,%eax), %ebx
+; X86-BMI2-NEXT: movl 108(%esp,%eax), %eax
+; X86-BMI2-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-BMI2-NEXT: shrdl %cl, %eax, %edx
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT: movl %edx, 56(%eax)
+; X86-BMI2-NEXT: movl %edi, 48(%eax)
+; X86-BMI2-NEXT: movl %esi, 52(%eax)
+; X86-BMI2-NEXT: movl %ebp, 40(%eax)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-BMI2-NEXT: movl %edx, 44(%eax)
+; X86-BMI2-NEXT: movl (%esp), %edx # 4-byte Reload
+; X86-BMI2-NEXT: movl %edx, 32(%eax)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-BMI2-NEXT: movl %edx, 36(%eax)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-BMI2-NEXT: movl %edx, 24(%eax)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-BMI2-NEXT: movl %edx, 28(%eax)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-BMI2-NEXT: movl %edx, 16(%eax)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-BMI2-NEXT: movl %edx, 20(%eax)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-BMI2-NEXT: movl %edx, 8(%eax)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-BMI2-NEXT: movl %edx, 12(%eax)
+; X86-BMI2-NEXT: sarxl %ecx, {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X86-BMI2-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-BMI2-NEXT: shrdl %cl, %esi, %ebx
+; X86-BMI2-NEXT: movl %ebx, (%eax)
+; X86-BMI2-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-BMI2-NEXT: movl %ecx, 4(%eax)
+; X86-BMI2-NEXT: movl %edx, 60(%eax)
+; X86-BMI2-NEXT: addl $188, %esp
+; X86-BMI2-NEXT: popl %esi
+; X86-BMI2-NEXT: popl %edi
+; X86-BMI2-NEXT: popl %ebx
+; X86-BMI2-NEXT: popl %ebp
+; X86-BMI2-NEXT: retl
%src = load i512, ptr %src.ptr, align 1
%bitOff = load i512, ptr %bitOff.ptr, align 1
%res = ashr i512 %src, %bitOff
@@ -6537,8 +2811,16 @@ define void @ashr_64bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; ALL: {{.*}}
; X64: {{.*}}
+; X64-HAVE-BMI2-HAVE-SHLD: {{.*}}
+; X64-HAVE-BMI2-NO-SHLD: {{.*}}
+; X64-NO-BMI2-HAVE-SHLD: {{.*}}
+; X64-NO-BMI2-NO-SHLD: {{.*}}
; X64-NO-SHLD: {{.*}}
; X64-SHLD: {{.*}}
; X86: {{.*}}
+; X86-HAVE-BMI2-HAVE-SHLD: {{.*}}
+; X86-HAVE-BMI2-NO-SHLD: {{.*}}
+; X86-NO-BMI2-HAVE-SHLD: {{.*}}
+; X86-NO-BMI2-NO-SHLD: {{.*}}
; X86-NO-SHLD: {{.*}}
; X86-SHLD: {{.*}}
diff --git a/llvm/test/CodeGen/X86/widen-load-of-small-alloca-with-zero-upper-half.ll b/llvm/test/CodeGen/X86/widen-load-of-small-alloca-with-zero-upper-half.ll
index fde915247760a..c436002369a50 100644
--- a/llvm/test/CodeGen/X86/widen-load-of-small-alloca-with-zero-upper-half.ll
+++ b/llvm/test/CodeGen/X86/widen-load-of-small-alloca-with-zero-upper-half.ll
@@ -186,52 +186,21 @@ define void @load_1byte_chunk_of_8byte_alloca_with_zero_upper_half(ptr %src, i64
; X64-BMI2-NEXT: movb %al, (%rdx)
; X64-BMI2-NEXT: retq
;
-; X86-NO-BMI2-NO-SHLD-LABEL: load_1byte_chunk_of_8byte_alloca_with_zero_upper_half:
-; X86-NO-BMI2-NO-SHLD: # %bb.0:
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl (%edx), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll $3, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: xorl %ebx, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: testb $32, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: cmovel %edx, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %bl, (%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: retl
-;
-; X86-SHLD-LABEL: load_1byte_chunk_of_8byte_alloca_with_zero_upper_half:
-; X86-SHLD: # %bb.0:
-; X86-SHLD-NEXT: pushl %esi
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-SHLD-NEXT: movl (%edx), %edx
-; X86-SHLD-NEXT: shll $3, %ecx
-; X86-SHLD-NEXT: xorl %esi, %esi
-; X86-SHLD-NEXT: shrdl %cl, %esi, %edx
-; X86-SHLD-NEXT: testb $32, %cl
-; X86-SHLD-NEXT: cmovnel %esi, %edx
-; X86-SHLD-NEXT: movb %dl, (%eax)
-; X86-SHLD-NEXT: popl %esi
-; X86-SHLD-NEXT: retl
-;
-; X86-HAVE-BMI2-NO-SHLD-LABEL: load_1byte_chunk_of_8byte_alloca_with_zero_upper_half:
-; X86-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shll $3, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, (%edx), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: xorl %ebx, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: testb $32, %cl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: cmovel %edx, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movb %bl, (%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: retl
+; X86-LABEL: load_1byte_chunk_of_8byte_alloca_with_zero_upper_half:
+; X86: # %bb.0:
+; X86-NEXT: pushl %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movl (%edx), %edx
+; X86-NEXT: shll $3, %ecx
+; X86-NEXT: xorl %esi, %esi
+; X86-NEXT: shrdl %cl, %esi, %edx
+; X86-NEXT: testb $32, %cl
+; X86-NEXT: cmovnel %esi, %edx
+; X86-NEXT: movb %dl, (%eax)
+; X86-NEXT: popl %esi
+; X86-NEXT: retl
%init = load <4 x i8>, ptr %src, align 1
%intermediate.sroa.0.0.vec.expand = shufflevector <4 x i8> %init, <4 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
%intermediate.sroa.0.0.vecblend = shufflevector <8 x i8> %intermediate.sroa.0.0.vec.expand, <8 x i8> <i8 poison, i8 poison, i8 poison, i8 poison, i8 0, i8 0, i8 0, i8 0>, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
@@ -263,52 +232,21 @@ define void @load_2byte_chunk_of_8byte_alloca_with_zero_upper_half(ptr %src, i64
; X64-BMI2-NEXT: movw %ax, (%rdx)
; X64-BMI2-NEXT: retq
;
-; X86-NO-BMI2-NO-SHLD-LABEL: load_2byte_chunk_of_8byte_alloca_with_zero_upper_half:
-; X86-NO-BMI2-NO-SHLD: # %bb.0:
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl (%edx), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll $3, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: xorl %esi, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: testb $32, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: cmovel %edx, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movw %si, (%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: retl
-;
-; X86-SHLD-LABEL: load_2byte_chunk_of_8byte_alloca_with_zero_upper_half:
-; X86-SHLD: # %bb.0:
-; X86-SHLD-NEXT: pushl %esi
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-SHLD-NEXT: movl (%edx), %edx
-; X86-SHLD-NEXT: shll $3, %ecx
-; X86-SHLD-NEXT: xorl %esi, %esi
-; X86-SHLD-NEXT: shrdl %cl, %esi, %edx
-; X86-SHLD-NEXT: testb $32, %cl
-; X86-SHLD-NEXT: cmovnel %esi, %edx
-; X86-SHLD-NEXT: movw %dx, (%eax)
-; X86-SHLD-NEXT: popl %esi
-; X86-SHLD-NEXT: retl
-;
-; X86-HAVE-BMI2-NO-SHLD-LABEL: load_2byte_chunk_of_8byte_alloca_with_zero_upper_half:
-; X86-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shll $3, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %eax, (%edx), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: xorl %esi, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: testb $32, %al
-; X86-HAVE-BMI2-NO-SHLD-NEXT: cmovel %edx, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movw %si, (%ecx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: retl
+; X86-LABEL: load_2byte_chunk_of_8byte_alloca_with_zero_upper_half:
+; X86: # %bb.0:
+; X86-NEXT: pushl %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movl (%edx), %edx
+; X86-NEXT: shll $3, %ecx
+; X86-NEXT: xorl %esi, %esi
+; X86-NEXT: shrdl %cl, %esi, %edx
+; X86-NEXT: testb $32, %cl
+; X86-NEXT: cmovnel %esi, %edx
+; X86-NEXT: movw %dx, (%eax)
+; X86-NEXT: popl %esi
+; X86-NEXT: retl
%init = load <4 x i8>, ptr %src, align 1
%intermediate.sroa.0.0.vec.expand = shufflevector <4 x i8> %init, <4 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
%intermediate.sroa.0.0.vecblend = shufflevector <8 x i8> %intermediate.sroa.0.0.vec.expand, <8 x i8> <i8 poison, i8 poison, i8 poison, i8 poison, i8 0, i8 0, i8 0, i8 0>, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
@@ -339,52 +277,21 @@ define void @load_4byte_chunk_of_8byte_alloca_with_zero_upper_half(ptr %src, i64
; X64-BMI2-NEXT: movl %eax, (%rdx)
; X64-BMI2-NEXT: retq
;
-; X86-NO-BMI2-NO-SHLD-LABEL: load_4byte_chunk_of_8byte_alloca_with_zero_upper_half:
-; X86-NO-BMI2-NO-SHLD: # %bb.0:
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl (%edx), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll $3, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: xorl %esi, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: testb $32, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: cmovel %edx, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %esi, (%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: retl
-;
-; X86-SHLD-LABEL: load_4byte_chunk_of_8byte_alloca_with_zero_upper_half:
-; X86-SHLD: # %bb.0:
-; X86-SHLD-NEXT: pushl %esi
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-SHLD-NEXT: movl (%edx), %edx
-; X86-SHLD-NEXT: shll $3, %ecx
-; X86-SHLD-NEXT: xorl %esi, %esi
-; X86-SHLD-NEXT: shrdl %cl, %esi, %edx
-; X86-SHLD-NEXT: testb $32, %cl
-; X86-SHLD-NEXT: cmovnel %esi, %edx
-; X86-SHLD-NEXT: movl %edx, (%eax)
-; X86-SHLD-NEXT: popl %esi
-; X86-SHLD-NEXT: retl
-;
-; X86-HAVE-BMI2-NO-SHLD-LABEL: load_4byte_chunk_of_8byte_alloca_with_zero_upper_half:
-; X86-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shll $3, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %eax, (%edx), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: xorl %esi, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: testb $32, %al
-; X86-HAVE-BMI2-NO-SHLD-NEXT: cmovel %edx, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, (%ecx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: retl
+; X86-LABEL: load_4byte_chunk_of_8byte_alloca_with_zero_upper_half:
+; X86: # %bb.0:
+; X86-NEXT: pushl %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movl (%edx), %edx
+; X86-NEXT: shll $3, %ecx
+; X86-NEXT: xorl %esi, %esi
+; X86-NEXT: shrdl %cl, %esi, %edx
+; X86-NEXT: testb $32, %cl
+; X86-NEXT: cmovnel %esi, %edx
+; X86-NEXT: movl %edx, (%eax)
+; X86-NEXT: popl %esi
+; X86-NEXT: retl
%init = load <4 x i8>, ptr %src, align 1
%intermediate.sroa.0.0.vec.expand = shufflevector <4 x i8> %init, <4 x i8> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
%intermediate.sroa.0.0.vecblend = shufflevector <8 x i8> %intermediate.sroa.0.0.vec.expand, <8 x i8> <i8 poison, i8 poison, i8 poison, i8 poison, i8 0, i8 0, i8 0, i8 0>, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 12, i32 13, i32 14, i32 15>
@@ -398,123 +305,43 @@ define void @load_4byte_chunk_of_8byte_alloca_with_zero_upper_half(ptr %src, i64
}
define void @load_1byte_chunk_of_16byte_alloca_with_zero_upper_half(ptr %src, i64 %byteOff, ptr %dst) nounwind {
-; X64-NO-BMI2-NO-SHLD-LABEL: load_1byte_chunk_of_16byte_alloca_with_zero_upper_half:
-; X64-NO-BMI2-NO-SHLD: # %bb.0:
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rsi, %rcx
-; X64-NO-BMI2-NO-SHLD-NEXT: movq (%rdi), %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: shll $3, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: xorl %esi, %esi
-; X64-NO-BMI2-NO-SHLD-NEXT: testb $64, %cl
-; X64-NO-BMI2-NO-SHLD-NEXT: cmoveq %rax, %rsi
-; X64-NO-BMI2-NO-SHLD-NEXT: movb %sil, (%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: retq
-;
-; X64-SHLD-LABEL: load_1byte_chunk_of_16byte_alloca_with_zero_upper_half:
-; X64-SHLD: # %bb.0:
-; X64-SHLD-NEXT: movq %rsi, %rcx
-; X64-SHLD-NEXT: movq (%rdi), %rax
-; X64-SHLD-NEXT: shll $3, %ecx
-; X64-SHLD-NEXT: xorl %esi, %esi
-; X64-SHLD-NEXT: shrdq %cl, %rsi, %rax
-; X64-SHLD-NEXT: testb $64, %cl
-; X64-SHLD-NEXT: cmovneq %rsi, %rax
-; X64-SHLD-NEXT: movb %al, (%rdx)
-; X64-SHLD-NEXT: retq
-;
-; X64-HAVE-BMI2-NO-SHLD-LABEL: load_1byte_chunk_of_16byte_alloca_with_zero_upper_half:
-; X64-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shll $3, %esi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rsi, (%rdi), %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: xorl %ecx, %ecx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: testb $64, %sil
-; X64-HAVE-BMI2-NO-SHLD-NEXT: cmoveq %rax, %rcx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movb %cl, (%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: retq
-;
-; X86-NO-BMI2-NO-SHLD-LABEL: load_1byte_chunk_of_16byte_alloca_with_zero_upper_half:
-; X86-NO-BMI2-NO-SHLD: # %bb.0:
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: subl $40, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movq {{.*#+}} xmm0 = mem[0],zero
-; X86-NO-BMI2-NO-SHLD-NEXT: movq {{.*#+}} xmm0 = xmm0[0],zero
-; X86-NO-BMI2-NO-SHLD-NEXT: shll $3, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: xorps %xmm1, %xmm1
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movdqa %xmm0, (%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrb $3, %dl
-; X86-NO-BMI2-NO-SHLD-NEXT: andb $12, %dl
-; X86-NO-BMI2-NO-SHLD-NEXT: movzbl %dl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl (%esp,%edx), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 4(%esp,%edx), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: notb %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %edx, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dl, (%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: addl $40, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: retl
-;
-; X86-SHLD-LABEL: load_1byte_chunk_of_16byte_alloca_with_zero_upper_half:
-; X86-SHLD: # %bb.0:
-; X86-SHLD-NEXT: pushl %ebx
-; X86-SHLD-NEXT: subl $40, %esp
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-SHLD-NEXT: movq {{.*#+}} xmm0 = mem[0],zero
-; X86-SHLD-NEXT: movq {{.*#+}} xmm0 = xmm0[0],zero
-; X86-SHLD-NEXT: shll $3, %ecx
-; X86-SHLD-NEXT: xorps %xmm1, %xmm1
-; X86-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movdqa %xmm0, (%esp)
-; X86-SHLD-NEXT: movl %ecx, %edx
-; X86-SHLD-NEXT: shrb $3, %dl
-; X86-SHLD-NEXT: andb $12, %dl
-; X86-SHLD-NEXT: movzbl %dl, %edx
-; X86-SHLD-NEXT: movl (%esp,%edx), %ebx
-; X86-SHLD-NEXT: movl 4(%esp,%edx), %edx
-; X86-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-SHLD-NEXT: shrdl %cl, %edx, %ebx
-; X86-SHLD-NEXT: movb %bl, (%eax)
-; X86-SHLD-NEXT: addl $40, %esp
-; X86-SHLD-NEXT: popl %ebx
-; X86-SHLD-NEXT: retl
+; X64-LABEL: load_1byte_chunk_of_16byte_alloca_with_zero_upper_half:
+; X64: # %bb.0:
+; X64-NEXT: movq %rsi, %rcx
+; X64-NEXT: movq (%rdi), %rax
+; X64-NEXT: shll $3, %ecx
+; X64-NEXT: xorl %esi, %esi
+; X64-NEXT: shrdq %cl, %rsi, %rax
+; X64-NEXT: testb $64, %cl
+; X64-NEXT: cmovneq %rsi, %rax
+; X64-NEXT: movb %al, (%rdx)
+; X64-NEXT: retq
;
-; X86-HAVE-BMI2-NO-SHLD-LABEL: load_1byte_chunk_of_16byte_alloca_with_zero_upper_half:
-; X86-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: subl $40, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movq {{.*#+}} xmm0 = mem[0],zero
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movq {{.*#+}} xmm0 = xmm0[0],zero
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shll $3, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: xorps %xmm1, %xmm1
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movdqa %xmm0, (%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrb $3, %dl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: andb $12, %dl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movzbl %dl, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, (%esp,%edx), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: notb %cl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 4(%esp,%edx), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %edx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %edx, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %esi, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movb %cl, (%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl $40, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: retl
+; X86-LABEL: load_1byte_chunk_of_16byte_alloca_with_zero_upper_half:
+; X86: # %bb.0:
+; X86-NEXT: pushl %ebx
+; X86-NEXT: subl $40, %esp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movq {{.*#+}} xmm0 = mem[0],zero
+; X86-NEXT: movq {{.*#+}} xmm0 = xmm0[0],zero
+; X86-NEXT: shll $3, %ecx
+; X86-NEXT: xorps %xmm1, %xmm1
+; X86-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NEXT: movdqa %xmm0, (%esp)
+; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: shrb $3, %dl
+; X86-NEXT: andb $12, %dl
+; X86-NEXT: movzbl %dl, %edx
+; X86-NEXT: movl (%esp,%edx), %ebx
+; X86-NEXT: movl 4(%esp,%edx), %edx
+; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: shrdl %cl, %edx, %ebx
+; X86-NEXT: movb %bl, (%eax)
+; X86-NEXT: addl $40, %esp
+; X86-NEXT: popl %ebx
+; X86-NEXT: retl
%init = load <8 x i8>, ptr %src, align 1
%intermediate.sroa.0.0.vec.expand = shufflevector <8 x i8> %init, <8 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
%intermediate.sroa.0.0.vecblend = shufflevector <16 x i8> %intermediate.sroa.0.0.vec.expand, <16 x i8> <i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
@@ -530,123 +357,43 @@ define void @load_1byte_chunk_of_16byte_alloca_with_zero_upper_half(ptr %src, i6
}
define void @load_2byte_chunk_of_16byte_alloca_with_zero_upper_half(ptr %src, i64 %byteOff, ptr %dst) nounwind {
-; X64-NO-BMI2-NO-SHLD-LABEL: load_2byte_chunk_of_16byte_alloca_with_zero_upper_half:
-; X64-NO-BMI2-NO-SHLD: # %bb.0:
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rsi, %rcx
-; X64-NO-BMI2-NO-SHLD-NEXT: movq (%rdi), %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: shll $3, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: xorl %esi, %esi
-; X64-NO-BMI2-NO-SHLD-NEXT: testb $64, %cl
-; X64-NO-BMI2-NO-SHLD-NEXT: cmoveq %rax, %rsi
-; X64-NO-BMI2-NO-SHLD-NEXT: movw %si, (%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: retq
-;
-; X64-SHLD-LABEL: load_2byte_chunk_of_16byte_alloca_with_zero_upper_half:
-; X64-SHLD: # %bb.0:
-; X64-SHLD-NEXT: movq %rsi, %rcx
-; X64-SHLD-NEXT: movq (%rdi), %rax
-; X64-SHLD-NEXT: shll $3, %ecx
-; X64-SHLD-NEXT: xorl %esi, %esi
-; X64-SHLD-NEXT: shrdq %cl, %rsi, %rax
-; X64-SHLD-NEXT: testb $64, %cl
-; X64-SHLD-NEXT: cmovneq %rsi, %rax
-; X64-SHLD-NEXT: movw %ax, (%rdx)
-; X64-SHLD-NEXT: retq
-;
-; X64-HAVE-BMI2-NO-SHLD-LABEL: load_2byte_chunk_of_16byte_alloca_with_zero_upper_half:
-; X64-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shll $3, %esi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rsi, (%rdi), %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: xorl %ecx, %ecx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: testb $64, %sil
-; X64-HAVE-BMI2-NO-SHLD-NEXT: cmoveq %rax, %rcx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movw %cx, (%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: retq
-;
-; X86-NO-BMI2-NO-SHLD-LABEL: load_2byte_chunk_of_16byte_alloca_with_zero_upper_half:
-; X86-NO-BMI2-NO-SHLD: # %bb.0:
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: subl $40, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movq {{.*#+}} xmm0 = mem[0],zero
-; X86-NO-BMI2-NO-SHLD-NEXT: movq {{.*#+}} xmm0 = xmm0[0],zero
-; X86-NO-BMI2-NO-SHLD-NEXT: shll $3, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: xorps %xmm1, %xmm1
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movdqa %xmm0, (%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrb $3, %dl
-; X86-NO-BMI2-NO-SHLD-NEXT: andb $12, %dl
-; X86-NO-BMI2-NO-SHLD-NEXT: movzbl %dl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl (%esp,%edx), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 4(%esp,%edx), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: notb %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %edx, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movw %dx, (%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: addl $40, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: retl
-;
-; X86-SHLD-LABEL: load_2byte_chunk_of_16byte_alloca_with_zero_upper_half:
-; X86-SHLD: # %bb.0:
-; X86-SHLD-NEXT: pushl %esi
-; X86-SHLD-NEXT: subl $40, %esp
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-SHLD-NEXT: movq {{.*#+}} xmm0 = mem[0],zero
-; X86-SHLD-NEXT: movq {{.*#+}} xmm0 = xmm0[0],zero
-; X86-SHLD-NEXT: shll $3, %ecx
-; X86-SHLD-NEXT: xorps %xmm1, %xmm1
-; X86-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movdqa %xmm0, (%esp)
-; X86-SHLD-NEXT: movl %ecx, %edx
-; X86-SHLD-NEXT: shrb $3, %dl
-; X86-SHLD-NEXT: andb $12, %dl
-; X86-SHLD-NEXT: movzbl %dl, %edx
-; X86-SHLD-NEXT: movl (%esp,%edx), %esi
-; X86-SHLD-NEXT: movl 4(%esp,%edx), %edx
-; X86-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-SHLD-NEXT: shrdl %cl, %edx, %esi
-; X86-SHLD-NEXT: movw %si, (%eax)
-; X86-SHLD-NEXT: addl $40, %esp
-; X86-SHLD-NEXT: popl %esi
-; X86-SHLD-NEXT: retl
+; X64-LABEL: load_2byte_chunk_of_16byte_alloca_with_zero_upper_half:
+; X64: # %bb.0:
+; X64-NEXT: movq %rsi, %rcx
+; X64-NEXT: movq (%rdi), %rax
+; X64-NEXT: shll $3, %ecx
+; X64-NEXT: xorl %esi, %esi
+; X64-NEXT: shrdq %cl, %rsi, %rax
+; X64-NEXT: testb $64, %cl
+; X64-NEXT: cmovneq %rsi, %rax
+; X64-NEXT: movw %ax, (%rdx)
+; X64-NEXT: retq
;
-; X86-HAVE-BMI2-NO-SHLD-LABEL: load_2byte_chunk_of_16byte_alloca_with_zero_upper_half:
-; X86-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: subl $40, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movq {{.*#+}} xmm0 = mem[0],zero
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movq {{.*#+}} xmm0 = xmm0[0],zero
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shll $3, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: xorps %xmm1, %xmm1
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movdqa %xmm0, (%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrb $3, %dl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: andb $12, %dl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movzbl %dl, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, (%esp,%edx), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: notb %cl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 4(%esp,%edx), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %edx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %edx, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %esi, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movw %cx, (%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl $40, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: retl
+; X86-LABEL: load_2byte_chunk_of_16byte_alloca_with_zero_upper_half:
+; X86: # %bb.0:
+; X86-NEXT: pushl %esi
+; X86-NEXT: subl $40, %esp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movq {{.*#+}} xmm0 = mem[0],zero
+; X86-NEXT: movq {{.*#+}} xmm0 = xmm0[0],zero
+; X86-NEXT: shll $3, %ecx
+; X86-NEXT: xorps %xmm1, %xmm1
+; X86-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NEXT: movdqa %xmm0, (%esp)
+; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: shrb $3, %dl
+; X86-NEXT: andb $12, %dl
+; X86-NEXT: movzbl %dl, %edx
+; X86-NEXT: movl (%esp,%edx), %esi
+; X86-NEXT: movl 4(%esp,%edx), %edx
+; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: shrdl %cl, %edx, %esi
+; X86-NEXT: movw %si, (%eax)
+; X86-NEXT: addl $40, %esp
+; X86-NEXT: popl %esi
+; X86-NEXT: retl
%init = load <8 x i8>, ptr %src, align 1
%intermediate.sroa.0.0.vec.expand = shufflevector <8 x i8> %init, <8 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
%intermediate.sroa.0.0.vecblend = shufflevector <16 x i8> %intermediate.sroa.0.0.vec.expand, <16 x i8> <i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
@@ -661,123 +408,43 @@ define void @load_2byte_chunk_of_16byte_alloca_with_zero_upper_half(ptr %src, i6
}
define void @load_4byte_chunk_of_16byte_alloca_with_zero_upper_half(ptr %src, i64 %byteOff, ptr %dst) nounwind {
-; X64-NO-BMI2-NO-SHLD-LABEL: load_4byte_chunk_of_16byte_alloca_with_zero_upper_half:
-; X64-NO-BMI2-NO-SHLD: # %bb.0:
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rsi, %rcx
-; X64-NO-BMI2-NO-SHLD-NEXT: movq (%rdi), %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: shll $3, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: xorl %esi, %esi
-; X64-NO-BMI2-NO-SHLD-NEXT: testb $64, %cl
-; X64-NO-BMI2-NO-SHLD-NEXT: cmoveq %rax, %rsi
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %esi, (%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: retq
-;
-; X64-SHLD-LABEL: load_4byte_chunk_of_16byte_alloca_with_zero_upper_half:
-; X64-SHLD: # %bb.0:
-; X64-SHLD-NEXT: movq %rsi, %rcx
-; X64-SHLD-NEXT: movq (%rdi), %rax
-; X64-SHLD-NEXT: shll $3, %ecx
-; X64-SHLD-NEXT: xorl %esi, %esi
-; X64-SHLD-NEXT: shrdq %cl, %rsi, %rax
-; X64-SHLD-NEXT: testb $64, %cl
-; X64-SHLD-NEXT: cmovneq %rsi, %rax
-; X64-SHLD-NEXT: movl %eax, (%rdx)
-; X64-SHLD-NEXT: retq
-;
-; X64-HAVE-BMI2-NO-SHLD-LABEL: load_4byte_chunk_of_16byte_alloca_with_zero_upper_half:
-; X64-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shll $3, %esi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rsi, (%rdi), %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: xorl %ecx, %ecx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: testb $64, %sil
-; X64-HAVE-BMI2-NO-SHLD-NEXT: cmoveq %rax, %rcx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, (%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: retq
-;
-; X86-NO-BMI2-NO-SHLD-LABEL: load_4byte_chunk_of_16byte_alloca_with_zero_upper_half:
-; X86-NO-BMI2-NO-SHLD: # %bb.0:
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: subl $40, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movq {{.*#+}} xmm0 = mem[0],zero
-; X86-NO-BMI2-NO-SHLD-NEXT: movq {{.*#+}} xmm0 = xmm0[0],zero
-; X86-NO-BMI2-NO-SHLD-NEXT: shll $3, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: xorps %xmm1, %xmm1
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movdqa %xmm0, (%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrb $3, %dl
-; X86-NO-BMI2-NO-SHLD-NEXT: andb $12, %dl
-; X86-NO-BMI2-NO-SHLD-NEXT: movzbl %dl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl (%esp,%edx), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 4(%esp,%edx), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: notb %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %edx, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, (%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: addl $40, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: retl
-;
-; X86-SHLD-LABEL: load_4byte_chunk_of_16byte_alloca_with_zero_upper_half:
-; X86-SHLD: # %bb.0:
-; X86-SHLD-NEXT: pushl %esi
-; X86-SHLD-NEXT: subl $40, %esp
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-SHLD-NEXT: movq {{.*#+}} xmm0 = mem[0],zero
-; X86-SHLD-NEXT: movq {{.*#+}} xmm0 = xmm0[0],zero
-; X86-SHLD-NEXT: shll $3, %ecx
-; X86-SHLD-NEXT: xorps %xmm1, %xmm1
-; X86-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movdqa %xmm0, (%esp)
-; X86-SHLD-NEXT: movl %ecx, %edx
-; X86-SHLD-NEXT: shrb $3, %dl
-; X86-SHLD-NEXT: andb $12, %dl
-; X86-SHLD-NEXT: movzbl %dl, %edx
-; X86-SHLD-NEXT: movl (%esp,%edx), %esi
-; X86-SHLD-NEXT: movl 4(%esp,%edx), %edx
-; X86-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-SHLD-NEXT: shrdl %cl, %edx, %esi
-; X86-SHLD-NEXT: movl %esi, (%eax)
-; X86-SHLD-NEXT: addl $40, %esp
-; X86-SHLD-NEXT: popl %esi
-; X86-SHLD-NEXT: retl
+; X64-LABEL: load_4byte_chunk_of_16byte_alloca_with_zero_upper_half:
+; X64: # %bb.0:
+; X64-NEXT: movq %rsi, %rcx
+; X64-NEXT: movq (%rdi), %rax
+; X64-NEXT: shll $3, %ecx
+; X64-NEXT: xorl %esi, %esi
+; X64-NEXT: shrdq %cl, %rsi, %rax
+; X64-NEXT: testb $64, %cl
+; X64-NEXT: cmovneq %rsi, %rax
+; X64-NEXT: movl %eax, (%rdx)
+; X64-NEXT: retq
;
-; X86-HAVE-BMI2-NO-SHLD-LABEL: load_4byte_chunk_of_16byte_alloca_with_zero_upper_half:
-; X86-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: subl $40, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movq {{.*#+}} xmm0 = mem[0],zero
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movq {{.*#+}} xmm0 = xmm0[0],zero
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shll $3, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: xorps %xmm1, %xmm1
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movdqa %xmm0, (%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrb $3, %dl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: andb $12, %dl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movzbl %dl, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, (%esp,%edx), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: notb %cl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 4(%esp,%edx), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %edx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %edx, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %esi, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, (%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl $40, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: retl
+; X86-LABEL: load_4byte_chunk_of_16byte_alloca_with_zero_upper_half:
+; X86: # %bb.0:
+; X86-NEXT: pushl %esi
+; X86-NEXT: subl $40, %esp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movq {{.*#+}} xmm0 = mem[0],zero
+; X86-NEXT: movq {{.*#+}} xmm0 = xmm0[0],zero
+; X86-NEXT: shll $3, %ecx
+; X86-NEXT: xorps %xmm1, %xmm1
+; X86-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NEXT: movdqa %xmm0, (%esp)
+; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: shrb $3, %dl
+; X86-NEXT: andb $12, %dl
+; X86-NEXT: movzbl %dl, %edx
+; X86-NEXT: movl (%esp,%edx), %esi
+; X86-NEXT: movl 4(%esp,%edx), %edx
+; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: shrdl %cl, %edx, %esi
+; X86-NEXT: movl %esi, (%eax)
+; X86-NEXT: addl $40, %esp
+; X86-NEXT: popl %esi
+; X86-NEXT: retl
%init = load <8 x i8>, ptr %src, align 1
%intermediate.sroa.0.0.vec.expand = shufflevector <8 x i8> %init, <8 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
%intermediate.sroa.0.0.vecblend = shufflevector <16 x i8> %intermediate.sroa.0.0.vec.expand, <16 x i8> <i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
@@ -792,162 +459,51 @@ define void @load_4byte_chunk_of_16byte_alloca_with_zero_upper_half(ptr %src, i6
}
define void @load_8byte_chunk_of_16byte_alloca_with_zero_upper_half(ptr %src, i64 %byteOff, ptr %dst) nounwind {
-; X64-NO-BMI2-NO-SHLD-LABEL: load_8byte_chunk_of_16byte_alloca_with_zero_upper_half:
-; X64-NO-BMI2-NO-SHLD: # %bb.0:
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rsi, %rcx
-; X64-NO-BMI2-NO-SHLD-NEXT: movq (%rdi), %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: shll $3, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: xorl %esi, %esi
-; X64-NO-BMI2-NO-SHLD-NEXT: testb $64, %cl
-; X64-NO-BMI2-NO-SHLD-NEXT: cmoveq %rax, %rsi
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rsi, (%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: retq
-;
-; X64-SHLD-LABEL: load_8byte_chunk_of_16byte_alloca_with_zero_upper_half:
-; X64-SHLD: # %bb.0:
-; X64-SHLD-NEXT: movq %rsi, %rcx
-; X64-SHLD-NEXT: movq (%rdi), %rax
-; X64-SHLD-NEXT: shll $3, %ecx
-; X64-SHLD-NEXT: xorl %esi, %esi
-; X64-SHLD-NEXT: shrdq %cl, %rsi, %rax
-; X64-SHLD-NEXT: testb $64, %cl
-; X64-SHLD-NEXT: cmovneq %rsi, %rax
-; X64-SHLD-NEXT: movq %rax, (%rdx)
-; X64-SHLD-NEXT: retq
-;
-; X64-HAVE-BMI2-NO-SHLD-LABEL: load_8byte_chunk_of_16byte_alloca_with_zero_upper_half:
-; X64-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shll $3, %esi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rsi, (%rdi), %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: xorl %ecx, %ecx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: testb $64, %sil
-; X64-HAVE-BMI2-NO-SHLD-NEXT: cmoveq %rax, %rcx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rcx, (%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: retq
-;
-; X86-NO-BMI2-NO-SHLD-LABEL: load_8byte_chunk_of_16byte_alloca_with_zero_upper_half:
-; X86-NO-BMI2-NO-SHLD: # %bb.0:
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: subl $44, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movq {{.*#+}} xmm0 = mem[0],zero
-; X86-NO-BMI2-NO-SHLD-NEXT: movq {{.*#+}} xmm0 = xmm0[0],zero
-; X86-NO-BMI2-NO-SHLD-NEXT: shll $3, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: xorps %xmm1, %xmm1
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movdqa %xmm0, (%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrb $3, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: andb $12, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: movzbl %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl (%esp,%ebx), %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 4(%esp,%ebx), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: notb %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%esi,%esi), %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %ebp, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: andb $24, %al
-; X86-NO-BMI2-NO-SHLD-NEXT: notb %al
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 8(%esp,%ebx), %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %ebx, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, 4(%edx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, (%edx)
-; X86-NO-BMI2-NO-SHLD-NEXT: addl $44, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: retl
-;
-; X86-SHLD-LABEL: load_8byte_chunk_of_16byte_alloca_with_zero_upper_half:
-; X86-SHLD: # %bb.0:
-; X86-SHLD-NEXT: pushl %ebx
-; X86-SHLD-NEXT: pushl %edi
-; X86-SHLD-NEXT: pushl %esi
-; X86-SHLD-NEXT: subl $32, %esp
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-SHLD-NEXT: movq {{.*#+}} xmm0 = mem[0],zero
-; X86-SHLD-NEXT: movq {{.*#+}} xmm0 = xmm0[0],zero
-; X86-SHLD-NEXT: shll $3, %ecx
-; X86-SHLD-NEXT: xorps %xmm1, %xmm1
-; X86-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movdqa %xmm0, (%esp)
-; X86-SHLD-NEXT: movl %ecx, %edx
-; X86-SHLD-NEXT: shrb $3, %dl
-; X86-SHLD-NEXT: andb $12, %dl
-; X86-SHLD-NEXT: movzbl %dl, %edx
-; X86-SHLD-NEXT: movl 8(%esp,%edx), %esi
-; X86-SHLD-NEXT: movl (%esp,%edx), %edi
-; X86-SHLD-NEXT: movl 4(%esp,%edx), %edx
-; X86-SHLD-NEXT: movl %edx, %ebx
-; X86-SHLD-NEXT: shrdl %cl, %esi, %ebx
-; X86-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-SHLD-NEXT: shrdl %cl, %edx, %edi
-; X86-SHLD-NEXT: movl %ebx, 4(%eax)
-; X86-SHLD-NEXT: movl %edi, (%eax)
-; X86-SHLD-NEXT: addl $32, %esp
-; X86-SHLD-NEXT: popl %esi
-; X86-SHLD-NEXT: popl %edi
-; X86-SHLD-NEXT: popl %ebx
-; X86-SHLD-NEXT: retl
+; X64-LABEL: load_8byte_chunk_of_16byte_alloca_with_zero_upper_half:
+; X64: # %bb.0:
+; X64-NEXT: movq %rsi, %rcx
+; X64-NEXT: movq (%rdi), %rax
+; X64-NEXT: shll $3, %ecx
+; X64-NEXT: xorl %esi, %esi
+; X64-NEXT: shrdq %cl, %rsi, %rax
+; X64-NEXT: testb $64, %cl
+; X64-NEXT: cmovneq %rsi, %rax
+; X64-NEXT: movq %rax, (%rdx)
+; X64-NEXT: retq
;
-; X86-HAVE-BMI2-NO-SHLD-LABEL: load_8byte_chunk_of_16byte_alloca_with_zero_upper_half:
-; X86-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: subl $44, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movq {{.*#+}} xmm0 = mem[0],zero
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movq {{.*#+}} xmm0 = xmm0[0],zero
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shll $3, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: xorps %xmm1, %xmm1
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movdqa %xmm0, (%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrb $3, %dl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: andb $12, %dl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movzbl %dl, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, (%esp,%edx), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: notb %bl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 4(%esp,%edx), %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 8(%esp,%edx), %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%edi,%edi), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ebx, %edx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %esi, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, %edi, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: andb $24, %cl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: notb %cl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %ebp, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %ebp, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %esi, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, 4(%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edx, (%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl $44, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: retl
+; X86-LABEL: load_8byte_chunk_of_16byte_alloca_with_zero_upper_half:
+; X86: # %bb.0:
+; X86-NEXT: pushl %ebx
+; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %esi
+; X86-NEXT: subl $32, %esp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movq {{.*#+}} xmm0 = mem[0],zero
+; X86-NEXT: movq {{.*#+}} xmm0 = xmm0[0],zero
+; X86-NEXT: shll $3, %ecx
+; X86-NEXT: xorps %xmm1, %xmm1
+; X86-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NEXT: movdqa %xmm0, (%esp)
+; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: shrb $3, %dl
+; X86-NEXT: andb $12, %dl
+; X86-NEXT: movzbl %dl, %edx
+; X86-NEXT: movl 8(%esp,%edx), %esi
+; X86-NEXT: movl (%esp,%edx), %edi
+; X86-NEXT: movl 4(%esp,%edx), %edx
+; X86-NEXT: movl %edx, %ebx
+; X86-NEXT: shrdl %cl, %esi, %ebx
+; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: shrdl %cl, %edx, %edi
+; X86-NEXT: movl %ebx, 4(%eax)
+; X86-NEXT: movl %edi, (%eax)
+; X86-NEXT: addl $32, %esp
+; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
+; X86-NEXT: popl %ebx
+; X86-NEXT: retl
%init = load <8 x i8>, ptr %src, align 1
%intermediate.sroa.0.0.vec.expand = shufflevector <8 x i8> %init, <8 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
%intermediate.sroa.0.0.vecblend = shufflevector <16 x i8> %intermediate.sroa.0.0.vec.expand, <16 x i8> <i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
@@ -979,89 +535,31 @@ define void @load_1byte_chunk_of_32byte_alloca_with_zero_upper_half(ptr %src, i6
; X64-NEXT: movb %al, (%rdx)
; X64-NEXT: retq
;
-; X86-NO-BMI2-NO-SHLD-LABEL: load_1byte_chunk_of_32byte_alloca_with_zero_upper_half:
-; X86-NO-BMI2-NO-SHLD: # %bb.0:
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: subl $72, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movups (%edx), %xmm0
-; X86-NO-BMI2-NO-SHLD-NEXT: xorps %xmm1, %xmm1
-; X86-NO-BMI2-NO-SHLD-NEXT: shll $3, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrb $5, %dl
-; X86-NO-BMI2-NO-SHLD-NEXT: movzbl %dl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl (%esp,%edx,4), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 4(%esp,%edx,4), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: notb %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %edx, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dl, (%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: addl $72, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: retl
-;
-; X86-SHLD-LABEL: load_1byte_chunk_of_32byte_alloca_with_zero_upper_half:
-; X86-SHLD: # %bb.0:
-; X86-SHLD-NEXT: pushl %ebx
-; X86-SHLD-NEXT: subl $72, %esp
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-SHLD-NEXT: movups (%edx), %xmm0
-; X86-SHLD-NEXT: xorps %xmm1, %xmm1
-; X86-SHLD-NEXT: shll $3, %ecx
-; X86-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-SHLD-NEXT: movl %ecx, %edx
-; X86-SHLD-NEXT: shrb $5, %dl
-; X86-SHLD-NEXT: movzbl %dl, %edx
-; X86-SHLD-NEXT: movl (%esp,%edx,4), %ebx
-; X86-SHLD-NEXT: movl 4(%esp,%edx,4), %edx
-; X86-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-SHLD-NEXT: shrdl %cl, %edx, %ebx
-; X86-SHLD-NEXT: movb %bl, (%eax)
-; X86-SHLD-NEXT: addl $72, %esp
-; X86-SHLD-NEXT: popl %ebx
-; X86-SHLD-NEXT: retl
-;
-; X86-HAVE-BMI2-NO-SHLD-LABEL: load_1byte_chunk_of_32byte_alloca_with_zero_upper_half:
-; X86-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: subl $72, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movups (%edx), %xmm0
-; X86-HAVE-BMI2-NO-SHLD-NEXT: xorps %xmm1, %xmm1
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shll $3, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrb $5, %dl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movzbl %dl, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, (%esp,%edx,4), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: notb %cl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 4(%esp,%edx,4), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %edx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %edx, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %esi, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movb %cl, (%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl $72, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: retl
+; X86-LABEL: load_1byte_chunk_of_32byte_alloca_with_zero_upper_half:
+; X86: # %bb.0:
+; X86-NEXT: pushl %ebx
+; X86-NEXT: subl $72, %esp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movups (%edx), %xmm0
+; X86-NEXT: xorps %xmm1, %xmm1
+; X86-NEXT: shll $3, %ecx
+; X86-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm0, (%esp)
+; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: shrb $5, %dl
+; X86-NEXT: movzbl %dl, %edx
+; X86-NEXT: movl (%esp,%edx,4), %ebx
+; X86-NEXT: movl 4(%esp,%edx,4), %edx
+; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: shrdl %cl, %edx, %ebx
+; X86-NEXT: movb %bl, (%eax)
+; X86-NEXT: addl $72, %esp
+; X86-NEXT: popl %ebx
+; X86-NEXT: retl
%init = load <16 x i8>, ptr %src, align 1
%intermediate.sroa.0.0.vec.expand = shufflevector <16 x i8> %init, <16 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
%intermediate.sroa.0.0.vecblend = shufflevector <32 x i8> %intermediate.sroa.0.0.vec.expand, <32 x i8> <i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
@@ -1124,89 +622,31 @@ define void @load_2byte_chunk_of_32byte_alloca_with_zero_upper_half(ptr %src, i6
; X64-BMI2-NEXT: movw %cx, (%rdx)
; X64-BMI2-NEXT: retq
;
-; X86-NO-BMI2-NO-SHLD-LABEL: load_2byte_chunk_of_32byte_alloca_with_zero_upper_half:
-; X86-NO-BMI2-NO-SHLD: # %bb.0:
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: subl $72, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movups (%edx), %xmm0
-; X86-NO-BMI2-NO-SHLD-NEXT: xorps %xmm1, %xmm1
-; X86-NO-BMI2-NO-SHLD-NEXT: shll $3, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrb $5, %dl
-; X86-NO-BMI2-NO-SHLD-NEXT: movzbl %dl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl (%esp,%edx,4), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 4(%esp,%edx,4), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: notb %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %edx, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movw %dx, (%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: addl $72, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: retl
-;
-; X86-SHLD-LABEL: load_2byte_chunk_of_32byte_alloca_with_zero_upper_half:
-; X86-SHLD: # %bb.0:
-; X86-SHLD-NEXT: pushl %esi
-; X86-SHLD-NEXT: subl $72, %esp
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-SHLD-NEXT: movups (%edx), %xmm0
-; X86-SHLD-NEXT: xorps %xmm1, %xmm1
-; X86-SHLD-NEXT: shll $3, %ecx
-; X86-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-SHLD-NEXT: movl %ecx, %edx
-; X86-SHLD-NEXT: shrb $5, %dl
-; X86-SHLD-NEXT: movzbl %dl, %edx
-; X86-SHLD-NEXT: movl (%esp,%edx,4), %esi
-; X86-SHLD-NEXT: movl 4(%esp,%edx,4), %edx
-; X86-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-SHLD-NEXT: shrdl %cl, %edx, %esi
-; X86-SHLD-NEXT: movw %si, (%eax)
-; X86-SHLD-NEXT: addl $72, %esp
-; X86-SHLD-NEXT: popl %esi
-; X86-SHLD-NEXT: retl
-;
-; X86-HAVE-BMI2-NO-SHLD-LABEL: load_2byte_chunk_of_32byte_alloca_with_zero_upper_half:
-; X86-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: subl $72, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movups (%edx), %xmm0
-; X86-HAVE-BMI2-NO-SHLD-NEXT: xorps %xmm1, %xmm1
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shll $3, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrb $5, %dl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movzbl %dl, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, (%esp,%edx,4), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: notb %cl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 4(%esp,%edx,4), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %edx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %edx, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %esi, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movw %cx, (%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl $72, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: retl
+; X86-LABEL: load_2byte_chunk_of_32byte_alloca_with_zero_upper_half:
+; X86: # %bb.0:
+; X86-NEXT: pushl %esi
+; X86-NEXT: subl $72, %esp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movups (%edx), %xmm0
+; X86-NEXT: xorps %xmm1, %xmm1
+; X86-NEXT: shll $3, %ecx
+; X86-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm0, (%esp)
+; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: shrb $5, %dl
+; X86-NEXT: movzbl %dl, %edx
+; X86-NEXT: movl (%esp,%edx,4), %esi
+; X86-NEXT: movl 4(%esp,%edx,4), %edx
+; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: shrdl %cl, %edx, %esi
+; X86-NEXT: movw %si, (%eax)
+; X86-NEXT: addl $72, %esp
+; X86-NEXT: popl %esi
+; X86-NEXT: retl
%init = load <16 x i8>, ptr %src, align 1
%intermediate.sroa.0.0.vec.expand = shufflevector <16 x i8> %init, <16 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
%intermediate.sroa.0.0.vecblend = shufflevector <32 x i8> %intermediate.sroa.0.0.vec.expand, <32 x i8> <i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
@@ -1268,89 +708,31 @@ define void @load_4byte_chunk_of_32byte_alloca_with_zero_upper_half(ptr %src, i6
; X64-BMI2-NEXT: movl %ecx, (%rdx)
; X64-BMI2-NEXT: retq
;
-; X86-NO-BMI2-NO-SHLD-LABEL: load_4byte_chunk_of_32byte_alloca_with_zero_upper_half:
-; X86-NO-BMI2-NO-SHLD: # %bb.0:
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: subl $72, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movups (%edx), %xmm0
-; X86-NO-BMI2-NO-SHLD-NEXT: xorps %xmm1, %xmm1
-; X86-NO-BMI2-NO-SHLD-NEXT: shll $3, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrb $5, %dl
-; X86-NO-BMI2-NO-SHLD-NEXT: movzbl %dl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl (%esp,%edx,4), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 4(%esp,%edx,4), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: notb %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %edx, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, (%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: addl $72, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: retl
-;
-; X86-SHLD-LABEL: load_4byte_chunk_of_32byte_alloca_with_zero_upper_half:
-; X86-SHLD: # %bb.0:
-; X86-SHLD-NEXT: pushl %esi
-; X86-SHLD-NEXT: subl $72, %esp
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-SHLD-NEXT: movups (%edx), %xmm0
-; X86-SHLD-NEXT: xorps %xmm1, %xmm1
-; X86-SHLD-NEXT: shll $3, %ecx
-; X86-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-SHLD-NEXT: movl %ecx, %edx
-; X86-SHLD-NEXT: shrb $5, %dl
-; X86-SHLD-NEXT: movzbl %dl, %edx
-; X86-SHLD-NEXT: movl (%esp,%edx,4), %esi
-; X86-SHLD-NEXT: movl 4(%esp,%edx,4), %edx
-; X86-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-SHLD-NEXT: shrdl %cl, %edx, %esi
-; X86-SHLD-NEXT: movl %esi, (%eax)
-; X86-SHLD-NEXT: addl $72, %esp
-; X86-SHLD-NEXT: popl %esi
-; X86-SHLD-NEXT: retl
-;
-; X86-HAVE-BMI2-NO-SHLD-LABEL: load_4byte_chunk_of_32byte_alloca_with_zero_upper_half:
-; X86-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: subl $72, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movups (%edx), %xmm0
-; X86-HAVE-BMI2-NO-SHLD-NEXT: xorps %xmm1, %xmm1
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shll $3, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrb $5, %dl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movzbl %dl, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, (%esp,%edx,4), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: notb %cl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 4(%esp,%edx,4), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %edx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %edx, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %esi, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, (%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl $72, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: retl
+; X86-LABEL: load_4byte_chunk_of_32byte_alloca_with_zero_upper_half:
+; X86: # %bb.0:
+; X86-NEXT: pushl %esi
+; X86-NEXT: subl $72, %esp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movups (%edx), %xmm0
+; X86-NEXT: xorps %xmm1, %xmm1
+; X86-NEXT: shll $3, %ecx
+; X86-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm0, (%esp)
+; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: shrb $5, %dl
+; X86-NEXT: movzbl %dl, %edx
+; X86-NEXT: movl (%esp,%edx,4), %esi
+; X86-NEXT: movl 4(%esp,%edx,4), %edx
+; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: shrdl %cl, %edx, %esi
+; X86-NEXT: movl %esi, (%eax)
+; X86-NEXT: addl $72, %esp
+; X86-NEXT: popl %esi
+; X86-NEXT: retl
%init = load <16 x i8>, ptr %src, align 1
%intermediate.sroa.0.0.vec.expand = shufflevector <16 x i8> %init, <16 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
%intermediate.sroa.0.0.vecblend = shufflevector <32 x i8> %intermediate.sroa.0.0.vec.expand, <32 x i8> <i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
@@ -1365,191 +747,58 @@ define void @load_4byte_chunk_of_32byte_alloca_with_zero_upper_half(ptr %src, i6
}
define void @load_8byte_chunk_of_32byte_alloca_with_zero_upper_half(ptr %src, i64 %byteOff, ptr %dst) nounwind {
-; X64-NO-BMI2-NO-SHLD-LABEL: load_8byte_chunk_of_32byte_alloca_with_zero_upper_half:
-; X64-NO-BMI2-NO-SHLD: # %bb.0:
-; X64-NO-BMI2-NO-SHLD-NEXT: movups (%rdi), %xmm0
-; X64-NO-BMI2-NO-SHLD-NEXT: xorps %xmm1, %xmm1
-; X64-NO-BMI2-NO-SHLD-NEXT: leal (,%rsi,8), %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %ecx, %eax
-; X64-NO-BMI2-NO-SHLD-NEXT: shrb $6, %al
-; X64-NO-BMI2-NO-SHLD-NEXT: movzbl %al, %eax
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -72(%rsp,%rax,8), %rsi
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -64(%rsp,%rax,8), %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %rsi
-; X64-NO-BMI2-NO-SHLD-NEXT: notb %cl
-; X64-NO-BMI2-NO-SHLD-NEXT: addq %rax, %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %rsi, %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rax, (%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: retq
-;
-; X64-SHLD-LABEL: load_8byte_chunk_of_32byte_alloca_with_zero_upper_half:
-; X64-SHLD: # %bb.0:
-; X64-SHLD-NEXT: movups (%rdi), %xmm0
-; X64-SHLD-NEXT: xorps %xmm1, %xmm1
-; X64-SHLD-NEXT: leal (,%rsi,8), %ecx
-; X64-SHLD-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; X64-SHLD-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; X64-SHLD-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; X64-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-SHLD-NEXT: movl %ecx, %eax
-; X64-SHLD-NEXT: shrb $6, %al
-; X64-SHLD-NEXT: movzbl %al, %eax
-; X64-SHLD-NEXT: movq -72(%rsp,%rax,8), %rsi
-; X64-SHLD-NEXT: movq -64(%rsp,%rax,8), %rax
-; X64-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X64-SHLD-NEXT: shrdq %cl, %rax, %rsi
-; X64-SHLD-NEXT: movq %rsi, (%rdx)
-; X64-SHLD-NEXT: retq
-;
-; X64-HAVE-BMI2-NO-SHLD-LABEL: load_8byte_chunk_of_32byte_alloca_with_zero_upper_half:
-; X64-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movups (%rdi), %xmm0
-; X64-HAVE-BMI2-NO-SHLD-NEXT: xorps %xmm1, %xmm1
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shll $3, %esi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, %eax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrb $6, %al
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movzbl %al, %eax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rsi, -72(%rsp,%rax,8), %rcx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: notb %sil
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq -64(%rsp,%rax,8), %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: addq %rax, %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rsi, %rax, %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %rcx, %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rax, (%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: retq
-;
-; X86-NO-BMI2-NO-SHLD-LABEL: load_8byte_chunk_of_32byte_alloca_with_zero_upper_half:
-; X86-NO-BMI2-NO-SHLD: # %bb.0:
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: subl $76, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movups (%ecx), %xmm0
-; X86-NO-BMI2-NO-SHLD-NEXT: xorps %xmm1, %xmm1
-; X86-NO-BMI2-NO-SHLD-NEXT: shll $3, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrb $5, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: movzbl %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl (%esp,%ebx,4), %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 4(%esp,%ebx,4), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: notb %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%esi,%esi), %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %ebp, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: andb $24, %al
-; X86-NO-BMI2-NO-SHLD-NEXT: notb %al
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 8(%esp,%ebx,4), %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %ebx, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, 4(%edx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, (%edx)
-; X86-NO-BMI2-NO-SHLD-NEXT: addl $76, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: retl
-;
-; X86-SHLD-LABEL: load_8byte_chunk_of_32byte_alloca_with_zero_upper_half:
-; X86-SHLD: # %bb.0:
-; X86-SHLD-NEXT: pushl %ebx
-; X86-SHLD-NEXT: pushl %edi
-; X86-SHLD-NEXT: pushl %esi
-; X86-SHLD-NEXT: subl $64, %esp
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-SHLD-NEXT: movups (%edx), %xmm0
-; X86-SHLD-NEXT: xorps %xmm1, %xmm1
-; X86-SHLD-NEXT: shll $3, %ecx
-; X86-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-SHLD-NEXT: movl %ecx, %edx
-; X86-SHLD-NEXT: shrb $5, %dl
-; X86-SHLD-NEXT: movzbl %dl, %edx
-; X86-SHLD-NEXT: movl 8(%esp,%edx,4), %esi
-; X86-SHLD-NEXT: movl (%esp,%edx,4), %edi
-; X86-SHLD-NEXT: movl 4(%esp,%edx,4), %edx
-; X86-SHLD-NEXT: movl %edx, %ebx
-; X86-SHLD-NEXT: shrdl %cl, %esi, %ebx
-; X86-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-SHLD-NEXT: shrdl %cl, %edx, %edi
-; X86-SHLD-NEXT: movl %ebx, 4(%eax)
-; X86-SHLD-NEXT: movl %edi, (%eax)
-; X86-SHLD-NEXT: addl $64, %esp
-; X86-SHLD-NEXT: popl %esi
-; X86-SHLD-NEXT: popl %edi
-; X86-SHLD-NEXT: popl %ebx
-; X86-SHLD-NEXT: retl
+; X64-LABEL: load_8byte_chunk_of_32byte_alloca_with_zero_upper_half:
+; X64: # %bb.0:
+; X64-NEXT: movups (%rdi), %xmm0
+; X64-NEXT: xorps %xmm1, %xmm1
+; X64-NEXT: leal (,%rsi,8), %ecx
+; X64-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NEXT: movl %ecx, %eax
+; X64-NEXT: shrb $6, %al
+; X64-NEXT: movzbl %al, %eax
+; X64-NEXT: movq -72(%rsp,%rax,8), %rsi
+; X64-NEXT: movq -64(%rsp,%rax,8), %rax
+; X64-NEXT: # kill: def $cl killed $cl killed $ecx
+; X64-NEXT: shrdq %cl, %rax, %rsi
+; X64-NEXT: movq %rsi, (%rdx)
+; X64-NEXT: retq
;
-; X86-HAVE-BMI2-NO-SHLD-LABEL: load_8byte_chunk_of_32byte_alloca_with_zero_upper_half:
-; X86-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: subl $76, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movups (%edx), %xmm0
-; X86-HAVE-BMI2-NO-SHLD-NEXT: xorps %xmm1, %xmm1
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shll $3, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrb $5, %dl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movzbl %dl, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, (%esp,%edx,4), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: notb %bl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 4(%esp,%edx,4), %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 8(%esp,%edx,4), %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%edi,%edi), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ebx, %edx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %esi, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, %edi, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: andb $24, %cl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: notb %cl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %ebp, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %ebp, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %esi, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, 4(%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edx, (%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl $76, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: retl
+; X86-LABEL: load_8byte_chunk_of_32byte_alloca_with_zero_upper_half:
+; X86: # %bb.0:
+; X86-NEXT: pushl %ebx
+; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %esi
+; X86-NEXT: subl $64, %esp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movups (%edx), %xmm0
+; X86-NEXT: xorps %xmm1, %xmm1
+; X86-NEXT: shll $3, %ecx
+; X86-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm0, (%esp)
+; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: shrb $5, %dl
+; X86-NEXT: movzbl %dl, %edx
+; X86-NEXT: movl 8(%esp,%edx,4), %esi
+; X86-NEXT: movl (%esp,%edx,4), %edi
+; X86-NEXT: movl 4(%esp,%edx,4), %edx
+; X86-NEXT: movl %edx, %ebx
+; X86-NEXT: shrdl %cl, %esi, %ebx
+; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: shrdl %cl, %edx, %edi
+; X86-NEXT: movl %ebx, 4(%eax)
+; X86-NEXT: movl %edi, (%eax)
+; X86-NEXT: addl $64, %esp
+; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
+; X86-NEXT: popl %ebx
+; X86-NEXT: retl
%init = load <16 x i8>, ptr %src, align 1
%intermediate.sroa.0.0.vec.expand = shufflevector <16 x i8> %init, <16 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
%intermediate.sroa.0.0.vecblend = shufflevector <32 x i8> %intermediate.sroa.0.0.vec.expand, <32 x i8> <i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
@@ -1564,292 +813,104 @@ define void @load_8byte_chunk_of_32byte_alloca_with_zero_upper_half(ptr %src, i6
}
define void @load_16byte_chunk_of_32byte_alloca_with_zero_upper_half(ptr %src, i64 %byteOff, ptr %dst) nounwind {
-; X64-NO-BMI2-NO-SHLD-LABEL: load_16byte_chunk_of_32byte_alloca_with_zero_upper_half:
-; X64-NO-BMI2-NO-SHLD: # %bb.0:
-; X64-NO-BMI2-NO-SHLD-NEXT: movups (%rdi), %xmm0
-; X64-NO-BMI2-NO-SHLD-NEXT: xorps %xmm1, %xmm1
-; X64-NO-BMI2-NO-SHLD-NEXT: leal (,%rsi,8), %eax
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrb $6, %cl
-; X64-NO-BMI2-NO-SHLD-NEXT: movzbl %cl, %edi
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -72(%rsp,%rdi,8), %r8
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -64(%rsp,%rdi,8), %r9
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %r8
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %esi
-; X64-NO-BMI2-NO-SHLD-NEXT: notb %sil
-; X64-NO-BMI2-NO-SHLD-NEXT: leaq (%r9,%r9), %r10
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %esi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %r10
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %r8, %r10
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %r9
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -56(%rsp,%rdi,8), %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: addq %rax, %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %esi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %r9, %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rax, 8(%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r10, (%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: retq
-;
-; X64-NO-BMI2-HAVE-SHLD-LABEL: load_16byte_chunk_of_32byte_alloca_with_zero_upper_half:
-; X64-NO-BMI2-HAVE-SHLD: # %bb.0:
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movups (%rdi), %xmm0
-; X64-NO-BMI2-HAVE-SHLD-NEXT: xorps %xmm1, %xmm1
-; X64-NO-BMI2-HAVE-SHLD-NEXT: leal (,%rsi,8), %eax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrb $6, %cl
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movzbl %cl, %esi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -72(%rsp,%rsi,8), %rdi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -64(%rsp,%rsi,8), %r8
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r8, %r9
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrq %cl, %r9
-; X64-NO-BMI2-HAVE-SHLD-NEXT: notb %cl
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -56(%rsp,%rsi,8), %rsi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: addq %rsi, %rsi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shlq %cl, %rsi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: orq %r9, %rsi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %r8, %rdi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rdi, (%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rsi, 8(%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: retq
-;
-; X64-HAVE-BMI2-NO-SHLD-LABEL: load_16byte_chunk_of_32byte_alloca_with_zero_upper_half:
-; X64-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movups (%rdi), %xmm0
-; X64-HAVE-BMI2-NO-SHLD-NEXT: xorps %xmm1, %xmm1
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shll $3, %esi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, %eax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, %ecx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrb $6, %cl
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movzbl %cl, %ecx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rax, -72(%rsp,%rcx,8), %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: notb %sil
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq -64(%rsp,%rcx,8), %r8
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq -56(%rsp,%rcx,8), %rcx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: leaq (%r8,%r8), %r9
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rsi, %r9, %r9
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %rdi, %r9
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rax, %r8, %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: addq %rcx, %rcx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rsi, %rcx, %rcx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %rax, %rcx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rcx, 8(%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %r9, (%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: retq
-;
-; X64-HAVE-BMI2-HAVE-SHLD-LABEL: load_16byte_chunk_of_32byte_alloca_with_zero_upper_half:
-; X64-HAVE-BMI2-HAVE-SHLD: # %bb.0:
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rsi, %rcx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movups (%rdi), %xmm0
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: xorps %xmm1, %xmm1
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shll $3, %ecx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, %eax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrb $6, %al
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movzbl %al, %eax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -72(%rsp,%rax,8), %rsi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -64(%rsp,%rax,8), %rdi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrxq %rcx, %rdi, %r8
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, %r9d
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: notb %r9b
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -56(%rsp,%rax,8), %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: addq %rax, %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shlxq %r9, %rax, %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: orq %r8, %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: # kill: def $cl killed $cl killed $rcx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %rdi, %rsi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rsi, (%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rax, 8(%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: retq
-;
-; X86-NO-BMI2-NO-SHLD-LABEL: load_16byte_chunk_of_32byte_alloca_with_zero_upper_half:
-; X86-NO-BMI2-NO-SHLD: # %bb.0:
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: subl $92, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movups (%ecx), %xmm0
-; X86-NO-BMI2-NO-SHLD-NEXT: xorps %xmm1, %xmm1
-; X86-NO-BMI2-NO-SHLD-NEXT: shll $3, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrb $5, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: movzbl %cl, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 16(%esp,%edi,4), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 20(%esp,%edi,4), %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: notb %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%ebp,%ebp), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %edx, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %al, %ch
-; X86-NO-BMI2-NO-SHLD-NEXT: andb $24, %ch
-; X86-NO-BMI2-NO-SHLD-NEXT: xorb $31, %ch
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 24(%esp,%edi,4), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%edx,%edx), %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %ebp, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %al, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 28(%esp,%edi,4), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%esi,%esi), %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %edx, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %al, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 32(%esp,%edi,4), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %eax, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, 12(%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebp, 8(%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, 4(%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, (%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: addl $92, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: retl
+; X64-NO-BMI2-LABEL: load_16byte_chunk_of_32byte_alloca_with_zero_upper_half:
+; X64-NO-BMI2: # %bb.0:
+; X64-NO-BMI2-NEXT: movups (%rdi), %xmm0
+; X64-NO-BMI2-NEXT: xorps %xmm1, %xmm1
+; X64-NO-BMI2-NEXT: leal (,%rsi,8), %eax
+; X64-NO-BMI2-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movl %eax, %ecx
+; X64-NO-BMI2-NEXT: shrb $6, %cl
+; X64-NO-BMI2-NEXT: movzbl %cl, %esi
+; X64-NO-BMI2-NEXT: movq -72(%rsp,%rsi,8), %rdi
+; X64-NO-BMI2-NEXT: movq -64(%rsp,%rsi,8), %r8
+; X64-NO-BMI2-NEXT: movq %r8, %r9
+; X64-NO-BMI2-NEXT: movl %eax, %ecx
+; X64-NO-BMI2-NEXT: shrq %cl, %r9
+; X64-NO-BMI2-NEXT: notb %cl
+; X64-NO-BMI2-NEXT: movq -56(%rsp,%rsi,8), %rsi
+; X64-NO-BMI2-NEXT: addq %rsi, %rsi
+; X64-NO-BMI2-NEXT: shlq %cl, %rsi
+; X64-NO-BMI2-NEXT: orq %r9, %rsi
+; X64-NO-BMI2-NEXT: movl %eax, %ecx
+; X64-NO-BMI2-NEXT: shrdq %cl, %r8, %rdi
+; X64-NO-BMI2-NEXT: movq %rdi, (%rdx)
+; X64-NO-BMI2-NEXT: movq %rsi, 8(%rdx)
+; X64-NO-BMI2-NEXT: retq
;
-; X86-SHLD-LABEL: load_16byte_chunk_of_32byte_alloca_with_zero_upper_half:
-; X86-SHLD: # %bb.0:
-; X86-SHLD-NEXT: pushl %ebp
-; X86-SHLD-NEXT: pushl %ebx
-; X86-SHLD-NEXT: pushl %edi
-; X86-SHLD-NEXT: pushl %esi
-; X86-SHLD-NEXT: subl $92, %esp
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-SHLD-NEXT: movups (%eax), %xmm0
-; X86-SHLD-NEXT: xorps %xmm1, %xmm1
-; X86-SHLD-NEXT: shll $3, %ecx
-; X86-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movl %ecx, %eax
-; X86-SHLD-NEXT: shrb $5, %al
-; X86-SHLD-NEXT: movzbl %al, %ebx
-; X86-SHLD-NEXT: movl 24(%esp,%ebx,4), %esi
-; X86-SHLD-NEXT: movl 16(%esp,%ebx,4), %eax
-; X86-SHLD-NEXT: movl 20(%esp,%ebx,4), %edi
-; X86-SHLD-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-SHLD-NEXT: shrdl %cl, %esi, %edi
-; X86-SHLD-NEXT: movl 28(%esp,%ebx,4), %ebp
-; X86-SHLD-NEXT: shrdl %cl, %ebp, %esi
-; X86-SHLD-NEXT: movl 32(%esp,%ebx,4), %ebx
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-SHLD-NEXT: shrdl %cl, %ebx, %ebp
-; X86-SHLD-NEXT: movl %ebp, 12(%edx)
-; X86-SHLD-NEXT: movl %esi, 8(%edx)
-; X86-SHLD-NEXT: movl %edi, 4(%edx)
-; X86-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-SHLD-NEXT: shrdl %cl, %esi, %eax
-; X86-SHLD-NEXT: movl %eax, (%edx)
-; X86-SHLD-NEXT: addl $92, %esp
-; X86-SHLD-NEXT: popl %esi
-; X86-SHLD-NEXT: popl %edi
-; X86-SHLD-NEXT: popl %ebx
-; X86-SHLD-NEXT: popl %ebp
-; X86-SHLD-NEXT: retl
+; X64-BMI2-LABEL: load_16byte_chunk_of_32byte_alloca_with_zero_upper_half:
+; X64-BMI2: # %bb.0:
+; X64-BMI2-NEXT: movq %rsi, %rcx
+; X64-BMI2-NEXT: movups (%rdi), %xmm0
+; X64-BMI2-NEXT: xorps %xmm1, %xmm1
+; X64-BMI2-NEXT: shll $3, %ecx
+; X64-BMI2-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movl %ecx, %eax
+; X64-BMI2-NEXT: shrb $6, %al
+; X64-BMI2-NEXT: movzbl %al, %eax
+; X64-BMI2-NEXT: movq -72(%rsp,%rax,8), %rsi
+; X64-BMI2-NEXT: movq -64(%rsp,%rax,8), %rdi
+; X64-BMI2-NEXT: shrxq %rcx, %rdi, %r8
+; X64-BMI2-NEXT: movl %ecx, %r9d
+; X64-BMI2-NEXT: notb %r9b
+; X64-BMI2-NEXT: movq -56(%rsp,%rax,8), %rax
+; X64-BMI2-NEXT: addq %rax, %rax
+; X64-BMI2-NEXT: shlxq %r9, %rax, %rax
+; X64-BMI2-NEXT: orq %r8, %rax
+; X64-BMI2-NEXT: # kill: def $cl killed $cl killed $rcx
+; X64-BMI2-NEXT: shrdq %cl, %rdi, %rsi
+; X64-BMI2-NEXT: movq %rsi, (%rdx)
+; X64-BMI2-NEXT: movq %rax, 8(%rdx)
+; X64-BMI2-NEXT: retq
;
-; X86-HAVE-BMI2-NO-SHLD-LABEL: load_16byte_chunk_of_32byte_alloca_with_zero_upper_half:
-; X86-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: subl $92, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movups (%ecx), %xmm0
-; X86-HAVE-BMI2-NO-SHLD-NEXT: xorps %xmm1, %xmm1
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shll $3, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrb $5, %dl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movzbl %dl, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, 16(%esp,%esi,4), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: notb %bl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 20(%esp,%esi,4), %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%edi,%edi), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ebx, %edx, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 24(%esp,%esi,4), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ebp, %edi, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: andb $24, %al
-; X86-HAVE-BMI2-NO-SHLD-NEXT: xorb $31, %al
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%ecx,%ecx), %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %eax, %edi, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %edx, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 28(%esp,%esi,4), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%ecx,%ecx), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ebx, %edx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ebp, {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %ebx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ebp, %ecx, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 32(%esp,%esi,4), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %esi, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %eax, %esi, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %ecx, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, 12(%ecx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edx, 8(%ecx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edi, 4(%ecx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, (%ecx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl $92, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: retl
+; X86-LABEL: load_16byte_chunk_of_32byte_alloca_with_zero_upper_half:
+; X86: # %bb.0:
+; X86-NEXT: pushl %ebp
+; X86-NEXT: pushl %ebx
+; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %esi
+; X86-NEXT: subl $92, %esp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movups (%eax), %xmm0
+; X86-NEXT: xorps %xmm1, %xmm1
+; X86-NEXT: shll $3, %ecx
+; X86-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: shrb $5, %al
+; X86-NEXT: movzbl %al, %ebx
+; X86-NEXT: movl 24(%esp,%ebx,4), %esi
+; X86-NEXT: movl 16(%esp,%ebx,4), %eax
+; X86-NEXT: movl 20(%esp,%ebx,4), %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shrdl %cl, %esi, %edi
+; X86-NEXT: movl 28(%esp,%ebx,4), %ebp
+; X86-NEXT: shrdl %cl, %ebp, %esi
+; X86-NEXT: movl 32(%esp,%ebx,4), %ebx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: shrdl %cl, %ebx, %ebp
+; X86-NEXT: movl %ebp, 12(%edx)
+; X86-NEXT: movl %esi, 8(%edx)
+; X86-NEXT: movl %edi, 4(%edx)
+; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: shrdl %cl, %esi, %eax
+; X86-NEXT: movl %eax, (%edx)
+; X86-NEXT: addl $92, %esp
+; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
+; X86-NEXT: popl %ebx
+; X86-NEXT: popl %ebp
+; X86-NEXT: retl
%init = load <16 x i8>, ptr %src, align 1
%intermediate.sroa.0.0.vec.expand = shufflevector <16 x i8> %init, <16 x i8> poison, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
%intermediate.sroa.0.0.vecblend = shufflevector <32 x i8> %intermediate.sroa.0.0.vec.expand, <32 x i8> <i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
@@ -1921,98 +982,34 @@ define void @load_1byte_chunk_of_64byte_alloca_with_zero_upper_half(ptr %src, i6
; X64-BMI2-NEXT: popq %rax
; X64-BMI2-NEXT: retq
;
-; X86-NO-BMI2-NO-SHLD-LABEL: load_1byte_chunk_of_64byte_alloca_with_zero_upper_half:
-; X86-NO-BMI2-NO-SHLD: # %bb.0:
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: subl $136, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movups (%ecx), %xmm0
-; X86-NO-BMI2-NO-SHLD-NEXT: movups 16(%ecx), %xmm1
-; X86-NO-BMI2-NO-SHLD-NEXT: xorps %xmm2, %xmm2
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (,%edx,8), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: andl $60, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl (%esp,%edx), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 4(%esp,%edx), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: notb %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %edx, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dl, (%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: addl $136, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: retl
-;
-; X86-SHLD-LABEL: load_1byte_chunk_of_64byte_alloca_with_zero_upper_half:
-; X86-SHLD: # %bb.0:
-; X86-SHLD-NEXT: pushl %ebx
-; X86-SHLD-NEXT: subl $136, %esp
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-SHLD-NEXT: movups (%ecx), %xmm0
-; X86-SHLD-NEXT: movups 16(%ecx), %xmm1
-; X86-SHLD-NEXT: xorps %xmm2, %xmm2
-; X86-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-SHLD-NEXT: leal (,%edx,8), %ecx
-; X86-SHLD-NEXT: andl $60, %edx
-; X86-SHLD-NEXT: movl (%esp,%edx), %ebx
-; X86-SHLD-NEXT: movl 4(%esp,%edx), %edx
-; X86-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-SHLD-NEXT: shrdl %cl, %edx, %ebx
-; X86-SHLD-NEXT: movb %bl, (%eax)
-; X86-SHLD-NEXT: addl $136, %esp
-; X86-SHLD-NEXT: popl %ebx
-; X86-SHLD-NEXT: retl
-;
-; X86-HAVE-BMI2-NO-SHLD-LABEL: load_1byte_chunk_of_64byte_alloca_with_zero_upper_half:
-; X86-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: subl $136, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movups (%edx), %xmm0
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movups 16(%edx), %xmm1
-; X86-HAVE-BMI2-NO-SHLD-NEXT: xorps %xmm2, %xmm2
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (,%ecx,8), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: andl $60, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %edx, (%esp,%ecx), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: notb %dl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 4(%esp,%ecx), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %ecx, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %edx, %ecx, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %esi, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movb %cl, (%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl $136, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: retl
+; X86-LABEL: load_1byte_chunk_of_64byte_alloca_with_zero_upper_half:
+; X86: # %bb.0:
+; X86-NEXT: pushl %ebx
+; X86-NEXT: subl $136, %esp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movups (%ecx), %xmm0
+; X86-NEXT: movups 16(%ecx), %xmm1
+; X86-NEXT: xorps %xmm2, %xmm2
+; X86-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm0, (%esp)
+; X86-NEXT: leal (,%edx,8), %ecx
+; X86-NEXT: andl $60, %edx
+; X86-NEXT: movl (%esp,%edx), %ebx
+; X86-NEXT: movl 4(%esp,%edx), %edx
+; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: shrdl %cl, %edx, %ebx
+; X86-NEXT: movb %bl, (%eax)
+; X86-NEXT: addl $136, %esp
+; X86-NEXT: popl %ebx
+; X86-NEXT: retl
%init = load <32 x i8>, ptr %src, align 1
%intermediate.sroa.0.0.vec.expand = shufflevector <32 x i8> %init, <32 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
%intermediate.sroa.0.0.vecblend = shufflevector <64 x i8> %intermediate.sroa.0.0.vec.expand, <64 x i8> <i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
@@ -2085,98 +1082,34 @@ define void @load_2byte_chunk_of_64byte_alloca_with_zero_upper_half(ptr %src, i6
; X64-BMI2-NEXT: popq %rax
; X64-BMI2-NEXT: retq
;
-; X86-NO-BMI2-NO-SHLD-LABEL: load_2byte_chunk_of_64byte_alloca_with_zero_upper_half:
-; X86-NO-BMI2-NO-SHLD: # %bb.0:
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: subl $136, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movups (%ecx), %xmm0
-; X86-NO-BMI2-NO-SHLD-NEXT: movups 16(%ecx), %xmm1
-; X86-NO-BMI2-NO-SHLD-NEXT: xorps %xmm2, %xmm2
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (,%edx,8), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: andl $60, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl (%esp,%edx), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 4(%esp,%edx), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: notb %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %edx, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movw %dx, (%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: addl $136, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: retl
-;
-; X86-SHLD-LABEL: load_2byte_chunk_of_64byte_alloca_with_zero_upper_half:
-; X86-SHLD: # %bb.0:
-; X86-SHLD-NEXT: pushl %esi
-; X86-SHLD-NEXT: subl $136, %esp
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-SHLD-NEXT: movups (%ecx), %xmm0
-; X86-SHLD-NEXT: movups 16(%ecx), %xmm1
-; X86-SHLD-NEXT: xorps %xmm2, %xmm2
-; X86-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-SHLD-NEXT: leal (,%edx,8), %ecx
-; X86-SHLD-NEXT: andl $60, %edx
-; X86-SHLD-NEXT: movl (%esp,%edx), %esi
-; X86-SHLD-NEXT: movl 4(%esp,%edx), %edx
-; X86-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-SHLD-NEXT: shrdl %cl, %edx, %esi
-; X86-SHLD-NEXT: movw %si, (%eax)
-; X86-SHLD-NEXT: addl $136, %esp
-; X86-SHLD-NEXT: popl %esi
-; X86-SHLD-NEXT: retl
-;
-; X86-HAVE-BMI2-NO-SHLD-LABEL: load_2byte_chunk_of_64byte_alloca_with_zero_upper_half:
-; X86-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: subl $136, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movups (%edx), %xmm0
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movups 16(%edx), %xmm1
-; X86-HAVE-BMI2-NO-SHLD-NEXT: xorps %xmm2, %xmm2
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (,%ecx,8), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: andl $60, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %edx, (%esp,%ecx), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: notb %dl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 4(%esp,%ecx), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %ecx, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %edx, %ecx, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %esi, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movw %cx, (%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl $136, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: retl
+; X86-LABEL: load_2byte_chunk_of_64byte_alloca_with_zero_upper_half:
+; X86: # %bb.0:
+; X86-NEXT: pushl %esi
+; X86-NEXT: subl $136, %esp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movups (%ecx), %xmm0
+; X86-NEXT: movups 16(%ecx), %xmm1
+; X86-NEXT: xorps %xmm2, %xmm2
+; X86-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm0, (%esp)
+; X86-NEXT: leal (,%edx,8), %ecx
+; X86-NEXT: andl $60, %edx
+; X86-NEXT: movl (%esp,%edx), %esi
+; X86-NEXT: movl 4(%esp,%edx), %edx
+; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: shrdl %cl, %edx, %esi
+; X86-NEXT: movw %si, (%eax)
+; X86-NEXT: addl $136, %esp
+; X86-NEXT: popl %esi
+; X86-NEXT: retl
%init = load <32 x i8>, ptr %src, align 1
%intermediate.sroa.0.0.vec.expand = shufflevector <32 x i8> %init, <32 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
%intermediate.sroa.0.0.vecblend = shufflevector <64 x i8> %intermediate.sroa.0.0.vec.expand, <64 x i8> <i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
@@ -2248,98 +1181,34 @@ define void @load_4byte_chunk_of_64byte_alloca_with_zero_upper_half(ptr %src, i6
; X64-BMI2-NEXT: popq %rax
; X64-BMI2-NEXT: retq
;
-; X86-NO-BMI2-NO-SHLD-LABEL: load_4byte_chunk_of_64byte_alloca_with_zero_upper_half:
-; X86-NO-BMI2-NO-SHLD: # %bb.0:
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: subl $136, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movups (%ecx), %xmm0
-; X86-NO-BMI2-NO-SHLD-NEXT: movups 16(%ecx), %xmm1
-; X86-NO-BMI2-NO-SHLD-NEXT: xorps %xmm2, %xmm2
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (,%edx,8), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: andl $60, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl (%esp,%edx), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 4(%esp,%edx), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: notb %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %edx, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, (%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: addl $136, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: retl
-;
-; X86-SHLD-LABEL: load_4byte_chunk_of_64byte_alloca_with_zero_upper_half:
-; X86-SHLD: # %bb.0:
-; X86-SHLD-NEXT: pushl %esi
-; X86-SHLD-NEXT: subl $136, %esp
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-SHLD-NEXT: movups (%ecx), %xmm0
-; X86-SHLD-NEXT: movups 16(%ecx), %xmm1
-; X86-SHLD-NEXT: xorps %xmm2, %xmm2
-; X86-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-SHLD-NEXT: leal (,%edx,8), %ecx
-; X86-SHLD-NEXT: andl $60, %edx
-; X86-SHLD-NEXT: movl (%esp,%edx), %esi
-; X86-SHLD-NEXT: movl 4(%esp,%edx), %edx
-; X86-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-SHLD-NEXT: shrdl %cl, %edx, %esi
-; X86-SHLD-NEXT: movl %esi, (%eax)
-; X86-SHLD-NEXT: addl $136, %esp
-; X86-SHLD-NEXT: popl %esi
-; X86-SHLD-NEXT: retl
-;
-; X86-HAVE-BMI2-NO-SHLD-LABEL: load_4byte_chunk_of_64byte_alloca_with_zero_upper_half:
-; X86-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: subl $136, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movups (%edx), %xmm0
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movups 16(%edx), %xmm1
-; X86-HAVE-BMI2-NO-SHLD-NEXT: xorps %xmm2, %xmm2
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (,%ecx,8), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: andl $60, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %edx, (%esp,%ecx), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: notb %dl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 4(%esp,%ecx), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %ecx, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %edx, %ecx, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %esi, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, (%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl $136, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: retl
+; X86-LABEL: load_4byte_chunk_of_64byte_alloca_with_zero_upper_half:
+; X86: # %bb.0:
+; X86-NEXT: pushl %esi
+; X86-NEXT: subl $136, %esp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movups (%ecx), %xmm0
+; X86-NEXT: movups 16(%ecx), %xmm1
+; X86-NEXT: xorps %xmm2, %xmm2
+; X86-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm0, (%esp)
+; X86-NEXT: leal (,%edx,8), %ecx
+; X86-NEXT: andl $60, %edx
+; X86-NEXT: movl (%esp,%edx), %esi
+; X86-NEXT: movl 4(%esp,%edx), %edx
+; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: shrdl %cl, %edx, %esi
+; X86-NEXT: movl %esi, (%eax)
+; X86-NEXT: addl $136, %esp
+; X86-NEXT: popl %esi
+; X86-NEXT: retl
%init = load <32 x i8>, ptr %src, align 1
%intermediate.sroa.0.0.vec.expand = shufflevector <32 x i8> %init, <32 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
%intermediate.sroa.0.0.vecblend = shufflevector <64 x i8> %intermediate.sroa.0.0.vec.expand, <64 x i8> <i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
@@ -2354,218 +1223,68 @@ define void @load_4byte_chunk_of_64byte_alloca_with_zero_upper_half(ptr %src, i6
}
define void @load_8byte_chunk_of_64byte_alloca_with_zero_upper_half(ptr %src, i64 %byteOff, ptr %dst) nounwind {
-; X64-NO-BMI2-NO-SHLD-LABEL: load_8byte_chunk_of_64byte_alloca_with_zero_upper_half:
-; X64-NO-BMI2-NO-SHLD: # %bb.0:
-; X64-NO-BMI2-NO-SHLD-NEXT: pushq %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: movups (%rdi), %xmm0
-; X64-NO-BMI2-NO-SHLD-NEXT: movups 16(%rdi), %xmm1
-; X64-NO-BMI2-NO-SHLD-NEXT: xorps %xmm2, %xmm2
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: leal (,%rsi,8), %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: andl $56, %esi
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -128(%rsp,%rsi), %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -120(%rsp,%rsi), %rsi
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: notb %cl
-; X64-NO-BMI2-NO-SHLD-NEXT: addq %rsi, %rsi
-; X64-NO-BMI2-NO-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %rsi
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %rax, %rsi
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rsi, (%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: popq %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: retq
-;
-; X64-SHLD-LABEL: load_8byte_chunk_of_64byte_alloca_with_zero_upper_half:
-; X64-SHLD: # %bb.0:
-; X64-SHLD-NEXT: pushq %rax
-; X64-SHLD-NEXT: movups (%rdi), %xmm0
-; X64-SHLD-NEXT: movups 16(%rdi), %xmm1
-; X64-SHLD-NEXT: xorps %xmm2, %xmm2
-; X64-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-SHLD-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; X64-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-SHLD-NEXT: leal (,%rsi,8), %ecx
-; X64-SHLD-NEXT: andl $56, %esi
-; X64-SHLD-NEXT: movq -128(%rsp,%rsi), %rax
-; X64-SHLD-NEXT: movq -120(%rsp,%rsi), %rsi
-; X64-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X64-SHLD-NEXT: shrdq %cl, %rsi, %rax
-; X64-SHLD-NEXT: movq %rax, (%rdx)
-; X64-SHLD-NEXT: popq %rax
-; X64-SHLD-NEXT: retq
-;
-; X64-HAVE-BMI2-NO-SHLD-LABEL: load_8byte_chunk_of_64byte_alloca_with_zero_upper_half:
-; X64-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X64-HAVE-BMI2-NO-SHLD-NEXT: pushq %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movups (%rdi), %xmm0
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movups 16(%rdi), %xmm1
-; X64-HAVE-BMI2-NO-SHLD-NEXT: xorps %xmm2, %xmm2
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: leal (,%rsi,8), %eax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: andl $56, %esi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rax, -128(%rsp,%rsi), %rcx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: notb %al
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq -120(%rsp,%rsi), %rsi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: addq %rsi, %rsi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rax, %rsi, %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %rcx, %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rax, (%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: popq %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: retq
-;
-; X86-NO-BMI2-NO-SHLD-LABEL: load_8byte_chunk_of_64byte_alloca_with_zero_upper_half:
-; X86-NO-BMI2-NO-SHLD: # %bb.0:
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: subl $140, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movups (%ecx), %xmm0
-; X86-NO-BMI2-NO-SHLD-NEXT: movups 16(%ecx), %xmm1
-; X86-NO-BMI2-NO-SHLD-NEXT: xorps %xmm2, %xmm2
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: andl $60, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl (%esp,%ebx), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 4(%esp,%ebx), %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: shll $3, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: andl $24, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%edi,%edi), %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: notb %dl
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 8(%esp,%ebx), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %eax, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %edi, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, 4(%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebp, (%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: addl $140, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: retl
-;
-; X86-SHLD-LABEL: load_8byte_chunk_of_64byte_alloca_with_zero_upper_half:
-; X86-SHLD: # %bb.0:
-; X86-SHLD-NEXT: pushl %ebx
-; X86-SHLD-NEXT: pushl %edi
-; X86-SHLD-NEXT: pushl %esi
-; X86-SHLD-NEXT: subl $128, %esp
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-SHLD-NEXT: movups (%edx), %xmm0
-; X86-SHLD-NEXT: movups 16(%edx), %xmm1
-; X86-SHLD-NEXT: xorps %xmm2, %xmm2
-; X86-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-SHLD-NEXT: movl %ecx, %esi
-; X86-SHLD-NEXT: andl $60, %esi
-; X86-SHLD-NEXT: movl 8(%esp,%esi), %edi
-; X86-SHLD-NEXT: movl (%esp,%esi), %edx
-; X86-SHLD-NEXT: movl 4(%esp,%esi), %esi
-; X86-SHLD-NEXT: shll $3, %ecx
-; X86-SHLD-NEXT: andl $24, %ecx
-; X86-SHLD-NEXT: movl %esi, %ebx
-; X86-SHLD-NEXT: shrdl %cl, %edi, %ebx
-; X86-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-SHLD-NEXT: shrdl %cl, %esi, %edx
-; X86-SHLD-NEXT: movl %ebx, 4(%eax)
-; X86-SHLD-NEXT: movl %edx, (%eax)
-; X86-SHLD-NEXT: addl $128, %esp
-; X86-SHLD-NEXT: popl %esi
-; X86-SHLD-NEXT: popl %edi
-; X86-SHLD-NEXT: popl %ebx
-; X86-SHLD-NEXT: retl
+; X64-LABEL: load_8byte_chunk_of_64byte_alloca_with_zero_upper_half:
+; X64: # %bb.0:
+; X64-NEXT: pushq %rax
+; X64-NEXT: movups (%rdi), %xmm0
+; X64-NEXT: movups 16(%rdi), %xmm1
+; X64-NEXT: xorps %xmm2, %xmm2
+; X64-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NEXT: leal (,%rsi,8), %ecx
+; X64-NEXT: andl $56, %esi
+; X64-NEXT: movq -128(%rsp,%rsi), %rax
+; X64-NEXT: movq -120(%rsp,%rsi), %rsi
+; X64-NEXT: # kill: def $cl killed $cl killed $ecx
+; X64-NEXT: shrdq %cl, %rsi, %rax
+; X64-NEXT: movq %rax, (%rdx)
+; X64-NEXT: popq %rax
+; X64-NEXT: retq
;
-; X86-HAVE-BMI2-NO-SHLD-LABEL: load_8byte_chunk_of_64byte_alloca_with_zero_upper_half:
-; X86-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: subl $140, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movups (%edx), %xmm0
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movups 16(%edx), %xmm1
-; X86-HAVE-BMI2-NO-SHLD-NEXT: xorps %xmm2, %xmm2
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (,%ecx,8), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: andl $24, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edx, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: andl $60, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ebx, (%esp,%ecx), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: notb %dl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 4(%esp,%ecx), %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 8(%esp,%ecx), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%edi,%edi), %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %edx, %ebp, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %esi, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ebx, %edi, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %ecx, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %edx, %ecx, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %esi, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, 4(%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ebp, (%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl $140, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: retl
+; X86-LABEL: load_8byte_chunk_of_64byte_alloca_with_zero_upper_half:
+; X86: # %bb.0:
+; X86-NEXT: pushl %ebx
+; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %esi
+; X86-NEXT: subl $128, %esp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movups (%edx), %xmm0
+; X86-NEXT: movups 16(%edx), %xmm1
+; X86-NEXT: xorps %xmm2, %xmm2
+; X86-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm0, (%esp)
+; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: andl $60, %esi
+; X86-NEXT: movl 8(%esp,%esi), %edi
+; X86-NEXT: movl (%esp,%esi), %edx
+; X86-NEXT: movl 4(%esp,%esi), %esi
+; X86-NEXT: shll $3, %ecx
+; X86-NEXT: andl $24, %ecx
+; X86-NEXT: movl %esi, %ebx
+; X86-NEXT: shrdl %cl, %edi, %ebx
+; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: shrdl %cl, %esi, %edx
+; X86-NEXT: movl %ebx, 4(%eax)
+; X86-NEXT: movl %edx, (%eax)
+; X86-NEXT: addl $128, %esp
+; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
+; X86-NEXT: popl %ebx
+; X86-NEXT: retl
%init = load <32 x i8>, ptr %src, align 1
%intermediate.sroa.0.0.vec.expand = shufflevector <32 x i8> %init, <32 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
%intermediate.sroa.0.0.vecblend = shufflevector <64 x i8> %intermediate.sroa.0.0.vec.expand, <64 x i8> <i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
@@ -2580,326 +1299,122 @@ define void @load_8byte_chunk_of_64byte_alloca_with_zero_upper_half(ptr %src, i6
}
define void @load_16byte_chunk_of_64byte_alloca_with_zero_upper_half(ptr %src, i64 %byteOff, ptr %dst) nounwind {
-; X64-NO-BMI2-NO-SHLD-LABEL: load_16byte_chunk_of_64byte_alloca_with_zero_upper_half:
-; X64-NO-BMI2-NO-SHLD: # %bb.0:
-; X64-NO-BMI2-NO-SHLD-NEXT: pushq %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: movups (%rdi), %xmm0
-; X64-NO-BMI2-NO-SHLD-NEXT: movups 16(%rdi), %xmm1
-; X64-NO-BMI2-NO-SHLD-NEXT: xorps %xmm2, %xmm2
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: leal (,%rsi,8), %eax
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %edi
-; X64-NO-BMI2-NO-SHLD-NEXT: andl $56, %edi
-; X64-NO-BMI2-NO-SHLD-NEXT: andl $56, %esi
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -128(%rsp,%rsi), %r8
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -120(%rsp,%rsi), %r9
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %edi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %r8
-; X64-NO-BMI2-NO-SHLD-NEXT: notb %cl
-; X64-NO-BMI2-NO-SHLD-NEXT: leaq (%r9,%r9), %r10
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %r10
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %r8, %r10
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %edi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %r9
-; X64-NO-BMI2-NO-SHLD-NEXT: notl %eax
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -112(%rsp,%rsi), %rsi
-; X64-NO-BMI2-NO-SHLD-NEXT: addq %rsi, %rsi
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %rsi
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %r9, %rsi
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rsi, 8(%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r10, (%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: popq %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: retq
-;
-; X64-NO-BMI2-HAVE-SHLD-LABEL: load_16byte_chunk_of_64byte_alloca_with_zero_upper_half:
-; X64-NO-BMI2-HAVE-SHLD: # %bb.0:
-; X64-NO-BMI2-HAVE-SHLD-NEXT: pushq %rax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movups (%rdi), %xmm0
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movups 16(%rdi), %xmm1
-; X64-NO-BMI2-HAVE-SHLD-NEXT: xorps %xmm2, %xmm2
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: leal (,%rsi,8), %eax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, %edi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: andl $56, %edi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: andl $56, %esi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -128(%rsp,%rsi), %r8
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -120(%rsp,%rsi), %r9
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r9, %r10
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movl %edi, %ecx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrq %cl, %r10
-; X64-NO-BMI2-HAVE-SHLD-NEXT: notl %eax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -112(%rsp,%rsi), %rsi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: addq %rsi, %rsi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shlq %cl, %rsi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: orq %r10, %rsi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movl %edi, %ecx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %r9, %r8
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r8, (%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rsi, 8(%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: popq %rax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: retq
-;
-; X64-HAVE-BMI2-NO-SHLD-LABEL: load_16byte_chunk_of_64byte_alloca_with_zero_upper_half:
-; X64-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X64-HAVE-BMI2-NO-SHLD-NEXT: pushq %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movups (%rdi), %xmm0
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movups 16(%rdi), %xmm1
-; X64-HAVE-BMI2-NO-SHLD-NEXT: xorps %xmm2, %xmm2
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: leal (,%rsi,8), %eax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: andl $56, %ecx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, %edi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: andl $56, %esi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rdi, -128(%rsp,%rsi), %r8
-; X64-HAVE-BMI2-NO-SHLD-NEXT: notb %cl
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq -120(%rsp,%rsi), %r9
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq -112(%rsp,%rsi), %rsi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: leaq (%r9,%r9), %r10
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rcx, %r10, %rcx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %r8, %rcx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: notl %eax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: addq %rsi, %rsi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rax, %rsi, %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rdi, %r9, %rsi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %rax, %rsi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rsi, 8(%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rcx, (%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: popq %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: retq
-;
-; X64-HAVE-BMI2-HAVE-SHLD-LABEL: load_16byte_chunk_of_64byte_alloca_with_zero_upper_half:
-; X64-HAVE-BMI2-HAVE-SHLD: # %bb.0:
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: pushq %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movups (%rdi), %xmm0
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movups 16(%rdi), %xmm1
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: xorps %xmm2, %xmm2
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: leal (,%rsi,8), %ecx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, %eax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: notl %eax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: andl $56, %esi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -112(%rsp,%rsi), %rdi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: addq %rdi, %rdi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shlxq %rax, %rdi, %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: andl $56, %ecx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -128(%rsp,%rsi), %rdi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -120(%rsp,%rsi), %rsi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrxq %rcx, %rsi, %r8
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: orq %rax, %r8
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: # kill: def $cl killed $cl killed $rcx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %rsi, %rdi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rdi, (%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %r8, 8(%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: popq %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: retq
-;
-; X86-NO-BMI2-NO-SHLD-LABEL: load_16byte_chunk_of_64byte_alloca_with_zero_upper_half:
-; X86-NO-BMI2-NO-SHLD: # %bb.0:
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: subl $156, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movups (%ecx), %xmm0
-; X86-NO-BMI2-NO-SHLD-NEXT: movups 16(%ecx), %xmm1
-; X86-NO-BMI2-NO-SHLD-NEXT: xorps %xmm2, %xmm2
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: andl $60, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 16(%esp,%esi), %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 20(%esp,%esi), %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: shll $3, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: andl $24, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%ebp,%ebp), %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: notb %dl
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %ebx, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 24(%esp,%esi), %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%ebx,%ebx), %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %ebp, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 28(%esp,%esi), %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%edi,%edi), %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %ebx, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 32(%esp,%esi), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %eax, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %edi, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, 12(%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebp, 8(%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, 4(%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, (%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: addl $156, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: retl
+; X64-NO-BMI2-LABEL: load_16byte_chunk_of_64byte_alloca_with_zero_upper_half:
+; X64-NO-BMI2: # %bb.0:
+; X64-NO-BMI2-NEXT: pushq %rax
+; X64-NO-BMI2-NEXT: movups (%rdi), %xmm0
+; X64-NO-BMI2-NEXT: movups 16(%rdi), %xmm1
+; X64-NO-BMI2-NEXT: xorps %xmm2, %xmm2
+; X64-NO-BMI2-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: leal (,%rsi,8), %eax
+; X64-NO-BMI2-NEXT: movl %eax, %edi
+; X64-NO-BMI2-NEXT: andl $56, %edi
+; X64-NO-BMI2-NEXT: andl $56, %esi
+; X64-NO-BMI2-NEXT: movq -128(%rsp,%rsi), %r8
+; X64-NO-BMI2-NEXT: movq -120(%rsp,%rsi), %r9
+; X64-NO-BMI2-NEXT: movq %r9, %r10
+; X64-NO-BMI2-NEXT: movl %edi, %ecx
+; X64-NO-BMI2-NEXT: shrq %cl, %r10
+; X64-NO-BMI2-NEXT: notl %eax
+; X64-NO-BMI2-NEXT: movq -112(%rsp,%rsi), %rsi
+; X64-NO-BMI2-NEXT: addq %rsi, %rsi
+; X64-NO-BMI2-NEXT: movl %eax, %ecx
+; X64-NO-BMI2-NEXT: shlq %cl, %rsi
+; X64-NO-BMI2-NEXT: orq %r10, %rsi
+; X64-NO-BMI2-NEXT: movl %edi, %ecx
+; X64-NO-BMI2-NEXT: shrdq %cl, %r9, %r8
+; X64-NO-BMI2-NEXT: movq %r8, (%rdx)
+; X64-NO-BMI2-NEXT: movq %rsi, 8(%rdx)
+; X64-NO-BMI2-NEXT: popq %rax
+; X64-NO-BMI2-NEXT: retq
;
-; X86-SHLD-LABEL: load_16byte_chunk_of_64byte_alloca_with_zero_upper_half:
-; X86-SHLD: # %bb.0:
-; X86-SHLD-NEXT: pushl %ebp
-; X86-SHLD-NEXT: pushl %ebx
-; X86-SHLD-NEXT: pushl %edi
-; X86-SHLD-NEXT: pushl %esi
-; X86-SHLD-NEXT: subl $156, %esp
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-SHLD-NEXT: movups (%eax), %xmm0
-; X86-SHLD-NEXT: movups 16(%eax), %xmm1
-; X86-SHLD-NEXT: xorps %xmm2, %xmm2
-; X86-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movl %ecx, %edi
-; X86-SHLD-NEXT: andl $60, %edi
-; X86-SHLD-NEXT: movl 24(%esp,%edi), %esi
-; X86-SHLD-NEXT: movl 16(%esp,%edi), %eax
-; X86-SHLD-NEXT: movl 20(%esp,%edi), %ebx
-; X86-SHLD-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-SHLD-NEXT: shll $3, %ecx
-; X86-SHLD-NEXT: andl $24, %ecx
-; X86-SHLD-NEXT: shrdl %cl, %esi, %ebx
-; X86-SHLD-NEXT: movl 28(%esp,%edi), %ebp
-; X86-SHLD-NEXT: shrdl %cl, %ebp, %esi
-; X86-SHLD-NEXT: movl 32(%esp,%edi), %edi
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-SHLD-NEXT: shrdl %cl, %edi, %ebp
-; X86-SHLD-NEXT: movl %ebp, 12(%edx)
-; X86-SHLD-NEXT: movl %esi, 8(%edx)
-; X86-SHLD-NEXT: movl %ebx, 4(%edx)
-; X86-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-SHLD-NEXT: shrdl %cl, %esi, %eax
-; X86-SHLD-NEXT: movl %eax, (%edx)
-; X86-SHLD-NEXT: addl $156, %esp
-; X86-SHLD-NEXT: popl %esi
-; X86-SHLD-NEXT: popl %edi
-; X86-SHLD-NEXT: popl %ebx
-; X86-SHLD-NEXT: popl %ebp
-; X86-SHLD-NEXT: retl
+; X64-BMI2-LABEL: load_16byte_chunk_of_64byte_alloca_with_zero_upper_half:
+; X64-BMI2: # %bb.0:
+; X64-BMI2-NEXT: pushq %rax
+; X64-BMI2-NEXT: movups (%rdi), %xmm0
+; X64-BMI2-NEXT: movups 16(%rdi), %xmm1
+; X64-BMI2-NEXT: xorps %xmm2, %xmm2
+; X64-BMI2-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: leal (,%rsi,8), %ecx
+; X64-BMI2-NEXT: movl %ecx, %eax
+; X64-BMI2-NEXT: notl %eax
+; X64-BMI2-NEXT: andl $56, %esi
+; X64-BMI2-NEXT: movq -112(%rsp,%rsi), %rdi
+; X64-BMI2-NEXT: addq %rdi, %rdi
+; X64-BMI2-NEXT: shlxq %rax, %rdi, %rax
+; X64-BMI2-NEXT: andl $56, %ecx
+; X64-BMI2-NEXT: movq -128(%rsp,%rsi), %rdi
+; X64-BMI2-NEXT: movq -120(%rsp,%rsi), %rsi
+; X64-BMI2-NEXT: shrxq %rcx, %rsi, %r8
+; X64-BMI2-NEXT: orq %rax, %r8
+; X64-BMI2-NEXT: # kill: def $cl killed $cl killed $rcx
+; X64-BMI2-NEXT: shrdq %cl, %rsi, %rdi
+; X64-BMI2-NEXT: movq %rdi, (%rdx)
+; X64-BMI2-NEXT: movq %r8, 8(%rdx)
+; X64-BMI2-NEXT: popq %rax
+; X64-BMI2-NEXT: retq
;
-; X86-HAVE-BMI2-NO-SHLD-LABEL: load_16byte_chunk_of_64byte_alloca_with_zero_upper_half:
-; X86-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: subl $156, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movups (%ecx), %xmm0
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movups 16(%ecx), %xmm1
-; X86-HAVE-BMI2-NO-SHLD-NEXT: xorps %xmm2, %xmm2
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (,%eax,8), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: andl $24, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: andl $60, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ebx, 16(%esp,%eax), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: notb %cl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 20(%esp,%eax), %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 24(%esp,%eax), %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%edi,%edi), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %edx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %esi, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ebx, %edi, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%ebp,%ebp), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %esi, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %edi, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ebx, %ebp, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 28(%esp,%eax), %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%ebp,%ebp), %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %edi, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ebx, %ebp, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 32(%esp,%eax), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %eax, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %eax, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %ebx, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, 12(%ecx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edi, 8(%ecx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, 4(%ecx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edx, (%ecx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl $156, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: retl
+; X86-LABEL: load_16byte_chunk_of_64byte_alloca_with_zero_upper_half:
+; X86: # %bb.0:
+; X86-NEXT: pushl %ebp
+; X86-NEXT: pushl %ebx
+; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %esi
+; X86-NEXT: subl $156, %esp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movups (%eax), %xmm0
+; X86-NEXT: movups 16(%eax), %xmm1
+; X86-NEXT: xorps %xmm2, %xmm2
+; X86-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %ecx, %edi
+; X86-NEXT: andl $60, %edi
+; X86-NEXT: movl 24(%esp,%edi), %esi
+; X86-NEXT: movl 16(%esp,%edi), %eax
+; X86-NEXT: movl 20(%esp,%edi), %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shll $3, %ecx
+; X86-NEXT: andl $24, %ecx
+; X86-NEXT: shrdl %cl, %esi, %ebx
+; X86-NEXT: movl 28(%esp,%edi), %ebp
+; X86-NEXT: shrdl %cl, %ebp, %esi
+; X86-NEXT: movl 32(%esp,%edi), %edi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: shrdl %cl, %edi, %ebp
+; X86-NEXT: movl %ebp, 12(%edx)
+; X86-NEXT: movl %esi, 8(%edx)
+; X86-NEXT: movl %ebx, 4(%edx)
+; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: shrdl %cl, %esi, %eax
+; X86-NEXT: movl %eax, (%edx)
+; X86-NEXT: addl $156, %esp
+; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
+; X86-NEXT: popl %ebx
+; X86-NEXT: popl %ebp
+; X86-NEXT: retl
%init = load <32 x i8>, ptr %src, align 1
%intermediate.sroa.0.0.vec.expand = shufflevector <32 x i8> %init, <32 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
%intermediate.sroa.0.0.vecblend = shufflevector <64 x i8> %intermediate.sroa.0.0.vec.expand, <64 x i8> <i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
@@ -2914,480 +1429,168 @@ define void @load_16byte_chunk_of_64byte_alloca_with_zero_upper_half(ptr %src, i
}
define void @load_32byte_chunk_of_64byte_alloca_with_zero_upper_half(ptr %src, i64 %byteOff, ptr %dst) nounwind {
-; X64-NO-BMI2-NO-SHLD-LABEL: load_32byte_chunk_of_64byte_alloca_with_zero_upper_half:
-; X64-NO-BMI2-NO-SHLD: # %bb.0:
-; X64-NO-BMI2-NO-SHLD-NEXT: pushq %r14
-; X64-NO-BMI2-NO-SHLD-NEXT: pushq %rbx
-; X64-NO-BMI2-NO-SHLD-NEXT: pushq %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: movups (%rdi), %xmm0
-; X64-NO-BMI2-NO-SHLD-NEXT: movups 16(%rdi), %xmm1
-; X64-NO-BMI2-NO-SHLD-NEXT: xorps %xmm2, %xmm2
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: leal (,%rsi,8), %eax
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %edi
-; X64-NO-BMI2-NO-SHLD-NEXT: andl $56, %edi
-; X64-NO-BMI2-NO-SHLD-NEXT: andl $56, %esi
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -128(%rsp,%rsi), %r10
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -120(%rsp,%rsi), %r11
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %edi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %r10
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %edi, %r8d
-; X64-NO-BMI2-NO-SHLD-NEXT: notb %r8b
-; X64-NO-BMI2-NO-SHLD-NEXT: leaq (%r11,%r11), %r9
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %r8d, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %r9
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %r10, %r9
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %edi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %r11
-; X64-NO-BMI2-NO-SHLD-NEXT: notl %eax
-; X64-NO-BMI2-NO-SHLD-NEXT: andl $63, %eax
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -112(%rsp,%rsi), %r10
-; X64-NO-BMI2-NO-SHLD-NEXT: leaq (%r10,%r10), %rbx
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %rbx
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %r11, %rbx
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %edi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %r10
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -104(%rsp,%rsi), %r11
-; X64-NO-BMI2-NO-SHLD-NEXT: leaq (%r11,%r11), %r14
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %r8d, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %r14
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %r10, %r14
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %edi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %r11
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -96(%rsp,%rsi), %rsi
-; X64-NO-BMI2-NO-SHLD-NEXT: addq %rsi, %rsi
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %rsi
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %r11, %rsi
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rsi, 24(%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r14, 16(%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rbx, 8(%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r9, (%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: addq $8, %rsp
-; X64-NO-BMI2-NO-SHLD-NEXT: popq %rbx
-; X64-NO-BMI2-NO-SHLD-NEXT: popq %r14
-; X64-NO-BMI2-NO-SHLD-NEXT: retq
-;
-; X64-NO-BMI2-HAVE-SHLD-LABEL: load_32byte_chunk_of_64byte_alloca_with_zero_upper_half:
-; X64-NO-BMI2-HAVE-SHLD: # %bb.0:
-; X64-NO-BMI2-HAVE-SHLD-NEXT: pushq %r14
-; X64-NO-BMI2-HAVE-SHLD-NEXT: pushq %rbx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: pushq %rax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movups (%rdi), %xmm0
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movups 16(%rdi), %xmm1
-; X64-NO-BMI2-HAVE-SHLD-NEXT: xorps %xmm2, %xmm2
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: leal (,%rsi,8), %edi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movl %edi, %eax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: andl $56, %eax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: andl $56, %esi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -128(%rsp,%rsi), %r8
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -120(%rsp,%rsi), %r9
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r9, %r10
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrq %cl, %r10
-; X64-NO-BMI2-HAVE-SHLD-NEXT: notl %edi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: andl $63, %edi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -112(%rsp,%rsi), %r11
-; X64-NO-BMI2-HAVE-SHLD-NEXT: leaq (%r11,%r11), %rbx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movl %edi, %ecx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shlq %cl, %rbx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: orq %r10, %rbx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -104(%rsp,%rsi), %r10
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r10, %r14
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrq %cl, %r14
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -96(%rsp,%rsi), %rsi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: addq %rsi, %rsi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movl %edi, %ecx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shlq %cl, %rsi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: orq %r14, %rsi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %r10, %r11
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %r9, %r8
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r11, 16(%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r8, (%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rsi, 24(%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rbx, 8(%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: addq $8, %rsp
-; X64-NO-BMI2-HAVE-SHLD-NEXT: popq %rbx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: popq %r14
-; X64-NO-BMI2-HAVE-SHLD-NEXT: retq
-;
-; X64-HAVE-BMI2-NO-SHLD-LABEL: load_32byte_chunk_of_64byte_alloca_with_zero_upper_half:
-; X64-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X64-HAVE-BMI2-NO-SHLD-NEXT: pushq %rbx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movups (%rdi), %xmm0
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movups 16(%rdi), %xmm1
-; X64-HAVE-BMI2-NO-SHLD-NEXT: xorps %xmm2, %xmm2
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: leal (,%rsi,8), %eax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: andl $56, %ecx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, %edi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: andl $56, %esi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rdi, -128(%rsp,%rsi), %r8
-; X64-HAVE-BMI2-NO-SHLD-NEXT: notb %cl
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq -120(%rsp,%rsi), %r9
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq -112(%rsp,%rsi), %r10
-; X64-HAVE-BMI2-NO-SHLD-NEXT: leaq (%r9,%r9), %r11
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rcx, %r11, %r11
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %r8, %r11
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rdi, %r9, %r8
-; X64-HAVE-BMI2-NO-SHLD-NEXT: notl %eax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: andl $63, %eax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: leaq (%r10,%r10), %r9
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rax, %r9, %r9
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %r8, %r9
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rdi, %r10, %r8
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq -104(%rsp,%rsi), %r10
-; X64-HAVE-BMI2-NO-SHLD-NEXT: leaq (%r10,%r10), %rbx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rcx, %rbx, %rcx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %r8, %rcx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rdi, %r10, %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq -96(%rsp,%rsi), %rsi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: addq %rsi, %rsi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rax, %rsi, %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %rdi, %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rax, 24(%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rcx, 16(%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %r9, 8(%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %r11, (%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: popq %rbx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: retq
-;
-; X64-HAVE-BMI2-HAVE-SHLD-LABEL: load_32byte_chunk_of_64byte_alloca_with_zero_upper_half:
-; X64-HAVE-BMI2-HAVE-SHLD: # %bb.0:
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: pushq %rbx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movups (%rdi), %xmm0
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movups 16(%rdi), %xmm1
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: xorps %xmm2, %xmm2
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: leal (,%rsi,8), %eax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movl %eax, %ecx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: andl $56, %ecx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: andl $56, %esi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -128(%rsp,%rsi), %rdi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -120(%rsp,%rsi), %r8
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrxq %rcx, %r8, %r9
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: notl %eax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: andl $63, %eax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -112(%rsp,%rsi), %r10
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: leaq (%r10,%r10), %r11
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shlxq %rax, %r11, %r11
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: orq %r9, %r11
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -104(%rsp,%rsi), %r9
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrxq %rcx, %r9, %rbx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -96(%rsp,%rsi), %rsi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: addq %rsi, %rsi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shlxq %rax, %rsi, %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: orq %rbx, %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %r9, %r10
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: # kill: def $cl killed $cl killed $rcx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %r8, %rdi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %r10, 16(%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rdi, (%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rax, 24(%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %r11, 8(%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: popq %rbx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: retq
-;
-; X86-NO-BMI2-NO-SHLD-LABEL: load_32byte_chunk_of_64byte_alloca_with_zero_upper_half:
-; X86-NO-BMI2-NO-SHLD: # %bb.0:
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: subl $172, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movups (%ecx), %xmm0
-; X86-NO-BMI2-NO-SHLD-NEXT: movups 16(%ecx), %xmm1
-; X86-NO-BMI2-NO-SHLD-NEXT: xorps %xmm2, %xmm2
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: andl $60, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 32(%esp,%edi), %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 36(%esp,%edi), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: shll $3, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: andl $24, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%esi,%esi), %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: notb %dl
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %ebx, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 40(%esp,%edi), %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%ebx,%ebx), %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 44(%esp,%edi), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%esi,%esi), %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %ebx, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 48(%esp,%edi), %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%ebx,%ebx), %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 52(%esp,%edi), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%esi,%esi), %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %ebx, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 56(%esp,%edi), %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%ebp,%ebp), %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 60(%esp,%edi), %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%ebx,%ebx), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %ebp, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 64(%esp,%edi), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %eax, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %ebx, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, 28(%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %esi, 24(%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, 20(%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, 16(%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, 12(%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, 8(%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, 4(%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, (%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: addl $172, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: retl
+; X64-NO-BMI2-LABEL: load_32byte_chunk_of_64byte_alloca_with_zero_upper_half:
+; X64-NO-BMI2: # %bb.0:
+; X64-NO-BMI2-NEXT: pushq %r14
+; X64-NO-BMI2-NEXT: pushq %rbx
+; X64-NO-BMI2-NEXT: pushq %rax
+; X64-NO-BMI2-NEXT: movups (%rdi), %xmm0
+; X64-NO-BMI2-NEXT: movups 16(%rdi), %xmm1
+; X64-NO-BMI2-NEXT: xorps %xmm2, %xmm2
+; X64-NO-BMI2-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: leal (,%rsi,8), %edi
+; X64-NO-BMI2-NEXT: movl %edi, %eax
+; X64-NO-BMI2-NEXT: andl $56, %eax
+; X64-NO-BMI2-NEXT: andl $56, %esi
+; X64-NO-BMI2-NEXT: movq -128(%rsp,%rsi), %r8
+; X64-NO-BMI2-NEXT: movq -120(%rsp,%rsi), %r9
+; X64-NO-BMI2-NEXT: movq %r9, %r10
+; X64-NO-BMI2-NEXT: movl %eax, %ecx
+; X64-NO-BMI2-NEXT: shrq %cl, %r10
+; X64-NO-BMI2-NEXT: notl %edi
+; X64-NO-BMI2-NEXT: andl $63, %edi
+; X64-NO-BMI2-NEXT: movq -112(%rsp,%rsi), %r11
+; X64-NO-BMI2-NEXT: leaq (%r11,%r11), %rbx
+; X64-NO-BMI2-NEXT: movl %edi, %ecx
+; X64-NO-BMI2-NEXT: shlq %cl, %rbx
+; X64-NO-BMI2-NEXT: orq %r10, %rbx
+; X64-NO-BMI2-NEXT: movq -104(%rsp,%rsi), %r10
+; X64-NO-BMI2-NEXT: movq %r10, %r14
+; X64-NO-BMI2-NEXT: movl %eax, %ecx
+; X64-NO-BMI2-NEXT: shrq %cl, %r14
+; X64-NO-BMI2-NEXT: movq -96(%rsp,%rsi), %rsi
+; X64-NO-BMI2-NEXT: addq %rsi, %rsi
+; X64-NO-BMI2-NEXT: movl %edi, %ecx
+; X64-NO-BMI2-NEXT: shlq %cl, %rsi
+; X64-NO-BMI2-NEXT: orq %r14, %rsi
+; X64-NO-BMI2-NEXT: movl %eax, %ecx
+; X64-NO-BMI2-NEXT: shrdq %cl, %r10, %r11
+; X64-NO-BMI2-NEXT: shrdq %cl, %r9, %r8
+; X64-NO-BMI2-NEXT: movq %r11, 16(%rdx)
+; X64-NO-BMI2-NEXT: movq %r8, (%rdx)
+; X64-NO-BMI2-NEXT: movq %rsi, 24(%rdx)
+; X64-NO-BMI2-NEXT: movq %rbx, 8(%rdx)
+; X64-NO-BMI2-NEXT: addq $8, %rsp
+; X64-NO-BMI2-NEXT: popq %rbx
+; X64-NO-BMI2-NEXT: popq %r14
+; X64-NO-BMI2-NEXT: retq
;
-; X86-SHLD-LABEL: load_32byte_chunk_of_64byte_alloca_with_zero_upper_half:
-; X86-SHLD: # %bb.0:
-; X86-SHLD-NEXT: pushl %ebp
-; X86-SHLD-NEXT: pushl %ebx
-; X86-SHLD-NEXT: pushl %edi
-; X86-SHLD-NEXT: pushl %esi
-; X86-SHLD-NEXT: subl $156, %esp
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-SHLD-NEXT: movups (%eax), %xmm0
-; X86-SHLD-NEXT: movups 16(%eax), %xmm1
-; X86-SHLD-NEXT: xorps %xmm2, %xmm2
-; X86-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movl %ecx, %edi
-; X86-SHLD-NEXT: andl $60, %edi
-; X86-SHLD-NEXT: movl 24(%esp,%edi), %edx
-; X86-SHLD-NEXT: movl 20(%esp,%edi), %esi
-; X86-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-SHLD-NEXT: shll $3, %ecx
-; X86-SHLD-NEXT: andl $24, %ecx
-; X86-SHLD-NEXT: movl %edx, %eax
-; X86-SHLD-NEXT: shrdl %cl, %edx, %esi
-; X86-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-SHLD-NEXT: movl 28(%esp,%edi), %edx
-; X86-SHLD-NEXT: shrdl %cl, %edx, %eax
-; X86-SHLD-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-SHLD-NEXT: movl 32(%esp,%edi), %ebp
-; X86-SHLD-NEXT: shrdl %cl, %ebp, %edx
-; X86-SHLD-NEXT: movl %edx, (%esp) # 4-byte Spill
-; X86-SHLD-NEXT: movl 36(%esp,%edi), %esi
-; X86-SHLD-NEXT: shrdl %cl, %esi, %ebp
-; X86-SHLD-NEXT: movl 40(%esp,%edi), %edx
-; X86-SHLD-NEXT: shrdl %cl, %edx, %esi
-; X86-SHLD-NEXT: movl 44(%esp,%edi), %eax
-; X86-SHLD-NEXT: shrdl %cl, %eax, %edx
-; X86-SHLD-NEXT: movl 16(%esp,%edi), %ebx
-; X86-SHLD-NEXT: movl 48(%esp,%edi), %edi
-; X86-SHLD-NEXT: shrdl %cl, %edi, %eax
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X86-SHLD-NEXT: movl %eax, 28(%edi)
-; X86-SHLD-NEXT: movl %edx, 24(%edi)
-; X86-SHLD-NEXT: movl %esi, 20(%edi)
-; X86-SHLD-NEXT: movl %ebp, 16(%edi)
-; X86-SHLD-NEXT: movl (%esp), %eax # 4-byte Reload
-; X86-SHLD-NEXT: movl %eax, 12(%edi)
-; X86-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-SHLD-NEXT: movl %eax, 8(%edi)
-; X86-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-SHLD-NEXT: movl %eax, 4(%edi)
-; X86-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-SHLD-NEXT: shrdl %cl, %eax, %ebx
-; X86-SHLD-NEXT: movl %ebx, (%edi)
-; X86-SHLD-NEXT: addl $156, %esp
-; X86-SHLD-NEXT: popl %esi
-; X86-SHLD-NEXT: popl %edi
-; X86-SHLD-NEXT: popl %ebx
-; X86-SHLD-NEXT: popl %ebp
-; X86-SHLD-NEXT: retl
+; X64-BMI2-LABEL: load_32byte_chunk_of_64byte_alloca_with_zero_upper_half:
+; X64-BMI2: # %bb.0:
+; X64-BMI2-NEXT: pushq %rbx
+; X64-BMI2-NEXT: movups (%rdi), %xmm0
+; X64-BMI2-NEXT: movups 16(%rdi), %xmm1
+; X64-BMI2-NEXT: xorps %xmm2, %xmm2
+; X64-BMI2-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: leal (,%rsi,8), %eax
+; X64-BMI2-NEXT: movl %eax, %ecx
+; X64-BMI2-NEXT: andl $56, %ecx
+; X64-BMI2-NEXT: andl $56, %esi
+; X64-BMI2-NEXT: movq -128(%rsp,%rsi), %rdi
+; X64-BMI2-NEXT: movq -120(%rsp,%rsi), %r8
+; X64-BMI2-NEXT: shrxq %rcx, %r8, %r9
+; X64-BMI2-NEXT: notl %eax
+; X64-BMI2-NEXT: andl $63, %eax
+; X64-BMI2-NEXT: movq -112(%rsp,%rsi), %r10
+; X64-BMI2-NEXT: leaq (%r10,%r10), %r11
+; X64-BMI2-NEXT: shlxq %rax, %r11, %r11
+; X64-BMI2-NEXT: orq %r9, %r11
+; X64-BMI2-NEXT: movq -104(%rsp,%rsi), %r9
+; X64-BMI2-NEXT: shrxq %rcx, %r9, %rbx
+; X64-BMI2-NEXT: movq -96(%rsp,%rsi), %rsi
+; X64-BMI2-NEXT: addq %rsi, %rsi
+; X64-BMI2-NEXT: shlxq %rax, %rsi, %rax
+; X64-BMI2-NEXT: orq %rbx, %rax
+; X64-BMI2-NEXT: shrdq %cl, %r9, %r10
+; X64-BMI2-NEXT: # kill: def $cl killed $cl killed $rcx
+; X64-BMI2-NEXT: shrdq %cl, %r8, %rdi
+; X64-BMI2-NEXT: movq %r10, 16(%rdx)
+; X64-BMI2-NEXT: movq %rdi, (%rdx)
+; X64-BMI2-NEXT: movq %rax, 24(%rdx)
+; X64-BMI2-NEXT: movq %r11, 8(%rdx)
+; X64-BMI2-NEXT: popq %rbx
+; X64-BMI2-NEXT: retq
;
-; X86-HAVE-BMI2-NO-SHLD-LABEL: load_32byte_chunk_of_64byte_alloca_with_zero_upper_half:
-; X86-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: subl $172, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movups (%ecx), %xmm0
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movups 16(%ecx), %xmm1
-; X86-HAVE-BMI2-NO-SHLD-NEXT: xorps %xmm2, %xmm2
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (,%eax,8), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: andl $24, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: andl $60, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ebx, 32(%esp,%eax), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: notb %cl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 36(%esp,%eax), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 40(%esp,%eax), %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%esi,%esi), %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %ebp, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %edx, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ebx, %esi, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%edi,%edi), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %esi, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %edx, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ebx, %edi, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 44(%esp,%eax), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%esi,%esi), %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %edi, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %edx, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ebx, %esi, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 48(%esp,%eax), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%esi,%esi), %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %ebp, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %edx, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ebx, %esi, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 52(%esp,%eax), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%edx,%edx), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %esi, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %ebp, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ebx, %edx, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 56(%esp,%eax), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%edx,%edx), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %esi, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %ebp, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ebx, %edx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 60(%esp,%eax), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%edx,%edx), %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %ebp, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ebx, %edx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 64(%esp,%eax), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %eax, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %eax, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %edx, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, 28(%ecx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ebp, 24(%ecx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, 20(%ecx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edi, 16(%ecx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, 12(%ecx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, 8(%ecx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, 4(%ecx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, (%ecx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl $172, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: retl
+; X86-LABEL: load_32byte_chunk_of_64byte_alloca_with_zero_upper_half:
+; X86: # %bb.0:
+; X86-NEXT: pushl %ebp
+; X86-NEXT: pushl %ebx
+; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %esi
+; X86-NEXT: subl $156, %esp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movups (%eax), %xmm0
+; X86-NEXT: movups 16(%eax), %xmm1
+; X86-NEXT: xorps %xmm2, %xmm2
+; X86-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %ecx, %edi
+; X86-NEXT: andl $60, %edi
+; X86-NEXT: movl 24(%esp,%edi), %edx
+; X86-NEXT: movl 20(%esp,%edi), %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shll $3, %ecx
+; X86-NEXT: andl $24, %ecx
+; X86-NEXT: movl %edx, %eax
+; X86-NEXT: shrdl %cl, %edx, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 28(%esp,%edi), %edx
+; X86-NEXT: shrdl %cl, %edx, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 32(%esp,%edi), %ebp
+; X86-NEXT: shrdl %cl, %ebp, %edx
+; X86-NEXT: movl %edx, (%esp) # 4-byte Spill
+; X86-NEXT: movl 36(%esp,%edi), %esi
+; X86-NEXT: shrdl %cl, %esi, %ebp
+; X86-NEXT: movl 40(%esp,%edi), %edx
+; X86-NEXT: shrdl %cl, %edx, %esi
+; X86-NEXT: movl 44(%esp,%edi), %eax
+; X86-NEXT: shrdl %cl, %eax, %edx
+; X86-NEXT: movl 16(%esp,%edi), %ebx
+; X86-NEXT: movl 48(%esp,%edi), %edi
+; X86-NEXT: shrdl %cl, %edi, %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT: movl %eax, 28(%edi)
+; X86-NEXT: movl %edx, 24(%edi)
+; X86-NEXT: movl %esi, 20(%edi)
+; X86-NEXT: movl %ebp, 16(%edi)
+; X86-NEXT: movl (%esp), %eax # 4-byte Reload
+; X86-NEXT: movl %eax, 12(%edi)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl %eax, 8(%edi)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl %eax, 4(%edi)
+; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: shrdl %cl, %eax, %ebx
+; X86-NEXT: movl %ebx, (%edi)
+; X86-NEXT: addl $156, %esp
+; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
+; X86-NEXT: popl %ebx
+; X86-NEXT: popl %ebp
+; X86-NEXT: retl
%init = load <32 x i8>, ptr %src, align 1
%intermediate.sroa.0.0.vec.expand = shufflevector <32 x i8> %init, <32 x i8> poison, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
%intermediate.sroa.0.0.vecblend = shufflevector <64 x i8> %intermediate.sroa.0.0.vec.expand, <64 x i8> <i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 poison, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0, i8 0>, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 96, i32 97, i32 98, i32 99, i32 100, i32 101, i32 102, i32 103, i32 104, i32 105, i32 106, i32 107, i32 108, i32 109, i32 110, i32 111, i32 112, i32 113, i32 114, i32 115, i32 116, i32 117, i32 118, i32 119, i32 120, i32 121, i32 122, i32 123, i32 124, i32 125, i32 126, i32 127>
@@ -3402,8 +1605,15 @@ define void @load_32byte_chunk_of_64byte_alloca_with_zero_upper_half(ptr %src, i
}
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; ALL: {{.*}}
+; X64-HAVE-BMI2-HAVE-SHLD: {{.*}}
+; X64-HAVE-BMI2-NO-SHLD: {{.*}}
+; X64-NO-BMI2-HAVE-SHLD: {{.*}}
+; X64-NO-BMI2-NO-SHLD: {{.*}}
; X64-NO-SHLD: {{.*}}
-; X86: {{.*}}
+; X64-SHLD: {{.*}}
; X86-HAVE-BMI2-HAVE-SHLD: {{.*}}
+; X86-HAVE-BMI2-NO-SHLD: {{.*}}
; X86-NO-BMI2-HAVE-SHLD: {{.*}}
+; X86-NO-BMI2-NO-SHLD: {{.*}}
; X86-NO-SHLD: {{.*}}
+; X86-SHLD: {{.*}}
diff --git a/llvm/test/CodeGen/X86/widen-load-of-small-alloca.ll b/llvm/test/CodeGen/X86/widen-load-of-small-alloca.ll
index bed8e5806380c..a629fdbbddd16 100644
--- a/llvm/test/CodeGen/X86/widen-load-of-small-alloca.ll
+++ b/llvm/test/CodeGen/X86/widen-load-of-small-alloca.ll
@@ -182,91 +182,39 @@ define void @load_1byte_chunk_of_8byte_alloca(ptr %src, i64 %byteOff, ptr %dst)
; X64-BMI2-NEXT: movb %al, (%rdx)
; X64-BMI2-NEXT: retq
;
-; X86-NO-BMI2-NO-SHLD-LABEL: load_1byte_chunk_of_8byte_alloca:
-; X86-NO-BMI2-NO-SHLD: # %bb.0:
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: shll $3, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl (%ecx), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 4(%ecx), %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%ebx,%ebx), %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: notb %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: testb $32, %al
-; X86-NO-BMI2-NO-SHLD-NEXT: cmovel %edi, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %bl, (%edx)
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: retl
-;
-; X86-NO-BMI2-HAVE-SHLD-LABEL: load_1byte_chunk_of_8byte_alloca:
-; X86-NO-BMI2-HAVE-SHLD: # %bb.0:
-; X86-NO-BMI2-HAVE-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shll $3, %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl (%edx), %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 4(%edx), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edx, %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrl %cl, %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: testb $32, %cl
-; X86-NO-BMI2-HAVE-SHLD-NEXT: cmovel %esi, %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movb %dl, (%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: retl
-;
-; X86-HAVE-BMI2-NO-SHLD-LABEL: load_1byte_chunk_of_8byte_alloca:
-; X86-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shll $3, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 4(%edx), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, (%edx), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: notb %bl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%esi,%esi), %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ebx, %edi, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %edx, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, %esi, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: testb $32, %cl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: cmovel %edi, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movb %dl, (%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: retl
+; X86-NO-BMI2-LABEL: load_1byte_chunk_of_8byte_alloca:
+; X86-NO-BMI2: # %bb.0:
+; X86-NO-BMI2-NEXT: pushl %esi
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-BMI2-NEXT: shll $3, %ecx
+; X86-NO-BMI2-NEXT: movl (%edx), %esi
+; X86-NO-BMI2-NEXT: movl 4(%edx), %edx
+; X86-NO-BMI2-NEXT: shrdl %cl, %edx, %esi
+; X86-NO-BMI2-NEXT: shrl %cl, %edx
+; X86-NO-BMI2-NEXT: testb $32, %cl
+; X86-NO-BMI2-NEXT: cmovel %esi, %edx
+; X86-NO-BMI2-NEXT: movb %dl, (%eax)
+; X86-NO-BMI2-NEXT: popl %esi
+; X86-NO-BMI2-NEXT: retl
;
-; X86-HAVE-BMI2-HAVE-SHLD-LABEL: load_1byte_chunk_of_8byte_alloca:
-; X86-HAVE-BMI2-HAVE-SHLD: # %bb.0:
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shll $3, %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl (%edx), %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 4(%edx), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edx, %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrxl %ecx, %edx, %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: testb $32, %cl
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: cmovel %esi, %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movb %dl, (%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: retl
+; X86-BMI2-LABEL: load_1byte_chunk_of_8byte_alloca:
+; X86-BMI2: # %bb.0:
+; X86-BMI2-NEXT: pushl %esi
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-BMI2-NEXT: shll $3, %ecx
+; X86-BMI2-NEXT: movl (%edx), %esi
+; X86-BMI2-NEXT: movl 4(%edx), %edx
+; X86-BMI2-NEXT: shrdl %cl, %edx, %esi
+; X86-BMI2-NEXT: shrxl %ecx, %edx, %edx
+; X86-BMI2-NEXT: testb $32, %cl
+; X86-BMI2-NEXT: cmovel %esi, %edx
+; X86-BMI2-NEXT: movb %dl, (%eax)
+; X86-BMI2-NEXT: popl %esi
+; X86-BMI2-NEXT: retl
%init = load <8 x i8>, ptr %src, align 1
%byteOff.numbits = shl nuw nsw i64 %byteOff, 3
%intermediate.val.frozen = freeze <8 x i8> %init
@@ -295,91 +243,39 @@ define void @load_2byte_chunk_of_8byte_alloca(ptr %src, i64 %byteOff, ptr %dst)
; X64-BMI2-NEXT: movw %ax, (%rdx)
; X64-BMI2-NEXT: retq
;
-; X86-NO-BMI2-NO-SHLD-LABEL: load_2byte_chunk_of_8byte_alloca:
-; X86-NO-BMI2-NO-SHLD: # %bb.0:
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: shll $3, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl (%ecx), %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 4(%ecx), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%esi,%esi), %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: notb %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %edi, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: testb $32, %al
-; X86-NO-BMI2-NO-SHLD-NEXT: cmovel %ebx, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movw %si, (%edx)
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: retl
-;
-; X86-NO-BMI2-HAVE-SHLD-LABEL: load_2byte_chunk_of_8byte_alloca:
-; X86-NO-BMI2-HAVE-SHLD: # %bb.0:
-; X86-NO-BMI2-HAVE-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shll $3, %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl (%edx), %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 4(%edx), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edx, %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrl %cl, %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: testb $32, %cl
-; X86-NO-BMI2-HAVE-SHLD-NEXT: cmovel %esi, %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movw %dx, (%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: retl
-;
-; X86-HAVE-BMI2-NO-SHLD-LABEL: load_2byte_chunk_of_8byte_alloca:
-; X86-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shll $3, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 4(%edx), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, (%edx), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: notb %bl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%esi,%esi), %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ebx, %edi, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %edx, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, %esi, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: testb $32, %cl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: cmovel %edi, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movw %dx, (%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: retl
+; X86-NO-BMI2-LABEL: load_2byte_chunk_of_8byte_alloca:
+; X86-NO-BMI2: # %bb.0:
+; X86-NO-BMI2-NEXT: pushl %esi
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-BMI2-NEXT: shll $3, %ecx
+; X86-NO-BMI2-NEXT: movl (%edx), %esi
+; X86-NO-BMI2-NEXT: movl 4(%edx), %edx
+; X86-NO-BMI2-NEXT: shrdl %cl, %edx, %esi
+; X86-NO-BMI2-NEXT: shrl %cl, %edx
+; X86-NO-BMI2-NEXT: testb $32, %cl
+; X86-NO-BMI2-NEXT: cmovel %esi, %edx
+; X86-NO-BMI2-NEXT: movw %dx, (%eax)
+; X86-NO-BMI2-NEXT: popl %esi
+; X86-NO-BMI2-NEXT: retl
;
-; X86-HAVE-BMI2-HAVE-SHLD-LABEL: load_2byte_chunk_of_8byte_alloca:
-; X86-HAVE-BMI2-HAVE-SHLD: # %bb.0:
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shll $3, %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl (%edx), %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 4(%edx), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edx, %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrxl %ecx, %edx, %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: testb $32, %cl
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: cmovel %esi, %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movw %dx, (%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: retl
+; X86-BMI2-LABEL: load_2byte_chunk_of_8byte_alloca:
+; X86-BMI2: # %bb.0:
+; X86-BMI2-NEXT: pushl %esi
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-BMI2-NEXT: shll $3, %ecx
+; X86-BMI2-NEXT: movl (%edx), %esi
+; X86-BMI2-NEXT: movl 4(%edx), %edx
+; X86-BMI2-NEXT: shrdl %cl, %edx, %esi
+; X86-BMI2-NEXT: shrxl %ecx, %edx, %edx
+; X86-BMI2-NEXT: testb $32, %cl
+; X86-BMI2-NEXT: cmovel %esi, %edx
+; X86-BMI2-NEXT: movw %dx, (%eax)
+; X86-BMI2-NEXT: popl %esi
+; X86-BMI2-NEXT: retl
%init = load <8 x i8>, ptr %src, align 1
%byteOff.numbits = shl nuw nsw i64 %byteOff, 3
%intermediate.val.frozen = freeze <8 x i8> %init
@@ -407,91 +303,39 @@ define void @load_4byte_chunk_of_8byte_alloca(ptr %src, i64 %byteOff, ptr %dst)
; X64-BMI2-NEXT: movl %eax, (%rdx)
; X64-BMI2-NEXT: retq
;
-; X86-NO-BMI2-NO-SHLD-LABEL: load_4byte_chunk_of_8byte_alloca:
-; X86-NO-BMI2-NO-SHLD: # %bb.0:
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: shll $3, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl (%ecx), %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 4(%ecx), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%esi,%esi), %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: notb %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %edi, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: testb $32, %al
-; X86-NO-BMI2-NO-SHLD-NEXT: cmovel %ebx, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %esi, (%edx)
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: retl
-;
-; X86-NO-BMI2-HAVE-SHLD-LABEL: load_4byte_chunk_of_8byte_alloca:
-; X86-NO-BMI2-HAVE-SHLD: # %bb.0:
-; X86-NO-BMI2-HAVE-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shll $3, %ecx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl (%edx), %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl 4(%edx), %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edx, %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: shrl %cl, %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: testb $32, %cl
-; X86-NO-BMI2-HAVE-SHLD-NEXT: cmovel %esi, %edx
-; X86-NO-BMI2-HAVE-SHLD-NEXT: movl %edx, (%eax)
-; X86-NO-BMI2-HAVE-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-HAVE-SHLD-NEXT: retl
-;
-; X86-HAVE-BMI2-NO-SHLD-LABEL: load_4byte_chunk_of_8byte_alloca:
-; X86-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shll $3, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 4(%edx), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, (%edx), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: notb %bl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%esi,%esi), %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ebx, %edi, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %edx, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, %esi, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: testb $32, %cl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: cmovel %edi, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edx, (%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: retl
+; X86-NO-BMI2-LABEL: load_4byte_chunk_of_8byte_alloca:
+; X86-NO-BMI2: # %bb.0:
+; X86-NO-BMI2-NEXT: pushl %esi
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NO-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NO-BMI2-NEXT: shll $3, %ecx
+; X86-NO-BMI2-NEXT: movl (%edx), %esi
+; X86-NO-BMI2-NEXT: movl 4(%edx), %edx
+; X86-NO-BMI2-NEXT: shrdl %cl, %edx, %esi
+; X86-NO-BMI2-NEXT: shrl %cl, %edx
+; X86-NO-BMI2-NEXT: testb $32, %cl
+; X86-NO-BMI2-NEXT: cmovel %esi, %edx
+; X86-NO-BMI2-NEXT: movl %edx, (%eax)
+; X86-NO-BMI2-NEXT: popl %esi
+; X86-NO-BMI2-NEXT: retl
;
-; X86-HAVE-BMI2-HAVE-SHLD-LABEL: load_4byte_chunk_of_8byte_alloca:
-; X86-HAVE-BMI2-HAVE-SHLD: # %bb.0:
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shll $3, %ecx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl (%edx), %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl 4(%edx), %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrdl %cl, %edx, %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: shrxl %ecx, %edx, %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: testb $32, %cl
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: cmovel %esi, %edx
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: movl %edx, (%eax)
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-HAVE-SHLD-NEXT: retl
+; X86-BMI2-LABEL: load_4byte_chunk_of_8byte_alloca:
+; X86-BMI2: # %bb.0:
+; X86-BMI2-NEXT: pushl %esi
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-BMI2-NEXT: shll $3, %ecx
+; X86-BMI2-NEXT: movl (%edx), %esi
+; X86-BMI2-NEXT: movl 4(%edx), %edx
+; X86-BMI2-NEXT: shrdl %cl, %edx, %esi
+; X86-BMI2-NEXT: shrxl %ecx, %edx, %edx
+; X86-BMI2-NEXT: testb $32, %cl
+; X86-BMI2-NEXT: cmovel %esi, %edx
+; X86-BMI2-NEXT: movl %edx, (%eax)
+; X86-BMI2-NEXT: popl %esi
+; X86-BMI2-NEXT: retl
%init = load <8 x i8>, ptr %src, align 1
%byteOff.numbits = shl nuw nsw i64 %byteOff, 3
%intermediate.val.frozen = freeze <8 x i8> %init
@@ -505,155 +349,60 @@ define void @load_4byte_chunk_of_8byte_alloca(ptr %src, i64 %byteOff, ptr %dst)
; no @load_8byte_chunk_of_8byte_alloca
define void @load_1byte_chunk_of_16byte_alloca(ptr %src, i64 %byteOff, ptr %dst) nounwind {
-; X64-NO-BMI2-NO-SHLD-LABEL: load_1byte_chunk_of_16byte_alloca:
-; X64-NO-BMI2-NO-SHLD: # %bb.0:
-; X64-NO-BMI2-NO-SHLD-NEXT: shll $3, %esi
-; X64-NO-BMI2-NO-SHLD-NEXT: movdqu (%rdi), %xmm0
-; X64-NO-BMI2-NO-SHLD-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %xmm1, %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %xmm0, %rdi
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %esi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %rdi
-; X64-NO-BMI2-NO-SHLD-NEXT: leal (%rax,%rax), %r8d
-; X64-NO-BMI2-NO-SHLD-NEXT: notb %cl
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %r8
-; X64-NO-BMI2-NO-SHLD-NEXT: orl %edi, %r8d
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %esi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: testb $64, %sil
-; X64-NO-BMI2-NO-SHLD-NEXT: cmoveq %r8, %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: movb %al, (%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: retq
-;
-; X64-NO-BMI2-HAVE-SHLD-LABEL: load_1byte_chunk_of_16byte_alloca:
-; X64-NO-BMI2-HAVE-SHLD: # %bb.0:
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rsi, %rcx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shll $3, %ecx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movdqu (%rdi), %xmm0
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %xmm0, %rax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %xmm0, %rsi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %rsi, %rax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrq %cl, %rsi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: testb $64, %cl
-; X64-NO-BMI2-HAVE-SHLD-NEXT: cmoveq %rax, %rsi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movb %sil, (%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: retq
-;
-; X64-HAVE-BMI2-NO-SHLD-LABEL: load_1byte_chunk_of_16byte_alloca:
-; X64-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shll $3, %esi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movdqu (%rdi), %xmm0
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %xmm0, %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %xmm0, %rcx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, %edi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: notb %dil
-; X64-HAVE-BMI2-NO-SHLD-NEXT: leal (%rcx,%rcx), %r8d
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rdi, %r8, %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rsi, %rax, %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orl %edi, %eax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rsi, %rcx, %rcx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: testb $64, %sil
-; X64-HAVE-BMI2-NO-SHLD-NEXT: cmoveq %rax, %rcx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movb %cl, (%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: retq
-;
-; X64-HAVE-BMI2-HAVE-SHLD-LABEL: load_1byte_chunk_of_16byte_alloca:
-; X64-HAVE-BMI2-HAVE-SHLD: # %bb.0:
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rsi, %rcx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shll $3, %ecx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movdqu (%rdi), %xmm0
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %xmm0, %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %xmm0, %rsi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %rsi, %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrxq %rcx, %rsi, %rsi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: testb $64, %cl
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: cmoveq %rax, %rsi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movb %sil, (%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: retq
-;
-; X86-NO-BMI2-NO-SHLD-LABEL: load_1byte_chunk_of_16byte_alloca:
-; X86-NO-BMI2-NO-SHLD: # %bb.0:
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: subl $40, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movups (%edx), %xmm0
-; X86-NO-BMI2-NO-SHLD-NEXT: shll $3, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: xorps %xmm1, %xmm1
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrb $3, %dl
-; X86-NO-BMI2-NO-SHLD-NEXT: andb $12, %dl
-; X86-NO-BMI2-NO-SHLD-NEXT: movzbl %dl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl (%esp,%edx), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 4(%esp,%edx), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: notb %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %edx, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dl, (%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: addl $40, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: retl
+; X64-NO-BMI2-LABEL: load_1byte_chunk_of_16byte_alloca:
+; X64-NO-BMI2: # %bb.0:
+; X64-NO-BMI2-NEXT: movq %rsi, %rcx
+; X64-NO-BMI2-NEXT: shll $3, %ecx
+; X64-NO-BMI2-NEXT: movdqu (%rdi), %xmm0
+; X64-NO-BMI2-NEXT: movq %xmm0, %rax
+; X64-NO-BMI2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; X64-NO-BMI2-NEXT: movq %xmm0, %rsi
+; X64-NO-BMI2-NEXT: shrdq %cl, %rsi, %rax
+; X64-NO-BMI2-NEXT: shrq %cl, %rsi
+; X64-NO-BMI2-NEXT: testb $64, %cl
+; X64-NO-BMI2-NEXT: cmoveq %rax, %rsi
+; X64-NO-BMI2-NEXT: movb %sil, (%rdx)
+; X64-NO-BMI2-NEXT: retq
;
-; X86-SHLD-LABEL: load_1byte_chunk_of_16byte_alloca:
-; X86-SHLD: # %bb.0:
-; X86-SHLD-NEXT: pushl %ebx
-; X86-SHLD-NEXT: subl $40, %esp
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-SHLD-NEXT: movups (%edx), %xmm0
-; X86-SHLD-NEXT: shll $3, %ecx
-; X86-SHLD-NEXT: xorps %xmm1, %xmm1
-; X86-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-SHLD-NEXT: movl %ecx, %edx
-; X86-SHLD-NEXT: shrb $3, %dl
-; X86-SHLD-NEXT: andb $12, %dl
-; X86-SHLD-NEXT: movzbl %dl, %edx
-; X86-SHLD-NEXT: movl (%esp,%edx), %ebx
-; X86-SHLD-NEXT: movl 4(%esp,%edx), %edx
-; X86-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-SHLD-NEXT: shrdl %cl, %edx, %ebx
-; X86-SHLD-NEXT: movb %bl, (%eax)
-; X86-SHLD-NEXT: addl $40, %esp
-; X86-SHLD-NEXT: popl %ebx
-; X86-SHLD-NEXT: retl
+; X64-BMI2-LABEL: load_1byte_chunk_of_16byte_alloca:
+; X64-BMI2: # %bb.0:
+; X64-BMI2-NEXT: movq %rsi, %rcx
+; X64-BMI2-NEXT: shll $3, %ecx
+; X64-BMI2-NEXT: movdqu (%rdi), %xmm0
+; X64-BMI2-NEXT: movq %xmm0, %rax
+; X64-BMI2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; X64-BMI2-NEXT: movq %xmm0, %rsi
+; X64-BMI2-NEXT: shrdq %cl, %rsi, %rax
+; X64-BMI2-NEXT: shrxq %rcx, %rsi, %rsi
+; X64-BMI2-NEXT: testb $64, %cl
+; X64-BMI2-NEXT: cmoveq %rax, %rsi
+; X64-BMI2-NEXT: movb %sil, (%rdx)
+; X64-BMI2-NEXT: retq
;
-; X86-HAVE-BMI2-NO-SHLD-LABEL: load_1byte_chunk_of_16byte_alloca:
-; X86-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: subl $40, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movups (%edx), %xmm0
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shll $3, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: xorps %xmm1, %xmm1
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrb $3, %dl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: andb $12, %dl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movzbl %dl, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, (%esp,%edx), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: notb %cl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 4(%esp,%edx), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %edx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %edx, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %esi, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movb %cl, (%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl $40, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: retl
+; X86-LABEL: load_1byte_chunk_of_16byte_alloca:
+; X86: # %bb.0:
+; X86-NEXT: pushl %ebx
+; X86-NEXT: subl $40, %esp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movups (%edx), %xmm0
+; X86-NEXT: shll $3, %ecx
+; X86-NEXT: xorps %xmm1, %xmm1
+; X86-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm0, (%esp)
+; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: shrb $3, %dl
+; X86-NEXT: andb $12, %dl
+; X86-NEXT: movzbl %dl, %edx
+; X86-NEXT: movl (%esp,%edx), %ebx
+; X86-NEXT: movl 4(%esp,%edx), %edx
+; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: shrdl %cl, %edx, %ebx
+; X86-NEXT: movb %bl, (%eax)
+; X86-NEXT: addl $40, %esp
+; X86-NEXT: popl %ebx
+; X86-NEXT: retl
%init = load <16 x i8>, ptr %src, align 1
%byteOff.numbits = shl nuw nsw i64 %byteOff, 3
%intermediate.val.frozen = freeze <16 x i8> %init
@@ -667,155 +416,60 @@ define void @load_1byte_chunk_of_16byte_alloca(ptr %src, i64 %byteOff, ptr %dst)
}
define void @load_2byte_chunk_of_16byte_alloca(ptr %src, i64 %byteOff, ptr %dst) nounwind {
-; X64-NO-BMI2-NO-SHLD-LABEL: load_2byte_chunk_of_16byte_alloca:
-; X64-NO-BMI2-NO-SHLD: # %bb.0:
-; X64-NO-BMI2-NO-SHLD-NEXT: shll $3, %esi
-; X64-NO-BMI2-NO-SHLD-NEXT: movdqu (%rdi), %xmm0
-; X64-NO-BMI2-NO-SHLD-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %xmm1, %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %xmm0, %rdi
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %esi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %rdi
-; X64-NO-BMI2-NO-SHLD-NEXT: leal (%rax,%rax), %r8d
-; X64-NO-BMI2-NO-SHLD-NEXT: notb %cl
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %r8
-; X64-NO-BMI2-NO-SHLD-NEXT: orl %edi, %r8d
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %esi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: testb $64, %sil
-; X64-NO-BMI2-NO-SHLD-NEXT: cmoveq %r8, %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: movw %ax, (%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: retq
-;
-; X64-NO-BMI2-HAVE-SHLD-LABEL: load_2byte_chunk_of_16byte_alloca:
-; X64-NO-BMI2-HAVE-SHLD: # %bb.0:
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rsi, %rcx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shll $3, %ecx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movdqu (%rdi), %xmm0
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %xmm0, %rax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %xmm0, %rsi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %rsi, %rax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrq %cl, %rsi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: testb $64, %cl
-; X64-NO-BMI2-HAVE-SHLD-NEXT: cmoveq %rax, %rsi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movw %si, (%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: retq
-;
-; X64-HAVE-BMI2-NO-SHLD-LABEL: load_2byte_chunk_of_16byte_alloca:
-; X64-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shll $3, %esi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movdqu (%rdi), %xmm0
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %xmm0, %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %xmm0, %rcx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, %edi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: notb %dil
-; X64-HAVE-BMI2-NO-SHLD-NEXT: leal (%rcx,%rcx), %r8d
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rdi, %r8, %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rsi, %rax, %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orl %edi, %eax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rsi, %rcx, %rcx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: testb $64, %sil
-; X64-HAVE-BMI2-NO-SHLD-NEXT: cmoveq %rax, %rcx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movw %cx, (%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: retq
-;
-; X64-HAVE-BMI2-HAVE-SHLD-LABEL: load_2byte_chunk_of_16byte_alloca:
-; X64-HAVE-BMI2-HAVE-SHLD: # %bb.0:
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rsi, %rcx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shll $3, %ecx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movdqu (%rdi), %xmm0
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %xmm0, %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %xmm0, %rsi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %rsi, %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrxq %rcx, %rsi, %rsi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: testb $64, %cl
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: cmoveq %rax, %rsi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movw %si, (%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: retq
-;
-; X86-NO-BMI2-NO-SHLD-LABEL: load_2byte_chunk_of_16byte_alloca:
-; X86-NO-BMI2-NO-SHLD: # %bb.0:
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: subl $40, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movups (%edx), %xmm0
-; X86-NO-BMI2-NO-SHLD-NEXT: shll $3, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: xorps %xmm1, %xmm1
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrb $3, %dl
-; X86-NO-BMI2-NO-SHLD-NEXT: andb $12, %dl
-; X86-NO-BMI2-NO-SHLD-NEXT: movzbl %dl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl (%esp,%edx), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 4(%esp,%edx), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: notb %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %edx, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movw %dx, (%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: addl $40, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: retl
+; X64-NO-BMI2-LABEL: load_2byte_chunk_of_16byte_alloca:
+; X64-NO-BMI2: # %bb.0:
+; X64-NO-BMI2-NEXT: movq %rsi, %rcx
+; X64-NO-BMI2-NEXT: shll $3, %ecx
+; X64-NO-BMI2-NEXT: movdqu (%rdi), %xmm0
+; X64-NO-BMI2-NEXT: movq %xmm0, %rax
+; X64-NO-BMI2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; X64-NO-BMI2-NEXT: movq %xmm0, %rsi
+; X64-NO-BMI2-NEXT: shrdq %cl, %rsi, %rax
+; X64-NO-BMI2-NEXT: shrq %cl, %rsi
+; X64-NO-BMI2-NEXT: testb $64, %cl
+; X64-NO-BMI2-NEXT: cmoveq %rax, %rsi
+; X64-NO-BMI2-NEXT: movw %si, (%rdx)
+; X64-NO-BMI2-NEXT: retq
;
-; X86-SHLD-LABEL: load_2byte_chunk_of_16byte_alloca:
-; X86-SHLD: # %bb.0:
-; X86-SHLD-NEXT: pushl %esi
-; X86-SHLD-NEXT: subl $40, %esp
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-SHLD-NEXT: movups (%edx), %xmm0
-; X86-SHLD-NEXT: shll $3, %ecx
-; X86-SHLD-NEXT: xorps %xmm1, %xmm1
-; X86-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-SHLD-NEXT: movl %ecx, %edx
-; X86-SHLD-NEXT: shrb $3, %dl
-; X86-SHLD-NEXT: andb $12, %dl
-; X86-SHLD-NEXT: movzbl %dl, %edx
-; X86-SHLD-NEXT: movl (%esp,%edx), %esi
-; X86-SHLD-NEXT: movl 4(%esp,%edx), %edx
-; X86-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-SHLD-NEXT: shrdl %cl, %edx, %esi
-; X86-SHLD-NEXT: movw %si, (%eax)
-; X86-SHLD-NEXT: addl $40, %esp
-; X86-SHLD-NEXT: popl %esi
-; X86-SHLD-NEXT: retl
+; X64-BMI2-LABEL: load_2byte_chunk_of_16byte_alloca:
+; X64-BMI2: # %bb.0:
+; X64-BMI2-NEXT: movq %rsi, %rcx
+; X64-BMI2-NEXT: shll $3, %ecx
+; X64-BMI2-NEXT: movdqu (%rdi), %xmm0
+; X64-BMI2-NEXT: movq %xmm0, %rax
+; X64-BMI2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; X64-BMI2-NEXT: movq %xmm0, %rsi
+; X64-BMI2-NEXT: shrdq %cl, %rsi, %rax
+; X64-BMI2-NEXT: shrxq %rcx, %rsi, %rsi
+; X64-BMI2-NEXT: testb $64, %cl
+; X64-BMI2-NEXT: cmoveq %rax, %rsi
+; X64-BMI2-NEXT: movw %si, (%rdx)
+; X64-BMI2-NEXT: retq
;
-; X86-HAVE-BMI2-NO-SHLD-LABEL: load_2byte_chunk_of_16byte_alloca:
-; X86-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: subl $40, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movups (%edx), %xmm0
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shll $3, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: xorps %xmm1, %xmm1
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrb $3, %dl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: andb $12, %dl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movzbl %dl, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, (%esp,%edx), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: notb %cl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 4(%esp,%edx), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %edx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %edx, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %esi, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movw %cx, (%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl $40, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: retl
+; X86-LABEL: load_2byte_chunk_of_16byte_alloca:
+; X86: # %bb.0:
+; X86-NEXT: pushl %esi
+; X86-NEXT: subl $40, %esp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movups (%edx), %xmm0
+; X86-NEXT: shll $3, %ecx
+; X86-NEXT: xorps %xmm1, %xmm1
+; X86-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm0, (%esp)
+; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: shrb $3, %dl
+; X86-NEXT: andb $12, %dl
+; X86-NEXT: movzbl %dl, %edx
+; X86-NEXT: movl (%esp,%edx), %esi
+; X86-NEXT: movl 4(%esp,%edx), %edx
+; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: shrdl %cl, %edx, %esi
+; X86-NEXT: movw %si, (%eax)
+; X86-NEXT: addl $40, %esp
+; X86-NEXT: popl %esi
+; X86-NEXT: retl
%init = load <16 x i8>, ptr %src, align 1
%byteOff.numbits = shl nuw nsw i64 %byteOff, 3
%intermediate.val.frozen = freeze <16 x i8> %init
@@ -828,155 +482,60 @@ define void @load_2byte_chunk_of_16byte_alloca(ptr %src, i64 %byteOff, ptr %dst)
}
define void @load_4byte_chunk_of_16byte_alloca(ptr %src, i64 %byteOff, ptr %dst) nounwind {
-; X64-NO-BMI2-NO-SHLD-LABEL: load_4byte_chunk_of_16byte_alloca:
-; X64-NO-BMI2-NO-SHLD: # %bb.0:
-; X64-NO-BMI2-NO-SHLD-NEXT: shll $3, %esi
-; X64-NO-BMI2-NO-SHLD-NEXT: movdqu (%rdi), %xmm0
-; X64-NO-BMI2-NO-SHLD-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %xmm1, %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %xmm0, %rdi
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %esi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %rdi
-; X64-NO-BMI2-NO-SHLD-NEXT: leal (%rax,%rax), %r8d
-; X64-NO-BMI2-NO-SHLD-NEXT: notb %cl
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %r8
-; X64-NO-BMI2-NO-SHLD-NEXT: orl %edi, %r8d
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %esi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: testb $64, %sil
-; X64-NO-BMI2-NO-SHLD-NEXT: cmoveq %r8, %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, (%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: retq
-;
-; X64-NO-BMI2-HAVE-SHLD-LABEL: load_4byte_chunk_of_16byte_alloca:
-; X64-NO-BMI2-HAVE-SHLD: # %bb.0:
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rsi, %rcx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shll $3, %ecx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movdqu (%rdi), %xmm0
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %xmm0, %rax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %xmm0, %rsi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %rsi, %rax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrq %cl, %rsi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: testb $64, %cl
-; X64-NO-BMI2-HAVE-SHLD-NEXT: cmoveq %rax, %rsi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movl %esi, (%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: retq
-;
-; X64-HAVE-BMI2-NO-SHLD-LABEL: load_4byte_chunk_of_16byte_alloca:
-; X64-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shll $3, %esi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movdqu (%rdi), %xmm0
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %xmm0, %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %xmm0, %rcx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, %edi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: notb %dil
-; X64-HAVE-BMI2-NO-SHLD-NEXT: leal (%rcx,%rcx), %r8d
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rdi, %r8, %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rsi, %rax, %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orl %edi, %eax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rsi, %rcx, %rcx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: testb $64, %sil
-; X64-HAVE-BMI2-NO-SHLD-NEXT: cmoveq %rax, %rcx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, (%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: retq
-;
-; X64-HAVE-BMI2-HAVE-SHLD-LABEL: load_4byte_chunk_of_16byte_alloca:
-; X64-HAVE-BMI2-HAVE-SHLD: # %bb.0:
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rsi, %rcx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shll $3, %ecx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movdqu (%rdi), %xmm0
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %xmm0, %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %xmm0, %rsi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %rsi, %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrxq %rcx, %rsi, %rsi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: testb $64, %cl
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: cmoveq %rax, %rsi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movl %esi, (%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: retq
-;
-; X86-NO-BMI2-NO-SHLD-LABEL: load_4byte_chunk_of_16byte_alloca:
-; X86-NO-BMI2-NO-SHLD: # %bb.0:
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: subl $40, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movups (%edx), %xmm0
-; X86-NO-BMI2-NO-SHLD-NEXT: shll $3, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: xorps %xmm1, %xmm1
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrb $3, %dl
-; X86-NO-BMI2-NO-SHLD-NEXT: andb $12, %dl
-; X86-NO-BMI2-NO-SHLD-NEXT: movzbl %dl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl (%esp,%edx), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 4(%esp,%edx), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: notb %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %edx, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, (%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: addl $40, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: retl
+; X64-NO-BMI2-LABEL: load_4byte_chunk_of_16byte_alloca:
+; X64-NO-BMI2: # %bb.0:
+; X64-NO-BMI2-NEXT: movq %rsi, %rcx
+; X64-NO-BMI2-NEXT: shll $3, %ecx
+; X64-NO-BMI2-NEXT: movdqu (%rdi), %xmm0
+; X64-NO-BMI2-NEXT: movq %xmm0, %rax
+; X64-NO-BMI2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; X64-NO-BMI2-NEXT: movq %xmm0, %rsi
+; X64-NO-BMI2-NEXT: shrdq %cl, %rsi, %rax
+; X64-NO-BMI2-NEXT: shrq %cl, %rsi
+; X64-NO-BMI2-NEXT: testb $64, %cl
+; X64-NO-BMI2-NEXT: cmoveq %rax, %rsi
+; X64-NO-BMI2-NEXT: movl %esi, (%rdx)
+; X64-NO-BMI2-NEXT: retq
;
-; X86-SHLD-LABEL: load_4byte_chunk_of_16byte_alloca:
-; X86-SHLD: # %bb.0:
-; X86-SHLD-NEXT: pushl %esi
-; X86-SHLD-NEXT: subl $40, %esp
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-SHLD-NEXT: movups (%edx), %xmm0
-; X86-SHLD-NEXT: shll $3, %ecx
-; X86-SHLD-NEXT: xorps %xmm1, %xmm1
-; X86-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-SHLD-NEXT: movl %ecx, %edx
-; X86-SHLD-NEXT: shrb $3, %dl
-; X86-SHLD-NEXT: andb $12, %dl
-; X86-SHLD-NEXT: movzbl %dl, %edx
-; X86-SHLD-NEXT: movl (%esp,%edx), %esi
-; X86-SHLD-NEXT: movl 4(%esp,%edx), %edx
-; X86-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-SHLD-NEXT: shrdl %cl, %edx, %esi
-; X86-SHLD-NEXT: movl %esi, (%eax)
-; X86-SHLD-NEXT: addl $40, %esp
-; X86-SHLD-NEXT: popl %esi
-; X86-SHLD-NEXT: retl
+; X64-BMI2-LABEL: load_4byte_chunk_of_16byte_alloca:
+; X64-BMI2: # %bb.0:
+; X64-BMI2-NEXT: movq %rsi, %rcx
+; X64-BMI2-NEXT: shll $3, %ecx
+; X64-BMI2-NEXT: movdqu (%rdi), %xmm0
+; X64-BMI2-NEXT: movq %xmm0, %rax
+; X64-BMI2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; X64-BMI2-NEXT: movq %xmm0, %rsi
+; X64-BMI2-NEXT: shrdq %cl, %rsi, %rax
+; X64-BMI2-NEXT: shrxq %rcx, %rsi, %rsi
+; X64-BMI2-NEXT: testb $64, %cl
+; X64-BMI2-NEXT: cmoveq %rax, %rsi
+; X64-BMI2-NEXT: movl %esi, (%rdx)
+; X64-BMI2-NEXT: retq
;
-; X86-HAVE-BMI2-NO-SHLD-LABEL: load_4byte_chunk_of_16byte_alloca:
-; X86-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: subl $40, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movups (%edx), %xmm0
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shll $3, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: xorps %xmm1, %xmm1
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrb $3, %dl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: andb $12, %dl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movzbl %dl, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, (%esp,%edx), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: notb %cl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 4(%esp,%edx), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %edx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %edx, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %esi, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, (%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl $40, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: retl
+; X86-LABEL: load_4byte_chunk_of_16byte_alloca:
+; X86: # %bb.0:
+; X86-NEXT: pushl %esi
+; X86-NEXT: subl $40, %esp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movups (%edx), %xmm0
+; X86-NEXT: shll $3, %ecx
+; X86-NEXT: xorps %xmm1, %xmm1
+; X86-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm0, (%esp)
+; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: shrb $3, %dl
+; X86-NEXT: andb $12, %dl
+; X86-NEXT: movzbl %dl, %edx
+; X86-NEXT: movl (%esp,%edx), %esi
+; X86-NEXT: movl 4(%esp,%edx), %edx
+; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: shrdl %cl, %edx, %esi
+; X86-NEXT: movl %esi, (%eax)
+; X86-NEXT: addl $40, %esp
+; X86-NEXT: popl %esi
+; X86-NEXT: retl
%init = load <16 x i8>, ptr %src, align 1
%byteOff.numbits = shl nuw nsw i64 %byteOff, 3
%intermediate.val.frozen = freeze <16 x i8> %init
@@ -989,194 +548,68 @@ define void @load_4byte_chunk_of_16byte_alloca(ptr %src, i64 %byteOff, ptr %dst)
}
define void @load_8byte_chunk_of_16byte_alloca(ptr %src, i64 %byteOff, ptr %dst) nounwind {
-; X64-NO-BMI2-NO-SHLD-LABEL: load_8byte_chunk_of_16byte_alloca:
-; X64-NO-BMI2-NO-SHLD: # %bb.0:
-; X64-NO-BMI2-NO-SHLD-NEXT: shll $3, %esi
-; X64-NO-BMI2-NO-SHLD-NEXT: movdqu (%rdi), %xmm0
-; X64-NO-BMI2-NO-SHLD-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %xmm1, %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %xmm0, %rdi
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %esi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %rdi
-; X64-NO-BMI2-NO-SHLD-NEXT: notb %cl
-; X64-NO-BMI2-NO-SHLD-NEXT: leaq (%rax,%rax), %r8
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %r8
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %rdi, %r8
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %esi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: testb $64, %sil
-; X64-NO-BMI2-NO-SHLD-NEXT: cmoveq %r8, %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rax, (%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: retq
-;
-; X64-NO-BMI2-HAVE-SHLD-LABEL: load_8byte_chunk_of_16byte_alloca:
-; X64-NO-BMI2-HAVE-SHLD: # %bb.0:
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rsi, %rcx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shll $3, %ecx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movdqu (%rdi), %xmm0
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %xmm0, %rax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %xmm0, %rsi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %rsi, %rax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrq %cl, %rsi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: testb $64, %cl
-; X64-NO-BMI2-HAVE-SHLD-NEXT: cmoveq %rax, %rsi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rsi, (%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: retq
-;
-; X64-HAVE-BMI2-NO-SHLD-LABEL: load_8byte_chunk_of_16byte_alloca:
-; X64-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shll $3, %esi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movdqu (%rdi), %xmm0
-; X64-HAVE-BMI2-NO-SHLD-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %xmm1, %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %xmm0, %rcx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rsi, %rcx, %rcx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, %edi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: notb %dil
-; X64-HAVE-BMI2-NO-SHLD-NEXT: leaq (%rax,%rax), %r8
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rdi, %r8, %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %rcx, %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rsi, %rax, %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: testb $64, %sil
-; X64-HAVE-BMI2-NO-SHLD-NEXT: cmoveq %rdi, %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rax, (%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: retq
-;
-; X64-HAVE-BMI2-HAVE-SHLD-LABEL: load_8byte_chunk_of_16byte_alloca:
-; X64-HAVE-BMI2-HAVE-SHLD: # %bb.0:
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rsi, %rcx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shll $3, %ecx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movdqu (%rdi), %xmm0
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %xmm0, %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %xmm0, %rsi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %rsi, %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrxq %rcx, %rsi, %rsi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: testb $64, %cl
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: cmoveq %rax, %rsi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rsi, (%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: retq
-;
-; X86-NO-BMI2-NO-SHLD-LABEL: load_8byte_chunk_of_16byte_alloca:
-; X86-NO-BMI2-NO-SHLD: # %bb.0:
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: subl $44, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movups (%ecx), %xmm0
-; X86-NO-BMI2-NO-SHLD-NEXT: shll $3, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: xorps %xmm1, %xmm1
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrb $3, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: andb $12, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: movzbl %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl (%esp,%ebx), %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 4(%esp,%ebx), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: notb %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%esi,%esi), %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %ebp, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: andb $24, %al
-; X86-NO-BMI2-NO-SHLD-NEXT: notb %al
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 8(%esp,%ebx), %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %ebx, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, 4(%edx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, (%edx)
-; X86-NO-BMI2-NO-SHLD-NEXT: addl $44, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: retl
+; X64-NO-BMI2-LABEL: load_8byte_chunk_of_16byte_alloca:
+; X64-NO-BMI2: # %bb.0:
+; X64-NO-BMI2-NEXT: movq %rsi, %rcx
+; X64-NO-BMI2-NEXT: shll $3, %ecx
+; X64-NO-BMI2-NEXT: movdqu (%rdi), %xmm0
+; X64-NO-BMI2-NEXT: movq %xmm0, %rax
+; X64-NO-BMI2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; X64-NO-BMI2-NEXT: movq %xmm0, %rsi
+; X64-NO-BMI2-NEXT: shrdq %cl, %rsi, %rax
+; X64-NO-BMI2-NEXT: shrq %cl, %rsi
+; X64-NO-BMI2-NEXT: testb $64, %cl
+; X64-NO-BMI2-NEXT: cmoveq %rax, %rsi
+; X64-NO-BMI2-NEXT: movq %rsi, (%rdx)
+; X64-NO-BMI2-NEXT: retq
;
-; X86-SHLD-LABEL: load_8byte_chunk_of_16byte_alloca:
-; X86-SHLD: # %bb.0:
-; X86-SHLD-NEXT: pushl %ebx
-; X86-SHLD-NEXT: pushl %edi
-; X86-SHLD-NEXT: pushl %esi
-; X86-SHLD-NEXT: subl $32, %esp
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-SHLD-NEXT: movups (%edx), %xmm0
-; X86-SHLD-NEXT: shll $3, %ecx
-; X86-SHLD-NEXT: xorps %xmm1, %xmm1
-; X86-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-SHLD-NEXT: movl %ecx, %edx
-; X86-SHLD-NEXT: shrb $3, %dl
-; X86-SHLD-NEXT: andb $12, %dl
-; X86-SHLD-NEXT: movzbl %dl, %edx
-; X86-SHLD-NEXT: movl 8(%esp,%edx), %esi
-; X86-SHLD-NEXT: movl (%esp,%edx), %edi
-; X86-SHLD-NEXT: movl 4(%esp,%edx), %edx
-; X86-SHLD-NEXT: movl %edx, %ebx
-; X86-SHLD-NEXT: shrdl %cl, %esi, %ebx
-; X86-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-SHLD-NEXT: shrdl %cl, %edx, %edi
-; X86-SHLD-NEXT: movl %ebx, 4(%eax)
-; X86-SHLD-NEXT: movl %edi, (%eax)
-; X86-SHLD-NEXT: addl $32, %esp
-; X86-SHLD-NEXT: popl %esi
-; X86-SHLD-NEXT: popl %edi
-; X86-SHLD-NEXT: popl %ebx
-; X86-SHLD-NEXT: retl
+; X64-BMI2-LABEL: load_8byte_chunk_of_16byte_alloca:
+; X64-BMI2: # %bb.0:
+; X64-BMI2-NEXT: movq %rsi, %rcx
+; X64-BMI2-NEXT: shll $3, %ecx
+; X64-BMI2-NEXT: movdqu (%rdi), %xmm0
+; X64-BMI2-NEXT: movq %xmm0, %rax
+; X64-BMI2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; X64-BMI2-NEXT: movq %xmm0, %rsi
+; X64-BMI2-NEXT: shrdq %cl, %rsi, %rax
+; X64-BMI2-NEXT: shrxq %rcx, %rsi, %rsi
+; X64-BMI2-NEXT: testb $64, %cl
+; X64-BMI2-NEXT: cmoveq %rax, %rsi
+; X64-BMI2-NEXT: movq %rsi, (%rdx)
+; X64-BMI2-NEXT: retq
;
-; X86-HAVE-BMI2-NO-SHLD-LABEL: load_8byte_chunk_of_16byte_alloca:
-; X86-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: subl $44, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movups (%edx), %xmm0
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shll $3, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: xorps %xmm1, %xmm1
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrb $3, %dl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: andb $12, %dl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movzbl %dl, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, (%esp,%edx), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: notb %bl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 4(%esp,%edx), %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 8(%esp,%edx), %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%edi,%edi), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ebx, %edx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %esi, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, %edi, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: andb $24, %cl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: notb %cl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %ebp, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %ebp, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %esi, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, 4(%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edx, (%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl $44, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: retl
+; X86-LABEL: load_8byte_chunk_of_16byte_alloca:
+; X86: # %bb.0:
+; X86-NEXT: pushl %ebx
+; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %esi
+; X86-NEXT: subl $32, %esp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movups (%edx), %xmm0
+; X86-NEXT: shll $3, %ecx
+; X86-NEXT: xorps %xmm1, %xmm1
+; X86-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm0, (%esp)
+; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: shrb $3, %dl
+; X86-NEXT: andb $12, %dl
+; X86-NEXT: movzbl %dl, %edx
+; X86-NEXT: movl 8(%esp,%edx), %esi
+; X86-NEXT: movl (%esp,%edx), %edi
+; X86-NEXT: movl 4(%esp,%edx), %edx
+; X86-NEXT: movl %edx, %ebx
+; X86-NEXT: shrdl %cl, %esi, %ebx
+; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: shrdl %cl, %edx, %edi
+; X86-NEXT: movl %ebx, 4(%eax)
+; X86-NEXT: movl %edi, (%eax)
+; X86-NEXT: addl $32, %esp
+; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
+; X86-NEXT: popl %ebx
+; X86-NEXT: retl
%init = load <16 x i8>, ptr %src, align 1
%byteOff.numbits = shl nuw nsw i64 %byteOff, 3
%intermediate.val.frozen = freeze <16 x i8> %init
@@ -1209,92 +642,32 @@ define void @load_1byte_chunk_of_32byte_alloca(ptr %src, i64 %byteOff, ptr %dst)
; X64-NEXT: movb %al, (%rdx)
; X64-NEXT: retq
;
-; X86-NO-BMI2-NO-SHLD-LABEL: load_1byte_chunk_of_32byte_alloca:
-; X86-NO-BMI2-NO-SHLD: # %bb.0:
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: subl $72, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movups (%edx), %xmm0
-; X86-NO-BMI2-NO-SHLD-NEXT: movups 16(%edx), %xmm1
-; X86-NO-BMI2-NO-SHLD-NEXT: shll $3, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: xorps %xmm2, %xmm2
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrb $5, %dl
-; X86-NO-BMI2-NO-SHLD-NEXT: movzbl %dl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl (%esp,%edx,4), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 4(%esp,%edx,4), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: notb %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %edx, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %dl, (%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: addl $72, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: retl
-;
-; X86-SHLD-LABEL: load_1byte_chunk_of_32byte_alloca:
-; X86-SHLD: # %bb.0:
-; X86-SHLD-NEXT: pushl %ebx
-; X86-SHLD-NEXT: subl $72, %esp
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-SHLD-NEXT: movups (%edx), %xmm0
-; X86-SHLD-NEXT: movups 16(%edx), %xmm1
-; X86-SHLD-NEXT: shll $3, %ecx
-; X86-SHLD-NEXT: xorps %xmm2, %xmm2
-; X86-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-SHLD-NEXT: movl %ecx, %edx
-; X86-SHLD-NEXT: shrb $5, %dl
-; X86-SHLD-NEXT: movzbl %dl, %edx
-; X86-SHLD-NEXT: movl (%esp,%edx,4), %ebx
-; X86-SHLD-NEXT: movl 4(%esp,%edx,4), %edx
-; X86-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-SHLD-NEXT: shrdl %cl, %edx, %ebx
-; X86-SHLD-NEXT: movb %bl, (%eax)
-; X86-SHLD-NEXT: addl $72, %esp
-; X86-SHLD-NEXT: popl %ebx
-; X86-SHLD-NEXT: retl
-;
-; X86-HAVE-BMI2-NO-SHLD-LABEL: load_1byte_chunk_of_32byte_alloca:
-; X86-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: subl $72, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movups (%edx), %xmm0
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movups 16(%edx), %xmm1
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shll $3, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: xorps %xmm2, %xmm2
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrb $5, %dl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movzbl %dl, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, (%esp,%edx,4), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: notb %cl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 4(%esp,%edx,4), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %edx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %edx, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %esi, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movb %cl, (%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl $72, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: retl
+; X86-LABEL: load_1byte_chunk_of_32byte_alloca:
+; X86: # %bb.0:
+; X86-NEXT: pushl %ebx
+; X86-NEXT: subl $72, %esp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movups (%edx), %xmm0
+; X86-NEXT: movups 16(%edx), %xmm1
+; X86-NEXT: shll $3, %ecx
+; X86-NEXT: xorps %xmm2, %xmm2
+; X86-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm0, (%esp)
+; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: shrb $5, %dl
+; X86-NEXT: movzbl %dl, %edx
+; X86-NEXT: movl (%esp,%edx,4), %ebx
+; X86-NEXT: movl 4(%esp,%edx,4), %edx
+; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: shrdl %cl, %edx, %ebx
+; X86-NEXT: movb %bl, (%eax)
+; X86-NEXT: addl $72, %esp
+; X86-NEXT: popl %ebx
+; X86-NEXT: retl
%init = load <32 x i8>, ptr %src, align 1
%byteOff.numbits = shl nuw nsw i64 %byteOff, 3
%intermediate.val.frozen = freeze <32 x i8> %init
@@ -1357,92 +730,32 @@ define void @load_2byte_chunk_of_32byte_alloca(ptr %src, i64 %byteOff, ptr %dst)
; X64-BMI2-NEXT: movw %cx, (%rdx)
; X64-BMI2-NEXT: retq
;
-; X86-NO-BMI2-NO-SHLD-LABEL: load_2byte_chunk_of_32byte_alloca:
-; X86-NO-BMI2-NO-SHLD: # %bb.0:
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: subl $72, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movups (%edx), %xmm0
-; X86-NO-BMI2-NO-SHLD-NEXT: movups 16(%edx), %xmm1
-; X86-NO-BMI2-NO-SHLD-NEXT: shll $3, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: xorps %xmm2, %xmm2
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrb $5, %dl
-; X86-NO-BMI2-NO-SHLD-NEXT: movzbl %dl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl (%esp,%edx,4), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 4(%esp,%edx,4), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: notb %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %edx, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movw %dx, (%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: addl $72, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: retl
-;
-; X86-SHLD-LABEL: load_2byte_chunk_of_32byte_alloca:
-; X86-SHLD: # %bb.0:
-; X86-SHLD-NEXT: pushl %esi
-; X86-SHLD-NEXT: subl $72, %esp
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-SHLD-NEXT: movups (%edx), %xmm0
-; X86-SHLD-NEXT: movups 16(%edx), %xmm1
-; X86-SHLD-NEXT: shll $3, %ecx
-; X86-SHLD-NEXT: xorps %xmm2, %xmm2
-; X86-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-SHLD-NEXT: movl %ecx, %edx
-; X86-SHLD-NEXT: shrb $5, %dl
-; X86-SHLD-NEXT: movzbl %dl, %edx
-; X86-SHLD-NEXT: movl (%esp,%edx,4), %esi
-; X86-SHLD-NEXT: movl 4(%esp,%edx,4), %edx
-; X86-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-SHLD-NEXT: shrdl %cl, %edx, %esi
-; X86-SHLD-NEXT: movw %si, (%eax)
-; X86-SHLD-NEXT: addl $72, %esp
-; X86-SHLD-NEXT: popl %esi
-; X86-SHLD-NEXT: retl
-;
-; X86-HAVE-BMI2-NO-SHLD-LABEL: load_2byte_chunk_of_32byte_alloca:
-; X86-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: subl $72, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movups (%edx), %xmm0
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movups 16(%edx), %xmm1
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shll $3, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: xorps %xmm2, %xmm2
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrb $5, %dl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movzbl %dl, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, (%esp,%edx,4), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: notb %cl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 4(%esp,%edx,4), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %edx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %edx, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %esi, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movw %cx, (%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl $72, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: retl
+; X86-LABEL: load_2byte_chunk_of_32byte_alloca:
+; X86: # %bb.0:
+; X86-NEXT: pushl %esi
+; X86-NEXT: subl $72, %esp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movups (%edx), %xmm0
+; X86-NEXT: movups 16(%edx), %xmm1
+; X86-NEXT: shll $3, %ecx
+; X86-NEXT: xorps %xmm2, %xmm2
+; X86-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm0, (%esp)
+; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: shrb $5, %dl
+; X86-NEXT: movzbl %dl, %edx
+; X86-NEXT: movl (%esp,%edx,4), %esi
+; X86-NEXT: movl 4(%esp,%edx,4), %edx
+; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: shrdl %cl, %edx, %esi
+; X86-NEXT: movw %si, (%eax)
+; X86-NEXT: addl $72, %esp
+; X86-NEXT: popl %esi
+; X86-NEXT: retl
%init = load <32 x i8>, ptr %src, align 1
%byteOff.numbits = shl nuw nsw i64 %byteOff, 3
%intermediate.val.frozen = freeze <32 x i8> %init
@@ -1504,92 +817,32 @@ define void @load_4byte_chunk_of_32byte_alloca(ptr %src, i64 %byteOff, ptr %dst)
; X64-BMI2-NEXT: movl %ecx, (%rdx)
; X64-BMI2-NEXT: retq
;
-; X86-NO-BMI2-NO-SHLD-LABEL: load_4byte_chunk_of_32byte_alloca:
-; X86-NO-BMI2-NO-SHLD: # %bb.0:
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: subl $72, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movups (%edx), %xmm0
-; X86-NO-BMI2-NO-SHLD-NEXT: movups 16(%edx), %xmm1
-; X86-NO-BMI2-NO-SHLD-NEXT: shll $3, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: xorps %xmm2, %xmm2
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ecx, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrb $5, %dl
-; X86-NO-BMI2-NO-SHLD-NEXT: movzbl %dl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl (%esp,%edx,4), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 4(%esp,%edx,4), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: notb %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %edx, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edx, (%eax)
-; X86-NO-BMI2-NO-SHLD-NEXT: addl $72, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: retl
-;
-; X86-SHLD-LABEL: load_4byte_chunk_of_32byte_alloca:
-; X86-SHLD: # %bb.0:
-; X86-SHLD-NEXT: pushl %esi
-; X86-SHLD-NEXT: subl $72, %esp
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-SHLD-NEXT: movups (%edx), %xmm0
-; X86-SHLD-NEXT: movups 16(%edx), %xmm1
-; X86-SHLD-NEXT: shll $3, %ecx
-; X86-SHLD-NEXT: xorps %xmm2, %xmm2
-; X86-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-SHLD-NEXT: movl %ecx, %edx
-; X86-SHLD-NEXT: shrb $5, %dl
-; X86-SHLD-NEXT: movzbl %dl, %edx
-; X86-SHLD-NEXT: movl (%esp,%edx,4), %esi
-; X86-SHLD-NEXT: movl 4(%esp,%edx,4), %edx
-; X86-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-SHLD-NEXT: shrdl %cl, %edx, %esi
-; X86-SHLD-NEXT: movl %esi, (%eax)
-; X86-SHLD-NEXT: addl $72, %esp
-; X86-SHLD-NEXT: popl %esi
-; X86-SHLD-NEXT: retl
-;
-; X86-HAVE-BMI2-NO-SHLD-LABEL: load_4byte_chunk_of_32byte_alloca:
-; X86-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: subl $72, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movups (%edx), %xmm0
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movups 16(%edx), %xmm1
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shll $3, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: xorps %xmm2, %xmm2
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrb $5, %dl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movzbl %dl, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, (%esp,%edx,4), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: notb %cl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 4(%esp,%edx,4), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %edx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %edx, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %esi, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, (%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl $72, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: retl
+; X86-LABEL: load_4byte_chunk_of_32byte_alloca:
+; X86: # %bb.0:
+; X86-NEXT: pushl %esi
+; X86-NEXT: subl $72, %esp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movups (%edx), %xmm0
+; X86-NEXT: movups 16(%edx), %xmm1
+; X86-NEXT: shll $3, %ecx
+; X86-NEXT: xorps %xmm2, %xmm2
+; X86-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm0, (%esp)
+; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: shrb $5, %dl
+; X86-NEXT: movzbl %dl, %edx
+; X86-NEXT: movl (%esp,%edx,4), %esi
+; X86-NEXT: movl 4(%esp,%edx,4), %edx
+; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: shrdl %cl, %edx, %esi
+; X86-NEXT: movl %esi, (%eax)
+; X86-NEXT: addl $72, %esp
+; X86-NEXT: popl %esi
+; X86-NEXT: retl
%init = load <32 x i8>, ptr %src, align 1
%byteOff.numbits = shl nuw nsw i64 %byteOff, 3
%intermediate.val.frozen = freeze <32 x i8> %init
@@ -1602,197 +855,60 @@ define void @load_4byte_chunk_of_32byte_alloca(ptr %src, i64 %byteOff, ptr %dst)
}
define void @load_8byte_chunk_of_32byte_alloca(ptr %src, i64 %byteOff, ptr %dst) nounwind {
-; X64-NO-BMI2-NO-SHLD-LABEL: load_8byte_chunk_of_32byte_alloca:
-; X64-NO-BMI2-NO-SHLD: # %bb.0:
-; X64-NO-BMI2-NO-SHLD-NEXT: movups (%rdi), %xmm0
-; X64-NO-BMI2-NO-SHLD-NEXT: movups 16(%rdi), %xmm1
-; X64-NO-BMI2-NO-SHLD-NEXT: leal (,%rsi,8), %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: xorps %xmm2, %xmm2
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %ecx, %eax
-; X64-NO-BMI2-NO-SHLD-NEXT: shrb $6, %al
-; X64-NO-BMI2-NO-SHLD-NEXT: movzbl %al, %eax
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -72(%rsp,%rax,8), %rsi
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -64(%rsp,%rax,8), %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %rsi
-; X64-NO-BMI2-NO-SHLD-NEXT: notb %cl
-; X64-NO-BMI2-NO-SHLD-NEXT: addq %rax, %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %rsi, %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rax, (%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: retq
-;
-; X64-SHLD-LABEL: load_8byte_chunk_of_32byte_alloca:
-; X64-SHLD: # %bb.0:
-; X64-SHLD-NEXT: movups (%rdi), %xmm0
-; X64-SHLD-NEXT: movups 16(%rdi), %xmm1
-; X64-SHLD-NEXT: leal (,%rsi,8), %ecx
-; X64-SHLD-NEXT: xorps %xmm2, %xmm2
-; X64-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-SHLD-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; X64-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-SHLD-NEXT: movl %ecx, %eax
-; X64-SHLD-NEXT: shrb $6, %al
-; X64-SHLD-NEXT: movzbl %al, %eax
-; X64-SHLD-NEXT: movq -72(%rsp,%rax,8), %rsi
-; X64-SHLD-NEXT: movq -64(%rsp,%rax,8), %rax
-; X64-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X64-SHLD-NEXT: shrdq %cl, %rax, %rsi
-; X64-SHLD-NEXT: movq %rsi, (%rdx)
-; X64-SHLD-NEXT: retq
-;
-; X64-HAVE-BMI2-NO-SHLD-LABEL: load_8byte_chunk_of_32byte_alloca:
-; X64-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movups (%rdi), %xmm0
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movups 16(%rdi), %xmm1
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shll $3, %esi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: xorps %xmm2, %xmm2
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, %eax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrb $6, %al
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movzbl %al, %eax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rsi, -72(%rsp,%rax,8), %rcx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: notb %sil
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq -64(%rsp,%rax,8), %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: addq %rax, %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rsi, %rax, %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %rcx, %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rax, (%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: retq
-;
-; X86-NO-BMI2-NO-SHLD-LABEL: load_8byte_chunk_of_32byte_alloca:
-; X86-NO-BMI2-NO-SHLD: # %bb.0:
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: subl $76, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movups (%ecx), %xmm0
-; X86-NO-BMI2-NO-SHLD-NEXT: movups 16(%ecx), %xmm1
-; X86-NO-BMI2-NO-SHLD-NEXT: shll $3, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: xorps %xmm2, %xmm2
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrb $5, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: movzbl %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl (%esp,%ebx,4), %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 4(%esp,%ebx,4), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: notb %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%esi,%esi), %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %ebp, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: andb $24, %al
-; X86-NO-BMI2-NO-SHLD-NEXT: notb %al
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 8(%esp,%ebx,4), %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %ebx, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, 4(%edx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %edi, (%edx)
-; X86-NO-BMI2-NO-SHLD-NEXT: addl $76, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: retl
-;
-; X86-SHLD-LABEL: load_8byte_chunk_of_32byte_alloca:
-; X86-SHLD: # %bb.0:
-; X86-SHLD-NEXT: pushl %ebx
-; X86-SHLD-NEXT: pushl %edi
-; X86-SHLD-NEXT: pushl %esi
-; X86-SHLD-NEXT: subl $64, %esp
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-SHLD-NEXT: movups (%edx), %xmm0
-; X86-SHLD-NEXT: movups 16(%edx), %xmm1
-; X86-SHLD-NEXT: shll $3, %ecx
-; X86-SHLD-NEXT: xorps %xmm2, %xmm2
-; X86-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-SHLD-NEXT: movl %ecx, %edx
-; X86-SHLD-NEXT: shrb $5, %dl
-; X86-SHLD-NEXT: movzbl %dl, %edx
-; X86-SHLD-NEXT: movl 8(%esp,%edx,4), %esi
-; X86-SHLD-NEXT: movl (%esp,%edx,4), %edi
-; X86-SHLD-NEXT: movl 4(%esp,%edx,4), %edx
-; X86-SHLD-NEXT: movl %edx, %ebx
-; X86-SHLD-NEXT: shrdl %cl, %esi, %ebx
-; X86-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-SHLD-NEXT: shrdl %cl, %edx, %edi
-; X86-SHLD-NEXT: movl %ebx, 4(%eax)
-; X86-SHLD-NEXT: movl %edi, (%eax)
-; X86-SHLD-NEXT: addl $64, %esp
-; X86-SHLD-NEXT: popl %esi
-; X86-SHLD-NEXT: popl %edi
-; X86-SHLD-NEXT: popl %ebx
-; X86-SHLD-NEXT: retl
+; X64-LABEL: load_8byte_chunk_of_32byte_alloca:
+; X64: # %bb.0:
+; X64-NEXT: movups (%rdi), %xmm0
+; X64-NEXT: movups 16(%rdi), %xmm1
+; X64-NEXT: leal (,%rsi,8), %ecx
+; X64-NEXT: xorps %xmm2, %xmm2
+; X64-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NEXT: movl %ecx, %eax
+; X64-NEXT: shrb $6, %al
+; X64-NEXT: movzbl %al, %eax
+; X64-NEXT: movq -72(%rsp,%rax,8), %rsi
+; X64-NEXT: movq -64(%rsp,%rax,8), %rax
+; X64-NEXT: # kill: def $cl killed $cl killed $ecx
+; X64-NEXT: shrdq %cl, %rax, %rsi
+; X64-NEXT: movq %rsi, (%rdx)
+; X64-NEXT: retq
;
-; X86-HAVE-BMI2-NO-SHLD-LABEL: load_8byte_chunk_of_32byte_alloca:
-; X86-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: subl $76, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movups (%edx), %xmm0
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movups 16(%edx), %xmm1
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shll $3, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: xorps %xmm2, %xmm2
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, (%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrb $5, %dl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movzbl %dl, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, (%esp,%edx,4), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: notb %bl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 4(%esp,%edx,4), %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 8(%esp,%edx,4), %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%edi,%edi), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ebx, %edx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %esi, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, %edi, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: andb $24, %cl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: notb %cl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %ebp, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ecx, %ebp, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %esi, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, 4(%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edx, (%eax)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl $76, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: retl
+; X86-LABEL: load_8byte_chunk_of_32byte_alloca:
+; X86: # %bb.0:
+; X86-NEXT: pushl %ebx
+; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %esi
+; X86-NEXT: subl $64, %esp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movups (%edx), %xmm0
+; X86-NEXT: movups 16(%edx), %xmm1
+; X86-NEXT: shll $3, %ecx
+; X86-NEXT: xorps %xmm2, %xmm2
+; X86-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm0, (%esp)
+; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: shrb $5, %dl
+; X86-NEXT: movzbl %dl, %edx
+; X86-NEXT: movl 8(%esp,%edx,4), %esi
+; X86-NEXT: movl (%esp,%edx,4), %edi
+; X86-NEXT: movl 4(%esp,%edx,4), %edx
+; X86-NEXT: movl %edx, %ebx
+; X86-NEXT: shrdl %cl, %esi, %ebx
+; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: shrdl %cl, %edx, %edi
+; X86-NEXT: movl %ebx, 4(%eax)
+; X86-NEXT: movl %edi, (%eax)
+; X86-NEXT: addl $64, %esp
+; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
+; X86-NEXT: popl %ebx
+; X86-NEXT: retl
%init = load <32 x i8>, ptr %src, align 1
%byteOff.numbits = shl nuw nsw i64 %byteOff, 3
%intermediate.val.frozen = freeze <32 x i8> %init
@@ -1805,299 +921,107 @@ define void @load_8byte_chunk_of_32byte_alloca(ptr %src, i64 %byteOff, ptr %dst)
}
define void @load_16byte_chunk_of_32byte_alloca(ptr %src, i64 %byteOff, ptr %dst) nounwind {
-; X64-NO-BMI2-NO-SHLD-LABEL: load_16byte_chunk_of_32byte_alloca:
-; X64-NO-BMI2-NO-SHLD: # %bb.0:
-; X64-NO-BMI2-NO-SHLD-NEXT: movups (%rdi), %xmm0
-; X64-NO-BMI2-NO-SHLD-NEXT: movups 16(%rdi), %xmm1
-; X64-NO-BMI2-NO-SHLD-NEXT: leal (,%rsi,8), %eax
-; X64-NO-BMI2-NO-SHLD-NEXT: xorps %xmm2, %xmm2
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrb $6, %cl
-; X64-NO-BMI2-NO-SHLD-NEXT: movzbl %cl, %edi
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -72(%rsp,%rdi,8), %r8
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -64(%rsp,%rdi,8), %r9
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %r8
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %esi
-; X64-NO-BMI2-NO-SHLD-NEXT: notb %sil
-; X64-NO-BMI2-NO-SHLD-NEXT: leaq (%r9,%r9), %r10
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %esi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %r10
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %r8, %r10
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shrq %cl, %r9
-; X64-NO-BMI2-NO-SHLD-NEXT: movq -56(%rsp,%rdi,8), %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: addq %rax, %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: movl %esi, %ecx
-; X64-NO-BMI2-NO-SHLD-NEXT: shlq %cl, %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: orq %r9, %rax
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %rax, 8(%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: movq %r10, (%rdx)
-; X64-NO-BMI2-NO-SHLD-NEXT: retq
-;
-; X64-NO-BMI2-HAVE-SHLD-LABEL: load_16byte_chunk_of_32byte_alloca:
-; X64-NO-BMI2-HAVE-SHLD: # %bb.0:
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movups (%rdi), %xmm0
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movups 16(%rdi), %xmm1
-; X64-NO-BMI2-HAVE-SHLD-NEXT: leal (,%rsi,8), %eax
-; X64-NO-BMI2-HAVE-SHLD-NEXT: xorps %xmm2, %xmm2
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrb $6, %cl
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movzbl %cl, %esi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -72(%rsp,%rsi,8), %rdi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -64(%rsp,%rsi,8), %r8
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %r8, %r9
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrq %cl, %r9
-; X64-NO-BMI2-HAVE-SHLD-NEXT: notb %cl
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq -56(%rsp,%rsi,8), %rsi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: addq %rsi, %rsi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shlq %cl, %rsi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: orq %r9, %rsi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movl %eax, %ecx
-; X64-NO-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %r8, %rdi
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rdi, (%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: movq %rsi, 8(%rdx)
-; X64-NO-BMI2-HAVE-SHLD-NEXT: retq
-;
-; X64-HAVE-BMI2-NO-SHLD-LABEL: load_16byte_chunk_of_32byte_alloca:
-; X64-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movups (%rdi), %xmm0
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movups 16(%rdi), %xmm1
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shll $3, %esi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: xorps %xmm2, %xmm2
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, %eax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movl %esi, %ecx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrb $6, %cl
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movzbl %cl, %ecx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rax, -72(%rsp,%rcx,8), %rdi
-; X64-HAVE-BMI2-NO-SHLD-NEXT: notb %sil
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq -64(%rsp,%rcx,8), %r8
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq -56(%rsp,%rcx,8), %rcx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: leaq (%r8,%r8), %r9
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rsi, %r9, %r9
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %rdi, %r9
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shrxq %rax, %r8, %rax
-; X64-HAVE-BMI2-NO-SHLD-NEXT: addq %rcx, %rcx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: shlxq %rsi, %rcx, %rcx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: orq %rax, %rcx
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %rcx, 8(%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: movq %r9, (%rdx)
-; X64-HAVE-BMI2-NO-SHLD-NEXT: retq
-;
-; X64-HAVE-BMI2-HAVE-SHLD-LABEL: load_16byte_chunk_of_32byte_alloca:
-; X64-HAVE-BMI2-HAVE-SHLD: # %bb.0:
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rsi, %rcx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movups (%rdi), %xmm0
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movups 16(%rdi), %xmm1
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shll $3, %ecx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: xorps %xmm2, %xmm2
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, %eax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrb $6, %al
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movzbl %al, %eax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -72(%rsp,%rax,8), %rsi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -64(%rsp,%rax,8), %rdi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrxq %rcx, %rdi, %r8
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movl %ecx, %r9d
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: notb %r9b
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq -56(%rsp,%rax,8), %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: addq %rax, %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shlxq %r9, %rax, %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: orq %r8, %rax
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: # kill: def $cl killed $cl killed $rcx
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: shrdq %cl, %rdi, %rsi
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rsi, (%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: movq %rax, 8(%rdx)
-; X64-HAVE-BMI2-HAVE-SHLD-NEXT: retq
-;
-; X86-NO-BMI2-NO-SHLD-LABEL: load_16byte_chunk_of_32byte_alloca:
-; X86-NO-BMI2-NO-SHLD: # %bb.0:
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: subl $92, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movups (%ecx), %xmm0
-; X86-NO-BMI2-NO-SHLD-NEXT: movups 16(%ecx), %xmm1
-; X86-NO-BMI2-NO-SHLD-NEXT: shll $3, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: xorps %xmm2, %xmm2
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrb $5, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: movzbl %cl, %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 16(%esp,%edi,4), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 20(%esp,%edi,4), %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: notb %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%ebp,%ebp), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %edx, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %al, %ch
-; X86-NO-BMI2-NO-SHLD-NEXT: andb $24, %ch
-; X86-NO-BMI2-NO-SHLD-NEXT: xorb $31, %ch
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 24(%esp,%edi,4), %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%edx,%edx), %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %ebp, %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %al, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %edx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 28(%esp,%edi,4), %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: leal (%esi,%esi), %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %edx, %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %al, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shrl %cl, %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: movl 32(%esp,%edi,4), %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: addl %eax, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movb %ch, %cl
-; X86-NO-BMI2-NO-SHLD-NEXT: shll %cl, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: orl %esi, %eax
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, 12(%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebp, 8(%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %ebx, 4(%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NO-BMI2-NO-SHLD-NEXT: movl %eax, (%ecx)
-; X86-NO-BMI2-NO-SHLD-NEXT: addl $92, %esp
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-NO-BMI2-NO-SHLD-NEXT: popl %ebp
-; X86-NO-BMI2-NO-SHLD-NEXT: retl
+; X64-NO-BMI2-LABEL: load_16byte_chunk_of_32byte_alloca:
+; X64-NO-BMI2: # %bb.0:
+; X64-NO-BMI2-NEXT: movups (%rdi), %xmm0
+; X64-NO-BMI2-NEXT: movups 16(%rdi), %xmm1
+; X64-NO-BMI2-NEXT: leal (,%rsi,8), %eax
+; X64-NO-BMI2-NEXT: xorps %xmm2, %xmm2
+; X64-NO-BMI2-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NO-BMI2-NEXT: movl %eax, %ecx
+; X64-NO-BMI2-NEXT: shrb $6, %cl
+; X64-NO-BMI2-NEXT: movzbl %cl, %esi
+; X64-NO-BMI2-NEXT: movq -72(%rsp,%rsi,8), %rdi
+; X64-NO-BMI2-NEXT: movq -64(%rsp,%rsi,8), %r8
+; X64-NO-BMI2-NEXT: movq %r8, %r9
+; X64-NO-BMI2-NEXT: movl %eax, %ecx
+; X64-NO-BMI2-NEXT: shrq %cl, %r9
+; X64-NO-BMI2-NEXT: notb %cl
+; X64-NO-BMI2-NEXT: movq -56(%rsp,%rsi,8), %rsi
+; X64-NO-BMI2-NEXT: addq %rsi, %rsi
+; X64-NO-BMI2-NEXT: shlq %cl, %rsi
+; X64-NO-BMI2-NEXT: orq %r9, %rsi
+; X64-NO-BMI2-NEXT: movl %eax, %ecx
+; X64-NO-BMI2-NEXT: shrdq %cl, %r8, %rdi
+; X64-NO-BMI2-NEXT: movq %rdi, (%rdx)
+; X64-NO-BMI2-NEXT: movq %rsi, 8(%rdx)
+; X64-NO-BMI2-NEXT: retq
;
-; X86-SHLD-LABEL: load_16byte_chunk_of_32byte_alloca:
-; X86-SHLD: # %bb.0:
-; X86-SHLD-NEXT: pushl %ebp
-; X86-SHLD-NEXT: pushl %ebx
-; X86-SHLD-NEXT: pushl %edi
-; X86-SHLD-NEXT: pushl %esi
-; X86-SHLD-NEXT: subl $92, %esp
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-SHLD-NEXT: movups (%eax), %xmm0
-; X86-SHLD-NEXT: movups 16(%eax), %xmm1
-; X86-SHLD-NEXT: shll $3, %ecx
-; X86-SHLD-NEXT: xorps %xmm2, %xmm2
-; X86-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-SHLD-NEXT: movl %ecx, %eax
-; X86-SHLD-NEXT: shrb $5, %al
-; X86-SHLD-NEXT: movzbl %al, %ebx
-; X86-SHLD-NEXT: movl 24(%esp,%ebx,4), %esi
-; X86-SHLD-NEXT: movl 16(%esp,%ebx,4), %eax
-; X86-SHLD-NEXT: movl 20(%esp,%ebx,4), %edi
-; X86-SHLD-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-SHLD-NEXT: shrdl %cl, %esi, %edi
-; X86-SHLD-NEXT: movl 28(%esp,%ebx,4), %ebp
-; X86-SHLD-NEXT: shrdl %cl, %ebp, %esi
-; X86-SHLD-NEXT: movl 32(%esp,%ebx,4), %ebx
-; X86-SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-SHLD-NEXT: shrdl %cl, %ebx, %ebp
-; X86-SHLD-NEXT: movl %ebp, 12(%edx)
-; X86-SHLD-NEXT: movl %esi, 8(%edx)
-; X86-SHLD-NEXT: movl %edi, 4(%edx)
-; X86-SHLD-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-SHLD-NEXT: shrdl %cl, %esi, %eax
-; X86-SHLD-NEXT: movl %eax, (%edx)
-; X86-SHLD-NEXT: addl $92, %esp
-; X86-SHLD-NEXT: popl %esi
-; X86-SHLD-NEXT: popl %edi
-; X86-SHLD-NEXT: popl %ebx
-; X86-SHLD-NEXT: popl %ebp
-; X86-SHLD-NEXT: retl
+; X64-BMI2-LABEL: load_16byte_chunk_of_32byte_alloca:
+; X64-BMI2: # %bb.0:
+; X64-BMI2-NEXT: movq %rsi, %rcx
+; X64-BMI2-NEXT: movups (%rdi), %xmm0
+; X64-BMI2-NEXT: movups 16(%rdi), %xmm1
+; X64-BMI2-NEXT: shll $3, %ecx
+; X64-BMI2-NEXT: xorps %xmm2, %xmm2
+; X64-BMI2-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-BMI2-NEXT: movl %ecx, %eax
+; X64-BMI2-NEXT: shrb $6, %al
+; X64-BMI2-NEXT: movzbl %al, %eax
+; X64-BMI2-NEXT: movq -72(%rsp,%rax,8), %rsi
+; X64-BMI2-NEXT: movq -64(%rsp,%rax,8), %rdi
+; X64-BMI2-NEXT: shrxq %rcx, %rdi, %r8
+; X64-BMI2-NEXT: movl %ecx, %r9d
+; X64-BMI2-NEXT: notb %r9b
+; X64-BMI2-NEXT: movq -56(%rsp,%rax,8), %rax
+; X64-BMI2-NEXT: addq %rax, %rax
+; X64-BMI2-NEXT: shlxq %r9, %rax, %rax
+; X64-BMI2-NEXT: orq %r8, %rax
+; X64-BMI2-NEXT: # kill: def $cl killed $cl killed $rcx
+; X64-BMI2-NEXT: shrdq %cl, %rdi, %rsi
+; X64-BMI2-NEXT: movq %rsi, (%rdx)
+; X64-BMI2-NEXT: movq %rax, 8(%rdx)
+; X64-BMI2-NEXT: retq
;
-; X86-HAVE-BMI2-NO-SHLD-LABEL: load_16byte_chunk_of_32byte_alloca:
-; X86-HAVE-BMI2-NO-SHLD: # %bb.0:
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: pushl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: subl $92, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movups (%ecx), %xmm0
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movups 16(%ecx), %xmm1
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shll $3, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: xorps %xmm2, %xmm2
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrb $5, %dl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movzbl %dl, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ecx, 16(%esp,%esi,4), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: notb %bl
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 20(%esp,%esi,4), %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%edi,%edi), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ebx, %edx, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 24(%esp,%esi,4), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ebp, %edi, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: andb $24, %al
-; X86-HAVE-BMI2-NO-SHLD-NEXT: xorb $31, %al
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%ecx,%ecx), %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %eax, %edi, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %edx, %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 28(%esp,%esi,4), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: leal (%ecx,%ecx), %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %ebx, %edx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ebp, {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %ebx, %edx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shrxl %ebp, %ecx, %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl 32(%esp,%esi,4), %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl %esi, %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: shlxl %eax, %esi, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: orl %ecx, %eax
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, 12(%ecx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edx, 8(%ecx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %edi, 4(%ecx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-HAVE-BMI2-NO-SHLD-NEXT: movl %eax, (%ecx)
-; X86-HAVE-BMI2-NO-SHLD-NEXT: addl $92, %esp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %esi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %edi
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %ebx
-; X86-HAVE-BMI2-NO-SHLD-NEXT: popl %ebp
-; X86-HAVE-BMI2-NO-SHLD-NEXT: retl
+; X86-LABEL: load_16byte_chunk_of_32byte_alloca:
+; X86: # %bb.0:
+; X86-NEXT: pushl %ebp
+; X86-NEXT: pushl %ebx
+; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %esi
+; X86-NEXT: subl $92, %esp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movups (%eax), %xmm0
+; X86-NEXT: movups 16(%eax), %xmm1
+; X86-NEXT: shll $3, %ecx
+; X86-NEXT: xorps %xmm2, %xmm2
+; X86-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm2, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm1, {{[0-9]+}}(%esp)
+; X86-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: shrb $5, %al
+; X86-NEXT: movzbl %al, %ebx
+; X86-NEXT: movl 24(%esp,%ebx,4), %esi
+; X86-NEXT: movl 16(%esp,%ebx,4), %eax
+; X86-NEXT: movl 20(%esp,%ebx,4), %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shrdl %cl, %esi, %edi
+; X86-NEXT: movl 28(%esp,%ebx,4), %ebp
+; X86-NEXT: shrdl %cl, %ebp, %esi
+; X86-NEXT: movl 32(%esp,%ebx,4), %ebx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: shrdl %cl, %ebx, %ebp
+; X86-NEXT: movl %ebp, 12(%edx)
+; X86-NEXT: movl %esi, 8(%edx)
+; X86-NEXT: movl %edi, 4(%edx)
+; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: shrdl %cl, %esi, %eax
+; X86-NEXT: movl %eax, (%edx)
+; X86-NEXT: addl $92, %esp
+; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
+; X86-NEXT: popl %ebx
+; X86-NEXT: popl %ebp
+; X86-NEXT: retl
%init = load <32 x i8>, ptr %src, align 1
%byteOff.numbits = shl nuw nsw i64 %byteOff, 3
%intermediate.val.frozen = freeze <32 x i8> %init
@@ -2112,6 +1036,15 @@ define void @load_16byte_chunk_of_32byte_alloca(ptr %src, i64 %byteOff, ptr %dst
; no @load_32byte_chunk_of_32byte_alloca
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; ALL: {{.*}}
+; X64-HAVE-BMI2-HAVE-SHLD: {{.*}}
+; X64-HAVE-BMI2-NO-SHLD: {{.*}}
+; X64-NO-BMI2-HAVE-SHLD: {{.*}}
+; X64-NO-BMI2-NO-SHLD: {{.*}}
; X64-NO-SHLD: {{.*}}
-; X86: {{.*}}
+; X64-SHLD: {{.*}}
+; X86-HAVE-BMI2-HAVE-SHLD: {{.*}}
+; X86-HAVE-BMI2-NO-SHLD: {{.*}}
+; X86-NO-BMI2-HAVE-SHLD: {{.*}}
+; X86-NO-BMI2-NO-SHLD: {{.*}}
; X86-NO-SHLD: {{.*}}
+; X86-SHLD: {{.*}}
diff --git a/llvm/test/CodeGen/X86/x86-64-double-precision-shift-left.ll b/llvm/test/CodeGen/X86/x86-64-double-precision-shift-left.ll
index 9d72386f35271..cea7162ebcc62 100644
--- a/llvm/test/CodeGen/X86/x86-64-double-precision-shift-left.ll
+++ b/llvm/test/CodeGen/X86/x86-64-double-precision-shift-left.ll
@@ -13,8 +13,8 @@
define i64 @lshift1(i64 %a, i64 %b) nounwind readnone uwtable {
; CHECK-LABEL: lshift1:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: shrq $63, %rsi
-; CHECK-NEXT: leaq (%rsi,%rdi,2), %rax
+; CHECK-NEXT: movq %rdi, %rax
+; CHECK-NEXT: shldq $1, %rsi, %rax
; CHECK-NEXT: retq
entry:
%shl = shl i64 %a, 1
@@ -31,8 +31,8 @@ entry:
define i64 @lshift2(i64 %a, i64 %b) nounwind readnone uwtable {
; CHECK-LABEL: lshift2:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: shrq $62, %rsi
-; CHECK-NEXT: leaq (%rsi,%rdi,4), %rax
+; CHECK-NEXT: movq %rdi, %rax
+; CHECK-NEXT: shldq $2, %rsi, %rax
; CHECK-NEXT: retq
entry:
%shl = shl i64 %a, 2
@@ -49,9 +49,8 @@ entry:
define i64 @lshift7(i64 %a, i64 %b) nounwind readnone uwtable {
; CHECK-LABEL: lshift7:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: shrq $57, %rsi
-; CHECK-NEXT: shlq $7, %rdi
-; CHECK-NEXT: leaq (%rdi,%rsi), %rax
+; CHECK-NEXT: movq %rdi, %rax
+; CHECK-NEXT: shldq $7, %rsi, %rax
; CHECK-NEXT: retq
entry:
%shl = shl i64 %a, 7
@@ -68,9 +67,8 @@ entry:
define i64 @lshift63(i64 %a, i64 %b) nounwind readnone uwtable {
; CHECK-LABEL: lshift63:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: shrq %rsi
-; CHECK-NEXT: shlq $63, %rdi
-; CHECK-NEXT: leaq (%rdi,%rsi), %rax
+; CHECK-NEXT: movq %rdi, %rax
+; CHECK-NEXT: shldq $63, %rsi, %rax
; CHECK-NEXT: retq
entry:
%shl = shl i64 %a, 63
diff --git a/llvm/test/CodeGen/X86/x86-64-double-precision-shift-right.ll b/llvm/test/CodeGen/X86/x86-64-double-precision-shift-right.ll
index 42df39f98c21a..420306881e06d 100644
--- a/llvm/test/CodeGen/X86/x86-64-double-precision-shift-right.ll
+++ b/llvm/test/CodeGen/X86/x86-64-double-precision-shift-right.ll
@@ -13,9 +13,8 @@
define i64 @rshift1(i64 %a, i64 %b) nounwind readnone uwtable {
; CHECK-LABEL: rshift1:
; CHECK: # %bb.0:
-; CHECK-NEXT: shrq %rdi
-; CHECK-NEXT: shlq $63, %rsi
-; CHECK-NEXT: leaq (%rsi,%rdi), %rax
+; CHECK-NEXT: movq %rdi, %rax
+; CHECK-NEXT: shrdq $1, %rsi, %rax
; CHECK-NEXT: retq
%1 = lshr i64 %a, 1
%2 = shl i64 %b, 63
@@ -31,9 +30,8 @@ define i64 @rshift1(i64 %a, i64 %b) nounwind readnone uwtable {
define i64 @rshift2(i64 %a, i64 %b) nounwind readnone uwtable {
; CHECK-LABEL: rshift2:
; CHECK: # %bb.0:
-; CHECK-NEXT: shrq $2, %rdi
-; CHECK-NEXT: shlq $62, %rsi
-; CHECK-NEXT: leaq (%rsi,%rdi), %rax
+; CHECK-NEXT: movq %rdi, %rax
+; CHECK-NEXT: shrdq $2, %rsi, %rax
; CHECK-NEXT: retq
%1 = lshr i64 %a, 2
%2 = shl i64 %b, 62
@@ -49,9 +47,8 @@ define i64 @rshift2(i64 %a, i64 %b) nounwind readnone uwtable {
define i64 @rshift7(i64 %a, i64 %b) nounwind readnone uwtable {
; CHECK-LABEL: rshift7:
; CHECK: # %bb.0:
-; CHECK-NEXT: shrq $7, %rdi
-; CHECK-NEXT: shlq $57, %rsi
-; CHECK-NEXT: leaq (%rsi,%rdi), %rax
+; CHECK-NEXT: movq %rdi, %rax
+; CHECK-NEXT: shrdq $7, %rsi, %rax
; CHECK-NEXT: retq
%1 = lshr i64 %a, 7
%2 = shl i64 %b, 57
@@ -67,8 +64,8 @@ define i64 @rshift7(i64 %a, i64 %b) nounwind readnone uwtable {
define i64 @rshift63(i64 %a, i64 %b) nounwind readnone uwtable {
; CHECK-LABEL: rshift63:
; CHECK: # %bb.0:
-; CHECK-NEXT: shrq $63, %rdi
-; CHECK-NEXT: leaq (%rdi,%rsi,2), %rax
+; CHECK-NEXT: movq %rdi, %rax
+; CHECK-NEXT: shrdq $63, %rsi, %rax
; CHECK-NEXT: retq
%1 = lshr i64 %a, 63
%2 = shl i64 %b, 1
diff --git a/llvm/test/CodeGen/X86/x86-64-double-shifts-Oz-Os-O2.ll b/llvm/test/CodeGen/X86/x86-64-double-shifts-Oz-Os-O2.ll
index 9f888f8a7fada..46afc8f88e086 100644
--- a/llvm/test/CodeGen/X86/x86-64-double-shifts-Oz-Os-O2.ll
+++ b/llvm/test/CodeGen/X86/x86-64-double-shifts-Oz-Os-O2.ll
@@ -78,9 +78,8 @@ attributes #1 = { nounwind optsize readnone uwtable "less-precise-fpmad"="false"
define i64 @_Z8lshift12mm(i64 %a, i64 %b) #2 {
; CHECK-LABEL: _Z8lshift12mm:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: shrq $52, %rsi
-; CHECK-NEXT: shlq $12, %rdi
-; CHECK-NEXT: leaq (%rdi,%rsi), %rax
+; CHECK-NEXT: movq %rdi, %rax
+; CHECK-NEXT: shldq $12, %rsi, %rax
; CHECK-NEXT: retq
entry:
%shl = shl i64 %a, 12
diff --git a/llvm/test/CodeGen/X86/x86-64-double-shifts-var.ll b/llvm/test/CodeGen/X86/x86-64-double-shifts-var.ll
index bb1a4e5fcb75b..090fb46f53381 100644
--- a/llvm/test/CodeGen/X86/x86-64-double-shifts-var.ll
+++ b/llvm/test/CodeGen/X86/x86-64-double-shifts-var.ll
@@ -33,35 +33,25 @@ define i64 @lshift(i64 %a, i64 %b, i32 %c) nounwind readnone {
; X64-LABEL: lshift:
; X64: # %bb.0: # %entry
; X64-NEXT: movl %edx, %ecx
-; X64-NEXT: movq %rsi, %rax
-; X64-NEXT: shlq %cl, %rdi
-; X64-NEXT: shrq %rax
-; X64-NEXT: notb %cl
+; X64-NEXT: movq %rdi, %rax
; X64-NEXT: # kill: def $cl killed $cl killed $ecx
-; X64-NEXT: shrq %cl, %rax
-; X64-NEXT: orq %rdi, %rax
+; X64-NEXT: shldq %cl, %rsi, %rax
; X64-NEXT: retq
;
; BMI-LABEL: lshift:
; BMI: # %bb.0: # %entry
-; BMI-NEXT: movq %rsi, %rax
+; BMI-NEXT: movq %rdi, %rax
; BMI-NEXT: movl %edx, %ecx
-; BMI-NEXT: shrq %rax
-; BMI-NEXT: shlq %cl, %rdi
-; BMI-NEXT: notb %cl
; BMI-NEXT: # kill: def $cl killed $cl killed $ecx
-; BMI-NEXT: shrq %cl, %rax
-; BMI-NEXT: orq %rdi, %rax
+; BMI-NEXT: shldq %cl, %rsi, %rax
; BMI-NEXT: retq
;
; BMI2-SLOW-LABEL: lshift:
; BMI2-SLOW: # %bb.0: # %entry
-; BMI2-SLOW-NEXT: # kill: def $edx killed $edx def $rdx
-; BMI2-SLOW-NEXT: shlxq %rdx, %rdi, %rcx
-; BMI2-SLOW-NEXT: notb %dl
-; BMI2-SLOW-NEXT: shrq %rsi
-; BMI2-SLOW-NEXT: shrxq %rdx, %rsi, %rax
-; BMI2-SLOW-NEXT: orq %rcx, %rax
+; BMI2-SLOW-NEXT: movq %rdi, %rax
+; BMI2-SLOW-NEXT: movl %edx, %ecx
+; BMI2-SLOW-NEXT: # kill: def $cl killed $cl killed $ecx
+; BMI2-SLOW-NEXT: shldq %cl, %rsi, %rax
; BMI2-SLOW-NEXT: retq
;
; BMI2-FAST-LABEL: lshift:
@@ -90,33 +80,25 @@ define i64 @rshift(i64 %a, i64 %b, i32 %c) nounwind readnone {
; X64-LABEL: rshift:
; X64: # %bb.0: # %entry
; X64-NEXT: movl %edx, %ecx
-; X64-NEXT: shrq %cl, %rdi
-; X64-NEXT: leaq (%rsi,%rsi), %rax
-; X64-NEXT: notb %cl
+; X64-NEXT: movq %rdi, %rax
; X64-NEXT: # kill: def $cl killed $cl killed $ecx
-; X64-NEXT: shlq %cl, %rax
-; X64-NEXT: orq %rdi, %rax
+; X64-NEXT: shrdq %cl, %rsi, %rax
; X64-NEXT: retq
;
; BMI-LABEL: rshift:
; BMI: # %bb.0: # %entry
+; BMI-NEXT: movq %rdi, %rax
; BMI-NEXT: movl %edx, %ecx
-; BMI-NEXT: leaq (%rsi,%rsi), %rax
-; BMI-NEXT: shrq %cl, %rdi
-; BMI-NEXT: notb %cl
; BMI-NEXT: # kill: def $cl killed $cl killed $ecx
-; BMI-NEXT: shlq %cl, %rax
-; BMI-NEXT: orq %rdi, %rax
+; BMI-NEXT: shrdq %cl, %rsi, %rax
; BMI-NEXT: retq
;
; BMI2-SLOW-LABEL: rshift:
; BMI2-SLOW: # %bb.0: # %entry
-; BMI2-SLOW-NEXT: # kill: def $edx killed $edx def $rdx
-; BMI2-SLOW-NEXT: shrxq %rdx, %rdi, %rcx
-; BMI2-SLOW-NEXT: notb %dl
-; BMI2-SLOW-NEXT: addq %rsi, %rsi
-; BMI2-SLOW-NEXT: shlxq %rdx, %rsi, %rax
-; BMI2-SLOW-NEXT: orq %rcx, %rax
+; BMI2-SLOW-NEXT: movq %rdi, %rax
+; BMI2-SLOW-NEXT: movl %edx, %ecx
+; BMI2-SLOW-NEXT: # kill: def $cl killed $cl killed $ecx
+; BMI2-SLOW-NEXT: shrdq %cl, %rsi, %rax
; BMI2-SLOW-NEXT: retq
;
; BMI2-FAST-LABEL: rshift:
>From 28074b275d237e98a4afcd13e0b63a3064076d85 Mon Sep 17 00:00:00 2001
From: stomfaig <stomfaig at gmail.com>
Date: Mon, 16 Mar 2026 17:11:19 +0000
Subject: [PATCH 02/11] add MC pattern shld->shl,shr,or
---
llvm/lib/Target/X86/X86InstrInfo.cpp | 92 ++
llvm/lib/Target/X86/X86InstrInfo.h | 2 +
llvm/test/CodeGen/X86/2008-07-11-SHLBy1.ll | 6 +-
llvm/test/CodeGen/X86/apx/shld.ll | 8 +-
llvm/test/CodeGen/X86/avgflooru-i128.ll | 51 +-
llvm/test/CodeGen/X86/avgflooru-scalar.ll | 48 +-
llvm/test/CodeGen/X86/bitreverse.ll | 557 ++++++---
llvm/test/CodeGen/X86/bswap.ll | 140 ++-
llvm/test/CodeGen/X86/clmul.ll | 24 +-
llvm/test/CodeGen/X86/combine-bswap.ll | 7 +-
.../CodeGen/X86/const-shift-of-constmasked.ll | 41 +-
llvm/test/CodeGen/X86/dagcombine-cse.ll | 5 +-
.../X86/div-rem-pair-recomposition-signed.ll | 544 +++++----
.../div-rem-pair-recomposition-unsigned.ll | 480 ++++----
llvm/test/CodeGen/X86/div_i129_v_pow2k.ll | 163 ++-
llvm/test/CodeGen/X86/divide-by-constant.ll | 30 +-
llvm/test/CodeGen/X86/divmod128.ll | 64 +-
.../CodeGen/X86/expand-large-fp-optnone.ll | 28 +-
llvm/test/CodeGen/X86/fold-tied-op.ll | 112 +-
llvm/test/CodeGen/X86/freeze-binary.ll | 16 +-
llvm/test/CodeGen/X86/fshl.ll | 31 +-
.../CodeGen/X86/funnel-shift-logic-fold.ll | 22 +-
llvm/test/CodeGen/X86/funnel-shift-rot.ll | 12 +-
llvm/test/CodeGen/X86/funnel-shift.ll | 119 +-
llvm/test/CodeGen/X86/icmp-shift-opt.ll | 74 +-
llvm/test/CodeGen/X86/imul.ll | 27 +-
llvm/test/CodeGen/X86/isel-shift.ll | 29 +-
llvm/test/CodeGen/X86/known-bits.ll | 18 +-
llvm/test/CodeGen/X86/legalize-shl-vec.ll | 242 ++--
llvm/test/CodeGen/X86/logic-shift.ll | 24 +-
llvm/test/CodeGen/X86/mul-constant-i64.ll | 76 +-
llvm/test/CodeGen/X86/or-lea.ll | 8 +-
llvm/test/CodeGen/X86/pr38539.ll | 170 +--
llvm/test/CodeGen/X86/pr43820.ll | 413 +++----
llvm/test/CodeGen/X86/pr49162.ll | 10 +-
llvm/test/CodeGen/X86/rot32.ll | 60 +-
llvm/test/CodeGen/X86/rot64.ll | 48 +-
llvm/test/CodeGen/X86/rotate-add.ll | 29 +-
llvm/test/CodeGen/X86/rotate-extract.ll | 33 +-
llvm/test/CodeGen/X86/rotate.ll | 66 +-
llvm/test/CodeGen/X86/rotate2.ll | 13 +-
llvm/test/CodeGen/X86/scmp.ll | 1056 ++++++++++-------
llvm/test/CodeGen/X86/sdiv_fix.ll | 160 +--
llvm/test/CodeGen/X86/sdiv_fix_sat.ll | 167 +--
llvm/test/CodeGen/X86/setcc-fsh.ll | 15 +-
llvm/test/CodeGen/X86/shift-bmi2.ll | 18 +-
llvm/test/CodeGen/X86/shift-i256.ll | 238 ++--
llvm/test/CodeGen/X86/shift-i512.ll | 282 +++--
llvm/test/CodeGen/X86/shift-mask.ll | 15 +-
.../CodeGen/X86/shld-machine-combiner.mir | 100 ++
llvm/test/CodeGen/X86/udiv_fix.ll | 7 +-
llvm/test/CodeGen/X86/udiv_fix_sat.ll | 17 +-
llvm/test/CodeGen/X86/vector-sext.ll | 41 +-
.../X86/x86-64-double-precision-shift-left.ll | 22 +-
.../x86-64-double-precision-shift-right.ll | 17 +-
.../X86/x86-64-double-shifts-Oz-Os-O2.ll | 24 +-
llvm/test/CodeGen/X86/xor-lea.ll | 8 +-
57 files changed, 3789 insertions(+), 2340 deletions(-)
create mode 100644 llvm/test/CodeGen/X86/shld-machine-combiner.mir
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index 2479a8dccfb00..c8646fe77d213 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -10663,11 +10663,99 @@ bool X86InstrInfo::getMachineCombinerPatterns(
}
break;
}
+ // We do not support CL variant,
+ // as requires a lot of bookeeping
+ case X86::SHLD32rri8:
+ case X86::SHLD32mri8:
+ case X86::SHLD64rri8:
+ case X86::SHLD64mri8: {
+ Patterns.push_back(X86MachineCombinerPattern::USHLD);
+ return true;
+ }
}
return TargetInstrInfo::getMachineCombinerPatterns(Root,
Patterns, DoRegPressureReduce);
}
+static void
+genAlternativeShldSequence(MachineInstr &Root, const TargetInstrInfo &TII,
+ SmallVectorImpl<MachineInstr *> &InsInstrs,
+ SmallVectorImpl<MachineInstr *> &DelInstrs,
+ DenseMap<Register, unsigned> &InstrIdxForVirtReg) {
+ auto *MF = Root.getMF();
+ auto OpCode = Root.getOpcode();
+
+ MachineRegisterInfo &RegInfo = MF->getRegInfo();
+ const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
+ auto GetRC = [&](Register Reg) {
+ return Reg.isVirtual() ? RegInfo.getRegClass(Reg)
+ : TRI->getMinimalPhysRegClass(Reg);
+ };
+
+ unsigned BW = 0;
+ if (OpCode == X86::SHLD32rri8 || OpCode == X86::SHLD32mri8)
+ BW = 32;
+ else
+ BW = 64;
+
+ if (OpCode == X86::SHLD32rri8 || OpCode == X86::SHLD64rri8) {
+ const TargetRegisterClass *RC = GetRC(Root.getOperand(0).getReg());
+ Register ShlReg = RegInfo.createVirtualRegister(RC);
+ Register ShrReg = RegInfo.createVirtualRegister(RC);
+ unsigned ShlOpc = (BW == 64) ? X86::SHL64ri : X86::SHL32ri;
+ unsigned ShrOpc = (BW == 64) ? X86::SHR64ri : X86::SHR32ri;
+ unsigned OrOpc = (BW == 64) ? X86::OR64rr : X86::OR32rr;
+ int64_t Imm = Root.getOperand(3).getImm();
+ MachineInstr *Shl = BuildMI(*MF, MIMetadata(Root), TII.get(ShlOpc), ShlReg)
+ .addReg(Root.getOperand(1).getReg())
+ .addImm(Imm);
+ MachineInstr *Shr = BuildMI(*MF, MIMetadata(Root), TII.get(ShrOpc), ShrReg)
+ .addReg(Root.getOperand(2).getReg())
+ .addImm(BW - Imm);
+ MachineInstr *Or = BuildMI(*MF, MIMetadata(Root), TII.get(OrOpc),
+ Root.getOperand(0).getReg())
+ .addReg(ShlReg)
+ .addReg(ShrReg);
+ InsInstrs.push_back(Shl);
+ InsInstrs.push_back(Shr);
+ InsInstrs.push_back(Or);
+ DelInstrs.push_back(&Root);
+ return;
+ }
+
+ if (OpCode == X86::SHLD32mri8 || OpCode == X86::SHLD64mri8) {
+ const TargetRegisterClass *RC = GetRC(Root.getOperand(5).getReg());
+ Register ShrReg = RegInfo.createVirtualRegister(RC);
+ unsigned ShlOpc = (BW == 64) ? X86::SHL64mi : X86::SHL32mi;
+ unsigned ShrOpc = (BW == 64) ? X86::SHR64ri : X86::SHR32ri;
+ unsigned OrOpc = (BW == 64) ? X86::OR64mr : X86::OR32mr;
+ int64_t Imm = Root.getOperand(6).getImm();
+ MachineInstr *Shl = BuildMI(*MF, MIMetadata(Root), TII.get(ShlOpc))
+ .add(Root.getOperand(0 + X86::AddrBaseReg))
+ .add(Root.getOperand(0 + X86::AddrScaleAmt))
+ .add(Root.getOperand(0 + X86::AddrIndexReg))
+ .add(Root.getOperand(0 + X86::AddrDisp))
+ .add(Root.getOperand(0 + X86::AddrSegmentReg))
+ .addImm(Imm);
+ MachineInstr *Shr = BuildMI(*MF, MIMetadata(Root), TII.get(ShrOpc), ShrReg)
+ .addReg(Root.getOperand(5).getReg())
+ .addImm(BW - Imm);
+ MachineInstr *Or = BuildMI(*MF, MIMetadata(Root), TII.get(OrOpc))
+ .add(Root.getOperand(0 + X86::AddrBaseReg))
+ .add(Root.getOperand(0 + X86::AddrScaleAmt))
+ .add(Root.getOperand(0 + X86::AddrIndexReg))
+ .add(Root.getOperand(0 + X86::AddrDisp))
+ .add(Root.getOperand(0 + X86::AddrSegmentReg))
+ .addReg(ShrReg);
+ InsInstrs.push_back(Shl);
+ InsInstrs.push_back(Shr);
+ InsInstrs.push_back(Or);
+ DelInstrs.push_back(&Root);
+ return;
+ }
+ llvm_unreachable("Unexpected opcode in genAlternativeShldSequence");
+}
+
static void
genAlternativeDpCodeSequence(MachineInstr &Root, const TargetInstrInfo &TII,
SmallVectorImpl<MachineInstr *> &InsInstrs,
@@ -10773,6 +10861,10 @@ void X86InstrInfo::genAlternativeCodeSequence(
genAlternativeDpCodeSequence(Root, *this, InsInstrs, DelInstrs,
InstrIdxForVirtReg);
return;
+ case X86MachineCombinerPattern::USHLD:
+ genAlternativeShldSequence(Root, *this, InsInstrs, DelInstrs,
+ InstrIdxForVirtReg);
+ return;
}
}
diff --git a/llvm/lib/Target/X86/X86InstrInfo.h b/llvm/lib/Target/X86/X86InstrInfo.h
index de8ccb44578a3..65a5ca95233f3 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.h
+++ b/llvm/lib/Target/X86/X86InstrInfo.h
@@ -30,6 +30,8 @@ class X86Subtarget;
enum X86MachineCombinerPattern : unsigned {
// X86 VNNI
DPWSSD = MachineCombinerPattern::TARGET_PATTERN_START,
+ // Unfold SHLD
+ USHLD,
};
namespace X86 {
diff --git a/llvm/test/CodeGen/X86/2008-07-11-SHLBy1.ll b/llvm/test/CodeGen/X86/2008-07-11-SHLBy1.ll
index 7779d2eb32ab8..70370533cebde 100644
--- a/llvm/test/CodeGen/X86/2008-07-11-SHLBy1.ll
+++ b/llvm/test/CodeGen/X86/2008-07-11-SHLBy1.ll
@@ -4,9 +4,11 @@
define i128 @sl(i128 %x) {
; CHECK-LABEL: sl:
; CHECK: # %bb.0:
-; CHECK-NEXT: movq %rsi, %rdx
-; CHECK-NEXT: shldq $1, %rdi, %rdx
+; CHECK-NEXT: leaq (,%rsi,2), %rcx
; CHECK-NEXT: leaq (%rdi,%rdi), %rax
+; CHECK-NEXT: movq %rdi, %rdx
+; CHECK-NEXT: shrq $63, %rdx
+; CHECK-NEXT: orq %rcx, %rdx
; CHECK-NEXT: retq
%t = shl i128 %x, 1
ret i128 %t
diff --git a/llvm/test/CodeGen/X86/apx/shld.ll b/llvm/test/CodeGen/X86/apx/shld.ll
index 5b99719ba537c..ee2ead8eaee61 100644
--- a/llvm/test/CodeGen/X86/apx/shld.ll
+++ b/llvm/test/CodeGen/X86/apx/shld.ll
@@ -243,7 +243,9 @@ entry:
define void @shld32mri8_legacy(ptr %ptr, i32 noundef %b) {
; CHECK-LABEL: shld32mri8_legacy:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: shldl $12, %esi, (%rdi)
+; CHECK-NEXT: shll $12, (%rdi)
+; CHECK-NEXT: shrl $20, %esi
+; CHECK-NEXT: orl %esi, (%rdi)
; CHECK-NEXT: retq
entry:
%a = load i32, ptr %ptr
@@ -255,7 +257,9 @@ entry:
define void @shld64mri8_legacy(ptr %ptr, i64 noundef %b) {
; CHECK-LABEL: shld64mri8_legacy:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: shldq $12, %rsi, (%rdi)
+; CHECK-NEXT: shlq $12, (%rdi)
+; CHECK-NEXT: shrq $52, %rsi
+; CHECK-NEXT: orq %rsi, (%rdi)
; CHECK-NEXT: retq
entry:
%a = load i64, ptr %ptr
diff --git a/llvm/test/CodeGen/X86/avgflooru-i128.ll b/llvm/test/CodeGen/X86/avgflooru-i128.ll
index 11e886e25ba4e..1cb184406d69a 100644
--- a/llvm/test/CodeGen/X86/avgflooru-i128.ll
+++ b/llvm/test/CodeGen/X86/avgflooru-i128.ll
@@ -10,7 +10,9 @@ define i128 @avgflooru_i128(i128 %x, i128 %y) {
; CHECK-NEXT: setb %cl
; CHECK-NEXT: shrdq $1, %rsi, %rax
; CHECK-NEXT: movzbl %cl, %edx
-; CHECK-NEXT: shldq $63, %rsi, %rdx
+; CHECK-NEXT: shlq $63, %rdx
+; CHECK-NEXT: shrq %rsi
+; CHECK-NEXT: orq %rsi, %rdx
; CHECK-NEXT: retq
start:
%xor = xor i128 %y, %x
@@ -32,10 +34,10 @@ define i128 @avgflooru_i128_multi_use(i128 %x, i128 %y) nounwind {
; CHECK-NEXT: pushq %r12
; CHECK-NEXT: pushq %rbx
; CHECK-NEXT: pushq %rax
-; CHECK-NEXT: movq %rcx, %rbx
-; CHECK-NEXT: movq %rdx, %r14
-; CHECK-NEXT: movq %rsi, %r15
-; CHECK-NEXT: movq %rdi, %r12
+; CHECK-NEXT: movq %rcx, %r15
+; CHECK-NEXT: movq %rdx, %r12
+; CHECK-NEXT: movq %rsi, %rbx
+; CHECK-NEXT: movq %rdi, %r14
; CHECK-NEXT: movq %rdx, %r13
; CHECK-NEXT: xorq %rdi, %r13
; CHECK-NEXT: movq %rcx, %rbp
@@ -48,13 +50,16 @@ define i128 @avgflooru_i128_multi_use(i128 %x, i128 %y) nounwind {
; CHECK-NEXT: movq %r13, %rdi
; CHECK-NEXT: movq %rbp, %rsi
; CHECK-NEXT: callq use at PLT
-; CHECK-NEXT: addq %r14, %r12
-; CHECK-NEXT: adcq %rbx, %r15
+; CHECK-NEXT: addq %r12, %r14
+; CHECK-NEXT: adcq %r15, %rbx
; CHECK-NEXT: setb %al
-; CHECK-NEXT: shrdq $1, %r15, %r12
-; CHECK-NEXT: movzbl %al, %edx
-; CHECK-NEXT: shldq $63, %r15, %rdx
-; CHECK-NEXT: movq %r12, %rax
+; CHECK-NEXT: shrdq $1, %rbx, %r14
+; CHECK-NEXT: movzbl %al, %eax
+; CHECK-NEXT: shlq $63, %rax
+; CHECK-NEXT: shrq %rbx
+; CHECK-NEXT: orq %rax, %rbx
+; CHECK-NEXT: movq %r14, %rax
+; CHECK-NEXT: movq %rbx, %rdx
; CHECK-NEXT: addq $8, %rsp
; CHECK-NEXT: popq %rbx
; CHECK-NEXT: popq %r12
@@ -123,18 +128,22 @@ define <2 x i128> @avgflooru_i128_vec(<2 x i128> %x, <2 x i128> %y) {
; CHECK-NEXT: adcq {{[0-9]+}}(%rsp), %rdx
; CHECK-NEXT: setb %dil
; CHECK-NEXT: movzbl %dil, %edi
-; CHECK-NEXT: shldq $63, %rdx, %rdi
+; CHECK-NEXT: shlq $63, %rdi
+; CHECK-NEXT: shrdq $1, %rdx, %rsi
+; CHECK-NEXT: shrq %rdx
+; CHECK-NEXT: orq %rdi, %rdx
; CHECK-NEXT: addq {{[0-9]+}}(%rsp), %rcx
; CHECK-NEXT: adcq {{[0-9]+}}(%rsp), %r8
-; CHECK-NEXT: setb %r9b
-; CHECK-NEXT: movzbl %r9b, %r9d
-; CHECK-NEXT: shldq $63, %r8, %r9
-; CHECK-NEXT: shldq $63, %rsi, %rdx
-; CHECK-NEXT: shldq $63, %rcx, %r8
-; CHECK-NEXT: movq %r8, 16(%rax)
-; CHECK-NEXT: movq %rdx, (%rax)
-; CHECK-NEXT: movq %r9, 24(%rax)
-; CHECK-NEXT: movq %rdi, 8(%rax)
+; CHECK-NEXT: setb %dil
+; CHECK-NEXT: movzbl %dil, %edi
+; CHECK-NEXT: shlq $63, %rdi
+; CHECK-NEXT: shrdq $1, %r8, %rcx
+; CHECK-NEXT: shrq %r8
+; CHECK-NEXT: orq %rdi, %r8
+; CHECK-NEXT: movq %rcx, 16(%rax)
+; CHECK-NEXT: movq %rsi, (%rax)
+; CHECK-NEXT: movq %r8, 24(%rax)
+; CHECK-NEXT: movq %rdx, 8(%rax)
; CHECK-NEXT: retq
start:
%xor = xor <2 x i128> %y, %x
diff --git a/llvm/test/CodeGen/X86/avgflooru-scalar.ll b/llvm/test/CodeGen/X86/avgflooru-scalar.ll
index bb070370316a8..dac0a9e41b2b1 100644
--- a/llvm/test/CodeGen/X86/avgflooru-scalar.ll
+++ b/llvm/test/CodeGen/X86/avgflooru-scalar.ll
@@ -253,14 +253,16 @@ define i32 @test_ext_i32(i32 %a0, i32 %a1) nounwind {
define i64 @test_fixed_i64(i64 %a0, i64 %a1) nounwind {
; X86-LABEL: test_fixed_i64:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: addl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: adcl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: setb %dl
-; X86-NEXT: movzbl %dl, %edx
-; X86-NEXT: shldl $31, %eax, %edx
-; X86-NEXT: shldl $31, %ecx, %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: addl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: adcl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: setb %cl
+; X86-NEXT: movzbl %cl, %ecx
+; X86-NEXT: shll $31, %ecx
+; X86-NEXT: shrdl $1, %edx, %eax
+; X86-NEXT: shrl %edx
+; X86-NEXT: orl %ecx, %edx
; X86-NEXT: retl
;
; X64-LABEL: test_fixed_i64:
@@ -281,14 +283,16 @@ define i64 @test_fixed_i64(i64 %a0, i64 %a1) nounwind {
define i64 @test_lsb_i64(i64 %a0, i64 %a1) nounwind {
; X86-LABEL: test_lsb_i64:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: addl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: adcl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: setb %dl
-; X86-NEXT: movzbl %dl, %edx
-; X86-NEXT: shldl $31, %eax, %edx
-; X86-NEXT: shldl $31, %ecx, %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: addl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: adcl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: setb %cl
+; X86-NEXT: movzbl %cl, %ecx
+; X86-NEXT: shll $31, %ecx
+; X86-NEXT: shrdl $1, %edx, %eax
+; X86-NEXT: shrl %edx
+; X86-NEXT: orl %ecx, %edx
; X86-NEXT: retl
;
; X64-LABEL: test_lsb_i64:
@@ -311,14 +315,16 @@ define i64 @test_lsb_i64(i64 %a0, i64 %a1) nounwind {
define i64 @test_ext_i64(i64 %a0, i64 %a1) nounwind {
; X86-LABEL: test_ext_i64:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: addl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: adcl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: setb %dl
-; X86-NEXT: movzbl %dl, %edx
-; X86-NEXT: shldl $31, %eax, %edx
-; X86-NEXT: shldl $31, %ecx, %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: addl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: adcl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: setb %cl
+; X86-NEXT: movzbl %cl, %ecx
+; X86-NEXT: shll $31, %ecx
+; X86-NEXT: shrdl $1, %edx, %eax
+; X86-NEXT: shrl %edx
+; X86-NEXT: orl %ecx, %edx
; X86-NEXT: retl
;
; X64-LABEL: test_ext_i64:
diff --git a/llvm/test/CodeGen/X86/bitreverse.ll b/llvm/test/CodeGen/X86/bitreverse.ll
index d92e1a1e7b9d4..a14c8e564182a 100644
--- a/llvm/test/CodeGen/X86/bitreverse.ll
+++ b/llvm/test/CodeGen/X86/bitreverse.ll
@@ -700,7 +700,7 @@ define i528 @large_promotion(i528 %A) nounwind {
; X86-NEXT: pushl %ebx
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
-; X86-NEXT: subl $60, %esp
+; X86-NEXT: subl $44, %esp
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
@@ -740,8 +740,12 @@ define i528 @large_promotion(i528 %A) nounwind {
; X86-NEXT: andl $1431655765, %ebx # imm = 0x55555555
; X86-NEXT: shrl %edi
; X86-NEXT: andl $1431655765, %edi # imm = 0x55555555
-; X86-NEXT: leal (%edi,%ebx,2), %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: leal (%edi,%ebx,2), %ebx
+; X86-NEXT: movl %ebx, %edi
+; X86-NEXT: shll $16, %edi
+; X86-NEXT: shrl $16, %ebp
+; X86-NEXT: orl %edi, %ebp
+; X86-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: bswapl %esi
; X86-NEXT: movl %esi, %edi
; X86-NEXT: andl $252645135, %edi # imm = 0xF0F0F0F
@@ -758,7 +762,12 @@ define i528 @large_promotion(i528 %A) nounwind {
; X86-NEXT: andl $1431655765, %edi # imm = 0x55555555
; X86-NEXT: shrl %esi
; X86-NEXT: andl $1431655765, %esi # imm = 0x55555555
-; X86-NEXT: leal (%esi,%edi,2), %ebx
+; X86-NEXT: leal (%esi,%edi,2), %edi
+; X86-NEXT: movl %edi, %esi
+; X86-NEXT: shll $16, %esi
+; X86-NEXT: shrl $16, %ebx
+; X86-NEXT: orl %esi, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: bswapl %edx
; X86-NEXT: movl %edx, %esi
; X86-NEXT: andl $252645135, %esi # imm = 0xF0F0F0F
@@ -775,8 +784,12 @@ define i528 @large_promotion(i528 %A) nounwind {
; X86-NEXT: andl $1431655765, %esi # imm = 0x55555555
; X86-NEXT: shrl %edx
; X86-NEXT: andl $1431655765, %edx # imm = 0x55555555
-; X86-NEXT: leal (%edx,%esi,2), %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: leal (%edx,%esi,2), %esi
+; X86-NEXT: movl %esi, %edx
+; X86-NEXT: shll $16, %edx
+; X86-NEXT: shrl $16, %edi
+; X86-NEXT: orl %edx, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: bswapl %ecx
; X86-NEXT: movl %ecx, %edx
; X86-NEXT: andl $252645135, %edx # imm = 0xF0F0F0F
@@ -793,8 +806,12 @@ define i528 @large_promotion(i528 %A) nounwind {
; X86-NEXT: andl $1431655765, %edx # imm = 0x55555555
; X86-NEXT: shrl %ecx
; X86-NEXT: andl $1431655765, %ecx # imm = 0x55555555
-; X86-NEXT: leal (%ecx,%edx,2), %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: leal (%ecx,%edx,2), %edx
+; X86-NEXT: movl %edx, %ecx
+; X86-NEXT: shll $16, %ecx
+; X86-NEXT: shrl $16, %esi
+; X86-NEXT: orl %ecx, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: bswapl %eax
; X86-NEXT: movl %eax, %ecx
; X86-NEXT: andl $252645135, %ecx # imm = 0xF0F0F0F
@@ -811,8 +828,12 @@ define i528 @large_promotion(i528 %A) nounwind {
; X86-NEXT: andl $1431655765, %ecx # imm = 0x55555555
; X86-NEXT: shrl %eax
; X86-NEXT: andl $1431655765, %eax # imm = 0x55555555
-; X86-NEXT: leal (%eax,%ecx,2), %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: leal (%eax,%ecx,2), %esi
+; X86-NEXT: movl %esi, %eax
+; X86-NEXT: shll $16, %eax
+; X86-NEXT: shrl $16, %edx
+; X86-NEXT: orl %eax, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: bswapl %eax
; X86-NEXT: movl %eax, %ecx
@@ -830,8 +851,12 @@ define i528 @large_promotion(i528 %A) nounwind {
; X86-NEXT: andl $1431655765, %ecx # imm = 0x55555555
; X86-NEXT: shrl %eax
; X86-NEXT: andl $1431655765, %eax # imm = 0x55555555
-; X86-NEXT: leal (%eax,%ecx,2), %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: leal (%eax,%ecx,2), %edx
+; X86-NEXT: movl %edx, %eax
+; X86-NEXT: shll $16, %eax
+; X86-NEXT: shrl $16, %esi
+; X86-NEXT: orl %eax, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: bswapl %eax
; X86-NEXT: movl %eax, %ecx
@@ -849,8 +874,12 @@ define i528 @large_promotion(i528 %A) nounwind {
; X86-NEXT: andl $1431655765, %ecx # imm = 0x55555555
; X86-NEXT: shrl %eax
; X86-NEXT: andl $1431655765, %eax # imm = 0x55555555
-; X86-NEXT: leal (%eax,%ecx,2), %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: leal (%eax,%ecx,2), %esi
+; X86-NEXT: movl %esi, %eax
+; X86-NEXT: shll $16, %eax
+; X86-NEXT: shrl $16, %edx
+; X86-NEXT: orl %eax, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: bswapl %eax
; X86-NEXT: movl %eax, %ecx
@@ -868,8 +897,12 @@ define i528 @large_promotion(i528 %A) nounwind {
; X86-NEXT: andl $1431655765, %ecx # imm = 0x55555555
; X86-NEXT: shrl %eax
; X86-NEXT: andl $1431655765, %eax # imm = 0x55555555
-; X86-NEXT: leal (%eax,%ecx,2), %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: leal (%eax,%ecx,2), %edx
+; X86-NEXT: movl %edx, %eax
+; X86-NEXT: shll $16, %eax
+; X86-NEXT: shrl $16, %esi
+; X86-NEXT: orl %eax, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: bswapl %eax
; X86-NEXT: movl %eax, %ecx
@@ -887,7 +920,12 @@ define i528 @large_promotion(i528 %A) nounwind {
; X86-NEXT: andl $1431655765, %ecx # imm = 0x55555555
; X86-NEXT: shrl %eax
; X86-NEXT: andl $1431655765, %eax # imm = 0x55555555
-; X86-NEXT: leal (%eax,%ecx,2), %edi
+; X86-NEXT: leal (%eax,%ecx,2), %esi
+; X86-NEXT: movl %esi, %eax
+; X86-NEXT: shll $16, %eax
+; X86-NEXT: shrl $16, %edx
+; X86-NEXT: orl %eax, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: bswapl %eax
; X86-NEXT: movl %eax, %ecx
@@ -905,8 +943,12 @@ define i528 @large_promotion(i528 %A) nounwind {
; X86-NEXT: andl $1431655765, %ecx # imm = 0x55555555
; X86-NEXT: shrl %eax
; X86-NEXT: andl $1431655765, %eax # imm = 0x55555555
-; X86-NEXT: leal (%eax,%ecx,2), %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: leal (%eax,%ecx,2), %edx
+; X86-NEXT: movl %edx, %eax
+; X86-NEXT: shll $16, %eax
+; X86-NEXT: shrl $16, %esi
+; X86-NEXT: orl %eax, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: bswapl %eax
; X86-NEXT: movl %eax, %ecx
@@ -924,8 +966,12 @@ define i528 @large_promotion(i528 %A) nounwind {
; X86-NEXT: andl $1431655765, %ecx # imm = 0x55555555
; X86-NEXT: shrl %eax
; X86-NEXT: andl $1431655765, %eax # imm = 0x55555555
-; X86-NEXT: leal (%eax,%ecx,2), %eax
-; X86-NEXT: movl %eax, (%esp) # 4-byte Spill
+; X86-NEXT: leal (%eax,%ecx,2), %ebp
+; X86-NEXT: movl %ebp, %eax
+; X86-NEXT: shll $16, %eax
+; X86-NEXT: shrl $16, %edx
+; X86-NEXT: orl %eax, %edx
+; X86-NEXT: movl %edx, (%esp) # 4-byte Spill
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: bswapl %eax
; X86-NEXT: movl %eax, %ecx
@@ -943,8 +989,11 @@ define i528 @large_promotion(i528 %A) nounwind {
; X86-NEXT: andl $1431655765, %ecx # imm = 0x55555555
; X86-NEXT: shrl %eax
; X86-NEXT: andl $1431655765, %eax # imm = 0x55555555
-; X86-NEXT: leal (%eax,%ecx,2), %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: leal (%eax,%ecx,2), %ebx
+; X86-NEXT: movl %ebx, %eax
+; X86-NEXT: shll $16, %eax
+; X86-NEXT: shrl $16, %ebp
+; X86-NEXT: orl %eax, %ebp
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: bswapl %eax
; X86-NEXT: movl %eax, %ecx
@@ -962,8 +1011,11 @@ define i528 @large_promotion(i528 %A) nounwind {
; X86-NEXT: andl $1431655765, %ecx # imm = 0x55555555
; X86-NEXT: shrl %eax
; X86-NEXT: andl $1431655765, %eax # imm = 0x55555555
-; X86-NEXT: leal (%eax,%ecx,2), %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: leal (%eax,%ecx,2), %edi
+; X86-NEXT: movl %edi, %eax
+; X86-NEXT: shll $16, %eax
+; X86-NEXT: shrl $16, %ebx
+; X86-NEXT: orl %eax, %ebx
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: bswapl %eax
; X86-NEXT: movl %eax, %ecx
@@ -981,8 +1033,11 @@ define i528 @large_promotion(i528 %A) nounwind {
; X86-NEXT: andl $1431655765, %ecx # imm = 0x55555555
; X86-NEXT: shrl %eax
; X86-NEXT: andl $1431655765, %eax # imm = 0x55555555
-; X86-NEXT: leal (%eax,%ecx,2), %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: leal (%eax,%ecx,2), %esi
+; X86-NEXT: movl %esi, %eax
+; X86-NEXT: shll $16, %eax
+; X86-NEXT: shrl $16, %edi
+; X86-NEXT: orl %eax, %edi
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: bswapl %eax
; X86-NEXT: movl %eax, %ecx
@@ -1000,76 +1055,40 @@ define i528 @large_promotion(i528 %A) nounwind {
; X86-NEXT: andl $1431655765, %ecx # imm = 0x55555555
; X86-NEXT: shrl %eax
; X86-NEXT: andl $1431655765, %eax # imm = 0x55555555
-; X86-NEXT: leal (%eax,%ecx,2), %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: leal (%eax,%ecx,2), %ecx
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: shll $16, %eax
+; X86-NEXT: shrl $16, %esi
+; X86-NEXT: orl %eax, %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: bswapl %eax
-; X86-NEXT: movl %eax, %ecx
-; X86-NEXT: andl $252645135, %ecx # imm = 0xF0F0F0F
-; X86-NEXT: shll $4, %ecx
+; X86-NEXT: movl %eax, %edx
+; X86-NEXT: andl $252645135, %edx # imm = 0xF0F0F0F
+; X86-NEXT: shll $4, %edx
; X86-NEXT: shrl $4, %eax
; X86-NEXT: andl $252645135, %eax # imm = 0xF0F0F0F
-; X86-NEXT: orl %ecx, %eax
-; X86-NEXT: movl %eax, %ecx
-; X86-NEXT: andl $858993459, %ecx # imm = 0x33333333
+; X86-NEXT: orl %edx, %eax
+; X86-NEXT: movl %eax, %edx
+; X86-NEXT: andl $858993459, %edx # imm = 0x33333333
; X86-NEXT: shrl $2, %eax
; X86-NEXT: andl $858993459, %eax # imm = 0x33333333
-; X86-NEXT: leal (%eax,%ecx,4), %eax
-; X86-NEXT: movl %eax, %ecx
-; X86-NEXT: andl $1431655765, %ecx # imm = 0x55555555
+; X86-NEXT: leal (%eax,%edx,4), %eax
+; X86-NEXT: movl %eax, %edx
+; X86-NEXT: andl $1431655765, %edx # imm = 0x55555555
; X86-NEXT: shrl %eax
; X86-NEXT: andl $1431655765, %eax # imm = 0x55555555
-; X86-NEXT: leal (%eax,%ecx,2), %edx
-; X86-NEXT: movl %ebp, %esi
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: shrdl $16, %ecx, %esi
-; X86-NEXT: movl %ebx, %eax
-; X86-NEXT: shrdl $16, %ebx, %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: shrdl $16, %ecx, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shrdl $16, %eax, %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: shrdl $16, %ecx, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shrdl $16, %eax, %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: shrdl $16, %ecx, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shrdl $16, %eax, %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shrdl $16, %edi, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shrdl $16, %eax, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl (%esp), %ecx # 4-byte Reload
-; X86-NEXT: shrdl $16, %ecx, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X86-NEXT: shrdl $16, %ebp, %ecx
-; X86-NEXT: movl %ecx, (%esp) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: shrdl $16, %ebx, %ebp
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: shrdl $16, %edi, %ebx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: shrdl $16, %ecx, %edi
-; X86-NEXT: shrdl $16, %edx, %ecx
+; X86-NEXT: leal (%eax,%edx,2), %edx
+; X86-NEXT: movl %edx, %eax
+; X86-NEXT: shll $16, %eax
+; X86-NEXT: shrl $16, %ecx
+; X86-NEXT: orl %eax, %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl %ecx, 60(%eax)
-; X86-NEXT: movl %edi, 56(%eax)
-; X86-NEXT: movl %ebx, 52(%eax)
-; X86-NEXT: movl %ebp, 48(%eax)
+; X86-NEXT: movl %esi, 56(%eax)
+; X86-NEXT: movl %edi, 52(%eax)
+; X86-NEXT: movl %ebx, 48(%eax)
+; X86-NEXT: movl %ebp, 44(%eax)
; X86-NEXT: movl (%esp), %ecx # 4-byte Reload
-; X86-NEXT: movl %ecx, 44(%eax)
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: movl %ecx, 40(%eax)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: movl %ecx, 36(%eax)
@@ -1089,10 +1108,11 @@ define i528 @large_promotion(i528 %A) nounwind {
; X86-NEXT: movl %ecx, 8(%eax)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: movl %ecx, 4(%eax)
-; X86-NEXT: movl %esi, (%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %ecx, (%eax)
; X86-NEXT: shrl $16, %edx
; X86-NEXT: movw %dx, 64(%eax)
-; X86-NEXT: addl $60, %esp
+; X86-NEXT: addl $44, %esp
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
; X86-NEXT: popl %ebx
@@ -1149,7 +1169,10 @@ define i528 @large_promotion(i528 %A) nounwind {
; X64-NEXT: shrq %rbx
; X64-NEXT: andq %r14, %rbx
; X64-NEXT: leaq (%rbx,%r13,2), %rbx
-; X64-NEXT: shrdq $48, %rbx, %rdi
+; X64-NEXT: movq %rbx, %r13
+; X64-NEXT: shlq $16, %r13
+; X64-NEXT: shrq $48, %rdi
+; X64-NEXT: orq %r13, %rdi
; X64-NEXT: bswapq %r15
; X64-NEXT: movq %r15, %r13
; X64-NEXT: shrq $4, %r13
@@ -1167,7 +1190,10 @@ define i528 @large_promotion(i528 %A) nounwind {
; X64-NEXT: shrq %r15
; X64-NEXT: andq %r14, %r15
; X64-NEXT: leaq (%r15,%r13,2), %r15
-; X64-NEXT: shrdq $48, %r15, %rbx
+; X64-NEXT: movq %r15, %r13
+; X64-NEXT: shlq $16, %r13
+; X64-NEXT: shrq $48, %rbx
+; X64-NEXT: orq %r13, %rbx
; X64-NEXT: bswapq %r12
; X64-NEXT: movq %r12, %r13
; X64-NEXT: shrq $4, %r13
@@ -1185,7 +1211,10 @@ define i528 @large_promotion(i528 %A) nounwind {
; X64-NEXT: shrq %r12
; X64-NEXT: andq %r14, %r12
; X64-NEXT: leaq (%r12,%r13,2), %r12
-; X64-NEXT: shrdq $48, %r12, %r15
+; X64-NEXT: movq %r12, %r13
+; X64-NEXT: shlq $16, %r13
+; X64-NEXT: shrq $48, %r15
+; X64-NEXT: orq %r13, %r15
; X64-NEXT: bswapq %r9
; X64-NEXT: movq %r9, %r13
; X64-NEXT: shrq $4, %r13
@@ -1203,7 +1232,10 @@ define i528 @large_promotion(i528 %A) nounwind {
; X64-NEXT: shrq %r9
; X64-NEXT: andq %r14, %r9
; X64-NEXT: leaq (%r9,%r13,2), %r9
-; X64-NEXT: shrdq $48, %r9, %r12
+; X64-NEXT: movq %r9, %r13
+; X64-NEXT: shlq $16, %r13
+; X64-NEXT: shrq $48, %r12
+; X64-NEXT: orq %r13, %r12
; X64-NEXT: bswapq %r8
; X64-NEXT: movq %r8, %r13
; X64-NEXT: shrq $4, %r13
@@ -1221,7 +1253,10 @@ define i528 @large_promotion(i528 %A) nounwind {
; X64-NEXT: shrq %r8
; X64-NEXT: andq %r14, %r8
; X64-NEXT: leaq (%r8,%r13,2), %r8
-; X64-NEXT: shrdq $48, %r8, %r9
+; X64-NEXT: movq %r8, %r13
+; X64-NEXT: shlq $16, %r13
+; X64-NEXT: shrq $48, %r9
+; X64-NEXT: orq %r13, %r9
; X64-NEXT: bswapq %rcx
; X64-NEXT: movq %rcx, %r13
; X64-NEXT: shrq $4, %r13
@@ -1239,7 +1274,10 @@ define i528 @large_promotion(i528 %A) nounwind {
; X64-NEXT: shrq %rcx
; X64-NEXT: andq %r14, %rcx
; X64-NEXT: leaq (%rcx,%r13,2), %rcx
-; X64-NEXT: shrdq $48, %rcx, %r8
+; X64-NEXT: movq %rcx, %r13
+; X64-NEXT: shlq $16, %r13
+; X64-NEXT: shrq $48, %r8
+; X64-NEXT: orq %r13, %r8
; X64-NEXT: bswapq %rdx
; X64-NEXT: movq %rdx, %r13
; X64-NEXT: shrq $4, %r13
@@ -1257,7 +1295,10 @@ define i528 @large_promotion(i528 %A) nounwind {
; X64-NEXT: shrq %rdx
; X64-NEXT: andq %r14, %rdx
; X64-NEXT: leaq (%rdx,%r13,2), %rdx
-; X64-NEXT: shrdq $48, %rdx, %rcx
+; X64-NEXT: movq %rdx, %r13
+; X64-NEXT: shlq $16, %r13
+; X64-NEXT: shrq $48, %rcx
+; X64-NEXT: orq %r13, %rcx
; X64-NEXT: bswapq %rsi
; X64-NEXT: movq %rsi, %r13
; X64-NEXT: shrq $4, %r13
@@ -1275,7 +1316,10 @@ define i528 @large_promotion(i528 %A) nounwind {
; X64-NEXT: shrq %rsi
; X64-NEXT: andq %r14, %rsi
; X64-NEXT: leaq (%rsi,%r10,2), %rsi
-; X64-NEXT: shrdq $48, %rsi, %rdx
+; X64-NEXT: movq %rsi, %r10
+; X64-NEXT: shlq $16, %r10
+; X64-NEXT: shrq $48, %rdx
+; X64-NEXT: orq %r10, %rdx
; X64-NEXT: shrq $48, %rsi
; X64-NEXT: movq %rdx, 56(%rax)
; X64-NEXT: movq %rcx, 48(%rax)
@@ -1303,71 +1347,119 @@ define i528 @large_promotion(i528 %A) nounwind {
; X86XOP-NEXT: vmovdqa {{.*#+}} xmm0 = [87,86,85,84,83,82,81,80,95,94,93,92,91,90,89,88]
; X86XOP-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
; X86XOP-NEXT: vpperm %xmm0, %xmm1, %xmm0, %xmm1
-; X86XOP-NEXT: vpextrd $1, %xmm1, %eax
+; X86XOP-NEXT: vpextrd $1, %xmm1, %ecx
; X86XOP-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
; X86XOP-NEXT: vpperm %xmm0, %xmm1, %xmm0, %xmm1
-; X86XOP-NEXT: vmovd %xmm1, %ecx
-; X86XOP-NEXT: shrdl $16, %ecx, %eax
-; X86XOP-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86XOP-NEXT: vpextrd $1, %xmm1, %eax
-; X86XOP-NEXT: shrdl $16, %eax, %ecx
+; X86XOP-NEXT: vmovd %xmm1, %edx
+; X86XOP-NEXT: movl %edx, %eax
+; X86XOP-NEXT: shll $16, %eax
+; X86XOP-NEXT: shrl $16, %ecx
+; X86XOP-NEXT: orl %eax, %ecx
; X86XOP-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86XOP-NEXT: vpextrd $1, %xmm1, %ecx
+; X86XOP-NEXT: movl %ecx, %eax
+; X86XOP-NEXT: shll $16, %eax
+; X86XOP-NEXT: shrl $16, %edx
+; X86XOP-NEXT: orl %eax, %edx
+; X86XOP-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86XOP-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
; X86XOP-NEXT: vpperm %xmm0, %xmm1, %xmm0, %xmm1
-; X86XOP-NEXT: vmovd %xmm1, %ecx
-; X86XOP-NEXT: shrdl $16, %ecx, %eax
-; X86XOP-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86XOP-NEXT: vpextrd $1, %xmm1, %eax
-; X86XOP-NEXT: shrdl $16, %eax, %ecx
+; X86XOP-NEXT: vmovd %xmm1, %edx
+; X86XOP-NEXT: movl %edx, %eax
+; X86XOP-NEXT: shll $16, %eax
+; X86XOP-NEXT: shrl $16, %ecx
+; X86XOP-NEXT: orl %eax, %ecx
; X86XOP-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86XOP-NEXT: vpextrd $1, %xmm1, %ecx
+; X86XOP-NEXT: movl %ecx, %eax
+; X86XOP-NEXT: shll $16, %eax
+; X86XOP-NEXT: shrl $16, %edx
+; X86XOP-NEXT: orl %eax, %edx
+; X86XOP-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86XOP-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
; X86XOP-NEXT: vpperm %xmm0, %xmm1, %xmm0, %xmm1
-; X86XOP-NEXT: vmovd %xmm1, %ecx
-; X86XOP-NEXT: shrdl $16, %ecx, %eax
-; X86XOP-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86XOP-NEXT: vpextrd $1, %xmm1, %eax
-; X86XOP-NEXT: shrdl $16, %eax, %ecx
+; X86XOP-NEXT: vmovd %xmm1, %edx
+; X86XOP-NEXT: movl %edx, %eax
+; X86XOP-NEXT: shll $16, %eax
+; X86XOP-NEXT: shrl $16, %ecx
+; X86XOP-NEXT: orl %eax, %ecx
; X86XOP-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86XOP-NEXT: vpextrd $1, %xmm1, %ecx
+; X86XOP-NEXT: movl %ecx, %eax
+; X86XOP-NEXT: shll $16, %eax
+; X86XOP-NEXT: shrl $16, %edx
+; X86XOP-NEXT: orl %eax, %edx
+; X86XOP-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86XOP-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
; X86XOP-NEXT: vpperm %xmm0, %xmm1, %xmm0, %xmm1
-; X86XOP-NEXT: vmovd %xmm1, %ecx
-; X86XOP-NEXT: shrdl $16, %ecx, %eax
-; X86XOP-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86XOP-NEXT: vpextrd $1, %xmm1, %eax
-; X86XOP-NEXT: shrdl $16, %eax, %ecx
+; X86XOP-NEXT: vmovd %xmm1, %edx
+; X86XOP-NEXT: movl %edx, %eax
+; X86XOP-NEXT: shll $16, %eax
+; X86XOP-NEXT: shrl $16, %ecx
+; X86XOP-NEXT: orl %eax, %ecx
; X86XOP-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86XOP-NEXT: vpextrd $1, %xmm1, %ecx
+; X86XOP-NEXT: movl %ecx, %eax
+; X86XOP-NEXT: shll $16, %eax
+; X86XOP-NEXT: shrl $16, %edx
+; X86XOP-NEXT: orl %eax, %edx
+; X86XOP-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86XOP-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
; X86XOP-NEXT: vpperm %xmm0, %xmm1, %xmm0, %xmm1
-; X86XOP-NEXT: vmovd %xmm1, %ecx
-; X86XOP-NEXT: shrdl $16, %ecx, %eax
-; X86XOP-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86XOP-NEXT: vpextrd $1, %xmm1, %eax
-; X86XOP-NEXT: shrdl $16, %eax, %ecx
+; X86XOP-NEXT: vmovd %xmm1, %edx
+; X86XOP-NEXT: movl %edx, %eax
+; X86XOP-NEXT: shll $16, %eax
+; X86XOP-NEXT: shrl $16, %ecx
+; X86XOP-NEXT: orl %eax, %ecx
; X86XOP-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86XOP-NEXT: vpextrd $1, %xmm1, %ecx
+; X86XOP-NEXT: movl %ecx, %eax
+; X86XOP-NEXT: shll $16, %eax
+; X86XOP-NEXT: shrl $16, %edx
+; X86XOP-NEXT: orl %eax, %edx
+; X86XOP-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86XOP-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
; X86XOP-NEXT: vpperm %xmm0, %xmm1, %xmm0, %xmm1
; X86XOP-NEXT: vmovd %xmm1, %ebp
-; X86XOP-NEXT: shrdl $16, %ebp, %eax
-; X86XOP-NEXT: movl %eax, (%esp) # 4-byte Spill
-; X86XOP-NEXT: vpextrd $1, %xmm1, %ebx
-; X86XOP-NEXT: shrdl $16, %ebx, %ebp
+; X86XOP-NEXT: movl %ebp, %eax
+; X86XOP-NEXT: shll $16, %eax
+; X86XOP-NEXT: shrl $16, %ecx
+; X86XOP-NEXT: orl %eax, %ecx
+; X86XOP-NEXT: movl %ecx, (%esp) # 4-byte Spill
+; X86XOP-NEXT: vpextrd $1, %xmm1, %edi
+; X86XOP-NEXT: movl %edi, %eax
+; X86XOP-NEXT: shll $16, %eax
+; X86XOP-NEXT: shrl $16, %ebp
+; X86XOP-NEXT: orl %eax, %ebp
; X86XOP-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
; X86XOP-NEXT: vpperm %xmm0, %xmm1, %xmm0, %xmm1
; X86XOP-NEXT: vmovd %xmm1, %esi
-; X86XOP-NEXT: shrdl $16, %esi, %ebx
+; X86XOP-NEXT: movl %esi, %eax
+; X86XOP-NEXT: shll $16, %eax
+; X86XOP-NEXT: shrl $16, %edi
+; X86XOP-NEXT: orl %eax, %edi
; X86XOP-NEXT: vpextrd $1, %xmm1, %edx
-; X86XOP-NEXT: shrdl $16, %edx, %esi
+; X86XOP-NEXT: movl %edx, %eax
+; X86XOP-NEXT: shll $16, %eax
+; X86XOP-NEXT: shrl $16, %esi
+; X86XOP-NEXT: orl %eax, %esi
; X86XOP-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
; X86XOP-NEXT: vpperm %xmm0, %xmm1, %xmm0, %xmm0
; X86XOP-NEXT: vmovd %xmm0, %ecx
-; X86XOP-NEXT: shrdl $16, %ecx, %edx
-; X86XOP-NEXT: vpextrd $1, %xmm0, %edi
-; X86XOP-NEXT: shrdl $16, %edi, %ecx
+; X86XOP-NEXT: movl %ecx, %eax
+; X86XOP-NEXT: shll $16, %eax
+; X86XOP-NEXT: shrl $16, %edx
+; X86XOP-NEXT: orl %eax, %edx
+; X86XOP-NEXT: vpextrd $1, %xmm0, %ebx
+; X86XOP-NEXT: movl %ebx, %eax
+; X86XOP-NEXT: shll $16, %eax
+; X86XOP-NEXT: shrl $16, %ecx
+; X86XOP-NEXT: orl %eax, %ecx
; X86XOP-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86XOP-NEXT: movl %ecx, 60(%eax)
; X86XOP-NEXT: movl %edx, 56(%eax)
; X86XOP-NEXT: movl %esi, 52(%eax)
-; X86XOP-NEXT: movl %ebx, 48(%eax)
+; X86XOP-NEXT: movl %edi, 48(%eax)
; X86XOP-NEXT: movl %ebp, 44(%eax)
; X86XOP-NEXT: movl (%esp), %ecx # 4-byte Reload
; X86XOP-NEXT: movl %ecx, 40(%eax)
@@ -1391,8 +1483,8 @@ define i528 @large_promotion(i528 %A) nounwind {
; X86XOP-NEXT: movl %ecx, 4(%eax)
; X86XOP-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86XOP-NEXT: movl %ecx, (%eax)
-; X86XOP-NEXT: shrl $16, %edi
-; X86XOP-NEXT: movw %di, 64(%eax)
+; X86XOP-NEXT: shrl $16, %ebx
+; X86XOP-NEXT: movw %bx, 64(%eax)
; X86XOP-NEXT: addl $44, %esp
; X86XOP-NEXT: popl %esi
; X86XOP-NEXT: popl %edi
@@ -1410,87 +1502,135 @@ define i528 @large_promotion(i528 %A) nounwind {
; X86GFNI-NEXT: vpbroadcastq {{.*#+}} xmm0 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
; X86GFNI-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
; X86GFNI-NEXT: vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
-; X86GFNI-NEXT: vmovd %xmm1, %eax
-; X86GFNI-NEXT: bswapl %eax
+; X86GFNI-NEXT: vmovd %xmm1, %ecx
+; X86GFNI-NEXT: bswapl %ecx
; X86GFNI-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
; X86GFNI-NEXT: vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
-; X86GFNI-NEXT: vpextrd $1, %xmm1, %ecx
-; X86GFNI-NEXT: bswapl %ecx
-; X86GFNI-NEXT: shrdl $16, %ecx, %eax
-; X86GFNI-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86GFNI-NEXT: vmovd %xmm1, %eax
-; X86GFNI-NEXT: bswapl %eax
-; X86GFNI-NEXT: shrdl $16, %eax, %ecx
+; X86GFNI-NEXT: vpextrd $1, %xmm1, %edx
+; X86GFNI-NEXT: bswapl %edx
+; X86GFNI-NEXT: movl %edx, %eax
+; X86GFNI-NEXT: shll $16, %eax
+; X86GFNI-NEXT: shrl $16, %ecx
+; X86GFNI-NEXT: orl %eax, %ecx
; X86GFNI-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86GFNI-NEXT: vmovd %xmm1, %ecx
+; X86GFNI-NEXT: bswapl %ecx
+; X86GFNI-NEXT: movl %ecx, %eax
+; X86GFNI-NEXT: shll $16, %eax
+; X86GFNI-NEXT: shrl $16, %edx
+; X86GFNI-NEXT: orl %eax, %edx
+; X86GFNI-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86GFNI-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
; X86GFNI-NEXT: vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
-; X86GFNI-NEXT: vpextrd $1, %xmm1, %ecx
-; X86GFNI-NEXT: bswapl %ecx
-; X86GFNI-NEXT: shrdl $16, %ecx, %eax
-; X86GFNI-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86GFNI-NEXT: vmovd %xmm1, %eax
-; X86GFNI-NEXT: bswapl %eax
-; X86GFNI-NEXT: shrdl $16, %eax, %ecx
+; X86GFNI-NEXT: vpextrd $1, %xmm1, %edx
+; X86GFNI-NEXT: bswapl %edx
+; X86GFNI-NEXT: movl %edx, %eax
+; X86GFNI-NEXT: shll $16, %eax
+; X86GFNI-NEXT: shrl $16, %ecx
+; X86GFNI-NEXT: orl %eax, %ecx
; X86GFNI-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86GFNI-NEXT: vmovd %xmm1, %ecx
+; X86GFNI-NEXT: bswapl %ecx
+; X86GFNI-NEXT: movl %ecx, %eax
+; X86GFNI-NEXT: shll $16, %eax
+; X86GFNI-NEXT: shrl $16, %edx
+; X86GFNI-NEXT: orl %eax, %edx
+; X86GFNI-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86GFNI-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
; X86GFNI-NEXT: vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
-; X86GFNI-NEXT: vpextrd $1, %xmm1, %ecx
-; X86GFNI-NEXT: bswapl %ecx
-; X86GFNI-NEXT: shrdl $16, %ecx, %eax
-; X86GFNI-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86GFNI-NEXT: vmovd %xmm1, %eax
-; X86GFNI-NEXT: bswapl %eax
-; X86GFNI-NEXT: shrdl $16, %eax, %ecx
+; X86GFNI-NEXT: vpextrd $1, %xmm1, %edx
+; X86GFNI-NEXT: bswapl %edx
+; X86GFNI-NEXT: movl %edx, %eax
+; X86GFNI-NEXT: shll $16, %eax
+; X86GFNI-NEXT: shrl $16, %ecx
+; X86GFNI-NEXT: orl %eax, %ecx
; X86GFNI-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86GFNI-NEXT: vmovd %xmm1, %ecx
+; X86GFNI-NEXT: bswapl %ecx
+; X86GFNI-NEXT: movl %ecx, %eax
+; X86GFNI-NEXT: shll $16, %eax
+; X86GFNI-NEXT: shrl $16, %edx
+; X86GFNI-NEXT: orl %eax, %edx
+; X86GFNI-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86GFNI-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
; X86GFNI-NEXT: vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
-; X86GFNI-NEXT: vpextrd $1, %xmm1, %ecx
-; X86GFNI-NEXT: bswapl %ecx
-; X86GFNI-NEXT: shrdl $16, %ecx, %eax
-; X86GFNI-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86GFNI-NEXT: vmovd %xmm1, %eax
-; X86GFNI-NEXT: bswapl %eax
-; X86GFNI-NEXT: shrdl $16, %eax, %ecx
+; X86GFNI-NEXT: vpextrd $1, %xmm1, %edx
+; X86GFNI-NEXT: bswapl %edx
+; X86GFNI-NEXT: movl %edx, %eax
+; X86GFNI-NEXT: shll $16, %eax
+; X86GFNI-NEXT: shrl $16, %ecx
+; X86GFNI-NEXT: orl %eax, %ecx
; X86GFNI-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86GFNI-NEXT: vmovd %xmm1, %ecx
+; X86GFNI-NEXT: bswapl %ecx
+; X86GFNI-NEXT: movl %ecx, %eax
+; X86GFNI-NEXT: shll $16, %eax
+; X86GFNI-NEXT: shrl $16, %edx
+; X86GFNI-NEXT: orl %eax, %edx
+; X86GFNI-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86GFNI-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
; X86GFNI-NEXT: vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
-; X86GFNI-NEXT: vpextrd $1, %xmm1, %ecx
-; X86GFNI-NEXT: bswapl %ecx
-; X86GFNI-NEXT: shrdl $16, %ecx, %eax
-; X86GFNI-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86GFNI-NEXT: vmovd %xmm1, %eax
-; X86GFNI-NEXT: bswapl %eax
-; X86GFNI-NEXT: shrdl $16, %eax, %ecx
+; X86GFNI-NEXT: vpextrd $1, %xmm1, %edx
+; X86GFNI-NEXT: bswapl %edx
+; X86GFNI-NEXT: movl %edx, %eax
+; X86GFNI-NEXT: shll $16, %eax
+; X86GFNI-NEXT: shrl $16, %ecx
+; X86GFNI-NEXT: orl %eax, %ecx
; X86GFNI-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86GFNI-NEXT: vmovd %xmm1, %ecx
+; X86GFNI-NEXT: bswapl %ecx
+; X86GFNI-NEXT: movl %ecx, %eax
+; X86GFNI-NEXT: shll $16, %eax
+; X86GFNI-NEXT: shrl $16, %edx
+; X86GFNI-NEXT: orl %eax, %edx
+; X86GFNI-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86GFNI-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
; X86GFNI-NEXT: vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
; X86GFNI-NEXT: vpextrd $1, %xmm1, %ebp
; X86GFNI-NEXT: bswapl %ebp
-; X86GFNI-NEXT: shrdl $16, %ebp, %eax
-; X86GFNI-NEXT: movl %eax, (%esp) # 4-byte Spill
+; X86GFNI-NEXT: movl %ebp, %eax
+; X86GFNI-NEXT: shll $16, %eax
+; X86GFNI-NEXT: shrl $16, %ecx
+; X86GFNI-NEXT: orl %eax, %ecx
+; X86GFNI-NEXT: movl %ecx, (%esp) # 4-byte Spill
; X86GFNI-NEXT: vmovd %xmm1, %ebx
; X86GFNI-NEXT: bswapl %ebx
-; X86GFNI-NEXT: shrdl $16, %ebx, %ebp
+; X86GFNI-NEXT: movl %ebx, %eax
+; X86GFNI-NEXT: shll $16, %eax
+; X86GFNI-NEXT: shrl $16, %ebp
+; X86GFNI-NEXT: orl %eax, %ebp
; X86GFNI-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
; X86GFNI-NEXT: vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
-; X86GFNI-NEXT: vpextrd $1, %xmm1, %edi
-; X86GFNI-NEXT: bswapl %edi
-; X86GFNI-NEXT: shrdl $16, %edi, %ebx
+; X86GFNI-NEXT: vpextrd $1, %xmm1, %esi
+; X86GFNI-NEXT: bswapl %esi
+; X86GFNI-NEXT: movl %esi, %eax
+; X86GFNI-NEXT: shll $16, %eax
+; X86GFNI-NEXT: shrl $16, %ebx
+; X86GFNI-NEXT: orl %eax, %ebx
; X86GFNI-NEXT: vmovd %xmm1, %edx
; X86GFNI-NEXT: bswapl %edx
-; X86GFNI-NEXT: shrdl $16, %edx, %edi
+; X86GFNI-NEXT: movl %edx, %eax
+; X86GFNI-NEXT: shll $16, %eax
+; X86GFNI-NEXT: shrl $16, %esi
+; X86GFNI-NEXT: orl %eax, %esi
; X86GFNI-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
; X86GFNI-NEXT: vgf2p8affineqb $0, %xmm0, %xmm1, %xmm0
; X86GFNI-NEXT: vpextrd $1, %xmm0, %ecx
; X86GFNI-NEXT: bswapl %ecx
-; X86GFNI-NEXT: shrdl $16, %ecx, %edx
-; X86GFNI-NEXT: vmovd %xmm0, %esi
-; X86GFNI-NEXT: bswapl %esi
-; X86GFNI-NEXT: shrdl $16, %esi, %ecx
+; X86GFNI-NEXT: movl %ecx, %eax
+; X86GFNI-NEXT: shll $16, %eax
+; X86GFNI-NEXT: shrl $16, %edx
+; X86GFNI-NEXT: orl %eax, %edx
+; X86GFNI-NEXT: vmovd %xmm0, %edi
+; X86GFNI-NEXT: bswapl %edi
+; X86GFNI-NEXT: movl %edi, %eax
+; X86GFNI-NEXT: shll $16, %eax
+; X86GFNI-NEXT: shrl $16, %ecx
+; X86GFNI-NEXT: orl %eax, %ecx
; X86GFNI-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86GFNI-NEXT: movl %ecx, 60(%eax)
; X86GFNI-NEXT: movl %edx, 56(%eax)
-; X86GFNI-NEXT: movl %edi, 52(%eax)
+; X86GFNI-NEXT: movl %esi, 52(%eax)
; X86GFNI-NEXT: movl %ebx, 48(%eax)
; X86GFNI-NEXT: movl %ebp, 44(%eax)
; X86GFNI-NEXT: movl (%esp), %ecx # 4-byte Reload
@@ -1515,8 +1655,8 @@ define i528 @large_promotion(i528 %A) nounwind {
; X86GFNI-NEXT: movl %ecx, 4(%eax)
; X86GFNI-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86GFNI-NEXT: movl %ecx, (%eax)
-; X86GFNI-NEXT: shrl $16, %esi
-; X86GFNI-NEXT: movw %si, 64(%eax)
+; X86GFNI-NEXT: shrl $16, %edi
+; X86GFNI-NEXT: movw %di, 64(%eax)
; X86GFNI-NEXT: addl $44, %esp
; X86GFNI-NEXT: popl %esi
; X86GFNI-NEXT: popl %edi
@@ -1526,6 +1666,7 @@ define i528 @large_promotion(i528 %A) nounwind {
;
; X64GFNI-LABEL: large_promotion:
; X64GFNI: # %bb.0:
+; X64GFNI-NEXT: pushq %r15
; X64GFNI-NEXT: pushq %r14
; X64GFNI-NEXT: pushq %rbx
; X64GFNI-NEXT: movq %rdi, %rax
@@ -1538,28 +1679,43 @@ define i528 @large_promotion(i528 %A) nounwind {
; X64GFNI-NEXT: vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
; X64GFNI-NEXT: vmovq %xmm1, %rdi
; X64GFNI-NEXT: bswapq %rdi
+; X64GFNI-NEXT: movq %rdi, %r11
+; X64GFNI-NEXT: shlq $16, %r11
+; X64GFNI-NEXT: movq %r10, %r9
+; X64GFNI-NEXT: shrq $48, %r9
+; X64GFNI-NEXT: orq %r11, %r9
; X64GFNI-NEXT: vmovq %r8, %xmm1
; X64GFNI-NEXT: vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
; X64GFNI-NEXT: vmovq %xmm1, %r8
; X64GFNI-NEXT: bswapq %r8
-; X64GFNI-NEXT: movq %r8, %r9
-; X64GFNI-NEXT: shldq $16, %rdi, %r9
-; X64GFNI-NEXT: shldq $16, %r10, %rdi
+; X64GFNI-NEXT: movq %r8, %r11
+; X64GFNI-NEXT: shlq $16, %r11
+; X64GFNI-NEXT: shrq $48, %rdi
+; X64GFNI-NEXT: orq %r11, %rdi
; X64GFNI-NEXT: vmovq %rcx, %xmm1
; X64GFNI-NEXT: vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
; X64GFNI-NEXT: vmovq %xmm1, %rcx
; X64GFNI-NEXT: bswapq %rcx
-; X64GFNI-NEXT: shrdq $48, %rcx, %r8
+; X64GFNI-NEXT: movq %rcx, %r11
+; X64GFNI-NEXT: shlq $16, %r11
+; X64GFNI-NEXT: shrq $48, %r8
+; X64GFNI-NEXT: orq %r11, %r8
; X64GFNI-NEXT: vmovq %rdx, %xmm1
; X64GFNI-NEXT: vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
; X64GFNI-NEXT: vmovq %xmm1, %rdx
; X64GFNI-NEXT: bswapq %rdx
-; X64GFNI-NEXT: shrdq $48, %rdx, %rcx
+; X64GFNI-NEXT: movq %rdx, %r11
+; X64GFNI-NEXT: shlq $16, %r11
+; X64GFNI-NEXT: shrq $48, %rcx
+; X64GFNI-NEXT: orq %r11, %rcx
; X64GFNI-NEXT: vmovq %rsi, %xmm1
; X64GFNI-NEXT: vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
; X64GFNI-NEXT: vmovq %xmm1, %rsi
; X64GFNI-NEXT: bswapq %rsi
-; X64GFNI-NEXT: shrdq $48, %rsi, %rdx
+; X64GFNI-NEXT: movq %rsi, %r11
+; X64GFNI-NEXT: shlq $16, %r11
+; X64GFNI-NEXT: shrq $48, %rdx
+; X64GFNI-NEXT: orq %r11, %rdx
; X64GFNI-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
; X64GFNI-NEXT: vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
; X64GFNI-NEXT: vmovq %xmm1, %r11
@@ -1568,13 +1724,21 @@ define i528 @large_promotion(i528 %A) nounwind {
; X64GFNI-NEXT: vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
; X64GFNI-NEXT: vmovq %xmm1, %rbx
; X64GFNI-NEXT: bswapq %rbx
-; X64GFNI-NEXT: shrdq $48, %rbx, %r11
+; X64GFNI-NEXT: movq %rbx, %r14
+; X64GFNI-NEXT: shlq $16, %r14
+; X64GFNI-NEXT: shrq $48, %r11
+; X64GFNI-NEXT: orq %r14, %r11
; X64GFNI-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
; X64GFNI-NEXT: vgf2p8affineqb $0, %xmm0, %xmm1, %xmm0
; X64GFNI-NEXT: vmovq %xmm0, %r14
; X64GFNI-NEXT: bswapq %r14
-; X64GFNI-NEXT: shrdq $48, %r14, %rbx
-; X64GFNI-NEXT: shrdq $48, %r10, %r14
+; X64GFNI-NEXT: movq %r14, %r15
+; X64GFNI-NEXT: shlq $16, %r15
+; X64GFNI-NEXT: shrq $48, %rbx
+; X64GFNI-NEXT: orq %r15, %rbx
+; X64GFNI-NEXT: shlq $16, %r10
+; X64GFNI-NEXT: shrq $48, %r14
+; X64GFNI-NEXT: orq %r10, %r14
; X64GFNI-NEXT: shrq $48, %rsi
; X64GFNI-NEXT: movq %r14, 16(%rax)
; X64GFNI-NEXT: movq %rbx, 8(%rax)
@@ -1582,11 +1746,12 @@ define i528 @large_promotion(i528 %A) nounwind {
; X64GFNI-NEXT: movq %rdx, 56(%rax)
; X64GFNI-NEXT: movq %rcx, 48(%rax)
; X64GFNI-NEXT: movq %r8, 40(%rax)
-; X64GFNI-NEXT: movq %r9, 32(%rax)
-; X64GFNI-NEXT: movq %rdi, 24(%rax)
+; X64GFNI-NEXT: movq %rdi, 32(%rax)
+; X64GFNI-NEXT: movq %r9, 24(%rax)
; X64GFNI-NEXT: movw %si, 64(%rax)
; X64GFNI-NEXT: popq %rbx
; X64GFNI-NEXT: popq %r14
+; X64GFNI-NEXT: popq %r15
; X64GFNI-NEXT: retq
%Z = call i528 @llvm.bitreverse.i528(i528 %A)
ret i528 %Z
diff --git a/llvm/test/CodeGen/X86/bswap.ll b/llvm/test/CodeGen/X86/bswap.ll
index 81eac5676bb5c..fd4c281d0c7ba 100644
--- a/llvm/test/CodeGen/X86/bswap.ll
+++ b/llvm/test/CodeGen/X86/bswap.ll
@@ -257,61 +257,109 @@ define i528 @large_promotion(i528 %A) nounwind {
; CHECK-NEXT: movl {{[0-9]+}}(%esp), %esi
; CHECK-NEXT: movl {{[0-9]+}}(%esp), %edx
; CHECK-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax
-; CHECK-NEXT: bswapl %eax
; CHECK-NEXT: bswapl %ecx
-; CHECK-NEXT: shrdl $16, %ecx, %eax
-; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; CHECK-NEXT: bswapl %edx
-; CHECK-NEXT: shrdl $16, %edx, %ecx
+; CHECK-NEXT: movl %edx, %eax
+; CHECK-NEXT: shll $16, %eax
+; CHECK-NEXT: shrl $16, %ecx
+; CHECK-NEXT: orl %eax, %ecx
; CHECK-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; CHECK-NEXT: bswapl %esi
-; CHECK-NEXT: shrdl $16, %esi, %edx
+; CHECK-NEXT: movl %esi, %eax
+; CHECK-NEXT: shll $16, %eax
+; CHECK-NEXT: shrl $16, %edx
+; CHECK-NEXT: orl %eax, %edx
; CHECK-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; CHECK-NEXT: bswapl %edi
-; CHECK-NEXT: shrdl $16, %edi, %esi
+; CHECK-NEXT: movl %edi, %eax
+; CHECK-NEXT: shll $16, %eax
+; CHECK-NEXT: shrl $16, %esi
+; CHECK-NEXT: orl %eax, %esi
; CHECK-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; CHECK-NEXT: bswapl %ebx
-; CHECK-NEXT: shrdl $16, %ebx, %edi
+; CHECK-NEXT: movl %ebx, %eax
+; CHECK-NEXT: shll $16, %eax
+; CHECK-NEXT: shrl $16, %edi
+; CHECK-NEXT: orl %eax, %edi
; CHECK-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; CHECK-NEXT: bswapl %ebp
-; CHECK-NEXT: shrdl $16, %ebp, %ebx
+; CHECK-NEXT: movl %ebp, %eax
+; CHECK-NEXT: shll $16, %eax
+; CHECK-NEXT: shrl $16, %ebx
+; CHECK-NEXT: orl %eax, %ebx
; CHECK-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax
-; CHECK-NEXT: bswapl %eax
-; CHECK-NEXT: shrdl $16, %eax, %ebp
+; CHECK-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT: bswapl %ecx
+; CHECK-NEXT: movl %ecx, %eax
+; CHECK-NEXT: shll $16, %eax
+; CHECK-NEXT: shrl $16, %ebp
+; CHECK-NEXT: orl %eax, %ebp
; CHECK-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT: movl {{[0-9]+}}(%esp), %edx
+; CHECK-NEXT: bswapl %edx
+; CHECK-NEXT: movl %edx, %eax
+; CHECK-NEXT: shll $16, %eax
+; CHECK-NEXT: shrl $16, %ecx
+; CHECK-NEXT: orl %eax, %ecx
+; CHECK-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; CHECK-NEXT: movl {{[0-9]+}}(%esp), %ecx
; CHECK-NEXT: bswapl %ecx
-; CHECK-NEXT: shrdl $16, %ecx, %eax
-; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax
-; CHECK-NEXT: bswapl %eax
-; CHECK-NEXT: shrdl $16, %eax, %ecx
+; CHECK-NEXT: movl %ecx, %eax
+; CHECK-NEXT: shll $16, %eax
+; CHECK-NEXT: shrl $16, %edx
+; CHECK-NEXT: orl %eax, %edx
+; CHECK-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT: movl {{[0-9]+}}(%esp), %edx
+; CHECK-NEXT: bswapl %edx
+; CHECK-NEXT: movl %edx, %eax
+; CHECK-NEXT: shll $16, %eax
+; CHECK-NEXT: shrl $16, %ecx
+; CHECK-NEXT: orl %eax, %ecx
; CHECK-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; CHECK-NEXT: movl {{[0-9]+}}(%esp), %ecx
; CHECK-NEXT: bswapl %ecx
-; CHECK-NEXT: shrdl $16, %ecx, %eax
-; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT: movl %ecx, %eax
+; CHECK-NEXT: shll $16, %eax
+; CHECK-NEXT: shrl $16, %edx
+; CHECK-NEXT: orl %eax, %edx
+; CHECK-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; CHECK-NEXT: movl {{[0-9]+}}(%esp), %ebp
; CHECK-NEXT: bswapl %ebp
-; CHECK-NEXT: shrdl $16, %ebp, %ecx
+; CHECK-NEXT: movl %ebp, %eax
+; CHECK-NEXT: shll $16, %eax
+; CHECK-NEXT: shrl $16, %ecx
+; CHECK-NEXT: orl %eax, %ecx
; CHECK-NEXT: movl %ecx, (%esp) # 4-byte Spill
; CHECK-NEXT: movl {{[0-9]+}}(%esp), %ebx
; CHECK-NEXT: bswapl %ebx
-; CHECK-NEXT: shrdl $16, %ebx, %ebp
+; CHECK-NEXT: movl %ebx, %eax
+; CHECK-NEXT: shll $16, %eax
+; CHECK-NEXT: shrl $16, %ebp
+; CHECK-NEXT: orl %eax, %ebp
; CHECK-NEXT: movl {{[0-9]+}}(%esp), %esi
; CHECK-NEXT: bswapl %esi
-; CHECK-NEXT: shrdl $16, %esi, %ebx
+; CHECK-NEXT: movl %esi, %eax
+; CHECK-NEXT: shll $16, %eax
+; CHECK-NEXT: shrl $16, %ebx
+; CHECK-NEXT: orl %eax, %ebx
; CHECK-NEXT: movl {{[0-9]+}}(%esp), %edx
; CHECK-NEXT: bswapl %edx
-; CHECK-NEXT: shrdl $16, %edx, %esi
+; CHECK-NEXT: movl %edx, %eax
+; CHECK-NEXT: shll $16, %eax
+; CHECK-NEXT: shrl $16, %esi
+; CHECK-NEXT: orl %eax, %esi
; CHECK-NEXT: movl {{[0-9]+}}(%esp), %ecx
; CHECK-NEXT: bswapl %ecx
-; CHECK-NEXT: shrdl $16, %ecx, %edx
+; CHECK-NEXT: movl %ecx, %eax
+; CHECK-NEXT: shll $16, %eax
+; CHECK-NEXT: shrl $16, %edx
+; CHECK-NEXT: orl %eax, %edx
; CHECK-NEXT: movl {{[0-9]+}}(%esp), %edi
; CHECK-NEXT: bswapl %edi
-; CHECK-NEXT: shrdl $16, %edi, %ecx
+; CHECK-NEXT: movl %edi, %eax
+; CHECK-NEXT: shll $16, %eax
+; CHECK-NEXT: shrl $16, %ecx
+; CHECK-NEXT: orl %eax, %ecx
; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax
; CHECK-NEXT: movl %ecx, 60(%eax)
; CHECK-NEXT: movl %edx, 56(%eax)
@@ -351,6 +399,7 @@ define i528 @large_promotion(i528 %A) nounwind {
;
; CHECK64-LABEL: large_promotion:
; CHECK64: # %bb.0:
+; CHECK64-NEXT: pushq %r14
; CHECK64-NEXT: pushq %rbx
; CHECK64-NEXT: movq %rdi, %rax
; CHECK64-NEXT: movq {{[0-9]+}}(%rsp), %rbx
@@ -359,21 +408,45 @@ define i528 @large_promotion(i528 %A) nounwind {
; CHECK64-NEXT: movq {{[0-9]+}}(%rsp), %rdi
; CHECK64-NEXT: bswapq %rdi
; CHECK64-NEXT: bswapq %r10
-; CHECK64-NEXT: shrdq $48, %r10, %rdi
+; CHECK64-NEXT: movq %r10, %r14
+; CHECK64-NEXT: shlq $16, %r14
+; CHECK64-NEXT: shrq $48, %rdi
+; CHECK64-NEXT: orq %r14, %rdi
; CHECK64-NEXT: bswapq %r11
-; CHECK64-NEXT: shrdq $48, %r11, %r10
+; CHECK64-NEXT: movq %r11, %r14
+; CHECK64-NEXT: shlq $16, %r14
+; CHECK64-NEXT: shrq $48, %r10
+; CHECK64-NEXT: orq %r14, %r10
; CHECK64-NEXT: bswapq %rbx
-; CHECK64-NEXT: shrdq $48, %rbx, %r11
+; CHECK64-NEXT: movq %rbx, %r14
+; CHECK64-NEXT: shlq $16, %r14
+; CHECK64-NEXT: shrq $48, %r11
+; CHECK64-NEXT: orq %r14, %r11
; CHECK64-NEXT: bswapq %r9
-; CHECK64-NEXT: shrdq $48, %r9, %rbx
+; CHECK64-NEXT: movq %r9, %r14
+; CHECK64-NEXT: shlq $16, %r14
+; CHECK64-NEXT: shrq $48, %rbx
+; CHECK64-NEXT: orq %r14, %rbx
; CHECK64-NEXT: bswapq %r8
-; CHECK64-NEXT: shrdq $48, %r8, %r9
+; CHECK64-NEXT: movq %r8, %r14
+; CHECK64-NEXT: shlq $16, %r14
+; CHECK64-NEXT: shrq $48, %r9
+; CHECK64-NEXT: orq %r14, %r9
; CHECK64-NEXT: bswapq %rcx
-; CHECK64-NEXT: shrdq $48, %rcx, %r8
+; CHECK64-NEXT: movq %rcx, %r14
+; CHECK64-NEXT: shlq $16, %r14
+; CHECK64-NEXT: shrq $48, %r8
+; CHECK64-NEXT: orq %r14, %r8
; CHECK64-NEXT: bswapq %rdx
-; CHECK64-NEXT: shrdq $48, %rdx, %rcx
+; CHECK64-NEXT: movq %rdx, %r14
+; CHECK64-NEXT: shlq $16, %r14
+; CHECK64-NEXT: shrq $48, %rcx
+; CHECK64-NEXT: orq %r14, %rcx
; CHECK64-NEXT: bswapq %rsi
-; CHECK64-NEXT: shrdq $48, %rsi, %rdx
+; CHECK64-NEXT: movq %rsi, %r14
+; CHECK64-NEXT: shlq $16, %r14
+; CHECK64-NEXT: shrq $48, %rdx
+; CHECK64-NEXT: orq %r14, %rdx
; CHECK64-NEXT: shrq $48, %rsi
; CHECK64-NEXT: movq %rdx, 56(%rax)
; CHECK64-NEXT: movq %rcx, 48(%rax)
@@ -385,6 +458,7 @@ define i528 @large_promotion(i528 %A) nounwind {
; CHECK64-NEXT: movq %rdi, (%rax)
; CHECK64-NEXT: movw %si, 64(%rax)
; CHECK64-NEXT: popq %rbx
+; CHECK64-NEXT: popq %r14
; CHECK64-NEXT: retq
%Z = call i528 @llvm.bswap.i528(i528 %A)
ret i528 %Z
diff --git a/llvm/test/CodeGen/X86/clmul.ll b/llvm/test/CodeGen/X86/clmul.ll
index d8f522ae06e62..9e06d282cd0f1 100644
--- a/llvm/test/CodeGen/X86/clmul.ll
+++ b/llvm/test/CodeGen/X86/clmul.ll
@@ -1905,10 +1905,12 @@ define i64 @clmulr_i64(i64 %a, i64 %b) nounwind {
; SSE2-PCLMUL-NEXT: movq %rsi, %xmm0
; SSE2-PCLMUL-NEXT: movq %rdi, %xmm1
; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm0, %xmm1
-; SSE2-PCLMUL-NEXT: movq %xmm1, %rcx
+; SSE2-PCLMUL-NEXT: movq %xmm1, %rax
; SSE2-PCLMUL-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
-; SSE2-PCLMUL-NEXT: movq %xmm0, %rax
-; SSE2-PCLMUL-NEXT: shldq $1, %rcx, %rax
+; SSE2-PCLMUL-NEXT: movq %xmm0, %rcx
+; SSE2-PCLMUL-NEXT: shlq %rcx
+; SSE2-PCLMUL-NEXT: shrq $63, %rax
+; SSE2-PCLMUL-NEXT: orq %rcx, %rax
; SSE2-PCLMUL-NEXT: retq
;
; SSE42-PCLMUL-LABEL: clmulr_i64:
@@ -1916,9 +1918,11 @@ define i64 @clmulr_i64(i64 %a, i64 %b) nounwind {
; SSE42-PCLMUL-NEXT: movq %rsi, %xmm0
; SSE42-PCLMUL-NEXT: movq %rdi, %xmm1
; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm0, %xmm1
-; SSE42-PCLMUL-NEXT: movq %xmm1, %rcx
-; SSE42-PCLMUL-NEXT: pextrq $1, %xmm1, %rax
-; SSE42-PCLMUL-NEXT: shldq $1, %rcx, %rax
+; SSE42-PCLMUL-NEXT: movq %xmm1, %rax
+; SSE42-PCLMUL-NEXT: pextrq $1, %xmm1, %rcx
+; SSE42-PCLMUL-NEXT: shlq %rcx
+; SSE42-PCLMUL-NEXT: shrq $63, %rax
+; SSE42-PCLMUL-NEXT: orq %rcx, %rax
; SSE42-PCLMUL-NEXT: retq
;
; AVX-LABEL: clmulr_i64:
@@ -1926,9 +1930,11 @@ define i64 @clmulr_i64(i64 %a, i64 %b) nounwind {
; AVX-NEXT: vmovq %rsi, %xmm0
; AVX-NEXT: vmovq %rdi, %xmm1
; AVX-NEXT: vpclmulqdq $0, %xmm0, %xmm1, %xmm0
-; AVX-NEXT: vmovq %xmm0, %rcx
-; AVX-NEXT: vpextrq $1, %xmm0, %rax
-; AVX-NEXT: shldq $1, %rcx, %rax
+; AVX-NEXT: vmovq %xmm0, %rax
+; AVX-NEXT: vpextrq $1, %xmm0, %rcx
+; AVX-NEXT: shlq %rcx
+; AVX-NEXT: shrq $63, %rax
+; AVX-NEXT: orq %rcx, %rax
; AVX-NEXT: retq
%a.ext = zext i64 %a to i128
%b.ext = zext i64 %b to i128
diff --git a/llvm/test/CodeGen/X86/combine-bswap.ll b/llvm/test/CodeGen/X86/combine-bswap.ll
index 1f074c877f3ae..821b9a1dae344 100644
--- a/llvm/test/CodeGen/X86/combine-bswap.ll
+++ b/llvm/test/CodeGen/X86/combine-bswap.ll
@@ -411,10 +411,13 @@ define i32 @bs_and_rhs_bs32_multiuse2(i32 %a, i32 %b) #0 {
define i64 @test_bswap64_shift17(i64 %a0) {
; X86-LABEL: test_bswap64_shift17:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: shldl $17, %edx, %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl %eax, %edx
; X86-NEXT: shll $17, %edx
+; X86-NEXT: shll $17, %ecx
+; X86-NEXT: shrl $15, %eax
+; X86-NEXT: orl %ecx, %eax
; X86-NEXT: bswapl %eax
; X86-NEXT: bswapl %edx
; X86-NEXT: retl
diff --git a/llvm/test/CodeGen/X86/const-shift-of-constmasked.ll b/llvm/test/CodeGen/X86/const-shift-of-constmasked.ll
index 142ac754c3f7e..d9eff05d566f5 100644
--- a/llvm/test/CodeGen/X86/const-shift-of-constmasked.ll
+++ b/llvm/test/CodeGen/X86/const-shift-of-constmasked.ll
@@ -1995,11 +1995,14 @@ define i64 @test_i64_2147483647_mask_shl_34(i64 %a0) {
define i64 @test_i64_140737488289792_mask_shl_15(i64 %a0) {
; X86-LABEL: test_i64_140737488289792_mask_shl_15:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl $32767, %edx # imm = 0x7FFF
-; X86-NEXT: andl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: shldl $15, %eax, %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movl %edx, %eax
; X86-NEXT: andl $65536, %eax # imm = 0x10000
+; X86-NEXT: movl $32767, %ecx # imm = 0x7FFF
+; X86-NEXT: andl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: shll $15, %ecx
+; X86-NEXT: shrl $17, %edx
+; X86-NEXT: orl %ecx, %edx
; X86-NEXT: shll $15, %eax
; X86-NEXT: retl
;
@@ -2016,10 +2019,12 @@ define i64 @test_i64_140737488289792_mask_shl_15(i64 %a0) {
define i64 @test_i64_140737488289792_mask_shl_16(i64 %a0) {
; X86-LABEL: test_i64_140737488289792_mask_shl_16:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl $32767, %edx # imm = 0x7FFF
-; X86-NEXT: andl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: shldl $16, %eax, %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movl $32767, %eax # imm = 0x7FFF
+; X86-NEXT: andl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: shll $16, %eax
+; X86-NEXT: shrl $16, %edx
+; X86-NEXT: orl %eax, %edx
; X86-NEXT: xorl %eax, %eax
; X86-NEXT: retl
;
@@ -2036,10 +2041,12 @@ define i64 @test_i64_140737488289792_mask_shl_16(i64 %a0) {
define i64 @test_i64_140737488289792_mask_shl_17(i64 %a0) {
; X86-LABEL: test_i64_140737488289792_mask_shl_17:
; X86: # %bb.0:
-; X86-NEXT: movzwl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: shll $16, %eax
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: shldl $17, %eax, %edx
+; X86-NEXT: movzwl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: shll $16, %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: shll $17, %eax
+; X86-NEXT: shrl $15, %edx
+; X86-NEXT: orl %eax, %edx
; X86-NEXT: xorl %eax, %eax
; X86-NEXT: retl
;
@@ -2056,10 +2063,12 @@ define i64 @test_i64_140737488289792_mask_shl_17(i64 %a0) {
define i64 @test_i64_140737488289792_mask_shl_18(i64 %a0) {
; X86-LABEL: test_i64_140737488289792_mask_shl_18:
; X86: # %bb.0:
-; X86-NEXT: movzwl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: shll $16, %eax
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: shldl $18, %eax, %edx
+; X86-NEXT: movzwl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: shll $16, %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: shll $18, %eax
+; X86-NEXT: shrl $14, %edx
+; X86-NEXT: orl %eax, %edx
; X86-NEXT: xorl %eax, %eax
; X86-NEXT: retl
;
diff --git a/llvm/test/CodeGen/X86/dagcombine-cse.ll b/llvm/test/CodeGen/X86/dagcombine-cse.ll
index 3efd536adc4d1..d68f8c4a08845 100644
--- a/llvm/test/CodeGen/X86/dagcombine-cse.ll
+++ b/llvm/test/CodeGen/X86/dagcombine-cse.ll
@@ -120,7 +120,10 @@ define i96 @square_high(i96 %x) nounwind {
; X64-NEXT: adcq %rsi, %rax
; X64-NEXT: imulq %rcx, %rcx
; X64-NEXT: addq %rax, %rcx
-; X64-NEXT: shrdq $32, %rcx, %r8
+; X64-NEXT: movq %rcx, %rax
+; X64-NEXT: shlq $32, %rax
+; X64-NEXT: shrq $32, %r8
+; X64-NEXT: orq %rax, %r8
; X64-NEXT: shrq $32, %rcx
; X64-NEXT: movq %r8, %rax
; X64-NEXT: movq %rcx, %rdx
diff --git a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
index 00f2e012c8b12..15521219438cd 100644
--- a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
+++ b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
@@ -152,87 +152,88 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: pushl %esi
; X86-NEXT: andl $-16, %esp
; X86-NEXT: subl $176, %esp
-; X86-NEXT: movl 32(%ebp), %eax
-; X86-NEXT: movl 36(%ebp), %ecx
-; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: movl 32(%ebp), %ecx
+; X86-NEXT: movl 36(%ebp), %eax
+; X86-NEXT: movl %eax, %edx
; X86-NEXT: sarl $31, %edx
-; X86-NEXT: xorl %edx, %ecx
-; X86-NEXT: movl %ecx, %edi
; X86-NEXT: xorl %edx, %eax
-; X86-NEXT: movl 28(%ebp), %esi
-; X86-NEXT: xorl %edx, %esi
-; X86-NEXT: movl 24(%ebp), %ecx
+; X86-NEXT: movl %eax, %esi
; X86-NEXT: xorl %edx, %ecx
-; X86-NEXT: subl %edx, %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: sbbl %edx, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: sbbl %edx, %eax
+; X86-NEXT: movl 28(%ebp), %edi
+; X86-NEXT: xorl %edx, %edi
+; X86-NEXT: movl 24(%ebp), %eax
+; X86-NEXT: xorl %edx, %eax
+; X86-NEXT: subl %edx, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: sbbl %edx, %edi
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 52(%ebp), %ecx
-; X86-NEXT: movl %ecx, %edx
-; X86-NEXT: sarl $31, %edx
-; X86-NEXT: movl %ecx, %ebx
-; X86-NEXT: xorl %edx, %ebx
+; X86-NEXT: sbbl %edx, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %edx, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 52(%ebp), %ebx
+; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: sarl $31, %ecx
+; X86-NEXT: xorl %ecx, %ebx
; X86-NEXT: movl 48(%ebp), %esi
-; X86-NEXT: xorl %edx, %esi
-; X86-NEXT: movl 44(%ebp), %eax
-; X86-NEXT: xorl %edx, %eax
+; X86-NEXT: xorl %ecx, %esi
+; X86-NEXT: movl 44(%ebp), %edx
+; X86-NEXT: xorl %ecx, %edx
; X86-NEXT: movl 40(%ebp), %edi
-; X86-NEXT: xorl %edx, %edi
-; X86-NEXT: subl %edx, %edi
-; X86-NEXT: sbbl %edx, %eax
-; X86-NEXT: sbbl %edx, %esi
+; X86-NEXT: xorl %ecx, %edi
+; X86-NEXT: subl %ecx, %edi
+; X86-NEXT: sbbl %ecx, %edx
+; X86-NEXT: sbbl %ecx, %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: sbbl %edx, %ebx
-; X86-NEXT: movl %eax, %ecx
-; X86-NEXT: orl %ebx, %ecx
-; X86-NEXT: movl %edi, %edx
-; X86-NEXT: orl %esi, %edx
-; X86-NEXT: orl %ecx, %edx
-; X86-NEXT: sete %dl
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %ecx, %ebx
+; X86-NEXT: movl %edx, %eax
+; X86-NEXT: orl %ebx, %eax
+; X86-NEXT: movl %edi, %ecx
+; X86-NEXT: orl %esi, %ecx
+; X86-NEXT: orl %eax, %ecx
+; X86-NEXT: sete %cl
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X86-NEXT: orl %ecx, %esi
-; X86-NEXT: sete %cl
-; X86-NEXT: orb %dl, %cl
-; X86-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; X86-NEXT: orl %eax, %esi
+; X86-NEXT: sete %al
+; X86-NEXT: orb %cl, %al
+; X86-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
; X86-NEXT: bsrl %ebx, %esi
; X86-NEXT: xorl $31, %esi
-; X86-NEXT: bsrl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X86-NEXT: xorl $31, %edx
-; X86-NEXT: orl $32, %edx
-; X86-NEXT: testl %ebx, %ebx
-; X86-NEXT: cmovnel %esi, %edx
-; X86-NEXT: bsrl %eax, %esi
-; X86-NEXT: xorl $31, %esi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: bsrl %edi, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: bsrl %eax, %ecx
; X86-NEXT: xorl $31, %ecx
; X86-NEXT: orl $32, %ecx
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: testl %eax, %eax
+; X86-NEXT: testl %ebx, %ebx
; X86-NEXT: cmovnel %esi, %ecx
-; X86-NEXT: orl $64, %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: bsrl %edx, %esi
+; X86-NEXT: xorl $31, %esi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: bsrl %edi, %edi
+; X86-NEXT: xorl $31, %edi
+; X86-NEXT: orl $32, %edi
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: testl %edx, %edx
+; X86-NEXT: movl %edi, %edx
+; X86-NEXT: cmovnel %esi, %edx
+; X86-NEXT: orl $64, %edx
+; X86-NEXT: movl %eax, %esi
; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: orl %ebx, %esi
-; X86-NEXT: cmovnel %edx, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: cmovnel %ecx, %edx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: bsrl %eax, %esi
; X86-NEXT: xorl $31, %esi
-; X86-NEXT: bsrl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X86-NEXT: xorl $31, %edx
-; X86-NEXT: orl $32, %edx
+; X86-NEXT: bsrl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: xorl $31, %ecx
+; X86-NEXT: orl $32, %ecx
; X86-NEXT: testl %eax, %eax
-; X86-NEXT: cmovnel %esi, %edx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: cmovnel %esi, %ecx
; X86-NEXT: bsrl %ebx, %edi
; X86-NEXT: xorl $31, %edi
; X86-NEXT: bsrl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
@@ -243,57 +244,59 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: orl $64, %esi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-NEXT: orl %eax, %edi
-; X86-NEXT: cmovnel %edx, %esi
-; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: xorl %ebx, %ebx
-; X86-NEXT: subl %esi, %eax
-; X86-NEXT: movl $0, %edx
-; X86-NEXT: sbbl %edx, %edx
-; X86-NEXT: movl $0, %esi
-; X86-NEXT: sbbl %esi, %esi
+; X86-NEXT: cmovnel %ecx, %esi
+; X86-NEXT: subl %esi, %edx
; X86-NEXT: movl $0, %edi
; X86-NEXT: sbbl %edi, %edi
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: sbbl %ecx, %ecx
+; X86-NEXT: movl $0, %esi
+; X86-NEXT: sbbl %esi, %esi
; X86-NEXT: cmpb $0, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Reload
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: jne .LBB4_1
; X86-NEXT: # %bb.2: # %select.false.sink
-; X86-NEXT: movl $127, %ecx
-; X86-NEXT: cmpl %eax, %ecx
-; X86-NEXT: movl $0, %ecx
-; X86-NEXT: sbbl %edx, %ecx
-; X86-NEXT: movl $0, %ecx
-; X86-NEXT: sbbl %esi, %ecx
-; X86-NEXT: movl $0, %ecx
-; X86-NEXT: sbbl %edi, %ecx
-; X86-NEXT: setb %cl
+; X86-NEXT: movl $127, %eax
+; X86-NEXT: cmpl %edx, %eax
+; X86-NEXT: movl $0, %eax
+; X86-NEXT: sbbl %edi, %eax
+; X86-NEXT: movl $0, %eax
+; X86-NEXT: sbbl %ecx, %eax
+; X86-NEXT: movl $0, %eax
+; X86-NEXT: sbbl %esi, %eax
+; X86-NEXT: setb %al
; X86-NEXT: .LBB4_3: # %select.end
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X86-NEXT: testb %cl, %cl
+; X86-NEXT: movl 56(%ebp), %esi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: testb %al, %al
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl $0, %ebx
; X86-NEXT: cmovnel %ebx, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $0, %edi
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: cmovnel %edi, %ebx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: cmovnel %edi, %esi
-; X86-NEXT: cmovel {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: cmovnel %ebx, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: cmovnel %ebx, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: cmovel {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: jne .LBB4_4
; X86-NEXT: # %bb.10: # %select.end
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: xorl $127, %eax
-; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X86-NEXT: orl %eax, %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %edx, %ebx
+; X86-NEXT: xorl $127, %ebx
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: movl %edi, %edx
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X86-NEXT: orl %ebx, %edx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NEXT: je .LBB4_11
; X86-NEXT: # %bb.8: # %udiv-bb1
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: xorps %xmm0, %xmm0
@@ -302,8 +305,8 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NEXT: movl %ebx, %ecx
; X86-NEXT: xorb $127, %cl
@@ -312,25 +315,25 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: andb $12, %al
; X86-NEXT: negb %al
; X86-NEXT: movsbl %al, %eax
-; X86-NEXT: movl 152(%esp,%eax), %edx
-; X86-NEXT: movl 156(%esp,%eax), %esi
-; X86-NEXT: shldl %cl, %edx, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 144(%esp,%eax), %esi
-; X86-NEXT: movl 148(%esp,%eax), %eax
-; X86-NEXT: shldl %cl, %eax, %edx
+; X86-NEXT: movl 152(%esp,%eax), %esi
+; X86-NEXT: movl 156(%esp,%eax), %edx
+; X86-NEXT: shldl %cl, %esi, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shldl %cl, %esi, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shll %cl, %esi
+; X86-NEXT: movl 144(%esp,%eax), %edx
+; X86-NEXT: movl 148(%esp,%eax), %eax
+; X86-NEXT: shldl %cl, %eax, %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shldl %cl, %edx, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shll %cl, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: addl $1, %ebx
-; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: adcl $0, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: jb .LBB4_9
; X86-NEXT: # %bb.5: # %udiv-preheader
-; X86-NEXT: movl %ebx, %ecx
; X86-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
@@ -338,25 +341,29 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %ebx, %eax
; X86-NEXT: shrb $3, %al
; X86-NEXT: andb $12, %al
; X86-NEXT: movzbl %al, %eax
-; X86-NEXT: movl 108(%esp,%eax), %ebx
-; X86-NEXT: movl 104(%esp,%eax), %edx
-; X86-NEXT: movl %edx, %esi
-; X86-NEXT: shrdl %cl, %ebx, %esi
+; X86-NEXT: movl 108(%esp,%eax), %esi
+; X86-NEXT: movl 104(%esp,%eax), %edi
+; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: movl %edi, %edx
+; X86-NEXT: shrdl %cl, %esi, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 96(%esp,%eax), %edx
+; X86-NEXT: movl 100(%esp,%eax), %ebx
+; X86-NEXT: movl %ebx, %eax
+; X86-NEXT: shrdl %cl, %edi, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shrl %cl, %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 96(%esp,%eax), %esi
-; X86-NEXT: movl 100(%esp,%eax), %edi
-; X86-NEXT: movl %edi, %eax
-; X86-NEXT: shrdl %cl, %edx, %eax
-; X86-NEXT: shrl %cl, %ebx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NEXT: shrdl %cl, %edi, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shrdl %cl, %ebx, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: addl $-1, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
@@ -366,177 +373,208 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: adcl $-1, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: adcl $-1, %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: adcl $-1, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: .p2align 4
; X86-NEXT: .LBB4_6: # %udiv-do-while
; X86-NEXT: # =>This Inner Loop Header: Depth=1
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: shldl $1, %edx, %ebx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %eax, %esi
-; X86-NEXT: shldl $1, %eax, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shldl $1, %eax, %esi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: shldl $1, %ebx, %eax
+; X86-NEXT: shll %ebx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: shldl $1, %edi, %ebx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: orl %edx, %ebx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shldl $1, %ecx, %edi
-; X86-NEXT: orl %edx, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %edi, %edx
+; X86-NEXT: shrl $31, %edx
+; X86-NEXT: orl %ebx, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shll %edi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: movl %ebx, %edx
+; X86-NEXT: shrl $31, %edx
+; X86-NEXT: orl %edi, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shll %ebx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: shldl $1, %edi, %ecx
-; X86-NEXT: orl %edx, %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: addl %edi, %edi
-; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: cmpl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: sbbl %esi, %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: sbbl %edx, %ecx
-; X86-NEXT: sarl $31, %ecx
-; X86-NEXT: movl %ecx, %edi
-; X86-NEXT: andl $1, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %ecx, %edi
-; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %edi, %edx
+; X86-NEXT: shrl $31, %edx
+; X86-NEXT: orl %ebx, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shll %edi
+; X86-NEXT: movl %esi, %edx
+; X86-NEXT: shrl $31, %edx
+; X86-NEXT: orl %edi, %edx
+; X86-NEXT: movl %edx, %edi
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shll %esi
+; X86-NEXT: movl %eax, %edx
+; X86-NEXT: shrl $31, %edx
+; X86-NEXT: orl %esi, %edx
+; X86-NEXT: movl %edx, %esi
+; X86-NEXT: shll %eax
; X86-NEXT: movl %ecx, %ebx
-; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X86-NEXT: movl %ecx, %edi
-; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: subl %ecx, %eax
+; X86-NEXT: shrl $31, %ebx
+; X86-NEXT: orl %eax, %ebx
+; X86-NEXT: shll %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl %eax, %edx
+; X86-NEXT: shrl $31, %edx
+; X86-NEXT: orl %ecx, %edx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: orl %ecx, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %ecx, %ebx
+; X86-NEXT: orl %ecx, %edx
+; X86-NEXT: addl %eax, %eax
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: cmpl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-NEXT: sbbl %edi, %esi
-; X86-NEXT: movl %esi, %eax
-; X86-NEXT: sbbl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: movl %edx, %ebx
-; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: addl $-1, %edx
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NEXT: sarl $31, %esi
+; X86-NEXT: movl %esi, %eax
+; X86-NEXT: andl $1, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %esi, %eax
+; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %esi, %eax
+; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NEXT: movl %esi, %ecx
+; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: subl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: sbbl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: sbbl %eax, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: sbbl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: adcl $-1, %ecx
+; X86-NEXT: addl $-1, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: adcl $-1, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NEXT: adcl $-1, %esi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-NEXT: adcl $-1, %edi
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %edi, %ecx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %esi, %edx
-; X86-NEXT: orl %ecx, %edx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: orl %edi, %eax
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %esi, %ecx
+; X86-NEXT: orl %eax, %ecx
+; X86-NEXT: movl %edx, %ecx
+; X86-NEXT: movl %ebx, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: movl %esi, %edi
; X86-NEXT: jne .LBB4_6
; X86-NEXT: .LBB4_7: # %udiv-loop-exit
+; X86-NEXT: leal (,%edx,2), %eax
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shldl $1, %esi, %eax
-; X86-NEXT: shldl $1, %ecx, %esi
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: shldl $1, %edx, %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: leal (%edi,%edx,2), %ebx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: movl %eax, %edi
-; X86-NEXT: .LBB4_11: # %udiv-end
-; X86-NEXT: xorl %edx, %edi
-; X86-NEXT: xorl %edx, %esi
-; X86-NEXT: xorl %edx, %ecx
-; X86-NEXT: xorl %edx, %ebx
-; X86-NEXT: subl %edx, %ebx
-; X86-NEXT: sbbl %edx, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: leal (%ecx,%esi,2), %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: sbbl %edx, %esi
-; X86-NEXT: sbbl %edx, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 56(%ebp), %eax
-; X86-NEXT: movl %ebx, (%eax)
-; X86-NEXT: movl %ecx, 4(%eax)
-; X86-NEXT: movl %esi, 8(%eax)
-; X86-NEXT: movl %edi, 12(%eax)
-; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: movl 40(%ebp), %ecx
-; X86-NEXT: mull %ecx
-; X86-NEXT: movl %edx, %edi
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %ebx, %eax
+; X86-NEXT: shrl $31, %esi
+; X86-NEXT: orl %eax, %esi
+; X86-NEXT: leal (,%ebx,2), %eax
+; X86-NEXT: shrl $31, %edx
+; X86-NEXT: orl %eax, %edx
+; X86-NEXT: shll %edi
+; X86-NEXT: shrl $31, %ebx
+; X86-NEXT: orl %edi, %ebx
; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: mull %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %esi, %eax
+; X86-NEXT: movl 56(%ebp), %esi
+; X86-NEXT: .LBB4_11: # %udiv-end
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: xorl %ecx, %edi
+; X86-NEXT: xorl %ecx, %edx
+; X86-NEXT: xorl %ecx, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: xorl %ecx, %ebx
+; X86-NEXT: subl %ecx, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %ecx, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %edx, %ecx
-; X86-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X86-NEXT: adcl $0, %edi
+; X86-NEXT: sbbl %ecx, %edx
+; X86-NEXT: sbbl %ecx, %edi
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %ebx, %eax
-; X86-NEXT: movl %esi, %edi
-; X86-NEXT: movl 44(%ebp), %esi
-; X86-NEXT: mull %esi
-; X86-NEXT: addl %ecx, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X86-NEXT: movl %ebx, (%esi)
+; X86-NEXT: movl %eax, 4(%esi)
+; X86-NEXT: movl %edx, 8(%esi)
+; X86-NEXT: movl %edi, 12(%esi)
+; X86-NEXT: movl 40(%ebp), %edi
; X86-NEXT: movl %edx, %ecx
-; X86-NEXT: setb %bl
+; X86-NEXT: mull %edi
+; X86-NEXT: movl %edx, %ebx
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NEXT: movl %esi, %eax
-; X86-NEXT: mull 44(%ebp)
-; X86-NEXT: addl %ecx, %eax
+; X86-NEXT: mull %edi
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %edx, %edi
+; X86-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X86-NEXT: adcl $0, %ebx
+; X86-NEXT: movl %esi, %eax
+; X86-NEXT: movl 44(%ebp), %esi
+; X86-NEXT: mull %esi
+; X86-NEXT: addl %edi, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: adcl %ebx, %edx
+; X86-NEXT: movl %edx, %edi
+; X86-NEXT: setb %bl
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: mull %esi
+; X86-NEXT: addl %edi, %eax
+; X86-NEXT: movl %eax, %edi
; X86-NEXT: movzbl %bl, %eax
; X86-NEXT: adcl %eax, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NEXT: movl 40(%ebp), %eax
-; X86-NEXT: imull %eax, %ecx
-; X86-NEXT: mull %edi
+; X86-NEXT: imull %eax, %ebx
+; X86-NEXT: mull %ecx
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: imull 44(%ebp), %edi
-; X86-NEXT: addl %edx, %edi
-; X86-NEXT: addl %ecx, %edi
-; X86-NEXT: movl 48(%ebp), %eax
-; X86-NEXT: movl %eax, %ecx
; X86-NEXT: imull %esi, %ecx
-; X86-NEXT: movl 52(%ebp), %esi
+; X86-NEXT: addl %edx, %ecx
+; X86-NEXT: addl %ebx, %ecx
+; X86-NEXT: movl 48(%ebp), %eax
+; X86-NEXT: movl %eax, %esi
+; X86-NEXT: imull {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NEXT: movl 52(%ebp), %ebx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: imull %edx, %esi
+; X86-NEXT: imull %edx, %ebx
; X86-NEXT: mull %edx
-; X86-NEXT: addl %edx, %esi
-; X86-NEXT: addl %ecx, %esi
-; X86-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NEXT: adcl %edi, %esi
+; X86-NEXT: addl %edx, %ebx
+; X86-NEXT: addl %esi, %ebx
; X86-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NEXT: adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NEXT: adcl %ecx, %ebx
+; X86-NEXT: addl %edi, %eax
+; X86-NEXT: adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
; X86-NEXT: movl 24(%ebp), %edx
; X86-NEXT: subl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
; X86-NEXT: movl 28(%ebp), %ecx
; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X86-NEXT: movl 32(%ebp), %ebx
-; X86-NEXT: sbbl %eax, %ebx
-; X86-NEXT: movl 36(%ebp), %edi
-; X86-NEXT: sbbl %esi, %edi
+; X86-NEXT: movl 32(%ebp), %edi
+; X86-NEXT: sbbl %eax, %edi
+; X86-NEXT: movl 36(%ebp), %esi
+; X86-NEXT: sbbl %ebx, %esi
; X86-NEXT: movl 8(%ebp), %eax
; X86-NEXT: movl %edx, (%eax)
; X86-NEXT: movl %ecx, 4(%eax)
-; X86-NEXT: movl %ebx, 8(%eax)
-; X86-NEXT: movl %edi, 12(%eax)
+; X86-NEXT: movl %edi, 8(%eax)
+; X86-NEXT: movl %esi, 12(%eax)
; X86-NEXT: leal -12(%ebp), %esp
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
@@ -544,14 +582,16 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: popl %ebp
; X86-NEXT: retl $4
; X86-NEXT: .LBB4_1:
-; X86-NEXT: movb $1, %cl
+; X86-NEXT: movb $1, %al
; X86-NEXT: jmp .LBB4_3
; X86-NEXT: .LBB4_9:
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: jmp .LBB4_7
; X86-NEXT: .LBB4_4:
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NEXT: jmp .LBB4_11
;
; X64-LABEL: scalar_i128:
diff --git a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll
index 3d756f3cf2141..1c8e35e807e64 100644
--- a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll
+++ b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll
@@ -152,13 +152,13 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: pushl %esi
; X86-NEXT: andl $-16, %esp
; X86-NEXT: subl $160, %esp
-; X86-NEXT: movl 48(%ebp), %ebx
+; X86-NEXT: movl 48(%ebp), %edi
; X86-NEXT: movl 40(%ebp), %ecx
; X86-NEXT: movl 52(%ebp), %esi
-; X86-NEXT: movl 44(%ebp), %edi
-; X86-NEXT: movl %edi, %eax
+; X86-NEXT: movl 44(%ebp), %ebx
+; X86-NEXT: movl %ebx, %eax
; X86-NEXT: orl %esi, %eax
-; X86-NEXT: orl %ebx, %ecx
+; X86-NEXT: orl %edi, %ecx
; X86-NEXT: orl %eax, %ecx
; X86-NEXT: sete %cl
; X86-NEXT: movl 28(%ebp), %eax
@@ -171,103 +171,103 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
; X86-NEXT: bsrl %esi, %edx
; X86-NEXT: xorl $31, %edx
-; X86-NEXT: bsrl %ebx, %ecx
-; X86-NEXT: movl %ebx, %eax
+; X86-NEXT: movl %edi, %eax
+; X86-NEXT: bsrl %edi, %edi
+; X86-NEXT: xorl $31, %edi
+; X86-NEXT: orl $32, %edi
+; X86-NEXT: testl %esi, %esi
+; X86-NEXT: cmovnel %edx, %edi
+; X86-NEXT: bsrl %ebx, %edx
+; X86-NEXT: xorl $31, %edx
+; X86-NEXT: bsrl 40(%ebp), %ecx
; X86-NEXT: xorl $31, %ecx
; X86-NEXT: orl $32, %ecx
-; X86-NEXT: testl %esi, %esi
+; X86-NEXT: testl %ebx, %ebx
; X86-NEXT: cmovnel %edx, %ecx
-; X86-NEXT: bsrl %edi, %edx
-; X86-NEXT: xorl $31, %edx
-; X86-NEXT: bsrl 40(%ebp), %ebx
-; X86-NEXT: xorl $31, %ebx
-; X86-NEXT: orl $32, %ebx
-; X86-NEXT: testl %edi, %edi
-; X86-NEXT: cmovnel %edx, %ebx
-; X86-NEXT: orl $64, %ebx
+; X86-NEXT: orl $64, %ecx
; X86-NEXT: movl %eax, %edx
; X86-NEXT: orl %esi, %edx
-; X86-NEXT: cmovnel %ecx, %ebx
-; X86-NEXT: movl 36(%ebp), %edi
-; X86-NEXT: bsrl %edi, %edx
+; X86-NEXT: cmovnel %edi, %ecx
+; X86-NEXT: movl 36(%ebp), %ebx
+; X86-NEXT: bsrl %ebx, %edx
; X86-NEXT: xorl $31, %edx
-; X86-NEXT: bsrl 32(%ebp), %ecx
-; X86-NEXT: xorl $31, %ecx
-; X86-NEXT: orl $32, %ecx
-; X86-NEXT: testl %edi, %edi
-; X86-NEXT: cmovnel %edx, %ecx
+; X86-NEXT: bsrl 32(%ebp), %edi
+; X86-NEXT: xorl $31, %edi
+; X86-NEXT: orl $32, %edi
+; X86-NEXT: testl %ebx, %ebx
+; X86-NEXT: cmovnel %edx, %edi
; X86-NEXT: movl 28(%ebp), %eax
-; X86-NEXT: bsrl %eax, %esi
-; X86-NEXT: xorl $31, %esi
-; X86-NEXT: bsrl 24(%ebp), %edx
+; X86-NEXT: bsrl %eax, %edx
; X86-NEXT: xorl $31, %edx
-; X86-NEXT: orl $32, %edx
+; X86-NEXT: bsrl 24(%ebp), %esi
+; X86-NEXT: xorl $31, %esi
+; X86-NEXT: orl $32, %esi
; X86-NEXT: testl %eax, %eax
-; X86-NEXT: movl 32(%ebp), %eax
-; X86-NEXT: cmovnel %esi, %edx
-; X86-NEXT: orl $64, %edx
-; X86-NEXT: movl %eax, %esi
-; X86-NEXT: orl %edi, %esi
-; X86-NEXT: cmovnel %ecx, %edx
-; X86-NEXT: xorl %ecx, %ecx
-; X86-NEXT: subl %edx, %ebx
-; X86-NEXT: movl %ebx, %edx
+; X86-NEXT: cmovnel %edx, %esi
+; X86-NEXT: orl $64, %esi
+; X86-NEXT: movl 32(%ebp), %edx
+; X86-NEXT: orl %ebx, %edx
+; X86-NEXT: cmovnel %edi, %esi
+; X86-NEXT: xorl %edx, %edx
+; X86-NEXT: subl %esi, %ecx
; X86-NEXT: movl $0, %esi
; X86-NEXT: sbbl %esi, %esi
-; X86-NEXT: movl $0, %ebx
-; X86-NEXT: sbbl %ebx, %ebx
; X86-NEXT: movl $0, %edi
; X86-NEXT: sbbl %edi, %edi
+; X86-NEXT: movl $0, %ebx
+; X86-NEXT: sbbl %ebx, %ebx
; X86-NEXT: cmpb $0, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Reload
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: jne .LBB4_1
; X86-NEXT: # %bb.2: # %select.false.sink
; X86-NEXT: movl $127, %eax
-; X86-NEXT: cmpl %edx, %eax
+; X86-NEXT: cmpl %ecx, %eax
; X86-NEXT: movl $0, %eax
; X86-NEXT: sbbl %esi, %eax
; X86-NEXT: movl $0, %eax
-; X86-NEXT: sbbl %ebx, %eax
-; X86-NEXT: movl $0, %eax
; X86-NEXT: sbbl %edi, %eax
+; X86-NEXT: movl $0, %eax
+; X86-NEXT: sbbl %ebx, %eax
; X86-NEXT: setb %al
; X86-NEXT: .LBB4_3: # %select.end
; X86-NEXT: testb %al, %al
-; X86-NEXT: movl 28(%ebp), %edx
-; X86-NEXT: cmovnel %ecx, %edx
+; X86-NEXT: movl 28(%ebp), %esi
+; X86-NEXT: cmovnel %edx, %esi
; X86-NEXT: movl 24(%ebp), %eax
-; X86-NEXT: cmovnel %ecx, %eax
+; X86-NEXT: cmovnel %edx, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 32(%ebp), %eax
+; X86-NEXT: cmovnel %edx, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 32(%ebp), %edi
-; X86-NEXT: cmovnel %ecx, %edi
; X86-NEXT: movl 36(%ebp), %ebx
-; X86-NEXT: cmovnel %ecx, %ebx
-; X86-NEXT: jne .LBB4_9
-; X86-NEXT: # %bb.4: # %select.end
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: movl %esi, %eax
+; X86-NEXT: cmovnel %edx, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 56(%ebp), %edi
+; X86-NEXT: jne .LBB4_4
+; X86-NEXT: # %bb.10: # %select.end
+; X86-NEXT: movl %ecx, %eax
; X86-NEXT: xorl $127, %eax
; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X86-NEXT: orl %eax, %ecx
-; X86-NEXT: movl 28(%ebp), %ecx
-; X86-NEXT: je .LBB4_9
-; X86-NEXT: # %bb.5: # %udiv-bb1
-; X86-NEXT: movl 24(%ebp), %edi
-; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X86-NEXT: orl %eax, %edx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: je .LBB4_11
+; X86-NEXT: # %bb.8: # %udiv-bb1
+; X86-NEXT: movl 24(%ebp), %ebx
+; X86-NEXT: movl %ebx, {{[0-9]+}}(%esp)
; X86-NEXT: xorps %xmm0, %xmm0
; X86-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 32(%ebp), %edx
+; X86-NEXT: movl 28(%ebp), %edx
; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 36(%ebp), %ecx
-; X86-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NEXT: movl %esi, %edx
-; X86-NEXT: movl %edx, %ecx
+; X86-NEXT: movl 32(%ebp), %esi
+; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 36(%ebp), %edi
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %ecx, %edx
; X86-NEXT: xorb $127, %cl
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: shrb $3, %al
@@ -278,50 +278,48 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: movl 140(%esp,%eax), %ebx
; X86-NEXT: shldl %cl, %esi, %ebx
; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 128(%esp,%eax), %edi
-; X86-NEXT: movl 132(%esp,%eax), %ebx
-; X86-NEXT: shldl %cl, %ebx, %esi
+; X86-NEXT: movl 128(%esp,%eax), %ebx
+; X86-NEXT: movl 132(%esp,%eax), %eax
+; X86-NEXT: shldl %cl, %eax, %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shldl %cl, %edi, %ebx
-; X86-NEXT: shll %cl, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: addl $1, %edx
+; X86-NEXT: shldl %cl, %ebx, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shll %cl, %ebx
+; X86-NEXT: movl %edx, %ecx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: addl $1, %ecx
; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: jb .LBB4_10
-; X86-NEXT: # %bb.6: # %udiv-preheader
+; X86-NEXT: jb .LBB4_9
+; X86-NEXT: # %bb.5: # %udiv-preheader
; X86-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 24(%ebp), %eax
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 28(%ebp), %eax
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 32(%ebp), %eax
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 36(%ebp), %eax
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl %edx, %eax
+; X86-NEXT: movl 24(%ebp), %edx
+; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 28(%ebp), %edx
+; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 32(%ebp), %edx
+; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %ecx, %eax
; X86-NEXT: shrb $3, %al
; X86-NEXT: andb $12, %al
; X86-NEXT: movzbl %al, %eax
-; X86-NEXT: movl 92(%esp,%eax), %esi
-; X86-NEXT: movl 88(%esp,%eax), %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %edx, %ecx
-; X86-NEXT: shrdl %cl, %esi, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 80(%esp,%eax), %edx
-; X86-NEXT: movl 84(%esp,%eax), %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: shrdl %cl, %edi, %eax
+; X86-NEXT: movl 92(%esp,%eax), %edi
+; X86-NEXT: movl 88(%esp,%eax), %edx
+; X86-NEXT: movl %edx, %esi
+; X86-NEXT: shrdl %cl, %edi, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 80(%esp,%eax), %esi
+; X86-NEXT: movl 84(%esp,%eax), %ebx
+; X86-NEXT: movl %ebx, %eax
+; X86-NEXT: shrdl %cl, %edx, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shrl %cl, %esi
+; X86-NEXT: shrl %cl, %edi
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shrdl %cl, %eax, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shrdl %cl, %ebx, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl 40(%ebp), %ecx
; X86-NEXT: addl $-1, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
@@ -331,170 +329,197 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: movl 48(%ebp), %ecx
; X86-NEXT: adcl $-1, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 52(%ebp), %ecx
-; X86-NEXT: adcl $-1, %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 52(%ebp), %eax
+; X86-NEXT: adcl $-1, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NEXT: .p2align 4
-; X86-NEXT: .LBB4_7: # %udiv-do-while
+; X86-NEXT: .LBB4_6: # %udiv-do-while
; X86-NEXT: # =>This Inner Loop Header: Depth=1
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shldl $1, %eax, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shll %edi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: movl %ebx, %eax
+; X86-NEXT: shrl $31, %eax
+; X86-NEXT: orl %edi, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shll %ebx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: shldl $1, %edi, %eax
+; X86-NEXT: movl %edi, %eax
+; X86-NEXT: shrl $31, %eax
+; X86-NEXT: orl %ebx, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shldl $1, %eax, %edi
+; X86-NEXT: shll %edi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: movl %ebx, %eax
+; X86-NEXT: shrl $31, %eax
+; X86-NEXT: orl %edi, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shll %ebx
+; X86-NEXT: movl %edx, %edi
+; X86-NEXT: shrl $31, %edi
+; X86-NEXT: orl %ebx, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shll %edx
+; X86-NEXT: movl %esi, %eax
+; X86-NEXT: shrl $31, %eax
+; X86-NEXT: orl %edx, %eax
+; X86-NEXT: movl %eax, %edx
+; X86-NEXT: shll %esi
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: shrl $31, %eax
+; X86-NEXT: orl %esi, %eax
+; X86-NEXT: shll %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: shldl $1, %esi, %eax
-; X86-NEXT: shldl $1, %ecx, %esi
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: orl %edx, %esi
+; X86-NEXT: movl %esi, %ebx
+; X86-NEXT: shrl $31, %ebx
+; X86-NEXT: orl %ecx, %ebx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: orl %ecx, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %ecx, %eax
+; X86-NEXT: orl %ecx, %ebx
+; X86-NEXT: addl %esi, %esi
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shldl $1, %ebx, %ecx
-; X86-NEXT: orl %edx, %ecx
+; X86-NEXT: cmpl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: sbbl %edx, %esi
+; X86-NEXT: sarl $31, %esi
+; X86-NEXT: movl %esi, %ecx
+; X86-NEXT: andl $1, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: shldl $1, %ecx, %ebx
-; X86-NEXT: orl %edx, %ebx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: addl %ecx, %ecx
-; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: movl %esi, %ecx
+; X86-NEXT: andl 52(%ebp), %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: cmpl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: sbbl %edi, %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: movl %esi, %edi
+; X86-NEXT: andl 48(%ebp), %edi
+; X86-NEXT: movl %esi, %ecx
+; X86-NEXT: andl 44(%ebp), %ecx
+; X86-NEXT: andl 40(%ebp), %esi
+; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: subl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: sbbl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: sbbl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: addl $-1, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: adcl $-1, %esi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: adcl $-1, %edi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: sbbl %edx, %ecx
-; X86-NEXT: sarl $31, %ecx
-; X86-NEXT: movl %ecx, %esi
-; X86-NEXT: andl $1, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %ecx, %esi
-; X86-NEXT: andl 52(%ebp), %esi
+; X86-NEXT: adcl $-1, %edx
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %ecx, %esi
-; X86-NEXT: andl 48(%ebp), %esi
-; X86-NEXT: movl %ecx, %ebx
-; X86-NEXT: andl 44(%ebp), %ebx
-; X86-NEXT: andl 40(%ebp), %ecx
-; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: subl %ecx, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: sbbl %ebx, %edi
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %edx, %esi
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: sbbl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: orl %edi, %ecx
+; X86-NEXT: orl %esi, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: movl %edx, %esi
-; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: movl %eax, %esi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: addl $-1, %edx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: adcl $-1, %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: adcl $-1, %eax
-; X86-NEXT: adcl $-1, %edi
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %edi, %ecx
+; X86-NEXT: jne .LBB4_6
+; X86-NEXT: .LBB4_7: # %udiv-loop-exit
+; X86-NEXT: leal (,%ebx,2), %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: leal (%edx,%esi,2), %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %eax, %edx
-; X86-NEXT: orl %ecx, %edx
+; X86-NEXT: shrl $31, %esi
+; X86-NEXT: orl %ecx, %esi
+; X86-NEXT: leal (,%eax,2), %ecx
+; X86-NEXT: shrl $31, %ebx
+; X86-NEXT: orl %ecx, %ebx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: jne .LBB4_7
-; X86-NEXT: .LBB4_8: # %udiv-loop-exit
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shldl $1, %ecx, %eax
-; X86-NEXT: shldl $1, %ebx, %ecx
+; X86-NEXT: shll %ecx
+; X86-NEXT: shrl $31, %eax
+; X86-NEXT: orl %ecx, %eax
+; X86-NEXT: movl 56(%ebp), %edi
+; X86-NEXT: .LBB4_11: # %udiv-end
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: shldl $1, %edx, %ebx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: leal (%esi,%edx,2), %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %ecx, %edi
-; X86-NEXT: movl %ebx, %edx
+; X86-NEXT: movl %edx, (%edi)
+; X86-NEXT: movl %esi, 4(%edi)
+; X86-NEXT: movl %ebx, 8(%edi)
+; X86-NEXT: movl %eax, 12(%edi)
; X86-NEXT: movl %eax, %ebx
-; X86-NEXT: .LBB4_9: # %udiv-end
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 56(%ebp), %eax
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: movl %esi, (%eax)
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %edx, 4(%eax)
-; X86-NEXT: movl %edi, 8(%eax)
-; X86-NEXT: movl %ebx, 12(%eax)
; X86-NEXT: movl 48(%ebp), %eax
; X86-NEXT: movl %eax, %ecx
-; X86-NEXT: imull %edx, %ecx
-; X86-NEXT: mull %esi
+; X86-NEXT: imull %esi, %ecx
+; X86-NEXT: movl %edx, %edi
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: mull %edx
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: addl %ecx, %edx
-; X86-NEXT: movl 52(%ebp), %edi
-; X86-NEXT: imull %esi, %edi
-; X86-NEXT: addl %edx, %edi
-; X86-NEXT: movl 40(%ebp), %eax
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: mull %esi
-; X86-NEXT: movl %eax, %ecx
-; X86-NEXT: imull 40(%ebp), %ebx
+; X86-NEXT: movl 52(%ebp), %esi
+; X86-NEXT: imull %edi, %esi
+; X86-NEXT: addl %edx, %esi
+; X86-NEXT: movl 40(%ebp), %edi
+; X86-NEXT: movl %edi, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: mull %ecx
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: imull %edi, %ebx
; X86-NEXT: addl %edx, %ebx
; X86-NEXT: movl 44(%ebp), %eax
-; X86-NEXT: imull %eax, %esi
-; X86-NEXT: addl %ebx, %esi
-; X86-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: imull %eax, %ecx
+; X86-NEXT: addl %ebx, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: addl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: adcl %esi, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: adcl %edi, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: movl %edi, %eax
-; X86-NEXT: movl 40(%ebp), %ecx
-; X86-NEXT: mull %ecx
-; X86-NEXT: movl %edx, %esi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: movl %esi, %eax
+; X86-NEXT: mull %edi
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NEXT: movl %ebx, %eax
-; X86-NEXT: mull %ecx
+; X86-NEXT: mull %edi
+; X86-NEXT: movl %edx, %edi
; X86-NEXT: movl %eax, %ecx
-; X86-NEXT: addl %esi, %ecx
-; X86-NEXT: adcl $0, %edx
-; X86-NEXT: movl %edx, %esi
-; X86-NEXT: movl %edi, %eax
+; X86-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: adcl $0, %edi
+; X86-NEXT: movl %esi, %eax
; X86-NEXT: mull 44(%ebp)
-; X86-NEXT: movl %edx, %edi
+; X86-NEXT: movl %edx, %esi
; X86-NEXT: addl %ecx, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: adcl %esi, %edi
+; X86-NEXT: adcl %edi, %esi
; X86-NEXT: setb %cl
; X86-NEXT: movl %ebx, %eax
; X86-NEXT: mull 44(%ebp)
-; X86-NEXT: addl %edi, %eax
+; X86-NEXT: addl %esi, %eax
; X86-NEXT: movzbl %cl, %ecx
; X86-NEXT: adcl %ecx, %edx
; X86-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
; X86-NEXT: adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X86-NEXT: movl 24(%ebp), %edi
-; X86-NEXT: subl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X86-NEXT: movl 28(%ebp), %ebx
-; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X86-NEXT: movl 32(%ebp), %ecx
-; X86-NEXT: sbbl %eax, %ecx
-; X86-NEXT: movl 36(%ebp), %esi
-; X86-NEXT: sbbl %edx, %esi
+; X86-NEXT: movl 24(%ebp), %ebx
+; X86-NEXT: subl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X86-NEXT: movl 28(%ebp), %ecx
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: movl 32(%ebp), %esi
+; X86-NEXT: sbbl %eax, %esi
+; X86-NEXT: movl 36(%ebp), %edi
+; X86-NEXT: sbbl %edx, %edi
; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: movl %edi, (%eax)
-; X86-NEXT: movl %ebx, 4(%eax)
-; X86-NEXT: movl %ecx, 8(%eax)
-; X86-NEXT: movl %esi, 12(%eax)
+; X86-NEXT: movl %ebx, (%eax)
+; X86-NEXT: movl %ecx, 4(%eax)
+; X86-NEXT: movl %esi, 8(%eax)
+; X86-NEXT: movl %edi, 12(%eax)
; X86-NEXT: leal -12(%ebp), %esp
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
@@ -504,10 +529,17 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: .LBB4_1:
; X86-NEXT: movb $1, %al
; X86-NEXT: jmp .LBB4_3
-; X86-NEXT: .LBB4_10:
+; X86-NEXT: .LBB4_9:
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: jmp .LBB4_8
+; X86-NEXT: jmp .LBB4_7
+; X86-NEXT: .LBB4_4:
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: jmp .LBB4_11
;
; X64-LABEL: scalar_i128:
; X64: # %bb.0:
diff --git a/llvm/test/CodeGen/X86/div_i129_v_pow2k.ll b/llvm/test/CodeGen/X86/div_i129_v_pow2k.ll
index 0924ea85a0126..628315d80408f 100644
--- a/llvm/test/CodeGen/X86/div_i129_v_pow2k.ll
+++ b/llvm/test/CodeGen/X86/div_i129_v_pow2k.ll
@@ -8,11 +8,14 @@ define i129 @v_sdiv_i129_v_pow2k(i129 %lhs) nounwind {
; X64-LABEL: v_sdiv_i129_v_pow2k:
; X64: # %bb.0:
; X64-NEXT: movq %rdx, %rcx
-; X64-NEXT: andl $1, %edx
-; X64-NEXT: negq %rdx
-; X64-NEXT: movl %edx, %eax
+; X64-NEXT: movl %ecx, %eax
; X64-NEXT: andl $1, %eax
-; X64-NEXT: shldq $32, %rdx, %rax
+; X64-NEXT: negq %rax
+; X64-NEXT: movl %eax, %edx
+; X64-NEXT: andl $1, %edx
+; X64-NEXT: shlq $32, %rdx
+; X64-NEXT: shrq $32, %rax
+; X64-NEXT: orq %rdx, %rax
; X64-NEXT: addq %rdi, %rax
; X64-NEXT: adcq $0, %rsi
; X64-NEXT: adcq $0, %rcx
@@ -20,7 +23,9 @@ define i129 @v_sdiv_i129_v_pow2k(i129 %lhs) nounwind {
; X64-NEXT: movq %rcx, %rdx
; X64-NEXT: negq %rdx
; X64-NEXT: shrdq $33, %rsi, %rax
-; X64-NEXT: shldq $31, %rsi, %rdx
+; X64-NEXT: shlq $31, %rdx
+; X64-NEXT: shrq $33, %rsi
+; X64-NEXT: orq %rsi, %rdx
; X64-NEXT: retq
;
; X64-O0-LABEL: v_sdiv_i129_v_pow2k:
@@ -49,40 +54,46 @@ define i129 @v_sdiv_i129_v_pow2k(i129 %lhs) nounwind {
;
; X86-LABEL: v_sdiv_i129_v_pow2k:
; X86: # %bb.0:
-; X86-NEXT: pushl %ebp
; X86-NEXT: pushl %ebx
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ecx, %ebx
-; X86-NEXT: andl $1, %ebx
-; X86-NEXT: negl %ebx
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl %ebx, %edi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT: movl %esi, %edi
; X86-NEXT: andl $1, %edi
-; X86-NEXT: addl {{[0-9]+}}(%esp), %ebx
-; X86-NEXT: adcl {{[0-9]+}}(%esp), %edi
-; X86-NEXT: adcl $0, %esi
+; X86-NEXT: negl %edi
+; X86-NEXT: movl %edi, %ecx
+; X86-NEXT: andl $1, %ecx
+; X86-NEXT: addl {{[0-9]+}}(%esp), %edi
+; X86-NEXT: adcl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: adcl $0, %eax
; X86-NEXT: adcl $0, %edx
-; X86-NEXT: adcl $0, %ecx
-; X86-NEXT: movl %ecx, %ebx
+; X86-NEXT: adcl $0, %esi
+; X86-NEXT: movl %esi, %ebx
+; X86-NEXT: shll $31, %esi
+; X86-NEXT: movl %edx, %edi
+; X86-NEXT: shrl %edi
+; X86-NEXT: orl %esi, %edi
+; X86-NEXT: shll $31, %edx
+; X86-NEXT: movl %eax, %esi
+; X86-NEXT: shrl %esi
+; X86-NEXT: orl %edx, %esi
+; X86-NEXT: shll $31, %eax
+; X86-NEXT: shrl %ecx
+; X86-NEXT: orl %eax, %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %ecx, (%eax)
+; X86-NEXT: movl %esi, 4(%eax)
; X86-NEXT: andl $1, %ebx
-; X86-NEXT: movl %ebx, %ebp
-; X86-NEXT: negl %ebp
-; X86-NEXT: shldl $31, %edx, %ecx
-; X86-NEXT: shldl $31, %esi, %edx
-; X86-NEXT: shldl $31, %edi, %esi
-; X86-NEXT: movl %esi, (%eax)
-; X86-NEXT: movl %edx, 4(%eax)
-; X86-NEXT: movl %ecx, 8(%eax)
-; X86-NEXT: movl %ebp, 12(%eax)
+; X86-NEXT: movl %edi, 8(%eax)
+; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: negl %ecx
+; X86-NEXT: movl %ecx, 12(%eax)
; X86-NEXT: movb %bl, 16(%eax)
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
; X86-NEXT: popl %ebx
-; X86-NEXT: popl %ebp
; X86-NEXT: retl $4
;
; X86-O0-LABEL: v_sdiv_i129_v_pow2k:
@@ -145,7 +156,9 @@ define i129 @v_sdiv_exact_i129_v_pow2k(i129 %lhs) nounwind {
; X64-NEXT: movq %rcx, %rdx
; X64-NEXT: negq %rdx
; X64-NEXT: shrdq $33, %rsi, %rax
-; X64-NEXT: shldq $31, %rsi, %rdx
+; X64-NEXT: shlq $31, %rdx
+; X64-NEXT: shrq $33, %rsi
+; X64-NEXT: orq %rsi, %rdx
; X64-NEXT: retq
;
; X64-O0-LABEL: v_sdiv_exact_i129_v_pow2k:
@@ -166,22 +179,30 @@ define i129 @v_sdiv_exact_i129_v_pow2k(i129 %lhs) nounwind {
; X86-NEXT: pushl %ebx
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl %edx, %ecx
-; X86-NEXT: andl $1, %ecx
-; X86-NEXT: movl %ecx, %esi
-; X86-NEXT: negl %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: andl $1, %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X86-NEXT: shrdl $1, %ebx, %edi
-; X86-NEXT: shldl $31, %ebp, %edx
-; X86-NEXT: shldl $31, %ebx, %ebp
-; X86-NEXT: movl %esi, 12(%eax)
-; X86-NEXT: movl %edx, 8(%eax)
-; X86-NEXT: movl %ebp, 4(%eax)
-; X86-NEXT: movl %edi, (%eax)
+; X86-NEXT: movl %esi, %ebx
+; X86-NEXT: shll $31, %ebx
+; X86-NEXT: shrl %edx
+; X86-NEXT: orl %ebx, %edx
+; X86-NEXT: shll $31, %eax
+; X86-NEXT: movl %edi, %ebx
+; X86-NEXT: shrl %ebx
+; X86-NEXT: orl %eax, %ebx
+; X86-NEXT: movl %ecx, %ebp
+; X86-NEXT: negl %ebp
+; X86-NEXT: shll $31, %edi
+; X86-NEXT: shrl %esi
+; X86-NEXT: orl %edi, %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %ebp, 12(%eax)
+; X86-NEXT: movl %ebx, 8(%eax)
+; X86-NEXT: movl %esi, 4(%eax)
+; X86-NEXT: movl %edx, (%eax)
; X86-NEXT: movb %cl, 16(%eax)
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
@@ -230,7 +251,9 @@ define i129 @v_udiv_i129_v_pow2k(i129 %lhs) nounwind {
; X64-NEXT: movq %rdi, %rax
; X64-NEXT: andl $1, %edx
; X64-NEXT: shrdq $33, %rsi, %rax
-; X64-NEXT: shldq $31, %rsi, %rdx
+; X64-NEXT: shlq $31, %rdx
+; X64-NEXT: shrq $33, %rsi
+; X64-NEXT: orq %rsi, %rdx
; X64-NEXT: xorl %ecx, %ecx
; X64-NEXT: retq
;
@@ -248,23 +271,33 @@ define i129 @v_udiv_i129_v_pow2k(i129 %lhs) nounwind {
;
; X86-LABEL: v_udiv_i129_v_pow2k:
; X86: # %bb.0:
+; X86-NEXT: pushl %ebx
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT: shrdl $1, %esi, %edx
-; X86-NEXT: shldl $31, %edi, %ecx
-; X86-NEXT: shldl $31, %esi, %edi
-; X86-NEXT: movl %ecx, 8(%eax)
-; X86-NEXT: movl %edi, 4(%eax)
-; X86-NEXT: movl %edx, (%eax)
+; X86-NEXT: movl %edx, %ebx
+; X86-NEXT: shll $31, %ebx
+; X86-NEXT: shrl %ecx
+; X86-NEXT: orl %ebx, %ecx
+; X86-NEXT: shll $31, %edi
+; X86-NEXT: movl %esi, %ebx
+; X86-NEXT: shrl %ebx
+; X86-NEXT: orl %edi, %ebx
+; X86-NEXT: shll $31, %esi
+; X86-NEXT: shrl %edx
+; X86-NEXT: orl %esi, %edx
+; X86-NEXT: movl %ebx, 8(%eax)
+; X86-NEXT: movl %edx, 4(%eax)
+; X86-NEXT: movl %ecx, (%eax)
; X86-NEXT: movl $0, 12(%eax)
; X86-NEXT: movb $0, 16(%eax)
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
+; X86-NEXT: popl %ebx
; X86-NEXT: retl $4
;
; X86-O0-LABEL: v_udiv_i129_v_pow2k:
@@ -301,7 +334,9 @@ define i129 @v_udiv_exact_i129_v_pow2k(i129 %lhs) nounwind {
; X64-NEXT: movq %rdi, %rax
; X64-NEXT: andl $1, %edx
; X64-NEXT: shrdq $33, %rsi, %rax
-; X64-NEXT: shldq $31, %rsi, %rdx
+; X64-NEXT: shlq $31, %rdx
+; X64-NEXT: shrq $33, %rsi
+; X64-NEXT: orq %rsi, %rdx
; X64-NEXT: xorl %ecx, %ecx
; X64-NEXT: retq
;
@@ -319,23 +354,33 @@ define i129 @v_udiv_exact_i129_v_pow2k(i129 %lhs) nounwind {
;
; X86-LABEL: v_udiv_exact_i129_v_pow2k:
; X86: # %bb.0:
+; X86-NEXT: pushl %ebx
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT: shrdl $1, %esi, %edx
-; X86-NEXT: shldl $31, %edi, %ecx
-; X86-NEXT: shldl $31, %esi, %edi
-; X86-NEXT: movl %ecx, 8(%eax)
-; X86-NEXT: movl %edi, 4(%eax)
-; X86-NEXT: movl %edx, (%eax)
+; X86-NEXT: movl %edx, %ebx
+; X86-NEXT: shll $31, %ebx
+; X86-NEXT: shrl %ecx
+; X86-NEXT: orl %ebx, %ecx
+; X86-NEXT: shll $31, %edi
+; X86-NEXT: movl %esi, %ebx
+; X86-NEXT: shrl %ebx
+; X86-NEXT: orl %edi, %ebx
+; X86-NEXT: shll $31, %esi
+; X86-NEXT: shrl %edx
+; X86-NEXT: orl %esi, %edx
+; X86-NEXT: movl %ebx, 8(%eax)
+; X86-NEXT: movl %edx, 4(%eax)
+; X86-NEXT: movl %ecx, (%eax)
; X86-NEXT: movl $0, 12(%eax)
; X86-NEXT: movb $0, 16(%eax)
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
+; X86-NEXT: popl %ebx
; X86-NEXT: retl $4
;
; X86-O0-LABEL: v_udiv_exact_i129_v_pow2k:
diff --git a/llvm/test/CodeGen/X86/divide-by-constant.ll b/llvm/test/CodeGen/X86/divide-by-constant.ll
index ac78136b9d8ea..dd9419e6658a9 100644
--- a/llvm/test/CodeGen/X86/divide-by-constant.ll
+++ b/llvm/test/CodeGen/X86/divide-by-constant.ll
@@ -699,9 +699,10 @@ define i64 @urem_i64_65537(i64 %x) nounwind {
; X86-NEXT: movl $-65535, %edx # imm = 0xFFFF0001
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: mull %edx
+; X86-NEXT: shrl $16, %edx
; X86-NEXT: movl %edx, %eax
-; X86-NEXT: shrl $16, %eax
-; X86-NEXT: shldl $16, %edx, %eax
+; X86-NEXT: shll $16, %eax
+; X86-NEXT: orl %edx, %eax
; X86-NEXT: subl %eax, %ecx
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: xorl %edx, %edx
@@ -729,11 +730,14 @@ define i64 @urem_i64_12(i64 %x) nounwind {
; X86: # %bb.0: # %entry
; X86-NEXT: pushl %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: shrl $2, %eax
-; X86-NEXT: shldl $30, %esi, %ecx
-; X86-NEXT: addl %eax, %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %eax, %edx
+; X86-NEXT: shrl $2, %edx
+; X86-NEXT: shll $30, %eax
+; X86-NEXT: movl %esi, %ecx
+; X86-NEXT: shrl $2, %ecx
+; X86-NEXT: orl %eax, %ecx
+; X86-NEXT: addl %edx, %ecx
; X86-NEXT: adcl $0, %ecx
; X86-NEXT: movl $-1431655765, %edx # imm = 0xAAAAAAAB
; X86-NEXT: movl %ecx, %eax
@@ -1084,9 +1088,10 @@ define i64 @udiv_i64_65537(i64 %x) nounwind {
; X86-NEXT: movl $-65535, %ebx # imm = 0xFFFF0001
; X86-NEXT: movl %esi, %eax
; X86-NEXT: mull %ebx
+; X86-NEXT: shrl $16, %edx
; X86-NEXT: movl %edx, %eax
-; X86-NEXT: shrl $16, %eax
-; X86-NEXT: shldl $16, %edx, %eax
+; X86-NEXT: shll $16, %eax
+; X86-NEXT: orl %edx, %eax
; X86-NEXT: subl %eax, %esi
; X86-NEXT: subl %esi, %ecx
; X86-NEXT: sbbl $0, %edi
@@ -1123,9 +1128,12 @@ define i64 @udiv_i64_12(i64 %x) nounwind {
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT: shrdl $2, %edi, %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %eax, %edi
; X86-NEXT: shrl $2, %edi
+; X86-NEXT: shll $30, %eax
+; X86-NEXT: shrl $2, %ecx
+; X86-NEXT: orl %eax, %ecx
; X86-NEXT: movl %ecx, %esi
; X86-NEXT: addl %edi, %esi
; X86-NEXT: adcl $0, %esi
diff --git a/llvm/test/CodeGen/X86/divmod128.ll b/llvm/test/CodeGen/X86/divmod128.ll
index 3796dd796eaf9..2523310dfca71 100644
--- a/llvm/test/CodeGen/X86/divmod128.ll
+++ b/llvm/test/CodeGen/X86/divmod128.ll
@@ -425,8 +425,11 @@ entry:
define i128 @urem_i128_12(i128 %x) nounwind {
; X86-64-LABEL: urem_i128_12:
; X86-64: # %bb.0: # %entry
-; X86-64-NEXT: movq %rsi, %rcx
-; X86-64-NEXT: shldq $62, %rdi, %rcx
+; X86-64-NEXT: movq %rsi, %rax
+; X86-64-NEXT: shlq $62, %rax
+; X86-64-NEXT: movq %rdi, %rcx
+; X86-64-NEXT: shrq $2, %rcx
+; X86-64-NEXT: orq %rax, %rcx
; X86-64-NEXT: shrq $2, %rsi
; X86-64-NEXT: addq %rsi, %rcx
; X86-64-NEXT: adcq $0, %rcx
@@ -443,8 +446,11 @@ define i128 @urem_i128_12(i128 %x) nounwind {
;
; WIN64-LABEL: urem_i128_12:
; WIN64: # %bb.0: # %entry
-; WIN64-NEXT: movq %rdx, %r8
-; WIN64-NEXT: shldq $62, %rcx, %r8
+; WIN64-NEXT: movq %rdx, %rax
+; WIN64-NEXT: shlq $62, %rax
+; WIN64-NEXT: movq %rcx, %r8
+; WIN64-NEXT: shrq $2, %r8
+; WIN64-NEXT: orq %rax, %r8
; WIN64-NEXT: shrq $2, %rdx
; WIN64-NEXT: addq %rdx, %r8
; WIN64-NEXT: adcq $0, %r8
@@ -898,24 +904,27 @@ entry:
define i128 @udiv_i128_12(i128 %x) nounwind {
; X86-64-LABEL: udiv_i128_12:
; X86-64: # %bb.0: # %entry
-; X86-64-NEXT: shrdq $2, %rsi, %rdi
+; X86-64-NEXT: movq %rsi, %rcx
+; X86-64-NEXT: shlq $62, %rcx
+; X86-64-NEXT: shrq $2, %rdi
+; X86-64-NEXT: orq %rdi, %rcx
; X86-64-NEXT: shrq $2, %rsi
-; X86-64-NEXT: movq %rdi, %rcx
-; X86-64-NEXT: addq %rsi, %rcx
-; X86-64-NEXT: adcq $0, %rcx
+; X86-64-NEXT: movq %rcx, %rdi
+; X86-64-NEXT: addq %rsi, %rdi
+; X86-64-NEXT: adcq $0, %rdi
; X86-64-NEXT: movabsq $-6148914691236517205, %r8 # imm = 0xAAAAAAAAAAAAAAAB
-; X86-64-NEXT: movq %rcx, %rax
+; X86-64-NEXT: movq %rdi, %rax
; X86-64-NEXT: mulq %r8
; X86-64-NEXT: shrq %rdx
; X86-64-NEXT: leaq (%rdx,%rdx,2), %rax
-; X86-64-NEXT: subq %rax, %rcx
-; X86-64-NEXT: subq %rcx, %rdi
+; X86-64-NEXT: subq %rax, %rdi
+; X86-64-NEXT: subq %rdi, %rcx
; X86-64-NEXT: sbbq $0, %rsi
-; X86-64-NEXT: movabsq $-6148914691236517206, %rcx # imm = 0xAAAAAAAAAAAAAAAA
-; X86-64-NEXT: imulq %rdi, %rcx
-; X86-64-NEXT: movq %rdi, %rax
+; X86-64-NEXT: movabsq $-6148914691236517206, %rdi # imm = 0xAAAAAAAAAAAAAAAA
+; X86-64-NEXT: imulq %rcx, %rdi
+; X86-64-NEXT: movq %rcx, %rax
; X86-64-NEXT: mulq %r8
-; X86-64-NEXT: addq %rcx, %rdx
+; X86-64-NEXT: addq %rdi, %rdx
; X86-64-NEXT: imulq %rsi, %r8
; X86-64-NEXT: addq %r8, %rdx
; X86-64-NEXT: retq
@@ -923,24 +932,27 @@ define i128 @udiv_i128_12(i128 %x) nounwind {
; WIN64-LABEL: udiv_i128_12:
; WIN64: # %bb.0: # %entry
; WIN64-NEXT: movq %rdx, %r8
-; WIN64-NEXT: shrdq $2, %rdx, %rcx
+; WIN64-NEXT: movq %rdx, %r9
+; WIN64-NEXT: shlq $62, %r9
+; WIN64-NEXT: shrq $2, %rcx
+; WIN64-NEXT: orq %rcx, %r9
; WIN64-NEXT: shrq $2, %r8
-; WIN64-NEXT: movq %rcx, %r9
-; WIN64-NEXT: addq %r8, %r9
-; WIN64-NEXT: adcq $0, %r9
+; WIN64-NEXT: movq %r9, %rcx
+; WIN64-NEXT: addq %r8, %rcx
+; WIN64-NEXT: adcq $0, %rcx
; WIN64-NEXT: movabsq $-6148914691236517205, %r10 # imm = 0xAAAAAAAAAAAAAAAB
-; WIN64-NEXT: movq %r9, %rax
+; WIN64-NEXT: movq %rcx, %rax
; WIN64-NEXT: mulq %r10
; WIN64-NEXT: shrq %rdx
; WIN64-NEXT: leaq (%rdx,%rdx,2), %rax
-; WIN64-NEXT: subq %rax, %r9
-; WIN64-NEXT: subq %r9, %rcx
+; WIN64-NEXT: subq %rax, %rcx
+; WIN64-NEXT: subq %rcx, %r9
; WIN64-NEXT: sbbq $0, %r8
-; WIN64-NEXT: movabsq $-6148914691236517206, %r9 # imm = 0xAAAAAAAAAAAAAAAA
-; WIN64-NEXT: imulq %rcx, %r9
-; WIN64-NEXT: movq %rcx, %rax
+; WIN64-NEXT: movabsq $-6148914691236517206, %rcx # imm = 0xAAAAAAAAAAAAAAAA
+; WIN64-NEXT: imulq %r9, %rcx
+; WIN64-NEXT: movq %r9, %rax
; WIN64-NEXT: mulq %r10
-; WIN64-NEXT: addq %r9, %rdx
+; WIN64-NEXT: addq %rcx, %rdx
; WIN64-NEXT: imulq %r10, %r8
; WIN64-NEXT: addq %r8, %rdx
; WIN64-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/expand-large-fp-optnone.ll b/llvm/test/CodeGen/X86/expand-large-fp-optnone.ll
index 16fe756bfc4e6..8eaf864e8f5b7 100644
--- a/llvm/test/CodeGen/X86/expand-large-fp-optnone.ll
+++ b/llvm/test/CodeGen/X86/expand-large-fp-optnone.ll
@@ -50,18 +50,27 @@ define double @main(i224 %0) #0 {
; CHECK-NEXT: sbbq %rax, %rdx
; CHECK-NEXT: sbbq %rax, %rcx
; CHECK-NEXT: movq %rcx, %r8
-; CHECK-NEXT: shldq $32, %rdx, %r8
+; CHECK-NEXT: shlq $32, %r8
+; CHECK-NEXT: movq %rdx, %rax
+; CHECK-NEXT: shrq $32, %rax
+; CHECK-NEXT: orq %rax, %r8
; CHECK-NEXT: bsrq %r8, %rax
; CHECK-NEXT: xorl $63, %eax
; CHECK-NEXT: movq %rdx, %r10
-; CHECK-NEXT: shldq $32, %rsi, %r10
+; CHECK-NEXT: shlq $32, %r10
+; CHECK-NEXT: movq %rsi, %r11
+; CHECK-NEXT: shrq $32, %r11
+; CHECK-NEXT: orq %r11, %r10
; CHECK-NEXT: bsrq %r10, %r11
; CHECK-NEXT: xorl $63, %r11d
; CHECK-NEXT: orl $64, %r11d
; CHECK-NEXT: testq %r8, %r8
; CHECK-NEXT: cmovnel %eax, %r11d
; CHECK-NEXT: movq %rsi, %rbx
-; CHECK-NEXT: shldq $32, %rdi, %rbx
+; CHECK-NEXT: shlq $32, %rbx
+; CHECK-NEXT: movq %rdi, %rax
+; CHECK-NEXT: shrq $32, %rax
+; CHECK-NEXT: orq %rax, %rbx
; CHECK-NEXT: bsrq %rbx, %r14
; CHECK-NEXT: xorl $63, %r14d
; CHECK-NEXT: movq %rdi, %rax
@@ -93,10 +102,17 @@ define double @main(i224 %0) #0 {
; CHECK-NEXT: jmp .LBB0_6
; CHECK-NEXT: .LBB0_4: # %itofp-sw-bb
; CHECK-NEXT: movq %rsi, %rax
-; CHECK-NEXT: shldq $1, %rdi, %rax
+; CHECK-NEXT: shlq %rax
+; CHECK-NEXT: movq %rdi, %r8
+; CHECK-NEXT: shrq $63, %r8
+; CHECK-NEXT: orq %r8, %rax
; CHECK-NEXT: movq %rdx, %r8
-; CHECK-NEXT: shldq $1, %rsi, %r8
-; CHECK-NEXT: shldq $1, %rdx, %rcx
+; CHECK-NEXT: shlq %r8
+; CHECK-NEXT: shrq $63, %rsi
+; CHECK-NEXT: orq %rsi, %r8
+; CHECK-NEXT: shlq %rcx
+; CHECK-NEXT: shrq $63, %rdx
+; CHECK-NEXT: orq %rdx, %rcx
; CHECK-NEXT: addq %rdi, %rdi
; CHECK-NEXT: movq %rax, %rsi
; CHECK-NEXT: movq %r8, %rdx
diff --git a/llvm/test/CodeGen/X86/fold-tied-op.ll b/llvm/test/CodeGen/X86/fold-tied-op.ll
index 3dc083fbd673b..99254ce3393ca 100644
--- a/llvm/test/CodeGen/X86/fold-tied-op.ll
+++ b/llvm/test/CodeGen/X86/fold-tied-op.ll
@@ -20,70 +20,77 @@ define i64 @fn1() #0 {
; CHECK-NEXT: pushl %ebx
; CHECK-NEXT: pushl %edi
; CHECK-NEXT: pushl %esi
-; CHECK-NEXT: subl $12, %esp
+; CHECK-NEXT: subl $16, %esp
; CHECK-NEXT: .cfi_offset %esi, -20
; CHECK-NEXT: .cfi_offset %edi, -16
; CHECK-NEXT: .cfi_offset %ebx, -12
-; CHECK-NEXT: movl $-1028477379, %eax # imm = 0xC2B2AE3D
-; CHECK-NEXT: movl $668265295, %ebx # imm = 0x27D4EB4F
-; CHECK-NEXT: movl a, %edi
-; CHECK-NEXT: cmpl $0, (%edi)
+; CHECK-NEXT: movl $-1028477379, %ecx # imm = 0xC2B2AE3D
+; CHECK-NEXT: movl a, %eax
+; CHECK-NEXT: cmpl $0, (%eax)
; CHECK-NEXT: je .LBB0_2
; CHECK-NEXT: # %bb.1: # %if.then
-; CHECK-NEXT: movl 8(%edi), %esi
-; CHECK-NEXT: movl 12(%edi), %edx
-; CHECK-NEXT: movl %edx, %eax
-; CHECK-NEXT: shldl $1, %esi, %eax
-; CHECK-NEXT: orl %edx, %eax
-; CHECK-NEXT: leal (%esi,%esi), %ecx
-; CHECK-NEXT: orl %esi, %ecx
-; CHECK-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; CHECK-NEXT: movl 16(%edi), %ecx
-; CHECK-NEXT: movl 20(%edi), %esi
-; CHECK-NEXT: movl %esi, %edx
-; CHECK-NEXT: shldl $2, %ecx, %edx
-; CHECK-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; CHECK-NEXT: leal (,%ecx,4), %edx
+; CHECK-NEXT: movl %eax, %edi
+; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT: movl 8(%eax), %eax
+; CHECK-NEXT: leal (%eax,%eax), %edx
+; CHECK-NEXT: orl %eax, %edx
; CHECK-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; CHECK-NEXT: shrdl $1, %esi, %ecx
-; CHECK-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; CHECK-NEXT: shrl %esi
-; CHECK-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; CHECK-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; CHECK-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; CHECK-NEXT: adcl %eax, %esi
+; CHECK-NEXT: shrl $31, %eax
+; CHECK-NEXT: movl 12(%edi), %esi
+; CHECK-NEXT: leal (,%esi,2), %ebx
+; CHECK-NEXT: orl %ebx, %eax
+; CHECK-NEXT: orl %esi, %eax
+; CHECK-NEXT: movl 16(%edi), %esi
; CHECK-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT: shrl $30, %esi
+; CHECK-NEXT: movl 20(%edi), %ebx
+; CHECK-NEXT: leal (,%ebx,4), %edx
+; CHECK-NEXT: orl %edx, %esi
+; CHECK-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; CHECK-NEXT: leal (,%edi,4), %edx
+; CHECK-NEXT: shrdl $1, %ebx, %edi
+; CHECK-NEXT: orl %edx, %edi
+; CHECK-NEXT: shrl %ebx
+; CHECK-NEXT: orl %esi, %ebx
+; CHECK-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; CHECK-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT: adcl %eax, %ebx
+; CHECK-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; CHECK-NEXT: movl 24(%edi), %eax
; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; CHECK-NEXT: movl $-1028477379, %ecx # imm = 0xC2B2AE3D
; CHECK-NEXT: imull %eax, %ecx
-; CHECK-NEXT: mull %ebx
+; CHECK-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT: movl $668265295, %ecx # imm = 0x27D4EB4F
+; CHECK-NEXT: mull %ecx
; CHECK-NEXT: movl %eax, %esi
-; CHECK-NEXT: addl %ecx, %edx
-; CHECK-NEXT: movl 28(%edi), %edi
-; CHECK-NEXT: imull %edi, %ebx
-; CHECK-NEXT: addl %edx, %ebx
+; CHECK-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; CHECK-NEXT: movl 28(%edi), %eax
+; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT: imull %eax, %ecx
+; CHECK-NEXT: addl %edx, %ecx
; CHECK-NEXT: movl $1336530590, %edx # imm = 0x4FA9D69E
-; CHECK-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; CHECK-NEXT: movl %ecx, %eax
+; CHECK-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; CHECK-NEXT: movl %edi, %eax
; CHECK-NEXT: mull %edx
-; CHECK-NEXT: imull $-2056954758, %ecx, %ecx # imm = 0x85655C7A
-; CHECK-NEXT: addl %edx, %ecx
-; CHECK-NEXT: imull $1336530590, %edi, %edx # imm = 0x4FA9D69E
-; CHECK-NEXT: addl %ecx, %edx
-; CHECK-NEXT: shrdl $3, %ebx, %esi
-; CHECK-NEXT: sarl $3, %ebx
-; CHECK-NEXT: orl %edx, %ebx
-; CHECK-NEXT: orl %eax, %esi
-; CHECK-NEXT: movl $-66860409, %ecx # imm = 0xFC03CA87
+; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT: imull $-2056954758, %edi, %eax # imm = 0x85655C7A
+; CHECK-NEXT: addl %edx, %eax
+; CHECK-NEXT: imull $1336530590, {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; CHECK-NEXT: # imm = 0x4FA9D69E
+; CHECK-NEXT: addl %eax, %edx
+; CHECK-NEXT: shrdl $3, %ecx, %esi
+; CHECK-NEXT: sarl $3, %ecx
+; CHECK-NEXT: orl %edx, %ecx
+; CHECK-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; CHECK-NEXT: movl $-66860409, %edx # imm = 0xFC03CA87
; CHECK-NEXT: movl %esi, %eax
-; CHECK-NEXT: mull %ecx
+; CHECK-NEXT: mull %edx
; CHECK-NEXT: movl %eax, %edi
; CHECK-NEXT: imull $326129324, %esi, %eax # imm = 0x137056AC
; CHECK-NEXT: addl %edx, %eax
-; CHECK-NEXT: imull $-66860409, %ebx, %ecx # imm = 0xFC03CA87
+; CHECK-NEXT: imull $-66860409, %ecx, %ecx # imm = 0xFC03CA87
; CHECK-NEXT: addl %eax, %ecx
-; CHECK-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; CHECK-NEXT: xorl %ebx, %ecx
; CHECK-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
; CHECK-NEXT: movl %edi, b
; CHECK-NEXT: movl %edi, %eax
@@ -95,13 +102,14 @@ define i64 @fn1() #0 {
; CHECK-NEXT: imull $-66860409, %ecx, %ecx # imm = 0xFC03CA87
; CHECK-NEXT: jmp .LBB0_3
; CHECK-NEXT: .LBB0_2: # %if.else
-; CHECK-NEXT: xorl b+4, %eax
-; CHECK-NEXT: xorl b, %ebx
+; CHECK-NEXT: xorl b+4, %ecx
+; CHECK-NEXT: movl $668265295, %esi # imm = 0x27D4EB4F
+; CHECK-NEXT: xorl b, %esi
+; CHECK-NEXT: movl %ecx, %edi
; CHECK-NEXT: movl $1419758215, %ecx # imm = 0x549FCA87
-; CHECK-NEXT: movl %eax, %edi
-; CHECK-NEXT: movl %ebx, %eax
+; CHECK-NEXT: movl %esi, %eax
; CHECK-NEXT: mull %ecx
-; CHECK-NEXT: imull $93298681, %ebx, %esi # imm = 0x58F9FF9
+; CHECK-NEXT: imull $93298681, %esi, %esi # imm = 0x58F9FF9
; CHECK-NEXT: addl %edx, %esi
; CHECK-NEXT: imull $1419758215, %edi, %ecx # imm = 0x549FCA87
; CHECK-NEXT: .LBB0_3: # %if.end
@@ -110,7 +118,7 @@ define i64 @fn1() #0 {
; CHECK-NEXT: adcl $-2048144777, %ecx # imm = 0x85EBCA77
; CHECK-NEXT: movl %eax, b
; CHECK-NEXT: movl %ecx, b+4
-; CHECK-NEXT: addl $12, %esp
+; CHECK-NEXT: addl $16, %esp
; CHECK-NEXT: popl %esi
; CHECK-NEXT: popl %edi
; CHECK-NEXT: popl %ebx
diff --git a/llvm/test/CodeGen/X86/freeze-binary.ll b/llvm/test/CodeGen/X86/freeze-binary.ll
index 46b2571e196bb..35e0bd4a460eb 100644
--- a/llvm/test/CodeGen/X86/freeze-binary.ll
+++ b/llvm/test/CodeGen/X86/freeze-binary.ll
@@ -763,17 +763,21 @@ declare <4 x i32> @llvm.fshr.v4i32(<4 x i32>, <4 x i32>, <4 x i32>)
define i32 @freeze_fshl(i32 %a0, i32 %a1, i32 %a2) nounwind {
; X86-LABEL: freeze_fshl:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: shrl $27, %eax
-; X86-NEXT: shldl $27, %ecx, %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: shrl $27, %ecx
+; X86-NEXT: shll $27, %ecx
+; X86-NEXT: shrl $5, %eax
+; X86-NEXT: orl %ecx, %eax
; X86-NEXT: retl
;
; X64-LABEL: freeze_fshl:
; X64: # %bb.0:
-; X64-NEXT: movl %esi, %eax
-; X64-NEXT: shrl $27, %eax
-; X64-NEXT: shldl $27, %edx, %eax
+; X64-NEXT: movl %edx, %eax
+; X64-NEXT: shrl $27, %esi
+; X64-NEXT: shll $27, %esi
+; X64-NEXT: shrl $5, %eax
+; X64-NEXT: orl %esi, %eax
; X64-NEXT: retq
%f1 = freeze i32 %a1
%f2 = freeze i32 %a2
diff --git a/llvm/test/CodeGen/X86/fshl.ll b/llvm/test/CodeGen/X86/fshl.ll
index 7a6e5f825b97c..455999b4900b7 100644
--- a/llvm/test/CodeGen/X86/fshl.ll
+++ b/llvm/test/CodeGen/X86/fshl.ll
@@ -338,15 +338,19 @@ define i16 @const_shift_i16(i16 %x, i16 %y) nounwind {
define i32 @const_shift_i32(i32 %x, i32 %y) nounwind {
; X86-LABEL: const_shift_i32:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: shldl $7, %ecx, %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: shll $7, %ecx
+; X86-NEXT: shrl $25, %eax
+; X86-NEXT: orl %ecx, %eax
; X86-NEXT: retl
;
; X64-LABEL: const_shift_i32:
; X64: # %bb.0:
-; X64-NEXT: movl %edi, %eax
-; X64-NEXT: shldl $7, %esi, %eax
+; X64-NEXT: movl %esi, %eax
+; X64-NEXT: shll $7, %edi
+; X64-NEXT: shrl $25, %eax
+; X64-NEXT: orl %edi, %eax
; X64-NEXT: retq
%tmp = tail call i32 @llvm.fshl.i32(i32 %x, i32 %y, i32 7)
ret i32 %tmp
@@ -355,17 +359,26 @@ define i32 @const_shift_i32(i32 %x, i32 %y) nounwind {
define i64 @const_shift_i64(i64 %x, i64 %y) nounwind {
; X86-LABEL: const_shift_i64:
; X86: # %bb.0:
+; X86-NEXT: pushl %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: shrdl $25, %ecx, %eax
-; X86-NEXT: shldl $7, %ecx, %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl %edx, %esi
+; X86-NEXT: shll $7, %esi
+; X86-NEXT: shrl $25, %eax
+; X86-NEXT: orl %esi, %eax
+; X86-NEXT: shll $7, %ecx
+; X86-NEXT: shrl $25, %edx
+; X86-NEXT: orl %ecx, %edx
+; X86-NEXT: popl %esi
; X86-NEXT: retl
;
; X64-LABEL: const_shift_i64:
; X64: # %bb.0:
-; X64-NEXT: movq %rdi, %rax
-; X64-NEXT: shldq $7, %rsi, %rax
+; X64-NEXT: movq %rsi, %rax
+; X64-NEXT: shlq $7, %rdi
+; X64-NEXT: shrq $57, %rax
+; X64-NEXT: orq %rdi, %rax
; X64-NEXT: retq
%tmp = tail call i64 @llvm.fshl.i64(i64 %x, i64 %y, i64 7)
ret i64 %tmp
diff --git a/llvm/test/CodeGen/X86/funnel-shift-logic-fold.ll b/llvm/test/CodeGen/X86/funnel-shift-logic-fold.ll
index fb875837cb836..6949661e03c77 100644
--- a/llvm/test/CodeGen/X86/funnel-shift-logic-fold.ll
+++ b/llvm/test/CodeGen/X86/funnel-shift-logic-fold.ll
@@ -52,10 +52,14 @@ define i64 @hoist_fshl_from_xor(i64 %a, i64 %b, i64 %c, i64 %d, i64 %s) nounwind
define i64 @fshl_or_with_different_shift_value(i64 %a, i64 %b, i64 %c, i64 %d) nounwind {
; X64-LABEL: fshl_or_with_different_shift_value:
; X64: # %bb.0:
-; X64-NEXT: movq %rdx, %rax
-; X64-NEXT: shldq $12, %rsi, %rdi
-; X64-NEXT: shldq $13, %rcx, %rax
-; X64-NEXT: orq %rdi, %rax
+; X64-NEXT: movq %rcx, %rax
+; X64-NEXT: shlq $12, %rdi
+; X64-NEXT: shrq $52, %rsi
+; X64-NEXT: orq %rdi, %rsi
+; X64-NEXT: shlq $13, %rdx
+; X64-NEXT: shrq $51, %rax
+; X64-NEXT: orq %rdx, %rax
+; X64-NEXT: orq %rsi, %rax
; X64-NEXT: retq
%fshl.0 = call i64 @llvm.fshl.i64(i64 %a, i64 %b, i64 12)
%fshl.1 = call i64 @llvm.fshl.i64(i64 %c, i64 %d, i64 13)
@@ -66,10 +70,12 @@ define i64 @fshl_or_with_different_shift_value(i64 %a, i64 %b, i64 %c, i64 %d) n
define i64 @hoist_fshl_from_or_const_shift(i64 %a, i64 %b, i64 %c, i64 %d) nounwind {
; X64-LABEL: hoist_fshl_from_or_const_shift:
; X64: # %bb.0:
-; X64-NEXT: movq %rdi, %rax
-; X64-NEXT: orq %rcx, %rsi
-; X64-NEXT: orq %rdx, %rax
-; X64-NEXT: shldq $15, %rsi, %rax
+; X64-NEXT: movq %rsi, %rax
+; X64-NEXT: orq %rcx, %rax
+; X64-NEXT: orq %rdx, %rdi
+; X64-NEXT: shlq $15, %rdi
+; X64-NEXT: shrq $49, %rax
+; X64-NEXT: orq %rdi, %rax
; X64-NEXT: retq
%fshl.0 = call i64 @llvm.fshl.i64(i64 %a, i64 %b, i64 15)
%fshl.1 = call i64 @llvm.fshl.i64(i64 %c, i64 %d, i64 15)
diff --git a/llvm/test/CodeGen/X86/funnel-shift-rot.ll b/llvm/test/CodeGen/X86/funnel-shift-rot.ll
index 7d106fce44555..183cc18aa7401 100644
--- a/llvm/test/CodeGen/X86/funnel-shift-rot.ll
+++ b/llvm/test/CodeGen/X86/funnel-shift-rot.ll
@@ -70,11 +70,17 @@ define i8 @rotl_i8_const_shift7(i8 %x) nounwind {
define i64 @rotl_i64_const_shift(i64 %x) nounwind {
; X86-SSE2-LABEL: rotl_i64_const_shift:
; X86-SSE2: # %bb.0:
-; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE2-NEXT: pushl %esi
; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE2-NEXT: leal (,%edx,8), %esi
; X86-SSE2-NEXT: movl %ecx, %eax
-; X86-SSE2-NEXT: shldl $3, %edx, %eax
-; X86-SSE2-NEXT: shldl $3, %ecx, %edx
+; X86-SSE2-NEXT: shrl $29, %eax
+; X86-SSE2-NEXT: orl %esi, %eax
+; X86-SSE2-NEXT: shll $3, %ecx
+; X86-SSE2-NEXT: shrl $29, %edx
+; X86-SSE2-NEXT: orl %ecx, %edx
+; X86-SSE2-NEXT: popl %esi
; X86-SSE2-NEXT: retl
;
; X64-AVX2-LABEL: rotl_i64_const_shift:
diff --git a/llvm/test/CodeGen/X86/funnel-shift.ll b/llvm/test/CodeGen/X86/funnel-shift.ll
index 318a48b92cb28..3a29310614d69 100644
--- a/llvm/test/CodeGen/X86/funnel-shift.ll
+++ b/llvm/test/CodeGen/X86/funnel-shift.ll
@@ -158,9 +158,12 @@ define i37 @fshl_i37(i37 %x, i37 %y, i37 %z) nounwind {
; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-SSE2-NEXT: andl $31, %eax
; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ebx
; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X86-SSE2-NEXT: shldl $27, %ebx, %edi
+; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE2-NEXT: shll $27, %ecx
+; X86-SSE2-NEXT: movl %edi, %ebx
+; X86-SSE2-NEXT: shrl $5, %ebx
+; X86-SSE2-NEXT: orl %ecx, %ebx
; X86-SSE2-NEXT: pushl $0
; X86-SSE2-NEXT: pushl $37
; X86-SSE2-NEXT: pushl %eax
@@ -171,17 +174,17 @@ define i37 @fshl_i37(i37 %x, i37 %y, i37 %z) nounwind {
; X86-SSE2-NEXT: testb $32, %cl
; X86-SSE2-NEXT: jne .LBB3_1
; X86-SSE2-NEXT: # %bb.2:
-; X86-SSE2-NEXT: movl %edi, %ebx
-; X86-SSE2-NEXT: movl %esi, %edi
+; X86-SSE2-NEXT: movl %ebx, %edi
+; X86-SSE2-NEXT: movl %esi, %ebx
; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %esi
; X86-SSE2-NEXT: jmp .LBB3_3
; X86-SSE2-NEXT: .LBB3_1:
-; X86-SSE2-NEXT: shll $27, %ebx
+; X86-SSE2-NEXT: shll $27, %edi
; X86-SSE2-NEXT: .LBB3_3:
-; X86-SSE2-NEXT: movl %edi, %eax
-; X86-SSE2-NEXT: shldl %cl, %ebx, %eax
+; X86-SSE2-NEXT: movl %ebx, %eax
+; X86-SSE2-NEXT: shldl %cl, %edi, %eax
; X86-SSE2-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-SSE2-NEXT: shldl %cl, %edi, %esi
+; X86-SSE2-NEXT: shldl %cl, %ebx, %esi
; X86-SSE2-NEXT: movl %esi, %edx
; X86-SSE2-NEXT: popl %esi
; X86-SSE2-NEXT: popl %edi
@@ -224,15 +227,19 @@ define i7 @fshl_i7_const_fold() {
define i32 @fshl_i32_const_shift(i32 %x, i32 %y) nounwind {
; X86-SSE2-LABEL: fshl_i32_const_shift:
; X86-SSE2: # %bb.0:
-; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-SSE2-NEXT: shldl $9, %ecx, %eax
+; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE2-NEXT: shll $9, %ecx
+; X86-SSE2-NEXT: shrl $23, %eax
+; X86-SSE2-NEXT: orl %ecx, %eax
; X86-SSE2-NEXT: retl
;
; X64-AVX-LABEL: fshl_i32_const_shift:
; X64-AVX: # %bb.0:
-; X64-AVX-NEXT: movl %edi, %eax
-; X64-AVX-NEXT: shldl $9, %esi, %eax
+; X64-AVX-NEXT: movl %esi, %eax
+; X64-AVX-NEXT: shll $9, %edi
+; X64-AVX-NEXT: shrl $23, %eax
+; X64-AVX-NEXT: orl %edi, %eax
; X64-AVX-NEXT: retq
%f = call i32 @llvm.fshl.i32(i32 %x, i32 %y, i32 9)
ret i32 %f
@@ -243,15 +250,19 @@ define i32 @fshl_i32_const_shift(i32 %x, i32 %y) nounwind {
define i32 @fshl_i32_const_overshift(i32 %x, i32 %y) nounwind {
; X86-SSE2-LABEL: fshl_i32_const_overshift:
; X86-SSE2: # %bb.0:
-; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-SSE2-NEXT: shldl $9, %ecx, %eax
+; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE2-NEXT: shll $9, %ecx
+; X86-SSE2-NEXT: shrl $23, %eax
+; X86-SSE2-NEXT: orl %ecx, %eax
; X86-SSE2-NEXT: retl
;
; X64-AVX-LABEL: fshl_i32_const_overshift:
; X64-AVX: # %bb.0:
-; X64-AVX-NEXT: movl %edi, %eax
-; X64-AVX-NEXT: shldl $9, %esi, %eax
+; X64-AVX-NEXT: movl %esi, %eax
+; X64-AVX-NEXT: shll $9, %edi
+; X64-AVX-NEXT: shrl $23, %eax
+; X64-AVX-NEXT: orl %edi, %eax
; X64-AVX-NEXT: retq
%f = call i32 @llvm.fshl.i32(i32 %x, i32 %y, i32 41)
ret i32 %f
@@ -262,17 +273,26 @@ define i32 @fshl_i32_const_overshift(i32 %x, i32 %y) nounwind {
define i64 @fshl_i64_const_overshift(i64 %x, i64 %y) nounwind {
; X86-SSE2-LABEL: fshl_i64_const_overshift:
; X86-SSE2: # %bb.0:
-; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: pushl %esi
; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-SSE2-NEXT: shldl $9, %ecx, %edx
-; X86-SSE2-NEXT: shrdl $23, %ecx, %eax
+; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-SSE2-NEXT: shll $9, %ecx
+; X86-SSE2-NEXT: movl %esi, %edx
+; X86-SSE2-NEXT: shrl $23, %edx
+; X86-SSE2-NEXT: orl %ecx, %edx
+; X86-SSE2-NEXT: shll $9, %esi
+; X86-SSE2-NEXT: shrl $23, %eax
+; X86-SSE2-NEXT: orl %esi, %eax
+; X86-SSE2-NEXT: popl %esi
; X86-SSE2-NEXT: retl
;
; X64-AVX-LABEL: fshl_i64_const_overshift:
; X64-AVX: # %bb.0:
-; X64-AVX-NEXT: movq %rdi, %rax
-; X64-AVX-NEXT: shldq $41, %rsi, %rax
+; X64-AVX-NEXT: movq %rsi, %rax
+; X64-AVX-NEXT: shlq $41, %rdi
+; X64-AVX-NEXT: shrq $23, %rax
+; X64-AVX-NEXT: orq %rdi, %rax
; X64-AVX-NEXT: retq
%f = call i64 @llvm.fshl.i64(i64 %x, i64 %y, i64 105)
ret i64 %f
@@ -325,8 +345,11 @@ define i37 @fshr_i37(i37 %x, i37 %y, i37 %z) nounwind {
; X86-SSE2-NEXT: andl $31, %eax
; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %edi
; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-SSE2-NEXT: shldl $27, %ebx, %esi
+; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE2-NEXT: shll $27, %ecx
+; X86-SSE2-NEXT: movl %ebx, %esi
+; X86-SSE2-NEXT: shrl $5, %esi
+; X86-SSE2-NEXT: orl %ecx, %esi
; X86-SSE2-NEXT: pushl $0
; X86-SSE2-NEXT: pushl $37
; X86-SSE2-NEXT: pushl %eax
@@ -392,15 +415,19 @@ define i7 @fshr_i7_const_fold() nounwind {
define i32 @fshl_i32_demandedbits(i32 %a0, i32 %a1) nounwind {
; X86-SSE2-LABEL: fshl_i32_demandedbits:
; X86-SSE2: # %bb.0:
-; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-SSE2-NEXT: shldl $9, %ecx, %eax
+; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE2-NEXT: shll $9, %ecx
+; X86-SSE2-NEXT: shrl $23, %eax
+; X86-SSE2-NEXT: orl %ecx, %eax
; X86-SSE2-NEXT: retl
;
; X64-AVX-LABEL: fshl_i32_demandedbits:
; X64-AVX: # %bb.0:
-; X64-AVX-NEXT: movl %edi, %eax
-; X64-AVX-NEXT: shldl $9, %esi, %eax
+; X64-AVX-NEXT: movl %esi, %eax
+; X64-AVX-NEXT: shll $9, %edi
+; X64-AVX-NEXT: shrl $23, %eax
+; X64-AVX-NEXT: orl %edi, %eax
; X64-AVX-NEXT: retq
%x = or i32 %a0, 2147483648
%y = or i32 %a1, 1
@@ -1087,24 +1114,28 @@ define void @PR45265(i32 %0, ptr nocapture readonly %1) nounwind {
; X86-SSE2: # %bb.0:
; X86-SSE2-NEXT: pushl %edi
; X86-SSE2-NEXT: pushl %esi
-; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-SSE2-NEXT: leal (%eax,%eax,2), %esi
-; X86-SSE2-NEXT: movzwl 8(%ecx,%esi,4), %edx
-; X86-SSE2-NEXT: movl 4(%ecx,%esi,4), %edi
-; X86-SSE2-NEXT: shrdl $8, %edx, %edi
-; X86-SSE2-NEXT: xorl %eax, %edi
-; X86-SSE2-NEXT: sarl $31, %eax
-; X86-SSE2-NEXT: movzbl 10(%ecx,%esi,4), %ecx
-; X86-SSE2-NEXT: shll $16, %ecx
-; X86-SSE2-NEXT: orl %edx, %ecx
-; X86-SSE2-NEXT: shll $8, %ecx
-; X86-SSE2-NEXT: movl %ecx, %edx
-; X86-SSE2-NEXT: sarl $8, %edx
+; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-SSE2-NEXT: leal (%ecx,%ecx,2), %edi
+; X86-SSE2-NEXT: movzwl 8(%edx,%edi,4), %esi
+; X86-SSE2-NEXT: movzbl 10(%edx,%edi,4), %eax
+; X86-SSE2-NEXT: shll $16, %eax
+; X86-SSE2-NEXT: orl %esi, %eax
+; X86-SSE2-NEXT: movl 4(%edx,%edi,4), %edx
+; X86-SSE2-NEXT: shll $24, %esi
+; X86-SSE2-NEXT: shrl $8, %edx
+; X86-SSE2-NEXT: orl %esi, %edx
+; X86-SSE2-NEXT: xorl %ecx, %edx
; X86-SSE2-NEXT: sarl $31, %ecx
-; X86-SSE2-NEXT: shldl $24, %edx, %ecx
-; X86-SSE2-NEXT: xorl %eax, %ecx
-; X86-SSE2-NEXT: orl %ecx, %edi
+; X86-SSE2-NEXT: shll $8, %eax
+; X86-SSE2-NEXT: movl %eax, %esi
+; X86-SSE2-NEXT: sarl $8, %esi
+; X86-SSE2-NEXT: sarl $31, %eax
+; X86-SSE2-NEXT: shll $24, %eax
+; X86-SSE2-NEXT: shrl $8, %esi
+; X86-SSE2-NEXT: orl %eax, %esi
+; X86-SSE2-NEXT: xorl %ecx, %esi
+; X86-SSE2-NEXT: orl %esi, %edx
; X86-SSE2-NEXT: jne .LBB50_1
; X86-SSE2-NEXT: # %bb.2:
; X86-SSE2-NEXT: popl %esi
diff --git a/llvm/test/CodeGen/X86/icmp-shift-opt.ll b/llvm/test/CodeGen/X86/icmp-shift-opt.ll
index 0296c2a011e25..49d292a3a03fa 100644
--- a/llvm/test/CodeGen/X86/icmp-shift-opt.ll
+++ b/llvm/test/CodeGen/X86/icmp-shift-opt.ll
@@ -33,7 +33,9 @@ define i128 @opt_setcc_lt_power_of_2(i128 %a) nounwind {
; X86-NEXT: movl %edi, %esi
; X86-NEXT: orl %edx, %esi
; X86-NEXT: orl %ecx, %esi
-; X86-NEXT: shrdl $28, %ebx, %esi
+; X86-NEXT: shll $4, %ebx
+; X86-NEXT: shrl $28, %esi
+; X86-NEXT: orl %ebx, %esi
; X86-NEXT: movl %eax, %esi
; X86-NEXT: jne .LBB0_1
; X86-NEXT: # %bb.2: # %exit
@@ -204,23 +206,32 @@ define i1 @opt_setcc_shl_eq_zero_multiple_shl_users(i128 %a) nounwind {
; X86-NEXT: andl $-16, %esp
; X86-NEXT: subl $16, %esp
; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: movl 12(%ebp), %ecx
-; X86-NEXT: movl 16(%ebp), %edx
-; X86-NEXT: movl 20(%ebp), %esi
-; X86-NEXT: shldl $17, %edx, %esi
-; X86-NEXT: shldl $17, %ecx, %edx
-; X86-NEXT: shldl $17, %eax, %ecx
-; X86-NEXT: shll $17, %eax
+; X86-NEXT: movl 12(%ebp), %esi
+; X86-NEXT: movl 16(%ebp), %ecx
; X86-NEXT: movl %ecx, %edi
-; X86-NEXT: orl %esi, %edi
-; X86-NEXT: movl %eax, %ebx
+; X86-NEXT: shll $17, %edi
+; X86-NEXT: movl %esi, %edx
+; X86-NEXT: shrl $15, %edx
+; X86-NEXT: orl %edi, %edx
+; X86-NEXT: movl 20(%ebp), %edi
+; X86-NEXT: shll $17, %edi
+; X86-NEXT: shrl $15, %ecx
+; X86-NEXT: orl %edi, %ecx
+; X86-NEXT: movl %eax, %edi
+; X86-NEXT: shll $17, %edi
+; X86-NEXT: shll $17, %esi
+; X86-NEXT: shrl $15, %eax
+; X86-NEXT: orl %esi, %eax
+; X86-NEXT: movl %eax, %esi
+; X86-NEXT: orl %ecx, %esi
+; X86-NEXT: movl %edi, %ebx
; X86-NEXT: orl %edx, %ebx
-; X86-NEXT: orl %edi, %ebx
+; X86-NEXT: orl %esi, %ebx
; X86-NEXT: sete %bl
-; X86-NEXT: pushl %esi
-; X86-NEXT: pushl %edx
; X86-NEXT: pushl %ecx
+; X86-NEXT: pushl %edx
; X86-NEXT: pushl %eax
+; X86-NEXT: pushl %edi
; X86-NEXT: calll use at PLT
; X86-NEXT: addl $16, %esp
; X86-NEXT: movl %ebx, %eax
@@ -234,7 +245,10 @@ define i1 @opt_setcc_shl_eq_zero_multiple_shl_users(i128 %a) nounwind {
; X64-LABEL: opt_setcc_shl_eq_zero_multiple_shl_users:
; X64: # %bb.0:
; X64-NEXT: pushq %rbx
-; X64-NEXT: shldq $17, %rdi, %rsi
+; X64-NEXT: shlq $17, %rsi
+; X64-NEXT: movq %rdi, %rax
+; X64-NEXT: shrq $47, %rax
+; X64-NEXT: orq %rax, %rsi
; X64-NEXT: shlq $17, %rdi
; X64-NEXT: movq %rdi, %rax
; X64-NEXT: orq %rsi, %rax
@@ -283,27 +297,41 @@ define i1 @opt_setcc_expanded_shl_correct_shifts(i64 %a, i64 %b) nounwind {
define i1 @opt_setcc_expanded_shl_wrong_shifts(i64 %a, i64 %b) nounwind {
; X86-LABEL: opt_setcc_expanded_shl_wrong_shifts:
; X86: # %bb.0:
+; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT: shldl $17, %edx, %esi
-; X86-NEXT: shldl $17, %ecx, %edx
-; X86-NEXT: shldl $18, %eax, %ecx
-; X86-NEXT: shll $18, %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT: shll $17, %edi
+; X86-NEXT: movl %esi, %edx
+; X86-NEXT: shrl $15, %edx
+; X86-NEXT: orl %edi, %edx
+; X86-NEXT: shll $17, %esi
+; X86-NEXT: movl %ecx, %edi
+; X86-NEXT: shrl $15, %edi
+; X86-NEXT: orl %esi, %edi
+; X86-NEXT: movl %eax, %esi
+; X86-NEXT: shll $18, %esi
+; X86-NEXT: orl %edi, %esi
+; X86-NEXT: shll $18, %ecx
+; X86-NEXT: shrl $14, %eax
+; X86-NEXT: orl %ecx, %eax
; X86-NEXT: orl %edx, %eax
-; X86-NEXT: orl %esi, %ecx
-; X86-NEXT: orl %eax, %ecx
+; X86-NEXT: orl %esi, %eax
; X86-NEXT: sete %al
; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
; X86-NEXT: retl
;
; X64-LABEL: opt_setcc_expanded_shl_wrong_shifts:
; X64: # %bb.0:
-; X64-NEXT: shldq $17, %rsi, %rdi
+; X64-NEXT: shlq $17, %rdi
+; X64-NEXT: movq %rsi, %rax
+; X64-NEXT: shrq $47, %rax
+; X64-NEXT: orq %rdi, %rax
; X64-NEXT: shlq $18, %rsi
-; X64-NEXT: orq %rdi, %rsi
+; X64-NEXT: orq %rax, %rsi
; X64-NEXT: sete %al
; X64-NEXT: retq
%shl.a = shl i64 %a, 17
diff --git a/llvm/test/CodeGen/X86/imul.ll b/llvm/test/CodeGen/X86/imul.ll
index 9131688c4efcc..873f30e7a1394 100644
--- a/llvm/test/CodeGen/X86/imul.ll
+++ b/llvm/test/CodeGen/X86/imul.ll
@@ -29,10 +29,12 @@ define i64 @mul4_64(i64 %A) {
;
; X86-LABEL: mul4_64:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: shldl $2, %eax, %edx
-; X86-NEXT: shll $2, %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: leal (,%edx,4), %eax
+; X86-NEXT: shll $2, %ecx
+; X86-NEXT: shrl $30, %edx
+; X86-NEXT: orl %ecx, %edx
; X86-NEXT: retl
%mul = mul i64 %A, 4
ret i64 %mul
@@ -63,10 +65,13 @@ define i64 @mul4096_64(i64 %A) {
;
; X86-LABEL: mul4096_64:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: shldl $12, %eax, %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl %edx, %eax
; X86-NEXT: shll $12, %eax
+; X86-NEXT: shll $12, %ecx
+; X86-NEXT: shrl $20, %edx
+; X86-NEXT: orl %ecx, %edx
; X86-NEXT: retl
%mul = mul i64 %A, 4096
ret i64 %mul
@@ -100,13 +105,21 @@ define i64 @mulmin4096_64(i64 %A) {
;
; X86-LABEL: mulmin4096_64:
; X86: # %bb.0:
+; X86-NEXT: pushl %esi
+; X86-NEXT: .cfi_def_cfa_offset 8
+; X86-NEXT: .cfi_offset %esi, -8
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: shldl $12, %eax, %ecx
+; X86-NEXT: shll $12, %ecx
+; X86-NEXT: movl %eax, %esi
+; X86-NEXT: shrl $20, %esi
+; X86-NEXT: orl %ecx, %esi
; X86-NEXT: shll $12, %eax
; X86-NEXT: xorl %edx, %edx
; X86-NEXT: negl %eax
-; X86-NEXT: sbbl %ecx, %edx
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: popl %esi
+; X86-NEXT: .cfi_def_cfa_offset 4
; X86-NEXT: retl
%mul = mul i64 %A, -4096
ret i64 %mul
diff --git a/llvm/test/CodeGen/X86/isel-shift.ll b/llvm/test/CodeGen/X86/isel-shift.ll
index 476dcf04dbaa2..7afc5ab1236cf 100644
--- a/llvm/test/CodeGen/X86/isel-shift.ll
+++ b/llvm/test/CodeGen/X86/isel-shift.ll
@@ -693,17 +693,22 @@ define i32 @shl_imm1_i32(i32 %a) {
define i64 @shl_imm1_i64(i64 %a) {
; SDAG-X86-LABEL: shl_imm1_i64:
; SDAG-X86: ## %bb.0:
-; SDAG-X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; SDAG-X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; SDAG-X86-NEXT: shldl $1, %eax, %edx
-; SDAG-X86-NEXT: addl %eax, %eax
+; SDAG-X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; SDAG-X86-NEXT: leal (%edx,%edx), %eax
+; SDAG-X86-NEXT: shll %ecx
+; SDAG-X86-NEXT: shrl $31, %edx
+; SDAG-X86-NEXT: orl %ecx, %edx
; SDAG-X86-NEXT: retl
;
; FASTISEL-X86-LABEL: shl_imm1_i64:
; FASTISEL-X86: ## %bb.0:
-; FASTISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; FASTISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; FASTISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; FASTISEL-X86-NEXT: shldl $1, %eax, %edx
+; FASTISEL-X86-NEXT: shll %ecx
+; FASTISEL-X86-NEXT: movl %eax, %edx
+; FASTISEL-X86-NEXT: shrl $31, %edx
+; FASTISEL-X86-NEXT: orl %ecx, %edx
; FASTISEL-X86-NEXT: addl %eax, %eax
; FASTISEL-X86-NEXT: retl
;
@@ -993,17 +998,23 @@ define i32 @shl_imm4_i32(i32 %a) {
define i64 @shl_imm4_i64(i64 %a) {
; SDAG-X86-LABEL: shl_imm4_i64:
; SDAG-X86: ## %bb.0:
-; SDAG-X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; SDAG-X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; SDAG-X86-NEXT: shldl $4, %eax, %edx
+; SDAG-X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; SDAG-X86-NEXT: movl %edx, %eax
; SDAG-X86-NEXT: shll $4, %eax
+; SDAG-X86-NEXT: shll $4, %ecx
+; SDAG-X86-NEXT: shrl $28, %edx
+; SDAG-X86-NEXT: orl %ecx, %edx
; SDAG-X86-NEXT: retl
;
; FASTISEL-X86-LABEL: shl_imm4_i64:
; FASTISEL-X86: ## %bb.0:
-; FASTISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; FASTISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; FASTISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; FASTISEL-X86-NEXT: shldl $4, %eax, %edx
+; FASTISEL-X86-NEXT: shll $4, %ecx
+; FASTISEL-X86-NEXT: movl %eax, %edx
+; FASTISEL-X86-NEXT: shrl $28, %edx
+; FASTISEL-X86-NEXT: orl %ecx, %edx
; FASTISEL-X86-NEXT: shll $4, %eax
; FASTISEL-X86-NEXT: retl
;
diff --git a/llvm/test/CodeGen/X86/known-bits.ll b/llvm/test/CodeGen/X86/known-bits.ll
index 58a0595e4322a..c80520bb32556 100644
--- a/llvm/test/CodeGen/X86/known-bits.ll
+++ b/llvm/test/CodeGen/X86/known-bits.ll
@@ -91,18 +91,20 @@ define i128 @knownbits_mask_addc_shl(i64 %a0, i64 %a1, i64 %a2) nounwind {
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl $-1024, %esi # imm = 0xFC00
; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
; X86-NEXT: andl %esi, %edi
; X86-NEXT: andl {{[0-9]+}}(%esp), %esi
; X86-NEXT: addl %edi, %esi
-; X86-NEXT: adcl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: adcl $0, %ecx
-; X86-NEXT: shldl $22, %edx, %ecx
-; X86-NEXT: shldl $22, %esi, %edx
-; X86-NEXT: movl %edx, 8(%eax)
+; X86-NEXT: adcl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: adcl $0, %edx
+; X86-NEXT: shll $22, %edx
+; X86-NEXT: shrdl $10, %ecx, %esi
+; X86-NEXT: shrl $10, %ecx
+; X86-NEXT: orl %edx, %ecx
+; X86-NEXT: movl %esi, 8(%eax)
; X86-NEXT: movl %ecx, 12(%eax)
; X86-NEXT: movl $0, 4(%eax)
; X86-NEXT: movl $0, (%eax)
@@ -116,7 +118,9 @@ define i128 @knownbits_mask_addc_shl(i64 %a0, i64 %a1, i64 %a2) nounwind {
; X64-NEXT: andq $-1024, %rsi # imm = 0xFC00
; X64-NEXT: addq %rdi, %rsi
; X64-NEXT: adcl $0, %edx
-; X64-NEXT: shldq $54, %rsi, %rdx
+; X64-NEXT: shlq $54, %rdx
+; X64-NEXT: shrq $10, %rsi
+; X64-NEXT: orq %rsi, %rdx
; X64-NEXT: xorl %eax, %eax
; X64-NEXT: retq
%1 = and i64 %a0, -1024
diff --git a/llvm/test/CodeGen/X86/legalize-shl-vec.ll b/llvm/test/CodeGen/X86/legalize-shl-vec.ll
index 4cfb050958abb..645a400bfc496 100644
--- a/llvm/test/CodeGen/X86/legalize-shl-vec.ll
+++ b/llvm/test/CodeGen/X86/legalize-shl-vec.ll
@@ -5,30 +5,60 @@
define <2 x i256> @test_shl(<2 x i256> %In) nounwind {
; X86-LABEL: test_shl:
; X86: # %bb.0:
+; X86-NEXT: pushl %ebp
+; X86-NEXT: pushl %ebx
+; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %esi
+; X86-NEXT: subl $12, %esp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT: leal (,%esi,4), %eax
+; X86-NEXT: movl %edx, %ecx
+; X86-NEXT: shrl $30, %ecx
+; X86-NEXT: orl %eax, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: leal (,%edi,4), %eax
+; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: shrl $30, %ecx
+; X86-NEXT: orl %eax, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %eax, %ebx
+; X86-NEXT: shrl $30, %ebx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT: leal (,%ebp,4), %ecx
+; X86-NEXT: orl %ecx, %ebx
+; X86-NEXT: shll $2, %edx
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: leal (,%ecx,4), %edi
+; X86-NEXT: movl %edi, (%esp) # 4-byte Spill
+; X86-NEXT: shrl $30, %ecx
+; X86-NEXT: orl %edx, %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: shldl $2, %ecx, %edx
-; X86-NEXT: movl %edx, 60(%eax)
+; X86-NEXT: shll $2, %edx
+; X86-NEXT: shrl $30, %esi
+; X86-NEXT: orl %edx, %esi
+; X86-NEXT: shll $2, %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: shldl $2, %edx, %ecx
-; X86-NEXT: movl %ecx, 56(%eax)
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: shldl $2, %ecx, %edx
+; X86-NEXT: shrl $30, %edx
+; X86-NEXT: orl %eax, %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: shll $2, %eax
+; X86-NEXT: shrl $30, %ebp
+; X86-NEXT: orl %eax, %ebp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %ebp, 60(%eax)
+; X86-NEXT: movl %ebx, 56(%eax)
; X86-NEXT: movl %edx, 52(%eax)
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: shldl $2, %edx, %ecx
-; X86-NEXT: movl %ecx, 48(%eax)
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: shldl $2, %ecx, %edx
-; X86-NEXT: movl %edx, 44(%eax)
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: shldl $2, %edx, %ecx
-; X86-NEXT: movl %ecx, 40(%eax)
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: shldl $2, %ecx, %edx
-; X86-NEXT: movl %edx, 36(%eax)
-; X86-NEXT: shll $2, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: movl %edx, 48(%eax)
+; X86-NEXT: movl %esi, 44(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: movl %edx, 40(%eax)
+; X86-NEXT: movl %ecx, 36(%eax)
+; X86-NEXT: movl (%esp), %ecx # 4-byte Reload
; X86-NEXT: movl %ecx, 32(%eax)
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: shll $31, %ecx
@@ -40,6 +70,11 @@ define <2 x i256> @test_shl(<2 x i256> %In) nounwind {
; X86-NEXT: movl $0, 8(%eax)
; X86-NEXT: movl $0, 4(%eax)
; X86-NEXT: movl $0, (%eax)
+; X86-NEXT: addl $12, %esp
+; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
+; X86-NEXT: popl %ebx
+; X86-NEXT: popl %ebp
; X86-NEXT: retl $4
;
; X64-LABEL: test_shl:
@@ -48,14 +83,21 @@ define <2 x i256> @test_shl(<2 x i256> %In) nounwind {
; X64-NEXT: movq {{[0-9]+}}(%rsp), %rcx
; X64-NEXT: movq {{[0-9]+}}(%rsp), %rdx
; X64-NEXT: movq {{[0-9]+}}(%rsp), %rdi
-; X64-NEXT: shldq $2, %rdx, %rcx
-; X64-NEXT: shldq $2, %rdi, %rdx
-; X64-NEXT: shldq $2, %r9, %rdi
+; X64-NEXT: leaq (,%rdi,4), %r8
+; X64-NEXT: movq %r9, %r10
+; X64-NEXT: shrq $62, %r10
+; X64-NEXT: orq %r8, %r10
+; X64-NEXT: leaq (,%rdx,4), %r8
+; X64-NEXT: shrq $62, %rdi
+; X64-NEXT: orq %r8, %rdi
+; X64-NEXT: shlq $2, %rcx
+; X64-NEXT: shrq $62, %rdx
+; X64-NEXT: orq %rcx, %rdx
; X64-NEXT: shlq $63, %rsi
; X64-NEXT: shlq $2, %r9
-; X64-NEXT: movq %rcx, 56(%rax)
-; X64-NEXT: movq %rdx, 48(%rax)
-; X64-NEXT: movq %rdi, 40(%rax)
+; X64-NEXT: movq %rdx, 56(%rax)
+; X64-NEXT: movq %rdi, 48(%rax)
+; X64-NEXT: movq %r10, 40(%rax)
; X64-NEXT: movq %r9, 32(%rax)
; X64-NEXT: movq %rsi, 24(%rax)
; X64-NEXT: xorps %xmm0, %xmm0
@@ -75,37 +117,43 @@ define <2 x i256> @test_srl(<2 x i256> %In) nounwind {
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
; X86-NEXT: subl $8, %esp
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT: movl %edi, %esi
-; X86-NEXT: shldl $28, %eax, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shrdl $4, %eax, %ecx
-; X86-NEXT: movl %ecx, (%esp) # 4-byte Spill
-; X86-NEXT: movl %edx, %ebp
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: shldl $28, %eax, %ebp
-; X86-NEXT: shrdl $4, %eax, %edi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl %ebx, %esi
-; X86-NEXT: shldl $28, %eax, %esi
-; X86-NEXT: shrdl $4, %eax, %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT: movl %ebx, %eax
+; X86-NEXT: shll $28, %eax
+; X86-NEXT: shrdl $4, %esi, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shrl $4, %esi
+; X86-NEXT: orl %eax, %esi
+; X86-NEXT: movl %edx, %eax
+; X86-NEXT: shll $28, %eax
+; X86-NEXT: shrdl $4, %ebp, %ebx
+; X86-NEXT: shrl $4, %ebp
+; X86-NEXT: orl %eax, %ebp
+; X86-NEXT: movl %edi, %eax
+; X86-NEXT: shll $28, %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: shrdl $4, %ecx, %ebx
+; X86-NEXT: shrdl $4, %ecx, %edx
+; X86-NEXT: movl %edx, (%esp) # 4-byte Spill
; X86-NEXT: shrl $4, %ecx
+; X86-NEXT: orl %eax, %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: shrdl $4, %edx, %edi
+; X86-NEXT: shrl $4, %edx
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl %ecx, 60(%eax)
-; X86-NEXT: movl %ebx, 56(%eax)
-; X86-NEXT: movl %esi, 52(%eax)
-; X86-NEXT: movl %edx, 48(%eax)
+; X86-NEXT: movl %edx, 60(%eax)
+; X86-NEXT: movl %edi, 56(%eax)
+; X86-NEXT: movl %ecx, 52(%eax)
+; X86-NEXT: movl (%esp), %ecx # 4-byte Reload
+; X86-NEXT: movl %ecx, 48(%eax)
; X86-NEXT: movl %ebp, 44(%eax)
-; X86-NEXT: movl %edi, 40(%eax)
+; X86-NEXT: movl %ebx, 40(%eax)
+; X86-NEXT: movl %esi, 36(%eax)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl %ecx, 36(%eax)
-; X86-NEXT: movl (%esp), %ecx # 4-byte Reload
; X86-NEXT: movl %ecx, 32(%eax)
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: shrl $31, %ecx
@@ -131,18 +179,21 @@ define <2 x i256> @test_srl(<2 x i256> %In) nounwind {
; X64-NEXT: movq {{[0-9]+}}(%rsp), %rdx
; X64-NEXT: movq {{[0-9]+}}(%rsp), %rsi
; X64-NEXT: shrdq $4, %rsi, %r9
-; X64-NEXT: shrdq $4, %rdx, %rsi
+; X64-NEXT: movq %rdx, %rdi
+; X64-NEXT: shlq $60, %rdi
+; X64-NEXT: shrq $4, %rsi
+; X64-NEXT: orq %rdi, %rsi
; X64-NEXT: shrdq $4, %rcx, %rdx
; X64-NEXT: shrq $63, %r8
; X64-NEXT: shrq $4, %rcx
-; X64-NEXT: movq %rcx, 56(%rdi)
-; X64-NEXT: movq %rdx, 48(%rdi)
-; X64-NEXT: movq %rsi, 40(%rdi)
-; X64-NEXT: movq %r9, 32(%rdi)
-; X64-NEXT: movq %r8, (%rdi)
+; X64-NEXT: movq %rcx, 56(%rax)
+; X64-NEXT: movq %rdx, 48(%rax)
+; X64-NEXT: movq %rsi, 40(%rax)
+; X64-NEXT: movq %r9, 32(%rax)
+; X64-NEXT: movq %r8, (%rax)
; X64-NEXT: xorps %xmm0, %xmm0
-; X64-NEXT: movaps %xmm0, 16(%rdi)
-; X64-NEXT: movq $0, 8(%rdi)
+; X64-NEXT: movaps %xmm0, 16(%rax)
+; X64-NEXT: movq $0, 8(%rax)
; X64-NEXT: retq
%Amt = insertelement <2 x i256> <i256 3, i256 4>, i256 255, i32 0
%Out = lshr <2 x i256> %In, %Amt
@@ -157,37 +208,43 @@ define <2 x i256> @test_sra(<2 x i256> %In) nounwind {
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
; X86-NEXT: subl $8, %esp
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT: movl %edi, %esi
-; X86-NEXT: shldl $26, %eax, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shrdl $6, %eax, %ecx
-; X86-NEXT: movl %ecx, (%esp) # 4-byte Spill
-; X86-NEXT: movl %edx, %ebp
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: shldl $26, %eax, %ebp
-; X86-NEXT: shrdl $6, %eax, %edi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl %ebx, %esi
-; X86-NEXT: shldl $26, %eax, %esi
-; X86-NEXT: shrdl $6, %eax, %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT: movl %ebx, %eax
+; X86-NEXT: shll $26, %eax
+; X86-NEXT: shrdl $6, %esi, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shrl $6, %esi
+; X86-NEXT: orl %eax, %esi
+; X86-NEXT: movl %edx, %eax
+; X86-NEXT: shll $26, %eax
+; X86-NEXT: shrdl $6, %ebp, %ebx
+; X86-NEXT: shrl $6, %ebp
+; X86-NEXT: orl %eax, %ebp
+; X86-NEXT: movl %edi, %eax
+; X86-NEXT: shll $26, %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: shrdl $6, %ecx, %ebx
-; X86-NEXT: sarl $6, %ecx
+; X86-NEXT: shrdl $6, %ecx, %edx
+; X86-NEXT: movl %edx, (%esp) # 4-byte Spill
+; X86-NEXT: shrl $6, %ecx
+; X86-NEXT: orl %eax, %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: shrdl $6, %edx, %edi
+; X86-NEXT: sarl $6, %edx
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl %ecx, 60(%eax)
-; X86-NEXT: movl %ebx, 56(%eax)
-; X86-NEXT: movl %esi, 52(%eax)
-; X86-NEXT: movl %edx, 48(%eax)
+; X86-NEXT: movl %edx, 60(%eax)
+; X86-NEXT: movl %edi, 56(%eax)
+; X86-NEXT: movl %ecx, 52(%eax)
+; X86-NEXT: movl (%esp), %ecx # 4-byte Reload
+; X86-NEXT: movl %ecx, 48(%eax)
; X86-NEXT: movl %ebp, 44(%eax)
-; X86-NEXT: movl %edi, 40(%eax)
+; X86-NEXT: movl %ebx, 40(%eax)
+; X86-NEXT: movl %esi, 36(%eax)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl %ecx, 36(%eax)
-; X86-NEXT: movl (%esp), %ecx # 4-byte Reload
; X86-NEXT: movl %ecx, 32(%eax)
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: sarl $31, %ecx
@@ -213,18 +270,21 @@ define <2 x i256> @test_sra(<2 x i256> %In) nounwind {
; X64-NEXT: movq {{[0-9]+}}(%rsp), %rdx
; X64-NEXT: movq {{[0-9]+}}(%rsp), %rsi
; X64-NEXT: shrdq $6, %rsi, %r9
-; X64-NEXT: shrdq $6, %rdx, %rsi
+; X64-NEXT: movq %rdx, %rdi
+; X64-NEXT: shlq $58, %rdi
+; X64-NEXT: shrq $6, %rsi
+; X64-NEXT: orq %rdi, %rsi
; X64-NEXT: shrdq $6, %rcx, %rdx
; X64-NEXT: sarq $63, %r8
; X64-NEXT: sarq $6, %rcx
-; X64-NEXT: movq %rcx, 56(%rdi)
-; X64-NEXT: movq %rdx, 48(%rdi)
-; X64-NEXT: movq %rsi, 40(%rdi)
-; X64-NEXT: movq %r9, 32(%rdi)
-; X64-NEXT: movq %r8, 24(%rdi)
-; X64-NEXT: movq %r8, 16(%rdi)
-; X64-NEXT: movq %r8, 8(%rdi)
-; X64-NEXT: movq %r8, (%rdi)
+; X64-NEXT: movq %rcx, 56(%rax)
+; X64-NEXT: movq %rdx, 48(%rax)
+; X64-NEXT: movq %rsi, 40(%rax)
+; X64-NEXT: movq %r9, 32(%rax)
+; X64-NEXT: movq %r8, 24(%rax)
+; X64-NEXT: movq %r8, 16(%rax)
+; X64-NEXT: movq %r8, 8(%rax)
+; X64-NEXT: movq %r8, (%rax)
; X64-NEXT: retq
%Amt = insertelement <2 x i256> <i256 5, i256 6>, i256 255, i32 0
%Out = ashr <2 x i256> %In, %Amt
diff --git a/llvm/test/CodeGen/X86/logic-shift.ll b/llvm/test/CodeGen/X86/logic-shift.ll
index 104151c76bc5d..97d6d12a4a903 100644
--- a/llvm/test/CodeGen/X86/logic-shift.ll
+++ b/llvm/test/CodeGen/X86/logic-shift.ll
@@ -858,9 +858,11 @@ define i64 @mix_logic_shl(i64 %x0, i64 %x1, i64 %y, i64 %z) {
define i32 @or_fshl_commute0(i32 %x, i32 %y) {
; CHECK-LABEL: or_fshl_commute0:
; CHECK: # %bb.0:
-; CHECK-NEXT: movl %esi, %eax
-; CHECK-NEXT: orl %edi, %eax
-; CHECK-NEXT: shldl $5, %edi, %eax
+; CHECK-NEXT: movl %edi, %eax
+; CHECK-NEXT: orl %edi, %esi
+; CHECK-NEXT: shll $5, %esi
+; CHECK-NEXT: shrl $27, %eax
+; CHECK-NEXT: orl %esi, %eax
; CHECK-NEXT: retq
%or1 = or i32 %x, %y
%sh1 = shl i32 %or1, 5
@@ -874,7 +876,9 @@ define i64 @or_fshl_commute1(i64 %x, i64 %y) {
; CHECK: # %bb.0:
; CHECK-NEXT: movl %edi, %eax
; CHECK-NEXT: orl %esi, %eax
-; CHECK-NEXT: shldq $35, %rdi, %rax
+; CHECK-NEXT: shlq $35, %rax
+; CHECK-NEXT: shrq $29, %rdi
+; CHECK-NEXT: orq %rdi, %rax
; CHECK-NEXT: retq
%or1 = or i64 %y, %x
%sh1 = shl i64 %or1, 35
@@ -938,7 +942,9 @@ define i64 @or_fshr_commute0(i64 %x, i64 %y) {
; CHECK: # %bb.0:
; CHECK-NEXT: movq %rsi, %rax
; CHECK-NEXT: orq %rdi, %rax
-; CHECK-NEXT: shrdq $24, %rdi, %rax
+; CHECK-NEXT: shlq $40, %rdi
+; CHECK-NEXT: shrq $24, %rax
+; CHECK-NEXT: orq %rdi, %rax
; CHECK-NEXT: retq
%or1 = or i64 %x, %y
%sh1 = shl i64 %x, 40
@@ -950,9 +956,11 @@ define i64 @or_fshr_commute0(i64 %x, i64 %y) {
define i32 @or_fshr_commute1(i32 %x, i32 %y) {
; CHECK-LABEL: or_fshr_commute1:
; CHECK: # %bb.0:
-; CHECK-NEXT: movl %esi, %eax
-; CHECK-NEXT: orl %edi, %eax
-; CHECK-NEXT: shrdl $29, %edi, %eax
+; CHECK-NEXT: # kill: def $edi killed $edi def $rdi
+; CHECK-NEXT: orl %edi, %esi
+; CHECK-NEXT: leal (,%rdi,8), %eax
+; CHECK-NEXT: shrl $29, %esi
+; CHECK-NEXT: orl %esi, %eax
; CHECK-NEXT: retq
%or1 = or i32 %y, %x
%sh1 = shl i32 %x, 3
diff --git a/llvm/test/CodeGen/X86/mul-constant-i64.ll b/llvm/test/CodeGen/X86/mul-constant-i64.ll
index 40d591f8d1be8..77d1252fa076e 100644
--- a/llvm/test/CodeGen/X86/mul-constant-i64.ll
+++ b/llvm/test/CodeGen/X86/mul-constant-i64.ll
@@ -32,18 +32,22 @@ define i64 @test_mul_by_1(i64 %x) nounwind {
define i64 @test_mul_by_2(i64 %x) {
; X86-LABEL: test_mul_by_2:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: shldl $1, %eax, %edx
-; X86-NEXT: addl %eax, %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: leal (%edx,%edx), %eax
+; X86-NEXT: shll %ecx
+; X86-NEXT: shrl $31, %edx
+; X86-NEXT: orl %ecx, %edx
; X86-NEXT: retl
;
; X86-NOOPT-LABEL: test_mul_by_2:
; X86-NOOPT: # %bb.0:
-; X86-NOOPT-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NOOPT-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NOOPT-NEXT: shldl $1, %eax, %edx
-; X86-NOOPT-NEXT: addl %eax, %eax
+; X86-NOOPT-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NOOPT-NEXT: leal (%edx,%edx), %eax
+; X86-NOOPT-NEXT: shll %ecx
+; X86-NOOPT-NEXT: shrl $31, %edx
+; X86-NOOPT-NEXT: orl %ecx, %edx
; X86-NOOPT-NEXT: retl
;
; X64-LABEL: test_mul_by_2:
@@ -83,18 +87,22 @@ define i64 @test_mul_by_3(i64 %x) {
define i64 @test_mul_by_4(i64 %x) {
; X86-LABEL: test_mul_by_4:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: shldl $2, %eax, %edx
-; X86-NEXT: shll $2, %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: leal (,%edx,4), %eax
+; X86-NEXT: shll $2, %ecx
+; X86-NEXT: shrl $30, %edx
+; X86-NEXT: orl %ecx, %edx
; X86-NEXT: retl
;
; X86-NOOPT-LABEL: test_mul_by_4:
; X86-NOOPT: # %bb.0:
-; X86-NOOPT-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NOOPT-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NOOPT-NEXT: shldl $2, %eax, %edx
-; X86-NOOPT-NEXT: shll $2, %eax
+; X86-NOOPT-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NOOPT-NEXT: leal (,%edx,4), %eax
+; X86-NOOPT-NEXT: shll $2, %ecx
+; X86-NOOPT-NEXT: shrl $30, %edx
+; X86-NOOPT-NEXT: orl %ecx, %edx
; X86-NOOPT-NEXT: retl
;
; X64-LABEL: test_mul_by_4:
@@ -198,18 +206,22 @@ define i64 @test_mul_by_7(i64 %x) {
define i64 @test_mul_by_8(i64 %x) {
; X86-LABEL: test_mul_by_8:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: shldl $3, %eax, %edx
-; X86-NEXT: shll $3, %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: leal (,%edx,8), %eax
+; X86-NEXT: shll $3, %ecx
+; X86-NEXT: shrl $29, %edx
+; X86-NEXT: orl %ecx, %edx
; X86-NEXT: retl
;
; X86-NOOPT-LABEL: test_mul_by_8:
; X86-NOOPT: # %bb.0:
-; X86-NOOPT-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NOOPT-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NOOPT-NEXT: shldl $3, %eax, %edx
-; X86-NOOPT-NEXT: shll $3, %eax
+; X86-NOOPT-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NOOPT-NEXT: leal (,%edx,8), %eax
+; X86-NOOPT-NEXT: shll $3, %ecx
+; X86-NOOPT-NEXT: shrl $29, %edx
+; X86-NOOPT-NEXT: orl %ecx, %edx
; X86-NOOPT-NEXT: retl
;
; X64-LABEL: test_mul_by_8:
@@ -470,18 +482,24 @@ define i64 @test_mul_by_15(i64 %x) {
define i64 @test_mul_by_16(i64 %x) {
; X86-LABEL: test_mul_by_16:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: shldl $4, %eax, %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl %edx, %eax
; X86-NEXT: shll $4, %eax
+; X86-NEXT: shll $4, %ecx
+; X86-NEXT: shrl $28, %edx
+; X86-NEXT: orl %ecx, %edx
; X86-NEXT: retl
;
; X86-NOOPT-LABEL: test_mul_by_16:
; X86-NOOPT: # %bb.0:
-; X86-NOOPT-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NOOPT-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NOOPT-NEXT: shldl $4, %eax, %edx
+; X86-NOOPT-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NOOPT-NEXT: movl %edx, %eax
; X86-NOOPT-NEXT: shll $4, %eax
+; X86-NOOPT-NEXT: shll $4, %ecx
+; X86-NOOPT-NEXT: shrl $28, %edx
+; X86-NOOPT-NEXT: orl %ecx, %edx
; X86-NOOPT-NEXT: retl
;
; X64-LABEL: test_mul_by_16:
@@ -1111,18 +1129,24 @@ define i64 @test_mul_by_31(i64 %x) {
define i64 @test_mul_by_32(i64 %x) {
; X86-LABEL: test_mul_by_32:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: shldl $5, %eax, %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl %edx, %eax
; X86-NEXT: shll $5, %eax
+; X86-NEXT: shll $5, %ecx
+; X86-NEXT: shrl $27, %edx
+; X86-NEXT: orl %ecx, %edx
; X86-NEXT: retl
;
; X86-NOOPT-LABEL: test_mul_by_32:
; X86-NOOPT: # %bb.0:
-; X86-NOOPT-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NOOPT-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NOOPT-NEXT: shldl $5, %eax, %edx
+; X86-NOOPT-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NOOPT-NEXT: movl %edx, %eax
; X86-NOOPT-NEXT: shll $5, %eax
+; X86-NOOPT-NEXT: shll $5, %ecx
+; X86-NOOPT-NEXT: shrl $27, %edx
+; X86-NOOPT-NEXT: orl %ecx, %edx
; X86-NOOPT-NEXT: retl
;
; X64-LABEL: test_mul_by_32:
diff --git a/llvm/test/CodeGen/X86/or-lea.ll b/llvm/test/CodeGen/X86/or-lea.ll
index 616ab99437892..fb77b25fd2302 100644
--- a/llvm/test/CodeGen/X86/or-lea.ll
+++ b/llvm/test/CodeGen/X86/or-lea.ll
@@ -175,11 +175,13 @@ define i64 @or_shift1_and1_64(i64 %x, i64 %y) {
; X86-LABEL: or_shift1_and1_64:
; X86: # %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: shldl $1, %ecx, %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: shll %ecx
; X86-NEXT: andl $1, %eax
-; X86-NEXT: leal (%eax,%ecx,2), %eax
+; X86-NEXT: leal (%eax,%edx,2), %eax
+; X86-NEXT: shrl $31, %edx
+; X86-NEXT: orl %ecx, %edx
; X86-NEXT: retl
;
; X64-LABEL: or_shift1_and1_64:
diff --git a/llvm/test/CodeGen/X86/pr38539.ll b/llvm/test/CodeGen/X86/pr38539.ll
index eecd15dd2afe9..42cb41424daa5 100644
--- a/llvm/test/CodeGen/X86/pr38539.ll
+++ b/llvm/test/CodeGen/X86/pr38539.ll
@@ -22,7 +22,7 @@ define void @f() nounwind {
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
; X86-NEXT: andl $-16, %esp
-; X86-NEXT: subl $176, %esp
+; X86-NEXT: subl $160, %esp
; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx
@@ -43,8 +43,11 @@ define void @f() nounwind {
; X86-NEXT: subl %ecx, %esi
; X86-NEXT: sbbl %eax, %ebx
; X86-NEXT: sbbl %eax, %edi
-; X86-NEXT: movl %edi, %ecx
-; X86-NEXT: shldl $30, %ebx, %ecx
+; X86-NEXT: movl %edi, %eax
+; X86-NEXT: shll $30, %eax
+; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: shrl $2, %ecx
+; X86-NEXT: orl %eax, %ecx
; X86-NEXT: movl %esi, %edx
; X86-NEXT: shrdl $2, %ebx, %edx
; X86-NEXT: testl %ecx, %ecx
@@ -100,16 +103,17 @@ define void @f() nounwind {
; X86-NEXT: testb %cl, %cl
; X86-NEXT: jne .LBB0_15
; X86-NEXT: # %bb.11: # %select.end
+; X86-NEXT: movl %ebx, %edx
; X86-NEXT: movl %eax, %ecx
; X86-NEXT: xorl $65, %ecx
; X86-NEXT: orl %esi, %ecx
; X86-NEXT: orl %ebx, %ecx
; X86-NEXT: je .LBB0_15
; X86-NEXT: # %bb.12: # %udiv-bb1
-; X86-NEXT: movl %eax, %ecx
-; X86-NEXT: addl $1, %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: adcl $0, %ebx
+; X86-NEXT: movl %eax, %ebx
+; X86-NEXT: addl $1, %ebx
+; X86-NEXT: adcl $0, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: adcl $0, %esi
; X86-NEXT: andl $3, %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
@@ -119,7 +123,7 @@ define void @f() nounwind {
; X86-NEXT: shrb $3, %al
; X86-NEXT: andb $12, %al
; X86-NEXT: negb %al
-; X86-NEXT: movsbl %al, %edx
+; X86-NEXT: movsbl %al, %eax
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
@@ -128,24 +132,23 @@ define void @f() nounwind {
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 128(%esp,%edx), %eax
-; X86-NEXT: movl 132(%esp,%edx), %edi
-; X86-NEXT: movl 136(%esp,%edx), %edx
-; X86-NEXT: shldl %cl, %edi, %edx
+; X86-NEXT: movl 112(%esp,%eax), %edx
+; X86-NEXT: movl 116(%esp,%eax), %edi
+; X86-NEXT: movl 120(%esp,%eax), %eax
+; X86-NEXT: shldl %cl, %edi, %eax
+; X86-NEXT: shldl %cl, %edx, %edi
+; X86-NEXT: shll %cl, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shldl %cl, %eax, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shll %cl, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: orl %esi, %eax
; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %ebx, %eax
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: orl %esi, %ecx
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NEXT: je .LBB0_15
; X86-NEXT: # %bb.13: # %udiv-preheader
-; X86-NEXT: andl $3, %esi
-; X86-NEXT: andl $3, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: andl $3, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: andl $3, %eax
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
@@ -155,59 +158,75 @@ define void @f() nounwind {
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: shrb $3, %al
-; X86-NEXT: andb $12, %al
-; X86-NEXT: movzbl %al, %eax
-; X86-NEXT: movl 88(%esp,%eax), %edi
-; X86-NEXT: movl 80(%esp,%eax), %ebx
-; X86-NEXT: movl 84(%esp,%eax), %eax
-; X86-NEXT: movl %eax, %edx
-; X86-NEXT: shrdl %cl, %edi, %edx
+; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: shrb $3, %dl
+; X86-NEXT: andb $12, %dl
+; X86-NEXT: movzbl %dl, %esi
+; X86-NEXT: movl 72(%esp,%esi), %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 64(%esp,%esi), %edx
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 68(%esp,%esi), %edi
+; X86-NEXT: movl %edi, %esi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: shrdl %cl, %ebx, %esi
; X86-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NEXT: shrdl %cl, %eax, %ebx
+; X86-NEXT: shrdl %cl, %edi, %edx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: addl $-1, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: adcl $-1, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: adcl $3, %esi
-; X86-NEXT: andl $3, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: addl $-1, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: adcl $-1, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: adcl $3, %ebx
+; X86-NEXT: andl $3, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: xorl %ecx, %ecx
; X86-NEXT: .p2align 4
; X86-NEXT: .LBB0_14: # %udiv-do-while
; X86-NEXT: # =>This Inner Loop Header: Depth=1
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: shldl $1, %edx, %ecx
-; X86-NEXT: shldl $1, %ebx, %edx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: movl %edi, %esi
-; X86-NEXT: andl $2, %esi
-; X86-NEXT: shrl %esi
-; X86-NEXT: leal (%esi,%ebx,2), %ebx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: shldl $1, %esi, %edi
-; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X86-NEXT: shll %ecx
+; X86-NEXT: movl %esi, %ebx
+; X86-NEXT: shrl $31, %ebx
+; X86-NEXT: orl %ecx, %ebx
+; X86-NEXT: shll %esi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: shrl $31, %edx
+; X86-NEXT: orl %esi, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %eax, %edx
+; X86-NEXT: andl $2, %edx
+; X86-NEXT: shrl %edx
+; X86-NEXT: leal (%edx,%ecx,2), %ecx
+; X86-NEXT: shll %eax
+; X86-NEXT: movl %edi, %edx
+; X86-NEXT: shrl $31, %edx
+; X86-NEXT: orl %eax, %edx
+; X86-NEXT: shll %edi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shldl $1, %eax, %esi
-; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %eax, %esi
+; X86-NEXT: shrl $31, %esi
+; X86-NEXT: orl %edi, %esi
+; X86-NEXT: movl %esi, %edi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: orl %esi, %edx
+; X86-NEXT: orl %esi, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: addl %eax, %eax
; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: andl $3, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: cmpl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; X86-NEXT: andl $3, %edx
+; X86-NEXT: cmpl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: sbbl %edx, %esi
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: sbbl %ecx, %esi
+; X86-NEXT: sbbl %ebx, %esi
; X86-NEXT: shll $30, %esi
; X86-NEXT: movl %esi, %edi
; X86-NEXT: sarl $31, %edi
@@ -220,25 +239,28 @@ define void @f() nounwind {
; X86-NEXT: movl %edi, %eax
; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X86-NEXT: subl %esi, %ebx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: sbbl %edi, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: sbbl %eax, %ecx
-; X86-NEXT: andl $3, %ecx
+; X86-NEXT: subl %esi, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: sbbl %edi, %ecx
+; X86-NEXT: sbbl %eax, %ebx
+; X86-NEXT: andl $3, %ebx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: addl $-1, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: adcl $-1, %edi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: adcl $-1, %esi
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: adcl $3, %ebx
-; X86-NEXT: andl $3, %ebx
+; X86-NEXT: adcl $3, %esi
+; X86-NEXT: andl $3, %esi
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %ebx, %eax
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: orl %esi, %eax
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %edi, %eax
+; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: movl %edx, %eax
; X86-NEXT: jne .LBB0_14
; X86-NEXT: .LBB0_15: # %udiv-end
; X86-NEXT: cmpb $0, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Reload
diff --git a/llvm/test/CodeGen/X86/pr43820.ll b/llvm/test/CodeGen/X86/pr43820.ll
index bf553c02fea3f..4c17cfd080afb 100644
--- a/llvm/test/CodeGen/X86/pr43820.ll
+++ b/llvm/test/CodeGen/X86/pr43820.ll
@@ -10,17 +10,18 @@ define i1000 @square(i1000 %A) nounwind {
; CHECK-NEXT: pushq %r13
; CHECK-NEXT: pushq %r12
; CHECK-NEXT: pushq %rbx
-; CHECK-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT: movq %rsi, %rbp
+; CHECK-NEXT: movq %rdi, %r11
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rsi
; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rdi
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r14
; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r15
; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rbx
; CHECK-NEXT: bswapq %rbx
; CHECK-NEXT: movq %rbx, %r10
; CHECK-NEXT: shrq $4, %r10
-; CHECK-NEXT: movabsq $1085102592571150095, %r11 # imm = 0xF0F0F0F0F0F0F0F
-; CHECK-NEXT: andq %r11, %r10
-; CHECK-NEXT: andq %r11, %rbx
+; CHECK-NEXT: movabsq $1085102592571150095, %r14 # imm = 0xF0F0F0F0F0F0F0F
+; CHECK-NEXT: andq %r14, %r10
+; CHECK-NEXT: andq %r14, %rbx
; CHECK-NEXT: shlq $4, %rbx
; CHECK-NEXT: orq %r10, %rbx
; CHECK-NEXT: movabsq $3689348814741910323, %r10 # imm = 0x3333333333333333
@@ -35,12 +36,11 @@ define i1000 @square(i1000 %A) nounwind {
; CHECK-NEXT: shrq %r12
; CHECK-NEXT: andq %rbx, %r12
; CHECK-NEXT: leaq (%r12,%r13,2), %rax
-; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-NEXT: bswapq %r15
; CHECK-NEXT: movq %r15, %r12
; CHECK-NEXT: shrq $4, %r12
-; CHECK-NEXT: andq %r11, %r12
-; CHECK-NEXT: andq %r11, %r15
+; CHECK-NEXT: andq %r14, %r12
+; CHECK-NEXT: andq %r14, %r15
; CHECK-NEXT: shlq $4, %r15
; CHECK-NEXT: orq %r12, %r15
; CHECK-NEXT: movq %r15, %r12
@@ -54,13 +54,14 @@ define i1000 @square(i1000 %A) nounwind {
; CHECK-NEXT: shrq %r15
; CHECK-NEXT: andq %r12, %r15
; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r12
-; CHECK-NEXT: leaq (%r15,%r13,2), %rax
-; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT: leaq (%r15,%r13,2), %r15
+; CHECK-NEXT: shrdq $24, %rax, %r15
+; CHECK-NEXT: movq %r15, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-NEXT: bswapq %r12
; CHECK-NEXT: movq %r12, %r15
; CHECK-NEXT: shrq $4, %r15
-; CHECK-NEXT: andq %r11, %r15
-; CHECK-NEXT: andq %r11, %r12
+; CHECK-NEXT: andq %r14, %r15
+; CHECK-NEXT: andq %r14, %r12
; CHECK-NEXT: shlq $4, %r12
; CHECK-NEXT: orq %r15, %r12
; CHECK-NEXT: movq %r12, %r15
@@ -72,145 +73,162 @@ define i1000 @square(i1000 %A) nounwind {
; CHECK-NEXT: andq %rbx, %r12
; CHECK-NEXT: shrq %r15
; CHECK-NEXT: andq %rbx, %r15
-; CHECK-NEXT: leaq (%r15,%r12,2), %rax
-; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: bswapq %r14
-; CHECK-NEXT: movq %r14, %r15
-; CHECK-NEXT: shrq $4, %r15
-; CHECK-NEXT: andq %r11, %r15
-; CHECK-NEXT: andq %r11, %r14
-; CHECK-NEXT: shlq $4, %r14
-; CHECK-NEXT: orq %r15, %r14
-; CHECK-NEXT: movq %r14, %r15
-; CHECK-NEXT: andq %r10, %r15
-; CHECK-NEXT: shrq $2, %r14
-; CHECK-NEXT: andq %r10, %r14
-; CHECK-NEXT: leaq (%r14,%r15,4), %r14
-; CHECK-NEXT: movq %r14, %r15
-; CHECK-NEXT: andq %rbx, %r15
-; CHECK-NEXT: shrq %r14
-; CHECK-NEXT: andq %rbx, %r14
-; CHECK-NEXT: leaq (%r14,%r15,2), %rax
+; CHECK-NEXT: leaq (%r15,%r12,2), %r12
+; CHECK-NEXT: movq %r12, %r15
+; CHECK-NEXT: shlq $40, %r15
+; CHECK-NEXT: shrq $24, %rax
+; CHECK-NEXT: orq %r15, %rax
; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-NEXT: bswapq %rdi
-; CHECK-NEXT: movq %rdi, %r14
-; CHECK-NEXT: shrq $4, %r14
-; CHECK-NEXT: andq %r11, %r14
-; CHECK-NEXT: andq %r11, %rdi
+; CHECK-NEXT: movq %rdi, %r15
+; CHECK-NEXT: shrq $4, %r15
+; CHECK-NEXT: andq %r14, %r15
+; CHECK-NEXT: andq %r14, %rdi
; CHECK-NEXT: shlq $4, %rdi
-; CHECK-NEXT: orq %r14, %rdi
-; CHECK-NEXT: movq %rdi, %r14
-; CHECK-NEXT: andq %r10, %r14
+; CHECK-NEXT: orq %r15, %rdi
+; CHECK-NEXT: movq %rdi, %r15
+; CHECK-NEXT: andq %r10, %r15
; CHECK-NEXT: shrq $2, %rdi
; CHECK-NEXT: andq %r10, %rdi
-; CHECK-NEXT: leaq (%rdi,%r14,4), %rdi
-; CHECK-NEXT: movq %rdi, %r14
-; CHECK-NEXT: andq %rbx, %r14
+; CHECK-NEXT: leaq (%rdi,%r15,4), %rdi
+; CHECK-NEXT: movq %rdi, %r15
+; CHECK-NEXT: andq %rbx, %r15
; CHECK-NEXT: shrq %rdi
; CHECK-NEXT: andq %rbx, %rdi
-; CHECK-NEXT: leaq (%rdi,%r14,2), %rax
-; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rdi
-; CHECK-NEXT: bswapq %rdi
-; CHECK-NEXT: movq %rdi, %r14
-; CHECK-NEXT: shrq $4, %r14
-; CHECK-NEXT: andq %r11, %r14
-; CHECK-NEXT: andq %r11, %rdi
-; CHECK-NEXT: shlq $4, %rdi
-; CHECK-NEXT: orq %r14, %rdi
-; CHECK-NEXT: movq %rdi, %r14
-; CHECK-NEXT: andq %r10, %r14
-; CHECK-NEXT: shrq $2, %rdi
+; CHECK-NEXT: leaq (%rdi,%r15,2), %rax
+; CHECK-NEXT: shrdq $24, %rax, %r12
+; CHECK-NEXT: movq %r12, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT: bswapq %rsi
+; CHECK-NEXT: movq %rsi, %rdi
+; CHECK-NEXT: shrq $4, %rdi
+; CHECK-NEXT: andq %r14, %rdi
+; CHECK-NEXT: andq %r14, %rsi
+; CHECK-NEXT: shlq $4, %rsi
+; CHECK-NEXT: orq %rdi, %rsi
+; CHECK-NEXT: movq %rsi, %rdi
; CHECK-NEXT: andq %r10, %rdi
-; CHECK-NEXT: leaq (%rdi,%r14,4), %rdi
-; CHECK-NEXT: movq %rdi, %r14
-; CHECK-NEXT: andq %rbx, %r14
-; CHECK-NEXT: shrq %rdi
+; CHECK-NEXT: shrq $2, %rsi
+; CHECK-NEXT: andq %r10, %rsi
+; CHECK-NEXT: leaq (%rsi,%rdi,4), %rsi
+; CHECK-NEXT: movq %rsi, %rdi
; CHECK-NEXT: andq %rbx, %rdi
-; CHECK-NEXT: leaq (%rdi,%r14,2), %rax
+; CHECK-NEXT: shrq %rsi
+; CHECK-NEXT: andq %rbx, %rsi
+; CHECK-NEXT: leaq (%rsi,%rdi,2), %rdi
+; CHECK-NEXT: movq %rdi, %rsi
+; CHECK-NEXT: movq %rdi, %r15
+; CHECK-NEXT: shlq $40, %rsi
+; CHECK-NEXT: shrq $24, %rax
+; CHECK-NEXT: orq %rsi, %rax
; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rdi
-; CHECK-NEXT: bswapq %rdi
-; CHECK-NEXT: movq %rdi, %r14
-; CHECK-NEXT: shrq $4, %r14
-; CHECK-NEXT: andq %r11, %r14
-; CHECK-NEXT: andq %r11, %rdi
-; CHECK-NEXT: shlq $4, %rdi
-; CHECK-NEXT: orq %r14, %rdi
-; CHECK-NEXT: movq %rdi, %r14
-; CHECK-NEXT: andq %r10, %r14
-; CHECK-NEXT: shrq $2, %rdi
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rsi
+; CHECK-NEXT: bswapq %rsi
+; CHECK-NEXT: movq %rsi, %rdi
+; CHECK-NEXT: shrq $4, %rdi
+; CHECK-NEXT: andq %r14, %rdi
+; CHECK-NEXT: andq %r14, %rsi
+; CHECK-NEXT: shlq $4, %rsi
+; CHECK-NEXT: orq %rdi, %rsi
+; CHECK-NEXT: movq %rsi, %rdi
; CHECK-NEXT: andq %r10, %rdi
-; CHECK-NEXT: leaq (%rdi,%r14,4), %rdi
-; CHECK-NEXT: movq %rdi, %r14
-; CHECK-NEXT: andq %rbx, %r14
-; CHECK-NEXT: shrq %rdi
+; CHECK-NEXT: shrq $2, %rsi
+; CHECK-NEXT: andq %r10, %rsi
+; CHECK-NEXT: leaq (%rsi,%rdi,4), %rsi
+; CHECK-NEXT: movq %rsi, %rdi
; CHECK-NEXT: andq %rbx, %rdi
-; CHECK-NEXT: leaq (%rdi,%r14,2), %rax
-; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rdi
-; CHECK-NEXT: bswapq %rdi
-; CHECK-NEXT: movq %rdi, %r14
-; CHECK-NEXT: shrq $4, %r14
-; CHECK-NEXT: andq %r11, %r14
-; CHECK-NEXT: andq %r11, %rdi
-; CHECK-NEXT: shlq $4, %rdi
-; CHECK-NEXT: orq %r14, %rdi
-; CHECK-NEXT: movq %rdi, %r14
-; CHECK-NEXT: andq %r10, %r14
-; CHECK-NEXT: shrq $2, %rdi
+; CHECK-NEXT: shrq %rsi
+; CHECK-NEXT: andq %rbx, %rsi
+; CHECK-NEXT: leaq (%rsi,%rdi,2), %r12
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rsi
+; CHECK-NEXT: bswapq %rsi
+; CHECK-NEXT: movq %rsi, %rdi
+; CHECK-NEXT: shrq $4, %rdi
+; CHECK-NEXT: andq %r14, %rdi
+; CHECK-NEXT: andq %r14, %rsi
+; CHECK-NEXT: shlq $4, %rsi
+; CHECK-NEXT: orq %rdi, %rsi
+; CHECK-NEXT: movq %rsi, %rdi
; CHECK-NEXT: andq %r10, %rdi
-; CHECK-NEXT: leaq (%rdi,%r14,4), %rdi
-; CHECK-NEXT: movq %rdi, %r14
-; CHECK-NEXT: andq %rbx, %r14
-; CHECK-NEXT: shrq %rdi
+; CHECK-NEXT: shrq $2, %rsi
+; CHECK-NEXT: andq %r10, %rsi
+; CHECK-NEXT: leaq (%rsi,%rdi,4), %rsi
+; CHECK-NEXT: movq %rsi, %rdi
; CHECK-NEXT: andq %rbx, %rdi
-; CHECK-NEXT: leaq (%rdi,%r14,2), %rax
-; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rdi
-; CHECK-NEXT: bswapq %rdi
-; CHECK-NEXT: movq %rdi, %r14
-; CHECK-NEXT: shrq $4, %r14
-; CHECK-NEXT: andq %r11, %r14
-; CHECK-NEXT: andq %r11, %rdi
-; CHECK-NEXT: shlq $4, %rdi
-; CHECK-NEXT: orq %r14, %rdi
-; CHECK-NEXT: movq %rdi, %r14
-; CHECK-NEXT: andq %r10, %r14
-; CHECK-NEXT: shrq $2, %rdi
+; CHECK-NEXT: shrq %rsi
+; CHECK-NEXT: andq %rbx, %rsi
+; CHECK-NEXT: leaq (%rsi,%rdi,2), %rax
+; CHECK-NEXT: shrdq $24, %r12, %r15
+; CHECK-NEXT: movq %r15, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT: movq %rax, %rsi
+; CHECK-NEXT: shlq $40, %rsi
+; CHECK-NEXT: shrq $24, %r12
+; CHECK-NEXT: orq %rsi, %r12
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rsi
+; CHECK-NEXT: bswapq %rsi
+; CHECK-NEXT: movq %rsi, %rdi
+; CHECK-NEXT: shrq $4, %rdi
+; CHECK-NEXT: andq %r14, %rdi
+; CHECK-NEXT: andq %r14, %rsi
+; CHECK-NEXT: shlq $4, %rsi
+; CHECK-NEXT: orq %rdi, %rsi
+; CHECK-NEXT: movq %rsi, %rdi
; CHECK-NEXT: andq %r10, %rdi
-; CHECK-NEXT: leaq (%rdi,%r14,4), %rdi
-; CHECK-NEXT: movq %rdi, %r14
-; CHECK-NEXT: andq %rbx, %r14
-; CHECK-NEXT: shrq %rdi
+; CHECK-NEXT: shrq $2, %rsi
+; CHECK-NEXT: andq %r10, %rsi
+; CHECK-NEXT: leaq (%rsi,%rdi,4), %rsi
+; CHECK-NEXT: movq %rsi, %rdi
; CHECK-NEXT: andq %rbx, %rdi
-; CHECK-NEXT: leaq (%rdi,%r14,2), %rax
-; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rdi
-; CHECK-NEXT: bswapq %rdi
-; CHECK-NEXT: movq %rdi, %r14
-; CHECK-NEXT: shrq $4, %r14
-; CHECK-NEXT: andq %r11, %r14
-; CHECK-NEXT: andq %r11, %rdi
-; CHECK-NEXT: shlq $4, %rdi
-; CHECK-NEXT: orq %r14, %rdi
-; CHECK-NEXT: movq %rdi, %r14
-; CHECK-NEXT: andq %r10, %r14
-; CHECK-NEXT: shrq $2, %rdi
+; CHECK-NEXT: shrq %rsi
+; CHECK-NEXT: andq %rbx, %rsi
+; CHECK-NEXT: leaq (%rsi,%rdi,2), %r15
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rsi
+; CHECK-NEXT: bswapq %rsi
+; CHECK-NEXT: movq %rsi, %rdi
+; CHECK-NEXT: shrq $4, %rdi
+; CHECK-NEXT: andq %r14, %rdi
+; CHECK-NEXT: andq %r14, %rsi
+; CHECK-NEXT: shlq $4, %rsi
+; CHECK-NEXT: orq %rdi, %rsi
+; CHECK-NEXT: movq %rsi, %rdi
; CHECK-NEXT: andq %r10, %rdi
-; CHECK-NEXT: leaq (%rdi,%r14,4), %rdi
-; CHECK-NEXT: movq %rdi, %r14
-; CHECK-NEXT: andq %rbx, %r14
-; CHECK-NEXT: shrq %rdi
+; CHECK-NEXT: shrq $2, %rsi
+; CHECK-NEXT: andq %r10, %rsi
+; CHECK-NEXT: leaq (%rsi,%rdi,4), %rsi
+; CHECK-NEXT: movq %rsi, %rdi
; CHECK-NEXT: andq %rbx, %rdi
-; CHECK-NEXT: leaq (%rdi,%r14,2), %rax
+; CHECK-NEXT: shrq %rsi
+; CHECK-NEXT: andq %rbx, %rsi
+; CHECK-NEXT: leaq (%rsi,%rdi,2), %r13
+; CHECK-NEXT: shrdq $24, %r15, %rax
; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT: movq %r13, %rsi
+; CHECK-NEXT: shlq $40, %rsi
+; CHECK-NEXT: shrq $24, %r15
+; CHECK-NEXT: orq %rsi, %r15
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rsi
+; CHECK-NEXT: bswapq %rsi
+; CHECK-NEXT: movq %rsi, %rdi
+; CHECK-NEXT: shrq $4, %rdi
+; CHECK-NEXT: andq %r14, %rdi
+; CHECK-NEXT: andq %r14, %rsi
+; CHECK-NEXT: shlq $4, %rsi
+; CHECK-NEXT: orq %rdi, %rsi
+; CHECK-NEXT: movq %rsi, %rdi
+; CHECK-NEXT: andq %r10, %rdi
+; CHECK-NEXT: shrq $2, %rsi
+; CHECK-NEXT: andq %r10, %rsi
+; CHECK-NEXT: leaq (%rsi,%rdi,4), %rsi
+; CHECK-NEXT: movq %rsi, %rdi
+; CHECK-NEXT: andq %rbx, %rdi
+; CHECK-NEXT: shrq %rsi
+; CHECK-NEXT: andq %rbx, %rsi
+; CHECK-NEXT: leaq (%rsi,%rdi,2), %rsi
; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rdi
; CHECK-NEXT: bswapq %rdi
; CHECK-NEXT: movq %rdi, %rax
; CHECK-NEXT: shrq $4, %rax
-; CHECK-NEXT: andq %r11, %rax
-; CHECK-NEXT: andq %r11, %rdi
+; CHECK-NEXT: andq %r14, %rax
+; CHECK-NEXT: andq %r14, %rdi
; CHECK-NEXT: shlq $4, %rdi
; CHECK-NEXT: orq %rax, %rdi
; CHECK-NEXT: movq %rdi, %rax
@@ -223,11 +241,16 @@ define i1000 @square(i1000 %A) nounwind {
; CHECK-NEXT: shrq %rax
; CHECK-NEXT: andq %rbx, %rax
; CHECK-NEXT: leaq (%rax,%rdi,2), %rdi
+; CHECK-NEXT: shrdq $24, %rsi, %r13
+; CHECK-NEXT: movq %rdi, %rax
+; CHECK-NEXT: shlq $40, %rax
+; CHECK-NEXT: shrq $24, %rsi
+; CHECK-NEXT: orq %rax, %rsi
; CHECK-NEXT: bswapq %r9
; CHECK-NEXT: movq %r9, %rax
; CHECK-NEXT: shrq $4, %rax
-; CHECK-NEXT: andq %r11, %rax
-; CHECK-NEXT: andq %r11, %r9
+; CHECK-NEXT: andq %r14, %rax
+; CHECK-NEXT: andq %r14, %r9
; CHECK-NEXT: shlq $4, %r9
; CHECK-NEXT: orq %rax, %r9
; CHECK-NEXT: movq %r9, %rax
@@ -239,13 +262,12 @@ define i1000 @square(i1000 %A) nounwind {
; CHECK-NEXT: andq %rbx, %r9
; CHECK-NEXT: shrq %rax
; CHECK-NEXT: andq %rbx, %rax
-; CHECK-NEXT: leaq (%rax,%r9,2), %rax
-; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT: leaq (%rax,%r9,2), %r9
; CHECK-NEXT: bswapq %r8
; CHECK-NEXT: movq %r8, %rax
; CHECK-NEXT: shrq $4, %rax
-; CHECK-NEXT: andq %r11, %rax
-; CHECK-NEXT: andq %r11, %r8
+; CHECK-NEXT: andq %r14, %rax
+; CHECK-NEXT: andq %r14, %r8
; CHECK-NEXT: shlq $4, %r8
; CHECK-NEXT: orq %rax, %r8
; CHECK-NEXT: movq %r8, %rax
@@ -257,13 +279,17 @@ define i1000 @square(i1000 %A) nounwind {
; CHECK-NEXT: andq %rbx, %r8
; CHECK-NEXT: shrq %rax
; CHECK-NEXT: andq %rbx, %rax
-; CHECK-NEXT: leaq (%rax,%r8,2), %rax
-; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT: leaq (%rax,%r8,2), %r8
+; CHECK-NEXT: shrdq $24, %r9, %rdi
+; CHECK-NEXT: movq %r8, %rax
+; CHECK-NEXT: shlq $40, %rax
+; CHECK-NEXT: shrq $24, %r9
+; CHECK-NEXT: orq %rax, %r9
; CHECK-NEXT: bswapq %rcx
; CHECK-NEXT: movq %rcx, %rax
; CHECK-NEXT: shrq $4, %rax
-; CHECK-NEXT: andq %r11, %rax
-; CHECK-NEXT: andq %r11, %rcx
+; CHECK-NEXT: andq %r14, %rax
+; CHECK-NEXT: andq %r14, %rcx
; CHECK-NEXT: shlq $4, %rcx
; CHECK-NEXT: orq %rax, %rcx
; CHECK-NEXT: movq %rcx, %rax
@@ -275,12 +301,12 @@ define i1000 @square(i1000 %A) nounwind {
; CHECK-NEXT: andq %rbx, %rcx
; CHECK-NEXT: shrq %rax
; CHECK-NEXT: andq %rbx, %rax
-; CHECK-NEXT: leaq (%rax,%rcx,2), %r14
+; CHECK-NEXT: leaq (%rax,%rcx,2), %rcx
; CHECK-NEXT: bswapq %rdx
; CHECK-NEXT: movq %rdx, %rax
; CHECK-NEXT: shrq $4, %rax
-; CHECK-NEXT: andq %r11, %rax
-; CHECK-NEXT: andq %r11, %rdx
+; CHECK-NEXT: andq %r14, %rax
+; CHECK-NEXT: andq %r14, %rdx
; CHECK-NEXT: shlq $4, %rdx
; CHECK-NEXT: orq %rax, %rdx
; CHECK-NEXT: movq %rdx, %rax
@@ -293,77 +319,56 @@ define i1000 @square(i1000 %A) nounwind {
; CHECK-NEXT: shrq %rax
; CHECK-NEXT: andq %rbx, %rax
; CHECK-NEXT: leaq (%rax,%rdx,2), %rdx
-; CHECK-NEXT: bswapq %rsi
-; CHECK-NEXT: movq %rsi, %rax
+; CHECK-NEXT: shrdq $24, %rcx, %r8
+; CHECK-NEXT: movq %rdx, %rax
+; CHECK-NEXT: shlq $40, %rax
+; CHECK-NEXT: shrq $24, %rcx
+; CHECK-NEXT: orq %rax, %rcx
+; CHECK-NEXT: bswapq %rbp
+; CHECK-NEXT: movq %rbp, %rax
; CHECK-NEXT: shrq $4, %rax
-; CHECK-NEXT: andq %r11, %rax
-; CHECK-NEXT: andq %r11, %rsi
-; CHECK-NEXT: shlq $4, %rsi
-; CHECK-NEXT: orq %rax, %rsi
-; CHECK-NEXT: movq %rsi, %rax
+; CHECK-NEXT: andq %r14, %rax
+; CHECK-NEXT: andq %r14, %rbp
+; CHECK-NEXT: shlq $4, %rbp
+; CHECK-NEXT: orq %rax, %rbp
+; CHECK-NEXT: movq %rbp, %rax
; CHECK-NEXT: andq %r10, %rax
-; CHECK-NEXT: shrq $2, %rsi
-; CHECK-NEXT: andq %r10, %rsi
-; CHECK-NEXT: leaq (%rsi,%rax,4), %rax
-; CHECK-NEXT: movq %rax, %rsi
-; CHECK-NEXT: andq %rbx, %rsi
+; CHECK-NEXT: shrq $2, %rbp
+; CHECK-NEXT: andq %r10, %rbp
+; CHECK-NEXT: leaq (%rbp,%rax,4), %rax
+; CHECK-NEXT: movq %rax, %r10
+; CHECK-NEXT: andq %rbx, %r10
; CHECK-NEXT: shrq %rax
; CHECK-NEXT: andq %rbx, %rax
-; CHECK-NEXT: leaq (%rax,%rsi,2), %rsi
-; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Reload
-; CHECK-NEXT: shrdq $24, %rax, %r10
+; CHECK-NEXT: leaq (%rax,%r10,2), %rax
+; CHECK-NEXT: shrdq $24, %rax, %rdx
+; CHECK-NEXT: movq %rdx, 112(%r11)
+; CHECK-NEXT: movq %rcx, 104(%r11)
+; CHECK-NEXT: movq %r8, 96(%r11)
+; CHECK-NEXT: movq %r9, 88(%r11)
+; CHECK-NEXT: movq %rdi, 80(%r11)
+; CHECK-NEXT: movq %rsi, 72(%r11)
+; CHECK-NEXT: movq %r13, 64(%r11)
+; CHECK-NEXT: movq %r15, 56(%r11)
; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; CHECK-NEXT: shrdq $24, %rcx, %rax
-; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Reload
-; CHECK-NEXT: shrdq $24, %rbp, %rcx
-; CHECK-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Reload
-; CHECK-NEXT: shrdq $24, %r13, %rbp
-; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r12 # 8-byte Reload
-; CHECK-NEXT: shrdq $24, %r12, %r13
-; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r15 # 8-byte Reload
-; CHECK-NEXT: shrdq $24, %r15, %r12
-; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Reload
-; CHECK-NEXT: shrdq $24, %rbx, %r15
-; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Reload
-; CHECK-NEXT: shrdq $24, %r11, %rbx
-; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r9 # 8-byte Reload
-; CHECK-NEXT: shrdq $24, %r9, %r11
-; CHECK-NEXT: movq %rdi, %r8
-; CHECK-NEXT: shrdq $24, %rdi, %r9
-; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
-; CHECK-NEXT: shrdq $24, %rdi, %r8
-; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; CHECK-NEXT: shrdq $24, %rax, %rdi
-; CHECK-NEXT: shrdq $24, %r14, %rax
-; CHECK-NEXT: movq %rax, %rcx
-; CHECK-NEXT: shrdq $24, %rdx, %r14
-; CHECK-NEXT: shrdq $24, %rsi, %rdx
-; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; CHECK-NEXT: movq %rdx, 112(%rax)
-; CHECK-NEXT: movq %r14, 104(%rax)
-; CHECK-NEXT: movq %rcx, 96(%rax)
-; CHECK-NEXT: movq %rdi, 88(%rax)
-; CHECK-NEXT: movq %r8, 80(%rax)
-; CHECK-NEXT: movq %r9, 72(%rax)
-; CHECK-NEXT: movq %r11, 64(%rax)
-; CHECK-NEXT: movq %rbx, 56(%rax)
-; CHECK-NEXT: movq %r15, 48(%rax)
-; CHECK-NEXT: movq %r12, 40(%rax)
-; CHECK-NEXT: movq %r13, 32(%rax)
-; CHECK-NEXT: movq %rbp, 24(%rax)
+; CHECK-NEXT: movq %rcx, 48(%r11)
+; CHECK-NEXT: movq %r12, 40(%r11)
+; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; CHECK-NEXT: movq %rcx, 32(%r11)
+; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; CHECK-NEXT: movq %rcx, 24(%r11)
; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; CHECK-NEXT: movq %rcx, 16(%rax)
+; CHECK-NEXT: movq %rcx, 16(%r11)
; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; CHECK-NEXT: movq %rcx, 8(%rax)
-; CHECK-NEXT: movq %r10, (%rax)
-; CHECK-NEXT: movq %rsi, %rcx
-; CHECK-NEXT: shrq $56, %rsi
-; CHECK-NEXT: movb %sil, 124(%rax)
+; CHECK-NEXT: movq %rcx, 8(%r11)
+; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; CHECK-NEXT: movq %rcx, (%r11)
+; CHECK-NEXT: movq %rax, %rcx
+; CHECK-NEXT: shrq $56, %rax
+; CHECK-NEXT: movb %al, 124(%r11)
; CHECK-NEXT: shrq $24, %rcx
-; CHECK-NEXT: movl %ecx, 120(%rax)
+; CHECK-NEXT: movl %ecx, 120(%r11)
+; CHECK-NEXT: movq %r11, %rax
; CHECK-NEXT: popq %rbx
; CHECK-NEXT: popq %r12
; CHECK-NEXT: popq %r13
diff --git a/llvm/test/CodeGen/X86/pr49162.ll b/llvm/test/CodeGen/X86/pr49162.ll
index db8cec61acd6b..8409a27a76763 100644
--- a/llvm/test/CodeGen/X86/pr49162.ll
+++ b/llvm/test/CodeGen/X86/pr49162.ll
@@ -6,11 +6,13 @@ define ptr @PR49162(ptr %base, ptr %ptr160) {
; X86-LABEL: PR49162:
; X86: # %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl 8(%eax), %ecx
+; X86-NEXT: movl 8(%eax), %eax
+; X86-NEXT: shll $16, %eax
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: sarl $31, %ecx
; X86-NEXT: shll $16, %ecx
-; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: sarl $31, %eax
-; X86-NEXT: shldl $16, %ecx, %eax
+; X86-NEXT: shrl $16, %eax
+; X86-NEXT: orl %ecx, %eax
; X86-NEXT: shll $2, %eax
; X86-NEXT: addl {{[0-9]+}}(%esp), %eax
; X86-NEXT: retl
diff --git a/llvm/test/CodeGen/X86/rot32.ll b/llvm/test/CodeGen/X86/rot32.ll
index 114d558954f4c..eb02931e66ff3 100644
--- a/llvm/test/CodeGen/X86/rot32.ll
+++ b/llvm/test/CodeGen/X86/rot32.ll
@@ -189,16 +189,32 @@ entry:
define i32 @xbar(i32 %x, i32 %y, i32 %z) nounwind readnone {
; CHECK32-LABEL: xbar:
; CHECK32: # %bb.0: # %entry
-; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %ecx
; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; CHECK32-NEXT: shldl $7, %ecx, %eax
+; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; CHECK32-NEXT: shll $7, %ecx
+; CHECK32-NEXT: shrl $25, %eax
+; CHECK32-NEXT: orl %ecx, %eax
; CHECK32-NEXT: retl
;
-; CHECK64-LABEL: xbar:
-; CHECK64: # %bb.0: # %entry
-; CHECK64-NEXT: movl %edi, %eax
-; CHECK64-NEXT: shrdl $25, %esi, %eax
-; CHECK64-NEXT: retq
+; X64-LABEL: xbar:
+; X64: # %bb.0: # %entry
+; X64-NEXT: movl %edi, %eax
+; X64-NEXT: shrdl $25, %esi, %eax
+; X64-NEXT: retq
+;
+; SHLD64-LABEL: xbar:
+; SHLD64: # %bb.0: # %entry
+; SHLD64-NEXT: movl %edi, %eax
+; SHLD64-NEXT: shrdl $25, %esi, %eax
+; SHLD64-NEXT: retq
+;
+; BMI264-LABEL: xbar:
+; BMI264: # %bb.0: # %entry
+; BMI264-NEXT: movl %edi, %eax
+; BMI264-NEXT: shll $7, %esi
+; BMI264-NEXT: shrl $25, %eax
+; BMI264-NEXT: orl %esi, %eax
+; BMI264-NEXT: retq
entry:
%0 = shl i32 %y, 7
%1 = lshr i32 %x, 25
@@ -295,16 +311,32 @@ entry:
define i32 @xbu(i32 %x, i32 %y, i32 %z) nounwind readnone {
; CHECK32-LABEL: xbu:
; CHECK32: # %bb.0: # %entry
-; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %ecx
; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; CHECK32-NEXT: shldl $25, %ecx, %eax
+; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; CHECK32-NEXT: shll $25, %ecx
+; CHECK32-NEXT: shrl $7, %eax
+; CHECK32-NEXT: orl %ecx, %eax
; CHECK32-NEXT: retl
;
-; CHECK64-LABEL: xbu:
-; CHECK64: # %bb.0: # %entry
-; CHECK64-NEXT: movl %edi, %eax
-; CHECK64-NEXT: shldl $25, %esi, %eax
-; CHECK64-NEXT: retq
+; X64-LABEL: xbu:
+; X64: # %bb.0: # %entry
+; X64-NEXT: movl %edi, %eax
+; X64-NEXT: shldl $25, %esi, %eax
+; X64-NEXT: retq
+;
+; SHLD64-LABEL: xbu:
+; SHLD64: # %bb.0: # %entry
+; SHLD64-NEXT: movl %edi, %eax
+; SHLD64-NEXT: shldl $25, %esi, %eax
+; SHLD64-NEXT: retq
+;
+; BMI264-LABEL: xbu:
+; BMI264: # %bb.0: # %entry
+; BMI264-NEXT: movl %esi, %eax
+; BMI264-NEXT: shll $25, %edi
+; BMI264-NEXT: shrl $7, %eax
+; BMI264-NEXT: orl %edi, %eax
+; BMI264-NEXT: retq
entry:
%0 = lshr i32 %y, 7
%1 = shl i32 %x, 25
diff --git a/llvm/test/CodeGen/X86/rot64.ll b/llvm/test/CodeGen/X86/rot64.ll
index b53a686038e91..526ed08ef45a1 100644
--- a/llvm/test/CodeGen/X86/rot64.ll
+++ b/llvm/test/CodeGen/X86/rot64.ll
@@ -117,11 +117,25 @@ entry:
}
define i64 @xbar(i64 %x, i64 %y, i64 %z) nounwind readnone {
-; ALL-LABEL: xbar:
-; ALL: # %bb.0: # %entry
-; ALL-NEXT: movq %rdi, %rax
-; ALL-NEXT: shrdq $57, %rsi, %rax
-; ALL-NEXT: retq
+; X64-LABEL: xbar:
+; X64: # %bb.0: # %entry
+; X64-NEXT: movq %rdi, %rax
+; X64-NEXT: shrdq $57, %rsi, %rax
+; X64-NEXT: retq
+;
+; SHLD-LABEL: xbar:
+; SHLD: # %bb.0: # %entry
+; SHLD-NEXT: movq %rdi, %rax
+; SHLD-NEXT: shrdq $57, %rsi, %rax
+; SHLD-NEXT: retq
+;
+; BMI2-LABEL: xbar:
+; BMI2: # %bb.0: # %entry
+; BMI2-NEXT: movq %rdi, %rax
+; BMI2-NEXT: shlq $7, %rsi
+; BMI2-NEXT: shrq $57, %rax
+; BMI2-NEXT: orq %rsi, %rax
+; BMI2-NEXT: retq
entry:
%0 = shl i64 %y, 7
%1 = lshr i64 %x, 57
@@ -179,11 +193,25 @@ entry:
}
define i64 @xbu(i64 %x, i64 %y, i64 %z) nounwind readnone {
-; ALL-LABEL: xbu:
-; ALL: # %bb.0: # %entry
-; ALL-NEXT: movq %rdi, %rax
-; ALL-NEXT: shldq $57, %rsi, %rax
-; ALL-NEXT: retq
+; X64-LABEL: xbu:
+; X64: # %bb.0: # %entry
+; X64-NEXT: movq %rdi, %rax
+; X64-NEXT: shldq $57, %rsi, %rax
+; X64-NEXT: retq
+;
+; SHLD-LABEL: xbu:
+; SHLD: # %bb.0: # %entry
+; SHLD-NEXT: movq %rdi, %rax
+; SHLD-NEXT: shldq $57, %rsi, %rax
+; SHLD-NEXT: retq
+;
+; BMI2-LABEL: xbu:
+; BMI2: # %bb.0: # %entry
+; BMI2-NEXT: movq %rsi, %rax
+; BMI2-NEXT: shlq $57, %rdi
+; BMI2-NEXT: shrq $7, %rax
+; BMI2-NEXT: orq %rdi, %rax
+; BMI2-NEXT: retq
entry:
%0 = lshr i64 %y, 7
%1 = shl i64 %x, 57
diff --git a/llvm/test/CodeGen/X86/rotate-add.ll b/llvm/test/CodeGen/X86/rotate-add.ll
index c705505bbbf2a..e0c06f022eb80 100644
--- a/llvm/test/CodeGen/X86/rotate-add.ll
+++ b/llvm/test/CodeGen/X86/rotate-add.ll
@@ -221,9 +221,14 @@ define i64 @test_rotl_udiv_special_case(i64 %i) {
; X86-NEXT: addl %ecx, %edx
; X86-NEXT: imull $-1431655765, %edi, %ecx # imm = 0xAAAAAAAB
; X86-NEXT: addl %ecx, %edx
-; X86-NEXT: movl %edx, %ecx
-; X86-NEXT: shldl $28, %eax, %ecx
-; X86-NEXT: shrdl $4, %eax, %edx
+; X86-NEXT: movl %edx, %esi
+; X86-NEXT: shll $28, %esi
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: shrl $4, %ecx
+; X86-NEXT: orl %esi, %ecx
+; X86-NEXT: shll $28, %eax
+; X86-NEXT: shrl $4, %edx
+; X86-NEXT: orl %eax, %edx
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: popl %esi
; X86-NEXT: .cfi_def_cfa_offset 12
@@ -278,7 +283,9 @@ define i64 @test_rotl_mul_with_mask_special_case(i64 %i) {
; X86-NEXT: movl $9, %eax
; X86-NEXT: mull {{[0-9]+}}(%esp)
; X86-NEXT: addl %ecx, %edx
-; X86-NEXT: shrdl $25, %eax, %edx
+; X86-NEXT: shll $7, %eax
+; X86-NEXT: shrl $25, %edx
+; X86-NEXT: orl %eax, %edx
; X86-NEXT: movzbl %dl, %eax
; X86-NEXT: xorl %edx, %edx
; X86-NEXT: retl
@@ -304,10 +311,12 @@ define i64 @test_rotl_mul_with_mask_special_case(i64 %i) {
define i32 @test_fshl_with_mask_special_case(i32 %x) {
; X86-LABEL: test_fshl_with_mask_special_case:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: orl $1, %eax
-; X86-NEXT: shldl $5, %ecx, %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: orl $1, %ecx
+; X86-NEXT: shll $5, %ecx
+; X86-NEXT: shrl $27, %eax
+; X86-NEXT: orl %ecx, %eax
; X86-NEXT: andl $-31, %eax
; X86-NEXT: retl
;
@@ -315,7 +324,9 @@ define i32 @test_fshl_with_mask_special_case(i32 %x) {
; X64: # %bb.0:
; X64-NEXT: movl %edi, %eax
; X64-NEXT: orl $1, %eax
-; X64-NEXT: shldl $5, %edi, %eax
+; X64-NEXT: shll $5, %eax
+; X64-NEXT: shrl $27, %edi
+; X64-NEXT: orl %edi, %eax
; X64-NEXT: andl $-31, %eax
; X64-NEXT: retq
%or1 = or i32 %x, 1
diff --git a/llvm/test/CodeGen/X86/rotate-extract.ll b/llvm/test/CodeGen/X86/rotate-extract.ll
index b5332068d7edd..d70c7846e3544 100644
--- a/llvm/test/CodeGen/X86/rotate-extract.ll
+++ b/llvm/test/CodeGen/X86/rotate-extract.ll
@@ -12,13 +12,23 @@
define i64 @rolq_extract_shl(i64 %i) nounwind {
; X86-LABEL: rolq_extract_shl:
; X86: # %bb.0:
+; X86-NEXT: pushl %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: shldl $3, %edx, %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: shll $3, %eax
+; X86-NEXT: movl %edx, %ecx
+; X86-NEXT: shrl $29, %ecx
+; X86-NEXT: orl %eax, %ecx
; X86-NEXT: shll $3, %edx
-; X86-NEXT: movl %edx, %eax
-; X86-NEXT: shldl $7, %ecx, %eax
-; X86-NEXT: shrdl $25, %ecx, %edx
+; X86-NEXT: movl %edx, %esi
+; X86-NEXT: shll $7, %esi
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: shrl $25, %eax
+; X86-NEXT: orl %esi, %eax
+; X86-NEXT: shll $7, %ecx
+; X86-NEXT: shrl $25, %edx
+; X86-NEXT: orl %ecx, %edx
+; X86-NEXT: popl %esi
; X86-NEXT: retl
;
; X64-LABEL: rolq_extract_shl:
@@ -110,7 +120,9 @@ define i64 @rolq_extract_mul_with_mask(i64 %i) nounwind {
; X86-NEXT: movl $9, %eax
; X86-NEXT: mull {{[0-9]+}}(%esp)
; X86-NEXT: addl %ecx, %edx
-; X86-NEXT: shrdl $25, %eax, %edx
+; X86-NEXT: shll $7, %eax
+; X86-NEXT: shrl $25, %edx
+; X86-NEXT: orl %eax, %edx
; X86-NEXT: movzbl %dl, %eax
; X86-NEXT: xorl %edx, %edx
; X86-NEXT: retl
@@ -133,14 +145,19 @@ define i64 @rolq_extract_mul_with_mask(i64 %i) nounwind {
define i64 @no_extract_shl(i64 %i) nounwind {
; X86-LABEL: no_extract_shl:
; X86: # %bb.0:
+; X86-NEXT: pushl %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl %eax, %edx
-; X86-NEXT: shldl $10, %ecx, %edx
+; X86-NEXT: movl %eax, %esi
+; X86-NEXT: shll $10, %esi
+; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: shrl $22, %edx
+; X86-NEXT: orl %esi, %edx
; X86-NEXT: shll $10, %ecx
; X86-NEXT: shrl $20, %eax
; X86-NEXT: andl $127, %eax
; X86-NEXT: orl %ecx, %eax
+; X86-NEXT: popl %esi
; X86-NEXT: retl
;
; X64-LABEL: no_extract_shl:
diff --git a/llvm/test/CodeGen/X86/rotate.ll b/llvm/test/CodeGen/X86/rotate.ll
index ea32edba62822..e76821e2f3325 100644
--- a/llvm/test/CodeGen/X86/rotate.ll
+++ b/llvm/test/CodeGen/X86/rotate.ll
@@ -113,11 +113,18 @@ define i64 @rotr64(i64 %A, i8 %Amt) nounwind {
define i64 @rotli64(i64 %A) nounwind {
; X86-LABEL: rotli64:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: pushl %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl %edx, %esi
+; X86-NEXT: shll $5, %esi
; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: shldl $5, %edx, %eax
-; X86-NEXT: shldl $5, %ecx, %edx
+; X86-NEXT: shrl $27, %eax
+; X86-NEXT: orl %esi, %eax
+; X86-NEXT: shll $5, %ecx
+; X86-NEXT: shrl $27, %edx
+; X86-NEXT: orl %ecx, %edx
+; X86-NEXT: popl %esi
; X86-NEXT: retl
;
; X64-LABEL: rotli64:
@@ -134,11 +141,18 @@ define i64 @rotli64(i64 %A) nounwind {
define i64 @rotri64(i64 %A) nounwind {
; X86-LABEL: rotri64:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: pushl %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movl %edx, %esi
+; X86-NEXT: shll $27, %esi
; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: shldl $27, %edx, %eax
-; X86-NEXT: shldl $27, %ecx, %edx
+; X86-NEXT: shrl $5, %eax
+; X86-NEXT: orl %esi, %eax
+; X86-NEXT: shll $27, %ecx
+; X86-NEXT: shrl $5, %edx
+; X86-NEXT: orl %ecx, %edx
+; X86-NEXT: popl %esi
; X86-NEXT: retl
;
; X64-LABEL: rotri64:
@@ -155,11 +169,17 @@ define i64 @rotri64(i64 %A) nounwind {
define i64 @rotl1_64(i64 %A) nounwind {
; X86-LABEL: rotl1_64:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: pushl %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: leal (,%edx,2), %esi
; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: shldl $1, %edx, %eax
-; X86-NEXT: shldl $1, %ecx, %edx
+; X86-NEXT: shrl $31, %eax
+; X86-NEXT: orl %esi, %eax
+; X86-NEXT: shll %ecx
+; X86-NEXT: shrl $31, %edx
+; X86-NEXT: orl %ecx, %edx
+; X86-NEXT: popl %esi
; X86-NEXT: retl
;
; X64-LABEL: rotl1_64:
@@ -176,11 +196,18 @@ define i64 @rotl1_64(i64 %A) nounwind {
define i64 @rotr1_64(i64 %A) nounwind {
; X86-LABEL: rotr1_64:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: pushl %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movl %edx, %esi
+; X86-NEXT: shll $31, %esi
; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: shldl $31, %edx, %eax
-; X86-NEXT: shldl $31, %ecx, %edx
+; X86-NEXT: shrl %eax
+; X86-NEXT: orl %esi, %eax
+; X86-NEXT: shll $31, %ecx
+; X86-NEXT: shrl %edx
+; X86-NEXT: orl %ecx, %edx
+; X86-NEXT: popl %esi
; X86-NEXT: retl
;
; X64-LABEL: rotr1_64:
@@ -565,16 +592,23 @@ define i8 @rotr1_8(i8 %A) nounwind {
define void @rotr1_64_mem(ptr %Aptr) nounwind {
; X86-LABEL: rotr1_64_mem:
; X86: # %bb.0:
+; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl (%eax), %ecx
; X86-NEXT: movl 4(%eax), %edx
; X86-NEXT: movl %ecx, %esi
-; X86-NEXT: shldl $31, %edx, %esi
-; X86-NEXT: shldl $31, %ecx, %edx
-; X86-NEXT: movl %edx, (%eax)
-; X86-NEXT: movl %esi, 4(%eax)
+; X86-NEXT: shll $31, %esi
+; X86-NEXT: movl %edx, %edi
+; X86-NEXT: shrl %edi
+; X86-NEXT: orl %esi, %edi
+; X86-NEXT: shll $31, %edx
+; X86-NEXT: shrl %ecx
+; X86-NEXT: orl %edx, %ecx
+; X86-NEXT: movl %ecx, (%eax)
+; X86-NEXT: movl %edi, 4(%eax)
; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
; X86-NEXT: retl
;
; X64-LABEL: rotr1_64_mem:
diff --git a/llvm/test/CodeGen/X86/rotate2.ll b/llvm/test/CodeGen/X86/rotate2.ll
index ac299581dd2f8..2c58e0a9de3ab 100644
--- a/llvm/test/CodeGen/X86/rotate2.ll
+++ b/llvm/test/CodeGen/X86/rotate2.ll
@@ -5,11 +5,18 @@
define i64 @test1(i64 %x) nounwind {
; X86-LABEL: test1:
; X86: # %bb.0: # %entry
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: pushl %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl %edx, %esi
+; X86-NEXT: shll $9, %esi
; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: shldl $9, %edx, %eax
-; X86-NEXT: shldl $9, %ecx, %edx
+; X86-NEXT: shrl $23, %eax
+; X86-NEXT: orl %esi, %eax
+; X86-NEXT: shll $9, %ecx
+; X86-NEXT: shrl $23, %edx
+; X86-NEXT: orl %ecx, %edx
+; X86-NEXT: popl %esi
; X86-NEXT: retl
;
; X64-LABEL: test1:
diff --git a/llvm/test/CodeGen/X86/scmp.ll b/llvm/test/CodeGen/X86/scmp.ll
index 393e05bfd0cc6..2687c70d7b948 100644
--- a/llvm/test/CodeGen/X86/scmp.ll
+++ b/llvm/test/CodeGen/X86/scmp.ll
@@ -2518,8 +2518,8 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; SSE2-NEXT: pushq %r13
; SSE2-NEXT: pushq %r12
; SSE2-NEXT: pushq %rbx
-; SSE2-NEXT: movq %rdi, %rax
-; SSE2-NEXT: movzbl {{[0-9]+}}(%rsp), %edi
+; SSE2-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE2-NEXT: movzbl {{[0-9]+}}(%rsp), %eax
; SSE2-NEXT: movzbl {{[0-9]+}}(%rsp), %r10d
; SSE2-NEXT: movzbl {{[0-9]+}}(%rsp), %r11d
; SSE2-NEXT: movzbl {{[0-9]+}}(%rsp), %ebx
@@ -2535,9 +2535,10 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; SSE2-NEXT: setg %r15b
; SSE2-NEXT: subb %sil, %r15b
; SSE2-NEXT: movsbq %r15b, %rsi
-; SSE2-NEXT: movq %rsi, (%rax)
+; SSE2-NEXT: movq %rsi, (%rdi)
; SSE2-NEXT: movq %rsi, %xmm0
; SSE2-NEXT: sarq $63, %rsi
+; SSE2-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; SSE2-NEXT: addb %r14b, %r14b
; SSE2-NEXT: sarb %r14b
; SSE2-NEXT: movzbl {{[0-9]+}}(%rsp), %r15d
@@ -2558,9 +2559,9 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; SSE2-NEXT: setl %dl
; SSE2-NEXT: setg %bpl
; SSE2-NEXT: subb %dl, %bpl
-; SSE2-NEXT: movsbq %bpl, %rdx
-; SSE2-NEXT: movq %rdx, %r12
-; SSE2-NEXT: sarq $63, %r12
+; SSE2-NEXT: movsbq %bpl, %r12
+; SSE2-NEXT: movq %r12, %rsi
+; SSE2-NEXT: sarq $63, %rsi
; SSE2-NEXT: addb %bl, %bl
; SSE2-NEXT: sarb %bl
; SSE2-NEXT: addb %cl, %cl
@@ -2569,9 +2570,9 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; SSE2-NEXT: setl %cl
; SSE2-NEXT: setg %bl
; SSE2-NEXT: subb %cl, %bl
-; SSE2-NEXT: movsbq %bl, %rbx
-; SSE2-NEXT: movq %rbx, %rcx
-; SSE2-NEXT: sarq $63, %rcx
+; SSE2-NEXT: movsbq %bl, %rdx
+; SSE2-NEXT: movq %rdx, %rbx
+; SSE2-NEXT: sarq $63, %rbx
; SSE2-NEXT: addb %r11b, %r11b
; SSE2-NEXT: sarb %r11b
; SSE2-NEXT: addb %r8b, %r8b
@@ -2594,72 +2595,78 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; SSE2-NEXT: movsbq %r10b, %r9
; SSE2-NEXT: movq %r9, %r10
; SSE2-NEXT: sarq $63, %r10
-; SSE2-NEXT: addb %dil, %dil
-; SSE2-NEXT: sarb %dil
+; SSE2-NEXT: addb %al, %al
+; SSE2-NEXT: sarb %al
; SSE2-NEXT: movzbl {{[0-9]+}}(%rsp), %ebp
; SSE2-NEXT: addb %bpl, %bpl
; SSE2-NEXT: sarb %bpl
-; SSE2-NEXT: cmpb %dil, %bpl
+; SSE2-NEXT: cmpb %al, %bpl
; SSE2-NEXT: setl %dil
; SSE2-NEXT: setg %bpl
; SSE2-NEXT: subb %dil, %bpl
-; SSE2-NEXT: movsbq %bpl, %rdi
-; SSE2-NEXT: movq %rdi, %r13
+; SSE2-NEXT: movsbq %bpl, %rax
+; SSE2-NEXT: movq %rax, %r13
; SSE2-NEXT: sarq $63, %r13
-; SSE2-NEXT: movl %r13d, 96(%rax)
+; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; SSE2-NEXT: movl %r13d, 96(%rcx)
; SSE2-NEXT: movabsq $2251799813685247, %rbp # imm = 0x7FFFFFFFFFFFF
; SSE2-NEXT: andq %r13, %rbp
-; SSE2-NEXT: shldq $62, %rdi, %r13
-; SSE2-NEXT: movq %r13, 88(%rax)
-; SSE2-NEXT: movq %r9, %r13
-; SSE2-NEXT: shrdq $44, %r10, %r13
-; SSE2-NEXT: movq %r13, 64(%rax)
-; SSE2-NEXT: movq %r8, %r13
-; SSE2-NEXT: shrdq $33, %r11, %r13
-; SSE2-NEXT: movq %r13, 48(%rax)
-; SSE2-NEXT: movq %rbx, %r13
-; SSE2-NEXT: shrdq $22, %rcx, %r13
-; SSE2-NEXT: movq %r13, 32(%rax)
-; SSE2-NEXT: movq %rdx, %r13
-; SSE2-NEXT: shrdq $11, %r12, %r13
-; SSE2-NEXT: movq %r13, 16(%rax)
-; SSE2-NEXT: movq %rbp, %r13
-; SSE2-NEXT: shrq $48, %r13
-; SSE2-NEXT: movb %r13b, 102(%rax)
+; SSE2-NEXT: shlq $62, %r13
+; SSE2-NEXT: movq %rax, %rdi
+; SSE2-NEXT: shrq $2, %rdi
+; SSE2-NEXT: orq %r13, %rdi
+; SSE2-NEXT: movq %rdi, 88(%rcx)
+; SSE2-NEXT: movq %r9, %rdi
+; SSE2-NEXT: shrdq $44, %r10, %rdi
+; SSE2-NEXT: movq %rdi, 64(%rcx)
+; SSE2-NEXT: movq %r8, %rdi
+; SSE2-NEXT: shrdq $33, %r11, %rdi
+; SSE2-NEXT: movq %rdi, 48(%rcx)
+; SSE2-NEXT: movq %rdx, %rdi
+; SSE2-NEXT: shrdq $22, %rbx, %rdi
+; SSE2-NEXT: movq %rdi, 32(%rcx)
+; SSE2-NEXT: movq %r12, %rdi
+; SSE2-NEXT: shrdq $11, %rsi, %rdi
+; SSE2-NEXT: movq %rdi, 16(%rcx)
+; SSE2-NEXT: movq %rbp, %rdi
+; SSE2-NEXT: shrq $48, %rdi
+; SSE2-NEXT: movb %dil, 102(%rcx)
; SSE2-NEXT: shrq $32, %rbp
-; SSE2-NEXT: movw %bp, 100(%rax)
-; SSE2-NEXT: movabsq $9007199254740991, %r13 # imm = 0x1FFFFFFFFFFFFF
-; SSE2-NEXT: andq %r13, %r15
+; SSE2-NEXT: movw %bp, 100(%rcx)
+; SSE2-NEXT: movabsq $9007199254740991, %rdi # imm = 0x1FFFFFFFFFFFFF
+; SSE2-NEXT: andq %rdi, %r15
; SSE2-NEXT: shldq $9, %r14, %r15
-; SSE2-NEXT: shlq $62, %rdi
-; SSE2-NEXT: orq %r15, %rdi
-; SSE2-NEXT: movq %rdi, 80(%rax)
-; SSE2-NEXT: movabsq $9007199254738944, %rdi # imm = 0x1FFFFFFFFFF800
-; SSE2-NEXT: andq %r12, %rdi
-; SSE2-NEXT: shlq $42, %rbx
-; SSE2-NEXT: shrq $11, %rdi
-; SSE2-NEXT: orq %rbx, %rdi
-; SSE2-NEXT: movq %rdi, 24(%rax)
+; SSE2-NEXT: shlq $62, %rax
+; SSE2-NEXT: orq %r15, %rax
+; SSE2-NEXT: movq %rax, 80(%rcx)
+; SSE2-NEXT: movabsq $9007199254738944, %rax # imm = 0x1FFFFFFFFFF800
+; SSE2-NEXT: andq %rsi, %rax
+; SSE2-NEXT: shlq $42, %rdx
+; SSE2-NEXT: shrq $11, %rax
+; SSE2-NEXT: orq %rdx, %rax
+; SSE2-NEXT: movq %rax, 24(%rcx)
; SSE2-NEXT: shlq $9, %r14
; SSE2-NEXT: andl $511, %r10d # imm = 0x1FF
; SSE2-NEXT: orq %r14, %r10
-; SSE2-NEXT: movq %r10, 72(%rax)
+; SSE2-NEXT: movq %r10, 72(%rcx)
; SSE2-NEXT: shlq $20, %r9
; SSE2-NEXT: andl $1048575, %r11d # imm = 0xFFFFF
; SSE2-NEXT: orq %r9, %r11
-; SSE2-NEXT: movq %r11, 56(%rax)
+; SSE2-NEXT: movq %r11, 56(%rcx)
; SSE2-NEXT: shlq $31, %r8
-; SSE2-NEXT: andl $2147483647, %ecx # imm = 0x7FFFFFFF
-; SSE2-NEXT: orq %r8, %rcx
-; SSE2-NEXT: movq %rcx, 40(%rax)
-; SSE2-NEXT: movq %rsi, %xmm1
+; SSE2-NEXT: andl $2147483647, %ebx # imm = 0x7FFFFFFF
+; SSE2-NEXT: orq %r8, %rbx
+; SSE2-NEXT: movq %rbx, 40(%rcx)
+; SSE2-NEXT: movq %rcx, %rax
+; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Folded Reload
+; SSE2-NEXT: # xmm1 = mem[0],zero
; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
; SSE2-NEXT: movq %xmm0, %rcx
-; SSE2-NEXT: andq %r13, %rcx
-; SSE2-NEXT: shlq $53, %rdx
-; SSE2-NEXT: orq %rcx, %rdx
-; SSE2-NEXT: movq %rdx, 8(%rax)
+; SSE2-NEXT: andq %rdi, %rcx
+; SSE2-NEXT: shlq $53, %r12
+; SSE2-NEXT: orq %rcx, %r12
+; SSE2-NEXT: movq %r12, 8(%rax)
; SSE2-NEXT: popq %rbx
; SSE2-NEXT: popq %r12
; SSE2-NEXT: popq %r13
@@ -2821,157 +2828,314 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; SSE4-NEXT: popq %rbp
; SSE4-NEXT: retq
;
-; AVX-LABEL: scmp_uncommon_vectors:
-; AVX: # %bb.0:
-; AVX-NEXT: pushq %rbp
-; AVX-NEXT: pushq %r15
-; AVX-NEXT: pushq %r14
-; AVX-NEXT: pushq %r13
-; AVX-NEXT: pushq %r12
-; AVX-NEXT: pushq %rbx
-; AVX-NEXT: movq %rdi, %rax
-; AVX-NEXT: movzbl {{[0-9]+}}(%rsp), %edi
-; AVX-NEXT: movzbl {{[0-9]+}}(%rsp), %r10d
-; AVX-NEXT: movzbl {{[0-9]+}}(%rsp), %r11d
-; AVX-NEXT: movzbl {{[0-9]+}}(%rsp), %ebx
-; AVX-NEXT: movzbl {{[0-9]+}}(%rsp), %ebp
-; AVX-NEXT: movzbl {{[0-9]+}}(%rsp), %r15d
-; AVX-NEXT: movzbl {{[0-9]+}}(%rsp), %r14d
-; AVX-NEXT: addb %r14b, %r14b
-; AVX-NEXT: sarb %r14b
-; AVX-NEXT: addb %sil, %sil
-; AVX-NEXT: sarb %sil
-; AVX-NEXT: cmpb %r14b, %sil
-; AVX-NEXT: setl %sil
-; AVX-NEXT: setg %r14b
-; AVX-NEXT: subb %sil, %r14b
-; AVX-NEXT: movsbq %r14b, %r14
-; AVX-NEXT: movq %r14, (%rax)
-; AVX-NEXT: sarq $63, %r14
-; AVX-NEXT: addb %r15b, %r15b
-; AVX-NEXT: sarb %r15b
-; AVX-NEXT: movzbl {{[0-9]+}}(%rsp), %esi
-; AVX-NEXT: addb %sil, %sil
-; AVX-NEXT: sarb %sil
-; AVX-NEXT: cmpb %r15b, %sil
-; AVX-NEXT: setl %sil
-; AVX-NEXT: setg %r15b
-; AVX-NEXT: subb %sil, %r15b
-; AVX-NEXT: movsbq %r15b, %rsi
-; AVX-NEXT: movq %rsi, %r12
-; AVX-NEXT: sarq $63, %r12
-; AVX-NEXT: addb %bpl, %bpl
-; AVX-NEXT: sarb %bpl
-; AVX-NEXT: addb %dl, %dl
-; AVX-NEXT: sarb %dl
-; AVX-NEXT: cmpb %bpl, %dl
-; AVX-NEXT: setl %dl
-; AVX-NEXT: setg %bpl
-; AVX-NEXT: subb %dl, %bpl
-; AVX-NEXT: movsbq %bpl, %r15
-; AVX-NEXT: movq %r15, %r13
-; AVX-NEXT: sarq $63, %r13
-; AVX-NEXT: addb %bl, %bl
-; AVX-NEXT: sarb %bl
-; AVX-NEXT: addb %cl, %cl
-; AVX-NEXT: sarb %cl
-; AVX-NEXT: cmpb %bl, %cl
-; AVX-NEXT: setl %cl
-; AVX-NEXT: setg %dl
-; AVX-NEXT: subb %cl, %dl
-; AVX-NEXT: movsbq %dl, %rbx
-; AVX-NEXT: movq %rbx, %rcx
-; AVX-NEXT: sarq $63, %rcx
-; AVX-NEXT: addb %r11b, %r11b
-; AVX-NEXT: sarb %r11b
-; AVX-NEXT: addb %r8b, %r8b
-; AVX-NEXT: sarb %r8b
-; AVX-NEXT: cmpb %r11b, %r8b
-; AVX-NEXT: setl %dl
-; AVX-NEXT: setg %r8b
-; AVX-NEXT: subb %dl, %r8b
-; AVX-NEXT: movsbq %r8b, %rdx
-; AVX-NEXT: movq %rdx, %r8
-; AVX-NEXT: sarq $63, %r8
-; AVX-NEXT: addb %r10b, %r10b
-; AVX-NEXT: sarb %r10b
-; AVX-NEXT: addb %r9b, %r9b
-; AVX-NEXT: sarb %r9b
-; AVX-NEXT: cmpb %r10b, %r9b
-; AVX-NEXT: setl %r9b
-; AVX-NEXT: setg %r10b
-; AVX-NEXT: subb %r9b, %r10b
-; AVX-NEXT: movsbq %r10b, %r9
-; AVX-NEXT: movq %r9, %r10
-; AVX-NEXT: sarq $63, %r10
-; AVX-NEXT: addb %dil, %dil
-; AVX-NEXT: sarb %dil
-; AVX-NEXT: movzbl {{[0-9]+}}(%rsp), %r11d
-; AVX-NEXT: addb %r11b, %r11b
-; AVX-NEXT: sarb %r11b
-; AVX-NEXT: cmpb %dil, %r11b
-; AVX-NEXT: setl %dil
-; AVX-NEXT: setg %r11b
-; AVX-NEXT: subb %dil, %r11b
-; AVX-NEXT: movsbq %r11b, %rdi
-; AVX-NEXT: movq %rdi, %r11
-; AVX-NEXT: sarq $63, %r11
-; AVX-NEXT: movl %r11d, 96(%rax)
-; AVX-NEXT: movb $51, %bpl
-; AVX-NEXT: bzhiq %rbp, %r11, %rbp
-; AVX-NEXT: shldq $62, %rdi, %r11
-; AVX-NEXT: movq %r11, 88(%rax)
-; AVX-NEXT: movq %r9, %r11
-; AVX-NEXT: shrdq $44, %r10, %r11
-; AVX-NEXT: movq %r11, 64(%rax)
-; AVX-NEXT: movq %rdx, %r11
-; AVX-NEXT: shrdq $33, %r8, %r11
-; AVX-NEXT: movq %r11, 48(%rax)
-; AVX-NEXT: movq %rbx, %r11
-; AVX-NEXT: shrdq $22, %rcx, %r11
-; AVX-NEXT: movq %r11, 32(%rax)
-; AVX-NEXT: movq %r15, %r11
-; AVX-NEXT: shrdq $11, %r13, %r11
-; AVX-NEXT: movq %r11, 16(%rax)
-; AVX-NEXT: movq %rbp, %r11
-; AVX-NEXT: shrq $48, %r11
-; AVX-NEXT: movb %r11b, 102(%rax)
-; AVX-NEXT: shrq $32, %rbp
-; AVX-NEXT: movw %bp, 100(%rax)
-; AVX-NEXT: movb $53, %r11b
-; AVX-NEXT: bzhiq %r11, %r12, %r12
-; AVX-NEXT: shldq $9, %rsi, %r12
-; AVX-NEXT: shlq $62, %rdi
-; AVX-NEXT: orq %r12, %rdi
-; AVX-NEXT: movq %rdi, 80(%rax)
-; AVX-NEXT: movb $42, %dil
-; AVX-NEXT: bzhiq %rdi, %r13, %rdi
-; AVX-NEXT: shlq $42, %rbx
-; AVX-NEXT: orq %rdi, %rbx
-; AVX-NEXT: movq %rbx, 24(%rax)
-; AVX-NEXT: bzhiq %r11, %r14, %rdi
-; AVX-NEXT: shlq $53, %r15
-; AVX-NEXT: orq %rdi, %r15
-; AVX-NEXT: movq %r15, 8(%rax)
-; AVX-NEXT: shlq $9, %rsi
-; AVX-NEXT: andl $511, %r10d # imm = 0x1FF
-; AVX-NEXT: orq %rsi, %r10
-; AVX-NEXT: movq %r10, 72(%rax)
-; AVX-NEXT: shlq $20, %r9
-; AVX-NEXT: andl $1048575, %r8d # imm = 0xFFFFF
-; AVX-NEXT: orq %r9, %r8
-; AVX-NEXT: movq %r8, 56(%rax)
-; AVX-NEXT: shlq $31, %rdx
-; AVX-NEXT: andl $2147483647, %ecx # imm = 0x7FFFFFFF
-; AVX-NEXT: orq %rdx, %rcx
-; AVX-NEXT: movq %rcx, 40(%rax)
-; AVX-NEXT: popq %rbx
-; AVX-NEXT: popq %r12
-; AVX-NEXT: popq %r13
-; AVX-NEXT: popq %r14
-; AVX-NEXT: popq %r15
-; AVX-NEXT: popq %rbp
-; AVX-NEXT: retq
+; AVX2-LABEL: scmp_uncommon_vectors:
+; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %rbp
+; AVX2-NEXT: pushq %r15
+; AVX2-NEXT: pushq %r14
+; AVX2-NEXT: pushq %r13
+; AVX2-NEXT: pushq %r12
+; AVX2-NEXT: pushq %rbx
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: movzbl {{[0-9]+}}(%rsp), %edi
+; AVX2-NEXT: movzbl {{[0-9]+}}(%rsp), %r10d
+; AVX2-NEXT: movzbl {{[0-9]+}}(%rsp), %r11d
+; AVX2-NEXT: movzbl {{[0-9]+}}(%rsp), %ebx
+; AVX2-NEXT: movzbl {{[0-9]+}}(%rsp), %ebp
+; AVX2-NEXT: movzbl {{[0-9]+}}(%rsp), %r15d
+; AVX2-NEXT: movzbl {{[0-9]+}}(%rsp), %r14d
+; AVX2-NEXT: addb %r14b, %r14b
+; AVX2-NEXT: sarb %r14b
+; AVX2-NEXT: addb %sil, %sil
+; AVX2-NEXT: sarb %sil
+; AVX2-NEXT: cmpb %r14b, %sil
+; AVX2-NEXT: setl %sil
+; AVX2-NEXT: setg %r14b
+; AVX2-NEXT: subb %sil, %r14b
+; AVX2-NEXT: movsbq %r14b, %r14
+; AVX2-NEXT: movq %r14, (%rax)
+; AVX2-NEXT: sarq $63, %r14
+; AVX2-NEXT: addb %r15b, %r15b
+; AVX2-NEXT: sarb %r15b
+; AVX2-NEXT: movzbl {{[0-9]+}}(%rsp), %esi
+; AVX2-NEXT: addb %sil, %sil
+; AVX2-NEXT: sarb %sil
+; AVX2-NEXT: cmpb %r15b, %sil
+; AVX2-NEXT: setl %sil
+; AVX2-NEXT: setg %r15b
+; AVX2-NEXT: subb %sil, %r15b
+; AVX2-NEXT: movsbq %r15b, %rsi
+; AVX2-NEXT: movq %rsi, %r12
+; AVX2-NEXT: sarq $63, %r12
+; AVX2-NEXT: addb %bpl, %bpl
+; AVX2-NEXT: sarb %bpl
+; AVX2-NEXT: addb %dl, %dl
+; AVX2-NEXT: sarb %dl
+; AVX2-NEXT: cmpb %bpl, %dl
+; AVX2-NEXT: setl %dl
+; AVX2-NEXT: setg %bpl
+; AVX2-NEXT: subb %dl, %bpl
+; AVX2-NEXT: movsbq %bpl, %r15
+; AVX2-NEXT: movq %r15, %r13
+; AVX2-NEXT: sarq $63, %r13
+; AVX2-NEXT: addb %bl, %bl
+; AVX2-NEXT: sarb %bl
+; AVX2-NEXT: addb %cl, %cl
+; AVX2-NEXT: sarb %cl
+; AVX2-NEXT: cmpb %bl, %cl
+; AVX2-NEXT: setl %cl
+; AVX2-NEXT: setg %dl
+; AVX2-NEXT: subb %cl, %dl
+; AVX2-NEXT: movsbq %dl, %rbx
+; AVX2-NEXT: movq %rbx, %rcx
+; AVX2-NEXT: sarq $63, %rcx
+; AVX2-NEXT: addb %r11b, %r11b
+; AVX2-NEXT: sarb %r11b
+; AVX2-NEXT: addb %r8b, %r8b
+; AVX2-NEXT: sarb %r8b
+; AVX2-NEXT: cmpb %r11b, %r8b
+; AVX2-NEXT: setl %dl
+; AVX2-NEXT: setg %r8b
+; AVX2-NEXT: subb %dl, %r8b
+; AVX2-NEXT: movsbq %r8b, %rdx
+; AVX2-NEXT: movq %rdx, %r8
+; AVX2-NEXT: sarq $63, %r8
+; AVX2-NEXT: addb %r10b, %r10b
+; AVX2-NEXT: sarb %r10b
+; AVX2-NEXT: addb %r9b, %r9b
+; AVX2-NEXT: sarb %r9b
+; AVX2-NEXT: cmpb %r10b, %r9b
+; AVX2-NEXT: setl %r9b
+; AVX2-NEXT: setg %r10b
+; AVX2-NEXT: subb %r9b, %r10b
+; AVX2-NEXT: movsbq %r10b, %r9
+; AVX2-NEXT: movq %r9, %r10
+; AVX2-NEXT: sarq $63, %r10
+; AVX2-NEXT: addb %dil, %dil
+; AVX2-NEXT: sarb %dil
+; AVX2-NEXT: movzbl {{[0-9]+}}(%rsp), %r11d
+; AVX2-NEXT: addb %r11b, %r11b
+; AVX2-NEXT: sarb %r11b
+; AVX2-NEXT: cmpb %dil, %r11b
+; AVX2-NEXT: setl %dil
+; AVX2-NEXT: setg %r11b
+; AVX2-NEXT: subb %dil, %r11b
+; AVX2-NEXT: movsbq %r11b, %rdi
+; AVX2-NEXT: movq %rdi, %r11
+; AVX2-NEXT: sarq $63, %r11
+; AVX2-NEXT: movl %r11d, 96(%rax)
+; AVX2-NEXT: movb $51, %bpl
+; AVX2-NEXT: bzhiq %rbp, %r11, %rbp
+; AVX2-NEXT: shldq $62, %rdi, %r11
+; AVX2-NEXT: movq %r11, 88(%rax)
+; AVX2-NEXT: movq %r9, %r11
+; AVX2-NEXT: shrdq $44, %r10, %r11
+; AVX2-NEXT: movq %r11, 64(%rax)
+; AVX2-NEXT: movq %rdx, %r11
+; AVX2-NEXT: shrdq $33, %r8, %r11
+; AVX2-NEXT: movq %r11, 48(%rax)
+; AVX2-NEXT: movq %rbx, %r11
+; AVX2-NEXT: shrdq $22, %rcx, %r11
+; AVX2-NEXT: movq %r11, 32(%rax)
+; AVX2-NEXT: movq %r15, %r11
+; AVX2-NEXT: shrdq $11, %r13, %r11
+; AVX2-NEXT: movq %r11, 16(%rax)
+; AVX2-NEXT: movq %rbp, %r11
+; AVX2-NEXT: shrq $48, %r11
+; AVX2-NEXT: movb %r11b, 102(%rax)
+; AVX2-NEXT: shrq $32, %rbp
+; AVX2-NEXT: movw %bp, 100(%rax)
+; AVX2-NEXT: movb $53, %r11b
+; AVX2-NEXT: bzhiq %r11, %r12, %r12
+; AVX2-NEXT: shldq $9, %rsi, %r12
+; AVX2-NEXT: shlq $62, %rdi
+; AVX2-NEXT: orq %r12, %rdi
+; AVX2-NEXT: movq %rdi, 80(%rax)
+; AVX2-NEXT: movb $42, %dil
+; AVX2-NEXT: bzhiq %rdi, %r13, %rdi
+; AVX2-NEXT: shlq $42, %rbx
+; AVX2-NEXT: orq %rdi, %rbx
+; AVX2-NEXT: movq %rbx, 24(%rax)
+; AVX2-NEXT: bzhiq %r11, %r14, %rdi
+; AVX2-NEXT: shlq $53, %r15
+; AVX2-NEXT: orq %rdi, %r15
+; AVX2-NEXT: movq %r15, 8(%rax)
+; AVX2-NEXT: shlq $9, %rsi
+; AVX2-NEXT: andl $511, %r10d # imm = 0x1FF
+; AVX2-NEXT: orq %rsi, %r10
+; AVX2-NEXT: movq %r10, 72(%rax)
+; AVX2-NEXT: shlq $20, %r9
+; AVX2-NEXT: andl $1048575, %r8d # imm = 0xFFFFF
+; AVX2-NEXT: orq %r9, %r8
+; AVX2-NEXT: movq %r8, 56(%rax)
+; AVX2-NEXT: shlq $31, %rdx
+; AVX2-NEXT: andl $2147483647, %ecx # imm = 0x7FFFFFFF
+; AVX2-NEXT: orq %rdx, %rcx
+; AVX2-NEXT: movq %rcx, 40(%rax)
+; AVX2-NEXT: popq %rbx
+; AVX2-NEXT: popq %r12
+; AVX2-NEXT: popq %r13
+; AVX2-NEXT: popq %r14
+; AVX2-NEXT: popq %r15
+; AVX2-NEXT: popq %rbp
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: scmp_uncommon_vectors:
+; AVX512: # %bb.0:
+; AVX512-NEXT: pushq %rbp
+; AVX512-NEXT: pushq %r15
+; AVX512-NEXT: pushq %r14
+; AVX512-NEXT: pushq %r13
+; AVX512-NEXT: pushq %r12
+; AVX512-NEXT: pushq %rbx
+; AVX512-NEXT: movq %rdi, %r12
+; AVX512-NEXT: movzbl {{[0-9]+}}(%rsp), %edi
+; AVX512-NEXT: movzbl {{[0-9]+}}(%rsp), %r10d
+; AVX512-NEXT: movzbl {{[0-9]+}}(%rsp), %r11d
+; AVX512-NEXT: movzbl {{[0-9]+}}(%rsp), %ebx
+; AVX512-NEXT: movzbl {{[0-9]+}}(%rsp), %ebp
+; AVX512-NEXT: movzbl {{[0-9]+}}(%rsp), %r15d
+; AVX512-NEXT: movzbl {{[0-9]+}}(%rsp), %r14d
+; AVX512-NEXT: addb %r14b, %r14b
+; AVX512-NEXT: sarb %r14b
+; AVX512-NEXT: addb %sil, %sil
+; AVX512-NEXT: sarb %sil
+; AVX512-NEXT: cmpb %r14b, %sil
+; AVX512-NEXT: setl %sil
+; AVX512-NEXT: setg %r14b
+; AVX512-NEXT: subb %sil, %r14b
+; AVX512-NEXT: movsbq %r14b, %r14
+; AVX512-NEXT: movq %r14, (%r12)
+; AVX512-NEXT: sarq $63, %r14
+; AVX512-NEXT: movq %r14, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT: addb %r15b, %r15b
+; AVX512-NEXT: sarb %r15b
+; AVX512-NEXT: movzbl {{[0-9]+}}(%rsp), %esi
+; AVX512-NEXT: addb %sil, %sil
+; AVX512-NEXT: sarb %sil
+; AVX512-NEXT: cmpb %r15b, %sil
+; AVX512-NEXT: setl %sil
+; AVX512-NEXT: setg %r15b
+; AVX512-NEXT: subb %sil, %r15b
+; AVX512-NEXT: movsbq %r15b, %rsi
+; AVX512-NEXT: movq %rsi, %r14
+; AVX512-NEXT: sarq $63, %r14
+; AVX512-NEXT: addb %bpl, %bpl
+; AVX512-NEXT: sarb %bpl
+; AVX512-NEXT: addb %dl, %dl
+; AVX512-NEXT: sarb %dl
+; AVX512-NEXT: cmpb %bpl, %dl
+; AVX512-NEXT: setl %dl
+; AVX512-NEXT: setg %bpl
+; AVX512-NEXT: subb %dl, %bpl
+; AVX512-NEXT: movsbq %bpl, %r15
+; AVX512-NEXT: movq %r15, %r13
+; AVX512-NEXT: sarq $63, %r13
+; AVX512-NEXT: addb %bl, %bl
+; AVX512-NEXT: sarb %bl
+; AVX512-NEXT: addb %cl, %cl
+; AVX512-NEXT: sarb %cl
+; AVX512-NEXT: cmpb %bl, %cl
+; AVX512-NEXT: setl %cl
+; AVX512-NEXT: setg %dl
+; AVX512-NEXT: subb %cl, %dl
+; AVX512-NEXT: movsbq %dl, %rbx
+; AVX512-NEXT: movq %rbx, %rcx
+; AVX512-NEXT: sarq $63, %rcx
+; AVX512-NEXT: addb %r11b, %r11b
+; AVX512-NEXT: sarb %r11b
+; AVX512-NEXT: addb %r8b, %r8b
+; AVX512-NEXT: sarb %r8b
+; AVX512-NEXT: cmpb %r11b, %r8b
+; AVX512-NEXT: setl %dl
+; AVX512-NEXT: setg %r8b
+; AVX512-NEXT: subb %dl, %r8b
+; AVX512-NEXT: movsbq %r8b, %rdx
+; AVX512-NEXT: movq %rdx, %r8
+; AVX512-NEXT: sarq $63, %r8
+; AVX512-NEXT: addb %r10b, %r10b
+; AVX512-NEXT: sarb %r10b
+; AVX512-NEXT: addb %r9b, %r9b
+; AVX512-NEXT: sarb %r9b
+; AVX512-NEXT: cmpb %r10b, %r9b
+; AVX512-NEXT: setl %r9b
+; AVX512-NEXT: setg %r10b
+; AVX512-NEXT: subb %r9b, %r10b
+; AVX512-NEXT: movsbq %r10b, %r9
+; AVX512-NEXT: movq %r9, %r10
+; AVX512-NEXT: sarq $63, %r10
+; AVX512-NEXT: addb %dil, %dil
+; AVX512-NEXT: sarb %dil
+; AVX512-NEXT: movzbl {{[0-9]+}}(%rsp), %r11d
+; AVX512-NEXT: addb %r11b, %r11b
+; AVX512-NEXT: sarb %r11b
+; AVX512-NEXT: cmpb %dil, %r11b
+; AVX512-NEXT: setl %dil
+; AVX512-NEXT: setg %r11b
+; AVX512-NEXT: subb %dil, %r11b
+; AVX512-NEXT: movsbq %r11b, %rax
+; AVX512-NEXT: movq %rax, %r11
+; AVX512-NEXT: sarq $63, %r11
+; AVX512-NEXT: movl %r11d, 96(%r12)
+; AVX512-NEXT: movb $51, %bpl
+; AVX512-NEXT: bzhiq %rbp, %r11, %rbp
+; AVX512-NEXT: shlq $62, %r11
+; AVX512-NEXT: movq %rax, %rdi
+; AVX512-NEXT: shrq $2, %rdi
+; AVX512-NEXT: orq %r11, %rdi
+; AVX512-NEXT: movq %rdi, 88(%r12)
+; AVX512-NEXT: movq %r9, %rdi
+; AVX512-NEXT: shrdq $44, %r10, %rdi
+; AVX512-NEXT: movq %rdi, 64(%r12)
+; AVX512-NEXT: movq %rdx, %rdi
+; AVX512-NEXT: shrdq $33, %r8, %rdi
+; AVX512-NEXT: movq %rdi, 48(%r12)
+; AVX512-NEXT: movq %rbx, %rdi
+; AVX512-NEXT: shrdq $22, %rcx, %rdi
+; AVX512-NEXT: movq %rdi, 32(%r12)
+; AVX512-NEXT: movq %r15, %rdi
+; AVX512-NEXT: shrdq $11, %r13, %rdi
+; AVX512-NEXT: movq %rdi, 16(%r12)
+; AVX512-NEXT: movq %rbp, %rdi
+; AVX512-NEXT: shrq $48, %rdi
+; AVX512-NEXT: movb %dil, 102(%r12)
+; AVX512-NEXT: shrq $32, %rbp
+; AVX512-NEXT: movw %bp, 100(%r12)
+; AVX512-NEXT: movb $53, %dil
+; AVX512-NEXT: bzhiq %rdi, %r14, %r11
+; AVX512-NEXT: shldq $9, %rsi, %r11
+; AVX512-NEXT: shlq $62, %rax
+; AVX512-NEXT: orq %r11, %rax
+; AVX512-NEXT: movq %rax, 80(%r12)
+; AVX512-NEXT: movb $42, %al
+; AVX512-NEXT: bzhiq %rax, %r13, %rax
+; AVX512-NEXT: shlq $42, %rbx
+; AVX512-NEXT: orq %rax, %rbx
+; AVX512-NEXT: movq %rbx, 24(%r12)
+; AVX512-NEXT: bzhiq %rdi, {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Folded Reload
+; AVX512-NEXT: shlq $53, %r15
+; AVX512-NEXT: orq %rax, %r15
+; AVX512-NEXT: movq %r15, 8(%r12)
+; AVX512-NEXT: shlq $9, %rsi
+; AVX512-NEXT: andl $511, %r10d # imm = 0x1FF
+; AVX512-NEXT: orq %rsi, %r10
+; AVX512-NEXT: movq %r10, 72(%r12)
+; AVX512-NEXT: shlq $20, %r9
+; AVX512-NEXT: andl $1048575, %r8d # imm = 0xFFFFF
+; AVX512-NEXT: orq %r9, %r8
+; AVX512-NEXT: movq %r8, 56(%r12)
+; AVX512-NEXT: shlq $31, %rdx
+; AVX512-NEXT: andl $2147483647, %ecx # imm = 0x7FFFFFFF
+; AVX512-NEXT: orq %rdx, %rcx
+; AVX512-NEXT: movq %rcx, 40(%r12)
+; AVX512-NEXT: movq %r12, %rax
+; AVX512-NEXT: popq %rbx
+; AVX512-NEXT: popq %r12
+; AVX512-NEXT: popq %r13
+; AVX512-NEXT: popq %r14
+; AVX512-NEXT: popq %r15
+; AVX512-NEXT: popq %rbp
+; AVX512-NEXT: retq
;
; X86-LABEL: scmp_uncommon_vectors:
; X86: # %bb.0:
@@ -3004,174 +3168,180 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; X86-NEXT: addb %al, %al
; X86-NEXT: sarb %al
; X86-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT: addb %bl, %bl
+; X86-NEXT: sarb %bl
+; X86-NEXT: movb {{[0-9]+}}(%esp), %ch
+; X86-NEXT: addb %ch, %ch
+; X86-NEXT: sarb %ch
+; X86-NEXT: movb {{[0-9]+}}(%esp), %cl
+; X86-NEXT: addb %cl, %cl
+; X86-NEXT: sarb %cl
+; X86-NEXT: movb {{[0-9]+}}(%esp), %ah
+; X86-NEXT: addb %ah, %ah
+; X86-NEXT: sarb %ah
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: addb %dl, %dl
+; X86-NEXT: sarb %dl
; X86-NEXT: movb {{[0-9]+}}(%esp), %dh
; X86-NEXT: addb %dh, %dh
; X86-NEXT: sarb %dh
-; X86-NEXT: movb {{[0-9]+}}(%esp), %dl
-; X86-NEXT: addb %dl, %dl
-; X86-NEXT: sarb %dl
-; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movb {{[0-9]+}}(%esp), %al
; X86-NEXT: addb %al, %al
; X86-NEXT: sarb %al
-; X86-NEXT: movb {{[0-9]+}}(%esp), %ah
-; X86-NEXT: addb %ah, %ah
-; X86-NEXT: sarb %ah
-; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: addb %cl, %cl
-; X86-NEXT: sarb %cl
-; X86-NEXT: movb {{[0-9]+}}(%esp), %ch
-; X86-NEXT: addb %ch, %ch
-; X86-NEXT: sarb %ch
-; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ebx
-; X86-NEXT: addb %bl, %bl
-; X86-NEXT: sarb %bl
; X86-NEXT: movb {{[0-9]+}}(%esp), %bh
; X86-NEXT: addb %bh, %bh
; X86-NEXT: sarb %bh
-; X86-NEXT: cmpb %bl, %bh
-; X86-NEXT: setl %bl
+; X86-NEXT: cmpb %al, %bh
+; X86-NEXT: setl %al
; X86-NEXT: setg %bh
-; X86-NEXT: subb %bl, %bh
+; X86-NEXT: subb %al, %bh
; X86-NEXT: movsbl %bh, %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: sarl $31, %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: andl $2097151, %esi # imm = 0x1FFFFF
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: cmpb %cl, %ch
-; X86-NEXT: setl %cl
-; X86-NEXT: setg %ch
-; X86-NEXT: subb %cl, %ch
-; X86-NEXT: movsbl %ch, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: sarl $31, %esi
-; X86-NEXT: movl %esi, %ecx
-; X86-NEXT: movl %esi, %ebx
+; X86-NEXT: cmpb %dl, %dh
+; X86-NEXT: setl %al
+; X86-NEXT: setg %dl
+; X86-NEXT: subb %al, %dl
+; X86-NEXT: movsbl %dl, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sarl $31, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %edx, %esi
+; X86-NEXT: andl $2097151, %esi # imm = 0x1FFFFF
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: andl $2097151, %ecx # imm = 0x1FFFFF
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: cmpb %al, %ah
+; X86-NEXT: cmpb %cl, %ah
; X86-NEXT: setl %al
; X86-NEXT: setg %cl
; X86-NEXT: subb %al, %cl
-; X86-NEXT: movsbl %cl, %ecx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl %ecx, (%eax)
-; X86-NEXT: sarl $31, %ecx
-; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: movsbl %cl, %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movl %eax, (%edx)
+; X86-NEXT: sarl $31, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: andl $2097151, %eax # imm = 0x1FFFFF
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: cmpb %dh, %dl
-; X86-NEXT: setl %al
-; X86-NEXT: setg %dl
-; X86-NEXT: subb %al, %dl
-; X86-NEXT: movsbl %dl, %ebp
-; X86-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: sarl $31, %ebp
+; X86-NEXT: cmpb %bl, %ch
+; X86-NEXT: setl %cl
+; X86-NEXT: setg %ch
+; X86-NEXT: subb %cl, %ch
+; X86-NEXT: movsbl %ch, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sarl $31, %ecx
; X86-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
; X86-NEXT: cmpb {{[-0-9]+}}(%e{{[sb]}}p), %al # 1-byte Folded Reload
-; X86-NEXT: setl %al
-; X86-NEXT: setg %dl
-; X86-NEXT: subb %al, %dl
-; X86-NEXT: movsbl %dl, %edi
+; X86-NEXT: setl %bl
+; X86-NEXT: setg %bh
+; X86-NEXT: subb %bl, %bh
+; X86-NEXT: movsbl %bh, %edi
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: sarl $31, %edi
-; X86-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X86-NEXT: cmpb {{[-0-9]+}}(%e{{[sb]}}p), %al # 1-byte Folded Reload
-; X86-NEXT: setl %al
-; X86-NEXT: setg %dl
-; X86-NEXT: subb %al, %dl
-; X86-NEXT: movsbl %dl, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: sarl $31, %eax
-; X86-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 1-byte Folded Reload
-; X86-NEXT: cmpb {{[-0-9]+}}(%e{{[sb]}}p), %dl # 1-byte Folded Reload
-; X86-NEXT: setl %dl
-; X86-NEXT: setg %dh
-; X86-NEXT: subb %dl, %dh
-; X86-NEXT: movsbl %dh, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: sarl $31, %edx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT: movl %edx, 96(%esi)
-; X86-NEXT: movl %edx, 92(%esi)
-; X86-NEXT: movl %ebx, 80(%esi)
-; X86-NEXT: movl %eax, 68(%esi)
-; X86-NEXT: movl %eax, 64(%esi)
-; X86-NEXT: movl %edi, 52(%esi)
-; X86-NEXT: movl %edi, 48(%esi)
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: movl %ebx, 36(%esi)
-; X86-NEXT: movl %ebp, 24(%esi)
-; X86-NEXT: movl %ebp, 20(%esi)
-; X86-NEXT: movl %ecx, 8(%esi)
-; X86-NEXT: movl %ecx, 4(%esi)
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movw %dx, 100(%esi)
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: shrdl $2, %edx, %ecx
-; X86-NEXT: movl %ecx, 88(%esi)
-; X86-NEXT: movl %esi, %edx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: shldl $9, %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 1-byte Folded Reload
+; X86-NEXT: cmpb {{[-0-9]+}}(%e{{[sb]}}p), %bl # 1-byte Folded Reload
+; X86-NEXT: setl %bl
+; X86-NEXT: setg %bh
+; X86-NEXT: subb %bl, %bh
+; X86-NEXT: movsbl %bh, %ebp
+; X86-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sarl $31, %ebp
+; X86-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 1-byte Folded Reload
+; X86-NEXT: cmpb {{[-0-9]+}}(%e{{[sb]}}p), %bl # 1-byte Folded Reload
+; X86-NEXT: setl %bl
+; X86-NEXT: setg %bh
+; X86-NEXT: subb %bl, %bh
+; X86-NEXT: movsbl %bh, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sarl $31, %ebx
+; X86-NEXT: movl %edx, %eax
+; X86-NEXT: movl %ebx, 96(%edx)
+; X86-NEXT: movl %ebx, 92(%edx)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: shldl $9, %esi, %ecx
-; X86-NEXT: movl %edx, %esi
-; X86-NEXT: movl %ecx, 76(%edx)
-; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: movl %esi, 80(%edx)
+; X86-NEXT: movl %ebp, 68(%edx)
+; X86-NEXT: movl %ebp, 64(%edx)
+; X86-NEXT: movl %edi, 52(%edx)
+; X86-NEXT: movl %edi, 48(%edx)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: shldl $20, %edx, %ecx
-; X86-NEXT: movl %ecx, 60(%esi)
-; X86-NEXT: movl %esi, %ebx
-; X86-NEXT: movl %edi, %ecx
+; X86-NEXT: movl %edx, 36(%eax)
+; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: movl %ecx, 24(%eax)
+; X86-NEXT: movl %ecx, 20(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %ecx, 8(%eax)
+; X86-NEXT: movl %ecx, 4(%eax)
+; X86-NEXT: movw %bx, 100(%eax)
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: shrdl $2, %ebx, %eax
+; X86-NEXT: movl %eax, 88(%ecx)
+; X86-NEXT: movl %esi, %eax
+; X86-NEXT: shll $9, %eax
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: shldl $31, %esi, %ecx
-; X86-NEXT: movl %ecx, 44(%ebx)
-; X86-NEXT: movl %ebx, %edx
+; X86-NEXT: shrl $23, %esi
+; X86-NEXT: orl %eax, %esi
+; X86-NEXT: movl %esi, 76(%ecx)
+; X86-NEXT: movl %ebp, %eax
+; X86-NEXT: shll $20, %eax
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: movl %esi, %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: shrdl $22, %ebx, %ecx
-; X86-NEXT: movl %ecx, 32(%edx)
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: movl %ebx, %ecx
-; X86-NEXT: shrdl $11, %ebp, %ecx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl %ecx, 16(%edx)
+; X86-NEXT: shrl $12, %esi
+; X86-NEXT: orl %eax, %esi
+; X86-NEXT: movl %esi, 60(%ecx)
+; X86-NEXT: movl %edi, %eax
+; X86-NEXT: shll $31, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: shrl %esi
+; X86-NEXT: orl %eax, %esi
+; X86-NEXT: movl %esi, 44(%ecx)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: movl %esi, %eax
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: shll $9, %ecx
-; X86-NEXT: andl $511, %eax # imm = 0x1FF
-; X86-NEXT: orl %ecx, %eax
+; X86-NEXT: shrdl $22, %ecx, %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %eax, 72(%ecx)
+; X86-NEXT: movl %eax, 32(%ecx)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: shrdl $11, %edx, %eax
+; X86-NEXT: movl %eax, 16(%ecx)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: shll $9, %eax
+; X86-NEXT: andl $511, %ebp # imm = 0x1FF
+; X86-NEXT: orl %eax, %ebp
+; X86-NEXT: movl %ebp, 72(%ecx)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: shll $20, %eax
; X86-NEXT: andl $1048575, %edi # imm = 0xFFFFF
; X86-NEXT: orl %eax, %edi
; X86-NEXT: movl %edi, 56(%ecx)
; X86-NEXT: shll $10, %esi
-; X86-NEXT: andl $1023, %ebp # imm = 0x3FF
-; X86-NEXT: orl %esi, %ebp
-; X86-NEXT: movl %ebp, 28(%ecx)
-; X86-NEXT: movl %ebx, %eax
+; X86-NEXT: andl $1023, %edx # imm = 0x3FF
+; X86-NEXT: orl %esi, %edx
+; X86-NEXT: movl %edx, 28(%ecx)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: shll $21, %eax
; X86-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
; X86-NEXT: movl %eax, 12(%ecx)
+; X86-NEXT: andl $7, %ebx
+; X86-NEXT: movb %bl, 102(%ecx)
+; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: shll $9, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: andl $7, %eax
-; X86-NEXT: movb %al, 102(%ecx)
+; X86-NEXT: shrl $23, %eax
+; X86-NEXT: orl %ecx, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: shll $30, %ecx
+; X86-NEXT: orl %eax, %ecx
+; X86-NEXT: movl %ecx, 84(%edx)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shll $30, %eax
-; X86-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NEXT: movl %eax, 84(%ecx)
-; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: shldl $10, %ecx, %edx
+; X86-NEXT: shldl $10, %ecx, %eax
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: shll $31, %ecx
-; X86-NEXT: orl %edx, %ecx
-; X86-NEXT: movl %ecx, 40(%eax)
+; X86-NEXT: orl %eax, %ecx
+; X86-NEXT: movl %ecx, 40(%edx)
+; X86-NEXT: movl %edx, %eax
; X86-NEXT: addl $52, %esp
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
@@ -3187,8 +3357,8 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; SETZUCC-NEXT: pushq %r13
; SETZUCC-NEXT: pushq %r12
; SETZUCC-NEXT: pushq %rbx
-; SETZUCC-NEXT: movq %rdi, %rax
-; SETZUCC-NEXT: movzbl {{[0-9]+}}(%rsp), %edi
+; SETZUCC-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SETZUCC-NEXT: movzbl {{[0-9]+}}(%rsp), %eax
; SETZUCC-NEXT: movzbl {{[0-9]+}}(%rsp), %r10d
; SETZUCC-NEXT: movzbl {{[0-9]+}}(%rsp), %r11d
; SETZUCC-NEXT: movzbl {{[0-9]+}}(%rsp), %ebx
@@ -3204,9 +3374,10 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; SETZUCC-NEXT: setzug %r15b
; SETZUCC-NEXT: subb %sil, %r15b
; SETZUCC-NEXT: movsbq %r15b, %rsi
-; SETZUCC-NEXT: movq %rsi, (%rax)
+; SETZUCC-NEXT: movq %rsi, (%rdi)
; SETZUCC-NEXT: movq %rsi, %xmm0
; SETZUCC-NEXT: sarq $63, %rsi
+; SETZUCC-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; SETZUCC-NEXT: addb %r14b, %r14b
; SETZUCC-NEXT: sarb %r14b
; SETZUCC-NEXT: movzbl {{[0-9]+}}(%rsp), %r15d
@@ -3227,9 +3398,9 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; SETZUCC-NEXT: setzul %dl
; SETZUCC-NEXT: setzug %bpl
; SETZUCC-NEXT: subb %dl, %bpl
-; SETZUCC-NEXT: movsbq %bpl, %rdx
-; SETZUCC-NEXT: movq %rdx, %r12
-; SETZUCC-NEXT: sarq $63, %r12
+; SETZUCC-NEXT: movsbq %bpl, %r12
+; SETZUCC-NEXT: movq %r12, %rsi
+; SETZUCC-NEXT: sarq $63, %rsi
; SETZUCC-NEXT: addb %bl, %bl
; SETZUCC-NEXT: sarb %bl
; SETZUCC-NEXT: addb %cl, %cl
@@ -3238,9 +3409,9 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; SETZUCC-NEXT: setzul %cl
; SETZUCC-NEXT: setzug %bl
; SETZUCC-NEXT: subb %cl, %bl
-; SETZUCC-NEXT: movsbq %bl, %rbx
-; SETZUCC-NEXT: movq %rbx, %rcx
-; SETZUCC-NEXT: sarq $63, %rcx
+; SETZUCC-NEXT: movsbq %bl, %rdx
+; SETZUCC-NEXT: movq %rdx, %rbx
+; SETZUCC-NEXT: sarq $63, %rbx
; SETZUCC-NEXT: addb %r11b, %r11b
; SETZUCC-NEXT: sarb %r11b
; SETZUCC-NEXT: addb %r8b, %r8b
@@ -3263,72 +3434,78 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; SETZUCC-NEXT: movsbq %r10b, %r9
; SETZUCC-NEXT: movq %r9, %r10
; SETZUCC-NEXT: sarq $63, %r10
-; SETZUCC-NEXT: addb %dil, %dil
-; SETZUCC-NEXT: sarb %dil
+; SETZUCC-NEXT: addb %al, %al
+; SETZUCC-NEXT: sarb %al
; SETZUCC-NEXT: movzbl {{[0-9]+}}(%rsp), %ebp
; SETZUCC-NEXT: addb %bpl, %bpl
; SETZUCC-NEXT: sarb %bpl
-; SETZUCC-NEXT: cmpb %dil, %bpl
+; SETZUCC-NEXT: cmpb %al, %bpl
; SETZUCC-NEXT: setzul %dil
; SETZUCC-NEXT: setzug %bpl
; SETZUCC-NEXT: subb %dil, %bpl
-; SETZUCC-NEXT: movsbq %bpl, %rdi
-; SETZUCC-NEXT: movq %rdi, %r13
+; SETZUCC-NEXT: movsbq %bpl, %rax
+; SETZUCC-NEXT: movq %rax, %r13
; SETZUCC-NEXT: sarq $63, %r13
-; SETZUCC-NEXT: movl %r13d, 96(%rax)
+; SETZUCC-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; SETZUCC-NEXT: movl %r13d, 96(%rcx)
; SETZUCC-NEXT: movabsq $2251799813685247, %rbp # imm = 0x7FFFFFFFFFFFF
; SETZUCC-NEXT: andq %r13, %rbp
-; SETZUCC-NEXT: shldq $62, %rdi, %r13
-; SETZUCC-NEXT: movq %r13, 88(%rax)
-; SETZUCC-NEXT: movq %r9, %r13
-; SETZUCC-NEXT: shrdq $44, %r10, %r13
-; SETZUCC-NEXT: movq %r13, 64(%rax)
-; SETZUCC-NEXT: movq %r8, %r13
-; SETZUCC-NEXT: shrdq $33, %r11, %r13
-; SETZUCC-NEXT: movq %r13, 48(%rax)
-; SETZUCC-NEXT: movq %rbx, %r13
-; SETZUCC-NEXT: shrdq $22, %rcx, %r13
-; SETZUCC-NEXT: movq %r13, 32(%rax)
-; SETZUCC-NEXT: movq %rdx, %r13
-; SETZUCC-NEXT: shrdq $11, %r12, %r13
-; SETZUCC-NEXT: movq %r13, 16(%rax)
-; SETZUCC-NEXT: movq %rbp, %r13
-; SETZUCC-NEXT: shrq $48, %r13
-; SETZUCC-NEXT: movb %r13b, 102(%rax)
+; SETZUCC-NEXT: shlq $62, %r13
+; SETZUCC-NEXT: movq %rax, %rdi
+; SETZUCC-NEXT: shrq $2, %rdi
+; SETZUCC-NEXT: orq %r13, %rdi
+; SETZUCC-NEXT: movq %rdi, 88(%rcx)
+; SETZUCC-NEXT: movq %r9, %rdi
+; SETZUCC-NEXT: shrdq $44, %r10, %rdi
+; SETZUCC-NEXT: movq %rdi, 64(%rcx)
+; SETZUCC-NEXT: movq %r8, %rdi
+; SETZUCC-NEXT: shrdq $33, %r11, %rdi
+; SETZUCC-NEXT: movq %rdi, 48(%rcx)
+; SETZUCC-NEXT: movq %rdx, %rdi
+; SETZUCC-NEXT: shrdq $22, %rbx, %rdi
+; SETZUCC-NEXT: movq %rdi, 32(%rcx)
+; SETZUCC-NEXT: movq %r12, %rdi
+; SETZUCC-NEXT: shrdq $11, %rsi, %rdi
+; SETZUCC-NEXT: movq %rdi, 16(%rcx)
+; SETZUCC-NEXT: movq %rbp, %rdi
+; SETZUCC-NEXT: shrq $48, %rdi
+; SETZUCC-NEXT: movb %dil, 102(%rcx)
; SETZUCC-NEXT: shrq $32, %rbp
-; SETZUCC-NEXT: movw %bp, 100(%rax)
-; SETZUCC-NEXT: movabsq $9007199254740991, %r13 # imm = 0x1FFFFFFFFFFFFF
-; SETZUCC-NEXT: andq %r13, %r15
+; SETZUCC-NEXT: movw %bp, 100(%rcx)
+; SETZUCC-NEXT: movabsq $9007199254740991, %rdi # imm = 0x1FFFFFFFFFFFFF
+; SETZUCC-NEXT: andq %rdi, %r15
; SETZUCC-NEXT: shldq $9, %r14, %r15
-; SETZUCC-NEXT: shlq $62, %rdi
-; SETZUCC-NEXT: orq %r15, %rdi
-; SETZUCC-NEXT: movq %rdi, 80(%rax)
-; SETZUCC-NEXT: movabsq $9007199254738944, %rdi # imm = 0x1FFFFFFFFFF800
-; SETZUCC-NEXT: andq %r12, %rdi
-; SETZUCC-NEXT: shlq $42, %rbx
-; SETZUCC-NEXT: shrq $11, %rdi
-; SETZUCC-NEXT: orq %rbx, %rdi
-; SETZUCC-NEXT: movq %rdi, 24(%rax)
+; SETZUCC-NEXT: shlq $62, %rax
+; SETZUCC-NEXT: orq %r15, %rax
+; SETZUCC-NEXT: movq %rax, 80(%rcx)
+; SETZUCC-NEXT: movabsq $9007199254738944, %rax # imm = 0x1FFFFFFFFFF800
+; SETZUCC-NEXT: andq %rsi, %rax
+; SETZUCC-NEXT: shlq $42, %rdx
+; SETZUCC-NEXT: shrq $11, %rax
+; SETZUCC-NEXT: orq %rdx, %rax
+; SETZUCC-NEXT: movq %rax, 24(%rcx)
; SETZUCC-NEXT: shlq $9, %r14
; SETZUCC-NEXT: andl $511, %r10d # imm = 0x1FF
; SETZUCC-NEXT: orq %r14, %r10
-; SETZUCC-NEXT: movq %r10, 72(%rax)
+; SETZUCC-NEXT: movq %r10, 72(%rcx)
; SETZUCC-NEXT: shlq $20, %r9
; SETZUCC-NEXT: andl $1048575, %r11d # imm = 0xFFFFF
; SETZUCC-NEXT: orq %r9, %r11
-; SETZUCC-NEXT: movq %r11, 56(%rax)
+; SETZUCC-NEXT: movq %r11, 56(%rcx)
; SETZUCC-NEXT: shlq $31, %r8
-; SETZUCC-NEXT: andl $2147483647, %ecx # imm = 0x7FFFFFFF
-; SETZUCC-NEXT: orq %r8, %rcx
-; SETZUCC-NEXT: movq %rcx, 40(%rax)
-; SETZUCC-NEXT: movq %rsi, %xmm1
+; SETZUCC-NEXT: andl $2147483647, %ebx # imm = 0x7FFFFFFF
+; SETZUCC-NEXT: orq %r8, %rbx
+; SETZUCC-NEXT: movq %rbx, 40(%rcx)
+; SETZUCC-NEXT: movq %rcx, %rax
+; SETZUCC-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Folded Reload
+; SETZUCC-NEXT: # xmm1 = mem[0],zero
; SETZUCC-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; SETZUCC-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
; SETZUCC-NEXT: movq %xmm0, %rcx
-; SETZUCC-NEXT: andq %r13, %rcx
-; SETZUCC-NEXT: shlq $53, %rdx
-; SETZUCC-NEXT: orq %rcx, %rdx
-; SETZUCC-NEXT: movq %rdx, 8(%rax)
+; SETZUCC-NEXT: andq %rdi, %rcx
+; SETZUCC-NEXT: shlq $53, %r12
+; SETZUCC-NEXT: orq %rcx, %r12
+; SETZUCC-NEXT: movq %r12, 8(%rax)
; SETZUCC-NEXT: popq %rbx
; SETZUCC-NEXT: popq %r12
; SETZUCC-NEXT: popq %r13
@@ -3345,8 +3522,8 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; NO-SETZUCC-NEXT: pushq %r13
; NO-SETZUCC-NEXT: pushq %r12
; NO-SETZUCC-NEXT: pushq %rbx
-; NO-SETZUCC-NEXT: movq %rdi, %rax
-; NO-SETZUCC-NEXT: movzbl {{[0-9]+}}(%rsp), %edi
+; NO-SETZUCC-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; NO-SETZUCC-NEXT: movzbl {{[0-9]+}}(%rsp), %eax
; NO-SETZUCC-NEXT: movzbl {{[0-9]+}}(%rsp), %r10d
; NO-SETZUCC-NEXT: movzbl {{[0-9]+}}(%rsp), %r11d
; NO-SETZUCC-NEXT: movzbl {{[0-9]+}}(%rsp), %ebx
@@ -3362,9 +3539,10 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; NO-SETZUCC-NEXT: setg %r15b
; NO-SETZUCC-NEXT: subb %sil, %r15b
; NO-SETZUCC-NEXT: movsbq %r15b, %rsi
-; NO-SETZUCC-NEXT: movq %rsi, (%rax)
+; NO-SETZUCC-NEXT: movq %rsi, (%rdi)
; NO-SETZUCC-NEXT: movq %rsi, %xmm0
; NO-SETZUCC-NEXT: sarq $63, %rsi
+; NO-SETZUCC-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; NO-SETZUCC-NEXT: addb %r14b, %r14b
; NO-SETZUCC-NEXT: sarb %r14b
; NO-SETZUCC-NEXT: movzbl {{[0-9]+}}(%rsp), %r15d
@@ -3385,9 +3563,9 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; NO-SETZUCC-NEXT: setl %dl
; NO-SETZUCC-NEXT: setg %bpl
; NO-SETZUCC-NEXT: subb %dl, %bpl
-; NO-SETZUCC-NEXT: movsbq %bpl, %rdx
-; NO-SETZUCC-NEXT: movq %rdx, %r12
-; NO-SETZUCC-NEXT: sarq $63, %r12
+; NO-SETZUCC-NEXT: movsbq %bpl, %r12
+; NO-SETZUCC-NEXT: movq %r12, %rsi
+; NO-SETZUCC-NEXT: sarq $63, %rsi
; NO-SETZUCC-NEXT: addb %bl, %bl
; NO-SETZUCC-NEXT: sarb %bl
; NO-SETZUCC-NEXT: addb %cl, %cl
@@ -3396,9 +3574,9 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; NO-SETZUCC-NEXT: setl %cl
; NO-SETZUCC-NEXT: setg %bl
; NO-SETZUCC-NEXT: subb %cl, %bl
-; NO-SETZUCC-NEXT: movsbq %bl, %rbx
-; NO-SETZUCC-NEXT: movq %rbx, %rcx
-; NO-SETZUCC-NEXT: sarq $63, %rcx
+; NO-SETZUCC-NEXT: movsbq %bl, %rdx
+; NO-SETZUCC-NEXT: movq %rdx, %rbx
+; NO-SETZUCC-NEXT: sarq $63, %rbx
; NO-SETZUCC-NEXT: addb %r11b, %r11b
; NO-SETZUCC-NEXT: sarb %r11b
; NO-SETZUCC-NEXT: addb %r8b, %r8b
@@ -3421,72 +3599,78 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; NO-SETZUCC-NEXT: movsbq %r10b, %r9
; NO-SETZUCC-NEXT: movq %r9, %r10
; NO-SETZUCC-NEXT: sarq $63, %r10
-; NO-SETZUCC-NEXT: addb %dil, %dil
-; NO-SETZUCC-NEXT: sarb %dil
+; NO-SETZUCC-NEXT: addb %al, %al
+; NO-SETZUCC-NEXT: sarb %al
; NO-SETZUCC-NEXT: movzbl {{[0-9]+}}(%rsp), %ebp
; NO-SETZUCC-NEXT: addb %bpl, %bpl
; NO-SETZUCC-NEXT: sarb %bpl
-; NO-SETZUCC-NEXT: cmpb %dil, %bpl
+; NO-SETZUCC-NEXT: cmpb %al, %bpl
; NO-SETZUCC-NEXT: setl %dil
; NO-SETZUCC-NEXT: setg %bpl
; NO-SETZUCC-NEXT: subb %dil, %bpl
-; NO-SETZUCC-NEXT: movsbq %bpl, %rdi
-; NO-SETZUCC-NEXT: movq %rdi, %r13
+; NO-SETZUCC-NEXT: movsbq %bpl, %rax
+; NO-SETZUCC-NEXT: movq %rax, %r13
; NO-SETZUCC-NEXT: sarq $63, %r13
-; NO-SETZUCC-NEXT: movl %r13d, 96(%rax)
+; NO-SETZUCC-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; NO-SETZUCC-NEXT: movl %r13d, 96(%rcx)
; NO-SETZUCC-NEXT: movabsq $2251799813685247, %rbp # imm = 0x7FFFFFFFFFFFF
; NO-SETZUCC-NEXT: andq %r13, %rbp
-; NO-SETZUCC-NEXT: shldq $62, %rdi, %r13
-; NO-SETZUCC-NEXT: movq %r13, 88(%rax)
-; NO-SETZUCC-NEXT: movq %r9, %r13
-; NO-SETZUCC-NEXT: shrdq $44, %r10, %r13
-; NO-SETZUCC-NEXT: movq %r13, 64(%rax)
-; NO-SETZUCC-NEXT: movq %r8, %r13
-; NO-SETZUCC-NEXT: shrdq $33, %r11, %r13
-; NO-SETZUCC-NEXT: movq %r13, 48(%rax)
-; NO-SETZUCC-NEXT: movq %rbx, %r13
-; NO-SETZUCC-NEXT: shrdq $22, %rcx, %r13
-; NO-SETZUCC-NEXT: movq %r13, 32(%rax)
-; NO-SETZUCC-NEXT: movq %rdx, %r13
-; NO-SETZUCC-NEXT: shrdq $11, %r12, %r13
-; NO-SETZUCC-NEXT: movq %r13, 16(%rax)
-; NO-SETZUCC-NEXT: movq %rbp, %r13
-; NO-SETZUCC-NEXT: shrq $48, %r13
-; NO-SETZUCC-NEXT: movb %r13b, 102(%rax)
+; NO-SETZUCC-NEXT: shlq $62, %r13
+; NO-SETZUCC-NEXT: movq %rax, %rdi
+; NO-SETZUCC-NEXT: shrq $2, %rdi
+; NO-SETZUCC-NEXT: orq %r13, %rdi
+; NO-SETZUCC-NEXT: movq %rdi, 88(%rcx)
+; NO-SETZUCC-NEXT: movq %r9, %rdi
+; NO-SETZUCC-NEXT: shrdq $44, %r10, %rdi
+; NO-SETZUCC-NEXT: movq %rdi, 64(%rcx)
+; NO-SETZUCC-NEXT: movq %r8, %rdi
+; NO-SETZUCC-NEXT: shrdq $33, %r11, %rdi
+; NO-SETZUCC-NEXT: movq %rdi, 48(%rcx)
+; NO-SETZUCC-NEXT: movq %rdx, %rdi
+; NO-SETZUCC-NEXT: shrdq $22, %rbx, %rdi
+; NO-SETZUCC-NEXT: movq %rdi, 32(%rcx)
+; NO-SETZUCC-NEXT: movq %r12, %rdi
+; NO-SETZUCC-NEXT: shrdq $11, %rsi, %rdi
+; NO-SETZUCC-NEXT: movq %rdi, 16(%rcx)
+; NO-SETZUCC-NEXT: movq %rbp, %rdi
+; NO-SETZUCC-NEXT: shrq $48, %rdi
+; NO-SETZUCC-NEXT: movb %dil, 102(%rcx)
; NO-SETZUCC-NEXT: shrq $32, %rbp
-; NO-SETZUCC-NEXT: movw %bp, 100(%rax)
-; NO-SETZUCC-NEXT: movabsq $9007199254740991, %r13 # imm = 0x1FFFFFFFFFFFFF
-; NO-SETZUCC-NEXT: andq %r13, %r15
+; NO-SETZUCC-NEXT: movw %bp, 100(%rcx)
+; NO-SETZUCC-NEXT: movabsq $9007199254740991, %rdi # imm = 0x1FFFFFFFFFFFFF
+; NO-SETZUCC-NEXT: andq %rdi, %r15
; NO-SETZUCC-NEXT: shldq $9, %r14, %r15
-; NO-SETZUCC-NEXT: shlq $62, %rdi
-; NO-SETZUCC-NEXT: orq %r15, %rdi
-; NO-SETZUCC-NEXT: movq %rdi, 80(%rax)
-; NO-SETZUCC-NEXT: movabsq $9007199254738944, %rdi # imm = 0x1FFFFFFFFFF800
-; NO-SETZUCC-NEXT: andq %r12, %rdi
-; NO-SETZUCC-NEXT: shlq $42, %rbx
-; NO-SETZUCC-NEXT: shrq $11, %rdi
-; NO-SETZUCC-NEXT: orq %rbx, %rdi
-; NO-SETZUCC-NEXT: movq %rdi, 24(%rax)
+; NO-SETZUCC-NEXT: shlq $62, %rax
+; NO-SETZUCC-NEXT: orq %r15, %rax
+; NO-SETZUCC-NEXT: movq %rax, 80(%rcx)
+; NO-SETZUCC-NEXT: movabsq $9007199254738944, %rax # imm = 0x1FFFFFFFFFF800
+; NO-SETZUCC-NEXT: andq %rsi, %rax
+; NO-SETZUCC-NEXT: shlq $42, %rdx
+; NO-SETZUCC-NEXT: shrq $11, %rax
+; NO-SETZUCC-NEXT: orq %rdx, %rax
+; NO-SETZUCC-NEXT: movq %rax, 24(%rcx)
; NO-SETZUCC-NEXT: shlq $9, %r14
; NO-SETZUCC-NEXT: andl $511, %r10d # imm = 0x1FF
; NO-SETZUCC-NEXT: orq %r14, %r10
-; NO-SETZUCC-NEXT: movq %r10, 72(%rax)
+; NO-SETZUCC-NEXT: movq %r10, 72(%rcx)
; NO-SETZUCC-NEXT: shlq $20, %r9
; NO-SETZUCC-NEXT: andl $1048575, %r11d # imm = 0xFFFFF
; NO-SETZUCC-NEXT: orq %r9, %r11
-; NO-SETZUCC-NEXT: movq %r11, 56(%rax)
+; NO-SETZUCC-NEXT: movq %r11, 56(%rcx)
; NO-SETZUCC-NEXT: shlq $31, %r8
-; NO-SETZUCC-NEXT: andl $2147483647, %ecx # imm = 0x7FFFFFFF
-; NO-SETZUCC-NEXT: orq %r8, %rcx
-; NO-SETZUCC-NEXT: movq %rcx, 40(%rax)
-; NO-SETZUCC-NEXT: movq %rsi, %xmm1
+; NO-SETZUCC-NEXT: andl $2147483647, %ebx # imm = 0x7FFFFFFF
+; NO-SETZUCC-NEXT: orq %r8, %rbx
+; NO-SETZUCC-NEXT: movq %rbx, 40(%rcx)
+; NO-SETZUCC-NEXT: movq %rcx, %rax
+; NO-SETZUCC-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Folded Reload
+; NO-SETZUCC-NEXT: # xmm1 = mem[0],zero
; NO-SETZUCC-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; NO-SETZUCC-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
; NO-SETZUCC-NEXT: movq %xmm0, %rcx
-; NO-SETZUCC-NEXT: andq %r13, %rcx
-; NO-SETZUCC-NEXT: shlq $53, %rdx
-; NO-SETZUCC-NEXT: orq %rcx, %rdx
-; NO-SETZUCC-NEXT: movq %rdx, 8(%rax)
+; NO-SETZUCC-NEXT: andq %rdi, %rcx
+; NO-SETZUCC-NEXT: shlq $53, %r12
+; NO-SETZUCC-NEXT: orq %rcx, %r12
+; NO-SETZUCC-NEXT: movq %r12, 8(%rax)
; NO-SETZUCC-NEXT: popq %rbx
; NO-SETZUCC-NEXT: popq %r12
; NO-SETZUCC-NEXT: popq %r13
diff --git a/llvm/test/CodeGen/X86/sdiv_fix.ll b/llvm/test/CodeGen/X86/sdiv_fix.ll
index 392bc83d9d5d8..f3e441fc5b602 100644
--- a/llvm/test/CodeGen/X86/sdiv_fix.ll
+++ b/llvm/test/CodeGen/X86/sdiv_fix.ll
@@ -262,9 +262,12 @@ define i64 @func5(i64 %x, i64 %y) nounwind {
; X64-NEXT: pushq %rax
; X64-NEXT: movq %rsi, %rbx
; X64-NEXT: movq %rdi, %r14
+; X64-NEXT: movq %rdi, %rax
+; X64-NEXT: sarq $63, %rax
+; X64-NEXT: shlq $31, %rax
; X64-NEXT: movq %rdi, %r15
-; X64-NEXT: sarq $63, %r15
-; X64-NEXT: shldq $31, %rdi, %r15
+; X64-NEXT: shrq $33, %r15
+; X64-NEXT: orq %rax, %r15
; X64-NEXT: shlq $31, %r14
; X64-NEXT: movq %rsi, %r12
; X64-NEXT: sarq $63, %r12
@@ -309,23 +312,31 @@ define i64 @func5(i64 %x, i64 %y) nounwind {
; X86-NEXT: andl $-16, %esp
; X86-NEXT: subl $112, %esp
; X86-NEXT: movl 8(%ebp), %ecx
-; X86-NEXT: movl 12(%ebp), %edi
-; X86-NEXT: movl 16(%ebp), %eax
-; X86-NEXT: movl 20(%ebp), %edx
-; X86-NEXT: movl %edx, %ebx
+; X86-NEXT: movl 12(%ebp), %esi
+; X86-NEXT: movl 20(%ebp), %ebx
+; X86-NEXT: movl %esi, %eax
+; X86-NEXT: sarl $31, %eax
+; X86-NEXT: movl %eax, %edx
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %esi, %eax
+; X86-NEXT: shll $31, %eax
+; X86-NEXT: movl %ecx, %edi
+; X86-NEXT: shrl %edi
+; X86-NEXT: orl %eax, %edi
+; X86-NEXT: leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %eax, (%esp)
+; X86-NEXT: movl %edx, %eax
+; X86-NEXT: shll $31, %eax
+; X86-NEXT: shrl %esi
+; X86-NEXT: orl %eax, %esi
+; X86-NEXT: movl %ebx, %eax
; X86-NEXT: sarl $31, %ebx
; X86-NEXT: movl %ebx, {{[0-9]+}}(%esp)
; X86-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl %edi, %esi
-; X86-NEXT: sarl $31, %esi
-; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: leal {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl %eax, (%esp)
-; X86-NEXT: shldl $31, %edi, %esi
-; X86-NEXT: shldl $31, %ecx, %edi
+; X86-NEXT: movl 16(%ebp), %eax
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X86-NEXT: shll $31, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
@@ -537,115 +548,126 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
; X86-NEXT: pushl %esi
; X86-NEXT: subl $60, %esp
; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ebp
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT: movl %esi, %ebp
-; X86-NEXT: sarl $31, %ebp
-; X86-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %ecx, %edx
-; X86-NEXT: shll $31, %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %eax, %edx
+; X86-NEXT: movl %eax, %esi
+; X86-NEXT: sarl $31, %edx
+; X86-NEXT: movl %edx, %edi
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: shrl $31, %eax
-; X86-NEXT: shldl $31, %ecx, %eax
+; X86-NEXT: shll $31, %eax
+; X86-NEXT: movl %eax, %edx
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: pushl %ebp
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: shrl $31, %eax
+; X86-NEXT: shll $31, %eax
+; X86-NEXT: shrl %ecx
+; X86-NEXT: orl %eax, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
-; X86-NEXT: pushl %eax
+; X86-NEXT: pushl %ecx
; X86-NEXT: pushl %edx
; X86-NEXT: calll __divdi3
; X86-NEXT: addl $16, %esp
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %edi, %eax
+; X86-NEXT: movl %ebp, %eax
; X86-NEXT: sarl $31, %eax
-; X86-NEXT: movl %ebx, %ebp
-; X86-NEXT: shll $31, %ebp
-; X86-NEXT: movl %ebx, %ecx
-; X86-NEXT: shrl $31, %ecx
-; X86-NEXT: shldl $31, %ebx, %ecx
-; X86-NEXT: pushl %eax
-; X86-NEXT: movl %eax, %esi
+; X86-NEXT: movl %eax, %edi
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ebx, %esi
+; X86-NEXT: shll $31, %esi
+; X86-NEXT: movl %ebx, %eax
+; X86-NEXT: shrl $31, %eax
+; X86-NEXT: shll $31, %eax
+; X86-NEXT: shrl %ebx
+; X86-NEXT: orl %eax, %ebx
; X86-NEXT: pushl %edi
-; X86-NEXT: pushl %ecx
-; X86-NEXT: movl %ecx, %ebx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: pushl %ebp
+; X86-NEXT: pushl %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: pushl %esi
; X86-NEXT: calll __moddi3
; X86-NEXT: addl $16, %esp
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: pushl %esi
; X86-NEXT: pushl %edi
-; X86-NEXT: pushl %ebx
; X86-NEXT: pushl %ebp
+; X86-NEXT: pushl %ebx
+; X86-NEXT: pushl %esi
; X86-NEXT: calll __divdi3
; X86-NEXT: addl $16, %esp
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl %eax, %ecx
-; X86-NEXT: sarl $31, %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl %edx, %ebx
-; X86-NEXT: shll $31, %ebx
+; X86-NEXT: movl %edx, %eax
+; X86-NEXT: sarl $31, %eax
+; X86-NEXT: movl %eax, %ebx
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: shll $31, %esi
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: shrl $31, %eax
+; X86-NEXT: shll $31, %eax
+; X86-NEXT: shrl %ecx
+; X86-NEXT: orl %eax, %ecx
+; X86-NEXT: pushl %ebx
+; X86-NEXT: pushl %edx
; X86-NEXT: movl %edx, %edi
-; X86-NEXT: shrl $31, %edi
-; X86-NEXT: shldl $31, %edx, %edi
; X86-NEXT: pushl %ecx
; X86-NEXT: movl %ecx, %ebp
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: pushl %eax
-; X86-NEXT: movl %eax, %esi
-; X86-NEXT: pushl %edi
-; X86-NEXT: pushl %ebx
+; X86-NEXT: pushl %esi
; X86-NEXT: calll __moddi3
; X86-NEXT: addl $16, %esp
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: pushl %ebx
+; X86-NEXT: pushl %edi
; X86-NEXT: pushl %ebp
; X86-NEXT: pushl %esi
-; X86-NEXT: pushl %edi
-; X86-NEXT: pushl %ebx
; X86-NEXT: calll __divdi3
; X86-NEXT: addl $16, %esp
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl %eax, %ebx
-; X86-NEXT: sarl $31, %ebx
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ecx, %esi
-; X86-NEXT: shll $31, %esi
; X86-NEXT: movl %ecx, %ebp
-; X86-NEXT: shrl $31, %ebp
-; X86-NEXT: shldl $31, %ecx, %ebp
-; X86-NEXT: pushl %ebx
-; X86-NEXT: pushl %eax
+; X86-NEXT: sarl $31, %ebp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT: movl %esi, %edi
+; X86-NEXT: shll $31, %edi
+; X86-NEXT: movl %esi, %eax
+; X86-NEXT: shrl $31, %eax
+; X86-NEXT: shll $31, %eax
+; X86-NEXT: shrl %esi
+; X86-NEXT: orl %eax, %esi
; X86-NEXT: pushl %ebp
+; X86-NEXT: pushl %ecx
; X86-NEXT: pushl %esi
+; X86-NEXT: pushl %edi
; X86-NEXT: calll __moddi3
; X86-NEXT: addl $16, %esp
; X86-NEXT: movl %eax, (%esp) # 4-byte Spill
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: pushl %ebx
-; X86-NEXT: pushl {{[0-9]+}}(%esp)
+; X86-NEXT: movl %edx, %ebx
; X86-NEXT: pushl %ebp
+; X86-NEXT: pushl {{[0-9]+}}(%esp)
; X86-NEXT: pushl %esi
+; X86-NEXT: pushl %edi
; X86-NEXT: calll __divdi3
; X86-NEXT: addl $16, %esp
-; X86-NEXT: testl %ebp, %ebp
+; X86-NEXT: testl %esi, %esi
; X86-NEXT: sets %cl
-; X86-NEXT: testl %ebx, %ebx
+; X86-NEXT: testl %ebp, %ebp
; X86-NEXT: sets %dl
; X86-NEXT: xorb %cl, %dl
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: orl (%esp), %ecx # 4-byte Folded Reload
+; X86-NEXT: orl (%esp), %ebx # 4-byte Folded Reload
; X86-NEXT: setne %cl
; X86-NEXT: testb %dl, %cl
; X86-NEXT: leal -1(%eax), %ecx
; X86-NEXT: cmovel %eax, %ecx
; X86-NEXT: movl %ecx, (%esp) # 4-byte Spill
-; X86-NEXT: testl %edi, %edi
+; X86-NEXT: cmpl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
; X86-NEXT: sets %al
; X86-NEXT: cmpl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
; X86-NEXT: sets %cl
diff --git a/llvm/test/CodeGen/X86/sdiv_fix_sat.ll b/llvm/test/CodeGen/X86/sdiv_fix_sat.ll
index e7d41d5bebc8a..fe8364966cfd4 100644
--- a/llvm/test/CodeGen/X86/sdiv_fix_sat.ll
+++ b/llvm/test/CodeGen/X86/sdiv_fix_sat.ll
@@ -307,18 +307,20 @@ define i64 @func5(i64 %x, i64 %y) nounwind {
; X64-NEXT: pushq %r12
; X64-NEXT: pushq %rbx
; X64-NEXT: subq $24, %rsp
+; X64-NEXT: movq %rdi, %r15
+; X64-NEXT: leaq (%rdi,%rdi), %r14
+; X64-NEXT: movq %rdi, %rax
+; X64-NEXT: sarq $63, %rax
+; X64-NEXT: shlq $31, %rax
+; X64-NEXT: shrq $33, %r14
+; X64-NEXT: orq %rax, %r14
+; X64-NEXT: shlq $32, %r15
; X64-NEXT: movq %rsi, %rdx
; X64-NEXT: movq %rsi, (%rsp) # 8-byte Spill
-; X64-NEXT: movq %rdi, %r14
-; X64-NEXT: leaq (%rdi,%rdi), %rax
-; X64-NEXT: movq %rdi, %r15
-; X64-NEXT: sarq $63, %r15
-; X64-NEXT: shldq $31, %rax, %r15
-; X64-NEXT: shlq $32, %r14
; X64-NEXT: movq %rsi, %r12
; X64-NEXT: sarq $63, %r12
-; X64-NEXT: movq %r14, %rdi
-; X64-NEXT: movq %r15, %rsi
+; X64-NEXT: movq %r15, %rdi
+; X64-NEXT: movq %r14, %rsi
; X64-NEXT: movq %r12, %rcx
; X64-NEXT: callq __divti3 at PLT
; X64-NEXT: movq %rax, %r13
@@ -327,13 +329,13 @@ define i64 @func5(i64 %x, i64 %y) nounwind {
; X64-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; X64-NEXT: subq $1, %r13
; X64-NEXT: sbbq $0, %rbp
-; X64-NEXT: testq %r15, %r15
+; X64-NEXT: testq %r14, %r14
; X64-NEXT: sets %al
; X64-NEXT: testq %r12, %r12
; X64-NEXT: sets %bl
; X64-NEXT: xorb %al, %bl
-; X64-NEXT: movq %r14, %rdi
-; X64-NEXT: movq %r15, %rsi
+; X64-NEXT: movq %r15, %rdi
+; X64-NEXT: movq %r14, %rsi
; X64-NEXT: movq (%rsp), %rdx # 8-byte Reload
; X64-NEXT: movq %r12, %rcx
; X64-NEXT: callq __modti3 at PLT
@@ -371,65 +373,72 @@ define i64 @func5(i64 %x, i64 %y) nounwind {
; X86-NEXT: pushl %esi
; X86-NEXT: andl $-16, %esp
; X86-NEXT: subl $128, %esp
-; X86-NEXT: movl 8(%ebp), %esi
-; X86-NEXT: movl 12(%ebp), %edi
-; X86-NEXT: movl 16(%ebp), %ecx
-; X86-NEXT: movl 20(%ebp), %edx
-; X86-NEXT: movl %edx, %eax
+; X86-NEXT: movl 8(%ebp), %edx
+; X86-NEXT: movl 12(%ebp), %esi
+; X86-NEXT: movl 20(%ebp), %ebx
+; X86-NEXT: movl %esi, %eax
; X86-NEXT: sarl $31, %eax
+; X86-NEXT: movl %eax, %ecx
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %esi, %eax
+; X86-NEXT: shll $31, %eax
+; X86-NEXT: movl %edx, %edi
+; X86-NEXT: shrl %edi
+; X86-NEXT: orl %eax, %edi
+; X86-NEXT: leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %eax, (%esp)
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: shll $31, %eax
+; X86-NEXT: shrl %esi
+; X86-NEXT: orl %eax, %esi
+; X86-NEXT: movl %ebx, %eax
+; X86-NEXT: sarl $31, %ebx
+; X86-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %ebx, {{[0-9]+}}(%esp)
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 16(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X86-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NEXT: movl %edi, %ebx
-; X86-NEXT: sarl $31, %ebx
-; X86-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: leal {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl %eax, (%esp)
-; X86-NEXT: shldl $31, %edi, %ebx
-; X86-NEXT: shldl $31, %esi, %edi
; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-NEXT: shll $31, %esi
+; X86-NEXT: shll $31, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-NEXT: movl %ebx, {{[0-9]+}}(%esp)
; X86-NEXT: calll __divti3
; X86-NEXT: subl $4, %esp
; X86-NEXT: movl 20(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: movl 16(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %ebx, {{[0-9]+}}(%esp)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: leal {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl %eax, (%esp)
-; X86-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: subl $1, %edi
+; X86-NEXT: subl $1, %esi
; X86-NEXT: sbbl $0, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: sbbl $0, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT: movl %esi, %ebx
-; X86-NEXT: sbbl $0, %ebx
-; X86-NEXT: testl %ecx, %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl $0, %edi
+; X86-NEXT: testl %ebx, %ebx
; X86-NEXT: sets %al
-; X86-NEXT: testl %edx, %edx
-; X86-NEXT: sets %cl
-; X86-NEXT: xorb %al, %cl
-; X86-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; X86-NEXT: testl %ecx, %ecx
+; X86-NEXT: sets %bl
+; X86-NEXT: xorb %al, %bl
; X86-NEXT: calll __modti3
; X86-NEXT: subl $4, %esp
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
@@ -438,39 +447,41 @@ define i64 @func5(i64 %x, i64 %y) nounwind {
; X86-NEXT: orl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: orl %eax, %ecx
; X86-NEXT: setne %al
-; X86-NEXT: testb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Reload
-; X86-NEXT: cmovel %esi, %ebx
+; X86-NEXT: testb %bl, %al
+; X86-NEXT: cmovel {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: cmovel {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: cmovel {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: cmovel {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: cmovel {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X86-NEXT: cmpl $-1, %edi
-; X86-NEXT: sbbl $2147483647, %ecx # imm = 0x7FFFFFFF
+; X86-NEXT: cmovel {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NEXT: cmpl $-1, %esi
; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: movl %eax, %ebx
+; X86-NEXT: sbbl $2147483647, %ecx # imm = 0x7FFFFFFF
+; X86-NEXT: movl %edx, %ecx
+; X86-NEXT: movl %edx, %eax
; X86-NEXT: sbbl $0, %ecx
-; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: movl %edi, %ecx
; X86-NEXT: sbbl $0, %ecx
; X86-NEXT: movl $2147483647, %edx # imm = 0x7FFFFFFF
-; X86-NEXT: cmovll {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X86-NEXT: cmovll %ebx, %edx
; X86-NEXT: movl $0, %ecx
-; X86-NEXT: cmovgel %ecx, %ebx
+; X86-NEXT: cmovgel %ecx, %edi
; X86-NEXT: cmovgel %ecx, %eax
; X86-NEXT: movl $-1, %ecx
-; X86-NEXT: cmovgel %ecx, %edi
-; X86-NEXT: movl %edi, %esi
-; X86-NEXT: negl %esi
-; X86-NEXT: movl $-2147483648, %esi # imm = 0x80000000
-; X86-NEXT: sbbl %edx, %esi
-; X86-NEXT: movl $-1, %esi
-; X86-NEXT: sbbl %eax, %esi
-; X86-NEXT: sbbl %ebx, %ecx
+; X86-NEXT: cmovgel %ecx, %esi
+; X86-NEXT: movl %esi, %ebx
+; X86-NEXT: negl %ebx
+; X86-NEXT: movl $-2147483648, %ebx # imm = 0x80000000
+; X86-NEXT: sbbl %edx, %ebx
+; X86-NEXT: movl $-1, %ebx
+; X86-NEXT: sbbl %eax, %ebx
+; X86-NEXT: sbbl %edi, %ecx
; X86-NEXT: movl $0, %eax
-; X86-NEXT: cmovgel %eax, %edi
+; X86-NEXT: cmovgel %eax, %esi
; X86-NEXT: movl $-2147483648, %eax # imm = 0x80000000
; X86-NEXT: cmovgel %eax, %edx
-; X86-NEXT: movl %edi, %eax
+; X86-NEXT: movl %esi, %eax
; X86-NEXT: leal -12(%ebp), %esp
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
@@ -570,9 +581,12 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
; X64-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
; X64-NEXT: movdqa %xmm3, (%rsp) # 16-byte Spill
; X64-NEXT: movq %xmm3, %r15
+; X64-NEXT: movq %r15, %rax
+; X64-NEXT: sarq $63, %rax
+; X64-NEXT: shlq $31, %rax
; X64-NEXT: movq %r15, %rbx
-; X64-NEXT: sarq $63, %rbx
-; X64-NEXT: shldq $31, %r15, %rbx
+; X64-NEXT: shrq $33, %rbx
+; X64-NEXT: orq %rax, %rbx
; X64-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
; X64-NEXT: pxor %xmm0, %xmm0
; X64-NEXT: pcmpgtd %xmm1, %xmm0
@@ -623,9 +637,12 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
; X64-NEXT: pshufd $238, (%rsp), %xmm0 # 16-byte Folded Reload
; X64-NEXT: # xmm0 = mem[2,3,2,3]
; X64-NEXT: movq %xmm0, %r15
+; X64-NEXT: movq %r15, %rax
+; X64-NEXT: sarq $63, %rax
+; X64-NEXT: shlq $31, %rax
; X64-NEXT: movq %r15, %rbx
-; X64-NEXT: sarq $63, %rbx
-; X64-NEXT: shldq $31, %r15, %rbx
+; X64-NEXT: shrq $33, %rbx
+; X64-NEXT: orq %rax, %rbx
; X64-NEXT: pshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
; X64-NEXT: # xmm0 = mem[2,3,2,3]
; X64-NEXT: movq %xmm0, %r12
@@ -681,9 +698,12 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
; X64-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
; X64-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; X64-NEXT: movq %xmm0, %r15
+; X64-NEXT: movq %r15, %rax
+; X64-NEXT: sarq $63, %rax
+; X64-NEXT: shlq $31, %rax
; X64-NEXT: movq %r15, %rbx
-; X64-NEXT: sarq $63, %rbx
-; X64-NEXT: shldq $31, %r15, %rbx
+; X64-NEXT: shrq $33, %rbx
+; X64-NEXT: orq %rax, %rbx
; X64-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; X64-NEXT: pxor %xmm1, %xmm1
; X64-NEXT: pcmpgtd %xmm0, %xmm1
@@ -733,9 +753,12 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
; X64-NEXT: pshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
; X64-NEXT: # xmm0 = mem[2,3,2,3]
; X64-NEXT: movq %xmm0, %r15
+; X64-NEXT: movq %r15, %rax
+; X64-NEXT: sarq $63, %rax
+; X64-NEXT: shlq $31, %rax
; X64-NEXT: movq %r15, %rbx
-; X64-NEXT: sarq $63, %rbx
-; X64-NEXT: shldq $31, %r15, %rbx
+; X64-NEXT: shrq $33, %rbx
+; X64-NEXT: orq %rax, %rbx
; X64-NEXT: pshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
; X64-NEXT: # xmm0 = mem[2,3,2,3]
; X64-NEXT: movq %xmm0, %r12
diff --git a/llvm/test/CodeGen/X86/setcc-fsh.ll b/llvm/test/CodeGen/X86/setcc-fsh.ll
index 7ab63959f58b0..f12ed9152671e 100644
--- a/llvm/test/CodeGen/X86/setcc-fsh.ll
+++ b/llvm/test/CodeGen/X86/setcc-fsh.ll
@@ -484,7 +484,9 @@ define i1 @fshl_xor_eq_0(i32 %x, i32 %y) {
; CHECK-LABEL: fshl_xor_eq_0:
; CHECK: # %bb.0:
; CHECK-NEXT: xorl %edi, %esi
-; CHECK-NEXT: shldl $2, %edi, %esi
+; CHECK-NEXT: shll $2, %esi
+; CHECK-NEXT: shrl $30, %edi
+; CHECK-NEXT: orl %esi, %edi
; CHECK-NEXT: sete %al
; CHECK-NEXT: retq
%or = xor i32 %x, %y
@@ -497,8 +499,9 @@ define i1 @fshl_or_sgt_0(i32 %x, i32 %y) {
; CHECK-LABEL: fshl_or_sgt_0:
; CHECK: # %bb.0:
; CHECK-NEXT: orl %edi, %esi
-; CHECK-NEXT: shldl $2, %edi, %esi
-; CHECK-NEXT: testl %esi, %esi
+; CHECK-NEXT: shll $2, %esi
+; CHECK-NEXT: shrl $30, %edi
+; CHECK-NEXT: orl %esi, %edi
; CHECK-NEXT: setg %al
; CHECK-NEXT: retq
%or = or i32 %x, %y
@@ -511,8 +514,10 @@ define i1 @fshl_or_ne_2(i32 %x, i32 %y) {
; CHECK-LABEL: fshl_or_ne_2:
; CHECK: # %bb.0:
; CHECK-NEXT: orl %edi, %esi
-; CHECK-NEXT: shldl $2, %edi, %esi
-; CHECK-NEXT: cmpl $2, %esi
+; CHECK-NEXT: shll $2, %esi
+; CHECK-NEXT: shrl $30, %edi
+; CHECK-NEXT: orl %esi, %edi
+; CHECK-NEXT: cmpl $2, %edi
; CHECK-NEXT: setne %al
; CHECK-NEXT: retq
%or = or i32 %x, %y
diff --git a/llvm/test/CodeGen/X86/shift-bmi2.ll b/llvm/test/CodeGen/X86/shift-bmi2.ll
index bb0213891c976..dafcf3db4550e 100644
--- a/llvm/test/CodeGen/X86/shift-bmi2.ll
+++ b/llvm/test/CodeGen/X86/shift-bmi2.ll
@@ -126,10 +126,13 @@ define i64 @shl64(i64 %x, i64 %shamt) nounwind uwtable readnone {
define i64 @shl64i(i64 %x) nounwind uwtable readnone {
; BMI2-LABEL: shl64i:
; BMI2: # %bb.0:
-; BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
; BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx
-; BMI2-NEXT: shldl $7, %eax, %edx
+; BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; BMI2-NEXT: movl %edx, %eax
; BMI2-NEXT: shll $7, %eax
+; BMI2-NEXT: shll $7, %ecx
+; BMI2-NEXT: shrl $25, %edx
+; BMI2-NEXT: orl %ecx, %edx
; BMI2-NEXT: retl
;
; BMI264-LABEL: shl64i:
@@ -184,11 +187,14 @@ define i64 @shl64p(ptr %p, i64 %shamt) nounwind uwtable readnone {
define i64 @shl64pi(ptr %p) nounwind uwtable readnone {
; BMI2-LABEL: shl64pi:
; BMI2: # %bb.0:
-; BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; BMI2-NEXT: movl (%ecx), %eax
-; BMI2-NEXT: movl 4(%ecx), %edx
-; BMI2-NEXT: shldl $7, %eax, %edx
+; BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; BMI2-NEXT: movl (%eax), %edx
+; BMI2-NEXT: movl 4(%eax), %ecx
+; BMI2-NEXT: movl %edx, %eax
; BMI2-NEXT: shll $7, %eax
+; BMI2-NEXT: shll $7, %ecx
+; BMI2-NEXT: shrl $25, %edx
+; BMI2-NEXT: orl %ecx, %edx
; BMI2-NEXT: retl
;
; BMI264-LABEL: shl64pi:
diff --git a/llvm/test/CodeGen/X86/shift-i256.ll b/llvm/test/CodeGen/X86/shift-i256.ll
index 3b8e766ae1bf4..c541de6865eb1 100644
--- a/llvm/test/CodeGen/X86/shift-i256.ll
+++ b/llvm/test/CodeGen/X86/shift-i256.ll
@@ -1458,43 +1458,85 @@ define i256 @shl_i256_1(i256 %a0) nounwind {
; CHECK-LABEL: shl_i256_1:
; CHECK: # %bb.0:
; CHECK-NEXT: movq %rdi, %rax
-; CHECK-NEXT: shldq $1, %rcx, %r8
-; CHECK-NEXT: shldq $1, %rdx, %rcx
-; CHECK-NEXT: shldq $1, %rsi, %rdx
+; CHECK-NEXT: leaq (,%rdx,2), %rdi
+; CHECK-NEXT: movq %rsi, %r9
+; CHECK-NEXT: shrq $63, %r9
+; CHECK-NEXT: orq %rdi, %r9
+; CHECK-NEXT: leaq (,%rcx,2), %rdi
+; CHECK-NEXT: shrq $63, %rdx
+; CHECK-NEXT: orq %rdi, %rdx
+; CHECK-NEXT: shlq %r8
+; CHECK-NEXT: shrq $63, %rcx
+; CHECK-NEXT: orq %r8, %rcx
; CHECK-NEXT: addq %rsi, %rsi
-; CHECK-NEXT: movq %r8, 24(%rdi)
-; CHECK-NEXT: movq %rcx, 16(%rdi)
-; CHECK-NEXT: movq %rdx, 8(%rdi)
-; CHECK-NEXT: movq %rsi, (%rdi)
+; CHECK-NEXT: movq %rcx, 24(%rax)
+; CHECK-NEXT: movq %rdx, 16(%rax)
+; CHECK-NEXT: movq %r9, 8(%rax)
+; CHECK-NEXT: movq %rsi, (%rax)
; CHECK-NEXT: retq
;
; X86-LABEL: shl_i256_1:
; X86: # %bb.0:
+; X86-NEXT: pushl %ebp
+; X86-NEXT: pushl %ebx
+; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %esi
+; X86-NEXT: subl $12, %esp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT: leal (,%esi,2), %eax
+; X86-NEXT: movl %edx, %ecx
+; X86-NEXT: shrl $31, %ecx
+; X86-NEXT: orl %eax, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: leal (,%edi,2), %eax
+; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: shrl $31, %ecx
+; X86-NEXT: orl %eax, %ecx
+; X86-NEXT: movl %ecx, (%esp) # 4-byte Spill
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %eax, %ebx
+; X86-NEXT: shrl $31, %ebx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT: leal (,%ebp,2), %ecx
+; X86-NEXT: orl %ecx, %ebx
+; X86-NEXT: shll %edx
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: leal (%ecx,%ecx), %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shrl $31, %ecx
+; X86-NEXT: orl %edx, %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: shldl $1, %ecx, %edx
-; X86-NEXT: movl %edx, 28(%eax)
+; X86-NEXT: shll %edx
+; X86-NEXT: shrl $31, %esi
+; X86-NEXT: orl %edx, %esi
+; X86-NEXT: shll %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: shldl $1, %edx, %ecx
-; X86-NEXT: movl %ecx, 24(%eax)
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: shldl $1, %ecx, %edx
+; X86-NEXT: shrl $31, %edx
+; X86-NEXT: orl %eax, %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: shll %eax
+; X86-NEXT: shrl $31, %ebp
+; X86-NEXT: orl %eax, %ebp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %ebp, 28(%eax)
+; X86-NEXT: movl %ebx, 24(%eax)
; X86-NEXT: movl %edx, 20(%eax)
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: shldl $1, %edx, %ecx
-; X86-NEXT: movl %ecx, 16(%eax)
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: shldl $1, %ecx, %edx
-; X86-NEXT: movl %edx, 12(%eax)
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: shldl $1, %edx, %ecx
-; X86-NEXT: movl %ecx, 8(%eax)
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: shldl $1, %ecx, %edx
-; X86-NEXT: movl %edx, 4(%eax)
-; X86-NEXT: addl %ecx, %ecx
+; X86-NEXT: movl (%esp), %edx # 4-byte Reload
+; X86-NEXT: movl %edx, 16(%eax)
+; X86-NEXT: movl %esi, 12(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: movl %edx, 8(%eax)
+; X86-NEXT: movl %ecx, 4(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: movl %ecx, (%eax)
+; X86-NEXT: addl $12, %esp
+; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
+; X86-NEXT: popl %ebx
+; X86-NEXT: popl %ebp
; X86-NEXT: retl $4
%r = shl i256 %a0, 1
ret i256 %r
@@ -1505,13 +1547,16 @@ define i256 @lshr_i256_1(i256 %a0) nounwind {
; CHECK: # %bb.0:
; CHECK-NEXT: movq %rdi, %rax
; CHECK-NEXT: shrdq $1, %rdx, %rsi
-; CHECK-NEXT: shrdq $1, %rcx, %rdx
+; CHECK-NEXT: movq %rcx, %rdi
+; CHECK-NEXT: shlq $63, %rdi
+; CHECK-NEXT: shrq %rdx
+; CHECK-NEXT: orq %rdi, %rdx
; CHECK-NEXT: shrdq $1, %r8, %rcx
; CHECK-NEXT: shrq %r8
-; CHECK-NEXT: movq %r8, 24(%rdi)
-; CHECK-NEXT: movq %rcx, 16(%rdi)
-; CHECK-NEXT: movq %rdx, 8(%rdi)
-; CHECK-NEXT: movq %rsi, (%rdi)
+; CHECK-NEXT: movq %r8, 24(%rax)
+; CHECK-NEXT: movq %rcx, 16(%rax)
+; CHECK-NEXT: movq %rdx, 8(%rax)
+; CHECK-NEXT: movq %rsi, (%rax)
; CHECK-NEXT: retq
;
; X86-LABEL: lshr_i256_1:
@@ -1521,39 +1566,45 @@ define i256 @lshr_i256_1(i256 %a0) nounwind {
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
; X86-NEXT: subl $8, %esp
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT: movl %ebx, %eax
+; X86-NEXT: shll $31, %eax
+; X86-NEXT: shrdl $1, %esi, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shrl %esi
+; X86-NEXT: orl %eax, %esi
+; X86-NEXT: movl %edx, %eax
+; X86-NEXT: shll $31, %eax
+; X86-NEXT: shrdl $1, %ebp, %ebx
+; X86-NEXT: shrl %ebp
+; X86-NEXT: orl %eax, %ebp
+; X86-NEXT: movl %edi, %ecx
+; X86-NEXT: shll $31, %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT: movl %edi, %esi
-; X86-NEXT: shldl $31, %eax, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shrdl $1, %eax, %ecx
-; X86-NEXT: movl %ecx, (%esp) # 4-byte Spill
-; X86-NEXT: movl %edx, %ebp
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: shldl $31, %eax, %ebp
-; X86-NEXT: shrdl $1, %eax, %edi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl %ebx, %esi
-; X86-NEXT: shldl $31, %eax, %esi
; X86-NEXT: shrdl $1, %eax, %edx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: shrdl $1, %eax, %ebx
+; X86-NEXT: movl %edx, (%esp) # 4-byte Spill
; X86-NEXT: shrl %eax
+; X86-NEXT: orl %ecx, %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %eax, 28(%ecx)
-; X86-NEXT: movl %ebx, 24(%ecx)
-; X86-NEXT: movl %esi, 20(%ecx)
-; X86-NEXT: movl %edx, 16(%ecx)
-; X86-NEXT: movl %ebp, 12(%ecx)
-; X86-NEXT: movl %edi, 8(%ecx)
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: movl %eax, 4(%ecx)
+; X86-NEXT: shrdl $1, %ecx, %edi
+; X86-NEXT: shrl %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movl %ecx, 28(%edx)
+; X86-NEXT: movl %edi, 24(%edx)
+; X86-NEXT: movl %eax, 20(%edx)
; X86-NEXT: movl (%esp), %eax # 4-byte Reload
-; X86-NEXT: movl %eax, (%ecx)
-; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: movl %eax, 16(%edx)
+; X86-NEXT: movl %ebp, 12(%edx)
+; X86-NEXT: movl %ebx, 8(%edx)
+; X86-NEXT: movl %esi, 4(%edx)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl %eax, (%edx)
+; X86-NEXT: movl %edx, %eax
; X86-NEXT: addl $8, %esp
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
@@ -1569,13 +1620,16 @@ define i256 @ashr_i256_1(i256 %a0) nounwind {
; CHECK: # %bb.0:
; CHECK-NEXT: movq %rdi, %rax
; CHECK-NEXT: shrdq $1, %rdx, %rsi
-; CHECK-NEXT: shrdq $1, %rcx, %rdx
+; CHECK-NEXT: movq %rcx, %rdi
+; CHECK-NEXT: shlq $63, %rdi
+; CHECK-NEXT: shrq %rdx
+; CHECK-NEXT: orq %rdi, %rdx
; CHECK-NEXT: shrdq $1, %r8, %rcx
; CHECK-NEXT: sarq %r8
-; CHECK-NEXT: movq %r8, 24(%rdi)
-; CHECK-NEXT: movq %rcx, 16(%rdi)
-; CHECK-NEXT: movq %rdx, 8(%rdi)
-; CHECK-NEXT: movq %rsi, (%rdi)
+; CHECK-NEXT: movq %r8, 24(%rax)
+; CHECK-NEXT: movq %rcx, 16(%rax)
+; CHECK-NEXT: movq %rdx, 8(%rax)
+; CHECK-NEXT: movq %rsi, (%rax)
; CHECK-NEXT: retq
;
; X86-LABEL: ashr_i256_1:
@@ -1585,39 +1639,45 @@ define i256 @ashr_i256_1(i256 %a0) nounwind {
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
; X86-NEXT: subl $8, %esp
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT: movl %ebx, %eax
+; X86-NEXT: shll $31, %eax
+; X86-NEXT: shrdl $1, %esi, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shrl %esi
+; X86-NEXT: orl %eax, %esi
+; X86-NEXT: movl %edx, %eax
+; X86-NEXT: shll $31, %eax
+; X86-NEXT: shrdl $1, %ebp, %ebx
+; X86-NEXT: shrl %ebp
+; X86-NEXT: orl %eax, %ebp
+; X86-NEXT: movl %edi, %ecx
+; X86-NEXT: shll $31, %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT: movl %edi, %esi
-; X86-NEXT: shldl $31, %eax, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shrdl $1, %eax, %ecx
-; X86-NEXT: movl %ecx, (%esp) # 4-byte Spill
-; X86-NEXT: movl %edx, %ebp
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: shldl $31, %eax, %ebp
-; X86-NEXT: shrdl $1, %eax, %edi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl %ebx, %esi
-; X86-NEXT: shldl $31, %eax, %esi
; X86-NEXT: shrdl $1, %eax, %edx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: shrdl $1, %eax, %ebx
-; X86-NEXT: sarl %eax
+; X86-NEXT: movl %edx, (%esp) # 4-byte Spill
+; X86-NEXT: shrl %eax
+; X86-NEXT: orl %ecx, %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %eax, 28(%ecx)
-; X86-NEXT: movl %ebx, 24(%ecx)
-; X86-NEXT: movl %esi, 20(%ecx)
-; X86-NEXT: movl %edx, 16(%ecx)
-; X86-NEXT: movl %ebp, 12(%ecx)
-; X86-NEXT: movl %edi, 8(%ecx)
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: movl %eax, 4(%ecx)
+; X86-NEXT: shrdl $1, %ecx, %edi
+; X86-NEXT: sarl %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movl %ecx, 28(%edx)
+; X86-NEXT: movl %edi, 24(%edx)
+; X86-NEXT: movl %eax, 20(%edx)
; X86-NEXT: movl (%esp), %eax # 4-byte Reload
-; X86-NEXT: movl %eax, (%ecx)
-; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: movl %eax, 16(%edx)
+; X86-NEXT: movl %ebp, 12(%edx)
+; X86-NEXT: movl %ebx, 8(%edx)
+; X86-NEXT: movl %esi, 4(%edx)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl %eax, (%edx)
+; X86-NEXT: movl %edx, %eax
; X86-NEXT: addl $8, %esp
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
diff --git a/llvm/test/CodeGen/X86/shift-i512.ll b/llvm/test/CodeGen/X86/shift-i512.ll
index 01cd11b9e712c..381ef07291718 100644
--- a/llvm/test/CodeGen/X86/shift-i512.ll
+++ b/llvm/test/CodeGen/X86/shift-i512.ll
@@ -1258,26 +1258,45 @@ define i512 @ashr_i512_load(ptr %p0, i512 %a1) nounwind {
define i512 @shl_i512_1(i512 %a0) nounwind {
; CHECK-LABEL: shl_i512_1:
; CHECK: # %bb.0:
+; CHECK-NEXT: pushq %r14
+; CHECK-NEXT: pushq %rbx
; CHECK-NEXT: movq %rdi, %rax
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rdi
; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rbx
; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r11
-; CHECK-NEXT: shldq $1, %rdi, %r10
-; CHECK-NEXT: shldq $1, %r11, %rdi
-; CHECK-NEXT: shldq $1, %r9, %r11
-; CHECK-NEXT: shldq $1, %r8, %r9
-; CHECK-NEXT: shldq $1, %rcx, %r8
-; CHECK-NEXT: shldq $1, %rdx, %rcx
-; CHECK-NEXT: shldq $1, %rsi, %rdx
+; CHECK-NEXT: leaq (,%rdx,2), %r14
+; CHECK-NEXT: movq %rsi, %rdi
+; CHECK-NEXT: shrq $63, %rdi
+; CHECK-NEXT: orq %r14, %rdi
+; CHECK-NEXT: leaq (,%rcx,2), %r14
+; CHECK-NEXT: shrq $63, %rdx
+; CHECK-NEXT: orq %r14, %rdx
+; CHECK-NEXT: leaq (,%r8,2), %r14
+; CHECK-NEXT: shrq $63, %rcx
+; CHECK-NEXT: orq %r14, %rcx
+; CHECK-NEXT: leaq (,%r9,2), %r14
+; CHECK-NEXT: shrq $63, %r8
+; CHECK-NEXT: orq %r14, %r8
+; CHECK-NEXT: leaq (,%r11,2), %r14
+; CHECK-NEXT: shrq $63, %r9
+; CHECK-NEXT: orq %r14, %r9
+; CHECK-NEXT: leaq (,%r10,2), %r14
+; CHECK-NEXT: shrq $63, %r11
+; CHECK-NEXT: orq %r14, %r11
; CHECK-NEXT: addq %rsi, %rsi
+; CHECK-NEXT: shlq %rbx
+; CHECK-NEXT: shrq $63, %r10
+; CHECK-NEXT: orq %rbx, %r10
; CHECK-NEXT: movq %r10, 56(%rax)
-; CHECK-NEXT: movq %rdi, 48(%rax)
-; CHECK-NEXT: movq %r11, 40(%rax)
-; CHECK-NEXT: movq %r9, 32(%rax)
-; CHECK-NEXT: movq %r8, 24(%rax)
-; CHECK-NEXT: movq %rcx, 16(%rax)
-; CHECK-NEXT: movq %rdx, 8(%rax)
+; CHECK-NEXT: movq %r11, 48(%rax)
+; CHECK-NEXT: movq %r9, 40(%rax)
+; CHECK-NEXT: movq %r8, 32(%rax)
+; CHECK-NEXT: movq %rcx, 24(%rax)
+; CHECK-NEXT: movq %rdx, 16(%rax)
+; CHECK-NEXT: movq %rdi, 8(%rax)
; CHECK-NEXT: movq %rsi, (%rax)
+; CHECK-NEXT: popq %rbx
+; CHECK-NEXT: popq %r14
; CHECK-NEXT: retq
%r = shl i512 %a0, 1
ret i512 %r
@@ -1286,26 +1305,37 @@ define i512 @shl_i512_1(i512 %a0) nounwind {
define i512 @lshr_i512_1(i512 %a0) nounwind {
; CHECK-LABEL: lshr_i512_1:
; CHECK: # %bb.0:
+; CHECK-NEXT: pushq %rbx
; CHECK-NEXT: movq %rdi, %rax
; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rdi
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r10
; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r10
; CHECK-NEXT: shrdq $1, %rdx, %rsi
-; CHECK-NEXT: shrdq $1, %rcx, %rdx
+; CHECK-NEXT: movq %rcx, %rbx
+; CHECK-NEXT: shlq $63, %rbx
+; CHECK-NEXT: shrq %rdx
+; CHECK-NEXT: orq %rbx, %rdx
; CHECK-NEXT: shrdq $1, %r8, %rcx
-; CHECK-NEXT: shrdq $1, %r9, %r8
-; CHECK-NEXT: shrdq $1, %r11, %r9
-; CHECK-NEXT: shrdq $1, %rdi, %r11
-; CHECK-NEXT: shrdq $1, %r10, %rdi
+; CHECK-NEXT: movq %r9, %rbx
+; CHECK-NEXT: shlq $63, %rbx
+; CHECK-NEXT: shrq %r8
+; CHECK-NEXT: orq %rbx, %r8
+; CHECK-NEXT: shrdq $1, %r10, %r9
+; CHECK-NEXT: movq %rdi, %rbx
+; CHECK-NEXT: shlq $63, %rbx
; CHECK-NEXT: shrq %r10
-; CHECK-NEXT: movq %r10, 56(%rax)
+; CHECK-NEXT: orq %rbx, %r10
+; CHECK-NEXT: shrdq $1, %r11, %rdi
+; CHECK-NEXT: shrq %r11
+; CHECK-NEXT: movq %r11, 56(%rax)
; CHECK-NEXT: movq %rdi, 48(%rax)
-; CHECK-NEXT: movq %r11, 40(%rax)
+; CHECK-NEXT: movq %r10, 40(%rax)
; CHECK-NEXT: movq %r9, 32(%rax)
; CHECK-NEXT: movq %r8, 24(%rax)
; CHECK-NEXT: movq %rcx, 16(%rax)
; CHECK-NEXT: movq %rdx, 8(%rax)
; CHECK-NEXT: movq %rsi, (%rax)
+; CHECK-NEXT: popq %rbx
; CHECK-NEXT: retq
%r = lshr i512 %a0, 1
ret i512 %r
@@ -1314,26 +1344,37 @@ define i512 @lshr_i512_1(i512 %a0) nounwind {
define i512 @ashr_i512_1(i512 %a0) nounwind {
; CHECK-LABEL: ashr_i512_1:
; CHECK: # %bb.0:
+; CHECK-NEXT: pushq %rbx
; CHECK-NEXT: movq %rdi, %rax
; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rdi
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r10
; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r10
; CHECK-NEXT: shrdq $1, %rdx, %rsi
-; CHECK-NEXT: shrdq $1, %rcx, %rdx
+; CHECK-NEXT: movq %rcx, %rbx
+; CHECK-NEXT: shlq $63, %rbx
+; CHECK-NEXT: shrq %rdx
+; CHECK-NEXT: orq %rbx, %rdx
; CHECK-NEXT: shrdq $1, %r8, %rcx
-; CHECK-NEXT: shrdq $1, %r9, %r8
-; CHECK-NEXT: shrdq $1, %r11, %r9
-; CHECK-NEXT: shrdq $1, %rdi, %r11
-; CHECK-NEXT: shrdq $1, %r10, %rdi
-; CHECK-NEXT: sarq %r10
-; CHECK-NEXT: movq %r10, 56(%rax)
+; CHECK-NEXT: movq %r9, %rbx
+; CHECK-NEXT: shlq $63, %rbx
+; CHECK-NEXT: shrq %r8
+; CHECK-NEXT: orq %rbx, %r8
+; CHECK-NEXT: shrdq $1, %r10, %r9
+; CHECK-NEXT: movq %rdi, %rbx
+; CHECK-NEXT: shlq $63, %rbx
+; CHECK-NEXT: shrq %r10
+; CHECK-NEXT: orq %rbx, %r10
+; CHECK-NEXT: shrdq $1, %r11, %rdi
+; CHECK-NEXT: sarq %r11
+; CHECK-NEXT: movq %r11, 56(%rax)
; CHECK-NEXT: movq %rdi, 48(%rax)
-; CHECK-NEXT: movq %r11, 40(%rax)
+; CHECK-NEXT: movq %r10, 40(%rax)
; CHECK-NEXT: movq %r9, 32(%rax)
; CHECK-NEXT: movq %r8, 24(%rax)
; CHECK-NEXT: movq %rcx, 16(%rax)
; CHECK-NEXT: movq %rdx, 8(%rax)
; CHECK-NEXT: movq %rsi, (%rax)
+; CHECK-NEXT: popq %rbx
; CHECK-NEXT: retq
%r = ashr i512 %a0, 1
ret i512 %r
@@ -1342,17 +1383,22 @@ define i512 @ashr_i512_1(i512 %a0) nounwind {
define i512 @shl_i512_200(i512 %a0) nounwind {
; SSE-LABEL: shl_i512_200:
; SSE: # %bb.0:
+; SSE-NEXT: movq %rsi, %r10
+; SSE-NEXT: shrdq $56, %rdx, %r10
; SSE-NEXT: movq %rdi, %rax
-; SSE-NEXT: movq %rsi, %rdi
-; SSE-NEXT: shrdq $56, %rdx, %rdi
-; SSE-NEXT: shrdq $56, %rcx, %rdx
+; SSE-NEXT: movq %rcx, %rdi
+; SSE-NEXT: shlq $8, %rdi
+; SSE-NEXT: shrq $56, %rdx
+; SSE-NEXT: orq %rdi, %rdx
; SSE-NEXT: shrdq $56, %r8, %rcx
-; SSE-NEXT: shldq $8, %r8, %r9
+; SSE-NEXT: shlq $8, %r9
+; SSE-NEXT: shrq $56, %r8
+; SSE-NEXT: orq %r9, %r8
; SSE-NEXT: shlq $8, %rsi
-; SSE-NEXT: movq %r9, 56(%rax)
+; SSE-NEXT: movq %r8, 56(%rax)
; SSE-NEXT: movq %rcx, 48(%rax)
; SSE-NEXT: movq %rdx, 40(%rax)
-; SSE-NEXT: movq %rdi, 32(%rax)
+; SSE-NEXT: movq %r10, 32(%rax)
; SSE-NEXT: movq %rsi, 24(%rax)
; SSE-NEXT: xorps %xmm0, %xmm0
; SSE-NEXT: movaps %xmm0, (%rax)
@@ -1364,11 +1410,16 @@ define i512 @shl_i512_200(i512 %a0) nounwind {
; AVX2-NEXT: movq %rdi, %rax
; AVX2-NEXT: movq %rsi, %rdi
; AVX2-NEXT: shrdq $56, %rdx, %rdi
-; AVX2-NEXT: shrdq $56, %rcx, %rdx
+; AVX2-NEXT: movq %rcx, %r10
+; AVX2-NEXT: shlq $8, %r10
+; AVX2-NEXT: shrq $56, %rdx
+; AVX2-NEXT: orq %r10, %rdx
; AVX2-NEXT: shrdq $56, %r8, %rcx
-; AVX2-NEXT: shldq $8, %r8, %r9
+; AVX2-NEXT: shlq $8, %r9
+; AVX2-NEXT: shrq $56, %r8
+; AVX2-NEXT: orq %r9, %r8
; AVX2-NEXT: shlq $8, %rsi
-; AVX2-NEXT: movq %r9, 56(%rax)
+; AVX2-NEXT: movq %r8, 56(%rax)
; AVX2-NEXT: movq %rcx, 48(%rax)
; AVX2-NEXT: movq %rdx, 40(%rax)
; AVX2-NEXT: movq %rdi, 32(%rax)
@@ -1383,11 +1434,16 @@ define i512 @shl_i512_200(i512 %a0) nounwind {
; AVX512-NEXT: movq %rdi, %rax
; AVX512-NEXT: movq %rsi, %rdi
; AVX512-NEXT: shrdq $56, %rdx, %rdi
-; AVX512-NEXT: shrdq $56, %rcx, %rdx
+; AVX512-NEXT: movq %rcx, %r10
+; AVX512-NEXT: shlq $8, %r10
+; AVX512-NEXT: shrq $56, %rdx
+; AVX512-NEXT: orq %r10, %rdx
; AVX512-NEXT: shrdq $56, %r8, %rcx
-; AVX512-NEXT: shldq $8, %r8, %r9
+; AVX512-NEXT: shlq $8, %r9
+; AVX512-NEXT: shrq $56, %r8
+; AVX512-NEXT: orq %r9, %r8
; AVX512-NEXT: shlq $8, %rsi
-; AVX512-NEXT: movq %r9, 56(%rax)
+; AVX512-NEXT: movq %r8, 56(%rax)
; AVX512-NEXT: movq %rcx, 48(%rax)
; AVX512-NEXT: movq %rdx, 40(%rax)
; AVX512-NEXT: movq %rdi, 32(%rax)
@@ -1405,63 +1461,99 @@ define i512 @lshr_i512_200(i512 %a0) nounwind {
; SSE: # %bb.0:
; SSE-NEXT: movq %rdi, %rax
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rcx
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rdx
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rsi
-; SSE-NEXT: shrdq $8, %r9, %r8
-; SSE-NEXT: shrdq $8, %rsi, %r9
-; SSE-NEXT: shrdq $8, %rcx, %rsi
-; SSE-NEXT: shrdq $8, %rdx, %rcx
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; SSE-NEXT: movq %r9, %rdi
+; SSE-NEXT: shlq $56, %rdi
+; SSE-NEXT: shrq $8, %r8
+; SSE-NEXT: orq %rdi, %r8
+; SSE-NEXT: movq %rdx, %rdi
+; SSE-NEXT: shlq $56, %rdi
+; SSE-NEXT: shrq $8, %r9
+; SSE-NEXT: orq %rdi, %r9
+; SSE-NEXT: movq %rcx, %rdi
+; SSE-NEXT: shlq $56, %rdi
; SSE-NEXT: shrq $8, %rdx
+; SSE-NEXT: orq %rdi, %rdx
+; SSE-NEXT: movq %rsi, %rdi
+; SSE-NEXT: shlq $56, %rdi
+; SSE-NEXT: shrq $8, %rcx
+; SSE-NEXT: orq %rdi, %rcx
+; SSE-NEXT: shrq $8, %rsi
; SSE-NEXT: xorps %xmm0, %xmm0
-; SSE-NEXT: movups %xmm0, 40(%rdi)
-; SSE-NEXT: movq %rdx, 32(%rdi)
-; SSE-NEXT: movq %rcx, 24(%rdi)
-; SSE-NEXT: movq %rsi, 16(%rdi)
-; SSE-NEXT: movq %r9, 8(%rdi)
-; SSE-NEXT: movq %r8, (%rdi)
-; SSE-NEXT: movq $0, 56(%rdi)
+; SSE-NEXT: movups %xmm0, 40(%rax)
+; SSE-NEXT: movq %rsi, 32(%rax)
+; SSE-NEXT: movq %rcx, 24(%rax)
+; SSE-NEXT: movq %rdx, 16(%rax)
+; SSE-NEXT: movq %r9, 8(%rax)
+; SSE-NEXT: movq %r8, (%rax)
+; SSE-NEXT: movq $0, 56(%rax)
; SSE-NEXT: retq
;
; AVX2-LABEL: lshr_i512_200:
; AVX2: # %bb.0:
; AVX2-NEXT: movq %rdi, %rax
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rcx
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rsi
-; AVX2-NEXT: shrdq $8, %r9, %r8
-; AVX2-NEXT: shrdq $8, %rsi, %r9
-; AVX2-NEXT: shrdq $8, %rcx, %rsi
-; AVX2-NEXT: shrdq $8, %rdx, %rcx
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; AVX2-NEXT: movq %r9, %rdi
+; AVX2-NEXT: shlq $56, %rdi
+; AVX2-NEXT: shrq $8, %r8
+; AVX2-NEXT: orq %rdi, %r8
+; AVX2-NEXT: movq %rdx, %rdi
+; AVX2-NEXT: shlq $56, %rdi
+; AVX2-NEXT: shrq $8, %r9
+; AVX2-NEXT: orq %rdi, %r9
+; AVX2-NEXT: movq %rcx, %rdi
+; AVX2-NEXT: shlq $56, %rdi
; AVX2-NEXT: shrq $8, %rdx
+; AVX2-NEXT: orq %rdi, %rdx
+; AVX2-NEXT: movq %rsi, %rdi
+; AVX2-NEXT: shlq $56, %rdi
+; AVX2-NEXT: shrq $8, %rcx
+; AVX2-NEXT: orq %rdi, %rcx
+; AVX2-NEXT: shrq $8, %rsi
; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: vmovups %xmm0, 40(%rdi)
-; AVX2-NEXT: movq %rdx, 32(%rdi)
-; AVX2-NEXT: movq %rcx, 24(%rdi)
-; AVX2-NEXT: movq %rsi, 16(%rdi)
-; AVX2-NEXT: movq %r9, 8(%rdi)
-; AVX2-NEXT: movq %r8, (%rdi)
-; AVX2-NEXT: movq $0, 56(%rdi)
+; AVX2-NEXT: vmovups %xmm0, 40(%rax)
+; AVX2-NEXT: movq %rsi, 32(%rax)
+; AVX2-NEXT: movq %rcx, 24(%rax)
+; AVX2-NEXT: movq %rdx, 16(%rax)
+; AVX2-NEXT: movq %r9, 8(%rax)
+; AVX2-NEXT: movq %r8, (%rax)
+; AVX2-NEXT: movq $0, 56(%rax)
; AVX2-NEXT: retq
;
; AVX512-LABEL: lshr_i512_200:
; AVX512: # %bb.0:
; AVX512-NEXT: movq %rdi, %rax
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rcx
-; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rdx
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rsi
-; AVX512-NEXT: shrdq $8, %r9, %r8
-; AVX512-NEXT: shrdq $8, %rsi, %r9
-; AVX512-NEXT: shrdq $8, %rcx, %rsi
-; AVX512-NEXT: shrdq $8, %rdx, %rcx
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; AVX512-NEXT: movq %r9, %rdi
+; AVX512-NEXT: shlq $56, %rdi
+; AVX512-NEXT: shrq $8, %r8
+; AVX512-NEXT: orq %rdi, %r8
+; AVX512-NEXT: movq %rdx, %rdi
+; AVX512-NEXT: shlq $56, %rdi
+; AVX512-NEXT: shrq $8, %r9
+; AVX512-NEXT: orq %rdi, %r9
+; AVX512-NEXT: movq %rcx, %rdi
+; AVX512-NEXT: shlq $56, %rdi
; AVX512-NEXT: shrq $8, %rdx
+; AVX512-NEXT: orq %rdi, %rdx
+; AVX512-NEXT: movq %rsi, %rdi
+; AVX512-NEXT: shlq $56, %rdi
+; AVX512-NEXT: shrq $8, %rcx
+; AVX512-NEXT: orq %rdi, %rcx
+; AVX512-NEXT: shrq $8, %rsi
; AVX512-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; AVX512-NEXT: vmovups %xmm0, 40(%rdi)
-; AVX512-NEXT: movq %rdx, 32(%rdi)
-; AVX512-NEXT: movq %rcx, 24(%rdi)
-; AVX512-NEXT: movq %rsi, 16(%rdi)
-; AVX512-NEXT: movq %r9, 8(%rdi)
-; AVX512-NEXT: movq %r8, (%rdi)
-; AVX512-NEXT: movq $0, 56(%rdi)
+; AVX512-NEXT: vmovups %xmm0, 40(%rax)
+; AVX512-NEXT: movq %rsi, 32(%rax)
+; AVX512-NEXT: movq %rcx, 24(%rax)
+; AVX512-NEXT: movq %rdx, 16(%rax)
+; AVX512-NEXT: movq %r9, 8(%rax)
+; AVX512-NEXT: movq %r8, (%rax)
+; AVX512-NEXT: movq $0, 56(%rax)
; AVX512-NEXT: retq
%r = lshr i512 %a0, 200
ret i512 %r
@@ -1472,21 +1564,33 @@ define i512 @ashr_i512_200(i512 %a0) nounwind {
; CHECK: # %bb.0:
; CHECK-NEXT: movq %rdi, %rax
; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rcx
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rdx
; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rsi
-; CHECK-NEXT: shrdq $8, %r9, %r8
-; CHECK-NEXT: shrdq $8, %rsi, %r9
-; CHECK-NEXT: shrdq $8, %rcx, %rsi
-; CHECK-NEXT: shrdq $8, %rdx, %rcx
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; CHECK-NEXT: movq %r9, %rdi
+; CHECK-NEXT: shlq $56, %rdi
+; CHECK-NEXT: shrq $8, %r8
+; CHECK-NEXT: orq %rdi, %r8
; CHECK-NEXT: movq %rdx, %rdi
+; CHECK-NEXT: shlq $56, %rdi
+; CHECK-NEXT: shrq $8, %r9
+; CHECK-NEXT: orq %rdi, %r9
+; CHECK-NEXT: movq %rcx, %rdi
+; CHECK-NEXT: shlq $56, %rdi
+; CHECK-NEXT: shrq $8, %rdx
+; CHECK-NEXT: orq %rdi, %rdx
+; CHECK-NEXT: movq %rsi, %rdi
+; CHECK-NEXT: shlq $56, %rdi
+; CHECK-NEXT: shrq $8, %rcx
+; CHECK-NEXT: orq %rdi, %rcx
+; CHECK-NEXT: movq %rsi, %rdi
; CHECK-NEXT: sarq $8, %rdi
-; CHECK-NEXT: sarq $63, %rdx
-; CHECK-NEXT: movq %rdx, 56(%rax)
-; CHECK-NEXT: movq %rdx, 48(%rax)
-; CHECK-NEXT: movq %rdx, 40(%rax)
+; CHECK-NEXT: sarq $63, %rsi
+; CHECK-NEXT: movq %rsi, 56(%rax)
+; CHECK-NEXT: movq %rsi, 48(%rax)
+; CHECK-NEXT: movq %rsi, 40(%rax)
; CHECK-NEXT: movq %rdi, 32(%rax)
; CHECK-NEXT: movq %rcx, 24(%rax)
-; CHECK-NEXT: movq %rsi, 16(%rax)
+; CHECK-NEXT: movq %rdx, 16(%rax)
; CHECK-NEXT: movq %r9, 8(%rax)
; CHECK-NEXT: movq %r8, (%rax)
; CHECK-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/shift-mask.ll b/llvm/test/CodeGen/X86/shift-mask.ll
index 604f5c19b92e5..e3dbbdd0402e3 100644
--- a/llvm/test/CodeGen/X86/shift-mask.ll
+++ b/llvm/test/CodeGen/X86/shift-mask.ll
@@ -259,8 +259,11 @@ define i64 @test_i64_shl_lshr_1(i64 %a0) {
; X86-LABEL: test_i64_shl_lshr_1:
; X86: # %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: shldl $2, %eax, %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: shll $2, %ecx
+; X86-NEXT: movl %eax, %edx
+; X86-NEXT: shrl $30, %edx
+; X86-NEXT: orl %ecx, %edx
; X86-NEXT: shll $2, %eax
; X86-NEXT: andl $-32, %eax
; X86-NEXT: retl
@@ -611,10 +614,12 @@ define i64 @test_i64_lshr_lshr_1(i64 %a0) {
define i64 @test_i64_lshr_lshr_2(i64 %a0) {
; X86-LABEL: test_i64_lshr_lshr_2:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: shldl $2, %eax, %edx
-; X86-NEXT: shll $2, %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: leal (,%edx,4), %eax
+; X86-NEXT: shll $2, %ecx
+; X86-NEXT: shrl $30, %edx
+; X86-NEXT: orl %ecx, %edx
; X86-NEXT: andl $536870911, %edx # imm = 0x1FFFFFFF
; X86-NEXT: retl
;
diff --git a/llvm/test/CodeGen/X86/shld-machine-combiner.mir b/llvm/test/CodeGen/X86/shld-machine-combiner.mir
new file mode 100644
index 0000000000000..fceee416118e1
--- /dev/null
+++ b/llvm/test/CodeGen/X86/shld-machine-combiner.mir
@@ -0,0 +1,100 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=x86_64-gnu-linux -mcpu=sandybridge -run-pass=machine-combiner %s -o - | FileCheck %s --check-prefixes=FAST-SHLD
+# RUN: llc -mtriple=x86_64-gnu-linux -mcpu=znver1 -run-pass=machine-combiner %s -o - | FileCheck %s --check-prefixes=SLOW-SHLD
+
+---
+name: rri32
+alignment: 16
+debugInstrRef: true
+body: |
+ bb.0:
+ liveins: $eax, $ebx
+ ; FAST-SHLD-LABEL: name: rri32
+ ; FAST-SHLD: liveins: $eax, $ebx
+ ; FAST-SHLD-NEXT: {{ $}}
+ ; FAST-SHLD-NEXT: [[SHLD32rri8_:%[0-9]+]]:gr32 = SHLD32rri8 $eax, $ebx, 2, implicit-def $eflags
+ ; FAST-SHLD-NEXT: RET [[SHLD32rri8_]]
+ ;
+ ; SLOW-SHLD-LABEL: name: rri32
+ ; SLOW-SHLD: liveins: $eax, $ebx
+ ; SLOW-SHLD-NEXT: {{ $}}
+ ; SLOW-SHLD-NEXT: [[SHL32ri:%[0-9]+]]:gr32 = SHL32ri $eax, 2, implicit-def $eflags
+ ; SLOW-SHLD-NEXT: [[SHR32ri:%[0-9]+]]:gr32 = SHR32ri $ebx, 30, implicit-def $eflags
+ ; SLOW-SHLD-NEXT: [[OR32rr:%[0-9]+]]:gr32 = OR32rr [[SHL32ri]], [[SHR32ri]], implicit-def $eflags
+ ; SLOW-SHLD-NEXT: RET [[OR32rr]]
+ %3:gr32 = SHLD32rri8 $eax, $ebx, 2, implicit-def $eflags
+ RET %3
+...
+
+---
+name: rri64
+alignment: 16
+debugInstrRef: true
+body: |
+ bb.0:
+ liveins: $rax, $rbx
+ ; FAST-SHLD-LABEL: name: rri64
+ ; FAST-SHLD: liveins: $rax, $rbx
+ ; FAST-SHLD-NEXT: {{ $}}
+ ; FAST-SHLD-NEXT: [[SHLD64rri8_:%[0-9]+]]:gr64 = SHLD64rri8 $rax, $rbx, 2, implicit-def $eflags
+ ; FAST-SHLD-NEXT: RET [[SHLD64rri8_]]
+ ;
+ ; SLOW-SHLD-LABEL: name: rri64
+ ; SLOW-SHLD: liveins: $rax, $rbx
+ ; SLOW-SHLD-NEXT: {{ $}}
+ ; SLOW-SHLD-NEXT: [[SHL64ri:%[0-9]+]]:gr64 = SHL64ri $rax, 2, implicit-def $eflags
+ ; SLOW-SHLD-NEXT: [[SHR64ri:%[0-9]+]]:gr64 = SHR64ri $rbx, 62, implicit-def $eflags
+ ; SLOW-SHLD-NEXT: [[OR64rr:%[0-9]+]]:gr64 = OR64rr [[SHL64ri]], [[SHR64ri]], implicit-def $eflags
+ ; SLOW-SHLD-NEXT: RET [[OR64rr]]
+ %3:gr64 = SHLD64rri8 $rax, $rbx, 2, implicit-def $eflags
+ RET %3
+...
+
+---
+name: mri32
+alignment: 16
+debugInstrRef: true
+body: |
+ bb.0:
+ liveins: $rax, $ebx
+ ; FAST-SHLD-LABEL: name: mri32
+ ; FAST-SHLD: liveins: $rax, $ebx
+ ; FAST-SHLD-NEXT: {{ $}}
+ ; FAST-SHLD-NEXT: SHLD32mri8 $rax, 1, $noreg, 0, $noreg, $ebx, 2, implicit-def $eflags
+ ; FAST-SHLD-NEXT: RET 0
+ ;
+ ; SLOW-SHLD-LABEL: name: mri32
+ ; SLOW-SHLD: liveins: $rax, $ebx
+ ; SLOW-SHLD-NEXT: {{ $}}
+ ; SLOW-SHLD-NEXT: SHL32mi $rax, 1, $noreg, 0, $noreg, 2, implicit-def $eflags
+ ; SLOW-SHLD-NEXT: [[SHR32ri:%[0-9]+]]:gr32_abcd_and_gr32_bsi = SHR32ri $ebx, 30, implicit-def $eflags
+ ; SLOW-SHLD-NEXT: OR32mr $rax, 1, $noreg, 0, $noreg, [[SHR32ri]], implicit-def $eflags
+ ; SLOW-SHLD-NEXT: RET 0
+ SHLD32mri8 $rax, 1, $noreg, 0, $noreg, $ebx, 2, implicit-def $eflags
+ RET 0
+...
+
+---
+name: mri64
+alignment: 16
+debugInstrRef: true
+body: |
+ bb.0:
+ liveins: $rax, $ebx
+
+ ; FAST-SHLD-LABEL: name: mri64
+ ; FAST-SHLD: liveins: $rax, $ebx
+ ; FAST-SHLD-NEXT: {{ $}}
+ ; FAST-SHLD-NEXT: SHLD64mri8 $rax, 1, $noreg, 0, $noreg, $rbx, 2, implicit-def $eflags
+ ; FAST-SHLD-NEXT: RET 0
+ ;
+ ; SLOW-SHLD-LABEL: name: mri64
+ ; SLOW-SHLD: liveins: $rax, $ebx
+ ; SLOW-SHLD-NEXT: {{ $}}
+ ; SLOW-SHLD-NEXT: SHL64mi $rax, 1, $noreg, 0, $noreg, 2, implicit-def $eflags
+ ; SLOW-SHLD-NEXT: [[SHR64ri:%[0-9]+]]:gr64_with_sub_32bit_in_gr32_abcd_and_gr32_bsi = SHR64ri $rbx, 62, implicit-def $eflags
+ ; SLOW-SHLD-NEXT: OR64mr $rax, 1, $noreg, 0, $noreg, [[SHR64ri]], implicit-def $eflags
+ ; SLOW-SHLD-NEXT: RET 0
+ SHLD64mri8 $rax, 1, $noreg, 0, $noreg, $rbx, 2, implicit-def $eflags
+ RET 0
+...
diff --git a/llvm/test/CodeGen/X86/udiv_fix.ll b/llvm/test/CodeGen/X86/udiv_fix.ll
index 82dfeeee13293..8f613274e6e24 100644
--- a/llvm/test/CodeGen/X86/udiv_fix.ll
+++ b/llvm/test/CodeGen/X86/udiv_fix.ll
@@ -163,8 +163,11 @@ define i64 @func5(i64 %x, i64 %y) nounwind {
; X86-NEXT: movl %ecx, %edx
; X86-NEXT: shrl %edx
; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NEXT: shldl $31, %eax, %ecx
-; X86-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NEXT: shll $31, %ecx
+; X86-NEXT: movl %eax, %edx
+; X86-NEXT: shrl %edx
+; X86-NEXT: orl %ecx, %edx
+; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X86-NEXT: shll $31, %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: leal {{[0-9]+}}(%esp), %eax
diff --git a/llvm/test/CodeGen/X86/udiv_fix_sat.ll b/llvm/test/CodeGen/X86/udiv_fix_sat.ll
index d1e9145c4ccee..9d57ac8b6fb99 100644
--- a/llvm/test/CodeGen/X86/udiv_fix_sat.ll
+++ b/llvm/test/CodeGen/X86/udiv_fix_sat.ll
@@ -172,10 +172,12 @@ define i64 @func5(i64 %x, i64 %y) nounwind {
; X64: # %bb.0:
; X64-NEXT: pushq %rax
; X64-NEXT: movq %rsi, %rdx
-; X64-NEXT: leaq (%rdi,%rdi), %rax
-; X64-NEXT: movq %rdi, %rsi
-; X64-NEXT: shrq $63, %rsi
-; X64-NEXT: shldq $31, %rax, %rsi
+; X64-NEXT: leaq (%rdi,%rdi), %rsi
+; X64-NEXT: movq %rdi, %rax
+; X64-NEXT: shrq $63, %rax
+; X64-NEXT: shlq $31, %rax
+; X64-NEXT: shrq $33, %rsi
+; X64-NEXT: orq %rax, %rsi
; X64-NEXT: shlq $32, %rdi
; X64-NEXT: xorl %ecx, %ecx
; X64-NEXT: callq __udivti3 at PLT
@@ -204,8 +206,11 @@ define i64 @func5(i64 %x, i64 %y) nounwind {
; X86-NEXT: movl %ecx, %edx
; X86-NEXT: shrl %edx
; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NEXT: shldl $31, %eax, %ecx
-; X86-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NEXT: shll $31, %ecx
+; X86-NEXT: movl %eax, %edx
+; X86-NEXT: shrl %edx
+; X86-NEXT: orl %ecx, %edx
+; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X86-NEXT: shll $31, %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: leal {{[0-9]+}}(%esp), %eax
diff --git a/llvm/test/CodeGen/X86/vector-sext.ll b/llvm/test/CodeGen/X86/vector-sext.ll
index 6ce34991050ac..d05273b219f5d 100644
--- a/llvm/test/CodeGen/X86/vector-sext.ll
+++ b/llvm/test/CodeGen/X86/vector-sext.ll
@@ -3692,22 +3692,24 @@ define <4 x i32> @sext_4i17_to_4i32(ptr %ptr) {
;
; X86-SSE2-LABEL: sext_4i17_to_4i32:
; X86-SSE2: # %bb.0:
-; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-SSE2-NEXT: movl (%edx), %eax
-; X86-SSE2-NEXT: movl 4(%edx), %ecx
-; X86-SSE2-NEXT: movl 8(%edx), %edx
-; X86-SSE2-NEXT: shldl $13, %ecx, %edx
-; X86-SSE2-NEXT: movd %edx, %xmm0
-; X86-SSE2-NEXT: pslld $15, %xmm0
-; X86-SSE2-NEXT: psrad $15, %xmm0
-; X86-SSE2-NEXT: movd %ecx, %xmm1
+; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE2-NEXT: movl (%ecx), %eax
+; X86-SSE2-NEXT: movl 4(%ecx), %edx
+; X86-SSE2-NEXT: movl 8(%ecx), %ecx
+; X86-SSE2-NEXT: shll $13, %ecx
+; X86-SSE2-NEXT: movd %edx, %xmm1
+; X86-SSE2-NEXT: movd %eax, %xmm0
+; X86-SSE2-NEXT: shrdl $17, %edx, %eax
+; X86-SSE2-NEXT: shrl $19, %edx
+; X86-SSE2-NEXT: orl %ecx, %edx
+; X86-SSE2-NEXT: movd %edx, %xmm2
+; X86-SSE2-NEXT: pslld $15, %xmm2
+; X86-SSE2-NEXT: psrad $15, %xmm2
; X86-SSE2-NEXT: pslld $13, %xmm1
; X86-SSE2-NEXT: psrad $15, %xmm1
-; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; X86-SSE2-NEXT: movd %eax, %xmm0
+; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
; X86-SSE2-NEXT: pslld $15, %xmm0
; X86-SSE2-NEXT: psrad $15, %xmm0
-; X86-SSE2-NEXT: shrdl $17, %ecx, %eax
; X86-SSE2-NEXT: movd %eax, %xmm2
; X86-SSE2-NEXT: pslld $15, %xmm2
; X86-SSE2-NEXT: psrad $15, %xmm2
@@ -3727,14 +3729,17 @@ define <4 x i32> @sext_4i17_to_4i32(ptr %ptr) {
; X86-SSE41-NEXT: pslld $15, %xmm0
; X86-SSE41-NEXT: psrad $15, %xmm0
; X86-SSE41-NEXT: pinsrd $1, %edx, %xmm0
+; X86-SSE41-NEXT: movl %eax, %edx
+; X86-SSE41-NEXT: shll $13, %edx
+; X86-SSE41-NEXT: sarl $15, %edx
+; X86-SSE41-NEXT: pinsrd $2, %edx, %xmm0
; X86-SSE41-NEXT: movl 8(%ecx), %ecx
-; X86-SSE41-NEXT: shldl $13, %eax, %ecx
-; X86-SSE41-NEXT: shll $13, %eax
+; X86-SSE41-NEXT: shll $13, %ecx
+; X86-SSE41-NEXT: shrl $19, %eax
+; X86-SSE41-NEXT: orl %ecx, %eax
+; X86-SSE41-NEXT: shll $15, %eax
; X86-SSE41-NEXT: sarl $15, %eax
-; X86-SSE41-NEXT: pinsrd $2, %eax, %xmm0
-; X86-SSE41-NEXT: shll $15, %ecx
-; X86-SSE41-NEXT: sarl $15, %ecx
-; X86-SSE41-NEXT: pinsrd $3, %ecx, %xmm0
+; X86-SSE41-NEXT: pinsrd $3, %eax, %xmm0
; X86-SSE41-NEXT: retl
%a = load <4 x i17>, ptr %ptr
%b = sext <4 x i17> %a to <4 x i32>
diff --git a/llvm/test/CodeGen/X86/x86-64-double-precision-shift-left.ll b/llvm/test/CodeGen/X86/x86-64-double-precision-shift-left.ll
index cea7162ebcc62..1c7f6045961fe 100644
--- a/llvm/test/CodeGen/X86/x86-64-double-precision-shift-left.ll
+++ b/llvm/test/CodeGen/X86/x86-64-double-precision-shift-left.ll
@@ -13,8 +13,9 @@
define i64 @lshift1(i64 %a, i64 %b) nounwind readnone uwtable {
; CHECK-LABEL: lshift1:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: movq %rdi, %rax
-; CHECK-NEXT: shldq $1, %rsi, %rax
+; CHECK-NEXT: leaq (,%rdi,2), %rax
+; CHECK-NEXT: shrq $63, %rsi
+; CHECK-NEXT: orq %rsi, %rax
; CHECK-NEXT: retq
entry:
%shl = shl i64 %a, 1
@@ -31,8 +32,9 @@ entry:
define i64 @lshift2(i64 %a, i64 %b) nounwind readnone uwtable {
; CHECK-LABEL: lshift2:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: movq %rdi, %rax
-; CHECK-NEXT: shldq $2, %rsi, %rax
+; CHECK-NEXT: leaq (,%rdi,4), %rax
+; CHECK-NEXT: shrq $62, %rsi
+; CHECK-NEXT: orq %rsi, %rax
; CHECK-NEXT: retq
entry:
%shl = shl i64 %a, 2
@@ -49,8 +51,10 @@ entry:
define i64 @lshift7(i64 %a, i64 %b) nounwind readnone uwtable {
; CHECK-LABEL: lshift7:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: movq %rdi, %rax
-; CHECK-NEXT: shldq $7, %rsi, %rax
+; CHECK-NEXT: movq %rsi, %rax
+; CHECK-NEXT: shlq $7, %rdi
+; CHECK-NEXT: shrq $57, %rax
+; CHECK-NEXT: orq %rdi, %rax
; CHECK-NEXT: retq
entry:
%shl = shl i64 %a, 7
@@ -67,8 +71,10 @@ entry:
define i64 @lshift63(i64 %a, i64 %b) nounwind readnone uwtable {
; CHECK-LABEL: lshift63:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: movq %rdi, %rax
-; CHECK-NEXT: shldq $63, %rsi, %rax
+; CHECK-NEXT: movq %rsi, %rax
+; CHECK-NEXT: shlq $63, %rdi
+; CHECK-NEXT: shrq %rax
+; CHECK-NEXT: orq %rdi, %rax
; CHECK-NEXT: retq
entry:
%shl = shl i64 %a, 63
diff --git a/llvm/test/CodeGen/X86/x86-64-double-precision-shift-right.ll b/llvm/test/CodeGen/X86/x86-64-double-precision-shift-right.ll
index 420306881e06d..f2687599e3c6f 100644
--- a/llvm/test/CodeGen/X86/x86-64-double-precision-shift-right.ll
+++ b/llvm/test/CodeGen/X86/x86-64-double-precision-shift-right.ll
@@ -14,7 +14,9 @@ define i64 @rshift1(i64 %a, i64 %b) nounwind readnone uwtable {
; CHECK-LABEL: rshift1:
; CHECK: # %bb.0:
; CHECK-NEXT: movq %rdi, %rax
-; CHECK-NEXT: shrdq $1, %rsi, %rax
+; CHECK-NEXT: shlq $63, %rsi
+; CHECK-NEXT: shrq %rax
+; CHECK-NEXT: orq %rsi, %rax
; CHECK-NEXT: retq
%1 = lshr i64 %a, 1
%2 = shl i64 %b, 63
@@ -31,7 +33,9 @@ define i64 @rshift2(i64 %a, i64 %b) nounwind readnone uwtable {
; CHECK-LABEL: rshift2:
; CHECK: # %bb.0:
; CHECK-NEXT: movq %rdi, %rax
-; CHECK-NEXT: shrdq $2, %rsi, %rax
+; CHECK-NEXT: shlq $62, %rsi
+; CHECK-NEXT: shrq $2, %rax
+; CHECK-NEXT: orq %rsi, %rax
; CHECK-NEXT: retq
%1 = lshr i64 %a, 2
%2 = shl i64 %b, 62
@@ -48,7 +52,9 @@ define i64 @rshift7(i64 %a, i64 %b) nounwind readnone uwtable {
; CHECK-LABEL: rshift7:
; CHECK: # %bb.0:
; CHECK-NEXT: movq %rdi, %rax
-; CHECK-NEXT: shrdq $7, %rsi, %rax
+; CHECK-NEXT: shlq $57, %rsi
+; CHECK-NEXT: shrq $7, %rax
+; CHECK-NEXT: orq %rsi, %rax
; CHECK-NEXT: retq
%1 = lshr i64 %a, 7
%2 = shl i64 %b, 57
@@ -64,8 +70,9 @@ define i64 @rshift7(i64 %a, i64 %b) nounwind readnone uwtable {
define i64 @rshift63(i64 %a, i64 %b) nounwind readnone uwtable {
; CHECK-LABEL: rshift63:
; CHECK: # %bb.0:
-; CHECK-NEXT: movq %rdi, %rax
-; CHECK-NEXT: shrdq $63, %rsi, %rax
+; CHECK-NEXT: leaq (,%rsi,2), %rax
+; CHECK-NEXT: shrq $63, %rdi
+; CHECK-NEXT: orq %rdi, %rax
; CHECK-NEXT: retq
%1 = lshr i64 %a, 63
%2 = shl i64 %b, 1
diff --git a/llvm/test/CodeGen/X86/x86-64-double-shifts-Oz-Os-O2.ll b/llvm/test/CodeGen/X86/x86-64-double-shifts-Oz-Os-O2.ll
index 46afc8f88e086..e9444734884c6 100644
--- a/llvm/test/CodeGen/X86/x86-64-double-shifts-Oz-Os-O2.ll
+++ b/llvm/test/CodeGen/X86/x86-64-double-shifts-Oz-Os-O2.ll
@@ -14,8 +14,10 @@
define i64 @_Z8lshift10mm(i64 %a, i64 %b) #0 {
; CHECK-LABEL: _Z8lshift10mm:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: movq %rdi, %rax
-; CHECK-NEXT: shldq $10, %rsi, %rax
+; CHECK-NEXT: movq %rsi, %rax
+; CHECK-NEXT: shlq $10, %rdi
+; CHECK-NEXT: shrq $54, %rax
+; CHECK-NEXT: orq %rdi, %rax
; CHECK-NEXT: retq
entry:
%shl = shl i64 %a, 10
@@ -40,8 +42,10 @@ attributes #0 = { minsize nounwind readnone uwtable "less-precise-fpmad"="false"
define i64 @_Z8lshift11mm(i64 %a, i64 %b) #1 {
; CHECK-LABEL: _Z8lshift11mm:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: movq %rdi, %rax
-; CHECK-NEXT: shldq $11, %rsi, %rax
+; CHECK-NEXT: movq %rsi, %rax
+; CHECK-NEXT: shlq $11, %rdi
+; CHECK-NEXT: shrq $53, %rax
+; CHECK-NEXT: orq %rdi, %rax
; CHECK-NEXT: retq
entry:
%shl = shl i64 %a, 11
@@ -53,8 +57,10 @@ entry:
define i64 @_Z8lshift11mm_pgso(i64 %a, i64 %b) !prof !14 {
; CHECK-LABEL: _Z8lshift11mm_pgso:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: movq %rdi, %rax
-; CHECK-NEXT: shldq $11, %rsi, %rax
+; CHECK-NEXT: movq %rsi, %rax
+; CHECK-NEXT: shlq $11, %rdi
+; CHECK-NEXT: shrq $53, %rax
+; CHECK-NEXT: orq %rdi, %rax
; CHECK-NEXT: retq
entry:
%shl = shl i64 %a, 11
@@ -78,8 +84,10 @@ attributes #1 = { nounwind optsize readnone uwtable "less-precise-fpmad"="false"
define i64 @_Z8lshift12mm(i64 %a, i64 %b) #2 {
; CHECK-LABEL: _Z8lshift12mm:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: movq %rdi, %rax
-; CHECK-NEXT: shldq $12, %rsi, %rax
+; CHECK-NEXT: movq %rsi, %rax
+; CHECK-NEXT: shlq $12, %rdi
+; CHECK-NEXT: shrq $52, %rax
+; CHECK-NEXT: orq %rdi, %rax
; CHECK-NEXT: retq
entry:
%shl = shl i64 %a, 12
diff --git a/llvm/test/CodeGen/X86/xor-lea.ll b/llvm/test/CodeGen/X86/xor-lea.ll
index d50752e48d293..2be7811adfd9a 100644
--- a/llvm/test/CodeGen/X86/xor-lea.ll
+++ b/llvm/test/CodeGen/X86/xor-lea.ll
@@ -364,10 +364,12 @@ define i32 @xor_bigshl_sminval_i32(i32 %x) {
define i64 @xor_shl_sminval_i64(i64 %x) {
; X86-LABEL: xor_shl_sminval_i64:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: shldl $2, %eax, %edx
-; X86-NEXT: shll $2, %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: leal (,%edx,4), %eax
+; X86-NEXT: shll $2, %ecx
+; X86-NEXT: shrl $30, %edx
+; X86-NEXT: orl %ecx, %edx
; X86-NEXT: addl $-2147483648, %edx # imm = 0x80000000
; X86-NEXT: retl
;
>From b201abf15493c1c640ef1a610c8116fc75ced90f Mon Sep 17 00:00:00 2001
From: stomfaig <stomfaig at gmail.com>
Date: Mon, 16 Mar 2026 22:24:22 +0000
Subject: [PATCH 03/11] correct latency propagation
---
llvm/lib/Target/X86/X86InstrInfo.cpp | 9 +-
llvm/test/CodeGen/X86/2008-07-11-SHLBy1.ll | 6 +-
llvm/test/CodeGen/X86/apx/shld.ll | 8 +-
llvm/test/CodeGen/X86/avgflooru-i128.ll | 37 +-
llvm/test/CodeGen/X86/avgflooru-scalar.ll | 48 +-
llvm/test/CodeGen/X86/bitreverse.ll | 537 +++++++-----------
llvm/test/CodeGen/X86/bswap.ll | 118 +---
llvm/test/CodeGen/X86/clmul.ll | 24 +-
llvm/test/CodeGen/X86/combine-bswap.ll | 7 +-
.../CodeGen/X86/const-shift-of-constmasked.ll | 41 +-
llvm/test/CodeGen/X86/dagcombine-cse.ll | 5 +-
.../X86/div-rem-pair-recomposition-signed.ll | 453 ++++++++-------
.../div-rem-pair-recomposition-unsigned.ll | 449 ++++++++-------
llvm/test/CodeGen/X86/div_i129_v_pow2k.ll | 75 +--
llvm/test/CodeGen/X86/divide-by-constant.ll | 30 +-
llvm/test/CodeGen/X86/divmod128.ll | 64 +--
.../CodeGen/X86/expand-large-fp-optnone.ll | 10 +-
llvm/test/CodeGen/X86/freeze-binary.ll | 16 +-
llvm/test/CodeGen/X86/fshl.ll | 31 +-
.../CodeGen/X86/funnel-shift-logic-fold.ll | 22 +-
llvm/test/CodeGen/X86/funnel-shift-rot.ll | 12 +-
llvm/test/CodeGen/X86/funnel-shift.ll | 85 +--
llvm/test/CodeGen/X86/icmp-shift-opt.ll | 74 +--
llvm/test/CodeGen/X86/imul.ll | 15 +-
llvm/test/CodeGen/X86/isel-shift.ll | 29 +-
llvm/test/CodeGen/X86/known-bits.ll | 18 +-
llvm/test/CodeGen/X86/legalize-shl-vec.ll | 242 +++-----
llvm/test/CodeGen/X86/logic-shift.ll | 24 +-
llvm/test/CodeGen/X86/mul-constant-i64.ll | 76 +--
llvm/test/CodeGen/X86/pr43820.ll | 397 +++++++------
llvm/test/CodeGen/X86/pr49162.ll | 10 +-
llvm/test/CodeGen/X86/rotate-add.ll | 29 +-
llvm/test/CodeGen/X86/rotate-extract.ll | 24 +-
llvm/test/CodeGen/X86/rotate.ll | 66 +--
llvm/test/CodeGen/X86/rotate2.ll | 13 +-
llvm/test/CodeGen/X86/scmp.ll | 192 ++++---
llvm/test/CodeGen/X86/sdiv_fix.ll | 7 +-
llvm/test/CodeGen/X86/sdiv_fix_sat.ll | 31 +-
llvm/test/CodeGen/X86/setcc-fsh.ll | 15 +-
llvm/test/CodeGen/X86/shift-i256.ll | 191 +++----
llvm/test/CodeGen/X86/shift-mask.ll | 8 +-
.../CodeGen/X86/shld-machine-combiner.mir | 79 ++-
llvm/test/CodeGen/X86/udiv_fix.ll | 7 +-
llvm/test/CodeGen/X86/udiv_fix_sat.ll | 10 +-
llvm/test/CodeGen/X86/vector-sext.ll | 26 +-
llvm/test/CodeGen/X86/xor-lea.ll | 8 +-
46 files changed, 1530 insertions(+), 2148 deletions(-)
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index c8646fe77d213..c8578540ccb6e 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -10663,14 +10663,16 @@ bool X86InstrInfo::getMachineCombinerPatterns(
}
break;
}
- // We do not support CL variant,
- // as requires a lot of bookeeping
+ // We do not support CL variant, as it requires a lot of bookkeeping.
+ // Only expand when SHLD is slow; on CPUs with fast SHLD the replacement
+ // sequence (SHL + SHR + OR) is not profitable.
case X86::SHLD32rri8:
case X86::SHLD32mri8:
case X86::SHLD64rri8:
case X86::SHLD64mri8: {
Patterns.push_back(X86MachineCombinerPattern::USHLD);
return true;
+ break;
}
}
return TargetInstrInfo::getMachineCombinerPatterns(Root,
@@ -10716,6 +10718,8 @@ genAlternativeShldSequence(MachineInstr &Root, const TargetInstrInfo &TII,
Root.getOperand(0).getReg())
.addReg(ShlReg)
.addReg(ShrReg);
+ InstrIdxForVirtReg.insert({ShlReg, 0});
+ InstrIdxForVirtReg.insert({ShrReg, 1});
InsInstrs.push_back(Shl);
InsInstrs.push_back(Shr);
InsInstrs.push_back(Or);
@@ -10747,6 +10751,7 @@ genAlternativeShldSequence(MachineInstr &Root, const TargetInstrInfo &TII,
.add(Root.getOperand(0 + X86::AddrDisp))
.add(Root.getOperand(0 + X86::AddrSegmentReg))
.addReg(ShrReg);
+ InstrIdxForVirtReg.insert({ShrReg, 1});
InsInstrs.push_back(Shl);
InsInstrs.push_back(Shr);
InsInstrs.push_back(Or);
diff --git a/llvm/test/CodeGen/X86/2008-07-11-SHLBy1.ll b/llvm/test/CodeGen/X86/2008-07-11-SHLBy1.ll
index 70370533cebde..7779d2eb32ab8 100644
--- a/llvm/test/CodeGen/X86/2008-07-11-SHLBy1.ll
+++ b/llvm/test/CodeGen/X86/2008-07-11-SHLBy1.ll
@@ -4,11 +4,9 @@
define i128 @sl(i128 %x) {
; CHECK-LABEL: sl:
; CHECK: # %bb.0:
-; CHECK-NEXT: leaq (,%rsi,2), %rcx
+; CHECK-NEXT: movq %rsi, %rdx
+; CHECK-NEXT: shldq $1, %rdi, %rdx
; CHECK-NEXT: leaq (%rdi,%rdi), %rax
-; CHECK-NEXT: movq %rdi, %rdx
-; CHECK-NEXT: shrq $63, %rdx
-; CHECK-NEXT: orq %rcx, %rdx
; CHECK-NEXT: retq
%t = shl i128 %x, 1
ret i128 %t
diff --git a/llvm/test/CodeGen/X86/apx/shld.ll b/llvm/test/CodeGen/X86/apx/shld.ll
index ee2ead8eaee61..5b99719ba537c 100644
--- a/llvm/test/CodeGen/X86/apx/shld.ll
+++ b/llvm/test/CodeGen/X86/apx/shld.ll
@@ -243,9 +243,7 @@ entry:
define void @shld32mri8_legacy(ptr %ptr, i32 noundef %b) {
; CHECK-LABEL: shld32mri8_legacy:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: shll $12, (%rdi)
-; CHECK-NEXT: shrl $20, %esi
-; CHECK-NEXT: orl %esi, (%rdi)
+; CHECK-NEXT: shldl $12, %esi, (%rdi)
; CHECK-NEXT: retq
entry:
%a = load i32, ptr %ptr
@@ -257,9 +255,7 @@ entry:
define void @shld64mri8_legacy(ptr %ptr, i64 noundef %b) {
; CHECK-LABEL: shld64mri8_legacy:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: shlq $12, (%rdi)
-; CHECK-NEXT: shrq $52, %rsi
-; CHECK-NEXT: orq %rsi, (%rdi)
+; CHECK-NEXT: shldq $12, %rsi, (%rdi)
; CHECK-NEXT: retq
entry:
%a = load i64, ptr %ptr
diff --git a/llvm/test/CodeGen/X86/avgflooru-i128.ll b/llvm/test/CodeGen/X86/avgflooru-i128.ll
index 1cb184406d69a..f0ca5ea5ff255 100644
--- a/llvm/test/CodeGen/X86/avgflooru-i128.ll
+++ b/llvm/test/CodeGen/X86/avgflooru-i128.ll
@@ -10,9 +10,7 @@ define i128 @avgflooru_i128(i128 %x, i128 %y) {
; CHECK-NEXT: setb %cl
; CHECK-NEXT: shrdq $1, %rsi, %rax
; CHECK-NEXT: movzbl %cl, %edx
-; CHECK-NEXT: shlq $63, %rdx
-; CHECK-NEXT: shrq %rsi
-; CHECK-NEXT: orq %rsi, %rdx
+; CHECK-NEXT: shldq $63, %rsi, %rdx
; CHECK-NEXT: retq
start:
%xor = xor i128 %y, %x
@@ -34,10 +32,10 @@ define i128 @avgflooru_i128_multi_use(i128 %x, i128 %y) nounwind {
; CHECK-NEXT: pushq %r12
; CHECK-NEXT: pushq %rbx
; CHECK-NEXT: pushq %rax
-; CHECK-NEXT: movq %rcx, %r15
-; CHECK-NEXT: movq %rdx, %r12
-; CHECK-NEXT: movq %rsi, %rbx
-; CHECK-NEXT: movq %rdi, %r14
+; CHECK-NEXT: movq %rcx, %rbx
+; CHECK-NEXT: movq %rdx, %r14
+; CHECK-NEXT: movq %rsi, %r15
+; CHECK-NEXT: movq %rdi, %r12
; CHECK-NEXT: movq %rdx, %r13
; CHECK-NEXT: xorq %rdi, %r13
; CHECK-NEXT: movq %rcx, %rbp
@@ -50,16 +48,13 @@ define i128 @avgflooru_i128_multi_use(i128 %x, i128 %y) nounwind {
; CHECK-NEXT: movq %r13, %rdi
; CHECK-NEXT: movq %rbp, %rsi
; CHECK-NEXT: callq use at PLT
-; CHECK-NEXT: addq %r12, %r14
-; CHECK-NEXT: adcq %r15, %rbx
+; CHECK-NEXT: addq %r14, %r12
+; CHECK-NEXT: adcq %rbx, %r15
; CHECK-NEXT: setb %al
-; CHECK-NEXT: shrdq $1, %rbx, %r14
-; CHECK-NEXT: movzbl %al, %eax
-; CHECK-NEXT: shlq $63, %rax
-; CHECK-NEXT: shrq %rbx
-; CHECK-NEXT: orq %rax, %rbx
-; CHECK-NEXT: movq %r14, %rax
-; CHECK-NEXT: movq %rbx, %rdx
+; CHECK-NEXT: shrdq $1, %r15, %r12
+; CHECK-NEXT: movzbl %al, %edx
+; CHECK-NEXT: shldq $63, %r15, %rdx
+; CHECK-NEXT: movq %r12, %rax
; CHECK-NEXT: addq $8, %rsp
; CHECK-NEXT: popq %rbx
; CHECK-NEXT: popq %r12
@@ -136,13 +131,11 @@ define <2 x i128> @avgflooru_i128_vec(<2 x i128> %x, <2 x i128> %y) {
; CHECK-NEXT: adcq {{[0-9]+}}(%rsp), %r8
; CHECK-NEXT: setb %dil
; CHECK-NEXT: movzbl %dil, %edi
-; CHECK-NEXT: shlq $63, %rdi
-; CHECK-NEXT: shrdq $1, %r8, %rcx
-; CHECK-NEXT: shrq %r8
-; CHECK-NEXT: orq %rdi, %r8
-; CHECK-NEXT: movq %rcx, 16(%rax)
+; CHECK-NEXT: shldq $63, %r8, %rdi
+; CHECK-NEXT: shldq $63, %rcx, %r8
+; CHECK-NEXT: movq %r8, 16(%rax)
; CHECK-NEXT: movq %rsi, (%rax)
-; CHECK-NEXT: movq %r8, 24(%rax)
+; CHECK-NEXT: movq %rdi, 24(%rax)
; CHECK-NEXT: movq %rdx, 8(%rax)
; CHECK-NEXT: retq
start:
diff --git a/llvm/test/CodeGen/X86/avgflooru-scalar.ll b/llvm/test/CodeGen/X86/avgflooru-scalar.ll
index dac0a9e41b2b1..bb070370316a8 100644
--- a/llvm/test/CodeGen/X86/avgflooru-scalar.ll
+++ b/llvm/test/CodeGen/X86/avgflooru-scalar.ll
@@ -253,16 +253,14 @@ define i32 @test_ext_i32(i32 %a0, i32 %a1) nounwind {
define i64 @test_fixed_i64(i64 %a0, i64 %a1) nounwind {
; X86-LABEL: test_fixed_i64:
; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: addl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: adcl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: setb %cl
-; X86-NEXT: movzbl %cl, %ecx
-; X86-NEXT: shll $31, %ecx
-; X86-NEXT: shrdl $1, %edx, %eax
-; X86-NEXT: shrl %edx
-; X86-NEXT: orl %ecx, %edx
+; X86-NEXT: addl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: adcl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: setb %dl
+; X86-NEXT: movzbl %dl, %edx
+; X86-NEXT: shldl $31, %eax, %edx
+; X86-NEXT: shldl $31, %ecx, %eax
; X86-NEXT: retl
;
; X64-LABEL: test_fixed_i64:
@@ -283,16 +281,14 @@ define i64 @test_fixed_i64(i64 %a0, i64 %a1) nounwind {
define i64 @test_lsb_i64(i64 %a0, i64 %a1) nounwind {
; X86-LABEL: test_lsb_i64:
; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: addl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: adcl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: setb %cl
-; X86-NEXT: movzbl %cl, %ecx
-; X86-NEXT: shll $31, %ecx
-; X86-NEXT: shrdl $1, %edx, %eax
-; X86-NEXT: shrl %edx
-; X86-NEXT: orl %ecx, %edx
+; X86-NEXT: addl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: adcl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: setb %dl
+; X86-NEXT: movzbl %dl, %edx
+; X86-NEXT: shldl $31, %eax, %edx
+; X86-NEXT: shldl $31, %ecx, %eax
; X86-NEXT: retl
;
; X64-LABEL: test_lsb_i64:
@@ -315,16 +311,14 @@ define i64 @test_lsb_i64(i64 %a0, i64 %a1) nounwind {
define i64 @test_ext_i64(i64 %a0, i64 %a1) nounwind {
; X86-LABEL: test_ext_i64:
; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: addl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: adcl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: setb %cl
-; X86-NEXT: movzbl %cl, %ecx
-; X86-NEXT: shll $31, %ecx
-; X86-NEXT: shrdl $1, %edx, %eax
-; X86-NEXT: shrl %edx
-; X86-NEXT: orl %ecx, %edx
+; X86-NEXT: addl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: adcl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: setb %dl
+; X86-NEXT: movzbl %dl, %edx
+; X86-NEXT: shldl $31, %eax, %edx
+; X86-NEXT: shldl $31, %ecx, %eax
; X86-NEXT: retl
;
; X64-LABEL: test_ext_i64:
diff --git a/llvm/test/CodeGen/X86/bitreverse.ll b/llvm/test/CodeGen/X86/bitreverse.ll
index a14c8e564182a..3340056452be7 100644
--- a/llvm/test/CodeGen/X86/bitreverse.ll
+++ b/llvm/test/CodeGen/X86/bitreverse.ll
@@ -700,7 +700,7 @@ define i528 @large_promotion(i528 %A) nounwind {
; X86-NEXT: pushl %ebx
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
-; X86-NEXT: subl $44, %esp
+; X86-NEXT: subl $60, %esp
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
@@ -740,12 +740,8 @@ define i528 @large_promotion(i528 %A) nounwind {
; X86-NEXT: andl $1431655765, %ebx # imm = 0x55555555
; X86-NEXT: shrl %edi
; X86-NEXT: andl $1431655765, %edi # imm = 0x55555555
-; X86-NEXT: leal (%edi,%ebx,2), %ebx
-; X86-NEXT: movl %ebx, %edi
-; X86-NEXT: shll $16, %edi
-; X86-NEXT: shrl $16, %ebp
-; X86-NEXT: orl %edi, %ebp
-; X86-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: leal (%edi,%ebx,2), %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: bswapl %esi
; X86-NEXT: movl %esi, %edi
; X86-NEXT: andl $252645135, %edi # imm = 0xF0F0F0F
@@ -762,12 +758,7 @@ define i528 @large_promotion(i528 %A) nounwind {
; X86-NEXT: andl $1431655765, %edi # imm = 0x55555555
; X86-NEXT: shrl %esi
; X86-NEXT: andl $1431655765, %esi # imm = 0x55555555
-; X86-NEXT: leal (%esi,%edi,2), %edi
-; X86-NEXT: movl %edi, %esi
-; X86-NEXT: shll $16, %esi
-; X86-NEXT: shrl $16, %ebx
-; X86-NEXT: orl %esi, %ebx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: leal (%esi,%edi,2), %ebx
; X86-NEXT: bswapl %edx
; X86-NEXT: movl %edx, %esi
; X86-NEXT: andl $252645135, %esi # imm = 0xF0F0F0F
@@ -784,12 +775,8 @@ define i528 @large_promotion(i528 %A) nounwind {
; X86-NEXT: andl $1431655765, %esi # imm = 0x55555555
; X86-NEXT: shrl %edx
; X86-NEXT: andl $1431655765, %edx # imm = 0x55555555
-; X86-NEXT: leal (%edx,%esi,2), %esi
-; X86-NEXT: movl %esi, %edx
-; X86-NEXT: shll $16, %edx
-; X86-NEXT: shrl $16, %edi
-; X86-NEXT: orl %edx, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: leal (%edx,%esi,2), %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: bswapl %ecx
; X86-NEXT: movl %ecx, %edx
; X86-NEXT: andl $252645135, %edx # imm = 0xF0F0F0F
@@ -806,12 +793,8 @@ define i528 @large_promotion(i528 %A) nounwind {
; X86-NEXT: andl $1431655765, %edx # imm = 0x55555555
; X86-NEXT: shrl %ecx
; X86-NEXT: andl $1431655765, %ecx # imm = 0x55555555
-; X86-NEXT: leal (%ecx,%edx,2), %edx
-; X86-NEXT: movl %edx, %ecx
-; X86-NEXT: shll $16, %ecx
-; X86-NEXT: shrl $16, %esi
-; X86-NEXT: orl %ecx, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: leal (%ecx,%edx,2), %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: bswapl %eax
; X86-NEXT: movl %eax, %ecx
; X86-NEXT: andl $252645135, %ecx # imm = 0xF0F0F0F
@@ -828,12 +811,8 @@ define i528 @large_promotion(i528 %A) nounwind {
; X86-NEXT: andl $1431655765, %ecx # imm = 0x55555555
; X86-NEXT: shrl %eax
; X86-NEXT: andl $1431655765, %eax # imm = 0x55555555
-; X86-NEXT: leal (%eax,%ecx,2), %esi
-; X86-NEXT: movl %esi, %eax
-; X86-NEXT: shll $16, %eax
-; X86-NEXT: shrl $16, %edx
-; X86-NEXT: orl %eax, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: leal (%eax,%ecx,2), %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: bswapl %eax
; X86-NEXT: movl %eax, %ecx
@@ -851,12 +830,8 @@ define i528 @large_promotion(i528 %A) nounwind {
; X86-NEXT: andl $1431655765, %ecx # imm = 0x55555555
; X86-NEXT: shrl %eax
; X86-NEXT: andl $1431655765, %eax # imm = 0x55555555
-; X86-NEXT: leal (%eax,%ecx,2), %edx
-; X86-NEXT: movl %edx, %eax
-; X86-NEXT: shll $16, %eax
-; X86-NEXT: shrl $16, %esi
-; X86-NEXT: orl %eax, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: leal (%eax,%ecx,2), %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: bswapl %eax
; X86-NEXT: movl %eax, %ecx
@@ -874,12 +849,8 @@ define i528 @large_promotion(i528 %A) nounwind {
; X86-NEXT: andl $1431655765, %ecx # imm = 0x55555555
; X86-NEXT: shrl %eax
; X86-NEXT: andl $1431655765, %eax # imm = 0x55555555
-; X86-NEXT: leal (%eax,%ecx,2), %esi
-; X86-NEXT: movl %esi, %eax
-; X86-NEXT: shll $16, %eax
-; X86-NEXT: shrl $16, %edx
-; X86-NEXT: orl %eax, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: leal (%eax,%ecx,2), %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: bswapl %eax
; X86-NEXT: movl %eax, %ecx
@@ -897,12 +868,8 @@ define i528 @large_promotion(i528 %A) nounwind {
; X86-NEXT: andl $1431655765, %ecx # imm = 0x55555555
; X86-NEXT: shrl %eax
; X86-NEXT: andl $1431655765, %eax # imm = 0x55555555
-; X86-NEXT: leal (%eax,%ecx,2), %edx
-; X86-NEXT: movl %edx, %eax
-; X86-NEXT: shll $16, %eax
-; X86-NEXT: shrl $16, %esi
-; X86-NEXT: orl %eax, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: leal (%eax,%ecx,2), %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: bswapl %eax
; X86-NEXT: movl %eax, %ecx
@@ -920,12 +887,7 @@ define i528 @large_promotion(i528 %A) nounwind {
; X86-NEXT: andl $1431655765, %ecx # imm = 0x55555555
; X86-NEXT: shrl %eax
; X86-NEXT: andl $1431655765, %eax # imm = 0x55555555
-; X86-NEXT: leal (%eax,%ecx,2), %esi
-; X86-NEXT: movl %esi, %eax
-; X86-NEXT: shll $16, %eax
-; X86-NEXT: shrl $16, %edx
-; X86-NEXT: orl %eax, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: leal (%eax,%ecx,2), %edi
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: bswapl %eax
; X86-NEXT: movl %eax, %ecx
@@ -943,12 +905,8 @@ define i528 @large_promotion(i528 %A) nounwind {
; X86-NEXT: andl $1431655765, %ecx # imm = 0x55555555
; X86-NEXT: shrl %eax
; X86-NEXT: andl $1431655765, %eax # imm = 0x55555555
-; X86-NEXT: leal (%eax,%ecx,2), %edx
-; X86-NEXT: movl %edx, %eax
-; X86-NEXT: shll $16, %eax
-; X86-NEXT: shrl $16, %esi
-; X86-NEXT: orl %eax, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: leal (%eax,%ecx,2), %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: bswapl %eax
; X86-NEXT: movl %eax, %ecx
@@ -966,12 +924,8 @@ define i528 @large_promotion(i528 %A) nounwind {
; X86-NEXT: andl $1431655765, %ecx # imm = 0x55555555
; X86-NEXT: shrl %eax
; X86-NEXT: andl $1431655765, %eax # imm = 0x55555555
-; X86-NEXT: leal (%eax,%ecx,2), %ebp
-; X86-NEXT: movl %ebp, %eax
-; X86-NEXT: shll $16, %eax
-; X86-NEXT: shrl $16, %edx
-; X86-NEXT: orl %eax, %edx
-; X86-NEXT: movl %edx, (%esp) # 4-byte Spill
+; X86-NEXT: leal (%eax,%ecx,2), %eax
+; X86-NEXT: movl %eax, (%esp) # 4-byte Spill
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: bswapl %eax
; X86-NEXT: movl %eax, %ecx
@@ -989,11 +943,8 @@ define i528 @large_promotion(i528 %A) nounwind {
; X86-NEXT: andl $1431655765, %ecx # imm = 0x55555555
; X86-NEXT: shrl %eax
; X86-NEXT: andl $1431655765, %eax # imm = 0x55555555
-; X86-NEXT: leal (%eax,%ecx,2), %ebx
-; X86-NEXT: movl %ebx, %eax
-; X86-NEXT: shll $16, %eax
-; X86-NEXT: shrl $16, %ebp
-; X86-NEXT: orl %eax, %ebp
+; X86-NEXT: leal (%eax,%ecx,2), %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: bswapl %eax
; X86-NEXT: movl %eax, %ecx
@@ -1011,11 +962,8 @@ define i528 @large_promotion(i528 %A) nounwind {
; X86-NEXT: andl $1431655765, %ecx # imm = 0x55555555
; X86-NEXT: shrl %eax
; X86-NEXT: andl $1431655765, %eax # imm = 0x55555555
-; X86-NEXT: leal (%eax,%ecx,2), %edi
-; X86-NEXT: movl %edi, %eax
-; X86-NEXT: shll $16, %eax
-; X86-NEXT: shrl $16, %ebx
-; X86-NEXT: orl %eax, %ebx
+; X86-NEXT: leal (%eax,%ecx,2), %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: bswapl %eax
; X86-NEXT: movl %eax, %ecx
@@ -1033,11 +981,8 @@ define i528 @large_promotion(i528 %A) nounwind {
; X86-NEXT: andl $1431655765, %ecx # imm = 0x55555555
; X86-NEXT: shrl %eax
; X86-NEXT: andl $1431655765, %eax # imm = 0x55555555
-; X86-NEXT: leal (%eax,%ecx,2), %esi
-; X86-NEXT: movl %esi, %eax
-; X86-NEXT: shll $16, %eax
-; X86-NEXT: shrl $16, %edi
-; X86-NEXT: orl %eax, %edi
+; X86-NEXT: leal (%eax,%ecx,2), %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: bswapl %eax
; X86-NEXT: movl %eax, %ecx
@@ -1055,40 +1000,76 @@ define i528 @large_promotion(i528 %A) nounwind {
; X86-NEXT: andl $1431655765, %ecx # imm = 0x55555555
; X86-NEXT: shrl %eax
; X86-NEXT: andl $1431655765, %eax # imm = 0x55555555
-; X86-NEXT: leal (%eax,%ecx,2), %ecx
-; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: shll $16, %eax
-; X86-NEXT: shrl $16, %esi
-; X86-NEXT: orl %eax, %esi
+; X86-NEXT: leal (%eax,%ecx,2), %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: bswapl %eax
-; X86-NEXT: movl %eax, %edx
-; X86-NEXT: andl $252645135, %edx # imm = 0xF0F0F0F
-; X86-NEXT: shll $4, %edx
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: andl $252645135, %ecx # imm = 0xF0F0F0F
+; X86-NEXT: shll $4, %ecx
; X86-NEXT: shrl $4, %eax
; X86-NEXT: andl $252645135, %eax # imm = 0xF0F0F0F
-; X86-NEXT: orl %edx, %eax
-; X86-NEXT: movl %eax, %edx
-; X86-NEXT: andl $858993459, %edx # imm = 0x33333333
+; X86-NEXT: orl %ecx, %eax
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: andl $858993459, %ecx # imm = 0x33333333
; X86-NEXT: shrl $2, %eax
; X86-NEXT: andl $858993459, %eax # imm = 0x33333333
-; X86-NEXT: leal (%eax,%edx,4), %eax
-; X86-NEXT: movl %eax, %edx
-; X86-NEXT: andl $1431655765, %edx # imm = 0x55555555
+; X86-NEXT: leal (%eax,%ecx,4), %eax
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: andl $1431655765, %ecx # imm = 0x55555555
; X86-NEXT: shrl %eax
; X86-NEXT: andl $1431655765, %eax # imm = 0x55555555
-; X86-NEXT: leal (%eax,%edx,2), %edx
-; X86-NEXT: movl %edx, %eax
-; X86-NEXT: shll $16, %eax
-; X86-NEXT: shrl $16, %ecx
-; X86-NEXT: orl %eax, %ecx
+; X86-NEXT: leal (%eax,%ecx,2), %edx
+; X86-NEXT: movl %ebp, %esi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: shrdl $16, %ecx, %esi
+; X86-NEXT: movl %ebx, %eax
+; X86-NEXT: shrdl $16, %ebx, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: shrdl $16, %ecx, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: shrdl $16, %eax, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: shrdl $16, %ecx, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: shrdl $16, %eax, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: shrdl $16, %ecx, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: shrdl $16, %eax, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shrdl $16, %edi, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: shrdl $16, %eax, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl (%esp), %ecx # 4-byte Reload
+; X86-NEXT: shrdl $16, %ecx, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X86-NEXT: shrdl $16, %ebp, %ecx
+; X86-NEXT: movl %ecx, (%esp) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: shrdl $16, %ebx, %ebp
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: shrdl $16, %edi, %ebx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: shrdl $16, %ecx, %edi
+; X86-NEXT: shrdl $16, %edx, %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl %ecx, 60(%eax)
-; X86-NEXT: movl %esi, 56(%eax)
-; X86-NEXT: movl %edi, 52(%eax)
-; X86-NEXT: movl %ebx, 48(%eax)
-; X86-NEXT: movl %ebp, 44(%eax)
+; X86-NEXT: movl %edi, 56(%eax)
+; X86-NEXT: movl %ebx, 52(%eax)
+; X86-NEXT: movl %ebp, 48(%eax)
; X86-NEXT: movl (%esp), %ecx # 4-byte Reload
+; X86-NEXT: movl %ecx, 44(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: movl %ecx, 40(%eax)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: movl %ecx, 36(%eax)
@@ -1108,11 +1089,10 @@ define i528 @large_promotion(i528 %A) nounwind {
; X86-NEXT: movl %ecx, 8(%eax)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: movl %ecx, 4(%eax)
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl %ecx, (%eax)
+; X86-NEXT: movl %esi, (%eax)
; X86-NEXT: shrl $16, %edx
; X86-NEXT: movw %dx, 64(%eax)
-; X86-NEXT: addl $44, %esp
+; X86-NEXT: addl $60, %esp
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
; X86-NEXT: popl %ebx
@@ -1169,10 +1149,7 @@ define i528 @large_promotion(i528 %A) nounwind {
; X64-NEXT: shrq %rbx
; X64-NEXT: andq %r14, %rbx
; X64-NEXT: leaq (%rbx,%r13,2), %rbx
-; X64-NEXT: movq %rbx, %r13
-; X64-NEXT: shlq $16, %r13
-; X64-NEXT: shrq $48, %rdi
-; X64-NEXT: orq %r13, %rdi
+; X64-NEXT: shrdq $48, %rbx, %rdi
; X64-NEXT: bswapq %r15
; X64-NEXT: movq %r15, %r13
; X64-NEXT: shrq $4, %r13
@@ -1190,10 +1167,7 @@ define i528 @large_promotion(i528 %A) nounwind {
; X64-NEXT: shrq %r15
; X64-NEXT: andq %r14, %r15
; X64-NEXT: leaq (%r15,%r13,2), %r15
-; X64-NEXT: movq %r15, %r13
-; X64-NEXT: shlq $16, %r13
-; X64-NEXT: shrq $48, %rbx
-; X64-NEXT: orq %r13, %rbx
+; X64-NEXT: shrdq $48, %r15, %rbx
; X64-NEXT: bswapq %r12
; X64-NEXT: movq %r12, %r13
; X64-NEXT: shrq $4, %r13
@@ -1211,10 +1185,7 @@ define i528 @large_promotion(i528 %A) nounwind {
; X64-NEXT: shrq %r12
; X64-NEXT: andq %r14, %r12
; X64-NEXT: leaq (%r12,%r13,2), %r12
-; X64-NEXT: movq %r12, %r13
-; X64-NEXT: shlq $16, %r13
-; X64-NEXT: shrq $48, %r15
-; X64-NEXT: orq %r13, %r15
+; X64-NEXT: shrdq $48, %r12, %r15
; X64-NEXT: bswapq %r9
; X64-NEXT: movq %r9, %r13
; X64-NEXT: shrq $4, %r13
@@ -1232,10 +1203,7 @@ define i528 @large_promotion(i528 %A) nounwind {
; X64-NEXT: shrq %r9
; X64-NEXT: andq %r14, %r9
; X64-NEXT: leaq (%r9,%r13,2), %r9
-; X64-NEXT: movq %r9, %r13
-; X64-NEXT: shlq $16, %r13
-; X64-NEXT: shrq $48, %r12
-; X64-NEXT: orq %r13, %r12
+; X64-NEXT: shrdq $48, %r9, %r12
; X64-NEXT: bswapq %r8
; X64-NEXT: movq %r8, %r13
; X64-NEXT: shrq $4, %r13
@@ -1347,119 +1315,71 @@ define i528 @large_promotion(i528 %A) nounwind {
; X86XOP-NEXT: vmovdqa {{.*#+}} xmm0 = [87,86,85,84,83,82,81,80,95,94,93,92,91,90,89,88]
; X86XOP-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
; X86XOP-NEXT: vpperm %xmm0, %xmm1, %xmm0, %xmm1
-; X86XOP-NEXT: vpextrd $1, %xmm1, %ecx
+; X86XOP-NEXT: vpextrd $1, %xmm1, %eax
; X86XOP-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
; X86XOP-NEXT: vpperm %xmm0, %xmm1, %xmm0, %xmm1
-; X86XOP-NEXT: vmovd %xmm1, %edx
-; X86XOP-NEXT: movl %edx, %eax
-; X86XOP-NEXT: shll $16, %eax
-; X86XOP-NEXT: shrl $16, %ecx
-; X86XOP-NEXT: orl %eax, %ecx
+; X86XOP-NEXT: vmovd %xmm1, %ecx
+; X86XOP-NEXT: shrdl $16, %ecx, %eax
+; X86XOP-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86XOP-NEXT: vpextrd $1, %xmm1, %eax
+; X86XOP-NEXT: shrdl $16, %eax, %ecx
; X86XOP-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86XOP-NEXT: vpextrd $1, %xmm1, %ecx
-; X86XOP-NEXT: movl %ecx, %eax
-; X86XOP-NEXT: shll $16, %eax
-; X86XOP-NEXT: shrl $16, %edx
-; X86XOP-NEXT: orl %eax, %edx
-; X86XOP-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86XOP-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
; X86XOP-NEXT: vpperm %xmm0, %xmm1, %xmm0, %xmm1
-; X86XOP-NEXT: vmovd %xmm1, %edx
-; X86XOP-NEXT: movl %edx, %eax
-; X86XOP-NEXT: shll $16, %eax
-; X86XOP-NEXT: shrl $16, %ecx
-; X86XOP-NEXT: orl %eax, %ecx
+; X86XOP-NEXT: vmovd %xmm1, %ecx
+; X86XOP-NEXT: shrdl $16, %ecx, %eax
+; X86XOP-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86XOP-NEXT: vpextrd $1, %xmm1, %eax
+; X86XOP-NEXT: shrdl $16, %eax, %ecx
; X86XOP-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86XOP-NEXT: vpextrd $1, %xmm1, %ecx
-; X86XOP-NEXT: movl %ecx, %eax
-; X86XOP-NEXT: shll $16, %eax
-; X86XOP-NEXT: shrl $16, %edx
-; X86XOP-NEXT: orl %eax, %edx
-; X86XOP-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86XOP-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
; X86XOP-NEXT: vpperm %xmm0, %xmm1, %xmm0, %xmm1
-; X86XOP-NEXT: vmovd %xmm1, %edx
-; X86XOP-NEXT: movl %edx, %eax
-; X86XOP-NEXT: shll $16, %eax
-; X86XOP-NEXT: shrl $16, %ecx
-; X86XOP-NEXT: orl %eax, %ecx
+; X86XOP-NEXT: vmovd %xmm1, %ecx
+; X86XOP-NEXT: shrdl $16, %ecx, %eax
+; X86XOP-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86XOP-NEXT: vpextrd $1, %xmm1, %eax
+; X86XOP-NEXT: shrdl $16, %eax, %ecx
; X86XOP-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86XOP-NEXT: vpextrd $1, %xmm1, %ecx
-; X86XOP-NEXT: movl %ecx, %eax
-; X86XOP-NEXT: shll $16, %eax
-; X86XOP-NEXT: shrl $16, %edx
-; X86XOP-NEXT: orl %eax, %edx
-; X86XOP-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86XOP-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
; X86XOP-NEXT: vpperm %xmm0, %xmm1, %xmm0, %xmm1
-; X86XOP-NEXT: vmovd %xmm1, %edx
-; X86XOP-NEXT: movl %edx, %eax
-; X86XOP-NEXT: shll $16, %eax
-; X86XOP-NEXT: shrl $16, %ecx
-; X86XOP-NEXT: orl %eax, %ecx
+; X86XOP-NEXT: vmovd %xmm1, %ecx
+; X86XOP-NEXT: shrdl $16, %ecx, %eax
+; X86XOP-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86XOP-NEXT: vpextrd $1, %xmm1, %eax
+; X86XOP-NEXT: shrdl $16, %eax, %ecx
; X86XOP-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86XOP-NEXT: vpextrd $1, %xmm1, %ecx
-; X86XOP-NEXT: movl %ecx, %eax
-; X86XOP-NEXT: shll $16, %eax
-; X86XOP-NEXT: shrl $16, %edx
-; X86XOP-NEXT: orl %eax, %edx
-; X86XOP-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86XOP-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
; X86XOP-NEXT: vpperm %xmm0, %xmm1, %xmm0, %xmm1
-; X86XOP-NEXT: vmovd %xmm1, %edx
-; X86XOP-NEXT: movl %edx, %eax
-; X86XOP-NEXT: shll $16, %eax
-; X86XOP-NEXT: shrl $16, %ecx
-; X86XOP-NEXT: orl %eax, %ecx
+; X86XOP-NEXT: vmovd %xmm1, %ecx
+; X86XOP-NEXT: shrdl $16, %ecx, %eax
+; X86XOP-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86XOP-NEXT: vpextrd $1, %xmm1, %eax
+; X86XOP-NEXT: shrdl $16, %eax, %ecx
; X86XOP-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86XOP-NEXT: vpextrd $1, %xmm1, %ecx
-; X86XOP-NEXT: movl %ecx, %eax
-; X86XOP-NEXT: shll $16, %eax
-; X86XOP-NEXT: shrl $16, %edx
-; X86XOP-NEXT: orl %eax, %edx
-; X86XOP-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86XOP-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
; X86XOP-NEXT: vpperm %xmm0, %xmm1, %xmm0, %xmm1
; X86XOP-NEXT: vmovd %xmm1, %ebp
-; X86XOP-NEXT: movl %ebp, %eax
-; X86XOP-NEXT: shll $16, %eax
-; X86XOP-NEXT: shrl $16, %ecx
-; X86XOP-NEXT: orl %eax, %ecx
-; X86XOP-NEXT: movl %ecx, (%esp) # 4-byte Spill
-; X86XOP-NEXT: vpextrd $1, %xmm1, %edi
-; X86XOP-NEXT: movl %edi, %eax
-; X86XOP-NEXT: shll $16, %eax
-; X86XOP-NEXT: shrl $16, %ebp
-; X86XOP-NEXT: orl %eax, %ebp
+; X86XOP-NEXT: shrdl $16, %ebp, %eax
+; X86XOP-NEXT: movl %eax, (%esp) # 4-byte Spill
+; X86XOP-NEXT: vpextrd $1, %xmm1, %ebx
+; X86XOP-NEXT: shrdl $16, %ebx, %ebp
; X86XOP-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
; X86XOP-NEXT: vpperm %xmm0, %xmm1, %xmm0, %xmm1
; X86XOP-NEXT: vmovd %xmm1, %esi
-; X86XOP-NEXT: movl %esi, %eax
-; X86XOP-NEXT: shll $16, %eax
-; X86XOP-NEXT: shrl $16, %edi
-; X86XOP-NEXT: orl %eax, %edi
+; X86XOP-NEXT: shrdl $16, %esi, %ebx
; X86XOP-NEXT: vpextrd $1, %xmm1, %edx
-; X86XOP-NEXT: movl %edx, %eax
-; X86XOP-NEXT: shll $16, %eax
-; X86XOP-NEXT: shrl $16, %esi
-; X86XOP-NEXT: orl %eax, %esi
+; X86XOP-NEXT: shrdl $16, %edx, %esi
; X86XOP-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
; X86XOP-NEXT: vpperm %xmm0, %xmm1, %xmm0, %xmm0
; X86XOP-NEXT: vmovd %xmm0, %ecx
-; X86XOP-NEXT: movl %ecx, %eax
-; X86XOP-NEXT: shll $16, %eax
-; X86XOP-NEXT: shrl $16, %edx
-; X86XOP-NEXT: orl %eax, %edx
-; X86XOP-NEXT: vpextrd $1, %xmm0, %ebx
-; X86XOP-NEXT: movl %ebx, %eax
-; X86XOP-NEXT: shll $16, %eax
-; X86XOP-NEXT: shrl $16, %ecx
-; X86XOP-NEXT: orl %eax, %ecx
+; X86XOP-NEXT: shrdl $16, %ecx, %edx
+; X86XOP-NEXT: vpextrd $1, %xmm0, %edi
+; X86XOP-NEXT: shrdl $16, %edi, %ecx
; X86XOP-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86XOP-NEXT: movl %ecx, 60(%eax)
; X86XOP-NEXT: movl %edx, 56(%eax)
; X86XOP-NEXT: movl %esi, 52(%eax)
-; X86XOP-NEXT: movl %edi, 48(%eax)
+; X86XOP-NEXT: movl %ebx, 48(%eax)
; X86XOP-NEXT: movl %ebp, 44(%eax)
; X86XOP-NEXT: movl (%esp), %ecx # 4-byte Reload
; X86XOP-NEXT: movl %ecx, 40(%eax)
@@ -1483,8 +1403,8 @@ define i528 @large_promotion(i528 %A) nounwind {
; X86XOP-NEXT: movl %ecx, 4(%eax)
; X86XOP-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86XOP-NEXT: movl %ecx, (%eax)
-; X86XOP-NEXT: shrl $16, %ebx
-; X86XOP-NEXT: movw %bx, 64(%eax)
+; X86XOP-NEXT: shrl $16, %edi
+; X86XOP-NEXT: movw %di, 64(%eax)
; X86XOP-NEXT: addl $44, %esp
; X86XOP-NEXT: popl %esi
; X86XOP-NEXT: popl %edi
@@ -1502,135 +1422,87 @@ define i528 @large_promotion(i528 %A) nounwind {
; X86GFNI-NEXT: vpbroadcastq {{.*#+}} xmm0 = [1,2,4,8,16,32,64,128,1,2,4,8,16,32,64,128]
; X86GFNI-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
; X86GFNI-NEXT: vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
-; X86GFNI-NEXT: vmovd %xmm1, %ecx
-; X86GFNI-NEXT: bswapl %ecx
+; X86GFNI-NEXT: vmovd %xmm1, %eax
+; X86GFNI-NEXT: bswapl %eax
; X86GFNI-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
; X86GFNI-NEXT: vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
-; X86GFNI-NEXT: vpextrd $1, %xmm1, %edx
-; X86GFNI-NEXT: bswapl %edx
-; X86GFNI-NEXT: movl %edx, %eax
-; X86GFNI-NEXT: shll $16, %eax
-; X86GFNI-NEXT: shrl $16, %ecx
-; X86GFNI-NEXT: orl %eax, %ecx
-; X86GFNI-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86GFNI-NEXT: vmovd %xmm1, %ecx
+; X86GFNI-NEXT: vpextrd $1, %xmm1, %ecx
; X86GFNI-NEXT: bswapl %ecx
-; X86GFNI-NEXT: movl %ecx, %eax
-; X86GFNI-NEXT: shll $16, %eax
-; X86GFNI-NEXT: shrl $16, %edx
-; X86GFNI-NEXT: orl %eax, %edx
-; X86GFNI-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86GFNI-NEXT: shrdl $16, %ecx, %eax
+; X86GFNI-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86GFNI-NEXT: vmovd %xmm1, %eax
+; X86GFNI-NEXT: bswapl %eax
+; X86GFNI-NEXT: shrdl $16, %eax, %ecx
+; X86GFNI-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86GFNI-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
; X86GFNI-NEXT: vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
-; X86GFNI-NEXT: vpextrd $1, %xmm1, %edx
-; X86GFNI-NEXT: bswapl %edx
-; X86GFNI-NEXT: movl %edx, %eax
-; X86GFNI-NEXT: shll $16, %eax
-; X86GFNI-NEXT: shrl $16, %ecx
-; X86GFNI-NEXT: orl %eax, %ecx
-; X86GFNI-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86GFNI-NEXT: vmovd %xmm1, %ecx
+; X86GFNI-NEXT: vpextrd $1, %xmm1, %ecx
; X86GFNI-NEXT: bswapl %ecx
-; X86GFNI-NEXT: movl %ecx, %eax
-; X86GFNI-NEXT: shll $16, %eax
-; X86GFNI-NEXT: shrl $16, %edx
-; X86GFNI-NEXT: orl %eax, %edx
-; X86GFNI-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86GFNI-NEXT: shrdl $16, %ecx, %eax
+; X86GFNI-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86GFNI-NEXT: vmovd %xmm1, %eax
+; X86GFNI-NEXT: bswapl %eax
+; X86GFNI-NEXT: shrdl $16, %eax, %ecx
+; X86GFNI-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86GFNI-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
; X86GFNI-NEXT: vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
-; X86GFNI-NEXT: vpextrd $1, %xmm1, %edx
-; X86GFNI-NEXT: bswapl %edx
-; X86GFNI-NEXT: movl %edx, %eax
-; X86GFNI-NEXT: shll $16, %eax
-; X86GFNI-NEXT: shrl $16, %ecx
-; X86GFNI-NEXT: orl %eax, %ecx
-; X86GFNI-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86GFNI-NEXT: vmovd %xmm1, %ecx
+; X86GFNI-NEXT: vpextrd $1, %xmm1, %ecx
; X86GFNI-NEXT: bswapl %ecx
-; X86GFNI-NEXT: movl %ecx, %eax
-; X86GFNI-NEXT: shll $16, %eax
-; X86GFNI-NEXT: shrl $16, %edx
-; X86GFNI-NEXT: orl %eax, %edx
-; X86GFNI-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86GFNI-NEXT: shrdl $16, %ecx, %eax
+; X86GFNI-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86GFNI-NEXT: vmovd %xmm1, %eax
+; X86GFNI-NEXT: bswapl %eax
+; X86GFNI-NEXT: shrdl $16, %eax, %ecx
+; X86GFNI-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86GFNI-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
; X86GFNI-NEXT: vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
-; X86GFNI-NEXT: vpextrd $1, %xmm1, %edx
-; X86GFNI-NEXT: bswapl %edx
-; X86GFNI-NEXT: movl %edx, %eax
-; X86GFNI-NEXT: shll $16, %eax
-; X86GFNI-NEXT: shrl $16, %ecx
-; X86GFNI-NEXT: orl %eax, %ecx
-; X86GFNI-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86GFNI-NEXT: vmovd %xmm1, %ecx
+; X86GFNI-NEXT: vpextrd $1, %xmm1, %ecx
; X86GFNI-NEXT: bswapl %ecx
-; X86GFNI-NEXT: movl %ecx, %eax
-; X86GFNI-NEXT: shll $16, %eax
-; X86GFNI-NEXT: shrl $16, %edx
-; X86GFNI-NEXT: orl %eax, %edx
-; X86GFNI-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86GFNI-NEXT: shrdl $16, %ecx, %eax
+; X86GFNI-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86GFNI-NEXT: vmovd %xmm1, %eax
+; X86GFNI-NEXT: bswapl %eax
+; X86GFNI-NEXT: shrdl $16, %eax, %ecx
+; X86GFNI-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86GFNI-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
; X86GFNI-NEXT: vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
-; X86GFNI-NEXT: vpextrd $1, %xmm1, %edx
-; X86GFNI-NEXT: bswapl %edx
-; X86GFNI-NEXT: movl %edx, %eax
-; X86GFNI-NEXT: shll $16, %eax
-; X86GFNI-NEXT: shrl $16, %ecx
-; X86GFNI-NEXT: orl %eax, %ecx
-; X86GFNI-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86GFNI-NEXT: vmovd %xmm1, %ecx
+; X86GFNI-NEXT: vpextrd $1, %xmm1, %ecx
; X86GFNI-NEXT: bswapl %ecx
-; X86GFNI-NEXT: movl %ecx, %eax
-; X86GFNI-NEXT: shll $16, %eax
-; X86GFNI-NEXT: shrl $16, %edx
-; X86GFNI-NEXT: orl %eax, %edx
-; X86GFNI-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86GFNI-NEXT: shrdl $16, %ecx, %eax
+; X86GFNI-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86GFNI-NEXT: vmovd %xmm1, %eax
+; X86GFNI-NEXT: bswapl %eax
+; X86GFNI-NEXT: shrdl $16, %eax, %ecx
+; X86GFNI-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86GFNI-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
; X86GFNI-NEXT: vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
; X86GFNI-NEXT: vpextrd $1, %xmm1, %ebp
; X86GFNI-NEXT: bswapl %ebp
-; X86GFNI-NEXT: movl %ebp, %eax
-; X86GFNI-NEXT: shll $16, %eax
-; X86GFNI-NEXT: shrl $16, %ecx
-; X86GFNI-NEXT: orl %eax, %ecx
-; X86GFNI-NEXT: movl %ecx, (%esp) # 4-byte Spill
+; X86GFNI-NEXT: shrdl $16, %ebp, %eax
+; X86GFNI-NEXT: movl %eax, (%esp) # 4-byte Spill
; X86GFNI-NEXT: vmovd %xmm1, %ebx
; X86GFNI-NEXT: bswapl %ebx
-; X86GFNI-NEXT: movl %ebx, %eax
-; X86GFNI-NEXT: shll $16, %eax
-; X86GFNI-NEXT: shrl $16, %ebp
-; X86GFNI-NEXT: orl %eax, %ebp
+; X86GFNI-NEXT: shrdl $16, %ebx, %ebp
; X86GFNI-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
; X86GFNI-NEXT: vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
-; X86GFNI-NEXT: vpextrd $1, %xmm1, %esi
-; X86GFNI-NEXT: bswapl %esi
-; X86GFNI-NEXT: movl %esi, %eax
-; X86GFNI-NEXT: shll $16, %eax
-; X86GFNI-NEXT: shrl $16, %ebx
-; X86GFNI-NEXT: orl %eax, %ebx
+; X86GFNI-NEXT: vpextrd $1, %xmm1, %edi
+; X86GFNI-NEXT: bswapl %edi
+; X86GFNI-NEXT: shrdl $16, %edi, %ebx
; X86GFNI-NEXT: vmovd %xmm1, %edx
; X86GFNI-NEXT: bswapl %edx
-; X86GFNI-NEXT: movl %edx, %eax
-; X86GFNI-NEXT: shll $16, %eax
-; X86GFNI-NEXT: shrl $16, %esi
-; X86GFNI-NEXT: orl %eax, %esi
+; X86GFNI-NEXT: shrdl $16, %edx, %edi
; X86GFNI-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
; X86GFNI-NEXT: vgf2p8affineqb $0, %xmm0, %xmm1, %xmm0
; X86GFNI-NEXT: vpextrd $1, %xmm0, %ecx
; X86GFNI-NEXT: bswapl %ecx
-; X86GFNI-NEXT: movl %ecx, %eax
-; X86GFNI-NEXT: shll $16, %eax
-; X86GFNI-NEXT: shrl $16, %edx
-; X86GFNI-NEXT: orl %eax, %edx
-; X86GFNI-NEXT: vmovd %xmm0, %edi
-; X86GFNI-NEXT: bswapl %edi
-; X86GFNI-NEXT: movl %edi, %eax
-; X86GFNI-NEXT: shll $16, %eax
-; X86GFNI-NEXT: shrl $16, %ecx
-; X86GFNI-NEXT: orl %eax, %ecx
+; X86GFNI-NEXT: shrdl $16, %ecx, %edx
+; X86GFNI-NEXT: vmovd %xmm0, %esi
+; X86GFNI-NEXT: bswapl %esi
+; X86GFNI-NEXT: shrdl $16, %esi, %ecx
; X86GFNI-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86GFNI-NEXT: movl %ecx, 60(%eax)
; X86GFNI-NEXT: movl %edx, 56(%eax)
-; X86GFNI-NEXT: movl %esi, 52(%eax)
+; X86GFNI-NEXT: movl %edi, 52(%eax)
; X86GFNI-NEXT: movl %ebx, 48(%eax)
; X86GFNI-NEXT: movl %ebp, 44(%eax)
; X86GFNI-NEXT: movl (%esp), %ecx # 4-byte Reload
@@ -1655,8 +1527,8 @@ define i528 @large_promotion(i528 %A) nounwind {
; X86GFNI-NEXT: movl %ecx, 4(%eax)
; X86GFNI-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86GFNI-NEXT: movl %ecx, (%eax)
-; X86GFNI-NEXT: shrl $16, %edi
-; X86GFNI-NEXT: movw %di, 64(%eax)
+; X86GFNI-NEXT: shrl $16, %esi
+; X86GFNI-NEXT: movw %si, 64(%eax)
; X86GFNI-NEXT: addl $44, %esp
; X86GFNI-NEXT: popl %esi
; X86GFNI-NEXT: popl %edi
@@ -1666,7 +1538,6 @@ define i528 @large_promotion(i528 %A) nounwind {
;
; X64GFNI-LABEL: large_promotion:
; X64GFNI: # %bb.0:
-; X64GFNI-NEXT: pushq %r15
; X64GFNI-NEXT: pushq %r14
; X64GFNI-NEXT: pushq %rbx
; X64GFNI-NEXT: movq %rdi, %rax
@@ -1679,43 +1550,28 @@ define i528 @large_promotion(i528 %A) nounwind {
; X64GFNI-NEXT: vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
; X64GFNI-NEXT: vmovq %xmm1, %rdi
; X64GFNI-NEXT: bswapq %rdi
-; X64GFNI-NEXT: movq %rdi, %r11
-; X64GFNI-NEXT: shlq $16, %r11
-; X64GFNI-NEXT: movq %r10, %r9
-; X64GFNI-NEXT: shrq $48, %r9
-; X64GFNI-NEXT: orq %r11, %r9
; X64GFNI-NEXT: vmovq %r8, %xmm1
; X64GFNI-NEXT: vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
; X64GFNI-NEXT: vmovq %xmm1, %r8
; X64GFNI-NEXT: bswapq %r8
-; X64GFNI-NEXT: movq %r8, %r11
-; X64GFNI-NEXT: shlq $16, %r11
-; X64GFNI-NEXT: shrq $48, %rdi
-; X64GFNI-NEXT: orq %r11, %rdi
+; X64GFNI-NEXT: movq %r8, %r9
+; X64GFNI-NEXT: shldq $16, %rdi, %r9
+; X64GFNI-NEXT: shldq $16, %r10, %rdi
; X64GFNI-NEXT: vmovq %rcx, %xmm1
; X64GFNI-NEXT: vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
; X64GFNI-NEXT: vmovq %xmm1, %rcx
; X64GFNI-NEXT: bswapq %rcx
-; X64GFNI-NEXT: movq %rcx, %r11
-; X64GFNI-NEXT: shlq $16, %r11
-; X64GFNI-NEXT: shrq $48, %r8
-; X64GFNI-NEXT: orq %r11, %r8
+; X64GFNI-NEXT: shrdq $48, %rcx, %r8
; X64GFNI-NEXT: vmovq %rdx, %xmm1
; X64GFNI-NEXT: vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
; X64GFNI-NEXT: vmovq %xmm1, %rdx
; X64GFNI-NEXT: bswapq %rdx
-; X64GFNI-NEXT: movq %rdx, %r11
-; X64GFNI-NEXT: shlq $16, %r11
-; X64GFNI-NEXT: shrq $48, %rcx
-; X64GFNI-NEXT: orq %r11, %rcx
+; X64GFNI-NEXT: shrdq $48, %rdx, %rcx
; X64GFNI-NEXT: vmovq %rsi, %xmm1
; X64GFNI-NEXT: vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
; X64GFNI-NEXT: vmovq %xmm1, %rsi
; X64GFNI-NEXT: bswapq %rsi
-; X64GFNI-NEXT: movq %rsi, %r11
-; X64GFNI-NEXT: shlq $16, %r11
-; X64GFNI-NEXT: shrq $48, %rdx
-; X64GFNI-NEXT: orq %r11, %rdx
+; X64GFNI-NEXT: shrdq $48, %rsi, %rdx
; X64GFNI-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
; X64GFNI-NEXT: vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
; X64GFNI-NEXT: vmovq %xmm1, %r11
@@ -1724,21 +1580,13 @@ define i528 @large_promotion(i528 %A) nounwind {
; X64GFNI-NEXT: vgf2p8affineqb $0, %xmm0, %xmm1, %xmm1
; X64GFNI-NEXT: vmovq %xmm1, %rbx
; X64GFNI-NEXT: bswapq %rbx
-; X64GFNI-NEXT: movq %rbx, %r14
-; X64GFNI-NEXT: shlq $16, %r14
-; X64GFNI-NEXT: shrq $48, %r11
-; X64GFNI-NEXT: orq %r14, %r11
+; X64GFNI-NEXT: shrdq $48, %rbx, %r11
; X64GFNI-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
; X64GFNI-NEXT: vgf2p8affineqb $0, %xmm0, %xmm1, %xmm0
; X64GFNI-NEXT: vmovq %xmm0, %r14
; X64GFNI-NEXT: bswapq %r14
-; X64GFNI-NEXT: movq %r14, %r15
-; X64GFNI-NEXT: shlq $16, %r15
-; X64GFNI-NEXT: shrq $48, %rbx
-; X64GFNI-NEXT: orq %r15, %rbx
-; X64GFNI-NEXT: shlq $16, %r10
-; X64GFNI-NEXT: shrq $48, %r14
-; X64GFNI-NEXT: orq %r10, %r14
+; X64GFNI-NEXT: shrdq $48, %r14, %rbx
+; X64GFNI-NEXT: shrdq $48, %r10, %r14
; X64GFNI-NEXT: shrq $48, %rsi
; X64GFNI-NEXT: movq %r14, 16(%rax)
; X64GFNI-NEXT: movq %rbx, 8(%rax)
@@ -1746,12 +1594,11 @@ define i528 @large_promotion(i528 %A) nounwind {
; X64GFNI-NEXT: movq %rdx, 56(%rax)
; X64GFNI-NEXT: movq %rcx, 48(%rax)
; X64GFNI-NEXT: movq %r8, 40(%rax)
-; X64GFNI-NEXT: movq %rdi, 32(%rax)
-; X64GFNI-NEXT: movq %r9, 24(%rax)
+; X64GFNI-NEXT: movq %r9, 32(%rax)
+; X64GFNI-NEXT: movq %rdi, 24(%rax)
; X64GFNI-NEXT: movw %si, 64(%rax)
; X64GFNI-NEXT: popq %rbx
; X64GFNI-NEXT: popq %r14
-; X64GFNI-NEXT: popq %r15
; X64GFNI-NEXT: retq
%Z = call i528 @llvm.bitreverse.i528(i528 %A)
ret i528 %Z
diff --git a/llvm/test/CodeGen/X86/bswap.ll b/llvm/test/CodeGen/X86/bswap.ll
index fd4c281d0c7ba..ab398b65bd3a1 100644
--- a/llvm/test/CodeGen/X86/bswap.ll
+++ b/llvm/test/CodeGen/X86/bswap.ll
@@ -257,109 +257,61 @@ define i528 @large_promotion(i528 %A) nounwind {
; CHECK-NEXT: movl {{[0-9]+}}(%esp), %esi
; CHECK-NEXT: movl {{[0-9]+}}(%esp), %edx
; CHECK-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: bswapl %eax
; CHECK-NEXT: bswapl %ecx
+; CHECK-NEXT: shrdl $16, %ecx, %eax
+; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; CHECK-NEXT: bswapl %edx
-; CHECK-NEXT: movl %edx, %eax
-; CHECK-NEXT: shll $16, %eax
-; CHECK-NEXT: shrl $16, %ecx
-; CHECK-NEXT: orl %eax, %ecx
+; CHECK-NEXT: shrdl $16, %edx, %ecx
; CHECK-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; CHECK-NEXT: bswapl %esi
-; CHECK-NEXT: movl %esi, %eax
-; CHECK-NEXT: shll $16, %eax
-; CHECK-NEXT: shrl $16, %edx
-; CHECK-NEXT: orl %eax, %edx
+; CHECK-NEXT: shrdl $16, %esi, %edx
; CHECK-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; CHECK-NEXT: bswapl %edi
-; CHECK-NEXT: movl %edi, %eax
-; CHECK-NEXT: shll $16, %eax
-; CHECK-NEXT: shrl $16, %esi
-; CHECK-NEXT: orl %eax, %esi
+; CHECK-NEXT: shrdl $16, %edi, %esi
; CHECK-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; CHECK-NEXT: bswapl %ebx
-; CHECK-NEXT: movl %ebx, %eax
-; CHECK-NEXT: shll $16, %eax
-; CHECK-NEXT: shrl $16, %edi
-; CHECK-NEXT: orl %eax, %edi
+; CHECK-NEXT: shrdl $16, %ebx, %edi
; CHECK-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; CHECK-NEXT: bswapl %ebp
-; CHECK-NEXT: movl %ebp, %eax
-; CHECK-NEXT: shll $16, %eax
-; CHECK-NEXT: shrl $16, %ebx
-; CHECK-NEXT: orl %eax, %ebx
+; CHECK-NEXT: shrdl $16, %ebp, %ebx
; CHECK-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; CHECK-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; CHECK-NEXT: bswapl %ecx
-; CHECK-NEXT: movl %ecx, %eax
-; CHECK-NEXT: shll $16, %eax
-; CHECK-NEXT: shrl $16, %ebp
-; CHECK-NEXT: orl %eax, %ebp
+; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: bswapl %eax
+; CHECK-NEXT: shrdl $16, %eax, %ebp
; CHECK-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; CHECK-NEXT: movl {{[0-9]+}}(%esp), %edx
-; CHECK-NEXT: bswapl %edx
-; CHECK-NEXT: movl %edx, %eax
-; CHECK-NEXT: shll $16, %eax
-; CHECK-NEXT: shrl $16, %ecx
-; CHECK-NEXT: orl %eax, %ecx
-; CHECK-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; CHECK-NEXT: movl {{[0-9]+}}(%esp), %ecx
; CHECK-NEXT: bswapl %ecx
-; CHECK-NEXT: movl %ecx, %eax
-; CHECK-NEXT: shll $16, %eax
-; CHECK-NEXT: shrl $16, %edx
-; CHECK-NEXT: orl %eax, %edx
-; CHECK-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; CHECK-NEXT: movl {{[0-9]+}}(%esp), %edx
-; CHECK-NEXT: bswapl %edx
-; CHECK-NEXT: movl %edx, %eax
-; CHECK-NEXT: shll $16, %eax
-; CHECK-NEXT: shrl $16, %ecx
-; CHECK-NEXT: orl %eax, %ecx
+; CHECK-NEXT: shrdl $16, %ecx, %eax
+; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: bswapl %eax
+; CHECK-NEXT: shrdl $16, %eax, %ecx
; CHECK-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; CHECK-NEXT: movl {{[0-9]+}}(%esp), %ecx
; CHECK-NEXT: bswapl %ecx
-; CHECK-NEXT: movl %ecx, %eax
-; CHECK-NEXT: shll $16, %eax
-; CHECK-NEXT: shrl $16, %edx
-; CHECK-NEXT: orl %eax, %edx
-; CHECK-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT: shrdl $16, %ecx, %eax
+; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; CHECK-NEXT: movl {{[0-9]+}}(%esp), %ebp
; CHECK-NEXT: bswapl %ebp
-; CHECK-NEXT: movl %ebp, %eax
-; CHECK-NEXT: shll $16, %eax
-; CHECK-NEXT: shrl $16, %ecx
-; CHECK-NEXT: orl %eax, %ecx
+; CHECK-NEXT: shrdl $16, %ebp, %ecx
; CHECK-NEXT: movl %ecx, (%esp) # 4-byte Spill
; CHECK-NEXT: movl {{[0-9]+}}(%esp), %ebx
; CHECK-NEXT: bswapl %ebx
-; CHECK-NEXT: movl %ebx, %eax
-; CHECK-NEXT: shll $16, %eax
-; CHECK-NEXT: shrl $16, %ebp
-; CHECK-NEXT: orl %eax, %ebp
+; CHECK-NEXT: shrdl $16, %ebx, %ebp
; CHECK-NEXT: movl {{[0-9]+}}(%esp), %esi
; CHECK-NEXT: bswapl %esi
-; CHECK-NEXT: movl %esi, %eax
-; CHECK-NEXT: shll $16, %eax
-; CHECK-NEXT: shrl $16, %ebx
-; CHECK-NEXT: orl %eax, %ebx
+; CHECK-NEXT: shrdl $16, %esi, %ebx
; CHECK-NEXT: movl {{[0-9]+}}(%esp), %edx
; CHECK-NEXT: bswapl %edx
-; CHECK-NEXT: movl %edx, %eax
-; CHECK-NEXT: shll $16, %eax
-; CHECK-NEXT: shrl $16, %esi
-; CHECK-NEXT: orl %eax, %esi
+; CHECK-NEXT: shrdl $16, %edx, %esi
; CHECK-NEXT: movl {{[0-9]+}}(%esp), %ecx
; CHECK-NEXT: bswapl %ecx
-; CHECK-NEXT: movl %ecx, %eax
-; CHECK-NEXT: shll $16, %eax
-; CHECK-NEXT: shrl $16, %edx
-; CHECK-NEXT: orl %eax, %edx
+; CHECK-NEXT: shrdl $16, %ecx, %edx
; CHECK-NEXT: movl {{[0-9]+}}(%esp), %edi
; CHECK-NEXT: bswapl %edi
-; CHECK-NEXT: movl %edi, %eax
-; CHECK-NEXT: shll $16, %eax
-; CHECK-NEXT: shrl $16, %ecx
-; CHECK-NEXT: orl %eax, %ecx
+; CHECK-NEXT: shrdl $16, %edi, %ecx
; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax
; CHECK-NEXT: movl %ecx, 60(%eax)
; CHECK-NEXT: movl %edx, 56(%eax)
@@ -408,25 +360,13 @@ define i528 @large_promotion(i528 %A) nounwind {
; CHECK64-NEXT: movq {{[0-9]+}}(%rsp), %rdi
; CHECK64-NEXT: bswapq %rdi
; CHECK64-NEXT: bswapq %r10
-; CHECK64-NEXT: movq %r10, %r14
-; CHECK64-NEXT: shlq $16, %r14
-; CHECK64-NEXT: shrq $48, %rdi
-; CHECK64-NEXT: orq %r14, %rdi
+; CHECK64-NEXT: shrdq $48, %r10, %rdi
; CHECK64-NEXT: bswapq %r11
-; CHECK64-NEXT: movq %r11, %r14
-; CHECK64-NEXT: shlq $16, %r14
-; CHECK64-NEXT: shrq $48, %r10
-; CHECK64-NEXT: orq %r14, %r10
+; CHECK64-NEXT: shrdq $48, %r11, %r10
; CHECK64-NEXT: bswapq %rbx
-; CHECK64-NEXT: movq %rbx, %r14
-; CHECK64-NEXT: shlq $16, %r14
-; CHECK64-NEXT: shrq $48, %r11
-; CHECK64-NEXT: orq %r14, %r11
+; CHECK64-NEXT: shrdq $48, %rbx, %r11
; CHECK64-NEXT: bswapq %r9
-; CHECK64-NEXT: movq %r9, %r14
-; CHECK64-NEXT: shlq $16, %r14
-; CHECK64-NEXT: shrq $48, %rbx
-; CHECK64-NEXT: orq %r14, %rbx
+; CHECK64-NEXT: shrdq $48, %r9, %rbx
; CHECK64-NEXT: bswapq %r8
; CHECK64-NEXT: movq %r8, %r14
; CHECK64-NEXT: shlq $16, %r14
diff --git a/llvm/test/CodeGen/X86/clmul.ll b/llvm/test/CodeGen/X86/clmul.ll
index 9e06d282cd0f1..d8f522ae06e62 100644
--- a/llvm/test/CodeGen/X86/clmul.ll
+++ b/llvm/test/CodeGen/X86/clmul.ll
@@ -1905,12 +1905,10 @@ define i64 @clmulr_i64(i64 %a, i64 %b) nounwind {
; SSE2-PCLMUL-NEXT: movq %rsi, %xmm0
; SSE2-PCLMUL-NEXT: movq %rdi, %xmm1
; SSE2-PCLMUL-NEXT: pclmulqdq $0, %xmm0, %xmm1
-; SSE2-PCLMUL-NEXT: movq %xmm1, %rax
+; SSE2-PCLMUL-NEXT: movq %xmm1, %rcx
; SSE2-PCLMUL-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
-; SSE2-PCLMUL-NEXT: movq %xmm0, %rcx
-; SSE2-PCLMUL-NEXT: shlq %rcx
-; SSE2-PCLMUL-NEXT: shrq $63, %rax
-; SSE2-PCLMUL-NEXT: orq %rcx, %rax
+; SSE2-PCLMUL-NEXT: movq %xmm0, %rax
+; SSE2-PCLMUL-NEXT: shldq $1, %rcx, %rax
; SSE2-PCLMUL-NEXT: retq
;
; SSE42-PCLMUL-LABEL: clmulr_i64:
@@ -1918,11 +1916,9 @@ define i64 @clmulr_i64(i64 %a, i64 %b) nounwind {
; SSE42-PCLMUL-NEXT: movq %rsi, %xmm0
; SSE42-PCLMUL-NEXT: movq %rdi, %xmm1
; SSE42-PCLMUL-NEXT: pclmulqdq $0, %xmm0, %xmm1
-; SSE42-PCLMUL-NEXT: movq %xmm1, %rax
-; SSE42-PCLMUL-NEXT: pextrq $1, %xmm1, %rcx
-; SSE42-PCLMUL-NEXT: shlq %rcx
-; SSE42-PCLMUL-NEXT: shrq $63, %rax
-; SSE42-PCLMUL-NEXT: orq %rcx, %rax
+; SSE42-PCLMUL-NEXT: movq %xmm1, %rcx
+; SSE42-PCLMUL-NEXT: pextrq $1, %xmm1, %rax
+; SSE42-PCLMUL-NEXT: shldq $1, %rcx, %rax
; SSE42-PCLMUL-NEXT: retq
;
; AVX-LABEL: clmulr_i64:
@@ -1930,11 +1926,9 @@ define i64 @clmulr_i64(i64 %a, i64 %b) nounwind {
; AVX-NEXT: vmovq %rsi, %xmm0
; AVX-NEXT: vmovq %rdi, %xmm1
; AVX-NEXT: vpclmulqdq $0, %xmm0, %xmm1, %xmm0
-; AVX-NEXT: vmovq %xmm0, %rax
-; AVX-NEXT: vpextrq $1, %xmm0, %rcx
-; AVX-NEXT: shlq %rcx
-; AVX-NEXT: shrq $63, %rax
-; AVX-NEXT: orq %rcx, %rax
+; AVX-NEXT: vmovq %xmm0, %rcx
+; AVX-NEXT: vpextrq $1, %xmm0, %rax
+; AVX-NEXT: shldq $1, %rcx, %rax
; AVX-NEXT: retq
%a.ext = zext i64 %a to i128
%b.ext = zext i64 %b to i128
diff --git a/llvm/test/CodeGen/X86/combine-bswap.ll b/llvm/test/CodeGen/X86/combine-bswap.ll
index 821b9a1dae344..1f074c877f3ae 100644
--- a/llvm/test/CodeGen/X86/combine-bswap.ll
+++ b/llvm/test/CodeGen/X86/combine-bswap.ll
@@ -411,13 +411,10 @@ define i32 @bs_and_rhs_bs32_multiuse2(i32 %a, i32 %b) #0 {
define i64 @test_bswap64_shift17(i64 %a0) {
; X86-LABEL: test_bswap64_shift17:
; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %eax, %edx
+; X86-NEXT: shldl $17, %edx, %eax
; X86-NEXT: shll $17, %edx
-; X86-NEXT: shll $17, %ecx
-; X86-NEXT: shrl $15, %eax
-; X86-NEXT: orl %ecx, %eax
; X86-NEXT: bswapl %eax
; X86-NEXT: bswapl %edx
; X86-NEXT: retl
diff --git a/llvm/test/CodeGen/X86/const-shift-of-constmasked.ll b/llvm/test/CodeGen/X86/const-shift-of-constmasked.ll
index d9eff05d566f5..142ac754c3f7e 100644
--- a/llvm/test/CodeGen/X86/const-shift-of-constmasked.ll
+++ b/llvm/test/CodeGen/X86/const-shift-of-constmasked.ll
@@ -1995,14 +1995,11 @@ define i64 @test_i64_2147483647_mask_shl_34(i64 %a0) {
define i64 @test_i64_140737488289792_mask_shl_15(i64 %a0) {
; X86-LABEL: test_i64_140737488289792_mask_shl_15:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl %edx, %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl $32767, %edx # imm = 0x7FFF
+; X86-NEXT: andl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: shldl $15, %eax, %edx
; X86-NEXT: andl $65536, %eax # imm = 0x10000
-; X86-NEXT: movl $32767, %ecx # imm = 0x7FFF
-; X86-NEXT: andl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: shll $15, %ecx
-; X86-NEXT: shrl $17, %edx
-; X86-NEXT: orl %ecx, %edx
; X86-NEXT: shll $15, %eax
; X86-NEXT: retl
;
@@ -2019,12 +2016,10 @@ define i64 @test_i64_140737488289792_mask_shl_15(i64 %a0) {
define i64 @test_i64_140737488289792_mask_shl_16(i64 %a0) {
; X86-LABEL: test_i64_140737488289792_mask_shl_16:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl $32767, %eax # imm = 0x7FFF
-; X86-NEXT: andl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: shll $16, %eax
-; X86-NEXT: shrl $16, %edx
-; X86-NEXT: orl %eax, %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl $32767, %edx # imm = 0x7FFF
+; X86-NEXT: andl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: shldl $16, %eax, %edx
; X86-NEXT: xorl %eax, %eax
; X86-NEXT: retl
;
@@ -2041,12 +2036,10 @@ define i64 @test_i64_140737488289792_mask_shl_16(i64 %a0) {
define i64 @test_i64_140737488289792_mask_shl_17(i64 %a0) {
; X86-LABEL: test_i64_140737488289792_mask_shl_17:
; X86: # %bb.0:
-; X86-NEXT: movzwl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: shll $16, %edx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: shll $17, %eax
-; X86-NEXT: shrl $15, %edx
-; X86-NEXT: orl %eax, %edx
+; X86-NEXT: movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: shll $16, %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: shldl $17, %eax, %edx
; X86-NEXT: xorl %eax, %eax
; X86-NEXT: retl
;
@@ -2063,12 +2056,10 @@ define i64 @test_i64_140737488289792_mask_shl_17(i64 %a0) {
define i64 @test_i64_140737488289792_mask_shl_18(i64 %a0) {
; X86-LABEL: test_i64_140737488289792_mask_shl_18:
; X86: # %bb.0:
-; X86-NEXT: movzwl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: shll $16, %edx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: shll $18, %eax
-; X86-NEXT: shrl $14, %edx
-; X86-NEXT: orl %eax, %edx
+; X86-NEXT: movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: shll $16, %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: shldl $18, %eax, %edx
; X86-NEXT: xorl %eax, %eax
; X86-NEXT: retl
;
diff --git a/llvm/test/CodeGen/X86/dagcombine-cse.ll b/llvm/test/CodeGen/X86/dagcombine-cse.ll
index d68f8c4a08845..3efd536adc4d1 100644
--- a/llvm/test/CodeGen/X86/dagcombine-cse.ll
+++ b/llvm/test/CodeGen/X86/dagcombine-cse.ll
@@ -120,10 +120,7 @@ define i96 @square_high(i96 %x) nounwind {
; X64-NEXT: adcq %rsi, %rax
; X64-NEXT: imulq %rcx, %rcx
; X64-NEXT: addq %rax, %rcx
-; X64-NEXT: movq %rcx, %rax
-; X64-NEXT: shlq $32, %rax
-; X64-NEXT: shrq $32, %r8
-; X64-NEXT: orq %rax, %r8
+; X64-NEXT: shrdq $32, %rcx, %r8
; X64-NEXT: shrq $32, %rcx
; X64-NEXT: movq %r8, %rax
; X64-NEXT: movq %rcx, %rdx
diff --git a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
index 15521219438cd..b94a0c697df64 100644
--- a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
+++ b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
@@ -152,151 +152,148 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: pushl %esi
; X86-NEXT: andl $-16, %esp
; X86-NEXT: subl $176, %esp
-; X86-NEXT: movl 32(%ebp), %ecx
-; X86-NEXT: movl 36(%ebp), %eax
-; X86-NEXT: movl %eax, %edx
+; X86-NEXT: movl 32(%ebp), %eax
+; X86-NEXT: movl 36(%ebp), %ecx
+; X86-NEXT: movl %ecx, %edx
; X86-NEXT: sarl $31, %edx
-; X86-NEXT: xorl %edx, %eax
-; X86-NEXT: movl %eax, %esi
; X86-NEXT: xorl %edx, %ecx
-; X86-NEXT: movl 28(%ebp), %edi
-; X86-NEXT: xorl %edx, %edi
-; X86-NEXT: movl 24(%ebp), %eax
+; X86-NEXT: movl %ecx, %edi
; X86-NEXT: xorl %edx, %eax
-; X86-NEXT: subl %edx, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: sbbl %edx, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: sbbl %edx, %ecx
+; X86-NEXT: movl 28(%ebp), %esi
+; X86-NEXT: xorl %edx, %esi
+; X86-NEXT: movl 24(%ebp), %ecx
+; X86-NEXT: xorl %edx, %ecx
+; X86-NEXT: subl %edx, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: sbbl %edx, %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 52(%ebp), %ebx
-; X86-NEXT: movl %ebx, %ecx
-; X86-NEXT: sarl $31, %ecx
-; X86-NEXT: xorl %ecx, %ebx
+; X86-NEXT: sbbl %edx, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %edx, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 52(%ebp), %ecx
+; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: sarl $31, %edx
+; X86-NEXT: movl %ecx, %edi
+; X86-NEXT: xorl %edx, %edi
; X86-NEXT: movl 48(%ebp), %esi
-; X86-NEXT: xorl %ecx, %esi
-; X86-NEXT: movl 44(%ebp), %edx
-; X86-NEXT: xorl %ecx, %edx
-; X86-NEXT: movl 40(%ebp), %edi
-; X86-NEXT: xorl %ecx, %edi
-; X86-NEXT: subl %ecx, %edi
-; X86-NEXT: sbbl %ecx, %edx
-; X86-NEXT: sbbl %ecx, %esi
+; X86-NEXT: xorl %edx, %esi
+; X86-NEXT: movl 44(%ebp), %eax
+; X86-NEXT: xorl %edx, %eax
+; X86-NEXT: movl 40(%ebp), %ebx
+; X86-NEXT: xorl %edx, %ebx
+; X86-NEXT: subl %edx, %ebx
+; X86-NEXT: sbbl %edx, %eax
+; X86-NEXT: sbbl %edx, %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: sbbl %ecx, %ebx
-; X86-NEXT: movl %edx, %eax
-; X86-NEXT: orl %ebx, %eax
-; X86-NEXT: movl %edi, %ecx
-; X86-NEXT: orl %esi, %ecx
-; X86-NEXT: orl %eax, %ecx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %edx, %edi
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: orl %edi, %ecx
+; X86-NEXT: movl %ebx, %edx
+; X86-NEXT: orl %esi, %edx
+; X86-NEXT: orl %ecx, %edx
; X86-NEXT: sete %cl
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X86-NEXT: orl %eax, %esi
-; X86-NEXT: sete %al
-; X86-NEXT: orb %cl, %al
-; X86-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X86-NEXT: bsrl %ebx, %esi
+; X86-NEXT: orl %edx, %esi
+; X86-NEXT: sete %dl
+; X86-NEXT: orb %cl, %dl
+; X86-NEXT: movb %dl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; X86-NEXT: bsrl %edi, %esi
; X86-NEXT: xorl $31, %esi
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: bsrl %eax, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: bsrl %edx, %ecx
; X86-NEXT: xorl $31, %ecx
; X86-NEXT: orl $32, %ecx
-; X86-NEXT: testl %ebx, %ebx
+; X86-NEXT: testl %edi, %edi
; X86-NEXT: cmovnel %esi, %ecx
-; X86-NEXT: bsrl %edx, %esi
+; X86-NEXT: bsrl %eax, %esi
; X86-NEXT: xorl $31, %esi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: bsrl %edi, %edi
-; X86-NEXT: xorl $31, %edi
-; X86-NEXT: orl $32, %edi
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: testl %edx, %edx
-; X86-NEXT: movl %edi, %edx
-; X86-NEXT: cmovnel %esi, %edx
-; X86-NEXT: orl $64, %edx
-; X86-NEXT: movl %eax, %esi
; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %ebx, %esi
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: cmovnel %ecx, %edx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: bsrl %eax, %esi
+; X86-NEXT: bsrl %ebx, %ebx
+; X86-NEXT: xorl $31, %ebx
+; X86-NEXT: orl $32, %ebx
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: testl %eax, %eax
+; X86-NEXT: cmovnel %esi, %ebx
+; X86-NEXT: orl $64, %ebx
+; X86-NEXT: movl %edx, %esi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %edi, %esi
+; X86-NEXT: cmovnel %ecx, %ebx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: bsrl %edx, %esi
; X86-NEXT: xorl $31, %esi
; X86-NEXT: bsrl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
; X86-NEXT: xorl $31, %ecx
; X86-NEXT: orl $32, %ecx
-; X86-NEXT: testl %eax, %eax
+; X86-NEXT: testl %edx, %edx
; X86-NEXT: cmovnel %esi, %ecx
-; X86-NEXT: bsrl %ebx, %edi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: bsrl %eax, %edi
; X86-NEXT: xorl $31, %edi
; X86-NEXT: bsrl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
; X86-NEXT: xorl $31, %esi
; X86-NEXT: orl $32, %esi
-; X86-NEXT: testl %ebx, %ebx
+; X86-NEXT: testl %eax, %eax
; X86-NEXT: cmovnel %edi, %esi
; X86-NEXT: orl $64, %esi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: orl %eax, %edi
+; X86-NEXT: orl %edx, %edi
; X86-NEXT: cmovnel %ecx, %esi
-; X86-NEXT: subl %esi, %edx
+; X86-NEXT: subl %esi, %ebx
; X86-NEXT: movl $0, %edi
; X86-NEXT: sbbl %edi, %edi
-; X86-NEXT: movl $0, %ecx
-; X86-NEXT: sbbl %ecx, %ecx
+; X86-NEXT: movl $0, %eax
+; X86-NEXT: sbbl %eax, %eax
; X86-NEXT: movl $0, %esi
; X86-NEXT: sbbl %esi, %esi
; X86-NEXT: cmpb $0, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Reload
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: jne .LBB4_1
; X86-NEXT: # %bb.2: # %select.false.sink
-; X86-NEXT: movl $127, %eax
-; X86-NEXT: cmpl %edx, %eax
-; X86-NEXT: movl $0, %eax
-; X86-NEXT: sbbl %edi, %eax
-; X86-NEXT: movl $0, %eax
-; X86-NEXT: sbbl %ecx, %eax
-; X86-NEXT: movl $0, %eax
-; X86-NEXT: sbbl %esi, %eax
-; X86-NEXT: setb %al
+; X86-NEXT: movl $127, %ecx
+; X86-NEXT: cmpl %ebx, %ecx
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: sbbl %edi, %ecx
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: sbbl %eax, %ecx
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: sbbl %esi, %ecx
+; X86-NEXT: setb %cl
; X86-NEXT: .LBB4_3: # %select.end
; X86-NEXT: movl 56(%ebp), %esi
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X86-NEXT: testb %al, %al
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: movl $0, %ebx
-; X86-NEXT: cmovnel %ebx, %eax
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: cmovnel %ebx, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: cmovnel %ebx, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: cmovel {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: xorl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: testb %cl, %cl
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: cmovnel %ecx, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: cmovnel %ecx, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: cmovnel %ecx, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: cmovel {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: jne .LBB4_4
; X86-NEXT: # %bb.10: # %select.end
-; X86-NEXT: movl %edx, %ebx
-; X86-NEXT: xorl $127, %ebx
-; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: movl %edi, %edx
+; X86-NEXT: movl %ebx, %edx
+; X86-NEXT: xorl $127, %edx
; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X86-NEXT: orl %ebx, %edx
+; X86-NEXT: movl %edi, %ecx
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: orl %edx, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: je .LBB4_11
; X86-NEXT: # %bb.8: # %udiv-bb1
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: xorps %xmm0, %xmm0
@@ -307,7 +304,6 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NEXT: movl %ebx, %ecx
; X86-NEXT: xorb $127, %cl
; X86-NEXT: movl %ecx, %eax
@@ -346,24 +342,26 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: movl %ebx, %eax
; X86-NEXT: shrb $3, %al
; X86-NEXT: andb $12, %al
-; X86-NEXT: movzbl %al, %eax
-; X86-NEXT: movl 108(%esp,%eax), %esi
-; X86-NEXT: movl 104(%esp,%eax), %edi
+; X86-NEXT: movzbl %al, %edx
+; X86-NEXT: movl 108(%esp,%edx), %esi
+; X86-NEXT: movl 104(%esp,%edx), %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %ebx, %ecx
-; X86-NEXT: movl %edi, %edx
-; X86-NEXT: shrdl %cl, %esi, %edx
+; X86-NEXT: shrdl %cl, %esi, %eax
+; X86-NEXT: movl 96(%esp,%edx), %edi
+; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: movl 100(%esp,%edx), %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: shrdl %cl, %ebx, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 96(%esp,%eax), %edx
-; X86-NEXT: movl 100(%esp,%eax), %ebx
-; X86-NEXT: movl %ebx, %eax
-; X86-NEXT: shrdl %cl, %edi, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: shrl %cl, %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NEXT: shrdl %cl, %ebx, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: shrdl %cl, %edx, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: addl $-1, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
@@ -373,208 +371,205 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: adcl $-1, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: adcl $-1, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: adcl $-1, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NEXT: .p2align 4
; X86-NEXT: .LBB4_6: # %udiv-do-while
; X86-NEXT: # =>This Inner Loop Header: Depth=1
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: shll %ebx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: movl %edi, %edx
-; X86-NEXT: shrl $31, %edx
-; X86-NEXT: orl %ebx, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: shll %edi
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: movl %ebx, %edx
+; X86-NEXT: movl %eax, %edx
; X86-NEXT: shrl $31, %edx
; X86-NEXT: orl %edi, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shll %ebx
+; X86-NEXT: shll %eax
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-NEXT: movl %edi, %edx
; X86-NEXT: shrl $31, %edx
-; X86-NEXT: orl %ebx, %edx
+; X86-NEXT: orl %eax, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: shll %edi
-; X86-NEXT: movl %esi, %edx
-; X86-NEXT: shrl $31, %edx
-; X86-NEXT: orl %edi, %edx
-; X86-NEXT: movl %edx, %edi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: movl %edx, %eax
+; X86-NEXT: shrl $31, %eax
+; X86-NEXT: orl %edi, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shldl $1, %ebx, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shll %esi
-; X86-NEXT: movl %eax, %edx
+; X86-NEXT: shll %ebx
+; X86-NEXT: movl %ecx, %edx
; X86-NEXT: shrl $31, %edx
-; X86-NEXT: orl %esi, %edx
-; X86-NEXT: movl %edx, %esi
-; X86-NEXT: shll %eax
-; X86-NEXT: movl %ecx, %ebx
-; X86-NEXT: shrl $31, %ebx
-; X86-NEXT: orl %eax, %ebx
+; X86-NEXT: orl %ebx, %edx
; X86-NEXT: shll %ecx
+; X86-NEXT: movl %esi, %ebx
+; X86-NEXT: shrl $31, %ebx
+; X86-NEXT: orl %ecx, %ebx
+; X86-NEXT: shll %esi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: movl %eax, %edx
-; X86-NEXT: shrl $31, %edx
-; X86-NEXT: orl %ecx, %edx
+; X86-NEXT: movl %eax, %edi
+; X86-NEXT: shrl $31, %edi
+; X86-NEXT: orl %esi, %edi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: orl %ecx, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %ecx, %ebx
; X86-NEXT: orl %ecx, %edx
+; X86-NEXT: orl %ecx, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %ecx, %edi
; X86-NEXT: addl %eax, %eax
; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: cmpl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: sbbl %edi, %esi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X86-NEXT: sarl $31, %esi
-; X86-NEXT: movl %esi, %eax
-; X86-NEXT: andl $1, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %esi, %eax
-; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %esi, %eax
-; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NEXT: movl %esi, %ecx
+; X86-NEXT: cmpl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NEXT: sarl $31, %eax
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: andl $1, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %eax, %ebx
+; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %eax, %ecx
; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: subl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: subl %eax, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: sbbl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: sbbl %eax, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: sbbl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: sbbl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: addl $-1, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: adcl $-1, %eax
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NEXT: adcl $-1, %esi
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: adcl $-1, %edi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: adcl $-1, %ebx
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %edi, %eax
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %ebx, %eax
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: orl %esi, %ecx
; X86-NEXT: orl %eax, %ecx
-; X86-NEXT: movl %edx, %ecx
-; X86-NEXT: movl %ebx, %eax
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: movl %esi, %edi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl %edi, %esi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %edx, %ebx
; X86-NEXT: jne .LBB4_6
; X86-NEXT: .LBB4_7: # %udiv-loop-exit
-; X86-NEXT: leal (,%edx,2), %eax
+; X86-NEXT: leal (,%edi,2), %eax
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: leal (%ecx,%esi,2), %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: shrl $31, %esi
; X86-NEXT: orl %eax, %esi
-; X86-NEXT: leal (,%ebx,2), %eax
-; X86-NEXT: shrl $31, %edx
-; X86-NEXT: orl %eax, %edx
-; X86-NEXT: shll %edi
-; X86-NEXT: shrl $31, %ebx
-; X86-NEXT: orl %edi, %ebx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: leal (,%ecx,2), %eax
+; X86-NEXT: shrl $31, %edi
+; X86-NEXT: orl %eax, %edi
+; X86-NEXT: shll %edx
+; X86-NEXT: shrl $31, %ecx
+; X86-NEXT: orl %edx, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: movl %esi, %eax
; X86-NEXT: movl 56(%ebp), %esi
+; X86-NEXT: movl %edi, %edx
; X86-NEXT: .LBB4_11: # %udiv-end
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: xorl %ecx, %edi
-; X86-NEXT: xorl %ecx, %edx
-; X86-NEXT: xorl %ecx, %eax
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NEXT: xorl %ecx, %ebx
-; X86-NEXT: subl %ecx, %ebx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: xorl %ecx, %edx
+; X86-NEXT: xorl %ecx, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: xorl %ecx, %edi
+; X86-NEXT: subl %ecx, %edi
; X86-NEXT: sbbl %ecx, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: sbbl %ecx, %edx
-; X86-NEXT: sbbl %ecx, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %ebx, (%esi)
+; X86-NEXT: sbbl %ecx, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %edi, (%esi)
; X86-NEXT: movl %eax, 4(%esi)
; X86-NEXT: movl %edx, 8(%esi)
-; X86-NEXT: movl %edi, 12(%esi)
-; X86-NEXT: movl 40(%ebp), %edi
-; X86-NEXT: movl %edx, %ecx
-; X86-NEXT: mull %edi
+; X86-NEXT: movl %ebx, 12(%esi)
+; X86-NEXT: movl 40(%ebp), %ecx
; X86-NEXT: movl %edx, %ebx
+; X86-NEXT: mull %ecx
+; X86-NEXT: movl %edx, %ecx
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: movl %esi, %eax
-; X86-NEXT: mull %edi
+; X86-NEXT: movl %edi, %esi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %edi, %eax
+; X86-NEXT: mull 40(%ebp)
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %edx, %edi
; X86-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X86-NEXT: adcl $0, %ebx
+; X86-NEXT: adcl $0, %ecx
; X86-NEXT: movl %esi, %eax
; X86-NEXT: movl 44(%ebp), %esi
; X86-NEXT: mull %esi
; X86-NEXT: addl %edi, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: adcl %ebx, %edx
-; X86-NEXT: movl %edx, %edi
-; X86-NEXT: setb %bl
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: mull %esi
-; X86-NEXT: addl %edi, %eax
+; X86-NEXT: adcl %ecx, %edx
+; X86-NEXT: movl %edx, %ecx
+; X86-NEXT: setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: movl %esi, %eax
+; X86-NEXT: mull 44(%ebp)
; X86-NEXT: movl %eax, %edi
-; X86-NEXT: movzbl %bl, %eax
+; X86-NEXT: addl %ecx, %edi
+; X86-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
; X86-NEXT: adcl %eax, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NEXT: movl 40(%ebp), %eax
-; X86-NEXT: imull %eax, %ebx
-; X86-NEXT: mull %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: imull %eax, %ecx
+; X86-NEXT: mull %ebx
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: imull %esi, %ecx
-; X86-NEXT: addl %edx, %ecx
-; X86-NEXT: addl %ebx, %ecx
+; X86-NEXT: imull 44(%ebp), %ebx
+; X86-NEXT: addl %edx, %ebx
+; X86-NEXT: addl %ecx, %ebx
; X86-NEXT: movl 48(%ebp), %eax
-; X86-NEXT: movl %eax, %esi
-; X86-NEXT: imull {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X86-NEXT: movl 52(%ebp), %ebx
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: imull %esi, %ecx
+; X86-NEXT: movl 52(%ebp), %esi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: imull %edx, %ebx
+; X86-NEXT: imull %edx, %esi
; X86-NEXT: mull %edx
-; X86-NEXT: addl %edx, %ebx
-; X86-NEXT: addl %esi, %ebx
+; X86-NEXT: addl %edx, %esi
+; X86-NEXT: addl %ecx, %esi
; X86-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NEXT: adcl %ecx, %ebx
+; X86-NEXT: adcl %ebx, %esi
; X86-NEXT: addl %edi, %eax
-; X86-NEXT: adcl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X86-NEXT: adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
; X86-NEXT: movl 24(%ebp), %edx
; X86-NEXT: subl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
; X86-NEXT: movl 28(%ebp), %ecx
; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X86-NEXT: movl 32(%ebp), %edi
-; X86-NEXT: sbbl %eax, %edi
-; X86-NEXT: movl 36(%ebp), %esi
-; X86-NEXT: sbbl %ebx, %esi
+; X86-NEXT: movl 32(%ebp), %ebx
+; X86-NEXT: sbbl %eax, %ebx
+; X86-NEXT: movl 36(%ebp), %edi
+; X86-NEXT: sbbl %esi, %edi
; X86-NEXT: movl 8(%ebp), %eax
; X86-NEXT: movl %edx, (%eax)
; X86-NEXT: movl %ecx, 4(%eax)
-; X86-NEXT: movl %edi, 8(%eax)
-; X86-NEXT: movl %esi, 12(%eax)
+; X86-NEXT: movl %ebx, 8(%eax)
+; X86-NEXT: movl %edi, 12(%eax)
; X86-NEXT: leal -12(%ebp), %esp
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
@@ -582,16 +577,18 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: popl %ebp
; X86-NEXT: retl $4
; X86-NEXT: .LBB4_1:
-; X86-NEXT: movb $1, %al
+; X86-NEXT: movb $1, %cl
; X86-NEXT: jmp .LBB4_3
; X86-NEXT: .LBB4_9:
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: jmp .LBB4_7
; X86-NEXT: .LBB4_4:
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: jmp .LBB4_11
;
; X64-LABEL: scalar_i128:
diff --git a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll
index 1c8e35e807e64..f66da12114efa 100644
--- a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll
+++ b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll
@@ -171,31 +171,32 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
; X86-NEXT: bsrl %esi, %edx
; X86-NEXT: xorl $31, %edx
+; X86-NEXT: bsrl %edi, %ecx
; X86-NEXT: movl %edi, %eax
-; X86-NEXT: bsrl %edi, %edi
-; X86-NEXT: xorl $31, %edi
-; X86-NEXT: orl $32, %edi
-; X86-NEXT: testl %esi, %esi
-; X86-NEXT: cmovnel %edx, %edi
-; X86-NEXT: bsrl %ebx, %edx
-; X86-NEXT: xorl $31, %edx
-; X86-NEXT: bsrl 40(%ebp), %ecx
; X86-NEXT: xorl $31, %ecx
; X86-NEXT: orl $32, %ecx
-; X86-NEXT: testl %ebx, %ebx
+; X86-NEXT: testl %esi, %esi
; X86-NEXT: cmovnel %edx, %ecx
-; X86-NEXT: orl $64, %ecx
-; X86-NEXT: movl %eax, %edx
-; X86-NEXT: orl %esi, %edx
-; X86-NEXT: cmovnel %edi, %ecx
-; X86-NEXT: movl 36(%ebp), %ebx
; X86-NEXT: bsrl %ebx, %edx
; X86-NEXT: xorl $31, %edx
-; X86-NEXT: bsrl 32(%ebp), %edi
+; X86-NEXT: bsrl 40(%ebp), %edi
; X86-NEXT: xorl $31, %edi
; X86-NEXT: orl $32, %edi
; X86-NEXT: testl %ebx, %ebx
; X86-NEXT: cmovnel %edx, %edi
+; X86-NEXT: orl $64, %edi
+; X86-NEXT: movl %eax, %edx
+; X86-NEXT: movl %edi, %ebx
+; X86-NEXT: movl 36(%ebp), %edi
+; X86-NEXT: orl %esi, %edx
+; X86-NEXT: cmovnel %ecx, %ebx
+; X86-NEXT: bsrl %edi, %edx
+; X86-NEXT: xorl $31, %edx
+; X86-NEXT: bsrl 32(%ebp), %ecx
+; X86-NEXT: xorl $31, %ecx
+; X86-NEXT: orl $32, %ecx
+; X86-NEXT: testl %edi, %edi
+; X86-NEXT: cmovnel %edx, %ecx
; X86-NEXT: movl 28(%ebp), %eax
; X86-NEXT: bsrl %eax, %edx
; X86-NEXT: xorl $31, %edx
@@ -206,68 +207,64 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: cmovnel %edx, %esi
; X86-NEXT: orl $64, %esi
; X86-NEXT: movl 32(%ebp), %edx
-; X86-NEXT: orl %ebx, %edx
-; X86-NEXT: cmovnel %edi, %esi
+; X86-NEXT: orl %edi, %edx
+; X86-NEXT: cmovnel %ecx, %esi
; X86-NEXT: xorl %edx, %edx
-; X86-NEXT: subl %esi, %ecx
+; X86-NEXT: subl %esi, %ebx
; X86-NEXT: movl $0, %esi
; X86-NEXT: sbbl %esi, %esi
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: sbbl %ecx, %ecx
; X86-NEXT: movl $0, %edi
; X86-NEXT: sbbl %edi, %edi
-; X86-NEXT: movl $0, %ebx
-; X86-NEXT: sbbl %ebx, %ebx
; X86-NEXT: cmpb $0, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Reload
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: jne .LBB4_1
; X86-NEXT: # %bb.2: # %select.false.sink
; X86-NEXT: movl $127, %eax
-; X86-NEXT: cmpl %ecx, %eax
+; X86-NEXT: cmpl %ebx, %eax
; X86-NEXT: movl $0, %eax
; X86-NEXT: sbbl %esi, %eax
; X86-NEXT: movl $0, %eax
-; X86-NEXT: sbbl %edi, %eax
+; X86-NEXT: sbbl %ecx, %eax
; X86-NEXT: movl $0, %eax
-; X86-NEXT: sbbl %ebx, %eax
+; X86-NEXT: sbbl %edi, %eax
; X86-NEXT: setb %al
; X86-NEXT: .LBB4_3: # %select.end
; X86-NEXT: testb %al, %al
-; X86-NEXT: movl 28(%ebp), %esi
-; X86-NEXT: cmovnel %edx, %esi
+; X86-NEXT: movl 28(%ebp), %edi
+; X86-NEXT: cmovnel %edx, %edi
; X86-NEXT: movl 24(%ebp), %eax
; X86-NEXT: cmovnel %edx, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl 32(%ebp), %eax
; X86-NEXT: cmovnel %edx, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 36(%ebp), %ebx
-; X86-NEXT: cmovnel %edx, %ebx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 56(%ebp), %edi
-; X86-NEXT: jne .LBB4_4
-; X86-NEXT: # %bb.10: # %select.end
-; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: movl 36(%ebp), %ecx
+; X86-NEXT: cmovnel %edx, %ecx
+; X86-NEXT: jne .LBB4_9
+; X86-NEXT: # %bb.4: # %select.end
+; X86-NEXT: movl %ebx, %eax
; X86-NEXT: xorl $127, %eax
; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: movl %esi, %edx
; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
; X86-NEXT: orl %eax, %edx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: je .LBB4_11
-; X86-NEXT: # %bb.8: # %udiv-bb1
-; X86-NEXT: movl 24(%ebp), %ebx
-; X86-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 28(%ebp), %eax
+; X86-NEXT: je .LBB4_9
+; X86-NEXT: # %bb.5: # %udiv-bb1
+; X86-NEXT: movl 24(%ebp), %esi
+; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
; X86-NEXT: xorps %xmm0, %xmm0
; X86-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 28(%ebp), %edx
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 32(%ebp), %edx
; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 32(%ebp), %esi
-; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 36(%ebp), %edi
-; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: movl 36(%ebp), %eax
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %ebx, %ecx
; X86-NEXT: xorb $127, %cl
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: shrb $3, %al
@@ -275,51 +272,54 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: negb %al
; X86-NEXT: movsbl %al, %eax
; X86-NEXT: movl 136(%esp,%eax), %esi
-; X86-NEXT: movl 140(%esp,%eax), %ebx
-; X86-NEXT: shldl %cl, %esi, %ebx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 128(%esp,%eax), %ebx
+; X86-NEXT: movl 140(%esp,%eax), %edi
+; X86-NEXT: shldl %cl, %esi, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 128(%esp,%eax), %edi
; X86-NEXT: movl 132(%esp,%eax), %eax
; X86-NEXT: shldl %cl, %eax, %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shldl %cl, %ebx, %eax
+; X86-NEXT: shldl %cl, %edi, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shll %cl, %ebx
-; X86-NEXT: movl %edx, %ecx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: addl $1, %ecx
+; X86-NEXT: shll %cl, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: addl $1, %ebx
; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: jb .LBB4_9
-; X86-NEXT: # %bb.5: # %udiv-preheader
+; X86-NEXT: jb .LBB4_10
+; X86-NEXT: # %bb.6: # %udiv-preheader
; X86-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 24(%ebp), %edx
-; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 28(%ebp), %edx
-; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 32(%ebp), %edx
+; X86-NEXT: movl 24(%ebp), %eax
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 28(%ebp), %eax
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: movl 36(%ebp), %eax
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %ebx, %eax
; X86-NEXT: shrb $3, %al
; X86-NEXT: andb $12, %al
-; X86-NEXT: movzbl %al, %eax
-; X86-NEXT: movl 92(%esp,%eax), %edi
-; X86-NEXT: movl 88(%esp,%eax), %edx
-; X86-NEXT: movl %edx, %esi
-; X86-NEXT: shrdl %cl, %edi, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 80(%esp,%eax), %esi
-; X86-NEXT: movl 84(%esp,%eax), %ebx
-; X86-NEXT: movl %ebx, %eax
-; X86-NEXT: shrdl %cl, %edx, %eax
+; X86-NEXT: movzbl %al, %esi
+; X86-NEXT: movl 92(%esp,%esi), %edx
+; X86-NEXT: movl 88(%esp,%esi), %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shrl %cl, %edi
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: shrdl %cl, %edx, %eax
+; X86-NEXT: movl 80(%esp,%esi), %edi
+; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: movl 84(%esp,%esi), %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NEXT: shrdl %cl, %ebx, %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shrl %cl, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: shrdl %cl, %edx, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl 40(%ebp), %ecx
; X86-NEXT: addl $-1, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
@@ -329,196 +329,201 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: movl 48(%ebp), %ecx
; X86-NEXT: adcl $-1, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 52(%ebp), %eax
-; X86-NEXT: adcl $-1, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 52(%ebp), %ecx
+; X86-NEXT: adcl $-1, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-NEXT: .p2align 4
-; X86-NEXT: .LBB4_6: # %udiv-do-while
+; X86-NEXT: .LBB4_7: # %udiv-do-while
; X86-NEXT: # =>This Inner Loop Header: Depth=1
-; X86-NEXT: shll %edi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: shll %esi
+; X86-NEXT: movl %eax, %ebx
+; X86-NEXT: shrl $31, %ebx
+; X86-NEXT: orl %esi, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shll %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: movl %esi, %ebx
+; X86-NEXT: shrl $31, %ebx
+; X86-NEXT: orl %eax, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shll %esi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NEXT: movl %ebx, %eax
; X86-NEXT: shrl $31, %eax
-; X86-NEXT: orl %edi, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shll %ebx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: movl %edi, %eax
-; X86-NEXT: shrl $31, %eax
-; X86-NEXT: orl %ebx, %eax
+; X86-NEXT: orl %esi, %eax
+; X86-NEXT: movl %eax, %esi
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shldl $1, %edi, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: shll %edi
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: movl %ebx, %eax
-; X86-NEXT: shrl $31, %eax
-; X86-NEXT: orl %edi, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shll %ebx
-; X86-NEXT: movl %edx, %edi
-; X86-NEXT: shrl $31, %edi
-; X86-NEXT: orl %ebx, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shll %edx
-; X86-NEXT: movl %esi, %eax
-; X86-NEXT: shrl $31, %eax
-; X86-NEXT: orl %edx, %eax
-; X86-NEXT: movl %eax, %edx
-; X86-NEXT: shll %esi
-; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: shrl $31, %eax
-; X86-NEXT: orl %esi, %eax
-; X86-NEXT: shll %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: movl %esi, %ebx
+; X86-NEXT: movl %ecx, %ebx
; X86-NEXT: shrl $31, %ebx
-; X86-NEXT: orl %ecx, %ebx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: orl %ecx, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %edi, %ebx
+; X86-NEXT: shll %ecx
+; X86-NEXT: movl %edx, %eax
+; X86-NEXT: shrl $31, %eax
; X86-NEXT: orl %ecx, %eax
-; X86-NEXT: orl %ecx, %ebx
-; X86-NEXT: addl %esi, %esi
-; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: cmpl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NEXT: movl %eax, %edi
+; X86-NEXT: shll %edx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: shrl $31, %ecx
+; X86-NEXT: orl %edx, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: orl %edx, %ebx
+; X86-NEXT: orl %edx, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %edx, %ecx
+; X86-NEXT: addl %eax, %eax
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: cmpl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: sbbl %esi, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: sbbl %edx, %esi
; X86-NEXT: sarl $31, %esi
-; X86-NEXT: movl %esi, %ecx
-; X86-NEXT: andl $1, %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %esi, %ecx
-; X86-NEXT: andl 52(%ebp), %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %esi, %eax
+; X86-NEXT: andl $1, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %esi, %eax
+; X86-NEXT: andl 52(%ebp), %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %esi, %edi
; X86-NEXT: andl 48(%ebp), %edi
-; X86-NEXT: movl %esi, %ecx
-; X86-NEXT: andl 44(%ebp), %ecx
+; X86-NEXT: movl %esi, %eax
+; X86-NEXT: andl 44(%ebp), %eax
; X86-NEXT: andl 40(%ebp), %esi
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: subl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: sbbl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: sbbl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X86-NEXT: subl %esi, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: sbbl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: movl %ecx, %edi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: sbbl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: addl $-1, %edx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: adcl $-1, %eax
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: addl $-1, %ecx
+; X86-NEXT: adcl $-1, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NEXT: adcl $-1, %esi
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: adcl $-1, %edi
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: adcl $-1, %edx
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %esi, %eax
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %edx, %esi
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %edi, %ecx
-; X86-NEXT: orl %esi, %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: movl %ebx, %ecx
-; X86-NEXT: movl %eax, %esi
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: jne .LBB4_6
-; X86-NEXT: .LBB4_7: # %udiv-loop-exit
-; X86-NEXT: leal (,%ebx,2), %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: orl %ecx, %edx
+; X86-NEXT: orl %eax, %edx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %edi, %esi
+; X86-NEXT: movl %edi, %edx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %ebx, %edi
+; X86-NEXT: jne .LBB4_7
+; X86-NEXT: .LBB4_8: # %udiv-loop-exit
+; X86-NEXT: leal (,%esi,2), %eax
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: leal (%edx,%esi,2), %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shrl $31, %esi
-; X86-NEXT: orl %ecx, %esi
-; X86-NEXT: leal (,%eax,2), %ecx
-; X86-NEXT: shrl $31, %ebx
-; X86-NEXT: orl %ecx, %ebx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: shll %ecx
-; X86-NEXT: shrl $31, %eax
-; X86-NEXT: orl %ecx, %eax
-; X86-NEXT: movl 56(%ebp), %edi
-; X86-NEXT: .LBB4_11: # %udiv-end
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: leal (%ecx,%edx,2), %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shrl $31, %edx
+; X86-NEXT: orl %eax, %edx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: leal (,%ecx,2), %eax
+; X86-NEXT: shrl $31, %esi
+; X86-NEXT: orl %eax, %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shll %ebx
+; X86-NEXT: shrl $31, %ecx
+; X86-NEXT: orl %ebx, %ecx
+; X86-NEXT: movl %edx, %edi
+; X86-NEXT: .LBB4_9: # %udiv-end
+; X86-NEXT: movl 56(%ebp), %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: movl %ebx, (%eax)
+; X86-NEXT: movl %edi, 4(%eax)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: movl %edx, (%edi)
-; X86-NEXT: movl %esi, 4(%edi)
-; X86-NEXT: movl %ebx, 8(%edi)
-; X86-NEXT: movl %eax, 12(%edi)
-; X86-NEXT: movl %eax, %ebx
+; X86-NEXT: movl %edx, 8(%eax)
+; X86-NEXT: movl %ecx, 12(%eax)
; X86-NEXT: movl 48(%ebp), %eax
-; X86-NEXT: movl %eax, %ecx
-; X86-NEXT: imull %esi, %ecx
-; X86-NEXT: movl %edx, %edi
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: mull %edx
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: addl %ecx, %edx
-; X86-NEXT: movl 52(%ebp), %esi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %eax, %esi
; X86-NEXT: imull %edi, %esi
-; X86-NEXT: addl %edx, %esi
-; X86-NEXT: movl 40(%ebp), %edi
-; X86-NEXT: movl %edi, %eax
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: mull %ecx
+; X86-NEXT: mull %ebx
+; X86-NEXT: movl %ebx, %edi
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: addl %esi, %edx
+; X86-NEXT: movl 52(%ebp), %ebx
; X86-NEXT: imull %edi, %ebx
; X86-NEXT: addl %edx, %ebx
-; X86-NEXT: movl 44(%ebp), %eax
-; X86-NEXT: imull %eax, %ecx
-; X86-NEXT: addl %ebx, %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: addl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: adcl %esi, %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 40(%ebp), %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: mull %edi
+; X86-NEXT: movl %eax, %esi
+; X86-NEXT: imull 40(%ebp), %ecx
+; X86-NEXT: addl %edx, %ecx
+; X86-NEXT: movl 44(%ebp), %edx
+; X86-NEXT: imull %edx, %edi
+; X86-NEXT: addl %ecx, %edi
+; X86-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: adcl %ebx, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NEXT: movl %esi, %eax
-; X86-NEXT: mull %edi
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 40(%ebp), %ecx
+; X86-NEXT: mull %ecx
+; X86-NEXT: movl %edx, %ebx
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: movl %ebx, %eax
-; X86-NEXT: mull %edi
-; X86-NEXT: movl %edx, %edi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: movl %edi, %eax
+; X86-NEXT: mull %ecx
; X86-NEXT: movl %eax, %ecx
-; X86-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X86-NEXT: adcl $0, %edi
+; X86-NEXT: addl %ebx, %ecx
+; X86-NEXT: adcl $0, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %esi, %eax
-; X86-NEXT: mull 44(%ebp)
+; X86-NEXT: movl 44(%ebp), %ebx
+; X86-NEXT: mull %ebx
; X86-NEXT: movl %edx, %esi
; X86-NEXT: addl %ecx, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: adcl %edi, %esi
+; X86-NEXT: adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
; X86-NEXT: setb %cl
-; X86-NEXT: movl %ebx, %eax
-; X86-NEXT: mull 44(%ebp)
+; X86-NEXT: movl %edi, %eax
+; X86-NEXT: mull %ebx
; X86-NEXT: addl %esi, %eax
; X86-NEXT: movzbl %cl, %ecx
; X86-NEXT: adcl %ecx, %edx
; X86-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
; X86-NEXT: adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X86-NEXT: movl 24(%ebp), %ebx
-; X86-NEXT: subl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X86-NEXT: movl 28(%ebp), %ecx
-; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X86-NEXT: movl 32(%ebp), %esi
-; X86-NEXT: sbbl %eax, %esi
+; X86-NEXT: movl 24(%ebp), %esi
+; X86-NEXT: subl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NEXT: movl 28(%ebp), %ebx
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X86-NEXT: movl 32(%ebp), %ecx
+; X86-NEXT: sbbl %eax, %ecx
; X86-NEXT: movl 36(%ebp), %edi
; X86-NEXT: sbbl %edx, %edi
; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: movl %ebx, (%eax)
-; X86-NEXT: movl %ecx, 4(%eax)
-; X86-NEXT: movl %esi, 8(%eax)
+; X86-NEXT: movl %esi, (%eax)
+; X86-NEXT: movl %ebx, 4(%eax)
+; X86-NEXT: movl %ecx, 8(%eax)
; X86-NEXT: movl %edi, 12(%eax)
; X86-NEXT: leal -12(%ebp), %esp
; X86-NEXT: popl %esi
@@ -529,17 +534,11 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: .LBB4_1:
; X86-NEXT: movb $1, %al
; X86-NEXT: jmp .LBB4_3
-; X86-NEXT: .LBB4_9:
+; X86-NEXT: .LBB4_10:
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: jmp .LBB4_7
-; X86-NEXT: .LBB4_4:
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: jmp .LBB4_11
+; X86-NEXT: jmp .LBB4_8
;
; X64-LABEL: scalar_i128:
; X64: # %bb.0:
diff --git a/llvm/test/CodeGen/X86/div_i129_v_pow2k.ll b/llvm/test/CodeGen/X86/div_i129_v_pow2k.ll
index 628315d80408f..f915a8d27e6b3 100644
--- a/llvm/test/CodeGen/X86/div_i129_v_pow2k.ll
+++ b/llvm/test/CodeGen/X86/div_i129_v_pow2k.ll
@@ -8,14 +8,11 @@ define i129 @v_sdiv_i129_v_pow2k(i129 %lhs) nounwind {
; X64-LABEL: v_sdiv_i129_v_pow2k:
; X64: # %bb.0:
; X64-NEXT: movq %rdx, %rcx
-; X64-NEXT: movl %ecx, %eax
-; X64-NEXT: andl $1, %eax
-; X64-NEXT: negq %rax
-; X64-NEXT: movl %eax, %edx
; X64-NEXT: andl $1, %edx
-; X64-NEXT: shlq $32, %rdx
-; X64-NEXT: shrq $32, %rax
-; X64-NEXT: orq %rdx, %rax
+; X64-NEXT: negq %rdx
+; X64-NEXT: movl %edx, %eax
+; X64-NEXT: andl $1, %eax
+; X64-NEXT: shldq $32, %rdx, %rax
; X64-NEXT: addq %rdi, %rax
; X64-NEXT: adcq $0, %rsi
; X64-NEXT: adcq $0, %rcx
@@ -23,9 +20,7 @@ define i129 @v_sdiv_i129_v_pow2k(i129 %lhs) nounwind {
; X64-NEXT: movq %rcx, %rdx
; X64-NEXT: negq %rdx
; X64-NEXT: shrdq $33, %rsi, %rax
-; X64-NEXT: shlq $31, %rdx
-; X64-NEXT: shrq $33, %rsi
-; X64-NEXT: orq %rsi, %rdx
+; X64-NEXT: shldq $31, %rsi, %rdx
; X64-NEXT: retq
;
; X64-O0-LABEL: v_sdiv_i129_v_pow2k:
@@ -156,9 +151,7 @@ define i129 @v_sdiv_exact_i129_v_pow2k(i129 %lhs) nounwind {
; X64-NEXT: movq %rcx, %rdx
; X64-NEXT: negq %rdx
; X64-NEXT: shrdq $33, %rsi, %rax
-; X64-NEXT: shlq $31, %rdx
-; X64-NEXT: shrq $33, %rsi
-; X64-NEXT: orq %rsi, %rdx
+; X64-NEXT: shldq $31, %rsi, %rdx
; X64-NEXT: retq
;
; X64-O0-LABEL: v_sdiv_exact_i129_v_pow2k:
@@ -251,9 +244,7 @@ define i129 @v_udiv_i129_v_pow2k(i129 %lhs) nounwind {
; X64-NEXT: movq %rdi, %rax
; X64-NEXT: andl $1, %edx
; X64-NEXT: shrdq $33, %rsi, %rax
-; X64-NEXT: shlq $31, %rdx
-; X64-NEXT: shrq $33, %rsi
-; X64-NEXT: orq %rsi, %rdx
+; X64-NEXT: shldq $31, %rsi, %rdx
; X64-NEXT: xorl %ecx, %ecx
; X64-NEXT: retq
;
@@ -271,33 +262,23 @@ define i129 @v_udiv_i129_v_pow2k(i129 %lhs) nounwind {
;
; X86-LABEL: v_udiv_i129_v_pow2k:
; X86: # %bb.0:
-; X86-NEXT: pushl %ebx
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT: movl %edx, %ebx
-; X86-NEXT: shll $31, %ebx
-; X86-NEXT: shrl %ecx
-; X86-NEXT: orl %ebx, %ecx
-; X86-NEXT: shll $31, %edi
-; X86-NEXT: movl %esi, %ebx
-; X86-NEXT: shrl %ebx
-; X86-NEXT: orl %edi, %ebx
-; X86-NEXT: shll $31, %esi
-; X86-NEXT: shrl %edx
-; X86-NEXT: orl %esi, %edx
-; X86-NEXT: movl %ebx, 8(%eax)
-; X86-NEXT: movl %edx, 4(%eax)
-; X86-NEXT: movl %ecx, (%eax)
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT: shrdl $1, %esi, %edx
+; X86-NEXT: shldl $31, %edi, %ecx
+; X86-NEXT: shldl $31, %esi, %edi
+; X86-NEXT: movl %ecx, 8(%eax)
+; X86-NEXT: movl %edi, 4(%eax)
+; X86-NEXT: movl %edx, (%eax)
; X86-NEXT: movl $0, 12(%eax)
; X86-NEXT: movb $0, 16(%eax)
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
-; X86-NEXT: popl %ebx
; X86-NEXT: retl $4
;
; X86-O0-LABEL: v_udiv_i129_v_pow2k:
@@ -334,9 +315,7 @@ define i129 @v_udiv_exact_i129_v_pow2k(i129 %lhs) nounwind {
; X64-NEXT: movq %rdi, %rax
; X64-NEXT: andl $1, %edx
; X64-NEXT: shrdq $33, %rsi, %rax
-; X64-NEXT: shlq $31, %rdx
-; X64-NEXT: shrq $33, %rsi
-; X64-NEXT: orq %rsi, %rdx
+; X64-NEXT: shldq $31, %rsi, %rdx
; X64-NEXT: xorl %ecx, %ecx
; X64-NEXT: retq
;
@@ -354,33 +333,23 @@ define i129 @v_udiv_exact_i129_v_pow2k(i129 %lhs) nounwind {
;
; X86-LABEL: v_udiv_exact_i129_v_pow2k:
; X86: # %bb.0:
-; X86-NEXT: pushl %ebx
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT: movl %edx, %ebx
-; X86-NEXT: shll $31, %ebx
-; X86-NEXT: shrl %ecx
-; X86-NEXT: orl %ebx, %ecx
-; X86-NEXT: shll $31, %edi
-; X86-NEXT: movl %esi, %ebx
-; X86-NEXT: shrl %ebx
-; X86-NEXT: orl %edi, %ebx
-; X86-NEXT: shll $31, %esi
-; X86-NEXT: shrl %edx
-; X86-NEXT: orl %esi, %edx
-; X86-NEXT: movl %ebx, 8(%eax)
-; X86-NEXT: movl %edx, 4(%eax)
-; X86-NEXT: movl %ecx, (%eax)
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT: shrdl $1, %esi, %edx
+; X86-NEXT: shldl $31, %edi, %ecx
+; X86-NEXT: shldl $31, %esi, %edi
+; X86-NEXT: movl %ecx, 8(%eax)
+; X86-NEXT: movl %edi, 4(%eax)
+; X86-NEXT: movl %edx, (%eax)
; X86-NEXT: movl $0, 12(%eax)
; X86-NEXT: movb $0, 16(%eax)
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
-; X86-NEXT: popl %ebx
; X86-NEXT: retl $4
;
; X86-O0-LABEL: v_udiv_exact_i129_v_pow2k:
diff --git a/llvm/test/CodeGen/X86/divide-by-constant.ll b/llvm/test/CodeGen/X86/divide-by-constant.ll
index dd9419e6658a9..ac78136b9d8ea 100644
--- a/llvm/test/CodeGen/X86/divide-by-constant.ll
+++ b/llvm/test/CodeGen/X86/divide-by-constant.ll
@@ -699,10 +699,9 @@ define i64 @urem_i64_65537(i64 %x) nounwind {
; X86-NEXT: movl $-65535, %edx # imm = 0xFFFF0001
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: mull %edx
-; X86-NEXT: shrl $16, %edx
; X86-NEXT: movl %edx, %eax
-; X86-NEXT: shll $16, %eax
-; X86-NEXT: orl %edx, %eax
+; X86-NEXT: shrl $16, %eax
+; X86-NEXT: shldl $16, %edx, %eax
; X86-NEXT: subl %eax, %ecx
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: xorl %edx, %edx
@@ -730,14 +729,11 @@ define i64 @urem_i64_12(i64 %x) nounwind {
; X86: # %bb.0: # %entry
; X86-NEXT: pushl %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl %eax, %edx
-; X86-NEXT: shrl $2, %edx
-; X86-NEXT: shll $30, %eax
-; X86-NEXT: movl %esi, %ecx
-; X86-NEXT: shrl $2, %ecx
-; X86-NEXT: orl %eax, %ecx
-; X86-NEXT: addl %edx, %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: shrl $2, %eax
+; X86-NEXT: shldl $30, %esi, %ecx
+; X86-NEXT: addl %eax, %ecx
; X86-NEXT: adcl $0, %ecx
; X86-NEXT: movl $-1431655765, %edx # imm = 0xAAAAAAAB
; X86-NEXT: movl %ecx, %eax
@@ -1088,10 +1084,9 @@ define i64 @udiv_i64_65537(i64 %x) nounwind {
; X86-NEXT: movl $-65535, %ebx # imm = 0xFFFF0001
; X86-NEXT: movl %esi, %eax
; X86-NEXT: mull %ebx
-; X86-NEXT: shrl $16, %edx
; X86-NEXT: movl %edx, %eax
-; X86-NEXT: shll $16, %eax
-; X86-NEXT: orl %edx, %eax
+; X86-NEXT: shrl $16, %eax
+; X86-NEXT: shldl $16, %edx, %eax
; X86-NEXT: subl %eax, %esi
; X86-NEXT: subl %esi, %ecx
; X86-NEXT: sbbl $0, %edi
@@ -1128,12 +1123,9 @@ define i64 @udiv_i64_12(i64 %x) nounwind {
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl %eax, %edi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT: shrdl $2, %edi, %ecx
; X86-NEXT: shrl $2, %edi
-; X86-NEXT: shll $30, %eax
-; X86-NEXT: shrl $2, %ecx
-; X86-NEXT: orl %eax, %ecx
; X86-NEXT: movl %ecx, %esi
; X86-NEXT: addl %edi, %esi
; X86-NEXT: adcl $0, %esi
diff --git a/llvm/test/CodeGen/X86/divmod128.ll b/llvm/test/CodeGen/X86/divmod128.ll
index 2523310dfca71..3796dd796eaf9 100644
--- a/llvm/test/CodeGen/X86/divmod128.ll
+++ b/llvm/test/CodeGen/X86/divmod128.ll
@@ -425,11 +425,8 @@ entry:
define i128 @urem_i128_12(i128 %x) nounwind {
; X86-64-LABEL: urem_i128_12:
; X86-64: # %bb.0: # %entry
-; X86-64-NEXT: movq %rsi, %rax
-; X86-64-NEXT: shlq $62, %rax
-; X86-64-NEXT: movq %rdi, %rcx
-; X86-64-NEXT: shrq $2, %rcx
-; X86-64-NEXT: orq %rax, %rcx
+; X86-64-NEXT: movq %rsi, %rcx
+; X86-64-NEXT: shldq $62, %rdi, %rcx
; X86-64-NEXT: shrq $2, %rsi
; X86-64-NEXT: addq %rsi, %rcx
; X86-64-NEXT: adcq $0, %rcx
@@ -446,11 +443,8 @@ define i128 @urem_i128_12(i128 %x) nounwind {
;
; WIN64-LABEL: urem_i128_12:
; WIN64: # %bb.0: # %entry
-; WIN64-NEXT: movq %rdx, %rax
-; WIN64-NEXT: shlq $62, %rax
-; WIN64-NEXT: movq %rcx, %r8
-; WIN64-NEXT: shrq $2, %r8
-; WIN64-NEXT: orq %rax, %r8
+; WIN64-NEXT: movq %rdx, %r8
+; WIN64-NEXT: shldq $62, %rcx, %r8
; WIN64-NEXT: shrq $2, %rdx
; WIN64-NEXT: addq %rdx, %r8
; WIN64-NEXT: adcq $0, %r8
@@ -904,27 +898,24 @@ entry:
define i128 @udiv_i128_12(i128 %x) nounwind {
; X86-64-LABEL: udiv_i128_12:
; X86-64: # %bb.0: # %entry
-; X86-64-NEXT: movq %rsi, %rcx
-; X86-64-NEXT: shlq $62, %rcx
-; X86-64-NEXT: shrq $2, %rdi
-; X86-64-NEXT: orq %rdi, %rcx
+; X86-64-NEXT: shrdq $2, %rsi, %rdi
; X86-64-NEXT: shrq $2, %rsi
-; X86-64-NEXT: movq %rcx, %rdi
-; X86-64-NEXT: addq %rsi, %rdi
-; X86-64-NEXT: adcq $0, %rdi
+; X86-64-NEXT: movq %rdi, %rcx
+; X86-64-NEXT: addq %rsi, %rcx
+; X86-64-NEXT: adcq $0, %rcx
; X86-64-NEXT: movabsq $-6148914691236517205, %r8 # imm = 0xAAAAAAAAAAAAAAAB
-; X86-64-NEXT: movq %rdi, %rax
+; X86-64-NEXT: movq %rcx, %rax
; X86-64-NEXT: mulq %r8
; X86-64-NEXT: shrq %rdx
; X86-64-NEXT: leaq (%rdx,%rdx,2), %rax
-; X86-64-NEXT: subq %rax, %rdi
-; X86-64-NEXT: subq %rdi, %rcx
+; X86-64-NEXT: subq %rax, %rcx
+; X86-64-NEXT: subq %rcx, %rdi
; X86-64-NEXT: sbbq $0, %rsi
-; X86-64-NEXT: movabsq $-6148914691236517206, %rdi # imm = 0xAAAAAAAAAAAAAAAA
-; X86-64-NEXT: imulq %rcx, %rdi
-; X86-64-NEXT: movq %rcx, %rax
+; X86-64-NEXT: movabsq $-6148914691236517206, %rcx # imm = 0xAAAAAAAAAAAAAAAA
+; X86-64-NEXT: imulq %rdi, %rcx
+; X86-64-NEXT: movq %rdi, %rax
; X86-64-NEXT: mulq %r8
-; X86-64-NEXT: addq %rdi, %rdx
+; X86-64-NEXT: addq %rcx, %rdx
; X86-64-NEXT: imulq %rsi, %r8
; X86-64-NEXT: addq %r8, %rdx
; X86-64-NEXT: retq
@@ -932,27 +923,24 @@ define i128 @udiv_i128_12(i128 %x) nounwind {
; WIN64-LABEL: udiv_i128_12:
; WIN64: # %bb.0: # %entry
; WIN64-NEXT: movq %rdx, %r8
-; WIN64-NEXT: movq %rdx, %r9
-; WIN64-NEXT: shlq $62, %r9
-; WIN64-NEXT: shrq $2, %rcx
-; WIN64-NEXT: orq %rcx, %r9
+; WIN64-NEXT: shrdq $2, %rdx, %rcx
; WIN64-NEXT: shrq $2, %r8
-; WIN64-NEXT: movq %r9, %rcx
-; WIN64-NEXT: addq %r8, %rcx
-; WIN64-NEXT: adcq $0, %rcx
+; WIN64-NEXT: movq %rcx, %r9
+; WIN64-NEXT: addq %r8, %r9
+; WIN64-NEXT: adcq $0, %r9
; WIN64-NEXT: movabsq $-6148914691236517205, %r10 # imm = 0xAAAAAAAAAAAAAAAB
-; WIN64-NEXT: movq %rcx, %rax
+; WIN64-NEXT: movq %r9, %rax
; WIN64-NEXT: mulq %r10
; WIN64-NEXT: shrq %rdx
; WIN64-NEXT: leaq (%rdx,%rdx,2), %rax
-; WIN64-NEXT: subq %rax, %rcx
-; WIN64-NEXT: subq %rcx, %r9
+; WIN64-NEXT: subq %rax, %r9
+; WIN64-NEXT: subq %r9, %rcx
; WIN64-NEXT: sbbq $0, %r8
-; WIN64-NEXT: movabsq $-6148914691236517206, %rcx # imm = 0xAAAAAAAAAAAAAAAA
-; WIN64-NEXT: imulq %r9, %rcx
-; WIN64-NEXT: movq %r9, %rax
+; WIN64-NEXT: movabsq $-6148914691236517206, %r9 # imm = 0xAAAAAAAAAAAAAAAA
+; WIN64-NEXT: imulq %rcx, %r9
+; WIN64-NEXT: movq %rcx, %rax
; WIN64-NEXT: mulq %r10
-; WIN64-NEXT: addq %rcx, %rdx
+; WIN64-NEXT: addq %r9, %rdx
; WIN64-NEXT: imulq %r10, %r8
; WIN64-NEXT: addq %r8, %rdx
; WIN64-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/expand-large-fp-optnone.ll b/llvm/test/CodeGen/X86/expand-large-fp-optnone.ll
index 8eaf864e8f5b7..ce38792e60258 100644
--- a/llvm/test/CodeGen/X86/expand-large-fp-optnone.ll
+++ b/llvm/test/CodeGen/X86/expand-large-fp-optnone.ll
@@ -50,17 +50,11 @@ define double @main(i224 %0) #0 {
; CHECK-NEXT: sbbq %rax, %rdx
; CHECK-NEXT: sbbq %rax, %rcx
; CHECK-NEXT: movq %rcx, %r8
-; CHECK-NEXT: shlq $32, %r8
-; CHECK-NEXT: movq %rdx, %rax
-; CHECK-NEXT: shrq $32, %rax
-; CHECK-NEXT: orq %rax, %r8
+; CHECK-NEXT: shldq $32, %rdx, %r8
; CHECK-NEXT: bsrq %r8, %rax
; CHECK-NEXT: xorl $63, %eax
; CHECK-NEXT: movq %rdx, %r10
-; CHECK-NEXT: shlq $32, %r10
-; CHECK-NEXT: movq %rsi, %r11
-; CHECK-NEXT: shrq $32, %r11
-; CHECK-NEXT: orq %r11, %r10
+; CHECK-NEXT: shldq $32, %rsi, %r10
; CHECK-NEXT: bsrq %r10, %r11
; CHECK-NEXT: xorl $63, %r11d
; CHECK-NEXT: orl $64, %r11d
diff --git a/llvm/test/CodeGen/X86/freeze-binary.ll b/llvm/test/CodeGen/X86/freeze-binary.ll
index 35e0bd4a460eb..46b2571e196bb 100644
--- a/llvm/test/CodeGen/X86/freeze-binary.ll
+++ b/llvm/test/CodeGen/X86/freeze-binary.ll
@@ -763,21 +763,17 @@ declare <4 x i32> @llvm.fshr.v4i32(<4 x i32>, <4 x i32>, <4 x i32>)
define i32 @freeze_fshl(i32 %a0, i32 %a1, i32 %a2) nounwind {
; X86-LABEL: freeze_fshl:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: shrl $27, %ecx
-; X86-NEXT: shll $27, %ecx
-; X86-NEXT: shrl $5, %eax
-; X86-NEXT: orl %ecx, %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: shrl $27, %eax
+; X86-NEXT: shldl $27, %ecx, %eax
; X86-NEXT: retl
;
; X64-LABEL: freeze_fshl:
; X64: # %bb.0:
-; X64-NEXT: movl %edx, %eax
-; X64-NEXT: shrl $27, %esi
-; X64-NEXT: shll $27, %esi
-; X64-NEXT: shrl $5, %eax
-; X64-NEXT: orl %esi, %eax
+; X64-NEXT: movl %esi, %eax
+; X64-NEXT: shrl $27, %eax
+; X64-NEXT: shldl $27, %edx, %eax
; X64-NEXT: retq
%f1 = freeze i32 %a1
%f2 = freeze i32 %a2
diff --git a/llvm/test/CodeGen/X86/fshl.ll b/llvm/test/CodeGen/X86/fshl.ll
index 455999b4900b7..7a6e5f825b97c 100644
--- a/llvm/test/CodeGen/X86/fshl.ll
+++ b/llvm/test/CodeGen/X86/fshl.ll
@@ -338,19 +338,15 @@ define i16 @const_shift_i16(i16 %x, i16 %y) nounwind {
define i32 @const_shift_i32(i32 %x, i32 %y) nounwind {
; X86-LABEL: const_shift_i32:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: shll $7, %ecx
-; X86-NEXT: shrl $25, %eax
-; X86-NEXT: orl %ecx, %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: shldl $7, %ecx, %eax
; X86-NEXT: retl
;
; X64-LABEL: const_shift_i32:
; X64: # %bb.0:
-; X64-NEXT: movl %esi, %eax
-; X64-NEXT: shll $7, %edi
-; X64-NEXT: shrl $25, %eax
-; X64-NEXT: orl %edi, %eax
+; X64-NEXT: movl %edi, %eax
+; X64-NEXT: shldl $7, %esi, %eax
; X64-NEXT: retq
%tmp = tail call i32 @llvm.fshl.i32(i32 %x, i32 %y, i32 7)
ret i32 %tmp
@@ -359,26 +355,17 @@ define i32 @const_shift_i32(i32 %x, i32 %y) nounwind {
define i64 @const_shift_i64(i64 %x, i64 %y) nounwind {
; X86-LABEL: const_shift_i64:
; X86: # %bb.0:
-; X86-NEXT: pushl %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %edx, %esi
-; X86-NEXT: shll $7, %esi
-; X86-NEXT: shrl $25, %eax
-; X86-NEXT: orl %esi, %eax
-; X86-NEXT: shll $7, %ecx
-; X86-NEXT: shrl $25, %edx
-; X86-NEXT: orl %ecx, %edx
-; X86-NEXT: popl %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: shrdl $25, %ecx, %eax
+; X86-NEXT: shldl $7, %ecx, %edx
; X86-NEXT: retl
;
; X64-LABEL: const_shift_i64:
; X64: # %bb.0:
-; X64-NEXT: movq %rsi, %rax
-; X64-NEXT: shlq $7, %rdi
-; X64-NEXT: shrq $57, %rax
-; X64-NEXT: orq %rdi, %rax
+; X64-NEXT: movq %rdi, %rax
+; X64-NEXT: shldq $7, %rsi, %rax
; X64-NEXT: retq
%tmp = tail call i64 @llvm.fshl.i64(i64 %x, i64 %y, i64 7)
ret i64 %tmp
diff --git a/llvm/test/CodeGen/X86/funnel-shift-logic-fold.ll b/llvm/test/CodeGen/X86/funnel-shift-logic-fold.ll
index 6949661e03c77..fb875837cb836 100644
--- a/llvm/test/CodeGen/X86/funnel-shift-logic-fold.ll
+++ b/llvm/test/CodeGen/X86/funnel-shift-logic-fold.ll
@@ -52,14 +52,10 @@ define i64 @hoist_fshl_from_xor(i64 %a, i64 %b, i64 %c, i64 %d, i64 %s) nounwind
define i64 @fshl_or_with_different_shift_value(i64 %a, i64 %b, i64 %c, i64 %d) nounwind {
; X64-LABEL: fshl_or_with_different_shift_value:
; X64: # %bb.0:
-; X64-NEXT: movq %rcx, %rax
-; X64-NEXT: shlq $12, %rdi
-; X64-NEXT: shrq $52, %rsi
-; X64-NEXT: orq %rdi, %rsi
-; X64-NEXT: shlq $13, %rdx
-; X64-NEXT: shrq $51, %rax
-; X64-NEXT: orq %rdx, %rax
-; X64-NEXT: orq %rsi, %rax
+; X64-NEXT: movq %rdx, %rax
+; X64-NEXT: shldq $12, %rsi, %rdi
+; X64-NEXT: shldq $13, %rcx, %rax
+; X64-NEXT: orq %rdi, %rax
; X64-NEXT: retq
%fshl.0 = call i64 @llvm.fshl.i64(i64 %a, i64 %b, i64 12)
%fshl.1 = call i64 @llvm.fshl.i64(i64 %c, i64 %d, i64 13)
@@ -70,12 +66,10 @@ define i64 @fshl_or_with_different_shift_value(i64 %a, i64 %b, i64 %c, i64 %d) n
define i64 @hoist_fshl_from_or_const_shift(i64 %a, i64 %b, i64 %c, i64 %d) nounwind {
; X64-LABEL: hoist_fshl_from_or_const_shift:
; X64: # %bb.0:
-; X64-NEXT: movq %rsi, %rax
-; X64-NEXT: orq %rcx, %rax
-; X64-NEXT: orq %rdx, %rdi
-; X64-NEXT: shlq $15, %rdi
-; X64-NEXT: shrq $49, %rax
-; X64-NEXT: orq %rdi, %rax
+; X64-NEXT: movq %rdi, %rax
+; X64-NEXT: orq %rcx, %rsi
+; X64-NEXT: orq %rdx, %rax
+; X64-NEXT: shldq $15, %rsi, %rax
; X64-NEXT: retq
%fshl.0 = call i64 @llvm.fshl.i64(i64 %a, i64 %b, i64 15)
%fshl.1 = call i64 @llvm.fshl.i64(i64 %c, i64 %d, i64 15)
diff --git a/llvm/test/CodeGen/X86/funnel-shift-rot.ll b/llvm/test/CodeGen/X86/funnel-shift-rot.ll
index 183cc18aa7401..7d106fce44555 100644
--- a/llvm/test/CodeGen/X86/funnel-shift-rot.ll
+++ b/llvm/test/CodeGen/X86/funnel-shift-rot.ll
@@ -70,17 +70,11 @@ define i8 @rotl_i8_const_shift7(i8 %x) nounwind {
define i64 @rotl_i64_const_shift(i64 %x) nounwind {
; X86-SSE2-LABEL: rotl_i64_const_shift:
; X86-SSE2: # %bb.0:
-; X86-SSE2-NEXT: pushl %esi
-; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-SSE2-NEXT: leal (,%edx,8), %esi
+; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-SSE2-NEXT: movl %ecx, %eax
-; X86-SSE2-NEXT: shrl $29, %eax
-; X86-SSE2-NEXT: orl %esi, %eax
-; X86-SSE2-NEXT: shll $3, %ecx
-; X86-SSE2-NEXT: shrl $29, %edx
-; X86-SSE2-NEXT: orl %ecx, %edx
-; X86-SSE2-NEXT: popl %esi
+; X86-SSE2-NEXT: shldl $3, %edx, %eax
+; X86-SSE2-NEXT: shldl $3, %ecx, %edx
; X86-SSE2-NEXT: retl
;
; X64-AVX2-LABEL: rotl_i64_const_shift:
diff --git a/llvm/test/CodeGen/X86/funnel-shift.ll b/llvm/test/CodeGen/X86/funnel-shift.ll
index 3a29310614d69..180d17baefa1a 100644
--- a/llvm/test/CodeGen/X86/funnel-shift.ll
+++ b/llvm/test/CodeGen/X86/funnel-shift.ll
@@ -227,19 +227,15 @@ define i7 @fshl_i7_const_fold() {
define i32 @fshl_i32_const_shift(i32 %x, i32 %y) nounwind {
; X86-SSE2-LABEL: fshl_i32_const_shift:
; X86-SSE2: # %bb.0:
-; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-SSE2-NEXT: shll $9, %ecx
-; X86-SSE2-NEXT: shrl $23, %eax
-; X86-SSE2-NEXT: orl %ecx, %eax
+; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: shldl $9, %ecx, %eax
; X86-SSE2-NEXT: retl
;
; X64-AVX-LABEL: fshl_i32_const_shift:
; X64-AVX: # %bb.0:
-; X64-AVX-NEXT: movl %esi, %eax
-; X64-AVX-NEXT: shll $9, %edi
-; X64-AVX-NEXT: shrl $23, %eax
-; X64-AVX-NEXT: orl %edi, %eax
+; X64-AVX-NEXT: movl %edi, %eax
+; X64-AVX-NEXT: shldl $9, %esi, %eax
; X64-AVX-NEXT: retq
%f = call i32 @llvm.fshl.i32(i32 %x, i32 %y, i32 9)
ret i32 %f
@@ -250,19 +246,15 @@ define i32 @fshl_i32_const_shift(i32 %x, i32 %y) nounwind {
define i32 @fshl_i32_const_overshift(i32 %x, i32 %y) nounwind {
; X86-SSE2-LABEL: fshl_i32_const_overshift:
; X86-SSE2: # %bb.0:
-; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-SSE2-NEXT: shll $9, %ecx
-; X86-SSE2-NEXT: shrl $23, %eax
-; X86-SSE2-NEXT: orl %ecx, %eax
+; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: shldl $9, %ecx, %eax
; X86-SSE2-NEXT: retl
;
; X64-AVX-LABEL: fshl_i32_const_overshift:
; X64-AVX: # %bb.0:
-; X64-AVX-NEXT: movl %esi, %eax
-; X64-AVX-NEXT: shll $9, %edi
-; X64-AVX-NEXT: shrl $23, %eax
-; X64-AVX-NEXT: orl %edi, %eax
+; X64-AVX-NEXT: movl %edi, %eax
+; X64-AVX-NEXT: shldl $9, %esi, %eax
; X64-AVX-NEXT: retq
%f = call i32 @llvm.fshl.i32(i32 %x, i32 %y, i32 41)
ret i32 %f
@@ -273,26 +265,17 @@ define i32 @fshl_i32_const_overshift(i32 %x, i32 %y) nounwind {
define i64 @fshl_i64_const_overshift(i64 %x, i64 %y) nounwind {
; X86-SSE2-LABEL: fshl_i64_const_overshift:
; X86-SSE2: # %bb.0:
-; X86-SSE2-NEXT: pushl %esi
-; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-SSE2-NEXT: shll $9, %ecx
-; X86-SSE2-NEXT: movl %esi, %edx
-; X86-SSE2-NEXT: shrl $23, %edx
-; X86-SSE2-NEXT: orl %ecx, %edx
-; X86-SSE2-NEXT: shll $9, %esi
-; X86-SSE2-NEXT: shrl $23, %eax
-; X86-SSE2-NEXT: orl %esi, %eax
-; X86-SSE2-NEXT: popl %esi
+; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SSE2-NEXT: shldl $9, %ecx, %edx
+; X86-SSE2-NEXT: shrdl $23, %ecx, %eax
; X86-SSE2-NEXT: retl
;
; X64-AVX-LABEL: fshl_i64_const_overshift:
; X64-AVX: # %bb.0:
-; X64-AVX-NEXT: movq %rsi, %rax
-; X64-AVX-NEXT: shlq $41, %rdi
-; X64-AVX-NEXT: shrq $23, %rax
-; X64-AVX-NEXT: orq %rdi, %rax
+; X64-AVX-NEXT: movq %rdi, %rax
+; X64-AVX-NEXT: shldq $41, %rsi, %rax
; X64-AVX-NEXT: retq
%f = call i64 @llvm.fshl.i64(i64 %x, i64 %y, i64 105)
ret i64 %f
@@ -415,19 +398,15 @@ define i7 @fshr_i7_const_fold() nounwind {
define i32 @fshl_i32_demandedbits(i32 %a0, i32 %a1) nounwind {
; X86-SSE2-LABEL: fshl_i32_demandedbits:
; X86-SSE2: # %bb.0:
-; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-SSE2-NEXT: shll $9, %ecx
-; X86-SSE2-NEXT: shrl $23, %eax
-; X86-SSE2-NEXT: orl %ecx, %eax
+; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: shldl $9, %ecx, %eax
; X86-SSE2-NEXT: retl
;
; X64-AVX-LABEL: fshl_i32_demandedbits:
; X64-AVX: # %bb.0:
-; X64-AVX-NEXT: movl %esi, %eax
-; X64-AVX-NEXT: shll $9, %edi
-; X64-AVX-NEXT: shrl $23, %eax
-; X64-AVX-NEXT: orl %edi, %eax
+; X64-AVX-NEXT: movl %edi, %eax
+; X64-AVX-NEXT: shldl $9, %esi, %eax
; X64-AVX-NEXT: retq
%x = or i32 %a0, 2147483648
%y = or i32 %a1, 1
@@ -1115,27 +1094,25 @@ define void @PR45265(i32 %0, ptr nocapture readonly %1) nounwind {
; X86-SSE2-NEXT: pushl %edi
; X86-SSE2-NEXT: pushl %esi
; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %esi
; X86-SSE2-NEXT: leal (%ecx,%ecx,2), %edi
-; X86-SSE2-NEXT: movzwl 8(%edx,%edi,4), %esi
-; X86-SSE2-NEXT: movzbl 10(%edx,%edi,4), %eax
+; X86-SSE2-NEXT: movzwl 8(%esi,%edi,4), %edx
+; X86-SSE2-NEXT: movzbl 10(%esi,%edi,4), %eax
; X86-SSE2-NEXT: shll $16, %eax
-; X86-SSE2-NEXT: orl %esi, %eax
-; X86-SSE2-NEXT: movl 4(%edx,%edi,4), %edx
-; X86-SSE2-NEXT: shll $24, %esi
-; X86-SSE2-NEXT: shrl $8, %edx
-; X86-SSE2-NEXT: orl %esi, %edx
-; X86-SSE2-NEXT: xorl %ecx, %edx
+; X86-SSE2-NEXT: orl %edx, %eax
+; X86-SSE2-NEXT: movl 4(%esi,%edi,4), %esi
+; X86-SSE2-NEXT: shll $24, %edx
+; X86-SSE2-NEXT: shrl $8, %esi
+; X86-SSE2-NEXT: orl %edx, %esi
+; X86-SSE2-NEXT: xorl %ecx, %esi
; X86-SSE2-NEXT: sarl $31, %ecx
; X86-SSE2-NEXT: shll $8, %eax
-; X86-SSE2-NEXT: movl %eax, %esi
-; X86-SSE2-NEXT: sarl $8, %esi
+; X86-SSE2-NEXT: movl %eax, %edx
+; X86-SSE2-NEXT: sarl $8, %edx
; X86-SSE2-NEXT: sarl $31, %eax
-; X86-SSE2-NEXT: shll $24, %eax
-; X86-SSE2-NEXT: shrl $8, %esi
+; X86-SSE2-NEXT: shldl $24, %edx, %eax
+; X86-SSE2-NEXT: xorl %ecx, %eax
; X86-SSE2-NEXT: orl %eax, %esi
-; X86-SSE2-NEXT: xorl %ecx, %esi
-; X86-SSE2-NEXT: orl %esi, %edx
; X86-SSE2-NEXT: jne .LBB50_1
; X86-SSE2-NEXT: # %bb.2:
; X86-SSE2-NEXT: popl %esi
diff --git a/llvm/test/CodeGen/X86/icmp-shift-opt.ll b/llvm/test/CodeGen/X86/icmp-shift-opt.ll
index 49d292a3a03fa..0296c2a011e25 100644
--- a/llvm/test/CodeGen/X86/icmp-shift-opt.ll
+++ b/llvm/test/CodeGen/X86/icmp-shift-opt.ll
@@ -33,9 +33,7 @@ define i128 @opt_setcc_lt_power_of_2(i128 %a) nounwind {
; X86-NEXT: movl %edi, %esi
; X86-NEXT: orl %edx, %esi
; X86-NEXT: orl %ecx, %esi
-; X86-NEXT: shll $4, %ebx
-; X86-NEXT: shrl $28, %esi
-; X86-NEXT: orl %ebx, %esi
+; X86-NEXT: shrdl $28, %ebx, %esi
; X86-NEXT: movl %eax, %esi
; X86-NEXT: jne .LBB0_1
; X86-NEXT: # %bb.2: # %exit
@@ -206,32 +204,23 @@ define i1 @opt_setcc_shl_eq_zero_multiple_shl_users(i128 %a) nounwind {
; X86-NEXT: andl $-16, %esp
; X86-NEXT: subl $16, %esp
; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: movl 12(%ebp), %esi
-; X86-NEXT: movl 16(%ebp), %ecx
+; X86-NEXT: movl 12(%ebp), %ecx
+; X86-NEXT: movl 16(%ebp), %edx
+; X86-NEXT: movl 20(%ebp), %esi
+; X86-NEXT: shldl $17, %edx, %esi
+; X86-NEXT: shldl $17, %ecx, %edx
+; X86-NEXT: shldl $17, %eax, %ecx
+; X86-NEXT: shll $17, %eax
; X86-NEXT: movl %ecx, %edi
-; X86-NEXT: shll $17, %edi
-; X86-NEXT: movl %esi, %edx
-; X86-NEXT: shrl $15, %edx
-; X86-NEXT: orl %edi, %edx
-; X86-NEXT: movl 20(%ebp), %edi
-; X86-NEXT: shll $17, %edi
-; X86-NEXT: shrl $15, %ecx
-; X86-NEXT: orl %edi, %ecx
-; X86-NEXT: movl %eax, %edi
-; X86-NEXT: shll $17, %edi
-; X86-NEXT: shll $17, %esi
-; X86-NEXT: shrl $15, %eax
-; X86-NEXT: orl %esi, %eax
-; X86-NEXT: movl %eax, %esi
-; X86-NEXT: orl %ecx, %esi
-; X86-NEXT: movl %edi, %ebx
+; X86-NEXT: orl %esi, %edi
+; X86-NEXT: movl %eax, %ebx
; X86-NEXT: orl %edx, %ebx
-; X86-NEXT: orl %esi, %ebx
+; X86-NEXT: orl %edi, %ebx
; X86-NEXT: sete %bl
-; X86-NEXT: pushl %ecx
+; X86-NEXT: pushl %esi
; X86-NEXT: pushl %edx
+; X86-NEXT: pushl %ecx
; X86-NEXT: pushl %eax
-; X86-NEXT: pushl %edi
; X86-NEXT: calll use at PLT
; X86-NEXT: addl $16, %esp
; X86-NEXT: movl %ebx, %eax
@@ -245,10 +234,7 @@ define i1 @opt_setcc_shl_eq_zero_multiple_shl_users(i128 %a) nounwind {
; X64-LABEL: opt_setcc_shl_eq_zero_multiple_shl_users:
; X64: # %bb.0:
; X64-NEXT: pushq %rbx
-; X64-NEXT: shlq $17, %rsi
-; X64-NEXT: movq %rdi, %rax
-; X64-NEXT: shrq $47, %rax
-; X64-NEXT: orq %rax, %rsi
+; X64-NEXT: shldq $17, %rdi, %rsi
; X64-NEXT: shlq $17, %rdi
; X64-NEXT: movq %rdi, %rax
; X64-NEXT: orq %rsi, %rax
@@ -297,41 +283,27 @@ define i1 @opt_setcc_expanded_shl_correct_shifts(i64 %a, i64 %b) nounwind {
define i1 @opt_setcc_expanded_shl_wrong_shifts(i64 %a, i64 %b) nounwind {
; X86-LABEL: opt_setcc_expanded_shl_wrong_shifts:
; X86: # %bb.0:
-; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT: shll $17, %edi
-; X86-NEXT: movl %esi, %edx
-; X86-NEXT: shrl $15, %edx
-; X86-NEXT: orl %edi, %edx
-; X86-NEXT: shll $17, %esi
-; X86-NEXT: movl %ecx, %edi
-; X86-NEXT: shrl $15, %edi
-; X86-NEXT: orl %esi, %edi
-; X86-NEXT: movl %eax, %esi
-; X86-NEXT: shll $18, %esi
-; X86-NEXT: orl %edi, %esi
-; X86-NEXT: shll $18, %ecx
-; X86-NEXT: shrl $14, %eax
-; X86-NEXT: orl %ecx, %eax
+; X86-NEXT: shldl $17, %edx, %esi
+; X86-NEXT: shldl $17, %ecx, %edx
+; X86-NEXT: shldl $18, %eax, %ecx
+; X86-NEXT: shll $18, %eax
; X86-NEXT: orl %edx, %eax
-; X86-NEXT: orl %esi, %eax
+; X86-NEXT: orl %esi, %ecx
+; X86-NEXT: orl %eax, %ecx
; X86-NEXT: sete %al
; X86-NEXT: popl %esi
-; X86-NEXT: popl %edi
; X86-NEXT: retl
;
; X64-LABEL: opt_setcc_expanded_shl_wrong_shifts:
; X64: # %bb.0:
-; X64-NEXT: shlq $17, %rdi
-; X64-NEXT: movq %rsi, %rax
-; X64-NEXT: shrq $47, %rax
-; X64-NEXT: orq %rdi, %rax
+; X64-NEXT: shldq $17, %rsi, %rdi
; X64-NEXT: shlq $18, %rsi
-; X64-NEXT: orq %rax, %rsi
+; X64-NEXT: orq %rdi, %rsi
; X64-NEXT: sete %al
; X64-NEXT: retq
%shl.a = shl i64 %a, 17
diff --git a/llvm/test/CodeGen/X86/imul.ll b/llvm/test/CodeGen/X86/imul.ll
index 873f30e7a1394..cc623edc2691f 100644
--- a/llvm/test/CodeGen/X86/imul.ll
+++ b/llvm/test/CodeGen/X86/imul.ll
@@ -29,12 +29,10 @@ define i64 @mul4_64(i64 %A) {
;
; X86-LABEL: mul4_64:
; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: leal (,%edx,4), %eax
-; X86-NEXT: shll $2, %ecx
-; X86-NEXT: shrl $30, %edx
-; X86-NEXT: orl %ecx, %edx
+; X86-NEXT: shldl $2, %eax, %edx
+; X86-NEXT: shll $2, %eax
; X86-NEXT: retl
%mul = mul i64 %A, 4
ret i64 %mul
@@ -65,13 +63,10 @@ define i64 @mul4096_64(i64 %A) {
;
; X86-LABEL: mul4096_64:
; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %edx, %eax
+; X86-NEXT: shldl $12, %eax, %edx
; X86-NEXT: shll $12, %eax
-; X86-NEXT: shll $12, %ecx
-; X86-NEXT: shrl $20, %edx
-; X86-NEXT: orl %ecx, %edx
; X86-NEXT: retl
%mul = mul i64 %A, 4096
ret i64 %mul
diff --git a/llvm/test/CodeGen/X86/isel-shift.ll b/llvm/test/CodeGen/X86/isel-shift.ll
index 7afc5ab1236cf..476dcf04dbaa2 100644
--- a/llvm/test/CodeGen/X86/isel-shift.ll
+++ b/llvm/test/CodeGen/X86/isel-shift.ll
@@ -693,22 +693,17 @@ define i32 @shl_imm1_i32(i32 %a) {
define i64 @shl_imm1_i64(i64 %a) {
; SDAG-X86-LABEL: shl_imm1_i64:
; SDAG-X86: ## %bb.0:
+; SDAG-X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; SDAG-X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; SDAG-X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; SDAG-X86-NEXT: leal (%edx,%edx), %eax
-; SDAG-X86-NEXT: shll %ecx
-; SDAG-X86-NEXT: shrl $31, %edx
-; SDAG-X86-NEXT: orl %ecx, %edx
+; SDAG-X86-NEXT: shldl $1, %eax, %edx
+; SDAG-X86-NEXT: addl %eax, %eax
; SDAG-X86-NEXT: retl
;
; FASTISEL-X86-LABEL: shl_imm1_i64:
; FASTISEL-X86: ## %bb.0:
-; FASTISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; FASTISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; FASTISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; FASTISEL-X86-NEXT: shll %ecx
-; FASTISEL-X86-NEXT: movl %eax, %edx
-; FASTISEL-X86-NEXT: shrl $31, %edx
-; FASTISEL-X86-NEXT: orl %ecx, %edx
+; FASTISEL-X86-NEXT: shldl $1, %eax, %edx
; FASTISEL-X86-NEXT: addl %eax, %eax
; FASTISEL-X86-NEXT: retl
;
@@ -998,23 +993,17 @@ define i32 @shl_imm4_i32(i32 %a) {
define i64 @shl_imm4_i64(i64 %a) {
; SDAG-X86-LABEL: shl_imm4_i64:
; SDAG-X86: ## %bb.0:
+; SDAG-X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; SDAG-X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; SDAG-X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; SDAG-X86-NEXT: movl %edx, %eax
+; SDAG-X86-NEXT: shldl $4, %eax, %edx
; SDAG-X86-NEXT: shll $4, %eax
-; SDAG-X86-NEXT: shll $4, %ecx
-; SDAG-X86-NEXT: shrl $28, %edx
-; SDAG-X86-NEXT: orl %ecx, %edx
; SDAG-X86-NEXT: retl
;
; FASTISEL-X86-LABEL: shl_imm4_i64:
; FASTISEL-X86: ## %bb.0:
-; FASTISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; FASTISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; FASTISEL-X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; FASTISEL-X86-NEXT: shll $4, %ecx
-; FASTISEL-X86-NEXT: movl %eax, %edx
-; FASTISEL-X86-NEXT: shrl $28, %edx
-; FASTISEL-X86-NEXT: orl %ecx, %edx
+; FASTISEL-X86-NEXT: shldl $4, %eax, %edx
; FASTISEL-X86-NEXT: shll $4, %eax
; FASTISEL-X86-NEXT: retl
;
diff --git a/llvm/test/CodeGen/X86/known-bits.ll b/llvm/test/CodeGen/X86/known-bits.ll
index c80520bb32556..58a0595e4322a 100644
--- a/llvm/test/CodeGen/X86/known-bits.ll
+++ b/llvm/test/CodeGen/X86/known-bits.ll
@@ -91,20 +91,18 @@ define i128 @knownbits_mask_addc_shl(i64 %a0, i64 %a1, i64 %a2) nounwind {
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-NEXT: movl $-1024, %esi # imm = 0xFC00
; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
; X86-NEXT: andl %esi, %edi
; X86-NEXT: andl {{[0-9]+}}(%esp), %esi
; X86-NEXT: addl %edi, %esi
-; X86-NEXT: adcl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: adcl $0, %edx
-; X86-NEXT: shll $22, %edx
-; X86-NEXT: shrdl $10, %ecx, %esi
-; X86-NEXT: shrl $10, %ecx
-; X86-NEXT: orl %edx, %ecx
-; X86-NEXT: movl %esi, 8(%eax)
+; X86-NEXT: adcl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: adcl $0, %ecx
+; X86-NEXT: shldl $22, %edx, %ecx
+; X86-NEXT: shldl $22, %esi, %edx
+; X86-NEXT: movl %edx, 8(%eax)
; X86-NEXT: movl %ecx, 12(%eax)
; X86-NEXT: movl $0, 4(%eax)
; X86-NEXT: movl $0, (%eax)
@@ -118,9 +116,7 @@ define i128 @knownbits_mask_addc_shl(i64 %a0, i64 %a1, i64 %a2) nounwind {
; X64-NEXT: andq $-1024, %rsi # imm = 0xFC00
; X64-NEXT: addq %rdi, %rsi
; X64-NEXT: adcl $0, %edx
-; X64-NEXT: shlq $54, %rdx
-; X64-NEXT: shrq $10, %rsi
-; X64-NEXT: orq %rsi, %rdx
+; X64-NEXT: shldq $54, %rsi, %rdx
; X64-NEXT: xorl %eax, %eax
; X64-NEXT: retq
%1 = and i64 %a0, -1024
diff --git a/llvm/test/CodeGen/X86/legalize-shl-vec.ll b/llvm/test/CodeGen/X86/legalize-shl-vec.ll
index 645a400bfc496..4cfb050958abb 100644
--- a/llvm/test/CodeGen/X86/legalize-shl-vec.ll
+++ b/llvm/test/CodeGen/X86/legalize-shl-vec.ll
@@ -5,60 +5,30 @@
define <2 x i256> @test_shl(<2 x i256> %In) nounwind {
; X86-LABEL: test_shl:
; X86: # %bb.0:
-; X86-NEXT: pushl %ebp
-; X86-NEXT: pushl %ebx
-; X86-NEXT: pushl %edi
-; X86-NEXT: pushl %esi
-; X86-NEXT: subl $12, %esp
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X86-NEXT: leal (,%esi,4), %eax
-; X86-NEXT: movl %edx, %ecx
-; X86-NEXT: shrl $30, %ecx
-; X86-NEXT: orl %eax, %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: leal (,%edi,4), %eax
-; X86-NEXT: movl %ebx, %ecx
-; X86-NEXT: shrl $30, %ecx
-; X86-NEXT: orl %eax, %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl %eax, %ebx
-; X86-NEXT: shrl $30, %ebx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X86-NEXT: leal (,%ebp,4), %ecx
-; X86-NEXT: orl %ecx, %ebx
-; X86-NEXT: shll $2, %edx
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: leal (,%ecx,4), %edi
-; X86-NEXT: movl %edi, (%esp) # 4-byte Spill
-; X86-NEXT: shrl $30, %ecx
-; X86-NEXT: orl %edx, %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: shll $2, %edx
-; X86-NEXT: shrl $30, %esi
-; X86-NEXT: orl %edx, %esi
-; X86-NEXT: shll $2, %eax
+; X86-NEXT: shldl $2, %ecx, %edx
+; X86-NEXT: movl %edx, 60(%eax)
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: shrl $30, %edx
-; X86-NEXT: orl %eax, %edx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: shll $2, %eax
-; X86-NEXT: shrl $30, %ebp
-; X86-NEXT: orl %eax, %ebp
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl %ebp, 60(%eax)
-; X86-NEXT: movl %ebx, 56(%eax)
+; X86-NEXT: shldl $2, %edx, %ecx
+; X86-NEXT: movl %ecx, 56(%eax)
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: shldl $2, %ecx, %edx
; X86-NEXT: movl %edx, 52(%eax)
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: movl %edx, 48(%eax)
-; X86-NEXT: movl %esi, 44(%eax)
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: movl %edx, 40(%eax)
-; X86-NEXT: movl %ecx, 36(%eax)
-; X86-NEXT: movl (%esp), %ecx # 4-byte Reload
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: shldl $2, %edx, %ecx
+; X86-NEXT: movl %ecx, 48(%eax)
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: shldl $2, %ecx, %edx
+; X86-NEXT: movl %edx, 44(%eax)
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: shldl $2, %edx, %ecx
+; X86-NEXT: movl %ecx, 40(%eax)
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: shldl $2, %ecx, %edx
+; X86-NEXT: movl %edx, 36(%eax)
+; X86-NEXT: shll $2, %ecx
; X86-NEXT: movl %ecx, 32(%eax)
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: shll $31, %ecx
@@ -70,11 +40,6 @@ define <2 x i256> @test_shl(<2 x i256> %In) nounwind {
; X86-NEXT: movl $0, 8(%eax)
; X86-NEXT: movl $0, 4(%eax)
; X86-NEXT: movl $0, (%eax)
-; X86-NEXT: addl $12, %esp
-; X86-NEXT: popl %esi
-; X86-NEXT: popl %edi
-; X86-NEXT: popl %ebx
-; X86-NEXT: popl %ebp
; X86-NEXT: retl $4
;
; X64-LABEL: test_shl:
@@ -83,21 +48,14 @@ define <2 x i256> @test_shl(<2 x i256> %In) nounwind {
; X64-NEXT: movq {{[0-9]+}}(%rsp), %rcx
; X64-NEXT: movq {{[0-9]+}}(%rsp), %rdx
; X64-NEXT: movq {{[0-9]+}}(%rsp), %rdi
-; X64-NEXT: leaq (,%rdi,4), %r8
-; X64-NEXT: movq %r9, %r10
-; X64-NEXT: shrq $62, %r10
-; X64-NEXT: orq %r8, %r10
-; X64-NEXT: leaq (,%rdx,4), %r8
-; X64-NEXT: shrq $62, %rdi
-; X64-NEXT: orq %r8, %rdi
-; X64-NEXT: shlq $2, %rcx
-; X64-NEXT: shrq $62, %rdx
-; X64-NEXT: orq %rcx, %rdx
+; X64-NEXT: shldq $2, %rdx, %rcx
+; X64-NEXT: shldq $2, %rdi, %rdx
+; X64-NEXT: shldq $2, %r9, %rdi
; X64-NEXT: shlq $63, %rsi
; X64-NEXT: shlq $2, %r9
-; X64-NEXT: movq %rdx, 56(%rax)
-; X64-NEXT: movq %rdi, 48(%rax)
-; X64-NEXT: movq %r10, 40(%rax)
+; X64-NEXT: movq %rcx, 56(%rax)
+; X64-NEXT: movq %rdx, 48(%rax)
+; X64-NEXT: movq %rdi, 40(%rax)
; X64-NEXT: movq %r9, 32(%rax)
; X64-NEXT: movq %rsi, 24(%rax)
; X64-NEXT: xorps %xmm0, %xmm0
@@ -117,43 +75,37 @@ define <2 x i256> @test_srl(<2 x i256> %In) nounwind {
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
; X86-NEXT: subl $8, %esp
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X86-NEXT: movl %ebx, %eax
-; X86-NEXT: shll $28, %eax
-; X86-NEXT: shrdl $4, %esi, %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shrl $4, %esi
-; X86-NEXT: orl %eax, %esi
-; X86-NEXT: movl %edx, %eax
-; X86-NEXT: shll $28, %eax
-; X86-NEXT: shrdl $4, %ebp, %ebx
-; X86-NEXT: shrl $4, %ebp
-; X86-NEXT: orl %eax, %ebp
-; X86-NEXT: movl %edi, %eax
-; X86-NEXT: shll $28, %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT: movl %edi, %esi
+; X86-NEXT: shldl $28, %eax, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shrdl $4, %eax, %ecx
+; X86-NEXT: movl %ecx, (%esp) # 4-byte Spill
+; X86-NEXT: movl %edx, %ebp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: shldl $28, %eax, %ebp
+; X86-NEXT: shrdl $4, %eax, %edi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %ebx, %esi
+; X86-NEXT: shldl $28, %eax, %esi
+; X86-NEXT: shrdl $4, %eax, %edx
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: shrdl $4, %ecx, %edx
-; X86-NEXT: movl %edx, (%esp) # 4-byte Spill
+; X86-NEXT: shrdl $4, %ecx, %ebx
; X86-NEXT: shrl $4, %ecx
-; X86-NEXT: orl %eax, %ecx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: shrdl $4, %edx, %edi
-; X86-NEXT: shrl $4, %edx
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl %edx, 60(%eax)
-; X86-NEXT: movl %edi, 56(%eax)
-; X86-NEXT: movl %ecx, 52(%eax)
-; X86-NEXT: movl (%esp), %ecx # 4-byte Reload
-; X86-NEXT: movl %ecx, 48(%eax)
+; X86-NEXT: movl %ecx, 60(%eax)
+; X86-NEXT: movl %ebx, 56(%eax)
+; X86-NEXT: movl %esi, 52(%eax)
+; X86-NEXT: movl %edx, 48(%eax)
; X86-NEXT: movl %ebp, 44(%eax)
-; X86-NEXT: movl %ebx, 40(%eax)
-; X86-NEXT: movl %esi, 36(%eax)
+; X86-NEXT: movl %edi, 40(%eax)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %ecx, 36(%eax)
+; X86-NEXT: movl (%esp), %ecx # 4-byte Reload
; X86-NEXT: movl %ecx, 32(%eax)
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: shrl $31, %ecx
@@ -179,21 +131,18 @@ define <2 x i256> @test_srl(<2 x i256> %In) nounwind {
; X64-NEXT: movq {{[0-9]+}}(%rsp), %rdx
; X64-NEXT: movq {{[0-9]+}}(%rsp), %rsi
; X64-NEXT: shrdq $4, %rsi, %r9
-; X64-NEXT: movq %rdx, %rdi
-; X64-NEXT: shlq $60, %rdi
-; X64-NEXT: shrq $4, %rsi
-; X64-NEXT: orq %rdi, %rsi
+; X64-NEXT: shrdq $4, %rdx, %rsi
; X64-NEXT: shrdq $4, %rcx, %rdx
; X64-NEXT: shrq $63, %r8
; X64-NEXT: shrq $4, %rcx
-; X64-NEXT: movq %rcx, 56(%rax)
-; X64-NEXT: movq %rdx, 48(%rax)
-; X64-NEXT: movq %rsi, 40(%rax)
-; X64-NEXT: movq %r9, 32(%rax)
-; X64-NEXT: movq %r8, (%rax)
+; X64-NEXT: movq %rcx, 56(%rdi)
+; X64-NEXT: movq %rdx, 48(%rdi)
+; X64-NEXT: movq %rsi, 40(%rdi)
+; X64-NEXT: movq %r9, 32(%rdi)
+; X64-NEXT: movq %r8, (%rdi)
; X64-NEXT: xorps %xmm0, %xmm0
-; X64-NEXT: movaps %xmm0, 16(%rax)
-; X64-NEXT: movq $0, 8(%rax)
+; X64-NEXT: movaps %xmm0, 16(%rdi)
+; X64-NEXT: movq $0, 8(%rdi)
; X64-NEXT: retq
%Amt = insertelement <2 x i256> <i256 3, i256 4>, i256 255, i32 0
%Out = lshr <2 x i256> %In, %Amt
@@ -208,43 +157,37 @@ define <2 x i256> @test_sra(<2 x i256> %In) nounwind {
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
; X86-NEXT: subl $8, %esp
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X86-NEXT: movl %ebx, %eax
-; X86-NEXT: shll $26, %eax
-; X86-NEXT: shrdl $6, %esi, %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shrl $6, %esi
-; X86-NEXT: orl %eax, %esi
-; X86-NEXT: movl %edx, %eax
-; X86-NEXT: shll $26, %eax
-; X86-NEXT: shrdl $6, %ebp, %ebx
-; X86-NEXT: shrl $6, %ebp
-; X86-NEXT: orl %eax, %ebp
-; X86-NEXT: movl %edi, %eax
-; X86-NEXT: shll $26, %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT: movl %edi, %esi
+; X86-NEXT: shldl $26, %eax, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shrdl $6, %eax, %ecx
+; X86-NEXT: movl %ecx, (%esp) # 4-byte Spill
+; X86-NEXT: movl %edx, %ebp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: shldl $26, %eax, %ebp
+; X86-NEXT: shrdl $6, %eax, %edi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %ebx, %esi
+; X86-NEXT: shldl $26, %eax, %esi
+; X86-NEXT: shrdl $6, %eax, %edx
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: shrdl $6, %ecx, %edx
-; X86-NEXT: movl %edx, (%esp) # 4-byte Spill
-; X86-NEXT: shrl $6, %ecx
-; X86-NEXT: orl %eax, %ecx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: shrdl $6, %edx, %edi
-; X86-NEXT: sarl $6, %edx
+; X86-NEXT: shrdl $6, %ecx, %ebx
+; X86-NEXT: sarl $6, %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl %edx, 60(%eax)
-; X86-NEXT: movl %edi, 56(%eax)
-; X86-NEXT: movl %ecx, 52(%eax)
-; X86-NEXT: movl (%esp), %ecx # 4-byte Reload
-; X86-NEXT: movl %ecx, 48(%eax)
+; X86-NEXT: movl %ecx, 60(%eax)
+; X86-NEXT: movl %ebx, 56(%eax)
+; X86-NEXT: movl %esi, 52(%eax)
+; X86-NEXT: movl %edx, 48(%eax)
; X86-NEXT: movl %ebp, 44(%eax)
-; X86-NEXT: movl %ebx, 40(%eax)
-; X86-NEXT: movl %esi, 36(%eax)
+; X86-NEXT: movl %edi, 40(%eax)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %ecx, 36(%eax)
+; X86-NEXT: movl (%esp), %ecx # 4-byte Reload
; X86-NEXT: movl %ecx, 32(%eax)
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: sarl $31, %ecx
@@ -270,21 +213,18 @@ define <2 x i256> @test_sra(<2 x i256> %In) nounwind {
; X64-NEXT: movq {{[0-9]+}}(%rsp), %rdx
; X64-NEXT: movq {{[0-9]+}}(%rsp), %rsi
; X64-NEXT: shrdq $6, %rsi, %r9
-; X64-NEXT: movq %rdx, %rdi
-; X64-NEXT: shlq $58, %rdi
-; X64-NEXT: shrq $6, %rsi
-; X64-NEXT: orq %rdi, %rsi
+; X64-NEXT: shrdq $6, %rdx, %rsi
; X64-NEXT: shrdq $6, %rcx, %rdx
; X64-NEXT: sarq $63, %r8
; X64-NEXT: sarq $6, %rcx
-; X64-NEXT: movq %rcx, 56(%rax)
-; X64-NEXT: movq %rdx, 48(%rax)
-; X64-NEXT: movq %rsi, 40(%rax)
-; X64-NEXT: movq %r9, 32(%rax)
-; X64-NEXT: movq %r8, 24(%rax)
-; X64-NEXT: movq %r8, 16(%rax)
-; X64-NEXT: movq %r8, 8(%rax)
-; X64-NEXT: movq %r8, (%rax)
+; X64-NEXT: movq %rcx, 56(%rdi)
+; X64-NEXT: movq %rdx, 48(%rdi)
+; X64-NEXT: movq %rsi, 40(%rdi)
+; X64-NEXT: movq %r9, 32(%rdi)
+; X64-NEXT: movq %r8, 24(%rdi)
+; X64-NEXT: movq %r8, 16(%rdi)
+; X64-NEXT: movq %r8, 8(%rdi)
+; X64-NEXT: movq %r8, (%rdi)
; X64-NEXT: retq
%Amt = insertelement <2 x i256> <i256 5, i256 6>, i256 255, i32 0
%Out = ashr <2 x i256> %In, %Amt
diff --git a/llvm/test/CodeGen/X86/logic-shift.ll b/llvm/test/CodeGen/X86/logic-shift.ll
index 97d6d12a4a903..104151c76bc5d 100644
--- a/llvm/test/CodeGen/X86/logic-shift.ll
+++ b/llvm/test/CodeGen/X86/logic-shift.ll
@@ -858,11 +858,9 @@ define i64 @mix_logic_shl(i64 %x0, i64 %x1, i64 %y, i64 %z) {
define i32 @or_fshl_commute0(i32 %x, i32 %y) {
; CHECK-LABEL: or_fshl_commute0:
; CHECK: # %bb.0:
-; CHECK-NEXT: movl %edi, %eax
-; CHECK-NEXT: orl %edi, %esi
-; CHECK-NEXT: shll $5, %esi
-; CHECK-NEXT: shrl $27, %eax
-; CHECK-NEXT: orl %esi, %eax
+; CHECK-NEXT: movl %esi, %eax
+; CHECK-NEXT: orl %edi, %eax
+; CHECK-NEXT: shldl $5, %edi, %eax
; CHECK-NEXT: retq
%or1 = or i32 %x, %y
%sh1 = shl i32 %or1, 5
@@ -876,9 +874,7 @@ define i64 @or_fshl_commute1(i64 %x, i64 %y) {
; CHECK: # %bb.0:
; CHECK-NEXT: movl %edi, %eax
; CHECK-NEXT: orl %esi, %eax
-; CHECK-NEXT: shlq $35, %rax
-; CHECK-NEXT: shrq $29, %rdi
-; CHECK-NEXT: orq %rdi, %rax
+; CHECK-NEXT: shldq $35, %rdi, %rax
; CHECK-NEXT: retq
%or1 = or i64 %y, %x
%sh1 = shl i64 %or1, 35
@@ -942,9 +938,7 @@ define i64 @or_fshr_commute0(i64 %x, i64 %y) {
; CHECK: # %bb.0:
; CHECK-NEXT: movq %rsi, %rax
; CHECK-NEXT: orq %rdi, %rax
-; CHECK-NEXT: shlq $40, %rdi
-; CHECK-NEXT: shrq $24, %rax
-; CHECK-NEXT: orq %rdi, %rax
+; CHECK-NEXT: shrdq $24, %rdi, %rax
; CHECK-NEXT: retq
%or1 = or i64 %x, %y
%sh1 = shl i64 %x, 40
@@ -956,11 +950,9 @@ define i64 @or_fshr_commute0(i64 %x, i64 %y) {
define i32 @or_fshr_commute1(i32 %x, i32 %y) {
; CHECK-LABEL: or_fshr_commute1:
; CHECK: # %bb.0:
-; CHECK-NEXT: # kill: def $edi killed $edi def $rdi
-; CHECK-NEXT: orl %edi, %esi
-; CHECK-NEXT: leal (,%rdi,8), %eax
-; CHECK-NEXT: shrl $29, %esi
-; CHECK-NEXT: orl %esi, %eax
+; CHECK-NEXT: movl %esi, %eax
+; CHECK-NEXT: orl %edi, %eax
+; CHECK-NEXT: shrdl $29, %edi, %eax
; CHECK-NEXT: retq
%or1 = or i32 %y, %x
%sh1 = shl i32 %x, 3
diff --git a/llvm/test/CodeGen/X86/mul-constant-i64.ll b/llvm/test/CodeGen/X86/mul-constant-i64.ll
index 77d1252fa076e..40d591f8d1be8 100644
--- a/llvm/test/CodeGen/X86/mul-constant-i64.ll
+++ b/llvm/test/CodeGen/X86/mul-constant-i64.ll
@@ -32,22 +32,18 @@ define i64 @test_mul_by_1(i64 %x) nounwind {
define i64 @test_mul_by_2(i64 %x) {
; X86-LABEL: test_mul_by_2:
; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: leal (%edx,%edx), %eax
-; X86-NEXT: shll %ecx
-; X86-NEXT: shrl $31, %edx
-; X86-NEXT: orl %ecx, %edx
+; X86-NEXT: shldl $1, %eax, %edx
+; X86-NEXT: addl %eax, %eax
; X86-NEXT: retl
;
; X86-NOOPT-LABEL: test_mul_by_2:
; X86-NOOPT: # %bb.0:
+; X86-NOOPT-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NOOPT-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NOOPT-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NOOPT-NEXT: leal (%edx,%edx), %eax
-; X86-NOOPT-NEXT: shll %ecx
-; X86-NOOPT-NEXT: shrl $31, %edx
-; X86-NOOPT-NEXT: orl %ecx, %edx
+; X86-NOOPT-NEXT: shldl $1, %eax, %edx
+; X86-NOOPT-NEXT: addl %eax, %eax
; X86-NOOPT-NEXT: retl
;
; X64-LABEL: test_mul_by_2:
@@ -87,22 +83,18 @@ define i64 @test_mul_by_3(i64 %x) {
define i64 @test_mul_by_4(i64 %x) {
; X86-LABEL: test_mul_by_4:
; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: leal (,%edx,4), %eax
-; X86-NEXT: shll $2, %ecx
-; X86-NEXT: shrl $30, %edx
-; X86-NEXT: orl %ecx, %edx
+; X86-NEXT: shldl $2, %eax, %edx
+; X86-NEXT: shll $2, %eax
; X86-NEXT: retl
;
; X86-NOOPT-LABEL: test_mul_by_4:
; X86-NOOPT: # %bb.0:
+; X86-NOOPT-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NOOPT-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NOOPT-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NOOPT-NEXT: leal (,%edx,4), %eax
-; X86-NOOPT-NEXT: shll $2, %ecx
-; X86-NOOPT-NEXT: shrl $30, %edx
-; X86-NOOPT-NEXT: orl %ecx, %edx
+; X86-NOOPT-NEXT: shldl $2, %eax, %edx
+; X86-NOOPT-NEXT: shll $2, %eax
; X86-NOOPT-NEXT: retl
;
; X64-LABEL: test_mul_by_4:
@@ -206,22 +198,18 @@ define i64 @test_mul_by_7(i64 %x) {
define i64 @test_mul_by_8(i64 %x) {
; X86-LABEL: test_mul_by_8:
; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: leal (,%edx,8), %eax
-; X86-NEXT: shll $3, %ecx
-; X86-NEXT: shrl $29, %edx
-; X86-NEXT: orl %ecx, %edx
+; X86-NEXT: shldl $3, %eax, %edx
+; X86-NEXT: shll $3, %eax
; X86-NEXT: retl
;
; X86-NOOPT-LABEL: test_mul_by_8:
; X86-NOOPT: # %bb.0:
+; X86-NOOPT-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NOOPT-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NOOPT-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NOOPT-NEXT: leal (,%edx,8), %eax
-; X86-NOOPT-NEXT: shll $3, %ecx
-; X86-NOOPT-NEXT: shrl $29, %edx
-; X86-NOOPT-NEXT: orl %ecx, %edx
+; X86-NOOPT-NEXT: shldl $3, %eax, %edx
+; X86-NOOPT-NEXT: shll $3, %eax
; X86-NOOPT-NEXT: retl
;
; X64-LABEL: test_mul_by_8:
@@ -482,24 +470,18 @@ define i64 @test_mul_by_15(i64 %x) {
define i64 @test_mul_by_16(i64 %x) {
; X86-LABEL: test_mul_by_16:
; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %edx, %eax
+; X86-NEXT: shldl $4, %eax, %edx
; X86-NEXT: shll $4, %eax
-; X86-NEXT: shll $4, %ecx
-; X86-NEXT: shrl $28, %edx
-; X86-NEXT: orl %ecx, %edx
; X86-NEXT: retl
;
; X86-NOOPT-LABEL: test_mul_by_16:
; X86-NOOPT: # %bb.0:
+; X86-NOOPT-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NOOPT-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NOOPT-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NOOPT-NEXT: movl %edx, %eax
+; X86-NOOPT-NEXT: shldl $4, %eax, %edx
; X86-NOOPT-NEXT: shll $4, %eax
-; X86-NOOPT-NEXT: shll $4, %ecx
-; X86-NOOPT-NEXT: shrl $28, %edx
-; X86-NOOPT-NEXT: orl %ecx, %edx
; X86-NOOPT-NEXT: retl
;
; X64-LABEL: test_mul_by_16:
@@ -1129,24 +1111,18 @@ define i64 @test_mul_by_31(i64 %x) {
define i64 @test_mul_by_32(i64 %x) {
; X86-LABEL: test_mul_by_32:
; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %edx, %eax
+; X86-NEXT: shldl $5, %eax, %edx
; X86-NEXT: shll $5, %eax
-; X86-NEXT: shll $5, %ecx
-; X86-NEXT: shrl $27, %edx
-; X86-NEXT: orl %ecx, %edx
; X86-NEXT: retl
;
; X86-NOOPT-LABEL: test_mul_by_32:
; X86-NOOPT: # %bb.0:
+; X86-NOOPT-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NOOPT-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NOOPT-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NOOPT-NEXT: movl %edx, %eax
+; X86-NOOPT-NEXT: shldl $5, %eax, %edx
; X86-NOOPT-NEXT: shll $5, %eax
-; X86-NOOPT-NEXT: shll $5, %ecx
-; X86-NOOPT-NEXT: shrl $27, %edx
-; X86-NOOPT-NEXT: orl %ecx, %edx
; X86-NOOPT-NEXT: retl
;
; X64-LABEL: test_mul_by_32:
diff --git a/llvm/test/CodeGen/X86/pr43820.ll b/llvm/test/CodeGen/X86/pr43820.ll
index 4c17cfd080afb..10270d7ae430f 100644
--- a/llvm/test/CodeGen/X86/pr43820.ll
+++ b/llvm/test/CodeGen/X86/pr43820.ll
@@ -10,18 +10,19 @@ define i1000 @square(i1000 %A) nounwind {
; CHECK-NEXT: pushq %r13
; CHECK-NEXT: pushq %r12
; CHECK-NEXT: pushq %rbx
-; CHECK-NEXT: movq %rsi, %rbp
-; CHECK-NEXT: movq %rdi, %r11
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rsi
+; CHECK-NEXT: movq %rcx, %r11
+; CHECK-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT: movq %rdi, %rsi
; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rdi
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r14
; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r15
; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rbx
; CHECK-NEXT: bswapq %rbx
; CHECK-NEXT: movq %rbx, %r10
; CHECK-NEXT: shrq $4, %r10
-; CHECK-NEXT: movabsq $1085102592571150095, %r14 # imm = 0xF0F0F0F0F0F0F0F
-; CHECK-NEXT: andq %r14, %r10
-; CHECK-NEXT: andq %r14, %rbx
+; CHECK-NEXT: movabsq $1085102592571150095, %rbp # imm = 0xF0F0F0F0F0F0F0F
+; CHECK-NEXT: andq %rbp, %r10
+; CHECK-NEXT: andq %rbp, %rbx
; CHECK-NEXT: shlq $4, %rbx
; CHECK-NEXT: orq %r10, %rbx
; CHECK-NEXT: movabsq $3689348814741910323, %r10 # imm = 0x3333333333333333
@@ -35,12 +36,12 @@ define i1000 @square(i1000 %A) nounwind {
; CHECK-NEXT: andq %rbx, %r13
; CHECK-NEXT: shrq %r12
; CHECK-NEXT: andq %rbx, %r12
-; CHECK-NEXT: leaq (%r12,%r13,2), %rax
+; CHECK-NEXT: leaq (%r12,%r13,2), %rcx
; CHECK-NEXT: bswapq %r15
; CHECK-NEXT: movq %r15, %r12
; CHECK-NEXT: shrq $4, %r12
-; CHECK-NEXT: andq %r14, %r12
-; CHECK-NEXT: andq %r14, %r15
+; CHECK-NEXT: andq %rbp, %r12
+; CHECK-NEXT: andq %rbp, %r15
; CHECK-NEXT: shlq $4, %r15
; CHECK-NEXT: orq %r12, %r15
; CHECK-NEXT: movq %r15, %r12
@@ -54,14 +55,13 @@ define i1000 @square(i1000 %A) nounwind {
; CHECK-NEXT: shrq %r15
; CHECK-NEXT: andq %r12, %r15
; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r12
-; CHECK-NEXT: leaq (%r15,%r13,2), %r15
-; CHECK-NEXT: shrdq $24, %rax, %r15
-; CHECK-NEXT: movq %r15, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT: leaq (%r15,%r13,2), %rax
+; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-NEXT: bswapq %r12
; CHECK-NEXT: movq %r12, %r15
; CHECK-NEXT: shrq $4, %r15
-; CHECK-NEXT: andq %r14, %r15
-; CHECK-NEXT: andq %r14, %r12
+; CHECK-NEXT: andq %rbp, %r15
+; CHECK-NEXT: andq %rbp, %r12
; CHECK-NEXT: shlq $4, %r12
; CHECK-NEXT: orq %r15, %r12
; CHECK-NEXT: movq %r12, %r15
@@ -73,162 +73,142 @@ define i1000 @square(i1000 %A) nounwind {
; CHECK-NEXT: andq %rbx, %r12
; CHECK-NEXT: shrq %r15
; CHECK-NEXT: andq %rbx, %r15
-; CHECK-NEXT: leaq (%r15,%r12,2), %r12
-; CHECK-NEXT: movq %r12, %r15
-; CHECK-NEXT: shlq $40, %r15
-; CHECK-NEXT: shrq $24, %rax
-; CHECK-NEXT: orq %r15, %rax
+; CHECK-NEXT: leaq (%r15,%r12,2), %rax
; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: bswapq %rdi
-; CHECK-NEXT: movq %rdi, %r15
+; CHECK-NEXT: bswapq %r14
+; CHECK-NEXT: movq %r14, %r15
; CHECK-NEXT: shrq $4, %r15
-; CHECK-NEXT: andq %r14, %r15
-; CHECK-NEXT: andq %r14, %rdi
-; CHECK-NEXT: shlq $4, %rdi
-; CHECK-NEXT: orq %r15, %rdi
-; CHECK-NEXT: movq %rdi, %r15
+; CHECK-NEXT: andq %rbp, %r15
+; CHECK-NEXT: andq %rbp, %r14
+; CHECK-NEXT: shlq $4, %r14
+; CHECK-NEXT: orq %r15, %r14
+; CHECK-NEXT: movq %r14, %r15
; CHECK-NEXT: andq %r10, %r15
+; CHECK-NEXT: shrq $2, %r14
+; CHECK-NEXT: andq %r10, %r14
+; CHECK-NEXT: leaq (%r14,%r15,4), %r14
+; CHECK-NEXT: movq %r14, %r15
+; CHECK-NEXT: andq %rbx, %r15
+; CHECK-NEXT: shrq %r14
+; CHECK-NEXT: andq %rbx, %r14
+; CHECK-NEXT: leaq (%r14,%r15,2), %rax
+; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT: bswapq %rdi
+; CHECK-NEXT: movq %rdi, %r14
+; CHECK-NEXT: shrq $4, %r14
+; CHECK-NEXT: andq %rbp, %r14
+; CHECK-NEXT: andq %rbp, %rdi
+; CHECK-NEXT: shlq $4, %rdi
+; CHECK-NEXT: orq %r14, %rdi
+; CHECK-NEXT: movq %rdi, %r14
+; CHECK-NEXT: andq %r10, %r14
; CHECK-NEXT: shrq $2, %rdi
; CHECK-NEXT: andq %r10, %rdi
-; CHECK-NEXT: leaq (%rdi,%r15,4), %rdi
-; CHECK-NEXT: movq %rdi, %r15
-; CHECK-NEXT: andq %rbx, %r15
+; CHECK-NEXT: leaq (%rdi,%r14,4), %rdi
+; CHECK-NEXT: movq %rdi, %r14
+; CHECK-NEXT: andq %rbx, %r14
; CHECK-NEXT: shrq %rdi
; CHECK-NEXT: andq %rbx, %rdi
-; CHECK-NEXT: leaq (%rdi,%r15,2), %rax
-; CHECK-NEXT: shrdq $24, %rax, %r12
-; CHECK-NEXT: movq %r12, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: bswapq %rsi
-; CHECK-NEXT: movq %rsi, %rdi
-; CHECK-NEXT: shrq $4, %rdi
-; CHECK-NEXT: andq %r14, %rdi
-; CHECK-NEXT: andq %r14, %rsi
-; CHECK-NEXT: shlq $4, %rsi
-; CHECK-NEXT: orq %rdi, %rsi
-; CHECK-NEXT: movq %rsi, %rdi
-; CHECK-NEXT: andq %r10, %rdi
-; CHECK-NEXT: shrq $2, %rsi
-; CHECK-NEXT: andq %r10, %rsi
-; CHECK-NEXT: leaq (%rsi,%rdi,4), %rsi
-; CHECK-NEXT: movq %rsi, %rdi
-; CHECK-NEXT: andq %rbx, %rdi
-; CHECK-NEXT: shrq %rsi
-; CHECK-NEXT: andq %rbx, %rsi
-; CHECK-NEXT: leaq (%rsi,%rdi,2), %rdi
-; CHECK-NEXT: movq %rdi, %rsi
-; CHECK-NEXT: movq %rdi, %r15
-; CHECK-NEXT: shlq $40, %rsi
-; CHECK-NEXT: shrq $24, %rax
-; CHECK-NEXT: orq %rsi, %rax
+; CHECK-NEXT: leaq (%rdi,%r14,2), %rax
; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rsi
-; CHECK-NEXT: bswapq %rsi
-; CHECK-NEXT: movq %rsi, %rdi
-; CHECK-NEXT: shrq $4, %rdi
-; CHECK-NEXT: andq %r14, %rdi
-; CHECK-NEXT: andq %r14, %rsi
-; CHECK-NEXT: shlq $4, %rsi
-; CHECK-NEXT: orq %rdi, %rsi
-; CHECK-NEXT: movq %rsi, %rdi
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rdi
+; CHECK-NEXT: bswapq %rdi
+; CHECK-NEXT: movq %rdi, %r14
+; CHECK-NEXT: shrq $4, %r14
+; CHECK-NEXT: andq %rbp, %r14
+; CHECK-NEXT: andq %rbp, %rdi
+; CHECK-NEXT: shlq $4, %rdi
+; CHECK-NEXT: orq %r14, %rdi
+; CHECK-NEXT: movq %rdi, %r14
+; CHECK-NEXT: andq %r10, %r14
+; CHECK-NEXT: shrq $2, %rdi
; CHECK-NEXT: andq %r10, %rdi
-; CHECK-NEXT: shrq $2, %rsi
-; CHECK-NEXT: andq %r10, %rsi
-; CHECK-NEXT: leaq (%rsi,%rdi,4), %rsi
-; CHECK-NEXT: movq %rsi, %rdi
+; CHECK-NEXT: leaq (%rdi,%r14,4), %rdi
+; CHECK-NEXT: movq %rdi, %r14
+; CHECK-NEXT: andq %rbx, %r14
+; CHECK-NEXT: shrq %rdi
; CHECK-NEXT: andq %rbx, %rdi
-; CHECK-NEXT: shrq %rsi
-; CHECK-NEXT: andq %rbx, %rsi
-; CHECK-NEXT: leaq (%rsi,%rdi,2), %r12
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rsi
-; CHECK-NEXT: bswapq %rsi
-; CHECK-NEXT: movq %rsi, %rdi
-; CHECK-NEXT: shrq $4, %rdi
-; CHECK-NEXT: andq %r14, %rdi
-; CHECK-NEXT: andq %r14, %rsi
-; CHECK-NEXT: shlq $4, %rsi
-; CHECK-NEXT: orq %rdi, %rsi
-; CHECK-NEXT: movq %rsi, %rdi
+; CHECK-NEXT: leaq (%rdi,%r14,2), %rax
+; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rdi
+; CHECK-NEXT: bswapq %rdi
+; CHECK-NEXT: movq %rdi, %r14
+; CHECK-NEXT: shrq $4, %r14
+; CHECK-NEXT: andq %rbp, %r14
+; CHECK-NEXT: andq %rbp, %rdi
+; CHECK-NEXT: shlq $4, %rdi
+; CHECK-NEXT: orq %r14, %rdi
+; CHECK-NEXT: movq %rdi, %r14
+; CHECK-NEXT: andq %r10, %r14
+; CHECK-NEXT: shrq $2, %rdi
; CHECK-NEXT: andq %r10, %rdi
-; CHECK-NEXT: shrq $2, %rsi
-; CHECK-NEXT: andq %r10, %rsi
-; CHECK-NEXT: leaq (%rsi,%rdi,4), %rsi
-; CHECK-NEXT: movq %rsi, %rdi
+; CHECK-NEXT: leaq (%rdi,%r14,4), %rdi
+; CHECK-NEXT: movq %rdi, %r14
+; CHECK-NEXT: andq %rbx, %r14
+; CHECK-NEXT: shrq %rdi
; CHECK-NEXT: andq %rbx, %rdi
-; CHECK-NEXT: shrq %rsi
-; CHECK-NEXT: andq %rbx, %rsi
-; CHECK-NEXT: leaq (%rsi,%rdi,2), %rax
-; CHECK-NEXT: shrdq $24, %r12, %r15
-; CHECK-NEXT: movq %r15, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: movq %rax, %rsi
-; CHECK-NEXT: shlq $40, %rsi
-; CHECK-NEXT: shrq $24, %r12
-; CHECK-NEXT: orq %rsi, %r12
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rsi
-; CHECK-NEXT: bswapq %rsi
-; CHECK-NEXT: movq %rsi, %rdi
-; CHECK-NEXT: shrq $4, %rdi
-; CHECK-NEXT: andq %r14, %rdi
-; CHECK-NEXT: andq %r14, %rsi
-; CHECK-NEXT: shlq $4, %rsi
-; CHECK-NEXT: orq %rdi, %rsi
-; CHECK-NEXT: movq %rsi, %rdi
+; CHECK-NEXT: leaq (%rdi,%r14,2), %rax
+; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rdi
+; CHECK-NEXT: bswapq %rdi
+; CHECK-NEXT: movq %rdi, %r14
+; CHECK-NEXT: shrq $4, %r14
+; CHECK-NEXT: andq %rbp, %r14
+; CHECK-NEXT: andq %rbp, %rdi
+; CHECK-NEXT: shlq $4, %rdi
+; CHECK-NEXT: orq %r14, %rdi
+; CHECK-NEXT: movq %rdi, %r14
+; CHECK-NEXT: andq %r10, %r14
+; CHECK-NEXT: shrq $2, %rdi
; CHECK-NEXT: andq %r10, %rdi
-; CHECK-NEXT: shrq $2, %rsi
-; CHECK-NEXT: andq %r10, %rsi
-; CHECK-NEXT: leaq (%rsi,%rdi,4), %rsi
-; CHECK-NEXT: movq %rsi, %rdi
+; CHECK-NEXT: leaq (%rdi,%r14,4), %rdi
+; CHECK-NEXT: movq %rdi, %r14
+; CHECK-NEXT: andq %rbx, %r14
+; CHECK-NEXT: shrq %rdi
; CHECK-NEXT: andq %rbx, %rdi
-; CHECK-NEXT: shrq %rsi
-; CHECK-NEXT: andq %rbx, %rsi
-; CHECK-NEXT: leaq (%rsi,%rdi,2), %r15
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rsi
-; CHECK-NEXT: bswapq %rsi
-; CHECK-NEXT: movq %rsi, %rdi
-; CHECK-NEXT: shrq $4, %rdi
-; CHECK-NEXT: andq %r14, %rdi
-; CHECK-NEXT: andq %r14, %rsi
-; CHECK-NEXT: shlq $4, %rsi
-; CHECK-NEXT: orq %rdi, %rsi
-; CHECK-NEXT: movq %rsi, %rdi
+; CHECK-NEXT: leaq (%rdi,%r14,2), %r13
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rdi
+; CHECK-NEXT: bswapq %rdi
+; CHECK-NEXT: movq %rdi, %r14
+; CHECK-NEXT: shrq $4, %r14
+; CHECK-NEXT: andq %rbp, %r14
+; CHECK-NEXT: andq %rbp, %rdi
+; CHECK-NEXT: shlq $4, %rdi
+; CHECK-NEXT: orq %r14, %rdi
+; CHECK-NEXT: movq %rdi, %r14
+; CHECK-NEXT: andq %r10, %r14
+; CHECK-NEXT: shrq $2, %rdi
; CHECK-NEXT: andq %r10, %rdi
-; CHECK-NEXT: shrq $2, %rsi
-; CHECK-NEXT: andq %r10, %rsi
-; CHECK-NEXT: leaq (%rsi,%rdi,4), %rsi
-; CHECK-NEXT: movq %rsi, %rdi
+; CHECK-NEXT: leaq (%rdi,%r14,4), %rdi
+; CHECK-NEXT: movq %rdi, %r14
+; CHECK-NEXT: andq %rbx, %r14
+; CHECK-NEXT: shrq %rdi
; CHECK-NEXT: andq %rbx, %rdi
-; CHECK-NEXT: shrq %rsi
-; CHECK-NEXT: andq %rbx, %rsi
-; CHECK-NEXT: leaq (%rsi,%rdi,2), %r13
-; CHECK-NEXT: shrdq $24, %r15, %rax
-; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: movq %r13, %rsi
-; CHECK-NEXT: shlq $40, %rsi
-; CHECK-NEXT: shrq $24, %r15
-; CHECK-NEXT: orq %rsi, %r15
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rsi
-; CHECK-NEXT: bswapq %rsi
-; CHECK-NEXT: movq %rsi, %rdi
-; CHECK-NEXT: shrq $4, %rdi
-; CHECK-NEXT: andq %r14, %rdi
-; CHECK-NEXT: andq %r14, %rsi
-; CHECK-NEXT: shlq $4, %rsi
-; CHECK-NEXT: orq %rdi, %rsi
-; CHECK-NEXT: movq %rsi, %rdi
+; CHECK-NEXT: leaq (%rdi,%r14,2), %r12
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rdi
+; CHECK-NEXT: bswapq %rdi
+; CHECK-NEXT: movq %rdi, %r14
+; CHECK-NEXT: shrq $4, %r14
+; CHECK-NEXT: andq %rbp, %r14
+; CHECK-NEXT: andq %rbp, %rdi
+; CHECK-NEXT: shlq $4, %rdi
+; CHECK-NEXT: orq %r14, %rdi
+; CHECK-NEXT: movq %rdi, %r14
+; CHECK-NEXT: andq %r10, %r14
+; CHECK-NEXT: shrq $2, %rdi
; CHECK-NEXT: andq %r10, %rdi
-; CHECK-NEXT: shrq $2, %rsi
-; CHECK-NEXT: andq %r10, %rsi
-; CHECK-NEXT: leaq (%rsi,%rdi,4), %rsi
-; CHECK-NEXT: movq %rsi, %rdi
+; CHECK-NEXT: leaq (%rdi,%r14,4), %rdi
+; CHECK-NEXT: movq %rdi, %r14
+; CHECK-NEXT: andq %rbx, %r14
+; CHECK-NEXT: shrq %rdi
; CHECK-NEXT: andq %rbx, %rdi
-; CHECK-NEXT: shrq %rsi
-; CHECK-NEXT: andq %rbx, %rsi
-; CHECK-NEXT: leaq (%rsi,%rdi,2), %rsi
+; CHECK-NEXT: leaq (%rdi,%r14,2), %r15
; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rdi
; CHECK-NEXT: bswapq %rdi
; CHECK-NEXT: movq %rdi, %rax
; CHECK-NEXT: shrq $4, %rax
-; CHECK-NEXT: andq %r14, %rax
-; CHECK-NEXT: andq %r14, %rdi
+; CHECK-NEXT: andq %rbp, %rax
+; CHECK-NEXT: andq %rbp, %rdi
; CHECK-NEXT: shlq $4, %rdi
; CHECK-NEXT: orq %rax, %rdi
; CHECK-NEXT: movq %rdi, %rax
@@ -240,17 +220,12 @@ define i1000 @square(i1000 %A) nounwind {
; CHECK-NEXT: andq %rbx, %rdi
; CHECK-NEXT: shrq %rax
; CHECK-NEXT: andq %rbx, %rax
-; CHECK-NEXT: leaq (%rax,%rdi,2), %rdi
-; CHECK-NEXT: shrdq $24, %rsi, %r13
-; CHECK-NEXT: movq %rdi, %rax
-; CHECK-NEXT: shlq $40, %rax
-; CHECK-NEXT: shrq $24, %rsi
-; CHECK-NEXT: orq %rax, %rsi
+; CHECK-NEXT: leaq (%rax,%rdi,2), %r14
; CHECK-NEXT: bswapq %r9
; CHECK-NEXT: movq %r9, %rax
; CHECK-NEXT: shrq $4, %rax
-; CHECK-NEXT: andq %r14, %rax
-; CHECK-NEXT: andq %r14, %r9
+; CHECK-NEXT: andq %rbp, %rax
+; CHECK-NEXT: andq %rbp, %r9
; CHECK-NEXT: shlq $4, %r9
; CHECK-NEXT: orq %rax, %r9
; CHECK-NEXT: movq %r9, %rax
@@ -266,8 +241,8 @@ define i1000 @square(i1000 %A) nounwind {
; CHECK-NEXT: bswapq %r8
; CHECK-NEXT: movq %r8, %rax
; CHECK-NEXT: shrq $4, %rax
-; CHECK-NEXT: andq %r14, %rax
-; CHECK-NEXT: andq %r14, %r8
+; CHECK-NEXT: andq %rbp, %rax
+; CHECK-NEXT: andq %rbp, %r8
; CHECK-NEXT: shlq $4, %r8
; CHECK-NEXT: orq %rax, %r8
; CHECK-NEXT: movq %r8, %rax
@@ -280,23 +255,46 @@ define i1000 @square(i1000 %A) nounwind {
; CHECK-NEXT: shrq %rax
; CHECK-NEXT: andq %rbx, %rax
; CHECK-NEXT: leaq (%rax,%r8,2), %r8
-; CHECK-NEXT: shrdq $24, %r9, %rdi
+; CHECK-NEXT: movq %rcx, %rax
+; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
+; CHECK-NEXT: shrdq $24, %rcx, %rdi
+; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; CHECK-NEXT: shrdq $24, %rcx, %rax
+; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; CHECK-NEXT: shrdq $24, %rax, %rcx
+; CHECK-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; CHECK-NEXT: shrdq $24, %rcx, %rax
+; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; CHECK-NEXT: shrdq $24, %rax, %rcx
+; CHECK-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; CHECK-NEXT: shrdq $24, %rcx, %rax
+; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT: shrdq $24, %r13, %rcx
+; CHECK-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT: shrdq $24, %r12, %r13
+; CHECK-NEXT: shrdq $24, %r15, %r12
+; CHECK-NEXT: shrdq $24, %r14, %r15
+; CHECK-NEXT: shrdq $24, %r9, %r14
; CHECK-NEXT: movq %r8, %rax
; CHECK-NEXT: shlq $40, %rax
; CHECK-NEXT: shrq $24, %r9
; CHECK-NEXT: orq %rax, %r9
-; CHECK-NEXT: bswapq %rcx
-; CHECK-NEXT: movq %rcx, %rax
+; CHECK-NEXT: bswapq %r11
+; CHECK-NEXT: movq %r11, %rax
; CHECK-NEXT: shrq $4, %rax
-; CHECK-NEXT: andq %r14, %rax
-; CHECK-NEXT: andq %r14, %rcx
-; CHECK-NEXT: shlq $4, %rcx
-; CHECK-NEXT: orq %rax, %rcx
-; CHECK-NEXT: movq %rcx, %rax
+; CHECK-NEXT: andq %rbp, %rax
+; CHECK-NEXT: andq %rbp, %r11
+; CHECK-NEXT: shlq $4, %r11
+; CHECK-NEXT: orq %rax, %r11
+; CHECK-NEXT: movq %r11, %rax
; CHECK-NEXT: andq %r10, %rax
-; CHECK-NEXT: shrq $2, %rcx
-; CHECK-NEXT: andq %r10, %rcx
-; CHECK-NEXT: leaq (%rcx,%rax,4), %rax
+; CHECK-NEXT: shrq $2, %r11
+; CHECK-NEXT: andq %r10, %r11
+; CHECK-NEXT: leaq (%r11,%rax,4), %rax
; CHECK-NEXT: movq %rax, %rcx
; CHECK-NEXT: andq %rbx, %rcx
; CHECK-NEXT: shrq %rax
@@ -305,8 +303,8 @@ define i1000 @square(i1000 %A) nounwind {
; CHECK-NEXT: bswapq %rdx
; CHECK-NEXT: movq %rdx, %rax
; CHECK-NEXT: shrq $4, %rax
-; CHECK-NEXT: andq %r14, %rax
-; CHECK-NEXT: andq %r14, %rdx
+; CHECK-NEXT: andq %rbp, %rax
+; CHECK-NEXT: andq %rbp, %rdx
; CHECK-NEXT: shlq $4, %rdx
; CHECK-NEXT: orq %rax, %rdx
; CHECK-NEXT: movq %rdx, %rax
@@ -324,51 +322,52 @@ define i1000 @square(i1000 %A) nounwind {
; CHECK-NEXT: shlq $40, %rax
; CHECK-NEXT: shrq $24, %rcx
; CHECK-NEXT: orq %rax, %rcx
-; CHECK-NEXT: bswapq %rbp
-; CHECK-NEXT: movq %rbp, %rax
+; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Reload
+; CHECK-NEXT: bswapq %r11
+; CHECK-NEXT: movq %r11, %rax
; CHECK-NEXT: shrq $4, %rax
-; CHECK-NEXT: andq %r14, %rax
-; CHECK-NEXT: andq %r14, %rbp
-; CHECK-NEXT: shlq $4, %rbp
-; CHECK-NEXT: orq %rax, %rbp
-; CHECK-NEXT: movq %rbp, %rax
+; CHECK-NEXT: andq %rbp, %rax
+; CHECK-NEXT: andq %rbp, %r11
+; CHECK-NEXT: shlq $4, %r11
+; CHECK-NEXT: orq %rax, %r11
+; CHECK-NEXT: movq %r11, %rax
; CHECK-NEXT: andq %r10, %rax
-; CHECK-NEXT: shrq $2, %rbp
-; CHECK-NEXT: andq %r10, %rbp
-; CHECK-NEXT: leaq (%rbp,%rax,4), %rax
+; CHECK-NEXT: shrq $2, %r11
+; CHECK-NEXT: andq %r10, %r11
+; CHECK-NEXT: leaq (%r11,%rax,4), %rax
; CHECK-NEXT: movq %rax, %r10
; CHECK-NEXT: andq %rbx, %r10
; CHECK-NEXT: shrq %rax
; CHECK-NEXT: andq %rbx, %rax
; CHECK-NEXT: leaq (%rax,%r10,2), %rax
; CHECK-NEXT: shrdq $24, %rax, %rdx
-; CHECK-NEXT: movq %rdx, 112(%r11)
-; CHECK-NEXT: movq %rcx, 104(%r11)
-; CHECK-NEXT: movq %r8, 96(%r11)
-; CHECK-NEXT: movq %r9, 88(%r11)
-; CHECK-NEXT: movq %rdi, 80(%r11)
-; CHECK-NEXT: movq %rsi, 72(%r11)
-; CHECK-NEXT: movq %r13, 64(%r11)
-; CHECK-NEXT: movq %r15, 56(%r11)
+; CHECK-NEXT: movq %rdx, 112(%rsi)
+; CHECK-NEXT: movq %rcx, 104(%rsi)
+; CHECK-NEXT: movq %r8, 96(%rsi)
+; CHECK-NEXT: movq %r9, 88(%rsi)
+; CHECK-NEXT: movq %r14, 80(%rsi)
+; CHECK-NEXT: movq %r15, 72(%rsi)
+; CHECK-NEXT: movq %r12, 64(%rsi)
+; CHECK-NEXT: movq %r13, 56(%rsi)
; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; CHECK-NEXT: movq %rcx, 48(%r11)
-; CHECK-NEXT: movq %r12, 40(%r11)
+; CHECK-NEXT: movq %rcx, 48(%rsi)
; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; CHECK-NEXT: movq %rcx, 32(%r11)
+; CHECK-NEXT: movq %rcx, 40(%rsi)
; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; CHECK-NEXT: movq %rcx, 24(%r11)
+; CHECK-NEXT: movq %rcx, 32(%rsi)
; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; CHECK-NEXT: movq %rcx, 16(%r11)
+; CHECK-NEXT: movq %rcx, 24(%rsi)
; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; CHECK-NEXT: movq %rcx, 8(%r11)
+; CHECK-NEXT: movq %rcx, 16(%rsi)
; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; CHECK-NEXT: movq %rcx, (%r11)
+; CHECK-NEXT: movq %rcx, 8(%rsi)
+; CHECK-NEXT: movq %rdi, (%rsi)
; CHECK-NEXT: movq %rax, %rcx
; CHECK-NEXT: shrq $56, %rax
-; CHECK-NEXT: movb %al, 124(%r11)
+; CHECK-NEXT: movb %al, 124(%rsi)
; CHECK-NEXT: shrq $24, %rcx
-; CHECK-NEXT: movl %ecx, 120(%r11)
-; CHECK-NEXT: movq %r11, %rax
+; CHECK-NEXT: movl %ecx, 120(%rsi)
+; CHECK-NEXT: movq %rsi, %rax
; CHECK-NEXT: popq %rbx
; CHECK-NEXT: popq %r12
; CHECK-NEXT: popq %r13
diff --git a/llvm/test/CodeGen/X86/pr49162.ll b/llvm/test/CodeGen/X86/pr49162.ll
index 8409a27a76763..db8cec61acd6b 100644
--- a/llvm/test/CodeGen/X86/pr49162.ll
+++ b/llvm/test/CodeGen/X86/pr49162.ll
@@ -6,13 +6,11 @@ define ptr @PR49162(ptr %base, ptr %ptr160) {
; X86-LABEL: PR49162:
; X86: # %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl 8(%eax), %eax
-; X86-NEXT: shll $16, %eax
-; X86-NEXT: movl %eax, %ecx
-; X86-NEXT: sarl $31, %ecx
+; X86-NEXT: movl 8(%eax), %ecx
; X86-NEXT: shll $16, %ecx
-; X86-NEXT: shrl $16, %eax
-; X86-NEXT: orl %ecx, %eax
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: sarl $31, %eax
+; X86-NEXT: shldl $16, %ecx, %eax
; X86-NEXT: shll $2, %eax
; X86-NEXT: addl {{[0-9]+}}(%esp), %eax
; X86-NEXT: retl
diff --git a/llvm/test/CodeGen/X86/rotate-add.ll b/llvm/test/CodeGen/X86/rotate-add.ll
index e0c06f022eb80..c705505bbbf2a 100644
--- a/llvm/test/CodeGen/X86/rotate-add.ll
+++ b/llvm/test/CodeGen/X86/rotate-add.ll
@@ -221,14 +221,9 @@ define i64 @test_rotl_udiv_special_case(i64 %i) {
; X86-NEXT: addl %ecx, %edx
; X86-NEXT: imull $-1431655765, %edi, %ecx # imm = 0xAAAAAAAB
; X86-NEXT: addl %ecx, %edx
-; X86-NEXT: movl %edx, %esi
-; X86-NEXT: shll $28, %esi
-; X86-NEXT: movl %eax, %ecx
-; X86-NEXT: shrl $4, %ecx
-; X86-NEXT: orl %esi, %ecx
-; X86-NEXT: shll $28, %eax
-; X86-NEXT: shrl $4, %edx
-; X86-NEXT: orl %eax, %edx
+; X86-NEXT: movl %edx, %ecx
+; X86-NEXT: shldl $28, %eax, %ecx
+; X86-NEXT: shrdl $4, %eax, %edx
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: popl %esi
; X86-NEXT: .cfi_def_cfa_offset 12
@@ -283,9 +278,7 @@ define i64 @test_rotl_mul_with_mask_special_case(i64 %i) {
; X86-NEXT: movl $9, %eax
; X86-NEXT: mull {{[0-9]+}}(%esp)
; X86-NEXT: addl %ecx, %edx
-; X86-NEXT: shll $7, %eax
-; X86-NEXT: shrl $25, %edx
-; X86-NEXT: orl %eax, %edx
+; X86-NEXT: shrdl $25, %eax, %edx
; X86-NEXT: movzbl %dl, %eax
; X86-NEXT: xorl %edx, %edx
; X86-NEXT: retl
@@ -311,12 +304,10 @@ define i64 @test_rotl_mul_with_mask_special_case(i64 %i) {
define i32 @test_fshl_with_mask_special_case(i32 %x) {
; X86-LABEL: test_fshl_with_mask_special_case:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl %eax, %ecx
-; X86-NEXT: orl $1, %ecx
-; X86-NEXT: shll $5, %ecx
-; X86-NEXT: shrl $27, %eax
-; X86-NEXT: orl %ecx, %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: orl $1, %eax
+; X86-NEXT: shldl $5, %ecx, %eax
; X86-NEXT: andl $-31, %eax
; X86-NEXT: retl
;
@@ -324,9 +315,7 @@ define i32 @test_fshl_with_mask_special_case(i32 %x) {
; X64: # %bb.0:
; X64-NEXT: movl %edi, %eax
; X64-NEXT: orl $1, %eax
-; X64-NEXT: shll $5, %eax
-; X64-NEXT: shrl $27, %edi
-; X64-NEXT: orl %edi, %eax
+; X64-NEXT: shldl $5, %edi, %eax
; X64-NEXT: andl $-31, %eax
; X64-NEXT: retq
%or1 = or i32 %x, 1
diff --git a/llvm/test/CodeGen/X86/rotate-extract.ll b/llvm/test/CodeGen/X86/rotate-extract.ll
index d70c7846e3544..2b87b639679dd 100644
--- a/llvm/test/CodeGen/X86/rotate-extract.ll
+++ b/llvm/test/CodeGen/X86/rotate-extract.ll
@@ -12,23 +12,13 @@
define i64 @rolq_extract_shl(i64 %i) nounwind {
; X86-LABEL: rolq_extract_shl:
; X86: # %bb.0:
-; X86-NEXT: pushl %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: shll $3, %eax
-; X86-NEXT: movl %edx, %ecx
-; X86-NEXT: shrl $29, %ecx
-; X86-NEXT: orl %eax, %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: shldl $3, %edx, %ecx
; X86-NEXT: shll $3, %edx
-; X86-NEXT: movl %edx, %esi
-; X86-NEXT: shll $7, %esi
-; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: shrl $25, %eax
-; X86-NEXT: orl %esi, %eax
-; X86-NEXT: shll $7, %ecx
-; X86-NEXT: shrl $25, %edx
-; X86-NEXT: orl %ecx, %edx
-; X86-NEXT: popl %esi
+; X86-NEXT: movl %edx, %eax
+; X86-NEXT: shldl $7, %ecx, %eax
+; X86-NEXT: shrdl $25, %ecx, %edx
; X86-NEXT: retl
;
; X64-LABEL: rolq_extract_shl:
@@ -120,9 +110,7 @@ define i64 @rolq_extract_mul_with_mask(i64 %i) nounwind {
; X86-NEXT: movl $9, %eax
; X86-NEXT: mull {{[0-9]+}}(%esp)
; X86-NEXT: addl %ecx, %edx
-; X86-NEXT: shll $7, %eax
-; X86-NEXT: shrl $25, %edx
-; X86-NEXT: orl %eax, %edx
+; X86-NEXT: shrdl $25, %eax, %edx
; X86-NEXT: movzbl %dl, %eax
; X86-NEXT: xorl %edx, %edx
; X86-NEXT: retl
diff --git a/llvm/test/CodeGen/X86/rotate.ll b/llvm/test/CodeGen/X86/rotate.ll
index e76821e2f3325..ea32edba62822 100644
--- a/llvm/test/CodeGen/X86/rotate.ll
+++ b/llvm/test/CodeGen/X86/rotate.ll
@@ -113,18 +113,11 @@ define i64 @rotr64(i64 %A, i8 %Amt) nounwind {
define i64 @rotli64(i64 %A) nounwind {
; X86-LABEL: rotli64:
; X86: # %bb.0:
-; X86-NEXT: pushl %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %edx, %esi
-; X86-NEXT: shll $5, %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: shrl $27, %eax
-; X86-NEXT: orl %esi, %eax
-; X86-NEXT: shll $5, %ecx
-; X86-NEXT: shrl $27, %edx
-; X86-NEXT: orl %ecx, %edx
-; X86-NEXT: popl %esi
+; X86-NEXT: shldl $5, %edx, %eax
+; X86-NEXT: shldl $5, %ecx, %edx
; X86-NEXT: retl
;
; X64-LABEL: rotli64:
@@ -141,18 +134,11 @@ define i64 @rotli64(i64 %A) nounwind {
define i64 @rotri64(i64 %A) nounwind {
; X86-LABEL: rotri64:
; X86: # %bb.0:
-; X86-NEXT: pushl %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl %edx, %esi
-; X86-NEXT: shll $27, %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: shrl $5, %eax
-; X86-NEXT: orl %esi, %eax
-; X86-NEXT: shll $27, %ecx
-; X86-NEXT: shrl $5, %edx
-; X86-NEXT: orl %ecx, %edx
-; X86-NEXT: popl %esi
+; X86-NEXT: shldl $27, %edx, %eax
+; X86-NEXT: shldl $27, %ecx, %edx
; X86-NEXT: retl
;
; X64-LABEL: rotri64:
@@ -169,17 +155,11 @@ define i64 @rotri64(i64 %A) nounwind {
define i64 @rotl1_64(i64 %A) nounwind {
; X86-LABEL: rotl1_64:
; X86: # %bb.0:
-; X86-NEXT: pushl %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: leal (,%edx,2), %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: shrl $31, %eax
-; X86-NEXT: orl %esi, %eax
-; X86-NEXT: shll %ecx
-; X86-NEXT: shrl $31, %edx
-; X86-NEXT: orl %ecx, %edx
-; X86-NEXT: popl %esi
+; X86-NEXT: shldl $1, %edx, %eax
+; X86-NEXT: shldl $1, %ecx, %edx
; X86-NEXT: retl
;
; X64-LABEL: rotl1_64:
@@ -196,18 +176,11 @@ define i64 @rotl1_64(i64 %A) nounwind {
define i64 @rotr1_64(i64 %A) nounwind {
; X86-LABEL: rotr1_64:
; X86: # %bb.0:
-; X86-NEXT: pushl %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl %edx, %esi
-; X86-NEXT: shll $31, %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: shrl %eax
-; X86-NEXT: orl %esi, %eax
-; X86-NEXT: shll $31, %ecx
-; X86-NEXT: shrl %edx
-; X86-NEXT: orl %ecx, %edx
-; X86-NEXT: popl %esi
+; X86-NEXT: shldl $31, %edx, %eax
+; X86-NEXT: shldl $31, %ecx, %edx
; X86-NEXT: retl
;
; X64-LABEL: rotr1_64:
@@ -592,23 +565,16 @@ define i8 @rotr1_8(i8 %A) nounwind {
define void @rotr1_64_mem(ptr %Aptr) nounwind {
; X86-LABEL: rotr1_64_mem:
; X86: # %bb.0:
-; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl (%eax), %ecx
; X86-NEXT: movl 4(%eax), %edx
; X86-NEXT: movl %ecx, %esi
-; X86-NEXT: shll $31, %esi
-; X86-NEXT: movl %edx, %edi
-; X86-NEXT: shrl %edi
-; X86-NEXT: orl %esi, %edi
-; X86-NEXT: shll $31, %edx
-; X86-NEXT: shrl %ecx
-; X86-NEXT: orl %edx, %ecx
-; X86-NEXT: movl %ecx, (%eax)
-; X86-NEXT: movl %edi, 4(%eax)
+; X86-NEXT: shldl $31, %edx, %esi
+; X86-NEXT: shldl $31, %ecx, %edx
+; X86-NEXT: movl %edx, (%eax)
+; X86-NEXT: movl %esi, 4(%eax)
; X86-NEXT: popl %esi
-; X86-NEXT: popl %edi
; X86-NEXT: retl
;
; X64-LABEL: rotr1_64_mem:
diff --git a/llvm/test/CodeGen/X86/rotate2.ll b/llvm/test/CodeGen/X86/rotate2.ll
index 2c58e0a9de3ab..ac299581dd2f8 100644
--- a/llvm/test/CodeGen/X86/rotate2.ll
+++ b/llvm/test/CodeGen/X86/rotate2.ll
@@ -5,18 +5,11 @@
define i64 @test1(i64 %x) nounwind {
; X86-LABEL: test1:
; X86: # %bb.0: # %entry
-; X86-NEXT: pushl %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %edx, %esi
-; X86-NEXT: shll $9, %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: shrl $23, %eax
-; X86-NEXT: orl %esi, %eax
-; X86-NEXT: shll $9, %ecx
-; X86-NEXT: shrl $23, %edx
-; X86-NEXT: orl %ecx, %edx
-; X86-NEXT: popl %esi
+; X86-NEXT: shldl $9, %edx, %eax
+; X86-NEXT: shldl $9, %ecx, %edx
; X86-NEXT: retl
;
; X64-LABEL: test1:
diff --git a/llvm/test/CodeGen/X86/scmp.ll b/llvm/test/CodeGen/X86/scmp.ll
index 2687c70d7b948..31382d017926d 100644
--- a/llvm/test/CodeGen/X86/scmp.ll
+++ b/llvm/test/CodeGen/X86/scmp.ll
@@ -3143,7 +3143,7 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; X86-NEXT: pushl %ebx
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
-; X86-NEXT: subl $52, %esp
+; X86-NEXT: subl $60, %esp
; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
; X86-NEXT: addb %al, %al
; X86-NEXT: sarb %al
@@ -3174,15 +3174,15 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; X86-NEXT: movb {{[0-9]+}}(%esp), %ch
; X86-NEXT: addb %ch, %ch
; X86-NEXT: sarb %ch
-; X86-NEXT: movb {{[0-9]+}}(%esp), %cl
-; X86-NEXT: addb %cl, %cl
-; X86-NEXT: sarb %cl
-; X86-NEXT: movb {{[0-9]+}}(%esp), %ah
-; X86-NEXT: addb %ah, %ah
-; X86-NEXT: sarb %ah
; X86-NEXT: movzbl {{[0-9]+}}(%esp), %edx
; X86-NEXT: addb %dl, %dl
; X86-NEXT: sarb %dl
+; X86-NEXT: movb {{[0-9]+}}(%esp), %ah
+; X86-NEXT: addb %ah, %ah
+; X86-NEXT: sarb %ah
+; X86-NEXT: movb {{[0-9]+}}(%esp), %cl
+; X86-NEXT: addb %cl, %cl
+; X86-NEXT: sarb %cl
; X86-NEXT: movb {{[0-9]+}}(%esp), %dh
; X86-NEXT: addb %dh, %dh
; X86-NEXT: sarb %dh
@@ -3202,18 +3202,17 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: andl $2097151, %esi # imm = 0x1FFFFF
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: cmpb %dl, %dh
+; X86-NEXT: cmpb %cl, %dh
; X86-NEXT: setl %al
-; X86-NEXT: setg %dl
-; X86-NEXT: subb %al, %dl
-; X86-NEXT: movsbl %dl, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: sarl $31, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %edx, %esi
+; X86-NEXT: setg %cl
+; X86-NEXT: subb %al, %cl
+; X86-NEXT: movsbl %cl, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sarl $31, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: andl $2097151, %esi # imm = 0x1FFFFF
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: cmpb %cl, %ah
+; X86-NEXT: cmpb %dl, %ah
; X86-NEXT: setl %al
; X86-NEXT: setg %cl
; X86-NEXT: subb %al, %cl
@@ -3221,9 +3220,9 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-NEXT: movl %eax, (%edx)
; X86-NEXT: sarl $31, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: andl $2097151, %eax # imm = 0x1FFFFF
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %eax, %esi
+; X86-NEXT: andl $2097151, %esi # imm = 0x1FFFFF
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: cmpb %bl, %ch
; X86-NEXT: setl %cl
; X86-NEXT: setg %ch
@@ -3231,20 +3230,21 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; X86-NEXT: movsbl %ch, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: sarl $31, %ecx
-; X86-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
-; X86-NEXT: cmpb {{[-0-9]+}}(%e{{[sb]}}p), %al # 1-byte Folded Reload
-; X86-NEXT: setl %bl
-; X86-NEXT: setg %bh
-; X86-NEXT: subb %bl, %bh
-; X86-NEXT: movsbl %bh, %edi
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X86-NEXT: cmpb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Folded Reload
+; X86-NEXT: setl %cl
+; X86-NEXT: setg %ch
+; X86-NEXT: subb %cl, %ch
+; X86-NEXT: movsbl %ch, %edi
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: sarl $31, %edi
-; X86-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 1-byte Folded Reload
-; X86-NEXT: cmpb {{[-0-9]+}}(%e{{[sb]}}p), %bl # 1-byte Folded Reload
-; X86-NEXT: setl %bl
-; X86-NEXT: setg %bh
-; X86-NEXT: subb %bl, %bh
-; X86-NEXT: movsbl %bh, %ebp
+; X86-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X86-NEXT: cmpb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Folded Reload
+; X86-NEXT: setl %cl
+; X86-NEXT: setg %ch
+; X86-NEXT: subb %cl, %ch
+; X86-NEXT: movsbl %ch, %ebp
; X86-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: sarl $31, %ebp
; X86-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 1-byte Folded Reload
@@ -3254,95 +3254,93 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; X86-NEXT: subb %bl, %bh
; X86-NEXT: movsbl %bh, %ebx
; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: sarl $31, %ebx
-; X86-NEXT: movl %edx, %eax
-; X86-NEXT: movl %ebx, 96(%edx)
-; X86-NEXT: movl %ebx, 92(%edx)
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: movl %esi, 80(%edx)
+; X86-NEXT: movl %ebx, %esi
+; X86-NEXT: sarl $31, %esi
+; X86-NEXT: movl %esi, 96(%edx)
+; X86-NEXT: movl %esi, 92(%edx)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %ecx, 80(%edx)
; X86-NEXT: movl %ebp, 68(%edx)
; X86-NEXT: movl %ebp, 64(%edx)
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %edi, 52(%edx)
; X86-NEXT: movl %edi, 48(%edx)
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: movl %edx, 36(%eax)
-; X86-NEXT: movl %ecx, %edx
-; X86-NEXT: movl %ecx, 24(%eax)
-; X86-NEXT: movl %ecx, 20(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: movl %edi, 36(%edx)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl %ecx, 8(%eax)
-; X86-NEXT: movl %ecx, 4(%eax)
-; X86-NEXT: movw %bx, 100(%eax)
-; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: movl %ecx, 24(%edx)
+; X86-NEXT: movl %ecx, 20(%edx)
+; X86-NEXT: movl %eax, 8(%edx)
+; X86-NEXT: movl %eax, 4(%edx)
+; X86-NEXT: movl %esi, %ecx
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movw %cx, 100(%edx)
+; X86-NEXT: shrdl $2, %esi, %ebx
+; X86-NEXT: movl %ebx, 88(%edx)
+; X86-NEXT: movl %edx, %esi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shrdl $2, %ebx, %eax
-; X86-NEXT: movl %eax, 88(%ecx)
-; X86-NEXT: movl %esi, %eax
+; X86-NEXT: shldl $9, %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: shll $9, %eax
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: shrl $23, %esi
-; X86-NEXT: orl %eax, %esi
-; X86-NEXT: movl %esi, 76(%ecx)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: shrl $23, %edi
+; X86-NEXT: orl %eax, %edi
+; X86-NEXT: movl %edi, 76(%edx)
; X86-NEXT: movl %ebp, %eax
; X86-NEXT: shll $20, %eax
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: shrl $12, %esi
-; X86-NEXT: orl %eax, %esi
-; X86-NEXT: movl %esi, 60(%ecx)
-; X86-NEXT: movl %edi, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: shrl $12, %edi
+; X86-NEXT: orl %eax, %edi
+; X86-NEXT: movl %edi, 60(%edx)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: movl %ebx, %eax
; X86-NEXT: shll $31, %eax
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: shrl %esi
-; X86-NEXT: orl %eax, %esi
-; X86-NEXT: movl %esi, 44(%ecx)
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: movl %esi, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: shrl %edi
+; X86-NEXT: orl %eax, %edi
+; X86-NEXT: movl %edi, 44(%edx)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: movl %edx, %eax
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: shrdl $22, %ecx, %eax
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %eax, 32(%ecx)
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shrdl $11, %edx, %eax
-; X86-NEXT: movl %eax, 16(%ecx)
+; X86-NEXT: movl %eax, 32(%esi)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: movl %edi, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: shrdl $11, %ecx, %eax
+; X86-NEXT: movl %eax, 16(%esi)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: shll $9, %eax
; X86-NEXT: andl $511, %ebp # imm = 0x1FF
; X86-NEXT: orl %eax, %ebp
-; X86-NEXT: movl %ebp, 72(%ecx)
+; X86-NEXT: movl %ebp, 72(%esi)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: shll $20, %eax
-; X86-NEXT: andl $1048575, %edi # imm = 0xFFFFF
-; X86-NEXT: orl %eax, %edi
-; X86-NEXT: movl %edi, 56(%ecx)
-; X86-NEXT: shll $10, %esi
-; X86-NEXT: andl $1023, %edx # imm = 0x3FF
-; X86-NEXT: orl %esi, %edx
-; X86-NEXT: movl %edx, 28(%ecx)
+; X86-NEXT: andl $1048575, %ebx # imm = 0xFFFFF
+; X86-NEXT: orl %eax, %ebx
+; X86-NEXT: movl %ebx, 56(%esi)
+; X86-NEXT: shll $10, %edx
+; X86-NEXT: andl $1023, %ecx # imm = 0x3FF
+; X86-NEXT: orl %edx, %ecx
+; X86-NEXT: movl %ecx, 28(%esi)
+; X86-NEXT: shll $21, %edi
+; X86-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X86-NEXT: movl %edi, 12(%esi)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shll $21, %eax
-; X86-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NEXT: movl %eax, 12(%ecx)
-; X86-NEXT: andl $7, %ebx
-; X86-NEXT: movb %bl, 102(%ecx)
-; X86-NEXT: movl %ecx, %edx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: shll $9, %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shrl $23, %eax
-; X86-NEXT: orl %ecx, %eax
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: shll $30, %ecx
-; X86-NEXT: orl %eax, %ecx
-; X86-NEXT: movl %ecx, 84(%edx)
+; X86-NEXT: andl $7, %eax
+; X86-NEXT: movb %al, 102(%esi)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: shll $30, %eax
+; X86-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NEXT: movl %eax, 84(%esi)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: shldl $10, %ecx, %eax
+; X86-NEXT: shldl $10, %ecx, %edx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: shll $31, %ecx
-; X86-NEXT: orl %eax, %ecx
-; X86-NEXT: movl %ecx, 40(%edx)
-; X86-NEXT: movl %edx, %eax
-; X86-NEXT: addl $52, %esp
+; X86-NEXT: orl %edx, %ecx
+; X86-NEXT: movl %ecx, 40(%esi)
+; X86-NEXT: movl %esi, %eax
+; X86-NEXT: addl $60, %esp
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
; X86-NEXT: popl %ebx
diff --git a/llvm/test/CodeGen/X86/sdiv_fix.ll b/llvm/test/CodeGen/X86/sdiv_fix.ll
index f3e441fc5b602..2511ab46d24b6 100644
--- a/llvm/test/CodeGen/X86/sdiv_fix.ll
+++ b/llvm/test/CodeGen/X86/sdiv_fix.ll
@@ -262,12 +262,9 @@ define i64 @func5(i64 %x, i64 %y) nounwind {
; X64-NEXT: pushq %rax
; X64-NEXT: movq %rsi, %rbx
; X64-NEXT: movq %rdi, %r14
-; X64-NEXT: movq %rdi, %rax
-; X64-NEXT: sarq $63, %rax
-; X64-NEXT: shlq $31, %rax
; X64-NEXT: movq %rdi, %r15
-; X64-NEXT: shrq $33, %r15
-; X64-NEXT: orq %rax, %r15
+; X64-NEXT: sarq $63, %r15
+; X64-NEXT: shldq $31, %rdi, %r15
; X64-NEXT: shlq $31, %r14
; X64-NEXT: movq %rsi, %r12
; X64-NEXT: sarq $63, %r12
diff --git a/llvm/test/CodeGen/X86/sdiv_fix_sat.ll b/llvm/test/CodeGen/X86/sdiv_fix_sat.ll
index fe8364966cfd4..d503b8f1d4e38 100644
--- a/llvm/test/CodeGen/X86/sdiv_fix_sat.ll
+++ b/llvm/test/CodeGen/X86/sdiv_fix_sat.ll
@@ -307,20 +307,18 @@ define i64 @func5(i64 %x, i64 %y) nounwind {
; X64-NEXT: pushq %r12
; X64-NEXT: pushq %rbx
; X64-NEXT: subq $24, %rsp
-; X64-NEXT: movq %rdi, %r15
-; X64-NEXT: leaq (%rdi,%rdi), %r14
-; X64-NEXT: movq %rdi, %rax
-; X64-NEXT: sarq $63, %rax
-; X64-NEXT: shlq $31, %rax
-; X64-NEXT: shrq $33, %r14
-; X64-NEXT: orq %rax, %r14
-; X64-NEXT: shlq $32, %r15
; X64-NEXT: movq %rsi, %rdx
; X64-NEXT: movq %rsi, (%rsp) # 8-byte Spill
+; X64-NEXT: movq %rdi, %r14
+; X64-NEXT: leaq (%rdi,%rdi), %rax
+; X64-NEXT: movq %rdi, %r15
+; X64-NEXT: sarq $63, %r15
+; X64-NEXT: shldq $31, %rax, %r15
+; X64-NEXT: shlq $32, %r14
; X64-NEXT: movq %rsi, %r12
; X64-NEXT: sarq $63, %r12
-; X64-NEXT: movq %r15, %rdi
-; X64-NEXT: movq %r14, %rsi
+; X64-NEXT: movq %r14, %rdi
+; X64-NEXT: movq %r15, %rsi
; X64-NEXT: movq %r12, %rcx
; X64-NEXT: callq __divti3 at PLT
; X64-NEXT: movq %rax, %r13
@@ -329,13 +327,13 @@ define i64 @func5(i64 %x, i64 %y) nounwind {
; X64-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; X64-NEXT: subq $1, %r13
; X64-NEXT: sbbq $0, %rbp
-; X64-NEXT: testq %r14, %r14
+; X64-NEXT: testq %r15, %r15
; X64-NEXT: sets %al
; X64-NEXT: testq %r12, %r12
; X64-NEXT: sets %bl
; X64-NEXT: xorb %al, %bl
-; X64-NEXT: movq %r15, %rdi
-; X64-NEXT: movq %r14, %rsi
+; X64-NEXT: movq %r14, %rdi
+; X64-NEXT: movq %r15, %rsi
; X64-NEXT: movq (%rsp), %rdx # 8-byte Reload
; X64-NEXT: movq %r12, %rcx
; X64-NEXT: callq __modti3 at PLT
@@ -581,12 +579,9 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
; X64-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
; X64-NEXT: movdqa %xmm3, (%rsp) # 16-byte Spill
; X64-NEXT: movq %xmm3, %r15
-; X64-NEXT: movq %r15, %rax
-; X64-NEXT: sarq $63, %rax
-; X64-NEXT: shlq $31, %rax
; X64-NEXT: movq %r15, %rbx
-; X64-NEXT: shrq $33, %rbx
-; X64-NEXT: orq %rax, %rbx
+; X64-NEXT: sarq $63, %rbx
+; X64-NEXT: shldq $31, %r15, %rbx
; X64-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
; X64-NEXT: pxor %xmm0, %xmm0
; X64-NEXT: pcmpgtd %xmm1, %xmm0
diff --git a/llvm/test/CodeGen/X86/setcc-fsh.ll b/llvm/test/CodeGen/X86/setcc-fsh.ll
index f12ed9152671e..7ab63959f58b0 100644
--- a/llvm/test/CodeGen/X86/setcc-fsh.ll
+++ b/llvm/test/CodeGen/X86/setcc-fsh.ll
@@ -484,9 +484,7 @@ define i1 @fshl_xor_eq_0(i32 %x, i32 %y) {
; CHECK-LABEL: fshl_xor_eq_0:
; CHECK: # %bb.0:
; CHECK-NEXT: xorl %edi, %esi
-; CHECK-NEXT: shll $2, %esi
-; CHECK-NEXT: shrl $30, %edi
-; CHECK-NEXT: orl %esi, %edi
+; CHECK-NEXT: shldl $2, %edi, %esi
; CHECK-NEXT: sete %al
; CHECK-NEXT: retq
%or = xor i32 %x, %y
@@ -499,9 +497,8 @@ define i1 @fshl_or_sgt_0(i32 %x, i32 %y) {
; CHECK-LABEL: fshl_or_sgt_0:
; CHECK: # %bb.0:
; CHECK-NEXT: orl %edi, %esi
-; CHECK-NEXT: shll $2, %esi
-; CHECK-NEXT: shrl $30, %edi
-; CHECK-NEXT: orl %esi, %edi
+; CHECK-NEXT: shldl $2, %edi, %esi
+; CHECK-NEXT: testl %esi, %esi
; CHECK-NEXT: setg %al
; CHECK-NEXT: retq
%or = or i32 %x, %y
@@ -514,10 +511,8 @@ define i1 @fshl_or_ne_2(i32 %x, i32 %y) {
; CHECK-LABEL: fshl_or_ne_2:
; CHECK: # %bb.0:
; CHECK-NEXT: orl %edi, %esi
-; CHECK-NEXT: shll $2, %esi
-; CHECK-NEXT: shrl $30, %edi
-; CHECK-NEXT: orl %esi, %edi
-; CHECK-NEXT: cmpl $2, %edi
+; CHECK-NEXT: shldl $2, %edi, %esi
+; CHECK-NEXT: cmpl $2, %esi
; CHECK-NEXT: setne %al
; CHECK-NEXT: retq
%or = or i32 %x, %y
diff --git a/llvm/test/CodeGen/X86/shift-i256.ll b/llvm/test/CodeGen/X86/shift-i256.ll
index c541de6865eb1..8a160aef154bb 100644
--- a/llvm/test/CodeGen/X86/shift-i256.ll
+++ b/llvm/test/CodeGen/X86/shift-i256.ll
@@ -1477,66 +1477,31 @@ define i256 @shl_i256_1(i256 %a0) nounwind {
;
; X86-LABEL: shl_i256_1:
; X86: # %bb.0:
-; X86-NEXT: pushl %ebp
-; X86-NEXT: pushl %ebx
-; X86-NEXT: pushl %edi
-; X86-NEXT: pushl %esi
-; X86-NEXT: subl $12, %esp
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X86-NEXT: leal (,%esi,2), %eax
-; X86-NEXT: movl %edx, %ecx
-; X86-NEXT: shrl $31, %ecx
-; X86-NEXT: orl %eax, %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: leal (,%edi,2), %eax
-; X86-NEXT: movl %ebx, %ecx
-; X86-NEXT: shrl $31, %ecx
-; X86-NEXT: orl %eax, %ecx
-; X86-NEXT: movl %ecx, (%esp) # 4-byte Spill
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl %eax, %ebx
-; X86-NEXT: shrl $31, %ebx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X86-NEXT: leal (,%ebp,2), %ecx
-; X86-NEXT: orl %ecx, %ebx
-; X86-NEXT: shll %edx
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: leal (%ecx,%ecx), %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shrl $31, %ecx
-; X86-NEXT: orl %edx, %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: shll %edx
-; X86-NEXT: shrl $31, %esi
-; X86-NEXT: orl %edx, %esi
-; X86-NEXT: shll %eax
+; X86-NEXT: shldl $1, %ecx, %edx
+; X86-NEXT: movl %edx, 28(%eax)
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: shrl $31, %edx
-; X86-NEXT: orl %eax, %edx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: shll %eax
-; X86-NEXT: shrl $31, %ebp
-; X86-NEXT: orl %eax, %ebp
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl %ebp, 28(%eax)
-; X86-NEXT: movl %ebx, 24(%eax)
+; X86-NEXT: shldl $1, %edx, %ecx
+; X86-NEXT: movl %ecx, 24(%eax)
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: shldl $1, %ecx, %edx
; X86-NEXT: movl %edx, 20(%eax)
-; X86-NEXT: movl (%esp), %edx # 4-byte Reload
-; X86-NEXT: movl %edx, 16(%eax)
-; X86-NEXT: movl %esi, 12(%eax)
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: movl %edx, 8(%eax)
-; X86-NEXT: movl %ecx, 4(%eax)
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: shldl $1, %edx, %ecx
+; X86-NEXT: movl %ecx, 16(%eax)
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: shldl $1, %ecx, %edx
+; X86-NEXT: movl %edx, 12(%eax)
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: shldl $1, %edx, %ecx
+; X86-NEXT: movl %ecx, 8(%eax)
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: shldl $1, %ecx, %edx
+; X86-NEXT: movl %edx, 4(%eax)
+; X86-NEXT: addl %ecx, %ecx
; X86-NEXT: movl %ecx, (%eax)
-; X86-NEXT: addl $12, %esp
-; X86-NEXT: popl %esi
-; X86-NEXT: popl %edi
-; X86-NEXT: popl %ebx
-; X86-NEXT: popl %ebp
; X86-NEXT: retl $4
%r = shl i256 %a0, 1
ret i256 %r
@@ -1566,45 +1531,39 @@ define i256 @lshr_i256_1(i256 %a0) nounwind {
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
; X86-NEXT: subl $8, %esp
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X86-NEXT: movl %ebx, %eax
-; X86-NEXT: shll $31, %eax
-; X86-NEXT: shrdl $1, %esi, %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shrl %esi
-; X86-NEXT: orl %eax, %esi
-; X86-NEXT: movl %edx, %eax
-; X86-NEXT: shll $31, %eax
-; X86-NEXT: shrdl $1, %ebp, %ebx
-; X86-NEXT: shrl %ebp
-; X86-NEXT: orl %eax, %ebp
-; X86-NEXT: movl %edi, %ecx
-; X86-NEXT: shll $31, %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT: movl %edi, %esi
+; X86-NEXT: shldl $31, %eax, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shrdl $1, %eax, %ecx
+; X86-NEXT: movl %ecx, (%esp) # 4-byte Spill
+; X86-NEXT: movl %edx, %ebp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: shldl $31, %eax, %ebp
+; X86-NEXT: shrdl $1, %eax, %edi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %ebx, %esi
+; X86-NEXT: shldl $31, %eax, %esi
; X86-NEXT: shrdl $1, %eax, %edx
-; X86-NEXT: movl %edx, (%esp) # 4-byte Spill
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: shrdl $1, %eax, %ebx
; X86-NEXT: shrl %eax
-; X86-NEXT: orl %ecx, %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: shrdl $1, %ecx, %edi
-; X86-NEXT: shrl %ecx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl %ecx, 28(%edx)
-; X86-NEXT: movl %edi, 24(%edx)
-; X86-NEXT: movl %eax, 20(%edx)
-; X86-NEXT: movl (%esp), %eax # 4-byte Reload
-; X86-NEXT: movl %eax, 16(%edx)
-; X86-NEXT: movl %ebp, 12(%edx)
-; X86-NEXT: movl %ebx, 8(%edx)
-; X86-NEXT: movl %esi, 4(%edx)
+; X86-NEXT: movl %eax, 28(%ecx)
+; X86-NEXT: movl %ebx, 24(%ecx)
+; X86-NEXT: movl %esi, 20(%ecx)
+; X86-NEXT: movl %edx, 16(%ecx)
+; X86-NEXT: movl %ebp, 12(%ecx)
+; X86-NEXT: movl %edi, 8(%ecx)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: movl %eax, (%edx)
-; X86-NEXT: movl %edx, %eax
+; X86-NEXT: movl %eax, 4(%ecx)
+; X86-NEXT: movl (%esp), %eax # 4-byte Reload
+; X86-NEXT: movl %eax, (%ecx)
+; X86-NEXT: movl %ecx, %eax
; X86-NEXT: addl $8, %esp
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
@@ -1639,45 +1598,39 @@ define i256 @ashr_i256_1(i256 %a0) nounwind {
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
; X86-NEXT: subl $8, %esp
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X86-NEXT: movl %ebx, %eax
-; X86-NEXT: shll $31, %eax
-; X86-NEXT: shrdl $1, %esi, %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shrl %esi
-; X86-NEXT: orl %eax, %esi
-; X86-NEXT: movl %edx, %eax
-; X86-NEXT: shll $31, %eax
-; X86-NEXT: shrdl $1, %ebp, %ebx
-; X86-NEXT: shrl %ebp
-; X86-NEXT: orl %eax, %ebp
-; X86-NEXT: movl %edi, %ecx
-; X86-NEXT: shll $31, %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT: movl %edi, %esi
+; X86-NEXT: shldl $31, %eax, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shrdl $1, %eax, %ecx
+; X86-NEXT: movl %ecx, (%esp) # 4-byte Spill
+; X86-NEXT: movl %edx, %ebp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: shldl $31, %eax, %ebp
+; X86-NEXT: shrdl $1, %eax, %edi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %ebx, %esi
+; X86-NEXT: shldl $31, %eax, %esi
; X86-NEXT: shrdl $1, %eax, %edx
-; X86-NEXT: movl %edx, (%esp) # 4-byte Spill
-; X86-NEXT: shrl %eax
-; X86-NEXT: orl %ecx, %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: shrdl $1, %eax, %ebx
+; X86-NEXT: sarl %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: shrdl $1, %ecx, %edi
-; X86-NEXT: sarl %ecx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl %ecx, 28(%edx)
-; X86-NEXT: movl %edi, 24(%edx)
-; X86-NEXT: movl %eax, 20(%edx)
-; X86-NEXT: movl (%esp), %eax # 4-byte Reload
-; X86-NEXT: movl %eax, 16(%edx)
-; X86-NEXT: movl %ebp, 12(%edx)
-; X86-NEXT: movl %ebx, 8(%edx)
-; X86-NEXT: movl %esi, 4(%edx)
+; X86-NEXT: movl %eax, 28(%ecx)
+; X86-NEXT: movl %ebx, 24(%ecx)
+; X86-NEXT: movl %esi, 20(%ecx)
+; X86-NEXT: movl %edx, 16(%ecx)
+; X86-NEXT: movl %ebp, 12(%ecx)
+; X86-NEXT: movl %edi, 8(%ecx)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: movl %eax, (%edx)
-; X86-NEXT: movl %edx, %eax
+; X86-NEXT: movl %eax, 4(%ecx)
+; X86-NEXT: movl (%esp), %eax # 4-byte Reload
+; X86-NEXT: movl %eax, (%ecx)
+; X86-NEXT: movl %ecx, %eax
; X86-NEXT: addl $8, %esp
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
diff --git a/llvm/test/CodeGen/X86/shift-mask.ll b/llvm/test/CodeGen/X86/shift-mask.ll
index e3dbbdd0402e3..09dcc8cb52118 100644
--- a/llvm/test/CodeGen/X86/shift-mask.ll
+++ b/llvm/test/CodeGen/X86/shift-mask.ll
@@ -614,12 +614,10 @@ define i64 @test_i64_lshr_lshr_1(i64 %a0) {
define i64 @test_i64_lshr_lshr_2(i64 %a0) {
; X86-LABEL: test_i64_lshr_lshr_2:
; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: leal (,%edx,4), %eax
-; X86-NEXT: shll $2, %ecx
-; X86-NEXT: shrl $30, %edx
-; X86-NEXT: orl %ecx, %edx
+; X86-NEXT: shldl $2, %eax, %edx
+; X86-NEXT: shll $2, %eax
; X86-NEXT: andl $536870911, %edx # imm = 0x1FFFFFFF
; X86-NEXT: retl
;
diff --git a/llvm/test/CodeGen/X86/shld-machine-combiner.mir b/llvm/test/CodeGen/X86/shld-machine-combiner.mir
index fceee416118e1..c0f4319f320d2 100644
--- a/llvm/test/CodeGen/X86/shld-machine-combiner.mir
+++ b/llvm/test/CodeGen/X86/shld-machine-combiner.mir
@@ -5,96 +5,119 @@
---
name: rri32
alignment: 16
-debugInstrRef: true
+tracksRegLiveness: true
body: |
bb.0:
liveins: $eax, $ebx
; FAST-SHLD-LABEL: name: rri32
; FAST-SHLD: liveins: $eax, $ebx
; FAST-SHLD-NEXT: {{ $}}
- ; FAST-SHLD-NEXT: [[SHLD32rri8_:%[0-9]+]]:gr32 = SHLD32rri8 $eax, $ebx, 2, implicit-def $eflags
- ; FAST-SHLD-NEXT: RET [[SHLD32rri8_]]
+ ; FAST-SHLD-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $eax
+ ; FAST-SHLD-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY $ebx
+ ; FAST-SHLD-NEXT: [[SHLD32rri8_:%[0-9]+]]:gr32 = SHLD32rri8 [[COPY]], [[COPY1]], 2, implicit-def $eflags
+ ; FAST-SHLD-NEXT: $eax = COPY [[SHLD32rri8_]]
+ ; FAST-SHLD-NEXT: RET 0
;
; SLOW-SHLD-LABEL: name: rri32
; SLOW-SHLD: liveins: $eax, $ebx
; SLOW-SHLD-NEXT: {{ $}}
- ; SLOW-SHLD-NEXT: [[SHL32ri:%[0-9]+]]:gr32 = SHL32ri $eax, 2, implicit-def $eflags
- ; SLOW-SHLD-NEXT: [[SHR32ri:%[0-9]+]]:gr32 = SHR32ri $ebx, 30, implicit-def $eflags
+ ; SLOW-SHLD-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $eax
+ ; SLOW-SHLD-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY $ebx
+ ; SLOW-SHLD-NEXT: [[SHL32ri:%[0-9]+]]:gr32 = SHL32ri [[COPY]], 2, implicit-def $eflags
+ ; SLOW-SHLD-NEXT: [[SHR32ri:%[0-9]+]]:gr32 = SHR32ri [[COPY1]], 30, implicit-def $eflags
; SLOW-SHLD-NEXT: [[OR32rr:%[0-9]+]]:gr32 = OR32rr [[SHL32ri]], [[SHR32ri]], implicit-def $eflags
- ; SLOW-SHLD-NEXT: RET [[OR32rr]]
- %3:gr32 = SHLD32rri8 $eax, $ebx, 2, implicit-def $eflags
- RET %3
+ ; SLOW-SHLD-NEXT: $eax = COPY [[OR32rr]]
+ ; SLOW-SHLD-NEXT: RET 0
+ %0:gr32 = COPY $eax
+ %1:gr32 = COPY $ebx
+ %2:gr32 = SHLD32rri8 %0, %1, 2, implicit-def $eflags
+ $eax = COPY %2
+ RET 0, implicit $rax
...
---
name: rri64
alignment: 16
-debugInstrRef: true
+tracksRegLiveness: true
body: |
bb.0:
liveins: $rax, $rbx
; FAST-SHLD-LABEL: name: rri64
; FAST-SHLD: liveins: $rax, $rbx
; FAST-SHLD-NEXT: {{ $}}
- ; FAST-SHLD-NEXT: [[SHLD64rri8_:%[0-9]+]]:gr64 = SHLD64rri8 $rax, $rbx, 2, implicit-def $eflags
- ; FAST-SHLD-NEXT: RET [[SHLD64rri8_]]
+ ; FAST-SHLD-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rax
+ ; FAST-SHLD-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rbx
+ ; FAST-SHLD-NEXT: [[SHLD64rri8_:%[0-9]+]]:gr64 = SHLD64rri8 [[COPY]], [[COPY1]], 2, implicit-def $eflags
+ ; FAST-SHLD-NEXT: $rax = COPY [[SHLD64rri8_]]
+ ; FAST-SHLD-NEXT: RET 0
;
; SLOW-SHLD-LABEL: name: rri64
; SLOW-SHLD: liveins: $rax, $rbx
; SLOW-SHLD-NEXT: {{ $}}
- ; SLOW-SHLD-NEXT: [[SHL64ri:%[0-9]+]]:gr64 = SHL64ri $rax, 2, implicit-def $eflags
- ; SLOW-SHLD-NEXT: [[SHR64ri:%[0-9]+]]:gr64 = SHR64ri $rbx, 62, implicit-def $eflags
+ ; SLOW-SHLD-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rax
+ ; SLOW-SHLD-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rbx
+ ; SLOW-SHLD-NEXT: [[SHL64ri:%[0-9]+]]:gr64 = SHL64ri [[COPY]], 2, implicit-def $eflags
+ ; SLOW-SHLD-NEXT: [[SHR64ri:%[0-9]+]]:gr64 = SHR64ri [[COPY1]], 62, implicit-def $eflags
; SLOW-SHLD-NEXT: [[OR64rr:%[0-9]+]]:gr64 = OR64rr [[SHL64ri]], [[SHR64ri]], implicit-def $eflags
- ; SLOW-SHLD-NEXT: RET [[OR64rr]]
- %3:gr64 = SHLD64rri8 $rax, $rbx, 2, implicit-def $eflags
- RET %3
+ ; SLOW-SHLD-NEXT: $rax = COPY [[OR64rr]]
+ ; SLOW-SHLD-NEXT: RET 0
+ %0:gr64 = COPY $rax
+ %1:gr64 = COPY $rbx
+ %2:gr64 = SHLD64rri8 %0, %1, 2, implicit-def $eflags
+ $rax = COPY %2
+ RET 0, implicit $rax
...
---
name: mri32
alignment: 16
-debugInstrRef: true
+tracksRegLiveness: true
body: |
bb.0:
liveins: $rax, $ebx
; FAST-SHLD-LABEL: name: mri32
; FAST-SHLD: liveins: $rax, $ebx
; FAST-SHLD-NEXT: {{ $}}
- ; FAST-SHLD-NEXT: SHLD32mri8 $rax, 1, $noreg, 0, $noreg, $ebx, 2, implicit-def $eflags
+ ; FAST-SHLD-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $ebx
+ ; FAST-SHLD-NEXT: SHLD32mri8 $rax, 1, $noreg, 0, $noreg, [[COPY]], 2, implicit-def $eflags
; FAST-SHLD-NEXT: RET 0
;
; SLOW-SHLD-LABEL: name: mri32
; SLOW-SHLD: liveins: $rax, $ebx
; SLOW-SHLD-NEXT: {{ $}}
+ ; SLOW-SHLD-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $ebx
; SLOW-SHLD-NEXT: SHL32mi $rax, 1, $noreg, 0, $noreg, 2, implicit-def $eflags
- ; SLOW-SHLD-NEXT: [[SHR32ri:%[0-9]+]]:gr32_abcd_and_gr32_bsi = SHR32ri $ebx, 30, implicit-def $eflags
+ ; SLOW-SHLD-NEXT: [[SHR32ri:%[0-9]+]]:gr32 = SHR32ri [[COPY]], 30, implicit-def $eflags
; SLOW-SHLD-NEXT: OR32mr $rax, 1, $noreg, 0, $noreg, [[SHR32ri]], implicit-def $eflags
; SLOW-SHLD-NEXT: RET 0
- SHLD32mri8 $rax, 1, $noreg, 0, $noreg, $ebx, 2, implicit-def $eflags
+ %0:gr32 = COPY $ebx
+ SHLD32mri8 $rax, 1, $noreg, 0, $noreg, %0, 2, implicit-def $eflags
RET 0
...
---
name: mri64
alignment: 16
-debugInstrRef: true
+tracksRegLiveness: true
body: |
bb.0:
- liveins: $rax, $ebx
-
+ liveins: $rax, $rbx
; FAST-SHLD-LABEL: name: mri64
- ; FAST-SHLD: liveins: $rax, $ebx
+ ; FAST-SHLD: liveins: $rax, $rbx
; FAST-SHLD-NEXT: {{ $}}
- ; FAST-SHLD-NEXT: SHLD64mri8 $rax, 1, $noreg, 0, $noreg, $rbx, 2, implicit-def $eflags
+ ; FAST-SHLD-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rbx
+ ; FAST-SHLD-NEXT: SHLD64mri8 $rax, 1, $noreg, 0, $noreg, [[COPY]], 2, implicit-def $eflags
; FAST-SHLD-NEXT: RET 0
;
; SLOW-SHLD-LABEL: name: mri64
- ; SLOW-SHLD: liveins: $rax, $ebx
+ ; SLOW-SHLD: liveins: $rax, $rbx
; SLOW-SHLD-NEXT: {{ $}}
+ ; SLOW-SHLD-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rbx
; SLOW-SHLD-NEXT: SHL64mi $rax, 1, $noreg, 0, $noreg, 2, implicit-def $eflags
- ; SLOW-SHLD-NEXT: [[SHR64ri:%[0-9]+]]:gr64_with_sub_32bit_in_gr32_abcd_and_gr32_bsi = SHR64ri $rbx, 62, implicit-def $eflags
+ ; SLOW-SHLD-NEXT: [[SHR64ri:%[0-9]+]]:gr64 = SHR64ri [[COPY]], 62, implicit-def $eflags
; SLOW-SHLD-NEXT: OR64mr $rax, 1, $noreg, 0, $noreg, [[SHR64ri]], implicit-def $eflags
; SLOW-SHLD-NEXT: RET 0
- SHLD64mri8 $rax, 1, $noreg, 0, $noreg, $rbx, 2, implicit-def $eflags
+ %0:gr64 = COPY $rbx
+ SHLD64mri8 $rax, 1, $noreg, 0, $noreg, %0, 2, implicit-def $eflags
RET 0
...
diff --git a/llvm/test/CodeGen/X86/udiv_fix.ll b/llvm/test/CodeGen/X86/udiv_fix.ll
index 8f613274e6e24..82dfeeee13293 100644
--- a/llvm/test/CodeGen/X86/udiv_fix.ll
+++ b/llvm/test/CodeGen/X86/udiv_fix.ll
@@ -163,11 +163,8 @@ define i64 @func5(i64 %x, i64 %y) nounwind {
; X86-NEXT: movl %ecx, %edx
; X86-NEXT: shrl %edx
; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NEXT: shll $31, %ecx
-; X86-NEXT: movl %eax, %edx
-; X86-NEXT: shrl %edx
-; X86-NEXT: orl %ecx, %edx
-; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NEXT: shldl $31, %eax, %ecx
+; X86-NEXT: movl %ecx, {{[0-9]+}}(%esp)
; X86-NEXT: shll $31, %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: leal {{[0-9]+}}(%esp), %eax
diff --git a/llvm/test/CodeGen/X86/udiv_fix_sat.ll b/llvm/test/CodeGen/X86/udiv_fix_sat.ll
index 9d57ac8b6fb99..421426f3ad959 100644
--- a/llvm/test/CodeGen/X86/udiv_fix_sat.ll
+++ b/llvm/test/CodeGen/X86/udiv_fix_sat.ll
@@ -172,12 +172,10 @@ define i64 @func5(i64 %x, i64 %y) nounwind {
; X64: # %bb.0:
; X64-NEXT: pushq %rax
; X64-NEXT: movq %rsi, %rdx
-; X64-NEXT: leaq (%rdi,%rdi), %rsi
-; X64-NEXT: movq %rdi, %rax
-; X64-NEXT: shrq $63, %rax
-; X64-NEXT: shlq $31, %rax
-; X64-NEXT: shrq $33, %rsi
-; X64-NEXT: orq %rax, %rsi
+; X64-NEXT: leaq (%rdi,%rdi), %rax
+; X64-NEXT: movq %rdi, %rsi
+; X64-NEXT: shrq $63, %rsi
+; X64-NEXT: shldq $31, %rax, %rsi
; X64-NEXT: shlq $32, %rdi
; X64-NEXT: xorl %ecx, %ecx
; X64-NEXT: callq __udivti3 at PLT
diff --git a/llvm/test/CodeGen/X86/vector-sext.ll b/llvm/test/CodeGen/X86/vector-sext.ll
index d05273b219f5d..693255e80b353 100644
--- a/llvm/test/CodeGen/X86/vector-sext.ll
+++ b/llvm/test/CodeGen/X86/vector-sext.ll
@@ -3692,24 +3692,22 @@ define <4 x i32> @sext_4i17_to_4i32(ptr %ptr) {
;
; X86-SSE2-LABEL: sext_4i17_to_4i32:
; X86-SSE2: # %bb.0:
-; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-SSE2-NEXT: movl (%ecx), %eax
-; X86-SSE2-NEXT: movl 4(%ecx), %edx
-; X86-SSE2-NEXT: movl 8(%ecx), %ecx
-; X86-SSE2-NEXT: shll $13, %ecx
-; X86-SSE2-NEXT: movd %edx, %xmm1
-; X86-SSE2-NEXT: movd %eax, %xmm0
-; X86-SSE2-NEXT: shrdl $17, %edx, %eax
-; X86-SSE2-NEXT: shrl $19, %edx
-; X86-SSE2-NEXT: orl %ecx, %edx
-; X86-SSE2-NEXT: movd %edx, %xmm2
-; X86-SSE2-NEXT: pslld $15, %xmm2
-; X86-SSE2-NEXT: psrad $15, %xmm2
+; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-SSE2-NEXT: movl (%edx), %eax
+; X86-SSE2-NEXT: movl 4(%edx), %ecx
+; X86-SSE2-NEXT: movl 8(%edx), %edx
+; X86-SSE2-NEXT: shldl $13, %ecx, %edx
+; X86-SSE2-NEXT: movd %edx, %xmm0
+; X86-SSE2-NEXT: pslld $15, %xmm0
+; X86-SSE2-NEXT: psrad $15, %xmm0
+; X86-SSE2-NEXT: movd %ecx, %xmm1
; X86-SSE2-NEXT: pslld $13, %xmm1
; X86-SSE2-NEXT: psrad $15, %xmm1
-; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; X86-SSE2-NEXT: movd %eax, %xmm0
; X86-SSE2-NEXT: pslld $15, %xmm0
; X86-SSE2-NEXT: psrad $15, %xmm0
+; X86-SSE2-NEXT: shrdl $17, %ecx, %eax
; X86-SSE2-NEXT: movd %eax, %xmm2
; X86-SSE2-NEXT: pslld $15, %xmm2
; X86-SSE2-NEXT: psrad $15, %xmm2
diff --git a/llvm/test/CodeGen/X86/xor-lea.ll b/llvm/test/CodeGen/X86/xor-lea.ll
index 2be7811adfd9a..d50752e48d293 100644
--- a/llvm/test/CodeGen/X86/xor-lea.ll
+++ b/llvm/test/CodeGen/X86/xor-lea.ll
@@ -364,12 +364,10 @@ define i32 @xor_bigshl_sminval_i32(i32 %x) {
define i64 @xor_shl_sminval_i64(i64 %x) {
; X86-LABEL: xor_shl_sminval_i64:
; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: leal (,%edx,4), %eax
-; X86-NEXT: shll $2, %ecx
-; X86-NEXT: shrl $30, %edx
-; X86-NEXT: orl %ecx, %edx
+; X86-NEXT: shldl $2, %eax, %edx
+; X86-NEXT: shll $2, %eax
; X86-NEXT: addl $-2147483648, %edx # imm = 0x80000000
; X86-NEXT: retl
;
>From fddc3562491b739ff1a0e3aeb004a40e663a7a08 Mon Sep 17 00:00:00 2001
From: stomfaig <stomfaig at gmail.com>
Date: Tue, 17 Mar 2026 11:50:53 +0000
Subject: [PATCH 04/11] add kill flags
---
llvm/lib/Target/X86/X86InstrInfo.cpp | 30 +++++++++++++++++-----------
1 file changed, 18 insertions(+), 12 deletions(-)
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index c8578540ccb6e..04e7a02fe92ae 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -10708,16 +10708,20 @@ genAlternativeShldSequence(MachineInstr &Root, const TargetInstrInfo &TII,
unsigned ShrOpc = (BW == 64) ? X86::SHR64ri : X86::SHR32ri;
unsigned OrOpc = (BW == 64) ? X86::OR64rr : X86::OR32rr;
int64_t Imm = Root.getOperand(3).getImm();
- MachineInstr *Shl = BuildMI(*MF, MIMetadata(Root), TII.get(ShlOpc), ShlReg)
- .addReg(Root.getOperand(1).getReg())
- .addImm(Imm);
- MachineInstr *Shr = BuildMI(*MF, MIMetadata(Root), TII.get(ShrOpc), ShrReg)
- .addReg(Root.getOperand(2).getReg())
- .addImm(BW - Imm);
+ MachineInstr *Shl =
+ BuildMI(*MF, MIMetadata(Root), TII.get(ShlOpc), ShlReg)
+ .addReg(Root.getOperand(1).getReg(),
+ getKillRegState(Root.getOperand(1).isKill()))
+ .addImm(Imm);
+ MachineInstr *Shr =
+ BuildMI(*MF, MIMetadata(Root), TII.get(ShrOpc), ShrReg)
+ .addReg(Root.getOperand(2).getReg(),
+ getKillRegState(Root.getOperand(2).isKill()))
+ .addImm(BW - Imm);
MachineInstr *Or = BuildMI(*MF, MIMetadata(Root), TII.get(OrOpc),
Root.getOperand(0).getReg())
- .addReg(ShlReg)
- .addReg(ShrReg);
+ .addReg(ShlReg, RegState::Kill)
+ .addReg(ShrReg, RegState::Kill);
InstrIdxForVirtReg.insert({ShlReg, 0});
InstrIdxForVirtReg.insert({ShrReg, 1});
InsInstrs.push_back(Shl);
@@ -10741,16 +10745,18 @@ genAlternativeShldSequence(MachineInstr &Root, const TargetInstrInfo &TII,
.add(Root.getOperand(0 + X86::AddrDisp))
.add(Root.getOperand(0 + X86::AddrSegmentReg))
.addImm(Imm);
- MachineInstr *Shr = BuildMI(*MF, MIMetadata(Root), TII.get(ShrOpc), ShrReg)
- .addReg(Root.getOperand(5).getReg())
- .addImm(BW - Imm);
+ MachineInstr *Shr =
+ BuildMI(*MF, MIMetadata(Root), TII.get(ShrOpc), ShrReg)
+ .addReg(Root.getOperand(5).getReg(),
+ getKillRegState(Root.getOperand(5).isKill()))
+ .addImm(BW - Imm);
MachineInstr *Or = BuildMI(*MF, MIMetadata(Root), TII.get(OrOpc))
.add(Root.getOperand(0 + X86::AddrBaseReg))
.add(Root.getOperand(0 + X86::AddrScaleAmt))
.add(Root.getOperand(0 + X86::AddrIndexReg))
.add(Root.getOperand(0 + X86::AddrDisp))
.add(Root.getOperand(0 + X86::AddrSegmentReg))
- .addReg(ShrReg);
+ .addReg(ShrReg, RegState::Kill);
InstrIdxForVirtReg.insert({ShrReg, 1});
InsInstrs.push_back(Shl);
InsInstrs.push_back(Shr);
>From a9f040bfa5b427e3580279553bb1859c82698e48 Mon Sep 17 00:00:00 2001
From: stomfaig <stomfaig at gmail.com>
Date: Tue, 17 Mar 2026 11:51:14 +0000
Subject: [PATCH 05/11] add implicit register usage
---
.../test/CodeGen/X86/shld-machine-combiner.mir | 18 +++++++++---------
1 file changed, 9 insertions(+), 9 deletions(-)
diff --git a/llvm/test/CodeGen/X86/shld-machine-combiner.mir b/llvm/test/CodeGen/X86/shld-machine-combiner.mir
index c0f4319f320d2..d3e4b52f9bd12 100644
--- a/llvm/test/CodeGen/X86/shld-machine-combiner.mir
+++ b/llvm/test/CodeGen/X86/shld-machine-combiner.mir
@@ -16,7 +16,7 @@ body: |
; FAST-SHLD-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY $ebx
; FAST-SHLD-NEXT: [[SHLD32rri8_:%[0-9]+]]:gr32 = SHLD32rri8 [[COPY]], [[COPY1]], 2, implicit-def $eflags
; FAST-SHLD-NEXT: $eax = COPY [[SHLD32rri8_]]
- ; FAST-SHLD-NEXT: RET 0
+ ; FAST-SHLD-NEXT: RET 0, implicit $rax
;
; SLOW-SHLD-LABEL: name: rri32
; SLOW-SHLD: liveins: $eax, $ebx
@@ -25,14 +25,14 @@ body: |
; SLOW-SHLD-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY $ebx
; SLOW-SHLD-NEXT: [[SHL32ri:%[0-9]+]]:gr32 = SHL32ri [[COPY]], 2, implicit-def $eflags
; SLOW-SHLD-NEXT: [[SHR32ri:%[0-9]+]]:gr32 = SHR32ri [[COPY1]], 30, implicit-def $eflags
- ; SLOW-SHLD-NEXT: [[OR32rr:%[0-9]+]]:gr32 = OR32rr [[SHL32ri]], [[SHR32ri]], implicit-def $eflags
+ ; SLOW-SHLD-NEXT: [[OR32rr:%[0-9]+]]:gr32 = OR32rr killed [[SHL32ri]], killed [[SHR32ri]], implicit-def $eflags
; SLOW-SHLD-NEXT: $eax = COPY [[OR32rr]]
- ; SLOW-SHLD-NEXT: RET 0
+ ; SLOW-SHLD-NEXT: RET 0, implicit $rax
%0:gr32 = COPY $eax
%1:gr32 = COPY $ebx
%2:gr32 = SHLD32rri8 %0, %1, 2, implicit-def $eflags
$eax = COPY %2
- RET 0, implicit $rax
+ RET 0, implicit $rax
...
---
@@ -49,7 +49,7 @@ body: |
; FAST-SHLD-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rbx
; FAST-SHLD-NEXT: [[SHLD64rri8_:%[0-9]+]]:gr64 = SHLD64rri8 [[COPY]], [[COPY1]], 2, implicit-def $eflags
; FAST-SHLD-NEXT: $rax = COPY [[SHLD64rri8_]]
- ; FAST-SHLD-NEXT: RET 0
+ ; FAST-SHLD-NEXT: RET 0, implicit $rax
;
; SLOW-SHLD-LABEL: name: rri64
; SLOW-SHLD: liveins: $rax, $rbx
@@ -58,9 +58,9 @@ body: |
; SLOW-SHLD-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rbx
; SLOW-SHLD-NEXT: [[SHL64ri:%[0-9]+]]:gr64 = SHL64ri [[COPY]], 2, implicit-def $eflags
; SLOW-SHLD-NEXT: [[SHR64ri:%[0-9]+]]:gr64 = SHR64ri [[COPY1]], 62, implicit-def $eflags
- ; SLOW-SHLD-NEXT: [[OR64rr:%[0-9]+]]:gr64 = OR64rr [[SHL64ri]], [[SHR64ri]], implicit-def $eflags
+ ; SLOW-SHLD-NEXT: [[OR64rr:%[0-9]+]]:gr64 = OR64rr killed [[SHL64ri]], killed [[SHR64ri]], implicit-def $eflags
; SLOW-SHLD-NEXT: $rax = COPY [[OR64rr]]
- ; SLOW-SHLD-NEXT: RET 0
+ ; SLOW-SHLD-NEXT: RET 0, implicit $rax
%0:gr64 = COPY $rax
%1:gr64 = COPY $rbx
%2:gr64 = SHLD64rri8 %0, %1, 2, implicit-def $eflags
@@ -88,7 +88,7 @@ body: |
; SLOW-SHLD-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $ebx
; SLOW-SHLD-NEXT: SHL32mi $rax, 1, $noreg, 0, $noreg, 2, implicit-def $eflags
; SLOW-SHLD-NEXT: [[SHR32ri:%[0-9]+]]:gr32 = SHR32ri [[COPY]], 30, implicit-def $eflags
- ; SLOW-SHLD-NEXT: OR32mr $rax, 1, $noreg, 0, $noreg, [[SHR32ri]], implicit-def $eflags
+ ; SLOW-SHLD-NEXT: OR32mr $rax, 1, $noreg, 0, $noreg, killed [[SHR32ri]], implicit-def $eflags
; SLOW-SHLD-NEXT: RET 0
%0:gr32 = COPY $ebx
SHLD32mri8 $rax, 1, $noreg, 0, $noreg, %0, 2, implicit-def $eflags
@@ -115,7 +115,7 @@ body: |
; SLOW-SHLD-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rbx
; SLOW-SHLD-NEXT: SHL64mi $rax, 1, $noreg, 0, $noreg, 2, implicit-def $eflags
; SLOW-SHLD-NEXT: [[SHR64ri:%[0-9]+]]:gr64 = SHR64ri [[COPY]], 62, implicit-def $eflags
- ; SLOW-SHLD-NEXT: OR64mr $rax, 1, $noreg, 0, $noreg, [[SHR64ri]], implicit-def $eflags
+ ; SLOW-SHLD-NEXT: OR64mr $rax, 1, $noreg, 0, $noreg, killed [[SHR64ri]], implicit-def $eflags
; SLOW-SHLD-NEXT: RET 0
%0:gr64 = COPY $rbx
SHLD64mri8 $rax, 1, $noreg, 0, $noreg, %0, 2, implicit-def $eflags
>From bf222d4891b8de77735203eb58daf7105b353f4b Mon Sep 17 00:00:00 2001
From: stomfaig <stomfaig at gmail.com>
Date: Thu, 19 Mar 2026 17:59:07 +0000
Subject: [PATCH 06/11] add shrd rewrites
---
llvm/lib/Target/X86/X86InstrInfo.cpp | 133 +--
llvm/lib/Target/X86/X86InstrInfo.h | 4 +-
llvm/test/CodeGen/X86/avgflooru-i128.ll | 59 +-
llvm/test/CodeGen/X86/avgflooru-scalar.ll | 12 +-
llvm/test/CodeGen/X86/div_i129_v_pow2k.ll | 22 +-
.../CodeGen/X86/expand-large-fp-optnone.ll | 9 +-
llvm/test/CodeGen/X86/fold-tied-op.ll | 96 +--
llvm/test/CodeGen/X86/known-bits.ll | 4 +-
llvm/test/CodeGen/X86/midpoint-int.ll | 78 +-
llvm/test/CodeGen/X86/pr32282.ll | 19 +-
llvm/test/CodeGen/X86/pr43820.ll | 580 ++++++-------
llvm/test/CodeGen/X86/scmp.ll | 775 ++++++++++--------
llvm/test/CodeGen/X86/sdiv_fix_sat.ll | 22 +-
...-combiner.mir => shd-machine-combiner.mir} | 120 +++
llvm/test/CodeGen/X86/shift-and.ll | 19 +-
llvm/test/CodeGen/X86/shift-i256.ll | 36 +-
llvm/test/CodeGen/X86/shift-i512.ll | 146 ++--
llvm/test/CodeGen/X86/shift-mask.ll | 7 +-
llvm/test/CodeGen/X86/smul_fix.ll | 28 +-
llvm/test/CodeGen/X86/smul_fix_sat.ll | 100 ++-
llvm/test/CodeGen/X86/udiv_fix_sat.ll | 66 +-
llvm/test/CodeGen/X86/umul_fix.ll | 8 +-
llvm/test/CodeGen/X86/umul_fix_sat.ll | 51 +-
llvm/test/CodeGen/X86/vector-sext.ll | 46 +-
llvm/test/CodeGen/X86/vector-zext.ll | 52 +-
25 files changed, 1433 insertions(+), 1059 deletions(-)
rename llvm/test/CodeGen/X86/{shld-machine-combiner.mir => shd-machine-combiner.mir} (51%)
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index 04e7a02fe92ae..07639eaf2c0c6 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -10664,13 +10664,15 @@ bool X86InstrInfo::getMachineCombinerPatterns(
break;
}
// We do not support CL variant, as it requires a lot of bookkeeping.
- // Only expand when SHLD is slow; on CPUs with fast SHLD the replacement
- // sequence (SHL + SHR + OR) is not profitable.
case X86::SHLD32rri8:
case X86::SHLD32mri8:
case X86::SHLD64rri8:
- case X86::SHLD64mri8: {
- Patterns.push_back(X86MachineCombinerPattern::USHLD);
+ case X86::SHLD64mri8:
+ case X86::SHRD32rri8:
+ case X86::SHRD32mri8:
+ case X86::SHRD64rri8:
+ case X86::SHRD64mri8: {
+ Patterns.push_back(X86MachineCombinerPattern::USHD);
return true;
break;
}
@@ -10680,10 +10682,10 @@ bool X86InstrInfo::getMachineCombinerPatterns(
}
static void
-genAlternativeShldSequence(MachineInstr &Root, const TargetInstrInfo &TII,
- SmallVectorImpl<MachineInstr *> &InsInstrs,
- SmallVectorImpl<MachineInstr *> &DelInstrs,
- DenseMap<Register, unsigned> &InstrIdxForVirtReg) {
+genAlternativeShdSequence(MachineInstr &Root, const TargetInstrInfo &TII,
+ SmallVectorImpl<MachineInstr *> &InsInstrs,
+ SmallVectorImpl<MachineInstr *> &DelInstrs,
+ DenseMap<Register, unsigned> &InstrIdxForVirtReg) {
auto *MF = Root.getMF();
auto OpCode = Root.getOpcode();
@@ -10694,59 +10696,88 @@ genAlternativeShldSequence(MachineInstr &Root, const TargetInstrInfo &TII,
: TRI->getMinimalPhysRegClass(Reg);
};
+ // SHLD: dst = (dst << imm) | (src >> (BW-imm)) — direct=SHL, complement=SHR
+ // SHRD: dst = (dst >> imm) | (src << (BW-imm)) — direct=SHR, complement=SHL
unsigned BW = 0;
- if (OpCode == X86::SHLD32rri8 || OpCode == X86::SHLD32mri8)
+ bool IsShrd = false;
+ bool IsMem = false;
+ switch (OpCode) {
+ case X86::SHRD32rri8:
+ IsShrd = true;
+ [[fallthrough]];
+ case X86::SHLD32rri8:
BW = 32;
- else
+ break;
+ case X86::SHRD64rri8:
+ IsShrd = true;
+ [[fallthrough]];
+ case X86::SHLD64rri8:
BW = 64;
+ break;
+ case X86::SHRD32mri8:
+ IsShrd = true;
+ [[fallthrough]];
+ case X86::SHLD32mri8:
+ BW = 32;
+ IsMem = true;
+ break;
+ case X86::SHRD64mri8:
+ IsShrd = true;
+ [[fallthrough]];
+ case X86::SHLD64mri8:
+ BW = 64;
+ IsMem = true;
+ break;
+ default:
+ llvm_unreachable("Unexpected opcode in genAlternativeShdSequence");
+ }
+
+ unsigned ShlOpc = (BW == 64) ? X86::SHL64ri : X86::SHL32ri;
+ unsigned ShrOpc = (BW == 64) ? X86::SHR64ri : X86::SHR32ri;
+ unsigned DirectOpc = IsShrd ? ShrOpc : ShlOpc;
+ unsigned ComplOpc = IsShrd ? ShlOpc : ShrOpc;
- if (OpCode == X86::SHLD32rri8 || OpCode == X86::SHLD64rri8) {
+ if (!IsMem) {
const TargetRegisterClass *RC = GetRC(Root.getOperand(0).getReg());
- Register ShlReg = RegInfo.createVirtualRegister(RC);
- Register ShrReg = RegInfo.createVirtualRegister(RC);
- unsigned ShlOpc = (BW == 64) ? X86::SHL64ri : X86::SHL32ri;
- unsigned ShrOpc = (BW == 64) ? X86::SHR64ri : X86::SHR32ri;
+ Register Reg1 = RegInfo.createVirtualRegister(RC);
+ Register Reg2 = RegInfo.createVirtualRegister(RC);
unsigned OrOpc = (BW == 64) ? X86::OR64rr : X86::OR32rr;
int64_t Imm = Root.getOperand(3).getImm();
- MachineInstr *Shl =
- BuildMI(*MF, MIMetadata(Root), TII.get(ShlOpc), ShlReg)
+ MachineInstr *MI1 =
+ BuildMI(*MF, MIMetadata(Root), TII.get(DirectOpc), Reg1)
.addReg(Root.getOperand(1).getReg(),
getKillRegState(Root.getOperand(1).isKill()))
.addImm(Imm);
- MachineInstr *Shr =
- BuildMI(*MF, MIMetadata(Root), TII.get(ShrOpc), ShrReg)
+ MachineInstr *MI2 =
+ BuildMI(*MF, MIMetadata(Root), TII.get(ComplOpc), Reg2)
.addReg(Root.getOperand(2).getReg(),
getKillRegState(Root.getOperand(2).isKill()))
.addImm(BW - Imm);
MachineInstr *Or = BuildMI(*MF, MIMetadata(Root), TII.get(OrOpc),
Root.getOperand(0).getReg())
- .addReg(ShlReg, RegState::Kill)
- .addReg(ShrReg, RegState::Kill);
- InstrIdxForVirtReg.insert({ShlReg, 0});
- InstrIdxForVirtReg.insert({ShrReg, 1});
- InsInstrs.push_back(Shl);
- InsInstrs.push_back(Shr);
+ .addReg(Reg1, RegState::Kill)
+ .addReg(Reg2, RegState::Kill);
+ InstrIdxForVirtReg.insert({Reg1, 0});
+ InstrIdxForVirtReg.insert({Reg2, 1});
+ InsInstrs.push_back(MI1);
+ InsInstrs.push_back(MI2);
InsInstrs.push_back(Or);
- DelInstrs.push_back(&Root);
- return;
- }
-
- if (OpCode == X86::SHLD32mri8 || OpCode == X86::SHLD64mri8) {
+ } else {
const TargetRegisterClass *RC = GetRC(Root.getOperand(5).getReg());
- Register ShrReg = RegInfo.createVirtualRegister(RC);
- unsigned ShlOpc = (BW == 64) ? X86::SHL64mi : X86::SHL32mi;
- unsigned ShrOpc = (BW == 64) ? X86::SHR64ri : X86::SHR32ri;
+ Register RegSrc = RegInfo.createVirtualRegister(RC);
+ unsigned MemOpc = IsShrd ? ((BW == 64) ? X86::SHR64mi : X86::SHR32mi)
+ : ((BW == 64) ? X86::SHL64mi : X86::SHL32mi);
unsigned OrOpc = (BW == 64) ? X86::OR64mr : X86::OR32mr;
int64_t Imm = Root.getOperand(6).getImm();
- MachineInstr *Shl = BuildMI(*MF, MIMetadata(Root), TII.get(ShlOpc))
- .add(Root.getOperand(0 + X86::AddrBaseReg))
- .add(Root.getOperand(0 + X86::AddrScaleAmt))
- .add(Root.getOperand(0 + X86::AddrIndexReg))
- .add(Root.getOperand(0 + X86::AddrDisp))
- .add(Root.getOperand(0 + X86::AddrSegmentReg))
- .addImm(Imm);
- MachineInstr *Shr =
- BuildMI(*MF, MIMetadata(Root), TII.get(ShrOpc), ShrReg)
+ MachineInstr *MemShift = BuildMI(*MF, MIMetadata(Root), TII.get(MemOpc))
+ .add(Root.getOperand(0 + X86::AddrBaseReg))
+ .add(Root.getOperand(0 + X86::AddrScaleAmt))
+ .add(Root.getOperand(0 + X86::AddrIndexReg))
+ .add(Root.getOperand(0 + X86::AddrDisp))
+ .add(Root.getOperand(0 + X86::AddrSegmentReg))
+ .addImm(Imm);
+ MachineInstr *RegShift =
+ BuildMI(*MF, MIMetadata(Root), TII.get(ComplOpc), RegSrc)
.addReg(Root.getOperand(5).getReg(),
getKillRegState(Root.getOperand(5).isKill()))
.addImm(BW - Imm);
@@ -10756,15 +10787,13 @@ genAlternativeShldSequence(MachineInstr &Root, const TargetInstrInfo &TII,
.add(Root.getOperand(0 + X86::AddrIndexReg))
.add(Root.getOperand(0 + X86::AddrDisp))
.add(Root.getOperand(0 + X86::AddrSegmentReg))
- .addReg(ShrReg, RegState::Kill);
- InstrIdxForVirtReg.insert({ShrReg, 1});
- InsInstrs.push_back(Shl);
- InsInstrs.push_back(Shr);
+ .addReg(RegSrc, RegState::Kill);
+ InstrIdxForVirtReg.insert({RegSrc, 1});
+ InsInstrs.push_back(MemShift);
+ InsInstrs.push_back(RegShift);
InsInstrs.push_back(Or);
- DelInstrs.push_back(&Root);
- return;
}
- llvm_unreachable("Unexpected opcode in genAlternativeShldSequence");
+ DelInstrs.push_back(&Root);
}
static void
@@ -10872,9 +10901,9 @@ void X86InstrInfo::genAlternativeCodeSequence(
genAlternativeDpCodeSequence(Root, *this, InsInstrs, DelInstrs,
InstrIdxForVirtReg);
return;
- case X86MachineCombinerPattern::USHLD:
- genAlternativeShldSequence(Root, *this, InsInstrs, DelInstrs,
- InstrIdxForVirtReg);
+ case X86MachineCombinerPattern::USHD:
+ genAlternativeShdSequence(Root, *this, InsInstrs, DelInstrs,
+ InstrIdxForVirtReg);
return;
}
}
diff --git a/llvm/lib/Target/X86/X86InstrInfo.h b/llvm/lib/Target/X86/X86InstrInfo.h
index 65a5ca95233f3..fbfbc13df59cb 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.h
+++ b/llvm/lib/Target/X86/X86InstrInfo.h
@@ -30,8 +30,8 @@ class X86Subtarget;
enum X86MachineCombinerPattern : unsigned {
// X86 VNNI
DPWSSD = MachineCombinerPattern::TARGET_PATTERN_START,
- // Unfold SHLD
- USHLD,
+ // Unfold SHD
+ USHD,
};
namespace X86 {
diff --git a/llvm/test/CodeGen/X86/avgflooru-i128.ll b/llvm/test/CodeGen/X86/avgflooru-i128.ll
index f0ca5ea5ff255..c48ea1d75f65b 100644
--- a/llvm/test/CodeGen/X86/avgflooru-i128.ll
+++ b/llvm/test/CodeGen/X86/avgflooru-i128.ll
@@ -4,13 +4,15 @@
define i128 @avgflooru_i128(i128 %x, i128 %y) {
; CHECK-LABEL: avgflooru_i128:
; CHECK: # %bb.0: # %start
-; CHECK-NEXT: movq %rdi, %rax
-; CHECK-NEXT: addq %rdx, %rax
+; CHECK-NEXT: addq %rdx, %rdi
; CHECK-NEXT: adcq %rcx, %rsi
-; CHECK-NEXT: setb %cl
-; CHECK-NEXT: shrdq $1, %rsi, %rax
-; CHECK-NEXT: movzbl %cl, %edx
+; CHECK-NEXT: setb %al
+; CHECK-NEXT: shrq %rdi
+; CHECK-NEXT: movzbl %al, %edx
; CHECK-NEXT: shldq $63, %rsi, %rdx
+; CHECK-NEXT: movq %rsi, %rax
+; CHECK-NEXT: shlq $63, %rax
+; CHECK-NEXT: orq %rdi, %rax
; CHECK-NEXT: retq
start:
%xor = xor i128 %y, %x
@@ -32,10 +34,10 @@ define i128 @avgflooru_i128_multi_use(i128 %x, i128 %y) nounwind {
; CHECK-NEXT: pushq %r12
; CHECK-NEXT: pushq %rbx
; CHECK-NEXT: pushq %rax
-; CHECK-NEXT: movq %rcx, %rbx
-; CHECK-NEXT: movq %rdx, %r14
-; CHECK-NEXT: movq %rsi, %r15
-; CHECK-NEXT: movq %rdi, %r12
+; CHECK-NEXT: movq %rcx, %r15
+; CHECK-NEXT: movq %rdx, %r12
+; CHECK-NEXT: movq %rsi, %rbx
+; CHECK-NEXT: movq %rdi, %r14
; CHECK-NEXT: movq %rdx, %r13
; CHECK-NEXT: xorq %rdi, %r13
; CHECK-NEXT: movq %rcx, %rbp
@@ -43,18 +45,22 @@ define i128 @avgflooru_i128_multi_use(i128 %x, i128 %y) nounwind {
; CHECK-NEXT: movq %r13, %rdi
; CHECK-NEXT: movq %rbp, %rsi
; CHECK-NEXT: callq use at PLT
-; CHECK-NEXT: shrdq $1, %rbp, %r13
+; CHECK-NEXT: shrq %r13
+; CHECK-NEXT: movq %rbp, %rdi
+; CHECK-NEXT: shlq $63, %rdi
+; CHECK-NEXT: orq %r13, %rdi
; CHECK-NEXT: shrq %rbp
-; CHECK-NEXT: movq %r13, %rdi
; CHECK-NEXT: movq %rbp, %rsi
; CHECK-NEXT: callq use at PLT
-; CHECK-NEXT: addq %r14, %r12
-; CHECK-NEXT: adcq %rbx, %r15
-; CHECK-NEXT: setb %al
-; CHECK-NEXT: shrdq $1, %r15, %r12
-; CHECK-NEXT: movzbl %al, %edx
-; CHECK-NEXT: shldq $63, %r15, %rdx
-; CHECK-NEXT: movq %r12, %rax
+; CHECK-NEXT: addq %r12, %r14
+; CHECK-NEXT: adcq %r15, %rbx
+; CHECK-NEXT: setb %cl
+; CHECK-NEXT: shrq %r14
+; CHECK-NEXT: movq %rbx, %rax
+; CHECK-NEXT: shlq $63, %rax
+; CHECK-NEXT: orq %r14, %rax
+; CHECK-NEXT: movzbl %cl, %edx
+; CHECK-NEXT: shldq $63, %rbx, %rdx
; CHECK-NEXT: addq $8, %rsp
; CHECK-NEXT: popq %rbx
; CHECK-NEXT: popq %r12
@@ -124,19 +130,24 @@ define <2 x i128> @avgflooru_i128_vec(<2 x i128> %x, <2 x i128> %y) {
; CHECK-NEXT: setb %dil
; CHECK-NEXT: movzbl %dil, %edi
; CHECK-NEXT: shlq $63, %rdi
-; CHECK-NEXT: shrdq $1, %rdx, %rsi
-; CHECK-NEXT: shrq %rdx
-; CHECK-NEXT: orq %rdi, %rdx
+; CHECK-NEXT: movq %rdx, %r9
+; CHECK-NEXT: shrq %r9
+; CHECK-NEXT: orq %rdi, %r9
; CHECK-NEXT: addq {{[0-9]+}}(%rsp), %rcx
; CHECK-NEXT: adcq {{[0-9]+}}(%rsp), %r8
; CHECK-NEXT: setb %dil
; CHECK-NEXT: movzbl %dil, %edi
; CHECK-NEXT: shldq $63, %r8, %rdi
-; CHECK-NEXT: shldq $63, %rcx, %r8
+; CHECK-NEXT: shrq %rsi
+; CHECK-NEXT: shlq $63, %rdx
+; CHECK-NEXT: orq %rsi, %rdx
+; CHECK-NEXT: shrq %rcx
+; CHECK-NEXT: shlq $63, %r8
+; CHECK-NEXT: orq %rcx, %r8
; CHECK-NEXT: movq %r8, 16(%rax)
-; CHECK-NEXT: movq %rsi, (%rax)
+; CHECK-NEXT: movq %rdx, (%rax)
; CHECK-NEXT: movq %rdi, 24(%rax)
-; CHECK-NEXT: movq %rdx, 8(%rax)
+; CHECK-NEXT: movq %r9, 8(%rax)
; CHECK-NEXT: retq
start:
%xor = xor <2 x i128> %y, %x
diff --git a/llvm/test/CodeGen/X86/avgflooru-scalar.ll b/llvm/test/CodeGen/X86/avgflooru-scalar.ll
index bb070370316a8..bb41ae6f62a2f 100644
--- a/llvm/test/CodeGen/X86/avgflooru-scalar.ll
+++ b/llvm/test/CodeGen/X86/avgflooru-scalar.ll
@@ -260,7 +260,9 @@ define i64 @test_fixed_i64(i64 %a0, i64 %a1) nounwind {
; X86-NEXT: setb %dl
; X86-NEXT: movzbl %dl, %edx
; X86-NEXT: shldl $31, %eax, %edx
-; X86-NEXT: shldl $31, %ecx, %eax
+; X86-NEXT: shrl %ecx
+; X86-NEXT: shll $31, %eax
+; X86-NEXT: orl %ecx, %eax
; X86-NEXT: retl
;
; X64-LABEL: test_fixed_i64:
@@ -288,7 +290,9 @@ define i64 @test_lsb_i64(i64 %a0, i64 %a1) nounwind {
; X86-NEXT: setb %dl
; X86-NEXT: movzbl %dl, %edx
; X86-NEXT: shldl $31, %eax, %edx
-; X86-NEXT: shldl $31, %ecx, %eax
+; X86-NEXT: shrl %ecx
+; X86-NEXT: shll $31, %eax
+; X86-NEXT: orl %ecx, %eax
; X86-NEXT: retl
;
; X64-LABEL: test_lsb_i64:
@@ -318,7 +322,9 @@ define i64 @test_ext_i64(i64 %a0, i64 %a1) nounwind {
; X86-NEXT: setb %dl
; X86-NEXT: movzbl %dl, %edx
; X86-NEXT: shldl $31, %eax, %edx
-; X86-NEXT: shldl $31, %ecx, %eax
+; X86-NEXT: shrl %ecx
+; X86-NEXT: shll $31, %eax
+; X86-NEXT: orl %ecx, %eax
; X86-NEXT: retl
;
; X64-LABEL: test_ext_i64:
diff --git a/llvm/test/CodeGen/X86/div_i129_v_pow2k.ll b/llvm/test/CodeGen/X86/div_i129_v_pow2k.ll
index f915a8d27e6b3..6df156f71a0af 100644
--- a/llvm/test/CodeGen/X86/div_i129_v_pow2k.ll
+++ b/llvm/test/CodeGen/X86/div_i129_v_pow2k.ll
@@ -19,8 +19,10 @@ define i129 @v_sdiv_i129_v_pow2k(i129 %lhs) nounwind {
; X64-NEXT: andl $1, %ecx
; X64-NEXT: movq %rcx, %rdx
; X64-NEXT: negq %rdx
-; X64-NEXT: shrdq $33, %rsi, %rax
+; X64-NEXT: shrq $33, %rax
; X64-NEXT: shldq $31, %rsi, %rdx
+; X64-NEXT: shlq $31, %rsi
+; X64-NEXT: orq %rsi, %rax
; X64-NEXT: retq
;
; X64-O0-LABEL: v_sdiv_i129_v_pow2k:
@@ -146,12 +148,14 @@ define i129 @v_sdiv_exact_i129_v_pow2k(i129 %lhs) nounwind {
; X64-LABEL: v_sdiv_exact_i129_v_pow2k:
; X64: # %bb.0:
; X64-NEXT: movq %rdx, %rcx
-; X64-NEXT: movq %rdi, %rax
+; X64-NEXT: movq %rsi, %rax
; X64-NEXT: andl $1, %ecx
; X64-NEXT: movq %rcx, %rdx
; X64-NEXT: negq %rdx
-; X64-NEXT: shrdq $33, %rsi, %rax
+; X64-NEXT: shrq $33, %rdi
; X64-NEXT: shldq $31, %rsi, %rdx
+; X64-NEXT: shlq $31, %rax
+; X64-NEXT: orq %rdi, %rax
; X64-NEXT: retq
;
; X64-O0-LABEL: v_sdiv_exact_i129_v_pow2k:
@@ -241,10 +245,12 @@ define i129 @v_sdiv_exact_i129_v_pow2k(i129 %lhs) nounwind {
define i129 @v_udiv_i129_v_pow2k(i129 %lhs) nounwind {
; X64-LABEL: v_udiv_i129_v_pow2k:
; X64: # %bb.0:
-; X64-NEXT: movq %rdi, %rax
+; X64-NEXT: movq %rsi, %rax
; X64-NEXT: andl $1, %edx
-; X64-NEXT: shrdq $33, %rsi, %rax
+; X64-NEXT: shrq $33, %rdi
; X64-NEXT: shldq $31, %rsi, %rdx
+; X64-NEXT: shlq $31, %rax
+; X64-NEXT: orq %rdi, %rax
; X64-NEXT: xorl %ecx, %ecx
; X64-NEXT: retq
;
@@ -312,10 +318,12 @@ define i129 @v_udiv_i129_v_pow2k(i129 %lhs) nounwind {
define i129 @v_udiv_exact_i129_v_pow2k(i129 %lhs) nounwind {
; X64-LABEL: v_udiv_exact_i129_v_pow2k:
; X64: # %bb.0:
-; X64-NEXT: movq %rdi, %rax
+; X64-NEXT: movq %rsi, %rax
; X64-NEXT: andl $1, %edx
-; X64-NEXT: shrdq $33, %rsi, %rax
+; X64-NEXT: shrq $33, %rdi
; X64-NEXT: shldq $31, %rsi, %rdx
+; X64-NEXT: shlq $31, %rax
+; X64-NEXT: orq %rdi, %rax
; X64-NEXT: xorl %ecx, %ecx
; X64-NEXT: retq
;
diff --git a/llvm/test/CodeGen/X86/expand-large-fp-optnone.ll b/llvm/test/CodeGen/X86/expand-large-fp-optnone.ll
index ce38792e60258..ccb186175a9ed 100644
--- a/llvm/test/CodeGen/X86/expand-large-fp-optnone.ll
+++ b/llvm/test/CodeGen/X86/expand-large-fp-optnone.ll
@@ -192,14 +192,19 @@ define double @main(i224 %0) #0 {
; CHECK-NEXT: adcq $0, %rdx
; CHECK-NEXT: adcq $0, %rcx
; CHECK-NEXT: movq %rdi, %rdx
-; CHECK-NEXT: shrdq $2, %rsi, %rdx
+; CHECK-NEXT: shrq $2, %rdx
+; CHECK-NEXT: movq %rsi, %rax
+; CHECK-NEXT: shlq $62, %rax
+; CHECK-NEXT: orq %rax, %rdx
; CHECK-NEXT: movq %rdx, %rax
; CHECK-NEXT: shrq $32, %rax
; CHECK-NEXT: btq $55, %rdi
; CHECK-NEXT: jae .LBB0_9
; CHECK-NEXT: jmp .LBB0_7
; CHECK-NEXT: .LBB0_7: # %itofp-if-then20
-; CHECK-NEXT: shrdq $3, %rsi, %rdi
+; CHECK-NEXT: shrq $3, %rdi
+; CHECK-NEXT: shlq $61, %rsi
+; CHECK-NEXT: orq %rsi, %rdi
; CHECK-NEXT: movq %rdi, %rax
; CHECK-NEXT: shrq $32, %rax
; CHECK-NEXT: movq %rdi, %rdx
diff --git a/llvm/test/CodeGen/X86/fold-tied-op.ll b/llvm/test/CodeGen/X86/fold-tied-op.ll
index 99254ce3393ca..2fec5397ae627 100644
--- a/llvm/test/CodeGen/X86/fold-tied-op.ll
+++ b/llvm/test/CodeGen/X86/fold-tied-op.ll
@@ -24,92 +24,92 @@ define i64 @fn1() #0 {
; CHECK-NEXT: .cfi_offset %esi, -20
; CHECK-NEXT: .cfi_offset %edi, -16
; CHECK-NEXT: .cfi_offset %ebx, -12
-; CHECK-NEXT: movl $-1028477379, %ecx # imm = 0xC2B2AE3D
+; CHECK-NEXT: movl $-1028477379, %edi # imm = 0xC2B2AE3D
+; CHECK-NEXT: movl $668265295, %ecx # imm = 0x27D4EB4F
; CHECK-NEXT: movl a, %eax
; CHECK-NEXT: cmpl $0, (%eax)
; CHECK-NEXT: je .LBB0_2
; CHECK-NEXT: # %bb.1: # %if.then
-; CHECK-NEXT: movl %eax, %edi
-; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT: movl %eax, %esi
; CHECK-NEXT: movl 8(%eax), %eax
; CHECK-NEXT: leal (%eax,%eax), %edx
; CHECK-NEXT: orl %eax, %edx
; CHECK-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; CHECK-NEXT: shrl $31, %eax
-; CHECK-NEXT: movl 12(%edi), %esi
-; CHECK-NEXT: leal (,%esi,2), %ebx
-; CHECK-NEXT: orl %ebx, %eax
-; CHECK-NEXT: orl %esi, %eax
-; CHECK-NEXT: movl 16(%edi), %esi
+; CHECK-NEXT: movl %esi, %edx
; CHECK-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; CHECK-NEXT: shrl $30, %esi
-; CHECK-NEXT: movl 20(%edi), %ebx
+; CHECK-NEXT: movl 12(%esi), %esi
+; CHECK-NEXT: leal (,%esi,2), %edi
+; CHECK-NEXT: orl %edi, %eax
+; CHECK-NEXT: orl %esi, %eax
+; CHECK-NEXT: movl 16(%edx), %esi
+; CHECK-NEXT: movl %esi, %edi
+; CHECK-NEXT: shrl $30, %edi
+; CHECK-NEXT: movl 20(%edx), %ebx
; CHECK-NEXT: leal (,%ebx,4), %edx
-; CHECK-NEXT: orl %edx, %esi
-; CHECK-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; CHECK-NEXT: leal (,%edi,4), %edx
-; CHECK-NEXT: shrdl $1, %ebx, %edi
; CHECK-NEXT: orl %edx, %edi
+; CHECK-NEXT: leal (,%esi,4), %edx
+; CHECK-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT: shrl %esi
+; CHECK-NEXT: movl %ebx, %edx
+; CHECK-NEXT: shll $31, %edx
+; CHECK-NEXT: orl %esi, %edx
+; CHECK-NEXT: movl %edx, %esi
+; CHECK-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
; CHECK-NEXT: shrl %ebx
-; CHECK-NEXT: orl %esi, %ebx
-; CHECK-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; CHECK-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT: orl %edi, %ebx
+; CHECK-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; CHECK-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; CHECK-NEXT: adcl %eax, %ebx
-; CHECK-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; CHECK-NEXT: movl 24(%edi), %eax
+; CHECK-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; CHECK-NEXT: movl 24(%ebx), %eax
; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; CHECK-NEXT: imull %eax, %ecx
-; CHECK-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; CHECK-NEXT: movl $668265295, %ecx # imm = 0x27D4EB4F
+; CHECK-NEXT: movl $-1028477379, %edi # imm = 0xC2B2AE3D
+; CHECK-NEXT: imull %eax, %edi
; CHECK-NEXT: mull %ecx
; CHECK-NEXT: movl %eax, %esi
-; CHECK-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; CHECK-NEXT: movl 28(%edi), %eax
-; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; CHECK-NEXT: imull %eax, %ecx
+; CHECK-NEXT: addl %edi, %edx
+; CHECK-NEXT: movl 28(%ebx), %edi
+; CHECK-NEXT: imull %edi, %ecx
; CHECK-NEXT: addl %edx, %ecx
; CHECK-NEXT: movl $1336530590, %edx # imm = 0x4FA9D69E
-; CHECK-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; CHECK-NEXT: movl %edi, %eax
+; CHECK-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; CHECK-NEXT: movl %ebx, %eax
; CHECK-NEXT: mull %edx
-; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; CHECK-NEXT: imull $-2056954758, %edi, %eax # imm = 0x85655C7A
-; CHECK-NEXT: addl %edx, %eax
-; CHECK-NEXT: imull $1336530590, {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; CHECK-NEXT: # imm = 0x4FA9D69E
-; CHECK-NEXT: addl %eax, %edx
+; CHECK-NEXT: imull $-2056954758, %ebx, %ebx # imm = 0x85655C7A
+; CHECK-NEXT: addl %edx, %ebx
+; CHECK-NEXT: imull $1336530590, %edi, %edx # imm = 0x4FA9D69E
+; CHECK-NEXT: addl %ebx, %edx
; CHECK-NEXT: shrdl $3, %ecx, %esi
; CHECK-NEXT: sarl $3, %ecx
; CHECK-NEXT: orl %edx, %ecx
-; CHECK-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; CHECK-NEXT: movl $-66860409, %edx # imm = 0xFC03CA87
+; CHECK-NEXT: orl %eax, %esi
+; CHECK-NEXT: movl $-66860409, %ebx # imm = 0xFC03CA87
; CHECK-NEXT: movl %esi, %eax
-; CHECK-NEXT: mull %edx
+; CHECK-NEXT: mull %ebx
; CHECK-NEXT: movl %eax, %edi
; CHECK-NEXT: imull $326129324, %esi, %eax # imm = 0x137056AC
; CHECK-NEXT: addl %edx, %eax
; CHECK-NEXT: imull $-66860409, %ecx, %ecx # imm = 0xFC03CA87
; CHECK-NEXT: addl %eax, %ecx
-; CHECK-NEXT: xorl %ebx, %ecx
+; CHECK-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
; CHECK-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
; CHECK-NEXT: movl %edi, b
; CHECK-NEXT: movl %edi, %eax
-; CHECK-NEXT: movl $-66860409, %edx # imm = 0xFC03CA87
-; CHECK-NEXT: mull %edx
+; CHECK-NEXT: mull %ebx
; CHECK-NEXT: imull $326129324, %edi, %esi # imm = 0x137056AC
; CHECK-NEXT: addl %edx, %esi
; CHECK-NEXT: movl %ecx, b+4
; CHECK-NEXT: imull $-66860409, %ecx, %ecx # imm = 0xFC03CA87
; CHECK-NEXT: jmp .LBB0_3
; CHECK-NEXT: .LBB0_2: # %if.else
-; CHECK-NEXT: xorl b+4, %ecx
-; CHECK-NEXT: movl $668265295, %esi # imm = 0x27D4EB4F
-; CHECK-NEXT: xorl b, %esi
-; CHECK-NEXT: movl %ecx, %edi
-; CHECK-NEXT: movl $1419758215, %ecx # imm = 0x549FCA87
-; CHECK-NEXT: movl %esi, %eax
-; CHECK-NEXT: mull %ecx
-; CHECK-NEXT: imull $93298681, %esi, %esi # imm = 0x58F9FF9
+; CHECK-NEXT: xorl b+4, %edi
+; CHECK-NEXT: xorl b, %ecx
+; CHECK-NEXT: movl $1419758215, %edx # imm = 0x549FCA87
+; CHECK-NEXT: movl %ecx, %eax
+; CHECK-NEXT: mull %edx
+; CHECK-NEXT: imull $93298681, %ecx, %esi # imm = 0x58F9FF9
; CHECK-NEXT: addl %edx, %esi
; CHECK-NEXT: imull $1419758215, %edi, %ecx # imm = 0x549FCA87
; CHECK-NEXT: .LBB0_3: # %if.end
diff --git a/llvm/test/CodeGen/X86/known-bits.ll b/llvm/test/CodeGen/X86/known-bits.ll
index 58a0595e4322a..3d94b89ac8a61 100644
--- a/llvm/test/CodeGen/X86/known-bits.ll
+++ b/llvm/test/CodeGen/X86/known-bits.ll
@@ -101,7 +101,9 @@ define i128 @knownbits_mask_addc_shl(i64 %a0, i64 %a1, i64 %a2) nounwind {
; X86-NEXT: adcl {{[0-9]+}}(%esp), %edx
; X86-NEXT: adcl $0, %ecx
; X86-NEXT: shldl $22, %edx, %ecx
-; X86-NEXT: shldl $22, %esi, %edx
+; X86-NEXT: shrl $10, %esi
+; X86-NEXT: shll $22, %edx
+; X86-NEXT: orl %esi, %edx
; X86-NEXT: movl %edx, 8(%eax)
; X86-NEXT: movl %ecx, 12(%eax)
; X86-NEXT: movl $0, 4(%eax)
diff --git a/llvm/test/CodeGen/X86/midpoint-int.ll b/llvm/test/CodeGen/X86/midpoint-int.ll
index ffbb4bf06debc..3348c1693883c 100644
--- a/llvm/test/CodeGen/X86/midpoint-int.ll
+++ b/llvm/test/CodeGen/X86/midpoint-int.ll
@@ -303,25 +303,28 @@ define i64 @scalar_i64_signed_reg_reg(i64 %a1, i64 %a2) nounwind {
; X86-NEXT: pushl %ebx
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %ebp
; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %esi, %eax
-; X86-NEXT: subl %edx, %eax
+; X86-NEXT: movl %esi, %edx
+; X86-NEXT: subl %eax, %edx
; X86-NEXT: movl %ecx, %edi
; X86-NEXT: sbbl %ebp, %edi
-; X86-NEXT: subl %esi, %edx
+; X86-NEXT: subl %esi, %eax
; X86-NEXT: sbbl %ecx, %ebp
; X86-NEXT: setl %bl
; X86-NEXT: movzbl %bl, %ebx
; X86-NEXT: jl .LBB5_2
; X86-NEXT: # %bb.1:
; X86-NEXT: movl %ebp, %edi
-; X86-NEXT: movl %edx, %eax
+; X86-NEXT: movl %eax, %edx
; X86-NEXT: .LBB5_2:
; X86-NEXT: negl %ebx
-; X86-NEXT: shrdl $1, %edi, %eax
+; X86-NEXT: shrl %edx
+; X86-NEXT: movl %edi, %eax
+; X86-NEXT: shll $31, %eax
+; X86-NEXT: orl %edx, %eax
; X86-NEXT: shrl %edi
; X86-NEXT: movl %eax, %ebp
; X86-NEXT: imull %ebx, %ebp
@@ -439,26 +442,29 @@ define i64 @scalar_i64_signed_mem_reg(ptr %a1_addr, i64 %a2) nounwind {
; X86-NEXT: pushl %ebx
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ebp
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl (%eax), %esi
-; X86-NEXT: movl 4(%eax), %ecx
-; X86-NEXT: movl %esi, %eax
-; X86-NEXT: subl %edx, %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl (%ecx), %esi
+; X86-NEXT: movl 4(%ecx), %ecx
+; X86-NEXT: movl %esi, %edx
+; X86-NEXT: subl %eax, %edx
; X86-NEXT: movl %ecx, %edi
; X86-NEXT: sbbl %ebp, %edi
-; X86-NEXT: subl %esi, %edx
+; X86-NEXT: subl %esi, %eax
; X86-NEXT: sbbl %ecx, %ebp
; X86-NEXT: setl %bl
; X86-NEXT: movzbl %bl, %ebx
; X86-NEXT: jl .LBB7_2
; X86-NEXT: # %bb.1:
; X86-NEXT: movl %ebp, %edi
-; X86-NEXT: movl %edx, %eax
+; X86-NEXT: movl %eax, %edx
; X86-NEXT: .LBB7_2:
; X86-NEXT: negl %ebx
-; X86-NEXT: shrdl $1, %edi, %eax
+; X86-NEXT: shrl %edx
+; X86-NEXT: movl %edi, %eax
+; X86-NEXT: shll $31, %eax
+; X86-NEXT: orl %edx, %eax
; X86-NEXT: shrl %edi
; X86-NEXT: movl %eax, %ebp
; X86-NEXT: imull %ebx, %ebp
@@ -510,24 +516,27 @@ define i64 @scalar_i64_signed_reg_mem(i64 %a1, ptr %a2_addr) nounwind {
; X86-NEXT: pushl %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl (%eax), %edx
-; X86-NEXT: movl 4(%eax), %ebp
-; X86-NEXT: movl %esi, %eax
-; X86-NEXT: subl %edx, %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movl (%edx), %eax
+; X86-NEXT: movl 4(%edx), %ebp
+; X86-NEXT: movl %esi, %edx
+; X86-NEXT: subl %eax, %edx
; X86-NEXT: movl %ecx, %edi
; X86-NEXT: sbbl %ebp, %edi
-; X86-NEXT: subl %esi, %edx
+; X86-NEXT: subl %esi, %eax
; X86-NEXT: sbbl %ecx, %ebp
; X86-NEXT: setl %bl
; X86-NEXT: movzbl %bl, %ebx
; X86-NEXT: jl .LBB8_2
; X86-NEXT: # %bb.1:
; X86-NEXT: movl %ebp, %edi
-; X86-NEXT: movl %edx, %eax
+; X86-NEXT: movl %eax, %edx
; X86-NEXT: .LBB8_2:
; X86-NEXT: negl %ebx
-; X86-NEXT: shrdl $1, %edi, %eax
+; X86-NEXT: shrl %edx
+; X86-NEXT: movl %edi, %eax
+; X86-NEXT: shll $31, %eax
+; X86-NEXT: orl %edx, %eax
; X86-NEXT: shrl %edi
; X86-NEXT: movl %eax, %ebp
; X86-NEXT: imull %ebx, %ebp
@@ -578,27 +587,30 @@ define i64 @scalar_i64_signed_mem_mem(ptr %a1_addr, ptr %a2_addr) nounwind {
; X86-NEXT: pushl %ebx
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl (%ecx), %esi
-; X86-NEXT: movl 4(%ecx), %ecx
-; X86-NEXT: movl (%eax), %edx
-; X86-NEXT: movl 4(%eax), %ebp
-; X86-NEXT: movl %esi, %eax
-; X86-NEXT: subl %edx, %eax
+; X86-NEXT: movl (%eax), %esi
+; X86-NEXT: movl 4(%eax), %ecx
+; X86-NEXT: movl (%edx), %eax
+; X86-NEXT: movl 4(%edx), %ebp
+; X86-NEXT: movl %esi, %edx
+; X86-NEXT: subl %eax, %edx
; X86-NEXT: movl %ecx, %edi
; X86-NEXT: sbbl %ebp, %edi
-; X86-NEXT: subl %esi, %edx
+; X86-NEXT: subl %esi, %eax
; X86-NEXT: sbbl %ecx, %ebp
; X86-NEXT: setl %bl
; X86-NEXT: movzbl %bl, %ebx
; X86-NEXT: jl .LBB9_2
; X86-NEXT: # %bb.1:
; X86-NEXT: movl %ebp, %edi
-; X86-NEXT: movl %edx, %eax
+; X86-NEXT: movl %eax, %edx
; X86-NEXT: .LBB9_2:
; X86-NEXT: negl %ebx
-; X86-NEXT: shrdl $1, %edi, %eax
+; X86-NEXT: shrl %edx
+; X86-NEXT: movl %edi, %eax
+; X86-NEXT: shll $31, %eax
+; X86-NEXT: orl %edx, %eax
; X86-NEXT: shrl %edi
; X86-NEXT: movl %eax, %ebp
; X86-NEXT: imull %ebx, %ebp
diff --git a/llvm/test/CodeGen/X86/pr32282.ll b/llvm/test/CodeGen/X86/pr32282.ll
index a5bb7316a9673..54b8b5df0e415 100644
--- a/llvm/test/CodeGen/X86/pr32282.ll
+++ b/llvm/test/CodeGen/X86/pr32282.ll
@@ -13,17 +13,20 @@ define dso_local void @foo(i64 %x) nounwind {
; X86-LABEL: foo:
; X86: # %bb.0:
; X86-NEXT: pushl %eax
-; X86-NEXT: movl d, %eax
-; X86-NEXT: notl %eax
-; X86-NEXT: movl d+4, %ecx
+; X86-NEXT: movl d, %ecx
; X86-NEXT: notl %ecx
-; X86-NEXT: andl $701685459, %ecx # imm = 0x29D2DED3
-; X86-NEXT: andl $-566231040, %eax # imm = 0xDE400000
-; X86-NEXT: shrdl $21, %ecx, %eax
+; X86-NEXT: movl d+4, %eax
+; X86-NEXT: notl %eax
+; X86-NEXT: andl $701685459, %eax # imm = 0x29D2DED3
+; X86-NEXT: andl $-566231040, %ecx # imm = 0xDE400000
; X86-NEXT: shrl $21, %ecx
-; X86-NEXT: addl $7, %eax
-; X86-NEXT: pushl %ecx
+; X86-NEXT: movl %eax, %edx
+; X86-NEXT: shll $11, %edx
+; X86-NEXT: orl %ecx, %edx
+; X86-NEXT: shrl $21, %eax
+; X86-NEXT: addl $7, %edx
; X86-NEXT: pushl %eax
+; X86-NEXT: pushl %edx
; X86-NEXT: pushl {{[0-9]+}}(%esp)
; X86-NEXT: pushl {{[0-9]+}}(%esp)
; X86-NEXT: calll __divdi3
diff --git a/llvm/test/CodeGen/X86/pr43820.ll b/llvm/test/CodeGen/X86/pr43820.ll
index 10270d7ae430f..3c260af2e504a 100644
--- a/llvm/test/CodeGen/X86/pr43820.ll
+++ b/llvm/test/CodeGen/X86/pr43820.ll
@@ -10,222 +10,224 @@ define i1000 @square(i1000 %A) nounwind {
; CHECK-NEXT: pushq %r13
; CHECK-NEXT: pushq %r12
; CHECK-NEXT: pushq %rbx
-; CHECK-NEXT: movq %rcx, %r11
-; CHECK-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: movq %rdi, %rsi
+; CHECK-NEXT: movq %rdx, %rbx
+; CHECK-NEXT: movq %rsi, %rbp
+; CHECK-NEXT: movq %rdi, %r12
; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rdi
; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r14
; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r15
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rbx
-; CHECK-NEXT: bswapq %rbx
-; CHECK-NEXT: movq %rbx, %r10
-; CHECK-NEXT: shrq $4, %r10
-; CHECK-NEXT: movabsq $1085102592571150095, %rbp # imm = 0xF0F0F0F0F0F0F0F
-; CHECK-NEXT: andq %rbp, %r10
-; CHECK-NEXT: andq %rbp, %rbx
-; CHECK-NEXT: shlq $4, %rbx
-; CHECK-NEXT: orq %r10, %rbx
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; CHECK-NEXT: bswapq %rax
+; CHECK-NEXT: movq %rax, %rsi
+; CHECK-NEXT: shrq $4, %rsi
+; CHECK-NEXT: movabsq $1085102592571150095, %r11 # imm = 0xF0F0F0F0F0F0F0F
+; CHECK-NEXT: andq %r11, %rsi
+; CHECK-NEXT: andq %r11, %rax
+; CHECK-NEXT: shlq $4, %rax
+; CHECK-NEXT: orq %rsi, %rax
; CHECK-NEXT: movabsq $3689348814741910323, %r10 # imm = 0x3333333333333333
-; CHECK-NEXT: movq %rbx, %r12
-; CHECK-NEXT: andq %r10, %r12
-; CHECK-NEXT: shrq $2, %rbx
-; CHECK-NEXT: andq %r10, %rbx
-; CHECK-NEXT: leaq (%rbx,%r12,4), %r12
-; CHECK-NEXT: movabsq $6148914691236517205, %rbx # imm = 0x5555555555555555
-; CHECK-NEXT: movq %r12, %r13
-; CHECK-NEXT: andq %rbx, %r13
-; CHECK-NEXT: shrq %r12
-; CHECK-NEXT: andq %rbx, %r12
-; CHECK-NEXT: leaq (%r12,%r13,2), %rcx
+; CHECK-NEXT: movq %rax, %rsi
+; CHECK-NEXT: andq %r10, %rsi
+; CHECK-NEXT: shrq $2, %rax
+; CHECK-NEXT: andq %r10, %rax
+; CHECK-NEXT: leaq (%rax,%rsi,4), %rax
+; CHECK-NEXT: movabsq $6148914691236517205, %rsi # imm = 0x5555555555555555
+; CHECK-NEXT: movq %rax, %r13
+; CHECK-NEXT: andq %rsi, %r13
+; CHECK-NEXT: shrq %rax
+; CHECK-NEXT: andq %rsi, %rax
+; CHECK-NEXT: leaq (%rax,%r13,2), %rax
+; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-NEXT: bswapq %r15
-; CHECK-NEXT: movq %r15, %r12
-; CHECK-NEXT: shrq $4, %r12
-; CHECK-NEXT: andq %rbp, %r12
-; CHECK-NEXT: andq %rbp, %r15
+; CHECK-NEXT: movq %r15, %rax
+; CHECK-NEXT: shrq $4, %rax
+; CHECK-NEXT: andq %r11, %rax
+; CHECK-NEXT: andq %r11, %r15
; CHECK-NEXT: shlq $4, %r15
-; CHECK-NEXT: orq %r12, %r15
-; CHECK-NEXT: movq %r15, %r12
-; CHECK-NEXT: andq %r10, %r12
+; CHECK-NEXT: orq %rax, %r15
+; CHECK-NEXT: movq %r15, %rax
+; CHECK-NEXT: andq %r10, %rax
; CHECK-NEXT: shrq $2, %r15
; CHECK-NEXT: andq %r10, %r15
-; CHECK-NEXT: leaq (%r15,%r12,4), %r15
-; CHECK-NEXT: movabsq $6148914691230924800, %r12 # imm = 0x5555555555000000
-; CHECK-NEXT: movq %r15, %r13
-; CHECK-NEXT: andq %r12, %r13
-; CHECK-NEXT: shrq %r15
-; CHECK-NEXT: andq %r12, %r15
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r12
-; CHECK-NEXT: leaq (%r15,%r13,2), %rax
+; CHECK-NEXT: leaq (%r15,%rax,4), %rax
+; CHECK-NEXT: movabsq $6148914691230924800, %r15 # imm = 0x5555555555000000
+; CHECK-NEXT: movq %rax, %r13
+; CHECK-NEXT: andq %r15, %r13
+; CHECK-NEXT: shrq %rax
+; CHECK-NEXT: andq %r15, %rax
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r15
+; CHECK-NEXT: leaq (%rax,%r13,2), %rax
; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: bswapq %r12
-; CHECK-NEXT: movq %r12, %r15
-; CHECK-NEXT: shrq $4, %r15
-; CHECK-NEXT: andq %rbp, %r15
-; CHECK-NEXT: andq %rbp, %r12
-; CHECK-NEXT: shlq $4, %r12
-; CHECK-NEXT: orq %r15, %r12
-; CHECK-NEXT: movq %r12, %r15
+; CHECK-NEXT: bswapq %r15
+; CHECK-NEXT: movq %r15, %rax
+; CHECK-NEXT: shrq $4, %rax
+; CHECK-NEXT: andq %r11, %rax
+; CHECK-NEXT: andq %r11, %r15
+; CHECK-NEXT: shlq $4, %r15
+; CHECK-NEXT: orq %rax, %r15
+; CHECK-NEXT: movq %r15, %rax
+; CHECK-NEXT: andq %r10, %rax
+; CHECK-NEXT: shrq $2, %r15
; CHECK-NEXT: andq %r10, %r15
-; CHECK-NEXT: shrq $2, %r12
-; CHECK-NEXT: andq %r10, %r12
-; CHECK-NEXT: leaq (%r12,%r15,4), %r15
-; CHECK-NEXT: movq %r15, %r12
-; CHECK-NEXT: andq %rbx, %r12
-; CHECK-NEXT: shrq %r15
-; CHECK-NEXT: andq %rbx, %r15
-; CHECK-NEXT: leaq (%r15,%r12,2), %rax
+; CHECK-NEXT: leaq (%r15,%rax,4), %rax
+; CHECK-NEXT: movq %rax, %r15
+; CHECK-NEXT: andq %rsi, %r15
+; CHECK-NEXT: shrq %rax
+; CHECK-NEXT: andq %rsi, %rax
+; CHECK-NEXT: leaq (%rax,%r15,2), %rax
; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-NEXT: bswapq %r14
-; CHECK-NEXT: movq %r14, %r15
-; CHECK-NEXT: shrq $4, %r15
-; CHECK-NEXT: andq %rbp, %r15
-; CHECK-NEXT: andq %rbp, %r14
+; CHECK-NEXT: movq %r14, %rax
+; CHECK-NEXT: shrq $4, %rax
+; CHECK-NEXT: andq %r11, %rax
+; CHECK-NEXT: andq %r11, %r14
; CHECK-NEXT: shlq $4, %r14
-; CHECK-NEXT: orq %r15, %r14
-; CHECK-NEXT: movq %r14, %r15
-; CHECK-NEXT: andq %r10, %r15
+; CHECK-NEXT: orq %rax, %r14
+; CHECK-NEXT: movq %r14, %rax
+; CHECK-NEXT: andq %r10, %rax
; CHECK-NEXT: shrq $2, %r14
; CHECK-NEXT: andq %r10, %r14
-; CHECK-NEXT: leaq (%r14,%r15,4), %r14
-; CHECK-NEXT: movq %r14, %r15
-; CHECK-NEXT: andq %rbx, %r15
-; CHECK-NEXT: shrq %r14
-; CHECK-NEXT: andq %rbx, %r14
-; CHECK-NEXT: leaq (%r14,%r15,2), %rax
+; CHECK-NEXT: leaq (%r14,%rax,4), %rax
+; CHECK-NEXT: movq %rax, %r14
+; CHECK-NEXT: andq %rsi, %r14
+; CHECK-NEXT: shrq %rax
+; CHECK-NEXT: andq %rsi, %rax
+; CHECK-NEXT: leaq (%rax,%r14,2), %rax
; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-NEXT: bswapq %rdi
-; CHECK-NEXT: movq %rdi, %r14
-; CHECK-NEXT: shrq $4, %r14
-; CHECK-NEXT: andq %rbp, %r14
-; CHECK-NEXT: andq %rbp, %rdi
+; CHECK-NEXT: movq %rdi, %rax
+; CHECK-NEXT: shrq $4, %rax
+; CHECK-NEXT: andq %r11, %rax
+; CHECK-NEXT: andq %r11, %rdi
; CHECK-NEXT: shlq $4, %rdi
-; CHECK-NEXT: orq %r14, %rdi
-; CHECK-NEXT: movq %rdi, %r14
-; CHECK-NEXT: andq %r10, %r14
+; CHECK-NEXT: orq %rax, %rdi
+; CHECK-NEXT: movq %rdi, %rax
+; CHECK-NEXT: andq %r10, %rax
; CHECK-NEXT: shrq $2, %rdi
; CHECK-NEXT: andq %r10, %rdi
-; CHECK-NEXT: leaq (%rdi,%r14,4), %rdi
-; CHECK-NEXT: movq %rdi, %r14
-; CHECK-NEXT: andq %rbx, %r14
-; CHECK-NEXT: shrq %rdi
-; CHECK-NEXT: andq %rbx, %rdi
-; CHECK-NEXT: leaq (%rdi,%r14,2), %rax
+; CHECK-NEXT: leaq (%rdi,%rax,4), %rax
+; CHECK-NEXT: movq %rax, %rdi
+; CHECK-NEXT: andq %rsi, %rdi
+; CHECK-NEXT: shrq %rax
+; CHECK-NEXT: andq %rsi, %rax
+; CHECK-NEXT: leaq (%rax,%rdi,2), %rax
; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rdi
-; CHECK-NEXT: bswapq %rdi
-; CHECK-NEXT: movq %rdi, %r14
-; CHECK-NEXT: shrq $4, %r14
-; CHECK-NEXT: andq %rbp, %r14
-; CHECK-NEXT: andq %rbp, %rdi
-; CHECK-NEXT: shlq $4, %rdi
-; CHECK-NEXT: orq %r14, %rdi
-; CHECK-NEXT: movq %rdi, %r14
-; CHECK-NEXT: andq %r10, %r14
-; CHECK-NEXT: shrq $2, %rdi
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; CHECK-NEXT: bswapq %rax
+; CHECK-NEXT: movq %rax, %rdi
+; CHECK-NEXT: shrq $4, %rdi
+; CHECK-NEXT: andq %r11, %rdi
+; CHECK-NEXT: andq %r11, %rax
+; CHECK-NEXT: shlq $4, %rax
+; CHECK-NEXT: orq %rdi, %rax
+; CHECK-NEXT: movq %rax, %rdi
; CHECK-NEXT: andq %r10, %rdi
-; CHECK-NEXT: leaq (%rdi,%r14,4), %rdi
-; CHECK-NEXT: movq %rdi, %r14
-; CHECK-NEXT: andq %rbx, %r14
-; CHECK-NEXT: shrq %rdi
-; CHECK-NEXT: andq %rbx, %rdi
-; CHECK-NEXT: leaq (%rdi,%r14,2), %rax
+; CHECK-NEXT: shrq $2, %rax
+; CHECK-NEXT: andq %r10, %rax
+; CHECK-NEXT: leaq (%rax,%rdi,4), %rax
+; CHECK-NEXT: movq %rax, %rdi
+; CHECK-NEXT: andq %rsi, %rdi
+; CHECK-NEXT: shrq %rax
+; CHECK-NEXT: andq %rsi, %rax
+; CHECK-NEXT: leaq (%rax,%rdi,2), %rax
; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rdi
-; CHECK-NEXT: bswapq %rdi
-; CHECK-NEXT: movq %rdi, %r14
-; CHECK-NEXT: shrq $4, %r14
-; CHECK-NEXT: andq %rbp, %r14
-; CHECK-NEXT: andq %rbp, %rdi
-; CHECK-NEXT: shlq $4, %rdi
-; CHECK-NEXT: orq %r14, %rdi
-; CHECK-NEXT: movq %rdi, %r14
-; CHECK-NEXT: andq %r10, %r14
-; CHECK-NEXT: shrq $2, %rdi
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; CHECK-NEXT: bswapq %rax
+; CHECK-NEXT: movq %rax, %rdi
+; CHECK-NEXT: shrq $4, %rdi
+; CHECK-NEXT: andq %r11, %rdi
+; CHECK-NEXT: andq %r11, %rax
+; CHECK-NEXT: shlq $4, %rax
+; CHECK-NEXT: orq %rdi, %rax
+; CHECK-NEXT: movq %rax, %rdi
; CHECK-NEXT: andq %r10, %rdi
-; CHECK-NEXT: leaq (%rdi,%r14,4), %rdi
-; CHECK-NEXT: movq %rdi, %r14
-; CHECK-NEXT: andq %rbx, %r14
-; CHECK-NEXT: shrq %rdi
-; CHECK-NEXT: andq %rbx, %rdi
-; CHECK-NEXT: leaq (%rdi,%r14,2), %rax
+; CHECK-NEXT: shrq $2, %rax
+; CHECK-NEXT: andq %r10, %rax
+; CHECK-NEXT: leaq (%rax,%rdi,4), %rax
+; CHECK-NEXT: movq %rax, %rdi
+; CHECK-NEXT: andq %rsi, %rdi
+; CHECK-NEXT: shrq %rax
+; CHECK-NEXT: andq %rsi, %rax
+; CHECK-NEXT: leaq (%rax,%rdi,2), %rax
; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rdi
-; CHECK-NEXT: bswapq %rdi
-; CHECK-NEXT: movq %rdi, %r14
-; CHECK-NEXT: shrq $4, %r14
-; CHECK-NEXT: andq %rbp, %r14
-; CHECK-NEXT: andq %rbp, %rdi
-; CHECK-NEXT: shlq $4, %rdi
-; CHECK-NEXT: orq %r14, %rdi
-; CHECK-NEXT: movq %rdi, %r14
-; CHECK-NEXT: andq %r10, %r14
-; CHECK-NEXT: shrq $2, %rdi
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; CHECK-NEXT: bswapq %rax
+; CHECK-NEXT: movq %rax, %rdi
+; CHECK-NEXT: shrq $4, %rdi
+; CHECK-NEXT: andq %r11, %rdi
+; CHECK-NEXT: andq %r11, %rax
+; CHECK-NEXT: shlq $4, %rax
+; CHECK-NEXT: orq %rdi, %rax
+; CHECK-NEXT: movq %rax, %rdi
; CHECK-NEXT: andq %r10, %rdi
-; CHECK-NEXT: leaq (%rdi,%r14,4), %rdi
-; CHECK-NEXT: movq %rdi, %r14
-; CHECK-NEXT: andq %rbx, %r14
-; CHECK-NEXT: shrq %rdi
-; CHECK-NEXT: andq %rbx, %rdi
-; CHECK-NEXT: leaq (%rdi,%r14,2), %r13
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rdi
-; CHECK-NEXT: bswapq %rdi
-; CHECK-NEXT: movq %rdi, %r14
-; CHECK-NEXT: shrq $4, %r14
-; CHECK-NEXT: andq %rbp, %r14
-; CHECK-NEXT: andq %rbp, %rdi
-; CHECK-NEXT: shlq $4, %rdi
-; CHECK-NEXT: orq %r14, %rdi
-; CHECK-NEXT: movq %rdi, %r14
-; CHECK-NEXT: andq %r10, %r14
-; CHECK-NEXT: shrq $2, %rdi
+; CHECK-NEXT: shrq $2, %rax
+; CHECK-NEXT: andq %r10, %rax
+; CHECK-NEXT: leaq (%rax,%rdi,4), %rax
+; CHECK-NEXT: movq %rax, %rdi
+; CHECK-NEXT: andq %rsi, %rdi
+; CHECK-NEXT: shrq %rax
+; CHECK-NEXT: andq %rsi, %rax
+; CHECK-NEXT: leaq (%rax,%rdi,2), %rax
+; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; CHECK-NEXT: bswapq %rax
+; CHECK-NEXT: movq %rax, %rdi
+; CHECK-NEXT: shrq $4, %rdi
+; CHECK-NEXT: andq %r11, %rdi
+; CHECK-NEXT: andq %r11, %rax
+; CHECK-NEXT: shlq $4, %rax
+; CHECK-NEXT: orq %rdi, %rax
+; CHECK-NEXT: movq %rax, %rdi
; CHECK-NEXT: andq %r10, %rdi
-; CHECK-NEXT: leaq (%rdi,%r14,4), %rdi
-; CHECK-NEXT: movq %rdi, %r14
-; CHECK-NEXT: andq %rbx, %r14
-; CHECK-NEXT: shrq %rdi
-; CHECK-NEXT: andq %rbx, %rdi
-; CHECK-NEXT: leaq (%rdi,%r14,2), %r12
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rdi
-; CHECK-NEXT: bswapq %rdi
-; CHECK-NEXT: movq %rdi, %r14
-; CHECK-NEXT: shrq $4, %r14
-; CHECK-NEXT: andq %rbp, %r14
-; CHECK-NEXT: andq %rbp, %rdi
-; CHECK-NEXT: shlq $4, %rdi
-; CHECK-NEXT: orq %r14, %rdi
-; CHECK-NEXT: movq %rdi, %r14
-; CHECK-NEXT: andq %r10, %r14
-; CHECK-NEXT: shrq $2, %rdi
+; CHECK-NEXT: shrq $2, %rax
+; CHECK-NEXT: andq %r10, %rax
+; CHECK-NEXT: leaq (%rax,%rdi,4), %rax
+; CHECK-NEXT: movq %rax, %rdi
+; CHECK-NEXT: andq %rsi, %rdi
+; CHECK-NEXT: shrq %rax
+; CHECK-NEXT: andq %rsi, %rax
+; CHECK-NEXT: leaq (%rax,%rdi,2), %r13
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; CHECK-NEXT: bswapq %rax
+; CHECK-NEXT: movq %rax, %rdi
+; CHECK-NEXT: shrq $4, %rdi
+; CHECK-NEXT: andq %r11, %rdi
+; CHECK-NEXT: andq %r11, %rax
+; CHECK-NEXT: shlq $4, %rax
+; CHECK-NEXT: orq %rdi, %rax
+; CHECK-NEXT: movq %rax, %rdi
; CHECK-NEXT: andq %r10, %rdi
-; CHECK-NEXT: leaq (%rdi,%r14,4), %rdi
-; CHECK-NEXT: movq %rdi, %r14
-; CHECK-NEXT: andq %rbx, %r14
-; CHECK-NEXT: shrq %rdi
-; CHECK-NEXT: andq %rbx, %rdi
-; CHECK-NEXT: leaq (%rdi,%r14,2), %r15
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rdi
-; CHECK-NEXT: bswapq %rdi
-; CHECK-NEXT: movq %rdi, %rax
-; CHECK-NEXT: shrq $4, %rax
-; CHECK-NEXT: andq %rbp, %rax
-; CHECK-NEXT: andq %rbp, %rdi
-; CHECK-NEXT: shlq $4, %rdi
-; CHECK-NEXT: orq %rax, %rdi
-; CHECK-NEXT: movq %rdi, %rax
+; CHECK-NEXT: shrq $2, %rax
; CHECK-NEXT: andq %r10, %rax
-; CHECK-NEXT: shrq $2, %rdi
+; CHECK-NEXT: leaq (%rax,%rdi,4), %rax
+; CHECK-NEXT: movq %rax, %rdi
+; CHECK-NEXT: andq %rsi, %rdi
+; CHECK-NEXT: shrq %rax
+; CHECK-NEXT: andq %rsi, %rax
+; CHECK-NEXT: leaq (%rax,%rdi,2), %r15
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; CHECK-NEXT: bswapq %rax
+; CHECK-NEXT: movq %rax, %rdi
+; CHECK-NEXT: shrq $4, %rdi
+; CHECK-NEXT: andq %r11, %rdi
+; CHECK-NEXT: andq %r11, %rax
+; CHECK-NEXT: shlq $4, %rax
+; CHECK-NEXT: orq %rdi, %rax
+; CHECK-NEXT: movq %rax, %rdi
; CHECK-NEXT: andq %r10, %rdi
-; CHECK-NEXT: leaq (%rdi,%rax,4), %rax
+; CHECK-NEXT: shrq $2, %rax
+; CHECK-NEXT: andq %r10, %rax
+; CHECK-NEXT: leaq (%rax,%rdi,4), %rax
; CHECK-NEXT: movq %rax, %rdi
-; CHECK-NEXT: andq %rbx, %rdi
+; CHECK-NEXT: andq %rsi, %rdi
; CHECK-NEXT: shrq %rax
-; CHECK-NEXT: andq %rbx, %rax
+; CHECK-NEXT: andq %rsi, %rax
; CHECK-NEXT: leaq (%rax,%rdi,2), %r14
; CHECK-NEXT: bswapq %r9
; CHECK-NEXT: movq %r9, %rax
; CHECK-NEXT: shrq $4, %rax
-; CHECK-NEXT: andq %rbp, %rax
-; CHECK-NEXT: andq %rbp, %r9
+; CHECK-NEXT: andq %r11, %rax
+; CHECK-NEXT: andq %r11, %r9
; CHECK-NEXT: shlq $4, %r9
; CHECK-NEXT: orq %rax, %r9
; CHECK-NEXT: movq %r9, %rax
@@ -234,15 +236,15 @@ define i1000 @square(i1000 %A) nounwind {
; CHECK-NEXT: andq %r10, %r9
; CHECK-NEXT: leaq (%r9,%rax,4), %rax
; CHECK-NEXT: movq %rax, %r9
-; CHECK-NEXT: andq %rbx, %r9
+; CHECK-NEXT: andq %rsi, %r9
; CHECK-NEXT: shrq %rax
-; CHECK-NEXT: andq %rbx, %rax
+; CHECK-NEXT: andq %rsi, %rax
; CHECK-NEXT: leaq (%rax,%r9,2), %r9
; CHECK-NEXT: bswapq %r8
; CHECK-NEXT: movq %r8, %rax
; CHECK-NEXT: shrq $4, %rax
-; CHECK-NEXT: andq %rbp, %rax
-; CHECK-NEXT: andq %rbp, %r8
+; CHECK-NEXT: andq %r11, %rax
+; CHECK-NEXT: andq %r11, %r8
; CHECK-NEXT: shlq $4, %r8
; CHECK-NEXT: orq %rax, %r8
; CHECK-NEXT: movq %r8, %rax
@@ -251,123 +253,131 @@ define i1000 @square(i1000 %A) nounwind {
; CHECK-NEXT: andq %r10, %r8
; CHECK-NEXT: leaq (%r8,%rax,4), %rax
; CHECK-NEXT: movq %rax, %r8
-; CHECK-NEXT: andq %rbx, %r8
+; CHECK-NEXT: andq %rsi, %r8
; CHECK-NEXT: shrq %rax
-; CHECK-NEXT: andq %rbx, %rax
-; CHECK-NEXT: leaq (%rax,%r8,2), %r8
-; CHECK-NEXT: movq %rcx, %rax
+; CHECK-NEXT: andq %rsi, %rax
+; CHECK-NEXT: leaq (%rax,%r8,2), %rdx
+; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
-; CHECK-NEXT: shrdq $24, %rcx, %rdi
-; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; CHECK-NEXT: shrdq $24, %rcx, %rax
+; CHECK-NEXT: shrdq $24, %r8, %rdi
+; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; CHECK-NEXT: shrdq $24, %rax, %r8
+; CHECK-NEXT: movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
+; CHECK-NEXT: shrdq $24, %r8, %rax
; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; CHECK-NEXT: shrdq $24, %rax, %rcx
-; CHECK-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; CHECK-NEXT: shrdq $24, %rcx, %rax
+; CHECK-NEXT: shrdq $24, %rax, %r8
+; CHECK-NEXT: movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
+; CHECK-NEXT: shrdq $24, %r8, %rax
; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; CHECK-NEXT: shrdq $24, %rax, %rcx
-; CHECK-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; CHECK-NEXT: shrdq $24, %rcx, %rax
+; CHECK-NEXT: shrdq $24, %rax, %r8
+; CHECK-NEXT: movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
+; CHECK-NEXT: shrdq $24, %r8, %rax
; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: shrdq $24, %r13, %rcx
-; CHECK-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: shrdq $24, %r12, %r13
-; CHECK-NEXT: shrdq $24, %r15, %r12
+; CHECK-NEXT: shrdq $24, %r13, %r8
+; CHECK-NEXT: movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT: shrdq $24, %r15, %r13
; CHECK-NEXT: shrdq $24, %r14, %r15
; CHECK-NEXT: shrdq $24, %r9, %r14
-; CHECK-NEXT: movq %r8, %rax
-; CHECK-NEXT: shlq $40, %rax
+; CHECK-NEXT: movq %rdx, %r8
+; CHECK-NEXT: shlq $40, %r8
; CHECK-NEXT: shrq $24, %r9
-; CHECK-NEXT: orq %rax, %r9
-; CHECK-NEXT: bswapq %r11
-; CHECK-NEXT: movq %r11, %rax
-; CHECK-NEXT: shrq $4, %rax
-; CHECK-NEXT: andq %rbp, %rax
-; CHECK-NEXT: andq %rbp, %r11
-; CHECK-NEXT: shlq $4, %r11
-; CHECK-NEXT: orq %rax, %r11
-; CHECK-NEXT: movq %r11, %rax
-; CHECK-NEXT: andq %r10, %rax
-; CHECK-NEXT: shrq $2, %r11
-; CHECK-NEXT: andq %r10, %r11
-; CHECK-NEXT: leaq (%r11,%rax,4), %rax
-; CHECK-NEXT: movq %rax, %rcx
-; CHECK-NEXT: andq %rbx, %rcx
-; CHECK-NEXT: shrq %rax
-; CHECK-NEXT: andq %rbx, %rax
-; CHECK-NEXT: leaq (%rax,%rcx,2), %rcx
-; CHECK-NEXT: bswapq %rdx
-; CHECK-NEXT: movq %rdx, %rax
+; CHECK-NEXT: orq %r8, %r9
+; CHECK-NEXT: bswapq %rcx
+; CHECK-NEXT: movq %rcx, %r8
+; CHECK-NEXT: shrq $4, %r8
+; CHECK-NEXT: andq %r11, %r8
+; CHECK-NEXT: andq %r11, %rcx
+; CHECK-NEXT: shlq $4, %rcx
+; CHECK-NEXT: orq %r8, %rcx
+; CHECK-NEXT: movq %rcx, %r8
+; CHECK-NEXT: andq %r10, %r8
+; CHECK-NEXT: shrq $2, %rcx
+; CHECK-NEXT: andq %r10, %rcx
+; CHECK-NEXT: leaq (%rcx,%r8,4), %rcx
+; CHECK-NEXT: movq %rcx, %r8
+; CHECK-NEXT: andq %rsi, %r8
+; CHECK-NEXT: shrq %rcx
+; CHECK-NEXT: andq %rsi, %rcx
+; CHECK-NEXT: leaq (%rcx,%r8,2), %rcx
+; CHECK-NEXT: shrq $24, %rdx
+; CHECK-NEXT: movq %rcx, %r8
+; CHECK-NEXT: shlq $40, %r8
+; CHECK-NEXT: orq %rdx, %r8
+; CHECK-NEXT: bswapq %rbx
+; CHECK-NEXT: movq %rbx, %rax
; CHECK-NEXT: shrq $4, %rax
-; CHECK-NEXT: andq %rbp, %rax
-; CHECK-NEXT: andq %rbp, %rdx
-; CHECK-NEXT: shlq $4, %rdx
-; CHECK-NEXT: orq %rax, %rdx
-; CHECK-NEXT: movq %rdx, %rax
+; CHECK-NEXT: andq %r11, %rax
+; CHECK-NEXT: andq %r11, %rbx
+; CHECK-NEXT: shlq $4, %rbx
+; CHECK-NEXT: orq %rax, %rbx
+; CHECK-NEXT: movq %rbx, %rax
; CHECK-NEXT: andq %r10, %rax
-; CHECK-NEXT: shrq $2, %rdx
-; CHECK-NEXT: andq %r10, %rdx
-; CHECK-NEXT: leaq (%rdx,%rax,4), %rax
-; CHECK-NEXT: movq %rax, %rdx
-; CHECK-NEXT: andq %rbx, %rdx
+; CHECK-NEXT: shrq $2, %rbx
+; CHECK-NEXT: andq %r10, %rbx
+; CHECK-NEXT: leaq (%rbx,%rax,4), %rax
+; CHECK-NEXT: movq %rax, %rbx
+; CHECK-NEXT: andq %rsi, %rbx
; CHECK-NEXT: shrq %rax
-; CHECK-NEXT: andq %rbx, %rax
-; CHECK-NEXT: leaq (%rax,%rdx,2), %rdx
-; CHECK-NEXT: shrdq $24, %rcx, %r8
-; CHECK-NEXT: movq %rdx, %rax
-; CHECK-NEXT: shlq $40, %rax
+; CHECK-NEXT: andq %rsi, %rax
+; CHECK-NEXT: leaq (%rax,%rbx,2), %rax
+; CHECK-NEXT: movq %rax, %rbx
+; CHECK-NEXT: shlq $40, %rbx
; CHECK-NEXT: shrq $24, %rcx
-; CHECK-NEXT: orq %rax, %rcx
-; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Reload
-; CHECK-NEXT: bswapq %r11
-; CHECK-NEXT: movq %r11, %rax
-; CHECK-NEXT: shrq $4, %rax
-; CHECK-NEXT: andq %rbp, %rax
-; CHECK-NEXT: andq %rbp, %r11
-; CHECK-NEXT: shlq $4, %r11
-; CHECK-NEXT: orq %rax, %r11
-; CHECK-NEXT: movq %r11, %rax
-; CHECK-NEXT: andq %r10, %rax
-; CHECK-NEXT: shrq $2, %r11
+; CHECK-NEXT: orq %rbx, %rcx
+; CHECK-NEXT: bswapq %rbp
+; CHECK-NEXT: movq %rbp, %rbx
+; CHECK-NEXT: shrq $4, %rbx
+; CHECK-NEXT: andq %r11, %rbx
+; CHECK-NEXT: andq %r11, %rbp
+; CHECK-NEXT: shlq $4, %rbp
+; CHECK-NEXT: orq %rbx, %rbp
+; CHECK-NEXT: movq %rbp, %r11
; CHECK-NEXT: andq %r10, %r11
-; CHECK-NEXT: leaq (%r11,%rax,4), %rax
-; CHECK-NEXT: movq %rax, %r10
-; CHECK-NEXT: andq %rbx, %r10
-; CHECK-NEXT: shrq %rax
-; CHECK-NEXT: andq %rbx, %rax
-; CHECK-NEXT: leaq (%rax,%r10,2), %rax
-; CHECK-NEXT: shrdq $24, %rax, %rdx
-; CHECK-NEXT: movq %rdx, 112(%rsi)
-; CHECK-NEXT: movq %rcx, 104(%rsi)
-; CHECK-NEXT: movq %r8, 96(%rsi)
-; CHECK-NEXT: movq %r9, 88(%rsi)
-; CHECK-NEXT: movq %r14, 80(%rsi)
-; CHECK-NEXT: movq %r15, 72(%rsi)
-; CHECK-NEXT: movq %r12, 64(%rsi)
-; CHECK-NEXT: movq %r13, 56(%rsi)
-; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; CHECK-NEXT: movq %rcx, 48(%rsi)
-; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; CHECK-NEXT: movq %rcx, 40(%rsi)
-; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; CHECK-NEXT: movq %rcx, 32(%rsi)
-; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; CHECK-NEXT: movq %rcx, 24(%rsi)
-; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; CHECK-NEXT: movq %rcx, 16(%rsi)
-; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; CHECK-NEXT: movq %rcx, 8(%rsi)
-; CHECK-NEXT: movq %rdi, (%rsi)
-; CHECK-NEXT: movq %rax, %rcx
-; CHECK-NEXT: shrq $56, %rax
-; CHECK-NEXT: movb %al, 124(%rsi)
-; CHECK-NEXT: shrq $24, %rcx
-; CHECK-NEXT: movl %ecx, 120(%rsi)
+; CHECK-NEXT: shrq $2, %rbp
+; CHECK-NEXT: andq %r10, %rbp
+; CHECK-NEXT: leaq (%rbp,%r11,4), %r10
+; CHECK-NEXT: movq %r10, %r11
+; CHECK-NEXT: andq %rsi, %r11
+; CHECK-NEXT: shrq %r10
+; CHECK-NEXT: andq %rsi, %r10
+; CHECK-NEXT: leaq (%r10,%r11,2), %rsi
+; CHECK-NEXT: shrq $24, %rax
+; CHECK-NEXT: movq %rsi, %r10
+; CHECK-NEXT: shlq $40, %r10
+; CHECK-NEXT: orq %rax, %r10
+; CHECK-NEXT: movq %r10, 112(%r12)
+; CHECK-NEXT: movq %rcx, 104(%r12)
+; CHECK-NEXT: movq %r8, 96(%r12)
+; CHECK-NEXT: movq %r9, 88(%r12)
+; CHECK-NEXT: movq %r14, 80(%r12)
+; CHECK-NEXT: movq %r15, 72(%r12)
+; CHECK-NEXT: movq %r13, 64(%r12)
+; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; CHECK-NEXT: movq %rax, 56(%r12)
+; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; CHECK-NEXT: movq %rax, 48(%r12)
+; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; CHECK-NEXT: movq %rax, 40(%r12)
+; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; CHECK-NEXT: movq %rax, 32(%r12)
+; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; CHECK-NEXT: movq %rax, 24(%r12)
+; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; CHECK-NEXT: movq %rax, 16(%r12)
+; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; CHECK-NEXT: movq %rax, 8(%r12)
+; CHECK-NEXT: movq %rdi, (%r12)
; CHECK-NEXT: movq %rsi, %rax
+; CHECK-NEXT: shrq $56, %rsi
+; CHECK-NEXT: movb %sil, 124(%r12)
+; CHECK-NEXT: shrq $24, %rax
+; CHECK-NEXT: movl %eax, 120(%r12)
+; CHECK-NEXT: movq %r12, %rax
; CHECK-NEXT: popq %rbx
; CHECK-NEXT: popq %r12
; CHECK-NEXT: popq %r13
diff --git a/llvm/test/CodeGen/X86/scmp.ll b/llvm/test/CodeGen/X86/scmp.ll
index 31382d017926d..19c808185824c 100644
--- a/llvm/test/CodeGen/X86/scmp.ll
+++ b/llvm/test/CodeGen/X86/scmp.ll
@@ -2518,7 +2518,6 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; SSE2-NEXT: pushq %r13
; SSE2-NEXT: pushq %r12
; SSE2-NEXT: pushq %rbx
-; SSE2-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; SSE2-NEXT: movzbl {{[0-9]+}}(%rsp), %eax
; SSE2-NEXT: movzbl {{[0-9]+}}(%rsp), %r10d
; SSE2-NEXT: movzbl {{[0-9]+}}(%rsp), %r11d
@@ -2559,9 +2558,9 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; SSE2-NEXT: setl %dl
; SSE2-NEXT: setg %bpl
; SSE2-NEXT: subb %dl, %bpl
-; SSE2-NEXT: movsbq %bpl, %r12
-; SSE2-NEXT: movq %r12, %rsi
-; SSE2-NEXT: sarq $63, %rsi
+; SSE2-NEXT: movsbq %bpl, %rdx
+; SSE2-NEXT: movq %rdx, %r12
+; SSE2-NEXT: sarq $63, %r12
; SSE2-NEXT: addb %bl, %bl
; SSE2-NEXT: sarb %bl
; SSE2-NEXT: addb %cl, %cl
@@ -2570,8 +2569,8 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; SSE2-NEXT: setl %cl
; SSE2-NEXT: setg %bl
; SSE2-NEXT: subb %cl, %bl
-; SSE2-NEXT: movsbq %bl, %rdx
-; SSE2-NEXT: movq %rdx, %rbx
+; SSE2-NEXT: movsbq %bl, %rsi
+; SSE2-NEXT: movq %rsi, %rbx
; SSE2-NEXT: sarq $63, %rbx
; SSE2-NEXT: addb %r11b, %r11b
; SSE2-NEXT: sarb %r11b
@@ -2601,72 +2600,85 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; SSE2-NEXT: addb %bpl, %bpl
; SSE2-NEXT: sarb %bpl
; SSE2-NEXT: cmpb %al, %bpl
-; SSE2-NEXT: setl %dil
+; SSE2-NEXT: setl %al
; SSE2-NEXT: setg %bpl
-; SSE2-NEXT: subb %dil, %bpl
-; SSE2-NEXT: movsbq %bpl, %rax
-; SSE2-NEXT: movq %rax, %r13
-; SSE2-NEXT: sarq $63, %r13
-; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; SSE2-NEXT: movl %r13d, 96(%rcx)
-; SSE2-NEXT: movabsq $2251799813685247, %rbp # imm = 0x7FFFFFFFFFFFF
-; SSE2-NEXT: andq %r13, %rbp
-; SSE2-NEXT: shlq $62, %r13
-; SSE2-NEXT: movq %rax, %rdi
+; SSE2-NEXT: subb %al, %bpl
+; SSE2-NEXT: movsbq %bpl, %rcx
+; SSE2-NEXT: movq %rcx, %rbp
+; SSE2-NEXT: sarq $63, %rbp
+; SSE2-NEXT: movq %rdi, %rax
+; SSE2-NEXT: movl %ebp, 96(%rdi)
+; SSE2-NEXT: movabsq $2251799813685247, %r13 # imm = 0x7FFFFFFFFFFFF
+; SSE2-NEXT: andq %rbp, %r13
+; SSE2-NEXT: shlq $62, %rbp
+; SSE2-NEXT: movq %rcx, %rdi
; SSE2-NEXT: shrq $2, %rdi
-; SSE2-NEXT: orq %r13, %rdi
-; SSE2-NEXT: movq %rdi, 88(%rcx)
+; SSE2-NEXT: orq %rbp, %rdi
+; SSE2-NEXT: movq %rdi, 88(%rax)
; SSE2-NEXT: movq %r9, %rdi
-; SSE2-NEXT: shrdq $44, %r10, %rdi
-; SSE2-NEXT: movq %rdi, 64(%rcx)
+; SSE2-NEXT: shrq $44, %rdi
+; SSE2-NEXT: movq %r10, %rbp
+; SSE2-NEXT: shlq $20, %rbp
+; SSE2-NEXT: orq %rdi, %rbp
+; SSE2-NEXT: movq %rbp, 64(%rax)
; SSE2-NEXT: movq %r8, %rdi
-; SSE2-NEXT: shrdq $33, %r11, %rdi
-; SSE2-NEXT: movq %rdi, 48(%rcx)
+; SSE2-NEXT: shrq $33, %rdi
+; SSE2-NEXT: movq %r11, %rbp
+; SSE2-NEXT: shlq $31, %rbp
+; SSE2-NEXT: orq %rdi, %rbp
+; SSE2-NEXT: movq %rbp, 48(%rax)
+; SSE2-NEXT: movq %rsi, %rdi
+; SSE2-NEXT: shrq $22, %rdi
+; SSE2-NEXT: movq %rbx, %rbp
+; SSE2-NEXT: shlq $42, %rbp
+; SSE2-NEXT: orq %rdi, %rbp
+; SSE2-NEXT: movq %rbp, 32(%rax)
; SSE2-NEXT: movq %rdx, %rdi
-; SSE2-NEXT: shrdq $22, %rbx, %rdi
-; SSE2-NEXT: movq %rdi, 32(%rcx)
-; SSE2-NEXT: movq %r12, %rdi
-; SSE2-NEXT: shrdq $11, %rsi, %rdi
-; SSE2-NEXT: movq %rdi, 16(%rcx)
-; SSE2-NEXT: movq %rbp, %rdi
+; SSE2-NEXT: shrq $11, %rdi
+; SSE2-NEXT: movabsq $9007199254738944, %rbp # imm = 0x1FFFFFFFFFF800
+; SSE2-NEXT: andq %r12, %rbp
+; SSE2-NEXT: shlq $53, %r12
+; SSE2-NEXT: orq %rdi, %r12
+; SSE2-NEXT: movq %r12, 16(%rax)
+; SSE2-NEXT: movq %r13, %rdi
; SSE2-NEXT: shrq $48, %rdi
-; SSE2-NEXT: movb %dil, 102(%rcx)
-; SSE2-NEXT: shrq $32, %rbp
-; SSE2-NEXT: movw %bp, 100(%rcx)
-; SSE2-NEXT: movabsq $9007199254740991, %rdi # imm = 0x1FFFFFFFFFFFFF
-; SSE2-NEXT: andq %rdi, %r15
-; SSE2-NEXT: shldq $9, %r14, %r15
-; SSE2-NEXT: shlq $62, %rax
-; SSE2-NEXT: orq %r15, %rax
-; SSE2-NEXT: movq %rax, 80(%rcx)
-; SSE2-NEXT: movabsq $9007199254738944, %rax # imm = 0x1FFFFFFFFFF800
-; SSE2-NEXT: andq %rsi, %rax
-; SSE2-NEXT: shlq $42, %rdx
-; SSE2-NEXT: shrq $11, %rax
-; SSE2-NEXT: orq %rdx, %rax
-; SSE2-NEXT: movq %rax, 24(%rcx)
+; SSE2-NEXT: movb %dil, 102(%rax)
+; SSE2-NEXT: shrq $32, %r13
+; SSE2-NEXT: movw %r13w, 100(%rax)
+; SSE2-NEXT: movabsq $9007199254740991, %r12 # imm = 0x1FFFFFFFFFFFFF
+; SSE2-NEXT: andq %r12, %r15
+; SSE2-NEXT: movq %r14, %rdi
+; SSE2-NEXT: shrq $55, %rdi
+; SSE2-NEXT: shlq $9, %r15
+; SSE2-NEXT: orq %rdi, %r15
+; SSE2-NEXT: shlq $62, %rcx
+; SSE2-NEXT: orq %r15, %rcx
+; SSE2-NEXT: movq %rcx, 80(%rax)
+; SSE2-NEXT: shlq $42, %rsi
+; SSE2-NEXT: shrq $11, %rbp
+; SSE2-NEXT: orq %rsi, %rbp
+; SSE2-NEXT: movq %rbp, 24(%rax)
; SSE2-NEXT: shlq $9, %r14
; SSE2-NEXT: andl $511, %r10d # imm = 0x1FF
; SSE2-NEXT: orq %r14, %r10
-; SSE2-NEXT: movq %r10, 72(%rcx)
+; SSE2-NEXT: movq %r10, 72(%rax)
; SSE2-NEXT: shlq $20, %r9
; SSE2-NEXT: andl $1048575, %r11d # imm = 0xFFFFF
; SSE2-NEXT: orq %r9, %r11
-; SSE2-NEXT: movq %r11, 56(%rcx)
+; SSE2-NEXT: movq %r11, 56(%rax)
; SSE2-NEXT: shlq $31, %r8
; SSE2-NEXT: andl $2147483647, %ebx # imm = 0x7FFFFFFF
; SSE2-NEXT: orq %r8, %rbx
-; SSE2-NEXT: movq %rbx, 40(%rcx)
-; SSE2-NEXT: movq %rcx, %rax
+; SSE2-NEXT: movq %rbx, 40(%rax)
; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Folded Reload
; SSE2-NEXT: # xmm1 = mem[0],zero
; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
; SSE2-NEXT: movq %xmm0, %rcx
-; SSE2-NEXT: andq %rdi, %rcx
-; SSE2-NEXT: shlq $53, %r12
-; SSE2-NEXT: orq %rcx, %r12
-; SSE2-NEXT: movq %r12, 8(%rax)
+; SSE2-NEXT: andq %r12, %rcx
+; SSE2-NEXT: shlq $53, %rdx
+; SSE2-NEXT: orq %rcx, %rdx
+; SSE2-NEXT: movq %rdx, 8(%rax)
; SSE2-NEXT: popq %rbx
; SSE2-NEXT: popq %r12
; SSE2-NEXT: popq %r13
@@ -2988,16 +3000,29 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; AVX512-NEXT: pushq %r13
; AVX512-NEXT: pushq %r12
; AVX512-NEXT: pushq %rbx
-; AVX512-NEXT: movq %rdi, %r12
+; AVX512-NEXT: movq %rdi, %r15
; AVX512-NEXT: movzbl {{[0-9]+}}(%rsp), %edi
-; AVX512-NEXT: movzbl {{[0-9]+}}(%rsp), %r10d
; AVX512-NEXT: movzbl {{[0-9]+}}(%rsp), %r11d
-; AVX512-NEXT: movzbl {{[0-9]+}}(%rsp), %ebx
; AVX512-NEXT: movzbl {{[0-9]+}}(%rsp), %ebp
-; AVX512-NEXT: movzbl {{[0-9]+}}(%rsp), %r15d
+; AVX512-NEXT: movzbl {{[0-9]+}}(%rsp), %eax
+; AVX512-NEXT: movzbl {{[0-9]+}}(%rsp), %ebx
; AVX512-NEXT: movzbl {{[0-9]+}}(%rsp), %r14d
+; AVX512-NEXT: movzbl {{[0-9]+}}(%rsp), %r10d
+; AVX512-NEXT: addb %r10b, %r10b
+; AVX512-NEXT: sarb %r10b
+; AVX512-NEXT: addb %sil, %sil
+; AVX512-NEXT: sarb %sil
+; AVX512-NEXT: cmpb %r10b, %sil
+; AVX512-NEXT: setl %sil
+; AVX512-NEXT: setg %r10b
+; AVX512-NEXT: subb %sil, %r10b
+; AVX512-NEXT: movsbq %r10b, %r10
+; AVX512-NEXT: movq %r10, (%r15)
+; AVX512-NEXT: sarq $63, %r10
+; AVX512-NEXT: movq %r10, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX512-NEXT: addb %r14b, %r14b
; AVX512-NEXT: sarb %r14b
+; AVX512-NEXT: movzbl {{[0-9]+}}(%rsp), %esi
; AVX512-NEXT: addb %sil, %sil
; AVX512-NEXT: sarb %sil
; AVX512-NEXT: cmpb %r14b, %sil
@@ -3005,130 +3030,131 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; AVX512-NEXT: setg %r14b
; AVX512-NEXT: subb %sil, %r14b
; AVX512-NEXT: movsbq %r14b, %r14
-; AVX512-NEXT: movq %r14, (%r12)
-; AVX512-NEXT: sarq $63, %r14
-; AVX512-NEXT: movq %r14, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX512-NEXT: addb %r15b, %r15b
-; AVX512-NEXT: sarb %r15b
-; AVX512-NEXT: movzbl {{[0-9]+}}(%rsp), %esi
-; AVX512-NEXT: addb %sil, %sil
-; AVX512-NEXT: sarb %sil
-; AVX512-NEXT: cmpb %r15b, %sil
-; AVX512-NEXT: setl %sil
-; AVX512-NEXT: setg %r15b
-; AVX512-NEXT: subb %sil, %r15b
-; AVX512-NEXT: movsbq %r15b, %rsi
-; AVX512-NEXT: movq %rsi, %r14
-; AVX512-NEXT: sarq $63, %r14
-; AVX512-NEXT: addb %bpl, %bpl
-; AVX512-NEXT: sarb %bpl
+; AVX512-NEXT: movq %r14, %r10
+; AVX512-NEXT: sarq $63, %r10
+; AVX512-NEXT: addb %bl, %bl
+; AVX512-NEXT: sarb %bl
; AVX512-NEXT: addb %dl, %dl
; AVX512-NEXT: sarb %dl
-; AVX512-NEXT: cmpb %bpl, %dl
+; AVX512-NEXT: cmpb %bl, %dl
; AVX512-NEXT: setl %dl
-; AVX512-NEXT: setg %bpl
-; AVX512-NEXT: subb %dl, %bpl
-; AVX512-NEXT: movsbq %bpl, %r15
-; AVX512-NEXT: movq %r15, %r13
-; AVX512-NEXT: sarq $63, %r13
-; AVX512-NEXT: addb %bl, %bl
-; AVX512-NEXT: sarb %bl
+; AVX512-NEXT: setg %bl
+; AVX512-NEXT: subb %dl, %bl
+; AVX512-NEXT: movsbq %bl, %rbx
+; AVX512-NEXT: movq %rbx, %r12
+; AVX512-NEXT: sarq $63, %r12
+; AVX512-NEXT: addb %al, %al
+; AVX512-NEXT: sarb %al
; AVX512-NEXT: addb %cl, %cl
; AVX512-NEXT: sarb %cl
-; AVX512-NEXT: cmpb %bl, %cl
+; AVX512-NEXT: cmpb %al, %cl
; AVX512-NEXT: setl %cl
; AVX512-NEXT: setg %dl
; AVX512-NEXT: subb %cl, %dl
-; AVX512-NEXT: movsbq %dl, %rbx
-; AVX512-NEXT: movq %rbx, %rcx
-; AVX512-NEXT: sarq $63, %rcx
-; AVX512-NEXT: addb %r11b, %r11b
-; AVX512-NEXT: sarb %r11b
+; AVX512-NEXT: movsbq %dl, %rcx
+; AVX512-NEXT: movq %rcx, %rsi
+; AVX512-NEXT: sarq $63, %rsi
+; AVX512-NEXT: addb %bpl, %bpl
+; AVX512-NEXT: sarb %bpl
; AVX512-NEXT: addb %r8b, %r8b
; AVX512-NEXT: sarb %r8b
-; AVX512-NEXT: cmpb %r11b, %r8b
+; AVX512-NEXT: cmpb %bpl, %r8b
; AVX512-NEXT: setl %dl
; AVX512-NEXT: setg %r8b
; AVX512-NEXT: subb %dl, %r8b
; AVX512-NEXT: movsbq %r8b, %rdx
; AVX512-NEXT: movq %rdx, %r8
; AVX512-NEXT: sarq $63, %r8
-; AVX512-NEXT: addb %r10b, %r10b
-; AVX512-NEXT: sarb %r10b
+; AVX512-NEXT: addb %r11b, %r11b
+; AVX512-NEXT: sarb %r11b
; AVX512-NEXT: addb %r9b, %r9b
; AVX512-NEXT: sarb %r9b
-; AVX512-NEXT: cmpb %r10b, %r9b
+; AVX512-NEXT: cmpb %r11b, %r9b
; AVX512-NEXT: setl %r9b
-; AVX512-NEXT: setg %r10b
-; AVX512-NEXT: subb %r9b, %r10b
-; AVX512-NEXT: movsbq %r10b, %r9
-; AVX512-NEXT: movq %r9, %r10
-; AVX512-NEXT: sarq $63, %r10
+; AVX512-NEXT: setg %r11b
+; AVX512-NEXT: subb %r9b, %r11b
+; AVX512-NEXT: movsbq %r11b, %r9
+; AVX512-NEXT: movq %r9, %r11
+; AVX512-NEXT: sarq $63, %r11
; AVX512-NEXT: addb %dil, %dil
; AVX512-NEXT: sarb %dil
-; AVX512-NEXT: movzbl {{[0-9]+}}(%rsp), %r11d
-; AVX512-NEXT: addb %r11b, %r11b
-; AVX512-NEXT: sarb %r11b
-; AVX512-NEXT: cmpb %dil, %r11b
+; AVX512-NEXT: movzbl {{[0-9]+}}(%rsp), %ebp
+; AVX512-NEXT: addb %bpl, %bpl
+; AVX512-NEXT: sarb %bpl
+; AVX512-NEXT: cmpb %dil, %bpl
; AVX512-NEXT: setl %dil
-; AVX512-NEXT: setg %r11b
-; AVX512-NEXT: subb %dil, %r11b
-; AVX512-NEXT: movsbq %r11b, %rax
-; AVX512-NEXT: movq %rax, %r11
-; AVX512-NEXT: sarq $63, %r11
-; AVX512-NEXT: movl %r11d, 96(%r12)
-; AVX512-NEXT: movb $51, %bpl
-; AVX512-NEXT: bzhiq %rbp, %r11, %rbp
-; AVX512-NEXT: shlq $62, %r11
+; AVX512-NEXT: setg %bpl
+; AVX512-NEXT: subb %dil, %bpl
+; AVX512-NEXT: movsbq %bpl, %rax
+; AVX512-NEXT: movq %rax, %rbp
+; AVX512-NEXT: sarq $63, %rbp
+; AVX512-NEXT: movl %ebp, 96(%r15)
+; AVX512-NEXT: movb $51, %r13b
+; AVX512-NEXT: bzhiq %r13, %rbp, %r13
+; AVX512-NEXT: shlq $62, %rbp
; AVX512-NEXT: movq %rax, %rdi
; AVX512-NEXT: shrq $2, %rdi
-; AVX512-NEXT: orq %r11, %rdi
-; AVX512-NEXT: movq %rdi, 88(%r12)
+; AVX512-NEXT: orq %rbp, %rdi
+; AVX512-NEXT: movq %rdi, 88(%r15)
; AVX512-NEXT: movq %r9, %rdi
-; AVX512-NEXT: shrdq $44, %r10, %rdi
-; AVX512-NEXT: movq %rdi, 64(%r12)
+; AVX512-NEXT: shrq $44, %rdi
+; AVX512-NEXT: movq %r11, %rbp
+; AVX512-NEXT: shlq $20, %rbp
+; AVX512-NEXT: orq %rdi, %rbp
+; AVX512-NEXT: movq %rbp, 64(%r15)
; AVX512-NEXT: movq %rdx, %rdi
-; AVX512-NEXT: shrdq $33, %r8, %rdi
-; AVX512-NEXT: movq %rdi, 48(%r12)
+; AVX512-NEXT: shrq $33, %rdi
+; AVX512-NEXT: movq %r8, %rbp
+; AVX512-NEXT: shlq $31, %rbp
+; AVX512-NEXT: orq %rdi, %rbp
+; AVX512-NEXT: movq %rbp, 48(%r15)
+; AVX512-NEXT: movq %rcx, %rdi
+; AVX512-NEXT: shrq $22, %rdi
+; AVX512-NEXT: movq %rsi, %rbp
+; AVX512-NEXT: shlq $42, %rbp
+; AVX512-NEXT: orq %rdi, %rbp
+; AVX512-NEXT: movq %rbp, 32(%r15)
; AVX512-NEXT: movq %rbx, %rdi
-; AVX512-NEXT: shrdq $22, %rcx, %rdi
-; AVX512-NEXT: movq %rdi, 32(%r12)
-; AVX512-NEXT: movq %r15, %rdi
-; AVX512-NEXT: shrdq $11, %r13, %rdi
-; AVX512-NEXT: movq %rdi, 16(%r12)
-; AVX512-NEXT: movq %rbp, %rdi
+; AVX512-NEXT: shrq $11, %rdi
+; AVX512-NEXT: movb $42, %bpl
+; AVX512-NEXT: bzhiq %rbp, %r12, %rbp
+; AVX512-NEXT: shlq $53, %r12
+; AVX512-NEXT: orq %rdi, %r12
+; AVX512-NEXT: movq %r12, 16(%r15)
+; AVX512-NEXT: movq %r13, %rdi
; AVX512-NEXT: shrq $48, %rdi
-; AVX512-NEXT: movb %dil, 102(%r12)
-; AVX512-NEXT: shrq $32, %rbp
-; AVX512-NEXT: movw %bp, 100(%r12)
+; AVX512-NEXT: movb %dil, 102(%r15)
+; AVX512-NEXT: shrq $32, %r13
+; AVX512-NEXT: movw %r13w, 100(%r15)
; AVX512-NEXT: movb $53, %dil
-; AVX512-NEXT: bzhiq %rdi, %r14, %r11
-; AVX512-NEXT: shldq $9, %rsi, %r11
+; AVX512-NEXT: bzhiq %rdi, %r10, %r10
+; AVX512-NEXT: movq %r14, %r12
+; AVX512-NEXT: shrq $55, %r12
+; AVX512-NEXT: shlq $9, %r10
+; AVX512-NEXT: orq %r12, %r10
; AVX512-NEXT: shlq $62, %rax
-; AVX512-NEXT: orq %r11, %rax
-; AVX512-NEXT: movq %rax, 80(%r12)
-; AVX512-NEXT: movb $42, %al
-; AVX512-NEXT: bzhiq %rax, %r13, %rax
-; AVX512-NEXT: shlq $42, %rbx
-; AVX512-NEXT: orq %rax, %rbx
-; AVX512-NEXT: movq %rbx, 24(%r12)
+; AVX512-NEXT: orq %r10, %rax
+; AVX512-NEXT: movq %rax, 80(%r15)
+; AVX512-NEXT: shlq $42, %rcx
+; AVX512-NEXT: orq %rbp, %rcx
+; AVX512-NEXT: movq %rcx, 24(%r15)
; AVX512-NEXT: bzhiq %rdi, {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Folded Reload
-; AVX512-NEXT: shlq $53, %r15
-; AVX512-NEXT: orq %rax, %r15
-; AVX512-NEXT: movq %r15, 8(%r12)
-; AVX512-NEXT: shlq $9, %rsi
-; AVX512-NEXT: andl $511, %r10d # imm = 0x1FF
-; AVX512-NEXT: orq %rsi, %r10
-; AVX512-NEXT: movq %r10, 72(%r12)
+; AVX512-NEXT: shlq $53, %rbx
+; AVX512-NEXT: orq %rax, %rbx
+; AVX512-NEXT: movq %rbx, 8(%r15)
+; AVX512-NEXT: shlq $9, %r14
+; AVX512-NEXT: andl $511, %r11d # imm = 0x1FF
+; AVX512-NEXT: orq %r14, %r11
+; AVX512-NEXT: movq %r11, 72(%r15)
; AVX512-NEXT: shlq $20, %r9
; AVX512-NEXT: andl $1048575, %r8d # imm = 0xFFFFF
; AVX512-NEXT: orq %r9, %r8
-; AVX512-NEXT: movq %r8, 56(%r12)
+; AVX512-NEXT: movq %r8, 56(%r15)
; AVX512-NEXT: shlq $31, %rdx
-; AVX512-NEXT: andl $2147483647, %ecx # imm = 0x7FFFFFFF
-; AVX512-NEXT: orq %rdx, %rcx
-; AVX512-NEXT: movq %rcx, 40(%r12)
-; AVX512-NEXT: movq %r12, %rax
+; AVX512-NEXT: andl $2147483647, %esi # imm = 0x7FFFFFFF
+; AVX512-NEXT: orq %rdx, %rsi
+; AVX512-NEXT: movq %rsi, 40(%r15)
+; AVX512-NEXT: movq %r15, %rax
; AVX512-NEXT: popq %rbx
; AVX512-NEXT: popq %r12
; AVX512-NEXT: popq %r13
@@ -3143,7 +3169,7 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; X86-NEXT: pushl %ebx
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
-; X86-NEXT: subl $60, %esp
+; X86-NEXT: subl $52, %esp
; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
; X86-NEXT: addb %al, %al
; X86-NEXT: sarb %al
@@ -3168,75 +3194,74 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; X86-NEXT: addb %al, %al
; X86-NEXT: sarb %al
; X86-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ebx
-; X86-NEXT: addb %bl, %bl
-; X86-NEXT: sarb %bl
-; X86-NEXT: movb {{[0-9]+}}(%esp), %ch
-; X86-NEXT: addb %ch, %ch
-; X86-NEXT: sarb %ch
; X86-NEXT: movzbl {{[0-9]+}}(%esp), %edx
; X86-NEXT: addb %dl, %dl
; X86-NEXT: sarb %dl
-; X86-NEXT: movb {{[0-9]+}}(%esp), %ah
-; X86-NEXT: addb %ah, %ah
-; X86-NEXT: sarb %ah
+; X86-NEXT: movb {{[0-9]+}}(%esp), %ch
+; X86-NEXT: addb %ch, %ch
+; X86-NEXT: sarb %ch
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT: addb %bl, %bl
+; X86-NEXT: sarb %bl
+; X86-NEXT: movb {{[0-9]+}}(%esp), %bh
+; X86-NEXT: addb %bh, %bh
+; X86-NEXT: sarb %bh
; X86-NEXT: movb {{[0-9]+}}(%esp), %cl
; X86-NEXT: addb %cl, %cl
; X86-NEXT: sarb %cl
-; X86-NEXT: movb {{[0-9]+}}(%esp), %dh
-; X86-NEXT: addb %dh, %dh
-; X86-NEXT: sarb %dh
+; X86-NEXT: movb {{[0-9]+}}(%esp), %ah
+; X86-NEXT: addb %ah, %ah
+; X86-NEXT: sarb %ah
; X86-NEXT: movb {{[0-9]+}}(%esp), %al
; X86-NEXT: addb %al, %al
; X86-NEXT: sarb %al
-; X86-NEXT: movb {{[0-9]+}}(%esp), %bh
-; X86-NEXT: addb %bh, %bh
-; X86-NEXT: sarb %bh
-; X86-NEXT: cmpb %al, %bh
+; X86-NEXT: movb {{[0-9]+}}(%esp), %dh
+; X86-NEXT: addb %dh, %dh
+; X86-NEXT: sarb %dh
+; X86-NEXT: cmpb %al, %dh
; X86-NEXT: setl %al
-; X86-NEXT: setg %bh
-; X86-NEXT: subb %al, %bh
-; X86-NEXT: movsbl %bh, %esi
+; X86-NEXT: setg %dh
+; X86-NEXT: subb %al, %dh
+; X86-NEXT: movsbl %dh, %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: sarl $31, %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: andl $2097151, %esi # imm = 0x1FFFFF
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: cmpb %cl, %dh
+; X86-NEXT: cmpb %cl, %ah
; X86-NEXT: setl %al
; X86-NEXT: setg %cl
; X86-NEXT: subb %al, %cl
-; X86-NEXT: movsbl %cl, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: sarl $31, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: andl $2097151, %esi # imm = 0x1FFFFF
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: cmpb %dl, %ah
+; X86-NEXT: movsbl %cl, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sarl $31, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: andl $2097151, %eax # imm = 0x1FFFFF
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: cmpb %bl, %bh
; X86-NEXT: setl %al
; X86-NEXT: setg %cl
; X86-NEXT: subb %al, %cl
; X86-NEXT: movsbl %cl, %eax
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl %eax, (%edx)
+; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT: movl %eax, (%esi)
; X86-NEXT: sarl $31, %eax
; X86-NEXT: movl %eax, %esi
; X86-NEXT: andl $2097151, %esi # imm = 0x1FFFFF
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: cmpb %bl, %ch
+; X86-NEXT: cmpb %dl, %ch
; X86-NEXT: setl %cl
-; X86-NEXT: setg %ch
-; X86-NEXT: subb %cl, %ch
-; X86-NEXT: movsbl %ch, %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: sarl $31, %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: setg %dl
+; X86-NEXT: subb %cl, %dl
+; X86-NEXT: movsbl %dl, %ebp
+; X86-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sarl $31, %ebp
; X86-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
; X86-NEXT: cmpb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Folded Reload
; X86-NEXT: setl %cl
-; X86-NEXT: setg %ch
-; X86-NEXT: subb %cl, %ch
-; X86-NEXT: movsbl %ch, %edi
+; X86-NEXT: setg %dl
+; X86-NEXT: subb %cl, %dl
+; X86-NEXT: movsbl %dl, %edi
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: sarl $31, %edi
; X86-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
@@ -3244,103 +3269,109 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; X86-NEXT: setl %cl
; X86-NEXT: setg %ch
; X86-NEXT: subb %cl, %ch
-; X86-NEXT: movsbl %ch, %ebp
-; X86-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: sarl $31, %ebp
-; X86-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 1-byte Folded Reload
-; X86-NEXT: cmpb {{[-0-9]+}}(%e{{[sb]}}p), %bl # 1-byte Folded Reload
+; X86-NEXT: movsbl %ch, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sarl $31, %esi
+; X86-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X86-NEXT: cmpb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Folded Reload
; X86-NEXT: setl %bl
; X86-NEXT: setg %bh
; X86-NEXT: subb %bl, %bh
-; X86-NEXT: movsbl %bh, %ebx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %ebx, %esi
-; X86-NEXT: sarl $31, %esi
-; X86-NEXT: movl %esi, 96(%edx)
-; X86-NEXT: movl %esi, 92(%edx)
+; X86-NEXT: movsbl %bh, %ecx
+; X86-NEXT: movl %ecx, %ebx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sarl $31, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movl %ecx, 96(%edx)
+; X86-NEXT: movl %ecx, 92(%edx)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: movl %ecx, 80(%edx)
-; X86-NEXT: movl %ebp, 68(%edx)
-; X86-NEXT: movl %ebp, 64(%edx)
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %esi, 68(%edx)
+; X86-NEXT: movl %esi, 64(%edx)
; X86-NEXT: movl %edi, 52(%edx)
; X86-NEXT: movl %edi, 48(%edx)
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: movl %edi, 36(%edx)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl %ecx, 24(%edx)
-; X86-NEXT: movl %ecx, 20(%edx)
+; X86-NEXT: movl %ecx, 36(%edx)
+; X86-NEXT: movl %ebp, 24(%edx)
+; X86-NEXT: movl %ebp, 20(%edx)
; X86-NEXT: movl %eax, 8(%edx)
; X86-NEXT: movl %eax, 4(%edx)
-; X86-NEXT: movl %esi, %ecx
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ebx, %eax
+; X86-NEXT: shrl $2, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %ecx, %ebx
+; X86-NEXT: shll $30, %ebx
+; X86-NEXT: orl %eax, %ebx
; X86-NEXT: movw %cx, 100(%edx)
-; X86-NEXT: shrdl $2, %esi, %ebx
; X86-NEXT: movl %ebx, 88(%edx)
-; X86-NEXT: movl %edx, %esi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: shldl $9, %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: shll $9, %eax
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: shrl $23, %edi
-; X86-NEXT: orl %eax, %edi
-; X86-NEXT: movl %edi, 76(%edx)
-; X86-NEXT: movl %ebp, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: shrl $23, %ebx
+; X86-NEXT: orl %eax, %ebx
+; X86-NEXT: movl %ebx, 76(%edx)
+; X86-NEXT: movl %esi, %eax
; X86-NEXT: shll $20, %eax
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: shrl $12, %edi
-; X86-NEXT: orl %eax, %edi
-; X86-NEXT: movl %edi, 60(%edx)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: movl %ebx, %eax
+; X86-NEXT: shrl $12, %ebx
+; X86-NEXT: orl %eax, %ebx
+; X86-NEXT: movl %ebx, 60(%edx)
+; X86-NEXT: movl %edi, %eax
; X86-NEXT: shll $31, %eax
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: shrl %edi
-; X86-NEXT: orl %eax, %edi
-; X86-NEXT: movl %edi, 44(%edx)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: shrl %ebx
+; X86-NEXT: orl %eax, %ebx
+; X86-NEXT: movl %ebx, 44(%edx)
+; X86-NEXT: movl %edx, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: shrl $22, %eax
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: movl %edx, %eax
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: shrdl $22, %ecx, %eax
-; X86-NEXT: movl %eax, 32(%esi)
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: movl %edi, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: shldl $10, %ebx, %edx
+; X86-NEXT: shll $10, %ebx
+; X86-NEXT: orl %eax, %ebx
+; X86-NEXT: movl %ebx, 32(%ecx)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: shrdl $11, %ecx, %eax
-; X86-NEXT: movl %eax, 16(%esi)
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shll $9, %eax
-; X86-NEXT: andl $511, %ebp # imm = 0x1FF
-; X86-NEXT: orl %eax, %ebp
-; X86-NEXT: movl %ebp, 72(%esi)
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shll $20, %eax
-; X86-NEXT: andl $1048575, %ebx # imm = 0xFFFFF
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: shrl $11, %eax
+; X86-NEXT: movl %ebp, %ebx
+; X86-NEXT: shll $21, %ebx
; X86-NEXT: orl %eax, %ebx
-; X86-NEXT: movl %ebx, 56(%esi)
-; X86-NEXT: shll $10, %edx
-; X86-NEXT: andl $1023, %ecx # imm = 0x3FF
-; X86-NEXT: orl %edx, %ecx
-; X86-NEXT: movl %ecx, 28(%esi)
-; X86-NEXT: shll $21, %edi
-; X86-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X86-NEXT: movl %edi, 12(%esi)
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: andl $7, %eax
-; X86-NEXT: movb %al, 102(%esi)
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %ebx, 16(%eax)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shll $30, %eax
-; X86-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NEXT: movl %eax, 84(%esi)
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: shll $9, %eax
+; X86-NEXT: andl $511, %esi # imm = 0x1FF
+; X86-NEXT: orl %eax, %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %esi, 72(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: shll $20, %esi
+; X86-NEXT: andl $1048575, %edi # imm = 0xFFFFF
+; X86-NEXT: orl %esi, %edi
+; X86-NEXT: movl %edi, 56(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: shll $10, %esi
+; X86-NEXT: andl $1023, %ebp # imm = 0x3FF
+; X86-NEXT: orl %esi, %ebp
+; X86-NEXT: movl %ebp, 28(%eax)
+; X86-NEXT: shll $21, %ecx
+; X86-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: movl %ecx, 12(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: andl $7, %ecx
+; X86-NEXT: movb %cl, 102(%eax)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: shldl $10, %ecx, %edx
+; X86-NEXT: shll $30, %ecx
+; X86-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: movl %ecx, 84(%eax)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: shll $31, %ecx
; X86-NEXT: orl %edx, %ecx
-; X86-NEXT: movl %ecx, 40(%esi)
-; X86-NEXT: movl %esi, %eax
-; X86-NEXT: addl $60, %esp
+; X86-NEXT: movl %ecx, 40(%eax)
+; X86-NEXT: addl $52, %esp
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
; X86-NEXT: popl %ebx
@@ -3355,7 +3386,6 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; SETZUCC-NEXT: pushq %r13
; SETZUCC-NEXT: pushq %r12
; SETZUCC-NEXT: pushq %rbx
-; SETZUCC-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; SETZUCC-NEXT: movzbl {{[0-9]+}}(%rsp), %eax
; SETZUCC-NEXT: movzbl {{[0-9]+}}(%rsp), %r10d
; SETZUCC-NEXT: movzbl {{[0-9]+}}(%rsp), %r11d
@@ -3396,9 +3426,9 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; SETZUCC-NEXT: setzul %dl
; SETZUCC-NEXT: setzug %bpl
; SETZUCC-NEXT: subb %dl, %bpl
-; SETZUCC-NEXT: movsbq %bpl, %r12
-; SETZUCC-NEXT: movq %r12, %rsi
-; SETZUCC-NEXT: sarq $63, %rsi
+; SETZUCC-NEXT: movsbq %bpl, %rdx
+; SETZUCC-NEXT: movq %rdx, %r12
+; SETZUCC-NEXT: sarq $63, %r12
; SETZUCC-NEXT: addb %bl, %bl
; SETZUCC-NEXT: sarb %bl
; SETZUCC-NEXT: addb %cl, %cl
@@ -3407,8 +3437,8 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; SETZUCC-NEXT: setzul %cl
; SETZUCC-NEXT: setzug %bl
; SETZUCC-NEXT: subb %cl, %bl
-; SETZUCC-NEXT: movsbq %bl, %rdx
-; SETZUCC-NEXT: movq %rdx, %rbx
+; SETZUCC-NEXT: movsbq %bl, %rsi
+; SETZUCC-NEXT: movq %rsi, %rbx
; SETZUCC-NEXT: sarq $63, %rbx
; SETZUCC-NEXT: addb %r11b, %r11b
; SETZUCC-NEXT: sarb %r11b
@@ -3438,72 +3468,85 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; SETZUCC-NEXT: addb %bpl, %bpl
; SETZUCC-NEXT: sarb %bpl
; SETZUCC-NEXT: cmpb %al, %bpl
-; SETZUCC-NEXT: setzul %dil
+; SETZUCC-NEXT: setzul %al
; SETZUCC-NEXT: setzug %bpl
-; SETZUCC-NEXT: subb %dil, %bpl
-; SETZUCC-NEXT: movsbq %bpl, %rax
-; SETZUCC-NEXT: movq %rax, %r13
-; SETZUCC-NEXT: sarq $63, %r13
-; SETZUCC-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; SETZUCC-NEXT: movl %r13d, 96(%rcx)
-; SETZUCC-NEXT: movabsq $2251799813685247, %rbp # imm = 0x7FFFFFFFFFFFF
-; SETZUCC-NEXT: andq %r13, %rbp
-; SETZUCC-NEXT: shlq $62, %r13
-; SETZUCC-NEXT: movq %rax, %rdi
+; SETZUCC-NEXT: subb %al, %bpl
+; SETZUCC-NEXT: movsbq %bpl, %rcx
+; SETZUCC-NEXT: movq %rcx, %rbp
+; SETZUCC-NEXT: sarq $63, %rbp
+; SETZUCC-NEXT: movq %rdi, %rax
+; SETZUCC-NEXT: movl %ebp, 96(%rdi)
+; SETZUCC-NEXT: movabsq $2251799813685247, %r13 # imm = 0x7FFFFFFFFFFFF
+; SETZUCC-NEXT: andq %rbp, %r13
+; SETZUCC-NEXT: shlq $62, %rbp
+; SETZUCC-NEXT: movq %rcx, %rdi
; SETZUCC-NEXT: shrq $2, %rdi
-; SETZUCC-NEXT: orq %r13, %rdi
-; SETZUCC-NEXT: movq %rdi, 88(%rcx)
+; SETZUCC-NEXT: orq %rbp, %rdi
+; SETZUCC-NEXT: movq %rdi, 88(%rax)
; SETZUCC-NEXT: movq %r9, %rdi
-; SETZUCC-NEXT: shrdq $44, %r10, %rdi
-; SETZUCC-NEXT: movq %rdi, 64(%rcx)
+; SETZUCC-NEXT: shrq $44, %rdi
+; SETZUCC-NEXT: movq %r10, %rbp
+; SETZUCC-NEXT: shlq $20, %rbp
+; SETZUCC-NEXT: orq %rdi, %rbp
+; SETZUCC-NEXT: movq %rbp, 64(%rax)
; SETZUCC-NEXT: movq %r8, %rdi
-; SETZUCC-NEXT: shrdq $33, %r11, %rdi
-; SETZUCC-NEXT: movq %rdi, 48(%rcx)
+; SETZUCC-NEXT: shrq $33, %rdi
+; SETZUCC-NEXT: movq %r11, %rbp
+; SETZUCC-NEXT: shlq $31, %rbp
+; SETZUCC-NEXT: orq %rdi, %rbp
+; SETZUCC-NEXT: movq %rbp, 48(%rax)
+; SETZUCC-NEXT: movq %rsi, %rdi
+; SETZUCC-NEXT: shrq $22, %rdi
+; SETZUCC-NEXT: movq %rbx, %rbp
+; SETZUCC-NEXT: shlq $42, %rbp
+; SETZUCC-NEXT: orq %rdi, %rbp
+; SETZUCC-NEXT: movq %rbp, 32(%rax)
; SETZUCC-NEXT: movq %rdx, %rdi
-; SETZUCC-NEXT: shrdq $22, %rbx, %rdi
-; SETZUCC-NEXT: movq %rdi, 32(%rcx)
-; SETZUCC-NEXT: movq %r12, %rdi
-; SETZUCC-NEXT: shrdq $11, %rsi, %rdi
-; SETZUCC-NEXT: movq %rdi, 16(%rcx)
-; SETZUCC-NEXT: movq %rbp, %rdi
+; SETZUCC-NEXT: shrq $11, %rdi
+; SETZUCC-NEXT: movabsq $9007199254738944, %rbp # imm = 0x1FFFFFFFFFF800
+; SETZUCC-NEXT: andq %r12, %rbp
+; SETZUCC-NEXT: shlq $53, %r12
+; SETZUCC-NEXT: orq %rdi, %r12
+; SETZUCC-NEXT: movq %r12, 16(%rax)
+; SETZUCC-NEXT: movq %r13, %rdi
; SETZUCC-NEXT: shrq $48, %rdi
-; SETZUCC-NEXT: movb %dil, 102(%rcx)
-; SETZUCC-NEXT: shrq $32, %rbp
-; SETZUCC-NEXT: movw %bp, 100(%rcx)
-; SETZUCC-NEXT: movabsq $9007199254740991, %rdi # imm = 0x1FFFFFFFFFFFFF
-; SETZUCC-NEXT: andq %rdi, %r15
-; SETZUCC-NEXT: shldq $9, %r14, %r15
-; SETZUCC-NEXT: shlq $62, %rax
-; SETZUCC-NEXT: orq %r15, %rax
-; SETZUCC-NEXT: movq %rax, 80(%rcx)
-; SETZUCC-NEXT: movabsq $9007199254738944, %rax # imm = 0x1FFFFFFFFFF800
-; SETZUCC-NEXT: andq %rsi, %rax
-; SETZUCC-NEXT: shlq $42, %rdx
-; SETZUCC-NEXT: shrq $11, %rax
-; SETZUCC-NEXT: orq %rdx, %rax
-; SETZUCC-NEXT: movq %rax, 24(%rcx)
+; SETZUCC-NEXT: movb %dil, 102(%rax)
+; SETZUCC-NEXT: shrq $32, %r13
+; SETZUCC-NEXT: movw %r13w, 100(%rax)
+; SETZUCC-NEXT: movabsq $9007199254740991, %r12 # imm = 0x1FFFFFFFFFFFFF
+; SETZUCC-NEXT: andq %r12, %r15
+; SETZUCC-NEXT: movq %r14, %rdi
+; SETZUCC-NEXT: shrq $55, %rdi
+; SETZUCC-NEXT: shlq $9, %r15
+; SETZUCC-NEXT: orq %rdi, %r15
+; SETZUCC-NEXT: shlq $62, %rcx
+; SETZUCC-NEXT: orq %r15, %rcx
+; SETZUCC-NEXT: movq %rcx, 80(%rax)
+; SETZUCC-NEXT: shlq $42, %rsi
+; SETZUCC-NEXT: shrq $11, %rbp
+; SETZUCC-NEXT: orq %rsi, %rbp
+; SETZUCC-NEXT: movq %rbp, 24(%rax)
; SETZUCC-NEXT: shlq $9, %r14
; SETZUCC-NEXT: andl $511, %r10d # imm = 0x1FF
; SETZUCC-NEXT: orq %r14, %r10
-; SETZUCC-NEXT: movq %r10, 72(%rcx)
+; SETZUCC-NEXT: movq %r10, 72(%rax)
; SETZUCC-NEXT: shlq $20, %r9
; SETZUCC-NEXT: andl $1048575, %r11d # imm = 0xFFFFF
; SETZUCC-NEXT: orq %r9, %r11
-; SETZUCC-NEXT: movq %r11, 56(%rcx)
+; SETZUCC-NEXT: movq %r11, 56(%rax)
; SETZUCC-NEXT: shlq $31, %r8
; SETZUCC-NEXT: andl $2147483647, %ebx # imm = 0x7FFFFFFF
; SETZUCC-NEXT: orq %r8, %rbx
-; SETZUCC-NEXT: movq %rbx, 40(%rcx)
-; SETZUCC-NEXT: movq %rcx, %rax
+; SETZUCC-NEXT: movq %rbx, 40(%rax)
; SETZUCC-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Folded Reload
; SETZUCC-NEXT: # xmm1 = mem[0],zero
; SETZUCC-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; SETZUCC-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
; SETZUCC-NEXT: movq %xmm0, %rcx
-; SETZUCC-NEXT: andq %rdi, %rcx
-; SETZUCC-NEXT: shlq $53, %r12
-; SETZUCC-NEXT: orq %rcx, %r12
-; SETZUCC-NEXT: movq %r12, 8(%rax)
+; SETZUCC-NEXT: andq %r12, %rcx
+; SETZUCC-NEXT: shlq $53, %rdx
+; SETZUCC-NEXT: orq %rcx, %rdx
+; SETZUCC-NEXT: movq %rdx, 8(%rax)
; SETZUCC-NEXT: popq %rbx
; SETZUCC-NEXT: popq %r12
; SETZUCC-NEXT: popq %r13
@@ -3520,7 +3563,6 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; NO-SETZUCC-NEXT: pushq %r13
; NO-SETZUCC-NEXT: pushq %r12
; NO-SETZUCC-NEXT: pushq %rbx
-; NO-SETZUCC-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; NO-SETZUCC-NEXT: movzbl {{[0-9]+}}(%rsp), %eax
; NO-SETZUCC-NEXT: movzbl {{[0-9]+}}(%rsp), %r10d
; NO-SETZUCC-NEXT: movzbl {{[0-9]+}}(%rsp), %r11d
@@ -3561,9 +3603,9 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; NO-SETZUCC-NEXT: setl %dl
; NO-SETZUCC-NEXT: setg %bpl
; NO-SETZUCC-NEXT: subb %dl, %bpl
-; NO-SETZUCC-NEXT: movsbq %bpl, %r12
-; NO-SETZUCC-NEXT: movq %r12, %rsi
-; NO-SETZUCC-NEXT: sarq $63, %rsi
+; NO-SETZUCC-NEXT: movsbq %bpl, %rdx
+; NO-SETZUCC-NEXT: movq %rdx, %r12
+; NO-SETZUCC-NEXT: sarq $63, %r12
; NO-SETZUCC-NEXT: addb %bl, %bl
; NO-SETZUCC-NEXT: sarb %bl
; NO-SETZUCC-NEXT: addb %cl, %cl
@@ -3572,8 +3614,8 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; NO-SETZUCC-NEXT: setl %cl
; NO-SETZUCC-NEXT: setg %bl
; NO-SETZUCC-NEXT: subb %cl, %bl
-; NO-SETZUCC-NEXT: movsbq %bl, %rdx
-; NO-SETZUCC-NEXT: movq %rdx, %rbx
+; NO-SETZUCC-NEXT: movsbq %bl, %rsi
+; NO-SETZUCC-NEXT: movq %rsi, %rbx
; NO-SETZUCC-NEXT: sarq $63, %rbx
; NO-SETZUCC-NEXT: addb %r11b, %r11b
; NO-SETZUCC-NEXT: sarb %r11b
@@ -3603,72 +3645,85 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; NO-SETZUCC-NEXT: addb %bpl, %bpl
; NO-SETZUCC-NEXT: sarb %bpl
; NO-SETZUCC-NEXT: cmpb %al, %bpl
-; NO-SETZUCC-NEXT: setl %dil
+; NO-SETZUCC-NEXT: setl %al
; NO-SETZUCC-NEXT: setg %bpl
-; NO-SETZUCC-NEXT: subb %dil, %bpl
-; NO-SETZUCC-NEXT: movsbq %bpl, %rax
-; NO-SETZUCC-NEXT: movq %rax, %r13
-; NO-SETZUCC-NEXT: sarq $63, %r13
-; NO-SETZUCC-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
-; NO-SETZUCC-NEXT: movl %r13d, 96(%rcx)
-; NO-SETZUCC-NEXT: movabsq $2251799813685247, %rbp # imm = 0x7FFFFFFFFFFFF
-; NO-SETZUCC-NEXT: andq %r13, %rbp
-; NO-SETZUCC-NEXT: shlq $62, %r13
-; NO-SETZUCC-NEXT: movq %rax, %rdi
+; NO-SETZUCC-NEXT: subb %al, %bpl
+; NO-SETZUCC-NEXT: movsbq %bpl, %rcx
+; NO-SETZUCC-NEXT: movq %rcx, %rbp
+; NO-SETZUCC-NEXT: sarq $63, %rbp
+; NO-SETZUCC-NEXT: movq %rdi, %rax
+; NO-SETZUCC-NEXT: movl %ebp, 96(%rdi)
+; NO-SETZUCC-NEXT: movabsq $2251799813685247, %r13 # imm = 0x7FFFFFFFFFFFF
+; NO-SETZUCC-NEXT: andq %rbp, %r13
+; NO-SETZUCC-NEXT: shlq $62, %rbp
+; NO-SETZUCC-NEXT: movq %rcx, %rdi
; NO-SETZUCC-NEXT: shrq $2, %rdi
-; NO-SETZUCC-NEXT: orq %r13, %rdi
-; NO-SETZUCC-NEXT: movq %rdi, 88(%rcx)
+; NO-SETZUCC-NEXT: orq %rbp, %rdi
+; NO-SETZUCC-NEXT: movq %rdi, 88(%rax)
; NO-SETZUCC-NEXT: movq %r9, %rdi
-; NO-SETZUCC-NEXT: shrdq $44, %r10, %rdi
-; NO-SETZUCC-NEXT: movq %rdi, 64(%rcx)
+; NO-SETZUCC-NEXT: shrq $44, %rdi
+; NO-SETZUCC-NEXT: movq %r10, %rbp
+; NO-SETZUCC-NEXT: shlq $20, %rbp
+; NO-SETZUCC-NEXT: orq %rdi, %rbp
+; NO-SETZUCC-NEXT: movq %rbp, 64(%rax)
; NO-SETZUCC-NEXT: movq %r8, %rdi
-; NO-SETZUCC-NEXT: shrdq $33, %r11, %rdi
-; NO-SETZUCC-NEXT: movq %rdi, 48(%rcx)
+; NO-SETZUCC-NEXT: shrq $33, %rdi
+; NO-SETZUCC-NEXT: movq %r11, %rbp
+; NO-SETZUCC-NEXT: shlq $31, %rbp
+; NO-SETZUCC-NEXT: orq %rdi, %rbp
+; NO-SETZUCC-NEXT: movq %rbp, 48(%rax)
+; NO-SETZUCC-NEXT: movq %rsi, %rdi
+; NO-SETZUCC-NEXT: shrq $22, %rdi
+; NO-SETZUCC-NEXT: movq %rbx, %rbp
+; NO-SETZUCC-NEXT: shlq $42, %rbp
+; NO-SETZUCC-NEXT: orq %rdi, %rbp
+; NO-SETZUCC-NEXT: movq %rbp, 32(%rax)
; NO-SETZUCC-NEXT: movq %rdx, %rdi
-; NO-SETZUCC-NEXT: shrdq $22, %rbx, %rdi
-; NO-SETZUCC-NEXT: movq %rdi, 32(%rcx)
-; NO-SETZUCC-NEXT: movq %r12, %rdi
-; NO-SETZUCC-NEXT: shrdq $11, %rsi, %rdi
-; NO-SETZUCC-NEXT: movq %rdi, 16(%rcx)
-; NO-SETZUCC-NEXT: movq %rbp, %rdi
+; NO-SETZUCC-NEXT: shrq $11, %rdi
+; NO-SETZUCC-NEXT: movabsq $9007199254738944, %rbp # imm = 0x1FFFFFFFFFF800
+; NO-SETZUCC-NEXT: andq %r12, %rbp
+; NO-SETZUCC-NEXT: shlq $53, %r12
+; NO-SETZUCC-NEXT: orq %rdi, %r12
+; NO-SETZUCC-NEXT: movq %r12, 16(%rax)
+; NO-SETZUCC-NEXT: movq %r13, %rdi
; NO-SETZUCC-NEXT: shrq $48, %rdi
-; NO-SETZUCC-NEXT: movb %dil, 102(%rcx)
-; NO-SETZUCC-NEXT: shrq $32, %rbp
-; NO-SETZUCC-NEXT: movw %bp, 100(%rcx)
-; NO-SETZUCC-NEXT: movabsq $9007199254740991, %rdi # imm = 0x1FFFFFFFFFFFFF
-; NO-SETZUCC-NEXT: andq %rdi, %r15
-; NO-SETZUCC-NEXT: shldq $9, %r14, %r15
-; NO-SETZUCC-NEXT: shlq $62, %rax
-; NO-SETZUCC-NEXT: orq %r15, %rax
-; NO-SETZUCC-NEXT: movq %rax, 80(%rcx)
-; NO-SETZUCC-NEXT: movabsq $9007199254738944, %rax # imm = 0x1FFFFFFFFFF800
-; NO-SETZUCC-NEXT: andq %rsi, %rax
-; NO-SETZUCC-NEXT: shlq $42, %rdx
-; NO-SETZUCC-NEXT: shrq $11, %rax
-; NO-SETZUCC-NEXT: orq %rdx, %rax
-; NO-SETZUCC-NEXT: movq %rax, 24(%rcx)
+; NO-SETZUCC-NEXT: movb %dil, 102(%rax)
+; NO-SETZUCC-NEXT: shrq $32, %r13
+; NO-SETZUCC-NEXT: movw %r13w, 100(%rax)
+; NO-SETZUCC-NEXT: movabsq $9007199254740991, %r12 # imm = 0x1FFFFFFFFFFFFF
+; NO-SETZUCC-NEXT: andq %r12, %r15
+; NO-SETZUCC-NEXT: movq %r14, %rdi
+; NO-SETZUCC-NEXT: shrq $55, %rdi
+; NO-SETZUCC-NEXT: shlq $9, %r15
+; NO-SETZUCC-NEXT: orq %rdi, %r15
+; NO-SETZUCC-NEXT: shlq $62, %rcx
+; NO-SETZUCC-NEXT: orq %r15, %rcx
+; NO-SETZUCC-NEXT: movq %rcx, 80(%rax)
+; NO-SETZUCC-NEXT: shlq $42, %rsi
+; NO-SETZUCC-NEXT: shrq $11, %rbp
+; NO-SETZUCC-NEXT: orq %rsi, %rbp
+; NO-SETZUCC-NEXT: movq %rbp, 24(%rax)
; NO-SETZUCC-NEXT: shlq $9, %r14
; NO-SETZUCC-NEXT: andl $511, %r10d # imm = 0x1FF
; NO-SETZUCC-NEXT: orq %r14, %r10
-; NO-SETZUCC-NEXT: movq %r10, 72(%rcx)
+; NO-SETZUCC-NEXT: movq %r10, 72(%rax)
; NO-SETZUCC-NEXT: shlq $20, %r9
; NO-SETZUCC-NEXT: andl $1048575, %r11d # imm = 0xFFFFF
; NO-SETZUCC-NEXT: orq %r9, %r11
-; NO-SETZUCC-NEXT: movq %r11, 56(%rcx)
+; NO-SETZUCC-NEXT: movq %r11, 56(%rax)
; NO-SETZUCC-NEXT: shlq $31, %r8
; NO-SETZUCC-NEXT: andl $2147483647, %ebx # imm = 0x7FFFFFFF
; NO-SETZUCC-NEXT: orq %r8, %rbx
-; NO-SETZUCC-NEXT: movq %rbx, 40(%rcx)
-; NO-SETZUCC-NEXT: movq %rcx, %rax
+; NO-SETZUCC-NEXT: movq %rbx, 40(%rax)
; NO-SETZUCC-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Folded Reload
; NO-SETZUCC-NEXT: # xmm1 = mem[0],zero
; NO-SETZUCC-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; NO-SETZUCC-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
; NO-SETZUCC-NEXT: movq %xmm0, %rcx
-; NO-SETZUCC-NEXT: andq %rdi, %rcx
-; NO-SETZUCC-NEXT: shlq $53, %r12
-; NO-SETZUCC-NEXT: orq %rcx, %r12
-; NO-SETZUCC-NEXT: movq %r12, 8(%rax)
+; NO-SETZUCC-NEXT: andq %r12, %rcx
+; NO-SETZUCC-NEXT: shlq $53, %rdx
+; NO-SETZUCC-NEXT: orq %rcx, %rdx
+; NO-SETZUCC-NEXT: movq %rdx, 8(%rax)
; NO-SETZUCC-NEXT: popq %rbx
; NO-SETZUCC-NEXT: popq %r12
; NO-SETZUCC-NEXT: popq %r13
diff --git a/llvm/test/CodeGen/X86/sdiv_fix_sat.ll b/llvm/test/CodeGen/X86/sdiv_fix_sat.ll
index d503b8f1d4e38..35f70aa5f0120 100644
--- a/llvm/test/CodeGen/X86/sdiv_fix_sat.ll
+++ b/llvm/test/CodeGen/X86/sdiv_fix_sat.ll
@@ -342,17 +342,19 @@ define i64 @func5(i64 %x, i64 %y) nounwind {
; X64-NEXT: testb %bl, %al
; X64-NEXT: cmoveq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Folded Reload
; X64-NEXT: cmoveq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Folded Reload
-; X64-NEXT: movq %rbp, %rcx
-; X64-NEXT: sarq $63, %rcx
-; X64-NEXT: andq %rbp, %rcx
+; X64-NEXT: movq %rbp, %rax
+; X64-NEXT: sarq $63, %rax
+; X64-NEXT: andq %rbp, %rax
; X64-NEXT: testq %rbp, %rbp
-; X64-NEXT: movq $-1, %rdx
-; X64-NEXT: cmovgq %rdx, %r13
-; X64-NEXT: xorl %eax, %eax
-; X64-NEXT: cmpq $-1, %rcx
-; X64-NEXT: cmovlq %rdx, %rcx
-; X64-NEXT: cmovgeq %r13, %rax
-; X64-NEXT: shrdq $1, %rcx, %rax
+; X64-NEXT: movq $-1, %rcx
+; X64-NEXT: cmovgq %rcx, %r13
+; X64-NEXT: xorl %edx, %edx
+; X64-NEXT: cmpq $-1, %rax
+; X64-NEXT: cmovlq %rcx, %rax
+; X64-NEXT: cmovgeq %r13, %rdx
+; X64-NEXT: shrq %rdx
+; X64-NEXT: shlq $63, %rax
+; X64-NEXT: orq %rdx, %rax
; X64-NEXT: addq $24, %rsp
; X64-NEXT: popq %rbx
; X64-NEXT: popq %r12
diff --git a/llvm/test/CodeGen/X86/shld-machine-combiner.mir b/llvm/test/CodeGen/X86/shd-machine-combiner.mir
similarity index 51%
rename from llvm/test/CodeGen/X86/shld-machine-combiner.mir
rename to llvm/test/CodeGen/X86/shd-machine-combiner.mir
index d3e4b52f9bd12..484f6896372dd 100644
--- a/llvm/test/CodeGen/X86/shld-machine-combiner.mir
+++ b/llvm/test/CodeGen/X86/shd-machine-combiner.mir
@@ -121,3 +121,123 @@ body: |
SHLD64mri8 $rax, 1, $noreg, 0, $noreg, %0, 2, implicit-def $eflags
RET 0
...
+
+---
+name: shrd_rri32
+alignment: 16
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $eax, $ebx
+ ; FAST-SHLD-LABEL: name: shrd_rri32
+ ; FAST-SHLD: liveins: $eax, $ebx
+ ; FAST-SHLD-NEXT: {{ $}}
+ ; FAST-SHLD-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $eax
+ ; FAST-SHLD-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY $ebx
+ ; FAST-SHLD-NEXT: [[SHRD32rri8_:%[0-9]+]]:gr32 = SHRD32rri8 [[COPY]], [[COPY1]], 2, implicit-def $eflags
+ ; FAST-SHLD-NEXT: $eax = COPY [[SHRD32rri8_]]
+ ; FAST-SHLD-NEXT: RET 0, implicit $rax
+ ;
+ ; SLOW-SHLD-LABEL: name: shrd_rri32
+ ; SLOW-SHLD: liveins: $eax, $ebx
+ ; SLOW-SHLD-NEXT: {{ $}}
+ ; SLOW-SHLD-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $eax
+ ; SLOW-SHLD-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY $ebx
+ ; SLOW-SHLD-NEXT: [[SHR32ri:%[0-9]+]]:gr32 = SHR32ri [[COPY]], 2, implicit-def $eflags
+ ; SLOW-SHLD-NEXT: [[SHL32ri:%[0-9]+]]:gr32 = SHL32ri [[COPY1]], 30, implicit-def $eflags
+ ; SLOW-SHLD-NEXT: [[OR32rr:%[0-9]+]]:gr32 = OR32rr killed [[SHR32ri]], killed [[SHL32ri]], implicit-def $eflags
+ ; SLOW-SHLD-NEXT: $eax = COPY [[OR32rr]]
+ ; SLOW-SHLD-NEXT: RET 0, implicit $rax
+ %0:gr32 = COPY $eax
+ %1:gr32 = COPY $ebx
+ %2:gr32 = SHRD32rri8 %0, %1, 2, implicit-def $eflags
+ $eax = COPY %2
+ RET 0, implicit $rax
+...
+
+---
+name: shrd_rri64
+alignment: 16
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $rax, $rbx
+ ; FAST-SHLD-LABEL: name: shrd_rri64
+ ; FAST-SHLD: liveins: $rax, $rbx
+ ; FAST-SHLD-NEXT: {{ $}}
+ ; FAST-SHLD-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rax
+ ; FAST-SHLD-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rbx
+ ; FAST-SHLD-NEXT: [[SHRD64rri8_:%[0-9]+]]:gr64 = SHRD64rri8 [[COPY]], [[COPY1]], 2, implicit-def $eflags
+ ; FAST-SHLD-NEXT: $rax = COPY [[SHRD64rri8_]]
+ ; FAST-SHLD-NEXT: RET 0, implicit $rax
+ ;
+ ; SLOW-SHLD-LABEL: name: shrd_rri64
+ ; SLOW-SHLD: liveins: $rax, $rbx
+ ; SLOW-SHLD-NEXT: {{ $}}
+ ; SLOW-SHLD-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rax
+ ; SLOW-SHLD-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rbx
+ ; SLOW-SHLD-NEXT: [[SHR64ri:%[0-9]+]]:gr64 = SHR64ri [[COPY]], 2, implicit-def $eflags
+ ; SLOW-SHLD-NEXT: [[SHL64ri:%[0-9]+]]:gr64 = SHL64ri [[COPY1]], 62, implicit-def $eflags
+ ; SLOW-SHLD-NEXT: [[OR64rr:%[0-9]+]]:gr64 = OR64rr killed [[SHR64ri]], killed [[SHL64ri]], implicit-def $eflags
+ ; SLOW-SHLD-NEXT: $rax = COPY [[OR64rr]]
+ ; SLOW-SHLD-NEXT: RET 0, implicit $rax
+ %0:gr64 = COPY $rax
+ %1:gr64 = COPY $rbx
+ %2:gr64 = SHRD64rri8 %0, %1, 2, implicit-def $eflags
+ $rax = COPY %2
+ RET 0, implicit $rax
+...
+
+---
+name: shrd_mri32
+alignment: 16
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $rax, $ebx
+ ; FAST-SHLD-LABEL: name: shrd_mri32
+ ; FAST-SHLD: liveins: $rax, $ebx
+ ; FAST-SHLD-NEXT: {{ $}}
+ ; FAST-SHLD-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $ebx
+ ; FAST-SHLD-NEXT: SHRD32mri8 $rax, 1, $noreg, 0, $noreg, [[COPY]], 2, implicit-def $eflags
+ ; FAST-SHLD-NEXT: RET 0
+ ;
+ ; SLOW-SHLD-LABEL: name: shrd_mri32
+ ; SLOW-SHLD: liveins: $rax, $ebx
+ ; SLOW-SHLD-NEXT: {{ $}}
+ ; SLOW-SHLD-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $ebx
+ ; SLOW-SHLD-NEXT: SHR32mi $rax, 1, $noreg, 0, $noreg, 2, implicit-def $eflags
+ ; SLOW-SHLD-NEXT: [[SHL32ri:%[0-9]+]]:gr32 = SHL32ri [[COPY]], 30, implicit-def $eflags
+ ; SLOW-SHLD-NEXT: OR32mr $rax, 1, $noreg, 0, $noreg, killed [[SHL32ri]], implicit-def $eflags
+ ; SLOW-SHLD-NEXT: RET 0
+ %0:gr32 = COPY $ebx
+ SHRD32mri8 $rax, 1, $noreg, 0, $noreg, %0, 2, implicit-def $eflags
+ RET 0
+...
+
+---
+name: shrd_mri64
+alignment: 16
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $rax, $rbx
+ ; FAST-SHLD-LABEL: name: shrd_mri64
+ ; FAST-SHLD: liveins: $rax, $rbx
+ ; FAST-SHLD-NEXT: {{ $}}
+ ; FAST-SHLD-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rbx
+ ; FAST-SHLD-NEXT: SHRD64mri8 $rax, 1, $noreg, 0, $noreg, [[COPY]], 2, implicit-def $eflags
+ ; FAST-SHLD-NEXT: RET 0
+ ;
+ ; SLOW-SHLD-LABEL: name: shrd_mri64
+ ; SLOW-SHLD: liveins: $rax, $rbx
+ ; SLOW-SHLD-NEXT: {{ $}}
+ ; SLOW-SHLD-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rbx
+ ; SLOW-SHLD-NEXT: SHR64mi $rax, 1, $noreg, 0, $noreg, 2, implicit-def $eflags
+ ; SLOW-SHLD-NEXT: [[SHL64ri:%[0-9]+]]:gr64 = SHL64ri [[COPY]], 62, implicit-def $eflags
+ ; SLOW-SHLD-NEXT: OR64mr $rax, 1, $noreg, 0, $noreg, killed [[SHL64ri]], implicit-def $eflags
+ ; SLOW-SHLD-NEXT: RET 0
+ %0:gr64 = COPY $rbx
+ SHRD64mri8 $rax, 1, $noreg, 0, $noreg, %0, 2, implicit-def $eflags
+ RET 0
+...
diff --git a/llvm/test/CodeGen/X86/shift-and.ll b/llvm/test/CodeGen/X86/shift-and.ll
index a467720fbbe61..9fcab4c243b12 100644
--- a/llvm/test/CodeGen/X86/shift-and.ll
+++ b/llvm/test/CodeGen/X86/shift-and.ll
@@ -168,21 +168,22 @@ define void @t5ptr(i64 %t, ptr %ptr) nounwind {
define i64 @t6(i64 %key, ptr nocapture %val) nounwind {
; X86-LABEL: t6:
; X86: # %bb.0:
-; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT: shrdl $3, %edi, %esi
-; X86-NEXT: shrl $3, %edi
-; X86-NEXT: movl (%ecx), %eax
-; X86-NEXT: movl 4(%ecx), %edx
+; X86-NEXT: movl %esi, %ecx
+; X86-NEXT: shrl $3, %ecx
+; X86-NEXT: shrl $3, %eax
+; X86-NEXT: shll $29, %esi
+; X86-NEXT: orl %eax, %esi
+; X86-NEXT: movl (%edx), %eax
+; X86-NEXT: movl 4(%edx), %edx
; X86-NEXT: addl $-1, %eax
; X86-NEXT: adcl $-1, %edx
; X86-NEXT: andl %esi, %eax
-; X86-NEXT: andl %edi, %edx
+; X86-NEXT: andl %ecx, %edx
; X86-NEXT: popl %esi
-; X86-NEXT: popl %edi
; X86-NEXT: retl
;
; X64-LABEL: t6:
diff --git a/llvm/test/CodeGen/X86/shift-i256.ll b/llvm/test/CodeGen/X86/shift-i256.ll
index 8a160aef154bb..3cd6df8779db4 100644
--- a/llvm/test/CodeGen/X86/shift-i256.ll
+++ b/llvm/test/CodeGen/X86/shift-i256.ll
@@ -1511,17 +1511,23 @@ define i256 @lshr_i256_1(i256 %a0) nounwind {
; CHECK-LABEL: lshr_i256_1:
; CHECK: # %bb.0:
; CHECK-NEXT: movq %rdi, %rax
-; CHECK-NEXT: shrdq $1, %rdx, %rsi
-; CHECK-NEXT: movq %rcx, %rdi
+; CHECK-NEXT: shrq %rsi
+; CHECK-NEXT: movq %rdx, %rdi
; CHECK-NEXT: shlq $63, %rdi
+; CHECK-NEXT: orq %rsi, %rdi
+; CHECK-NEXT: movq %rcx, %rsi
+; CHECK-NEXT: shlq $63, %rsi
; CHECK-NEXT: shrq %rdx
-; CHECK-NEXT: orq %rdi, %rdx
-; CHECK-NEXT: shrdq $1, %r8, %rcx
+; CHECK-NEXT: orq %rsi, %rdx
+; CHECK-NEXT: shrq %rcx
+; CHECK-NEXT: movq %r8, %rsi
+; CHECK-NEXT: shlq $63, %rsi
+; CHECK-NEXT: orq %rcx, %rsi
; CHECK-NEXT: shrq %r8
; CHECK-NEXT: movq %r8, 24(%rax)
-; CHECK-NEXT: movq %rcx, 16(%rax)
+; CHECK-NEXT: movq %rsi, 16(%rax)
; CHECK-NEXT: movq %rdx, 8(%rax)
-; CHECK-NEXT: movq %rsi, (%rax)
+; CHECK-NEXT: movq %rdi, (%rax)
; CHECK-NEXT: retq
;
; X86-LABEL: lshr_i256_1:
@@ -1578,17 +1584,23 @@ define i256 @ashr_i256_1(i256 %a0) nounwind {
; CHECK-LABEL: ashr_i256_1:
; CHECK: # %bb.0:
; CHECK-NEXT: movq %rdi, %rax
-; CHECK-NEXT: shrdq $1, %rdx, %rsi
-; CHECK-NEXT: movq %rcx, %rdi
+; CHECK-NEXT: shrq %rsi
+; CHECK-NEXT: movq %rdx, %rdi
; CHECK-NEXT: shlq $63, %rdi
+; CHECK-NEXT: orq %rsi, %rdi
+; CHECK-NEXT: movq %rcx, %rsi
+; CHECK-NEXT: shlq $63, %rsi
; CHECK-NEXT: shrq %rdx
-; CHECK-NEXT: orq %rdi, %rdx
-; CHECK-NEXT: shrdq $1, %r8, %rcx
+; CHECK-NEXT: orq %rsi, %rdx
+; CHECK-NEXT: shrq %rcx
+; CHECK-NEXT: movq %r8, %rsi
+; CHECK-NEXT: shlq $63, %rsi
+; CHECK-NEXT: orq %rcx, %rsi
; CHECK-NEXT: sarq %r8
; CHECK-NEXT: movq %r8, 24(%rax)
-; CHECK-NEXT: movq %rcx, 16(%rax)
+; CHECK-NEXT: movq %rsi, 16(%rax)
; CHECK-NEXT: movq %rdx, 8(%rax)
-; CHECK-NEXT: movq %rsi, (%rax)
+; CHECK-NEXT: movq %rdi, (%rax)
; CHECK-NEXT: retq
;
; X86-LABEL: ashr_i256_1:
diff --git a/llvm/test/CodeGen/X86/shift-i512.ll b/llvm/test/CodeGen/X86/shift-i512.ll
index 381ef07291718..cc072a775f40b 100644
--- a/llvm/test/CodeGen/X86/shift-i512.ll
+++ b/llvm/test/CodeGen/X86/shift-i512.ll
@@ -1307,34 +1307,46 @@ define i512 @lshr_i512_1(i512 %a0) nounwind {
; CHECK: # %bb.0:
; CHECK-NEXT: pushq %rbx
; CHECK-NEXT: movq %rdi, %rax
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rdi
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rbx
; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r11
; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r10
-; CHECK-NEXT: shrdq $1, %rdx, %rsi
-; CHECK-NEXT: movq %rcx, %rbx
-; CHECK-NEXT: shlq $63, %rbx
+; CHECK-NEXT: shrq %rsi
+; CHECK-NEXT: movq %rdx, %rdi
+; CHECK-NEXT: shlq $63, %rdi
+; CHECK-NEXT: orq %rsi, %rdi
+; CHECK-NEXT: movq %rcx, %rsi
+; CHECK-NEXT: shlq $63, %rsi
; CHECK-NEXT: shrq %rdx
-; CHECK-NEXT: orq %rbx, %rdx
-; CHECK-NEXT: shrdq $1, %r8, %rcx
-; CHECK-NEXT: movq %r9, %rbx
-; CHECK-NEXT: shlq $63, %rbx
+; CHECK-NEXT: orq %rsi, %rdx
+; CHECK-NEXT: shrq %rcx
+; CHECK-NEXT: movq %r8, %rsi
+; CHECK-NEXT: shlq $63, %rsi
+; CHECK-NEXT: orq %rcx, %rsi
+; CHECK-NEXT: movq %r9, %rcx
+; CHECK-NEXT: shlq $63, %rcx
; CHECK-NEXT: shrq %r8
-; CHECK-NEXT: orq %rbx, %r8
-; CHECK-NEXT: shrdq $1, %r10, %r9
-; CHECK-NEXT: movq %rdi, %rbx
-; CHECK-NEXT: shlq $63, %rbx
+; CHECK-NEXT: orq %rcx, %r8
+; CHECK-NEXT: shrq %r9
+; CHECK-NEXT: movq %r10, %rcx
+; CHECK-NEXT: shlq $63, %rcx
+; CHECK-NEXT: orq %r9, %rcx
+; CHECK-NEXT: movq %rbx, %r9
+; CHECK-NEXT: shlq $63, %r9
; CHECK-NEXT: shrq %r10
-; CHECK-NEXT: orq %rbx, %r10
-; CHECK-NEXT: shrdq $1, %r11, %rdi
+; CHECK-NEXT: orq %r9, %r10
+; CHECK-NEXT: shrq %rbx
+; CHECK-NEXT: movq %r11, %r9
+; CHECK-NEXT: shlq $63, %r9
+; CHECK-NEXT: orq %rbx, %r9
; CHECK-NEXT: shrq %r11
; CHECK-NEXT: movq %r11, 56(%rax)
-; CHECK-NEXT: movq %rdi, 48(%rax)
+; CHECK-NEXT: movq %r9, 48(%rax)
; CHECK-NEXT: movq %r10, 40(%rax)
-; CHECK-NEXT: movq %r9, 32(%rax)
+; CHECK-NEXT: movq %rcx, 32(%rax)
; CHECK-NEXT: movq %r8, 24(%rax)
-; CHECK-NEXT: movq %rcx, 16(%rax)
+; CHECK-NEXT: movq %rsi, 16(%rax)
; CHECK-NEXT: movq %rdx, 8(%rax)
-; CHECK-NEXT: movq %rsi, (%rax)
+; CHECK-NEXT: movq %rdi, (%rax)
; CHECK-NEXT: popq %rbx
; CHECK-NEXT: retq
%r = lshr i512 %a0, 1
@@ -1346,34 +1358,46 @@ define i512 @ashr_i512_1(i512 %a0) nounwind {
; CHECK: # %bb.0:
; CHECK-NEXT: pushq %rbx
; CHECK-NEXT: movq %rdi, %rax
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rdi
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rbx
; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r11
; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r10
-; CHECK-NEXT: shrdq $1, %rdx, %rsi
-; CHECK-NEXT: movq %rcx, %rbx
-; CHECK-NEXT: shlq $63, %rbx
+; CHECK-NEXT: shrq %rsi
+; CHECK-NEXT: movq %rdx, %rdi
+; CHECK-NEXT: shlq $63, %rdi
+; CHECK-NEXT: orq %rsi, %rdi
+; CHECK-NEXT: movq %rcx, %rsi
+; CHECK-NEXT: shlq $63, %rsi
; CHECK-NEXT: shrq %rdx
-; CHECK-NEXT: orq %rbx, %rdx
-; CHECK-NEXT: shrdq $1, %r8, %rcx
-; CHECK-NEXT: movq %r9, %rbx
-; CHECK-NEXT: shlq $63, %rbx
+; CHECK-NEXT: orq %rsi, %rdx
+; CHECK-NEXT: shrq %rcx
+; CHECK-NEXT: movq %r8, %rsi
+; CHECK-NEXT: shlq $63, %rsi
+; CHECK-NEXT: orq %rcx, %rsi
+; CHECK-NEXT: movq %r9, %rcx
+; CHECK-NEXT: shlq $63, %rcx
; CHECK-NEXT: shrq %r8
-; CHECK-NEXT: orq %rbx, %r8
-; CHECK-NEXT: shrdq $1, %r10, %r9
-; CHECK-NEXT: movq %rdi, %rbx
-; CHECK-NEXT: shlq $63, %rbx
+; CHECK-NEXT: orq %rcx, %r8
+; CHECK-NEXT: shrq %r9
+; CHECK-NEXT: movq %r10, %rcx
+; CHECK-NEXT: shlq $63, %rcx
+; CHECK-NEXT: orq %r9, %rcx
+; CHECK-NEXT: movq %rbx, %r9
+; CHECK-NEXT: shlq $63, %r9
; CHECK-NEXT: shrq %r10
-; CHECK-NEXT: orq %rbx, %r10
-; CHECK-NEXT: shrdq $1, %r11, %rdi
+; CHECK-NEXT: orq %r9, %r10
+; CHECK-NEXT: shrq %rbx
+; CHECK-NEXT: movq %r11, %r9
+; CHECK-NEXT: shlq $63, %r9
+; CHECK-NEXT: orq %rbx, %r9
; CHECK-NEXT: sarq %r11
; CHECK-NEXT: movq %r11, 56(%rax)
-; CHECK-NEXT: movq %rdi, 48(%rax)
+; CHECK-NEXT: movq %r9, 48(%rax)
; CHECK-NEXT: movq %r10, 40(%rax)
-; CHECK-NEXT: movq %r9, 32(%rax)
+; CHECK-NEXT: movq %rcx, 32(%rax)
; CHECK-NEXT: movq %r8, 24(%rax)
-; CHECK-NEXT: movq %rcx, 16(%rax)
+; CHECK-NEXT: movq %rsi, 16(%rax)
; CHECK-NEXT: movq %rdx, 8(%rax)
-; CHECK-NEXT: movq %rsi, (%rax)
+; CHECK-NEXT: movq %rdi, (%rax)
; CHECK-NEXT: popq %rbx
; CHECK-NEXT: retq
%r = ashr i512 %a0, 1
@@ -1383,20 +1407,26 @@ define i512 @ashr_i512_1(i512 %a0) nounwind {
define i512 @shl_i512_200(i512 %a0) nounwind {
; SSE-LABEL: shl_i512_200:
; SSE: # %bb.0:
-; SSE-NEXT: movq %rsi, %r10
-; SSE-NEXT: shrdq $56, %rdx, %r10
; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: movq %rsi, %rdi
+; SSE-NEXT: shrq $56, %rdi
+; SSE-NEXT: movq %rdx, %r10
+; SSE-NEXT: shlq $8, %r10
+; SSE-NEXT: orq %rdi, %r10
; SSE-NEXT: movq %rcx, %rdi
; SSE-NEXT: shlq $8, %rdi
; SSE-NEXT: shrq $56, %rdx
; SSE-NEXT: orq %rdi, %rdx
-; SSE-NEXT: shrdq $56, %r8, %rcx
+; SSE-NEXT: shrq $56, %rcx
+; SSE-NEXT: movq %r8, %rdi
+; SSE-NEXT: shlq $8, %rdi
+; SSE-NEXT: orq %rcx, %rdi
; SSE-NEXT: shlq $8, %r9
; SSE-NEXT: shrq $56, %r8
; SSE-NEXT: orq %r9, %r8
; SSE-NEXT: shlq $8, %rsi
; SSE-NEXT: movq %r8, 56(%rax)
-; SSE-NEXT: movq %rcx, 48(%rax)
+; SSE-NEXT: movq %rdi, 48(%rax)
; SSE-NEXT: movq %rdx, 40(%rax)
; SSE-NEXT: movq %r10, 32(%rax)
; SSE-NEXT: movq %rsi, 24(%rax)
@@ -1409,20 +1439,26 @@ define i512 @shl_i512_200(i512 %a0) nounwind {
; AVX2: # %bb.0:
; AVX2-NEXT: movq %rdi, %rax
; AVX2-NEXT: movq %rsi, %rdi
-; AVX2-NEXT: shrdq $56, %rdx, %rdi
-; AVX2-NEXT: movq %rcx, %r10
+; AVX2-NEXT: shrq $56, %rdi
+; AVX2-NEXT: movq %rdx, %r10
; AVX2-NEXT: shlq $8, %r10
+; AVX2-NEXT: orq %rdi, %r10
+; AVX2-NEXT: movq %rcx, %rdi
+; AVX2-NEXT: shlq $8, %rdi
; AVX2-NEXT: shrq $56, %rdx
-; AVX2-NEXT: orq %r10, %rdx
-; AVX2-NEXT: shrdq $56, %r8, %rcx
+; AVX2-NEXT: orq %rdi, %rdx
+; AVX2-NEXT: shrq $56, %rcx
+; AVX2-NEXT: movq %r8, %rdi
+; AVX2-NEXT: shlq $8, %rdi
+; AVX2-NEXT: orq %rcx, %rdi
; AVX2-NEXT: shlq $8, %r9
; AVX2-NEXT: shrq $56, %r8
; AVX2-NEXT: orq %r9, %r8
; AVX2-NEXT: shlq $8, %rsi
; AVX2-NEXT: movq %r8, 56(%rax)
-; AVX2-NEXT: movq %rcx, 48(%rax)
+; AVX2-NEXT: movq %rdi, 48(%rax)
; AVX2-NEXT: movq %rdx, 40(%rax)
-; AVX2-NEXT: movq %rdi, 32(%rax)
+; AVX2-NEXT: movq %r10, 32(%rax)
; AVX2-NEXT: movq %rsi, 24(%rax)
; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX2-NEXT: vmovaps %xmm0, (%rax)
@@ -1433,20 +1469,26 @@ define i512 @shl_i512_200(i512 %a0) nounwind {
; AVX512: # %bb.0:
; AVX512-NEXT: movq %rdi, %rax
; AVX512-NEXT: movq %rsi, %rdi
-; AVX512-NEXT: shrdq $56, %rdx, %rdi
-; AVX512-NEXT: movq %rcx, %r10
+; AVX512-NEXT: shrq $56, %rdi
+; AVX512-NEXT: movq %rdx, %r10
; AVX512-NEXT: shlq $8, %r10
+; AVX512-NEXT: orq %rdi, %r10
+; AVX512-NEXT: movq %rcx, %rdi
+; AVX512-NEXT: shlq $8, %rdi
; AVX512-NEXT: shrq $56, %rdx
-; AVX512-NEXT: orq %r10, %rdx
-; AVX512-NEXT: shrdq $56, %r8, %rcx
+; AVX512-NEXT: orq %rdi, %rdx
+; AVX512-NEXT: shrq $56, %rcx
+; AVX512-NEXT: movq %r8, %rdi
+; AVX512-NEXT: shlq $8, %rdi
+; AVX512-NEXT: orq %rcx, %rdi
; AVX512-NEXT: shlq $8, %r9
; AVX512-NEXT: shrq $56, %r8
; AVX512-NEXT: orq %r9, %r8
; AVX512-NEXT: shlq $8, %rsi
; AVX512-NEXT: movq %r8, 56(%rax)
-; AVX512-NEXT: movq %rcx, 48(%rax)
+; AVX512-NEXT: movq %rdi, 48(%rax)
; AVX512-NEXT: movq %rdx, 40(%rax)
-; AVX512-NEXT: movq %rdi, 32(%rax)
+; AVX512-NEXT: movq %r10, 32(%rax)
; AVX512-NEXT: movq %rsi, 24(%rax)
; AVX512-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX512-NEXT: vmovaps %xmm0, (%rax)
diff --git a/llvm/test/CodeGen/X86/shift-mask.ll b/llvm/test/CodeGen/X86/shift-mask.ll
index 09dcc8cb52118..3d4f7f1d48575 100644
--- a/llvm/test/CodeGen/X86/shift-mask.ll
+++ b/llvm/test/CodeGen/X86/shift-mask.ll
@@ -587,9 +587,12 @@ define i64 @test_i64_lshr_lshr_0(i64 %a0) {
define i64 @test_i64_lshr_lshr_1(i64 %a0) {
; X86-LABEL: test_i64_lshr_lshr_1:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: shrdl $2, %edx, %eax
+; X86-NEXT: shrl $2, %ecx
+; X86-NEXT: movl %edx, %eax
+; X86-NEXT: shll $30, %eax
+; X86-NEXT: orl %ecx, %eax
; X86-NEXT: shrl $2, %edx
; X86-NEXT: andl $134217727, %edx # imm = 0x7FFFFFF
; X86-NEXT: retl
diff --git a/llvm/test/CodeGen/X86/smul_fix.ll b/llvm/test/CodeGen/X86/smul_fix.ll
index 8cb032776114b..aeee992dc7937 100644
--- a/llvm/test/CodeGen/X86/smul_fix.ll
+++ b/llvm/test/CodeGen/X86/smul_fix.ll
@@ -80,7 +80,9 @@ define i64 @func2(i64 %x, i64 %y) {
; X86-NEXT: testl %ebx, %ebx
; X86-NEXT: cmovsl %ecx, %edx
; X86-NEXT: shldl $30, %eax, %edx
-; X86-NEXT: shldl $30, %esi, %eax
+; X86-NEXT: shrl $2, %esi
+; X86-NEXT: shll $30, %eax
+; X86-NEXT: orl %esi, %eax
; X86-NEXT: popl %esi
; X86-NEXT: .cfi_def_cfa_offset 16
; X86-NEXT: popl %edi
@@ -390,20 +392,22 @@ define i64 @func8(i64 %x, i64 %y) nounwind {
; X86-NEXT: adcl $0, %ebx
; X86-NEXT: movl %edi, %ecx
; X86-NEXT: subl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ebx, %esi
-; X86-NEXT: sbbl $0, %esi
-; X86-NEXT: cmpl $0, {{[0-9]+}}(%esp)
-; X86-NEXT: cmovnsl %ebx, %esi
+; X86-NEXT: movl %ebx, %ebp
+; X86-NEXT: sbbl $0, %ebp
+; X86-NEXT: testl %esi, %esi
+; X86-NEXT: cmovnsl %ebx, %ebp
; X86-NEXT: cmovnsl %edi, %ecx
-; X86-NEXT: movl %ecx, %edi
-; X86-NEXT: subl {{[0-9]+}}(%esp), %edi
-; X86-NEXT: movl %esi, %edx
+; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: subl {{[0-9]+}}(%esp), %esi
+; X86-NEXT: movl %ebp, %edx
; X86-NEXT: sbbl $0, %edx
; X86-NEXT: cmpl $0, {{[0-9]+}}(%esp)
-; X86-NEXT: cmovnsl %esi, %edx
-; X86-NEXT: cmovnsl %ecx, %edi
-; X86-NEXT: shldl $1, %edi, %edx
-; X86-NEXT: shrdl $31, %edi, %eax
+; X86-NEXT: cmovnsl %ebp, %edx
+; X86-NEXT: cmovnsl %ecx, %esi
+; X86-NEXT: shldl $1, %esi, %edx
+; X86-NEXT: shrl $31, %eax
+; X86-NEXT: shll %esi
+; X86-NEXT: orl %esi, %eax
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
; X86-NEXT: popl %ebx
diff --git a/llvm/test/CodeGen/X86/smul_fix_sat.ll b/llvm/test/CodeGen/X86/smul_fix_sat.ll
index e68b6e328b723..82143cfbbbdd0 100644
--- a/llvm/test/CodeGen/X86/smul_fix_sat.ll
+++ b/llvm/test/CodeGen/X86/smul_fix_sat.ll
@@ -60,73 +60,77 @@ define i64 @func2(i64 %x, i64 %y) nounwind {
; X86-NEXT: pushl %ebx
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
-; X86-NEXT: subl $8, %esp
+; X86-NEXT: pushl %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx
; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: mull %esi
; X86-NEXT: movl %edx, %edi
; X86-NEXT: movl %eax, %ebp
; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: mull %ebx
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: mull {{[0-9]+}}(%esp)
+; X86-NEXT: movl %eax, (%esp) # 4-byte Spill
; X86-NEXT: addl %edx, %ebp
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: adcl $0, %edi
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: imull %esi
; X86-NEXT: movl %edx, %esi
-; X86-NEXT: movl %eax, (%esp) # 4-byte Spill
-; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: mull %ebx
; X86-NEXT: movl %eax, %ebx
-; X86-NEXT: addl %ebp, %ebx
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: mull {{[0-9]+}}(%esp)
+; X86-NEXT: addl %ebp, %eax
; X86-NEXT: adcl %edi, %edx
; X86-NEXT: adcl $0, %esi
-; X86-NEXT: addl (%esp), %edx # 4-byte Folded Reload
+; X86-NEXT: addl %ebx, %edx
; X86-NEXT: adcl $0, %esi
-; X86-NEXT: movl %edx, %edi
-; X86-NEXT: subl {{[0-9]+}}(%esp), %edi
+; X86-NEXT: movl %edx, %ebx
+; X86-NEXT: subl {{[0-9]+}}(%esp), %ebx
; X86-NEXT: movl %esi, %ebp
; X86-NEXT: sbbl $0, %ebp
; X86-NEXT: testl %ecx, %ecx
; X86-NEXT: cmovnsl %esi, %ebp
-; X86-NEXT: cmovnsl %edx, %edi
-; X86-NEXT: movl %edi, %ecx
+; X86-NEXT: cmovnsl %edx, %ebx
+; X86-NEXT: movl %ebx, %ecx
; X86-NEXT: subl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ebp, %edx
-; X86-NEXT: sbbl $0, %edx
+; X86-NEXT: movl %ebp, %edi
+; X86-NEXT: sbbl $0, %edi
; X86-NEXT: cmpl $0, {{[0-9]+}}(%esp)
-; X86-NEXT: cmovnsl %ebp, %edx
-; X86-NEXT: cmovnsl %edi, %ecx
-; X86-NEXT: testl %edx, %edx
-; X86-NEXT: setg %ah
-; X86-NEXT: sete (%esp) # 1-byte Folded Spill
+; X86-NEXT: cmovnsl %ebp, %edi
+; X86-NEXT: cmovnsl %ebx, %ecx
+; X86-NEXT: testl %edi, %edi
+; X86-NEXT: setg %bl
+; X86-NEXT: sete %bh
; X86-NEXT: cmpl $2, %ecx
-; X86-NEXT: setae %al
-; X86-NEXT: andb (%esp), %al # 1-byte Folded Reload
-; X86-NEXT: orb %ah, %al
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X86-NEXT: shrdl $2, %ebx, %ebp
-; X86-NEXT: shrdl $2, %ecx, %ebx
-; X86-NEXT: testb %al, %al
+; X86-NEXT: setae %dl
+; X86-NEXT: movl %eax, %esi
+; X86-NEXT: shrl $2, %esi
+; X86-NEXT: movl %ecx, %ebp
+; X86-NEXT: shll $30, %ebp
+; X86-NEXT: orl %esi, %ebp
+; X86-NEXT: movl (%esp), %esi # 4-byte Reload
+; X86-NEXT: shrl $2, %esi
+; X86-NEXT: shll $30, %eax
+; X86-NEXT: orl %esi, %eax
+; X86-NEXT: andb %bh, %dl
+; X86-NEXT: orb %bl, %dl
+; X86-NEXT: testb %dl, %dl
; X86-NEXT: movl $2147483647, %esi # imm = 0x7FFFFFFF
-; X86-NEXT: cmovel %ebx, %esi
-; X86-NEXT: movl $-1, %edi
-; X86-NEXT: cmovel %ebp, %edi
-; X86-NEXT: cmpl $-1, %edx
+; X86-NEXT: cmovel %ebp, %esi
+; X86-NEXT: movl $-1, %edx
+; X86-NEXT: cmovnel %edx, %eax
+; X86-NEXT: cmpl $-1, %edi
; X86-NEXT: setl %dl
-; X86-NEXT: sete %al
+; X86-NEXT: sete %dh
; X86-NEXT: cmpl $-2, %ecx
; X86-NEXT: setb %cl
-; X86-NEXT: andb %al, %cl
-; X86-NEXT: xorl %eax, %eax
+; X86-NEXT: andb %dh, %cl
+; X86-NEXT: xorl %edi, %edi
; X86-NEXT: orb %dl, %cl
-; X86-NEXT: cmovel %edi, %eax
+; X86-NEXT: cmovnel %edi, %eax
; X86-NEXT: movl $-2147483648, %edx # imm = 0x80000000
; X86-NEXT: cmovel %esi, %edx
-; X86-NEXT: addl $8, %esp
+; X86-NEXT: addl $4, %esp
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
; X86-NEXT: popl %ebx
@@ -233,12 +237,15 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
; X64-NEXT: imulq %rdx, %rsi
; X64-NEXT: movq %rsi, %rdx
; X64-NEXT: shrq $32, %rdx
-; X64-NEXT: shrdl $2, %edx, %esi
+; X64-NEXT: shrl $2, %esi
+; X64-NEXT: movl %edx, %edi
+; X64-NEXT: shll $30, %edi
+; X64-NEXT: orl %esi, %edi
; X64-NEXT: cmpl $2, %edx
-; X64-NEXT: cmovgel %eax, %esi
+; X64-NEXT: cmovgel %eax, %edi
; X64-NEXT: cmpl $-2, %edx
-; X64-NEXT: cmovll %ecx, %esi
-; X64-NEXT: movd %esi, %xmm2
+; X64-NEXT: cmovll %ecx, %edi
+; X64-NEXT: movd %edi, %xmm2
; X64-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
; X64-NEXT: movd %xmm1, %edx
; X64-NEXT: movslq %edx, %rdx
@@ -737,18 +744,21 @@ define i64 @func8(i64 %x, i64 %y) nounwind {
; X86-NEXT: cmpl $0, {{[0-9]+}}(%esp)
; X86-NEXT: cmovnsl %ebp, %ecx
; X86-NEXT: cmovnsl %edx, %esi
-; X86-NEXT: shrdl $31, %esi, %eax
-; X86-NEXT: shrdl $31, %ecx, %esi
+; X86-NEXT: movl %esi, %edx
+; X86-NEXT: shrdl $31, %ecx, %edx
+; X86-NEXT: shrl $31, %eax
+; X86-NEXT: shll %esi
+; X86-NEXT: orl %esi, %eax
; X86-NEXT: cmpl $1073741824, %ecx # imm = 0x40000000
-; X86-NEXT: movl $2147483647, %edi # imm = 0x7FFFFFFF
-; X86-NEXT: cmovll %esi, %edi
+; X86-NEXT: movl $2147483647, %esi # imm = 0x7FFFFFFF
+; X86-NEXT: cmovll %edx, %esi
; X86-NEXT: movl $-1, %edx
; X86-NEXT: cmovgel %edx, %eax
; X86-NEXT: xorl %edx, %edx
; X86-NEXT: cmpl $-1073741824, %ecx # imm = 0xC0000000
; X86-NEXT: cmovll %edx, %eax
; X86-NEXT: movl $-2147483648, %edx # imm = 0x80000000
-; X86-NEXT: cmovgel %edi, %edx
+; X86-NEXT: cmovgel %esi, %edx
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
; X86-NEXT: popl %ebx
diff --git a/llvm/test/CodeGen/X86/udiv_fix_sat.ll b/llvm/test/CodeGen/X86/udiv_fix_sat.ll
index 421426f3ad959..5b432a2374f76 100644
--- a/llvm/test/CodeGen/X86/udiv_fix_sat.ll
+++ b/llvm/test/CodeGen/X86/udiv_fix_sat.ll
@@ -387,9 +387,9 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
; X86-NEXT: subl $8, %esp
-; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ebp
; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx
; X86-NEXT: pushl $0
; X86-NEXT: pushl {{[0-9]+}}(%esp)
@@ -401,36 +401,41 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
; X86-NEXT: movl %edx, (%esp) # 4-byte Spill
; X86-NEXT: pushl $0
; X86-NEXT: pushl %ebx
-; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %esi
; X86-NEXT: pushl $0
; X86-NEXT: calll __udivdi3
; X86-NEXT: addl $16, %esp
-; X86-NEXT: movl %eax, %ebx
-; X86-NEXT: movl %edx, %edi
+; X86-NEXT: movl %eax, %esi
+; X86-NEXT: movl %edx, %ebx
; X86-NEXT: pushl $0
; X86-NEXT: pushl %ebp
-; X86-NEXT: pushl %esi
+; X86-NEXT: pushl %edi
; X86-NEXT: pushl $0
; X86-NEXT: calll __udivdi3
; X86-NEXT: addl $16, %esp
-; X86-NEXT: movl %eax, %esi
+; X86-NEXT: movl %edx, %ebp
; X86-NEXT: cmpl $2, %edx
-; X86-NEXT: movl $1, %ebp
-; X86-NEXT: cmovael %ebp, %edx
-; X86-NEXT: movl $-1, %eax
-; X86-NEXT: cmovael %eax, %esi
-; X86-NEXT: shrdl $1, %edx, %esi
-; X86-NEXT: cmpl $2, %edi
-; X86-NEXT: cmovael %ebp, %edi
+; X86-NEXT: movl $1, %edi
+; X86-NEXT: cmovael %edi, %ebp
; X86-NEXT: movl $-1, %ecx
-; X86-NEXT: cmovael %ecx, %ebx
-; X86-NEXT: shrdl $1, %edi, %ebx
-; X86-NEXT: movl (%esp), %eax # 4-byte Reload
-; X86-NEXT: cmpl $2, %eax
-; X86-NEXT: cmovael %ebp, %eax
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: cmovael %ecx, %edi
-; X86-NEXT: shrdl $1, %eax, %edi
+; X86-NEXT: cmovael %ecx, %eax
+; X86-NEXT: shrl %eax
+; X86-NEXT: shll $31, %ebp
+; X86-NEXT: orl %eax, %ebp
+; X86-NEXT: cmpl $2, %ebx
+; X86-NEXT: cmovael %edi, %ebx
+; X86-NEXT: cmovael %ecx, %esi
+; X86-NEXT: shrl %esi
+; X86-NEXT: shll $31, %ebx
+; X86-NEXT: orl %esi, %ebx
+; X86-NEXT: movl (%esp), %esi # 4-byte Reload
+; X86-NEXT: cmpl $2, %esi
+; X86-NEXT: cmovael %edi, %esi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: cmovael %ecx, %eax
+; X86-NEXT: shrl %eax
+; X86-NEXT: shll $31, %esi
+; X86-NEXT: orl %eax, %esi
; X86-NEXT: pushl $0
; X86-NEXT: pushl {{[0-9]+}}(%esp)
; X86-NEXT: pushl {{[0-9]+}}(%esp)
@@ -438,16 +443,17 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
; X86-NEXT: calll __udivdi3
; X86-NEXT: addl $16, %esp
; X86-NEXT: cmpl $2, %edx
-; X86-NEXT: cmovbl %edx, %ebp
+; X86-NEXT: cmovbl %edx, %edi
; X86-NEXT: movl $-1, %ecx
; X86-NEXT: cmovael %ecx, %eax
-; X86-NEXT: shrdl $1, %ebp, %eax
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %eax, 12(%ecx)
-; X86-NEXT: movl %edi, 8(%ecx)
-; X86-NEXT: movl %ebx, 4(%ecx)
-; X86-NEXT: movl %esi, (%ecx)
-; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: shrl %eax
+; X86-NEXT: shll $31, %edi
+; X86-NEXT: orl %eax, %edi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %edi, 12(%eax)
+; X86-NEXT: movl %esi, 8(%eax)
+; X86-NEXT: movl %ebx, 4(%eax)
+; X86-NEXT: movl %ebp, (%eax)
; X86-NEXT: addl $8, %esp
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
diff --git a/llvm/test/CodeGen/X86/umul_fix.ll b/llvm/test/CodeGen/X86/umul_fix.ll
index 5a68484596a2f..d8b0052d09f72 100644
--- a/llvm/test/CodeGen/X86/umul_fix.ll
+++ b/llvm/test/CodeGen/X86/umul_fix.ll
@@ -62,7 +62,9 @@ define i64 @func2(i64 %x, i64 %y) nounwind {
; X86-NEXT: imull {{[0-9]+}}(%esp), %ecx
; X86-NEXT: addl %ecx, %edx
; X86-NEXT: shldl $30, %eax, %edx
-; X86-NEXT: shldl $30, %esi, %eax
+; X86-NEXT: shrl $2, %esi
+; X86-NEXT: shll $30, %eax
+; X86-NEXT: orl %esi, %eax
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
; X86-NEXT: popl %ebx
@@ -329,7 +331,9 @@ define i64 @func8(i64 %x, i64 %y) nounwind {
; X86-NEXT: addl %ebp, %ecx
; X86-NEXT: adcl $0, %esi
; X86-NEXT: shldl $1, %ecx, %esi
-; X86-NEXT: shrdl $31, %ecx, %eax
+; X86-NEXT: shrl $31, %eax
+; X86-NEXT: shll %ecx
+; X86-NEXT: orl %ecx, %eax
; X86-NEXT: movl %esi, %edx
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
diff --git a/llvm/test/CodeGen/X86/umul_fix_sat.ll b/llvm/test/CodeGen/X86/umul_fix_sat.ll
index 8c7078c726328..4120c21eaecd9 100644
--- a/llvm/test/CodeGen/X86/umul_fix_sat.ll
+++ b/llvm/test/CodeGen/X86/umul_fix_sat.ll
@@ -52,35 +52,41 @@ define i64 @func2(i64 %x, i64 %y) nounwind {
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: mull %edi
-; X86-NEXT: movl %edx, %esi
+; X86-NEXT: mull %esi
+; X86-NEXT: movl %edx, %edi
; X86-NEXT: movl %eax, %ebx
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: mull {{[0-9]+}}(%esp)
; X86-NEXT: movl %eax, %ecx
; X86-NEXT: addl %edx, %ebx
-; X86-NEXT: adcl $0, %esi
+; X86-NEXT: adcl $0, %edi
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: mull %edi
-; X86-NEXT: movl %edx, %edi
+; X86-NEXT: mull %esi
+; X86-NEXT: movl %edx, %esi
; X86-NEXT: movl %eax, %ebp
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: mull {{[0-9]+}}(%esp)
; X86-NEXT: addl %ebx, %eax
-; X86-NEXT: adcl %esi, %edx
-; X86-NEXT: adcl $0, %edi
+; X86-NEXT: adcl %edi, %edx
+; X86-NEXT: adcl $0, %esi
; X86-NEXT: addl %ebp, %edx
-; X86-NEXT: adcl $0, %edi
-; X86-NEXT: shrdl $2, %eax, %ecx
-; X86-NEXT: shrdl $2, %edx, %eax
+; X86-NEXT: adcl $0, %esi
+; X86-NEXT: movl %eax, %ebx
+; X86-NEXT: shrl $2, %ebx
+; X86-NEXT: movl %edx, %edi
+; X86-NEXT: shll $30, %edi
+; X86-NEXT: orl %ebx, %edi
; X86-NEXT: shrl $2, %edx
-; X86-NEXT: orl %edi, %edx
-; X86-NEXT: movl $-1, %edx
-; X86-NEXT: cmovnel %edx, %ecx
-; X86-NEXT: cmovel %eax, %edx
-; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: shrl $2, %ecx
+; X86-NEXT: shll $30, %eax
+; X86-NEXT: orl %ecx, %eax
+; X86-NEXT: orl %esi, %edx
+; X86-NEXT: movl $-1, %ecx
+; X86-NEXT: cmovnel %ecx, %eax
+; X86-NEXT: cmovnel %ecx, %edi
+; X86-NEXT: movl %edi, %edx
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
; X86-NEXT: popl %ebx
@@ -167,10 +173,13 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
; X64-NEXT: imulq %rcx, %rdx
; X64-NEXT: movq %rdx, %rcx
; X64-NEXT: shrq $32, %rcx
-; X64-NEXT: shrdl $2, %ecx, %edx
+; X64-NEXT: shrl $2, %edx
+; X64-NEXT: movl %ecx, %esi
+; X64-NEXT: shll $30, %esi
+; X64-NEXT: orl %edx, %esi
; X64-NEXT: cmpl $4, %ecx
-; X64-NEXT: cmovael %eax, %edx
-; X64-NEXT: movd %edx, %xmm2
+; X64-NEXT: cmovael %eax, %esi
+; X64-NEXT: movd %esi, %xmm2
; X64-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
; X64-NEXT: movd %xmm1, %ecx
; X64-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
@@ -518,7 +527,9 @@ define i64 @func8(i64 %x, i64 %y) nounwind {
; X86-NEXT: addl %ebp, %edx
; X86-NEXT: adcl $0, %ecx
; X86-NEXT: shldl $1, %edx, %ecx
-; X86-NEXT: shrdl $31, %edx, %eax
+; X86-NEXT: shrl $31, %eax
+; X86-NEXT: leal (,%edx,2), %esi
+; X86-NEXT: orl %esi, %eax
; X86-NEXT: testl $-2147483648, %edx # imm = 0x80000000
; X86-NEXT: movl $-1, %edx
; X86-NEXT: cmovnel %edx, %eax
diff --git a/llvm/test/CodeGen/X86/vector-sext.ll b/llvm/test/CodeGen/X86/vector-sext.ll
index 693255e80b353..239ff0075ded0 100644
--- a/llvm/test/CodeGen/X86/vector-sext.ll
+++ b/llvm/test/CodeGen/X86/vector-sext.ll
@@ -3649,45 +3649,51 @@ define <4 x i32> @sext_4i17_to_4i32(ptr %ptr) {
; SSE41-LABEL: sext_4i17_to_4i32:
; SSE41: # %bb.0:
; SSE41-NEXT: movq (%rdi), %rax
-; SSE41-NEXT: movd %eax, %xmm0
; SSE41-NEXT: movq %rax, %rcx
-; SSE41-NEXT: movl 8(%rdi), %edx
-; SSE41-NEXT: shldq $13, %rax, %rdx
-; SSE41-NEXT: shrq $17, %rax
-; SSE41-NEXT: shll $15, %eax
-; SSE41-NEXT: sarl $15, %eax
+; SSE41-NEXT: shrq $17, %rcx
+; SSE41-NEXT: shll $15, %ecx
+; SSE41-NEXT: sarl $15, %ecx
+; SSE41-NEXT: movd %eax, %xmm0
; SSE41-NEXT: pslld $15, %xmm0
; SSE41-NEXT: psrad $15, %xmm0
-; SSE41-NEXT: pinsrd $1, %eax, %xmm0
+; SSE41-NEXT: pinsrd $1, %ecx, %xmm0
+; SSE41-NEXT: movq %rax, %rcx
; SSE41-NEXT: shrq $34, %rcx
; SSE41-NEXT: shll $15, %ecx
; SSE41-NEXT: sarl $15, %ecx
; SSE41-NEXT: pinsrd $2, %ecx, %xmm0
-; SSE41-NEXT: shll $15, %edx
-; SSE41-NEXT: sarl $15, %edx
-; SSE41-NEXT: pinsrd $3, %edx, %xmm0
+; SSE41-NEXT: movl 8(%rdi), %ecx
+; SSE41-NEXT: shrq $51, %rax
+; SSE41-NEXT: shlq $13, %rcx
+; SSE41-NEXT: orq %rax, %rcx
+; SSE41-NEXT: shll $15, %ecx
+; SSE41-NEXT: sarl $15, %ecx
+; SSE41-NEXT: pinsrd $3, %ecx, %xmm0
; SSE41-NEXT: retq
;
; AVX-LABEL: sext_4i17_to_4i32:
; AVX: # %bb.0:
; AVX-NEXT: movq (%rdi), %rax
-; AVX-NEXT: vmovd %eax, %xmm0
; AVX-NEXT: movq %rax, %rcx
-; AVX-NEXT: movl 8(%rdi), %edx
-; AVX-NEXT: shldq $13, %rax, %rdx
-; AVX-NEXT: shrq $17, %rax
-; AVX-NEXT: shll $15, %eax
-; AVX-NEXT: sarl $15, %eax
+; AVX-NEXT: shrq $17, %rcx
+; AVX-NEXT: shll $15, %ecx
+; AVX-NEXT: sarl $15, %ecx
+; AVX-NEXT: vmovd %eax, %xmm0
; AVX-NEXT: vpslld $15, %xmm0, %xmm0
; AVX-NEXT: vpsrad $15, %xmm0, %xmm0
-; AVX-NEXT: vpinsrd $1, %eax, %xmm0, %xmm0
+; AVX-NEXT: vpinsrd $1, %ecx, %xmm0, %xmm0
+; AVX-NEXT: movq %rax, %rcx
; AVX-NEXT: shrq $34, %rcx
; AVX-NEXT: shll $15, %ecx
; AVX-NEXT: sarl $15, %ecx
; AVX-NEXT: vpinsrd $2, %ecx, %xmm0, %xmm0
-; AVX-NEXT: shll $15, %edx
-; AVX-NEXT: sarl $15, %edx
-; AVX-NEXT: vpinsrd $3, %edx, %xmm0, %xmm0
+; AVX-NEXT: movl 8(%rdi), %ecx
+; AVX-NEXT: shrq $51, %rax
+; AVX-NEXT: shlq $13, %rcx
+; AVX-NEXT: orq %rax, %rcx
+; AVX-NEXT: shll $15, %ecx
+; AVX-NEXT: sarl $15, %ecx
+; AVX-NEXT: vpinsrd $3, %ecx, %xmm0, %xmm0
; AVX-NEXT: retq
;
; X86-SSE2-LABEL: sext_4i17_to_4i32:
diff --git a/llvm/test/CodeGen/X86/vector-zext.ll b/llvm/test/CodeGen/X86/vector-zext.ll
index e061e4e35e1e7..edb00220d8a81 100644
--- a/llvm/test/CodeGen/X86/vector-zext.ll
+++ b/llvm/test/CodeGen/X86/vector-zext.ll
@@ -2362,45 +2362,54 @@ define <4 x i32> @zext_4i17_to_4i32(ptr %ptr) {
; SSE41-LABEL: zext_4i17_to_4i32:
; SSE41: # %bb.0:
; SSE41-NEXT: movq (%rdi), %rax
+; SSE41-NEXT: movq %rax, %rcx
+; SSE41-NEXT: shrq $17, %rcx
; SSE41-NEXT: movd %eax, %xmm0
+; SSE41-NEXT: pinsrd $1, %ecx, %xmm0
; SSE41-NEXT: movq %rax, %rcx
-; SSE41-NEXT: movl 8(%rdi), %edx
-; SSE41-NEXT: shldq $13, %rax, %rdx
-; SSE41-NEXT: shrq $17, %rax
-; SSE41-NEXT: pinsrd $1, %eax, %xmm0
; SSE41-NEXT: shrq $34, %rcx
; SSE41-NEXT: pinsrd $2, %ecx, %xmm0
-; SSE41-NEXT: pinsrd $3, %edx, %xmm0
+; SSE41-NEXT: movl 8(%rdi), %ecx
+; SSE41-NEXT: shrq $51, %rax
+; SSE41-NEXT: shlq $13, %rcx
+; SSE41-NEXT: orq %rax, %rcx
+; SSE41-NEXT: pinsrd $3, %ecx, %xmm0
; SSE41-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
; SSE41-NEXT: retq
;
; AVX1-LABEL: zext_4i17_to_4i32:
; AVX1: # %bb.0:
; AVX1-NEXT: movq (%rdi), %rax
+; AVX1-NEXT: movq %rax, %rcx
+; AVX1-NEXT: shrq $17, %rcx
; AVX1-NEXT: vmovd %eax, %xmm0
+; AVX1-NEXT: vpinsrd $1, %ecx, %xmm0, %xmm0
; AVX1-NEXT: movq %rax, %rcx
-; AVX1-NEXT: movl 8(%rdi), %edx
-; AVX1-NEXT: shldq $13, %rax, %rdx
-; AVX1-NEXT: shrq $17, %rax
-; AVX1-NEXT: vpinsrd $1, %eax, %xmm0, %xmm0
; AVX1-NEXT: shrq $34, %rcx
; AVX1-NEXT: vpinsrd $2, %ecx, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrd $3, %edx, %xmm0, %xmm0
+; AVX1-NEXT: movl 8(%rdi), %ecx
+; AVX1-NEXT: shrq $51, %rax
+; AVX1-NEXT: shlq $13, %rcx
+; AVX1-NEXT: orq %rax, %rcx
+; AVX1-NEXT: vpinsrd $3, %ecx, %xmm0, %xmm0
; AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
; AVX1-NEXT: retq
;
; AVX2-LABEL: zext_4i17_to_4i32:
; AVX2: # %bb.0:
; AVX2-NEXT: movq (%rdi), %rax
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: shrq $17, %rcx
; AVX2-NEXT: vmovd %eax, %xmm0
+; AVX2-NEXT: vpinsrd $1, %ecx, %xmm0, %xmm0
; AVX2-NEXT: movq %rax, %rcx
-; AVX2-NEXT: movl 8(%rdi), %edx
-; AVX2-NEXT: shldq $13, %rax, %rdx
-; AVX2-NEXT: shrq $17, %rax
-; AVX2-NEXT: vpinsrd $1, %eax, %xmm0, %xmm0
; AVX2-NEXT: shrq $34, %rcx
; AVX2-NEXT: vpinsrd $2, %ecx, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrd $3, %edx, %xmm0, %xmm0
+; AVX2-NEXT: movl 8(%rdi), %ecx
+; AVX2-NEXT: shrq $51, %rax
+; AVX2-NEXT: shlq $13, %rcx
+; AVX2-NEXT: orq %rax, %rcx
+; AVX2-NEXT: vpinsrd $3, %ecx, %xmm0, %xmm0
; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm1 = [131071,131071,131071,131071]
; AVX2-NEXT: vpand %xmm1, %xmm0, %xmm0
; AVX2-NEXT: retq
@@ -2408,15 +2417,18 @@ define <4 x i32> @zext_4i17_to_4i32(ptr %ptr) {
; AVX512-LABEL: zext_4i17_to_4i32:
; AVX512: # %bb.0:
; AVX512-NEXT: movq (%rdi), %rax
+; AVX512-NEXT: movq %rax, %rcx
+; AVX512-NEXT: shrq $17, %rcx
; AVX512-NEXT: vmovd %eax, %xmm0
+; AVX512-NEXT: vpinsrd $1, %ecx, %xmm0, %xmm0
; AVX512-NEXT: movq %rax, %rcx
-; AVX512-NEXT: movl 8(%rdi), %edx
-; AVX512-NEXT: shldq $13, %rax, %rdx
-; AVX512-NEXT: shrq $17, %rax
-; AVX512-NEXT: vpinsrd $1, %eax, %xmm0, %xmm0
; AVX512-NEXT: shrq $34, %rcx
; AVX512-NEXT: vpinsrd $2, %ecx, %xmm0, %xmm0
-; AVX512-NEXT: vpinsrd $3, %edx, %xmm0, %xmm0
+; AVX512-NEXT: movl 8(%rdi), %ecx
+; AVX512-NEXT: shrq $51, %rax
+; AVX512-NEXT: shlq $13, %rcx
+; AVX512-NEXT: orq %rax, %rcx
+; AVX512-NEXT: vpinsrd $3, %ecx, %xmm0, %xmm0
; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm1 = [131071,131071,131071,131071]
; AVX512-NEXT: vpand %xmm1, %xmm0, %xmm0
; AVX512-NEXT: retq
>From 4acc2dfabe572ab386753090009e39c157c97c32 Mon Sep 17 00:00:00 2001
From: stomfaig <stomfaig at gmail.com>
Date: Fri, 20 Mar 2026 14:12:30 +0000
Subject: [PATCH 07/11] add MustReduceLatency MC pattern
---
.../llvm/CodeGen/MachineCombinerPattern.h | 1 +
llvm/lib/CodeGen/MachineCombiner.cpp | 17 +++++++++++++----
2 files changed, 14 insertions(+), 4 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/MachineCombinerPattern.h b/llvm/include/llvm/CodeGen/MachineCombinerPattern.h
index 25fce679323ee..74216af9788d3 100644
--- a/llvm/include/llvm/CodeGen/MachineCombinerPattern.h
+++ b/llvm/include/llvm/CodeGen/MachineCombinerPattern.h
@@ -21,6 +21,7 @@ namespace llvm {
enum class CombinerObjective {
MustReduceDepth, // The data dependency chain must be improved.
MustReduceRegisterPressure, // The register pressure must be reduced.
+ MustReduceLatency, // The new pattern must have lower latency.
Default // The critical path must not be lengthened.
};
diff --git a/llvm/lib/CodeGen/MachineCombiner.cpp b/llvm/lib/CodeGen/MachineCombiner.cpp
index b86e70f265786..09a0b1a65203b 100644
--- a/llvm/lib/CodeGen/MachineCombiner.cpp
+++ b/llvm/lib/CodeGen/MachineCombiner.cpp
@@ -396,13 +396,22 @@ bool MachineCombiner::improvesCriticalPathLen(
<< "\n\tNewRootDepth + NewRootLatency = " << NewCycleCount
<< "\n\tRootDepth + RootLatency + RootSlack = "
<< OldCycleCount);
- LLVM_DEBUG(NewCycleCount <= OldCycleCount
- ? dbgs() << "\n\t It IMPROVES PathLen because"
- : dbgs() << "\n\t It DOES NOT improve PathLen because");
+
+ bool IsStrictImprovementRequired =
+ getCombinerObjective(Pattern) == CombinerObjective::MustReduceLatency;
+ bool Improves = IsStrictImprovementRequired ? NewCycleCount < OldCycleCount
+ : NewCycleCount <= OldCycleCount;
+
+ if (IsStrictImprovementRequired)
+ LLVM_DEBUG(
+ dbgs() << "\n\t (MustReduceLatency: strict improvement required)");
+
+ LLVM_DEBUG(Improves ? dbgs() << "\n\t It IMPROVES PathLen because"
+ : dbgs() << "\n\t It DOES NOT improve PathLen because");
LLVM_DEBUG(dbgs() << "\n\t\tNewCycleCount = " << NewCycleCount
<< ", OldCycleCount = " << OldCycleCount << "\n");
- return NewCycleCount <= OldCycleCount;
+ return Improves;
}
/// helper routine to convert instructions into SC
>From a0a9ee0b8c49c929fdaaddfd0a64eabb5f359098 Mon Sep 17 00:00:00 2001
From: stomfaig <stomfaig at gmail.com>
Date: Tue, 24 Mar 2026 17:13:28 +0000
Subject: [PATCH 08/11] stricter MustReduceLatency objective
---
llvm/lib/CodeGen/MachineCombiner.cpp | 19 ++++++++++---------
llvm/lib/Target/X86/X86InstrInfo.cpp | 9 +++++++++
llvm/lib/Target/X86/X86InstrInfo.h | 2 ++
3 files changed, 21 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/CodeGen/MachineCombiner.cpp b/llvm/lib/CodeGen/MachineCombiner.cpp
index 09a0b1a65203b..1a8746ac713ae 100644
--- a/llvm/lib/CodeGen/MachineCombiner.cpp
+++ b/llvm/lib/CodeGen/MachineCombiner.cpp
@@ -397,19 +397,20 @@ bool MachineCombiner::improvesCriticalPathLen(
<< "\n\tRootDepth + RootLatency + RootSlack = "
<< OldCycleCount);
- bool IsStrictImprovementRequired =
+ bool IsMustReduceLatency =
getCombinerObjective(Pattern) == CombinerObjective::MustReduceLatency;
- bool Improves = IsStrictImprovementRequired ? NewCycleCount < OldCycleCount
- : NewCycleCount <= OldCycleCount;
-
- if (IsStrictImprovementRequired)
- LLVM_DEBUG(
- dbgs() << "\n\t (MustReduceLatency: strict improvement required)");
-
+ unsigned CompareAgainst =
+ IsMustReduceLatency ? RootDepth + RootLatency : OldCycleCount;
+ bool Improves = IsMustReduceLatency ? NewCycleCount < CompareAgainst
+ : NewCycleCount <= CompareAgainst;
+
+ if (IsMustReduceLatency)
+ LLVM_DEBUG(dbgs() << "\n\t (MustReduceLatency: chain must be strictly "
+ "shorter, slack excluded)");
LLVM_DEBUG(Improves ? dbgs() << "\n\t It IMPROVES PathLen because"
: dbgs() << "\n\t It DOES NOT improve PathLen because");
LLVM_DEBUG(dbgs() << "\n\t\tNewCycleCount = " << NewCycleCount
- << ", OldCycleCount = " << OldCycleCount << "\n");
+ << ", OldCycleCount = " << CompareAgainst << "\n");
return Improves;
}
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index 07639eaf2c0c6..3f865acd4add3 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -10636,6 +10636,15 @@ void X86InstrInfo::buildClearRegister(Register Reg, MachineBasicBlock &MBB,
}
}
+CombinerObjective X86InstrInfo::getCombinerObjective(unsigned Pattern) const {
+ switch (Pattern) {
+ case X86MachineCombinerPattern::USHD:
+ return CombinerObjective::MustReduceLatency;
+ default:
+ return TargetInstrInfo::getCombinerObjective(Pattern);
+ }
+}
+
bool X86InstrInfo::getMachineCombinerPatterns(
MachineInstr &Root, SmallVectorImpl<unsigned> &Patterns,
bool DoRegPressureReduce) const {
diff --git a/llvm/lib/Target/X86/X86InstrInfo.h b/llvm/lib/Target/X86/X86InstrInfo.h
index fbfbc13df59cb..2a4b516538510 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.h
+++ b/llvm/lib/Target/X86/X86InstrInfo.h
@@ -674,6 +674,8 @@ class X86InstrInfo final : public X86GenInstrInfo {
SmallVectorImpl<unsigned> &Patterns,
bool DoRegPressureReduce) const override;
+ CombinerObjective getCombinerObjective(unsigned Pattern) const override;
+
/// When getMachineCombinerPatterns() finds potential patterns,
/// this function generates the instructions that could replace the
/// original code sequence.
>From b658ef55dd3e9d5ea8307a4d208365deca270f2f Mon Sep 17 00:00:00 2001
From: stomfaig <stomfaig at gmail.com>
Date: Tue, 24 Mar 2026 17:16:34 +0000
Subject: [PATCH 09/11] update tests
---
llvm/test/CodeGen/X86/avgflooru-i128.ll | 65 +-
llvm/test/CodeGen/X86/avgflooru-scalar.ll | 12 +-
llvm/test/CodeGen/X86/bitreverse.ll | 20 +-
llvm/test/CodeGen/X86/bswap.ll | 22 +-
.../X86/div-rem-pair-recomposition-signed.ll | 453 ++++++------
.../div-rem-pair-recomposition-unsigned.ll | 417 +++++------
llvm/test/CodeGen/X86/div_i129_v_pow2k.ll | 110 ++-
.../CodeGen/X86/expand-large-fp-optnone.ll | 27 +-
llvm/test/CodeGen/X86/fold-tied-op.ll | 112 ++-
llvm/test/CodeGen/X86/funnel-shift.ll | 60 +-
llvm/test/CodeGen/X86/imul.ll | 12 +-
llvm/test/CodeGen/X86/known-bits.ll | 4 +-
llvm/test/CodeGen/X86/midpoint-int.ll | 78 +-
llvm/test/CodeGen/X86/or-lea.ll | 8 +-
llvm/test/CodeGen/X86/pr38539.ll | 170 ++---
llvm/test/CodeGen/X86/pr43820.ll | 550 +++++++--------
llvm/test/CodeGen/X86/rot32.ll | 60 +-
llvm/test/CodeGen/X86/rotate-extract.ll | 9 +-
llvm/test/CodeGen/X86/scmp.ll | 624 ++++++++--------
llvm/test/CodeGen/X86/sdiv_fix.ll | 153 ++--
llvm/test/CodeGen/X86/sdiv_fix_sat.ll | 158 ++---
llvm/test/CodeGen/X86/shift-and.ll | 19 +-
llvm/test/CodeGen/X86/shift-i256.ll | 225 ++++--
llvm/test/CodeGen/X86/shift-i512.ll | 664 ++++++++++++------
llvm/test/CodeGen/X86/shift-mask.ll | 14 +-
llvm/test/CodeGen/X86/smul_fix.ll | 28 +-
llvm/test/CodeGen/X86/smul_fix_sat.ll | 100 ++-
llvm/test/CodeGen/X86/udiv_fix_sat.ll | 73 +-
llvm/test/CodeGen/X86/umul_fix.ll | 8 +-
llvm/test/CodeGen/X86/umul_fix_sat.ll | 51 +-
llvm/test/CodeGen/X86/vector-sext.ll | 61 +-
llvm/test/CodeGen/X86/vector-zext.ll | 52 +-
32 files changed, 2196 insertions(+), 2223 deletions(-)
diff --git a/llvm/test/CodeGen/X86/avgflooru-i128.ll b/llvm/test/CodeGen/X86/avgflooru-i128.ll
index c48ea1d75f65b..11e886e25ba4e 100644
--- a/llvm/test/CodeGen/X86/avgflooru-i128.ll
+++ b/llvm/test/CodeGen/X86/avgflooru-i128.ll
@@ -4,15 +4,13 @@
define i128 @avgflooru_i128(i128 %x, i128 %y) {
; CHECK-LABEL: avgflooru_i128:
; CHECK: # %bb.0: # %start
-; CHECK-NEXT: addq %rdx, %rdi
+; CHECK-NEXT: movq %rdi, %rax
+; CHECK-NEXT: addq %rdx, %rax
; CHECK-NEXT: adcq %rcx, %rsi
-; CHECK-NEXT: setb %al
-; CHECK-NEXT: shrq %rdi
-; CHECK-NEXT: movzbl %al, %edx
+; CHECK-NEXT: setb %cl
+; CHECK-NEXT: shrdq $1, %rsi, %rax
+; CHECK-NEXT: movzbl %cl, %edx
; CHECK-NEXT: shldq $63, %rsi, %rdx
-; CHECK-NEXT: movq %rsi, %rax
-; CHECK-NEXT: shlq $63, %rax
-; CHECK-NEXT: orq %rdi, %rax
; CHECK-NEXT: retq
start:
%xor = xor i128 %y, %x
@@ -34,10 +32,10 @@ define i128 @avgflooru_i128_multi_use(i128 %x, i128 %y) nounwind {
; CHECK-NEXT: pushq %r12
; CHECK-NEXT: pushq %rbx
; CHECK-NEXT: pushq %rax
-; CHECK-NEXT: movq %rcx, %r15
-; CHECK-NEXT: movq %rdx, %r12
-; CHECK-NEXT: movq %rsi, %rbx
-; CHECK-NEXT: movq %rdi, %r14
+; CHECK-NEXT: movq %rcx, %rbx
+; CHECK-NEXT: movq %rdx, %r14
+; CHECK-NEXT: movq %rsi, %r15
+; CHECK-NEXT: movq %rdi, %r12
; CHECK-NEXT: movq %rdx, %r13
; CHECK-NEXT: xorq %rdi, %r13
; CHECK-NEXT: movq %rcx, %rbp
@@ -45,22 +43,18 @@ define i128 @avgflooru_i128_multi_use(i128 %x, i128 %y) nounwind {
; CHECK-NEXT: movq %r13, %rdi
; CHECK-NEXT: movq %rbp, %rsi
; CHECK-NEXT: callq use at PLT
-; CHECK-NEXT: shrq %r13
-; CHECK-NEXT: movq %rbp, %rdi
-; CHECK-NEXT: shlq $63, %rdi
-; CHECK-NEXT: orq %r13, %rdi
+; CHECK-NEXT: shrdq $1, %rbp, %r13
; CHECK-NEXT: shrq %rbp
+; CHECK-NEXT: movq %r13, %rdi
; CHECK-NEXT: movq %rbp, %rsi
; CHECK-NEXT: callq use at PLT
-; CHECK-NEXT: addq %r12, %r14
-; CHECK-NEXT: adcq %r15, %rbx
-; CHECK-NEXT: setb %cl
-; CHECK-NEXT: shrq %r14
-; CHECK-NEXT: movq %rbx, %rax
-; CHECK-NEXT: shlq $63, %rax
-; CHECK-NEXT: orq %r14, %rax
-; CHECK-NEXT: movzbl %cl, %edx
-; CHECK-NEXT: shldq $63, %rbx, %rdx
+; CHECK-NEXT: addq %r14, %r12
+; CHECK-NEXT: adcq %rbx, %r15
+; CHECK-NEXT: setb %al
+; CHECK-NEXT: shrdq $1, %r15, %r12
+; CHECK-NEXT: movzbl %al, %edx
+; CHECK-NEXT: shldq $63, %r15, %rdx
+; CHECK-NEXT: movq %r12, %rax
; CHECK-NEXT: addq $8, %rsp
; CHECK-NEXT: popq %rbx
; CHECK-NEXT: popq %r12
@@ -129,25 +123,18 @@ define <2 x i128> @avgflooru_i128_vec(<2 x i128> %x, <2 x i128> %y) {
; CHECK-NEXT: adcq {{[0-9]+}}(%rsp), %rdx
; CHECK-NEXT: setb %dil
; CHECK-NEXT: movzbl %dil, %edi
-; CHECK-NEXT: shlq $63, %rdi
-; CHECK-NEXT: movq %rdx, %r9
-; CHECK-NEXT: shrq %r9
-; CHECK-NEXT: orq %rdi, %r9
+; CHECK-NEXT: shldq $63, %rdx, %rdi
; CHECK-NEXT: addq {{[0-9]+}}(%rsp), %rcx
; CHECK-NEXT: adcq {{[0-9]+}}(%rsp), %r8
-; CHECK-NEXT: setb %dil
-; CHECK-NEXT: movzbl %dil, %edi
-; CHECK-NEXT: shldq $63, %r8, %rdi
-; CHECK-NEXT: shrq %rsi
-; CHECK-NEXT: shlq $63, %rdx
-; CHECK-NEXT: orq %rsi, %rdx
-; CHECK-NEXT: shrq %rcx
-; CHECK-NEXT: shlq $63, %r8
-; CHECK-NEXT: orq %rcx, %r8
+; CHECK-NEXT: setb %r9b
+; CHECK-NEXT: movzbl %r9b, %r9d
+; CHECK-NEXT: shldq $63, %r8, %r9
+; CHECK-NEXT: shldq $63, %rsi, %rdx
+; CHECK-NEXT: shldq $63, %rcx, %r8
; CHECK-NEXT: movq %r8, 16(%rax)
; CHECK-NEXT: movq %rdx, (%rax)
-; CHECK-NEXT: movq %rdi, 24(%rax)
-; CHECK-NEXT: movq %r9, 8(%rax)
+; CHECK-NEXT: movq %r9, 24(%rax)
+; CHECK-NEXT: movq %rdi, 8(%rax)
; CHECK-NEXT: retq
start:
%xor = xor <2 x i128> %y, %x
diff --git a/llvm/test/CodeGen/X86/avgflooru-scalar.ll b/llvm/test/CodeGen/X86/avgflooru-scalar.ll
index bb41ae6f62a2f..bb070370316a8 100644
--- a/llvm/test/CodeGen/X86/avgflooru-scalar.ll
+++ b/llvm/test/CodeGen/X86/avgflooru-scalar.ll
@@ -260,9 +260,7 @@ define i64 @test_fixed_i64(i64 %a0, i64 %a1) nounwind {
; X86-NEXT: setb %dl
; X86-NEXT: movzbl %dl, %edx
; X86-NEXT: shldl $31, %eax, %edx
-; X86-NEXT: shrl %ecx
-; X86-NEXT: shll $31, %eax
-; X86-NEXT: orl %ecx, %eax
+; X86-NEXT: shldl $31, %ecx, %eax
; X86-NEXT: retl
;
; X64-LABEL: test_fixed_i64:
@@ -290,9 +288,7 @@ define i64 @test_lsb_i64(i64 %a0, i64 %a1) nounwind {
; X86-NEXT: setb %dl
; X86-NEXT: movzbl %dl, %edx
; X86-NEXT: shldl $31, %eax, %edx
-; X86-NEXT: shrl %ecx
-; X86-NEXT: shll $31, %eax
-; X86-NEXT: orl %ecx, %eax
+; X86-NEXT: shldl $31, %ecx, %eax
; X86-NEXT: retl
;
; X64-LABEL: test_lsb_i64:
@@ -322,9 +318,7 @@ define i64 @test_ext_i64(i64 %a0, i64 %a1) nounwind {
; X86-NEXT: setb %dl
; X86-NEXT: movzbl %dl, %edx
; X86-NEXT: shldl $31, %eax, %edx
-; X86-NEXT: shrl %ecx
-; X86-NEXT: shll $31, %eax
-; X86-NEXT: orl %ecx, %eax
+; X86-NEXT: shldl $31, %ecx, %eax
; X86-NEXT: retl
;
; X64-LABEL: test_ext_i64:
diff --git a/llvm/test/CodeGen/X86/bitreverse.ll b/llvm/test/CodeGen/X86/bitreverse.ll
index 3340056452be7..d92e1a1e7b9d4 100644
--- a/llvm/test/CodeGen/X86/bitreverse.ll
+++ b/llvm/test/CodeGen/X86/bitreverse.ll
@@ -1221,10 +1221,7 @@ define i528 @large_promotion(i528 %A) nounwind {
; X64-NEXT: shrq %r8
; X64-NEXT: andq %r14, %r8
; X64-NEXT: leaq (%r8,%r13,2), %r8
-; X64-NEXT: movq %r8, %r13
-; X64-NEXT: shlq $16, %r13
-; X64-NEXT: shrq $48, %r9
-; X64-NEXT: orq %r13, %r9
+; X64-NEXT: shrdq $48, %r8, %r9
; X64-NEXT: bswapq %rcx
; X64-NEXT: movq %rcx, %r13
; X64-NEXT: shrq $4, %r13
@@ -1242,10 +1239,7 @@ define i528 @large_promotion(i528 %A) nounwind {
; X64-NEXT: shrq %rcx
; X64-NEXT: andq %r14, %rcx
; X64-NEXT: leaq (%rcx,%r13,2), %rcx
-; X64-NEXT: movq %rcx, %r13
-; X64-NEXT: shlq $16, %r13
-; X64-NEXT: shrq $48, %r8
-; X64-NEXT: orq %r13, %r8
+; X64-NEXT: shrdq $48, %rcx, %r8
; X64-NEXT: bswapq %rdx
; X64-NEXT: movq %rdx, %r13
; X64-NEXT: shrq $4, %r13
@@ -1263,10 +1257,7 @@ define i528 @large_promotion(i528 %A) nounwind {
; X64-NEXT: shrq %rdx
; X64-NEXT: andq %r14, %rdx
; X64-NEXT: leaq (%rdx,%r13,2), %rdx
-; X64-NEXT: movq %rdx, %r13
-; X64-NEXT: shlq $16, %r13
-; X64-NEXT: shrq $48, %rcx
-; X64-NEXT: orq %r13, %rcx
+; X64-NEXT: shrdq $48, %rdx, %rcx
; X64-NEXT: bswapq %rsi
; X64-NEXT: movq %rsi, %r13
; X64-NEXT: shrq $4, %r13
@@ -1284,10 +1275,7 @@ define i528 @large_promotion(i528 %A) nounwind {
; X64-NEXT: shrq %rsi
; X64-NEXT: andq %r14, %rsi
; X64-NEXT: leaq (%rsi,%r10,2), %rsi
-; X64-NEXT: movq %rsi, %r10
-; X64-NEXT: shlq $16, %r10
-; X64-NEXT: shrq $48, %rdx
-; X64-NEXT: orq %r10, %rdx
+; X64-NEXT: shrdq $48, %rsi, %rdx
; X64-NEXT: shrq $48, %rsi
; X64-NEXT: movq %rdx, 56(%rax)
; X64-NEXT: movq %rcx, 48(%rax)
diff --git a/llvm/test/CodeGen/X86/bswap.ll b/llvm/test/CodeGen/X86/bswap.ll
index ab398b65bd3a1..81eac5676bb5c 100644
--- a/llvm/test/CodeGen/X86/bswap.ll
+++ b/llvm/test/CodeGen/X86/bswap.ll
@@ -351,7 +351,6 @@ define i528 @large_promotion(i528 %A) nounwind {
;
; CHECK64-LABEL: large_promotion:
; CHECK64: # %bb.0:
-; CHECK64-NEXT: pushq %r14
; CHECK64-NEXT: pushq %rbx
; CHECK64-NEXT: movq %rdi, %rax
; CHECK64-NEXT: movq {{[0-9]+}}(%rsp), %rbx
@@ -368,25 +367,13 @@ define i528 @large_promotion(i528 %A) nounwind {
; CHECK64-NEXT: bswapq %r9
; CHECK64-NEXT: shrdq $48, %r9, %rbx
; CHECK64-NEXT: bswapq %r8
-; CHECK64-NEXT: movq %r8, %r14
-; CHECK64-NEXT: shlq $16, %r14
-; CHECK64-NEXT: shrq $48, %r9
-; CHECK64-NEXT: orq %r14, %r9
+; CHECK64-NEXT: shrdq $48, %r8, %r9
; CHECK64-NEXT: bswapq %rcx
-; CHECK64-NEXT: movq %rcx, %r14
-; CHECK64-NEXT: shlq $16, %r14
-; CHECK64-NEXT: shrq $48, %r8
-; CHECK64-NEXT: orq %r14, %r8
+; CHECK64-NEXT: shrdq $48, %rcx, %r8
; CHECK64-NEXT: bswapq %rdx
-; CHECK64-NEXT: movq %rdx, %r14
-; CHECK64-NEXT: shlq $16, %r14
-; CHECK64-NEXT: shrq $48, %rcx
-; CHECK64-NEXT: orq %r14, %rcx
+; CHECK64-NEXT: shrdq $48, %rdx, %rcx
; CHECK64-NEXT: bswapq %rsi
-; CHECK64-NEXT: movq %rsi, %r14
-; CHECK64-NEXT: shlq $16, %r14
-; CHECK64-NEXT: shrq $48, %rdx
-; CHECK64-NEXT: orq %r14, %rdx
+; CHECK64-NEXT: shrdq $48, %rsi, %rdx
; CHECK64-NEXT: shrq $48, %rsi
; CHECK64-NEXT: movq %rdx, 56(%rax)
; CHECK64-NEXT: movq %rcx, 48(%rax)
@@ -398,7 +385,6 @@ define i528 @large_promotion(i528 %A) nounwind {
; CHECK64-NEXT: movq %rdi, (%rax)
; CHECK64-NEXT: movw %si, 64(%rax)
; CHECK64-NEXT: popq %rbx
-; CHECK64-NEXT: popq %r14
; CHECK64-NEXT: retq
%Z = call i528 @llvm.bswap.i528(i528 %A)
ret i528 %Z
diff --git a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
index b94a0c697df64..00f2e012c8b12 100644
--- a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
+++ b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
@@ -175,125 +175,125 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: movl 52(%ebp), %ecx
; X86-NEXT: movl %ecx, %edx
; X86-NEXT: sarl $31, %edx
-; X86-NEXT: movl %ecx, %edi
-; X86-NEXT: xorl %edx, %edi
+; X86-NEXT: movl %ecx, %ebx
+; X86-NEXT: xorl %edx, %ebx
; X86-NEXT: movl 48(%ebp), %esi
; X86-NEXT: xorl %edx, %esi
; X86-NEXT: movl 44(%ebp), %eax
; X86-NEXT: xorl %edx, %eax
-; X86-NEXT: movl 40(%ebp), %ebx
-; X86-NEXT: xorl %edx, %ebx
-; X86-NEXT: subl %edx, %ebx
+; X86-NEXT: movl 40(%ebp), %edi
+; X86-NEXT: xorl %edx, %edi
+; X86-NEXT: subl %edx, %edi
; X86-NEXT: sbbl %edx, %eax
; X86-NEXT: sbbl %edx, %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: sbbl %edx, %edi
+; X86-NEXT: sbbl %edx, %ebx
; X86-NEXT: movl %eax, %ecx
-; X86-NEXT: orl %edi, %ecx
-; X86-NEXT: movl %ebx, %edx
+; X86-NEXT: orl %ebx, %ecx
+; X86-NEXT: movl %edi, %edx
; X86-NEXT: orl %esi, %edx
; X86-NEXT: orl %ecx, %edx
-; X86-NEXT: sete %cl
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X86-NEXT: sete %dl
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X86-NEXT: orl %edx, %esi
-; X86-NEXT: sete %dl
-; X86-NEXT: orb %cl, %dl
-; X86-NEXT: movb %dl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X86-NEXT: bsrl %edi, %esi
+; X86-NEXT: orl %ecx, %esi
+; X86-NEXT: sete %cl
+; X86-NEXT: orb %dl, %cl
+; X86-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; X86-NEXT: bsrl %ebx, %esi
; X86-NEXT: xorl $31, %esi
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: bsrl %edx, %ecx
-; X86-NEXT: xorl $31, %ecx
-; X86-NEXT: orl $32, %ecx
-; X86-NEXT: testl %edi, %edi
-; X86-NEXT: cmovnel %esi, %ecx
+; X86-NEXT: bsrl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X86-NEXT: xorl $31, %edx
+; X86-NEXT: orl $32, %edx
+; X86-NEXT: testl %ebx, %ebx
+; X86-NEXT: cmovnel %esi, %edx
; X86-NEXT: bsrl %eax, %esi
; X86-NEXT: xorl $31, %esi
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: bsrl %ebx, %ebx
-; X86-NEXT: xorl $31, %ebx
-; X86-NEXT: orl $32, %ebx
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: testl %eax, %eax
-; X86-NEXT: cmovnel %esi, %ebx
-; X86-NEXT: orl $64, %ebx
-; X86-NEXT: movl %edx, %esi
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %edi, %esi
-; X86-NEXT: cmovnel %ecx, %ebx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: bsrl %edx, %esi
-; X86-NEXT: xorl $31, %esi
-; X86-NEXT: bsrl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: bsrl %edi, %ecx
; X86-NEXT: xorl $31, %ecx
; X86-NEXT: orl $32, %ecx
-; X86-NEXT: testl %edx, %edx
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: testl %eax, %eax
; X86-NEXT: cmovnel %esi, %ecx
+; X86-NEXT: orl $64, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %ebx, %esi
+; X86-NEXT: cmovnel %edx, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: bsrl %eax, %edi
+; X86-NEXT: bsrl %eax, %esi
+; X86-NEXT: xorl $31, %esi
+; X86-NEXT: bsrl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X86-NEXT: xorl $31, %edx
+; X86-NEXT: orl $32, %edx
+; X86-NEXT: testl %eax, %eax
+; X86-NEXT: cmovnel %esi, %edx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: bsrl %ebx, %edi
; X86-NEXT: xorl $31, %edi
; X86-NEXT: bsrl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
; X86-NEXT: xorl $31, %esi
; X86-NEXT: orl $32, %esi
-; X86-NEXT: testl %eax, %eax
+; X86-NEXT: testl %ebx, %ebx
; X86-NEXT: cmovnel %edi, %esi
; X86-NEXT: orl $64, %esi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: orl %edx, %edi
-; X86-NEXT: cmovnel %ecx, %esi
-; X86-NEXT: subl %esi, %ebx
-; X86-NEXT: movl $0, %edi
-; X86-NEXT: sbbl %edi, %edi
-; X86-NEXT: movl $0, %eax
-; X86-NEXT: sbbl %eax, %eax
+; X86-NEXT: orl %eax, %edi
+; X86-NEXT: cmovnel %edx, %esi
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: xorl %ebx, %ebx
+; X86-NEXT: subl %esi, %eax
+; X86-NEXT: movl $0, %edx
+; X86-NEXT: sbbl %edx, %edx
; X86-NEXT: movl $0, %esi
; X86-NEXT: sbbl %esi, %esi
+; X86-NEXT: movl $0, %edi
+; X86-NEXT: sbbl %edi, %edi
; X86-NEXT: cmpb $0, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Reload
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: jne .LBB4_1
; X86-NEXT: # %bb.2: # %select.false.sink
; X86-NEXT: movl $127, %ecx
-; X86-NEXT: cmpl %ebx, %ecx
+; X86-NEXT: cmpl %eax, %ecx
; X86-NEXT: movl $0, %ecx
-; X86-NEXT: sbbl %edi, %ecx
-; X86-NEXT: movl $0, %ecx
-; X86-NEXT: sbbl %eax, %ecx
+; X86-NEXT: sbbl %edx, %ecx
; X86-NEXT: movl $0, %ecx
; X86-NEXT: sbbl %esi, %ecx
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: sbbl %edi, %ecx
; X86-NEXT: setb %cl
; X86-NEXT: .LBB4_3: # %select.end
-; X86-NEXT: movl 56(%ebp), %esi
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: xorl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
; X86-NEXT: testb %cl, %cl
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: movl $0, %ecx
-; X86-NEXT: cmovnel %ecx, %eax
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: cmovnel %ecx, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: cmovnel %ecx, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: cmovel {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: cmovnel %ebx, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, %edi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: cmovnel %edi, %ebx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: cmovnel %edi, %esi
+; X86-NEXT: cmovel {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
; X86-NEXT: jne .LBB4_4
; X86-NEXT: # %bb.10: # %select.end
-; X86-NEXT: movl %ebx, %edx
-; X86-NEXT: xorl $127, %edx
-; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X86-NEXT: movl %edi, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: xorl $127, %eax
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X86-NEXT: orl %edx, %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: orl %eax, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: je .LBB4_11
; X86-NEXT: # %bb.8: # %udiv-bb1
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: xorps %xmm0, %xmm0
@@ -302,8 +302,9 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NEXT: movl %ebx, %ecx
; X86-NEXT: xorb $127, %cl
; X86-NEXT: movl %ecx, %eax
@@ -311,25 +312,25 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: andb $12, %al
; X86-NEXT: negb %al
; X86-NEXT: movsbl %al, %eax
-; X86-NEXT: movl 152(%esp,%eax), %esi
-; X86-NEXT: movl 156(%esp,%eax), %edx
-; X86-NEXT: shldl %cl, %esi, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 144(%esp,%eax), %edx
-; X86-NEXT: movl 148(%esp,%eax), %eax
-; X86-NEXT: shldl %cl, %eax, %esi
+; X86-NEXT: movl 152(%esp,%eax), %edx
+; X86-NEXT: movl 156(%esp,%eax), %esi
+; X86-NEXT: shldl %cl, %edx, %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shldl %cl, %edx, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shll %cl, %edx
+; X86-NEXT: movl 144(%esp,%eax), %esi
+; X86-NEXT: movl 148(%esp,%eax), %eax
+; X86-NEXT: shldl %cl, %eax, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shldl %cl, %esi, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shll %cl, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: addl $1, %ebx
-; X86-NEXT: adcl $0, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: jb .LBB4_9
; X86-NEXT: # %bb.5: # %udiv-preheader
+; X86-NEXT: movl %ebx, %ecx
; X86-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
@@ -337,31 +338,25 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl %ebx, %eax
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %ecx, %eax
; X86-NEXT: shrb $3, %al
; X86-NEXT: andb $12, %al
-; X86-NEXT: movzbl %al, %edx
-; X86-NEXT: movl 108(%esp,%edx), %esi
-; X86-NEXT: movl 104(%esp,%edx), %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %ebx, %ecx
-; X86-NEXT: shrdl %cl, %esi, %eax
-; X86-NEXT: movl 96(%esp,%edx), %edi
-; X86-NEXT: movl %ebx, %ecx
-; X86-NEXT: movl 100(%esp,%edx), %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: shrdl %cl, %ebx, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shrl %cl, %esi
+; X86-NEXT: movzbl %al, %eax
+; X86-NEXT: movl 108(%esp,%eax), %ebx
+; X86-NEXT: movl 104(%esp,%eax), %edx
+; X86-NEXT: movl %edx, %esi
+; X86-NEXT: shrdl %cl, %ebx, %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 96(%esp,%eax), %esi
+; X86-NEXT: movl 100(%esp,%eax), %edi
+; X86-NEXT: movl %edi, %eax
+; X86-NEXT: shrdl %cl, %edx, %eax
+; X86-NEXT: shrl %cl, %ebx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: shrdl %cl, %edx, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shrdl %cl, %edi, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: addl $-1, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
@@ -371,179 +366,151 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: adcl $-1, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: adcl $-1, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: adcl $-1, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: .p2align 4
; X86-NEXT: .LBB4_6: # %udiv-do-while
; X86-NEXT: # =>This Inner Loop Header: Depth=1
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: shll %edi
-; X86-NEXT: movl %eax, %edx
-; X86-NEXT: shrl $31, %edx
-; X86-NEXT: orl %edi, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shll %eax
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: movl %edi, %edx
-; X86-NEXT: shrl $31, %edx
-; X86-NEXT: orl %eax, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shll %edi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: movl %edx, %eax
-; X86-NEXT: shrl $31, %eax
-; X86-NEXT: orl %edi, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shldl $1, %ebx, %edx
+; X86-NEXT: shldl $1, %edx, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %eax, %esi
+; X86-NEXT: shldl $1, %eax, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shll %ebx
-; X86-NEXT: movl %ecx, %edx
-; X86-NEXT: shrl $31, %edx
-; X86-NEXT: orl %ebx, %edx
-; X86-NEXT: shll %ecx
-; X86-NEXT: movl %esi, %ebx
-; X86-NEXT: shrl $31, %ebx
-; X86-NEXT: orl %ecx, %ebx
-; X86-NEXT: shll %esi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: movl %eax, %edi
-; X86-NEXT: shrl $31, %edi
-; X86-NEXT: orl %esi, %edi
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: orl %ecx, %edx
-; X86-NEXT: orl %ecx, %ebx
+; X86-NEXT: shldl $1, %eax, %esi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: shldl $1, %ebx, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: shldl $1, %edi, %ebx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: orl %edx, %ebx
; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %ecx, %edi
-; X86-NEXT: addl %eax, %eax
-; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: cmpl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NEXT: sarl $31, %eax
-; X86-NEXT: movl %eax, %ecx
-; X86-NEXT: andl $1, %ecx
+; X86-NEXT: shldl $1, %ecx, %edi
+; X86-NEXT: orl %edx, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: shldl $1, %edi, %ecx
+; X86-NEXT: orl %edx, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %eax, %ebx
+; X86-NEXT: addl %edi, %edi
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: cmpl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: sbbl %esi, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: sbbl %edx, %ecx
+; X86-NEXT: sarl $31, %ecx
+; X86-NEXT: movl %ecx, %edi
+; X86-NEXT: andl $1, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ecx, %edi
+; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ecx, %ebx
; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X86-NEXT: movl %eax, %ecx
-; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: movl %ecx, %edi
+; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: subl %eax, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: sbbl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: sbbl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: subl %ecx, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %edi, %esi
+; X86-NEXT: movl %esi, %eax
; X86-NEXT: sbbl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: addl $-1, %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: adcl $-1, %eax
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: movl %edx, %ebx
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: addl $-1, %edx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: adcl $-1, %ecx
; X86-NEXT: adcl $-1, %esi
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: adcl $-1, %ebx
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %ebx, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: adcl $-1, %edi
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %esi, %ecx
-; X86-NEXT: orl %eax, %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: movl %edi, %esi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %edi, %ecx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %esi, %edx
+; X86-NEXT: orl %ecx, %edx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl %edx, %ebx
; X86-NEXT: jne .LBB4_6
; X86-NEXT: .LBB4_7: # %udiv-loop-exit
-; X86-NEXT: leal (,%edi,2), %eax
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: leal (%ecx,%esi,2), %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shrl $31, %esi
-; X86-NEXT: orl %eax, %esi
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: leal (,%ecx,2), %eax
-; X86-NEXT: shrl $31, %edi
-; X86-NEXT: orl %eax, %edi
-; X86-NEXT: shll %edx
-; X86-NEXT: shrl $31, %ecx
-; X86-NEXT: orl %edx, %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl %esi, %eax
-; X86-NEXT: movl 56(%ebp), %esi
-; X86-NEXT: movl %edi, %edx
-; X86-NEXT: .LBB4_11: # %udiv-end
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: xorl %ecx, %ebx
-; X86-NEXT: xorl %ecx, %edx
-; X86-NEXT: xorl %ecx, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: shldl $1, %esi, %eax
+; X86-NEXT: shldl $1, %ecx, %esi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: shldl $1, %edx, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: xorl %ecx, %edi
-; X86-NEXT: subl %ecx, %edi
-; X86-NEXT: sbbl %ecx, %eax
+; X86-NEXT: leal (%edi,%edx,2), %ebx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: movl %eax, %edi
+; X86-NEXT: .LBB4_11: # %udiv-end
+; X86-NEXT: xorl %edx, %edi
+; X86-NEXT: xorl %edx, %esi
+; X86-NEXT: xorl %edx, %ecx
+; X86-NEXT: xorl %edx, %ebx
+; X86-NEXT: subl %edx, %ebx
+; X86-NEXT: sbbl %edx, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %edx, %esi
+; X86-NEXT: sbbl %edx, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 56(%ebp), %eax
+; X86-NEXT: movl %ebx, (%eax)
+; X86-NEXT: movl %ecx, 4(%eax)
+; X86-NEXT: movl %esi, 8(%eax)
+; X86-NEXT: movl %edi, 12(%eax)
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: movl 40(%ebp), %ecx
+; X86-NEXT: mull %ecx
+; X86-NEXT: movl %edx, %edi
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: sbbl %ecx, %edx
-; X86-NEXT: sbbl %ecx, %ebx
+; X86-NEXT: movl %ebx, %eax
; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %edi, (%esi)
-; X86-NEXT: movl %eax, 4(%esi)
-; X86-NEXT: movl %edx, 8(%esi)
-; X86-NEXT: movl %ebx, 12(%esi)
-; X86-NEXT: movl 40(%ebp), %ecx
-; X86-NEXT: movl %edx, %ebx
; X86-NEXT: mull %ecx
-; X86-NEXT: movl %edx, %ecx
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %edi, %esi
+; X86-NEXT: movl %edx, %ecx
+; X86-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: adcl $0, %edi
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %edi, %eax
-; X86-NEXT: mull 40(%ebp)
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %edx, %edi
-; X86-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X86-NEXT: adcl $0, %ecx
-; X86-NEXT: movl %esi, %eax
+; X86-NEXT: movl %ebx, %eax
+; X86-NEXT: movl %esi, %edi
; X86-NEXT: movl 44(%ebp), %esi
; X86-NEXT: mull %esi
-; X86-NEXT: addl %edi, %eax
+; X86-NEXT: addl %ecx, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: adcl %ecx, %edx
+; X86-NEXT: adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
; X86-NEXT: movl %edx, %ecx
-; X86-NEXT: setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X86-NEXT: setb %bl
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NEXT: movl %esi, %eax
; X86-NEXT: mull 44(%ebp)
-; X86-NEXT: movl %eax, %edi
-; X86-NEXT: addl %ecx, %edi
-; X86-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X86-NEXT: addl %ecx, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movzbl %bl, %eax
; X86-NEXT: adcl %eax, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 40(%ebp), %eax
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl 40(%ebp), %eax
; X86-NEXT: imull %eax, %ecx
-; X86-NEXT: mull %ebx
+; X86-NEXT: mull %edi
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: imull 44(%ebp), %ebx
-; X86-NEXT: addl %edx, %ebx
-; X86-NEXT: addl %ecx, %ebx
+; X86-NEXT: imull 44(%ebp), %edi
+; X86-NEXT: addl %edx, %edi
+; X86-NEXT: addl %ecx, %edi
; X86-NEXT: movl 48(%ebp), %eax
; X86-NEXT: movl %eax, %ecx
; X86-NEXT: imull %esi, %ecx
@@ -554,8 +521,8 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: addl %edx, %esi
; X86-NEXT: addl %ecx, %esi
; X86-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NEXT: adcl %ebx, %esi
-; X86-NEXT: addl %edi, %eax
+; X86-NEXT: adcl %edi, %esi
+; X86-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
; X86-NEXT: adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
; X86-NEXT: movl 24(%ebp), %edx
; X86-NEXT: subl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
@@ -580,14 +547,10 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: movb $1, %cl
; X86-NEXT: jmp .LBB4_3
; X86-NEXT: .LBB4_9:
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: jmp .LBB4_7
; X86-NEXT: .LBB4_4:
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: jmp .LBB4_11
;
diff --git a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll
index f66da12114efa..3d756f3cf2141 100644
--- a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll
+++ b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll
@@ -152,13 +152,13 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: pushl %esi
; X86-NEXT: andl $-16, %esp
; X86-NEXT: subl $160, %esp
-; X86-NEXT: movl 48(%ebp), %edi
+; X86-NEXT: movl 48(%ebp), %ebx
; X86-NEXT: movl 40(%ebp), %ecx
; X86-NEXT: movl 52(%ebp), %esi
-; X86-NEXT: movl 44(%ebp), %ebx
-; X86-NEXT: movl %ebx, %eax
+; X86-NEXT: movl 44(%ebp), %edi
+; X86-NEXT: movl %edi, %eax
; X86-NEXT: orl %esi, %eax
-; X86-NEXT: orl %edi, %ecx
+; X86-NEXT: orl %ebx, %ecx
; X86-NEXT: orl %eax, %ecx
; X86-NEXT: sete %cl
; X86-NEXT: movl 28(%ebp), %eax
@@ -171,25 +171,24 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
; X86-NEXT: bsrl %esi, %edx
; X86-NEXT: xorl $31, %edx
-; X86-NEXT: bsrl %edi, %ecx
-; X86-NEXT: movl %edi, %eax
+; X86-NEXT: bsrl %ebx, %ecx
+; X86-NEXT: movl %ebx, %eax
; X86-NEXT: xorl $31, %ecx
; X86-NEXT: orl $32, %ecx
; X86-NEXT: testl %esi, %esi
; X86-NEXT: cmovnel %edx, %ecx
-; X86-NEXT: bsrl %ebx, %edx
+; X86-NEXT: bsrl %edi, %edx
; X86-NEXT: xorl $31, %edx
-; X86-NEXT: bsrl 40(%ebp), %edi
-; X86-NEXT: xorl $31, %edi
-; X86-NEXT: orl $32, %edi
-; X86-NEXT: testl %ebx, %ebx
-; X86-NEXT: cmovnel %edx, %edi
-; X86-NEXT: orl $64, %edi
+; X86-NEXT: bsrl 40(%ebp), %ebx
+; X86-NEXT: xorl $31, %ebx
+; X86-NEXT: orl $32, %ebx
+; X86-NEXT: testl %edi, %edi
+; X86-NEXT: cmovnel %edx, %ebx
+; X86-NEXT: orl $64, %ebx
; X86-NEXT: movl %eax, %edx
-; X86-NEXT: movl %edi, %ebx
-; X86-NEXT: movl 36(%ebp), %edi
; X86-NEXT: orl %esi, %edx
; X86-NEXT: cmovnel %ecx, %ebx
+; X86-NEXT: movl 36(%ebp), %edi
; X86-NEXT: bsrl %edi, %edx
; X86-NEXT: xorl $31, %edx
; X86-NEXT: bsrl 32(%ebp), %ecx
@@ -198,73 +197,77 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: testl %edi, %edi
; X86-NEXT: cmovnel %edx, %ecx
; X86-NEXT: movl 28(%ebp), %eax
-; X86-NEXT: bsrl %eax, %edx
-; X86-NEXT: xorl $31, %edx
-; X86-NEXT: bsrl 24(%ebp), %esi
+; X86-NEXT: bsrl %eax, %esi
; X86-NEXT: xorl $31, %esi
-; X86-NEXT: orl $32, %esi
+; X86-NEXT: bsrl 24(%ebp), %edx
+; X86-NEXT: xorl $31, %edx
+; X86-NEXT: orl $32, %edx
; X86-NEXT: testl %eax, %eax
-; X86-NEXT: cmovnel %edx, %esi
-; X86-NEXT: orl $64, %esi
-; X86-NEXT: movl 32(%ebp), %edx
-; X86-NEXT: orl %edi, %edx
-; X86-NEXT: cmovnel %ecx, %esi
-; X86-NEXT: xorl %edx, %edx
-; X86-NEXT: subl %esi, %ebx
+; X86-NEXT: movl 32(%ebp), %eax
+; X86-NEXT: cmovnel %esi, %edx
+; X86-NEXT: orl $64, %edx
+; X86-NEXT: movl %eax, %esi
+; X86-NEXT: orl %edi, %esi
+; X86-NEXT: cmovnel %ecx, %edx
+; X86-NEXT: xorl %ecx, %ecx
+; X86-NEXT: subl %edx, %ebx
+; X86-NEXT: movl %ebx, %edx
; X86-NEXT: movl $0, %esi
; X86-NEXT: sbbl %esi, %esi
-; X86-NEXT: movl $0, %ecx
-; X86-NEXT: sbbl %ecx, %ecx
+; X86-NEXT: movl $0, %ebx
+; X86-NEXT: sbbl %ebx, %ebx
; X86-NEXT: movl $0, %edi
; X86-NEXT: sbbl %edi, %edi
; X86-NEXT: cmpb $0, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Reload
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: jne .LBB4_1
; X86-NEXT: # %bb.2: # %select.false.sink
; X86-NEXT: movl $127, %eax
-; X86-NEXT: cmpl %ebx, %eax
+; X86-NEXT: cmpl %edx, %eax
; X86-NEXT: movl $0, %eax
; X86-NEXT: sbbl %esi, %eax
; X86-NEXT: movl $0, %eax
-; X86-NEXT: sbbl %ecx, %eax
+; X86-NEXT: sbbl %ebx, %eax
; X86-NEXT: movl $0, %eax
; X86-NEXT: sbbl %edi, %eax
; X86-NEXT: setb %al
; X86-NEXT: .LBB4_3: # %select.end
; X86-NEXT: testb %al, %al
-; X86-NEXT: movl 28(%ebp), %edi
-; X86-NEXT: cmovnel %edx, %edi
+; X86-NEXT: movl 28(%ebp), %edx
+; X86-NEXT: cmovnel %ecx, %edx
; X86-NEXT: movl 24(%ebp), %eax
-; X86-NEXT: cmovnel %edx, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 32(%ebp), %eax
-; X86-NEXT: cmovnel %edx, %eax
+; X86-NEXT: cmovnel %ecx, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 36(%ebp), %ecx
-; X86-NEXT: cmovnel %edx, %ecx
+; X86-NEXT: movl 32(%ebp), %edi
+; X86-NEXT: cmovnel %ecx, %edi
+; X86-NEXT: movl 36(%ebp), %ebx
+; X86-NEXT: cmovnel %ecx, %ebx
; X86-NEXT: jne .LBB4_9
; X86-NEXT: # %bb.4: # %select.end
-; X86-NEXT: movl %ebx, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: movl %esi, %eax
; X86-NEXT: xorl $127, %eax
; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NEXT: movl %esi, %edx
-; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X86-NEXT: orl %eax, %edx
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 28(%ebp), %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: orl %eax, %ecx
+; X86-NEXT: movl 28(%ebp), %ecx
; X86-NEXT: je .LBB4_9
; X86-NEXT: # %bb.5: # %udiv-bb1
-; X86-NEXT: movl 24(%ebp), %esi
-; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 24(%ebp), %edi
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X86-NEXT: xorps %xmm0, %xmm0
; X86-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %ecx, {{[0-9]+}}(%esp)
; X86-NEXT: movl 32(%ebp), %edx
; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 36(%ebp), %eax
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: movl 36(%ebp), %ecx
+; X86-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %esi, %edx
+; X86-NEXT: movl %edx, %ecx
; X86-NEXT: xorb $127, %cl
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: shrb $3, %al
@@ -272,18 +275,17 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: negb %al
; X86-NEXT: movsbl %al, %eax
; X86-NEXT: movl 136(%esp,%eax), %esi
-; X86-NEXT: movl 140(%esp,%eax), %edi
-; X86-NEXT: shldl %cl, %esi, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 140(%esp,%eax), %ebx
+; X86-NEXT: shldl %cl, %esi, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl 128(%esp,%eax), %edi
-; X86-NEXT: movl 132(%esp,%eax), %eax
-; X86-NEXT: shldl %cl, %eax, %esi
+; X86-NEXT: movl 132(%esp,%eax), %ebx
+; X86-NEXT: shldl %cl, %ebx, %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shldl %cl, %edi, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shldl %cl, %edi, %ebx
; X86-NEXT: shll %cl, %edi
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: addl $1, %ebx
+; X86-NEXT: addl $1, %edx
; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
@@ -294,32 +296,32 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: movl 28(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 32(%ebp), %eax
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: movl 36(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl %ebx, %eax
+; X86-NEXT: movl %edx, %eax
; X86-NEXT: shrb $3, %al
; X86-NEXT: andb $12, %al
-; X86-NEXT: movzbl %al, %esi
-; X86-NEXT: movl 92(%esp,%esi), %edx
-; X86-NEXT: movl 88(%esp,%esi), %eax
+; X86-NEXT: movzbl %al, %eax
+; X86-NEXT: movl 92(%esp,%eax), %esi
+; X86-NEXT: movl 88(%esp,%eax), %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %edx, %ecx
+; X86-NEXT: shrdl %cl, %esi, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 80(%esp,%eax), %edx
+; X86-NEXT: movl 84(%esp,%eax), %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %ebx, %ecx
-; X86-NEXT: shrdl %cl, %edx, %eax
-; X86-NEXT: movl 80(%esp,%esi), %edi
-; X86-NEXT: movl %ebx, %ecx
-; X86-NEXT: movl 84(%esp,%esi), %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: shrdl %cl, %ebx, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shrl %cl, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: shrdl %cl, %edi, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shrl %cl, %esi
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: shrdl %cl, %edx, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: shrdl %cl, %eax, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl 40(%ebp), %ecx
; X86-NEXT: addl $-1, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
@@ -333,198 +335,166 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: adcl $-1, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: .p2align 4
; X86-NEXT: .LBB4_7: # %udiv-do-while
; X86-NEXT: # =>This Inner Loop Header: Depth=1
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: shll %esi
-; X86-NEXT: movl %eax, %ebx
-; X86-NEXT: shrl $31, %ebx
-; X86-NEXT: orl %esi, %ebx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shll %eax
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: movl %esi, %ebx
-; X86-NEXT: shrl $31, %ebx
-; X86-NEXT: orl %eax, %ebx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shll %esi
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: movl %ebx, %eax
-; X86-NEXT: shrl $31, %eax
-; X86-NEXT: orl %esi, %eax
-; X86-NEXT: movl %eax, %esi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: shldl $1, %eax, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: shldl $1, %edi, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shldl $1, %edi, %ebx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shll %edi
-; X86-NEXT: movl %ecx, %ebx
-; X86-NEXT: shrl $31, %ebx
-; X86-NEXT: orl %edi, %ebx
-; X86-NEXT: shll %ecx
-; X86-NEXT: movl %edx, %eax
-; X86-NEXT: shrl $31, %eax
-; X86-NEXT: orl %ecx, %eax
-; X86-NEXT: movl %eax, %edi
-; X86-NEXT: shll %edx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: movl %eax, %ecx
-; X86-NEXT: shrl $31, %ecx
-; X86-NEXT: orl %edx, %ecx
+; X86-NEXT: shldl $1, %eax, %edi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: shldl $1, %esi, %eax
+; X86-NEXT: shldl $1, %ecx, %esi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: orl %edx, %ebx
-; X86-NEXT: orl %edx, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %edx, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shldl $1, %ebx, %ecx
; X86-NEXT: orl %edx, %ecx
-; X86-NEXT: addl %eax, %eax
-; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: shldl $1, %ecx, %ebx
+; X86-NEXT: orl %edx, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: addl %ecx, %ecx
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: cmpl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: sbbl %edi, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: cmpl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: sbbl %esi, %eax
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X86-NEXT: sarl $31, %esi
-; X86-NEXT: movl %esi, %eax
-; X86-NEXT: andl $1, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %esi, %eax
-; X86-NEXT: andl 52(%ebp), %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %esi, %edi
-; X86-NEXT: andl 48(%ebp), %edi
-; X86-NEXT: movl %esi, %eax
-; X86-NEXT: andl 44(%ebp), %eax
-; X86-NEXT: andl 40(%ebp), %esi
+; X86-NEXT: sbbl %edx, %ecx
+; X86-NEXT: sarl $31, %ecx
+; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: andl $1, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: andl 52(%ebp), %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: andl 48(%ebp), %esi
+; X86-NEXT: movl %ecx, %ebx
+; X86-NEXT: andl 44(%ebp), %ebx
+; X86-NEXT: andl 40(%ebp), %ecx
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: subl %esi, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: sbbl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: sbbl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl %ecx, %edi
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: sbbl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: subl %ecx, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %ebx, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: sbbl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: movl %edx, %esi
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NEXT: addl $-1, %edx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: adcl $-1, %eax
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: adcl $-1, %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: adcl $-1, %esi
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %esi, %eax
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %ecx, %edx
-; X86-NEXT: orl %eax, %edx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: adcl $-1, %eax
+; X86-NEXT: adcl $-1, %edi
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %edi, %esi
-; X86-NEXT: movl %edi, %edx
+; X86-NEXT: orl %edi, %ecx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %eax, %edx
+; X86-NEXT: orl %ecx, %edx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl %ebx, %edi
; X86-NEXT: jne .LBB4_7
; X86-NEXT: .LBB4_8: # %udiv-loop-exit
-; X86-NEXT: leal (,%esi,2), %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: shldl $1, %ecx, %eax
+; X86-NEXT: shldl $1, %ebx, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: leal (%ecx,%edx,2), %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shrl $31, %edx
-; X86-NEXT: orl %eax, %edx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: leal (,%ecx,2), %eax
-; X86-NEXT: shrl $31, %esi
-; X86-NEXT: orl %eax, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shll %ebx
-; X86-NEXT: shrl $31, %ecx
-; X86-NEXT: orl %ebx, %ecx
-; X86-NEXT: movl %edx, %edi
+; X86-NEXT: shldl $1, %edx, %ebx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: leal (%esi,%edx,2), %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ecx, %edi
+; X86-NEXT: movl %ebx, %edx
+; X86-NEXT: movl %eax, %ebx
; X86-NEXT: .LBB4_9: # %udiv-end
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl 56(%ebp), %eax
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: movl %ebx, (%eax)
-; X86-NEXT: movl %edi, 4(%eax)
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: movl %edx, 8(%eax)
-; X86-NEXT: movl %ecx, 12(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: movl %esi, (%eax)
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %edx, 4(%eax)
+; X86-NEXT: movl %edi, 8(%eax)
+; X86-NEXT: movl %ebx, 12(%eax)
; X86-NEXT: movl 48(%ebp), %eax
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %eax, %esi
-; X86-NEXT: imull %edi, %esi
-; X86-NEXT: mull %ebx
-; X86-NEXT: movl %ebx, %edi
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: imull %edx, %ecx
+; X86-NEXT: mull %esi
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: addl %esi, %edx
-; X86-NEXT: movl 52(%ebp), %ebx
-; X86-NEXT: imull %edi, %ebx
-; X86-NEXT: addl %edx, %ebx
+; X86-NEXT: addl %ecx, %edx
+; X86-NEXT: movl 52(%ebp), %edi
+; X86-NEXT: imull %esi, %edi
+; X86-NEXT: addl %edx, %edi
; X86-NEXT: movl 40(%ebp), %eax
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: mull %edi
-; X86-NEXT: movl %eax, %esi
-; X86-NEXT: imull 40(%ebp), %ecx
-; X86-NEXT: addl %edx, %ecx
-; X86-NEXT: movl 44(%ebp), %edx
-; X86-NEXT: imull %edx, %edi
-; X86-NEXT: addl %ecx, %edi
-; X86-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: adcl %ebx, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: movl %esi, %eax
+; X86-NEXT: mull %esi
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: imull 40(%ebp), %ebx
+; X86-NEXT: addl %edx, %ebx
+; X86-NEXT: movl 44(%ebp), %eax
+; X86-NEXT: imull %eax, %esi
+; X86-NEXT: addl %ebx, %esi
+; X86-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: adcl %edi, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: movl %edi, %eax
; X86-NEXT: movl 40(%ebp), %ecx
; X86-NEXT: mull %ecx
-; X86-NEXT: movl %edx, %ebx
+; X86-NEXT: movl %edx, %esi
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: movl %edi, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: movl %ebx, %eax
; X86-NEXT: mull %ecx
; X86-NEXT: movl %eax, %ecx
-; X86-NEXT: addl %ebx, %ecx
+; X86-NEXT: addl %esi, %ecx
; X86-NEXT: adcl $0, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %esi, %eax
-; X86-NEXT: movl 44(%ebp), %ebx
-; X86-NEXT: mull %ebx
; X86-NEXT: movl %edx, %esi
+; X86-NEXT: movl %edi, %eax
+; X86-NEXT: mull 44(%ebp)
+; X86-NEXT: movl %edx, %edi
; X86-NEXT: addl %ecx, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: adcl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NEXT: adcl %esi, %edi
; X86-NEXT: setb %cl
-; X86-NEXT: movl %edi, %eax
-; X86-NEXT: mull %ebx
-; X86-NEXT: addl %esi, %eax
+; X86-NEXT: movl %ebx, %eax
+; X86-NEXT: mull 44(%ebp)
+; X86-NEXT: addl %edi, %eax
; X86-NEXT: movzbl %cl, %ecx
; X86-NEXT: adcl %ecx, %edx
; X86-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
; X86-NEXT: adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X86-NEXT: movl 24(%ebp), %esi
-; X86-NEXT: subl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NEXT: movl 24(%ebp), %edi
+; X86-NEXT: subl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
; X86-NEXT: movl 28(%ebp), %ebx
; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
; X86-NEXT: movl 32(%ebp), %ecx
; X86-NEXT: sbbl %eax, %ecx
-; X86-NEXT: movl 36(%ebp), %edi
-; X86-NEXT: sbbl %edx, %edi
+; X86-NEXT: movl 36(%ebp), %esi
+; X86-NEXT: sbbl %edx, %esi
; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: movl %esi, (%eax)
+; X86-NEXT: movl %edi, (%eax)
; X86-NEXT: movl %ebx, 4(%eax)
; X86-NEXT: movl %ecx, 8(%eax)
-; X86-NEXT: movl %edi, 12(%eax)
+; X86-NEXT: movl %esi, 12(%eax)
; X86-NEXT: leal -12(%ebp), %esp
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
@@ -535,8 +505,7 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: movb $1, %al
; X86-NEXT: jmp .LBB4_3
; X86-NEXT: .LBB4_10:
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: jmp .LBB4_8
;
diff --git a/llvm/test/CodeGen/X86/div_i129_v_pow2k.ll b/llvm/test/CodeGen/X86/div_i129_v_pow2k.ll
index 6df156f71a0af..0924ea85a0126 100644
--- a/llvm/test/CodeGen/X86/div_i129_v_pow2k.ll
+++ b/llvm/test/CodeGen/X86/div_i129_v_pow2k.ll
@@ -19,10 +19,8 @@ define i129 @v_sdiv_i129_v_pow2k(i129 %lhs) nounwind {
; X64-NEXT: andl $1, %ecx
; X64-NEXT: movq %rcx, %rdx
; X64-NEXT: negq %rdx
-; X64-NEXT: shrq $33, %rax
+; X64-NEXT: shrdq $33, %rsi, %rax
; X64-NEXT: shldq $31, %rsi, %rdx
-; X64-NEXT: shlq $31, %rsi
-; X64-NEXT: orq %rsi, %rax
; X64-NEXT: retq
;
; X64-O0-LABEL: v_sdiv_i129_v_pow2k:
@@ -51,46 +49,40 @@ define i129 @v_sdiv_i129_v_pow2k(i129 %lhs) nounwind {
;
; X86-LABEL: v_sdiv_i129_v_pow2k:
; X86: # %bb.0:
+; X86-NEXT: pushl %ebp
; X86-NEXT: pushl %ebx
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT: movl %esi, %edi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl %ecx, %ebx
+; X86-NEXT: andl $1, %ebx
+; X86-NEXT: negl %ebx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %ebx, %edi
; X86-NEXT: andl $1, %edi
-; X86-NEXT: negl %edi
-; X86-NEXT: movl %edi, %ecx
-; X86-NEXT: andl $1, %ecx
-; X86-NEXT: addl {{[0-9]+}}(%esp), %edi
-; X86-NEXT: adcl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: adcl $0, %eax
-; X86-NEXT: adcl $0, %edx
+; X86-NEXT: addl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT: adcl {{[0-9]+}}(%esp), %edi
; X86-NEXT: adcl $0, %esi
-; X86-NEXT: movl %esi, %ebx
-; X86-NEXT: shll $31, %esi
-; X86-NEXT: movl %edx, %edi
-; X86-NEXT: shrl %edi
-; X86-NEXT: orl %esi, %edi
-; X86-NEXT: shll $31, %edx
-; X86-NEXT: movl %eax, %esi
-; X86-NEXT: shrl %esi
-; X86-NEXT: orl %edx, %esi
-; X86-NEXT: shll $31, %eax
-; X86-NEXT: shrl %ecx
-; X86-NEXT: orl %eax, %ecx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl %ecx, (%eax)
-; X86-NEXT: movl %esi, 4(%eax)
+; X86-NEXT: adcl $0, %edx
+; X86-NEXT: adcl $0, %ecx
+; X86-NEXT: movl %ecx, %ebx
; X86-NEXT: andl $1, %ebx
-; X86-NEXT: movl %edi, 8(%eax)
-; X86-NEXT: movl %ebx, %ecx
-; X86-NEXT: negl %ecx
-; X86-NEXT: movl %ecx, 12(%eax)
+; X86-NEXT: movl %ebx, %ebp
+; X86-NEXT: negl %ebp
+; X86-NEXT: shldl $31, %edx, %ecx
+; X86-NEXT: shldl $31, %esi, %edx
+; X86-NEXT: shldl $31, %edi, %esi
+; X86-NEXT: movl %esi, (%eax)
+; X86-NEXT: movl %edx, 4(%eax)
+; X86-NEXT: movl %ecx, 8(%eax)
+; X86-NEXT: movl %ebp, 12(%eax)
; X86-NEXT: movb %bl, 16(%eax)
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
; X86-NEXT: popl %ebx
+; X86-NEXT: popl %ebp
; X86-NEXT: retl $4
;
; X86-O0-LABEL: v_sdiv_i129_v_pow2k:
@@ -148,14 +140,12 @@ define i129 @v_sdiv_exact_i129_v_pow2k(i129 %lhs) nounwind {
; X64-LABEL: v_sdiv_exact_i129_v_pow2k:
; X64: # %bb.0:
; X64-NEXT: movq %rdx, %rcx
-; X64-NEXT: movq %rsi, %rax
+; X64-NEXT: movq %rdi, %rax
; X64-NEXT: andl $1, %ecx
; X64-NEXT: movq %rcx, %rdx
; X64-NEXT: negq %rdx
-; X64-NEXT: shrq $33, %rdi
+; X64-NEXT: shrdq $33, %rsi, %rax
; X64-NEXT: shldq $31, %rsi, %rdx
-; X64-NEXT: shlq $31, %rax
-; X64-NEXT: orq %rdi, %rax
; X64-NEXT: retq
;
; X64-O0-LABEL: v_sdiv_exact_i129_v_pow2k:
@@ -176,30 +166,22 @@ define i129 @v_sdiv_exact_i129_v_pow2k(i129 %lhs) nounwind {
; X86-NEXT: pushl %ebx
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl %eax, %ecx
-; X86-NEXT: andl $1, %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT: movl %esi, %ebx
-; X86-NEXT: shll $31, %ebx
-; X86-NEXT: shrl %edx
-; X86-NEXT: orl %ebx, %edx
-; X86-NEXT: shll $31, %eax
-; X86-NEXT: movl %edi, %ebx
-; X86-NEXT: shrl %ebx
-; X86-NEXT: orl %eax, %ebx
-; X86-NEXT: movl %ecx, %ebp
-; X86-NEXT: negl %ebp
-; X86-NEXT: shll $31, %edi
-; X86-NEXT: shrl %esi
-; X86-NEXT: orl %edi, %esi
+; X86-NEXT: movl %edx, %ecx
+; X86-NEXT: andl $1, %ecx
+; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: negl %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl %ebp, 12(%eax)
-; X86-NEXT: movl %ebx, 8(%eax)
-; X86-NEXT: movl %esi, 4(%eax)
-; X86-NEXT: movl %edx, (%eax)
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT: shrdl $1, %ebx, %edi
+; X86-NEXT: shldl $31, %ebp, %edx
+; X86-NEXT: shldl $31, %ebx, %ebp
+; X86-NEXT: movl %esi, 12(%eax)
+; X86-NEXT: movl %edx, 8(%eax)
+; X86-NEXT: movl %ebp, 4(%eax)
+; X86-NEXT: movl %edi, (%eax)
; X86-NEXT: movb %cl, 16(%eax)
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
@@ -245,12 +227,10 @@ define i129 @v_sdiv_exact_i129_v_pow2k(i129 %lhs) nounwind {
define i129 @v_udiv_i129_v_pow2k(i129 %lhs) nounwind {
; X64-LABEL: v_udiv_i129_v_pow2k:
; X64: # %bb.0:
-; X64-NEXT: movq %rsi, %rax
+; X64-NEXT: movq %rdi, %rax
; X64-NEXT: andl $1, %edx
-; X64-NEXT: shrq $33, %rdi
+; X64-NEXT: shrdq $33, %rsi, %rax
; X64-NEXT: shldq $31, %rsi, %rdx
-; X64-NEXT: shlq $31, %rax
-; X64-NEXT: orq %rdi, %rax
; X64-NEXT: xorl %ecx, %ecx
; X64-NEXT: retq
;
@@ -318,12 +298,10 @@ define i129 @v_udiv_i129_v_pow2k(i129 %lhs) nounwind {
define i129 @v_udiv_exact_i129_v_pow2k(i129 %lhs) nounwind {
; X64-LABEL: v_udiv_exact_i129_v_pow2k:
; X64: # %bb.0:
-; X64-NEXT: movq %rsi, %rax
+; X64-NEXT: movq %rdi, %rax
; X64-NEXT: andl $1, %edx
-; X64-NEXT: shrq $33, %rdi
+; X64-NEXT: shrdq $33, %rsi, %rax
; X64-NEXT: shldq $31, %rsi, %rdx
-; X64-NEXT: shlq $31, %rax
-; X64-NEXT: orq %rdi, %rax
; X64-NEXT: xorl %ecx, %ecx
; X64-NEXT: retq
;
diff --git a/llvm/test/CodeGen/X86/expand-large-fp-optnone.ll b/llvm/test/CodeGen/X86/expand-large-fp-optnone.ll
index ccb186175a9ed..16fe756bfc4e6 100644
--- a/llvm/test/CodeGen/X86/expand-large-fp-optnone.ll
+++ b/llvm/test/CodeGen/X86/expand-large-fp-optnone.ll
@@ -61,10 +61,7 @@ define double @main(i224 %0) #0 {
; CHECK-NEXT: testq %r8, %r8
; CHECK-NEXT: cmovnel %eax, %r11d
; CHECK-NEXT: movq %rsi, %rbx
-; CHECK-NEXT: shlq $32, %rbx
-; CHECK-NEXT: movq %rdi, %rax
-; CHECK-NEXT: shrq $32, %rax
-; CHECK-NEXT: orq %rax, %rbx
+; CHECK-NEXT: shldq $32, %rdi, %rbx
; CHECK-NEXT: bsrq %rbx, %r14
; CHECK-NEXT: xorl $63, %r14d
; CHECK-NEXT: movq %rdi, %rax
@@ -96,17 +93,10 @@ define double @main(i224 %0) #0 {
; CHECK-NEXT: jmp .LBB0_6
; CHECK-NEXT: .LBB0_4: # %itofp-sw-bb
; CHECK-NEXT: movq %rsi, %rax
-; CHECK-NEXT: shlq %rax
-; CHECK-NEXT: movq %rdi, %r8
-; CHECK-NEXT: shrq $63, %r8
-; CHECK-NEXT: orq %r8, %rax
+; CHECK-NEXT: shldq $1, %rdi, %rax
; CHECK-NEXT: movq %rdx, %r8
-; CHECK-NEXT: shlq %r8
-; CHECK-NEXT: shrq $63, %rsi
-; CHECK-NEXT: orq %rsi, %r8
-; CHECK-NEXT: shlq %rcx
-; CHECK-NEXT: shrq $63, %rdx
-; CHECK-NEXT: orq %rdx, %rcx
+; CHECK-NEXT: shldq $1, %rsi, %r8
+; CHECK-NEXT: shldq $1, %rdx, %rcx
; CHECK-NEXT: addq %rdi, %rdi
; CHECK-NEXT: movq %rax, %rsi
; CHECK-NEXT: movq %r8, %rdx
@@ -192,19 +182,14 @@ define double @main(i224 %0) #0 {
; CHECK-NEXT: adcq $0, %rdx
; CHECK-NEXT: adcq $0, %rcx
; CHECK-NEXT: movq %rdi, %rdx
-; CHECK-NEXT: shrq $2, %rdx
-; CHECK-NEXT: movq %rsi, %rax
-; CHECK-NEXT: shlq $62, %rax
-; CHECK-NEXT: orq %rax, %rdx
+; CHECK-NEXT: shrdq $2, %rsi, %rdx
; CHECK-NEXT: movq %rdx, %rax
; CHECK-NEXT: shrq $32, %rax
; CHECK-NEXT: btq $55, %rdi
; CHECK-NEXT: jae .LBB0_9
; CHECK-NEXT: jmp .LBB0_7
; CHECK-NEXT: .LBB0_7: # %itofp-if-then20
-; CHECK-NEXT: shrq $3, %rdi
-; CHECK-NEXT: shlq $61, %rsi
-; CHECK-NEXT: orq %rsi, %rdi
+; CHECK-NEXT: shrdq $3, %rsi, %rdi
; CHECK-NEXT: movq %rdi, %rax
; CHECK-NEXT: shrq $32, %rax
; CHECK-NEXT: movq %rdi, %rdx
diff --git a/llvm/test/CodeGen/X86/fold-tied-op.ll b/llvm/test/CodeGen/X86/fold-tied-op.ll
index 2fec5397ae627..3dc083fbd673b 100644
--- a/llvm/test/CodeGen/X86/fold-tied-op.ll
+++ b/llvm/test/CodeGen/X86/fold-tied-op.ll
@@ -20,96 +20,88 @@ define i64 @fn1() #0 {
; CHECK-NEXT: pushl %ebx
; CHECK-NEXT: pushl %edi
; CHECK-NEXT: pushl %esi
-; CHECK-NEXT: subl $16, %esp
+; CHECK-NEXT: subl $12, %esp
; CHECK-NEXT: .cfi_offset %esi, -20
; CHECK-NEXT: .cfi_offset %edi, -16
; CHECK-NEXT: .cfi_offset %ebx, -12
-; CHECK-NEXT: movl $-1028477379, %edi # imm = 0xC2B2AE3D
-; CHECK-NEXT: movl $668265295, %ecx # imm = 0x27D4EB4F
-; CHECK-NEXT: movl a, %eax
-; CHECK-NEXT: cmpl $0, (%eax)
+; CHECK-NEXT: movl $-1028477379, %eax # imm = 0xC2B2AE3D
+; CHECK-NEXT: movl $668265295, %ebx # imm = 0x27D4EB4F
+; CHECK-NEXT: movl a, %edi
+; CHECK-NEXT: cmpl $0, (%edi)
; CHECK-NEXT: je .LBB0_2
; CHECK-NEXT: # %bb.1: # %if.then
-; CHECK-NEXT: movl %eax, %esi
-; CHECK-NEXT: movl 8(%eax), %eax
-; CHECK-NEXT: leal (%eax,%eax), %edx
-; CHECK-NEXT: orl %eax, %edx
-; CHECK-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; CHECK-NEXT: shrl $31, %eax
+; CHECK-NEXT: movl 8(%edi), %esi
+; CHECK-NEXT: movl 12(%edi), %edx
+; CHECK-NEXT: movl %edx, %eax
+; CHECK-NEXT: shldl $1, %esi, %eax
+; CHECK-NEXT: orl %edx, %eax
+; CHECK-NEXT: leal (%esi,%esi), %ecx
+; CHECK-NEXT: orl %esi, %ecx
+; CHECK-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT: movl 16(%edi), %ecx
+; CHECK-NEXT: movl 20(%edi), %esi
; CHECK-NEXT: movl %esi, %edx
-; CHECK-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; CHECK-NEXT: movl 12(%esi), %esi
-; CHECK-NEXT: leal (,%esi,2), %edi
-; CHECK-NEXT: orl %edi, %eax
-; CHECK-NEXT: orl %esi, %eax
-; CHECK-NEXT: movl 16(%edx), %esi
-; CHECK-NEXT: movl %esi, %edi
-; CHECK-NEXT: shrl $30, %edi
-; CHECK-NEXT: movl 20(%edx), %ebx
-; CHECK-NEXT: leal (,%ebx,4), %edx
-; CHECK-NEXT: orl %edx, %edi
-; CHECK-NEXT: leal (,%esi,4), %edx
+; CHECK-NEXT: shldl $2, %ecx, %edx
; CHECK-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT: leal (,%ecx,4), %edx
+; CHECK-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT: shrdl $1, %esi, %ecx
+; CHECK-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
; CHECK-NEXT: shrl %esi
-; CHECK-NEXT: movl %ebx, %edx
-; CHECK-NEXT: shll $31, %edx
-; CHECK-NEXT: orl %esi, %edx
-; CHECK-NEXT: movl %edx, %esi
; CHECK-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; CHECK-NEXT: shrl %ebx
-; CHECK-NEXT: orl %edi, %ebx
-; CHECK-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; CHECK-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; CHECK-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT: adcl %eax, %esi
; CHECK-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; CHECK-NEXT: adcl %eax, %ebx
-; CHECK-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; CHECK-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; CHECK-NEXT: movl 24(%ebx), %eax
+; CHECK-NEXT: movl 24(%edi), %eax
; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; CHECK-NEXT: movl $-1028477379, %edi # imm = 0xC2B2AE3D
-; CHECK-NEXT: imull %eax, %edi
-; CHECK-NEXT: mull %ecx
+; CHECK-NEXT: movl $-1028477379, %ecx # imm = 0xC2B2AE3D
+; CHECK-NEXT: imull %eax, %ecx
+; CHECK-NEXT: mull %ebx
; CHECK-NEXT: movl %eax, %esi
-; CHECK-NEXT: addl %edi, %edx
-; CHECK-NEXT: movl 28(%ebx), %edi
-; CHECK-NEXT: imull %edi, %ecx
-; CHECK-NEXT: addl %edx, %ecx
+; CHECK-NEXT: addl %ecx, %edx
+; CHECK-NEXT: movl 28(%edi), %edi
+; CHECK-NEXT: imull %edi, %ebx
+; CHECK-NEXT: addl %edx, %ebx
; CHECK-NEXT: movl $1336530590, %edx # imm = 0x4FA9D69E
-; CHECK-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; CHECK-NEXT: movl %ebx, %eax
+; CHECK-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; CHECK-NEXT: movl %ecx, %eax
; CHECK-NEXT: mull %edx
-; CHECK-NEXT: imull $-2056954758, %ebx, %ebx # imm = 0x85655C7A
-; CHECK-NEXT: addl %edx, %ebx
+; CHECK-NEXT: imull $-2056954758, %ecx, %ecx # imm = 0x85655C7A
+; CHECK-NEXT: addl %edx, %ecx
; CHECK-NEXT: imull $1336530590, %edi, %edx # imm = 0x4FA9D69E
-; CHECK-NEXT: addl %ebx, %edx
-; CHECK-NEXT: shrdl $3, %ecx, %esi
-; CHECK-NEXT: sarl $3, %ecx
-; CHECK-NEXT: orl %edx, %ecx
+; CHECK-NEXT: addl %ecx, %edx
+; CHECK-NEXT: shrdl $3, %ebx, %esi
+; CHECK-NEXT: sarl $3, %ebx
+; CHECK-NEXT: orl %edx, %ebx
; CHECK-NEXT: orl %eax, %esi
-; CHECK-NEXT: movl $-66860409, %ebx # imm = 0xFC03CA87
+; CHECK-NEXT: movl $-66860409, %ecx # imm = 0xFC03CA87
; CHECK-NEXT: movl %esi, %eax
-; CHECK-NEXT: mull %ebx
+; CHECK-NEXT: mull %ecx
; CHECK-NEXT: movl %eax, %edi
; CHECK-NEXT: imull $326129324, %esi, %eax # imm = 0x137056AC
; CHECK-NEXT: addl %edx, %eax
-; CHECK-NEXT: imull $-66860409, %ecx, %ecx # imm = 0xFC03CA87
+; CHECK-NEXT: imull $-66860409, %ebx, %ecx # imm = 0xFC03CA87
; CHECK-NEXT: addl %eax, %ecx
; CHECK-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
; CHECK-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
; CHECK-NEXT: movl %edi, b
; CHECK-NEXT: movl %edi, %eax
-; CHECK-NEXT: mull %ebx
+; CHECK-NEXT: movl $-66860409, %edx # imm = 0xFC03CA87
+; CHECK-NEXT: mull %edx
; CHECK-NEXT: imull $326129324, %edi, %esi # imm = 0x137056AC
; CHECK-NEXT: addl %edx, %esi
; CHECK-NEXT: movl %ecx, b+4
; CHECK-NEXT: imull $-66860409, %ecx, %ecx # imm = 0xFC03CA87
; CHECK-NEXT: jmp .LBB0_3
; CHECK-NEXT: .LBB0_2: # %if.else
-; CHECK-NEXT: xorl b+4, %edi
-; CHECK-NEXT: xorl b, %ecx
-; CHECK-NEXT: movl $1419758215, %edx # imm = 0x549FCA87
-; CHECK-NEXT: movl %ecx, %eax
-; CHECK-NEXT: mull %edx
-; CHECK-NEXT: imull $93298681, %ecx, %esi # imm = 0x58F9FF9
+; CHECK-NEXT: xorl b+4, %eax
+; CHECK-NEXT: xorl b, %ebx
+; CHECK-NEXT: movl $1419758215, %ecx # imm = 0x549FCA87
+; CHECK-NEXT: movl %eax, %edi
+; CHECK-NEXT: movl %ebx, %eax
+; CHECK-NEXT: mull %ecx
+; CHECK-NEXT: imull $93298681, %ebx, %esi # imm = 0x58F9FF9
; CHECK-NEXT: addl %edx, %esi
; CHECK-NEXT: imull $1419758215, %edi, %ecx # imm = 0x549FCA87
; CHECK-NEXT: .LBB0_3: # %if.end
@@ -118,7 +110,7 @@ define i64 @fn1() #0 {
; CHECK-NEXT: adcl $-2048144777, %ecx # imm = 0x85EBCA77
; CHECK-NEXT: movl %eax, b
; CHECK-NEXT: movl %ecx, b+4
-; CHECK-NEXT: addl $16, %esp
+; CHECK-NEXT: addl $12, %esp
; CHECK-NEXT: popl %esi
; CHECK-NEXT: popl %edi
; CHECK-NEXT: popl %ebx
diff --git a/llvm/test/CodeGen/X86/funnel-shift.ll b/llvm/test/CodeGen/X86/funnel-shift.ll
index 180d17baefa1a..318a48b92cb28 100644
--- a/llvm/test/CodeGen/X86/funnel-shift.ll
+++ b/llvm/test/CodeGen/X86/funnel-shift.ll
@@ -158,12 +158,9 @@ define i37 @fshl_i37(i37 %x, i37 %y, i37 %z) nounwind {
; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-SSE2-NEXT: andl $31, %eax
; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ebx
; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-SSE2-NEXT: shll $27, %ecx
-; X86-SSE2-NEXT: movl %edi, %ebx
-; X86-SSE2-NEXT: shrl $5, %ebx
-; X86-SSE2-NEXT: orl %ecx, %ebx
+; X86-SSE2-NEXT: shldl $27, %ebx, %edi
; X86-SSE2-NEXT: pushl $0
; X86-SSE2-NEXT: pushl $37
; X86-SSE2-NEXT: pushl %eax
@@ -174,17 +171,17 @@ define i37 @fshl_i37(i37 %x, i37 %y, i37 %z) nounwind {
; X86-SSE2-NEXT: testb $32, %cl
; X86-SSE2-NEXT: jne .LBB3_1
; X86-SSE2-NEXT: # %bb.2:
-; X86-SSE2-NEXT: movl %ebx, %edi
-; X86-SSE2-NEXT: movl %esi, %ebx
+; X86-SSE2-NEXT: movl %edi, %ebx
+; X86-SSE2-NEXT: movl %esi, %edi
; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %esi
; X86-SSE2-NEXT: jmp .LBB3_3
; X86-SSE2-NEXT: .LBB3_1:
-; X86-SSE2-NEXT: shll $27, %edi
+; X86-SSE2-NEXT: shll $27, %ebx
; X86-SSE2-NEXT: .LBB3_3:
-; X86-SSE2-NEXT: movl %ebx, %eax
-; X86-SSE2-NEXT: shldl %cl, %edi, %eax
+; X86-SSE2-NEXT: movl %edi, %eax
+; X86-SSE2-NEXT: shldl %cl, %ebx, %eax
; X86-SSE2-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-SSE2-NEXT: shldl %cl, %ebx, %esi
+; X86-SSE2-NEXT: shldl %cl, %edi, %esi
; X86-SSE2-NEXT: movl %esi, %edx
; X86-SSE2-NEXT: popl %esi
; X86-SSE2-NEXT: popl %edi
@@ -328,11 +325,8 @@ define i37 @fshr_i37(i37 %x, i37 %y, i37 %z) nounwind {
; X86-SSE2-NEXT: andl $31, %eax
; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %edi
; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-SSE2-NEXT: shll $27, %ecx
-; X86-SSE2-NEXT: movl %ebx, %esi
-; X86-SSE2-NEXT: shrl $5, %esi
-; X86-SSE2-NEXT: orl %ecx, %esi
+; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-SSE2-NEXT: shldl $27, %ebx, %esi
; X86-SSE2-NEXT: pushl $0
; X86-SSE2-NEXT: pushl $37
; X86-SSE2-NEXT: pushl %eax
@@ -1093,26 +1087,24 @@ define void @PR45265(i32 %0, ptr nocapture readonly %1) nounwind {
; X86-SSE2: # %bb.0:
; X86-SSE2-NEXT: pushl %edi
; X86-SSE2-NEXT: pushl %esi
+; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-SSE2-NEXT: leal (%ecx,%ecx,2), %edi
-; X86-SSE2-NEXT: movzwl 8(%esi,%edi,4), %edx
-; X86-SSE2-NEXT: movzbl 10(%esi,%edi,4), %eax
-; X86-SSE2-NEXT: shll $16, %eax
-; X86-SSE2-NEXT: orl %edx, %eax
-; X86-SSE2-NEXT: movl 4(%esi,%edi,4), %esi
-; X86-SSE2-NEXT: shll $24, %edx
-; X86-SSE2-NEXT: shrl $8, %esi
-; X86-SSE2-NEXT: orl %edx, %esi
-; X86-SSE2-NEXT: xorl %ecx, %esi
-; X86-SSE2-NEXT: sarl $31, %ecx
-; X86-SSE2-NEXT: shll $8, %eax
-; X86-SSE2-NEXT: movl %eax, %edx
-; X86-SSE2-NEXT: sarl $8, %edx
+; X86-SSE2-NEXT: leal (%eax,%eax,2), %esi
+; X86-SSE2-NEXT: movzwl 8(%ecx,%esi,4), %edx
+; X86-SSE2-NEXT: movl 4(%ecx,%esi,4), %edi
+; X86-SSE2-NEXT: shrdl $8, %edx, %edi
+; X86-SSE2-NEXT: xorl %eax, %edi
; X86-SSE2-NEXT: sarl $31, %eax
-; X86-SSE2-NEXT: shldl $24, %edx, %eax
-; X86-SSE2-NEXT: xorl %ecx, %eax
-; X86-SSE2-NEXT: orl %eax, %esi
+; X86-SSE2-NEXT: movzbl 10(%ecx,%esi,4), %ecx
+; X86-SSE2-NEXT: shll $16, %ecx
+; X86-SSE2-NEXT: orl %edx, %ecx
+; X86-SSE2-NEXT: shll $8, %ecx
+; X86-SSE2-NEXT: movl %ecx, %edx
+; X86-SSE2-NEXT: sarl $8, %edx
+; X86-SSE2-NEXT: sarl $31, %ecx
+; X86-SSE2-NEXT: shldl $24, %edx, %ecx
+; X86-SSE2-NEXT: xorl %eax, %ecx
+; X86-SSE2-NEXT: orl %ecx, %edi
; X86-SSE2-NEXT: jne .LBB50_1
; X86-SSE2-NEXT: # %bb.2:
; X86-SSE2-NEXT: popl %esi
diff --git a/llvm/test/CodeGen/X86/imul.ll b/llvm/test/CodeGen/X86/imul.ll
index cc623edc2691f..9131688c4efcc 100644
--- a/llvm/test/CodeGen/X86/imul.ll
+++ b/llvm/test/CodeGen/X86/imul.ll
@@ -100,21 +100,13 @@ define i64 @mulmin4096_64(i64 %A) {
;
; X86-LABEL: mulmin4096_64:
; X86: # %bb.0:
-; X86-NEXT: pushl %esi
-; X86-NEXT: .cfi_def_cfa_offset 8
-; X86-NEXT: .cfi_offset %esi, -8
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: shll $12, %ecx
-; X86-NEXT: movl %eax, %esi
-; X86-NEXT: shrl $20, %esi
-; X86-NEXT: orl %ecx, %esi
+; X86-NEXT: shldl $12, %eax, %ecx
; X86-NEXT: shll $12, %eax
; X86-NEXT: xorl %edx, %edx
; X86-NEXT: negl %eax
-; X86-NEXT: sbbl %esi, %edx
-; X86-NEXT: popl %esi
-; X86-NEXT: .cfi_def_cfa_offset 4
+; X86-NEXT: sbbl %ecx, %edx
; X86-NEXT: retl
%mul = mul i64 %A, -4096
ret i64 %mul
diff --git a/llvm/test/CodeGen/X86/known-bits.ll b/llvm/test/CodeGen/X86/known-bits.ll
index 3d94b89ac8a61..58a0595e4322a 100644
--- a/llvm/test/CodeGen/X86/known-bits.ll
+++ b/llvm/test/CodeGen/X86/known-bits.ll
@@ -101,9 +101,7 @@ define i128 @knownbits_mask_addc_shl(i64 %a0, i64 %a1, i64 %a2) nounwind {
; X86-NEXT: adcl {{[0-9]+}}(%esp), %edx
; X86-NEXT: adcl $0, %ecx
; X86-NEXT: shldl $22, %edx, %ecx
-; X86-NEXT: shrl $10, %esi
-; X86-NEXT: shll $22, %edx
-; X86-NEXT: orl %esi, %edx
+; X86-NEXT: shldl $22, %esi, %edx
; X86-NEXT: movl %edx, 8(%eax)
; X86-NEXT: movl %ecx, 12(%eax)
; X86-NEXT: movl $0, 4(%eax)
diff --git a/llvm/test/CodeGen/X86/midpoint-int.ll b/llvm/test/CodeGen/X86/midpoint-int.ll
index 3348c1693883c..ffbb4bf06debc 100644
--- a/llvm/test/CodeGen/X86/midpoint-int.ll
+++ b/llvm/test/CodeGen/X86/midpoint-int.ll
@@ -303,28 +303,25 @@ define i64 @scalar_i64_signed_reg_reg(i64 %a1, i64 %a2) nounwind {
; X86-NEXT: pushl %ebx
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-NEXT: movl {{[0-9]+}}(%esp), %ebp
; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %esi, %edx
-; X86-NEXT: subl %eax, %edx
+; X86-NEXT: movl %esi, %eax
+; X86-NEXT: subl %edx, %eax
; X86-NEXT: movl %ecx, %edi
; X86-NEXT: sbbl %ebp, %edi
-; X86-NEXT: subl %esi, %eax
+; X86-NEXT: subl %esi, %edx
; X86-NEXT: sbbl %ecx, %ebp
; X86-NEXT: setl %bl
; X86-NEXT: movzbl %bl, %ebx
; X86-NEXT: jl .LBB5_2
; X86-NEXT: # %bb.1:
; X86-NEXT: movl %ebp, %edi
-; X86-NEXT: movl %eax, %edx
+; X86-NEXT: movl %edx, %eax
; X86-NEXT: .LBB5_2:
; X86-NEXT: negl %ebx
-; X86-NEXT: shrl %edx
-; X86-NEXT: movl %edi, %eax
-; X86-NEXT: shll $31, %eax
-; X86-NEXT: orl %edx, %eax
+; X86-NEXT: shrdl $1, %edi, %eax
; X86-NEXT: shrl %edi
; X86-NEXT: movl %eax, %ebp
; X86-NEXT: imull %ebx, %ebp
@@ -442,29 +439,26 @@ define i64 @scalar_i64_signed_mem_reg(ptr %a1_addr, i64 %a2) nounwind {
; X86-NEXT: pushl %ebx
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl (%ecx), %esi
-; X86-NEXT: movl 4(%ecx), %ecx
-; X86-NEXT: movl %esi, %edx
-; X86-NEXT: subl %eax, %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl (%eax), %esi
+; X86-NEXT: movl 4(%eax), %ecx
+; X86-NEXT: movl %esi, %eax
+; X86-NEXT: subl %edx, %eax
; X86-NEXT: movl %ecx, %edi
; X86-NEXT: sbbl %ebp, %edi
-; X86-NEXT: subl %esi, %eax
+; X86-NEXT: subl %esi, %edx
; X86-NEXT: sbbl %ecx, %ebp
; X86-NEXT: setl %bl
; X86-NEXT: movzbl %bl, %ebx
; X86-NEXT: jl .LBB7_2
; X86-NEXT: # %bb.1:
; X86-NEXT: movl %ebp, %edi
-; X86-NEXT: movl %eax, %edx
+; X86-NEXT: movl %edx, %eax
; X86-NEXT: .LBB7_2:
; X86-NEXT: negl %ebx
-; X86-NEXT: shrl %edx
-; X86-NEXT: movl %edi, %eax
-; X86-NEXT: shll $31, %eax
-; X86-NEXT: orl %edx, %eax
+; X86-NEXT: shrdl $1, %edi, %eax
; X86-NEXT: shrl %edi
; X86-NEXT: movl %eax, %ebp
; X86-NEXT: imull %ebx, %ebp
@@ -516,27 +510,24 @@ define i64 @scalar_i64_signed_reg_mem(i64 %a1, ptr %a2_addr) nounwind {
; X86-NEXT: pushl %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl (%edx), %eax
-; X86-NEXT: movl 4(%edx), %ebp
-; X86-NEXT: movl %esi, %edx
-; X86-NEXT: subl %eax, %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl (%eax), %edx
+; X86-NEXT: movl 4(%eax), %ebp
+; X86-NEXT: movl %esi, %eax
+; X86-NEXT: subl %edx, %eax
; X86-NEXT: movl %ecx, %edi
; X86-NEXT: sbbl %ebp, %edi
-; X86-NEXT: subl %esi, %eax
+; X86-NEXT: subl %esi, %edx
; X86-NEXT: sbbl %ecx, %ebp
; X86-NEXT: setl %bl
; X86-NEXT: movzbl %bl, %ebx
; X86-NEXT: jl .LBB8_2
; X86-NEXT: # %bb.1:
; X86-NEXT: movl %ebp, %edi
-; X86-NEXT: movl %eax, %edx
+; X86-NEXT: movl %edx, %eax
; X86-NEXT: .LBB8_2:
; X86-NEXT: negl %ebx
-; X86-NEXT: shrl %edx
-; X86-NEXT: movl %edi, %eax
-; X86-NEXT: shll $31, %eax
-; X86-NEXT: orl %edx, %eax
+; X86-NEXT: shrdl $1, %edi, %eax
; X86-NEXT: shrl %edi
; X86-NEXT: movl %eax, %ebp
; X86-NEXT: imull %ebx, %ebp
@@ -587,30 +578,27 @@ define i64 @scalar_i64_signed_mem_mem(ptr %a1_addr, ptr %a2_addr) nounwind {
; X86-NEXT: pushl %ebx
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl (%eax), %esi
-; X86-NEXT: movl 4(%eax), %ecx
-; X86-NEXT: movl (%edx), %eax
-; X86-NEXT: movl 4(%edx), %ebp
-; X86-NEXT: movl %esi, %edx
-; X86-NEXT: subl %eax, %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl (%ecx), %esi
+; X86-NEXT: movl 4(%ecx), %ecx
+; X86-NEXT: movl (%eax), %edx
+; X86-NEXT: movl 4(%eax), %ebp
+; X86-NEXT: movl %esi, %eax
+; X86-NEXT: subl %edx, %eax
; X86-NEXT: movl %ecx, %edi
; X86-NEXT: sbbl %ebp, %edi
-; X86-NEXT: subl %esi, %eax
+; X86-NEXT: subl %esi, %edx
; X86-NEXT: sbbl %ecx, %ebp
; X86-NEXT: setl %bl
; X86-NEXT: movzbl %bl, %ebx
; X86-NEXT: jl .LBB9_2
; X86-NEXT: # %bb.1:
; X86-NEXT: movl %ebp, %edi
-; X86-NEXT: movl %eax, %edx
+; X86-NEXT: movl %edx, %eax
; X86-NEXT: .LBB9_2:
; X86-NEXT: negl %ebx
-; X86-NEXT: shrl %edx
-; X86-NEXT: movl %edi, %eax
-; X86-NEXT: shll $31, %eax
-; X86-NEXT: orl %edx, %eax
+; X86-NEXT: shrdl $1, %edi, %eax
; X86-NEXT: shrl %edi
; X86-NEXT: movl %eax, %ebp
; X86-NEXT: imull %ebx, %ebp
diff --git a/llvm/test/CodeGen/X86/or-lea.ll b/llvm/test/CodeGen/X86/or-lea.ll
index fb77b25fd2302..616ab99437892 100644
--- a/llvm/test/CodeGen/X86/or-lea.ll
+++ b/llvm/test/CodeGen/X86/or-lea.ll
@@ -175,13 +175,11 @@ define i64 @or_shift1_and1_64(i64 %x, i64 %y) {
; X86-LABEL: or_shift1_and1_64:
; X86: # %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: shll %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: shldl $1, %ecx, %edx
; X86-NEXT: andl $1, %eax
-; X86-NEXT: leal (%eax,%edx,2), %eax
-; X86-NEXT: shrl $31, %edx
-; X86-NEXT: orl %ecx, %edx
+; X86-NEXT: leal (%eax,%ecx,2), %eax
; X86-NEXT: retl
;
; X64-LABEL: or_shift1_and1_64:
diff --git a/llvm/test/CodeGen/X86/pr38539.ll b/llvm/test/CodeGen/X86/pr38539.ll
index 42cb41424daa5..eecd15dd2afe9 100644
--- a/llvm/test/CodeGen/X86/pr38539.ll
+++ b/llvm/test/CodeGen/X86/pr38539.ll
@@ -22,7 +22,7 @@ define void @f() nounwind {
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
; X86-NEXT: andl $-16, %esp
-; X86-NEXT: subl $160, %esp
+; X86-NEXT: subl $176, %esp
; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx
@@ -43,11 +43,8 @@ define void @f() nounwind {
; X86-NEXT: subl %ecx, %esi
; X86-NEXT: sbbl %eax, %ebx
; X86-NEXT: sbbl %eax, %edi
-; X86-NEXT: movl %edi, %eax
-; X86-NEXT: shll $30, %eax
-; X86-NEXT: movl %ebx, %ecx
-; X86-NEXT: shrl $2, %ecx
-; X86-NEXT: orl %eax, %ecx
+; X86-NEXT: movl %edi, %ecx
+; X86-NEXT: shldl $30, %ebx, %ecx
; X86-NEXT: movl %esi, %edx
; X86-NEXT: shrdl $2, %ebx, %edx
; X86-NEXT: testl %ecx, %ecx
@@ -103,17 +100,16 @@ define void @f() nounwind {
; X86-NEXT: testb %cl, %cl
; X86-NEXT: jne .LBB0_15
; X86-NEXT: # %bb.11: # %select.end
-; X86-NEXT: movl %ebx, %edx
; X86-NEXT: movl %eax, %ecx
; X86-NEXT: xorl $65, %ecx
; X86-NEXT: orl %esi, %ecx
; X86-NEXT: orl %ebx, %ecx
; X86-NEXT: je .LBB0_15
; X86-NEXT: # %bb.12: # %udiv-bb1
-; X86-NEXT: movl %eax, %ebx
-; X86-NEXT: addl $1, %ebx
-; X86-NEXT: adcl $0, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: addl $1, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: adcl $0, %ebx
; X86-NEXT: adcl $0, %esi
; X86-NEXT: andl $3, %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
@@ -123,7 +119,7 @@ define void @f() nounwind {
; X86-NEXT: shrb $3, %al
; X86-NEXT: andb $12, %al
; X86-NEXT: negb %al
-; X86-NEXT: movsbl %al, %eax
+; X86-NEXT: movsbl %al, %edx
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
@@ -132,23 +128,24 @@ define void @f() nounwind {
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 112(%esp,%eax), %edx
-; X86-NEXT: movl 116(%esp,%eax), %edi
-; X86-NEXT: movl 120(%esp,%eax), %eax
-; X86-NEXT: shldl %cl, %edi, %eax
-; X86-NEXT: shldl %cl, %edx, %edi
-; X86-NEXT: shll %cl, %edx
+; X86-NEXT: movl 128(%esp,%edx), %eax
+; X86-NEXT: movl 132(%esp,%edx), %edi
+; X86-NEXT: movl 136(%esp,%edx), %edx
+; X86-NEXT: shldl %cl, %edi, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shldl %cl, %eax, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shll %cl, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: orl %esi, %eax
; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %ebx, %ecx
-; X86-NEXT: orl %esi, %ecx
-; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: orl %ebx, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NEXT: je .LBB0_15
; X86-NEXT: # %bb.13: # %udiv-preheader
-; X86-NEXT: andl $3, %ebx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: andl $3, %eax
+; X86-NEXT: andl $3, %esi
+; X86-NEXT: andl $3, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
@@ -158,75 +155,59 @@ define void @f() nounwind {
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl %ecx, %edx
-; X86-NEXT: shrb $3, %dl
-; X86-NEXT: andb $12, %dl
-; X86-NEXT: movzbl %dl, %esi
-; X86-NEXT: movl 72(%esp,%esi), %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 64(%esp,%esi), %edx
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 68(%esp,%esi), %edi
-; X86-NEXT: movl %edi, %esi
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: shrdl %cl, %ebx, %esi
-; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: shrb $3, %al
+; X86-NEXT: andb $12, %al
+; X86-NEXT: movzbl %al, %eax
+; X86-NEXT: movl 88(%esp,%eax), %edi
+; X86-NEXT: movl 80(%esp,%eax), %ebx
+; X86-NEXT: movl 84(%esp,%eax), %eax
+; X86-NEXT: movl %eax, %edx
; X86-NEXT: shrdl %cl, %edi, %edx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: shrdl %cl, %eax, %ebx
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: addl $-1, %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: adcl $-1, %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: adcl $3, %ebx
-; X86-NEXT: andl $3, %ebx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: addl $-1, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: adcl $-1, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: adcl $3, %esi
+; X86-NEXT: andl $3, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: xorl %ecx, %ecx
; X86-NEXT: .p2align 4
; X86-NEXT: .LBB0_14: # %udiv-do-while
; X86-NEXT: # =>This Inner Loop Header: Depth=1
-; X86-NEXT: shll %ecx
-; X86-NEXT: movl %esi, %ebx
-; X86-NEXT: shrl $31, %ebx
-; X86-NEXT: orl %ecx, %ebx
-; X86-NEXT: shll %esi
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl %ecx, %edx
-; X86-NEXT: shrl $31, %edx
-; X86-NEXT: orl %esi, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %eax, %edx
-; X86-NEXT: andl $2, %edx
-; X86-NEXT: shrl %edx
-; X86-NEXT: leal (%edx,%ecx,2), %ecx
-; X86-NEXT: shll %eax
-; X86-NEXT: movl %edi, %edx
-; X86-NEXT: shrl $31, %edx
-; X86-NEXT: orl %eax, %edx
-; X86-NEXT: shll %edi
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: movl %eax, %esi
-; X86-NEXT: shrl $31, %esi
-; X86-NEXT: orl %edi, %esi
-; X86-NEXT: movl %esi, %edi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: shldl $1, %edx, %ecx
+; X86-NEXT: shldl $1, %ebx, %edx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: movl %edi, %esi
+; X86-NEXT: andl $2, %esi
+; X86-NEXT: shrl %esi
+; X86-NEXT: leal (%esi,%ebx,2), %ebx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: orl %esi, %edx
-; X86-NEXT: orl %esi, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shldl $1, %esi, %edi
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: shldl $1, %eax, %esi
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: addl %eax, %eax
; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: andl $3, %edx
-; X86-NEXT: cmpl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; X86-NEXT: andl $3, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: cmpl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NEXT: sbbl %edx, %esi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: sbbl %ebx, %esi
+; X86-NEXT: sbbl %ecx, %esi
; X86-NEXT: shll $30, %esi
; X86-NEXT: movl %esi, %edi
; X86-NEXT: sarl $31, %edi
@@ -239,28 +220,25 @@ define void @f() nounwind {
; X86-NEXT: movl %edi, %eax
; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X86-NEXT: subl %esi, %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: sbbl %edi, %ecx
-; X86-NEXT: sbbl %eax, %ebx
-; X86-NEXT: andl $3, %ebx
+; X86-NEXT: subl %esi, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %edi, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %eax, %ecx
+; X86-NEXT: andl $3, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: addl $-1, %eax
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: adcl $-1, %edi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: adcl $3, %esi
-; X86-NEXT: andl $3, %esi
+; X86-NEXT: adcl $-1, %esi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: adcl $3, %ebx
+; X86-NEXT: andl $3, %ebx
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %ebx, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: orl %esi, %eax
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %edi, %eax
-; X86-NEXT: movl %ecx, %esi
-; X86-NEXT: movl %ebx, %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: movl %edx, %eax
; X86-NEXT: jne .LBB0_14
; X86-NEXT: .LBB0_15: # %udiv-end
; X86-NEXT: cmpb $0, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Reload
diff --git a/llvm/test/CodeGen/X86/pr43820.ll b/llvm/test/CodeGen/X86/pr43820.ll
index 3c260af2e504a..bf553c02fea3f 100644
--- a/llvm/test/CodeGen/X86/pr43820.ll
+++ b/llvm/test/CodeGen/X86/pr43820.ll
@@ -10,219 +10,219 @@ define i1000 @square(i1000 %A) nounwind {
; CHECK-NEXT: pushq %r13
; CHECK-NEXT: pushq %r12
; CHECK-NEXT: pushq %rbx
-; CHECK-NEXT: movq %rdx, %rbx
-; CHECK-NEXT: movq %rsi, %rbp
-; CHECK-NEXT: movq %rdi, %r12
+; CHECK-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rdi
; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r14
; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r15
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; CHECK-NEXT: bswapq %rax
-; CHECK-NEXT: movq %rax, %rsi
-; CHECK-NEXT: shrq $4, %rsi
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; CHECK-NEXT: bswapq %rbx
+; CHECK-NEXT: movq %rbx, %r10
+; CHECK-NEXT: shrq $4, %r10
; CHECK-NEXT: movabsq $1085102592571150095, %r11 # imm = 0xF0F0F0F0F0F0F0F
-; CHECK-NEXT: andq %r11, %rsi
-; CHECK-NEXT: andq %r11, %rax
-; CHECK-NEXT: shlq $4, %rax
-; CHECK-NEXT: orq %rsi, %rax
+; CHECK-NEXT: andq %r11, %r10
+; CHECK-NEXT: andq %r11, %rbx
+; CHECK-NEXT: shlq $4, %rbx
+; CHECK-NEXT: orq %r10, %rbx
; CHECK-NEXT: movabsq $3689348814741910323, %r10 # imm = 0x3333333333333333
-; CHECK-NEXT: movq %rax, %rsi
-; CHECK-NEXT: andq %r10, %rsi
-; CHECK-NEXT: shrq $2, %rax
-; CHECK-NEXT: andq %r10, %rax
-; CHECK-NEXT: leaq (%rax,%rsi,4), %rax
-; CHECK-NEXT: movabsq $6148914691236517205, %rsi # imm = 0x5555555555555555
-; CHECK-NEXT: movq %rax, %r13
-; CHECK-NEXT: andq %rsi, %r13
-; CHECK-NEXT: shrq %rax
-; CHECK-NEXT: andq %rsi, %rax
-; CHECK-NEXT: leaq (%rax,%r13,2), %rax
+; CHECK-NEXT: movq %rbx, %r12
+; CHECK-NEXT: andq %r10, %r12
+; CHECK-NEXT: shrq $2, %rbx
+; CHECK-NEXT: andq %r10, %rbx
+; CHECK-NEXT: leaq (%rbx,%r12,4), %r12
+; CHECK-NEXT: movabsq $6148914691236517205, %rbx # imm = 0x5555555555555555
+; CHECK-NEXT: movq %r12, %r13
+; CHECK-NEXT: andq %rbx, %r13
+; CHECK-NEXT: shrq %r12
+; CHECK-NEXT: andq %rbx, %r12
+; CHECK-NEXT: leaq (%r12,%r13,2), %rax
; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-NEXT: bswapq %r15
-; CHECK-NEXT: movq %r15, %rax
-; CHECK-NEXT: shrq $4, %rax
-; CHECK-NEXT: andq %r11, %rax
+; CHECK-NEXT: movq %r15, %r12
+; CHECK-NEXT: shrq $4, %r12
+; CHECK-NEXT: andq %r11, %r12
; CHECK-NEXT: andq %r11, %r15
; CHECK-NEXT: shlq $4, %r15
-; CHECK-NEXT: orq %rax, %r15
-; CHECK-NEXT: movq %r15, %rax
-; CHECK-NEXT: andq %r10, %rax
+; CHECK-NEXT: orq %r12, %r15
+; CHECK-NEXT: movq %r15, %r12
+; CHECK-NEXT: andq %r10, %r12
; CHECK-NEXT: shrq $2, %r15
; CHECK-NEXT: andq %r10, %r15
-; CHECK-NEXT: leaq (%r15,%rax,4), %rax
-; CHECK-NEXT: movabsq $6148914691230924800, %r15 # imm = 0x5555555555000000
-; CHECK-NEXT: movq %rax, %r13
-; CHECK-NEXT: andq %r15, %r13
-; CHECK-NEXT: shrq %rax
-; CHECK-NEXT: andq %r15, %rax
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r15
-; CHECK-NEXT: leaq (%rax,%r13,2), %rax
+; CHECK-NEXT: leaq (%r15,%r12,4), %r15
+; CHECK-NEXT: movabsq $6148914691230924800, %r12 # imm = 0x5555555555000000
+; CHECK-NEXT: movq %r15, %r13
+; CHECK-NEXT: andq %r12, %r13
+; CHECK-NEXT: shrq %r15
+; CHECK-NEXT: andq %r12, %r15
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r12
+; CHECK-NEXT: leaq (%r15,%r13,2), %rax
; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: bswapq %r15
-; CHECK-NEXT: movq %r15, %rax
-; CHECK-NEXT: shrq $4, %rax
-; CHECK-NEXT: andq %r11, %rax
+; CHECK-NEXT: bswapq %r12
+; CHECK-NEXT: movq %r12, %r15
+; CHECK-NEXT: shrq $4, %r15
; CHECK-NEXT: andq %r11, %r15
-; CHECK-NEXT: shlq $4, %r15
-; CHECK-NEXT: orq %rax, %r15
-; CHECK-NEXT: movq %r15, %rax
-; CHECK-NEXT: andq %r10, %rax
-; CHECK-NEXT: shrq $2, %r15
+; CHECK-NEXT: andq %r11, %r12
+; CHECK-NEXT: shlq $4, %r12
+; CHECK-NEXT: orq %r15, %r12
+; CHECK-NEXT: movq %r12, %r15
; CHECK-NEXT: andq %r10, %r15
-; CHECK-NEXT: leaq (%r15,%rax,4), %rax
-; CHECK-NEXT: movq %rax, %r15
-; CHECK-NEXT: andq %rsi, %r15
-; CHECK-NEXT: shrq %rax
-; CHECK-NEXT: andq %rsi, %rax
-; CHECK-NEXT: leaq (%rax,%r15,2), %rax
+; CHECK-NEXT: shrq $2, %r12
+; CHECK-NEXT: andq %r10, %r12
+; CHECK-NEXT: leaq (%r12,%r15,4), %r15
+; CHECK-NEXT: movq %r15, %r12
+; CHECK-NEXT: andq %rbx, %r12
+; CHECK-NEXT: shrq %r15
+; CHECK-NEXT: andq %rbx, %r15
+; CHECK-NEXT: leaq (%r15,%r12,2), %rax
; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-NEXT: bswapq %r14
-; CHECK-NEXT: movq %r14, %rax
-; CHECK-NEXT: shrq $4, %rax
-; CHECK-NEXT: andq %r11, %rax
+; CHECK-NEXT: movq %r14, %r15
+; CHECK-NEXT: shrq $4, %r15
+; CHECK-NEXT: andq %r11, %r15
; CHECK-NEXT: andq %r11, %r14
; CHECK-NEXT: shlq $4, %r14
-; CHECK-NEXT: orq %rax, %r14
-; CHECK-NEXT: movq %r14, %rax
-; CHECK-NEXT: andq %r10, %rax
+; CHECK-NEXT: orq %r15, %r14
+; CHECK-NEXT: movq %r14, %r15
+; CHECK-NEXT: andq %r10, %r15
; CHECK-NEXT: shrq $2, %r14
; CHECK-NEXT: andq %r10, %r14
-; CHECK-NEXT: leaq (%r14,%rax,4), %rax
-; CHECK-NEXT: movq %rax, %r14
-; CHECK-NEXT: andq %rsi, %r14
-; CHECK-NEXT: shrq %rax
-; CHECK-NEXT: andq %rsi, %rax
-; CHECK-NEXT: leaq (%rax,%r14,2), %rax
+; CHECK-NEXT: leaq (%r14,%r15,4), %r14
+; CHECK-NEXT: movq %r14, %r15
+; CHECK-NEXT: andq %rbx, %r15
+; CHECK-NEXT: shrq %r14
+; CHECK-NEXT: andq %rbx, %r14
+; CHECK-NEXT: leaq (%r14,%r15,2), %rax
; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-NEXT: bswapq %rdi
-; CHECK-NEXT: movq %rdi, %rax
-; CHECK-NEXT: shrq $4, %rax
-; CHECK-NEXT: andq %r11, %rax
+; CHECK-NEXT: movq %rdi, %r14
+; CHECK-NEXT: shrq $4, %r14
+; CHECK-NEXT: andq %r11, %r14
; CHECK-NEXT: andq %r11, %rdi
; CHECK-NEXT: shlq $4, %rdi
-; CHECK-NEXT: orq %rax, %rdi
-; CHECK-NEXT: movq %rdi, %rax
-; CHECK-NEXT: andq %r10, %rax
+; CHECK-NEXT: orq %r14, %rdi
+; CHECK-NEXT: movq %rdi, %r14
+; CHECK-NEXT: andq %r10, %r14
; CHECK-NEXT: shrq $2, %rdi
; CHECK-NEXT: andq %r10, %rdi
-; CHECK-NEXT: leaq (%rdi,%rax,4), %rax
-; CHECK-NEXT: movq %rax, %rdi
-; CHECK-NEXT: andq %rsi, %rdi
-; CHECK-NEXT: shrq %rax
-; CHECK-NEXT: andq %rsi, %rax
-; CHECK-NEXT: leaq (%rax,%rdi,2), %rax
+; CHECK-NEXT: leaq (%rdi,%r14,4), %rdi
+; CHECK-NEXT: movq %rdi, %r14
+; CHECK-NEXT: andq %rbx, %r14
+; CHECK-NEXT: shrq %rdi
+; CHECK-NEXT: andq %rbx, %rdi
+; CHECK-NEXT: leaq (%rdi,%r14,2), %rax
; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; CHECK-NEXT: bswapq %rax
-; CHECK-NEXT: movq %rax, %rdi
-; CHECK-NEXT: shrq $4, %rdi
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rdi
+; CHECK-NEXT: bswapq %rdi
+; CHECK-NEXT: movq %rdi, %r14
+; CHECK-NEXT: shrq $4, %r14
+; CHECK-NEXT: andq %r11, %r14
; CHECK-NEXT: andq %r11, %rdi
-; CHECK-NEXT: andq %r11, %rax
-; CHECK-NEXT: shlq $4, %rax
-; CHECK-NEXT: orq %rdi, %rax
-; CHECK-NEXT: movq %rax, %rdi
+; CHECK-NEXT: shlq $4, %rdi
+; CHECK-NEXT: orq %r14, %rdi
+; CHECK-NEXT: movq %rdi, %r14
+; CHECK-NEXT: andq %r10, %r14
+; CHECK-NEXT: shrq $2, %rdi
; CHECK-NEXT: andq %r10, %rdi
-; CHECK-NEXT: shrq $2, %rax
-; CHECK-NEXT: andq %r10, %rax
-; CHECK-NEXT: leaq (%rax,%rdi,4), %rax
-; CHECK-NEXT: movq %rax, %rdi
-; CHECK-NEXT: andq %rsi, %rdi
-; CHECK-NEXT: shrq %rax
-; CHECK-NEXT: andq %rsi, %rax
-; CHECK-NEXT: leaq (%rax,%rdi,2), %rax
+; CHECK-NEXT: leaq (%rdi,%r14,4), %rdi
+; CHECK-NEXT: movq %rdi, %r14
+; CHECK-NEXT: andq %rbx, %r14
+; CHECK-NEXT: shrq %rdi
+; CHECK-NEXT: andq %rbx, %rdi
+; CHECK-NEXT: leaq (%rdi,%r14,2), %rax
; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; CHECK-NEXT: bswapq %rax
-; CHECK-NEXT: movq %rax, %rdi
-; CHECK-NEXT: shrq $4, %rdi
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rdi
+; CHECK-NEXT: bswapq %rdi
+; CHECK-NEXT: movq %rdi, %r14
+; CHECK-NEXT: shrq $4, %r14
+; CHECK-NEXT: andq %r11, %r14
; CHECK-NEXT: andq %r11, %rdi
-; CHECK-NEXT: andq %r11, %rax
-; CHECK-NEXT: shlq $4, %rax
-; CHECK-NEXT: orq %rdi, %rax
-; CHECK-NEXT: movq %rax, %rdi
+; CHECK-NEXT: shlq $4, %rdi
+; CHECK-NEXT: orq %r14, %rdi
+; CHECK-NEXT: movq %rdi, %r14
+; CHECK-NEXT: andq %r10, %r14
+; CHECK-NEXT: shrq $2, %rdi
; CHECK-NEXT: andq %r10, %rdi
-; CHECK-NEXT: shrq $2, %rax
-; CHECK-NEXT: andq %r10, %rax
-; CHECK-NEXT: leaq (%rax,%rdi,4), %rax
-; CHECK-NEXT: movq %rax, %rdi
-; CHECK-NEXT: andq %rsi, %rdi
-; CHECK-NEXT: shrq %rax
-; CHECK-NEXT: andq %rsi, %rax
-; CHECK-NEXT: leaq (%rax,%rdi,2), %rax
+; CHECK-NEXT: leaq (%rdi,%r14,4), %rdi
+; CHECK-NEXT: movq %rdi, %r14
+; CHECK-NEXT: andq %rbx, %r14
+; CHECK-NEXT: shrq %rdi
+; CHECK-NEXT: andq %rbx, %rdi
+; CHECK-NEXT: leaq (%rdi,%r14,2), %rax
; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; CHECK-NEXT: bswapq %rax
-; CHECK-NEXT: movq %rax, %rdi
-; CHECK-NEXT: shrq $4, %rdi
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rdi
+; CHECK-NEXT: bswapq %rdi
+; CHECK-NEXT: movq %rdi, %r14
+; CHECK-NEXT: shrq $4, %r14
+; CHECK-NEXT: andq %r11, %r14
; CHECK-NEXT: andq %r11, %rdi
-; CHECK-NEXT: andq %r11, %rax
-; CHECK-NEXT: shlq $4, %rax
-; CHECK-NEXT: orq %rdi, %rax
-; CHECK-NEXT: movq %rax, %rdi
+; CHECK-NEXT: shlq $4, %rdi
+; CHECK-NEXT: orq %r14, %rdi
+; CHECK-NEXT: movq %rdi, %r14
+; CHECK-NEXT: andq %r10, %r14
+; CHECK-NEXT: shrq $2, %rdi
; CHECK-NEXT: andq %r10, %rdi
-; CHECK-NEXT: shrq $2, %rax
-; CHECK-NEXT: andq %r10, %rax
-; CHECK-NEXT: leaq (%rax,%rdi,4), %rax
-; CHECK-NEXT: movq %rax, %rdi
-; CHECK-NEXT: andq %rsi, %rdi
-; CHECK-NEXT: shrq %rax
-; CHECK-NEXT: andq %rsi, %rax
-; CHECK-NEXT: leaq (%rax,%rdi,2), %rax
+; CHECK-NEXT: leaq (%rdi,%r14,4), %rdi
+; CHECK-NEXT: movq %rdi, %r14
+; CHECK-NEXT: andq %rbx, %r14
+; CHECK-NEXT: shrq %rdi
+; CHECK-NEXT: andq %rbx, %rdi
+; CHECK-NEXT: leaq (%rdi,%r14,2), %rax
; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; CHECK-NEXT: bswapq %rax
-; CHECK-NEXT: movq %rax, %rdi
-; CHECK-NEXT: shrq $4, %rdi
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rdi
+; CHECK-NEXT: bswapq %rdi
+; CHECK-NEXT: movq %rdi, %r14
+; CHECK-NEXT: shrq $4, %r14
+; CHECK-NEXT: andq %r11, %r14
; CHECK-NEXT: andq %r11, %rdi
-; CHECK-NEXT: andq %r11, %rax
-; CHECK-NEXT: shlq $4, %rax
-; CHECK-NEXT: orq %rdi, %rax
-; CHECK-NEXT: movq %rax, %rdi
+; CHECK-NEXT: shlq $4, %rdi
+; CHECK-NEXT: orq %r14, %rdi
+; CHECK-NEXT: movq %rdi, %r14
+; CHECK-NEXT: andq %r10, %r14
+; CHECK-NEXT: shrq $2, %rdi
; CHECK-NEXT: andq %r10, %rdi
-; CHECK-NEXT: shrq $2, %rax
-; CHECK-NEXT: andq %r10, %rax
-; CHECK-NEXT: leaq (%rax,%rdi,4), %rax
-; CHECK-NEXT: movq %rax, %rdi
-; CHECK-NEXT: andq %rsi, %rdi
-; CHECK-NEXT: shrq %rax
-; CHECK-NEXT: andq %rsi, %rax
-; CHECK-NEXT: leaq (%rax,%rdi,2), %r13
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; CHECK-NEXT: bswapq %rax
-; CHECK-NEXT: movq %rax, %rdi
-; CHECK-NEXT: shrq $4, %rdi
+; CHECK-NEXT: leaq (%rdi,%r14,4), %rdi
+; CHECK-NEXT: movq %rdi, %r14
+; CHECK-NEXT: andq %rbx, %r14
+; CHECK-NEXT: shrq %rdi
+; CHECK-NEXT: andq %rbx, %rdi
+; CHECK-NEXT: leaq (%rdi,%r14,2), %rax
+; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rdi
+; CHECK-NEXT: bswapq %rdi
+; CHECK-NEXT: movq %rdi, %r14
+; CHECK-NEXT: shrq $4, %r14
+; CHECK-NEXT: andq %r11, %r14
; CHECK-NEXT: andq %r11, %rdi
-; CHECK-NEXT: andq %r11, %rax
-; CHECK-NEXT: shlq $4, %rax
-; CHECK-NEXT: orq %rdi, %rax
-; CHECK-NEXT: movq %rax, %rdi
+; CHECK-NEXT: shlq $4, %rdi
+; CHECK-NEXT: orq %r14, %rdi
+; CHECK-NEXT: movq %rdi, %r14
+; CHECK-NEXT: andq %r10, %r14
+; CHECK-NEXT: shrq $2, %rdi
; CHECK-NEXT: andq %r10, %rdi
-; CHECK-NEXT: shrq $2, %rax
-; CHECK-NEXT: andq %r10, %rax
-; CHECK-NEXT: leaq (%rax,%rdi,4), %rax
-; CHECK-NEXT: movq %rax, %rdi
-; CHECK-NEXT: andq %rsi, %rdi
-; CHECK-NEXT: shrq %rax
-; CHECK-NEXT: andq %rsi, %rax
-; CHECK-NEXT: leaq (%rax,%rdi,2), %r15
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; CHECK-NEXT: bswapq %rax
-; CHECK-NEXT: movq %rax, %rdi
-; CHECK-NEXT: shrq $4, %rdi
-; CHECK-NEXT: andq %r11, %rdi
+; CHECK-NEXT: leaq (%rdi,%r14,4), %rdi
+; CHECK-NEXT: movq %rdi, %r14
+; CHECK-NEXT: andq %rbx, %r14
+; CHECK-NEXT: shrq %rdi
+; CHECK-NEXT: andq %rbx, %rdi
+; CHECK-NEXT: leaq (%rdi,%r14,2), %rax
+; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rdi
+; CHECK-NEXT: bswapq %rdi
+; CHECK-NEXT: movq %rdi, %rax
+; CHECK-NEXT: shrq $4, %rax
; CHECK-NEXT: andq %r11, %rax
-; CHECK-NEXT: shlq $4, %rax
-; CHECK-NEXT: orq %rdi, %rax
-; CHECK-NEXT: movq %rax, %rdi
-; CHECK-NEXT: andq %r10, %rdi
-; CHECK-NEXT: shrq $2, %rax
+; CHECK-NEXT: andq %r11, %rdi
+; CHECK-NEXT: shlq $4, %rdi
+; CHECK-NEXT: orq %rax, %rdi
+; CHECK-NEXT: movq %rdi, %rax
; CHECK-NEXT: andq %r10, %rax
-; CHECK-NEXT: leaq (%rax,%rdi,4), %rax
+; CHECK-NEXT: shrq $2, %rdi
+; CHECK-NEXT: andq %r10, %rdi
+; CHECK-NEXT: leaq (%rdi,%rax,4), %rax
; CHECK-NEXT: movq %rax, %rdi
-; CHECK-NEXT: andq %rsi, %rdi
+; CHECK-NEXT: andq %rbx, %rdi
; CHECK-NEXT: shrq %rax
-; CHECK-NEXT: andq %rsi, %rax
-; CHECK-NEXT: leaq (%rax,%rdi,2), %r14
+; CHECK-NEXT: andq %rbx, %rax
+; CHECK-NEXT: leaq (%rax,%rdi,2), %rdi
; CHECK-NEXT: bswapq %r9
; CHECK-NEXT: movq %r9, %rax
; CHECK-NEXT: shrq $4, %rax
@@ -236,10 +236,11 @@ define i1000 @square(i1000 %A) nounwind {
; CHECK-NEXT: andq %r10, %r9
; CHECK-NEXT: leaq (%r9,%rax,4), %rax
; CHECK-NEXT: movq %rax, %r9
-; CHECK-NEXT: andq %rsi, %r9
+; CHECK-NEXT: andq %rbx, %r9
; CHECK-NEXT: shrq %rax
-; CHECK-NEXT: andq %rsi, %rax
-; CHECK-NEXT: leaq (%rax,%r9,2), %r9
+; CHECK-NEXT: andq %rbx, %rax
+; CHECK-NEXT: leaq (%rax,%r9,2), %rax
+; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-NEXT: bswapq %r8
; CHECK-NEXT: movq %r8, %rax
; CHECK-NEXT: shrq $4, %rax
@@ -253,131 +254,116 @@ define i1000 @square(i1000 %A) nounwind {
; CHECK-NEXT: andq %r10, %r8
; CHECK-NEXT: leaq (%r8,%rax,4), %rax
; CHECK-NEXT: movq %rax, %r8
-; CHECK-NEXT: andq %rsi, %r8
+; CHECK-NEXT: andq %rbx, %r8
; CHECK-NEXT: shrq %rax
-; CHECK-NEXT: andq %rsi, %rax
-; CHECK-NEXT: leaq (%rax,%r8,2), %rdx
-; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
-; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
-; CHECK-NEXT: shrdq $24, %r8, %rdi
-; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; CHECK-NEXT: shrdq $24, %rax, %r8
-; CHECK-NEXT: movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
-; CHECK-NEXT: shrdq $24, %r8, %rax
-; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; CHECK-NEXT: shrdq $24, %rax, %r8
-; CHECK-NEXT: movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
-; CHECK-NEXT: shrdq $24, %r8, %rax
-; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; CHECK-NEXT: shrdq $24, %rax, %r8
-; CHECK-NEXT: movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
-; CHECK-NEXT: shrdq $24, %r8, %rax
+; CHECK-NEXT: andq %rbx, %rax
+; CHECK-NEXT: leaq (%rax,%r8,2), %rax
; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: shrdq $24, %r13, %r8
-; CHECK-NEXT: movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; CHECK-NEXT: shrdq $24, %r15, %r13
-; CHECK-NEXT: shrdq $24, %r14, %r15
-; CHECK-NEXT: shrdq $24, %r9, %r14
-; CHECK-NEXT: movq %rdx, %r8
-; CHECK-NEXT: shlq $40, %r8
-; CHECK-NEXT: shrq $24, %r9
-; CHECK-NEXT: orq %r8, %r9
; CHECK-NEXT: bswapq %rcx
-; CHECK-NEXT: movq %rcx, %r8
-; CHECK-NEXT: shrq $4, %r8
-; CHECK-NEXT: andq %r11, %r8
+; CHECK-NEXT: movq %rcx, %rax
+; CHECK-NEXT: shrq $4, %rax
+; CHECK-NEXT: andq %r11, %rax
; CHECK-NEXT: andq %r11, %rcx
; CHECK-NEXT: shlq $4, %rcx
-; CHECK-NEXT: orq %r8, %rcx
-; CHECK-NEXT: movq %rcx, %r8
-; CHECK-NEXT: andq %r10, %r8
+; CHECK-NEXT: orq %rax, %rcx
+; CHECK-NEXT: movq %rcx, %rax
+; CHECK-NEXT: andq %r10, %rax
; CHECK-NEXT: shrq $2, %rcx
; CHECK-NEXT: andq %r10, %rcx
-; CHECK-NEXT: leaq (%rcx,%r8,4), %rcx
-; CHECK-NEXT: movq %rcx, %r8
-; CHECK-NEXT: andq %rsi, %r8
-; CHECK-NEXT: shrq %rcx
-; CHECK-NEXT: andq %rsi, %rcx
-; CHECK-NEXT: leaq (%rcx,%r8,2), %rcx
-; CHECK-NEXT: shrq $24, %rdx
-; CHECK-NEXT: movq %rcx, %r8
-; CHECK-NEXT: shlq $40, %r8
-; CHECK-NEXT: orq %rdx, %r8
-; CHECK-NEXT: bswapq %rbx
-; CHECK-NEXT: movq %rbx, %rax
+; CHECK-NEXT: leaq (%rcx,%rax,4), %rax
+; CHECK-NEXT: movq %rax, %rcx
+; CHECK-NEXT: andq %rbx, %rcx
+; CHECK-NEXT: shrq %rax
+; CHECK-NEXT: andq %rbx, %rax
+; CHECK-NEXT: leaq (%rax,%rcx,2), %r14
+; CHECK-NEXT: bswapq %rdx
+; CHECK-NEXT: movq %rdx, %rax
; CHECK-NEXT: shrq $4, %rax
; CHECK-NEXT: andq %r11, %rax
-; CHECK-NEXT: andq %r11, %rbx
-; CHECK-NEXT: shlq $4, %rbx
-; CHECK-NEXT: orq %rax, %rbx
-; CHECK-NEXT: movq %rbx, %rax
+; CHECK-NEXT: andq %r11, %rdx
+; CHECK-NEXT: shlq $4, %rdx
+; CHECK-NEXT: orq %rax, %rdx
+; CHECK-NEXT: movq %rdx, %rax
; CHECK-NEXT: andq %r10, %rax
-; CHECK-NEXT: shrq $2, %rbx
-; CHECK-NEXT: andq %r10, %rbx
-; CHECK-NEXT: leaq (%rbx,%rax,4), %rax
-; CHECK-NEXT: movq %rax, %rbx
-; CHECK-NEXT: andq %rsi, %rbx
+; CHECK-NEXT: shrq $2, %rdx
+; CHECK-NEXT: andq %r10, %rdx
+; CHECK-NEXT: leaq (%rdx,%rax,4), %rax
+; CHECK-NEXT: movq %rax, %rdx
+; CHECK-NEXT: andq %rbx, %rdx
; CHECK-NEXT: shrq %rax
-; CHECK-NEXT: andq %rsi, %rax
-; CHECK-NEXT: leaq (%rax,%rbx,2), %rax
-; CHECK-NEXT: movq %rax, %rbx
-; CHECK-NEXT: shlq $40, %rbx
-; CHECK-NEXT: shrq $24, %rcx
-; CHECK-NEXT: orq %rbx, %rcx
-; CHECK-NEXT: bswapq %rbp
-; CHECK-NEXT: movq %rbp, %rbx
-; CHECK-NEXT: shrq $4, %rbx
-; CHECK-NEXT: andq %r11, %rbx
-; CHECK-NEXT: andq %r11, %rbp
-; CHECK-NEXT: shlq $4, %rbp
-; CHECK-NEXT: orq %rbx, %rbp
-; CHECK-NEXT: movq %rbp, %r11
-; CHECK-NEXT: andq %r10, %r11
-; CHECK-NEXT: shrq $2, %rbp
-; CHECK-NEXT: andq %r10, %rbp
-; CHECK-NEXT: leaq (%rbp,%r11,4), %r10
-; CHECK-NEXT: movq %r10, %r11
-; CHECK-NEXT: andq %rsi, %r11
-; CHECK-NEXT: shrq %r10
-; CHECK-NEXT: andq %rsi, %r10
-; CHECK-NEXT: leaq (%r10,%r11,2), %rsi
-; CHECK-NEXT: shrq $24, %rax
-; CHECK-NEXT: movq %rsi, %r10
-; CHECK-NEXT: shlq $40, %r10
-; CHECK-NEXT: orq %rax, %r10
-; CHECK-NEXT: movq %r10, 112(%r12)
-; CHECK-NEXT: movq %rcx, 104(%r12)
-; CHECK-NEXT: movq %r8, 96(%r12)
-; CHECK-NEXT: movq %r9, 88(%r12)
-; CHECK-NEXT: movq %r14, 80(%r12)
-; CHECK-NEXT: movq %r15, 72(%r12)
-; CHECK-NEXT: movq %r13, 64(%r12)
-; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; CHECK-NEXT: movq %rax, 56(%r12)
-; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; CHECK-NEXT: movq %rax, 48(%r12)
-; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; CHECK-NEXT: movq %rax, 40(%r12)
-; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; CHECK-NEXT: movq %rax, 32(%r12)
+; CHECK-NEXT: andq %rbx, %rax
+; CHECK-NEXT: leaq (%rax,%rdx,2), %rdx
+; CHECK-NEXT: bswapq %rsi
+; CHECK-NEXT: movq %rsi, %rax
+; CHECK-NEXT: shrq $4, %rax
+; CHECK-NEXT: andq %r11, %rax
+; CHECK-NEXT: andq %r11, %rsi
+; CHECK-NEXT: shlq $4, %rsi
+; CHECK-NEXT: orq %rax, %rsi
+; CHECK-NEXT: movq %rsi, %rax
+; CHECK-NEXT: andq %r10, %rax
+; CHECK-NEXT: shrq $2, %rsi
+; CHECK-NEXT: andq %r10, %rsi
+; CHECK-NEXT: leaq (%rsi,%rax,4), %rax
+; CHECK-NEXT: movq %rax, %rsi
+; CHECK-NEXT: andq %rbx, %rsi
+; CHECK-NEXT: shrq %rax
+; CHECK-NEXT: andq %rbx, %rax
+; CHECK-NEXT: leaq (%rax,%rsi,2), %rsi
; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; CHECK-NEXT: movq %rax, 24(%r12)
+; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Reload
+; CHECK-NEXT: shrdq $24, %rax, %r10
+; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; CHECK-NEXT: shrdq $24, %rcx, %rax
+; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Reload
+; CHECK-NEXT: shrdq $24, %rbp, %rcx
+; CHECK-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Reload
+; CHECK-NEXT: shrdq $24, %r13, %rbp
+; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r12 # 8-byte Reload
+; CHECK-NEXT: shrdq $24, %r12, %r13
+; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r15 # 8-byte Reload
+; CHECK-NEXT: shrdq $24, %r15, %r12
+; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Reload
+; CHECK-NEXT: shrdq $24, %rbx, %r15
+; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Reload
+; CHECK-NEXT: shrdq $24, %r11, %rbx
+; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r9 # 8-byte Reload
+; CHECK-NEXT: shrdq $24, %r9, %r11
+; CHECK-NEXT: movq %rdi, %r8
+; CHECK-NEXT: shrdq $24, %rdi, %r9
+; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
+; CHECK-NEXT: shrdq $24, %rdi, %r8
; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; CHECK-NEXT: movq %rax, 16(%r12)
+; CHECK-NEXT: shrdq $24, %rax, %rdi
+; CHECK-NEXT: shrdq $24, %r14, %rax
+; CHECK-NEXT: movq %rax, %rcx
+; CHECK-NEXT: shrdq $24, %rdx, %r14
+; CHECK-NEXT: shrdq $24, %rsi, %rdx
; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; CHECK-NEXT: movq %rax, 8(%r12)
-; CHECK-NEXT: movq %rdi, (%r12)
-; CHECK-NEXT: movq %rsi, %rax
+; CHECK-NEXT: movq %rdx, 112(%rax)
+; CHECK-NEXT: movq %r14, 104(%rax)
+; CHECK-NEXT: movq %rcx, 96(%rax)
+; CHECK-NEXT: movq %rdi, 88(%rax)
+; CHECK-NEXT: movq %r8, 80(%rax)
+; CHECK-NEXT: movq %r9, 72(%rax)
+; CHECK-NEXT: movq %r11, 64(%rax)
+; CHECK-NEXT: movq %rbx, 56(%rax)
+; CHECK-NEXT: movq %r15, 48(%rax)
+; CHECK-NEXT: movq %r12, 40(%rax)
+; CHECK-NEXT: movq %r13, 32(%rax)
+; CHECK-NEXT: movq %rbp, 24(%rax)
+; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; CHECK-NEXT: movq %rcx, 16(%rax)
+; CHECK-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; CHECK-NEXT: movq %rcx, 8(%rax)
+; CHECK-NEXT: movq %r10, (%rax)
+; CHECK-NEXT: movq %rsi, %rcx
; CHECK-NEXT: shrq $56, %rsi
-; CHECK-NEXT: movb %sil, 124(%r12)
-; CHECK-NEXT: shrq $24, %rax
-; CHECK-NEXT: movl %eax, 120(%r12)
-; CHECK-NEXT: movq %r12, %rax
+; CHECK-NEXT: movb %sil, 124(%rax)
+; CHECK-NEXT: shrq $24, %rcx
+; CHECK-NEXT: movl %ecx, 120(%rax)
; CHECK-NEXT: popq %rbx
; CHECK-NEXT: popq %r12
; CHECK-NEXT: popq %r13
diff --git a/llvm/test/CodeGen/X86/rot32.ll b/llvm/test/CodeGen/X86/rot32.ll
index eb02931e66ff3..8da0e0e6d23cd 100644
--- a/llvm/test/CodeGen/X86/rot32.ll
+++ b/llvm/test/CodeGen/X86/rot32.ll
@@ -187,14 +187,28 @@ entry:
}
define i32 @xbar(i32 %x, i32 %y, i32 %z) nounwind readnone {
-; CHECK32-LABEL: xbar:
-; CHECK32: # %bb.0: # %entry
-; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; CHECK32-NEXT: shll $7, %ecx
-; CHECK32-NEXT: shrl $25, %eax
-; CHECK32-NEXT: orl %ecx, %eax
-; CHECK32-NEXT: retl
+; X86-LABEL: xbar:
+; X86: # %bb.0: # %entry
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: shldl $7, %ecx, %eax
+; X86-NEXT: retl
+;
+; SHLD-LABEL: xbar:
+; SHLD: # %bb.0: # %entry
+; SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
+; SHLD-NEXT: shldl $7, %ecx, %eax
+; SHLD-NEXT: retl
+;
+; BMI2-LABEL: xbar:
+; BMI2: # %bb.0: # %entry
+; BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; BMI2-NEXT: shll $7, %ecx
+; BMI2-NEXT: shrl $25, %eax
+; BMI2-NEXT: orl %ecx, %eax
+; BMI2-NEXT: retl
;
; X64-LABEL: xbar:
; X64: # %bb.0: # %entry
@@ -309,14 +323,28 @@ entry:
}
define i32 @xbu(i32 %x, i32 %y, i32 %z) nounwind readnone {
-; CHECK32-LABEL: xbu:
-; CHECK32: # %bb.0: # %entry
-; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; CHECK32-NEXT: shll $25, %ecx
-; CHECK32-NEXT: shrl $7, %eax
-; CHECK32-NEXT: orl %ecx, %eax
-; CHECK32-NEXT: retl
+; X86-LABEL: xbu:
+; X86: # %bb.0: # %entry
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: shldl $25, %ecx, %eax
+; X86-NEXT: retl
+;
+; SHLD-LABEL: xbu:
+; SHLD: # %bb.0: # %entry
+; SHLD-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
+; SHLD-NEXT: shldl $25, %ecx, %eax
+; SHLD-NEXT: retl
+;
+; BMI2-LABEL: xbu:
+; BMI2: # %bb.0: # %entry
+; BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; BMI2-NEXT: shll $25, %ecx
+; BMI2-NEXT: shrl $7, %eax
+; BMI2-NEXT: orl %ecx, %eax
+; BMI2-NEXT: retl
;
; X64-LABEL: xbu:
; X64: # %bb.0: # %entry
diff --git a/llvm/test/CodeGen/X86/rotate-extract.ll b/llvm/test/CodeGen/X86/rotate-extract.ll
index 2b87b639679dd..b5332068d7edd 100644
--- a/llvm/test/CodeGen/X86/rotate-extract.ll
+++ b/llvm/test/CodeGen/X86/rotate-extract.ll
@@ -133,19 +133,14 @@ define i64 @rolq_extract_mul_with_mask(i64 %i) nounwind {
define i64 @no_extract_shl(i64 %i) nounwind {
; X86-LABEL: no_extract_shl:
; X86: # %bb.0:
-; X86-NEXT: pushl %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl %eax, %esi
-; X86-NEXT: shll $10, %esi
-; X86-NEXT: movl %ecx, %edx
-; X86-NEXT: shrl $22, %edx
-; X86-NEXT: orl %esi, %edx
+; X86-NEXT: movl %eax, %edx
+; X86-NEXT: shldl $10, %ecx, %edx
; X86-NEXT: shll $10, %ecx
; X86-NEXT: shrl $20, %eax
; X86-NEXT: andl $127, %eax
; X86-NEXT: orl %ecx, %eax
-; X86-NEXT: popl %esi
; X86-NEXT: retl
;
; X64-LABEL: no_extract_shl:
diff --git a/llvm/test/CodeGen/X86/scmp.ll b/llvm/test/CodeGen/X86/scmp.ll
index 19c808185824c..aef81841a458c 100644
--- a/llvm/test/CodeGen/X86/scmp.ll
+++ b/llvm/test/CodeGen/X86/scmp.ll
@@ -2518,7 +2518,8 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; SSE2-NEXT: pushq %r13
; SSE2-NEXT: pushq %r12
; SSE2-NEXT: pushq %rbx
-; SSE2-NEXT: movzbl {{[0-9]+}}(%rsp), %eax
+; SSE2-NEXT: movq %rdi, %rax
+; SSE2-NEXT: movzbl {{[0-9]+}}(%rsp), %edi
; SSE2-NEXT: movzbl {{[0-9]+}}(%rsp), %r10d
; SSE2-NEXT: movzbl {{[0-9]+}}(%rsp), %r11d
; SSE2-NEXT: movzbl {{[0-9]+}}(%rsp), %ebx
@@ -2534,10 +2535,9 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; SSE2-NEXT: setg %r15b
; SSE2-NEXT: subb %sil, %r15b
; SSE2-NEXT: movsbq %r15b, %rsi
-; SSE2-NEXT: movq %rsi, (%rdi)
+; SSE2-NEXT: movq %rsi, (%rax)
; SSE2-NEXT: movq %rsi, %xmm0
; SSE2-NEXT: sarq $63, %rsi
-; SSE2-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; SSE2-NEXT: addb %r14b, %r14b
; SSE2-NEXT: sarb %r14b
; SSE2-NEXT: movzbl {{[0-9]+}}(%rsp), %r15d
@@ -2569,9 +2569,9 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; SSE2-NEXT: setl %cl
; SSE2-NEXT: setg %bl
; SSE2-NEXT: subb %cl, %bl
-; SSE2-NEXT: movsbq %bl, %rsi
-; SSE2-NEXT: movq %rsi, %rbx
-; SSE2-NEXT: sarq $63, %rbx
+; SSE2-NEXT: movsbq %bl, %rbx
+; SSE2-NEXT: movq %rbx, %rcx
+; SSE2-NEXT: sarq $63, %rcx
; SSE2-NEXT: addb %r11b, %r11b
; SSE2-NEXT: sarb %r11b
; SSE2-NEXT: addb %r8b, %r8b
@@ -2594,70 +2594,52 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; SSE2-NEXT: movsbq %r10b, %r9
; SSE2-NEXT: movq %r9, %r10
; SSE2-NEXT: sarq $63, %r10
-; SSE2-NEXT: addb %al, %al
-; SSE2-NEXT: sarb %al
+; SSE2-NEXT: addb %dil, %dil
+; SSE2-NEXT: sarb %dil
; SSE2-NEXT: movzbl {{[0-9]+}}(%rsp), %ebp
; SSE2-NEXT: addb %bpl, %bpl
; SSE2-NEXT: sarb %bpl
-; SSE2-NEXT: cmpb %al, %bpl
-; SSE2-NEXT: setl %al
+; SSE2-NEXT: cmpb %dil, %bpl
+; SSE2-NEXT: setl %dil
; SSE2-NEXT: setg %bpl
-; SSE2-NEXT: subb %al, %bpl
-; SSE2-NEXT: movsbq %bpl, %rcx
-; SSE2-NEXT: movq %rcx, %rbp
-; SSE2-NEXT: sarq $63, %rbp
-; SSE2-NEXT: movq %rdi, %rax
-; SSE2-NEXT: movl %ebp, 96(%rdi)
-; SSE2-NEXT: movabsq $2251799813685247, %r13 # imm = 0x7FFFFFFFFFFFF
-; SSE2-NEXT: andq %rbp, %r13
-; SSE2-NEXT: shlq $62, %rbp
-; SSE2-NEXT: movq %rcx, %rdi
-; SSE2-NEXT: shrq $2, %rdi
-; SSE2-NEXT: orq %rbp, %rdi
-; SSE2-NEXT: movq %rdi, 88(%rax)
-; SSE2-NEXT: movq %r9, %rdi
-; SSE2-NEXT: shrq $44, %rdi
-; SSE2-NEXT: movq %r10, %rbp
-; SSE2-NEXT: shlq $20, %rbp
-; SSE2-NEXT: orq %rdi, %rbp
-; SSE2-NEXT: movq %rbp, 64(%rax)
-; SSE2-NEXT: movq %r8, %rdi
-; SSE2-NEXT: shrq $33, %rdi
-; SSE2-NEXT: movq %r11, %rbp
-; SSE2-NEXT: shlq $31, %rbp
-; SSE2-NEXT: orq %rdi, %rbp
-; SSE2-NEXT: movq %rbp, 48(%rax)
-; SSE2-NEXT: movq %rsi, %rdi
-; SSE2-NEXT: shrq $22, %rdi
-; SSE2-NEXT: movq %rbx, %rbp
-; SSE2-NEXT: shlq $42, %rbp
-; SSE2-NEXT: orq %rdi, %rbp
-; SSE2-NEXT: movq %rbp, 32(%rax)
-; SSE2-NEXT: movq %rdx, %rdi
+; SSE2-NEXT: subb %dil, %bpl
+; SSE2-NEXT: movsbq %bpl, %rdi
+; SSE2-NEXT: movq %rdi, %r13
+; SSE2-NEXT: sarq $63, %r13
+; SSE2-NEXT: movl %r13d, 96(%rax)
+; SSE2-NEXT: movabsq $2251799813685247, %rbp # imm = 0x7FFFFFFFFFFFF
+; SSE2-NEXT: andq %r13, %rbp
+; SSE2-NEXT: shldq $62, %rdi, %r13
+; SSE2-NEXT: movq %r13, 88(%rax)
+; SSE2-NEXT: movq %r9, %r13
+; SSE2-NEXT: shrdq $44, %r10, %r13
+; SSE2-NEXT: movq %r13, 64(%rax)
+; SSE2-NEXT: movq %r8, %r13
+; SSE2-NEXT: shrdq $33, %r11, %r13
+; SSE2-NEXT: movq %r13, 48(%rax)
+; SSE2-NEXT: movq %rbx, %r13
+; SSE2-NEXT: shrdq $22, %rcx, %r13
+; SSE2-NEXT: movq %r13, 32(%rax)
+; SSE2-NEXT: movq %rdx, %r13
+; SSE2-NEXT: shrdq $11, %r12, %r13
+; SSE2-NEXT: movq %r13, 16(%rax)
+; SSE2-NEXT: movq %rbp, %r13
+; SSE2-NEXT: shrq $48, %r13
+; SSE2-NEXT: movb %r13b, 102(%rax)
+; SSE2-NEXT: shrq $32, %rbp
+; SSE2-NEXT: movw %bp, 100(%rax)
+; SSE2-NEXT: movabsq $9007199254740991, %r13 # imm = 0x1FFFFFFFFFFFFF
+; SSE2-NEXT: andq %r13, %r15
+; SSE2-NEXT: shldq $9, %r14, %r15
+; SSE2-NEXT: shlq $62, %rdi
+; SSE2-NEXT: orq %r15, %rdi
+; SSE2-NEXT: movq %rdi, 80(%rax)
+; SSE2-NEXT: movabsq $9007199254738944, %rdi # imm = 0x1FFFFFFFFFF800
+; SSE2-NEXT: andq %r12, %rdi
+; SSE2-NEXT: shlq $42, %rbx
; SSE2-NEXT: shrq $11, %rdi
-; SSE2-NEXT: movabsq $9007199254738944, %rbp # imm = 0x1FFFFFFFFFF800
-; SSE2-NEXT: andq %r12, %rbp
-; SSE2-NEXT: shlq $53, %r12
-; SSE2-NEXT: orq %rdi, %r12
-; SSE2-NEXT: movq %r12, 16(%rax)
-; SSE2-NEXT: movq %r13, %rdi
-; SSE2-NEXT: shrq $48, %rdi
-; SSE2-NEXT: movb %dil, 102(%rax)
-; SSE2-NEXT: shrq $32, %r13
-; SSE2-NEXT: movw %r13w, 100(%rax)
-; SSE2-NEXT: movabsq $9007199254740991, %r12 # imm = 0x1FFFFFFFFFFFFF
-; SSE2-NEXT: andq %r12, %r15
-; SSE2-NEXT: movq %r14, %rdi
-; SSE2-NEXT: shrq $55, %rdi
-; SSE2-NEXT: shlq $9, %r15
-; SSE2-NEXT: orq %rdi, %r15
-; SSE2-NEXT: shlq $62, %rcx
-; SSE2-NEXT: orq %r15, %rcx
-; SSE2-NEXT: movq %rcx, 80(%rax)
-; SSE2-NEXT: shlq $42, %rsi
-; SSE2-NEXT: shrq $11, %rbp
-; SSE2-NEXT: orq %rsi, %rbp
-; SSE2-NEXT: movq %rbp, 24(%rax)
+; SSE2-NEXT: orq %rbx, %rdi
+; SSE2-NEXT: movq %rdi, 24(%rax)
; SSE2-NEXT: shlq $9, %r14
; SSE2-NEXT: andl $511, %r10d # imm = 0x1FF
; SSE2-NEXT: orq %r14, %r10
@@ -2667,15 +2649,14 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; SSE2-NEXT: orq %r9, %r11
; SSE2-NEXT: movq %r11, 56(%rax)
; SSE2-NEXT: shlq $31, %r8
-; SSE2-NEXT: andl $2147483647, %ebx # imm = 0x7FFFFFFF
-; SSE2-NEXT: orq %r8, %rbx
-; SSE2-NEXT: movq %rbx, 40(%rax)
-; SSE2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Folded Reload
-; SSE2-NEXT: # xmm1 = mem[0],zero
+; SSE2-NEXT: andl $2147483647, %ecx # imm = 0x7FFFFFFF
+; SSE2-NEXT: orq %r8, %rcx
+; SSE2-NEXT: movq %rcx, 40(%rax)
+; SSE2-NEXT: movq %rsi, %xmm1
; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
; SSE2-NEXT: movq %xmm0, %rcx
-; SSE2-NEXT: andq %r12, %rcx
+; SSE2-NEXT: andq %r13, %rcx
; SSE2-NEXT: shlq $53, %rdx
; SSE2-NEXT: orq %rcx, %rdx
; SSE2-NEXT: movq %rdx, 8(%rax)
@@ -3194,9 +3175,21 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; X86-NEXT: addb %al, %al
; X86-NEXT: sarb %al
; X86-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X86-NEXT: movzbl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movb {{[0-9]+}}(%esp), %dh
+; X86-NEXT: addb %dh, %dh
+; X86-NEXT: sarb %dh
+; X86-NEXT: movb {{[0-9]+}}(%esp), %dl
; X86-NEXT: addb %dl, %dl
; X86-NEXT: sarb %dl
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: addb %al, %al
+; X86-NEXT: sarb %al
+; X86-NEXT: movb {{[0-9]+}}(%esp), %ah
+; X86-NEXT: addb %ah, %ah
+; X86-NEXT: sarb %ah
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: addb %cl, %cl
+; X86-NEXT: sarb %cl
; X86-NEXT: movb {{[0-9]+}}(%esp), %ch
; X86-NEXT: addb %ch, %ch
; X86-NEXT: sarb %ch
@@ -3206,167 +3199,146 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; X86-NEXT: movb {{[0-9]+}}(%esp), %bh
; X86-NEXT: addb %bh, %bh
; X86-NEXT: sarb %bh
-; X86-NEXT: movb {{[0-9]+}}(%esp), %cl
-; X86-NEXT: addb %cl, %cl
-; X86-NEXT: sarb %cl
-; X86-NEXT: movb {{[0-9]+}}(%esp), %ah
-; X86-NEXT: addb %ah, %ah
-; X86-NEXT: sarb %ah
-; X86-NEXT: movb {{[0-9]+}}(%esp), %al
-; X86-NEXT: addb %al, %al
-; X86-NEXT: sarb %al
-; X86-NEXT: movb {{[0-9]+}}(%esp), %dh
-; X86-NEXT: addb %dh, %dh
-; X86-NEXT: sarb %dh
-; X86-NEXT: cmpb %al, %dh
-; X86-NEXT: setl %al
-; X86-NEXT: setg %dh
-; X86-NEXT: subb %al, %dh
-; X86-NEXT: movsbl %dh, %esi
+; X86-NEXT: cmpb %bl, %bh
+; X86-NEXT: setl %bl
+; X86-NEXT: setg %bh
+; X86-NEXT: subb %bl, %bh
+; X86-NEXT: movsbl %bh, %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: sarl $31, %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: andl $2097151, %esi # imm = 0x1FFFFF
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: cmpb %cl, %ah
+; X86-NEXT: cmpb %cl, %ch
+; X86-NEXT: setl %cl
+; X86-NEXT: setg %ch
+; X86-NEXT: subb %cl, %ch
+; X86-NEXT: movsbl %ch, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sarl $31, %esi
+; X86-NEXT: movl %esi, %ecx
+; X86-NEXT: movl %esi, %ebx
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: andl $2097151, %ecx # imm = 0x1FFFFF
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: cmpb %al, %ah
; X86-NEXT: setl %al
; X86-NEXT: setg %cl
; X86-NEXT: subb %al, %cl
-; X86-NEXT: movsbl %cl, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: sarl $31, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movsbl %cl, %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %ecx, (%eax)
+; X86-NEXT: sarl $31, %ecx
+; X86-NEXT: movl %ecx, %eax
; X86-NEXT: andl $2097151, %eax # imm = 0x1FFFFF
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: cmpb %bl, %bh
+; X86-NEXT: cmpb %dh, %dl
; X86-NEXT: setl %al
-; X86-NEXT: setg %cl
-; X86-NEXT: subb %al, %cl
-; X86-NEXT: movsbl %cl, %eax
-; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT: movl %eax, (%esi)
-; X86-NEXT: sarl $31, %eax
-; X86-NEXT: movl %eax, %esi
-; X86-NEXT: andl $2097151, %esi # imm = 0x1FFFFF
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: cmpb %dl, %ch
-; X86-NEXT: setl %cl
; X86-NEXT: setg %dl
-; X86-NEXT: subb %cl, %dl
+; X86-NEXT: subb %al, %dl
; X86-NEXT: movsbl %dl, %ebp
; X86-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: sarl $31, %ebp
-; X86-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
-; X86-NEXT: cmpb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Folded Reload
-; X86-NEXT: setl %cl
+; X86-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X86-NEXT: cmpb {{[-0-9]+}}(%e{{[sb]}}p), %al # 1-byte Folded Reload
+; X86-NEXT: setl %al
; X86-NEXT: setg %dl
-; X86-NEXT: subb %cl, %dl
+; X86-NEXT: subb %al, %dl
; X86-NEXT: movsbl %dl, %edi
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: sarl $31, %edi
-; X86-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
-; X86-NEXT: cmpb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Folded Reload
-; X86-NEXT: setl %cl
-; X86-NEXT: setg %ch
-; X86-NEXT: subb %cl, %ch
-; X86-NEXT: movsbl %ch, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: sarl $31, %esi
-; X86-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
-; X86-NEXT: cmpb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Folded Reload
-; X86-NEXT: setl %bl
-; X86-NEXT: setg %bh
-; X86-NEXT: subb %bl, %bh
-; X86-NEXT: movsbl %bh, %ecx
-; X86-NEXT: movl %ecx, %ebx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: sarl $31, %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl %ecx, 96(%edx)
-; X86-NEXT: movl %ecx, 92(%edx)
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl %ecx, 80(%edx)
-; X86-NEXT: movl %esi, 68(%edx)
-; X86-NEXT: movl %esi, 64(%edx)
-; X86-NEXT: movl %edi, 52(%edx)
-; X86-NEXT: movl %edi, 48(%edx)
+; X86-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X86-NEXT: cmpb {{[-0-9]+}}(%e{{[sb]}}p), %al # 1-byte Folded Reload
+; X86-NEXT: setl %al
+; X86-NEXT: setg %dl
+; X86-NEXT: subb %al, %dl
+; X86-NEXT: movsbl %dl, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sarl $31, %eax
+; X86-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 1-byte Folded Reload
+; X86-NEXT: cmpb {{[-0-9]+}}(%e{{[sb]}}p), %dl # 1-byte Folded Reload
+; X86-NEXT: setl %dl
+; X86-NEXT: setg %dh
+; X86-NEXT: subb %dl, %dh
+; X86-NEXT: movsbl %dh, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sarl $31, %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT: movl %edx, 96(%esi)
+; X86-NEXT: movl %edx, 92(%esi)
+; X86-NEXT: movl %ebx, 80(%esi)
+; X86-NEXT: movl %eax, 68(%esi)
+; X86-NEXT: movl %eax, 64(%esi)
+; X86-NEXT: movl %edi, 52(%esi)
+; X86-NEXT: movl %edi, 48(%esi)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: movl %ebx, 36(%esi)
+; X86-NEXT: movl %ebp, 24(%esi)
+; X86-NEXT: movl %ebp, 20(%esi)
+; X86-NEXT: movl %ecx, 8(%esi)
+; X86-NEXT: movl %ecx, 4(%esi)
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movw %dx, 100(%esi)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl %ecx, 36(%edx)
-; X86-NEXT: movl %ebp, 24(%edx)
-; X86-NEXT: movl %ebp, 20(%edx)
-; X86-NEXT: movl %eax, 8(%edx)
-; X86-NEXT: movl %eax, 4(%edx)
-; X86-NEXT: movl %ebx, %eax
-; X86-NEXT: shrl $2, %eax
+; X86-NEXT: shrdl $2, %edx, %ecx
+; X86-NEXT: movl %ecx, 88(%esi)
+; X86-NEXT: movl %esi, %edx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl %ecx, %ebx
-; X86-NEXT: shll $30, %ebx
-; X86-NEXT: orl %eax, %ebx
-; X86-NEXT: movw %cx, 100(%edx)
-; X86-NEXT: movl %ebx, 88(%edx)
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shldl $9, %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: shll $9, %eax
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: shrl $23, %ebx
-; X86-NEXT: orl %eax, %ebx
-; X86-NEXT: movl %ebx, 76(%edx)
-; X86-NEXT: movl %esi, %eax
-; X86-NEXT: shll $20, %eax
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: shrl $12, %ebx
-; X86-NEXT: orl %eax, %ebx
-; X86-NEXT: movl %ebx, 60(%edx)
-; X86-NEXT: movl %edi, %eax
-; X86-NEXT: shll $31, %eax
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: shrl %ebx
-; X86-NEXT: orl %eax, %ebx
-; X86-NEXT: movl %ebx, 44(%edx)
-; X86-NEXT: movl %edx, %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shrl $22, %eax
+; X86-NEXT: shldl $9, %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: shldl $9, %esi, %ecx
+; X86-NEXT: movl %edx, %esi
+; X86-NEXT: movl %ecx, 76(%edx)
+; X86-NEXT: movl %eax, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: shldl $20, %edx, %ecx
+; X86-NEXT: movl %ecx, 60(%esi)
+; X86-NEXT: movl %esi, %ebx
+; X86-NEXT: movl %edi, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: shldl $31, %esi, %ecx
+; X86-NEXT: movl %ecx, 44(%ebx)
+; X86-NEXT: movl %ebx, %edx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: movl %esi, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: shrdl $22, %ebx, %ecx
+; X86-NEXT: movl %ecx, 32(%edx)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: shldl $10, %ebx, %edx
-; X86-NEXT: shll $10, %ebx
-; X86-NEXT: orl %eax, %ebx
-; X86-NEXT: movl %ebx, 32(%ecx)
+; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: shrdl $11, %ebp, %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movl %ecx, 16(%edx)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: shrl $11, %eax
-; X86-NEXT: movl %ebp, %ebx
-; X86-NEXT: shll $21, %ebx
-; X86-NEXT: orl %eax, %ebx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl %ebx, 16(%eax)
+; X86-NEXT: shll $9, %ecx
+; X86-NEXT: andl $511, %eax # imm = 0x1FF
+; X86-NEXT: orl %ecx, %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl %eax, 72(%ecx)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shll $9, %eax
-; X86-NEXT: andl $511, %esi # imm = 0x1FF
-; X86-NEXT: orl %eax, %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl %esi, 72(%eax)
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: shll $20, %esi
+; X86-NEXT: shll $20, %eax
; X86-NEXT: andl $1048575, %edi # imm = 0xFFFFF
-; X86-NEXT: orl %esi, %edi
-; X86-NEXT: movl %edi, 56(%eax)
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: orl %eax, %edi
+; X86-NEXT: movl %edi, 56(%ecx)
; X86-NEXT: shll $10, %esi
; X86-NEXT: andl $1023, %ebp # imm = 0x3FF
; X86-NEXT: orl %esi, %ebp
-; X86-NEXT: movl %ebp, 28(%eax)
-; X86-NEXT: shll $21, %ecx
-; X86-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X86-NEXT: movl %ecx, 12(%eax)
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: andl $7, %ecx
-; X86-NEXT: movb %cl, 102(%eax)
+; X86-NEXT: movl %ebp, 28(%ecx)
+; X86-NEXT: movl %ebx, %eax
+; X86-NEXT: shll $21, %eax
+; X86-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NEXT: movl %eax, 12(%ecx)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: andl $7, %eax
+; X86-NEXT: movb %al, 102(%ecx)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: shll $30, %eax
+; X86-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NEXT: movl %eax, 84(%ecx)
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: shll $30, %ecx
-; X86-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X86-NEXT: movl %ecx, 84(%eax)
+; X86-NEXT: shldl $10, %ecx, %edx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: shll $31, %ecx
; X86-NEXT: orl %edx, %ecx
@@ -3386,7 +3358,8 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; SETZUCC-NEXT: pushq %r13
; SETZUCC-NEXT: pushq %r12
; SETZUCC-NEXT: pushq %rbx
-; SETZUCC-NEXT: movzbl {{[0-9]+}}(%rsp), %eax
+; SETZUCC-NEXT: movq %rdi, %rax
+; SETZUCC-NEXT: movzbl {{[0-9]+}}(%rsp), %edi
; SETZUCC-NEXT: movzbl {{[0-9]+}}(%rsp), %r10d
; SETZUCC-NEXT: movzbl {{[0-9]+}}(%rsp), %r11d
; SETZUCC-NEXT: movzbl {{[0-9]+}}(%rsp), %ebx
@@ -3402,10 +3375,9 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; SETZUCC-NEXT: setzug %r15b
; SETZUCC-NEXT: subb %sil, %r15b
; SETZUCC-NEXT: movsbq %r15b, %rsi
-; SETZUCC-NEXT: movq %rsi, (%rdi)
+; SETZUCC-NEXT: movq %rsi, (%rax)
; SETZUCC-NEXT: movq %rsi, %xmm0
; SETZUCC-NEXT: sarq $63, %rsi
-; SETZUCC-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; SETZUCC-NEXT: addb %r14b, %r14b
; SETZUCC-NEXT: sarb %r14b
; SETZUCC-NEXT: movzbl {{[0-9]+}}(%rsp), %r15d
@@ -3437,9 +3409,9 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; SETZUCC-NEXT: setzul %cl
; SETZUCC-NEXT: setzug %bl
; SETZUCC-NEXT: subb %cl, %bl
-; SETZUCC-NEXT: movsbq %bl, %rsi
-; SETZUCC-NEXT: movq %rsi, %rbx
-; SETZUCC-NEXT: sarq $63, %rbx
+; SETZUCC-NEXT: movsbq %bl, %rbx
+; SETZUCC-NEXT: movq %rbx, %rcx
+; SETZUCC-NEXT: sarq $63, %rcx
; SETZUCC-NEXT: addb %r11b, %r11b
; SETZUCC-NEXT: sarb %r11b
; SETZUCC-NEXT: addb %r8b, %r8b
@@ -3462,70 +3434,52 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; SETZUCC-NEXT: movsbq %r10b, %r9
; SETZUCC-NEXT: movq %r9, %r10
; SETZUCC-NEXT: sarq $63, %r10
-; SETZUCC-NEXT: addb %al, %al
-; SETZUCC-NEXT: sarb %al
+; SETZUCC-NEXT: addb %dil, %dil
+; SETZUCC-NEXT: sarb %dil
; SETZUCC-NEXT: movzbl {{[0-9]+}}(%rsp), %ebp
; SETZUCC-NEXT: addb %bpl, %bpl
; SETZUCC-NEXT: sarb %bpl
-; SETZUCC-NEXT: cmpb %al, %bpl
-; SETZUCC-NEXT: setzul %al
+; SETZUCC-NEXT: cmpb %dil, %bpl
+; SETZUCC-NEXT: setzul %dil
; SETZUCC-NEXT: setzug %bpl
-; SETZUCC-NEXT: subb %al, %bpl
-; SETZUCC-NEXT: movsbq %bpl, %rcx
-; SETZUCC-NEXT: movq %rcx, %rbp
-; SETZUCC-NEXT: sarq $63, %rbp
-; SETZUCC-NEXT: movq %rdi, %rax
-; SETZUCC-NEXT: movl %ebp, 96(%rdi)
-; SETZUCC-NEXT: movabsq $2251799813685247, %r13 # imm = 0x7FFFFFFFFFFFF
-; SETZUCC-NEXT: andq %rbp, %r13
-; SETZUCC-NEXT: shlq $62, %rbp
-; SETZUCC-NEXT: movq %rcx, %rdi
-; SETZUCC-NEXT: shrq $2, %rdi
-; SETZUCC-NEXT: orq %rbp, %rdi
-; SETZUCC-NEXT: movq %rdi, 88(%rax)
-; SETZUCC-NEXT: movq %r9, %rdi
-; SETZUCC-NEXT: shrq $44, %rdi
-; SETZUCC-NEXT: movq %r10, %rbp
-; SETZUCC-NEXT: shlq $20, %rbp
-; SETZUCC-NEXT: orq %rdi, %rbp
-; SETZUCC-NEXT: movq %rbp, 64(%rax)
-; SETZUCC-NEXT: movq %r8, %rdi
-; SETZUCC-NEXT: shrq $33, %rdi
-; SETZUCC-NEXT: movq %r11, %rbp
-; SETZUCC-NEXT: shlq $31, %rbp
-; SETZUCC-NEXT: orq %rdi, %rbp
-; SETZUCC-NEXT: movq %rbp, 48(%rax)
-; SETZUCC-NEXT: movq %rsi, %rdi
-; SETZUCC-NEXT: shrq $22, %rdi
-; SETZUCC-NEXT: movq %rbx, %rbp
-; SETZUCC-NEXT: shlq $42, %rbp
-; SETZUCC-NEXT: orq %rdi, %rbp
-; SETZUCC-NEXT: movq %rbp, 32(%rax)
-; SETZUCC-NEXT: movq %rdx, %rdi
+; SETZUCC-NEXT: subb %dil, %bpl
+; SETZUCC-NEXT: movsbq %bpl, %rdi
+; SETZUCC-NEXT: movq %rdi, %r13
+; SETZUCC-NEXT: sarq $63, %r13
+; SETZUCC-NEXT: movl %r13d, 96(%rax)
+; SETZUCC-NEXT: movabsq $2251799813685247, %rbp # imm = 0x7FFFFFFFFFFFF
+; SETZUCC-NEXT: andq %r13, %rbp
+; SETZUCC-NEXT: shldq $62, %rdi, %r13
+; SETZUCC-NEXT: movq %r13, 88(%rax)
+; SETZUCC-NEXT: movq %r9, %r13
+; SETZUCC-NEXT: shrdq $44, %r10, %r13
+; SETZUCC-NEXT: movq %r13, 64(%rax)
+; SETZUCC-NEXT: movq %r8, %r13
+; SETZUCC-NEXT: shrdq $33, %r11, %r13
+; SETZUCC-NEXT: movq %r13, 48(%rax)
+; SETZUCC-NEXT: movq %rbx, %r13
+; SETZUCC-NEXT: shrdq $22, %rcx, %r13
+; SETZUCC-NEXT: movq %r13, 32(%rax)
+; SETZUCC-NEXT: movq %rdx, %r13
+; SETZUCC-NEXT: shrdq $11, %r12, %r13
+; SETZUCC-NEXT: movq %r13, 16(%rax)
+; SETZUCC-NEXT: movq %rbp, %r13
+; SETZUCC-NEXT: shrq $48, %r13
+; SETZUCC-NEXT: movb %r13b, 102(%rax)
+; SETZUCC-NEXT: shrq $32, %rbp
+; SETZUCC-NEXT: movw %bp, 100(%rax)
+; SETZUCC-NEXT: movabsq $9007199254740991, %r13 # imm = 0x1FFFFFFFFFFFFF
+; SETZUCC-NEXT: andq %r13, %r15
+; SETZUCC-NEXT: shldq $9, %r14, %r15
+; SETZUCC-NEXT: shlq $62, %rdi
+; SETZUCC-NEXT: orq %r15, %rdi
+; SETZUCC-NEXT: movq %rdi, 80(%rax)
+; SETZUCC-NEXT: movabsq $9007199254738944, %rdi # imm = 0x1FFFFFFFFFF800
+; SETZUCC-NEXT: andq %r12, %rdi
+; SETZUCC-NEXT: shlq $42, %rbx
; SETZUCC-NEXT: shrq $11, %rdi
-; SETZUCC-NEXT: movabsq $9007199254738944, %rbp # imm = 0x1FFFFFFFFFF800
-; SETZUCC-NEXT: andq %r12, %rbp
-; SETZUCC-NEXT: shlq $53, %r12
-; SETZUCC-NEXT: orq %rdi, %r12
-; SETZUCC-NEXT: movq %r12, 16(%rax)
-; SETZUCC-NEXT: movq %r13, %rdi
-; SETZUCC-NEXT: shrq $48, %rdi
-; SETZUCC-NEXT: movb %dil, 102(%rax)
-; SETZUCC-NEXT: shrq $32, %r13
-; SETZUCC-NEXT: movw %r13w, 100(%rax)
-; SETZUCC-NEXT: movabsq $9007199254740991, %r12 # imm = 0x1FFFFFFFFFFFFF
-; SETZUCC-NEXT: andq %r12, %r15
-; SETZUCC-NEXT: movq %r14, %rdi
-; SETZUCC-NEXT: shrq $55, %rdi
-; SETZUCC-NEXT: shlq $9, %r15
-; SETZUCC-NEXT: orq %rdi, %r15
-; SETZUCC-NEXT: shlq $62, %rcx
-; SETZUCC-NEXT: orq %r15, %rcx
-; SETZUCC-NEXT: movq %rcx, 80(%rax)
-; SETZUCC-NEXT: shlq $42, %rsi
-; SETZUCC-NEXT: shrq $11, %rbp
-; SETZUCC-NEXT: orq %rsi, %rbp
-; SETZUCC-NEXT: movq %rbp, 24(%rax)
+; SETZUCC-NEXT: orq %rbx, %rdi
+; SETZUCC-NEXT: movq %rdi, 24(%rax)
; SETZUCC-NEXT: shlq $9, %r14
; SETZUCC-NEXT: andl $511, %r10d # imm = 0x1FF
; SETZUCC-NEXT: orq %r14, %r10
@@ -3535,15 +3489,14 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; SETZUCC-NEXT: orq %r9, %r11
; SETZUCC-NEXT: movq %r11, 56(%rax)
; SETZUCC-NEXT: shlq $31, %r8
-; SETZUCC-NEXT: andl $2147483647, %ebx # imm = 0x7FFFFFFF
-; SETZUCC-NEXT: orq %r8, %rbx
-; SETZUCC-NEXT: movq %rbx, 40(%rax)
-; SETZUCC-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Folded Reload
-; SETZUCC-NEXT: # xmm1 = mem[0],zero
+; SETZUCC-NEXT: andl $2147483647, %ecx # imm = 0x7FFFFFFF
+; SETZUCC-NEXT: orq %r8, %rcx
+; SETZUCC-NEXT: movq %rcx, 40(%rax)
+; SETZUCC-NEXT: movq %rsi, %xmm1
; SETZUCC-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; SETZUCC-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
; SETZUCC-NEXT: movq %xmm0, %rcx
-; SETZUCC-NEXT: andq %r12, %rcx
+; SETZUCC-NEXT: andq %r13, %rcx
; SETZUCC-NEXT: shlq $53, %rdx
; SETZUCC-NEXT: orq %rcx, %rdx
; SETZUCC-NEXT: movq %rdx, 8(%rax)
@@ -3563,7 +3516,8 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; NO-SETZUCC-NEXT: pushq %r13
; NO-SETZUCC-NEXT: pushq %r12
; NO-SETZUCC-NEXT: pushq %rbx
-; NO-SETZUCC-NEXT: movzbl {{[0-9]+}}(%rsp), %eax
+; NO-SETZUCC-NEXT: movq %rdi, %rax
+; NO-SETZUCC-NEXT: movzbl {{[0-9]+}}(%rsp), %edi
; NO-SETZUCC-NEXT: movzbl {{[0-9]+}}(%rsp), %r10d
; NO-SETZUCC-NEXT: movzbl {{[0-9]+}}(%rsp), %r11d
; NO-SETZUCC-NEXT: movzbl {{[0-9]+}}(%rsp), %ebx
@@ -3579,10 +3533,9 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; NO-SETZUCC-NEXT: setg %r15b
; NO-SETZUCC-NEXT: subb %sil, %r15b
; NO-SETZUCC-NEXT: movsbq %r15b, %rsi
-; NO-SETZUCC-NEXT: movq %rsi, (%rdi)
+; NO-SETZUCC-NEXT: movq %rsi, (%rax)
; NO-SETZUCC-NEXT: movq %rsi, %xmm0
; NO-SETZUCC-NEXT: sarq $63, %rsi
-; NO-SETZUCC-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; NO-SETZUCC-NEXT: addb %r14b, %r14b
; NO-SETZUCC-NEXT: sarb %r14b
; NO-SETZUCC-NEXT: movzbl {{[0-9]+}}(%rsp), %r15d
@@ -3614,9 +3567,9 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; NO-SETZUCC-NEXT: setl %cl
; NO-SETZUCC-NEXT: setg %bl
; NO-SETZUCC-NEXT: subb %cl, %bl
-; NO-SETZUCC-NEXT: movsbq %bl, %rsi
-; NO-SETZUCC-NEXT: movq %rsi, %rbx
-; NO-SETZUCC-NEXT: sarq $63, %rbx
+; NO-SETZUCC-NEXT: movsbq %bl, %rbx
+; NO-SETZUCC-NEXT: movq %rbx, %rcx
+; NO-SETZUCC-NEXT: sarq $63, %rcx
; NO-SETZUCC-NEXT: addb %r11b, %r11b
; NO-SETZUCC-NEXT: sarb %r11b
; NO-SETZUCC-NEXT: addb %r8b, %r8b
@@ -3639,70 +3592,52 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; NO-SETZUCC-NEXT: movsbq %r10b, %r9
; NO-SETZUCC-NEXT: movq %r9, %r10
; NO-SETZUCC-NEXT: sarq $63, %r10
-; NO-SETZUCC-NEXT: addb %al, %al
-; NO-SETZUCC-NEXT: sarb %al
+; NO-SETZUCC-NEXT: addb %dil, %dil
+; NO-SETZUCC-NEXT: sarb %dil
; NO-SETZUCC-NEXT: movzbl {{[0-9]+}}(%rsp), %ebp
; NO-SETZUCC-NEXT: addb %bpl, %bpl
; NO-SETZUCC-NEXT: sarb %bpl
-; NO-SETZUCC-NEXT: cmpb %al, %bpl
-; NO-SETZUCC-NEXT: setl %al
+; NO-SETZUCC-NEXT: cmpb %dil, %bpl
+; NO-SETZUCC-NEXT: setl %dil
; NO-SETZUCC-NEXT: setg %bpl
-; NO-SETZUCC-NEXT: subb %al, %bpl
-; NO-SETZUCC-NEXT: movsbq %bpl, %rcx
-; NO-SETZUCC-NEXT: movq %rcx, %rbp
-; NO-SETZUCC-NEXT: sarq $63, %rbp
-; NO-SETZUCC-NEXT: movq %rdi, %rax
-; NO-SETZUCC-NEXT: movl %ebp, 96(%rdi)
-; NO-SETZUCC-NEXT: movabsq $2251799813685247, %r13 # imm = 0x7FFFFFFFFFFFF
-; NO-SETZUCC-NEXT: andq %rbp, %r13
-; NO-SETZUCC-NEXT: shlq $62, %rbp
-; NO-SETZUCC-NEXT: movq %rcx, %rdi
-; NO-SETZUCC-NEXT: shrq $2, %rdi
-; NO-SETZUCC-NEXT: orq %rbp, %rdi
-; NO-SETZUCC-NEXT: movq %rdi, 88(%rax)
-; NO-SETZUCC-NEXT: movq %r9, %rdi
-; NO-SETZUCC-NEXT: shrq $44, %rdi
-; NO-SETZUCC-NEXT: movq %r10, %rbp
-; NO-SETZUCC-NEXT: shlq $20, %rbp
-; NO-SETZUCC-NEXT: orq %rdi, %rbp
-; NO-SETZUCC-NEXT: movq %rbp, 64(%rax)
-; NO-SETZUCC-NEXT: movq %r8, %rdi
-; NO-SETZUCC-NEXT: shrq $33, %rdi
-; NO-SETZUCC-NEXT: movq %r11, %rbp
-; NO-SETZUCC-NEXT: shlq $31, %rbp
-; NO-SETZUCC-NEXT: orq %rdi, %rbp
-; NO-SETZUCC-NEXT: movq %rbp, 48(%rax)
-; NO-SETZUCC-NEXT: movq %rsi, %rdi
-; NO-SETZUCC-NEXT: shrq $22, %rdi
-; NO-SETZUCC-NEXT: movq %rbx, %rbp
-; NO-SETZUCC-NEXT: shlq $42, %rbp
-; NO-SETZUCC-NEXT: orq %rdi, %rbp
-; NO-SETZUCC-NEXT: movq %rbp, 32(%rax)
-; NO-SETZUCC-NEXT: movq %rdx, %rdi
+; NO-SETZUCC-NEXT: subb %dil, %bpl
+; NO-SETZUCC-NEXT: movsbq %bpl, %rdi
+; NO-SETZUCC-NEXT: movq %rdi, %r13
+; NO-SETZUCC-NEXT: sarq $63, %r13
+; NO-SETZUCC-NEXT: movl %r13d, 96(%rax)
+; NO-SETZUCC-NEXT: movabsq $2251799813685247, %rbp # imm = 0x7FFFFFFFFFFFF
+; NO-SETZUCC-NEXT: andq %r13, %rbp
+; NO-SETZUCC-NEXT: shldq $62, %rdi, %r13
+; NO-SETZUCC-NEXT: movq %r13, 88(%rax)
+; NO-SETZUCC-NEXT: movq %r9, %r13
+; NO-SETZUCC-NEXT: shrdq $44, %r10, %r13
+; NO-SETZUCC-NEXT: movq %r13, 64(%rax)
+; NO-SETZUCC-NEXT: movq %r8, %r13
+; NO-SETZUCC-NEXT: shrdq $33, %r11, %r13
+; NO-SETZUCC-NEXT: movq %r13, 48(%rax)
+; NO-SETZUCC-NEXT: movq %rbx, %r13
+; NO-SETZUCC-NEXT: shrdq $22, %rcx, %r13
+; NO-SETZUCC-NEXT: movq %r13, 32(%rax)
+; NO-SETZUCC-NEXT: movq %rdx, %r13
+; NO-SETZUCC-NEXT: shrdq $11, %r12, %r13
+; NO-SETZUCC-NEXT: movq %r13, 16(%rax)
+; NO-SETZUCC-NEXT: movq %rbp, %r13
+; NO-SETZUCC-NEXT: shrq $48, %r13
+; NO-SETZUCC-NEXT: movb %r13b, 102(%rax)
+; NO-SETZUCC-NEXT: shrq $32, %rbp
+; NO-SETZUCC-NEXT: movw %bp, 100(%rax)
+; NO-SETZUCC-NEXT: movabsq $9007199254740991, %r13 # imm = 0x1FFFFFFFFFFFFF
+; NO-SETZUCC-NEXT: andq %r13, %r15
+; NO-SETZUCC-NEXT: shldq $9, %r14, %r15
+; NO-SETZUCC-NEXT: shlq $62, %rdi
+; NO-SETZUCC-NEXT: orq %r15, %rdi
+; NO-SETZUCC-NEXT: movq %rdi, 80(%rax)
+; NO-SETZUCC-NEXT: movabsq $9007199254738944, %rdi # imm = 0x1FFFFFFFFFF800
+; NO-SETZUCC-NEXT: andq %r12, %rdi
+; NO-SETZUCC-NEXT: shlq $42, %rbx
; NO-SETZUCC-NEXT: shrq $11, %rdi
-; NO-SETZUCC-NEXT: movabsq $9007199254738944, %rbp # imm = 0x1FFFFFFFFFF800
-; NO-SETZUCC-NEXT: andq %r12, %rbp
-; NO-SETZUCC-NEXT: shlq $53, %r12
-; NO-SETZUCC-NEXT: orq %rdi, %r12
-; NO-SETZUCC-NEXT: movq %r12, 16(%rax)
-; NO-SETZUCC-NEXT: movq %r13, %rdi
-; NO-SETZUCC-NEXT: shrq $48, %rdi
-; NO-SETZUCC-NEXT: movb %dil, 102(%rax)
-; NO-SETZUCC-NEXT: shrq $32, %r13
-; NO-SETZUCC-NEXT: movw %r13w, 100(%rax)
-; NO-SETZUCC-NEXT: movabsq $9007199254740991, %r12 # imm = 0x1FFFFFFFFFFFFF
-; NO-SETZUCC-NEXT: andq %r12, %r15
-; NO-SETZUCC-NEXT: movq %r14, %rdi
-; NO-SETZUCC-NEXT: shrq $55, %rdi
-; NO-SETZUCC-NEXT: shlq $9, %r15
-; NO-SETZUCC-NEXT: orq %rdi, %r15
-; NO-SETZUCC-NEXT: shlq $62, %rcx
-; NO-SETZUCC-NEXT: orq %r15, %rcx
-; NO-SETZUCC-NEXT: movq %rcx, 80(%rax)
-; NO-SETZUCC-NEXT: shlq $42, %rsi
-; NO-SETZUCC-NEXT: shrq $11, %rbp
-; NO-SETZUCC-NEXT: orq %rsi, %rbp
-; NO-SETZUCC-NEXT: movq %rbp, 24(%rax)
+; NO-SETZUCC-NEXT: orq %rbx, %rdi
+; NO-SETZUCC-NEXT: movq %rdi, 24(%rax)
; NO-SETZUCC-NEXT: shlq $9, %r14
; NO-SETZUCC-NEXT: andl $511, %r10d # imm = 0x1FF
; NO-SETZUCC-NEXT: orq %r14, %r10
@@ -3712,15 +3647,14 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; NO-SETZUCC-NEXT: orq %r9, %r11
; NO-SETZUCC-NEXT: movq %r11, 56(%rax)
; NO-SETZUCC-NEXT: shlq $31, %r8
-; NO-SETZUCC-NEXT: andl $2147483647, %ebx # imm = 0x7FFFFFFF
-; NO-SETZUCC-NEXT: orq %r8, %rbx
-; NO-SETZUCC-NEXT: movq %rbx, 40(%rax)
-; NO-SETZUCC-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 8-byte Folded Reload
-; NO-SETZUCC-NEXT: # xmm1 = mem[0],zero
+; NO-SETZUCC-NEXT: andl $2147483647, %ecx # imm = 0x7FFFFFFF
+; NO-SETZUCC-NEXT: orq %r8, %rcx
+; NO-SETZUCC-NEXT: movq %rcx, 40(%rax)
+; NO-SETZUCC-NEXT: movq %rsi, %xmm1
; NO-SETZUCC-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; NO-SETZUCC-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
; NO-SETZUCC-NEXT: movq %xmm0, %rcx
-; NO-SETZUCC-NEXT: andq %r12, %rcx
+; NO-SETZUCC-NEXT: andq %r13, %rcx
; NO-SETZUCC-NEXT: shlq $53, %rdx
; NO-SETZUCC-NEXT: orq %rcx, %rdx
; NO-SETZUCC-NEXT: movq %rdx, 8(%rax)
diff --git a/llvm/test/CodeGen/X86/sdiv_fix.ll b/llvm/test/CodeGen/X86/sdiv_fix.ll
index 2511ab46d24b6..392bc83d9d5d8 100644
--- a/llvm/test/CodeGen/X86/sdiv_fix.ll
+++ b/llvm/test/CodeGen/X86/sdiv_fix.ll
@@ -309,31 +309,23 @@ define i64 @func5(i64 %x, i64 %y) nounwind {
; X86-NEXT: andl $-16, %esp
; X86-NEXT: subl $112, %esp
; X86-NEXT: movl 8(%ebp), %ecx
-; X86-NEXT: movl 12(%ebp), %esi
-; X86-NEXT: movl 20(%ebp), %ebx
-; X86-NEXT: movl %esi, %eax
-; X86-NEXT: sarl $31, %eax
-; X86-NEXT: movl %eax, %edx
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %esi, %eax
-; X86-NEXT: shll $31, %eax
-; X86-NEXT: movl %ecx, %edi
-; X86-NEXT: shrl %edi
-; X86-NEXT: orl %eax, %edi
-; X86-NEXT: leal {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl %eax, (%esp)
-; X86-NEXT: movl %edx, %eax
-; X86-NEXT: shll $31, %eax
-; X86-NEXT: shrl %esi
-; X86-NEXT: orl %eax, %esi
-; X86-NEXT: movl %ebx, %eax
+; X86-NEXT: movl 12(%ebp), %edi
+; X86-NEXT: movl 16(%ebp), %eax
+; X86-NEXT: movl 20(%ebp), %edx
+; X86-NEXT: movl %edx, %ebx
; X86-NEXT: sarl $31, %ebx
; X86-NEXT: movl %ebx, {{[0-9]+}}(%esp)
; X86-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 16(%ebp), %eax
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %edi, %esi
+; X86-NEXT: sarl $31, %esi
+; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %eax, (%esp)
+; X86-NEXT: shldl $31, %edi, %esi
+; X86-NEXT: shldl $31, %ecx, %edi
; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X86-NEXT: shll $31, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
@@ -545,126 +537,115 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
; X86-NEXT: pushl %esi
; X86-NEXT: subl $60, %esp
; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl %eax, %edx
-; X86-NEXT: movl %eax, %esi
-; X86-NEXT: sarl $31, %edx
-; X86-NEXT: movl %edx, %edi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT: movl %esi, %ebp
+; X86-NEXT: sarl $31, %ebp
+; X86-NEXT: movl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: shll $31, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: shll $31, %eax
-; X86-NEXT: movl %eax, %edx
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %ecx, %eax
; X86-NEXT: shrl $31, %eax
-; X86-NEXT: shll $31, %eax
-; X86-NEXT: shrl %ecx
-; X86-NEXT: orl %eax, %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: pushl %edi
+; X86-NEXT: shldl $31, %ecx, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: pushl %ebp
; X86-NEXT: pushl %esi
-; X86-NEXT: pushl %ecx
+; X86-NEXT: pushl %eax
; X86-NEXT: pushl %edx
; X86-NEXT: calll __divdi3
; X86-NEXT: addl $16, %esp
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %ebp, %eax
+; X86-NEXT: movl %edi, %eax
; X86-NEXT: sarl $31, %eax
-; X86-NEXT: movl %eax, %edi
+; X86-NEXT: movl %ebx, %ebp
+; X86-NEXT: shll $31, %ebp
+; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: shrl $31, %ecx
+; X86-NEXT: shldl $31, %ebx, %ecx
+; X86-NEXT: pushl %eax
+; X86-NEXT: movl %eax, %esi
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %ebx, %esi
-; X86-NEXT: shll $31, %esi
-; X86-NEXT: movl %ebx, %eax
-; X86-NEXT: shrl $31, %eax
-; X86-NEXT: shll $31, %eax
-; X86-NEXT: shrl %ebx
-; X86-NEXT: orl %eax, %ebx
; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %ecx
+; X86-NEXT: movl %ecx, %ebx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: pushl %ebp
-; X86-NEXT: pushl %ebx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: pushl %esi
; X86-NEXT: calll __moddi3
; X86-NEXT: addl $16, %esp
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: pushl %esi
; X86-NEXT: pushl %edi
-; X86-NEXT: pushl %ebp
; X86-NEXT: pushl %ebx
-; X86-NEXT: pushl %esi
+; X86-NEXT: pushl %ebp
; X86-NEXT: calll __divdi3
; X86-NEXT: addl $16, %esp
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: sarl $31, %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl %edx, %eax
-; X86-NEXT: sarl $31, %eax
-; X86-NEXT: movl %eax, %ebx
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ecx, %esi
-; X86-NEXT: shll $31, %esi
-; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: shrl $31, %eax
-; X86-NEXT: shll $31, %eax
-; X86-NEXT: shrl %ecx
-; X86-NEXT: orl %eax, %ecx
-; X86-NEXT: pushl %ebx
-; X86-NEXT: pushl %edx
+; X86-NEXT: movl %edx, %ebx
+; X86-NEXT: shll $31, %ebx
; X86-NEXT: movl %edx, %edi
+; X86-NEXT: shrl $31, %edi
+; X86-NEXT: shldl $31, %edx, %edi
; X86-NEXT: pushl %ecx
; X86-NEXT: movl %ecx, %ebp
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: pushl %esi
+; X86-NEXT: pushl %eax
+; X86-NEXT: movl %eax, %esi
+; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %ebx
; X86-NEXT: calll __moddi3
; X86-NEXT: addl $16, %esp
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: pushl %ebx
-; X86-NEXT: pushl %edi
; X86-NEXT: pushl %ebp
; X86-NEXT: pushl %esi
+; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %ebx
; X86-NEXT: calll __divdi3
; X86-NEXT: addl $16, %esp
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %eax, %ebx
+; X86-NEXT: sarl $31, %ebx
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: shll $31, %esi
; X86-NEXT: movl %ecx, %ebp
-; X86-NEXT: sarl $31, %ebp
-; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT: movl %esi, %edi
-; X86-NEXT: shll $31, %edi
-; X86-NEXT: movl %esi, %eax
-; X86-NEXT: shrl $31, %eax
-; X86-NEXT: shll $31, %eax
-; X86-NEXT: shrl %esi
-; X86-NEXT: orl %eax, %esi
+; X86-NEXT: shrl $31, %ebp
+; X86-NEXT: shldl $31, %ecx, %ebp
+; X86-NEXT: pushl %ebx
+; X86-NEXT: pushl %eax
; X86-NEXT: pushl %ebp
-; X86-NEXT: pushl %ecx
; X86-NEXT: pushl %esi
-; X86-NEXT: pushl %edi
; X86-NEXT: calll __moddi3
; X86-NEXT: addl $16, %esp
; X86-NEXT: movl %eax, (%esp) # 4-byte Spill
-; X86-NEXT: movl %edx, %ebx
-; X86-NEXT: pushl %ebp
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: pushl %ebx
; X86-NEXT: pushl {{[0-9]+}}(%esp)
+; X86-NEXT: pushl %ebp
; X86-NEXT: pushl %esi
-; X86-NEXT: pushl %edi
; X86-NEXT: calll __divdi3
; X86-NEXT: addl $16, %esp
-; X86-NEXT: testl %esi, %esi
-; X86-NEXT: sets %cl
; X86-NEXT: testl %ebp, %ebp
+; X86-NEXT: sets %cl
+; X86-NEXT: testl %ebx, %ebx
; X86-NEXT: sets %dl
; X86-NEXT: xorb %cl, %dl
-; X86-NEXT: orl (%esp), %ebx # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: orl (%esp), %ecx # 4-byte Folded Reload
; X86-NEXT: setne %cl
; X86-NEXT: testb %dl, %cl
; X86-NEXT: leal -1(%eax), %ecx
; X86-NEXT: cmovel %eax, %ecx
; X86-NEXT: movl %ecx, (%esp) # 4-byte Spill
-; X86-NEXT: cmpl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; X86-NEXT: testl %edi, %edi
; X86-NEXT: sets %al
; X86-NEXT: cmpl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
; X86-NEXT: sets %cl
diff --git a/llvm/test/CodeGen/X86/sdiv_fix_sat.ll b/llvm/test/CodeGen/X86/sdiv_fix_sat.ll
index 35f70aa5f0120..e7d41d5bebc8a 100644
--- a/llvm/test/CodeGen/X86/sdiv_fix_sat.ll
+++ b/llvm/test/CodeGen/X86/sdiv_fix_sat.ll
@@ -342,19 +342,17 @@ define i64 @func5(i64 %x, i64 %y) nounwind {
; X64-NEXT: testb %bl, %al
; X64-NEXT: cmoveq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Folded Reload
; X64-NEXT: cmoveq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Folded Reload
-; X64-NEXT: movq %rbp, %rax
-; X64-NEXT: sarq $63, %rax
-; X64-NEXT: andq %rbp, %rax
+; X64-NEXT: movq %rbp, %rcx
+; X64-NEXT: sarq $63, %rcx
+; X64-NEXT: andq %rbp, %rcx
; X64-NEXT: testq %rbp, %rbp
-; X64-NEXT: movq $-1, %rcx
-; X64-NEXT: cmovgq %rcx, %r13
-; X64-NEXT: xorl %edx, %edx
-; X64-NEXT: cmpq $-1, %rax
-; X64-NEXT: cmovlq %rcx, %rax
-; X64-NEXT: cmovgeq %r13, %rdx
-; X64-NEXT: shrq %rdx
-; X64-NEXT: shlq $63, %rax
-; X64-NEXT: orq %rdx, %rax
+; X64-NEXT: movq $-1, %rdx
+; X64-NEXT: cmovgq %rdx, %r13
+; X64-NEXT: xorl %eax, %eax
+; X64-NEXT: cmpq $-1, %rcx
+; X64-NEXT: cmovlq %rdx, %rcx
+; X64-NEXT: cmovgeq %r13, %rax
+; X64-NEXT: shrdq $1, %rcx, %rax
; X64-NEXT: addq $24, %rsp
; X64-NEXT: popq %rbx
; X64-NEXT: popq %r12
@@ -373,72 +371,65 @@ define i64 @func5(i64 %x, i64 %y) nounwind {
; X86-NEXT: pushl %esi
; X86-NEXT: andl $-16, %esp
; X86-NEXT: subl $128, %esp
-; X86-NEXT: movl 8(%ebp), %edx
-; X86-NEXT: movl 12(%ebp), %esi
-; X86-NEXT: movl 20(%ebp), %ebx
-; X86-NEXT: movl %esi, %eax
+; X86-NEXT: movl 8(%ebp), %esi
+; X86-NEXT: movl 12(%ebp), %edi
+; X86-NEXT: movl 16(%ebp), %ecx
+; X86-NEXT: movl 20(%ebp), %edx
+; X86-NEXT: movl %edx, %eax
; X86-NEXT: sarl $31, %eax
-; X86-NEXT: movl %eax, %ecx
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %esi, %eax
-; X86-NEXT: shll $31, %eax
-; X86-NEXT: movl %edx, %edi
-; X86-NEXT: shrl %edi
-; X86-NEXT: orl %eax, %edi
-; X86-NEXT: leal {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl %eax, (%esp)
-; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: shll $31, %eax
-; X86-NEXT: shrl %esi
-; X86-NEXT: orl %eax, %esi
-; X86-NEXT: movl %ebx, %eax
-; X86-NEXT: sarl $31, %ebx
-; X86-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X86-NEXT: movl %ebx, {{[0-9]+}}(%esp)
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 16(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X86-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %edi, %ebx
+; X86-NEXT: sarl $31, %ebx
+; X86-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: leal {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %eax, (%esp)
+; X86-NEXT: shldl $31, %edi, %ebx
+; X86-NEXT: shldl $31, %esi, %edi
; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-NEXT: shll $31, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NEXT: shll $31, %esi
; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %ebx, {{[0-9]+}}(%esp)
; X86-NEXT: calll __divti3
; X86-NEXT: subl $4, %esp
; X86-NEXT: movl 20(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: movl 16(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X86-NEXT: movl %ebx, {{[0-9]+}}(%esp)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
; X86-NEXT: leal {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl %eax, (%esp)
-; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: subl $1, %esi
+; X86-NEXT: subl $1, %edi
; X86-NEXT: sbbl $0, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: sbbl $0, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: sbbl $0, %edi
-; X86-NEXT: testl %ebx, %ebx
-; X86-NEXT: sets %al
+; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT: movl %esi, %ebx
+; X86-NEXT: sbbl $0, %ebx
; X86-NEXT: testl %ecx, %ecx
-; X86-NEXT: sets %bl
-; X86-NEXT: xorb %al, %bl
+; X86-NEXT: sets %al
+; X86-NEXT: testl %edx, %edx
+; X86-NEXT: sets %cl
+; X86-NEXT: xorb %al, %cl
+; X86-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
; X86-NEXT: calll __modti3
; X86-NEXT: subl $4, %esp
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
@@ -447,41 +438,39 @@ define i64 @func5(i64 %x, i64 %y) nounwind {
; X86-NEXT: orl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: orl %eax, %ecx
; X86-NEXT: setne %al
-; X86-NEXT: testb %bl, %al
-; X86-NEXT: cmovel {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: cmovel {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X86-NEXT: testb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Reload
+; X86-NEXT: cmovel %esi, %ebx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: cmovel {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NEXT: cmovel {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X86-NEXT: cmpl $-1, %esi
-; X86-NEXT: movl %eax, %ecx
-; X86-NEXT: movl %eax, %ebx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: cmovel {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: cmovel {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X86-NEXT: cmpl $-1, %edi
; X86-NEXT: sbbl $2147483647, %ecx # imm = 0x7FFFFFFF
-; X86-NEXT: movl %edx, %ecx
-; X86-NEXT: movl %edx, %eax
+; X86-NEXT: movl %eax, %ecx
; X86-NEXT: sbbl $0, %ecx
-; X86-NEXT: movl %edi, %ecx
+; X86-NEXT: movl %ebx, %ecx
; X86-NEXT: sbbl $0, %ecx
; X86-NEXT: movl $2147483647, %edx # imm = 0x7FFFFFFF
-; X86-NEXT: cmovll %ebx, %edx
+; X86-NEXT: cmovll {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
; X86-NEXT: movl $0, %ecx
-; X86-NEXT: cmovgel %ecx, %edi
+; X86-NEXT: cmovgel %ecx, %ebx
; X86-NEXT: cmovgel %ecx, %eax
; X86-NEXT: movl $-1, %ecx
-; X86-NEXT: cmovgel %ecx, %esi
-; X86-NEXT: movl %esi, %ebx
-; X86-NEXT: negl %ebx
-; X86-NEXT: movl $-2147483648, %ebx # imm = 0x80000000
-; X86-NEXT: sbbl %edx, %ebx
-; X86-NEXT: movl $-1, %ebx
-; X86-NEXT: sbbl %eax, %ebx
-; X86-NEXT: sbbl %edi, %ecx
+; X86-NEXT: cmovgel %ecx, %edi
+; X86-NEXT: movl %edi, %esi
+; X86-NEXT: negl %esi
+; X86-NEXT: movl $-2147483648, %esi # imm = 0x80000000
+; X86-NEXT: sbbl %edx, %esi
+; X86-NEXT: movl $-1, %esi
+; X86-NEXT: sbbl %eax, %esi
+; X86-NEXT: sbbl %ebx, %ecx
; X86-NEXT: movl $0, %eax
-; X86-NEXT: cmovgel %eax, %esi
+; X86-NEXT: cmovgel %eax, %edi
; X86-NEXT: movl $-2147483648, %eax # imm = 0x80000000
; X86-NEXT: cmovgel %eax, %edx
-; X86-NEXT: movl %esi, %eax
+; X86-NEXT: movl %edi, %eax
; X86-NEXT: leal -12(%ebp), %esp
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
@@ -634,12 +623,9 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
; X64-NEXT: pshufd $238, (%rsp), %xmm0 # 16-byte Folded Reload
; X64-NEXT: # xmm0 = mem[2,3,2,3]
; X64-NEXT: movq %xmm0, %r15
-; X64-NEXT: movq %r15, %rax
-; X64-NEXT: sarq $63, %rax
-; X64-NEXT: shlq $31, %rax
; X64-NEXT: movq %r15, %rbx
-; X64-NEXT: shrq $33, %rbx
-; X64-NEXT: orq %rax, %rbx
+; X64-NEXT: sarq $63, %rbx
+; X64-NEXT: shldq $31, %r15, %rbx
; X64-NEXT: pshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
; X64-NEXT: # xmm0 = mem[2,3,2,3]
; X64-NEXT: movq %xmm0, %r12
@@ -695,12 +681,9 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
; X64-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
; X64-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; X64-NEXT: movq %xmm0, %r15
-; X64-NEXT: movq %r15, %rax
-; X64-NEXT: sarq $63, %rax
-; X64-NEXT: shlq $31, %rax
; X64-NEXT: movq %r15, %rbx
-; X64-NEXT: shrq $33, %rbx
-; X64-NEXT: orq %rax, %rbx
+; X64-NEXT: sarq $63, %rbx
+; X64-NEXT: shldq $31, %r15, %rbx
; X64-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
; X64-NEXT: pxor %xmm1, %xmm1
; X64-NEXT: pcmpgtd %xmm0, %xmm1
@@ -750,12 +733,9 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
; X64-NEXT: pshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
; X64-NEXT: # xmm0 = mem[2,3,2,3]
; X64-NEXT: movq %xmm0, %r15
-; X64-NEXT: movq %r15, %rax
-; X64-NEXT: sarq $63, %rax
-; X64-NEXT: shlq $31, %rax
; X64-NEXT: movq %r15, %rbx
-; X64-NEXT: shrq $33, %rbx
-; X64-NEXT: orq %rax, %rbx
+; X64-NEXT: sarq $63, %rbx
+; X64-NEXT: shldq $31, %r15, %rbx
; X64-NEXT: pshufd $238, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
; X64-NEXT: # xmm0 = mem[2,3,2,3]
; X64-NEXT: movq %xmm0, %r12
diff --git a/llvm/test/CodeGen/X86/shift-and.ll b/llvm/test/CodeGen/X86/shift-and.ll
index 9fcab4c243b12..a467720fbbe61 100644
--- a/llvm/test/CodeGen/X86/shift-and.ll
+++ b/llvm/test/CodeGen/X86/shift-and.ll
@@ -168,22 +168,21 @@ define void @t5ptr(i64 %t, ptr %ptr) nounwind {
define i64 @t6(i64 %key, ptr nocapture %val) nounwind {
; X86-LABEL: t6:
; X86: # %bb.0:
+; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT: movl %esi, %ecx
-; X86-NEXT: shrl $3, %ecx
-; X86-NEXT: shrl $3, %eax
-; X86-NEXT: shll $29, %esi
-; X86-NEXT: orl %eax, %esi
-; X86-NEXT: movl (%edx), %eax
-; X86-NEXT: movl 4(%edx), %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT: shrdl $3, %edi, %esi
+; X86-NEXT: shrl $3, %edi
+; X86-NEXT: movl (%ecx), %eax
+; X86-NEXT: movl 4(%ecx), %edx
; X86-NEXT: addl $-1, %eax
; X86-NEXT: adcl $-1, %edx
; X86-NEXT: andl %esi, %eax
-; X86-NEXT: andl %ecx, %edx
+; X86-NEXT: andl %edi, %edx
; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
; X86-NEXT: retl
;
; X64-LABEL: t6:
diff --git a/llvm/test/CodeGen/X86/shift-i256.ll b/llvm/test/CodeGen/X86/shift-i256.ll
index 3cd6df8779db4..d82172c964398 100644
--- a/llvm/test/CodeGen/X86/shift-i256.ll
+++ b/llvm/test/CodeGen/X86/shift-i256.ll
@@ -1455,25 +1455,58 @@ define i256 @ashr_i256_load(ptr %p0, i256 %a1) nounwind {
}
define i256 @shl_i256_1(i256 %a0) nounwind {
-; CHECK-LABEL: shl_i256_1:
-; CHECK: # %bb.0:
-; CHECK-NEXT: movq %rdi, %rax
-; CHECK-NEXT: leaq (,%rdx,2), %rdi
-; CHECK-NEXT: movq %rsi, %r9
-; CHECK-NEXT: shrq $63, %r9
-; CHECK-NEXT: orq %rdi, %r9
-; CHECK-NEXT: leaq (,%rcx,2), %rdi
-; CHECK-NEXT: shrq $63, %rdx
-; CHECK-NEXT: orq %rdi, %rdx
-; CHECK-NEXT: shlq %r8
-; CHECK-NEXT: shrq $63, %rcx
-; CHECK-NEXT: orq %r8, %rcx
-; CHECK-NEXT: addq %rsi, %rsi
-; CHECK-NEXT: movq %rcx, 24(%rax)
-; CHECK-NEXT: movq %rdx, 16(%rax)
-; CHECK-NEXT: movq %r9, 8(%rax)
-; CHECK-NEXT: movq %rsi, (%rax)
-; CHECK-NEXT: retq
+; SSE-LABEL: shl_i256_1:
+; SSE: # %bb.0:
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: shldq $1, %rcx, %r8
+; SSE-NEXT: shldq $1, %rdx, %rcx
+; SSE-NEXT: shldq $1, %rsi, %rdx
+; SSE-NEXT: addq %rsi, %rsi
+; SSE-NEXT: movq %r8, 24(%rdi)
+; SSE-NEXT: movq %rcx, 16(%rdi)
+; SSE-NEXT: movq %rdx, 8(%rdi)
+; SSE-NEXT: movq %rsi, (%rdi)
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: shl_i256_1:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: leaq (,%rdx,2), %rdi
+; AVX2-NEXT: movq %rsi, %r9
+; AVX2-NEXT: shrq $63, %r9
+; AVX2-NEXT: orq %rdi, %r9
+; AVX2-NEXT: leaq (,%rcx,2), %rdi
+; AVX2-NEXT: shrq $63, %rdx
+; AVX2-NEXT: orq %rdi, %rdx
+; AVX2-NEXT: shlq %r8
+; AVX2-NEXT: shrq $63, %rcx
+; AVX2-NEXT: orq %r8, %rcx
+; AVX2-NEXT: addq %rsi, %rsi
+; AVX2-NEXT: movq %rcx, 24(%rax)
+; AVX2-NEXT: movq %rdx, 16(%rax)
+; AVX2-NEXT: movq %r9, 8(%rax)
+; AVX2-NEXT: movq %rsi, (%rax)
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: shl_i256_1:
+; AVX512: # %bb.0:
+; AVX512-NEXT: movq %rdi, %rax
+; AVX512-NEXT: leaq (,%rdx,2), %rdi
+; AVX512-NEXT: movq %rsi, %r9
+; AVX512-NEXT: shrq $63, %r9
+; AVX512-NEXT: orq %rdi, %r9
+; AVX512-NEXT: leaq (,%rcx,2), %rdi
+; AVX512-NEXT: shrq $63, %rdx
+; AVX512-NEXT: orq %rdi, %rdx
+; AVX512-NEXT: shlq %r8
+; AVX512-NEXT: shrq $63, %rcx
+; AVX512-NEXT: orq %r8, %rcx
+; AVX512-NEXT: addq %rsi, %rsi
+; AVX512-NEXT: movq %rcx, 24(%rax)
+; AVX512-NEXT: movq %rdx, 16(%rax)
+; AVX512-NEXT: movq %r9, 8(%rax)
+; AVX512-NEXT: movq %rsi, (%rax)
+; AVX512-NEXT: retq
;
; X86-LABEL: shl_i256_1:
; X86: # %bb.0:
@@ -1508,27 +1541,62 @@ define i256 @shl_i256_1(i256 %a0) nounwind {
}
define i256 @lshr_i256_1(i256 %a0) nounwind {
-; CHECK-LABEL: lshr_i256_1:
-; CHECK: # %bb.0:
-; CHECK-NEXT: movq %rdi, %rax
-; CHECK-NEXT: shrq %rsi
-; CHECK-NEXT: movq %rdx, %rdi
-; CHECK-NEXT: shlq $63, %rdi
-; CHECK-NEXT: orq %rsi, %rdi
-; CHECK-NEXT: movq %rcx, %rsi
-; CHECK-NEXT: shlq $63, %rsi
-; CHECK-NEXT: shrq %rdx
-; CHECK-NEXT: orq %rsi, %rdx
-; CHECK-NEXT: shrq %rcx
-; CHECK-NEXT: movq %r8, %rsi
-; CHECK-NEXT: shlq $63, %rsi
-; CHECK-NEXT: orq %rcx, %rsi
-; CHECK-NEXT: shrq %r8
-; CHECK-NEXT: movq %r8, 24(%rax)
-; CHECK-NEXT: movq %rsi, 16(%rax)
-; CHECK-NEXT: movq %rdx, 8(%rax)
-; CHECK-NEXT: movq %rdi, (%rax)
-; CHECK-NEXT: retq
+; SSE-LABEL: lshr_i256_1:
+; SSE: # %bb.0:
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: shrdq $1, %rdx, %rsi
+; SSE-NEXT: shrdq $1, %rcx, %rdx
+; SSE-NEXT: shrdq $1, %r8, %rcx
+; SSE-NEXT: shrq %r8
+; SSE-NEXT: movq %r8, 24(%rdi)
+; SSE-NEXT: movq %rcx, 16(%rdi)
+; SSE-NEXT: movq %rdx, 8(%rdi)
+; SSE-NEXT: movq %rsi, (%rdi)
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: lshr_i256_1:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: shrq %rsi
+; AVX2-NEXT: movq %rdx, %rdi
+; AVX2-NEXT: shlq $63, %rdi
+; AVX2-NEXT: orq %rsi, %rdi
+; AVX2-NEXT: movq %rcx, %rsi
+; AVX2-NEXT: shlq $63, %rsi
+; AVX2-NEXT: shrq %rdx
+; AVX2-NEXT: orq %rsi, %rdx
+; AVX2-NEXT: shrq %rcx
+; AVX2-NEXT: movq %r8, %rsi
+; AVX2-NEXT: shlq $63, %rsi
+; AVX2-NEXT: orq %rcx, %rsi
+; AVX2-NEXT: shrq %r8
+; AVX2-NEXT: movq %r8, 24(%rax)
+; AVX2-NEXT: movq %rsi, 16(%rax)
+; AVX2-NEXT: movq %rdx, 8(%rax)
+; AVX2-NEXT: movq %rdi, (%rax)
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: lshr_i256_1:
+; AVX512: # %bb.0:
+; AVX512-NEXT: movq %rdi, %rax
+; AVX512-NEXT: shrq %rsi
+; AVX512-NEXT: movq %rdx, %rdi
+; AVX512-NEXT: shlq $63, %rdi
+; AVX512-NEXT: orq %rsi, %rdi
+; AVX512-NEXT: movq %rcx, %rsi
+; AVX512-NEXT: shlq $63, %rsi
+; AVX512-NEXT: shrq %rdx
+; AVX512-NEXT: orq %rsi, %rdx
+; AVX512-NEXT: shrq %rcx
+; AVX512-NEXT: movq %r8, %rsi
+; AVX512-NEXT: shlq $63, %rsi
+; AVX512-NEXT: orq %rcx, %rsi
+; AVX512-NEXT: shrq %r8
+; AVX512-NEXT: movq %r8, 24(%rax)
+; AVX512-NEXT: movq %rsi, 16(%rax)
+; AVX512-NEXT: movq %rdx, 8(%rax)
+; AVX512-NEXT: movq %rdi, (%rax)
+; AVX512-NEXT: retq
;
; X86-LABEL: lshr_i256_1:
; X86: # %bb.0:
@@ -1581,27 +1649,62 @@ define i256 @lshr_i256_1(i256 %a0) nounwind {
}
define i256 @ashr_i256_1(i256 %a0) nounwind {
-; CHECK-LABEL: ashr_i256_1:
-; CHECK: # %bb.0:
-; CHECK-NEXT: movq %rdi, %rax
-; CHECK-NEXT: shrq %rsi
-; CHECK-NEXT: movq %rdx, %rdi
-; CHECK-NEXT: shlq $63, %rdi
-; CHECK-NEXT: orq %rsi, %rdi
-; CHECK-NEXT: movq %rcx, %rsi
-; CHECK-NEXT: shlq $63, %rsi
-; CHECK-NEXT: shrq %rdx
-; CHECK-NEXT: orq %rsi, %rdx
-; CHECK-NEXT: shrq %rcx
-; CHECK-NEXT: movq %r8, %rsi
-; CHECK-NEXT: shlq $63, %rsi
-; CHECK-NEXT: orq %rcx, %rsi
-; CHECK-NEXT: sarq %r8
-; CHECK-NEXT: movq %r8, 24(%rax)
-; CHECK-NEXT: movq %rsi, 16(%rax)
-; CHECK-NEXT: movq %rdx, 8(%rax)
-; CHECK-NEXT: movq %rdi, (%rax)
-; CHECK-NEXT: retq
+; SSE-LABEL: ashr_i256_1:
+; SSE: # %bb.0:
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: shrdq $1, %rdx, %rsi
+; SSE-NEXT: shrdq $1, %rcx, %rdx
+; SSE-NEXT: shrdq $1, %r8, %rcx
+; SSE-NEXT: sarq %r8
+; SSE-NEXT: movq %r8, 24(%rdi)
+; SSE-NEXT: movq %rcx, 16(%rdi)
+; SSE-NEXT: movq %rdx, 8(%rdi)
+; SSE-NEXT: movq %rsi, (%rdi)
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: ashr_i256_1:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: shrq %rsi
+; AVX2-NEXT: movq %rdx, %rdi
+; AVX2-NEXT: shlq $63, %rdi
+; AVX2-NEXT: orq %rsi, %rdi
+; AVX2-NEXT: movq %rcx, %rsi
+; AVX2-NEXT: shlq $63, %rsi
+; AVX2-NEXT: shrq %rdx
+; AVX2-NEXT: orq %rsi, %rdx
+; AVX2-NEXT: shrq %rcx
+; AVX2-NEXT: movq %r8, %rsi
+; AVX2-NEXT: shlq $63, %rsi
+; AVX2-NEXT: orq %rcx, %rsi
+; AVX2-NEXT: sarq %r8
+; AVX2-NEXT: movq %r8, 24(%rax)
+; AVX2-NEXT: movq %rsi, 16(%rax)
+; AVX2-NEXT: movq %rdx, 8(%rax)
+; AVX2-NEXT: movq %rdi, (%rax)
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: ashr_i256_1:
+; AVX512: # %bb.0:
+; AVX512-NEXT: movq %rdi, %rax
+; AVX512-NEXT: shrq %rsi
+; AVX512-NEXT: movq %rdx, %rdi
+; AVX512-NEXT: shlq $63, %rdi
+; AVX512-NEXT: orq %rsi, %rdi
+; AVX512-NEXT: movq %rcx, %rsi
+; AVX512-NEXT: shlq $63, %rsi
+; AVX512-NEXT: shrq %rdx
+; AVX512-NEXT: orq %rsi, %rdx
+; AVX512-NEXT: shrq %rcx
+; AVX512-NEXT: movq %r8, %rsi
+; AVX512-NEXT: shlq $63, %rsi
+; AVX512-NEXT: orq %rcx, %rsi
+; AVX512-NEXT: sarq %r8
+; AVX512-NEXT: movq %r8, 24(%rax)
+; AVX512-NEXT: movq %rsi, 16(%rax)
+; AVX512-NEXT: movq %rdx, 8(%rax)
+; AVX512-NEXT: movq %rdi, (%rax)
+; AVX512-NEXT: retq
;
; X86-LABEL: ashr_i256_1:
; X86: # %bb.0:
diff --git a/llvm/test/CodeGen/X86/shift-i512.ll b/llvm/test/CodeGen/X86/shift-i512.ll
index cc072a775f40b..3c49212378c64 100644
--- a/llvm/test/CodeGen/X86/shift-i512.ll
+++ b/llvm/test/CodeGen/X86/shift-i512.ll
@@ -1256,150 +1256,359 @@ define i512 @ashr_i512_load(ptr %p0, i512 %a1) nounwind {
}
define i512 @shl_i512_1(i512 %a0) nounwind {
-; CHECK-LABEL: shl_i512_1:
-; CHECK: # %bb.0:
-; CHECK-NEXT: pushq %r14
-; CHECK-NEXT: pushq %rbx
-; CHECK-NEXT: movq %rdi, %rax
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r10
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rbx
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r11
-; CHECK-NEXT: leaq (,%rdx,2), %r14
-; CHECK-NEXT: movq %rsi, %rdi
-; CHECK-NEXT: shrq $63, %rdi
-; CHECK-NEXT: orq %r14, %rdi
-; CHECK-NEXT: leaq (,%rcx,2), %r14
-; CHECK-NEXT: shrq $63, %rdx
-; CHECK-NEXT: orq %r14, %rdx
-; CHECK-NEXT: leaq (,%r8,2), %r14
-; CHECK-NEXT: shrq $63, %rcx
-; CHECK-NEXT: orq %r14, %rcx
-; CHECK-NEXT: leaq (,%r9,2), %r14
-; CHECK-NEXT: shrq $63, %r8
-; CHECK-NEXT: orq %r14, %r8
-; CHECK-NEXT: leaq (,%r11,2), %r14
-; CHECK-NEXT: shrq $63, %r9
-; CHECK-NEXT: orq %r14, %r9
-; CHECK-NEXT: leaq (,%r10,2), %r14
-; CHECK-NEXT: shrq $63, %r11
-; CHECK-NEXT: orq %r14, %r11
-; CHECK-NEXT: addq %rsi, %rsi
-; CHECK-NEXT: shlq %rbx
-; CHECK-NEXT: shrq $63, %r10
-; CHECK-NEXT: orq %rbx, %r10
-; CHECK-NEXT: movq %r10, 56(%rax)
-; CHECK-NEXT: movq %r11, 48(%rax)
-; CHECK-NEXT: movq %r9, 40(%rax)
-; CHECK-NEXT: movq %r8, 32(%rax)
-; CHECK-NEXT: movq %rcx, 24(%rax)
-; CHECK-NEXT: movq %rdx, 16(%rax)
-; CHECK-NEXT: movq %rdi, 8(%rax)
-; CHECK-NEXT: movq %rsi, (%rax)
-; CHECK-NEXT: popq %rbx
-; CHECK-NEXT: popq %r14
-; CHECK-NEXT: retq
+; SSE-LABEL: shl_i512_1:
+; SSE: # %bb.0:
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rdi
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; SSE-NEXT: shldq $1, %rdi, %r10
+; SSE-NEXT: shldq $1, %r11, %rdi
+; SSE-NEXT: shldq $1, %r9, %r11
+; SSE-NEXT: shldq $1, %r8, %r9
+; SSE-NEXT: shldq $1, %rcx, %r8
+; SSE-NEXT: shldq $1, %rdx, %rcx
+; SSE-NEXT: shldq $1, %rsi, %rdx
+; SSE-NEXT: addq %rsi, %rsi
+; SSE-NEXT: movq %r10, 56(%rax)
+; SSE-NEXT: movq %rdi, 48(%rax)
+; SSE-NEXT: movq %r11, 40(%rax)
+; SSE-NEXT: movq %r9, 32(%rax)
+; SSE-NEXT: movq %r8, 24(%rax)
+; SSE-NEXT: movq %rcx, 16(%rax)
+; SSE-NEXT: movq %rdx, 8(%rax)
+; SSE-NEXT: movq %rsi, (%rax)
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: shl_i512_1:
+; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %r14
+; AVX2-NEXT: pushq %rbx
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX2-NEXT: leaq (,%rdx,2), %r14
+; AVX2-NEXT: movq %rsi, %rdi
+; AVX2-NEXT: shrq $63, %rdi
+; AVX2-NEXT: orq %r14, %rdi
+; AVX2-NEXT: leaq (,%rcx,2), %r14
+; AVX2-NEXT: shrq $63, %rdx
+; AVX2-NEXT: orq %r14, %rdx
+; AVX2-NEXT: leaq (,%r8,2), %r14
+; AVX2-NEXT: shrq $63, %rcx
+; AVX2-NEXT: orq %r14, %rcx
+; AVX2-NEXT: leaq (,%r9,2), %r14
+; AVX2-NEXT: shrq $63, %r8
+; AVX2-NEXT: orq %r14, %r8
+; AVX2-NEXT: leaq (,%r11,2), %r14
+; AVX2-NEXT: shrq $63, %r9
+; AVX2-NEXT: orq %r14, %r9
+; AVX2-NEXT: leaq (,%r10,2), %r14
+; AVX2-NEXT: shrq $63, %r11
+; AVX2-NEXT: orq %r14, %r11
+; AVX2-NEXT: addq %rsi, %rsi
+; AVX2-NEXT: shlq %rbx
+; AVX2-NEXT: shrq $63, %r10
+; AVX2-NEXT: orq %rbx, %r10
+; AVX2-NEXT: movq %r10, 56(%rax)
+; AVX2-NEXT: movq %r11, 48(%rax)
+; AVX2-NEXT: movq %r9, 40(%rax)
+; AVX2-NEXT: movq %r8, 32(%rax)
+; AVX2-NEXT: movq %rcx, 24(%rax)
+; AVX2-NEXT: movq %rdx, 16(%rax)
+; AVX2-NEXT: movq %rdi, 8(%rax)
+; AVX2-NEXT: movq %rsi, (%rax)
+; AVX2-NEXT: popq %rbx
+; AVX2-NEXT: popq %r14
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: shl_i512_1:
+; AVX512: # %bb.0:
+; AVX512-NEXT: pushq %r14
+; AVX512-NEXT: pushq %rbx
+; AVX512-NEXT: movq %rdi, %rax
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX512-NEXT: leaq (,%rdx,2), %r14
+; AVX512-NEXT: movq %rsi, %rdi
+; AVX512-NEXT: shrq $63, %rdi
+; AVX512-NEXT: orq %r14, %rdi
+; AVX512-NEXT: leaq (,%rcx,2), %r14
+; AVX512-NEXT: shrq $63, %rdx
+; AVX512-NEXT: orq %r14, %rdx
+; AVX512-NEXT: leaq (,%r8,2), %r14
+; AVX512-NEXT: shrq $63, %rcx
+; AVX512-NEXT: orq %r14, %rcx
+; AVX512-NEXT: leaq (,%r9,2), %r14
+; AVX512-NEXT: shrq $63, %r8
+; AVX512-NEXT: orq %r14, %r8
+; AVX512-NEXT: leaq (,%r11,2), %r14
+; AVX512-NEXT: shrq $63, %r9
+; AVX512-NEXT: orq %r14, %r9
+; AVX512-NEXT: leaq (,%r10,2), %r14
+; AVX512-NEXT: shrq $63, %r11
+; AVX512-NEXT: orq %r14, %r11
+; AVX512-NEXT: addq %rsi, %rsi
+; AVX512-NEXT: shlq %rbx
+; AVX512-NEXT: shrq $63, %r10
+; AVX512-NEXT: orq %rbx, %r10
+; AVX512-NEXT: movq %r10, 56(%rax)
+; AVX512-NEXT: movq %r11, 48(%rax)
+; AVX512-NEXT: movq %r9, 40(%rax)
+; AVX512-NEXT: movq %r8, 32(%rax)
+; AVX512-NEXT: movq %rcx, 24(%rax)
+; AVX512-NEXT: movq %rdx, 16(%rax)
+; AVX512-NEXT: movq %rdi, 8(%rax)
+; AVX512-NEXT: movq %rsi, (%rax)
+; AVX512-NEXT: popq %rbx
+; AVX512-NEXT: popq %r14
+; AVX512-NEXT: retq
%r = shl i512 %a0, 1
ret i512 %r
}
define i512 @lshr_i512_1(i512 %a0) nounwind {
-; CHECK-LABEL: lshr_i512_1:
-; CHECK: # %bb.0:
-; CHECK-NEXT: pushq %rbx
-; CHECK-NEXT: movq %rdi, %rax
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rbx
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r11
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r10
-; CHECK-NEXT: shrq %rsi
-; CHECK-NEXT: movq %rdx, %rdi
-; CHECK-NEXT: shlq $63, %rdi
-; CHECK-NEXT: orq %rsi, %rdi
-; CHECK-NEXT: movq %rcx, %rsi
-; CHECK-NEXT: shlq $63, %rsi
-; CHECK-NEXT: shrq %rdx
-; CHECK-NEXT: orq %rsi, %rdx
-; CHECK-NEXT: shrq %rcx
-; CHECK-NEXT: movq %r8, %rsi
-; CHECK-NEXT: shlq $63, %rsi
-; CHECK-NEXT: orq %rcx, %rsi
-; CHECK-NEXT: movq %r9, %rcx
-; CHECK-NEXT: shlq $63, %rcx
-; CHECK-NEXT: shrq %r8
-; CHECK-NEXT: orq %rcx, %r8
-; CHECK-NEXT: shrq %r9
-; CHECK-NEXT: movq %r10, %rcx
-; CHECK-NEXT: shlq $63, %rcx
-; CHECK-NEXT: orq %r9, %rcx
-; CHECK-NEXT: movq %rbx, %r9
-; CHECK-NEXT: shlq $63, %r9
-; CHECK-NEXT: shrq %r10
-; CHECK-NEXT: orq %r9, %r10
-; CHECK-NEXT: shrq %rbx
-; CHECK-NEXT: movq %r11, %r9
-; CHECK-NEXT: shlq $63, %r9
-; CHECK-NEXT: orq %rbx, %r9
-; CHECK-NEXT: shrq %r11
-; CHECK-NEXT: movq %r11, 56(%rax)
-; CHECK-NEXT: movq %r9, 48(%rax)
-; CHECK-NEXT: movq %r10, 40(%rax)
-; CHECK-NEXT: movq %rcx, 32(%rax)
-; CHECK-NEXT: movq %r8, 24(%rax)
-; CHECK-NEXT: movq %rsi, 16(%rax)
-; CHECK-NEXT: movq %rdx, 8(%rax)
-; CHECK-NEXT: movq %rdi, (%rax)
-; CHECK-NEXT: popq %rbx
-; CHECK-NEXT: retq
+; SSE-LABEL: lshr_i512_1:
+; SSE: # %bb.0:
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rdi
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; SSE-NEXT: shrdq $1, %rdx, %rsi
+; SSE-NEXT: shrdq $1, %rcx, %rdx
+; SSE-NEXT: shrdq $1, %r8, %rcx
+; SSE-NEXT: shrdq $1, %r9, %r8
+; SSE-NEXT: shrdq $1, %r11, %r9
+; SSE-NEXT: shrdq $1, %rdi, %r11
+; SSE-NEXT: shrdq $1, %r10, %rdi
+; SSE-NEXT: shrq %r10
+; SSE-NEXT: movq %r10, 56(%rax)
+; SSE-NEXT: movq %rdi, 48(%rax)
+; SSE-NEXT: movq %r11, 40(%rax)
+; SSE-NEXT: movq %r9, 32(%rax)
+; SSE-NEXT: movq %r8, 24(%rax)
+; SSE-NEXT: movq %rcx, 16(%rax)
+; SSE-NEXT: movq %rdx, 8(%rax)
+; SSE-NEXT: movq %rsi, (%rax)
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: lshr_i512_1:
+; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %rbx
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX2-NEXT: shrq %rsi
+; AVX2-NEXT: movq %rdx, %rdi
+; AVX2-NEXT: shlq $63, %rdi
+; AVX2-NEXT: orq %rsi, %rdi
+; AVX2-NEXT: movq %rcx, %rsi
+; AVX2-NEXT: shlq $63, %rsi
+; AVX2-NEXT: shrq %rdx
+; AVX2-NEXT: orq %rsi, %rdx
+; AVX2-NEXT: shrq %rcx
+; AVX2-NEXT: movq %r8, %rsi
+; AVX2-NEXT: shlq $63, %rsi
+; AVX2-NEXT: orq %rcx, %rsi
+; AVX2-NEXT: movq %r9, %rcx
+; AVX2-NEXT: shlq $63, %rcx
+; AVX2-NEXT: shrq %r8
+; AVX2-NEXT: orq %rcx, %r8
+; AVX2-NEXT: shrq %r9
+; AVX2-NEXT: movq %r10, %rcx
+; AVX2-NEXT: shlq $63, %rcx
+; AVX2-NEXT: orq %r9, %rcx
+; AVX2-NEXT: movq %rbx, %r9
+; AVX2-NEXT: shlq $63, %r9
+; AVX2-NEXT: shrq %r10
+; AVX2-NEXT: orq %r9, %r10
+; AVX2-NEXT: shrq %rbx
+; AVX2-NEXT: movq %r11, %r9
+; AVX2-NEXT: shlq $63, %r9
+; AVX2-NEXT: orq %rbx, %r9
+; AVX2-NEXT: shrq %r11
+; AVX2-NEXT: movq %r11, 56(%rax)
+; AVX2-NEXT: movq %r9, 48(%rax)
+; AVX2-NEXT: movq %r10, 40(%rax)
+; AVX2-NEXT: movq %rcx, 32(%rax)
+; AVX2-NEXT: movq %r8, 24(%rax)
+; AVX2-NEXT: movq %rsi, 16(%rax)
+; AVX2-NEXT: movq %rdx, 8(%rax)
+; AVX2-NEXT: movq %rdi, (%rax)
+; AVX2-NEXT: popq %rbx
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: lshr_i512_1:
+; AVX512: # %bb.0:
+; AVX512-NEXT: pushq %rbx
+; AVX512-NEXT: movq %rdi, %rax
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX512-NEXT: shrq %rsi
+; AVX512-NEXT: movq %rdx, %rdi
+; AVX512-NEXT: shlq $63, %rdi
+; AVX512-NEXT: orq %rsi, %rdi
+; AVX512-NEXT: movq %rcx, %rsi
+; AVX512-NEXT: shlq $63, %rsi
+; AVX512-NEXT: shrq %rdx
+; AVX512-NEXT: orq %rsi, %rdx
+; AVX512-NEXT: shrq %rcx
+; AVX512-NEXT: movq %r8, %rsi
+; AVX512-NEXT: shlq $63, %rsi
+; AVX512-NEXT: orq %rcx, %rsi
+; AVX512-NEXT: movq %r9, %rcx
+; AVX512-NEXT: shlq $63, %rcx
+; AVX512-NEXT: shrq %r8
+; AVX512-NEXT: orq %rcx, %r8
+; AVX512-NEXT: shrq %r9
+; AVX512-NEXT: movq %r10, %rcx
+; AVX512-NEXT: shlq $63, %rcx
+; AVX512-NEXT: orq %r9, %rcx
+; AVX512-NEXT: movq %rbx, %r9
+; AVX512-NEXT: shlq $63, %r9
+; AVX512-NEXT: shrq %r10
+; AVX512-NEXT: orq %r9, %r10
+; AVX512-NEXT: shrq %rbx
+; AVX512-NEXT: movq %r11, %r9
+; AVX512-NEXT: shlq $63, %r9
+; AVX512-NEXT: orq %rbx, %r9
+; AVX512-NEXT: shrq %r11
+; AVX512-NEXT: movq %r11, 56(%rax)
+; AVX512-NEXT: movq %r9, 48(%rax)
+; AVX512-NEXT: movq %r10, 40(%rax)
+; AVX512-NEXT: movq %rcx, 32(%rax)
+; AVX512-NEXT: movq %r8, 24(%rax)
+; AVX512-NEXT: movq %rsi, 16(%rax)
+; AVX512-NEXT: movq %rdx, 8(%rax)
+; AVX512-NEXT: movq %rdi, (%rax)
+; AVX512-NEXT: popq %rbx
+; AVX512-NEXT: retq
%r = lshr i512 %a0, 1
ret i512 %r
}
define i512 @ashr_i512_1(i512 %a0) nounwind {
-; CHECK-LABEL: ashr_i512_1:
-; CHECK: # %bb.0:
-; CHECK-NEXT: pushq %rbx
-; CHECK-NEXT: movq %rdi, %rax
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rbx
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r11
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r10
-; CHECK-NEXT: shrq %rsi
-; CHECK-NEXT: movq %rdx, %rdi
-; CHECK-NEXT: shlq $63, %rdi
-; CHECK-NEXT: orq %rsi, %rdi
-; CHECK-NEXT: movq %rcx, %rsi
-; CHECK-NEXT: shlq $63, %rsi
-; CHECK-NEXT: shrq %rdx
-; CHECK-NEXT: orq %rsi, %rdx
-; CHECK-NEXT: shrq %rcx
-; CHECK-NEXT: movq %r8, %rsi
-; CHECK-NEXT: shlq $63, %rsi
-; CHECK-NEXT: orq %rcx, %rsi
-; CHECK-NEXT: movq %r9, %rcx
-; CHECK-NEXT: shlq $63, %rcx
-; CHECK-NEXT: shrq %r8
-; CHECK-NEXT: orq %rcx, %r8
-; CHECK-NEXT: shrq %r9
-; CHECK-NEXT: movq %r10, %rcx
-; CHECK-NEXT: shlq $63, %rcx
-; CHECK-NEXT: orq %r9, %rcx
-; CHECK-NEXT: movq %rbx, %r9
-; CHECK-NEXT: shlq $63, %r9
-; CHECK-NEXT: shrq %r10
-; CHECK-NEXT: orq %r9, %r10
-; CHECK-NEXT: shrq %rbx
-; CHECK-NEXT: movq %r11, %r9
-; CHECK-NEXT: shlq $63, %r9
-; CHECK-NEXT: orq %rbx, %r9
-; CHECK-NEXT: sarq %r11
-; CHECK-NEXT: movq %r11, 56(%rax)
-; CHECK-NEXT: movq %r9, 48(%rax)
-; CHECK-NEXT: movq %r10, 40(%rax)
-; CHECK-NEXT: movq %rcx, 32(%rax)
-; CHECK-NEXT: movq %r8, 24(%rax)
-; CHECK-NEXT: movq %rsi, 16(%rax)
-; CHECK-NEXT: movq %rdx, 8(%rax)
-; CHECK-NEXT: movq %rdi, (%rax)
-; CHECK-NEXT: popq %rbx
-; CHECK-NEXT: retq
+; SSE-LABEL: ashr_i512_1:
+; SSE: # %bb.0:
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rdi
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; SSE-NEXT: shrdq $1, %rdx, %rsi
+; SSE-NEXT: shrdq $1, %rcx, %rdx
+; SSE-NEXT: shrdq $1, %r8, %rcx
+; SSE-NEXT: shrdq $1, %r9, %r8
+; SSE-NEXT: shrdq $1, %r11, %r9
+; SSE-NEXT: shrdq $1, %rdi, %r11
+; SSE-NEXT: shrdq $1, %r10, %rdi
+; SSE-NEXT: sarq %r10
+; SSE-NEXT: movq %r10, 56(%rax)
+; SSE-NEXT: movq %rdi, 48(%rax)
+; SSE-NEXT: movq %r11, 40(%rax)
+; SSE-NEXT: movq %r9, 32(%rax)
+; SSE-NEXT: movq %r8, 24(%rax)
+; SSE-NEXT: movq %rcx, 16(%rax)
+; SSE-NEXT: movq %rdx, 8(%rax)
+; SSE-NEXT: movq %rsi, (%rax)
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: ashr_i512_1:
+; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %rbx
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX2-NEXT: shrq %rsi
+; AVX2-NEXT: movq %rdx, %rdi
+; AVX2-NEXT: shlq $63, %rdi
+; AVX2-NEXT: orq %rsi, %rdi
+; AVX2-NEXT: movq %rcx, %rsi
+; AVX2-NEXT: shlq $63, %rsi
+; AVX2-NEXT: shrq %rdx
+; AVX2-NEXT: orq %rsi, %rdx
+; AVX2-NEXT: shrq %rcx
+; AVX2-NEXT: movq %r8, %rsi
+; AVX2-NEXT: shlq $63, %rsi
+; AVX2-NEXT: orq %rcx, %rsi
+; AVX2-NEXT: movq %r9, %rcx
+; AVX2-NEXT: shlq $63, %rcx
+; AVX2-NEXT: shrq %r8
+; AVX2-NEXT: orq %rcx, %r8
+; AVX2-NEXT: shrq %r9
+; AVX2-NEXT: movq %r10, %rcx
+; AVX2-NEXT: shlq $63, %rcx
+; AVX2-NEXT: orq %r9, %rcx
+; AVX2-NEXT: movq %rbx, %r9
+; AVX2-NEXT: shlq $63, %r9
+; AVX2-NEXT: shrq %r10
+; AVX2-NEXT: orq %r9, %r10
+; AVX2-NEXT: shrq %rbx
+; AVX2-NEXT: movq %r11, %r9
+; AVX2-NEXT: shlq $63, %r9
+; AVX2-NEXT: orq %rbx, %r9
+; AVX2-NEXT: sarq %r11
+; AVX2-NEXT: movq %r11, 56(%rax)
+; AVX2-NEXT: movq %r9, 48(%rax)
+; AVX2-NEXT: movq %r10, 40(%rax)
+; AVX2-NEXT: movq %rcx, 32(%rax)
+; AVX2-NEXT: movq %r8, 24(%rax)
+; AVX2-NEXT: movq %rsi, 16(%rax)
+; AVX2-NEXT: movq %rdx, 8(%rax)
+; AVX2-NEXT: movq %rdi, (%rax)
+; AVX2-NEXT: popq %rbx
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: ashr_i512_1:
+; AVX512: # %bb.0:
+; AVX512-NEXT: pushq %rbx
+; AVX512-NEXT: movq %rdi, %rax
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX512-NEXT: shrq %rsi
+; AVX512-NEXT: movq %rdx, %rdi
+; AVX512-NEXT: shlq $63, %rdi
+; AVX512-NEXT: orq %rsi, %rdi
+; AVX512-NEXT: movq %rcx, %rsi
+; AVX512-NEXT: shlq $63, %rsi
+; AVX512-NEXT: shrq %rdx
+; AVX512-NEXT: orq %rsi, %rdx
+; AVX512-NEXT: shrq %rcx
+; AVX512-NEXT: movq %r8, %rsi
+; AVX512-NEXT: shlq $63, %rsi
+; AVX512-NEXT: orq %rcx, %rsi
+; AVX512-NEXT: movq %r9, %rcx
+; AVX512-NEXT: shlq $63, %rcx
+; AVX512-NEXT: shrq %r8
+; AVX512-NEXT: orq %rcx, %r8
+; AVX512-NEXT: shrq %r9
+; AVX512-NEXT: movq %r10, %rcx
+; AVX512-NEXT: shlq $63, %rcx
+; AVX512-NEXT: orq %r9, %rcx
+; AVX512-NEXT: movq %rbx, %r9
+; AVX512-NEXT: shlq $63, %r9
+; AVX512-NEXT: shrq %r10
+; AVX512-NEXT: orq %r9, %r10
+; AVX512-NEXT: shrq %rbx
+; AVX512-NEXT: movq %r11, %r9
+; AVX512-NEXT: shlq $63, %r9
+; AVX512-NEXT: orq %rbx, %r9
+; AVX512-NEXT: sarq %r11
+; AVX512-NEXT: movq %r11, 56(%rax)
+; AVX512-NEXT: movq %r9, 48(%rax)
+; AVX512-NEXT: movq %r10, 40(%rax)
+; AVX512-NEXT: movq %rcx, 32(%rax)
+; AVX512-NEXT: movq %r8, 24(%rax)
+; AVX512-NEXT: movq %rsi, 16(%rax)
+; AVX512-NEXT: movq %rdx, 8(%rax)
+; AVX512-NEXT: movq %rdi, (%rax)
+; AVX512-NEXT: popq %rbx
+; AVX512-NEXT: retq
%r = ashr i512 %a0, 1
ret i512 %r
}
@@ -1409,26 +1618,15 @@ define i512 @shl_i512_200(i512 %a0) nounwind {
; SSE: # %bb.0:
; SSE-NEXT: movq %rdi, %rax
; SSE-NEXT: movq %rsi, %rdi
-; SSE-NEXT: shrq $56, %rdi
-; SSE-NEXT: movq %rdx, %r10
-; SSE-NEXT: shlq $8, %r10
-; SSE-NEXT: orq %rdi, %r10
-; SSE-NEXT: movq %rcx, %rdi
-; SSE-NEXT: shlq $8, %rdi
-; SSE-NEXT: shrq $56, %rdx
-; SSE-NEXT: orq %rdi, %rdx
-; SSE-NEXT: shrq $56, %rcx
-; SSE-NEXT: movq %r8, %rdi
-; SSE-NEXT: shlq $8, %rdi
-; SSE-NEXT: orq %rcx, %rdi
-; SSE-NEXT: shlq $8, %r9
-; SSE-NEXT: shrq $56, %r8
-; SSE-NEXT: orq %r9, %r8
+; SSE-NEXT: shrdq $56, %rdx, %rdi
+; SSE-NEXT: shrdq $56, %rcx, %rdx
+; SSE-NEXT: shrdq $56, %r8, %rcx
+; SSE-NEXT: shldq $8, %r8, %r9
; SSE-NEXT: shlq $8, %rsi
-; SSE-NEXT: movq %r8, 56(%rax)
-; SSE-NEXT: movq %rdi, 48(%rax)
+; SSE-NEXT: movq %r9, 56(%rax)
+; SSE-NEXT: movq %rcx, 48(%rax)
; SSE-NEXT: movq %rdx, 40(%rax)
-; SSE-NEXT: movq %r10, 32(%rax)
+; SSE-NEXT: movq %rdi, 32(%rax)
; SSE-NEXT: movq %rsi, 24(%rax)
; SSE-NEXT: xorps %xmm0, %xmm0
; SSE-NEXT: movaps %xmm0, (%rax)
@@ -1503,33 +1701,21 @@ define i512 @lshr_i512_200(i512 %a0) nounwind {
; SSE: # %bb.0:
; SSE-NEXT: movq %rdi, %rax
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rcx
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rsi
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rdx
-; SSE-NEXT: movq %r9, %rdi
-; SSE-NEXT: shlq $56, %rdi
-; SSE-NEXT: shrq $8, %r8
-; SSE-NEXT: orq %rdi, %r8
-; SSE-NEXT: movq %rdx, %rdi
-; SSE-NEXT: shlq $56, %rdi
-; SSE-NEXT: shrq $8, %r9
-; SSE-NEXT: orq %rdi, %r9
-; SSE-NEXT: movq %rcx, %rdi
-; SSE-NEXT: shlq $56, %rdi
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rsi
+; SSE-NEXT: shrdq $8, %r9, %r8
+; SSE-NEXT: shrdq $8, %rsi, %r9
+; SSE-NEXT: shrdq $8, %rcx, %rsi
+; SSE-NEXT: shrdq $8, %rdx, %rcx
; SSE-NEXT: shrq $8, %rdx
-; SSE-NEXT: orq %rdi, %rdx
-; SSE-NEXT: movq %rsi, %rdi
-; SSE-NEXT: shlq $56, %rdi
-; SSE-NEXT: shrq $8, %rcx
-; SSE-NEXT: orq %rdi, %rcx
-; SSE-NEXT: shrq $8, %rsi
; SSE-NEXT: xorps %xmm0, %xmm0
-; SSE-NEXT: movups %xmm0, 40(%rax)
-; SSE-NEXT: movq %rsi, 32(%rax)
-; SSE-NEXT: movq %rcx, 24(%rax)
-; SSE-NEXT: movq %rdx, 16(%rax)
-; SSE-NEXT: movq %r9, 8(%rax)
-; SSE-NEXT: movq %r8, (%rax)
-; SSE-NEXT: movq $0, 56(%rax)
+; SSE-NEXT: movups %xmm0, 40(%rdi)
+; SSE-NEXT: movq %rdx, 32(%rdi)
+; SSE-NEXT: movq %rcx, 24(%rdi)
+; SSE-NEXT: movq %rsi, 16(%rdi)
+; SSE-NEXT: movq %r9, 8(%rdi)
+; SSE-NEXT: movq %r8, (%rdi)
+; SSE-NEXT: movq $0, 56(%rdi)
; SSE-NEXT: retq
;
; AVX2-LABEL: lshr_i512_200:
@@ -1602,40 +1788,98 @@ define i512 @lshr_i512_200(i512 %a0) nounwind {
}
define i512 @ashr_i512_200(i512 %a0) nounwind {
-; CHECK-LABEL: ashr_i512_200:
-; CHECK: # %bb.0:
-; CHECK-NEXT: movq %rdi, %rax
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rcx
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rsi
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rdx
-; CHECK-NEXT: movq %r9, %rdi
-; CHECK-NEXT: shlq $56, %rdi
-; CHECK-NEXT: shrq $8, %r8
-; CHECK-NEXT: orq %rdi, %r8
-; CHECK-NEXT: movq %rdx, %rdi
-; CHECK-NEXT: shlq $56, %rdi
-; CHECK-NEXT: shrq $8, %r9
-; CHECK-NEXT: orq %rdi, %r9
-; CHECK-NEXT: movq %rcx, %rdi
-; CHECK-NEXT: shlq $56, %rdi
-; CHECK-NEXT: shrq $8, %rdx
-; CHECK-NEXT: orq %rdi, %rdx
-; CHECK-NEXT: movq %rsi, %rdi
-; CHECK-NEXT: shlq $56, %rdi
-; CHECK-NEXT: shrq $8, %rcx
-; CHECK-NEXT: orq %rdi, %rcx
-; CHECK-NEXT: movq %rsi, %rdi
-; CHECK-NEXT: sarq $8, %rdi
-; CHECK-NEXT: sarq $63, %rsi
-; CHECK-NEXT: movq %rsi, 56(%rax)
-; CHECK-NEXT: movq %rsi, 48(%rax)
-; CHECK-NEXT: movq %rsi, 40(%rax)
-; CHECK-NEXT: movq %rdi, 32(%rax)
-; CHECK-NEXT: movq %rcx, 24(%rax)
-; CHECK-NEXT: movq %rdx, 16(%rax)
-; CHECK-NEXT: movq %r9, 8(%rax)
-; CHECK-NEXT: movq %r8, (%rax)
-; CHECK-NEXT: retq
+; SSE-LABEL: ashr_i512_200:
+; SSE: # %bb.0:
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rcx
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rsi
+; SSE-NEXT: shrdq $8, %r9, %r8
+; SSE-NEXT: shrdq $8, %rsi, %r9
+; SSE-NEXT: shrdq $8, %rcx, %rsi
+; SSE-NEXT: shrdq $8, %rdx, %rcx
+; SSE-NEXT: movq %rdx, %rdi
+; SSE-NEXT: sarq $8, %rdi
+; SSE-NEXT: sarq $63, %rdx
+; SSE-NEXT: movq %rdx, 56(%rax)
+; SSE-NEXT: movq %rdx, 48(%rax)
+; SSE-NEXT: movq %rdx, 40(%rax)
+; SSE-NEXT: movq %rdi, 32(%rax)
+; SSE-NEXT: movq %rcx, 24(%rax)
+; SSE-NEXT: movq %rsi, 16(%rax)
+; SSE-NEXT: movq %r9, 8(%rax)
+; SSE-NEXT: movq %r8, (%rax)
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: ashr_i512_200:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rcx
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rsi
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; AVX2-NEXT: movq %r9, %rdi
+; AVX2-NEXT: shlq $56, %rdi
+; AVX2-NEXT: shrq $8, %r8
+; AVX2-NEXT: orq %rdi, %r8
+; AVX2-NEXT: movq %rdx, %rdi
+; AVX2-NEXT: shlq $56, %rdi
+; AVX2-NEXT: shrq $8, %r9
+; AVX2-NEXT: orq %rdi, %r9
+; AVX2-NEXT: movq %rcx, %rdi
+; AVX2-NEXT: shlq $56, %rdi
+; AVX2-NEXT: shrq $8, %rdx
+; AVX2-NEXT: orq %rdi, %rdx
+; AVX2-NEXT: movq %rsi, %rdi
+; AVX2-NEXT: shlq $56, %rdi
+; AVX2-NEXT: shrq $8, %rcx
+; AVX2-NEXT: orq %rdi, %rcx
+; AVX2-NEXT: movq %rsi, %rdi
+; AVX2-NEXT: sarq $8, %rdi
+; AVX2-NEXT: sarq $63, %rsi
+; AVX2-NEXT: movq %rsi, 56(%rax)
+; AVX2-NEXT: movq %rsi, 48(%rax)
+; AVX2-NEXT: movq %rsi, 40(%rax)
+; AVX2-NEXT: movq %rdi, 32(%rax)
+; AVX2-NEXT: movq %rcx, 24(%rax)
+; AVX2-NEXT: movq %rdx, 16(%rax)
+; AVX2-NEXT: movq %r9, 8(%rax)
+; AVX2-NEXT: movq %r8, (%rax)
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: ashr_i512_200:
+; AVX512: # %bb.0:
+; AVX512-NEXT: movq %rdi, %rax
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rcx
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rsi
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; AVX512-NEXT: movq %r9, %rdi
+; AVX512-NEXT: shlq $56, %rdi
+; AVX512-NEXT: shrq $8, %r8
+; AVX512-NEXT: orq %rdi, %r8
+; AVX512-NEXT: movq %rdx, %rdi
+; AVX512-NEXT: shlq $56, %rdi
+; AVX512-NEXT: shrq $8, %r9
+; AVX512-NEXT: orq %rdi, %r9
+; AVX512-NEXT: movq %rcx, %rdi
+; AVX512-NEXT: shlq $56, %rdi
+; AVX512-NEXT: shrq $8, %rdx
+; AVX512-NEXT: orq %rdi, %rdx
+; AVX512-NEXT: movq %rsi, %rdi
+; AVX512-NEXT: shlq $56, %rdi
+; AVX512-NEXT: shrq $8, %rcx
+; AVX512-NEXT: orq %rdi, %rcx
+; AVX512-NEXT: movq %rsi, %rdi
+; AVX512-NEXT: sarq $8, %rdi
+; AVX512-NEXT: sarq $63, %rsi
+; AVX512-NEXT: movq %rsi, 56(%rax)
+; AVX512-NEXT: movq %rsi, 48(%rax)
+; AVX512-NEXT: movq %rsi, 40(%rax)
+; AVX512-NEXT: movq %rdi, 32(%rax)
+; AVX512-NEXT: movq %rcx, 24(%rax)
+; AVX512-NEXT: movq %rdx, 16(%rax)
+; AVX512-NEXT: movq %r9, 8(%rax)
+; AVX512-NEXT: movq %r8, (%rax)
+; AVX512-NEXT: retq
%r = ashr i512 %a0, 200
ret i512 %r
}
diff --git a/llvm/test/CodeGen/X86/shift-mask.ll b/llvm/test/CodeGen/X86/shift-mask.ll
index 3d4f7f1d48575..604f5c19b92e5 100644
--- a/llvm/test/CodeGen/X86/shift-mask.ll
+++ b/llvm/test/CodeGen/X86/shift-mask.ll
@@ -259,11 +259,8 @@ define i64 @test_i64_shl_lshr_1(i64 %a0) {
; X86-LABEL: test_i64_shl_lshr_1:
; X86: # %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: shll $2, %ecx
-; X86-NEXT: movl %eax, %edx
-; X86-NEXT: shrl $30, %edx
-; X86-NEXT: orl %ecx, %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: shldl $2, %eax, %edx
; X86-NEXT: shll $2, %eax
; X86-NEXT: andl $-32, %eax
; X86-NEXT: retl
@@ -587,12 +584,9 @@ define i64 @test_i64_lshr_lshr_0(i64 %a0) {
define i64 @test_i64_lshr_lshr_1(i64 %a0) {
; X86-LABEL: test_i64_lshr_lshr_1:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: shrl $2, %ecx
-; X86-NEXT: movl %edx, %eax
-; X86-NEXT: shll $30, %eax
-; X86-NEXT: orl %ecx, %eax
+; X86-NEXT: shrdl $2, %edx, %eax
; X86-NEXT: shrl $2, %edx
; X86-NEXT: andl $134217727, %edx # imm = 0x7FFFFFF
; X86-NEXT: retl
diff --git a/llvm/test/CodeGen/X86/smul_fix.ll b/llvm/test/CodeGen/X86/smul_fix.ll
index aeee992dc7937..8cb032776114b 100644
--- a/llvm/test/CodeGen/X86/smul_fix.ll
+++ b/llvm/test/CodeGen/X86/smul_fix.ll
@@ -80,9 +80,7 @@ define i64 @func2(i64 %x, i64 %y) {
; X86-NEXT: testl %ebx, %ebx
; X86-NEXT: cmovsl %ecx, %edx
; X86-NEXT: shldl $30, %eax, %edx
-; X86-NEXT: shrl $2, %esi
-; X86-NEXT: shll $30, %eax
-; X86-NEXT: orl %esi, %eax
+; X86-NEXT: shldl $30, %esi, %eax
; X86-NEXT: popl %esi
; X86-NEXT: .cfi_def_cfa_offset 16
; X86-NEXT: popl %edi
@@ -392,22 +390,20 @@ define i64 @func8(i64 %x, i64 %y) nounwind {
; X86-NEXT: adcl $0, %ebx
; X86-NEXT: movl %edi, %ecx
; X86-NEXT: subl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ebx, %ebp
-; X86-NEXT: sbbl $0, %ebp
-; X86-NEXT: testl %esi, %esi
-; X86-NEXT: cmovnsl %ebx, %ebp
+; X86-NEXT: movl %ebx, %esi
+; X86-NEXT: sbbl $0, %esi
+; X86-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: cmovnsl %ebx, %esi
; X86-NEXT: cmovnsl %edi, %ecx
-; X86-NEXT: movl %ecx, %esi
-; X86-NEXT: subl {{[0-9]+}}(%esp), %esi
-; X86-NEXT: movl %ebp, %edx
+; X86-NEXT: movl %ecx, %edi
+; X86-NEXT: subl {{[0-9]+}}(%esp), %edi
+; X86-NEXT: movl %esi, %edx
; X86-NEXT: sbbl $0, %edx
; X86-NEXT: cmpl $0, {{[0-9]+}}(%esp)
-; X86-NEXT: cmovnsl %ebp, %edx
-; X86-NEXT: cmovnsl %ecx, %esi
-; X86-NEXT: shldl $1, %esi, %edx
-; X86-NEXT: shrl $31, %eax
-; X86-NEXT: shll %esi
-; X86-NEXT: orl %esi, %eax
+; X86-NEXT: cmovnsl %esi, %edx
+; X86-NEXT: cmovnsl %ecx, %edi
+; X86-NEXT: shldl $1, %edi, %edx
+; X86-NEXT: shrdl $31, %edi, %eax
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
; X86-NEXT: popl %ebx
diff --git a/llvm/test/CodeGen/X86/smul_fix_sat.ll b/llvm/test/CodeGen/X86/smul_fix_sat.ll
index 82143cfbbbdd0..e68b6e328b723 100644
--- a/llvm/test/CodeGen/X86/smul_fix_sat.ll
+++ b/llvm/test/CodeGen/X86/smul_fix_sat.ll
@@ -60,77 +60,73 @@ define i64 @func2(i64 %x, i64 %y) nounwind {
; X86-NEXT: pushl %ebx
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
-; X86-NEXT: pushl %eax
+; X86-NEXT: subl $8, %esp
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx
; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: mull %esi
; X86-NEXT: movl %edx, %edi
; X86-NEXT: movl %eax, %ebp
; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: mull {{[0-9]+}}(%esp)
-; X86-NEXT: movl %eax, (%esp) # 4-byte Spill
+; X86-NEXT: mull %ebx
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: addl %edx, %ebp
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: adcl $0, %edi
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: imull %esi
; X86-NEXT: movl %edx, %esi
-; X86-NEXT: movl %eax, %ebx
+; X86-NEXT: movl %eax, (%esp) # 4-byte Spill
; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: mull {{[0-9]+}}(%esp)
-; X86-NEXT: addl %ebp, %eax
+; X86-NEXT: mull %ebx
+; X86-NEXT: movl %eax, %ebx
+; X86-NEXT: addl %ebp, %ebx
; X86-NEXT: adcl %edi, %edx
; X86-NEXT: adcl $0, %esi
-; X86-NEXT: addl %ebx, %edx
+; X86-NEXT: addl (%esp), %edx # 4-byte Folded Reload
; X86-NEXT: adcl $0, %esi
-; X86-NEXT: movl %edx, %ebx
-; X86-NEXT: subl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT: movl %edx, %edi
+; X86-NEXT: subl {{[0-9]+}}(%esp), %edi
; X86-NEXT: movl %esi, %ebp
; X86-NEXT: sbbl $0, %ebp
; X86-NEXT: testl %ecx, %ecx
; X86-NEXT: cmovnsl %esi, %ebp
-; X86-NEXT: cmovnsl %edx, %ebx
-; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: cmovnsl %edx, %edi
+; X86-NEXT: movl %edi, %ecx
; X86-NEXT: subl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ebp, %edi
-; X86-NEXT: sbbl $0, %edi
+; X86-NEXT: movl %ebp, %edx
+; X86-NEXT: sbbl $0, %edx
; X86-NEXT: cmpl $0, {{[0-9]+}}(%esp)
-; X86-NEXT: cmovnsl %ebp, %edi
-; X86-NEXT: cmovnsl %ebx, %ecx
-; X86-NEXT: testl %edi, %edi
-; X86-NEXT: setg %bl
-; X86-NEXT: sete %bh
+; X86-NEXT: cmovnsl %ebp, %edx
+; X86-NEXT: cmovnsl %edi, %ecx
+; X86-NEXT: testl %edx, %edx
+; X86-NEXT: setg %ah
+; X86-NEXT: sete (%esp) # 1-byte Folded Spill
; X86-NEXT: cmpl $2, %ecx
-; X86-NEXT: setae %dl
-; X86-NEXT: movl %eax, %esi
-; X86-NEXT: shrl $2, %esi
-; X86-NEXT: movl %ecx, %ebp
-; X86-NEXT: shll $30, %ebp
-; X86-NEXT: orl %esi, %ebp
-; X86-NEXT: movl (%esp), %esi # 4-byte Reload
-; X86-NEXT: shrl $2, %esi
-; X86-NEXT: shll $30, %eax
-; X86-NEXT: orl %esi, %eax
-; X86-NEXT: andb %bh, %dl
-; X86-NEXT: orb %bl, %dl
-; X86-NEXT: testb %dl, %dl
+; X86-NEXT: setae %al
+; X86-NEXT: andb (%esp), %al # 1-byte Folded Reload
+; X86-NEXT: orb %ah, %al
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X86-NEXT: shrdl $2, %ebx, %ebp
+; X86-NEXT: shrdl $2, %ecx, %ebx
+; X86-NEXT: testb %al, %al
; X86-NEXT: movl $2147483647, %esi # imm = 0x7FFFFFFF
-; X86-NEXT: cmovel %ebp, %esi
-; X86-NEXT: movl $-1, %edx
-; X86-NEXT: cmovnel %edx, %eax
-; X86-NEXT: cmpl $-1, %edi
+; X86-NEXT: cmovel %ebx, %esi
+; X86-NEXT: movl $-1, %edi
+; X86-NEXT: cmovel %ebp, %edi
+; X86-NEXT: cmpl $-1, %edx
; X86-NEXT: setl %dl
-; X86-NEXT: sete %dh
+; X86-NEXT: sete %al
; X86-NEXT: cmpl $-2, %ecx
; X86-NEXT: setb %cl
-; X86-NEXT: andb %dh, %cl
-; X86-NEXT: xorl %edi, %edi
+; X86-NEXT: andb %al, %cl
+; X86-NEXT: xorl %eax, %eax
; X86-NEXT: orb %dl, %cl
-; X86-NEXT: cmovnel %edi, %eax
+; X86-NEXT: cmovel %edi, %eax
; X86-NEXT: movl $-2147483648, %edx # imm = 0x80000000
; X86-NEXT: cmovel %esi, %edx
-; X86-NEXT: addl $4, %esp
+; X86-NEXT: addl $8, %esp
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
; X86-NEXT: popl %ebx
@@ -237,15 +233,12 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
; X64-NEXT: imulq %rdx, %rsi
; X64-NEXT: movq %rsi, %rdx
; X64-NEXT: shrq $32, %rdx
-; X64-NEXT: shrl $2, %esi
-; X64-NEXT: movl %edx, %edi
-; X64-NEXT: shll $30, %edi
-; X64-NEXT: orl %esi, %edi
+; X64-NEXT: shrdl $2, %edx, %esi
; X64-NEXT: cmpl $2, %edx
-; X64-NEXT: cmovgel %eax, %edi
+; X64-NEXT: cmovgel %eax, %esi
; X64-NEXT: cmpl $-2, %edx
-; X64-NEXT: cmovll %ecx, %edi
-; X64-NEXT: movd %edi, %xmm2
+; X64-NEXT: cmovll %ecx, %esi
+; X64-NEXT: movd %esi, %xmm2
; X64-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
; X64-NEXT: movd %xmm1, %edx
; X64-NEXT: movslq %edx, %rdx
@@ -744,21 +737,18 @@ define i64 @func8(i64 %x, i64 %y) nounwind {
; X86-NEXT: cmpl $0, {{[0-9]+}}(%esp)
; X86-NEXT: cmovnsl %ebp, %ecx
; X86-NEXT: cmovnsl %edx, %esi
-; X86-NEXT: movl %esi, %edx
-; X86-NEXT: shrdl $31, %ecx, %edx
-; X86-NEXT: shrl $31, %eax
-; X86-NEXT: shll %esi
-; X86-NEXT: orl %esi, %eax
+; X86-NEXT: shrdl $31, %esi, %eax
+; X86-NEXT: shrdl $31, %ecx, %esi
; X86-NEXT: cmpl $1073741824, %ecx # imm = 0x40000000
-; X86-NEXT: movl $2147483647, %esi # imm = 0x7FFFFFFF
-; X86-NEXT: cmovll %edx, %esi
+; X86-NEXT: movl $2147483647, %edi # imm = 0x7FFFFFFF
+; X86-NEXT: cmovll %esi, %edi
; X86-NEXT: movl $-1, %edx
; X86-NEXT: cmovgel %edx, %eax
; X86-NEXT: xorl %edx, %edx
; X86-NEXT: cmpl $-1073741824, %ecx # imm = 0xC0000000
; X86-NEXT: cmovll %edx, %eax
; X86-NEXT: movl $-2147483648, %edx # imm = 0x80000000
-; X86-NEXT: cmovgel %esi, %edx
+; X86-NEXT: cmovgel %edi, %edx
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
; X86-NEXT: popl %ebx
diff --git a/llvm/test/CodeGen/X86/udiv_fix_sat.ll b/llvm/test/CodeGen/X86/udiv_fix_sat.ll
index 5b432a2374f76..d1e9145c4ccee 100644
--- a/llvm/test/CodeGen/X86/udiv_fix_sat.ll
+++ b/llvm/test/CodeGen/X86/udiv_fix_sat.ll
@@ -204,11 +204,8 @@ define i64 @func5(i64 %x, i64 %y) nounwind {
; X86-NEXT: movl %ecx, %edx
; X86-NEXT: shrl %edx
; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NEXT: shll $31, %ecx
-; X86-NEXT: movl %eax, %edx
-; X86-NEXT: shrl %edx
-; X86-NEXT: orl %ecx, %edx
-; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NEXT: shldl $31, %eax, %ecx
+; X86-NEXT: movl %ecx, {{[0-9]+}}(%esp)
; X86-NEXT: shll $31, %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: leal {{[0-9]+}}(%esp), %eax
@@ -387,9 +384,9 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
; X86-NEXT: subl $8, %esp
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ebp
; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx
; X86-NEXT: pushl $0
; X86-NEXT: pushl {{[0-9]+}}(%esp)
@@ -401,41 +398,36 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
; X86-NEXT: movl %edx, (%esp) # 4-byte Spill
; X86-NEXT: pushl $0
; X86-NEXT: pushl %ebx
-; X86-NEXT: pushl %esi
+; X86-NEXT: pushl %edi
; X86-NEXT: pushl $0
; X86-NEXT: calll __udivdi3
; X86-NEXT: addl $16, %esp
-; X86-NEXT: movl %eax, %esi
-; X86-NEXT: movl %edx, %ebx
+; X86-NEXT: movl %eax, %ebx
+; X86-NEXT: movl %edx, %edi
; X86-NEXT: pushl $0
; X86-NEXT: pushl %ebp
-; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %esi
; X86-NEXT: pushl $0
; X86-NEXT: calll __udivdi3
; X86-NEXT: addl $16, %esp
-; X86-NEXT: movl %edx, %ebp
+; X86-NEXT: movl %eax, %esi
; X86-NEXT: cmpl $2, %edx
-; X86-NEXT: movl $1, %edi
-; X86-NEXT: cmovael %edi, %ebp
+; X86-NEXT: movl $1, %ebp
+; X86-NEXT: cmovael %ebp, %edx
+; X86-NEXT: movl $-1, %eax
+; X86-NEXT: cmovael %eax, %esi
+; X86-NEXT: shrdl $1, %edx, %esi
+; X86-NEXT: cmpl $2, %edi
+; X86-NEXT: cmovael %ebp, %edi
; X86-NEXT: movl $-1, %ecx
-; X86-NEXT: cmovael %ecx, %eax
-; X86-NEXT: shrl %eax
-; X86-NEXT: shll $31, %ebp
-; X86-NEXT: orl %eax, %ebp
-; X86-NEXT: cmpl $2, %ebx
-; X86-NEXT: cmovael %edi, %ebx
-; X86-NEXT: cmovael %ecx, %esi
-; X86-NEXT: shrl %esi
-; X86-NEXT: shll $31, %ebx
-; X86-NEXT: orl %esi, %ebx
-; X86-NEXT: movl (%esp), %esi # 4-byte Reload
-; X86-NEXT: cmpl $2, %esi
-; X86-NEXT: cmovael %edi, %esi
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: cmovael %ecx, %eax
-; X86-NEXT: shrl %eax
-; X86-NEXT: shll $31, %esi
-; X86-NEXT: orl %eax, %esi
+; X86-NEXT: cmovael %ecx, %ebx
+; X86-NEXT: shrdl $1, %edi, %ebx
+; X86-NEXT: movl (%esp), %eax # 4-byte Reload
+; X86-NEXT: cmpl $2, %eax
+; X86-NEXT: cmovael %ebp, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: cmovael %ecx, %edi
+; X86-NEXT: shrdl $1, %eax, %edi
; X86-NEXT: pushl $0
; X86-NEXT: pushl {{[0-9]+}}(%esp)
; X86-NEXT: pushl {{[0-9]+}}(%esp)
@@ -443,17 +435,16 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
; X86-NEXT: calll __udivdi3
; X86-NEXT: addl $16, %esp
; X86-NEXT: cmpl $2, %edx
-; X86-NEXT: cmovbl %edx, %edi
+; X86-NEXT: cmovbl %edx, %ebp
; X86-NEXT: movl $-1, %ecx
; X86-NEXT: cmovael %ecx, %eax
-; X86-NEXT: shrl %eax
-; X86-NEXT: shll $31, %edi
-; X86-NEXT: orl %eax, %edi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl %edi, 12(%eax)
-; X86-NEXT: movl %esi, 8(%eax)
-; X86-NEXT: movl %ebx, 4(%eax)
-; X86-NEXT: movl %ebp, (%eax)
+; X86-NEXT: shrdl $1, %ebp, %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl %eax, 12(%ecx)
+; X86-NEXT: movl %edi, 8(%ecx)
+; X86-NEXT: movl %ebx, 4(%ecx)
+; X86-NEXT: movl %esi, (%ecx)
+; X86-NEXT: movl %ecx, %eax
; X86-NEXT: addl $8, %esp
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
diff --git a/llvm/test/CodeGen/X86/umul_fix.ll b/llvm/test/CodeGen/X86/umul_fix.ll
index d8b0052d09f72..5a68484596a2f 100644
--- a/llvm/test/CodeGen/X86/umul_fix.ll
+++ b/llvm/test/CodeGen/X86/umul_fix.ll
@@ -62,9 +62,7 @@ define i64 @func2(i64 %x, i64 %y) nounwind {
; X86-NEXT: imull {{[0-9]+}}(%esp), %ecx
; X86-NEXT: addl %ecx, %edx
; X86-NEXT: shldl $30, %eax, %edx
-; X86-NEXT: shrl $2, %esi
-; X86-NEXT: shll $30, %eax
-; X86-NEXT: orl %esi, %eax
+; X86-NEXT: shldl $30, %esi, %eax
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
; X86-NEXT: popl %ebx
@@ -331,9 +329,7 @@ define i64 @func8(i64 %x, i64 %y) nounwind {
; X86-NEXT: addl %ebp, %ecx
; X86-NEXT: adcl $0, %esi
; X86-NEXT: shldl $1, %ecx, %esi
-; X86-NEXT: shrl $31, %eax
-; X86-NEXT: shll %ecx
-; X86-NEXT: orl %ecx, %eax
+; X86-NEXT: shrdl $31, %ecx, %eax
; X86-NEXT: movl %esi, %edx
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
diff --git a/llvm/test/CodeGen/X86/umul_fix_sat.ll b/llvm/test/CodeGen/X86/umul_fix_sat.ll
index 4120c21eaecd9..8c7078c726328 100644
--- a/llvm/test/CodeGen/X86/umul_fix_sat.ll
+++ b/llvm/test/CodeGen/X86/umul_fix_sat.ll
@@ -52,41 +52,35 @@ define i64 @func2(i64 %x, i64 %y) nounwind {
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: mull %esi
-; X86-NEXT: movl %edx, %edi
+; X86-NEXT: mull %edi
+; X86-NEXT: movl %edx, %esi
; X86-NEXT: movl %eax, %ebx
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: mull {{[0-9]+}}(%esp)
; X86-NEXT: movl %eax, %ecx
; X86-NEXT: addl %edx, %ebx
-; X86-NEXT: adcl $0, %edi
+; X86-NEXT: adcl $0, %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: mull %esi
-; X86-NEXT: movl %edx, %esi
+; X86-NEXT: mull %edi
+; X86-NEXT: movl %edx, %edi
; X86-NEXT: movl %eax, %ebp
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: mull {{[0-9]+}}(%esp)
; X86-NEXT: addl %ebx, %eax
-; X86-NEXT: adcl %edi, %edx
-; X86-NEXT: adcl $0, %esi
+; X86-NEXT: adcl %esi, %edx
+; X86-NEXT: adcl $0, %edi
; X86-NEXT: addl %ebp, %edx
-; X86-NEXT: adcl $0, %esi
-; X86-NEXT: movl %eax, %ebx
-; X86-NEXT: shrl $2, %ebx
-; X86-NEXT: movl %edx, %edi
-; X86-NEXT: shll $30, %edi
-; X86-NEXT: orl %ebx, %edi
+; X86-NEXT: adcl $0, %edi
+; X86-NEXT: shrdl $2, %eax, %ecx
+; X86-NEXT: shrdl $2, %edx, %eax
; X86-NEXT: shrl $2, %edx
-; X86-NEXT: shrl $2, %ecx
-; X86-NEXT: shll $30, %eax
-; X86-NEXT: orl %ecx, %eax
-; X86-NEXT: orl %esi, %edx
-; X86-NEXT: movl $-1, %ecx
-; X86-NEXT: cmovnel %ecx, %eax
-; X86-NEXT: cmovnel %ecx, %edi
-; X86-NEXT: movl %edi, %edx
+; X86-NEXT: orl %edi, %edx
+; X86-NEXT: movl $-1, %edx
+; X86-NEXT: cmovnel %edx, %ecx
+; X86-NEXT: cmovel %eax, %edx
+; X86-NEXT: movl %ecx, %eax
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
; X86-NEXT: popl %ebx
@@ -173,13 +167,10 @@ define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
; X64-NEXT: imulq %rcx, %rdx
; X64-NEXT: movq %rdx, %rcx
; X64-NEXT: shrq $32, %rcx
-; X64-NEXT: shrl $2, %edx
-; X64-NEXT: movl %ecx, %esi
-; X64-NEXT: shll $30, %esi
-; X64-NEXT: orl %edx, %esi
+; X64-NEXT: shrdl $2, %ecx, %edx
; X64-NEXT: cmpl $4, %ecx
-; X64-NEXT: cmovael %eax, %esi
-; X64-NEXT: movd %esi, %xmm2
+; X64-NEXT: cmovael %eax, %edx
+; X64-NEXT: movd %edx, %xmm2
; X64-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
; X64-NEXT: movd %xmm1, %ecx
; X64-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
@@ -527,9 +518,7 @@ define i64 @func8(i64 %x, i64 %y) nounwind {
; X86-NEXT: addl %ebp, %edx
; X86-NEXT: adcl $0, %ecx
; X86-NEXT: shldl $1, %edx, %ecx
-; X86-NEXT: shrl $31, %eax
-; X86-NEXT: leal (,%edx,2), %esi
-; X86-NEXT: orl %esi, %eax
+; X86-NEXT: shrdl $31, %edx, %eax
; X86-NEXT: testl $-2147483648, %edx # imm = 0x80000000
; X86-NEXT: movl $-1, %edx
; X86-NEXT: cmovnel %edx, %eax
diff --git a/llvm/test/CodeGen/X86/vector-sext.ll b/llvm/test/CodeGen/X86/vector-sext.ll
index 239ff0075ded0..6ce34991050ac 100644
--- a/llvm/test/CodeGen/X86/vector-sext.ll
+++ b/llvm/test/CodeGen/X86/vector-sext.ll
@@ -3649,51 +3649,45 @@ define <4 x i32> @sext_4i17_to_4i32(ptr %ptr) {
; SSE41-LABEL: sext_4i17_to_4i32:
; SSE41: # %bb.0:
; SSE41-NEXT: movq (%rdi), %rax
-; SSE41-NEXT: movq %rax, %rcx
-; SSE41-NEXT: shrq $17, %rcx
-; SSE41-NEXT: shll $15, %ecx
-; SSE41-NEXT: sarl $15, %ecx
; SSE41-NEXT: movd %eax, %xmm0
+; SSE41-NEXT: movq %rax, %rcx
+; SSE41-NEXT: movl 8(%rdi), %edx
+; SSE41-NEXT: shldq $13, %rax, %rdx
+; SSE41-NEXT: shrq $17, %rax
+; SSE41-NEXT: shll $15, %eax
+; SSE41-NEXT: sarl $15, %eax
; SSE41-NEXT: pslld $15, %xmm0
; SSE41-NEXT: psrad $15, %xmm0
-; SSE41-NEXT: pinsrd $1, %ecx, %xmm0
-; SSE41-NEXT: movq %rax, %rcx
+; SSE41-NEXT: pinsrd $1, %eax, %xmm0
; SSE41-NEXT: shrq $34, %rcx
; SSE41-NEXT: shll $15, %ecx
; SSE41-NEXT: sarl $15, %ecx
; SSE41-NEXT: pinsrd $2, %ecx, %xmm0
-; SSE41-NEXT: movl 8(%rdi), %ecx
-; SSE41-NEXT: shrq $51, %rax
-; SSE41-NEXT: shlq $13, %rcx
-; SSE41-NEXT: orq %rax, %rcx
-; SSE41-NEXT: shll $15, %ecx
-; SSE41-NEXT: sarl $15, %ecx
-; SSE41-NEXT: pinsrd $3, %ecx, %xmm0
+; SSE41-NEXT: shll $15, %edx
+; SSE41-NEXT: sarl $15, %edx
+; SSE41-NEXT: pinsrd $3, %edx, %xmm0
; SSE41-NEXT: retq
;
; AVX-LABEL: sext_4i17_to_4i32:
; AVX: # %bb.0:
; AVX-NEXT: movq (%rdi), %rax
-; AVX-NEXT: movq %rax, %rcx
-; AVX-NEXT: shrq $17, %rcx
-; AVX-NEXT: shll $15, %ecx
-; AVX-NEXT: sarl $15, %ecx
; AVX-NEXT: vmovd %eax, %xmm0
+; AVX-NEXT: movq %rax, %rcx
+; AVX-NEXT: movl 8(%rdi), %edx
+; AVX-NEXT: shldq $13, %rax, %rdx
+; AVX-NEXT: shrq $17, %rax
+; AVX-NEXT: shll $15, %eax
+; AVX-NEXT: sarl $15, %eax
; AVX-NEXT: vpslld $15, %xmm0, %xmm0
; AVX-NEXT: vpsrad $15, %xmm0, %xmm0
-; AVX-NEXT: vpinsrd $1, %ecx, %xmm0, %xmm0
-; AVX-NEXT: movq %rax, %rcx
+; AVX-NEXT: vpinsrd $1, %eax, %xmm0, %xmm0
; AVX-NEXT: shrq $34, %rcx
; AVX-NEXT: shll $15, %ecx
; AVX-NEXT: sarl $15, %ecx
; AVX-NEXT: vpinsrd $2, %ecx, %xmm0, %xmm0
-; AVX-NEXT: movl 8(%rdi), %ecx
-; AVX-NEXT: shrq $51, %rax
-; AVX-NEXT: shlq $13, %rcx
-; AVX-NEXT: orq %rax, %rcx
-; AVX-NEXT: shll $15, %ecx
-; AVX-NEXT: sarl $15, %ecx
-; AVX-NEXT: vpinsrd $3, %ecx, %xmm0, %xmm0
+; AVX-NEXT: shll $15, %edx
+; AVX-NEXT: sarl $15, %edx
+; AVX-NEXT: vpinsrd $3, %edx, %xmm0, %xmm0
; AVX-NEXT: retq
;
; X86-SSE2-LABEL: sext_4i17_to_4i32:
@@ -3733,17 +3727,14 @@ define <4 x i32> @sext_4i17_to_4i32(ptr %ptr) {
; X86-SSE41-NEXT: pslld $15, %xmm0
; X86-SSE41-NEXT: psrad $15, %xmm0
; X86-SSE41-NEXT: pinsrd $1, %edx, %xmm0
-; X86-SSE41-NEXT: movl %eax, %edx
-; X86-SSE41-NEXT: shll $13, %edx
-; X86-SSE41-NEXT: sarl $15, %edx
-; X86-SSE41-NEXT: pinsrd $2, %edx, %xmm0
; X86-SSE41-NEXT: movl 8(%ecx), %ecx
-; X86-SSE41-NEXT: shll $13, %ecx
-; X86-SSE41-NEXT: shrl $19, %eax
-; X86-SSE41-NEXT: orl %ecx, %eax
-; X86-SSE41-NEXT: shll $15, %eax
+; X86-SSE41-NEXT: shldl $13, %eax, %ecx
+; X86-SSE41-NEXT: shll $13, %eax
; X86-SSE41-NEXT: sarl $15, %eax
-; X86-SSE41-NEXT: pinsrd $3, %eax, %xmm0
+; X86-SSE41-NEXT: pinsrd $2, %eax, %xmm0
+; X86-SSE41-NEXT: shll $15, %ecx
+; X86-SSE41-NEXT: sarl $15, %ecx
+; X86-SSE41-NEXT: pinsrd $3, %ecx, %xmm0
; X86-SSE41-NEXT: retl
%a = load <4 x i17>, ptr %ptr
%b = sext <4 x i17> %a to <4 x i32>
diff --git a/llvm/test/CodeGen/X86/vector-zext.ll b/llvm/test/CodeGen/X86/vector-zext.ll
index edb00220d8a81..e061e4e35e1e7 100644
--- a/llvm/test/CodeGen/X86/vector-zext.ll
+++ b/llvm/test/CodeGen/X86/vector-zext.ll
@@ -2362,54 +2362,45 @@ define <4 x i32> @zext_4i17_to_4i32(ptr %ptr) {
; SSE41-LABEL: zext_4i17_to_4i32:
; SSE41: # %bb.0:
; SSE41-NEXT: movq (%rdi), %rax
-; SSE41-NEXT: movq %rax, %rcx
-; SSE41-NEXT: shrq $17, %rcx
; SSE41-NEXT: movd %eax, %xmm0
-; SSE41-NEXT: pinsrd $1, %ecx, %xmm0
; SSE41-NEXT: movq %rax, %rcx
+; SSE41-NEXT: movl 8(%rdi), %edx
+; SSE41-NEXT: shldq $13, %rax, %rdx
+; SSE41-NEXT: shrq $17, %rax
+; SSE41-NEXT: pinsrd $1, %eax, %xmm0
; SSE41-NEXT: shrq $34, %rcx
; SSE41-NEXT: pinsrd $2, %ecx, %xmm0
-; SSE41-NEXT: movl 8(%rdi), %ecx
-; SSE41-NEXT: shrq $51, %rax
-; SSE41-NEXT: shlq $13, %rcx
-; SSE41-NEXT: orq %rax, %rcx
-; SSE41-NEXT: pinsrd $3, %ecx, %xmm0
+; SSE41-NEXT: pinsrd $3, %edx, %xmm0
; SSE41-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
; SSE41-NEXT: retq
;
; AVX1-LABEL: zext_4i17_to_4i32:
; AVX1: # %bb.0:
; AVX1-NEXT: movq (%rdi), %rax
-; AVX1-NEXT: movq %rax, %rcx
-; AVX1-NEXT: shrq $17, %rcx
; AVX1-NEXT: vmovd %eax, %xmm0
-; AVX1-NEXT: vpinsrd $1, %ecx, %xmm0, %xmm0
; AVX1-NEXT: movq %rax, %rcx
+; AVX1-NEXT: movl 8(%rdi), %edx
+; AVX1-NEXT: shldq $13, %rax, %rdx
+; AVX1-NEXT: shrq $17, %rax
+; AVX1-NEXT: vpinsrd $1, %eax, %xmm0, %xmm0
; AVX1-NEXT: shrq $34, %rcx
; AVX1-NEXT: vpinsrd $2, %ecx, %xmm0, %xmm0
-; AVX1-NEXT: movl 8(%rdi), %ecx
-; AVX1-NEXT: shrq $51, %rax
-; AVX1-NEXT: shlq $13, %rcx
-; AVX1-NEXT: orq %rax, %rcx
-; AVX1-NEXT: vpinsrd $3, %ecx, %xmm0, %xmm0
+; AVX1-NEXT: vpinsrd $3, %edx, %xmm0, %xmm0
; AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
; AVX1-NEXT: retq
;
; AVX2-LABEL: zext_4i17_to_4i32:
; AVX2: # %bb.0:
; AVX2-NEXT: movq (%rdi), %rax
-; AVX2-NEXT: movq %rax, %rcx
-; AVX2-NEXT: shrq $17, %rcx
; AVX2-NEXT: vmovd %eax, %xmm0
-; AVX2-NEXT: vpinsrd $1, %ecx, %xmm0, %xmm0
; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: movl 8(%rdi), %edx
+; AVX2-NEXT: shldq $13, %rax, %rdx
+; AVX2-NEXT: shrq $17, %rax
+; AVX2-NEXT: vpinsrd $1, %eax, %xmm0, %xmm0
; AVX2-NEXT: shrq $34, %rcx
; AVX2-NEXT: vpinsrd $2, %ecx, %xmm0, %xmm0
-; AVX2-NEXT: movl 8(%rdi), %ecx
-; AVX2-NEXT: shrq $51, %rax
-; AVX2-NEXT: shlq $13, %rcx
-; AVX2-NEXT: orq %rax, %rcx
-; AVX2-NEXT: vpinsrd $3, %ecx, %xmm0, %xmm0
+; AVX2-NEXT: vpinsrd $3, %edx, %xmm0, %xmm0
; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm1 = [131071,131071,131071,131071]
; AVX2-NEXT: vpand %xmm1, %xmm0, %xmm0
; AVX2-NEXT: retq
@@ -2417,18 +2408,15 @@ define <4 x i32> @zext_4i17_to_4i32(ptr %ptr) {
; AVX512-LABEL: zext_4i17_to_4i32:
; AVX512: # %bb.0:
; AVX512-NEXT: movq (%rdi), %rax
-; AVX512-NEXT: movq %rax, %rcx
-; AVX512-NEXT: shrq $17, %rcx
; AVX512-NEXT: vmovd %eax, %xmm0
-; AVX512-NEXT: vpinsrd $1, %ecx, %xmm0, %xmm0
; AVX512-NEXT: movq %rax, %rcx
+; AVX512-NEXT: movl 8(%rdi), %edx
+; AVX512-NEXT: shldq $13, %rax, %rdx
+; AVX512-NEXT: shrq $17, %rax
+; AVX512-NEXT: vpinsrd $1, %eax, %xmm0, %xmm0
; AVX512-NEXT: shrq $34, %rcx
; AVX512-NEXT: vpinsrd $2, %ecx, %xmm0, %xmm0
-; AVX512-NEXT: movl 8(%rdi), %ecx
-; AVX512-NEXT: shrq $51, %rax
-; AVX512-NEXT: shlq $13, %rcx
-; AVX512-NEXT: orq %rax, %rcx
-; AVX512-NEXT: vpinsrd $3, %ecx, %xmm0, %xmm0
+; AVX512-NEXT: vpinsrd $3, %edx, %xmm0, %xmm0
; AVX512-NEXT: vpbroadcastd {{.*#+}} xmm1 = [131071,131071,131071,131071]
; AVX512-NEXT: vpand %xmm1, %xmm0, %xmm0
; AVX512-NEXT: retq
>From 9fa7ec9215062cc5131b2aa7e623d5859a45aa7b Mon Sep 17 00:00:00 2001
From: stomfaig <stomfaig at gmail.com>
Date: Thu, 26 Mar 2026 13:03:00 +0000
Subject: [PATCH 10/11] add lea based patterns and update tests
---
llvm/lib/Target/X86/X86InstrInfo.cpp | 150 ++++-
llvm/lib/Target/X86/X86InstrInfo.h | 3 +-
llvm/test/CodeGen/X86/fshl.ll | 535 +++++++++------
llvm/test/CodeGen/X86/fshr.ll | 435 +++++++-----
llvm/test/CodeGen/X86/load-local-v3i129.ll | 11 +-
llvm/test/CodeGen/X86/pr32282.ll | 6 +-
llvm/test/CodeGen/X86/rot32.ll | 14 +-
llvm/test/CodeGen/X86/rot64.ll | 10 +-
llvm/test/CodeGen/X86/scmp.ll | 98 +--
.../test/CodeGen/X86/shd-machine-combiner.mir | 630 +++++++++++++++---
llvm/test/CodeGen/X86/shift-bmi2.ll | 22 +-
llvm/test/CodeGen/X86/shift-i256.ll | 102 ++-
llvm/test/CodeGen/X86/shift-i512.ll | 532 +++++++--------
...lar-shift-by-byte-multiple-legalization.ll | 48 +-
.../X86/wide-scalar-shift-legalization.ll | 12 +-
...ad-of-small-alloca-with-zero-upper-half.ll | 12 +-
.../CodeGen/X86/widen-load-of-small-alloca.ll | 12 +-
.../X86/x86-64-double-precision-shift-left.ll | 18 +-
.../x86-64-double-precision-shift-right.ll | 15 +-
.../X86/x86-64-double-shifts-Oz-Os-O2.ll | 20 +-
20 files changed, 1727 insertions(+), 958 deletions(-)
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index 3f865acd4add3..33ff43ebe2d34 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -10638,7 +10638,8 @@ void X86InstrInfo::buildClearRegister(Register Reg, MachineBasicBlock &MBB,
CombinerObjective X86InstrInfo::getCombinerObjective(unsigned Pattern) const {
switch (Pattern) {
- case X86MachineCombinerPattern::USHD:
+ case X86MachineCombinerPattern::USHD_OR:
+ case X86MachineCombinerPattern::USHD_LEA:
return CombinerObjective::MustReduceLatency;
default:
return TargetInstrInfo::getCombinerObjective(Pattern);
@@ -10674,27 +10675,27 @@ bool X86InstrInfo::getMachineCombinerPatterns(
}
// We do not support CL variant, as it requires a lot of bookkeeping.
case X86::SHLD32rri8:
- case X86::SHLD32mri8:
case X86::SHLD64rri8:
- case X86::SHLD64mri8:
case X86::SHRD32rri8:
- case X86::SHRD32mri8:
case X86::SHRD64rri8:
- case X86::SHRD64mri8: {
- Patterns.push_back(X86MachineCombinerPattern::USHD);
+ // Only offer LEA variant for register operands
+ Patterns.push_back(X86MachineCombinerPattern::USHD_LEA);
+ [[fallthrough]];
+ case X86::SHLD32mri8:
+ case X86::SHLD64mri8:
+ case X86::SHRD32mri8:
+ case X86::SHRD64mri8:
+ Patterns.push_back(X86MachineCombinerPattern::USHD_OR);
return true;
- break;
- }
}
return TargetInstrInfo::getMachineCombinerPatterns(Root,
Patterns, DoRegPressureReduce);
}
-static void
-genAlternativeShdSequence(MachineInstr &Root, const TargetInstrInfo &TII,
- SmallVectorImpl<MachineInstr *> &InsInstrs,
- SmallVectorImpl<MachineInstr *> &DelInstrs,
- DenseMap<Register, unsigned> &InstrIdxForVirtReg) {
+static void genShdOrSequence(MachineInstr &Root, const TargetInstrInfo &TII,
+ SmallVectorImpl<MachineInstr *> &InsInstrs,
+ SmallVectorImpl<MachineInstr *> &DelInstrs,
+ DenseMap<Register, unsigned> &InstrIdxForVirtReg) {
auto *MF = Root.getMF();
auto OpCode = Root.getOpcode();
@@ -10738,7 +10739,7 @@ genAlternativeShdSequence(MachineInstr &Root, const TargetInstrInfo &TII,
IsMem = true;
break;
default:
- llvm_unreachable("Unexpected opcode in genAlternativeShdSequence");
+ llvm_unreachable("Unexpected opcode in genShdOrSequence");
}
unsigned ShlOpc = (BW == 64) ? X86::SHL64ri : X86::SHL32ri;
@@ -10805,6 +10806,120 @@ genAlternativeShdSequence(MachineInstr &Root, const TargetInstrInfo &TII,
DelInstrs.push_back(&Root);
}
+// Expand SHD op using LEA as the OR node. Either to:
+// - incorporate one of the shifts into the lea
+// - allow less copying
+static void
+genShdLeaSequence(MachineInstr &Root, const TargetInstrInfo &TII,
+ SmallVectorImpl<MachineInstr *> &InsInstrs,
+ SmallVectorImpl<MachineInstr *> &DelInstrs,
+ DenseMap<Register, unsigned> &InstrIdxForVirtReg) {
+ auto *MF = Root.getMF();
+ MachineRegisterInfo &RegInfo = MF->getRegInfo();
+ const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
+ auto GetRC = [&](Register Reg) {
+ return Reg.isVirtual() ? RegInfo.getRegClass(Reg)
+ : TRI->getMinimalPhysRegClass(Reg);
+ };
+
+ unsigned BW = 0;
+ bool IsShrd = false;
+ switch (Root.getOpcode()) {
+ case X86::SHRD32rri8:
+ IsShrd = true;
+ [[fallthrough]];
+ case X86::SHLD32rri8:
+ BW = 32;
+ break;
+ case X86::SHRD64rri8:
+ IsShrd = true;
+ [[fallthrough]];
+ case X86::SHLD64rri8:
+ BW = 64;
+ break;
+ default:
+ llvm_unreachable("Unexpected opcode in genShdLeaSequence");
+ }
+
+ int64_t Imm = Root.getOperand(3).getImm();
+ int64_t ScaleBits = IsShrd ? (BW - Imm) : Imm;
+
+ unsigned ShlOpc = (BW == 64) ? X86::SHL64ri : X86::SHL32ri;
+ unsigned ShrOpc = (BW == 64) ? X86::SHR64ri : X86::SHR32ri;
+ unsigned LeaOpc = (BW == 64) ? X86::LEA64r : X86::LEA32r;
+ const TargetRegisterClass *RC = GetRC(Root.getOperand(0).getReg());
+ Register OutReg = Root.getOperand(0).getReg();
+
+ if (ScaleBits >= 1 && ScaleBits <= 3) {
+ // One of the shifts can be done in LEA
+ Register Reg1 = RegInfo.createVirtualRegister(RC);
+ if (!IsShrd) {
+ MachineInstr *Shr =
+ BuildMI(*MF, MIMetadata(Root), TII.get(ShrOpc), Reg1)
+ .addReg(Root.getOperand(2).getReg(),
+ getKillRegState(Root.getOperand(2).isKill()))
+ .addImm(BW - Imm);
+ MachineInstr *Lea =
+ BuildMI(*MF, MIMetadata(Root), TII.get(LeaOpc), OutReg)
+ .addReg(Reg1, RegState::Kill)
+ .addImm(1 << Imm)
+ .addReg(Root.getOperand(1).getReg(),
+ getKillRegState(Root.getOperand(1).isKill()))
+ .addImm(0)
+ .addReg(0);
+ InstrIdxForVirtReg.insert({Reg1, 0});
+ InsInstrs.push_back(Shr);
+ InsInstrs.push_back(Lea);
+ } else {
+ MachineInstr *Shr =
+ BuildMI(*MF, MIMetadata(Root), TII.get(ShrOpc), Reg1)
+ .addReg(Root.getOperand(1).getReg(),
+ getKillRegState(Root.getOperand(1).isKill()))
+ .addImm(Imm);
+ MachineInstr *Lea =
+ BuildMI(*MF, MIMetadata(Root), TII.get(LeaOpc), OutReg)
+ .addReg(Reg1, RegState::Kill)
+ .addImm(1 << (BW - Imm))
+ .addReg(Root.getOperand(2).getReg(),
+ getKillRegState(Root.getOperand(2).isKill()))
+ .addImm(0)
+ .addReg(0);
+ InstrIdxForVirtReg.insert({Reg1, 0});
+ InsInstrs.push_back(Shr);
+ InsInstrs.push_back(Lea);
+ }
+ } else {
+ // Neither shift fits a LEA scale. Make shifts
+ // explicit and emit LEA as OR.
+ unsigned DirectOpc = IsShrd ? ShrOpc : ShlOpc;
+ unsigned ComplOpc = IsShrd ? ShlOpc : ShrOpc;
+ Register Reg1 = RegInfo.createVirtualRegister(RC);
+ Register Reg2 = RegInfo.createVirtualRegister(RC);
+ MachineInstr *MI1 =
+ BuildMI(*MF, MIMetadata(Root), TII.get(DirectOpc), Reg1)
+ .addReg(Root.getOperand(1).getReg(),
+ getKillRegState(Root.getOperand(1).isKill()))
+ .addImm(Imm);
+ MachineInstr *MI2 =
+ BuildMI(*MF, MIMetadata(Root), TII.get(ComplOpc), Reg2)
+ .addReg(Root.getOperand(2).getReg(),
+ getKillRegState(Root.getOperand(2).isKill()))
+ .addImm(BW - Imm);
+ MachineInstr *Lea = BuildMI(*MF, MIMetadata(Root), TII.get(LeaOpc), OutReg)
+ .addReg(Reg1, RegState::Kill)
+ .addImm(1)
+ .addReg(Reg2, RegState::Kill)
+ .addImm(0)
+ .addReg(0);
+ InstrIdxForVirtReg.insert({Reg1, 0});
+ InstrIdxForVirtReg.insert({Reg2, 1});
+ InsInstrs.push_back(MI1);
+ InsInstrs.push_back(MI2);
+ InsInstrs.push_back(Lea);
+ }
+ DelInstrs.push_back(&Root);
+}
+
static void
genAlternativeDpCodeSequence(MachineInstr &Root, const TargetInstrInfo &TII,
SmallVectorImpl<MachineInstr *> &InsInstrs,
@@ -10910,10 +11025,11 @@ void X86InstrInfo::genAlternativeCodeSequence(
genAlternativeDpCodeSequence(Root, *this, InsInstrs, DelInstrs,
InstrIdxForVirtReg);
return;
- case X86MachineCombinerPattern::USHD:
- genAlternativeShdSequence(Root, *this, InsInstrs, DelInstrs,
- InstrIdxForVirtReg);
+ case X86MachineCombinerPattern::USHD_OR:
+ genShdOrSequence(Root, *this, InsInstrs, DelInstrs, InstrIdxForVirtReg);
return;
+ case X86MachineCombinerPattern::USHD_LEA:
+ genShdLeaSequence(Root, *this, InsInstrs, DelInstrs, InstrIdxForVirtReg);
}
}
diff --git a/llvm/lib/Target/X86/X86InstrInfo.h b/llvm/lib/Target/X86/X86InstrInfo.h
index 2a4b516538510..0286756603eaf 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.h
+++ b/llvm/lib/Target/X86/X86InstrInfo.h
@@ -31,7 +31,8 @@ enum X86MachineCombinerPattern : unsigned {
// X86 VNNI
DPWSSD = MachineCombinerPattern::TARGET_PATTERN_START,
// Unfold SHD
- USHD,
+ USHD_OR,
+ USHD_LEA,
};
namespace X86 {
diff --git a/llvm/test/CodeGen/X86/fshl.ll b/llvm/test/CodeGen/X86/fshl.ll
index 7a6e5f825b97c..c4bd4b940008d 100644
--- a/llvm/test/CodeGen/X86/fshl.ll
+++ b/llvm/test/CodeGen/X86/fshl.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=i686-unknown-unknown | FileCheck %s --check-prefixes=X86,X86-FAST
-; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+slow-shld | FileCheck %s --check-prefixes=X86,X86-SLOW
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefixes=X64,X64-FAST
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+slow-shld | FileCheck %s --check-prefixes=X64,X64-SLOW
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mcpu=znver3 | FileCheck %s --check-prefixes=X86,X86-FAST
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mcpu=znver2 | FileCheck %s --check-prefixes=X86,X86-SLOW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver3 | FileCheck %s --check-prefixes=X64,X64-FAST
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver2 | FileCheck %s --check-prefixes=X64,X64-SLOW
declare i8 @llvm.fshl.i8(i8, i8, i8) nounwind readnone
declare i16 @llvm.fshl.i16(i16, i16, i16) nounwind readnone
@@ -17,25 +17,23 @@ declare i128 @llvm.fshl.i128(i128, i128, i128) nounwind readnone
define i8 @var_shift_i8(i8 %x, i8 %y, i8 %z) nounwind {
; X86-LABEL: var_shift_i8:
; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movzbl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: shll $8, %eax
-; X86-NEXT: orl %edx, %eax
-; X86-NEXT: andb $7, %cl
-; X86-NEXT: shll %cl, %eax
+; X86-NEXT: shll $8, %edx
+; X86-NEXT: andb $7, %al
+; X86-NEXT: orl %ecx, %edx
+; X86-NEXT: shlxl %eax, %edx, %eax
; X86-NEXT: movb %ah, %al
; X86-NEXT: retl
;
; X64-LABEL: var_shift_i8:
; X64: # %bb.0:
-; X64-NEXT: movl %edx, %ecx
; X64-NEXT: shll $8, %edi
; X64-NEXT: movzbl %sil, %eax
+; X64-NEXT: andb $7, %dl
; X64-NEXT: orl %edi, %eax
-; X64-NEXT: andb $7, %cl
-; X64-NEXT: # kill: def $cl killed $cl killed $ecx
-; X64-NEXT: shll %cl, %eax
+; X64-NEXT: shlxl %edx, %eax, %eax
; X64-NEXT: shrl $8, %eax
; X64-NEXT: # kill: def $al killed $al killed $eax
; X64-NEXT: retq
@@ -46,22 +44,22 @@ define i8 @var_shift_i8(i8 %x, i8 %y, i8 %z) nounwind {
define i16 @var_shift_i16(i16 %x, i16 %y, i16 %z) nounwind {
; X86-FAST-LABEL: var_shift_i16:
; X86-FAST: # %bb.0:
+; X86-FAST-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
; X86-FAST-NEXT: movzwl {{[0-9]+}}(%esp), %edx
; X86-FAST-NEXT: movzwl {{[0-9]+}}(%esp), %eax
-; X86-FAST-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
; X86-FAST-NEXT: andb $15, %cl
; X86-FAST-NEXT: shldw %cl, %dx, %ax
; X86-FAST-NEXT: retl
;
; X86-SLOW-LABEL: var_shift_i16:
; X86-SLOW: # %bb.0:
-; X86-SLOW-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X86-SLOW-NEXT: movzwl {{[0-9]+}}(%esp), %edx
-; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-SLOW-NEXT: shll $16, %eax
-; X86-SLOW-NEXT: orl %edx, %eax
-; X86-SLOW-NEXT: andb $15, %cl
-; X86-SLOW-NEXT: shll %cl, %eax
+; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-SLOW-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SLOW-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; X86-SLOW-NEXT: shll $16, %edx
+; X86-SLOW-NEXT: andb $15, %al
+; X86-SLOW-NEXT: orl %ecx, %edx
+; X86-SLOW-NEXT: shlxl %eax, %edx, %eax
; X86-SLOW-NEXT: shrl $16, %eax
; X86-SLOW-NEXT: # kill: def $ax killed $ax killed $eax
; X86-SLOW-NEXT: retl
@@ -69,8 +67,8 @@ define i16 @var_shift_i16(i16 %x, i16 %y, i16 %z) nounwind {
; X64-FAST-LABEL: var_shift_i16:
; X64-FAST: # %bb.0:
; X64-FAST-NEXT: movl %edx, %ecx
-; X64-FAST-NEXT: movl %edi, %eax
; X64-FAST-NEXT: andb $15, %cl
+; X64-FAST-NEXT: movl %edi, %eax
; X64-FAST-NEXT: # kill: def $cl killed $cl killed $ecx
; X64-FAST-NEXT: shldw %cl, %si, %ax
; X64-FAST-NEXT: # kill: def $ax killed $ax killed $eax
@@ -78,13 +76,11 @@ define i16 @var_shift_i16(i16 %x, i16 %y, i16 %z) nounwind {
;
; X64-SLOW-LABEL: var_shift_i16:
; X64-SLOW: # %bb.0:
-; X64-SLOW-NEXT: movl %edx, %ecx
; X64-SLOW-NEXT: shll $16, %edi
; X64-SLOW-NEXT: movzwl %si, %eax
+; X64-SLOW-NEXT: andb $15, %dl
; X64-SLOW-NEXT: orl %edi, %eax
-; X64-SLOW-NEXT: andb $15, %cl
-; X64-SLOW-NEXT: # kill: def $cl killed $cl killed $ecx
-; X64-SLOW-NEXT: shll %cl, %eax
+; X64-SLOW-NEXT: shlxl %edx, %eax, %eax
; X64-SLOW-NEXT: shrl $16, %eax
; X64-SLOW-NEXT: # kill: def $ax killed $ax killed $eax
; X64-SLOW-NEXT: retq
@@ -101,13 +97,21 @@ define i32 @var_shift_i32(i32 %x, i32 %y, i32 %z) nounwind {
; X86-NEXT: shldl %cl, %edx, %eax
; X86-NEXT: retl
;
-; X64-LABEL: var_shift_i32:
-; X64: # %bb.0:
-; X64-NEXT: movl %edx, %ecx
-; X64-NEXT: movl %edi, %eax
-; X64-NEXT: # kill: def $cl killed $cl killed $ecx
-; X64-NEXT: shldl %cl, %esi, %eax
-; X64-NEXT: retq
+; X64-FAST-LABEL: var_shift_i32:
+; X64-FAST: # %bb.0:
+; X64-FAST-NEXT: movl %edx, %ecx
+; X64-FAST-NEXT: movl %edi, %eax
+; X64-FAST-NEXT: # kill: def $cl killed $cl killed $ecx
+; X64-FAST-NEXT: shldl %cl, %esi, %eax
+; X64-FAST-NEXT: retq
+;
+; X64-SLOW-LABEL: var_shift_i32:
+; X64-SLOW: # %bb.0:
+; X64-SLOW-NEXT: movl %edi, %eax
+; X64-SLOW-NEXT: movl %edx, %ecx
+; X64-SLOW-NEXT: # kill: def $cl killed $cl killed $ecx
+; X64-SLOW-NEXT: shldl %cl, %esi, %eax
+; X64-SLOW-NEXT: retq
%tmp = tail call i32 @llvm.fshl.i32(i32 %x, i32 %y, i32 %z)
ret i32 %tmp
}
@@ -121,13 +125,21 @@ define i32 @var_shift_i32_optsize(i32 %x, i32 %y, i32 %z) nounwind optsize {
; X86-NEXT: shldl %cl, %edx, %eax
; X86-NEXT: retl
;
-; X64-LABEL: var_shift_i32_optsize:
-; X64: # %bb.0:
-; X64-NEXT: movl %edx, %ecx
-; X64-NEXT: movl %edi, %eax
-; X64-NEXT: # kill: def $cl killed $cl killed $ecx
-; X64-NEXT: shldl %cl, %esi, %eax
-; X64-NEXT: retq
+; X64-FAST-LABEL: var_shift_i32_optsize:
+; X64-FAST: # %bb.0:
+; X64-FAST-NEXT: movl %edx, %ecx
+; X64-FAST-NEXT: movl %edi, %eax
+; X64-FAST-NEXT: # kill: def $cl killed $cl killed $ecx
+; X64-FAST-NEXT: shldl %cl, %esi, %eax
+; X64-FAST-NEXT: retq
+;
+; X64-SLOW-LABEL: var_shift_i32_optsize:
+; X64-SLOW: # %bb.0:
+; X64-SLOW-NEXT: movl %edi, %eax
+; X64-SLOW-NEXT: movl %edx, %ecx
+; X64-SLOW-NEXT: # kill: def $cl killed $cl killed $ecx
+; X64-SLOW-NEXT: shldl %cl, %esi, %eax
+; X64-SLOW-NEXT: retq
%tmp = tail call i32 @llvm.fshl.i32(i32 %x, i32 %y, i32 %z)
ret i32 %tmp
}
@@ -141,132 +153,231 @@ define i32 @var_shift_i32_pgso(i32 %x, i32 %y, i32 %z) nounwind !prof !14 {
; X86-NEXT: shldl %cl, %edx, %eax
; X86-NEXT: retl
;
-; X64-LABEL: var_shift_i32_pgso:
-; X64: # %bb.0:
-; X64-NEXT: movl %edx, %ecx
-; X64-NEXT: movl %edi, %eax
-; X64-NEXT: # kill: def $cl killed $cl killed $ecx
-; X64-NEXT: shldl %cl, %esi, %eax
-; X64-NEXT: retq
+; X64-FAST-LABEL: var_shift_i32_pgso:
+; X64-FAST: # %bb.0:
+; X64-FAST-NEXT: movl %edx, %ecx
+; X64-FAST-NEXT: movl %edi, %eax
+; X64-FAST-NEXT: # kill: def $cl killed $cl killed $ecx
+; X64-FAST-NEXT: shldl %cl, %esi, %eax
+; X64-FAST-NEXT: retq
+;
+; X64-SLOW-LABEL: var_shift_i32_pgso:
+; X64-SLOW: # %bb.0:
+; X64-SLOW-NEXT: movl %edi, %eax
+; X64-SLOW-NEXT: movl %edx, %ecx
+; X64-SLOW-NEXT: # kill: def $cl killed $cl killed $ecx
+; X64-SLOW-NEXT: shldl %cl, %esi, %eax
+; X64-SLOW-NEXT: retq
%tmp = tail call i32 @llvm.fshl.i32(i32 %x, i32 %y, i32 %z)
ret i32 %tmp
}
define i64 @var_shift_i64(i64 %x, i64 %y, i64 %z) nounwind {
-; X86-LABEL: var_shift_i64:
-; X86: # %bb.0:
-; X86-NEXT: pushl %edi
-; X86-NEXT: pushl %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: testb $32, %cl
-; X86-NEXT: jne .LBB5_1
-; X86-NEXT: # %bb.2:
-; X86-NEXT: movl %edx, %edi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: jmp .LBB5_3
-; X86-NEXT: .LBB5_1:
-; X86-NEXT: movl %esi, %edi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT: .LBB5_3:
-; X86-NEXT: movl %edi, %eax
-; X86-NEXT: shldl %cl, %esi, %eax
-; X86-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NEXT: shldl %cl, %edi, %edx
-; X86-NEXT: popl %esi
-; X86-NEXT: popl %edi
-; X86-NEXT: retl
+; X86-FAST-LABEL: var_shift_i64:
+; X86-FAST: # %bb.0:
+; X86-FAST-NEXT: pushl %edi
+; X86-FAST-NEXT: pushl %esi
+; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-FAST-NEXT: testb $32, %cl
+; X86-FAST-NEXT: movl %edx, %edi
+; X86-FAST-NEXT: cmovnel %esi, %edi
+; X86-FAST-NEXT: cmovel {{[0-9]+}}(%esp), %edx
+; X86-FAST-NEXT: cmovnel {{[0-9]+}}(%esp), %esi
+; X86-FAST-NEXT: movl %edi, %eax
+; X86-FAST-NEXT: shldl %cl, %esi, %eax
+; X86-FAST-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-FAST-NEXT: shldl %cl, %edi, %edx
+; X86-FAST-NEXT: popl %esi
+; X86-FAST-NEXT: popl %edi
+; X86-FAST-NEXT: retl
;
-; X64-LABEL: var_shift_i64:
-; X64: # %bb.0:
-; X64-NEXT: movq %rdx, %rcx
-; X64-NEXT: movq %rdi, %rax
-; X64-NEXT: # kill: def $cl killed $cl killed $rcx
-; X64-NEXT: shldq %cl, %rsi, %rax
-; X64-NEXT: retq
+; X86-SLOW-LABEL: var_shift_i64:
+; X86-SLOW: # %bb.0:
+; X86-SLOW-NEXT: pushl %edi
+; X86-SLOW-NEXT: pushl %esi
+; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-SLOW-NEXT: testb $32, %cl
+; X86-SLOW-NEXT: movl %edx, %edi
+; X86-SLOW-NEXT: cmovnel %esi, %edi
+; X86-SLOW-NEXT: cmovel {{[0-9]+}}(%esp), %edx
+; X86-SLOW-NEXT: cmovnel {{[0-9]+}}(%esp), %esi
+; X86-SLOW-NEXT: movl %edi, %eax
+; X86-SLOW-NEXT: shldl %cl, %esi, %eax
+; X86-SLOW-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-SLOW-NEXT: shldl %cl, %edi, %edx
+; X86-SLOW-NEXT: popl %esi
+; X86-SLOW-NEXT: popl %edi
+; X86-SLOW-NEXT: retl
+;
+; X64-FAST-LABEL: var_shift_i64:
+; X64-FAST: # %bb.0:
+; X64-FAST-NEXT: movq %rdx, %rcx
+; X64-FAST-NEXT: movq %rdi, %rax
+; X64-FAST-NEXT: # kill: def $cl killed $cl killed $rcx
+; X64-FAST-NEXT: shldq %cl, %rsi, %rax
+; X64-FAST-NEXT: retq
+;
+; X64-SLOW-LABEL: var_shift_i64:
+; X64-SLOW: # %bb.0:
+; X64-SLOW-NEXT: movq %rdi, %rax
+; X64-SLOW-NEXT: movq %rdx, %rcx
+; X64-SLOW-NEXT: # kill: def $cl killed $cl killed $rcx
+; X64-SLOW-NEXT: shldq %cl, %rsi, %rax
+; X64-SLOW-NEXT: retq
%tmp = tail call i64 @llvm.fshl.i64(i64 %x, i64 %y, i64 %z)
ret i64 %tmp
}
define i128 @var_shift_i128(i128 %x, i128 %y, i128 %z) nounwind {
-; X86-LABEL: var_shift_i128:
-; X86: # %bb.0:
-; X86-NEXT: pushl %ebp
-; X86-NEXT: movl %esp, %ebp
-; X86-NEXT: pushl %ebx
-; X86-NEXT: pushl %edi
-; X86-NEXT: pushl %esi
-; X86-NEXT: andl $-16, %esp
-; X86-NEXT: subl $16, %esp
-; X86-NEXT: movl 24(%ebp), %edi
-; X86-NEXT: movl 28(%ebp), %edx
-; X86-NEXT: movl 48(%ebp), %esi
-; X86-NEXT: movl 56(%ebp), %ecx
-; X86-NEXT: testb $64, %cl
-; X86-NEXT: movl 52(%ebp), %eax
-; X86-NEXT: jne .LBB6_1
-; X86-NEXT: # %bb.2:
-; X86-NEXT: movl %esi, %ebx
-; X86-NEXT: movl %edi, %esi
-; X86-NEXT: movl 32(%ebp), %edi
-; X86-NEXT: movl %eax, (%esp) # 4-byte Spill
-; X86-NEXT: movl %edx, %eax
-; X86-NEXT: movl 36(%ebp), %edx
-; X86-NEXT: testb $32, %cl
-; X86-NEXT: je .LBB6_5
-; X86-NEXT: .LBB6_4:
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %esi, %eax
-; X86-NEXT: movl (%esp), %esi # 4-byte Reload
-; X86-NEXT: jmp .LBB6_6
-; X86-NEXT: .LBB6_1:
-; X86-NEXT: movl 44(%ebp), %ebx
-; X86-NEXT: movl %ebx, (%esp) # 4-byte Spill
-; X86-NEXT: movl 40(%ebp), %ebx
-; X86-NEXT: testb $32, %cl
-; X86-NEXT: jne .LBB6_4
-; X86-NEXT: .LBB6_5:
-; X86-NEXT: movl (%esp), %ebx # 4-byte Reload
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: .LBB6_6:
-; X86-NEXT: movl %esi, %edi
-; X86-NEXT: shldl %cl, %ebx, %edi
-; X86-NEXT: movl %eax, %edx
-; X86-NEXT: movl %eax, %ebx
-; X86-NEXT: shldl %cl, %esi, %ebx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: movl %eax, %esi
-; X86-NEXT: shldl %cl, %edx, %esi
-; X86-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: shldl %cl, %eax, %edx
-; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: movl %edx, 12(%eax)
-; X86-NEXT: movl %esi, 8(%eax)
-; X86-NEXT: movl %ebx, 4(%eax)
-; X86-NEXT: movl %edi, (%eax)
-; X86-NEXT: leal -12(%ebp), %esp
-; X86-NEXT: popl %esi
-; X86-NEXT: popl %edi
-; X86-NEXT: popl %ebx
-; X86-NEXT: popl %ebp
-; X86-NEXT: retl $4
-;
-; X64-LABEL: var_shift_i128:
-; X64: # %bb.0:
-; X64-NEXT: testb $64, %r8b
-; X64-NEXT: cmovneq %rdi, %rsi
-; X64-NEXT: cmoveq %rcx, %rdx
-; X64-NEXT: cmovneq %rcx, %rdi
-; X64-NEXT: movq %rdi, %rax
-; X64-NEXT: movl %r8d, %ecx
-; X64-NEXT: shldq %cl, %rdx, %rax
-; X64-NEXT: shldq %cl, %rdi, %rsi
-; X64-NEXT: movq %rsi, %rdx
-; X64-NEXT: retq
+; X86-FAST-LABEL: var_shift_i128:
+; X86-FAST: # %bb.0:
+; X86-FAST-NEXT: pushl %ebp
+; X86-FAST-NEXT: movl %esp, %ebp
+; X86-FAST-NEXT: pushl %ebx
+; X86-FAST-NEXT: pushl %edi
+; X86-FAST-NEXT: pushl %esi
+; X86-FAST-NEXT: andl $-16, %esp
+; X86-FAST-NEXT: subl $16, %esp
+; X86-FAST-NEXT: movl 56(%ebp), %ecx
+; X86-FAST-NEXT: movl 48(%ebp), %edi
+; X86-FAST-NEXT: movl 24(%ebp), %eax
+; X86-FAST-NEXT: movl 28(%ebp), %edx
+; X86-FAST-NEXT: movl 52(%ebp), %esi
+; X86-FAST-NEXT: testb $64, %cl
+; X86-FAST-NEXT: movl %eax, %ecx
+; X86-FAST-NEXT: movl %edx, %ebx
+; X86-FAST-NEXT: cmovnel %edi, %ecx
+; X86-FAST-NEXT: cmovnel %esi, %ebx
+; X86-FAST-NEXT: cmovnel 44(%ebp), %esi
+; X86-FAST-NEXT: cmovnel 40(%ebp), %edi
+; X86-FAST-NEXT: cmovel 36(%ebp), %edx
+; X86-FAST-NEXT: cmovel 32(%ebp), %eax
+; X86-FAST-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-FAST-NEXT: movl 56(%ebp), %ecx
+; X86-FAST-NEXT: testb $32, %cl
+; X86-FAST-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-FAST-NEXT: cmovnel %eax, %edx
+; X86-FAST-NEXT: cmovnel %ebx, %eax
+; X86-FAST-NEXT: cmovel %esi, %edi
+; X86-FAST-NEXT: cmovel %ecx, %esi
+; X86-FAST-NEXT: cmovnel %ecx, %ebx
+; X86-FAST-NEXT: movl 56(%ebp), %ecx
+; X86-FAST-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-FAST-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-FAST-NEXT: shldl %cl, %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-FAST-NEXT: movl 56(%ebp), %ecx
+; X86-FAST-NEXT: movl %ebx, %edi
+; X86-FAST-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-FAST-NEXT: shldl %cl, %esi, %edi
+; X86-FAST-NEXT: movl 56(%ebp), %ecx
+; X86-FAST-NEXT: movl %eax, %esi
+; X86-FAST-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-FAST-NEXT: shldl %cl, %ebx, %esi
+; X86-FAST-NEXT: movl 56(%ebp), %ecx
+; X86-FAST-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-FAST-NEXT: shldl %cl, %eax, %edx
+; X86-FAST-NEXT: movl 8(%ebp), %eax
+; X86-FAST-NEXT: movl %edx, 12(%eax)
+; X86-FAST-NEXT: movl %esi, 8(%eax)
+; X86-FAST-NEXT: movl %edi, 4(%eax)
+; X86-FAST-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-FAST-NEXT: movl %ecx, (%eax)
+; X86-FAST-NEXT: leal -12(%ebp), %esp
+; X86-FAST-NEXT: popl %esi
+; X86-FAST-NEXT: popl %edi
+; X86-FAST-NEXT: popl %ebx
+; X86-FAST-NEXT: popl %ebp
+; X86-FAST-NEXT: retl $4
+;
+; X86-SLOW-LABEL: var_shift_i128:
+; X86-SLOW: # %bb.0:
+; X86-SLOW-NEXT: pushl %ebp
+; X86-SLOW-NEXT: movl %esp, %ebp
+; X86-SLOW-NEXT: pushl %ebx
+; X86-SLOW-NEXT: pushl %edi
+; X86-SLOW-NEXT: pushl %esi
+; X86-SLOW-NEXT: andl $-16, %esp
+; X86-SLOW-NEXT: subl $16, %esp
+; X86-SLOW-NEXT: movl 56(%ebp), %ecx
+; X86-SLOW-NEXT: movl 24(%ebp), %eax
+; X86-SLOW-NEXT: movl 48(%ebp), %edi
+; X86-SLOW-NEXT: movl 28(%ebp), %edx
+; X86-SLOW-NEXT: movl 52(%ebp), %esi
+; X86-SLOW-NEXT: testb $64, %cl
+; X86-SLOW-NEXT: movl %eax, %ecx
+; X86-SLOW-NEXT: movl %edx, %ebx
+; X86-SLOW-NEXT: cmovnel %edi, %ecx
+; X86-SLOW-NEXT: cmovnel %esi, %ebx
+; X86-SLOW-NEXT: cmovnel 44(%ebp), %esi
+; X86-SLOW-NEXT: cmovnel 40(%ebp), %edi
+; X86-SLOW-NEXT: cmovel 36(%ebp), %edx
+; X86-SLOW-NEXT: cmovel 32(%ebp), %eax
+; X86-SLOW-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-SLOW-NEXT: movl 56(%ebp), %ecx
+; X86-SLOW-NEXT: testb $32, %cl
+; X86-SLOW-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-SLOW-NEXT: cmovnel %eax, %edx
+; X86-SLOW-NEXT: cmovnel %ebx, %eax
+; X86-SLOW-NEXT: cmovel %esi, %edi
+; X86-SLOW-NEXT: cmovel %ecx, %esi
+; X86-SLOW-NEXT: cmovnel %ecx, %ebx
+; X86-SLOW-NEXT: movl 56(%ebp), %ecx
+; X86-SLOW-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-SLOW-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-SLOW-NEXT: shldl %cl, %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-SLOW-NEXT: movl %ebx, %edi
+; X86-SLOW-NEXT: movl 56(%ebp), %ecx
+; X86-SLOW-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-SLOW-NEXT: shldl %cl, %esi, %edi
+; X86-SLOW-NEXT: movl %eax, %esi
+; X86-SLOW-NEXT: movl 56(%ebp), %ecx
+; X86-SLOW-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-SLOW-NEXT: shldl %cl, %ebx, %esi
+; X86-SLOW-NEXT: movl 56(%ebp), %ecx
+; X86-SLOW-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-SLOW-NEXT: shldl %cl, %eax, %edx
+; X86-SLOW-NEXT: movl 8(%ebp), %eax
+; X86-SLOW-NEXT: movl %edx, 12(%eax)
+; X86-SLOW-NEXT: movl %esi, 8(%eax)
+; X86-SLOW-NEXT: movl %edi, 4(%eax)
+; X86-SLOW-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-SLOW-NEXT: movl %ecx, (%eax)
+; X86-SLOW-NEXT: leal -12(%ebp), %esp
+; X86-SLOW-NEXT: popl %esi
+; X86-SLOW-NEXT: popl %edi
+; X86-SLOW-NEXT: popl %ebx
+; X86-SLOW-NEXT: popl %ebp
+; X86-SLOW-NEXT: retl $4
+;
+; X64-FAST-LABEL: var_shift_i128:
+; X64-FAST: # %bb.0:
+; X64-FAST-NEXT: testb $64, %r8b
+; X64-FAST-NEXT: cmovneq %rdi, %rsi
+; X64-FAST-NEXT: cmovneq %rcx, %rdi
+; X64-FAST-NEXT: cmoveq %rcx, %rdx
+; X64-FAST-NEXT: movl %r8d, %ecx
+; X64-FAST-NEXT: movq %rdi, %rax
+; X64-FAST-NEXT: shldq %cl, %rdx, %rax
+; X64-FAST-NEXT: shldq %cl, %rdi, %rsi
+; X64-FAST-NEXT: movq %rsi, %rdx
+; X64-FAST-NEXT: retq
+;
+; X64-SLOW-LABEL: var_shift_i128:
+; X64-SLOW: # %bb.0:
+; X64-SLOW-NEXT: testb $64, %r8b
+; X64-SLOW-NEXT: cmovneq %rdi, %rsi
+; X64-SLOW-NEXT: cmovneq %rcx, %rdi
+; X64-SLOW-NEXT: cmoveq %rcx, %rdx
+; X64-SLOW-NEXT: movl %r8d, %ecx
+; X64-SLOW-NEXT: movq %rdi, %rax
+; X64-SLOW-NEXT: shldq %cl, %rdi, %rsi
+; X64-SLOW-NEXT: shldq %cl, %rdx, %rax
+; X64-SLOW-NEXT: movq %rsi, %rdx
+; X64-SLOW-NEXT: retq
%tmp = tail call i128 @llvm.fshl.i128(i128 %x, i128 %y, i128 %z)
ret i128 %tmp
}
@@ -336,37 +447,76 @@ define i16 @const_shift_i16(i16 %x, i16 %y) nounwind {
}
define i32 @const_shift_i32(i32 %x, i32 %y) nounwind {
-; X86-LABEL: const_shift_i32:
-; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: shldl $7, %ecx, %eax
-; X86-NEXT: retl
+; X86-FAST-LABEL: const_shift_i32:
+; X86-FAST: # %bb.0:
+; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-FAST-NEXT: shldl $7, %ecx, %eax
+; X86-FAST-NEXT: retl
;
-; X64-LABEL: const_shift_i32:
-; X64: # %bb.0:
-; X64-NEXT: movl %edi, %eax
-; X64-NEXT: shldl $7, %esi, %eax
-; X64-NEXT: retq
+; X86-SLOW-LABEL: const_shift_i32:
+; X86-SLOW: # %bb.0:
+; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SLOW-NEXT: shll $7, %ecx
+; X86-SLOW-NEXT: shrl $25, %eax
+; X86-SLOW-NEXT: addl %ecx, %eax
+; X86-SLOW-NEXT: retl
+;
+; X64-FAST-LABEL: const_shift_i32:
+; X64-FAST: # %bb.0:
+; X64-FAST-NEXT: movl %edi, %eax
+; X64-FAST-NEXT: shldl $7, %esi, %eax
+; X64-FAST-NEXT: retq
+;
+; X64-SLOW-LABEL: const_shift_i32:
+; X64-SLOW: # %bb.0:
+; X64-SLOW-NEXT: shll $7, %edi
+; X64-SLOW-NEXT: shrl $25, %esi
+; X64-SLOW-NEXT: leal (%edi,%esi), %eax
+; X64-SLOW-NEXT: retq
%tmp = tail call i32 @llvm.fshl.i32(i32 %x, i32 %y, i32 7)
ret i32 %tmp
}
define i64 @const_shift_i64(i64 %x, i64 %y) nounwind {
-; X86-LABEL: const_shift_i64:
-; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: shrdl $25, %ecx, %eax
-; X86-NEXT: shldl $7, %ecx, %edx
-; X86-NEXT: retl
+; X86-FAST-LABEL: const_shift_i64:
+; X86-FAST: # %bb.0:
+; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-FAST-NEXT: shrdl $25, %ecx, %eax
+; X86-FAST-NEXT: shldl $7, %ecx, %edx
+; X86-FAST-NEXT: retl
;
-; X64-LABEL: const_shift_i64:
-; X64: # %bb.0:
-; X64-NEXT: movq %rdi, %rax
-; X64-NEXT: shldq $7, %rsi, %rax
-; X64-NEXT: retq
+; X86-SLOW-LABEL: const_shift_i64:
+; X86-SLOW: # %bb.0:
+; X86-SLOW-NEXT: pushl %esi
+; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-SLOW-NEXT: movl %ecx, %esi
+; X86-SLOW-NEXT: shrl $25, %eax
+; X86-SLOW-NEXT: shll $7, %edx
+; X86-SLOW-NEXT: shrl $25, %ecx
+; X86-SLOW-NEXT: shll $7, %esi
+; X86-SLOW-NEXT: addl %ecx, %edx
+; X86-SLOW-NEXT: addl %esi, %eax
+; X86-SLOW-NEXT: popl %esi
+; X86-SLOW-NEXT: retl
+;
+; X64-FAST-LABEL: const_shift_i64:
+; X64-FAST: # %bb.0:
+; X64-FAST-NEXT: movq %rdi, %rax
+; X64-FAST-NEXT: shldq $7, %rsi, %rax
+; X64-FAST-NEXT: retq
+;
+; X64-SLOW-LABEL: const_shift_i64:
+; X64-SLOW: # %bb.0:
+; X64-SLOW-NEXT: shlq $7, %rdi
+; X64-SLOW-NEXT: shrq $57, %rsi
+; X64-SLOW-NEXT: leaq (%rdi,%rsi), %rax
+; X64-SLOW-NEXT: retq
%tmp = tail call i64 @llvm.fshl.i64(i64 %x, i64 %y, i64 7)
ret i64 %tmp
}
@@ -451,14 +601,23 @@ define i64 @combine_fshl_load_i64(ptr %p) nounwind {
}
define i16 @combine_fshl_load_i16_chain_use(ptr %p, ptr %q, i16 %r) nounwind {
-; X86-LABEL: combine_fshl_load_i16_chain_use:
-; X86: # %bb.0:
-; X86-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movzwl 1(%eax), %eax
-; X86-NEXT: movw %cx, (%edx)
-; X86-NEXT: retl
+; X86-FAST-LABEL: combine_fshl_load_i16_chain_use:
+; X86-FAST: # %bb.0:
+; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-FAST-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-FAST-NEXT: movzwl 1(%eax), %eax
+; X86-FAST-NEXT: movw %cx, (%edx)
+; X86-FAST-NEXT: retl
+;
+; X86-SLOW-LABEL: combine_fshl_load_i16_chain_use:
+; X86-SLOW: # %bb.0:
+; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SLOW-NEXT: movzwl 1(%eax), %eax
+; X86-SLOW-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-SLOW-NEXT: movw %cx, (%edx)
+; X86-SLOW-NEXT: retl
;
; X64-LABEL: combine_fshl_load_i16_chain_use:
; X64: # %bb.0:
diff --git a/llvm/test/CodeGen/X86/fshr.ll b/llvm/test/CodeGen/X86/fshr.ll
index f77ffe92472a2..25400eef54f39 100644
--- a/llvm/test/CodeGen/X86/fshr.ll
+++ b/llvm/test/CodeGen/X86/fshr.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=i686-unknown-unknown | FileCheck %s --check-prefixes=X86,X86-FAST
-; RUN: llc < %s -mtriple=i686-unknown-unknown -mattr=+slow-shld | FileCheck %s --check-prefixes=X86,X86-SLOW
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefixes=X64,X64-FAST
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+slow-shld | FileCheck %s --check-prefixes=X64,X64-SLOW
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mcpu=znver3 | FileCheck %s --check-prefixes=X86,X86-FAST
+; RUN: llc < %s -mtriple=i686-unknown-unknown -mcpu=znver2 | FileCheck %s --check-prefixes=X86,X86-SLOW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver3 | FileCheck %s --check-prefixes=X64,X64-FAST
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver2 | FileCheck %s --check-prefixes=X64,X64-SLOW
declare i8 @llvm.fshr.i8(i8, i8, i8) nounwind readnone
declare i16 @llvm.fshr.i16(i16, i16, i16) nounwind readnone
@@ -17,25 +17,23 @@ declare i128 @llvm.fshr.i128(i128, i128, i128) nounwind readnone
define i8 @var_shift_i8(i8 %x, i8 %y, i8 %z) nounwind {
; X86-LABEL: var_shift_i8:
; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movzbl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: shll $8, %eax
-; X86-NEXT: orl %edx, %eax
-; X86-NEXT: andb $7, %cl
-; X86-NEXT: shrl %cl, %eax
+; X86-NEXT: shll $8, %edx
+; X86-NEXT: andb $7, %al
+; X86-NEXT: orl %ecx, %edx
+; X86-NEXT: shrxl %eax, %edx, %eax
; X86-NEXT: # kill: def $al killed $al killed $eax
; X86-NEXT: retl
;
; X64-LABEL: var_shift_i8:
; X64: # %bb.0:
-; X64-NEXT: movl %edx, %ecx
; X64-NEXT: shll $8, %edi
; X64-NEXT: movzbl %sil, %eax
+; X64-NEXT: andb $7, %dl
; X64-NEXT: orl %edi, %eax
-; X64-NEXT: andb $7, %cl
-; X64-NEXT: # kill: def $cl killed $cl killed $ecx
-; X64-NEXT: shrl %cl, %eax
+; X64-NEXT: shrxl %edx, %eax, %eax
; X64-NEXT: # kill: def $al killed $al killed $eax
; X64-NEXT: retq
%tmp = tail call i8 @llvm.fshr.i8(i8 %x, i8 %y, i8 %z)
@@ -45,30 +43,30 @@ define i8 @var_shift_i8(i8 %x, i8 %y, i8 %z) nounwind {
define i16 @var_shift_i16(i16 %x, i16 %y, i16 %z) nounwind {
; X86-FAST-LABEL: var_shift_i16:
; X86-FAST: # %bb.0:
+; X86-FAST-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
; X86-FAST-NEXT: movzwl {{[0-9]+}}(%esp), %edx
; X86-FAST-NEXT: movzwl {{[0-9]+}}(%esp), %eax
-; X86-FAST-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
; X86-FAST-NEXT: andb $15, %cl
; X86-FAST-NEXT: shrdw %cl, %dx, %ax
; X86-FAST-NEXT: retl
;
; X86-SLOW-LABEL: var_shift_i16:
; X86-SLOW: # %bb.0:
-; X86-SLOW-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X86-SLOW-NEXT: movzwl {{[0-9]+}}(%esp), %edx
-; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-SLOW-NEXT: shll $16, %eax
-; X86-SLOW-NEXT: orl %edx, %eax
-; X86-SLOW-NEXT: andb $15, %cl
-; X86-SLOW-NEXT: shrl %cl, %eax
+; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-SLOW-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-SLOW-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; X86-SLOW-NEXT: shll $16, %edx
+; X86-SLOW-NEXT: andb $15, %al
+; X86-SLOW-NEXT: orl %ecx, %edx
+; X86-SLOW-NEXT: shrxl %eax, %edx, %eax
; X86-SLOW-NEXT: # kill: def $ax killed $ax killed $eax
; X86-SLOW-NEXT: retl
;
; X64-FAST-LABEL: var_shift_i16:
; X64-FAST: # %bb.0:
; X64-FAST-NEXT: movl %edx, %ecx
-; X64-FAST-NEXT: movl %esi, %eax
; X64-FAST-NEXT: andb $15, %cl
+; X64-FAST-NEXT: movl %esi, %eax
; X64-FAST-NEXT: # kill: def $cl killed $cl killed $ecx
; X64-FAST-NEXT: shrdw %cl, %di, %ax
; X64-FAST-NEXT: # kill: def $ax killed $ax killed $eax
@@ -76,13 +74,11 @@ define i16 @var_shift_i16(i16 %x, i16 %y, i16 %z) nounwind {
;
; X64-SLOW-LABEL: var_shift_i16:
; X64-SLOW: # %bb.0:
-; X64-SLOW-NEXT: movl %edx, %ecx
; X64-SLOW-NEXT: shll $16, %edi
; X64-SLOW-NEXT: movzwl %si, %eax
+; X64-SLOW-NEXT: andb $15, %dl
; X64-SLOW-NEXT: orl %edi, %eax
-; X64-SLOW-NEXT: andb $15, %cl
-; X64-SLOW-NEXT: # kill: def $cl killed $cl killed $ecx
-; X64-SLOW-NEXT: shrl %cl, %eax
+; X64-SLOW-NEXT: shrxl %edx, %eax, %eax
; X64-SLOW-NEXT: # kill: def $ax killed $ax killed $eax
; X64-SLOW-NEXT: retq
%tmp = tail call i16 @llvm.fshr.i16(i16 %x, i16 %y, i16 %z)
@@ -98,13 +94,21 @@ define i32 @var_shift_i32(i32 %x, i32 %y, i32 %z) nounwind {
; X86-NEXT: shrdl %cl, %edx, %eax
; X86-NEXT: retl
;
-; X64-LABEL: var_shift_i32:
-; X64: # %bb.0:
-; X64-NEXT: movl %edx, %ecx
-; X64-NEXT: movl %esi, %eax
-; X64-NEXT: # kill: def $cl killed $cl killed $ecx
-; X64-NEXT: shrdl %cl, %edi, %eax
-; X64-NEXT: retq
+; X64-FAST-LABEL: var_shift_i32:
+; X64-FAST: # %bb.0:
+; X64-FAST-NEXT: movl %edx, %ecx
+; X64-FAST-NEXT: movl %esi, %eax
+; X64-FAST-NEXT: # kill: def $cl killed $cl killed $ecx
+; X64-FAST-NEXT: shrdl %cl, %edi, %eax
+; X64-FAST-NEXT: retq
+;
+; X64-SLOW-LABEL: var_shift_i32:
+; X64-SLOW: # %bb.0:
+; X64-SLOW-NEXT: movl %esi, %eax
+; X64-SLOW-NEXT: movl %edx, %ecx
+; X64-SLOW-NEXT: # kill: def $cl killed $cl killed $ecx
+; X64-SLOW-NEXT: shrdl %cl, %edi, %eax
+; X64-SLOW-NEXT: retq
%tmp = tail call i32 @llvm.fshr.i32(i32 %x, i32 %y, i32 %z)
ret i32 %tmp
}
@@ -118,13 +122,21 @@ define i32 @var_shift_i32_optsize(i32 %x, i32 %y, i32 %z) nounwind optsize {
; X86-NEXT: shrdl %cl, %edx, %eax
; X86-NEXT: retl
;
-; X64-LABEL: var_shift_i32_optsize:
-; X64: # %bb.0:
-; X64-NEXT: movl %edx, %ecx
-; X64-NEXT: movl %esi, %eax
-; X64-NEXT: # kill: def $cl killed $cl killed $ecx
-; X64-NEXT: shrdl %cl, %edi, %eax
-; X64-NEXT: retq
+; X64-FAST-LABEL: var_shift_i32_optsize:
+; X64-FAST: # %bb.0:
+; X64-FAST-NEXT: movl %edx, %ecx
+; X64-FAST-NEXT: movl %esi, %eax
+; X64-FAST-NEXT: # kill: def $cl killed $cl killed $ecx
+; X64-FAST-NEXT: shrdl %cl, %edi, %eax
+; X64-FAST-NEXT: retq
+;
+; X64-SLOW-LABEL: var_shift_i32_optsize:
+; X64-SLOW: # %bb.0:
+; X64-SLOW-NEXT: movl %esi, %eax
+; X64-SLOW-NEXT: movl %edx, %ecx
+; X64-SLOW-NEXT: # kill: def $cl killed $cl killed $ecx
+; X64-SLOW-NEXT: shrdl %cl, %edi, %eax
+; X64-SLOW-NEXT: retq
%tmp = tail call i32 @llvm.fshr.i32(i32 %x, i32 %y, i32 %z)
ret i32 %tmp
}
@@ -138,112 +150,184 @@ define i32 @var_shift_i32_pgso(i32 %x, i32 %y, i32 %z) nounwind !prof !14 {
; X86-NEXT: shrdl %cl, %edx, %eax
; X86-NEXT: retl
;
-; X64-LABEL: var_shift_i32_pgso:
-; X64: # %bb.0:
-; X64-NEXT: movl %edx, %ecx
-; X64-NEXT: movl %esi, %eax
-; X64-NEXT: # kill: def $cl killed $cl killed $ecx
-; X64-NEXT: shrdl %cl, %edi, %eax
-; X64-NEXT: retq
+; X64-FAST-LABEL: var_shift_i32_pgso:
+; X64-FAST: # %bb.0:
+; X64-FAST-NEXT: movl %edx, %ecx
+; X64-FAST-NEXT: movl %esi, %eax
+; X64-FAST-NEXT: # kill: def $cl killed $cl killed $ecx
+; X64-FAST-NEXT: shrdl %cl, %edi, %eax
+; X64-FAST-NEXT: retq
+;
+; X64-SLOW-LABEL: var_shift_i32_pgso:
+; X64-SLOW: # %bb.0:
+; X64-SLOW-NEXT: movl %esi, %eax
+; X64-SLOW-NEXT: movl %edx, %ecx
+; X64-SLOW-NEXT: # kill: def $cl killed $cl killed $ecx
+; X64-SLOW-NEXT: shrdl %cl, %edi, %eax
+; X64-SLOW-NEXT: retq
%tmp = tail call i32 @llvm.fshr.i32(i32 %x, i32 %y, i32 %z)
ret i32 %tmp
}
define i64 @var_shift_i64(i64 %x, i64 %y, i64 %z) nounwind {
-; X86-LABEL: var_shift_i64:
-; X86: # %bb.0:
-; X86-NEXT: pushl %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: testb $32, %cl
-; X86-NEXT: je .LBB5_1
-; X86-NEXT: # %bb.2:
-; X86-NEXT: movl %esi, %edx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT: jmp .LBB5_3
-; X86-NEXT: .LBB5_1:
-; X86-NEXT: movl %eax, %edx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: .LBB5_3:
-; X86-NEXT: shrdl %cl, %edx, %eax
-; X86-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NEXT: shrdl %cl, %esi, %edx
-; X86-NEXT: popl %esi
-; X86-NEXT: retl
+; X86-FAST-LABEL: var_shift_i64:
+; X86-FAST: # %bb.0:
+; X86-FAST-NEXT: pushl %esi
+; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-FAST-NEXT: testb $32, %cl
+; X86-FAST-NEXT: movl %esi, %edx
+; X86-FAST-NEXT: cmovel %eax, %edx
+; X86-FAST-NEXT: cmovnel {{[0-9]+}}(%esp), %esi
+; X86-FAST-NEXT: cmovel {{[0-9]+}}(%esp), %eax
+; X86-FAST-NEXT: shrdl %cl, %edx, %eax
+; X86-FAST-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-FAST-NEXT: shrdl %cl, %esi, %edx
+; X86-FAST-NEXT: popl %esi
+; X86-FAST-NEXT: retl
;
-; X64-LABEL: var_shift_i64:
-; X64: # %bb.0:
-; X64-NEXT: movq %rdx, %rcx
-; X64-NEXT: movq %rsi, %rax
-; X64-NEXT: # kill: def $cl killed $cl killed $rcx
-; X64-NEXT: shrdq %cl, %rdi, %rax
-; X64-NEXT: retq
+; X86-SLOW-LABEL: var_shift_i64:
+; X86-SLOW: # %bb.0:
+; X86-SLOW-NEXT: pushl %esi
+; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SLOW-NEXT: testb $32, %cl
+; X86-SLOW-NEXT: movl %esi, %edx
+; X86-SLOW-NEXT: cmovel %eax, %edx
+; X86-SLOW-NEXT: cmovnel {{[0-9]+}}(%esp), %esi
+; X86-SLOW-NEXT: cmovel {{[0-9]+}}(%esp), %eax
+; X86-SLOW-NEXT: shrdl %cl, %edx, %eax
+; X86-SLOW-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-SLOW-NEXT: shrdl %cl, %esi, %edx
+; X86-SLOW-NEXT: popl %esi
+; X86-SLOW-NEXT: retl
+;
+; X64-FAST-LABEL: var_shift_i64:
+; X64-FAST: # %bb.0:
+; X64-FAST-NEXT: movq %rdx, %rcx
+; X64-FAST-NEXT: movq %rsi, %rax
+; X64-FAST-NEXT: # kill: def $cl killed $cl killed $rcx
+; X64-FAST-NEXT: shrdq %cl, %rdi, %rax
+; X64-FAST-NEXT: retq
+;
+; X64-SLOW-LABEL: var_shift_i64:
+; X64-SLOW: # %bb.0:
+; X64-SLOW-NEXT: movq %rsi, %rax
+; X64-SLOW-NEXT: movq %rdx, %rcx
+; X64-SLOW-NEXT: # kill: def $cl killed $cl killed $rcx
+; X64-SLOW-NEXT: shrdq %cl, %rdi, %rax
+; X64-SLOW-NEXT: retq
%tmp = tail call i64 @llvm.fshr.i64(i64 %x, i64 %y, i64 %z)
ret i64 %tmp
}
define i128 @var_shift_i128(i128 %x, i128 %y, i128 %z) nounwind {
-; X86-LABEL: var_shift_i128:
-; X86: # %bb.0:
-; X86-NEXT: pushl %ebp
-; X86-NEXT: movl %esp, %ebp
-; X86-NEXT: pushl %ebx
-; X86-NEXT: pushl %edi
-; X86-NEXT: pushl %esi
-; X86-NEXT: andl $-16, %esp
-; X86-NEXT: subl $16, %esp
-; X86-NEXT: movl 24(%ebp), %esi
-; X86-NEXT: movl 28(%ebp), %eax
-; X86-NEXT: movl 48(%ebp), %edx
-; X86-NEXT: movl 56(%ebp), %ecx
-; X86-NEXT: testb $64, %cl
-; X86-NEXT: movl 52(%ebp), %ebx
-; X86-NEXT: je .LBB6_1
-; X86-NEXT: # %bb.2:
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %esi, %edx
-; X86-NEXT: movl 32(%ebp), %esi
-; X86-NEXT: movl %ebx, %edi
-; X86-NEXT: movl %eax, %ebx
-; X86-NEXT: movl 36(%ebp), %eax
-; X86-NEXT: testb $32, %cl
-; X86-NEXT: je .LBB6_4
-; X86-NEXT: jmp .LBB6_5
-; X86-NEXT: .LBB6_1:
-; X86-NEXT: movl 40(%ebp), %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 44(%ebp), %edi
-; X86-NEXT: testb $32, %cl
-; X86-NEXT: jne .LBB6_5
-; X86-NEXT: .LBB6_4:
-; X86-NEXT: movl %esi, %eax
-; X86-NEXT: movl %ebx, %esi
-; X86-NEXT: movl %edx, %ebx
-; X86-NEXT: movl %edi, %edx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: .LBB6_5:
-; X86-NEXT: shrdl %cl, %edx, %edi
-; X86-NEXT: shrdl %cl, %ebx, %edx
-; X86-NEXT: shrdl %cl, %esi, %ebx
-; X86-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NEXT: shrdl %cl, %eax, %esi
-; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: movl %esi, 12(%eax)
-; X86-NEXT: movl %ebx, 8(%eax)
-; X86-NEXT: movl %edx, 4(%eax)
-; X86-NEXT: movl %edi, (%eax)
-; X86-NEXT: leal -12(%ebp), %esp
-; X86-NEXT: popl %esi
-; X86-NEXT: popl %edi
-; X86-NEXT: popl %ebx
-; X86-NEXT: popl %ebp
-; X86-NEXT: retl $4
+; X86-FAST-LABEL: var_shift_i128:
+; X86-FAST: # %bb.0:
+; X86-FAST-NEXT: pushl %ebp
+; X86-FAST-NEXT: movl %esp, %ebp
+; X86-FAST-NEXT: pushl %ebx
+; X86-FAST-NEXT: pushl %edi
+; X86-FAST-NEXT: pushl %esi
+; X86-FAST-NEXT: andl $-16, %esp
+; X86-FAST-NEXT: subl $16, %esp
+; X86-FAST-NEXT: movl 56(%ebp), %eax
+; X86-FAST-NEXT: movl 24(%ebp), %esi
+; X86-FAST-NEXT: movl 48(%ebp), %ecx
+; X86-FAST-NEXT: movl 52(%ebp), %edx
+; X86-FAST-NEXT: testb $64, %al
+; X86-FAST-NEXT: movl %esi, %edi
+; X86-FAST-NEXT: movl 28(%ebp), %eax
+; X86-FAST-NEXT: cmovel %ecx, %edi
+; X86-FAST-NEXT: cmovel 40(%ebp), %ecx
+; X86-FAST-NEXT: cmovnel 32(%ebp), %esi
+; X86-FAST-NEXT: movl %eax, %ebx
+; X86-FAST-NEXT: cmovel %edx, %ebx
+; X86-FAST-NEXT: cmovel 44(%ebp), %edx
+; X86-FAST-NEXT: cmovnel 36(%ebp), %eax
+; X86-FAST-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-FAST-NEXT: movl 56(%ebp), %ecx
+; X86-FAST-NEXT: testb $32, %cl
+; X86-FAST-NEXT: cmovel %esi, %eax
+; X86-FAST-NEXT: cmovel %ebx, %esi
+; X86-FAST-NEXT: cmovel %edi, %ebx
+; X86-FAST-NEXT: cmovel %edx, %edi
+; X86-FAST-NEXT: cmovel {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X86-FAST-NEXT: shrdl %cl, %edi, %edx
+; X86-FAST-NEXT: shrdl %cl, %ebx, %edi
+; X86-FAST-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-FAST-NEXT: shrdl %cl, %esi, %ebx
+; X86-FAST-NEXT: movl 56(%ebp), %ecx
+; X86-FAST-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-FAST-NEXT: shrdl %cl, %eax, %esi
+; X86-FAST-NEXT: movl 8(%ebp), %eax
+; X86-FAST-NEXT: movl %esi, 12(%eax)
+; X86-FAST-NEXT: movl %ebx, 8(%eax)
+; X86-FAST-NEXT: movl %edi, 4(%eax)
+; X86-FAST-NEXT: movl %edx, (%eax)
+; X86-FAST-NEXT: leal -12(%ebp), %esp
+; X86-FAST-NEXT: popl %esi
+; X86-FAST-NEXT: popl %edi
+; X86-FAST-NEXT: popl %ebx
+; X86-FAST-NEXT: popl %ebp
+; X86-FAST-NEXT: retl $4
+;
+; X86-SLOW-LABEL: var_shift_i128:
+; X86-SLOW: # %bb.0:
+; X86-SLOW-NEXT: pushl %ebp
+; X86-SLOW-NEXT: movl %esp, %ebp
+; X86-SLOW-NEXT: pushl %ebx
+; X86-SLOW-NEXT: pushl %edi
+; X86-SLOW-NEXT: pushl %esi
+; X86-SLOW-NEXT: andl $-16, %esp
+; X86-SLOW-NEXT: subl $16, %esp
+; X86-SLOW-NEXT: movl 56(%ebp), %eax
+; X86-SLOW-NEXT: movl 24(%ebp), %esi
+; X86-SLOW-NEXT: movl 48(%ebp), %ecx
+; X86-SLOW-NEXT: movl 52(%ebp), %edx
+; X86-SLOW-NEXT: testb $64, %al
+; X86-SLOW-NEXT: movl %esi, %edi
+; X86-SLOW-NEXT: movl 28(%ebp), %eax
+; X86-SLOW-NEXT: cmovel %ecx, %edi
+; X86-SLOW-NEXT: cmovel 40(%ebp), %ecx
+; X86-SLOW-NEXT: cmovnel 32(%ebp), %esi
+; X86-SLOW-NEXT: movl %eax, %ebx
+; X86-SLOW-NEXT: cmovnel 36(%ebp), %eax
+; X86-SLOW-NEXT: cmovel %edx, %ebx
+; X86-SLOW-NEXT: cmovel 44(%ebp), %edx
+; X86-SLOW-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-SLOW-NEXT: movl 56(%ebp), %ecx
+; X86-SLOW-NEXT: testb $32, %cl
+; X86-SLOW-NEXT: cmovel %esi, %eax
+; X86-SLOW-NEXT: cmovel %ebx, %esi
+; X86-SLOW-NEXT: cmovel %edi, %ebx
+; X86-SLOW-NEXT: cmovel %edx, %edi
+; X86-SLOW-NEXT: cmovel {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X86-SLOW-NEXT: shrdl %cl, %edi, %edx
+; X86-SLOW-NEXT: shrdl %cl, %ebx, %edi
+; X86-SLOW-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-SLOW-NEXT: shrdl %cl, %esi, %ebx
+; X86-SLOW-NEXT: movl 56(%ebp), %ecx
+; X86-SLOW-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-SLOW-NEXT: shrdl %cl, %eax, %esi
+; X86-SLOW-NEXT: movl 8(%ebp), %eax
+; X86-SLOW-NEXT: movl %esi, 12(%eax)
+; X86-SLOW-NEXT: movl %ebx, 8(%eax)
+; X86-SLOW-NEXT: movl %edi, 4(%eax)
+; X86-SLOW-NEXT: movl %edx, (%eax)
+; X86-SLOW-NEXT: leal -12(%ebp), %esp
+; X86-SLOW-NEXT: popl %esi
+; X86-SLOW-NEXT: popl %edi
+; X86-SLOW-NEXT: popl %ebx
+; X86-SLOW-NEXT: popl %ebp
+; X86-SLOW-NEXT: retl $4
;
; X64-LABEL: var_shift_i128:
; X64: # %bb.0:
-; X64-NEXT: movq %rdx, %rax
; X64-NEXT: testb $64, %r8b
+; X64-NEXT: movq %rdx, %rax
; X64-NEXT: cmoveq %rdi, %rsi
; X64-NEXT: cmoveq %rcx, %rdi
; X64-NEXT: cmovneq %rcx, %rax
@@ -320,37 +404,76 @@ define i16 @const_shift_i16(i16 %x, i16 %y) nounwind {
}
define i32 @const_shift_i32(i32 %x, i32 %y) nounwind {
-; X86-LABEL: const_shift_i32:
-; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: shrdl $7, %ecx, %eax
-; X86-NEXT: retl
+; X86-FAST-LABEL: const_shift_i32:
+; X86-FAST: # %bb.0:
+; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-FAST-NEXT: shrdl $7, %ecx, %eax
+; X86-FAST-NEXT: retl
;
-; X64-LABEL: const_shift_i32:
-; X64: # %bb.0:
-; X64-NEXT: movl %edi, %eax
-; X64-NEXT: shldl $25, %esi, %eax
-; X64-NEXT: retq
+; X86-SLOW-LABEL: const_shift_i32:
+; X86-SLOW: # %bb.0:
+; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SLOW-NEXT: shrl $7, %ecx
+; X86-SLOW-NEXT: shll $25, %eax
+; X86-SLOW-NEXT: addl %ecx, %eax
+; X86-SLOW-NEXT: retl
+;
+; X64-FAST-LABEL: const_shift_i32:
+; X64-FAST: # %bb.0:
+; X64-FAST-NEXT: movl %edi, %eax
+; X64-FAST-NEXT: shldl $25, %esi, %eax
+; X64-FAST-NEXT: retq
+;
+; X64-SLOW-LABEL: const_shift_i32:
+; X64-SLOW: # %bb.0:
+; X64-SLOW-NEXT: shrl $7, %esi
+; X64-SLOW-NEXT: shll $25, %edi
+; X64-SLOW-NEXT: leal (%esi,%edi), %eax
+; X64-SLOW-NEXT: retq
%tmp = tail call i32 @llvm.fshr.i32(i32 %x, i32 %y, i32 7)
ret i32 %tmp
}
define i64 @const_shift_i64(i64 %x, i64 %y) nounwind {
-; X86-LABEL: const_shift_i64:
-; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: shldl $25, %ecx, %edx
-; X86-NEXT: shrdl $7, %ecx, %eax
-; X86-NEXT: retl
+; X86-FAST-LABEL: const_shift_i64:
+; X86-FAST: # %bb.0:
+; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-FAST-NEXT: shldl $25, %ecx, %edx
+; X86-FAST-NEXT: shrdl $7, %ecx, %eax
+; X86-FAST-NEXT: retl
;
-; X64-LABEL: const_shift_i64:
-; X64: # %bb.0:
-; X64-NEXT: movq %rdi, %rax
-; X64-NEXT: shldq $57, %rsi, %rax
-; X64-NEXT: retq
+; X86-SLOW-LABEL: const_shift_i64:
+; X86-SLOW: # %bb.0:
+; X86-SLOW-NEXT: pushl %esi
+; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-SLOW-NEXT: movl %esi, %edx
+; X86-SLOW-NEXT: shll $25, %eax
+; X86-SLOW-NEXT: shrl $7, %ecx
+; X86-SLOW-NEXT: shll $25, %esi
+; X86-SLOW-NEXT: shrl $7, %edx
+; X86-SLOW-NEXT: addl %eax, %edx
+; X86-SLOW-NEXT: leal (%ecx,%esi), %eax
+; X86-SLOW-NEXT: popl %esi
+; X86-SLOW-NEXT: retl
+;
+; X64-FAST-LABEL: const_shift_i64:
+; X64-FAST: # %bb.0:
+; X64-FAST-NEXT: movq %rdi, %rax
+; X64-FAST-NEXT: shldq $57, %rsi, %rax
+; X64-FAST-NEXT: retq
+;
+; X64-SLOW-LABEL: const_shift_i64:
+; X64-SLOW: # %bb.0:
+; X64-SLOW-NEXT: shrq $7, %rsi
+; X64-SLOW-NEXT: shlq $57, %rdi
+; X64-SLOW-NEXT: leaq (%rsi,%rdi), %rax
+; X64-SLOW-NEXT: retq
%tmp = tail call i64 @llvm.fshr.i64(i64 %x, i64 %y, i64 7)
ret i64 %tmp
}
diff --git a/llvm/test/CodeGen/X86/load-local-v3i129.ll b/llvm/test/CodeGen/X86/load-local-v3i129.ll
index e9a1407df488c..d58b74c8ec983 100644
--- a/llvm/test/CodeGen/X86/load-local-v3i129.ll
+++ b/llvm/test/CodeGen/X86/load-local-v3i129.ll
@@ -1,16 +1,16 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --no_x86_scrub_sp
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+slow-shld | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver3 | FileCheck %s --check-prefixes=CHECK,FAST
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver2 | FileCheck %s --check-prefixes=CHECK,SLOW
define void @_start() nounwind {
; CHECK-LABEL: _start:
; CHECK: # %bb.0: # %Entry
; CHECK-NEXT: movq -40(%rsp), %rax
+; CHECK-NEXT: orq $-2, -56(%rsp)
+; CHECK-NEXT: movq $-1, -48(%rsp)
; CHECK-NEXT: andq $-4, %rax
; CHECK-NEXT: incq %rax
; CHECK-NEXT: movq %rax, -40(%rsp)
-; CHECK-NEXT: orq $-2, -56(%rsp)
-; CHECK-NEXT: movq $-1, -48(%rsp)
; CHECK-NEXT: retq
Entry:
%y = alloca <3 x i129>, align 16
@@ -19,3 +19,6 @@ Entry:
store <3 x i129> %I1, ptr %y
ret void
}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; FAST: {{.*}}
+; SLOW: {{.*}}
diff --git a/llvm/test/CodeGen/X86/pr32282.ll b/llvm/test/CodeGen/X86/pr32282.ll
index 54b8b5df0e415..01e194ec73139 100644
--- a/llvm/test/CodeGen/X86/pr32282.ll
+++ b/llvm/test/CodeGen/X86/pr32282.ll
@@ -22,11 +22,11 @@ define dso_local void @foo(i64 %x) nounwind {
; X86-NEXT: shrl $21, %ecx
; X86-NEXT: movl %eax, %edx
; X86-NEXT: shll $11, %edx
-; X86-NEXT: orl %ecx, %edx
+; X86-NEXT: addl %edx, %ecx
; X86-NEXT: shrl $21, %eax
-; X86-NEXT: addl $7, %edx
+; X86-NEXT: addl $7, %ecx
; X86-NEXT: pushl %eax
-; X86-NEXT: pushl %edx
+; X86-NEXT: pushl %ecx
; X86-NEXT: pushl {{[0-9]+}}(%esp)
; X86-NEXT: pushl {{[0-9]+}}(%esp)
; X86-NEXT: calll __divdi3
diff --git a/llvm/test/CodeGen/X86/rot32.ll b/llvm/test/CodeGen/X86/rot32.ll
index 8da0e0e6d23cd..95da4eba415b7 100644
--- a/llvm/test/CodeGen/X86/rot32.ll
+++ b/llvm/test/CodeGen/X86/rot32.ll
@@ -207,7 +207,7 @@ define i32 @xbar(i32 %x, i32 %y, i32 %z) nounwind readnone {
; BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
; BMI2-NEXT: shll $7, %ecx
; BMI2-NEXT: shrl $25, %eax
-; BMI2-NEXT: orl %ecx, %eax
+; BMI2-NEXT: addl %ecx, %eax
; BMI2-NEXT: retl
;
; X64-LABEL: xbar:
@@ -224,10 +224,9 @@ define i32 @xbar(i32 %x, i32 %y, i32 %z) nounwind readnone {
;
; BMI264-LABEL: xbar:
; BMI264: # %bb.0: # %entry
-; BMI264-NEXT: movl %edi, %eax
; BMI264-NEXT: shll $7, %esi
-; BMI264-NEXT: shrl $25, %eax
-; BMI264-NEXT: orl %esi, %eax
+; BMI264-NEXT: shrl $25, %edi
+; BMI264-NEXT: leal (%esi,%edi), %eax
; BMI264-NEXT: retq
entry:
%0 = shl i32 %y, 7
@@ -343,7 +342,7 @@ define i32 @xbu(i32 %x, i32 %y, i32 %z) nounwind readnone {
; BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
; BMI2-NEXT: shll $25, %ecx
; BMI2-NEXT: shrl $7, %eax
-; BMI2-NEXT: orl %ecx, %eax
+; BMI2-NEXT: addl %ecx, %eax
; BMI2-NEXT: retl
;
; X64-LABEL: xbu:
@@ -360,10 +359,9 @@ define i32 @xbu(i32 %x, i32 %y, i32 %z) nounwind readnone {
;
; BMI264-LABEL: xbu:
; BMI264: # %bb.0: # %entry
-; BMI264-NEXT: movl %esi, %eax
; BMI264-NEXT: shll $25, %edi
-; BMI264-NEXT: shrl $7, %eax
-; BMI264-NEXT: orl %edi, %eax
+; BMI264-NEXT: shrl $7, %esi
+; BMI264-NEXT: leal (%edi,%esi), %eax
; BMI264-NEXT: retq
entry:
%0 = lshr i32 %y, 7
diff --git a/llvm/test/CodeGen/X86/rot64.ll b/llvm/test/CodeGen/X86/rot64.ll
index 526ed08ef45a1..c03e31ef50985 100644
--- a/llvm/test/CodeGen/X86/rot64.ll
+++ b/llvm/test/CodeGen/X86/rot64.ll
@@ -131,10 +131,9 @@ define i64 @xbar(i64 %x, i64 %y, i64 %z) nounwind readnone {
;
; BMI2-LABEL: xbar:
; BMI2: # %bb.0: # %entry
-; BMI2-NEXT: movq %rdi, %rax
; BMI2-NEXT: shlq $7, %rsi
-; BMI2-NEXT: shrq $57, %rax
-; BMI2-NEXT: orq %rsi, %rax
+; BMI2-NEXT: shrq $57, %rdi
+; BMI2-NEXT: leaq (%rsi,%rdi), %rax
; BMI2-NEXT: retq
entry:
%0 = shl i64 %y, 7
@@ -207,10 +206,9 @@ define i64 @xbu(i64 %x, i64 %y, i64 %z) nounwind readnone {
;
; BMI2-LABEL: xbu:
; BMI2: # %bb.0: # %entry
-; BMI2-NEXT: movq %rsi, %rax
; BMI2-NEXT: shlq $57, %rdi
-; BMI2-NEXT: shrq $7, %rax
-; BMI2-NEXT: orq %rdi, %rax
+; BMI2-NEXT: shrq $7, %rsi
+; BMI2-NEXT: leaq (%rdi,%rsi), %rax
; BMI2-NEXT: retq
entry:
%0 = lshr i64 %y, 7
diff --git a/llvm/test/CodeGen/X86/scmp.ll b/llvm/test/CodeGen/X86/scmp.ll
index aef81841a458c..d89be0f6d587c 100644
--- a/llvm/test/CodeGen/X86/scmp.ll
+++ b/llvm/test/CodeGen/X86/scmp.ll
@@ -2981,11 +2981,11 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; AVX512-NEXT: pushq %r13
; AVX512-NEXT: pushq %r12
; AVX512-NEXT: pushq %rbx
-; AVX512-NEXT: movq %rdi, %r15
+; AVX512-NEXT: movq %rdi, %r12
; AVX512-NEXT: movzbl {{[0-9]+}}(%rsp), %edi
; AVX512-NEXT: movzbl {{[0-9]+}}(%rsp), %r11d
; AVX512-NEXT: movzbl {{[0-9]+}}(%rsp), %ebp
-; AVX512-NEXT: movzbl {{[0-9]+}}(%rsp), %eax
+; AVX512-NEXT: movzbl {{[0-9]+}}(%rsp), %r15d
; AVX512-NEXT: movzbl {{[0-9]+}}(%rsp), %ebx
; AVX512-NEXT: movzbl {{[0-9]+}}(%rsp), %r14d
; AVX512-NEXT: movzbl {{[0-9]+}}(%rsp), %r10d
@@ -2998,7 +2998,7 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; AVX512-NEXT: setg %r10b
; AVX512-NEXT: subb %sil, %r10b
; AVX512-NEXT: movsbq %r10b, %r10
-; AVX512-NEXT: movq %r10, (%r15)
+; AVX512-NEXT: movq %r10, (%r12)
; AVX512-NEXT: sarq $63, %r10
; AVX512-NEXT: movq %r10, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX512-NEXT: addb %r14b, %r14b
@@ -3022,19 +3022,19 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; AVX512-NEXT: setg %bl
; AVX512-NEXT: subb %dl, %bl
; AVX512-NEXT: movsbq %bl, %rbx
-; AVX512-NEXT: movq %rbx, %r12
-; AVX512-NEXT: sarq $63, %r12
-; AVX512-NEXT: addb %al, %al
-; AVX512-NEXT: sarb %al
+; AVX512-NEXT: movq %rbx, %rsi
+; AVX512-NEXT: sarq $63, %rsi
+; AVX512-NEXT: addb %r15b, %r15b
+; AVX512-NEXT: sarb %r15b
; AVX512-NEXT: addb %cl, %cl
; AVX512-NEXT: sarb %cl
-; AVX512-NEXT: cmpb %al, %cl
+; AVX512-NEXT: cmpb %r15b, %cl
; AVX512-NEXT: setl %cl
; AVX512-NEXT: setg %dl
; AVX512-NEXT: subb %cl, %dl
-; AVX512-NEXT: movsbq %dl, %rcx
-; AVX512-NEXT: movq %rcx, %rsi
-; AVX512-NEXT: sarq $63, %rsi
+; AVX512-NEXT: movsbq %dl, %r15
+; AVX512-NEXT: movq %r15, %rcx
+; AVX512-NEXT: sarq $63, %rcx
; AVX512-NEXT: addb %bpl, %bpl
; AVX512-NEXT: sarb %bpl
; AVX512-NEXT: addb %r8b, %r8b
@@ -3069,73 +3069,73 @@ define <7 x i117> @scmp_uncommon_vectors(<7 x i7> %x, <7 x i7> %y) nounwind {
; AVX512-NEXT: movsbq %bpl, %rax
; AVX512-NEXT: movq %rax, %rbp
; AVX512-NEXT: sarq $63, %rbp
-; AVX512-NEXT: movl %ebp, 96(%r15)
+; AVX512-NEXT: movl %ebp, 96(%r12)
; AVX512-NEXT: movb $51, %r13b
; AVX512-NEXT: bzhiq %r13, %rbp, %r13
; AVX512-NEXT: shlq $62, %rbp
; AVX512-NEXT: movq %rax, %rdi
; AVX512-NEXT: shrq $2, %rdi
-; AVX512-NEXT: orq %rbp, %rdi
-; AVX512-NEXT: movq %rdi, 88(%r15)
+; AVX512-NEXT: addq %rbp, %rdi
+; AVX512-NEXT: movq %rdi, 88(%r12)
; AVX512-NEXT: movq %r9, %rdi
; AVX512-NEXT: shrq $44, %rdi
; AVX512-NEXT: movq %r11, %rbp
; AVX512-NEXT: shlq $20, %rbp
-; AVX512-NEXT: orq %rdi, %rbp
-; AVX512-NEXT: movq %rbp, 64(%r15)
+; AVX512-NEXT: addq %rbp, %rdi
+; AVX512-NEXT: movq %rdi, 64(%r12)
; AVX512-NEXT: movq %rdx, %rdi
; AVX512-NEXT: shrq $33, %rdi
; AVX512-NEXT: movq %r8, %rbp
; AVX512-NEXT: shlq $31, %rbp
-; AVX512-NEXT: orq %rdi, %rbp
-; AVX512-NEXT: movq %rbp, 48(%r15)
-; AVX512-NEXT: movq %rcx, %rdi
+; AVX512-NEXT: addq %rbp, %rdi
+; AVX512-NEXT: movq %rdi, 48(%r12)
+; AVX512-NEXT: movq %r15, %rdi
; AVX512-NEXT: shrq $22, %rdi
-; AVX512-NEXT: movq %rsi, %rbp
+; AVX512-NEXT: movq %rcx, %rbp
; AVX512-NEXT: shlq $42, %rbp
-; AVX512-NEXT: orq %rdi, %rbp
-; AVX512-NEXT: movq %rbp, 32(%r15)
+; AVX512-NEXT: addq %rbp, %rdi
+; AVX512-NEXT: movq %rdi, 32(%r12)
; AVX512-NEXT: movq %rbx, %rdi
; AVX512-NEXT: shrq $11, %rdi
; AVX512-NEXT: movb $42, %bpl
-; AVX512-NEXT: bzhiq %rbp, %r12, %rbp
-; AVX512-NEXT: shlq $53, %r12
-; AVX512-NEXT: orq %rdi, %r12
-; AVX512-NEXT: movq %r12, 16(%r15)
-; AVX512-NEXT: movq %r13, %rdi
-; AVX512-NEXT: shrq $48, %rdi
-; AVX512-NEXT: movb %dil, 102(%r15)
+; AVX512-NEXT: bzhiq %rbp, %rsi, %rbp
+; AVX512-NEXT: shlq $53, %rsi
+; AVX512-NEXT: addq %rdi, %rsi
+; AVX512-NEXT: movq %rsi, 16(%r12)
+; AVX512-NEXT: movq %r13, %rsi
+; AVX512-NEXT: shrq $48, %rsi
+; AVX512-NEXT: movb %sil, 102(%r12)
; AVX512-NEXT: shrq $32, %r13
-; AVX512-NEXT: movw %r13w, 100(%r15)
-; AVX512-NEXT: movb $53, %dil
-; AVX512-NEXT: bzhiq %rdi, %r10, %r10
-; AVX512-NEXT: movq %r14, %r12
-; AVX512-NEXT: shrq $55, %r12
-; AVX512-NEXT: shlq $9, %r10
-; AVX512-NEXT: orq %r12, %r10
+; AVX512-NEXT: movw %r13w, 100(%r12)
+; AVX512-NEXT: movb $53, %sil
+; AVX512-NEXT: bzhiq %rsi, %r10, %rdi
+; AVX512-NEXT: movq %r14, %r10
+; AVX512-NEXT: shrq $55, %r10
+; AVX512-NEXT: shlq $9, %rdi
+; AVX512-NEXT: addq %r10, %rdi
; AVX512-NEXT: shlq $62, %rax
-; AVX512-NEXT: orq %r10, %rax
-; AVX512-NEXT: movq %rax, 80(%r15)
-; AVX512-NEXT: shlq $42, %rcx
-; AVX512-NEXT: orq %rbp, %rcx
-; AVX512-NEXT: movq %rcx, 24(%r15)
-; AVX512-NEXT: bzhiq %rdi, {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Folded Reload
+; AVX512-NEXT: orq %rdi, %rax
+; AVX512-NEXT: movq %rax, 80(%r12)
+; AVX512-NEXT: shlq $42, %r15
+; AVX512-NEXT: orq %rbp, %r15
+; AVX512-NEXT: movq %r15, 24(%r12)
+; AVX512-NEXT: bzhiq %rsi, {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Folded Reload
; AVX512-NEXT: shlq $53, %rbx
; AVX512-NEXT: orq %rax, %rbx
-; AVX512-NEXT: movq %rbx, 8(%r15)
+; AVX512-NEXT: movq %rbx, 8(%r12)
; AVX512-NEXT: shlq $9, %r14
; AVX512-NEXT: andl $511, %r11d # imm = 0x1FF
; AVX512-NEXT: orq %r14, %r11
-; AVX512-NEXT: movq %r11, 72(%r15)
+; AVX512-NEXT: movq %r11, 72(%r12)
; AVX512-NEXT: shlq $20, %r9
; AVX512-NEXT: andl $1048575, %r8d # imm = 0xFFFFF
; AVX512-NEXT: orq %r9, %r8
-; AVX512-NEXT: movq %r8, 56(%r15)
+; AVX512-NEXT: movq %r8, 56(%r12)
; AVX512-NEXT: shlq $31, %rdx
-; AVX512-NEXT: andl $2147483647, %esi # imm = 0x7FFFFFFF
-; AVX512-NEXT: orq %rdx, %rsi
-; AVX512-NEXT: movq %rsi, 40(%r15)
-; AVX512-NEXT: movq %r15, %rax
+; AVX512-NEXT: andl $2147483647, %ecx # imm = 0x7FFFFFFF
+; AVX512-NEXT: orq %rdx, %rcx
+; AVX512-NEXT: movq %rcx, 40(%r12)
+; AVX512-NEXT: movq %r12, %rax
; AVX512-NEXT: popq %rbx
; AVX512-NEXT: popq %r12
; AVX512-NEXT: popq %r13
diff --git a/llvm/test/CodeGen/X86/shd-machine-combiner.mir b/llvm/test/CodeGen/X86/shd-machine-combiner.mir
index 484f6896372dd..7d984e65eb30d 100644
--- a/llvm/test/CodeGen/X86/shd-machine-combiner.mir
+++ b/llvm/test/CodeGen/X86/shd-machine-combiner.mir
@@ -1,15 +1,16 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
# RUN: llc -mtriple=x86_64-gnu-linux -mcpu=sandybridge -run-pass=machine-combiner %s -o - | FileCheck %s --check-prefixes=FAST-SHLD
-# RUN: llc -mtriple=x86_64-gnu-linux -mcpu=znver1 -run-pass=machine-combiner %s -o - | FileCheck %s --check-prefixes=SLOW-SHLD
+# RUN: llc -mtriple=x86_64-gnu-linux -mcpu=znver1 -run-pass=machine-combiner %s -o - | FileCheck %s --check-prefixes=SLOW-SHLD
+# RUN: llc -mtriple=x86_64-gnu-linux -mcpu=btver2 -run-pass=machine-combiner %s -o - | FileCheck %s --check-prefixes=SLOW-SHLD-SLOW-LEA
---
-name: rri32
+name: shld_rri32_small
alignment: 16
tracksRegLiveness: true
body: |
bb.0:
liveins: $eax, $ebx
- ; FAST-SHLD-LABEL: name: rri32
+ ; FAST-SHLD-LABEL: name: shld_rri32_small
; FAST-SHLD: liveins: $eax, $ebx
; FAST-SHLD-NEXT: {{ $}}
; FAST-SHLD-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $eax
@@ -18,151 +19,422 @@ body: |
; FAST-SHLD-NEXT: $eax = COPY [[SHLD32rri8_]]
; FAST-SHLD-NEXT: RET 0, implicit $rax
;
- ; SLOW-SHLD-LABEL: name: rri32
+ ; SLOW-SHLD-LABEL: name: shld_rri32_small
; SLOW-SHLD: liveins: $eax, $ebx
; SLOW-SHLD-NEXT: {{ $}}
; SLOW-SHLD-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $eax
; SLOW-SHLD-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY $ebx
- ; SLOW-SHLD-NEXT: [[SHL32ri:%[0-9]+]]:gr32 = SHL32ri [[COPY]], 2, implicit-def $eflags
; SLOW-SHLD-NEXT: [[SHR32ri:%[0-9]+]]:gr32 = SHR32ri [[COPY1]], 30, implicit-def $eflags
- ; SLOW-SHLD-NEXT: [[OR32rr:%[0-9]+]]:gr32 = OR32rr killed [[SHL32ri]], killed [[SHR32ri]], implicit-def $eflags
- ; SLOW-SHLD-NEXT: $eax = COPY [[OR32rr]]
+ ; SLOW-SHLD-NEXT: [[LEA32r:%[0-9]+]]:gr32 = LEA32r killed [[SHR32ri]], 4, [[COPY]], 0, $noreg
+ ; SLOW-SHLD-NEXT: $eax = COPY [[LEA32r]]
; SLOW-SHLD-NEXT: RET 0, implicit $rax
- %0:gr32 = COPY $eax
- %1:gr32 = COPY $ebx
- %2:gr32 = SHLD32rri8 %0, %1, 2, implicit-def $eflags
+ ;
+ ; SLOW-SHLD-SLOW-LEA-LABEL: name: shld_rri32_small
+ ; SLOW-SHLD-SLOW-LEA: liveins: $eax, $ebx
+ ; SLOW-SHLD-SLOW-LEA-NEXT: {{ $}}
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $eax
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY $ebx
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[SHL32ri:%[0-9]+]]:gr32 = SHL32ri [[COPY]], 2, implicit-def $eflags
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[SHR32ri:%[0-9]+]]:gr32 = SHR32ri [[COPY1]], 30, implicit-def $eflags
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[OR32rr:%[0-9]+]]:gr32 = OR32rr killed [[SHL32ri]], killed [[SHR32ri]], implicit-def $eflags
+ ; SLOW-SHLD-SLOW-LEA-NEXT: $eax = COPY [[OR32rr]]
+ ; SLOW-SHLD-SLOW-LEA-NEXT: RET 0, implicit $rax
+ %1:gr32 = COPY $eax
+ %0:gr32 = COPY $ebx
+ %2:gr32 = SHLD32rri8 %1, %0, 2, implicit-def $eflags
$eax = COPY %2
RET 0, implicit $rax
...
+
---
-name: rri64
+name: shld_rri32_mid
alignment: 16
tracksRegLiveness: true
body: |
bb.0:
- liveins: $rax, $rbx
- ; FAST-SHLD-LABEL: name: rri64
- ; FAST-SHLD: liveins: $rax, $rbx
+ liveins: $eax, $ebx
+ ; FAST-SHLD-LABEL: name: shld_rri32_mid
+ ; FAST-SHLD: liveins: $eax, $ebx
; FAST-SHLD-NEXT: {{ $}}
- ; FAST-SHLD-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rax
- ; FAST-SHLD-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rbx
- ; FAST-SHLD-NEXT: [[SHLD64rri8_:%[0-9]+]]:gr64 = SHLD64rri8 [[COPY]], [[COPY1]], 2, implicit-def $eflags
- ; FAST-SHLD-NEXT: $rax = COPY [[SHLD64rri8_]]
+ ; FAST-SHLD-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $eax
+ ; FAST-SHLD-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY $ebx
+ ; FAST-SHLD-NEXT: [[SHLD32rri8_:%[0-9]+]]:gr32 = SHLD32rri8 [[COPY]], [[COPY1]], 16, implicit-def $eflags
+ ; FAST-SHLD-NEXT: $eax = COPY [[SHLD32rri8_]]
; FAST-SHLD-NEXT: RET 0, implicit $rax
;
- ; SLOW-SHLD-LABEL: name: rri64
- ; SLOW-SHLD: liveins: $rax, $rbx
+ ; SLOW-SHLD-LABEL: name: shld_rri32_mid
+ ; SLOW-SHLD: liveins: $eax, $ebx
; SLOW-SHLD-NEXT: {{ $}}
- ; SLOW-SHLD-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rax
- ; SLOW-SHLD-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rbx
- ; SLOW-SHLD-NEXT: [[SHL64ri:%[0-9]+]]:gr64 = SHL64ri [[COPY]], 2, implicit-def $eflags
- ; SLOW-SHLD-NEXT: [[SHR64ri:%[0-9]+]]:gr64 = SHR64ri [[COPY1]], 62, implicit-def $eflags
- ; SLOW-SHLD-NEXT: [[OR64rr:%[0-9]+]]:gr64 = OR64rr killed [[SHL64ri]], killed [[SHR64ri]], implicit-def $eflags
- ; SLOW-SHLD-NEXT: $rax = COPY [[OR64rr]]
+ ; SLOW-SHLD-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $eax
+ ; SLOW-SHLD-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY $ebx
+ ; SLOW-SHLD-NEXT: [[SHL32ri:%[0-9]+]]:gr32 = SHL32ri [[COPY]], 16, implicit-def $eflags
+ ; SLOW-SHLD-NEXT: [[SHR32ri:%[0-9]+]]:gr32 = SHR32ri [[COPY1]], 16, implicit-def $eflags
+ ; SLOW-SHLD-NEXT: [[LEA32r:%[0-9]+]]:gr32 = LEA32r killed [[SHL32ri]], 1, killed [[SHR32ri]], 0, $noreg
+ ; SLOW-SHLD-NEXT: $eax = COPY [[LEA32r]]
; SLOW-SHLD-NEXT: RET 0, implicit $rax
- %0:gr64 = COPY $rax
- %1:gr64 = COPY $rbx
- %2:gr64 = SHLD64rri8 %0, %1, 2, implicit-def $eflags
- $rax = COPY %2
+ ;
+ ; SLOW-SHLD-SLOW-LEA-LABEL: name: shld_rri32_mid
+ ; SLOW-SHLD-SLOW-LEA: liveins: $eax, $ebx
+ ; SLOW-SHLD-SLOW-LEA-NEXT: {{ $}}
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $eax
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY $ebx
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[SHL32ri:%[0-9]+]]:gr32 = SHL32ri [[COPY]], 16, implicit-def $eflags
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[SHR32ri:%[0-9]+]]:gr32 = SHR32ri [[COPY1]], 16, implicit-def $eflags
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[LEA32r:%[0-9]+]]:gr32 = LEA32r killed [[SHL32ri]], 1, killed [[SHR32ri]], 0, $noreg
+ ; SLOW-SHLD-SLOW-LEA-NEXT: $eax = COPY [[LEA32r]]
+ ; SLOW-SHLD-SLOW-LEA-NEXT: RET 0, implicit $rax
+ %1:gr32 = COPY $eax
+ %0:gr32 = COPY $ebx
+ %2:gr32 = SHLD32rri8 %1, %0, 16, implicit-def $eflags
+ $eax = COPY %2
+ RET 0, implicit $rax
+...
+
+
+---
+name: shrd_rri32_small
+alignment: 16
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $eax, $ebx
+ ; FAST-SHLD-LABEL: name: shrd_rri32_small
+ ; FAST-SHLD: liveins: $eax, $ebx
+ ; FAST-SHLD-NEXT: {{ $}}
+ ; FAST-SHLD-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $eax
+ ; FAST-SHLD-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY $ebx
+ ; FAST-SHLD-NEXT: [[SHRD32rri8_:%[0-9]+]]:gr32 = SHRD32rri8 [[COPY]], [[COPY1]], 2, implicit-def $eflags
+ ; FAST-SHLD-NEXT: $eax = COPY [[SHRD32rri8_]]
+ ; FAST-SHLD-NEXT: RET 0, implicit $rax
+ ;
+ ; SLOW-SHLD-LABEL: name: shrd_rri32_small
+ ; SLOW-SHLD: liveins: $eax, $ebx
+ ; SLOW-SHLD-NEXT: {{ $}}
+ ; SLOW-SHLD-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $eax
+ ; SLOW-SHLD-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY $ebx
+ ; SLOW-SHLD-NEXT: [[SHR32ri:%[0-9]+]]:gr32 = SHR32ri [[COPY]], 2, implicit-def $eflags
+ ; SLOW-SHLD-NEXT: [[SHL32ri:%[0-9]+]]:gr32 = SHL32ri [[COPY1]], 30, implicit-def $eflags
+ ; SLOW-SHLD-NEXT: [[LEA32r:%[0-9]+]]:gr32 = LEA32r killed [[SHR32ri]], 1, killed [[SHL32ri]], 0, $noreg
+ ; SLOW-SHLD-NEXT: $eax = COPY [[LEA32r]]
+ ; SLOW-SHLD-NEXT: RET 0, implicit $rax
+ ;
+ ; SLOW-SHLD-SLOW-LEA-LABEL: name: shrd_rri32_small
+ ; SLOW-SHLD-SLOW-LEA: liveins: $eax, $ebx
+ ; SLOW-SHLD-SLOW-LEA-NEXT: {{ $}}
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $eax
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY $ebx
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[SHR32ri:%[0-9]+]]:gr32 = SHR32ri [[COPY]], 2, implicit-def $eflags
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[SHL32ri:%[0-9]+]]:gr32 = SHL32ri [[COPY1]], 30, implicit-def $eflags
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[LEA32r:%[0-9]+]]:gr32 = LEA32r killed [[SHR32ri]], 1, killed [[SHL32ri]], 0, $noreg
+ ; SLOW-SHLD-SLOW-LEA-NEXT: $eax = COPY [[LEA32r]]
+ ; SLOW-SHLD-SLOW-LEA-NEXT: RET 0, implicit $rax
+ %1:gr32 = COPY $eax
+ %0:gr32 = COPY $ebx
+ %2:gr32 = SHRD32rri8 %1, %0, 2, implicit-def $eflags
+ $eax = COPY %2
+ RET 0, implicit $rax
+...
+
+
+---
+name: shrd_rri32_mid
+alignment: 16
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $eax, $ebx
+ ; FAST-SHLD-LABEL: name: shrd_rri32_mid
+ ; FAST-SHLD: liveins: $eax, $ebx
+ ; FAST-SHLD-NEXT: {{ $}}
+ ; FAST-SHLD-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $eax
+ ; FAST-SHLD-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY $ebx
+ ; FAST-SHLD-NEXT: [[SHRD32rri8_:%[0-9]+]]:gr32 = SHRD32rri8 [[COPY]], [[COPY1]], 16, implicit-def $eflags
+ ; FAST-SHLD-NEXT: $eax = COPY [[SHRD32rri8_]]
+ ; FAST-SHLD-NEXT: RET 0, implicit $rax
+ ;
+ ; SLOW-SHLD-LABEL: name: shrd_rri32_mid
+ ; SLOW-SHLD: liveins: $eax, $ebx
+ ; SLOW-SHLD-NEXT: {{ $}}
+ ; SLOW-SHLD-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $eax
+ ; SLOW-SHLD-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY $ebx
+ ; SLOW-SHLD-NEXT: [[SHR32ri:%[0-9]+]]:gr32 = SHR32ri [[COPY]], 16, implicit-def $eflags
+ ; SLOW-SHLD-NEXT: [[SHL32ri:%[0-9]+]]:gr32 = SHL32ri [[COPY1]], 16, implicit-def $eflags
+ ; SLOW-SHLD-NEXT: [[LEA32r:%[0-9]+]]:gr32 = LEA32r killed [[SHR32ri]], 1, killed [[SHL32ri]], 0, $noreg
+ ; SLOW-SHLD-NEXT: $eax = COPY [[LEA32r]]
+ ; SLOW-SHLD-NEXT: RET 0, implicit $rax
+ ;
+ ; SLOW-SHLD-SLOW-LEA-LABEL: name: shrd_rri32_mid
+ ; SLOW-SHLD-SLOW-LEA: liveins: $eax, $ebx
+ ; SLOW-SHLD-SLOW-LEA-NEXT: {{ $}}
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $eax
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY $ebx
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[SHR32ri:%[0-9]+]]:gr32 = SHR32ri [[COPY]], 16, implicit-def $eflags
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[SHL32ri:%[0-9]+]]:gr32 = SHL32ri [[COPY1]], 16, implicit-def $eflags
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[LEA32r:%[0-9]+]]:gr32 = LEA32r killed [[SHR32ri]], 1, killed [[SHL32ri]], 0, $noreg
+ ; SLOW-SHLD-SLOW-LEA-NEXT: $eax = COPY [[LEA32r]]
+ ; SLOW-SHLD-SLOW-LEA-NEXT: RET 0, implicit $rax
+ %1:gr32 = COPY $eax
+ %0:gr32 = COPY $ebx
+ %2:gr32 = SHRD32rri8 %1, %0, 16, implicit-def $eflags
+ $eax = COPY %2
RET 0, implicit $rax
...
+
---
-name: mri32
+name: shld_mri32_small
alignment: 16
tracksRegLiveness: true
body: |
bb.0:
liveins: $rax, $ebx
- ; FAST-SHLD-LABEL: name: mri32
+
+ ; FAST-SHLD-LABEL: name: shld_mri32_small
; FAST-SHLD: liveins: $rax, $ebx
; FAST-SHLD-NEXT: {{ $}}
; FAST-SHLD-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $ebx
; FAST-SHLD-NEXT: SHLD32mri8 $rax, 1, $noreg, 0, $noreg, [[COPY]], 2, implicit-def $eflags
- ; FAST-SHLD-NEXT: RET 0
+ ; FAST-SHLD-NEXT: RET 0, implicit $rax
;
- ; SLOW-SHLD-LABEL: name: mri32
+ ; SLOW-SHLD-LABEL: name: shld_mri32_small
; SLOW-SHLD: liveins: $rax, $ebx
; SLOW-SHLD-NEXT: {{ $}}
; SLOW-SHLD-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $ebx
; SLOW-SHLD-NEXT: SHL32mi $rax, 1, $noreg, 0, $noreg, 2, implicit-def $eflags
; SLOW-SHLD-NEXT: [[SHR32ri:%[0-9]+]]:gr32 = SHR32ri [[COPY]], 30, implicit-def $eflags
; SLOW-SHLD-NEXT: OR32mr $rax, 1, $noreg, 0, $noreg, killed [[SHR32ri]], implicit-def $eflags
- ; SLOW-SHLD-NEXT: RET 0
+ ; SLOW-SHLD-NEXT: RET 0, implicit $rax
+ ;
+ ; SLOW-SHLD-SLOW-LEA-LABEL: name: shld_mri32_small
+ ; SLOW-SHLD-SLOW-LEA: liveins: $rax, $ebx
+ ; SLOW-SHLD-SLOW-LEA-NEXT: {{ $}}
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $ebx
+ ; SLOW-SHLD-SLOW-LEA-NEXT: SHL32mi $rax, 1, $noreg, 0, $noreg, 2, implicit-def $eflags
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[SHR32ri:%[0-9]+]]:gr32 = SHR32ri [[COPY]], 30, implicit-def $eflags
+ ; SLOW-SHLD-SLOW-LEA-NEXT: OR32mr $rax, 1, $noreg, 0, $noreg, killed [[SHR32ri]], implicit-def $eflags
+ ; SLOW-SHLD-SLOW-LEA-NEXT: RET 0, implicit $rax
%0:gr32 = COPY $ebx
SHLD32mri8 $rax, 1, $noreg, 0, $noreg, %0, 2, implicit-def $eflags
- RET 0
+
+ RET 0, implicit $rax
+...
+
+
+---
+name: shld_mri32_mid
+alignment: 16
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $rax, $ebx
+
+ ; FAST-SHLD-LABEL: name: shld_mri32_mid
+ ; FAST-SHLD: liveins: $rax, $ebx
+ ; FAST-SHLD-NEXT: {{ $}}
+ ; FAST-SHLD-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $ebx
+ ; FAST-SHLD-NEXT: SHLD32mri8 $rax, 1, $noreg, 0, $noreg, [[COPY]], 16, implicit-def $eflags
+ ; FAST-SHLD-NEXT: RET 0, implicit $rax
+ ;
+ ; SLOW-SHLD-LABEL: name: shld_mri32_mid
+ ; SLOW-SHLD: liveins: $rax, $ebx
+ ; SLOW-SHLD-NEXT: {{ $}}
+ ; SLOW-SHLD-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $ebx
+ ; SLOW-SHLD-NEXT: SHL32mi $rax, 1, $noreg, 0, $noreg, 16, implicit-def $eflags
+ ; SLOW-SHLD-NEXT: [[SHR32ri:%[0-9]+]]:gr32 = SHR32ri [[COPY]], 16, implicit-def $eflags
+ ; SLOW-SHLD-NEXT: OR32mr $rax, 1, $noreg, 0, $noreg, killed [[SHR32ri]], implicit-def $eflags
+ ; SLOW-SHLD-NEXT: RET 0, implicit $rax
+ ;
+ ; SLOW-SHLD-SLOW-LEA-LABEL: name: shld_mri32_mid
+ ; SLOW-SHLD-SLOW-LEA: liveins: $rax, $ebx
+ ; SLOW-SHLD-SLOW-LEA-NEXT: {{ $}}
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $ebx
+ ; SLOW-SHLD-SLOW-LEA-NEXT: SHL32mi $rax, 1, $noreg, 0, $noreg, 16, implicit-def $eflags
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[SHR32ri:%[0-9]+]]:gr32 = SHR32ri [[COPY]], 16, implicit-def $eflags
+ ; SLOW-SHLD-SLOW-LEA-NEXT: OR32mr $rax, 1, $noreg, 0, $noreg, killed [[SHR32ri]], implicit-def $eflags
+ ; SLOW-SHLD-SLOW-LEA-NEXT: RET 0, implicit $rax
+ %0:gr32 = COPY $ebx
+ SHLD32mri8 $rax, 1, $noreg, 0, $noreg, %0, 16, implicit-def $eflags
+
+ RET 0, implicit $rax
...
+
---
-name: mri64
+name: shrd_mri32_small
+alignment: 16
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $rax, $ebx
+
+ ; FAST-SHLD-LABEL: name: shrd_mri32_small
+ ; FAST-SHLD: liveins: $rax, $ebx
+ ; FAST-SHLD-NEXT: {{ $}}
+ ; FAST-SHLD-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $ebx
+ ; FAST-SHLD-NEXT: SHRD32mri8 $rax, 1, $noreg, 0, $noreg, [[COPY]], 2, implicit-def $eflags
+ ; FAST-SHLD-NEXT: RET 0, implicit $rax
+ ;
+ ; SLOW-SHLD-LABEL: name: shrd_mri32_small
+ ; SLOW-SHLD: liveins: $rax, $ebx
+ ; SLOW-SHLD-NEXT: {{ $}}
+ ; SLOW-SHLD-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $ebx
+ ; SLOW-SHLD-NEXT: SHR32mi $rax, 1, $noreg, 0, $noreg, 2, implicit-def $eflags
+ ; SLOW-SHLD-NEXT: [[SHL32ri:%[0-9]+]]:gr32 = SHL32ri [[COPY]], 30, implicit-def $eflags
+ ; SLOW-SHLD-NEXT: OR32mr $rax, 1, $noreg, 0, $noreg, killed [[SHL32ri]], implicit-def $eflags
+ ; SLOW-SHLD-NEXT: RET 0, implicit $rax
+ ;
+ ; SLOW-SHLD-SLOW-LEA-LABEL: name: shrd_mri32_small
+ ; SLOW-SHLD-SLOW-LEA: liveins: $rax, $ebx
+ ; SLOW-SHLD-SLOW-LEA-NEXT: {{ $}}
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $ebx
+ ; SLOW-SHLD-SLOW-LEA-NEXT: SHR32mi $rax, 1, $noreg, 0, $noreg, 2, implicit-def $eflags
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[SHL32ri:%[0-9]+]]:gr32 = SHL32ri [[COPY]], 30, implicit-def $eflags
+ ; SLOW-SHLD-SLOW-LEA-NEXT: OR32mr $rax, 1, $noreg, 0, $noreg, killed [[SHL32ri]], implicit-def $eflags
+ ; SLOW-SHLD-SLOW-LEA-NEXT: RET 0, implicit $rax
+ %0:gr32 = COPY $ebx
+ SHRD32mri8 $rax, 1, $noreg, 0, $noreg, %0, 2, implicit-def $eflags
+
+ RET 0, implicit $rax
+...
+
+
+---
+name: shrd_mri32_mid
+alignment: 16
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $rax, $ebx
+
+ ; FAST-SHLD-LABEL: name: shrd_mri32_mid
+ ; FAST-SHLD: liveins: $rax, $ebx
+ ; FAST-SHLD-NEXT: {{ $}}
+ ; FAST-SHLD-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $ebx
+ ; FAST-SHLD-NEXT: SHRD32mri8 $rax, 1, $noreg, 0, $noreg, [[COPY]], 16, implicit-def $eflags
+ ; FAST-SHLD-NEXT: RET 0, implicit $rax
+ ;
+ ; SLOW-SHLD-LABEL: name: shrd_mri32_mid
+ ; SLOW-SHLD: liveins: $rax, $ebx
+ ; SLOW-SHLD-NEXT: {{ $}}
+ ; SLOW-SHLD-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $ebx
+ ; SLOW-SHLD-NEXT: SHR32mi $rax, 1, $noreg, 0, $noreg, 16, implicit-def $eflags
+ ; SLOW-SHLD-NEXT: [[SHL32ri:%[0-9]+]]:gr32 = SHL32ri [[COPY]], 16, implicit-def $eflags
+ ; SLOW-SHLD-NEXT: OR32mr $rax, 1, $noreg, 0, $noreg, killed [[SHL32ri]], implicit-def $eflags
+ ; SLOW-SHLD-NEXT: RET 0, implicit $rax
+ ;
+ ; SLOW-SHLD-SLOW-LEA-LABEL: name: shrd_mri32_mid
+ ; SLOW-SHLD-SLOW-LEA: liveins: $rax, $ebx
+ ; SLOW-SHLD-SLOW-LEA-NEXT: {{ $}}
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $ebx
+ ; SLOW-SHLD-SLOW-LEA-NEXT: SHR32mi $rax, 1, $noreg, 0, $noreg, 16, implicit-def $eflags
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[SHL32ri:%[0-9]+]]:gr32 = SHL32ri [[COPY]], 16, implicit-def $eflags
+ ; SLOW-SHLD-SLOW-LEA-NEXT: OR32mr $rax, 1, $noreg, 0, $noreg, killed [[SHL32ri]], implicit-def $eflags
+ ; SLOW-SHLD-SLOW-LEA-NEXT: RET 0, implicit $rax
+ %0:gr32 = COPY $ebx
+ SHRD32mri8 $rax, 1, $noreg, 0, $noreg, %0, 16, implicit-def $eflags
+
+ RET 0, implicit $rax
+...
+
+
+---
+name: shld_rri64_small
alignment: 16
tracksRegLiveness: true
body: |
bb.0:
liveins: $rax, $rbx
- ; FAST-SHLD-LABEL: name: mri64
+ ; FAST-SHLD-LABEL: name: shld_rri64_small
; FAST-SHLD: liveins: $rax, $rbx
; FAST-SHLD-NEXT: {{ $}}
- ; FAST-SHLD-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rbx
- ; FAST-SHLD-NEXT: SHLD64mri8 $rax, 1, $noreg, 0, $noreg, [[COPY]], 2, implicit-def $eflags
- ; FAST-SHLD-NEXT: RET 0
+ ; FAST-SHLD-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rax
+ ; FAST-SHLD-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rbx
+ ; FAST-SHLD-NEXT: [[SHLD64rri8_:%[0-9]+]]:gr64 = SHLD64rri8 [[COPY]], [[COPY1]], 2, implicit-def $eflags
+ ; FAST-SHLD-NEXT: $rax = COPY [[SHLD64rri8_]]
+ ; FAST-SHLD-NEXT: RET 0, implicit $rax
;
- ; SLOW-SHLD-LABEL: name: mri64
+ ; SLOW-SHLD-LABEL: name: shld_rri64_small
; SLOW-SHLD: liveins: $rax, $rbx
; SLOW-SHLD-NEXT: {{ $}}
- ; SLOW-SHLD-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rbx
- ; SLOW-SHLD-NEXT: SHL64mi $rax, 1, $noreg, 0, $noreg, 2, implicit-def $eflags
- ; SLOW-SHLD-NEXT: [[SHR64ri:%[0-9]+]]:gr64 = SHR64ri [[COPY]], 62, implicit-def $eflags
- ; SLOW-SHLD-NEXT: OR64mr $rax, 1, $noreg, 0, $noreg, killed [[SHR64ri]], implicit-def $eflags
- ; SLOW-SHLD-NEXT: RET 0
+ ; SLOW-SHLD-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rax
+ ; SLOW-SHLD-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rbx
+ ; SLOW-SHLD-NEXT: [[SHR64ri:%[0-9]+]]:gr64 = SHR64ri [[COPY1]], 62, implicit-def $eflags
+ ; SLOW-SHLD-NEXT: [[LEA64r:%[0-9]+]]:gr64 = LEA64r killed [[SHR64ri]], 4, [[COPY]], 0, $noreg
+ ; SLOW-SHLD-NEXT: $rax = COPY [[LEA64r]]
+ ; SLOW-SHLD-NEXT: RET 0, implicit $rax
+ ;
+ ; SLOW-SHLD-SLOW-LEA-LABEL: name: shld_rri64_small
+ ; SLOW-SHLD-SLOW-LEA: liveins: $rax, $rbx
+ ; SLOW-SHLD-SLOW-LEA-NEXT: {{ $}}
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rax
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rbx
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[SHL64ri:%[0-9]+]]:gr64 = SHL64ri [[COPY]], 2, implicit-def $eflags
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[SHR64ri:%[0-9]+]]:gr64 = SHR64ri [[COPY1]], 62, implicit-def $eflags
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[OR64rr:%[0-9]+]]:gr64 = OR64rr killed [[SHL64ri]], killed [[SHR64ri]], implicit-def $eflags
+ ; SLOW-SHLD-SLOW-LEA-NEXT: $rax = COPY [[OR64rr]]
+ ; SLOW-SHLD-SLOW-LEA-NEXT: RET 0, implicit $rax
+ %1:gr64 = COPY $rax
%0:gr64 = COPY $rbx
- SHLD64mri8 $rax, 1, $noreg, 0, $noreg, %0, 2, implicit-def $eflags
- RET 0
+ %2:gr64 = SHLD64rri8 %1, %0, 2, implicit-def $eflags
+ $rax = COPY %2
+ RET 0, implicit $rax
...
+
---
-name: shrd_rri32
+name: shld_rri64_mid
alignment: 16
tracksRegLiveness: true
body: |
bb.0:
- liveins: $eax, $ebx
- ; FAST-SHLD-LABEL: name: shrd_rri32
- ; FAST-SHLD: liveins: $eax, $ebx
+ liveins: $rax, $rbx
+ ; FAST-SHLD-LABEL: name: shld_rri64_mid
+ ; FAST-SHLD: liveins: $rax, $rbx
; FAST-SHLD-NEXT: {{ $}}
- ; FAST-SHLD-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $eax
- ; FAST-SHLD-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY $ebx
- ; FAST-SHLD-NEXT: [[SHRD32rri8_:%[0-9]+]]:gr32 = SHRD32rri8 [[COPY]], [[COPY1]], 2, implicit-def $eflags
- ; FAST-SHLD-NEXT: $eax = COPY [[SHRD32rri8_]]
+ ; FAST-SHLD-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rax
+ ; FAST-SHLD-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rbx
+ ; FAST-SHLD-NEXT: [[SHLD64rri8_:%[0-9]+]]:gr64 = SHLD64rri8 [[COPY]], [[COPY1]], 32, implicit-def $eflags
+ ; FAST-SHLD-NEXT: $rax = COPY [[SHLD64rri8_]]
; FAST-SHLD-NEXT: RET 0, implicit $rax
;
- ; SLOW-SHLD-LABEL: name: shrd_rri32
- ; SLOW-SHLD: liveins: $eax, $ebx
+ ; SLOW-SHLD-LABEL: name: shld_rri64_mid
+ ; SLOW-SHLD: liveins: $rax, $rbx
; SLOW-SHLD-NEXT: {{ $}}
- ; SLOW-SHLD-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $eax
- ; SLOW-SHLD-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY $ebx
- ; SLOW-SHLD-NEXT: [[SHR32ri:%[0-9]+]]:gr32 = SHR32ri [[COPY]], 2, implicit-def $eflags
- ; SLOW-SHLD-NEXT: [[SHL32ri:%[0-9]+]]:gr32 = SHL32ri [[COPY1]], 30, implicit-def $eflags
- ; SLOW-SHLD-NEXT: [[OR32rr:%[0-9]+]]:gr32 = OR32rr killed [[SHR32ri]], killed [[SHL32ri]], implicit-def $eflags
- ; SLOW-SHLD-NEXT: $eax = COPY [[OR32rr]]
+ ; SLOW-SHLD-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rax
+ ; SLOW-SHLD-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rbx
+ ; SLOW-SHLD-NEXT: [[SHL64ri:%[0-9]+]]:gr64 = SHL64ri [[COPY]], 32, implicit-def $eflags
+ ; SLOW-SHLD-NEXT: [[SHR64ri:%[0-9]+]]:gr64 = SHR64ri [[COPY1]], 32, implicit-def $eflags
+ ; SLOW-SHLD-NEXT: [[LEA64r:%[0-9]+]]:gr64 = LEA64r killed [[SHL64ri]], 1, killed [[SHR64ri]], 0, $noreg
+ ; SLOW-SHLD-NEXT: $rax = COPY [[LEA64r]]
; SLOW-SHLD-NEXT: RET 0, implicit $rax
- %0:gr32 = COPY $eax
- %1:gr32 = COPY $ebx
- %2:gr32 = SHRD32rri8 %0, %1, 2, implicit-def $eflags
- $eax = COPY %2
+ ;
+ ; SLOW-SHLD-SLOW-LEA-LABEL: name: shld_rri64_mid
+ ; SLOW-SHLD-SLOW-LEA: liveins: $rax, $rbx
+ ; SLOW-SHLD-SLOW-LEA-NEXT: {{ $}}
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rax
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rbx
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[SHL64ri:%[0-9]+]]:gr64 = SHL64ri [[COPY]], 32, implicit-def $eflags
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[SHR64ri:%[0-9]+]]:gr64 = SHR64ri [[COPY1]], 32, implicit-def $eflags
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[LEA64r:%[0-9]+]]:gr64 = LEA64r killed [[SHL64ri]], 1, killed [[SHR64ri]], 0, $noreg
+ ; SLOW-SHLD-SLOW-LEA-NEXT: $rax = COPY [[LEA64r]]
+ ; SLOW-SHLD-SLOW-LEA-NEXT: RET 0, implicit $rax
+ %1:gr64 = COPY $rax
+ %0:gr64 = COPY $rbx
+ %2:gr64 = SHLD64rri8 %1, %0, 32, implicit-def $eflags
+ $rax = COPY %2
RET 0, implicit $rax
...
+
---
-name: shrd_rri64
+name: shrd_rri64_small
alignment: 16
tracksRegLiveness: true
body: |
bb.0:
liveins: $rax, $rbx
- ; FAST-SHLD-LABEL: name: shrd_rri64
+ ; FAST-SHLD-LABEL: name: shrd_rri64_small
; FAST-SHLD: liveins: $rax, $rbx
; FAST-SHLD-NEXT: {{ $}}
; FAST-SHLD-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rax
@@ -171,73 +443,231 @@ body: |
; FAST-SHLD-NEXT: $rax = COPY [[SHRD64rri8_]]
; FAST-SHLD-NEXT: RET 0, implicit $rax
;
- ; SLOW-SHLD-LABEL: name: shrd_rri64
+ ; SLOW-SHLD-LABEL: name: shrd_rri64_small
; SLOW-SHLD: liveins: $rax, $rbx
; SLOW-SHLD-NEXT: {{ $}}
; SLOW-SHLD-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rax
; SLOW-SHLD-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rbx
; SLOW-SHLD-NEXT: [[SHR64ri:%[0-9]+]]:gr64 = SHR64ri [[COPY]], 2, implicit-def $eflags
; SLOW-SHLD-NEXT: [[SHL64ri:%[0-9]+]]:gr64 = SHL64ri [[COPY1]], 62, implicit-def $eflags
- ; SLOW-SHLD-NEXT: [[OR64rr:%[0-9]+]]:gr64 = OR64rr killed [[SHR64ri]], killed [[SHL64ri]], implicit-def $eflags
- ; SLOW-SHLD-NEXT: $rax = COPY [[OR64rr]]
+ ; SLOW-SHLD-NEXT: [[LEA64r:%[0-9]+]]:gr64 = LEA64r killed [[SHR64ri]], 1, killed [[SHL64ri]], 0, $noreg
+ ; SLOW-SHLD-NEXT: $rax = COPY [[LEA64r]]
; SLOW-SHLD-NEXT: RET 0, implicit $rax
- %0:gr64 = COPY $rax
- %1:gr64 = COPY $rbx
- %2:gr64 = SHRD64rri8 %0, %1, 2, implicit-def $eflags
+ ;
+ ; SLOW-SHLD-SLOW-LEA-LABEL: name: shrd_rri64_small
+ ; SLOW-SHLD-SLOW-LEA: liveins: $rax, $rbx
+ ; SLOW-SHLD-SLOW-LEA-NEXT: {{ $}}
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rax
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rbx
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[SHR64ri:%[0-9]+]]:gr64 = SHR64ri [[COPY]], 2, implicit-def $eflags
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[SHL64ri:%[0-9]+]]:gr64 = SHL64ri [[COPY1]], 62, implicit-def $eflags
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[LEA64r:%[0-9]+]]:gr64 = LEA64r killed [[SHR64ri]], 1, killed [[SHL64ri]], 0, $noreg
+ ; SLOW-SHLD-SLOW-LEA-NEXT: $rax = COPY [[LEA64r]]
+ ; SLOW-SHLD-SLOW-LEA-NEXT: RET 0, implicit $rax
+ %1:gr64 = COPY $rax
+ %0:gr64 = COPY $rbx
+ %2:gr64 = SHRD64rri8 %1, %0, 2, implicit-def $eflags
$rax = COPY %2
RET 0, implicit $rax
...
+
---
-name: shrd_mri32
+name: shrd_rri64_mid
alignment: 16
tracksRegLiveness: true
body: |
bb.0:
- liveins: $rax, $ebx
- ; FAST-SHLD-LABEL: name: shrd_mri32
- ; FAST-SHLD: liveins: $rax, $ebx
+ liveins: $rax, $rbx
+ ; FAST-SHLD-LABEL: name: shrd_rri64_mid
+ ; FAST-SHLD: liveins: $rax, $rbx
; FAST-SHLD-NEXT: {{ $}}
- ; FAST-SHLD-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $ebx
- ; FAST-SHLD-NEXT: SHRD32mri8 $rax, 1, $noreg, 0, $noreg, [[COPY]], 2, implicit-def $eflags
- ; FAST-SHLD-NEXT: RET 0
+ ; FAST-SHLD-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rax
+ ; FAST-SHLD-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rbx
+ ; FAST-SHLD-NEXT: [[SHRD64rri8_:%[0-9]+]]:gr64 = SHRD64rri8 [[COPY]], [[COPY1]], 32, implicit-def $eflags
+ ; FAST-SHLD-NEXT: $rax = COPY [[SHRD64rri8_]]
+ ; FAST-SHLD-NEXT: RET 0, implicit $rax
;
- ; SLOW-SHLD-LABEL: name: shrd_mri32
- ; SLOW-SHLD: liveins: $rax, $ebx
+ ; SLOW-SHLD-LABEL: name: shrd_rri64_mid
+ ; SLOW-SHLD: liveins: $rax, $rbx
; SLOW-SHLD-NEXT: {{ $}}
- ; SLOW-SHLD-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY $ebx
- ; SLOW-SHLD-NEXT: SHR32mi $rax, 1, $noreg, 0, $noreg, 2, implicit-def $eflags
- ; SLOW-SHLD-NEXT: [[SHL32ri:%[0-9]+]]:gr32 = SHL32ri [[COPY]], 30, implicit-def $eflags
- ; SLOW-SHLD-NEXT: OR32mr $rax, 1, $noreg, 0, $noreg, killed [[SHL32ri]], implicit-def $eflags
- ; SLOW-SHLD-NEXT: RET 0
- %0:gr32 = COPY $ebx
- SHRD32mri8 $rax, 1, $noreg, 0, $noreg, %0, 2, implicit-def $eflags
- RET 0
+ ; SLOW-SHLD-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rax
+ ; SLOW-SHLD-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rbx
+ ; SLOW-SHLD-NEXT: [[SHR64ri:%[0-9]+]]:gr64 = SHR64ri [[COPY]], 32, implicit-def $eflags
+ ; SLOW-SHLD-NEXT: [[SHL64ri:%[0-9]+]]:gr64 = SHL64ri [[COPY1]], 32, implicit-def $eflags
+ ; SLOW-SHLD-NEXT: [[LEA64r:%[0-9]+]]:gr64 = LEA64r killed [[SHR64ri]], 1, killed [[SHL64ri]], 0, $noreg
+ ; SLOW-SHLD-NEXT: $rax = COPY [[LEA64r]]
+ ; SLOW-SHLD-NEXT: RET 0, implicit $rax
+ ;
+ ; SLOW-SHLD-SLOW-LEA-LABEL: name: shrd_rri64_mid
+ ; SLOW-SHLD-SLOW-LEA: liveins: $rax, $rbx
+ ; SLOW-SHLD-SLOW-LEA-NEXT: {{ $}}
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rax
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rbx
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[SHR64ri:%[0-9]+]]:gr64 = SHR64ri [[COPY]], 32, implicit-def $eflags
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[SHL64ri:%[0-9]+]]:gr64 = SHL64ri [[COPY1]], 32, implicit-def $eflags
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[LEA64r:%[0-9]+]]:gr64 = LEA64r killed [[SHR64ri]], 1, killed [[SHL64ri]], 0, $noreg
+ ; SLOW-SHLD-SLOW-LEA-NEXT: $rax = COPY [[LEA64r]]
+ ; SLOW-SHLD-SLOW-LEA-NEXT: RET 0, implicit $rax
+ %1:gr64 = COPY $rax
+ %0:gr64 = COPY $rbx
+ %2:gr64 = SHRD64rri8 %1, %0, 32, implicit-def $eflags
+ $rax = COPY %2
+ RET 0, implicit $rax
+...
+
+
+---
+name: shld_mri64_small
+alignment: 16
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $rax, $rbx
+
+ ; FAST-SHLD-LABEL: name: shld_mri64_small
+ ; FAST-SHLD: liveins: $rax, $rbx
+ ; FAST-SHLD-NEXT: {{ $}}
+ ; FAST-SHLD-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rbx
+ ; FAST-SHLD-NEXT: SHLD64mri8 $rax, 1, $noreg, 0, $noreg, [[COPY]], 2, implicit-def $eflags
+ ; FAST-SHLD-NEXT: RET 0, implicit $rax
+ ;
+ ; SLOW-SHLD-LABEL: name: shld_mri64_small
+ ; SLOW-SHLD: liveins: $rax, $rbx
+ ; SLOW-SHLD-NEXT: {{ $}}
+ ; SLOW-SHLD-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rbx
+ ; SLOW-SHLD-NEXT: SHL64mi $rax, 1, $noreg, 0, $noreg, 2, implicit-def $eflags
+ ; SLOW-SHLD-NEXT: [[SHR64ri:%[0-9]+]]:gr64 = SHR64ri [[COPY]], 62, implicit-def $eflags
+ ; SLOW-SHLD-NEXT: OR64mr $rax, 1, $noreg, 0, $noreg, killed [[SHR64ri]], implicit-def $eflags
+ ; SLOW-SHLD-NEXT: RET 0, implicit $rax
+ ;
+ ; SLOW-SHLD-SLOW-LEA-LABEL: name: shld_mri64_small
+ ; SLOW-SHLD-SLOW-LEA: liveins: $rax, $rbx
+ ; SLOW-SHLD-SLOW-LEA-NEXT: {{ $}}
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rbx
+ ; SLOW-SHLD-SLOW-LEA-NEXT: SHL64mi $rax, 1, $noreg, 0, $noreg, 2, implicit-def $eflags
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[SHR64ri:%[0-9]+]]:gr64 = SHR64ri [[COPY]], 62, implicit-def $eflags
+ ; SLOW-SHLD-SLOW-LEA-NEXT: OR64mr $rax, 1, $noreg, 0, $noreg, killed [[SHR64ri]], implicit-def $eflags
+ ; SLOW-SHLD-SLOW-LEA-NEXT: RET 0, implicit $rax
+ %0:gr64 = COPY $rbx
+ SHLD64mri8 $rax, 1, $noreg, 0, $noreg, %0, 2, implicit-def $eflags
+
+ RET 0, implicit $rax
...
+
---
-name: shrd_mri64
+name: shld_mri64_mid
alignment: 16
tracksRegLiveness: true
body: |
bb.0:
liveins: $rax, $rbx
- ; FAST-SHLD-LABEL: name: shrd_mri64
+
+ ; FAST-SHLD-LABEL: name: shld_mri64_mid
+ ; FAST-SHLD: liveins: $rax, $rbx
+ ; FAST-SHLD-NEXT: {{ $}}
+ ; FAST-SHLD-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rbx
+ ; FAST-SHLD-NEXT: SHLD64mri8 $rax, 1, $noreg, 0, $noreg, [[COPY]], 32, implicit-def $eflags
+ ; FAST-SHLD-NEXT: RET 0, implicit $rax
+ ;
+ ; SLOW-SHLD-LABEL: name: shld_mri64_mid
+ ; SLOW-SHLD: liveins: $rax, $rbx
+ ; SLOW-SHLD-NEXT: {{ $}}
+ ; SLOW-SHLD-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rbx
+ ; SLOW-SHLD-NEXT: SHL64mi $rax, 1, $noreg, 0, $noreg, 32, implicit-def $eflags
+ ; SLOW-SHLD-NEXT: [[SHR64ri:%[0-9]+]]:gr64 = SHR64ri [[COPY]], 32, implicit-def $eflags
+ ; SLOW-SHLD-NEXT: OR64mr $rax, 1, $noreg, 0, $noreg, killed [[SHR64ri]], implicit-def $eflags
+ ; SLOW-SHLD-NEXT: RET 0, implicit $rax
+ ;
+ ; SLOW-SHLD-SLOW-LEA-LABEL: name: shld_mri64_mid
+ ; SLOW-SHLD-SLOW-LEA: liveins: $rax, $rbx
+ ; SLOW-SHLD-SLOW-LEA-NEXT: {{ $}}
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rbx
+ ; SLOW-SHLD-SLOW-LEA-NEXT: SHL64mi $rax, 1, $noreg, 0, $noreg, 32, implicit-def $eflags
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[SHR64ri:%[0-9]+]]:gr64 = SHR64ri [[COPY]], 32, implicit-def $eflags
+ ; SLOW-SHLD-SLOW-LEA-NEXT: OR64mr $rax, 1, $noreg, 0, $noreg, killed [[SHR64ri]], implicit-def $eflags
+ ; SLOW-SHLD-SLOW-LEA-NEXT: RET 0, implicit $rax
+ %0:gr64 = COPY $rbx
+ SHLD64mri8 $rax, 1, $noreg, 0, $noreg, %0, 32, implicit-def $eflags
+
+ RET 0, implicit $rax
+...
+
+
+---
+name: shrd_mri64_small
+alignment: 16
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $rax, $rbx
+
+ ; FAST-SHLD-LABEL: name: shrd_mri64_small
; FAST-SHLD: liveins: $rax, $rbx
; FAST-SHLD-NEXT: {{ $}}
; FAST-SHLD-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rbx
; FAST-SHLD-NEXT: SHRD64mri8 $rax, 1, $noreg, 0, $noreg, [[COPY]], 2, implicit-def $eflags
- ; FAST-SHLD-NEXT: RET 0
+ ; FAST-SHLD-NEXT: RET 0, implicit $rax
;
- ; SLOW-SHLD-LABEL: name: shrd_mri64
+ ; SLOW-SHLD-LABEL: name: shrd_mri64_small
; SLOW-SHLD: liveins: $rax, $rbx
; SLOW-SHLD-NEXT: {{ $}}
; SLOW-SHLD-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rbx
; SLOW-SHLD-NEXT: SHR64mi $rax, 1, $noreg, 0, $noreg, 2, implicit-def $eflags
; SLOW-SHLD-NEXT: [[SHL64ri:%[0-9]+]]:gr64 = SHL64ri [[COPY]], 62, implicit-def $eflags
; SLOW-SHLD-NEXT: OR64mr $rax, 1, $noreg, 0, $noreg, killed [[SHL64ri]], implicit-def $eflags
- ; SLOW-SHLD-NEXT: RET 0
+ ; SLOW-SHLD-NEXT: RET 0, implicit $rax
+ ;
+ ; SLOW-SHLD-SLOW-LEA-LABEL: name: shrd_mri64_small
+ ; SLOW-SHLD-SLOW-LEA: liveins: $rax, $rbx
+ ; SLOW-SHLD-SLOW-LEA-NEXT: {{ $}}
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rbx
+ ; SLOW-SHLD-SLOW-LEA-NEXT: SHR64mi $rax, 1, $noreg, 0, $noreg, 2, implicit-def $eflags
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[SHL64ri:%[0-9]+]]:gr64 = SHL64ri [[COPY]], 62, implicit-def $eflags
+ ; SLOW-SHLD-SLOW-LEA-NEXT: OR64mr $rax, 1, $noreg, 0, $noreg, killed [[SHL64ri]], implicit-def $eflags
+ ; SLOW-SHLD-SLOW-LEA-NEXT: RET 0, implicit $rax
%0:gr64 = COPY $rbx
SHRD64mri8 $rax, 1, $noreg, 0, $noreg, %0, 2, implicit-def $eflags
- RET 0
+
+ RET 0, implicit $rax
+...
+
+
+---
+name: shrd_mri64_mid
+alignment: 16
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $rax, $rbx
+
+ ; FAST-SHLD-LABEL: name: shrd_mri64_mid
+ ; FAST-SHLD: liveins: $rax, $rbx
+ ; FAST-SHLD-NEXT: {{ $}}
+ ; FAST-SHLD-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rbx
+ ; FAST-SHLD-NEXT: SHRD64mri8 $rax, 1, $noreg, 0, $noreg, [[COPY]], 32, implicit-def $eflags
+ ; FAST-SHLD-NEXT: RET 0, implicit $rax
+ ;
+ ; SLOW-SHLD-LABEL: name: shrd_mri64_mid
+ ; SLOW-SHLD: liveins: $rax, $rbx
+ ; SLOW-SHLD-NEXT: {{ $}}
+ ; SLOW-SHLD-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rbx
+ ; SLOW-SHLD-NEXT: SHR64mi $rax, 1, $noreg, 0, $noreg, 32, implicit-def $eflags
+ ; SLOW-SHLD-NEXT: [[SHL64ri:%[0-9]+]]:gr64 = SHL64ri [[COPY]], 32, implicit-def $eflags
+ ; SLOW-SHLD-NEXT: OR64mr $rax, 1, $noreg, 0, $noreg, killed [[SHL64ri]], implicit-def $eflags
+ ; SLOW-SHLD-NEXT: RET 0, implicit $rax
+ ;
+ ; SLOW-SHLD-SLOW-LEA-LABEL: name: shrd_mri64_mid
+ ; SLOW-SHLD-SLOW-LEA: liveins: $rax, $rbx
+ ; SLOW-SHLD-SLOW-LEA-NEXT: {{ $}}
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rbx
+ ; SLOW-SHLD-SLOW-LEA-NEXT: SHR64mi $rax, 1, $noreg, 0, $noreg, 32, implicit-def $eflags
+ ; SLOW-SHLD-SLOW-LEA-NEXT: [[SHL64ri:%[0-9]+]]:gr64 = SHL64ri [[COPY]], 32, implicit-def $eflags
+ ; SLOW-SHLD-SLOW-LEA-NEXT: OR64mr $rax, 1, $noreg, 0, $noreg, killed [[SHL64ri]], implicit-def $eflags
+ ; SLOW-SHLD-SLOW-LEA-NEXT: RET 0, implicit $rax
+ %0:gr64 = COPY $rbx
+ SHRD64mri8 $rax, 1, $noreg, 0, $noreg, %0, 32, implicit-def $eflags
+
+ RET 0, implicit $rax
...
diff --git a/llvm/test/CodeGen/X86/shift-bmi2.ll b/llvm/test/CodeGen/X86/shift-bmi2.ll
index dafcf3db4550e..e3eade10123ed 100644
--- a/llvm/test/CodeGen/X86/shift-bmi2.ll
+++ b/llvm/test/CodeGen/X86/shift-bmi2.ll
@@ -126,13 +126,13 @@ define i64 @shl64(i64 %x, i64 %shamt) nounwind uwtable readnone {
define i64 @shl64i(i64 %x) nounwind uwtable readnone {
; BMI2-LABEL: shl64i:
; BMI2: # %bb.0:
-; BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx
; BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; BMI2-NEXT: movl %edx, %eax
+; BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx
+; BMI2-NEXT: movl %ecx, %eax
; BMI2-NEXT: shll $7, %eax
-; BMI2-NEXT: shll $7, %ecx
-; BMI2-NEXT: shrl $25, %edx
-; BMI2-NEXT: orl %ecx, %edx
+; BMI2-NEXT: shll $7, %edx
+; BMI2-NEXT: shrl $25, %ecx
+; BMI2-NEXT: addl %ecx, %edx
; BMI2-NEXT: retl
;
; BMI264-LABEL: shl64i:
@@ -188,13 +188,13 @@ define i64 @shl64pi(ptr %p) nounwind uwtable readnone {
; BMI2-LABEL: shl64pi:
; BMI2: # %bb.0:
; BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
-; BMI2-NEXT: movl (%eax), %edx
-; BMI2-NEXT: movl 4(%eax), %ecx
-; BMI2-NEXT: movl %edx, %eax
+; BMI2-NEXT: movl (%eax), %ecx
+; BMI2-NEXT: movl 4(%eax), %edx
+; BMI2-NEXT: movl %ecx, %eax
; BMI2-NEXT: shll $7, %eax
-; BMI2-NEXT: shll $7, %ecx
-; BMI2-NEXT: shrl $25, %edx
-; BMI2-NEXT: orl %ecx, %edx
+; BMI2-NEXT: shll $7, %edx
+; BMI2-NEXT: shrl $25, %ecx
+; BMI2-NEXT: addl %ecx, %edx
; BMI2-NEXT: retl
;
; BMI264-LABEL: shl64pi:
diff --git a/llvm/test/CodeGen/X86/shift-i256.ll b/llvm/test/CodeGen/X86/shift-i256.ll
index d82172c964398..68d9103e6079b 100644
--- a/llvm/test/CodeGen/X86/shift-i256.ll
+++ b/llvm/test/CodeGen/X86/shift-i256.ll
@@ -1471,40 +1471,34 @@ define i256 @shl_i256_1(i256 %a0) nounwind {
; AVX2-LABEL: shl_i256_1:
; AVX2: # %bb.0:
; AVX2-NEXT: movq %rdi, %rax
-; AVX2-NEXT: leaq (,%rdx,2), %rdi
-; AVX2-NEXT: movq %rsi, %r9
-; AVX2-NEXT: shrq $63, %r9
-; AVX2-NEXT: orq %rdi, %r9
-; AVX2-NEXT: leaq (,%rcx,2), %rdi
+; AVX2-NEXT: movq %rsi, %rdi
+; AVX2-NEXT: shrq $63, %rdi
+; AVX2-NEXT: leaq (%rdi,%rdx,2), %rdi
; AVX2-NEXT: shrq $63, %rdx
-; AVX2-NEXT: orq %rdi, %rdx
-; AVX2-NEXT: shlq %r8
+; AVX2-NEXT: leaq (%rdx,%rcx,2), %rdx
; AVX2-NEXT: shrq $63, %rcx
-; AVX2-NEXT: orq %r8, %rcx
+; AVX2-NEXT: leaq (%rcx,%r8,2), %rcx
; AVX2-NEXT: addq %rsi, %rsi
; AVX2-NEXT: movq %rcx, 24(%rax)
; AVX2-NEXT: movq %rdx, 16(%rax)
-; AVX2-NEXT: movq %r9, 8(%rax)
+; AVX2-NEXT: movq %rdi, 8(%rax)
; AVX2-NEXT: movq %rsi, (%rax)
; AVX2-NEXT: retq
;
; AVX512-LABEL: shl_i256_1:
; AVX512: # %bb.0:
; AVX512-NEXT: movq %rdi, %rax
-; AVX512-NEXT: leaq (,%rdx,2), %rdi
-; AVX512-NEXT: movq %rsi, %r9
-; AVX512-NEXT: shrq $63, %r9
-; AVX512-NEXT: orq %rdi, %r9
-; AVX512-NEXT: leaq (,%rcx,2), %rdi
+; AVX512-NEXT: movq %rsi, %rdi
+; AVX512-NEXT: shrq $63, %rdi
+; AVX512-NEXT: leaq (%rdi,%rdx,2), %rdi
; AVX512-NEXT: shrq $63, %rdx
-; AVX512-NEXT: orq %rdi, %rdx
-; AVX512-NEXT: shlq %r8
+; AVX512-NEXT: leaq (%rdx,%rcx,2), %rdx
; AVX512-NEXT: shrq $63, %rcx
-; AVX512-NEXT: orq %r8, %rcx
+; AVX512-NEXT: leaq (%rcx,%r8,2), %rcx
; AVX512-NEXT: addq %rsi, %rsi
; AVX512-NEXT: movq %rcx, 24(%rax)
; AVX512-NEXT: movq %rdx, 16(%rax)
-; AVX512-NEXT: movq %r9, 8(%rax)
+; AVX512-NEXT: movq %rdi, 8(%rax)
; AVX512-NEXT: movq %rsi, (%rax)
; AVX512-NEXT: retq
;
@@ -1560,20 +1554,20 @@ define i256 @lshr_i256_1(i256 %a0) nounwind {
; AVX2-NEXT: shrq %rsi
; AVX2-NEXT: movq %rdx, %rdi
; AVX2-NEXT: shlq $63, %rdi
-; AVX2-NEXT: orq %rsi, %rdi
-; AVX2-NEXT: movq %rcx, %rsi
-; AVX2-NEXT: shlq $63, %rsi
+; AVX2-NEXT: addq %rdi, %rsi
+; AVX2-NEXT: movq %rcx, %rdi
+; AVX2-NEXT: shlq $63, %rdi
; AVX2-NEXT: shrq %rdx
-; AVX2-NEXT: orq %rsi, %rdx
+; AVX2-NEXT: addq %rdi, %rdx
; AVX2-NEXT: shrq %rcx
-; AVX2-NEXT: movq %r8, %rsi
-; AVX2-NEXT: shlq $63, %rsi
-; AVX2-NEXT: orq %rcx, %rsi
+; AVX2-NEXT: movq %r8, %rdi
+; AVX2-NEXT: shlq $63, %rdi
+; AVX2-NEXT: addq %rdi, %rcx
; AVX2-NEXT: shrq %r8
; AVX2-NEXT: movq %r8, 24(%rax)
-; AVX2-NEXT: movq %rsi, 16(%rax)
+; AVX2-NEXT: movq %rcx, 16(%rax)
; AVX2-NEXT: movq %rdx, 8(%rax)
-; AVX2-NEXT: movq %rdi, (%rax)
+; AVX2-NEXT: movq %rsi, (%rax)
; AVX2-NEXT: retq
;
; AVX512-LABEL: lshr_i256_1:
@@ -1582,20 +1576,20 @@ define i256 @lshr_i256_1(i256 %a0) nounwind {
; AVX512-NEXT: shrq %rsi
; AVX512-NEXT: movq %rdx, %rdi
; AVX512-NEXT: shlq $63, %rdi
-; AVX512-NEXT: orq %rsi, %rdi
-; AVX512-NEXT: movq %rcx, %rsi
-; AVX512-NEXT: shlq $63, %rsi
+; AVX512-NEXT: addq %rdi, %rsi
+; AVX512-NEXT: movq %rcx, %rdi
+; AVX512-NEXT: shlq $63, %rdi
; AVX512-NEXT: shrq %rdx
-; AVX512-NEXT: orq %rsi, %rdx
+; AVX512-NEXT: addq %rdi, %rdx
; AVX512-NEXT: shrq %rcx
-; AVX512-NEXT: movq %r8, %rsi
-; AVX512-NEXT: shlq $63, %rsi
-; AVX512-NEXT: orq %rcx, %rsi
+; AVX512-NEXT: movq %r8, %rdi
+; AVX512-NEXT: shlq $63, %rdi
+; AVX512-NEXT: addq %rdi, %rcx
; AVX512-NEXT: shrq %r8
; AVX512-NEXT: movq %r8, 24(%rax)
-; AVX512-NEXT: movq %rsi, 16(%rax)
+; AVX512-NEXT: movq %rcx, 16(%rax)
; AVX512-NEXT: movq %rdx, 8(%rax)
-; AVX512-NEXT: movq %rdi, (%rax)
+; AVX512-NEXT: movq %rsi, (%rax)
; AVX512-NEXT: retq
;
; X86-LABEL: lshr_i256_1:
@@ -1668,20 +1662,20 @@ define i256 @ashr_i256_1(i256 %a0) nounwind {
; AVX2-NEXT: shrq %rsi
; AVX2-NEXT: movq %rdx, %rdi
; AVX2-NEXT: shlq $63, %rdi
-; AVX2-NEXT: orq %rsi, %rdi
-; AVX2-NEXT: movq %rcx, %rsi
-; AVX2-NEXT: shlq $63, %rsi
+; AVX2-NEXT: addq %rdi, %rsi
+; AVX2-NEXT: movq %rcx, %rdi
+; AVX2-NEXT: shlq $63, %rdi
; AVX2-NEXT: shrq %rdx
-; AVX2-NEXT: orq %rsi, %rdx
+; AVX2-NEXT: addq %rdi, %rdx
; AVX2-NEXT: shrq %rcx
-; AVX2-NEXT: movq %r8, %rsi
-; AVX2-NEXT: shlq $63, %rsi
-; AVX2-NEXT: orq %rcx, %rsi
+; AVX2-NEXT: movq %r8, %rdi
+; AVX2-NEXT: shlq $63, %rdi
+; AVX2-NEXT: addq %rdi, %rcx
; AVX2-NEXT: sarq %r8
; AVX2-NEXT: movq %r8, 24(%rax)
-; AVX2-NEXT: movq %rsi, 16(%rax)
+; AVX2-NEXT: movq %rcx, 16(%rax)
; AVX2-NEXT: movq %rdx, 8(%rax)
-; AVX2-NEXT: movq %rdi, (%rax)
+; AVX2-NEXT: movq %rsi, (%rax)
; AVX2-NEXT: retq
;
; AVX512-LABEL: ashr_i256_1:
@@ -1690,20 +1684,20 @@ define i256 @ashr_i256_1(i256 %a0) nounwind {
; AVX512-NEXT: shrq %rsi
; AVX512-NEXT: movq %rdx, %rdi
; AVX512-NEXT: shlq $63, %rdi
-; AVX512-NEXT: orq %rsi, %rdi
-; AVX512-NEXT: movq %rcx, %rsi
-; AVX512-NEXT: shlq $63, %rsi
+; AVX512-NEXT: addq %rdi, %rsi
+; AVX512-NEXT: movq %rcx, %rdi
+; AVX512-NEXT: shlq $63, %rdi
; AVX512-NEXT: shrq %rdx
-; AVX512-NEXT: orq %rsi, %rdx
+; AVX512-NEXT: addq %rdi, %rdx
; AVX512-NEXT: shrq %rcx
-; AVX512-NEXT: movq %r8, %rsi
-; AVX512-NEXT: shlq $63, %rsi
-; AVX512-NEXT: orq %rcx, %rsi
+; AVX512-NEXT: movq %r8, %rdi
+; AVX512-NEXT: shlq $63, %rdi
+; AVX512-NEXT: addq %rdi, %rcx
; AVX512-NEXT: sarq %r8
; AVX512-NEXT: movq %r8, 24(%rax)
-; AVX512-NEXT: movq %rsi, 16(%rax)
+; AVX512-NEXT: movq %rcx, 16(%rax)
; AVX512-NEXT: movq %rdx, 8(%rax)
-; AVX512-NEXT: movq %rdi, (%rax)
+; AVX512-NEXT: movq %rsi, (%rax)
; AVX512-NEXT: retq
;
; X86-LABEL: ashr_i256_1:
diff --git a/llvm/test/CodeGen/X86/shift-i512.ll b/llvm/test/CodeGen/X86/shift-i512.ll
index 3c49212378c64..765665b904f25 100644
--- a/llvm/test/CodeGen/X86/shift-i512.ll
+++ b/llvm/test/CodeGen/X86/shift-i512.ll
@@ -1258,112 +1258,98 @@ define i512 @ashr_i512_load(ptr %p0, i512 %a1) nounwind {
define i512 @shl_i512_1(i512 %a0) nounwind {
; SSE-LABEL: shl_i512_1:
; SSE: # %bb.0:
+; SSE-NEXT: pushq %rbx
; SSE-NEXT: movq %rdi, %rax
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rdi
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r10
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r11
-; SSE-NEXT: shldq $1, %rdi, %r10
-; SSE-NEXT: shldq $1, %r11, %rdi
-; SSE-NEXT: shldq $1, %r9, %r11
-; SSE-NEXT: shldq $1, %r8, %r9
+; SSE-NEXT: movq %r9, %rbx
+; SSE-NEXT: shldq $1, %r8, %rbx
; SSE-NEXT: shldq $1, %rcx, %r8
; SSE-NEXT: shldq $1, %rdx, %rcx
; SSE-NEXT: shldq $1, %rsi, %rdx
+; SSE-NEXT: shrq $63, %r9
+; SSE-NEXT: leaq (%r9,%r11,2), %r9
+; SSE-NEXT: shrdq $63, %rdi, %r11
; SSE-NEXT: addq %rsi, %rsi
+; SSE-NEXT: shldq $1, %rdi, %r10
; SSE-NEXT: movq %r10, 56(%rax)
-; SSE-NEXT: movq %rdi, 48(%rax)
-; SSE-NEXT: movq %r11, 40(%rax)
-; SSE-NEXT: movq %r9, 32(%rax)
+; SSE-NEXT: movq %r11, 48(%rax)
+; SSE-NEXT: movq %r9, 40(%rax)
+; SSE-NEXT: movq %rbx, 32(%rax)
; SSE-NEXT: movq %r8, 24(%rax)
; SSE-NEXT: movq %rcx, 16(%rax)
; SSE-NEXT: movq %rdx, 8(%rax)
; SSE-NEXT: movq %rsi, (%rax)
+; SSE-NEXT: popq %rbx
; SSE-NEXT: retq
;
; AVX2-LABEL: shl_i512_1:
; AVX2: # %bb.0:
-; AVX2-NEXT: pushq %r14
; AVX2-NEXT: pushq %rbx
; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdi
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rbx
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
-; AVX2-NEXT: leaq (,%rdx,2), %r14
-; AVX2-NEXT: movq %rsi, %rdi
-; AVX2-NEXT: shrq $63, %rdi
-; AVX2-NEXT: orq %r14, %rdi
-; AVX2-NEXT: leaq (,%rcx,2), %r14
+; AVX2-NEXT: movq %rsi, %rbx
+; AVX2-NEXT: shrq $63, %rbx
+; AVX2-NEXT: leaq (%rbx,%rdx,2), %rbx
; AVX2-NEXT: shrq $63, %rdx
-; AVX2-NEXT: orq %r14, %rdx
-; AVX2-NEXT: leaq (,%r8,2), %r14
+; AVX2-NEXT: leaq (%rdx,%rcx,2), %rdx
; AVX2-NEXT: shrq $63, %rcx
-; AVX2-NEXT: orq %r14, %rcx
-; AVX2-NEXT: leaq (,%r9,2), %r14
+; AVX2-NEXT: leaq (%rcx,%r8,2), %rcx
; AVX2-NEXT: shrq $63, %r8
-; AVX2-NEXT: orq %r14, %r8
-; AVX2-NEXT: leaq (,%r11,2), %r14
+; AVX2-NEXT: leaq (%r8,%r9,2), %r8
; AVX2-NEXT: shrq $63, %r9
-; AVX2-NEXT: orq %r14, %r9
-; AVX2-NEXT: leaq (,%r10,2), %r14
+; AVX2-NEXT: leaq (%r9,%r11,2), %r9
; AVX2-NEXT: shrq $63, %r11
-; AVX2-NEXT: orq %r14, %r11
+; AVX2-NEXT: leaq (%r11,%rdi,2), %r11
; AVX2-NEXT: addq %rsi, %rsi
-; AVX2-NEXT: shlq %rbx
-; AVX2-NEXT: shrq $63, %r10
-; AVX2-NEXT: orq %rbx, %r10
-; AVX2-NEXT: movq %r10, 56(%rax)
+; AVX2-NEXT: shrq $63, %rdi
+; AVX2-NEXT: leaq (%rdi,%r10,2), %rdi
+; AVX2-NEXT: movq %rdi, 56(%rax)
; AVX2-NEXT: movq %r11, 48(%rax)
; AVX2-NEXT: movq %r9, 40(%rax)
; AVX2-NEXT: movq %r8, 32(%rax)
; AVX2-NEXT: movq %rcx, 24(%rax)
; AVX2-NEXT: movq %rdx, 16(%rax)
-; AVX2-NEXT: movq %rdi, 8(%rax)
+; AVX2-NEXT: movq %rbx, 8(%rax)
; AVX2-NEXT: movq %rsi, (%rax)
; AVX2-NEXT: popq %rbx
-; AVX2-NEXT: popq %r14
; AVX2-NEXT: retq
;
; AVX512-LABEL: shl_i512_1:
; AVX512: # %bb.0:
-; AVX512-NEXT: pushq %r14
; AVX512-NEXT: pushq %rbx
; AVX512-NEXT: movq %rdi, %rax
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rdi
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r10
-; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rbx
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r11
-; AVX512-NEXT: leaq (,%rdx,2), %r14
-; AVX512-NEXT: movq %rsi, %rdi
-; AVX512-NEXT: shrq $63, %rdi
-; AVX512-NEXT: orq %r14, %rdi
-; AVX512-NEXT: leaq (,%rcx,2), %r14
+; AVX512-NEXT: movq %rsi, %rbx
+; AVX512-NEXT: shrq $63, %rbx
+; AVX512-NEXT: leaq (%rbx,%rdx,2), %rbx
; AVX512-NEXT: shrq $63, %rdx
-; AVX512-NEXT: orq %r14, %rdx
-; AVX512-NEXT: leaq (,%r8,2), %r14
+; AVX512-NEXT: leaq (%rdx,%rcx,2), %rdx
; AVX512-NEXT: shrq $63, %rcx
-; AVX512-NEXT: orq %r14, %rcx
-; AVX512-NEXT: leaq (,%r9,2), %r14
+; AVX512-NEXT: leaq (%rcx,%r8,2), %rcx
; AVX512-NEXT: shrq $63, %r8
-; AVX512-NEXT: orq %r14, %r8
-; AVX512-NEXT: leaq (,%r11,2), %r14
+; AVX512-NEXT: leaq (%r8,%r9,2), %r8
; AVX512-NEXT: shrq $63, %r9
-; AVX512-NEXT: orq %r14, %r9
-; AVX512-NEXT: leaq (,%r10,2), %r14
+; AVX512-NEXT: leaq (%r9,%r11,2), %r9
; AVX512-NEXT: shrq $63, %r11
-; AVX512-NEXT: orq %r14, %r11
+; AVX512-NEXT: leaq (%r11,%rdi,2), %r11
; AVX512-NEXT: addq %rsi, %rsi
-; AVX512-NEXT: shlq %rbx
-; AVX512-NEXT: shrq $63, %r10
-; AVX512-NEXT: orq %rbx, %r10
-; AVX512-NEXT: movq %r10, 56(%rax)
+; AVX512-NEXT: shrq $63, %rdi
+; AVX512-NEXT: leaq (%rdi,%r10,2), %rdi
+; AVX512-NEXT: movq %rdi, 56(%rax)
; AVX512-NEXT: movq %r11, 48(%rax)
; AVX512-NEXT: movq %r9, 40(%rax)
; AVX512-NEXT: movq %r8, 32(%rax)
; AVX512-NEXT: movq %rcx, 24(%rax)
; AVX512-NEXT: movq %rdx, 16(%rax)
-; AVX512-NEXT: movq %rdi, 8(%rax)
+; AVX512-NEXT: movq %rbx, 8(%rax)
; AVX512-NEXT: movq %rsi, (%rax)
; AVX512-NEXT: popq %rbx
-; AVX512-NEXT: popq %r14
; AVX512-NEXT: retq
%r = shl i512 %a0, 1
ret i512 %r
@@ -1398,46 +1384,46 @@ define i512 @lshr_i512_1(i512 %a0) nounwind {
; AVX2: # %bb.0:
; AVX2-NEXT: pushq %rbx
; AVX2-NEXT: movq %rdi, %rax
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rbx
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdi
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
; AVX2-NEXT: shrq %rsi
-; AVX2-NEXT: movq %rdx, %rdi
-; AVX2-NEXT: shlq $63, %rdi
-; AVX2-NEXT: orq %rsi, %rdi
-; AVX2-NEXT: movq %rcx, %rsi
-; AVX2-NEXT: shlq $63, %rsi
+; AVX2-NEXT: movq %rdx, %rbx
+; AVX2-NEXT: shlq $63, %rbx
+; AVX2-NEXT: addq %rbx, %rsi
+; AVX2-NEXT: movq %rcx, %rbx
+; AVX2-NEXT: shlq $63, %rbx
; AVX2-NEXT: shrq %rdx
-; AVX2-NEXT: orq %rsi, %rdx
+; AVX2-NEXT: addq %rbx, %rdx
; AVX2-NEXT: shrq %rcx
-; AVX2-NEXT: movq %r8, %rsi
-; AVX2-NEXT: shlq $63, %rsi
-; AVX2-NEXT: orq %rcx, %rsi
-; AVX2-NEXT: movq %r9, %rcx
-; AVX2-NEXT: shlq $63, %rcx
+; AVX2-NEXT: movq %r8, %rbx
+; AVX2-NEXT: shlq $63, %rbx
+; AVX2-NEXT: addq %rbx, %rcx
+; AVX2-NEXT: movq %r9, %rbx
+; AVX2-NEXT: shlq $63, %rbx
; AVX2-NEXT: shrq %r8
-; AVX2-NEXT: orq %rcx, %r8
+; AVX2-NEXT: addq %rbx, %r8
; AVX2-NEXT: shrq %r9
-; AVX2-NEXT: movq %r10, %rcx
-; AVX2-NEXT: shlq $63, %rcx
-; AVX2-NEXT: orq %r9, %rcx
-; AVX2-NEXT: movq %rbx, %r9
-; AVX2-NEXT: shlq $63, %r9
-; AVX2-NEXT: shrq %r10
-; AVX2-NEXT: orq %r9, %r10
-; AVX2-NEXT: shrq %rbx
-; AVX2-NEXT: movq %r11, %r9
-; AVX2-NEXT: shlq $63, %r9
-; AVX2-NEXT: orq %rbx, %r9
+; AVX2-NEXT: movq %r11, %rbx
+; AVX2-NEXT: shlq $63, %rbx
+; AVX2-NEXT: addq %rbx, %r9
+; AVX2-NEXT: movq %r10, %rbx
+; AVX2-NEXT: shlq $63, %rbx
; AVX2-NEXT: shrq %r11
-; AVX2-NEXT: movq %r11, 56(%rax)
-; AVX2-NEXT: movq %r9, 48(%rax)
-; AVX2-NEXT: movq %r10, 40(%rax)
-; AVX2-NEXT: movq %rcx, 32(%rax)
+; AVX2-NEXT: addq %rbx, %r11
+; AVX2-NEXT: shrq %r10
+; AVX2-NEXT: movq %rdi, %rbx
+; AVX2-NEXT: shlq $63, %rbx
+; AVX2-NEXT: addq %rbx, %r10
+; AVX2-NEXT: shrq %rdi
+; AVX2-NEXT: movq %rdi, 56(%rax)
+; AVX2-NEXT: movq %r10, 48(%rax)
+; AVX2-NEXT: movq %r11, 40(%rax)
+; AVX2-NEXT: movq %r9, 32(%rax)
; AVX2-NEXT: movq %r8, 24(%rax)
-; AVX2-NEXT: movq %rsi, 16(%rax)
+; AVX2-NEXT: movq %rcx, 16(%rax)
; AVX2-NEXT: movq %rdx, 8(%rax)
-; AVX2-NEXT: movq %rdi, (%rax)
+; AVX2-NEXT: movq %rsi, (%rax)
; AVX2-NEXT: popq %rbx
; AVX2-NEXT: retq
;
@@ -1445,46 +1431,46 @@ define i512 @lshr_i512_1(i512 %a0) nounwind {
; AVX512: # %bb.0:
; AVX512-NEXT: pushq %rbx
; AVX512-NEXT: movq %rdi, %rax
-; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rbx
-; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r11
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rdi
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r11
; AVX512-NEXT: shrq %rsi
-; AVX512-NEXT: movq %rdx, %rdi
-; AVX512-NEXT: shlq $63, %rdi
-; AVX512-NEXT: orq %rsi, %rdi
-; AVX512-NEXT: movq %rcx, %rsi
-; AVX512-NEXT: shlq $63, %rsi
+; AVX512-NEXT: movq %rdx, %rbx
+; AVX512-NEXT: shlq $63, %rbx
+; AVX512-NEXT: addq %rbx, %rsi
+; AVX512-NEXT: movq %rcx, %rbx
+; AVX512-NEXT: shlq $63, %rbx
; AVX512-NEXT: shrq %rdx
-; AVX512-NEXT: orq %rsi, %rdx
+; AVX512-NEXT: addq %rbx, %rdx
; AVX512-NEXT: shrq %rcx
-; AVX512-NEXT: movq %r8, %rsi
-; AVX512-NEXT: shlq $63, %rsi
-; AVX512-NEXT: orq %rcx, %rsi
-; AVX512-NEXT: movq %r9, %rcx
-; AVX512-NEXT: shlq $63, %rcx
+; AVX512-NEXT: movq %r8, %rbx
+; AVX512-NEXT: shlq $63, %rbx
+; AVX512-NEXT: addq %rbx, %rcx
+; AVX512-NEXT: movq %r9, %rbx
+; AVX512-NEXT: shlq $63, %rbx
; AVX512-NEXT: shrq %r8
-; AVX512-NEXT: orq %rcx, %r8
+; AVX512-NEXT: addq %rbx, %r8
; AVX512-NEXT: shrq %r9
-; AVX512-NEXT: movq %r10, %rcx
-; AVX512-NEXT: shlq $63, %rcx
-; AVX512-NEXT: orq %r9, %rcx
-; AVX512-NEXT: movq %rbx, %r9
-; AVX512-NEXT: shlq $63, %r9
-; AVX512-NEXT: shrq %r10
-; AVX512-NEXT: orq %r9, %r10
-; AVX512-NEXT: shrq %rbx
-; AVX512-NEXT: movq %r11, %r9
-; AVX512-NEXT: shlq $63, %r9
-; AVX512-NEXT: orq %rbx, %r9
+; AVX512-NEXT: movq %r11, %rbx
+; AVX512-NEXT: shlq $63, %rbx
+; AVX512-NEXT: addq %rbx, %r9
+; AVX512-NEXT: movq %r10, %rbx
+; AVX512-NEXT: shlq $63, %rbx
; AVX512-NEXT: shrq %r11
-; AVX512-NEXT: movq %r11, 56(%rax)
-; AVX512-NEXT: movq %r9, 48(%rax)
-; AVX512-NEXT: movq %r10, 40(%rax)
-; AVX512-NEXT: movq %rcx, 32(%rax)
+; AVX512-NEXT: addq %rbx, %r11
+; AVX512-NEXT: shrq %r10
+; AVX512-NEXT: movq %rdi, %rbx
+; AVX512-NEXT: shlq $63, %rbx
+; AVX512-NEXT: addq %rbx, %r10
+; AVX512-NEXT: shrq %rdi
+; AVX512-NEXT: movq %rdi, 56(%rax)
+; AVX512-NEXT: movq %r10, 48(%rax)
+; AVX512-NEXT: movq %r11, 40(%rax)
+; AVX512-NEXT: movq %r9, 32(%rax)
; AVX512-NEXT: movq %r8, 24(%rax)
-; AVX512-NEXT: movq %rsi, 16(%rax)
+; AVX512-NEXT: movq %rcx, 16(%rax)
; AVX512-NEXT: movq %rdx, 8(%rax)
-; AVX512-NEXT: movq %rdi, (%rax)
+; AVX512-NEXT: movq %rsi, (%rax)
; AVX512-NEXT: popq %rbx
; AVX512-NEXT: retq
%r = lshr i512 %a0, 1
@@ -1520,46 +1506,46 @@ define i512 @ashr_i512_1(i512 %a0) nounwind {
; AVX2: # %bb.0:
; AVX2-NEXT: pushq %rbx
; AVX2-NEXT: movq %rdi, %rax
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rbx
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdi
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
; AVX2-NEXT: shrq %rsi
-; AVX2-NEXT: movq %rdx, %rdi
-; AVX2-NEXT: shlq $63, %rdi
-; AVX2-NEXT: orq %rsi, %rdi
-; AVX2-NEXT: movq %rcx, %rsi
-; AVX2-NEXT: shlq $63, %rsi
+; AVX2-NEXT: movq %rdx, %rbx
+; AVX2-NEXT: shlq $63, %rbx
+; AVX2-NEXT: addq %rbx, %rsi
+; AVX2-NEXT: movq %rcx, %rbx
+; AVX2-NEXT: shlq $63, %rbx
; AVX2-NEXT: shrq %rdx
-; AVX2-NEXT: orq %rsi, %rdx
+; AVX2-NEXT: addq %rbx, %rdx
; AVX2-NEXT: shrq %rcx
-; AVX2-NEXT: movq %r8, %rsi
-; AVX2-NEXT: shlq $63, %rsi
-; AVX2-NEXT: orq %rcx, %rsi
-; AVX2-NEXT: movq %r9, %rcx
-; AVX2-NEXT: shlq $63, %rcx
+; AVX2-NEXT: movq %r8, %rbx
+; AVX2-NEXT: shlq $63, %rbx
+; AVX2-NEXT: addq %rbx, %rcx
+; AVX2-NEXT: movq %r9, %rbx
+; AVX2-NEXT: shlq $63, %rbx
; AVX2-NEXT: shrq %r8
-; AVX2-NEXT: orq %rcx, %r8
+; AVX2-NEXT: addq %rbx, %r8
; AVX2-NEXT: shrq %r9
-; AVX2-NEXT: movq %r10, %rcx
-; AVX2-NEXT: shlq $63, %rcx
-; AVX2-NEXT: orq %r9, %rcx
-; AVX2-NEXT: movq %rbx, %r9
-; AVX2-NEXT: shlq $63, %r9
+; AVX2-NEXT: movq %r11, %rbx
+; AVX2-NEXT: shlq $63, %rbx
+; AVX2-NEXT: addq %rbx, %r9
+; AVX2-NEXT: movq %r10, %rbx
+; AVX2-NEXT: shlq $63, %rbx
+; AVX2-NEXT: shrq %r11
+; AVX2-NEXT: addq %rbx, %r11
; AVX2-NEXT: shrq %r10
-; AVX2-NEXT: orq %r9, %r10
-; AVX2-NEXT: shrq %rbx
-; AVX2-NEXT: movq %r11, %r9
-; AVX2-NEXT: shlq $63, %r9
-; AVX2-NEXT: orq %rbx, %r9
-; AVX2-NEXT: sarq %r11
-; AVX2-NEXT: movq %r11, 56(%rax)
-; AVX2-NEXT: movq %r9, 48(%rax)
-; AVX2-NEXT: movq %r10, 40(%rax)
-; AVX2-NEXT: movq %rcx, 32(%rax)
+; AVX2-NEXT: movq %rdi, %rbx
+; AVX2-NEXT: shlq $63, %rbx
+; AVX2-NEXT: addq %rbx, %r10
+; AVX2-NEXT: sarq %rdi
+; AVX2-NEXT: movq %rdi, 56(%rax)
+; AVX2-NEXT: movq %r10, 48(%rax)
+; AVX2-NEXT: movq %r11, 40(%rax)
+; AVX2-NEXT: movq %r9, 32(%rax)
; AVX2-NEXT: movq %r8, 24(%rax)
-; AVX2-NEXT: movq %rsi, 16(%rax)
+; AVX2-NEXT: movq %rcx, 16(%rax)
; AVX2-NEXT: movq %rdx, 8(%rax)
-; AVX2-NEXT: movq %rdi, (%rax)
+; AVX2-NEXT: movq %rsi, (%rax)
; AVX2-NEXT: popq %rbx
; AVX2-NEXT: retq
;
@@ -1567,46 +1553,46 @@ define i512 @ashr_i512_1(i512 %a0) nounwind {
; AVX512: # %bb.0:
; AVX512-NEXT: pushq %rbx
; AVX512-NEXT: movq %rdi, %rax
-; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rbx
-; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r11
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rdi
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %r11
; AVX512-NEXT: shrq %rsi
-; AVX512-NEXT: movq %rdx, %rdi
-; AVX512-NEXT: shlq $63, %rdi
-; AVX512-NEXT: orq %rsi, %rdi
-; AVX512-NEXT: movq %rcx, %rsi
-; AVX512-NEXT: shlq $63, %rsi
+; AVX512-NEXT: movq %rdx, %rbx
+; AVX512-NEXT: shlq $63, %rbx
+; AVX512-NEXT: addq %rbx, %rsi
+; AVX512-NEXT: movq %rcx, %rbx
+; AVX512-NEXT: shlq $63, %rbx
; AVX512-NEXT: shrq %rdx
-; AVX512-NEXT: orq %rsi, %rdx
+; AVX512-NEXT: addq %rbx, %rdx
; AVX512-NEXT: shrq %rcx
-; AVX512-NEXT: movq %r8, %rsi
-; AVX512-NEXT: shlq $63, %rsi
-; AVX512-NEXT: orq %rcx, %rsi
-; AVX512-NEXT: movq %r9, %rcx
-; AVX512-NEXT: shlq $63, %rcx
+; AVX512-NEXT: movq %r8, %rbx
+; AVX512-NEXT: shlq $63, %rbx
+; AVX512-NEXT: addq %rbx, %rcx
+; AVX512-NEXT: movq %r9, %rbx
+; AVX512-NEXT: shlq $63, %rbx
; AVX512-NEXT: shrq %r8
-; AVX512-NEXT: orq %rcx, %r8
+; AVX512-NEXT: addq %rbx, %r8
; AVX512-NEXT: shrq %r9
-; AVX512-NEXT: movq %r10, %rcx
-; AVX512-NEXT: shlq $63, %rcx
-; AVX512-NEXT: orq %r9, %rcx
-; AVX512-NEXT: movq %rbx, %r9
-; AVX512-NEXT: shlq $63, %r9
+; AVX512-NEXT: movq %r11, %rbx
+; AVX512-NEXT: shlq $63, %rbx
+; AVX512-NEXT: addq %rbx, %r9
+; AVX512-NEXT: movq %r10, %rbx
+; AVX512-NEXT: shlq $63, %rbx
+; AVX512-NEXT: shrq %r11
+; AVX512-NEXT: addq %rbx, %r11
; AVX512-NEXT: shrq %r10
-; AVX512-NEXT: orq %r9, %r10
-; AVX512-NEXT: shrq %rbx
-; AVX512-NEXT: movq %r11, %r9
-; AVX512-NEXT: shlq $63, %r9
-; AVX512-NEXT: orq %rbx, %r9
-; AVX512-NEXT: sarq %r11
-; AVX512-NEXT: movq %r11, 56(%rax)
-; AVX512-NEXT: movq %r9, 48(%rax)
-; AVX512-NEXT: movq %r10, 40(%rax)
-; AVX512-NEXT: movq %rcx, 32(%rax)
+; AVX512-NEXT: movq %rdi, %rbx
+; AVX512-NEXT: shlq $63, %rbx
+; AVX512-NEXT: addq %rbx, %r10
+; AVX512-NEXT: sarq %rdi
+; AVX512-NEXT: movq %rdi, 56(%rax)
+; AVX512-NEXT: movq %r10, 48(%rax)
+; AVX512-NEXT: movq %r11, 40(%rax)
+; AVX512-NEXT: movq %r9, 32(%rax)
; AVX512-NEXT: movq %r8, 24(%rax)
-; AVX512-NEXT: movq %rsi, 16(%rax)
+; AVX512-NEXT: movq %rcx, 16(%rax)
; AVX512-NEXT: movq %rdx, 8(%rax)
-; AVX512-NEXT: movq %rdi, (%rax)
+; AVX512-NEXT: movq %rsi, (%rax)
; AVX512-NEXT: popq %rbx
; AVX512-NEXT: retq
%r = ashr i512 %a0, 1
@@ -1640,23 +1626,23 @@ define i512 @shl_i512_200(i512 %a0) nounwind {
; AVX2-NEXT: shrq $56, %rdi
; AVX2-NEXT: movq %rdx, %r10
; AVX2-NEXT: shlq $8, %r10
-; AVX2-NEXT: orq %rdi, %r10
-; AVX2-NEXT: movq %rcx, %rdi
-; AVX2-NEXT: shlq $8, %rdi
+; AVX2-NEXT: addq %r10, %rdi
+; AVX2-NEXT: movq %rcx, %r10
+; AVX2-NEXT: shlq $8, %r10
; AVX2-NEXT: shrq $56, %rdx
-; AVX2-NEXT: orq %rdi, %rdx
+; AVX2-NEXT: addq %r10, %rdx
; AVX2-NEXT: shrq $56, %rcx
-; AVX2-NEXT: movq %r8, %rdi
-; AVX2-NEXT: shlq $8, %rdi
-; AVX2-NEXT: orq %rcx, %rdi
+; AVX2-NEXT: movq %r8, %r10
+; AVX2-NEXT: shlq $8, %r10
+; AVX2-NEXT: addq %r10, %rcx
; AVX2-NEXT: shlq $8, %r9
; AVX2-NEXT: shrq $56, %r8
-; AVX2-NEXT: orq %r9, %r8
+; AVX2-NEXT: addq %r9, %r8
; AVX2-NEXT: shlq $8, %rsi
; AVX2-NEXT: movq %r8, 56(%rax)
-; AVX2-NEXT: movq %rdi, 48(%rax)
+; AVX2-NEXT: movq %rcx, 48(%rax)
; AVX2-NEXT: movq %rdx, 40(%rax)
-; AVX2-NEXT: movq %r10, 32(%rax)
+; AVX2-NEXT: movq %rdi, 32(%rax)
; AVX2-NEXT: movq %rsi, 24(%rax)
; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX2-NEXT: vmovaps %xmm0, (%rax)
@@ -1670,23 +1656,23 @@ define i512 @shl_i512_200(i512 %a0) nounwind {
; AVX512-NEXT: shrq $56, %rdi
; AVX512-NEXT: movq %rdx, %r10
; AVX512-NEXT: shlq $8, %r10
-; AVX512-NEXT: orq %rdi, %r10
-; AVX512-NEXT: movq %rcx, %rdi
-; AVX512-NEXT: shlq $8, %rdi
+; AVX512-NEXT: addq %r10, %rdi
+; AVX512-NEXT: movq %rcx, %r10
+; AVX512-NEXT: shlq $8, %r10
; AVX512-NEXT: shrq $56, %rdx
-; AVX512-NEXT: orq %rdi, %rdx
+; AVX512-NEXT: addq %r10, %rdx
; AVX512-NEXT: shrq $56, %rcx
-; AVX512-NEXT: movq %r8, %rdi
-; AVX512-NEXT: shlq $8, %rdi
-; AVX512-NEXT: orq %rcx, %rdi
+; AVX512-NEXT: movq %r8, %r10
+; AVX512-NEXT: shlq $8, %r10
+; AVX512-NEXT: addq %r10, %rcx
; AVX512-NEXT: shlq $8, %r9
; AVX512-NEXT: shrq $56, %r8
-; AVX512-NEXT: orq %r9, %r8
+; AVX512-NEXT: addq %r9, %r8
; AVX512-NEXT: shlq $8, %rsi
; AVX512-NEXT: movq %r8, 56(%rax)
-; AVX512-NEXT: movq %rdi, 48(%rax)
+; AVX512-NEXT: movq %rcx, 48(%rax)
; AVX512-NEXT: movq %rdx, 40(%rax)
-; AVX512-NEXT: movq %r10, 32(%rax)
+; AVX512-NEXT: movq %rdi, 32(%rax)
; AVX512-NEXT: movq %rsi, 24(%rax)
; AVX512-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX512-NEXT: vmovaps %xmm0, (%rax)
@@ -1722,32 +1708,32 @@ define i512 @lshr_i512_200(i512 %a0) nounwind {
; AVX2: # %bb.0:
; AVX2-NEXT: movq %rdi, %rax
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rcx
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rsi
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rsi
; AVX2-NEXT: movq %r9, %rdi
; AVX2-NEXT: shlq $56, %rdi
; AVX2-NEXT: shrq $8, %r8
-; AVX2-NEXT: orq %rdi, %r8
-; AVX2-NEXT: movq %rdx, %rdi
-; AVX2-NEXT: shlq $56, %rdi
+; AVX2-NEXT: addq %r8, %rdi
+; AVX2-NEXT: movq %rsi, %r8
+; AVX2-NEXT: shlq $56, %r8
; AVX2-NEXT: shrq $8, %r9
-; AVX2-NEXT: orq %rdi, %r9
-; AVX2-NEXT: movq %rcx, %rdi
-; AVX2-NEXT: shlq $56, %rdi
-; AVX2-NEXT: shrq $8, %rdx
-; AVX2-NEXT: orq %rdi, %rdx
-; AVX2-NEXT: movq %rsi, %rdi
-; AVX2-NEXT: shlq $56, %rdi
-; AVX2-NEXT: shrq $8, %rcx
-; AVX2-NEXT: orq %rdi, %rcx
+; AVX2-NEXT: addq %r9, %r8
+; AVX2-NEXT: movq %rcx, %r9
+; AVX2-NEXT: shlq $56, %r9
; AVX2-NEXT: shrq $8, %rsi
+; AVX2-NEXT: addq %r9, %rsi
+; AVX2-NEXT: movq %rdx, %r9
+; AVX2-NEXT: shlq $56, %r9
+; AVX2-NEXT: shrq $8, %rcx
+; AVX2-NEXT: addq %r9, %rcx
+; AVX2-NEXT: shrq $8, %rdx
; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX2-NEXT: vmovups %xmm0, 40(%rax)
-; AVX2-NEXT: movq %rsi, 32(%rax)
+; AVX2-NEXT: movq %rdx, 32(%rax)
; AVX2-NEXT: movq %rcx, 24(%rax)
-; AVX2-NEXT: movq %rdx, 16(%rax)
-; AVX2-NEXT: movq %r9, 8(%rax)
-; AVX2-NEXT: movq %r8, (%rax)
+; AVX2-NEXT: movq %rsi, 16(%rax)
+; AVX2-NEXT: movq %r8, 8(%rax)
+; AVX2-NEXT: movq %rdi, (%rax)
; AVX2-NEXT: movq $0, 56(%rax)
; AVX2-NEXT: retq
;
@@ -1755,32 +1741,32 @@ define i512 @lshr_i512_200(i512 %a0) nounwind {
; AVX512: # %bb.0:
; AVX512-NEXT: movq %rdi, %rax
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rcx
-; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rsi
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rsi
; AVX512-NEXT: movq %r9, %rdi
; AVX512-NEXT: shlq $56, %rdi
; AVX512-NEXT: shrq $8, %r8
-; AVX512-NEXT: orq %rdi, %r8
-; AVX512-NEXT: movq %rdx, %rdi
-; AVX512-NEXT: shlq $56, %rdi
+; AVX512-NEXT: addq %r8, %rdi
+; AVX512-NEXT: movq %rsi, %r8
+; AVX512-NEXT: shlq $56, %r8
; AVX512-NEXT: shrq $8, %r9
-; AVX512-NEXT: orq %rdi, %r9
-; AVX512-NEXT: movq %rcx, %rdi
-; AVX512-NEXT: shlq $56, %rdi
-; AVX512-NEXT: shrq $8, %rdx
-; AVX512-NEXT: orq %rdi, %rdx
-; AVX512-NEXT: movq %rsi, %rdi
-; AVX512-NEXT: shlq $56, %rdi
-; AVX512-NEXT: shrq $8, %rcx
-; AVX512-NEXT: orq %rdi, %rcx
+; AVX512-NEXT: addq %r9, %r8
+; AVX512-NEXT: movq %rcx, %r9
+; AVX512-NEXT: shlq $56, %r9
; AVX512-NEXT: shrq $8, %rsi
+; AVX512-NEXT: addq %r9, %rsi
+; AVX512-NEXT: movq %rdx, %r9
+; AVX512-NEXT: shlq $56, %r9
+; AVX512-NEXT: shrq $8, %rcx
+; AVX512-NEXT: addq %r9, %rcx
+; AVX512-NEXT: shrq $8, %rdx
; AVX512-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX512-NEXT: vmovups %xmm0, 40(%rax)
-; AVX512-NEXT: movq %rsi, 32(%rax)
+; AVX512-NEXT: movq %rdx, 32(%rax)
; AVX512-NEXT: movq %rcx, 24(%rax)
-; AVX512-NEXT: movq %rdx, 16(%rax)
-; AVX512-NEXT: movq %r9, 8(%rax)
-; AVX512-NEXT: movq %r8, (%rax)
+; AVX512-NEXT: movq %rsi, 16(%rax)
+; AVX512-NEXT: movq %r8, 8(%rax)
+; AVX512-NEXT: movq %rdi, (%rax)
; AVX512-NEXT: movq $0, 56(%rax)
; AVX512-NEXT: retq
%r = lshr i512 %a0, 200
@@ -1815,70 +1801,70 @@ define i512 @ashr_i512_200(i512 %a0) nounwind {
; AVX2: # %bb.0:
; AVX2-NEXT: movq %rdi, %rax
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rcx
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rsi
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rsi
; AVX2-NEXT: movq %r9, %rdi
; AVX2-NEXT: shlq $56, %rdi
; AVX2-NEXT: shrq $8, %r8
-; AVX2-NEXT: orq %rdi, %r8
-; AVX2-NEXT: movq %rdx, %rdi
-; AVX2-NEXT: shlq $56, %rdi
+; AVX2-NEXT: addq %r8, %rdi
+; AVX2-NEXT: movq %rsi, %r8
+; AVX2-NEXT: shlq $56, %r8
; AVX2-NEXT: shrq $8, %r9
-; AVX2-NEXT: orq %rdi, %r9
-; AVX2-NEXT: movq %rcx, %rdi
-; AVX2-NEXT: shlq $56, %rdi
-; AVX2-NEXT: shrq $8, %rdx
-; AVX2-NEXT: orq %rdi, %rdx
-; AVX2-NEXT: movq %rsi, %rdi
-; AVX2-NEXT: shlq $56, %rdi
+; AVX2-NEXT: addq %r9, %r8
+; AVX2-NEXT: movq %rcx, %r9
+; AVX2-NEXT: shlq $56, %r9
+; AVX2-NEXT: shrq $8, %rsi
+; AVX2-NEXT: addq %r9, %rsi
+; AVX2-NEXT: movq %rdx, %r9
+; AVX2-NEXT: shlq $56, %r9
; AVX2-NEXT: shrq $8, %rcx
-; AVX2-NEXT: orq %rdi, %rcx
-; AVX2-NEXT: movq %rsi, %rdi
-; AVX2-NEXT: sarq $8, %rdi
-; AVX2-NEXT: sarq $63, %rsi
-; AVX2-NEXT: movq %rsi, 56(%rax)
-; AVX2-NEXT: movq %rsi, 48(%rax)
-; AVX2-NEXT: movq %rsi, 40(%rax)
-; AVX2-NEXT: movq %rdi, 32(%rax)
+; AVX2-NEXT: addq %r9, %rcx
+; AVX2-NEXT: movq %rdx, %r9
+; AVX2-NEXT: sarq $8, %r9
+; AVX2-NEXT: sarq $63, %rdx
+; AVX2-NEXT: movq %rdx, 56(%rax)
+; AVX2-NEXT: movq %rdx, 48(%rax)
+; AVX2-NEXT: movq %rdx, 40(%rax)
+; AVX2-NEXT: movq %r9, 32(%rax)
; AVX2-NEXT: movq %rcx, 24(%rax)
-; AVX2-NEXT: movq %rdx, 16(%rax)
-; AVX2-NEXT: movq %r9, 8(%rax)
-; AVX2-NEXT: movq %r8, (%rax)
+; AVX2-NEXT: movq %rsi, 16(%rax)
+; AVX2-NEXT: movq %r8, 8(%rax)
+; AVX2-NEXT: movq %rdi, (%rax)
; AVX2-NEXT: retq
;
; AVX512-LABEL: ashr_i512_200:
; AVX512: # %bb.0:
; AVX512-NEXT: movq %rdi, %rax
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rcx
-; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rsi
; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rsi
; AVX512-NEXT: movq %r9, %rdi
; AVX512-NEXT: shlq $56, %rdi
; AVX512-NEXT: shrq $8, %r8
-; AVX512-NEXT: orq %rdi, %r8
-; AVX512-NEXT: movq %rdx, %rdi
-; AVX512-NEXT: shlq $56, %rdi
+; AVX512-NEXT: addq %r8, %rdi
+; AVX512-NEXT: movq %rsi, %r8
+; AVX512-NEXT: shlq $56, %r8
; AVX512-NEXT: shrq $8, %r9
-; AVX512-NEXT: orq %rdi, %r9
-; AVX512-NEXT: movq %rcx, %rdi
-; AVX512-NEXT: shlq $56, %rdi
-; AVX512-NEXT: shrq $8, %rdx
-; AVX512-NEXT: orq %rdi, %rdx
-; AVX512-NEXT: movq %rsi, %rdi
-; AVX512-NEXT: shlq $56, %rdi
+; AVX512-NEXT: addq %r9, %r8
+; AVX512-NEXT: movq %rcx, %r9
+; AVX512-NEXT: shlq $56, %r9
+; AVX512-NEXT: shrq $8, %rsi
+; AVX512-NEXT: addq %r9, %rsi
+; AVX512-NEXT: movq %rdx, %r9
+; AVX512-NEXT: shlq $56, %r9
; AVX512-NEXT: shrq $8, %rcx
-; AVX512-NEXT: orq %rdi, %rcx
-; AVX512-NEXT: movq %rsi, %rdi
-; AVX512-NEXT: sarq $8, %rdi
-; AVX512-NEXT: sarq $63, %rsi
-; AVX512-NEXT: movq %rsi, 56(%rax)
-; AVX512-NEXT: movq %rsi, 48(%rax)
-; AVX512-NEXT: movq %rsi, 40(%rax)
-; AVX512-NEXT: movq %rdi, 32(%rax)
+; AVX512-NEXT: addq %r9, %rcx
+; AVX512-NEXT: movq %rdx, %r9
+; AVX512-NEXT: sarq $8, %r9
+; AVX512-NEXT: sarq $63, %rdx
+; AVX512-NEXT: movq %rdx, 56(%rax)
+; AVX512-NEXT: movq %rdx, 48(%rax)
+; AVX512-NEXT: movq %rdx, 40(%rax)
+; AVX512-NEXT: movq %r9, 32(%rax)
; AVX512-NEXT: movq %rcx, 24(%rax)
-; AVX512-NEXT: movq %rdx, 16(%rax)
-; AVX512-NEXT: movq %r9, 8(%rax)
-; AVX512-NEXT: movq %r8, (%rax)
+; AVX512-NEXT: movq %rsi, 16(%rax)
+; AVX512-NEXT: movq %r8, 8(%rax)
+; AVX512-NEXT: movq %rdi, (%rax)
; AVX512-NEXT: retq
%r = ashr i512 %a0, 200
ret i512 %r
diff --git a/llvm/test/CodeGen/X86/wide-scalar-shift-by-byte-multiple-legalization.ll b/llvm/test/CodeGen/X86/wide-scalar-shift-by-byte-multiple-legalization.ll
index 77180c6aa1b10..8f4a960ea8c3f 100644
--- a/llvm/test/CodeGen/X86/wide-scalar-shift-by-byte-multiple-legalization.ll
+++ b/llvm/test/CodeGen/X86/wide-scalar-shift-by-byte-multiple-legalization.ll
@@ -1,36 +1,20 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2,-bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-SSE2,X64-NO-BMI2,X64-NO-SHLD-NO-BMI2,X64-NO-SHLD-NO-BMI2-SSE2
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2,-bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-SSE2,X64-NO-BMI2,X64-HAVE-SHLD-NO-BMI2,X64-HAVE-SHLD-NO-BMI2-SSE2
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2,+bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-SSE2,X64-HAVE-BMI2,X64-NO-SHLD-HAVE-BMI2,X64-NO-SHLD-HAVE-BMI2-SSE2
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2,+bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-SSE2,X64-HAVE-BMI2,X64-HAVE-SHLD-HAVE-BMI2,X64-HAVE-SHLD-HAVE-BMI2-SSE2
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse4.2,-bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-SSE42,X64-NO-BMI2,X64-NO-SHLD-NO-BMI2,X64-NO-SHLD-NO-BMI2-SSE4
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse4.2,-bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-SSE42,X64-NO-BMI2,X64-HAVE-SHLD-NO-BMI2,X64-HAVE-SHLD-NO-BMI2-SSE4
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse4.2,+bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-SSE42,X64-HAVE-BMI2,X64-NO-SHLD-HAVE-BMI2,X64-NO-SHLD-HAVE-BMI2-SSE4
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse4.2,+bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-SSE42,X64-HAVE-BMI2,X64-HAVE-SHLD-HAVE-BMI2,X64-HAVE-SHLD-HAVE-BMI2-SSE4
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx,-bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-AVX,X64-AVX1,X64-NO-BMI2,X64-NO-SHLD-NO-BMI2,X64-NO-SHLD-NO-BMI2-AVX,X64-NO-SHLD-NO-BMI2-AVX1
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx,-bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-AVX,X64-AVX1,X64-NO-BMI2,X64-HAVE-SHLD-NO-BMI2,X64-HAVE-SHLD-NO-BMI2-AVX,X64-HAVE-SHLD-NO-BMI2-AVX1
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx,+bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-AVX,X64-AVX1,X64-HAVE-BMI2,X64-NO-SHLD-HAVE-BMI2,X64-NO-SHLD-HAVE-BMI2-AVX,X64-NO-SHLD-HAVE-BMI2-AVX1
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx,+bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-AVX,X64-AVX1,X64-HAVE-BMI2,X64-HAVE-SHLD-HAVE-BMI2,X64-HAVE-SHLD-HAVE-BMI2-AVX,X64-HAVE-SHLD-HAVE-BMI2-AVX1
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512vl,-bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-AVX,X64-AVX512,X64-NO-BMI2,X64-NO-SHLD-NO-BMI2,X64-NO-SHLD-NO-BMI2-AVX,X64-NO-BMI2-AVX512,X64-NO-SHLD-NO-BMI2-AVX512
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512vl,-bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-AVX,X64-AVX512,X64-NO-BMI2,X64-HAVE-SHLD-NO-BMI2,X64-HAVE-SHLD-NO-BMI2-AVX,X64-NO-BMI2-AVX512,X64-HAVE-SHLD-NO-BMI2-AVX512
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512vl,+bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-AVX,X64-AVX512,X64-HAVE-BMI2,X64-NO-SHLD-HAVE-BMI2,X64-NO-SHLD-HAVE-BMI2-AVX,X64-HAVE-BMI2-AVX512,X64-NO-SHLD-HAVE-BMI2-AVX512
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512vl,+bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-AVX,X64-AVX512,X64-HAVE-BMI2,X64-HAVE-SHLD-HAVE-BMI2,X64-HAVE-SHLD-HAVE-BMI2-AVX,X64-HAVE-BMI2-AVX512,X64-HAVE-SHLD-HAVE-BMI2-AVX512
-; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse2,-bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-SSE2,X86-NO-BMI2,X86-NO-SHLD-NO-BMI2,X86-NO-SHLD-NO-BMI2-SSE2
-; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse2,-bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-SSE2,X86-NO-BMI2,X86-HAVE-SHLD-NO-BMI2,X86-HAVE-SHLD-NO-BMI2-SSE2
-; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse2,+bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-SSE2,X86-HAVE-BMI2,X86-NO-SHLD-HAVE-BMI2,X86-NO-SHLD-HAVE-BMI2-SSE2
-; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse2,+bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-SSE2,X86-HAVE-BMI2,X86-HAVE-SHLD-HAVE-BMI2,X86-HAVE-SHLD-HAVE-BMI2-SSE2
-; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse4.2,-bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-SSE42,X86-NO-BMI2,X86-NO-SHLD-NO-BMI2,X86-NO-SHLD-NO-BMI2-SSE4
-; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse4.2,-bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-SSE42,X86-NO-BMI2,X86-HAVE-SHLD-NO-BMI2,X86-HAVE-SHLD-NO-BMI2-SSE4
-; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse4.2,+bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-SSE42,X86-HAVE-BMI2,X86-NO-SHLD-HAVE-BMI2,X86-NO-SHLD-HAVE-BMI2-SSE4
-; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse4.2,+bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-SSE42,X86-HAVE-BMI2,X86-HAVE-SHLD-HAVE-BMI2,X86-HAVE-SHLD-HAVE-BMI2-SSE4
-; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+avx,-bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-AVX,X86-AVX1,X86-NO-BMI2,X86-NO-SHLD-NO-BMI2,X86-NO-SHLD-NO-BMI2-AVX,X86-NO-SHLD-NO-BMI2-AVX1
-; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+avx,-bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-AVX,X86-AVX1,X86-NO-BMI2,X86-HAVE-SHLD-NO-BMI2,X86-HAVE-SHLD-NO-BMI2-AVX,X86-HAVE-SHLD-NO-BMI2-AVX1
-; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+avx,+bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-AVX,X86-AVX1,X86-HAVE-BMI2,X86-NO-SHLD-HAVE-BMI2,X86-NO-SHLD-HAVE-BMI2-AVX,X86-NO-SHLD-HAVE-BMI2-AVX1
-; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+avx,+bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-AVX,X86-AVX1,X86-HAVE-BMI2,X86-HAVE-SHLD-HAVE-BMI2,X86-HAVE-SHLD-HAVE-BMI2-AVX,X86-HAVE-SHLD-HAVE-BMI2-AVX1
-; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+avx512vl,-bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-AVX,X86-AVX512,X86-NO-BMI2,X86-NO-SHLD-NO-BMI2,X86-NO-SHLD-NO-BMI2-AVX,X86-NO-BMI2-AVX512,X86-NO-SHLD-NO-BMI2-AVX512
-; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+avx512vl,-bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-AVX,X86-AVX512,X86-NO-BMI2,X86-HAVE-SHLD-NO-BMI2,X86-HAVE-SHLD-NO-BMI2-AVX,X86-NO-BMI2-AVX512,X86-HAVE-SHLD-NO-BMI2-AVX512
-; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+avx512vl,+bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-AVX,X86-AVX512,X86-HAVE-BMI2,X86-NO-SHLD-HAVE-BMI2,X86-NO-SHLD-HAVE-BMI2-AVX,X86-HAVE-BMI2-AVX512,X86-NO-SHLD-HAVE-BMI2-AVX512
-; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+avx512vl,+bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-AVX,X86-AVX512,X86-HAVE-BMI2,X86-HAVE-SHLD-HAVE-BMI2,X86-HAVE-SHLD-HAVE-BMI2-AVX,X86-HAVE-BMI2-AVX512,X86-HAVE-SHLD-HAVE-BMI2-AVX512
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2,-bmi2 | FileCheck %s --check-prefixes=ALL,X64,X64-SSE2,X64-NO-BMI2
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2,+bmi2 | FileCheck %s --check-prefixes=ALL,X64,X64-SSE2,X64-HAVE-BMI2
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse4.2,-bmi2 | FileCheck %s --check-prefixes=ALL,X64,X64-SSE42,X64-NO-BMI2
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse4.2,+bmi2 | FileCheck %s --check-prefixes=ALL,X64,X64-SSE42,X64-HAVE-BMI2
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx,-bmi2 | FileCheck %s --check-prefixes=ALL,X64,X64-AVX,X64-AVX1,X64-NO-BMI2
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx,+bmi2 | FileCheck %s --check-prefixes=ALL,X64,X64-AVX,X64-AVX1,X64-HAVE-BMI2
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512vl,-bmi2 | FileCheck %s --check-prefixes=ALL,X64,X64-AVX,X64-AVX512,X64-NO-BMI2,X64-NO-BMI2-AVX512
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx512vl,+bmi2 | FileCheck %s --check-prefixes=ALL,X64,X64-AVX,X64-AVX512,X64-HAVE-BMI2,X64-HAVE-BMI2-AVX512
+; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse2,-bmi2 | FileCheck %s --check-prefixes=ALL,X86,X86-SSE2,X86-NO-BMI2
+; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse2,+bmi2 | FileCheck %s --check-prefixes=ALL,X86,X86-SSE2,X86-HAVE-BMI2
+; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse4.2,-bmi2 | FileCheck %s --check-prefixes=ALL,X86,X86-SSE42,X86-NO-BMI2
+; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse4.2,+bmi2 | FileCheck %s --check-prefixes=ALL,X86,X86-SSE42,X86-HAVE-BMI2
+; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+avx,-bmi2 | FileCheck %s --check-prefixes=ALL,X86,X86-AVX,X86-AVX1,X86-NO-BMI2
+; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+avx,+bmi2 | FileCheck %s --check-prefixes=ALL,X86,X86-AVX,X86-AVX1,X86-HAVE-BMI2
+; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+avx512vl,-bmi2 | FileCheck %s --check-prefixes=ALL,X86,X86-AVX,X86-AVX512,X86-NO-BMI2,X86-NO-BMI2-AVX512
+; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+avx512vl,+bmi2 | FileCheck %s --check-prefixes=ALL,X86,X86-AVX,X86-AVX512,X86-HAVE-BMI2,X86-HAVE-BMI2-AVX512
define void @lshr_4bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-BMI2-LABEL: lshr_4bytes:
diff --git a/llvm/test/CodeGen/X86/wide-scalar-shift-legalization.ll b/llvm/test/CodeGen/X86/wide-scalar-shift-legalization.ll
index 98bc1b0b95747..78a153fbb5918 100644
--- a/llvm/test/CodeGen/X86/wide-scalar-shift-legalization.ll
+++ b/llvm/test/CodeGen/X86/wide-scalar-shift-legalization.ll
@@ -1,12 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2,-bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-NO-BMI2,X64-NO-SHLD,X64-NO-BMI2-NO-SHLD
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2,-bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-NO-BMI2,X64-SHLD,X64-NO-BMI2-HAVE-SHLD
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2,+bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-BMI2,X64-NO-SHLD,X64-HAVE-BMI2-NO-SHLD
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2,+bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-BMI2,X64-SHLD,X64-HAVE-BMI2-HAVE-SHLD
-; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse2,-bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-NO-BMI2,X86-NO-SHLD,X86-NO-BMI2-NO-SHLD
-; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse2,-bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-NO-BMI2,X86-SHLD,X86-NO-BMI2-HAVE-SHLD
-; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse2,+bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-BMI2,X86-NO-SHLD,X86-HAVE-BMI2-NO-SHLD
-; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse2,+bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-BMI2,X86-SHLD,X86-HAVE-BMI2-HAVE-SHLD
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2,-bmi2 | FileCheck %s --check-prefixes=ALL,X64,X64-NO-BMI2
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2,+bmi2 | FileCheck %s --check-prefixes=ALL,X64,X64-BMI2
+; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse2,-bmi2 | FileCheck %s --check-prefixes=ALL,X86,X86-NO-BMI2
+; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse2,+bmi2 | FileCheck %s --check-prefixes=ALL,X86,X86-BMI2
define void @lshr_4bytes(ptr %src.ptr, ptr %bitOff.ptr, ptr %dst) nounwind {
; X64-NO-BMI2-LABEL: lshr_4bytes:
diff --git a/llvm/test/CodeGen/X86/widen-load-of-small-alloca-with-zero-upper-half.ll b/llvm/test/CodeGen/X86/widen-load-of-small-alloca-with-zero-upper-half.ll
index c436002369a50..dcee54e621b8b 100644
--- a/llvm/test/CodeGen/X86/widen-load-of-small-alloca-with-zero-upper-half.ll
+++ b/llvm/test/CodeGen/X86/widen-load-of-small-alloca-with-zero-upper-half.ll
@@ -1,12 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2,-bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-NO-BMI2,X64-NO-SHLD,X64-NO-BMI2-NO-SHLD
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2,-bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-NO-BMI2,X64-SHLD,X64-NO-BMI2-HAVE-SHLD
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2,+bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-BMI2,X64-NO-SHLD,X64-HAVE-BMI2-NO-SHLD
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2,+bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-BMI2,X64-SHLD,X64-HAVE-BMI2-HAVE-SHLD
-; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse2,-bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-NO-BMI2,X86-NO-SHLD,X86-NO-BMI2-NO-SHLD
-; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse2,-bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-NO-BMI2,X86-SHLD,X86-NO-BMI2-HAVE-SHLD
-; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse2,+bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-BMI2,X86-NO-SHLD,X86-HAVE-BMI2-NO-SHLD
-; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse2,+bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-BMI2,X86-SHLD,X86-HAVE-BMI2-HAVE-SHLD
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2,-bmi2 | FileCheck %s --check-prefixes=ALL,X64,X64-NO-BMI2
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2,+bmi2 | FileCheck %s --check-prefixes=ALL,X64,X64-BMI2
+; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse2,-bmi2 | FileCheck %s --check-prefixes=ALL,X86,X86-NO-BMI2
+; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse2,+bmi2 | FileCheck %s --check-prefixes=ALL,X86,X86-BMI2
define void @load_1byte_chunk_of_2byte_alloca_with_zero_upper_half(ptr %src, i64 %byteOff, ptr %dst) nounwind {
; X64-NO-BMI2-LABEL: load_1byte_chunk_of_2byte_alloca_with_zero_upper_half:
diff --git a/llvm/test/CodeGen/X86/widen-load-of-small-alloca.ll b/llvm/test/CodeGen/X86/widen-load-of-small-alloca.ll
index a629fdbbddd16..71c273a31a70f 100644
--- a/llvm/test/CodeGen/X86/widen-load-of-small-alloca.ll
+++ b/llvm/test/CodeGen/X86/widen-load-of-small-alloca.ll
@@ -1,12 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2,-bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-NO-BMI2,X64-NO-SHLD,X64-NO-BMI2-NO-SHLD
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2,-bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-NO-BMI2,X64-SHLD,X64-NO-BMI2-HAVE-SHLD
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2,+bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-BMI2,X64-NO-SHLD,X64-HAVE-BMI2-NO-SHLD
-; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2,+bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X64,X64-BMI2,X64-SHLD,X64-HAVE-BMI2-HAVE-SHLD
-; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse2,-bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-NO-BMI2,X86-NO-SHLD,X86-NO-BMI2-NO-SHLD
-; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse2,-bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-NO-BMI2,X86-SHLD,X86-NO-BMI2-HAVE-SHLD
-; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse2,+bmi2,+slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-BMI2,X86-NO-SHLD,X86-HAVE-BMI2-NO-SHLD
-; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse2,+bmi2,-slow-shld | FileCheck %s --check-prefixes=ALL,X86,X86-BMI2,X86-SHLD,X86-HAVE-BMI2-HAVE-SHLD
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2,-bmi2 | FileCheck %s --check-prefixes=ALL,X64,X64-NO-BMI2
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2,+bmi2 | FileCheck %s --check-prefixes=ALL,X64,X64-BMI2
+; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse2,-bmi2 | FileCheck %s --check-prefixes=ALL,X86,X86-NO-BMI2
+; RUN: llc < %s -mtriple=i686-unknown-linux-gnu -mattr=+sse2,+bmi2 | FileCheck %s --check-prefixes=ALL,X86,X86-BMI2
; no @load_1byte_chunk_of_1byte_alloca
diff --git a/llvm/test/CodeGen/X86/x86-64-double-precision-shift-left.ll b/llvm/test/CodeGen/X86/x86-64-double-precision-shift-left.ll
index 1c7f6045961fe..f4138be3c9b7c 100644
--- a/llvm/test/CodeGen/X86/x86-64-double-precision-shift-left.ll
+++ b/llvm/test/CodeGen/X86/x86-64-double-precision-shift-left.ll
@@ -1,5 +1,5 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=bdver1 | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=znver1 | FileCheck %s
; Verify that for the architectures that are known to have poor latency
; double precision shift instructions we generate alternative sequence
@@ -13,9 +13,8 @@
define i64 @lshift1(i64 %a, i64 %b) nounwind readnone uwtable {
; CHECK-LABEL: lshift1:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: leaq (,%rdi,2), %rax
; CHECK-NEXT: shrq $63, %rsi
-; CHECK-NEXT: orq %rsi, %rax
+; CHECK-NEXT: leaq (%rsi,%rdi,2), %rax
; CHECK-NEXT: retq
entry:
%shl = shl i64 %a, 1
@@ -32,9 +31,8 @@ entry:
define i64 @lshift2(i64 %a, i64 %b) nounwind readnone uwtable {
; CHECK-LABEL: lshift2:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: leaq (,%rdi,4), %rax
; CHECK-NEXT: shrq $62, %rsi
-; CHECK-NEXT: orq %rsi, %rax
+; CHECK-NEXT: leaq (%rsi,%rdi,4), %rax
; CHECK-NEXT: retq
entry:
%shl = shl i64 %a, 2
@@ -51,10 +49,9 @@ entry:
define i64 @lshift7(i64 %a, i64 %b) nounwind readnone uwtable {
; CHECK-LABEL: lshift7:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: movq %rsi, %rax
; CHECK-NEXT: shlq $7, %rdi
-; CHECK-NEXT: shrq $57, %rax
-; CHECK-NEXT: orq %rdi, %rax
+; CHECK-NEXT: shrq $57, %rsi
+; CHECK-NEXT: leaq (%rdi,%rsi), %rax
; CHECK-NEXT: retq
entry:
%shl = shl i64 %a, 7
@@ -71,10 +68,9 @@ entry:
define i64 @lshift63(i64 %a, i64 %b) nounwind readnone uwtable {
; CHECK-LABEL: lshift63:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: movq %rsi, %rax
; CHECK-NEXT: shlq $63, %rdi
-; CHECK-NEXT: shrq %rax
-; CHECK-NEXT: orq %rdi, %rax
+; CHECK-NEXT: shrq %rsi
+; CHECK-NEXT: leaq (%rdi,%rsi), %rax
; CHECK-NEXT: retq
entry:
%shl = shl i64 %a, 63
diff --git a/llvm/test/CodeGen/X86/x86-64-double-precision-shift-right.ll b/llvm/test/CodeGen/X86/x86-64-double-precision-shift-right.ll
index f2687599e3c6f..4fbda3e1e2dfd 100644
--- a/llvm/test/CodeGen/X86/x86-64-double-precision-shift-right.ll
+++ b/llvm/test/CodeGen/X86/x86-64-double-precision-shift-right.ll
@@ -13,10 +13,9 @@
define i64 @rshift1(i64 %a, i64 %b) nounwind readnone uwtable {
; CHECK-LABEL: rshift1:
; CHECK: # %bb.0:
-; CHECK-NEXT: movq %rdi, %rax
; CHECK-NEXT: shlq $63, %rsi
-; CHECK-NEXT: shrq %rax
-; CHECK-NEXT: orq %rsi, %rax
+; CHECK-NEXT: shrq %rdi
+; CHECK-NEXT: leaq (%rsi,%rdi), %rax
; CHECK-NEXT: retq
%1 = lshr i64 %a, 1
%2 = shl i64 %b, 63
@@ -32,10 +31,9 @@ define i64 @rshift1(i64 %a, i64 %b) nounwind readnone uwtable {
define i64 @rshift2(i64 %a, i64 %b) nounwind readnone uwtable {
; CHECK-LABEL: rshift2:
; CHECK: # %bb.0:
-; CHECK-NEXT: movq %rdi, %rax
; CHECK-NEXT: shlq $62, %rsi
-; CHECK-NEXT: shrq $2, %rax
-; CHECK-NEXT: orq %rsi, %rax
+; CHECK-NEXT: shrq $2, %rdi
+; CHECK-NEXT: leaq (%rsi,%rdi), %rax
; CHECK-NEXT: retq
%1 = lshr i64 %a, 2
%2 = shl i64 %b, 62
@@ -51,10 +49,9 @@ define i64 @rshift2(i64 %a, i64 %b) nounwind readnone uwtable {
define i64 @rshift7(i64 %a, i64 %b) nounwind readnone uwtable {
; CHECK-LABEL: rshift7:
; CHECK: # %bb.0:
-; CHECK-NEXT: movq %rdi, %rax
; CHECK-NEXT: shlq $57, %rsi
-; CHECK-NEXT: shrq $7, %rax
-; CHECK-NEXT: orq %rsi, %rax
+; CHECK-NEXT: shrq $7, %rdi
+; CHECK-NEXT: leaq (%rsi,%rdi), %rax
; CHECK-NEXT: retq
%1 = lshr i64 %a, 7
%2 = shl i64 %b, 57
diff --git a/llvm/test/CodeGen/X86/x86-64-double-shifts-Oz-Os-O2.ll b/llvm/test/CodeGen/X86/x86-64-double-shifts-Oz-Os-O2.ll
index e9444734884c6..6e6e348a58739 100644
--- a/llvm/test/CodeGen/X86/x86-64-double-shifts-Oz-Os-O2.ll
+++ b/llvm/test/CodeGen/X86/x86-64-double-shifts-Oz-Os-O2.ll
@@ -14,10 +14,9 @@
define i64 @_Z8lshift10mm(i64 %a, i64 %b) #0 {
; CHECK-LABEL: _Z8lshift10mm:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: movq %rsi, %rax
; CHECK-NEXT: shlq $10, %rdi
-; CHECK-NEXT: shrq $54, %rax
-; CHECK-NEXT: orq %rdi, %rax
+; CHECK-NEXT: shrq $54, %rsi
+; CHECK-NEXT: leaq (%rdi,%rsi), %rax
; CHECK-NEXT: retq
entry:
%shl = shl i64 %a, 10
@@ -42,10 +41,9 @@ attributes #0 = { minsize nounwind readnone uwtable "less-precise-fpmad"="false"
define i64 @_Z8lshift11mm(i64 %a, i64 %b) #1 {
; CHECK-LABEL: _Z8lshift11mm:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: movq %rsi, %rax
; CHECK-NEXT: shlq $11, %rdi
-; CHECK-NEXT: shrq $53, %rax
-; CHECK-NEXT: orq %rdi, %rax
+; CHECK-NEXT: shrq $53, %rsi
+; CHECK-NEXT: leaq (%rdi,%rsi), %rax
; CHECK-NEXT: retq
entry:
%shl = shl i64 %a, 11
@@ -57,10 +55,9 @@ entry:
define i64 @_Z8lshift11mm_pgso(i64 %a, i64 %b) !prof !14 {
; CHECK-LABEL: _Z8lshift11mm_pgso:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: movq %rsi, %rax
; CHECK-NEXT: shlq $11, %rdi
-; CHECK-NEXT: shrq $53, %rax
-; CHECK-NEXT: orq %rdi, %rax
+; CHECK-NEXT: shrq $53, %rsi
+; CHECK-NEXT: leaq (%rdi,%rsi), %rax
; CHECK-NEXT: retq
entry:
%shl = shl i64 %a, 11
@@ -84,10 +81,9 @@ attributes #1 = { nounwind optsize readnone uwtable "less-precise-fpmad"="false"
define i64 @_Z8lshift12mm(i64 %a, i64 %b) #2 {
; CHECK-LABEL: _Z8lshift12mm:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: movq %rsi, %rax
; CHECK-NEXT: shlq $12, %rdi
-; CHECK-NEXT: shrq $52, %rax
-; CHECK-NEXT: orq %rdi, %rax
+; CHECK-NEXT: shrq $52, %rsi
+; CHECK-NEXT: leaq (%rdi,%rsi), %rax
; CHECK-NEXT: retq
entry:
%shl = shl i64 %a, 12
>From a5276dad07244ff6d63ccafb7b2caa8b94e752b2 Mon Sep 17 00:00:00 2001
From: stomfaig <stomfaig at gmail.com>
Date: Fri, 1 May 2026 15:58:09 +0100
Subject: [PATCH 11/11] handle cl cases
---
llvm/lib/Target/X86/X86InstrInfo.cpp | 147 +-
llvm/lib/Target/X86/X86InstrInfo.h | 1 +
llvm/test/CodeGen/X86/bit-manip-i256.ll | 320 ++-
llvm/test/CodeGen/X86/extract-bits.ll | 134 +-
llvm/test/CodeGen/X86/extract-lowbits.ll | 164 +-
llvm/test/CodeGen/X86/fshl.ll | 207 +-
llvm/test/CodeGen/X86/fshr.ll | 203 +-
llvm/test/CodeGen/X86/rot32.ll | 66 +-
llvm/test/CodeGen/X86/shift-i256.ll | 2426 ++++++++++++++--------
llvm/test/CodeGen/X86/shift-i512.ll | 1397 +++++++++----
10 files changed, 3445 insertions(+), 1620 deletions(-)
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index 33ff43ebe2d34..805fff17e4a2d 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -10640,6 +10640,7 @@ CombinerObjective X86InstrInfo::getCombinerObjective(unsigned Pattern) const {
switch (Pattern) {
case X86MachineCombinerPattern::USHD_OR:
case X86MachineCombinerPattern::USHD_LEA:
+ case X86MachineCombinerPattern::USHD_SHLX_OR:
return CombinerObjective::MustReduceLatency;
default:
return TargetInstrInfo::getCombinerObjective(Pattern);
@@ -10673,7 +10674,6 @@ bool X86InstrInfo::getMachineCombinerPatterns(
}
break;
}
- // We do not support CL variant, as it requires a lot of bookkeeping.
case X86::SHLD32rri8:
case X86::SHLD64rri8:
case X86::SHRD32rri8:
@@ -10687,6 +10687,23 @@ bool X86InstrInfo::getMachineCombinerPatterns(
case X86::SHRD64mri8:
Patterns.push_back(X86MachineCombinerPattern::USHD_OR);
return true;
+ // With BMI2, one SH*rCL/SH*mCL uses the existing CL directly; the
+ // complement shift uses SHLX/SHRX with a negated copy of CL, avoiding
+ // physical-register conflicts. LEA is inapplicable (needs compile-time
+ // scale).
+ case X86::SHLD32rrCL:
+ case X86::SHLD64rrCL:
+ case X86::SHRD32rrCL:
+ case X86::SHRD64rrCL:
+ case X86::SHLD32mrCL:
+ case X86::SHLD64mrCL:
+ case X86::SHRD32mrCL:
+ case X86::SHRD64mrCL:
+ if (Subtarget.hasBMI2()) {
+ Patterns.push_back(X86MachineCombinerPattern::USHD_SHLX_OR);
+ return true;
+ }
+ break;
}
return TargetInstrInfo::getMachineCombinerPatterns(Root,
Patterns, DoRegPressureReduce);
@@ -10920,6 +10937,131 @@ genShdLeaSequence(MachineInstr &Root, const TargetInstrInfo &TII,
DelInstrs.push_back(&Root);
}
+// Expand SH_D_rCL into SH_rCL and SH_X + OR, the idea being
+// that we can spare a register by knowing that one of the operands is
+// already in CL.
+// We have to introduce a new register, otherwise all the operations would
+// serialise on CL accesses. This is why this optimization only applies
+// when BMI2 is available.
+static void
+genShdShlxOrSequence(MachineInstr &Root, const TargetInstrInfo &TII,
+ SmallVectorImpl<MachineInstr *> &InsInstrs,
+ SmallVectorImpl<MachineInstr *> &DelInstrs,
+ DenseMap<Register, unsigned> &InstrIdxForVirtReg) {
+ auto *MF = Root.getMF();
+ MachineRegisterInfo &RegInfo = MF->getRegInfo();
+ const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
+ auto GetRC = [&](Register Reg) {
+ return Reg.isVirtual() ? RegInfo.getRegClass(Reg)
+ : TRI->getMinimalPhysRegClass(Reg);
+ };
+
+ unsigned OpCode = Root.getOpcode();
+ bool Is64 = OpCode == X86::SHLD64rrCL || OpCode == X86::SHRD64rrCL ||
+ OpCode == X86::SHLD64mrCL || OpCode == X86::SHRD64mrCL;
+ bool IsShrd = OpCode == X86::SHRD32rrCL || OpCode == X86::SHRD64rrCL ||
+ OpCode == X86::SHRD32mrCL || OpCode == X86::SHRD64mrCL;
+ bool IsMem = OpCode == X86::SHLD32mrCL || OpCode == X86::SHLD64mrCL ||
+ OpCode == X86::SHRD32mrCL || OpCode == X86::SHRD64mrCL;
+
+ unsigned DirectClOpc, MemClOpc, ComplXOpc, MovzxOpc, NegOpc, OrOpc;
+ const TargetRegisterClass *RC;
+ if (Is64) {
+ DirectClOpc = IsShrd ? X86::SHR64rCL : X86::SHL64rCL;
+ MemClOpc = IsShrd ? X86::SHR64mCL : X86::SHL64mCL;
+ ComplXOpc = IsShrd ? X86::SHLX64rr : X86::SHRX64rr;
+ MovzxOpc = X86::MOVZX64rr8;
+ NegOpc = X86::NEG64r;
+ OrOpc = IsMem ? X86::OR64mr : X86::OR64rr;
+ RC = &X86::GR64RegClass;
+ } else {
+ DirectClOpc = IsShrd ? X86::SHR32rCL : X86::SHL32rCL;
+ MemClOpc = IsShrd ? X86::SHR32mCL : X86::SHL32mCL;
+ ComplXOpc = IsShrd ? X86::SHLX32rr : X86::SHRX32rr;
+ MovzxOpc = X86::MOVZX32rr8;
+ NegOpc = X86::NEG32r;
+ OrOpc = IsMem ? X86::OR32mr : X86::OR32rr;
+ RC = &X86::GR32RegClass;
+ }
+
+ // Copy physical CL into a virtual register to be negated.
+ Register CntVirt = RegInfo.createVirtualRegister(RC);
+ MachineInstr *Movzx =
+ BuildMI(*MF, MIMetadata(Root), TII.get(MovzxOpc), CntVirt)
+ .addReg(X86::CL);
+
+ Register CntNeg = RegInfo.createVirtualRegister(RC);
+ MachineInstr *Neg = BuildMI(*MF, MIMetadata(Root), TII.get(NegOpc), CntNeg)
+ .addReg(CntVirt, RegState::Kill);
+
+ if (!IsMem) {
+ // Register case
+ Register Src1 = Root.getOperand(1).getReg();
+ Register Src2 = Root.getOperand(2).getReg();
+ Register Dst = Root.getOperand(0).getReg();
+
+ Register Tmp1 = RegInfo.createVirtualRegister(RC);
+ MachineInstr *DirectShift =
+ BuildMI(*MF, MIMetadata(Root), TII.get(DirectClOpc), Tmp1)
+ .addReg(Src1, getKillRegState(Root.getOperand(1).isKill()));
+
+ Register Tmp2 = RegInfo.createVirtualRegister(RC);
+ MachineInstr *ComplShift =
+ BuildMI(*MF, MIMetadata(Root), TII.get(ComplXOpc), Tmp2)
+ .addReg(Src2, getKillRegState(Root.getOperand(2).isKill()))
+ .addReg(CntNeg, RegState::Kill);
+
+ MachineInstr *Or = BuildMI(*MF, MIMetadata(Root), TII.get(OrOpc), Dst)
+ .addReg(Tmp1, RegState::Kill)
+ .addReg(Tmp2, RegState::Kill);
+
+ InstrIdxForVirtReg.insert({CntVirt, 0});
+ InstrIdxForVirtReg.insert({CntNeg, 1});
+ InstrIdxForVirtReg.insert({Tmp1, 2});
+ InstrIdxForVirtReg.insert({Tmp2, 3});
+ InsInstrs.push_back(Movzx);
+ InsInstrs.push_back(Neg);
+ InsInstrs.push_back(DirectShift);
+ InsInstrs.push_back(ComplShift);
+ InsInstrs.push_back(Or);
+ } else {
+ // Memory case
+ const TargetRegisterClass *SrcRC = GetRC(Root.getOperand(5).getReg());
+
+ MachineInstr *MemShift = BuildMI(*MF, MIMetadata(Root), TII.get(MemClOpc))
+ .add(Root.getOperand(0 + X86::AddrBaseReg))
+ .add(Root.getOperand(0 + X86::AddrScaleAmt))
+ .add(Root.getOperand(0 + X86::AddrIndexReg))
+ .add(Root.getOperand(0 + X86::AddrDisp))
+ .add(Root.getOperand(0 + X86::AddrSegmentReg));
+
+ Register RegSrc = RegInfo.createVirtualRegister(SrcRC);
+ MachineInstr *ComplShift =
+ BuildMI(*MF, MIMetadata(Root), TII.get(ComplXOpc), RegSrc)
+ .addReg(Root.getOperand(5).getReg(),
+ getKillRegState(Root.getOperand(5).isKill()))
+ .addReg(CntNeg, RegState::Kill);
+
+ MachineInstr *Or = BuildMI(*MF, MIMetadata(Root), TII.get(OrOpc))
+ .add(Root.getOperand(0 + X86::AddrBaseReg))
+ .add(Root.getOperand(0 + X86::AddrScaleAmt))
+ .add(Root.getOperand(0 + X86::AddrIndexReg))
+ .add(Root.getOperand(0 + X86::AddrDisp))
+ .add(Root.getOperand(0 + X86::AddrSegmentReg))
+ .addReg(RegSrc, RegState::Kill);
+
+ InstrIdxForVirtReg.insert({CntVirt, 0});
+ InstrIdxForVirtReg.insert({CntNeg, 1});
+ InstrIdxForVirtReg.insert({RegSrc, 3});
+ InsInstrs.push_back(Movzx);
+ InsInstrs.push_back(Neg);
+ InsInstrs.push_back(MemShift);
+ InsInstrs.push_back(ComplShift);
+ InsInstrs.push_back(Or);
+ }
+ DelInstrs.push_back(&Root);
+}
+
static void
genAlternativeDpCodeSequence(MachineInstr &Root, const TargetInstrInfo &TII,
SmallVectorImpl<MachineInstr *> &InsInstrs,
@@ -11030,6 +11172,9 @@ void X86InstrInfo::genAlternativeCodeSequence(
return;
case X86MachineCombinerPattern::USHD_LEA:
genShdLeaSequence(Root, *this, InsInstrs, DelInstrs, InstrIdxForVirtReg);
+ return;
+ case X86MachineCombinerPattern::USHD_SHLX_OR:
+ genShdShlxOrSequence(Root, *this, InsInstrs, DelInstrs, InstrIdxForVirtReg);
}
}
diff --git a/llvm/lib/Target/X86/X86InstrInfo.h b/llvm/lib/Target/X86/X86InstrInfo.h
index 0286756603eaf..b67ff4e76ed70 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.h
+++ b/llvm/lib/Target/X86/X86InstrInfo.h
@@ -33,6 +33,7 @@ enum X86MachineCombinerPattern : unsigned {
// Unfold SHD
USHD_OR,
USHD_LEA,
+ USHD_SHLX_OR,
};
namespace X86 {
diff --git a/llvm/test/CodeGen/X86/bit-manip-i256.ll b/llvm/test/CodeGen/X86/bit-manip-i256.ll
index dba8d0d3dd07f..ce32fea04e6db 100644
--- a/llvm/test/CodeGen/X86/bit-manip-i256.ll
+++ b/llvm/test/CodeGen/X86/bit-manip-i256.ll
@@ -200,7 +200,9 @@ define i256 @bext_i256(i256 %a0, i256 %idx, i256 %len) nounwind {
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: pushq %r15
; AVX512VL-NEXT: pushq %r14
+; AVX512VL-NEXT: pushq %r12
; AVX512VL-NEXT: pushq %rbx
+; AVX512VL-NEXT: pushq %rax
; AVX512VL-NEXT: movq %rcx, %r10
; AVX512VL-NEXT: movq %rdi, %rax
; AVX512VL-NEXT: movq {{[0-9]+}}(%rsp), %rcx
@@ -212,14 +214,26 @@ define i256 @bext_i256(i256 %a0, i256 %idx, i256 %len) nounwind {
; AVX512VL-NEXT: shrb $3, %dil
; AVX512VL-NEXT: andb $24, %dil
; AVX512VL-NEXT: negb %dil
-; AVX512VL-NEXT: movsbq %dil, %rbx
-; AVX512VL-NEXT: movq -16(%rsp,%rbx), %r11
-; AVX512VL-NEXT: movq -8(%rsp,%rbx), %rdi
-; AVX512VL-NEXT: shldq %cl, %r11, %rdi
-; AVX512VL-NEXT: movq -32(%rsp,%rbx), %r15
-; AVX512VL-NEXT: movq -24(%rsp,%rbx), %r14
-; AVX512VL-NEXT: shldq %cl, %r14, %r11
-; AVX512VL-NEXT: shldq %cl, %r15, %r14
+; AVX512VL-NEXT: movsbq %dil, %r11
+; AVX512VL-NEXT: movq -16(%rsp,%r11), %rbx
+; AVX512VL-NEXT: movq -8(%rsp,%r11), %r14
+; AVX512VL-NEXT: movzbq %cl, %rdi
+; AVX512VL-NEXT: shlq %cl, %r14
+; AVX512VL-NEXT: negq %rdi
+; AVX512VL-NEXT: shrxq %rdi, %rbx, %rdi
+; AVX512VL-NEXT: orq %r14, %rdi
+; AVX512VL-NEXT: movq -32(%rsp,%r11), %r15
+; AVX512VL-NEXT: movq -24(%rsp,%r11), %r12
+; AVX512VL-NEXT: movzbq %cl, %r11
+; AVX512VL-NEXT: negq %r11
+; AVX512VL-NEXT: shlq %cl, %rbx
+; AVX512VL-NEXT: shrxq %r11, %r12, %r11
+; AVX512VL-NEXT: orq %rbx, %r11
+; AVX512VL-NEXT: movzbq %cl, %rbx
+; AVX512VL-NEXT: negq %rbx
+; AVX512VL-NEXT: shlq %cl, %r12
+; AVX512VL-NEXT: shrxq %rbx, %r15, %r14
+; AVX512VL-NEXT: orq %r12, %r14
; AVX512VL-NEXT: shlxq %rcx, %r15, %rbx
; AVX512VL-NEXT: addq $-1, %rbx
; AVX512VL-NEXT: adcq $-1, %r14
@@ -232,26 +246,40 @@ define i256 @bext_i256(i256 %a0, i256 %idx, i256 %len) nounwind {
; AVX512VL-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movl %r9d, %ecx
; AVX512VL-NEXT: shrb $6, %cl
-; AVX512VL-NEXT: movzbl %cl, %edx
-; AVX512VL-NEXT: movq -112(%rsp,%rdx,8), %rsi
-; AVX512VL-NEXT: movq -128(%rsp,%rdx,8), %r8
-; AVX512VL-NEXT: movq -120(%rsp,%rdx,8), %r10
-; AVX512VL-NEXT: movq %r10, %r15
+; AVX512VL-NEXT: movzbl %cl, %r10d
+; AVX512VL-NEXT: movq -112(%rsp,%r10,8), %r15
+; AVX512VL-NEXT: movq -128(%rsp,%r10,8), %rsi
+; AVX512VL-NEXT: movq -120(%rsp,%r10,8), %r8
; AVX512VL-NEXT: movl %r9d, %ecx
-; AVX512VL-NEXT: shrdq %cl, %rsi, %r15
-; AVX512VL-NEXT: andq %r14, %r15
-; AVX512VL-NEXT: movq -104(%rsp,%rdx,8), %rdx
-; AVX512VL-NEXT: shrdq %cl, %rdx, %rsi
-; AVX512VL-NEXT: andq %r11, %rsi
-; AVX512VL-NEXT: shrdq %cl, %r10, %r8
-; AVX512VL-NEXT: andq %rbx, %r8
-; AVX512VL-NEXT: shrxq %r9, %rdx, %rcx
-; AVX512VL-NEXT: andq %rdi, %rcx
-; AVX512VL-NEXT: movq %r8, (%rax)
-; AVX512VL-NEXT: movq %r15, 8(%rax)
-; AVX512VL-NEXT: movq %rsi, 16(%rax)
-; AVX512VL-NEXT: movq %rcx, 24(%rax)
+; AVX512VL-NEXT: movzbq %cl, %rdx
+; AVX512VL-NEXT: negq %rdx
+; AVX512VL-NEXT: movq %r8, %r12
+; AVX512VL-NEXT: shrq %cl, %r12
+; AVX512VL-NEXT: shlxq %rdx, %r15, %rdx
+; AVX512VL-NEXT: orq %r12, %rdx
+; AVX512VL-NEXT: andq %r14, %rdx
+; AVX512VL-NEXT: movq -104(%rsp,%r10,8), %r10
+; AVX512VL-NEXT: movzbq %cl, %r14
+; AVX512VL-NEXT: shrq %cl, %r15
+; AVX512VL-NEXT: negq %r14
+; AVX512VL-NEXT: shlxq %r14, %r10, %r14
+; AVX512VL-NEXT: orq %r15, %r14
+; AVX512VL-NEXT: andq %r11, %r14
+; AVX512VL-NEXT: movzbq %cl, %r11
+; AVX512VL-NEXT: negq %r11
+; AVX512VL-NEXT: shrq %cl, %rsi
+; AVX512VL-NEXT: shlxq %r11, %r8, %rcx
+; AVX512VL-NEXT: orq %rsi, %rcx
+; AVX512VL-NEXT: andq %rbx, %rcx
+; AVX512VL-NEXT: shrxq %r9, %r10, %rsi
+; AVX512VL-NEXT: andq %rdi, %rsi
+; AVX512VL-NEXT: movq %rcx, (%rax)
+; AVX512VL-NEXT: movq %rdx, 8(%rax)
+; AVX512VL-NEXT: movq %r14, 16(%rax)
+; AVX512VL-NEXT: movq %rsi, 24(%rax)
+; AVX512VL-NEXT: addq $8, %rsp
; AVX512VL-NEXT: popq %rbx
+; AVX512VL-NEXT: popq %r12
; AVX512VL-NEXT: popq %r14
; AVX512VL-NEXT: popq %r15
; AVX512VL-NEXT: vzeroupper
@@ -261,7 +289,9 @@ define i256 @bext_i256(i256 %a0, i256 %idx, i256 %len) nounwind {
; AVX512VBMI: # %bb.0:
; AVX512VBMI-NEXT: pushq %r15
; AVX512VBMI-NEXT: pushq %r14
+; AVX512VBMI-NEXT: pushq %r12
; AVX512VBMI-NEXT: pushq %rbx
+; AVX512VBMI-NEXT: pushq %rax
; AVX512VBMI-NEXT: movq %rcx, %r10
; AVX512VBMI-NEXT: movq %rdi, %rax
; AVX512VBMI-NEXT: movq {{[0-9]+}}(%rsp), %rcx
@@ -273,14 +303,26 @@ define i256 @bext_i256(i256 %a0, i256 %idx, i256 %len) nounwind {
; AVX512VBMI-NEXT: shrb $3, %dil
; AVX512VBMI-NEXT: andb $24, %dil
; AVX512VBMI-NEXT: negb %dil
-; AVX512VBMI-NEXT: movsbq %dil, %rbx
-; AVX512VBMI-NEXT: movq -16(%rsp,%rbx), %r11
-; AVX512VBMI-NEXT: movq -8(%rsp,%rbx), %rdi
-; AVX512VBMI-NEXT: shldq %cl, %r11, %rdi
-; AVX512VBMI-NEXT: movq -32(%rsp,%rbx), %r15
-; AVX512VBMI-NEXT: movq -24(%rsp,%rbx), %r14
-; AVX512VBMI-NEXT: shldq %cl, %r14, %r11
-; AVX512VBMI-NEXT: shldq %cl, %r15, %r14
+; AVX512VBMI-NEXT: movsbq %dil, %r11
+; AVX512VBMI-NEXT: movq -16(%rsp,%r11), %rbx
+; AVX512VBMI-NEXT: movq -8(%rsp,%r11), %r14
+; AVX512VBMI-NEXT: movzbq %cl, %rdi
+; AVX512VBMI-NEXT: shlq %cl, %r14
+; AVX512VBMI-NEXT: negq %rdi
+; AVX512VBMI-NEXT: shrxq %rdi, %rbx, %rdi
+; AVX512VBMI-NEXT: orq %r14, %rdi
+; AVX512VBMI-NEXT: movq -32(%rsp,%r11), %r15
+; AVX512VBMI-NEXT: movq -24(%rsp,%r11), %r12
+; AVX512VBMI-NEXT: movzbq %cl, %r11
+; AVX512VBMI-NEXT: negq %r11
+; AVX512VBMI-NEXT: shlq %cl, %rbx
+; AVX512VBMI-NEXT: shrxq %r11, %r12, %r11
+; AVX512VBMI-NEXT: orq %rbx, %r11
+; AVX512VBMI-NEXT: movzbq %cl, %rbx
+; AVX512VBMI-NEXT: negq %rbx
+; AVX512VBMI-NEXT: shlq %cl, %r12
+; AVX512VBMI-NEXT: shrxq %rbx, %r15, %r14
+; AVX512VBMI-NEXT: orq %r12, %r14
; AVX512VBMI-NEXT: shlxq %rcx, %r15, %rbx
; AVX512VBMI-NEXT: addq $-1, %rbx
; AVX512VBMI-NEXT: adcq $-1, %r14
@@ -293,26 +335,40 @@ define i256 @bext_i256(i256 %a0, i256 %idx, i256 %len) nounwind {
; AVX512VBMI-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movl %r9d, %ecx
; AVX512VBMI-NEXT: shrb $6, %cl
-; AVX512VBMI-NEXT: movzbl %cl, %edx
-; AVX512VBMI-NEXT: movq -112(%rsp,%rdx,8), %rsi
-; AVX512VBMI-NEXT: movq -128(%rsp,%rdx,8), %r8
-; AVX512VBMI-NEXT: movq -120(%rsp,%rdx,8), %r10
-; AVX512VBMI-NEXT: movq %r10, %r15
+; AVX512VBMI-NEXT: movzbl %cl, %r10d
+; AVX512VBMI-NEXT: movq -112(%rsp,%r10,8), %r15
+; AVX512VBMI-NEXT: movq -128(%rsp,%r10,8), %rsi
+; AVX512VBMI-NEXT: movq -120(%rsp,%r10,8), %r8
; AVX512VBMI-NEXT: movl %r9d, %ecx
-; AVX512VBMI-NEXT: shrdq %cl, %rsi, %r15
-; AVX512VBMI-NEXT: andq %r14, %r15
-; AVX512VBMI-NEXT: movq -104(%rsp,%rdx,8), %rdx
-; AVX512VBMI-NEXT: shrdq %cl, %rdx, %rsi
-; AVX512VBMI-NEXT: andq %r11, %rsi
-; AVX512VBMI-NEXT: shrdq %cl, %r10, %r8
-; AVX512VBMI-NEXT: andq %rbx, %r8
-; AVX512VBMI-NEXT: shrxq %r9, %rdx, %rcx
-; AVX512VBMI-NEXT: andq %rdi, %rcx
-; AVX512VBMI-NEXT: movq %r8, (%rax)
-; AVX512VBMI-NEXT: movq %r15, 8(%rax)
-; AVX512VBMI-NEXT: movq %rsi, 16(%rax)
-; AVX512VBMI-NEXT: movq %rcx, 24(%rax)
+; AVX512VBMI-NEXT: movzbq %cl, %rdx
+; AVX512VBMI-NEXT: negq %rdx
+; AVX512VBMI-NEXT: movq %r8, %r12
+; AVX512VBMI-NEXT: shrq %cl, %r12
+; AVX512VBMI-NEXT: shlxq %rdx, %r15, %rdx
+; AVX512VBMI-NEXT: orq %r12, %rdx
+; AVX512VBMI-NEXT: andq %r14, %rdx
+; AVX512VBMI-NEXT: movq -104(%rsp,%r10,8), %r10
+; AVX512VBMI-NEXT: movzbq %cl, %r14
+; AVX512VBMI-NEXT: shrq %cl, %r15
+; AVX512VBMI-NEXT: negq %r14
+; AVX512VBMI-NEXT: shlxq %r14, %r10, %r14
+; AVX512VBMI-NEXT: orq %r15, %r14
+; AVX512VBMI-NEXT: andq %r11, %r14
+; AVX512VBMI-NEXT: movzbq %cl, %r11
+; AVX512VBMI-NEXT: negq %r11
+; AVX512VBMI-NEXT: shrq %cl, %rsi
+; AVX512VBMI-NEXT: shlxq %r11, %r8, %rcx
+; AVX512VBMI-NEXT: orq %rsi, %rcx
+; AVX512VBMI-NEXT: andq %rbx, %rcx
+; AVX512VBMI-NEXT: shrxq %r9, %r10, %rsi
+; AVX512VBMI-NEXT: andq %rdi, %rsi
+; AVX512VBMI-NEXT: movq %rcx, (%rax)
+; AVX512VBMI-NEXT: movq %rdx, 8(%rax)
+; AVX512VBMI-NEXT: movq %r14, 16(%rax)
+; AVX512VBMI-NEXT: movq %rsi, 24(%rax)
+; AVX512VBMI-NEXT: addq $8, %rsp
; AVX512VBMI-NEXT: popq %rbx
+; AVX512VBMI-NEXT: popq %r12
; AVX512VBMI-NEXT: popq %r14
; AVX512VBMI-NEXT: popq %r15
; AVX512VBMI-NEXT: vzeroupper
@@ -817,7 +873,9 @@ define i256 @bext_i256_load(i256 %a0, i256 %idx, i256 %len) nounwind {
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: pushq %r15
; AVX512VL-NEXT: pushq %r14
+; AVX512VL-NEXT: pushq %r12
; AVX512VL-NEXT: pushq %rbx
+; AVX512VL-NEXT: pushq %rax
; AVX512VL-NEXT: movq %rcx, %r10
; AVX512VL-NEXT: movq %rdi, %rax
; AVX512VL-NEXT: movq {{[0-9]+}}(%rsp), %rcx
@@ -829,14 +887,26 @@ define i256 @bext_i256_load(i256 %a0, i256 %idx, i256 %len) nounwind {
; AVX512VL-NEXT: shrb $3, %dil
; AVX512VL-NEXT: andb $24, %dil
; AVX512VL-NEXT: negb %dil
-; AVX512VL-NEXT: movsbq %dil, %rbx
-; AVX512VL-NEXT: movq -16(%rsp,%rbx), %r11
-; AVX512VL-NEXT: movq -8(%rsp,%rbx), %rdi
-; AVX512VL-NEXT: shldq %cl, %r11, %rdi
-; AVX512VL-NEXT: movq -32(%rsp,%rbx), %r15
-; AVX512VL-NEXT: movq -24(%rsp,%rbx), %r14
-; AVX512VL-NEXT: shldq %cl, %r14, %r11
-; AVX512VL-NEXT: shldq %cl, %r15, %r14
+; AVX512VL-NEXT: movsbq %dil, %r11
+; AVX512VL-NEXT: movq -16(%rsp,%r11), %rbx
+; AVX512VL-NEXT: movq -8(%rsp,%r11), %r14
+; AVX512VL-NEXT: movzbq %cl, %rdi
+; AVX512VL-NEXT: shlq %cl, %r14
+; AVX512VL-NEXT: negq %rdi
+; AVX512VL-NEXT: shrxq %rdi, %rbx, %rdi
+; AVX512VL-NEXT: orq %r14, %rdi
+; AVX512VL-NEXT: movq -32(%rsp,%r11), %r15
+; AVX512VL-NEXT: movq -24(%rsp,%r11), %r12
+; AVX512VL-NEXT: movzbq %cl, %r11
+; AVX512VL-NEXT: negq %r11
+; AVX512VL-NEXT: shlq %cl, %rbx
+; AVX512VL-NEXT: shrxq %r11, %r12, %r11
+; AVX512VL-NEXT: orq %rbx, %r11
+; AVX512VL-NEXT: movzbq %cl, %rbx
+; AVX512VL-NEXT: negq %rbx
+; AVX512VL-NEXT: shlq %cl, %r12
+; AVX512VL-NEXT: shrxq %rbx, %r15, %r14
+; AVX512VL-NEXT: orq %r12, %r14
; AVX512VL-NEXT: shlxq %rcx, %r15, %rbx
; AVX512VL-NEXT: addq $-1, %rbx
; AVX512VL-NEXT: adcq $-1, %r14
@@ -849,26 +919,40 @@ define i256 @bext_i256_load(i256 %a0, i256 %idx, i256 %len) nounwind {
; AVX512VL-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movl %r9d, %ecx
; AVX512VL-NEXT: shrb $6, %cl
-; AVX512VL-NEXT: movzbl %cl, %edx
-; AVX512VL-NEXT: movq -112(%rsp,%rdx,8), %rsi
-; AVX512VL-NEXT: movq -128(%rsp,%rdx,8), %r8
-; AVX512VL-NEXT: movq -120(%rsp,%rdx,8), %r10
-; AVX512VL-NEXT: movq %r10, %r15
+; AVX512VL-NEXT: movzbl %cl, %r10d
+; AVX512VL-NEXT: movq -112(%rsp,%r10,8), %r15
+; AVX512VL-NEXT: movq -128(%rsp,%r10,8), %rsi
+; AVX512VL-NEXT: movq -120(%rsp,%r10,8), %r8
; AVX512VL-NEXT: movl %r9d, %ecx
-; AVX512VL-NEXT: shrdq %cl, %rsi, %r15
-; AVX512VL-NEXT: andq %r14, %r15
-; AVX512VL-NEXT: movq -104(%rsp,%rdx,8), %rdx
-; AVX512VL-NEXT: shrdq %cl, %rdx, %rsi
-; AVX512VL-NEXT: andq %r11, %rsi
-; AVX512VL-NEXT: shrdq %cl, %r10, %r8
-; AVX512VL-NEXT: andq %rbx, %r8
-; AVX512VL-NEXT: shrxq %r9, %rdx, %rcx
-; AVX512VL-NEXT: andq %rdi, %rcx
-; AVX512VL-NEXT: movq %r8, (%rax)
-; AVX512VL-NEXT: movq %r15, 8(%rax)
-; AVX512VL-NEXT: movq %rsi, 16(%rax)
-; AVX512VL-NEXT: movq %rcx, 24(%rax)
+; AVX512VL-NEXT: movzbq %cl, %rdx
+; AVX512VL-NEXT: negq %rdx
+; AVX512VL-NEXT: movq %r8, %r12
+; AVX512VL-NEXT: shrq %cl, %r12
+; AVX512VL-NEXT: shlxq %rdx, %r15, %rdx
+; AVX512VL-NEXT: orq %r12, %rdx
+; AVX512VL-NEXT: andq %r14, %rdx
+; AVX512VL-NEXT: movq -104(%rsp,%r10,8), %r10
+; AVX512VL-NEXT: movzbq %cl, %r14
+; AVX512VL-NEXT: shrq %cl, %r15
+; AVX512VL-NEXT: negq %r14
+; AVX512VL-NEXT: shlxq %r14, %r10, %r14
+; AVX512VL-NEXT: orq %r15, %r14
+; AVX512VL-NEXT: andq %r11, %r14
+; AVX512VL-NEXT: movzbq %cl, %r11
+; AVX512VL-NEXT: negq %r11
+; AVX512VL-NEXT: shrq %cl, %rsi
+; AVX512VL-NEXT: shlxq %r11, %r8, %rcx
+; AVX512VL-NEXT: orq %rsi, %rcx
+; AVX512VL-NEXT: andq %rbx, %rcx
+; AVX512VL-NEXT: shrxq %r9, %r10, %rsi
+; AVX512VL-NEXT: andq %rdi, %rsi
+; AVX512VL-NEXT: movq %rcx, (%rax)
+; AVX512VL-NEXT: movq %rdx, 8(%rax)
+; AVX512VL-NEXT: movq %r14, 16(%rax)
+; AVX512VL-NEXT: movq %rsi, 24(%rax)
+; AVX512VL-NEXT: addq $8, %rsp
; AVX512VL-NEXT: popq %rbx
+; AVX512VL-NEXT: popq %r12
; AVX512VL-NEXT: popq %r14
; AVX512VL-NEXT: popq %r15
; AVX512VL-NEXT: vzeroupper
@@ -878,7 +962,9 @@ define i256 @bext_i256_load(i256 %a0, i256 %idx, i256 %len) nounwind {
; AVX512VBMI: # %bb.0:
; AVX512VBMI-NEXT: pushq %r15
; AVX512VBMI-NEXT: pushq %r14
+; AVX512VBMI-NEXT: pushq %r12
; AVX512VBMI-NEXT: pushq %rbx
+; AVX512VBMI-NEXT: pushq %rax
; AVX512VBMI-NEXT: movq %rcx, %r10
; AVX512VBMI-NEXT: movq %rdi, %rax
; AVX512VBMI-NEXT: movq {{[0-9]+}}(%rsp), %rcx
@@ -890,14 +976,26 @@ define i256 @bext_i256_load(i256 %a0, i256 %idx, i256 %len) nounwind {
; AVX512VBMI-NEXT: shrb $3, %dil
; AVX512VBMI-NEXT: andb $24, %dil
; AVX512VBMI-NEXT: negb %dil
-; AVX512VBMI-NEXT: movsbq %dil, %rbx
-; AVX512VBMI-NEXT: movq -16(%rsp,%rbx), %r11
-; AVX512VBMI-NEXT: movq -8(%rsp,%rbx), %rdi
-; AVX512VBMI-NEXT: shldq %cl, %r11, %rdi
-; AVX512VBMI-NEXT: movq -32(%rsp,%rbx), %r15
-; AVX512VBMI-NEXT: movq -24(%rsp,%rbx), %r14
-; AVX512VBMI-NEXT: shldq %cl, %r14, %r11
-; AVX512VBMI-NEXT: shldq %cl, %r15, %r14
+; AVX512VBMI-NEXT: movsbq %dil, %r11
+; AVX512VBMI-NEXT: movq -16(%rsp,%r11), %rbx
+; AVX512VBMI-NEXT: movq -8(%rsp,%r11), %r14
+; AVX512VBMI-NEXT: movzbq %cl, %rdi
+; AVX512VBMI-NEXT: shlq %cl, %r14
+; AVX512VBMI-NEXT: negq %rdi
+; AVX512VBMI-NEXT: shrxq %rdi, %rbx, %rdi
+; AVX512VBMI-NEXT: orq %r14, %rdi
+; AVX512VBMI-NEXT: movq -32(%rsp,%r11), %r15
+; AVX512VBMI-NEXT: movq -24(%rsp,%r11), %r12
+; AVX512VBMI-NEXT: movzbq %cl, %r11
+; AVX512VBMI-NEXT: negq %r11
+; AVX512VBMI-NEXT: shlq %cl, %rbx
+; AVX512VBMI-NEXT: shrxq %r11, %r12, %r11
+; AVX512VBMI-NEXT: orq %rbx, %r11
+; AVX512VBMI-NEXT: movzbq %cl, %rbx
+; AVX512VBMI-NEXT: negq %rbx
+; AVX512VBMI-NEXT: shlq %cl, %r12
+; AVX512VBMI-NEXT: shrxq %rbx, %r15, %r14
+; AVX512VBMI-NEXT: orq %r12, %r14
; AVX512VBMI-NEXT: shlxq %rcx, %r15, %rbx
; AVX512VBMI-NEXT: addq $-1, %rbx
; AVX512VBMI-NEXT: adcq $-1, %r14
@@ -910,26 +1008,40 @@ define i256 @bext_i256_load(i256 %a0, i256 %idx, i256 %len) nounwind {
; AVX512VBMI-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movl %r9d, %ecx
; AVX512VBMI-NEXT: shrb $6, %cl
-; AVX512VBMI-NEXT: movzbl %cl, %edx
-; AVX512VBMI-NEXT: movq -112(%rsp,%rdx,8), %rsi
-; AVX512VBMI-NEXT: movq -128(%rsp,%rdx,8), %r8
-; AVX512VBMI-NEXT: movq -120(%rsp,%rdx,8), %r10
-; AVX512VBMI-NEXT: movq %r10, %r15
+; AVX512VBMI-NEXT: movzbl %cl, %r10d
+; AVX512VBMI-NEXT: movq -112(%rsp,%r10,8), %r15
+; AVX512VBMI-NEXT: movq -128(%rsp,%r10,8), %rsi
+; AVX512VBMI-NEXT: movq -120(%rsp,%r10,8), %r8
; AVX512VBMI-NEXT: movl %r9d, %ecx
-; AVX512VBMI-NEXT: shrdq %cl, %rsi, %r15
-; AVX512VBMI-NEXT: andq %r14, %r15
-; AVX512VBMI-NEXT: movq -104(%rsp,%rdx,8), %rdx
-; AVX512VBMI-NEXT: shrdq %cl, %rdx, %rsi
-; AVX512VBMI-NEXT: andq %r11, %rsi
-; AVX512VBMI-NEXT: shrdq %cl, %r10, %r8
-; AVX512VBMI-NEXT: andq %rbx, %r8
-; AVX512VBMI-NEXT: shrxq %r9, %rdx, %rcx
-; AVX512VBMI-NEXT: andq %rdi, %rcx
-; AVX512VBMI-NEXT: movq %r8, (%rax)
-; AVX512VBMI-NEXT: movq %r15, 8(%rax)
-; AVX512VBMI-NEXT: movq %rsi, 16(%rax)
-; AVX512VBMI-NEXT: movq %rcx, 24(%rax)
+; AVX512VBMI-NEXT: movzbq %cl, %rdx
+; AVX512VBMI-NEXT: negq %rdx
+; AVX512VBMI-NEXT: movq %r8, %r12
+; AVX512VBMI-NEXT: shrq %cl, %r12
+; AVX512VBMI-NEXT: shlxq %rdx, %r15, %rdx
+; AVX512VBMI-NEXT: orq %r12, %rdx
+; AVX512VBMI-NEXT: andq %r14, %rdx
+; AVX512VBMI-NEXT: movq -104(%rsp,%r10,8), %r10
+; AVX512VBMI-NEXT: movzbq %cl, %r14
+; AVX512VBMI-NEXT: shrq %cl, %r15
+; AVX512VBMI-NEXT: negq %r14
+; AVX512VBMI-NEXT: shlxq %r14, %r10, %r14
+; AVX512VBMI-NEXT: orq %r15, %r14
+; AVX512VBMI-NEXT: andq %r11, %r14
+; AVX512VBMI-NEXT: movzbq %cl, %r11
+; AVX512VBMI-NEXT: negq %r11
+; AVX512VBMI-NEXT: shrq %cl, %rsi
+; AVX512VBMI-NEXT: shlxq %r11, %r8, %rcx
+; AVX512VBMI-NEXT: orq %rsi, %rcx
+; AVX512VBMI-NEXT: andq %rbx, %rcx
+; AVX512VBMI-NEXT: shrxq %r9, %r10, %rsi
+; AVX512VBMI-NEXT: andq %rdi, %rsi
+; AVX512VBMI-NEXT: movq %rcx, (%rax)
+; AVX512VBMI-NEXT: movq %rdx, 8(%rax)
+; AVX512VBMI-NEXT: movq %r14, 16(%rax)
+; AVX512VBMI-NEXT: movq %rsi, 24(%rax)
+; AVX512VBMI-NEXT: addq $8, %rsp
; AVX512VBMI-NEXT: popq %rbx
+; AVX512VBMI-NEXT: popq %r12
; AVX512VBMI-NEXT: popq %r14
; AVX512VBMI-NEXT: popq %r15
; AVX512VBMI-NEXT: vzeroupper
diff --git a/llvm/test/CodeGen/X86/extract-bits.ll b/llvm/test/CodeGen/X86/extract-bits.ll
index 90e075bfabf0a..5bc6eec4e5631 100644
--- a/llvm/test/CodeGen/X86/extract-bits.ll
+++ b/llvm/test/CodeGen/X86/extract-bits.ll
@@ -614,6 +614,7 @@ define i64 @bextr64_a0(i64 %val, i64 %numskipbits, i64 %numlowbits) nounwind {
;
; X86-BMI2-LABEL: bextr64_a0:
; X86-BMI2: # %bb.0:
+; X86-BMI2-NEXT: pushl %ebp
; X86-BMI2-NEXT: pushl %ebx
; X86-BMI2-NEXT: pushl %edi
; X86-BMI2-NEXT: pushl %esi
@@ -623,17 +624,21 @@ define i64 @bextr64_a0(i64 %val, i64 %numskipbits, i64 %numlowbits) nounwind {
; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-BMI2-NEXT: shrdl %cl, %eax, %esi
; X86-BMI2-NEXT: shrxl %ecx, %eax, %edi
+; X86-BMI2-NEXT: xorl %eax, %eax
; X86-BMI2-NEXT: testb $32, %cl
; X86-BMI2-NEXT: je .LBB7_2
; X86-BMI2-NEXT: # %bb.1:
; X86-BMI2-NEXT: movl %edi, %esi
; X86-BMI2-NEXT: xorl %edi, %edi
; X86-BMI2-NEXT: .LBB7_2:
-; X86-BMI2-NEXT: movl $1, %eax
-; X86-BMI2-NEXT: xorl %edx, %edx
; X86-BMI2-NEXT: movl %ebx, %ecx
-; X86-BMI2-NEXT: shldl %cl, %eax, %edx
-; X86-BMI2-NEXT: shlxl %ebx, %eax, %eax
+; X86-BMI2-NEXT: movzbl %cl, %edx
+; X86-BMI2-NEXT: movl $1, %ebp
+; X86-BMI2-NEXT: negl %edx
+; X86-BMI2-NEXT: shll %cl, %eax
+; X86-BMI2-NEXT: shrxl %edx, %ebp, %edx
+; X86-BMI2-NEXT: orl %eax, %edx
+; X86-BMI2-NEXT: shlxl %ebx, %ebp, %eax
; X86-BMI2-NEXT: testb $32, %bl
; X86-BMI2-NEXT: je .LBB7_4
; X86-BMI2-NEXT: # %bb.3:
@@ -647,6 +652,7 @@ define i64 @bextr64_a0(i64 %val, i64 %numskipbits, i64 %numlowbits) nounwind {
; X86-BMI2-NEXT: popl %esi
; X86-BMI2-NEXT: popl %edi
; X86-BMI2-NEXT: popl %ebx
+; X86-BMI2-NEXT: popl %ebp
; X86-BMI2-NEXT: retl
;
; X64-NOBMI-LABEL: bextr64_a0:
@@ -758,10 +764,11 @@ define i64 @bextr64_a0_arithmetic(i64 %val, i64 %numskipbits, i64 %numlowbits) n
;
; X86-BMI2-LABEL: bextr64_a0_arithmetic:
; X86-BMI2: # %bb.0:
+; X86-BMI2-NEXT: pushl %ebp
; X86-BMI2-NEXT: pushl %ebx
; X86-BMI2-NEXT: pushl %edi
; X86-BMI2-NEXT: pushl %esi
-; X86-BMI2-NEXT: movzbl {{[0-9]+}}(%esp), %ebx
+; X86-BMI2-NEXT: movzbl {{[0-9]+}}(%esp), %edx
; X86-BMI2-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %esi
; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
@@ -774,12 +781,17 @@ define i64 @bextr64_a0_arithmetic(i64 %val, i64 %numskipbits, i64 %numlowbits) n
; X86-BMI2-NEXT: movl %edi, %esi
; X86-BMI2-NEXT: movl %eax, %edi
; X86-BMI2-NEXT: .LBB8_2:
-; X86-BMI2-NEXT: movl $1, %eax
-; X86-BMI2-NEXT: xorl %edx, %edx
-; X86-BMI2-NEXT: movl %ebx, %ecx
-; X86-BMI2-NEXT: shldl %cl, %eax, %edx
-; X86-BMI2-NEXT: shlxl %ebx, %eax, %eax
-; X86-BMI2-NEXT: testb $32, %bl
+; X86-BMI2-NEXT: movl %edx, %ecx
+; X86-BMI2-NEXT: movzbl %cl, %eax
+; X86-BMI2-NEXT: movl $1, %ebp
+; X86-BMI2-NEXT: xorl %ebx, %ebx
+; X86-BMI2-NEXT: negl %eax
+; X86-BMI2-NEXT: shll %cl, %ebx
+; X86-BMI2-NEXT: movl %edx, %ecx
+; X86-BMI2-NEXT: shrxl %eax, %ebp, %edx
+; X86-BMI2-NEXT: orl %ebx, %edx
+; X86-BMI2-NEXT: shlxl %ecx, %ebp, %eax
+; X86-BMI2-NEXT: testb $32, %cl
; X86-BMI2-NEXT: je .LBB8_4
; X86-BMI2-NEXT: # %bb.3:
; X86-BMI2-NEXT: movl %eax, %edx
@@ -792,6 +804,7 @@ define i64 @bextr64_a0_arithmetic(i64 %val, i64 %numskipbits, i64 %numlowbits) n
; X86-BMI2-NEXT: popl %esi
; X86-BMI2-NEXT: popl %edi
; X86-BMI2-NEXT: popl %ebx
+; X86-BMI2-NEXT: popl %ebp
; X86-BMI2-NEXT: retl
;
; X64-NOBMI-LABEL: bextr64_a0_arithmetic:
@@ -902,6 +915,7 @@ define i64 @bextr64_a1_indexzext(i64 %val, i8 zeroext %numskipbits, i8 zeroext %
;
; X86-BMI2-LABEL: bextr64_a1_indexzext:
; X86-BMI2: # %bb.0:
+; X86-BMI2-NEXT: pushl %ebp
; X86-BMI2-NEXT: pushl %ebx
; X86-BMI2-NEXT: pushl %edi
; X86-BMI2-NEXT: pushl %esi
@@ -911,17 +925,21 @@ define i64 @bextr64_a1_indexzext(i64 %val, i8 zeroext %numskipbits, i8 zeroext %
; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-BMI2-NEXT: shrdl %cl, %eax, %esi
; X86-BMI2-NEXT: shrxl %ecx, %eax, %edi
+; X86-BMI2-NEXT: xorl %eax, %eax
; X86-BMI2-NEXT: testb $32, %cl
; X86-BMI2-NEXT: je .LBB9_2
; X86-BMI2-NEXT: # %bb.1:
; X86-BMI2-NEXT: movl %edi, %esi
; X86-BMI2-NEXT: xorl %edi, %edi
; X86-BMI2-NEXT: .LBB9_2:
-; X86-BMI2-NEXT: movl $1, %eax
-; X86-BMI2-NEXT: xorl %edx, %edx
; X86-BMI2-NEXT: movl %ebx, %ecx
-; X86-BMI2-NEXT: shldl %cl, %eax, %edx
-; X86-BMI2-NEXT: shlxl %ebx, %eax, %eax
+; X86-BMI2-NEXT: movzbl %cl, %edx
+; X86-BMI2-NEXT: movl $1, %ebp
+; X86-BMI2-NEXT: negl %edx
+; X86-BMI2-NEXT: shll %cl, %eax
+; X86-BMI2-NEXT: shrxl %edx, %ebp, %edx
+; X86-BMI2-NEXT: orl %eax, %edx
+; X86-BMI2-NEXT: shlxl %ebx, %ebp, %eax
; X86-BMI2-NEXT: testb $32, %bl
; X86-BMI2-NEXT: je .LBB9_4
; X86-BMI2-NEXT: # %bb.3:
@@ -935,6 +953,7 @@ define i64 @bextr64_a1_indexzext(i64 %val, i8 zeroext %numskipbits, i8 zeroext %
; X86-BMI2-NEXT: popl %esi
; X86-BMI2-NEXT: popl %edi
; X86-BMI2-NEXT: popl %ebx
+; X86-BMI2-NEXT: popl %ebp
; X86-BMI2-NEXT: retl
;
; X64-NOBMI-LABEL: bextr64_a1_indexzext:
@@ -1050,6 +1069,7 @@ define i64 @bextr64_a2_load(ptr %w, i64 %numskipbits, i64 %numlowbits) nounwind
;
; X86-BMI2-LABEL: bextr64_a2_load:
; X86-BMI2: # %bb.0:
+; X86-BMI2-NEXT: pushl %ebp
; X86-BMI2-NEXT: pushl %ebx
; X86-BMI2-NEXT: pushl %edi
; X86-BMI2-NEXT: pushl %esi
@@ -1060,17 +1080,21 @@ define i64 @bextr64_a2_load(ptr %w, i64 %numskipbits, i64 %numlowbits) nounwind
; X86-BMI2-NEXT: movl 4(%eax), %eax
; X86-BMI2-NEXT: shrxl %ecx, %eax, %edi
; X86-BMI2-NEXT: shrdl %cl, %eax, %esi
+; X86-BMI2-NEXT: xorl %eax, %eax
; X86-BMI2-NEXT: testb $32, %cl
; X86-BMI2-NEXT: je .LBB10_2
; X86-BMI2-NEXT: # %bb.1:
; X86-BMI2-NEXT: movl %edi, %esi
; X86-BMI2-NEXT: xorl %edi, %edi
; X86-BMI2-NEXT: .LBB10_2:
-; X86-BMI2-NEXT: movl $1, %eax
-; X86-BMI2-NEXT: xorl %edx, %edx
; X86-BMI2-NEXT: movl %ebx, %ecx
-; X86-BMI2-NEXT: shldl %cl, %eax, %edx
-; X86-BMI2-NEXT: shlxl %ebx, %eax, %eax
+; X86-BMI2-NEXT: movzbl %cl, %edx
+; X86-BMI2-NEXT: movl $1, %ebp
+; X86-BMI2-NEXT: negl %edx
+; X86-BMI2-NEXT: shll %cl, %eax
+; X86-BMI2-NEXT: shrxl %edx, %ebp, %edx
+; X86-BMI2-NEXT: orl %eax, %edx
+; X86-BMI2-NEXT: shlxl %ebx, %ebp, %eax
; X86-BMI2-NEXT: testb $32, %bl
; X86-BMI2-NEXT: je .LBB10_4
; X86-BMI2-NEXT: # %bb.3:
@@ -1084,6 +1108,7 @@ define i64 @bextr64_a2_load(ptr %w, i64 %numskipbits, i64 %numlowbits) nounwind
; X86-BMI2-NEXT: popl %esi
; X86-BMI2-NEXT: popl %edi
; X86-BMI2-NEXT: popl %ebx
+; X86-BMI2-NEXT: popl %ebp
; X86-BMI2-NEXT: retl
;
; X64-NOBMI-LABEL: bextr64_a2_load:
@@ -1197,6 +1222,7 @@ define i64 @bextr64_a3_load_indexzext(ptr %w, i8 zeroext %numskipbits, i8 zeroex
;
; X86-BMI2-LABEL: bextr64_a3_load_indexzext:
; X86-BMI2: # %bb.0:
+; X86-BMI2-NEXT: pushl %ebp
; X86-BMI2-NEXT: pushl %ebx
; X86-BMI2-NEXT: pushl %edi
; X86-BMI2-NEXT: pushl %esi
@@ -1207,17 +1233,21 @@ define i64 @bextr64_a3_load_indexzext(ptr %w, i8 zeroext %numskipbits, i8 zeroex
; X86-BMI2-NEXT: movl 4(%eax), %eax
; X86-BMI2-NEXT: shrxl %ecx, %eax, %edi
; X86-BMI2-NEXT: shrdl %cl, %eax, %esi
+; X86-BMI2-NEXT: xorl %eax, %eax
; X86-BMI2-NEXT: testb $32, %cl
; X86-BMI2-NEXT: je .LBB11_2
; X86-BMI2-NEXT: # %bb.1:
; X86-BMI2-NEXT: movl %edi, %esi
; X86-BMI2-NEXT: xorl %edi, %edi
; X86-BMI2-NEXT: .LBB11_2:
-; X86-BMI2-NEXT: movl $1, %eax
-; X86-BMI2-NEXT: xorl %edx, %edx
; X86-BMI2-NEXT: movl %ebx, %ecx
-; X86-BMI2-NEXT: shldl %cl, %eax, %edx
-; X86-BMI2-NEXT: shlxl %ebx, %eax, %eax
+; X86-BMI2-NEXT: movzbl %cl, %edx
+; X86-BMI2-NEXT: movl $1, %ebp
+; X86-BMI2-NEXT: negl %edx
+; X86-BMI2-NEXT: shll %cl, %eax
+; X86-BMI2-NEXT: shrxl %edx, %ebp, %edx
+; X86-BMI2-NEXT: orl %eax, %edx
+; X86-BMI2-NEXT: shlxl %ebx, %ebp, %eax
; X86-BMI2-NEXT: testb $32, %bl
; X86-BMI2-NEXT: je .LBB11_4
; X86-BMI2-NEXT: # %bb.3:
@@ -1231,6 +1261,7 @@ define i64 @bextr64_a3_load_indexzext(ptr %w, i8 zeroext %numskipbits, i8 zeroex
; X86-BMI2-NEXT: popl %esi
; X86-BMI2-NEXT: popl %edi
; X86-BMI2-NEXT: popl %ebx
+; X86-BMI2-NEXT: popl %ebp
; X86-BMI2-NEXT: retl
;
; X64-NOBMI-LABEL: bextr64_a3_load_indexzext:
@@ -1346,6 +1377,7 @@ define i64 @bextr64_a4_commutative(i64 %val, i64 %numskipbits, i64 %numlowbits)
;
; X86-BMI2-LABEL: bextr64_a4_commutative:
; X86-BMI2: # %bb.0:
+; X86-BMI2-NEXT: pushl %ebp
; X86-BMI2-NEXT: pushl %ebx
; X86-BMI2-NEXT: pushl %edi
; X86-BMI2-NEXT: pushl %esi
@@ -1355,30 +1387,35 @@ define i64 @bextr64_a4_commutative(i64 %val, i64 %numskipbits, i64 %numlowbits)
; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-BMI2-NEXT: shrdl %cl, %edx, %eax
; X86-BMI2-NEXT: shrxl %ecx, %edx, %edx
+; X86-BMI2-NEXT: xorl %esi, %esi
; X86-BMI2-NEXT: testb $32, %cl
; X86-BMI2-NEXT: je .LBB12_2
; X86-BMI2-NEXT: # %bb.1:
; X86-BMI2-NEXT: movl %edx, %eax
; X86-BMI2-NEXT: xorl %edx, %edx
; X86-BMI2-NEXT: .LBB12_2:
-; X86-BMI2-NEXT: movl $1, %edi
-; X86-BMI2-NEXT: xorl %esi, %esi
; X86-BMI2-NEXT: movl %ebx, %ecx
-; X86-BMI2-NEXT: shldl %cl, %edi, %esi
-; X86-BMI2-NEXT: shlxl %ebx, %edi, %ecx
+; X86-BMI2-NEXT: movzbl %cl, %edi
+; X86-BMI2-NEXT: movl $1, %ebp
+; X86-BMI2-NEXT: negl %edi
+; X86-BMI2-NEXT: shll %cl, %esi
+; X86-BMI2-NEXT: shrxl %edi, %ebp, %ecx
+; X86-BMI2-NEXT: orl %esi, %ecx
+; X86-BMI2-NEXT: shlxl %ebx, %ebp, %esi
; X86-BMI2-NEXT: testb $32, %bl
; X86-BMI2-NEXT: je .LBB12_4
; X86-BMI2-NEXT: # %bb.3:
-; X86-BMI2-NEXT: movl %ecx, %esi
-; X86-BMI2-NEXT: xorl %ecx, %ecx
+; X86-BMI2-NEXT: movl %esi, %ecx
+; X86-BMI2-NEXT: xorl %esi, %esi
; X86-BMI2-NEXT: .LBB12_4:
-; X86-BMI2-NEXT: addl $-1, %ecx
-; X86-BMI2-NEXT: adcl $-1, %esi
-; X86-BMI2-NEXT: andl %ecx, %eax
-; X86-BMI2-NEXT: andl %esi, %edx
+; X86-BMI2-NEXT: addl $-1, %esi
+; X86-BMI2-NEXT: adcl $-1, %ecx
+; X86-BMI2-NEXT: andl %esi, %eax
+; X86-BMI2-NEXT: andl %ecx, %edx
; X86-BMI2-NEXT: popl %esi
; X86-BMI2-NEXT: popl %edi
; X86-BMI2-NEXT: popl %ebx
+; X86-BMI2-NEXT: popl %ebp
; X86-BMI2-NEXT: retl
;
; X64-NOBMI-LABEL: bextr64_a4_commutative:
@@ -1524,21 +1561,24 @@ define i64 @bextr64_a5_skipextrauses(i64 %val, i64 %numskipbits, i64 %numlowbits
; X86-BMI2-NEXT: movzbl {{[0-9]+}}(%esp), %edx
; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ebx
; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-BMI2-NEXT: movl %eax, %ecx
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-BMI2-NEXT: shrdl %cl, %esi, %ebx
-; X86-BMI2-NEXT: shrxl %eax, %esi, %ebp
-; X86-BMI2-NEXT: testb $32, %al
+; X86-BMI2-NEXT: shrxl %ecx, %esi, %ebp
+; X86-BMI2-NEXT: xorl %edi, %edi
+; X86-BMI2-NEXT: testb $32, %cl
; X86-BMI2-NEXT: je .LBB13_2
; X86-BMI2-NEXT: # %bb.1:
; X86-BMI2-NEXT: movl %ebp, %ebx
; X86-BMI2-NEXT: xorl %ebp, %ebp
; X86-BMI2-NEXT: .LBB13_2:
-; X86-BMI2-NEXT: movl $1, %edi
-; X86-BMI2-NEXT: xorl %esi, %esi
; X86-BMI2-NEXT: movl %edx, %ecx
-; X86-BMI2-NEXT: shldl %cl, %edi, %esi
-; X86-BMI2-NEXT: shlxl %edx, %edi, %edi
+; X86-BMI2-NEXT: movzbl %cl, %esi
+; X86-BMI2-NEXT: movl $1, %eax
+; X86-BMI2-NEXT: negl %esi
+; X86-BMI2-NEXT: shll %cl, %edi
+; X86-BMI2-NEXT: shrxl %esi, %eax, %esi
+; X86-BMI2-NEXT: orl %edi, %esi
+; X86-BMI2-NEXT: shlxl %edx, %eax, %edi
; X86-BMI2-NEXT: testb $32, %dl
; X86-BMI2-NEXT: je .LBB13_4
; X86-BMI2-NEXT: # %bb.3:
@@ -1551,7 +1591,7 @@ define i64 @bextr64_a5_skipextrauses(i64 %val, i64 %numskipbits, i64 %numlowbits
; X86-BMI2-NEXT: andl %ebp, %esi
; X86-BMI2-NEXT: subl $8, %esp
; X86-BMI2-NEXT: pushl {{[0-9]+}}(%esp)
-; X86-BMI2-NEXT: pushl %eax
+; X86-BMI2-NEXT: pushl {{[0-9]+}}(%esp)
; X86-BMI2-NEXT: calll use64 at PLT
; X86-BMI2-NEXT: addl $16, %esp
; X86-BMI2-NEXT: movl %edi, %eax
@@ -6465,6 +6505,7 @@ define i32 @bextr64_32_c3(i64 %val, i64 %numskipbits, i64 %numlowbits) nounwind
;
; X86-BMI2-LABEL: bextr64_32_c3:
; X86-BMI2: # %bb.0:
+; X86-BMI2-NEXT: pushl %edi
; X86-BMI2-NEXT: pushl %esi
; X86-BMI2-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx
@@ -6477,9 +6518,13 @@ define i32 @bextr64_32_c3(i64 %val, i64 %numskipbits, i64 %numlowbits) nounwind
; X86-BMI2-NEXT: .LBB50_2:
; X86-BMI2-NEXT: movb $64, %cl
; X86-BMI2-NEXT: subb {{[0-9]+}}(%esp), %cl
+; X86-BMI2-NEXT: movzbl %cl, %esi
; X86-BMI2-NEXT: xorl %eax, %eax
-; X86-BMI2-NEXT: movl $-1, %esi
-; X86-BMI2-NEXT: shrdl %cl, %eax, %esi
+; X86-BMI2-NEXT: movl $-1, %edi
+; X86-BMI2-NEXT: negl %esi
+; X86-BMI2-NEXT: shrl %cl, %edi
+; X86-BMI2-NEXT: shlxl %esi, %eax, %esi
+; X86-BMI2-NEXT: orl %edi, %esi
; X86-BMI2-NEXT: testb $32, %cl
; X86-BMI2-NEXT: jne .LBB50_4
; X86-BMI2-NEXT: # %bb.3:
@@ -6487,6 +6532,7 @@ define i32 @bextr64_32_c3(i64 %val, i64 %numskipbits, i64 %numlowbits) nounwind
; X86-BMI2-NEXT: .LBB50_4:
; X86-BMI2-NEXT: andl %edx, %eax
; X86-BMI2-NEXT: popl %esi
+; X86-BMI2-NEXT: popl %edi
; X86-BMI2-NEXT: retl
;
; X64-NOBMI-LABEL: bextr64_32_c3:
diff --git a/llvm/test/CodeGen/X86/extract-lowbits.ll b/llvm/test/CodeGen/X86/extract-lowbits.ll
index aff5ac7437a29..a9b82546b1873 100644
--- a/llvm/test/CodeGen/X86/extract-lowbits.ll
+++ b/llvm/test/CodeGen/X86/extract-lowbits.ll
@@ -325,11 +325,17 @@ define i64 @bzhi64_a0(i64 %val, i64 %numlowbits) nounwind {
;
; X86-BMI2-LABEL: bzhi64_a0:
; X86-BMI2: # %bb.0:
+; X86-BMI2-NEXT: pushl %edi
+; X86-BMI2-NEXT: pushl %esi
; X86-BMI2-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X86-BMI2-NEXT: movl $1, %eax
-; X86-BMI2-NEXT: xorl %edx, %edx
-; X86-BMI2-NEXT: shldl %cl, %eax, %edx
-; X86-BMI2-NEXT: shlxl %ecx, %eax, %eax
+; X86-BMI2-NEXT: movzbl %cl, %eax
+; X86-BMI2-NEXT: movl $1, %esi
+; X86-BMI2-NEXT: xorl %edi, %edi
+; X86-BMI2-NEXT: negl %eax
+; X86-BMI2-NEXT: shll %cl, %edi
+; X86-BMI2-NEXT: shrxl %eax, %esi, %edx
+; X86-BMI2-NEXT: orl %edi, %edx
+; X86-BMI2-NEXT: shlxl %ecx, %esi, %eax
; X86-BMI2-NEXT: testb $32, %cl
; X86-BMI2-NEXT: je .LBB5_2
; X86-BMI2-NEXT: # %bb.1:
@@ -340,6 +346,8 @@ define i64 @bzhi64_a0(i64 %val, i64 %numlowbits) nounwind {
; X86-BMI2-NEXT: adcl $-1, %edx
; X86-BMI2-NEXT: andl {{[0-9]+}}(%esp), %edx
; X86-BMI2-NEXT: andl {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT: popl %esi
+; X86-BMI2-NEXT: popl %edi
; X86-BMI2-NEXT: retl
;
; X64-NOBMI-LABEL: bzhi64_a0:
@@ -410,11 +418,17 @@ define i64 @bzhi64_a0_masked(i64 %val, i64 %numlowbits) nounwind {
;
; X86-BMI2-LABEL: bzhi64_a0_masked:
; X86-BMI2: # %bb.0:
+; X86-BMI2-NEXT: pushl %edi
+; X86-BMI2-NEXT: pushl %esi
; X86-BMI2-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X86-BMI2-NEXT: movl $1, %eax
-; X86-BMI2-NEXT: xorl %edx, %edx
-; X86-BMI2-NEXT: shldl %cl, %eax, %edx
-; X86-BMI2-NEXT: shlxl %ecx, %eax, %eax
+; X86-BMI2-NEXT: movzbl %cl, %eax
+; X86-BMI2-NEXT: movl $1, %esi
+; X86-BMI2-NEXT: xorl %edi, %edi
+; X86-BMI2-NEXT: negl %eax
+; X86-BMI2-NEXT: shll %cl, %edi
+; X86-BMI2-NEXT: shrxl %eax, %esi, %edx
+; X86-BMI2-NEXT: orl %edi, %edx
+; X86-BMI2-NEXT: shlxl %ecx, %esi, %eax
; X86-BMI2-NEXT: testb $32, %cl
; X86-BMI2-NEXT: je .LBB6_2
; X86-BMI2-NEXT: # %bb.1:
@@ -425,6 +439,8 @@ define i64 @bzhi64_a0_masked(i64 %val, i64 %numlowbits) nounwind {
; X86-BMI2-NEXT: adcl $-1, %edx
; X86-BMI2-NEXT: andl {{[0-9]+}}(%esp), %edx
; X86-BMI2-NEXT: andl {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT: popl %esi
+; X86-BMI2-NEXT: popl %edi
; X86-BMI2-NEXT: retl
;
; X64-NOBMI-LABEL: bzhi64_a0_masked:
@@ -497,11 +513,17 @@ define i64 @bzhi64_a1_indexzext(i64 %val, i8 zeroext %numlowbits) nounwind {
;
; X86-BMI2-LABEL: bzhi64_a1_indexzext:
; X86-BMI2: # %bb.0:
+; X86-BMI2-NEXT: pushl %edi
+; X86-BMI2-NEXT: pushl %esi
; X86-BMI2-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X86-BMI2-NEXT: movl $1, %eax
-; X86-BMI2-NEXT: xorl %edx, %edx
-; X86-BMI2-NEXT: shldl %cl, %eax, %edx
-; X86-BMI2-NEXT: shlxl %ecx, %eax, %eax
+; X86-BMI2-NEXT: movzbl %cl, %eax
+; X86-BMI2-NEXT: movl $1, %esi
+; X86-BMI2-NEXT: xorl %edi, %edi
+; X86-BMI2-NEXT: negl %eax
+; X86-BMI2-NEXT: shll %cl, %edi
+; X86-BMI2-NEXT: shrxl %eax, %esi, %edx
+; X86-BMI2-NEXT: orl %edi, %edx
+; X86-BMI2-NEXT: shlxl %ecx, %esi, %eax
; X86-BMI2-NEXT: testb $32, %cl
; X86-BMI2-NEXT: je .LBB7_2
; X86-BMI2-NEXT: # %bb.1:
@@ -512,6 +534,8 @@ define i64 @bzhi64_a1_indexzext(i64 %val, i8 zeroext %numlowbits) nounwind {
; X86-BMI2-NEXT: adcl $-1, %edx
; X86-BMI2-NEXT: andl {{[0-9]+}}(%esp), %edx
; X86-BMI2-NEXT: andl {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT: popl %esi
+; X86-BMI2-NEXT: popl %edi
; X86-BMI2-NEXT: retl
;
; X64-NOBMI-LABEL: bzhi64_a1_indexzext:
@@ -590,13 +614,19 @@ define i64 @bzhi64_a2_load(ptr %w, i64 %numlowbits) nounwind {
;
; X86-BMI2-LABEL: bzhi64_a2_load:
; X86-BMI2: # %bb.0:
+; X86-BMI2-NEXT: pushl %ebx
+; X86-BMI2-NEXT: pushl %edi
; X86-BMI2-NEXT: pushl %esi
-; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %esi
; X86-BMI2-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X86-BMI2-NEXT: movl $1, %eax
-; X86-BMI2-NEXT: xorl %edx, %edx
-; X86-BMI2-NEXT: shldl %cl, %eax, %edx
-; X86-BMI2-NEXT: shlxl %ecx, %eax, %eax
+; X86-BMI2-NEXT: movzbl %cl, %eax
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-BMI2-NEXT: movl $1, %edi
+; X86-BMI2-NEXT: xorl %ebx, %ebx
+; X86-BMI2-NEXT: negl %eax
+; X86-BMI2-NEXT: shll %cl, %ebx
+; X86-BMI2-NEXT: shrxl %eax, %edi, %edx
+; X86-BMI2-NEXT: orl %ebx, %edx
+; X86-BMI2-NEXT: shlxl %ecx, %edi, %eax
; X86-BMI2-NEXT: testb $32, %cl
; X86-BMI2-NEXT: je .LBB8_2
; X86-BMI2-NEXT: # %bb.1:
@@ -608,6 +638,8 @@ define i64 @bzhi64_a2_load(ptr %w, i64 %numlowbits) nounwind {
; X86-BMI2-NEXT: andl 4(%esi), %edx
; X86-BMI2-NEXT: andl (%esi), %eax
; X86-BMI2-NEXT: popl %esi
+; X86-BMI2-NEXT: popl %edi
+; X86-BMI2-NEXT: popl %ebx
; X86-BMI2-NEXT: retl
;
; X64-NOBMI-LABEL: bzhi64_a2_load:
@@ -684,13 +716,19 @@ define i64 @bzhi64_a3_load_indexzext(ptr %w, i8 zeroext %numlowbits) nounwind {
;
; X86-BMI2-LABEL: bzhi64_a3_load_indexzext:
; X86-BMI2: # %bb.0:
+; X86-BMI2-NEXT: pushl %ebx
+; X86-BMI2-NEXT: pushl %edi
; X86-BMI2-NEXT: pushl %esi
-; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %esi
; X86-BMI2-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X86-BMI2-NEXT: movl $1, %eax
-; X86-BMI2-NEXT: xorl %edx, %edx
-; X86-BMI2-NEXT: shldl %cl, %eax, %edx
-; X86-BMI2-NEXT: shlxl %ecx, %eax, %eax
+; X86-BMI2-NEXT: movzbl %cl, %eax
+; X86-BMI2-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-BMI2-NEXT: movl $1, %edi
+; X86-BMI2-NEXT: xorl %ebx, %ebx
+; X86-BMI2-NEXT: negl %eax
+; X86-BMI2-NEXT: shll %cl, %ebx
+; X86-BMI2-NEXT: shrxl %eax, %edi, %edx
+; X86-BMI2-NEXT: orl %ebx, %edx
+; X86-BMI2-NEXT: shlxl %ecx, %edi, %eax
; X86-BMI2-NEXT: testb $32, %cl
; X86-BMI2-NEXT: je .LBB9_2
; X86-BMI2-NEXT: # %bb.1:
@@ -702,6 +740,8 @@ define i64 @bzhi64_a3_load_indexzext(ptr %w, i8 zeroext %numlowbits) nounwind {
; X86-BMI2-NEXT: andl 4(%esi), %edx
; X86-BMI2-NEXT: andl (%esi), %eax
; X86-BMI2-NEXT: popl %esi
+; X86-BMI2-NEXT: popl %edi
+; X86-BMI2-NEXT: popl %ebx
; X86-BMI2-NEXT: retl
;
; X64-NOBMI-LABEL: bzhi64_a3_load_indexzext:
@@ -775,11 +815,17 @@ define i64 @bzhi64_a4_commutative(i64 %val, i64 %numlowbits) nounwind {
;
; X86-BMI2-LABEL: bzhi64_a4_commutative:
; X86-BMI2: # %bb.0:
+; X86-BMI2-NEXT: pushl %edi
+; X86-BMI2-NEXT: pushl %esi
; X86-BMI2-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X86-BMI2-NEXT: movl $1, %eax
-; X86-BMI2-NEXT: xorl %edx, %edx
-; X86-BMI2-NEXT: shldl %cl, %eax, %edx
-; X86-BMI2-NEXT: shlxl %ecx, %eax, %eax
+; X86-BMI2-NEXT: movzbl %cl, %eax
+; X86-BMI2-NEXT: movl $1, %esi
+; X86-BMI2-NEXT: xorl %edi, %edi
+; X86-BMI2-NEXT: negl %eax
+; X86-BMI2-NEXT: shll %cl, %edi
+; X86-BMI2-NEXT: shrxl %eax, %esi, %edx
+; X86-BMI2-NEXT: orl %edi, %edx
+; X86-BMI2-NEXT: shlxl %ecx, %esi, %eax
; X86-BMI2-NEXT: testb $32, %cl
; X86-BMI2-NEXT: je .LBB10_2
; X86-BMI2-NEXT: # %bb.1:
@@ -790,6 +836,8 @@ define i64 @bzhi64_a4_commutative(i64 %val, i64 %numlowbits) nounwind {
; X86-BMI2-NEXT: adcl $-1, %edx
; X86-BMI2-NEXT: andl {{[0-9]+}}(%esp), %edx
; X86-BMI2-NEXT: andl {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT: popl %esi
+; X86-BMI2-NEXT: popl %edi
; X86-BMI2-NEXT: retl
;
; X64-NOBMI-LABEL: bzhi64_a4_commutative:
@@ -3219,20 +3267,56 @@ define i32 @bzhi64_32_c2(i64 %val, i32 %numlowbits) nounwind {
; Shifting happens in 64-bit. Mask is 32-bit, but calculated in 64-bit.
; Masking is 64-bit. Then truncation.
define i32 @bzhi64_32_c3(i64 %val, i64 %numlowbits) nounwind {
-; X86-LABEL: bzhi64_32_c3:
-; X86: # %bb.0:
-; X86-NEXT: movb $64, %cl
-; X86-NEXT: subb {{[0-9]+}}(%esp), %cl
-; X86-NEXT: xorl %eax, %eax
-; X86-NEXT: movl $-1, %edx
-; X86-NEXT: shrdl %cl, %eax, %edx
-; X86-NEXT: testb $32, %cl
-; X86-NEXT: jne .LBB43_2
-; X86-NEXT: # %bb.1:
-; X86-NEXT: movl %edx, %eax
-; X86-NEXT: .LBB43_2:
-; X86-NEXT: andl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: retl
+; X86-NOBMI-LABEL: bzhi64_32_c3:
+; X86-NOBMI: # %bb.0:
+; X86-NOBMI-NEXT: movb $64, %cl
+; X86-NOBMI-NEXT: subb {{[0-9]+}}(%esp), %cl
+; X86-NOBMI-NEXT: xorl %eax, %eax
+; X86-NOBMI-NEXT: movl $-1, %edx
+; X86-NOBMI-NEXT: shrdl %cl, %eax, %edx
+; X86-NOBMI-NEXT: testb $32, %cl
+; X86-NOBMI-NEXT: jne .LBB43_2
+; X86-NOBMI-NEXT: # %bb.1:
+; X86-NOBMI-NEXT: movl %edx, %eax
+; X86-NOBMI-NEXT: .LBB43_2:
+; X86-NOBMI-NEXT: andl {{[0-9]+}}(%esp), %eax
+; X86-NOBMI-NEXT: retl
+;
+; X86-BMI1-LABEL: bzhi64_32_c3:
+; X86-BMI1: # %bb.0:
+; X86-BMI1-NEXT: movb $64, %cl
+; X86-BMI1-NEXT: subb {{[0-9]+}}(%esp), %cl
+; X86-BMI1-NEXT: xorl %eax, %eax
+; X86-BMI1-NEXT: movl $-1, %edx
+; X86-BMI1-NEXT: shrdl %cl, %eax, %edx
+; X86-BMI1-NEXT: testb $32, %cl
+; X86-BMI1-NEXT: jne .LBB43_2
+; X86-BMI1-NEXT: # %bb.1:
+; X86-BMI1-NEXT: movl %edx, %eax
+; X86-BMI1-NEXT: .LBB43_2:
+; X86-BMI1-NEXT: andl {{[0-9]+}}(%esp), %eax
+; X86-BMI1-NEXT: retl
+;
+; X86-BMI2-LABEL: bzhi64_32_c3:
+; X86-BMI2: # %bb.0:
+; X86-BMI2-NEXT: pushl %esi
+; X86-BMI2-NEXT: movb $64, %cl
+; X86-BMI2-NEXT: subb {{[0-9]+}}(%esp), %cl
+; X86-BMI2-NEXT: movzbl %cl, %edx
+; X86-BMI2-NEXT: xorl %eax, %eax
+; X86-BMI2-NEXT: movl $-1, %esi
+; X86-BMI2-NEXT: negl %edx
+; X86-BMI2-NEXT: shrl %cl, %esi
+; X86-BMI2-NEXT: shlxl %edx, %eax, %edx
+; X86-BMI2-NEXT: orl %esi, %edx
+; X86-BMI2-NEXT: testb $32, %cl
+; X86-BMI2-NEXT: jne .LBB43_2
+; X86-BMI2-NEXT: # %bb.1:
+; X86-BMI2-NEXT: movl %edx, %eax
+; X86-BMI2-NEXT: .LBB43_2:
+; X86-BMI2-NEXT: andl {{[0-9]+}}(%esp), %eax
+; X86-BMI2-NEXT: popl %esi
+; X86-BMI2-NEXT: retl
;
; X64-NOBMI-LABEL: bzhi64_32_c3:
; X64-NOBMI: # %bb.0:
diff --git a/llvm/test/CodeGen/X86/fshl.ll b/llvm/test/CodeGen/X86/fshl.ll
index c4bd4b940008d..256c0460d90ef 100644
--- a/llvm/test/CodeGen/X86/fshl.ll
+++ b/llvm/test/CodeGen/X86/fshl.ll
@@ -89,13 +89,24 @@ define i16 @var_shift_i16(i16 %x, i16 %y, i16 %z) nounwind {
}
define i32 @var_shift_i32(i32 %x, i32 %y, i32 %z) nounwind {
-; X86-LABEL: var_shift_i32:
-; X86: # %bb.0:
-; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: shldl %cl, %edx, %eax
-; X86-NEXT: retl
+; X86-FAST-LABEL: var_shift_i32:
+; X86-FAST: # %bb.0:
+; X86-FAST-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-FAST-NEXT: shldl %cl, %edx, %eax
+; X86-FAST-NEXT: retl
+;
+; X86-SLOW-LABEL: var_shift_i32:
+; X86-SLOW: # %bb.0:
+; X86-SLOW-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-SLOW-NEXT: movzbl %cl, %eax
+; X86-SLOW-NEXT: shll %cl, %edx
+; X86-SLOW-NEXT: negl %eax
+; X86-SLOW-NEXT: shrxl %eax, {{[0-9]+}}(%esp), %eax
+; X86-SLOW-NEXT: orl %edx, %eax
+; X86-SLOW-NEXT: retl
;
; X64-FAST-LABEL: var_shift_i32:
; X64-FAST: # %bb.0:
@@ -117,13 +128,24 @@ define i32 @var_shift_i32(i32 %x, i32 %y, i32 %z) nounwind {
}
define i32 @var_shift_i32_optsize(i32 %x, i32 %y, i32 %z) nounwind optsize {
-; X86-LABEL: var_shift_i32_optsize:
-; X86: # %bb.0:
-; X86-NEXT: movb {{[0-9]+}}(%esp), %cl
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: shldl %cl, %edx, %eax
-; X86-NEXT: retl
+; X86-FAST-LABEL: var_shift_i32_optsize:
+; X86-FAST: # %bb.0:
+; X86-FAST-NEXT: movb {{[0-9]+}}(%esp), %cl
+; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-FAST-NEXT: shldl %cl, %edx, %eax
+; X86-FAST-NEXT: retl
+;
+; X86-SLOW-LABEL: var_shift_i32_optsize:
+; X86-SLOW: # %bb.0:
+; X86-SLOW-NEXT: movb {{[0-9]+}}(%esp), %cl
+; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-SLOW-NEXT: movzbl %cl, %eax
+; X86-SLOW-NEXT: shll %cl, %edx
+; X86-SLOW-NEXT: negl %eax
+; X86-SLOW-NEXT: shrxl %eax, {{[0-9]+}}(%esp), %eax
+; X86-SLOW-NEXT: orl %edx, %eax
+; X86-SLOW-NEXT: retl
;
; X64-FAST-LABEL: var_shift_i32_optsize:
; X64-FAST: # %bb.0:
@@ -145,13 +167,24 @@ define i32 @var_shift_i32_optsize(i32 %x, i32 %y, i32 %z) nounwind optsize {
}
define i32 @var_shift_i32_pgso(i32 %x, i32 %y, i32 %z) nounwind !prof !14 {
-; X86-LABEL: var_shift_i32_pgso:
-; X86: # %bb.0:
-; X86-NEXT: movb {{[0-9]+}}(%esp), %cl
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: shldl %cl, %edx, %eax
-; X86-NEXT: retl
+; X86-FAST-LABEL: var_shift_i32_pgso:
+; X86-FAST: # %bb.0:
+; X86-FAST-NEXT: movb {{[0-9]+}}(%esp), %cl
+; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-FAST-NEXT: shldl %cl, %edx, %eax
+; X86-FAST-NEXT: retl
+;
+; X86-SLOW-LABEL: var_shift_i32_pgso:
+; X86-SLOW: # %bb.0:
+; X86-SLOW-NEXT: movb {{[0-9]+}}(%esp), %cl
+; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-SLOW-NEXT: movzbl %cl, %eax
+; X86-SLOW-NEXT: shll %cl, %edx
+; X86-SLOW-NEXT: negl %eax
+; X86-SLOW-NEXT: shrxl %eax, {{[0-9]+}}(%esp), %eax
+; X86-SLOW-NEXT: orl %edx, %eax
+; X86-SLOW-NEXT: retl
;
; X64-FAST-LABEL: var_shift_i32_pgso:
; X64-FAST: # %bb.0:
@@ -195,22 +228,34 @@ define i64 @var_shift_i64(i64 %x, i64 %y, i64 %z) nounwind {
;
; X86-SLOW-LABEL: var_shift_i64:
; X86-SLOW: # %bb.0:
+; X86-SLOW-NEXT: pushl %ebp
+; X86-SLOW-NEXT: pushl %ebx
; X86-SLOW-NEXT: pushl %edi
; X86-SLOW-NEXT: pushl %esi
-; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %ebx
; X86-SLOW-NEXT: testb $32, %cl
-; X86-SLOW-NEXT: movl %edx, %edi
-; X86-SLOW-NEXT: cmovnel %esi, %edi
-; X86-SLOW-NEXT: cmovel {{[0-9]+}}(%esp), %edx
-; X86-SLOW-NEXT: cmovnel {{[0-9]+}}(%esp), %esi
-; X86-SLOW-NEXT: movl %edi, %eax
-; X86-SLOW-NEXT: shldl %cl, %esi, %eax
+; X86-SLOW-NEXT: movl %esi, %edx
+; X86-SLOW-NEXT: movzbl %cl, %edi
+; X86-SLOW-NEXT: cmovnel %ebx, %edx
+; X86-SLOW-NEXT: cmovel {{[0-9]+}}(%esp), %esi
+; X86-SLOW-NEXT: cmovnel {{[0-9]+}}(%esp), %ebx
+; X86-SLOW-NEXT: negl %edi
+; X86-SLOW-NEXT: movl %edx, %eax
+; X86-SLOW-NEXT: shll %cl, %eax
; X86-SLOW-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-SLOW-NEXT: shldl %cl, %edi, %edx
+; X86-SLOW-NEXT: movzbl %cl, %ebp
+; X86-SLOW-NEXT: negl %ebp
+; X86-SLOW-NEXT: shrxl %ebp, %edx, %edx
+; X86-SLOW-NEXT: shrxl %edi, %ebx, %edi
+; X86-SLOW-NEXT: shll %cl, %esi
+; X86-SLOW-NEXT: orl %edi, %eax
+; X86-SLOW-NEXT: orl %esi, %edx
; X86-SLOW-NEXT: popl %esi
; X86-SLOW-NEXT: popl %edi
+; X86-SLOW-NEXT: popl %ebx
+; X86-SLOW-NEXT: popl %ebp
; X86-SLOW-NEXT: retl
;
; X64-FAST-LABEL: var_shift_i64:
@@ -302,50 +347,67 @@ define i128 @var_shift_i128(i128 %x, i128 %y, i128 %z) nounwind {
; X86-SLOW-NEXT: pushl %esi
; X86-SLOW-NEXT: andl $-16, %esp
; X86-SLOW-NEXT: subl $16, %esp
-; X86-SLOW-NEXT: movl 56(%ebp), %ecx
-; X86-SLOW-NEXT: movl 24(%ebp), %eax
-; X86-SLOW-NEXT: movl 48(%ebp), %edi
-; X86-SLOW-NEXT: movl 28(%ebp), %edx
-; X86-SLOW-NEXT: movl 52(%ebp), %esi
-; X86-SLOW-NEXT: testb $64, %cl
-; X86-SLOW-NEXT: movl %eax, %ecx
-; X86-SLOW-NEXT: movl %edx, %ebx
+; X86-SLOW-NEXT: movl 56(%ebp), %eax
+; X86-SLOW-NEXT: movl 28(%ebp), %esi
+; X86-SLOW-NEXT: movl 52(%ebp), %edi
+; X86-SLOW-NEXT: movl 24(%ebp), %edx
+; X86-SLOW-NEXT: movl 48(%ebp), %ebx
+; X86-SLOW-NEXT: testb $64, %al
+; X86-SLOW-NEXT: movl %esi, %ecx
+; X86-SLOW-NEXT: movl %edx, %eax
; X86-SLOW-NEXT: cmovnel %edi, %ecx
-; X86-SLOW-NEXT: cmovnel %esi, %ebx
-; X86-SLOW-NEXT: cmovnel 44(%ebp), %esi
-; X86-SLOW-NEXT: cmovnel 40(%ebp), %edi
-; X86-SLOW-NEXT: cmovel 36(%ebp), %edx
-; X86-SLOW-NEXT: cmovel 32(%ebp), %eax
-; X86-SLOW-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-SLOW-NEXT: movl 56(%ebp), %ecx
-; X86-SLOW-NEXT: testb $32, %cl
-; X86-SLOW-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-SLOW-NEXT: cmovnel %eax, %edx
+; X86-SLOW-NEXT: cmovel 36(%ebp), %esi
+; X86-SLOW-NEXT: cmovel 32(%ebp), %edx
; X86-SLOW-NEXT: cmovnel %ebx, %eax
-; X86-SLOW-NEXT: cmovel %esi, %edi
-; X86-SLOW-NEXT: cmovel %ecx, %esi
-; X86-SLOW-NEXT: cmovnel %ecx, %ebx
+; X86-SLOW-NEXT: cmovnel 44(%ebp), %edi
+; X86-SLOW-NEXT: cmovnel 40(%ebp), %ebx
+; X86-SLOW-NEXT: movl %ecx, (%esp) # 4-byte Spill
; X86-SLOW-NEXT: movl 56(%ebp), %ecx
-; X86-SLOW-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-SLOW-NEXT: testb $32, %cl
+; X86-SLOW-NEXT: movl (%esp), %ecx # 4-byte Reload
+; X86-SLOW-NEXT: cmovnel %edx, %esi
+; X86-SLOW-NEXT: cmovel %edi, %ebx
+; X86-SLOW-NEXT: cmovel %eax, %edi
; X86-SLOW-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-SLOW-NEXT: shldl %cl, %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-SLOW-NEXT: movl %ebx, %edi
-; X86-SLOW-NEXT: movl 56(%ebp), %ecx
-; X86-SLOW-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-SLOW-NEXT: shldl %cl, %esi, %edi
-; X86-SLOW-NEXT: movl %eax, %esi
+; X86-SLOW-NEXT: cmovnel %ecx, %edx
+; X86-SLOW-NEXT: cmovnel %eax, %ecx
+; X86-SLOW-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-SLOW-NEXT: movl %ecx, %edx
; X86-SLOW-NEXT: movl 56(%ebp), %ecx
+; X86-SLOW-NEXT: movzbl %cl, %eax
+; X86-SLOW-NEXT: negl %eax
+; X86-SLOW-NEXT: shrxl %eax, %ebx, %eax
+; X86-SLOW-NEXT: movl %edi, %ebx
+; X86-SLOW-NEXT: shll %cl, %ebx
+; X86-SLOW-NEXT: orl %eax, %ebx
+; X86-SLOW-NEXT: movzbl %cl, %eax
+; X86-SLOW-NEXT: negl %eax
+; X86-SLOW-NEXT: shrxl %eax, %edi, %edi
+; X86-SLOW-NEXT: movl %edx, %eax
+; X86-SLOW-NEXT: shll %cl, %eax
; X86-SLOW-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-SLOW-NEXT: shldl %cl, %ebx, %esi
+; X86-SLOW-NEXT: orl %eax, %edi
+; X86-SLOW-NEXT: movzbl %cl, %eax
+; X86-SLOW-NEXT: negl %eax
+; X86-SLOW-NEXT: shrxl %eax, %edx, %esi
+; X86-SLOW-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-SLOW-NEXT: movl %edx, %eax
+; X86-SLOW-NEXT: shll %cl, %eax
; X86-SLOW-NEXT: movl 56(%ebp), %ecx
; X86-SLOW-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-SLOW-NEXT: shldl %cl, %eax, %edx
-; X86-SLOW-NEXT: movl 8(%ebp), %eax
-; X86-SLOW-NEXT: movl %edx, 12(%eax)
-; X86-SLOW-NEXT: movl %esi, 8(%eax)
-; X86-SLOW-NEXT: movl %edi, 4(%eax)
-; X86-SLOW-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-SLOW-NEXT: movl %ecx, (%eax)
+; X86-SLOW-NEXT: orl %eax, %esi
+; X86-SLOW-NEXT: movzbl %cl, %eax
+; X86-SLOW-NEXT: negl %eax
+; X86-SLOW-NEXT: shrxl %eax, %edx, %eax
+; X86-SLOW-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-SLOW-NEXT: shll %cl, %edx
+; X86-SLOW-NEXT: orl %edx, %eax
+; X86-SLOW-NEXT: movl 8(%ebp), %edx
+; X86-SLOW-NEXT: movl %eax, 12(%edx)
+; X86-SLOW-NEXT: movl %esi, 8(%edx)
+; X86-SLOW-NEXT: movl %edi, 4(%edx)
+; X86-SLOW-NEXT: movl %ebx, (%edx)
+; X86-SLOW-NEXT: movl %edx, %eax
; X86-SLOW-NEXT: leal -12(%ebp), %esp
; X86-SLOW-NEXT: popl %esi
; X86-SLOW-NEXT: popl %edi
@@ -370,13 +432,20 @@ define i128 @var_shift_i128(i128 %x, i128 %y, i128 %z) nounwind {
; X64-SLOW: # %bb.0:
; X64-SLOW-NEXT: testb $64, %r8b
; X64-SLOW-NEXT: cmovneq %rdi, %rsi
-; X64-SLOW-NEXT: cmovneq %rcx, %rdi
; X64-SLOW-NEXT: cmoveq %rcx, %rdx
+; X64-SLOW-NEXT: cmovneq %rcx, %rdi
; X64-SLOW-NEXT: movl %r8d, %ecx
+; X64-SLOW-NEXT: movzbq %cl, %rax
+; X64-SLOW-NEXT: shlq %cl, %rsi
+; X64-SLOW-NEXT: negq %rax
+; X64-SLOW-NEXT: shrxq %rax, %rdx, %rdx
; X64-SLOW-NEXT: movq %rdi, %rax
-; X64-SLOW-NEXT: shldq %cl, %rdi, %rsi
-; X64-SLOW-NEXT: shldq %cl, %rdx, %rax
-; X64-SLOW-NEXT: movq %rsi, %rdx
+; X64-SLOW-NEXT: shlq %cl, %rax
+; X64-SLOW-NEXT: orq %rdx, %rax
+; X64-SLOW-NEXT: movzbq %cl, %rdx
+; X64-SLOW-NEXT: negq %rdx
+; X64-SLOW-NEXT: shrxq %rdx, %rdi, %rdx
+; X64-SLOW-NEXT: orq %rsi, %rdx
; X64-SLOW-NEXT: retq
%tmp = tail call i128 @llvm.fshl.i128(i128 %x, i128 %y, i128 %z)
ret i128 %tmp
diff --git a/llvm/test/CodeGen/X86/fshr.ll b/llvm/test/CodeGen/X86/fshr.ll
index 25400eef54f39..00f04b1012fde 100644
--- a/llvm/test/CodeGen/X86/fshr.ll
+++ b/llvm/test/CodeGen/X86/fshr.ll
@@ -86,13 +86,24 @@ define i16 @var_shift_i16(i16 %x, i16 %y, i16 %z) nounwind {
}
define i32 @var_shift_i32(i32 %x, i32 %y, i32 %z) nounwind {
-; X86-LABEL: var_shift_i32:
-; X86: # %bb.0:
-; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: shrdl %cl, %edx, %eax
-; X86-NEXT: retl
+; X86-FAST-LABEL: var_shift_i32:
+; X86-FAST: # %bb.0:
+; X86-FAST-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-FAST-NEXT: shrdl %cl, %edx, %eax
+; X86-FAST-NEXT: retl
+;
+; X86-SLOW-LABEL: var_shift_i32:
+; X86-SLOW: # %bb.0:
+; X86-SLOW-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-SLOW-NEXT: movzbl %cl, %eax
+; X86-SLOW-NEXT: shrl %cl, %edx
+; X86-SLOW-NEXT: negl %eax
+; X86-SLOW-NEXT: shlxl %eax, {{[0-9]+}}(%esp), %eax
+; X86-SLOW-NEXT: orl %edx, %eax
+; X86-SLOW-NEXT: retl
;
; X64-FAST-LABEL: var_shift_i32:
; X64-FAST: # %bb.0:
@@ -114,13 +125,24 @@ define i32 @var_shift_i32(i32 %x, i32 %y, i32 %z) nounwind {
}
define i32 @var_shift_i32_optsize(i32 %x, i32 %y, i32 %z) nounwind optsize {
-; X86-LABEL: var_shift_i32_optsize:
-; X86: # %bb.0:
-; X86-NEXT: movb {{[0-9]+}}(%esp), %cl
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: shrdl %cl, %edx, %eax
-; X86-NEXT: retl
+; X86-FAST-LABEL: var_shift_i32_optsize:
+; X86-FAST: # %bb.0:
+; X86-FAST-NEXT: movb {{[0-9]+}}(%esp), %cl
+; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-FAST-NEXT: shrdl %cl, %edx, %eax
+; X86-FAST-NEXT: retl
+;
+; X86-SLOW-LABEL: var_shift_i32_optsize:
+; X86-SLOW: # %bb.0:
+; X86-SLOW-NEXT: movb {{[0-9]+}}(%esp), %cl
+; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-SLOW-NEXT: movzbl %cl, %eax
+; X86-SLOW-NEXT: shrl %cl, %edx
+; X86-SLOW-NEXT: negl %eax
+; X86-SLOW-NEXT: shlxl %eax, {{[0-9]+}}(%esp), %eax
+; X86-SLOW-NEXT: orl %edx, %eax
+; X86-SLOW-NEXT: retl
;
; X64-FAST-LABEL: var_shift_i32_optsize:
; X64-FAST: # %bb.0:
@@ -142,13 +164,24 @@ define i32 @var_shift_i32_optsize(i32 %x, i32 %y, i32 %z) nounwind optsize {
}
define i32 @var_shift_i32_pgso(i32 %x, i32 %y, i32 %z) nounwind !prof !14 {
-; X86-LABEL: var_shift_i32_pgso:
-; X86: # %bb.0:
-; X86-NEXT: movb {{[0-9]+}}(%esp), %cl
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: shrdl %cl, %edx, %eax
-; X86-NEXT: retl
+; X86-FAST-LABEL: var_shift_i32_pgso:
+; X86-FAST: # %bb.0:
+; X86-FAST-NEXT: movb {{[0-9]+}}(%esp), %cl
+; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-FAST-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-FAST-NEXT: shrdl %cl, %edx, %eax
+; X86-FAST-NEXT: retl
+;
+; X86-SLOW-LABEL: var_shift_i32_pgso:
+; X86-SLOW: # %bb.0:
+; X86-SLOW-NEXT: movb {{[0-9]+}}(%esp), %cl
+; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-SLOW-NEXT: movzbl %cl, %eax
+; X86-SLOW-NEXT: shrl %cl, %edx
+; X86-SLOW-NEXT: negl %eax
+; X86-SLOW-NEXT: shlxl %eax, {{[0-9]+}}(%esp), %eax
+; X86-SLOW-NEXT: orl %edx, %eax
+; X86-SLOW-NEXT: retl
;
; X64-FAST-LABEL: var_shift_i32_pgso:
; X64-FAST: # %bb.0:
@@ -189,19 +222,31 @@ define i64 @var_shift_i64(i64 %x, i64 %y, i64 %z) nounwind {
;
; X86-SLOW-LABEL: var_shift_i64:
; X86-SLOW: # %bb.0:
+; X86-SLOW-NEXT: pushl %ebx
+; X86-SLOW-NEXT: pushl %edi
; X86-SLOW-NEXT: pushl %esi
-; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %esi
; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-SLOW-NEXT: movl {{[0-9]+}}(%esp), %edi
; X86-SLOW-NEXT: testb $32, %cl
; X86-SLOW-NEXT: movl %esi, %edx
-; X86-SLOW-NEXT: cmovel %eax, %edx
+; X86-SLOW-NEXT: movzbl %cl, %eax
+; X86-SLOW-NEXT: cmovel %edi, %edx
+; X86-SLOW-NEXT: cmovel {{[0-9]+}}(%esp), %edi
; X86-SLOW-NEXT: cmovnel {{[0-9]+}}(%esp), %esi
-; X86-SLOW-NEXT: cmovel {{[0-9]+}}(%esp), %eax
-; X86-SLOW-NEXT: shrdl %cl, %edx, %eax
+; X86-SLOW-NEXT: negl %eax
+; X86-SLOW-NEXT: shlxl %eax, %edx, %eax
+; X86-SLOW-NEXT: shrl %cl, %edi
; X86-SLOW-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-SLOW-NEXT: shrdl %cl, %esi, %edx
+; X86-SLOW-NEXT: movzbl %cl, %ebx
+; X86-SLOW-NEXT: shrl %cl, %edx
+; X86-SLOW-NEXT: negl %ebx
+; X86-SLOW-NEXT: orl %edi, %eax
+; X86-SLOW-NEXT: shlxl %ebx, %esi, %esi
+; X86-SLOW-NEXT: orl %esi, %edx
; X86-SLOW-NEXT: popl %esi
+; X86-SLOW-NEXT: popl %edi
+; X86-SLOW-NEXT: popl %ebx
; X86-SLOW-NEXT: retl
;
; X64-FAST-LABEL: var_shift_i64:
@@ -284,39 +329,60 @@ define i128 @var_shift_i128(i128 %x, i128 %y, i128 %z) nounwind {
; X86-SLOW-NEXT: andl $-16, %esp
; X86-SLOW-NEXT: subl $16, %esp
; X86-SLOW-NEXT: movl 56(%ebp), %eax
-; X86-SLOW-NEXT: movl 24(%ebp), %esi
+; X86-SLOW-NEXT: movl 24(%ebp), %edx
; X86-SLOW-NEXT: movl 48(%ebp), %ecx
-; X86-SLOW-NEXT: movl 52(%ebp), %edx
+; X86-SLOW-NEXT: movl 52(%ebp), %ebx
; X86-SLOW-NEXT: testb $64, %al
-; X86-SLOW-NEXT: movl %esi, %edi
+; X86-SLOW-NEXT: movl %edx, %edi
; X86-SLOW-NEXT: movl 28(%ebp), %eax
; X86-SLOW-NEXT: cmovel %ecx, %edi
; X86-SLOW-NEXT: cmovel 40(%ebp), %ecx
-; X86-SLOW-NEXT: cmovnel 32(%ebp), %esi
-; X86-SLOW-NEXT: movl %eax, %ebx
+; X86-SLOW-NEXT: cmovnel 32(%ebp), %edx
+; X86-SLOW-NEXT: movl %eax, %esi
; X86-SLOW-NEXT: cmovnel 36(%ebp), %eax
-; X86-SLOW-NEXT: cmovel %edx, %ebx
-; X86-SLOW-NEXT: cmovel 44(%ebp), %edx
+; X86-SLOW-NEXT: cmovel %ebx, %esi
+; X86-SLOW-NEXT: cmovel 44(%ebp), %ebx
; X86-SLOW-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-SLOW-NEXT: movl 56(%ebp), %ecx
; X86-SLOW-NEXT: testb $32, %cl
-; X86-SLOW-NEXT: cmovel %esi, %eax
-; X86-SLOW-NEXT: cmovel %ebx, %esi
-; X86-SLOW-NEXT: cmovel %edi, %ebx
-; X86-SLOW-NEXT: cmovel %edx, %edi
-; X86-SLOW-NEXT: cmovel {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X86-SLOW-NEXT: shrdl %cl, %edi, %edx
-; X86-SLOW-NEXT: shrdl %cl, %ebx, %edi
; X86-SLOW-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-SLOW-NEXT: shrdl %cl, %esi, %ebx
+; X86-SLOW-NEXT: cmovel %edx, %eax
+; X86-SLOW-NEXT: cmovel %esi, %edx
+; X86-SLOW-NEXT: cmovel %edi, %esi
+; X86-SLOW-NEXT: cmovel %ebx, %edi
+; X86-SLOW-NEXT: cmovel {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X86-SLOW-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-SLOW-NEXT: movzbl %cl, %eax
+; X86-SLOW-NEXT: negl %eax
+; X86-SLOW-NEXT: shlxl %eax, %edi, %eax
+; X86-SLOW-NEXT: shrl %cl, %ebx
+; X86-SLOW-NEXT: movl 56(%ebp), %ecx
+; X86-SLOW-NEXT: orl %ebx, %eax
+; X86-SLOW-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-SLOW-NEXT: movzbl %cl, %eax
+; X86-SLOW-NEXT: shrl %cl, %edi
+; X86-SLOW-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-SLOW-NEXT: negl %eax
+; X86-SLOW-NEXT: shlxl %eax, %esi, %ebx
+; X86-SLOW-NEXT: movzbl %cl, %eax
+; X86-SLOW-NEXT: shrl %cl, %esi
; X86-SLOW-NEXT: movl 56(%ebp), %ecx
; X86-SLOW-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-SLOW-NEXT: shrdl %cl, %eax, %esi
+; X86-SLOW-NEXT: negl %eax
+; X86-SLOW-NEXT: orl %edi, %ebx
+; X86-SLOW-NEXT: shlxl %eax, %edx, %edi
+; X86-SLOW-NEXT: movzbl %cl, %eax
+; X86-SLOW-NEXT: shrl %cl, %edx
+; X86-SLOW-NEXT: negl %eax
+; X86-SLOW-NEXT: orl %esi, %edi
+; X86-SLOW-NEXT: shlxl %eax, {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-SLOW-NEXT: orl %eax, %edx
; X86-SLOW-NEXT: movl 8(%ebp), %eax
-; X86-SLOW-NEXT: movl %esi, 12(%eax)
-; X86-SLOW-NEXT: movl %ebx, 8(%eax)
-; X86-SLOW-NEXT: movl %edi, 4(%eax)
-; X86-SLOW-NEXT: movl %edx, (%eax)
+; X86-SLOW-NEXT: movl %edx, 12(%eax)
+; X86-SLOW-NEXT: movl %edi, 8(%eax)
+; X86-SLOW-NEXT: movl %ebx, 4(%eax)
+; X86-SLOW-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-SLOW-NEXT: movl %ecx, (%eax)
; X86-SLOW-NEXT: leal -12(%ebp), %esp
; X86-SLOW-NEXT: popl %esi
; X86-SLOW-NEXT: popl %edi
@@ -324,18 +390,37 @@ define i128 @var_shift_i128(i128 %x, i128 %y, i128 %z) nounwind {
; X86-SLOW-NEXT: popl %ebp
; X86-SLOW-NEXT: retl $4
;
-; X64-LABEL: var_shift_i128:
-; X64: # %bb.0:
-; X64-NEXT: testb $64, %r8b
-; X64-NEXT: movq %rdx, %rax
-; X64-NEXT: cmoveq %rdi, %rsi
-; X64-NEXT: cmoveq %rcx, %rdi
-; X64-NEXT: cmovneq %rcx, %rax
-; X64-NEXT: movl %r8d, %ecx
-; X64-NEXT: shrdq %cl, %rdi, %rax
-; X64-NEXT: shrdq %cl, %rsi, %rdi
-; X64-NEXT: movq %rdi, %rdx
-; X64-NEXT: retq
+; X64-FAST-LABEL: var_shift_i128:
+; X64-FAST: # %bb.0:
+; X64-FAST-NEXT: testb $64, %r8b
+; X64-FAST-NEXT: movq %rdx, %rax
+; X64-FAST-NEXT: cmoveq %rdi, %rsi
+; X64-FAST-NEXT: cmoveq %rcx, %rdi
+; X64-FAST-NEXT: cmovneq %rcx, %rax
+; X64-FAST-NEXT: movl %r8d, %ecx
+; X64-FAST-NEXT: shrdq %cl, %rdi, %rax
+; X64-FAST-NEXT: shrdq %cl, %rsi, %rdi
+; X64-FAST-NEXT: movq %rdi, %rdx
+; X64-FAST-NEXT: retq
+;
+; X64-SLOW-LABEL: var_shift_i128:
+; X64-SLOW: # %bb.0:
+; X64-SLOW-NEXT: testb $64, %r8b
+; X64-SLOW-NEXT: cmoveq %rdi, %rsi
+; X64-SLOW-NEXT: cmoveq %rcx, %rdi
+; X64-SLOW-NEXT: cmovneq %rcx, %rdx
+; X64-SLOW-NEXT: movl %r8d, %ecx
+; X64-SLOW-NEXT: movzbq %cl, %rax
+; X64-SLOW-NEXT: shrq %cl, %rdx
+; X64-SLOW-NEXT: negq %rax
+; X64-SLOW-NEXT: shlxq %rax, %rdi, %rax
+; X64-SLOW-NEXT: shrq %cl, %rdi
+; X64-SLOW-NEXT: orq %rdx, %rax
+; X64-SLOW-NEXT: movzbq %cl, %rdx
+; X64-SLOW-NEXT: negq %rdx
+; X64-SLOW-NEXT: shlxq %rdx, %rsi, %rdx
+; X64-SLOW-NEXT: orq %rdi, %rdx
+; X64-SLOW-NEXT: retq
%tmp = tail call i128 @llvm.fshr.i128(i128 %x, i128 %y, i128 %z)
ret i128 %tmp
}
diff --git a/llvm/test/CodeGen/X86/rot32.ll b/llvm/test/CodeGen/X86/rot32.ll
index 95da4eba415b7..821f2b049f387 100644
--- a/llvm/test/CodeGen/X86/rot32.ll
+++ b/llvm/test/CodeGen/X86/rot32.ll
@@ -30,13 +30,32 @@ entry:
}
define i32 @bar(i32 %x, i32 %y, i32 %z) nounwind readnone {
-; CHECK32-LABEL: bar:
-; CHECK32: # %bb.0: # %entry
-; CHECK32-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; CHECK32-NEXT: shldl %cl, %edx, %eax
-; CHECK32-NEXT: retl
+; X86-LABEL: bar:
+; X86: # %bb.0: # %entry
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: shldl %cl, %edx, %eax
+; X86-NEXT: retl
+;
+; SHLD-LABEL: bar:
+; SHLD: # %bb.0: # %entry
+; SHLD-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
+; SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
+; SHLD-NEXT: shldl %cl, %edx, %eax
+; SHLD-NEXT: retl
+;
+; BMI2-LABEL: bar:
+; BMI2: # %bb.0: # %entry
+; BMI2-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; BMI2-NEXT: movzbl %cl, %eax
+; BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx
+; BMI2-NEXT: shll %cl, %edx
+; BMI2-NEXT: negl %eax
+; BMI2-NEXT: shrxl %eax, {{[0-9]+}}(%esp), %eax
+; BMI2-NEXT: orl %edx, %eax
+; BMI2-NEXT: retl
;
; CHECK64-LABEL: bar:
; CHECK64: # %bb.0: # %entry
@@ -77,13 +96,32 @@ entry:
}
define i32 @bu(i32 %x, i32 %y, i32 %z) nounwind readnone {
-; CHECK32-LABEL: bu:
-; CHECK32: # %bb.0: # %entry
-; CHECK32-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; CHECK32-NEXT: shrdl %cl, %edx, %eax
-; CHECK32-NEXT: retl
+; X86-LABEL: bu:
+; X86: # %bb.0: # %entry
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: shrdl %cl, %edx, %eax
+; X86-NEXT: retl
+;
+; SHLD-LABEL: bu:
+; SHLD: # %bb.0: # %entry
+; SHLD-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; SHLD-NEXT: movl {{[0-9]+}}(%esp), %edx
+; SHLD-NEXT: movl {{[0-9]+}}(%esp), %eax
+; SHLD-NEXT: shrdl %cl, %edx, %eax
+; SHLD-NEXT: retl
+;
+; BMI2-LABEL: bu:
+; BMI2: # %bb.0: # %entry
+; BMI2-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; BMI2-NEXT: movzbl %cl, %eax
+; BMI2-NEXT: movl {{[0-9]+}}(%esp), %edx
+; BMI2-NEXT: shrl %cl, %edx
+; BMI2-NEXT: negl %eax
+; BMI2-NEXT: shlxl %eax, {{[0-9]+}}(%esp), %eax
+; BMI2-NEXT: orl %edx, %eax
+; BMI2-NEXT: retl
;
; CHECK64-LABEL: bu:
; CHECK64: # %bb.0: # %entry
diff --git a/llvm/test/CodeGen/X86/shift-i256.ll b/llvm/test/CodeGen/X86/shift-i256.ll
index 68d9103e6079b..6f48db675cc76 100644
--- a/llvm/test/CodeGen/X86/shift-i256.ll
+++ b/llvm/test/CodeGen/X86/shift-i256.ll
@@ -42,124 +42,178 @@ define i256 @shl_i256(i256 %a0, i256 %a1) nounwind {
;
; AVX2-LABEL: shl_i256:
; AVX2: # %bb.0:
+; AVX2-NEXT: movq %rdi, %rax
; AVX2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movl %r9d, %eax
-; AVX2-NEXT: shrb $3, %al
-; AVX2-NEXT: andb $24, %al
-; AVX2-NEXT: negb %al
-; AVX2-NEXT: movsbq %al, %rdx
-; AVX2-NEXT: movq -32(%rsp,%rdx), %rsi
-; AVX2-NEXT: movq -24(%rsp,%rdx), %rax
-; AVX2-NEXT: movq %rax, %r8
; AVX2-NEXT: movl %r9d, %ecx
-; AVX2-NEXT: shldq %cl, %rsi, %r8
-; AVX2-NEXT: movq -16(%rsp,%rdx), %r10
-; AVX2-NEXT: shldq %cl, %rax, %r10
-; AVX2-NEXT: movq %rdi, %rax
-; AVX2-NEXT: movq -40(%rsp,%rdx), %rdx
-; AVX2-NEXT: shlxq %r9, %rdx, %rdi
-; AVX2-NEXT: shldq %cl, %rdx, %rsi
-; AVX2-NEXT: movq %r10, 24(%rax)
-; AVX2-NEXT: movq %r8, 16(%rax)
-; AVX2-NEXT: movq %rsi, 8(%rax)
+; AVX2-NEXT: shrb $3, %cl
+; AVX2-NEXT: andb $24, %cl
+; AVX2-NEXT: negb %cl
+; AVX2-NEXT: movsbq %cl, %rdi
+; AVX2-NEXT: movq -40(%rsp,%rdi), %rsi
+; AVX2-NEXT: movq -32(%rsp,%rdi), %rdx
+; AVX2-NEXT: movq -24(%rsp,%rdi), %r8
+; AVX2-NEXT: movq %r8, %r10
+; AVX2-NEXT: movl %r9d, %ecx
+; AVX2-NEXT: shlq %cl, %r10
+; AVX2-NEXT: movzbq %cl, %rcx
+; AVX2-NEXT: negq %rcx
+; AVX2-NEXT: shrxq %rcx, %rdx, %r11
+; AVX2-NEXT: orq %r10, %r11
+; AVX2-NEXT: movq -16(%rsp,%rdi), %rdi
+; AVX2-NEXT: movl %r9d, %ecx
+; AVX2-NEXT: shlq %cl, %rdi
+; AVX2-NEXT: movzbq %cl, %rcx
+; AVX2-NEXT: negq %rcx
+; AVX2-NEXT: shrxq %rcx, %r8, %r8
+; AVX2-NEXT: orq %rdi, %r8
+; AVX2-NEXT: shlxq %r9, %rsi, %rdi
+; AVX2-NEXT: movl %r9d, %ecx
+; AVX2-NEXT: shlq %cl, %rdx
+; AVX2-NEXT: movzbq %cl, %rcx
+; AVX2-NEXT: negq %rcx
+; AVX2-NEXT: shrxq %rcx, %rsi, %rcx
+; AVX2-NEXT: orq %rdx, %rcx
+; AVX2-NEXT: movq %r8, 24(%rax)
+; AVX2-NEXT: movq %r11, 16(%rax)
+; AVX2-NEXT: movq %rcx, 8(%rax)
; AVX2-NEXT: movq %rdi, (%rax)
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512F-LABEL: shl_i256:
; AVX512F: # %bb.0:
+; AVX512F-NEXT: movq %rdi, %rax
; AVX512F-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX512F-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512F-NEXT: movl %r9d, %eax
-; AVX512F-NEXT: shrb $3, %al
-; AVX512F-NEXT: andb $24, %al
-; AVX512F-NEXT: negb %al
-; AVX512F-NEXT: movsbq %al, %rdx
-; AVX512F-NEXT: movq -32(%rsp,%rdx), %rsi
-; AVX512F-NEXT: movq -24(%rsp,%rdx), %rax
-; AVX512F-NEXT: movq %rax, %r8
; AVX512F-NEXT: movl %r9d, %ecx
-; AVX512F-NEXT: shldq %cl, %rsi, %r8
-; AVX512F-NEXT: movq -16(%rsp,%rdx), %r10
-; AVX512F-NEXT: shldq %cl, %rax, %r10
-; AVX512F-NEXT: movq %rdi, %rax
-; AVX512F-NEXT: movq -40(%rsp,%rdx), %rdx
-; AVX512F-NEXT: shlxq %r9, %rdx, %rdi
-; AVX512F-NEXT: shldq %cl, %rdx, %rsi
-; AVX512F-NEXT: movq %r10, 24(%rax)
-; AVX512F-NEXT: movq %r8, 16(%rax)
-; AVX512F-NEXT: movq %rsi, 8(%rax)
+; AVX512F-NEXT: shrb $3, %cl
+; AVX512F-NEXT: andb $24, %cl
+; AVX512F-NEXT: negb %cl
+; AVX512F-NEXT: movsbq %cl, %rdi
+; AVX512F-NEXT: movq -40(%rsp,%rdi), %rsi
+; AVX512F-NEXT: movq -32(%rsp,%rdi), %rdx
+; AVX512F-NEXT: movq -24(%rsp,%rdi), %r8
+; AVX512F-NEXT: movq %r8, %r10
+; AVX512F-NEXT: movl %r9d, %ecx
+; AVX512F-NEXT: shlq %cl, %r10
+; AVX512F-NEXT: movzbq %cl, %rcx
+; AVX512F-NEXT: negq %rcx
+; AVX512F-NEXT: shrxq %rcx, %rdx, %r11
+; AVX512F-NEXT: orq %r10, %r11
+; AVX512F-NEXT: movq -16(%rsp,%rdi), %rdi
+; AVX512F-NEXT: movl %r9d, %ecx
+; AVX512F-NEXT: shlq %cl, %rdi
+; AVX512F-NEXT: movzbq %cl, %rcx
+; AVX512F-NEXT: negq %rcx
+; AVX512F-NEXT: shrxq %rcx, %r8, %r8
+; AVX512F-NEXT: orq %rdi, %r8
+; AVX512F-NEXT: shlxq %r9, %rsi, %rdi
+; AVX512F-NEXT: movl %r9d, %ecx
+; AVX512F-NEXT: shlq %cl, %rdx
+; AVX512F-NEXT: movzbq %cl, %rcx
+; AVX512F-NEXT: negq %rcx
+; AVX512F-NEXT: shrxq %rcx, %rsi, %rcx
+; AVX512F-NEXT: orq %rdx, %rcx
+; AVX512F-NEXT: movq %r8, 24(%rax)
+; AVX512F-NEXT: movq %r11, 16(%rax)
+; AVX512F-NEXT: movq %rcx, 8(%rax)
; AVX512F-NEXT: movq %rdi, (%rax)
; AVX512F-NEXT: retq
;
; AVX512VL-LABEL: shl_i256:
; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: movq %rdi, %rax
; AVX512VL-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: movl %r9d, %eax
-; AVX512VL-NEXT: shrb $3, %al
-; AVX512VL-NEXT: andb $24, %al
-; AVX512VL-NEXT: negb %al
-; AVX512VL-NEXT: movsbq %al, %rax
-; AVX512VL-NEXT: movq -32(%rsp,%rax), %rdx
-; AVX512VL-NEXT: movq -24(%rsp,%rax), %rsi
-; AVX512VL-NEXT: movq %rsi, %r8
; AVX512VL-NEXT: movl %r9d, %ecx
-; AVX512VL-NEXT: shldq %cl, %rdx, %r8
-; AVX512VL-NEXT: movq -16(%rsp,%rax), %r10
-; AVX512VL-NEXT: shldq %cl, %rsi, %r10
-; AVX512VL-NEXT: movq -40(%rsp,%rax), %rsi
-; AVX512VL-NEXT: shldq %cl, %rsi, %rdx
-; AVX512VL-NEXT: movq %rdi, %rax
-; AVX512VL-NEXT: shlxq %r9, %rsi, %rcx
-; AVX512VL-NEXT: movq %r10, 24(%rdi)
-; AVX512VL-NEXT: movq %r8, 16(%rdi)
-; AVX512VL-NEXT: movq %rdx, 8(%rdi)
-; AVX512VL-NEXT: movq %rcx, (%rdi)
+; AVX512VL-NEXT: shrb $3, %cl
+; AVX512VL-NEXT: andb $24, %cl
+; AVX512VL-NEXT: negb %cl
+; AVX512VL-NEXT: movsbq %cl, %rdi
+; AVX512VL-NEXT: movq -40(%rsp,%rdi), %rsi
+; AVX512VL-NEXT: movq -32(%rsp,%rdi), %rdx
+; AVX512VL-NEXT: movq -24(%rsp,%rdi), %r8
+; AVX512VL-NEXT: movq %r8, %r10
+; AVX512VL-NEXT: movl %r9d, %ecx
+; AVX512VL-NEXT: shlq %cl, %r10
+; AVX512VL-NEXT: movzbq %cl, %rcx
+; AVX512VL-NEXT: negq %rcx
+; AVX512VL-NEXT: shrxq %rcx, %rdx, %r11
+; AVX512VL-NEXT: orq %r10, %r11
+; AVX512VL-NEXT: movq -16(%rsp,%rdi), %rdi
+; AVX512VL-NEXT: movl %r9d, %ecx
+; AVX512VL-NEXT: movzbq %cl, %r10
+; AVX512VL-NEXT: negq %r10
+; AVX512VL-NEXT: shlq %cl, %rdi
+; AVX512VL-NEXT: shrxq %r10, %r8, %r8
+; AVX512VL-NEXT: orq %rdi, %r8
+; AVX512VL-NEXT: shlxq %r9, %rsi, %rdi
+; AVX512VL-NEXT: movzbq %cl, %r9
+; AVX512VL-NEXT: negq %r9
+; AVX512VL-NEXT: shlq %cl, %rdx
+; AVX512VL-NEXT: shrxq %r9, %rsi, %rcx
+; AVX512VL-NEXT: orq %rdx, %rcx
+; AVX512VL-NEXT: movq %r8, 24(%rax)
+; AVX512VL-NEXT: movq %r11, 16(%rax)
+; AVX512VL-NEXT: movq %rcx, 8(%rax)
+; AVX512VL-NEXT: movq %rdi, (%rax)
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VBMI-LABEL: shl_i256:
; AVX512VBMI: # %bb.0:
+; AVX512VBMI-NEXT: movq %rdi, %rax
; AVX512VBMI-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: movl %r9d, %eax
-; AVX512VBMI-NEXT: shrb $3, %al
-; AVX512VBMI-NEXT: andb $24, %al
-; AVX512VBMI-NEXT: negb %al
-; AVX512VBMI-NEXT: movsbq %al, %rax
-; AVX512VBMI-NEXT: movq -32(%rsp,%rax), %rdx
-; AVX512VBMI-NEXT: movq -24(%rsp,%rax), %rsi
-; AVX512VBMI-NEXT: movq %rsi, %r8
; AVX512VBMI-NEXT: movl %r9d, %ecx
-; AVX512VBMI-NEXT: shldq %cl, %rdx, %r8
-; AVX512VBMI-NEXT: movq -16(%rsp,%rax), %r10
-; AVX512VBMI-NEXT: shldq %cl, %rsi, %r10
-; AVX512VBMI-NEXT: movq -40(%rsp,%rax), %rsi
-; AVX512VBMI-NEXT: shldq %cl, %rsi, %rdx
-; AVX512VBMI-NEXT: movq %rdi, %rax
-; AVX512VBMI-NEXT: shlxq %r9, %rsi, %rcx
-; AVX512VBMI-NEXT: movq %r10, 24(%rdi)
-; AVX512VBMI-NEXT: movq %r8, 16(%rdi)
-; AVX512VBMI-NEXT: movq %rdx, 8(%rdi)
-; AVX512VBMI-NEXT: movq %rcx, (%rdi)
+; AVX512VBMI-NEXT: shrb $3, %cl
+; AVX512VBMI-NEXT: andb $24, %cl
+; AVX512VBMI-NEXT: negb %cl
+; AVX512VBMI-NEXT: movsbq %cl, %rdi
+; AVX512VBMI-NEXT: movq -40(%rsp,%rdi), %rsi
+; AVX512VBMI-NEXT: movq -32(%rsp,%rdi), %rdx
+; AVX512VBMI-NEXT: movq -24(%rsp,%rdi), %r8
+; AVX512VBMI-NEXT: movq %r8, %r10
+; AVX512VBMI-NEXT: movl %r9d, %ecx
+; AVX512VBMI-NEXT: shlq %cl, %r10
+; AVX512VBMI-NEXT: movzbq %cl, %rcx
+; AVX512VBMI-NEXT: negq %rcx
+; AVX512VBMI-NEXT: shrxq %rcx, %rdx, %r11
+; AVX512VBMI-NEXT: orq %r10, %r11
+; AVX512VBMI-NEXT: movq -16(%rsp,%rdi), %rdi
+; AVX512VBMI-NEXT: movl %r9d, %ecx
+; AVX512VBMI-NEXT: movzbq %cl, %r10
+; AVX512VBMI-NEXT: negq %r10
+; AVX512VBMI-NEXT: shlq %cl, %rdi
+; AVX512VBMI-NEXT: shrxq %r10, %r8, %r8
+; AVX512VBMI-NEXT: orq %rdi, %r8
+; AVX512VBMI-NEXT: shlxq %r9, %rsi, %rdi
+; AVX512VBMI-NEXT: movzbq %cl, %r9
+; AVX512VBMI-NEXT: negq %r9
+; AVX512VBMI-NEXT: shlq %cl, %rdx
+; AVX512VBMI-NEXT: shrxq %r9, %rsi, %rcx
+; AVX512VBMI-NEXT: orq %rdx, %rcx
+; AVX512VBMI-NEXT: movq %r8, 24(%rax)
+; AVX512VBMI-NEXT: movq %r11, 16(%rax)
+; AVX512VBMI-NEXT: movq %rcx, 8(%rax)
+; AVX512VBMI-NEXT: movq %rdi, (%rax)
; AVX512VBMI-NEXT: vzeroupper
; AVX512VBMI-NEXT: retq
;
@@ -286,116 +340,170 @@ define i256 @lshr_i256(i256 %a0, i256 %a1) nounwind {
;
; AVX2-LABEL: lshr_i256:
; AVX2: # %bb.0:
+; AVX2-NEXT: movq %rdi, %rax
; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movl %r9d, %eax
-; AVX2-NEXT: shrb $6, %al
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: movq -56(%rsp,%rax,8), %rdx
-; AVX2-NEXT: movq -64(%rsp,%rax,8), %rsi
-; AVX2-NEXT: movq %rsi, %r8
; AVX2-NEXT: movl %r9d, %ecx
-; AVX2-NEXT: shrdq %cl, %rdx, %r8
-; AVX2-NEXT: movq -72(%rsp,%rax,8), %r10
-; AVX2-NEXT: movq -48(%rsp,%rax,8), %r11
-; AVX2-NEXT: shrdq %cl, %r11, %rdx
-; AVX2-NEXT: shrdq %cl, %rsi, %r10
-; AVX2-NEXT: movq %rdi, %rax
-; AVX2-NEXT: shrxq %r9, %r11, %rcx
-; AVX2-NEXT: movq %rcx, 24(%rdi)
-; AVX2-NEXT: movq %rdx, 16(%rdi)
-; AVX2-NEXT: movq %r8, 8(%rdi)
-; AVX2-NEXT: movq %r10, (%rdi)
+; AVX2-NEXT: shrb $6, %cl
+; AVX2-NEXT: movzbl %cl, %edi
+; AVX2-NEXT: movq -56(%rsp,%rdi,8), %r8
+; AVX2-NEXT: movq -72(%rsp,%rdi,8), %rdx
+; AVX2-NEXT: movq -64(%rsp,%rdi,8), %rsi
+; AVX2-NEXT: movq %rsi, %r10
+; AVX2-NEXT: movl %r9d, %ecx
+; AVX2-NEXT: shrq %cl, %r10
+; AVX2-NEXT: movzbq %cl, %rcx
+; AVX2-NEXT: negq %rcx
+; AVX2-NEXT: shlxq %rcx, %r8, %r11
+; AVX2-NEXT: orq %r10, %r11
+; AVX2-NEXT: movq -48(%rsp,%rdi,8), %rdi
+; AVX2-NEXT: movl %r9d, %ecx
+; AVX2-NEXT: shrq %cl, %r8
+; AVX2-NEXT: movzbq %cl, %rcx
+; AVX2-NEXT: negq %rcx
+; AVX2-NEXT: shlxq %rcx, %rdi, %r10
+; AVX2-NEXT: orq %r8, %r10
+; AVX2-NEXT: movl %r9d, %ecx
+; AVX2-NEXT: movzbq %cl, %r8
+; AVX2-NEXT: shrq %cl, %rdx
+; AVX2-NEXT: negq %r8
+; AVX2-NEXT: shlxq %r8, %rsi, %rcx
+; AVX2-NEXT: orq %rdx, %rcx
+; AVX2-NEXT: shrxq %r9, %rdi, %rdx
+; AVX2-NEXT: movq %rdx, 24(%rax)
+; AVX2-NEXT: movq %r10, 16(%rax)
+; AVX2-NEXT: movq %r11, 8(%rax)
+; AVX2-NEXT: movq %rcx, (%rax)
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512F-LABEL: lshr_i256:
; AVX512F: # %bb.0:
+; AVX512F-NEXT: movq %rdi, %rax
; AVX512F-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX512F-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
-; AVX512F-NEXT: movl %r9d, %eax
-; AVX512F-NEXT: shrb $6, %al
-; AVX512F-NEXT: movzbl %al, %eax
-; AVX512F-NEXT: movq -56(%rsp,%rax,8), %rdx
-; AVX512F-NEXT: movq -64(%rsp,%rax,8), %rsi
-; AVX512F-NEXT: movq %rsi, %r8
; AVX512F-NEXT: movl %r9d, %ecx
-; AVX512F-NEXT: shrdq %cl, %rdx, %r8
-; AVX512F-NEXT: movq -72(%rsp,%rax,8), %r10
-; AVX512F-NEXT: movq -48(%rsp,%rax,8), %r11
-; AVX512F-NEXT: shrdq %cl, %r11, %rdx
-; AVX512F-NEXT: shrdq %cl, %rsi, %r10
-; AVX512F-NEXT: movq %rdi, %rax
-; AVX512F-NEXT: shrxq %r9, %r11, %rcx
-; AVX512F-NEXT: movq %rcx, 24(%rdi)
-; AVX512F-NEXT: movq %rdx, 16(%rdi)
-; AVX512F-NEXT: movq %r8, 8(%rdi)
-; AVX512F-NEXT: movq %r10, (%rdi)
+; AVX512F-NEXT: shrb $6, %cl
+; AVX512F-NEXT: movzbl %cl, %edi
+; AVX512F-NEXT: movq -56(%rsp,%rdi,8), %r8
+; AVX512F-NEXT: movq -72(%rsp,%rdi,8), %rdx
+; AVX512F-NEXT: movq -64(%rsp,%rdi,8), %rsi
+; AVX512F-NEXT: movq %rsi, %r10
+; AVX512F-NEXT: movl %r9d, %ecx
+; AVX512F-NEXT: shrq %cl, %r10
+; AVX512F-NEXT: movzbq %cl, %rcx
+; AVX512F-NEXT: negq %rcx
+; AVX512F-NEXT: shlxq %rcx, %r8, %r11
+; AVX512F-NEXT: orq %r10, %r11
+; AVX512F-NEXT: movq -48(%rsp,%rdi,8), %rdi
+; AVX512F-NEXT: movl %r9d, %ecx
+; AVX512F-NEXT: shrq %cl, %r8
+; AVX512F-NEXT: movzbq %cl, %rcx
+; AVX512F-NEXT: negq %rcx
+; AVX512F-NEXT: shlxq %rcx, %rdi, %r10
+; AVX512F-NEXT: orq %r8, %r10
+; AVX512F-NEXT: movl %r9d, %ecx
+; AVX512F-NEXT: movzbq %cl, %r8
+; AVX512F-NEXT: shrq %cl, %rdx
+; AVX512F-NEXT: negq %r8
+; AVX512F-NEXT: shlxq %r8, %rsi, %rcx
+; AVX512F-NEXT: orq %rdx, %rcx
+; AVX512F-NEXT: shrxq %r9, %rdi, %rdx
+; AVX512F-NEXT: movq %rdx, 24(%rax)
+; AVX512F-NEXT: movq %r10, 16(%rax)
+; AVX512F-NEXT: movq %r11, 8(%rax)
+; AVX512F-NEXT: movq %rcx, (%rax)
; AVX512F-NEXT: retq
;
; AVX512VL-LABEL: lshr_i256:
; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: movq %rdi, %rax
; AVX512VL-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: movl %r9d, %eax
-; AVX512VL-NEXT: shrb $6, %al
-; AVX512VL-NEXT: movzbl %al, %eax
-; AVX512VL-NEXT: movq -56(%rsp,%rax,8), %rdx
-; AVX512VL-NEXT: movq -64(%rsp,%rax,8), %rsi
-; AVX512VL-NEXT: movq %rsi, %r8
; AVX512VL-NEXT: movl %r9d, %ecx
-; AVX512VL-NEXT: shrdq %cl, %rdx, %r8
-; AVX512VL-NEXT: movq -48(%rsp,%rax,8), %r10
-; AVX512VL-NEXT: shrdq %cl, %r10, %rdx
-; AVX512VL-NEXT: movq -72(%rsp,%rax,8), %r11
-; AVX512VL-NEXT: shrdq %cl, %rsi, %r11
-; AVX512VL-NEXT: movq %rdi, %rax
-; AVX512VL-NEXT: shrxq %r9, %r10, %rcx
-; AVX512VL-NEXT: movq %rcx, 24(%rdi)
-; AVX512VL-NEXT: movq %rdx, 16(%rdi)
-; AVX512VL-NEXT: movq %r8, 8(%rdi)
-; AVX512VL-NEXT: movq %r11, (%rdi)
+; AVX512VL-NEXT: shrb $6, %cl
+; AVX512VL-NEXT: movzbl %cl, %esi
+; AVX512VL-NEXT: movq -56(%rsp,%rsi,8), %rdi
+; AVX512VL-NEXT: movq -72(%rsp,%rsi,8), %rdx
+; AVX512VL-NEXT: movq -64(%rsp,%rsi,8), %r8
+; AVX512VL-NEXT: movq %r8, %r10
+; AVX512VL-NEXT: movl %r9d, %ecx
+; AVX512VL-NEXT: shrq %cl, %r10
+; AVX512VL-NEXT: movzbq %cl, %rcx
+; AVX512VL-NEXT: negq %rcx
+; AVX512VL-NEXT: shlxq %rcx, %rdi, %r11
+; AVX512VL-NEXT: orq %r10, %r11
+; AVX512VL-NEXT: movq -48(%rsp,%rsi,8), %rsi
+; AVX512VL-NEXT: movl %r9d, %ecx
+; AVX512VL-NEXT: movzbq %cl, %r10
+; AVX512VL-NEXT: negq %r10
+; AVX512VL-NEXT: shrq %cl, %rdi
+; AVX512VL-NEXT: shlxq %r10, %rsi, %r10
+; AVX512VL-NEXT: orq %rdi, %r10
+; AVX512VL-NEXT: movzbq %cl, %rdi
+; AVX512VL-NEXT: negq %rdi
+; AVX512VL-NEXT: shrq %cl, %rdx
+; AVX512VL-NEXT: shlxq %rdi, %r8, %rcx
+; AVX512VL-NEXT: orq %rdx, %rcx
+; AVX512VL-NEXT: shrxq %r9, %rsi, %rdx
+; AVX512VL-NEXT: movq %rdx, 24(%rax)
+; AVX512VL-NEXT: movq %r10, 16(%rax)
+; AVX512VL-NEXT: movq %r11, 8(%rax)
+; AVX512VL-NEXT: movq %rcx, (%rax)
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VBMI-LABEL: lshr_i256:
; AVX512VBMI: # %bb.0:
+; AVX512VBMI-NEXT: movq %rdi, %rax
; AVX512VBMI-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: movl %r9d, %eax
-; AVX512VBMI-NEXT: shrb $6, %al
-; AVX512VBMI-NEXT: movzbl %al, %eax
-; AVX512VBMI-NEXT: movq -56(%rsp,%rax,8), %rdx
-; AVX512VBMI-NEXT: movq -64(%rsp,%rax,8), %rsi
-; AVX512VBMI-NEXT: movq %rsi, %r8
; AVX512VBMI-NEXT: movl %r9d, %ecx
-; AVX512VBMI-NEXT: shrdq %cl, %rdx, %r8
-; AVX512VBMI-NEXT: movq -48(%rsp,%rax,8), %r10
-; AVX512VBMI-NEXT: shrdq %cl, %r10, %rdx
-; AVX512VBMI-NEXT: movq -72(%rsp,%rax,8), %r11
-; AVX512VBMI-NEXT: shrdq %cl, %rsi, %r11
-; AVX512VBMI-NEXT: movq %rdi, %rax
-; AVX512VBMI-NEXT: shrxq %r9, %r10, %rcx
-; AVX512VBMI-NEXT: movq %rcx, 24(%rdi)
-; AVX512VBMI-NEXT: movq %rdx, 16(%rdi)
-; AVX512VBMI-NEXT: movq %r8, 8(%rdi)
-; AVX512VBMI-NEXT: movq %r11, (%rdi)
+; AVX512VBMI-NEXT: shrb $6, %cl
+; AVX512VBMI-NEXT: movzbl %cl, %esi
+; AVX512VBMI-NEXT: movq -56(%rsp,%rsi,8), %rdi
+; AVX512VBMI-NEXT: movq -72(%rsp,%rsi,8), %rdx
+; AVX512VBMI-NEXT: movq -64(%rsp,%rsi,8), %r8
+; AVX512VBMI-NEXT: movq %r8, %r10
+; AVX512VBMI-NEXT: movl %r9d, %ecx
+; AVX512VBMI-NEXT: shrq %cl, %r10
+; AVX512VBMI-NEXT: movzbq %cl, %rcx
+; AVX512VBMI-NEXT: negq %rcx
+; AVX512VBMI-NEXT: shlxq %rcx, %rdi, %r11
+; AVX512VBMI-NEXT: orq %r10, %r11
+; AVX512VBMI-NEXT: movq -48(%rsp,%rsi,8), %rsi
+; AVX512VBMI-NEXT: movl %r9d, %ecx
+; AVX512VBMI-NEXT: movzbq %cl, %r10
+; AVX512VBMI-NEXT: negq %r10
+; AVX512VBMI-NEXT: shrq %cl, %rdi
+; AVX512VBMI-NEXT: shlxq %r10, %rsi, %r10
+; AVX512VBMI-NEXT: orq %rdi, %r10
+; AVX512VBMI-NEXT: movzbq %cl, %rdi
+; AVX512VBMI-NEXT: negq %rdi
+; AVX512VBMI-NEXT: shrq %cl, %rdx
+; AVX512VBMI-NEXT: shlxq %rdi, %r8, %rcx
+; AVX512VBMI-NEXT: orq %rdx, %rcx
+; AVX512VBMI-NEXT: shrxq %r9, %rsi, %rdx
+; AVX512VBMI-NEXT: movq %rdx, 24(%rax)
+; AVX512VBMI-NEXT: movq %r10, 16(%rax)
+; AVX512VBMI-NEXT: movq %r11, 8(%rax)
+; AVX512VBMI-NEXT: movq %rcx, (%rax)
; AVX512VBMI-NEXT: vzeroupper
; AVX512VBMI-NEXT: retq
;
@@ -516,6 +624,7 @@ define i256 @ashr_i256(i256 %a0, i256 %a1) nounwind {
;
; AVX2-LABEL: ashr_i256:
; AVX2: # %bb.0:
+; AVX2-NEXT: movq %rdi, %rax
; AVX2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
@@ -525,28 +634,42 @@ define i256 @ashr_i256(i256 %a0, i256 %a1) nounwind {
; AVX2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movl %r9d, %eax
-; AVX2-NEXT: shrb $6, %al
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: movq -56(%rsp,%rax,8), %rdx
-; AVX2-NEXT: movq -64(%rsp,%rax,8), %rsi
-; AVX2-NEXT: movq %rsi, %r8
; AVX2-NEXT: movl %r9d, %ecx
-; AVX2-NEXT: shrdq %cl, %rdx, %r8
-; AVX2-NEXT: movq -72(%rsp,%rax,8), %r10
-; AVX2-NEXT: movq -48(%rsp,%rax,8), %r11
-; AVX2-NEXT: shrdq %cl, %r11, %rdx
-; AVX2-NEXT: shrdq %cl, %rsi, %r10
-; AVX2-NEXT: movq %rdi, %rax
-; AVX2-NEXT: sarxq %r9, %r11, %rcx
-; AVX2-NEXT: movq %rcx, 24(%rdi)
-; AVX2-NEXT: movq %rdx, 16(%rdi)
-; AVX2-NEXT: movq %r8, 8(%rdi)
-; AVX2-NEXT: movq %r10, (%rdi)
+; AVX2-NEXT: shrb $6, %cl
+; AVX2-NEXT: movzbl %cl, %edi
+; AVX2-NEXT: movq -56(%rsp,%rdi,8), %r8
+; AVX2-NEXT: movq -72(%rsp,%rdi,8), %rdx
+; AVX2-NEXT: movq -64(%rsp,%rdi,8), %rsi
+; AVX2-NEXT: movq %rsi, %r10
+; AVX2-NEXT: movl %r9d, %ecx
+; AVX2-NEXT: shrq %cl, %r10
+; AVX2-NEXT: movzbq %cl, %rcx
+; AVX2-NEXT: negq %rcx
+; AVX2-NEXT: shlxq %rcx, %r8, %r11
+; AVX2-NEXT: orq %r10, %r11
+; AVX2-NEXT: movq -48(%rsp,%rdi,8), %rdi
+; AVX2-NEXT: movl %r9d, %ecx
+; AVX2-NEXT: shrq %cl, %r8
+; AVX2-NEXT: movzbq %cl, %rcx
+; AVX2-NEXT: negq %rcx
+; AVX2-NEXT: shlxq %rcx, %rdi, %r10
+; AVX2-NEXT: orq %r8, %r10
+; AVX2-NEXT: movl %r9d, %ecx
+; AVX2-NEXT: movzbq %cl, %r8
+; AVX2-NEXT: shrq %cl, %rdx
+; AVX2-NEXT: negq %r8
+; AVX2-NEXT: shlxq %r8, %rsi, %rcx
+; AVX2-NEXT: orq %rdx, %rcx
+; AVX2-NEXT: sarxq %r9, %rdi, %rdx
+; AVX2-NEXT: movq %rdx, 24(%rax)
+; AVX2-NEXT: movq %r10, 16(%rax)
+; AVX2-NEXT: movq %r11, 8(%rax)
+; AVX2-NEXT: movq %rcx, (%rax)
; AVX2-NEXT: retq
;
; AVX512F-LABEL: ashr_i256:
; AVX512F: # %bb.0:
+; AVX512F-NEXT: movq %rdi, %rax
; AVX512F-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
@@ -556,28 +679,42 @@ define i256 @ashr_i256(i256 %a0, i256 %a1) nounwind {
; AVX512F-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
-; AVX512F-NEXT: movl %r9d, %eax
-; AVX512F-NEXT: shrb $6, %al
-; AVX512F-NEXT: movzbl %al, %eax
-; AVX512F-NEXT: movq -56(%rsp,%rax,8), %rdx
-; AVX512F-NEXT: movq -64(%rsp,%rax,8), %rsi
-; AVX512F-NEXT: movq %rsi, %r8
; AVX512F-NEXT: movl %r9d, %ecx
-; AVX512F-NEXT: shrdq %cl, %rdx, %r8
-; AVX512F-NEXT: movq -72(%rsp,%rax,8), %r10
-; AVX512F-NEXT: movq -48(%rsp,%rax,8), %r11
-; AVX512F-NEXT: shrdq %cl, %r11, %rdx
-; AVX512F-NEXT: shrdq %cl, %rsi, %r10
-; AVX512F-NEXT: movq %rdi, %rax
-; AVX512F-NEXT: sarxq %r9, %r11, %rcx
-; AVX512F-NEXT: movq %rcx, 24(%rdi)
-; AVX512F-NEXT: movq %rdx, 16(%rdi)
-; AVX512F-NEXT: movq %r8, 8(%rdi)
-; AVX512F-NEXT: movq %r10, (%rdi)
+; AVX512F-NEXT: shrb $6, %cl
+; AVX512F-NEXT: movzbl %cl, %edi
+; AVX512F-NEXT: movq -56(%rsp,%rdi,8), %r8
+; AVX512F-NEXT: movq -72(%rsp,%rdi,8), %rdx
+; AVX512F-NEXT: movq -64(%rsp,%rdi,8), %rsi
+; AVX512F-NEXT: movq %rsi, %r10
+; AVX512F-NEXT: movl %r9d, %ecx
+; AVX512F-NEXT: shrq %cl, %r10
+; AVX512F-NEXT: movzbq %cl, %rcx
+; AVX512F-NEXT: negq %rcx
+; AVX512F-NEXT: shlxq %rcx, %r8, %r11
+; AVX512F-NEXT: orq %r10, %r11
+; AVX512F-NEXT: movq -48(%rsp,%rdi,8), %rdi
+; AVX512F-NEXT: movl %r9d, %ecx
+; AVX512F-NEXT: shrq %cl, %r8
+; AVX512F-NEXT: movzbq %cl, %rcx
+; AVX512F-NEXT: negq %rcx
+; AVX512F-NEXT: shlxq %rcx, %rdi, %r10
+; AVX512F-NEXT: orq %r8, %r10
+; AVX512F-NEXT: movl %r9d, %ecx
+; AVX512F-NEXT: movzbq %cl, %r8
+; AVX512F-NEXT: shrq %cl, %rdx
+; AVX512F-NEXT: negq %r8
+; AVX512F-NEXT: shlxq %r8, %rsi, %rcx
+; AVX512F-NEXT: orq %rdx, %rcx
+; AVX512F-NEXT: sarxq %r9, %rdi, %rdx
+; AVX512F-NEXT: movq %rdx, 24(%rax)
+; AVX512F-NEXT: movq %r10, 16(%rax)
+; AVX512F-NEXT: movq %r11, 8(%rax)
+; AVX512F-NEXT: movq %rcx, (%rax)
; AVX512F-NEXT: retq
;
; AVX512VL-LABEL: ashr_i256:
; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: movq %rdi, %rax
; AVX512VL-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
@@ -587,28 +724,41 @@ define i256 @ashr_i256(i256 %a0, i256 %a1) nounwind {
; AVX512VL-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: movl %r9d, %eax
-; AVX512VL-NEXT: shrb $6, %al
-; AVX512VL-NEXT: movzbl %al, %eax
-; AVX512VL-NEXT: movq -56(%rsp,%rax,8), %rdx
-; AVX512VL-NEXT: movq -64(%rsp,%rax,8), %rsi
-; AVX512VL-NEXT: movq %rsi, %r8
; AVX512VL-NEXT: movl %r9d, %ecx
-; AVX512VL-NEXT: shrdq %cl, %rdx, %r8
-; AVX512VL-NEXT: movq -48(%rsp,%rax,8), %r10
-; AVX512VL-NEXT: shrdq %cl, %r10, %rdx
-; AVX512VL-NEXT: movq -72(%rsp,%rax,8), %r11
-; AVX512VL-NEXT: shrdq %cl, %rsi, %r11
-; AVX512VL-NEXT: movq %rdi, %rax
-; AVX512VL-NEXT: sarxq %r9, %r10, %rcx
-; AVX512VL-NEXT: movq %rcx, 24(%rdi)
-; AVX512VL-NEXT: movq %rdx, 16(%rdi)
-; AVX512VL-NEXT: movq %r8, 8(%rdi)
-; AVX512VL-NEXT: movq %r11, (%rdi)
+; AVX512VL-NEXT: shrb $6, %cl
+; AVX512VL-NEXT: movzbl %cl, %esi
+; AVX512VL-NEXT: movq -56(%rsp,%rsi,8), %rdi
+; AVX512VL-NEXT: movq -72(%rsp,%rsi,8), %rdx
+; AVX512VL-NEXT: movq -64(%rsp,%rsi,8), %r8
+; AVX512VL-NEXT: movq %r8, %r10
+; AVX512VL-NEXT: movl %r9d, %ecx
+; AVX512VL-NEXT: shrq %cl, %r10
+; AVX512VL-NEXT: movzbq %cl, %rcx
+; AVX512VL-NEXT: negq %rcx
+; AVX512VL-NEXT: shlxq %rcx, %rdi, %r11
+; AVX512VL-NEXT: orq %r10, %r11
+; AVX512VL-NEXT: movq -48(%rsp,%rsi,8), %rsi
+; AVX512VL-NEXT: movl %r9d, %ecx
+; AVX512VL-NEXT: movzbq %cl, %r10
+; AVX512VL-NEXT: negq %r10
+; AVX512VL-NEXT: shrq %cl, %rdi
+; AVX512VL-NEXT: shlxq %r10, %rsi, %r10
+; AVX512VL-NEXT: orq %rdi, %r10
+; AVX512VL-NEXT: movzbq %cl, %rdi
+; AVX512VL-NEXT: negq %rdi
+; AVX512VL-NEXT: shrq %cl, %rdx
+; AVX512VL-NEXT: shlxq %rdi, %r8, %rcx
+; AVX512VL-NEXT: orq %rdx, %rcx
+; AVX512VL-NEXT: sarxq %r9, %rsi, %rdx
+; AVX512VL-NEXT: movq %rdx, 24(%rax)
+; AVX512VL-NEXT: movq %r10, 16(%rax)
+; AVX512VL-NEXT: movq %r11, 8(%rax)
+; AVX512VL-NEXT: movq %rcx, (%rax)
; AVX512VL-NEXT: retq
;
; AVX512VBMI-LABEL: ashr_i256:
; AVX512VBMI: # %bb.0:
+; AVX512VBMI-NEXT: movq %rdi, %rax
; AVX512VBMI-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
@@ -618,24 +768,36 @@ define i256 @ashr_i256(i256 %a0, i256 %a1) nounwind {
; AVX512VBMI-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: movl %r9d, %eax
-; AVX512VBMI-NEXT: shrb $6, %al
-; AVX512VBMI-NEXT: movzbl %al, %eax
-; AVX512VBMI-NEXT: movq -56(%rsp,%rax,8), %rdx
-; AVX512VBMI-NEXT: movq -64(%rsp,%rax,8), %rsi
-; AVX512VBMI-NEXT: movq %rsi, %r8
; AVX512VBMI-NEXT: movl %r9d, %ecx
-; AVX512VBMI-NEXT: shrdq %cl, %rdx, %r8
-; AVX512VBMI-NEXT: movq -48(%rsp,%rax,8), %r10
-; AVX512VBMI-NEXT: shrdq %cl, %r10, %rdx
-; AVX512VBMI-NEXT: movq -72(%rsp,%rax,8), %r11
-; AVX512VBMI-NEXT: shrdq %cl, %rsi, %r11
-; AVX512VBMI-NEXT: movq %rdi, %rax
-; AVX512VBMI-NEXT: sarxq %r9, %r10, %rcx
-; AVX512VBMI-NEXT: movq %rcx, 24(%rdi)
-; AVX512VBMI-NEXT: movq %rdx, 16(%rdi)
-; AVX512VBMI-NEXT: movq %r8, 8(%rdi)
-; AVX512VBMI-NEXT: movq %r11, (%rdi)
+; AVX512VBMI-NEXT: shrb $6, %cl
+; AVX512VBMI-NEXT: movzbl %cl, %esi
+; AVX512VBMI-NEXT: movq -56(%rsp,%rsi,8), %rdi
+; AVX512VBMI-NEXT: movq -72(%rsp,%rsi,8), %rdx
+; AVX512VBMI-NEXT: movq -64(%rsp,%rsi,8), %r8
+; AVX512VBMI-NEXT: movq %r8, %r10
+; AVX512VBMI-NEXT: movl %r9d, %ecx
+; AVX512VBMI-NEXT: shrq %cl, %r10
+; AVX512VBMI-NEXT: movzbq %cl, %rcx
+; AVX512VBMI-NEXT: negq %rcx
+; AVX512VBMI-NEXT: shlxq %rcx, %rdi, %r11
+; AVX512VBMI-NEXT: orq %r10, %r11
+; AVX512VBMI-NEXT: movq -48(%rsp,%rsi,8), %rsi
+; AVX512VBMI-NEXT: movl %r9d, %ecx
+; AVX512VBMI-NEXT: movzbq %cl, %r10
+; AVX512VBMI-NEXT: negq %r10
+; AVX512VBMI-NEXT: shrq %cl, %rdi
+; AVX512VBMI-NEXT: shlxq %r10, %rsi, %r10
+; AVX512VBMI-NEXT: orq %rdi, %r10
+; AVX512VBMI-NEXT: movzbq %cl, %rdi
+; AVX512VBMI-NEXT: negq %rdi
+; AVX512VBMI-NEXT: shrq %cl, %rdx
+; AVX512VBMI-NEXT: shlxq %rdi, %r8, %rcx
+; AVX512VBMI-NEXT: orq %rdx, %rcx
+; AVX512VBMI-NEXT: sarxq %r9, %rsi, %rdx
+; AVX512VBMI-NEXT: movq %rdx, 24(%rax)
+; AVX512VBMI-NEXT: movq %r10, 16(%rax)
+; AVX512VBMI-NEXT: movq %r11, 8(%rax)
+; AVX512VBMI-NEXT: movq %rcx, (%rax)
; AVX512VBMI-NEXT: retq
;
; X86-LABEL: ashr_i256:
@@ -758,118 +920,170 @@ define i256 @shl_i256_load(ptr %p0, i256 %a1) nounwind {
;
; AVX2-LABEL: shl_i256_load:
; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %rbx
; AVX2-NEXT: movq %rdx, %rcx
+; AVX2-NEXT: movzbq %cl, %rax
; AVX2-NEXT: vmovups (%rsi), %ymm0
; AVX2-NEXT: vxorps %xmm1, %xmm1, %xmm1
; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movl %ecx, %eax
-; AVX2-NEXT: shrb $3, %al
-; AVX2-NEXT: andb $24, %al
-; AVX2-NEXT: negb %al
-; AVX2-NEXT: movsbq %al, %rdx
-; AVX2-NEXT: movq -32(%rsp,%rdx), %rsi
-; AVX2-NEXT: movq -24(%rsp,%rdx), %rax
-; AVX2-NEXT: movq %rax, %r8
-; AVX2-NEXT: shldq %cl, %rsi, %r8
-; AVX2-NEXT: movq -16(%rsp,%rdx), %r9
-; AVX2-NEXT: shldq %cl, %rax, %r9
+; AVX2-NEXT: shrb $3, %dl
+; AVX2-NEXT: andb $24, %dl
+; AVX2-NEXT: negb %dl
+; AVX2-NEXT: movsbq %dl, %rsi
+; AVX2-NEXT: movq -16(%rsp,%rsi), %rdx
+; AVX2-NEXT: movq %rdx, %r8
+; AVX2-NEXT: shlq %cl, %r8
+; AVX2-NEXT: movq -8(%rsp,%rsi), %r9
+; AVX2-NEXT: shlq %cl, %r9
+; AVX2-NEXT: movq -24(%rsp,%rsi), %r10
+; AVX2-NEXT: negq %rax
+; AVX2-NEXT: shrxq %rax, %r10, %r11
+; AVX2-NEXT: movzbq %cl, %rbx
+; AVX2-NEXT: shlq %cl, %r10
; AVX2-NEXT: movq %rdi, %rax
-; AVX2-NEXT: movq -40(%rsp,%rdx), %rdx
-; AVX2-NEXT: shlxq %rcx, %rdx, %rdi
-; AVX2-NEXT: # kill: def $cl killed $cl killed $rcx
-; AVX2-NEXT: shldq %cl, %rdx, %rsi
-; AVX2-NEXT: movq %r9, 24(%rax)
-; AVX2-NEXT: movq %r8, 16(%rax)
-; AVX2-NEXT: movq %rsi, 8(%rax)
+; AVX2-NEXT: orq %r8, %r11
+; AVX2-NEXT: movq -32(%rsp,%rsi), %rsi
+; AVX2-NEXT: negq %rbx
+; AVX2-NEXT: shrxq %rbx, %rdx, %rdx
+; AVX2-NEXT: orq %r9, %rdx
+; AVX2-NEXT: shlxq %rcx, %rsi, %rdi
+; AVX2-NEXT: movzbq %cl, %rcx
+; AVX2-NEXT: negq %rcx
+; AVX2-NEXT: shrxq %rcx, %rsi, %rcx
+; AVX2-NEXT: orq %r10, %rcx
+; AVX2-NEXT: movq %rdx, 24(%rax)
+; AVX2-NEXT: movq %r11, 16(%rax)
+; AVX2-NEXT: movq %rcx, 8(%rax)
; AVX2-NEXT: movq %rdi, (%rax)
+; AVX2-NEXT: popq %rbx
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512F-LABEL: shl_i256_load:
; AVX512F: # %bb.0:
+; AVX512F-NEXT: pushq %rbx
; AVX512F-NEXT: movq %rdx, %rcx
+; AVX512F-NEXT: movzbq %cl, %rax
; AVX512F-NEXT: vmovups (%rsi), %ymm0
; AVX512F-NEXT: vxorps %xmm1, %xmm1, %xmm1
; AVX512F-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512F-NEXT: movl %ecx, %eax
-; AVX512F-NEXT: shrb $3, %al
-; AVX512F-NEXT: andb $24, %al
-; AVX512F-NEXT: negb %al
-; AVX512F-NEXT: movsbq %al, %rdx
-; AVX512F-NEXT: movq -32(%rsp,%rdx), %rsi
-; AVX512F-NEXT: movq -24(%rsp,%rdx), %rax
-; AVX512F-NEXT: movq %rax, %r8
-; AVX512F-NEXT: shldq %cl, %rsi, %r8
-; AVX512F-NEXT: movq -16(%rsp,%rdx), %r9
-; AVX512F-NEXT: shldq %cl, %rax, %r9
+; AVX512F-NEXT: shrb $3, %dl
+; AVX512F-NEXT: andb $24, %dl
+; AVX512F-NEXT: negb %dl
+; AVX512F-NEXT: movsbq %dl, %rsi
+; AVX512F-NEXT: movq -16(%rsp,%rsi), %rdx
+; AVX512F-NEXT: movq %rdx, %r8
+; AVX512F-NEXT: shlq %cl, %r8
+; AVX512F-NEXT: movq -8(%rsp,%rsi), %r9
+; AVX512F-NEXT: shlq %cl, %r9
+; AVX512F-NEXT: movq -24(%rsp,%rsi), %r10
+; AVX512F-NEXT: negq %rax
+; AVX512F-NEXT: shrxq %rax, %r10, %r11
+; AVX512F-NEXT: movzbq %cl, %rbx
+; AVX512F-NEXT: shlq %cl, %r10
; AVX512F-NEXT: movq %rdi, %rax
-; AVX512F-NEXT: movq -40(%rsp,%rdx), %rdx
-; AVX512F-NEXT: shlxq %rcx, %rdx, %rdi
-; AVX512F-NEXT: # kill: def $cl killed $cl killed $rcx
-; AVX512F-NEXT: shldq %cl, %rdx, %rsi
-; AVX512F-NEXT: movq %r9, 24(%rax)
-; AVX512F-NEXT: movq %r8, 16(%rax)
-; AVX512F-NEXT: movq %rsi, 8(%rax)
+; AVX512F-NEXT: orq %r8, %r11
+; AVX512F-NEXT: movq -32(%rsp,%rsi), %rsi
+; AVX512F-NEXT: negq %rbx
+; AVX512F-NEXT: shrxq %rbx, %rdx, %rdx
+; AVX512F-NEXT: orq %r9, %rdx
+; AVX512F-NEXT: shlxq %rcx, %rsi, %rdi
+; AVX512F-NEXT: movzbq %cl, %rcx
+; AVX512F-NEXT: negq %rcx
+; AVX512F-NEXT: shrxq %rcx, %rsi, %rcx
+; AVX512F-NEXT: orq %r10, %rcx
+; AVX512F-NEXT: movq %rdx, 24(%rax)
+; AVX512F-NEXT: movq %r11, 16(%rax)
+; AVX512F-NEXT: movq %rcx, 8(%rax)
; AVX512F-NEXT: movq %rdi, (%rax)
+; AVX512F-NEXT: popq %rbx
; AVX512F-NEXT: retq
;
; AVX512VL-LABEL: shl_i256_load:
; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: pushq %rbx
; AVX512VL-NEXT: movq %rdx, %rcx
+; AVX512VL-NEXT: movq %rdi, %rax
; AVX512VL-NEXT: vmovups (%rsi), %ymm0
; AVX512VL-NEXT: vxorps %xmm1, %xmm1, %xmm1
; AVX512VL-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movzbq %cl, %rdx
; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: movl %ecx, %eax
-; AVX512VL-NEXT: shrb $3, %al
-; AVX512VL-NEXT: andb $24, %al
-; AVX512VL-NEXT: negb %al
-; AVX512VL-NEXT: movsbq %al, %rax
-; AVX512VL-NEXT: movq -32(%rsp,%rax), %rdx
-; AVX512VL-NEXT: movq -24(%rsp,%rax), %rsi
-; AVX512VL-NEXT: movq %rsi, %r8
-; AVX512VL-NEXT: shldq %cl, %rdx, %r8
-; AVX512VL-NEXT: movq -16(%rsp,%rax), %r9
-; AVX512VL-NEXT: shldq %cl, %rsi, %r9
-; AVX512VL-NEXT: movq -40(%rsp,%rax), %rsi
-; AVX512VL-NEXT: shldq %cl, %rsi, %rdx
-; AVX512VL-NEXT: movq %rdi, %rax
-; AVX512VL-NEXT: shlxq %rcx, %rsi, %rcx
-; AVX512VL-NEXT: movq %r9, 24(%rdi)
-; AVX512VL-NEXT: movq %r8, 16(%rdi)
-; AVX512VL-NEXT: movq %rdx, 8(%rdi)
-; AVX512VL-NEXT: movq %rcx, (%rdi)
+; AVX512VL-NEXT: movl %ecx, %esi
+; AVX512VL-NEXT: shrb $3, %sil
+; AVX512VL-NEXT: andb $24, %sil
+; AVX512VL-NEXT: negb %sil
+; AVX512VL-NEXT: movsbq %sil, %rsi
+; AVX512VL-NEXT: movq -16(%rsp,%rsi), %rdi
+; AVX512VL-NEXT: movq %rdi, %r8
+; AVX512VL-NEXT: shlq %cl, %r8
+; AVX512VL-NEXT: movzbq %cl, %r9
+; AVX512VL-NEXT: movq -8(%rsp,%rsi), %r10
+; AVX512VL-NEXT: shlq %cl, %r10
+; AVX512VL-NEXT: movzbq %cl, %r11
+; AVX512VL-NEXT: movq -24(%rsp,%rsi), %rbx
+; AVX512VL-NEXT: negq %rdx
+; AVX512VL-NEXT: shrxq %rdx, %rbx, %rdx
+; AVX512VL-NEXT: orq %r8, %rdx
+; AVX512VL-NEXT: negq %r9
+; AVX512VL-NEXT: shrxq %r9, %rdi, %rdi
+; AVX512VL-NEXT: movq -32(%rsp,%rsi), %rsi
+; AVX512VL-NEXT: orq %r10, %rdi
+; AVX512VL-NEXT: shlxq %rcx, %rsi, %r8
+; AVX512VL-NEXT: negq %r11
+; AVX512VL-NEXT: shlq %cl, %rbx
+; AVX512VL-NEXT: shrxq %r11, %rsi, %rcx
+; AVX512VL-NEXT: orq %rbx, %rcx
+; AVX512VL-NEXT: movq %rdi, 24(%rax)
+; AVX512VL-NEXT: movq %rdx, 16(%rax)
+; AVX512VL-NEXT: movq %rcx, 8(%rax)
+; AVX512VL-NEXT: movq %r8, (%rax)
+; AVX512VL-NEXT: popq %rbx
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VBMI-LABEL: shl_i256_load:
; AVX512VBMI: # %bb.0:
+; AVX512VBMI-NEXT: pushq %rbx
; AVX512VBMI-NEXT: movq %rdx, %rcx
+; AVX512VBMI-NEXT: movq %rdi, %rax
; AVX512VBMI-NEXT: vmovups (%rsi), %ymm0
; AVX512VBMI-NEXT: vxorps %xmm1, %xmm1, %xmm1
; AVX512VBMI-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movzbq %cl, %rdx
; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: movl %ecx, %eax
-; AVX512VBMI-NEXT: shrb $3, %al
-; AVX512VBMI-NEXT: andb $24, %al
-; AVX512VBMI-NEXT: negb %al
-; AVX512VBMI-NEXT: movsbq %al, %rax
-; AVX512VBMI-NEXT: movq -32(%rsp,%rax), %rdx
-; AVX512VBMI-NEXT: movq -24(%rsp,%rax), %rsi
-; AVX512VBMI-NEXT: movq %rsi, %r8
-; AVX512VBMI-NEXT: shldq %cl, %rdx, %r8
-; AVX512VBMI-NEXT: movq -16(%rsp,%rax), %r9
-; AVX512VBMI-NEXT: shldq %cl, %rsi, %r9
-; AVX512VBMI-NEXT: movq -40(%rsp,%rax), %rsi
-; AVX512VBMI-NEXT: shldq %cl, %rsi, %rdx
-; AVX512VBMI-NEXT: movq %rdi, %rax
-; AVX512VBMI-NEXT: shlxq %rcx, %rsi, %rcx
-; AVX512VBMI-NEXT: movq %r9, 24(%rdi)
-; AVX512VBMI-NEXT: movq %r8, 16(%rdi)
-; AVX512VBMI-NEXT: movq %rdx, 8(%rdi)
-; AVX512VBMI-NEXT: movq %rcx, (%rdi)
+; AVX512VBMI-NEXT: movl %ecx, %esi
+; AVX512VBMI-NEXT: shrb $3, %sil
+; AVX512VBMI-NEXT: andb $24, %sil
+; AVX512VBMI-NEXT: negb %sil
+; AVX512VBMI-NEXT: movsbq %sil, %rsi
+; AVX512VBMI-NEXT: movq -16(%rsp,%rsi), %rdi
+; AVX512VBMI-NEXT: movq %rdi, %r8
+; AVX512VBMI-NEXT: shlq %cl, %r8
+; AVX512VBMI-NEXT: movzbq %cl, %r9
+; AVX512VBMI-NEXT: movq -8(%rsp,%rsi), %r10
+; AVX512VBMI-NEXT: shlq %cl, %r10
+; AVX512VBMI-NEXT: movzbq %cl, %r11
+; AVX512VBMI-NEXT: movq -24(%rsp,%rsi), %rbx
+; AVX512VBMI-NEXT: negq %rdx
+; AVX512VBMI-NEXT: shrxq %rdx, %rbx, %rdx
+; AVX512VBMI-NEXT: orq %r8, %rdx
+; AVX512VBMI-NEXT: negq %r9
+; AVX512VBMI-NEXT: shrxq %r9, %rdi, %rdi
+; AVX512VBMI-NEXT: movq -32(%rsp,%rsi), %rsi
+; AVX512VBMI-NEXT: orq %r10, %rdi
+; AVX512VBMI-NEXT: shlxq %rcx, %rsi, %r8
+; AVX512VBMI-NEXT: negq %r11
+; AVX512VBMI-NEXT: shlq %cl, %rbx
+; AVX512VBMI-NEXT: shrxq %r11, %rsi, %rcx
+; AVX512VBMI-NEXT: orq %rbx, %rcx
+; AVX512VBMI-NEXT: movq %rdi, 24(%rax)
+; AVX512VBMI-NEXT: movq %rdx, 16(%rax)
+; AVX512VBMI-NEXT: movq %rcx, 8(%rax)
+; AVX512VBMI-NEXT: movq %r8, (%rax)
+; AVX512VBMI-NEXT: popq %rbx
; AVX512VBMI-NEXT: vzeroupper
; AVX512VBMI-NEXT: retq
;
@@ -1004,107 +1218,151 @@ define i256 @lshr_i256_load(ptr %p0, i256 %a1) nounwind {
; AVX2-LABEL: lshr_i256_load:
; AVX2: # %bb.0:
; AVX2-NEXT: movq %rdx, %rcx
+; AVX2-NEXT: movzbq %cl, %rax
; AVX2-NEXT: vmovups (%rsi), %ymm0
; AVX2-NEXT: vxorps %xmm1, %xmm1, %xmm1
; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movl %ecx, %eax
-; AVX2-NEXT: shrb $6, %al
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: movq -56(%rsp,%rax,8), %rdx
-; AVX2-NEXT: movq -64(%rsp,%rax,8), %rsi
-; AVX2-NEXT: movq %rsi, %r8
-; AVX2-NEXT: shrdq %cl, %rdx, %r8
-; AVX2-NEXT: movq -72(%rsp,%rax,8), %r9
-; AVX2-NEXT: movq -48(%rsp,%rax,8), %r10
-; AVX2-NEXT: shrdq %cl, %r10, %rdx
-; AVX2-NEXT: shrdq %cl, %rsi, %r9
+; AVX2-NEXT: shrb $6, %dl
+; AVX2-NEXT: movzbl %dl, %r8d
+; AVX2-NEXT: movq -56(%rsp,%r8,8), %r9
+; AVX2-NEXT: movq -64(%rsp,%r8,8), %rsi
+; AVX2-NEXT: negq %rax
+; AVX2-NEXT: movq %rsi, %r10
+; AVX2-NEXT: shrq %cl, %r10
+; AVX2-NEXT: shlxq %rax, %r9, %rdx
+; AVX2-NEXT: shrq %cl, %r9
; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: movzbq %cl, %rdi
+; AVX2-NEXT: movzbq %cl, %r11
+; AVX2-NEXT: orq %r10, %rdx
+; AVX2-NEXT: movq -48(%rsp,%r8,8), %r10
+; AVX2-NEXT: negq %rdi
+; AVX2-NEXT: shlxq %rdi, %r10, %rdi
+; AVX2-NEXT: orq %r9, %rdi
+; AVX2-NEXT: movq -72(%rsp,%r8,8), %r8
+; AVX2-NEXT: shrq %cl, %r8
+; AVX2-NEXT: negq %r11
+; AVX2-NEXT: shlxq %r11, %rsi, %rsi
+; AVX2-NEXT: orq %r8, %rsi
; AVX2-NEXT: shrxq %rcx, %r10, %rcx
-; AVX2-NEXT: movq %rcx, 24(%rdi)
-; AVX2-NEXT: movq %rdx, 16(%rdi)
-; AVX2-NEXT: movq %r8, 8(%rdi)
-; AVX2-NEXT: movq %r9, (%rdi)
+; AVX2-NEXT: movq %rcx, 24(%rax)
+; AVX2-NEXT: movq %rdi, 16(%rax)
+; AVX2-NEXT: movq %rdx, 8(%rax)
+; AVX2-NEXT: movq %rsi, (%rax)
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512F-LABEL: lshr_i256_load:
; AVX512F: # %bb.0:
; AVX512F-NEXT: movq %rdx, %rcx
+; AVX512F-NEXT: movzbq %cl, %rax
; AVX512F-NEXT: vmovups (%rsi), %ymm0
; AVX512F-NEXT: vxorps %xmm1, %xmm1, %xmm1
; AVX512F-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512F-NEXT: movl %ecx, %eax
-; AVX512F-NEXT: shrb $6, %al
-; AVX512F-NEXT: movzbl %al, %eax
-; AVX512F-NEXT: movq -56(%rsp,%rax,8), %rdx
-; AVX512F-NEXT: movq -64(%rsp,%rax,8), %rsi
-; AVX512F-NEXT: movq %rsi, %r8
-; AVX512F-NEXT: shrdq %cl, %rdx, %r8
-; AVX512F-NEXT: movq -72(%rsp,%rax,8), %r9
-; AVX512F-NEXT: movq -48(%rsp,%rax,8), %r10
-; AVX512F-NEXT: shrdq %cl, %r10, %rdx
-; AVX512F-NEXT: shrdq %cl, %rsi, %r9
+; AVX512F-NEXT: shrb $6, %dl
+; AVX512F-NEXT: movzbl %dl, %r8d
+; AVX512F-NEXT: movq -56(%rsp,%r8,8), %r9
+; AVX512F-NEXT: movq -64(%rsp,%r8,8), %rsi
+; AVX512F-NEXT: negq %rax
+; AVX512F-NEXT: movq %rsi, %r10
+; AVX512F-NEXT: shrq %cl, %r10
+; AVX512F-NEXT: shlxq %rax, %r9, %rdx
+; AVX512F-NEXT: shrq %cl, %r9
; AVX512F-NEXT: movq %rdi, %rax
+; AVX512F-NEXT: movzbq %cl, %rdi
+; AVX512F-NEXT: movzbq %cl, %r11
+; AVX512F-NEXT: orq %r10, %rdx
+; AVX512F-NEXT: movq -48(%rsp,%r8,8), %r10
+; AVX512F-NEXT: negq %rdi
+; AVX512F-NEXT: shlxq %rdi, %r10, %rdi
+; AVX512F-NEXT: orq %r9, %rdi
+; AVX512F-NEXT: movq -72(%rsp,%r8,8), %r8
+; AVX512F-NEXT: shrq %cl, %r8
+; AVX512F-NEXT: negq %r11
+; AVX512F-NEXT: shlxq %r11, %rsi, %rsi
+; AVX512F-NEXT: orq %r8, %rsi
; AVX512F-NEXT: shrxq %rcx, %r10, %rcx
-; AVX512F-NEXT: movq %rcx, 24(%rdi)
-; AVX512F-NEXT: movq %rdx, 16(%rdi)
-; AVX512F-NEXT: movq %r8, 8(%rdi)
-; AVX512F-NEXT: movq %r9, (%rdi)
+; AVX512F-NEXT: movq %rcx, 24(%rax)
+; AVX512F-NEXT: movq %rdi, 16(%rax)
+; AVX512F-NEXT: movq %rdx, 8(%rax)
+; AVX512F-NEXT: movq %rsi, (%rax)
; AVX512F-NEXT: retq
;
; AVX512VL-LABEL: lshr_i256_load:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: movq %rdx, %rcx
+; AVX512VL-NEXT: movq %rdi, %rax
+; AVX512VL-NEXT: movzbq %cl, %rdi
; AVX512VL-NEXT: vmovups (%rsi), %ymm0
; AVX512VL-NEXT: vxorps %xmm1, %xmm1, %xmm1
; AVX512VL-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: movl %ecx, %eax
-; AVX512VL-NEXT: shrb $6, %al
-; AVX512VL-NEXT: movzbl %al, %eax
-; AVX512VL-NEXT: movq -56(%rsp,%rax,8), %rdx
-; AVX512VL-NEXT: movq -64(%rsp,%rax,8), %rsi
-; AVX512VL-NEXT: movq %rsi, %r8
-; AVX512VL-NEXT: shrdq %cl, %rdx, %r8
-; AVX512VL-NEXT: movq -48(%rsp,%rax,8), %r9
-; AVX512VL-NEXT: shrdq %cl, %r9, %rdx
-; AVX512VL-NEXT: movq -72(%rsp,%rax,8), %r10
-; AVX512VL-NEXT: shrdq %cl, %rsi, %r10
-; AVX512VL-NEXT: movq %rdi, %rax
-; AVX512VL-NEXT: shrxq %rcx, %r9, %rcx
-; AVX512VL-NEXT: movq %rcx, 24(%rdi)
-; AVX512VL-NEXT: movq %rdx, 16(%rdi)
-; AVX512VL-NEXT: movq %r8, 8(%rdi)
-; AVX512VL-NEXT: movq %r10, (%rdi)
+; AVX512VL-NEXT: shrb $6, %dl
+; AVX512VL-NEXT: movzbl %dl, %esi
+; AVX512VL-NEXT: movq -64(%rsp,%rsi,8), %rdx
+; AVX512VL-NEXT: movq %rdx, %r8
+; AVX512VL-NEXT: shrq %cl, %r8
+; AVX512VL-NEXT: movzbq %cl, %r9
+; AVX512VL-NEXT: movq -56(%rsp,%rsi,8), %r10
+; AVX512VL-NEXT: negq %rdi
+; AVX512VL-NEXT: shlxq %rdi, %r10, %rdi
+; AVX512VL-NEXT: shrq %cl, %r10
+; AVX512VL-NEXT: movzbq %cl, %r11
+; AVX512VL-NEXT: orq %r8, %rdi
+; AVX512VL-NEXT: movq -48(%rsp,%rsi,8), %r8
+; AVX512VL-NEXT: negq %r9
+; AVX512VL-NEXT: shlxq %r9, %r8, %r9
+; AVX512VL-NEXT: orq %r10, %r9
+; AVX512VL-NEXT: movq -72(%rsp,%rsi,8), %rsi
+; AVX512VL-NEXT: negq %r11
+; AVX512VL-NEXT: shrq %cl, %rsi
+; AVX512VL-NEXT: shlxq %r11, %rdx, %rdx
+; AVX512VL-NEXT: orq %rsi, %rdx
+; AVX512VL-NEXT: shrxq %rcx, %r8, %rcx
+; AVX512VL-NEXT: movq %rcx, 24(%rax)
+; AVX512VL-NEXT: movq %r9, 16(%rax)
+; AVX512VL-NEXT: movq %rdi, 8(%rax)
+; AVX512VL-NEXT: movq %rdx, (%rax)
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VBMI-LABEL: lshr_i256_load:
; AVX512VBMI: # %bb.0:
; AVX512VBMI-NEXT: movq %rdx, %rcx
+; AVX512VBMI-NEXT: movq %rdi, %rax
+; AVX512VBMI-NEXT: movzbq %cl, %rdi
; AVX512VBMI-NEXT: vmovups (%rsi), %ymm0
; AVX512VBMI-NEXT: vxorps %xmm1, %xmm1, %xmm1
; AVX512VBMI-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: movl %ecx, %eax
-; AVX512VBMI-NEXT: shrb $6, %al
-; AVX512VBMI-NEXT: movzbl %al, %eax
-; AVX512VBMI-NEXT: movq -56(%rsp,%rax,8), %rdx
-; AVX512VBMI-NEXT: movq -64(%rsp,%rax,8), %rsi
-; AVX512VBMI-NEXT: movq %rsi, %r8
-; AVX512VBMI-NEXT: shrdq %cl, %rdx, %r8
-; AVX512VBMI-NEXT: movq -48(%rsp,%rax,8), %r9
-; AVX512VBMI-NEXT: shrdq %cl, %r9, %rdx
-; AVX512VBMI-NEXT: movq -72(%rsp,%rax,8), %r10
-; AVX512VBMI-NEXT: shrdq %cl, %rsi, %r10
-; AVX512VBMI-NEXT: movq %rdi, %rax
-; AVX512VBMI-NEXT: shrxq %rcx, %r9, %rcx
-; AVX512VBMI-NEXT: movq %rcx, 24(%rdi)
-; AVX512VBMI-NEXT: movq %rdx, 16(%rdi)
-; AVX512VBMI-NEXT: movq %r8, 8(%rdi)
-; AVX512VBMI-NEXT: movq %r10, (%rdi)
+; AVX512VBMI-NEXT: shrb $6, %dl
+; AVX512VBMI-NEXT: movzbl %dl, %esi
+; AVX512VBMI-NEXT: movq -64(%rsp,%rsi,8), %rdx
+; AVX512VBMI-NEXT: movq %rdx, %r8
+; AVX512VBMI-NEXT: shrq %cl, %r8
+; AVX512VBMI-NEXT: movzbq %cl, %r9
+; AVX512VBMI-NEXT: movq -56(%rsp,%rsi,8), %r10
+; AVX512VBMI-NEXT: negq %rdi
+; AVX512VBMI-NEXT: shlxq %rdi, %r10, %rdi
+; AVX512VBMI-NEXT: shrq %cl, %r10
+; AVX512VBMI-NEXT: movzbq %cl, %r11
+; AVX512VBMI-NEXT: orq %r8, %rdi
+; AVX512VBMI-NEXT: movq -48(%rsp,%rsi,8), %r8
+; AVX512VBMI-NEXT: negq %r9
+; AVX512VBMI-NEXT: shlxq %r9, %r8, %r9
+; AVX512VBMI-NEXT: orq %r10, %r9
+; AVX512VBMI-NEXT: movq -72(%rsp,%rsi,8), %rsi
+; AVX512VBMI-NEXT: negq %r11
+; AVX512VBMI-NEXT: shrq %cl, %rsi
+; AVX512VBMI-NEXT: shlxq %r11, %rdx, %rdx
+; AVX512VBMI-NEXT: orq %rsi, %rdx
+; AVX512VBMI-NEXT: shrxq %rcx, %r8, %rcx
+; AVX512VBMI-NEXT: movq %rcx, 24(%rax)
+; AVX512VBMI-NEXT: movq %r9, 16(%rax)
+; AVX512VBMI-NEXT: movq %rdi, 8(%rax)
+; AVX512VBMI-NEXT: movq %rdx, (%rax)
; AVX512VBMI-NEXT: vzeroupper
; AVX512VBMI-NEXT: retq
;
@@ -1235,133 +1493,181 @@ define i256 @ashr_i256_load(ptr %p0, i256 %a1) nounwind {
; AVX2-LABEL: ashr_i256_load:
; AVX2: # %bb.0:
; AVX2-NEXT: movq %rdx, %rcx
+; AVX2-NEXT: movzbq %cl, %rax
; AVX2-NEXT: vmovaps (%rsi), %xmm0
-; AVX2-NEXT: movq 16(%rsi), %rax
-; AVX2-NEXT: movq 24(%rsi), %rdx
+; AVX2-NEXT: movq 16(%rsi), %rdx
+; AVX2-NEXT: movq 24(%rsi), %rsi
+; AVX2-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: sarq $63, %rdx
-; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movl %ecx, %eax
-; AVX2-NEXT: shrb $6, %al
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: movq -56(%rsp,%rax,8), %rdx
-; AVX2-NEXT: movq -64(%rsp,%rax,8), %rsi
-; AVX2-NEXT: movq %rsi, %r8
-; AVX2-NEXT: shrdq %cl, %rdx, %r8
-; AVX2-NEXT: movq -72(%rsp,%rax,8), %r9
-; AVX2-NEXT: movq -48(%rsp,%rax,8), %r10
-; AVX2-NEXT: shrdq %cl, %r10, %rdx
-; AVX2-NEXT: shrdq %cl, %rsi, %r9
+; AVX2-NEXT: sarq $63, %rsi
+; AVX2-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movl %ecx, %edx
+; AVX2-NEXT: shrb $6, %dl
+; AVX2-NEXT: movzbl %dl, %r8d
+; AVX2-NEXT: movq -56(%rsp,%r8,8), %r9
+; AVX2-NEXT: movq -64(%rsp,%r8,8), %rsi
+; AVX2-NEXT: negq %rax
+; AVX2-NEXT: movq %rsi, %r10
+; AVX2-NEXT: shrq %cl, %r10
+; AVX2-NEXT: shlxq %rax, %r9, %rdx
+; AVX2-NEXT: shrq %cl, %r9
; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: movzbq %cl, %rdi
+; AVX2-NEXT: movzbq %cl, %r11
+; AVX2-NEXT: orq %r10, %rdx
+; AVX2-NEXT: movq -48(%rsp,%r8,8), %r10
+; AVX2-NEXT: negq %rdi
+; AVX2-NEXT: shlxq %rdi, %r10, %rdi
+; AVX2-NEXT: orq %r9, %rdi
+; AVX2-NEXT: movq -72(%rsp,%r8,8), %r8
+; AVX2-NEXT: shrq %cl, %r8
+; AVX2-NEXT: negq %r11
+; AVX2-NEXT: shlxq %r11, %rsi, %rsi
+; AVX2-NEXT: orq %r8, %rsi
; AVX2-NEXT: sarxq %rcx, %r10, %rcx
-; AVX2-NEXT: movq %rcx, 24(%rdi)
-; AVX2-NEXT: movq %rdx, 16(%rdi)
-; AVX2-NEXT: movq %r8, 8(%rdi)
-; AVX2-NEXT: movq %r9, (%rdi)
+; AVX2-NEXT: movq %rcx, 24(%rax)
+; AVX2-NEXT: movq %rdi, 16(%rax)
+; AVX2-NEXT: movq %rdx, 8(%rax)
+; AVX2-NEXT: movq %rsi, (%rax)
; AVX2-NEXT: retq
;
; AVX512F-LABEL: ashr_i256_load:
; AVX512F: # %bb.0:
; AVX512F-NEXT: movq %rdx, %rcx
+; AVX512F-NEXT: movzbq %cl, %rax
; AVX512F-NEXT: vmovaps (%rsi), %xmm0
-; AVX512F-NEXT: movq 16(%rsi), %rax
-; AVX512F-NEXT: movq 24(%rsi), %rdx
+; AVX512F-NEXT: movq 16(%rsi), %rdx
+; AVX512F-NEXT: movq 24(%rsi), %rsi
+; AVX512F-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; AVX512F-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
-; AVX512F-NEXT: sarq $63, %rdx
-; AVX512F-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; AVX512F-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; AVX512F-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; AVX512F-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; AVX512F-NEXT: movl %ecx, %eax
-; AVX512F-NEXT: shrb $6, %al
-; AVX512F-NEXT: movzbl %al, %eax
-; AVX512F-NEXT: movq -56(%rsp,%rax,8), %rdx
-; AVX512F-NEXT: movq -64(%rsp,%rax,8), %rsi
-; AVX512F-NEXT: movq %rsi, %r8
-; AVX512F-NEXT: shrdq %cl, %rdx, %r8
-; AVX512F-NEXT: movq -72(%rsp,%rax,8), %r9
-; AVX512F-NEXT: movq -48(%rsp,%rax,8), %r10
-; AVX512F-NEXT: shrdq %cl, %r10, %rdx
-; AVX512F-NEXT: shrdq %cl, %rsi, %r9
+; AVX512F-NEXT: sarq $63, %rsi
+; AVX512F-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movl %ecx, %edx
+; AVX512F-NEXT: shrb $6, %dl
+; AVX512F-NEXT: movzbl %dl, %r8d
+; AVX512F-NEXT: movq -56(%rsp,%r8,8), %r9
+; AVX512F-NEXT: movq -64(%rsp,%r8,8), %rsi
+; AVX512F-NEXT: negq %rax
+; AVX512F-NEXT: movq %rsi, %r10
+; AVX512F-NEXT: shrq %cl, %r10
+; AVX512F-NEXT: shlxq %rax, %r9, %rdx
+; AVX512F-NEXT: shrq %cl, %r9
; AVX512F-NEXT: movq %rdi, %rax
+; AVX512F-NEXT: movzbq %cl, %rdi
+; AVX512F-NEXT: movzbq %cl, %r11
+; AVX512F-NEXT: orq %r10, %rdx
+; AVX512F-NEXT: movq -48(%rsp,%r8,8), %r10
+; AVX512F-NEXT: negq %rdi
+; AVX512F-NEXT: shlxq %rdi, %r10, %rdi
+; AVX512F-NEXT: orq %r9, %rdi
+; AVX512F-NEXT: movq -72(%rsp,%r8,8), %r8
+; AVX512F-NEXT: shrq %cl, %r8
+; AVX512F-NEXT: negq %r11
+; AVX512F-NEXT: shlxq %r11, %rsi, %rsi
+; AVX512F-NEXT: orq %r8, %rsi
; AVX512F-NEXT: sarxq %rcx, %r10, %rcx
-; AVX512F-NEXT: movq %rcx, 24(%rdi)
-; AVX512F-NEXT: movq %rdx, 16(%rdi)
-; AVX512F-NEXT: movq %r8, 8(%rdi)
-; AVX512F-NEXT: movq %r9, (%rdi)
+; AVX512F-NEXT: movq %rcx, 24(%rax)
+; AVX512F-NEXT: movq %rdi, 16(%rax)
+; AVX512F-NEXT: movq %rdx, 8(%rax)
+; AVX512F-NEXT: movq %rsi, (%rax)
; AVX512F-NEXT: retq
;
; AVX512VL-LABEL: ashr_i256_load:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: movq %rdx, %rcx
+; AVX512VL-NEXT: movq %rdi, %rax
+; AVX512VL-NEXT: movzbq %cl, %rdi
; AVX512VL-NEXT: vmovaps (%rsi), %xmm0
-; AVX512VL-NEXT: movq 16(%rsi), %rax
-; AVX512VL-NEXT: movq 24(%rsi), %rdx
+; AVX512VL-NEXT: movq 16(%rsi), %rdx
+; AVX512VL-NEXT: movq 24(%rsi), %rsi
+; AVX512VL-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: sarq $63, %rdx
-; AVX512VL-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: movl %ecx, %eax
-; AVX512VL-NEXT: shrb $6, %al
-; AVX512VL-NEXT: movzbl %al, %eax
-; AVX512VL-NEXT: movq -56(%rsp,%rax,8), %rdx
-; AVX512VL-NEXT: movq -64(%rsp,%rax,8), %rsi
-; AVX512VL-NEXT: movq %rsi, %r8
-; AVX512VL-NEXT: shrdq %cl, %rdx, %r8
-; AVX512VL-NEXT: movq -48(%rsp,%rax,8), %r9
-; AVX512VL-NEXT: shrdq %cl, %r9, %rdx
-; AVX512VL-NEXT: movq -72(%rsp,%rax,8), %r10
-; AVX512VL-NEXT: shrdq %cl, %rsi, %r10
-; AVX512VL-NEXT: movq %rdi, %rax
-; AVX512VL-NEXT: sarxq %rcx, %r9, %rcx
-; AVX512VL-NEXT: movq %rcx, 24(%rdi)
-; AVX512VL-NEXT: movq %rdx, 16(%rdi)
-; AVX512VL-NEXT: movq %r8, 8(%rdi)
-; AVX512VL-NEXT: movq %r10, (%rdi)
+; AVX512VL-NEXT: sarq $63, %rsi
+; AVX512VL-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movl %ecx, %edx
+; AVX512VL-NEXT: shrb $6, %dl
+; AVX512VL-NEXT: movzbl %dl, %esi
+; AVX512VL-NEXT: movq -64(%rsp,%rsi,8), %rdx
+; AVX512VL-NEXT: movq %rdx, %r8
+; AVX512VL-NEXT: shrq %cl, %r8
+; AVX512VL-NEXT: movzbq %cl, %r9
+; AVX512VL-NEXT: movq -56(%rsp,%rsi,8), %r10
+; AVX512VL-NEXT: negq %rdi
+; AVX512VL-NEXT: shlxq %rdi, %r10, %rdi
+; AVX512VL-NEXT: shrq %cl, %r10
+; AVX512VL-NEXT: movzbq %cl, %r11
+; AVX512VL-NEXT: orq %r8, %rdi
+; AVX512VL-NEXT: movq -48(%rsp,%rsi,8), %r8
+; AVX512VL-NEXT: negq %r9
+; AVX512VL-NEXT: shlxq %r9, %r8, %r9
+; AVX512VL-NEXT: orq %r10, %r9
+; AVX512VL-NEXT: movq -72(%rsp,%rsi,8), %rsi
+; AVX512VL-NEXT: negq %r11
+; AVX512VL-NEXT: shrq %cl, %rsi
+; AVX512VL-NEXT: shlxq %r11, %rdx, %rdx
+; AVX512VL-NEXT: orq %rsi, %rdx
+; AVX512VL-NEXT: sarxq %rcx, %r8, %rcx
+; AVX512VL-NEXT: movq %rcx, 24(%rax)
+; AVX512VL-NEXT: movq %r9, 16(%rax)
+; AVX512VL-NEXT: movq %rdi, 8(%rax)
+; AVX512VL-NEXT: movq %rdx, (%rax)
; AVX512VL-NEXT: retq
;
; AVX512VBMI-LABEL: ashr_i256_load:
; AVX512VBMI: # %bb.0:
; AVX512VBMI-NEXT: movq %rdx, %rcx
+; AVX512VBMI-NEXT: movq %rdi, %rax
+; AVX512VBMI-NEXT: movzbq %cl, %rdi
; AVX512VBMI-NEXT: vmovaps (%rsi), %xmm0
-; AVX512VBMI-NEXT: movq 16(%rsi), %rax
-; AVX512VBMI-NEXT: movq 24(%rsi), %rdx
+; AVX512VBMI-NEXT: movq 16(%rsi), %rdx
+; AVX512VBMI-NEXT: movq 24(%rsi), %rsi
+; AVX512VBMI-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: sarq $63, %rdx
-; AVX512VBMI-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: movl %ecx, %eax
-; AVX512VBMI-NEXT: shrb $6, %al
-; AVX512VBMI-NEXT: movzbl %al, %eax
-; AVX512VBMI-NEXT: movq -56(%rsp,%rax,8), %rdx
-; AVX512VBMI-NEXT: movq -64(%rsp,%rax,8), %rsi
-; AVX512VBMI-NEXT: movq %rsi, %r8
-; AVX512VBMI-NEXT: shrdq %cl, %rdx, %r8
-; AVX512VBMI-NEXT: movq -48(%rsp,%rax,8), %r9
-; AVX512VBMI-NEXT: shrdq %cl, %r9, %rdx
-; AVX512VBMI-NEXT: movq -72(%rsp,%rax,8), %r10
-; AVX512VBMI-NEXT: shrdq %cl, %rsi, %r10
-; AVX512VBMI-NEXT: movq %rdi, %rax
-; AVX512VBMI-NEXT: sarxq %rcx, %r9, %rcx
-; AVX512VBMI-NEXT: movq %rcx, 24(%rdi)
-; AVX512VBMI-NEXT: movq %rdx, 16(%rdi)
-; AVX512VBMI-NEXT: movq %r8, 8(%rdi)
-; AVX512VBMI-NEXT: movq %r10, (%rdi)
+; AVX512VBMI-NEXT: sarq $63, %rsi
+; AVX512VBMI-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movl %ecx, %edx
+; AVX512VBMI-NEXT: shrb $6, %dl
+; AVX512VBMI-NEXT: movzbl %dl, %esi
+; AVX512VBMI-NEXT: movq -64(%rsp,%rsi,8), %rdx
+; AVX512VBMI-NEXT: movq %rdx, %r8
+; AVX512VBMI-NEXT: shrq %cl, %r8
+; AVX512VBMI-NEXT: movzbq %cl, %r9
+; AVX512VBMI-NEXT: movq -56(%rsp,%rsi,8), %r10
+; AVX512VBMI-NEXT: negq %rdi
+; AVX512VBMI-NEXT: shlxq %rdi, %r10, %rdi
+; AVX512VBMI-NEXT: shrq %cl, %r10
+; AVX512VBMI-NEXT: movzbq %cl, %r11
+; AVX512VBMI-NEXT: orq %r8, %rdi
+; AVX512VBMI-NEXT: movq -48(%rsp,%rsi,8), %r8
+; AVX512VBMI-NEXT: negq %r9
+; AVX512VBMI-NEXT: shlxq %r9, %r8, %r9
+; AVX512VBMI-NEXT: orq %r10, %r9
+; AVX512VBMI-NEXT: movq -72(%rsp,%rsi,8), %rsi
+; AVX512VBMI-NEXT: negq %r11
+; AVX512VBMI-NEXT: shrq %cl, %rsi
+; AVX512VBMI-NEXT: shlxq %r11, %rdx, %rdx
+; AVX512VBMI-NEXT: orq %rsi, %rdx
+; AVX512VBMI-NEXT: sarxq %rcx, %r8, %rcx
+; AVX512VBMI-NEXT: movq %rcx, 24(%rax)
+; AVX512VBMI-NEXT: movq %r9, 16(%rax)
+; AVX512VBMI-NEXT: movq %rdi, 8(%rax)
+; AVX512VBMI-NEXT: movq %rdx, (%rax)
; AVX512VBMI-NEXT: retq
;
; X86-LABEL: ashr_i256_load:
@@ -2048,116 +2354,168 @@ define i256 @shl_1_i256(i256 %a0) nounwind {
;
; AVX2-LABEL: shl_1_i256:
; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %rbx
; AVX2-NEXT: movq %rsi, %rcx
; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movzbq %cl, %rax
+; AVX2-NEXT: movl %ecx, %edx
+; AVX2-NEXT: shrb $3, %dl
+; AVX2-NEXT: andb $24, %dl
+; AVX2-NEXT: negb %dl
+; AVX2-NEXT: movsbq %dl, %rdx
; AVX2-NEXT: vmovss {{.*#+}} xmm0 = [1,0,0,0]
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movl %ecx, %eax
-; AVX2-NEXT: shrb $3, %al
-; AVX2-NEXT: andb $24, %al
-; AVX2-NEXT: negb %al
-; AVX2-NEXT: movsbq %al, %rdx
-; AVX2-NEXT: movq -32(%rsp,%rdx), %rsi
-; AVX2-NEXT: movq -24(%rsp,%rdx), %rax
-; AVX2-NEXT: movq %rax, %r8
-; AVX2-NEXT: shldq %cl, %rsi, %r8
-; AVX2-NEXT: movq -16(%rsp,%rdx), %r9
-; AVX2-NEXT: shldq %cl, %rax, %r9
+; AVX2-NEXT: movq -16(%rsp,%rdx), %rsi
+; AVX2-NEXT: movq %rsi, %r8
+; AVX2-NEXT: shlq %cl, %r8
+; AVX2-NEXT: movq -8(%rsp,%rdx), %r9
+; AVX2-NEXT: shlq %cl, %r9
+; AVX2-NEXT: movq -24(%rsp,%rdx), %r10
+; AVX2-NEXT: negq %rax
+; AVX2-NEXT: shrxq %rax, %r10, %r11
+; AVX2-NEXT: movzbq %cl, %rbx
+; AVX2-NEXT: shlq %cl, %r10
; AVX2-NEXT: movq %rdi, %rax
-; AVX2-NEXT: movq -40(%rsp,%rdx), %rdx
+; AVX2-NEXT: orq %r8, %r11
+; AVX2-NEXT: movq -32(%rsp,%rdx), %rdx
+; AVX2-NEXT: negq %rbx
+; AVX2-NEXT: shrxq %rbx, %rsi, %rsi
+; AVX2-NEXT: orq %r9, %rsi
; AVX2-NEXT: shlxq %rcx, %rdx, %rdi
-; AVX2-NEXT: # kill: def $cl killed $cl killed $rcx
-; AVX2-NEXT: shldq %cl, %rdx, %rsi
-; AVX2-NEXT: movq %r9, 24(%rax)
-; AVX2-NEXT: movq %r8, 16(%rax)
-; AVX2-NEXT: movq %rsi, 8(%rax)
+; AVX2-NEXT: movzbq %cl, %rcx
+; AVX2-NEXT: negq %rcx
+; AVX2-NEXT: shrxq %rcx, %rdx, %rcx
+; AVX2-NEXT: orq %r10, %rcx
+; AVX2-NEXT: movq %rsi, 24(%rax)
+; AVX2-NEXT: movq %r11, 16(%rax)
+; AVX2-NEXT: movq %rcx, 8(%rax)
; AVX2-NEXT: movq %rdi, (%rax)
+; AVX2-NEXT: popq %rbx
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512F-LABEL: shl_1_i256:
; AVX512F: # %bb.0:
+; AVX512F-NEXT: pushq %rbx
; AVX512F-NEXT: movq %rsi, %rcx
+; AVX512F-NEXT: movzbq %cl, %rax
+; AVX512F-NEXT: movl %ecx, %edx
+; AVX512F-NEXT: shrb $3, %dl
+; AVX512F-NEXT: andb $24, %dl
+; AVX512F-NEXT: negb %dl
+; AVX512F-NEXT: movsbq %dl, %rdx
; AVX512F-NEXT: vmovaps {{.*#+}} zmm0 = [0,0,0,0,1,0,0,0]
; AVX512F-NEXT: vmovups %zmm0, -{{[0-9]+}}(%rsp)
-; AVX512F-NEXT: movl %ecx, %eax
-; AVX512F-NEXT: shrb $3, %al
-; AVX512F-NEXT: andb $24, %al
-; AVX512F-NEXT: negb %al
-; AVX512F-NEXT: movsbq %al, %rdx
-; AVX512F-NEXT: movq -32(%rsp,%rdx), %rsi
-; AVX512F-NEXT: movq -24(%rsp,%rdx), %rax
-; AVX512F-NEXT: movq %rax, %r8
-; AVX512F-NEXT: shldq %cl, %rsi, %r8
-; AVX512F-NEXT: movq -16(%rsp,%rdx), %r9
-; AVX512F-NEXT: shldq %cl, %rax, %r9
+; AVX512F-NEXT: movq -16(%rsp,%rdx), %rsi
+; AVX512F-NEXT: movq %rsi, %r8
+; AVX512F-NEXT: shlq %cl, %r8
+; AVX512F-NEXT: movq -8(%rsp,%rdx), %r9
+; AVX512F-NEXT: shlq %cl, %r9
+; AVX512F-NEXT: movq -24(%rsp,%rdx), %r10
+; AVX512F-NEXT: negq %rax
+; AVX512F-NEXT: shrxq %rax, %r10, %r11
+; AVX512F-NEXT: movzbq %cl, %rbx
+; AVX512F-NEXT: shlq %cl, %r10
; AVX512F-NEXT: movq %rdi, %rax
-; AVX512F-NEXT: movq -40(%rsp,%rdx), %rdx
+; AVX512F-NEXT: orq %r8, %r11
+; AVX512F-NEXT: movq -32(%rsp,%rdx), %rdx
+; AVX512F-NEXT: negq %rbx
+; AVX512F-NEXT: shrxq %rbx, %rsi, %rsi
+; AVX512F-NEXT: orq %r9, %rsi
; AVX512F-NEXT: shlxq %rcx, %rdx, %rdi
-; AVX512F-NEXT: # kill: def $cl killed $cl killed $rcx
-; AVX512F-NEXT: shldq %cl, %rdx, %rsi
-; AVX512F-NEXT: movq %r9, 24(%rax)
-; AVX512F-NEXT: movq %r8, 16(%rax)
-; AVX512F-NEXT: movq %rsi, 8(%rax)
+; AVX512F-NEXT: movzbq %cl, %rcx
+; AVX512F-NEXT: negq %rcx
+; AVX512F-NEXT: shrxq %rcx, %rdx, %rcx
+; AVX512F-NEXT: orq %r10, %rcx
+; AVX512F-NEXT: movq %rsi, 24(%rax)
+; AVX512F-NEXT: movq %r11, 16(%rax)
+; AVX512F-NEXT: movq %rcx, 8(%rax)
; AVX512F-NEXT: movq %rdi, (%rax)
+; AVX512F-NEXT: popq %rbx
; AVX512F-NEXT: retq
;
; AVX512VL-LABEL: shl_1_i256:
; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: pushq %rbx
; AVX512VL-NEXT: movq %rsi, %rcx
+; AVX512VL-NEXT: movq %rdi, %rax
; AVX512VL-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: vmovaps {{.*#+}} xmm0 = [1,0,0,0]
+; AVX512VL-NEXT: movzbq %cl, %rdx
; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: movl %ecx, %eax
-; AVX512VL-NEXT: shrb $3, %al
-; AVX512VL-NEXT: andb $24, %al
-; AVX512VL-NEXT: negb %al
-; AVX512VL-NEXT: movsbq %al, %rax
-; AVX512VL-NEXT: movq -32(%rsp,%rax), %rdx
-; AVX512VL-NEXT: movq -24(%rsp,%rax), %rsi
-; AVX512VL-NEXT: movq %rsi, %r8
-; AVX512VL-NEXT: shldq %cl, %rdx, %r8
-; AVX512VL-NEXT: movq -16(%rsp,%rax), %r9
-; AVX512VL-NEXT: shldq %cl, %rsi, %r9
-; AVX512VL-NEXT: movq -40(%rsp,%rax), %rsi
-; AVX512VL-NEXT: shldq %cl, %rsi, %rdx
-; AVX512VL-NEXT: movq %rdi, %rax
-; AVX512VL-NEXT: shlxq %rcx, %rsi, %rcx
-; AVX512VL-NEXT: movq %r9, 24(%rdi)
-; AVX512VL-NEXT: movq %r8, 16(%rdi)
-; AVX512VL-NEXT: movq %rdx, 8(%rdi)
-; AVX512VL-NEXT: movq %rcx, (%rdi)
+; AVX512VL-NEXT: shrb $3, %sil
+; AVX512VL-NEXT: andb $24, %sil
+; AVX512VL-NEXT: negb %sil
+; AVX512VL-NEXT: movsbq %sil, %rsi
+; AVX512VL-NEXT: movq -16(%rsp,%rsi), %rdi
+; AVX512VL-NEXT: movq %rdi, %r8
+; AVX512VL-NEXT: shlq %cl, %r8
+; AVX512VL-NEXT: movzbq %cl, %r9
+; AVX512VL-NEXT: movq -8(%rsp,%rsi), %r10
+; AVX512VL-NEXT: shlq %cl, %r10
+; AVX512VL-NEXT: movzbq %cl, %r11
+; AVX512VL-NEXT: movq -24(%rsp,%rsi), %rbx
+; AVX512VL-NEXT: negq %rdx
+; AVX512VL-NEXT: shrxq %rdx, %rbx, %rdx
+; AVX512VL-NEXT: orq %r8, %rdx
+; AVX512VL-NEXT: negq %r9
+; AVX512VL-NEXT: shrxq %r9, %rdi, %rdi
+; AVX512VL-NEXT: movq -32(%rsp,%rsi), %rsi
+; AVX512VL-NEXT: orq %r10, %rdi
+; AVX512VL-NEXT: shlxq %rcx, %rsi, %r8
+; AVX512VL-NEXT: negq %r11
+; AVX512VL-NEXT: shlq %cl, %rbx
+; AVX512VL-NEXT: shrxq %r11, %rsi, %rcx
+; AVX512VL-NEXT: orq %rbx, %rcx
+; AVX512VL-NEXT: movq %rdi, 24(%rax)
+; AVX512VL-NEXT: movq %rdx, 16(%rax)
+; AVX512VL-NEXT: movq %rcx, 8(%rax)
+; AVX512VL-NEXT: movq %r8, (%rax)
+; AVX512VL-NEXT: popq %rbx
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VBMI-LABEL: shl_1_i256:
; AVX512VBMI: # %bb.0:
+; AVX512VBMI-NEXT: pushq %rbx
; AVX512VBMI-NEXT: movq %rsi, %rcx
+; AVX512VBMI-NEXT: movq %rdi, %rax
; AVX512VBMI-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: vmovaps {{.*#+}} xmm0 = [1,0,0,0]
+; AVX512VBMI-NEXT: movzbq %cl, %rdx
; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: movl %ecx, %eax
-; AVX512VBMI-NEXT: shrb $3, %al
-; AVX512VBMI-NEXT: andb $24, %al
-; AVX512VBMI-NEXT: negb %al
-; AVX512VBMI-NEXT: movsbq %al, %rax
-; AVX512VBMI-NEXT: movq -32(%rsp,%rax), %rdx
-; AVX512VBMI-NEXT: movq -24(%rsp,%rax), %rsi
-; AVX512VBMI-NEXT: movq %rsi, %r8
-; AVX512VBMI-NEXT: shldq %cl, %rdx, %r8
-; AVX512VBMI-NEXT: movq -16(%rsp,%rax), %r9
-; AVX512VBMI-NEXT: shldq %cl, %rsi, %r9
-; AVX512VBMI-NEXT: movq -40(%rsp,%rax), %rsi
-; AVX512VBMI-NEXT: shldq %cl, %rsi, %rdx
-; AVX512VBMI-NEXT: movq %rdi, %rax
-; AVX512VBMI-NEXT: shlxq %rcx, %rsi, %rcx
-; AVX512VBMI-NEXT: movq %r9, 24(%rdi)
-; AVX512VBMI-NEXT: movq %r8, 16(%rdi)
-; AVX512VBMI-NEXT: movq %rdx, 8(%rdi)
-; AVX512VBMI-NEXT: movq %rcx, (%rdi)
+; AVX512VBMI-NEXT: shrb $3, %sil
+; AVX512VBMI-NEXT: andb $24, %sil
+; AVX512VBMI-NEXT: negb %sil
+; AVX512VBMI-NEXT: movsbq %sil, %rsi
+; AVX512VBMI-NEXT: movq -16(%rsp,%rsi), %rdi
+; AVX512VBMI-NEXT: movq %rdi, %r8
+; AVX512VBMI-NEXT: shlq %cl, %r8
+; AVX512VBMI-NEXT: movzbq %cl, %r9
+; AVX512VBMI-NEXT: movq -8(%rsp,%rsi), %r10
+; AVX512VBMI-NEXT: shlq %cl, %r10
+; AVX512VBMI-NEXT: movzbq %cl, %r11
+; AVX512VBMI-NEXT: movq -24(%rsp,%rsi), %rbx
+; AVX512VBMI-NEXT: negq %rdx
+; AVX512VBMI-NEXT: shrxq %rdx, %rbx, %rdx
+; AVX512VBMI-NEXT: orq %r8, %rdx
+; AVX512VBMI-NEXT: negq %r9
+; AVX512VBMI-NEXT: shrxq %r9, %rdi, %rdi
+; AVX512VBMI-NEXT: movq -32(%rsp,%rsi), %rsi
+; AVX512VBMI-NEXT: orq %r10, %rdi
+; AVX512VBMI-NEXT: shlxq %rcx, %rsi, %r8
+; AVX512VBMI-NEXT: negq %r11
+; AVX512VBMI-NEXT: shlq %cl, %rbx
+; AVX512VBMI-NEXT: shrxq %r11, %rsi, %rcx
+; AVX512VBMI-NEXT: orq %rbx, %rcx
+; AVX512VBMI-NEXT: movq %rdi, 24(%rax)
+; AVX512VBMI-NEXT: movq %rdx, 16(%rax)
+; AVX512VBMI-NEXT: movq %rcx, 8(%rax)
+; AVX512VBMI-NEXT: movq %r8, (%rax)
+; AVX512VBMI-NEXT: popq %rbx
; AVX512VBMI-NEXT: vzeroupper
; AVX512VBMI-NEXT: retq
;
@@ -2277,106 +2635,154 @@ define i256 @lshr_signbit_i256(i256 %a0) nounwind {
;
; AVX2-LABEL: lshr_signbit_i256:
; AVX2: # %bb.0:
+; AVX2-NEXT: movq %rsi, %rcx
+; AVX2-NEXT: movzbq %cl, %rax
+; AVX2-NEXT: movl %ecx, %edx
+; AVX2-NEXT: shrb $6, %dl
+; AVX2-NEXT: movzbl %dl, %esi
; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movq %rsi, %rcx
; AVX2-NEXT: vmovaps {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movl %ecx, %eax
-; AVX2-NEXT: shrb $6, %al
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: movq -56(%rsp,%rax,8), %rdx
-; AVX2-NEXT: movq -64(%rsp,%rax,8), %rsi
-; AVX2-NEXT: movq %rsi, %r8
-; AVX2-NEXT: shrdq %cl, %rdx, %r8
-; AVX2-NEXT: movq -72(%rsp,%rax,8), %r9
-; AVX2-NEXT: movq -48(%rsp,%rax,8), %r10
-; AVX2-NEXT: shrdq %cl, %r10, %rdx
-; AVX2-NEXT: shrdq %cl, %rsi, %r9
+; AVX2-NEXT: movq -56(%rsp,%rsi,8), %r9
+; AVX2-NEXT: movq -64(%rsp,%rsi,8), %r8
+; AVX2-NEXT: negq %rax
+; AVX2-NEXT: movq %r8, %r10
+; AVX2-NEXT: shrq %cl, %r10
+; AVX2-NEXT: shlxq %rax, %r9, %rdx
+; AVX2-NEXT: shrq %cl, %r9
; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: movzbq %cl, %rdi
+; AVX2-NEXT: movzbq %cl, %r11
+; AVX2-NEXT: orq %r10, %rdx
+; AVX2-NEXT: movq -48(%rsp,%rsi,8), %r10
+; AVX2-NEXT: negq %rdi
+; AVX2-NEXT: shlxq %rdi, %r10, %rdi
+; AVX2-NEXT: orq %r9, %rdi
+; AVX2-NEXT: movq -72(%rsp,%rsi,8), %rsi
+; AVX2-NEXT: shrq %cl, %rsi
+; AVX2-NEXT: negq %r11
+; AVX2-NEXT: shlxq %r11, %r8, %r8
+; AVX2-NEXT: orq %rsi, %r8
; AVX2-NEXT: shrxq %rcx, %r10, %rcx
-; AVX2-NEXT: movq %rcx, 24(%rdi)
-; AVX2-NEXT: movq %rdx, 16(%rdi)
-; AVX2-NEXT: movq %r8, 8(%rdi)
-; AVX2-NEXT: movq %r9, (%rdi)
+; AVX2-NEXT: movq %rcx, 24(%rax)
+; AVX2-NEXT: movq %rdi, 16(%rax)
+; AVX2-NEXT: movq %rdx, 8(%rax)
+; AVX2-NEXT: movq %r8, (%rax)
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512F-LABEL: lshr_signbit_i256:
; AVX512F: # %bb.0:
; AVX512F-NEXT: movq %rsi, %rcx
+; AVX512F-NEXT: movzbq %cl, %rax
+; AVX512F-NEXT: movl %ecx, %edx
+; AVX512F-NEXT: shrb $6, %dl
+; AVX512F-NEXT: movzbl %dl, %r8d
; AVX512F-NEXT: vmovaps {{.*#+}} zmm0 = [0,0,0,9223372036854775808,0,0,0,0]
; AVX512F-NEXT: vmovups %zmm0, -{{[0-9]+}}(%rsp)
-; AVX512F-NEXT: movl %ecx, %eax
-; AVX512F-NEXT: shrb $6, %al
-; AVX512F-NEXT: movzbl %al, %eax
-; AVX512F-NEXT: movq -56(%rsp,%rax,8), %rdx
-; AVX512F-NEXT: movq -64(%rsp,%rax,8), %rsi
-; AVX512F-NEXT: movq %rsi, %r8
-; AVX512F-NEXT: shrdq %cl, %rdx, %r8
-; AVX512F-NEXT: movq -72(%rsp,%rax,8), %r9
-; AVX512F-NEXT: movq -48(%rsp,%rax,8), %r10
-; AVX512F-NEXT: shrdq %cl, %r10, %rdx
-; AVX512F-NEXT: shrdq %cl, %rsi, %r9
+; AVX512F-NEXT: movq -56(%rsp,%r8,8), %r9
+; AVX512F-NEXT: movq -64(%rsp,%r8,8), %rsi
+; AVX512F-NEXT: negq %rax
+; AVX512F-NEXT: movq %rsi, %r10
+; AVX512F-NEXT: shrq %cl, %r10
+; AVX512F-NEXT: shlxq %rax, %r9, %rdx
+; AVX512F-NEXT: shrq %cl, %r9
; AVX512F-NEXT: movq %rdi, %rax
+; AVX512F-NEXT: movzbq %cl, %rdi
+; AVX512F-NEXT: movzbq %cl, %r11
+; AVX512F-NEXT: orq %r10, %rdx
+; AVX512F-NEXT: movq -48(%rsp,%r8,8), %r10
+; AVX512F-NEXT: negq %rdi
+; AVX512F-NEXT: shlxq %rdi, %r10, %rdi
+; AVX512F-NEXT: orq %r9, %rdi
+; AVX512F-NEXT: movq -72(%rsp,%r8,8), %r8
+; AVX512F-NEXT: shrq %cl, %r8
+; AVX512F-NEXT: negq %r11
+; AVX512F-NEXT: shlxq %r11, %rsi, %rsi
+; AVX512F-NEXT: orq %r8, %rsi
; AVX512F-NEXT: shrxq %rcx, %r10, %rcx
-; AVX512F-NEXT: movq %rcx, 24(%rdi)
-; AVX512F-NEXT: movq %rdx, 16(%rdi)
-; AVX512F-NEXT: movq %r8, 8(%rdi)
-; AVX512F-NEXT: movq %r9, (%rdi)
+; AVX512F-NEXT: movq %rcx, 24(%rax)
+; AVX512F-NEXT: movq %rdi, 16(%rax)
+; AVX512F-NEXT: movq %rdx, 8(%rax)
+; AVX512F-NEXT: movq %rsi, (%rax)
; AVX512F-NEXT: retq
;
; AVX512VL-LABEL: lshr_signbit_i256:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: movq %rsi, %rcx
+; AVX512VL-NEXT: movq %rdi, %rax
; AVX512VL-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movzbq %cl, %rsi
; AVX512VL-NEXT: vmovaps {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: movl %ecx, %eax
-; AVX512VL-NEXT: shrb $6, %al
-; AVX512VL-NEXT: movzbl %al, %eax
-; AVX512VL-NEXT: movq -56(%rsp,%rax,8), %rdx
-; AVX512VL-NEXT: movq -64(%rsp,%rax,8), %rsi
-; AVX512VL-NEXT: movq %rsi, %r8
-; AVX512VL-NEXT: shrdq %cl, %rdx, %r8
-; AVX512VL-NEXT: movq -48(%rsp,%rax,8), %r9
-; AVX512VL-NEXT: shrdq %cl, %r9, %rdx
-; AVX512VL-NEXT: movq -72(%rsp,%rax,8), %r10
-; AVX512VL-NEXT: shrdq %cl, %rsi, %r10
-; AVX512VL-NEXT: movq %rdi, %rax
-; AVX512VL-NEXT: shrxq %rcx, %r9, %rcx
-; AVX512VL-NEXT: movq %rcx, 24(%rdi)
-; AVX512VL-NEXT: movq %rdx, 16(%rdi)
-; AVX512VL-NEXT: movq %r8, 8(%rdi)
-; AVX512VL-NEXT: movq %r10, (%rdi)
+; AVX512VL-NEXT: movl %ecx, %edx
+; AVX512VL-NEXT: shrb $6, %dl
+; AVX512VL-NEXT: movzbl %dl, %edi
+; AVX512VL-NEXT: movq -64(%rsp,%rdi,8), %rdx
+; AVX512VL-NEXT: movq %rdx, %r8
+; AVX512VL-NEXT: shrq %cl, %r8
+; AVX512VL-NEXT: movzbq %cl, %r9
+; AVX512VL-NEXT: movq -56(%rsp,%rdi,8), %r10
+; AVX512VL-NEXT: negq %rsi
+; AVX512VL-NEXT: shlxq %rsi, %r10, %rsi
+; AVX512VL-NEXT: shrq %cl, %r10
+; AVX512VL-NEXT: movzbq %cl, %r11
+; AVX512VL-NEXT: orq %r8, %rsi
+; AVX512VL-NEXT: movq -48(%rsp,%rdi,8), %r8
+; AVX512VL-NEXT: negq %r9
+; AVX512VL-NEXT: shlxq %r9, %r8, %r9
+; AVX512VL-NEXT: orq %r10, %r9
+; AVX512VL-NEXT: movq -72(%rsp,%rdi,8), %rdi
+; AVX512VL-NEXT: negq %r11
+; AVX512VL-NEXT: shrq %cl, %rdi
+; AVX512VL-NEXT: shlxq %r11, %rdx, %rdx
+; AVX512VL-NEXT: orq %rdi, %rdx
+; AVX512VL-NEXT: shrxq %rcx, %r8, %rcx
+; AVX512VL-NEXT: movq %rcx, 24(%rax)
+; AVX512VL-NEXT: movq %r9, 16(%rax)
+; AVX512VL-NEXT: movq %rsi, 8(%rax)
+; AVX512VL-NEXT: movq %rdx, (%rax)
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VBMI-LABEL: lshr_signbit_i256:
; AVX512VBMI: # %bb.0:
; AVX512VBMI-NEXT: movq %rsi, %rcx
+; AVX512VBMI-NEXT: movq %rdi, %rax
; AVX512VBMI-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movzbq %cl, %rsi
; AVX512VBMI-NEXT: vmovaps {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: movl %ecx, %eax
-; AVX512VBMI-NEXT: shrb $6, %al
-; AVX512VBMI-NEXT: movzbl %al, %eax
-; AVX512VBMI-NEXT: movq -56(%rsp,%rax,8), %rdx
-; AVX512VBMI-NEXT: movq -64(%rsp,%rax,8), %rsi
-; AVX512VBMI-NEXT: movq %rsi, %r8
-; AVX512VBMI-NEXT: shrdq %cl, %rdx, %r8
-; AVX512VBMI-NEXT: movq -48(%rsp,%rax,8), %r9
-; AVX512VBMI-NEXT: shrdq %cl, %r9, %rdx
-; AVX512VBMI-NEXT: movq -72(%rsp,%rax,8), %r10
-; AVX512VBMI-NEXT: shrdq %cl, %rsi, %r10
-; AVX512VBMI-NEXT: movq %rdi, %rax
-; AVX512VBMI-NEXT: shrxq %rcx, %r9, %rcx
-; AVX512VBMI-NEXT: movq %rcx, 24(%rdi)
-; AVX512VBMI-NEXT: movq %rdx, 16(%rdi)
-; AVX512VBMI-NEXT: movq %r8, 8(%rdi)
-; AVX512VBMI-NEXT: movq %r10, (%rdi)
+; AVX512VBMI-NEXT: movl %ecx, %edx
+; AVX512VBMI-NEXT: shrb $6, %dl
+; AVX512VBMI-NEXT: movzbl %dl, %edi
+; AVX512VBMI-NEXT: movq -64(%rsp,%rdi,8), %rdx
+; AVX512VBMI-NEXT: movq %rdx, %r8
+; AVX512VBMI-NEXT: shrq %cl, %r8
+; AVX512VBMI-NEXT: movzbq %cl, %r9
+; AVX512VBMI-NEXT: movq -56(%rsp,%rdi,8), %r10
+; AVX512VBMI-NEXT: negq %rsi
+; AVX512VBMI-NEXT: shlxq %rsi, %r10, %rsi
+; AVX512VBMI-NEXT: shrq %cl, %r10
+; AVX512VBMI-NEXT: movzbq %cl, %r11
+; AVX512VBMI-NEXT: orq %r8, %rsi
+; AVX512VBMI-NEXT: movq -48(%rsp,%rdi,8), %r8
+; AVX512VBMI-NEXT: negq %r9
+; AVX512VBMI-NEXT: shlxq %r9, %r8, %r9
+; AVX512VBMI-NEXT: orq %r10, %r9
+; AVX512VBMI-NEXT: movq -72(%rsp,%rdi,8), %rdi
+; AVX512VBMI-NEXT: negq %r11
+; AVX512VBMI-NEXT: shrq %cl, %rdi
+; AVX512VBMI-NEXT: shlxq %r11, %rdx, %rdx
+; AVX512VBMI-NEXT: orq %rdi, %rdx
+; AVX512VBMI-NEXT: shrxq %rcx, %r8, %rcx
+; AVX512VBMI-NEXT: movq %rcx, 24(%rax)
+; AVX512VBMI-NEXT: movq %r9, 16(%rax)
+; AVX512VBMI-NEXT: movq %rsi, 8(%rax)
+; AVX512VBMI-NEXT: movq %rdx, (%rax)
; AVX512VBMI-NEXT: vzeroupper
; AVX512VBMI-NEXT: retq
;
@@ -2491,106 +2897,154 @@ define i256 @ashr_signbit_i256(i256 %a0) nounwind {
;
; AVX2-LABEL: ashr_signbit_i256:
; AVX2: # %bb.0:
+; AVX2-NEXT: movq %rsi, %rcx
+; AVX2-NEXT: movzbq %cl, %rax
+; AVX2-NEXT: movl %ecx, %edx
+; AVX2-NEXT: shrb $6, %dl
+; AVX2-NEXT: movzbl %dl, %esi
; AVX2-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
; AVX2-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movq %rsi, %rcx
; AVX2-NEXT: vmovaps {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movl %ecx, %eax
-; AVX2-NEXT: shrb $6, %al
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: movq -56(%rsp,%rax,8), %rdx
-; AVX2-NEXT: movq -64(%rsp,%rax,8), %rsi
-; AVX2-NEXT: movq %rsi, %r8
-; AVX2-NEXT: shrdq %cl, %rdx, %r8
-; AVX2-NEXT: movq -72(%rsp,%rax,8), %r9
-; AVX2-NEXT: movq -48(%rsp,%rax,8), %r10
-; AVX2-NEXT: shrdq %cl, %r10, %rdx
-; AVX2-NEXT: shrdq %cl, %rsi, %r9
+; AVX2-NEXT: movq -56(%rsp,%rsi,8), %r9
+; AVX2-NEXT: movq -64(%rsp,%rsi,8), %r8
+; AVX2-NEXT: negq %rax
+; AVX2-NEXT: movq %r8, %r10
+; AVX2-NEXT: shrq %cl, %r10
+; AVX2-NEXT: shlxq %rax, %r9, %rdx
+; AVX2-NEXT: shrq %cl, %r9
; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: movzbq %cl, %rdi
+; AVX2-NEXT: movzbq %cl, %r11
+; AVX2-NEXT: orq %r10, %rdx
+; AVX2-NEXT: movq -48(%rsp,%rsi,8), %r10
+; AVX2-NEXT: negq %rdi
+; AVX2-NEXT: shlxq %rdi, %r10, %rdi
+; AVX2-NEXT: orq %r9, %rdi
+; AVX2-NEXT: movq -72(%rsp,%rsi,8), %rsi
+; AVX2-NEXT: shrq %cl, %rsi
+; AVX2-NEXT: negq %r11
+; AVX2-NEXT: shlxq %r11, %r8, %r8
+; AVX2-NEXT: orq %rsi, %r8
; AVX2-NEXT: sarxq %rcx, %r10, %rcx
-; AVX2-NEXT: movq %rcx, 24(%rdi)
-; AVX2-NEXT: movq %rdx, 16(%rdi)
-; AVX2-NEXT: movq %r8, 8(%rdi)
-; AVX2-NEXT: movq %r9, (%rdi)
+; AVX2-NEXT: movq %rcx, 24(%rax)
+; AVX2-NEXT: movq %rdi, 16(%rax)
+; AVX2-NEXT: movq %rdx, 8(%rax)
+; AVX2-NEXT: movq %r8, (%rax)
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512F-LABEL: ashr_signbit_i256:
; AVX512F: # %bb.0:
; AVX512F-NEXT: movq %rsi, %rcx
+; AVX512F-NEXT: movzbq %cl, %rax
+; AVX512F-NEXT: movl %ecx, %edx
+; AVX512F-NEXT: shrb $6, %dl
+; AVX512F-NEXT: movzbl %dl, %r8d
; AVX512F-NEXT: vmovaps {{.*#+}} zmm0 = [0,0,0,9223372036854775808,18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615]
; AVX512F-NEXT: vmovups %zmm0, -{{[0-9]+}}(%rsp)
-; AVX512F-NEXT: movl %ecx, %eax
-; AVX512F-NEXT: shrb $6, %al
-; AVX512F-NEXT: movzbl %al, %eax
-; AVX512F-NEXT: movq -56(%rsp,%rax,8), %rdx
-; AVX512F-NEXT: movq -64(%rsp,%rax,8), %rsi
-; AVX512F-NEXT: movq %rsi, %r8
-; AVX512F-NEXT: shrdq %cl, %rdx, %r8
-; AVX512F-NEXT: movq -72(%rsp,%rax,8), %r9
-; AVX512F-NEXT: movq -48(%rsp,%rax,8), %r10
-; AVX512F-NEXT: shrdq %cl, %r10, %rdx
-; AVX512F-NEXT: shrdq %cl, %rsi, %r9
+; AVX512F-NEXT: movq -56(%rsp,%r8,8), %r9
+; AVX512F-NEXT: movq -64(%rsp,%r8,8), %rsi
+; AVX512F-NEXT: negq %rax
+; AVX512F-NEXT: movq %rsi, %r10
+; AVX512F-NEXT: shrq %cl, %r10
+; AVX512F-NEXT: shlxq %rax, %r9, %rdx
+; AVX512F-NEXT: shrq %cl, %r9
; AVX512F-NEXT: movq %rdi, %rax
+; AVX512F-NEXT: movzbq %cl, %rdi
+; AVX512F-NEXT: movzbq %cl, %r11
+; AVX512F-NEXT: orq %r10, %rdx
+; AVX512F-NEXT: movq -48(%rsp,%r8,8), %r10
+; AVX512F-NEXT: negq %rdi
+; AVX512F-NEXT: shlxq %rdi, %r10, %rdi
+; AVX512F-NEXT: orq %r9, %rdi
+; AVX512F-NEXT: movq -72(%rsp,%r8,8), %r8
+; AVX512F-NEXT: shrq %cl, %r8
+; AVX512F-NEXT: negq %r11
+; AVX512F-NEXT: shlxq %r11, %rsi, %rsi
+; AVX512F-NEXT: orq %r8, %rsi
; AVX512F-NEXT: sarxq %rcx, %r10, %rcx
-; AVX512F-NEXT: movq %rcx, 24(%rdi)
-; AVX512F-NEXT: movq %rdx, 16(%rdi)
-; AVX512F-NEXT: movq %r8, 8(%rdi)
-; AVX512F-NEXT: movq %r9, (%rdi)
+; AVX512F-NEXT: movq %rcx, 24(%rax)
+; AVX512F-NEXT: movq %rdi, 16(%rax)
+; AVX512F-NEXT: movq %rdx, 8(%rax)
+; AVX512F-NEXT: movq %rsi, (%rax)
; AVX512F-NEXT: retq
;
; AVX512VL-LABEL: ashr_signbit_i256:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: movq %rsi, %rcx
+; AVX512VL-NEXT: movq %rdi, %rax
; AVX512VL-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
; AVX512VL-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movzbq %cl, %rsi
; AVX512VL-NEXT: vmovaps {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: movl %ecx, %eax
-; AVX512VL-NEXT: shrb $6, %al
-; AVX512VL-NEXT: movzbl %al, %eax
-; AVX512VL-NEXT: movq -56(%rsp,%rax,8), %rdx
-; AVX512VL-NEXT: movq -64(%rsp,%rax,8), %rsi
-; AVX512VL-NEXT: movq %rsi, %r8
-; AVX512VL-NEXT: shrdq %cl, %rdx, %r8
-; AVX512VL-NEXT: movq -48(%rsp,%rax,8), %r9
-; AVX512VL-NEXT: shrdq %cl, %r9, %rdx
-; AVX512VL-NEXT: movq -72(%rsp,%rax,8), %r10
-; AVX512VL-NEXT: shrdq %cl, %rsi, %r10
-; AVX512VL-NEXT: movq %rdi, %rax
-; AVX512VL-NEXT: sarxq %rcx, %r9, %rcx
-; AVX512VL-NEXT: movq %rcx, 24(%rdi)
-; AVX512VL-NEXT: movq %rdx, 16(%rdi)
-; AVX512VL-NEXT: movq %r8, 8(%rdi)
-; AVX512VL-NEXT: movq %r10, (%rdi)
+; AVX512VL-NEXT: movl %ecx, %edx
+; AVX512VL-NEXT: shrb $6, %dl
+; AVX512VL-NEXT: movzbl %dl, %edi
+; AVX512VL-NEXT: movq -64(%rsp,%rdi,8), %rdx
+; AVX512VL-NEXT: movq %rdx, %r8
+; AVX512VL-NEXT: shrq %cl, %r8
+; AVX512VL-NEXT: movzbq %cl, %r9
+; AVX512VL-NEXT: movq -56(%rsp,%rdi,8), %r10
+; AVX512VL-NEXT: negq %rsi
+; AVX512VL-NEXT: shlxq %rsi, %r10, %rsi
+; AVX512VL-NEXT: shrq %cl, %r10
+; AVX512VL-NEXT: movzbq %cl, %r11
+; AVX512VL-NEXT: orq %r8, %rsi
+; AVX512VL-NEXT: movq -48(%rsp,%rdi,8), %r8
+; AVX512VL-NEXT: negq %r9
+; AVX512VL-NEXT: shlxq %r9, %r8, %r9
+; AVX512VL-NEXT: orq %r10, %r9
+; AVX512VL-NEXT: movq -72(%rsp,%rdi,8), %rdi
+; AVX512VL-NEXT: negq %r11
+; AVX512VL-NEXT: shrq %cl, %rdi
+; AVX512VL-NEXT: shlxq %r11, %rdx, %rdx
+; AVX512VL-NEXT: orq %rdi, %rdx
+; AVX512VL-NEXT: sarxq %rcx, %r8, %rcx
+; AVX512VL-NEXT: movq %rcx, 24(%rax)
+; AVX512VL-NEXT: movq %r9, 16(%rax)
+; AVX512VL-NEXT: movq %rsi, 8(%rax)
+; AVX512VL-NEXT: movq %rdx, (%rax)
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VBMI-LABEL: ashr_signbit_i256:
; AVX512VBMI: # %bb.0:
; AVX512VBMI-NEXT: movq %rsi, %rcx
+; AVX512VBMI-NEXT: movq %rdi, %rax
; AVX512VBMI-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
; AVX512VBMI-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movzbq %cl, %rsi
; AVX512VBMI-NEXT: vmovaps {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: movl %ecx, %eax
-; AVX512VBMI-NEXT: shrb $6, %al
-; AVX512VBMI-NEXT: movzbl %al, %eax
-; AVX512VBMI-NEXT: movq -56(%rsp,%rax,8), %rdx
-; AVX512VBMI-NEXT: movq -64(%rsp,%rax,8), %rsi
-; AVX512VBMI-NEXT: movq %rsi, %r8
-; AVX512VBMI-NEXT: shrdq %cl, %rdx, %r8
-; AVX512VBMI-NEXT: movq -48(%rsp,%rax,8), %r9
-; AVX512VBMI-NEXT: shrdq %cl, %r9, %rdx
-; AVX512VBMI-NEXT: movq -72(%rsp,%rax,8), %r10
-; AVX512VBMI-NEXT: shrdq %cl, %rsi, %r10
-; AVX512VBMI-NEXT: movq %rdi, %rax
-; AVX512VBMI-NEXT: sarxq %rcx, %r9, %rcx
-; AVX512VBMI-NEXT: movq %rcx, 24(%rdi)
-; AVX512VBMI-NEXT: movq %rdx, 16(%rdi)
-; AVX512VBMI-NEXT: movq %r8, 8(%rdi)
-; AVX512VBMI-NEXT: movq %r10, (%rdi)
+; AVX512VBMI-NEXT: movl %ecx, %edx
+; AVX512VBMI-NEXT: shrb $6, %dl
+; AVX512VBMI-NEXT: movzbl %dl, %edi
+; AVX512VBMI-NEXT: movq -64(%rsp,%rdi,8), %rdx
+; AVX512VBMI-NEXT: movq %rdx, %r8
+; AVX512VBMI-NEXT: shrq %cl, %r8
+; AVX512VBMI-NEXT: movzbq %cl, %r9
+; AVX512VBMI-NEXT: movq -56(%rsp,%rdi,8), %r10
+; AVX512VBMI-NEXT: negq %rsi
+; AVX512VBMI-NEXT: shlxq %rsi, %r10, %rsi
+; AVX512VBMI-NEXT: shrq %cl, %r10
+; AVX512VBMI-NEXT: movzbq %cl, %r11
+; AVX512VBMI-NEXT: orq %r8, %rsi
+; AVX512VBMI-NEXT: movq -48(%rsp,%rdi,8), %r8
+; AVX512VBMI-NEXT: negq %r9
+; AVX512VBMI-NEXT: shlxq %r9, %r8, %r9
+; AVX512VBMI-NEXT: orq %r10, %r9
+; AVX512VBMI-NEXT: movq -72(%rsp,%rdi,8), %rdi
+; AVX512VBMI-NEXT: negq %r11
+; AVX512VBMI-NEXT: shrq %cl, %rdi
+; AVX512VBMI-NEXT: shlxq %r11, %rdx, %rdx
+; AVX512VBMI-NEXT: orq %rdi, %rdx
+; AVX512VBMI-NEXT: sarxq %rcx, %r8, %rcx
+; AVX512VBMI-NEXT: movq %rcx, 24(%rax)
+; AVX512VBMI-NEXT: movq %r9, 16(%rax)
+; AVX512VBMI-NEXT: movq %rsi, 8(%rax)
+; AVX512VBMI-NEXT: movq %rdx, (%rax)
; AVX512VBMI-NEXT: vzeroupper
; AVX512VBMI-NEXT: retq
;
@@ -2706,116 +3160,168 @@ define i256 @shl_allbits_i256(i256 %a0) nounwind {
;
; AVX2-LABEL: shl_allbits_i256:
; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %rbx
; AVX2-NEXT: movq %rsi, %rcx
; AVX2-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
; AVX2-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movzbq %cl, %rax
+; AVX2-NEXT: movl %ecx, %edx
+; AVX2-NEXT: shrb $3, %dl
+; AVX2-NEXT: andb $24, %dl
+; AVX2-NEXT: negb %dl
+; AVX2-NEXT: movsbq %dl, %rdx
; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
; AVX2-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movl %ecx, %eax
-; AVX2-NEXT: shrb $3, %al
-; AVX2-NEXT: andb $24, %al
-; AVX2-NEXT: negb %al
-; AVX2-NEXT: movsbq %al, %rdx
-; AVX2-NEXT: movq -32(%rsp,%rdx), %rsi
-; AVX2-NEXT: movq -24(%rsp,%rdx), %rax
-; AVX2-NEXT: movq %rax, %r8
-; AVX2-NEXT: shldq %cl, %rsi, %r8
-; AVX2-NEXT: movq -16(%rsp,%rdx), %r9
-; AVX2-NEXT: shldq %cl, %rax, %r9
+; AVX2-NEXT: movq -16(%rsp,%rdx), %rsi
+; AVX2-NEXT: movq %rsi, %r8
+; AVX2-NEXT: shlq %cl, %r8
+; AVX2-NEXT: movq -8(%rsp,%rdx), %r9
+; AVX2-NEXT: shlq %cl, %r9
+; AVX2-NEXT: movq -24(%rsp,%rdx), %r10
+; AVX2-NEXT: negq %rax
+; AVX2-NEXT: shrxq %rax, %r10, %r11
+; AVX2-NEXT: movzbq %cl, %rbx
+; AVX2-NEXT: shlq %cl, %r10
; AVX2-NEXT: movq %rdi, %rax
-; AVX2-NEXT: movq -40(%rsp,%rdx), %rdx
+; AVX2-NEXT: orq %r8, %r11
+; AVX2-NEXT: movq -32(%rsp,%rdx), %rdx
+; AVX2-NEXT: negq %rbx
+; AVX2-NEXT: shrxq %rbx, %rsi, %rsi
+; AVX2-NEXT: orq %r9, %rsi
; AVX2-NEXT: shlxq %rcx, %rdx, %rdi
-; AVX2-NEXT: # kill: def $cl killed $cl killed $rcx
-; AVX2-NEXT: shldq %cl, %rdx, %rsi
-; AVX2-NEXT: movq %r9, 24(%rax)
-; AVX2-NEXT: movq %r8, 16(%rax)
-; AVX2-NEXT: movq %rsi, 8(%rax)
+; AVX2-NEXT: movzbq %cl, %rcx
+; AVX2-NEXT: negq %rcx
+; AVX2-NEXT: shrxq %rcx, %rdx, %rcx
+; AVX2-NEXT: orq %r10, %rcx
+; AVX2-NEXT: movq %rsi, 24(%rax)
+; AVX2-NEXT: movq %r11, 16(%rax)
+; AVX2-NEXT: movq %rcx, 8(%rax)
; AVX2-NEXT: movq %rdi, (%rax)
+; AVX2-NEXT: popq %rbx
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512F-LABEL: shl_allbits_i256:
; AVX512F: # %bb.0:
+; AVX512F-NEXT: pushq %rbx
; AVX512F-NEXT: movq %rsi, %rcx
+; AVX512F-NEXT: movzbq %cl, %rax
+; AVX512F-NEXT: movl %ecx, %edx
+; AVX512F-NEXT: shrb $3, %dl
+; AVX512F-NEXT: andb $24, %dl
+; AVX512F-NEXT: negb %dl
+; AVX512F-NEXT: movsbq %dl, %rdx
; AVX512F-NEXT: vmovaps {{.*#+}} zmm0 = [0,0,0,0,18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615]
; AVX512F-NEXT: vmovups %zmm0, -{{[0-9]+}}(%rsp)
-; AVX512F-NEXT: movl %ecx, %eax
-; AVX512F-NEXT: shrb $3, %al
-; AVX512F-NEXT: andb $24, %al
-; AVX512F-NEXT: negb %al
-; AVX512F-NEXT: movsbq %al, %rdx
-; AVX512F-NEXT: movq -32(%rsp,%rdx), %rsi
-; AVX512F-NEXT: movq -24(%rsp,%rdx), %rax
-; AVX512F-NEXT: movq %rax, %r8
-; AVX512F-NEXT: shldq %cl, %rsi, %r8
-; AVX512F-NEXT: movq -16(%rsp,%rdx), %r9
-; AVX512F-NEXT: shldq %cl, %rax, %r9
+; AVX512F-NEXT: movq -16(%rsp,%rdx), %rsi
+; AVX512F-NEXT: movq %rsi, %r8
+; AVX512F-NEXT: shlq %cl, %r8
+; AVX512F-NEXT: movq -8(%rsp,%rdx), %r9
+; AVX512F-NEXT: shlq %cl, %r9
+; AVX512F-NEXT: movq -24(%rsp,%rdx), %r10
+; AVX512F-NEXT: negq %rax
+; AVX512F-NEXT: shrxq %rax, %r10, %r11
+; AVX512F-NEXT: movzbq %cl, %rbx
+; AVX512F-NEXT: shlq %cl, %r10
; AVX512F-NEXT: movq %rdi, %rax
-; AVX512F-NEXT: movq -40(%rsp,%rdx), %rdx
+; AVX512F-NEXT: orq %r8, %r11
+; AVX512F-NEXT: movq -32(%rsp,%rdx), %rdx
+; AVX512F-NEXT: negq %rbx
+; AVX512F-NEXT: shrxq %rbx, %rsi, %rsi
+; AVX512F-NEXT: orq %r9, %rsi
; AVX512F-NEXT: shlxq %rcx, %rdx, %rdi
-; AVX512F-NEXT: # kill: def $cl killed $cl killed $rcx
-; AVX512F-NEXT: shldq %cl, %rdx, %rsi
-; AVX512F-NEXT: movq %r9, 24(%rax)
-; AVX512F-NEXT: movq %r8, 16(%rax)
-; AVX512F-NEXT: movq %rsi, 8(%rax)
+; AVX512F-NEXT: movzbq %cl, %rcx
+; AVX512F-NEXT: negq %rcx
+; AVX512F-NEXT: shrxq %rcx, %rdx, %rcx
+; AVX512F-NEXT: orq %r10, %rcx
+; AVX512F-NEXT: movq %rsi, 24(%rax)
+; AVX512F-NEXT: movq %r11, 16(%rax)
+; AVX512F-NEXT: movq %rcx, 8(%rax)
; AVX512F-NEXT: movq %rdi, (%rax)
+; AVX512F-NEXT: popq %rbx
; AVX512F-NEXT: retq
;
; AVX512VL-LABEL: shl_allbits_i256:
; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: pushq %rbx
; AVX512VL-NEXT: movq %rsi, %rcx
+; AVX512VL-NEXT: movq %rdi, %rax
+; AVX512VL-NEXT: movzbq %cl, %rdx
; AVX512VL-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
; AVX512VL-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: vpxor %xmm0, %xmm0, %xmm0
; AVX512VL-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: movl %ecx, %eax
-; AVX512VL-NEXT: shrb $3, %al
-; AVX512VL-NEXT: andb $24, %al
-; AVX512VL-NEXT: negb %al
-; AVX512VL-NEXT: movsbq %al, %rax
-; AVX512VL-NEXT: movq -32(%rsp,%rax), %rdx
-; AVX512VL-NEXT: movq -24(%rsp,%rax), %rsi
-; AVX512VL-NEXT: movq %rsi, %r8
-; AVX512VL-NEXT: shldq %cl, %rdx, %r8
-; AVX512VL-NEXT: movq -16(%rsp,%rax), %r9
-; AVX512VL-NEXT: shldq %cl, %rsi, %r9
-; AVX512VL-NEXT: movq -40(%rsp,%rax), %rsi
-; AVX512VL-NEXT: shldq %cl, %rsi, %rdx
-; AVX512VL-NEXT: movq %rdi, %rax
-; AVX512VL-NEXT: shlxq %rcx, %rsi, %rcx
-; AVX512VL-NEXT: movq %r9, 24(%rdi)
-; AVX512VL-NEXT: movq %r8, 16(%rdi)
-; AVX512VL-NEXT: movq %rdx, 8(%rdi)
-; AVX512VL-NEXT: movq %rcx, (%rdi)
+; AVX512VL-NEXT: shrb $3, %sil
+; AVX512VL-NEXT: andb $24, %sil
+; AVX512VL-NEXT: negb %sil
+; AVX512VL-NEXT: movsbq %sil, %rsi
+; AVX512VL-NEXT: movq -16(%rsp,%rsi), %rdi
+; AVX512VL-NEXT: movq %rdi, %r8
+; AVX512VL-NEXT: shlq %cl, %r8
+; AVX512VL-NEXT: movzbq %cl, %r9
+; AVX512VL-NEXT: movq -8(%rsp,%rsi), %r10
+; AVX512VL-NEXT: shlq %cl, %r10
+; AVX512VL-NEXT: movzbq %cl, %r11
+; AVX512VL-NEXT: movq -24(%rsp,%rsi), %rbx
+; AVX512VL-NEXT: negq %rdx
+; AVX512VL-NEXT: shrxq %rdx, %rbx, %rdx
+; AVX512VL-NEXT: orq %r8, %rdx
+; AVX512VL-NEXT: negq %r9
+; AVX512VL-NEXT: shrxq %r9, %rdi, %rdi
+; AVX512VL-NEXT: movq -32(%rsp,%rsi), %rsi
+; AVX512VL-NEXT: orq %r10, %rdi
+; AVX512VL-NEXT: shlxq %rcx, %rsi, %r8
+; AVX512VL-NEXT: negq %r11
+; AVX512VL-NEXT: shlq %cl, %rbx
+; AVX512VL-NEXT: shrxq %r11, %rsi, %rcx
+; AVX512VL-NEXT: orq %rbx, %rcx
+; AVX512VL-NEXT: movq %rdi, 24(%rax)
+; AVX512VL-NEXT: movq %rdx, 16(%rax)
+; AVX512VL-NEXT: movq %rcx, 8(%rax)
+; AVX512VL-NEXT: movq %r8, (%rax)
+; AVX512VL-NEXT: popq %rbx
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VBMI-LABEL: shl_allbits_i256:
; AVX512VBMI: # %bb.0:
+; AVX512VBMI-NEXT: pushq %rbx
; AVX512VBMI-NEXT: movq %rsi, %rcx
+; AVX512VBMI-NEXT: movq %rdi, %rax
+; AVX512VBMI-NEXT: movzbq %cl, %rdx
; AVX512VBMI-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
; AVX512VBMI-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: vpxor %xmm0, %xmm0, %xmm0
; AVX512VBMI-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: movl %ecx, %eax
-; AVX512VBMI-NEXT: shrb $3, %al
-; AVX512VBMI-NEXT: andb $24, %al
-; AVX512VBMI-NEXT: negb %al
-; AVX512VBMI-NEXT: movsbq %al, %rax
-; AVX512VBMI-NEXT: movq -32(%rsp,%rax), %rdx
-; AVX512VBMI-NEXT: movq -24(%rsp,%rax), %rsi
-; AVX512VBMI-NEXT: movq %rsi, %r8
-; AVX512VBMI-NEXT: shldq %cl, %rdx, %r8
-; AVX512VBMI-NEXT: movq -16(%rsp,%rax), %r9
-; AVX512VBMI-NEXT: shldq %cl, %rsi, %r9
-; AVX512VBMI-NEXT: movq -40(%rsp,%rax), %rsi
-; AVX512VBMI-NEXT: shldq %cl, %rsi, %rdx
-; AVX512VBMI-NEXT: movq %rdi, %rax
-; AVX512VBMI-NEXT: shlxq %rcx, %rsi, %rcx
-; AVX512VBMI-NEXT: movq %r9, 24(%rdi)
-; AVX512VBMI-NEXT: movq %r8, 16(%rdi)
-; AVX512VBMI-NEXT: movq %rdx, 8(%rdi)
-; AVX512VBMI-NEXT: movq %rcx, (%rdi)
+; AVX512VBMI-NEXT: shrb $3, %sil
+; AVX512VBMI-NEXT: andb $24, %sil
+; AVX512VBMI-NEXT: negb %sil
+; AVX512VBMI-NEXT: movsbq %sil, %rsi
+; AVX512VBMI-NEXT: movq -16(%rsp,%rsi), %rdi
+; AVX512VBMI-NEXT: movq %rdi, %r8
+; AVX512VBMI-NEXT: shlq %cl, %r8
+; AVX512VBMI-NEXT: movzbq %cl, %r9
+; AVX512VBMI-NEXT: movq -8(%rsp,%rsi), %r10
+; AVX512VBMI-NEXT: shlq %cl, %r10
+; AVX512VBMI-NEXT: movzbq %cl, %r11
+; AVX512VBMI-NEXT: movq -24(%rsp,%rsi), %rbx
+; AVX512VBMI-NEXT: negq %rdx
+; AVX512VBMI-NEXT: shrxq %rdx, %rbx, %rdx
+; AVX512VBMI-NEXT: orq %r8, %rdx
+; AVX512VBMI-NEXT: negq %r9
+; AVX512VBMI-NEXT: shrxq %r9, %rdi, %rdi
+; AVX512VBMI-NEXT: movq -32(%rsp,%rsi), %rsi
+; AVX512VBMI-NEXT: orq %r10, %rdi
+; AVX512VBMI-NEXT: shlxq %rcx, %rsi, %r8
+; AVX512VBMI-NEXT: negq %r11
+; AVX512VBMI-NEXT: shlq %cl, %rbx
+; AVX512VBMI-NEXT: shrxq %r11, %rsi, %rcx
+; AVX512VBMI-NEXT: orq %rbx, %rcx
+; AVX512VBMI-NEXT: movq %rdi, 24(%rax)
+; AVX512VBMI-NEXT: movq %rdx, 16(%rax)
+; AVX512VBMI-NEXT: movq %rcx, 8(%rax)
+; AVX512VBMI-NEXT: movq %r8, (%rax)
+; AVX512VBMI-NEXT: popq %rbx
; AVX512VBMI-NEXT: vzeroupper
; AVX512VBMI-NEXT: retq
;
@@ -2936,106 +3442,154 @@ define i256 @lshr_allbits_i256(i256 %a0) nounwind {
;
; AVX2-LABEL: lshr_allbits_i256:
; AVX2: # %bb.0:
+; AVX2-NEXT: movq %rsi, %rcx
+; AVX2-NEXT: movzbq %cl, %rax
+; AVX2-NEXT: movl %ecx, %edx
+; AVX2-NEXT: shrb $6, %dl
+; AVX2-NEXT: movzbl %dl, %esi
; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
; AVX2-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movq %rsi, %rcx
; AVX2-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
; AVX2-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movl %ecx, %eax
-; AVX2-NEXT: shrb $6, %al
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: movq -56(%rsp,%rax,8), %rdx
-; AVX2-NEXT: movq -64(%rsp,%rax,8), %rsi
-; AVX2-NEXT: movq %rsi, %r8
-; AVX2-NEXT: shrdq %cl, %rdx, %r8
-; AVX2-NEXT: movq -72(%rsp,%rax,8), %r9
-; AVX2-NEXT: movq -48(%rsp,%rax,8), %r10
-; AVX2-NEXT: shrdq %cl, %r10, %rdx
-; AVX2-NEXT: shrdq %cl, %rsi, %r9
+; AVX2-NEXT: movq -56(%rsp,%rsi,8), %r9
+; AVX2-NEXT: movq -64(%rsp,%rsi,8), %r8
+; AVX2-NEXT: negq %rax
+; AVX2-NEXT: movq %r8, %r10
+; AVX2-NEXT: shrq %cl, %r10
+; AVX2-NEXT: shlxq %rax, %r9, %rdx
+; AVX2-NEXT: shrq %cl, %r9
; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: movzbq %cl, %rdi
+; AVX2-NEXT: movzbq %cl, %r11
+; AVX2-NEXT: orq %r10, %rdx
+; AVX2-NEXT: movq -48(%rsp,%rsi,8), %r10
+; AVX2-NEXT: negq %rdi
+; AVX2-NEXT: shlxq %rdi, %r10, %rdi
+; AVX2-NEXT: orq %r9, %rdi
+; AVX2-NEXT: movq -72(%rsp,%rsi,8), %rsi
+; AVX2-NEXT: shrq %cl, %rsi
+; AVX2-NEXT: negq %r11
+; AVX2-NEXT: shlxq %r11, %r8, %r8
+; AVX2-NEXT: orq %rsi, %r8
; AVX2-NEXT: shrxq %rcx, %r10, %rcx
-; AVX2-NEXT: movq %rcx, 24(%rdi)
-; AVX2-NEXT: movq %rdx, 16(%rdi)
-; AVX2-NEXT: movq %r8, 8(%rdi)
-; AVX2-NEXT: movq %r9, (%rdi)
+; AVX2-NEXT: movq %rcx, 24(%rax)
+; AVX2-NEXT: movq %rdi, 16(%rax)
+; AVX2-NEXT: movq %rdx, 8(%rax)
+; AVX2-NEXT: movq %r8, (%rax)
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512F-LABEL: lshr_allbits_i256:
; AVX512F: # %bb.0:
; AVX512F-NEXT: movq %rsi, %rcx
+; AVX512F-NEXT: movzbq %cl, %rax
+; AVX512F-NEXT: movl %ecx, %edx
+; AVX512F-NEXT: shrb $6, %dl
+; AVX512F-NEXT: movzbl %dl, %r8d
; AVX512F-NEXT: vmovaps {{.*#+}} zmm0 = [18446744073709551615,18446744073709551615,18446744073709551615,18446744073709551615,0,0,0,0]
; AVX512F-NEXT: vmovups %zmm0, -{{[0-9]+}}(%rsp)
-; AVX512F-NEXT: movl %ecx, %eax
-; AVX512F-NEXT: shrb $6, %al
-; AVX512F-NEXT: movzbl %al, %eax
-; AVX512F-NEXT: movq -56(%rsp,%rax,8), %rdx
-; AVX512F-NEXT: movq -64(%rsp,%rax,8), %rsi
-; AVX512F-NEXT: movq %rsi, %r8
-; AVX512F-NEXT: shrdq %cl, %rdx, %r8
-; AVX512F-NEXT: movq -72(%rsp,%rax,8), %r9
-; AVX512F-NEXT: movq -48(%rsp,%rax,8), %r10
-; AVX512F-NEXT: shrdq %cl, %r10, %rdx
-; AVX512F-NEXT: shrdq %cl, %rsi, %r9
+; AVX512F-NEXT: movq -56(%rsp,%r8,8), %r9
+; AVX512F-NEXT: movq -64(%rsp,%r8,8), %rsi
+; AVX512F-NEXT: negq %rax
+; AVX512F-NEXT: movq %rsi, %r10
+; AVX512F-NEXT: shrq %cl, %r10
+; AVX512F-NEXT: shlxq %rax, %r9, %rdx
+; AVX512F-NEXT: shrq %cl, %r9
; AVX512F-NEXT: movq %rdi, %rax
+; AVX512F-NEXT: movzbq %cl, %rdi
+; AVX512F-NEXT: movzbq %cl, %r11
+; AVX512F-NEXT: orq %r10, %rdx
+; AVX512F-NEXT: movq -48(%rsp,%r8,8), %r10
+; AVX512F-NEXT: negq %rdi
+; AVX512F-NEXT: shlxq %rdi, %r10, %rdi
+; AVX512F-NEXT: orq %r9, %rdi
+; AVX512F-NEXT: movq -72(%rsp,%r8,8), %r8
+; AVX512F-NEXT: shrq %cl, %r8
+; AVX512F-NEXT: negq %r11
+; AVX512F-NEXT: shlxq %r11, %rsi, %rsi
+; AVX512F-NEXT: orq %r8, %rsi
; AVX512F-NEXT: shrxq %rcx, %r10, %rcx
-; AVX512F-NEXT: movq %rcx, 24(%rdi)
-; AVX512F-NEXT: movq %rdx, 16(%rdi)
-; AVX512F-NEXT: movq %r8, 8(%rdi)
-; AVX512F-NEXT: movq %r9, (%rdi)
+; AVX512F-NEXT: movq %rcx, 24(%rax)
+; AVX512F-NEXT: movq %rdi, 16(%rax)
+; AVX512F-NEXT: movq %rdx, 8(%rax)
+; AVX512F-NEXT: movq %rsi, (%rax)
; AVX512F-NEXT: retq
;
; AVX512VL-LABEL: lshr_allbits_i256:
; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: movq %rsi, %rcx
+; AVX512VL-NEXT: movq %rdi, %rax
+; AVX512VL-NEXT: movzbq %cl, %rsi
; AVX512VL-NEXT: vpxor %xmm0, %xmm0, %xmm0
; AVX512VL-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: movq %rsi, %rcx
; AVX512VL-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
; AVX512VL-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: movl %ecx, %eax
-; AVX512VL-NEXT: shrb $6, %al
-; AVX512VL-NEXT: movzbl %al, %eax
-; AVX512VL-NEXT: movq -56(%rsp,%rax,8), %rdx
-; AVX512VL-NEXT: movq -64(%rsp,%rax,8), %rsi
-; AVX512VL-NEXT: movq %rsi, %r8
-; AVX512VL-NEXT: shrdq %cl, %rdx, %r8
-; AVX512VL-NEXT: movq -48(%rsp,%rax,8), %r9
-; AVX512VL-NEXT: shrdq %cl, %r9, %rdx
-; AVX512VL-NEXT: movq -72(%rsp,%rax,8), %r10
-; AVX512VL-NEXT: shrdq %cl, %rsi, %r10
-; AVX512VL-NEXT: movq %rdi, %rax
-; AVX512VL-NEXT: shrxq %rcx, %r9, %rcx
-; AVX512VL-NEXT: movq %rcx, 24(%rdi)
-; AVX512VL-NEXT: movq %rdx, 16(%rdi)
-; AVX512VL-NEXT: movq %r8, 8(%rdi)
-; AVX512VL-NEXT: movq %r10, (%rdi)
+; AVX512VL-NEXT: movl %ecx, %edx
+; AVX512VL-NEXT: shrb $6, %dl
+; AVX512VL-NEXT: movzbl %dl, %edi
+; AVX512VL-NEXT: movq -64(%rsp,%rdi,8), %rdx
+; AVX512VL-NEXT: movq %rdx, %r8
+; AVX512VL-NEXT: shrq %cl, %r8
+; AVX512VL-NEXT: movzbq %cl, %r9
+; AVX512VL-NEXT: movq -56(%rsp,%rdi,8), %r10
+; AVX512VL-NEXT: negq %rsi
+; AVX512VL-NEXT: shlxq %rsi, %r10, %rsi
+; AVX512VL-NEXT: shrq %cl, %r10
+; AVX512VL-NEXT: movzbq %cl, %r11
+; AVX512VL-NEXT: orq %r8, %rsi
+; AVX512VL-NEXT: movq -48(%rsp,%rdi,8), %r8
+; AVX512VL-NEXT: negq %r9
+; AVX512VL-NEXT: shlxq %r9, %r8, %r9
+; AVX512VL-NEXT: orq %r10, %r9
+; AVX512VL-NEXT: movq -72(%rsp,%rdi,8), %rdi
+; AVX512VL-NEXT: negq %r11
+; AVX512VL-NEXT: shrq %cl, %rdi
+; AVX512VL-NEXT: shlxq %r11, %rdx, %rdx
+; AVX512VL-NEXT: orq %rdi, %rdx
+; AVX512VL-NEXT: shrxq %rcx, %r8, %rcx
+; AVX512VL-NEXT: movq %rcx, 24(%rax)
+; AVX512VL-NEXT: movq %r9, 16(%rax)
+; AVX512VL-NEXT: movq %rsi, 8(%rax)
+; AVX512VL-NEXT: movq %rdx, (%rax)
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VBMI-LABEL: lshr_allbits_i256:
; AVX512VBMI: # %bb.0:
+; AVX512VBMI-NEXT: movq %rsi, %rcx
+; AVX512VBMI-NEXT: movq %rdi, %rax
+; AVX512VBMI-NEXT: movzbq %cl, %rsi
; AVX512VBMI-NEXT: vpxor %xmm0, %xmm0, %xmm0
; AVX512VBMI-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: movq %rsi, %rcx
; AVX512VBMI-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
; AVX512VBMI-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: movl %ecx, %eax
-; AVX512VBMI-NEXT: shrb $6, %al
-; AVX512VBMI-NEXT: movzbl %al, %eax
-; AVX512VBMI-NEXT: movq -56(%rsp,%rax,8), %rdx
-; AVX512VBMI-NEXT: movq -64(%rsp,%rax,8), %rsi
-; AVX512VBMI-NEXT: movq %rsi, %r8
-; AVX512VBMI-NEXT: shrdq %cl, %rdx, %r8
-; AVX512VBMI-NEXT: movq -48(%rsp,%rax,8), %r9
-; AVX512VBMI-NEXT: shrdq %cl, %r9, %rdx
-; AVX512VBMI-NEXT: movq -72(%rsp,%rax,8), %r10
-; AVX512VBMI-NEXT: shrdq %cl, %rsi, %r10
-; AVX512VBMI-NEXT: movq %rdi, %rax
-; AVX512VBMI-NEXT: shrxq %rcx, %r9, %rcx
-; AVX512VBMI-NEXT: movq %rcx, 24(%rdi)
-; AVX512VBMI-NEXT: movq %rdx, 16(%rdi)
-; AVX512VBMI-NEXT: movq %r8, 8(%rdi)
-; AVX512VBMI-NEXT: movq %r10, (%rdi)
+; AVX512VBMI-NEXT: movl %ecx, %edx
+; AVX512VBMI-NEXT: shrb $6, %dl
+; AVX512VBMI-NEXT: movzbl %dl, %edi
+; AVX512VBMI-NEXT: movq -64(%rsp,%rdi,8), %rdx
+; AVX512VBMI-NEXT: movq %rdx, %r8
+; AVX512VBMI-NEXT: shrq %cl, %r8
+; AVX512VBMI-NEXT: movzbq %cl, %r9
+; AVX512VBMI-NEXT: movq -56(%rsp,%rdi,8), %r10
+; AVX512VBMI-NEXT: negq %rsi
+; AVX512VBMI-NEXT: shlxq %rsi, %r10, %rsi
+; AVX512VBMI-NEXT: shrq %cl, %r10
+; AVX512VBMI-NEXT: movzbq %cl, %r11
+; AVX512VBMI-NEXT: orq %r8, %rsi
+; AVX512VBMI-NEXT: movq -48(%rsp,%rdi,8), %r8
+; AVX512VBMI-NEXT: negq %r9
+; AVX512VBMI-NEXT: shlxq %r9, %r8, %r9
+; AVX512VBMI-NEXT: orq %r10, %r9
+; AVX512VBMI-NEXT: movq -72(%rsp,%rdi,8), %rdi
+; AVX512VBMI-NEXT: negq %r11
+; AVX512VBMI-NEXT: shrq %cl, %rdi
+; AVX512VBMI-NEXT: shlxq %r11, %rdx, %rdx
+; AVX512VBMI-NEXT: orq %rdi, %rdx
+; AVX512VBMI-NEXT: shrxq %rcx, %r8, %rcx
+; AVX512VBMI-NEXT: movq %rcx, 24(%rax)
+; AVX512VBMI-NEXT: movq %r9, 16(%rax)
+; AVX512VBMI-NEXT: movq %rsi, 8(%rax)
+; AVX512VBMI-NEXT: movq %rdx, (%rax)
; AVX512VBMI-NEXT: vzeroupper
; AVX512VBMI-NEXT: retq
;
@@ -3143,11 +3697,14 @@ define i64 @lshr_extract_i256_i64(i256 %a0, i256 %a1) nounwind {
; AVX2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movl %r8d, %eax
; AVX2-NEXT: shrb $6, %al
-; AVX2-NEXT: movzbl %al, %ecx
-; AVX2-NEXT: movq -72(%rsp,%rcx,8), %rax
-; AVX2-NEXT: movq -64(%rsp,%rcx,8), %rdx
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: movq -72(%rsp,%rax,8), %rdx
; AVX2-NEXT: movl %r8d, %ecx
-; AVX2-NEXT: shrdq %cl, %rdx, %rax
+; AVX2-NEXT: movzbq %cl, %rsi
+; AVX2-NEXT: shrq %cl, %rdx
+; AVX2-NEXT: negq %rsi
+; AVX2-NEXT: shlxq %rsi, -64(%rsp,%rax,8), %rax
+; AVX2-NEXT: orq %rdx, %rax
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
@@ -3161,11 +3718,14 @@ define i64 @lshr_extract_i256_i64(i256 %a0, i256 %a1) nounwind {
; AVX512F-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movl %r8d, %eax
; AVX512F-NEXT: shrb $6, %al
-; AVX512F-NEXT: movzbl %al, %ecx
-; AVX512F-NEXT: movq -72(%rsp,%rcx,8), %rax
-; AVX512F-NEXT: movq -64(%rsp,%rcx,8), %rdx
+; AVX512F-NEXT: movzbl %al, %eax
+; AVX512F-NEXT: movq -72(%rsp,%rax,8), %rdx
; AVX512F-NEXT: movl %r8d, %ecx
-; AVX512F-NEXT: shrdq %cl, %rdx, %rax
+; AVX512F-NEXT: movzbq %cl, %rsi
+; AVX512F-NEXT: shrq %cl, %rdx
+; AVX512F-NEXT: negq %rsi
+; AVX512F-NEXT: shlxq %rsi, -64(%rsp,%rax,8), %rax
+; AVX512F-NEXT: orq %rdx, %rax
; AVX512F-NEXT: retq
;
; AVX512VL-LABEL: lshr_extract_i256_i64:
@@ -3178,11 +3738,14 @@ define i64 @lshr_extract_i256_i64(i256 %a0, i256 %a1) nounwind {
; AVX512VL-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movl %r8d, %eax
; AVX512VL-NEXT: shrb $6, %al
-; AVX512VL-NEXT: movzbl %al, %ecx
-; AVX512VL-NEXT: movq -72(%rsp,%rcx,8), %rax
-; AVX512VL-NEXT: movq -64(%rsp,%rcx,8), %rdx
+; AVX512VL-NEXT: movzbl %al, %eax
+; AVX512VL-NEXT: movq -72(%rsp,%rax,8), %rdx
; AVX512VL-NEXT: movl %r8d, %ecx
-; AVX512VL-NEXT: shrdq %cl, %rdx, %rax
+; AVX512VL-NEXT: movzbq %cl, %rsi
+; AVX512VL-NEXT: negq %rsi
+; AVX512VL-NEXT: shrq %cl, %rdx
+; AVX512VL-NEXT: shlxq %rsi, -64(%rsp,%rax,8), %rax
+; AVX512VL-NEXT: orq %rdx, %rax
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
@@ -3196,11 +3759,14 @@ define i64 @lshr_extract_i256_i64(i256 %a0, i256 %a1) nounwind {
; AVX512VBMI-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movl %r8d, %eax
; AVX512VBMI-NEXT: shrb $6, %al
-; AVX512VBMI-NEXT: movzbl %al, %ecx
-; AVX512VBMI-NEXT: movq -72(%rsp,%rcx,8), %rax
-; AVX512VBMI-NEXT: movq -64(%rsp,%rcx,8), %rdx
+; AVX512VBMI-NEXT: movzbl %al, %eax
+; AVX512VBMI-NEXT: movq -72(%rsp,%rax,8), %rdx
; AVX512VBMI-NEXT: movl %r8d, %ecx
-; AVX512VBMI-NEXT: shrdq %cl, %rdx, %rax
+; AVX512VBMI-NEXT: movzbq %cl, %rsi
+; AVX512VBMI-NEXT: negq %rsi
+; AVX512VBMI-NEXT: shrq %cl, %rdx
+; AVX512VBMI-NEXT: shlxq %rsi, -64(%rsp,%rax,8), %rax
+; AVX512VBMI-NEXT: orq %rdx, %rax
; AVX512VBMI-NEXT: vzeroupper
; AVX512VBMI-NEXT: retq
;
@@ -3258,25 +3824,117 @@ define i64 @lshr_extract_i256_i64(i256 %a0, i256 %a1) nounwind {
}
define i64 @ashr_extract_i256_i64(i256 %a0, i256 %a1) nounwind {
-; CHECK-LABEL: ashr_extract_i256_i64:
-; CHECK: # %bb.0:
-; CHECK-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT: sarq $63, %rcx
-; CHECK-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT: movl %r8d, %eax
-; CHECK-NEXT: shrb $6, %al
-; CHECK-NEXT: movzbl %al, %ecx
-; CHECK-NEXT: movq -72(%rsp,%rcx,8), %rax
-; CHECK-NEXT: movq -64(%rsp,%rcx,8), %rdx
-; CHECK-NEXT: movl %r8d, %ecx
-; CHECK-NEXT: shrdq %cl, %rdx, %rax
-; CHECK-NEXT: retq
+; SSE-LABEL: ashr_extract_i256_i64:
+; SSE: # %bb.0:
+; SSE-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: sarq $63, %rcx
+; SSE-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movl %r8d, %eax
+; SSE-NEXT: shrb $6, %al
+; SSE-NEXT: movzbl %al, %ecx
+; SSE-NEXT: movq -72(%rsp,%rcx,8), %rax
+; SSE-NEXT: movq -64(%rsp,%rcx,8), %rdx
+; SSE-NEXT: movl %r8d, %ecx
+; SSE-NEXT: shrdq %cl, %rdx, %rax
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: ashr_extract_i256_i64:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: sarq $63, %rcx
+; AVX2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movl %r8d, %eax
+; AVX2-NEXT: shrb $6, %al
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: movq -72(%rsp,%rax,8), %rdx
+; AVX2-NEXT: movl %r8d, %ecx
+; AVX2-NEXT: movzbq %cl, %rsi
+; AVX2-NEXT: shrq %cl, %rdx
+; AVX2-NEXT: negq %rsi
+; AVX2-NEXT: shlxq %rsi, -64(%rsp,%rax,8), %rax
+; AVX2-NEXT: orq %rdx, %rax
+; AVX2-NEXT: retq
+;
+; AVX512F-LABEL: ashr_extract_i256_i64:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: sarq $63, %rcx
+; AVX512F-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movl %r8d, %eax
+; AVX512F-NEXT: shrb $6, %al
+; AVX512F-NEXT: movzbl %al, %eax
+; AVX512F-NEXT: movq -72(%rsp,%rax,8), %rdx
+; AVX512F-NEXT: movl %r8d, %ecx
+; AVX512F-NEXT: movzbq %cl, %rsi
+; AVX512F-NEXT: shrq %cl, %rdx
+; AVX512F-NEXT: negq %rsi
+; AVX512F-NEXT: shlxq %rsi, -64(%rsp,%rax,8), %rax
+; AVX512F-NEXT: orq %rdx, %rax
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: ashr_extract_i256_i64:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: sarq $63, %rcx
+; AVX512VL-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movl %r8d, %eax
+; AVX512VL-NEXT: shrb $6, %al
+; AVX512VL-NEXT: movzbl %al, %eax
+; AVX512VL-NEXT: movq -72(%rsp,%rax,8), %rdx
+; AVX512VL-NEXT: movl %r8d, %ecx
+; AVX512VL-NEXT: movzbq %cl, %rsi
+; AVX512VL-NEXT: negq %rsi
+; AVX512VL-NEXT: shrq %cl, %rdx
+; AVX512VL-NEXT: shlxq %rsi, -64(%rsp,%rax,8), %rax
+; AVX512VL-NEXT: orq %rdx, %rax
+; AVX512VL-NEXT: retq
+;
+; AVX512VBMI-LABEL: ashr_extract_i256_i64:
+; AVX512VBMI: # %bb.0:
+; AVX512VBMI-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: sarq $63, %rcx
+; AVX512VBMI-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movl %r8d, %eax
+; AVX512VBMI-NEXT: shrb $6, %al
+; AVX512VBMI-NEXT: movzbl %al, %eax
+; AVX512VBMI-NEXT: movq -72(%rsp,%rax,8), %rdx
+; AVX512VBMI-NEXT: movl %r8d, %ecx
+; AVX512VBMI-NEXT: movzbq %cl, %rsi
+; AVX512VBMI-NEXT: negq %rsi
+; AVX512VBMI-NEXT: shrq %cl, %rdx
+; AVX512VBMI-NEXT: shlxq %rsi, -64(%rsp,%rax,8), %rax
+; AVX512VBMI-NEXT: orq %rdx, %rax
+; AVX512VBMI-NEXT: retq
;
; X86-LABEL: ashr_extract_i256_i64:
; X86: # %bb.0:
@@ -3507,48 +4165,102 @@ define i64 @ashr_extract_load_i256_i64(ptr %p0, i256 %a1) nounwind {
; AVX2-LABEL: ashr_extract_load_i256_i64:
; AVX2: # %bb.0:
; AVX2-NEXT: movq %rsi, %rcx
+; AVX2-NEXT: movzbq %cl, %rax
; AVX2-NEXT: vmovaps (%rdi), %xmm0
-; AVX2-NEXT: movq 16(%rdi), %rax
-; AVX2-NEXT: movq 24(%rdi), %rdx
+; AVX2-NEXT: movq 16(%rdi), %rdx
+; AVX2-NEXT: movq 24(%rdi), %rsi
; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: sarq $63, %rdx
-; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movl %ecx, %eax
-; AVX2-NEXT: shrb $6, %al
-; AVX2-NEXT: movzbl %al, %edx
-; AVX2-NEXT: movq -72(%rsp,%rdx,8), %rax
-; AVX2-NEXT: movq -64(%rsp,%rdx,8), %rdx
-; AVX2-NEXT: # kill: def $cl killed $cl killed $rcx
-; AVX2-NEXT: shrdq %cl, %rdx, %rax
+; AVX2-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: sarq $63, %rsi
+; AVX2-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movl %ecx, %edx
+; AVX2-NEXT: shrb $6, %dl
+; AVX2-NEXT: movzbl %dl, %edx
+; AVX2-NEXT: movq -72(%rsp,%rdx,8), %rsi
+; AVX2-NEXT: shrq %cl, %rsi
+; AVX2-NEXT: negq %rax
+; AVX2-NEXT: shlxq %rax, -64(%rsp,%rdx,8), %rax
+; AVX2-NEXT: orq %rsi, %rax
; AVX2-NEXT: retq
;
-; AVX512-LABEL: ashr_extract_load_i256_i64:
-; AVX512: # %bb.0:
-; AVX512-NEXT: movq %rsi, %rcx
-; AVX512-NEXT: vmovaps (%rdi), %xmm0
-; AVX512-NEXT: movq 16(%rdi), %rax
-; AVX512-NEXT: movq 24(%rdi), %rdx
-; AVX512-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; AVX512-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
-; AVX512-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
-; AVX512-NEXT: sarq $63, %rdx
-; AVX512-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; AVX512-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; AVX512-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; AVX512-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; AVX512-NEXT: movl %ecx, %eax
-; AVX512-NEXT: shrb $6, %al
-; AVX512-NEXT: movzbl %al, %edx
-; AVX512-NEXT: movq -72(%rsp,%rdx,8), %rax
-; AVX512-NEXT: movq -64(%rsp,%rdx,8), %rdx
-; AVX512-NEXT: # kill: def $cl killed $cl killed $rcx
-; AVX512-NEXT: shrdq %cl, %rdx, %rax
-; AVX512-NEXT: retq
+; AVX512F-LABEL: ashr_extract_load_i256_i64:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: movq %rsi, %rcx
+; AVX512F-NEXT: movzbq %cl, %rax
+; AVX512F-NEXT: vmovaps (%rdi), %xmm0
+; AVX512F-NEXT: movq 16(%rdi), %rdx
+; AVX512F-NEXT: movq 24(%rdi), %rsi
+; AVX512F-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: sarq $63, %rsi
+; AVX512F-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movl %ecx, %edx
+; AVX512F-NEXT: shrb $6, %dl
+; AVX512F-NEXT: movzbl %dl, %edx
+; AVX512F-NEXT: movq -72(%rsp,%rdx,8), %rsi
+; AVX512F-NEXT: shrq %cl, %rsi
+; AVX512F-NEXT: negq %rax
+; AVX512F-NEXT: shlxq %rax, -64(%rsp,%rdx,8), %rax
+; AVX512F-NEXT: orq %rsi, %rax
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: ashr_extract_load_i256_i64:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: movq %rsi, %rcx
+; AVX512VL-NEXT: movzbq %cl, %rax
+; AVX512VL-NEXT: vmovaps (%rdi), %xmm0
+; AVX512VL-NEXT: movq 16(%rdi), %rdx
+; AVX512VL-NEXT: movq 24(%rdi), %rsi
+; AVX512VL-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: sarq $63, %rsi
+; AVX512VL-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movl %ecx, %edx
+; AVX512VL-NEXT: shrb $6, %dl
+; AVX512VL-NEXT: movzbl %dl, %edx
+; AVX512VL-NEXT: movq -72(%rsp,%rdx,8), %rsi
+; AVX512VL-NEXT: negq %rax
+; AVX512VL-NEXT: shrq %cl, %rsi
+; AVX512VL-NEXT: shlxq %rax, -64(%rsp,%rdx,8), %rax
+; AVX512VL-NEXT: orq %rsi, %rax
+; AVX512VL-NEXT: retq
+;
+; AVX512VBMI-LABEL: ashr_extract_load_i256_i64:
+; AVX512VBMI: # %bb.0:
+; AVX512VBMI-NEXT: movq %rsi, %rcx
+; AVX512VBMI-NEXT: movzbq %cl, %rax
+; AVX512VBMI-NEXT: vmovaps (%rdi), %xmm0
+; AVX512VBMI-NEXT: movq 16(%rdi), %rdx
+; AVX512VBMI-NEXT: movq 24(%rdi), %rsi
+; AVX512VBMI-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: sarq $63, %rsi
+; AVX512VBMI-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movl %ecx, %edx
+; AVX512VBMI-NEXT: shrb $6, %dl
+; AVX512VBMI-NEXT: movzbl %dl, %edx
+; AVX512VBMI-NEXT: movq -72(%rsp,%rdx,8), %rsi
+; AVX512VBMI-NEXT: negq %rax
+; AVX512VBMI-NEXT: shrq %cl, %rsi
+; AVX512VBMI-NEXT: shlxq %rax, -64(%rsp,%rdx,8), %rax
+; AVX512VBMI-NEXT: orq %rsi, %rax
+; AVX512VBMI-NEXT: retq
;
; X86-LABEL: ashr_extract_load_i256_i64:
; X86: # %bb.0:
diff --git a/llvm/test/CodeGen/X86/shift-i512.ll b/llvm/test/CodeGen/X86/shift-i512.ll
index 765665b904f25..f1f24e479dc2d 100644
--- a/llvm/test/CodeGen/X86/shift-i512.ll
+++ b/llvm/test/CodeGen/X86/shift-i512.ll
@@ -72,10 +72,13 @@ define i512 @shl_i512(i512 %a0, i512 %a1) nounwind {
;
; AVX2-LABEL: shl_i512:
; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %r15
; AVX2-NEXT: pushq %r14
+; AVX2-NEXT: pushq %r12
; AVX2-NEXT: pushq %rbx
; AVX2-NEXT: pushq %rax
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdi
; AVX2-NEXT: vmovaps {{[0-9]+}}(%rsp), %xmm0
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10
; AVX2-NEXT: movq %r10, -{{[0-9]+}}(%rsp)
@@ -88,46 +91,75 @@ define i512 @shl_i512(i512 %a0, i512 %a1) nounwind {
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movl %eax, %ecx
+; AVX2-NEXT: movl %edi, %ecx
; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: shrl $3, %eax
-; AVX2-NEXT: andl $56, %eax
-; AVX2-NEXT: negl %eax
-; AVX2-NEXT: movslq %eax, %r8
-; AVX2-NEXT: movq -56(%rsp,%r8), %rdx
-; AVX2-NEXT: movq -48(%rsp,%r8), %rax
-; AVX2-NEXT: movq %rax, %rsi
-; AVX2-NEXT: shldq %cl, %rdx, %rsi
-; AVX2-NEXT: movq -40(%rsp,%r8), %r10
-; AVX2-NEXT: movq %r10, %r9
-; AVX2-NEXT: shldq %cl, %rax, %r9
-; AVX2-NEXT: movq -32(%rsp,%r8), %rax
-; AVX2-NEXT: movq %rax, %r11
-; AVX2-NEXT: shldq %cl, %r10, %r11
-; AVX2-NEXT: movq -24(%rsp,%r8), %r10
-; AVX2-NEXT: movq %r10, %rbx
-; AVX2-NEXT: shldq %cl, %rax, %rbx
-; AVX2-NEXT: movq -16(%rsp,%r8), %rax
-; AVX2-NEXT: movq %rax, %r14
-; AVX2-NEXT: shldq %cl, %r10, %r14
-; AVX2-NEXT: movq -8(%rsp,%r8), %r10
-; AVX2-NEXT: shldq %cl, %rax, %r10
-; AVX2-NEXT: movq %rdi, %rax
-; AVX2-NEXT: movq -64(%rsp,%r8), %rdi
-; AVX2-NEXT: shlxq %rcx, %rdi, %r8
+; AVX2-NEXT: shrl $3, %edi
+; AVX2-NEXT: andl $56, %edi
+; AVX2-NEXT: negl %edi
+; AVX2-NEXT: movslq %edi, %r9
+; AVX2-NEXT: movq -64(%rsp,%r9), %rsi
+; AVX2-NEXT: movq -56(%rsp,%r9), %rdx
+; AVX2-NEXT: movq -48(%rsp,%r9), %r8
+; AVX2-NEXT: movq %r8, %r10
+; AVX2-NEXT: shlq %cl, %r10
+; AVX2-NEXT: movzbq %cl, %rdi
+; AVX2-NEXT: negq %rdi
+; AVX2-NEXT: shrxq %rdi, %rdx, %rdi
+; AVX2-NEXT: orq %r10, %rdi
+; AVX2-NEXT: movq -40(%rsp,%r9), %r10
+; AVX2-NEXT: movq %r10, %r11
+; AVX2-NEXT: shlq %cl, %r11
+; AVX2-NEXT: movzbq %cl, %rbx
+; AVX2-NEXT: negq %rbx
+; AVX2-NEXT: shrxq %rbx, %r8, %r8
+; AVX2-NEXT: orq %r11, %r8
+; AVX2-NEXT: movq -32(%rsp,%r9), %r11
+; AVX2-NEXT: movq %r11, %rbx
+; AVX2-NEXT: shlq %cl, %rbx
+; AVX2-NEXT: movzbq %cl, %r14
+; AVX2-NEXT: negq %r14
+; AVX2-NEXT: shrxq %r14, %r10, %r10
+; AVX2-NEXT: orq %rbx, %r10
+; AVX2-NEXT: movq -24(%rsp,%r9), %rbx
+; AVX2-NEXT: movq %rbx, %r14
+; AVX2-NEXT: shlq %cl, %r14
+; AVX2-NEXT: movzbq %cl, %r15
+; AVX2-NEXT: negq %r15
+; AVX2-NEXT: shrxq %r15, %r11, %r11
+; AVX2-NEXT: orq %r14, %r11
+; AVX2-NEXT: movq -16(%rsp,%r9), %r14
+; AVX2-NEXT: movq %r14, %r15
+; AVX2-NEXT: shlq %cl, %r15
+; AVX2-NEXT: movzbq %cl, %r12
+; AVX2-NEXT: negq %r12
+; AVX2-NEXT: shrxq %r12, %rbx, %rbx
+; AVX2-NEXT: orq %r15, %rbx
+; AVX2-NEXT: movq -8(%rsp,%r9), %r9
+; AVX2-NEXT: shlq %cl, %r9
+; AVX2-NEXT: movzbq %cl, %r15
+; AVX2-NEXT: negq %r15
+; AVX2-NEXT: shrxq %r15, %r14, %r14
+; AVX2-NEXT: orq %r9, %r14
+; AVX2-NEXT: shlxq %rcx, %rsi, %r9
; AVX2-NEXT: # kill: def $cl killed $cl killed $rcx
-; AVX2-NEXT: shldq %cl, %rdi, %rdx
-; AVX2-NEXT: movq %r10, 56(%rax)
-; AVX2-NEXT: movq %r14, 48(%rax)
-; AVX2-NEXT: movq %rbx, 40(%rax)
-; AVX2-NEXT: movq %r11, 32(%rax)
-; AVX2-NEXT: movq %r9, 24(%rax)
-; AVX2-NEXT: movq %rsi, 16(%rax)
-; AVX2-NEXT: movq %rdx, 8(%rax)
-; AVX2-NEXT: movq %r8, (%rax)
+; AVX2-NEXT: shlq %cl, %rdx
+; AVX2-NEXT: movzbq %cl, %rcx
+; AVX2-NEXT: negq %rcx
+; AVX2-NEXT: shrxq %rcx, %rsi, %rcx
+; AVX2-NEXT: orq %rdx, %rcx
+; AVX2-NEXT: movq %r14, 56(%rax)
+; AVX2-NEXT: movq %rbx, 48(%rax)
+; AVX2-NEXT: movq %r11, 40(%rax)
+; AVX2-NEXT: movq %r10, 32(%rax)
+; AVX2-NEXT: movq %r8, 24(%rax)
+; AVX2-NEXT: movq %rdi, 16(%rax)
+; AVX2-NEXT: movq %rcx, 8(%rax)
+; AVX2-NEXT: movq %r9, (%rax)
; AVX2-NEXT: addq $8, %rsp
; AVX2-NEXT: popq %rbx
+; AVX2-NEXT: popq %r12
; AVX2-NEXT: popq %r14
+; AVX2-NEXT: popq %r15
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
@@ -302,7 +334,8 @@ define i512 @lshr_i512(i512 %a0, i512 %a1) nounwind {
; AVX2-NEXT: pushq %r15
; AVX2-NEXT: pushq %r14
; AVX2-NEXT: pushq %rbx
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdi
; AVX2-NEXT: vmovaps {{[0-9]+}}(%rsp), %xmm0
; AVX2-NEXT: vxorps %xmm1, %xmm1, %xmm1
; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
@@ -315,36 +348,63 @@ define i512 @lshr_i512(i512 %a0, i512 %a1) nounwind {
; AVX2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movl %eax, %ecx
+; AVX2-NEXT: movl %edi, %ecx
; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: shrl $3, %eax
-; AVX2-NEXT: andl $56, %eax
-; AVX2-NEXT: movq -112(%rsp,%rax), %rdx
-; AVX2-NEXT: movq -120(%rsp,%rax), %r9
-; AVX2-NEXT: movq %r9, %rsi
-; AVX2-NEXT: shrdq %cl, %rdx, %rsi
-; AVX2-NEXT: movq -104(%rsp,%rax), %r8
-; AVX2-NEXT: shrdq %cl, %r8, %rdx
-; AVX2-NEXT: movq -96(%rsp,%rax), %r10
-; AVX2-NEXT: shrdq %cl, %r10, %r8
-; AVX2-NEXT: movq -88(%rsp,%rax), %r11
-; AVX2-NEXT: shrdq %cl, %r11, %r10
-; AVX2-NEXT: movq -80(%rsp,%rax), %rbx
-; AVX2-NEXT: shrdq %cl, %rbx, %r11
-; AVX2-NEXT: movq -128(%rsp,%rax), %r14
-; AVX2-NEXT: movq -72(%rsp,%rax), %r15
-; AVX2-NEXT: shrdq %cl, %r15, %rbx
-; AVX2-NEXT: shrdq %cl, %r9, %r14
-; AVX2-NEXT: movq %rdi, %rax
-; AVX2-NEXT: shrxq %rcx, %r15, %rcx
-; AVX2-NEXT: movq %rcx, 56(%rdi)
-; AVX2-NEXT: movq %rbx, 48(%rdi)
-; AVX2-NEXT: movq %r11, 40(%rdi)
-; AVX2-NEXT: movq %r10, 32(%rdi)
-; AVX2-NEXT: movq %r8, 24(%rdi)
-; AVX2-NEXT: movq %rdx, 16(%rdi)
-; AVX2-NEXT: movq %rsi, 8(%rdi)
-; AVX2-NEXT: movq %r14, (%rdi)
+; AVX2-NEXT: shrl $3, %edi
+; AVX2-NEXT: andl $56, %edi
+; AVX2-NEXT: movq -112(%rsp,%rdi), %r10
+; AVX2-NEXT: movq -128(%rsp,%rdi), %rdx
+; AVX2-NEXT: movq -120(%rsp,%rdi), %r8
+; AVX2-NEXT: movq %r8, %r9
+; AVX2-NEXT: shrq %cl, %r9
+; AVX2-NEXT: movzbq %cl, %rsi
+; AVX2-NEXT: negq %rsi
+; AVX2-NEXT: shlxq %rsi, %r10, %rsi
+; AVX2-NEXT: orq %r9, %rsi
+; AVX2-NEXT: movq -104(%rsp,%rdi), %r11
+; AVX2-NEXT: shrq %cl, %r10
+; AVX2-NEXT: movzbq %cl, %r9
+; AVX2-NEXT: negq %r9
+; AVX2-NEXT: shlxq %r9, %r11, %r9
+; AVX2-NEXT: orq %r10, %r9
+; AVX2-NEXT: movq -96(%rsp,%rdi), %rbx
+; AVX2-NEXT: shrq %cl, %r11
+; AVX2-NEXT: movzbq %cl, %r10
+; AVX2-NEXT: negq %r10
+; AVX2-NEXT: shlxq %r10, %rbx, %r10
+; AVX2-NEXT: orq %r11, %r10
+; AVX2-NEXT: movq -88(%rsp,%rdi), %r14
+; AVX2-NEXT: shrq %cl, %rbx
+; AVX2-NEXT: movzbq %cl, %r11
+; AVX2-NEXT: negq %r11
+; AVX2-NEXT: shlxq %r11, %r14, %r11
+; AVX2-NEXT: orq %rbx, %r11
+; AVX2-NEXT: movq -80(%rsp,%rdi), %rbx
+; AVX2-NEXT: shrq %cl, %r14
+; AVX2-NEXT: movzbq %cl, %r15
+; AVX2-NEXT: negq %r15
+; AVX2-NEXT: shlxq %r15, %rbx, %r15
+; AVX2-NEXT: orq %r14, %r15
+; AVX2-NEXT: movq -72(%rsp,%rdi), %rdi
+; AVX2-NEXT: shrq %cl, %rbx
+; AVX2-NEXT: movzbq %cl, %r14
+; AVX2-NEXT: negq %r14
+; AVX2-NEXT: shlxq %r14, %rdi, %r14
+; AVX2-NEXT: orq %rbx, %r14
+; AVX2-NEXT: movzbq %cl, %rbx
+; AVX2-NEXT: shrq %cl, %rdx
+; AVX2-NEXT: negq %rbx
+; AVX2-NEXT: shlxq %rbx, %r8, %r8
+; AVX2-NEXT: orq %rdx, %r8
+; AVX2-NEXT: shrxq %rcx, %rdi, %rcx
+; AVX2-NEXT: movq %rcx, 56(%rax)
+; AVX2-NEXT: movq %r14, 48(%rax)
+; AVX2-NEXT: movq %r15, 40(%rax)
+; AVX2-NEXT: movq %r11, 32(%rax)
+; AVX2-NEXT: movq %r10, 24(%rax)
+; AVX2-NEXT: movq %r9, 16(%rax)
+; AVX2-NEXT: movq %rsi, 8(%rax)
+; AVX2-NEXT: movq %r8, (%rax)
; AVX2-NEXT: popq %rbx
; AVX2-NEXT: popq %r14
; AVX2-NEXT: popq %r15
@@ -520,11 +580,12 @@ define i512 @ashr_i512(i512 %a0, i512 %a1) nounwind {
; AVX2-NEXT: pushq %r15
; AVX2-NEXT: pushq %r14
; AVX2-NEXT: pushq %rbx
+; AVX2-NEXT: movq %rdi, %rax
; AVX2-NEXT: vmovaps {{[0-9]+}}(%rsp), %xmm0
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10
; AVX2-NEXT: movq %r10, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovups %xmm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdi
; AVX2-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
@@ -539,36 +600,63 @@ define i512 @ashr_i512(i512 %a0, i512 %a1) nounwind {
; AVX2-NEXT: movq %r10, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %r10, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %r10, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movl %eax, %ecx
+; AVX2-NEXT: movl %edi, %ecx
; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: shrl $3, %eax
-; AVX2-NEXT: andl $56, %eax
-; AVX2-NEXT: movq -112(%rsp,%rax), %rdx
-; AVX2-NEXT: movq -120(%rsp,%rax), %r9
-; AVX2-NEXT: movq %r9, %rsi
-; AVX2-NEXT: shrdq %cl, %rdx, %rsi
-; AVX2-NEXT: movq -104(%rsp,%rax), %r8
-; AVX2-NEXT: shrdq %cl, %r8, %rdx
-; AVX2-NEXT: movq -96(%rsp,%rax), %r10
-; AVX2-NEXT: shrdq %cl, %r10, %r8
-; AVX2-NEXT: movq -88(%rsp,%rax), %r11
-; AVX2-NEXT: shrdq %cl, %r11, %r10
-; AVX2-NEXT: movq -80(%rsp,%rax), %rbx
-; AVX2-NEXT: shrdq %cl, %rbx, %r11
-; AVX2-NEXT: movq -128(%rsp,%rax), %r14
-; AVX2-NEXT: movq -72(%rsp,%rax), %r15
-; AVX2-NEXT: shrdq %cl, %r15, %rbx
-; AVX2-NEXT: shrdq %cl, %r9, %r14
-; AVX2-NEXT: movq %rdi, %rax
-; AVX2-NEXT: sarxq %rcx, %r15, %rcx
-; AVX2-NEXT: movq %rcx, 56(%rdi)
-; AVX2-NEXT: movq %rbx, 48(%rdi)
-; AVX2-NEXT: movq %r11, 40(%rdi)
-; AVX2-NEXT: movq %r10, 32(%rdi)
-; AVX2-NEXT: movq %r8, 24(%rdi)
-; AVX2-NEXT: movq %rdx, 16(%rdi)
-; AVX2-NEXT: movq %rsi, 8(%rdi)
-; AVX2-NEXT: movq %r14, (%rdi)
+; AVX2-NEXT: shrl $3, %edi
+; AVX2-NEXT: andl $56, %edi
+; AVX2-NEXT: movq -112(%rsp,%rdi), %r10
+; AVX2-NEXT: movq -128(%rsp,%rdi), %rdx
+; AVX2-NEXT: movq -120(%rsp,%rdi), %r8
+; AVX2-NEXT: movq %r8, %r9
+; AVX2-NEXT: shrq %cl, %r9
+; AVX2-NEXT: movzbq %cl, %rsi
+; AVX2-NEXT: negq %rsi
+; AVX2-NEXT: shlxq %rsi, %r10, %rsi
+; AVX2-NEXT: orq %r9, %rsi
+; AVX2-NEXT: movq -104(%rsp,%rdi), %r11
+; AVX2-NEXT: shrq %cl, %r10
+; AVX2-NEXT: movzbq %cl, %r9
+; AVX2-NEXT: negq %r9
+; AVX2-NEXT: shlxq %r9, %r11, %r9
+; AVX2-NEXT: orq %r10, %r9
+; AVX2-NEXT: movq -96(%rsp,%rdi), %rbx
+; AVX2-NEXT: shrq %cl, %r11
+; AVX2-NEXT: movzbq %cl, %r10
+; AVX2-NEXT: negq %r10
+; AVX2-NEXT: shlxq %r10, %rbx, %r10
+; AVX2-NEXT: orq %r11, %r10
+; AVX2-NEXT: movq -88(%rsp,%rdi), %r14
+; AVX2-NEXT: shrq %cl, %rbx
+; AVX2-NEXT: movzbq %cl, %r11
+; AVX2-NEXT: negq %r11
+; AVX2-NEXT: shlxq %r11, %r14, %r11
+; AVX2-NEXT: orq %rbx, %r11
+; AVX2-NEXT: movq -80(%rsp,%rdi), %rbx
+; AVX2-NEXT: shrq %cl, %r14
+; AVX2-NEXT: movzbq %cl, %r15
+; AVX2-NEXT: negq %r15
+; AVX2-NEXT: shlxq %r15, %rbx, %r15
+; AVX2-NEXT: orq %r14, %r15
+; AVX2-NEXT: movq -72(%rsp,%rdi), %rdi
+; AVX2-NEXT: shrq %cl, %rbx
+; AVX2-NEXT: movzbq %cl, %r14
+; AVX2-NEXT: negq %r14
+; AVX2-NEXT: shlxq %r14, %rdi, %r14
+; AVX2-NEXT: orq %rbx, %r14
+; AVX2-NEXT: movzbq %cl, %rbx
+; AVX2-NEXT: shrq %cl, %rdx
+; AVX2-NEXT: negq %rbx
+; AVX2-NEXT: shlxq %rbx, %r8, %r8
+; AVX2-NEXT: orq %rdx, %r8
+; AVX2-NEXT: sarxq %rcx, %rdi, %rcx
+; AVX2-NEXT: movq %rcx, 56(%rax)
+; AVX2-NEXT: movq %r14, 48(%rax)
+; AVX2-NEXT: movq %r15, 40(%rax)
+; AVX2-NEXT: movq %r11, 32(%rax)
+; AVX2-NEXT: movq %r10, 24(%rax)
+; AVX2-NEXT: movq %r9, 16(%rax)
+; AVX2-NEXT: movq %rsi, 8(%rax)
+; AVX2-NEXT: movq %r8, (%rax)
; AVX2-NEXT: popq %rbx
; AVX2-NEXT: popq %r14
; AVX2-NEXT: popq %r15
@@ -764,9 +852,13 @@ define i512 @shl_i512_load(ptr %p0, i512 %a1) nounwind {
;
; AVX2-LABEL: shl_i512_load:
; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %rbp
+; AVX2-NEXT: pushq %r15
; AVX2-NEXT: pushq %r14
+; AVX2-NEXT: pushq %r13
+; AVX2-NEXT: pushq %r12
; AVX2-NEXT: pushq %rbx
-; AVX2-NEXT: pushq %rax
+; AVX2-NEXT: subq $24, %rsp
; AVX2-NEXT: vmovups (%rsi), %ymm0
; AVX2-NEXT: vmovups 32(%rsi), %ymm1
; AVX2-NEXT: vxorps %xmm2, %xmm2, %xmm2
@@ -779,41 +871,72 @@ define i512 @shl_i512_load(ptr %p0, i512 %a1) nounwind {
; AVX2-NEXT: shrl $3, %edx
; AVX2-NEXT: andl $56, %edx
; AVX2-NEXT: negl %edx
-; AVX2-NEXT: movslq %edx, %r8
-; AVX2-NEXT: movq -56(%rsp,%r8), %rdx
-; AVX2-NEXT: movq -48(%rsp,%r8), %rax
-; AVX2-NEXT: movq %rax, %rsi
-; AVX2-NEXT: shldq %cl, %rdx, %rsi
-; AVX2-NEXT: movq -40(%rsp,%r8), %r10
-; AVX2-NEXT: movq %r10, %r9
-; AVX2-NEXT: shldq %cl, %rax, %r9
-; AVX2-NEXT: movq -32(%rsp,%r8), %rax
-; AVX2-NEXT: movq %rax, %r11
-; AVX2-NEXT: shldq %cl, %r10, %r11
-; AVX2-NEXT: movq -24(%rsp,%r8), %r10
+; AVX2-NEXT: movslq %edx, %rdx
+; AVX2-NEXT: movq -32(%rsp,%rdx), %r10
; AVX2-NEXT: movq %r10, %rbx
-; AVX2-NEXT: shldq %cl, %rax, %rbx
-; AVX2-NEXT: movq -16(%rsp,%r8), %rax
-; AVX2-NEXT: movq %rax, %r14
-; AVX2-NEXT: shldq %cl, %r10, %r14
-; AVX2-NEXT: movq -8(%rsp,%r8), %r10
-; AVX2-NEXT: shldq %cl, %rax, %r10
+; AVX2-NEXT: shlq %cl, %rbx
+; AVX2-NEXT: movzbq %cl, %rsi
+; AVX2-NEXT: movzbq %cl, %r12
+; AVX2-NEXT: movq -24(%rsp,%rdx), %r11
+; AVX2-NEXT: movq %r11, %rbp
+; AVX2-NEXT: shlq %cl, %rbp
+; AVX2-NEXT: movzbq %cl, %r13
+; AVX2-NEXT: movq -16(%rsp,%rdx), %r9
+; AVX2-NEXT: movq %r9, %r15
+; AVX2-NEXT: shlq %cl, %r15
+; AVX2-NEXT: movq -8(%rsp,%rdx), %r14
+; AVX2-NEXT: movq %r14, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: shlq %cl, %r14
; AVX2-NEXT: movq %rdi, %rax
-; AVX2-NEXT: movq -64(%rsp,%r8), %rdi
-; AVX2-NEXT: shlxq %rcx, %rdi, %r8
-; AVX2-NEXT: # kill: def $cl killed $cl killed $rcx
-; AVX2-NEXT: shldq %cl, %rdi, %rdx
-; AVX2-NEXT: movq %r10, 56(%rax)
-; AVX2-NEXT: movq %r14, 48(%rax)
-; AVX2-NEXT: movq %rbx, 40(%rax)
-; AVX2-NEXT: movq %r11, 32(%rax)
-; AVX2-NEXT: movq %r9, 24(%rax)
+; AVX2-NEXT: negq %rsi
+; AVX2-NEXT: movq -40(%rsp,%rdx), %r8
+; AVX2-NEXT: shrxq %rsi, %r8, %rsi
+; AVX2-NEXT: orq %rbx, %rsi
+; AVX2-NEXT: negq %r12
+; AVX2-NEXT: shrxq %r12, %r10, %rdi
+; AVX2-NEXT: orq %rbp, %rdi
+; AVX2-NEXT: movzbq %cl, %r12
+; AVX2-NEXT: movzbq %cl, %rbx
+; AVX2-NEXT: negq %r13
+; AVX2-NEXT: shrxq %r13, %r11, %r10
+; AVX2-NEXT: orq %r15, %r10
+; AVX2-NEXT: negq %r12
+; AVX2-NEXT: shrxq %r12, %r9, %r11
+; AVX2-NEXT: movq (%rsp,%rdx), %r15
+; AVX2-NEXT: movq %r15, %r9
+; AVX2-NEXT: shlq %cl, %r9
+; AVX2-NEXT: movq 8(%rsp,%rdx), %r12
+; AVX2-NEXT: shlq %cl, %r12
+; AVX2-NEXT: orq %r14, %r11
+; AVX2-NEXT: movzbq %cl, %r14
+; AVX2-NEXT: shlq %cl, %r8
+; AVX2-NEXT: negq %rbx
+; AVX2-NEXT: shrxq %rbx, {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Folded Reload
+; AVX2-NEXT: movq -48(%rsp,%rdx), %rdx
+; AVX2-NEXT: orq %r9, %rbx
+; AVX2-NEXT: negq %r14
+; AVX2-NEXT: shrxq %r14, %r15, %r9
+; AVX2-NEXT: orq %r12, %r9
+; AVX2-NEXT: shlxq %rcx, %rdx, %r14
+; AVX2-NEXT: movzbq %cl, %rcx
+; AVX2-NEXT: negq %rcx
+; AVX2-NEXT: shrxq %rcx, %rdx, %rcx
+; AVX2-NEXT: orq %r8, %rcx
+; AVX2-NEXT: movq %r9, 56(%rax)
+; AVX2-NEXT: movq %rbx, 48(%rax)
+; AVX2-NEXT: movq %r11, 40(%rax)
+; AVX2-NEXT: movq %r10, 32(%rax)
+; AVX2-NEXT: movq %rdi, 24(%rax)
; AVX2-NEXT: movq %rsi, 16(%rax)
-; AVX2-NEXT: movq %rdx, 8(%rax)
-; AVX2-NEXT: movq %r8, (%rax)
-; AVX2-NEXT: addq $8, %rsp
+; AVX2-NEXT: movq %rcx, 8(%rax)
+; AVX2-NEXT: movq %r14, (%rax)
+; AVX2-NEXT: addq $24, %rsp
; AVX2-NEXT: popq %rbx
+; AVX2-NEXT: popq %r12
+; AVX2-NEXT: popq %r13
; AVX2-NEXT: popq %r14
+; AVX2-NEXT: popq %r15
+; AVX2-NEXT: popq %rbp
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
@@ -941,9 +1064,16 @@ define i512 @lshr_i512_load(ptr %p0, i512 %a1) nounwind {
;
; AVX2-LABEL: lshr_i512_load:
; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %rbp
+; AVX2-NEXT: pushq %r15
; AVX2-NEXT: pushq %r14
+; AVX2-NEXT: pushq %r13
+; AVX2-NEXT: pushq %r12
; AVX2-NEXT: pushq %rbx
-; AVX2-NEXT: pushq %rax
+; AVX2-NEXT: subq $24, %rsp
+; AVX2-NEXT: movl %edx, %ecx
+; AVX2-NEXT: andl $63, %ecx
+; AVX2-NEXT: movzbq %cl, %rax
; AVX2-NEXT: vmovups (%rsi), %ymm0
; AVX2-NEXT: vmovups 32(%rsi), %ymm1
; AVX2-NEXT: vxorps %xmm2, %xmm2, %xmm2
@@ -951,39 +1081,68 @@ define i512 @lshr_i512_load(ptr %p0, i512 %a1) nounwind {
; AVX2-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movl %edx, %ecx
-; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: shrl $3, %edx
; AVX2-NEXT: andl $56, %edx
-; AVX2-NEXT: movq -112(%rsp,%rdx), %rsi
-; AVX2-NEXT: movq -120(%rsp,%rdx), %rax
-; AVX2-NEXT: movq %rax, %r8
-; AVX2-NEXT: shrdq %cl, %rsi, %r8
-; AVX2-NEXT: movq -104(%rsp,%rdx), %r9
-; AVX2-NEXT: shrdq %cl, %r9, %rsi
; AVX2-NEXT: movq -96(%rsp,%rdx), %r10
-; AVX2-NEXT: shrdq %cl, %r10, %r9
-; AVX2-NEXT: movq -88(%rsp,%rdx), %r11
-; AVX2-NEXT: shrdq %cl, %r11, %r10
-; AVX2-NEXT: movq -80(%rsp,%rdx), %rbx
-; AVX2-NEXT: shrdq %cl, %rbx, %r11
-; AVX2-NEXT: movq -128(%rsp,%rdx), %r14
-; AVX2-NEXT: movq -72(%rsp,%rdx), %rdx
-; AVX2-NEXT: shrdq %cl, %rdx, %rbx
-; AVX2-NEXT: shrdq %cl, %rax, %r14
+; AVX2-NEXT: movq -104(%rsp,%rdx), %r11
+; AVX2-NEXT: movq %r11, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: negq %rax
+; AVX2-NEXT: shrq %cl, %r11
+; AVX2-NEXT: shlxq %rax, %r10, %rsi
+; AVX2-NEXT: shrq %cl, %r10
+; AVX2-NEXT: movzbq %cl, %rax
+; AVX2-NEXT: movzbq %cl, %rbx
+; AVX2-NEXT: movq -88(%rsp,%rdx), %r14
+; AVX2-NEXT: negq %rax
+; AVX2-NEXT: shlxq %rax, %r14, %r9
+; AVX2-NEXT: shrq %cl, %r14
+; AVX2-NEXT: movzbq %cl, %rax
+; AVX2-NEXT: movq -80(%rsp,%rdx), %r12
+; AVX2-NEXT: negq %rbx
+; AVX2-NEXT: shlxq %rbx, %r12, %rbx
+; AVX2-NEXT: shrq %cl, %r12
+; AVX2-NEXT: movzbq %cl, %r13
+; AVX2-NEXT: movq -72(%rsp,%rdx), %rbp
+; AVX2-NEXT: negq %rax
+; AVX2-NEXT: shlxq %rax, %rbp, %r15
+; AVX2-NEXT: movq -64(%rsp,%rdx), %r8
+; AVX2-NEXT: negq %r13
+; AVX2-NEXT: shrq %cl, %rbp
+; AVX2-NEXT: shlxq %r13, %r8, %r13
+; AVX2-NEXT: shrq %cl, %r8
; AVX2-NEXT: movq %rdi, %rax
-; AVX2-NEXT: shrxq %rcx, %rdx, %rcx
-; AVX2-NEXT: movq %rcx, 56(%rdi)
-; AVX2-NEXT: movq %rbx, 48(%rdi)
-; AVX2-NEXT: movq %r11, 40(%rdi)
-; AVX2-NEXT: movq %r10, 32(%rdi)
-; AVX2-NEXT: movq %r9, 24(%rdi)
-; AVX2-NEXT: movq %rsi, 16(%rdi)
-; AVX2-NEXT: movq %r8, 8(%rdi)
-; AVX2-NEXT: movq %r14, (%rdi)
-; AVX2-NEXT: addq $8, %rsp
+; AVX2-NEXT: orq %r11, %rsi
+; AVX2-NEXT: orq %r10, %r9
+; AVX2-NEXT: orq %r14, %rbx
+; AVX2-NEXT: movzbq %cl, %rdi
+; AVX2-NEXT: movzbq %cl, %r10
+; AVX2-NEXT: orq %r12, %r15
+; AVX2-NEXT: orq %rbp, %r13
+; AVX2-NEXT: movq -56(%rsp,%rdx), %r11
+; AVX2-NEXT: negq %rdi
+; AVX2-NEXT: shlxq %rdi, %r11, %rdi
+; AVX2-NEXT: orq %r8, %rdi
+; AVX2-NEXT: movq -112(%rsp,%rdx), %rdx
+; AVX2-NEXT: shrq %cl, %rdx
+; AVX2-NEXT: negq %r10
+; AVX2-NEXT: shlxq %r10, {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Folded Reload
+; AVX2-NEXT: orq %rdx, %r8
+; AVX2-NEXT: shrxq %rcx, %r11, %rcx
+; AVX2-NEXT: movq %rcx, 56(%rax)
+; AVX2-NEXT: movq %rdi, 48(%rax)
+; AVX2-NEXT: movq %r13, 40(%rax)
+; AVX2-NEXT: movq %r15, 32(%rax)
+; AVX2-NEXT: movq %rbx, 24(%rax)
+; AVX2-NEXT: movq %r9, 16(%rax)
+; AVX2-NEXT: movq %rsi, 8(%rax)
+; AVX2-NEXT: movq %r8, (%rax)
+; AVX2-NEXT: addq $24, %rsp
; AVX2-NEXT: popq %rbx
+; AVX2-NEXT: popq %r12
+; AVX2-NEXT: popq %r13
; AVX2-NEXT: popq %r14
+; AVX2-NEXT: popq %r15
+; AVX2-NEXT: popq %rbp
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
@@ -1120,59 +1279,95 @@ define i512 @ashr_i512_load(ptr %p0, i512 %a1) nounwind {
;
; AVX2-LABEL: ashr_i512_load:
; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %rbp
+; AVX2-NEXT: pushq %r15
; AVX2-NEXT: pushq %r14
+; AVX2-NEXT: pushq %r13
+; AVX2-NEXT: pushq %r12
; AVX2-NEXT: pushq %rbx
-; AVX2-NEXT: pushq %rax
+; AVX2-NEXT: subq $24, %rsp
+; AVX2-NEXT: movl %edx, %ecx
+; AVX2-NEXT: andl $63, %ecx
+; AVX2-NEXT: movzbq %cl, %rax
; AVX2-NEXT: vmovups (%rsi), %ymm0
; AVX2-NEXT: vmovaps 32(%rsi), %xmm1
-; AVX2-NEXT: movq 48(%rsi), %rax
-; AVX2-NEXT: movq 56(%rsi), %rcx
-; AVX2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq 48(%rsi), %r8
+; AVX2-NEXT: movq 56(%rsi), %rsi
+; AVX2-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovaps %xmm1, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: sarq $63, %rcx
-; AVX2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movl %edx, %ecx
-; AVX2-NEXT: andl $63, %ecx
+; AVX2-NEXT: sarq $63, %rsi
+; AVX2-NEXT: movq %rsi, {{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rsi, (%rsp)
+; AVX2-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: shrl $3, %edx
; AVX2-NEXT: andl $56, %edx
-; AVX2-NEXT: movq -112(%rsp,%rdx), %rsi
-; AVX2-NEXT: movq -120(%rsp,%rdx), %rax
-; AVX2-NEXT: movq %rax, %r8
-; AVX2-NEXT: shrdq %cl, %rsi, %r8
-; AVX2-NEXT: movq -104(%rsp,%rdx), %r9
-; AVX2-NEXT: shrdq %cl, %r9, %rsi
; AVX2-NEXT: movq -96(%rsp,%rdx), %r10
-; AVX2-NEXT: shrdq %cl, %r10, %r9
-; AVX2-NEXT: movq -88(%rsp,%rdx), %r11
-; AVX2-NEXT: shrdq %cl, %r11, %r10
-; AVX2-NEXT: movq -80(%rsp,%rdx), %rbx
-; AVX2-NEXT: shrdq %cl, %rbx, %r11
-; AVX2-NEXT: movq -128(%rsp,%rdx), %r14
-; AVX2-NEXT: movq -72(%rsp,%rdx), %rdx
-; AVX2-NEXT: shrdq %cl, %rdx, %rbx
-; AVX2-NEXT: shrdq %cl, %rax, %r14
+; AVX2-NEXT: movq -104(%rsp,%rdx), %r11
+; AVX2-NEXT: movq %r11, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: negq %rax
+; AVX2-NEXT: shrq %cl, %r11
+; AVX2-NEXT: shlxq %rax, %r10, %rsi
+; AVX2-NEXT: shrq %cl, %r10
+; AVX2-NEXT: movzbq %cl, %rax
+; AVX2-NEXT: movzbq %cl, %rbx
+; AVX2-NEXT: movq -88(%rsp,%rdx), %r14
+; AVX2-NEXT: negq %rax
+; AVX2-NEXT: shlxq %rax, %r14, %r9
+; AVX2-NEXT: shrq %cl, %r14
+; AVX2-NEXT: movzbq %cl, %rax
+; AVX2-NEXT: movq -80(%rsp,%rdx), %r12
+; AVX2-NEXT: negq %rbx
+; AVX2-NEXT: shlxq %rbx, %r12, %rbx
+; AVX2-NEXT: shrq %cl, %r12
+; AVX2-NEXT: movzbq %cl, %r13
+; AVX2-NEXT: movq -72(%rsp,%rdx), %rbp
+; AVX2-NEXT: negq %rax
+; AVX2-NEXT: shlxq %rax, %rbp, %r15
+; AVX2-NEXT: movq -64(%rsp,%rdx), %r8
+; AVX2-NEXT: negq %r13
+; AVX2-NEXT: shrq %cl, %rbp
+; AVX2-NEXT: shlxq %r13, %r8, %r13
+; AVX2-NEXT: shrq %cl, %r8
; AVX2-NEXT: movq %rdi, %rax
-; AVX2-NEXT: sarxq %rcx, %rdx, %rcx
-; AVX2-NEXT: movq %rcx, 56(%rdi)
-; AVX2-NEXT: movq %rbx, 48(%rdi)
-; AVX2-NEXT: movq %r11, 40(%rdi)
-; AVX2-NEXT: movq %r10, 32(%rdi)
-; AVX2-NEXT: movq %r9, 24(%rdi)
-; AVX2-NEXT: movq %rsi, 16(%rdi)
-; AVX2-NEXT: movq %r8, 8(%rdi)
-; AVX2-NEXT: movq %r14, (%rdi)
-; AVX2-NEXT: addq $8, %rsp
+; AVX2-NEXT: orq %r11, %rsi
+; AVX2-NEXT: orq %r10, %r9
+; AVX2-NEXT: orq %r14, %rbx
+; AVX2-NEXT: movzbq %cl, %rdi
+; AVX2-NEXT: movzbq %cl, %r10
+; AVX2-NEXT: orq %r12, %r15
+; AVX2-NEXT: orq %rbp, %r13
+; AVX2-NEXT: movq -56(%rsp,%rdx), %r11
+; AVX2-NEXT: negq %rdi
+; AVX2-NEXT: shlxq %rdi, %r11, %rdi
+; AVX2-NEXT: orq %r8, %rdi
+; AVX2-NEXT: movq -112(%rsp,%rdx), %rdx
+; AVX2-NEXT: shrq %cl, %rdx
+; AVX2-NEXT: negq %r10
+; AVX2-NEXT: shlxq %r10, {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Folded Reload
+; AVX2-NEXT: orq %rdx, %r8
+; AVX2-NEXT: sarxq %rcx, %r11, %rcx
+; AVX2-NEXT: movq %rcx, 56(%rax)
+; AVX2-NEXT: movq %rdi, 48(%rax)
+; AVX2-NEXT: movq %r13, 40(%rax)
+; AVX2-NEXT: movq %r15, 32(%rax)
+; AVX2-NEXT: movq %rbx, 24(%rax)
+; AVX2-NEXT: movq %r9, 16(%rax)
+; AVX2-NEXT: movq %rsi, 8(%rax)
+; AVX2-NEXT: movq %r8, (%rax)
+; AVX2-NEXT: addq $24, %rsp
; AVX2-NEXT: popq %rbx
+; AVX2-NEXT: popq %r12
+; AVX2-NEXT: popq %r13
; AVX2-NEXT: popq %r14
+; AVX2-NEXT: popq %r15
+; AVX2-NEXT: popq %rbp
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
@@ -2089,9 +2284,13 @@ define i512 @shl_1_i512(i512 %a0) nounwind {
;
; AVX2-LABEL: shl_1_i512:
; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %rbp
+; AVX2-NEXT: pushq %r15
; AVX2-NEXT: pushq %r14
+; AVX2-NEXT: pushq %r13
+; AVX2-NEXT: pushq %r12
; AVX2-NEXT: pushq %rbx
-; AVX2-NEXT: pushq %rax
+; AVX2-NEXT: subq $24, %rsp
; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
@@ -2103,41 +2302,72 @@ define i512 @shl_1_i512(i512 %a0) nounwind {
; AVX2-NEXT: shrl $3, %esi
; AVX2-NEXT: andl $56, %esi
; AVX2-NEXT: negl %esi
-; AVX2-NEXT: movslq %esi, %r8
-; AVX2-NEXT: movq -56(%rsp,%r8), %rdx
-; AVX2-NEXT: movq -48(%rsp,%r8), %rax
-; AVX2-NEXT: movq %rax, %rsi
-; AVX2-NEXT: shldq %cl, %rdx, %rsi
-; AVX2-NEXT: movq -40(%rsp,%r8), %r10
-; AVX2-NEXT: movq %r10, %r9
-; AVX2-NEXT: shldq %cl, %rax, %r9
-; AVX2-NEXT: movq -32(%rsp,%r8), %rax
-; AVX2-NEXT: movq %rax, %r11
-; AVX2-NEXT: shldq %cl, %r10, %r11
-; AVX2-NEXT: movq -24(%rsp,%r8), %r10
+; AVX2-NEXT: movslq %esi, %rdx
+; AVX2-NEXT: movq -32(%rsp,%rdx), %r10
; AVX2-NEXT: movq %r10, %rbx
-; AVX2-NEXT: shldq %cl, %rax, %rbx
-; AVX2-NEXT: movq -16(%rsp,%r8), %rax
-; AVX2-NEXT: movq %rax, %r14
-; AVX2-NEXT: shldq %cl, %r10, %r14
-; AVX2-NEXT: movq -8(%rsp,%r8), %r10
-; AVX2-NEXT: shldq %cl, %rax, %r10
+; AVX2-NEXT: shlq %cl, %rbx
+; AVX2-NEXT: movzbq %cl, %rsi
+; AVX2-NEXT: movzbq %cl, %r12
+; AVX2-NEXT: movq -24(%rsp,%rdx), %r11
+; AVX2-NEXT: movq %r11, %rbp
+; AVX2-NEXT: shlq %cl, %rbp
+; AVX2-NEXT: movzbq %cl, %r13
+; AVX2-NEXT: movq -16(%rsp,%rdx), %r9
+; AVX2-NEXT: movq %r9, %r15
+; AVX2-NEXT: shlq %cl, %r15
+; AVX2-NEXT: movq -8(%rsp,%rdx), %r14
+; AVX2-NEXT: movq %r14, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: shlq %cl, %r14
; AVX2-NEXT: movq %rdi, %rax
-; AVX2-NEXT: movq -64(%rsp,%r8), %rdi
-; AVX2-NEXT: shlxq %rcx, %rdi, %r8
-; AVX2-NEXT: # kill: def $cl killed $cl killed $rcx
-; AVX2-NEXT: shldq %cl, %rdi, %rdx
-; AVX2-NEXT: movq %r10, 56(%rax)
-; AVX2-NEXT: movq %r14, 48(%rax)
-; AVX2-NEXT: movq %rbx, 40(%rax)
-; AVX2-NEXT: movq %r11, 32(%rax)
-; AVX2-NEXT: movq %r9, 24(%rax)
+; AVX2-NEXT: negq %rsi
+; AVX2-NEXT: movq -40(%rsp,%rdx), %r8
+; AVX2-NEXT: shrxq %rsi, %r8, %rsi
+; AVX2-NEXT: orq %rbx, %rsi
+; AVX2-NEXT: negq %r12
+; AVX2-NEXT: shrxq %r12, %r10, %rdi
+; AVX2-NEXT: orq %rbp, %rdi
+; AVX2-NEXT: movzbq %cl, %r12
+; AVX2-NEXT: movzbq %cl, %rbx
+; AVX2-NEXT: negq %r13
+; AVX2-NEXT: shrxq %r13, %r11, %r10
+; AVX2-NEXT: orq %r15, %r10
+; AVX2-NEXT: negq %r12
+; AVX2-NEXT: shrxq %r12, %r9, %r11
+; AVX2-NEXT: movq (%rsp,%rdx), %r15
+; AVX2-NEXT: movq %r15, %r9
+; AVX2-NEXT: shlq %cl, %r9
+; AVX2-NEXT: movq 8(%rsp,%rdx), %r12
+; AVX2-NEXT: shlq %cl, %r12
+; AVX2-NEXT: orq %r14, %r11
+; AVX2-NEXT: movzbq %cl, %r14
+; AVX2-NEXT: shlq %cl, %r8
+; AVX2-NEXT: negq %rbx
+; AVX2-NEXT: shrxq %rbx, {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Folded Reload
+; AVX2-NEXT: movq -48(%rsp,%rdx), %rdx
+; AVX2-NEXT: orq %r9, %rbx
+; AVX2-NEXT: negq %r14
+; AVX2-NEXT: shrxq %r14, %r15, %r9
+; AVX2-NEXT: orq %r12, %r9
+; AVX2-NEXT: shlxq %rcx, %rdx, %r14
+; AVX2-NEXT: movzbq %cl, %rcx
+; AVX2-NEXT: negq %rcx
+; AVX2-NEXT: shrxq %rcx, %rdx, %rcx
+; AVX2-NEXT: orq %r8, %rcx
+; AVX2-NEXT: movq %r9, 56(%rax)
+; AVX2-NEXT: movq %rbx, 48(%rax)
+; AVX2-NEXT: movq %r11, 40(%rax)
+; AVX2-NEXT: movq %r10, 32(%rax)
+; AVX2-NEXT: movq %rdi, 24(%rax)
; AVX2-NEXT: movq %rsi, 16(%rax)
-; AVX2-NEXT: movq %rdx, 8(%rax)
-; AVX2-NEXT: movq %r8, (%rax)
-; AVX2-NEXT: addq $8, %rsp
+; AVX2-NEXT: movq %rcx, 8(%rax)
+; AVX2-NEXT: movq %r14, (%rax)
+; AVX2-NEXT: addq $24, %rsp
; AVX2-NEXT: popq %rbx
+; AVX2-NEXT: popq %r12
+; AVX2-NEXT: popq %r13
; AVX2-NEXT: popq %r14
+; AVX2-NEXT: popq %r15
+; AVX2-NEXT: popq %rbp
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
@@ -2237,48 +2467,84 @@ define i512 @lshr_signbit_i512(i512 %a0) nounwind {
;
; AVX2-LABEL: lshr_signbit_i512:
; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %rbp
+; AVX2-NEXT: pushq %r15
; AVX2-NEXT: pushq %r14
+; AVX2-NEXT: pushq %r13
+; AVX2-NEXT: pushq %r12
; AVX2-NEXT: pushq %rbx
-; AVX2-NEXT: pushq %rax
+; AVX2-NEXT: subq $24, %rsp
; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movl %esi, %ecx
+; AVX2-NEXT: andl $63, %ecx
+; AVX2-NEXT: movzbq %cl, %rax
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovaps {{.*#+}} ymm1 = [0,0,0,9223372036854775808]
; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movl %esi, %ecx
-; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: shrl $3, %esi
; AVX2-NEXT: andl $56, %esi
-; AVX2-NEXT: movq -112(%rsp,%rsi), %rdx
-; AVX2-NEXT: movq -120(%rsp,%rsi), %rax
-; AVX2-NEXT: movq %rax, %r8
-; AVX2-NEXT: shrdq %cl, %rdx, %r8
-; AVX2-NEXT: movq -104(%rsp,%rsi), %r9
-; AVX2-NEXT: shrdq %cl, %r9, %rdx
; AVX2-NEXT: movq -96(%rsp,%rsi), %r10
-; AVX2-NEXT: shrdq %cl, %r10, %r9
-; AVX2-NEXT: movq -88(%rsp,%rsi), %r11
-; AVX2-NEXT: shrdq %cl, %r11, %r10
-; AVX2-NEXT: movq -80(%rsp,%rsi), %rbx
-; AVX2-NEXT: shrdq %cl, %rbx, %r11
-; AVX2-NEXT: movq -128(%rsp,%rsi), %r14
-; AVX2-NEXT: movq -72(%rsp,%rsi), %rsi
-; AVX2-NEXT: shrdq %cl, %rsi, %rbx
-; AVX2-NEXT: shrdq %cl, %rax, %r14
+; AVX2-NEXT: movq -104(%rsp,%rsi), %r11
+; AVX2-NEXT: movq %r11, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: negq %rax
+; AVX2-NEXT: shrq %cl, %r11
+; AVX2-NEXT: shlxq %rax, %r10, %rdx
+; AVX2-NEXT: shrq %cl, %r10
+; AVX2-NEXT: movzbq %cl, %rax
+; AVX2-NEXT: movzbq %cl, %rbx
+; AVX2-NEXT: movq -88(%rsp,%rsi), %r14
+; AVX2-NEXT: negq %rax
+; AVX2-NEXT: shlxq %rax, %r14, %r9
+; AVX2-NEXT: shrq %cl, %r14
+; AVX2-NEXT: movzbq %cl, %rax
+; AVX2-NEXT: movq -80(%rsp,%rsi), %r12
+; AVX2-NEXT: negq %rbx
+; AVX2-NEXT: shlxq %rbx, %r12, %rbx
+; AVX2-NEXT: shrq %cl, %r12
+; AVX2-NEXT: movzbq %cl, %r13
+; AVX2-NEXT: movq -72(%rsp,%rsi), %rbp
+; AVX2-NEXT: negq %rax
+; AVX2-NEXT: shlxq %rax, %rbp, %r15
+; AVX2-NEXT: movq -64(%rsp,%rsi), %r8
+; AVX2-NEXT: negq %r13
+; AVX2-NEXT: shrq %cl, %rbp
+; AVX2-NEXT: shlxq %r13, %r8, %r13
+; AVX2-NEXT: shrq %cl, %r8
; AVX2-NEXT: movq %rdi, %rax
-; AVX2-NEXT: shrxq %rcx, %rsi, %rcx
-; AVX2-NEXT: movq %rcx, 56(%rdi)
-; AVX2-NEXT: movq %rbx, 48(%rdi)
-; AVX2-NEXT: movq %r11, 40(%rdi)
-; AVX2-NEXT: movq %r10, 32(%rdi)
-; AVX2-NEXT: movq %r9, 24(%rdi)
-; AVX2-NEXT: movq %rdx, 16(%rdi)
-; AVX2-NEXT: movq %r8, 8(%rdi)
-; AVX2-NEXT: movq %r14, (%rdi)
-; AVX2-NEXT: addq $8, %rsp
+; AVX2-NEXT: orq %r11, %rdx
+; AVX2-NEXT: orq %r10, %r9
+; AVX2-NEXT: orq %r14, %rbx
+; AVX2-NEXT: movzbq %cl, %rdi
+; AVX2-NEXT: movzbq %cl, %r10
+; AVX2-NEXT: orq %r12, %r15
+; AVX2-NEXT: orq %rbp, %r13
+; AVX2-NEXT: movq -56(%rsp,%rsi), %r11
+; AVX2-NEXT: negq %rdi
+; AVX2-NEXT: shlxq %rdi, %r11, %rdi
+; AVX2-NEXT: orq %r8, %rdi
+; AVX2-NEXT: movq -112(%rsp,%rsi), %rsi
+; AVX2-NEXT: shrq %cl, %rsi
+; AVX2-NEXT: negq %r10
+; AVX2-NEXT: shlxq %r10, {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Folded Reload
+; AVX2-NEXT: orq %rsi, %r8
+; AVX2-NEXT: shrxq %rcx, %r11, %rcx
+; AVX2-NEXT: movq %rcx, 56(%rax)
+; AVX2-NEXT: movq %rdi, 48(%rax)
+; AVX2-NEXT: movq %r13, 40(%rax)
+; AVX2-NEXT: movq %r15, 32(%rax)
+; AVX2-NEXT: movq %rbx, 24(%rax)
+; AVX2-NEXT: movq %r9, 16(%rax)
+; AVX2-NEXT: movq %rdx, 8(%rax)
+; AVX2-NEXT: movq %r8, (%rax)
+; AVX2-NEXT: addq $24, %rsp
; AVX2-NEXT: popq %rbx
+; AVX2-NEXT: popq %r12
+; AVX2-NEXT: popq %r13
; AVX2-NEXT: popq %r14
+; AVX2-NEXT: popq %r15
+; AVX2-NEXT: popq %rbp
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
@@ -2386,49 +2652,85 @@ define i512 @ashr_signbit_i512(i512 %a0) nounwind {
;
; AVX2-LABEL: ashr_signbit_i512:
; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %rbp
+; AVX2-NEXT: pushq %r15
; AVX2-NEXT: pushq %r14
+; AVX2-NEXT: pushq %r13
+; AVX2-NEXT: pushq %r12
; AVX2-NEXT: pushq %rbx
-; AVX2-NEXT: pushq %rax
+; AVX2-NEXT: subq $24, %rsp
; AVX2-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
; AVX2-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovaps {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movl %esi, %ecx
; AVX2-NEXT: andl $63, %ecx
+; AVX2-NEXT: movzbq %cl, %rax
+; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: shrl $3, %esi
; AVX2-NEXT: andl $56, %esi
-; AVX2-NEXT: movq -112(%rsp,%rsi), %rdx
-; AVX2-NEXT: movq -120(%rsp,%rsi), %rax
-; AVX2-NEXT: movq %rax, %r8
-; AVX2-NEXT: shrdq %cl, %rdx, %r8
-; AVX2-NEXT: movq -104(%rsp,%rsi), %r9
-; AVX2-NEXT: shrdq %cl, %r9, %rdx
; AVX2-NEXT: movq -96(%rsp,%rsi), %r10
-; AVX2-NEXT: shrdq %cl, %r10, %r9
-; AVX2-NEXT: movq -88(%rsp,%rsi), %r11
-; AVX2-NEXT: shrdq %cl, %r11, %r10
-; AVX2-NEXT: movq -80(%rsp,%rsi), %rbx
-; AVX2-NEXT: shrdq %cl, %rbx, %r11
-; AVX2-NEXT: movq -128(%rsp,%rsi), %r14
-; AVX2-NEXT: movq -72(%rsp,%rsi), %rsi
-; AVX2-NEXT: shrdq %cl, %rsi, %rbx
-; AVX2-NEXT: shrdq %cl, %rax, %r14
+; AVX2-NEXT: movq -104(%rsp,%rsi), %r11
+; AVX2-NEXT: movq %r11, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: negq %rax
+; AVX2-NEXT: shrq %cl, %r11
+; AVX2-NEXT: shlxq %rax, %r10, %rdx
+; AVX2-NEXT: shrq %cl, %r10
+; AVX2-NEXT: movzbq %cl, %rax
+; AVX2-NEXT: movzbq %cl, %rbx
+; AVX2-NEXT: movq -88(%rsp,%rsi), %r14
+; AVX2-NEXT: negq %rax
+; AVX2-NEXT: shlxq %rax, %r14, %r9
+; AVX2-NEXT: shrq %cl, %r14
+; AVX2-NEXT: movzbq %cl, %rax
+; AVX2-NEXT: movq -80(%rsp,%rsi), %r12
+; AVX2-NEXT: negq %rbx
+; AVX2-NEXT: shlxq %rbx, %r12, %rbx
+; AVX2-NEXT: shrq %cl, %r12
+; AVX2-NEXT: movzbq %cl, %r13
+; AVX2-NEXT: movq -72(%rsp,%rsi), %rbp
+; AVX2-NEXT: negq %rax
+; AVX2-NEXT: shlxq %rax, %rbp, %r15
+; AVX2-NEXT: movq -64(%rsp,%rsi), %r8
+; AVX2-NEXT: negq %r13
+; AVX2-NEXT: shrq %cl, %rbp
+; AVX2-NEXT: shlxq %r13, %r8, %r13
+; AVX2-NEXT: shrq %cl, %r8
; AVX2-NEXT: movq %rdi, %rax
-; AVX2-NEXT: sarxq %rcx, %rsi, %rcx
-; AVX2-NEXT: movq %rcx, 56(%rdi)
-; AVX2-NEXT: movq %rbx, 48(%rdi)
-; AVX2-NEXT: movq %r11, 40(%rdi)
-; AVX2-NEXT: movq %r10, 32(%rdi)
-; AVX2-NEXT: movq %r9, 24(%rdi)
-; AVX2-NEXT: movq %rdx, 16(%rdi)
-; AVX2-NEXT: movq %r8, 8(%rdi)
-; AVX2-NEXT: movq %r14, (%rdi)
-; AVX2-NEXT: addq $8, %rsp
+; AVX2-NEXT: orq %r11, %rdx
+; AVX2-NEXT: orq %r10, %r9
+; AVX2-NEXT: orq %r14, %rbx
+; AVX2-NEXT: movzbq %cl, %rdi
+; AVX2-NEXT: movzbq %cl, %r10
+; AVX2-NEXT: orq %r12, %r15
+; AVX2-NEXT: orq %rbp, %r13
+; AVX2-NEXT: movq -56(%rsp,%rsi), %r11
+; AVX2-NEXT: negq %rdi
+; AVX2-NEXT: shlxq %rdi, %r11, %rdi
+; AVX2-NEXT: orq %r8, %rdi
+; AVX2-NEXT: movq -112(%rsp,%rsi), %rsi
+; AVX2-NEXT: shrq %cl, %rsi
+; AVX2-NEXT: negq %r10
+; AVX2-NEXT: shlxq %r10, {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Folded Reload
+; AVX2-NEXT: orq %rsi, %r8
+; AVX2-NEXT: sarxq %rcx, %r11, %rcx
+; AVX2-NEXT: movq %rcx, 56(%rax)
+; AVX2-NEXT: movq %rdi, 48(%rax)
+; AVX2-NEXT: movq %r13, 40(%rax)
+; AVX2-NEXT: movq %r15, 32(%rax)
+; AVX2-NEXT: movq %rbx, 24(%rax)
+; AVX2-NEXT: movq %r9, 16(%rax)
+; AVX2-NEXT: movq %rdx, 8(%rax)
+; AVX2-NEXT: movq %r8, (%rax)
+; AVX2-NEXT: addq $24, %rsp
; AVX2-NEXT: popq %rbx
+; AVX2-NEXT: popq %r12
+; AVX2-NEXT: popq %r13
; AVX2-NEXT: popq %r14
+; AVX2-NEXT: popq %r15
+; AVX2-NEXT: popq %rbp
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
@@ -2569,9 +2871,13 @@ define i512 @shl_allbits_i512(i512 %a0) nounwind {
;
; AVX2-LABEL: shl_allbits_i512:
; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %rbp
+; AVX2-NEXT: pushq %r15
; AVX2-NEXT: pushq %r14
+; AVX2-NEXT: pushq %r13
+; AVX2-NEXT: pushq %r12
; AVX2-NEXT: pushq %rbx
-; AVX2-NEXT: pushq %rax
+; AVX2-NEXT: subq $24, %rsp
; AVX2-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
; AVX2-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
@@ -2583,41 +2889,72 @@ define i512 @shl_allbits_i512(i512 %a0) nounwind {
; AVX2-NEXT: shrl $3, %esi
; AVX2-NEXT: andl $56, %esi
; AVX2-NEXT: negl %esi
-; AVX2-NEXT: movslq %esi, %r8
-; AVX2-NEXT: movq -56(%rsp,%r8), %rdx
-; AVX2-NEXT: movq -48(%rsp,%r8), %rax
-; AVX2-NEXT: movq %rax, %rsi
-; AVX2-NEXT: shldq %cl, %rdx, %rsi
-; AVX2-NEXT: movq -40(%rsp,%r8), %r10
-; AVX2-NEXT: movq %r10, %r9
-; AVX2-NEXT: shldq %cl, %rax, %r9
-; AVX2-NEXT: movq -32(%rsp,%r8), %rax
-; AVX2-NEXT: movq %rax, %r11
-; AVX2-NEXT: shldq %cl, %r10, %r11
-; AVX2-NEXT: movq -24(%rsp,%r8), %r10
+; AVX2-NEXT: movslq %esi, %rdx
+; AVX2-NEXT: movq -32(%rsp,%rdx), %r10
; AVX2-NEXT: movq %r10, %rbx
-; AVX2-NEXT: shldq %cl, %rax, %rbx
-; AVX2-NEXT: movq -16(%rsp,%r8), %rax
-; AVX2-NEXT: movq %rax, %r14
-; AVX2-NEXT: shldq %cl, %r10, %r14
-; AVX2-NEXT: movq -8(%rsp,%r8), %r10
-; AVX2-NEXT: shldq %cl, %rax, %r10
+; AVX2-NEXT: shlq %cl, %rbx
+; AVX2-NEXT: movzbq %cl, %rsi
+; AVX2-NEXT: movzbq %cl, %r12
+; AVX2-NEXT: movq -24(%rsp,%rdx), %r11
+; AVX2-NEXT: movq %r11, %rbp
+; AVX2-NEXT: shlq %cl, %rbp
+; AVX2-NEXT: movzbq %cl, %r13
+; AVX2-NEXT: movq -16(%rsp,%rdx), %r9
+; AVX2-NEXT: movq %r9, %r15
+; AVX2-NEXT: shlq %cl, %r15
+; AVX2-NEXT: movq -8(%rsp,%rdx), %r14
+; AVX2-NEXT: movq %r14, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: shlq %cl, %r14
; AVX2-NEXT: movq %rdi, %rax
-; AVX2-NEXT: movq -64(%rsp,%r8), %rdi
-; AVX2-NEXT: shlxq %rcx, %rdi, %r8
-; AVX2-NEXT: # kill: def $cl killed $cl killed $rcx
-; AVX2-NEXT: shldq %cl, %rdi, %rdx
-; AVX2-NEXT: movq %r10, 56(%rax)
-; AVX2-NEXT: movq %r14, 48(%rax)
-; AVX2-NEXT: movq %rbx, 40(%rax)
-; AVX2-NEXT: movq %r11, 32(%rax)
-; AVX2-NEXT: movq %r9, 24(%rax)
+; AVX2-NEXT: negq %rsi
+; AVX2-NEXT: movq -40(%rsp,%rdx), %r8
+; AVX2-NEXT: shrxq %rsi, %r8, %rsi
+; AVX2-NEXT: orq %rbx, %rsi
+; AVX2-NEXT: negq %r12
+; AVX2-NEXT: shrxq %r12, %r10, %rdi
+; AVX2-NEXT: orq %rbp, %rdi
+; AVX2-NEXT: movzbq %cl, %r12
+; AVX2-NEXT: movzbq %cl, %rbx
+; AVX2-NEXT: negq %r13
+; AVX2-NEXT: shrxq %r13, %r11, %r10
+; AVX2-NEXT: orq %r15, %r10
+; AVX2-NEXT: negq %r12
+; AVX2-NEXT: shrxq %r12, %r9, %r11
+; AVX2-NEXT: movq (%rsp,%rdx), %r15
+; AVX2-NEXT: movq %r15, %r9
+; AVX2-NEXT: shlq %cl, %r9
+; AVX2-NEXT: movq 8(%rsp,%rdx), %r12
+; AVX2-NEXT: shlq %cl, %r12
+; AVX2-NEXT: orq %r14, %r11
+; AVX2-NEXT: movzbq %cl, %r14
+; AVX2-NEXT: shlq %cl, %r8
+; AVX2-NEXT: negq %rbx
+; AVX2-NEXT: shrxq %rbx, {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Folded Reload
+; AVX2-NEXT: movq -48(%rsp,%rdx), %rdx
+; AVX2-NEXT: orq %r9, %rbx
+; AVX2-NEXT: negq %r14
+; AVX2-NEXT: shrxq %r14, %r15, %r9
+; AVX2-NEXT: orq %r12, %r9
+; AVX2-NEXT: shlxq %rcx, %rdx, %r14
+; AVX2-NEXT: movzbq %cl, %rcx
+; AVX2-NEXT: negq %rcx
+; AVX2-NEXT: shrxq %rcx, %rdx, %rcx
+; AVX2-NEXT: orq %r8, %rcx
+; AVX2-NEXT: movq %r9, 56(%rax)
+; AVX2-NEXT: movq %rbx, 48(%rax)
+; AVX2-NEXT: movq %r11, 40(%rax)
+; AVX2-NEXT: movq %r10, 32(%rax)
+; AVX2-NEXT: movq %rdi, 24(%rax)
; AVX2-NEXT: movq %rsi, 16(%rax)
-; AVX2-NEXT: movq %rdx, 8(%rax)
-; AVX2-NEXT: movq %r8, (%rax)
-; AVX2-NEXT: addq $8, %rsp
+; AVX2-NEXT: movq %rcx, 8(%rax)
+; AVX2-NEXT: movq %r14, (%rax)
+; AVX2-NEXT: addq $24, %rsp
; AVX2-NEXT: popq %rbx
+; AVX2-NEXT: popq %r12
+; AVX2-NEXT: popq %r13
; AVX2-NEXT: popq %r14
+; AVX2-NEXT: popq %r15
+; AVX2-NEXT: popq %rbp
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
@@ -2747,48 +3084,84 @@ define i512 @lshr_allbits_i512(i512 %a0) nounwind {
;
; AVX2-LABEL: lshr_allbits_i512:
; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %rbp
+; AVX2-NEXT: pushq %r15
; AVX2-NEXT: pushq %r14
+; AVX2-NEXT: pushq %r13
+; AVX2-NEXT: pushq %r12
; AVX2-NEXT: pushq %rbx
-; AVX2-NEXT: pushq %rax
+; AVX2-NEXT: subq $24, %rsp
; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
; AVX2-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movl %esi, %ecx
+; AVX2-NEXT: andl $63, %ecx
+; AVX2-NEXT: movzbq %cl, %rax
; AVX2-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
; AVX2-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movl %esi, %ecx
-; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: shrl $3, %esi
; AVX2-NEXT: andl $56, %esi
-; AVX2-NEXT: movq -112(%rsp,%rsi), %rdx
-; AVX2-NEXT: movq -120(%rsp,%rsi), %rax
-; AVX2-NEXT: movq %rax, %r8
-; AVX2-NEXT: shrdq %cl, %rdx, %r8
-; AVX2-NEXT: movq -104(%rsp,%rsi), %r9
-; AVX2-NEXT: shrdq %cl, %r9, %rdx
; AVX2-NEXT: movq -96(%rsp,%rsi), %r10
-; AVX2-NEXT: shrdq %cl, %r10, %r9
-; AVX2-NEXT: movq -88(%rsp,%rsi), %r11
-; AVX2-NEXT: shrdq %cl, %r11, %r10
-; AVX2-NEXT: movq -80(%rsp,%rsi), %rbx
-; AVX2-NEXT: shrdq %cl, %rbx, %r11
-; AVX2-NEXT: movq -128(%rsp,%rsi), %r14
-; AVX2-NEXT: movq -72(%rsp,%rsi), %rsi
-; AVX2-NEXT: shrdq %cl, %rsi, %rbx
-; AVX2-NEXT: shrdq %cl, %rax, %r14
+; AVX2-NEXT: movq -104(%rsp,%rsi), %r11
+; AVX2-NEXT: movq %r11, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: negq %rax
+; AVX2-NEXT: shrq %cl, %r11
+; AVX2-NEXT: shlxq %rax, %r10, %rdx
+; AVX2-NEXT: shrq %cl, %r10
+; AVX2-NEXT: movzbq %cl, %rax
+; AVX2-NEXT: movzbq %cl, %rbx
+; AVX2-NEXT: movq -88(%rsp,%rsi), %r14
+; AVX2-NEXT: negq %rax
+; AVX2-NEXT: shlxq %rax, %r14, %r9
+; AVX2-NEXT: shrq %cl, %r14
+; AVX2-NEXT: movzbq %cl, %rax
+; AVX2-NEXT: movq -80(%rsp,%rsi), %r12
+; AVX2-NEXT: negq %rbx
+; AVX2-NEXT: shlxq %rbx, %r12, %rbx
+; AVX2-NEXT: shrq %cl, %r12
+; AVX2-NEXT: movzbq %cl, %r13
+; AVX2-NEXT: movq -72(%rsp,%rsi), %rbp
+; AVX2-NEXT: negq %rax
+; AVX2-NEXT: shlxq %rax, %rbp, %r15
+; AVX2-NEXT: movq -64(%rsp,%rsi), %r8
+; AVX2-NEXT: negq %r13
+; AVX2-NEXT: shrq %cl, %rbp
+; AVX2-NEXT: shlxq %r13, %r8, %r13
+; AVX2-NEXT: shrq %cl, %r8
; AVX2-NEXT: movq %rdi, %rax
-; AVX2-NEXT: shrxq %rcx, %rsi, %rcx
-; AVX2-NEXT: movq %rcx, 56(%rdi)
-; AVX2-NEXT: movq %rbx, 48(%rdi)
-; AVX2-NEXT: movq %r11, 40(%rdi)
-; AVX2-NEXT: movq %r10, 32(%rdi)
-; AVX2-NEXT: movq %r9, 24(%rdi)
-; AVX2-NEXT: movq %rdx, 16(%rdi)
-; AVX2-NEXT: movq %r8, 8(%rdi)
-; AVX2-NEXT: movq %r14, (%rdi)
-; AVX2-NEXT: addq $8, %rsp
+; AVX2-NEXT: orq %r11, %rdx
+; AVX2-NEXT: orq %r10, %r9
+; AVX2-NEXT: orq %r14, %rbx
+; AVX2-NEXT: movzbq %cl, %rdi
+; AVX2-NEXT: movzbq %cl, %r10
+; AVX2-NEXT: orq %r12, %r15
+; AVX2-NEXT: orq %rbp, %r13
+; AVX2-NEXT: movq -56(%rsp,%rsi), %r11
+; AVX2-NEXT: negq %rdi
+; AVX2-NEXT: shlxq %rdi, %r11, %rdi
+; AVX2-NEXT: orq %r8, %rdi
+; AVX2-NEXT: movq -112(%rsp,%rsi), %rsi
+; AVX2-NEXT: shrq %cl, %rsi
+; AVX2-NEXT: negq %r10
+; AVX2-NEXT: shlxq %r10, {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Folded Reload
+; AVX2-NEXT: orq %rsi, %r8
+; AVX2-NEXT: shrxq %rcx, %r11, %rcx
+; AVX2-NEXT: movq %rcx, 56(%rax)
+; AVX2-NEXT: movq %rdi, 48(%rax)
+; AVX2-NEXT: movq %r13, 40(%rax)
+; AVX2-NEXT: movq %r15, 32(%rax)
+; AVX2-NEXT: movq %rbx, 24(%rax)
+; AVX2-NEXT: movq %r9, 16(%rax)
+; AVX2-NEXT: movq %rdx, 8(%rax)
+; AVX2-NEXT: movq %r8, (%rax)
+; AVX2-NEXT: addq $24, %rsp
; AVX2-NEXT: popq %rbx
+; AVX2-NEXT: popq %r12
+; AVX2-NEXT: popq %r13
; AVX2-NEXT: popq %r14
+; AVX2-NEXT: popq %r15
+; AVX2-NEXT: popq %rbp
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
@@ -2892,11 +3265,11 @@ define i64 @lshr_extract_i512_i64(i512 %a0, i512 %a1) nounwind {
; AVX2-LABEL: lshr_extract_i512_i64:
; AVX2: # %bb.0:
; AVX2-NEXT: pushq %rax
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovaps {{[0-9]+}}(%rsp), %xmm0
-; AVX2-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
@@ -2904,13 +3277,16 @@ define i64 @lshr_extract_i512_i64(i512 %a0, i512 %a1) nounwind {
; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movl %r10d, %ecx
-; AVX2-NEXT: shrl $3, %ecx
-; AVX2-NEXT: andl $56, %ecx
-; AVX2-NEXT: movq -128(%rsp,%rcx), %rax
-; AVX2-NEXT: movq -120(%rsp,%rcx), %rdx
-; AVX2-NEXT: movl %r10d, %ecx
-; AVX2-NEXT: shrdq %cl, %rdx, %rax
+; AVX2-NEXT: movl %eax, %edx
+; AVX2-NEXT: shrl $3, %edx
+; AVX2-NEXT: andl $56, %edx
+; AVX2-NEXT: movq -128(%rsp,%rdx), %rsi
+; AVX2-NEXT: movl %eax, %ecx
+; AVX2-NEXT: movzbq %cl, %rax
+; AVX2-NEXT: shrq %cl, %rsi
+; AVX2-NEXT: negq %rax
+; AVX2-NEXT: shlxq %rax, -120(%rsp,%rdx), %rax
+; AVX2-NEXT: orq %rsi, %rax
; AVX2-NEXT: popq %rcx
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
@@ -2918,10 +3294,10 @@ define i64 @lshr_extract_i512_i64(i512 %a0, i512 %a1) nounwind {
; AVX512F-LABEL: lshr_extract_i512_i64:
; AVX512F: # %bb.0:
; AVX512F-NEXT: pushq %rax
-; AVX512F-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX512F-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX512F-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT: vmovups %zmm0, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: vmovaps {{[0-9]+}}(%rsp), %xmm0
-; AVX512F-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT: vmovups %zmm1, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
@@ -2929,38 +3305,44 @@ define i64 @lshr_extract_i512_i64(i512 %a0, i512 %a1) nounwind {
; AVX512F-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; AVX512F-NEXT: movl %r10d, %ecx
-; AVX512F-NEXT: shrl $3, %ecx
-; AVX512F-NEXT: andl $56, %ecx
-; AVX512F-NEXT: movq -128(%rsp,%rcx), %rax
-; AVX512F-NEXT: movq -120(%rsp,%rcx), %rdx
-; AVX512F-NEXT: movl %r10d, %ecx
-; AVX512F-NEXT: shrdq %cl, %rdx, %rax
+; AVX512F-NEXT: movl %eax, %edx
+; AVX512F-NEXT: shrl $3, %edx
+; AVX512F-NEXT: andl $56, %edx
+; AVX512F-NEXT: movq -128(%rsp,%rdx), %rsi
+; AVX512F-NEXT: movl %eax, %ecx
+; AVX512F-NEXT: movzbq %cl, %rax
+; AVX512F-NEXT: shrq %cl, %rsi
+; AVX512F-NEXT: negq %rax
+; AVX512F-NEXT: shlxq %rax, -120(%rsp,%rdx), %rax
+; AVX512F-NEXT: orq %rsi, %rax
; AVX512F-NEXT: popq %rcx
; AVX512F-NEXT: retq
;
; AVX512VL-LABEL: lshr_extract_i512_i64:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: pushq %rax
-; AVX512VL-NEXT: movq {{[0-9]+}}(%rsp), %r10
; AVX512VL-NEXT: vmovaps {{[0-9]+}}(%rsp), %xmm0
; AVX512VL-NEXT: vxorps %xmm1, %xmm1, %xmm1
; AVX512VL-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq {{[0-9]+}}(%rsp), %rax
; AVX512VL-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: movl %r10d, %ecx
-; AVX512VL-NEXT: shrl $3, %ecx
-; AVX512VL-NEXT: andl $56, %ecx
-; AVX512VL-NEXT: movq -128(%rsp,%rcx), %rax
-; AVX512VL-NEXT: movq -120(%rsp,%rcx), %rdx
-; AVX512VL-NEXT: movl %r10d, %ecx
-; AVX512VL-NEXT: shrdq %cl, %rdx, %rax
+; AVX512VL-NEXT: movl %eax, %edx
+; AVX512VL-NEXT: shrl $3, %edx
+; AVX512VL-NEXT: andl $56, %edx
+; AVX512VL-NEXT: movq -128(%rsp,%rdx), %rsi
+; AVX512VL-NEXT: movl %eax, %ecx
+; AVX512VL-NEXT: movzbq %cl, %rax
+; AVX512VL-NEXT: negq %rax
+; AVX512VL-NEXT: shrq %cl, %rsi
+; AVX512VL-NEXT: shlxq %rax, -120(%rsp,%rdx), %rax
+; AVX512VL-NEXT: orq %rsi, %rax
; AVX512VL-NEXT: popq %rcx
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
@@ -2968,25 +3350,28 @@ define i64 @lshr_extract_i512_i64(i512 %a0, i512 %a1) nounwind {
; AVX512VBMI-LABEL: lshr_extract_i512_i64:
; AVX512VBMI: # %bb.0:
; AVX512VBMI-NEXT: pushq %rax
-; AVX512VBMI-NEXT: movq {{[0-9]+}}(%rsp), %r10
; AVX512VBMI-NEXT: vmovaps {{[0-9]+}}(%rsp), %xmm0
; AVX512VBMI-NEXT: vxorps %xmm1, %xmm1, %xmm1
; AVX512VBMI-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq {{[0-9]+}}(%rsp), %rax
; AVX512VBMI-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: movl %r10d, %ecx
-; AVX512VBMI-NEXT: shrl $3, %ecx
-; AVX512VBMI-NEXT: andl $56, %ecx
-; AVX512VBMI-NEXT: movq -128(%rsp,%rcx), %rax
-; AVX512VBMI-NEXT: movq -120(%rsp,%rcx), %rdx
-; AVX512VBMI-NEXT: movl %r10d, %ecx
-; AVX512VBMI-NEXT: shrdq %cl, %rdx, %rax
+; AVX512VBMI-NEXT: movl %eax, %edx
+; AVX512VBMI-NEXT: shrl $3, %edx
+; AVX512VBMI-NEXT: andl $56, %edx
+; AVX512VBMI-NEXT: movq -128(%rsp,%rdx), %rsi
+; AVX512VBMI-NEXT: movl %eax, %ecx
+; AVX512VBMI-NEXT: movzbq %cl, %rax
+; AVX512VBMI-NEXT: negq %rax
+; AVX512VBMI-NEXT: shrq %cl, %rsi
+; AVX512VBMI-NEXT: shlxq %rax, -120(%rsp,%rdx), %rax
+; AVX512VBMI-NEXT: orq %rsi, %rax
; AVX512VBMI-NEXT: popq %rcx
; AVX512VBMI-NEXT: vzeroupper
; AVX512VBMI-NEXT: retq
@@ -2996,39 +3381,183 @@ define i64 @lshr_extract_i512_i64(i512 %a0, i512 %a1) nounwind {
}
define i64 @ashr_extract_i512_i64(i512 %a0, i512 %a1) nounwind {
-; CHECK-LABEL: ashr_extract_i512_i64:
-; CHECK: # %bb.0:
-; CHECK-NEXT: pushq %rax
-; CHECK-NEXT: movq %rcx, %rax
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rcx
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r10
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r11
-; CHECK-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT: movq %r10, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT: sarq $63, %r11
-; CHECK-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
-; CHECK-NEXT: movl %ecx, %edx
-; CHECK-NEXT: shrl $3, %edx
-; CHECK-NEXT: andl $56, %edx
-; CHECK-NEXT: movq -128(%rsp,%rdx), %rax
-; CHECK-NEXT: movq -120(%rsp,%rdx), %rdx
-; CHECK-NEXT: # kill: def $cl killed $cl killed $rcx
-; CHECK-NEXT: shrdq %cl, %rdx, %rax
-; CHECK-NEXT: popq %rcx
-; CHECK-NEXT: retq
+; SSE-LABEL: ashr_extract_i512_i64:
+; SSE: # %bb.0:
+; SSE-NEXT: pushq %rax
+; SSE-NEXT: movq %rcx, %rax
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rcx
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; SSE-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %r10, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: sarq $63, %r11
+; SSE-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movl %ecx, %edx
+; SSE-NEXT: shrl $3, %edx
+; SSE-NEXT: andl $56, %edx
+; SSE-NEXT: movq -128(%rsp,%rdx), %rax
+; SSE-NEXT: movq -120(%rsp,%rdx), %rdx
+; SSE-NEXT: # kill: def $cl killed $cl killed $rcx
+; SSE-NEXT: shrdq %cl, %rdx, %rax
+; SSE-NEXT: popq %rcx
+; SSE-NEXT: retq
+;
+; AVX2-LABEL: ashr_extract_i512_i64:
+; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %rax
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX2-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %r10, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: sarq $63, %r11
+; AVX2-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movl %eax, %edx
+; AVX2-NEXT: shrl $3, %edx
+; AVX2-NEXT: andl $56, %edx
+; AVX2-NEXT: movq -128(%rsp,%rdx), %rsi
+; AVX2-NEXT: movl %eax, %ecx
+; AVX2-NEXT: movzbq %cl, %rax
+; AVX2-NEXT: shrq %cl, %rsi
+; AVX2-NEXT: negq %rax
+; AVX2-NEXT: shlxq %rax, -120(%rsp,%rdx), %rax
+; AVX2-NEXT: orq %rsi, %rax
+; AVX2-NEXT: popq %rcx
+; AVX2-NEXT: retq
+;
+; AVX512F-LABEL: ashr_extract_i512_i64:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: pushq %rax
+; AVX512F-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX512F-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX512F-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX512F-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %r10, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: sarq $63, %r11
+; AVX512F-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movl %eax, %edx
+; AVX512F-NEXT: shrl $3, %edx
+; AVX512F-NEXT: andl $56, %edx
+; AVX512F-NEXT: movq -128(%rsp,%rdx), %rsi
+; AVX512F-NEXT: movl %eax, %ecx
+; AVX512F-NEXT: movzbq %cl, %rax
+; AVX512F-NEXT: shrq %cl, %rsi
+; AVX512F-NEXT: negq %rax
+; AVX512F-NEXT: shlxq %rax, -120(%rsp,%rdx), %rax
+; AVX512F-NEXT: orq %rsi, %rax
+; AVX512F-NEXT: popq %rcx
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: ashr_extract_i512_i64:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: pushq %rax
+; AVX512VL-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX512VL-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX512VL-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX512VL-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %r10, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: sarq $63, %r11
+; AVX512VL-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movl %eax, %edx
+; AVX512VL-NEXT: shrl $3, %edx
+; AVX512VL-NEXT: andl $56, %edx
+; AVX512VL-NEXT: movq -128(%rsp,%rdx), %rsi
+; AVX512VL-NEXT: movl %eax, %ecx
+; AVX512VL-NEXT: movzbq %cl, %rax
+; AVX512VL-NEXT: negq %rax
+; AVX512VL-NEXT: shrq %cl, %rsi
+; AVX512VL-NEXT: shlxq %rax, -120(%rsp,%rdx), %rax
+; AVX512VL-NEXT: orq %rsi, %rax
+; AVX512VL-NEXT: popq %rcx
+; AVX512VL-NEXT: retq
+;
+; AVX512VBMI-LABEL: ashr_extract_i512_i64:
+; AVX512VBMI: # %bb.0:
+; AVX512VBMI-NEXT: pushq %rax
+; AVX512VBMI-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX512VBMI-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX512VBMI-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX512VBMI-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %r10, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: sarq $63, %r11
+; AVX512VBMI-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movq %r11, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movl %eax, %edx
+; AVX512VBMI-NEXT: shrl $3, %edx
+; AVX512VBMI-NEXT: andl $56, %edx
+; AVX512VBMI-NEXT: movq -128(%rsp,%rdx), %rsi
+; AVX512VBMI-NEXT: movl %eax, %ecx
+; AVX512VBMI-NEXT: movzbq %cl, %rax
+; AVX512VBMI-NEXT: negq %rax
+; AVX512VBMI-NEXT: shrq %cl, %rsi
+; AVX512VBMI-NEXT: shlxq %rax, -120(%rsp,%rdx), %rax
+; AVX512VBMI-NEXT: orq %rsi, %rax
+; AVX512VBMI-NEXT: popq %rcx
+; AVX512VBMI-NEXT: retq
%b = ashr i512 %a0, %a1
%r = trunc i512 %b to i64
ret i64 %r
@@ -3066,6 +3595,7 @@ define i64 @lshr_extract_load_i512_i64(ptr %p0, i512 %a1) nounwind {
; AVX2: # %bb.0:
; AVX2-NEXT: pushq %rax
; AVX2-NEXT: movq %rsi, %rcx
+; AVX2-NEXT: movzbq %cl, %rax
; AVX2-NEXT: vmovups (%rdi), %ymm0
; AVX2-NEXT: vmovups 32(%rdi), %ymm1
; AVX2-NEXT: vxorps %xmm2, %xmm2, %xmm2
@@ -3076,10 +3606,11 @@ define i64 @lshr_extract_load_i512_i64(ptr %p0, i512 %a1) nounwind {
; AVX2-NEXT: movl %ecx, %edx
; AVX2-NEXT: shrl $3, %edx
; AVX2-NEXT: andl $56, %edx
-; AVX2-NEXT: movq -128(%rsp,%rdx), %rax
-; AVX2-NEXT: movq -120(%rsp,%rdx), %rdx
-; AVX2-NEXT: # kill: def $cl killed $cl killed $rcx
-; AVX2-NEXT: shrdq %cl, %rdx, %rax
+; AVX2-NEXT: movq -128(%rsp,%rdx), %rsi
+; AVX2-NEXT: shrq %cl, %rsi
+; AVX2-NEXT: negq %rax
+; AVX2-NEXT: shlxq %rax, -120(%rsp,%rdx), %rax
+; AVX2-NEXT: orq %rsi, %rax
; AVX2-NEXT: popq %rcx
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
@@ -3177,30 +3708,32 @@ define i64 @ashr_extract_load_i512_i64(ptr %p0, i512 %a1) nounwind {
; AVX2: # %bb.0:
; AVX2-NEXT: pushq %rax
; AVX2-NEXT: movq %rsi, %rcx
+; AVX2-NEXT: movzbq %cl, %rax
; AVX2-NEXT: vmovups (%rdi), %ymm0
; AVX2-NEXT: vmovaps 32(%rdi), %xmm1
-; AVX2-NEXT: movq 48(%rdi), %rax
-; AVX2-NEXT: movq 56(%rdi), %rdx
+; AVX2-NEXT: movq 48(%rdi), %rdx
+; AVX2-NEXT: movq 56(%rdi), %rsi
; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovaps %xmm1, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: sarq $63, %rdx
-; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: sarq $63, %rsi
+; AVX2-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movl %ecx, %edx
; AVX2-NEXT: shrl $3, %edx
; AVX2-NEXT: andl $56, %edx
-; AVX2-NEXT: movq -128(%rsp,%rdx), %rax
-; AVX2-NEXT: movq -120(%rsp,%rdx), %rdx
-; AVX2-NEXT: # kill: def $cl killed $cl killed $rcx
-; AVX2-NEXT: shrdq %cl, %rdx, %rax
+; AVX2-NEXT: movq -128(%rsp,%rdx), %rsi
+; AVX2-NEXT: shrq %cl, %rsi
+; AVX2-NEXT: negq %rax
+; AVX2-NEXT: shlxq %rax, -120(%rsp,%rdx), %rax
+; AVX2-NEXT: orq %rsi, %rax
; AVX2-NEXT: popq %rcx
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
More information about the llvm-commits
mailing list