[llvm] [X86] fold mov dec/inc to lea +- 1 (PR #185194)
Takashi Idobe via llvm-commits
llvm-commits at lists.llvm.org
Thu Mar 26 05:31:17 PDT 2026
https://github.com/Takashiidobe updated https://github.com/llvm/llvm-project/pull/185194
>From 8763eb21238e7a8e606a310e7aac1c3f4992fa0f Mon Sep 17 00:00:00 2001
From: Takashiidobe <idobetakashi at gmail.com>
Date: Sat, 7 Mar 2026 09:09:35 -0500
Subject: [PATCH 1/2] fuse mov + inc/dec pattern into just lea reg [reg +- 1]
---
.../lib/CodeGen/TwoAddressInstructionPass.cpp | 33 +-
llvm/lib/Target/X86/X86FixupLEAs.cpp | 6 +
.../CodeGen/X86/avx512-intrinsics-upgrade.ll | 46 +-
llvm/test/CodeGen/X86/bit-manip-i128.ll | 641 +--
llvm/test/CodeGen/X86/bitcast-vector-bool.ll | 177 +-
llvm/test/CodeGen/X86/bitcnt-big-integer.ll | 5120 ++++++++++-------
llvm/test/CodeGen/X86/bittest-big-integer.ll | 376 +-
llvm/test/CodeGen/X86/extract-lowbits.ll | 4 +-
llvm/test/CodeGen/X86/i128-udiv.ll | 3316 +++++------
llvm/test/CodeGen/X86/icmp-pow2-diff.ll | 10 +-
.../test/CodeGen/X86/icmp-pow2-logic-npow2.ll | 4 +-
.../CodeGen/X86/isel-llvm.set.rounding.ll | 25 +-
llvm/test/CodeGen/X86/mov-lea-folds.ll | 103 +
llvm/test/CodeGen/X86/neg-abs.ll | 8 +-
llvm/test/CodeGen/X86/sat-add.ll | 21 +-
llvm/test/CodeGen/X86/setcc-logic.ll | 10 +-
llvm/test/CodeGen/X86/vector-idiv-udiv-128.ll | 612 +-
.../CodeGen/X86/vector-reduce-fmin-nnan.ll | 3 +-
llvm/test/CodeGen/X86/vector-reduce-smin.ll | 161 +-
llvm/test/CodeGen/X86/vector-trunc-usat.ll | 766 ++-
...ad-of-small-alloca-with-zero-upper-half.ll | 12 +-
21 files changed, 6025 insertions(+), 5429 deletions(-)
create mode 100644 llvm/test/CodeGen/X86/mov-lea-folds.ll
diff --git a/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp b/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp
index ace5c6e49596e..43790412cda43 100644
--- a/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp
+++ b/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp
@@ -158,7 +158,7 @@ class TwoAddressInstructionImpl {
bool commuteInstruction(MachineInstr *MI, unsigned DstIdx,
unsigned RegBIdx, unsigned RegCIdx, unsigned Dist);
- bool isProfitableToConv3Addr(Register RegA, Register RegB);
+ bool isProfitableToConv3Addr(MachineInstr &MI, Register RegA, Register RegB);
bool convertInstTo3Addr(MachineBasicBlock::iterator &mi,
MachineBasicBlock::iterator &nmi, Register RegA,
@@ -786,13 +786,38 @@ bool TwoAddressInstructionImpl::commuteInstruction(MachineInstr *MI,
/// Return true if it is profitable to convert the given 2-address instruction
/// to a 3-address one.
-bool TwoAddressInstructionImpl::isProfitableToConv3Addr(Register RegA,
+bool TwoAddressInstructionImpl::isProfitableToConv3Addr(MachineInstr &MI,
+ Register RegA,
Register RegB) {
+ // Narrowly target copy+op patterns:
+ // %tmp = COPY %src
+ // %tmp = OP %tmp, ...
+ // and only when removing that copy is obvious.
+ if (RegA.isVirtual() && RegB.isVirtual() && MRI->hasOneNonDBGUse(RegB)) {
+
+ MachineInstr *DefMI = MRI->getUniqueVRegDef(RegB);
+ bool HasCopyDef = DefMI && DefMI->isCopy();
+ bool InSameBlock = HasCopyDef && DefMI->getParent() == MI.getParent();
+ bool HasRegOperands = InSameBlock && DefMI->getOperand(0).isReg() &&
+ DefMI->getOperand(1).isReg();
+ bool DefinesRegB = HasRegOperands && DefMI->getOperand(0).getReg() == RegB;
+ bool CopySrcIsVirtual =
+ DefinesRegB && DefMI->getOperand(1).getReg().isVirtual();
+
+ if (CopySrcIsVirtual) {
+ auto NextNonDbg = std::next(DefMI->getIterator());
+ while (NextNonDbg != MI.getIterator() && NextNonDbg->isDebugInstr())
+ ++NextNonDbg;
+ if (NextNonDbg == MI.getIterator())
+ return true;
+ }
+ }
+
// Look for situations like this:
// %reg1024 = MOV r1
// %reg1025 = MOV r0
// %reg1026 = ADD %reg1024, %reg1025
- // r2 = MOV %reg1026
+ // r2 = MOV %reg1026
// Turn ADD into a 3-address instruction to avoid a copy.
MCRegister FromRegB = getMappedReg(RegB, SrcRegMap);
if (!FromRegB)
@@ -1387,7 +1412,7 @@ bool TwoAddressInstructionImpl::tryInstructionTransform(
if (ConvertibleTo3Addr) {
// This instruction is potentially convertible to a true
// three-address instruction. Check if it is profitable.
- if (!regBKilled || isProfitableToConv3Addr(regA, regB)) {
+ if (!regBKilled || isProfitableToConv3Addr(MI, regA, regB)) {
// Try to convert it.
if (convertInstTo3Addr(mi, nmi, regA, regB, Dist)) {
++NumConvertedTo3Addr;
diff --git a/llvm/lib/Target/X86/X86FixupLEAs.cpp b/llvm/lib/Target/X86/X86FixupLEAs.cpp
index 07f656fc5ccfd..d33a754ae2d09 100644
--- a/llvm/lib/Target/X86/X86FixupLEAs.cpp
+++ b/llvm/lib/Target/X86/X86FixupLEAs.cpp
@@ -643,6 +643,9 @@ bool FixupLEAsImpl::optTwoAddrLEA(MachineBasicBlock::iterator &I,
} else
return false;
+ // optTwoAddrLEA only rewrites when EFLAGS are dead; preserve that on the
+ // replacement so later transforms can still reason about dead flags.
+ NewMI->addRegisterDead(X86::EFLAGS, TRI);
MBB.getParent()->substituteDebugValuesForInst(*I, *NewMI, 1);
MBB.erase(I);
I = NewMI;
@@ -733,6 +736,9 @@ void FixupLEAsImpl::processInstructionForSlowLEA(MachineBasicBlock::iterator &I,
LLVM_DEBUG(NewMI->dump(););
}
if (NewMI) {
+ // processInstructionForSlowLEA is guarded by dead EFLAGS liveness.
+ // Preserve that on the replacement instruction.
+ NewMI->addRegisterDead(X86::EFLAGS, TRI);
MBB.getParent()->substituteDebugValuesForInst(*I, *NewMI, 1);
MBB.erase(I);
I = NewMI;
diff --git a/llvm/test/CodeGen/X86/avx512-intrinsics-upgrade.ll b/llvm/test/CodeGen/X86/avx512-intrinsics-upgrade.ll
index 5a0a9b7c3c25c..6755ee0b174df 100644
--- a/llvm/test/CodeGen/X86/avx512-intrinsics-upgrade.ll
+++ b/llvm/test/CodeGen/X86/avx512-intrinsics-upgrade.ll
@@ -5638,10 +5638,7 @@ define <8 x i64> @test_mask_mul_epu32_rmb(<16 x i32> %a, ptr %ptr_b) {
; X86-LABEL: test_mask_mul_epu32_rmb:
; X86: ## %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT: vpbroadcastd (%eax), %zmm1 ## encoding: [0x62,0xf2,0x7d,0x48,0x58,0x08]
-; X86-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}{1to8}, %zmm1, %zmm1 ## encoding: [0x62,0xf1,0xf5,0x58,0xdb,0x0d,A,A,A,A]
-; X86-NEXT: ## fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
-; X86-NEXT: vpmuludq %zmm1, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0xf4,0xc1]
+; X86-NEXT: vpmuludq (%eax){1to8}, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x58,0xf4,0x00]
; X86-NEXT: retl ## encoding: [0xc3]
;
; X64-LABEL: test_mask_mul_epu32_rmb:
@@ -5660,12 +5657,9 @@ define <8 x i64> @test_mask_mul_epu32_rmbk(<16 x i32> %a, ptr %ptr_b, <8 x i64>
; X86-LABEL: test_mask_mul_epu32_rmbk:
; X86: ## %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT: vpbroadcastd (%eax), %zmm2 ## encoding: [0x62,0xf2,0x7d,0x48,0x58,0x10]
-; X86-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}{1to8}, %zmm2, %zmm2 ## encoding: [0x62,0xf1,0xed,0x58,0xdb,0x15,A,A,A,A]
-; X86-NEXT: ## fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
-; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x08]
-; X86-NEXT: kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT: vpmuludq %zmm2, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0xfd,0x49,0xf4,0xca]
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx ## encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT: kmovw %ecx, %k1 ## encoding: [0xc5,0xf8,0x92,0xc9]
+; X86-NEXT: vpmuludq (%eax){1to8}, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0xfd,0x59,0xf4,0x08]
; X86-NEXT: vmovdqa64 %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
; X86-NEXT: retl ## encoding: [0xc3]
;
@@ -5687,12 +5681,9 @@ define <8 x i64> @test_mask_mul_epu32_rmbkz(<16 x i32> %a, ptr %ptr_b, i8 %mask)
; X86-LABEL: test_mask_mul_epu32_rmbkz:
; X86: ## %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT: vpbroadcastd (%eax), %zmm1 ## encoding: [0x62,0xf2,0x7d,0x48,0x58,0x08]
-; X86-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}{1to8}, %zmm1, %zmm1 ## encoding: [0x62,0xf1,0xf5,0x58,0xdb,0x0d,A,A,A,A]
-; X86-NEXT: ## fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
-; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x08]
-; X86-NEXT: kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT: vpmuludq %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xc9,0xf4,0xc1]
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx ## encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT: kmovw %ecx, %k1 ## encoding: [0xc5,0xf8,0x92,0xc9]
+; X86-NEXT: vpmuludq (%eax){1to8}, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xd9,0xf4,0x00]
; X86-NEXT: retl ## encoding: [0xc3]
;
; X64-LABEL: test_mask_mul_epu32_rmbkz:
@@ -7386,10 +7377,7 @@ define <8 x i64> @test_mul_epu32_rmb(<16 x i32> %a, ptr %ptr_b) {
; X86-LABEL: test_mul_epu32_rmb:
; X86: ## %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT: vpbroadcastd (%eax), %zmm1 ## encoding: [0x62,0xf2,0x7d,0x48,0x58,0x08]
-; X86-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}{1to8}, %zmm1, %zmm1 ## encoding: [0x62,0xf1,0xf5,0x58,0xdb,0x0d,A,A,A,A]
-; X86-NEXT: ## fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
-; X86-NEXT: vpmuludq %zmm1, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0xf4,0xc1]
+; X86-NEXT: vpmuludq (%eax){1to8}, %zmm0, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x58,0xf4,0x00]
; X86-NEXT: retl ## encoding: [0xc3]
;
; X64-LABEL: test_mul_epu32_rmb:
@@ -7408,12 +7396,9 @@ define <8 x i64> @test_mul_epu32_rmbk(<16 x i32> %a, ptr %ptr_b, <8 x i64> %pass
; X86-LABEL: test_mul_epu32_rmbk:
; X86: ## %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT: vpbroadcastd (%eax), %zmm2 ## encoding: [0x62,0xf2,0x7d,0x48,0x58,0x10]
-; X86-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}{1to8}, %zmm2, %zmm2 ## encoding: [0x62,0xf1,0xed,0x58,0xdb,0x15,A,A,A,A]
-; X86-NEXT: ## fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
-; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x08]
-; X86-NEXT: kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT: vpmuludq %zmm2, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0xfd,0x49,0xf4,0xca]
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx ## encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT: kmovw %ecx, %k1 ## encoding: [0xc5,0xf8,0x92,0xc9]
+; X86-NEXT: vpmuludq (%eax){1to8}, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf1,0xfd,0x59,0xf4,0x08]
; X86-NEXT: vmovdqa64 %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
; X86-NEXT: retl ## encoding: [0xc3]
;
@@ -7437,12 +7422,9 @@ define <8 x i64> @test_mul_epu32_rmbkz(<16 x i32> %a, ptr %ptr_b, i8 %mask) {
; X86-LABEL: test_mul_epu32_rmbkz:
; X86: ## %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax ## encoding: [0x8b,0x44,0x24,0x04]
-; X86-NEXT: vpbroadcastd (%eax), %zmm1 ## encoding: [0x62,0xf2,0x7d,0x48,0x58,0x08]
-; X86-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}{1to8}, %zmm1, %zmm1 ## encoding: [0x62,0xf1,0xf5,0x58,0xdb,0x0d,A,A,A,A]
-; X86-NEXT: ## fixup A - offset: 6, value: {{\.?LCPI[0-9]+_[0-9]+}}, kind: FK_Data_4
-; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x08]
-; X86-NEXT: kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT: vpmuludq %zmm1, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xc9,0xf4,0xc1]
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx ## encoding: [0x0f,0xb6,0x4c,0x24,0x08]
+; X86-NEXT: kmovw %ecx, %k1 ## encoding: [0xc5,0xf8,0x92,0xc9]
+; X86-NEXT: vpmuludq (%eax){1to8}, %zmm0, %zmm0 {%k1} {z} ## encoding: [0x62,0xf1,0xfd,0xd9,0xf4,0x00]
; X86-NEXT: retl ## encoding: [0xc3]
;
; X64-LABEL: test_mul_epu32_rmbkz:
diff --git a/llvm/test/CodeGen/X86/bit-manip-i128.ll b/llvm/test/CodeGen/X86/bit-manip-i128.ll
index 451ce3b1b7578..5d048bfead2f5 100644
--- a/llvm/test/CodeGen/X86/bit-manip-i128.ll
+++ b/llvm/test/CodeGen/X86/bit-manip-i128.ll
@@ -1153,142 +1153,46 @@ define i128 @bswap_i128_load(ptr %p0) nounwind {
}
define i128 @bitreverse_i128(i128 %a0) nounwind {
-; SSE2-LABEL: bitreverse_i128:
-; SSE2: # %bb.0:
-; SSE2-NEXT: bswapq %rsi
-; SSE2-NEXT: movq %rsi, %rax
-; SSE2-NEXT: shrq $4, %rax
-; SSE2-NEXT: movabsq $1085102592571150095, %rdx # imm = 0xF0F0F0F0F0F0F0F
-; SSE2-NEXT: andq %rdx, %rax
-; SSE2-NEXT: andq %rdx, %rsi
-; SSE2-NEXT: shlq $4, %rsi
-; SSE2-NEXT: orq %rax, %rsi
-; SSE2-NEXT: movabsq $3689348814741910323, %rcx # imm = 0x3333333333333333
-; SSE2-NEXT: movq %rsi, %rax
-; SSE2-NEXT: andq %rcx, %rax
-; SSE2-NEXT: shrq $2, %rsi
-; SSE2-NEXT: andq %rcx, %rsi
-; SSE2-NEXT: leaq (%rsi,%rax,4), %rax
-; SSE2-NEXT: movabsq $6148914691236517205, %rsi # imm = 0x5555555555555555
-; SSE2-NEXT: movq %rax, %r8
-; SSE2-NEXT: andq %rsi, %r8
-; SSE2-NEXT: shrq %rax
-; SSE2-NEXT: andq %rsi, %rax
-; SSE2-NEXT: leaq (%rax,%r8,2), %rax
-; SSE2-NEXT: bswapq %rdi
-; SSE2-NEXT: movq %rdi, %r8
-; SSE2-NEXT: shrq $4, %r8
-; SSE2-NEXT: andq %rdx, %r8
-; SSE2-NEXT: andq %rdx, %rdi
-; SSE2-NEXT: shlq $4, %rdi
-; SSE2-NEXT: orq %r8, %rdi
-; SSE2-NEXT: movq %rdi, %rdx
-; SSE2-NEXT: andq %rcx, %rdx
-; SSE2-NEXT: shrq $2, %rdi
-; SSE2-NEXT: andq %rcx, %rdi
-; SSE2-NEXT: leaq (%rdi,%rdx,4), %rcx
-; SSE2-NEXT: movq %rcx, %rdx
-; SSE2-NEXT: andq %rsi, %rdx
-; SSE2-NEXT: shrq %rcx
-; SSE2-NEXT: andq %rsi, %rcx
-; SSE2-NEXT: leaq (%rcx,%rdx,2), %rdx
-; SSE2-NEXT: retq
-;
-; SSE42-LABEL: bitreverse_i128:
-; SSE42: # %bb.0:
-; SSE42-NEXT: movq %rdi, %xmm0
-; SSE42-NEXT: movq %rsi, %xmm1
-; SSE42-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
-; SSE42-NEXT: pshufb {{.*#+}} xmm1 = xmm1[7,6,5,4,3,2,1,0,15,14,13,12,11,10,9,8]
-; SSE42-NEXT: movdqa {{.*#+}} xmm0 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; SSE42-NEXT: movdqa %xmm1, %xmm2
-; SSE42-NEXT: pand %xmm0, %xmm2
-; SSE42-NEXT: movdqa {{.*#+}} xmm3 = [0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240]
-; SSE42-NEXT: pshufb %xmm2, %xmm3
-; SSE42-NEXT: psrlw $4, %xmm1
-; SSE42-NEXT: pand %xmm0, %xmm1
-; SSE42-NEXT: movdqa {{.*#+}} xmm0 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15]
-; SSE42-NEXT: pshufb %xmm1, %xmm0
-; SSE42-NEXT: por %xmm3, %xmm0
-; SSE42-NEXT: movq %xmm0, %rax
-; SSE42-NEXT: pextrq $1, %xmm0, %rdx
-; SSE42-NEXT: retq
-;
-; AVX2-LABEL: bitreverse_i128:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vmovq %rdi, %xmm0
-; AVX2-NEXT: vmovq %rsi, %xmm1
-; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[7,6,5,4,3,2,1,0,15,14,13,12,11,10,9,8]
-; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX2-NEXT: vpand %xmm1, %xmm0, %xmm2
-; AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = [0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240]
-; AVX2-NEXT: vpshufb %xmm2, %xmm3, %xmm2
-; AVX2-NEXT: vpsrlw $4, %xmm0, %xmm0
-; AVX2-NEXT: vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vmovdqa {{.*#+}} xmm1 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15]
-; AVX2-NEXT: vpshufb %xmm0, %xmm1, %xmm0
-; AVX2-NEXT: vpor %xmm0, %xmm2, %xmm0
-; AVX2-NEXT: vmovq %xmm0, %rax
-; AVX2-NEXT: vpextrq $1, %xmm0, %rdx
-; AVX2-NEXT: retq
-;
-; AVX512F-LABEL: bitreverse_i128:
-; AVX512F: # %bb.0:
-; AVX512F-NEXT: vmovq %rdi, %xmm0
-; AVX512F-NEXT: vmovq %rsi, %xmm1
-; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX512F-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[7,6,5,4,3,2,1,0,15,14,13,12,11,10,9,8]
-; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512F-NEXT: vpand %xmm1, %xmm0, %xmm2
-; AVX512F-NEXT: vmovdqa {{.*#+}} xmm3 = [0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240]
-; AVX512F-NEXT: vpshufb %xmm2, %xmm3, %xmm2
-; AVX512F-NEXT: vpsrlw $4, %xmm0, %xmm0
-; AVX512F-NEXT: vpand %xmm1, %xmm0, %xmm0
-; AVX512F-NEXT: vmovdqa {{.*#+}} xmm1 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15]
-; AVX512F-NEXT: vpshufb %xmm0, %xmm1, %xmm0
-; AVX512F-NEXT: vpor %xmm0, %xmm2, %xmm0
-; AVX512F-NEXT: vmovq %xmm0, %rax
-; AVX512F-NEXT: vpextrq $1, %xmm0, %rdx
-; AVX512F-NEXT: retq
-;
-; AVX512VL-LABEL: bitreverse_i128:
-; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vmovq %rdi, %xmm0
-; AVX512VL-NEXT: vmovq %rsi, %xmm1
-; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX512VL-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[7,6,5,4,3,2,1,0,15,14,13,12,11,10,9,8]
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512VL-NEXT: vpand %xmm1, %xmm0, %xmm2
-; AVX512VL-NEXT: vmovdqa {{.*#+}} xmm3 = [0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240]
-; AVX512VL-NEXT: vpshufb %xmm2, %xmm3, %xmm2
-; AVX512VL-NEXT: vpsrlw $4, %xmm0, %xmm0
-; AVX512VL-NEXT: vmovdqa {{.*#+}} xmm3 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15]
-; AVX512VL-NEXT: vpand %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT: vpshufb %xmm0, %xmm3, %xmm0
-; AVX512VL-NEXT: vpor %xmm0, %xmm2, %xmm0
-; AVX512VL-NEXT: vmovq %xmm0, %rax
-; AVX512VL-NEXT: vpextrq $1, %xmm0, %rdx
-; AVX512VL-NEXT: retq
-;
-; AVX512VBMI-LABEL: bitreverse_i128:
-; AVX512VBMI: # %bb.0:
-; AVX512VBMI-NEXT: vmovq %rdi, %xmm0
-; AVX512VBMI-NEXT: vmovq %rsi, %xmm1
-; AVX512VBMI-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX512VBMI-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[7,6,5,4,3,2,1,0,15,14,13,12,11,10,9,8]
-; AVX512VBMI-NEXT: vpbroadcastd {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512VBMI-NEXT: vpand %xmm1, %xmm0, %xmm2
-; AVX512VBMI-NEXT: vmovdqa {{.*#+}} xmm3 = [0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240]
-; AVX512VBMI-NEXT: vpshufb %xmm2, %xmm3, %xmm2
-; AVX512VBMI-NEXT: vpsrlw $4, %xmm0, %xmm0
-; AVX512VBMI-NEXT: vmovdqa {{.*#+}} xmm3 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15]
-; AVX512VBMI-NEXT: vpand %xmm1, %xmm0, %xmm0
-; AVX512VBMI-NEXT: vpshufb %xmm0, %xmm3, %xmm0
-; AVX512VBMI-NEXT: vpor %xmm0, %xmm2, %xmm0
-; AVX512VBMI-NEXT: vmovq %xmm0, %rax
-; AVX512VBMI-NEXT: vpextrq $1, %xmm0, %rdx
-; AVX512VBMI-NEXT: retq
+; CHECK-LABEL: bitreverse_i128:
+; CHECK: # %bb.0:
+; CHECK-NEXT: bswapq %rsi
+; CHECK-NEXT: movq %rsi, %rax
+; CHECK-NEXT: shrq $4, %rax
+; CHECK-NEXT: movabsq $1085102592571150095, %rdx # imm = 0xF0F0F0F0F0F0F0F
+; CHECK-NEXT: andq %rdx, %rax
+; CHECK-NEXT: andq %rdx, %rsi
+; CHECK-NEXT: shlq $4, %rsi
+; CHECK-NEXT: orq %rax, %rsi
+; CHECK-NEXT: movabsq $3689348814741910323, %rcx # imm = 0x3333333333333333
+; CHECK-NEXT: movq %rsi, %rax
+; CHECK-NEXT: andq %rcx, %rax
+; CHECK-NEXT: shrq $2, %rsi
+; CHECK-NEXT: andq %rcx, %rsi
+; CHECK-NEXT: leaq (%rsi,%rax,4), %rax
+; CHECK-NEXT: movabsq $6148914691236517205, %rsi # imm = 0x5555555555555555
+; CHECK-NEXT: movq %rax, %r8
+; CHECK-NEXT: andq %rsi, %r8
+; CHECK-NEXT: shrq %rax
+; CHECK-NEXT: andq %rsi, %rax
+; CHECK-NEXT: leaq (%rax,%r8,2), %rax
+; CHECK-NEXT: bswapq %rdi
+; CHECK-NEXT: movq %rdi, %r8
+; CHECK-NEXT: shrq $4, %r8
+; CHECK-NEXT: andq %rdx, %r8
+; CHECK-NEXT: andq %rdx, %rdi
+; CHECK-NEXT: shlq $4, %rdi
+; CHECK-NEXT: orq %r8, %rdi
+; CHECK-NEXT: movq %rdi, %rdx
+; CHECK-NEXT: andq %rcx, %rdx
+; CHECK-NEXT: shrq $2, %rdi
+; CHECK-NEXT: andq %rcx, %rdi
+; CHECK-NEXT: leaq (%rdi,%rdx,4), %rcx
+; CHECK-NEXT: movq %rcx, %rdx
+; CHECK-NEXT: andq %rsi, %rdx
+; CHECK-NEXT: shrq %rcx
+; CHECK-NEXT: andq %rsi, %rcx
+; CHECK-NEXT: leaq (%rcx,%rdx,2), %rdx
+; CHECK-NEXT: retq
%res = call i128 @llvm.bitreverse.i128(i128 %a0)
ret i128 %res
}
@@ -1340,83 +1244,217 @@ define i128 @bitreverse_i128_vector(<2 x i64> %v0) nounwind {
;
; SSE42-LABEL: bitreverse_i128_vector:
; SSE42: # %bb.0:
-; SSE42-NEXT: pshufb {{.*#+}} xmm0 = xmm0[15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]
-; SSE42-NEXT: movdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; SSE42-NEXT: movdqa %xmm0, %xmm2
-; SSE42-NEXT: pand %xmm1, %xmm2
-; SSE42-NEXT: movdqa {{.*#+}} xmm3 = [0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240]
-; SSE42-NEXT: pshufb %xmm2, %xmm3
-; SSE42-NEXT: psrlw $4, %xmm0
-; SSE42-NEXT: pand %xmm1, %xmm0
-; SSE42-NEXT: movdqa {{.*#+}} xmm1 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15]
-; SSE42-NEXT: pshufb %xmm0, %xmm1
-; SSE42-NEXT: por %xmm3, %xmm1
-; SSE42-NEXT: movq %xmm1, %rax
-; SSE42-NEXT: pextrq $1, %xmm1, %rdx
+; SSE42-NEXT: pextrq $1, %xmm0, %rax
+; SSE42-NEXT: bswapq %rax
+; SSE42-NEXT: movq %xmm0, %rcx
+; SSE42-NEXT: movq %rax, %rdx
+; SSE42-NEXT: shrq $4, %rdx
+; SSE42-NEXT: movabsq $1085102592571150095, %rsi # imm = 0xF0F0F0F0F0F0F0F
+; SSE42-NEXT: andq %rsi, %rdx
+; SSE42-NEXT: andq %rsi, %rax
+; SSE42-NEXT: shlq $4, %rax
+; SSE42-NEXT: orq %rdx, %rax
+; SSE42-NEXT: movabsq $3689348814741910323, %rdx # imm = 0x3333333333333333
+; SSE42-NEXT: movq %rax, %rdi
+; SSE42-NEXT: andq %rdx, %rdi
+; SSE42-NEXT: shrq $2, %rax
+; SSE42-NEXT: andq %rdx, %rax
+; SSE42-NEXT: leaq (%rax,%rdi,4), %rax
+; SSE42-NEXT: movabsq $6148914691236517205, %rdi # imm = 0x5555555555555555
+; SSE42-NEXT: movq %rax, %r8
+; SSE42-NEXT: andq %rdi, %r8
+; SSE42-NEXT: shrq %rax
+; SSE42-NEXT: andq %rdi, %rax
+; SSE42-NEXT: leaq (%rax,%r8,2), %rax
+; SSE42-NEXT: bswapq %rcx
+; SSE42-NEXT: movq %rcx, %r8
+; SSE42-NEXT: shrq $4, %r8
+; SSE42-NEXT: andq %rsi, %r8
+; SSE42-NEXT: andq %rsi, %rcx
+; SSE42-NEXT: shlq $4, %rcx
+; SSE42-NEXT: orq %r8, %rcx
+; SSE42-NEXT: movq %rcx, %rsi
+; SSE42-NEXT: andq %rdx, %rsi
+; SSE42-NEXT: shrq $2, %rcx
+; SSE42-NEXT: andq %rdx, %rcx
+; SSE42-NEXT: leaq (%rcx,%rsi,4), %rcx
+; SSE42-NEXT: movq %rcx, %rdx
+; SSE42-NEXT: andq %rdi, %rdx
+; SSE42-NEXT: shrq %rcx
+; SSE42-NEXT: andq %rdi, %rcx
+; SSE42-NEXT: leaq (%rcx,%rdx,2), %rdx
; SSE42-NEXT: retq
;
; AVX2-LABEL: bitreverse_i128_vector:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]
-; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX2-NEXT: vpand %xmm1, %xmm0, %xmm2
-; AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = [0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240]
-; AVX2-NEXT: vpshufb %xmm2, %xmm3, %xmm2
-; AVX2-NEXT: vpsrlw $4, %xmm0, %xmm0
-; AVX2-NEXT: vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vmovdqa {{.*#+}} xmm1 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15]
-; AVX2-NEXT: vpshufb %xmm0, %xmm1, %xmm0
-; AVX2-NEXT: vpor %xmm0, %xmm2, %xmm0
-; AVX2-NEXT: vmovq %xmm0, %rax
-; AVX2-NEXT: vpextrq $1, %xmm0, %rdx
+; AVX2-NEXT: vpextrq $1, %xmm0, %rax
+; AVX2-NEXT: bswapq %rax
+; AVX2-NEXT: vmovq %xmm0, %rcx
+; AVX2-NEXT: movq %rax, %rdx
+; AVX2-NEXT: shrq $4, %rdx
+; AVX2-NEXT: movabsq $1085102592571150095, %rsi # imm = 0xF0F0F0F0F0F0F0F
+; AVX2-NEXT: andq %rsi, %rdx
+; AVX2-NEXT: andq %rsi, %rax
+; AVX2-NEXT: shlq $4, %rax
+; AVX2-NEXT: orq %rdx, %rax
+; AVX2-NEXT: movabsq $3689348814741910323, %rdx # imm = 0x3333333333333333
+; AVX2-NEXT: movq %rax, %rdi
+; AVX2-NEXT: andq %rdx, %rdi
+; AVX2-NEXT: shrq $2, %rax
+; AVX2-NEXT: andq %rdx, %rax
+; AVX2-NEXT: leaq (%rax,%rdi,4), %rax
+; AVX2-NEXT: movabsq $6148914691236517205, %rdi # imm = 0x5555555555555555
+; AVX2-NEXT: movq %rax, %r8
+; AVX2-NEXT: andq %rdi, %r8
+; AVX2-NEXT: shrq %rax
+; AVX2-NEXT: andq %rdi, %rax
+; AVX2-NEXT: leaq (%rax,%r8,2), %rax
+; AVX2-NEXT: bswapq %rcx
+; AVX2-NEXT: movq %rcx, %r8
+; AVX2-NEXT: shrq $4, %r8
+; AVX2-NEXT: andq %rsi, %r8
+; AVX2-NEXT: andq %rsi, %rcx
+; AVX2-NEXT: shlq $4, %rcx
+; AVX2-NEXT: orq %r8, %rcx
+; AVX2-NEXT: movq %rcx, %rsi
+; AVX2-NEXT: andq %rdx, %rsi
+; AVX2-NEXT: shrq $2, %rcx
+; AVX2-NEXT: andq %rdx, %rcx
+; AVX2-NEXT: leaq (%rcx,%rsi,4), %rcx
+; AVX2-NEXT: movq %rcx, %rdx
+; AVX2-NEXT: andq %rdi, %rdx
+; AVX2-NEXT: shrq %rcx
+; AVX2-NEXT: andq %rdi, %rcx
+; AVX2-NEXT: leaq (%rcx,%rdx,2), %rdx
; AVX2-NEXT: retq
;
; AVX512F-LABEL: bitreverse_i128_vector:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]
-; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512F-NEXT: vpand %xmm1, %xmm0, %xmm2
-; AVX512F-NEXT: vmovdqa {{.*#+}} xmm3 = [0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240]
-; AVX512F-NEXT: vpshufb %xmm2, %xmm3, %xmm2
-; AVX512F-NEXT: vpsrlw $4, %xmm0, %xmm0
-; AVX512F-NEXT: vpand %xmm1, %xmm0, %xmm0
-; AVX512F-NEXT: vmovdqa {{.*#+}} xmm1 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15]
-; AVX512F-NEXT: vpshufb %xmm0, %xmm1, %xmm0
-; AVX512F-NEXT: vpor %xmm0, %xmm2, %xmm0
-; AVX512F-NEXT: vmovq %xmm0, %rax
-; AVX512F-NEXT: vpextrq $1, %xmm0, %rdx
+; AVX512F-NEXT: vpextrq $1, %xmm0, %rax
+; AVX512F-NEXT: bswapq %rax
+; AVX512F-NEXT: vmovq %xmm0, %rcx
+; AVX512F-NEXT: movq %rax, %rdx
+; AVX512F-NEXT: shrq $4, %rdx
+; AVX512F-NEXT: movabsq $1085102592571150095, %rsi # imm = 0xF0F0F0F0F0F0F0F
+; AVX512F-NEXT: andq %rsi, %rdx
+; AVX512F-NEXT: andq %rsi, %rax
+; AVX512F-NEXT: shlq $4, %rax
+; AVX512F-NEXT: orq %rdx, %rax
+; AVX512F-NEXT: movabsq $3689348814741910323, %rdx # imm = 0x3333333333333333
+; AVX512F-NEXT: movq %rax, %rdi
+; AVX512F-NEXT: andq %rdx, %rdi
+; AVX512F-NEXT: shrq $2, %rax
+; AVX512F-NEXT: andq %rdx, %rax
+; AVX512F-NEXT: leaq (%rax,%rdi,4), %rax
+; AVX512F-NEXT: movabsq $6148914691236517205, %rdi # imm = 0x5555555555555555
+; AVX512F-NEXT: movq %rax, %r8
+; AVX512F-NEXT: andq %rdi, %r8
+; AVX512F-NEXT: shrq %rax
+; AVX512F-NEXT: andq %rdi, %rax
+; AVX512F-NEXT: leaq (%rax,%r8,2), %rax
+; AVX512F-NEXT: bswapq %rcx
+; AVX512F-NEXT: movq %rcx, %r8
+; AVX512F-NEXT: shrq $4, %r8
+; AVX512F-NEXT: andq %rsi, %r8
+; AVX512F-NEXT: andq %rsi, %rcx
+; AVX512F-NEXT: shlq $4, %rcx
+; AVX512F-NEXT: orq %r8, %rcx
+; AVX512F-NEXT: movq %rcx, %rsi
+; AVX512F-NEXT: andq %rdx, %rsi
+; AVX512F-NEXT: shrq $2, %rcx
+; AVX512F-NEXT: andq %rdx, %rcx
+; AVX512F-NEXT: leaq (%rcx,%rsi,4), %rcx
+; AVX512F-NEXT: movq %rcx, %rdx
+; AVX512F-NEXT: andq %rdi, %rdx
+; AVX512F-NEXT: shrq %rcx
+; AVX512F-NEXT: andq %rdi, %rcx
+; AVX512F-NEXT: leaq (%rcx,%rdx,2), %rdx
; AVX512F-NEXT: retq
;
; AVX512VL-LABEL: bitreverse_i128_vector:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512VL-NEXT: vpand %xmm1, %xmm0, %xmm2
-; AVX512VL-NEXT: vmovdqa {{.*#+}} xmm3 = [0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240]
-; AVX512VL-NEXT: vpshufb %xmm2, %xmm3, %xmm2
-; AVX512VL-NEXT: vpsrlw $4, %xmm0, %xmm0
-; AVX512VL-NEXT: vmovdqa {{.*#+}} xmm3 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15]
-; AVX512VL-NEXT: vpand %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT: vpshufb %xmm0, %xmm3, %xmm0
-; AVX512VL-NEXT: vpor %xmm0, %xmm2, %xmm0
-; AVX512VL-NEXT: vmovq %xmm0, %rax
-; AVX512VL-NEXT: vpextrq $1, %xmm0, %rdx
+; AVX512VL-NEXT: vmovq %xmm0, %rcx
+; AVX512VL-NEXT: vpextrq $1, %xmm0, %rax
+; AVX512VL-NEXT: bswapq %rax
+; AVX512VL-NEXT: movq %rax, %rdx
+; AVX512VL-NEXT: shrq $4, %rdx
+; AVX512VL-NEXT: movabsq $1085102592571150095, %rsi # imm = 0xF0F0F0F0F0F0F0F
+; AVX512VL-NEXT: andq %rsi, %rdx
+; AVX512VL-NEXT: andq %rsi, %rax
+; AVX512VL-NEXT: shlq $4, %rax
+; AVX512VL-NEXT: orq %rdx, %rax
+; AVX512VL-NEXT: movabsq $3689348814741910323, %rdx # imm = 0x3333333333333333
+; AVX512VL-NEXT: movq %rax, %rdi
+; AVX512VL-NEXT: andq %rdx, %rdi
+; AVX512VL-NEXT: shrq $2, %rax
+; AVX512VL-NEXT: andq %rdx, %rax
+; AVX512VL-NEXT: leaq (%rax,%rdi,4), %rax
+; AVX512VL-NEXT: movabsq $6148914691236517205, %rdi # imm = 0x5555555555555555
+; AVX512VL-NEXT: movq %rax, %r8
+; AVX512VL-NEXT: andq %rdi, %r8
+; AVX512VL-NEXT: shrq %rax
+; AVX512VL-NEXT: andq %rdi, %rax
+; AVX512VL-NEXT: leaq (%rax,%r8,2), %rax
+; AVX512VL-NEXT: bswapq %rcx
+; AVX512VL-NEXT: movq %rcx, %r8
+; AVX512VL-NEXT: shrq $4, %r8
+; AVX512VL-NEXT: andq %rsi, %r8
+; AVX512VL-NEXT: andq %rsi, %rcx
+; AVX512VL-NEXT: shlq $4, %rcx
+; AVX512VL-NEXT: orq %r8, %rcx
+; AVX512VL-NEXT: movq %rcx, %rsi
+; AVX512VL-NEXT: andq %rdx, %rsi
+; AVX512VL-NEXT: shrq $2, %rcx
+; AVX512VL-NEXT: andq %rdx, %rcx
+; AVX512VL-NEXT: leaq (%rcx,%rsi,4), %rcx
+; AVX512VL-NEXT: movq %rcx, %rdx
+; AVX512VL-NEXT: andq %rdi, %rdx
+; AVX512VL-NEXT: shrq %rcx
+; AVX512VL-NEXT: andq %rdi, %rcx
+; AVX512VL-NEXT: leaq (%rcx,%rdx,2), %rdx
; AVX512VL-NEXT: retq
;
; AVX512VBMI-LABEL: bitreverse_i128_vector:
; AVX512VBMI: # %bb.0:
-; AVX512VBMI-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]
-; AVX512VBMI-NEXT: vpbroadcastd {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512VBMI-NEXT: vpand %xmm1, %xmm0, %xmm2
-; AVX512VBMI-NEXT: vmovdqa {{.*#+}} xmm3 = [0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240]
-; AVX512VBMI-NEXT: vpshufb %xmm2, %xmm3, %xmm2
-; AVX512VBMI-NEXT: vpsrlw $4, %xmm0, %xmm0
-; AVX512VBMI-NEXT: vmovdqa {{.*#+}} xmm3 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15]
-; AVX512VBMI-NEXT: vpand %xmm1, %xmm0, %xmm0
-; AVX512VBMI-NEXT: vpshufb %xmm0, %xmm3, %xmm0
-; AVX512VBMI-NEXT: vpor %xmm0, %xmm2, %xmm0
-; AVX512VBMI-NEXT: vmovq %xmm0, %rax
-; AVX512VBMI-NEXT: vpextrq $1, %xmm0, %rdx
+; AVX512VBMI-NEXT: vmovq %xmm0, %rcx
+; AVX512VBMI-NEXT: vpextrq $1, %xmm0, %rax
+; AVX512VBMI-NEXT: bswapq %rax
+; AVX512VBMI-NEXT: movq %rax, %rdx
+; AVX512VBMI-NEXT: shrq $4, %rdx
+; AVX512VBMI-NEXT: movabsq $1085102592571150095, %rsi # imm = 0xF0F0F0F0F0F0F0F
+; AVX512VBMI-NEXT: andq %rsi, %rdx
+; AVX512VBMI-NEXT: andq %rsi, %rax
+; AVX512VBMI-NEXT: shlq $4, %rax
+; AVX512VBMI-NEXT: orq %rdx, %rax
+; AVX512VBMI-NEXT: movabsq $3689348814741910323, %rdx # imm = 0x3333333333333333
+; AVX512VBMI-NEXT: movq %rax, %rdi
+; AVX512VBMI-NEXT: andq %rdx, %rdi
+; AVX512VBMI-NEXT: shrq $2, %rax
+; AVX512VBMI-NEXT: andq %rdx, %rax
+; AVX512VBMI-NEXT: leaq (%rax,%rdi,4), %rax
+; AVX512VBMI-NEXT: movabsq $6148914691236517205, %rdi # imm = 0x5555555555555555
+; AVX512VBMI-NEXT: movq %rax, %r8
+; AVX512VBMI-NEXT: andq %rdi, %r8
+; AVX512VBMI-NEXT: shrq %rax
+; AVX512VBMI-NEXT: andq %rdi, %rax
+; AVX512VBMI-NEXT: leaq (%rax,%r8,2), %rax
+; AVX512VBMI-NEXT: bswapq %rcx
+; AVX512VBMI-NEXT: movq %rcx, %r8
+; AVX512VBMI-NEXT: shrq $4, %r8
+; AVX512VBMI-NEXT: andq %rsi, %r8
+; AVX512VBMI-NEXT: andq %rsi, %rcx
+; AVX512VBMI-NEXT: shlq $4, %rcx
+; AVX512VBMI-NEXT: orq %r8, %rcx
+; AVX512VBMI-NEXT: movq %rcx, %rsi
+; AVX512VBMI-NEXT: andq %rdx, %rsi
+; AVX512VBMI-NEXT: shrq $2, %rcx
+; AVX512VBMI-NEXT: andq %rdx, %rcx
+; AVX512VBMI-NEXT: leaq (%rcx,%rsi,4), %rcx
+; AVX512VBMI-NEXT: movq %rcx, %rdx
+; AVX512VBMI-NEXT: andq %rdi, %rdx
+; AVX512VBMI-NEXT: shrq %rcx
+; AVX512VBMI-NEXT: andq %rdi, %rcx
+; AVX512VBMI-NEXT: leaq (%rcx,%rdx,2), %rdx
; AVX512VBMI-NEXT: retq
%a0 = bitcast <2 x i64> %v0 to i128
%res = call i128 @llvm.bitreverse.i128(i128 %a0)
@@ -1424,134 +1462,89 @@ define i128 @bitreverse_i128_vector(<2 x i64> %v0) nounwind {
}
define i128 @bitreverse_i128_load(ptr %p0) nounwind {
-; SSE2-LABEL: bitreverse_i128_load:
-; SSE2: # %bb.0:
-; SSE2-NEXT: movq 8(%rdi), %rax
-; SSE2-NEXT: bswapq %rax
-; SSE2-NEXT: movq (%rdi), %rcx
-; SSE2-NEXT: movq %rax, %rdx
-; SSE2-NEXT: shrq $4, %rdx
-; SSE2-NEXT: movabsq $1085102592571150095, %rsi # imm = 0xF0F0F0F0F0F0F0F
-; SSE2-NEXT: andq %rsi, %rdx
-; SSE2-NEXT: andq %rsi, %rax
-; SSE2-NEXT: shlq $4, %rax
-; SSE2-NEXT: orq %rdx, %rax
-; SSE2-NEXT: movabsq $3689348814741910323, %rdx # imm = 0x3333333333333333
-; SSE2-NEXT: movq %rax, %rdi
-; SSE2-NEXT: andq %rdx, %rdi
-; SSE2-NEXT: shrq $2, %rax
-; SSE2-NEXT: andq %rdx, %rax
-; SSE2-NEXT: leaq (%rax,%rdi,4), %rax
-; SSE2-NEXT: movabsq $6148914691236517205, %rdi # imm = 0x5555555555555555
-; SSE2-NEXT: movq %rax, %r8
-; SSE2-NEXT: andq %rdi, %r8
-; SSE2-NEXT: shrq %rax
-; SSE2-NEXT: andq %rdi, %rax
-; SSE2-NEXT: leaq (%rax,%r8,2), %rax
-; SSE2-NEXT: bswapq %rcx
-; SSE2-NEXT: movq %rcx, %r8
-; SSE2-NEXT: shrq $4, %r8
-; SSE2-NEXT: andq %rsi, %r8
-; SSE2-NEXT: andq %rsi, %rcx
-; SSE2-NEXT: shlq $4, %rcx
-; SSE2-NEXT: orq %r8, %rcx
-; SSE2-NEXT: movq %rcx, %rsi
-; SSE2-NEXT: andq %rdx, %rsi
-; SSE2-NEXT: shrq $2, %rcx
-; SSE2-NEXT: andq %rdx, %rcx
-; SSE2-NEXT: leaq (%rcx,%rsi,4), %rcx
-; SSE2-NEXT: movq %rcx, %rdx
-; SSE2-NEXT: andq %rdi, %rdx
-; SSE2-NEXT: shrq %rcx
-; SSE2-NEXT: andq %rdi, %rcx
-; SSE2-NEXT: leaq (%rcx,%rdx,2), %rdx
-; SSE2-NEXT: retq
-;
-; SSE42-LABEL: bitreverse_i128_load:
-; SSE42: # %bb.0:
-; SSE42-NEXT: movdqa (%rdi), %xmm0
-; SSE42-NEXT: pshufb {{.*#+}} xmm0 = xmm0[15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]
-; SSE42-NEXT: movdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; SSE42-NEXT: movdqa %xmm0, %xmm2
-; SSE42-NEXT: pand %xmm1, %xmm2
-; SSE42-NEXT: movdqa {{.*#+}} xmm3 = [0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240]
-; SSE42-NEXT: pshufb %xmm2, %xmm3
-; SSE42-NEXT: psrlw $4, %xmm0
-; SSE42-NEXT: pand %xmm1, %xmm0
-; SSE42-NEXT: movdqa {{.*#+}} xmm1 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15]
-; SSE42-NEXT: pshufb %xmm0, %xmm1
-; SSE42-NEXT: por %xmm3, %xmm1
-; SSE42-NEXT: movq %xmm1, %rax
-; SSE42-NEXT: pextrq $1, %xmm1, %rdx
-; SSE42-NEXT: retq
-;
-; AVX2-LABEL: bitreverse_i128_load:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vmovdqa (%rdi), %xmm0
-; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]
-; AVX2-NEXT: vpbroadcastd {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX2-NEXT: vpand %xmm1, %xmm0, %xmm2
-; AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = [0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240]
-; AVX2-NEXT: vpshufb %xmm2, %xmm3, %xmm2
-; AVX2-NEXT: vpsrlw $4, %xmm0, %xmm0
-; AVX2-NEXT: vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vmovdqa {{.*#+}} xmm1 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15]
-; AVX2-NEXT: vpshufb %xmm0, %xmm1, %xmm0
-; AVX2-NEXT: vpor %xmm0, %xmm2, %xmm0
-; AVX2-NEXT: vmovq %xmm0, %rax
-; AVX2-NEXT: vpextrq $1, %xmm0, %rdx
-; AVX2-NEXT: retq
-;
-; AVX512F-LABEL: bitreverse_i128_load:
-; AVX512F: # %bb.0:
-; AVX512F-NEXT: vmovdqa (%rdi), %xmm0
-; AVX512F-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]
-; AVX512F-NEXT: vpbroadcastd {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512F-NEXT: vpand %xmm1, %xmm0, %xmm2
-; AVX512F-NEXT: vmovdqa {{.*#+}} xmm3 = [0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240]
-; AVX512F-NEXT: vpshufb %xmm2, %xmm3, %xmm2
-; AVX512F-NEXT: vpsrlw $4, %xmm0, %xmm0
-; AVX512F-NEXT: vpand %xmm1, %xmm0, %xmm0
-; AVX512F-NEXT: vmovdqa {{.*#+}} xmm1 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15]
-; AVX512F-NEXT: vpshufb %xmm0, %xmm1, %xmm0
-; AVX512F-NEXT: vpor %xmm0, %xmm2, %xmm0
-; AVX512F-NEXT: vmovq %xmm0, %rax
-; AVX512F-NEXT: vpextrq $1, %xmm0, %rdx
-; AVX512F-NEXT: retq
-;
-; AVX512VL-LABEL: bitreverse_i128_load:
-; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vmovdqa (%rdi), %xmm0
-; AVX512VL-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]
-; AVX512VL-NEXT: vpbroadcastd {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512VL-NEXT: vpand %xmm1, %xmm0, %xmm2
-; AVX512VL-NEXT: vmovdqa {{.*#+}} xmm3 = [0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240]
-; AVX512VL-NEXT: vpshufb %xmm2, %xmm3, %xmm2
-; AVX512VL-NEXT: vpsrlw $4, %xmm0, %xmm0
-; AVX512VL-NEXT: vmovdqa {{.*#+}} xmm3 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15]
-; AVX512VL-NEXT: vpand %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT: vpshufb %xmm0, %xmm3, %xmm0
-; AVX512VL-NEXT: vpor %xmm0, %xmm2, %xmm0
-; AVX512VL-NEXT: vmovq %xmm0, %rax
-; AVX512VL-NEXT: vpextrq $1, %xmm0, %rdx
-; AVX512VL-NEXT: retq
+; SSE-LABEL: bitreverse_i128_load:
+; SSE: # %bb.0:
+; SSE-NEXT: movq 8(%rdi), %rax
+; SSE-NEXT: bswapq %rax
+; SSE-NEXT: movq (%rdi), %rcx
+; SSE-NEXT: movq %rax, %rdx
+; SSE-NEXT: shrq $4, %rdx
+; SSE-NEXT: movabsq $1085102592571150095, %rsi # imm = 0xF0F0F0F0F0F0F0F
+; SSE-NEXT: andq %rsi, %rdx
+; SSE-NEXT: andq %rsi, %rax
+; SSE-NEXT: shlq $4, %rax
+; SSE-NEXT: orq %rdx, %rax
+; SSE-NEXT: movabsq $3689348814741910323, %rdx # imm = 0x3333333333333333
+; SSE-NEXT: movq %rax, %rdi
+; SSE-NEXT: andq %rdx, %rdi
+; SSE-NEXT: shrq $2, %rax
+; SSE-NEXT: andq %rdx, %rax
+; SSE-NEXT: leaq (%rax,%rdi,4), %rax
+; SSE-NEXT: movabsq $6148914691236517205, %rdi # imm = 0x5555555555555555
+; SSE-NEXT: movq %rax, %r8
+; SSE-NEXT: andq %rdi, %r8
+; SSE-NEXT: shrq %rax
+; SSE-NEXT: andq %rdi, %rax
+; SSE-NEXT: leaq (%rax,%r8,2), %rax
+; SSE-NEXT: bswapq %rcx
+; SSE-NEXT: movq %rcx, %r8
+; SSE-NEXT: shrq $4, %r8
+; SSE-NEXT: andq %rsi, %r8
+; SSE-NEXT: andq %rsi, %rcx
+; SSE-NEXT: shlq $4, %rcx
+; SSE-NEXT: orq %r8, %rcx
+; SSE-NEXT: movq %rcx, %rsi
+; SSE-NEXT: andq %rdx, %rsi
+; SSE-NEXT: shrq $2, %rcx
+; SSE-NEXT: andq %rdx, %rcx
+; SSE-NEXT: leaq (%rcx,%rsi,4), %rcx
+; SSE-NEXT: movq %rcx, %rdx
+; SSE-NEXT: andq %rdi, %rdx
+; SSE-NEXT: shrq %rcx
+; SSE-NEXT: andq %rdi, %rcx
+; SSE-NEXT: leaq (%rcx,%rdx,2), %rdx
+; SSE-NEXT: retq
;
-; AVX512VBMI-LABEL: bitreverse_i128_load:
-; AVX512VBMI: # %bb.0:
-; AVX512VBMI-NEXT: vmovdqa (%rdi), %xmm0
-; AVX512VBMI-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0]
-; AVX512VBMI-NEXT: vpbroadcastd {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512VBMI-NEXT: vpand %xmm1, %xmm0, %xmm2
-; AVX512VBMI-NEXT: vmovdqa {{.*#+}} xmm3 = [0,128,64,192,32,160,96,224,16,144,80,208,48,176,112,240]
-; AVX512VBMI-NEXT: vpshufb %xmm2, %xmm3, %xmm2
-; AVX512VBMI-NEXT: vpsrlw $4, %xmm0, %xmm0
-; AVX512VBMI-NEXT: vmovdqa {{.*#+}} xmm3 = [0,8,4,12,2,10,6,14,1,9,5,13,3,11,7,15]
-; AVX512VBMI-NEXT: vpand %xmm1, %xmm0, %xmm0
-; AVX512VBMI-NEXT: vpshufb %xmm0, %xmm3, %xmm0
-; AVX512VBMI-NEXT: vpor %xmm0, %xmm2, %xmm0
-; AVX512VBMI-NEXT: vmovq %xmm0, %rax
-; AVX512VBMI-NEXT: vpextrq $1, %xmm0, %rdx
-; AVX512VBMI-NEXT: retq
+; AVX-LABEL: bitreverse_i128_load:
+; AVX: # %bb.0:
+; AVX-NEXT: movbeq 8(%rdi), %rax
+; AVX-NEXT: movq %rax, %rcx
+; AVX-NEXT: shrq $4, %rcx
+; AVX-NEXT: movabsq $1085102592571150095, %rdx # imm = 0xF0F0F0F0F0F0F0F
+; AVX-NEXT: andq %rdx, %rcx
+; AVX-NEXT: andq %rdx, %rax
+; AVX-NEXT: shlq $4, %rax
+; AVX-NEXT: orq %rcx, %rax
+; AVX-NEXT: movabsq $3689348814741910323, %rcx # imm = 0x3333333333333333
+; AVX-NEXT: movq %rax, %rsi
+; AVX-NEXT: andq %rcx, %rsi
+; AVX-NEXT: shrq $2, %rax
+; AVX-NEXT: andq %rcx, %rax
+; AVX-NEXT: leaq (%rax,%rsi,4), %rax
+; AVX-NEXT: movabsq $6148914691236517205, %rsi # imm = 0x5555555555555555
+; AVX-NEXT: movq %rax, %r8
+; AVX-NEXT: andq %rsi, %r8
+; AVX-NEXT: shrq %rax
+; AVX-NEXT: andq %rsi, %rax
+; AVX-NEXT: leaq (%rax,%r8,2), %rax
+; AVX-NEXT: movbeq (%rdi), %rdi
+; AVX-NEXT: movq %rdi, %r8
+; AVX-NEXT: shrq $4, %r8
+; AVX-NEXT: andq %rdx, %r8
+; AVX-NEXT: andq %rdx, %rdi
+; AVX-NEXT: shlq $4, %rdi
+; AVX-NEXT: orq %r8, %rdi
+; AVX-NEXT: movq %rdi, %rdx
+; AVX-NEXT: andq %rcx, %rdx
+; AVX-NEXT: shrq $2, %rdi
+; AVX-NEXT: andq %rcx, %rdi
+; AVX-NEXT: leaq (%rdi,%rdx,4), %rcx
+; AVX-NEXT: movq %rcx, %rdx
+; AVX-NEXT: andq %rsi, %rdx
+; AVX-NEXT: shrq %rcx
+; AVX-NEXT: andq %rsi, %rcx
+; AVX-NEXT: leaq (%rcx,%rdx,2), %rdx
+; AVX-NEXT: retq
%a0 = load i128, ptr %p0
%res = call i128 @llvm.bitreverse.i128(i128 %a0)
ret i128 %res
diff --git a/llvm/test/CodeGen/X86/bitcast-vector-bool.ll b/llvm/test/CodeGen/X86/bitcast-vector-bool.ll
index f0688482fbeb8..fae1ff90dd8d5 100644
--- a/llvm/test/CodeGen/X86/bitcast-vector-bool.ll
+++ b/llvm/test/CodeGen/X86/bitcast-vector-bool.ll
@@ -634,11 +634,7 @@ define i1 @trunc_v32i8_cmp(<32 x i8> %a0) nounwind {
define i4 @bitcast_v8i64_to_v2i4(<8 x i64> %a0) nounwind {
; SSE-LABEL: bitcast_v8i64_to_v2i4:
; SSE: # %bb.0:
-; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
; SSE-NEXT: packssdw %xmm3, %xmm2
-; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
; SSE-NEXT: packssdw %xmm1, %xmm0
; SSE-NEXT: packssdw %xmm2, %xmm0
; SSE-NEXT: packsswb %xmm0, %xmm0
@@ -652,15 +648,13 @@ define i4 @bitcast_v8i64_to_v2i4(<8 x i64> %a0) nounwind {
;
; AVX1-LABEL: bitcast_v8i64_to_v2i4:
; AVX1: # %bb.0:
-; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT: vpcmpgtq %xmm2, %xmm3, %xmm2
-; AVX1-NEXT: vpcmpgtq %xmm1, %xmm3, %xmm1
-; AVX1-NEXT: vpackssdw %xmm2, %xmm1, %xmm1
; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2
+; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3
; AVX1-NEXT: vpcmpgtq %xmm2, %xmm3, %xmm2
; AVX1-NEXT: vpcmpgtq %xmm0, %xmm3, %xmm0
; AVX1-NEXT: vpackssdw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT: vpackssdw %xmm2, %xmm1, %xmm1
; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
; AVX1-NEXT: vmovmskps %ymm0, %eax
; AVX1-NEXT: movl %eax, %ecx
@@ -673,9 +667,6 @@ define i4 @bitcast_v8i64_to_v2i4(<8 x i64> %a0) nounwind {
;
; AVX2-LABEL: bitcast_v8i64_to_v2i4:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT: vpcmpgtq %ymm1, %ymm2, %ymm1
-; AVX2-NEXT: vpcmpgtq %ymm0, %ymm2, %ymm0
; AVX2-NEXT: vpackssdw %ymm1, %ymm0, %ymm0
; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
; AVX2-NEXT: vmovmskps %ymm0, %eax
@@ -1202,60 +1193,162 @@ define i64 @bitcast_v128i8_to_v2i64(<128 x i8> %a0) nounwind {
define i1 @trunc_v128i8_cmp(<128 x i8> %a0) nounwind {
; SSE2-SSSE3-LABEL: trunc_v128i8_cmp:
; SSE2-SSSE3: # %bb.0:
-; SSE2-SSSE3-NEXT: pand %xmm7, %xmm3
-; SSE2-SSSE3-NEXT: pand %xmm5, %xmm1
-; SSE2-SSSE3-NEXT: pand %xmm3, %xmm1
-; SSE2-SSSE3-NEXT: pand %xmm6, %xmm2
-; SSE2-SSSE3-NEXT: pand %xmm4, %xmm0
-; SSE2-SSSE3-NEXT: pand %xmm2, %xmm0
-; SSE2-SSSE3-NEXT: pand %xmm1, %xmm0
; SSE2-SSSE3-NEXT: psllw $7, %xmm0
; SSE2-SSSE3-NEXT: pmovmskb %xmm0, %eax
-; SSE2-SSSE3-NEXT: xorl $65535, %eax # imm = 0xFFFF
+; SSE2-SSSE3-NEXT: psllw $7, %xmm1
+; SSE2-SSSE3-NEXT: pmovmskb %xmm1, %ecx
+; SSE2-SSSE3-NEXT: shll $16, %ecx
+; SSE2-SSSE3-NEXT: orl %eax, %ecx
+; SSE2-SSSE3-NEXT: psllw $7, %xmm2
+; SSE2-SSSE3-NEXT: pmovmskb %xmm2, %edx
+; SSE2-SSSE3-NEXT: psllw $7, %xmm3
+; SSE2-SSSE3-NEXT: pmovmskb %xmm3, %eax
+; SSE2-SSSE3-NEXT: shll $16, %eax
+; SSE2-SSSE3-NEXT: orl %edx, %eax
+; SSE2-SSSE3-NEXT: shlq $32, %rax
+; SSE2-SSSE3-NEXT: orq %rcx, %rax
+; SSE2-SSSE3-NEXT: psllw $7, %xmm4
+; SSE2-SSSE3-NEXT: pmovmskb %xmm4, %ecx
+; SSE2-SSSE3-NEXT: psllw $7, %xmm5
+; SSE2-SSSE3-NEXT: pmovmskb %xmm5, %edx
+; SSE2-SSSE3-NEXT: shll $16, %edx
+; SSE2-SSSE3-NEXT: orl %ecx, %edx
+; SSE2-SSSE3-NEXT: psllw $7, %xmm6
+; SSE2-SSSE3-NEXT: pmovmskb %xmm6, %ecx
+; SSE2-SSSE3-NEXT: psllw $7, %xmm7
+; SSE2-SSSE3-NEXT: pmovmskb %xmm7, %esi
+; SSE2-SSSE3-NEXT: shll $16, %esi
+; SSE2-SSSE3-NEXT: orl %ecx, %esi
+; SSE2-SSSE3-NEXT: shlq $32, %rsi
+; SSE2-SSSE3-NEXT: orq %rdx, %rsi
+; SSE2-SSSE3-NEXT: movq %rsi, %xmm0
+; SSE2-SSSE3-NEXT: movq %rax, %xmm1
+; SSE2-SSSE3-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE2-SSSE3-NEXT: pcmpeqd %xmm0, %xmm0
+; SSE2-SSSE3-NEXT: pcmpeqb %xmm1, %xmm0
+; SSE2-SSSE3-NEXT: pmovmskb %xmm0, %eax
+; SSE2-SSSE3-NEXT: cmpl $65535, %eax # imm = 0xFFFF
; SSE2-SSSE3-NEXT: setne %al
; SSE2-SSSE3-NEXT: retq
;
; SSE41-LABEL: trunc_v128i8_cmp:
; SSE41: # %bb.0:
-; SSE41-NEXT: pand %xmm7, %xmm3
-; SSE41-NEXT: pand %xmm5, %xmm1
-; SSE41-NEXT: pand %xmm3, %xmm1
-; SSE41-NEXT: pand %xmm6, %xmm2
-; SSE41-NEXT: pand %xmm4, %xmm0
-; SSE41-NEXT: pand %xmm2, %xmm0
-; SSE41-NEXT: pand %xmm1, %xmm0
-; SSE41-NEXT: ptest {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE41-NEXT: psllw $7, %xmm0
+; SSE41-NEXT: pmovmskb %xmm0, %eax
+; SSE41-NEXT: psllw $7, %xmm1
+; SSE41-NEXT: pmovmskb %xmm1, %ecx
+; SSE41-NEXT: shll $16, %ecx
+; SSE41-NEXT: orl %eax, %ecx
+; SSE41-NEXT: psllw $7, %xmm2
+; SSE41-NEXT: pmovmskb %xmm2, %edx
+; SSE41-NEXT: psllw $7, %xmm3
+; SSE41-NEXT: pmovmskb %xmm3, %eax
+; SSE41-NEXT: shll $16, %eax
+; SSE41-NEXT: orl %edx, %eax
+; SSE41-NEXT: shlq $32, %rax
+; SSE41-NEXT: orq %rcx, %rax
+; SSE41-NEXT: psllw $7, %xmm4
+; SSE41-NEXT: pmovmskb %xmm4, %ecx
+; SSE41-NEXT: psllw $7, %xmm5
+; SSE41-NEXT: pmovmskb %xmm5, %edx
+; SSE41-NEXT: shll $16, %edx
+; SSE41-NEXT: orl %ecx, %edx
+; SSE41-NEXT: psllw $7, %xmm6
+; SSE41-NEXT: pmovmskb %xmm6, %ecx
+; SSE41-NEXT: psllw $7, %xmm7
+; SSE41-NEXT: pmovmskb %xmm7, %esi
+; SSE41-NEXT: shll $16, %esi
+; SSE41-NEXT: orl %ecx, %esi
+; SSE41-NEXT: shlq $32, %rsi
+; SSE41-NEXT: orq %rdx, %rsi
+; SSE41-NEXT: movq %rsi, %xmm0
+; SSE41-NEXT: movq %rax, %xmm1
+; SSE41-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0]
+; SSE41-NEXT: pcmpeqd %xmm0, %xmm0
+; SSE41-NEXT: ptest %xmm0, %xmm1
; SSE41-NEXT: setae %al
; SSE41-NEXT: retq
;
; AVX1-LABEL: trunc_v128i8_cmp:
; AVX1: # %bb.0:
-; AVX1-NEXT: vandps %ymm3, %ymm1, %ymm1
-; AVX1-NEXT: vandps %ymm2, %ymm0, %ymm0
-; AVX1-NEXT: vandps %ymm1, %ymm0, %ymm0
-; AVX1-NEXT: vptest {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0
+; AVX1-NEXT: vpsllw $7, %xmm0, %xmm4
+; AVX1-NEXT: vpmovmskb %xmm4, %eax
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT: vpsllw $7, %xmm0, %xmm0
+; AVX1-NEXT: vpmovmskb %xmm0, %ecx
+; AVX1-NEXT: shll $16, %ecx
+; AVX1-NEXT: orl %eax, %ecx
+; AVX1-NEXT: vpsllw $7, %xmm1, %xmm0
+; AVX1-NEXT: vpmovmskb %xmm0, %edx
+; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm0
+; AVX1-NEXT: vpsllw $7, %xmm0, %xmm0
+; AVX1-NEXT: vpmovmskb %xmm0, %eax
+; AVX1-NEXT: shll $16, %eax
+; AVX1-NEXT: orl %edx, %eax
+; AVX1-NEXT: shlq $32, %rax
+; AVX1-NEXT: orq %rcx, %rax
+; AVX1-NEXT: vpsllw $7, %xmm2, %xmm0
+; AVX1-NEXT: vpmovmskb %xmm0, %ecx
+; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm0
+; AVX1-NEXT: vpsllw $7, %xmm0, %xmm0
+; AVX1-NEXT: vpmovmskb %xmm0, %edx
+; AVX1-NEXT: shll $16, %edx
+; AVX1-NEXT: orl %ecx, %edx
+; AVX1-NEXT: vpsllw $7, %xmm3, %xmm0
+; AVX1-NEXT: vpmovmskb %xmm0, %ecx
+; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm0
+; AVX1-NEXT: vpsllw $7, %xmm0, %xmm0
+; AVX1-NEXT: vpmovmskb %xmm0, %esi
+; AVX1-NEXT: shll $16, %esi
+; AVX1-NEXT: orl %ecx, %esi
+; AVX1-NEXT: shlq $32, %rsi
+; AVX1-NEXT: orq %rdx, %rsi
+; AVX1-NEXT: vmovq %rsi, %xmm0
+; AVX1-NEXT: vmovq %rax, %xmm1
+; AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX1-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1
+; AVX1-NEXT: vptest %xmm1, %xmm0
; AVX1-NEXT: setae %al
; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
; AVX2-LABEL: trunc_v128i8_cmp:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpand %ymm3, %ymm1, %ymm1
-; AVX2-NEXT: vpand %ymm2, %ymm0, %ymm0
-; AVX2-NEXT: vpand %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: vpbroadcastq {{.*#+}} ymm1 = [72340172838076673,72340172838076673,72340172838076673,72340172838076673]
-; AVX2-NEXT: vptest %ymm1, %ymm0
+; AVX2-NEXT: vpsllw $7, %ymm1, %ymm1
+; AVX2-NEXT: vpmovmskb %ymm1, %eax
+; AVX2-NEXT: shlq $32, %rax
+; AVX2-NEXT: vpsllw $7, %ymm0, %ymm0
+; AVX2-NEXT: vpmovmskb %ymm0, %ecx
+; AVX2-NEXT: orq %rax, %rcx
+; AVX2-NEXT: vpsllw $7, %ymm3, %ymm0
+; AVX2-NEXT: vpmovmskb %ymm0, %eax
+; AVX2-NEXT: shlq $32, %rax
+; AVX2-NEXT: vpsllw $7, %ymm2, %ymm0
+; AVX2-NEXT: vpmovmskb %ymm0, %edx
+; AVX2-NEXT: orq %rax, %rdx
+; AVX2-NEXT: vmovq %rdx, %xmm0
+; AVX2-NEXT: vmovq %rcx, %xmm1
+; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX2-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vptest %xmm1, %xmm0
; AVX2-NEXT: setae %al
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512-LABEL: trunc_v128i8_cmp:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpbroadcastb {{.*#+}} zmm2 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX512-NEXT: vpternlogq {{.*#+}} zmm0 = zmm0 & zmm2 & zmm1
-; AVX512-NEXT: vpcmpneqd %zmm2, %zmm0, %k0
-; AVX512-NEXT: kortestw %k0, %k0
-; AVX512-NEXT: setne %al
+; AVX512-NEXT: vpsllw $7, %zmm0, %zmm0
+; AVX512-NEXT: vpmovb2m %zmm0, %k0
+; AVX512-NEXT: kmovq %k0, %rax
+; AVX512-NEXT: vpsllw $7, %zmm1, %zmm0
+; AVX512-NEXT: vpmovb2m %zmm0, %k0
+; AVX512-NEXT: kmovq %k0, %rcx
+; AVX512-NEXT: vmovq %rcx, %xmm0
+; AVX512-NEXT: vmovq %rax, %xmm1
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vptest %xmm1, %xmm0
+; AVX512-NEXT: setae %al
; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
%1 = trunc <128 x i8> %a0 to <128 x i1>
diff --git a/llvm/test/CodeGen/X86/bitcnt-big-integer.ll b/llvm/test/CodeGen/X86/bitcnt-big-integer.ll
index 5c9a9338bc32f..ed64278c3fc3e 100644
--- a/llvm/test/CodeGen/X86/bitcnt-big-integer.ll
+++ b/llvm/test/CodeGen/X86/bitcnt-big-integer.ll
@@ -495,12 +495,14 @@ define i32 @load_ctpop_i256(ptr %p0) nounwind {
;
; AVX512POPCNT-LABEL: load_ctpop_i256:
; AVX512POPCNT: # %bb.0:
-; AVX512POPCNT-NEXT: vmovdqu (%rdi), %ymm0
-; AVX512POPCNT-NEXT: vpopcntq %zmm0, %zmm0
-; AVX512POPCNT-NEXT: vpmovqb %zmm0, %xmm0
-; AVX512POPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512POPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512POPCNT-NEXT: vmovd %xmm0, %eax
+; AVX512POPCNT-NEXT: popcntq 24(%rdi), %rax
+; AVX512POPCNT-NEXT: popcntq 16(%rdi), %rcx
+; AVX512POPCNT-NEXT: addl %eax, %ecx
+; AVX512POPCNT-NEXT: popcntq 8(%rdi), %rdx
+; AVX512POPCNT-NEXT: popcntq (%rdi), %rax
+; AVX512POPCNT-NEXT: addl %edx, %eax
+; AVX512POPCNT-NEXT: addl %ecx, %eax
+; AVX512POPCNT-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: load_ctpop_i256:
@@ -518,12 +520,15 @@ define i32 @load_ctpop_i256(ptr %p0) nounwind {
;
; AVX512VLPOPCNT-LABEL: load_ctpop_i256:
; AVX512VLPOPCNT: # %bb.0:
-; AVX512VLPOPCNT-NEXT: vpopcntq (%rdi), %ymm0
-; AVX512VLPOPCNT-NEXT: vpmovqb %ymm0, %xmm0
-; AVX512VLPOPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512VLPOPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
-; AVX512VLPOPCNT-NEXT: vzeroupper
+; AVX512VLPOPCNT-NEXT: popcntq 24(%rdi), %rax
+; AVX512VLPOPCNT-NEXT: popcntq 16(%rdi), %rcx
+; AVX512VLPOPCNT-NEXT: addl %eax, %ecx
+; AVX512VLPOPCNT-NEXT: popcntq 8(%rdi), %rdx
+; AVX512VLPOPCNT-NEXT: xorl %eax, %eax
+; AVX512VLPOPCNT-NEXT: popcntq (%rdi), %rax
+; AVX512VLPOPCNT-NEXT: addl %edx, %eax
+; AVX512VLPOPCNT-NEXT: addl %ecx, %eax
+; AVX512VLPOPCNT-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512VLPOPCNT-NEXT: retq
%a0 = load i256, ptr %p0
%cnt = call i256 @llvm.ctpop.i256(i256 %a0)
@@ -589,12 +594,19 @@ define i32 @vector_ctpop_i256(<8 x i32> %v0) nounwind {
;
; AVX512POPCNT-LABEL: vector_ctpop_i256:
; AVX512POPCNT: # %bb.0:
-; AVX512POPCNT-NEXT: vmovdqa %ymm0, %ymm0
-; AVX512POPCNT-NEXT: vpopcntq %zmm0, %zmm0
-; AVX512POPCNT-NEXT: vpmovqb %zmm0, %xmm0
-; AVX512POPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512POPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512POPCNT-NEXT: vmovd %xmm0, %eax
+; AVX512POPCNT-NEXT: vpextrq $1, %xmm0, %rax
+; AVX512POPCNT-NEXT: vmovq %xmm0, %rcx
+; AVX512POPCNT-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX512POPCNT-NEXT: vpextrq $1, %xmm0, %rdx
+; AVX512POPCNT-NEXT: vmovq %xmm0, %rsi
+; AVX512POPCNT-NEXT: popcntq %rdx, %rdx
+; AVX512POPCNT-NEXT: popcntq %rsi, %rsi
+; AVX512POPCNT-NEXT: addl %edx, %esi
+; AVX512POPCNT-NEXT: popcntq %rax, %rdx
+; AVX512POPCNT-NEXT: popcntq %rcx, %rax
+; AVX512POPCNT-NEXT: addl %edx, %eax
+; AVX512POPCNT-NEXT: addl %esi, %eax
+; AVX512POPCNT-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: vector_ctpop_i256:
@@ -619,11 +631,21 @@ define i32 @vector_ctpop_i256(<8 x i32> %v0) nounwind {
;
; AVX512VLPOPCNT-LABEL: vector_ctpop_i256:
; AVX512VLPOPCNT: # %bb.0:
-; AVX512VLPOPCNT-NEXT: vpopcntq %ymm0, %ymm0
-; AVX512VLPOPCNT-NEXT: vpmovqb %ymm0, %xmm0
-; AVX512VLPOPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512VLPOPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
+; AVX512VLPOPCNT-NEXT: vpextrq $1, %xmm0, %rax
+; AVX512VLPOPCNT-NEXT: vmovq %xmm0, %rcx
+; AVX512VLPOPCNT-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX512VLPOPCNT-NEXT: vmovq %xmm0, %rdx
+; AVX512VLPOPCNT-NEXT: vpextrq $1, %xmm0, %rsi
+; AVX512VLPOPCNT-NEXT: popcntq %rsi, %rsi
+; AVX512VLPOPCNT-NEXT: popcntq %rdx, %rdx
+; AVX512VLPOPCNT-NEXT: addl %esi, %edx
+; AVX512VLPOPCNT-NEXT: xorl %esi, %esi
+; AVX512VLPOPCNT-NEXT: popcntq %rax, %rsi
+; AVX512VLPOPCNT-NEXT: xorl %eax, %eax
+; AVX512VLPOPCNT-NEXT: popcntq %rcx, %rax
+; AVX512VLPOPCNT-NEXT: addl %esi, %eax
+; AVX512VLPOPCNT-NEXT: addl %edx, %eax
+; AVX512VLPOPCNT-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%a0 = bitcast <8 x i32> %v0 to i256
@@ -705,16 +727,22 @@ define <2 x i32> @load_ctpop_v2i256_v2i32(ptr %p0) nounwind {
;
; AVX512POPCNT-LABEL: load_ctpop_v2i256_v2i32:
; AVX512POPCNT: # %bb.0:
-; AVX512POPCNT-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512POPCNT-NEXT: vmovdqa 32(%rdi), %ymm1
-; AVX512POPCNT-NEXT: vpopcntq %zmm1, %zmm1
-; AVX512POPCNT-NEXT: vpmovqb %zmm1, %xmm1
-; AVX512POPCNT-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512POPCNT-NEXT: vpsadbw %xmm2, %xmm1, %xmm1
-; AVX512POPCNT-NEXT: vpopcntq %zmm0, %zmm0
-; AVX512POPCNT-NEXT: vpmovqb %zmm0, %xmm0
-; AVX512POPCNT-NEXT: vpsadbw %xmm2, %xmm0, %xmm0
-; AVX512POPCNT-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX512POPCNT-NEXT: popcntq 56(%rdi), %rax
+; AVX512POPCNT-NEXT: popcntq 48(%rdi), %rcx
+; AVX512POPCNT-NEXT: addl %eax, %ecx
+; AVX512POPCNT-NEXT: popcntq 40(%rdi), %rax
+; AVX512POPCNT-NEXT: popcntq 32(%rdi), %rdx
+; AVX512POPCNT-NEXT: addl %eax, %edx
+; AVX512POPCNT-NEXT: addl %ecx, %edx
+; AVX512POPCNT-NEXT: popcntq 24(%rdi), %rax
+; AVX512POPCNT-NEXT: popcntq 16(%rdi), %rcx
+; AVX512POPCNT-NEXT: popcntq 8(%rdi), %rsi
+; AVX512POPCNT-NEXT: addl %eax, %ecx
+; AVX512POPCNT-NEXT: popcntq (%rdi), %rax
+; AVX512POPCNT-NEXT: addl %esi, %eax
+; AVX512POPCNT-NEXT: addl %ecx, %eax
+; AVX512POPCNT-NEXT: vmovd %eax, %xmm0
+; AVX512POPCNT-NEXT: vpinsrd $1, %edx, %xmm0, %xmm0
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: load_ctpop_v2i256_v2i32:
@@ -742,15 +770,25 @@ define <2 x i32> @load_ctpop_v2i256_v2i32(ptr %p0) nounwind {
;
; AVX512VLPOPCNT-LABEL: load_ctpop_v2i256_v2i32:
; AVX512VLPOPCNT: # %bb.0:
-; AVX512VLPOPCNT-NEXT: vpopcntq 32(%rdi), %ymm0
-; AVX512VLPOPCNT-NEXT: vpmovqb %ymm0, %xmm0
-; AVX512VLPOPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512VLPOPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512VLPOPCNT-NEXT: vpopcntq (%rdi), %ymm2
-; AVX512VLPOPCNT-NEXT: vpmovqb %ymm2, %xmm2
-; AVX512VLPOPCNT-NEXT: vpsadbw %xmm1, %xmm2, %xmm1
-; AVX512VLPOPCNT-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; AVX512VLPOPCNT-NEXT: vzeroupper
+; AVX512VLPOPCNT-NEXT: popcntq 56(%rdi), %rax
+; AVX512VLPOPCNT-NEXT: popcntq 48(%rdi), %rcx
+; AVX512VLPOPCNT-NEXT: addl %eax, %ecx
+; AVX512VLPOPCNT-NEXT: xorl %eax, %eax
+; AVX512VLPOPCNT-NEXT: popcntq 40(%rdi), %rax
+; AVX512VLPOPCNT-NEXT: popcntq 32(%rdi), %rdx
+; AVX512VLPOPCNT-NEXT: addl %eax, %edx
+; AVX512VLPOPCNT-NEXT: addl %ecx, %edx
+; AVX512VLPOPCNT-NEXT: xorl %eax, %eax
+; AVX512VLPOPCNT-NEXT: popcntq 24(%rdi), %rax
+; AVX512VLPOPCNT-NEXT: xorl %ecx, %ecx
+; AVX512VLPOPCNT-NEXT: popcntq 16(%rdi), %rcx
+; AVX512VLPOPCNT-NEXT: popcntq 8(%rdi), %rsi
+; AVX512VLPOPCNT-NEXT: popcntq (%rdi), %rdi
+; AVX512VLPOPCNT-NEXT: addl %eax, %ecx
+; AVX512VLPOPCNT-NEXT: addl %esi, %edi
+; AVX512VLPOPCNT-NEXT: addl %ecx, %edi
+; AVX512VLPOPCNT-NEXT: vmovd %edi, %xmm0
+; AVX512VLPOPCNT-NEXT: vpinsrd $1, %edx, %xmm0, %xmm0
; AVX512VLPOPCNT-NEXT: retq
%a0 = load <2 x i256>, ptr %p0
%cnt = call <2 x i256> @llvm.ctpop.v2i256(<2 x i256> %a0)
@@ -832,15 +870,23 @@ define <2 x i64> @load_ctpop_v2i256_v2i64(ptr %p0) nounwind {
;
; AVX512POPCNT-LABEL: load_ctpop_v2i256_v2i64:
; AVX512POPCNT: # %bb.0:
-; AVX512POPCNT-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512POPCNT-NEXT: vmovdqa 32(%rdi), %ymm1
-; AVX512POPCNT-NEXT: vpopcntq %zmm1, %zmm1
-; AVX512POPCNT-NEXT: vpmovqb %zmm1, %xmm1
-; AVX512POPCNT-NEXT: vpopcntq %zmm0, %zmm0
-; AVX512POPCNT-NEXT: vpmovqb %zmm0, %xmm0
-; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX512POPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512POPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512POPCNT-NEXT: popcntq 24(%rdi), %rax
+; AVX512POPCNT-NEXT: popcntq 16(%rdi), %rcx
+; AVX512POPCNT-NEXT: addl %eax, %ecx
+; AVX512POPCNT-NEXT: popcntq 8(%rdi), %rax
+; AVX512POPCNT-NEXT: popcntq (%rdi), %rdx
+; AVX512POPCNT-NEXT: addl %eax, %edx
+; AVX512POPCNT-NEXT: addl %ecx, %edx
+; AVX512POPCNT-NEXT: popcntq 56(%rdi), %rax
+; AVX512POPCNT-NEXT: popcntq 48(%rdi), %rcx
+; AVX512POPCNT-NEXT: popcntq 40(%rdi), %rsi
+; AVX512POPCNT-NEXT: addl %eax, %ecx
+; AVX512POPCNT-NEXT: popcntq 32(%rdi), %rax
+; AVX512POPCNT-NEXT: addl %esi, %eax
+; AVX512POPCNT-NEXT: addl %ecx, %eax
+; AVX512POPCNT-NEXT: vmovd %eax, %xmm0
+; AVX512POPCNT-NEXT: vmovd %edx, %xmm1
+; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: load_ctpop_v2i256_v2i64:
@@ -869,14 +915,26 @@ define <2 x i64> @load_ctpop_v2i256_v2i64(ptr %p0) nounwind {
;
; AVX512VLPOPCNT-LABEL: load_ctpop_v2i256_v2i64:
; AVX512VLPOPCNT: # %bb.0:
-; AVX512VLPOPCNT-NEXT: vpopcntq 32(%rdi), %ymm0
-; AVX512VLPOPCNT-NEXT: vpmovqb %ymm0, %xmm0
-; AVX512VLPOPCNT-NEXT: vpopcntq (%rdi), %ymm1
-; AVX512VLPOPCNT-NEXT: vpmovqb %ymm1, %xmm1
+; AVX512VLPOPCNT-NEXT: popcntq 24(%rdi), %rax
+; AVX512VLPOPCNT-NEXT: popcntq 16(%rdi), %rcx
+; AVX512VLPOPCNT-NEXT: addl %eax, %ecx
+; AVX512VLPOPCNT-NEXT: xorl %eax, %eax
+; AVX512VLPOPCNT-NEXT: popcntq 8(%rdi), %rax
+; AVX512VLPOPCNT-NEXT: popcntq (%rdi), %rdx
+; AVX512VLPOPCNT-NEXT: addl %eax, %edx
+; AVX512VLPOPCNT-NEXT: addl %ecx, %edx
+; AVX512VLPOPCNT-NEXT: xorl %eax, %eax
+; AVX512VLPOPCNT-NEXT: popcntq 56(%rdi), %rax
+; AVX512VLPOPCNT-NEXT: xorl %ecx, %ecx
+; AVX512VLPOPCNT-NEXT: popcntq 48(%rdi), %rcx
+; AVX512VLPOPCNT-NEXT: popcntq 40(%rdi), %rsi
+; AVX512VLPOPCNT-NEXT: popcntq 32(%rdi), %rdi
+; AVX512VLPOPCNT-NEXT: addl %eax, %ecx
+; AVX512VLPOPCNT-NEXT: addl %esi, %edi
+; AVX512VLPOPCNT-NEXT: addl %ecx, %edi
+; AVX512VLPOPCNT-NEXT: vmovd %edi, %xmm0
+; AVX512VLPOPCNT-NEXT: vmovd %edx, %xmm1
; AVX512VLPOPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX512VLPOPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512VLPOPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%a0 = load <2 x i256>, ptr %p0
%cnt = call <2 x i256> @llvm.ctpop.v2i256(<2 x i256> %a0)
@@ -1074,11 +1132,22 @@ define i32 @load_ctpop_i512(ptr %p0) nounwind {
;
; AVX512POPCNT-LABEL: load_ctpop_i512:
; AVX512POPCNT: # %bb.0:
-; AVX512POPCNT-NEXT: vpopcntq (%rdi), %zmm0
-; AVX512POPCNT-NEXT: vpmovqb %zmm0, %xmm0
-; AVX512POPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512POPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512POPCNT-NEXT: vmovd %xmm0, %eax
+; AVX512POPCNT-NEXT: popcntq 56(%rdi), %rax
+; AVX512POPCNT-NEXT: popcntq 48(%rdi), %rcx
+; AVX512POPCNT-NEXT: addl %eax, %ecx
+; AVX512POPCNT-NEXT: popcntq 40(%rdi), %rax
+; AVX512POPCNT-NEXT: popcntq 32(%rdi), %rdx
+; AVX512POPCNT-NEXT: addl %eax, %edx
+; AVX512POPCNT-NEXT: addl %ecx, %edx
+; AVX512POPCNT-NEXT: popcntq 24(%rdi), %rcx
+; AVX512POPCNT-NEXT: popcntq 16(%rdi), %rsi
+; AVX512POPCNT-NEXT: popcntq 8(%rdi), %r8
+; AVX512POPCNT-NEXT: popcntq (%rdi), %rax
+; AVX512POPCNT-NEXT: addl %ecx, %esi
+; AVX512POPCNT-NEXT: addl %r8d, %eax
+; AVX512POPCNT-NEXT: addl %esi, %eax
+; AVX512POPCNT-NEXT: addl %edx, %eax
+; AVX512POPCNT-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: load_ctpop_i512:
@@ -1107,12 +1176,26 @@ define i32 @load_ctpop_i512(ptr %p0) nounwind {
;
; AVX512VLPOPCNT-LABEL: load_ctpop_i512:
; AVX512VLPOPCNT: # %bb.0:
-; AVX512VLPOPCNT-NEXT: vpopcntq (%rdi), %zmm0
-; AVX512VLPOPCNT-NEXT: vpmovqb %zmm0, %xmm0
-; AVX512VLPOPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512VLPOPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
-; AVX512VLPOPCNT-NEXT: vzeroupper
+; AVX512VLPOPCNT-NEXT: popcntq 56(%rdi), %rax
+; AVX512VLPOPCNT-NEXT: popcntq 48(%rdi), %rcx
+; AVX512VLPOPCNT-NEXT: addl %eax, %ecx
+; AVX512VLPOPCNT-NEXT: xorl %eax, %eax
+; AVX512VLPOPCNT-NEXT: popcntq 40(%rdi), %rax
+; AVX512VLPOPCNT-NEXT: popcntq 32(%rdi), %rdx
+; AVX512VLPOPCNT-NEXT: addl %eax, %edx
+; AVX512VLPOPCNT-NEXT: addl %ecx, %edx
+; AVX512VLPOPCNT-NEXT: xorl %eax, %eax
+; AVX512VLPOPCNT-NEXT: popcntq 24(%rdi), %rax
+; AVX512VLPOPCNT-NEXT: xorl %ecx, %ecx
+; AVX512VLPOPCNT-NEXT: popcntq 16(%rdi), %rcx
+; AVX512VLPOPCNT-NEXT: popcntq 8(%rdi), %rsi
+; AVX512VLPOPCNT-NEXT: addl %eax, %ecx
+; AVX512VLPOPCNT-NEXT: xorl %eax, %eax
+; AVX512VLPOPCNT-NEXT: popcntq (%rdi), %rax
+; AVX512VLPOPCNT-NEXT: addl %esi, %eax
+; AVX512VLPOPCNT-NEXT: addl %ecx, %eax
+; AVX512VLPOPCNT-NEXT: addl %edx, %eax
+; AVX512VLPOPCNT-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512VLPOPCNT-NEXT: retq
%a0 = load i512, ptr %p0
%cnt = call i512 @llvm.ctpop.i512(i512 %a0)
@@ -1213,11 +1296,33 @@ define i32 @vector_ctpop_i512(<16 x i32> %v0) nounwind {
;
; AVX512POPCNT-LABEL: vector_ctpop_i512:
; AVX512POPCNT: # %bb.0:
-; AVX512POPCNT-NEXT: vpopcntq %zmm0, %zmm0
-; AVX512POPCNT-NEXT: vpmovqb %zmm0, %xmm0
-; AVX512POPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512POPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512POPCNT-NEXT: vmovd %xmm0, %eax
+; AVX512POPCNT-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512POPCNT-NEXT: vmovq %xmm1, %rax
+; AVX512POPCNT-NEXT: vpextrq $1, %xmm1, %rcx
+; AVX512POPCNT-NEXT: vpextrq $1, %xmm0, %rdx
+; AVX512POPCNT-NEXT: vmovq %xmm0, %rsi
+; AVX512POPCNT-NEXT: vextracti32x4 $2, %zmm0, %xmm1
+; AVX512POPCNT-NEXT: vpextrq $1, %xmm1, %rdi
+; AVX512POPCNT-NEXT: vmovq %xmm1, %r8
+; AVX512POPCNT-NEXT: vextracti32x4 $3, %zmm0, %xmm0
+; AVX512POPCNT-NEXT: vpextrq $1, %xmm0, %r9
+; AVX512POPCNT-NEXT: vmovq %xmm0, %r10
+; AVX512POPCNT-NEXT: popcntq %r9, %r9
+; AVX512POPCNT-NEXT: popcntq %r10, %r10
+; AVX512POPCNT-NEXT: addl %r9d, %r10d
+; AVX512POPCNT-NEXT: popcntq %rdi, %rdi
+; AVX512POPCNT-NEXT: popcntq %r8, %r8
+; AVX512POPCNT-NEXT: addl %edi, %r8d
+; AVX512POPCNT-NEXT: addl %r10d, %r8d
+; AVX512POPCNT-NEXT: popcntq %rdx, %rdx
+; AVX512POPCNT-NEXT: popcntq %rsi, %rsi
+; AVX512POPCNT-NEXT: addl %edx, %esi
+; AVX512POPCNT-NEXT: popcntq %rcx, %rcx
+; AVX512POPCNT-NEXT: popcntq %rax, %rax
+; AVX512POPCNT-NEXT: addl %ecx, %eax
+; AVX512POPCNT-NEXT: addl %esi, %eax
+; AVX512POPCNT-NEXT: addl %r8d, %eax
+; AVX512POPCNT-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: vector_ctpop_i512:
@@ -1254,11 +1359,33 @@ define i32 @vector_ctpop_i512(<16 x i32> %v0) nounwind {
;
; AVX512VLPOPCNT-LABEL: vector_ctpop_i512:
; AVX512VLPOPCNT: # %bb.0:
-; AVX512VLPOPCNT-NEXT: vpopcntq %zmm0, %zmm0
-; AVX512VLPOPCNT-NEXT: vpmovqb %zmm0, %xmm0
-; AVX512VLPOPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512VLPOPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
+; AVX512VLPOPCNT-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512VLPOPCNT-NEXT: vmovq %xmm1, %rax
+; AVX512VLPOPCNT-NEXT: vpextrq $1, %xmm1, %rcx
+; AVX512VLPOPCNT-NEXT: vpextrq $1, %xmm0, %rdx
+; AVX512VLPOPCNT-NEXT: vmovq %xmm0, %rsi
+; AVX512VLPOPCNT-NEXT: vextracti32x4 $2, %zmm0, %xmm1
+; AVX512VLPOPCNT-NEXT: vmovq %xmm1, %rdi
+; AVX512VLPOPCNT-NEXT: vpextrq $1, %xmm1, %r8
+; AVX512VLPOPCNT-NEXT: vextracti32x4 $3, %zmm0, %xmm0
+; AVX512VLPOPCNT-NEXT: vmovq %xmm0, %r9
+; AVX512VLPOPCNT-NEXT: vpextrq $1, %xmm0, %r10
+; AVX512VLPOPCNT-NEXT: popcntq %r10, %r10
+; AVX512VLPOPCNT-NEXT: popcntq %r9, %r9
+; AVX512VLPOPCNT-NEXT: addl %r10d, %r9d
+; AVX512VLPOPCNT-NEXT: popcntq %r8, %r8
+; AVX512VLPOPCNT-NEXT: popcntq %rdi, %rdi
+; AVX512VLPOPCNT-NEXT: addl %r8d, %edi
+; AVX512VLPOPCNT-NEXT: addl %r9d, %edi
+; AVX512VLPOPCNT-NEXT: popcntq %rdx, %rdx
+; AVX512VLPOPCNT-NEXT: popcntq %rsi, %rsi
+; AVX512VLPOPCNT-NEXT: addl %edx, %esi
+; AVX512VLPOPCNT-NEXT: popcntq %rcx, %rcx
+; AVX512VLPOPCNT-NEXT: popcntq %rax, %rax
+; AVX512VLPOPCNT-NEXT: addl %ecx, %eax
+; AVX512VLPOPCNT-NEXT: addl %esi, %eax
+; AVX512VLPOPCNT-NEXT: addl %edi, %eax
+; AVX512VLPOPCNT-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%a0 = bitcast <16 x i32> %v0 to i512
@@ -1412,27 +1539,42 @@ define i32 @test_ctpop_i1024(i1024 %a0) nounwind {
;
; AVX512POPCNT-LABEL: test_ctpop_i1024:
; AVX512POPCNT: # %bb.0:
+; AVX512POPCNT-NEXT: pushq %r14
+; AVX512POPCNT-NEXT: pushq %rbx
; AVX512POPCNT-NEXT: popcntq {{[0-9]+}}(%rsp), %rax
; AVX512POPCNT-NEXT: popcntq {{[0-9]+}}(%rsp), %r10
; AVX512POPCNT-NEXT: addl %eax, %r10d
+; AVX512POPCNT-NEXT: popcntq {{[0-9]+}}(%rsp), %rax
+; AVX512POPCNT-NEXT: popcntq {{[0-9]+}}(%rsp), %r11
+; AVX512POPCNT-NEXT: addl %eax, %r11d
+; AVX512POPCNT-NEXT: addl %r10d, %r11d
+; AVX512POPCNT-NEXT: popcntq {{[0-9]+}}(%rsp), %rax
+; AVX512POPCNT-NEXT: popcntq {{[0-9]+}}(%rsp), %rbx
+; AVX512POPCNT-NEXT: popcntq {{[0-9]+}}(%rsp), %r14
+; AVX512POPCNT-NEXT: addl %eax, %ebx
+; AVX512POPCNT-NEXT: popcntq {{[0-9]+}}(%rsp), %r10
+; AVX512POPCNT-NEXT: addl %r14d, %r10d
+; AVX512POPCNT-NEXT: addl %ebx, %r10d
+; AVX512POPCNT-NEXT: addl %r11d, %r10d
+; AVX512POPCNT-NEXT: popcntq {{[0-9]+}}(%rsp), %rax
+; AVX512POPCNT-NEXT: popcntq {{[0-9]+}}(%rsp), %r11
+; AVX512POPCNT-NEXT: addl %eax, %r11d
; AVX512POPCNT-NEXT: popcntq %r9, %rax
; AVX512POPCNT-NEXT: popcntq %r8, %r8
; AVX512POPCNT-NEXT: addl %eax, %r8d
-; AVX512POPCNT-NEXT: addl %r10d, %r8d
+; AVX512POPCNT-NEXT: addl %r11d, %r8d
; AVX512POPCNT-NEXT: popcntq %rcx, %rax
; AVX512POPCNT-NEXT: popcntq %rdx, %rcx
; AVX512POPCNT-NEXT: addl %eax, %ecx
-; AVX512POPCNT-NEXT: popcntq %rsi, %rax
-; AVX512POPCNT-NEXT: popcntq %rdi, %rdx
-; AVX512POPCNT-NEXT: addl %eax, %edx
-; AVX512POPCNT-NEXT: addl %ecx, %edx
-; AVX512POPCNT-NEXT: addl %r8d, %edx
-; AVX512POPCNT-NEXT: vpopcntq {{[0-9]+}}(%rsp), %zmm0
-; AVX512POPCNT-NEXT: vpmovqb %zmm0, %xmm0
-; AVX512POPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512POPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512POPCNT-NEXT: vmovd %xmm0, %eax
+; AVX512POPCNT-NEXT: popcntq %rsi, %rdx
+; AVX512POPCNT-NEXT: popcntq %rdi, %rax
; AVX512POPCNT-NEXT: addl %edx, %eax
+; AVX512POPCNT-NEXT: addl %ecx, %eax
+; AVX512POPCNT-NEXT: addl %r8d, %eax
+; AVX512POPCNT-NEXT: addl %r10d, %eax
+; AVX512POPCNT-NEXT: # kill: def $eax killed $eax killed $rax
+; AVX512POPCNT-NEXT: popq %rbx
+; AVX512POPCNT-NEXT: popq %r14
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: test_ctpop_i1024:
@@ -1489,33 +1631,54 @@ define i32 @test_ctpop_i1024(i1024 %a0) nounwind {
;
; AVX512VLPOPCNT-LABEL: test_ctpop_i1024:
; AVX512VLPOPCNT: # %bb.0:
+; AVX512VLPOPCNT-NEXT: pushq %r14
+; AVX512VLPOPCNT-NEXT: pushq %rbx
; AVX512VLPOPCNT-NEXT: popcntq {{[0-9]+}}(%rsp), %rax
; AVX512VLPOPCNT-NEXT: popcntq {{[0-9]+}}(%rsp), %r10
; AVX512VLPOPCNT-NEXT: addl %eax, %r10d
; AVX512VLPOPCNT-NEXT: xorl %eax, %eax
+; AVX512VLPOPCNT-NEXT: popcntq {{[0-9]+}}(%rsp), %rax
+; AVX512VLPOPCNT-NEXT: popcntq {{[0-9]+}}(%rsp), %r11
+; AVX512VLPOPCNT-NEXT: addl %eax, %r11d
+; AVX512VLPOPCNT-NEXT: addl %r10d, %r11d
+; AVX512VLPOPCNT-NEXT: xorl %eax, %eax
+; AVX512VLPOPCNT-NEXT: popcntq {{[0-9]+}}(%rsp), %rax
+; AVX512VLPOPCNT-NEXT: xorl %ebx, %ebx
+; AVX512VLPOPCNT-NEXT: popcntq {{[0-9]+}}(%rsp), %rbx
+; AVX512VLPOPCNT-NEXT: xorl %r14d, %r14d
+; AVX512VLPOPCNT-NEXT: popcntq {{[0-9]+}}(%rsp), %r14
+; AVX512VLPOPCNT-NEXT: addl %eax, %ebx
+; AVX512VLPOPCNT-NEXT: xorl %r10d, %r10d
+; AVX512VLPOPCNT-NEXT: popcntq {{[0-9]+}}(%rsp), %r10
+; AVX512VLPOPCNT-NEXT: addl %r14d, %r10d
+; AVX512VLPOPCNT-NEXT: addl %ebx, %r10d
+; AVX512VLPOPCNT-NEXT: addl %r11d, %r10d
+; AVX512VLPOPCNT-NEXT: xorl %eax, %eax
+; AVX512VLPOPCNT-NEXT: popcntq {{[0-9]+}}(%rsp), %rax
+; AVX512VLPOPCNT-NEXT: xorl %r11d, %r11d
+; AVX512VLPOPCNT-NEXT: popcntq {{[0-9]+}}(%rsp), %r11
+; AVX512VLPOPCNT-NEXT: addl %eax, %r11d
+; AVX512VLPOPCNT-NEXT: xorl %eax, %eax
; AVX512VLPOPCNT-NEXT: popcntq %r9, %rax
; AVX512VLPOPCNT-NEXT: popcntq %r8, %r8
; AVX512VLPOPCNT-NEXT: addl %eax, %r8d
-; AVX512VLPOPCNT-NEXT: addl %r10d, %r8d
+; AVX512VLPOPCNT-NEXT: addl %r11d, %r8d
; AVX512VLPOPCNT-NEXT: xorl %eax, %eax
; AVX512VLPOPCNT-NEXT: popcntq %rcx, %rax
; AVX512VLPOPCNT-NEXT: xorl %ecx, %ecx
; AVX512VLPOPCNT-NEXT: popcntq %rdx, %rcx
; AVX512VLPOPCNT-NEXT: addl %eax, %ecx
-; AVX512VLPOPCNT-NEXT: xorl %eax, %eax
-; AVX512VLPOPCNT-NEXT: popcntq %rsi, %rax
; AVX512VLPOPCNT-NEXT: xorl %edx, %edx
-; AVX512VLPOPCNT-NEXT: popcntq %rdi, %rdx
-; AVX512VLPOPCNT-NEXT: addl %eax, %edx
-; AVX512VLPOPCNT-NEXT: addl %ecx, %edx
-; AVX512VLPOPCNT-NEXT: addl %r8d, %edx
-; AVX512VLPOPCNT-NEXT: vpopcntq {{[0-9]+}}(%rsp), %zmm0
-; AVX512VLPOPCNT-NEXT: vpmovqb %zmm0, %xmm0
-; AVX512VLPOPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512VLPOPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
+; AVX512VLPOPCNT-NEXT: popcntq %rsi, %rdx
+; AVX512VLPOPCNT-NEXT: xorl %eax, %eax
+; AVX512VLPOPCNT-NEXT: popcntq %rdi, %rax
; AVX512VLPOPCNT-NEXT: addl %edx, %eax
-; AVX512VLPOPCNT-NEXT: vzeroupper
+; AVX512VLPOPCNT-NEXT: addl %ecx, %eax
+; AVX512VLPOPCNT-NEXT: addl %r8d, %eax
+; AVX512VLPOPCNT-NEXT: addl %r10d, %eax
+; AVX512VLPOPCNT-NEXT: # kill: def $eax killed $eax killed $rax
+; AVX512VLPOPCNT-NEXT: popq %rbx
+; AVX512VLPOPCNT-NEXT: popq %r14
; AVX512VLPOPCNT-NEXT: retq
%cnt = call i1024 @llvm.ctpop.i1024(i1024 %a0)
%res = trunc i1024 %cnt to i32
@@ -1655,16 +1818,38 @@ define i32 @load_ctpop_i1024(ptr %p0) nounwind {
;
; AVX512POPCNT-LABEL: load_ctpop_i1024:
; AVX512POPCNT: # %bb.0:
-; AVX512POPCNT-NEXT: vpopcntq 64(%rdi), %zmm0
-; AVX512POPCNT-NEXT: vpmovqb %zmm0, %xmm0
-; AVX512POPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512POPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512POPCNT-NEXT: vmovd %xmm0, %ecx
-; AVX512POPCNT-NEXT: vpopcntq (%rdi), %zmm0
-; AVX512POPCNT-NEXT: vpmovqb %zmm0, %xmm0
-; AVX512POPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512POPCNT-NEXT: vmovd %xmm0, %eax
+; AVX512POPCNT-NEXT: popcntq 120(%rdi), %rax
+; AVX512POPCNT-NEXT: popcntq 112(%rdi), %rcx
+; AVX512POPCNT-NEXT: addl %eax, %ecx
+; AVX512POPCNT-NEXT: popcntq 104(%rdi), %rax
+; AVX512POPCNT-NEXT: popcntq 96(%rdi), %rdx
+; AVX512POPCNT-NEXT: addl %eax, %edx
+; AVX512POPCNT-NEXT: addl %ecx, %edx
+; AVX512POPCNT-NEXT: popcntq 88(%rdi), %rax
+; AVX512POPCNT-NEXT: popcntq 80(%rdi), %rsi
+; AVX512POPCNT-NEXT: popcntq 72(%rdi), %r8
+; AVX512POPCNT-NEXT: popcntq 64(%rdi), %rcx
+; AVX512POPCNT-NEXT: addl %eax, %esi
+; AVX512POPCNT-NEXT: addl %r8d, %ecx
+; AVX512POPCNT-NEXT: addl %esi, %ecx
+; AVX512POPCNT-NEXT: addl %edx, %ecx
+; AVX512POPCNT-NEXT: popcntq 56(%rdi), %rax
+; AVX512POPCNT-NEXT: popcntq 48(%rdi), %rdx
+; AVX512POPCNT-NEXT: popcntq 40(%rdi), %rsi
+; AVX512POPCNT-NEXT: popcntq 32(%rdi), %r8
+; AVX512POPCNT-NEXT: addl %eax, %edx
+; AVX512POPCNT-NEXT: addl %esi, %r8d
+; AVX512POPCNT-NEXT: popcntq 24(%rdi), %rax
+; AVX512POPCNT-NEXT: addl %edx, %r8d
+; AVX512POPCNT-NEXT: popcntq 16(%rdi), %rdx
+; AVX512POPCNT-NEXT: addl %eax, %edx
+; AVX512POPCNT-NEXT: popcntq 8(%rdi), %rsi
+; AVX512POPCNT-NEXT: popcntq (%rdi), %rax
+; AVX512POPCNT-NEXT: addl %esi, %eax
+; AVX512POPCNT-NEXT: addl %edx, %eax
+; AVX512POPCNT-NEXT: addl %r8d, %eax
; AVX512POPCNT-NEXT: addl %ecx, %eax
+; AVX512POPCNT-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: load_ctpop_i1024:
@@ -1716,17 +1901,49 @@ define i32 @load_ctpop_i1024(ptr %p0) nounwind {
;
; AVX512VLPOPCNT-LABEL: load_ctpop_i1024:
; AVX512VLPOPCNT: # %bb.0:
-; AVX512VLPOPCNT-NEXT: vpopcntq 64(%rdi), %zmm0
-; AVX512VLPOPCNT-NEXT: vpmovqb %zmm0, %xmm0
-; AVX512VLPOPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512VLPOPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512VLPOPCNT-NEXT: vpopcntq (%rdi), %zmm2
-; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %ecx
-; AVX512VLPOPCNT-NEXT: vpmovqb %zmm2, %xmm0
-; AVX512VLPOPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
+; AVX512VLPOPCNT-NEXT: popcntq 120(%rdi), %rax
+; AVX512VLPOPCNT-NEXT: popcntq 112(%rdi), %rcx
+; AVX512VLPOPCNT-NEXT: addl %eax, %ecx
+; AVX512VLPOPCNT-NEXT: xorl %eax, %eax
+; AVX512VLPOPCNT-NEXT: popcntq 104(%rdi), %rax
+; AVX512VLPOPCNT-NEXT: popcntq 96(%rdi), %rdx
+; AVX512VLPOPCNT-NEXT: addl %eax, %edx
+; AVX512VLPOPCNT-NEXT: addl %ecx, %edx
+; AVX512VLPOPCNT-NEXT: xorl %eax, %eax
+; AVX512VLPOPCNT-NEXT: popcntq 88(%rdi), %rax
+; AVX512VLPOPCNT-NEXT: popcntq 80(%rdi), %rsi
+; AVX512VLPOPCNT-NEXT: popcntq 72(%rdi), %r8
+; AVX512VLPOPCNT-NEXT: addl %eax, %esi
+; AVX512VLPOPCNT-NEXT: xorl %ecx, %ecx
+; AVX512VLPOPCNT-NEXT: popcntq 64(%rdi), %rcx
+; AVX512VLPOPCNT-NEXT: addl %r8d, %ecx
+; AVX512VLPOPCNT-NEXT: addl %esi, %ecx
+; AVX512VLPOPCNT-NEXT: addl %edx, %ecx
+; AVX512VLPOPCNT-NEXT: xorl %eax, %eax
+; AVX512VLPOPCNT-NEXT: popcntq 56(%rdi), %rax
+; AVX512VLPOPCNT-NEXT: xorl %edx, %edx
+; AVX512VLPOPCNT-NEXT: popcntq 48(%rdi), %rdx
+; AVX512VLPOPCNT-NEXT: xorl %esi, %esi
+; AVX512VLPOPCNT-NEXT: popcntq 40(%rdi), %rsi
+; AVX512VLPOPCNT-NEXT: addl %eax, %edx
+; AVX512VLPOPCNT-NEXT: xorl %r8d, %r8d
+; AVX512VLPOPCNT-NEXT: popcntq 32(%rdi), %r8
+; AVX512VLPOPCNT-NEXT: addl %esi, %r8d
+; AVX512VLPOPCNT-NEXT: xorl %eax, %eax
+; AVX512VLPOPCNT-NEXT: popcntq 24(%rdi), %rax
+; AVX512VLPOPCNT-NEXT: addl %edx, %r8d
+; AVX512VLPOPCNT-NEXT: xorl %edx, %edx
+; AVX512VLPOPCNT-NEXT: popcntq 16(%rdi), %rdx
+; AVX512VLPOPCNT-NEXT: addl %eax, %edx
+; AVX512VLPOPCNT-NEXT: xorl %esi, %esi
+; AVX512VLPOPCNT-NEXT: popcntq 8(%rdi), %rsi
+; AVX512VLPOPCNT-NEXT: xorl %eax, %eax
+; AVX512VLPOPCNT-NEXT: popcntq (%rdi), %rax
+; AVX512VLPOPCNT-NEXT: addl %esi, %eax
+; AVX512VLPOPCNT-NEXT: addl %edx, %eax
+; AVX512VLPOPCNT-NEXT: addl %r8d, %eax
; AVX512VLPOPCNT-NEXT: addl %ecx, %eax
-; AVX512VLPOPCNT-NEXT: vzeroupper
+; AVX512VLPOPCNT-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512VLPOPCNT-NEXT: retq
%a0 = load i1024, ptr %p0
%cnt = call i1024 @llvm.ctpop.i1024(i1024 %a0)
@@ -2384,100 +2601,117 @@ define i32 @test_ctlz_i512(i512 %a0) nounwind {
define i32 @load_ctlz_i512(ptr %p0) nounwind {
; SSE-LABEL: load_ctlz_i512:
; SSE: # %bb.0:
-; SSE-NEXT: movdqa 32(%rdi), %xmm0
-; SSE-NEXT: movq 16(%rdi), %rcx
-; SSE-NEXT: movq 24(%rdi), %rdx
-; SSE-NEXT: movq 40(%rdi), %r8
-; SSE-NEXT: movq 48(%rdi), %rax
-; SSE-NEXT: movq 56(%rdi), %r9
-; SSE-NEXT: bsrq %r9, %rsi
-; SSE-NEXT: xorl $63, %esi
-; SSE-NEXT: bsrq %rax, %r10
-; SSE-NEXT: xorl $63, %r10d
-; SSE-NEXT: orl $64, %r10d
-; SSE-NEXT: testq %r9, %r9
-; SSE-NEXT: cmovnel %esi, %r10d
-; SSE-NEXT: bsrq %r8, %r11
-; SSE-NEXT: xorl $63, %r11d
-; SSE-NEXT: bsrq 32(%rdi), %rsi
-; SSE-NEXT: xorl $63, %esi
-; SSE-NEXT: orl $64, %esi
+; SSE-NEXT: pushq %r15
+; SSE-NEXT: pushq %r14
+; SSE-NEXT: pushq %rbx
+; SSE-NEXT: movq 8(%rdi), %r10
+; SSE-NEXT: movq 16(%rdi), %r9
+; SSE-NEXT: movq 32(%rdi), %rcx
+; SSE-NEXT: movq 40(%rdi), %rdx
+; SSE-NEXT: movq 48(%rdi), %rsi
+; SSE-NEXT: movq 56(%rdi), %r8
+; SSE-NEXT: bsrq %r8, %rax
+; SSE-NEXT: xorl $63, %eax
+; SSE-NEXT: bsrq %rsi, %r14
+; SSE-NEXT: xorl $63, %r14d
+; SSE-NEXT: orl $64, %r14d
; SSE-NEXT: testq %r8, %r8
-; SSE-NEXT: cmovnel %r11d, %esi
-; SSE-NEXT: subl $-128, %esi
-; SSE-NEXT: orq %r9, %rax
-; SSE-NEXT: cmovnel %r10d, %esi
+; SSE-NEXT: cmovnel %eax, %r14d
; SSE-NEXT: bsrq %rdx, %rax
; SSE-NEXT: xorl $63, %eax
-; SSE-NEXT: bsrq %rcx, %r8
-; SSE-NEXT: xorl $63, %r8d
-; SSE-NEXT: orl $64, %r8d
+; SSE-NEXT: bsrq %rcx, %r11
+; SSE-NEXT: xorl $63, %r11d
+; SSE-NEXT: orl $64, %r11d
; SSE-NEXT: testq %rdx, %rdx
-; SSE-NEXT: cmovnel %eax, %r8d
-; SSE-NEXT: movq 8(%rdi), %r9
-; SSE-NEXT: bsrq %r9, %r10
-; SSE-NEXT: xorl $63, %r10d
+; SSE-NEXT: cmovnel %eax, %r11d
+; SSE-NEXT: movq 24(%rdi), %rbx
+; SSE-NEXT: subl $-128, %r11d
+; SSE-NEXT: movq %rsi, %rax
+; SSE-NEXT: orq %r8, %rax
+; SSE-NEXT: cmovnel %r14d, %r11d
+; SSE-NEXT: bsrq %rbx, %rax
+; SSE-NEXT: xorl $63, %eax
+; SSE-NEXT: bsrq %r9, %r14
+; SSE-NEXT: xorl $63, %r14d
+; SSE-NEXT: orl $64, %r14d
+; SSE-NEXT: testq %rbx, %rbx
+; SSE-NEXT: cmovnel %eax, %r14d
+; SSE-NEXT: bsrq %r10, %r15
+; SSE-NEXT: xorl $63, %r15d
; SSE-NEXT: movl $127, %eax
; SSE-NEXT: bsrq (%rdi), %rax
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: addl $64, %eax
-; SSE-NEXT: testq %r9, %r9
-; SSE-NEXT: cmovnel %r10d, %eax
+; SSE-NEXT: testq %r10, %r10
+; SSE-NEXT: cmovnel %r15d, %eax
; SSE-NEXT: subl $-128, %eax
-; SSE-NEXT: orq %rdx, %rcx
-; SSE-NEXT: cmovnel %r8d, %eax
+; SSE-NEXT: orq %rbx, %r9
+; SSE-NEXT: cmovnel %r14d, %eax
; SSE-NEXT: addl $256, %eax # imm = 0x100
-; SSE-NEXT: por 48(%rdi), %xmm0
-; SSE-NEXT: ptest %xmm0, %xmm0
-; SSE-NEXT: cmovnel %esi, %eax
+; SSE-NEXT: orq %r8, %rdx
+; SSE-NEXT: orq %rsi, %rcx
+; SSE-NEXT: orq %rdx, %rcx
+; SSE-NEXT: cmovnel %r11d, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
+; SSE-NEXT: popq %rbx
+; SSE-NEXT: popq %r14
+; SSE-NEXT: popq %r15
; SSE-NEXT: retq
;
; AVX2-LABEL: load_ctlz_i512:
; AVX2: # %bb.0:
-; AVX2-NEXT: movq 16(%rdi), %rcx
-; AVX2-NEXT: movq 24(%rdi), %rdx
-; AVX2-NEXT: movq 40(%rdi), %rax
-; AVX2-NEXT: movq 48(%rdi), %r8
-; AVX2-NEXT: movq 56(%rdi), %r9
-; AVX2-NEXT: lzcntq %r9, %rsi
-; AVX2-NEXT: lzcntq %r8, %r10
-; AVX2-NEXT: addl $64, %r10d
-; AVX2-NEXT: testq %r9, %r9
-; AVX2-NEXT: cmovnel %esi, %r10d
-; AVX2-NEXT: lzcntq %rax, %r11
-; AVX2-NEXT: xorl %esi, %esi
-; AVX2-NEXT: lzcntq 32(%rdi), %rsi
-; AVX2-NEXT: addl $64, %esi
-; AVX2-NEXT: testq %rax, %rax
-; AVX2-NEXT: cmovnel %r11d, %esi
-; AVX2-NEXT: subl $-128, %esi
-; AVX2-NEXT: orq %r9, %r8
-; AVX2-NEXT: cmovnel %r10d, %esi
-; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: lzcntq %rdx, %rax
-; AVX2-NEXT: xorl %r8d, %r8d
-; AVX2-NEXT: lzcntq %rcx, %r8
-; AVX2-NEXT: addl $64, %r8d
+; AVX2-NEXT: pushq %r15
+; AVX2-NEXT: pushq %r14
+; AVX2-NEXT: pushq %rbx
+; AVX2-NEXT: movq 8(%rdi), %r10
+; AVX2-NEXT: movq 16(%rdi), %r9
+; AVX2-NEXT: movq 32(%rdi), %rcx
+; AVX2-NEXT: movq 40(%rdi), %rdx
+; AVX2-NEXT: movq 48(%rdi), %rsi
+; AVX2-NEXT: movq 56(%rdi), %r8
+; AVX2-NEXT: lzcntq %r8, %rax
+; AVX2-NEXT: xorl %ebx, %ebx
+; AVX2-NEXT: lzcntq %rsi, %rbx
+; AVX2-NEXT: addl $64, %ebx
+; AVX2-NEXT: testq %r8, %r8
+; AVX2-NEXT: cmovnel %eax, %ebx
+; AVX2-NEXT: xorl %eax, %eax
+; AVX2-NEXT: lzcntq %rdx, %rax
+; AVX2-NEXT: lzcntq %rcx, %r11
+; AVX2-NEXT: addl $64, %r11d
; AVX2-NEXT: testq %rdx, %rdx
-; AVX2-NEXT: cmovnel %eax, %r8d
-; AVX2-NEXT: movq 8(%rdi), %r9
-; AVX2-NEXT: xorl %r10d, %r10d
-; AVX2-NEXT: lzcntq %r9, %r10
+; AVX2-NEXT: cmovnel %eax, %r11d
+; AVX2-NEXT: subl $-128, %r11d
+; AVX2-NEXT: movq %rsi, %rax
+; AVX2-NEXT: orq %r8, %rax
+; AVX2-NEXT: cmovnel %ebx, %r11d
+; AVX2-NEXT: movq 24(%rdi), %rbx
+; AVX2-NEXT: xorl %eax, %eax
+; AVX2-NEXT: lzcntq %rbx, %rax
+; AVX2-NEXT: xorl %r14d, %r14d
+; AVX2-NEXT: lzcntq %r9, %r14
+; AVX2-NEXT: addl $64, %r14d
+; AVX2-NEXT: testq %rbx, %rbx
+; AVX2-NEXT: cmovnel %eax, %r14d
+; AVX2-NEXT: xorl %r15d, %r15d
+; AVX2-NEXT: lzcntq %r10, %r15
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq (%rdi), %rax
; AVX2-NEXT: addl $64, %eax
-; AVX2-NEXT: testq %r9, %r9
-; AVX2-NEXT: cmovnel %r10d, %eax
+; AVX2-NEXT: testq %r10, %r10
+; AVX2-NEXT: cmovnel %r15d, %eax
; AVX2-NEXT: subl $-128, %eax
-; AVX2-NEXT: orq %rdx, %rcx
-; AVX2-NEXT: cmovnel %r8d, %eax
-; AVX2-NEXT: vmovdqa 32(%rdi), %xmm0
+; AVX2-NEXT: orq %rbx, %r9
+; AVX2-NEXT: cmovnel %r14d, %eax
; AVX2-NEXT: addl $256, %eax # imm = 0x100
-; AVX2-NEXT: vpor 48(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vptest %xmm0, %xmm0
-; AVX2-NEXT: cmovnel %esi, %eax
+; AVX2-NEXT: orq %r8, %rdx
+; AVX2-NEXT: orq %rsi, %rcx
+; AVX2-NEXT: orq %rdx, %rcx
+; AVX2-NEXT: cmovnel %r11d, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
+; AVX2-NEXT: popq %rbx
+; AVX2-NEXT: popq %r14
+; AVX2-NEXT: popq %r15
; AVX2-NEXT: retq
;
; AVX512F-LABEL: load_ctlz_i512:
@@ -2695,359 +2929,445 @@ define i32 @vector_ctlz_i512(<16 x i32> %v0) nounwind {
define i32 @test_ctlz_i1024(i1024 %a0) nounwind {
; SSE-LABEL: test_ctlz_i1024:
; SSE: # %bb.0:
+; SSE-NEXT: pushq %rbp
+; SSE-NEXT: pushq %r15
; SSE-NEXT: pushq %r14
+; SSE-NEXT: pushq %r13
+; SSE-NEXT: pushq %r12
; SSE-NEXT: pushq %rbx
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r11
-; SSE-NEXT: bsrq %r11, %r10
-; SSE-NEXT: xorl $63, %r10d
-; SSE-NEXT: bsrq %rax, %r14
-; SSE-NEXT: xorl $63, %r14d
-; SSE-NEXT: orl $64, %r14d
-; SSE-NEXT: testq %r11, %r11
-; SSE-NEXT: cmovnel %r10d, %r14d
-; SSE-NEXT: bsrq %r9, %r10
-; SSE-NEXT: xorl $63, %r10d
-; SSE-NEXT: bsrq %r8, %rbx
-; SSE-NEXT: xorl $63, %ebx
-; SSE-NEXT: orl $64, %ebx
-; SSE-NEXT: testq %r9, %r9
-; SSE-NEXT: cmovnel %r10d, %ebx
-; SSE-NEXT: subl $-128, %ebx
-; SSE-NEXT: movq %rax, %r10
-; SSE-NEXT: orq %r11, %r10
-; SSE-NEXT: cmovnel %r14d, %ebx
-; SSE-NEXT: bsrq %rcx, %r10
-; SSE-NEXT: xorl $63, %r10d
-; SSE-NEXT: bsrq %rdx, %r14
-; SSE-NEXT: xorl $63, %r14d
-; SSE-NEXT: orl $64, %r14d
-; SSE-NEXT: testq %rcx, %rcx
-; SSE-NEXT: cmovnel %r10d, %r14d
-; SSE-NEXT: movl $127, %r10d
-; SSE-NEXT: bsrq %rdi, %r10
-; SSE-NEXT: bsrq %rsi, %rdi
-; SSE-NEXT: xorl $63, %edi
-; SSE-NEXT: xorl $63, %r10d
-; SSE-NEXT: addl $64, %r10d
-; SSE-NEXT: testq %rsi, %rsi
-; SSE-NEXT: cmovnel %edi, %r10d
-; SSE-NEXT: subl $-128, %r10d
-; SSE-NEXT: orq %rcx, %rdx
-; SSE-NEXT: cmovnel %r14d, %r10d
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rcx
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rdi
-; SSE-NEXT: orq %r11, %r9
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rsi
-; SSE-NEXT: orq %rax, %r8
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; SSE-NEXT: addl $256, %r10d # imm = 0x100
-; SSE-NEXT: orq %r9, %r8
-; SSE-NEXT: cmovnel %ebx, %r10d
-; SSE-NEXT: addl $512, %r10d # imm = 0x200
-; SSE-NEXT: bsrq %rax, %rdx
-; SSE-NEXT: xorl $63, %edx
-; SSE-NEXT: bsrq %rsi, %r9
-; SSE-NEXT: xorl $63, %r9d
-; SSE-NEXT: orl $64, %r9d
-; SSE-NEXT: testq %rax, %rax
-; SSE-NEXT: cmovnel %edx, %r9d
-; SSE-NEXT: bsrq %rdi, %r8
-; SSE-NEXT: xorl $63, %r8d
-; SSE-NEXT: bsrq {{[0-9]+}}(%rsp), %rdx
-; SSE-NEXT: xorl $63, %edx
-; SSE-NEXT: orl $64, %edx
-; SSE-NEXT: testq %rdi, %rdi
-; SSE-NEXT: cmovnel %r8d, %edx
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rdi
+; SSE-NEXT: movq %r9, %r11
+; SSE-NEXT: movq %r8, %r9
+; SSE-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE-NEXT: movq %rdx, %r12
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r14
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r15
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r8
-; SSE-NEXT: subl $-128, %edx
-; SSE-NEXT: orq %rax, %rsi
-; SSE-NEXT: cmovnel %r9d, %edx
; SSE-NEXT: bsrq %r8, %rax
; SSE-NEXT: xorl $63, %eax
-; SSE-NEXT: bsrq %rdi, %rsi
-; SSE-NEXT: xorl $63, %esi
-; SSE-NEXT: orl $64, %esi
+; SSE-NEXT: bsrq %r15, %rcx
+; SSE-NEXT: xorl $63, %ecx
+; SSE-NEXT: orl $64, %ecx
; SSE-NEXT: testq %r8, %r8
-; SSE-NEXT: cmovnel %eax, %esi
-; SSE-NEXT: bsrq %rcx, %r9
-; SSE-NEXT: xorl $63, %r9d
+; SSE-NEXT: cmovnel %eax, %ecx
+; SSE-NEXT: bsrq %r14, %rdx
+; SSE-NEXT: xorl $63, %edx
; SSE-NEXT: bsrq {{[0-9]+}}(%rsp), %rax
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: orl $64, %eax
-; SSE-NEXT: testq %rcx, %rcx
-; SSE-NEXT: cmovnel %r9d, %eax
+; SSE-NEXT: testq %r14, %r14
+; SSE-NEXT: cmovnel %edx, %eax
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r13
; SSE-NEXT: subl $-128, %eax
-; SSE-NEXT: orq %r8, %rdi
-; SSE-NEXT: cmovnel %esi, %eax
-; SSE-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm0
-; SSE-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm1
+; SSE-NEXT: movq %r15, %rdx
+; SSE-NEXT: orq %r8, %rdx
+; SSE-NEXT: movq %r8, %r14
+; SSE-NEXT: cmovnel %ecx, %eax
+; SSE-NEXT: bsrq %r13, %rcx
+; SSE-NEXT: xorl $63, %ecx
+; SSE-NEXT: bsrq %rbx, %rdx
+; SSE-NEXT: xorl $63, %edx
+; SSE-NEXT: orl $64, %edx
+; SSE-NEXT: testq %r13, %r13
+; SSE-NEXT: cmovnel %ecx, %edx
+; SSE-NEXT: bsrq %r10, %rcx
+; SSE-NEXT: xorl $63, %ecx
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r8
+; SSE-NEXT: bsrq %r8, %rbp
+; SSE-NEXT: xorl $63, %ebp
+; SSE-NEXT: orl $64, %ebp
+; SSE-NEXT: testq %r10, %r10
+; SSE-NEXT: cmovnel %ecx, %ebp
+; SSE-NEXT: subl $-128, %ebp
+; SSE-NEXT: movq %rbx, %rcx
+; SSE-NEXT: orq %r13, %rcx
+; SSE-NEXT: cmovnel %edx, %ebp
+; SSE-NEXT: addl $256, %ebp # imm = 0x100
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rcx
+; SSE-NEXT: orq %r14, %rcx
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; SSE-NEXT: orq %r15, %rdx
+; SSE-NEXT: orq %rcx, %rdx
+; SSE-NEXT: cmovnel %eax, %ebp
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r14
+; SSE-NEXT: bsrq %r14, %rax
+; SSE-NEXT: xorl $63, %eax
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r15
+; SSE-NEXT: bsrq %r15, %rcx
+; SSE-NEXT: xorl $63, %ecx
+; SSE-NEXT: orl $64, %ecx
+; SSE-NEXT: testq %r14, %r14
+; SSE-NEXT: cmovnel %eax, %ecx
+; SSE-NEXT: bsrq %r11, %rax
+; SSE-NEXT: xorl $63, %eax
+; SSE-NEXT: bsrq %r9, %rdx
+; SSE-NEXT: xorl $63, %edx
+; SSE-NEXT: orl $64, %edx
+; SSE-NEXT: testq %r11, %r11
+; SSE-NEXT: cmovnel %eax, %edx
+; SSE-NEXT: subl $-128, %edx
+; SSE-NEXT: movq %r15, %rax
+; SSE-NEXT: orq %r14, %rax
+; SSE-NEXT: cmovnel %ecx, %edx
+; SSE-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r15 # 8-byte Reload
+; SSE-NEXT: bsrq %r15, %rax
+; SSE-NEXT: xorl $63, %eax
+; SSE-NEXT: bsrq %r12, %rcx
+; SSE-NEXT: xorl $63, %ecx
+; SSE-NEXT: orl $64, %ecx
+; SSE-NEXT: testq %r15, %r15
+; SSE-NEXT: cmovnel %eax, %ecx
+; SSE-NEXT: movl $127, %eax
+; SSE-NEXT: bsrq %rdi, %rax
+; SSE-NEXT: bsrq %rsi, %rdi
+; SSE-NEXT: xorl $63, %edi
+; SSE-NEXT: xorl $63, %eax
+; SSE-NEXT: addl $64, %eax
+; SSE-NEXT: testq %rsi, %rsi
+; SSE-NEXT: cmovnel %edi, %eax
+; SSE-NEXT: subl $-128, %eax
+; SSE-NEXT: orq %r15, %r12
+; SSE-NEXT: cmovnel %ecx, %eax
+; SSE-NEXT: orq %r14, %r11
+; SSE-NEXT: orq {{[0-9]+}}(%rsp), %r9
; SSE-NEXT: addl $256, %eax # imm = 0x100
-; SSE-NEXT: movdqa %xmm1, %xmm2
-; SSE-NEXT: por %xmm0, %xmm2
-; SSE-NEXT: ptest %xmm2, %xmm2
+; SSE-NEXT: orq %r11, %r9
; SSE-NEXT: cmovnel %edx, %eax
-; SSE-NEXT: por {{[0-9]+}}(%rsp), %xmm0
-; SSE-NEXT: por {{[0-9]+}}(%rsp), %xmm1
-; SSE-NEXT: por %xmm0, %xmm1
-; SSE-NEXT: ptest %xmm1, %xmm1
-; SSE-NEXT: cmovel %r10d, %eax
+; SSE-NEXT: orq {{[0-9]+}}(%rsp), %r13
+; SSE-NEXT: orq {{[0-9]+}}(%rsp), %r10
+; SSE-NEXT: orq %r13, %r10
+; SSE-NEXT: orq {{[0-9]+}}(%rsp), %rbx
+; SSE-NEXT: orq {{[0-9]+}}(%rsp), %r8
+; SSE-NEXT: orq %rbx, %r8
+; SSE-NEXT: addl $512, %eax # imm = 0x200
+; SSE-NEXT: orq %r10, %r8
+; SSE-NEXT: cmovnel %ebp, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: popq %rbx
+; SSE-NEXT: popq %r12
+; SSE-NEXT: popq %r13
; SSE-NEXT: popq %r14
+; SSE-NEXT: popq %r15
+; SSE-NEXT: popq %rbp
; SSE-NEXT: retq
;
; AVX2-LABEL: test_ctlz_i1024:
; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %rbp
; AVX2-NEXT: pushq %r15
; AVX2-NEXT: pushq %r14
; AVX2-NEXT: pushq %r13
; AVX2-NEXT: pushq %r12
; AVX2-NEXT: pushq %rbx
+; AVX2-NEXT: movq %r9, %r14
+; AVX2-NEXT: movq %r8, %r11
+; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r15
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rbx
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r15
-; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: lzcntq %r15, %rbx
-; AVX2-NEXT: xorl %r12d, %r12d
-; AVX2-NEXT: lzcntq %r11, %r12
-; AVX2-NEXT: addl $64, %r12d
-; AVX2-NEXT: testq %r15, %r15
-; AVX2-NEXT: cmovnel %ebx, %r12d
-; AVX2-NEXT: xorl %r14d, %r14d
-; AVX2-NEXT: lzcntq {{[0-9]+}}(%rsp), %r14
-; AVX2-NEXT: xorl %r13d, %r13d
-; AVX2-NEXT: lzcntq %r10, %r13
-; AVX2-NEXT: addl $64, %r14d
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r8
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r12
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: lzcntq %r12, %rcx
+; AVX2-NEXT: xorl %r9d, %r9d
+; AVX2-NEXT: lzcntq %r8, %r9
+; AVX2-NEXT: addl $64, %r9d
+; AVX2-NEXT: testq %r12, %r12
+; AVX2-NEXT: cmovnel %ecx, %r9d
+; AVX2-NEXT: xorl %esi, %esi
+; AVX2-NEXT: lzcntq %r10, %rsi
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: lzcntq %rax, %rcx
+; AVX2-NEXT: addl $64, %ecx
; AVX2-NEXT: testq %r10, %r10
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rbx
-; AVX2-NEXT: cmovnel %r13d, %r14d
-; AVX2-NEXT: subl $-128, %r14d
-; AVX2-NEXT: orq %r15, %r11
-; AVX2-NEXT: cmovnel %r12d, %r14d
-; AVX2-NEXT: xorl %r10d, %r10d
-; AVX2-NEXT: lzcntq %rbx, %r10
-; AVX2-NEXT: xorl %r15d, %r15d
-; AVX2-NEXT: lzcntq %rax, %r15
-; AVX2-NEXT: addl $64, %r15d
-; AVX2-NEXT: testq %rbx, %rbx
-; AVX2-NEXT: cmovnel %r10d, %r15d
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
-; AVX2-NEXT: xorl %r12d, %r12d
-; AVX2-NEXT: lzcntq %r11, %r12
-; AVX2-NEXT: xorl %r10d, %r10d
-; AVX2-NEXT: lzcntq {{[0-9]+}}(%rsp), %r10
-; AVX2-NEXT: addl $64, %r10d
-; AVX2-NEXT: testq %r11, %r11
-; AVX2-NEXT: cmovnel %r12d, %r10d
-; AVX2-NEXT: vmovdqu {{[0-9]+}}(%rsp), %ymm0
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
-; AVX2-NEXT: subl $-128, %r10d
-; AVX2-NEXT: orq %rbx, %rax
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rbx
-; AVX2-NEXT: cmovnel %r15d, %r10d
-; AVX2-NEXT: addl $256, %r10d # imm = 0x100
-; AVX2-NEXT: vpor {{[0-9]+}}(%rsp), %xmm0, %xmm1
-; AVX2-NEXT: vptest %xmm1, %xmm1
-; AVX2-NEXT: cmovnel %r14d, %r10d
-; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: lzcntq %rbx, %rax
-; AVX2-NEXT: xorl %r15d, %r15d
-; AVX2-NEXT: lzcntq %r11, %r15
-; AVX2-NEXT: addl $64, %r15d
+; AVX2-NEXT: cmovnel %esi, %ecx
+; AVX2-NEXT: subl $-128, %ecx
+; AVX2-NEXT: movq %r8, %rsi
+; AVX2-NEXT: orq %r12, %rsi
+; AVX2-NEXT: cmovnel %r9d, %ecx
+; AVX2-NEXT: xorl %edi, %edi
+; AVX2-NEXT: lzcntq %rbx, %rdi
+; AVX2-NEXT: xorl %esi, %esi
+; AVX2-NEXT: lzcntq %r15, %rsi
+; AVX2-NEXT: addl $64, %esi
; AVX2-NEXT: testq %rbx, %rbx
-; AVX2-NEXT: cmovnel %eax, %r15d
-; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: lzcntq %r9, %rax
-; AVX2-NEXT: xorl %r14d, %r14d
-; AVX2-NEXT: lzcntq %r8, %r14
-; AVX2-NEXT: addl $64, %r14d
+; AVX2-NEXT: cmovnel %edi, %esi
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r13
+; AVX2-NEXT: xorl %ebp, %ebp
+; AVX2-NEXT: lzcntq %r13, %rbp
+; AVX2-NEXT: addl $64, %ebp
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r9
+; AVX2-NEXT: xorl %edi, %edi
+; AVX2-NEXT: lzcntq %r9, %rdi
; AVX2-NEXT: testq %r9, %r9
-; AVX2-NEXT: cmovnel %eax, %r14d
-; AVX2-NEXT: subl $-128, %r14d
-; AVX2-NEXT: movq %r11, %rax
-; AVX2-NEXT: orq %rbx, %rax
-; AVX2-NEXT: cmovnel %r15d, %r14d
-; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: lzcntq %rcx, %rax
-; AVX2-NEXT: xorl %r15d, %r15d
-; AVX2-NEXT: lzcntq %rdx, %r15
-; AVX2-NEXT: addl $64, %r15d
-; AVX2-NEXT: testq %rcx, %rcx
-; AVX2-NEXT: cmovnel %eax, %r15d
-; AVX2-NEXT: xorl %r12d, %r12d
-; AVX2-NEXT: lzcntq %rsi, %r12
+; AVX2-NEXT: cmovnel %edi, %ebp
+; AVX2-NEXT: subl $-128, %ebp
+; AVX2-NEXT: movq %r15, %rdi
+; AVX2-NEXT: orq %rbx, %rdi
+; AVX2-NEXT: cmovnel %esi, %ebp
+; AVX2-NEXT: addl $256, %ebp # imm = 0x100
+; AVX2-NEXT: movq %r10, %rdi
+; AVX2-NEXT: orq %r12, %rdi
+; AVX2-NEXT: movq %rax, %rsi
+; AVX2-NEXT: orq %r8, %rsi
+; AVX2-NEXT: orq %rdi, %rsi
+; AVX2-NEXT: cmovnel %ecx, %ebp
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdi
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq %rdi, %rax
; AVX2-NEXT: addl $64, %eax
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r12
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: lzcntq %r12, %rcx
+; AVX2-NEXT: testq %r12, %r12
+; AVX2-NEXT: cmovnel %ecx, %eax
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: lzcntq %r11, %rcx
+; AVX2-NEXT: addl $64, %ecx
+; AVX2-NEXT: xorl %esi, %esi
+; AVX2-NEXT: lzcntq %r14, %rsi
+; AVX2-NEXT: testq %r14, %r14
+; AVX2-NEXT: cmovnel %esi, %ecx
+; AVX2-NEXT: subl $-128, %ecx
+; AVX2-NEXT: movq %rdi, %rsi
+; AVX2-NEXT: orq %r12, %rsi
+; AVX2-NEXT: cmovnel %eax, %ecx
+; AVX2-NEXT: movq %rdx, %rdi
+; AVX2-NEXT: lzcntq %rdx, %rdx
+; AVX2-NEXT: addl $64, %edx
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Reload
+; AVX2-NEXT: xorl %eax, %eax
+; AVX2-NEXT: lzcntq %r10, %rax
+; AVX2-NEXT: testq %r10, %r10
+; AVX2-NEXT: cmovnel %eax, %edx
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; AVX2-NEXT: lzcntq %rax, %rax
+; AVX2-NEXT: addl $64, %eax
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
+; AVX2-NEXT: lzcntq %rsi, %r8
; AVX2-NEXT: testq %rsi, %rsi
-; AVX2-NEXT: cmovnel %r12d, %eax
+; AVX2-NEXT: cmovnel %r8d, %eax
; AVX2-NEXT: subl $-128, %eax
-; AVX2-NEXT: orq %rcx, %rdx
-; AVX2-NEXT: cmovnel %r15d, %eax
-; AVX2-NEXT: orq %rbx, %r9
-; AVX2-NEXT: orq %r11, %r8
+; AVX2-NEXT: orq %r10, %rdi
+; AVX2-NEXT: cmovnel %edx, %eax
+; AVX2-NEXT: orq %r12, %r14
+; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r11
; AVX2-NEXT: addl $256, %eax # imm = 0x100
-; AVX2-NEXT: orq %r9, %r8
-; AVX2-NEXT: cmovnel %r14d, %eax
+; AVX2-NEXT: orq %r14, %r11
+; AVX2-NEXT: cmovnel %ecx, %eax
+; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %rbx
+; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r9
+; AVX2-NEXT: orq %rbx, %r9
+; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r15
+; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r13
+; AVX2-NEXT: orq %r15, %r13
; AVX2-NEXT: addl $512, %eax # imm = 0x200
-; AVX2-NEXT: vpor {{[0-9]+}}(%rsp), %ymm0, %ymm0
-; AVX2-NEXT: vptest %ymm0, %ymm0
-; AVX2-NEXT: cmovnel %r10d, %eax
+; AVX2-NEXT: orq %r9, %r13
+; AVX2-NEXT: cmovnel %ebp, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: popq %rbx
; AVX2-NEXT: popq %r12
; AVX2-NEXT: popq %r13
; AVX2-NEXT: popq %r14
; AVX2-NEXT: popq %r15
-; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: popq %rbp
; AVX2-NEXT: retq
;
; AVX512F-LABEL: test_ctlz_i1024:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
-; AVX512F-NEXT: vmovq %rdi, %xmm1
-; AVX512F-NEXT: vmovq %rsi, %xmm2
+; AVX512F-NEXT: pushq %r14
+; AVX512F-NEXT: pushq %rbx
+; AVX512F-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; AVX512F-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX512F-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX512F-NEXT: movq {{[0-9]+}}(%rsp), %r14
+; AVX512F-NEXT: vmovq %rdi, %xmm0
+; AVX512F-NEXT: vmovq %rsi, %xmm1
+; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512F-NEXT: vmovq %rdx, %xmm1
+; AVX512F-NEXT: vmovq %rcx, %xmm2
; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX512F-NEXT: vmovq %rdx, %xmm2
-; AVX512F-NEXT: vmovq %rcx, %xmm3
-; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
-; AVX512F-NEXT: vpshufd {{.*#+}} xmm3 = mem[2,3,0,1]
+; AVX512F-NEXT: vpshufd {{.*#+}} xmm2 = mem[2,3,0,1]
+; AVX512F-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512F-NEXT: vmovq %r8, %xmm1
+; AVX512F-NEXT: vmovq %r9, %xmm3
+; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm3[0],xmm1[0]
; AVX512F-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
-; AVX512F-NEXT: vmovq %r8, %xmm2
-; AVX512F-NEXT: vmovq %r9, %xmm4
-; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm4[0],xmm2[0]
-; AVX512F-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
-; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
-; AVX512F-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448]
-; AVX512F-NEXT: vpaddq %zmm3, %zmm2, %zmm2
-; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
-; AVX512F-NEXT: vpcompressq %zmm2, %zmm1 {%k1}
-; AVX512F-NEXT: vmovd %xmm1, %ecx
+; AVX512F-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512F-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,64,128,192,256,320,384,448]
+; AVX512F-NEXT: vpaddq %zmm2, %zmm1, %zmm1
+; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
+; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
+; AVX512F-NEXT: vmovd %xmm0, %ecx
; AVX512F-NEXT: addl $512, %ecx # imm = 0x200
-; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
-; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm1
-; AVX512F-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512F-NEXT: vpaddq %zmm3, %zmm2, %zmm2
-; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512F-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
-; AVX512F-NEXT: vmovd %xmm1, %eax
-; AVX512F-NEXT: vpor {{[0-9]+}}(%rsp), %ymm0, %ymm0
-; AVX512F-NEXT: vptest %ymm0, %ymm0
+; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]
+; AVX512F-NEXT: vpermq {{[0-9]+}}(%rsp), %zmm0, %zmm0
+; AVX512F-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512F-NEXT: vpaddq %zmm2, %zmm1, %zmm1
+; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; AVX512F-NEXT: orq {{[0-9]+}}(%rsp), %r14
+; AVX512F-NEXT: vmovd %xmm0, %eax
+; AVX512F-NEXT: orq {{[0-9]+}}(%rsp), %r11
+; AVX512F-NEXT: orq %r14, %r11
+; AVX512F-NEXT: orq {{[0-9]+}}(%rsp), %rbx
+; AVX512F-NEXT: orq {{[0-9]+}}(%rsp), %r10
+; AVX512F-NEXT: orq %rbx, %r10
+; AVX512F-NEXT: orq %r11, %r10
; AVX512F-NEXT: cmovel %ecx, %eax
+; AVX512F-NEXT: popq %rbx
+; AVX512F-NEXT: popq %r14
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: test_ctlz_i1024:
; AVX512POPCNT: # %bb.0:
-; AVX512POPCNT-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
-; AVX512POPCNT-NEXT: vmovq %rdi, %xmm1
-; AVX512POPCNT-NEXT: vmovq %rsi, %xmm2
+; AVX512POPCNT-NEXT: pushq %r14
+; AVX512POPCNT-NEXT: pushq %rbx
+; AVX512POPCNT-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; AVX512POPCNT-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX512POPCNT-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX512POPCNT-NEXT: movq {{[0-9]+}}(%rsp), %r14
+; AVX512POPCNT-NEXT: vmovq %rdi, %xmm0
+; AVX512POPCNT-NEXT: vmovq %rsi, %xmm1
+; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512POPCNT-NEXT: vmovq %rdx, %xmm1
+; AVX512POPCNT-NEXT: vmovq %rcx, %xmm2
; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX512POPCNT-NEXT: vmovq %rdx, %xmm2
-; AVX512POPCNT-NEXT: vmovq %rcx, %xmm3
-; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
-; AVX512POPCNT-NEXT: vpshufd {{.*#+}} xmm3 = mem[2,3,0,1]
+; AVX512POPCNT-NEXT: vpshufd {{.*#+}} xmm2 = mem[2,3,0,1]
+; AVX512POPCNT-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512POPCNT-NEXT: vmovq %r8, %xmm1
+; AVX512POPCNT-NEXT: vmovq %r9, %xmm3
+; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm3[0],xmm1[0]
; AVX512POPCNT-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
-; AVX512POPCNT-NEXT: vmovq %r8, %xmm2
-; AVX512POPCNT-NEXT: vmovq %r9, %xmm4
-; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm4[0],xmm2[0]
-; AVX512POPCNT-NEXT: vinserti128 $1, %xmm2, %ymm3, %ymm2
-; AVX512POPCNT-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
-; AVX512POPCNT-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448]
-; AVX512POPCNT-NEXT: vpaddq %zmm3, %zmm2, %zmm2
-; AVX512POPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512POPCNT-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
-; AVX512POPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1}
-; AVX512POPCNT-NEXT: vmovd %xmm1, %ecx
+; AVX512POPCNT-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512POPCNT-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,64,128,192,256,320,384,448]
+; AVX512POPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm1
+; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512POPCNT-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
+; AVX512POPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
+; AVX512POPCNT-NEXT: vmovd %xmm0, %ecx
; AVX512POPCNT-NEXT: addl $512, %ecx # imm = 0x200
-; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
-; AVX512POPCNT-NEXT: vpermq %zmm0, %zmm1, %zmm1
-; AVX512POPCNT-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512POPCNT-NEXT: vpaddq %zmm3, %zmm2, %zmm2
-; AVX512POPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512POPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
-; AVX512POPCNT-NEXT: vmovd %xmm1, %eax
-; AVX512POPCNT-NEXT: vpor {{[0-9]+}}(%rsp), %ymm0, %ymm0
-; AVX512POPCNT-NEXT: vptest %ymm0, %ymm0
+; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]
+; AVX512POPCNT-NEXT: vpermq {{[0-9]+}}(%rsp), %zmm0, %zmm0
+; AVX512POPCNT-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512POPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm1
+; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512POPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; AVX512POPCNT-NEXT: orq {{[0-9]+}}(%rsp), %r14
+; AVX512POPCNT-NEXT: vmovd %xmm0, %eax
+; AVX512POPCNT-NEXT: orq {{[0-9]+}}(%rsp), %r11
+; AVX512POPCNT-NEXT: orq %r14, %r11
+; AVX512POPCNT-NEXT: orq {{[0-9]+}}(%rsp), %rbx
+; AVX512POPCNT-NEXT: orq {{[0-9]+}}(%rsp), %r10
+; AVX512POPCNT-NEXT: orq %rbx, %r10
+; AVX512POPCNT-NEXT: orq %r11, %r10
; AVX512POPCNT-NEXT: cmovel %ecx, %eax
+; AVX512POPCNT-NEXT: popq %rbx
+; AVX512POPCNT-NEXT: popq %r14
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: test_ctlz_i1024:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
-; AVX512VL-NEXT: vmovq %rdi, %xmm1
-; AVX512VL-NEXT: vmovq %rsi, %xmm2
+; AVX512VL-NEXT: pushq %r14
+; AVX512VL-NEXT: pushq %rbx
+; AVX512VL-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; AVX512VL-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX512VL-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX512VL-NEXT: movq {{[0-9]+}}(%rsp), %r14
+; AVX512VL-NEXT: vmovq %rdi, %xmm0
+; AVX512VL-NEXT: vmovq %rsi, %xmm1
+; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512VL-NEXT: vmovq %rdx, %xmm1
+; AVX512VL-NEXT: vmovq %rcx, %xmm2
; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX512VL-NEXT: vmovq %rdx, %xmm2
-; AVX512VL-NEXT: vmovq %rcx, %xmm3
+; AVX512VL-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512VL-NEXT: vpshufd {{.*#+}} xmm1 = mem[2,3,0,1]
+; AVX512VL-NEXT: vmovq %r8, %xmm2
+; AVX512VL-NEXT: vmovq %r9, %xmm3
; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
-; AVX512VL-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
-; AVX512VL-NEXT: vpshufd {{.*#+}} xmm2 = mem[2,3,0,1]
-; AVX512VL-NEXT: vmovq %r8, %xmm3
-; AVX512VL-NEXT: vmovq %r9, %xmm4
-; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]
-; AVX512VL-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
-; AVX512VL-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
-; AVX512VL-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448]
-; AVX512VL-NEXT: vpaddq %zmm3, %zmm2, %zmm2
-; AVX512VL-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512VL-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
-; AVX512VL-NEXT: vpcompressq %zmm2, %zmm1 {%k1}
-; AVX512VL-NEXT: vmovd %xmm1, %ecx
+; AVX512VL-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512VL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512VL-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,64,128,192,256,320,384,448]
+; AVX512VL-NEXT: vpaddq %zmm2, %zmm1, %zmm1
+; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512VL-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
+; AVX512VL-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
+; AVX512VL-NEXT: vmovd %xmm0, %ecx
; AVX512VL-NEXT: addl $512, %ecx # imm = 0x200
-; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
-; AVX512VL-NEXT: vpermq %zmm0, %zmm1, %zmm1
-; AVX512VL-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512VL-NEXT: vpaddq %zmm3, %zmm2, %zmm2
-; AVX512VL-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512VL-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
-; AVX512VL-NEXT: vmovd %xmm1, %eax
-; AVX512VL-NEXT: vpor {{[0-9]+}}(%rsp), %ymm0, %ymm0
-; AVX512VL-NEXT: vptest %ymm0, %ymm0
+; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]
+; AVX512VL-NEXT: vpermq {{[0-9]+}}(%rsp), %zmm0, %zmm0
+; AVX512VL-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512VL-NEXT: vpaddq %zmm2, %zmm1, %zmm1
+; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512VL-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; AVX512VL-NEXT: vmovd %xmm0, %eax
+; AVX512VL-NEXT: orq {{[0-9]+}}(%rsp), %r14
+; AVX512VL-NEXT: orq {{[0-9]+}}(%rsp), %r11
+; AVX512VL-NEXT: orq {{[0-9]+}}(%rsp), %rbx
+; AVX512VL-NEXT: orq %r14, %r11
+; AVX512VL-NEXT: orq {{[0-9]+}}(%rsp), %r10
+; AVX512VL-NEXT: orq %rbx, %r10
+; AVX512VL-NEXT: orq %r11, %r10
; AVX512VL-NEXT: cmovel %ecx, %eax
+; AVX512VL-NEXT: popq %rbx
+; AVX512VL-NEXT: popq %r14
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: test_ctlz_i1024:
; AVX512VLPOPCNT: # %bb.0:
-; AVX512VLPOPCNT-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
-; AVX512VLPOPCNT-NEXT: vmovq %rdi, %xmm1
-; AVX512VLPOPCNT-NEXT: vmovq %rsi, %xmm2
+; AVX512VLPOPCNT-NEXT: pushq %r14
+; AVX512VLPOPCNT-NEXT: pushq %rbx
+; AVX512VLPOPCNT-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; AVX512VLPOPCNT-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX512VLPOPCNT-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX512VLPOPCNT-NEXT: movq {{[0-9]+}}(%rsp), %r14
+; AVX512VLPOPCNT-NEXT: vmovq %rdi, %xmm0
+; AVX512VLPOPCNT-NEXT: vmovq %rsi, %xmm1
+; AVX512VLPOPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512VLPOPCNT-NEXT: vmovq %rdx, %xmm1
+; AVX512VLPOPCNT-NEXT: vmovq %rcx, %xmm2
; AVX512VLPOPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX512VLPOPCNT-NEXT: vmovq %rdx, %xmm2
-; AVX512VLPOPCNT-NEXT: vmovq %rcx, %xmm3
+; AVX512VLPOPCNT-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512VLPOPCNT-NEXT: vpshufd {{.*#+}} xmm1 = mem[2,3,0,1]
+; AVX512VLPOPCNT-NEXT: vmovq %r8, %xmm2
+; AVX512VLPOPCNT-NEXT: vmovq %r9, %xmm3
; AVX512VLPOPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
-; AVX512VLPOPCNT-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
-; AVX512VLPOPCNT-NEXT: vpshufd {{.*#+}} xmm2 = mem[2,3,0,1]
-; AVX512VLPOPCNT-NEXT: vmovq %r8, %xmm3
-; AVX512VLPOPCNT-NEXT: vmovq %r9, %xmm4
-; AVX512VLPOPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]
-; AVX512VLPOPCNT-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
-; AVX512VLPOPCNT-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
-; AVX512VLPOPCNT-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448]
-; AVX512VLPOPCNT-NEXT: vpaddq %zmm3, %zmm2, %zmm2
-; AVX512VLPOPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512VLPOPCNT-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
-; AVX512VLPOPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1}
-; AVX512VLPOPCNT-NEXT: vmovd %xmm1, %ecx
+; AVX512VLPOPCNT-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512VLPOPCNT-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512VLPOPCNT-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,64,128,192,256,320,384,448]
+; AVX512VLPOPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm1
+; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512VLPOPCNT-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
+; AVX512VLPOPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
+; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %ecx
; AVX512VLPOPCNT-NEXT: addl $512, %ecx # imm = 0x200
-; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
-; AVX512VLPOPCNT-NEXT: vpermq %zmm0, %zmm1, %zmm1
-; AVX512VLPOPCNT-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512VLPOPCNT-NEXT: vpaddq %zmm3, %zmm2, %zmm2
-; AVX512VLPOPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512VLPOPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
-; AVX512VLPOPCNT-NEXT: vmovd %xmm1, %eax
-; AVX512VLPOPCNT-NEXT: vpor {{[0-9]+}}(%rsp), %ymm0, %ymm0
-; AVX512VLPOPCNT-NEXT: vptest %ymm0, %ymm0
+; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]
+; AVX512VLPOPCNT-NEXT: vpermq {{[0-9]+}}(%rsp), %zmm0, %zmm0
+; AVX512VLPOPCNT-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512VLPOPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm1
+; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512VLPOPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
+; AVX512VLPOPCNT-NEXT: orq {{[0-9]+}}(%rsp), %r14
+; AVX512VLPOPCNT-NEXT: orq {{[0-9]+}}(%rsp), %r11
+; AVX512VLPOPCNT-NEXT: orq {{[0-9]+}}(%rsp), %rbx
+; AVX512VLPOPCNT-NEXT: orq %r14, %r11
+; AVX512VLPOPCNT-NEXT: orq {{[0-9]+}}(%rsp), %r10
+; AVX512VLPOPCNT-NEXT: orq %rbx, %r10
+; AVX512VLPOPCNT-NEXT: orq %r11, %r10
; AVX512VLPOPCNT-NEXT: cmovel %ecx, %eax
+; AVX512VLPOPCNT-NEXT: popq %rbx
+; AVX512VLPOPCNT-NEXT: popq %r14
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%cnt = call i1024 @llvm.ctlz.i1024(i1024 %a0, i1 0)
@@ -3058,305 +3378,390 @@ define i32 @test_ctlz_i1024(i1024 %a0) nounwind {
define i32 @load_ctlz_i1024(ptr %p0) nounwind {
; SSE-LABEL: load_ctlz_i1024:
; SSE: # %bb.0:
+; SSE-NEXT: pushq %rbp
+; SSE-NEXT: pushq %r15
+; SSE-NEXT: pushq %r14
+; SSE-NEXT: pushq %r13
+; SSE-NEXT: pushq %r12
; SSE-NEXT: pushq %rbx
-; SSE-NEXT: movq 8(%rdi), %rcx
-; SSE-NEXT: movq 72(%rdi), %rax
+; SSE-NEXT: movq 40(%rdi), %rbp
+; SSE-NEXT: movq 64(%rdi), %rbx
+; SSE-NEXT: movq 72(%rdi), %r11
+; SSE-NEXT: movq 80(%rdi), %r12
+; SSE-NEXT: movq 88(%rdi), %r14
+; SSE-NEXT: movq 96(%rdi), %rsi
; SSE-NEXT: movq 104(%rdi), %r9
-; SSE-NEXT: movq 112(%rdi), %rdx
+; SSE-NEXT: movq 112(%rdi), %r10
; SSE-NEXT: movq 120(%rdi), %r8
-; SSE-NEXT: bsrq %r8, %rsi
-; SSE-NEXT: xorl $63, %esi
-; SSE-NEXT: bsrq %rdx, %r11
-; SSE-NEXT: xorl $63, %r11d
-; SSE-NEXT: orl $64, %r11d
+; SSE-NEXT: bsrq %r8, %rax
+; SSE-NEXT: xorl $63, %eax
+; SSE-NEXT: bsrq %r10, %rcx
+; SSE-NEXT: xorl $63, %ecx
+; SSE-NEXT: orl $64, %ecx
; SSE-NEXT: testq %r8, %r8
-; SSE-NEXT: cmovnel %esi, %r11d
-; SSE-NEXT: bsrq %r9, %r10
-; SSE-NEXT: xorl $63, %r10d
-; SSE-NEXT: bsrq 96(%rdi), %rsi
-; SSE-NEXT: xorl $63, %esi
-; SSE-NEXT: orl $64, %esi
+; SSE-NEXT: cmovnel %eax, %ecx
+; SSE-NEXT: bsrq %r9, %rdx
+; SSE-NEXT: xorl $63, %edx
+; SSE-NEXT: bsrq %rsi, %rax
+; SSE-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE-NEXT: xorl $63, %eax
+; SSE-NEXT: orl $64, %eax
; SSE-NEXT: testq %r9, %r9
-; SSE-NEXT: cmovnel %r10d, %esi
-; SSE-NEXT: movq 80(%rdi), %r9
-; SSE-NEXT: movq 88(%rdi), %r10
-; SSE-NEXT: subl $-128, %esi
+; SSE-NEXT: movq %r9, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE-NEXT: cmovnel %edx, %eax
+; SSE-NEXT: subl $-128, %eax
+; SSE-NEXT: movq %r10, %rdx
+; SSE-NEXT: movq %r10, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; SSE-NEXT: orq %r8, %rdx
-; SSE-NEXT: cmovnel %r11d, %esi
-; SSE-NEXT: bsrq %r10, %rdx
-; SSE-NEXT: xorl $63, %edx
-; SSE-NEXT: bsrq %r9, %r11
-; SSE-NEXT: xorl $63, %r11d
-; SSE-NEXT: orl $64, %r11d
-; SSE-NEXT: testq %r10, %r10
-; SSE-NEXT: cmovnel %edx, %r11d
-; SSE-NEXT: bsrq %rax, %rbx
-; SSE-NEXT: xorl $63, %ebx
-; SSE-NEXT: bsrq 64(%rdi), %rdx
+; SSE-NEXT: movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE-NEXT: cmovnel %ecx, %eax
+; SSE-NEXT: bsrq %r14, %rcx
+; SSE-NEXT: xorl $63, %ecx
+; SSE-NEXT: movq %r12, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE-NEXT: bsrq %r12, %rdx
; SSE-NEXT: xorl $63, %edx
; SSE-NEXT: orl $64, %edx
-; SSE-NEXT: testq %rax, %rax
-; SSE-NEXT: movq 40(%rdi), %r8
-; SSE-NEXT: cmovnel %ebx, %edx
-; SSE-NEXT: movq 48(%rdi), %rax
-; SSE-NEXT: movdqa 112(%rdi), %xmm0
-; SSE-NEXT: movdqa 96(%rdi), %xmm1
-; SSE-NEXT: subl $-128, %edx
-; SSE-NEXT: orq %r10, %r9
-; SSE-NEXT: cmovnel %r11d, %edx
-; SSE-NEXT: addl $256, %edx # imm = 0x100
-; SSE-NEXT: movdqa %xmm1, %xmm2
-; SSE-NEXT: por %xmm0, %xmm2
-; SSE-NEXT: ptest %xmm2, %xmm2
-; SSE-NEXT: movq 56(%rdi), %r10
-; SSE-NEXT: cmovnel %esi, %edx
-; SSE-NEXT: bsrq %r10, %rsi
-; SSE-NEXT: xorl $63, %esi
-; SSE-NEXT: bsrq %rax, %r11
-; SSE-NEXT: xorl $63, %r11d
-; SSE-NEXT: orl $64, %r11d
+; SSE-NEXT: testq %r14, %r14
+; SSE-NEXT: cmovnel %ecx, %edx
+; SSE-NEXT: bsrq %r11, %rcx
+; SSE-NEXT: xorl $63, %ecx
+; SSE-NEXT: bsrq %rbx, %r15
+; SSE-NEXT: xorl $63, %r15d
+; SSE-NEXT: orl $64, %r15d
+; SSE-NEXT: testq %r11, %r11
+; SSE-NEXT: cmovnel %ecx, %r15d
+; SSE-NEXT: subl $-128, %r15d
+; SSE-NEXT: movq %r12, %rcx
+; SSE-NEXT: orq %r14, %rcx
+; SSE-NEXT: cmovnel %edx, %r15d
+; SSE-NEXT: movq 48(%rdi), %r12
+; SSE-NEXT: addl $256, %r15d # imm = 0x100
+; SSE-NEXT: movq %r9, %rcx
+; SSE-NEXT: orq %r8, %rcx
+; SSE-NEXT: movq %rsi, %rdx
+; SSE-NEXT: orq %r10, %rdx
+; SSE-NEXT: orq %rcx, %rdx
+; SSE-NEXT: movq 56(%rdi), %r13
+; SSE-NEXT: cmovnel %eax, %r15d
+; SSE-NEXT: bsrq %r13, %rax
+; SSE-NEXT: xorl $63, %eax
+; SSE-NEXT: bsrq %r12, %rcx
+; SSE-NEXT: xorl $63, %ecx
+; SSE-NEXT: orl $64, %ecx
+; SSE-NEXT: testq %r13, %r13
+; SSE-NEXT: cmovnel %eax, %ecx
+; SSE-NEXT: movq %rbp, %r10
+; SSE-NEXT: bsrq %rbp, %rax
+; SSE-NEXT: xorl $63, %eax
+; SSE-NEXT: movq 32(%rdi), %r8
+; SSE-NEXT: bsrq %r8, %rbp
+; SSE-NEXT: xorl $63, %ebp
+; SSE-NEXT: orl $64, %ebp
; SSE-NEXT: testq %r10, %r10
-; SSE-NEXT: cmovnel %esi, %r11d
-; SSE-NEXT: bsrq %r8, %r9
-; SSE-NEXT: xorl $63, %r9d
-; SSE-NEXT: bsrq 32(%rdi), %rsi
-; SSE-NEXT: xorl $63, %esi
-; SSE-NEXT: orl $64, %esi
-; SSE-NEXT: testq %r8, %r8
-; SSE-NEXT: cmovnel %r9d, %esi
-; SSE-NEXT: movq 16(%rdi), %r8
+; SSE-NEXT: cmovnel %eax, %ebp
+; SSE-NEXT: subl $-128, %ebp
+; SSE-NEXT: movq %r12, %rax
+; SSE-NEXT: orq %r13, %rax
+; SSE-NEXT: cmovnel %ecx, %ebp
; SSE-NEXT: movq 24(%rdi), %r9
-; SSE-NEXT: subl $-128, %esi
-; SSE-NEXT: orq %r10, %rax
-; SSE-NEXT: cmovnel %r11d, %esi
; SSE-NEXT: bsrq %r9, %rax
; SSE-NEXT: xorl $63, %eax
-; SSE-NEXT: bsrq %r8, %r10
-; SSE-NEXT: xorl $63, %r10d
-; SSE-NEXT: orl $64, %r10d
+; SSE-NEXT: movq 16(%rdi), %rsi
+; SSE-NEXT: bsrq %rsi, %rcx
+; SSE-NEXT: xorl $63, %ecx
+; SSE-NEXT: orl $64, %ecx
; SSE-NEXT: testq %r9, %r9
-; SSE-NEXT: cmovnel %eax, %r10d
-; SSE-NEXT: bsrq %rcx, %r11
-; SSE-NEXT: xorl $63, %r11d
+; SSE-NEXT: cmovnel %eax, %ecx
; SSE-NEXT: movl $127, %eax
; SSE-NEXT: bsrq (%rdi), %rax
+; SSE-NEXT: movq 8(%rdi), %rdi
+; SSE-NEXT: bsrq %rdi, %rdx
+; SSE-NEXT: xorl $63, %edx
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: addl $64, %eax
-; SSE-NEXT: testq %rcx, %rcx
-; SSE-NEXT: cmovnel %r11d, %eax
+; SSE-NEXT: testq %rdi, %rdi
+; SSE-NEXT: cmovnel %edx, %eax
; SSE-NEXT: subl $-128, %eax
-; SSE-NEXT: orq %r9, %r8
-; SSE-NEXT: cmovnel %r10d, %eax
-; SSE-NEXT: movdqa 32(%rdi), %xmm2
+; SSE-NEXT: orq %r9, %rsi
+; SSE-NEXT: cmovnel %ecx, %eax
+; SSE-NEXT: orq %r13, %r10
+; SSE-NEXT: orq %r12, %r8
; SSE-NEXT: addl $256, %eax # imm = 0x100
-; SSE-NEXT: por 48(%rdi), %xmm2
-; SSE-NEXT: ptest %xmm2, %xmm2
-; SSE-NEXT: cmovnel %esi, %eax
-; SSE-NEXT: por 80(%rdi), %xmm0
-; SSE-NEXT: por 64(%rdi), %xmm1
+; SSE-NEXT: orq %r10, %r8
+; SSE-NEXT: cmovnel %ebp, %eax
+; SSE-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r14 # 8-byte Folded Reload
+; SSE-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Folded Reload
+; SSE-NEXT: orq %r14, %r11
+; SSE-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; SSE-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Folded Reload
+; SSE-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Folded Reload
+; SSE-NEXT: orq %rcx, %rbx
; SSE-NEXT: addl $512, %eax # imm = 0x200
-; SSE-NEXT: por %xmm0, %xmm1
-; SSE-NEXT: ptest %xmm1, %xmm1
-; SSE-NEXT: cmovnel %edx, %eax
+; SSE-NEXT: orq %r11, %rbx
+; SSE-NEXT: cmovnel %r15d, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: popq %rbx
+; SSE-NEXT: popq %r12
+; SSE-NEXT: popq %r13
+; SSE-NEXT: popq %r14
+; SSE-NEXT: popq %r15
+; SSE-NEXT: popq %rbp
; SSE-NEXT: retq
;
; AVX2-LABEL: load_ctlz_i1024:
; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %rbp
+; AVX2-NEXT: pushq %r15
+; AVX2-NEXT: pushq %r14
+; AVX2-NEXT: pushq %r13
+; AVX2-NEXT: pushq %r12
; AVX2-NEXT: pushq %rbx
-; AVX2-NEXT: movq 16(%rdi), %rcx
-; AVX2-NEXT: movq 72(%rdi), %rsi
-; AVX2-NEXT: movq 104(%rdi), %rdx
-; AVX2-NEXT: movq 112(%rdi), %r8
-; AVX2-NEXT: movq 120(%rdi), %r10
-; AVX2-NEXT: lzcntq %r10, %rax
-; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: lzcntq %r8, %rbx
-; AVX2-NEXT: addl $64, %ebx
-; AVX2-NEXT: testq %r10, %r10
-; AVX2-NEXT: cmovnel %eax, %ebx
-; AVX2-NEXT: lzcntq %rdx, %r11
+; AVX2-NEXT: movq 48(%rdi), %r9
+; AVX2-NEXT: movq 56(%rdi), %rbp
+; AVX2-NEXT: movq 64(%rdi), %r11
+; AVX2-NEXT: movq 72(%rdi), %r10
+; AVX2-NEXT: movq 80(%rdi), %r14
+; AVX2-NEXT: movq 88(%rdi), %rbx
+; AVX2-NEXT: movq 96(%rdi), %rdx
+; AVX2-NEXT: movq 104(%rdi), %r8
+; AVX2-NEXT: movq 112(%rdi), %rsi
+; AVX2-NEXT: movq 120(%rdi), %r15
+; AVX2-NEXT: lzcntq %r15, %rax
+; AVX2-NEXT: lzcntq %rsi, %rcx
+; AVX2-NEXT: addl $64, %ecx
+; AVX2-NEXT: testq %r15, %r15
+; AVX2-NEXT: cmovnel %eax, %ecx
+; AVX2-NEXT: xorl %r12d, %r12d
+; AVX2-NEXT: lzcntq %r8, %r12
; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: lzcntq 96(%rdi), %rax
+; AVX2-NEXT: lzcntq %rdx, %rax
+; AVX2-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: addl $64, %eax
-; AVX2-NEXT: testq %rdx, %rdx
-; AVX2-NEXT: movq 80(%rdi), %r9
-; AVX2-NEXT: cmovnel %r11d, %eax
-; AVX2-NEXT: movq 88(%rdi), %r11
+; AVX2-NEXT: testq %r8, %r8
+; AVX2-NEXT: movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: cmovnel %r12d, %eax
; AVX2-NEXT: subl $-128, %eax
-; AVX2-NEXT: orq %r10, %r8
-; AVX2-NEXT: cmovnel %ebx, %eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: lzcntq %r11, %rdx
-; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: lzcntq %r9, %rbx
-; AVX2-NEXT: addl $64, %ebx
-; AVX2-NEXT: testq %r11, %r11
-; AVX2-NEXT: cmovnel %edx, %ebx
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: lzcntq 64(%rdi), %rdx
-; AVX2-NEXT: xorl %r8d, %r8d
-; AVX2-NEXT: lzcntq %rsi, %r8
-; AVX2-NEXT: addl $64, %edx
-; AVX2-NEXT: testq %rsi, %rsi
-; AVX2-NEXT: movq 40(%rdi), %r10
-; AVX2-NEXT: cmovnel %r8d, %edx
-; AVX2-NEXT: movq 48(%rdi), %r8
-; AVX2-NEXT: subl $-128, %edx
-; AVX2-NEXT: orq %r11, %r9
-; AVX2-NEXT: movq 56(%rdi), %r9
-; AVX2-NEXT: cmovnel %ebx, %edx
-; AVX2-NEXT: vmovdqu 96(%rdi), %ymm0
-; AVX2-NEXT: addl $256, %edx # imm = 0x100
-; AVX2-NEXT: vpor 112(%rdi), %xmm0, %xmm1
-; AVX2-NEXT: vptest %xmm1, %xmm1
-; AVX2-NEXT: cmovnel %eax, %edx
+; AVX2-NEXT: movq %rsi, %r12
+; AVX2-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: orq %r15, %r12
+; AVX2-NEXT: cmovnel %ecx, %eax
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: lzcntq %rbx, %rcx
+; AVX2-NEXT: movq %r14, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: xorl %r13d, %r13d
+; AVX2-NEXT: lzcntq %r14, %r13
+; AVX2-NEXT: addl $64, %r13d
+; AVX2-NEXT: testq %rbx, %rbx
+; AVX2-NEXT: cmovnel %ecx, %r13d
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: lzcntq %r10, %rcx
+; AVX2-NEXT: xorl %r12d, %r12d
+; AVX2-NEXT: lzcntq %r11, %r12
+; AVX2-NEXT: addl $64, %r12d
+; AVX2-NEXT: testq %r10, %r10
+; AVX2-NEXT: cmovnel %ecx, %r12d
+; AVX2-NEXT: subl $-128, %r12d
+; AVX2-NEXT: movq %r14, %rcx
+; AVX2-NEXT: orq %rbx, %rcx
+; AVX2-NEXT: cmovnel %r13d, %r12d
+; AVX2-NEXT: addl $256, %r12d # imm = 0x100
+; AVX2-NEXT: movq %r8, %rcx
+; AVX2-NEXT: orq %r15, %rcx
+; AVX2-NEXT: orq %rsi, %rdx
+; AVX2-NEXT: orq %rcx, %rdx
+; AVX2-NEXT: cmovnel %eax, %r12d
+; AVX2-NEXT: movq %rbp, %r14
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: lzcntq %rbp, %rcx
+; AVX2-NEXT: movq %r9, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq %r9, %rax
-; AVX2-NEXT: xorl %r11d, %r11d
-; AVX2-NEXT: lzcntq %r8, %r11
-; AVX2-NEXT: addl $64, %r11d
-; AVX2-NEXT: testq %r9, %r9
-; AVX2-NEXT: cmovnel %eax, %r11d
-; AVX2-NEXT: xorl %esi, %esi
-; AVX2-NEXT: lzcntq 32(%rdi), %rsi
-; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: lzcntq %r10, %rax
-; AVX2-NEXT: addl $64, %esi
-; AVX2-NEXT: testq %r10, %r10
-; AVX2-NEXT: movq 24(%rdi), %r10
-; AVX2-NEXT: cmovnel %eax, %esi
-; AVX2-NEXT: subl $-128, %esi
-; AVX2-NEXT: orq %r9, %r8
-; AVX2-NEXT: cmovnel %r11d, %esi
+; AVX2-NEXT: addl $64, %eax
+; AVX2-NEXT: testq %rbp, %rbp
+; AVX2-NEXT: cmovnel %ecx, %eax
+; AVX2-NEXT: movq 32(%rdi), %r13
+; AVX2-NEXT: xorl %ebp, %ebp
+; AVX2-NEXT: lzcntq %r13, %rbp
+; AVX2-NEXT: addl $64, %ebp
+; AVX2-NEXT: movq 40(%rdi), %r8
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: lzcntq %r8, %rdx
+; AVX2-NEXT: testq %r8, %r8
+; AVX2-NEXT: cmovnel %edx, %ebp
+; AVX2-NEXT: subl $-128, %ebp
+; AVX2-NEXT: movq %r9, %rdx
+; AVX2-NEXT: orq %r14, %rdx
+; AVX2-NEXT: cmovnel %eax, %ebp
+; AVX2-NEXT: movq 16(%rdi), %r9
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: lzcntq %r9, %rcx
+; AVX2-NEXT: addl $64, %ecx
+; AVX2-NEXT: movq 24(%rdi), %rdx
; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: lzcntq %r10, %rax
-; AVX2-NEXT: xorl %r8d, %r8d
-; AVX2-NEXT: lzcntq %rcx, %r8
-; AVX2-NEXT: addl $64, %r8d
-; AVX2-NEXT: testq %r10, %r10
-; AVX2-NEXT: cmovnel %eax, %r8d
-; AVX2-NEXT: movq 8(%rdi), %r9
-; AVX2-NEXT: xorl %r11d, %r11d
-; AVX2-NEXT: lzcntq %r9, %r11
+; AVX2-NEXT: lzcntq %rdx, %rax
+; AVX2-NEXT: testq %rdx, %rdx
+; AVX2-NEXT: cmovnel %eax, %ecx
+; AVX2-NEXT: movq 8(%rdi), %rsi
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq (%rdi), %rax
; AVX2-NEXT: addl $64, %eax
-; AVX2-NEXT: testq %r9, %r9
-; AVX2-NEXT: cmovnel %r11d, %eax
+; AVX2-NEXT: lzcntq %rsi, %rdi
+; AVX2-NEXT: testq %rsi, %rsi
+; AVX2-NEXT: cmovnel %edi, %eax
; AVX2-NEXT: subl $-128, %eax
-; AVX2-NEXT: orq %r10, %rcx
-; AVX2-NEXT: cmovnel %r8d, %eax
-; AVX2-NEXT: vmovdqa 32(%rdi), %xmm1
+; AVX2-NEXT: orq %rdx, %r9
+; AVX2-NEXT: cmovnel %ecx, %eax
+; AVX2-NEXT: orq %r14, %r8
+; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Folded Reload
; AVX2-NEXT: addl $256, %eax # imm = 0x100
-; AVX2-NEXT: vpor 48(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vptest %xmm1, %xmm1
-; AVX2-NEXT: cmovnel %esi, %eax
-; AVX2-NEXT: vpor 64(%rdi), %ymm0, %ymm0
+; AVX2-NEXT: orq %r8, %r13
+; AVX2-NEXT: cmovnel %ebp, %eax
+; AVX2-NEXT: orq %r15, %rbx
+; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Folded Reload
+; AVX2-NEXT: orq %rbx, %r10
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Folded Reload
+; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Folded Reload
+; AVX2-NEXT: orq %rcx, %r11
; AVX2-NEXT: addl $512, %eax # imm = 0x200
-; AVX2-NEXT: vptest %ymm0, %ymm0
-; AVX2-NEXT: cmovnel %edx, %eax
+; AVX2-NEXT: orq %r10, %r11
+; AVX2-NEXT: cmovnel %r12d, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: popq %rbx
-; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: popq %r12
+; AVX2-NEXT: popq %r13
+; AVX2-NEXT: popq %r14
+; AVX2-NEXT: popq %r15
+; AVX2-NEXT: popq %rbp
; AVX2-NEXT: retq
;
; AVX512F-LABEL: load_ctlz_i1024:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vmovdqu64 64(%rdi), %zmm0
-; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
-; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm2
-; AVX512F-NEXT: vplzcntq %zmm2, %zmm3
-; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm4 = [0,64,128,192,256,320,384,448]
-; AVX512F-NEXT: vpaddq %zmm4, %zmm3, %zmm3
-; AVX512F-NEXT: vptestmq %zmm2, %zmm2, %k1
-; AVX512F-NEXT: vpcompressq %zmm3, %zmm2 {%k1} {z}
-; AVX512F-NEXT: vmovd %xmm2, %ecx
-; AVX512F-NEXT: vpermq (%rdi), %zmm1, %zmm1
+; AVX512F-NEXT: movq 80(%rdi), %rsi
+; AVX512F-NEXT: movq 64(%rdi), %rcx
+; AVX512F-NEXT: movq 72(%rdi), %rdx
+; AVX512F-NEXT: movq 88(%rdi), %r8
+; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]
+; AVX512F-NEXT: vpermq 64(%rdi), %zmm0, %zmm1
; AVX512F-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512F-NEXT: vpaddq %zmm4, %zmm2, %zmm2
+; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448]
+; AVX512F-NEXT: vpaddq %zmm3, %zmm2, %zmm2
; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
-; AVX512F-NEXT: vpcompressq %zmm2, %zmm1 {%k1}
-; AVX512F-NEXT: vmovd %xmm1, %eax
+; AVX512F-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
+; AVX512F-NEXT: vmovd %xmm1, %r9d
+; AVX512F-NEXT: vpermq (%rdi), %zmm0, %zmm0
+; AVX512F-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512F-NEXT: vpaddq %zmm3, %zmm1, %zmm1
+; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
+; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
+; AVX512F-NEXT: vmovd %xmm0, %eax
+; AVX512F-NEXT: orq 120(%rdi), %r8
; AVX512F-NEXT: addl $512, %eax # imm = 0x200
-; AVX512F-NEXT: vpor 96(%rdi), %ymm0, %ymm0
-; AVX512F-NEXT: vptest %ymm0, %ymm0
-; AVX512F-NEXT: cmovnel %ecx, %eax
+; AVX512F-NEXT: orq 104(%rdi), %rdx
+; AVX512F-NEXT: orq %r8, %rdx
+; AVX512F-NEXT: orq 112(%rdi), %rsi
+; AVX512F-NEXT: orq 96(%rdi), %rcx
+; AVX512F-NEXT: orq %rsi, %rcx
+; AVX512F-NEXT: orq %rdx, %rcx
+; AVX512F-NEXT: cmovnel %r9d, %eax
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: load_ctlz_i1024:
; AVX512POPCNT: # %bb.0:
-; AVX512POPCNT-NEXT: vmovdqu64 64(%rdi), %zmm0
-; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
-; AVX512POPCNT-NEXT: vpermq %zmm0, %zmm1, %zmm2
-; AVX512POPCNT-NEXT: vplzcntq %zmm2, %zmm3
-; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm4 = [0,64,128,192,256,320,384,448]
-; AVX512POPCNT-NEXT: vpaddq %zmm4, %zmm3, %zmm3
-; AVX512POPCNT-NEXT: vptestmq %zmm2, %zmm2, %k1
-; AVX512POPCNT-NEXT: vpcompressq %zmm3, %zmm2 {%k1} {z}
-; AVX512POPCNT-NEXT: vmovd %xmm2, %ecx
-; AVX512POPCNT-NEXT: vpermq (%rdi), %zmm1, %zmm1
+; AVX512POPCNT-NEXT: movq 80(%rdi), %rsi
+; AVX512POPCNT-NEXT: movq 64(%rdi), %rcx
+; AVX512POPCNT-NEXT: movq 72(%rdi), %rdx
+; AVX512POPCNT-NEXT: movq 88(%rdi), %r8
+; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]
+; AVX512POPCNT-NEXT: vpermq 64(%rdi), %zmm0, %zmm1
; AVX512POPCNT-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512POPCNT-NEXT: vpaddq %zmm4, %zmm2, %zmm2
+; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448]
+; AVX512POPCNT-NEXT: vpaddq %zmm3, %zmm2, %zmm2
; AVX512POPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512POPCNT-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
-; AVX512POPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1}
-; AVX512POPCNT-NEXT: vmovd %xmm1, %eax
+; AVX512POPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
+; AVX512POPCNT-NEXT: vmovd %xmm1, %r9d
+; AVX512POPCNT-NEXT: vpermq (%rdi), %zmm0, %zmm0
+; AVX512POPCNT-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512POPCNT-NEXT: vpaddq %zmm3, %zmm1, %zmm1
+; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512POPCNT-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
+; AVX512POPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
+; AVX512POPCNT-NEXT: vmovd %xmm0, %eax
+; AVX512POPCNT-NEXT: orq 120(%rdi), %r8
; AVX512POPCNT-NEXT: addl $512, %eax # imm = 0x200
-; AVX512POPCNT-NEXT: vpor 96(%rdi), %ymm0, %ymm0
-; AVX512POPCNT-NEXT: vptest %ymm0, %ymm0
-; AVX512POPCNT-NEXT: cmovnel %ecx, %eax
+; AVX512POPCNT-NEXT: orq 104(%rdi), %rdx
+; AVX512POPCNT-NEXT: orq %r8, %rdx
+; AVX512POPCNT-NEXT: orq 112(%rdi), %rsi
+; AVX512POPCNT-NEXT: orq 96(%rdi), %rcx
+; AVX512POPCNT-NEXT: orq %rsi, %rcx
+; AVX512POPCNT-NEXT: orq %rdx, %rcx
+; AVX512POPCNT-NEXT: cmovnel %r9d, %eax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: load_ctlz_i1024:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vmovdqu64 64(%rdi), %zmm0
-; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
-; AVX512VL-NEXT: vpermq %zmm0, %zmm1, %zmm2
-; AVX512VL-NEXT: vplzcntq %zmm2, %zmm3
-; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm4 = [0,64,128,192,256,320,384,448]
-; AVX512VL-NEXT: vpaddq %zmm4, %zmm3, %zmm3
-; AVX512VL-NEXT: vptestmq %zmm2, %zmm2, %k1
-; AVX512VL-NEXT: vpcompressq %zmm3, %zmm2 {%k1} {z}
-; AVX512VL-NEXT: vmovd %xmm2, %ecx
-; AVX512VL-NEXT: vpermq (%rdi), %zmm1, %zmm1
+; AVX512VL-NEXT: movq 80(%rdi), %rsi
+; AVX512VL-NEXT: movq 64(%rdi), %rcx
+; AVX512VL-NEXT: movq 72(%rdi), %rdx
+; AVX512VL-NEXT: movq 88(%rdi), %r8
+; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]
+; AVX512VL-NEXT: vpermq 64(%rdi), %zmm0, %zmm1
; AVX512VL-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512VL-NEXT: vpaddq %zmm4, %zmm2, %zmm2
+; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448]
+; AVX512VL-NEXT: vpaddq %zmm3, %zmm2, %zmm2
; AVX512VL-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512VL-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
-; AVX512VL-NEXT: vpcompressq %zmm2, %zmm1 {%k1}
-; AVX512VL-NEXT: vmovd %xmm1, %eax
+; AVX512VL-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
+; AVX512VL-NEXT: vmovd %xmm1, %r9d
+; AVX512VL-NEXT: vpermq (%rdi), %zmm0, %zmm0
+; AVX512VL-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512VL-NEXT: vpaddq %zmm3, %zmm1, %zmm1
+; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512VL-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
+; AVX512VL-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
+; AVX512VL-NEXT: vmovd %xmm0, %eax
; AVX512VL-NEXT: addl $512, %eax # imm = 0x200
-; AVX512VL-NEXT: vpor 96(%rdi), %ymm0, %ymm0
-; AVX512VL-NEXT: vptest %ymm0, %ymm0
-; AVX512VL-NEXT: cmovnel %ecx, %eax
+; AVX512VL-NEXT: orq 120(%rdi), %r8
+; AVX512VL-NEXT: orq 104(%rdi), %rdx
+; AVX512VL-NEXT: orq 112(%rdi), %rsi
+; AVX512VL-NEXT: orq %r8, %rdx
+; AVX512VL-NEXT: orq 96(%rdi), %rcx
+; AVX512VL-NEXT: orq %rsi, %rcx
+; AVX512VL-NEXT: orq %rdx, %rcx
+; AVX512VL-NEXT: cmovnel %r9d, %eax
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: load_ctlz_i1024:
; AVX512VLPOPCNT: # %bb.0:
-; AVX512VLPOPCNT-NEXT: vmovdqu64 64(%rdi), %zmm0
-; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
-; AVX512VLPOPCNT-NEXT: vpermq %zmm0, %zmm1, %zmm2
-; AVX512VLPOPCNT-NEXT: vplzcntq %zmm2, %zmm3
-; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm4 = [0,64,128,192,256,320,384,448]
-; AVX512VLPOPCNT-NEXT: vpaddq %zmm4, %zmm3, %zmm3
-; AVX512VLPOPCNT-NEXT: vptestmq %zmm2, %zmm2, %k1
-; AVX512VLPOPCNT-NEXT: vpcompressq %zmm3, %zmm2 {%k1} {z}
-; AVX512VLPOPCNT-NEXT: vmovd %xmm2, %ecx
-; AVX512VLPOPCNT-NEXT: vpermq (%rdi), %zmm1, %zmm1
+; AVX512VLPOPCNT-NEXT: movq 80(%rdi), %rsi
+; AVX512VLPOPCNT-NEXT: movq 64(%rdi), %rcx
+; AVX512VLPOPCNT-NEXT: movq 72(%rdi), %rdx
+; AVX512VLPOPCNT-NEXT: movq 88(%rdi), %r8
+; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]
+; AVX512VLPOPCNT-NEXT: vpermq 64(%rdi), %zmm0, %zmm1
; AVX512VLPOPCNT-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512VLPOPCNT-NEXT: vpaddq %zmm4, %zmm2, %zmm2
+; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448]
+; AVX512VLPOPCNT-NEXT: vpaddq %zmm3, %zmm2, %zmm2
; AVX512VLPOPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512VLPOPCNT-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
-; AVX512VLPOPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1}
-; AVX512VLPOPCNT-NEXT: vmovd %xmm1, %eax
+; AVX512VLPOPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
+; AVX512VLPOPCNT-NEXT: vmovd %xmm1, %r9d
+; AVX512VLPOPCNT-NEXT: vpermq (%rdi), %zmm0, %zmm0
+; AVX512VLPOPCNT-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512VLPOPCNT-NEXT: vpaddq %zmm3, %zmm1, %zmm1
+; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512VLPOPCNT-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
+; AVX512VLPOPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
+; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
; AVX512VLPOPCNT-NEXT: addl $512, %eax # imm = 0x200
-; AVX512VLPOPCNT-NEXT: vpor 96(%rdi), %ymm0, %ymm0
-; AVX512VLPOPCNT-NEXT: vptest %ymm0, %ymm0
-; AVX512VLPOPCNT-NEXT: cmovnel %ecx, %eax
+; AVX512VLPOPCNT-NEXT: orq 120(%rdi), %r8
+; AVX512VLPOPCNT-NEXT: orq 104(%rdi), %rdx
+; AVX512VLPOPCNT-NEXT: orq 112(%rdi), %rsi
+; AVX512VLPOPCNT-NEXT: orq %r8, %rdx
+; AVX512VLPOPCNT-NEXT: orq 96(%rdi), %rcx
+; AVX512VLPOPCNT-NEXT: orq %rsi, %rcx
+; AVX512VLPOPCNT-NEXT: orq %rdx, %rcx
+; AVX512VLPOPCNT-NEXT: cmovnel %r9d, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%a0 = load i1024, ptr %p0
@@ -3996,101 +4401,116 @@ define i32 @test_ctlz_undef_i512(i512 %a0) nounwind {
define i32 @load_ctlz_undef_i512(ptr %p0) nounwind {
; SSE-LABEL: load_ctlz_undef_i512:
; SSE: # %bb.0:
+; SSE-NEXT: pushq %r15
+; SSE-NEXT: pushq %r14
; SSE-NEXT: pushq %rbx
-; SSE-NEXT: movdqa 32(%rdi), %xmm0
-; SSE-NEXT: movq 8(%rdi), %rsi
-; SSE-NEXT: movq 16(%rdi), %rcx
-; SSE-NEXT: movq 24(%rdi), %rdx
-; SSE-NEXT: movq 48(%rdi), %rax
-; SSE-NEXT: movq 56(%rdi), %r9
-; SSE-NEXT: bsrq %r9, %r8
-; SSE-NEXT: xorl $63, %r8d
-; SSE-NEXT: bsrq %rax, %r10
-; SSE-NEXT: xorl $63, %r10d
-; SSE-NEXT: orl $64, %r10d
-; SSE-NEXT: testq %r9, %r9
-; SSE-NEXT: cmovnel %r8d, %r10d
-; SSE-NEXT: movq 40(%rdi), %r11
-; SSE-NEXT: bsrq %r11, %rbx
-; SSE-NEXT: bsrq 32(%rdi), %r8
-; SSE-NEXT: xorl $63, %ebx
-; SSE-NEXT: xorl $63, %r8d
-; SSE-NEXT: orl $64, %r8d
-; SSE-NEXT: testq %r11, %r11
-; SSE-NEXT: cmovnel %ebx, %r8d
-; SSE-NEXT: subl $-128, %r8d
-; SSE-NEXT: orq %r9, %rax
-; SSE-NEXT: cmovnel %r10d, %r8d
+; SSE-NEXT: movq 8(%rdi), %r11
+; SSE-NEXT: movq 16(%rdi), %r9
+; SSE-NEXT: movq 24(%rdi), %r10
+; SSE-NEXT: movq 32(%rdi), %rcx
+; SSE-NEXT: movq 40(%rdi), %rdx
+; SSE-NEXT: movq 48(%rdi), %rsi
+; SSE-NEXT: movq 56(%rdi), %r8
+; SSE-NEXT: bsrq %r8, %rax
+; SSE-NEXT: xorl $63, %eax
+; SSE-NEXT: bsrq %rsi, %r14
+; SSE-NEXT: xorl $63, %r14d
+; SSE-NEXT: orl $64, %r14d
+; SSE-NEXT: testq %r8, %r8
+; SSE-NEXT: cmovnel %eax, %r14d
; SSE-NEXT: bsrq %rdx, %rax
; SSE-NEXT: xorl $63, %eax
-; SSE-NEXT: bsrq %rcx, %r9
-; SSE-NEXT: xorl $63, %r9d
-; SSE-NEXT: orl $64, %r9d
+; SSE-NEXT: bsrq %rcx, %rbx
+; SSE-NEXT: xorl $63, %ebx
+; SSE-NEXT: orl $64, %ebx
; SSE-NEXT: testq %rdx, %rdx
-; SSE-NEXT: cmovnel %eax, %r9d
-; SSE-NEXT: bsrq %rsi, %r10
-; SSE-NEXT: xorl $63, %r10d
+; SSE-NEXT: cmovnel %eax, %ebx
+; SSE-NEXT: subl $-128, %ebx
+; SSE-NEXT: movq %rsi, %rax
+; SSE-NEXT: orq %r8, %rax
+; SSE-NEXT: cmovnel %r14d, %ebx
+; SSE-NEXT: bsrq %r10, %rax
+; SSE-NEXT: xorl $63, %eax
+; SSE-NEXT: bsrq %r9, %r14
+; SSE-NEXT: xorl $63, %r14d
+; SSE-NEXT: orl $64, %r14d
+; SSE-NEXT: testq %r10, %r10
+; SSE-NEXT: cmovnel %eax, %r14d
+; SSE-NEXT: bsrq %r11, %r15
+; SSE-NEXT: xorl $63, %r15d
; SSE-NEXT: bsrq (%rdi), %rax
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: orl $64, %eax
-; SSE-NEXT: testq %rsi, %rsi
-; SSE-NEXT: cmovnel %r10d, %eax
+; SSE-NEXT: testq %r11, %r11
+; SSE-NEXT: cmovnel %r15d, %eax
; SSE-NEXT: subl $-128, %eax
-; SSE-NEXT: orq %rdx, %rcx
-; SSE-NEXT: cmovnel %r9d, %eax
+; SSE-NEXT: orq %r10, %r9
+; SSE-NEXT: cmovnel %r14d, %eax
; SSE-NEXT: addl $256, %eax # imm = 0x100
-; SSE-NEXT: por 48(%rdi), %xmm0
-; SSE-NEXT: ptest %xmm0, %xmm0
-; SSE-NEXT: cmovnel %r8d, %eax
+; SSE-NEXT: orq %r8, %rdx
+; SSE-NEXT: orq %rsi, %rcx
+; SSE-NEXT: orq %rdx, %rcx
+; SSE-NEXT: cmovnel %ebx, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: popq %rbx
+; SSE-NEXT: popq %r14
+; SSE-NEXT: popq %r15
; SSE-NEXT: retq
;
; AVX2-LABEL: load_ctlz_undef_i512:
; AVX2: # %bb.0:
-; AVX2-NEXT: movq 16(%rdi), %rcx
-; AVX2-NEXT: movq 24(%rdi), %rdx
-; AVX2-NEXT: movq 40(%rdi), %rax
-; AVX2-NEXT: movq 48(%rdi), %r8
-; AVX2-NEXT: movq 56(%rdi), %r9
-; AVX2-NEXT: lzcntq %r9, %rsi
-; AVX2-NEXT: lzcntq %r8, %r10
-; AVX2-NEXT: addl $64, %r10d
-; AVX2-NEXT: testq %r9, %r9
-; AVX2-NEXT: cmovnel %esi, %r10d
-; AVX2-NEXT: lzcntq %rax, %r11
-; AVX2-NEXT: xorl %esi, %esi
-; AVX2-NEXT: lzcntq 32(%rdi), %rsi
-; AVX2-NEXT: addl $64, %esi
-; AVX2-NEXT: testq %rax, %rax
-; AVX2-NEXT: cmovnel %r11d, %esi
-; AVX2-NEXT: subl $-128, %esi
-; AVX2-NEXT: orq %r9, %r8
-; AVX2-NEXT: cmovnel %r10d, %esi
+; AVX2-NEXT: pushq %r15
+; AVX2-NEXT: pushq %r14
+; AVX2-NEXT: pushq %rbx
+; AVX2-NEXT: movq 8(%rdi), %r10
+; AVX2-NEXT: movq 16(%rdi), %r9
+; AVX2-NEXT: movq 32(%rdi), %rcx
+; AVX2-NEXT: movq 40(%rdi), %rdx
+; AVX2-NEXT: movq 48(%rdi), %rsi
+; AVX2-NEXT: movq 56(%rdi), %r8
+; AVX2-NEXT: lzcntq %r8, %rax
+; AVX2-NEXT: xorl %ebx, %ebx
+; AVX2-NEXT: lzcntq %rsi, %rbx
+; AVX2-NEXT: addl $64, %ebx
+; AVX2-NEXT: testq %r8, %r8
+; AVX2-NEXT: cmovnel %eax, %ebx
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq %rdx, %rax
-; AVX2-NEXT: xorl %r8d, %r8d
-; AVX2-NEXT: lzcntq %rcx, %r8
-; AVX2-NEXT: addl $64, %r8d
+; AVX2-NEXT: lzcntq %rcx, %r11
+; AVX2-NEXT: addl $64, %r11d
; AVX2-NEXT: testq %rdx, %rdx
-; AVX2-NEXT: cmovnel %eax, %r8d
-; AVX2-NEXT: movq 8(%rdi), %r9
-; AVX2-NEXT: xorl %r10d, %r10d
-; AVX2-NEXT: lzcntq %r9, %r10
+; AVX2-NEXT: cmovnel %eax, %r11d
+; AVX2-NEXT: subl $-128, %r11d
+; AVX2-NEXT: movq %rsi, %rax
+; AVX2-NEXT: orq %r8, %rax
+; AVX2-NEXT: cmovnel %ebx, %r11d
+; AVX2-NEXT: movq 24(%rdi), %rbx
+; AVX2-NEXT: xorl %eax, %eax
+; AVX2-NEXT: lzcntq %rbx, %rax
+; AVX2-NEXT: xorl %r14d, %r14d
+; AVX2-NEXT: lzcntq %r9, %r14
+; AVX2-NEXT: addl $64, %r14d
+; AVX2-NEXT: testq %rbx, %rbx
+; AVX2-NEXT: cmovnel %eax, %r14d
+; AVX2-NEXT: xorl %r15d, %r15d
+; AVX2-NEXT: lzcntq %r10, %r15
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq (%rdi), %rax
; AVX2-NEXT: addl $64, %eax
-; AVX2-NEXT: testq %r9, %r9
-; AVX2-NEXT: cmovnel %r10d, %eax
+; AVX2-NEXT: testq %r10, %r10
+; AVX2-NEXT: cmovnel %r15d, %eax
; AVX2-NEXT: subl $-128, %eax
-; AVX2-NEXT: orq %rdx, %rcx
-; AVX2-NEXT: cmovnel %r8d, %eax
-; AVX2-NEXT: vmovdqa 32(%rdi), %xmm0
+; AVX2-NEXT: orq %rbx, %r9
+; AVX2-NEXT: cmovnel %r14d, %eax
; AVX2-NEXT: addl $256, %eax # imm = 0x100
-; AVX2-NEXT: vpor 48(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vptest %xmm0, %xmm0
-; AVX2-NEXT: cmovnel %esi, %eax
+; AVX2-NEXT: orq %r8, %rdx
+; AVX2-NEXT: orq %rsi, %rcx
+; AVX2-NEXT: orq %rdx, %rcx
+; AVX2-NEXT: cmovnel %r11d, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
+; AVX2-NEXT: popq %rbx
+; AVX2-NEXT: popq %r14
+; AVX2-NEXT: popq %r15
; AVX2-NEXT: retq
;
; AVX512F-LABEL: load_ctlz_undef_i512:
@@ -4299,354 +4719,443 @@ define i32 @vector_ctlz_undef_i512(<16 x i32> %v0) nounwind {
define i32 @test_ctlz_undef_i1024(i1024 %a0) nounwind {
; SSE-LABEL: test_ctlz_undef_i1024:
; SSE: # %bb.0:
+; SSE-NEXT: pushq %rbp
+; SSE-NEXT: pushq %r15
; SSE-NEXT: pushq %r14
+; SSE-NEXT: pushq %r13
+; SSE-NEXT: pushq %r12
; SSE-NEXT: pushq %rbx
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r10
-; SSE-NEXT: bsrq %r10, %r11
-; SSE-NEXT: xorl $63, %r11d
-; SSE-NEXT: bsrq %rax, %rbx
-; SSE-NEXT: xorl $63, %ebx
-; SSE-NEXT: orl $64, %ebx
-; SSE-NEXT: testq %r10, %r10
-; SSE-NEXT: cmovnel %r11d, %ebx
-; SSE-NEXT: bsrq %r9, %r14
-; SSE-NEXT: xorl $63, %r14d
-; SSE-NEXT: bsrq %r8, %r11
-; SSE-NEXT: xorl $63, %r11d
-; SSE-NEXT: orl $64, %r11d
-; SSE-NEXT: testq %r9, %r9
-; SSE-NEXT: cmovnel %r14d, %r11d
-; SSE-NEXT: subl $-128, %r11d
-; SSE-NEXT: movq %rax, %r14
-; SSE-NEXT: orq %r10, %r14
-; SSE-NEXT: cmovnel %ebx, %r11d
-; SSE-NEXT: bsrq %rcx, %rbx
-; SSE-NEXT: xorl $63, %ebx
-; SSE-NEXT: bsrq %rdx, %r14
-; SSE-NEXT: xorl $63, %r14d
-; SSE-NEXT: orl $64, %r14d
-; SSE-NEXT: testq %rcx, %rcx
-; SSE-NEXT: cmovnel %ebx, %r14d
-; SSE-NEXT: bsrq %rsi, %rbx
-; SSE-NEXT: xorl $63, %ebx
-; SSE-NEXT: bsrq %rdi, %rdi
-; SSE-NEXT: xorl $63, %edi
-; SSE-NEXT: orl $64, %edi
-; SSE-NEXT: testq %rsi, %rsi
-; SSE-NEXT: cmovnel %ebx, %edi
-; SSE-NEXT: subl $-128, %edi
-; SSE-NEXT: orq %rcx, %rdx
-; SSE-NEXT: cmovnel %r14d, %edi
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rcx
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rbx
-; SSE-NEXT: orq %r10, %r9
+; SSE-NEXT: movq %r9, %r12
+; SSE-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r14
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r13
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r15
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rdx
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rsi
-; SSE-NEXT: orq %rax, %r8
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; SSE-NEXT: addl $256, %edi # imm = 0x100
-; SSE-NEXT: orq %r9, %r8
-; SSE-NEXT: cmovnel %r11d, %edi
-; SSE-NEXT: addl $512, %edi # imm = 0x200
-; SSE-NEXT: bsrq %rax, %rdx
-; SSE-NEXT: xorl $63, %edx
-; SSE-NEXT: bsrq %rsi, %r10
-; SSE-NEXT: xorl $63, %r10d
-; SSE-NEXT: orl $64, %r10d
-; SSE-NEXT: testq %rax, %rax
-; SSE-NEXT: cmovnel %edx, %r10d
-; SSE-NEXT: bsrq %rbx, %r8
-; SSE-NEXT: xorl $63, %r8d
-; SSE-NEXT: bsrq {{[0-9]+}}(%rsp), %rdx
-; SSE-NEXT: xorl $63, %edx
-; SSE-NEXT: orl $64, %edx
-; SSE-NEXT: testq %rbx, %rbx
-; SSE-NEXT: cmovnel %r8d, %edx
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r8
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r9
-; SSE-NEXT: subl $-128, %edx
-; SSE-NEXT: orq %rax, %rsi
-; SSE-NEXT: cmovnel %r10d, %edx
-; SSE-NEXT: bsrq %r9, %rax
-; SSE-NEXT: xorl $63, %eax
-; SSE-NEXT: bsrq %r8, %rsi
-; SSE-NEXT: xorl $63, %esi
-; SSE-NEXT: orl $64, %esi
-; SSE-NEXT: testq %r9, %r9
-; SSE-NEXT: cmovnel %eax, %esi
-; SSE-NEXT: bsrq %rcx, %r10
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; SSE-NEXT: bsrq %r11, %rax
+; SSE-NEXT: xorl $63, %eax
+; SSE-NEXT: bsrq %rsi, %rcx
+; SSE-NEXT: xorl $63, %ecx
+; SSE-NEXT: orl $64, %ecx
+; SSE-NEXT: testq %r11, %r11
+; SSE-NEXT: cmovnel %eax, %ecx
+; SSE-NEXT: bsrq %rdx, %r10
; SSE-NEXT: xorl $63, %r10d
; SSE-NEXT: bsrq {{[0-9]+}}(%rsp), %rax
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: orl $64, %eax
-; SSE-NEXT: testq %rcx, %rcx
+; SSE-NEXT: testq %rdx, %rdx
; SSE-NEXT: cmovnel %r10d, %eax
; SSE-NEXT: subl $-128, %eax
-; SSE-NEXT: orq %r9, %r8
+; SSE-NEXT: movq %rsi, %r9
+; SSE-NEXT: movq %rsi, %rbx
+; SSE-NEXT: orq %r11, %r9
+; SSE-NEXT: cmovnel %ecx, %eax
+; SSE-NEXT: bsrq %r15, %rcx
+; SSE-NEXT: xorl $63, %ecx
+; SSE-NEXT: bsrq %r13, %rsi
+; SSE-NEXT: xorl $63, %esi
+; SSE-NEXT: orl $64, %esi
+; SSE-NEXT: testq %r15, %r15
+; SSE-NEXT: cmovnel %ecx, %esi
+; SSE-NEXT: bsrq %r14, %rcx
+; SSE-NEXT: xorl $63, %ecx
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r9
+; SSE-NEXT: bsrq %r9, %rbp
+; SSE-NEXT: xorl $63, %ebp
+; SSE-NEXT: orl $64, %ebp
+; SSE-NEXT: testq %r14, %r14
+; SSE-NEXT: cmovnel %ecx, %ebp
+; SSE-NEXT: movq %r8, %r10
+; SSE-NEXT: subl $-128, %ebp
+; SSE-NEXT: movq %r13, %rcx
+; SSE-NEXT: orq %r15, %rcx
+; SSE-NEXT: cmovnel %esi, %ebp
+; SSE-NEXT: addl $256, %ebp # imm = 0x100
+; SSE-NEXT: orq %r11, %rdx
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rsi
+; SSE-NEXT: orq %rbx, %rsi
+; SSE-NEXT: orq %rdx, %rsi
+; SSE-NEXT: cmovnel %eax, %ebp
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; SSE-NEXT: bsrq %rdx, %rcx
+; SSE-NEXT: xorl $63, %ecx
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r8
+; SSE-NEXT: bsrq %r8, %rax
+; SSE-NEXT: xorl $63, %eax
+; SSE-NEXT: orl $64, %eax
+; SSE-NEXT: testq %rdx, %rdx
+; SSE-NEXT: cmovnel %ecx, %eax
+; SSE-NEXT: bsrq %r12, %rsi
+; SSE-NEXT: xorl $63, %esi
+; SSE-NEXT: bsrq %r10, %rcx
+; SSE-NEXT: xorl $63, %ecx
+; SSE-NEXT: orl $64, %ecx
+; SSE-NEXT: testq %r12, %r12
+; SSE-NEXT: cmovnel %esi, %ecx
+; SSE-NEXT: movq %rdi, %rbx
+; SSE-NEXT: subl $-128, %ecx
+; SSE-NEXT: movq %r8, %rsi
+; SSE-NEXT: orq %rdx, %rsi
+; SSE-NEXT: cmovnel %eax, %ecx
+; SSE-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Reload
+; SSE-NEXT: bsrq %r11, %rax
+; SSE-NEXT: xorl $63, %eax
+; SSE-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
+; SSE-NEXT: bsrq %r8, %rdx
+; SSE-NEXT: xorl $63, %edx
+; SSE-NEXT: orl $64, %edx
+; SSE-NEXT: testq %r11, %r11
+; SSE-NEXT: cmovnel %eax, %edx
+; SSE-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
+; SSE-NEXT: bsrq %rdi, %rsi
+; SSE-NEXT: xorl $63, %esi
+; SSE-NEXT: bsrq %rbx, %rax
+; SSE-NEXT: xorl $63, %eax
+; SSE-NEXT: orl $64, %eax
+; SSE-NEXT: testq %rdi, %rdi
; SSE-NEXT: cmovnel %esi, %eax
-; SSE-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm0
-; SSE-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm1
-; SSE-NEXT: addl $256, %eax # imm = 0x100
-; SSE-NEXT: movdqa %xmm1, %xmm2
-; SSE-NEXT: por %xmm0, %xmm2
-; SSE-NEXT: ptest %xmm2, %xmm2
+; SSE-NEXT: subl $-128, %eax
+; SSE-NEXT: orq %r11, %r8
; SSE-NEXT: cmovnel %edx, %eax
-; SSE-NEXT: por {{[0-9]+}}(%rsp), %xmm0
-; SSE-NEXT: por {{[0-9]+}}(%rsp), %xmm1
-; SSE-NEXT: por %xmm0, %xmm1
-; SSE-NEXT: ptest %xmm1, %xmm1
-; SSE-NEXT: cmovel %edi, %eax
+; SSE-NEXT: orq {{[0-9]+}}(%rsp), %r12
+; SSE-NEXT: orq {{[0-9]+}}(%rsp), %r10
+; SSE-NEXT: addl $256, %eax # imm = 0x100
+; SSE-NEXT: orq %r12, %r10
+; SSE-NEXT: cmovnel %ecx, %eax
+; SSE-NEXT: orq {{[0-9]+}}(%rsp), %r15
+; SSE-NEXT: orq {{[0-9]+}}(%rsp), %r14
+; SSE-NEXT: orq %r15, %r14
+; SSE-NEXT: orq {{[0-9]+}}(%rsp), %r13
+; SSE-NEXT: orq {{[0-9]+}}(%rsp), %r9
+; SSE-NEXT: orq %r13, %r9
+; SSE-NEXT: addl $512, %eax # imm = 0x200
+; SSE-NEXT: orq %r14, %r9
+; SSE-NEXT: cmovnel %ebp, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: popq %rbx
+; SSE-NEXT: popq %r12
+; SSE-NEXT: popq %r13
; SSE-NEXT: popq %r14
+; SSE-NEXT: popq %r15
+; SSE-NEXT: popq %rbp
; SSE-NEXT: retq
;
; AVX2-LABEL: test_ctlz_undef_i1024:
; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %rbp
; AVX2-NEXT: pushq %r15
; AVX2-NEXT: pushq %r14
; AVX2-NEXT: pushq %r13
; AVX2-NEXT: pushq %r12
; AVX2-NEXT: pushq %rbx
+; AVX2-NEXT: movq %r9, %r14
+; AVX2-NEXT: movq %r8, %r11
+; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r15
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rbx
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r15
-; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: lzcntq %r15, %rbx
-; AVX2-NEXT: xorl %r12d, %r12d
-; AVX2-NEXT: lzcntq %r11, %r12
-; AVX2-NEXT: addl $64, %r12d
-; AVX2-NEXT: testq %r15, %r15
-; AVX2-NEXT: cmovnel %ebx, %r12d
-; AVX2-NEXT: xorl %r14d, %r14d
-; AVX2-NEXT: lzcntq {{[0-9]+}}(%rsp), %r14
-; AVX2-NEXT: xorl %r13d, %r13d
-; AVX2-NEXT: lzcntq %r10, %r13
-; AVX2-NEXT: addl $64, %r14d
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r8
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r12
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: lzcntq %r12, %rcx
+; AVX2-NEXT: xorl %r9d, %r9d
+; AVX2-NEXT: lzcntq %r8, %r9
+; AVX2-NEXT: addl $64, %r9d
+; AVX2-NEXT: testq %r12, %r12
+; AVX2-NEXT: cmovnel %ecx, %r9d
+; AVX2-NEXT: xorl %esi, %esi
+; AVX2-NEXT: lzcntq %r10, %rsi
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: lzcntq %rax, %rcx
+; AVX2-NEXT: addl $64, %ecx
; AVX2-NEXT: testq %r10, %r10
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rbx
-; AVX2-NEXT: cmovnel %r13d, %r14d
-; AVX2-NEXT: subl $-128, %r14d
-; AVX2-NEXT: orq %r15, %r11
-; AVX2-NEXT: cmovnel %r12d, %r14d
-; AVX2-NEXT: xorl %r10d, %r10d
-; AVX2-NEXT: lzcntq %rbx, %r10
-; AVX2-NEXT: xorl %r15d, %r15d
-; AVX2-NEXT: lzcntq %rax, %r15
-; AVX2-NEXT: addl $64, %r15d
-; AVX2-NEXT: testq %rbx, %rbx
-; AVX2-NEXT: cmovnel %r10d, %r15d
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
-; AVX2-NEXT: xorl %r12d, %r12d
-; AVX2-NEXT: lzcntq %r11, %r12
-; AVX2-NEXT: xorl %r10d, %r10d
-; AVX2-NEXT: lzcntq {{[0-9]+}}(%rsp), %r10
-; AVX2-NEXT: addl $64, %r10d
-; AVX2-NEXT: testq %r11, %r11
-; AVX2-NEXT: cmovnel %r12d, %r10d
-; AVX2-NEXT: vmovdqu {{[0-9]+}}(%rsp), %ymm0
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
-; AVX2-NEXT: subl $-128, %r10d
-; AVX2-NEXT: orq %rbx, %rax
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rbx
-; AVX2-NEXT: cmovnel %r15d, %r10d
-; AVX2-NEXT: addl $256, %r10d # imm = 0x100
-; AVX2-NEXT: vpor {{[0-9]+}}(%rsp), %xmm0, %xmm1
-; AVX2-NEXT: vptest %xmm1, %xmm1
-; AVX2-NEXT: cmovnel %r14d, %r10d
-; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: lzcntq %rbx, %rax
-; AVX2-NEXT: xorl %r15d, %r15d
-; AVX2-NEXT: lzcntq %r11, %r15
-; AVX2-NEXT: addl $64, %r15d
+; AVX2-NEXT: cmovnel %esi, %ecx
+; AVX2-NEXT: subl $-128, %ecx
+; AVX2-NEXT: movq %r8, %rsi
+; AVX2-NEXT: orq %r12, %rsi
+; AVX2-NEXT: cmovnel %r9d, %ecx
+; AVX2-NEXT: xorl %edi, %edi
+; AVX2-NEXT: lzcntq %rbx, %rdi
+; AVX2-NEXT: xorl %esi, %esi
+; AVX2-NEXT: lzcntq %r15, %rsi
+; AVX2-NEXT: addl $64, %esi
; AVX2-NEXT: testq %rbx, %rbx
-; AVX2-NEXT: cmovnel %eax, %r15d
-; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: lzcntq %r9, %rax
-; AVX2-NEXT: xorl %r14d, %r14d
-; AVX2-NEXT: lzcntq %r8, %r14
-; AVX2-NEXT: addl $64, %r14d
+; AVX2-NEXT: cmovnel %edi, %esi
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r13
+; AVX2-NEXT: xorl %ebp, %ebp
+; AVX2-NEXT: lzcntq %r13, %rbp
+; AVX2-NEXT: addl $64, %ebp
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r9
+; AVX2-NEXT: xorl %edi, %edi
+; AVX2-NEXT: lzcntq %r9, %rdi
; AVX2-NEXT: testq %r9, %r9
-; AVX2-NEXT: cmovnel %eax, %r14d
-; AVX2-NEXT: subl $-128, %r14d
-; AVX2-NEXT: movq %r11, %rax
-; AVX2-NEXT: orq %rbx, %rax
-; AVX2-NEXT: cmovnel %r15d, %r14d
-; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: lzcntq %rcx, %rax
-; AVX2-NEXT: xorl %r15d, %r15d
-; AVX2-NEXT: lzcntq %rdx, %r15
-; AVX2-NEXT: addl $64, %r15d
-; AVX2-NEXT: testq %rcx, %rcx
-; AVX2-NEXT: cmovnel %eax, %r15d
-; AVX2-NEXT: xorl %r12d, %r12d
-; AVX2-NEXT: lzcntq %rsi, %r12
+; AVX2-NEXT: cmovnel %edi, %ebp
+; AVX2-NEXT: subl $-128, %ebp
+; AVX2-NEXT: movq %r15, %rdi
+; AVX2-NEXT: orq %rbx, %rdi
+; AVX2-NEXT: cmovnel %esi, %ebp
+; AVX2-NEXT: addl $256, %ebp # imm = 0x100
+; AVX2-NEXT: movq %r10, %rdi
+; AVX2-NEXT: orq %r12, %rdi
+; AVX2-NEXT: movq %rax, %rsi
+; AVX2-NEXT: orq %r8, %rsi
+; AVX2-NEXT: orq %rdi, %rsi
+; AVX2-NEXT: cmovnel %ecx, %ebp
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdi
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq %rdi, %rax
; AVX2-NEXT: addl $64, %eax
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r12
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: lzcntq %r12, %rcx
+; AVX2-NEXT: testq %r12, %r12
+; AVX2-NEXT: cmovnel %ecx, %eax
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: lzcntq %r11, %rcx
+; AVX2-NEXT: addl $64, %ecx
+; AVX2-NEXT: xorl %esi, %esi
+; AVX2-NEXT: lzcntq %r14, %rsi
+; AVX2-NEXT: testq %r14, %r14
+; AVX2-NEXT: cmovnel %esi, %ecx
+; AVX2-NEXT: subl $-128, %ecx
+; AVX2-NEXT: movq %rdi, %rsi
+; AVX2-NEXT: orq %r12, %rsi
+; AVX2-NEXT: cmovnel %eax, %ecx
+; AVX2-NEXT: movq %rdx, %rdi
+; AVX2-NEXT: lzcntq %rdx, %rdx
+; AVX2-NEXT: addl $64, %edx
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Reload
+; AVX2-NEXT: xorl %eax, %eax
+; AVX2-NEXT: lzcntq %r10, %rax
+; AVX2-NEXT: testq %r10, %r10
+; AVX2-NEXT: cmovnel %eax, %edx
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
+; AVX2-NEXT: lzcntq %rax, %rax
+; AVX2-NEXT: addl $64, %eax
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
+; AVX2-NEXT: lzcntq %rsi, %r8
; AVX2-NEXT: testq %rsi, %rsi
-; AVX2-NEXT: cmovnel %r12d, %eax
+; AVX2-NEXT: cmovnel %r8d, %eax
; AVX2-NEXT: subl $-128, %eax
-; AVX2-NEXT: orq %rcx, %rdx
-; AVX2-NEXT: cmovnel %r15d, %eax
-; AVX2-NEXT: orq %rbx, %r9
-; AVX2-NEXT: orq %r11, %r8
+; AVX2-NEXT: orq %r10, %rdi
+; AVX2-NEXT: cmovnel %edx, %eax
+; AVX2-NEXT: orq %r12, %r14
+; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r11
; AVX2-NEXT: addl $256, %eax # imm = 0x100
-; AVX2-NEXT: orq %r9, %r8
-; AVX2-NEXT: cmovnel %r14d, %eax
+; AVX2-NEXT: orq %r14, %r11
+; AVX2-NEXT: cmovnel %ecx, %eax
+; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %rbx
+; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r9
+; AVX2-NEXT: orq %rbx, %r9
+; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r15
+; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r13
+; AVX2-NEXT: orq %r15, %r13
; AVX2-NEXT: addl $512, %eax # imm = 0x200
-; AVX2-NEXT: vpor {{[0-9]+}}(%rsp), %ymm0, %ymm0
-; AVX2-NEXT: vptest %ymm0, %ymm0
-; AVX2-NEXT: cmovnel %r10d, %eax
+; AVX2-NEXT: orq %r9, %r13
+; AVX2-NEXT: cmovnel %ebp, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: popq %rbx
; AVX2-NEXT: popq %r12
; AVX2-NEXT: popq %r13
; AVX2-NEXT: popq %r14
; AVX2-NEXT: popq %r15
-; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: popq %rbp
; AVX2-NEXT: retq
;
; AVX512F-LABEL: test_ctlz_undef_i1024:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
-; AVX512F-NEXT: vmovq %rdi, %xmm1
-; AVX512F-NEXT: vmovq %rsi, %xmm2
+; AVX512F-NEXT: pushq %r14
+; AVX512F-NEXT: pushq %rbx
+; AVX512F-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; AVX512F-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX512F-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX512F-NEXT: movq {{[0-9]+}}(%rsp), %r14
+; AVX512F-NEXT: vmovq %rdi, %xmm0
+; AVX512F-NEXT: vmovq %rsi, %xmm1
+; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512F-NEXT: vmovq %rdx, %xmm1
+; AVX512F-NEXT: vmovq %rcx, %xmm2
; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX512F-NEXT: vmovq %rdx, %xmm2
-; AVX512F-NEXT: vmovq %rcx, %xmm3
+; AVX512F-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512F-NEXT: vpshufd {{.*#+}} xmm1 = mem[2,3,0,1]
+; AVX512F-NEXT: vmovq %r8, %xmm2
+; AVX512F-NEXT: vmovq %r9, %xmm3
; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
-; AVX512F-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
-; AVX512F-NEXT: vpshufd {{.*#+}} xmm2 = mem[2,3,0,1]
-; AVX512F-NEXT: vmovq %r8, %xmm3
-; AVX512F-NEXT: vmovq %r9, %xmm4
-; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]
-; AVX512F-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
-; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
-; AVX512F-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448]
-; AVX512F-NEXT: vpaddq %zmm3, %zmm2, %zmm2
-; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512F-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
-; AVX512F-NEXT: vmovd %xmm1, %ecx
+; AVX512F-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512F-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512F-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,64,128,192,256,320,384,448]
+; AVX512F-NEXT: vpaddq %zmm2, %zmm1, %zmm1
+; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; AVX512F-NEXT: vmovd %xmm0, %ecx
; AVX512F-NEXT: addl $512, %ecx # imm = 0x200
-; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
-; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm1
-; AVX512F-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512F-NEXT: vpaddq %zmm3, %zmm2, %zmm2
-; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512F-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
-; AVX512F-NEXT: vmovd %xmm1, %eax
-; AVX512F-NEXT: vpor {{[0-9]+}}(%rsp), %ymm0, %ymm0
-; AVX512F-NEXT: vptest %ymm0, %ymm0
+; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]
+; AVX512F-NEXT: vpermq {{[0-9]+}}(%rsp), %zmm0, %zmm0
+; AVX512F-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512F-NEXT: vpaddq %zmm2, %zmm1, %zmm1
+; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; AVX512F-NEXT: orq {{[0-9]+}}(%rsp), %r14
+; AVX512F-NEXT: vmovd %xmm0, %eax
+; AVX512F-NEXT: orq {{[0-9]+}}(%rsp), %r11
+; AVX512F-NEXT: orq %r14, %r11
+; AVX512F-NEXT: orq {{[0-9]+}}(%rsp), %rbx
+; AVX512F-NEXT: orq {{[0-9]+}}(%rsp), %r10
+; AVX512F-NEXT: orq %rbx, %r10
+; AVX512F-NEXT: orq %r11, %r10
; AVX512F-NEXT: cmovel %ecx, %eax
+; AVX512F-NEXT: popq %rbx
+; AVX512F-NEXT: popq %r14
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: test_ctlz_undef_i1024:
; AVX512POPCNT: # %bb.0:
-; AVX512POPCNT-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
-; AVX512POPCNT-NEXT: vmovq %rdi, %xmm1
-; AVX512POPCNT-NEXT: vmovq %rsi, %xmm2
+; AVX512POPCNT-NEXT: pushq %r14
+; AVX512POPCNT-NEXT: pushq %rbx
+; AVX512POPCNT-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; AVX512POPCNT-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX512POPCNT-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX512POPCNT-NEXT: movq {{[0-9]+}}(%rsp), %r14
+; AVX512POPCNT-NEXT: vmovq %rdi, %xmm0
+; AVX512POPCNT-NEXT: vmovq %rsi, %xmm1
+; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512POPCNT-NEXT: vmovq %rdx, %xmm1
+; AVX512POPCNT-NEXT: vmovq %rcx, %xmm2
; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX512POPCNT-NEXT: vmovq %rdx, %xmm2
-; AVX512POPCNT-NEXT: vmovq %rcx, %xmm3
+; AVX512POPCNT-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512POPCNT-NEXT: vpshufd {{.*#+}} xmm1 = mem[2,3,0,1]
+; AVX512POPCNT-NEXT: vmovq %r8, %xmm2
+; AVX512POPCNT-NEXT: vmovq %r9, %xmm3
; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
-; AVX512POPCNT-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
-; AVX512POPCNT-NEXT: vpshufd {{.*#+}} xmm2 = mem[2,3,0,1]
-; AVX512POPCNT-NEXT: vmovq %r8, %xmm3
-; AVX512POPCNT-NEXT: vmovq %r9, %xmm4
-; AVX512POPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]
-; AVX512POPCNT-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
-; AVX512POPCNT-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
-; AVX512POPCNT-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448]
-; AVX512POPCNT-NEXT: vpaddq %zmm3, %zmm2, %zmm2
-; AVX512POPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512POPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
-; AVX512POPCNT-NEXT: vmovd %xmm1, %ecx
+; AVX512POPCNT-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512POPCNT-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512POPCNT-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,64,128,192,256,320,384,448]
+; AVX512POPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm1
+; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512POPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; AVX512POPCNT-NEXT: vmovd %xmm0, %ecx
; AVX512POPCNT-NEXT: addl $512, %ecx # imm = 0x200
-; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
-; AVX512POPCNT-NEXT: vpermq %zmm0, %zmm1, %zmm1
-; AVX512POPCNT-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512POPCNT-NEXT: vpaddq %zmm3, %zmm2, %zmm2
-; AVX512POPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512POPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
-; AVX512POPCNT-NEXT: vmovd %xmm1, %eax
-; AVX512POPCNT-NEXT: vpor {{[0-9]+}}(%rsp), %ymm0, %ymm0
-; AVX512POPCNT-NEXT: vptest %ymm0, %ymm0
+; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]
+; AVX512POPCNT-NEXT: vpermq {{[0-9]+}}(%rsp), %zmm0, %zmm0
+; AVX512POPCNT-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512POPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm1
+; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512POPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; AVX512POPCNT-NEXT: orq {{[0-9]+}}(%rsp), %r14
+; AVX512POPCNT-NEXT: vmovd %xmm0, %eax
+; AVX512POPCNT-NEXT: orq {{[0-9]+}}(%rsp), %r11
+; AVX512POPCNT-NEXT: orq %r14, %r11
+; AVX512POPCNT-NEXT: orq {{[0-9]+}}(%rsp), %rbx
+; AVX512POPCNT-NEXT: orq {{[0-9]+}}(%rsp), %r10
+; AVX512POPCNT-NEXT: orq %rbx, %r10
+; AVX512POPCNT-NEXT: orq %r11, %r10
; AVX512POPCNT-NEXT: cmovel %ecx, %eax
+; AVX512POPCNT-NEXT: popq %rbx
+; AVX512POPCNT-NEXT: popq %r14
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: test_ctlz_undef_i1024:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
-; AVX512VL-NEXT: vmovq %rdi, %xmm1
-; AVX512VL-NEXT: vmovq %rsi, %xmm2
+; AVX512VL-NEXT: pushq %r14
+; AVX512VL-NEXT: pushq %rbx
+; AVX512VL-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; AVX512VL-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX512VL-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX512VL-NEXT: movq {{[0-9]+}}(%rsp), %r14
+; AVX512VL-NEXT: vmovq %rdi, %xmm0
+; AVX512VL-NEXT: vmovq %rsi, %xmm1
+; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512VL-NEXT: vmovq %rdx, %xmm1
+; AVX512VL-NEXT: vmovq %rcx, %xmm2
; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX512VL-NEXT: vmovq %rdx, %xmm2
-; AVX512VL-NEXT: vmovq %rcx, %xmm3
+; AVX512VL-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512VL-NEXT: vpshufd {{.*#+}} xmm1 = mem[2,3,0,1]
+; AVX512VL-NEXT: vmovq %r8, %xmm2
+; AVX512VL-NEXT: vmovq %r9, %xmm3
; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
-; AVX512VL-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
-; AVX512VL-NEXT: vpshufd {{.*#+}} xmm2 = mem[2,3,0,1]
-; AVX512VL-NEXT: vmovq %r8, %xmm3
-; AVX512VL-NEXT: vmovq %r9, %xmm4
-; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]
-; AVX512VL-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
-; AVX512VL-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
-; AVX512VL-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448]
-; AVX512VL-NEXT: vpaddq %zmm3, %zmm2, %zmm2
-; AVX512VL-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512VL-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
-; AVX512VL-NEXT: vmovd %xmm1, %ecx
+; AVX512VL-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512VL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512VL-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,64,128,192,256,320,384,448]
+; AVX512VL-NEXT: vpaddq %zmm2, %zmm1, %zmm1
+; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512VL-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; AVX512VL-NEXT: vmovd %xmm0, %ecx
; AVX512VL-NEXT: addl $512, %ecx # imm = 0x200
-; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
-; AVX512VL-NEXT: vpermq %zmm0, %zmm1, %zmm1
-; AVX512VL-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512VL-NEXT: vpaddq %zmm3, %zmm2, %zmm2
-; AVX512VL-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512VL-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
-; AVX512VL-NEXT: vmovd %xmm1, %eax
-; AVX512VL-NEXT: vpor {{[0-9]+}}(%rsp), %ymm0, %ymm0
-; AVX512VL-NEXT: vptest %ymm0, %ymm0
+; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]
+; AVX512VL-NEXT: vpermq {{[0-9]+}}(%rsp), %zmm0, %zmm0
+; AVX512VL-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512VL-NEXT: vpaddq %zmm2, %zmm1, %zmm1
+; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512VL-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; AVX512VL-NEXT: vmovd %xmm0, %eax
+; AVX512VL-NEXT: orq {{[0-9]+}}(%rsp), %r14
+; AVX512VL-NEXT: orq {{[0-9]+}}(%rsp), %r11
+; AVX512VL-NEXT: orq {{[0-9]+}}(%rsp), %rbx
+; AVX512VL-NEXT: orq %r14, %r11
+; AVX512VL-NEXT: orq {{[0-9]+}}(%rsp), %r10
+; AVX512VL-NEXT: orq %rbx, %r10
+; AVX512VL-NEXT: orq %r11, %r10
; AVX512VL-NEXT: cmovel %ecx, %eax
+; AVX512VL-NEXT: popq %rbx
+; AVX512VL-NEXT: popq %r14
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: test_ctlz_undef_i1024:
; AVX512VLPOPCNT: # %bb.0:
-; AVX512VLPOPCNT-NEXT: vmovdqu64 {{[0-9]+}}(%rsp), %zmm0
-; AVX512VLPOPCNT-NEXT: vmovq %rdi, %xmm1
-; AVX512VLPOPCNT-NEXT: vmovq %rsi, %xmm2
+; AVX512VLPOPCNT-NEXT: pushq %r14
+; AVX512VLPOPCNT-NEXT: pushq %rbx
+; AVX512VLPOPCNT-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; AVX512VLPOPCNT-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX512VLPOPCNT-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX512VLPOPCNT-NEXT: movq {{[0-9]+}}(%rsp), %r14
+; AVX512VLPOPCNT-NEXT: vmovq %rdi, %xmm0
+; AVX512VLPOPCNT-NEXT: vmovq %rsi, %xmm1
+; AVX512VLPOPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512VLPOPCNT-NEXT: vmovq %rdx, %xmm1
+; AVX512VLPOPCNT-NEXT: vmovq %rcx, %xmm2
; AVX512VLPOPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX512VLPOPCNT-NEXT: vmovq %rdx, %xmm2
-; AVX512VLPOPCNT-NEXT: vmovq %rcx, %xmm3
+; AVX512VLPOPCNT-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512VLPOPCNT-NEXT: vpshufd {{.*#+}} xmm1 = mem[2,3,0,1]
+; AVX512VLPOPCNT-NEXT: vmovq %r8, %xmm2
+; AVX512VLPOPCNT-NEXT: vmovq %r9, %xmm3
; AVX512VLPOPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
-; AVX512VLPOPCNT-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
-; AVX512VLPOPCNT-NEXT: vpshufd {{.*#+}} xmm2 = mem[2,3,0,1]
-; AVX512VLPOPCNT-NEXT: vmovq %r8, %xmm3
-; AVX512VLPOPCNT-NEXT: vmovq %r9, %xmm4
-; AVX512VLPOPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm3 = xmm4[0],xmm3[0]
-; AVX512VLPOPCNT-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
-; AVX512VLPOPCNT-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
-; AVX512VLPOPCNT-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448]
-; AVX512VLPOPCNT-NEXT: vpaddq %zmm3, %zmm2, %zmm2
-; AVX512VLPOPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512VLPOPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
-; AVX512VLPOPCNT-NEXT: vmovd %xmm1, %ecx
+; AVX512VLPOPCNT-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512VLPOPCNT-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512VLPOPCNT-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm2 = [0,64,128,192,256,320,384,448]
+; AVX512VLPOPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm1
+; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512VLPOPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %ecx
; AVX512VLPOPCNT-NEXT: addl $512, %ecx # imm = 0x200
-; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
-; AVX512VLPOPCNT-NEXT: vpermq %zmm0, %zmm1, %zmm1
-; AVX512VLPOPCNT-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512VLPOPCNT-NEXT: vpaddq %zmm3, %zmm2, %zmm2
-; AVX512VLPOPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512VLPOPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
-; AVX512VLPOPCNT-NEXT: vmovd %xmm1, %eax
-; AVX512VLPOPCNT-NEXT: vpor {{[0-9]+}}(%rsp), %ymm0, %ymm0
-; AVX512VLPOPCNT-NEXT: vptest %ymm0, %ymm0
+; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]
+; AVX512VLPOPCNT-NEXT: vpermq {{[0-9]+}}(%rsp), %zmm0, %zmm0
+; AVX512VLPOPCNT-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512VLPOPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm1
+; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512VLPOPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
+; AVX512VLPOPCNT-NEXT: orq {{[0-9]+}}(%rsp), %r14
+; AVX512VLPOPCNT-NEXT: orq {{[0-9]+}}(%rsp), %r11
+; AVX512VLPOPCNT-NEXT: orq {{[0-9]+}}(%rsp), %rbx
+; AVX512VLPOPCNT-NEXT: orq %r14, %r11
+; AVX512VLPOPCNT-NEXT: orq {{[0-9]+}}(%rsp), %r10
+; AVX512VLPOPCNT-NEXT: orq %rbx, %r10
+; AVX512VLPOPCNT-NEXT: orq %r11, %r10
; AVX512VLPOPCNT-NEXT: cmovel %ecx, %eax
+; AVX512VLPOPCNT-NEXT: popq %rbx
+; AVX512VLPOPCNT-NEXT: popq %r14
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%cnt = call i1024 @llvm.ctlz.i1024(i1024 %a0, i1 -1)
@@ -4657,298 +5166,386 @@ define i32 @test_ctlz_undef_i1024(i1024 %a0) nounwind {
define i32 @load_ctlz_undef_i1024(ptr %p0) nounwind {
; SSE-LABEL: load_ctlz_undef_i1024:
; SSE: # %bb.0:
-; SSE-NEXT: movq 8(%rdi), %rcx
-; SSE-NEXT: movq 72(%rdi), %rax
-; SSE-NEXT: movq 104(%rdi), %r8
-; SSE-NEXT: movq 112(%rdi), %rdx
-; SSE-NEXT: movq 120(%rdi), %r9
-; SSE-NEXT: bsrq %r9, %rsi
-; SSE-NEXT: xorl $63, %esi
-; SSE-NEXT: bsrq %rdx, %r11
-; SSE-NEXT: xorl $63, %r11d
-; SSE-NEXT: orl $64, %r11d
-; SSE-NEXT: testq %r9, %r9
-; SSE-NEXT: cmovnel %esi, %r11d
-; SSE-NEXT: bsrq %r8, %r10
-; SSE-NEXT: xorl $63, %r10d
-; SSE-NEXT: bsrq 96(%rdi), %rsi
-; SSE-NEXT: xorl $63, %esi
-; SSE-NEXT: orl $64, %esi
+; SSE-NEXT: pushq %rbp
+; SSE-NEXT: pushq %r15
+; SSE-NEXT: pushq %r14
+; SSE-NEXT: pushq %r13
+; SSE-NEXT: pushq %r12
+; SSE-NEXT: pushq %rbx
+; SSE-NEXT: movq 40(%rdi), %rbp
+; SSE-NEXT: movq 64(%rdi), %rbx
+; SSE-NEXT: movq 72(%rdi), %r11
+; SSE-NEXT: movq 80(%rdi), %r12
+; SSE-NEXT: movq 88(%rdi), %r14
+; SSE-NEXT: movq 96(%rdi), %r13
+; SSE-NEXT: movq 104(%rdi), %r9
+; SSE-NEXT: movq 112(%rdi), %r10
+; SSE-NEXT: movq 120(%rdi), %r8
+; SSE-NEXT: bsrq %r8, %rax
+; SSE-NEXT: xorl $63, %eax
+; SSE-NEXT: bsrq %r10, %rcx
+; SSE-NEXT: xorl $63, %ecx
+; SSE-NEXT: orl $64, %ecx
; SSE-NEXT: testq %r8, %r8
-; SSE-NEXT: movq 80(%rdi), %r8
-; SSE-NEXT: cmovnel %r10d, %esi
-; SSE-NEXT: movq 88(%rdi), %r10
-; SSE-NEXT: subl $-128, %esi
-; SSE-NEXT: orq %r9, %rdx
-; SSE-NEXT: cmovnel %r11d, %esi
-; SSE-NEXT: bsrq %r10, %rdx
+; SSE-NEXT: cmovnel %eax, %ecx
+; SSE-NEXT: bsrq %r9, %rdx
; SSE-NEXT: xorl $63, %edx
-; SSE-NEXT: bsrq %r8, %r11
-; SSE-NEXT: xorl $63, %r11d
-; SSE-NEXT: orl $64, %r11d
-; SSE-NEXT: testq %r10, %r10
-; SSE-NEXT: cmovnel %edx, %r11d
-; SSE-NEXT: bsrq %rax, %r9
-; SSE-NEXT: xorl $63, %r9d
-; SSE-NEXT: bsrq 64(%rdi), %rdx
+; SSE-NEXT: bsrq %r13, %rax
+; SSE-NEXT: movq %r13, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE-NEXT: xorl $63, %eax
+; SSE-NEXT: orl $64, %eax
+; SSE-NEXT: testq %r9, %r9
+; SSE-NEXT: movq %r9, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE-NEXT: cmovnel %edx, %eax
+; SSE-NEXT: subl $-128, %eax
+; SSE-NEXT: movq %r10, %rdx
+; SSE-NEXT: movq %r10, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE-NEXT: orq %r8, %rdx
+; SSE-NEXT: movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE-NEXT: cmovnel %ecx, %eax
+; SSE-NEXT: bsrq %r14, %rcx
+; SSE-NEXT: xorl $63, %ecx
+; SSE-NEXT: movq %r12, %rsi
+; SSE-NEXT: movq %r12, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE-NEXT: bsrq %r12, %rdx
; SSE-NEXT: xorl $63, %edx
; SSE-NEXT: orl $64, %edx
-; SSE-NEXT: testq %rax, %rax
-; SSE-NEXT: cmovnel %r9d, %edx
-; SSE-NEXT: movq 40(%rdi), %r9
-; SSE-NEXT: movq 48(%rdi), %rax
-; SSE-NEXT: subl $-128, %edx
-; SSE-NEXT: orq %r10, %r8
-; SSE-NEXT: movq 56(%rdi), %r8
-; SSE-NEXT: movdqa 112(%rdi), %xmm0
-; SSE-NEXT: cmovnel %r11d, %edx
-; SSE-NEXT: movdqa 96(%rdi), %xmm1
-; SSE-NEXT: addl $256, %edx # imm = 0x100
-; SSE-NEXT: movdqa %xmm1, %xmm2
-; SSE-NEXT: por %xmm0, %xmm2
-; SSE-NEXT: ptest %xmm2, %xmm2
-; SSE-NEXT: cmovnel %esi, %edx
-; SSE-NEXT: bsrq %r8, %rsi
-; SSE-NEXT: xorl $63, %esi
-; SSE-NEXT: bsrq %rax, %r11
-; SSE-NEXT: xorl $63, %r11d
-; SSE-NEXT: orl $64, %r11d
-; SSE-NEXT: testq %r8, %r8
-; SSE-NEXT: cmovnel %esi, %r11d
-; SSE-NEXT: bsrq %r9, %r10
-; SSE-NEXT: xorl $63, %r10d
-; SSE-NEXT: bsrq 32(%rdi), %rsi
-; SSE-NEXT: xorl $63, %esi
-; SSE-NEXT: orl $64, %esi
-; SSE-NEXT: testq %r9, %r9
-; SSE-NEXT: movq 16(%rdi), %r9
-; SSE-NEXT: cmovnel %r10d, %esi
-; SSE-NEXT: movq 24(%rdi), %r10
-; SSE-NEXT: subl $-128, %esi
-; SSE-NEXT: orq %r8, %rax
-; SSE-NEXT: cmovnel %r11d, %esi
-; SSE-NEXT: bsrq %r10, %rax
+; SSE-NEXT: testq %r14, %r14
+; SSE-NEXT: cmovnel %ecx, %edx
+; SSE-NEXT: bsrq %r11, %rcx
+; SSE-NEXT: xorl $63, %ecx
+; SSE-NEXT: bsrq %rbx, %r15
+; SSE-NEXT: xorl $63, %r15d
+; SSE-NEXT: orl $64, %r15d
+; SSE-NEXT: testq %r11, %r11
+; SSE-NEXT: cmovnel %ecx, %r15d
+; SSE-NEXT: movq 48(%rdi), %r12
+; SSE-NEXT: subl $-128, %r15d
+; SSE-NEXT: movq %rsi, %rcx
+; SSE-NEXT: orq %r14, %rcx
+; SSE-NEXT: cmovnel %edx, %r15d
+; SSE-NEXT: addl $256, %r15d # imm = 0x100
+; SSE-NEXT: movq %r9, %rcx
+; SSE-NEXT: orq %r8, %rcx
+; SSE-NEXT: movq %r13, %rdx
+; SSE-NEXT: orq %r10, %rdx
+; SSE-NEXT: orq %rcx, %rdx
+; SSE-NEXT: movq 56(%rdi), %r13
+; SSE-NEXT: cmovnel %eax, %r15d
+; SSE-NEXT: bsrq %r13, %rax
; SSE-NEXT: xorl $63, %eax
-; SSE-NEXT: bsrq %r9, %r8
-; SSE-NEXT: xorl $63, %r8d
-; SSE-NEXT: orl $64, %r8d
+; SSE-NEXT: bsrq %r12, %rdx
+; SSE-NEXT: xorl $63, %edx
+; SSE-NEXT: orl $64, %edx
+; SSE-NEXT: testq %r13, %r13
+; SSE-NEXT: cmovnel %eax, %edx
+; SSE-NEXT: movq %rbp, %r10
+; SSE-NEXT: bsrq %rbp, %rax
+; SSE-NEXT: xorl $63, %eax
+; SSE-NEXT: movq 32(%rdi), %r8
+; SSE-NEXT: bsrq %r8, %rbp
+; SSE-NEXT: xorl $63, %ebp
+; SSE-NEXT: orl $64, %ebp
; SSE-NEXT: testq %r10, %r10
-; SSE-NEXT: cmovnel %eax, %r8d
-; SSE-NEXT: bsrq %rcx, %r11
-; SSE-NEXT: xorl $63, %r11d
+; SSE-NEXT: cmovnel %eax, %ebp
+; SSE-NEXT: subl $-128, %ebp
+; SSE-NEXT: movq %r12, %rax
+; SSE-NEXT: orq %r13, %rax
+; SSE-NEXT: cmovnel %edx, %ebp
+; SSE-NEXT: movq 24(%rdi), %r9
+; SSE-NEXT: bsrq %r9, %rax
+; SSE-NEXT: xorl $63, %eax
+; SSE-NEXT: movq 16(%rdi), %rsi
+; SSE-NEXT: bsrq %rsi, %rcx
+; SSE-NEXT: xorl $63, %ecx
+; SSE-NEXT: orl $64, %ecx
+; SSE-NEXT: testq %r9, %r9
+; SSE-NEXT: cmovnel %eax, %ecx
+; SSE-NEXT: movq 8(%rdi), %rdx
; SSE-NEXT: bsrq (%rdi), %rax
+; SSE-NEXT: bsrq %rdx, %rdi
+; SSE-NEXT: xorl $63, %edi
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: orl $64, %eax
-; SSE-NEXT: testq %rcx, %rcx
-; SSE-NEXT: cmovnel %r11d, %eax
+; SSE-NEXT: testq %rdx, %rdx
+; SSE-NEXT: cmovnel %edi, %eax
; SSE-NEXT: subl $-128, %eax
-; SSE-NEXT: orq %r10, %r9
-; SSE-NEXT: cmovnel %r8d, %eax
-; SSE-NEXT: movdqa 32(%rdi), %xmm2
+; SSE-NEXT: orq %r9, %rsi
+; SSE-NEXT: cmovnel %ecx, %eax
+; SSE-NEXT: orq %r13, %r10
+; SSE-NEXT: orq %r12, %r8
; SSE-NEXT: addl $256, %eax # imm = 0x100
-; SSE-NEXT: por 48(%rdi), %xmm2
-; SSE-NEXT: ptest %xmm2, %xmm2
-; SSE-NEXT: cmovnel %esi, %eax
-; SSE-NEXT: por 80(%rdi), %xmm0
-; SSE-NEXT: por 64(%rdi), %xmm1
+; SSE-NEXT: orq %r10, %r8
+; SSE-NEXT: cmovnel %ebp, %eax
+; SSE-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r14 # 8-byte Folded Reload
+; SSE-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Folded Reload
+; SSE-NEXT: orq %r14, %r11
+; SSE-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; SSE-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Folded Reload
+; SSE-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Folded Reload
+; SSE-NEXT: orq %rcx, %rbx
; SSE-NEXT: addl $512, %eax # imm = 0x200
-; SSE-NEXT: por %xmm0, %xmm1
-; SSE-NEXT: ptest %xmm1, %xmm1
-; SSE-NEXT: cmovnel %edx, %eax
+; SSE-NEXT: orq %r11, %rbx
+; SSE-NEXT: cmovnel %r15d, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
+; SSE-NEXT: popq %rbx
+; SSE-NEXT: popq %r12
+; SSE-NEXT: popq %r13
+; SSE-NEXT: popq %r14
+; SSE-NEXT: popq %r15
+; SSE-NEXT: popq %rbp
; SSE-NEXT: retq
;
; AVX2-LABEL: load_ctlz_undef_i1024:
; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %rbp
+; AVX2-NEXT: pushq %r15
+; AVX2-NEXT: pushq %r14
+; AVX2-NEXT: pushq %r13
+; AVX2-NEXT: pushq %r12
; AVX2-NEXT: pushq %rbx
-; AVX2-NEXT: movq 16(%rdi), %rcx
-; AVX2-NEXT: movq 72(%rdi), %rsi
-; AVX2-NEXT: movq 104(%rdi), %rdx
-; AVX2-NEXT: movq 112(%rdi), %r8
-; AVX2-NEXT: movq 120(%rdi), %r10
-; AVX2-NEXT: lzcntq %r10, %rax
-; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: lzcntq %r8, %rbx
-; AVX2-NEXT: addl $64, %ebx
-; AVX2-NEXT: testq %r10, %r10
-; AVX2-NEXT: cmovnel %eax, %ebx
-; AVX2-NEXT: lzcntq %rdx, %r11
+; AVX2-NEXT: movq 48(%rdi), %r9
+; AVX2-NEXT: movq 56(%rdi), %rbp
+; AVX2-NEXT: movq 64(%rdi), %r11
+; AVX2-NEXT: movq 72(%rdi), %r10
+; AVX2-NEXT: movq 80(%rdi), %r14
+; AVX2-NEXT: movq 88(%rdi), %rbx
+; AVX2-NEXT: movq 96(%rdi), %rdx
+; AVX2-NEXT: movq 104(%rdi), %r8
+; AVX2-NEXT: movq 112(%rdi), %rsi
+; AVX2-NEXT: movq 120(%rdi), %r15
+; AVX2-NEXT: lzcntq %r15, %rax
+; AVX2-NEXT: lzcntq %rsi, %rcx
+; AVX2-NEXT: addl $64, %ecx
+; AVX2-NEXT: testq %r15, %r15
+; AVX2-NEXT: cmovnel %eax, %ecx
+; AVX2-NEXT: xorl %r12d, %r12d
+; AVX2-NEXT: lzcntq %r8, %r12
; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: lzcntq 96(%rdi), %rax
+; AVX2-NEXT: lzcntq %rdx, %rax
+; AVX2-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: addl $64, %eax
-; AVX2-NEXT: testq %rdx, %rdx
-; AVX2-NEXT: movq 80(%rdi), %r9
-; AVX2-NEXT: cmovnel %r11d, %eax
-; AVX2-NEXT: movq 88(%rdi), %r11
+; AVX2-NEXT: testq %r8, %r8
+; AVX2-NEXT: movq %r8, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: cmovnel %r12d, %eax
; AVX2-NEXT: subl $-128, %eax
-; AVX2-NEXT: orq %r10, %r8
-; AVX2-NEXT: cmovnel %ebx, %eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: lzcntq %r11, %rdx
-; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: lzcntq %r9, %rbx
-; AVX2-NEXT: addl $64, %ebx
-; AVX2-NEXT: testq %r11, %r11
-; AVX2-NEXT: cmovnel %edx, %ebx
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: lzcntq 64(%rdi), %rdx
-; AVX2-NEXT: xorl %r8d, %r8d
-; AVX2-NEXT: lzcntq %rsi, %r8
-; AVX2-NEXT: addl $64, %edx
-; AVX2-NEXT: testq %rsi, %rsi
-; AVX2-NEXT: movq 40(%rdi), %r10
-; AVX2-NEXT: cmovnel %r8d, %edx
-; AVX2-NEXT: movq 48(%rdi), %r8
-; AVX2-NEXT: subl $-128, %edx
-; AVX2-NEXT: orq %r11, %r9
-; AVX2-NEXT: movq 56(%rdi), %r9
-; AVX2-NEXT: cmovnel %ebx, %edx
-; AVX2-NEXT: vmovdqu 96(%rdi), %ymm0
-; AVX2-NEXT: addl $256, %edx # imm = 0x100
-; AVX2-NEXT: vpor 112(%rdi), %xmm0, %xmm1
-; AVX2-NEXT: vptest %xmm1, %xmm1
-; AVX2-NEXT: cmovnel %eax, %edx
+; AVX2-NEXT: movq %rsi, %r12
+; AVX2-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: orq %r15, %r12
+; AVX2-NEXT: cmovnel %ecx, %eax
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: lzcntq %rbx, %rcx
+; AVX2-NEXT: movq %r14, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: xorl %r13d, %r13d
+; AVX2-NEXT: lzcntq %r14, %r13
+; AVX2-NEXT: addl $64, %r13d
+; AVX2-NEXT: testq %rbx, %rbx
+; AVX2-NEXT: cmovnel %ecx, %r13d
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: lzcntq %r10, %rcx
+; AVX2-NEXT: xorl %r12d, %r12d
+; AVX2-NEXT: lzcntq %r11, %r12
+; AVX2-NEXT: addl $64, %r12d
+; AVX2-NEXT: testq %r10, %r10
+; AVX2-NEXT: cmovnel %ecx, %r12d
+; AVX2-NEXT: subl $-128, %r12d
+; AVX2-NEXT: movq %r14, %rcx
+; AVX2-NEXT: orq %rbx, %rcx
+; AVX2-NEXT: cmovnel %r13d, %r12d
+; AVX2-NEXT: addl $256, %r12d # imm = 0x100
+; AVX2-NEXT: movq %r8, %rcx
+; AVX2-NEXT: orq %r15, %rcx
+; AVX2-NEXT: orq %rsi, %rdx
+; AVX2-NEXT: orq %rcx, %rdx
+; AVX2-NEXT: cmovnel %eax, %r12d
+; AVX2-NEXT: movq %rbp, %r14
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: lzcntq %rbp, %rcx
+; AVX2-NEXT: movq %r9, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq %r9, %rax
-; AVX2-NEXT: xorl %r11d, %r11d
-; AVX2-NEXT: lzcntq %r8, %r11
-; AVX2-NEXT: addl $64, %r11d
-; AVX2-NEXT: testq %r9, %r9
-; AVX2-NEXT: cmovnel %eax, %r11d
-; AVX2-NEXT: xorl %esi, %esi
-; AVX2-NEXT: lzcntq 32(%rdi), %rsi
-; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: lzcntq %r10, %rax
-; AVX2-NEXT: addl $64, %esi
-; AVX2-NEXT: testq %r10, %r10
-; AVX2-NEXT: movq 24(%rdi), %r10
-; AVX2-NEXT: cmovnel %eax, %esi
-; AVX2-NEXT: subl $-128, %esi
-; AVX2-NEXT: orq %r9, %r8
-; AVX2-NEXT: cmovnel %r11d, %esi
+; AVX2-NEXT: addl $64, %eax
+; AVX2-NEXT: testq %rbp, %rbp
+; AVX2-NEXT: cmovnel %ecx, %eax
+; AVX2-NEXT: movq 32(%rdi), %r13
+; AVX2-NEXT: xorl %ebp, %ebp
+; AVX2-NEXT: lzcntq %r13, %rbp
+; AVX2-NEXT: addl $64, %ebp
+; AVX2-NEXT: movq 40(%rdi), %r8
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: lzcntq %r8, %rdx
+; AVX2-NEXT: testq %r8, %r8
+; AVX2-NEXT: cmovnel %edx, %ebp
+; AVX2-NEXT: subl $-128, %ebp
+; AVX2-NEXT: movq %r9, %rdx
+; AVX2-NEXT: orq %r14, %rdx
+; AVX2-NEXT: cmovnel %eax, %ebp
+; AVX2-NEXT: movq 16(%rdi), %r9
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: lzcntq %r9, %rcx
+; AVX2-NEXT: addl $64, %ecx
+; AVX2-NEXT: movq 24(%rdi), %rdx
; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: lzcntq %r10, %rax
-; AVX2-NEXT: xorl %r8d, %r8d
-; AVX2-NEXT: lzcntq %rcx, %r8
-; AVX2-NEXT: addl $64, %r8d
-; AVX2-NEXT: testq %r10, %r10
-; AVX2-NEXT: cmovnel %eax, %r8d
-; AVX2-NEXT: movq 8(%rdi), %r9
-; AVX2-NEXT: xorl %r11d, %r11d
-; AVX2-NEXT: lzcntq %r9, %r11
+; AVX2-NEXT: lzcntq %rdx, %rax
+; AVX2-NEXT: testq %rdx, %rdx
+; AVX2-NEXT: cmovnel %eax, %ecx
+; AVX2-NEXT: movq 8(%rdi), %rsi
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: lzcntq (%rdi), %rax
; AVX2-NEXT: addl $64, %eax
-; AVX2-NEXT: testq %r9, %r9
-; AVX2-NEXT: cmovnel %r11d, %eax
+; AVX2-NEXT: lzcntq %rsi, %rdi
+; AVX2-NEXT: testq %rsi, %rsi
+; AVX2-NEXT: cmovnel %edi, %eax
; AVX2-NEXT: subl $-128, %eax
-; AVX2-NEXT: orq %r10, %rcx
-; AVX2-NEXT: cmovnel %r8d, %eax
-; AVX2-NEXT: vmovdqa 32(%rdi), %xmm1
+; AVX2-NEXT: orq %rdx, %r9
+; AVX2-NEXT: cmovnel %ecx, %eax
+; AVX2-NEXT: orq %r14, %r8
+; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r13 # 8-byte Folded Reload
; AVX2-NEXT: addl $256, %eax # imm = 0x100
-; AVX2-NEXT: vpor 48(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vptest %xmm1, %xmm1
-; AVX2-NEXT: cmovnel %esi, %eax
-; AVX2-NEXT: vpor 64(%rdi), %ymm0, %ymm0
+; AVX2-NEXT: orq %r8, %r13
+; AVX2-NEXT: cmovnel %ebp, %eax
+; AVX2-NEXT: orq %r15, %rbx
+; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Folded Reload
+; AVX2-NEXT: orq %rbx, %r10
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Folded Reload
+; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Folded Reload
+; AVX2-NEXT: orq %rcx, %r11
; AVX2-NEXT: addl $512, %eax # imm = 0x200
-; AVX2-NEXT: vptest %ymm0, %ymm0
-; AVX2-NEXT: cmovnel %edx, %eax
+; AVX2-NEXT: orq %r10, %r11
+; AVX2-NEXT: cmovnel %r12d, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: popq %rbx
-; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: popq %r12
+; AVX2-NEXT: popq %r13
+; AVX2-NEXT: popq %r14
+; AVX2-NEXT: popq %r15
+; AVX2-NEXT: popq %rbp
; AVX2-NEXT: retq
;
; AVX512F-LABEL: load_ctlz_undef_i1024:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vmovdqu64 64(%rdi), %zmm0
-; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
-; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm2
-; AVX512F-NEXT: vplzcntq %zmm2, %zmm3
-; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm4 = [0,64,128,192,256,320,384,448]
-; AVX512F-NEXT: vpaddq %zmm4, %zmm3, %zmm3
-; AVX512F-NEXT: vptestmq %zmm2, %zmm2, %k1
-; AVX512F-NEXT: vpcompressq %zmm3, %zmm2 {%k1} {z}
-; AVX512F-NEXT: vmovd %xmm2, %ecx
-; AVX512F-NEXT: vpermq (%rdi), %zmm1, %zmm1
+; AVX512F-NEXT: movq 80(%rdi), %rsi
+; AVX512F-NEXT: movq 64(%rdi), %rcx
+; AVX512F-NEXT: movq 72(%rdi), %rdx
+; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]
+; AVX512F-NEXT: vpermq 64(%rdi), %zmm0, %zmm1
+; AVX512F-NEXT: movq 88(%rdi), %r8
; AVX512F-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512F-NEXT: vpaddq %zmm4, %zmm2, %zmm2
+; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448]
+; AVX512F-NEXT: vpaddq %zmm3, %zmm2, %zmm2
; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512F-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
-; AVX512F-NEXT: vmovd %xmm1, %eax
+; AVX512F-NEXT: vpermq (%rdi), %zmm0, %zmm0
+; AVX512F-NEXT: vmovd %xmm1, %r9d
+; AVX512F-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512F-NEXT: vpaddq %zmm3, %zmm1, %zmm1
+; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; AVX512F-NEXT: vmovd %xmm0, %eax
+; AVX512F-NEXT: orq 120(%rdi), %r8
; AVX512F-NEXT: addl $512, %eax # imm = 0x200
-; AVX512F-NEXT: vpor 96(%rdi), %ymm0, %ymm0
-; AVX512F-NEXT: vptest %ymm0, %ymm0
-; AVX512F-NEXT: cmovnel %ecx, %eax
+; AVX512F-NEXT: orq 104(%rdi), %rdx
+; AVX512F-NEXT: orq %r8, %rdx
+; AVX512F-NEXT: orq 112(%rdi), %rsi
+; AVX512F-NEXT: orq 96(%rdi), %rcx
+; AVX512F-NEXT: orq %rsi, %rcx
+; AVX512F-NEXT: orq %rdx, %rcx
+; AVX512F-NEXT: cmovnel %r9d, %eax
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: load_ctlz_undef_i1024:
; AVX512POPCNT: # %bb.0:
-; AVX512POPCNT-NEXT: vmovdqu64 64(%rdi), %zmm0
-; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
-; AVX512POPCNT-NEXT: vpermq %zmm0, %zmm1, %zmm2
-; AVX512POPCNT-NEXT: vplzcntq %zmm2, %zmm3
-; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm4 = [0,64,128,192,256,320,384,448]
-; AVX512POPCNT-NEXT: vpaddq %zmm4, %zmm3, %zmm3
-; AVX512POPCNT-NEXT: vptestmq %zmm2, %zmm2, %k1
-; AVX512POPCNT-NEXT: vpcompressq %zmm3, %zmm2 {%k1} {z}
-; AVX512POPCNT-NEXT: vmovd %xmm2, %ecx
-; AVX512POPCNT-NEXT: vpermq (%rdi), %zmm1, %zmm1
+; AVX512POPCNT-NEXT: movq 80(%rdi), %rsi
+; AVX512POPCNT-NEXT: movq 64(%rdi), %rcx
+; AVX512POPCNT-NEXT: movq 72(%rdi), %rdx
+; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]
+; AVX512POPCNT-NEXT: vpermq 64(%rdi), %zmm0, %zmm1
+; AVX512POPCNT-NEXT: movq 88(%rdi), %r8
; AVX512POPCNT-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512POPCNT-NEXT: vpaddq %zmm4, %zmm2, %zmm2
+; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448]
+; AVX512POPCNT-NEXT: vpaddq %zmm3, %zmm2, %zmm2
; AVX512POPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512POPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
-; AVX512POPCNT-NEXT: vmovd %xmm1, %eax
+; AVX512POPCNT-NEXT: vpermq (%rdi), %zmm0, %zmm0
+; AVX512POPCNT-NEXT: vmovd %xmm1, %r9d
+; AVX512POPCNT-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512POPCNT-NEXT: vpaddq %zmm3, %zmm1, %zmm1
+; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512POPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; AVX512POPCNT-NEXT: vmovd %xmm0, %eax
+; AVX512POPCNT-NEXT: orq 120(%rdi), %r8
; AVX512POPCNT-NEXT: addl $512, %eax # imm = 0x200
-; AVX512POPCNT-NEXT: vpor 96(%rdi), %ymm0, %ymm0
-; AVX512POPCNT-NEXT: vptest %ymm0, %ymm0
-; AVX512POPCNT-NEXT: cmovnel %ecx, %eax
+; AVX512POPCNT-NEXT: orq 104(%rdi), %rdx
+; AVX512POPCNT-NEXT: orq %r8, %rdx
+; AVX512POPCNT-NEXT: orq 112(%rdi), %rsi
+; AVX512POPCNT-NEXT: orq 96(%rdi), %rcx
+; AVX512POPCNT-NEXT: orq %rsi, %rcx
+; AVX512POPCNT-NEXT: orq %rdx, %rcx
+; AVX512POPCNT-NEXT: cmovnel %r9d, %eax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: load_ctlz_undef_i1024:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vmovdqu64 64(%rdi), %zmm0
-; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
-; AVX512VL-NEXT: vpermq %zmm0, %zmm1, %zmm2
-; AVX512VL-NEXT: vplzcntq %zmm2, %zmm3
-; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm4 = [0,64,128,192,256,320,384,448]
-; AVX512VL-NEXT: vpaddq %zmm4, %zmm3, %zmm3
-; AVX512VL-NEXT: vptestmq %zmm2, %zmm2, %k1
-; AVX512VL-NEXT: vpcompressq %zmm3, %zmm2 {%k1} {z}
-; AVX512VL-NEXT: vmovd %xmm2, %ecx
-; AVX512VL-NEXT: vpermq (%rdi), %zmm1, %zmm1
+; AVX512VL-NEXT: movq 80(%rdi), %rsi
+; AVX512VL-NEXT: movq 64(%rdi), %rcx
+; AVX512VL-NEXT: movq 72(%rdi), %rdx
+; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]
+; AVX512VL-NEXT: vpermq 64(%rdi), %zmm0, %zmm1
+; AVX512VL-NEXT: movq 88(%rdi), %r8
; AVX512VL-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512VL-NEXT: vpaddq %zmm4, %zmm2, %zmm2
+; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448]
+; AVX512VL-NEXT: vpaddq %zmm3, %zmm2, %zmm2
; AVX512VL-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512VL-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
-; AVX512VL-NEXT: vmovd %xmm1, %eax
+; AVX512VL-NEXT: vmovd %xmm1, %r9d
+; AVX512VL-NEXT: vpermq (%rdi), %zmm0, %zmm0
+; AVX512VL-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512VL-NEXT: vpaddq %zmm3, %zmm1, %zmm1
+; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512VL-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; AVX512VL-NEXT: vmovd %xmm0, %eax
; AVX512VL-NEXT: addl $512, %eax # imm = 0x200
-; AVX512VL-NEXT: vpor 96(%rdi), %ymm0, %ymm0
-; AVX512VL-NEXT: vptest %ymm0, %ymm0
-; AVX512VL-NEXT: cmovnel %ecx, %eax
+; AVX512VL-NEXT: orq 120(%rdi), %r8
+; AVX512VL-NEXT: orq 104(%rdi), %rdx
+; AVX512VL-NEXT: orq 112(%rdi), %rsi
+; AVX512VL-NEXT: orq %r8, %rdx
+; AVX512VL-NEXT: orq 96(%rdi), %rcx
+; AVX512VL-NEXT: orq %rsi, %rcx
+; AVX512VL-NEXT: orq %rdx, %rcx
+; AVX512VL-NEXT: cmovnel %r9d, %eax
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: load_ctlz_undef_i1024:
; AVX512VLPOPCNT: # %bb.0:
-; AVX512VLPOPCNT-NEXT: vmovdqu64 64(%rdi), %zmm0
-; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
-; AVX512VLPOPCNT-NEXT: vpermq %zmm0, %zmm1, %zmm2
-; AVX512VLPOPCNT-NEXT: vplzcntq %zmm2, %zmm3
-; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm4 = [0,64,128,192,256,320,384,448]
-; AVX512VLPOPCNT-NEXT: vpaddq %zmm4, %zmm3, %zmm3
-; AVX512VLPOPCNT-NEXT: vptestmq %zmm2, %zmm2, %k1
-; AVX512VLPOPCNT-NEXT: vpcompressq %zmm3, %zmm2 {%k1} {z}
-; AVX512VLPOPCNT-NEXT: vmovd %xmm2, %ecx
-; AVX512VLPOPCNT-NEXT: vpermq (%rdi), %zmm1, %zmm1
+; AVX512VLPOPCNT-NEXT: movq 80(%rdi), %rsi
+; AVX512VLPOPCNT-NEXT: movq 64(%rdi), %rcx
+; AVX512VLPOPCNT-NEXT: movq 72(%rdi), %rdx
+; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]
+; AVX512VLPOPCNT-NEXT: vpermq 64(%rdi), %zmm0, %zmm1
+; AVX512VLPOPCNT-NEXT: movq 88(%rdi), %r8
; AVX512VLPOPCNT-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512VLPOPCNT-NEXT: vpaddq %zmm4, %zmm2, %zmm2
+; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm3 = [0,64,128,192,256,320,384,448]
+; AVX512VLPOPCNT-NEXT: vpaddq %zmm3, %zmm2, %zmm2
; AVX512VLPOPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
; AVX512VLPOPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
-; AVX512VLPOPCNT-NEXT: vmovd %xmm1, %eax
+; AVX512VLPOPCNT-NEXT: vmovd %xmm1, %r9d
+; AVX512VLPOPCNT-NEXT: vpermq (%rdi), %zmm0, %zmm0
+; AVX512VLPOPCNT-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512VLPOPCNT-NEXT: vpaddq %zmm3, %zmm1, %zmm1
+; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512VLPOPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
; AVX512VLPOPCNT-NEXT: addl $512, %eax # imm = 0x200
-; AVX512VLPOPCNT-NEXT: vpor 96(%rdi), %ymm0, %ymm0
-; AVX512VLPOPCNT-NEXT: vptest %ymm0, %ymm0
-; AVX512VLPOPCNT-NEXT: cmovnel %ecx, %eax
+; AVX512VLPOPCNT-NEXT: orq 120(%rdi), %r8
+; AVX512VLPOPCNT-NEXT: orq 104(%rdi), %rdx
+; AVX512VLPOPCNT-NEXT: orq 112(%rdi), %rsi
+; AVX512VLPOPCNT-NEXT: orq %r8, %rdx
+; AVX512VLPOPCNT-NEXT: orq 96(%rdi), %rcx
+; AVX512VLPOPCNT-NEXT: orq %rsi, %rcx
+; AVX512VLPOPCNT-NEXT: orq %rdx, %rcx
+; AVX512VLPOPCNT-NEXT: cmovnel %r9d, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%a0 = load i1024, ptr %p0
@@ -5573,92 +6170,109 @@ define i32 @test_cttz_i512(i512 %a0) nounwind {
define i32 @load_cttz_i512(ptr %p0) nounwind {
; SSE-LABEL: load_cttz_i512:
; SSE: # %bb.0:
-; SSE-NEXT: movdqa (%rdi), %xmm0
-; SSE-NEXT: movq 48(%rdi), %rdx
-; SSE-NEXT: movq 40(%rdi), %rcx
-; SSE-NEXT: movq (%rdi), %rax
-; SSE-NEXT: movq 8(%rdi), %r8
-; SSE-NEXT: rep bsfq %rax, %rsi
-; SSE-NEXT: rep bsfq %r8, %r9
-; SSE-NEXT: addl $64, %r9d
-; SSE-NEXT: testq %rax, %rax
-; SSE-NEXT: cmovnel %esi, %r9d
-; SSE-NEXT: movq 16(%rdi), %r10
-; SSE-NEXT: rep bsfq %r10, %r11
-; SSE-NEXT: rep bsfq 24(%rdi), %rsi
-; SSE-NEXT: addl $64, %esi
-; SSE-NEXT: testq %r10, %r10
-; SSE-NEXT: cmovnel %r11d, %esi
-; SSE-NEXT: subl $-128, %esi
-; SSE-NEXT: orq %r8, %rax
-; SSE-NEXT: cmovnel %r9d, %esi
-; SSE-NEXT: movq 32(%rdi), %r8
-; SSE-NEXT: rep bsfq %r8, %rax
-; SSE-NEXT: rep bsfq %rcx, %r9
-; SSE-NEXT: addl $64, %r9d
-; SSE-NEXT: testq %r8, %r8
-; SSE-NEXT: cmovnel %eax, %r9d
-; SSE-NEXT: rep bsfq %rdx, %r10
+; SSE-NEXT: pushq %r15
+; SSE-NEXT: pushq %r14
+; SSE-NEXT: pushq %rbx
+; SSE-NEXT: movq 48(%rdi), %r10
+; SSE-NEXT: movq 40(%rdi), %r9
+; SSE-NEXT: movq 24(%rdi), %r8
+; SSE-NEXT: movq 16(%rdi), %rdx
+; SSE-NEXT: movq (%rdi), %rcx
+; SSE-NEXT: movq 8(%rdi), %rsi
+; SSE-NEXT: rep bsfq %rcx, %rax
+; SSE-NEXT: rep bsfq %rsi, %rbx
+; SSE-NEXT: addl $64, %ebx
+; SSE-NEXT: testq %rcx, %rcx
+; SSE-NEXT: cmovnel %eax, %ebx
+; SSE-NEXT: rep bsfq %rdx, %rax
+; SSE-NEXT: rep bsfq %r8, %r11
+; SSE-NEXT: addl $64, %r11d
+; SSE-NEXT: testq %rdx, %rdx
+; SSE-NEXT: cmovnel %eax, %r11d
+; SSE-NEXT: movq 32(%rdi), %r14
+; SSE-NEXT: subl $-128, %r11d
+; SSE-NEXT: movq %rcx, %rax
+; SSE-NEXT: orq %rsi, %rax
+; SSE-NEXT: cmovnel %ebx, %r11d
+; SSE-NEXT: rep bsfq %r14, %rax
+; SSE-NEXT: rep bsfq %r9, %rbx
+; SSE-NEXT: addl $64, %ebx
+; SSE-NEXT: testq %r14, %r14
+; SSE-NEXT: cmovnel %eax, %ebx
+; SSE-NEXT: rep bsfq %r10, %r15
; SSE-NEXT: movl $64, %eax
; SSE-NEXT: rep bsfq 56(%rdi), %rax
; SSE-NEXT: addl $64, %eax
-; SSE-NEXT: testq %rdx, %rdx
-; SSE-NEXT: cmovnel %r10d, %eax
+; SSE-NEXT: testq %r10, %r10
+; SSE-NEXT: cmovnel %r15d, %eax
; SSE-NEXT: subl $-128, %eax
-; SSE-NEXT: orq %rcx, %r8
-; SSE-NEXT: cmovnel %r9d, %eax
+; SSE-NEXT: orq %r9, %r14
+; SSE-NEXT: cmovnel %ebx, %eax
; SSE-NEXT: addl $256, %eax # imm = 0x100
-; SSE-NEXT: por 16(%rdi), %xmm0
-; SSE-NEXT: ptest %xmm0, %xmm0
-; SSE-NEXT: cmovnel %esi, %eax
+; SSE-NEXT: orq %r8, %rsi
+; SSE-NEXT: orq %rdx, %rcx
+; SSE-NEXT: orq %rsi, %rcx
+; SSE-NEXT: cmovnel %r11d, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
+; SSE-NEXT: popq %rbx
+; SSE-NEXT: popq %r14
+; SSE-NEXT: popq %r15
; SSE-NEXT: retq
;
; AVX2-LABEL: load_cttz_i512:
; AVX2: # %bb.0:
-; AVX2-NEXT: movq 40(%rdi), %rcx
-; AVX2-NEXT: movq 32(%rdi), %rdx
-; AVX2-NEXT: movq 16(%rdi), %rax
-; AVX2-NEXT: movq (%rdi), %r8
-; AVX2-NEXT: movq 8(%rdi), %r9
-; AVX2-NEXT: tzcntq %r8, %rsi
-; AVX2-NEXT: tzcntq %r9, %r10
-; AVX2-NEXT: addl $64, %r10d
-; AVX2-NEXT: testq %r8, %r8
-; AVX2-NEXT: cmovnel %esi, %r10d
-; AVX2-NEXT: tzcntq %rax, %r11
-; AVX2-NEXT: xorl %esi, %esi
-; AVX2-NEXT: tzcntq 24(%rdi), %rsi
-; AVX2-NEXT: addl $64, %esi
-; AVX2-NEXT: testq %rax, %rax
-; AVX2-NEXT: cmovnel %r11d, %esi
-; AVX2-NEXT: subl $-128, %esi
-; AVX2-NEXT: orq %r9, %r8
-; AVX2-NEXT: cmovnel %r10d, %esi
+; AVX2-NEXT: pushq %r15
+; AVX2-NEXT: pushq %r14
+; AVX2-NEXT: pushq %rbx
+; AVX2-NEXT: movq 48(%rdi), %r10
+; AVX2-NEXT: movq 40(%rdi), %r9
+; AVX2-NEXT: movq 24(%rdi), %r8
+; AVX2-NEXT: movq 16(%rdi), %rdx
+; AVX2-NEXT: movq (%rdi), %rcx
+; AVX2-NEXT: movq 8(%rdi), %rsi
+; AVX2-NEXT: tzcntq %rcx, %rax
+; AVX2-NEXT: xorl %ebx, %ebx
+; AVX2-NEXT: tzcntq %rsi, %rbx
+; AVX2-NEXT: addl $64, %ebx
+; AVX2-NEXT: testq %rcx, %rcx
+; AVX2-NEXT: cmovnel %eax, %ebx
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq %rdx, %rax
-; AVX2-NEXT: xorl %r8d, %r8d
-; AVX2-NEXT: tzcntq %rcx, %r8
-; AVX2-NEXT: addl $64, %r8d
+; AVX2-NEXT: tzcntq %r8, %r11
+; AVX2-NEXT: addl $64, %r11d
; AVX2-NEXT: testq %rdx, %rdx
-; AVX2-NEXT: cmovnel %eax, %r8d
-; AVX2-NEXT: movq 48(%rdi), %r9
-; AVX2-NEXT: xorl %r10d, %r10d
-; AVX2-NEXT: tzcntq %r9, %r10
+; AVX2-NEXT: cmovnel %eax, %r11d
+; AVX2-NEXT: subl $-128, %r11d
+; AVX2-NEXT: movq %rcx, %rax
+; AVX2-NEXT: orq %rsi, %rax
+; AVX2-NEXT: cmovnel %ebx, %r11d
+; AVX2-NEXT: movq 32(%rdi), %rbx
+; AVX2-NEXT: xorl %eax, %eax
+; AVX2-NEXT: tzcntq %rbx, %rax
+; AVX2-NEXT: xorl %r14d, %r14d
+; AVX2-NEXT: tzcntq %r9, %r14
+; AVX2-NEXT: addl $64, %r14d
+; AVX2-NEXT: testq %rbx, %rbx
+; AVX2-NEXT: cmovnel %eax, %r14d
+; AVX2-NEXT: xorl %r15d, %r15d
+; AVX2-NEXT: tzcntq %r10, %r15
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq 56(%rdi), %rax
; AVX2-NEXT: addl $64, %eax
-; AVX2-NEXT: testq %r9, %r9
-; AVX2-NEXT: cmovnel %r10d, %eax
+; AVX2-NEXT: testq %r10, %r10
+; AVX2-NEXT: cmovnel %r15d, %eax
; AVX2-NEXT: subl $-128, %eax
-; AVX2-NEXT: orq %rcx, %rdx
-; AVX2-NEXT: cmovnel %r8d, %eax
-; AVX2-NEXT: vmovdqa (%rdi), %xmm0
+; AVX2-NEXT: orq %r9, %rbx
+; AVX2-NEXT: cmovnel %r14d, %eax
; AVX2-NEXT: addl $256, %eax # imm = 0x100
-; AVX2-NEXT: vpor 16(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vptest %xmm0, %xmm0
-; AVX2-NEXT: cmovnel %esi, %eax
+; AVX2-NEXT: orq %r8, %rsi
+; AVX2-NEXT: orq %rdx, %rcx
+; AVX2-NEXT: orq %rsi, %rcx
+; AVX2-NEXT: cmovnel %r11d, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
+; AVX2-NEXT: popq %rbx
+; AVX2-NEXT: popq %r14
+; AVX2-NEXT: popq %r15
; AVX2-NEXT: retq
;
; AVX512F-LABEL: load_cttz_i512:
@@ -5890,91 +6504,97 @@ define i32 @test_cttz_i1024(i1024 %a0) nounwind {
; SSE-NEXT: pushq %r13
; SSE-NEXT: pushq %r12
; SSE-NEXT: pushq %rbx
-; SSE-NEXT: movq %r9, %r10
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r14
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r9
-; SSE-NEXT: rep bsfq %rdi, %r11
-; SSE-NEXT: rep bsfq %rsi, %rbx
-; SSE-NEXT: addl $64, %ebx
-; SSE-NEXT: testq %rdi, %rdi
-; SSE-NEXT: cmovnel %r11d, %ebx
+; SSE-NEXT: movq %r9, %r13
+; SSE-NEXT: movq %r8, %r14
+; SSE-NEXT: movq %rcx, %rbx
+; SSE-NEXT: movq %rdx, %r10
+; SSE-NEXT: movq %rsi, %r9
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rsi
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r11
-; SSE-NEXT: rep bsfq %rdx, %r12
-; SSE-NEXT: rep bsfq %rcx, %r15
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; SSE-NEXT: rep bsfq %rdi, %rax
+; SSE-NEXT: rep bsfq %r9, %r15
; SSE-NEXT: addl $64, %r15d
-; SSE-NEXT: testq %rdx, %rdx
-; SSE-NEXT: cmovnel %r12d, %r15d
-; SSE-NEXT: subl $-128, %r15d
+; SSE-NEXT: testq %rdi, %rdi
+; SSE-NEXT: cmovnel %eax, %r15d
+; SSE-NEXT: rep bsfq %r10, %r12
+; SSE-NEXT: rep bsfq %rcx, %rax
+; SSE-NEXT: addl $64, %eax
+; SSE-NEXT: testq %r10, %r10
+; SSE-NEXT: cmovnel %r12d, %eax
+; SSE-NEXT: subl $-128, %eax
; SSE-NEXT: movq %rdi, %r12
-; SSE-NEXT: orq %rsi, %r12
-; SSE-NEXT: cmovnel %ebx, %r15d
-; SSE-NEXT: rep bsfq %r8, %rbx
-; SSE-NEXT: rep bsfq %r10, %r13
+; SSE-NEXT: orq %r9, %r12
+; SSE-NEXT: cmovnel %r15d, %eax
+; SSE-NEXT: rep bsfq %r8, %r15
+; SSE-NEXT: movq %r13, %rcx
+; SSE-NEXT: movq %r13, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE-NEXT: rep bsfq %r13, %r13
; SSE-NEXT: addl $64, %r13d
; SSE-NEXT: testq %r8, %r8
-; SSE-NEXT: cmovnel %ebx, %r13d
-; SSE-NEXT: rep bsfq %r9, %r12
-; SSE-NEXT: rep bsfq %r11, %rbx
-; SSE-NEXT: addl $64, %ebx
-; SSE-NEXT: testq %r9, %r9
-; SSE-NEXT: cmovnel %r12d, %ebx
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r12
-; SSE-NEXT: subl $-128, %ebx
-; SSE-NEXT: movq %r8, %rbp
-; SSE-NEXT: orq %r10, %rbp
-; SSE-NEXT: cmovnel %r13d, %ebx
-; SSE-NEXT: addl $256, %ebx # imm = 0x100
-; SSE-NEXT: movq %rsi, %r13
-; SSE-NEXT: orq %rcx, %r13
-; SSE-NEXT: movq %rdi, %rbp
-; SSE-NEXT: orq %rdx, %rbp
-; SSE-NEXT: orq %r13, %rbp
-; SSE-NEXT: cmovnel %r15d, %ebx
-; SSE-NEXT: rep bsfq %rax, %r15
-; SSE-NEXT: rep bsfq %r12, %r13
-; SSE-NEXT: addl $64, %r13d
-; SSE-NEXT: testq %rax, %rax
; SSE-NEXT: cmovnel %r15d, %r13d
-; SSE-NEXT: rep bsfq %r14, %rbp
+; SSE-NEXT: rep bsfq %rdx, %r12
; SSE-NEXT: rep bsfq {{[0-9]+}}(%rsp), %r15
; SSE-NEXT: addl $64, %r15d
-; SSE-NEXT: testq %r14, %r14
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r9
-; SSE-NEXT: cmovnel %ebp, %r15d
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r14
-; SSE-NEXT: subl $-128, %r15d
-; SSE-NEXT: orq %r12, %rax
+; SSE-NEXT: testq %rdx, %rdx
+; SSE-NEXT: cmovnel %r12d, %r15d
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r12
+; SSE-NEXT: subl $-128, %r15d
+; SSE-NEXT: movq %r8, %rbp
+; SSE-NEXT: orq %rcx, %rbp
; SSE-NEXT: cmovnel %r13d, %r15d
-; SSE-NEXT: rep bsfq %r14, %rax
-; SSE-NEXT: rep bsfq %r12, %r13
+; SSE-NEXT: addl $256, %r15d # imm = 0x100
+; SSE-NEXT: movq %r9, %r13
+; SSE-NEXT: orq %rbx, %r13
+; SSE-NEXT: movq %rdi, %rbp
+; SSE-NEXT: orq %r10, %rbp
+; SSE-NEXT: orq %r13, %rbp
+; SSE-NEXT: cmovnel %eax, %r15d
+; SSE-NEXT: rep bsfq %r11, %r13
+; SSE-NEXT: rep bsfq %r12, %rax
+; SSE-NEXT: addl $64, %eax
+; SSE-NEXT: testq %r11, %r11
+; SSE-NEXT: cmovnel %r13d, %eax
+; SSE-NEXT: rep bsfq {{[0-9]+}}(%rsp), %r13
; SSE-NEXT: addl $64, %r13d
-; SSE-NEXT: testq %r14, %r14
+; SSE-NEXT: rep bsfq %rsi, %rcx
+; SSE-NEXT: testq %rsi, %rsi
+; SSE-NEXT: cmovnel %ecx, %r13d
+; SSE-NEXT: subl $-128, %r13d
+; SSE-NEXT: movq %r11, %rcx
+; SSE-NEXT: orq %r12, %rcx
; SSE-NEXT: cmovnel %eax, %r13d
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rbp
+; SSE-NEXT: rep bsfq %rbp, %rcx
+; SSE-NEXT: addl $64, %ecx
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; SSE-NEXT: rep bsfq %rdx, %rax
+; SSE-NEXT: testq %rdx, %rdx
+; SSE-NEXT: cmovnel %eax, %ecx
; SSE-NEXT: movl $64, %eax
; SSE-NEXT: rep bsfq {{[0-9]+}}(%rsp), %rax
; SSE-NEXT: addl $64, %eax
-; SSE-NEXT: rep bsfq %r9, %rbp
-; SSE-NEXT: testq %r9, %r9
-; SSE-NEXT: cmovnel %ebp, %eax
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r8
+; SSE-NEXT: rep bsfq %r8, %rsi
+; SSE-NEXT: testq %r8, %r8
+; SSE-NEXT: cmovnel %esi, %eax
; SSE-NEXT: subl $-128, %eax
-; SSE-NEXT: orq %r12, %r14
-; SSE-NEXT: cmovnel %r13d, %eax
-; SSE-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm0
+; SSE-NEXT: orq %rbp, %rdx
+; SSE-NEXT: cmovnel %ecx, %eax
+; SSE-NEXT: orq {{[0-9]+}}(%rsp), %r12
+; SSE-NEXT: orq {{[0-9]+}}(%rsp), %r11
; SSE-NEXT: addl $256, %eax # imm = 0x100
-; SSE-NEXT: por {{[0-9]+}}(%rsp), %xmm0
-; SSE-NEXT: ptest %xmm0, %xmm0
-; SSE-NEXT: cmovnel %r15d, %eax
-; SSE-NEXT: orq %r11, %rcx
-; SSE-NEXT: orq %r10, %rsi
-; SSE-NEXT: orq %rcx, %rsi
-; SSE-NEXT: orq {{[0-9]+}}(%rsp), %rdx
-; SSE-NEXT: orq %r8, %rdi
-; SSE-NEXT: orq %rdx, %rdi
+; SSE-NEXT: orq %r12, %r11
+; SSE-NEXT: cmovnel %r13d, %eax
+; SSE-NEXT: orq {{[0-9]+}}(%rsp), %rbx
+; SSE-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r9 # 8-byte Folded Reload
+; SSE-NEXT: orq %rbx, %r9
+; SSE-NEXT: orq {{[0-9]+}}(%rsp), %r10
+; SSE-NEXT: orq %r14, %rdi
+; SSE-NEXT: orq %r10, %rdi
; SSE-NEXT: addl $512, %eax # imm = 0x200
-; SSE-NEXT: orq %rsi, %rdi
-; SSE-NEXT: cmovnel %ebx, %eax
+; SSE-NEXT: orq %r9, %rdi
+; SSE-NEXT: cmovnel %r15d, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: popq %rbx
; SSE-NEXT: popq %r12
@@ -5992,108 +6612,111 @@ define i32 @test_cttz_i1024(i1024 %a0) nounwind {
; AVX2-NEXT: pushq %r13
; AVX2-NEXT: pushq %r12
; AVX2-NEXT: pushq %rbx
-; AVX2-NEXT: movq %r9, %r10
-; AVX2-NEXT: movq %r8, %r9
-; AVX2-NEXT: movq %rcx, %r8
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r14
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX2-NEXT: movq %r9, %rbx
+; AVX2-NEXT: movq %r8, %r14
+; AVX2-NEXT: movq %rcx, %r11
+; AVX2-NEXT: movq %rdx, %r10
+; AVX2-NEXT: movq %rsi, %r9
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r8
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rsi
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rcx
-; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: tzcntq %rdi, %rbx
-; AVX2-NEXT: xorl %r12d, %r12d
-; AVX2-NEXT: tzcntq %rsi, %r12
-; AVX2-NEXT: addl $64, %r12d
-; AVX2-NEXT: testq %rdi, %rdi
-; AVX2-NEXT: cmovnel %ebx, %r12d
-; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: tzcntq %rdx, %rbx
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; AVX2-NEXT: tzcntq %rdi, %rax
; AVX2-NEXT: xorl %r15d, %r15d
-; AVX2-NEXT: tzcntq %r8, %r15
+; AVX2-NEXT: tzcntq %r9, %r15
; AVX2-NEXT: addl $64, %r15d
-; AVX2-NEXT: testq %rdx, %rdx
-; AVX2-NEXT: cmovnel %ebx, %r15d
-; AVX2-NEXT: subl $-128, %r15d
-; AVX2-NEXT: movq %rdi, %rbx
-; AVX2-NEXT: orq %rsi, %rbx
-; AVX2-NEXT: cmovnel %r12d, %r15d
-; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: tzcntq %r9, %rbx
+; AVX2-NEXT: testq %rdi, %rdi
+; AVX2-NEXT: cmovnel %eax, %r15d
+; AVX2-NEXT: xorl %r12d, %r12d
+; AVX2-NEXT: tzcntq %r10, %r12
+; AVX2-NEXT: xorl %eax, %eax
+; AVX2-NEXT: tzcntq %r11, %rax
+; AVX2-NEXT: addl $64, %eax
+; AVX2-NEXT: testq %r10, %r10
+; AVX2-NEXT: cmovnel %r12d, %eax
+; AVX2-NEXT: subl $-128, %eax
+; AVX2-NEXT: movq %rdi, %r12
+; AVX2-NEXT: orq %r9, %r12
+; AVX2-NEXT: cmovnel %r15d, %eax
+; AVX2-NEXT: xorl %r15d, %r15d
+; AVX2-NEXT: tzcntq %r14, %r15
+; AVX2-NEXT: movq %rbx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: xorl %r12d, %r12d
-; AVX2-NEXT: tzcntq %r10, %r12
+; AVX2-NEXT: tzcntq %rbx, %r12
; AVX2-NEXT: addl $64, %r12d
-; AVX2-NEXT: testq %r9, %r9
-; AVX2-NEXT: cmovnel %ebx, %r12d
-; AVX2-NEXT: xorl %r13d, %r13d
-; AVX2-NEXT: tzcntq %r11, %r13
-; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: tzcntq %rcx, %rbx
-; AVX2-NEXT: addl $64, %ebx
-; AVX2-NEXT: testq %r11, %r11
-; AVX2-NEXT: cmovnel %r13d, %ebx
-; AVX2-NEXT: subl $-128, %ebx
-; AVX2-NEXT: movq %r9, %r13
-; AVX2-NEXT: orq %r10, %r13
-; AVX2-NEXT: cmovnel %r12d, %ebx
-; AVX2-NEXT: addl $256, %ebx # imm = 0x100
-; AVX2-NEXT: movq %rsi, %r12
-; AVX2-NEXT: orq %r8, %r12
-; AVX2-NEXT: movq %rdi, %r13
-; AVX2-NEXT: orq %rdx, %r13
-; AVX2-NEXT: orq %r12, %r13
-; AVX2-NEXT: cmovnel %r15d, %ebx
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r12
-; AVX2-NEXT: xorl %r15d, %r15d
-; AVX2-NEXT: tzcntq %rax, %r15
+; AVX2-NEXT: testq %r14, %r14
+; AVX2-NEXT: cmovnel %r15d, %r12d
; AVX2-NEXT: xorl %r13d, %r13d
-; AVX2-NEXT: tzcntq %r12, %r13
-; AVX2-NEXT: addl $64, %r13d
-; AVX2-NEXT: testq %rax, %rax
-; AVX2-NEXT: cmovnel %r15d, %r13d
-; AVX2-NEXT: xorl %ebp, %ebp
-; AVX2-NEXT: tzcntq %r14, %rbp
+; AVX2-NEXT: tzcntq %rcx, %r13
; AVX2-NEXT: xorl %r15d, %r15d
-; AVX2-NEXT: tzcntq {{[0-9]+}}(%rsp), %r15
+; AVX2-NEXT: tzcntq %rdx, %r15
; AVX2-NEXT: addl $64, %r15d
-; AVX2-NEXT: testq %r14, %r14
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r14
-; AVX2-NEXT: cmovnel %ebp, %r15d
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rbp
-; AVX2-NEXT: subl $-128, %r15d
-; AVX2-NEXT: orq %r12, %rax
+; AVX2-NEXT: testq %rcx, %rcx
; AVX2-NEXT: cmovnel %r13d, %r15d
+; AVX2-NEXT: subl $-128, %r15d
+; AVX2-NEXT: movq %r14, %r13
+; AVX2-NEXT: orq %rbx, %r13
+; AVX2-NEXT: cmovnel %r12d, %r15d
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r12
+; AVX2-NEXT: addl $256, %r15d # imm = 0x100
+; AVX2-NEXT: movq %r9, %r13
+; AVX2-NEXT: orq %r11, %r13
+; AVX2-NEXT: movq %rdi, %rbp
+; AVX2-NEXT: orq %r10, %rbp
+; AVX2-NEXT: orq %r13, %rbp
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r13
+; AVX2-NEXT: cmovnel %eax, %r15d
+; AVX2-NEXT: xorl %ebp, %ebp
+; AVX2-NEXT: tzcntq %r12, %rbp
; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: tzcntq %rbp, %rax
-; AVX2-NEXT: xorl %r12d, %r12d
-; AVX2-NEXT: tzcntq %r14, %r12
-; AVX2-NEXT: addl $64, %r12d
-; AVX2-NEXT: testq %rbp, %rbp
-; AVX2-NEXT: cmovnel %eax, %r12d
+; AVX2-NEXT: tzcntq %r13, %rax
+; AVX2-NEXT: addl $64, %eax
+; AVX2-NEXT: testq %r12, %r12
+; AVX2-NEXT: cmovnel %ebp, %eax
+; AVX2-NEXT: xorl %ebp, %ebp
+; AVX2-NEXT: tzcntq %r8, %rbp
+; AVX2-NEXT: addl $64, %ebp
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: tzcntq %rsi, %rcx
+; AVX2-NEXT: testq %rsi, %rsi
+; AVX2-NEXT: cmovnel %ecx, %ebp
+; AVX2-NEXT: subl $-128, %ebp
+; AVX2-NEXT: movq %r12, %rcx
+; AVX2-NEXT: orq %r13, %rcx
+; AVX2-NEXT: cmovnel %eax, %ebp
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: tzcntq %rbx, %rcx
+; AVX2-NEXT: addl $64, %ecx
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; AVX2-NEXT: xorl %eax, %eax
+; AVX2-NEXT: tzcntq %rdx, %rax
+; AVX2-NEXT: testq %rdx, %rdx
+; AVX2-NEXT: cmovnel %eax, %ecx
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq {{[0-9]+}}(%rsp), %rax
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r13
; AVX2-NEXT: addl $64, %eax
-; AVX2-NEXT: xorl %ecx, %ecx
-; AVX2-NEXT: tzcntq %r13, %rcx
-; AVX2-NEXT: testq %r13, %r13
-; AVX2-NEXT: cmovnel %ecx, %eax
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r8
+; AVX2-NEXT: tzcntq %r8, %rsi
+; AVX2-NEXT: testq %r8, %r8
+; AVX2-NEXT: cmovnel %esi, %eax
; AVX2-NEXT: subl $-128, %eax
-; AVX2-NEXT: orq %r14, %rbp
-; AVX2-NEXT: cmovnel %r12d, %eax
-; AVX2-NEXT: vmovdqa {{[0-9]+}}(%rsp), %xmm0
+; AVX2-NEXT: orq %rbx, %rdx
+; AVX2-NEXT: cmovnel %ecx, %eax
+; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r13
+; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r12
; AVX2-NEXT: addl $256, %eax # imm = 0x100
-; AVX2-NEXT: vpor {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX2-NEXT: vptest %xmm0, %xmm0
-; AVX2-NEXT: cmovnel %r15d, %eax
-; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r8
-; AVX2-NEXT: orq %r10, %rsi
-; AVX2-NEXT: orq %r8, %rsi
-; AVX2-NEXT: orq %r11, %rdx
-; AVX2-NEXT: orq %r9, %rdi
-; AVX2-NEXT: orq %rdx, %rdi
+; AVX2-NEXT: orq %r13, %r12
+; AVX2-NEXT: cmovnel %ebp, %eax
+; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r11
+; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r9 # 8-byte Folded Reload
+; AVX2-NEXT: orq %r11, %r9
+; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r10
+; AVX2-NEXT: orq %r14, %rdi
+; AVX2-NEXT: orq %r10, %rdi
; AVX2-NEXT: addl $512, %eax # imm = 0x200
-; AVX2-NEXT: orq %rsi, %rdi
-; AVX2-NEXT: cmovnel %ebx, %eax
+; AVX2-NEXT: orq %r9, %rdi
+; AVX2-NEXT: cmovnel %r15d, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: popq %rbx
; AVX2-NEXT: popq %r12
@@ -6284,301 +6907,389 @@ define i32 @test_cttz_i1024(i1024 %a0) nounwind {
define i32 @load_cttz_i1024(ptr %p0) nounwind {
; SSE-LABEL: load_cttz_i1024:
; SSE: # %bb.0:
+; SSE-NEXT: pushq %rbp
+; SSE-NEXT: pushq %r15
+; SSE-NEXT: pushq %r14
+; SSE-NEXT: pushq %r13
+; SSE-NEXT: pushq %r12
; SSE-NEXT: pushq %rbx
-; SSE-NEXT: movq 112(%rdi), %rcx
-; SSE-NEXT: movq 48(%rdi), %rsi
-; SSE-NEXT: movq (%rdi), %rdx
-; SSE-NEXT: movq 8(%rdi), %r8
-; SSE-NEXT: rep bsfq %rdx, %rax
-; SSE-NEXT: rep bsfq %r8, %r9
-; SSE-NEXT: addl $64, %r9d
-; SSE-NEXT: testq %rdx, %rdx
-; SSE-NEXT: cmovnel %eax, %r9d
-; SSE-NEXT: movq 16(%rdi), %r10
-; SSE-NEXT: rep bsfq %r10, %r11
-; SSE-NEXT: rep bsfq 24(%rdi), %rax
-; SSE-NEXT: addl $64, %eax
-; SSE-NEXT: testq %r10, %r10
-; SSE-NEXT: cmovnel %r11d, %eax
-; SSE-NEXT: movq 40(%rdi), %r10
-; SSE-NEXT: movq 32(%rdi), %r11
-; SSE-NEXT: subl $-128, %eax
-; SSE-NEXT: orq %r8, %rdx
-; SSE-NEXT: cmovnel %r9d, %eax
+; SSE-NEXT: movq 88(%rdi), %r10
+; SSE-NEXT: movq %r10, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE-NEXT: movq 56(%rdi), %rcx
+; SSE-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE-NEXT: movq 40(%rdi), %rsi
+; SSE-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE-NEXT: movq 24(%rdi), %r9
+; SSE-NEXT: movq 16(%rdi), %r15
+; SSE-NEXT: movq (%rdi), %r8
+; SSE-NEXT: movq 8(%rdi), %r11
+; SSE-NEXT: rep bsfq %r8, %rax
; SSE-NEXT: rep bsfq %r11, %rdx
-; SSE-NEXT: rep bsfq %r10, %rbx
-; SSE-NEXT: addl $64, %ebx
-; SSE-NEXT: testq %r11, %r11
-; SSE-NEXT: cmovnel %edx, %ebx
-; SSE-NEXT: rep bsfq %rsi, %r9
-; SSE-NEXT: rep bsfq 56(%rdi), %rdx
; SSE-NEXT: addl $64, %edx
-; SSE-NEXT: testq %rsi, %rsi
-; SSE-NEXT: movq 72(%rdi), %r8
-; SSE-NEXT: cmovnel %r9d, %edx
-; SSE-NEXT: movq 64(%rdi), %r9
-; SSE-NEXT: movdqa 16(%rdi), %xmm0
-; SSE-NEXT: movdqa (%rdi), %xmm1
-; SSE-NEXT: subl $-128, %edx
-; SSE-NEXT: orq %r10, %r11
-; SSE-NEXT: cmovnel %ebx, %edx
-; SSE-NEXT: addl $256, %edx # imm = 0x100
-; SSE-NEXT: movdqa %xmm1, %xmm2
-; SSE-NEXT: por %xmm0, %xmm2
-; SSE-NEXT: ptest %xmm2, %xmm2
+; SSE-NEXT: testq %r8, %r8
; SSE-NEXT: cmovnel %eax, %edx
+; SSE-NEXT: rep bsfq %r15, %rbx
; SSE-NEXT: rep bsfq %r9, %rax
-; SSE-NEXT: rep bsfq %r8, %r11
-; SSE-NEXT: addl $64, %r11d
-; SSE-NEXT: testq %r9, %r9
-; SSE-NEXT: cmovnel %eax, %r11d
-; SSE-NEXT: movq 80(%rdi), %rax
-; SSE-NEXT: rep bsfq %rax, %r10
-; SSE-NEXT: rep bsfq 88(%rdi), %rsi
-; SSE-NEXT: addl $64, %esi
-; SSE-NEXT: testq %rax, %rax
-; SSE-NEXT: cmovnel %r10d, %esi
-; SSE-NEXT: movq 104(%rdi), %r10
-; SSE-NEXT: movq 96(%rdi), %rbx
-; SSE-NEXT: subl $-128, %esi
-; SSE-NEXT: orq %r8, %r9
-; SSE-NEXT: cmovnel %r11d, %esi
-; SSE-NEXT: rep bsfq %rbx, %rax
-; SSE-NEXT: rep bsfq %r10, %r8
-; SSE-NEXT: addl $64, %r8d
+; SSE-NEXT: movq %r9, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE-NEXT: addl $64, %eax
+; SSE-NEXT: testq %r15, %r15
+; SSE-NEXT: cmovnel %ebx, %eax
+; SSE-NEXT: movq 32(%rdi), %rbx
+; SSE-NEXT: subl $-128, %eax
+; SSE-NEXT: movq %r8, %r14
+; SSE-NEXT: orq %r11, %r14
+; SSE-NEXT: cmovnel %edx, %eax
+; SSE-NEXT: rep bsfq %rbx, %rdx
+; SSE-NEXT: rep bsfq %rsi, %r12
+; SSE-NEXT: addl $64, %r12d
; SSE-NEXT: testq %rbx, %rbx
-; SSE-NEXT: cmovnel %eax, %r8d
-; SSE-NEXT: rep bsfq %rcx, %r9
+; SSE-NEXT: cmovnel %edx, %r12d
+; SSE-NEXT: movq 48(%rdi), %r13
+; SSE-NEXT: movq %r13, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE-NEXT: rep bsfq %r13, %rdx
+; SSE-NEXT: rep bsfq %rcx, %r14
+; SSE-NEXT: addl $64, %r14d
+; SSE-NEXT: testq %r13, %r13
+; SSE-NEXT: cmovnel %edx, %r14d
+; SSE-NEXT: subl $-128, %r14d
+; SSE-NEXT: movq %rbx, %rdx
+; SSE-NEXT: orq %rsi, %rdx
+; SSE-NEXT: cmovnel %r12d, %r14d
+; SSE-NEXT: movq 72(%rdi), %r12
+; SSE-NEXT: addl $256, %r14d # imm = 0x100
+; SSE-NEXT: movq %r11, %rdx
+; SSE-NEXT: orq %r9, %rdx
+; SSE-NEXT: movq %r8, %r13
+; SSE-NEXT: orq %r15, %r13
+; SSE-NEXT: orq %rdx, %r13
+; SSE-NEXT: movq 64(%rdi), %r13
+; SSE-NEXT: cmovnel %eax, %r14d
+; SSE-NEXT: rep bsfq %r13, %rdx
+; SSE-NEXT: rep bsfq %r12, %rax
+; SSE-NEXT: addl $64, %eax
+; SSE-NEXT: testq %r13, %r13
+; SSE-NEXT: cmovnel %edx, %eax
+; SSE-NEXT: rep bsfq %r10, %rbp
+; SSE-NEXT: addl $64, %ebp
+; SSE-NEXT: movq 80(%rdi), %r10
+; SSE-NEXT: rep bsfq %r10, %rcx
+; SSE-NEXT: testq %r10, %r10
+; SSE-NEXT: cmovnel %ecx, %ebp
+; SSE-NEXT: subl $-128, %ebp
+; SSE-NEXT: movq %r13, %rcx
+; SSE-NEXT: orq %r12, %rcx
+; SSE-NEXT: cmovnel %eax, %ebp
+; SSE-NEXT: movq 104(%rdi), %r9
+; SSE-NEXT: rep bsfq %r9, %rcx
+; SSE-NEXT: addl $64, %ecx
+; SSE-NEXT: movq 96(%rdi), %rdx
+; SSE-NEXT: rep bsfq %rdx, %rax
+; SSE-NEXT: testq %rdx, %rdx
+; SSE-NEXT: cmovnel %eax, %ecx
; SSE-NEXT: movl $64, %eax
; SSE-NEXT: rep bsfq 120(%rdi), %rax
+; SSE-NEXT: movq 112(%rdi), %rdi
; SSE-NEXT: addl $64, %eax
-; SSE-NEXT: testq %rcx, %rcx
-; SSE-NEXT: cmovnel %r9d, %eax
+; SSE-NEXT: rep bsfq %rdi, %rsi
+; SSE-NEXT: testq %rdi, %rdi
+; SSE-NEXT: cmovnel %esi, %eax
; SSE-NEXT: subl $-128, %eax
-; SSE-NEXT: orq %r10, %rbx
-; SSE-NEXT: cmovnel %r8d, %eax
-; SSE-NEXT: movdqa 64(%rdi), %xmm2
+; SSE-NEXT: orq %r9, %rdx
+; SSE-NEXT: cmovnel %ecx, %eax
+; SSE-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r12 # 8-byte Folded Reload
+; SSE-NEXT: orq %r10, %r13
; SSE-NEXT: addl $256, %eax # imm = 0x100
-; SSE-NEXT: por 80(%rdi), %xmm2
-; SSE-NEXT: ptest %xmm2, %xmm2
-; SSE-NEXT: cmovnel %esi, %eax
-; SSE-NEXT: por 48(%rdi), %xmm0
-; SSE-NEXT: por 32(%rdi), %xmm1
+; SSE-NEXT: orq %r12, %r13
+; SSE-NEXT: cmovnel %ebp, %eax
+; SSE-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; SSE-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Folded Reload
+; SSE-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Folded Reload
+; SSE-NEXT: orq %rcx, %r11
+; SSE-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r15 # 8-byte Folded Reload
+; SSE-NEXT: orq %rbx, %r8
+; SSE-NEXT: orq %r15, %r8
; SSE-NEXT: addl $512, %eax # imm = 0x200
-; SSE-NEXT: por %xmm0, %xmm1
-; SSE-NEXT: ptest %xmm1, %xmm1
-; SSE-NEXT: cmovnel %edx, %eax
+; SSE-NEXT: orq %r11, %r8
+; SSE-NEXT: cmovnel %r14d, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: popq %rbx
+; SSE-NEXT: popq %r12
+; SSE-NEXT: popq %r13
+; SSE-NEXT: popq %r14
+; SSE-NEXT: popq %r15
+; SSE-NEXT: popq %rbp
; SSE-NEXT: retq
;
; AVX2-LABEL: load_cttz_i1024:
; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %rbp
+; AVX2-NEXT: pushq %r15
+; AVX2-NEXT: pushq %r14
+; AVX2-NEXT: pushq %r13
+; AVX2-NEXT: pushq %r12
; AVX2-NEXT: pushq %rbx
-; AVX2-NEXT: movq 104(%rdi), %rcx
-; AVX2-NEXT: movq 48(%rdi), %rsi
-; AVX2-NEXT: movq 16(%rdi), %rdx
+; AVX2-NEXT: movq 72(%rdi), %r14
+; AVX2-NEXT: movq 64(%rdi), %r15
+; AVX2-NEXT: movq 56(%rdi), %r9
+; AVX2-NEXT: movq %r9, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: movq 48(%rdi), %rcx
+; AVX2-NEXT: movq 40(%rdi), %r10
+; AVX2-NEXT: movq %r10, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: movq 32(%rdi), %rsi
+; AVX2-NEXT: movq 24(%rdi), %rbp
+; AVX2-NEXT: movq 16(%rdi), %rbx
; AVX2-NEXT: movq (%rdi), %r8
-; AVX2-NEXT: movq 8(%rdi), %r9
+; AVX2-NEXT: movq 8(%rdi), %r11
; AVX2-NEXT: tzcntq %r8, %rax
-; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: tzcntq %r9, %rbx
-; AVX2-NEXT: addl $64, %ebx
+; AVX2-NEXT: tzcntq %r11, %rdx
+; AVX2-NEXT: addl $64, %edx
; AVX2-NEXT: testq %r8, %r8
-; AVX2-NEXT: cmovnel %eax, %ebx
-; AVX2-NEXT: tzcntq %rdx, %r11
+; AVX2-NEXT: cmovnel %eax, %edx
+; AVX2-NEXT: xorl %r12d, %r12d
+; AVX2-NEXT: tzcntq %rbx, %r12
; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: tzcntq 24(%rdi), %rax
+; AVX2-NEXT: tzcntq %rbp, %rax
+; AVX2-NEXT: movq %rbp, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: addl $64, %eax
-; AVX2-NEXT: testq %rdx, %rdx
-; AVX2-NEXT: movq 40(%rdi), %r10
-; AVX2-NEXT: cmovnel %r11d, %eax
-; AVX2-NEXT: movq 32(%rdi), %r11
+; AVX2-NEXT: testq %rbx, %rbx
+; AVX2-NEXT: cmovnel %r12d, %eax
; AVX2-NEXT: subl $-128, %eax
-; AVX2-NEXT: orq %r9, %r8
-; AVX2-NEXT: cmovnel %ebx, %eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: tzcntq %r11, %rdx
-; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: tzcntq %r10, %rbx
-; AVX2-NEXT: addl $64, %ebx
-; AVX2-NEXT: testq %r11, %r11
-; AVX2-NEXT: cmovnel %edx, %ebx
+; AVX2-NEXT: movq %r8, %r12
+; AVX2-NEXT: orq %r11, %r12
+; AVX2-NEXT: cmovnel %edx, %eax
; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: tzcntq 56(%rdi), %rdx
-; AVX2-NEXT: xorl %r8d, %r8d
-; AVX2-NEXT: tzcntq %rsi, %r8
-; AVX2-NEXT: addl $64, %edx
+; AVX2-NEXT: tzcntq %rsi, %rdx
+; AVX2-NEXT: xorl %r13d, %r13d
+; AVX2-NEXT: tzcntq %r10, %r13
+; AVX2-NEXT: addl $64, %r13d
; AVX2-NEXT: testq %rsi, %rsi
-; AVX2-NEXT: movq 80(%rdi), %r9
-; AVX2-NEXT: cmovnel %r8d, %edx
-; AVX2-NEXT: movq 72(%rdi), %r8
-; AVX2-NEXT: subl $-128, %edx
-; AVX2-NEXT: orq %r10, %r11
-; AVX2-NEXT: movq 64(%rdi), %r10
-; AVX2-NEXT: cmovnel %ebx, %edx
-; AVX2-NEXT: vmovdqu (%rdi), %ymm0
-; AVX2-NEXT: addl $256, %edx # imm = 0x100
-; AVX2-NEXT: vpor 16(%rdi), %xmm0, %xmm1
-; AVX2-NEXT: vptest %xmm1, %xmm1
-; AVX2-NEXT: cmovnel %eax, %edx
+; AVX2-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: cmovnel %edx, %r13d
+; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: tzcntq %rcx, %rdx
+; AVX2-NEXT: xorl %r12d, %r12d
+; AVX2-NEXT: tzcntq %r9, %r12
+; AVX2-NEXT: addl $64, %r12d
+; AVX2-NEXT: testq %rcx, %rcx
+; AVX2-NEXT: cmovnel %edx, %r12d
+; AVX2-NEXT: subl $-128, %r12d
+; AVX2-NEXT: movq %rsi, %rdx
+; AVX2-NEXT: orq %r10, %rdx
+; AVX2-NEXT: cmovnel %r13d, %r12d
+; AVX2-NEXT: addl $256, %r12d # imm = 0x100
+; AVX2-NEXT: movq %r11, %rdx
+; AVX2-NEXT: orq %rbp, %rdx
+; AVX2-NEXT: movq %r8, %r13
+; AVX2-NEXT: orq %rbx, %r13
+; AVX2-NEXT: orq %rdx, %r13
+; AVX2-NEXT: cmovnel %eax, %r12d
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: tzcntq %r15, %rdx
; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: tzcntq %r10, %rax
-; AVX2-NEXT: xorl %r11d, %r11d
-; AVX2-NEXT: tzcntq %r8, %r11
-; AVX2-NEXT: addl $64, %r11d
+; AVX2-NEXT: tzcntq %r14, %rax
+; AVX2-NEXT: addl $64, %eax
+; AVX2-NEXT: testq %r15, %r15
+; AVX2-NEXT: cmovnel %edx, %eax
+; AVX2-NEXT: movq 88(%rdi), %rbp
+; AVX2-NEXT: xorl %r13d, %r13d
+; AVX2-NEXT: tzcntq %rbp, %r13
+; AVX2-NEXT: addl $64, %r13d
+; AVX2-NEXT: movq 80(%rdi), %r10
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: tzcntq %r10, %rcx
; AVX2-NEXT: testq %r10, %r10
-; AVX2-NEXT: cmovnel %eax, %r11d
-; AVX2-NEXT: xorl %esi, %esi
-; AVX2-NEXT: tzcntq 88(%rdi), %rsi
-; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: tzcntq %r9, %rax
-; AVX2-NEXT: addl $64, %esi
-; AVX2-NEXT: testq %r9, %r9
-; AVX2-NEXT: movq 96(%rdi), %r9
-; AVX2-NEXT: cmovnel %eax, %esi
-; AVX2-NEXT: subl $-128, %esi
-; AVX2-NEXT: orq %r8, %r10
-; AVX2-NEXT: cmovnel %r11d, %esi
+; AVX2-NEXT: cmovnel %ecx, %r13d
+; AVX2-NEXT: subl $-128, %r13d
+; AVX2-NEXT: movq %r15, %rcx
+; AVX2-NEXT: orq %r14, %rcx
+; AVX2-NEXT: cmovnel %eax, %r13d
+; AVX2-NEXT: movq 104(%rdi), %r9
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: tzcntq %r9, %rcx
+; AVX2-NEXT: addl $64, %ecx
+; AVX2-NEXT: movq 96(%rdi), %rdx
; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: tzcntq %r9, %rax
-; AVX2-NEXT: xorl %r8d, %r8d
-; AVX2-NEXT: tzcntq %rcx, %r8
-; AVX2-NEXT: addl $64, %r8d
-; AVX2-NEXT: testq %r9, %r9
-; AVX2-NEXT: cmovnel %eax, %r8d
-; AVX2-NEXT: movq 112(%rdi), %r10
-; AVX2-NEXT: xorl %r11d, %r11d
-; AVX2-NEXT: tzcntq %r10, %r11
+; AVX2-NEXT: tzcntq %rdx, %rax
+; AVX2-NEXT: testq %rdx, %rdx
+; AVX2-NEXT: cmovnel %eax, %ecx
+; AVX2-NEXT: movq 112(%rdi), %rsi
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq 120(%rdi), %rax
; AVX2-NEXT: addl $64, %eax
-; AVX2-NEXT: testq %r10, %r10
-; AVX2-NEXT: cmovnel %r11d, %eax
+; AVX2-NEXT: tzcntq %rsi, %rdi
+; AVX2-NEXT: testq %rsi, %rsi
+; AVX2-NEXT: cmovnel %edi, %eax
; AVX2-NEXT: subl $-128, %eax
-; AVX2-NEXT: orq %rcx, %r9
-; AVX2-NEXT: cmovnel %r8d, %eax
-; AVX2-NEXT: vmovdqa 64(%rdi), %xmm1
+; AVX2-NEXT: orq %r9, %rdx
+; AVX2-NEXT: cmovnel %ecx, %eax
+; AVX2-NEXT: orq %rbp, %r14
+; AVX2-NEXT: orq %r10, %r15
; AVX2-NEXT: addl $256, %eax # imm = 0x100
-; AVX2-NEXT: vpor 80(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vptest %xmm1, %xmm1
-; AVX2-NEXT: cmovnel %esi, %eax
-; AVX2-NEXT: vpor 32(%rdi), %ymm0, %ymm0
+; AVX2-NEXT: orq %r14, %r15
+; AVX2-NEXT: cmovnel %r13d, %eax
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Folded Reload
+; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Folded Reload
+; AVX2-NEXT: orq %rcx, %r11
+; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Folded Reload
+; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Folded Reload
+; AVX2-NEXT: orq %rbx, %r8
; AVX2-NEXT: addl $512, %eax # imm = 0x200
-; AVX2-NEXT: vptest %ymm0, %ymm0
-; AVX2-NEXT: cmovnel %edx, %eax
+; AVX2-NEXT: orq %r11, %r8
+; AVX2-NEXT: cmovnel %r12d, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: popq %rbx
-; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: popq %r12
+; AVX2-NEXT: popq %r13
+; AVX2-NEXT: popq %r14
+; AVX2-NEXT: popq %r15
+; AVX2-NEXT: popq %rbp
; AVX2-NEXT: retq
;
; AVX512F-LABEL: load_cttz_i1024:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vmovdqu64 (%rdi), %zmm0
-; AVX512F-NEXT: vmovdqu64 64(%rdi), %zmm1
+; AVX512F-NEXT: vmovdqu64 64(%rdi), %zmm0
+; AVX512F-NEXT: vmovdqu64 (%rdi), %zmm1
+; AVX512F-NEXT: movq 16(%rdi), %rax
+; AVX512F-NEXT: movq (%rdi), %rcx
+; AVX512F-NEXT: movq 8(%rdi), %rdx
+; AVX512F-NEXT: movq 24(%rdi), %rsi
+; AVX512F-NEXT: orq 56(%rdi), %rsi
+; AVX512F-NEXT: orq 40(%rdi), %rdx
+; AVX512F-NEXT: orq 48(%rdi), %rax
+; AVX512F-NEXT: orq 32(%rdi), %rcx
+; AVX512F-NEXT: orq %rsi, %rdx
+; AVX512F-NEXT: orq %rax, %rcx
; AVX512F-NEXT: vpternlogd {{.*#+}} zmm2 = -1
-; AVX512F-NEXT: vpaddq %zmm2, %zmm0, %zmm3
-; AVX512F-NEXT: vpandnq %zmm3, %zmm0, %zmm3
+; AVX512F-NEXT: vpaddq %zmm2, %zmm1, %zmm3
+; AVX512F-NEXT: vpandnq %zmm3, %zmm1, %zmm3
; AVX512F-NEXT: vplzcntq %zmm3, %zmm3
; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm4 = [64,128,192,256,320,384,448,512]
; AVX512F-NEXT: vpsubq %zmm3, %zmm4, %zmm3
-; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
-; AVX512F-NEXT: vpcompressq %zmm3, %zmm3 {%k1} {z}
-; AVX512F-NEXT: vmovd %xmm3, %ecx
-; AVX512F-NEXT: vpaddq %zmm2, %zmm1, %zmm2
-; AVX512F-NEXT: vpandnq %zmm2, %zmm1, %zmm2
-; AVX512F-NEXT: vplzcntq %zmm2, %zmm2
-; AVX512F-NEXT: vpsubq %zmm2, %zmm4, %zmm2
; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
-; AVX512F-NEXT: vpcompressq %zmm2, %zmm1 {%k1}
-; AVX512F-NEXT: vmovd %xmm1, %eax
+; AVX512F-NEXT: vpcompressq %zmm3, %zmm1 {%k1} {z}
+; AVX512F-NEXT: vmovd %xmm1, %esi
+; AVX512F-NEXT: vpaddq %zmm2, %zmm0, %zmm1
+; AVX512F-NEXT: vpandnq %zmm1, %zmm0, %zmm1
+; AVX512F-NEXT: vplzcntq %zmm1, %zmm1
+; AVX512F-NEXT: vpsubq %zmm1, %zmm4, %zmm1
+; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
+; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
+; AVX512F-NEXT: vmovd %xmm0, %eax
; AVX512F-NEXT: addl $512, %eax # imm = 0x200
-; AVX512F-NEXT: vpor 32(%rdi), %ymm0, %ymm0
-; AVX512F-NEXT: vptest %ymm0, %ymm0
-; AVX512F-NEXT: cmovnel %ecx, %eax
+; AVX512F-NEXT: orq %rdx, %rcx
+; AVX512F-NEXT: cmovnel %esi, %eax
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: load_cttz_i1024:
; AVX512POPCNT: # %bb.0:
-; AVX512POPCNT-NEXT: vmovdqu64 (%rdi), %zmm0
-; AVX512POPCNT-NEXT: vmovdqu64 64(%rdi), %zmm1
-; AVX512POPCNT-NEXT: vpternlogd {{.*#+}} zmm2 = -1
-; AVX512POPCNT-NEXT: vpaddq %zmm2, %zmm0, %zmm3
-; AVX512POPCNT-NEXT: vpandnq %zmm3, %zmm0, %zmm3
+; AVX512POPCNT-NEXT: vmovdqu64 64(%rdi), %zmm0
+; AVX512POPCNT-NEXT: vmovdqu64 (%rdi), %zmm1
+; AVX512POPCNT-NEXT: movq 16(%rdi), %rax
+; AVX512POPCNT-NEXT: movq (%rdi), %rcx
+; AVX512POPCNT-NEXT: movq 8(%rdi), %rdx
+; AVX512POPCNT-NEXT: movq 24(%rdi), %rsi
+; AVX512POPCNT-NEXT: orq 56(%rdi), %rsi
+; AVX512POPCNT-NEXT: orq 40(%rdi), %rdx
+; AVX512POPCNT-NEXT: orq 48(%rdi), %rax
+; AVX512POPCNT-NEXT: orq 32(%rdi), %rcx
+; AVX512POPCNT-NEXT: orq %rsi, %rdx
+; AVX512POPCNT-NEXT: orq %rax, %rcx
+; AVX512POPCNT-NEXT: vpternlogd {{.*#+}} zmm2 = -1
+; AVX512POPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm3
+; AVX512POPCNT-NEXT: vpandnq %zmm3, %zmm1, %zmm3
; AVX512POPCNT-NEXT: vpopcntq %zmm3, %zmm3
; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm4 = [0,64,128,192,256,320,384,448]
; AVX512POPCNT-NEXT: vpaddq %zmm4, %zmm3, %zmm3
-; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
-; AVX512POPCNT-NEXT: vpcompressq %zmm3, %zmm3 {%k1} {z}
-; AVX512POPCNT-NEXT: vmovd %xmm3, %ecx
-; AVX512POPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm2
-; AVX512POPCNT-NEXT: vpandnq %zmm2, %zmm1, %zmm2
-; AVX512POPCNT-NEXT: vpopcntq %zmm2, %zmm2
-; AVX512POPCNT-NEXT: vpaddq %zmm4, %zmm2, %zmm2
; AVX512POPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512POPCNT-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
-; AVX512POPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1}
-; AVX512POPCNT-NEXT: vmovd %xmm1, %eax
+; AVX512POPCNT-NEXT: vpcompressq %zmm3, %zmm1 {%k1} {z}
+; AVX512POPCNT-NEXT: vmovd %xmm1, %esi
+; AVX512POPCNT-NEXT: vpaddq %zmm2, %zmm0, %zmm1
+; AVX512POPCNT-NEXT: vpandnq %zmm1, %zmm0, %zmm1
+; AVX512POPCNT-NEXT: vpopcntq %zmm1, %zmm1
+; AVX512POPCNT-NEXT: vpaddq %zmm4, %zmm1, %zmm1
+; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512POPCNT-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
+; AVX512POPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
+; AVX512POPCNT-NEXT: vmovd %xmm0, %eax
; AVX512POPCNT-NEXT: addl $512, %eax # imm = 0x200
-; AVX512POPCNT-NEXT: vpor 32(%rdi), %ymm0, %ymm0
-; AVX512POPCNT-NEXT: vptest %ymm0, %ymm0
-; AVX512POPCNT-NEXT: cmovnel %ecx, %eax
+; AVX512POPCNT-NEXT: orq %rdx, %rcx
+; AVX512POPCNT-NEXT: cmovnel %esi, %eax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: load_cttz_i1024:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vmovdqu64 (%rdi), %zmm0
-; AVX512VL-NEXT: vmovdqu64 64(%rdi), %zmm1
+; AVX512VL-NEXT: vmovdqu64 64(%rdi), %zmm0
+; AVX512VL-NEXT: vmovdqu64 (%rdi), %zmm1
+; AVX512VL-NEXT: movq 16(%rdi), %rax
+; AVX512VL-NEXT: movq (%rdi), %rcx
+; AVX512VL-NEXT: movq 8(%rdi), %rdx
+; AVX512VL-NEXT: movq 24(%rdi), %rsi
+; AVX512VL-NEXT: orq 56(%rdi), %rsi
+; AVX512VL-NEXT: orq 40(%rdi), %rdx
+; AVX512VL-NEXT: orq %rsi, %rdx
+; AVX512VL-NEXT: orq 48(%rdi), %rax
+; AVX512VL-NEXT: orq 32(%rdi), %rcx
+; AVX512VL-NEXT: orq %rax, %rcx
; AVX512VL-NEXT: vpternlogd {{.*#+}} zmm2 = -1
-; AVX512VL-NEXT: vpaddq %zmm2, %zmm0, %zmm3
-; AVX512VL-NEXT: vpandnq %zmm3, %zmm0, %zmm3
+; AVX512VL-NEXT: vpaddq %zmm2, %zmm1, %zmm3
+; AVX512VL-NEXT: vpandnq %zmm3, %zmm1, %zmm3
; AVX512VL-NEXT: vplzcntq %zmm3, %zmm3
; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm4 = [64,128,192,256,320,384,448,512]
; AVX512VL-NEXT: vpsubq %zmm3, %zmm4, %zmm3
-; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
-; AVX512VL-NEXT: vpcompressq %zmm3, %zmm3 {%k1} {z}
-; AVX512VL-NEXT: vmovd %xmm3, %ecx
-; AVX512VL-NEXT: vpaddq %zmm2, %zmm1, %zmm2
-; AVX512VL-NEXT: vpandnq %zmm2, %zmm1, %zmm2
-; AVX512VL-NEXT: vplzcntq %zmm2, %zmm2
-; AVX512VL-NEXT: vpsubq %zmm2, %zmm4, %zmm2
; AVX512VL-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512VL-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
-; AVX512VL-NEXT: vpcompressq %zmm2, %zmm1 {%k1}
-; AVX512VL-NEXT: vmovd %xmm1, %eax
+; AVX512VL-NEXT: vpcompressq %zmm3, %zmm1 {%k1} {z}
+; AVX512VL-NEXT: vmovd %xmm1, %esi
+; AVX512VL-NEXT: vpaddq %zmm2, %zmm0, %zmm1
+; AVX512VL-NEXT: vpandnq %zmm1, %zmm0, %zmm1
+; AVX512VL-NEXT: vplzcntq %zmm1, %zmm1
+; AVX512VL-NEXT: vpsubq %zmm1, %zmm4, %zmm1
+; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512VL-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
+; AVX512VL-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
+; AVX512VL-NEXT: vmovd %xmm0, %eax
; AVX512VL-NEXT: addl $512, %eax # imm = 0x200
-; AVX512VL-NEXT: vpor 32(%rdi), %ymm0, %ymm0
-; AVX512VL-NEXT: vptest %ymm0, %ymm0
-; AVX512VL-NEXT: cmovnel %ecx, %eax
+; AVX512VL-NEXT: orq %rdx, %rcx
+; AVX512VL-NEXT: cmovnel %esi, %eax
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: load_cttz_i1024:
; AVX512VLPOPCNT: # %bb.0:
-; AVX512VLPOPCNT-NEXT: vmovdqu64 (%rdi), %zmm0
-; AVX512VLPOPCNT-NEXT: vmovdqu64 64(%rdi), %zmm1
+; AVX512VLPOPCNT-NEXT: vmovdqu64 64(%rdi), %zmm0
+; AVX512VLPOPCNT-NEXT: vmovdqu64 (%rdi), %zmm1
+; AVX512VLPOPCNT-NEXT: movq 16(%rdi), %rax
+; AVX512VLPOPCNT-NEXT: movq (%rdi), %rcx
+; AVX512VLPOPCNT-NEXT: movq 8(%rdi), %rdx
+; AVX512VLPOPCNT-NEXT: movq 24(%rdi), %rsi
+; AVX512VLPOPCNT-NEXT: orq 56(%rdi), %rsi
+; AVX512VLPOPCNT-NEXT: orq 40(%rdi), %rdx
+; AVX512VLPOPCNT-NEXT: orq %rsi, %rdx
+; AVX512VLPOPCNT-NEXT: orq 48(%rdi), %rax
+; AVX512VLPOPCNT-NEXT: orq 32(%rdi), %rcx
+; AVX512VLPOPCNT-NEXT: orq %rax, %rcx
; AVX512VLPOPCNT-NEXT: vpternlogd {{.*#+}} zmm2 = -1
-; AVX512VLPOPCNT-NEXT: vpaddq %zmm2, %zmm0, %zmm3
-; AVX512VLPOPCNT-NEXT: vpandnq %zmm3, %zmm0, %zmm3
+; AVX512VLPOPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm3
+; AVX512VLPOPCNT-NEXT: vpandnq %zmm3, %zmm1, %zmm3
; AVX512VLPOPCNT-NEXT: vpopcntq %zmm3, %zmm3
; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm4 = [0,64,128,192,256,320,384,448]
; AVX512VLPOPCNT-NEXT: vpaddq %zmm4, %zmm3, %zmm3
-; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
-; AVX512VLPOPCNT-NEXT: vpcompressq %zmm3, %zmm3 {%k1} {z}
-; AVX512VLPOPCNT-NEXT: vmovd %xmm3, %ecx
-; AVX512VLPOPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm2
-; AVX512VLPOPCNT-NEXT: vpandnq %zmm2, %zmm1, %zmm2
-; AVX512VLPOPCNT-NEXT: vpopcntq %zmm2, %zmm2
-; AVX512VLPOPCNT-NEXT: vpaddq %zmm4, %zmm2, %zmm2
; AVX512VLPOPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512VLPOPCNT-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
-; AVX512VLPOPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1}
-; AVX512VLPOPCNT-NEXT: vmovd %xmm1, %eax
+; AVX512VLPOPCNT-NEXT: vpcompressq %zmm3, %zmm1 {%k1} {z}
+; AVX512VLPOPCNT-NEXT: vmovd %xmm1, %esi
+; AVX512VLPOPCNT-NEXT: vpaddq %zmm2, %zmm0, %zmm1
+; AVX512VLPOPCNT-NEXT: vpandnq %zmm1, %zmm0, %zmm1
+; AVX512VLPOPCNT-NEXT: vpopcntq %zmm1, %zmm1
+; AVX512VLPOPCNT-NEXT: vpaddq %zmm4, %zmm1, %zmm1
+; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512VLPOPCNT-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
+; AVX512VLPOPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
+; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
; AVX512VLPOPCNT-NEXT: addl $512, %eax # imm = 0x200
-; AVX512VLPOPCNT-NEXT: vpor 32(%rdi), %ymm0, %ymm0
-; AVX512VLPOPCNT-NEXT: vptest %ymm0, %ymm0
-; AVX512VLPOPCNT-NEXT: cmovnel %ecx, %eax
+; AVX512VLPOPCNT-NEXT: orq %rdx, %rcx
+; AVX512VLPOPCNT-NEXT: cmovnel %esi, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%a0 = load i1024, ptr %p0
@@ -7184,93 +7895,108 @@ define i32 @test_cttz_undef_i512(i512 %a0) nounwind {
define i32 @load_cttz_undef_i512(ptr %p0) nounwind {
; SSE-LABEL: load_cttz_undef_i512:
; SSE: # %bb.0:
+; SSE-NEXT: pushq %r15
+; SSE-NEXT: pushq %r14
; SSE-NEXT: pushq %rbx
-; SSE-NEXT: movdqa (%rdi), %xmm0
-; SSE-NEXT: movq 48(%rdi), %rsi
-; SSE-NEXT: movq 40(%rdi), %rcx
-; SSE-NEXT: movq 32(%rdi), %rdx
-; SSE-NEXT: movq 16(%rdi), %rax
-; SSE-NEXT: movq (%rdi), %r9
-; SSE-NEXT: movq 8(%rdi), %r10
-; SSE-NEXT: rep bsfq %r9, %r8
-; SSE-NEXT: rep bsfq %r10, %r11
+; SSE-NEXT: movq 40(%rdi), %r9
+; SSE-NEXT: movq 24(%rdi), %r8
+; SSE-NEXT: movq 16(%rdi), %rdx
+; SSE-NEXT: movq (%rdi), %rcx
+; SSE-NEXT: movq 8(%rdi), %rsi
+; SSE-NEXT: rep bsfq %rcx, %rax
+; SSE-NEXT: rep bsfq %rsi, %r11
; SSE-NEXT: addl $64, %r11d
-; SSE-NEXT: testq %r9, %r9
-; SSE-NEXT: cmovnel %r8d, %r11d
-; SSE-NEXT: rep bsfq %rax, %rbx
-; SSE-NEXT: rep bsfq 24(%rdi), %r8
-; SSE-NEXT: addl $64, %r8d
-; SSE-NEXT: testq %rax, %rax
-; SSE-NEXT: cmovnel %ebx, %r8d
-; SSE-NEXT: subl $-128, %r8d
-; SSE-NEXT: orq %r10, %r9
-; SSE-NEXT: cmovnel %r11d, %r8d
+; SSE-NEXT: testq %rcx, %rcx
+; SSE-NEXT: cmovnel %eax, %r11d
; SSE-NEXT: rep bsfq %rdx, %rax
-; SSE-NEXT: rep bsfq %rcx, %r9
-; SSE-NEXT: addl $64, %r9d
+; SSE-NEXT: rep bsfq %r8, %r10
+; SSE-NEXT: addl $64, %r10d
; SSE-NEXT: testq %rdx, %rdx
-; SSE-NEXT: cmovnel %eax, %r9d
-; SSE-NEXT: rep bsfq %rsi, %r10
+; SSE-NEXT: cmovnel %eax, %r10d
+; SSE-NEXT: movq 32(%rdi), %rbx
+; SSE-NEXT: subl $-128, %r10d
+; SSE-NEXT: movq %rcx, %rax
+; SSE-NEXT: orq %rsi, %rax
+; SSE-NEXT: cmovnel %r11d, %r10d
+; SSE-NEXT: rep bsfq %rbx, %rax
+; SSE-NEXT: rep bsfq %r9, %r11
+; SSE-NEXT: addl $64, %r11d
+; SSE-NEXT: testq %rbx, %rbx
+; SSE-NEXT: cmovnel %eax, %r11d
+; SSE-NEXT: movq 48(%rdi), %r14
+; SSE-NEXT: rep bsfq %r14, %r15
; SSE-NEXT: rep bsfq 56(%rdi), %rax
; SSE-NEXT: addl $64, %eax
-; SSE-NEXT: testq %rsi, %rsi
-; SSE-NEXT: cmovnel %r10d, %eax
+; SSE-NEXT: testq %r14, %r14
+; SSE-NEXT: cmovnel %r15d, %eax
; SSE-NEXT: subl $-128, %eax
-; SSE-NEXT: orq %rcx, %rdx
-; SSE-NEXT: cmovnel %r9d, %eax
+; SSE-NEXT: orq %r9, %rbx
+; SSE-NEXT: cmovnel %r11d, %eax
; SSE-NEXT: addl $256, %eax # imm = 0x100
-; SSE-NEXT: por 16(%rdi), %xmm0
-; SSE-NEXT: ptest %xmm0, %xmm0
-; SSE-NEXT: cmovnel %r8d, %eax
+; SSE-NEXT: orq %r8, %rsi
+; SSE-NEXT: orq %rdx, %rcx
+; SSE-NEXT: orq %rsi, %rcx
+; SSE-NEXT: cmovnel %r10d, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: popq %rbx
+; SSE-NEXT: popq %r14
+; SSE-NEXT: popq %r15
; SSE-NEXT: retq
;
; AVX2-LABEL: load_cttz_undef_i512:
; AVX2: # %bb.0:
-; AVX2-NEXT: movq 40(%rdi), %rcx
-; AVX2-NEXT: movq 32(%rdi), %rdx
-; AVX2-NEXT: movq 16(%rdi), %rax
-; AVX2-NEXT: movq (%rdi), %r8
-; AVX2-NEXT: movq 8(%rdi), %r9
-; AVX2-NEXT: tzcntq %r8, %rsi
-; AVX2-NEXT: tzcntq %r9, %r10
-; AVX2-NEXT: addl $64, %r10d
-; AVX2-NEXT: testq %r8, %r8
-; AVX2-NEXT: cmovnel %esi, %r10d
-; AVX2-NEXT: tzcntq %rax, %r11
-; AVX2-NEXT: xorl %esi, %esi
-; AVX2-NEXT: tzcntq 24(%rdi), %rsi
-; AVX2-NEXT: addl $64, %esi
-; AVX2-NEXT: testq %rax, %rax
-; AVX2-NEXT: cmovnel %r11d, %esi
-; AVX2-NEXT: subl $-128, %esi
-; AVX2-NEXT: orq %r9, %r8
-; AVX2-NEXT: cmovnel %r10d, %esi
+; AVX2-NEXT: pushq %r15
+; AVX2-NEXT: pushq %r14
+; AVX2-NEXT: pushq %rbx
+; AVX2-NEXT: movq 48(%rdi), %r10
+; AVX2-NEXT: movq 40(%rdi), %r9
+; AVX2-NEXT: movq 24(%rdi), %r8
+; AVX2-NEXT: movq 16(%rdi), %rdx
+; AVX2-NEXT: movq (%rdi), %rcx
+; AVX2-NEXT: movq 8(%rdi), %rsi
+; AVX2-NEXT: tzcntq %rcx, %rax
+; AVX2-NEXT: xorl %ebx, %ebx
+; AVX2-NEXT: tzcntq %rsi, %rbx
+; AVX2-NEXT: addl $64, %ebx
+; AVX2-NEXT: testq %rcx, %rcx
+; AVX2-NEXT: cmovnel %eax, %ebx
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq %rdx, %rax
-; AVX2-NEXT: xorl %r8d, %r8d
-; AVX2-NEXT: tzcntq %rcx, %r8
-; AVX2-NEXT: addl $64, %r8d
+; AVX2-NEXT: tzcntq %r8, %r11
+; AVX2-NEXT: addl $64, %r11d
; AVX2-NEXT: testq %rdx, %rdx
-; AVX2-NEXT: cmovnel %eax, %r8d
-; AVX2-NEXT: movq 48(%rdi), %r9
-; AVX2-NEXT: xorl %r10d, %r10d
-; AVX2-NEXT: tzcntq %r9, %r10
+; AVX2-NEXT: cmovnel %eax, %r11d
+; AVX2-NEXT: subl $-128, %r11d
+; AVX2-NEXT: movq %rcx, %rax
+; AVX2-NEXT: orq %rsi, %rax
+; AVX2-NEXT: cmovnel %ebx, %r11d
+; AVX2-NEXT: movq 32(%rdi), %rbx
+; AVX2-NEXT: xorl %eax, %eax
+; AVX2-NEXT: tzcntq %rbx, %rax
+; AVX2-NEXT: xorl %r14d, %r14d
+; AVX2-NEXT: tzcntq %r9, %r14
+; AVX2-NEXT: addl $64, %r14d
+; AVX2-NEXT: testq %rbx, %rbx
+; AVX2-NEXT: cmovnel %eax, %r14d
+; AVX2-NEXT: xorl %r15d, %r15d
+; AVX2-NEXT: tzcntq %r10, %r15
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq 56(%rdi), %rax
; AVX2-NEXT: addl $64, %eax
-; AVX2-NEXT: testq %r9, %r9
-; AVX2-NEXT: cmovnel %r10d, %eax
+; AVX2-NEXT: testq %r10, %r10
+; AVX2-NEXT: cmovnel %r15d, %eax
; AVX2-NEXT: subl $-128, %eax
-; AVX2-NEXT: orq %rcx, %rdx
-; AVX2-NEXT: cmovnel %r8d, %eax
-; AVX2-NEXT: vmovdqa (%rdi), %xmm0
+; AVX2-NEXT: orq %r9, %rbx
+; AVX2-NEXT: cmovnel %r14d, %eax
; AVX2-NEXT: addl $256, %eax # imm = 0x100
-; AVX2-NEXT: vpor 16(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vptest %xmm0, %xmm0
-; AVX2-NEXT: cmovnel %esi, %eax
+; AVX2-NEXT: orq %r8, %rsi
+; AVX2-NEXT: orq %rdx, %rcx
+; AVX2-NEXT: orq %rsi, %rcx
+; AVX2-NEXT: cmovnel %r11d, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
+; AVX2-NEXT: popq %rbx
+; AVX2-NEXT: popq %r14
+; AVX2-NEXT: popq %r15
; AVX2-NEXT: retq
;
; AVX512F-LABEL: load_cttz_undef_i512:
@@ -7493,92 +8219,95 @@ define i32 @test_cttz_undef_i1024(i1024 %a0) nounwind {
; SSE-NEXT: pushq %r13
; SSE-NEXT: pushq %r12
; SSE-NEXT: pushq %rbx
-; SSE-NEXT: movq %r9, %r10
-; SSE-NEXT: movq %r8, %r9
-; SSE-NEXT: movq %rcx, %r8
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rcx
-; SSE-NEXT: rep bsfq %rdi, %r11
-; SSE-NEXT: rep bsfq %rsi, %rbx
-; SSE-NEXT: addl $64, %ebx
-; SSE-NEXT: testq %rdi, %rdi
-; SSE-NEXT: cmovnel %r11d, %ebx
+; SSE-NEXT: movq %r9, %r14
+; SSE-NEXT: movq %rcx, %rbx
+; SSE-NEXT: movq %rdx, %r10
+; SSE-NEXT: movq %rsi, %r9
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rdx
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r11
-; SSE-NEXT: rep bsfq %rdx, %r15
-; SSE-NEXT: rep bsfq %r8, %r14
-; SSE-NEXT: addl $64, %r14d
-; SSE-NEXT: testq %rdx, %rdx
-; SSE-NEXT: cmovnel %r15d, %r14d
-; SSE-NEXT: subl $-128, %r14d
-; SSE-NEXT: movq %rdi, %r15
-; SSE-NEXT: orq %rsi, %r15
-; SSE-NEXT: cmovnel %ebx, %r14d
-; SSE-NEXT: rep bsfq %r9, %rbx
-; SSE-NEXT: rep bsfq %r10, %r12
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rcx
+; SSE-NEXT: rep bsfq %rdi, %rax
+; SSE-NEXT: rep bsfq %rsi, %r12
; SSE-NEXT: addl $64, %r12d
-; SSE-NEXT: testq %r9, %r9
-; SSE-NEXT: cmovnel %ebx, %r12d
-; SSE-NEXT: rep bsfq %rcx, %r15
-; SSE-NEXT: rep bsfq %r11, %rbx
-; SSE-NEXT: addl $64, %ebx
-; SSE-NEXT: testq %rcx, %rcx
-; SSE-NEXT: cmovnel %r15d, %ebx
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r15
-; SSE-NEXT: subl $-128, %ebx
-; SSE-NEXT: movq %r9, %r13
-; SSE-NEXT: orq %r10, %r13
-; SSE-NEXT: cmovnel %r12d, %ebx
-; SSE-NEXT: addl $256, %ebx # imm = 0x100
-; SSE-NEXT: movq %rsi, %r12
-; SSE-NEXT: orq %r8, %r12
+; SSE-NEXT: testq %rdi, %rdi
+; SSE-NEXT: cmovnel %eax, %r12d
+; SSE-NEXT: rep bsfq %r10, %r15
+; SSE-NEXT: rep bsfq %rbx, %rax
+; SSE-NEXT: addl $64, %eax
+; SSE-NEXT: testq %r10, %r10
+; SSE-NEXT: cmovnel %r15d, %eax
+; SSE-NEXT: subl $-128, %eax
; SSE-NEXT: movq %rdi, %r13
-; SSE-NEXT: orq %rdx, %r13
-; SSE-NEXT: orq %r12, %r13
-; SSE-NEXT: cmovnel %r14d, %ebx
-; SSE-NEXT: rep bsfq %rax, %r14
-; SSE-NEXT: rep bsfq %r15, %r13
+; SSE-NEXT: orq %rsi, %r13
+; SSE-NEXT: cmovnel %r12d, %eax
+; SSE-NEXT: movq %r8, %r15
+; SSE-NEXT: rep bsfq %r8, %r12
+; SSE-NEXT: movq %r14, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE-NEXT: rep bsfq %r14, %r13
; SSE-NEXT: addl $64, %r13d
-; SSE-NEXT: testq %rax, %rax
-; SSE-NEXT: cmovnel %r14d, %r13d
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r12
-; SSE-NEXT: rep bsfq %r12, %rbp
-; SSE-NEXT: rep bsfq {{[0-9]+}}(%rsp), %r14
-; SSE-NEXT: addl $64, %r14d
-; SSE-NEXT: testq %r12, %r12
-; SSE-NEXT: cmovnel %ebp, %r14d
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r12
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rbp
-; SSE-NEXT: subl $-128, %r14d
-; SSE-NEXT: orq %r15, %rax
-; SSE-NEXT: cmovnel %r13d, %r14d
-; SSE-NEXT: rep bsfq %r12, %rax
-; SSE-NEXT: rep bsfq %rbp, %r15
-; SSE-NEXT: addl $64, %r15d
-; SSE-NEXT: testq %r12, %r12
-; SSE-NEXT: cmovnel %eax, %r15d
+; SSE-NEXT: testq %r8, %r8
+; SSE-NEXT: cmovnel %r12d, %r13d
+; SSE-NEXT: rep bsfq %rcx, %rbp
+; SSE-NEXT: rep bsfq {{[0-9]+}}(%rsp), %r12
+; SSE-NEXT: addl $64, %r12d
+; SSE-NEXT: testq %rcx, %rcx
+; SSE-NEXT: cmovnel %ebp, %r12d
+; SSE-NEXT: subl $-128, %r12d
+; SSE-NEXT: movq %r8, %rbp
+; SSE-NEXT: orq %r14, %rbp
+; SSE-NEXT: cmovnel %r13d, %r12d
+; SSE-NEXT: addl $256, %r12d # imm = 0x100
+; SSE-NEXT: movq %rsi, %r13
+; SSE-NEXT: orq %rbx, %r13
+; SSE-NEXT: movq %rdi, %rbp
+; SSE-NEXT: orq %r10, %rbp
+; SSE-NEXT: orq %r13, %rbp
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r13
+; SSE-NEXT: cmovnel %eax, %r12d
+; SSE-NEXT: rep bsfq %r11, %rbp
+; SSE-NEXT: rep bsfq %r13, %rax
+; SSE-NEXT: addl $64, %eax
+; SSE-NEXT: testq %r11, %r11
+; SSE-NEXT: cmovnel %ebp, %eax
+; SSE-NEXT: rep bsfq {{[0-9]+}}(%rsp), %rbp
+; SSE-NEXT: addl $64, %ebp
+; SSE-NEXT: rep bsfq %rdx, %rcx
+; SSE-NEXT: testq %rdx, %rdx
+; SSE-NEXT: cmovnel %ecx, %ebp
+; SSE-NEXT: subl $-128, %ebp
+; SSE-NEXT: movq %r11, %rcx
+; SSE-NEXT: orq %r13, %rcx
+; SSE-NEXT: cmovnel %eax, %ebp
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r14
+; SSE-NEXT: rep bsfq %r14, %rcx
+; SSE-NEXT: addl $64, %ecx
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; SSE-NEXT: rep bsfq %rdx, %rax
+; SSE-NEXT: testq %rdx, %rdx
+; SSE-NEXT: cmovnel %eax, %ecx
; SSE-NEXT: rep bsfq {{[0-9]+}}(%rsp), %rax
; SSE-NEXT: addl $64, %eax
-; SSE-NEXT: rep bsfq %r13, %rcx
-; SSE-NEXT: testq %r13, %r13
-; SSE-NEXT: cmovnel %ecx, %eax
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r8
+; SSE-NEXT: rep bsfq %r8, %rsi
+; SSE-NEXT: testq %r8, %r8
+; SSE-NEXT: cmovnel %esi, %eax
; SSE-NEXT: subl $-128, %eax
-; SSE-NEXT: orq %rbp, %r12
-; SSE-NEXT: cmovnel %r15d, %eax
-; SSE-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm0
+; SSE-NEXT: orq %r14, %rdx
+; SSE-NEXT: cmovnel %ecx, %eax
+; SSE-NEXT: orq {{[0-9]+}}(%rsp), %r13
+; SSE-NEXT: orq {{[0-9]+}}(%rsp), %r11
; SSE-NEXT: addl $256, %eax # imm = 0x100
-; SSE-NEXT: por {{[0-9]+}}(%rsp), %xmm0
-; SSE-NEXT: ptest %xmm0, %xmm0
-; SSE-NEXT: cmovnel %r14d, %eax
-; SSE-NEXT: orq %r11, %r8
-; SSE-NEXT: orq %r10, %rsi
-; SSE-NEXT: orq %r8, %rsi
-; SSE-NEXT: orq {{[0-9]+}}(%rsp), %rdx
-; SSE-NEXT: orq %r9, %rdi
-; SSE-NEXT: orq %rdx, %rdi
+; SSE-NEXT: orq %r13, %r11
+; SSE-NEXT: cmovnel %ebp, %eax
+; SSE-NEXT: orq {{[0-9]+}}(%rsp), %rbx
+; SSE-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r9 # 8-byte Folded Reload
+; SSE-NEXT: orq %rbx, %r9
+; SSE-NEXT: orq {{[0-9]+}}(%rsp), %r10
+; SSE-NEXT: orq %r15, %rdi
+; SSE-NEXT: orq %r10, %rdi
; SSE-NEXT: addl $512, %eax # imm = 0x200
-; SSE-NEXT: orq %rsi, %rdi
-; SSE-NEXT: cmovnel %ebx, %eax
+; SSE-NEXT: orq %r9, %rdi
+; SSE-NEXT: cmovnel %r12d, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: popq %rbx
; SSE-NEXT: popq %r12
@@ -7596,108 +8325,111 @@ define i32 @test_cttz_undef_i1024(i1024 %a0) nounwind {
; AVX2-NEXT: pushq %r13
; AVX2-NEXT: pushq %r12
; AVX2-NEXT: pushq %rbx
-; AVX2-NEXT: movq %r9, %r10
-; AVX2-NEXT: movq %r8, %r9
-; AVX2-NEXT: movq %rcx, %r8
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r14
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX2-NEXT: movq %r9, %rbx
+; AVX2-NEXT: movq %r8, %r14
+; AVX2-NEXT: movq %rcx, %r11
+; AVX2-NEXT: movq %rdx, %r10
+; AVX2-NEXT: movq %rsi, %r9
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r8
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rsi
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rcx
-; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: tzcntq %rdi, %rbx
-; AVX2-NEXT: xorl %r12d, %r12d
-; AVX2-NEXT: tzcntq %rsi, %r12
-; AVX2-NEXT: addl $64, %r12d
-; AVX2-NEXT: testq %rdi, %rdi
-; AVX2-NEXT: cmovnel %ebx, %r12d
-; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: tzcntq %rdx, %rbx
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; AVX2-NEXT: tzcntq %rdi, %rax
; AVX2-NEXT: xorl %r15d, %r15d
-; AVX2-NEXT: tzcntq %r8, %r15
+; AVX2-NEXT: tzcntq %r9, %r15
; AVX2-NEXT: addl $64, %r15d
-; AVX2-NEXT: testq %rdx, %rdx
-; AVX2-NEXT: cmovnel %ebx, %r15d
-; AVX2-NEXT: subl $-128, %r15d
-; AVX2-NEXT: movq %rdi, %rbx
-; AVX2-NEXT: orq %rsi, %rbx
-; AVX2-NEXT: cmovnel %r12d, %r15d
-; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: tzcntq %r9, %rbx
+; AVX2-NEXT: testq %rdi, %rdi
+; AVX2-NEXT: cmovnel %eax, %r15d
; AVX2-NEXT: xorl %r12d, %r12d
; AVX2-NEXT: tzcntq %r10, %r12
+; AVX2-NEXT: xorl %eax, %eax
+; AVX2-NEXT: tzcntq %r11, %rax
+; AVX2-NEXT: addl $64, %eax
+; AVX2-NEXT: testq %r10, %r10
+; AVX2-NEXT: cmovnel %r12d, %eax
+; AVX2-NEXT: subl $-128, %eax
+; AVX2-NEXT: movq %rdi, %r12
+; AVX2-NEXT: orq %r9, %r12
+; AVX2-NEXT: cmovnel %r15d, %eax
+; AVX2-NEXT: xorl %r15d, %r15d
+; AVX2-NEXT: tzcntq %r14, %r15
+; AVX2-NEXT: movq %rbx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: xorl %r12d, %r12d
+; AVX2-NEXT: tzcntq %rbx, %r12
; AVX2-NEXT: addl $64, %r12d
-; AVX2-NEXT: testq %r9, %r9
-; AVX2-NEXT: cmovnel %ebx, %r12d
+; AVX2-NEXT: testq %r14, %r14
+; AVX2-NEXT: cmovnel %r15d, %r12d
; AVX2-NEXT: xorl %r13d, %r13d
-; AVX2-NEXT: tzcntq %r11, %r13
-; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: tzcntq %rcx, %rbx
-; AVX2-NEXT: addl $64, %ebx
-; AVX2-NEXT: testq %r11, %r11
-; AVX2-NEXT: cmovnel %r13d, %ebx
-; AVX2-NEXT: subl $-128, %ebx
-; AVX2-NEXT: movq %r9, %r13
-; AVX2-NEXT: orq %r10, %r13
-; AVX2-NEXT: cmovnel %r12d, %ebx
-; AVX2-NEXT: addl $256, %ebx # imm = 0x100
-; AVX2-NEXT: movq %rsi, %r12
-; AVX2-NEXT: orq %r8, %r12
-; AVX2-NEXT: movq %rdi, %r13
-; AVX2-NEXT: orq %rdx, %r13
-; AVX2-NEXT: orq %r12, %r13
-; AVX2-NEXT: cmovnel %r15d, %ebx
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r12
+; AVX2-NEXT: tzcntq %rcx, %r13
; AVX2-NEXT: xorl %r15d, %r15d
-; AVX2-NEXT: tzcntq %rax, %r15
-; AVX2-NEXT: xorl %r13d, %r13d
-; AVX2-NEXT: tzcntq %r12, %r13
-; AVX2-NEXT: addl $64, %r13d
-; AVX2-NEXT: testq %rax, %rax
-; AVX2-NEXT: cmovnel %r15d, %r13d
+; AVX2-NEXT: tzcntq %rdx, %r15
+; AVX2-NEXT: addl $64, %r15d
+; AVX2-NEXT: testq %rcx, %rcx
+; AVX2-NEXT: cmovnel %r13d, %r15d
+; AVX2-NEXT: subl $-128, %r15d
+; AVX2-NEXT: movq %r14, %r13
+; AVX2-NEXT: orq %rbx, %r13
+; AVX2-NEXT: cmovnel %r12d, %r15d
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r12
+; AVX2-NEXT: addl $256, %r15d # imm = 0x100
+; AVX2-NEXT: movq %r9, %r13
+; AVX2-NEXT: orq %r11, %r13
+; AVX2-NEXT: movq %rdi, %rbp
+; AVX2-NEXT: orq %r10, %rbp
+; AVX2-NEXT: orq %r13, %rbp
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r13
+; AVX2-NEXT: cmovnel %eax, %r15d
+; AVX2-NEXT: xorl %ebp, %ebp
+; AVX2-NEXT: tzcntq %r12, %rbp
+; AVX2-NEXT: xorl %eax, %eax
+; AVX2-NEXT: tzcntq %r13, %rax
+; AVX2-NEXT: addl $64, %eax
+; AVX2-NEXT: testq %r12, %r12
+; AVX2-NEXT: cmovnel %ebp, %eax
; AVX2-NEXT: xorl %ebp, %ebp
-; AVX2-NEXT: tzcntq %r14, %rbp
-; AVX2-NEXT: xorl %r15d, %r15d
-; AVX2-NEXT: tzcntq {{[0-9]+}}(%rsp), %r15
-; AVX2-NEXT: addl $64, %r15d
-; AVX2-NEXT: testq %r14, %r14
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r14
-; AVX2-NEXT: cmovnel %ebp, %r15d
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rbp
-; AVX2-NEXT: subl $-128, %r15d
-; AVX2-NEXT: orq %r12, %rax
-; AVX2-NEXT: cmovnel %r13d, %r15d
+; AVX2-NEXT: tzcntq %r8, %rbp
+; AVX2-NEXT: addl $64, %ebp
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: tzcntq %rsi, %rcx
+; AVX2-NEXT: testq %rsi, %rsi
+; AVX2-NEXT: cmovnel %ecx, %ebp
+; AVX2-NEXT: subl $-128, %ebp
+; AVX2-NEXT: movq %r12, %rcx
+; AVX2-NEXT: orq %r13, %rcx
+; AVX2-NEXT: cmovnel %eax, %ebp
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: tzcntq %rbx, %rcx
+; AVX2-NEXT: addl $64, %ecx
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdx
; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: tzcntq %rbp, %rax
-; AVX2-NEXT: xorl %r12d, %r12d
-; AVX2-NEXT: tzcntq %r14, %r12
-; AVX2-NEXT: addl $64, %r12d
-; AVX2-NEXT: testq %rbp, %rbp
-; AVX2-NEXT: cmovnel %eax, %r12d
+; AVX2-NEXT: tzcntq %rdx, %rax
+; AVX2-NEXT: testq %rdx, %rdx
+; AVX2-NEXT: cmovnel %eax, %ecx
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq {{[0-9]+}}(%rsp), %rax
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r13
; AVX2-NEXT: addl $64, %eax
-; AVX2-NEXT: xorl %ecx, %ecx
-; AVX2-NEXT: tzcntq %r13, %rcx
-; AVX2-NEXT: testq %r13, %r13
-; AVX2-NEXT: cmovnel %ecx, %eax
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r8
+; AVX2-NEXT: tzcntq %r8, %rsi
+; AVX2-NEXT: testq %r8, %r8
+; AVX2-NEXT: cmovnel %esi, %eax
; AVX2-NEXT: subl $-128, %eax
-; AVX2-NEXT: orq %r14, %rbp
-; AVX2-NEXT: cmovnel %r12d, %eax
-; AVX2-NEXT: vmovdqa {{[0-9]+}}(%rsp), %xmm0
+; AVX2-NEXT: orq %rbx, %rdx
+; AVX2-NEXT: cmovnel %ecx, %eax
+; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r13
+; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r12
; AVX2-NEXT: addl $256, %eax # imm = 0x100
-; AVX2-NEXT: vpor {{[0-9]+}}(%rsp), %xmm0, %xmm0
-; AVX2-NEXT: vptest %xmm0, %xmm0
-; AVX2-NEXT: cmovnel %r15d, %eax
-; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r8
-; AVX2-NEXT: orq %r10, %rsi
-; AVX2-NEXT: orq %r8, %rsi
-; AVX2-NEXT: orq %r11, %rdx
-; AVX2-NEXT: orq %r9, %rdi
-; AVX2-NEXT: orq %rdx, %rdi
+; AVX2-NEXT: orq %r13, %r12
+; AVX2-NEXT: cmovnel %ebp, %eax
+; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r11
+; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r9 # 8-byte Folded Reload
+; AVX2-NEXT: orq %r11, %r9
+; AVX2-NEXT: orq {{[0-9]+}}(%rsp), %r10
+; AVX2-NEXT: orq %r14, %rdi
+; AVX2-NEXT: orq %r10, %rdi
; AVX2-NEXT: addl $512, %eax # imm = 0x200
-; AVX2-NEXT: orq %rsi, %rdi
-; AVX2-NEXT: cmovnel %ebx, %eax
+; AVX2-NEXT: orq %r9, %rdi
+; AVX2-NEXT: cmovnel %r15d, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: popq %rbx
; AVX2-NEXT: popq %r12
@@ -7884,296 +8616,384 @@ define i32 @test_cttz_undef_i1024(i1024 %a0) nounwind {
define i32 @load_cttz_undef_i1024(ptr %p0) nounwind {
; SSE-LABEL: load_cttz_undef_i1024:
; SSE: # %bb.0:
+; SSE-NEXT: pushq %rbp
+; SSE-NEXT: pushq %r15
+; SSE-NEXT: pushq %r14
+; SSE-NEXT: pushq %r13
+; SSE-NEXT: pushq %r12
; SSE-NEXT: pushq %rbx
-; SSE-NEXT: movq 104(%rdi), %rcx
-; SSE-NEXT: movq 40(%rdi), %rax
-; SSE-NEXT: movq 16(%rdi), %rdx
+; SSE-NEXT: movq 72(%rdi), %rbx
+; SSE-NEXT: movq 56(%rdi), %r9
+; SSE-NEXT: movq %r9, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE-NEXT: movq 48(%rdi), %rcx
+; SSE-NEXT: movq 40(%rdi), %r10
+; SSE-NEXT: movq %r10, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE-NEXT: movq 32(%rdi), %rsi
+; SSE-NEXT: movq 24(%rdi), %rbp
; SSE-NEXT: movq (%rdi), %r8
-; SSE-NEXT: movq 8(%rdi), %r9
-; SSE-NEXT: rep bsfq %r8, %rsi
-; SSE-NEXT: rep bsfq %r9, %r11
-; SSE-NEXT: addl $64, %r11d
-; SSE-NEXT: testq %r8, %r8
-; SSE-NEXT: cmovnel %esi, %r11d
-; SSE-NEXT: rep bsfq 24(%rdi), %rsi
-; SSE-NEXT: rep bsfq %rdx, %rbx
-; SSE-NEXT: addl $64, %esi
-; SSE-NEXT: testq %rdx, %rdx
-; SSE-NEXT: movq 32(%rdi), %r10
-; SSE-NEXT: cmovnel %ebx, %esi
-; SSE-NEXT: subl $-128, %esi
-; SSE-NEXT: orq %r9, %r8
-; SSE-NEXT: cmovnel %r11d, %esi
-; SSE-NEXT: rep bsfq %r10, %rdx
-; SSE-NEXT: rep bsfq %rax, %r11
-; SSE-NEXT: addl $64, %r11d
-; SSE-NEXT: testq %r10, %r10
-; SSE-NEXT: cmovnel %edx, %r11d
-; SSE-NEXT: movq 48(%rdi), %r8
-; SSE-NEXT: rep bsfq %r8, %r9
-; SSE-NEXT: rep bsfq 56(%rdi), %rdx
+; SSE-NEXT: movq 8(%rdi), %r11
+; SSE-NEXT: rep bsfq %r8, %rax
+; SSE-NEXT: rep bsfq %r11, %rdx
; SSE-NEXT: addl $64, %edx
; SSE-NEXT: testq %r8, %r8
-; SSE-NEXT: cmovnel %r9d, %edx
-; SSE-NEXT: movq 80(%rdi), %r9
-; SSE-NEXT: movq 72(%rdi), %r8
-; SSE-NEXT: subl $-128, %edx
-; SSE-NEXT: orq %rax, %r10
-; SSE-NEXT: movq 64(%rdi), %rax
-; SSE-NEXT: movdqa 16(%rdi), %xmm0
-; SSE-NEXT: cmovnel %r11d, %edx
-; SSE-NEXT: movdqa (%rdi), %xmm1
-; SSE-NEXT: addl $256, %edx # imm = 0x100
-; SSE-NEXT: movdqa %xmm1, %xmm2
-; SSE-NEXT: por %xmm0, %xmm2
-; SSE-NEXT: ptest %xmm2, %xmm2
-; SSE-NEXT: cmovnel %esi, %edx
-; SSE-NEXT: rep bsfq %rax, %rsi
-; SSE-NEXT: rep bsfq %r8, %r10
-; SSE-NEXT: addl $64, %r10d
-; SSE-NEXT: testq %rax, %rax
-; SSE-NEXT: cmovnel %esi, %r10d
-; SSE-NEXT: rep bsfq 88(%rdi), %rsi
-; SSE-NEXT: rep bsfq %r9, %r11
-; SSE-NEXT: addl $64, %esi
-; SSE-NEXT: testq %r9, %r9
-; SSE-NEXT: movq 96(%rdi), %r9
-; SSE-NEXT: cmovnel %r11d, %esi
-; SSE-NEXT: subl $-128, %esi
-; SSE-NEXT: orq %r8, %rax
-; SSE-NEXT: cmovnel %r10d, %esi
-; SSE-NEXT: rep bsfq %r9, %rax
-; SSE-NEXT: rep bsfq %rcx, %r8
-; SSE-NEXT: addl $64, %r8d
-; SSE-NEXT: testq %r9, %r9
-; SSE-NEXT: cmovnel %eax, %r8d
-; SSE-NEXT: movq 112(%rdi), %r10
-; SSE-NEXT: rep bsfq %r10, %r11
-; SSE-NEXT: rep bsfq 120(%rdi), %rax
+; SSE-NEXT: cmovnel %eax, %edx
+; SSE-NEXT: movq 16(%rdi), %r14
+; SSE-NEXT: rep bsfq %r14, %r15
+; SSE-NEXT: rep bsfq %rbp, %rax
+; SSE-NEXT: movq %rbp, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE-NEXT: addl $64, %eax
+; SSE-NEXT: testq %r14, %r14
+; SSE-NEXT: cmovnel %r15d, %eax
+; SSE-NEXT: subl $-128, %eax
+; SSE-NEXT: movq %r8, %r15
+; SSE-NEXT: orq %r11, %r15
+; SSE-NEXT: cmovnel %edx, %eax
+; SSE-NEXT: rep bsfq %rsi, %rdx
+; SSE-NEXT: rep bsfq %r10, %r13
+; SSE-NEXT: addl $64, %r13d
+; SSE-NEXT: testq %rsi, %rsi
+; SSE-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE-NEXT: cmovnel %edx, %r13d
+; SSE-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; SSE-NEXT: rep bsfq %rcx, %rdx
+; SSE-NEXT: rep bsfq %r9, %r15
+; SSE-NEXT: addl $64, %r15d
+; SSE-NEXT: testq %rcx, %rcx
+; SSE-NEXT: cmovnel %edx, %r15d
+; SSE-NEXT: movq 64(%rdi), %r12
+; SSE-NEXT: subl $-128, %r15d
+; SSE-NEXT: movq %rsi, %rdx
+; SSE-NEXT: orq %r10, %rdx
+; SSE-NEXT: cmovnel %r13d, %r15d
+; SSE-NEXT: addl $256, %r15d # imm = 0x100
+; SSE-NEXT: movq %r11, %rdx
+; SSE-NEXT: orq %rbp, %rdx
+; SSE-NEXT: movq %r8, %r13
+; SSE-NEXT: orq %r14, %r13
+; SSE-NEXT: orq %rdx, %r13
+; SSE-NEXT: cmovnel %eax, %r15d
+; SSE-NEXT: rep bsfq %r12, %rdx
+; SSE-NEXT: rep bsfq %rbx, %rax
; SSE-NEXT: addl $64, %eax
+; SSE-NEXT: testq %r12, %r12
+; SSE-NEXT: cmovnel %edx, %eax
+; SSE-NEXT: movq 88(%rdi), %rbp
+; SSE-NEXT: rep bsfq %rbp, %r13
+; SSE-NEXT: addl $64, %r13d
+; SSE-NEXT: movq 80(%rdi), %r10
+; SSE-NEXT: rep bsfq %r10, %rcx
; SSE-NEXT: testq %r10, %r10
-; SSE-NEXT: cmovnel %r11d, %eax
+; SSE-NEXT: cmovnel %ecx, %r13d
+; SSE-NEXT: subl $-128, %r13d
+; SSE-NEXT: movq %r12, %rcx
+; SSE-NEXT: orq %rbx, %rcx
+; SSE-NEXT: cmovnel %eax, %r13d
+; SSE-NEXT: movq 104(%rdi), %r9
+; SSE-NEXT: rep bsfq %r9, %rcx
+; SSE-NEXT: addl $64, %ecx
+; SSE-NEXT: movq 96(%rdi), %rdx
+; SSE-NEXT: rep bsfq %rdx, %rax
+; SSE-NEXT: testq %rdx, %rdx
+; SSE-NEXT: cmovnel %eax, %ecx
+; SSE-NEXT: rep bsfq 120(%rdi), %rax
+; SSE-NEXT: movq 112(%rdi), %rdi
+; SSE-NEXT: addl $64, %eax
+; SSE-NEXT: rep bsfq %rdi, %rsi
+; SSE-NEXT: testq %rdi, %rdi
+; SSE-NEXT: cmovnel %esi, %eax
; SSE-NEXT: subl $-128, %eax
-; SSE-NEXT: orq %rcx, %r9
-; SSE-NEXT: cmovnel %r8d, %eax
-; SSE-NEXT: movdqa 64(%rdi), %xmm2
+; SSE-NEXT: orq %r9, %rdx
+; SSE-NEXT: cmovnel %ecx, %eax
+; SSE-NEXT: orq %rbp, %rbx
+; SSE-NEXT: orq %r10, %r12
; SSE-NEXT: addl $256, %eax # imm = 0x100
-; SSE-NEXT: por 80(%rdi), %xmm2
-; SSE-NEXT: ptest %xmm2, %xmm2
-; SSE-NEXT: cmovnel %esi, %eax
-; SSE-NEXT: por 48(%rdi), %xmm0
-; SSE-NEXT: por 32(%rdi), %xmm1
+; SSE-NEXT: orq %rbx, %r12
+; SSE-NEXT: cmovnel %r13d, %eax
+; SSE-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; SSE-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Folded Reload
+; SSE-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Folded Reload
+; SSE-NEXT: orq %rcx, %r11
+; SSE-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r14 # 8-byte Folded Reload
+; SSE-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Folded Reload
+; SSE-NEXT: orq %r14, %r8
; SSE-NEXT: addl $512, %eax # imm = 0x200
-; SSE-NEXT: por %xmm0, %xmm1
-; SSE-NEXT: ptest %xmm1, %xmm1
-; SSE-NEXT: cmovnel %edx, %eax
+; SSE-NEXT: orq %r11, %r8
+; SSE-NEXT: cmovnel %r15d, %eax
; SSE-NEXT: # kill: def $eax killed $eax killed $rax
; SSE-NEXT: popq %rbx
+; SSE-NEXT: popq %r12
+; SSE-NEXT: popq %r13
+; SSE-NEXT: popq %r14
+; SSE-NEXT: popq %r15
+; SSE-NEXT: popq %rbp
; SSE-NEXT: retq
;
; AVX2-LABEL: load_cttz_undef_i1024:
; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %rbp
+; AVX2-NEXT: pushq %r15
+; AVX2-NEXT: pushq %r14
+; AVX2-NEXT: pushq %r13
+; AVX2-NEXT: pushq %r12
; AVX2-NEXT: pushq %rbx
-; AVX2-NEXT: movq 104(%rdi), %rcx
-; AVX2-NEXT: movq 48(%rdi), %rsi
-; AVX2-NEXT: movq 16(%rdi), %rdx
+; AVX2-NEXT: movq 72(%rdi), %r14
+; AVX2-NEXT: movq 64(%rdi), %r15
+; AVX2-NEXT: movq 56(%rdi), %r9
+; AVX2-NEXT: movq %r9, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: movq 48(%rdi), %rcx
+; AVX2-NEXT: movq 40(%rdi), %r10
+; AVX2-NEXT: movq %r10, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: movq 32(%rdi), %rsi
+; AVX2-NEXT: movq 24(%rdi), %rbp
+; AVX2-NEXT: movq 16(%rdi), %rbx
; AVX2-NEXT: movq (%rdi), %r8
-; AVX2-NEXT: movq 8(%rdi), %r9
+; AVX2-NEXT: movq 8(%rdi), %r11
; AVX2-NEXT: tzcntq %r8, %rax
-; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: tzcntq %r9, %rbx
-; AVX2-NEXT: addl $64, %ebx
+; AVX2-NEXT: tzcntq %r11, %rdx
+; AVX2-NEXT: addl $64, %edx
; AVX2-NEXT: testq %r8, %r8
-; AVX2-NEXT: cmovnel %eax, %ebx
-; AVX2-NEXT: tzcntq %rdx, %r11
+; AVX2-NEXT: cmovnel %eax, %edx
+; AVX2-NEXT: xorl %r12d, %r12d
+; AVX2-NEXT: tzcntq %rbx, %r12
; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: tzcntq 24(%rdi), %rax
+; AVX2-NEXT: tzcntq %rbp, %rax
+; AVX2-NEXT: movq %rbp, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: addl $64, %eax
-; AVX2-NEXT: testq %rdx, %rdx
-; AVX2-NEXT: movq 40(%rdi), %r10
-; AVX2-NEXT: cmovnel %r11d, %eax
-; AVX2-NEXT: movq 32(%rdi), %r11
+; AVX2-NEXT: testq %rbx, %rbx
+; AVX2-NEXT: cmovnel %r12d, %eax
; AVX2-NEXT: subl $-128, %eax
-; AVX2-NEXT: orq %r9, %r8
-; AVX2-NEXT: cmovnel %ebx, %eax
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: tzcntq %r11, %rdx
-; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: tzcntq %r10, %rbx
-; AVX2-NEXT: addl $64, %ebx
-; AVX2-NEXT: testq %r11, %r11
-; AVX2-NEXT: cmovnel %edx, %ebx
+; AVX2-NEXT: movq %r8, %r12
+; AVX2-NEXT: orq %r11, %r12
+; AVX2-NEXT: cmovnel %edx, %eax
; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: tzcntq 56(%rdi), %rdx
-; AVX2-NEXT: xorl %r8d, %r8d
-; AVX2-NEXT: tzcntq %rsi, %r8
-; AVX2-NEXT: addl $64, %edx
+; AVX2-NEXT: tzcntq %rsi, %rdx
+; AVX2-NEXT: xorl %r13d, %r13d
+; AVX2-NEXT: tzcntq %r10, %r13
+; AVX2-NEXT: addl $64, %r13d
; AVX2-NEXT: testq %rsi, %rsi
-; AVX2-NEXT: movq 80(%rdi), %r9
-; AVX2-NEXT: cmovnel %r8d, %edx
-; AVX2-NEXT: movq 72(%rdi), %r8
-; AVX2-NEXT: subl $-128, %edx
-; AVX2-NEXT: orq %r10, %r11
-; AVX2-NEXT: movq 64(%rdi), %r10
-; AVX2-NEXT: cmovnel %ebx, %edx
-; AVX2-NEXT: vmovdqu (%rdi), %ymm0
-; AVX2-NEXT: addl $256, %edx # imm = 0x100
-; AVX2-NEXT: vpor 16(%rdi), %xmm0, %xmm1
-; AVX2-NEXT: vptest %xmm1, %xmm1
-; AVX2-NEXT: cmovnel %eax, %edx
+; AVX2-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: cmovnel %edx, %r13d
+; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: tzcntq %rcx, %rdx
+; AVX2-NEXT: xorl %r12d, %r12d
+; AVX2-NEXT: tzcntq %r9, %r12
+; AVX2-NEXT: addl $64, %r12d
+; AVX2-NEXT: testq %rcx, %rcx
+; AVX2-NEXT: cmovnel %edx, %r12d
+; AVX2-NEXT: subl $-128, %r12d
+; AVX2-NEXT: movq %rsi, %rdx
+; AVX2-NEXT: orq %r10, %rdx
+; AVX2-NEXT: cmovnel %r13d, %r12d
+; AVX2-NEXT: addl $256, %r12d # imm = 0x100
+; AVX2-NEXT: movq %r11, %rdx
+; AVX2-NEXT: orq %rbp, %rdx
+; AVX2-NEXT: movq %r8, %r13
+; AVX2-NEXT: orq %rbx, %r13
+; AVX2-NEXT: orq %rdx, %r13
+; AVX2-NEXT: cmovnel %eax, %r12d
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: tzcntq %r15, %rdx
; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: tzcntq %r10, %rax
-; AVX2-NEXT: xorl %r11d, %r11d
-; AVX2-NEXT: tzcntq %r8, %r11
-; AVX2-NEXT: addl $64, %r11d
+; AVX2-NEXT: tzcntq %r14, %rax
+; AVX2-NEXT: addl $64, %eax
+; AVX2-NEXT: testq %r15, %r15
+; AVX2-NEXT: cmovnel %edx, %eax
+; AVX2-NEXT: movq 88(%rdi), %rbp
+; AVX2-NEXT: xorl %r13d, %r13d
+; AVX2-NEXT: tzcntq %rbp, %r13
+; AVX2-NEXT: addl $64, %r13d
+; AVX2-NEXT: movq 80(%rdi), %r10
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: tzcntq %r10, %rcx
; AVX2-NEXT: testq %r10, %r10
-; AVX2-NEXT: cmovnel %eax, %r11d
-; AVX2-NEXT: xorl %esi, %esi
-; AVX2-NEXT: tzcntq 88(%rdi), %rsi
-; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: tzcntq %r9, %rax
-; AVX2-NEXT: addl $64, %esi
-; AVX2-NEXT: testq %r9, %r9
-; AVX2-NEXT: movq 96(%rdi), %r9
-; AVX2-NEXT: cmovnel %eax, %esi
-; AVX2-NEXT: subl $-128, %esi
-; AVX2-NEXT: orq %r8, %r10
-; AVX2-NEXT: cmovnel %r11d, %esi
+; AVX2-NEXT: cmovnel %ecx, %r13d
+; AVX2-NEXT: subl $-128, %r13d
+; AVX2-NEXT: movq %r15, %rcx
+; AVX2-NEXT: orq %r14, %rcx
+; AVX2-NEXT: cmovnel %eax, %r13d
+; AVX2-NEXT: movq 104(%rdi), %r9
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: tzcntq %r9, %rcx
+; AVX2-NEXT: addl $64, %ecx
+; AVX2-NEXT: movq 96(%rdi), %rdx
; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: tzcntq %r9, %rax
-; AVX2-NEXT: xorl %r8d, %r8d
-; AVX2-NEXT: tzcntq %rcx, %r8
-; AVX2-NEXT: addl $64, %r8d
-; AVX2-NEXT: testq %r9, %r9
-; AVX2-NEXT: cmovnel %eax, %r8d
-; AVX2-NEXT: movq 112(%rdi), %r10
-; AVX2-NEXT: xorl %r11d, %r11d
-; AVX2-NEXT: tzcntq %r10, %r11
+; AVX2-NEXT: tzcntq %rdx, %rax
+; AVX2-NEXT: testq %rdx, %rdx
+; AVX2-NEXT: cmovnel %eax, %ecx
+; AVX2-NEXT: movq 112(%rdi), %rsi
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq 120(%rdi), %rax
; AVX2-NEXT: addl $64, %eax
-; AVX2-NEXT: testq %r10, %r10
-; AVX2-NEXT: cmovnel %r11d, %eax
+; AVX2-NEXT: tzcntq %rsi, %rdi
+; AVX2-NEXT: testq %rsi, %rsi
+; AVX2-NEXT: cmovnel %edi, %eax
; AVX2-NEXT: subl $-128, %eax
-; AVX2-NEXT: orq %rcx, %r9
-; AVX2-NEXT: cmovnel %r8d, %eax
-; AVX2-NEXT: vmovdqa 64(%rdi), %xmm1
+; AVX2-NEXT: orq %r9, %rdx
+; AVX2-NEXT: cmovnel %ecx, %eax
+; AVX2-NEXT: orq %rbp, %r14
+; AVX2-NEXT: orq %r10, %r15
; AVX2-NEXT: addl $256, %eax # imm = 0x100
-; AVX2-NEXT: vpor 80(%rdi), %xmm1, %xmm1
-; AVX2-NEXT: vptest %xmm1, %xmm1
-; AVX2-NEXT: cmovnel %esi, %eax
-; AVX2-NEXT: vpor 32(%rdi), %ymm0, %ymm0
+; AVX2-NEXT: orq %r14, %r15
+; AVX2-NEXT: cmovnel %r13d, %eax
+; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
+; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Folded Reload
+; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Folded Reload
+; AVX2-NEXT: orq %rcx, %r11
+; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Folded Reload
+; AVX2-NEXT: orq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Folded Reload
+; AVX2-NEXT: orq %rbx, %r8
; AVX2-NEXT: addl $512, %eax # imm = 0x200
-; AVX2-NEXT: vptest %ymm0, %ymm0
-; AVX2-NEXT: cmovnel %edx, %eax
+; AVX2-NEXT: orq %r11, %r8
+; AVX2-NEXT: cmovnel %r12d, %eax
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: popq %rbx
-; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: popq %r12
+; AVX2-NEXT: popq %r13
+; AVX2-NEXT: popq %r14
+; AVX2-NEXT: popq %r15
+; AVX2-NEXT: popq %rbp
; AVX2-NEXT: retq
;
; AVX512F-LABEL: load_cttz_undef_i1024:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vmovdqu64 (%rdi), %zmm0
-; AVX512F-NEXT: vmovdqu64 64(%rdi), %zmm1
+; AVX512F-NEXT: vmovdqu64 64(%rdi), %zmm0
+; AVX512F-NEXT: vmovdqu64 (%rdi), %zmm1
+; AVX512F-NEXT: movq 16(%rdi), %rax
+; AVX512F-NEXT: movq (%rdi), %rcx
+; AVX512F-NEXT: movq 8(%rdi), %rdx
+; AVX512F-NEXT: movq 24(%rdi), %rsi
+; AVX512F-NEXT: orq 56(%rdi), %rsi
+; AVX512F-NEXT: orq 40(%rdi), %rdx
+; AVX512F-NEXT: orq 48(%rdi), %rax
+; AVX512F-NEXT: orq %rsi, %rdx
+; AVX512F-NEXT: orq 32(%rdi), %rcx
+; AVX512F-NEXT: orq %rax, %rcx
; AVX512F-NEXT: vpternlogd {{.*#+}} zmm2 = -1
-; AVX512F-NEXT: vpaddq %zmm2, %zmm0, %zmm3
-; AVX512F-NEXT: vpandnq %zmm3, %zmm0, %zmm3
+; AVX512F-NEXT: vpaddq %zmm2, %zmm1, %zmm3
+; AVX512F-NEXT: vpandnq %zmm3, %zmm1, %zmm3
; AVX512F-NEXT: vplzcntq %zmm3, %zmm3
; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm4 = [64,128,192,256,320,384,448,512]
; AVX512F-NEXT: vpsubq %zmm3, %zmm4, %zmm3
-; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
-; AVX512F-NEXT: vpcompressq %zmm3, %zmm3 {%k1} {z}
-; AVX512F-NEXT: vmovd %xmm3, %ecx
-; AVX512F-NEXT: vpaddq %zmm2, %zmm1, %zmm2
-; AVX512F-NEXT: vpandnq %zmm2, %zmm1, %zmm2
-; AVX512F-NEXT: vplzcntq %zmm2, %zmm2
-; AVX512F-NEXT: vpsubq %zmm2, %zmm4, %zmm2
; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512F-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
-; AVX512F-NEXT: vmovd %xmm1, %eax
+; AVX512F-NEXT: vpcompressq %zmm3, %zmm1 {%k1} {z}
+; AVX512F-NEXT: vmovd %xmm1, %esi
+; AVX512F-NEXT: vpaddq %zmm2, %zmm0, %zmm1
+; AVX512F-NEXT: vpandnq %zmm1, %zmm0, %zmm1
+; AVX512F-NEXT: vplzcntq %zmm1, %zmm1
+; AVX512F-NEXT: vpsubq %zmm1, %zmm4, %zmm1
+; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; AVX512F-NEXT: vmovd %xmm0, %eax
; AVX512F-NEXT: addl $512, %eax # imm = 0x200
-; AVX512F-NEXT: vpor 32(%rdi), %ymm0, %ymm0
-; AVX512F-NEXT: vptest %ymm0, %ymm0
-; AVX512F-NEXT: cmovnel %ecx, %eax
+; AVX512F-NEXT: orq %rdx, %rcx
+; AVX512F-NEXT: cmovnel %esi, %eax
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: load_cttz_undef_i1024:
; AVX512POPCNT: # %bb.0:
-; AVX512POPCNT-NEXT: vmovdqu64 (%rdi), %zmm0
-; AVX512POPCNT-NEXT: vmovdqu64 64(%rdi), %zmm1
+; AVX512POPCNT-NEXT: vmovdqu64 64(%rdi), %zmm0
+; AVX512POPCNT-NEXT: vmovdqu64 (%rdi), %zmm1
+; AVX512POPCNT-NEXT: movq 16(%rdi), %rax
+; AVX512POPCNT-NEXT: movq (%rdi), %rcx
+; AVX512POPCNT-NEXT: movq 8(%rdi), %rdx
+; AVX512POPCNT-NEXT: movq 24(%rdi), %rsi
+; AVX512POPCNT-NEXT: orq 56(%rdi), %rsi
+; AVX512POPCNT-NEXT: orq 40(%rdi), %rdx
+; AVX512POPCNT-NEXT: orq 48(%rdi), %rax
+; AVX512POPCNT-NEXT: orq %rsi, %rdx
+; AVX512POPCNT-NEXT: orq 32(%rdi), %rcx
+; AVX512POPCNT-NEXT: orq %rax, %rcx
; AVX512POPCNT-NEXT: vpternlogd {{.*#+}} zmm2 = -1
-; AVX512POPCNT-NEXT: vpaddq %zmm2, %zmm0, %zmm3
-; AVX512POPCNT-NEXT: vpandnq %zmm3, %zmm0, %zmm3
+; AVX512POPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm3
+; AVX512POPCNT-NEXT: vpandnq %zmm3, %zmm1, %zmm3
; AVX512POPCNT-NEXT: vpopcntq %zmm3, %zmm3
; AVX512POPCNT-NEXT: vmovdqa64 {{.*#+}} zmm4 = [0,64,128,192,256,320,384,448]
; AVX512POPCNT-NEXT: vpaddq %zmm4, %zmm3, %zmm3
-; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
-; AVX512POPCNT-NEXT: vpcompressq %zmm3, %zmm3 {%k1} {z}
-; AVX512POPCNT-NEXT: vmovd %xmm3, %ecx
-; AVX512POPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm2
-; AVX512POPCNT-NEXT: vpandnq %zmm2, %zmm1, %zmm2
-; AVX512POPCNT-NEXT: vpopcntq %zmm2, %zmm2
-; AVX512POPCNT-NEXT: vpaddq %zmm4, %zmm2, %zmm2
; AVX512POPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512POPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
-; AVX512POPCNT-NEXT: vmovd %xmm1, %eax
+; AVX512POPCNT-NEXT: vpcompressq %zmm3, %zmm1 {%k1} {z}
+; AVX512POPCNT-NEXT: vmovd %xmm1, %esi
+; AVX512POPCNT-NEXT: vpaddq %zmm2, %zmm0, %zmm1
+; AVX512POPCNT-NEXT: vpandnq %zmm1, %zmm0, %zmm1
+; AVX512POPCNT-NEXT: vpopcntq %zmm1, %zmm1
+; AVX512POPCNT-NEXT: vpaddq %zmm4, %zmm1, %zmm1
+; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512POPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; AVX512POPCNT-NEXT: vmovd %xmm0, %eax
; AVX512POPCNT-NEXT: addl $512, %eax # imm = 0x200
-; AVX512POPCNT-NEXT: vpor 32(%rdi), %ymm0, %ymm0
-; AVX512POPCNT-NEXT: vptest %ymm0, %ymm0
-; AVX512POPCNT-NEXT: cmovnel %ecx, %eax
+; AVX512POPCNT-NEXT: orq %rdx, %rcx
+; AVX512POPCNT-NEXT: cmovnel %esi, %eax
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: load_cttz_undef_i1024:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vmovdqu64 (%rdi), %zmm0
-; AVX512VL-NEXT: vmovdqu64 64(%rdi), %zmm1
+; AVX512VL-NEXT: vmovdqu64 64(%rdi), %zmm0
+; AVX512VL-NEXT: vmovdqu64 (%rdi), %zmm1
+; AVX512VL-NEXT: movq 16(%rdi), %rax
+; AVX512VL-NEXT: movq (%rdi), %rcx
+; AVX512VL-NEXT: movq 8(%rdi), %rdx
+; AVX512VL-NEXT: movq 24(%rdi), %rsi
+; AVX512VL-NEXT: orq 56(%rdi), %rsi
+; AVX512VL-NEXT: orq 40(%rdi), %rdx
+; AVX512VL-NEXT: orq 48(%rdi), %rax
+; AVX512VL-NEXT: orq 32(%rdi), %rcx
+; AVX512VL-NEXT: orq %rsi, %rdx
+; AVX512VL-NEXT: orq %rax, %rcx
; AVX512VL-NEXT: vpternlogd {{.*#+}} zmm2 = -1
-; AVX512VL-NEXT: vpaddq %zmm2, %zmm0, %zmm3
-; AVX512VL-NEXT: vpandnq %zmm3, %zmm0, %zmm3
+; AVX512VL-NEXT: vpaddq %zmm2, %zmm1, %zmm3
+; AVX512VL-NEXT: vpandnq %zmm3, %zmm1, %zmm3
; AVX512VL-NEXT: vplzcntq %zmm3, %zmm3
; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm4 = [64,128,192,256,320,384,448,512]
; AVX512VL-NEXT: vpsubq %zmm3, %zmm4, %zmm3
-; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
-; AVX512VL-NEXT: vpcompressq %zmm3, %zmm3 {%k1} {z}
-; AVX512VL-NEXT: vmovd %xmm3, %ecx
-; AVX512VL-NEXT: vpaddq %zmm2, %zmm1, %zmm2
-; AVX512VL-NEXT: vpandnq %zmm2, %zmm1, %zmm2
-; AVX512VL-NEXT: vplzcntq %zmm2, %zmm2
-; AVX512VL-NEXT: vpsubq %zmm2, %zmm4, %zmm2
; AVX512VL-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512VL-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
-; AVX512VL-NEXT: vmovd %xmm1, %eax
+; AVX512VL-NEXT: vpcompressq %zmm3, %zmm1 {%k1} {z}
+; AVX512VL-NEXT: vmovd %xmm1, %esi
+; AVX512VL-NEXT: vpaddq %zmm2, %zmm0, %zmm1
+; AVX512VL-NEXT: vpandnq %zmm1, %zmm0, %zmm1
+; AVX512VL-NEXT: vplzcntq %zmm1, %zmm1
+; AVX512VL-NEXT: vpsubq %zmm1, %zmm4, %zmm1
+; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512VL-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; AVX512VL-NEXT: vmovd %xmm0, %eax
; AVX512VL-NEXT: addl $512, %eax # imm = 0x200
-; AVX512VL-NEXT: vpor 32(%rdi), %ymm0, %ymm0
-; AVX512VL-NEXT: vptest %ymm0, %ymm0
-; AVX512VL-NEXT: cmovnel %ecx, %eax
+; AVX512VL-NEXT: orq %rdx, %rcx
+; AVX512VL-NEXT: cmovnel %esi, %eax
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: load_cttz_undef_i1024:
; AVX512VLPOPCNT: # %bb.0:
-; AVX512VLPOPCNT-NEXT: vmovdqu64 (%rdi), %zmm0
-; AVX512VLPOPCNT-NEXT: vmovdqu64 64(%rdi), %zmm1
+; AVX512VLPOPCNT-NEXT: vmovdqu64 64(%rdi), %zmm0
+; AVX512VLPOPCNT-NEXT: vmovdqu64 (%rdi), %zmm1
+; AVX512VLPOPCNT-NEXT: movq 16(%rdi), %rax
+; AVX512VLPOPCNT-NEXT: movq (%rdi), %rcx
+; AVX512VLPOPCNT-NEXT: movq 8(%rdi), %rdx
+; AVX512VLPOPCNT-NEXT: movq 24(%rdi), %rsi
+; AVX512VLPOPCNT-NEXT: orq 56(%rdi), %rsi
+; AVX512VLPOPCNT-NEXT: orq 40(%rdi), %rdx
+; AVX512VLPOPCNT-NEXT: orq 48(%rdi), %rax
+; AVX512VLPOPCNT-NEXT: orq 32(%rdi), %rcx
+; AVX512VLPOPCNT-NEXT: orq %rsi, %rdx
+; AVX512VLPOPCNT-NEXT: orq %rax, %rcx
; AVX512VLPOPCNT-NEXT: vpternlogd {{.*#+}} zmm2 = -1
-; AVX512VLPOPCNT-NEXT: vpaddq %zmm2, %zmm0, %zmm3
-; AVX512VLPOPCNT-NEXT: vpandnq %zmm3, %zmm0, %zmm3
+; AVX512VLPOPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm3
+; AVX512VLPOPCNT-NEXT: vpandnq %zmm3, %zmm1, %zmm3
; AVX512VLPOPCNT-NEXT: vpopcntq %zmm3, %zmm3
; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm4 = [0,64,128,192,256,320,384,448]
; AVX512VLPOPCNT-NEXT: vpaddq %zmm4, %zmm3, %zmm3
-; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
-; AVX512VLPOPCNT-NEXT: vpcompressq %zmm3, %zmm3 {%k1} {z}
-; AVX512VLPOPCNT-NEXT: vmovd %xmm3, %ecx
-; AVX512VLPOPCNT-NEXT: vpaddq %zmm2, %zmm1, %zmm2
-; AVX512VLPOPCNT-NEXT: vpandnq %zmm2, %zmm1, %zmm2
-; AVX512VLPOPCNT-NEXT: vpopcntq %zmm2, %zmm2
-; AVX512VLPOPCNT-NEXT: vpaddq %zmm4, %zmm2, %zmm2
; AVX512VLPOPCNT-NEXT: vptestmq %zmm1, %zmm1, %k1
-; AVX512VLPOPCNT-NEXT: vpcompressq %zmm2, %zmm1 {%k1} {z}
-; AVX512VLPOPCNT-NEXT: vmovd %xmm1, %eax
+; AVX512VLPOPCNT-NEXT: vpcompressq %zmm3, %zmm1 {%k1} {z}
+; AVX512VLPOPCNT-NEXT: vmovd %xmm1, %esi
+; AVX512VLPOPCNT-NEXT: vpaddq %zmm2, %zmm0, %zmm1
+; AVX512VLPOPCNT-NEXT: vpandnq %zmm1, %zmm0, %zmm1
+; AVX512VLPOPCNT-NEXT: vpopcntq %zmm1, %zmm1
+; AVX512VLPOPCNT-NEXT: vpaddq %zmm4, %zmm1, %zmm1
+; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
+; AVX512VLPOPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
; AVX512VLPOPCNT-NEXT: addl $512, %eax # imm = 0x200
-; AVX512VLPOPCNT-NEXT: vpor 32(%rdi), %ymm0, %ymm0
-; AVX512VLPOPCNT-NEXT: vptest %ymm0, %ymm0
-; AVX512VLPOPCNT-NEXT: cmovnel %ecx, %eax
+; AVX512VLPOPCNT-NEXT: orq %rdx, %rcx
+; AVX512VLPOPCNT-NEXT: cmovnel %esi, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
; AVX512VLPOPCNT-NEXT: retq
%a0 = load i1024, ptr %p0
diff --git a/llvm/test/CodeGen/X86/bittest-big-integer.ll b/llvm/test/CodeGen/X86/bittest-big-integer.ll
index 96ccc7b0f7527..023fb5065b892 100644
--- a/llvm/test/CodeGen/X86/bittest-big-integer.ll
+++ b/llvm/test/CodeGen/X86/bittest-big-integer.ll
@@ -1039,14 +1039,14 @@ define <8 x i16> @complement_ne_i128_bitcast(ptr %word, i32 %position) nounwind
; SSE2: # %bb.0:
; SSE2-NEXT: # kill: def $esi killed $esi def $rsi
; SSE2-NEXT: movdqa (%rdi), %xmm0
-; SSE2-NEXT: movq %xmm0, %rax
-; SSE2-NEXT: movq 8(%rdi), %rdx
+; SSE2-NEXT: movq 8(%rdi), %rax
+; SSE2-NEXT: movq %xmm0, %rdx
; SSE2-NEXT: movl %esi, %ecx
; SSE2-NEXT: andb $32, %cl
-; SSE2-NEXT: shrq %cl, %rdx
+; SSE2-NEXT: shrdq %cl, %rax, %rdx
; SSE2-NEXT: shrq %cl, %rax
; SSE2-NEXT: testb $64, %sil
-; SSE2-NEXT: cmovneq %rdx, %rax
+; SSE2-NEXT: cmoveq %rdx, %rax
; SSE2-NEXT: btcl %esi, %eax
; SSE2-NEXT: andl $96, %esi
; SSE2-NEXT: shrl $3, %esi
@@ -1057,14 +1057,14 @@ define <8 x i16> @complement_ne_i128_bitcast(ptr %word, i32 %position) nounwind
; SSE4: # %bb.0:
; SSE4-NEXT: # kill: def $esi killed $esi def $rsi
; SSE4-NEXT: movdqa (%rdi), %xmm0
-; SSE4-NEXT: movq %xmm0, %rax
-; SSE4-NEXT: pextrq $1, %xmm0, %rdx
+; SSE4-NEXT: pextrq $1, %xmm0, %rax
+; SSE4-NEXT: movq %xmm0, %rdx
; SSE4-NEXT: movl %esi, %ecx
; SSE4-NEXT: andb $32, %cl
-; SSE4-NEXT: shrq %cl, %rdx
+; SSE4-NEXT: shrdq %cl, %rax, %rdx
; SSE4-NEXT: shrq %cl, %rax
; SSE4-NEXT: testb $64, %sil
-; SSE4-NEXT: cmovneq %rdx, %rax
+; SSE4-NEXT: cmoveq %rdx, %rax
; SSE4-NEXT: btcl %esi, %eax
; SSE4-NEXT: andl $96, %esi
; SSE4-NEXT: shrl $3, %esi
@@ -1075,14 +1075,14 @@ define <8 x i16> @complement_ne_i128_bitcast(ptr %word, i32 %position) nounwind
; AVX: # %bb.0:
; AVX-NEXT: # kill: def $esi killed $esi def $rsi
; AVX-NEXT: vmovdqa (%rdi), %xmm0
-; AVX-NEXT: vmovq %xmm0, %rax
-; AVX-NEXT: vpextrq $1, %xmm0, %rcx
-; AVX-NEXT: movl %esi, %edx
-; AVX-NEXT: andb $32, %dl
-; AVX-NEXT: shrxq %rdx, %rcx, %rcx
-; AVX-NEXT: shrxq %rdx, %rax, %rax
+; AVX-NEXT: vpextrq $1, %xmm0, %rax
+; AVX-NEXT: vmovq %xmm0, %rdx
+; AVX-NEXT: movl %esi, %ecx
+; AVX-NEXT: andb $32, %cl
+; AVX-NEXT: shrdq %cl, %rax, %rdx
+; AVX-NEXT: shrxq %rcx, %rax, %rax
; AVX-NEXT: testb $64, %sil
-; AVX-NEXT: cmovneq %rcx, %rax
+; AVX-NEXT: cmoveq %rdx, %rax
; AVX-NEXT: btcl %esi, %eax
; AVX-NEXT: andl $96, %esi
; AVX-NEXT: shrl $3, %esi
@@ -1199,47 +1199,26 @@ define i32 @chain_reset_i256(ptr %p0, ptr %p1, ptr %p2, i32 %position) nounwind
; X86-NEXT: popl %ebp
; X86-NEXT: retl
;
-; SSE2-LABEL: chain_reset_i256:
-; SSE2: # %bb.0:
-; SSE2-NEXT: # kill: def $ecx killed $ecx def $rcx
-; SSE2-NEXT: movl $-2, %eax
-; SSE2-NEXT: roll %cl, %eax
-; SSE2-NEXT: shrl $3, %ecx
-; SSE2-NEXT: andl $28, %ecx
-; SSE2-NEXT: andl %eax, (%rdi,%rcx)
-; SSE2-NEXT: movl (%rdi), %ecx
-; SSE2-NEXT: movdqa (%rdi), %xmm0
-; SSE2-NEXT: por 16(%rdi), %xmm0
-; SSE2-NEXT: pxor %xmm1, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm0, %xmm1
-; SSE2-NEXT: movmskps %xmm1, %edi
-; SSE2-NEXT: xorl $15, %edi
-; SSE2-NEXT: movl (%rsi), %eax
-; SSE2-NEXT: movl %ecx, (%rsi)
-; SSE2-NEXT: movl (%rdx), %ecx
-; SSE2-NEXT: addl %ecx, %eax
-; SSE2-NEXT: testl %edi, %edi
-; SSE2-NEXT: cmovnel %ecx, %eax
-; SSE2-NEXT: retq
-;
-; SSE4-LABEL: chain_reset_i256:
-; SSE4: # %bb.0:
-; SSE4-NEXT: # kill: def $ecx killed $ecx def $rcx
-; SSE4-NEXT: movl $-2, %eax
-; SSE4-NEXT: roll %cl, %eax
-; SSE4-NEXT: shrl $3, %ecx
-; SSE4-NEXT: andl $28, %ecx
-; SSE4-NEXT: andl %eax, (%rdi,%rcx)
-; SSE4-NEXT: movl (%rdi), %ecx
-; SSE4-NEXT: movdqa (%rdi), %xmm0
-; SSE4-NEXT: por 16(%rdi), %xmm0
-; SSE4-NEXT: movl (%rsi), %eax
-; SSE4-NEXT: movl %ecx, (%rsi)
-; SSE4-NEXT: movl (%rdx), %ecx
-; SSE4-NEXT: addl %ecx, %eax
-; SSE4-NEXT: ptest %xmm0, %xmm0
-; SSE4-NEXT: cmovnel %ecx, %eax
-; SSE4-NEXT: retq
+; SSE-LABEL: chain_reset_i256:
+; SSE: # %bb.0:
+; SSE-NEXT: # kill: def $ecx killed $ecx def $rcx
+; SSE-NEXT: movl $-2, %eax
+; SSE-NEXT: roll %cl, %eax
+; SSE-NEXT: shrl $3, %ecx
+; SSE-NEXT: andl $28, %ecx
+; SSE-NEXT: andl %eax, (%rdi,%rcx)
+; SSE-NEXT: movq (%rdi), %rcx
+; SSE-NEXT: movq 8(%rdi), %r8
+; SSE-NEXT: orq 24(%rdi), %r8
+; SSE-NEXT: movq 16(%rdi), %rdi
+; SSE-NEXT: orq %rcx, %rdi
+; SSE-NEXT: movl (%rsi), %eax
+; SSE-NEXT: movl %ecx, (%rsi)
+; SSE-NEXT: movl (%rdx), %ecx
+; SSE-NEXT: addl %ecx, %eax
+; SSE-NEXT: orq %r8, %rdi
+; SSE-NEXT: cmovnel %ecx, %eax
+; SSE-NEXT: retq
;
; AVX-LABEL: chain_reset_i256:
; AVX: # %bb.0:
@@ -1407,12 +1386,12 @@ define i1 @sequence_i128(ptr %word, i32 %pos0, i32 %pos1, i32 %pos2) nounwind {
; SSE-NEXT: xorq (%rdi), %r9
; SSE-NEXT: movl %edx, %ecx
; SSE-NEXT: andb $32, %cl
-; SSE-NEXT: movq %rsi, %rax
-; SSE-NEXT: shrq %cl, %rax
-; SSE-NEXT: movq %r9, %r11
+; SSE-NEXT: movq %r9, %rax
+; SSE-NEXT: shrdq %cl, %rsi, %rax
+; SSE-NEXT: movq %rsi, %r11
; SSE-NEXT: shrq %cl, %r11
; SSE-NEXT: testb $64, %dl
-; SSE-NEXT: cmovneq %rax, %r11
+; SSE-NEXT: cmoveq %rax, %r11
; SSE-NEXT: btl %edx, %r11d
; SSE-NEXT: setae %al
; SSE-NEXT: orq %r10, %rsi
@@ -1436,22 +1415,23 @@ define i1 @sequence_i128(ptr %word, i32 %pos0, i32 %pos1, i32 %pos2) nounwind {
; AVX2-NEXT: xorl %r11d, %r11d
; AVX2-NEXT: movl %eax, %ecx
; AVX2-NEXT: shldq %cl, %r10, %r11
-; AVX2-NEXT: shlxq %rax, %r10, %rcx
+; AVX2-NEXT: shlxq %rax, %r10, %r10
; AVX2-NEXT: testb $64, %al
-; AVX2-NEXT: cmovneq %rcx, %r11
-; AVX2-NEXT: cmovneq %r9, %rcx
+; AVX2-NEXT: cmovneq %r10, %r11
+; AVX2-NEXT: cmovneq %r9, %r10
; AVX2-NEXT: xorq 8(%rdi), %rsi
; AVX2-NEXT: xorq (%rdi), %r8
-; AVX2-NEXT: movl %edx, %eax
-; AVX2-NEXT: andb $32, %al
-; AVX2-NEXT: shrxq %rax, %rsi, %r9
-; AVX2-NEXT: shrxq %rax, %r8, %rax
+; AVX2-NEXT: movl %edx, %ecx
+; AVX2-NEXT: andb $32, %cl
+; AVX2-NEXT: movq %r8, %rax
+; AVX2-NEXT: shrdq %cl, %rsi, %rax
+; AVX2-NEXT: shrxq %rcx, %rsi, %rcx
; AVX2-NEXT: testb $64, %dl
-; AVX2-NEXT: cmovneq %r9, %rax
-; AVX2-NEXT: btl %edx, %eax
+; AVX2-NEXT: cmoveq %rax, %rcx
+; AVX2-NEXT: btl %edx, %ecx
; AVX2-NEXT: setae %al
; AVX2-NEXT: orq %r11, %rsi
-; AVX2-NEXT: orq %rcx, %r8
+; AVX2-NEXT: orq %r10, %r8
; AVX2-NEXT: movq %r8, (%rdi)
; AVX2-NEXT: movq %rsi, 8(%rdi)
; AVX2-NEXT: retq
@@ -1471,22 +1451,23 @@ define i1 @sequence_i128(ptr %word, i32 %pos0, i32 %pos1, i32 %pos2) nounwind {
; AVX512-NEXT: xorl %r11d, %r11d
; AVX512-NEXT: movl %eax, %ecx
; AVX512-NEXT: shldq %cl, %r9, %r11
-; AVX512-NEXT: shlxq %rax, %r9, %rcx
+; AVX512-NEXT: shlxq %rax, %r9, %r9
; AVX512-NEXT: testb $64, %al
-; AVX512-NEXT: cmovneq %rcx, %r11
-; AVX512-NEXT: cmovneq %r10, %rcx
+; AVX512-NEXT: cmovneq %r9, %r11
+; AVX512-NEXT: cmovneq %r10, %r9
; AVX512-NEXT: xorq 8(%rdi), %rsi
; AVX512-NEXT: xorq (%rdi), %r8
-; AVX512-NEXT: movl %edx, %eax
-; AVX512-NEXT: andb $32, %al
-; AVX512-NEXT: shrxq %rax, %rsi, %r9
-; AVX512-NEXT: shrxq %rax, %r8, %rax
+; AVX512-NEXT: movl %edx, %ecx
+; AVX512-NEXT: andb $32, %cl
+; AVX512-NEXT: movq %r8, %rax
+; AVX512-NEXT: shrdq %cl, %rsi, %rax
+; AVX512-NEXT: shrxq %rcx, %rsi, %rcx
; AVX512-NEXT: testb $64, %dl
-; AVX512-NEXT: cmovneq %r9, %rax
-; AVX512-NEXT: btl %edx, %eax
+; AVX512-NEXT: cmoveq %rax, %rcx
+; AVX512-NEXT: btl %edx, %ecx
; AVX512-NEXT: setae %al
; AVX512-NEXT: orq %r11, %rsi
-; AVX512-NEXT: orq %rcx, %r8
+; AVX512-NEXT: orq %r9, %r8
; AVX512-NEXT: movq %r8, (%rdi)
; AVX512-NEXT: movq %rsi, 8(%rdi)
; AVX512-NEXT: retq
@@ -1892,157 +1873,114 @@ define i32 @blsr_u512(ptr %word) nounwind {
; X86-NEXT: popl %ebp
; X86-NEXT: retl
;
-; SSE2-LABEL: blsr_u512:
-; SSE2: # %bb.0:
-; SSE2-NEXT: pushq %rbx
-; SSE2-NEXT: movdqa (%rdi), %xmm0
-; SSE2-NEXT: movq 48(%rdi), %r8
-; SSE2-NEXT: movq 40(%rdi), %rdx
-; SSE2-NEXT: movq 32(%rdi), %rsi
-; SSE2-NEXT: movq (%rdi), %rax
-; SSE2-NEXT: movq 8(%rdi), %r9
-; SSE2-NEXT: rep bsfq %rax, %rcx
-; SSE2-NEXT: rep bsfq %r9, %r10
-; SSE2-NEXT: addq $64, %r10
-; SSE2-NEXT: testq %rax, %rax
-; SSE2-NEXT: cmovneq %rcx, %r10
-; SSE2-NEXT: movq 16(%rdi), %r11
-; SSE2-NEXT: rep bsfq %r11, %rbx
-; SSE2-NEXT: rep bsfq 24(%rdi), %rcx
-; SSE2-NEXT: addq $64, %rcx
-; SSE2-NEXT: testq %r11, %r11
-; SSE2-NEXT: cmovneq %rbx, %rcx
-; SSE2-NEXT: subq $-128, %rcx
-; SSE2-NEXT: orq %r9, %rax
-; SSE2-NEXT: cmovneq %r10, %rcx
-; SSE2-NEXT: rep bsfq %rsi, %rax
-; SSE2-NEXT: rep bsfq %rdx, %r9
-; SSE2-NEXT: addq $64, %r9
-; SSE2-NEXT: testq %rsi, %rsi
-; SSE2-NEXT: cmovneq %rax, %r9
-; SSE2-NEXT: rep bsfq %r8, %r10
-; SSE2-NEXT: movl $64, %eax
-; SSE2-NEXT: rep bsfq 56(%rdi), %rax
-; SSE2-NEXT: addq $64, %rax
-; SSE2-NEXT: testq %r8, %r8
-; SSE2-NEXT: cmovneq %r10, %rax
-; SSE2-NEXT: subq $-128, %rax
-; SSE2-NEXT: orq %rdx, %rsi
-; SSE2-NEXT: cmovneq %r9, %rax
-; SSE2-NEXT: por 16(%rdi), %xmm0
-; SSE2-NEXT: addq $256, %rax # imm = 0x100
-; SSE2-NEXT: pxor %xmm1, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm0, %xmm1
-; SSE2-NEXT: movmskps %xmm1, %edx
-; SSE2-NEXT: xorl $15, %edx
-; SSE2-NEXT: cmovneq %rcx, %rax
-; SSE2-NEXT: movl $-2, %edx
-; SSE2-NEXT: movl %eax, %ecx
-; SSE2-NEXT: roll %cl, %edx
-; SSE2-NEXT: movl %eax, %ecx
-; SSE2-NEXT: shrl $3, %ecx
-; SSE2-NEXT: andl $60, %ecx
-; SSE2-NEXT: andl %edx, (%rdi,%rcx)
-; SSE2-NEXT: # kill: def $eax killed $eax killed $rax
-; SSE2-NEXT: popq %rbx
-; SSE2-NEXT: retq
-;
-; SSE4-LABEL: blsr_u512:
-; SSE4: # %bb.0:
-; SSE4-NEXT: movdqa (%rdi), %xmm0
-; SSE4-NEXT: movq 48(%rdi), %rdx
-; SSE4-NEXT: movq 40(%rdi), %rcx
-; SSE4-NEXT: movq (%rdi), %rax
-; SSE4-NEXT: movq 8(%rdi), %r8
-; SSE4-NEXT: rep bsfq %rax, %rsi
-; SSE4-NEXT: rep bsfq %r8, %r9
-; SSE4-NEXT: addq $64, %r9
-; SSE4-NEXT: testq %rax, %rax
-; SSE4-NEXT: cmovneq %rsi, %r9
-; SSE4-NEXT: movq 16(%rdi), %r10
-; SSE4-NEXT: rep bsfq %r10, %r11
-; SSE4-NEXT: rep bsfq 24(%rdi), %rsi
-; SSE4-NEXT: addq $64, %rsi
-; SSE4-NEXT: testq %r10, %r10
-; SSE4-NEXT: cmovneq %r11, %rsi
-; SSE4-NEXT: subq $-128, %rsi
-; SSE4-NEXT: orq %r8, %rax
-; SSE4-NEXT: cmovneq %r9, %rsi
-; SSE4-NEXT: movq 32(%rdi), %r8
-; SSE4-NEXT: rep bsfq %r8, %rax
-; SSE4-NEXT: rep bsfq %rcx, %r9
-; SSE4-NEXT: addq $64, %r9
-; SSE4-NEXT: testq %r8, %r8
-; SSE4-NEXT: cmovneq %rax, %r9
-; SSE4-NEXT: rep bsfq %rdx, %r10
-; SSE4-NEXT: movl $64, %eax
-; SSE4-NEXT: rep bsfq 56(%rdi), %rax
-; SSE4-NEXT: addq $64, %rax
-; SSE4-NEXT: testq %rdx, %rdx
-; SSE4-NEXT: cmovneq %r10, %rax
-; SSE4-NEXT: subq $-128, %rax
-; SSE4-NEXT: orq %rcx, %r8
-; SSE4-NEXT: cmovneq %r9, %rax
-; SSE4-NEXT: addq $256, %rax # imm = 0x100
-; SSE4-NEXT: por 16(%rdi), %xmm0
-; SSE4-NEXT: ptest %xmm0, %xmm0
-; SSE4-NEXT: cmovneq %rsi, %rax
-; SSE4-NEXT: movl $-2, %edx
-; SSE4-NEXT: movl %eax, %ecx
-; SSE4-NEXT: roll %cl, %edx
-; SSE4-NEXT: movl %eax, %ecx
-; SSE4-NEXT: shrl $3, %ecx
-; SSE4-NEXT: andl $60, %ecx
-; SSE4-NEXT: andl %edx, (%rdi,%rcx)
-; SSE4-NEXT: # kill: def $eax killed $eax killed $rax
-; SSE4-NEXT: retq
+; SSE-LABEL: blsr_u512:
+; SSE: # %bb.0:
+; SSE-NEXT: pushq %r15
+; SSE-NEXT: pushq %r14
+; SSE-NEXT: pushq %rbx
+; SSE-NEXT: movq 48(%rdi), %r11
+; SSE-NEXT: movq 40(%rdi), %r9
+; SSE-NEXT: movq 24(%rdi), %r8
+; SSE-NEXT: movq 16(%rdi), %rdx
+; SSE-NEXT: movq (%rdi), %rcx
+; SSE-NEXT: movq 8(%rdi), %rsi
+; SSE-NEXT: rep bsfq %rcx, %rax
+; SSE-NEXT: rep bsfq %rsi, %rbx
+; SSE-NEXT: addq $64, %rbx
+; SSE-NEXT: testq %rcx, %rcx
+; SSE-NEXT: cmovneq %rax, %rbx
+; SSE-NEXT: rep bsfq %rdx, %rax
+; SSE-NEXT: rep bsfq %r8, %r10
+; SSE-NEXT: addq $64, %r10
+; SSE-NEXT: testq %rdx, %rdx
+; SSE-NEXT: cmovneq %rax, %r10
+; SSE-NEXT: movq 32(%rdi), %r14
+; SSE-NEXT: subq $-128, %r10
+; SSE-NEXT: movq %rcx, %rax
+; SSE-NEXT: orq %rsi, %rax
+; SSE-NEXT: cmovneq %rbx, %r10
+; SSE-NEXT: rep bsfq %r14, %rax
+; SSE-NEXT: rep bsfq %r9, %rbx
+; SSE-NEXT: addq $64, %rbx
+; SSE-NEXT: testq %r14, %r14
+; SSE-NEXT: cmovneq %rax, %rbx
+; SSE-NEXT: rep bsfq %r11, %r15
+; SSE-NEXT: movl $64, %eax
+; SSE-NEXT: rep bsfq 56(%rdi), %rax
+; SSE-NEXT: addq $64, %rax
+; SSE-NEXT: testq %r11, %r11
+; SSE-NEXT: cmovneq %r15, %rax
+; SSE-NEXT: subq $-128, %rax
+; SSE-NEXT: orq %r9, %r14
+; SSE-NEXT: cmovneq %rbx, %rax
+; SSE-NEXT: addq $256, %rax # imm = 0x100
+; SSE-NEXT: orq %r8, %rsi
+; SSE-NEXT: orq %rdx, %rcx
+; SSE-NEXT: orq %rsi, %rcx
+; SSE-NEXT: cmovneq %r10, %rax
+; SSE-NEXT: movl $-2, %edx
+; SSE-NEXT: movl %eax, %ecx
+; SSE-NEXT: roll %cl, %edx
+; SSE-NEXT: movl %eax, %ecx
+; SSE-NEXT: shrl $3, %ecx
+; SSE-NEXT: andl $60, %ecx
+; SSE-NEXT: andl %edx, (%rdi,%rcx)
+; SSE-NEXT: # kill: def $eax killed $eax killed $rax
+; SSE-NEXT: popq %rbx
+; SSE-NEXT: popq %r14
+; SSE-NEXT: popq %r15
+; SSE-NEXT: retq
;
; AVX2-LABEL: blsr_u512:
; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %r15
+; AVX2-NEXT: pushq %r14
; AVX2-NEXT: pushq %rbx
-; AVX2-NEXT: vmovdqa (%rdi), %xmm0
-; AVX2-NEXT: movq 48(%rdi), %rsi
-; AVX2-NEXT: movq 40(%rdi), %rcx
-; AVX2-NEXT: movq 32(%rdi), %rdx
-; AVX2-NEXT: movq 16(%rdi), %rax
-; AVX2-NEXT: movq (%rdi), %r9
-; AVX2-NEXT: movq 8(%rdi), %r10
-; AVX2-NEXT: tzcntq %r9, %r8
-; AVX2-NEXT: tzcntq %r10, %r11
-; AVX2-NEXT: addq $64, %r11
-; AVX2-NEXT: testq %r9, %r9
-; AVX2-NEXT: cmovneq %r8, %r11
+; AVX2-NEXT: movq 40(%rdi), %r9
+; AVX2-NEXT: movq 32(%rdi), %r10
+; AVX2-NEXT: movq 24(%rdi), %r8
+; AVX2-NEXT: movq 16(%rdi), %rdx
+; AVX2-NEXT: movq (%rdi), %rcx
+; AVX2-NEXT: movq 8(%rdi), %rsi
+; AVX2-NEXT: tzcntq %rcx, %rax
; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: tzcntq %rax, %rbx
-; AVX2-NEXT: xorl %r8d, %r8d
-; AVX2-NEXT: tzcntq 24(%rdi), %r8
-; AVX2-NEXT: addq $64, %r8
-; AVX2-NEXT: testq %rax, %rax
-; AVX2-NEXT: cmovneq %rbx, %r8
-; AVX2-NEXT: subq $-128, %r8
-; AVX2-NEXT: orq %r10, %r9
-; AVX2-NEXT: cmovneq %r11, %r8
+; AVX2-NEXT: tzcntq %rsi, %rbx
+; AVX2-NEXT: addq $64, %rbx
+; AVX2-NEXT: testq %rcx, %rcx
+; AVX2-NEXT: cmovneq %rax, %rbx
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq %rdx, %rax
-; AVX2-NEXT: xorl %r9d, %r9d
-; AVX2-NEXT: tzcntq %rcx, %r9
-; AVX2-NEXT: addq $64, %r9
+; AVX2-NEXT: tzcntq %r8, %r11
+; AVX2-NEXT: addq $64, %r11
; AVX2-NEXT: testq %rdx, %rdx
-; AVX2-NEXT: cmovneq %rax, %r9
-; AVX2-NEXT: xorl %r10d, %r10d
-; AVX2-NEXT: tzcntq %rsi, %r10
+; AVX2-NEXT: cmovneq %rax, %r11
+; AVX2-NEXT: subq $-128, %r11
+; AVX2-NEXT: movq %rcx, %rax
+; AVX2-NEXT: orq %rsi, %rax
+; AVX2-NEXT: cmovneq %rbx, %r11
+; AVX2-NEXT: xorl %eax, %eax
+; AVX2-NEXT: tzcntq %r10, %rax
+; AVX2-NEXT: xorl %ebx, %ebx
+; AVX2-NEXT: tzcntq %r9, %rbx
+; AVX2-NEXT: addq $64, %rbx
+; AVX2-NEXT: testq %r10, %r10
+; AVX2-NEXT: cmovneq %rax, %rbx
+; AVX2-NEXT: movq 48(%rdi), %r14
+; AVX2-NEXT: xorl %r15d, %r15d
+; AVX2-NEXT: tzcntq %r14, %r15
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq 56(%rdi), %rax
; AVX2-NEXT: addq $64, %rax
-; AVX2-NEXT: testq %rsi, %rsi
-; AVX2-NEXT: cmovneq %r10, %rax
+; AVX2-NEXT: testq %r14, %r14
+; AVX2-NEXT: cmovneq %r15, %rax
; AVX2-NEXT: subq $-128, %rax
-; AVX2-NEXT: orq %rcx, %rdx
-; AVX2-NEXT: cmovneq %r9, %rax
+; AVX2-NEXT: orq %r9, %r10
+; AVX2-NEXT: cmovneq %rbx, %rax
; AVX2-NEXT: addq $256, %rax # imm = 0x100
-; AVX2-NEXT: vpor 16(%rdi), %xmm0, %xmm0
-; AVX2-NEXT: vptest %xmm0, %xmm0
-; AVX2-NEXT: cmovneq %r8, %rax
+; AVX2-NEXT: orq %r8, %rsi
+; AVX2-NEXT: orq %rdx, %rcx
+; AVX2-NEXT: orq %rsi, %rcx
+; AVX2-NEXT: cmovneq %r11, %rax
; AVX2-NEXT: movl $-2, %edx
; AVX2-NEXT: movl %eax, %ecx
; AVX2-NEXT: roll %cl, %edx
@@ -2052,6 +1990,8 @@ define i32 @blsr_u512(ptr %word) nounwind {
; AVX2-NEXT: andl %edx, (%rdi,%rcx)
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; AVX2-NEXT: popq %rbx
+; AVX2-NEXT: popq %r14
+; AVX2-NEXT: popq %r15
; AVX2-NEXT: retq
;
; AVX512-LABEL: blsr_u512:
diff --git a/llvm/test/CodeGen/X86/extract-lowbits.ll b/llvm/test/CodeGen/X86/extract-lowbits.ll
index aff5ac7437a29..077e7770c83cd 100644
--- a/llvm/test/CodeGen/X86/extract-lowbits.ll
+++ b/llvm/test/CodeGen/X86/extract-lowbits.ll
@@ -439,14 +439,14 @@ define i64 @bzhi64_a0_masked(i64 %val, i64 %numlowbits) nounwind {
;
; X64-BMI1-LABEL: bzhi64_a0_masked:
; X64-BMI1: # %bb.0:
-; X64-BMI1-NEXT: andl $63, %esi
+; X64-BMI1-NEXT: andb $63, %sil
; X64-BMI1-NEXT: shll $8, %esi
; X64-BMI1-NEXT: bextrq %rsi, %rdi, %rax
; X64-BMI1-NEXT: retq
;
; X64-BMI2-LABEL: bzhi64_a0_masked:
; X64-BMI2: # %bb.0:
-; X64-BMI2-NEXT: andl $63, %esi
+; X64-BMI2-NEXT: andb $63, %sil
; X64-BMI2-NEXT: bzhiq %rsi, %rdi, %rax
; X64-BMI2-NEXT: retq
%numlowbits.masked = and i64 %numlowbits, 63
diff --git a/llvm/test/CodeGen/X86/i128-udiv.ll b/llvm/test/CodeGen/X86/i128-udiv.ll
index e347c5944c2b9..5c41621650c36 100644
--- a/llvm/test/CodeGen/X86/i128-udiv.ll
+++ b/llvm/test/CodeGen/X86/i128-udiv.ll
@@ -44,95 +44,92 @@ define i128 @test2(i128 %x) nounwind {
; X86-NEXT: pushl %esi
; X86-NEXT: andl $-16, %esp
; X86-NEXT: subl $144, %esp
-; X86-NEXT: movl 32(%ebp), %esi
-; X86-NEXT: movl 36(%ebp), %edi
-; X86-NEXT: movl 28(%ebp), %ecx
-; X86-NEXT: testl %edi, %edi
+; X86-NEXT: movl 32(%ebp), %edx
+; X86-NEXT: movl 24(%ebp), %edi
+; X86-NEXT: movl 36(%ebp), %esi
+; X86-NEXT: movl 28(%ebp), %eax
+; X86-NEXT: orl %esi, %eax
+; X86-NEXT: movl %edi, %ecx
+; X86-NEXT: orl %edx, %ecx
+; X86-NEXT: orl %eax, %ecx
+; X86-NEXT: sete %bl
+; X86-NEXT: testl %esi, %esi
; X86-NEXT: jne .LBB1_1
; X86-NEXT: # %bb.2: # %_udiv-special-cases
-; X86-NEXT: bsrl %esi, %ebx
-; X86-NEXT: xorl $31, %ebx
-; X86-NEXT: orl $32, %ebx
-; X86-NEXT: jmp .LBB1_3
-; X86-NEXT: .LBB1_1:
-; X86-NEXT: bsrl %edi, %ebx
-; X86-NEXT: xorl $31, %ebx
-; X86-NEXT: .LBB1_3: # %_udiv-special-cases
-; X86-NEXT: movl 24(%ebp), %edx
-; X86-NEXT: testl %ecx, %ecx
-; X86-NEXT: jne .LBB1_4
-; X86-NEXT: # %bb.5: # %_udiv-special-cases
; X86-NEXT: bsrl %edx, %eax
; X86-NEXT: xorl $31, %eax
; X86-NEXT: orl $32, %eax
-; X86-NEXT: jmp .LBB1_6
+; X86-NEXT: cmpl $0, 28(%ebp)
+; X86-NEXT: je .LBB1_5
; X86-NEXT: .LBB1_4:
-; X86-NEXT: bsrl %ecx, %eax
+; X86-NEXT: bsrl 28(%ebp), %ecx
+; X86-NEXT: xorl $31, %ecx
+; X86-NEXT: jmp .LBB1_6
+; X86-NEXT: .LBB1_1:
+; X86-NEXT: bsrl %esi, %eax
; X86-NEXT: xorl $31, %eax
+; X86-NEXT: cmpl $0, 28(%ebp)
+; X86-NEXT: jne .LBB1_4
+; X86-NEXT: .LBB1_5: # %_udiv-special-cases
+; X86-NEXT: bsrl %edi, %ecx
+; X86-NEXT: xorl $31, %ecx
+; X86-NEXT: orl $32, %ecx
; X86-NEXT: .LBB1_6: # %_udiv-special-cases
-; X86-NEXT: orl %edi, %ecx
+; X86-NEXT: movl $0, (%esp) # 4-byte Folded Spill
; X86-NEXT: orl %esi, %edx
-; X86-NEXT: orl %edi, %esi
; X86-NEXT: jne .LBB1_8
; X86-NEXT: # %bb.7: # %_udiv-special-cases
-; X86-NEXT: orl $64, %eax
-; X86-NEXT: movl %eax, %ebx
+; X86-NEXT: orl $64, %ecx
+; X86-NEXT: movl %ecx, %eax
; X86-NEXT: .LBB1_8: # %_udiv-special-cases
-; X86-NEXT: negl %ebx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: negl %eax
; X86-NEXT: movl $0, %esi
; X86-NEXT: sbbl %esi, %esi
-; X86-NEXT: movl $0, %ebx
-; X86-NEXT: sbbl %ebx, %ebx
-; X86-NEXT: movl $0, %edi
-; X86-NEXT: sbbl %edi, %edi
-; X86-NEXT: orl %ecx, %edx
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: je .LBB1_9
-; X86-NEXT: # %bb.10: # %select.false.sink
-; X86-NEXT: xorl %eax, %eax
+; X86-NEXT: movl $0, %edx
+; X86-NEXT: sbbl %edx, %edx
+; X86-NEXT: movl %eax, %edi
+; X86-NEXT: movl $0, %eax
+; X86-NEXT: sbbl %eax, %eax
; X86-NEXT: movl $127, %ecx
-; X86-NEXT: cmpl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: cmpl %edi, %ecx
; X86-NEXT: movl $0, %ecx
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: sbbl %esi, %ecx
; X86-NEXT: movl $0, %ecx
-; X86-NEXT: sbbl %ebx, %ecx
-; X86-NEXT: sbbl %edi, %eax
-; X86-NEXT: setb %cl
-; X86-NEXT: .LBB1_11: # %select.end
-; X86-NEXT: movl %ebx, %esi
+; X86-NEXT: sbbl %edx, %ecx
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %eax, %ecx
; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: movl $0, (%esp) # 4-byte Folded Spill
-; X86-NEXT: testb %cl, %cl
-; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: setb %cl
+; X86-NEXT: orb %cl, %bl
+; X86-NEXT: testb %bl, %bl
+; X86-NEXT: movl $0, %esi
; X86-NEXT: movl $0, %edi
-; X86-NEXT: movl $0, %ecx
-; X86-NEXT: jne .LBB1_13
-; X86-NEXT: # %bb.12: # %select.end
-; X86-NEXT: movl 28(%ebp), %ebx
-; X86-NEXT: movl %ebx, (%esp) # 4-byte Spill
-; X86-NEXT: movl 24(%ebp), %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 32(%ebp), %edi
+; X86-NEXT: movl $0, %ebx
+; X86-NEXT: jne .LBB1_10
+; X86-NEXT: # %bb.9: # %_udiv-special-cases
; X86-NEXT: movl 36(%ebp), %ecx
-; X86-NEXT: .LBB1_13: # %select.end
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: jne .LBB1_14
-; X86-NEXT: # %bb.20: # %select.end
+; X86-NEXT: movl %ecx, (%esp) # 4-byte Spill
+; X86-NEXT: movl 32(%ebp), %esi
+; X86-NEXT: movl 28(%ebp), %edi
+; X86-NEXT: movl 24(%ebp), %ebx
+; X86-NEXT: .LBB1_10: # %_udiv-special-cases
+; X86-NEXT: jne .LBB1_11
+; X86-NEXT: # %bb.17: # %_udiv-special-cases
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl %ecx, %ebx
; X86-NEXT: xorl $127, %ecx
-; X86-NEXT: orl %esi, %ecx
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %edx, %ecx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
; X86-NEXT: orl %ecx, %edx
-; X86-NEXT: movl (%esp), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: je .LBB1_21
-; X86-NEXT: # %bb.18: # %udiv-bb1
+; X86-NEXT: movl (%esp), %edx # 4-byte Reload
+; X86-NEXT: je .LBB1_18
+; X86-NEXT: # %bb.15: # %udiv-bb1
; X86-NEXT: movl 24(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: movl 28(%ebp), %eax
@@ -145,6 +142,7 @@ define i128 @test2(i128 %x) nounwind {
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NEXT: movl %ebx, %ecx
; X86-NEXT: xorb $127, %cl
; X86-NEXT: movl %ecx, %eax
@@ -152,25 +150,35 @@ define i128 @test2(i128 %x) nounwind {
; X86-NEXT: andb $12, %al
; X86-NEXT: negb %al
; X86-NEXT: movsbl %al, %eax
-; X86-NEXT: movl 120(%esp,%eax), %edx
-; X86-NEXT: movl 124(%esp,%eax), %edi
-; X86-NEXT: shldl %cl, %edx, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 112(%esp,%eax), %edi
+; X86-NEXT: movl 120(%esp,%eax), %esi
+; X86-NEXT: movl 124(%esp,%eax), %edx
+; X86-NEXT: shldl %cl, %esi, %edx
+; X86-NEXT: movl %edx, (%esp) # 4-byte Spill
+; X86-NEXT: movl 112(%esp,%eax), %edx
; X86-NEXT: movl 116(%esp,%eax), %eax
-; X86-NEXT: shldl %cl, %eax, %edx
+; X86-NEXT: shldl %cl, %eax, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shldl %cl, %edx, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shll %cl, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shldl %cl, %edi, %eax
-; X86-NEXT: movl %eax, (%esp) # 4-byte Spill
-; X86-NEXT: shll %cl, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: addl $1, %ebx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: jb .LBB1_19
-; X86-NEXT: # %bb.15: # %udiv-preheader
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: adcl $0, %edx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: adcl $0, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: adcl $0, %esi
+; X86-NEXT: jae .LBB1_12
+; X86-NEXT: # %bb.16:
+; X86-NEXT: movl (%esp), %eax # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: jmp .LBB1_14
+; X86-NEXT: .LBB1_11:
+; X86-NEXT: movl (%esp), %edx # 4-byte Reload
+; X86-NEXT: jmp .LBB1_18
+; X86-NEXT: .LBB1_12: # %udiv-preheader
; X86-NEXT: movl 24(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: movl 28(%ebp), %eax
@@ -183,26 +191,29 @@ define i128 @test2(i128 %x) nounwind {
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ebx, %eax
; X86-NEXT: shrb $3, %al
; X86-NEXT: andb $12, %al
-; X86-NEXT: movzbl %al, %esi
-; X86-NEXT: movl 76(%esp,%esi), %eax
-; X86-NEXT: movl 72(%esp,%esi), %edi
-; X86-NEXT: movl %edi, %edx
-; X86-NEXT: shrdl %cl, %eax, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 64(%esp,%esi), %edx
-; X86-NEXT: movl 68(%esp,%esi), %esi
-; X86-NEXT: movl %esi, %ebx
-; X86-NEXT: shrdl %cl, %edi, %ebx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shrl %cl, %eax
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movzbl %al, %edx
+; X86-NEXT: movl 76(%esp,%edx), %esi
+; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: movl 72(%esp,%edx), %ebx
+; X86-NEXT: movl %ebx, %eax
+; X86-NEXT: shrdl %cl, %esi, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NEXT: shrdl %cl, %esi, %edx
+; X86-NEXT: movl 64(%esp,%edx), %eax
+; X86-NEXT: movl 68(%esp,%edx), %edi
+; X86-NEXT: movl %edi, %edx
+; X86-NEXT: shrdl %cl, %ebx, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shrl %cl, %esi
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: shrdl %cl, %edi, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $-4, %eax
; X86-NEXT: addl $-1, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
@@ -210,95 +221,94 @@ define i128 @test2(i128 %x) nounwind {
; X86-NEXT: adcl $-1, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: movl (%esp), %eax # 4-byte Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: .p2align 4
-; X86-NEXT: .LBB1_16: # %udiv-do-while
+; X86-NEXT: .LBB1_13: # %udiv-do-while
; X86-NEXT: # =>This Inner Loop Header: Depth=1
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: shldl $1, %edx, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: shldl $1, %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: shldl $1, %esi, %ebx
+; X86-NEXT: shldl $1, %ebx, %edx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: shldl $1, %ecx, %esi
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: shldl $1, %edi, %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shldl $1, %eax, %edi
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: orl %edx, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl (%esp), %edi # 4-byte Reload
+; X86-NEXT: shldl $1, %ecx, %ebx
+; X86-NEXT: shldl $1, %eax, %ecx
; X86-NEXT: shldl $1, %edi, %eax
-; X86-NEXT: orl %edx, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: orl %esi, %eax
+; X86-NEXT: movl %eax, (%esp) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: shldl $1, %eax, %edi
-; X86-NEXT: orl %edx, %edi
-; X86-NEXT: movl %edi, (%esp) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: addl %eax, %eax
+; X86-NEXT: orl %esi, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: shldl $1, %esi, %eax
; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl (%esp), %eax # 4-byte Reload
+; X86-NEXT: addl %esi, %esi
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $-1, %eax
-; X86-NEXT: cmpl %ecx, %eax
+; X86-NEXT: movl $-1, %esi
+; X86-NEXT: cmpl %ecx, %esi
; X86-NEXT: movl $-1, %ecx
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: sbbl %esi, %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: sbbl %ebx, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NEXT: sbbl %edx, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
; X86-NEXT: sarl $31, %ecx
-; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: andl $1, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: movl %ecx, %esi
-; X86-NEXT: movl $-1, %eax
-; X86-NEXT: andl %eax, %esi
-; X86-NEXT: movl $-4, %eax
-; X86-NEXT: andl %eax, %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: subl %ecx, %ebx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %edx, %ecx
-; X86-NEXT: sbbl %esi, %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: andl $1, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: movl %ecx, %ebx
+; X86-NEXT: movl $-1, %edi
+; X86-NEXT: andl %edi, %ebx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: movl $-4, %esi
+; X86-NEXT: andl %esi, %ecx
+; X86-NEXT: subl %ecx, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NEXT: addl $-1, %esi
-; X86-NEXT: adcl $-1, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: adcl $-1, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NEXT: adcl $-1, %edx
-; X86-NEXT: adcl $-1, %edi
-; X86-NEXT: movl %eax, %ecx
-; X86-NEXT: orl %edi, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: adcl $-1, %ebx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %ebx, %ecx
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %edx, %esi
-; X86-NEXT: orl %ecx, %esi
-; X86-NEXT: jne .LBB1_16
-; X86-NEXT: .LBB1_17: # %udiv-loop-exit
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: shldl $1, %edi, %edx
-; X86-NEXT: movl (%esp), %ecx # 4-byte Reload
+; X86-NEXT: movl %esi, %ebx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %edx, %ebx
+; X86-NEXT: orl %ecx, %ebx
+; X86-NEXT: jne .LBB1_13
+; X86-NEXT: .LBB1_14: # %udiv-loop-exit
+; X86-NEXT: shldl $1, %edi, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: shldl $1, %ecx, %edi
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shldl $1, %eax, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: shldl $1, %edx, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: leal (%esi,%eax,2), %esi
+; X86-NEXT: leal (%esi,%edx,2), %ebx
+; X86-NEXT: movl %eax, %edx
; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: .LBB1_21: # %udiv-end
-; X86-NEXT: movl %esi, (%eax)
-; X86-NEXT: movl %ecx, 4(%eax)
-; X86-NEXT: movl %edi, 8(%eax)
+; X86-NEXT: movl %edi, %esi
+; X86-NEXT: movl %ecx, %edi
+; X86-NEXT: .LBB1_18: # %udiv-end
+; X86-NEXT: movl %ebx, (%eax)
+; X86-NEXT: movl %edi, 4(%eax)
+; X86-NEXT: movl %esi, 8(%eax)
; X86-NEXT: movl %edx, 12(%eax)
; X86-NEXT: leal -12(%ebp), %esp
; X86-NEXT: popl %esi
@@ -306,17 +316,6 @@ define i128 @test2(i128 %x) nounwind {
; X86-NEXT: popl %ebx
; X86-NEXT: popl %ebp
; X86-NEXT: retl $4
-; X86-NEXT: .LBB1_9:
-; X86-NEXT: movb $1, %cl
-; X86-NEXT: jmp .LBB1_11
-; X86-NEXT: .LBB1_19:
-; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: jmp .LBB1_17
-; X86-NEXT: .LBB1_14:
-; X86-NEXT: movl (%esp), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: jmp .LBB1_21
;
; X64-LABEL: test2:
; X64: # %bb.0:
@@ -341,107 +340,105 @@ define i128 @test3(i128 %x) nounwind {
; X86-NEXT: pushl %esi
; X86-NEXT: andl $-16, %esp
; X86-NEXT: subl $160, %esp
-; X86-NEXT: movl 32(%ebp), %edi
-; X86-NEXT: movl 36(%ebp), %edx
-; X86-NEXT: movl 28(%ebp), %esi
-; X86-NEXT: testl %edx, %edx
+; X86-NEXT: movl 32(%ebp), %edx
+; X86-NEXT: movl 24(%ebp), %edi
+; X86-NEXT: movl 36(%ebp), %esi
+; X86-NEXT: movl 28(%ebp), %eax
+; X86-NEXT: orl %esi, %eax
+; X86-NEXT: movl %edi, %ecx
+; X86-NEXT: orl %edx, %ecx
+; X86-NEXT: orl %eax, %ecx
+; X86-NEXT: sete {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X86-NEXT: testl %esi, %esi
; X86-NEXT: jne .LBB2_1
; X86-NEXT: # %bb.2: # %_udiv-special-cases
-; X86-NEXT: bsrl %edi, %ebx
+; X86-NEXT: bsrl %edx, %ebx
; X86-NEXT: xorl $31, %ebx
; X86-NEXT: orl $32, %ebx
; X86-NEXT: jmp .LBB2_3
; X86-NEXT: .LBB2_1:
-; X86-NEXT: bsrl %edx, %ebx
+; X86-NEXT: bsrl %esi, %ebx
; X86-NEXT: xorl $31, %ebx
; X86-NEXT: .LBB2_3: # %_udiv-special-cases
-; X86-NEXT: movl 24(%ebp), %eax
-; X86-NEXT: testl %esi, %esi
+; X86-NEXT: movl 28(%ebp), %eax
+; X86-NEXT: testl %eax, %eax
; X86-NEXT: jne .LBB2_4
; X86-NEXT: # %bb.5: # %_udiv-special-cases
-; X86-NEXT: bsrl %eax, %ecx
+; X86-NEXT: bsrl %edi, %ecx
; X86-NEXT: xorl $31, %ecx
; X86-NEXT: orl $32, %ecx
-; X86-NEXT: jmp .LBB2_6
+; X86-NEXT: orl %esi, %edx
+; X86-NEXT: je .LBB2_7
+; X86-NEXT: jmp .LBB2_8
; X86-NEXT: .LBB2_4:
-; X86-NEXT: bsrl %esi, %ecx
+; X86-NEXT: bsrl %eax, %ecx
; X86-NEXT: xorl $31, %ecx
-; X86-NEXT: .LBB2_6: # %_udiv-special-cases
-; X86-NEXT: orl %edx, %esi
-; X86-NEXT: orl %edi, %eax
-; X86-NEXT: orl %edx, %edi
+; X86-NEXT: orl %esi, %edx
; X86-NEXT: jne .LBB2_8
-; X86-NEXT: # %bb.7: # %_udiv-special-cases
+; X86-NEXT: .LBB2_7: # %_udiv-special-cases
; X86-NEXT: orl $64, %ecx
; X86-NEXT: movl %ecx, %ebx
; X86-NEXT: .LBB2_8: # %_udiv-special-cases
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: negl %ebx
-; X86-NEXT: movl $0, %ecx
-; X86-NEXT: sbbl %ecx, %ecx
; X86-NEXT: movl $0, %edx
; X86-NEXT: sbbl %edx, %edx
-; X86-NEXT: movl %ebx, %edi
-; X86-NEXT: movl $0, %ebx
-; X86-NEXT: sbbl %ebx, %ebx
-; X86-NEXT: orl %esi, %eax
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: je .LBB2_9
-; X86-NEXT: # %bb.10: # %select.false.sink
-; X86-NEXT: movl $127, %eax
-; X86-NEXT: cmpl %edi, %eax
-; X86-NEXT: movl $0, %eax
-; X86-NEXT: movl %ecx, %edi
-; X86-NEXT: sbbl %ecx, %eax
-; X86-NEXT: movl $0, %eax
-; X86-NEXT: sbbl %edx, %eax
+; X86-NEXT: movl $0, %esi
+; X86-NEXT: sbbl %esi, %esi
; X86-NEXT: movl $0, %eax
-; X86-NEXT: sbbl %ebx, %eax
+; X86-NEXT: sbbl %eax, %eax
+; X86-NEXT: movl $127, %ecx
+; X86-NEXT: cmpl %ebx, %ecx
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: sbbl %edx, %ecx
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %esi, %ecx
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %eax, %ecx
; X86-NEXT: setb %cl
-; X86-NEXT: .LBB2_11: # %select.end
-; X86-NEXT: movl %ebx, %esi
+; X86-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X86-NEXT: orb %cl, %al
+; X86-NEXT: testb %al, %al
; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: testb %cl, %cl
-; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: movl $0, %ecx
-; X86-NEXT: movl $0, %ebx
-; X86-NEXT: jne .LBB2_13
-; X86-NEXT: # %bb.12: # %select.end
-; X86-NEXT: movl 28(%ebp), %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 24(%ebp), %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: movl $0, %esi
+; X86-NEXT: jne .LBB2_10
+; X86-NEXT: # %bb.9: # %_udiv-special-cases
+; X86-NEXT: movl 36(%ebp), %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl 32(%ebp), %ecx
-; X86-NEXT: movl 36(%ebp), %ebx
-; X86-NEXT: .LBB2_13: # %select.end
+; X86-NEXT: movl 28(%ebp), %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 24(%ebp), %esi
+; X86-NEXT: .LBB2_10: # %_udiv-special-cases
+; X86-NEXT: jne .LBB2_16
+; X86-NEXT: # %bb.11: # %_udiv-special-cases
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: jne .LBB2_14
-; X86-NEXT: # %bb.20: # %select.end
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %ebx, %ecx
; X86-NEXT: xorl $127, %ecx
-; X86-NEXT: orl %edx, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: orl %edi, %ecx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %edi, %edx
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %esi, %edx
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
; X86-NEXT: orl %ecx, %edx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: je .LBB2_21
-; X86-NEXT: # %bb.18: # %udiv-bb1
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: je .LBB2_16
+; X86-NEXT: # %bb.12: # %udiv-bb1
; X86-NEXT: movl 24(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: movl 28(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 32(%ebp), %ebx
-; X86-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 36(%ebp), %ebx
-; X86-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 32(%ebp), %eax
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 36(%ebp), %eax
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NEXT: movl %ebx, %ecx
; X86-NEXT: xorb $127, %cl
; X86-NEXT: movl %ecx, %eax
@@ -453,54 +450,67 @@ define i128 @test3(i128 %x) nounwind {
; X86-NEXT: movl 140(%esp,%eax), %esi
; X86-NEXT: shldl %cl, %edx, %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 128(%esp,%eax), %esi
-; X86-NEXT: movl 132(%esp,%eax), %eax
-; X86-NEXT: shldl %cl, %eax, %edx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 128(%esp,%eax), %ebx
+; X86-NEXT: movl 132(%esp,%eax), %esi
+; X86-NEXT: shldl %cl, %esi, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shldl %cl, %esi, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shll %cl, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: addl $1, %ebx
+; X86-NEXT: shldl %cl, %ebx, %esi
+; X86-NEXT: shll %cl, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: addl $1, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: adcl $0, %ebx
; X86-NEXT: adcl $0, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: jb .LBB2_19
-; X86-NEXT: # %bb.15: # %udiv-preheader
-; X86-NEXT: movl %ebx, %ecx
-; X86-NEXT: movl 24(%ebp), %eax
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 28(%ebp), %eax
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 32(%ebp), %eax
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 36(%ebp), %eax
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: adcl $0, %edx
+; X86-NEXT: jae .LBB2_13
+; X86-NEXT: # %bb.17:
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: jmp .LBB2_15
+; X86-NEXT: .LBB2_13: # %udiv-preheader
+; X86-NEXT: movl %edi, %eax
+; X86-NEXT: movl 24(%ebp), %edi
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 28(%ebp), %edi
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 32(%ebp), %edi
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 36(%ebp), %edi
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: shrb $3, %al
; X86-NEXT: andb $12, %al
-; X86-NEXT: movzbl %al, %edx
-; X86-NEXT: movl 92(%esp,%edx), %edi
-; X86-NEXT: movl 88(%esp,%edx), %esi
-; X86-NEXT: movl %esi, %eax
-; X86-NEXT: shrdl %cl, %edi, %eax
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movzbl %al, %esi
+; X86-NEXT: movl 92(%esp,%esi), %edi
+; X86-NEXT: movl 88(%esp,%esi), %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 80(%esp,%edx), %eax
-; X86-NEXT: movl 84(%esp,%edx), %ebx
-; X86-NEXT: movl %ebx, %edx
-; X86-NEXT: shrdl %cl, %esi, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %eax, %edx
+; X86-NEXT: shrdl %cl, %edi, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 80(%esp,%esi), %edx
+; X86-NEXT: movl 84(%esp,%esi), %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: shrdl %cl, %esi, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: shrl %cl, %edi
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NEXT: shrdl %cl, %ebx, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: shrdl %cl, %eax, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NEXT: movl $-3, %eax
; X86-NEXT: addl $-1, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
@@ -514,120 +524,103 @@ define i128 @test3(i128 %x) nounwind {
; X86-NEXT: adcl $-1, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: .p2align 4
-; X86-NEXT: .LBB2_16: # %udiv-do-while
+; X86-NEXT: .LBB2_14: # %udiv-do-while
; X86-NEXT: # =>This Inner Loop Header: Depth=1
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: shldl $1, %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: shldl $1, %edi, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: shldl $1, %edx, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: shldl $1, %ebx, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: shldl $1, %edi, %ebx
-; X86-NEXT: shldl $1, %ecx, %edi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: orl %eax, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shldl $1, %esi, %ecx
-; X86-NEXT: orl %eax, %ecx
+; X86-NEXT: shldl $1, %eax, %edx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: shldl $1, %ecx, %eax
+; X86-NEXT: shldl $1, %edi, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: orl %ebx, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shldl $1, %edx, %esi
-; X86-NEXT: orl %eax, %esi
+; X86-NEXT: shldl $1, %esi, %edi
+; X86-NEXT: orl %ebx, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: shldl $1, %ecx, %esi
+; X86-NEXT: orl %ebx, %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: addl %edx, %edx
-; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: cmpl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; X86-NEXT: addl %ecx, %ecx
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: cmpl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: sbbl %eax, %ecx
+; X86-NEXT: sbbl %edx, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
; X86-NEXT: sarl $31, %ecx
-; X86-NEXT: movl %ecx, %edx
-; X86-NEXT: andl $1, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: andl $1, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: movl %ecx, %edi
; X86-NEXT: movl $-1, %esi
; X86-NEXT: andl %esi, %edi
; X86-NEXT: movl %ecx, %esi
-; X86-NEXT: movl $-5, %edx
-; X86-NEXT: andl %edx, %esi
-; X86-NEXT: movl $-3, %edx
-; X86-NEXT: andl %edx, %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: subl %ecx, %ebx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: sbbl %edi, %eax
+; X86-NEXT: movl $-5, %ebx
+; X86-NEXT: andl %ebx, %esi
+; X86-NEXT: movl $-3, %ebx
+; X86-NEXT: andl %ebx, %ecx
+; X86-NEXT: subl %ecx, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: sbbl %edi, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: sbbl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: sbbl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: addl $-1, %esi
-; X86-NEXT: adcl $-1, %eax
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: addl $-1, %eax
; X86-NEXT: adcl $-1, %ebx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-NEXT: adcl $-1, %edi
-; X86-NEXT: movl %eax, %ecx
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %edi, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: adcl $-1, %esi
+; X86-NEXT: movl %ebx, %ecx
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %ebx, %esi
+; X86-NEXT: orl %esi, %ecx
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %eax, %esi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %edi, %esi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-NEXT: orl %ecx, %esi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: jne .LBB2_16
-; X86-NEXT: .LBB2_17: # %udiv-loop-exit
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: shldl $1, %ecx, %ebx
-; X86-NEXT: shldl $1, %esi, %ecx
-; X86-NEXT: shldl $1, %edx, %esi
+; X86-NEXT: jne .LBB2_14
+; X86-NEXT: .LBB2_15: # %udiv-loop-exit
+; X86-NEXT: shldl $1, %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: shldl $1, %esi, %edi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: leal (%eax,%edx,2), %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shldl $1, %eax, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: leal (%ecx,%eax,2), %esi
; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: movl %ecx, %edx
-; X86-NEXT: .LBB2_21: # %udiv-end
+; X86-NEXT: movl %edi, %ecx
+; X86-NEXT: .LBB2_16: # %udiv-end
+; X86-NEXT: movl %esi, (%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: movl %edx, 4(%eax)
+; X86-NEXT: movl %ecx, 8(%eax)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl %ecx, (%eax)
-; X86-NEXT: movl %esi, 4(%eax)
-; X86-NEXT: movl %edx, 8(%eax)
-; X86-NEXT: movl %ebx, 12(%eax)
+; X86-NEXT: movl %ecx, 12(%eax)
; X86-NEXT: leal -12(%ebp), %esp
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
; X86-NEXT: popl %ebx
; X86-NEXT: popl %ebp
; X86-NEXT: retl $4
-; X86-NEXT: .LBB2_9:
-; X86-NEXT: movl %ecx, %edi
-; X86-NEXT: movb $1, %cl
-; X86-NEXT: jmp .LBB2_11
-; X86-NEXT: .LBB2_19:
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: jmp .LBB2_17
-; X86-NEXT: .LBB2_14:
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: jmp .LBB2_21
;
; X64-LABEL: test3:
; X64: # %bb.0:
@@ -651,168 +644,178 @@ define i128 @div_by_7(i128 %x) nounwind {
; X86-NEXT: pushl %esi
; X86-NEXT: andl $-16, %esp
; X86-NEXT: subl $160, %esp
-; X86-NEXT: movl 32(%ebp), %edi
-; X86-NEXT: movl 36(%ebp), %ebx
-; X86-NEXT: movl 28(%ebp), %edx
-; X86-NEXT: testl %ebx, %ebx
+; X86-NEXT: movl 32(%ebp), %esi
+; X86-NEXT: movl 24(%ebp), %edx
+; X86-NEXT: movl 36(%ebp), %edi
+; X86-NEXT: movl 28(%ebp), %ebx
+; X86-NEXT: movl %ebx, %eax
+; X86-NEXT: orl %edi, %eax
+; X86-NEXT: movl %edx, %ecx
+; X86-NEXT: orl %esi, %ecx
+; X86-NEXT: orl %eax, %ecx
+; X86-NEXT: sete {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X86-NEXT: testl %edi, %edi
; X86-NEXT: jne .LBB3_1
; X86-NEXT: # %bb.2: # %entry_udiv-special-cases
-; X86-NEXT: bsrl %edi, %esi
-; X86-NEXT: xorl $31, %esi
-; X86-NEXT: orl $32, %esi
-; X86-NEXT: jmp .LBB3_3
-; X86-NEXT: .LBB3_1:
-; X86-NEXT: bsrl %ebx, %esi
-; X86-NEXT: xorl $31, %esi
-; X86-NEXT: .LBB3_3: # %entry_udiv-special-cases
-; X86-NEXT: movl 24(%ebp), %eax
-; X86-NEXT: testl %edx, %edx
-; X86-NEXT: jne .LBB3_4
-; X86-NEXT: # %bb.5: # %entry_udiv-special-cases
-; X86-NEXT: bsrl %eax, %ecx
+; X86-NEXT: bsrl %esi, %ecx
; X86-NEXT: xorl $31, %ecx
; X86-NEXT: orl $32, %ecx
-; X86-NEXT: jmp .LBB3_6
+; X86-NEXT: testl %ebx, %ebx
+; X86-NEXT: je .LBB3_5
; X86-NEXT: .LBB3_4:
-; X86-NEXT: bsrl %edx, %ecx
+; X86-NEXT: bsrl %ebx, %edx
+; X86-NEXT: xorl $31, %edx
+; X86-NEXT: jmp .LBB3_6
+; X86-NEXT: .LBB3_1:
+; X86-NEXT: bsrl %edi, %ecx
; X86-NEXT: xorl $31, %ecx
+; X86-NEXT: testl %ebx, %ebx
+; X86-NEXT: jne .LBB3_4
+; X86-NEXT: .LBB3_5: # %entry_udiv-special-cases
+; X86-NEXT: bsrl %edx, %edx
+; X86-NEXT: xorl $31, %edx
+; X86-NEXT: orl $32, %edx
; X86-NEXT: .LBB3_6: # %entry_udiv-special-cases
-; X86-NEXT: orl %ebx, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %edi, %eax
-; X86-NEXT: orl %ebx, %edi
+; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: orl %edi, %esi
; X86-NEXT: jne .LBB3_8
; X86-NEXT: # %bb.7: # %entry_udiv-special-cases
-; X86-NEXT: orl $64, %ecx
-; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: orl $64, %edx
+; X86-NEXT: movl %edx, %ecx
; X86-NEXT: .LBB3_8: # %entry_udiv-special-cases
-; X86-NEXT: movl $125, %ebx
-; X86-NEXT: subl %esi, %ebx
-; X86-NEXT: movl $0, %edx
-; X86-NEXT: sbbl %edx, %edx
+; X86-NEXT: movl $125, %edx
+; X86-NEXT: subl %ecx, %edx
+; X86-NEXT: movl $0, %ebx
+; X86-NEXT: sbbl %ebx, %ebx
; X86-NEXT: movl $0, %esi
; X86-NEXT: sbbl %esi, %esi
-; X86-NEXT: movl $0, %edi
-; X86-NEXT: sbbl %edi, %edi
-; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: je .LBB3_9
-; X86-NEXT: # %bb.10: # %select.false.sink
-; X86-NEXT: xorl %eax, %eax
+; X86-NEXT: movl $0, %eax
+; X86-NEXT: sbbl %eax, %eax
; X86-NEXT: movl $127, %ecx
-; X86-NEXT: cmpl %ebx, %ecx
+; X86-NEXT: cmpl %edx, %ecx
; X86-NEXT: movl $0, %ecx
-; X86-NEXT: sbbl %edx, %ecx
+; X86-NEXT: sbbl %ebx, %ecx
; X86-NEXT: movl $0, %ecx
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: sbbl %esi, %ecx
-; X86-NEXT: sbbl %edi, %eax
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %eax, %ecx
; X86-NEXT: setb %cl
-; X86-NEXT: .LBB3_11: # %select.end
+; X86-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X86-NEXT: orb %cl, %al
+; X86-NEXT: testb %al, %al
; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: xorl %edx, %edx
-; X86-NEXT: testb %cl, %cl
-; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: movl $0, %esi
; X86-NEXT: movl $0, %ecx
-; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: jne .LBB3_13
-; X86-NEXT: # %bb.12: # %select.end
-; X86-NEXT: movl 28(%ebp), %edx
-; X86-NEXT: movl 24(%ebp), %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 32(%ebp), %ecx
-; X86-NEXT: movl 36(%ebp), %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: .LBB3_13: # %select.end
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: jne .LBB3_14
-; X86-NEXT: # %bb.20: # %select.end
+; X86-NEXT: movl $0, %edi
+; X86-NEXT: jne .LBB3_10
+; X86-NEXT: # %bb.9: # %entry_udiv-special-cases
; X86-NEXT: movl %ebx, %ecx
-; X86-NEXT: xorl $127, %ecx
-; X86-NEXT: orl %esi, %ecx
+; X86-NEXT: movl 36(%ebp), %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ecx, %ebx
+; X86-NEXT: movl 32(%ebp), %esi
+; X86-NEXT: movl 28(%ebp), %ecx
+; X86-NEXT: movl 24(%ebp), %edi
+; X86-NEXT: .LBB3_10: # %entry_udiv-special-cases
+; X86-NEXT: jne .LBB3_16
+; X86-NEXT: # %bb.11: # %entry_udiv-special-cases
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: movl %esi, %edx
-; X86-NEXT: orl %edi, %edx
+; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: movl %edx, %ecx
+; X86-NEXT: xorl $127, %ecx
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ebx, %edx
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
; X86-NEXT: orl %ecx, %edx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: je .LBB3_21
-; X86-NEXT: # %bb.18: # %udiv-bb1
+; X86-NEXT: movl %esi, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: je .LBB3_16
+; X86-NEXT: # %bb.12: # %udiv-bb1
; X86-NEXT: movl 24(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: movl 28(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 32(%ebp), %eax
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 32(%ebp), %edi
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X86-NEXT: movl 36(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
-; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: movl %eax, %edi
; X86-NEXT: xorb $127, %cl
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: shrb $3, %al
; X86-NEXT: andb $12, %al
; X86-NEXT: negb %al
; X86-NEXT: movsbl %al, %eax
-; X86-NEXT: movl 136(%esp,%eax), %edx
-; X86-NEXT: movl 140(%esp,%eax), %edi
-; X86-NEXT: shldl %cl, %edx, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 128(%esp,%eax), %edi
-; X86-NEXT: movl 132(%esp,%eax), %eax
-; X86-NEXT: shldl %cl, %eax, %edx
+; X86-NEXT: movl 136(%esp,%eax), %esi
+; X86-NEXT: movl 140(%esp,%eax), %edx
+; X86-NEXT: shldl %cl, %esi, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shldl %cl, %edi, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shll %cl, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: addl $1, %ebx
-; X86-NEXT: adcl $0, %esi
+; X86-NEXT: movl 128(%esp,%eax), %edx
+; X86-NEXT: movl 132(%esp,%eax), %eax
+; X86-NEXT: shldl %cl, %eax, %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: jb .LBB3_19
-; X86-NEXT: # %bb.15: # %udiv-preheader
-; X86-NEXT: movl %ebx, %ecx
-; X86-NEXT: movl 24(%ebp), %eax
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 28(%ebp), %eax
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 32(%ebp), %eax
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 36(%ebp), %eax
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: shldl %cl, %edx, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shll %cl, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: addl $1, %edi
+; X86-NEXT: adcl $0, %ebx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: adcl $0, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: adcl $0, %edx
+; X86-NEXT: jae .LBB3_13
+; X86-NEXT: # %bb.17:
+; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: jmp .LBB3_15
+; X86-NEXT: .LBB3_13: # %udiv-preheader
+; X86-NEXT: movl %edi, %ecx
+; X86-NEXT: movl 24(%ebp), %esi
+; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 28(%ebp), %esi
+; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 32(%ebp), %esi
+; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 36(%ebp), %esi
+; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: shrb $3, %al
; X86-NEXT: andb $12, %al
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movzbl %al, %edi
; X86-NEXT: movl 92(%esp,%edi), %eax
-; X86-NEXT: movl 88(%esp,%edi), %ebx
-; X86-NEXT: movl %ebx, %esi
-; X86-NEXT: shrdl %cl, %eax, %esi
+; X86-NEXT: movl 88(%esp,%edi), %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 80(%esp,%edi), %esi
-; X86-NEXT: movl 84(%esp,%edi), %edi
-; X86-NEXT: movl %edi, %edx
-; X86-NEXT: shrdl %cl, %ebx, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %esi, %edx
+; X86-NEXT: shrdl %cl, %eax, %edx
+; X86-NEXT: movl 80(%esp,%edi), %ebx
+; X86-NEXT: movl 84(%esp,%edi), %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: shrdl %cl, %esi, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: shrl %cl, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NEXT: shrdl %cl, %edi, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: shrdl %cl, %eax, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $7, %eax
; X86-NEXT: addl $-1, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
@@ -826,79 +829,78 @@ define i128 @div_by_7(i128 %x) nounwind {
; X86-NEXT: adcl $-1, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: .p2align 4
-; X86-NEXT: .LBB3_16: # %udiv-do-while
+; X86-NEXT: .LBB3_14: # %udiv-do-while
; X86-NEXT: # =>This Inner Loop Header: Depth=1
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %edx, %edi
+; X86-NEXT: shldl $1, %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: shldl $1, %eax, %edi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: shldl $1, %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: shldl $1, %edi, %ebx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: shldl $1, %edx, %edi
-; X86-NEXT: shldl $1, %eax, %edx
-; X86-NEXT: shldl $1, %ecx, %eax
+; X86-NEXT: shldl $1, %ebx, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: shldl $1, %ecx, %ebx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: orl %esi, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shldl $1, %eax, %ecx
-; X86-NEXT: orl %esi, %ecx
+; X86-NEXT: shldl $1, %esi, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: orl %edx, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: shldl $1, %ecx, %eax
-; X86-NEXT: orl %esi, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: addl %ecx, %ecx
-; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: shldl $1, %ecx, %esi
+; X86-NEXT: orl %edx, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: shldl $1, %esi, %ecx
+; X86-NEXT: orl %edx, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: cmpl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; X86-NEXT: addl %esi, %esi
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: cmpl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: sbbl %edi, %ecx
+; X86-NEXT: sbbl %eax, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: sbbl %ebx, %ecx
+; X86-NEXT: sbbl %edi, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: sbbl %edx, %ecx
; X86-NEXT: sarl $31, %ecx
-; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: andl $1, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: andl $1, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl $7, %eax
-; X86-NEXT: andl %eax, %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: subl %ecx, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $7, %esi
+; X86-NEXT: andl %esi, %ecx
+; X86-NEXT: subl %ecx, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl $0, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: sbbl $0, %edi
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: sbbl $0, %ebx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: sbbl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: movl %edx, %eax
+; X86-NEXT: sbbl $0, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NEXT: addl $-1, %ebx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: adcl $-1, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-NEXT: adcl $-1, %edi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NEXT: adcl $-1, %edx
-; X86-NEXT: adcl $-1, %esi
-; X86-NEXT: movl %edi, %ecx
-; X86-NEXT: orl %esi, %ecx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %edx, %ebx
+; X86-NEXT: orl %edx, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %edi, %ebx
; X86-NEXT: orl %ecx, %ebx
+; X86-NEXT: jne .LBB3_14
+; X86-NEXT: .LBB3_15: # %udiv-loop-exit
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: jne .LBB3_16
-; X86-NEXT: .LBB3_17: # %udiv-loop-exit
-; X86-NEXT: shldl $1, %ecx, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shldl $1, %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NEXT: shldl $1, %edx, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
@@ -906,10 +908,12 @@ define i128 @div_by_7(i128 %x) nounwind {
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NEXT: leal (%esi,%eax,2), %edi
; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: .LBB3_21: # %udiv-end
+; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: movl %edx, %ecx
+; X86-NEXT: .LBB3_16: # %udiv-end
; X86-NEXT: movl %edi, (%eax)
-; X86-NEXT: movl %edx, 4(%eax)
-; X86-NEXT: movl %ecx, 8(%eax)
+; X86-NEXT: movl %ecx, 4(%eax)
+; X86-NEXT: movl %esi, 8(%eax)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: movl %ecx, 12(%eax)
; X86-NEXT: leal -12(%ebp), %esp
@@ -918,49 +922,14 @@ define i128 @div_by_7(i128 %x) nounwind {
; X86-NEXT: popl %ebx
; X86-NEXT: popl %ebp
; X86-NEXT: retl $4
-; X86-NEXT: .LBB3_9:
-; X86-NEXT: movb $1, %cl
-; X86-NEXT: jmp .LBB3_11
-; X86-NEXT: .LBB3_19:
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: jmp .LBB3_17
-; X86-NEXT: .LBB3_14:
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: jmp .LBB3_21
;
; X64-LABEL: div_by_7:
; X64: # %bb.0: # %entry
-; X64-NEXT: movabsq $1152921504606846975, %rax # imm = 0xFFFFFFFFFFFFFFF
-; X64-NEXT: movq %rdi, %rcx
-; X64-NEXT: andq %rax, %rcx
-; X64-NEXT: movq %rdi, %rdx
-; X64-NEXT: shrdq $60, %rsi, %rdx
-; X64-NEXT: andq %rax, %rdx
-; X64-NEXT: addq %rcx, %rdx
-; X64-NEXT: movq %rsi, %rcx
-; X64-NEXT: shrq $56, %rcx
-; X64-NEXT: addq %rdx, %rcx
-; X64-NEXT: movabsq $5270498306774157605, %rdx # imm = 0x4924924924924925
-; X64-NEXT: movq %rcx, %rax
-; X64-NEXT: mulq %rdx
-; X64-NEXT: shrq %rdx
-; X64-NEXT: leaq (,%rdx,8), %rax
-; X64-NEXT: subq %rax, %rdx
-; X64-NEXT: addq %rcx, %rdx
-; X64-NEXT: subq %rdx, %rdi
-; X64-NEXT: sbbq $0, %rsi
-; X64-NEXT: movabsq $-5270498306774157605, %rcx # imm = 0xB6DB6DB6DB6DB6DB
-; X64-NEXT: imulq %rdi, %rcx
-; X64-NEXT: movabsq $7905747460161236407, %r8 # imm = 0x6DB6DB6DB6DB6DB7
-; X64-NEXT: movq %rdi, %rax
-; X64-NEXT: mulq %r8
-; X64-NEXT: addq %rcx, %rdx
-; X64-NEXT: imulq %rsi, %r8
-; X64-NEXT: addq %r8, %rdx
+; X64-NEXT: pushq %rax
+; X64-NEXT: movl $7, %edx
+; X64-NEXT: xorl %ecx, %ecx
+; X64-NEXT: callq __udivti3 at PLT
+; X64-NEXT: popq %rcx
; X64-NEXT: retq
entry:
%div = udiv i128 %x, 7
@@ -977,168 +946,178 @@ define i128 @div_by_11(i128 %x) nounwind {
; X86-NEXT: pushl %esi
; X86-NEXT: andl $-16, %esp
; X86-NEXT: subl $160, %esp
-; X86-NEXT: movl 32(%ebp), %edi
-; X86-NEXT: movl 36(%ebp), %ebx
-; X86-NEXT: movl 28(%ebp), %edx
-; X86-NEXT: testl %ebx, %ebx
+; X86-NEXT: movl 32(%ebp), %esi
+; X86-NEXT: movl 24(%ebp), %edx
+; X86-NEXT: movl 36(%ebp), %edi
+; X86-NEXT: movl 28(%ebp), %ebx
+; X86-NEXT: movl %ebx, %eax
+; X86-NEXT: orl %edi, %eax
+; X86-NEXT: movl %edx, %ecx
+; X86-NEXT: orl %esi, %ecx
+; X86-NEXT: orl %eax, %ecx
+; X86-NEXT: sete {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X86-NEXT: testl %edi, %edi
; X86-NEXT: jne .LBB4_1
; X86-NEXT: # %bb.2: # %_udiv-special-cases
-; X86-NEXT: bsrl %edi, %esi
-; X86-NEXT: xorl $31, %esi
-; X86-NEXT: orl $32, %esi
-; X86-NEXT: jmp .LBB4_3
-; X86-NEXT: .LBB4_1:
-; X86-NEXT: bsrl %ebx, %esi
-; X86-NEXT: xorl $31, %esi
-; X86-NEXT: .LBB4_3: # %_udiv-special-cases
-; X86-NEXT: movl 24(%ebp), %eax
-; X86-NEXT: testl %edx, %edx
-; X86-NEXT: jne .LBB4_4
-; X86-NEXT: # %bb.5: # %_udiv-special-cases
-; X86-NEXT: bsrl %eax, %ecx
+; X86-NEXT: bsrl %esi, %ecx
; X86-NEXT: xorl $31, %ecx
; X86-NEXT: orl $32, %ecx
-; X86-NEXT: jmp .LBB4_6
+; X86-NEXT: testl %ebx, %ebx
+; X86-NEXT: je .LBB4_5
; X86-NEXT: .LBB4_4:
-; X86-NEXT: bsrl %edx, %ecx
+; X86-NEXT: bsrl %ebx, %edx
+; X86-NEXT: xorl $31, %edx
+; X86-NEXT: jmp .LBB4_6
+; X86-NEXT: .LBB4_1:
+; X86-NEXT: bsrl %edi, %ecx
; X86-NEXT: xorl $31, %ecx
+; X86-NEXT: testl %ebx, %ebx
+; X86-NEXT: jne .LBB4_4
+; X86-NEXT: .LBB4_5: # %_udiv-special-cases
+; X86-NEXT: bsrl %edx, %edx
+; X86-NEXT: xorl $31, %edx
+; X86-NEXT: orl $32, %edx
; X86-NEXT: .LBB4_6: # %_udiv-special-cases
-; X86-NEXT: orl %ebx, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %edi, %eax
-; X86-NEXT: orl %ebx, %edi
+; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: orl %edi, %esi
; X86-NEXT: jne .LBB4_8
; X86-NEXT: # %bb.7: # %_udiv-special-cases
-; X86-NEXT: orl $64, %ecx
-; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: orl $64, %edx
+; X86-NEXT: movl %edx, %ecx
; X86-NEXT: .LBB4_8: # %_udiv-special-cases
-; X86-NEXT: movl $124, %ebx
-; X86-NEXT: subl %esi, %ebx
-; X86-NEXT: movl $0, %edx
-; X86-NEXT: sbbl %edx, %edx
+; X86-NEXT: movl $124, %edx
+; X86-NEXT: subl %ecx, %edx
+; X86-NEXT: movl $0, %ebx
+; X86-NEXT: sbbl %ebx, %ebx
; X86-NEXT: movl $0, %esi
; X86-NEXT: sbbl %esi, %esi
-; X86-NEXT: movl $0, %edi
-; X86-NEXT: sbbl %edi, %edi
-; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: je .LBB4_9
-; X86-NEXT: # %bb.10: # %select.false.sink
-; X86-NEXT: xorl %eax, %eax
+; X86-NEXT: movl $0, %eax
+; X86-NEXT: sbbl %eax, %eax
; X86-NEXT: movl $127, %ecx
-; X86-NEXT: cmpl %ebx, %ecx
+; X86-NEXT: cmpl %edx, %ecx
; X86-NEXT: movl $0, %ecx
-; X86-NEXT: sbbl %edx, %ecx
+; X86-NEXT: sbbl %ebx, %ecx
; X86-NEXT: movl $0, %ecx
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: sbbl %esi, %ecx
-; X86-NEXT: sbbl %edi, %eax
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %eax, %ecx
; X86-NEXT: setb %cl
-; X86-NEXT: .LBB4_11: # %select.end
+; X86-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X86-NEXT: orb %cl, %al
+; X86-NEXT: testb %al, %al
; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: xorl %edx, %edx
-; X86-NEXT: testb %cl, %cl
-; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: movl $0, %esi
; X86-NEXT: movl $0, %ecx
-; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: jne .LBB4_13
-; X86-NEXT: # %bb.12: # %select.end
-; X86-NEXT: movl 28(%ebp), %edx
-; X86-NEXT: movl 24(%ebp), %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 32(%ebp), %ecx
-; X86-NEXT: movl 36(%ebp), %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: .LBB4_13: # %select.end
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: jne .LBB4_14
-; X86-NEXT: # %bb.20: # %select.end
+; X86-NEXT: movl $0, %edi
+; X86-NEXT: jne .LBB4_10
+; X86-NEXT: # %bb.9: # %_udiv-special-cases
; X86-NEXT: movl %ebx, %ecx
-; X86-NEXT: xorl $127, %ecx
-; X86-NEXT: orl %esi, %ecx
+; X86-NEXT: movl 36(%ebp), %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ecx, %ebx
+; X86-NEXT: movl 32(%ebp), %esi
+; X86-NEXT: movl 28(%ebp), %ecx
+; X86-NEXT: movl 24(%ebp), %edi
+; X86-NEXT: .LBB4_10: # %_udiv-special-cases
+; X86-NEXT: jne .LBB4_16
+; X86-NEXT: # %bb.11: # %_udiv-special-cases
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: movl %esi, %edx
-; X86-NEXT: orl %edi, %edx
+; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: movl %edx, %ecx
+; X86-NEXT: xorl $127, %ecx
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ebx, %edx
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
; X86-NEXT: orl %ecx, %edx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: je .LBB4_21
-; X86-NEXT: # %bb.18: # %udiv-bb1
+; X86-NEXT: movl %esi, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: je .LBB4_16
+; X86-NEXT: # %bb.12: # %udiv-bb1
; X86-NEXT: movl 24(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: movl 28(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 32(%ebp), %eax
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 32(%ebp), %edi
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X86-NEXT: movl 36(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
-; X86-NEXT: movl %ebx, %ecx
-; X86-NEXT: xorb $127, %cl
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: movl %eax, %edi
+; X86-NEXT: xorb $127, %cl
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: shrb $3, %al
; X86-NEXT: andb $12, %al
; X86-NEXT: negb %al
; X86-NEXT: movsbl %al, %eax
-; X86-NEXT: movl 136(%esp,%eax), %edx
-; X86-NEXT: movl 140(%esp,%eax), %edi
-; X86-NEXT: shldl %cl, %edx, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 128(%esp,%eax), %edi
-; X86-NEXT: movl 132(%esp,%eax), %eax
-; X86-NEXT: shldl %cl, %eax, %edx
+; X86-NEXT: movl 136(%esp,%eax), %esi
+; X86-NEXT: movl 140(%esp,%eax), %edx
+; X86-NEXT: shldl %cl, %esi, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shldl %cl, %edi, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shll %cl, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: addl $1, %ebx
-; X86-NEXT: adcl $0, %esi
+; X86-NEXT: movl 128(%esp,%eax), %edx
+; X86-NEXT: movl 132(%esp,%eax), %eax
+; X86-NEXT: shldl %cl, %eax, %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: jb .LBB4_19
-; X86-NEXT: # %bb.15: # %udiv-preheader
-; X86-NEXT: movl %ebx, %ecx
-; X86-NEXT: movl 24(%ebp), %eax
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 28(%ebp), %eax
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 32(%ebp), %eax
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 36(%ebp), %eax
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: shldl %cl, %edx, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shll %cl, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: addl $1, %edi
+; X86-NEXT: adcl $0, %ebx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: adcl $0, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: adcl $0, %edx
+; X86-NEXT: jae .LBB4_13
+; X86-NEXT: # %bb.17:
+; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: jmp .LBB4_15
+; X86-NEXT: .LBB4_13: # %udiv-preheader
+; X86-NEXT: movl %edi, %ecx
+; X86-NEXT: movl 24(%ebp), %esi
+; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 28(%ebp), %esi
+; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 32(%ebp), %esi
+; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 36(%ebp), %esi
+; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: shrb $3, %al
; X86-NEXT: andb $12, %al
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movzbl %al, %edi
; X86-NEXT: movl 92(%esp,%edi), %eax
-; X86-NEXT: movl 88(%esp,%edi), %ebx
-; X86-NEXT: movl %ebx, %esi
-; X86-NEXT: shrdl %cl, %eax, %esi
+; X86-NEXT: movl 88(%esp,%edi), %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 80(%esp,%edi), %esi
-; X86-NEXT: movl 84(%esp,%edi), %edi
-; X86-NEXT: movl %edi, %edx
-; X86-NEXT: shrdl %cl, %ebx, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %esi, %edx
+; X86-NEXT: shrdl %cl, %eax, %edx
+; X86-NEXT: movl 80(%esp,%edi), %ebx
+; X86-NEXT: movl 84(%esp,%edi), %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: shrdl %cl, %esi, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: shrl %cl, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NEXT: shrdl %cl, %edi, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: shrdl %cl, %eax, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $11, %eax
; X86-NEXT: addl $-1, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
@@ -1152,79 +1131,78 @@ define i128 @div_by_11(i128 %x) nounwind {
; X86-NEXT: adcl $-1, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: .p2align 4
-; X86-NEXT: .LBB4_16: # %udiv-do-while
+; X86-NEXT: .LBB4_14: # %udiv-do-while
; X86-NEXT: # =>This Inner Loop Header: Depth=1
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %edx, %edi
+; X86-NEXT: shldl $1, %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: shldl $1, %eax, %edi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: shldl $1, %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: shldl $1, %edi, %ebx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: shldl $1, %edx, %edi
-; X86-NEXT: shldl $1, %eax, %edx
-; X86-NEXT: shldl $1, %ecx, %eax
+; X86-NEXT: shldl $1, %ebx, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: shldl $1, %ecx, %ebx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: orl %esi, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shldl $1, %eax, %ecx
-; X86-NEXT: orl %esi, %ecx
+; X86-NEXT: shldl $1, %esi, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: orl %edx, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: shldl $1, %ecx, %eax
-; X86-NEXT: orl %esi, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: addl %ecx, %ecx
-; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: shldl $1, %ecx, %esi
+; X86-NEXT: orl %edx, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: shldl $1, %esi, %ecx
+; X86-NEXT: orl %edx, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: cmpl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; X86-NEXT: addl %esi, %esi
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: cmpl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: sbbl %edi, %ecx
+; X86-NEXT: sbbl %eax, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: sbbl %ebx, %ecx
+; X86-NEXT: sbbl %edi, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: sbbl %edx, %ecx
; X86-NEXT: sarl $31, %ecx
-; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: andl $1, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: andl $1, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl $11, %eax
-; X86-NEXT: andl %eax, %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: subl %ecx, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $11, %esi
+; X86-NEXT: andl %esi, %ecx
+; X86-NEXT: subl %ecx, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl $0, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: sbbl $0, %edi
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: sbbl $0, %ebx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: sbbl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: movl %edx, %eax
+; X86-NEXT: sbbl $0, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NEXT: addl $-1, %ebx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: adcl $-1, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-NEXT: adcl $-1, %edi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NEXT: adcl $-1, %edx
-; X86-NEXT: adcl $-1, %esi
-; X86-NEXT: movl %edi, %ecx
-; X86-NEXT: orl %esi, %ecx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %edx, %ebx
+; X86-NEXT: orl %edx, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %edi, %ebx
; X86-NEXT: orl %ecx, %ebx
+; X86-NEXT: jne .LBB4_14
+; X86-NEXT: .LBB4_15: # %udiv-loop-exit
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: jne .LBB4_16
-; X86-NEXT: .LBB4_17: # %udiv-loop-exit
-; X86-NEXT: shldl $1, %ecx, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shldl $1, %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NEXT: shldl $1, %edx, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
@@ -1232,10 +1210,12 @@ define i128 @div_by_11(i128 %x) nounwind {
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NEXT: leal (%esi,%eax,2), %edi
; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: .LBB4_21: # %udiv-end
+; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: movl %edx, %ecx
+; X86-NEXT: .LBB4_16: # %udiv-end
; X86-NEXT: movl %edi, (%eax)
-; X86-NEXT: movl %edx, 4(%eax)
-; X86-NEXT: movl %ecx, 8(%eax)
+; X86-NEXT: movl %ecx, 4(%eax)
+; X86-NEXT: movl %esi, 8(%eax)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: movl %ecx, 12(%eax)
; X86-NEXT: leal -12(%ebp), %esp
@@ -1244,48 +1224,14 @@ define i128 @div_by_11(i128 %x) nounwind {
; X86-NEXT: popl %ebx
; X86-NEXT: popl %ebp
; X86-NEXT: retl $4
-; X86-NEXT: .LBB4_9:
-; X86-NEXT: movb $1, %cl
-; X86-NEXT: jmp .LBB4_11
-; X86-NEXT: .LBB4_19:
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: jmp .LBB4_17
-; X86-NEXT: .LBB4_14:
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: jmp .LBB4_21
;
; X64-LABEL: div_by_11:
; X64: # %bb.0:
-; X64-NEXT: movabsq $1152921504606846975, %rax # imm = 0xFFFFFFFFFFFFFFF
-; X64-NEXT: movq %rdi, %rcx
-; X64-NEXT: andq %rax, %rcx
-; X64-NEXT: movq %rdi, %rdx
-; X64-NEXT: shrdq $60, %rsi, %rdx
-; X64-NEXT: andq %rax, %rdx
-; X64-NEXT: addq %rcx, %rdx
-; X64-NEXT: movq %rsi, %rcx
-; X64-NEXT: shrq $56, %rcx
-; X64-NEXT: addq %rdx, %rcx
-; X64-NEXT: movabsq $3353953467947191203, %r8 # imm = 0x2E8BA2E8BA2E8BA3
-; X64-NEXT: movq %rcx, %rax
-; X64-NEXT: mulq %r8
-; X64-NEXT: shrq %rdx
-; X64-NEXT: leaq (%rdx,%rdx,4), %rax
-; X64-NEXT: leaq (%rdx,%rax,2), %rax
-; X64-NEXT: subq %rax, %rcx
-; X64-NEXT: subq %rcx, %rdi
-; X64-NEXT: sbbq $0, %rsi
-; X64-NEXT: movabsq $-6707906935894382406, %rcx # imm = 0xA2E8BA2E8BA2E8BA
-; X64-NEXT: imulq %rdi, %rcx
-; X64-NEXT: movq %rdi, %rax
-; X64-NEXT: mulq %r8
-; X64-NEXT: addq %rcx, %rdx
-; X64-NEXT: imulq %rsi, %r8
-; X64-NEXT: addq %r8, %rdx
+; X64-NEXT: pushq %rax
+; X64-NEXT: movl $11, %edx
+; X64-NEXT: xorl %ecx, %ecx
+; X64-NEXT: callq __udivti3 at PLT
+; X64-NEXT: popq %rcx
; X64-NEXT: retq
%div = udiv i128 %x, 11
ret i128 %div
@@ -1301,168 +1247,178 @@ define i128 @div_by_22(i128 %x) nounwind {
; X86-NEXT: pushl %esi
; X86-NEXT: andl $-16, %esp
; X86-NEXT: subl $160, %esp
-; X86-NEXT: movl 32(%ebp), %edi
-; X86-NEXT: movl 36(%ebp), %ebx
-; X86-NEXT: movl 28(%ebp), %edx
-; X86-NEXT: testl %ebx, %ebx
+; X86-NEXT: movl 32(%ebp), %esi
+; X86-NEXT: movl 24(%ebp), %edx
+; X86-NEXT: movl 36(%ebp), %edi
+; X86-NEXT: movl 28(%ebp), %ebx
+; X86-NEXT: movl %ebx, %eax
+; X86-NEXT: orl %edi, %eax
+; X86-NEXT: movl %edx, %ecx
+; X86-NEXT: orl %esi, %ecx
+; X86-NEXT: orl %eax, %ecx
+; X86-NEXT: sete {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X86-NEXT: testl %edi, %edi
; X86-NEXT: jne .LBB5_1
; X86-NEXT: # %bb.2: # %entry_udiv-special-cases
-; X86-NEXT: bsrl %edi, %esi
-; X86-NEXT: xorl $31, %esi
-; X86-NEXT: orl $32, %esi
-; X86-NEXT: jmp .LBB5_3
-; X86-NEXT: .LBB5_1:
-; X86-NEXT: bsrl %ebx, %esi
-; X86-NEXT: xorl $31, %esi
-; X86-NEXT: .LBB5_3: # %entry_udiv-special-cases
-; X86-NEXT: movl 24(%ebp), %eax
-; X86-NEXT: testl %edx, %edx
-; X86-NEXT: jne .LBB5_4
-; X86-NEXT: # %bb.5: # %entry_udiv-special-cases
-; X86-NEXT: bsrl %eax, %ecx
+; X86-NEXT: bsrl %esi, %ecx
; X86-NEXT: xorl $31, %ecx
; X86-NEXT: orl $32, %ecx
-; X86-NEXT: jmp .LBB5_6
+; X86-NEXT: testl %ebx, %ebx
+; X86-NEXT: je .LBB5_5
; X86-NEXT: .LBB5_4:
-; X86-NEXT: bsrl %edx, %ecx
+; X86-NEXT: bsrl %ebx, %edx
+; X86-NEXT: xorl $31, %edx
+; X86-NEXT: jmp .LBB5_6
+; X86-NEXT: .LBB5_1:
+; X86-NEXT: bsrl %edi, %ecx
; X86-NEXT: xorl $31, %ecx
+; X86-NEXT: testl %ebx, %ebx
+; X86-NEXT: jne .LBB5_4
+; X86-NEXT: .LBB5_5: # %entry_udiv-special-cases
+; X86-NEXT: bsrl %edx, %edx
+; X86-NEXT: xorl $31, %edx
+; X86-NEXT: orl $32, %edx
; X86-NEXT: .LBB5_6: # %entry_udiv-special-cases
-; X86-NEXT: orl %ebx, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %edi, %eax
-; X86-NEXT: orl %ebx, %edi
+; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: orl %edi, %esi
; X86-NEXT: jne .LBB5_8
; X86-NEXT: # %bb.7: # %entry_udiv-special-cases
-; X86-NEXT: orl $64, %ecx
-; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: orl $64, %edx
+; X86-NEXT: movl %edx, %ecx
; X86-NEXT: .LBB5_8: # %entry_udiv-special-cases
-; X86-NEXT: movl $123, %ebx
-; X86-NEXT: subl %esi, %ebx
-; X86-NEXT: movl $0, %edx
-; X86-NEXT: sbbl %edx, %edx
+; X86-NEXT: movl $123, %edx
+; X86-NEXT: subl %ecx, %edx
+; X86-NEXT: movl $0, %ebx
+; X86-NEXT: sbbl %ebx, %ebx
; X86-NEXT: movl $0, %esi
; X86-NEXT: sbbl %esi, %esi
-; X86-NEXT: movl $0, %edi
-; X86-NEXT: sbbl %edi, %edi
-; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: je .LBB5_9
-; X86-NEXT: # %bb.10: # %select.false.sink
-; X86-NEXT: xorl %eax, %eax
+; X86-NEXT: movl $0, %eax
+; X86-NEXT: sbbl %eax, %eax
; X86-NEXT: movl $127, %ecx
-; X86-NEXT: cmpl %ebx, %ecx
+; X86-NEXT: cmpl %edx, %ecx
; X86-NEXT: movl $0, %ecx
-; X86-NEXT: sbbl %edx, %ecx
+; X86-NEXT: sbbl %ebx, %ecx
; X86-NEXT: movl $0, %ecx
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: sbbl %esi, %ecx
-; X86-NEXT: sbbl %edi, %eax
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %eax, %ecx
; X86-NEXT: setb %cl
-; X86-NEXT: .LBB5_11: # %select.end
+; X86-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X86-NEXT: orb %cl, %al
+; X86-NEXT: testb %al, %al
; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: xorl %edx, %edx
-; X86-NEXT: testb %cl, %cl
-; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: movl $0, %esi
; X86-NEXT: movl $0, %ecx
-; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: jne .LBB5_13
-; X86-NEXT: # %bb.12: # %select.end
-; X86-NEXT: movl 28(%ebp), %edx
-; X86-NEXT: movl 24(%ebp), %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 32(%ebp), %ecx
-; X86-NEXT: movl 36(%ebp), %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: .LBB5_13: # %select.end
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: jne .LBB5_14
-; X86-NEXT: # %bb.20: # %select.end
+; X86-NEXT: movl $0, %edi
+; X86-NEXT: jne .LBB5_10
+; X86-NEXT: # %bb.9: # %entry_udiv-special-cases
; X86-NEXT: movl %ebx, %ecx
-; X86-NEXT: xorl $127, %ecx
-; X86-NEXT: orl %esi, %ecx
+; X86-NEXT: movl 36(%ebp), %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ecx, %ebx
+; X86-NEXT: movl 32(%ebp), %esi
+; X86-NEXT: movl 28(%ebp), %ecx
+; X86-NEXT: movl 24(%ebp), %edi
+; X86-NEXT: .LBB5_10: # %entry_udiv-special-cases
+; X86-NEXT: jne .LBB5_16
+; X86-NEXT: # %bb.11: # %entry_udiv-special-cases
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: movl %esi, %edx
-; X86-NEXT: orl %edi, %edx
+; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: movl %edx, %ecx
+; X86-NEXT: xorl $127, %ecx
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ebx, %edx
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
; X86-NEXT: orl %ecx, %edx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: je .LBB5_21
-; X86-NEXT: # %bb.18: # %udiv-bb1
+; X86-NEXT: movl %esi, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: je .LBB5_16
+; X86-NEXT: # %bb.12: # %udiv-bb1
; X86-NEXT: movl 24(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: movl 28(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 32(%ebp), %eax
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 32(%ebp), %edi
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X86-NEXT: movl 36(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
-; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: movl %eax, %edi
; X86-NEXT: xorb $127, %cl
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: shrb $3, %al
; X86-NEXT: andb $12, %al
; X86-NEXT: negb %al
; X86-NEXT: movsbl %al, %eax
-; X86-NEXT: movl 136(%esp,%eax), %edx
-; X86-NEXT: movl 140(%esp,%eax), %edi
-; X86-NEXT: shldl %cl, %edx, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 128(%esp,%eax), %edi
-; X86-NEXT: movl 132(%esp,%eax), %eax
-; X86-NEXT: shldl %cl, %eax, %edx
+; X86-NEXT: movl 136(%esp,%eax), %esi
+; X86-NEXT: movl 140(%esp,%eax), %edx
+; X86-NEXT: shldl %cl, %esi, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shldl %cl, %edi, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shll %cl, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: addl $1, %ebx
-; X86-NEXT: adcl $0, %esi
+; X86-NEXT: movl 128(%esp,%eax), %edx
+; X86-NEXT: movl 132(%esp,%eax), %eax
+; X86-NEXT: shldl %cl, %eax, %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: jb .LBB5_19
-; X86-NEXT: # %bb.15: # %udiv-preheader
-; X86-NEXT: movl %ebx, %ecx
-; X86-NEXT: movl 24(%ebp), %eax
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 28(%ebp), %eax
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 32(%ebp), %eax
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 36(%ebp), %eax
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: shldl %cl, %edx, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shll %cl, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: addl $1, %edi
+; X86-NEXT: adcl $0, %ebx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: adcl $0, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: adcl $0, %edx
+; X86-NEXT: jae .LBB5_13
+; X86-NEXT: # %bb.17:
+; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: jmp .LBB5_15
+; X86-NEXT: .LBB5_13: # %udiv-preheader
+; X86-NEXT: movl %edi, %ecx
+; X86-NEXT: movl 24(%ebp), %esi
+; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 28(%ebp), %esi
+; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 32(%ebp), %esi
+; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 36(%ebp), %esi
+; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: shrb $3, %al
; X86-NEXT: andb $12, %al
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movzbl %al, %edi
; X86-NEXT: movl 92(%esp,%edi), %eax
-; X86-NEXT: movl 88(%esp,%edi), %ebx
-; X86-NEXT: movl %ebx, %esi
-; X86-NEXT: shrdl %cl, %eax, %esi
+; X86-NEXT: movl 88(%esp,%edi), %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 80(%esp,%edi), %esi
-; X86-NEXT: movl 84(%esp,%edi), %edi
-; X86-NEXT: movl %edi, %edx
-; X86-NEXT: shrdl %cl, %ebx, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %esi, %edx
+; X86-NEXT: shrdl %cl, %eax, %edx
+; X86-NEXT: movl 80(%esp,%edi), %ebx
+; X86-NEXT: movl 84(%esp,%edi), %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: shrdl %cl, %esi, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: shrl %cl, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NEXT: shrdl %cl, %edi, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: shrdl %cl, %eax, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $22, %eax
; X86-NEXT: addl $-1, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
@@ -1476,79 +1432,78 @@ define i128 @div_by_22(i128 %x) nounwind {
; X86-NEXT: adcl $-1, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: .p2align 4
-; X86-NEXT: .LBB5_16: # %udiv-do-while
+; X86-NEXT: .LBB5_14: # %udiv-do-while
; X86-NEXT: # =>This Inner Loop Header: Depth=1
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %edx, %edi
+; X86-NEXT: shldl $1, %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: shldl $1, %eax, %edi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: shldl $1, %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: shldl $1, %edi, %ebx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: shldl $1, %edx, %edi
-; X86-NEXT: shldl $1, %eax, %edx
-; X86-NEXT: shldl $1, %ecx, %eax
+; X86-NEXT: shldl $1, %ebx, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: shldl $1, %ecx, %ebx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: orl %esi, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shldl $1, %eax, %ecx
-; X86-NEXT: orl %esi, %ecx
+; X86-NEXT: shldl $1, %esi, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: orl %edx, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: shldl $1, %ecx, %eax
-; X86-NEXT: orl %esi, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: addl %ecx, %ecx
-; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: shldl $1, %ecx, %esi
+; X86-NEXT: orl %edx, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: shldl $1, %esi, %ecx
+; X86-NEXT: orl %edx, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: cmpl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; X86-NEXT: addl %esi, %esi
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: cmpl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: sbbl %edi, %ecx
+; X86-NEXT: sbbl %eax, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: sbbl %ebx, %ecx
+; X86-NEXT: sbbl %edi, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: sbbl %edx, %ecx
; X86-NEXT: sarl $31, %ecx
-; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: andl $1, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: andl $1, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl $22, %eax
-; X86-NEXT: andl %eax, %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: subl %ecx, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $22, %esi
+; X86-NEXT: andl %esi, %ecx
+; X86-NEXT: subl %ecx, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl $0, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: sbbl $0, %edi
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: sbbl $0, %ebx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: sbbl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: movl %edx, %eax
+; X86-NEXT: sbbl $0, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NEXT: addl $-1, %ebx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: adcl $-1, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-NEXT: adcl $-1, %edi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NEXT: adcl $-1, %edx
-; X86-NEXT: adcl $-1, %esi
-; X86-NEXT: movl %edi, %ecx
-; X86-NEXT: orl %esi, %ecx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %edx, %ebx
+; X86-NEXT: orl %edx, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %edi, %ebx
; X86-NEXT: orl %ecx, %ebx
+; X86-NEXT: jne .LBB5_14
+; X86-NEXT: .LBB5_15: # %udiv-loop-exit
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: jne .LBB5_16
-; X86-NEXT: .LBB5_17: # %udiv-loop-exit
-; X86-NEXT: shldl $1, %ecx, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shldl $1, %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NEXT: shldl $1, %edx, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
@@ -1556,10 +1511,12 @@ define i128 @div_by_22(i128 %x) nounwind {
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NEXT: leal (%esi,%eax,2), %edi
; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: .LBB5_21: # %udiv-end
+; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: movl %edx, %ecx
+; X86-NEXT: .LBB5_16: # %udiv-end
; X86-NEXT: movl %edi, (%eax)
-; X86-NEXT: movl %edx, 4(%eax)
-; X86-NEXT: movl %ecx, 8(%eax)
+; X86-NEXT: movl %ecx, 4(%eax)
+; X86-NEXT: movl %esi, 8(%eax)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: movl %ecx, 12(%eax)
; X86-NEXT: leal -12(%ebp), %esp
@@ -1568,49 +1525,14 @@ define i128 @div_by_22(i128 %x) nounwind {
; X86-NEXT: popl %ebx
; X86-NEXT: popl %ebp
; X86-NEXT: retl $4
-; X86-NEXT: .LBB5_9:
-; X86-NEXT: movb $1, %cl
-; X86-NEXT: jmp .LBB5_11
-; X86-NEXT: .LBB5_19:
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: jmp .LBB5_17
-; X86-NEXT: .LBB5_14:
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: jmp .LBB5_21
;
; X64-LABEL: div_by_22:
; X64: # %bb.0: # %entry
-; X64-NEXT: movq %rdi, %rax
-; X64-NEXT: shrdq $61, %rsi, %rax
-; X64-NEXT: movabsq $1152921504606846975, %rdx # imm = 0xFFFFFFFFFFFFFFF
-; X64-NEXT: andq %rdx, %rax
-; X64-NEXT: shrdq $1, %rsi, %rdi
-; X64-NEXT: andq %rdi, %rdx
-; X64-NEXT: addq %rax, %rdx
-; X64-NEXT: movq %rsi, %rcx
-; X64-NEXT: shrq $57, %rcx
-; X64-NEXT: addq %rdx, %rcx
-; X64-NEXT: movabsq $3353953467947191203, %r8 # imm = 0x2E8BA2E8BA2E8BA3
-; X64-NEXT: movq %rcx, %rax
-; X64-NEXT: mulq %r8
-; X64-NEXT: shrq %rdx
-; X64-NEXT: leaq (%rdx,%rdx,4), %rax
-; X64-NEXT: leaq (%rdx,%rax,2), %rax
-; X64-NEXT: subq %rax, %rcx
-; X64-NEXT: shrq %rsi
-; X64-NEXT: subq %rcx, %rdi
-; X64-NEXT: sbbq $0, %rsi
-; X64-NEXT: movabsq $-6707906935894382406, %rcx # imm = 0xA2E8BA2E8BA2E8BA
-; X64-NEXT: imulq %rdi, %rcx
-; X64-NEXT: movq %rdi, %rax
-; X64-NEXT: mulq %r8
-; X64-NEXT: addq %rcx, %rdx
-; X64-NEXT: imulq %rsi, %r8
-; X64-NEXT: addq %r8, %rdx
+; X64-NEXT: pushq %rax
+; X64-NEXT: movl $22, %edx
+; X64-NEXT: xorl %ecx, %ecx
+; X64-NEXT: callq __udivti3 at PLT
+; X64-NEXT: popq %rcx
; X64-NEXT: retq
entry:
%div = udiv i128 %x, 22
@@ -1627,168 +1549,178 @@ define i128 @div_by_56(i128 %x) nounwind {
; X86-NEXT: pushl %esi
; X86-NEXT: andl $-16, %esp
; X86-NEXT: subl $160, %esp
-; X86-NEXT: movl 32(%ebp), %edi
-; X86-NEXT: movl 36(%ebp), %ebx
-; X86-NEXT: movl 28(%ebp), %edx
-; X86-NEXT: testl %ebx, %ebx
+; X86-NEXT: movl 32(%ebp), %esi
+; X86-NEXT: movl 24(%ebp), %edx
+; X86-NEXT: movl 36(%ebp), %edi
+; X86-NEXT: movl 28(%ebp), %ebx
+; X86-NEXT: movl %ebx, %eax
+; X86-NEXT: orl %edi, %eax
+; X86-NEXT: movl %edx, %ecx
+; X86-NEXT: orl %esi, %ecx
+; X86-NEXT: orl %eax, %ecx
+; X86-NEXT: sete {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X86-NEXT: testl %edi, %edi
; X86-NEXT: jne .LBB6_1
; X86-NEXT: # %bb.2: # %_udiv-special-cases
-; X86-NEXT: bsrl %edi, %esi
-; X86-NEXT: xorl $31, %esi
-; X86-NEXT: orl $32, %esi
-; X86-NEXT: jmp .LBB6_3
-; X86-NEXT: .LBB6_1:
-; X86-NEXT: bsrl %ebx, %esi
-; X86-NEXT: xorl $31, %esi
-; X86-NEXT: .LBB6_3: # %_udiv-special-cases
-; X86-NEXT: movl 24(%ebp), %eax
-; X86-NEXT: testl %edx, %edx
-; X86-NEXT: jne .LBB6_4
-; X86-NEXT: # %bb.5: # %_udiv-special-cases
-; X86-NEXT: bsrl %eax, %ecx
+; X86-NEXT: bsrl %esi, %ecx
; X86-NEXT: xorl $31, %ecx
; X86-NEXT: orl $32, %ecx
-; X86-NEXT: jmp .LBB6_6
+; X86-NEXT: testl %ebx, %ebx
+; X86-NEXT: je .LBB6_5
; X86-NEXT: .LBB6_4:
-; X86-NEXT: bsrl %edx, %ecx
+; X86-NEXT: bsrl %ebx, %edx
+; X86-NEXT: xorl $31, %edx
+; X86-NEXT: jmp .LBB6_6
+; X86-NEXT: .LBB6_1:
+; X86-NEXT: bsrl %edi, %ecx
; X86-NEXT: xorl $31, %ecx
+; X86-NEXT: testl %ebx, %ebx
+; X86-NEXT: jne .LBB6_4
+; X86-NEXT: .LBB6_5: # %_udiv-special-cases
+; X86-NEXT: bsrl %edx, %edx
+; X86-NEXT: xorl $31, %edx
+; X86-NEXT: orl $32, %edx
; X86-NEXT: .LBB6_6: # %_udiv-special-cases
-; X86-NEXT: orl %ebx, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %edi, %eax
-; X86-NEXT: orl %ebx, %edi
+; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: orl %edi, %esi
; X86-NEXT: jne .LBB6_8
; X86-NEXT: # %bb.7: # %_udiv-special-cases
-; X86-NEXT: orl $64, %ecx
-; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: orl $64, %edx
+; X86-NEXT: movl %edx, %ecx
; X86-NEXT: .LBB6_8: # %_udiv-special-cases
-; X86-NEXT: movl $122, %ebx
-; X86-NEXT: subl %esi, %ebx
-; X86-NEXT: movl $0, %edx
-; X86-NEXT: sbbl %edx, %edx
+; X86-NEXT: movl $122, %edx
+; X86-NEXT: subl %ecx, %edx
+; X86-NEXT: movl $0, %ebx
+; X86-NEXT: sbbl %ebx, %ebx
; X86-NEXT: movl $0, %esi
; X86-NEXT: sbbl %esi, %esi
-; X86-NEXT: movl $0, %edi
-; X86-NEXT: sbbl %edi, %edi
-; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: je .LBB6_9
-; X86-NEXT: # %bb.10: # %select.false.sink
-; X86-NEXT: xorl %eax, %eax
+; X86-NEXT: movl $0, %eax
+; X86-NEXT: sbbl %eax, %eax
; X86-NEXT: movl $127, %ecx
-; X86-NEXT: cmpl %ebx, %ecx
+; X86-NEXT: cmpl %edx, %ecx
; X86-NEXT: movl $0, %ecx
-; X86-NEXT: sbbl %edx, %ecx
+; X86-NEXT: sbbl %ebx, %ecx
; X86-NEXT: movl $0, %ecx
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: sbbl %esi, %ecx
-; X86-NEXT: sbbl %edi, %eax
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %eax, %ecx
; X86-NEXT: setb %cl
-; X86-NEXT: .LBB6_11: # %select.end
+; X86-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X86-NEXT: orb %cl, %al
+; X86-NEXT: testb %al, %al
; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: xorl %edx, %edx
-; X86-NEXT: testb %cl, %cl
-; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: movl $0, %esi
; X86-NEXT: movl $0, %ecx
-; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: jne .LBB6_13
-; X86-NEXT: # %bb.12: # %select.end
-; X86-NEXT: movl 28(%ebp), %edx
-; X86-NEXT: movl 24(%ebp), %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 32(%ebp), %ecx
-; X86-NEXT: movl 36(%ebp), %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: .LBB6_13: # %select.end
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: jne .LBB6_14
-; X86-NEXT: # %bb.20: # %select.end
+; X86-NEXT: movl $0, %edi
+; X86-NEXT: jne .LBB6_10
+; X86-NEXT: # %bb.9: # %_udiv-special-cases
; X86-NEXT: movl %ebx, %ecx
-; X86-NEXT: xorl $127, %ecx
-; X86-NEXT: orl %esi, %ecx
+; X86-NEXT: movl 36(%ebp), %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ecx, %ebx
+; X86-NEXT: movl 32(%ebp), %esi
+; X86-NEXT: movl 28(%ebp), %ecx
+; X86-NEXT: movl 24(%ebp), %edi
+; X86-NEXT: .LBB6_10: # %_udiv-special-cases
+; X86-NEXT: jne .LBB6_16
+; X86-NEXT: # %bb.11: # %_udiv-special-cases
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: movl %esi, %edx
-; X86-NEXT: orl %edi, %edx
+; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: movl %edx, %ecx
+; X86-NEXT: xorl $127, %ecx
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ebx, %edx
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
; X86-NEXT: orl %ecx, %edx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: je .LBB6_21
-; X86-NEXT: # %bb.18: # %udiv-bb1
+; X86-NEXT: movl %esi, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: je .LBB6_16
+; X86-NEXT: # %bb.12: # %udiv-bb1
; X86-NEXT: movl 24(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: movl 28(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 32(%ebp), %eax
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 32(%ebp), %edi
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X86-NEXT: movl 36(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
-; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: movl %eax, %edi
; X86-NEXT: xorb $127, %cl
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: shrb $3, %al
; X86-NEXT: andb $12, %al
; X86-NEXT: negb %al
; X86-NEXT: movsbl %al, %eax
-; X86-NEXT: movl 136(%esp,%eax), %edx
-; X86-NEXT: movl 140(%esp,%eax), %edi
-; X86-NEXT: shldl %cl, %edx, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 128(%esp,%eax), %edi
-; X86-NEXT: movl 132(%esp,%eax), %eax
-; X86-NEXT: shldl %cl, %eax, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shldl %cl, %edi, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shll %cl, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: addl $1, %ebx
-; X86-NEXT: adcl $0, %esi
+; X86-NEXT: movl 136(%esp,%eax), %esi
+; X86-NEXT: movl 140(%esp,%eax), %edx
+; X86-NEXT: shldl %cl, %esi, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 128(%esp,%eax), %edx
+; X86-NEXT: movl 132(%esp,%eax), %eax
+; X86-NEXT: shldl %cl, %eax, %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: jb .LBB6_19
-; X86-NEXT: # %bb.15: # %udiv-preheader
-; X86-NEXT: movl %ebx, %ecx
-; X86-NEXT: movl 24(%ebp), %eax
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 28(%ebp), %eax
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 32(%ebp), %eax
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 36(%ebp), %eax
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: shldl %cl, %edx, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shll %cl, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: addl $1, %edi
+; X86-NEXT: adcl $0, %ebx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: adcl $0, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: adcl $0, %edx
+; X86-NEXT: jae .LBB6_13
+; X86-NEXT: # %bb.17:
+; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: jmp .LBB6_15
+; X86-NEXT: .LBB6_13: # %udiv-preheader
+; X86-NEXT: movl %edi, %ecx
+; X86-NEXT: movl 24(%ebp), %esi
+; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 28(%ebp), %esi
+; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 32(%ebp), %esi
+; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 36(%ebp), %esi
+; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: shrb $3, %al
; X86-NEXT: andb $12, %al
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movzbl %al, %edi
; X86-NEXT: movl 92(%esp,%edi), %eax
-; X86-NEXT: movl 88(%esp,%edi), %ebx
-; X86-NEXT: movl %ebx, %esi
-; X86-NEXT: shrdl %cl, %eax, %esi
+; X86-NEXT: movl 88(%esp,%edi), %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 80(%esp,%edi), %esi
-; X86-NEXT: movl 84(%esp,%edi), %edi
-; X86-NEXT: movl %edi, %edx
-; X86-NEXT: shrdl %cl, %ebx, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %esi, %edx
+; X86-NEXT: shrdl %cl, %eax, %edx
+; X86-NEXT: movl 80(%esp,%edi), %ebx
+; X86-NEXT: movl 84(%esp,%edi), %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: shrdl %cl, %esi, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: shrl %cl, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NEXT: shrdl %cl, %edi, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: shrdl %cl, %eax, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $56, %eax
; X86-NEXT: addl $-1, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
@@ -1802,79 +1734,78 @@ define i128 @div_by_56(i128 %x) nounwind {
; X86-NEXT: adcl $-1, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: .p2align 4
-; X86-NEXT: .LBB6_16: # %udiv-do-while
+; X86-NEXT: .LBB6_14: # %udiv-do-while
; X86-NEXT: # =>This Inner Loop Header: Depth=1
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %edx, %edi
+; X86-NEXT: shldl $1, %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: shldl $1, %eax, %edi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: shldl $1, %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: shldl $1, %edi, %ebx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: shldl $1, %edx, %edi
-; X86-NEXT: shldl $1, %eax, %edx
-; X86-NEXT: shldl $1, %ecx, %eax
+; X86-NEXT: shldl $1, %ebx, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: shldl $1, %ecx, %ebx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: orl %esi, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shldl $1, %eax, %ecx
-; X86-NEXT: orl %esi, %ecx
+; X86-NEXT: shldl $1, %esi, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: orl %edx, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: shldl $1, %ecx, %eax
-; X86-NEXT: orl %esi, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: addl %ecx, %ecx
-; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: shldl $1, %ecx, %esi
+; X86-NEXT: orl %edx, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: shldl $1, %esi, %ecx
+; X86-NEXT: orl %edx, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: cmpl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; X86-NEXT: addl %esi, %esi
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: cmpl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: sbbl %edi, %ecx
+; X86-NEXT: sbbl %eax, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: sbbl %ebx, %ecx
+; X86-NEXT: sbbl %edi, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: sbbl %edx, %ecx
; X86-NEXT: sarl $31, %ecx
-; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: andl $1, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: andl $1, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl $56, %eax
-; X86-NEXT: andl %eax, %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: subl %ecx, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $56, %esi
+; X86-NEXT: andl %esi, %ecx
+; X86-NEXT: subl %ecx, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl $0, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: sbbl $0, %edi
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: sbbl $0, %ebx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: sbbl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: movl %edx, %eax
+; X86-NEXT: sbbl $0, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NEXT: addl $-1, %ebx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: adcl $-1, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-NEXT: adcl $-1, %edi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NEXT: adcl $-1, %edx
-; X86-NEXT: adcl $-1, %esi
-; X86-NEXT: movl %edi, %ecx
-; X86-NEXT: orl %esi, %ecx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %edx, %ebx
+; X86-NEXT: orl %edx, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %edi, %ebx
; X86-NEXT: orl %ecx, %ebx
+; X86-NEXT: jne .LBB6_14
+; X86-NEXT: .LBB6_15: # %udiv-loop-exit
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: jne .LBB6_16
-; X86-NEXT: .LBB6_17: # %udiv-loop-exit
-; X86-NEXT: shldl $1, %ecx, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shldl $1, %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NEXT: shldl $1, %edx, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
@@ -1882,10 +1813,12 @@ define i128 @div_by_56(i128 %x) nounwind {
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NEXT: leal (%esi,%eax,2), %edi
; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: .LBB6_21: # %udiv-end
+; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: movl %edx, %ecx
+; X86-NEXT: .LBB6_16: # %udiv-end
; X86-NEXT: movl %edi, (%eax)
-; X86-NEXT: movl %edx, 4(%eax)
-; X86-NEXT: movl %ecx, 8(%eax)
+; X86-NEXT: movl %ecx, 4(%eax)
+; X86-NEXT: movl %esi, 8(%eax)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: movl %ecx, 12(%eax)
; X86-NEXT: leal -12(%ebp), %esp
@@ -1894,50 +1827,14 @@ define i128 @div_by_56(i128 %x) nounwind {
; X86-NEXT: popl %ebx
; X86-NEXT: popl %ebp
; X86-NEXT: retl $4
-; X86-NEXT: .LBB6_9:
-; X86-NEXT: movb $1, %cl
-; X86-NEXT: jmp .LBB6_11
-; X86-NEXT: .LBB6_19:
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: jmp .LBB6_17
-; X86-NEXT: .LBB6_14:
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: jmp .LBB6_21
;
; X64-LABEL: div_by_56:
; X64: # %bb.0:
-; X64-NEXT: movq %rdi, %rax
-; X64-NEXT: shrdq $63, %rsi, %rax
-; X64-NEXT: movabsq $1152921504606846975, %rdx # imm = 0xFFFFFFFFFFFFFFF
-; X64-NEXT: andq %rdx, %rax
-; X64-NEXT: shrdq $3, %rsi, %rdi
-; X64-NEXT: andq %rdi, %rdx
-; X64-NEXT: addq %rax, %rdx
-; X64-NEXT: movq %rsi, %rcx
-; X64-NEXT: shrq $59, %rcx
-; X64-NEXT: addq %rdx, %rcx
-; X64-NEXT: movabsq $5270498306774157605, %rdx # imm = 0x4924924924924925
-; X64-NEXT: movq %rcx, %rax
-; X64-NEXT: mulq %rdx
-; X64-NEXT: shrq %rdx
-; X64-NEXT: leaq (,%rdx,8), %rax
-; X64-NEXT: subq %rax, %rdx
-; X64-NEXT: addq %rcx, %rdx
-; X64-NEXT: shrq $3, %rsi
-; X64-NEXT: subq %rdx, %rdi
-; X64-NEXT: sbbq $0, %rsi
-; X64-NEXT: movabsq $-5270498306774157605, %rcx # imm = 0xB6DB6DB6DB6DB6DB
-; X64-NEXT: imulq %rdi, %rcx
-; X64-NEXT: movabsq $7905747460161236407, %r8 # imm = 0x6DB6DB6DB6DB6DB7
-; X64-NEXT: movq %rdi, %rax
-; X64-NEXT: mulq %r8
-; X64-NEXT: addq %rcx, %rdx
-; X64-NEXT: imulq %rsi, %r8
-; X64-NEXT: addq %r8, %rdx
+; X64-NEXT: pushq %rax
+; X64-NEXT: movl $56, %edx
+; X64-NEXT: xorl %ecx, %ecx
+; X64-NEXT: callq __udivti3 at PLT
+; X64-NEXT: popq %rcx
; X64-NEXT: retq
%div = udiv i128 %x, 56 ; 8 * 7
ret i128 %div
@@ -1954,209 +1851,213 @@ define i128 @rem_by_7(i128 %x) nounwind {
; X86-NEXT: andl $-16, %esp
; X86-NEXT: subl $160, %esp
; X86-NEXT: movl 32(%ebp), %edi
+; X86-NEXT: movl 24(%ebp), %edx
; X86-NEXT: movl 36(%ebp), %ebx
-; X86-NEXT: movl 28(%ebp), %eax
+; X86-NEXT: movl 28(%ebp), %esi
+; X86-NEXT: movl %esi, %eax
+; X86-NEXT: orl %ebx, %eax
+; X86-NEXT: movl %edx, %ecx
+; X86-NEXT: orl %edi, %ecx
+; X86-NEXT: orl %eax, %ecx
+; X86-NEXT: sete %al
; X86-NEXT: testl %ebx, %ebx
; X86-NEXT: jne .LBB7_1
; X86-NEXT: # %bb.2: # %_udiv-special-cases
-; X86-NEXT: bsrl %edi, %edx
-; X86-NEXT: xorl $31, %edx
-; X86-NEXT: orl $32, %edx
-; X86-NEXT: jmp .LBB7_3
-; X86-NEXT: .LBB7_1:
-; X86-NEXT: bsrl %ebx, %edx
-; X86-NEXT: xorl $31, %edx
-; X86-NEXT: .LBB7_3: # %_udiv-special-cases
-; X86-NEXT: movl 24(%ebp), %esi
-; X86-NEXT: testl %eax, %eax
-; X86-NEXT: jne .LBB7_4
-; X86-NEXT: # %bb.5: # %_udiv-special-cases
-; X86-NEXT: bsrl %esi, %ecx
+; X86-NEXT: bsrl %edi, %ecx
; X86-NEXT: xorl $31, %ecx
; X86-NEXT: orl $32, %ecx
-; X86-NEXT: jmp .LBB7_6
+; X86-NEXT: testl %esi, %esi
+; X86-NEXT: je .LBB7_5
; X86-NEXT: .LBB7_4:
-; X86-NEXT: bsrl %eax, %ecx
+; X86-NEXT: bsrl %esi, %edx
+; X86-NEXT: xorl $31, %edx
+; X86-NEXT: jmp .LBB7_6
+; X86-NEXT: .LBB7_1:
+; X86-NEXT: bsrl %ebx, %ecx
; X86-NEXT: xorl $31, %ecx
+; X86-NEXT: testl %esi, %esi
+; X86-NEXT: jne .LBB7_4
+; X86-NEXT: .LBB7_5: # %_udiv-special-cases
+; X86-NEXT: bsrl %edx, %edx
+; X86-NEXT: xorl $31, %edx
+; X86-NEXT: orl $32, %edx
; X86-NEXT: .LBB7_6: # %_udiv-special-cases
-; X86-NEXT: orl %ebx, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %edi, %esi
-; X86-NEXT: orl %ebx, %edi
+; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: movl %edi, %esi
+; X86-NEXT: orl %ebx, %esi
; X86-NEXT: jne .LBB7_8
; X86-NEXT: # %bb.7: # %_udiv-special-cases
-; X86-NEXT: orl $64, %ecx
-; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: orl $64, %edx
+; X86-NEXT: movl %edx, %ecx
; X86-NEXT: .LBB7_8: # %_udiv-special-cases
-; X86-NEXT: movl $125, %ebx
-; X86-NEXT: subl %edx, %ebx
-; X86-NEXT: movl $0, %edx
-; X86-NEXT: sbbl %edx, %edx
-; X86-NEXT: movl $0, %eax
-; X86-NEXT: sbbl %eax, %eax
+; X86-NEXT: movl $125, %edx
+; X86-NEXT: subl %ecx, %edx
+; X86-NEXT: movl $0, %esi
+; X86-NEXT: sbbl %esi, %esi
; X86-NEXT: movl $0, %edi
; X86-NEXT: sbbl %edi, %edi
-; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: je .LBB7_9
-; X86-NEXT: # %bb.10: # %select.false.sink
-; X86-NEXT: xorl %esi, %esi
+; X86-NEXT: movl $0, %ebx
+; X86-NEXT: sbbl %ebx, %ebx
; X86-NEXT: movl $127, %ecx
-; X86-NEXT: cmpl %ebx, %ecx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: cmpl %edx, %ecx
; X86-NEXT: movl $0, %ecx
-; X86-NEXT: sbbl %edx, %ecx
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %esi, %ecx
; X86-NEXT: movl $0, %ecx
-; X86-NEXT: sbbl %eax, %ecx
-; X86-NEXT: sbbl %edi, %esi
-; X86-NEXT: setb %al
-; X86-NEXT: .LBB7_11: # %select.end
-; X86-NEXT: movl 32(%ebp), %edi
-; X86-NEXT: movl %edx, %ebx
-; X86-NEXT: xorl %edx, %edx
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %edi, %ecx
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: sbbl %ebx, %ecx
+; X86-NEXT: setb %cl
+; X86-NEXT: orb %cl, %al
; X86-NEXT: testb %al, %al
-; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: movl $7, %edi
+; X86-NEXT: movl $0, %edx
; X86-NEXT: movl $0, %eax
; X86-NEXT: movl $0, %esi
-; X86-NEXT: jne .LBB7_13
-; X86-NEXT: # %bb.12: # %select.end
-; X86-NEXT: movl 28(%ebp), %edx
+; X86-NEXT: jne .LBB7_10
+; X86-NEXT: # %bb.9: # %_udiv-special-cases
+; X86-NEXT: movl 36(%ebp), %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 32(%ebp), %edx
+; X86-NEXT: movl 28(%ebp), %eax
; X86-NEXT: movl 24(%ebp), %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %edi, %eax
-; X86-NEXT: movl 36(%ebp), %esi
-; X86-NEXT: .LBB7_13: # %select.end
+; X86-NEXT: .LBB7_10: # %_udiv-special-cases
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: jne .LBB7_14
-; X86-NEXT: # %bb.20: # %select.end
+; X86-NEXT: jne .LBB7_16
+; X86-NEXT: # %bb.11: # %_udiv-special-cases
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: xorl $127, %eax
-; X86-NEXT: orl %ecx, %eax
-; X86-NEXT: movl %ebx, %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: orl %edi, %ecx
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: orl %ebx, %ecx
; X86-NEXT: orl %eax, %ecx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: je .LBB7_21
-; X86-NEXT: # %bb.18: # %udiv-bb1
-; X86-NEXT: movl 24(%ebp), %ebx
-; X86-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; X86-NEXT: je .LBB7_16
+; X86-NEXT: # %bb.12: # %udiv-bb1
+; X86-NEXT: movl 24(%ebp), %edx
+; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X86-NEXT: movl 28(%ebp), %esi
; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
; X86-NEXT: movl 32(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 36(%ebp), %edx
-; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 36(%ebp), %eax
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: movl %edx, %ecx
; X86-NEXT: xorb $127, %cl
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: shrb $3, %al
; X86-NEXT: andb $12, %al
; X86-NEXT: negb %al
; X86-NEXT: movsbl %al, %eax
-; X86-NEXT: movl 136(%esp,%eax), %esi
-; X86-NEXT: movl 140(%esp,%eax), %edx
-; X86-NEXT: shldl %cl, %esi, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 128(%esp,%eax), %edx
+; X86-NEXT: movl %ebx, %esi
+; X86-NEXT: movl 136(%esp,%eax), %ebx
+; X86-NEXT: movl 140(%esp,%eax), %edi
+; X86-NEXT: shldl %cl, %ebx, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 128(%esp,%eax), %edi
; X86-NEXT: movl 132(%esp,%eax), %eax
-; X86-NEXT: shldl %cl, %eax, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shldl %cl, %edx, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shll %cl, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: addl $1, %ebx
-; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: adcl $0, %edi
+; X86-NEXT: shldl %cl, %eax, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shldl %cl, %edi, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shll %cl, %edi
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: jb .LBB7_19
-; X86-NEXT: # %bb.15: # %udiv-preheader
-; X86-NEXT: movl 24(%ebp), %eax
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 28(%ebp), %eax
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 32(%ebp), %eax
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 36(%ebp), %eax
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: addl $1, %edx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: adcl $0, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: adcl $0, %edi
+; X86-NEXT: adcl $0, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: jae .LBB7_13
+; X86-NEXT: # %bb.17:
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: jmp .LBB7_15
+; X86-NEXT: .LBB7_13: # %udiv-preheader
+; X86-NEXT: movl %edx, %ecx
+; X86-NEXT: movl 24(%ebp), %edx
+; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 28(%ebp), %edx
+; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 32(%ebp), %edx
+; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 36(%ebp), %edx
+; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
-; X86-NEXT: movl %ebx, %ecx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: shrb $3, %al
; X86-NEXT: andb $12, %al
; X86-NEXT: movzbl %al, %eax
-; X86-NEXT: movl 92(%esp,%eax), %esi
-; X86-NEXT: movl 88(%esp,%eax), %edx
-; X86-NEXT: movl %edx, %edi
-; X86-NEXT: shrdl %cl, %esi, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 80(%esp,%eax), %edi
-; X86-NEXT: movl 84(%esp,%eax), %ebx
-; X86-NEXT: movl %ebx, %eax
-; X86-NEXT: shrdl %cl, %edx, %eax
-; X86-NEXT: shrl %cl, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NEXT: shrdl %cl, %ebx, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $7, %edx
-; X86-NEXT: addl $-1, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $0, %edx
-; X86-NEXT: adcl $-1, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $0, %edx
-; X86-NEXT: adcl $-1, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $0, %edx
-; X86-NEXT: adcl $-1, %edx
+; X86-NEXT: movl 92(%esp,%eax), %edx
+; X86-NEXT: movl 88(%esp,%eax), %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shrdl %cl, %edx, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %edi, %ebx
+; X86-NEXT: movl 80(%esp,%eax), %esi
+; X86-NEXT: movl 84(%esp,%eax), %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: shrdl %cl, %edi, %eax
+; X86-NEXT: shrl %cl, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: xorl %edx, %edx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: shrdl %cl, %edx, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $7, %ecx
+; X86-NEXT: addl $-1, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: adcl $-1, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: adcl $-1, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: adcl $-1, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: .p2align 4
-; X86-NEXT: .LBB7_16: # %udiv-do-while
+; X86-NEXT: .LBB7_14: # %udiv-do-while
; X86-NEXT: # =>This Inner Loop Header: Depth=1
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: shldl $1, %ebx, %esi
+; X86-NEXT: shldl $1, %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: shldl $1, %eax, %ebx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: shldl $1, %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: shldl $1, %ecx, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: shldl $1, %edx, %eax
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: shldl $1, %edi, %ecx
-; X86-NEXT: orl %edx, %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: shldl $1, %edi, %edx
+; X86-NEXT: orl %esi, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: shldl $1, %ecx, %edi
-; X86-NEXT: orl %edx, %edi
+; X86-NEXT: orl %esi, %edi
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-NEXT: shldl $1, %edi, %ecx
-; X86-NEXT: orl %edx, %ecx
+; X86-NEXT: orl %esi, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: addl %edi, %edi
; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
@@ -2166,13 +2067,14 @@ define i128 @rem_by_7(i128 %x) nounwind {
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: sbbl %ebx, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: sbbl %esi, %ecx
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
; X86-NEXT: sarl $31, %ecx
; X86-NEXT: movl %ecx, %edx
; X86-NEXT: andl $1, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $7, %edx
-; X86-NEXT: andl %edx, %ecx
+; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: movl $7, %esi
+; X86-NEXT: andl %esi, %ecx
; X86-NEXT: subl %ecx, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
@@ -2180,59 +2082,59 @@ define i128 @rem_by_7(i128 %x) nounwind {
; X86-NEXT: sbbl $0, %ebx
; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: sbbl $0, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: addl $-1, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: addl $-1, %esi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: adcl $-1, %ecx
; X86-NEXT: adcl $-1, %ebx
-; X86-NEXT: adcl $-1, %esi
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %esi, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: orl %ebx, %edx
-; X86-NEXT: orl %ecx, %edx
-; X86-NEXT: movl $0, %edx
+; X86-NEXT: adcl $-1, %edx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %edx, %ecx
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %ebx, %esi
+; X86-NEXT: orl %ecx, %esi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: jne .LBB7_16
-; X86-NEXT: .LBB7_17: # %udiv-loop-exit
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shldl $1, %eax, %ecx
+; X86-NEXT: jne .LBB7_14
+; X86-NEXT: .LBB7_15: # %udiv-loop-exit
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: shldl $1, %edx, %eax
-; X86-NEXT: shldl $1, %edi, %edx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: leal (%esi,%edi,2), %ebx
-; X86-NEXT: movl %ecx, %esi
-; X86-NEXT: .LBB7_21: # %udiv-end
-; X86-NEXT: movl $7, %ecx
-; X86-NEXT: imull %ecx, %esi
-; X86-NEXT: movl %edx, %edi
-; X86-NEXT: mull %ecx
+; X86-NEXT: shldl $1, %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: shldl $1, %ecx, %edx
+; X86-NEXT: shldl $1, %edi, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: leal (%eax,%edi,2), %esi
+; X86-NEXT: movl $7, %edi
+; X86-NEXT: .LBB7_16: # %udiv-end
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: imull %edi, %ebx
+; X86-NEXT: movl %edx, %eax
+; X86-NEXT: mull %edi
; X86-NEXT: movl %edx, %ecx
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %eax, %edi
; X86-NEXT: movl $7, %eax
-; X86-NEXT: mull %ebx
-; X86-NEXT: movl %edx, %ebx
+; X86-NEXT: mull %esi
+; X86-NEXT: movl %edx, %esi
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $7, %eax
-; X86-NEXT: mull %edi
-; X86-NEXT: addl %ebx, %eax
-; X86-NEXT: adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X86-NEXT: adcl %esi, %ecx
-; X86-NEXT: movl 24(%ebp), %ebx
-; X86-NEXT: subl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X86-NEXT: movl 28(%ebp), %esi
-; X86-NEXT: sbbl %eax, %esi
-; X86-NEXT: movl 32(%ebp), %edi
-; X86-NEXT: sbbl %edx, %edi
+; X86-NEXT: mull {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; X86-NEXT: addl %esi, %eax
+; X86-NEXT: adcl %edi, %edx
+; X86-NEXT: adcl %ebx, %ecx
+; X86-NEXT: movl 24(%ebp), %esi
+; X86-NEXT: subl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NEXT: movl 28(%ebp), %edi
+; X86-NEXT: sbbl %eax, %edi
+; X86-NEXT: movl 32(%ebp), %ebx
+; X86-NEXT: sbbl %edx, %ebx
; X86-NEXT: movl 36(%ebp), %edx
; X86-NEXT: sbbl %ecx, %edx
; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: movl %ebx, (%eax)
-; X86-NEXT: movl %esi, 4(%eax)
-; X86-NEXT: movl %edi, 8(%eax)
+; X86-NEXT: movl %esi, (%eax)
+; X86-NEXT: movl %edi, 4(%eax)
+; X86-NEXT: movl %ebx, 8(%eax)
; X86-NEXT: movl %edx, 12(%eax)
; X86-NEXT: leal -12(%ebp), %esp
; X86-NEXT: popl %esi
@@ -2240,38 +2142,14 @@ define i128 @rem_by_7(i128 %x) nounwind {
; X86-NEXT: popl %ebx
; X86-NEXT: popl %ebp
; X86-NEXT: retl $4
-; X86-NEXT: .LBB7_9:
-; X86-NEXT: movb $1, %al
-; X86-NEXT: jmp .LBB7_11
-; X86-NEXT: .LBB7_19:
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: jmp .LBB7_17
-; X86-NEXT: .LBB7_14:
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: jmp .LBB7_21
;
; X64-LABEL: rem_by_7:
; X64: # %bb.0:
-; X64-NEXT: movabsq $1152921504606846975, %rcx # imm = 0xFFFFFFFFFFFFFFF
-; X64-NEXT: movq %rdi, %rax
-; X64-NEXT: andq %rcx, %rax
-; X64-NEXT: shrdq $60, %rsi, %rdi
-; X64-NEXT: andq %rdi, %rcx
-; X64-NEXT: addq %rax, %rcx
-; X64-NEXT: shrq $56, %rsi
-; X64-NEXT: addq %rsi, %rcx
-; X64-NEXT: movabsq $5270498306774157605, %rdx # imm = 0x4924924924924925
-; X64-NEXT: movq %rcx, %rax
-; X64-NEXT: mulq %rdx
-; X64-NEXT: shrq %rdx
-; X64-NEXT: leaq (,%rdx,8), %rax
-; X64-NEXT: subq %rax, %rdx
-; X64-NEXT: addq %rdx, %rcx
-; X64-NEXT: movq %rcx, %rax
-; X64-NEXT: xorl %edx, %edx
+; X64-NEXT: pushq %rax
+; X64-NEXT: movl $7, %edx
+; X64-NEXT: xorl %ecx, %ecx
+; X64-NEXT: callq __umodti3 at PLT
+; X64-NEXT: popq %rcx
; X64-NEXT: retq
%rem = urem i128 %x, 7
ret i128 %rem
@@ -2288,209 +2166,213 @@ define i128 @rem_by_14(i128 %x) nounwind {
; X86-NEXT: andl $-16, %esp
; X86-NEXT: subl $160, %esp
; X86-NEXT: movl 32(%ebp), %edi
+; X86-NEXT: movl 24(%ebp), %edx
; X86-NEXT: movl 36(%ebp), %ebx
-; X86-NEXT: movl 28(%ebp), %eax
+; X86-NEXT: movl 28(%ebp), %esi
+; X86-NEXT: movl %esi, %eax
+; X86-NEXT: orl %ebx, %eax
+; X86-NEXT: movl %edx, %ecx
+; X86-NEXT: orl %edi, %ecx
+; X86-NEXT: orl %eax, %ecx
+; X86-NEXT: sete %al
; X86-NEXT: testl %ebx, %ebx
; X86-NEXT: jne .LBB8_1
; X86-NEXT: # %bb.2: # %_udiv-special-cases
-; X86-NEXT: bsrl %edi, %edx
-; X86-NEXT: xorl $31, %edx
-; X86-NEXT: orl $32, %edx
-; X86-NEXT: jmp .LBB8_3
-; X86-NEXT: .LBB8_1:
-; X86-NEXT: bsrl %ebx, %edx
-; X86-NEXT: xorl $31, %edx
-; X86-NEXT: .LBB8_3: # %_udiv-special-cases
-; X86-NEXT: movl 24(%ebp), %esi
-; X86-NEXT: testl %eax, %eax
-; X86-NEXT: jne .LBB8_4
-; X86-NEXT: # %bb.5: # %_udiv-special-cases
-; X86-NEXT: bsrl %esi, %ecx
+; X86-NEXT: bsrl %edi, %ecx
; X86-NEXT: xorl $31, %ecx
; X86-NEXT: orl $32, %ecx
-; X86-NEXT: jmp .LBB8_6
+; X86-NEXT: testl %esi, %esi
+; X86-NEXT: je .LBB8_5
; X86-NEXT: .LBB8_4:
-; X86-NEXT: bsrl %eax, %ecx
+; X86-NEXT: bsrl %esi, %edx
+; X86-NEXT: xorl $31, %edx
+; X86-NEXT: jmp .LBB8_6
+; X86-NEXT: .LBB8_1:
+; X86-NEXT: bsrl %ebx, %ecx
; X86-NEXT: xorl $31, %ecx
+; X86-NEXT: testl %esi, %esi
+; X86-NEXT: jne .LBB8_4
+; X86-NEXT: .LBB8_5: # %_udiv-special-cases
+; X86-NEXT: bsrl %edx, %edx
+; X86-NEXT: xorl $31, %edx
+; X86-NEXT: orl $32, %edx
; X86-NEXT: .LBB8_6: # %_udiv-special-cases
-; X86-NEXT: orl %ebx, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %edi, %esi
-; X86-NEXT: orl %ebx, %edi
+; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: movl %edi, %esi
+; X86-NEXT: orl %ebx, %esi
; X86-NEXT: jne .LBB8_8
; X86-NEXT: # %bb.7: # %_udiv-special-cases
-; X86-NEXT: orl $64, %ecx
-; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: orl $64, %edx
+; X86-NEXT: movl %edx, %ecx
; X86-NEXT: .LBB8_8: # %_udiv-special-cases
-; X86-NEXT: movl $124, %ebx
-; X86-NEXT: subl %edx, %ebx
-; X86-NEXT: movl $0, %edx
-; X86-NEXT: sbbl %edx, %edx
-; X86-NEXT: movl $0, %eax
-; X86-NEXT: sbbl %eax, %eax
+; X86-NEXT: movl $124, %edx
+; X86-NEXT: subl %ecx, %edx
+; X86-NEXT: movl $0, %esi
+; X86-NEXT: sbbl %esi, %esi
; X86-NEXT: movl $0, %edi
; X86-NEXT: sbbl %edi, %edi
-; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: je .LBB8_9
-; X86-NEXT: # %bb.10: # %select.false.sink
-; X86-NEXT: xorl %esi, %esi
+; X86-NEXT: movl $0, %ebx
+; X86-NEXT: sbbl %ebx, %ebx
; X86-NEXT: movl $127, %ecx
-; X86-NEXT: cmpl %ebx, %ecx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: cmpl %edx, %ecx
; X86-NEXT: movl $0, %ecx
-; X86-NEXT: sbbl %edx, %ecx
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %esi, %ecx
; X86-NEXT: movl $0, %ecx
-; X86-NEXT: sbbl %eax, %ecx
-; X86-NEXT: sbbl %edi, %esi
-; X86-NEXT: setb %al
-; X86-NEXT: .LBB8_11: # %select.end
-; X86-NEXT: movl 32(%ebp), %edi
-; X86-NEXT: movl %edx, %ebx
-; X86-NEXT: xorl %edx, %edx
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %edi, %ecx
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: sbbl %ebx, %ecx
+; X86-NEXT: setb %cl
+; X86-NEXT: orb %cl, %al
; X86-NEXT: testb %al, %al
-; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: movl $14, %edi
+; X86-NEXT: movl $0, %edx
; X86-NEXT: movl $0, %eax
; X86-NEXT: movl $0, %esi
-; X86-NEXT: jne .LBB8_13
-; X86-NEXT: # %bb.12: # %select.end
-; X86-NEXT: movl 28(%ebp), %edx
+; X86-NEXT: jne .LBB8_10
+; X86-NEXT: # %bb.9: # %_udiv-special-cases
+; X86-NEXT: movl 36(%ebp), %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 32(%ebp), %edx
+; X86-NEXT: movl 28(%ebp), %eax
; X86-NEXT: movl 24(%ebp), %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %edi, %eax
-; X86-NEXT: movl 36(%ebp), %esi
-; X86-NEXT: .LBB8_13: # %select.end
+; X86-NEXT: .LBB8_10: # %_udiv-special-cases
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: jne .LBB8_14
-; X86-NEXT: # %bb.20: # %select.end
+; X86-NEXT: jne .LBB8_16
+; X86-NEXT: # %bb.11: # %_udiv-special-cases
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: xorl $127, %eax
-; X86-NEXT: orl %ecx, %eax
-; X86-NEXT: movl %ebx, %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: orl %edi, %ecx
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: orl %ebx, %ecx
; X86-NEXT: orl %eax, %ecx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: je .LBB8_21
-; X86-NEXT: # %bb.18: # %udiv-bb1
-; X86-NEXT: movl 24(%ebp), %ebx
-; X86-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; X86-NEXT: je .LBB8_16
+; X86-NEXT: # %bb.12: # %udiv-bb1
+; X86-NEXT: movl 24(%ebp), %edx
+; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X86-NEXT: movl 28(%ebp), %esi
; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
; X86-NEXT: movl 32(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 36(%ebp), %edx
-; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 36(%ebp), %eax
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: movl %edx, %ecx
; X86-NEXT: xorb $127, %cl
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: shrb $3, %al
; X86-NEXT: andb $12, %al
; X86-NEXT: negb %al
; X86-NEXT: movsbl %al, %eax
-; X86-NEXT: movl 136(%esp,%eax), %esi
-; X86-NEXT: movl 140(%esp,%eax), %edx
-; X86-NEXT: shldl %cl, %esi, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 128(%esp,%eax), %edx
+; X86-NEXT: movl %ebx, %esi
+; X86-NEXT: movl 136(%esp,%eax), %ebx
+; X86-NEXT: movl 140(%esp,%eax), %edi
+; X86-NEXT: shldl %cl, %ebx, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 128(%esp,%eax), %edi
; X86-NEXT: movl 132(%esp,%eax), %eax
-; X86-NEXT: shldl %cl, %eax, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shldl %cl, %edx, %eax
+; X86-NEXT: shldl %cl, %eax, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shldl %cl, %edi, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shll %cl, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: addl $1, %ebx
-; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: adcl $0, %edi
+; X86-NEXT: shll %cl, %edi
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: jb .LBB8_19
-; X86-NEXT: # %bb.15: # %udiv-preheader
-; X86-NEXT: movl 24(%ebp), %eax
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 28(%ebp), %eax
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 32(%ebp), %eax
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 36(%ebp), %eax
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: addl $1, %edx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: adcl $0, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: adcl $0, %edi
+; X86-NEXT: adcl $0, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: jae .LBB8_13
+; X86-NEXT: # %bb.17:
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: jmp .LBB8_15
+; X86-NEXT: .LBB8_13: # %udiv-preheader
+; X86-NEXT: movl %edx, %ecx
+; X86-NEXT: movl 24(%ebp), %edx
+; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 28(%ebp), %edx
+; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 32(%ebp), %edx
+; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 36(%ebp), %edx
+; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
-; X86-NEXT: movl %ebx, %ecx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: shrb $3, %al
; X86-NEXT: andb $12, %al
; X86-NEXT: movzbl %al, %eax
-; X86-NEXT: movl 92(%esp,%eax), %esi
-; X86-NEXT: movl 88(%esp,%eax), %edx
-; X86-NEXT: movl %edx, %edi
-; X86-NEXT: shrdl %cl, %esi, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 80(%esp,%eax), %edi
-; X86-NEXT: movl 84(%esp,%eax), %ebx
-; X86-NEXT: movl %ebx, %eax
-; X86-NEXT: shrdl %cl, %edx, %eax
-; X86-NEXT: shrl %cl, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NEXT: shrdl %cl, %ebx, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $14, %edx
-; X86-NEXT: addl $-1, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $0, %edx
-; X86-NEXT: adcl $-1, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $0, %edx
-; X86-NEXT: adcl $-1, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $0, %edx
-; X86-NEXT: adcl $-1, %edx
+; X86-NEXT: movl 92(%esp,%eax), %edx
+; X86-NEXT: movl 88(%esp,%eax), %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shrdl %cl, %edx, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %edi, %ebx
+; X86-NEXT: movl 80(%esp,%eax), %esi
+; X86-NEXT: movl 84(%esp,%eax), %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: shrdl %cl, %edi, %eax
+; X86-NEXT: shrl %cl, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: xorl %edx, %edx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: # kill: def $cl killed $cl killed $ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: shrdl %cl, %edx, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $14, %ecx
+; X86-NEXT: addl $-1, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: adcl $-1, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: adcl $-1, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: adcl $-1, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: .p2align 4
-; X86-NEXT: .LBB8_16: # %udiv-do-while
+; X86-NEXT: .LBB8_14: # %udiv-do-while
; X86-NEXT: # =>This Inner Loop Header: Depth=1
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: shldl $1, %ebx, %esi
+; X86-NEXT: shldl $1, %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: shldl $1, %eax, %ebx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: shldl $1, %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: shldl $1, %ecx, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: shldl $1, %edx, %eax
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: shldl $1, %edi, %ecx
-; X86-NEXT: orl %edx, %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: shldl $1, %edi, %edx
+; X86-NEXT: orl %esi, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: shldl $1, %ecx, %edi
-; X86-NEXT: orl %edx, %edi
+; X86-NEXT: orl %esi, %edi
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-NEXT: shldl $1, %edi, %ecx
-; X86-NEXT: orl %edx, %ecx
+; X86-NEXT: orl %esi, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: addl %edi, %edi
; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
@@ -2500,13 +2382,14 @@ define i128 @rem_by_14(i128 %x) nounwind {
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: sbbl %ebx, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: sbbl %esi, %ecx
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
; X86-NEXT: sarl $31, %ecx
; X86-NEXT: movl %ecx, %edx
; X86-NEXT: andl $1, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $14, %edx
-; X86-NEXT: andl %edx, %ecx
+; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: movl $14, %esi
+; X86-NEXT: andl %esi, %ecx
; X86-NEXT: subl %ecx, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
@@ -2514,59 +2397,59 @@ define i128 @rem_by_14(i128 %x) nounwind {
; X86-NEXT: sbbl $0, %ebx
; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: sbbl $0, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: addl $-1, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: addl $-1, %esi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: adcl $-1, %ecx
; X86-NEXT: adcl $-1, %ebx
-; X86-NEXT: adcl $-1, %esi
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %esi, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: orl %ebx, %edx
-; X86-NEXT: orl %ecx, %edx
-; X86-NEXT: movl $0, %edx
+; X86-NEXT: adcl $-1, %edx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %edx, %ecx
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %ebx, %esi
+; X86-NEXT: orl %ecx, %esi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: jne .LBB8_16
-; X86-NEXT: .LBB8_17: # %udiv-loop-exit
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shldl $1, %eax, %ecx
+; X86-NEXT: jne .LBB8_14
+; X86-NEXT: .LBB8_15: # %udiv-loop-exit
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: shldl $1, %edx, %eax
-; X86-NEXT: shldl $1, %edi, %edx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: leal (%esi,%edi,2), %ebx
-; X86-NEXT: movl %ecx, %esi
-; X86-NEXT: .LBB8_21: # %udiv-end
-; X86-NEXT: movl $14, %ecx
-; X86-NEXT: imull %ecx, %esi
-; X86-NEXT: movl %edx, %edi
-; X86-NEXT: mull %ecx
+; X86-NEXT: shldl $1, %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: shldl $1, %ecx, %edx
+; X86-NEXT: shldl $1, %edi, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: leal (%eax,%edi,2), %esi
+; X86-NEXT: movl $14, %edi
+; X86-NEXT: .LBB8_16: # %udiv-end
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: imull %edi, %ebx
+; X86-NEXT: movl %edx, %eax
+; X86-NEXT: mull %edi
; X86-NEXT: movl %edx, %ecx
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %eax, %edi
; X86-NEXT: movl $14, %eax
-; X86-NEXT: mull %ebx
-; X86-NEXT: movl %edx, %ebx
+; X86-NEXT: mull %esi
+; X86-NEXT: movl %edx, %esi
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $14, %eax
-; X86-NEXT: mull %edi
-; X86-NEXT: addl %ebx, %eax
-; X86-NEXT: adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X86-NEXT: adcl %esi, %ecx
-; X86-NEXT: movl 24(%ebp), %ebx
-; X86-NEXT: subl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X86-NEXT: movl 28(%ebp), %esi
-; X86-NEXT: sbbl %eax, %esi
-; X86-NEXT: movl 32(%ebp), %edi
-; X86-NEXT: sbbl %edx, %edi
+; X86-NEXT: mull {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; X86-NEXT: addl %esi, %eax
+; X86-NEXT: adcl %edi, %edx
+; X86-NEXT: adcl %ebx, %ecx
+; X86-NEXT: movl 24(%ebp), %esi
+; X86-NEXT: subl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NEXT: movl 28(%ebp), %edi
+; X86-NEXT: sbbl %eax, %edi
+; X86-NEXT: movl 32(%ebp), %ebx
+; X86-NEXT: sbbl %edx, %ebx
; X86-NEXT: movl 36(%ebp), %edx
; X86-NEXT: sbbl %ecx, %edx
; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: movl %ebx, (%eax)
-; X86-NEXT: movl %esi, 4(%eax)
-; X86-NEXT: movl %edi, 8(%eax)
+; X86-NEXT: movl %esi, (%eax)
+; X86-NEXT: movl %edi, 4(%eax)
+; X86-NEXT: movl %ebx, 8(%eax)
; X86-NEXT: movl %edx, 12(%eax)
; X86-NEXT: leal -12(%ebp), %esp
; X86-NEXT: popl %esi
@@ -2574,41 +2457,14 @@ define i128 @rem_by_14(i128 %x) nounwind {
; X86-NEXT: popl %ebx
; X86-NEXT: popl %ebp
; X86-NEXT: retl $4
-; X86-NEXT: .LBB8_9:
-; X86-NEXT: movb $1, %al
-; X86-NEXT: jmp .LBB8_11
-; X86-NEXT: .LBB8_19:
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: jmp .LBB8_17
-; X86-NEXT: .LBB8_14:
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: jmp .LBB8_21
;
; X64-LABEL: rem_by_14:
; X64: # %bb.0:
-; X64-NEXT: movq %rdi, %rax
-; X64-NEXT: shrdq $61, %rsi, %rax
-; X64-NEXT: movabsq $1152921504606846975, %rcx # imm = 0xFFFFFFFFFFFFFFF
-; X64-NEXT: andq %rcx, %rax
-; X64-NEXT: movq %rdi, %rdx
-; X64-NEXT: shrq %rdx
-; X64-NEXT: andq %rcx, %rdx
-; X64-NEXT: addq %rax, %rdx
-; X64-NEXT: shrq $57, %rsi
-; X64-NEXT: addq %rdx, %rsi
-; X64-NEXT: movabsq $5270498306774157605, %rcx # imm = 0x4924924924924925
-; X64-NEXT: movq %rsi, %rax
-; X64-NEXT: mulq %rcx
-; X64-NEXT: shrq %rdx
-; X64-NEXT: leal (,%rdx,8), %eax
-; X64-NEXT: subl %eax, %edx
-; X64-NEXT: addl %edx, %esi
-; X64-NEXT: andl $1, %edi
-; X64-NEXT: leaq (%rdi,%rsi,2), %rax
-; X64-NEXT: xorl %edx, %edx
+; X64-NEXT: pushq %rax
+; X64-NEXT: movl $14, %edx
+; X64-NEXT: xorl %ecx, %ecx
+; X64-NEXT: callq __umodti3 at PLT
+; X64-NEXT: popq %rcx
; X64-NEXT: retq
%rem = urem i128 %x, 14
ret i128 %rem
@@ -2624,168 +2480,178 @@ define i128 @div_by_67(i128 %x) nounwind {
; X86-NEXT: pushl %esi
; X86-NEXT: andl $-16, %esp
; X86-NEXT: subl $160, %esp
-; X86-NEXT: movl 32(%ebp), %edi
-; X86-NEXT: movl 36(%ebp), %ebx
-; X86-NEXT: movl 28(%ebp), %edx
-; X86-NEXT: testl %ebx, %ebx
+; X86-NEXT: movl 32(%ebp), %esi
+; X86-NEXT: movl 24(%ebp), %edx
+; X86-NEXT: movl 36(%ebp), %edi
+; X86-NEXT: movl 28(%ebp), %ebx
+; X86-NEXT: movl %ebx, %eax
+; X86-NEXT: orl %edi, %eax
+; X86-NEXT: movl %edx, %ecx
+; X86-NEXT: orl %esi, %ecx
+; X86-NEXT: orl %eax, %ecx
+; X86-NEXT: sete {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X86-NEXT: testl %edi, %edi
; X86-NEXT: jne .LBB9_1
; X86-NEXT: # %bb.2: # %entry_udiv-special-cases
-; X86-NEXT: bsrl %edi, %esi
-; X86-NEXT: xorl $31, %esi
-; X86-NEXT: orl $32, %esi
-; X86-NEXT: jmp .LBB9_3
-; X86-NEXT: .LBB9_1:
-; X86-NEXT: bsrl %ebx, %esi
-; X86-NEXT: xorl $31, %esi
-; X86-NEXT: .LBB9_3: # %entry_udiv-special-cases
-; X86-NEXT: movl 24(%ebp), %eax
-; X86-NEXT: testl %edx, %edx
-; X86-NEXT: jne .LBB9_4
-; X86-NEXT: # %bb.5: # %entry_udiv-special-cases
-; X86-NEXT: bsrl %eax, %ecx
+; X86-NEXT: bsrl %esi, %ecx
; X86-NEXT: xorl $31, %ecx
; X86-NEXT: orl $32, %ecx
-; X86-NEXT: jmp .LBB9_6
+; X86-NEXT: testl %ebx, %ebx
+; X86-NEXT: je .LBB9_5
; X86-NEXT: .LBB9_4:
-; X86-NEXT: bsrl %edx, %ecx
+; X86-NEXT: bsrl %ebx, %edx
+; X86-NEXT: xorl $31, %edx
+; X86-NEXT: jmp .LBB9_6
+; X86-NEXT: .LBB9_1:
+; X86-NEXT: bsrl %edi, %ecx
; X86-NEXT: xorl $31, %ecx
+; X86-NEXT: testl %ebx, %ebx
+; X86-NEXT: jne .LBB9_4
+; X86-NEXT: .LBB9_5: # %entry_udiv-special-cases
+; X86-NEXT: bsrl %edx, %edx
+; X86-NEXT: xorl $31, %edx
+; X86-NEXT: orl $32, %edx
; X86-NEXT: .LBB9_6: # %entry_udiv-special-cases
-; X86-NEXT: orl %ebx, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %edi, %eax
-; X86-NEXT: orl %ebx, %edi
+; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: orl %edi, %esi
; X86-NEXT: jne .LBB9_8
; X86-NEXT: # %bb.7: # %entry_udiv-special-cases
-; X86-NEXT: orl $64, %ecx
-; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: orl $64, %edx
+; X86-NEXT: movl %edx, %ecx
; X86-NEXT: .LBB9_8: # %entry_udiv-special-cases
-; X86-NEXT: movl $121, %ebx
-; X86-NEXT: subl %esi, %ebx
-; X86-NEXT: movl $0, %edx
-; X86-NEXT: sbbl %edx, %edx
+; X86-NEXT: movl $121, %edx
+; X86-NEXT: subl %ecx, %edx
+; X86-NEXT: movl $0, %ebx
+; X86-NEXT: sbbl %ebx, %ebx
; X86-NEXT: movl $0, %esi
; X86-NEXT: sbbl %esi, %esi
-; X86-NEXT: movl $0, %edi
-; X86-NEXT: sbbl %edi, %edi
-; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: je .LBB9_9
-; X86-NEXT: # %bb.10: # %select.false.sink
-; X86-NEXT: xorl %eax, %eax
+; X86-NEXT: movl $0, %eax
+; X86-NEXT: sbbl %eax, %eax
; X86-NEXT: movl $127, %ecx
-; X86-NEXT: cmpl %ebx, %ecx
+; X86-NEXT: cmpl %edx, %ecx
; X86-NEXT: movl $0, %ecx
-; X86-NEXT: sbbl %edx, %ecx
+; X86-NEXT: sbbl %ebx, %ecx
; X86-NEXT: movl $0, %ecx
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: sbbl %esi, %ecx
-; X86-NEXT: sbbl %edi, %eax
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %eax, %ecx
; X86-NEXT: setb %cl
-; X86-NEXT: .LBB9_11: # %select.end
+; X86-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; X86-NEXT: orb %cl, %al
+; X86-NEXT: testb %al, %al
; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: xorl %edx, %edx
-; X86-NEXT: testb %cl, %cl
-; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: movl $0, %esi
; X86-NEXT: movl $0, %ecx
-; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: jne .LBB9_13
-; X86-NEXT: # %bb.12: # %select.end
-; X86-NEXT: movl 28(%ebp), %edx
-; X86-NEXT: movl 24(%ebp), %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 32(%ebp), %ecx
-; X86-NEXT: movl 36(%ebp), %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: .LBB9_13: # %select.end
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: jne .LBB9_14
-; X86-NEXT: # %bb.20: # %select.end
+; X86-NEXT: movl $0, %edi
+; X86-NEXT: jne .LBB9_10
+; X86-NEXT: # %bb.9: # %entry_udiv-special-cases
; X86-NEXT: movl %ebx, %ecx
-; X86-NEXT: xorl $127, %ecx
-; X86-NEXT: orl %esi, %ecx
+; X86-NEXT: movl 36(%ebp), %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ecx, %ebx
+; X86-NEXT: movl 32(%ebp), %esi
+; X86-NEXT: movl 28(%ebp), %ecx
+; X86-NEXT: movl 24(%ebp), %edi
+; X86-NEXT: .LBB9_10: # %entry_udiv-special-cases
+; X86-NEXT: jne .LBB9_16
+; X86-NEXT: # %bb.11: # %entry_udiv-special-cases
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: movl %esi, %edx
-; X86-NEXT: orl %edi, %edx
+; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: movl %edx, %ecx
+; X86-NEXT: xorl $127, %ecx
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ebx, %edx
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
; X86-NEXT: orl %ecx, %edx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: je .LBB9_21
-; X86-NEXT: # %bb.18: # %udiv-bb1
+; X86-NEXT: movl %esi, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: je .LBB9_16
+; X86-NEXT: # %bb.12: # %udiv-bb1
; X86-NEXT: movl 24(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: movl 28(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 32(%ebp), %eax
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 32(%ebp), %edi
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X86-NEXT: movl 36(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
-; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: movl %eax, %edi
; X86-NEXT: xorb $127, %cl
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: shrb $3, %al
; X86-NEXT: andb $12, %al
; X86-NEXT: negb %al
; X86-NEXT: movsbl %al, %eax
-; X86-NEXT: movl 136(%esp,%eax), %edx
-; X86-NEXT: movl 140(%esp,%eax), %edi
-; X86-NEXT: shldl %cl, %edx, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 128(%esp,%eax), %edi
-; X86-NEXT: movl 132(%esp,%eax), %eax
-; X86-NEXT: shldl %cl, %eax, %edx
+; X86-NEXT: movl 136(%esp,%eax), %esi
+; X86-NEXT: movl 140(%esp,%eax), %edx
+; X86-NEXT: shldl %cl, %esi, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shldl %cl, %edi, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shll %cl, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: addl $1, %ebx
-; X86-NEXT: adcl $0, %esi
+; X86-NEXT: movl 128(%esp,%eax), %edx
+; X86-NEXT: movl 132(%esp,%eax), %eax
+; X86-NEXT: shldl %cl, %eax, %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: jb .LBB9_19
-; X86-NEXT: # %bb.15: # %udiv-preheader
-; X86-NEXT: movl %ebx, %ecx
-; X86-NEXT: movl 24(%ebp), %eax
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 28(%ebp), %eax
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 32(%ebp), %eax
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 36(%ebp), %eax
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: shldl %cl, %edx, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shll %cl, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: addl $1, %edi
+; X86-NEXT: adcl $0, %ebx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: adcl $0, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: adcl $0, %edx
+; X86-NEXT: jae .LBB9_13
+; X86-NEXT: # %bb.17:
+; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: jmp .LBB9_15
+; X86-NEXT: .LBB9_13: # %udiv-preheader
+; X86-NEXT: movl %edi, %ecx
+; X86-NEXT: movl 24(%ebp), %esi
+; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 28(%ebp), %esi
+; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 32(%ebp), %esi
+; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 36(%ebp), %esi
+; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: shrb $3, %al
; X86-NEXT: andb $12, %al
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movzbl %al, %edi
; X86-NEXT: movl 92(%esp,%edi), %eax
-; X86-NEXT: movl 88(%esp,%edi), %ebx
-; X86-NEXT: movl %ebx, %esi
-; X86-NEXT: shrdl %cl, %eax, %esi
+; X86-NEXT: movl 88(%esp,%edi), %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 80(%esp,%edi), %esi
-; X86-NEXT: movl 84(%esp,%edi), %edi
-; X86-NEXT: movl %edi, %edx
-; X86-NEXT: shrdl %cl, %ebx, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %esi, %edx
+; X86-NEXT: shrdl %cl, %eax, %edx
+; X86-NEXT: movl 80(%esp,%edi), %ebx
+; X86-NEXT: movl 84(%esp,%edi), %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: shrdl %cl, %esi, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: shrl %cl, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NEXT: shrdl %cl, %edi, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: shrdl %cl, %eax, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $67, %eax
; X86-NEXT: addl $-1, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
@@ -2799,79 +2665,78 @@ define i128 @div_by_67(i128 %x) nounwind {
; X86-NEXT: adcl $-1, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: .p2align 4
-; X86-NEXT: .LBB9_16: # %udiv-do-while
+; X86-NEXT: .LBB9_14: # %udiv-do-while
; X86-NEXT: # =>This Inner Loop Header: Depth=1
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %edx, %edi
+; X86-NEXT: shldl $1, %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: shldl $1, %eax, %edi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: shldl $1, %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: shldl $1, %edi, %ebx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: shldl $1, %edx, %edi
-; X86-NEXT: shldl $1, %eax, %edx
-; X86-NEXT: shldl $1, %ecx, %eax
+; X86-NEXT: shldl $1, %ebx, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: shldl $1, %ecx, %ebx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: orl %esi, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shldl $1, %eax, %ecx
-; X86-NEXT: orl %esi, %ecx
+; X86-NEXT: shldl $1, %esi, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: orl %edx, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: shldl $1, %ecx, %eax
-; X86-NEXT: orl %esi, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: addl %ecx, %ecx
-; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: shldl $1, %ecx, %esi
+; X86-NEXT: orl %edx, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: shldl $1, %esi, %ecx
+; X86-NEXT: orl %edx, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: cmpl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; X86-NEXT: addl %esi, %esi
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: cmpl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: sbbl %edi, %ecx
+; X86-NEXT: sbbl %eax, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: sbbl %ebx, %ecx
+; X86-NEXT: sbbl %edi, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: sbbl %edx, %ecx
; X86-NEXT: sarl $31, %ecx
-; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: andl $1, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: andl $1, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl $67, %eax
-; X86-NEXT: andl %eax, %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: subl %ecx, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $67, %esi
+; X86-NEXT: andl %esi, %ecx
+; X86-NEXT: subl %ecx, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl $0, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: sbbl $0, %edi
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: sbbl $0, %ebx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: sbbl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: movl %edx, %eax
+; X86-NEXT: sbbl $0, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NEXT: addl $-1, %ebx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: adcl $-1, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-NEXT: adcl $-1, %edi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NEXT: adcl $-1, %edx
-; X86-NEXT: adcl $-1, %esi
-; X86-NEXT: movl %edi, %ecx
-; X86-NEXT: orl %esi, %ecx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %edx, %ebx
+; X86-NEXT: orl %edx, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %edi, %ebx
; X86-NEXT: orl %ecx, %ebx
+; X86-NEXT: jne .LBB9_14
+; X86-NEXT: .LBB9_15: # %udiv-loop-exit
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: jne .LBB9_16
-; X86-NEXT: .LBB9_17: # %udiv-loop-exit
-; X86-NEXT: shldl $1, %ecx, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shldl $1, %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NEXT: shldl $1, %edx, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
@@ -2879,10 +2744,12 @@ define i128 @div_by_67(i128 %x) nounwind {
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NEXT: leal (%esi,%eax,2), %edi
; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: .LBB9_21: # %udiv-end
+; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: movl %edx, %ecx
+; X86-NEXT: .LBB9_16: # %udiv-end
; X86-NEXT: movl %edi, (%eax)
-; X86-NEXT: movl %edx, 4(%eax)
-; X86-NEXT: movl %ecx, 8(%eax)
+; X86-NEXT: movl %ecx, 4(%eax)
+; X86-NEXT: movl %esi, 8(%eax)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: movl %ecx, 12(%eax)
; X86-NEXT: leal -12(%ebp), %esp
@@ -2891,19 +2758,6 @@ define i128 @div_by_67(i128 %x) nounwind {
; X86-NEXT: popl %ebx
; X86-NEXT: popl %ebp
; X86-NEXT: retl $4
-; X86-NEXT: .LBB9_9:
-; X86-NEXT: movb $1, %cl
-; X86-NEXT: jmp .LBB9_11
-; X86-NEXT: .LBB9_19:
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: jmp .LBB9_17
-; X86-NEXT: .LBB9_14:
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: jmp .LBB9_21
;
; X64-LABEL: div_by_67:
; X64: # %bb.0: # %entry
diff --git a/llvm/test/CodeGen/X86/icmp-pow2-diff.ll b/llvm/test/CodeGen/X86/icmp-pow2-diff.ll
index dada1726be424..88e238be5ba8b 100644
--- a/llvm/test/CodeGen/X86/icmp-pow2-diff.ll
+++ b/llvm/test/CodeGen/X86/icmp-pow2-diff.ll
@@ -151,7 +151,7 @@ define <8 x i1> @andnot_ne_v8i16_todo_no_splat(<8 x i16> %x) nounwind {
; AVX512-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1
; AVX512-NEXT: vpcmpeqw %xmm1, %xmm0, %xmm2
; AVX512-NEXT: vpcmpeqw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT: vpternlogq $54, %xmm2, %xmm1, %xmm0
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = xmm1 ^ (xmm0 | xmm2)
; AVX512-NEXT: retq
;
; AVX2-LABEL: andnot_ne_v8i16_todo_no_splat:
@@ -184,7 +184,7 @@ define <8 x i1> @andnot_ne_v8i16(<8 x i16> %x) nounwind {
; AVX512-NEXT: vpandnd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm0
; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512-NEXT: vpcmpeqw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vpternlogq $15, %xmm0, %xmm0, %xmm0
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~xmm0
; AVX512-NEXT: retq
;
; AVX2-LABEL: andnot_ne_v8i16:
@@ -217,7 +217,7 @@ define <16 x i1> @andnot_ne_v16i8_fail_max_not_n1(<16 x i8> %x) nounwind {
; AVX512-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1
; AVX512-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm2
; AVX512-NEXT: vpcmpgtb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT: vpternlogq $54, %xmm2, %xmm1, %xmm0
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = xmm1 ^ (xmm0 | xmm2)
; AVX512-NEXT: retq
;
; AVX2-LABEL: andnot_ne_v16i8_fail_max_not_n1:
@@ -250,7 +250,7 @@ define <16 x i1> @andnot_ne_v16i8(<16 x i8> %x) nounwind {
; AVX512-NEXT: vpandnd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm0
; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512-NEXT: vpcmpeqb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vpternlogq $15, %xmm0, %xmm0, %xmm0
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = ~xmm0
; AVX512-NEXT: retq
;
; AVX2-LABEL: andnot_ne_v16i8:
@@ -309,7 +309,7 @@ define <8 x i1> @addand_ne_v8i16_fail(<8 x i16> %x) nounwind {
; AVX512-NEXT: vpcmpeqw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1
; AVX512-NEXT: vpcmpeqd %xmm2, %xmm2, %xmm2
; AVX512-NEXT: vpcmpeqw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT: vpternlogq $86, %xmm2, %xmm1, %xmm0
+; AVX512-NEXT: vpternlogq {{.*#+}} xmm0 = xmm2 ^ (xmm0 | xmm1)
; AVX512-NEXT: retq
;
; AVX2-LABEL: addand_ne_v8i16_fail:
diff --git a/llvm/test/CodeGen/X86/icmp-pow2-logic-npow2.ll b/llvm/test/CodeGen/X86/icmp-pow2-logic-npow2.ll
index b14fedeaae57d..02078c3575dd6 100644
--- a/llvm/test/CodeGen/X86/icmp-pow2-logic-npow2.ll
+++ b/llvm/test/CodeGen/X86/icmp-pow2-logic-npow2.ll
@@ -196,8 +196,8 @@ define i1 @abs_ne_pow2(i64 %0) nounwind {
define i1 @abs_ne_nonpow2(i16 %0) nounwind {
; X86-LABEL: abs_ne_nonpow2:
; X86: # %bb.0:
-; X86-NEXT: movswl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movswl %ax, %ecx
; X86-NEXT: sarl $15, %ecx
; X86-NEXT: xorl %ecx, %eax
; X86-NEXT: subl %ecx, %eax
diff --git a/llvm/test/CodeGen/X86/isel-llvm.set.rounding.ll b/llvm/test/CodeGen/X86/isel-llvm.set.rounding.ll
index 859c6d1965b68..d271e97d8832a 100644
--- a/llvm/test/CodeGen/X86/isel-llvm.set.rounding.ll
+++ b/llvm/test/CodeGen/X86/isel-llvm.set.rounding.ll
@@ -117,7 +117,10 @@ define void @func_02() nounwind {
; GISEL-X86-NOSSE: # %bb.0:
; GISEL-X86-NOSSE-NEXT: pushl %eax
; GISEL-X86-NOSSE-NEXT: fnstcw (%esp)
-; GISEL-X86-NOSSE-NEXT: andw $-3073, (%esp) # imm = 0xF3FF
+; GISEL-X86-NOSSE-NEXT: movw $-3073, %ax # imm = 0xF3FF
+; GISEL-X86-NOSSE-NEXT: andw (%esp), %ax
+; GISEL-X86-NOSSE-NEXT: orw $0, %ax
+; GISEL-X86-NOSSE-NEXT: movw %ax, (%esp)
; GISEL-X86-NOSSE-NEXT: fldcw (%esp)
; GISEL-X86-NOSSE-NEXT: popl %eax
; GISEL-X86-NOSSE-NEXT: retl
@@ -132,7 +135,10 @@ define void @func_02() nounwind {
; GISEL-X64-NOSSE-LABEL: func_02:
; GISEL-X64-NOSSE: # %bb.0:
; GISEL-X64-NOSSE-NEXT: fnstcw -{{[0-9]+}}(%rsp)
-; GISEL-X64-NOSSE-NEXT: andw $-3073, -{{[0-9]+}}(%rsp) # imm = 0xF3FF
+; GISEL-X64-NOSSE-NEXT: movw $-3073, %ax # imm = 0xF3FF
+; GISEL-X64-NOSSE-NEXT: andw -{{[0-9]+}}(%rsp), %ax
+; GISEL-X64-NOSSE-NEXT: orw $0, %ax
+; GISEL-X64-NOSSE-NEXT: movw %ax, -{{[0-9]+}}(%rsp)
; GISEL-X64-NOSSE-NEXT: fldcw -{{[0-9]+}}(%rsp)
; GISEL-X64-NOSSE-NEXT: retq
;
@@ -149,7 +155,10 @@ define void @func_02() nounwind {
; GISEL-X86: # %bb.0:
; GISEL-X86-NEXT: pushl %eax
; GISEL-X86-NEXT: fnstcw (%esp)
-; GISEL-X86-NEXT: andw $-3073, (%esp) # imm = 0xF3FF
+; GISEL-X86-NEXT: movw $-3073, %ax # imm = 0xF3FF
+; GISEL-X86-NEXT: andw (%esp), %ax
+; GISEL-X86-NEXT: orw $0, %ax
+; GISEL-X86-NEXT: movw %ax, (%esp)
; GISEL-X86-NEXT: fldcw (%esp)
; GISEL-X86-NEXT: popl %eax
; GISEL-X86-NEXT: retl
@@ -167,10 +176,16 @@ define void @func_02() nounwind {
; GISEL-X64-LABEL: func_02:
; GISEL-X64: # %bb.0:
; GISEL-X64-NEXT: fnstcw -{{[0-9]+}}(%rsp)
-; GISEL-X64-NEXT: andw $-3073, -{{[0-9]+}}(%rsp) # imm = 0xF3FF
+; GISEL-X64-NEXT: movw $-3073, %ax # imm = 0xF3FF
+; GISEL-X64-NEXT: andw -{{[0-9]+}}(%rsp), %ax
+; GISEL-X64-NEXT: orw $0, %ax
+; GISEL-X64-NEXT: movw %ax, -{{[0-9]+}}(%rsp)
; GISEL-X64-NEXT: fldcw -{{[0-9]+}}(%rsp)
; GISEL-X64-NEXT: stmxcsr -{{[0-9]+}}(%rsp)
-; GISEL-X64-NEXT: andl $-24577, -{{[0-9]+}}(%rsp) # imm = 0x9FFF
+; GISEL-X64-NEXT: movl $-24577, %eax # imm = 0x9FFF
+; GISEL-X64-NEXT: andl -{{[0-9]+}}(%rsp), %eax
+; GISEL-X64-NEXT: orl $0, %eax
+; GISEL-X64-NEXT: movl %eax, -{{[0-9]+}}(%rsp)
; GISEL-X64-NEXT: ldmxcsr -{{[0-9]+}}(%rsp)
; GISEL-X64-NEXT: retq
call void @llvm.set.rounding(i32 1) ; ToNearestTiesToEven (CW[11-10] = 00)
diff --git a/llvm/test/CodeGen/X86/mov-lea-folds.ll b/llvm/test/CodeGen/X86/mov-lea-folds.ll
new file mode 100644
index 0000000000000..b9ac6076e8b89
--- /dev/null
+++ b/llvm/test/CodeGen/X86/mov-lea-folds.ll
@@ -0,0 +1,103 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx2,+slow-3ops-lea | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx2,-slow-3ops-lea | FileCheck %s
+
+define i64 @mov_dec(<32 x i8> %x) {
+; CHECK-LABEL: mov_dec:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovmskb %ymm0, %ecx
+; CHECK-NEXT: movl %ecx, %eax
+; CHECK-NEXT: decl %eax
+; CHECK-NEXT: shlq $32, %rcx
+; CHECK-NEXT: orq %rcx, %rax
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ %cmp = icmp slt <32 x i8> %x, zeroinitializer
+ %mvmsk = bitcast <32 x i1> %cmp to i32
+ %dec = add i32 %mvmsk, -1
+ %ext = zext i32 %mvmsk to i64
+ %shl = shl nuw i64 %ext, 32
+ %dec.ext = zext i32 %dec to i64
+ %res = or disjoint i64 %shl, %dec.ext
+ ret i64 %res
+}
+
+define i64 @mov_inc(<32 x i8> %x) {
+; CHECK-LABEL: mov_inc:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovmskb %ymm0, %ecx
+; CHECK-NEXT: movl %ecx, %eax
+; CHECK-NEXT: incl %eax
+; CHECK-NEXT: shlq $32, %rcx
+; CHECK-NEXT: orq %rcx, %rax
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ %cmp = icmp slt <32 x i8> %x, zeroinitializer
+ %mvmsk = bitcast <32 x i1> %cmp to i32
+ %inc = add i32 %mvmsk, 1
+ %ext = zext i32 %mvmsk to i64
+ %shl = shl nuw i64 %ext, 32
+ %inc.ext = zext i32 %inc to i64
+ %res = or disjoint i64 %shl, %inc.ext
+ ret i64 %res
+}
+
+define i64 @mov_add_imm(<32 x i8> %x) {
+; CHECK-LABEL: mov_add_imm:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovmskb %ymm0, %ecx
+; CHECK-NEXT: movl %ecx, %eax
+; CHECK-NEXT: addl $5, %eax
+; CHECK-NEXT: shlq $32, %rcx
+; CHECK-NEXT: orq %rcx, %rax
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ %cmp = icmp slt <32 x i8> %x, zeroinitializer
+ %mvmsk = bitcast <32 x i1> %cmp to i32
+ %add = add i32 %mvmsk, 5
+ %ext = zext i32 %mvmsk to i64
+ %shl = shl nuw i64 %ext, 32
+ %add.ext = zext i32 %add to i64
+ %res = or disjoint i64 %shl, %add.ext
+ ret i64 %res
+}
+
+define i64 @mov_add_rr(<32 x i8> %x, i32 %y) {
+; CHECK-LABEL: mov_add_rr:
+; CHECK: # %bb.0:
+; CHECK-NEXT: # kill: def $edi killed $edi def $rdi
+; CHECK-NEXT: vpmovmskb %ymm0, %ecx
+; CHECK-NEXT: leal (%rdi,%rcx), %eax
+; CHECK-NEXT: shlq $32, %rcx
+; CHECK-NEXT: orq %rcx, %rax
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ %cmp = icmp slt <32 x i8> %x, zeroinitializer
+ %mvmsk = bitcast <32 x i1> %cmp to i32
+ %add = add i32 %mvmsk, %y
+ %ext = zext i32 %mvmsk to i64
+ %shl = shl nuw i64 %ext, 32
+ %add.ext = zext i32 %add to i64
+ %res = or disjoint i64 %shl, %add.ext
+ ret i64 %res
+}
+
+define i64 @mov_shl(<32 x i8> %x) {
+; CHECK-LABEL: mov_shl:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovmskb %ymm0, %ecx
+; CHECK-NEXT: movl %ecx, %eax
+; CHECK-NEXT: shll $2, %eax
+; CHECK-NEXT: shlq $32, %rcx
+; CHECK-NEXT: orq %rcx, %rax
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ %cmp = icmp slt <32 x i8> %x, zeroinitializer
+ %mvmsk = bitcast <32 x i1> %cmp to i32
+ %shl_op = shl i32 %mvmsk, 2
+ %ext = zext i32 %mvmsk to i64
+ %shl = shl nuw i64 %ext, 32
+ %shl_op.ext = zext i32 %shl_op to i64
+ %res = or disjoint i64 %shl, %shl_op.ext
+ ret i64 %res
+}
diff --git a/llvm/test/CodeGen/X86/neg-abs.ll b/llvm/test/CodeGen/X86/neg-abs.ll
index 050610426e6bd..724b2dc4c431a 100644
--- a/llvm/test/CodeGen/X86/neg-abs.ll
+++ b/llvm/test/CodeGen/X86/neg-abs.ll
@@ -34,8 +34,8 @@ define i8 @neg_abs_i8(i8 %x) nounwind {
define i16 @neg_abs_i16(i16 %x) nounwind {
; X86-LABEL: neg_abs_i16:
; X86: # %bb.0:
-; X86-NEXT: movswl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movswl %cx, %eax
; X86-NEXT: sarl $15, %eax
; X86-NEXT: xorl %eax, %ecx
; X86-NEXT: subl %ecx, %eax
@@ -182,8 +182,8 @@ define i8 @sub_abs_i8(i8 %x, i8 %y) nounwind {
define i16 @sub_abs_i16(i16 %x, i16 %y) nounwind {
; X86-LABEL: sub_abs_i16:
; X86: # %bb.0:
-; X86-NEXT: movswl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movswl %cx, %eax
; X86-NEXT: sarl $15, %eax
; X86-NEXT: xorl %eax, %ecx
; X86-NEXT: subl %ecx, %eax
diff --git a/llvm/test/CodeGen/X86/sat-add.ll b/llvm/test/CodeGen/X86/sat-add.ll
index 69e6ff7770ebe..b12be7cb129d3 100644
--- a/llvm/test/CodeGen/X86/sat-add.ll
+++ b/llvm/test/CodeGen/X86/sat-add.ll
@@ -631,13 +631,12 @@ define <2 x i64> @unsigned_sat_constant_v2i64_using_min(<2 x i64> %x) {
; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456]
; SSE41-NEXT: pxor %xmm1, %xmm0
; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [9223372034707292117,9223372034707292117]
+; SSE41-NEXT: movdqa %xmm0, %xmm4
+; SSE41-NEXT: pcmpeqd %xmm3, %xmm4
; SSE41-NEXT: pcmpgtd %xmm0, %xmm3
-; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm3[0,0,2,2]
-; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
-; SSE41-NEXT: pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm5
-; SSE41-NEXT: pand %xmm4, %xmm5
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
-; SSE41-NEXT: por %xmm5, %xmm0
+; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
+; SSE41-NEXT: pand %xmm4, %xmm0
+; SSE41-NEXT: por %xmm3, %xmm0
; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm2
; SSE41-NEXT: paddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
; SSE41-NEXT: movdqa %xmm2, %xmm0
@@ -1164,12 +1163,10 @@ define <2 x i64> @unsigned_sat_variable_v2i64_using_min(<2 x i64> %x, <2 x i64>
; SSE41-NEXT: movdqa {{.*#+}} xmm4 = [9223372034707292159,9223372034707292159]
; SSE41-NEXT: pxor %xmm1, %xmm4
; SSE41-NEXT: movdqa %xmm4, %xmm5
-; SSE41-NEXT: pcmpgtd %xmm0, %xmm5
-; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE41-NEXT: pcmpeqd %xmm0, %xmm4
-; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE41-NEXT: pand %xmm6, %xmm4
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm5[1,1,3,3]
+; SSE41-NEXT: pcmpeqd %xmm0, %xmm5
+; SSE41-NEXT: pcmpgtd %xmm0, %xmm4
+; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT: pand %xmm5, %xmm0
; SSE41-NEXT: por %xmm4, %xmm0
; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm3
; SSE41-NEXT: paddq %xmm1, %xmm3
diff --git a/llvm/test/CodeGen/X86/setcc-logic.ll b/llvm/test/CodeGen/X86/setcc-logic.ll
index c577849b45ee8..3572d046bace7 100644
--- a/llvm/test/CodeGen/X86/setcc-logic.ll
+++ b/llvm/test/CodeGen/X86/setcc-logic.ll
@@ -628,9 +628,9 @@ define i1 @and_icmps_const_1bit_diff_common_op(i32 %x, i32 %y) {
define i1 @or_cmp_eq_i128(i128 %x, i128 %y) {
; NOBMI-LABEL: or_cmp_eq_i128:
; NOBMI: # %bb.0:
-; NOBMI-NEXT: notq %rdi
; NOBMI-NEXT: notq %rsi
; NOBMI-NEXT: andq %rcx, %rsi
+; NOBMI-NEXT: notq %rdi
; NOBMI-NEXT: andq %rdx, %rdi
; NOBMI-NEXT: orq %rsi, %rdi
; NOBMI-NEXT: sete %al
@@ -652,7 +652,7 @@ define i1 @or_cmp_eq_i64(i64 %x, i64 %y) {
; NOBMI-LABEL: or_cmp_eq_i64:
; NOBMI: # %bb.0:
; NOBMI-NEXT: notq %rdi
-; NOBMI-NEXT: testq %rdi, %rsi
+; NOBMI-NEXT: testq %rsi, %rdi
; NOBMI-NEXT: sete %al
; NOBMI-NEXT: retq
;
@@ -670,7 +670,7 @@ define i1 @or_cmp_ne_i32(i32 %x, i32 %y) {
; NOBMI-LABEL: or_cmp_ne_i32:
; NOBMI: # %bb.0:
; NOBMI-NEXT: notl %esi
-; NOBMI-NEXT: testl %esi, %edi
+; NOBMI-NEXT: testl %edi, %esi
; NOBMI-NEXT: setne %al
; NOBMI-NEXT: retq
;
@@ -688,7 +688,7 @@ define i1 @or_cmp_eq_i16(i16 zeroext %x, i16 zeroext %y) {
; NOBMI-LABEL: or_cmp_eq_i16:
; NOBMI: # %bb.0:
; NOBMI-NEXT: notl %edi
-; NOBMI-NEXT: testl %edi, %esi
+; NOBMI-NEXT: testl %esi, %edi
; NOBMI-NEXT: sete %al
; NOBMI-NEXT: retq
;
@@ -706,7 +706,7 @@ define i1 @or_cmp_ne_i8(i8 zeroext %x, i8 zeroext %y) {
; CHECK-LABEL: or_cmp_ne_i8:
; CHECK: # %bb.0:
; CHECK-NEXT: notb %sil
-; CHECK-NEXT: testb %sil, %dil
+; CHECK-NEXT: testb %dil, %sil
; CHECK-NEXT: setne %al
; CHECK-NEXT: retq
%o = or i8 %x, %y
diff --git a/llvm/test/CodeGen/X86/vector-idiv-udiv-128.ll b/llvm/test/CodeGen/X86/vector-idiv-udiv-128.ll
index 4a6cbaaa89e76..f006e7a35b271 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-udiv-128.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-udiv-128.ll
@@ -946,67 +946,33 @@ define <2 x i128> @v2i128_div_by_7(<2 x i128> %x) nounwind {
; SSE: # %bb.0: # %entry
; SSE-NEXT: pushq %r15
; SSE-NEXT: pushq %r14
+; SSE-NEXT: pushq %r13
+; SSE-NEXT: pushq %r12
; SSE-NEXT: pushq %rbx
-; SSE-NEXT: movq %rcx, %r9
-; SSE-NEXT: movq %rdx, %rcx
-; SSE-NEXT: movabsq $1152921504606846975, %r14 # imm = 0xFFFFFFFFFFFFFFF
-; SSE-NEXT: movq %rsi, %rax
-; SSE-NEXT: andq %r14, %rax
-; SSE-NEXT: movq %rsi, %rdx
-; SSE-NEXT: shrdq $60, %rcx, %rdx
-; SSE-NEXT: andq %r14, %rdx
-; SSE-NEXT: addq %rax, %rdx
-; SSE-NEXT: movq %rcx, %r10
-; SSE-NEXT: shrq $56, %r10
-; SSE-NEXT: addq %rdx, %r10
-; SSE-NEXT: movabsq $5270498306774157605, %r15 # imm = 0x4924924924924925
-; SSE-NEXT: movq %r10, %rax
-; SSE-NEXT: mulq %r15
-; SSE-NEXT: shrq %rdx
-; SSE-NEXT: leaq (,%rdx,8), %rax
-; SSE-NEXT: subq %rax, %rdx
-; SSE-NEXT: addq %r10, %rdx
-; SSE-NEXT: subq %rdx, %rsi
-; SSE-NEXT: sbbq $0, %rcx
-; SSE-NEXT: movabsq $-5270498306774157605, %r11 # imm = 0xB6DB6DB6DB6DB6DB
-; SSE-NEXT: movq %rsi, %r10
-; SSE-NEXT: imulq %r11, %r10
-; SSE-NEXT: movabsq $7905747460161236407, %rbx # imm = 0x6DB6DB6DB6DB6DB7
-; SSE-NEXT: movq %rsi, %rax
-; SSE-NEXT: mulq %rbx
+; SSE-NEXT: movq %r8, %rbx
+; SSE-NEXT: movq %rcx, %r14
+; SSE-NEXT: movq %rdx, %rax
+; SSE-NEXT: movq %rdi, %r15
+; SSE-NEXT: movl $7, %edx
+; SSE-NEXT: movq %rsi, %rdi
; SSE-NEXT: movq %rax, %rsi
-; SSE-NEXT: addq %r10, %rdx
-; SSE-NEXT: imulq %rbx, %rcx
-; SSE-NEXT: addq %rdx, %rcx
-; SSE-NEXT: movq %r9, %rax
-; SSE-NEXT: andq %r14, %rax
-; SSE-NEXT: movq %r9, %rdx
-; SSE-NEXT: shrdq $60, %r8, %rdx
-; SSE-NEXT: andq %r14, %rdx
-; SSE-NEXT: addq %rax, %rdx
-; SSE-NEXT: movq %r8, %r10
-; SSE-NEXT: shrq $56, %r10
-; SSE-NEXT: addq %rdx, %r10
-; SSE-NEXT: movq %r10, %rax
-; SSE-NEXT: mulq %r15
-; SSE-NEXT: shrq %rdx
-; SSE-NEXT: leaq (,%rdx,8), %rax
-; SSE-NEXT: subq %rax, %rdx
-; SSE-NEXT: addq %r10, %rdx
-; SSE-NEXT: subq %rdx, %r9
-; SSE-NEXT: sbbq $0, %r8
-; SSE-NEXT: imulq %r9, %r11
-; SSE-NEXT: movq %r9, %rax
-; SSE-NEXT: mulq %rbx
-; SSE-NEXT: addq %r11, %rdx
-; SSE-NEXT: imulq %rbx, %r8
-; SSE-NEXT: addq %rdx, %r8
-; SSE-NEXT: movq %rax, 16(%rdi)
-; SSE-NEXT: movq %rsi, (%rdi)
-; SSE-NEXT: movq %r8, 24(%rdi)
-; SSE-NEXT: movq %rcx, 8(%rdi)
-; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: xorl %ecx, %ecx
+; SSE-NEXT: callq __udivti3 at PLT
+; SSE-NEXT: movq %rax, %r12
+; SSE-NEXT: movq %rdx, %r13
+; SSE-NEXT: movl $7, %edx
+; SSE-NEXT: movq %r14, %rdi
+; SSE-NEXT: movq %rbx, %rsi
+; SSE-NEXT: xorl %ecx, %ecx
+; SSE-NEXT: callq __udivti3 at PLT
+; SSE-NEXT: movq %rdx, 24(%r15)
+; SSE-NEXT: movq %rax, 16(%r15)
+; SSE-NEXT: movq %r13, 8(%r15)
+; SSE-NEXT: movq %r12, (%r15)
+; SSE-NEXT: movq %r15, %rax
; SSE-NEXT: popq %rbx
+; SSE-NEXT: popq %r12
+; SSE-NEXT: popq %r13
; SSE-NEXT: popq %r14
; SSE-NEXT: popq %r15
; SSE-NEXT: retq
@@ -1015,67 +981,33 @@ define <2 x i128> @v2i128_div_by_7(<2 x i128> %x) nounwind {
; AVX: # %bb.0: # %entry
; AVX-NEXT: pushq %r15
; AVX-NEXT: pushq %r14
+; AVX-NEXT: pushq %r13
+; AVX-NEXT: pushq %r12
; AVX-NEXT: pushq %rbx
-; AVX-NEXT: movq %rcx, %r9
-; AVX-NEXT: movq %rdx, %rcx
-; AVX-NEXT: movabsq $1152921504606846975, %r14 # imm = 0xFFFFFFFFFFFFFFF
-; AVX-NEXT: movq %rsi, %rax
-; AVX-NEXT: andq %r14, %rax
-; AVX-NEXT: movq %rsi, %rdx
-; AVX-NEXT: shrdq $60, %rcx, %rdx
-; AVX-NEXT: andq %r14, %rdx
-; AVX-NEXT: addq %rax, %rdx
-; AVX-NEXT: movq %rcx, %r10
-; AVX-NEXT: shrq $56, %r10
-; AVX-NEXT: addq %rdx, %r10
-; AVX-NEXT: movabsq $5270498306774157605, %r15 # imm = 0x4924924924924925
-; AVX-NEXT: movq %r10, %rax
-; AVX-NEXT: mulq %r15
-; AVX-NEXT: shrq %rdx
-; AVX-NEXT: leaq (,%rdx,8), %rax
-; AVX-NEXT: subq %rax, %rdx
-; AVX-NEXT: addq %r10, %rdx
-; AVX-NEXT: subq %rdx, %rsi
-; AVX-NEXT: sbbq $0, %rcx
-; AVX-NEXT: movabsq $-5270498306774157605, %r11 # imm = 0xB6DB6DB6DB6DB6DB
-; AVX-NEXT: movq %rsi, %r10
-; AVX-NEXT: imulq %r11, %r10
-; AVX-NEXT: movabsq $7905747460161236407, %rbx # imm = 0x6DB6DB6DB6DB6DB7
-; AVX-NEXT: movq %rsi, %rax
-; AVX-NEXT: mulq %rbx
+; AVX-NEXT: movq %r8, %rbx
+; AVX-NEXT: movq %rcx, %r14
+; AVX-NEXT: movq %rdx, %rax
+; AVX-NEXT: movq %rdi, %r15
+; AVX-NEXT: movl $7, %edx
+; AVX-NEXT: movq %rsi, %rdi
; AVX-NEXT: movq %rax, %rsi
-; AVX-NEXT: addq %r10, %rdx
-; AVX-NEXT: imulq %rbx, %rcx
-; AVX-NEXT: addq %rdx, %rcx
-; AVX-NEXT: movq %r9, %rax
-; AVX-NEXT: andq %r14, %rax
-; AVX-NEXT: movq %r9, %rdx
-; AVX-NEXT: shrdq $60, %r8, %rdx
-; AVX-NEXT: andq %r14, %rdx
-; AVX-NEXT: addq %rax, %rdx
-; AVX-NEXT: movq %r8, %r10
-; AVX-NEXT: shrq $56, %r10
-; AVX-NEXT: addq %rdx, %r10
-; AVX-NEXT: movq %r10, %rax
-; AVX-NEXT: mulq %r15
-; AVX-NEXT: shrq %rdx
-; AVX-NEXT: leaq (,%rdx,8), %rax
-; AVX-NEXT: subq %rax, %rdx
-; AVX-NEXT: addq %r10, %rdx
-; AVX-NEXT: subq %rdx, %r9
-; AVX-NEXT: sbbq $0, %r8
-; AVX-NEXT: imulq %r9, %r11
-; AVX-NEXT: movq %r9, %rax
-; AVX-NEXT: mulq %rbx
-; AVX-NEXT: addq %r11, %rdx
-; AVX-NEXT: imulq %rbx, %r8
-; AVX-NEXT: addq %rdx, %r8
-; AVX-NEXT: movq %rax, 16(%rdi)
-; AVX-NEXT: movq %rsi, (%rdi)
-; AVX-NEXT: movq %r8, 24(%rdi)
-; AVX-NEXT: movq %rcx, 8(%rdi)
-; AVX-NEXT: movq %rdi, %rax
+; AVX-NEXT: xorl %ecx, %ecx
+; AVX-NEXT: callq __udivti3 at PLT
+; AVX-NEXT: movq %rax, %r12
+; AVX-NEXT: movq %rdx, %r13
+; AVX-NEXT: movl $7, %edx
+; AVX-NEXT: movq %r14, %rdi
+; AVX-NEXT: movq %rbx, %rsi
+; AVX-NEXT: xorl %ecx, %ecx
+; AVX-NEXT: callq __udivti3 at PLT
+; AVX-NEXT: movq %rdx, 24(%r15)
+; AVX-NEXT: movq %rax, 16(%r15)
+; AVX-NEXT: movq %r13, 8(%r15)
+; AVX-NEXT: movq %r12, (%r15)
+; AVX-NEXT: movq %r15, %rax
; AVX-NEXT: popq %rbx
+; AVX-NEXT: popq %r12
+; AVX-NEXT: popq %r13
; AVX-NEXT: popq %r14
; AVX-NEXT: popq %r15
; AVX-NEXT: retq
@@ -1089,70 +1021,33 @@ define <2 x i128> @v2i128_div_by_14(<2 x i128> %x) nounwind {
; SSE: # %bb.0: # %entry
; SSE-NEXT: pushq %r15
; SSE-NEXT: pushq %r14
+; SSE-NEXT: pushq %r13
+; SSE-NEXT: pushq %r12
; SSE-NEXT: pushq %rbx
-; SSE-NEXT: movq %rcx, %r9
-; SSE-NEXT: movq %rdx, %rcx
-; SSE-NEXT: movq %rsi, %rax
-; SSE-NEXT: shrdq $61, %rdx, %rax
-; SSE-NEXT: movabsq $1152921504606846975, %r11 # imm = 0xFFFFFFFFFFFFFFF
-; SSE-NEXT: andq %r11, %rax
-; SSE-NEXT: shrdq $1, %rdx, %rsi
-; SSE-NEXT: movq %rsi, %rdx
-; SSE-NEXT: andq %r11, %rdx
-; SSE-NEXT: addq %rax, %rdx
-; SSE-NEXT: movq %rcx, %r10
-; SSE-NEXT: shrq $57, %r10
-; SSE-NEXT: addq %rdx, %r10
-; SSE-NEXT: movabsq $5270498306774157605, %r15 # imm = 0x4924924924924925
-; SSE-NEXT: movq %r10, %rax
-; SSE-NEXT: mulq %r15
-; SSE-NEXT: shrq %rdx
-; SSE-NEXT: leaq (,%rdx,8), %rax
-; SSE-NEXT: subq %rax, %rdx
-; SSE-NEXT: addq %r10, %rdx
-; SSE-NEXT: shrq %rcx
-; SSE-NEXT: subq %rdx, %rsi
-; SSE-NEXT: sbbq $0, %rcx
-; SSE-NEXT: movabsq $-5270498306774157605, %rbx # imm = 0xB6DB6DB6DB6DB6DB
-; SSE-NEXT: movq %rsi, %r10
-; SSE-NEXT: imulq %rbx, %r10
-; SSE-NEXT: movabsq $7905747460161236407, %r14 # imm = 0x6DB6DB6DB6DB6DB7
-; SSE-NEXT: movq %rsi, %rax
-; SSE-NEXT: mulq %r14
+; SSE-NEXT: movq %r8, %rbx
+; SSE-NEXT: movq %rcx, %r14
+; SSE-NEXT: movq %rdx, %rax
+; SSE-NEXT: movq %rdi, %r15
+; SSE-NEXT: movl $14, %edx
+; SSE-NEXT: movq %rsi, %rdi
; SSE-NEXT: movq %rax, %rsi
-; SSE-NEXT: addq %r10, %rdx
-; SSE-NEXT: imulq %r14, %rcx
-; SSE-NEXT: addq %rdx, %rcx
-; SSE-NEXT: movq %r9, %rax
-; SSE-NEXT: shrdq $61, %r8, %rax
-; SSE-NEXT: andq %r11, %rax
-; SSE-NEXT: shrdq $1, %r8, %r9
-; SSE-NEXT: andq %r9, %r11
-; SSE-NEXT: addq %rax, %r11
-; SSE-NEXT: movq %r8, %r10
-; SSE-NEXT: shrq $57, %r10
-; SSE-NEXT: addq %r11, %r10
-; SSE-NEXT: movq %r10, %rax
-; SSE-NEXT: mulq %r15
-; SSE-NEXT: shrq %rdx
-; SSE-NEXT: leaq (,%rdx,8), %rax
-; SSE-NEXT: subq %rax, %rdx
-; SSE-NEXT: addq %r10, %rdx
-; SSE-NEXT: shrq %r8
-; SSE-NEXT: subq %rdx, %r9
-; SSE-NEXT: sbbq $0, %r8
-; SSE-NEXT: imulq %r9, %rbx
-; SSE-NEXT: movq %r9, %rax
-; SSE-NEXT: mulq %r14
-; SSE-NEXT: addq %rbx, %rdx
-; SSE-NEXT: imulq %r14, %r8
-; SSE-NEXT: addq %rdx, %r8
-; SSE-NEXT: movq %rax, 16(%rdi)
-; SSE-NEXT: movq %rsi, (%rdi)
-; SSE-NEXT: movq %r8, 24(%rdi)
-; SSE-NEXT: movq %rcx, 8(%rdi)
-; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: xorl %ecx, %ecx
+; SSE-NEXT: callq __udivti3 at PLT
+; SSE-NEXT: movq %rax, %r12
+; SSE-NEXT: movq %rdx, %r13
+; SSE-NEXT: movl $14, %edx
+; SSE-NEXT: movq %r14, %rdi
+; SSE-NEXT: movq %rbx, %rsi
+; SSE-NEXT: xorl %ecx, %ecx
+; SSE-NEXT: callq __udivti3 at PLT
+; SSE-NEXT: movq %rdx, 24(%r15)
+; SSE-NEXT: movq %rax, 16(%r15)
+; SSE-NEXT: movq %r13, 8(%r15)
+; SSE-NEXT: movq %r12, (%r15)
+; SSE-NEXT: movq %r15, %rax
; SSE-NEXT: popq %rbx
+; SSE-NEXT: popq %r12
+; SSE-NEXT: popq %r13
; SSE-NEXT: popq %r14
; SSE-NEXT: popq %r15
; SSE-NEXT: retq
@@ -1161,70 +1056,33 @@ define <2 x i128> @v2i128_div_by_14(<2 x i128> %x) nounwind {
; AVX: # %bb.0: # %entry
; AVX-NEXT: pushq %r15
; AVX-NEXT: pushq %r14
+; AVX-NEXT: pushq %r13
+; AVX-NEXT: pushq %r12
; AVX-NEXT: pushq %rbx
-; AVX-NEXT: movq %rcx, %r9
-; AVX-NEXT: movq %rdx, %rcx
-; AVX-NEXT: movq %rsi, %rax
-; AVX-NEXT: shrdq $61, %rdx, %rax
-; AVX-NEXT: movabsq $1152921504606846975, %r11 # imm = 0xFFFFFFFFFFFFFFF
-; AVX-NEXT: andq %r11, %rax
-; AVX-NEXT: shrdq $1, %rdx, %rsi
-; AVX-NEXT: movq %rsi, %rdx
-; AVX-NEXT: andq %r11, %rdx
-; AVX-NEXT: addq %rax, %rdx
-; AVX-NEXT: movq %rcx, %r10
-; AVX-NEXT: shrq $57, %r10
-; AVX-NEXT: addq %rdx, %r10
-; AVX-NEXT: movabsq $5270498306774157605, %r15 # imm = 0x4924924924924925
-; AVX-NEXT: movq %r10, %rax
-; AVX-NEXT: mulq %r15
-; AVX-NEXT: shrq %rdx
-; AVX-NEXT: leaq (,%rdx,8), %rax
-; AVX-NEXT: subq %rax, %rdx
-; AVX-NEXT: addq %r10, %rdx
-; AVX-NEXT: shrq %rcx
-; AVX-NEXT: subq %rdx, %rsi
-; AVX-NEXT: sbbq $0, %rcx
-; AVX-NEXT: movabsq $-5270498306774157605, %rbx # imm = 0xB6DB6DB6DB6DB6DB
-; AVX-NEXT: movq %rsi, %r10
-; AVX-NEXT: imulq %rbx, %r10
-; AVX-NEXT: movabsq $7905747460161236407, %r14 # imm = 0x6DB6DB6DB6DB6DB7
-; AVX-NEXT: movq %rsi, %rax
-; AVX-NEXT: mulq %r14
+; AVX-NEXT: movq %r8, %rbx
+; AVX-NEXT: movq %rcx, %r14
+; AVX-NEXT: movq %rdx, %rax
+; AVX-NEXT: movq %rdi, %r15
+; AVX-NEXT: movl $14, %edx
+; AVX-NEXT: movq %rsi, %rdi
; AVX-NEXT: movq %rax, %rsi
-; AVX-NEXT: addq %r10, %rdx
-; AVX-NEXT: imulq %r14, %rcx
-; AVX-NEXT: addq %rdx, %rcx
-; AVX-NEXT: movq %r9, %rax
-; AVX-NEXT: shrdq $61, %r8, %rax
-; AVX-NEXT: andq %r11, %rax
-; AVX-NEXT: shrdq $1, %r8, %r9
-; AVX-NEXT: andq %r9, %r11
-; AVX-NEXT: addq %rax, %r11
-; AVX-NEXT: movq %r8, %r10
-; AVX-NEXT: shrq $57, %r10
-; AVX-NEXT: addq %r11, %r10
-; AVX-NEXT: movq %r10, %rax
-; AVX-NEXT: mulq %r15
-; AVX-NEXT: shrq %rdx
-; AVX-NEXT: leaq (,%rdx,8), %rax
-; AVX-NEXT: subq %rax, %rdx
-; AVX-NEXT: addq %r10, %rdx
-; AVX-NEXT: shrq %r8
-; AVX-NEXT: subq %rdx, %r9
-; AVX-NEXT: sbbq $0, %r8
-; AVX-NEXT: imulq %r9, %rbx
-; AVX-NEXT: movq %r9, %rax
-; AVX-NEXT: mulq %r14
-; AVX-NEXT: addq %rbx, %rdx
-; AVX-NEXT: imulq %r14, %r8
-; AVX-NEXT: addq %rdx, %r8
-; AVX-NEXT: movq %rax, 16(%rdi)
-; AVX-NEXT: movq %rsi, (%rdi)
-; AVX-NEXT: movq %r8, 24(%rdi)
-; AVX-NEXT: movq %rcx, 8(%rdi)
-; AVX-NEXT: movq %rdi, %rax
+; AVX-NEXT: xorl %ecx, %ecx
+; AVX-NEXT: callq __udivti3 at PLT
+; AVX-NEXT: movq %rax, %r12
+; AVX-NEXT: movq %rdx, %r13
+; AVX-NEXT: movl $14, %edx
+; AVX-NEXT: movq %r14, %rdi
+; AVX-NEXT: movq %rbx, %rsi
+; AVX-NEXT: xorl %ecx, %ecx
+; AVX-NEXT: callq __udivti3 at PLT
+; AVX-NEXT: movq %rdx, 24(%r15)
+; AVX-NEXT: movq %rax, 16(%r15)
+; AVX-NEXT: movq %r13, 8(%r15)
+; AVX-NEXT: movq %r12, (%r15)
+; AVX-NEXT: movq %r15, %rax
; AVX-NEXT: popq %rbx
+; AVX-NEXT: popq %r12
+; AVX-NEXT: popq %r13
; AVX-NEXT: popq %r14
; AVX-NEXT: popq %r15
; AVX-NEXT: retq
@@ -1236,80 +1094,72 @@ entry:
define <2 x i128> @v2i128_rem_by_7(<2 x i128> %x) nounwind {
; SSE-LABEL: v2i128_rem_by_7:
; SSE: # %bb.0: # %entry
-; SSE-NEXT: movq %rdx, %r9
-; SSE-NEXT: movabsq $1152921504606846975, %r10 # imm = 0xFFFFFFFFFFFFFFF
-; SSE-NEXT: movq %rsi, %rax
-; SSE-NEXT: andq %r10, %rax
-; SSE-NEXT: shrdq $60, %rdx, %rsi
-; SSE-NEXT: andq %r10, %rsi
-; SSE-NEXT: addq %rax, %rsi
-; SSE-NEXT: shrq $56, %r9
-; SSE-NEXT: addq %rsi, %r9
-; SSE-NEXT: movabsq $5270498306774157605, %r11 # imm = 0x4924924924924925
-; SSE-NEXT: movq %r9, %rax
-; SSE-NEXT: mulq %r11
-; SSE-NEXT: movq %rdx, %rsi
-; SSE-NEXT: shrq %rsi
-; SSE-NEXT: leaq (,%rsi,8), %rax
-; SSE-NEXT: subq %rax, %rsi
-; SSE-NEXT: addq %r9, %rsi
-; SSE-NEXT: movq %rcx, %rax
-; SSE-NEXT: andq %r10, %rax
-; SSE-NEXT: shrdq $60, %r8, %rcx
-; SSE-NEXT: andq %r10, %rcx
-; SSE-NEXT: addq %rax, %rcx
-; SSE-NEXT: shrq $56, %r8
-; SSE-NEXT: addq %rcx, %r8
-; SSE-NEXT: movq %r8, %rax
-; SSE-NEXT: mulq %r11
-; SSE-NEXT: shrq %rdx
-; SSE-NEXT: leaq (,%rdx,8), %rax
-; SSE-NEXT: subq %rax, %rdx
-; SSE-NEXT: addq %r8, %rdx
-; SSE-NEXT: movq %rdx, 16(%rdi)
-; SSE-NEXT: movq %rsi, (%rdi)
-; SSE-NEXT: movq $0, 24(%rdi)
-; SSE-NEXT: movq $0, 8(%rdi)
-; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: pushq %r15
+; SSE-NEXT: pushq %r14
+; SSE-NEXT: pushq %r13
+; SSE-NEXT: pushq %r12
+; SSE-NEXT: pushq %rbx
+; SSE-NEXT: movq %r8, %rbx
+; SSE-NEXT: movq %rcx, %r14
+; SSE-NEXT: movq %rdx, %rax
+; SSE-NEXT: movq %rdi, %r15
+; SSE-NEXT: movl $7, %edx
+; SSE-NEXT: movq %rsi, %rdi
+; SSE-NEXT: movq %rax, %rsi
+; SSE-NEXT: xorl %ecx, %ecx
+; SSE-NEXT: callq __umodti3 at PLT
+; SSE-NEXT: movq %rax, %r12
+; SSE-NEXT: movq %rdx, %r13
+; SSE-NEXT: movl $7, %edx
+; SSE-NEXT: movq %r14, %rdi
+; SSE-NEXT: movq %rbx, %rsi
+; SSE-NEXT: xorl %ecx, %ecx
+; SSE-NEXT: callq __umodti3 at PLT
+; SSE-NEXT: movq %rdx, 24(%r15)
+; SSE-NEXT: movq %rax, 16(%r15)
+; SSE-NEXT: movq %r13, 8(%r15)
+; SSE-NEXT: movq %r12, (%r15)
+; SSE-NEXT: movq %r15, %rax
+; SSE-NEXT: popq %rbx
+; SSE-NEXT: popq %r12
+; SSE-NEXT: popq %r13
+; SSE-NEXT: popq %r14
+; SSE-NEXT: popq %r15
; SSE-NEXT: retq
;
; AVX-LABEL: v2i128_rem_by_7:
; AVX: # %bb.0: # %entry
-; AVX-NEXT: movq %rdx, %r9
-; AVX-NEXT: movabsq $1152921504606846975, %r10 # imm = 0xFFFFFFFFFFFFFFF
-; AVX-NEXT: movq %rsi, %rax
-; AVX-NEXT: andq %r10, %rax
-; AVX-NEXT: shrdq $60, %rdx, %rsi
-; AVX-NEXT: andq %r10, %rsi
-; AVX-NEXT: addq %rax, %rsi
-; AVX-NEXT: shrq $56, %r9
-; AVX-NEXT: addq %rsi, %r9
-; AVX-NEXT: movabsq $5270498306774157605, %r11 # imm = 0x4924924924924925
-; AVX-NEXT: movq %r9, %rax
-; AVX-NEXT: mulq %r11
-; AVX-NEXT: movq %rdx, %rsi
-; AVX-NEXT: shrq %rsi
-; AVX-NEXT: leaq (,%rsi,8), %rax
-; AVX-NEXT: subq %rax, %rsi
-; AVX-NEXT: addq %r9, %rsi
-; AVX-NEXT: movq %rcx, %rax
-; AVX-NEXT: andq %r10, %rax
-; AVX-NEXT: shrdq $60, %r8, %rcx
-; AVX-NEXT: andq %r10, %rcx
-; AVX-NEXT: addq %rax, %rcx
-; AVX-NEXT: shrq $56, %r8
-; AVX-NEXT: addq %rcx, %r8
-; AVX-NEXT: movq %r8, %rax
-; AVX-NEXT: mulq %r11
-; AVX-NEXT: shrq %rdx
-; AVX-NEXT: leaq (,%rdx,8), %rax
-; AVX-NEXT: subq %rax, %rdx
-; AVX-NEXT: addq %r8, %rdx
-; AVX-NEXT: movq %rdx, 16(%rdi)
-; AVX-NEXT: movq %rsi, (%rdi)
-; AVX-NEXT: movq $0, 24(%rdi)
-; AVX-NEXT: movq $0, 8(%rdi)
-; AVX-NEXT: movq %rdi, %rax
+; AVX-NEXT: pushq %r15
+; AVX-NEXT: pushq %r14
+; AVX-NEXT: pushq %r13
+; AVX-NEXT: pushq %r12
+; AVX-NEXT: pushq %rbx
+; AVX-NEXT: movq %r8, %rbx
+; AVX-NEXT: movq %rcx, %r14
+; AVX-NEXT: movq %rdx, %rax
+; AVX-NEXT: movq %rdi, %r15
+; AVX-NEXT: movl $7, %edx
+; AVX-NEXT: movq %rsi, %rdi
+; AVX-NEXT: movq %rax, %rsi
+; AVX-NEXT: xorl %ecx, %ecx
+; AVX-NEXT: callq __umodti3 at PLT
+; AVX-NEXT: movq %rax, %r12
+; AVX-NEXT: movq %rdx, %r13
+; AVX-NEXT: movl $7, %edx
+; AVX-NEXT: movq %r14, %rdi
+; AVX-NEXT: movq %rbx, %rsi
+; AVX-NEXT: xorl %ecx, %ecx
+; AVX-NEXT: callq __umodti3 at PLT
+; AVX-NEXT: movq %rdx, 24(%r15)
+; AVX-NEXT: movq %rax, 16(%r15)
+; AVX-NEXT: movq %r13, 8(%r15)
+; AVX-NEXT: movq %r12, (%r15)
+; AVX-NEXT: movq %r15, %rax
+; AVX-NEXT: popq %rbx
+; AVX-NEXT: popq %r12
+; AVX-NEXT: popq %r13
+; AVX-NEXT: popq %r14
+; AVX-NEXT: popq %r15
; AVX-NEXT: retq
entry:
%rem = urem <2 x i128> %x, <i128 7, i128 7>
@@ -1319,94 +1169,72 @@ entry:
define <2 x i128> @v2i128_rem_by_14(<2 x i128> %x) nounwind {
; SSE-LABEL: v2i128_rem_by_14:
; SSE: # %bb.0: # %entry
-; SSE-NEXT: movq %rdx, %r9
-; SSE-NEXT: movq %rsi, %rax
-; SSE-NEXT: shrdq $61, %rdx, %rax
-; SSE-NEXT: movabsq $1152921504606846975, %r10 # imm = 0xFFFFFFFFFFFFFFF
-; SSE-NEXT: andq %r10, %rax
-; SSE-NEXT: movq %rsi, %rdx
-; SSE-NEXT: shrq %rdx
-; SSE-NEXT: andq %r10, %rdx
-; SSE-NEXT: addq %rax, %rdx
-; SSE-NEXT: shrq $57, %r9
-; SSE-NEXT: addq %rdx, %r9
-; SSE-NEXT: movabsq $5270498306774157605, %r11 # imm = 0x4924924924924925
-; SSE-NEXT: movq %r9, %rax
-; SSE-NEXT: mulq %r11
-; SSE-NEXT: shrq %rdx
-; SSE-NEXT: leal (,%rdx,8), %eax
-; SSE-NEXT: subl %eax, %edx
-; SSE-NEXT: addl %edx, %r9d
-; SSE-NEXT: andl $1, %esi
-; SSE-NEXT: leaq (%rsi,%r9,2), %rsi
-; SSE-NEXT: movq %rcx, %rax
-; SSE-NEXT: shrdq $61, %r8, %rax
-; SSE-NEXT: andq %r10, %rax
-; SSE-NEXT: movq %rcx, %rdx
-; SSE-NEXT: shrq %rdx
-; SSE-NEXT: andq %r10, %rdx
-; SSE-NEXT: addq %rax, %rdx
-; SSE-NEXT: shrq $57, %r8
-; SSE-NEXT: addq %rdx, %r8
-; SSE-NEXT: movq %r8, %rax
-; SSE-NEXT: mulq %r11
-; SSE-NEXT: shrq %rdx
-; SSE-NEXT: leal (,%rdx,8), %eax
-; SSE-NEXT: subl %eax, %edx
-; SSE-NEXT: addl %edx, %r8d
-; SSE-NEXT: andl $1, %ecx
-; SSE-NEXT: leaq (%rcx,%r8,2), %rax
-; SSE-NEXT: movq %rax, 16(%rdi)
-; SSE-NEXT: movq %rsi, (%rdi)
-; SSE-NEXT: movq $0, 24(%rdi)
-; SSE-NEXT: movq $0, 8(%rdi)
-; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: pushq %r15
+; SSE-NEXT: pushq %r14
+; SSE-NEXT: pushq %r13
+; SSE-NEXT: pushq %r12
+; SSE-NEXT: pushq %rbx
+; SSE-NEXT: movq %r8, %rbx
+; SSE-NEXT: movq %rcx, %r14
+; SSE-NEXT: movq %rdx, %rax
+; SSE-NEXT: movq %rdi, %r15
+; SSE-NEXT: movl $14, %edx
+; SSE-NEXT: movq %rsi, %rdi
+; SSE-NEXT: movq %rax, %rsi
+; SSE-NEXT: xorl %ecx, %ecx
+; SSE-NEXT: callq __umodti3 at PLT
+; SSE-NEXT: movq %rax, %r12
+; SSE-NEXT: movq %rdx, %r13
+; SSE-NEXT: movl $14, %edx
+; SSE-NEXT: movq %r14, %rdi
+; SSE-NEXT: movq %rbx, %rsi
+; SSE-NEXT: xorl %ecx, %ecx
+; SSE-NEXT: callq __umodti3 at PLT
+; SSE-NEXT: movq %rdx, 24(%r15)
+; SSE-NEXT: movq %rax, 16(%r15)
+; SSE-NEXT: movq %r13, 8(%r15)
+; SSE-NEXT: movq %r12, (%r15)
+; SSE-NEXT: movq %r15, %rax
+; SSE-NEXT: popq %rbx
+; SSE-NEXT: popq %r12
+; SSE-NEXT: popq %r13
+; SSE-NEXT: popq %r14
+; SSE-NEXT: popq %r15
; SSE-NEXT: retq
;
; AVX-LABEL: v2i128_rem_by_14:
; AVX: # %bb.0: # %entry
-; AVX-NEXT: movq %rdx, %r9
-; AVX-NEXT: movq %rsi, %rax
-; AVX-NEXT: shrdq $61, %rdx, %rax
-; AVX-NEXT: movabsq $1152921504606846975, %r10 # imm = 0xFFFFFFFFFFFFFFF
-; AVX-NEXT: andq %r10, %rax
-; AVX-NEXT: movq %rsi, %rdx
-; AVX-NEXT: shrq %rdx
-; AVX-NEXT: andq %r10, %rdx
-; AVX-NEXT: addq %rax, %rdx
-; AVX-NEXT: shrq $57, %r9
-; AVX-NEXT: addq %rdx, %r9
-; AVX-NEXT: movabsq $5270498306774157605, %r11 # imm = 0x4924924924924925
-; AVX-NEXT: movq %r9, %rax
-; AVX-NEXT: mulq %r11
-; AVX-NEXT: shrq %rdx
-; AVX-NEXT: leal (,%rdx,8), %eax
-; AVX-NEXT: subl %eax, %edx
-; AVX-NEXT: addl %edx, %r9d
-; AVX-NEXT: andl $1, %esi
-; AVX-NEXT: leaq (%rsi,%r9,2), %rsi
-; AVX-NEXT: movq %rcx, %rax
-; AVX-NEXT: shrdq $61, %r8, %rax
-; AVX-NEXT: andq %r10, %rax
-; AVX-NEXT: movq %rcx, %rdx
-; AVX-NEXT: shrq %rdx
-; AVX-NEXT: andq %r10, %rdx
-; AVX-NEXT: addq %rax, %rdx
-; AVX-NEXT: shrq $57, %r8
-; AVX-NEXT: addq %rdx, %r8
-; AVX-NEXT: movq %r8, %rax
-; AVX-NEXT: mulq %r11
-; AVX-NEXT: shrq %rdx
-; AVX-NEXT: leal (,%rdx,8), %eax
-; AVX-NEXT: subl %eax, %edx
-; AVX-NEXT: addl %edx, %r8d
-; AVX-NEXT: andl $1, %ecx
-; AVX-NEXT: leaq (%rcx,%r8,2), %rax
-; AVX-NEXT: movq %rax, 16(%rdi)
-; AVX-NEXT: movq %rsi, (%rdi)
-; AVX-NEXT: movq $0, 24(%rdi)
-; AVX-NEXT: movq $0, 8(%rdi)
-; AVX-NEXT: movq %rdi, %rax
+; AVX-NEXT: pushq %r15
+; AVX-NEXT: pushq %r14
+; AVX-NEXT: pushq %r13
+; AVX-NEXT: pushq %r12
+; AVX-NEXT: pushq %rbx
+; AVX-NEXT: movq %r8, %rbx
+; AVX-NEXT: movq %rcx, %r14
+; AVX-NEXT: movq %rdx, %rax
+; AVX-NEXT: movq %rdi, %r15
+; AVX-NEXT: movl $14, %edx
+; AVX-NEXT: movq %rsi, %rdi
+; AVX-NEXT: movq %rax, %rsi
+; AVX-NEXT: xorl %ecx, %ecx
+; AVX-NEXT: callq __umodti3 at PLT
+; AVX-NEXT: movq %rax, %r12
+; AVX-NEXT: movq %rdx, %r13
+; AVX-NEXT: movl $14, %edx
+; AVX-NEXT: movq %r14, %rdi
+; AVX-NEXT: movq %rbx, %rsi
+; AVX-NEXT: xorl %ecx, %ecx
+; AVX-NEXT: callq __umodti3 at PLT
+; AVX-NEXT: movq %rdx, 24(%r15)
+; AVX-NEXT: movq %rax, 16(%r15)
+; AVX-NEXT: movq %r13, 8(%r15)
+; AVX-NEXT: movq %r12, (%r15)
+; AVX-NEXT: movq %r15, %rax
+; AVX-NEXT: popq %rbx
+; AVX-NEXT: popq %r12
+; AVX-NEXT: popq %r13
+; AVX-NEXT: popq %r14
+; AVX-NEXT: popq %r15
; AVX-NEXT: retq
entry:
%rem = urem <2 x i128> %x, <i128 14, i128 14>
diff --git a/llvm/test/CodeGen/X86/vector-reduce-fmin-nnan.ll b/llvm/test/CodeGen/X86/vector-reduce-fmin-nnan.ll
index d0a178dcc08b8..465988760d44a 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-fmin-nnan.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-fmin-nnan.ll
@@ -414,8 +414,9 @@ define half @test_v2f16(<2 x half> %a0) nounwind {
; AVX512F-NEXT: vpsrld $16, %xmm0, %xmm1
; AVX512F-NEXT: vcvtph2ps %xmm0, %xmm2
; AVX512F-NEXT: vcvtph2ps %xmm1, %xmm3
+; AVX512F-NEXT: xorl %eax, %eax
; AVX512F-NEXT: vucomiss %xmm3, %xmm2
-; AVX512F-NEXT: setb %al
+; AVX512F-NEXT: sbbl %eax, %eax
; AVX512F-NEXT: kmovd %eax, %k1
; AVX512F-NEXT: vmovdqu16 %zmm0, %zmm1 {%k1}
; AVX512F-NEXT: vmovdqa %xmm1, %xmm0
diff --git a/llvm/test/CodeGen/X86/vector-reduce-smin.ll b/llvm/test/CodeGen/X86/vector-reduce-smin.ll
index f86d3c1dd6e3f..bb87740c21538 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-smin.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-smin.ll
@@ -37,17 +37,14 @@ define i64 @test_v2i64(<2 x i64> %a0) {
; SSE41: # %bb.0:
; SSE41-NEXT: movdqa %xmm0, %xmm1
; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
-; SSE41-NEXT: pmovzxdq {{.*#+}} xmm0 = [2147483648,2147483648]
-; SSE41-NEXT: movdqa %xmm1, %xmm3
-; SSE41-NEXT: pxor %xmm0, %xmm3
-; SSE41-NEXT: pxor %xmm2, %xmm0
-; SSE41-NEXT: movdqa %xmm0, %xmm4
-; SSE41-NEXT: pcmpgtd %xmm3, %xmm4
-; SSE41-NEXT: pmovsxdq %xmm4, %xmm5
-; SSE41-NEXT: pcmpeqd %xmm3, %xmm0
-; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; SSE41-NEXT: pand %xmm5, %xmm3
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3]
+; SSE41-NEXT: pmovzxdq {{.*#+}} xmm3 = [2147483648,2147483648]
+; SSE41-NEXT: pxor %xmm3, %xmm0
+; SSE41-NEXT: pxor %xmm2, %xmm3
+; SSE41-NEXT: movdqa %xmm3, %xmm4
+; SSE41-NEXT: pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT: pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT: pmovsxdq %xmm3, %xmm0
+; SSE41-NEXT: pand %xmm4, %xmm0
; SSE41-NEXT: por %xmm3, %xmm0
; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm2
; SSE41-NEXT: movq %xmm2, %rax
@@ -135,12 +132,10 @@ define i64 @test_v4i64(<4 x i64> %a0) {
; SSE41-NEXT: movdqa %xmm1, %xmm4
; SSE41-NEXT: pxor %xmm3, %xmm4
; SSE41-NEXT: movdqa %xmm4, %xmm5
-; SSE41-NEXT: pcmpgtd %xmm0, %xmm5
-; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
-; SSE41-NEXT: pcmpeqd %xmm0, %xmm4
-; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; SSE41-NEXT: pand %xmm6, %xmm4
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm5[1,1,3,3]
+; SSE41-NEXT: pcmpeqd %xmm0, %xmm5
+; SSE41-NEXT: pcmpgtd %xmm0, %xmm4
+; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[0,0,2,2]
+; SSE41-NEXT: pand %xmm5, %xmm0
; SSE41-NEXT: por %xmm4, %xmm0
; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm1
; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
@@ -148,12 +143,10 @@ define i64 @test_v4i64(<4 x i64> %a0) {
; SSE41-NEXT: pxor %xmm3, %xmm0
; SSE41-NEXT: pxor %xmm2, %xmm3
; SSE41-NEXT: movdqa %xmm3, %xmm4
-; SSE41-NEXT: pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT: pmovsxdq %xmm4, %xmm5
-; SSE41-NEXT: pcmpeqd %xmm0, %xmm3
-; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm3[1,1,3,3]
-; SSE41-NEXT: pand %xmm5, %xmm3
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3]
+; SSE41-NEXT: pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT: pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT: pmovsxdq %xmm3, %xmm0
+; SSE41-NEXT: pand %xmm4, %xmm0
; SSE41-NEXT: por %xmm3, %xmm0
; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm2
; SSE41-NEXT: movq %xmm2, %rax
@@ -296,12 +289,10 @@ define i64 @test_v8i64(<8 x i64> %a0) {
; SSE41-NEXT: movdqa %xmm3, %xmm6
; SSE41-NEXT: pxor %xmm5, %xmm6
; SSE41-NEXT: movdqa %xmm6, %xmm7
-; SSE41-NEXT: pcmpgtd %xmm0, %xmm7
-; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
-; SSE41-NEXT: pcmpeqd %xmm0, %xmm6
-; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
-; SSE41-NEXT: pand %xmm8, %xmm6
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[1,1,3,3]
+; SSE41-NEXT: pcmpeqd %xmm0, %xmm7
+; SSE41-NEXT: pcmpgtd %xmm0, %xmm6
+; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT: pand %xmm7, %xmm0
; SSE41-NEXT: por %xmm6, %xmm0
; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm3
; SSE41-NEXT: movdqa %xmm4, %xmm0
@@ -309,12 +300,10 @@ define i64 @test_v8i64(<8 x i64> %a0) {
; SSE41-NEXT: movdqa %xmm2, %xmm1
; SSE41-NEXT: pxor %xmm5, %xmm1
; SSE41-NEXT: movdqa %xmm1, %xmm6
-; SSE41-NEXT: pcmpgtd %xmm0, %xmm6
-; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
-; SSE41-NEXT: pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE41-NEXT: pand %xmm7, %xmm1
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[1,1,3,3]
+; SSE41-NEXT: pcmpeqd %xmm0, %xmm6
+; SSE41-NEXT: pcmpgtd %xmm0, %xmm1
+; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2]
+; SSE41-NEXT: pand %xmm6, %xmm0
; SSE41-NEXT: por %xmm1, %xmm0
; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm2
; SSE41-NEXT: movapd %xmm2, %xmm0
@@ -322,12 +311,10 @@ define i64 @test_v8i64(<8 x i64> %a0) {
; SSE41-NEXT: movapd %xmm3, %xmm1
; SSE41-NEXT: xorpd %xmm5, %xmm1
; SSE41-NEXT: movapd %xmm1, %xmm4
-; SSE41-NEXT: pcmpgtd %xmm0, %xmm4
-; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm4[0,0,2,2]
-; SSE41-NEXT: pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE41-NEXT: pand %xmm6, %xmm1
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3]
+; SSE41-NEXT: pcmpeqd %xmm0, %xmm4
+; SSE41-NEXT: pcmpgtd %xmm0, %xmm1
+; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2]
+; SSE41-NEXT: pand %xmm4, %xmm0
; SSE41-NEXT: por %xmm1, %xmm0
; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm3
; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
@@ -335,12 +322,10 @@ define i64 @test_v8i64(<8 x i64> %a0) {
; SSE41-NEXT: pxor %xmm5, %xmm0
; SSE41-NEXT: pxor %xmm1, %xmm5
; SSE41-NEXT: movdqa %xmm5, %xmm2
-; SSE41-NEXT: pcmpgtd %xmm0, %xmm2
-; SSE41-NEXT: pmovsxdq %xmm2, %xmm4
-; SSE41-NEXT: pcmpeqd %xmm0, %xmm5
-; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; SSE41-NEXT: pand %xmm4, %xmm5
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3]
+; SSE41-NEXT: pcmpeqd %xmm0, %xmm2
+; SSE41-NEXT: pcmpgtd %xmm0, %xmm5
+; SSE41-NEXT: pmovsxdq %xmm5, %xmm0
+; SSE41-NEXT: pand %xmm2, %xmm0
; SSE41-NEXT: por %xmm5, %xmm0
; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm1
; SSE41-NEXT: movq %xmm1, %rax
@@ -559,12 +544,10 @@ define i64 @test_v16i64(<16 x i64> %a0) {
; SSE41-NEXT: movdqa %xmm6, %xmm10
; SSE41-NEXT: pxor %xmm9, %xmm10
; SSE41-NEXT: movdqa %xmm10, %xmm11
-; SSE41-NEXT: pcmpgtd %xmm0, %xmm11
-; SSE41-NEXT: pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2]
-; SSE41-NEXT: pcmpeqd %xmm0, %xmm10
-; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm10[1,1,3,3]
-; SSE41-NEXT: pand %xmm12, %xmm10
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm11[1,1,3,3]
+; SSE41-NEXT: pcmpeqd %xmm0, %xmm11
+; SSE41-NEXT: pcmpgtd %xmm0, %xmm10
+; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm10[0,0,2,2]
+; SSE41-NEXT: pand %xmm11, %xmm0
; SSE41-NEXT: por %xmm10, %xmm0
; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm6
; SSE41-NEXT: movdqa %xmm8, %xmm0
@@ -572,12 +555,10 @@ define i64 @test_v16i64(<16 x i64> %a0) {
; SSE41-NEXT: movdqa %xmm4, %xmm2
; SSE41-NEXT: pxor %xmm9, %xmm2
; SSE41-NEXT: movdqa %xmm2, %xmm10
-; SSE41-NEXT: pcmpgtd %xmm0, %xmm10
-; SSE41-NEXT: pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2]
-; SSE41-NEXT: pcmpeqd %xmm0, %xmm2
-; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE41-NEXT: pand %xmm11, %xmm2
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm10[1,1,3,3]
+; SSE41-NEXT: pcmpeqd %xmm0, %xmm10
+; SSE41-NEXT: pcmpgtd %xmm0, %xmm2
+; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2]
+; SSE41-NEXT: pand %xmm10, %xmm0
; SSE41-NEXT: por %xmm2, %xmm0
; SSE41-NEXT: blendvpd %xmm0, %xmm8, %xmm4
; SSE41-NEXT: movdqa %xmm3, %xmm0
@@ -585,12 +566,10 @@ define i64 @test_v16i64(<16 x i64> %a0) {
; SSE41-NEXT: movdqa %xmm7, %xmm2
; SSE41-NEXT: pxor %xmm9, %xmm2
; SSE41-NEXT: movdqa %xmm2, %xmm8
-; SSE41-NEXT: pcmpgtd %xmm0, %xmm8
-; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm8[0,0,2,2]
-; SSE41-NEXT: pcmpeqd %xmm0, %xmm2
-; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE41-NEXT: pand %xmm10, %xmm2
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm8[1,1,3,3]
+; SSE41-NEXT: pcmpeqd %xmm0, %xmm8
+; SSE41-NEXT: pcmpgtd %xmm0, %xmm2
+; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2]
+; SSE41-NEXT: pand %xmm8, %xmm0
; SSE41-NEXT: por %xmm2, %xmm0
; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm7
; SSE41-NEXT: movdqa %xmm1, %xmm0
@@ -598,12 +577,10 @@ define i64 @test_v16i64(<16 x i64> %a0) {
; SSE41-NEXT: movdqa %xmm5, %xmm2
; SSE41-NEXT: pxor %xmm9, %xmm2
; SSE41-NEXT: movdqa %xmm2, %xmm3
-; SSE41-NEXT: pcmpgtd %xmm0, %xmm3
-; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm3[0,0,2,2]
-; SSE41-NEXT: pcmpeqd %xmm0, %xmm2
-; SSE41-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE41-NEXT: pand %xmm8, %xmm2
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,1,3,3]
+; SSE41-NEXT: pcmpeqd %xmm0, %xmm3
+; SSE41-NEXT: pcmpgtd %xmm0, %xmm2
+; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,0,2,2]
+; SSE41-NEXT: pand %xmm3, %xmm0
; SSE41-NEXT: por %xmm2, %xmm0
; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm5
; SSE41-NEXT: movapd %xmm5, %xmm0
@@ -611,12 +588,10 @@ define i64 @test_v16i64(<16 x i64> %a0) {
; SSE41-NEXT: movapd %xmm7, %xmm1
; SSE41-NEXT: xorpd %xmm9, %xmm1
; SSE41-NEXT: movapd %xmm1, %xmm2
-; SSE41-NEXT: pcmpgtd %xmm0, %xmm2
-; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE41-NEXT: pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE41-NEXT: pand %xmm3, %xmm1
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3]
+; SSE41-NEXT: pcmpeqd %xmm0, %xmm2
+; SSE41-NEXT: pcmpgtd %xmm0, %xmm1
+; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2]
+; SSE41-NEXT: pand %xmm2, %xmm0
; SSE41-NEXT: por %xmm1, %xmm0
; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm7
; SSE41-NEXT: movapd %xmm4, %xmm0
@@ -624,12 +599,10 @@ define i64 @test_v16i64(<16 x i64> %a0) {
; SSE41-NEXT: movapd %xmm6, %xmm1
; SSE41-NEXT: xorpd %xmm9, %xmm1
; SSE41-NEXT: movapd %xmm1, %xmm2
-; SSE41-NEXT: pcmpgtd %xmm0, %xmm2
-; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE41-NEXT: pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE41-NEXT: pand %xmm3, %xmm1
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3]
+; SSE41-NEXT: pcmpeqd %xmm0, %xmm2
+; SSE41-NEXT: pcmpgtd %xmm0, %xmm1
+; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2]
+; SSE41-NEXT: pand %xmm2, %xmm0
; SSE41-NEXT: por %xmm1, %xmm0
; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm6
; SSE41-NEXT: movapd %xmm6, %xmm0
@@ -637,12 +610,10 @@ define i64 @test_v16i64(<16 x i64> %a0) {
; SSE41-NEXT: movapd %xmm7, %xmm1
; SSE41-NEXT: xorpd %xmm9, %xmm1
; SSE41-NEXT: movapd %xmm1, %xmm2
-; SSE41-NEXT: pcmpgtd %xmm0, %xmm2
-; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,0,2,2]
-; SSE41-NEXT: pcmpeqd %xmm0, %xmm1
-; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE41-NEXT: pand %xmm3, %xmm1
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3]
+; SSE41-NEXT: pcmpeqd %xmm0, %xmm2
+; SSE41-NEXT: pcmpgtd %xmm0, %xmm1
+; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,2,2]
+; SSE41-NEXT: pand %xmm2, %xmm0
; SSE41-NEXT: por %xmm1, %xmm0
; SSE41-NEXT: blendvpd %xmm0, %xmm6, %xmm7
; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm7[2,3,2,3]
@@ -650,13 +621,11 @@ define i64 @test_v16i64(<16 x i64> %a0) {
; SSE41-NEXT: pxor %xmm9, %xmm0
; SSE41-NEXT: pxor %xmm1, %xmm9
; SSE41-NEXT: movdqa %xmm9, %xmm2
-; SSE41-NEXT: pcmpgtd %xmm0, %xmm2
-; SSE41-NEXT: pmovsxdq %xmm2, %xmm3
-; SSE41-NEXT: pcmpeqd %xmm0, %xmm9
-; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm9[1,1,3,3]
-; SSE41-NEXT: pand %xmm3, %xmm4
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,3,3]
-; SSE41-NEXT: por %xmm4, %xmm0
+; SSE41-NEXT: pcmpeqd %xmm0, %xmm2
+; SSE41-NEXT: pcmpgtd %xmm0, %xmm9
+; SSE41-NEXT: pmovsxdq %xmm9, %xmm0
+; SSE41-NEXT: pand %xmm2, %xmm0
+; SSE41-NEXT: por %xmm9, %xmm0
; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm1
; SSE41-NEXT: movq %xmm1, %rax
; SSE41-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/vector-trunc-usat.ll b/llvm/test/CodeGen/X86/vector-trunc-usat.ll
index 9c82aad5fa6a3..0806e4960e48a 100644
--- a/llvm/test/CodeGen/X86/vector-trunc-usat.ll
+++ b/llvm/test/CodeGen/X86/vector-trunc-usat.ll
@@ -40,11 +40,10 @@ define <2 x i32> @trunc_usat_v2i64_v2i32(<2 x i64> %a0) {
; SSE41: # %bb.0:
; SSE41-NEXT: movdqa %xmm0, %xmm1
; SSE41-NEXT: movapd {{.*#+}} xmm2 = [4294967295,4294967295]
-; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456]
-; SSE41-NEXT: pxor %xmm1, %xmm0
-; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT: pxor %xmm0, %xmm3
+; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
; SSE41-NEXT: pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
; SSE41-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
; SSE41-NEXT: pandn %xmm3, %xmm0
; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm2
@@ -117,11 +116,10 @@ define void @trunc_usat_v2i64_v2i32_store(<2 x i64> %a0, ptr %p1) {
; SSE41: # %bb.0:
; SSE41-NEXT: movdqa %xmm0, %xmm1
; SSE41-NEXT: movapd {{.*#+}} xmm2 = [4294967295,4294967295]
-; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456]
-; SSE41-NEXT: pxor %xmm1, %xmm0
-; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT: pxor %xmm0, %xmm3
+; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
; SSE41-NEXT: pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
; SSE41-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
; SSE41-NEXT: pandn %xmm3, %xmm0
; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm2
@@ -209,21 +207,20 @@ define <4 x i32> @trunc_usat_v4i64_v4i32(<4 x i64> %a0) {
; SSE41: # %bb.0:
; SSE41-NEXT: movdqa %xmm0, %xmm2
; SSE41-NEXT: movdqa {{.*#+}} xmm4 = [9223372039002259456,9223372039002259456]
-; SSE41-NEXT: pxor %xmm4, %xmm0
-; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[0,0,2,2]
-; SSE41-NEXT: movdqa {{.*#+}} xmm5 = [2147483647,2147483647,2147483647,2147483647]
-; SSE41-NEXT: movdqa %xmm5, %xmm6
-; SSE41-NEXT: pcmpgtd %xmm3, %xmm6
-; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; SSE41-NEXT: pcmpeqd %xmm4, %xmm3
-; SSE41-NEXT: pand %xmm6, %xmm3
-; SSE41-NEXT: movdqa %xmm1, %xmm0
-; SSE41-NEXT: pxor %xmm4, %xmm0
-; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm0[0,0,2,2]
-; SSE41-NEXT: pcmpgtd %xmm6, %xmm5
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT: pcmpeqd %xmm4, %xmm0
-; SSE41-NEXT: pand %xmm5, %xmm0
+; SSE41-NEXT: movdqa %xmm0, %xmm5
+; SSE41-NEXT: pxor %xmm4, %xmm5
+; SSE41-NEXT: movdqa {{.*#+}} xmm6 = [9223372039002259455,9223372039002259455]
+; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm5[0,0,2,2]
+; SSE41-NEXT: pcmpeqd %xmm6, %xmm5
+; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [2147483647,2147483647,2147483647,2147483647]
+; SSE41-NEXT: movdqa %xmm0, %xmm3
+; SSE41-NEXT: pcmpgtd %xmm7, %xmm3
+; SSE41-NEXT: pand %xmm5, %xmm3
+; SSE41-NEXT: pxor %xmm1, %xmm4
+; SSE41-NEXT: pcmpeqd %xmm4, %xmm6
+; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,0,2,2]
+; SSE41-NEXT: pcmpgtd %xmm4, %xmm0
+; SSE41-NEXT: pand %xmm6, %xmm0
; SSE41-NEXT: movapd {{.*#+}} xmm4 = [4294967295,4294967295]
; SSE41-NEXT: movapd {{.*#+}} xmm5 = [4294967295,429496729]
; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm5
@@ -404,54 +401,51 @@ define <8 x i32> @trunc_usat_v8i64_v8i32(ptr %p0) {
;
; SSE41-LABEL: trunc_usat_v8i64_v8i32:
; SSE41: # %bb.0:
-; SSE41-NEXT: movdqa (%rdi), %xmm3
-; SSE41-NEXT: movdqa 16(%rdi), %xmm6
-; SSE41-NEXT: movdqa 32(%rdi), %xmm7
+; SSE41-NEXT: movdqa (%rdi), %xmm4
+; SSE41-NEXT: movdqa 16(%rdi), %xmm7
+; SSE41-NEXT: movdqa 32(%rdi), %xmm8
; SSE41-NEXT: movdqa 48(%rdi), %xmm1
-; SSE41-NEXT: movapd {{.*#+}} xmm2 = [4294967295,4294967295]
-; SSE41-NEXT: movdqa {{.*#+}} xmm4 = [9223372039002259456,9223372039002259456]
-; SSE41-NEXT: movdqa %xmm1, %xmm0
-; SSE41-NEXT: pxor %xmm4, %xmm0
-; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm0[0,0,2,2]
-; SSE41-NEXT: movdqa {{.*#+}} xmm5 = [2147483647,2147483647,2147483647,2147483647]
-; SSE41-NEXT: movdqa %xmm5, %xmm9
-; SSE41-NEXT: pcmpgtd %xmm8, %xmm9
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT: pcmpeqd %xmm4, %xmm0
-; SSE41-NEXT: pand %xmm9, %xmm0
-; SSE41-NEXT: movapd %xmm2, %xmm8
-; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm8
-; SSE41-NEXT: movdqa %xmm7, %xmm0
-; SSE41-NEXT: pxor %xmm4, %xmm0
-; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE41-NEXT: movdqa %xmm5, %xmm9
-; SSE41-NEXT: pcmpgtd %xmm1, %xmm9
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT: pcmpeqd %xmm4, %xmm0
+; SSE41-NEXT: movapd {{.*#+}} xmm3 = [4294967295,4294967295]
+; SSE41-NEXT: movdqa {{.*#+}} xmm6 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT: movdqa %xmm1, %xmm9
+; SSE41-NEXT: pxor %xmm6, %xmm9
+; SSE41-NEXT: movdqa {{.*#+}} xmm5 = [9223372039002259455,9223372039002259455]
+; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm9[0,0,2,2]
+; SSE41-NEXT: pcmpeqd %xmm5, %xmm9
+; SSE41-NEXT: movdqa {{.*#+}} xmm2 = [2147483647,2147483647,2147483647,2147483647]
+; SSE41-NEXT: movdqa %xmm2, %xmm0
+; SSE41-NEXT: pcmpgtd %xmm10, %xmm0
; SSE41-NEXT: pand %xmm9, %xmm0
-; SSE41-NEXT: movapd %xmm2, %xmm1
-; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm1
-; SSE41-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,2],xmm8[0,2]
-; SSE41-NEXT: movdqa %xmm6, %xmm0
-; SSE41-NEXT: pxor %xmm4, %xmm0
-; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm0[0,0,2,2]
-; SSE41-NEXT: movdqa %xmm5, %xmm8
-; SSE41-NEXT: pcmpgtd %xmm7, %xmm8
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT: pcmpeqd %xmm4, %xmm0
+; SSE41-NEXT: movapd %xmm3, %xmm9
+; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm9
+; SSE41-NEXT: movdqa %xmm8, %xmm1
+; SSE41-NEXT: pxor %xmm6, %xmm1
+; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm1[0,0,2,2]
+; SSE41-NEXT: pcmpeqd %xmm5, %xmm1
+; SSE41-NEXT: movdqa %xmm2, %xmm0
+; SSE41-NEXT: pcmpgtd %xmm10, %xmm0
+; SSE41-NEXT: pand %xmm1, %xmm0
+; SSE41-NEXT: movapd %xmm3, %xmm1
+; SSE41-NEXT: blendvpd %xmm0, %xmm8, %xmm1
+; SSE41-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,2],xmm9[0,2]
+; SSE41-NEXT: movdqa %xmm7, %xmm8
+; SSE41-NEXT: pxor %xmm6, %xmm8
+; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2]
+; SSE41-NEXT: pcmpeqd %xmm5, %xmm8
+; SSE41-NEXT: movdqa %xmm2, %xmm0
+; SSE41-NEXT: pcmpgtd %xmm9, %xmm0
; SSE41-NEXT: pand %xmm8, %xmm0
-; SSE41-NEXT: movapd %xmm2, %xmm7
-; SSE41-NEXT: blendvpd %xmm0, %xmm6, %xmm7
-; SSE41-NEXT: movdqa %xmm3, %xmm0
-; SSE41-NEXT: pxor %xmm4, %xmm0
-; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm0[0,0,2,2]
-; SSE41-NEXT: pcmpgtd %xmm6, %xmm5
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT: pcmpeqd %xmm4, %xmm0
-; SSE41-NEXT: pand %xmm5, %xmm0
-; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm2
-; SSE41-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,2],xmm7[0,2]
-; SSE41-NEXT: movaps %xmm2, %xmm0
+; SSE41-NEXT: movapd %xmm3, %xmm8
+; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm8
+; SSE41-NEXT: pxor %xmm4, %xmm6
+; SSE41-NEXT: pcmpeqd %xmm6, %xmm5
+; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT: pcmpgtd %xmm0, %xmm2
+; SSE41-NEXT: pand %xmm5, %xmm2
+; SSE41-NEXT: movdqa %xmm2, %xmm0
+; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm3
+; SSE41-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,2],xmm8[0,2]
+; SSE41-NEXT: movaps %xmm3, %xmm0
; SSE41-NEXT: retq
;
; AVX1-LABEL: trunc_usat_v8i64_v8i32:
@@ -544,11 +538,10 @@ define <2 x i16> @trunc_usat_v2i64_v2i16(<2 x i64> %a0) {
; SSE41: # %bb.0:
; SSE41-NEXT: movdqa %xmm0, %xmm1
; SSE41-NEXT: movapd {{.*#+}} xmm2 = [65535,65535]
-; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456]
-; SSE41-NEXT: pxor %xmm1, %xmm0
-; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT: pxor %xmm0, %xmm3
+; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
; SSE41-NEXT: pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
; SSE41-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
; SSE41-NEXT: pandn %xmm3, %xmm0
; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm2
@@ -639,11 +632,10 @@ define void @trunc_usat_v2i64_v2i16_store(<2 x i64> %a0, ptr %p1) {
; SSE41: # %bb.0:
; SSE41-NEXT: movdqa %xmm0, %xmm1
; SSE41-NEXT: movapd {{.*#+}} xmm2 = [65535,65535]
-; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456]
-; SSE41-NEXT: pxor %xmm1, %xmm0
-; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT: pxor %xmm0, %xmm3
+; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
; SSE41-NEXT: pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
; SSE41-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
; SSE41-NEXT: pandn %xmm3, %xmm0
; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm2
@@ -756,27 +748,26 @@ define <4 x i16> @trunc_usat_v4i64_v4i16(<4 x i64> %a0) {
; SSE41: # %bb.0:
; SSE41-NEXT: movdqa %xmm0, %xmm3
; SSE41-NEXT: movapd {{.*#+}} xmm2 = [65535,65535]
-; SSE41-NEXT: movdqa {{.*#+}} xmm4 = [9223372039002259456,9223372039002259456]
-; SSE41-NEXT: movdqa %xmm1, %xmm0
-; SSE41-NEXT: pxor %xmm4, %xmm0
-; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm0[0,0,2,2]
-; SSE41-NEXT: movdqa {{.*#+}} xmm6 = [2147549183,2147549183,2147549183,2147549183]
-; SSE41-NEXT: movdqa %xmm6, %xmm7
-; SSE41-NEXT: pcmpgtd %xmm5, %xmm7
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT: pcmpeqd %xmm4, %xmm0
-; SSE41-NEXT: pand %xmm7, %xmm0
-; SSE41-NEXT: movapd %xmm2, %xmm5
-; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm5
-; SSE41-NEXT: movdqa %xmm3, %xmm0
-; SSE41-NEXT: pxor %xmm4, %xmm0
-; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE41-NEXT: pcmpgtd %xmm1, %xmm6
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT: pcmpeqd %xmm4, %xmm0
+; SSE41-NEXT: movdqa {{.*#+}} xmm5 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT: movdqa %xmm1, %xmm6
+; SSE41-NEXT: pxor %xmm5, %xmm6
+; SSE41-NEXT: movdqa {{.*#+}} xmm7 = [9223372039002324991,9223372039002324991]
+; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2]
+; SSE41-NEXT: pcmpeqd %xmm7, %xmm6
+; SSE41-NEXT: movdqa {{.*#+}} xmm4 = [2147549183,2147549183,2147549183,2147549183]
+; SSE41-NEXT: movdqa %xmm4, %xmm0
+; SSE41-NEXT: pcmpgtd %xmm8, %xmm0
; SSE41-NEXT: pand %xmm6, %xmm0
+; SSE41-NEXT: movapd %xmm2, %xmm6
+; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm6
+; SSE41-NEXT: pxor %xmm3, %xmm5
+; SSE41-NEXT: pcmpeqd %xmm5, %xmm7
+; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
+; SSE41-NEXT: pcmpgtd %xmm0, %xmm4
+; SSE41-NEXT: pand %xmm7, %xmm4
+; SSE41-NEXT: movdqa %xmm4, %xmm0
; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm2
-; SSE41-NEXT: packusdw %xmm5, %xmm2
+; SSE41-NEXT: packusdw %xmm6, %xmm2
; SSE41-NEXT: packusdw %xmm2, %xmm2
; SSE41-NEXT: movdqa %xmm2, %xmm0
; SSE41-NEXT: retq
@@ -889,30 +880,29 @@ define void @trunc_usat_v4i64_v4i16_store(<4 x i64> %a0, ptr%p1) {
; SSE41-LABEL: trunc_usat_v4i64_v4i16_store:
; SSE41: # %bb.0:
; SSE41-NEXT: movdqa %xmm0, %xmm2
-; SSE41-NEXT: movapd {{.*#+}} xmm3 = [65535,65535]
-; SSE41-NEXT: movdqa {{.*#+}} xmm4 = [9223372039002259456,9223372039002259456]
-; SSE41-NEXT: movdqa %xmm1, %xmm0
-; SSE41-NEXT: pxor %xmm4, %xmm0
-; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm0[0,0,2,2]
-; SSE41-NEXT: movdqa {{.*#+}} xmm6 = [2147549183,2147549183,2147549183,2147549183]
-; SSE41-NEXT: movdqa %xmm6, %xmm7
-; SSE41-NEXT: pcmpgtd %xmm5, %xmm7
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT: pcmpeqd %xmm4, %xmm0
-; SSE41-NEXT: pand %xmm7, %xmm0
-; SSE41-NEXT: movapd %xmm3, %xmm5
-; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm5
-; SSE41-NEXT: movdqa %xmm2, %xmm0
-; SSE41-NEXT: pxor %xmm4, %xmm0
-; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE41-NEXT: pcmpgtd %xmm1, %xmm6
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT: pcmpeqd %xmm4, %xmm0
+; SSE41-NEXT: movapd {{.*#+}} xmm4 = [65535,65535]
+; SSE41-NEXT: movdqa {{.*#+}} xmm5 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT: movdqa %xmm1, %xmm6
+; SSE41-NEXT: pxor %xmm5, %xmm6
+; SSE41-NEXT: movdqa {{.*#+}} xmm7 = [9223372039002324991,9223372039002324991]
+; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2]
+; SSE41-NEXT: pcmpeqd %xmm7, %xmm6
+; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [2147549183,2147549183,2147549183,2147549183]
+; SSE41-NEXT: movdqa %xmm3, %xmm0
+; SSE41-NEXT: pcmpgtd %xmm8, %xmm0
; SSE41-NEXT: pand %xmm6, %xmm0
-; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm3
-; SSE41-NEXT: packusdw %xmm5, %xmm3
-; SSE41-NEXT: packusdw %xmm3, %xmm3
-; SSE41-NEXT: movq %xmm3, (%rdi)
+; SSE41-NEXT: movapd %xmm4, %xmm6
+; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm6
+; SSE41-NEXT: pxor %xmm2, %xmm5
+; SSE41-NEXT: pcmpeqd %xmm5, %xmm7
+; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
+; SSE41-NEXT: pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT: pand %xmm7, %xmm3
+; SSE41-NEXT: movdqa %xmm3, %xmm0
+; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm4
+; SSE41-NEXT: packusdw %xmm6, %xmm4
+; SSE41-NEXT: packusdw %xmm4, %xmm4
+; SSE41-NEXT: movq %xmm4, (%rdi)
; SSE41-NEXT: retq
;
; AVX1-LABEL: trunc_usat_v4i64_v4i16_store:
@@ -1055,55 +1045,52 @@ define <8 x i16> @trunc_usat_v8i64_v8i16(ptr %p0) {
;
; SSE41-LABEL: trunc_usat_v8i64_v8i16:
; SSE41: # %bb.0:
-; SSE41-NEXT: movdqa (%rdi), %xmm7
-; SSE41-NEXT: movdqa 16(%rdi), %xmm1
-; SSE41-NEXT: movdqa 32(%rdi), %xmm3
-; SSE41-NEXT: movdqa 48(%rdi), %xmm6
-; SSE41-NEXT: movapd {{.*#+}} xmm2 = [65535,65535]
-; SSE41-NEXT: movdqa {{.*#+}} xmm4 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT: movdqa (%rdi), %xmm8
+; SSE41-NEXT: movdqa 16(%rdi), %xmm2
+; SSE41-NEXT: movdqa 32(%rdi), %xmm4
+; SSE41-NEXT: movdqa 48(%rdi), %xmm7
+; SSE41-NEXT: movapd {{.*#+}} xmm3 = [65535,65535]
+; SSE41-NEXT: movdqa {{.*#+}} xmm6 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT: movdqa %xmm2, %xmm9
+; SSE41-NEXT: pxor %xmm6, %xmm9
+; SSE41-NEXT: movdqa {{.*#+}} xmm5 = [9223372039002324991,9223372039002324991]
+; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm9[0,0,2,2]
+; SSE41-NEXT: pcmpeqd %xmm5, %xmm9
+; SSE41-NEXT: movdqa {{.*#+}} xmm1 = [2147549183,2147549183,2147549183,2147549183]
; SSE41-NEXT: movdqa %xmm1, %xmm0
-; SSE41-NEXT: pxor %xmm4, %xmm0
-; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm0[0,0,2,2]
-; SSE41-NEXT: movdqa {{.*#+}} xmm5 = [2147549183,2147549183,2147549183,2147549183]
-; SSE41-NEXT: movdqa %xmm5, %xmm9
-; SSE41-NEXT: pcmpgtd %xmm8, %xmm9
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT: pcmpeqd %xmm4, %xmm0
-; SSE41-NEXT: pand %xmm9, %xmm0
-; SSE41-NEXT: movapd %xmm2, %xmm8
-; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm8
-; SSE41-NEXT: movdqa %xmm7, %xmm0
-; SSE41-NEXT: pxor %xmm4, %xmm0
-; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE41-NEXT: movdqa %xmm5, %xmm9
-; SSE41-NEXT: pcmpgtd %xmm1, %xmm9
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT: pcmpeqd %xmm4, %xmm0
+; SSE41-NEXT: pcmpgtd %xmm10, %xmm0
; SSE41-NEXT: pand %xmm9, %xmm0
-; SSE41-NEXT: movapd %xmm2, %xmm1
-; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm1
-; SSE41-NEXT: packusdw %xmm8, %xmm1
-; SSE41-NEXT: movdqa %xmm6, %xmm0
-; SSE41-NEXT: pxor %xmm4, %xmm0
-; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm0[0,0,2,2]
-; SSE41-NEXT: movdqa %xmm5, %xmm8
-; SSE41-NEXT: pcmpgtd %xmm7, %xmm8
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT: pcmpeqd %xmm4, %xmm0
+; SSE41-NEXT: movapd %xmm3, %xmm9
+; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm9
+; SSE41-NEXT: movdqa %xmm8, %xmm2
+; SSE41-NEXT: pxor %xmm6, %xmm2
+; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm2[0,0,2,2]
+; SSE41-NEXT: pcmpeqd %xmm5, %xmm2
+; SSE41-NEXT: movdqa %xmm1, %xmm0
+; SSE41-NEXT: pcmpgtd %xmm10, %xmm0
+; SSE41-NEXT: pand %xmm2, %xmm0
+; SSE41-NEXT: movapd %xmm3, %xmm2
+; SSE41-NEXT: blendvpd %xmm0, %xmm8, %xmm2
+; SSE41-NEXT: packusdw %xmm9, %xmm2
+; SSE41-NEXT: movdqa %xmm7, %xmm8
+; SSE41-NEXT: pxor %xmm6, %xmm8
+; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2]
+; SSE41-NEXT: pcmpeqd %xmm5, %xmm8
+; SSE41-NEXT: movdqa %xmm1, %xmm0
+; SSE41-NEXT: pcmpgtd %xmm9, %xmm0
; SSE41-NEXT: pand %xmm8, %xmm0
-; SSE41-NEXT: movapd %xmm2, %xmm7
-; SSE41-NEXT: blendvpd %xmm0, %xmm6, %xmm7
-; SSE41-NEXT: movdqa %xmm3, %xmm0
-; SSE41-NEXT: pxor %xmm4, %xmm0
-; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm0[0,0,2,2]
-; SSE41-NEXT: pcmpgtd %xmm6, %xmm5
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT: pcmpeqd %xmm4, %xmm0
-; SSE41-NEXT: pand %xmm5, %xmm0
-; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm2
-; SSE41-NEXT: packusdw %xmm7, %xmm2
-; SSE41-NEXT: packusdw %xmm2, %xmm1
+; SSE41-NEXT: movapd %xmm3, %xmm8
+; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm8
+; SSE41-NEXT: pxor %xmm4, %xmm6
+; SSE41-NEXT: pcmpeqd %xmm6, %xmm5
+; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT: pcmpgtd %xmm0, %xmm1
+; SSE41-NEXT: pand %xmm5, %xmm1
; SSE41-NEXT: movdqa %xmm1, %xmm0
+; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm3
+; SSE41-NEXT: packusdw %xmm8, %xmm3
+; SSE41-NEXT: packusdw %xmm3, %xmm2
+; SSE41-NEXT: movdqa %xmm2, %xmm0
; SSE41-NEXT: retq
;
; AVX1-LABEL: trunc_usat_v8i64_v8i16:
@@ -1647,11 +1634,10 @@ define <2 x i8> @trunc_usat_v2i64_v2i8(<2 x i64> %a0) {
; SSE41: # %bb.0:
; SSE41-NEXT: movdqa %xmm0, %xmm1
; SSE41-NEXT: movapd {{.*#+}} xmm2 = [255,255]
-; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456]
-; SSE41-NEXT: pxor %xmm1, %xmm0
-; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT: pxor %xmm0, %xmm3
+; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
; SSE41-NEXT: pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
; SSE41-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
; SSE41-NEXT: pandn %xmm3, %xmm0
; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm2
@@ -1743,11 +1729,10 @@ define void @trunc_usat_v2i64_v2i8_store(<2 x i64> %a0, ptr %p1) {
; SSE41: # %bb.0:
; SSE41-NEXT: movdqa %xmm0, %xmm1
; SSE41-NEXT: movapd {{.*#+}} xmm2 = [255,255]
-; SSE41-NEXT: movdqa {{.*#+}} xmm0 = [9223372039002259456,9223372039002259456]
-; SSE41-NEXT: pxor %xmm1, %xmm0
-; SSE41-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT: pxor %xmm0, %xmm3
+; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,0,2,2]
; SSE41-NEXT: pcmpeqd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
; SSE41-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
; SSE41-NEXT: pandn %xmm3, %xmm0
; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm2
@@ -1837,27 +1822,26 @@ define <4 x i8> @trunc_usat_v4i64_v4i8(<4 x i64> %a0) {
; SSE41: # %bb.0:
; SSE41-NEXT: movdqa %xmm0, %xmm3
; SSE41-NEXT: movapd {{.*#+}} xmm2 = [255,255]
-; SSE41-NEXT: movdqa {{.*#+}} xmm4 = [9223372039002259456,9223372039002259456]
-; SSE41-NEXT: movdqa %xmm1, %xmm0
-; SSE41-NEXT: pxor %xmm4, %xmm0
-; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm0[0,0,2,2]
-; SSE41-NEXT: movdqa {{.*#+}} xmm6 = [2147483903,2147483903,2147483903,2147483903]
-; SSE41-NEXT: movdqa %xmm6, %xmm7
-; SSE41-NEXT: pcmpgtd %xmm5, %xmm7
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT: pcmpeqd %xmm4, %xmm0
-; SSE41-NEXT: pand %xmm7, %xmm0
-; SSE41-NEXT: movapd %xmm2, %xmm5
-; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm5
-; SSE41-NEXT: movdqa %xmm3, %xmm0
-; SSE41-NEXT: pxor %xmm4, %xmm0
-; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE41-NEXT: pcmpgtd %xmm1, %xmm6
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT: pcmpeqd %xmm4, %xmm0
+; SSE41-NEXT: movdqa {{.*#+}} xmm5 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT: movdqa %xmm1, %xmm6
+; SSE41-NEXT: pxor %xmm5, %xmm6
+; SSE41-NEXT: movdqa {{.*#+}} xmm7 = [9223372039002259711,9223372039002259711]
+; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2]
+; SSE41-NEXT: pcmpeqd %xmm7, %xmm6
+; SSE41-NEXT: movdqa {{.*#+}} xmm4 = [2147483903,2147483903,2147483903,2147483903]
+; SSE41-NEXT: movdqa %xmm4, %xmm0
+; SSE41-NEXT: pcmpgtd %xmm8, %xmm0
; SSE41-NEXT: pand %xmm6, %xmm0
+; SSE41-NEXT: movapd %xmm2, %xmm6
+; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm6
+; SSE41-NEXT: pxor %xmm3, %xmm5
+; SSE41-NEXT: pcmpeqd %xmm5, %xmm7
+; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
+; SSE41-NEXT: pcmpgtd %xmm0, %xmm4
+; SSE41-NEXT: pand %xmm7, %xmm4
+; SSE41-NEXT: movdqa %xmm4, %xmm0
; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm2
-; SSE41-NEXT: packusdw %xmm5, %xmm2
+; SSE41-NEXT: packusdw %xmm6, %xmm2
; SSE41-NEXT: packusdw %xmm2, %xmm2
; SSE41-NEXT: packuswb %xmm2, %xmm2
; SSE41-NEXT: movdqa %xmm2, %xmm0
@@ -1972,31 +1956,30 @@ define void @trunc_usat_v4i64_v4i8_store(<4 x i64> %a0, ptr%p1) {
; SSE41-LABEL: trunc_usat_v4i64_v4i8_store:
; SSE41: # %bb.0:
; SSE41-NEXT: movdqa %xmm0, %xmm2
-; SSE41-NEXT: movapd {{.*#+}} xmm3 = [255,255]
-; SSE41-NEXT: movdqa {{.*#+}} xmm4 = [9223372039002259456,9223372039002259456]
-; SSE41-NEXT: movdqa %xmm1, %xmm0
-; SSE41-NEXT: pxor %xmm4, %xmm0
-; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm0[0,0,2,2]
-; SSE41-NEXT: movdqa {{.*#+}} xmm6 = [2147483903,2147483903,2147483903,2147483903]
-; SSE41-NEXT: movdqa %xmm6, %xmm7
-; SSE41-NEXT: pcmpgtd %xmm5, %xmm7
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT: pcmpeqd %xmm4, %xmm0
-; SSE41-NEXT: pand %xmm7, %xmm0
-; SSE41-NEXT: movapd %xmm3, %xmm5
-; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm5
-; SSE41-NEXT: movdqa %xmm2, %xmm0
-; SSE41-NEXT: pxor %xmm4, %xmm0
-; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE41-NEXT: pcmpgtd %xmm1, %xmm6
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT: pcmpeqd %xmm4, %xmm0
+; SSE41-NEXT: movapd {{.*#+}} xmm4 = [255,255]
+; SSE41-NEXT: movdqa {{.*#+}} xmm5 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT: movdqa %xmm1, %xmm6
+; SSE41-NEXT: pxor %xmm5, %xmm6
+; SSE41-NEXT: movdqa {{.*#+}} xmm7 = [9223372039002259711,9223372039002259711]
+; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm6[0,0,2,2]
+; SSE41-NEXT: pcmpeqd %xmm7, %xmm6
+; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [2147483903,2147483903,2147483903,2147483903]
+; SSE41-NEXT: movdqa %xmm3, %xmm0
+; SSE41-NEXT: pcmpgtd %xmm8, %xmm0
; SSE41-NEXT: pand %xmm6, %xmm0
-; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm3
-; SSE41-NEXT: packusdw %xmm5, %xmm3
-; SSE41-NEXT: packusdw %xmm3, %xmm3
-; SSE41-NEXT: packuswb %xmm3, %xmm3
-; SSE41-NEXT: movd %xmm3, (%rdi)
+; SSE41-NEXT: movapd %xmm4, %xmm6
+; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm6
+; SSE41-NEXT: pxor %xmm2, %xmm5
+; SSE41-NEXT: pcmpeqd %xmm5, %xmm7
+; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
+; SSE41-NEXT: pcmpgtd %xmm0, %xmm3
+; SSE41-NEXT: pand %xmm7, %xmm3
+; SSE41-NEXT: movdqa %xmm3, %xmm0
+; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm4
+; SSE41-NEXT: packusdw %xmm6, %xmm4
+; SSE41-NEXT: packusdw %xmm4, %xmm4
+; SSE41-NEXT: packuswb %xmm4, %xmm4
+; SSE41-NEXT: movd %xmm4, (%rdi)
; SSE41-NEXT: retq
;
; AVX1-LABEL: trunc_usat_v4i64_v4i8_store:
@@ -2138,56 +2121,53 @@ define <8 x i8> @trunc_usat_v8i64_v8i8(ptr %p0) {
;
; SSE41-LABEL: trunc_usat_v8i64_v8i8:
; SSE41: # %bb.0:
-; SSE41-NEXT: movdqa (%rdi), %xmm7
-; SSE41-NEXT: movdqa 16(%rdi), %xmm1
-; SSE41-NEXT: movdqa 32(%rdi), %xmm3
-; SSE41-NEXT: movdqa 48(%rdi), %xmm6
-; SSE41-NEXT: movapd {{.*#+}} xmm2 = [255,255]
-; SSE41-NEXT: movdqa {{.*#+}} xmm4 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT: movdqa (%rdi), %xmm8
+; SSE41-NEXT: movdqa 16(%rdi), %xmm2
+; SSE41-NEXT: movdqa 32(%rdi), %xmm4
+; SSE41-NEXT: movdqa 48(%rdi), %xmm7
+; SSE41-NEXT: movapd {{.*#+}} xmm3 = [255,255]
+; SSE41-NEXT: movdqa {{.*#+}} xmm6 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT: movdqa %xmm2, %xmm9
+; SSE41-NEXT: pxor %xmm6, %xmm9
+; SSE41-NEXT: movdqa {{.*#+}} xmm5 = [9223372039002259711,9223372039002259711]
+; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm9[0,0,2,2]
+; SSE41-NEXT: pcmpeqd %xmm5, %xmm9
+; SSE41-NEXT: movdqa {{.*#+}} xmm1 = [2147483903,2147483903,2147483903,2147483903]
; SSE41-NEXT: movdqa %xmm1, %xmm0
-; SSE41-NEXT: pxor %xmm4, %xmm0
-; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm0[0,0,2,2]
-; SSE41-NEXT: movdqa {{.*#+}} xmm5 = [2147483903,2147483903,2147483903,2147483903]
-; SSE41-NEXT: movdqa %xmm5, %xmm9
-; SSE41-NEXT: pcmpgtd %xmm8, %xmm9
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT: pcmpeqd %xmm4, %xmm0
+; SSE41-NEXT: pcmpgtd %xmm10, %xmm0
; SSE41-NEXT: pand %xmm9, %xmm0
-; SSE41-NEXT: movapd %xmm2, %xmm8
-; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm8
-; SSE41-NEXT: movdqa %xmm7, %xmm0
-; SSE41-NEXT: pxor %xmm4, %xmm0
-; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE41-NEXT: movdqa %xmm5, %xmm9
-; SSE41-NEXT: pcmpgtd %xmm1, %xmm9
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT: pcmpeqd %xmm4, %xmm0
-; SSE41-NEXT: pand %xmm9, %xmm0
-; SSE41-NEXT: movapd %xmm2, %xmm1
-; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm1
-; SSE41-NEXT: packusdw %xmm8, %xmm1
-; SSE41-NEXT: movdqa %xmm6, %xmm0
-; SSE41-NEXT: pxor %xmm4, %xmm0
-; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm0[0,0,2,2]
-; SSE41-NEXT: movdqa %xmm5, %xmm8
-; SSE41-NEXT: pcmpgtd %xmm7, %xmm8
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT: pcmpeqd %xmm4, %xmm0
+; SSE41-NEXT: movapd %xmm3, %xmm9
+; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm9
+; SSE41-NEXT: movdqa %xmm8, %xmm2
+; SSE41-NEXT: pxor %xmm6, %xmm2
+; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm2[0,0,2,2]
+; SSE41-NEXT: pcmpeqd %xmm5, %xmm2
+; SSE41-NEXT: movdqa %xmm1, %xmm0
+; SSE41-NEXT: pcmpgtd %xmm10, %xmm0
+; SSE41-NEXT: pand %xmm2, %xmm0
+; SSE41-NEXT: movapd %xmm3, %xmm2
+; SSE41-NEXT: blendvpd %xmm0, %xmm8, %xmm2
+; SSE41-NEXT: packusdw %xmm9, %xmm2
+; SSE41-NEXT: movdqa %xmm7, %xmm8
+; SSE41-NEXT: pxor %xmm6, %xmm8
+; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2]
+; SSE41-NEXT: pcmpeqd %xmm5, %xmm8
+; SSE41-NEXT: movdqa %xmm1, %xmm0
+; SSE41-NEXT: pcmpgtd %xmm9, %xmm0
; SSE41-NEXT: pand %xmm8, %xmm0
-; SSE41-NEXT: movapd %xmm2, %xmm7
-; SSE41-NEXT: blendvpd %xmm0, %xmm6, %xmm7
-; SSE41-NEXT: movdqa %xmm3, %xmm0
-; SSE41-NEXT: pxor %xmm4, %xmm0
-; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm0[0,0,2,2]
-; SSE41-NEXT: pcmpgtd %xmm6, %xmm5
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT: pcmpeqd %xmm4, %xmm0
-; SSE41-NEXT: pand %xmm5, %xmm0
-; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm2
-; SSE41-NEXT: packusdw %xmm7, %xmm2
-; SSE41-NEXT: packusdw %xmm2, %xmm1
-; SSE41-NEXT: packuswb %xmm1, %xmm1
+; SSE41-NEXT: movapd %xmm3, %xmm8
+; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm8
+; SSE41-NEXT: pxor %xmm4, %xmm6
+; SSE41-NEXT: pcmpeqd %xmm6, %xmm5
+; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,0,2,2]
+; SSE41-NEXT: pcmpgtd %xmm0, %xmm1
+; SSE41-NEXT: pand %xmm5, %xmm1
; SSE41-NEXT: movdqa %xmm1, %xmm0
+; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm3
+; SSE41-NEXT: packusdw %xmm8, %xmm3
+; SSE41-NEXT: packusdw %xmm3, %xmm2
+; SSE41-NEXT: packuswb %xmm2, %xmm2
+; SSE41-NEXT: movdqa %xmm2, %xmm0
; SSE41-NEXT: retq
;
; AVX1-LABEL: trunc_usat_v8i64_v8i8:
@@ -2323,56 +2303,53 @@ define void @trunc_usat_v8i64_v8i8_store(ptr %p0, ptr%p1) {
;
; SSE41-LABEL: trunc_usat_v8i64_v8i8_store:
; SSE41: # %bb.0:
-; SSE41-NEXT: movdqa (%rdi), %xmm7
-; SSE41-NEXT: movdqa 16(%rdi), %xmm6
-; SSE41-NEXT: movdqa 32(%rdi), %xmm2
-; SSE41-NEXT: movdqa 48(%rdi), %xmm5
-; SSE41-NEXT: movapd {{.*#+}} xmm1 = [255,255]
-; SSE41-NEXT: movdqa {{.*#+}} xmm3 = [9223372039002259456,9223372039002259456]
-; SSE41-NEXT: movdqa %xmm6, %xmm0
-; SSE41-NEXT: pxor %xmm3, %xmm0
-; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm0[0,0,2,2]
-; SSE41-NEXT: movdqa {{.*#+}} xmm4 = [2147483903,2147483903,2147483903,2147483903]
-; SSE41-NEXT: movdqa %xmm4, %xmm9
-; SSE41-NEXT: pcmpgtd %xmm8, %xmm9
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT: pcmpeqd %xmm3, %xmm0
-; SSE41-NEXT: pand %xmm9, %xmm0
-; SSE41-NEXT: movapd %xmm1, %xmm8
-; SSE41-NEXT: blendvpd %xmm0, %xmm6, %xmm8
-; SSE41-NEXT: movdqa %xmm7, %xmm0
-; SSE41-NEXT: pxor %xmm3, %xmm0
-; SSE41-NEXT: pshufd {{.*#+}} xmm6 = xmm0[0,0,2,2]
-; SSE41-NEXT: movdqa %xmm4, %xmm9
-; SSE41-NEXT: pcmpgtd %xmm6, %xmm9
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT: pcmpeqd %xmm3, %xmm0
+; SSE41-NEXT: movdqa (%rdi), %xmm8
+; SSE41-NEXT: movdqa 16(%rdi), %xmm7
+; SSE41-NEXT: movdqa 32(%rdi), %xmm3
+; SSE41-NEXT: movdqa 48(%rdi), %xmm6
+; SSE41-NEXT: movapd {{.*#+}} xmm2 = [255,255]
+; SSE41-NEXT: movdqa {{.*#+}} xmm5 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT: movdqa %xmm7, %xmm9
+; SSE41-NEXT: pxor %xmm5, %xmm9
+; SSE41-NEXT: movdqa {{.*#+}} xmm4 = [9223372039002259711,9223372039002259711]
+; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm9[0,0,2,2]
+; SSE41-NEXT: pcmpeqd %xmm4, %xmm9
+; SSE41-NEXT: movdqa {{.*#+}} xmm1 = [2147483903,2147483903,2147483903,2147483903]
+; SSE41-NEXT: movdqa %xmm1, %xmm0
+; SSE41-NEXT: pcmpgtd %xmm10, %xmm0
; SSE41-NEXT: pand %xmm9, %xmm0
-; SSE41-NEXT: movapd %xmm1, %xmm6
-; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm6
-; SSE41-NEXT: packusdw %xmm8, %xmm6
-; SSE41-NEXT: movdqa %xmm5, %xmm0
-; SSE41-NEXT: pxor %xmm3, %xmm0
-; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm0[0,0,2,2]
-; SSE41-NEXT: movdqa %xmm4, %xmm8
-; SSE41-NEXT: pcmpgtd %xmm7, %xmm8
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT: pcmpeqd %xmm3, %xmm0
+; SSE41-NEXT: movapd %xmm2, %xmm9
+; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm9
+; SSE41-NEXT: movdqa %xmm8, %xmm7
+; SSE41-NEXT: pxor %xmm5, %xmm7
+; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm7[0,0,2,2]
+; SSE41-NEXT: pcmpeqd %xmm4, %xmm7
+; SSE41-NEXT: movdqa %xmm1, %xmm0
+; SSE41-NEXT: pcmpgtd %xmm10, %xmm0
+; SSE41-NEXT: pand %xmm7, %xmm0
+; SSE41-NEXT: movapd %xmm2, %xmm7
+; SSE41-NEXT: blendvpd %xmm0, %xmm8, %xmm7
+; SSE41-NEXT: packusdw %xmm9, %xmm7
+; SSE41-NEXT: movdqa %xmm6, %xmm8
+; SSE41-NEXT: pxor %xmm5, %xmm8
+; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2]
+; SSE41-NEXT: pcmpeqd %xmm4, %xmm8
+; SSE41-NEXT: movdqa %xmm1, %xmm0
+; SSE41-NEXT: pcmpgtd %xmm9, %xmm0
; SSE41-NEXT: pand %xmm8, %xmm0
-; SSE41-NEXT: movapd %xmm1, %xmm7
-; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm7
-; SSE41-NEXT: movdqa %xmm2, %xmm0
-; SSE41-NEXT: pxor %xmm3, %xmm0
-; SSE41-NEXT: pshufd {{.*#+}} xmm5 = xmm0[0,0,2,2]
-; SSE41-NEXT: pcmpgtd %xmm5, %xmm4
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT: pcmpeqd %xmm3, %xmm0
-; SSE41-NEXT: pand %xmm4, %xmm0
-; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm1
-; SSE41-NEXT: packusdw %xmm7, %xmm1
-; SSE41-NEXT: packusdw %xmm1, %xmm6
-; SSE41-NEXT: packuswb %xmm6, %xmm6
-; SSE41-NEXT: movq %xmm6, (%rsi)
+; SSE41-NEXT: movapd %xmm2, %xmm8
+; SSE41-NEXT: blendvpd %xmm0, %xmm6, %xmm8
+; SSE41-NEXT: pxor %xmm3, %xmm5
+; SSE41-NEXT: pcmpeqd %xmm5, %xmm4
+; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm5[0,0,2,2]
+; SSE41-NEXT: pcmpgtd %xmm0, %xmm1
+; SSE41-NEXT: pand %xmm4, %xmm1
+; SSE41-NEXT: movdqa %xmm1, %xmm0
+; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm2
+; SSE41-NEXT: packusdw %xmm8, %xmm2
+; SSE41-NEXT: packusdw %xmm2, %xmm7
+; SSE41-NEXT: packuswb %xmm7, %xmm7
+; SSE41-NEXT: movq %xmm7, (%rsi)
; SSE41-NEXT: retq
;
; AVX1-LABEL: trunc_usat_v8i64_v8i8_store:
@@ -2561,103 +2538,96 @@ define <16 x i8> @trunc_usat_v16i64_v16i8(ptr %p0) {
;
; SSE41-LABEL: trunc_usat_v16i64_v16i8:
; SSE41: # %bb.0:
-; SSE41-NEXT: movdqa 96(%rdi), %xmm2
-; SSE41-NEXT: movdqa 112(%rdi), %xmm4
-; SSE41-NEXT: movdqa 64(%rdi), %xmm7
-; SSE41-NEXT: movdqa 80(%rdi), %xmm8
-; SSE41-NEXT: movdqa (%rdi), %xmm11
-; SSE41-NEXT: movdqa 16(%rdi), %xmm1
-; SSE41-NEXT: movdqa 32(%rdi), %xmm9
-; SSE41-NEXT: movdqa 48(%rdi), %xmm10
-; SSE41-NEXT: movapd {{.*#+}} xmm3 = [255,255]
-; SSE41-NEXT: movdqa {{.*#+}} xmm5 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT: movdqa 96(%rdi), %xmm3
+; SSE41-NEXT: movdqa 112(%rdi), %xmm5
+; SSE41-NEXT: movdqa 64(%rdi), %xmm8
+; SSE41-NEXT: movdqa 80(%rdi), %xmm9
+; SSE41-NEXT: movdqa (%rdi), %xmm12
+; SSE41-NEXT: movdqa 16(%rdi), %xmm2
+; SSE41-NEXT: movdqa 32(%rdi), %xmm10
+; SSE41-NEXT: movdqa 48(%rdi), %xmm11
+; SSE41-NEXT: movapd {{.*#+}} xmm4 = [255,255]
+; SSE41-NEXT: movdqa {{.*#+}} xmm7 = [9223372039002259456,9223372039002259456]
+; SSE41-NEXT: movdqa %xmm2, %xmm13
+; SSE41-NEXT: pxor %xmm7, %xmm13
+; SSE41-NEXT: movdqa {{.*#+}} xmm6 = [9223372039002259711,9223372039002259711]
+; SSE41-NEXT: pshufd {{.*#+}} xmm14 = xmm13[0,0,2,2]
+; SSE41-NEXT: pcmpeqd %xmm6, %xmm13
+; SSE41-NEXT: movdqa {{.*#+}} xmm1 = [2147483903,2147483903,2147483903,2147483903]
; SSE41-NEXT: movdqa %xmm1, %xmm0
-; SSE41-NEXT: pxor %xmm5, %xmm0
-; SSE41-NEXT: pshufd {{.*#+}} xmm12 = xmm0[0,0,2,2]
-; SSE41-NEXT: movdqa {{.*#+}} xmm6 = [2147483903,2147483903,2147483903,2147483903]
-; SSE41-NEXT: movdqa %xmm6, %xmm13
-; SSE41-NEXT: pcmpgtd %xmm12, %xmm13
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT: pcmpeqd %xmm5, %xmm0
+; SSE41-NEXT: pcmpgtd %xmm14, %xmm0
; SSE41-NEXT: pand %xmm13, %xmm0
-; SSE41-NEXT: movapd %xmm3, %xmm12
-; SSE41-NEXT: blendvpd %xmm0, %xmm1, %xmm12
-; SSE41-NEXT: movdqa %xmm11, %xmm0
-; SSE41-NEXT: pxor %xmm5, %xmm0
-; SSE41-NEXT: pshufd {{.*#+}} xmm1 = xmm0[0,0,2,2]
-; SSE41-NEXT: movdqa %xmm6, %xmm13
-; SSE41-NEXT: pcmpgtd %xmm1, %xmm13
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT: pcmpeqd %xmm5, %xmm0
-; SSE41-NEXT: pand %xmm13, %xmm0
-; SSE41-NEXT: movapd %xmm3, %xmm1
-; SSE41-NEXT: blendvpd %xmm0, %xmm11, %xmm1
-; SSE41-NEXT: packusdw %xmm12, %xmm1
-; SSE41-NEXT: movdqa %xmm10, %xmm0
-; SSE41-NEXT: pxor %xmm5, %xmm0
-; SSE41-NEXT: pshufd {{.*#+}} xmm11 = xmm0[0,0,2,2]
-; SSE41-NEXT: movdqa %xmm6, %xmm12
-; SSE41-NEXT: pcmpgtd %xmm11, %xmm12
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT: pcmpeqd %xmm5, %xmm0
+; SSE41-NEXT: movapd %xmm4, %xmm13
+; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm13
+; SSE41-NEXT: movdqa %xmm12, %xmm2
+; SSE41-NEXT: pxor %xmm7, %xmm2
+; SSE41-NEXT: pshufd {{.*#+}} xmm14 = xmm2[0,0,2,2]
+; SSE41-NEXT: pcmpeqd %xmm6, %xmm2
+; SSE41-NEXT: movdqa %xmm1, %xmm0
+; SSE41-NEXT: pcmpgtd %xmm14, %xmm0
+; SSE41-NEXT: pand %xmm2, %xmm0
+; SSE41-NEXT: movapd %xmm4, %xmm2
+; SSE41-NEXT: blendvpd %xmm0, %xmm12, %xmm2
+; SSE41-NEXT: packusdw %xmm13, %xmm2
+; SSE41-NEXT: movdqa %xmm11, %xmm12
+; SSE41-NEXT: pxor %xmm7, %xmm12
+; SSE41-NEXT: pshufd {{.*#+}} xmm13 = xmm12[0,0,2,2]
+; SSE41-NEXT: pcmpeqd %xmm6, %xmm12
+; SSE41-NEXT: movdqa %xmm1, %xmm0
+; SSE41-NEXT: pcmpgtd %xmm13, %xmm0
; SSE41-NEXT: pand %xmm12, %xmm0
-; SSE41-NEXT: movapd %xmm3, %xmm11
+; SSE41-NEXT: movapd %xmm4, %xmm12
+; SSE41-NEXT: blendvpd %xmm0, %xmm11, %xmm12
+; SSE41-NEXT: movdqa %xmm10, %xmm11
+; SSE41-NEXT: pxor %xmm7, %xmm11
+; SSE41-NEXT: pshufd {{.*#+}} xmm13 = xmm11[0,0,2,2]
+; SSE41-NEXT: pcmpeqd %xmm6, %xmm11
+; SSE41-NEXT: movdqa %xmm1, %xmm0
+; SSE41-NEXT: pcmpgtd %xmm13, %xmm0
+; SSE41-NEXT: pand %xmm11, %xmm0
+; SSE41-NEXT: movapd %xmm4, %xmm11
; SSE41-NEXT: blendvpd %xmm0, %xmm10, %xmm11
-; SSE41-NEXT: movdqa %xmm9, %xmm0
-; SSE41-NEXT: pxor %xmm5, %xmm0
-; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm0[0,0,2,2]
-; SSE41-NEXT: movdqa %xmm6, %xmm12
-; SSE41-NEXT: pcmpgtd %xmm10, %xmm12
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT: pcmpeqd %xmm5, %xmm0
-; SSE41-NEXT: pand %xmm12, %xmm0
-; SSE41-NEXT: movapd %xmm3, %xmm10
-; SSE41-NEXT: blendvpd %xmm0, %xmm9, %xmm10
-; SSE41-NEXT: packusdw %xmm11, %xmm10
-; SSE41-NEXT: packusdw %xmm10, %xmm1
-; SSE41-NEXT: movdqa %xmm8, %xmm0
-; SSE41-NEXT: pxor %xmm5, %xmm0
-; SSE41-NEXT: pshufd {{.*#+}} xmm9 = xmm0[0,0,2,2]
-; SSE41-NEXT: movdqa %xmm6, %xmm10
-; SSE41-NEXT: pcmpgtd %xmm9, %xmm10
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT: pcmpeqd %xmm5, %xmm0
-; SSE41-NEXT: pand %xmm10, %xmm0
-; SSE41-NEXT: movapd %xmm3, %xmm9
-; SSE41-NEXT: blendvpd %xmm0, %xmm8, %xmm9
-; SSE41-NEXT: movdqa %xmm7, %xmm0
-; SSE41-NEXT: pxor %xmm5, %xmm0
-; SSE41-NEXT: pshufd {{.*#+}} xmm8 = xmm0[0,0,2,2]
-; SSE41-NEXT: movdqa %xmm6, %xmm10
-; SSE41-NEXT: pcmpgtd %xmm8, %xmm10
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT: pcmpeqd %xmm5, %xmm0
+; SSE41-NEXT: packusdw %xmm12, %xmm11
+; SSE41-NEXT: packusdw %xmm11, %xmm2
+; SSE41-NEXT: movdqa %xmm9, %xmm10
+; SSE41-NEXT: pxor %xmm7, %xmm10
+; SSE41-NEXT: pshufd {{.*#+}} xmm11 = xmm10[0,0,2,2]
+; SSE41-NEXT: pcmpeqd %xmm6, %xmm10
+; SSE41-NEXT: movdqa %xmm1, %xmm0
+; SSE41-NEXT: pcmpgtd %xmm11, %xmm0
; SSE41-NEXT: pand %xmm10, %xmm0
-; SSE41-NEXT: movapd %xmm3, %xmm8
-; SSE41-NEXT: blendvpd %xmm0, %xmm7, %xmm8
-; SSE41-NEXT: packusdw %xmm9, %xmm8
-; SSE41-NEXT: movdqa %xmm4, %xmm0
-; SSE41-NEXT: pxor %xmm5, %xmm0
-; SSE41-NEXT: pshufd {{.*#+}} xmm7 = xmm0[0,0,2,2]
-; SSE41-NEXT: movdqa %xmm6, %xmm9
-; SSE41-NEXT: pcmpgtd %xmm7, %xmm9
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT: pcmpeqd %xmm5, %xmm0
+; SSE41-NEXT: movapd %xmm4, %xmm10
+; SSE41-NEXT: blendvpd %xmm0, %xmm9, %xmm10
+; SSE41-NEXT: movdqa %xmm8, %xmm9
+; SSE41-NEXT: pxor %xmm7, %xmm9
+; SSE41-NEXT: pshufd {{.*#+}} xmm11 = xmm9[0,0,2,2]
+; SSE41-NEXT: pcmpeqd %xmm6, %xmm9
+; SSE41-NEXT: movdqa %xmm1, %xmm0
+; SSE41-NEXT: pcmpgtd %xmm11, %xmm0
; SSE41-NEXT: pand %xmm9, %xmm0
-; SSE41-NEXT: movapd %xmm3, %xmm7
-; SSE41-NEXT: blendvpd %xmm0, %xmm4, %xmm7
-; SSE41-NEXT: movdqa %xmm2, %xmm0
-; SSE41-NEXT: pxor %xmm5, %xmm0
-; SSE41-NEXT: pshufd {{.*#+}} xmm4 = xmm0[0,0,2,2]
-; SSE41-NEXT: pcmpgtd %xmm4, %xmm6
-; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
-; SSE41-NEXT: pcmpeqd %xmm5, %xmm0
-; SSE41-NEXT: pand %xmm6, %xmm0
-; SSE41-NEXT: blendvpd %xmm0, %xmm2, %xmm3
-; SSE41-NEXT: packusdw %xmm7, %xmm3
-; SSE41-NEXT: packusdw %xmm3, %xmm8
-; SSE41-NEXT: packuswb %xmm8, %xmm1
+; SSE41-NEXT: movapd %xmm4, %xmm9
+; SSE41-NEXT: blendvpd %xmm0, %xmm8, %xmm9
+; SSE41-NEXT: packusdw %xmm10, %xmm9
+; SSE41-NEXT: movdqa %xmm5, %xmm8
+; SSE41-NEXT: pxor %xmm7, %xmm8
+; SSE41-NEXT: pshufd {{.*#+}} xmm10 = xmm8[0,0,2,2]
+; SSE41-NEXT: pcmpeqd %xmm6, %xmm8
+; SSE41-NEXT: movdqa %xmm1, %xmm0
+; SSE41-NEXT: pcmpgtd %xmm10, %xmm0
+; SSE41-NEXT: pand %xmm8, %xmm0
+; SSE41-NEXT: movapd %xmm4, %xmm8
+; SSE41-NEXT: blendvpd %xmm0, %xmm5, %xmm8
+; SSE41-NEXT: pxor %xmm3, %xmm7
+; SSE41-NEXT: pcmpeqd %xmm7, %xmm6
+; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm7[0,0,2,2]
+; SSE41-NEXT: pcmpgtd %xmm0, %xmm1
+; SSE41-NEXT: pand %xmm6, %xmm1
; SSE41-NEXT: movdqa %xmm1, %xmm0
+; SSE41-NEXT: blendvpd %xmm0, %xmm3, %xmm4
+; SSE41-NEXT: packusdw %xmm8, %xmm4
+; SSE41-NEXT: packusdw %xmm4, %xmm9
+; SSE41-NEXT: packuswb %xmm9, %xmm2
+; SSE41-NEXT: movdqa %xmm2, %xmm0
; SSE41-NEXT: retq
;
; AVX1-LABEL: trunc_usat_v16i64_v16i8:
diff --git a/llvm/test/CodeGen/X86/widen-load-of-small-alloca-with-zero-upper-half.ll b/llvm/test/CodeGen/X86/widen-load-of-small-alloca-with-zero-upper-half.ll
index fde915247760a..6b5c6049f025b 100644
--- a/llvm/test/CodeGen/X86/widen-load-of-small-alloca-with-zero-upper-half.ll
+++ b/llvm/test/CodeGen/X86/widen-load-of-small-alloca-with-zero-upper-half.ll
@@ -171,8 +171,8 @@ define void @load_2byte_chunk_of_4byte_alloca_with_zero_upper_half(ptr %src, i64
define void @load_1byte_chunk_of_8byte_alloca_with_zero_upper_half(ptr %src, i64 %byteOff, ptr %dst) nounwind {
; X64-NO-BMI2-LABEL: load_1byte_chunk_of_8byte_alloca_with_zero_upper_half:
; X64-NO-BMI2: # %bb.0:
-; X64-NO-BMI2-NEXT: movl (%rdi), %eax
; X64-NO-BMI2-NEXT: leal (,%rsi,8), %ecx
+; X64-NO-BMI2-NEXT: movl (%rdi), %eax
; X64-NO-BMI2-NEXT: # kill: def $cl killed $cl killed $ecx
; X64-NO-BMI2-NEXT: shrq %cl, %rax
; X64-NO-BMI2-NEXT: movb %al, (%rdx)
@@ -180,8 +180,8 @@ define void @load_1byte_chunk_of_8byte_alloca_with_zero_upper_half(ptr %src, i64
;
; X64-BMI2-LABEL: load_1byte_chunk_of_8byte_alloca_with_zero_upper_half:
; X64-BMI2: # %bb.0:
-; X64-BMI2-NEXT: movl (%rdi), %eax
; X64-BMI2-NEXT: shll $3, %esi
+; X64-BMI2-NEXT: movl (%rdi), %eax
; X64-BMI2-NEXT: shrxq %rsi, %rax, %rax
; X64-BMI2-NEXT: movb %al, (%rdx)
; X64-BMI2-NEXT: retq
@@ -248,8 +248,8 @@ define void @load_1byte_chunk_of_8byte_alloca_with_zero_upper_half(ptr %src, i64
define void @load_2byte_chunk_of_8byte_alloca_with_zero_upper_half(ptr %src, i64 %byteOff, ptr %dst) nounwind {
; X64-NO-BMI2-LABEL: load_2byte_chunk_of_8byte_alloca_with_zero_upper_half:
; X64-NO-BMI2: # %bb.0:
-; X64-NO-BMI2-NEXT: movl (%rdi), %eax
; X64-NO-BMI2-NEXT: leal (,%rsi,8), %ecx
+; X64-NO-BMI2-NEXT: movl (%rdi), %eax
; X64-NO-BMI2-NEXT: # kill: def $cl killed $cl killed $ecx
; X64-NO-BMI2-NEXT: shrq %cl, %rax
; X64-NO-BMI2-NEXT: movw %ax, (%rdx)
@@ -257,8 +257,8 @@ define void @load_2byte_chunk_of_8byte_alloca_with_zero_upper_half(ptr %src, i64
;
; X64-BMI2-LABEL: load_2byte_chunk_of_8byte_alloca_with_zero_upper_half:
; X64-BMI2: # %bb.0:
-; X64-BMI2-NEXT: movl (%rdi), %eax
; X64-BMI2-NEXT: shll $3, %esi
+; X64-BMI2-NEXT: movl (%rdi), %eax
; X64-BMI2-NEXT: shrxq %rsi, %rax, %rax
; X64-BMI2-NEXT: movw %ax, (%rdx)
; X64-BMI2-NEXT: retq
@@ -324,8 +324,8 @@ define void @load_2byte_chunk_of_8byte_alloca_with_zero_upper_half(ptr %src, i64
define void @load_4byte_chunk_of_8byte_alloca_with_zero_upper_half(ptr %src, i64 %byteOff, ptr %dst) nounwind {
; X64-NO-BMI2-LABEL: load_4byte_chunk_of_8byte_alloca_with_zero_upper_half:
; X64-NO-BMI2: # %bb.0:
-; X64-NO-BMI2-NEXT: movl (%rdi), %eax
; X64-NO-BMI2-NEXT: leal (,%rsi,8), %ecx
+; X64-NO-BMI2-NEXT: movl (%rdi), %eax
; X64-NO-BMI2-NEXT: # kill: def $cl killed $cl killed $ecx
; X64-NO-BMI2-NEXT: shrq %cl, %rax
; X64-NO-BMI2-NEXT: movl %eax, (%rdx)
@@ -333,8 +333,8 @@ define void @load_4byte_chunk_of_8byte_alloca_with_zero_upper_half(ptr %src, i64
;
; X64-BMI2-LABEL: load_4byte_chunk_of_8byte_alloca_with_zero_upper_half:
; X64-BMI2: # %bb.0:
-; X64-BMI2-NEXT: movl (%rdi), %eax
; X64-BMI2-NEXT: shll $3, %esi
+; X64-BMI2-NEXT: movl (%rdi), %eax
; X64-BMI2-NEXT: shrxq %rsi, %rax, %rax
; X64-BMI2-NEXT: movl %eax, (%rdx)
; X64-BMI2-NEXT: retq
>From 06c481dab57864c82887aa6ab9931b7880bfc9d3 Mon Sep 17 00:00:00 2001
From: Takashiidobe <idobetakashi at gmail.com>
Date: Wed, 25 Mar 2026 22:32:49 -0400
Subject: [PATCH 2/2] retry fixup pass
---
.../lib/CodeGen/TwoAddressInstructionPass.cpp | 33 +----
llvm/lib/Target/X86/X86FixupLEAs.cpp | 116 ++++++++++++++++++
llvm/test/CodeGen/X86/mov-lea-folds.ll | 50 ++++++--
3 files changed, 162 insertions(+), 37 deletions(-)
diff --git a/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp b/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp
index 43790412cda43..ace5c6e49596e 100644
--- a/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp
+++ b/llvm/lib/CodeGen/TwoAddressInstructionPass.cpp
@@ -158,7 +158,7 @@ class TwoAddressInstructionImpl {
bool commuteInstruction(MachineInstr *MI, unsigned DstIdx,
unsigned RegBIdx, unsigned RegCIdx, unsigned Dist);
- bool isProfitableToConv3Addr(MachineInstr &MI, Register RegA, Register RegB);
+ bool isProfitableToConv3Addr(Register RegA, Register RegB);
bool convertInstTo3Addr(MachineBasicBlock::iterator &mi,
MachineBasicBlock::iterator &nmi, Register RegA,
@@ -786,38 +786,13 @@ bool TwoAddressInstructionImpl::commuteInstruction(MachineInstr *MI,
/// Return true if it is profitable to convert the given 2-address instruction
/// to a 3-address one.
-bool TwoAddressInstructionImpl::isProfitableToConv3Addr(MachineInstr &MI,
- Register RegA,
+bool TwoAddressInstructionImpl::isProfitableToConv3Addr(Register RegA,
Register RegB) {
- // Narrowly target copy+op patterns:
- // %tmp = COPY %src
- // %tmp = OP %tmp, ...
- // and only when removing that copy is obvious.
- if (RegA.isVirtual() && RegB.isVirtual() && MRI->hasOneNonDBGUse(RegB)) {
-
- MachineInstr *DefMI = MRI->getUniqueVRegDef(RegB);
- bool HasCopyDef = DefMI && DefMI->isCopy();
- bool InSameBlock = HasCopyDef && DefMI->getParent() == MI.getParent();
- bool HasRegOperands = InSameBlock && DefMI->getOperand(0).isReg() &&
- DefMI->getOperand(1).isReg();
- bool DefinesRegB = HasRegOperands && DefMI->getOperand(0).getReg() == RegB;
- bool CopySrcIsVirtual =
- DefinesRegB && DefMI->getOperand(1).getReg().isVirtual();
-
- if (CopySrcIsVirtual) {
- auto NextNonDbg = std::next(DefMI->getIterator());
- while (NextNonDbg != MI.getIterator() && NextNonDbg->isDebugInstr())
- ++NextNonDbg;
- if (NextNonDbg == MI.getIterator())
- return true;
- }
- }
-
// Look for situations like this:
// %reg1024 = MOV r1
// %reg1025 = MOV r0
// %reg1026 = ADD %reg1024, %reg1025
- // r2 = MOV %reg1026
+ // r2 = MOV %reg1026
// Turn ADD into a 3-address instruction to avoid a copy.
MCRegister FromRegB = getMappedReg(RegB, SrcRegMap);
if (!FromRegB)
@@ -1412,7 +1387,7 @@ bool TwoAddressInstructionImpl::tryInstructionTransform(
if (ConvertibleTo3Addr) {
// This instruction is potentially convertible to a true
// three-address instruction. Check if it is profitable.
- if (!regBKilled || isProfitableToConv3Addr(MI, regA, regB)) {
+ if (!regBKilled || isProfitableToConv3Addr(regA, regB)) {
// Try to convert it.
if (convertInstTo3Addr(mi, nmi, regA, regB, Dist)) {
++NumConvertedTo3Addr;
diff --git a/llvm/lib/Target/X86/X86FixupLEAs.cpp b/llvm/lib/Target/X86/X86FixupLEAs.cpp
index d33a754ae2d09..dd052eaa59fda 100644
--- a/llvm/lib/Target/X86/X86FixupLEAs.cpp
+++ b/llvm/lib/Target/X86/X86FixupLEAs.cpp
@@ -120,6 +120,9 @@ class FixupLEAsImpl {
MachineInstr *postRAConvertToLEA(MachineBasicBlock &MBB,
MachineBasicBlock::iterator &MBBI) const;
+ /// Fold adjacent mov + LEA-able ALU operations into a single LEA.
+ bool foldCopyToLEA(MachineBasicBlock &MBB) const;
+
public:
FixupLEAsImpl(ProfileSummaryInfo *PSI, MachineBlockFrequencyInfo *MBFI)
: PSI(PSI), MBFI(MBFI) {}
@@ -229,6 +232,40 @@ static bool isLEA(unsigned Opcode) {
Opcode == X86::LEA64_32r;
}
+static MachineBasicBlock::iterator
+getPrevNonDebugInstr(MachineBasicBlock &MBB, MachineBasicBlock::iterator I) {
+ while (I != MBB.begin()) {
+ --I;
+ if (!I->isDebugInstr())
+ return I;
+ }
+ return MBB.end();
+}
+
+static bool isLEAableFromCopy(unsigned Opcode) {
+ switch (Opcode) {
+ default:
+ return false;
+ case X86::SHL64ri:
+ case X86::SHL32ri:
+ case X86::INC64r:
+ case X86::INC32r:
+ case X86::DEC64r:
+ case X86::DEC32r:
+ case X86::ADD64rr:
+ case X86::ADD64rr_DB:
+ case X86::ADD32rr:
+ case X86::ADD32rr_DB:
+ case X86::ADD64ri32:
+ case X86::ADD64ri32_DB:
+ case X86::ADD32ri:
+ case X86::ADD32ri_DB:
+ case X86::SUB64ri32:
+ case X86::SUB32ri:
+ return true;
+ }
+}
+
bool FixupLEAsImpl::runOnMachineFunction(MachineFunction &MF) {
const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
bool IsSlowLEA = ST.slowLEA();
@@ -244,6 +281,8 @@ bool FixupLEAsImpl::runOnMachineFunction(MachineFunction &MF) {
LLVM_DEBUG(dbgs() << "Start X86FixupLEAs\n";);
for (MachineBasicBlock &MBB : MF) {
+ foldCopyToLEA(MBB);
+
// First pass. Try to remove or optimize existing LEAs.
bool OptIncDecPerBB =
OptIncDec || llvm::shouldOptimizeForSize(&MBB, PSI, MBFI);
@@ -273,6 +312,83 @@ bool FixupLEAsImpl::runOnMachineFunction(MachineFunction &MF) {
return true;
}
+bool FixupLEAsImpl::foldCopyToLEA(MachineBasicBlock &MBB) const {
+ MachineFunction &MF = *MBB.getParent();
+ bool Changed = false;
+
+ for (auto I = MBB.begin(); I != MBB.end();) {
+ MachineInstr &MI = *I;
+ if (!isLEAableFromCopy(MI.getOpcode()) || !MI.isConvertibleTo3Addr()) {
+ ++I;
+ continue;
+ }
+
+ auto Prev = getPrevNonDebugInstr(MBB, I);
+ if (Prev == MBB.end()) {
+ ++I;
+ continue;
+ }
+
+ Register DstReg = MI.getOperand(0).getReg();
+ if ((Prev->getOpcode() != X86::MOV32rr &&
+ Prev->getOpcode() != X86::MOV64rr) ||
+ Prev->getOperand(0).getReg() != DstReg || !MI.getOperand(1).isReg() ||
+ MI.getOperand(1).getReg() != DstReg) {
+ ++I;
+ continue;
+ }
+
+ MachineInstr *TmpMI = MF.CloneMachineInstr(&MI);
+ MBB.insert(I, TmpMI);
+
+ Register SrcReg = Prev->getOperand(1).getReg();
+ SmallVector<MachineOperand *, 2> ReplacedOps;
+ for (MachineOperand &MO : TmpMI->operands()) {
+ if (!MO.isReg() || MO.isDef() || MO.getReg() != DstReg)
+ continue;
+ MO.setReg(SrcReg);
+ MO.setIsKill(false);
+ ReplacedOps.push_back(&MO);
+ }
+
+ if (ReplacedOps.empty()) {
+ MBB.erase(TmpMI);
+ ++I;
+ continue;
+ }
+
+ bool HasKillOnSrcReg = false;
+ for (MachineOperand &MO : TmpMI->operands()) {
+ if (&MO == ReplacedOps.back())
+ continue;
+ if (MO.isReg() && !MO.isDef() && MO.getReg() == SrcReg && MO.isKill()) {
+ HasKillOnSrcReg = true;
+ break;
+ }
+ }
+ if (Prev->getOperand(1).isKill() && !HasKillOnSrcReg)
+ ReplacedOps.back()->setIsKill(true);
+
+ MachineInstr *NewMI = TII->convertToThreeAddress(*TmpMI, nullptr, nullptr);
+ MBB.erase(TmpMI);
+ if (!NewMI) {
+ ++I;
+ continue;
+ }
+
+ ++NumLEAs;
+ Changed = true;
+ MBB.getParent()->substituteDebugValuesForInst(*Prev, *NewMI, 1);
+ MBB.getParent()->substituteDebugValuesForInst(MI, *NewMI, 1);
+
+ auto EraseMI = I++;
+ MBB.erase(Prev);
+ MBB.erase(EraseMI);
+ }
+
+ return Changed;
+}
+
FixupLEAsImpl::RegUsageState
FixupLEAsImpl::usesRegister(MachineOperand &p, MachineBasicBlock::iterator I) {
RegUsageState RegUsage = RU_NotUsed;
diff --git a/llvm/test/CodeGen/X86/mov-lea-folds.ll b/llvm/test/CodeGen/X86/mov-lea-folds.ll
index b9ac6076e8b89..08fd9365528a1 100644
--- a/llvm/test/CodeGen/X86/mov-lea-folds.ll
+++ b/llvm/test/CodeGen/X86/mov-lea-folds.ll
@@ -6,8 +6,7 @@ define i64 @mov_dec(<32 x i8> %x) {
; CHECK-LABEL: mov_dec:
; CHECK: # %bb.0:
; CHECK-NEXT: vpmovmskb %ymm0, %ecx
-; CHECK-NEXT: movl %ecx, %eax
-; CHECK-NEXT: decl %eax
+; CHECK-NEXT: leal -1(%rcx), %eax
; CHECK-NEXT: shlq $32, %rcx
; CHECK-NEXT: orq %rcx, %rax
; CHECK-NEXT: vzeroupper
@@ -26,8 +25,7 @@ define i64 @mov_inc(<32 x i8> %x) {
; CHECK-LABEL: mov_inc:
; CHECK: # %bb.0:
; CHECK-NEXT: vpmovmskb %ymm0, %ecx
-; CHECK-NEXT: movl %ecx, %eax
-; CHECK-NEXT: incl %eax
+; CHECK-NEXT: leal 1(%rcx), %eax
; CHECK-NEXT: shlq $32, %rcx
; CHECK-NEXT: orq %rcx, %rax
; CHECK-NEXT: vzeroupper
@@ -46,8 +44,7 @@ define i64 @mov_add_imm(<32 x i8> %x) {
; CHECK-LABEL: mov_add_imm:
; CHECK: # %bb.0:
; CHECK-NEXT: vpmovmskb %ymm0, %ecx
-; CHECK-NEXT: movl %ecx, %eax
-; CHECK-NEXT: addl $5, %eax
+; CHECK-NEXT: leal 5(%rcx), %eax
; CHECK-NEXT: shlq $32, %rcx
; CHECK-NEXT: orq %rcx, %rax
; CHECK-NEXT: vzeroupper
@@ -86,8 +83,7 @@ define i64 @mov_shl(<32 x i8> %x) {
; CHECK-LABEL: mov_shl:
; CHECK: # %bb.0:
; CHECK-NEXT: vpmovmskb %ymm0, %ecx
-; CHECK-NEXT: movl %ecx, %eax
-; CHECK-NEXT: shll $2, %eax
+; CHECK-NEXT: leal (,%rcx,4), %eax
; CHECK-NEXT: shlq $32, %rcx
; CHECK-NEXT: orq %rcx, %rax
; CHECK-NEXT: vzeroupper
@@ -101,3 +97,41 @@ define i64 @mov_shl(<32 x i8> %x) {
%res = or disjoint i64 %shl, %shl_op.ext
ret i64 %res
}
+
+define i64 @mov_sub_imm(<32 x i8> %x) {
+; CHECK-LABEL: mov_sub_imm:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovmskb %ymm0, %ecx
+; CHECK-NEXT: leal -5(%rcx), %eax
+; CHECK-NEXT: shlq $32, %rcx
+; CHECK-NEXT: orq %rcx, %rax
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ %cmp = icmp slt <32 x i8> %x, zeroinitializer
+ %mvmsk = bitcast <32 x i1> %cmp to i32
+ %sub = add i32 %mvmsk, -5
+ %ext = zext i32 %mvmsk to i64
+ %shl = shl nuw i64 %ext, 32
+ %sub.ext = zext i32 %sub to i64
+ %res = or disjoint i64 %shl, %sub.ext
+ ret i64 %res
+}
+
+define i64 @mov_add_self(<32 x i8> %x) {
+; CHECK-LABEL: mov_add_self:
+; CHECK: # %bb.0:
+; CHECK-NEXT: vpmovmskb %ymm0, %ecx
+; CHECK-NEXT: leal (%rcx,%rcx), %eax
+; CHECK-NEXT: shlq $32, %rcx
+; CHECK-NEXT: orq %rcx, %rax
+; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: retq
+ %cmp = icmp slt <32 x i8> %x, zeroinitializer
+ %mvmsk = bitcast <32 x i1> %cmp to i32
+ %add = add i32 %mvmsk, %mvmsk
+ %ext = zext i32 %mvmsk to i64
+ %shl = shl nuw i64 %ext, 32
+ %add.ext = zext i32 %add to i64
+ %res = or disjoint i64 %shl, %add.ext
+ ret i64 %res
+}
More information about the llvm-commits
mailing list