[llvm] [CodeGenPrepare] Recursively sink expensive operands when converting select to branch (PR #197869)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 23 06:10:10 PDT 2026
https://github.com/ayrai-gb updated https://github.com/llvm/llvm-project/pull/197869
>From 9a0977d9edceadffb4ee604829f512902162befa Mon Sep 17 00:00:00 2001
From: Ayush Rai <your-email at amd.com>
Date: Fri, 15 May 2026 10:31:25 +0530
Subject: [PATCH 1/8] [CodeGenPrepare] Recursively sink expensive operands when
converting select to branch When converting select instructions to branches,
improve operand sinking by: - Adding recursive sinking through
sinkSelectOperand1() with multi-use tracking - Handling grouped selects that
share the same condition - Skipping div/rem-by-constant (they lower to cheap
mul+shift sequences) This enables more profitable select-to-branch
conversions when expensive operands like divisions or loads can be sunk into
the taken branch.
Co-authored-by: Cursor <cursoragent at cursor.com>
---
llvm/lib/CodeGen/CodeGenPrepare.cpp | 103 ++++++++++++++++++++++------
1 file changed, 81 insertions(+), 22 deletions(-)
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index ff40f210c5790..b858e9598b030 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -7622,24 +7622,70 @@ bool CodeGenPrepare::optimizeLoadExt(LoadInst *Load) {
return true;
}
-/// Check if V (an operand of a select instruction) is an expensive instruction
-/// that is only used once.
-static bool sinkSelectOperand(const TargetTransformInfo *TTI, Value *V) {
+/// Recursively check if V or any instruction in its sinkable chain is
+/// expensive. Instructions are sinkable when all of their uses are accounted
+/// for on one side (true/false) of the select group. RemainingUses tracks
+/// unaccounted uses for multi-use instructions on that side. When Chain is
+/// non-null, sinkable instructions are collected in pre-order (users before
+/// operands); the caller must reverse the chain to get the correct sinking
+/// order.
+static bool sinkSelectOperand1(const TargetTransformInfo *TTI, Value *V,
+ DenseMap<Instruction *, unsigned> &RemainingUses,
+ SmallVectorImpl<Instruction *> *Chain = nullptr,
+ unsigned Depth = 4) {
auto *I = dyn_cast<Instruction>(V);
- // If it's safe to speculatively execute, then it should not have side
- // effects; therefore, it's safe to sink and possibly *not* execute.
- return I && I->hasOneUse() && isSafeToSpeculativelyExecute(I) &&
- TTI->isExpensiveToSpeculativelyExecute(I);
+ if (!I || Depth == 0)
+ return false;
+ if (!isSafeToSpeculativelyExecute(I))
+ return false;
+
+ if (!I->hasOneUse()) {
+ auto [It, _] = RemainingUses.try_emplace(I, I->getNumUses());
+ assert(It->second > 0 && "this operand has atleast one use.");
+ --It->second;
+ if (It->second != 0)
+ return false;
+ }
+
+ if (Chain)
+ Chain->push_back(I);
+
+ bool HasExpensive = TTI->isExpensiveToSpeculativelyExecute(I);
+ // Integer div/rem by a constant will be lowered to mul+shift by ISel,
+ // not an actual idiv. Don't treat it as expensive to speculatively execute.
+ if (HasExpensive) {
+ unsigned Op = I->getOpcode();
+ if ((Op == Instruction::SDiv || Op == Instruction::UDiv ||
+ Op == Instruction::SRem || Op == Instruction::URem) &&
+ isa<ConstantInt>(I->getOperand(1)))
+ HasExpensive = false;
+ }
+ if (HasExpensive) {
+ LLVM_DEBUG(dbgs() << "CGP: Found expensive instruction to sink: " << *I
+ << "\n");
+ if (!Chain)
+ return true;
+ }
+ for (Value *Op : I->operands()) {
+ if (sinkSelectOperand1(TTI, Op, RemainingUses, Chain, Depth - 1)) {
+ HasExpensive = true;
+ if (!Chain)
+ return true;
+ }
+ }
+ return HasExpensive;
}
/// Returns true if a SelectInst should be turned into an explicit branch.
static bool isFormingBranchFromSelectProfitable(const TargetTransformInfo *TTI,
const TargetLowering *TLI,
- SelectInst *SI) {
+ ArrayRef<SelectInst *> ASI) {
// If even a predictable select is cheap, then a branch can't be cheaper.
if (!TLI->isPredictableSelectExpensive())
return false;
+ SelectInst *SI = ASI.front();
+
// FIXME: This should use the same heuristics as IfConversion to determine
// whether a select is better represented as a branch.
@@ -7651,8 +7697,12 @@ static bool isFormingBranchFromSelectProfitable(const TargetTransformInfo *TTI,
uint64_t Sum = TrueWeight + FalseWeight;
if (Sum != 0) {
auto Probability = BranchProbability::getBranchProbability(Max, Sum);
- if (Probability > TTI->getPredictableBranchThreshold())
+ if (Probability > TTI->getPredictableBranchThreshold()) {
+ LLVM_DEBUG(dbgs() << "CGP: Forming branch from select due to profile "
+ "data: "
+ << *SI << " (probability " << Probability << ")\n");
return true;
+ }
}
}
@@ -7661,14 +7711,23 @@ static bool isFormingBranchFromSelectProfitable(const TargetTransformInfo *TTI,
// If a branch is predictable, an out-of-order CPU can avoid blocking on its
// comparison condition. If the compare has more than one use, there's
// probably another cmov or setcc around, so it's not worth emitting a branch.
- if (!Cmp || !Cmp->hasOneUse())
+ // For grouped selects, allow exactly one use per select.
+ if (!Cmp || !Cmp->hasNUses(ASI.size()))
return false;
- // If either operand of the select is expensive and only needed on one side
- // of the select, we should form a branch.
- if (sinkSelectOperand(TTI, SI->getTrueValue()) ||
- sinkSelectOperand(TTI, SI->getFalseValue()))
- return true;
+ // If any operand in the select group is expensive and only needed on one
+ // side, we should form a branch. Check recursively through each select's
+ // operand chains, using per-side use tracking for multi-use instructions.
+ DenseMap<Instruction *, unsigned> TrueRemainingUses, FalseRemainingUses;
+ for (SelectInst *SI : ASI) {
+ if (sinkSelectOperand1(TTI, SI->getTrueValue(), TrueRemainingUses) ||
+ sinkSelectOperand1(TTI, SI->getFalseValue(), FalseRemainingUses)) {
+ LLVM_DEBUG(dbgs() << "CGP: Forming branch from select due to expensive "
+ "operand: "
+ << *SI << "\n");
+ return true;
+ }
+ }
return false;
}
@@ -7807,7 +7866,7 @@ bool CodeGenPrepare::optimizeSelectInst(SelectInst *SI) {
SelectKind = TargetLowering::ScalarValSelect;
if (TLI->isSelectSupported(SelectKind) &&
- (!isFormingBranchFromSelectProfitable(TTI, TLI, SI) ||
+ (!isFormingBranchFromSelectProfitable(TTI, TLI, ASI) ||
llvm::shouldOptimizeForSize(SI->getParent(), PSI, BFI)))
return false;
@@ -7842,11 +7901,11 @@ bool CodeGenPrepare::optimizeSelectInst(SelectInst *SI) {
// Collect values that go on the true side and the values that go on the false
// side.
SmallVector<Instruction *> TrueInstrs, FalseInstrs;
+ DenseMap<Instruction *, unsigned> TrueRemainingUses, FalseRemainingUses;
for (SelectInst *SI : ASI) {
- if (Value *V = SI->getTrueValue(); sinkSelectOperand(TTI, V))
- TrueInstrs.push_back(cast<Instruction>(V));
- if (Value *V = SI->getFalseValue(); sinkSelectOperand(TTI, V))
- FalseInstrs.push_back(cast<Instruction>(V));
+ sinkSelectOperand1(TTI, SI->getTrueValue(), TrueRemainingUses, &TrueInstrs);
+ sinkSelectOperand1(TTI, SI->getFalseValue(), FalseRemainingUses,
+ &FalseInstrs);
}
// Split the select block, according to how many (if any) values go on each
@@ -7910,9 +7969,9 @@ bool CodeGenPrepare::optimizeSelectInst(SelectInst *SI) {
// Sink expensive instructions into the conditional blocks to avoid executing
// them speculatively.
- for (Instruction *I : TrueInstrs)
+ for (Instruction *I : llvm::reverse(TrueInstrs))
I->moveBefore(TrueBranch->getIterator());
- for (Instruction *I : FalseInstrs)
+ for (Instruction *I : llvm::reverse(FalseInstrs))
I->moveBefore(FalseBranch->getIterator());
// If we did not create a new block for one of the 'true' or 'false' paths
>From abf90504996531f577b311c0e3e945e35efd278e Mon Sep 17 00:00:00 2001
From: Ayush Rai <your-email at amd.com>
Date: Fri, 15 May 2026 17:24:40 +0530
Subject: [PATCH 2/8] Update test expectations for recursive select operand
sinking
---
llvm/test/CodeGen/X86/bit-manip-i128.ll | 93 +-
llvm/test/CodeGen/X86/bit-manip-i256.ll | 1408 ++++++-----
llvm/test/CodeGen/X86/bit-manip-i512.ll | 2141 ++++++++++-------
llvm/test/CodeGen/X86/bsr.ll | 157 +-
llvm/test/CodeGen/X86/freeze-unary.ll | 9 +-
llvm/test/CodeGen/X86/pseudo_cmov_lower.ll | 20 +-
llvm/test/CodeGen/X86/pseudo_cmov_lower2.ll | 21 +-
.../CodeGen/X86/shrink-wrap-chkstk-x86_64.ll | 25 +-
.../CodeGenPrepare/X86/optimizeSelect-DT.ll | 17 +-
.../Transforms/CodeGenPrepare/X86/select.ll | 32 +-
10 files changed, 2242 insertions(+), 1681 deletions(-)
diff --git a/llvm/test/CodeGen/X86/bit-manip-i128.ll b/llvm/test/CodeGen/X86/bit-manip-i128.ll
index 451ce3b1b7578..150c3486a6091 100644
--- a/llvm/test/CodeGen/X86/bit-manip-i128.ll
+++ b/llvm/test/CodeGen/X86/bit-manip-i128.ll
@@ -901,6 +901,10 @@ define i128 @bzhi_i128_load(ptr %p0, i128 %idx) nounwind {
define i128 @isolate_msb_i128(i128 %a0, i128 %idx) nounwind {
; SSE-LABEL: isolate_msb_i128:
; SSE: # %bb.0:
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: orq %rsi, %rax
+; SSE-NEXT: je .LBB15_1
+; SSE-NEXT: # %bb.2: # %select.false.sink
; SSE-NEXT: bsrq %rsi, %rax
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: bsrq %rdi, %rcx
@@ -912,14 +916,15 @@ define i128 @isolate_msb_i128(i128 %a0, i128 %idx) nounwind {
; SSE-NEXT: movabsq $-9223372036854775808, %rdx # imm = 0x8000000000000000
; SSE-NEXT: xorl %eax, %eax
; SSE-NEXT: shrdq %cl, %rdx, %rax
-; SSE-NEXT: xorl %r8d, %r8d
+; SSE-NEXT: xorl %esi, %esi
; SSE-NEXT: shrq %cl, %rdx
; SSE-NEXT: testb $64, %cl
; SSE-NEXT: cmovneq %rdx, %rax
-; SSE-NEXT: cmovneq %r8, %rdx
-; SSE-NEXT: orq %rsi, %rdi
-; SSE-NEXT: cmoveq %r8, %rax
-; SSE-NEXT: cmoveq %r8, %rdx
+; SSE-NEXT: cmovneq %rsi, %rdx
+; SSE-NEXT: retq
+; SSE-NEXT: .LBB15_1:
+; SSE-NEXT: xorl %eax, %eax
+; SSE-NEXT: xorl %edx, %edx
; SSE-NEXT: retq
;
; AVX-LABEL: isolate_msb_i128:
@@ -952,13 +957,15 @@ define i128 @isolate_msb_i128(i128 %a0, i128 %idx) nounwind {
define i128 @isolate_msb_i128_vector(<2 x i64> %v0, i128 %idx) nounwind {
; SSE2-LABEL: isolate_msb_i128_vector:
; SSE2: # %bb.0:
-; SSE2-NEXT: movq %xmm0, %rax
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; SSE2-NEXT: movq %xmm1, %rcx
; SSE2-NEXT: pxor %xmm1, %xmm1
; SSE2-NEXT: pcmpeqd %xmm0, %xmm1
-; SSE2-NEXT: movmskps %xmm1, %esi
-; SSE2-NEXT: xorl $15, %esi
+; SSE2-NEXT: movmskps %xmm1, %eax
+; SSE2-NEXT: xorl $15, %eax
+; SSE2-NEXT: je .LBB16_1
+; SSE2-NEXT: # %bb.2: # %select.false.sink
+; SSE2-NEXT: movq %xmm0, %rax
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; SSE2-NEXT: movq %xmm0, %rcx
; SSE2-NEXT: bsrq %rcx, %rdx
; SSE2-NEXT: xorl $63, %edx
; SSE2-NEXT: bsrq %rax, %rax
@@ -970,27 +977,31 @@ define i128 @isolate_msb_i128_vector(<2 x i64> %v0, i128 %idx) nounwind {
; SSE2-NEXT: movabsq $-9223372036854775808, %rdx # imm = 0x8000000000000000
; SSE2-NEXT: xorl %eax, %eax
; SSE2-NEXT: shrdq %cl, %rdx, %rax
-; SSE2-NEXT: xorl %edi, %edi
+; SSE2-NEXT: xorl %esi, %esi
; SSE2-NEXT: shrq %cl, %rdx
; SSE2-NEXT: testb $64, %cl
; SSE2-NEXT: cmovneq %rdx, %rax
-; SSE2-NEXT: cmovneq %rdi, %rdx
-; SSE2-NEXT: testl %esi, %esi
-; SSE2-NEXT: cmoveq %rdi, %rax
-; SSE2-NEXT: cmoveq %rdi, %rdx
+; SSE2-NEXT: cmovneq %rsi, %rdx
+; SSE2-NEXT: retq
+; SSE2-NEXT: .LBB16_1:
+; SSE2-NEXT: xorl %eax, %eax
+; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: retq
;
; SSE42-LABEL: isolate_msb_i128_vector:
; SSE42: # %bb.0:
-; SSE42-NEXT: movq %xmm0, %rax
-; SSE42-NEXT: pextrq $1, %xmm0, %rcx
-; SSE42-NEXT: bsrq %rcx, %rdx
+; SSE42-NEXT: ptest %xmm0, %xmm0
+; SSE42-NEXT: je .LBB16_1
+; SSE42-NEXT: # %bb.2: # %select.false.sink
+; SSE42-NEXT: pextrq $1, %xmm0, %rax
+; SSE42-NEXT: movq %xmm0, %rcx
+; SSE42-NEXT: bsrq %rax, %rdx
; SSE42-NEXT: xorl $63, %edx
-; SSE42-NEXT: bsrq %rax, %rax
-; SSE42-NEXT: xorl $63, %eax
-; SSE42-NEXT: orb $64, %al
-; SSE42-NEXT: testq %rcx, %rcx
-; SSE42-NEXT: movzbl %al, %ecx
+; SSE42-NEXT: bsrq %rcx, %rcx
+; SSE42-NEXT: xorl $63, %ecx
+; SSE42-NEXT: orb $64, %cl
+; SSE42-NEXT: testq %rax, %rax
+; SSE42-NEXT: movzbl %cl, %ecx
; SSE42-NEXT: cmovnel %edx, %ecx
; SSE42-NEXT: movabsq $-9223372036854775808, %rdx # imm = 0x8000000000000000
; SSE42-NEXT: xorl %eax, %eax
@@ -1000,9 +1011,10 @@ define i128 @isolate_msb_i128_vector(<2 x i64> %v0, i128 %idx) nounwind {
; SSE42-NEXT: testb $64, %cl
; SSE42-NEXT: cmovneq %rdx, %rax
; SSE42-NEXT: cmovneq %rsi, %rdx
-; SSE42-NEXT: ptest %xmm0, %xmm0
-; SSE42-NEXT: cmoveq %rsi, %rax
-; SSE42-NEXT: cmoveq %rsi, %rdx
+; SSE42-NEXT: retq
+; SSE42-NEXT: .LBB16_1:
+; SSE42-NEXT: xorl %eax, %eax
+; SSE42-NEXT: xorl %edx, %edx
; SSE42-NEXT: retq
;
; AVX-LABEL: isolate_msb_i128_vector:
@@ -1038,27 +1050,32 @@ define i128 @isolate_msb_i128_vector(<2 x i64> %v0, i128 %idx) nounwind {
define i128 @isolate_msb_i128_load(ptr %p0, i128 %idx) nounwind {
; SSE-LABEL: isolate_msb_i128_load:
; SSE: # %bb.0:
-; SSE-NEXT: movq (%rdi), %rsi
-; SSE-NEXT: movq 8(%rdi), %rdi
-; SSE-NEXT: bsrq %rdi, %rax
-; SSE-NEXT: xorl $63, %eax
-; SSE-NEXT: bsrq %rsi, %rcx
+; SSE-NEXT: movq (%rdi), %rcx
+; SSE-NEXT: movq 8(%rdi), %rax
+; SSE-NEXT: movq %rcx, %rdx
+; SSE-NEXT: orq %rax, %rdx
+; SSE-NEXT: je .LBB17_1
+; SSE-NEXT: # %bb.2: # %select.false.sink
+; SSE-NEXT: bsrq %rax, %rdx
+; SSE-NEXT: xorl $63, %edx
+; SSE-NEXT: bsrq %rcx, %rcx
; SSE-NEXT: xorl $63, %ecx
; SSE-NEXT: orb $64, %cl
-; SSE-NEXT: testq %rdi, %rdi
+; SSE-NEXT: testq %rax, %rax
; SSE-NEXT: movzbl %cl, %ecx
-; SSE-NEXT: cmovnel %eax, %ecx
+; SSE-NEXT: cmovnel %edx, %ecx
; SSE-NEXT: movabsq $-9223372036854775808, %rdx # imm = 0x8000000000000000
; SSE-NEXT: xorl %eax, %eax
; SSE-NEXT: shrdq %cl, %rdx, %rax
-; SSE-NEXT: xorl %r8d, %r8d
+; SSE-NEXT: xorl %esi, %esi
; SSE-NEXT: shrq %cl, %rdx
; SSE-NEXT: testb $64, %cl
; SSE-NEXT: cmovneq %rdx, %rax
-; SSE-NEXT: cmovneq %r8, %rdx
-; SSE-NEXT: orq %rdi, %rsi
-; SSE-NEXT: cmoveq %r8, %rax
-; SSE-NEXT: cmoveq %r8, %rdx
+; SSE-NEXT: cmovneq %rsi, %rdx
+; SSE-NEXT: retq
+; SSE-NEXT: .LBB17_1:
+; SSE-NEXT: xorl %eax, %eax
+; SSE-NEXT: xorl %edx, %edx
; SSE-NEXT: retq
;
; AVX-LABEL: isolate_msb_i128_load:
diff --git a/llvm/test/CodeGen/X86/bit-manip-i256.ll b/llvm/test/CodeGen/X86/bit-manip-i256.ll
index 042b7d428ba5c..6896cb0e9abf5 100644
--- a/llvm/test/CodeGen/X86/bit-manip-i256.ll
+++ b/llvm/test/CodeGen/X86/bit-manip-i256.ll
@@ -2407,267 +2407,288 @@ define i256 @bzhi_i256_load(ptr %p0, i256 %idx) nounwind {
define i256 @isolate_msb_i256(i256 %a0, i256 %idx) nounwind {
; SSE-LABEL: isolate_msb_i256:
; SSE: # %bb.0:
-; SSE-NEXT: pushq %rbx
-; SSE-NEXT: movq %rcx, %rax
-; SSE-NEXT: movq %rdx, %r9
-; SSE-NEXT: orq %r8, %r9
-; SSE-NEXT: bsrq %rsi, %rcx
-; SSE-NEXT: orq %rax, %rsi
-; SSE-NEXT: bsrq %r8, %r10
-; SSE-NEXT: xorq $63, %r10
-; SSE-NEXT: bsrq %rax, %r11
-; SSE-NEXT: xorq $63, %r11
-; SSE-NEXT: orq $64, %r11
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: movq %rdx, %rdi
+; SSE-NEXT: orq %r8, %rdi
+; SSE-NEXT: movq %rsi, %r9
+; SSE-NEXT: orq %rcx, %r9
+; SSE-NEXT: orq %rdi, %r9
+; SSE-NEXT: je .LBB15_1
+; SSE-NEXT: # %bb.2: # %select.false.sink
+; SSE-NEXT: bsrq %r8, %rdi
+; SSE-NEXT: xorq $63, %rdi
+; SSE-NEXT: bsrq %rcx, %r9
+; SSE-NEXT: xorq $63, %r9
+; SSE-NEXT: orq $64, %r9
; SSE-NEXT: testq %r8, %r8
-; SSE-NEXT: cmovneq %r10, %r11
-; SSE-NEXT: bsrq %rdx, %r10
-; SSE-NEXT: xorq $63, %r10
+; SSE-NEXT: cmovneq %rdi, %r9
+; SSE-NEXT: bsrq %rdx, %rdi
+; SSE-NEXT: xorq $63, %rdi
+; SSE-NEXT: movq %rcx, %r10
+; SSE-NEXT: bsrq %rsi, %rcx
; SSE-NEXT: xorq $63, %rcx
; SSE-NEXT: orq $64, %rcx
; SSE-NEXT: testq %rdx, %rdx
-; SSE-NEXT: cmovneq %r10, %rcx
+; SSE-NEXT: cmovneq %rdi, %rcx
; SSE-NEXT: orq $128, %rcx
-; SSE-NEXT: orq %r8, %rax
-; SSE-NEXT: cmovneq %r11, %rcx
+; SSE-NEXT: orq %r8, %r10
+; SSE-NEXT: cmovneq %r9, %rcx
; SSE-NEXT: xorps %xmm0, %xmm0
; SSE-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
; SSE-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; SSE-NEXT: movabsq $-9223372036854775808, %rax # imm = 0x8000000000000000
-; SSE-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movabsq $-9223372036854775808, %rdx # imm = 0x8000000000000000
+; SSE-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
; SSE-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; SSE-NEXT: movl %ecx, %eax
-; SSE-NEXT: shrb $6, %al
-; SSE-NEXT: movzbl %al, %eax
+; SSE-NEXT: movl %ecx, %edx
+; SSE-NEXT: shrb $6, %dl
+; SSE-NEXT: movzbl %dl, %edi
; SSE-NEXT: movq $0, -{{[0-9]+}}(%rsp)
-; SSE-NEXT: movq -40(%rsp,%rax,8), %rdx
-; SSE-NEXT: movq -48(%rsp,%rax,8), %r8
-; SSE-NEXT: movq %r8, %r10
-; SSE-NEXT: shrdq %cl, %rdx, %r10
-; SSE-NEXT: movq -56(%rsp,%rax,8), %r11
-; SSE-NEXT: movq %r11, %rbx
-; SSE-NEXT: shrdq %cl, %r8, %rbx
-; SSE-NEXT: movq -64(%rsp,%rax,8), %r8
+; SSE-NEXT: movq -48(%rsp,%rdi,8), %rdx
+; SSE-NEXT: movq -56(%rsp,%rdi,8), %r9
+; SSE-NEXT: movq %r9, %rsi
+; SSE-NEXT: shrdq %cl, %rdx, %rsi
+; SSE-NEXT: movq -64(%rsp,%rdi,8), %r10
+; SSE-NEXT: movq %r10, %r8
+; SSE-NEXT: shrdq %cl, %r9, %r8
+; SSE-NEXT: movq -72(%rsp,%rdi,8), %rdi
; SSE-NEXT: shrq %cl, %rdx
; SSE-NEXT: # kill: def $cl killed $cl killed $rcx
-; SSE-NEXT: shrdq %cl, %r11, %r8
-; SSE-NEXT: xorl %ecx, %ecx
-; SSE-NEXT: orq %r9, %rsi
-; SSE-NEXT: cmoveq %rcx, %rbx
-; SSE-NEXT: cmoveq %rcx, %r10
-; SSE-NEXT: cmoveq %rcx, %r8
-; SSE-NEXT: movq %rdi, %rax
-; SSE-NEXT: cmoveq %rcx, %rdx
-; SSE-NEXT: movq %rdx, 24(%rdi)
-; SSE-NEXT: movq %r10, 16(%rdi)
-; SSE-NEXT: movq %rbx, 8(%rdi)
-; SSE-NEXT: movq %r8, (%rdi)
-; SSE-NEXT: popq %rbx
+; SSE-NEXT: shrdq %cl, %r10, %rdi
+; SSE-NEXT: jmp .LBB15_3
+; SSE-NEXT: .LBB15_1:
+; SSE-NEXT: xorl %edi, %edi
+; SSE-NEXT: xorl %r8d, %r8d
+; SSE-NEXT: xorl %esi, %esi
+; SSE-NEXT: xorl %edx, %edx
+; SSE-NEXT: .LBB15_3: # %select.end
+; SSE-NEXT: movq %rdi, (%rax)
+; SSE-NEXT: movq %r8, 8(%rax)
+; SSE-NEXT: movq %rsi, 16(%rax)
+; SSE-NEXT: movq %rdx, 24(%rax)
; SSE-NEXT: retq
;
; AVX2-LABEL: isolate_msb_i256:
; AVX2: # %bb.0:
-; AVX2-NEXT: pushq %rbx
-; AVX2-NEXT: movq %rcx, %rax
-; AVX2-NEXT: movq %rdx, %r9
-; AVX2-NEXT: orq %r8, %r9
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: movq %rdx, %rdi
+; AVX2-NEXT: orq %r8, %rdi
+; AVX2-NEXT: movq %rsi, %r9
+; AVX2-NEXT: orq %rcx, %r9
+; AVX2-NEXT: orq %rdi, %r9
+; AVX2-NEXT: je .LBB15_1
+; AVX2-NEXT: # %bb.2: # %select.false.sink
+; AVX2-NEXT: xorl %edi, %edi
+; AVX2-NEXT: lzcntq %r8, %rdi
+; AVX2-NEXT: xorl %r9d, %r9d
+; AVX2-NEXT: lzcntq %rcx, %r9
+; AVX2-NEXT: addq $64, %r9
+; AVX2-NEXT: testq %r8, %r8
+; AVX2-NEXT: cmovneq %rdi, %r9
+; AVX2-NEXT: xorl %edi, %edi
+; AVX2-NEXT: lzcntq %rdx, %rdi
+; AVX2-NEXT: movq %rcx, %r10
; AVX2-NEXT: xorl %ecx, %ecx
; AVX2-NEXT: lzcntq %rsi, %rcx
-; AVX2-NEXT: orq %rax, %rsi
-; AVX2-NEXT: lzcntq %r8, %r10
-; AVX2-NEXT: lzcntq %rax, %r11
-; AVX2-NEXT: addq $64, %r11
-; AVX2-NEXT: testq %r8, %r8
-; AVX2-NEXT: cmovneq %r10, %r11
-; AVX2-NEXT: xorl %r10d, %r10d
-; AVX2-NEXT: lzcntq %rdx, %r10
; AVX2-NEXT: addq $64, %rcx
; AVX2-NEXT: testq %rdx, %rdx
-; AVX2-NEXT: cmovneq %r10, %rcx
+; AVX2-NEXT: cmovneq %rdi, %rcx
; AVX2-NEXT: subq $-128, %rcx
-; AVX2-NEXT: orq %r8, %rax
-; AVX2-NEXT: cmovneq %r11, %rcx
-; AVX2-NEXT: vmovaps {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
-; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: orq %r8, %r10
+; AVX2-NEXT: cmovneq %r9, %rcx
; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movl %ecx, %eax
-; AVX2-NEXT: shrb $6, %al
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: movq -40(%rsp,%rax,8), %rdx
-; AVX2-NEXT: movq -48(%rsp,%rax,8), %r8
-; AVX2-NEXT: movq %r8, %r10
-; AVX2-NEXT: shrdq %cl, %rdx, %r10
-; AVX2-NEXT: movq -64(%rsp,%rax,8), %r11
-; AVX2-NEXT: movq -56(%rsp,%rax,8), %rax
-; AVX2-NEXT: movq %rax, %rbx
-; AVX2-NEXT: shrdq %cl, %r8, %rbx
-; AVX2-NEXT: shrdq %cl, %rax, %r11
-; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: vmovaps {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
+; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movl %ecx, %edx
+; AVX2-NEXT: shrb $6, %dl
+; AVX2-NEXT: movzbl %dl, %esi
+; AVX2-NEXT: movq -48(%rsp,%rsi,8), %r8
+; AVX2-NEXT: movq -56(%rsp,%rsi,8), %r9
+; AVX2-NEXT: movq %r9, %rdx
+; AVX2-NEXT: shrdq %cl, %r8, %rdx
+; AVX2-NEXT: movq -64(%rsp,%rsi,8), %r10
+; AVX2-NEXT: movq %r10, %rdi
+; AVX2-NEXT: shrdq %cl, %r9, %rdi
+; AVX2-NEXT: movq -72(%rsp,%rsi,8), %rsi
+; AVX2-NEXT: shrxq %rcx, %r8, %r8
+; AVX2-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX2-NEXT: shrdq %cl, %r10, %rsi
+; AVX2-NEXT: jmp .LBB15_3
+; AVX2-NEXT: .LBB15_1:
+; AVX2-NEXT: xorl %esi, %esi
; AVX2-NEXT: xorl %edi, %edi
-; AVX2-NEXT: orq %r9, %rsi
-; AVX2-NEXT: shrxq %rcx, %rdx, %rcx
-; AVX2-NEXT: cmoveq %rdi, %rbx
-; AVX2-NEXT: cmoveq %rdi, %r10
-; AVX2-NEXT: cmoveq %rdi, %r11
-; AVX2-NEXT: cmoveq %rdi, %rcx
-; AVX2-NEXT: movq %rcx, 24(%rax)
-; AVX2-NEXT: movq %r10, 16(%rax)
-; AVX2-NEXT: movq %rbx, 8(%rax)
-; AVX2-NEXT: movq %r11, (%rax)
-; AVX2-NEXT: popq %rbx
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: xorl %r8d, %r8d
+; AVX2-NEXT: .LBB15_3: # %select.end
+; AVX2-NEXT: movq %rsi, (%rax)
+; AVX2-NEXT: movq %rdi, 8(%rax)
+; AVX2-NEXT: movq %rdx, 16(%rax)
+; AVX2-NEXT: movq %r8, 24(%rax)
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512F-LABEL: isolate_msb_i256:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: pushq %rbx
-; AVX512F-NEXT: movq %rcx, %rax
-; AVX512F-NEXT: movq %rdx, %r9
-; AVX512F-NEXT: orq %r8, %r9
-; AVX512F-NEXT: lzcntq %rsi, %rcx
-; AVX512F-NEXT: orq %rax, %rsi
-; AVX512F-NEXT: lzcntq %r8, %r10
-; AVX512F-NEXT: lzcntq %rax, %r11
-; AVX512F-NEXT: addq $64, %r11
+; AVX512F-NEXT: movq %rdi, %rax
+; AVX512F-NEXT: movq %rdx, %rdi
+; AVX512F-NEXT: orq %r8, %rdi
+; AVX512F-NEXT: movq %rsi, %r9
+; AVX512F-NEXT: orq %rcx, %r9
+; AVX512F-NEXT: orq %rdi, %r9
+; AVX512F-NEXT: je .LBB15_1
+; AVX512F-NEXT: # %bb.2: # %select.false.sink
+; AVX512F-NEXT: lzcntq %r8, %rdi
+; AVX512F-NEXT: lzcntq %rcx, %r9
+; AVX512F-NEXT: addq $64, %r9
; AVX512F-NEXT: testq %r8, %r8
-; AVX512F-NEXT: cmovneq %r10, %r11
-; AVX512F-NEXT: lzcntq %rdx, %r10
+; AVX512F-NEXT: cmovneq %rdi, %r9
+; AVX512F-NEXT: lzcntq %rdx, %rdi
+; AVX512F-NEXT: movq %rcx, %r10
+; AVX512F-NEXT: lzcntq %rsi, %rcx
; AVX512F-NEXT: addq $64, %rcx
; AVX512F-NEXT: testq %rdx, %rdx
-; AVX512F-NEXT: cmovneq %r10, %rcx
+; AVX512F-NEXT: cmovneq %rdi, %rcx
; AVX512F-NEXT: subq $-128, %rcx
-; AVX512F-NEXT: orq %r8, %rax
-; AVX512F-NEXT: cmovneq %r11, %rcx
+; AVX512F-NEXT: orq %r8, %r10
+; AVX512F-NEXT: cmovneq %r9, %rcx
; AVX512F-NEXT: vmovaps {{.*#+}} zmm0 = [0,0,0,9223372036854775808,0,0,0,0]
; AVX512F-NEXT: vmovups %zmm0, -{{[0-9]+}}(%rsp)
-; AVX512F-NEXT: movl %ecx, %eax
-; AVX512F-NEXT: shrb $6, %al
-; AVX512F-NEXT: movzbl %al, %eax
-; AVX512F-NEXT: movq -40(%rsp,%rax,8), %rdx
-; AVX512F-NEXT: movq -48(%rsp,%rax,8), %r8
-; AVX512F-NEXT: movq %r8, %r10
-; AVX512F-NEXT: shrdq %cl, %rdx, %r10
-; AVX512F-NEXT: movq -64(%rsp,%rax,8), %r11
-; AVX512F-NEXT: movq -56(%rsp,%rax,8), %rax
-; AVX512F-NEXT: movq %rax, %rbx
-; AVX512F-NEXT: shrdq %cl, %r8, %rbx
-; AVX512F-NEXT: shrdq %cl, %rax, %r11
-; AVX512F-NEXT: movq %rdi, %rax
+; AVX512F-NEXT: movl %ecx, %edx
+; AVX512F-NEXT: shrb $6, %dl
+; AVX512F-NEXT: movzbl %dl, %esi
+; AVX512F-NEXT: movq -48(%rsp,%rsi,8), %r8
+; AVX512F-NEXT: movq -56(%rsp,%rsi,8), %r9
+; AVX512F-NEXT: movq %r9, %rdx
+; AVX512F-NEXT: shrdq %cl, %r8, %rdx
+; AVX512F-NEXT: movq -64(%rsp,%rsi,8), %r10
+; AVX512F-NEXT: movq %r10, %rdi
+; AVX512F-NEXT: shrdq %cl, %r9, %rdi
+; AVX512F-NEXT: movq -72(%rsp,%rsi,8), %rsi
+; AVX512F-NEXT: shrxq %rcx, %r8, %r8
+; AVX512F-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX512F-NEXT: shrdq %cl, %r10, %rsi
+; AVX512F-NEXT: jmp .LBB15_3
+; AVX512F-NEXT: .LBB15_1:
+; AVX512F-NEXT: xorl %esi, %esi
; AVX512F-NEXT: xorl %edi, %edi
-; AVX512F-NEXT: orq %r9, %rsi
-; AVX512F-NEXT: shrxq %rcx, %rdx, %rcx
-; AVX512F-NEXT: cmoveq %rdi, %rbx
-; AVX512F-NEXT: cmoveq %rdi, %r10
-; AVX512F-NEXT: cmoveq %rdi, %r11
-; AVX512F-NEXT: cmoveq %rdi, %rcx
-; AVX512F-NEXT: movq %rcx, 24(%rax)
-; AVX512F-NEXT: movq %r10, 16(%rax)
-; AVX512F-NEXT: movq %rbx, 8(%rax)
-; AVX512F-NEXT: movq %r11, (%rax)
-; AVX512F-NEXT: popq %rbx
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: xorl %r8d, %r8d
+; AVX512F-NEXT: .LBB15_3: # %select.end
+; AVX512F-NEXT: movq %rsi, (%rax)
+; AVX512F-NEXT: movq %rdi, 8(%rax)
+; AVX512F-NEXT: movq %rdx, 16(%rax)
+; AVX512F-NEXT: movq %r8, 24(%rax)
; AVX512F-NEXT: retq
;
; AVX512VL-LABEL: isolate_msb_i256:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: pushq %rbx
-; AVX512VL-NEXT: movq %rcx, %rax
-; AVX512VL-NEXT: movq %rdx, %r9
-; AVX512VL-NEXT: orq %r8, %r9
-; AVX512VL-NEXT: lzcntq %rsi, %rcx
-; AVX512VL-NEXT: orq %rax, %rsi
-; AVX512VL-NEXT: lzcntq %r8, %r10
-; AVX512VL-NEXT: lzcntq %rax, %r11
-; AVX512VL-NEXT: addq $64, %r11
+; AVX512VL-NEXT: movq %rdi, %rax
+; AVX512VL-NEXT: movq %rdx, %rdi
+; AVX512VL-NEXT: orq %r8, %rdi
+; AVX512VL-NEXT: movq %rsi, %r9
+; AVX512VL-NEXT: orq %rcx, %r9
+; AVX512VL-NEXT: orq %rdi, %r9
+; AVX512VL-NEXT: je .LBB15_1
+; AVX512VL-NEXT: # %bb.2: # %select.false.sink
+; AVX512VL-NEXT: lzcntq %r8, %rdi
+; AVX512VL-NEXT: lzcntq %rcx, %r9
+; AVX512VL-NEXT: addq $64, %r9
; AVX512VL-NEXT: testq %r8, %r8
-; AVX512VL-NEXT: cmovneq %r10, %r11
-; AVX512VL-NEXT: lzcntq %rdx, %r10
+; AVX512VL-NEXT: cmovneq %rdi, %r9
+; AVX512VL-NEXT: lzcntq %rdx, %rdi
+; AVX512VL-NEXT: movq %rcx, %r10
+; AVX512VL-NEXT: lzcntq %rsi, %rcx
; AVX512VL-NEXT: addq $64, %rcx
; AVX512VL-NEXT: testq %rdx, %rdx
-; AVX512VL-NEXT: cmovneq %r10, %rcx
+; AVX512VL-NEXT: cmovneq %rdi, %rcx
; AVX512VL-NEXT: subq $-128, %rcx
-; AVX512VL-NEXT: orq %r8, %rax
-; AVX512VL-NEXT: vmovaps {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
-; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: cmovneq %r11, %rcx
+; AVX512VL-NEXT: orq %r8, %r10
+; AVX512VL-NEXT: cmovneq %r9, %rcx
; AVX512VL-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: movl %ecx, %eax
-; AVX512VL-NEXT: shrb $6, %al
-; AVX512VL-NEXT: movzbl %al, %eax
-; AVX512VL-NEXT: movq -40(%rsp,%rax,8), %rdx
-; AVX512VL-NEXT: movq -48(%rsp,%rax,8), %r8
-; AVX512VL-NEXT: movq %r8, %r10
-; AVX512VL-NEXT: shrdq %cl, %rdx, %r10
-; AVX512VL-NEXT: movq -56(%rsp,%rax,8), %r11
-; AVX512VL-NEXT: movq %r11, %rbx
-; AVX512VL-NEXT: shrdq %cl, %r8, %rbx
-; AVX512VL-NEXT: movq -64(%rsp,%rax,8), %r8
-; AVX512VL-NEXT: shrdq %cl, %r11, %r8
-; AVX512VL-NEXT: movq %rdi, %rax
+; AVX512VL-NEXT: vmovaps {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
+; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movl %ecx, %edx
+; AVX512VL-NEXT: shrb $6, %dl
+; AVX512VL-NEXT: movzbl %dl, %esi
+; AVX512VL-NEXT: movq -48(%rsp,%rsi,8), %r8
+; AVX512VL-NEXT: movq -56(%rsp,%rsi,8), %r9
+; AVX512VL-NEXT: movq %r9, %rdx
+; AVX512VL-NEXT: shrdq %cl, %r8, %rdx
+; AVX512VL-NEXT: movq -64(%rsp,%rsi,8), %r10
+; AVX512VL-NEXT: movq %r10, %rdi
+; AVX512VL-NEXT: shrdq %cl, %r9, %rdi
+; AVX512VL-NEXT: movq -72(%rsp,%rsi,8), %rsi
+; AVX512VL-NEXT: shrxq %rcx, %r8, %r8
+; AVX512VL-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX512VL-NEXT: shrdq %cl, %r10, %rsi
+; AVX512VL-NEXT: jmp .LBB15_3
+; AVX512VL-NEXT: .LBB15_1:
+; AVX512VL-NEXT: xorl %esi, %esi
; AVX512VL-NEXT: xorl %edi, %edi
-; AVX512VL-NEXT: orq %r9, %rsi
-; AVX512VL-NEXT: shrxq %rcx, %rdx, %rcx
-; AVX512VL-NEXT: cmoveq %rdi, %rbx
-; AVX512VL-NEXT: cmoveq %rdi, %r10
-; AVX512VL-NEXT: cmoveq %rdi, %r8
-; AVX512VL-NEXT: cmoveq %rdi, %rcx
-; AVX512VL-NEXT: movq %rcx, 24(%rax)
-; AVX512VL-NEXT: movq %r10, 16(%rax)
-; AVX512VL-NEXT: movq %rbx, 8(%rax)
-; AVX512VL-NEXT: movq %r8, (%rax)
-; AVX512VL-NEXT: popq %rbx
+; AVX512VL-NEXT: xorl %edx, %edx
+; AVX512VL-NEXT: xorl %r8d, %r8d
+; AVX512VL-NEXT: .LBB15_3: # %select.end
+; AVX512VL-NEXT: movq %rsi, (%rax)
+; AVX512VL-NEXT: movq %rdi, 8(%rax)
+; AVX512VL-NEXT: movq %rdx, 16(%rax)
+; AVX512VL-NEXT: movq %r8, 24(%rax)
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VBMI-LABEL: isolate_msb_i256:
; AVX512VBMI: # %bb.0:
-; AVX512VBMI-NEXT: pushq %rbx
-; AVX512VBMI-NEXT: movq %rcx, %rax
-; AVX512VBMI-NEXT: movq %rdx, %r9
-; AVX512VBMI-NEXT: orq %r8, %r9
-; AVX512VBMI-NEXT: lzcntq %rsi, %rcx
-; AVX512VBMI-NEXT: orq %rax, %rsi
-; AVX512VBMI-NEXT: lzcntq %r8, %r10
-; AVX512VBMI-NEXT: lzcntq %rax, %r11
-; AVX512VBMI-NEXT: addq $64, %r11
+; AVX512VBMI-NEXT: movq %rdi, %rax
+; AVX512VBMI-NEXT: movq %rdx, %rdi
+; AVX512VBMI-NEXT: orq %r8, %rdi
+; AVX512VBMI-NEXT: movq %rsi, %r9
+; AVX512VBMI-NEXT: orq %rcx, %r9
+; AVX512VBMI-NEXT: orq %rdi, %r9
+; AVX512VBMI-NEXT: je .LBB15_1
+; AVX512VBMI-NEXT: # %bb.2: # %select.false.sink
+; AVX512VBMI-NEXT: lzcntq %r8, %rdi
+; AVX512VBMI-NEXT: lzcntq %rcx, %r9
+; AVX512VBMI-NEXT: addq $64, %r9
; AVX512VBMI-NEXT: testq %r8, %r8
-; AVX512VBMI-NEXT: cmovneq %r10, %r11
-; AVX512VBMI-NEXT: lzcntq %rdx, %r10
+; AVX512VBMI-NEXT: cmovneq %rdi, %r9
+; AVX512VBMI-NEXT: lzcntq %rdx, %rdi
+; AVX512VBMI-NEXT: movq %rcx, %r10
+; AVX512VBMI-NEXT: lzcntq %rsi, %rcx
; AVX512VBMI-NEXT: addq $64, %rcx
; AVX512VBMI-NEXT: testq %rdx, %rdx
-; AVX512VBMI-NEXT: cmovneq %r10, %rcx
+; AVX512VBMI-NEXT: cmovneq %rdi, %rcx
; AVX512VBMI-NEXT: subq $-128, %rcx
-; AVX512VBMI-NEXT: orq %r8, %rax
-; AVX512VBMI-NEXT: vmovaps {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
-; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: cmovneq %r11, %rcx
+; AVX512VBMI-NEXT: orq %r8, %r10
+; AVX512VBMI-NEXT: cmovneq %r9, %rcx
; AVX512VBMI-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: movl %ecx, %eax
-; AVX512VBMI-NEXT: shrb $6, %al
-; AVX512VBMI-NEXT: movzbl %al, %eax
-; AVX512VBMI-NEXT: movq -40(%rsp,%rax,8), %rdx
-; AVX512VBMI-NEXT: movq -48(%rsp,%rax,8), %r8
-; AVX512VBMI-NEXT: movq %r8, %r10
-; AVX512VBMI-NEXT: shrdq %cl, %rdx, %r10
-; AVX512VBMI-NEXT: movq -56(%rsp,%rax,8), %r11
-; AVX512VBMI-NEXT: movq %r11, %rbx
-; AVX512VBMI-NEXT: shrdq %cl, %r8, %rbx
-; AVX512VBMI-NEXT: movq -64(%rsp,%rax,8), %r8
-; AVX512VBMI-NEXT: shrdq %cl, %r11, %r8
-; AVX512VBMI-NEXT: movq %rdi, %rax
+; AVX512VBMI-NEXT: vmovaps {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
+; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movl %ecx, %edx
+; AVX512VBMI-NEXT: shrb $6, %dl
+; AVX512VBMI-NEXT: movzbl %dl, %esi
+; AVX512VBMI-NEXT: movq -48(%rsp,%rsi,8), %r8
+; AVX512VBMI-NEXT: movq -56(%rsp,%rsi,8), %r9
+; AVX512VBMI-NEXT: movq %r9, %rdx
+; AVX512VBMI-NEXT: shrdq %cl, %r8, %rdx
+; AVX512VBMI-NEXT: movq -64(%rsp,%rsi,8), %r10
+; AVX512VBMI-NEXT: movq %r10, %rdi
+; AVX512VBMI-NEXT: shrdq %cl, %r9, %rdi
+; AVX512VBMI-NEXT: movq -72(%rsp,%rsi,8), %rsi
+; AVX512VBMI-NEXT: shrxq %rcx, %r8, %r8
+; AVX512VBMI-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX512VBMI-NEXT: shrdq %cl, %r10, %rsi
+; AVX512VBMI-NEXT: jmp .LBB15_3
+; AVX512VBMI-NEXT: .LBB15_1:
+; AVX512VBMI-NEXT: xorl %esi, %esi
; AVX512VBMI-NEXT: xorl %edi, %edi
-; AVX512VBMI-NEXT: orq %r9, %rsi
-; AVX512VBMI-NEXT: shrxq %rcx, %rdx, %rcx
-; AVX512VBMI-NEXT: cmoveq %rdi, %rbx
-; AVX512VBMI-NEXT: cmoveq %rdi, %r10
-; AVX512VBMI-NEXT: cmoveq %rdi, %r8
-; AVX512VBMI-NEXT: cmoveq %rdi, %rcx
-; AVX512VBMI-NEXT: movq %rcx, 24(%rax)
-; AVX512VBMI-NEXT: movq %r10, 16(%rax)
-; AVX512VBMI-NEXT: movq %rbx, 8(%rax)
-; AVX512VBMI-NEXT: movq %r8, (%rax)
-; AVX512VBMI-NEXT: popq %rbx
+; AVX512VBMI-NEXT: xorl %edx, %edx
+; AVX512VBMI-NEXT: xorl %r8d, %r8d
+; AVX512VBMI-NEXT: .LBB15_3: # %select.end
+; AVX512VBMI-NEXT: movq %rsi, (%rax)
+; AVX512VBMI-NEXT: movq %rdi, 8(%rax)
+; AVX512VBMI-NEXT: movq %rdx, 16(%rax)
+; AVX512VBMI-NEXT: movq %r8, 24(%rax)
; AVX512VBMI-NEXT: vzeroupper
; AVX512VBMI-NEXT: retq
%eqz = icmp eq i256 %a0, 0
@@ -2681,36 +2702,38 @@ define i256 @isolate_msb_i256(i256 %a0, i256 %idx) nounwind {
define i256 @isolate_msb_i256_vector(<4 x i64> %v0, i256 %idx) nounwind {
; SSE2-LABEL: isolate_msb_i256_vector:
; SSE2: # %bb.0:
+; SSE2-NEXT: movq %rdi, %rax
+; SSE2-NEXT: movdqa %xmm0, %xmm3
+; SSE2-NEXT: por %xmm1, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: pcmpeqd %xmm2, %xmm3
+; SSE2-NEXT: movmskps %xmm3, %ecx
+; SSE2-NEXT: xorl $15, %ecx
+; SSE2-NEXT: je .LBB16_1
+; SSE2-NEXT: # %bb.2: # %select.false.sink
+; SSE2-NEXT: movq %xmm1, %rdx
; SSE2-NEXT: movq %xmm0, %rcx
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
-; SSE2-NEXT: movq %xmm2, %rdx
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
; SSE2-NEXT: movq %xmm1, %rsi
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
-; SSE2-NEXT: movq %xmm2, %r8
-; SSE2-NEXT: por %xmm1, %xmm0
-; SSE2-NEXT: pxor %xmm2, %xmm2
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm0
-; SSE2-NEXT: movmskps %xmm0, %eax
-; SSE2-NEXT: xorl $15, %eax
-; SSE2-NEXT: bsrq %r8, %r9
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; SSE2-NEXT: movq %xmm0, %rdi
+; SSE2-NEXT: bsrq %rsi, %r8
+; SSE2-NEXT: xorq $63, %r8
+; SSE2-NEXT: bsrq %rdx, %r9
; SSE2-NEXT: xorq $63, %r9
-; SSE2-NEXT: bsrq %rsi, %rsi
-; SSE2-NEXT: xorq $63, %rsi
-; SSE2-NEXT: orq $64, %rsi
-; SSE2-NEXT: testq %r8, %r8
-; SSE2-NEXT: cmovneq %r9, %rsi
-; SSE2-NEXT: bsrq %rdx, %r8
+; SSE2-NEXT: orq $64, %r9
+; SSE2-NEXT: testq %rsi, %rsi
+; SSE2-NEXT: cmovneq %r8, %r9
+; SSE2-NEXT: bsrq %rdi, %r8
; SSE2-NEXT: xorq $63, %r8
; SSE2-NEXT: bsrq %rcx, %rcx
; SSE2-NEXT: xorq $63, %rcx
; SSE2-NEXT: orq $64, %rcx
-; SSE2-NEXT: testq %rdx, %rdx
+; SSE2-NEXT: testq %rdi, %rdi
; SSE2-NEXT: cmovneq %r8, %rcx
; SSE2-NEXT: orq $128, %rcx
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm1
-; SSE2-NEXT: movmskps %xmm1, %edx
-; SSE2-NEXT: xorl $15, %edx
-; SSE2-NEXT: cmovneq %rsi, %rcx
+; SSE2-NEXT: orq %rsi, %rdx
+; SSE2-NEXT: cmovneq %r9, %rcx
; SSE2-NEXT: movdqa %xmm2, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: movdqa %xmm2, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: movabsq $-9223372036854775808, %rdx # imm = 0x8000000000000000
@@ -2718,257 +2741,316 @@ define i256 @isolate_msb_i256_vector(<4 x i64> %v0, i256 %idx) nounwind {
; SSE2-NEXT: movdqa %xmm2, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: movl %ecx, %edx
; SSE2-NEXT: shrb $6, %dl
-; SSE2-NEXT: movzbl %dl, %esi
+; SSE2-NEXT: movzbl %dl, %edi
; SSE2-NEXT: movq $0, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movq -48(%rsp,%rsi,8), %rdx
-; SSE2-NEXT: movq -56(%rsp,%rsi,8), %r8
-; SSE2-NEXT: movq %r8, %r9
-; SSE2-NEXT: shrdq %cl, %rdx, %r9
-; SSE2-NEXT: movq -64(%rsp,%rsi,8), %r10
-; SSE2-NEXT: movq %r10, %r11
-; SSE2-NEXT: shrdq %cl, %r8, %r11
-; SSE2-NEXT: movq -72(%rsp,%rsi,8), %rsi
+; SSE2-NEXT: movq -48(%rsp,%rdi,8), %rdx
+; SSE2-NEXT: movq -56(%rsp,%rdi,8), %r9
+; SSE2-NEXT: movq %r9, %rsi
+; SSE2-NEXT: shrdq %cl, %rdx, %rsi
+; SSE2-NEXT: movq -64(%rsp,%rdi,8), %r10
+; SSE2-NEXT: movq %r10, %r8
+; SSE2-NEXT: shrdq %cl, %r9, %r8
+; SSE2-NEXT: movq -72(%rsp,%rdi,8), %rdi
; SSE2-NEXT: shrq %cl, %rdx
; SSE2-NEXT: # kill: def $cl killed $cl killed $rcx
-; SSE2-NEXT: shrdq %cl, %r10, %rsi
-; SSE2-NEXT: xorl %ecx, %ecx
-; SSE2-NEXT: testl %eax, %eax
-; SSE2-NEXT: cmoveq %rcx, %r11
-; SSE2-NEXT: cmoveq %rcx, %r9
-; SSE2-NEXT: cmoveq %rcx, %rsi
-; SSE2-NEXT: movq %rdi, %rax
-; SSE2-NEXT: cmoveq %rcx, %rdx
-; SSE2-NEXT: movq %rdx, 24(%rdi)
-; SSE2-NEXT: movq %r9, 16(%rdi)
-; SSE2-NEXT: movq %r11, 8(%rdi)
-; SSE2-NEXT: movq %rsi, (%rdi)
+; SSE2-NEXT: shrdq %cl, %r10, %rdi
+; SSE2-NEXT: jmp .LBB16_3
+; SSE2-NEXT: .LBB16_1:
+; SSE2-NEXT: xorl %edi, %edi
+; SSE2-NEXT: xorl %r8d, %r8d
+; SSE2-NEXT: xorl %esi, %esi
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: .LBB16_3: # %select.end
+; SSE2-NEXT: movq %rdi, (%rax)
+; SSE2-NEXT: movq %r8, 8(%rax)
+; SSE2-NEXT: movq %rsi, 16(%rax)
+; SSE2-NEXT: movq %rdx, 24(%rax)
; SSE2-NEXT: retq
;
; SSE42-LABEL: isolate_msb_i256_vector:
; SSE42: # %bb.0:
-; SSE42-NEXT: pextrq $1, %xmm0, %rdx
+; SSE42-NEXT: movq %rdi, %rax
+; SSE42-NEXT: movdqa %xmm0, %xmm2
+; SSE42-NEXT: por %xmm1, %xmm2
+; SSE42-NEXT: ptest %xmm2, %xmm2
+; SSE42-NEXT: je .LBB16_1
+; SSE42-NEXT: # %bb.2: # %select.false.sink
+; SSE42-NEXT: pextrq $1, %xmm1, %rdx
+; SSE42-NEXT: pextrq $1, %xmm0, %rsi
+; SSE42-NEXT: movq %xmm1, %rdi
; SSE42-NEXT: movq %xmm0, %rcx
-; SSE42-NEXT: movq %xmm1, %rax
-; SSE42-NEXT: pextrq $1, %xmm1, %rsi
+; SSE42-NEXT: bsrq %rdx, %r8
+; SSE42-NEXT: xorq $63, %r8
+; SSE42-NEXT: bsrq %rdi, %r9
+; SSE42-NEXT: xorq $63, %r9
+; SSE42-NEXT: orq $64, %r9
+; SSE42-NEXT: testq %rdx, %rdx
+; SSE42-NEXT: cmovneq %r8, %r9
; SSE42-NEXT: bsrq %rsi, %r8
; SSE42-NEXT: xorq $63, %r8
-; SSE42-NEXT: bsrq %rax, %rax
-; SSE42-NEXT: xorq $63, %rax
-; SSE42-NEXT: orq $64, %rax
-; SSE42-NEXT: testq %rsi, %rsi
-; SSE42-NEXT: cmovneq %r8, %rax
-; SSE42-NEXT: bsrq %rdx, %rsi
-; SSE42-NEXT: xorq $63, %rsi
; SSE42-NEXT: bsrq %rcx, %rcx
; SSE42-NEXT: xorq $63, %rcx
; SSE42-NEXT: orq $64, %rcx
-; SSE42-NEXT: testq %rdx, %rdx
-; SSE42-NEXT: cmovneq %rsi, %rcx
-; SSE42-NEXT: xorps %xmm2, %xmm2
-; SSE42-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: testq %rsi, %rsi
+; SSE42-NEXT: cmovneq %r8, %rcx
; SSE42-NEXT: orq $128, %rcx
-; SSE42-NEXT: ptest %xmm1, %xmm1
-; SSE42-NEXT: cmovneq %rax, %rcx
-; SSE42-NEXT: movabsq $-9223372036854775808, %rax # imm = 0x8000000000000000
-; SSE42-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movl %ecx, %eax
-; SSE42-NEXT: shrb $6, %al
-; SSE42-NEXT: movzbl %al, %eax
+; SSE42-NEXT: orq %rdx, %rdi
+; SSE42-NEXT: cmovneq %r9, %rcx
+; SSE42-NEXT: pxor %xmm0, %xmm0
+; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movabsq $-9223372036854775808, %rdx # imm = 0x8000000000000000
+; SSE42-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movl %ecx, %edx
+; SSE42-NEXT: shrb $6, %dl
+; SSE42-NEXT: movzbl %dl, %edi
; SSE42-NEXT: movq $0, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movq -48(%rsp,%rax,8), %rdx
-; SSE42-NEXT: movq -56(%rsp,%rax,8), %rsi
-; SSE42-NEXT: movq %rsi, %r8
-; SSE42-NEXT: shrdq %cl, %rdx, %r8
-; SSE42-NEXT: movq -64(%rsp,%rax,8), %r9
-; SSE42-NEXT: movq %r9, %r10
-; SSE42-NEXT: shrdq %cl, %rsi, %r10
-; SSE42-NEXT: movq -72(%rsp,%rax,8), %rsi
+; SSE42-NEXT: movq -48(%rsp,%rdi,8), %rdx
+; SSE42-NEXT: movq -56(%rsp,%rdi,8), %r9
+; SSE42-NEXT: movq %r9, %rsi
+; SSE42-NEXT: shrdq %cl, %rdx, %rsi
+; SSE42-NEXT: movq -64(%rsp,%rdi,8), %r10
+; SSE42-NEXT: movq %r10, %r8
+; SSE42-NEXT: shrdq %cl, %r9, %r8
+; SSE42-NEXT: movq -72(%rsp,%rdi,8), %rdi
; SSE42-NEXT: shrq %cl, %rdx
; SSE42-NEXT: # kill: def $cl killed $cl killed $rcx
-; SSE42-NEXT: shrdq %cl, %r9, %rsi
-; SSE42-NEXT: por %xmm1, %xmm0
-; SSE42-NEXT: xorl %ecx, %ecx
-; SSE42-NEXT: ptest %xmm0, %xmm0
-; SSE42-NEXT: cmoveq %rcx, %r10
-; SSE42-NEXT: cmoveq %rcx, %r8
-; SSE42-NEXT: cmoveq %rcx, %rsi
-; SSE42-NEXT: movq %rdi, %rax
-; SSE42-NEXT: cmoveq %rcx, %rdx
-; SSE42-NEXT: movq %rdx, 24(%rdi)
-; SSE42-NEXT: movq %r8, 16(%rdi)
-; SSE42-NEXT: movq %r10, 8(%rdi)
-; SSE42-NEXT: movq %rsi, (%rdi)
+; SSE42-NEXT: shrdq %cl, %r10, %rdi
+; SSE42-NEXT: jmp .LBB16_3
+; SSE42-NEXT: .LBB16_1:
+; SSE42-NEXT: xorl %edi, %edi
+; SSE42-NEXT: xorl %r8d, %r8d
+; SSE42-NEXT: xorl %esi, %esi
+; SSE42-NEXT: xorl %edx, %edx
+; SSE42-NEXT: .LBB16_3: # %select.end
+; SSE42-NEXT: movq %rdi, (%rax)
+; SSE42-NEXT: movq %r8, 8(%rax)
+; SSE42-NEXT: movq %rsi, 16(%rax)
+; SSE42-NEXT: movq %rdx, 24(%rax)
; SSE42-NEXT: retq
;
; AVX2-LABEL: isolate_msb_i256_vector:
; AVX2: # %bb.0:
-; AVX2-NEXT: vmovq %xmm0, %rax
-; AVX2-NEXT: vpextrq $1, %xmm0, %rdx
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: vptest %ymm0, %ymm0
+; AVX2-NEXT: je .LBB16_1
+; AVX2-NEXT: # %bb.2: # %select.false.sink
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpextrq $1, %xmm1, %rdx
; AVX2-NEXT: vmovq %xmm1, %rsi
-; AVX2-NEXT: vpextrq $1, %xmm1, %r8
-; AVX2-NEXT: lzcntq %r8, %rcx
+; AVX2-NEXT: vpextrq $1, %xmm0, %rdi
+; AVX2-NEXT: vmovq %xmm0, %rcx
+; AVX2-NEXT: lzcntq %rdx, %r8
; AVX2-NEXT: lzcntq %rsi, %r9
; AVX2-NEXT: addq $64, %r9
-; AVX2-NEXT: testq %r8, %r8
-; AVX2-NEXT: cmovneq %rcx, %r9
-; AVX2-NEXT: lzcntq %rdx, %r10
-; AVX2-NEXT: xorl %ecx, %ecx
-; AVX2-NEXT: lzcntq %rax, %rcx
-; AVX2-NEXT: addq $64, %rcx
; AVX2-NEXT: testq %rdx, %rdx
-; AVX2-NEXT: cmovneq %r10, %rcx
+; AVX2-NEXT: cmovneq %r8, %r9
+; AVX2-NEXT: xorl %r8d, %r8d
+; AVX2-NEXT: lzcntq %rdi, %r8
+; AVX2-NEXT: lzcntq %rcx, %rcx
+; AVX2-NEXT: addq $64, %rcx
+; AVX2-NEXT: testq %rdi, %rdi
+; AVX2-NEXT: cmovneq %r8, %rcx
; AVX2-NEXT: subq $-128, %rcx
-; AVX2-NEXT: orq %r8, %rsi
+; AVX2-NEXT: orq %rdx, %rsi
; AVX2-NEXT: cmovneq %r9, %rcx
-; AVX2-NEXT: vmovaps {{.*#+}} ymm1 = [0,0,0,9223372036854775808]
-; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movl %ecx, %eax
-; AVX2-NEXT: shrb $6, %al
-; AVX2-NEXT: movzbl %al, %edx
-; AVX2-NEXT: movq -48(%rsp,%rdx,8), %rsi
-; AVX2-NEXT: movq -56(%rsp,%rdx,8), %r8
-; AVX2-NEXT: movq %r8, %r9
-; AVX2-NEXT: shrdq %cl, %rsi, %r9
-; AVX2-NEXT: movq %rdi, %rax
-; AVX2-NEXT: movq -72(%rsp,%rdx,8), %rdi
-; AVX2-NEXT: movq -64(%rsp,%rdx,8), %rdx
-; AVX2-NEXT: movq %rdx, %r10
-; AVX2-NEXT: shrdq %cl, %r8, %r10
-; AVX2-NEXT: shrdq %cl, %rdx, %rdi
+; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovdqa {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
+; AVX2-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movl %ecx, %edx
+; AVX2-NEXT: shrb $6, %dl
+; AVX2-NEXT: movzbl %dl, %esi
+; AVX2-NEXT: movq -48(%rsp,%rsi,8), %r8
+; AVX2-NEXT: movq -56(%rsp,%rsi,8), %r9
+; AVX2-NEXT: movq %r9, %rdx
+; AVX2-NEXT: shrdq %cl, %r8, %rdx
+; AVX2-NEXT: movq -64(%rsp,%rsi,8), %r10
+; AVX2-NEXT: movq %r10, %rdi
+; AVX2-NEXT: shrdq %cl, %r9, %rdi
+; AVX2-NEXT: movq -72(%rsp,%rsi,8), %rsi
+; AVX2-NEXT: shrxq %rcx, %r8, %r8
+; AVX2-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX2-NEXT: shrdq %cl, %r10, %rsi
+; AVX2-NEXT: jmp .LBB16_3
+; AVX2-NEXT: .LBB16_1:
+; AVX2-NEXT: xorl %esi, %esi
+; AVX2-NEXT: xorl %edi, %edi
; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: vptest %ymm0, %ymm0
-; AVX2-NEXT: shrxq %rcx, %rsi, %rcx
-; AVX2-NEXT: cmoveq %rdx, %r10
-; AVX2-NEXT: cmoveq %rdx, %r9
-; AVX2-NEXT: cmoveq %rdx, %rdi
-; AVX2-NEXT: cmoveq %rdx, %rcx
-; AVX2-NEXT: movq %rcx, 24(%rax)
-; AVX2-NEXT: movq %r9, 16(%rax)
-; AVX2-NEXT: movq %r10, 8(%rax)
-; AVX2-NEXT: movq %rdi, (%rax)
+; AVX2-NEXT: xorl %r8d, %r8d
+; AVX2-NEXT: .LBB16_3: # %select.end
+; AVX2-NEXT: movq %rsi, (%rax)
+; AVX2-NEXT: movq %rdi, 8(%rax)
+; AVX2-NEXT: movq %rdx, 16(%rax)
+; AVX2-NEXT: movq %r8, 24(%rax)
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512F-LABEL: isolate_msb_i256_vector:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vmovaps {{.*#+}} zmm1 = [0,0,0,9223372036854775808,0,0,0,0]
-; AVX512F-NEXT: vmovups %zmm1, -{{[0-9]+}}(%rsp)
-; AVX512F-NEXT: vpermq {{.*#+}} ymm1 = ymm0[3,2,1,0]
-; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k0
-; AVX512F-NEXT: vplzcntq %zmm1, %zmm1
-; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,64,128,192]
-; AVX512F-NEXT: kshiftlw $12, %k0, %k0
-; AVX512F-NEXT: kshiftrw $12, %k0, %k1
-; AVX512F-NEXT: vpcompressq %zmm1, %zmm1 {%k1}
-; AVX512F-NEXT: vmovd %xmm1, %ecx
-; AVX512F-NEXT: movl %ecx, %eax
-; AVX512F-NEXT: shrb $6, %al
-; AVX512F-NEXT: movzbl %al, %edx
-; AVX512F-NEXT: movq -48(%rsp,%rdx,8), %rsi
-; AVX512F-NEXT: movq -56(%rsp,%rdx,8), %r8
-; AVX512F-NEXT: movq %r8, %r9
-; AVX512F-NEXT: shrdq %cl, %rsi, %r9
; AVX512F-NEXT: movq %rdi, %rax
-; AVX512F-NEXT: vmovdqa %ymm0, %ymm0
-; AVX512F-NEXT: movq -72(%rsp,%rdx,8), %rdi
-; AVX512F-NEXT: movq -64(%rsp,%rdx,8), %rdx
-; AVX512F-NEXT: movq %rdx, %r10
-; AVX512F-NEXT: shrdq %cl, %r8, %r10
-; AVX512F-NEXT: shrdq %cl, %rdx, %rdi
-; AVX512F-NEXT: vptestmd %zmm0, %zmm0, %k0
-; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: vmovdqa %ymm0, %ymm1
+; AVX512F-NEXT: vptestmd %zmm1, %zmm1, %k0
; AVX512F-NEXT: kortestw %k0, %k0
-; AVX512F-NEXT: shrxq %rcx, %rsi, %rcx
-; AVX512F-NEXT: cmoveq %rdx, %r10
-; AVX512F-NEXT: cmoveq %rdx, %r9
-; AVX512F-NEXT: cmoveq %rdx, %rdi
-; AVX512F-NEXT: cmoveq %rdx, %rcx
-; AVX512F-NEXT: movq %rcx, 24(%rax)
-; AVX512F-NEXT: movq %r9, 16(%rax)
-; AVX512F-NEXT: movq %r10, 8(%rax)
-; AVX512F-NEXT: movq %rdi, (%rax)
+; AVX512F-NEXT: je .LBB16_1
+; AVX512F-NEXT: # %bb.2: # %select.false.sink
+; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT: vpextrq $1, %xmm1, %rdx
+; AVX512F-NEXT: vmovq %xmm1, %rsi
+; AVX512F-NEXT: vpextrq $1, %xmm0, %rdi
+; AVX512F-NEXT: vmovq %xmm0, %rcx
+; AVX512F-NEXT: lzcntq %rdx, %r8
+; AVX512F-NEXT: lzcntq %rsi, %r9
+; AVX512F-NEXT: addq $64, %r9
+; AVX512F-NEXT: testq %rdx, %rdx
+; AVX512F-NEXT: cmovneq %r8, %r9
+; AVX512F-NEXT: lzcntq %rdi, %r8
+; AVX512F-NEXT: lzcntq %rcx, %rcx
+; AVX512F-NEXT: addq $64, %rcx
+; AVX512F-NEXT: testq %rdi, %rdi
+; AVX512F-NEXT: cmovneq %r8, %rcx
+; AVX512F-NEXT: subq $-128, %rcx
+; AVX512F-NEXT: orq %rdx, %rsi
+; AVX512F-NEXT: cmovneq %r9, %rcx
+; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm0 = [0,0,0,9223372036854775808,0,0,0,0]
+; AVX512F-NEXT: vmovdqu64 %zmm0, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movl %ecx, %edx
+; AVX512F-NEXT: shrb $6, %dl
+; AVX512F-NEXT: movzbl %dl, %esi
+; AVX512F-NEXT: movq -48(%rsp,%rsi,8), %r8
+; AVX512F-NEXT: movq -56(%rsp,%rsi,8), %r9
+; AVX512F-NEXT: movq %r9, %rdx
+; AVX512F-NEXT: shrdq %cl, %r8, %rdx
+; AVX512F-NEXT: movq -64(%rsp,%rsi,8), %r10
+; AVX512F-NEXT: movq %r10, %rdi
+; AVX512F-NEXT: shrdq %cl, %r9, %rdi
+; AVX512F-NEXT: movq -72(%rsp,%rsi,8), %rsi
+; AVX512F-NEXT: shrxq %rcx, %r8, %r8
+; AVX512F-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX512F-NEXT: shrdq %cl, %r10, %rsi
+; AVX512F-NEXT: jmp .LBB16_3
+; AVX512F-NEXT: .LBB16_1:
+; AVX512F-NEXT: xorl %esi, %esi
+; AVX512F-NEXT: xorl %edi, %edi
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: xorl %r8d, %r8d
+; AVX512F-NEXT: .LBB16_3: # %select.end
+; AVX512F-NEXT: movq %rsi, (%rax)
+; AVX512F-NEXT: movq %rdi, 8(%rax)
+; AVX512F-NEXT: movq %rdx, 16(%rax)
+; AVX512F-NEXT: movq %r8, 24(%rax)
; AVX512F-NEXT: retq
;
; AVX512VL-LABEL: isolate_msb_i256_vector:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vpermq {{.*#+}} ymm1 = ymm0[3,2,1,0]
-; AVX512VL-NEXT: vptestmq %ymm1, %ymm1, %k1
-; AVX512VL-NEXT: vplzcntq %ymm1, %ymm1
-; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,64,128,192]
-; AVX512VL-NEXT: vpcompressq %ymm1, %ymm1 {%k1}
-; AVX512VL-NEXT: vmovaps {{.*#+}} ymm2 = [0,0,0,9223372036854775808]
-; AVX512VL-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: vxorps %xmm2, %xmm2, %xmm2
-; AVX512VL-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: vmovd %xmm1, %ecx
-; AVX512VL-NEXT: movl %ecx, %eax
-; AVX512VL-NEXT: shrb $6, %al
-; AVX512VL-NEXT: movzbl %al, %edx
-; AVX512VL-NEXT: movq -48(%rsp,%rdx,8), %rsi
-; AVX512VL-NEXT: movq -56(%rsp,%rdx,8), %rax
-; AVX512VL-NEXT: movq %rax, %r8
-; AVX512VL-NEXT: shrdq %cl, %rsi, %r8
-; AVX512VL-NEXT: movq -64(%rsp,%rdx,8), %r9
-; AVX512VL-NEXT: movq %r9, %r10
-; AVX512VL-NEXT: shrdq %cl, %rax, %r10
; AVX512VL-NEXT: movq %rdi, %rax
-; AVX512VL-NEXT: movq -72(%rsp,%rdx,8), %rdx
-; AVX512VL-NEXT: shrdq %cl, %r9, %rdx
-; AVX512VL-NEXT: xorl %edi, %edi
; AVX512VL-NEXT: vptest %ymm0, %ymm0
-; AVX512VL-NEXT: shrxq %rcx, %rsi, %rcx
-; AVX512VL-NEXT: cmoveq %rdi, %r10
-; AVX512VL-NEXT: cmoveq %rdi, %r8
-; AVX512VL-NEXT: cmoveq %rdi, %rdx
-; AVX512VL-NEXT: cmoveq %rdi, %rcx
-; AVX512VL-NEXT: movq %rcx, 24(%rax)
-; AVX512VL-NEXT: movq %r8, 16(%rax)
-; AVX512VL-NEXT: movq %r10, 8(%rax)
-; AVX512VL-NEXT: movq %rdx, (%rax)
+; AVX512VL-NEXT: je .LBB16_1
+; AVX512VL-NEXT: # %bb.2: # %select.false.sink
+; AVX512VL-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512VL-NEXT: vpextrq $1, %xmm1, %rdx
+; AVX512VL-NEXT: vpextrq $1, %xmm0, %rsi
+; AVX512VL-NEXT: vmovq %xmm1, %rdi
+; AVX512VL-NEXT: vmovq %xmm0, %rcx
+; AVX512VL-NEXT: lzcntq %rdx, %r8
+; AVX512VL-NEXT: lzcntq %rdi, %r9
+; AVX512VL-NEXT: addq $64, %r9
+; AVX512VL-NEXT: testq %rdx, %rdx
+; AVX512VL-NEXT: cmovneq %r8, %r9
+; AVX512VL-NEXT: lzcntq %rsi, %r8
+; AVX512VL-NEXT: lzcntq %rcx, %rcx
+; AVX512VL-NEXT: addq $64, %rcx
+; AVX512VL-NEXT: testq %rsi, %rsi
+; AVX512VL-NEXT: cmovneq %r8, %rcx
+; AVX512VL-NEXT: subq $-128, %rcx
+; AVX512VL-NEXT: orq %rdx, %rdi
+; AVX512VL-NEXT: cmovneq %r9, %rcx
+; AVX512VL-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512VL-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: vmovdqa {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
+; AVX512VL-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movl %ecx, %edx
+; AVX512VL-NEXT: shrb $6, %dl
+; AVX512VL-NEXT: movzbl %dl, %esi
+; AVX512VL-NEXT: movq -48(%rsp,%rsi,8), %r8
+; AVX512VL-NEXT: movq -56(%rsp,%rsi,8), %r9
+; AVX512VL-NEXT: movq %r9, %rdx
+; AVX512VL-NEXT: shrdq %cl, %r8, %rdx
+; AVX512VL-NEXT: movq -64(%rsp,%rsi,8), %r10
+; AVX512VL-NEXT: movq %r10, %rdi
+; AVX512VL-NEXT: shrdq %cl, %r9, %rdi
+; AVX512VL-NEXT: movq -72(%rsp,%rsi,8), %rsi
+; AVX512VL-NEXT: shrxq %rcx, %r8, %r8
+; AVX512VL-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX512VL-NEXT: shrdq %cl, %r10, %rsi
+; AVX512VL-NEXT: jmp .LBB16_3
+; AVX512VL-NEXT: .LBB16_1:
+; AVX512VL-NEXT: xorl %esi, %esi
+; AVX512VL-NEXT: xorl %edi, %edi
+; AVX512VL-NEXT: xorl %edx, %edx
+; AVX512VL-NEXT: xorl %r8d, %r8d
+; AVX512VL-NEXT: .LBB16_3: # %select.end
+; AVX512VL-NEXT: movq %rsi, (%rax)
+; AVX512VL-NEXT: movq %rdi, 8(%rax)
+; AVX512VL-NEXT: movq %rdx, 16(%rax)
+; AVX512VL-NEXT: movq %r8, 24(%rax)
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VBMI-LABEL: isolate_msb_i256_vector:
; AVX512VBMI: # %bb.0:
-; AVX512VBMI-NEXT: vpermq {{.*#+}} ymm1 = ymm0[3,2,1,0]
-; AVX512VBMI-NEXT: vptestmq %ymm1, %ymm1, %k1
-; AVX512VBMI-NEXT: vplzcntq %ymm1, %ymm1
-; AVX512VBMI-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,64,128,192]
-; AVX512VBMI-NEXT: vpcompressq %ymm1, %ymm1 {%k1}
-; AVX512VBMI-NEXT: vmovaps {{.*#+}} ymm2 = [0,0,0,9223372036854775808]
-; AVX512VBMI-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: vxorps %xmm2, %xmm2, %xmm2
-; AVX512VBMI-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: vmovd %xmm1, %ecx
-; AVX512VBMI-NEXT: movl %ecx, %eax
-; AVX512VBMI-NEXT: shrb $6, %al
-; AVX512VBMI-NEXT: movzbl %al, %edx
-; AVX512VBMI-NEXT: movq -48(%rsp,%rdx,8), %rsi
-; AVX512VBMI-NEXT: movq -56(%rsp,%rdx,8), %rax
-; AVX512VBMI-NEXT: movq %rax, %r8
-; AVX512VBMI-NEXT: shrdq %cl, %rsi, %r8
-; AVX512VBMI-NEXT: movq -64(%rsp,%rdx,8), %r9
-; AVX512VBMI-NEXT: movq %r9, %r10
-; AVX512VBMI-NEXT: shrdq %cl, %rax, %r10
; AVX512VBMI-NEXT: movq %rdi, %rax
-; AVX512VBMI-NEXT: movq -72(%rsp,%rdx,8), %rdx
-; AVX512VBMI-NEXT: shrdq %cl, %r9, %rdx
-; AVX512VBMI-NEXT: xorl %edi, %edi
; AVX512VBMI-NEXT: vptest %ymm0, %ymm0
-; AVX512VBMI-NEXT: shrxq %rcx, %rsi, %rcx
-; AVX512VBMI-NEXT: cmoveq %rdi, %r10
-; AVX512VBMI-NEXT: cmoveq %rdi, %r8
-; AVX512VBMI-NEXT: cmoveq %rdi, %rdx
-; AVX512VBMI-NEXT: cmoveq %rdi, %rcx
-; AVX512VBMI-NEXT: movq %rcx, 24(%rax)
-; AVX512VBMI-NEXT: movq %r8, 16(%rax)
-; AVX512VBMI-NEXT: movq %r10, 8(%rax)
-; AVX512VBMI-NEXT: movq %rdx, (%rax)
+; AVX512VBMI-NEXT: je .LBB16_1
+; AVX512VBMI-NEXT: # %bb.2: # %select.false.sink
+; AVX512VBMI-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512VBMI-NEXT: vpextrq $1, %xmm1, %rdx
+; AVX512VBMI-NEXT: vpextrq $1, %xmm0, %rsi
+; AVX512VBMI-NEXT: vmovq %xmm1, %rdi
+; AVX512VBMI-NEXT: vmovq %xmm0, %rcx
+; AVX512VBMI-NEXT: lzcntq %rdx, %r8
+; AVX512VBMI-NEXT: lzcntq %rdi, %r9
+; AVX512VBMI-NEXT: addq $64, %r9
+; AVX512VBMI-NEXT: testq %rdx, %rdx
+; AVX512VBMI-NEXT: cmovneq %r8, %r9
+; AVX512VBMI-NEXT: lzcntq %rsi, %r8
+; AVX512VBMI-NEXT: lzcntq %rcx, %rcx
+; AVX512VBMI-NEXT: addq $64, %rcx
+; AVX512VBMI-NEXT: testq %rsi, %rsi
+; AVX512VBMI-NEXT: cmovneq %r8, %rcx
+; AVX512VBMI-NEXT: subq $-128, %rcx
+; AVX512VBMI-NEXT: orq %rdx, %rdi
+; AVX512VBMI-NEXT: cmovneq %r9, %rcx
+; AVX512VBMI-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512VBMI-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: vmovdqa {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
+; AVX512VBMI-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movl %ecx, %edx
+; AVX512VBMI-NEXT: shrb $6, %dl
+; AVX512VBMI-NEXT: movzbl %dl, %esi
+; AVX512VBMI-NEXT: movq -48(%rsp,%rsi,8), %r8
+; AVX512VBMI-NEXT: movq -56(%rsp,%rsi,8), %r9
+; AVX512VBMI-NEXT: movq %r9, %rdx
+; AVX512VBMI-NEXT: shrdq %cl, %r8, %rdx
+; AVX512VBMI-NEXT: movq -64(%rsp,%rsi,8), %r10
+; AVX512VBMI-NEXT: movq %r10, %rdi
+; AVX512VBMI-NEXT: shrdq %cl, %r9, %rdi
+; AVX512VBMI-NEXT: movq -72(%rsp,%rsi,8), %rsi
+; AVX512VBMI-NEXT: shrxq %rcx, %r8, %r8
+; AVX512VBMI-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX512VBMI-NEXT: shrdq %cl, %r10, %rsi
+; AVX512VBMI-NEXT: jmp .LBB16_3
+; AVX512VBMI-NEXT: .LBB16_1:
+; AVX512VBMI-NEXT: xorl %esi, %esi
+; AVX512VBMI-NEXT: xorl %edi, %edi
+; AVX512VBMI-NEXT: xorl %edx, %edx
+; AVX512VBMI-NEXT: xorl %r8d, %r8d
+; AVX512VBMI-NEXT: .LBB16_3: # %select.end
+; AVX512VBMI-NEXT: movq %rsi, (%rax)
+; AVX512VBMI-NEXT: movq %rdi, 8(%rax)
+; AVX512VBMI-NEXT: movq %rdx, 16(%rax)
+; AVX512VBMI-NEXT: movq %r8, 24(%rax)
; AVX512VBMI-NEXT: vzeroupper
; AVX512VBMI-NEXT: retq
%a0 = bitcast <4 x i64> %v0 to i256
@@ -2983,32 +3065,35 @@ define i256 @isolate_msb_i256_vector(<4 x i64> %v0, i256 %idx) nounwind {
define i256 @isolate_msb_i256_load(ptr %p0, i256 %idx) nounwind {
; SSE2-LABEL: isolate_msb_i256_load:
; SSE2: # %bb.0:
-; SSE2-NEXT: movq 8(%rsi), %r9
-; SSE2-NEXT: movq 16(%rsi), %rdx
-; SSE2-NEXT: movq 24(%rsi), %r8
+; SSE2-NEXT: movq %rdi, %rax
; SSE2-NEXT: movdqa (%rsi), %xmm1
; SSE2-NEXT: por 16(%rsi), %xmm1
; SSE2-NEXT: pxor %xmm0, %xmm0
; SSE2-NEXT: pcmpeqd %xmm0, %xmm1
-; SSE2-NEXT: movmskps %xmm1, %eax
-; SSE2-NEXT: xorl $15, %eax
-; SSE2-NEXT: bsrq %r8, %rcx
+; SSE2-NEXT: movmskps %xmm1, %ecx
+; SSE2-NEXT: xorl $15, %ecx
+; SSE2-NEXT: je .LBB17_1
+; SSE2-NEXT: # %bb.2: # %select.false.sink
+; SSE2-NEXT: movq 24(%rsi), %rdx
+; SSE2-NEXT: movq 16(%rsi), %rdi
+; SSE2-NEXT: movq 8(%rsi), %r8
+; SSE2-NEXT: bsrq %rdx, %rcx
; SSE2-NEXT: xorq $63, %rcx
-; SSE2-NEXT: bsrq %rdx, %r10
+; SSE2-NEXT: bsrq %rdi, %r9
+; SSE2-NEXT: xorq $63, %r9
+; SSE2-NEXT: orq $64, %r9
+; SSE2-NEXT: testq %rdx, %rdx
+; SSE2-NEXT: cmovneq %rcx, %r9
+; SSE2-NEXT: bsrq %r8, %r10
; SSE2-NEXT: xorq $63, %r10
-; SSE2-NEXT: orq $64, %r10
-; SSE2-NEXT: testq %r8, %r8
-; SSE2-NEXT: cmovneq %rcx, %r10
-; SSE2-NEXT: bsrq %r9, %r11
-; SSE2-NEXT: xorq $63, %r11
; SSE2-NEXT: bsrq (%rsi), %rcx
; SSE2-NEXT: xorq $63, %rcx
; SSE2-NEXT: orq $64, %rcx
-; SSE2-NEXT: testq %r9, %r9
-; SSE2-NEXT: cmovneq %r11, %rcx
-; SSE2-NEXT: orq $128, %rcx
-; SSE2-NEXT: orq %r8, %rdx
+; SSE2-NEXT: testq %r8, %r8
; SSE2-NEXT: cmovneq %r10, %rcx
+; SSE2-NEXT: orq $128, %rcx
+; SSE2-NEXT: orq %rdx, %rdi
+; SSE2-NEXT: cmovneq %r9, %rcx
; SSE2-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: movabsq $-9223372036854775808, %rdx # imm = 0x8000000000000000
@@ -3016,41 +3101,45 @@ define i256 @isolate_msb_i256_load(ptr %p0, i256 %idx) nounwind {
; SSE2-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: movl %ecx, %edx
; SSE2-NEXT: shrb $6, %dl
-; SSE2-NEXT: movzbl %dl, %esi
+; SSE2-NEXT: movzbl %dl, %edi
; SSE2-NEXT: movq $0, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movq -48(%rsp,%rsi,8), %rdx
-; SSE2-NEXT: movq -56(%rsp,%rsi,8), %r8
-; SSE2-NEXT: movq %r8, %r9
-; SSE2-NEXT: shrdq %cl, %rdx, %r9
-; SSE2-NEXT: movq -64(%rsp,%rsi,8), %r10
-; SSE2-NEXT: movq %r10, %r11
-; SSE2-NEXT: shrdq %cl, %r8, %r11
-; SSE2-NEXT: movq -72(%rsp,%rsi,8), %rsi
+; SSE2-NEXT: movq -48(%rsp,%rdi,8), %rdx
+; SSE2-NEXT: movq -56(%rsp,%rdi,8), %r9
+; SSE2-NEXT: movq %r9, %rsi
+; SSE2-NEXT: shrdq %cl, %rdx, %rsi
+; SSE2-NEXT: movq -64(%rsp,%rdi,8), %r10
+; SSE2-NEXT: movq %r10, %r8
+; SSE2-NEXT: shrdq %cl, %r9, %r8
+; SSE2-NEXT: movq -72(%rsp,%rdi,8), %rdi
; SSE2-NEXT: shrq %cl, %rdx
; SSE2-NEXT: # kill: def $cl killed $cl killed $rcx
-; SSE2-NEXT: shrdq %cl, %r10, %rsi
-; SSE2-NEXT: xorl %ecx, %ecx
-; SSE2-NEXT: testl %eax, %eax
-; SSE2-NEXT: cmoveq %rcx, %r11
-; SSE2-NEXT: cmoveq %rcx, %r9
-; SSE2-NEXT: cmoveq %rcx, %rsi
-; SSE2-NEXT: movq %rdi, %rax
-; SSE2-NEXT: cmoveq %rcx, %rdx
-; SSE2-NEXT: movq %rdx, 24(%rdi)
-; SSE2-NEXT: movq %r9, 16(%rdi)
-; SSE2-NEXT: movq %r11, 8(%rdi)
-; SSE2-NEXT: movq %rsi, (%rdi)
+; SSE2-NEXT: shrdq %cl, %r10, %rdi
+; SSE2-NEXT: jmp .LBB17_3
+; SSE2-NEXT: .LBB17_1:
+; SSE2-NEXT: xorl %edi, %edi
+; SSE2-NEXT: xorl %r8d, %r8d
+; SSE2-NEXT: xorl %esi, %esi
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: .LBB17_3: # %select.end
+; SSE2-NEXT: movq %rdi, (%rax)
+; SSE2-NEXT: movq %r8, 8(%rax)
+; SSE2-NEXT: movq %rsi, 16(%rax)
+; SSE2-NEXT: movq %rdx, 24(%rax)
; SSE2-NEXT: retq
;
; SSE42-LABEL: isolate_msb_i256_load:
; SSE42: # %bb.0:
-; SSE42-NEXT: movq 16(%rsi), %rax
-; SSE42-NEXT: movq 24(%rsi), %rdx
+; SSE42-NEXT: movq %rdi, %rax
; SSE42-NEXT: movdqa (%rsi), %xmm0
; SSE42-NEXT: por 16(%rsi), %xmm0
+; SSE42-NEXT: ptest %xmm0, %xmm0
+; SSE42-NEXT: je .LBB17_1
+; SSE42-NEXT: # %bb.2: # %select.false.sink
+; SSE42-NEXT: movq 24(%rsi), %rdx
+; SSE42-NEXT: movq 16(%rsi), %rdi
; SSE42-NEXT: bsrq %rdx, %rcx
; SSE42-NEXT: xorq $63, %rcx
-; SSE42-NEXT: bsrq %rax, %r8
+; SSE42-NEXT: bsrq %rdi, %r8
; SSE42-NEXT: xorq $63, %r8
; SSE42-NEXT: orq $64, %r8
; SSE42-NEXT: testq %rdx, %rdx
@@ -3064,210 +3153,259 @@ define i256 @isolate_msb_i256_load(ptr %p0, i256 %idx) nounwind {
; SSE42-NEXT: testq %r9, %r9
; SSE42-NEXT: cmovneq %r10, %rcx
; SSE42-NEXT: orq $128, %rcx
-; SSE42-NEXT: orq %rdx, %rax
+; SSE42-NEXT: orq %rdx, %rdi
; SSE42-NEXT: cmovneq %r8, %rcx
-; SSE42-NEXT: xorps %xmm1, %xmm1
-; SSE42-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movabsq $-9223372036854775808, %rax # imm = 0x8000000000000000
-; SSE42-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movl %ecx, %eax
-; SSE42-NEXT: shrb $6, %al
-; SSE42-NEXT: movzbl %al, %eax
+; SSE42-NEXT: pxor %xmm0, %xmm0
+; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movabsq $-9223372036854775808, %rdx # imm = 0x8000000000000000
+; SSE42-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movl %ecx, %edx
+; SSE42-NEXT: shrb $6, %dl
+; SSE42-NEXT: movzbl %dl, %edi
; SSE42-NEXT: movq $0, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movq -48(%rsp,%rax,8), %rdx
-; SSE42-NEXT: movq -56(%rsp,%rax,8), %rsi
-; SSE42-NEXT: movq %rsi, %r8
-; SSE42-NEXT: shrdq %cl, %rdx, %r8
-; SSE42-NEXT: movq -64(%rsp,%rax,8), %r9
-; SSE42-NEXT: movq %r9, %r10
-; SSE42-NEXT: shrdq %cl, %rsi, %r10
-; SSE42-NEXT: movq -72(%rsp,%rax,8), %rsi
+; SSE42-NEXT: movq -48(%rsp,%rdi,8), %rdx
+; SSE42-NEXT: movq -56(%rsp,%rdi,8), %r9
+; SSE42-NEXT: movq %r9, %rsi
+; SSE42-NEXT: shrdq %cl, %rdx, %rsi
+; SSE42-NEXT: movq -64(%rsp,%rdi,8), %r10
+; SSE42-NEXT: movq %r10, %r8
+; SSE42-NEXT: shrdq %cl, %r9, %r8
+; SSE42-NEXT: movq -72(%rsp,%rdi,8), %rdi
; SSE42-NEXT: shrq %cl, %rdx
; SSE42-NEXT: # kill: def $cl killed $cl killed $rcx
-; SSE42-NEXT: shrdq %cl, %r9, %rsi
-; SSE42-NEXT: xorl %ecx, %ecx
-; SSE42-NEXT: ptest %xmm0, %xmm0
-; SSE42-NEXT: cmoveq %rcx, %r10
-; SSE42-NEXT: cmoveq %rcx, %r8
-; SSE42-NEXT: cmoveq %rcx, %rsi
-; SSE42-NEXT: movq %rdi, %rax
-; SSE42-NEXT: cmoveq %rcx, %rdx
-; SSE42-NEXT: movq %rdx, 24(%rdi)
-; SSE42-NEXT: movq %r8, 16(%rdi)
-; SSE42-NEXT: movq %r10, 8(%rdi)
-; SSE42-NEXT: movq %rsi, (%rdi)
+; SSE42-NEXT: shrdq %cl, %r10, %rdi
+; SSE42-NEXT: jmp .LBB17_3
+; SSE42-NEXT: .LBB17_1:
+; SSE42-NEXT: xorl %edi, %edi
+; SSE42-NEXT: xorl %r8d, %r8d
+; SSE42-NEXT: xorl %esi, %esi
+; SSE42-NEXT: xorl %edx, %edx
+; SSE42-NEXT: .LBB17_3: # %select.end
+; SSE42-NEXT: movq %rdi, (%rax)
+; SSE42-NEXT: movq %r8, 8(%rax)
+; SSE42-NEXT: movq %rsi, 16(%rax)
+; SSE42-NEXT: movq %rdx, 24(%rax)
; SSE42-NEXT: retq
;
; AVX2-LABEL: isolate_msb_i256_load:
; AVX2: # %bb.0:
-; AVX2-NEXT: movq 16(%rsi), %rax
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: vmovdqu (%rsi), %ymm0
+; AVX2-NEXT: vptest %ymm0, %ymm0
+; AVX2-NEXT: je .LBB17_1
+; AVX2-NEXT: # %bb.2: # %select.false.sink
; AVX2-NEXT: movq 24(%rsi), %rdx
-; AVX2-NEXT: lzcntq %rdx, %rcx
-; AVX2-NEXT: lzcntq %rax, %r8
-; AVX2-NEXT: addq $64, %r8
+; AVX2-NEXT: movq 16(%rsi), %rdi
+; AVX2-NEXT: movq (%rsi), %rcx
+; AVX2-NEXT: movq 8(%rsi), %rsi
+; AVX2-NEXT: lzcntq %rdx, %r8
+; AVX2-NEXT: lzcntq %rdi, %r9
+; AVX2-NEXT: addq $64, %r9
; AVX2-NEXT: testq %rdx, %rdx
-; AVX2-NEXT: cmovneq %rcx, %r8
-; AVX2-NEXT: movq 8(%rsi), %r9
-; AVX2-NEXT: lzcntq %r9, %r10
-; AVX2-NEXT: xorl %ecx, %ecx
-; AVX2-NEXT: lzcntq (%rsi), %rcx
+; AVX2-NEXT: cmovneq %r8, %r9
+; AVX2-NEXT: xorl %r8d, %r8d
+; AVX2-NEXT: lzcntq %rsi, %r8
+; AVX2-NEXT: lzcntq %rcx, %rcx
; AVX2-NEXT: addq $64, %rcx
-; AVX2-NEXT: testq %r9, %r9
-; AVX2-NEXT: cmovneq %r10, %rcx
-; AVX2-NEXT: subq $-128, %rcx
-; AVX2-NEXT: orq %rdx, %rax
+; AVX2-NEXT: testq %rsi, %rsi
; AVX2-NEXT: cmovneq %r8, %rcx
-; AVX2-NEXT: vmovdqu (%rsi), %ymm0
-; AVX2-NEXT: vmovaps {{.*#+}} ymm1 = [0,0,0,9223372036854775808]
-; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movl %ecx, %eax
-; AVX2-NEXT: shrb $6, %al
-; AVX2-NEXT: movzbl %al, %edx
-; AVX2-NEXT: movq -48(%rsp,%rdx,8), %rsi
-; AVX2-NEXT: movq -56(%rsp,%rdx,8), %r8
-; AVX2-NEXT: movq %r8, %r9
-; AVX2-NEXT: shrdq %cl, %rsi, %r9
-; AVX2-NEXT: movq %rdi, %rax
-; AVX2-NEXT: movq -72(%rsp,%rdx,8), %rdi
-; AVX2-NEXT: movq -64(%rsp,%rdx,8), %rdx
-; AVX2-NEXT: movq %rdx, %r10
-; AVX2-NEXT: shrdq %cl, %r8, %r10
-; AVX2-NEXT: shrdq %cl, %rdx, %rdi
+; AVX2-NEXT: subq $-128, %rcx
+; AVX2-NEXT: orq %rdx, %rdi
+; AVX2-NEXT: cmovneq %r9, %rcx
+; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovdqa {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
+; AVX2-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movl %ecx, %edx
+; AVX2-NEXT: shrb $6, %dl
+; AVX2-NEXT: movzbl %dl, %esi
+; AVX2-NEXT: movq -48(%rsp,%rsi,8), %r8
+; AVX2-NEXT: movq -56(%rsp,%rsi,8), %r9
+; AVX2-NEXT: movq %r9, %rdx
+; AVX2-NEXT: shrdq %cl, %r8, %rdx
+; AVX2-NEXT: movq -64(%rsp,%rsi,8), %r10
+; AVX2-NEXT: movq %r10, %rdi
+; AVX2-NEXT: shrdq %cl, %r9, %rdi
+; AVX2-NEXT: movq -72(%rsp,%rsi,8), %rsi
+; AVX2-NEXT: shrxq %rcx, %r8, %r8
+; AVX2-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX2-NEXT: shrdq %cl, %r10, %rsi
+; AVX2-NEXT: jmp .LBB17_3
+; AVX2-NEXT: .LBB17_1:
+; AVX2-NEXT: xorl %esi, %esi
+; AVX2-NEXT: xorl %edi, %edi
; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: vptest %ymm0, %ymm0
-; AVX2-NEXT: shrxq %rcx, %rsi, %rcx
-; AVX2-NEXT: cmoveq %rdx, %r10
-; AVX2-NEXT: cmoveq %rdx, %r9
-; AVX2-NEXT: cmoveq %rdx, %rdi
-; AVX2-NEXT: cmoveq %rdx, %rcx
-; AVX2-NEXT: movq %rcx, 24(%rax)
-; AVX2-NEXT: movq %r9, 16(%rax)
-; AVX2-NEXT: movq %r10, 8(%rax)
-; AVX2-NEXT: movq %rdi, (%rax)
+; AVX2-NEXT: xorl %r8d, %r8d
+; AVX2-NEXT: .LBB17_3: # %select.end
+; AVX2-NEXT: movq %rsi, (%rax)
+; AVX2-NEXT: movq %rdi, 8(%rax)
+; AVX2-NEXT: movq %rdx, 16(%rax)
+; AVX2-NEXT: movq %r8, 24(%rax)
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512F-LABEL: isolate_msb_i256_load:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vmovdqu (%rsi), %ymm0
-; AVX512F-NEXT: vmovaps {{.*#+}} zmm1 = [0,0,0,9223372036854775808,0,0,0,0]
-; AVX512F-NEXT: vmovups %zmm1, -{{[0-9]+}}(%rsp)
-; AVX512F-NEXT: vpermq {{.*#+}} ymm1 = ymm0[3,2,1,0]
-; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k0
-; AVX512F-NEXT: vplzcntq %zmm1, %zmm1
-; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,64,128,192]
-; AVX512F-NEXT: kshiftlw $12, %k0, %k0
-; AVX512F-NEXT: kshiftrw $12, %k0, %k1
-; AVX512F-NEXT: vpcompressq %zmm1, %zmm1 {%k1} {z}
-; AVX512F-NEXT: vmovd %xmm1, %ecx
-; AVX512F-NEXT: movl %ecx, %eax
-; AVX512F-NEXT: shrb $6, %al
-; AVX512F-NEXT: movzbl %al, %edx
-; AVX512F-NEXT: movq -48(%rsp,%rdx,8), %rsi
-; AVX512F-NEXT: movq -56(%rsp,%rdx,8), %r8
-; AVX512F-NEXT: movq %r8, %r9
-; AVX512F-NEXT: shrdq %cl, %rsi, %r9
; AVX512F-NEXT: movq %rdi, %rax
-; AVX512F-NEXT: movq -72(%rsp,%rdx,8), %rdi
-; AVX512F-NEXT: movq -64(%rsp,%rdx,8), %rdx
-; AVX512F-NEXT: movq %rdx, %r10
-; AVX512F-NEXT: shrdq %cl, %r8, %r10
-; AVX512F-NEXT: shrdq %cl, %rdx, %rdi
+; AVX512F-NEXT: vmovdqu (%rsi), %ymm0
; AVX512F-NEXT: vptestmd %zmm0, %zmm0, %k0
-; AVX512F-NEXT: xorl %edx, %edx
; AVX512F-NEXT: kortestw %k0, %k0
-; AVX512F-NEXT: shrxq %rcx, %rsi, %rcx
-; AVX512F-NEXT: cmoveq %rdx, %r10
-; AVX512F-NEXT: cmoveq %rdx, %r9
-; AVX512F-NEXT: cmoveq %rdx, %rdi
-; AVX512F-NEXT: cmoveq %rdx, %rcx
-; AVX512F-NEXT: movq %rcx, 24(%rax)
-; AVX512F-NEXT: movq %r9, 16(%rax)
-; AVX512F-NEXT: movq %r10, 8(%rax)
-; AVX512F-NEXT: movq %rdi, (%rax)
+; AVX512F-NEXT: je .LBB17_1
+; AVX512F-NEXT: # %bb.2: # %select.false.sink
+; AVX512F-NEXT: movq 24(%rsi), %rdx
+; AVX512F-NEXT: movq 16(%rsi), %rdi
+; AVX512F-NEXT: lzcntq %rdx, %rcx
+; AVX512F-NEXT: lzcntq %rdi, %r8
+; AVX512F-NEXT: addq $64, %r8
+; AVX512F-NEXT: testq %rdx, %rdx
+; AVX512F-NEXT: cmovneq %rcx, %r8
+; AVX512F-NEXT: movq 8(%rsi), %r9
+; AVX512F-NEXT: lzcntq %r9, %r10
+; AVX512F-NEXT: lzcntq (%rsi), %rcx
+; AVX512F-NEXT: addq $64, %rcx
+; AVX512F-NEXT: testq %r9, %r9
+; AVX512F-NEXT: cmovneq %r10, %rcx
+; AVX512F-NEXT: subq $-128, %rcx
+; AVX512F-NEXT: orq %rdx, %rdi
+; AVX512F-NEXT: cmovneq %r8, %rcx
+; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm0 = [0,0,0,9223372036854775808,0,0,0,0]
+; AVX512F-NEXT: vmovdqu64 %zmm0, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movl %ecx, %edx
+; AVX512F-NEXT: shrb $6, %dl
+; AVX512F-NEXT: movzbl %dl, %esi
+; AVX512F-NEXT: movq -48(%rsp,%rsi,8), %r8
+; AVX512F-NEXT: movq -56(%rsp,%rsi,8), %r9
+; AVX512F-NEXT: movq %r9, %rdx
+; AVX512F-NEXT: shrdq %cl, %r8, %rdx
+; AVX512F-NEXT: movq -64(%rsp,%rsi,8), %r10
+; AVX512F-NEXT: movq %r10, %rdi
+; AVX512F-NEXT: shrdq %cl, %r9, %rdi
+; AVX512F-NEXT: movq -72(%rsp,%rsi,8), %rsi
+; AVX512F-NEXT: shrxq %rcx, %r8, %r8
+; AVX512F-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX512F-NEXT: shrdq %cl, %r10, %rsi
+; AVX512F-NEXT: jmp .LBB17_3
+; AVX512F-NEXT: .LBB17_1:
+; AVX512F-NEXT: xorl %esi, %esi
+; AVX512F-NEXT: xorl %edi, %edi
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: xorl %r8d, %r8d
+; AVX512F-NEXT: .LBB17_3: # %select.end
+; AVX512F-NEXT: movq %rsi, (%rax)
+; AVX512F-NEXT: movq %rdi, 8(%rax)
+; AVX512F-NEXT: movq %rdx, 16(%rax)
+; AVX512F-NEXT: movq %r8, 24(%rax)
; AVX512F-NEXT: retq
;
; AVX512VL-LABEL: isolate_msb_i256_load:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vmovdqu (%rsi), %ymm0
-; AVX512VL-NEXT: vpermq {{.*#+}} ymm1 = ymm0[3,2,1,0]
-; AVX512VL-NEXT: vptestmq %ymm1, %ymm1, %k1
-; AVX512VL-NEXT: vplzcntq %ymm1, %ymm1
-; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,64,128,192]
-; AVX512VL-NEXT: vpcompressq %ymm1, %ymm1 {%k1} {z}
-; AVX512VL-NEXT: vmovaps {{.*#+}} ymm2 = [0,0,0,9223372036854775808]
-; AVX512VL-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: vxorps %xmm2, %xmm2, %xmm2
-; AVX512VL-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: vmovd %xmm1, %ecx
-; AVX512VL-NEXT: movl %ecx, %eax
-; AVX512VL-NEXT: shrb $6, %al
-; AVX512VL-NEXT: movzbl %al, %edx
-; AVX512VL-NEXT: movq -48(%rsp,%rdx,8), %rsi
-; AVX512VL-NEXT: movq -56(%rsp,%rdx,8), %rax
-; AVX512VL-NEXT: movq %rax, %r8
-; AVX512VL-NEXT: shrdq %cl, %rsi, %r8
-; AVX512VL-NEXT: movq -64(%rsp,%rdx,8), %r9
-; AVX512VL-NEXT: movq %r9, %r10
-; AVX512VL-NEXT: shrdq %cl, %rax, %r10
; AVX512VL-NEXT: movq %rdi, %rax
-; AVX512VL-NEXT: movq -72(%rsp,%rdx,8), %rdx
-; AVX512VL-NEXT: shrdq %cl, %r9, %rdx
-; AVX512VL-NEXT: xorl %edi, %edi
+; AVX512VL-NEXT: vmovdqu (%rsi), %ymm0
; AVX512VL-NEXT: vptest %ymm0, %ymm0
-; AVX512VL-NEXT: shrxq %rcx, %rsi, %rcx
-; AVX512VL-NEXT: cmoveq %rdi, %r10
-; AVX512VL-NEXT: cmoveq %rdi, %r8
-; AVX512VL-NEXT: cmoveq %rdi, %rdx
-; AVX512VL-NEXT: cmoveq %rdi, %rcx
-; AVX512VL-NEXT: movq %rcx, 24(%rax)
-; AVX512VL-NEXT: movq %r8, 16(%rax)
-; AVX512VL-NEXT: movq %r10, 8(%rax)
-; AVX512VL-NEXT: movq %rdx, (%rax)
+; AVX512VL-NEXT: je .LBB17_1
+; AVX512VL-NEXT: # %bb.2: # %select.false.sink
+; AVX512VL-NEXT: movq 24(%rsi), %rdx
+; AVX512VL-NEXT: movq 16(%rsi), %rdi
+; AVX512VL-NEXT: movq 8(%rsi), %r8
+; AVX512VL-NEXT: lzcntq %rdx, %rcx
+; AVX512VL-NEXT: lzcntq %rdi, %r9
+; AVX512VL-NEXT: addq $64, %r9
+; AVX512VL-NEXT: testq %rdx, %rdx
+; AVX512VL-NEXT: cmovneq %rcx, %r9
+; AVX512VL-NEXT: lzcntq %r8, %r10
+; AVX512VL-NEXT: lzcntq (%rsi), %rcx
+; AVX512VL-NEXT: addq $64, %rcx
+; AVX512VL-NEXT: testq %r8, %r8
+; AVX512VL-NEXT: cmovneq %r10, %rcx
+; AVX512VL-NEXT: subq $-128, %rcx
+; AVX512VL-NEXT: orq %rdx, %rdi
+; AVX512VL-NEXT: cmovneq %r9, %rcx
+; AVX512VL-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512VL-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: vmovdqa {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
+; AVX512VL-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movl %ecx, %edx
+; AVX512VL-NEXT: shrb $6, %dl
+; AVX512VL-NEXT: movzbl %dl, %esi
+; AVX512VL-NEXT: movq -48(%rsp,%rsi,8), %r8
+; AVX512VL-NEXT: movq -56(%rsp,%rsi,8), %r9
+; AVX512VL-NEXT: movq %r9, %rdx
+; AVX512VL-NEXT: shrdq %cl, %r8, %rdx
+; AVX512VL-NEXT: movq -64(%rsp,%rsi,8), %r10
+; AVX512VL-NEXT: movq %r10, %rdi
+; AVX512VL-NEXT: shrdq %cl, %r9, %rdi
+; AVX512VL-NEXT: movq -72(%rsp,%rsi,8), %rsi
+; AVX512VL-NEXT: shrxq %rcx, %r8, %r8
+; AVX512VL-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX512VL-NEXT: shrdq %cl, %r10, %rsi
+; AVX512VL-NEXT: jmp .LBB17_3
+; AVX512VL-NEXT: .LBB17_1:
+; AVX512VL-NEXT: xorl %esi, %esi
+; AVX512VL-NEXT: xorl %edi, %edi
+; AVX512VL-NEXT: xorl %edx, %edx
+; AVX512VL-NEXT: xorl %r8d, %r8d
+; AVX512VL-NEXT: .LBB17_3: # %select.end
+; AVX512VL-NEXT: movq %rsi, (%rax)
+; AVX512VL-NEXT: movq %rdi, 8(%rax)
+; AVX512VL-NEXT: movq %rdx, 16(%rax)
+; AVX512VL-NEXT: movq %r8, 24(%rax)
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VBMI-LABEL: isolate_msb_i256_load:
; AVX512VBMI: # %bb.0:
-; AVX512VBMI-NEXT: vmovdqu (%rsi), %ymm0
-; AVX512VBMI-NEXT: vpermq {{.*#+}} ymm1 = ymm0[3,2,1,0]
-; AVX512VBMI-NEXT: vptestmq %ymm1, %ymm1, %k1
-; AVX512VBMI-NEXT: vplzcntq %ymm1, %ymm1
-; AVX512VBMI-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,64,128,192]
-; AVX512VBMI-NEXT: vpcompressq %ymm1, %ymm1 {%k1} {z}
-; AVX512VBMI-NEXT: vmovaps {{.*#+}} ymm2 = [0,0,0,9223372036854775808]
-; AVX512VBMI-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: vxorps %xmm2, %xmm2, %xmm2
-; AVX512VBMI-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: vmovd %xmm1, %ecx
-; AVX512VBMI-NEXT: movl %ecx, %eax
-; AVX512VBMI-NEXT: shrb $6, %al
-; AVX512VBMI-NEXT: movzbl %al, %edx
-; AVX512VBMI-NEXT: movq -48(%rsp,%rdx,8), %rsi
-; AVX512VBMI-NEXT: movq -56(%rsp,%rdx,8), %rax
-; AVX512VBMI-NEXT: movq %rax, %r8
-; AVX512VBMI-NEXT: shrdq %cl, %rsi, %r8
-; AVX512VBMI-NEXT: movq -64(%rsp,%rdx,8), %r9
-; AVX512VBMI-NEXT: movq %r9, %r10
-; AVX512VBMI-NEXT: shrdq %cl, %rax, %r10
; AVX512VBMI-NEXT: movq %rdi, %rax
-; AVX512VBMI-NEXT: movq -72(%rsp,%rdx,8), %rdx
-; AVX512VBMI-NEXT: shrdq %cl, %r9, %rdx
-; AVX512VBMI-NEXT: xorl %edi, %edi
+; AVX512VBMI-NEXT: vmovdqu (%rsi), %ymm0
; AVX512VBMI-NEXT: vptest %ymm0, %ymm0
-; AVX512VBMI-NEXT: shrxq %rcx, %rsi, %rcx
-; AVX512VBMI-NEXT: cmoveq %rdi, %r10
-; AVX512VBMI-NEXT: cmoveq %rdi, %r8
-; AVX512VBMI-NEXT: cmoveq %rdi, %rdx
-; AVX512VBMI-NEXT: cmoveq %rdi, %rcx
-; AVX512VBMI-NEXT: movq %rcx, 24(%rax)
-; AVX512VBMI-NEXT: movq %r8, 16(%rax)
-; AVX512VBMI-NEXT: movq %r10, 8(%rax)
-; AVX512VBMI-NEXT: movq %rdx, (%rax)
+; AVX512VBMI-NEXT: je .LBB17_1
+; AVX512VBMI-NEXT: # %bb.2: # %select.false.sink
+; AVX512VBMI-NEXT: movq 24(%rsi), %rdx
+; AVX512VBMI-NEXT: movq 16(%rsi), %rdi
+; AVX512VBMI-NEXT: movq 8(%rsi), %r8
+; AVX512VBMI-NEXT: lzcntq %rdx, %rcx
+; AVX512VBMI-NEXT: lzcntq %rdi, %r9
+; AVX512VBMI-NEXT: addq $64, %r9
+; AVX512VBMI-NEXT: testq %rdx, %rdx
+; AVX512VBMI-NEXT: cmovneq %rcx, %r9
+; AVX512VBMI-NEXT: lzcntq %r8, %r10
+; AVX512VBMI-NEXT: lzcntq (%rsi), %rcx
+; AVX512VBMI-NEXT: addq $64, %rcx
+; AVX512VBMI-NEXT: testq %r8, %r8
+; AVX512VBMI-NEXT: cmovneq %r10, %rcx
+; AVX512VBMI-NEXT: subq $-128, %rcx
+; AVX512VBMI-NEXT: orq %rdx, %rdi
+; AVX512VBMI-NEXT: cmovneq %r9, %rcx
+; AVX512VBMI-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512VBMI-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: vmovdqa {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
+; AVX512VBMI-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movl %ecx, %edx
+; AVX512VBMI-NEXT: shrb $6, %dl
+; AVX512VBMI-NEXT: movzbl %dl, %esi
+; AVX512VBMI-NEXT: movq -48(%rsp,%rsi,8), %r8
+; AVX512VBMI-NEXT: movq -56(%rsp,%rsi,8), %r9
+; AVX512VBMI-NEXT: movq %r9, %rdx
+; AVX512VBMI-NEXT: shrdq %cl, %r8, %rdx
+; AVX512VBMI-NEXT: movq -64(%rsp,%rsi,8), %r10
+; AVX512VBMI-NEXT: movq %r10, %rdi
+; AVX512VBMI-NEXT: shrdq %cl, %r9, %rdi
+; AVX512VBMI-NEXT: movq -72(%rsp,%rsi,8), %rsi
+; AVX512VBMI-NEXT: shrxq %rcx, %r8, %r8
+; AVX512VBMI-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX512VBMI-NEXT: shrdq %cl, %r10, %rsi
+; AVX512VBMI-NEXT: jmp .LBB17_3
+; AVX512VBMI-NEXT: .LBB17_1:
+; AVX512VBMI-NEXT: xorl %esi, %esi
+; AVX512VBMI-NEXT: xorl %edi, %edi
+; AVX512VBMI-NEXT: xorl %edx, %edx
+; AVX512VBMI-NEXT: xorl %r8d, %r8d
+; AVX512VBMI-NEXT: .LBB17_3: # %select.end
+; AVX512VBMI-NEXT: movq %rsi, (%rax)
+; AVX512VBMI-NEXT: movq %rdi, 8(%rax)
+; AVX512VBMI-NEXT: movq %rdx, 16(%rax)
+; AVX512VBMI-NEXT: movq %r8, 24(%rax)
; AVX512VBMI-NEXT: vzeroupper
; AVX512VBMI-NEXT: retq
%a0 = load i256, ptr %p0
diff --git a/llvm/test/CodeGen/X86/bit-manip-i512.ll b/llvm/test/CodeGen/X86/bit-manip-i512.ll
index 9e4d00650e613..81694e88b66b0 100644
--- a/llvm/test/CodeGen/X86/bit-manip-i512.ll
+++ b/llvm/test/CodeGen/X86/bit-manip-i512.ll
@@ -4101,62 +4101,64 @@ define i512 @isolate_msb_i512(i512 %a0, i512 %idx) nounwind {
; SSE: # %bb.0:
; SSE-NEXT: pushq %r15
; SSE-NEXT: pushq %r14
-; SSE-NEXT: pushq %r13
-; SSE-NEXT: pushq %r12
; SSE-NEXT: pushq %rbx
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r10
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r11
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rbx
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r14
-; SSE-NEXT: movq %r8, %r10
-; SSE-NEXT: orq %r14, %r10
-; SSE-NEXT: movq %rdx, %rax
-; SSE-NEXT: orq %r11, %rax
-; SSE-NEXT: orq %r10, %rax
-; SSE-NEXT: movq %rcx, %r15
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rdi
+; SSE-NEXT: movq %r8, %rbx
+; SSE-NEXT: orq %r10, %rbx
+; SSE-NEXT: movq %rdx, %r14
+; SSE-NEXT: orq %rdi, %r14
+; SSE-NEXT: orq %rbx, %r14
+; SSE-NEXT: movq %rcx, %rbx
+; SSE-NEXT: orq %r11, %rbx
+; SSE-NEXT: movq %rsi, %r15
+; SSE-NEXT: orq %r9, %r15
; SSE-NEXT: orq %rbx, %r15
-; SSE-NEXT: movq %rsi, %r10
-; SSE-NEXT: orq %r9, %r10
-; SSE-NEXT: orq %r15, %r10
-; SSE-NEXT: bsrq %r14, %r15
+; SSE-NEXT: orq %r14, %r15
+; SSE-NEXT: je .LBB15_1
+; SSE-NEXT: # %bb.2: # %select.false.sink
+; SSE-NEXT: bsrq %r10, %rbx
+; SSE-NEXT: xorq $63, %rbx
+; SSE-NEXT: bsrq %r11, %r14
+; SSE-NEXT: xorq $63, %r14
+; SSE-NEXT: orq $64, %r14
+; SSE-NEXT: testq %r10, %r10
+; SSE-NEXT: cmovneq %rbx, %r14
+; SSE-NEXT: bsrq %rdi, %r15
; SSE-NEXT: xorq $63, %r15
-; SSE-NEXT: bsrq %rbx, %r12
-; SSE-NEXT: xorq $63, %r12
-; SSE-NEXT: orq $64, %r12
-; SSE-NEXT: testq %r14, %r14
-; SSE-NEXT: cmovneq %r15, %r12
-; SSE-NEXT: bsrq %r11, %r13
-; SSE-NEXT: xorq $63, %r13
-; SSE-NEXT: bsrq %r9, %r15
+; SSE-NEXT: bsrq %r9, %rbx
+; SSE-NEXT: xorq $63, %rbx
+; SSE-NEXT: orq $64, %rbx
+; SSE-NEXT: testq %rdi, %rdi
+; SSE-NEXT: cmovneq %r15, %rbx
+; SSE-NEXT: orq $128, %rbx
+; SSE-NEXT: movq %r11, %r15
+; SSE-NEXT: orq %r10, %r15
+; SSE-NEXT: cmovneq %r14, %rbx
+; SSE-NEXT: bsrq %r8, %r14
+; SSE-NEXT: xorq $63, %r14
+; SSE-NEXT: bsrq %rcx, %r15
; SSE-NEXT: xorq $63, %r15
; SSE-NEXT: orq $64, %r15
-; SSE-NEXT: testq %r11, %r11
-; SSE-NEXT: cmovneq %r13, %r15
-; SSE-NEXT: orq $128, %r15
-; SSE-NEXT: movq %rbx, %r13
-; SSE-NEXT: orq %r14, %r13
-; SSE-NEXT: cmovneq %r12, %r15
-; SSE-NEXT: bsrq %r8, %r12
-; SSE-NEXT: xorq $63, %r12
-; SSE-NEXT: bsrq %rcx, %r13
-; SSE-NEXT: xorq $63, %r13
-; SSE-NEXT: orq $64, %r13
; SSE-NEXT: testq %r8, %r8
-; SSE-NEXT: cmovneq %r12, %r13
-; SSE-NEXT: bsrq %rdx, %r12
-; SSE-NEXT: xorq $63, %r12
+; SSE-NEXT: cmovneq %r14, %r15
+; SSE-NEXT: bsrq %rdx, %r14
+; SSE-NEXT: xorq $63, %r14
; SSE-NEXT: bsrq %rsi, %rsi
; SSE-NEXT: xorq $63, %rsi
; SSE-NEXT: orq $64, %rsi
; SSE-NEXT: testq %rdx, %rdx
-; SSE-NEXT: cmovneq %r12, %rsi
+; SSE-NEXT: cmovneq %r14, %rsi
; SSE-NEXT: orq $128, %rsi
; SSE-NEXT: orq %r8, %rcx
-; SSE-NEXT: cmovneq %r13, %rsi
+; SSE-NEXT: cmovneq %r15, %rsi
; SSE-NEXT: orq $256, %rsi # imm = 0x100
-; SSE-NEXT: orq %r14, %r11
-; SSE-NEXT: orq %rbx, %r9
; SSE-NEXT: orq %r11, %r9
-; SSE-NEXT: cmovneq %r15, %rsi
+; SSE-NEXT: orq %r10, %rdi
+; SSE-NEXT: orq %r9, %rdi
+; SSE-NEXT: cmovneq %rbx, %rsi
; SSE-NEXT: xorps %xmm0, %xmm0
; SSE-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
; SSE-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
@@ -4173,246 +4175,288 @@ define i512 @isolate_msb_i512(i512 %a0, i512 %idx) nounwind {
; SSE-NEXT: andl $56, %esi
; SSE-NEXT: movq $0, -{{[0-9]+}}(%rsp)
; SSE-NEXT: movq -72(%rsp,%rsi), %rdx
-; SSE-NEXT: movq -80(%rsp,%rsi), %r11
-; SSE-NEXT: movq %r11, %r8
-; SSE-NEXT: shrdq %cl, %rdx, %r8
-; SSE-NEXT: movq -88(%rsp,%rsi), %rbx
-; SSE-NEXT: movq %rbx, %r9
-; SSE-NEXT: shrdq %cl, %r11, %r9
-; SSE-NEXT: movq -96(%rsp,%rsi), %r14
+; SSE-NEXT: movq -80(%rsp,%rsi), %r9
+; SSE-NEXT: movq %r9, %rdi
+; SSE-NEXT: shrdq %cl, %rdx, %rdi
+; SSE-NEXT: movq -88(%rsp,%rsi), %r10
+; SSE-NEXT: movq %r10, %r8
+; SSE-NEXT: shrdq %cl, %r9, %r8
+; SSE-NEXT: movq -96(%rsp,%rsi), %r11
+; SSE-NEXT: movq %r11, %r9
+; SSE-NEXT: shrdq %cl, %r10, %r9
+; SSE-NEXT: movq -104(%rsp,%rsi), %rbx
+; SSE-NEXT: movq %rbx, %r10
+; SSE-NEXT: shrdq %cl, %r11, %r10
+; SSE-NEXT: movq -112(%rsp,%rsi), %r14
; SSE-NEXT: movq %r14, %r11
; SSE-NEXT: shrdq %cl, %rbx, %r11
-; SSE-NEXT: movq -104(%rsp,%rsi), %r15
+; SSE-NEXT: movq -120(%rsp,%rsi), %r15
; SSE-NEXT: movq %r15, %rbx
; SSE-NEXT: shrdq %cl, %r14, %rbx
-; SSE-NEXT: movq -112(%rsp,%rsi), %r12
-; SSE-NEXT: movq %r12, %r14
-; SSE-NEXT: shrdq %cl, %r15, %r14
-; SSE-NEXT: movq -120(%rsp,%rsi), %r13
-; SSE-NEXT: movq %r13, %r15
-; SSE-NEXT: shrdq %cl, %r12, %r15
; SSE-NEXT: movq -128(%rsp,%rsi), %rsi
; SSE-NEXT: shrq %cl, %rdx
; SSE-NEXT: # kill: def $cl killed $cl killed $ecx
-; SSE-NEXT: shrdq %cl, %r13, %rsi
-; SSE-NEXT: xorl %ecx, %ecx
-; SSE-NEXT: orq %rax, %r10
-; SSE-NEXT: cmoveq %rcx, %r15
-; SSE-NEXT: cmoveq %rcx, %r14
-; SSE-NEXT: cmoveq %rcx, %rbx
-; SSE-NEXT: cmoveq %rcx, %r11
-; SSE-NEXT: cmoveq %rcx, %r9
-; SSE-NEXT: cmoveq %rcx, %r8
-; SSE-NEXT: cmoveq %rcx, %rsi
-; SSE-NEXT: movq %rdi, %rax
-; SSE-NEXT: cmoveq %rcx, %rdx
-; SSE-NEXT: movq %rdx, 56(%rdi)
-; SSE-NEXT: movq %r8, 48(%rdi)
-; SSE-NEXT: movq %r9, 40(%rdi)
-; SSE-NEXT: movq %r11, 32(%rdi)
-; SSE-NEXT: movq %rbx, 24(%rdi)
-; SSE-NEXT: movq %r14, 16(%rdi)
-; SSE-NEXT: movq %r15, 8(%rdi)
-; SSE-NEXT: movq %rsi, (%rdi)
+; SSE-NEXT: shrdq %cl, %r15, %rsi
+; SSE-NEXT: jmp .LBB15_3
+; SSE-NEXT: .LBB15_1:
+; SSE-NEXT: xorl %esi, %esi
+; SSE-NEXT: xorl %ebx, %ebx
+; SSE-NEXT: xorl %r11d, %r11d
+; SSE-NEXT: xorl %r10d, %r10d
+; SSE-NEXT: xorl %r9d, %r9d
+; SSE-NEXT: xorl %r8d, %r8d
+; SSE-NEXT: xorl %edi, %edi
+; SSE-NEXT: xorl %edx, %edx
+; SSE-NEXT: .LBB15_3: # %select.end
+; SSE-NEXT: movq %rsi, (%rax)
+; SSE-NEXT: movq %rbx, 8(%rax)
+; SSE-NEXT: movq %r11, 16(%rax)
+; SSE-NEXT: movq %r10, 24(%rax)
+; SSE-NEXT: movq %r9, 32(%rax)
+; SSE-NEXT: movq %r8, 40(%rax)
+; SSE-NEXT: movq %rdi, 48(%rax)
+; SSE-NEXT: movq %rdx, 56(%rax)
; SSE-NEXT: popq %rbx
-; SSE-NEXT: popq %r12
-; SSE-NEXT: popq %r13
; SSE-NEXT: popq %r14
; SSE-NEXT: popq %r15
; SSE-NEXT: retq
;
; AVX2-LABEL: isolate_msb_i512:
; AVX2: # %bb.0:
-; AVX2-NEXT: pushq %rbp
; AVX2-NEXT: pushq %r15
; AVX2-NEXT: pushq %r14
-; AVX2-NEXT: pushq %r13
-; AVX2-NEXT: pushq %r12
; AVX2-NEXT: pushq %rbx
-; AVX2-NEXT: pushq %rax
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rbx
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r14
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r15
-; AVX2-NEXT: movq %r8, %rax
-; AVX2-NEXT: orq %r15, %rax
-; AVX2-NEXT: movq %rdx, %r10
-; AVX2-NEXT: orq %rbx, %r10
-; AVX2-NEXT: orq %rax, %r10
-; AVX2-NEXT: movq %rcx, %rax
-; AVX2-NEXT: orq %r14, %rax
-; AVX2-NEXT: movq %rsi, %r11
-; AVX2-NEXT: orq %r9, %r11
-; AVX2-NEXT: orq %rax, %r11
-; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: lzcntq %r15, %rax
-; AVX2-NEXT: xorl %r13d, %r13d
-; AVX2-NEXT: lzcntq %r14, %r13
-; AVX2-NEXT: addq $64, %r13
-; AVX2-NEXT: testq %r15, %r15
-; AVX2-NEXT: cmovneq %rax, %r13
-; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: lzcntq %rbx, %rax
-; AVX2-NEXT: xorl %r12d, %r12d
-; AVX2-NEXT: lzcntq %r9, %r12
-; AVX2-NEXT: addq $64, %r12
-; AVX2-NEXT: testq %rbx, %rbx
-; AVX2-NEXT: cmovneq %rax, %r12
-; AVX2-NEXT: subq $-128, %r12
-; AVX2-NEXT: movq %r14, %rax
-; AVX2-NEXT: orq %r15, %rax
-; AVX2-NEXT: cmovneq %r13, %r12
-; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: lzcntq %r8, %rax
-; AVX2-NEXT: xorl %r13d, %r13d
-; AVX2-NEXT: lzcntq %rcx, %r13
-; AVX2-NEXT: addq $64, %r13
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdi
+; AVX2-NEXT: movq %r8, %rbx
+; AVX2-NEXT: orq %r10, %rbx
+; AVX2-NEXT: movq %rdx, %r14
+; AVX2-NEXT: orq %rdi, %r14
+; AVX2-NEXT: orq %rbx, %r14
+; AVX2-NEXT: movq %rcx, %rbx
+; AVX2-NEXT: orq %r11, %rbx
+; AVX2-NEXT: movq %rsi, %r15
+; AVX2-NEXT: orq %r9, %r15
+; AVX2-NEXT: orq %rbx, %r15
+; AVX2-NEXT: orq %r14, %r15
+; AVX2-NEXT: je .LBB15_1
+; AVX2-NEXT: # %bb.2: # %select.false.sink
+; AVX2-NEXT: xorl %ebx, %ebx
+; AVX2-NEXT: lzcntq %r10, %rbx
+; AVX2-NEXT: xorl %r14d, %r14d
+; AVX2-NEXT: lzcntq %r11, %r14
+; AVX2-NEXT: addq $64, %r14
+; AVX2-NEXT: testq %r10, %r10
+; AVX2-NEXT: cmovneq %rbx, %r14
+; AVX2-NEXT: xorl %r15d, %r15d
+; AVX2-NEXT: lzcntq %rdi, %r15
+; AVX2-NEXT: xorl %ebx, %ebx
+; AVX2-NEXT: lzcntq %r9, %rbx
+; AVX2-NEXT: addq $64, %rbx
+; AVX2-NEXT: testq %rdi, %rdi
+; AVX2-NEXT: cmovneq %r15, %rbx
+; AVX2-NEXT: subq $-128, %rbx
+; AVX2-NEXT: movq %r11, %r15
+; AVX2-NEXT: orq %r10, %r15
+; AVX2-NEXT: cmovneq %r14, %rbx
+; AVX2-NEXT: xorl %r14d, %r14d
+; AVX2-NEXT: lzcntq %r8, %r14
+; AVX2-NEXT: xorl %r15d, %r15d
+; AVX2-NEXT: lzcntq %rcx, %r15
+; AVX2-NEXT: addq $64, %r15
; AVX2-NEXT: testq %r8, %r8
-; AVX2-NEXT: cmovneq %rax, %r13
-; AVX2-NEXT: xorl %ebp, %ebp
-; AVX2-NEXT: lzcntq %rdx, %rbp
-; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: lzcntq %rsi, %rax
-; AVX2-NEXT: addq $64, %rax
+; AVX2-NEXT: cmovneq %r14, %r15
+; AVX2-NEXT: xorl %r14d, %r14d
+; AVX2-NEXT: lzcntq %rdx, %r14
+; AVX2-NEXT: lzcntq %rsi, %rsi
+; AVX2-NEXT: addq $64, %rsi
; AVX2-NEXT: testq %rdx, %rdx
-; AVX2-NEXT: cmovneq %rbp, %rax
-; AVX2-NEXT: subq $-128, %rax
+; AVX2-NEXT: cmovneq %r14, %rsi
+; AVX2-NEXT: subq $-128, %rsi
; AVX2-NEXT: orq %r8, %rcx
-; AVX2-NEXT: cmovneq %r13, %rax
-; AVX2-NEXT: addq $256, %rax # imm = 0x100
-; AVX2-NEXT: orq %r15, %rbx
-; AVX2-NEXT: orq %r14, %r9
-; AVX2-NEXT: orq %rbx, %r9
-; AVX2-NEXT: cmovneq %r12, %rax
-; AVX2-NEXT: vmovaps {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
-; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: cmovneq %r15, %rsi
+; AVX2-NEXT: addq $256, %rsi # imm = 0x100
+; AVX2-NEXT: orq %r11, %r9
+; AVX2-NEXT: orq %r10, %rdi
+; AVX2-NEXT: orq %r9, %rdi
+; AVX2-NEXT: cmovneq %rbx, %rsi
; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movl %eax, %ecx
+; AVX2-NEXT: movl %esi, %ecx
; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: shrl $3, %eax
-; AVX2-NEXT: andl $56, %eax
-; AVX2-NEXT: movq -72(%rsp,%rax), %rbx
-; AVX2-NEXT: movq -80(%rsp,%rax), %r8
+; AVX2-NEXT: shrl $3, %esi
+; AVX2-NEXT: andl $56, %esi
+; AVX2-NEXT: movq -72(%rsp,%rsi), %rbx
+; AVX2-NEXT: movq -80(%rsp,%rsi), %r8
; AVX2-NEXT: movq %r8, %rdx
; AVX2-NEXT: shrdq %cl, %rbx, %rdx
-; AVX2-NEXT: movq -88(%rsp,%rax), %r9
-; AVX2-NEXT: movq %r9, %rsi
-; AVX2-NEXT: shrdq %cl, %r8, %rsi
-; AVX2-NEXT: movq -96(%rsp,%rax), %r14
-; AVX2-NEXT: movq %r14, %r8
+; AVX2-NEXT: movq -88(%rsp,%rsi), %r9
+; AVX2-NEXT: movq %r9, %rdi
+; AVX2-NEXT: shrdq %cl, %r8, %rdi
+; AVX2-NEXT: movq -96(%rsp,%rsi), %r10
+; AVX2-NEXT: movq %r10, %r8
; AVX2-NEXT: shrdq %cl, %r9, %r8
-; AVX2-NEXT: movq -104(%rsp,%rax), %r15
-; AVX2-NEXT: movq %r15, %r9
-; AVX2-NEXT: shrdq %cl, %r14, %r9
-; AVX2-NEXT: movq -112(%rsp,%rax), %r13
-; AVX2-NEXT: movq %r13, %r14
-; AVX2-NEXT: shrdq %cl, %r15, %r14
-; AVX2-NEXT: movq -128(%rsp,%rax), %r15
-; AVX2-NEXT: movq -120(%rsp,%rax), %rax
-; AVX2-NEXT: movq %rax, %r12
-; AVX2-NEXT: shrdq %cl, %r13, %r12
-; AVX2-NEXT: shrdq %cl, %rax, %r15
-; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: movq -104(%rsp,%rsi), %r11
+; AVX2-NEXT: movq %r11, %r9
+; AVX2-NEXT: shrdq %cl, %r10, %r9
+; AVX2-NEXT: movq -112(%rsp,%rsi), %r14
+; AVX2-NEXT: movq %r14, %r10
+; AVX2-NEXT: shrdq %cl, %r11, %r10
+; AVX2-NEXT: movq -120(%rsp,%rsi), %r15
+; AVX2-NEXT: movq %r15, %r11
+; AVX2-NEXT: shrdq %cl, %r14, %r11
+; AVX2-NEXT: movq -128(%rsp,%rsi), %rsi
+; AVX2-NEXT: shrxq %rcx, %rbx, %rbx
+; AVX2-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX2-NEXT: shrdq %cl, %r15, %rsi
+; AVX2-NEXT: jmp .LBB15_3
+; AVX2-NEXT: .LBB15_1:
+; AVX2-NEXT: xorl %esi, %esi
+; AVX2-NEXT: xorl %r11d, %r11d
+; AVX2-NEXT: xorl %r10d, %r10d
+; AVX2-NEXT: xorl %r9d, %r9d
+; AVX2-NEXT: xorl %r8d, %r8d
; AVX2-NEXT: xorl %edi, %edi
-; AVX2-NEXT: orq %r10, %r11
-; AVX2-NEXT: shrxq %rcx, %rbx, %rcx
-; AVX2-NEXT: cmoveq %rdi, %r12
-; AVX2-NEXT: cmoveq %rdi, %r14
-; AVX2-NEXT: cmoveq %rdi, %r9
-; AVX2-NEXT: cmoveq %rdi, %r8
-; AVX2-NEXT: cmoveq %rdi, %rsi
-; AVX2-NEXT: cmoveq %rdi, %rdx
-; AVX2-NEXT: cmoveq %rdi, %r15
-; AVX2-NEXT: cmoveq %rdi, %rcx
-; AVX2-NEXT: movq %rcx, 56(%rax)
-; AVX2-NEXT: movq %rdx, 48(%rax)
-; AVX2-NEXT: movq %rsi, 40(%rax)
-; AVX2-NEXT: movq %r8, 32(%rax)
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: xorl %ebx, %ebx
+; AVX2-NEXT: .LBB15_3: # %select.end
+; AVX2-NEXT: movq %rsi, (%rax)
+; AVX2-NEXT: movq %r11, 8(%rax)
+; AVX2-NEXT: movq %r10, 16(%rax)
; AVX2-NEXT: movq %r9, 24(%rax)
-; AVX2-NEXT: movq %r14, 16(%rax)
-; AVX2-NEXT: movq %r12, 8(%rax)
-; AVX2-NEXT: movq %r15, (%rax)
-; AVX2-NEXT: addq $8, %rsp
+; AVX2-NEXT: movq %r8, 32(%rax)
+; AVX2-NEXT: movq %rdi, 40(%rax)
+; AVX2-NEXT: movq %rdx, 48(%rax)
+; AVX2-NEXT: movq %rbx, 56(%rax)
; AVX2-NEXT: popq %rbx
-; AVX2-NEXT: popq %r12
-; AVX2-NEXT: popq %r13
; AVX2-NEXT: popq %r14
; AVX2-NEXT: popq %r15
-; AVX2-NEXT: popq %rbp
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512F-LABEL: isolate_msb_i512:
; AVX512F: # %bb.0:
+; AVX512F-NEXT: pushq %r15
+; AVX512F-NEXT: pushq %r14
+; AVX512F-NEXT: pushq %rbx
; AVX512F-NEXT: movq %rdi, %rax
; AVX512F-NEXT: movq {{[0-9]+}}(%rsp), %rdi
-; AVX512F-NEXT: vmovq %rdx, %xmm0
-; AVX512F-NEXT: orq %rdi, %rdx
-; AVX512F-NEXT: vmovq %r8, %xmm1
-; AVX512F-NEXT: orq {{[0-9]+}}(%rsp), %r8
-; AVX512F-NEXT: orq %rdx, %r8
-; AVX512F-NEXT: vmovq %rsi, %xmm2
-; AVX512F-NEXT: vmovq %rcx, %xmm3
-; AVX512F-NEXT: orq {{[0-9]+}}(%rsp), %rcx
-; AVX512F-NEXT: orq %r9, %rsi
-; AVX512F-NEXT: orq %rsi, %rcx
-; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: orq %r8, %rcx
-; AVX512F-NEXT: sete %dl
-; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; AVX512F-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX512F-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX512F-NEXT: movq %r8, %rbx
+; AVX512F-NEXT: orq %rdi, %rbx
+; AVX512F-NEXT: movq %rdx, %r14
+; AVX512F-NEXT: orq %r11, %r14
+; AVX512F-NEXT: orq %rbx, %r14
+; AVX512F-NEXT: movq %rcx, %rbx
+; AVX512F-NEXT: orq %r10, %rbx
+; AVX512F-NEXT: movq %rsi, %r15
+; AVX512F-NEXT: orq %r9, %r15
+; AVX512F-NEXT: orq %rbx, %r15
+; AVX512F-NEXT: orq %r14, %r15
+; AVX512F-NEXT: je .LBB15_1
+; AVX512F-NEXT: # %bb.2: # %select.false.sink
+; AVX512F-NEXT: vmovq %rsi, %xmm0
+; AVX512F-NEXT: vmovq %rdx, %xmm1
+; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512F-NEXT: vmovq %rcx, %xmm1
+; AVX512F-NEXT: vmovq %r8, %xmm2
+; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512F-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
-; AVX512F-NEXT: vpshufd {{.*#+}} xmm1 = mem[2,3,0,1]
-; AVX512F-NEXT: vmovq %r9, %xmm2
+; AVX512F-NEXT: vmovq %r9, %xmm1
+; AVX512F-NEXT: vmovq %r11, %xmm2
+; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX512F-NEXT: vmovq %r10, %xmm2
; AVX512F-NEXT: vmovq %rdi, %xmm3
; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
-; AVX512F-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512F-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
; AVX512F-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512F-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1 # [0,64,128,192,256,320,384,448]
; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
-; AVX512F-NEXT: vplzcntq %zmm0, %zmm0
-; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,64,128,192,256,320,384,448]
-; AVX512F-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
; AVX512F-NEXT: vmovq %xmm0, %rcx
-; AVX512F-NEXT: movabsq $-9223372036854775808, %rsi # imm = 0x8000000000000000
-; AVX512F-NEXT: shrxq %rcx, %rsi, %rsi
+; AVX512F-NEXT: movabsq $-9223372036854775808, %rdx # imm = 0x8000000000000000
+; AVX512F-NEXT: shrxq %rcx, %rdx, %rdx
; AVX512F-NEXT: shrl $6, %ecx
-; AVX512F-NEXT: movl $128, %edi
-; AVX512F-NEXT: shrxq %rcx, %rdi, %rcx
+; AVX512F-NEXT: movl $128, %esi
+; AVX512F-NEXT: shrxq %rcx, %rsi, %rcx
; AVX512F-NEXT: kmovw %ecx, %k1
-; AVX512F-NEXT: vpbroadcastq %rsi, %zmm0 {%k1} {z}
-; AVX512F-NEXT: negl %edx
-; AVX512F-NEXT: kmovw %edx, %k0
-; AVX512F-NEXT: knotw %k0, %k1
-; AVX512F-NEXT: vmovdqa32 %zmm0, %zmm0 {%k1} {z}
-; AVX512F-NEXT: vmovdqu64 %zmm0, (%rax)
+; AVX512F-NEXT: vpbroadcastq %rdx, %zmm0 {%k1} {z}
+; AVX512F-NEXT: vextracti32x4 $3, %zmm0, %xmm1
+; AVX512F-NEXT: vpextrq $1, %xmm1, %rcx
+; AVX512F-NEXT: vmovq %xmm1, %rdx
+; AVX512F-NEXT: vextracti32x4 $2, %zmm0, %xmm1
+; AVX512F-NEXT: vpextrq $1, %xmm1, %rsi
+; AVX512F-NEXT: vmovq %xmm1, %r8
+; AVX512F-NEXT: vpextrq $1, %xmm0, %r9
+; AVX512F-NEXT: vmovq %xmm0, %rdi
+; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX512F-NEXT: vpextrq $1, %xmm0, %r10
+; AVX512F-NEXT: vmovq %xmm0, %r11
+; AVX512F-NEXT: jmp .LBB15_3
+; AVX512F-NEXT: .LBB15_1:
+; AVX512F-NEXT: xorl %edi, %edi
+; AVX512F-NEXT: xorl %r9d, %r9d
+; AVX512F-NEXT: xorl %r11d, %r11d
+; AVX512F-NEXT: xorl %r10d, %r10d
+; AVX512F-NEXT: xorl %r8d, %r8d
+; AVX512F-NEXT: xorl %esi, %esi
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: xorl %ecx, %ecx
+; AVX512F-NEXT: .LBB15_3: # %select.end
+; AVX512F-NEXT: movq %rdi, (%rax)
+; AVX512F-NEXT: movq %r9, 8(%rax)
+; AVX512F-NEXT: movq %r11, 16(%rax)
+; AVX512F-NEXT: movq %r10, 24(%rax)
+; AVX512F-NEXT: movq %r8, 32(%rax)
+; AVX512F-NEXT: movq %rsi, 40(%rax)
+; AVX512F-NEXT: movq %rdx, 48(%rax)
+; AVX512F-NEXT: movq %rcx, 56(%rax)
+; AVX512F-NEXT: popq %rbx
+; AVX512F-NEXT: popq %r14
+; AVX512F-NEXT: popq %r15
; AVX512F-NEXT: retq
;
; AVX512VL-LABEL: isolate_msb_i512:
; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: pushq %r15
+; AVX512VL-NEXT: pushq %r14
+; AVX512VL-NEXT: pushq %rbx
; AVX512VL-NEXT: movq %rdi, %rax
; AVX512VL-NEXT: movq {{[0-9]+}}(%rsp), %rdi
-; AVX512VL-NEXT: vmovq %rdx, %xmm0
-; AVX512VL-NEXT: orq %rdi, %rdx
-; AVX512VL-NEXT: vmovq %r8, %xmm1
-; AVX512VL-NEXT: orq {{[0-9]+}}(%rsp), %r8
-; AVX512VL-NEXT: orq %rdx, %r8
-; AVX512VL-NEXT: vmovq %rsi, %xmm2
-; AVX512VL-NEXT: orq %r9, %rsi
-; AVX512VL-NEXT: vmovq %rcx, %xmm3
-; AVX512VL-NEXT: orq {{[0-9]+}}(%rsp), %rcx
-; AVX512VL-NEXT: orq %rsi, %rcx
-; AVX512VL-NEXT: xorl %edx, %edx
-; AVX512VL-NEXT: orq %r8, %rcx
-; AVX512VL-NEXT: sete %dl
-; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
-; AVX512VL-NEXT: vpshufd {{.*#+}} xmm2 = mem[2,3,0,1]
+; AVX512VL-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX512VL-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX512VL-NEXT: movq %r8, %rbx
+; AVX512VL-NEXT: orq %rdi, %rbx
+; AVX512VL-NEXT: movq %rdx, %r14
+; AVX512VL-NEXT: orq %r11, %r14
+; AVX512VL-NEXT: orq %rbx, %r14
+; AVX512VL-NEXT: movq %rcx, %rbx
+; AVX512VL-NEXT: orq %r10, %rbx
+; AVX512VL-NEXT: movq %rsi, %r15
+; AVX512VL-NEXT: orq %r9, %r15
+; AVX512VL-NEXT: orq %rbx, %r15
+; AVX512VL-NEXT: orq %r14, %r15
+; AVX512VL-NEXT: je .LBB15_1
+; AVX512VL-NEXT: # %bb.2: # %select.false.sink
+; AVX512VL-NEXT: vmovq %rsi, %xmm0
+; AVX512VL-NEXT: vmovq %rdx, %xmm1
+; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512VL-NEXT: vmovq %rcx, %xmm1
+; AVX512VL-NEXT: vmovq %r8, %xmm2
+; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512VL-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
; AVX512VL-NEXT: vmovq %r9, %xmm1
+; AVX512VL-NEXT: vmovq %r11, %xmm2
+; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX512VL-NEXT: vmovq %r10, %xmm2
; AVX512VL-NEXT: vmovq %rdi, %xmm3
-; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm3[0],xmm1[0]
+; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
; AVX512VL-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
; AVX512VL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
@@ -4420,45 +4464,84 @@ define i512 @isolate_msb_i512(i512 %a0, i512 %idx) nounwind {
; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,64,128,192,256,320,384,448]
; AVX512VL-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
; AVX512VL-NEXT: vmovq %xmm0, %rcx
-; AVX512VL-NEXT: movabsq $-9223372036854775808, %rsi # imm = 0x8000000000000000
-; AVX512VL-NEXT: shrxq %rcx, %rsi, %rsi
+; AVX512VL-NEXT: movabsq $-9223372036854775808, %rdx # imm = 0x8000000000000000
+; AVX512VL-NEXT: shrxq %rcx, %rdx, %rdx
; AVX512VL-NEXT: shrl $6, %ecx
-; AVX512VL-NEXT: movl $128, %edi
-; AVX512VL-NEXT: shrxq %rcx, %rdi, %rcx
+; AVX512VL-NEXT: movl $128, %esi
+; AVX512VL-NEXT: shrxq %rcx, %rsi, %rcx
; AVX512VL-NEXT: kmovd %ecx, %k1
-; AVX512VL-NEXT: vpbroadcastq %rsi, %zmm0 {%k1} {z}
-; AVX512VL-NEXT: negl %edx
-; AVX512VL-NEXT: kmovd %edx, %k0
-; AVX512VL-NEXT: knotw %k0, %k1
-; AVX512VL-NEXT: vmovdqa32 %zmm0, %zmm0 {%k1} {z}
-; AVX512VL-NEXT: vmovdqu64 %zmm0, (%rax)
+; AVX512VL-NEXT: vpbroadcastq %rdx, %zmm0 {%k1} {z}
+; AVX512VL-NEXT: vextracti32x4 $3, %zmm0, %xmm1
+; AVX512VL-NEXT: vpextrq $1, %xmm1, %rcx
+; AVX512VL-NEXT: vmovq %xmm1, %rdx
+; AVX512VL-NEXT: vextracti32x4 $2, %zmm0, %xmm1
+; AVX512VL-NEXT: vpextrq $1, %xmm1, %rsi
+; AVX512VL-NEXT: vpextrq $1, %xmm0, %r8
+; AVX512VL-NEXT: vmovq %xmm1, %r9
+; AVX512VL-NEXT: vmovq %xmm0, %rdi
+; AVX512VL-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX512VL-NEXT: vpextrq $1, %xmm0, %r10
+; AVX512VL-NEXT: vmovq %xmm0, %r11
+; AVX512VL-NEXT: jmp .LBB15_3
+; AVX512VL-NEXT: .LBB15_1:
+; AVX512VL-NEXT: xorl %edi, %edi
+; AVX512VL-NEXT: xorl %r8d, %r8d
+; AVX512VL-NEXT: xorl %r11d, %r11d
+; AVX512VL-NEXT: xorl %r10d, %r10d
+; AVX512VL-NEXT: xorl %r9d, %r9d
+; AVX512VL-NEXT: xorl %esi, %esi
+; AVX512VL-NEXT: xorl %edx, %edx
+; AVX512VL-NEXT: xorl %ecx, %ecx
+; AVX512VL-NEXT: .LBB15_3: # %select.end
+; AVX512VL-NEXT: movq %rdi, (%rax)
+; AVX512VL-NEXT: movq %r8, 8(%rax)
+; AVX512VL-NEXT: movq %r11, 16(%rax)
+; AVX512VL-NEXT: movq %r10, 24(%rax)
+; AVX512VL-NEXT: movq %r9, 32(%rax)
+; AVX512VL-NEXT: movq %rsi, 40(%rax)
+; AVX512VL-NEXT: movq %rdx, 48(%rax)
+; AVX512VL-NEXT: movq %rcx, 56(%rax)
+; AVX512VL-NEXT: popq %rbx
+; AVX512VL-NEXT: popq %r14
+; AVX512VL-NEXT: popq %r15
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VBMI-LABEL: isolate_msb_i512:
; AVX512VBMI: # %bb.0:
+; AVX512VBMI-NEXT: pushq %r15
+; AVX512VBMI-NEXT: pushq %r14
+; AVX512VBMI-NEXT: pushq %rbx
; AVX512VBMI-NEXT: movq %rdi, %rax
; AVX512VBMI-NEXT: movq {{[0-9]+}}(%rsp), %rdi
-; AVX512VBMI-NEXT: vmovq %rdx, %xmm0
-; AVX512VBMI-NEXT: orq %rdi, %rdx
-; AVX512VBMI-NEXT: vmovq %r8, %xmm1
-; AVX512VBMI-NEXT: orq {{[0-9]+}}(%rsp), %r8
-; AVX512VBMI-NEXT: orq %rdx, %r8
-; AVX512VBMI-NEXT: vmovq %rsi, %xmm2
-; AVX512VBMI-NEXT: orq %r9, %rsi
-; AVX512VBMI-NEXT: vmovq %rcx, %xmm3
-; AVX512VBMI-NEXT: orq {{[0-9]+}}(%rsp), %rcx
-; AVX512VBMI-NEXT: orq %rsi, %rcx
-; AVX512VBMI-NEXT: xorl %edx, %edx
-; AVX512VBMI-NEXT: orq %r8, %rcx
-; AVX512VBMI-NEXT: sete %dl
-; AVX512VBMI-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; AVX512VBMI-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
-; AVX512VBMI-NEXT: vpshufd {{.*#+}} xmm2 = mem[2,3,0,1]
+; AVX512VBMI-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX512VBMI-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; AVX512VBMI-NEXT: movq %r8, %rbx
+; AVX512VBMI-NEXT: orq %rdi, %rbx
+; AVX512VBMI-NEXT: movq %rdx, %r14
+; AVX512VBMI-NEXT: orq %r11, %r14
+; AVX512VBMI-NEXT: orq %rbx, %r14
+; AVX512VBMI-NEXT: movq %rcx, %rbx
+; AVX512VBMI-NEXT: orq %r10, %rbx
+; AVX512VBMI-NEXT: movq %rsi, %r15
+; AVX512VBMI-NEXT: orq %r9, %r15
+; AVX512VBMI-NEXT: orq %rbx, %r15
+; AVX512VBMI-NEXT: orq %r14, %r15
+; AVX512VBMI-NEXT: je .LBB15_1
+; AVX512VBMI-NEXT: # %bb.2: # %select.false.sink
+; AVX512VBMI-NEXT: vmovq %rsi, %xmm0
+; AVX512VBMI-NEXT: vmovq %rdx, %xmm1
+; AVX512VBMI-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512VBMI-NEXT: vmovq %rcx, %xmm1
+; AVX512VBMI-NEXT: vmovq %r8, %xmm2
+; AVX512VBMI-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
; AVX512VBMI-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
; AVX512VBMI-NEXT: vmovq %r9, %xmm1
+; AVX512VBMI-NEXT: vmovq %r11, %xmm2
+; AVX512VBMI-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX512VBMI-NEXT: vmovq %r10, %xmm2
; AVX512VBMI-NEXT: vmovq %rdi, %xmm3
-; AVX512VBMI-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm3[0],xmm1[0]
+; AVX512VBMI-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
; AVX512VBMI-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
; AVX512VBMI-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
; AVX512VBMI-NEXT: vptestmq %zmm0, %zmm0, %k1
@@ -4466,18 +4549,46 @@ define i512 @isolate_msb_i512(i512 %a0, i512 %idx) nounwind {
; AVX512VBMI-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,64,128,192,256,320,384,448]
; AVX512VBMI-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
; AVX512VBMI-NEXT: vmovq %xmm0, %rcx
-; AVX512VBMI-NEXT: movabsq $-9223372036854775808, %rsi # imm = 0x8000000000000000
-; AVX512VBMI-NEXT: shrxq %rcx, %rsi, %rsi
+; AVX512VBMI-NEXT: movabsq $-9223372036854775808, %rdx # imm = 0x8000000000000000
+; AVX512VBMI-NEXT: shrxq %rcx, %rdx, %rdx
; AVX512VBMI-NEXT: shrl $6, %ecx
-; AVX512VBMI-NEXT: movl $128, %edi
-; AVX512VBMI-NEXT: shrxq %rcx, %rdi, %rcx
+; AVX512VBMI-NEXT: movl $128, %esi
+; AVX512VBMI-NEXT: shrxq %rcx, %rsi, %rcx
; AVX512VBMI-NEXT: kmovd %ecx, %k1
-; AVX512VBMI-NEXT: vpbroadcastq %rsi, %zmm0 {%k1} {z}
-; AVX512VBMI-NEXT: negl %edx
-; AVX512VBMI-NEXT: kmovd %edx, %k0
-; AVX512VBMI-NEXT: knotw %k0, %k1
-; AVX512VBMI-NEXT: vmovdqa32 %zmm0, %zmm0 {%k1} {z}
-; AVX512VBMI-NEXT: vmovdqu64 %zmm0, (%rax)
+; AVX512VBMI-NEXT: vpbroadcastq %rdx, %zmm0 {%k1} {z}
+; AVX512VBMI-NEXT: vextracti32x4 $3, %zmm0, %xmm1
+; AVX512VBMI-NEXT: vpextrq $1, %xmm1, %rcx
+; AVX512VBMI-NEXT: vmovq %xmm1, %rdx
+; AVX512VBMI-NEXT: vextracti32x4 $2, %zmm0, %xmm1
+; AVX512VBMI-NEXT: vpextrq $1, %xmm1, %rsi
+; AVX512VBMI-NEXT: vpextrq $1, %xmm0, %r8
+; AVX512VBMI-NEXT: vmovq %xmm1, %r9
+; AVX512VBMI-NEXT: vmovq %xmm0, %rdi
+; AVX512VBMI-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX512VBMI-NEXT: vpextrq $1, %xmm0, %r10
+; AVX512VBMI-NEXT: vmovq %xmm0, %r11
+; AVX512VBMI-NEXT: jmp .LBB15_3
+; AVX512VBMI-NEXT: .LBB15_1:
+; AVX512VBMI-NEXT: xorl %edi, %edi
+; AVX512VBMI-NEXT: xorl %r8d, %r8d
+; AVX512VBMI-NEXT: xorl %r11d, %r11d
+; AVX512VBMI-NEXT: xorl %r10d, %r10d
+; AVX512VBMI-NEXT: xorl %r9d, %r9d
+; AVX512VBMI-NEXT: xorl %esi, %esi
+; AVX512VBMI-NEXT: xorl %edx, %edx
+; AVX512VBMI-NEXT: xorl %ecx, %ecx
+; AVX512VBMI-NEXT: .LBB15_3: # %select.end
+; AVX512VBMI-NEXT: movq %rdi, (%rax)
+; AVX512VBMI-NEXT: movq %r8, 8(%rax)
+; AVX512VBMI-NEXT: movq %r11, 16(%rax)
+; AVX512VBMI-NEXT: movq %r10, 24(%rax)
+; AVX512VBMI-NEXT: movq %r9, 32(%rax)
+; AVX512VBMI-NEXT: movq %rsi, 40(%rax)
+; AVX512VBMI-NEXT: movq %rdx, 48(%rax)
+; AVX512VBMI-NEXT: movq %rcx, 56(%rax)
+; AVX512VBMI-NEXT: popq %rbx
+; AVX512VBMI-NEXT: popq %r14
+; AVX512VBMI-NEXT: popq %r15
; AVX512VBMI-NEXT: vzeroupper
; AVX512VBMI-NEXT: retq
%eqz = icmp eq i512 %a0, 0
@@ -4493,73 +4604,71 @@ define i512 @isolate_msb_i512_vector(<8 x i64> %v0, i512 %idx) nounwind {
; SSE2: # %bb.0:
; SSE2-NEXT: pushq %r15
; SSE2-NEXT: pushq %r14
-; SSE2-NEXT: pushq %r12
; SSE2-NEXT: pushq %rbx
-; SSE2-NEXT: pushq %rax
-; SSE2-NEXT: movq %xmm0, %rdx
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
-; SSE2-NEXT: movq %xmm4, %rcx
-; SSE2-NEXT: movq %xmm1, %r8
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
-; SSE2-NEXT: movq %xmm4, %rsi
-; SSE2-NEXT: movq %xmm2, %r9
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]
-; SSE2-NEXT: movq %xmm4, %r11
-; SSE2-NEXT: movq %xmm3, %r10
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
-; SSE2-NEXT: movq %xmm4, %rbx
+; SSE2-NEXT: movq %rdi, %rax
; SSE2-NEXT: movdqa %xmm1, %xmm4
; SSE2-NEXT: por %xmm3, %xmm4
-; SSE2-NEXT: por %xmm2, %xmm0
-; SSE2-NEXT: por %xmm4, %xmm0
+; SSE2-NEXT: movdqa %xmm0, %xmm5
+; SSE2-NEXT: por %xmm2, %xmm5
+; SSE2-NEXT: por %xmm4, %xmm5
; SSE2-NEXT: pxor %xmm4, %xmm4
-; SSE2-NEXT: pcmpeqd %xmm4, %xmm0
-; SSE2-NEXT: movmskps %xmm0, %eax
-; SSE2-NEXT: xorl $15, %eax
+; SSE2-NEXT: pcmpeqd %xmm4, %xmm5
+; SSE2-NEXT: movmskps %xmm5, %ecx
+; SSE2-NEXT: xorl $15, %ecx
+; SSE2-NEXT: je .LBB16_1
+; SSE2-NEXT: # %bb.2: # %select.false.sink
+; SSE2-NEXT: movq %xmm3, %rsi
+; SSE2-NEXT: movq %xmm2, %rcx
+; SSE2-NEXT: movq %xmm1, %r9
+; SSE2-NEXT: movq %xmm0, %rdx
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
+; SSE2-NEXT: movq %xmm3, %r8
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
+; SSE2-NEXT: movq %xmm2, %rdi
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; SSE2-NEXT: movq %xmm1, %r10
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; SSE2-NEXT: movq %xmm0, %rbx
+; SSE2-NEXT: bsrq %r8, %r11
+; SSE2-NEXT: xorq $63, %r11
+; SSE2-NEXT: bsrq %rsi, %r14
+; SSE2-NEXT: xorq $63, %r14
+; SSE2-NEXT: orq $64, %r14
+; SSE2-NEXT: testq %r8, %r8
+; SSE2-NEXT: cmovneq %r11, %r14
+; SSE2-NEXT: bsrq %rdi, %r15
+; SSE2-NEXT: xorq $63, %r15
+; SSE2-NEXT: bsrq %rcx, %r11
+; SSE2-NEXT: xorq $63, %r11
+; SSE2-NEXT: orq $64, %r11
+; SSE2-NEXT: testq %rdi, %rdi
+; SSE2-NEXT: cmovneq %r15, %r11
+; SSE2-NEXT: orq $128, %r11
+; SSE2-NEXT: movq %rsi, %r15
+; SSE2-NEXT: orq %r8, %r15
+; SSE2-NEXT: cmovneq %r14, %r11
+; SSE2-NEXT: bsrq %r10, %r14
+; SSE2-NEXT: xorq $63, %r14
+; SSE2-NEXT: bsrq %r9, %r15
+; SSE2-NEXT: xorq $63, %r15
+; SSE2-NEXT: orq $64, %r15
+; SSE2-NEXT: testq %r10, %r10
+; SSE2-NEXT: cmovneq %r14, %r15
; SSE2-NEXT: bsrq %rbx, %r14
; SSE2-NEXT: xorq $63, %r14
-; SSE2-NEXT: bsrq %r10, %r10
-; SSE2-NEXT: xorq $63, %r10
-; SSE2-NEXT: orq $64, %r10
-; SSE2-NEXT: testq %rbx, %rbx
-; SSE2-NEXT: cmovneq %r14, %r10
-; SSE2-NEXT: bsrq %r11, %rbx
-; SSE2-NEXT: xorq $63, %rbx
-; SSE2-NEXT: bsrq %r9, %r9
-; SSE2-NEXT: xorq $63, %r9
-; SSE2-NEXT: orq $64, %r9
-; SSE2-NEXT: testq %r11, %r11
-; SSE2-NEXT: cmovneq %rbx, %r9
-; SSE2-NEXT: orq $128, %r9
-; SSE2-NEXT: por %xmm3, %xmm2
-; SSE2-NEXT: pcmpeqd %xmm4, %xmm3
-; SSE2-NEXT: movmskps %xmm3, %r11d
-; SSE2-NEXT: xorl $15, %r11d
-; SSE2-NEXT: cmovneq %r10, %r9
-; SSE2-NEXT: bsrq %rsi, %r10
-; SSE2-NEXT: xorq $63, %r10
-; SSE2-NEXT: bsrq %r8, %r8
-; SSE2-NEXT: xorq $63, %r8
-; SSE2-NEXT: orq $64, %r8
-; SSE2-NEXT: testq %rsi, %rsi
-; SSE2-NEXT: cmovneq %r10, %r8
-; SSE2-NEXT: bsrq %rcx, %rsi
-; SSE2-NEXT: xorq $63, %rsi
; SSE2-NEXT: bsrq %rdx, %rdx
; SSE2-NEXT: xorq $63, %rdx
; SSE2-NEXT: orq $64, %rdx
-; SSE2-NEXT: testq %rcx, %rcx
-; SSE2-NEXT: cmovneq %rsi, %rdx
+; SSE2-NEXT: testq %rbx, %rbx
+; SSE2-NEXT: cmovneq %r14, %rdx
; SSE2-NEXT: orq $128, %rdx
-; SSE2-NEXT: pcmpeqd %xmm4, %xmm1
-; SSE2-NEXT: movmskps %xmm1, %ecx
-; SSE2-NEXT: xorl $15, %ecx
-; SSE2-NEXT: cmovneq %r8, %rdx
+; SSE2-NEXT: orq %r10, %r9
+; SSE2-NEXT: cmovneq %r15, %rdx
; SSE2-NEXT: orq $256, %rdx # imm = 0x100
-; SSE2-NEXT: pcmpeqd %xmm4, %xmm2
-; SSE2-NEXT: movmskps %xmm2, %ecx
-; SSE2-NEXT: xorl $15, %ecx
-; SSE2-NEXT: cmovneq %r9, %rdx
+; SSE2-NEXT: orq %rsi, %rcx
+; SSE2-NEXT: orq %r8, %rdi
+; SSE2-NEXT: orq %rcx, %rdi
+; SSE2-NEXT: cmovneq %r11, %rdx
; SSE2-NEXT: movdqa %xmm4, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: movdqa %xmm4, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: movdqa %xmm4, -{{[0-9]+}}(%rsp)
@@ -4575,50 +4684,48 @@ define i512 @isolate_msb_i512_vector(<8 x i64> %v0, i512 %idx) nounwind {
; SSE2-NEXT: andl $56, %edx
; SSE2-NEXT: movq $0, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: movq -72(%rsp,%rdx), %rsi
-; SSE2-NEXT: movq -80(%rsp,%rdx), %r10
+; SSE2-NEXT: movq -80(%rsp,%rdx), %r9
+; SSE2-NEXT: movq %r9, %rdi
+; SSE2-NEXT: shrdq %cl, %rsi, %rdi
+; SSE2-NEXT: movq -88(%rsp,%rdx), %r10
; SSE2-NEXT: movq %r10, %r8
-; SSE2-NEXT: shrdq %cl, %rsi, %r8
-; SSE2-NEXT: movq -88(%rsp,%rdx), %r11
+; SSE2-NEXT: shrdq %cl, %r9, %r8
+; SSE2-NEXT: movq -96(%rsp,%rdx), %r11
; SSE2-NEXT: movq %r11, %r9
; SSE2-NEXT: shrdq %cl, %r10, %r9
-; SSE2-NEXT: movq -96(%rsp,%rdx), %rbx
+; SSE2-NEXT: movq -104(%rsp,%rdx), %rbx
; SSE2-NEXT: movq %rbx, %r10
; SSE2-NEXT: shrdq %cl, %r11, %r10
-; SSE2-NEXT: movq -104(%rsp,%rdx), %r14
+; SSE2-NEXT: movq -112(%rsp,%rdx), %r14
; SSE2-NEXT: movq %r14, %r11
; SSE2-NEXT: shrdq %cl, %rbx, %r11
-; SSE2-NEXT: movq -112(%rsp,%rdx), %r15
+; SSE2-NEXT: movq -120(%rsp,%rdx), %r15
; SSE2-NEXT: movq %r15, %rbx
; SSE2-NEXT: shrdq %cl, %r14, %rbx
-; SSE2-NEXT: movq -120(%rsp,%rdx), %r12
-; SSE2-NEXT: movq %r12, %r14
-; SSE2-NEXT: shrdq %cl, %r15, %r14
; SSE2-NEXT: movq -128(%rsp,%rdx), %rdx
; SSE2-NEXT: shrq %cl, %rsi
; SSE2-NEXT: # kill: def $cl killed $cl killed $ecx
-; SSE2-NEXT: shrdq %cl, %r12, %rdx
-; SSE2-NEXT: xorl %ecx, %ecx
-; SSE2-NEXT: testl %eax, %eax
-; SSE2-NEXT: cmoveq %rcx, %r14
-; SSE2-NEXT: cmoveq %rcx, %rbx
-; SSE2-NEXT: cmoveq %rcx, %r11
-; SSE2-NEXT: cmoveq %rcx, %r10
-; SSE2-NEXT: cmoveq %rcx, %r9
-; SSE2-NEXT: cmoveq %rcx, %r8
-; SSE2-NEXT: cmoveq %rcx, %rdx
-; SSE2-NEXT: movq %rdi, %rax
-; SSE2-NEXT: cmoveq %rcx, %rsi
-; SSE2-NEXT: movq %rsi, 56(%rdi)
-; SSE2-NEXT: movq %r8, 48(%rdi)
-; SSE2-NEXT: movq %r9, 40(%rdi)
-; SSE2-NEXT: movq %r10, 32(%rdi)
-; SSE2-NEXT: movq %r11, 24(%rdi)
-; SSE2-NEXT: movq %rbx, 16(%rdi)
-; SSE2-NEXT: movq %r14, 8(%rdi)
-; SSE2-NEXT: movq %rdx, (%rdi)
-; SSE2-NEXT: addq $8, %rsp
+; SSE2-NEXT: shrdq %cl, %r15, %rdx
+; SSE2-NEXT: jmp .LBB16_3
+; SSE2-NEXT: .LBB16_1:
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: xorl %ebx, %ebx
+; SSE2-NEXT: xorl %r11d, %r11d
+; SSE2-NEXT: xorl %r10d, %r10d
+; SSE2-NEXT: xorl %r9d, %r9d
+; SSE2-NEXT: xorl %r8d, %r8d
+; SSE2-NEXT: xorl %edi, %edi
+; SSE2-NEXT: xorl %esi, %esi
+; SSE2-NEXT: .LBB16_3: # %select.end
+; SSE2-NEXT: movq %rdx, (%rax)
+; SSE2-NEXT: movq %rbx, 8(%rax)
+; SSE2-NEXT: movq %r11, 16(%rax)
+; SSE2-NEXT: movq %r10, 24(%rax)
+; SSE2-NEXT: movq %r9, 32(%rax)
+; SSE2-NEXT: movq %r8, 40(%rax)
+; SSE2-NEXT: movq %rdi, 48(%rax)
+; SSE2-NEXT: movq %rsi, 56(%rax)
; SSE2-NEXT: popq %rbx
-; SSE2-NEXT: popq %r12
; SSE2-NEXT: popq %r14
; SSE2-NEXT: popq %r15
; SSE2-NEXT: retq
@@ -4628,112 +4735,120 @@ define i512 @isolate_msb_i512_vector(<8 x i64> %v0, i512 %idx) nounwind {
; SSE42-NEXT: pushq %r15
; SSE42-NEXT: pushq %r14
; SSE42-NEXT: pushq %rbx
-; SSE42-NEXT: movq %xmm0, %rax
-; SSE42-NEXT: pextrq $1, %xmm0, %rcx
-; SSE42-NEXT: pextrq $1, %xmm1, %rdx
-; SSE42-NEXT: movq %xmm1, %r8
-; SSE42-NEXT: movq %xmm2, %rsi
-; SSE42-NEXT: pextrq $1, %xmm2, %r9
-; SSE42-NEXT: movq %xmm3, %r10
-; SSE42-NEXT: pextrq $1, %xmm3, %r11
-; SSE42-NEXT: bsrq %r11, %rbx
-; SSE42-NEXT: xorq $63, %rbx
-; SSE42-NEXT: bsrq %r10, %r10
-; SSE42-NEXT: xorq $63, %r10
-; SSE42-NEXT: orq $64, %r10
-; SSE42-NEXT: testq %r11, %r11
-; SSE42-NEXT: cmovneq %rbx, %r10
-; SSE42-NEXT: bsrq %r9, %r11
-; SSE42-NEXT: xorq $63, %r11
-; SSE42-NEXT: bsrq %rsi, %rsi
-; SSE42-NEXT: xorq $63, %rsi
-; SSE42-NEXT: orq $64, %rsi
-; SSE42-NEXT: testq %r9, %r9
-; SSE42-NEXT: cmovneq %r11, %rsi
-; SSE42-NEXT: orq $128, %rsi
-; SSE42-NEXT: ptest %xmm3, %xmm3
-; SSE42-NEXT: cmovneq %r10, %rsi
-; SSE42-NEXT: bsrq %rdx, %r9
-; SSE42-NEXT: xorq $63, %r9
-; SSE42-NEXT: bsrq %r8, %r8
-; SSE42-NEXT: xorq $63, %r8
-; SSE42-NEXT: orq $64, %r8
-; SSE42-NEXT: testq %rdx, %rdx
-; SSE42-NEXT: cmovneq %r9, %r8
+; SSE42-NEXT: movq %rdi, %rax
+; SSE42-NEXT: movdqa %xmm1, %xmm4
+; SSE42-NEXT: por %xmm3, %xmm4
+; SSE42-NEXT: movdqa %xmm0, %xmm5
+; SSE42-NEXT: por %xmm2, %xmm5
+; SSE42-NEXT: por %xmm4, %xmm5
+; SSE42-NEXT: ptest %xmm5, %xmm5
+; SSE42-NEXT: je .LBB16_1
+; SSE42-NEXT: # %bb.2: # %select.false.sink
+; SSE42-NEXT: pextrq $1, %xmm3, %rsi
+; SSE42-NEXT: movq %xmm3, %rdi
+; SSE42-NEXT: pextrq $1, %xmm2, %rcx
+; SSE42-NEXT: pextrq $1, %xmm1, %r9
+; SSE42-NEXT: movq %xmm2, %r8
+; SSE42-NEXT: movq %xmm1, %r10
+; SSE42-NEXT: pextrq $1, %xmm0, %rbx
+; SSE42-NEXT: bsrq %rsi, %rdx
+; SSE42-NEXT: xorq $63, %rdx
+; SSE42-NEXT: bsrq %rdi, %r14
+; SSE42-NEXT: xorq $63, %r14
+; SSE42-NEXT: orq $64, %r14
+; SSE42-NEXT: testq %rsi, %rsi
+; SSE42-NEXT: cmovneq %rdx, %r14
; SSE42-NEXT: bsrq %rcx, %rdx
; SSE42-NEXT: xorq $63, %rdx
-; SSE42-NEXT: bsrq %rax, %rax
-; SSE42-NEXT: xorq $63, %rax
-; SSE42-NEXT: orq $64, %rax
+; SSE42-NEXT: bsrq %r8, %r11
+; SSE42-NEXT: xorq $63, %r11
+; SSE42-NEXT: orq $64, %r11
; SSE42-NEXT: testq %rcx, %rcx
-; SSE42-NEXT: cmovneq %rdx, %rax
-; SSE42-NEXT: por %xmm2, %xmm0
-; SSE42-NEXT: orq $128, %rax
-; SSE42-NEXT: ptest %xmm1, %xmm1
-; SSE42-NEXT: cmovneq %r8, %rax
-; SSE42-NEXT: orq $256, %rax # imm = 0x100
-; SSE42-NEXT: por %xmm3, %xmm2
-; SSE42-NEXT: ptest %xmm2, %xmm2
-; SSE42-NEXT: cmovneq %rsi, %rax
-; SSE42-NEXT: pxor %xmm2, %xmm2
-; SSE42-NEXT: movdqa %xmm2, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movdqa %xmm2, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movdqa %xmm2, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movdqa %xmm2, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: cmovneq %rdx, %r11
+; SSE42-NEXT: movq %xmm0, %rdx
+; SSE42-NEXT: orq $128, %r11
+; SSE42-NEXT: movq %rdi, %r15
+; SSE42-NEXT: orq %rsi, %r15
+; SSE42-NEXT: cmovneq %r14, %r11
+; SSE42-NEXT: bsrq %r9, %r14
+; SSE42-NEXT: xorq $63, %r14
+; SSE42-NEXT: bsrq %r10, %r15
+; SSE42-NEXT: xorq $63, %r15
+; SSE42-NEXT: orq $64, %r15
+; SSE42-NEXT: testq %r9, %r9
+; SSE42-NEXT: cmovneq %r14, %r15
+; SSE42-NEXT: bsrq %rbx, %r14
+; SSE42-NEXT: xorq $63, %r14
+; SSE42-NEXT: bsrq %rdx, %rdx
+; SSE42-NEXT: xorq $63, %rdx
+; SSE42-NEXT: orq $64, %rdx
+; SSE42-NEXT: testq %rbx, %rbx
+; SSE42-NEXT: cmovneq %r14, %rdx
+; SSE42-NEXT: orq $128, %rdx
+; SSE42-NEXT: orq %r9, %r10
+; SSE42-NEXT: cmovneq %r15, %rdx
+; SSE42-NEXT: orq $256, %rdx # imm = 0x100
+; SSE42-NEXT: orq %rdi, %r8
+; SSE42-NEXT: orq %rsi, %rcx
+; SSE42-NEXT: orq %r8, %rcx
+; SSE42-NEXT: cmovneq %r11, %rdx
+; SSE42-NEXT: pxor %xmm0, %xmm0
+; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
; SSE42-NEXT: movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000
; SSE42-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movdqa %xmm2, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movdqa %xmm2, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movdqa %xmm2, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movl %eax, %ecx
+; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movl %edx, %ecx
; SSE42-NEXT: andl $63, %ecx
-; SSE42-NEXT: shrl $3, %eax
-; SSE42-NEXT: andl $56, %eax
+; SSE42-NEXT: shrl $3, %edx
+; SSE42-NEXT: andl $56, %edx
; SSE42-NEXT: movq $0, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movq -72(%rsp,%rax), %rdx
-; SSE42-NEXT: movq -80(%rsp,%rax), %r9
-; SSE42-NEXT: movq %r9, %rsi
-; SSE42-NEXT: shrdq %cl, %rdx, %rsi
-; SSE42-NEXT: movq -88(%rsp,%rax), %r10
+; SSE42-NEXT: movq -72(%rsp,%rdx), %rsi
+; SSE42-NEXT: movq -80(%rsp,%rdx), %r9
+; SSE42-NEXT: movq %r9, %rdi
+; SSE42-NEXT: shrdq %cl, %rsi, %rdi
+; SSE42-NEXT: movq -88(%rsp,%rdx), %r10
; SSE42-NEXT: movq %r10, %r8
; SSE42-NEXT: shrdq %cl, %r9, %r8
-; SSE42-NEXT: movq -96(%rsp,%rax), %r11
+; SSE42-NEXT: movq -96(%rsp,%rdx), %r11
; SSE42-NEXT: movq %r11, %r9
; SSE42-NEXT: shrdq %cl, %r10, %r9
-; SSE42-NEXT: movq -104(%rsp,%rax), %rbx
+; SSE42-NEXT: movq -104(%rsp,%rdx), %rbx
; SSE42-NEXT: movq %rbx, %r10
; SSE42-NEXT: shrdq %cl, %r11, %r10
-; SSE42-NEXT: movq -112(%rsp,%rax), %r14
+; SSE42-NEXT: movq -112(%rsp,%rdx), %r14
; SSE42-NEXT: movq %r14, %r11
; SSE42-NEXT: shrdq %cl, %rbx, %r11
-; SSE42-NEXT: movq -120(%rsp,%rax), %r15
+; SSE42-NEXT: movq -120(%rsp,%rdx), %r15
; SSE42-NEXT: movq %r15, %rbx
; SSE42-NEXT: shrdq %cl, %r14, %rbx
-; SSE42-NEXT: movq -128(%rsp,%rax), %r14
-; SSE42-NEXT: shrq %cl, %rdx
+; SSE42-NEXT: movq -128(%rsp,%rdx), %rdx
+; SSE42-NEXT: shrq %cl, %rsi
; SSE42-NEXT: # kill: def $cl killed $cl killed $ecx
-; SSE42-NEXT: shrdq %cl, %r15, %r14
-; SSE42-NEXT: por %xmm3, %xmm1
-; SSE42-NEXT: por %xmm1, %xmm0
-; SSE42-NEXT: xorl %ecx, %ecx
-; SSE42-NEXT: ptest %xmm0, %xmm0
-; SSE42-NEXT: cmoveq %rcx, %rbx
-; SSE42-NEXT: cmoveq %rcx, %r11
-; SSE42-NEXT: cmoveq %rcx, %r10
-; SSE42-NEXT: cmoveq %rcx, %r9
-; SSE42-NEXT: cmoveq %rcx, %r8
-; SSE42-NEXT: cmoveq %rcx, %rsi
-; SSE42-NEXT: cmoveq %rcx, %r14
-; SSE42-NEXT: movq %rdi, %rax
-; SSE42-NEXT: cmoveq %rcx, %rdx
-; SSE42-NEXT: movq %rdx, 56(%rdi)
-; SSE42-NEXT: movq %rsi, 48(%rdi)
-; SSE42-NEXT: movq %r8, 40(%rdi)
-; SSE42-NEXT: movq %r9, 32(%rdi)
-; SSE42-NEXT: movq %r10, 24(%rdi)
-; SSE42-NEXT: movq %r11, 16(%rdi)
-; SSE42-NEXT: movq %rbx, 8(%rdi)
-; SSE42-NEXT: movq %r14, (%rdi)
+; SSE42-NEXT: shrdq %cl, %r15, %rdx
+; SSE42-NEXT: jmp .LBB16_3
+; SSE42-NEXT: .LBB16_1:
+; SSE42-NEXT: xorl %edx, %edx
+; SSE42-NEXT: xorl %ebx, %ebx
+; SSE42-NEXT: xorl %r11d, %r11d
+; SSE42-NEXT: xorl %r10d, %r10d
+; SSE42-NEXT: xorl %r9d, %r9d
+; SSE42-NEXT: xorl %r8d, %r8d
+; SSE42-NEXT: xorl %edi, %edi
+; SSE42-NEXT: xorl %esi, %esi
+; SSE42-NEXT: .LBB16_3: # %select.end
+; SSE42-NEXT: movq %rdx, (%rax)
+; SSE42-NEXT: movq %rbx, 8(%rax)
+; SSE42-NEXT: movq %r11, 16(%rax)
+; SSE42-NEXT: movq %r10, 24(%rax)
+; SSE42-NEXT: movq %r9, 32(%rax)
+; SSE42-NEXT: movq %r8, 40(%rax)
+; SSE42-NEXT: movq %rdi, 48(%rax)
+; SSE42-NEXT: movq %rsi, 56(%rax)
; SSE42-NEXT: popq %rbx
; SSE42-NEXT: popq %r14
; SSE42-NEXT: popq %r15
@@ -4744,103 +4859,111 @@ define i512 @isolate_msb_i512_vector(<8 x i64> %v0, i512 %idx) nounwind {
; AVX2-NEXT: pushq %r15
; AVX2-NEXT: pushq %r14
; AVX2-NEXT: pushq %rbx
-; AVX2-NEXT: vpextrq $1, %xmm0, %rsi
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2
-; AVX2-NEXT: vmovq %xmm2, %rax
-; AVX2-NEXT: vpextrq $1, %xmm2, %rcx
-; AVX2-NEXT: vmovq %xmm1, %rdx
-; AVX2-NEXT: vpextrq $1, %xmm1, %r9
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: vpor %ymm1, %ymm0, %ymm2
+; AVX2-NEXT: vptest %ymm2, %ymm2
+; AVX2-NEXT: je .LBB16_1
+; AVX2-NEXT: # %bb.2: # %select.false.sink
; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
-; AVX2-NEXT: vmovq %xmm2, %r10
-; AVX2-NEXT: vpextrq $1, %xmm2, %r11
-; AVX2-NEXT: lzcntq %r11, %r8
-; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: lzcntq %r10, %rbx
-; AVX2-NEXT: addq $64, %rbx
-; AVX2-NEXT: testq %r11, %r11
-; AVX2-NEXT: cmovneq %r8, %rbx
+; AVX2-NEXT: vpextrq $1, %xmm2, %rcx
+; AVX2-NEXT: vmovq %xmm2, %r8
+; AVX2-NEXT: vpextrq $1, %xmm1, %rsi
+; AVX2-NEXT: vmovq %xmm1, %rdi
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpextrq $1, %xmm1, %r9
+; AVX2-NEXT: vmovq %xmm1, %r10
+; AVX2-NEXT: vpextrq $1, %xmm0, %rbx
+; AVX2-NEXT: vmovq %xmm0, %rdx
+; AVX2-NEXT: lzcntq %rcx, %r11
+; AVX2-NEXT: xorl %r14d, %r14d
+; AVX2-NEXT: lzcntq %r8, %r14
+; AVX2-NEXT: addq $64, %r14
+; AVX2-NEXT: testq %rcx, %rcx
+; AVX2-NEXT: cmovneq %r11, %r14
+; AVX2-NEXT: xorl %r15d, %r15d
+; AVX2-NEXT: lzcntq %rsi, %r15
+; AVX2-NEXT: xorl %r11d, %r11d
+; AVX2-NEXT: lzcntq %rdi, %r11
+; AVX2-NEXT: addq $64, %r11
+; AVX2-NEXT: testq %rsi, %rsi
+; AVX2-NEXT: cmovneq %r15, %r11
+; AVX2-NEXT: subq $-128, %r11
+; AVX2-NEXT: movq %r8, %r15
+; AVX2-NEXT: orq %rcx, %r15
+; AVX2-NEXT: cmovneq %r14, %r11
; AVX2-NEXT: xorl %r14d, %r14d
; AVX2-NEXT: lzcntq %r9, %r14
-; AVX2-NEXT: xorl %r8d, %r8d
-; AVX2-NEXT: lzcntq %rdx, %r8
-; AVX2-NEXT: addq $64, %r8
+; AVX2-NEXT: xorl %r15d, %r15d
+; AVX2-NEXT: lzcntq %r10, %r15
+; AVX2-NEXT: addq $64, %r15
; AVX2-NEXT: testq %r9, %r9
-; AVX2-NEXT: cmovneq %r14, %r8
-; AVX2-NEXT: subq $-128, %r8
-; AVX2-NEXT: orq %r11, %r10
-; AVX2-NEXT: cmovneq %rbx, %r8
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: lzcntq %rcx, %rdx
-; AVX2-NEXT: xorl %r9d, %r9d
-; AVX2-NEXT: lzcntq %rax, %r9
-; AVX2-NEXT: addq $64, %r9
-; AVX2-NEXT: testq %rcx, %rcx
-; AVX2-NEXT: cmovneq %rdx, %r9
-; AVX2-NEXT: vmovq %xmm0, %rdx
-; AVX2-NEXT: xorl %r10d, %r10d
-; AVX2-NEXT: lzcntq %rsi, %r10
+; AVX2-NEXT: cmovneq %r14, %r15
+; AVX2-NEXT: xorl %r14d, %r14d
+; AVX2-NEXT: lzcntq %rbx, %r14
; AVX2-NEXT: lzcntq %rdx, %rdx
; AVX2-NEXT: addq $64, %rdx
-; AVX2-NEXT: testq %rsi, %rsi
-; AVX2-NEXT: cmovneq %r10, %rdx
+; AVX2-NEXT: testq %rbx, %rbx
+; AVX2-NEXT: cmovneq %r14, %rdx
; AVX2-NEXT: subq $-128, %rdx
-; AVX2-NEXT: orq %rcx, %rax
-; AVX2-NEXT: cmovneq %r9, %rdx
+; AVX2-NEXT: orq %r9, %r10
+; AVX2-NEXT: cmovneq %r15, %rdx
; AVX2-NEXT: addq $256, %rdx # imm = 0x100
-; AVX2-NEXT: vptest %ymm1, %ymm1
-; AVX2-NEXT: cmovneq %r8, %rdx
-; AVX2-NEXT: vmovaps {{.*#+}} ymm2 = [0,0,0,9223372036854775808]
-; AVX2-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vxorps %xmm2, %xmm2, %xmm2
-; AVX2-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: orq %r8, %rdi
+; AVX2-NEXT: orq %rcx, %rsi
+; AVX2-NEXT: orq %rdi, %rsi
+; AVX2-NEXT: cmovneq %r11, %rdx
+; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovdqa {{.*#+}} ymm1 = [0,0,0,9223372036854775808]
+; AVX2-NEXT: vmovdqu %ymm1, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movl %edx, %ecx
; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: shrl $3, %edx
; AVX2-NEXT: andl $56, %edx
-; AVX2-NEXT: movq -72(%rsp,%rdx), %r11
-; AVX2-NEXT: movq -80(%rsp,%rdx), %rax
-; AVX2-NEXT: movq %rax, %rsi
-; AVX2-NEXT: shrdq %cl, %r11, %rsi
-; AVX2-NEXT: movq -88(%rsp,%rdx), %r10
+; AVX2-NEXT: movq -72(%rsp,%rdx), %rbx
+; AVX2-NEXT: movq -80(%rsp,%rdx), %r8
+; AVX2-NEXT: movq %r8, %rsi
+; AVX2-NEXT: shrdq %cl, %rbx, %rsi
+; AVX2-NEXT: movq -88(%rsp,%rdx), %r9
+; AVX2-NEXT: movq %r9, %rdi
+; AVX2-NEXT: shrdq %cl, %r8, %rdi
+; AVX2-NEXT: movq -96(%rsp,%rdx), %r10
; AVX2-NEXT: movq %r10, %r8
-; AVX2-NEXT: shrdq %cl, %rax, %r8
-; AVX2-NEXT: movq -96(%rsp,%rdx), %rax
-; AVX2-NEXT: movq %rax, %r9
+; AVX2-NEXT: shrdq %cl, %r9, %r8
+; AVX2-NEXT: movq -104(%rsp,%rdx), %r11
+; AVX2-NEXT: movq %r11, %r9
; AVX2-NEXT: shrdq %cl, %r10, %r9
-; AVX2-NEXT: movq -104(%rsp,%rdx), %r14
+; AVX2-NEXT: movq -112(%rsp,%rdx), %r14
; AVX2-NEXT: movq %r14, %r10
-; AVX2-NEXT: shrdq %cl, %rax, %r10
-; AVX2-NEXT: movq -112(%rsp,%rdx), %r15
-; AVX2-NEXT: movq %r15, %rbx
-; AVX2-NEXT: shrdq %cl, %r14, %rbx
-; AVX2-NEXT: movq %rdi, %rax
-; AVX2-NEXT: vpor %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: movq -128(%rsp,%rdx), %rdi
-; AVX2-NEXT: movq -120(%rsp,%rdx), %r14
-; AVX2-NEXT: movq %r14, %rdx
+; AVX2-NEXT: shrdq %cl, %r11, %r10
+; AVX2-NEXT: movq -120(%rsp,%rdx), %r15
+; AVX2-NEXT: movq %r15, %r11
+; AVX2-NEXT: shrdq %cl, %r14, %r11
+; AVX2-NEXT: movq -128(%rsp,%rdx), %rdx
+; AVX2-NEXT: shrxq %rcx, %rbx, %rbx
+; AVX2-NEXT: # kill: def $cl killed $cl killed $rcx
; AVX2-NEXT: shrdq %cl, %r15, %rdx
-; AVX2-NEXT: shrdq %cl, %r14, %rdi
-; AVX2-NEXT: xorl %r14d, %r14d
-; AVX2-NEXT: vptest %ymm0, %ymm0
-; AVX2-NEXT: shrxq %rcx, %r11, %rcx
-; AVX2-NEXT: cmoveq %r14, %rdx
-; AVX2-NEXT: cmoveq %r14, %rbx
-; AVX2-NEXT: cmoveq %r14, %r10
-; AVX2-NEXT: cmoveq %r14, %r9
-; AVX2-NEXT: cmoveq %r14, %r8
-; AVX2-NEXT: cmoveq %r14, %rsi
-; AVX2-NEXT: cmoveq %r14, %rdi
-; AVX2-NEXT: cmoveq %r14, %rcx
-; AVX2-NEXT: movq %rcx, 56(%rax)
+; AVX2-NEXT: jmp .LBB16_3
+; AVX2-NEXT: .LBB16_1:
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: xorl %r11d, %r11d
+; AVX2-NEXT: xorl %r10d, %r10d
+; AVX2-NEXT: xorl %r9d, %r9d
+; AVX2-NEXT: xorl %r8d, %r8d
+; AVX2-NEXT: xorl %edi, %edi
+; AVX2-NEXT: xorl %esi, %esi
+; AVX2-NEXT: xorl %ebx, %ebx
+; AVX2-NEXT: .LBB16_3: # %select.end
+; AVX2-NEXT: movq %rdx, (%rax)
+; AVX2-NEXT: movq %r11, 8(%rax)
+; AVX2-NEXT: movq %r10, 16(%rax)
+; AVX2-NEXT: movq %r9, 24(%rax)
+; AVX2-NEXT: movq %r8, 32(%rax)
+; AVX2-NEXT: movq %rdi, 40(%rax)
; AVX2-NEXT: movq %rsi, 48(%rax)
-; AVX2-NEXT: movq %r8, 40(%rax)
-; AVX2-NEXT: movq %r9, 32(%rax)
-; AVX2-NEXT: movq %r10, 24(%rax)
-; AVX2-NEXT: movq %rbx, 16(%rax)
-; AVX2-NEXT: movq %rdx, 8(%rax)
-; AVX2-NEXT: movq %rdi, (%rax)
+; AVX2-NEXT: movq %rbx, 56(%rax)
; AVX2-NEXT: popq %rbx
; AVX2-NEXT: popq %r14
; AVX2-NEXT: popq %r15
@@ -4851,56 +4974,154 @@ define i512 @isolate_msb_i512_vector(<8 x i64> %v0, i512 %idx) nounwind {
; AVX512F: # %bb.0:
; AVX512F-NEXT: movq %rdi, %rax
; AVX512F-NEXT: vptestmd %zmm0, %zmm0, %k0
-; AVX512F-NEXT: xorl %ecx, %ecx
; AVX512F-NEXT: kortestw %k0, %k0
-; AVX512F-NEXT: sete %cl
-; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
-; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0
+; AVX512F-NEXT: je .LBB16_1
+; AVX512F-NEXT: # %bb.2: # %select.false.sink
+; AVX512F-NEXT: vextracti32x4 $3, %zmm0, %xmm1
+; AVX512F-NEXT: vpextrq $1, %xmm1, %rcx
+; AVX512F-NEXT: vmovq %xmm1, %rdx
+; AVX512F-NEXT: vextracti32x4 $2, %zmm0, %xmm1
+; AVX512F-NEXT: vpextrq $1, %xmm1, %rsi
+; AVX512F-NEXT: vpextrq $1, %xmm0, %rdi
+; AVX512F-NEXT: vmovq %xmm1, %r8
+; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT: vpextrq $1, %xmm1, %r9
+; AVX512F-NEXT: vmovq %xmm0, %r10
+; AVX512F-NEXT: vmovq %xmm1, %r11
+; AVX512F-NEXT: vmovq %r10, %xmm0
+; AVX512F-NEXT: vmovq %rdi, %xmm1
+; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512F-NEXT: vmovq %r11, %xmm1
+; AVX512F-NEXT: vmovq %r9, %xmm2
+; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX512F-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512F-NEXT: vmovq %r8, %xmm1
+; AVX512F-NEXT: vmovq %rsi, %xmm2
+; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX512F-NEXT: vmovq %rdx, %xmm2
+; AVX512F-NEXT: vmovq %rcx, %xmm3
+; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
+; AVX512F-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
+; AVX512F-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512F-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1 # [0,64,128,192,256,320,384,448]
; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
-; AVX512F-NEXT: vplzcntq %zmm0, %zmm0
-; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,64,128,192,256,320,384,448]
-; AVX512F-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
-; AVX512F-NEXT: vmovq %xmm0, %rdx
-; AVX512F-NEXT: movabsq $-9223372036854775808, %rsi # imm = 0x8000000000000000
-; AVX512F-NEXT: shrxq %rdx, %rsi, %rsi
-; AVX512F-NEXT: shrl $6, %edx
-; AVX512F-NEXT: movl $128, %edi
-; AVX512F-NEXT: shrxq %rdx, %rdi, %rdx
-; AVX512F-NEXT: kmovw %edx, %k1
-; AVX512F-NEXT: vpbroadcastq %rsi, %zmm0 {%k1} {z}
-; AVX512F-NEXT: negl %ecx
-; AVX512F-NEXT: kmovw %ecx, %k0
-; AVX512F-NEXT: knotw %k0, %k1
-; AVX512F-NEXT: vmovdqa32 %zmm0, %zmm0 {%k1} {z}
-; AVX512F-NEXT: vmovdqu64 %zmm0, (%rax)
+; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; AVX512F-NEXT: vmovq %xmm0, %rcx
+; AVX512F-NEXT: movabsq $-9223372036854775808, %rdx # imm = 0x8000000000000000
+; AVX512F-NEXT: shrxq %rcx, %rdx, %rdx
+; AVX512F-NEXT: shrl $6, %ecx
+; AVX512F-NEXT: movl $128, %esi
+; AVX512F-NEXT: shrxq %rcx, %rsi, %rcx
+; AVX512F-NEXT: kmovw %ecx, %k1
+; AVX512F-NEXT: vpbroadcastq %rdx, %zmm0 {%k1} {z}
+; AVX512F-NEXT: vextracti32x4 $3, %zmm0, %xmm1
+; AVX512F-NEXT: vpextrq $1, %xmm1, %rcx
+; AVX512F-NEXT: vmovq %xmm1, %rdx
+; AVX512F-NEXT: vextracti32x4 $2, %zmm0, %xmm1
+; AVX512F-NEXT: vpextrq $1, %xmm1, %rsi
+; AVX512F-NEXT: vmovq %xmm1, %r8
+; AVX512F-NEXT: vpextrq $1, %xmm0, %r9
+; AVX512F-NEXT: vmovq %xmm0, %rdi
+; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX512F-NEXT: vpextrq $1, %xmm0, %r10
+; AVX512F-NEXT: vmovq %xmm0, %r11
+; AVX512F-NEXT: jmp .LBB16_3
+; AVX512F-NEXT: .LBB16_1:
+; AVX512F-NEXT: xorl %edi, %edi
+; AVX512F-NEXT: xorl %r9d, %r9d
+; AVX512F-NEXT: xorl %r11d, %r11d
+; AVX512F-NEXT: xorl %r10d, %r10d
+; AVX512F-NEXT: xorl %r8d, %r8d
+; AVX512F-NEXT: xorl %esi, %esi
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: xorl %ecx, %ecx
+; AVX512F-NEXT: .LBB16_3: # %select.end
+; AVX512F-NEXT: movq %rdi, (%rax)
+; AVX512F-NEXT: movq %r9, 8(%rax)
+; AVX512F-NEXT: movq %r11, 16(%rax)
+; AVX512F-NEXT: movq %r10, 24(%rax)
+; AVX512F-NEXT: movq %r8, 32(%rax)
+; AVX512F-NEXT: movq %rsi, 40(%rax)
+; AVX512F-NEXT: movq %rdx, 48(%rax)
+; AVX512F-NEXT: movq %rcx, 56(%rax)
; AVX512F-NEXT: retq
;
; AVX512VL-LABEL: isolate_msb_i512_vector:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: movq %rdi, %rax
; AVX512VL-NEXT: vptestmd %zmm0, %zmm0, %k0
-; AVX512VL-NEXT: xorl %ecx, %ecx
; AVX512VL-NEXT: kortestw %k0, %k0
-; AVX512VL-NEXT: sete %cl
-; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
-; AVX512VL-NEXT: vpermq %zmm0, %zmm1, %zmm0
+; AVX512VL-NEXT: je .LBB16_1
+; AVX512VL-NEXT: # %bb.2: # %select.false.sink
+; AVX512VL-NEXT: vextracti32x4 $3, %zmm0, %xmm1
+; AVX512VL-NEXT: vpextrq $1, %xmm1, %rcx
+; AVX512VL-NEXT: vmovq %xmm1, %rdx
+; AVX512VL-NEXT: vextracti32x4 $2, %zmm0, %xmm1
+; AVX512VL-NEXT: vpextrq $1, %xmm1, %rsi
+; AVX512VL-NEXT: vmovq %xmm1, %rdi
+; AVX512VL-NEXT: vpextrq $1, %xmm0, %r8
+; AVX512VL-NEXT: vmovq %xmm0, %r9
+; AVX512VL-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX512VL-NEXT: vpextrq $1, %xmm0, %r10
+; AVX512VL-NEXT: vmovq %xmm0, %r11
+; AVX512VL-NEXT: vmovq %r9, %xmm0
+; AVX512VL-NEXT: vmovq %r8, %xmm1
+; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512VL-NEXT: vmovq %r11, %xmm1
+; AVX512VL-NEXT: vmovq %r10, %xmm2
+; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX512VL-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512VL-NEXT: vmovq %rdi, %xmm1
+; AVX512VL-NEXT: vmovq %rsi, %xmm2
+; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX512VL-NEXT: vmovq %rdx, %xmm2
+; AVX512VL-NEXT: vmovq %rcx, %xmm3
+; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
+; AVX512VL-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
+; AVX512VL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VL-NEXT: vplzcntq %zmm0, %zmm0
; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,64,128,192,256,320,384,448]
; AVX512VL-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
-; AVX512VL-NEXT: vmovq %xmm0, %rdx
-; AVX512VL-NEXT: movabsq $-9223372036854775808, %rsi # imm = 0x8000000000000000
-; AVX512VL-NEXT: shrxq %rdx, %rsi, %rsi
-; AVX512VL-NEXT: shrl $6, %edx
-; AVX512VL-NEXT: movl $128, %edi
-; AVX512VL-NEXT: shrxq %rdx, %rdi, %rdx
-; AVX512VL-NEXT: kmovd %edx, %k1
-; AVX512VL-NEXT: vpbroadcastq %rsi, %zmm0 {%k1} {z}
-; AVX512VL-NEXT: negl %ecx
-; AVX512VL-NEXT: kmovd %ecx, %k0
-; AVX512VL-NEXT: knotw %k0, %k1
-; AVX512VL-NEXT: vmovdqa32 %zmm0, %zmm0 {%k1} {z}
-; AVX512VL-NEXT: vmovdqu64 %zmm0, (%rax)
+; AVX512VL-NEXT: vmovq %xmm0, %rcx
+; AVX512VL-NEXT: movabsq $-9223372036854775808, %rdx # imm = 0x8000000000000000
+; AVX512VL-NEXT: shrxq %rcx, %rdx, %rdx
+; AVX512VL-NEXT: shrl $6, %ecx
+; AVX512VL-NEXT: movl $128, %esi
+; AVX512VL-NEXT: shrxq %rcx, %rsi, %rcx
+; AVX512VL-NEXT: kmovd %ecx, %k1
+; AVX512VL-NEXT: vpbroadcastq %rdx, %zmm0 {%k1} {z}
+; AVX512VL-NEXT: vextracti32x4 $3, %zmm0, %xmm1
+; AVX512VL-NEXT: vpextrq $1, %xmm1, %rcx
+; AVX512VL-NEXT: vmovq %xmm1, %rdx
+; AVX512VL-NEXT: vextracti32x4 $2, %zmm0, %xmm1
+; AVX512VL-NEXT: vpextrq $1, %xmm1, %rsi
+; AVX512VL-NEXT: vpextrq $1, %xmm0, %r8
+; AVX512VL-NEXT: vmovq %xmm1, %r9
+; AVX512VL-NEXT: vmovq %xmm0, %rdi
+; AVX512VL-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX512VL-NEXT: vpextrq $1, %xmm0, %r10
+; AVX512VL-NEXT: vmovq %xmm0, %r11
+; AVX512VL-NEXT: jmp .LBB16_3
+; AVX512VL-NEXT: .LBB16_1:
+; AVX512VL-NEXT: xorl %edi, %edi
+; AVX512VL-NEXT: xorl %r8d, %r8d
+; AVX512VL-NEXT: xorl %r11d, %r11d
+; AVX512VL-NEXT: xorl %r10d, %r10d
+; AVX512VL-NEXT: xorl %r9d, %r9d
+; AVX512VL-NEXT: xorl %esi, %esi
+; AVX512VL-NEXT: xorl %edx, %edx
+; AVX512VL-NEXT: xorl %ecx, %ecx
+; AVX512VL-NEXT: .LBB16_3: # %select.end
+; AVX512VL-NEXT: movq %rdi, (%rax)
+; AVX512VL-NEXT: movq %r8, 8(%rax)
+; AVX512VL-NEXT: movq %r11, 16(%rax)
+; AVX512VL-NEXT: movq %r10, 24(%rax)
+; AVX512VL-NEXT: movq %r9, 32(%rax)
+; AVX512VL-NEXT: movq %rsi, 40(%rax)
+; AVX512VL-NEXT: movq %rdx, 48(%rax)
+; AVX512VL-NEXT: movq %rcx, 56(%rax)
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
@@ -4908,28 +5129,77 @@ define i512 @isolate_msb_i512_vector(<8 x i64> %v0, i512 %idx) nounwind {
; AVX512VBMI: # %bb.0:
; AVX512VBMI-NEXT: movq %rdi, %rax
; AVX512VBMI-NEXT: vptestmd %zmm0, %zmm0, %k0
-; AVX512VBMI-NEXT: xorl %ecx, %ecx
; AVX512VBMI-NEXT: kortestw %k0, %k0
-; AVX512VBMI-NEXT: sete %cl
-; AVX512VBMI-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
-; AVX512VBMI-NEXT: vpermq %zmm0, %zmm1, %zmm0
+; AVX512VBMI-NEXT: je .LBB16_1
+; AVX512VBMI-NEXT: # %bb.2: # %select.false.sink
+; AVX512VBMI-NEXT: vextracti32x4 $3, %zmm0, %xmm1
+; AVX512VBMI-NEXT: vpextrq $1, %xmm1, %rcx
+; AVX512VBMI-NEXT: vmovq %xmm1, %rdx
+; AVX512VBMI-NEXT: vextracti32x4 $2, %zmm0, %xmm1
+; AVX512VBMI-NEXT: vpextrq $1, %xmm1, %rsi
+; AVX512VBMI-NEXT: vmovq %xmm1, %rdi
+; AVX512VBMI-NEXT: vpextrq $1, %xmm0, %r8
+; AVX512VBMI-NEXT: vmovq %xmm0, %r9
+; AVX512VBMI-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX512VBMI-NEXT: vpextrq $1, %xmm0, %r10
+; AVX512VBMI-NEXT: vmovq %xmm0, %r11
+; AVX512VBMI-NEXT: vmovq %r9, %xmm0
+; AVX512VBMI-NEXT: vmovq %r8, %xmm1
+; AVX512VBMI-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512VBMI-NEXT: vmovq %r11, %xmm1
+; AVX512VBMI-NEXT: vmovq %r10, %xmm2
+; AVX512VBMI-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX512VBMI-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512VBMI-NEXT: vmovq %rdi, %xmm1
+; AVX512VBMI-NEXT: vmovq %rsi, %xmm2
+; AVX512VBMI-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX512VBMI-NEXT: vmovq %rdx, %xmm2
+; AVX512VBMI-NEXT: vmovq %rcx, %xmm3
+; AVX512VBMI-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
+; AVX512VBMI-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
+; AVX512VBMI-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
; AVX512VBMI-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VBMI-NEXT: vplzcntq %zmm0, %zmm0
; AVX512VBMI-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,64,128,192,256,320,384,448]
; AVX512VBMI-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
-; AVX512VBMI-NEXT: vmovq %xmm0, %rdx
-; AVX512VBMI-NEXT: movabsq $-9223372036854775808, %rsi # imm = 0x8000000000000000
-; AVX512VBMI-NEXT: shrxq %rdx, %rsi, %rsi
-; AVX512VBMI-NEXT: shrl $6, %edx
-; AVX512VBMI-NEXT: movl $128, %edi
-; AVX512VBMI-NEXT: shrxq %rdx, %rdi, %rdx
-; AVX512VBMI-NEXT: kmovd %edx, %k1
-; AVX512VBMI-NEXT: vpbroadcastq %rsi, %zmm0 {%k1} {z}
-; AVX512VBMI-NEXT: negl %ecx
-; AVX512VBMI-NEXT: kmovd %ecx, %k0
-; AVX512VBMI-NEXT: knotw %k0, %k1
-; AVX512VBMI-NEXT: vmovdqa32 %zmm0, %zmm0 {%k1} {z}
-; AVX512VBMI-NEXT: vmovdqu64 %zmm0, (%rax)
+; AVX512VBMI-NEXT: vmovq %xmm0, %rcx
+; AVX512VBMI-NEXT: movabsq $-9223372036854775808, %rdx # imm = 0x8000000000000000
+; AVX512VBMI-NEXT: shrxq %rcx, %rdx, %rdx
+; AVX512VBMI-NEXT: shrl $6, %ecx
+; AVX512VBMI-NEXT: movl $128, %esi
+; AVX512VBMI-NEXT: shrxq %rcx, %rsi, %rcx
+; AVX512VBMI-NEXT: kmovd %ecx, %k1
+; AVX512VBMI-NEXT: vpbroadcastq %rdx, %zmm0 {%k1} {z}
+; AVX512VBMI-NEXT: vextracti32x4 $3, %zmm0, %xmm1
+; AVX512VBMI-NEXT: vpextrq $1, %xmm1, %rcx
+; AVX512VBMI-NEXT: vmovq %xmm1, %rdx
+; AVX512VBMI-NEXT: vextracti32x4 $2, %zmm0, %xmm1
+; AVX512VBMI-NEXT: vpextrq $1, %xmm1, %rsi
+; AVX512VBMI-NEXT: vpextrq $1, %xmm0, %r8
+; AVX512VBMI-NEXT: vmovq %xmm1, %r9
+; AVX512VBMI-NEXT: vmovq %xmm0, %rdi
+; AVX512VBMI-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX512VBMI-NEXT: vpextrq $1, %xmm0, %r10
+; AVX512VBMI-NEXT: vmovq %xmm0, %r11
+; AVX512VBMI-NEXT: jmp .LBB16_3
+; AVX512VBMI-NEXT: .LBB16_1:
+; AVX512VBMI-NEXT: xorl %edi, %edi
+; AVX512VBMI-NEXT: xorl %r8d, %r8d
+; AVX512VBMI-NEXT: xorl %r11d, %r11d
+; AVX512VBMI-NEXT: xorl %r10d, %r10d
+; AVX512VBMI-NEXT: xorl %r9d, %r9d
+; AVX512VBMI-NEXT: xorl %esi, %esi
+; AVX512VBMI-NEXT: xorl %edx, %edx
+; AVX512VBMI-NEXT: xorl %ecx, %ecx
+; AVX512VBMI-NEXT: .LBB16_3: # %select.end
+; AVX512VBMI-NEXT: movq %rdi, (%rax)
+; AVX512VBMI-NEXT: movq %r8, 8(%rax)
+; AVX512VBMI-NEXT: movq %r11, 16(%rax)
+; AVX512VBMI-NEXT: movq %r10, 24(%rax)
+; AVX512VBMI-NEXT: movq %r9, 32(%rax)
+; AVX512VBMI-NEXT: movq %rsi, 40(%rax)
+; AVX512VBMI-NEXT: movq %rdx, 48(%rax)
+; AVX512VBMI-NEXT: movq %rcx, 56(%rax)
; AVX512VBMI-NEXT: vzeroupper
; AVX512VBMI-NEXT: retq
%a0 = bitcast <8 x i64> %v0 to i512
@@ -4949,63 +5219,65 @@ define i512 @isolate_msb_i512_load(ptr %p0, i512 %idx) nounwind {
; SSE2-NEXT: pushq %r12
; SSE2-NEXT: pushq %rbx
; SSE2-NEXT: pushq %rax
-; SSE2-NEXT: movq 8(%rsi), %r9
-; SSE2-NEXT: movq 16(%rsi), %rcx
-; SSE2-NEXT: movq 24(%rsi), %r8
-; SSE2-NEXT: movq 48(%rsi), %rdx
-; SSE2-NEXT: movq 56(%rsi), %r11
-; SSE2-NEXT: movdqa 32(%rsi), %xmm1
-; SSE2-NEXT: movdqa 48(%rsi), %xmm2
+; SSE2-NEXT: movq %rdi, %rax
; SSE2-NEXT: movdqa 16(%rsi), %xmm0
-; SSE2-NEXT: por %xmm2, %xmm0
-; SSE2-NEXT: movdqa (%rsi), %xmm3
-; SSE2-NEXT: por %xmm1, %xmm3
-; SSE2-NEXT: por %xmm0, %xmm3
+; SSE2-NEXT: por 48(%rsi), %xmm0
+; SSE2-NEXT: movdqa (%rsi), %xmm1
+; SSE2-NEXT: por 32(%rsi), %xmm1
+; SSE2-NEXT: por %xmm0, %xmm1
; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: pcmpeqd %xmm0, %xmm3
-; SSE2-NEXT: movmskps %xmm3, %eax
-; SSE2-NEXT: xorl $15, %eax
-; SSE2-NEXT: bsrq %r11, %r10
-; SSE2-NEXT: xorq $63, %r10
-; SSE2-NEXT: bsrq %rdx, %rbx
-; SSE2-NEXT: xorq $63, %rbx
-; SSE2-NEXT: orq $64, %rbx
-; SSE2-NEXT: testq %r11, %r11
-; SSE2-NEXT: cmovneq %r10, %rbx
-; SSE2-NEXT: movq 40(%rsi), %r14
-; SSE2-NEXT: bsrq %r14, %r15
-; SSE2-NEXT: bsrq 32(%rsi), %r10
+; SSE2-NEXT: pcmpeqd %xmm0, %xmm1
+; SSE2-NEXT: movmskps %xmm1, %ecx
+; SSE2-NEXT: xorl $15, %ecx
+; SSE2-NEXT: je .LBB17_1
+; SSE2-NEXT: # %bb.2: # %select.false.sink
+; SSE2-NEXT: movq 56(%rsi), %rdi
+; SSE2-NEXT: movq 48(%rsi), %r9
+; SSE2-NEXT: movq 40(%rsi), %rcx
+; SSE2-NEXT: movq 32(%rsi), %r8
+; SSE2-NEXT: movq 24(%rsi), %r10
+; SSE2-NEXT: movq 16(%rsi), %r11
+; SSE2-NEXT: movq 8(%rsi), %r14
+; SSE2-NEXT: bsrq %rdi, %rdx
+; SSE2-NEXT: xorq $63, %rdx
+; SSE2-NEXT: bsrq %r9, %r15
; SSE2-NEXT: xorq $63, %r15
-; SSE2-NEXT: xorq $63, %r10
-; SSE2-NEXT: orq $64, %r10
-; SSE2-NEXT: testq %r14, %r14
-; SSE2-NEXT: cmovneq %r15, %r10
-; SSE2-NEXT: orq $128, %r10
-; SSE2-NEXT: orq %r11, %rdx
-; SSE2-NEXT: cmovneq %rbx, %r10
-; SSE2-NEXT: bsrq %r8, %rdx
+; SSE2-NEXT: orq $64, %r15
+; SSE2-NEXT: testq %rdi, %rdi
+; SSE2-NEXT: cmovneq %rdx, %r15
+; SSE2-NEXT: bsrq %rcx, %rdx
; SSE2-NEXT: xorq $63, %rdx
-; SSE2-NEXT: bsrq %rcx, %r11
-; SSE2-NEXT: xorq $63, %r11
-; SSE2-NEXT: orq $64, %r11
-; SSE2-NEXT: testq %r8, %r8
-; SSE2-NEXT: cmovneq %rdx, %r11
-; SSE2-NEXT: bsrq %r9, %rbx
+; SSE2-NEXT: bsrq %r8, %rbx
; SSE2-NEXT: xorq $63, %rbx
+; SSE2-NEXT: orq $64, %rbx
+; SSE2-NEXT: testq %rcx, %rcx
+; SSE2-NEXT: cmovneq %rdx, %rbx
+; SSE2-NEXT: orq $128, %rbx
+; SSE2-NEXT: movq %r9, %rdx
+; SSE2-NEXT: orq %rdi, %rdx
+; SSE2-NEXT: cmovneq %r15, %rbx
+; SSE2-NEXT: bsrq %r10, %rdx
+; SSE2-NEXT: xorq $63, %rdx
+; SSE2-NEXT: bsrq %r11, %r15
+; SSE2-NEXT: xorq $63, %r15
+; SSE2-NEXT: orq $64, %r15
+; SSE2-NEXT: testq %r10, %r10
+; SSE2-NEXT: cmovneq %rdx, %r15
+; SSE2-NEXT: bsrq %r14, %r12
+; SSE2-NEXT: xorq $63, %r12
; SSE2-NEXT: bsrq (%rsi), %rdx
; SSE2-NEXT: xorq $63, %rdx
; SSE2-NEXT: orq $64, %rdx
-; SSE2-NEXT: testq %r9, %r9
-; SSE2-NEXT: cmovneq %rbx, %rdx
+; SSE2-NEXT: testq %r14, %r14
+; SSE2-NEXT: cmovneq %r12, %rdx
; SSE2-NEXT: orq $128, %rdx
-; SSE2-NEXT: orq %r8, %rcx
-; SSE2-NEXT: cmovneq %r11, %rdx
+; SSE2-NEXT: orq %r10, %r11
+; SSE2-NEXT: cmovneq %r15, %rdx
; SSE2-NEXT: orq $256, %rdx # imm = 0x100
-; SSE2-NEXT: por %xmm2, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm0, %xmm1
-; SSE2-NEXT: movmskps %xmm1, %ecx
-; SSE2-NEXT: xorl $15, %ecx
-; SSE2-NEXT: cmovneq %r10, %rdx
+; SSE2-NEXT: orq %r9, %r8
+; SSE2-NEXT: orq %rdi, %rcx
+; SSE2-NEXT: orq %r8, %rcx
+; SSE2-NEXT: cmovneq %rbx, %rdx
; SSE2-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
@@ -5021,47 +5293,47 @@ define i512 @isolate_msb_i512_load(ptr %p0, i512 %idx) nounwind {
; SSE2-NEXT: andl $56, %edx
; SSE2-NEXT: movq $0, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: movq -72(%rsp,%rdx), %rsi
-; SSE2-NEXT: movq -80(%rsp,%rdx), %r10
+; SSE2-NEXT: movq -80(%rsp,%rdx), %r9
+; SSE2-NEXT: movq %r9, %rdi
+; SSE2-NEXT: shrdq %cl, %rsi, %rdi
+; SSE2-NEXT: movq -88(%rsp,%rdx), %r10
; SSE2-NEXT: movq %r10, %r8
-; SSE2-NEXT: shrdq %cl, %rsi, %r8
-; SSE2-NEXT: movq -88(%rsp,%rdx), %r11
+; SSE2-NEXT: shrdq %cl, %r9, %r8
+; SSE2-NEXT: movq -96(%rsp,%rdx), %r11
; SSE2-NEXT: movq %r11, %r9
; SSE2-NEXT: shrdq %cl, %r10, %r9
-; SSE2-NEXT: movq -96(%rsp,%rdx), %rbx
+; SSE2-NEXT: movq -104(%rsp,%rdx), %rbx
; SSE2-NEXT: movq %rbx, %r10
; SSE2-NEXT: shrdq %cl, %r11, %r10
-; SSE2-NEXT: movq -104(%rsp,%rdx), %r14
+; SSE2-NEXT: movq -112(%rsp,%rdx), %r14
; SSE2-NEXT: movq %r14, %r11
; SSE2-NEXT: shrdq %cl, %rbx, %r11
-; SSE2-NEXT: movq -112(%rsp,%rdx), %r15
+; SSE2-NEXT: movq -120(%rsp,%rdx), %r15
; SSE2-NEXT: movq %r15, %rbx
; SSE2-NEXT: shrdq %cl, %r14, %rbx
-; SSE2-NEXT: movq -120(%rsp,%rdx), %r12
-; SSE2-NEXT: movq %r12, %r14
-; SSE2-NEXT: shrdq %cl, %r15, %r14
; SSE2-NEXT: movq -128(%rsp,%rdx), %rdx
; SSE2-NEXT: shrq %cl, %rsi
; SSE2-NEXT: # kill: def $cl killed $cl killed $ecx
-; SSE2-NEXT: shrdq %cl, %r12, %rdx
-; SSE2-NEXT: xorl %ecx, %ecx
-; SSE2-NEXT: testl %eax, %eax
-; SSE2-NEXT: cmoveq %rcx, %r14
-; SSE2-NEXT: cmoveq %rcx, %rbx
-; SSE2-NEXT: cmoveq %rcx, %r11
-; SSE2-NEXT: cmoveq %rcx, %r10
-; SSE2-NEXT: cmoveq %rcx, %r9
-; SSE2-NEXT: cmoveq %rcx, %r8
-; SSE2-NEXT: cmoveq %rcx, %rdx
-; SSE2-NEXT: movq %rdi, %rax
-; SSE2-NEXT: cmoveq %rcx, %rsi
-; SSE2-NEXT: movq %rsi, 56(%rdi)
-; SSE2-NEXT: movq %r8, 48(%rdi)
-; SSE2-NEXT: movq %r9, 40(%rdi)
-; SSE2-NEXT: movq %r10, 32(%rdi)
-; SSE2-NEXT: movq %r11, 24(%rdi)
-; SSE2-NEXT: movq %rbx, 16(%rdi)
-; SSE2-NEXT: movq %r14, 8(%rdi)
-; SSE2-NEXT: movq %rdx, (%rdi)
+; SSE2-NEXT: shrdq %cl, %r15, %rdx
+; SSE2-NEXT: jmp .LBB17_3
+; SSE2-NEXT: .LBB17_1:
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: xorl %ebx, %ebx
+; SSE2-NEXT: xorl %r11d, %r11d
+; SSE2-NEXT: xorl %r10d, %r10d
+; SSE2-NEXT: xorl %r9d, %r9d
+; SSE2-NEXT: xorl %r8d, %r8d
+; SSE2-NEXT: xorl %edi, %edi
+; SSE2-NEXT: xorl %esi, %esi
+; SSE2-NEXT: .LBB17_3: # %select.end
+; SSE2-NEXT: movq %rdx, (%rax)
+; SSE2-NEXT: movq %rbx, 8(%rax)
+; SSE2-NEXT: movq %r11, 16(%rax)
+; SSE2-NEXT: movq %r10, 24(%rax)
+; SSE2-NEXT: movq %r9, 32(%rax)
+; SSE2-NEXT: movq %r8, 40(%rax)
+; SSE2-NEXT: movq %rdi, 48(%rax)
+; SSE2-NEXT: movq %rsi, 56(%rax)
; SSE2-NEXT: addq $8, %rsp
; SSE2-NEXT: popq %rbx
; SSE2-NEXT: popq %r12
@@ -5073,116 +5345,125 @@ define i512 @isolate_msb_i512_load(ptr %p0, i512 %idx) nounwind {
; SSE42: # %bb.0:
; SSE42-NEXT: pushq %r15
; SSE42-NEXT: pushq %r14
+; SSE42-NEXT: pushq %r12
; SSE42-NEXT: pushq %rbx
-; SSE42-NEXT: movq 8(%rsi), %r8
-; SSE42-NEXT: movq 16(%rsi), %rcx
-; SSE42-NEXT: movq 24(%rsi), %rdx
-; SSE42-NEXT: movq 40(%rsi), %r11
-; SSE42-NEXT: movq 48(%rsi), %rax
-; SSE42-NEXT: movq 56(%rsi), %r10
-; SSE42-NEXT: movdqa 32(%rsi), %xmm2
-; SSE42-NEXT: movdqa 48(%rsi), %xmm0
-; SSE42-NEXT: movdqa (%rsi), %xmm1
-; SSE42-NEXT: bsrq %r10, %r9
-; SSE42-NEXT: xorq $63, %r9
-; SSE42-NEXT: bsrq %rax, %rbx
+; SSE42-NEXT: pushq %rax
+; SSE42-NEXT: movdqa (%rsi), %xmm0
+; SSE42-NEXT: movdqa 16(%rsi), %xmm1
+; SSE42-NEXT: por 48(%rsi), %xmm1
+; SSE42-NEXT: por 32(%rsi), %xmm0
+; SSE42-NEXT: movq %rdi, %rax
+; SSE42-NEXT: por %xmm1, %xmm0
+; SSE42-NEXT: ptest %xmm0, %xmm0
+; SSE42-NEXT: je .LBB17_1
+; SSE42-NEXT: # %bb.2: # %select.false.sink
+; SSE42-NEXT: movq 56(%rsi), %rdi
+; SSE42-NEXT: movq 48(%rsi), %r9
+; SSE42-NEXT: movq 40(%rsi), %rcx
+; SSE42-NEXT: movq 32(%rsi), %r8
+; SSE42-NEXT: movq 24(%rsi), %r10
+; SSE42-NEXT: movq 16(%rsi), %r11
+; SSE42-NEXT: bsrq %rdi, %rdx
+; SSE42-NEXT: xorq $63, %rdx
+; SSE42-NEXT: bsrq %r9, %r14
+; SSE42-NEXT: xorq $63, %r14
+; SSE42-NEXT: orq $64, %r14
+; SSE42-NEXT: testq %rdi, %rdi
+; SSE42-NEXT: cmovneq %rdx, %r14
+; SSE42-NEXT: bsrq %rcx, %rdx
+; SSE42-NEXT: xorq $63, %rdx
+; SSE42-NEXT: bsrq %r8, %rbx
; SSE42-NEXT: xorq $63, %rbx
; SSE42-NEXT: orq $64, %rbx
-; SSE42-NEXT: testq %r10, %r10
-; SSE42-NEXT: cmovneq %r9, %rbx
+; SSE42-NEXT: testq %rcx, %rcx
+; SSE42-NEXT: cmovneq %rdx, %rbx
+; SSE42-NEXT: orq $128, %rbx
+; SSE42-NEXT: movq %r9, %rdx
+; SSE42-NEXT: orq %rdi, %rdx
+; SSE42-NEXT: cmovneq %r14, %rbx
+; SSE42-NEXT: bsrq %r10, %rdx
+; SSE42-NEXT: xorq $63, %rdx
; SSE42-NEXT: bsrq %r11, %r14
; SSE42-NEXT: xorq $63, %r14
-; SSE42-NEXT: bsrq 32(%rsi), %r9
-; SSE42-NEXT: xorq $63, %r9
-; SSE42-NEXT: orq $64, %r9
-; SSE42-NEXT: testq %r11, %r11
-; SSE42-NEXT: cmovneq %r14, %r9
-; SSE42-NEXT: orq $128, %r9
-; SSE42-NEXT: orq %r10, %rax
-; SSE42-NEXT: cmovneq %rbx, %r9
-; SSE42-NEXT: bsrq %rdx, %rax
-; SSE42-NEXT: xorq $63, %rax
-; SSE42-NEXT: bsrq %rcx, %r10
-; SSE42-NEXT: xorq $63, %r10
-; SSE42-NEXT: orq $64, %r10
-; SSE42-NEXT: testq %rdx, %rdx
-; SSE42-NEXT: cmovneq %rax, %r10
-; SSE42-NEXT: por %xmm2, %xmm1
-; SSE42-NEXT: bsrq %r8, %r11
-; SSE42-NEXT: bsrq (%rsi), %rax
-; SSE42-NEXT: xorq $63, %r11
-; SSE42-NEXT: xorq $63, %rax
-; SSE42-NEXT: orq $64, %rax
-; SSE42-NEXT: testq %r8, %r8
-; SSE42-NEXT: cmovneq %r11, %rax
-; SSE42-NEXT: orq $128, %rax
-; SSE42-NEXT: orq %rdx, %rcx
-; SSE42-NEXT: cmovneq %r10, %rax
-; SSE42-NEXT: orq $256, %rax # imm = 0x100
-; SSE42-NEXT: por %xmm0, %xmm2
-; SSE42-NEXT: ptest %xmm2, %xmm2
-; SSE42-NEXT: cmovneq %r9, %rax
-; SSE42-NEXT: movdqa 16(%rsi), %xmm2
-; SSE42-NEXT: xorps %xmm3, %xmm3
-; SSE42-NEXT: movaps %xmm3, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movaps %xmm3, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movaps %xmm3, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movaps %xmm3, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: orq $64, %r14
+; SSE42-NEXT: testq %r10, %r10
+; SSE42-NEXT: cmovneq %rdx, %r14
+; SSE42-NEXT: movq 8(%rsi), %r15
+; SSE42-NEXT: bsrq %r15, %r12
+; SSE42-NEXT: bsrq (%rsi), %rdx
+; SSE42-NEXT: xorq $63, %r12
+; SSE42-NEXT: xorq $63, %rdx
+; SSE42-NEXT: orq $64, %rdx
+; SSE42-NEXT: testq %r15, %r15
+; SSE42-NEXT: cmovneq %r12, %rdx
+; SSE42-NEXT: orq $128, %rdx
+; SSE42-NEXT: orq %r10, %r11
+; SSE42-NEXT: cmovneq %r14, %rdx
+; SSE42-NEXT: orq $256, %rdx # imm = 0x100
+; SSE42-NEXT: orq %r9, %r8
+; SSE42-NEXT: orq %rdi, %rcx
+; SSE42-NEXT: orq %r8, %rcx
+; SSE42-NEXT: cmovneq %rbx, %rdx
+; SSE42-NEXT: pxor %xmm0, %xmm0
+; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
; SSE42-NEXT: movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000
; SSE42-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movaps %xmm3, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movaps %xmm3, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movaps %xmm3, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movl %eax, %ecx
+; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movl %edx, %ecx
; SSE42-NEXT: andl $63, %ecx
-; SSE42-NEXT: shrl $3, %eax
-; SSE42-NEXT: andl $56, %eax
+; SSE42-NEXT: shrl $3, %edx
+; SSE42-NEXT: andl $56, %edx
; SSE42-NEXT: movq $0, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movq -72(%rsp,%rax), %rdx
-; SSE42-NEXT: movq -80(%rsp,%rax), %r9
-; SSE42-NEXT: movq %r9, %rsi
-; SSE42-NEXT: shrdq %cl, %rdx, %rsi
-; SSE42-NEXT: movq -88(%rsp,%rax), %r10
+; SSE42-NEXT: movq -72(%rsp,%rdx), %rsi
+; SSE42-NEXT: movq -80(%rsp,%rdx), %r9
+; SSE42-NEXT: movq %r9, %rdi
+; SSE42-NEXT: shrdq %cl, %rsi, %rdi
+; SSE42-NEXT: movq -88(%rsp,%rdx), %r10
; SSE42-NEXT: movq %r10, %r8
; SSE42-NEXT: shrdq %cl, %r9, %r8
-; SSE42-NEXT: movq -96(%rsp,%rax), %r11
+; SSE42-NEXT: movq -96(%rsp,%rdx), %r11
; SSE42-NEXT: movq %r11, %r9
; SSE42-NEXT: shrdq %cl, %r10, %r9
-; SSE42-NEXT: movq -104(%rsp,%rax), %rbx
+; SSE42-NEXT: movq -104(%rsp,%rdx), %rbx
; SSE42-NEXT: movq %rbx, %r10
; SSE42-NEXT: shrdq %cl, %r11, %r10
-; SSE42-NEXT: movq -112(%rsp,%rax), %r14
+; SSE42-NEXT: movq -112(%rsp,%rdx), %r14
; SSE42-NEXT: movq %r14, %r11
; SSE42-NEXT: shrdq %cl, %rbx, %r11
-; SSE42-NEXT: movq -120(%rsp,%rax), %r15
+; SSE42-NEXT: movq -120(%rsp,%rdx), %r15
; SSE42-NEXT: movq %r15, %rbx
; SSE42-NEXT: shrdq %cl, %r14, %rbx
-; SSE42-NEXT: movq -128(%rsp,%rax), %r14
-; SSE42-NEXT: shrq %cl, %rdx
+; SSE42-NEXT: movq -128(%rsp,%rdx), %rdx
+; SSE42-NEXT: shrq %cl, %rsi
; SSE42-NEXT: # kill: def $cl killed $cl killed $ecx
-; SSE42-NEXT: shrdq %cl, %r15, %r14
-; SSE42-NEXT: por %xmm0, %xmm2
-; SSE42-NEXT: por %xmm2, %xmm1
-; SSE42-NEXT: xorl %ecx, %ecx
-; SSE42-NEXT: ptest %xmm1, %xmm1
-; SSE42-NEXT: cmoveq %rcx, %rbx
-; SSE42-NEXT: cmoveq %rcx, %r11
-; SSE42-NEXT: cmoveq %rcx, %r10
-; SSE42-NEXT: cmoveq %rcx, %r9
-; SSE42-NEXT: cmoveq %rcx, %r8
-; SSE42-NEXT: cmoveq %rcx, %rsi
-; SSE42-NEXT: cmoveq %rcx, %r14
-; SSE42-NEXT: movq %rdi, %rax
-; SSE42-NEXT: cmoveq %rcx, %rdx
-; SSE42-NEXT: movq %rdx, 56(%rdi)
-; SSE42-NEXT: movq %rsi, 48(%rdi)
-; SSE42-NEXT: movq %r8, 40(%rdi)
-; SSE42-NEXT: movq %r9, 32(%rdi)
-; SSE42-NEXT: movq %r10, 24(%rdi)
-; SSE42-NEXT: movq %r11, 16(%rdi)
-; SSE42-NEXT: movq %rbx, 8(%rdi)
-; SSE42-NEXT: movq %r14, (%rdi)
+; SSE42-NEXT: shrdq %cl, %r15, %rdx
+; SSE42-NEXT: jmp .LBB17_3
+; SSE42-NEXT: .LBB17_1:
+; SSE42-NEXT: xorl %edx, %edx
+; SSE42-NEXT: xorl %ebx, %ebx
+; SSE42-NEXT: xorl %r11d, %r11d
+; SSE42-NEXT: xorl %r10d, %r10d
+; SSE42-NEXT: xorl %r9d, %r9d
+; SSE42-NEXT: xorl %r8d, %r8d
+; SSE42-NEXT: xorl %edi, %edi
+; SSE42-NEXT: xorl %esi, %esi
+; SSE42-NEXT: .LBB17_3: # %select.end
+; SSE42-NEXT: movq %rdx, (%rax)
+; SSE42-NEXT: movq %rbx, 8(%rax)
+; SSE42-NEXT: movq %r11, 16(%rax)
+; SSE42-NEXT: movq %r10, 24(%rax)
+; SSE42-NEXT: movq %r9, 32(%rax)
+; SSE42-NEXT: movq %r8, 40(%rax)
+; SSE42-NEXT: movq %rdi, 48(%rax)
+; SSE42-NEXT: movq %rsi, 56(%rax)
+; SSE42-NEXT: addq $8, %rsp
; SSE42-NEXT: popq %rbx
+; SSE42-NEXT: popq %r12
; SSE42-NEXT: popq %r14
; SSE42-NEXT: popq %r15
; SSE42-NEXT: retq
@@ -5192,101 +5473,111 @@ define i512 @isolate_msb_i512_load(ptr %p0, i512 %idx) nounwind {
; AVX2-NEXT: pushq %r15
; AVX2-NEXT: pushq %r14
; AVX2-NEXT: pushq %rbx
-; AVX2-NEXT: vmovdqu 32(%rsi), %ymm1
-; AVX2-NEXT: movq 16(%rsi), %rax
-; AVX2-NEXT: movq 24(%rsi), %rcx
-; AVX2-NEXT: movq 40(%rsi), %rdx
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: vmovdqu (%rsi), %ymm0
+; AVX2-NEXT: vpor 32(%rsi), %ymm0, %ymm0
+; AVX2-NEXT: vptest %ymm0, %ymm0
+; AVX2-NEXT: je .LBB17_1
+; AVX2-NEXT: # %bb.2: # %select.false.sink
+; AVX2-NEXT: movq 56(%rsi), %rdi
; AVX2-NEXT: movq 48(%rsi), %r9
-; AVX2-NEXT: vpor (%rsi), %ymm1, %ymm0
-; AVX2-NEXT: movq 56(%rsi), %r10
-; AVX2-NEXT: lzcntq %r10, %r8
-; AVX2-NEXT: lzcntq %r9, %r11
-; AVX2-NEXT: addq $64, %r11
-; AVX2-NEXT: testq %r10, %r10
-; AVX2-NEXT: cmovneq %r8, %r11
-; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: lzcntq %rdx, %rbx
-; AVX2-NEXT: xorl %r8d, %r8d
-; AVX2-NEXT: lzcntq 32(%rsi), %r8
-; AVX2-NEXT: addq $64, %r8
-; AVX2-NEXT: testq %rdx, %rdx
-; AVX2-NEXT: cmovneq %rbx, %r8
-; AVX2-NEXT: subq $-128, %r8
-; AVX2-NEXT: orq %r10, %r9
-; AVX2-NEXT: cmovneq %r11, %r8
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: lzcntq %rcx, %rdx
-; AVX2-NEXT: xorl %r9d, %r9d
-; AVX2-NEXT: lzcntq %rax, %r9
-; AVX2-NEXT: addq $64, %r9
+; AVX2-NEXT: movq 40(%rsi), %rcx
+; AVX2-NEXT: movq 32(%rsi), %r8
+; AVX2-NEXT: movq 24(%rsi), %r10
+; AVX2-NEXT: movq 16(%rsi), %r11
+; AVX2-NEXT: movq (%rsi), %rdx
+; AVX2-NEXT: movq 8(%rsi), %rbx
+; AVX2-NEXT: xorl %esi, %esi
+; AVX2-NEXT: lzcntq %rdi, %rsi
+; AVX2-NEXT: xorl %r14d, %r14d
+; AVX2-NEXT: lzcntq %r9, %r14
+; AVX2-NEXT: addq $64, %r14
+; AVX2-NEXT: testq %rdi, %rdi
+; AVX2-NEXT: cmovneq %rsi, %r14
+; AVX2-NEXT: xorl %r15d, %r15d
+; AVX2-NEXT: lzcntq %rcx, %r15
+; AVX2-NEXT: xorl %esi, %esi
+; AVX2-NEXT: lzcntq %r8, %rsi
+; AVX2-NEXT: addq $64, %rsi
; AVX2-NEXT: testq %rcx, %rcx
-; AVX2-NEXT: cmovneq %rdx, %r9
-; AVX2-NEXT: movq 8(%rsi), %r10
-; AVX2-NEXT: xorl %r11d, %r11d
-; AVX2-NEXT: lzcntq %r10, %r11
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: lzcntq (%rsi), %rdx
-; AVX2-NEXT: addq $64, %rdx
+; AVX2-NEXT: cmovneq %r15, %rsi
+; AVX2-NEXT: subq $-128, %rsi
+; AVX2-NEXT: movq %r9, %r15
+; AVX2-NEXT: orq %rdi, %r15
+; AVX2-NEXT: cmovneq %r14, %rsi
+; AVX2-NEXT: xorl %r14d, %r14d
+; AVX2-NEXT: lzcntq %r10, %r14
+; AVX2-NEXT: xorl %r15d, %r15d
+; AVX2-NEXT: lzcntq %r11, %r15
+; AVX2-NEXT: addq $64, %r15
; AVX2-NEXT: testq %r10, %r10
-; AVX2-NEXT: cmovneq %r11, %rdx
+; AVX2-NEXT: cmovneq %r14, %r15
+; AVX2-NEXT: xorl %r14d, %r14d
+; AVX2-NEXT: lzcntq %rbx, %r14
+; AVX2-NEXT: lzcntq %rdx, %rdx
+; AVX2-NEXT: addq $64, %rdx
+; AVX2-NEXT: testq %rbx, %rbx
+; AVX2-NEXT: cmovneq %r14, %rdx
; AVX2-NEXT: subq $-128, %rdx
-; AVX2-NEXT: orq %rcx, %rax
-; AVX2-NEXT: cmovneq %r9, %rdx
+; AVX2-NEXT: orq %r10, %r11
+; AVX2-NEXT: cmovneq %r15, %rdx
; AVX2-NEXT: addq $256, %rdx # imm = 0x100
-; AVX2-NEXT: vpor 48(%rsi), %xmm1, %xmm1
-; AVX2-NEXT: vptest %xmm1, %xmm1
-; AVX2-NEXT: cmovneq %r8, %rdx
+; AVX2-NEXT: orq %r9, %r8
+; AVX2-NEXT: orq %rdi, %rcx
+; AVX2-NEXT: orq %r8, %rcx
+; AVX2-NEXT: cmovneq %rsi, %rdx
+; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovaps {{.*#+}} ymm1 = [0,0,0,9223372036854775808]
; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movl %edx, %ecx
; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: shrl $3, %edx
; AVX2-NEXT: andl $56, %edx
-; AVX2-NEXT: movq -72(%rsp,%rdx), %r11
-; AVX2-NEXT: movq -80(%rsp,%rdx), %rax
-; AVX2-NEXT: movq %rax, %rsi
-; AVX2-NEXT: shrdq %cl, %r11, %rsi
-; AVX2-NEXT: movq -88(%rsp,%rdx), %r10
+; AVX2-NEXT: movq -72(%rsp,%rdx), %rbx
+; AVX2-NEXT: movq -80(%rsp,%rdx), %r8
+; AVX2-NEXT: movq %r8, %rsi
+; AVX2-NEXT: shrdq %cl, %rbx, %rsi
+; AVX2-NEXT: movq -88(%rsp,%rdx), %r9
+; AVX2-NEXT: movq %r9, %rdi
+; AVX2-NEXT: shrdq %cl, %r8, %rdi
+; AVX2-NEXT: movq -96(%rsp,%rdx), %r10
; AVX2-NEXT: movq %r10, %r8
-; AVX2-NEXT: shrdq %cl, %rax, %r8
-; AVX2-NEXT: movq -96(%rsp,%rdx), %rax
-; AVX2-NEXT: movq %rax, %r9
+; AVX2-NEXT: shrdq %cl, %r9, %r8
+; AVX2-NEXT: movq -104(%rsp,%rdx), %r11
+; AVX2-NEXT: movq %r11, %r9
; AVX2-NEXT: shrdq %cl, %r10, %r9
-; AVX2-NEXT: movq -104(%rsp,%rdx), %r14
+; AVX2-NEXT: movq -112(%rsp,%rdx), %r14
; AVX2-NEXT: movq %r14, %r10
-; AVX2-NEXT: shrdq %cl, %rax, %r10
-; AVX2-NEXT: movq -112(%rsp,%rdx), %r15
-; AVX2-NEXT: movq %r15, %rbx
-; AVX2-NEXT: shrdq %cl, %r14, %rbx
-; AVX2-NEXT: movq %rdi, %rax
-; AVX2-NEXT: movq -128(%rsp,%rdx), %rdi
-; AVX2-NEXT: movq -120(%rsp,%rdx), %r14
-; AVX2-NEXT: movq %r14, %rdx
+; AVX2-NEXT: shrdq %cl, %r11, %r10
+; AVX2-NEXT: movq -120(%rsp,%rdx), %r15
+; AVX2-NEXT: movq %r15, %r11
+; AVX2-NEXT: shrdq %cl, %r14, %r11
+; AVX2-NEXT: movq -128(%rsp,%rdx), %rdx
+; AVX2-NEXT: shrxq %rcx, %rbx, %rbx
+; AVX2-NEXT: # kill: def $cl killed $cl killed $rcx
; AVX2-NEXT: shrdq %cl, %r15, %rdx
-; AVX2-NEXT: shrdq %cl, %r14, %rdi
-; AVX2-NEXT: xorl %r14d, %r14d
-; AVX2-NEXT: vptest %ymm0, %ymm0
-; AVX2-NEXT: shrxq %rcx, %r11, %rcx
-; AVX2-NEXT: cmoveq %r14, %rdx
-; AVX2-NEXT: cmoveq %r14, %rbx
-; AVX2-NEXT: cmoveq %r14, %r10
-; AVX2-NEXT: cmoveq %r14, %r9
-; AVX2-NEXT: cmoveq %r14, %r8
-; AVX2-NEXT: cmoveq %r14, %rsi
-; AVX2-NEXT: cmoveq %r14, %rdi
-; AVX2-NEXT: cmoveq %r14, %rcx
-; AVX2-NEXT: movq %rcx, 56(%rax)
+; AVX2-NEXT: jmp .LBB17_3
+; AVX2-NEXT: .LBB17_1:
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: xorl %r11d, %r11d
+; AVX2-NEXT: xorl %r10d, %r10d
+; AVX2-NEXT: xorl %r9d, %r9d
+; AVX2-NEXT: xorl %r8d, %r8d
+; AVX2-NEXT: xorl %edi, %edi
+; AVX2-NEXT: xorl %esi, %esi
+; AVX2-NEXT: xorl %ebx, %ebx
+; AVX2-NEXT: .LBB17_3: # %select.end
+; AVX2-NEXT: movq %rdx, (%rax)
+; AVX2-NEXT: movq %r11, 8(%rax)
+; AVX2-NEXT: movq %r10, 16(%rax)
+; AVX2-NEXT: movq %r9, 24(%rax)
+; AVX2-NEXT: movq %r8, 32(%rax)
+; AVX2-NEXT: movq %rdi, 40(%rax)
; AVX2-NEXT: movq %rsi, 48(%rax)
-; AVX2-NEXT: movq %r8, 40(%rax)
-; AVX2-NEXT: movq %r9, 32(%rax)
-; AVX2-NEXT: movq %r10, 24(%rax)
-; AVX2-NEXT: movq %rbx, 16(%rax)
-; AVX2-NEXT: movq %rdx, 8(%rax)
-; AVX2-NEXT: movq %rdi, (%rax)
+; AVX2-NEXT: movq %rbx, 56(%rax)
; AVX2-NEXT: popq %rbx
; AVX2-NEXT: popq %r14
; AVX2-NEXT: popq %r15
@@ -5298,28 +5589,66 @@ define i512 @isolate_msb_i512_load(ptr %p0, i512 %idx) nounwind {
; AVX512F-NEXT: movq %rdi, %rax
; AVX512F-NEXT: vmovdqu64 (%rsi), %zmm0
; AVX512F-NEXT: vptestmd %zmm0, %zmm0, %k0
-; AVX512F-NEXT: xorl %ecx, %ecx
; AVX512F-NEXT: kortestw %k0, %k0
-; AVX512F-NEXT: sete %cl
-; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
-; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0
+; AVX512F-NEXT: je .LBB17_1
+; AVX512F-NEXT: # %bb.2: # %select.false.sink
+; AVX512F-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
+; AVX512F-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
+; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512F-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
+; AVX512F-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero
+; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX512F-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512F-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
+; AVX512F-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero
+; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX512F-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero
+; AVX512F-NEXT: vmovq {{.*#+}} xmm3 = mem[0],zero
+; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
+; AVX512F-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
+; AVX512F-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512F-NEXT: vplzcntq %zmm0, %zmm1
+; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1 # [0,64,128,192,256,320,384,448]
; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
-; AVX512F-NEXT: vplzcntq %zmm0, %zmm0
-; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,64,128,192,256,320,384,448]
-; AVX512F-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
-; AVX512F-NEXT: vmovq %xmm0, %rdx
-; AVX512F-NEXT: movabsq $-9223372036854775808, %rsi # imm = 0x8000000000000000
-; AVX512F-NEXT: shrxq %rdx, %rsi, %rsi
-; AVX512F-NEXT: shrl $6, %edx
-; AVX512F-NEXT: movl $128, %edi
-; AVX512F-NEXT: shrxq %rdx, %rdi, %rdx
-; AVX512F-NEXT: kmovw %edx, %k1
-; AVX512F-NEXT: vpbroadcastq %rsi, %zmm0 {%k1} {z}
-; AVX512F-NEXT: negl %ecx
-; AVX512F-NEXT: kmovw %ecx, %k0
-; AVX512F-NEXT: knotw %k0, %k1
-; AVX512F-NEXT: vmovdqa32 %zmm0, %zmm0 {%k1} {z}
-; AVX512F-NEXT: vmovdqu64 %zmm0, (%rax)
+; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; AVX512F-NEXT: vmovq %xmm0, %rcx
+; AVX512F-NEXT: movabsq $-9223372036854775808, %rdx # imm = 0x8000000000000000
+; AVX512F-NEXT: shrxq %rcx, %rdx, %rdx
+; AVX512F-NEXT: shrl $6, %ecx
+; AVX512F-NEXT: movl $128, %esi
+; AVX512F-NEXT: shrxq %rcx, %rsi, %rcx
+; AVX512F-NEXT: kmovw %ecx, %k1
+; AVX512F-NEXT: vpbroadcastq %rdx, %zmm0 {%k1} {z}
+; AVX512F-NEXT: vextracti32x4 $3, %zmm0, %xmm1
+; AVX512F-NEXT: vpextrq $1, %xmm1, %rcx
+; AVX512F-NEXT: vmovq %xmm1, %rdx
+; AVX512F-NEXT: vextracti32x4 $2, %zmm0, %xmm1
+; AVX512F-NEXT: vpextrq $1, %xmm1, %rsi
+; AVX512F-NEXT: vmovq %xmm1, %r8
+; AVX512F-NEXT: vpextrq $1, %xmm0, %r9
+; AVX512F-NEXT: vmovq %xmm0, %rdi
+; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX512F-NEXT: vpextrq $1, %xmm0, %r10
+; AVX512F-NEXT: vmovq %xmm0, %r11
+; AVX512F-NEXT: jmp .LBB17_3
+; AVX512F-NEXT: .LBB17_1:
+; AVX512F-NEXT: xorl %edi, %edi
+; AVX512F-NEXT: xorl %r9d, %r9d
+; AVX512F-NEXT: xorl %r11d, %r11d
+; AVX512F-NEXT: xorl %r10d, %r10d
+; AVX512F-NEXT: xorl %r8d, %r8d
+; AVX512F-NEXT: xorl %esi, %esi
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: xorl %ecx, %ecx
+; AVX512F-NEXT: .LBB17_3: # %select.end
+; AVX512F-NEXT: movq %rdi, (%rax)
+; AVX512F-NEXT: movq %r9, 8(%rax)
+; AVX512F-NEXT: movq %r11, 16(%rax)
+; AVX512F-NEXT: movq %r10, 24(%rax)
+; AVX512F-NEXT: movq %r8, 32(%rax)
+; AVX512F-NEXT: movq %rsi, 40(%rax)
+; AVX512F-NEXT: movq %rdx, 48(%rax)
+; AVX512F-NEXT: movq %rcx, 56(%rax)
; AVX512F-NEXT: retq
;
; AVX512VL-LABEL: isolate_msb_i512_load:
@@ -5327,28 +5656,66 @@ define i512 @isolate_msb_i512_load(ptr %p0, i512 %idx) nounwind {
; AVX512VL-NEXT: movq %rdi, %rax
; AVX512VL-NEXT: vmovdqu64 (%rsi), %zmm0
; AVX512VL-NEXT: vptestmd %zmm0, %zmm0, %k0
-; AVX512VL-NEXT: xorl %ecx, %ecx
; AVX512VL-NEXT: kortestw %k0, %k0
-; AVX512VL-NEXT: sete %cl
-; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
-; AVX512VL-NEXT: vpermq %zmm0, %zmm1, %zmm0
+; AVX512VL-NEXT: je .LBB17_1
+; AVX512VL-NEXT: # %bb.2: # %select.false.sink
+; AVX512VL-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
+; AVX512VL-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
+; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512VL-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
+; AVX512VL-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero
+; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX512VL-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512VL-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
+; AVX512VL-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero
+; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX512VL-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero
+; AVX512VL-NEXT: vmovq {{.*#+}} xmm3 = mem[0],zero
+; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
+; AVX512VL-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
+; AVX512VL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VL-NEXT: vplzcntq %zmm0, %zmm0
; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,64,128,192,256,320,384,448]
; AVX512VL-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
-; AVX512VL-NEXT: vmovq %xmm0, %rdx
-; AVX512VL-NEXT: movabsq $-9223372036854775808, %rsi # imm = 0x8000000000000000
-; AVX512VL-NEXT: shrxq %rdx, %rsi, %rsi
-; AVX512VL-NEXT: shrl $6, %edx
-; AVX512VL-NEXT: movl $128, %edi
-; AVX512VL-NEXT: shrxq %rdx, %rdi, %rdx
-; AVX512VL-NEXT: kmovd %edx, %k1
-; AVX512VL-NEXT: vpbroadcastq %rsi, %zmm0 {%k1} {z}
-; AVX512VL-NEXT: negl %ecx
-; AVX512VL-NEXT: kmovd %ecx, %k0
-; AVX512VL-NEXT: knotw %k0, %k1
-; AVX512VL-NEXT: vmovdqa32 %zmm0, %zmm0 {%k1} {z}
-; AVX512VL-NEXT: vmovdqu64 %zmm0, (%rax)
+; AVX512VL-NEXT: vmovq %xmm0, %rcx
+; AVX512VL-NEXT: movabsq $-9223372036854775808, %rdx # imm = 0x8000000000000000
+; AVX512VL-NEXT: shrxq %rcx, %rdx, %rdx
+; AVX512VL-NEXT: shrl $6, %ecx
+; AVX512VL-NEXT: movl $128, %esi
+; AVX512VL-NEXT: shrxq %rcx, %rsi, %rcx
+; AVX512VL-NEXT: kmovd %ecx, %k1
+; AVX512VL-NEXT: vpbroadcastq %rdx, %zmm0 {%k1} {z}
+; AVX512VL-NEXT: vextracti32x4 $3, %zmm0, %xmm1
+; AVX512VL-NEXT: vpextrq $1, %xmm1, %rcx
+; AVX512VL-NEXT: vmovq %xmm1, %rdx
+; AVX512VL-NEXT: vextracti32x4 $2, %zmm0, %xmm1
+; AVX512VL-NEXT: vpextrq $1, %xmm1, %rsi
+; AVX512VL-NEXT: vpextrq $1, %xmm0, %r8
+; AVX512VL-NEXT: vmovq %xmm1, %r9
+; AVX512VL-NEXT: vmovq %xmm0, %rdi
+; AVX512VL-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX512VL-NEXT: vpextrq $1, %xmm0, %r10
+; AVX512VL-NEXT: vmovq %xmm0, %r11
+; AVX512VL-NEXT: jmp .LBB17_3
+; AVX512VL-NEXT: .LBB17_1:
+; AVX512VL-NEXT: xorl %edi, %edi
+; AVX512VL-NEXT: xorl %r8d, %r8d
+; AVX512VL-NEXT: xorl %r11d, %r11d
+; AVX512VL-NEXT: xorl %r10d, %r10d
+; AVX512VL-NEXT: xorl %r9d, %r9d
+; AVX512VL-NEXT: xorl %esi, %esi
+; AVX512VL-NEXT: xorl %edx, %edx
+; AVX512VL-NEXT: xorl %ecx, %ecx
+; AVX512VL-NEXT: .LBB17_3: # %select.end
+; AVX512VL-NEXT: movq %rdi, (%rax)
+; AVX512VL-NEXT: movq %r8, 8(%rax)
+; AVX512VL-NEXT: movq %r11, 16(%rax)
+; AVX512VL-NEXT: movq %r10, 24(%rax)
+; AVX512VL-NEXT: movq %r9, 32(%rax)
+; AVX512VL-NEXT: movq %rsi, 40(%rax)
+; AVX512VL-NEXT: movq %rdx, 48(%rax)
+; AVX512VL-NEXT: movq %rcx, 56(%rax)
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
@@ -5357,28 +5724,66 @@ define i512 @isolate_msb_i512_load(ptr %p0, i512 %idx) nounwind {
; AVX512VBMI-NEXT: movq %rdi, %rax
; AVX512VBMI-NEXT: vmovdqu64 (%rsi), %zmm0
; AVX512VBMI-NEXT: vptestmd %zmm0, %zmm0, %k0
-; AVX512VBMI-NEXT: xorl %ecx, %ecx
; AVX512VBMI-NEXT: kortestw %k0, %k0
-; AVX512VBMI-NEXT: sete %cl
-; AVX512VBMI-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
-; AVX512VBMI-NEXT: vpermq %zmm0, %zmm1, %zmm0
+; AVX512VBMI-NEXT: je .LBB17_1
+; AVX512VBMI-NEXT: # %bb.2: # %select.false.sink
+; AVX512VBMI-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
+; AVX512VBMI-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
+; AVX512VBMI-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512VBMI-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
+; AVX512VBMI-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero
+; AVX512VBMI-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX512VBMI-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512VBMI-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
+; AVX512VBMI-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero
+; AVX512VBMI-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX512VBMI-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero
+; AVX512VBMI-NEXT: vmovq {{.*#+}} xmm3 = mem[0],zero
+; AVX512VBMI-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
+; AVX512VBMI-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
+; AVX512VBMI-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
; AVX512VBMI-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VBMI-NEXT: vplzcntq %zmm0, %zmm0
; AVX512VBMI-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,64,128,192,256,320,384,448]
; AVX512VBMI-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
-; AVX512VBMI-NEXT: vmovq %xmm0, %rdx
-; AVX512VBMI-NEXT: movabsq $-9223372036854775808, %rsi # imm = 0x8000000000000000
-; AVX512VBMI-NEXT: shrxq %rdx, %rsi, %rsi
-; AVX512VBMI-NEXT: shrl $6, %edx
-; AVX512VBMI-NEXT: movl $128, %edi
-; AVX512VBMI-NEXT: shrxq %rdx, %rdi, %rdx
-; AVX512VBMI-NEXT: kmovd %edx, %k1
-; AVX512VBMI-NEXT: vpbroadcastq %rsi, %zmm0 {%k1} {z}
-; AVX512VBMI-NEXT: negl %ecx
-; AVX512VBMI-NEXT: kmovd %ecx, %k0
-; AVX512VBMI-NEXT: knotw %k0, %k1
-; AVX512VBMI-NEXT: vmovdqa32 %zmm0, %zmm0 {%k1} {z}
-; AVX512VBMI-NEXT: vmovdqu64 %zmm0, (%rax)
+; AVX512VBMI-NEXT: vmovq %xmm0, %rcx
+; AVX512VBMI-NEXT: movabsq $-9223372036854775808, %rdx # imm = 0x8000000000000000
+; AVX512VBMI-NEXT: shrxq %rcx, %rdx, %rdx
+; AVX512VBMI-NEXT: shrl $6, %ecx
+; AVX512VBMI-NEXT: movl $128, %esi
+; AVX512VBMI-NEXT: shrxq %rcx, %rsi, %rcx
+; AVX512VBMI-NEXT: kmovd %ecx, %k1
+; AVX512VBMI-NEXT: vpbroadcastq %rdx, %zmm0 {%k1} {z}
+; AVX512VBMI-NEXT: vextracti32x4 $3, %zmm0, %xmm1
+; AVX512VBMI-NEXT: vpextrq $1, %xmm1, %rcx
+; AVX512VBMI-NEXT: vmovq %xmm1, %rdx
+; AVX512VBMI-NEXT: vextracti32x4 $2, %zmm0, %xmm1
+; AVX512VBMI-NEXT: vpextrq $1, %xmm1, %rsi
+; AVX512VBMI-NEXT: vpextrq $1, %xmm0, %r8
+; AVX512VBMI-NEXT: vmovq %xmm1, %r9
+; AVX512VBMI-NEXT: vmovq %xmm0, %rdi
+; AVX512VBMI-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX512VBMI-NEXT: vpextrq $1, %xmm0, %r10
+; AVX512VBMI-NEXT: vmovq %xmm0, %r11
+; AVX512VBMI-NEXT: jmp .LBB17_3
+; AVX512VBMI-NEXT: .LBB17_1:
+; AVX512VBMI-NEXT: xorl %edi, %edi
+; AVX512VBMI-NEXT: xorl %r8d, %r8d
+; AVX512VBMI-NEXT: xorl %r11d, %r11d
+; AVX512VBMI-NEXT: xorl %r10d, %r10d
+; AVX512VBMI-NEXT: xorl %r9d, %r9d
+; AVX512VBMI-NEXT: xorl %esi, %esi
+; AVX512VBMI-NEXT: xorl %edx, %edx
+; AVX512VBMI-NEXT: xorl %ecx, %ecx
+; AVX512VBMI-NEXT: .LBB17_3: # %select.end
+; AVX512VBMI-NEXT: movq %rdi, (%rax)
+; AVX512VBMI-NEXT: movq %r8, 8(%rax)
+; AVX512VBMI-NEXT: movq %r11, 16(%rax)
+; AVX512VBMI-NEXT: movq %r10, 24(%rax)
+; AVX512VBMI-NEXT: movq %r9, 32(%rax)
+; AVX512VBMI-NEXT: movq %rsi, 40(%rax)
+; AVX512VBMI-NEXT: movq %rdx, 48(%rax)
+; AVX512VBMI-NEXT: movq %rcx, 56(%rax)
; AVX512VBMI-NEXT: vzeroupper
; AVX512VBMI-NEXT: retq
%a0 = load i512, ptr %p0
diff --git a/llvm/test/CodeGen/X86/bsr.ll b/llvm/test/CodeGen/X86/bsr.ll
index affacc5ee6487..44e9f60bd49eb 100644
--- a/llvm/test/CodeGen/X86/bsr.ll
+++ b/llvm/test/CodeGen/X86/bsr.ll
@@ -248,26 +248,27 @@ define i64 @cmov_bsr64(i64 %x, i64 %y) nounwind {
define i64 @cmov_bsr64_undef(i64 %x, i64 %y) nounwind {
; X86-LABEL: cmov_bsr64_undef:
; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: testl %edx, %edx
-; X86-NEXT: jne .LBB7_1
-; X86-NEXT: # %bb.2:
+; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: orl %eax, %edx
+; X86-NEXT: je .LBB7_1
+; X86-NEXT: # %bb.2: # %select.false.sink
+; X86-NEXT: testl %eax, %eax
+; X86-NEXT: jne .LBB7_3
+; X86-NEXT: # %bb.4: # %select.false.sink
; X86-NEXT: bsrl %ecx, %eax
; X86-NEXT: xorl $31, %eax
; X86-NEXT: orl $32, %eax
-; X86-NEXT: orl %edx, %ecx
-; X86-NEXT: jne .LBB7_5
-; X86-NEXT: .LBB7_4:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: jmp .LBB7_5
+; X86-NEXT: .LBB7_1:
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: retl
-; X86-NEXT: .LBB7_1:
-; X86-NEXT: bsrl %edx, %eax
+; X86-NEXT: .LBB7_3:
+; X86-NEXT: bsrl %eax, %eax
; X86-NEXT: xorl $31, %eax
-; X86-NEXT: orl %edx, %ecx
-; X86-NEXT: je .LBB7_4
-; X86-NEXT: .LBB7_5:
+; X86-NEXT: .LBB7_5: # %select.false.sink
; X86-NEXT: xorl $63, %eax
; X86-NEXT: xorl %edx, %edx
; X86-NEXT: retl
@@ -372,20 +373,23 @@ define i128 @cmov_bsr128(i128 %x, i128 %y) nounwind {
;
; X64-LABEL: cmov_bsr128:
; X64: # %bb.0:
-; X64-NEXT: bsrq %rsi, %r8
-; X64-NEXT: xorq $63, %r8
+; X64-NEXT: movq %rdx, %rax
+; X64-NEXT: movq %rdi, %rdx
+; X64-NEXT: orq %rsi, %rdx
+; X64-NEXT: je .LBB8_2
+; X64-NEXT: # %bb.1: # %select.false.sink
+; X64-NEXT: bsrq %rsi, %rcx
+; X64-NEXT: xorq $63, %rcx
; X64-NEXT: movl $127, %eax
; X64-NEXT: bsrq %rdi, %rax
; X64-NEXT: xorq $63, %rax
; X64-NEXT: addq $64, %rax
; X64-NEXT: testq %rsi, %rsi
-; X64-NEXT: cmovneq %r8, %rax
+; X64-NEXT: cmovneq %rcx, %rax
; X64-NEXT: xorq $127, %rax
-; X64-NEXT: xorl %r8d, %r8d
-; X64-NEXT: orq %rsi, %rdi
-; X64-NEXT: cmoveq %rdx, %rax
-; X64-NEXT: cmoveq %rcx, %r8
-; X64-NEXT: movq %r8, %rdx
+; X64-NEXT: xorl %ecx, %ecx
+; X64-NEXT: .LBB8_2: # %select.end
+; X64-NEXT: movq %rcx, %rdx
; X64-NEXT: retq
%1 = tail call i128 @llvm.ctlz.i128(i128 %x, i1 false)
%2 = xor i128 %1, 127
@@ -404,60 +408,62 @@ define i128 @cmov_bsr128_undef(i128 %x, i128 %y) nounwind {
; X86-NEXT: pushl %esi
; X86-NEXT: andl $-16, %esp
; X86-NEXT: subl $16, %esp
-; X86-NEXT: movl 28(%ebp), %edx
-; X86-NEXT: movl 32(%ebp), %edi
-; X86-NEXT: movl 36(%ebp), %eax
-; X86-NEXT: testl %eax, %eax
-; X86-NEXT: jne .LBB9_1
-; X86-NEXT: # %bb.2:
-; X86-NEXT: bsrl %edi, %esi
-; X86-NEXT: xorl $31, %esi
-; X86-NEXT: orl $32, %esi
-; X86-NEXT: jmp .LBB9_3
-; X86-NEXT: .LBB9_1:
-; X86-NEXT: bsrl %eax, %esi
-; X86-NEXT: xorl $31, %esi
-; X86-NEXT: .LBB9_3:
+; X86-NEXT: movl 36(%ebp), %edx
+; X86-NEXT: movl 32(%ebp), %esi
+; X86-NEXT: movl 28(%ebp), %edi
; X86-NEXT: movl 24(%ebp), %ebx
+; X86-NEXT: movl %edi, %eax
+; X86-NEXT: orl %edx, %eax
+; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: orl %esi, %ecx
+; X86-NEXT: orl %eax, %ecx
+; X86-NEXT: movl 8(%ebp), %eax
+; X86-NEXT: je .LBB9_1
+; X86-NEXT: # %bb.2: # %select.true.sink
; X86-NEXT: testl %edx, %edx
-; X86-NEXT: jne .LBB9_4
-; X86-NEXT: # %bb.5:
-; X86-NEXT: bsrl %ebx, %ecx
+; X86-NEXT: jne .LBB9_3
+; X86-NEXT: # %bb.4: # %select.true.sink
+; X86-NEXT: bsrl %esi, %ecx
; X86-NEXT: xorl $31, %ecx
; X86-NEXT: orl $32, %ecx
-; X86-NEXT: orl %eax, %edi
+; X86-NEXT: testl %edi, %edi
; X86-NEXT: je .LBB9_7
-; X86-NEXT: jmp .LBB9_8
-; X86-NEXT: .LBB9_4:
+; X86-NEXT: .LBB9_6:
+; X86-NEXT: bsrl %edi, %edi
+; X86-NEXT: xorl $31, %edi
+; X86-NEXT: orl %edx, %esi
+; X86-NEXT: je .LBB9_9
+; X86-NEXT: jmp .LBB9_10
+; X86-NEXT: .LBB9_1:
+; X86-NEXT: movl 52(%ebp), %esi
+; X86-NEXT: movl 48(%ebp), %edi
+; X86-NEXT: movl 44(%ebp), %edx
+; X86-NEXT: movl 40(%ebp), %ecx
+; X86-NEXT: jmp .LBB9_11
+; X86-NEXT: .LBB9_3:
; X86-NEXT: bsrl %edx, %ecx
; X86-NEXT: xorl $31, %ecx
-; X86-NEXT: orl %eax, %edi
-; X86-NEXT: jne .LBB9_8
-; X86-NEXT: .LBB9_7:
-; X86-NEXT: orl $64, %ecx
-; X86-NEXT: movl %ecx, %esi
-; X86-NEXT: .LBB9_8:
-; X86-NEXT: orl %eax, %edx
-; X86-NEXT: orl 32(%ebp), %ebx
-; X86-NEXT: orl %edx, %ebx
-; X86-NEXT: jne .LBB9_9
-; X86-NEXT: # %bb.10:
-; X86-NEXT: movl 48(%ebp), %edx
-; X86-NEXT: movl 52(%ebp), %edi
-; X86-NEXT: movl 40(%ebp), %esi
-; X86-NEXT: movl 44(%ebp), %ecx
-; X86-NEXT: jmp .LBB9_11
-; X86-NEXT: .LBB9_9:
-; X86-NEXT: xorl $127, %esi
-; X86-NEXT: xorl %ecx, %ecx
+; X86-NEXT: testl %edi, %edi
+; X86-NEXT: jne .LBB9_6
+; X86-NEXT: .LBB9_7: # %select.true.sink
+; X86-NEXT: bsrl %ebx, %edi
+; X86-NEXT: xorl $31, %edi
+; X86-NEXT: orl $32, %edi
+; X86-NEXT: orl %edx, %esi
+; X86-NEXT: jne .LBB9_10
+; X86-NEXT: .LBB9_9: # %select.true.sink
+; X86-NEXT: orl $64, %edi
+; X86-NEXT: movl %edi, %ecx
+; X86-NEXT: .LBB9_10: # %select.true.sink
+; X86-NEXT: xorl $127, %ecx
; X86-NEXT: xorl %edx, %edx
; X86-NEXT: xorl %edi, %edi
-; X86-NEXT: .LBB9_11:
-; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: movl %edi, 12(%eax)
-; X86-NEXT: movl %edx, 8(%eax)
-; X86-NEXT: movl %ecx, 4(%eax)
-; X86-NEXT: movl %esi, (%eax)
+; X86-NEXT: xorl %esi, %esi
+; X86-NEXT: .LBB9_11: # %select.end
+; X86-NEXT: movl %ecx, (%eax)
+; X86-NEXT: movl %edx, 4(%eax)
+; X86-NEXT: movl %edi, 8(%eax)
+; X86-NEXT: movl %esi, 12(%eax)
; X86-NEXT: leal -12(%ebp), %esp
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
@@ -467,19 +473,22 @@ define i128 @cmov_bsr128_undef(i128 %x, i128 %y) nounwind {
;
; X64-LABEL: cmov_bsr128_undef:
; X64: # %bb.0:
-; X64-NEXT: bsrq %rsi, %r8
-; X64-NEXT: xorq $63, %r8
+; X64-NEXT: movq %rdx, %rax
+; X64-NEXT: movq %rdi, %rdx
+; X64-NEXT: orq %rsi, %rdx
+; X64-NEXT: je .LBB9_2
+; X64-NEXT: # %bb.1: # %select.true.sink
+; X64-NEXT: bsrq %rsi, %rcx
+; X64-NEXT: xorq $63, %rcx
; X64-NEXT: bsrq %rdi, %rax
; X64-NEXT: xorq $63, %rax
; X64-NEXT: orq $64, %rax
; X64-NEXT: testq %rsi, %rsi
-; X64-NEXT: cmovneq %r8, %rax
+; X64-NEXT: cmovneq %rcx, %rax
; X64-NEXT: xorq $127, %rax
-; X64-NEXT: xorl %r8d, %r8d
-; X64-NEXT: orq %rsi, %rdi
-; X64-NEXT: cmoveq %rdx, %rax
-; X64-NEXT: cmoveq %rcx, %r8
-; X64-NEXT: movq %r8, %rdx
+; X64-NEXT: xorl %ecx, %ecx
+; X64-NEXT: .LBB9_2: # %select.end
+; X64-NEXT: movq %rcx, %rdx
; X64-NEXT: retq
%1 = tail call i128 @llvm.ctlz.i128(i128 %x, i1 true)
%2 = xor i128 %1, 127
diff --git a/llvm/test/CodeGen/X86/freeze-unary.ll b/llvm/test/CodeGen/X86/freeze-unary.ll
index 222ec1eb3b9de..ad653d3f5134e 100644
--- a/llvm/test/CodeGen/X86/freeze-unary.ll
+++ b/llvm/test/CodeGen/X86/freeze-unary.ll
@@ -209,11 +209,18 @@ declare <4 x i32> @llvm.bitreverse.v4i32(<4 x i32>)
define i32 @freeze_ctlz(i32 %a0) nounwind {
; X86-LABEL: freeze_ctlz:
; X86: # %bb.0:
-; X86-NEXT: bsrl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: testl %eax, %eax
+; X86-NEXT: je .LBB12_1
+; X86-NEXT: # %bb.2: # %select.false.sink
+; X86-NEXT: bsrl %eax, %ecx
; X86-NEXT: movl $63, %eax
; X86-NEXT: cmovnel %ecx, %eax
; X86-NEXT: xorl $31, %eax
; X86-NEXT: retl
+; X86-NEXT: .LBB12_1:
+; X86-NEXT: movl $32, %eax
+; X86-NEXT: retl
;
; X64-LABEL: freeze_ctlz:
; X64: # %bb.0:
diff --git a/llvm/test/CodeGen/X86/pseudo_cmov_lower.ll b/llvm/test/CodeGen/X86/pseudo_cmov_lower.ll
index 7d7d72e4c89f6..956b8811cf283 100644
--- a/llvm/test/CodeGen/X86/pseudo_cmov_lower.ll
+++ b/llvm/test/CodeGen/X86/pseudo_cmov_lower.ll
@@ -119,13 +119,13 @@ entry:
ret <2 x double> %sub
}
-; This test checks that only a single ja gets generated in the final code
-; for lowering the CMOV pseudos that get created for this IR. This combines
-; all the supported types together into one long string of selects based
-; on the same condition.
+; This test checks that only a single conditional jump gets generated in the
+; final code for lowering the CMOV pseudos that get created for this IR. This
+; combines all the supported types together into one long string of selects
+; based on the same condition.
; CHECK-LABEL: foo8:
-; CHECK: ja
-; CHECK-NOT: ja
+; CHECK: jbe
+; CHECK-NOT: jbe
define void @foo8(i32 %v1,
i8 %v2, i8 %v3,
i16 %v12, i16 %v13,
@@ -200,8 +200,8 @@ entry:
ret void
}
-; This test checks that only a single ja gets generated in the final code
-; for lowering the CMOV pseudos that get created for this IR.
+; This test checks that only a single conditional jump gets generated in the
+; final code for lowering the CMOV pseudos that get created for this IR.
; on the same condition.
; Contrary to my expectations, this doesn't exercise the code for
; CMOV_V8I1, CMOV_V16I1, CMOV_V32I1, or CMOV_V64I1. Instead the selects all
@@ -213,8 +213,8 @@ entry:
; tests those opcodes.
;
; CHECK-LABEL: foo9:
-; CHECK: ja
-; CHECK-NOT: ja
+; CHECK: jae
+; CHECK-NOT: jae
define void @foo9(i32 %v1,
<8 x i1> %v12, <8 x i1> %v13,
<16 x i1> %v22, <16 x i1> %v23,
diff --git a/llvm/test/CodeGen/X86/pseudo_cmov_lower2.ll b/llvm/test/CodeGen/X86/pseudo_cmov_lower2.ll
index 193a07c8ad3b4..9ecdb23723faa 100644
--- a/llvm/test/CodeGen/X86/pseudo_cmov_lower2.ll
+++ b/llvm/test/CodeGen/X86/pseudo_cmov_lower2.ll
@@ -123,16 +123,16 @@ define double @foo5(float %p1, double %p2, double %p3) nounwind {
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: xorps %xmm3, %xmm3
; CHECK-NEXT: ucomiss %xmm3, %xmm0
-; CHECK-NEXT: movsd {{.*#+}} xmm0 = [1.25E+0,0.0E+0]
; CHECK-NEXT: jae .LBB4_1
-; CHECK-NEXT: # %bb.2: # %select.false
+; CHECK-NEXT: # %bb.2: # %select.false.sink
+; CHECK-NEXT: movsd {{.*#+}} xmm0 = [1.25E+0,0.0E+0]
; CHECK-NEXT: addsd %xmm2, %xmm0
; CHECK-NEXT: .LBB4_3: # %select.end
; CHECK-NEXT: subsd %xmm1, %xmm0
; CHECK-NEXT: addsd %xmm2, %xmm0
; CHECK-NEXT: retq
-; CHECK-NEXT: .LBB4_1:
-; CHECK-NEXT: addsd %xmm0, %xmm1
+; CHECK-NEXT: .LBB4_1: # %select.true.sink
+; CHECK-NEXT: addsd {{.*}}, %xmm1
; CHECK-NEXT: movapd %xmm1, %xmm0
; CHECK-NEXT: movapd %xmm1, %xmm2
; CHECK-NEXT: jmp .LBB4_3
@@ -157,27 +157,28 @@ define double @foo6(float %p1, double %p2, double %p3) nounwind {
; CHECK-NEXT: movaps %xmm0, %xmm3
; CHECK-NEXT: xorps %xmm0, %xmm0
; CHECK-NEXT: ucomiss %xmm0, %xmm3
-; CHECK-NEXT: movsd {{.*#+}} xmm0 = [1.25E+0,0.0E+0]
; CHECK-NEXT: jae .LBB5_1
-; CHECK-NEXT: # %bb.2: # %select.false
+; CHECK-NEXT: # %bb.2: # %select.false.sink
+; CHECK-NEXT: movsd {{.*#+}} xmm0 = [1.25E+0,0.0E+0]
; CHECK-NEXT: addsd %xmm2, %xmm0
; CHECK-NEXT: .LBB5_3: # %select.end
; CHECK-NEXT: ucomiss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
; CHECK-NEXT: movapd %xmm0, %xmm4
; CHECK-NEXT: jae .LBB5_5
-; CHECK-NEXT: # %bb.4: # %select.false2
+; CHECK-NEXT: # %bb.4: # %select.false
; CHECK-NEXT: movapd %xmm1, %xmm4
; CHECK-NEXT: .LBB5_5: # %select.end1
; CHECK-NEXT: ucomiss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
; CHECK-NEXT: movapd %xmm4, %xmm1
; CHECK-NEXT: jae .LBB5_7
-; CHECK-NEXT: # %bb.6: # %select.false4
+; CHECK-NEXT: # %bb.6: # %select.false3
; CHECK-NEXT: movapd %xmm2, %xmm1
-; CHECK-NEXT: .LBB5_7: # %select.end3
+; CHECK-NEXT: .LBB5_7: # %select.end2
; CHECK-NEXT: subsd %xmm4, %xmm0
; CHECK-NEXT: addsd %xmm1, %xmm0
; CHECK-NEXT: retq
-; CHECK-NEXT: .LBB5_1:
+; CHECK-NEXT: .LBB5_1: # %select.true.sink
+; CHECK-NEXT: movsd {{.*#+}} xmm0 = [1.25E+0,0.0E+0]
; CHECK-NEXT: addsd %xmm1, %xmm0
; CHECK-NEXT: jmp .LBB5_3
entry:
diff --git a/llvm/test/CodeGen/X86/shrink-wrap-chkstk-x86_64.ll b/llvm/test/CodeGen/X86/shrink-wrap-chkstk-x86_64.ll
index 0793c33d2e8eb..b0c07debbeda2 100644
--- a/llvm/test/CodeGen/X86/shrink-wrap-chkstk-x86_64.ll
+++ b/llvm/test/CodeGen/X86/shrink-wrap-chkstk-x86_64.ll
@@ -9,24 +9,19 @@
define void @fn1() nounwind uwtable {
; CHECK-LABEL: fn1:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: movl a(%rip), %eax
-; CHECK-NEXT: testl %eax, %eax
-; CHECK-NEXT: jne .LBB0_2
-; CHECK-NEXT: # %bb.1: # %select.true.sink
-; CHECK-NEXT: cltq
-; CHECK-NEXT: imulq $715827883, %rax, %rax # imm = 0x2AAAAAAB
-; CHECK-NEXT: movq %rax, %rcx
-; CHECK-NEXT: shrq $63, %rcx
-; CHECK-NEXT: shrq $32, %rax
-; CHECK-NEXT: addl %ecx, %eax
-; CHECK-NEXT: .LBB0_2: # %select.end
-; CHECK-NEXT: pushq %rax
-; CHECK-NEXT: movl $4128, %eax # imm = 0x1020
+; CHECK-NEXT: movl $4136, %eax # imm = 0x1028
; CHECK-NEXT: callq ___chkstk_ms
; CHECK-NEXT: subq %rax, %rsp
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rax
; CHECK-NEXT: .cfi_def_cfa_offset 4144
-; CHECK-NEXT: movl %eax, b(%rip)
+; CHECK-NEXT: movslq a(%rip), %rax
+; CHECK-NEXT: imulq $715827883, %rax, %rcx # imm = 0x2AAAAAAB
+; CHECK-NEXT: movq %rcx, %rdx
+; CHECK-NEXT: shrq $63, %rdx
+; CHECK-NEXT: shrq $32, %rcx
+; CHECK-NEXT: addl %edx, %ecx
+; CHECK-NEXT: testq %rax, %rax
+; CHECK-NEXT: cmovnel %eax, %ecx
+; CHECK-NEXT: movl %ecx, b(%rip)
; CHECK-NEXT: leaq {{[0-9]+}}(%rsp), %rcx
; CHECK-NEXT: # kill: def $ecx killed $ecx killed $rcx
; CHECK-NEXT: callq fn2
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/optimizeSelect-DT.ll b/llvm/test/Transforms/CodeGenPrepare/X86/optimizeSelect-DT.ll
index aaf3df0934681..1f82e2a4da57b 100644
--- a/llvm/test/Transforms/CodeGenPrepare/X86/optimizeSelect-DT.ll
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/optimizeSelect-DT.ll
@@ -4,21 +4,18 @@
target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"
+; srem by constant 2 is not expensive (lowers to cheap bit ops), so no sinking.
define i1 @PR41004(i32 %x, i32 %y, i32 %t1) {
; CHECK-LABEL: @PR41004(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[MUL_FR:%.*]] = freeze i32 [[Y:%.*]]
-; CHECK-NEXT: [[T0:%.*]] = icmp eq i32 [[MUL_FR]], 1
-; CHECK-NEXT: br i1 [[T0]], label [[SELECT_TRUE_SINK:%.*]], label [[SELECT_END:%.*]]
-; CHECK: select.true.sink:
; CHECK-NEXT: [[REM:%.*]] = srem i32 [[X:%.*]], 2
-; CHECK-NEXT: br label [[SELECT_END]]
-; CHECK: select.end:
-; CHECK-NEXT: [[MUL:%.*]] = phi i32 [ [[REM]], [[SELECT_TRUE_SINK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT: [[USUB:%.*]] = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 [[T1:%.*]], i32 1)
-; CHECK-NEXT: [[NEG:%.*]] = extractvalue { i32, i1 } [[USUB]], 0
-; CHECK-NEXT: [[TOBOOL:%.*]] = extractvalue { i32, i1 } [[USUB]], 1
+; CHECK-NEXT: [[T0:%.*]] = icmp eq i32 [[Y:%.*]], 1
+; CHECK-NEXT: [[MUL:%.*]] = select i1 [[T0]], i32 [[REM]], i32 0
+; CHECK-NEXT: [[NEG:%.*]] = add i32 [[T1:%.*]], -1
; CHECK-NEXT: [[ADD:%.*]] = add i32 [[NEG]], [[MUL]]
+; CHECK-NEXT: br label [[IF:%.*]]
+; CHECK: if:
+; CHECK-NEXT: [[TOBOOL:%.*]] = icmp eq i32 [[T1]], 0
; CHECK-NEXT: ret i1 [[TOBOOL]]
;
entry:
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/select.ll b/llvm/test/Transforms/CodeGenPrepare/X86/select.ll
index 1dafde445a3e2..eb2e16a4e269c 100644
--- a/llvm/test/Transforms/CodeGenPrepare/X86/select.ll
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/select.ll
@@ -200,31 +200,23 @@ define float @fadd_no_sink(float %a, float %b) {
ret float %sel
}
-; Possible enhancement: sinkability is only calculated with the direct
-; operand of the select, so we don't try to sink this. The fdiv cost is not
-; taken into account.
+; Recursive sinking: the fdiv is expensive, so sink the entire chain
+; (fdiv + fadd) into the taken branch.
-define float @fdiv_no_sink(float %a, float %b) {
-; CHECK-LABEL: @fdiv_no_sink(
+define float @fdiv_chain_sink(float %a, float %b) {
+; CHECK-LABEL: @fdiv_chain_sink(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[DIV:%.*]] = fdiv float [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[SEL_FROZEN:%.*]] = freeze float [[A:%.*]]
+; CHECK-NEXT: [[CMP:%.*]] = fcmp ogt float [[SEL_FROZEN]], 1.000000e+00
+; CHECK-NEXT: br i1 [[CMP]], label [[SELECT_TRUE_SINK:%.*]], label [[SELECT_END:%.*]]
+; CHECK: select.true.sink:
+; CHECK-NEXT: [[DIV:%.*]] = fdiv float [[A]], [[B:%.*]]
; CHECK-NEXT: [[ADD:%.*]] = fadd float [[DIV]], [[B]]
-; CHECK-NEXT: [[CMP:%.*]] = fcmp ogt float [[A]], 1.000000e+00
-; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], float [[ADD]], float 8.000000e+00
+; CHECK-NEXT: br label [[SELECT_END]]
+; CHECK: select.end:
+; CHECK-NEXT: [[SEL:%.*]] = phi float [ [[ADD]], [[SELECT_TRUE_SINK]] ], [ 8.000000e+00, [[ENTRY:%.*]] ]
; CHECK-NEXT: ret float [[SEL]]
;
-; DEBUG-LABEL: @fdiv_no_sink(
-; DEBUG-NEXT: entry:
-; DEBUG-NEXT: [[DIV:%.*]] = fdiv float [[A:%.*]], [[B:%.*]], !dbg [[DBG72:![0-9]+]]
-; DEBUG-NEXT: #dbg_value(float [[DIV]], [[META68:![0-9]+]], !DIExpression(), [[DBG72]])
-; DEBUG-NEXT: [[ADD:%.*]] = fadd float [[DIV]], [[B]], !dbg [[DBG73:![0-9]+]]
-; DEBUG-NEXT: #dbg_value(float [[ADD]], [[META69:![0-9]+]], !DIExpression(), [[DBG73]])
-; DEBUG-NEXT: [[CMP:%.*]] = fcmp ogt float [[A]], 1.000000e+00, !dbg [[DBG74:![0-9]+]]
-; DEBUG-NEXT: #dbg_value(i1 [[CMP]], [[META70:![0-9]+]], !DIExpression(), [[DBG74]])
-; DEBUG-NEXT: [[SEL:%.*]] = select i1 [[CMP]], float [[ADD]], float 8.000000e+00, !dbg [[DBG75:![0-9]+]]
-; DEBUG-NEXT: #dbg_value(float [[SEL]], [[META71:![0-9]+]], !DIExpression(), [[DBG75]])
-; DEBUG-NEXT: ret float [[SEL]], !dbg [[DBG76:![0-9]+]]
-;
entry:
%div = fdiv float %a, %b
%add = fadd float %div, %b
>From 7fc447221a2371556e6e3f22c9012295bea76db3 Mon Sep 17 00:00:00 2001
From: Ayush Rai <your-email at amd.com>
Date: Mon, 18 May 2026 10:33:21 +0530
Subject: [PATCH 3/8] Update ARM/SPARC test expectations for recursive select
operand sinking
Co-authored-by: Cursor <cursoragent at cursor.com>
---
.../CodeGen/ARM/fadd-select-fneg-combine.ll | 6 ++-
llvm/test/CodeGen/SPARC/predictable-select.ll | 48 ++++++++-----------
2 files changed, 24 insertions(+), 30 deletions(-)
diff --git a/llvm/test/CodeGen/ARM/fadd-select-fneg-combine.ll b/llvm/test/CodeGen/ARM/fadd-select-fneg-combine.ll
index 90bb02fdc0cd4..145beefdf2a96 100644
--- a/llvm/test/CodeGen/ARM/fadd-select-fneg-combine.ll
+++ b/llvm/test/CodeGen/ARM/fadd-select-fneg-combine.ll
@@ -50,7 +50,8 @@ define <2 x float> @fadd_select_fneg_fneg_v2f32(i32 %arg0, <2 x float> %x, <2 x
; CHECK-NEXT: add r0, sp, #8
; CHECK-NEXT: vldrw.u32 q1, [r0]
; CHECK-NEXT: b .LBB2_3
-; CHECK-NEXT: .LBB2_2:
+; CHECK-NEXT: .LBB2_2: @ %select.true.sink
+; CHECK-NEXT: vldr d3, [sp]
; CHECK-NEXT: vmov d2, r2, r3
; CHECK-NEXT: .LBB2_3: @ %select.end
; CHECK-NEXT: vneg.f32 q1, q1
@@ -77,7 +78,8 @@ define <2 x half> @fadd_select_fneg_fneg_v2f16(i32 %arg0, <2 x half> %x, <2 x ha
; CHECK-NEXT: add r0, sp, #8
; CHECK-NEXT: vldrw.u32 q1, [r0]
; CHECK-NEXT: b .LBB3_3
-; CHECK-NEXT: .LBB3_2:
+; CHECK-NEXT: .LBB3_2: @ %select.true.sink
+; CHECK-NEXT: vldr d3, [sp]
; CHECK-NEXT: vmov d2, r2, r3
; CHECK-NEXT: .LBB3_3: @ %select.end
; CHECK-NEXT: vneg.f16 q1, q1
diff --git a/llvm/test/CodeGen/SPARC/predictable-select.ll b/llvm/test/CodeGen/SPARC/predictable-select.ll
index cf200a121d0f1..a27aa9b0e6408 100644
--- a/llvm/test/CodeGen/SPARC/predictable-select.ll
+++ b/llvm/test/CodeGen/SPARC/predictable-select.ll
@@ -4,45 +4,37 @@
; RUN: llc -O3 < %s -relocation-model=pic -mtriple=sparc -mcpu=v9 -mattr=+no-predictor | FileCheck --check-prefix=SPARC-NO-PREDICTOR %s
; RUN: llc -O3 < %s -relocation-model=pic -mtriple=sparcv9 -mattr=+no-predictor | FileCheck --check-prefix=SPARC64-NO-PREDICTOR %s
-;; Normally, highly predictable selects should be turned into branches.
-;; On the other hand, early Niagara processors should prefer conditional moves
-;; over branches even when it's predictable.
+;; Test select lowering for SPARC targets.
+;; For this case (sdiv by constant), conditional moves are used since the
+;; division is strength-reduced to cheap multiply/shift operations.
define i32 @cdiv(i32 %cond, i32 %num) #0 {
; SPARC-LABEL: cdiv:
; SPARC: ! %bb.0: ! %entry
+; SPARC-NEXT: sethi 1398101, %o2
+; SPARC-NEXT: or %o2, 342, %o2
+; SPARC-NEXT: smul %o1, %o2, %o2
+; SPARC-NEXT: rd %y, %o2
+; SPARC-NEXT: srl %o2, 31, %o3
+; SPARC-NEXT: add %o2, %o3, %o2
; SPARC-NEXT: cmp %o0, 0
-; SPARC-NEXT: be %icc, .LBB0_2
-; SPARC-NEXT: mov %o1, %o0
-; SPARC-NEXT: ! %bb.1: ! %select.end
-; SPARC-NEXT: retl
-; SPARC-NEXT: nop
-; SPARC-NEXT: .LBB0_2: ! %select.true.sink
-; SPARC-NEXT: sethi 1398101, %o1
-; SPARC-NEXT: or %o1, 342, %o1
-; SPARC-NEXT: smul %o0, %o1, %o0
-; SPARC-NEXT: rd %y, %o0
-; SPARC-NEXT: srl %o0, 31, %o1
+; SPARC-NEXT: move %icc, %o2, %o1
; SPARC-NEXT: retl
-; SPARC-NEXT: add %o0, %o1, %o0
+; SPARC-NEXT: mov %o1, %o0
;
; SPARC64-LABEL: cdiv:
; SPARC64: ! %bb.0: ! %entry
+; SPARC64-NEXT: sra %o1, 0, %o2
+; SPARC64-NEXT: sethi 1398101, %o3
+; SPARC64-NEXT: or %o3, 342, %o3
+; SPARC64-NEXT: mulx %o2, %o3, %o2
+; SPARC64-NEXT: srlx %o2, 63, %o3
+; SPARC64-NEXT: srlx %o2, 32, %o2
+; SPARC64-NEXT: add %o2, %o3, %o2
; SPARC64-NEXT: cmp %o0, 0
-; SPARC64-NEXT: be %icc, .LBB0_2
-; SPARC64-NEXT: mov %o1, %o0
-; SPARC64-NEXT: ! %bb.1: ! %select.end
-; SPARC64-NEXT: retl
-; SPARC64-NEXT: nop
-; SPARC64-NEXT: .LBB0_2: ! %select.true.sink
-; SPARC64-NEXT: sra %o0, 0, %o0
-; SPARC64-NEXT: sethi 1398101, %o1
-; SPARC64-NEXT: or %o1, 342, %o1
-; SPARC64-NEXT: mulx %o0, %o1, %o0
-; SPARC64-NEXT: srlx %o0, 63, %o1
-; SPARC64-NEXT: srlx %o0, 32, %o0
+; SPARC64-NEXT: move %icc, %o2, %o1
; SPARC64-NEXT: retl
-; SPARC64-NEXT: add %o0, %o1, %o0
+; SPARC64-NEXT: mov %o1, %o0
;
; SPARC-NO-PREDICTOR-LABEL: cdiv:
; SPARC-NO-PREDICTOR: ! %bb.0: ! %entry
>From b005c129dfefdec8556ed393c2bff369c642907e Mon Sep 17 00:00:00 2001
From: Ayush Rai <ayrai at amd.com>
Date: Tue, 21 Jul 2026 14:00:41 +0530
Subject: [PATCH 4/8] [CodeGenPrepare] Add latency-based select sink heuristic
---
llvm/lib/CodeGen/CodeGenPrepare.cpp | 156 +++++++++++++++-------------
1 file changed, 81 insertions(+), 75 deletions(-)
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index b858e9598b030..58a0849f60145 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -7622,75 +7622,89 @@ bool CodeGenPrepare::optimizeLoadExt(LoadInst *Load) {
return true;
}
-/// Recursively check if V or any instruction in its sinkable chain is
-/// expensive. Instructions are sinkable when all of their uses are accounted
-/// for on one side (true/false) of the select group. RemainingUses tracks
-/// unaccounted uses for multi-use instructions on that side. When Chain is
-/// non-null, sinkable instructions are collected in pre-order (users before
-/// operands); the caller must reverse the chain to get the correct sinking
-/// order.
-static bool sinkSelectOperand1(const TargetTransformInfo *TTI, Value *V,
- DenseMap<Instruction *, unsigned> &RemainingUses,
- SmallVectorImpl<Instruction *> *Chain = nullptr,
- unsigned Depth = 4) {
+static bool isLatencyExpensiveToSpeculate(const TargetTransformInfo *TTI,
+ const Instruction *I) {
+ InstructionCost InstLat =
+ TTI->getInstructionCost(I, TargetTransformInfo::TCK_Latency);
+ InstructionCost BrPenalty = TTI->getBranchMispredictPenalty();
+
+ if (!InstLat.isValid() || !BrPenalty.isValid() || BrPenalty.getValue() == 0)
+ return TTI->isExpensiveToSpeculativelyExecute(I);
+
+ return InstLat >= BrPenalty;
+}
+
+static constexpr int SinkSelectChainMaxDepth = 8;
+
+static bool collectSinkableChain(const TargetTransformInfo *TTI, Value *V,
+ BasicBlock *StartBlock, Value *SharedCond,
+ SmallVectorImpl<Instruction *> &Chain,
+ int Depth) {
auto *I = dyn_cast<Instruction>(V);
- if (!I || Depth == 0)
+ if (!I || Depth <= 0)
+ return false;
+ if (I->getParent() != StartBlock)
+ return false;
+ if (auto *Inner = dyn_cast<SelectInst>(I))
+ if (Inner->getCondition() == SharedCond)
+ return false;
+ if (!I->hasOneUse())
return false;
if (!isSafeToSpeculativelyExecute(I))
return false;
- if (!I->hasOneUse()) {
- auto [It, _] = RemainingUses.try_emplace(I, I->getNumUses());
- assert(It->second > 0 && "this operand has atleast one use.");
- --It->second;
- if (It->second != 0)
- return false;
+ bool HasExpensive = isLatencyExpensiveToSpeculate(TTI, I);
+ for (Value *Op : I->operands()) {
+ if (collectSinkableChain(TTI, Op, StartBlock, SharedCond, Chain, Depth - 1))
+ HasExpensive = true;
}
- if (Chain)
- Chain->push_back(I);
-
- bool HasExpensive = TTI->isExpensiveToSpeculativelyExecute(I);
- // Integer div/rem by a constant will be lowered to mul+shift by ISel,
- // not an actual idiv. Don't treat it as expensive to speculatively execute.
- if (HasExpensive) {
- unsigned Op = I->getOpcode();
- if ((Op == Instruction::SDiv || Op == Instruction::UDiv ||
- Op == Instruction::SRem || Op == Instruction::URem) &&
- isa<ConstantInt>(I->getOperand(1)))
- HasExpensive = false;
- }
if (HasExpensive) {
- LLVM_DEBUG(dbgs() << "CGP: Found expensive instruction to sink: " << *I
- << "\n");
- if (!Chain)
- return true;
- }
- for (Value *Op : I->operands()) {
- if (sinkSelectOperand1(TTI, Op, RemainingUses, Chain, Depth - 1)) {
- HasExpensive = true;
- if (!Chain)
- return true;
- }
+ Chain.push_back(I);
+ LLVM_DEBUG(dbgs() << "CGP: Collected instruction to sink: " << *I << "\n");
}
return HasExpensive;
}
-/// Returns true if a SelectInst should be turned into an explicit branch.
+struct SelectSinkPlan {
+ SmallVector<Instruction *, 4> TrueChain;
+ SmallVector<Instruction *, 4> FalseChain;
+ bool HasExpensive = false;
+};
+
+static SelectSinkPlan planSelectSink(const TargetTransformInfo *TTI,
+ ArrayRef<SelectInst *> ASI) {
+ SelectSinkPlan Plan;
+ BasicBlock *StartBlock = ASI.front()->getParent();
+ Value *SharedCond = ASI.front()->getCondition();
+
+ InstructionCost BrPenalty = TTI->getBranchMispredictPenalty();
+ int MaxDepth = (BrPenalty.isValid() && BrPenalty.getValue() != 0)
+ ? SinkSelectChainMaxDepth
+ : 1;
+
+ for (SelectInst *SI : ASI) {
+ if (collectSinkableChain(TTI, SI->getTrueValue(), StartBlock, SharedCond,
+ Plan.TrueChain, MaxDepth))
+ Plan.HasExpensive = true;
+ if (collectSinkableChain(TTI, SI->getFalseValue(), StartBlock, SharedCond,
+ Plan.FalseChain, MaxDepth))
+ Plan.HasExpensive = true;
+ }
+
+ return Plan;
+}
+
static bool isFormingBranchFromSelectProfitable(const TargetTransformInfo *TTI,
const TargetLowering *TLI,
- ArrayRef<SelectInst *> ASI) {
+ ArrayRef<SelectInst *> ASI,
+ const SelectSinkPlan &Plan) {
// If even a predictable select is cheap, then a branch can't be cheaper.
if (!TLI->isPredictableSelectExpensive())
return false;
SelectInst *SI = ASI.front();
- // FIXME: This should use the same heuristics as IfConversion to determine
- // whether a select is better represented as a branch.
-
- // If metadata tells us that the select condition is obviously predictable,
- // then we want to replace the select with a branch.
uint64_t TrueWeight, FalseWeight;
if (extractBranchWeights(*SI, TrueWeight, FalseWeight)) {
uint64_t Max = std::max(TrueWeight, FalseWeight);
@@ -7711,22 +7725,19 @@ static bool isFormingBranchFromSelectProfitable(const TargetTransformInfo *TTI,
// If a branch is predictable, an out-of-order CPU can avoid blocking on its
// comparison condition. If the compare has more than one use, there's
// probably another cmov or setcc around, so it's not worth emitting a branch.
- // For grouped selects, allow exactly one use per select.
- if (!Cmp || !Cmp->hasNUses(ASI.size()))
+ if (!Cmp)
return false;
+ SmallPtrSet<const Value *, 4> ASISet(ASI.begin(), ASI.end());
+ for (const User *U : Cmp->users()) {
+ if (!ASISet.contains(U))
+ return false;
+ }
- // If any operand in the select group is expensive and only needed on one
- // side, we should form a branch. Check recursively through each select's
- // operand chains, using per-side use tracking for multi-use instructions.
- DenseMap<Instruction *, unsigned> TrueRemainingUses, FalseRemainingUses;
- for (SelectInst *SI : ASI) {
- if (sinkSelectOperand1(TTI, SI->getTrueValue(), TrueRemainingUses) ||
- sinkSelectOperand1(TTI, SI->getFalseValue(), FalseRemainingUses)) {
- LLVM_DEBUG(dbgs() << "CGP: Forming branch from select due to expensive "
- "operand: "
- << *SI << "\n");
- return true;
- }
+ if (Plan.HasExpensive) {
+ LLVM_DEBUG(dbgs() << "CGP: Forming branch from select due to expensive "
+ "operand in group of "
+ << ASI.size() << " selects\n");
+ return true;
}
return false;
@@ -7865,8 +7876,10 @@ bool CodeGenPrepare::optimizeSelectInst(SelectInst *SI) {
else
SelectKind = TargetLowering::ScalarValSelect;
+ SelectSinkPlan Plan = planSelectSink(TTI, ASI);
+
if (TLI->isSelectSupported(SelectKind) &&
- (!isFormingBranchFromSelectProfitable(TTI, TLI, ASI) ||
+ (!isFormingBranchFromSelectProfitable(TTI, TLI, ASI, Plan) ||
llvm::shouldOptimizeForSize(SI->getParent(), PSI, BFI)))
return false;
@@ -7898,15 +7911,8 @@ bool CodeGenPrepare::optimizeSelectInst(SelectInst *SI) {
// but we don't set a ModifiedDT flag to avoid restarting the function walk in
// runOnFunction for each select optimized.
- // Collect values that go on the true side and the values that go on the false
- // side.
- SmallVector<Instruction *> TrueInstrs, FalseInstrs;
- DenseMap<Instruction *, unsigned> TrueRemainingUses, FalseRemainingUses;
- for (SelectInst *SI : ASI) {
- sinkSelectOperand1(TTI, SI->getTrueValue(), TrueRemainingUses, &TrueInstrs);
- sinkSelectOperand1(TTI, SI->getFalseValue(), FalseRemainingUses,
- &FalseInstrs);
- }
+ SmallVectorImpl<Instruction *> &TrueInstrs = Plan.TrueChain;
+ SmallVectorImpl<Instruction *> &FalseInstrs = Plan.FalseChain;
// Split the select block, according to how many (if any) values go on each
// side.
@@ -7969,9 +7975,9 @@ bool CodeGenPrepare::optimizeSelectInst(SelectInst *SI) {
// Sink expensive instructions into the conditional blocks to avoid executing
// them speculatively.
- for (Instruction *I : llvm::reverse(TrueInstrs))
+ for (Instruction *I : TrueInstrs)
I->moveBefore(TrueBranch->getIterator());
- for (Instruction *I : llvm::reverse(FalseInstrs))
+ for (Instruction *I : FalseInstrs)
I->moveBefore(FalseBranch->getIterator());
// If we did not create a new block for one of the 'true' or 'false' paths
>From 1b09ffcc506238875ddd8ef479aaebd8207bc9a9 Mon Sep 17 00:00:00 2001
From: Ayush Rai <ayrai at amd.com>
Date: Tue, 21 Jul 2026 14:01:03 +0530
Subject: [PATCH 5/8] [X86] Use sched model branch mispredict penalty
---
llvm/lib/Target/X86/X86TargetTransformInfo.cpp | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 2ecc48cd37b42..2936dbcd4ee80 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -8036,8 +8036,10 @@ InstructionCost X86TTIImpl::getScalingFactorCost(Type *Ty, GlobalValue *BaseGV,
}
InstructionCost X86TTIImpl::getBranchMispredictPenalty() const {
- // TODO: Hook MispredictPenalty of SchedMachineModel into this.
- return 14;
+ unsigned Penalty = ST->getSchedModel().MispredictPenalty;
+ if (!Penalty)
+ return 14;
+ return Penalty;
}
bool X86TTIImpl::isVectorShiftByScalarCheap(Type *Ty) const {
>From 84be44c8adaaf411d97695754a1c19be3eb9d8cd Mon Sep 17 00:00:00 2001
From: Ayush Rai <ayrai at amd.com>
Date: Tue, 21 Jul 2026 15:59:44 +0530
Subject: [PATCH 6/8] [test] Revert stale select-sink expectations
---
.../CodeGen/ARM/fadd-select-fneg-combine.ll | 6 +-
llvm/test/CodeGen/SPARC/predictable-select.ll | 48 +-
llvm/test/CodeGen/X86/bit-manip-i128.ll | 93 +-
llvm/test/CodeGen/X86/bit-manip-i256.ll | 1396 +++++------
llvm/test/CodeGen/X86/bit-manip-i512.ll | 2160 +++++++----------
llvm/test/CodeGen/X86/bsr.ll | 157 +-
llvm/test/CodeGen/X86/freeze-unary.ll | 9 +-
llvm/test/CodeGen/X86/pseudo_cmov_lower.ll | 20 +-
llvm/test/CodeGen/X86/pseudo_cmov_lower2.ll | 21 +-
.../CodeGen/X86/shrink-wrap-chkstk-x86_64.ll | 25 +-
.../CodeGenPrepare/X86/optimizeSelect-DT.ll | 17 +-
.../Transforms/CodeGenPrepare/X86/select.ll | 32 +-
12 files changed, 1714 insertions(+), 2270 deletions(-)
diff --git a/llvm/test/CodeGen/ARM/fadd-select-fneg-combine.ll b/llvm/test/CodeGen/ARM/fadd-select-fneg-combine.ll
index 145beefdf2a96..90bb02fdc0cd4 100644
--- a/llvm/test/CodeGen/ARM/fadd-select-fneg-combine.ll
+++ b/llvm/test/CodeGen/ARM/fadd-select-fneg-combine.ll
@@ -50,8 +50,7 @@ define <2 x float> @fadd_select_fneg_fneg_v2f32(i32 %arg0, <2 x float> %x, <2 x
; CHECK-NEXT: add r0, sp, #8
; CHECK-NEXT: vldrw.u32 q1, [r0]
; CHECK-NEXT: b .LBB2_3
-; CHECK-NEXT: .LBB2_2: @ %select.true.sink
-; CHECK-NEXT: vldr d3, [sp]
+; CHECK-NEXT: .LBB2_2:
; CHECK-NEXT: vmov d2, r2, r3
; CHECK-NEXT: .LBB2_3: @ %select.end
; CHECK-NEXT: vneg.f32 q1, q1
@@ -78,8 +77,7 @@ define <2 x half> @fadd_select_fneg_fneg_v2f16(i32 %arg0, <2 x half> %x, <2 x ha
; CHECK-NEXT: add r0, sp, #8
; CHECK-NEXT: vldrw.u32 q1, [r0]
; CHECK-NEXT: b .LBB3_3
-; CHECK-NEXT: .LBB3_2: @ %select.true.sink
-; CHECK-NEXT: vldr d3, [sp]
+; CHECK-NEXT: .LBB3_2:
; CHECK-NEXT: vmov d2, r2, r3
; CHECK-NEXT: .LBB3_3: @ %select.end
; CHECK-NEXT: vneg.f16 q1, q1
diff --git a/llvm/test/CodeGen/SPARC/predictable-select.ll b/llvm/test/CodeGen/SPARC/predictable-select.ll
index a27aa9b0e6408..cf200a121d0f1 100644
--- a/llvm/test/CodeGen/SPARC/predictable-select.ll
+++ b/llvm/test/CodeGen/SPARC/predictable-select.ll
@@ -4,37 +4,45 @@
; RUN: llc -O3 < %s -relocation-model=pic -mtriple=sparc -mcpu=v9 -mattr=+no-predictor | FileCheck --check-prefix=SPARC-NO-PREDICTOR %s
; RUN: llc -O3 < %s -relocation-model=pic -mtriple=sparcv9 -mattr=+no-predictor | FileCheck --check-prefix=SPARC64-NO-PREDICTOR %s
-;; Test select lowering for SPARC targets.
-;; For this case (sdiv by constant), conditional moves are used since the
-;; division is strength-reduced to cheap multiply/shift operations.
+;; Normally, highly predictable selects should be turned into branches.
+;; On the other hand, early Niagara processors should prefer conditional moves
+;; over branches even when it's predictable.
define i32 @cdiv(i32 %cond, i32 %num) #0 {
; SPARC-LABEL: cdiv:
; SPARC: ! %bb.0: ! %entry
-; SPARC-NEXT: sethi 1398101, %o2
-; SPARC-NEXT: or %o2, 342, %o2
-; SPARC-NEXT: smul %o1, %o2, %o2
-; SPARC-NEXT: rd %y, %o2
-; SPARC-NEXT: srl %o2, 31, %o3
-; SPARC-NEXT: add %o2, %o3, %o2
; SPARC-NEXT: cmp %o0, 0
-; SPARC-NEXT: move %icc, %o2, %o1
-; SPARC-NEXT: retl
+; SPARC-NEXT: be %icc, .LBB0_2
; SPARC-NEXT: mov %o1, %o0
+; SPARC-NEXT: ! %bb.1: ! %select.end
+; SPARC-NEXT: retl
+; SPARC-NEXT: nop
+; SPARC-NEXT: .LBB0_2: ! %select.true.sink
+; SPARC-NEXT: sethi 1398101, %o1
+; SPARC-NEXT: or %o1, 342, %o1
+; SPARC-NEXT: smul %o0, %o1, %o0
+; SPARC-NEXT: rd %y, %o0
+; SPARC-NEXT: srl %o0, 31, %o1
+; SPARC-NEXT: retl
+; SPARC-NEXT: add %o0, %o1, %o0
;
; SPARC64-LABEL: cdiv:
; SPARC64: ! %bb.0: ! %entry
-; SPARC64-NEXT: sra %o1, 0, %o2
-; SPARC64-NEXT: sethi 1398101, %o3
-; SPARC64-NEXT: or %o3, 342, %o3
-; SPARC64-NEXT: mulx %o2, %o3, %o2
-; SPARC64-NEXT: srlx %o2, 63, %o3
-; SPARC64-NEXT: srlx %o2, 32, %o2
-; SPARC64-NEXT: add %o2, %o3, %o2
; SPARC64-NEXT: cmp %o0, 0
-; SPARC64-NEXT: move %icc, %o2, %o1
-; SPARC64-NEXT: retl
+; SPARC64-NEXT: be %icc, .LBB0_2
; SPARC64-NEXT: mov %o1, %o0
+; SPARC64-NEXT: ! %bb.1: ! %select.end
+; SPARC64-NEXT: retl
+; SPARC64-NEXT: nop
+; SPARC64-NEXT: .LBB0_2: ! %select.true.sink
+; SPARC64-NEXT: sra %o0, 0, %o0
+; SPARC64-NEXT: sethi 1398101, %o1
+; SPARC64-NEXT: or %o1, 342, %o1
+; SPARC64-NEXT: mulx %o0, %o1, %o0
+; SPARC64-NEXT: srlx %o0, 63, %o1
+; SPARC64-NEXT: srlx %o0, 32, %o0
+; SPARC64-NEXT: retl
+; SPARC64-NEXT: add %o0, %o1, %o0
;
; SPARC-NO-PREDICTOR-LABEL: cdiv:
; SPARC-NO-PREDICTOR: ! %bb.0: ! %entry
diff --git a/llvm/test/CodeGen/X86/bit-manip-i128.ll b/llvm/test/CodeGen/X86/bit-manip-i128.ll
index 150c3486a6091..451ce3b1b7578 100644
--- a/llvm/test/CodeGen/X86/bit-manip-i128.ll
+++ b/llvm/test/CodeGen/X86/bit-manip-i128.ll
@@ -901,10 +901,6 @@ define i128 @bzhi_i128_load(ptr %p0, i128 %idx) nounwind {
define i128 @isolate_msb_i128(i128 %a0, i128 %idx) nounwind {
; SSE-LABEL: isolate_msb_i128:
; SSE: # %bb.0:
-; SSE-NEXT: movq %rdi, %rax
-; SSE-NEXT: orq %rsi, %rax
-; SSE-NEXT: je .LBB15_1
-; SSE-NEXT: # %bb.2: # %select.false.sink
; SSE-NEXT: bsrq %rsi, %rax
; SSE-NEXT: xorl $63, %eax
; SSE-NEXT: bsrq %rdi, %rcx
@@ -916,15 +912,14 @@ define i128 @isolate_msb_i128(i128 %a0, i128 %idx) nounwind {
; SSE-NEXT: movabsq $-9223372036854775808, %rdx # imm = 0x8000000000000000
; SSE-NEXT: xorl %eax, %eax
; SSE-NEXT: shrdq %cl, %rdx, %rax
-; SSE-NEXT: xorl %esi, %esi
+; SSE-NEXT: xorl %r8d, %r8d
; SSE-NEXT: shrq %cl, %rdx
; SSE-NEXT: testb $64, %cl
; SSE-NEXT: cmovneq %rdx, %rax
-; SSE-NEXT: cmovneq %rsi, %rdx
-; SSE-NEXT: retq
-; SSE-NEXT: .LBB15_1:
-; SSE-NEXT: xorl %eax, %eax
-; SSE-NEXT: xorl %edx, %edx
+; SSE-NEXT: cmovneq %r8, %rdx
+; SSE-NEXT: orq %rsi, %rdi
+; SSE-NEXT: cmoveq %r8, %rax
+; SSE-NEXT: cmoveq %r8, %rdx
; SSE-NEXT: retq
;
; AVX-LABEL: isolate_msb_i128:
@@ -957,15 +952,13 @@ define i128 @isolate_msb_i128(i128 %a0, i128 %idx) nounwind {
define i128 @isolate_msb_i128_vector(<2 x i64> %v0, i128 %idx) nounwind {
; SSE2-LABEL: isolate_msb_i128_vector:
; SSE2: # %bb.0:
+; SSE2-NEXT: movq %xmm0, %rax
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE2-NEXT: movq %xmm1, %rcx
; SSE2-NEXT: pxor %xmm1, %xmm1
; SSE2-NEXT: pcmpeqd %xmm0, %xmm1
-; SSE2-NEXT: movmskps %xmm1, %eax
-; SSE2-NEXT: xorl $15, %eax
-; SSE2-NEXT: je .LBB16_1
-; SSE2-NEXT: # %bb.2: # %select.false.sink
-; SSE2-NEXT: movq %xmm0, %rax
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
-; SSE2-NEXT: movq %xmm0, %rcx
+; SSE2-NEXT: movmskps %xmm1, %esi
+; SSE2-NEXT: xorl $15, %esi
; SSE2-NEXT: bsrq %rcx, %rdx
; SSE2-NEXT: xorl $63, %edx
; SSE2-NEXT: bsrq %rax, %rax
@@ -977,31 +970,27 @@ define i128 @isolate_msb_i128_vector(<2 x i64> %v0, i128 %idx) nounwind {
; SSE2-NEXT: movabsq $-9223372036854775808, %rdx # imm = 0x8000000000000000
; SSE2-NEXT: xorl %eax, %eax
; SSE2-NEXT: shrdq %cl, %rdx, %rax
-; SSE2-NEXT: xorl %esi, %esi
+; SSE2-NEXT: xorl %edi, %edi
; SSE2-NEXT: shrq %cl, %rdx
; SSE2-NEXT: testb $64, %cl
; SSE2-NEXT: cmovneq %rdx, %rax
-; SSE2-NEXT: cmovneq %rsi, %rdx
-; SSE2-NEXT: retq
-; SSE2-NEXT: .LBB16_1:
-; SSE2-NEXT: xorl %eax, %eax
-; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: cmovneq %rdi, %rdx
+; SSE2-NEXT: testl %esi, %esi
+; SSE2-NEXT: cmoveq %rdi, %rax
+; SSE2-NEXT: cmoveq %rdi, %rdx
; SSE2-NEXT: retq
;
; SSE42-LABEL: isolate_msb_i128_vector:
; SSE42: # %bb.0:
-; SSE42-NEXT: ptest %xmm0, %xmm0
-; SSE42-NEXT: je .LBB16_1
-; SSE42-NEXT: # %bb.2: # %select.false.sink
-; SSE42-NEXT: pextrq $1, %xmm0, %rax
-; SSE42-NEXT: movq %xmm0, %rcx
-; SSE42-NEXT: bsrq %rax, %rdx
+; SSE42-NEXT: movq %xmm0, %rax
+; SSE42-NEXT: pextrq $1, %xmm0, %rcx
+; SSE42-NEXT: bsrq %rcx, %rdx
; SSE42-NEXT: xorl $63, %edx
-; SSE42-NEXT: bsrq %rcx, %rcx
-; SSE42-NEXT: xorl $63, %ecx
-; SSE42-NEXT: orb $64, %cl
-; SSE42-NEXT: testq %rax, %rax
-; SSE42-NEXT: movzbl %cl, %ecx
+; SSE42-NEXT: bsrq %rax, %rax
+; SSE42-NEXT: xorl $63, %eax
+; SSE42-NEXT: orb $64, %al
+; SSE42-NEXT: testq %rcx, %rcx
+; SSE42-NEXT: movzbl %al, %ecx
; SSE42-NEXT: cmovnel %edx, %ecx
; SSE42-NEXT: movabsq $-9223372036854775808, %rdx # imm = 0x8000000000000000
; SSE42-NEXT: xorl %eax, %eax
@@ -1011,10 +1000,9 @@ define i128 @isolate_msb_i128_vector(<2 x i64> %v0, i128 %idx) nounwind {
; SSE42-NEXT: testb $64, %cl
; SSE42-NEXT: cmovneq %rdx, %rax
; SSE42-NEXT: cmovneq %rsi, %rdx
-; SSE42-NEXT: retq
-; SSE42-NEXT: .LBB16_1:
-; SSE42-NEXT: xorl %eax, %eax
-; SSE42-NEXT: xorl %edx, %edx
+; SSE42-NEXT: ptest %xmm0, %xmm0
+; SSE42-NEXT: cmoveq %rsi, %rax
+; SSE42-NEXT: cmoveq %rsi, %rdx
; SSE42-NEXT: retq
;
; AVX-LABEL: isolate_msb_i128_vector:
@@ -1050,32 +1038,27 @@ define i128 @isolate_msb_i128_vector(<2 x i64> %v0, i128 %idx) nounwind {
define i128 @isolate_msb_i128_load(ptr %p0, i128 %idx) nounwind {
; SSE-LABEL: isolate_msb_i128_load:
; SSE: # %bb.0:
-; SSE-NEXT: movq (%rdi), %rcx
-; SSE-NEXT: movq 8(%rdi), %rax
-; SSE-NEXT: movq %rcx, %rdx
-; SSE-NEXT: orq %rax, %rdx
-; SSE-NEXT: je .LBB17_1
-; SSE-NEXT: # %bb.2: # %select.false.sink
-; SSE-NEXT: bsrq %rax, %rdx
-; SSE-NEXT: xorl $63, %edx
-; SSE-NEXT: bsrq %rcx, %rcx
+; SSE-NEXT: movq (%rdi), %rsi
+; SSE-NEXT: movq 8(%rdi), %rdi
+; SSE-NEXT: bsrq %rdi, %rax
+; SSE-NEXT: xorl $63, %eax
+; SSE-NEXT: bsrq %rsi, %rcx
; SSE-NEXT: xorl $63, %ecx
; SSE-NEXT: orb $64, %cl
-; SSE-NEXT: testq %rax, %rax
+; SSE-NEXT: testq %rdi, %rdi
; SSE-NEXT: movzbl %cl, %ecx
-; SSE-NEXT: cmovnel %edx, %ecx
+; SSE-NEXT: cmovnel %eax, %ecx
; SSE-NEXT: movabsq $-9223372036854775808, %rdx # imm = 0x8000000000000000
; SSE-NEXT: xorl %eax, %eax
; SSE-NEXT: shrdq %cl, %rdx, %rax
-; SSE-NEXT: xorl %esi, %esi
+; SSE-NEXT: xorl %r8d, %r8d
; SSE-NEXT: shrq %cl, %rdx
; SSE-NEXT: testb $64, %cl
; SSE-NEXT: cmovneq %rdx, %rax
-; SSE-NEXT: cmovneq %rsi, %rdx
-; SSE-NEXT: retq
-; SSE-NEXT: .LBB17_1:
-; SSE-NEXT: xorl %eax, %eax
-; SSE-NEXT: xorl %edx, %edx
+; SSE-NEXT: cmovneq %r8, %rdx
+; SSE-NEXT: orq %rdi, %rsi
+; SSE-NEXT: cmoveq %r8, %rax
+; SSE-NEXT: cmoveq %r8, %rdx
; SSE-NEXT: retq
;
; AVX-LABEL: isolate_msb_i128_load:
diff --git a/llvm/test/CodeGen/X86/bit-manip-i256.ll b/llvm/test/CodeGen/X86/bit-manip-i256.ll
index 6896cb0e9abf5..e4f5421c6fba9 100644
--- a/llvm/test/CodeGen/X86/bit-manip-i256.ll
+++ b/llvm/test/CodeGen/X86/bit-manip-i256.ll
@@ -2407,288 +2407,267 @@ define i256 @bzhi_i256_load(ptr %p0, i256 %idx) nounwind {
define i256 @isolate_msb_i256(i256 %a0, i256 %idx) nounwind {
; SSE-LABEL: isolate_msb_i256:
; SSE: # %bb.0:
-; SSE-NEXT: movq %rdi, %rax
-; SSE-NEXT: movq %rdx, %rdi
-; SSE-NEXT: orq %r8, %rdi
-; SSE-NEXT: movq %rsi, %r9
-; SSE-NEXT: orq %rcx, %r9
-; SSE-NEXT: orq %rdi, %r9
-; SSE-NEXT: je .LBB15_1
-; SSE-NEXT: # %bb.2: # %select.false.sink
-; SSE-NEXT: bsrq %r8, %rdi
-; SSE-NEXT: xorq $63, %rdi
-; SSE-NEXT: bsrq %rcx, %r9
-; SSE-NEXT: xorq $63, %r9
-; SSE-NEXT: orq $64, %r9
-; SSE-NEXT: testq %r8, %r8
-; SSE-NEXT: cmovneq %rdi, %r9
-; SSE-NEXT: bsrq %rdx, %rdi
-; SSE-NEXT: xorq $63, %rdi
-; SSE-NEXT: movq %rcx, %r10
+; SSE-NEXT: pushq %rbx
+; SSE-NEXT: movq %rcx, %rax
+; SSE-NEXT: movq %rdx, %r9
+; SSE-NEXT: orq %r8, %r9
; SSE-NEXT: bsrq %rsi, %rcx
+; SSE-NEXT: orq %rax, %rsi
+; SSE-NEXT: bsrq %r8, %r10
+; SSE-NEXT: xorq $63, %r10
+; SSE-NEXT: bsrq %rax, %r11
+; SSE-NEXT: xorq $63, %r11
+; SSE-NEXT: orq $64, %r11
+; SSE-NEXT: testq %r8, %r8
+; SSE-NEXT: cmovneq %r10, %r11
+; SSE-NEXT: bsrq %rdx, %r10
+; SSE-NEXT: xorq $63, %r10
; SSE-NEXT: xorq $63, %rcx
; SSE-NEXT: orq $64, %rcx
; SSE-NEXT: testq %rdx, %rdx
-; SSE-NEXT: cmovneq %rdi, %rcx
+; SSE-NEXT: cmovneq %r10, %rcx
; SSE-NEXT: orq $128, %rcx
-; SSE-NEXT: orq %r8, %r10
-; SSE-NEXT: cmovneq %r9, %rcx
+; SSE-NEXT: orq %r8, %rax
+; SSE-NEXT: cmovneq %r11, %rcx
; SSE-NEXT: xorps %xmm0, %xmm0
; SSE-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
; SSE-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; SSE-NEXT: movabsq $-9223372036854775808, %rdx # imm = 0x8000000000000000
-; SSE-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movabsq $-9223372036854775808, %rax # imm = 0x8000000000000000
+; SSE-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
; SSE-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; SSE-NEXT: movl %ecx, %edx
-; SSE-NEXT: shrb $6, %dl
-; SSE-NEXT: movzbl %dl, %edi
+; SSE-NEXT: movl %ecx, %eax
+; SSE-NEXT: shrb $6, %al
+; SSE-NEXT: movzbl %al, %eax
; SSE-NEXT: movq $0, -{{[0-9]+}}(%rsp)
-; SSE-NEXT: movq -48(%rsp,%rdi,8), %rdx
-; SSE-NEXT: movq -56(%rsp,%rdi,8), %r9
-; SSE-NEXT: movq %r9, %rsi
-; SSE-NEXT: shrdq %cl, %rdx, %rsi
-; SSE-NEXT: movq -64(%rsp,%rdi,8), %r10
-; SSE-NEXT: movq %r10, %r8
-; SSE-NEXT: shrdq %cl, %r9, %r8
-; SSE-NEXT: movq -72(%rsp,%rdi,8), %rdi
+; SSE-NEXT: movq -40(%rsp,%rax,8), %rdx
+; SSE-NEXT: movq -48(%rsp,%rax,8), %r8
+; SSE-NEXT: movq %r8, %r10
+; SSE-NEXT: shrdq %cl, %rdx, %r10
+; SSE-NEXT: movq -56(%rsp,%rax,8), %r11
+; SSE-NEXT: movq %r11, %rbx
+; SSE-NEXT: shrdq %cl, %r8, %rbx
+; SSE-NEXT: movq -64(%rsp,%rax,8), %r8
; SSE-NEXT: shrq %cl, %rdx
; SSE-NEXT: # kill: def $cl killed $cl killed $rcx
-; SSE-NEXT: shrdq %cl, %r10, %rdi
-; SSE-NEXT: jmp .LBB15_3
-; SSE-NEXT: .LBB15_1:
-; SSE-NEXT: xorl %edi, %edi
-; SSE-NEXT: xorl %r8d, %r8d
-; SSE-NEXT: xorl %esi, %esi
-; SSE-NEXT: xorl %edx, %edx
-; SSE-NEXT: .LBB15_3: # %select.end
-; SSE-NEXT: movq %rdi, (%rax)
-; SSE-NEXT: movq %r8, 8(%rax)
-; SSE-NEXT: movq %rsi, 16(%rax)
-; SSE-NEXT: movq %rdx, 24(%rax)
+; SSE-NEXT: shrdq %cl, %r11, %r8
+; SSE-NEXT: xorl %ecx, %ecx
+; SSE-NEXT: orq %r9, %rsi
+; SSE-NEXT: cmoveq %rcx, %rbx
+; SSE-NEXT: cmoveq %rcx, %r10
+; SSE-NEXT: cmoveq %rcx, %r8
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: cmoveq %rcx, %rdx
+; SSE-NEXT: movq %rdx, 24(%rdi)
+; SSE-NEXT: movq %r10, 16(%rdi)
+; SSE-NEXT: movq %rbx, 8(%rdi)
+; SSE-NEXT: movq %r8, (%rdi)
+; SSE-NEXT: popq %rbx
; SSE-NEXT: retq
;
; AVX2-LABEL: isolate_msb_i256:
; AVX2: # %bb.0:
-; AVX2-NEXT: movq %rdi, %rax
-; AVX2-NEXT: movq %rdx, %rdi
-; AVX2-NEXT: orq %r8, %rdi
-; AVX2-NEXT: movq %rsi, %r9
-; AVX2-NEXT: orq %rcx, %r9
-; AVX2-NEXT: orq %rdi, %r9
-; AVX2-NEXT: je .LBB15_1
-; AVX2-NEXT: # %bb.2: # %select.false.sink
-; AVX2-NEXT: xorl %edi, %edi
-; AVX2-NEXT: lzcntq %r8, %rdi
-; AVX2-NEXT: xorl %r9d, %r9d
-; AVX2-NEXT: lzcntq %rcx, %r9
-; AVX2-NEXT: addq $64, %r9
-; AVX2-NEXT: testq %r8, %r8
-; AVX2-NEXT: cmovneq %rdi, %r9
-; AVX2-NEXT: xorl %edi, %edi
-; AVX2-NEXT: lzcntq %rdx, %rdi
-; AVX2-NEXT: movq %rcx, %r10
+; AVX2-NEXT: pushq %rbx
+; AVX2-NEXT: movq %rcx, %rax
+; AVX2-NEXT: movq %rdx, %r9
+; AVX2-NEXT: orq %r8, %r9
; AVX2-NEXT: xorl %ecx, %ecx
; AVX2-NEXT: lzcntq %rsi, %rcx
+; AVX2-NEXT: orq %rax, %rsi
+; AVX2-NEXT: lzcntq %r8, %r10
+; AVX2-NEXT: lzcntq %rax, %r11
+; AVX2-NEXT: addq $64, %r11
+; AVX2-NEXT: testq %r8, %r8
+; AVX2-NEXT: cmovneq %r10, %r11
+; AVX2-NEXT: xorl %r10d, %r10d
+; AVX2-NEXT: lzcntq %rdx, %r10
; AVX2-NEXT: addq $64, %rcx
; AVX2-NEXT: testq %rdx, %rdx
-; AVX2-NEXT: cmovneq %rdi, %rcx
+; AVX2-NEXT: cmovneq %r10, %rcx
; AVX2-NEXT: subq $-128, %rcx
-; AVX2-NEXT: orq %r8, %r10
-; AVX2-NEXT: cmovneq %r9, %rcx
+; AVX2-NEXT: orq %r8, %rax
+; AVX2-NEXT: cmovneq %r11, %rcx
; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovaps {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movl %ecx, %edx
-; AVX2-NEXT: shrb $6, %dl
-; AVX2-NEXT: movzbl %dl, %esi
-; AVX2-NEXT: movq -48(%rsp,%rsi,8), %r8
-; AVX2-NEXT: movq -56(%rsp,%rsi,8), %r9
-; AVX2-NEXT: movq %r9, %rdx
-; AVX2-NEXT: shrdq %cl, %r8, %rdx
-; AVX2-NEXT: movq -64(%rsp,%rsi,8), %r10
-; AVX2-NEXT: movq %r10, %rdi
-; AVX2-NEXT: shrdq %cl, %r9, %rdi
-; AVX2-NEXT: movq -72(%rsp,%rsi,8), %rsi
-; AVX2-NEXT: shrxq %rcx, %r8, %r8
-; AVX2-NEXT: # kill: def $cl killed $cl killed $rcx
-; AVX2-NEXT: shrdq %cl, %r10, %rsi
-; AVX2-NEXT: jmp .LBB15_3
-; AVX2-NEXT: .LBB15_1:
-; AVX2-NEXT: xorl %esi, %esi
+; AVX2-NEXT: movl %ecx, %eax
+; AVX2-NEXT: shrb $6, %al
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: movq -40(%rsp,%rax,8), %rdx
+; AVX2-NEXT: movq -48(%rsp,%rax,8), %r8
+; AVX2-NEXT: movq %r8, %r10
+; AVX2-NEXT: shrdq %cl, %rdx, %r10
+; AVX2-NEXT: movq -64(%rsp,%rax,8), %r11
+; AVX2-NEXT: movq -56(%rsp,%rax,8), %rax
+; AVX2-NEXT: movq %rax, %rbx
+; AVX2-NEXT: shrdq %cl, %r8, %rbx
+; AVX2-NEXT: shrdq %cl, %rax, %r11
+; AVX2-NEXT: movq %rdi, %rax
; AVX2-NEXT: xorl %edi, %edi
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: xorl %r8d, %r8d
-; AVX2-NEXT: .LBB15_3: # %select.end
-; AVX2-NEXT: movq %rsi, (%rax)
-; AVX2-NEXT: movq %rdi, 8(%rax)
-; AVX2-NEXT: movq %rdx, 16(%rax)
-; AVX2-NEXT: movq %r8, 24(%rax)
+; AVX2-NEXT: orq %r9, %rsi
+; AVX2-NEXT: shrxq %rcx, %rdx, %rcx
+; AVX2-NEXT: cmoveq %rdi, %rbx
+; AVX2-NEXT: cmoveq %rdi, %r10
+; AVX2-NEXT: cmoveq %rdi, %r11
+; AVX2-NEXT: cmoveq %rdi, %rcx
+; AVX2-NEXT: movq %rcx, 24(%rax)
+; AVX2-NEXT: movq %r10, 16(%rax)
+; AVX2-NEXT: movq %rbx, 8(%rax)
+; AVX2-NEXT: movq %r11, (%rax)
+; AVX2-NEXT: popq %rbx
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512F-LABEL: isolate_msb_i256:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: movq %rdi, %rax
-; AVX512F-NEXT: movq %rdx, %rdi
-; AVX512F-NEXT: orq %r8, %rdi
-; AVX512F-NEXT: movq %rsi, %r9
-; AVX512F-NEXT: orq %rcx, %r9
-; AVX512F-NEXT: orq %rdi, %r9
-; AVX512F-NEXT: je .LBB15_1
-; AVX512F-NEXT: # %bb.2: # %select.false.sink
-; AVX512F-NEXT: lzcntq %r8, %rdi
-; AVX512F-NEXT: lzcntq %rcx, %r9
-; AVX512F-NEXT: addq $64, %r9
-; AVX512F-NEXT: testq %r8, %r8
-; AVX512F-NEXT: cmovneq %rdi, %r9
-; AVX512F-NEXT: lzcntq %rdx, %rdi
-; AVX512F-NEXT: movq %rcx, %r10
+; AVX512F-NEXT: pushq %rbx
+; AVX512F-NEXT: movq %rcx, %rax
+; AVX512F-NEXT: movq %rdx, %r9
+; AVX512F-NEXT: orq %r8, %r9
; AVX512F-NEXT: lzcntq %rsi, %rcx
+; AVX512F-NEXT: orq %rax, %rsi
+; AVX512F-NEXT: lzcntq %r8, %r10
+; AVX512F-NEXT: lzcntq %rax, %r11
+; AVX512F-NEXT: addq $64, %r11
+; AVX512F-NEXT: testq %r8, %r8
+; AVX512F-NEXT: cmovneq %r10, %r11
+; AVX512F-NEXT: lzcntq %rdx, %r10
; AVX512F-NEXT: addq $64, %rcx
; AVX512F-NEXT: testq %rdx, %rdx
-; AVX512F-NEXT: cmovneq %rdi, %rcx
+; AVX512F-NEXT: cmovneq %r10, %rcx
; AVX512F-NEXT: subq $-128, %rcx
-; AVX512F-NEXT: orq %r8, %r10
-; AVX512F-NEXT: cmovneq %r9, %rcx
+; AVX512F-NEXT: orq %r8, %rax
+; AVX512F-NEXT: cmovneq %r11, %rcx
; AVX512F-NEXT: vmovaps {{.*#+}} zmm0 = [0,0,0,9223372036854775808,0,0,0,0]
; AVX512F-NEXT: vmovups %zmm0, -{{[0-9]+}}(%rsp)
-; AVX512F-NEXT: movl %ecx, %edx
-; AVX512F-NEXT: shrb $6, %dl
-; AVX512F-NEXT: movzbl %dl, %esi
-; AVX512F-NEXT: movq -48(%rsp,%rsi,8), %r8
-; AVX512F-NEXT: movq -56(%rsp,%rsi,8), %r9
-; AVX512F-NEXT: movq %r9, %rdx
-; AVX512F-NEXT: shrdq %cl, %r8, %rdx
-; AVX512F-NEXT: movq -64(%rsp,%rsi,8), %r10
-; AVX512F-NEXT: movq %r10, %rdi
-; AVX512F-NEXT: shrdq %cl, %r9, %rdi
-; AVX512F-NEXT: movq -72(%rsp,%rsi,8), %rsi
-; AVX512F-NEXT: shrxq %rcx, %r8, %r8
-; AVX512F-NEXT: # kill: def $cl killed $cl killed $rcx
-; AVX512F-NEXT: shrdq %cl, %r10, %rsi
-; AVX512F-NEXT: jmp .LBB15_3
-; AVX512F-NEXT: .LBB15_1:
-; AVX512F-NEXT: xorl %esi, %esi
+; AVX512F-NEXT: movl %ecx, %eax
+; AVX512F-NEXT: shrb $6, %al
+; AVX512F-NEXT: movzbl %al, %eax
+; AVX512F-NEXT: movq -40(%rsp,%rax,8), %rdx
+; AVX512F-NEXT: movq -48(%rsp,%rax,8), %r8
+; AVX512F-NEXT: movq %r8, %r10
+; AVX512F-NEXT: shrdq %cl, %rdx, %r10
+; AVX512F-NEXT: movq -64(%rsp,%rax,8), %r11
+; AVX512F-NEXT: movq -56(%rsp,%rax,8), %rax
+; AVX512F-NEXT: movq %rax, %rbx
+; AVX512F-NEXT: shrdq %cl, %r8, %rbx
+; AVX512F-NEXT: shrdq %cl, %rax, %r11
+; AVX512F-NEXT: movq %rdi, %rax
; AVX512F-NEXT: xorl %edi, %edi
-; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: xorl %r8d, %r8d
-; AVX512F-NEXT: .LBB15_3: # %select.end
-; AVX512F-NEXT: movq %rsi, (%rax)
-; AVX512F-NEXT: movq %rdi, 8(%rax)
-; AVX512F-NEXT: movq %rdx, 16(%rax)
-; AVX512F-NEXT: movq %r8, 24(%rax)
+; AVX512F-NEXT: orq %r9, %rsi
+; AVX512F-NEXT: shrxq %rcx, %rdx, %rcx
+; AVX512F-NEXT: cmoveq %rdi, %rbx
+; AVX512F-NEXT: cmoveq %rdi, %r10
+; AVX512F-NEXT: cmoveq %rdi, %r11
+; AVX512F-NEXT: cmoveq %rdi, %rcx
+; AVX512F-NEXT: movq %rcx, 24(%rax)
+; AVX512F-NEXT: movq %r10, 16(%rax)
+; AVX512F-NEXT: movq %rbx, 8(%rax)
+; AVX512F-NEXT: movq %r11, (%rax)
+; AVX512F-NEXT: popq %rbx
; AVX512F-NEXT: retq
;
; AVX512VL-LABEL: isolate_msb_i256:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: movq %rdi, %rax
-; AVX512VL-NEXT: movq %rdx, %rdi
-; AVX512VL-NEXT: orq %r8, %rdi
-; AVX512VL-NEXT: movq %rsi, %r9
-; AVX512VL-NEXT: orq %rcx, %r9
-; AVX512VL-NEXT: orq %rdi, %r9
-; AVX512VL-NEXT: je .LBB15_1
-; AVX512VL-NEXT: # %bb.2: # %select.false.sink
-; AVX512VL-NEXT: lzcntq %r8, %rdi
-; AVX512VL-NEXT: lzcntq %rcx, %r9
-; AVX512VL-NEXT: addq $64, %r9
-; AVX512VL-NEXT: testq %r8, %r8
-; AVX512VL-NEXT: cmovneq %rdi, %r9
-; AVX512VL-NEXT: lzcntq %rdx, %rdi
-; AVX512VL-NEXT: movq %rcx, %r10
+; AVX512VL-NEXT: pushq %rbx
+; AVX512VL-NEXT: movq %rcx, %rax
+; AVX512VL-NEXT: movq %rdx, %r9
+; AVX512VL-NEXT: orq %r8, %r9
; AVX512VL-NEXT: lzcntq %rsi, %rcx
+; AVX512VL-NEXT: orq %rax, %rsi
+; AVX512VL-NEXT: lzcntq %r8, %r10
+; AVX512VL-NEXT: lzcntq %rax, %r11
+; AVX512VL-NEXT: addq $64, %r11
+; AVX512VL-NEXT: testq %r8, %r8
+; AVX512VL-NEXT: cmovneq %r10, %r11
+; AVX512VL-NEXT: lzcntq %rdx, %r10
; AVX512VL-NEXT: addq $64, %rcx
; AVX512VL-NEXT: testq %rdx, %rdx
-; AVX512VL-NEXT: cmovneq %rdi, %rcx
+; AVX512VL-NEXT: cmovneq %r10, %rcx
; AVX512VL-NEXT: subq $-128, %rcx
-; AVX512VL-NEXT: orq %r8, %r10
-; AVX512VL-NEXT: cmovneq %r9, %rcx
+; AVX512VL-NEXT: orq %r8, %rax
+; AVX512VL-NEXT: cmovneq %r11, %rcx
; AVX512VL-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: vmovaps {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: movl %ecx, %edx
-; AVX512VL-NEXT: shrb $6, %dl
-; AVX512VL-NEXT: movzbl %dl, %esi
-; AVX512VL-NEXT: movq -48(%rsp,%rsi,8), %r8
-; AVX512VL-NEXT: movq -56(%rsp,%rsi,8), %r9
-; AVX512VL-NEXT: movq %r9, %rdx
-; AVX512VL-NEXT: shrdq %cl, %r8, %rdx
-; AVX512VL-NEXT: movq -64(%rsp,%rsi,8), %r10
-; AVX512VL-NEXT: movq %r10, %rdi
-; AVX512VL-NEXT: shrdq %cl, %r9, %rdi
-; AVX512VL-NEXT: movq -72(%rsp,%rsi,8), %rsi
-; AVX512VL-NEXT: shrxq %rcx, %r8, %r8
-; AVX512VL-NEXT: # kill: def $cl killed $cl killed $rcx
-; AVX512VL-NEXT: shrdq %cl, %r10, %rsi
-; AVX512VL-NEXT: jmp .LBB15_3
-; AVX512VL-NEXT: .LBB15_1:
-; AVX512VL-NEXT: xorl %esi, %esi
+; AVX512VL-NEXT: movl %ecx, %eax
+; AVX512VL-NEXT: shrb $6, %al
+; AVX512VL-NEXT: movzbl %al, %eax
+; AVX512VL-NEXT: movq -40(%rsp,%rax,8), %rdx
+; AVX512VL-NEXT: movq -48(%rsp,%rax,8), %r8
+; AVX512VL-NEXT: movq %r8, %r10
+; AVX512VL-NEXT: shrdq %cl, %rdx, %r10
+; AVX512VL-NEXT: movq -56(%rsp,%rax,8), %r11
+; AVX512VL-NEXT: movq %r11, %rbx
+; AVX512VL-NEXT: shrdq %cl, %r8, %rbx
+; AVX512VL-NEXT: movq -64(%rsp,%rax,8), %r8
+; AVX512VL-NEXT: shrdq %cl, %r11, %r8
+; AVX512VL-NEXT: movq %rdi, %rax
; AVX512VL-NEXT: xorl %edi, %edi
-; AVX512VL-NEXT: xorl %edx, %edx
-; AVX512VL-NEXT: xorl %r8d, %r8d
-; AVX512VL-NEXT: .LBB15_3: # %select.end
-; AVX512VL-NEXT: movq %rsi, (%rax)
-; AVX512VL-NEXT: movq %rdi, 8(%rax)
-; AVX512VL-NEXT: movq %rdx, 16(%rax)
-; AVX512VL-NEXT: movq %r8, 24(%rax)
+; AVX512VL-NEXT: orq %r9, %rsi
+; AVX512VL-NEXT: shrxq %rcx, %rdx, %rcx
+; AVX512VL-NEXT: cmoveq %rdi, %rbx
+; AVX512VL-NEXT: cmoveq %rdi, %r10
+; AVX512VL-NEXT: cmoveq %rdi, %r8
+; AVX512VL-NEXT: cmoveq %rdi, %rcx
+; AVX512VL-NEXT: movq %rcx, 24(%rax)
+; AVX512VL-NEXT: movq %r10, 16(%rax)
+; AVX512VL-NEXT: movq %rbx, 8(%rax)
+; AVX512VL-NEXT: movq %r8, (%rax)
+; AVX512VL-NEXT: popq %rbx
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VBMI-LABEL: isolate_msb_i256:
; AVX512VBMI: # %bb.0:
-; AVX512VBMI-NEXT: movq %rdi, %rax
-; AVX512VBMI-NEXT: movq %rdx, %rdi
-; AVX512VBMI-NEXT: orq %r8, %rdi
-; AVX512VBMI-NEXT: movq %rsi, %r9
-; AVX512VBMI-NEXT: orq %rcx, %r9
-; AVX512VBMI-NEXT: orq %rdi, %r9
-; AVX512VBMI-NEXT: je .LBB15_1
-; AVX512VBMI-NEXT: # %bb.2: # %select.false.sink
-; AVX512VBMI-NEXT: lzcntq %r8, %rdi
-; AVX512VBMI-NEXT: lzcntq %rcx, %r9
-; AVX512VBMI-NEXT: addq $64, %r9
-; AVX512VBMI-NEXT: testq %r8, %r8
-; AVX512VBMI-NEXT: cmovneq %rdi, %r9
-; AVX512VBMI-NEXT: lzcntq %rdx, %rdi
-; AVX512VBMI-NEXT: movq %rcx, %r10
+; AVX512VBMI-NEXT: pushq %rbx
+; AVX512VBMI-NEXT: movq %rcx, %rax
+; AVX512VBMI-NEXT: movq %rdx, %r9
+; AVX512VBMI-NEXT: orq %r8, %r9
; AVX512VBMI-NEXT: lzcntq %rsi, %rcx
+; AVX512VBMI-NEXT: orq %rax, %rsi
+; AVX512VBMI-NEXT: lzcntq %r8, %r10
+; AVX512VBMI-NEXT: lzcntq %rax, %r11
+; AVX512VBMI-NEXT: addq $64, %r11
+; AVX512VBMI-NEXT: testq %r8, %r8
+; AVX512VBMI-NEXT: cmovneq %r10, %r11
+; AVX512VBMI-NEXT: lzcntq %rdx, %r10
; AVX512VBMI-NEXT: addq $64, %rcx
; AVX512VBMI-NEXT: testq %rdx, %rdx
-; AVX512VBMI-NEXT: cmovneq %rdi, %rcx
+; AVX512VBMI-NEXT: cmovneq %r10, %rcx
; AVX512VBMI-NEXT: subq $-128, %rcx
-; AVX512VBMI-NEXT: orq %r8, %r10
-; AVX512VBMI-NEXT: cmovneq %r9, %rcx
+; AVX512VBMI-NEXT: orq %r8, %rax
+; AVX512VBMI-NEXT: cmovneq %r11, %rcx
; AVX512VBMI-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: vmovaps {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: movl %ecx, %edx
-; AVX512VBMI-NEXT: shrb $6, %dl
-; AVX512VBMI-NEXT: movzbl %dl, %esi
-; AVX512VBMI-NEXT: movq -48(%rsp,%rsi,8), %r8
-; AVX512VBMI-NEXT: movq -56(%rsp,%rsi,8), %r9
-; AVX512VBMI-NEXT: movq %r9, %rdx
-; AVX512VBMI-NEXT: shrdq %cl, %r8, %rdx
-; AVX512VBMI-NEXT: movq -64(%rsp,%rsi,8), %r10
-; AVX512VBMI-NEXT: movq %r10, %rdi
-; AVX512VBMI-NEXT: shrdq %cl, %r9, %rdi
-; AVX512VBMI-NEXT: movq -72(%rsp,%rsi,8), %rsi
-; AVX512VBMI-NEXT: shrxq %rcx, %r8, %r8
-; AVX512VBMI-NEXT: # kill: def $cl killed $cl killed $rcx
-; AVX512VBMI-NEXT: shrdq %cl, %r10, %rsi
-; AVX512VBMI-NEXT: jmp .LBB15_3
-; AVX512VBMI-NEXT: .LBB15_1:
-; AVX512VBMI-NEXT: xorl %esi, %esi
+; AVX512VBMI-NEXT: movl %ecx, %eax
+; AVX512VBMI-NEXT: shrb $6, %al
+; AVX512VBMI-NEXT: movzbl %al, %eax
+; AVX512VBMI-NEXT: movq -40(%rsp,%rax,8), %rdx
+; AVX512VBMI-NEXT: movq -48(%rsp,%rax,8), %r8
+; AVX512VBMI-NEXT: movq %r8, %r10
+; AVX512VBMI-NEXT: shrdq %cl, %rdx, %r10
+; AVX512VBMI-NEXT: movq -56(%rsp,%rax,8), %r11
+; AVX512VBMI-NEXT: movq %r11, %rbx
+; AVX512VBMI-NEXT: shrdq %cl, %r8, %rbx
+; AVX512VBMI-NEXT: movq -64(%rsp,%rax,8), %r8
+; AVX512VBMI-NEXT: shrdq %cl, %r11, %r8
+; AVX512VBMI-NEXT: movq %rdi, %rax
; AVX512VBMI-NEXT: xorl %edi, %edi
-; AVX512VBMI-NEXT: xorl %edx, %edx
-; AVX512VBMI-NEXT: xorl %r8d, %r8d
-; AVX512VBMI-NEXT: .LBB15_3: # %select.end
-; AVX512VBMI-NEXT: movq %rsi, (%rax)
-; AVX512VBMI-NEXT: movq %rdi, 8(%rax)
-; AVX512VBMI-NEXT: movq %rdx, 16(%rax)
-; AVX512VBMI-NEXT: movq %r8, 24(%rax)
+; AVX512VBMI-NEXT: orq %r9, %rsi
+; AVX512VBMI-NEXT: shrxq %rcx, %rdx, %rcx
+; AVX512VBMI-NEXT: cmoveq %rdi, %rbx
+; AVX512VBMI-NEXT: cmoveq %rdi, %r10
+; AVX512VBMI-NEXT: cmoveq %rdi, %r8
+; AVX512VBMI-NEXT: cmoveq %rdi, %rcx
+; AVX512VBMI-NEXT: movq %rcx, 24(%rax)
+; AVX512VBMI-NEXT: movq %r10, 16(%rax)
+; AVX512VBMI-NEXT: movq %rbx, 8(%rax)
+; AVX512VBMI-NEXT: movq %r8, (%rax)
+; AVX512VBMI-NEXT: popq %rbx
; AVX512VBMI-NEXT: vzeroupper
; AVX512VBMI-NEXT: retq
%eqz = icmp eq i256 %a0, 0
@@ -2702,38 +2681,36 @@ define i256 @isolate_msb_i256(i256 %a0, i256 %idx) nounwind {
define i256 @isolate_msb_i256_vector(<4 x i64> %v0, i256 %idx) nounwind {
; SSE2-LABEL: isolate_msb_i256_vector:
; SSE2: # %bb.0:
-; SSE2-NEXT: movq %rdi, %rax
-; SSE2-NEXT: movdqa %xmm0, %xmm3
-; SSE2-NEXT: por %xmm1, %xmm3
-; SSE2-NEXT: pxor %xmm2, %xmm2
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm3
-; SSE2-NEXT: movmskps %xmm3, %ecx
-; SSE2-NEXT: xorl $15, %ecx
-; SSE2-NEXT: je .LBB16_1
-; SSE2-NEXT: # %bb.2: # %select.false.sink
-; SSE2-NEXT: movq %xmm1, %rdx
; SSE2-NEXT: movq %xmm0, %rcx
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
+; SSE2-NEXT: movq %xmm2, %rdx
; SSE2-NEXT: movq %xmm1, %rsi
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
-; SSE2-NEXT: movq %xmm0, %rdi
-; SSE2-NEXT: bsrq %rsi, %r8
-; SSE2-NEXT: xorq $63, %r8
-; SSE2-NEXT: bsrq %rdx, %r9
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; SSE2-NEXT: movq %xmm2, %r8
+; SSE2-NEXT: por %xmm1, %xmm0
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: pcmpeqd %xmm2, %xmm0
+; SSE2-NEXT: movmskps %xmm0, %eax
+; SSE2-NEXT: xorl $15, %eax
+; SSE2-NEXT: bsrq %r8, %r9
; SSE2-NEXT: xorq $63, %r9
-; SSE2-NEXT: orq $64, %r9
-; SSE2-NEXT: testq %rsi, %rsi
-; SSE2-NEXT: cmovneq %r8, %r9
-; SSE2-NEXT: bsrq %rdi, %r8
+; SSE2-NEXT: bsrq %rsi, %rsi
+; SSE2-NEXT: xorq $63, %rsi
+; SSE2-NEXT: orq $64, %rsi
+; SSE2-NEXT: testq %r8, %r8
+; SSE2-NEXT: cmovneq %r9, %rsi
+; SSE2-NEXT: bsrq %rdx, %r8
; SSE2-NEXT: xorq $63, %r8
; SSE2-NEXT: bsrq %rcx, %rcx
; SSE2-NEXT: xorq $63, %rcx
; SSE2-NEXT: orq $64, %rcx
-; SSE2-NEXT: testq %rdi, %rdi
+; SSE2-NEXT: testq %rdx, %rdx
; SSE2-NEXT: cmovneq %r8, %rcx
; SSE2-NEXT: orq $128, %rcx
-; SSE2-NEXT: orq %rsi, %rdx
-; SSE2-NEXT: cmovneq %r9, %rcx
+; SSE2-NEXT: pcmpeqd %xmm2, %xmm1
+; SSE2-NEXT: movmskps %xmm1, %edx
+; SSE2-NEXT: xorl $15, %edx
+; SSE2-NEXT: cmovneq %rsi, %rcx
; SSE2-NEXT: movdqa %xmm2, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: movdqa %xmm2, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: movabsq $-9223372036854775808, %rdx # imm = 0x8000000000000000
@@ -2741,316 +2718,257 @@ define i256 @isolate_msb_i256_vector(<4 x i64> %v0, i256 %idx) nounwind {
; SSE2-NEXT: movdqa %xmm2, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: movl %ecx, %edx
; SSE2-NEXT: shrb $6, %dl
-; SSE2-NEXT: movzbl %dl, %edi
+; SSE2-NEXT: movzbl %dl, %esi
; SSE2-NEXT: movq $0, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movq -48(%rsp,%rdi,8), %rdx
-; SSE2-NEXT: movq -56(%rsp,%rdi,8), %r9
-; SSE2-NEXT: movq %r9, %rsi
-; SSE2-NEXT: shrdq %cl, %rdx, %rsi
-; SSE2-NEXT: movq -64(%rsp,%rdi,8), %r10
-; SSE2-NEXT: movq %r10, %r8
-; SSE2-NEXT: shrdq %cl, %r9, %r8
-; SSE2-NEXT: movq -72(%rsp,%rdi,8), %rdi
+; SSE2-NEXT: movq -48(%rsp,%rsi,8), %rdx
+; SSE2-NEXT: movq -56(%rsp,%rsi,8), %r8
+; SSE2-NEXT: movq %r8, %r9
+; SSE2-NEXT: shrdq %cl, %rdx, %r9
+; SSE2-NEXT: movq -64(%rsp,%rsi,8), %r10
+; SSE2-NEXT: movq %r10, %r11
+; SSE2-NEXT: shrdq %cl, %r8, %r11
+; SSE2-NEXT: movq -72(%rsp,%rsi,8), %rsi
; SSE2-NEXT: shrq %cl, %rdx
; SSE2-NEXT: # kill: def $cl killed $cl killed $rcx
-; SSE2-NEXT: shrdq %cl, %r10, %rdi
-; SSE2-NEXT: jmp .LBB16_3
-; SSE2-NEXT: .LBB16_1:
-; SSE2-NEXT: xorl %edi, %edi
-; SSE2-NEXT: xorl %r8d, %r8d
-; SSE2-NEXT: xorl %esi, %esi
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: .LBB16_3: # %select.end
-; SSE2-NEXT: movq %rdi, (%rax)
-; SSE2-NEXT: movq %r8, 8(%rax)
-; SSE2-NEXT: movq %rsi, 16(%rax)
-; SSE2-NEXT: movq %rdx, 24(%rax)
+; SSE2-NEXT: shrdq %cl, %r10, %rsi
+; SSE2-NEXT: xorl %ecx, %ecx
+; SSE2-NEXT: testl %eax, %eax
+; SSE2-NEXT: cmoveq %rcx, %r11
+; SSE2-NEXT: cmoveq %rcx, %r9
+; SSE2-NEXT: cmoveq %rcx, %rsi
+; SSE2-NEXT: movq %rdi, %rax
+; SSE2-NEXT: cmoveq %rcx, %rdx
+; SSE2-NEXT: movq %rdx, 24(%rdi)
+; SSE2-NEXT: movq %r9, 16(%rdi)
+; SSE2-NEXT: movq %r11, 8(%rdi)
+; SSE2-NEXT: movq %rsi, (%rdi)
; SSE2-NEXT: retq
;
; SSE42-LABEL: isolate_msb_i256_vector:
; SSE42: # %bb.0:
-; SSE42-NEXT: movq %rdi, %rax
-; SSE42-NEXT: movdqa %xmm0, %xmm2
-; SSE42-NEXT: por %xmm1, %xmm2
-; SSE42-NEXT: ptest %xmm2, %xmm2
-; SSE42-NEXT: je .LBB16_1
-; SSE42-NEXT: # %bb.2: # %select.false.sink
-; SSE42-NEXT: pextrq $1, %xmm1, %rdx
-; SSE42-NEXT: pextrq $1, %xmm0, %rsi
-; SSE42-NEXT: movq %xmm1, %rdi
+; SSE42-NEXT: pextrq $1, %xmm0, %rdx
; SSE42-NEXT: movq %xmm0, %rcx
-; SSE42-NEXT: bsrq %rdx, %r8
-; SSE42-NEXT: xorq $63, %r8
-; SSE42-NEXT: bsrq %rdi, %r9
-; SSE42-NEXT: xorq $63, %r9
-; SSE42-NEXT: orq $64, %r9
-; SSE42-NEXT: testq %rdx, %rdx
-; SSE42-NEXT: cmovneq %r8, %r9
+; SSE42-NEXT: movq %xmm1, %rax
+; SSE42-NEXT: pextrq $1, %xmm1, %rsi
; SSE42-NEXT: bsrq %rsi, %r8
; SSE42-NEXT: xorq $63, %r8
+; SSE42-NEXT: bsrq %rax, %rax
+; SSE42-NEXT: xorq $63, %rax
+; SSE42-NEXT: orq $64, %rax
+; SSE42-NEXT: testq %rsi, %rsi
+; SSE42-NEXT: cmovneq %r8, %rax
+; SSE42-NEXT: bsrq %rdx, %rsi
+; SSE42-NEXT: xorq $63, %rsi
; SSE42-NEXT: bsrq %rcx, %rcx
; SSE42-NEXT: xorq $63, %rcx
; SSE42-NEXT: orq $64, %rcx
-; SSE42-NEXT: testq %rsi, %rsi
-; SSE42-NEXT: cmovneq %r8, %rcx
+; SSE42-NEXT: testq %rdx, %rdx
+; SSE42-NEXT: cmovneq %rsi, %rcx
+; SSE42-NEXT: xorps %xmm2, %xmm2
+; SSE42-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
; SSE42-NEXT: orq $128, %rcx
-; SSE42-NEXT: orq %rdx, %rdi
-; SSE42-NEXT: cmovneq %r9, %rcx
-; SSE42-NEXT: pxor %xmm0, %xmm0
-; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movabsq $-9223372036854775808, %rdx # imm = 0x8000000000000000
-; SSE42-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movl %ecx, %edx
-; SSE42-NEXT: shrb $6, %dl
-; SSE42-NEXT: movzbl %dl, %edi
+; SSE42-NEXT: ptest %xmm1, %xmm1
+; SSE42-NEXT: cmovneq %rax, %rcx
+; SSE42-NEXT: movabsq $-9223372036854775808, %rax # imm = 0x8000000000000000
+; SSE42-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movl %ecx, %eax
+; SSE42-NEXT: shrb $6, %al
+; SSE42-NEXT: movzbl %al, %eax
; SSE42-NEXT: movq $0, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movq -48(%rsp,%rdi,8), %rdx
-; SSE42-NEXT: movq -56(%rsp,%rdi,8), %r9
-; SSE42-NEXT: movq %r9, %rsi
-; SSE42-NEXT: shrdq %cl, %rdx, %rsi
-; SSE42-NEXT: movq -64(%rsp,%rdi,8), %r10
-; SSE42-NEXT: movq %r10, %r8
-; SSE42-NEXT: shrdq %cl, %r9, %r8
-; SSE42-NEXT: movq -72(%rsp,%rdi,8), %rdi
+; SSE42-NEXT: movq -48(%rsp,%rax,8), %rdx
+; SSE42-NEXT: movq -56(%rsp,%rax,8), %rsi
+; SSE42-NEXT: movq %rsi, %r8
+; SSE42-NEXT: shrdq %cl, %rdx, %r8
+; SSE42-NEXT: movq -64(%rsp,%rax,8), %r9
+; SSE42-NEXT: movq %r9, %r10
+; SSE42-NEXT: shrdq %cl, %rsi, %r10
+; SSE42-NEXT: movq -72(%rsp,%rax,8), %rsi
; SSE42-NEXT: shrq %cl, %rdx
; SSE42-NEXT: # kill: def $cl killed $cl killed $rcx
-; SSE42-NEXT: shrdq %cl, %r10, %rdi
-; SSE42-NEXT: jmp .LBB16_3
-; SSE42-NEXT: .LBB16_1:
-; SSE42-NEXT: xorl %edi, %edi
-; SSE42-NEXT: xorl %r8d, %r8d
-; SSE42-NEXT: xorl %esi, %esi
-; SSE42-NEXT: xorl %edx, %edx
-; SSE42-NEXT: .LBB16_3: # %select.end
-; SSE42-NEXT: movq %rdi, (%rax)
-; SSE42-NEXT: movq %r8, 8(%rax)
-; SSE42-NEXT: movq %rsi, 16(%rax)
-; SSE42-NEXT: movq %rdx, 24(%rax)
+; SSE42-NEXT: shrdq %cl, %r9, %rsi
+; SSE42-NEXT: por %xmm1, %xmm0
+; SSE42-NEXT: xorl %ecx, %ecx
+; SSE42-NEXT: ptest %xmm0, %xmm0
+; SSE42-NEXT: cmoveq %rcx, %r10
+; SSE42-NEXT: cmoveq %rcx, %r8
+; SSE42-NEXT: cmoveq %rcx, %rsi
+; SSE42-NEXT: movq %rdi, %rax
+; SSE42-NEXT: cmoveq %rcx, %rdx
+; SSE42-NEXT: movq %rdx, 24(%rdi)
+; SSE42-NEXT: movq %r8, 16(%rdi)
+; SSE42-NEXT: movq %r10, 8(%rdi)
+; SSE42-NEXT: movq %rsi, (%rdi)
; SSE42-NEXT: retq
;
; AVX2-LABEL: isolate_msb_i256_vector:
; AVX2: # %bb.0:
-; AVX2-NEXT: movq %rdi, %rax
-; AVX2-NEXT: vptest %ymm0, %ymm0
-; AVX2-NEXT: je .LBB16_1
-; AVX2-NEXT: # %bb.2: # %select.false.sink
+; AVX2-NEXT: vmovq %xmm0, %rax
+; AVX2-NEXT: vpextrq $1, %xmm0, %rdx
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT: vpextrq $1, %xmm1, %rdx
; AVX2-NEXT: vmovq %xmm1, %rsi
-; AVX2-NEXT: vpextrq $1, %xmm0, %rdi
-; AVX2-NEXT: vmovq %xmm0, %rcx
-; AVX2-NEXT: lzcntq %rdx, %r8
+; AVX2-NEXT: vpextrq $1, %xmm1, %r8
+; AVX2-NEXT: lzcntq %r8, %rcx
; AVX2-NEXT: lzcntq %rsi, %r9
; AVX2-NEXT: addq $64, %r9
-; AVX2-NEXT: testq %rdx, %rdx
-; AVX2-NEXT: cmovneq %r8, %r9
-; AVX2-NEXT: xorl %r8d, %r8d
-; AVX2-NEXT: lzcntq %rdi, %r8
-; AVX2-NEXT: lzcntq %rcx, %rcx
+; AVX2-NEXT: testq %r8, %r8
+; AVX2-NEXT: cmovneq %rcx, %r9
+; AVX2-NEXT: lzcntq %rdx, %r10
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: lzcntq %rax, %rcx
; AVX2-NEXT: addq $64, %rcx
-; AVX2-NEXT: testq %rdi, %rdi
-; AVX2-NEXT: cmovneq %r8, %rcx
+; AVX2-NEXT: testq %rdx, %rdx
+; AVX2-NEXT: cmovneq %r10, %rcx
; AVX2-NEXT: subq $-128, %rcx
-; AVX2-NEXT: orq %rdx, %rsi
+; AVX2-NEXT: orq %r8, %rsi
; AVX2-NEXT: cmovneq %r9, %rcx
-; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vmovdqa {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
-; AVX2-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movl %ecx, %edx
-; AVX2-NEXT: shrb $6, %dl
-; AVX2-NEXT: movzbl %dl, %esi
-; AVX2-NEXT: movq -48(%rsp,%rsi,8), %r8
-; AVX2-NEXT: movq -56(%rsp,%rsi,8), %r9
-; AVX2-NEXT: movq %r9, %rdx
-; AVX2-NEXT: shrdq %cl, %r8, %rdx
-; AVX2-NEXT: movq -64(%rsp,%rsi,8), %r10
-; AVX2-NEXT: movq %r10, %rdi
-; AVX2-NEXT: shrdq %cl, %r9, %rdi
-; AVX2-NEXT: movq -72(%rsp,%rsi,8), %rsi
-; AVX2-NEXT: shrxq %rcx, %r8, %r8
-; AVX2-NEXT: # kill: def $cl killed $cl killed $rcx
-; AVX2-NEXT: shrdq %cl, %r10, %rsi
-; AVX2-NEXT: jmp .LBB16_3
-; AVX2-NEXT: .LBB16_1:
-; AVX2-NEXT: xorl %esi, %esi
-; AVX2-NEXT: xorl %edi, %edi
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vmovdqu %ymm1, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovaps {{.*#+}} ymm1 = [0,0,0,9223372036854775808]
+; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movl %ecx, %eax
+; AVX2-NEXT: shrb $6, %al
+; AVX2-NEXT: movzbl %al, %edx
+; AVX2-NEXT: movq -48(%rsp,%rdx,8), %rsi
+; AVX2-NEXT: movq -56(%rsp,%rdx,8), %r8
+; AVX2-NEXT: movq %r8, %r9
+; AVX2-NEXT: shrdq %cl, %rsi, %r9
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: movq -72(%rsp,%rdx,8), %rdi
+; AVX2-NEXT: movq -64(%rsp,%rdx,8), %rdx
+; AVX2-NEXT: movq %rdx, %r10
+; AVX2-NEXT: shrdq %cl, %r8, %r10
+; AVX2-NEXT: shrdq %cl, %rdx, %rdi
; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: xorl %r8d, %r8d
-; AVX2-NEXT: .LBB16_3: # %select.end
-; AVX2-NEXT: movq %rsi, (%rax)
-; AVX2-NEXT: movq %rdi, 8(%rax)
-; AVX2-NEXT: movq %rdx, 16(%rax)
-; AVX2-NEXT: movq %r8, 24(%rax)
+; AVX2-NEXT: vptest %ymm0, %ymm0
+; AVX2-NEXT: shrxq %rcx, %rsi, %rcx
+; AVX2-NEXT: cmoveq %rdx, %r10
+; AVX2-NEXT: cmoveq %rdx, %r9
+; AVX2-NEXT: cmoveq %rdx, %rdi
+; AVX2-NEXT: cmoveq %rdx, %rcx
+; AVX2-NEXT: movq %rcx, 24(%rax)
+; AVX2-NEXT: movq %r9, 16(%rax)
+; AVX2-NEXT: movq %r10, 8(%rax)
+; AVX2-NEXT: movq %rdi, (%rax)
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512F-LABEL: isolate_msb_i256_vector:
; AVX512F: # %bb.0:
+; AVX512F-NEXT: vmovaps {{.*#+}} zmm1 = [0,0,0,9223372036854775808,0,0,0,0]
+; AVX512F-NEXT: vmovups %zmm1, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: vpermq {{.*#+}} ymm1 = ymm0[3,2,1,0]
+; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k0
+; AVX512F-NEXT: vplzcntq %zmm1, %zmm1
+; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,64,128,192]
+; AVX512F-NEXT: kshiftlw $12, %k0, %k0
+; AVX512F-NEXT: kshiftrw $12, %k0, %k1
+; AVX512F-NEXT: vpcompressq %zmm1, %zmm1 {%k1}
+; AVX512F-NEXT: vmovd %xmm1, %ecx
+; AVX512F-NEXT: movl %ecx, %eax
+; AVX512F-NEXT: shrb $6, %al
+; AVX512F-NEXT: movzbl %al, %edx
+; AVX512F-NEXT: movq -48(%rsp,%rdx,8), %rsi
+; AVX512F-NEXT: movq -56(%rsp,%rdx,8), %r8
+; AVX512F-NEXT: movq %r8, %r9
+; AVX512F-NEXT: shrdq %cl, %rsi, %r9
; AVX512F-NEXT: movq %rdi, %rax
-; AVX512F-NEXT: vmovdqa %ymm0, %ymm1
-; AVX512F-NEXT: vptestmd %zmm1, %zmm1, %k0
-; AVX512F-NEXT: kortestw %k0, %k0
-; AVX512F-NEXT: je .LBB16_1
-; AVX512F-NEXT: # %bb.2: # %select.false.sink
-; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX512F-NEXT: vpextrq $1, %xmm1, %rdx
-; AVX512F-NEXT: vmovq %xmm1, %rsi
-; AVX512F-NEXT: vpextrq $1, %xmm0, %rdi
-; AVX512F-NEXT: vmovq %xmm0, %rcx
-; AVX512F-NEXT: lzcntq %rdx, %r8
-; AVX512F-NEXT: lzcntq %rsi, %r9
-; AVX512F-NEXT: addq $64, %r9
-; AVX512F-NEXT: testq %rdx, %rdx
-; AVX512F-NEXT: cmovneq %r8, %r9
-; AVX512F-NEXT: lzcntq %rdi, %r8
-; AVX512F-NEXT: lzcntq %rcx, %rcx
-; AVX512F-NEXT: addq $64, %rcx
-; AVX512F-NEXT: testq %rdi, %rdi
-; AVX512F-NEXT: cmovneq %r8, %rcx
-; AVX512F-NEXT: subq $-128, %rcx
-; AVX512F-NEXT: orq %rdx, %rsi
-; AVX512F-NEXT: cmovneq %r9, %rcx
-; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm0 = [0,0,0,9223372036854775808,0,0,0,0]
-; AVX512F-NEXT: vmovdqu64 %zmm0, -{{[0-9]+}}(%rsp)
-; AVX512F-NEXT: movl %ecx, %edx
-; AVX512F-NEXT: shrb $6, %dl
-; AVX512F-NEXT: movzbl %dl, %esi
-; AVX512F-NEXT: movq -48(%rsp,%rsi,8), %r8
-; AVX512F-NEXT: movq -56(%rsp,%rsi,8), %r9
-; AVX512F-NEXT: movq %r9, %rdx
-; AVX512F-NEXT: shrdq %cl, %r8, %rdx
-; AVX512F-NEXT: movq -64(%rsp,%rsi,8), %r10
-; AVX512F-NEXT: movq %r10, %rdi
-; AVX512F-NEXT: shrdq %cl, %r9, %rdi
-; AVX512F-NEXT: movq -72(%rsp,%rsi,8), %rsi
-; AVX512F-NEXT: shrxq %rcx, %r8, %r8
-; AVX512F-NEXT: # kill: def $cl killed $cl killed $rcx
-; AVX512F-NEXT: shrdq %cl, %r10, %rsi
-; AVX512F-NEXT: jmp .LBB16_3
-; AVX512F-NEXT: .LBB16_1:
-; AVX512F-NEXT: xorl %esi, %esi
-; AVX512F-NEXT: xorl %edi, %edi
+; AVX512F-NEXT: vmovdqa %ymm0, %ymm0
+; AVX512F-NEXT: movq -72(%rsp,%rdx,8), %rdi
+; AVX512F-NEXT: movq -64(%rsp,%rdx,8), %rdx
+; AVX512F-NEXT: movq %rdx, %r10
+; AVX512F-NEXT: shrdq %cl, %r8, %r10
+; AVX512F-NEXT: shrdq %cl, %rdx, %rdi
+; AVX512F-NEXT: vptestmd %zmm0, %zmm0, %k0
; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: xorl %r8d, %r8d
-; AVX512F-NEXT: .LBB16_3: # %select.end
-; AVX512F-NEXT: movq %rsi, (%rax)
-; AVX512F-NEXT: movq %rdi, 8(%rax)
-; AVX512F-NEXT: movq %rdx, 16(%rax)
-; AVX512F-NEXT: movq %r8, 24(%rax)
+; AVX512F-NEXT: kortestw %k0, %k0
+; AVX512F-NEXT: shrxq %rcx, %rsi, %rcx
+; AVX512F-NEXT: cmoveq %rdx, %r10
+; AVX512F-NEXT: cmoveq %rdx, %r9
+; AVX512F-NEXT: cmoveq %rdx, %rdi
+; AVX512F-NEXT: cmoveq %rdx, %rcx
+; AVX512F-NEXT: movq %rcx, 24(%rax)
+; AVX512F-NEXT: movq %r9, 16(%rax)
+; AVX512F-NEXT: movq %r10, 8(%rax)
+; AVX512F-NEXT: movq %rdi, (%rax)
; AVX512F-NEXT: retq
;
; AVX512VL-LABEL: isolate_msb_i256_vector:
; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: vpermq {{.*#+}} ymm1 = ymm0[3,2,1,0]
+; AVX512VL-NEXT: vptestmq %ymm1, %ymm1, %k1
+; AVX512VL-NEXT: vplzcntq %ymm1, %ymm1
+; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,64,128,192]
+; AVX512VL-NEXT: vpcompressq %ymm1, %ymm1 {%k1}
+; AVX512VL-NEXT: vxorps %xmm2, %xmm2, %xmm2
+; AVX512VL-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: vmovaps {{.*#+}} ymm2 = [0,0,0,9223372036854775808]
+; AVX512VL-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: vmovd %xmm1, %ecx
+; AVX512VL-NEXT: movl %ecx, %eax
+; AVX512VL-NEXT: shrb $6, %al
+; AVX512VL-NEXT: movzbl %al, %edx
+; AVX512VL-NEXT: movq -48(%rsp,%rdx,8), %rsi
+; AVX512VL-NEXT: movq -56(%rsp,%rdx,8), %rax
+; AVX512VL-NEXT: movq %rax, %r8
+; AVX512VL-NEXT: shrdq %cl, %rsi, %r8
+; AVX512VL-NEXT: movq -64(%rsp,%rdx,8), %r9
+; AVX512VL-NEXT: movq %r9, %r10
+; AVX512VL-NEXT: shrdq %cl, %rax, %r10
; AVX512VL-NEXT: movq %rdi, %rax
-; AVX512VL-NEXT: vptest %ymm0, %ymm0
-; AVX512VL-NEXT: je .LBB16_1
-; AVX512VL-NEXT: # %bb.2: # %select.false.sink
-; AVX512VL-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX512VL-NEXT: vpextrq $1, %xmm1, %rdx
-; AVX512VL-NEXT: vpextrq $1, %xmm0, %rsi
-; AVX512VL-NEXT: vmovq %xmm1, %rdi
-; AVX512VL-NEXT: vmovq %xmm0, %rcx
-; AVX512VL-NEXT: lzcntq %rdx, %r8
-; AVX512VL-NEXT: lzcntq %rdi, %r9
-; AVX512VL-NEXT: addq $64, %r9
-; AVX512VL-NEXT: testq %rdx, %rdx
-; AVX512VL-NEXT: cmovneq %r8, %r9
-; AVX512VL-NEXT: lzcntq %rsi, %r8
-; AVX512VL-NEXT: lzcntq %rcx, %rcx
-; AVX512VL-NEXT: addq $64, %rcx
-; AVX512VL-NEXT: testq %rsi, %rsi
-; AVX512VL-NEXT: cmovneq %r8, %rcx
-; AVX512VL-NEXT: subq $-128, %rcx
-; AVX512VL-NEXT: orq %rdx, %rdi
-; AVX512VL-NEXT: cmovneq %r9, %rcx
-; AVX512VL-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX512VL-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: vmovdqa {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
-; AVX512VL-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: movl %ecx, %edx
-; AVX512VL-NEXT: shrb $6, %dl
-; AVX512VL-NEXT: movzbl %dl, %esi
-; AVX512VL-NEXT: movq -48(%rsp,%rsi,8), %r8
-; AVX512VL-NEXT: movq -56(%rsp,%rsi,8), %r9
-; AVX512VL-NEXT: movq %r9, %rdx
-; AVX512VL-NEXT: shrdq %cl, %r8, %rdx
-; AVX512VL-NEXT: movq -64(%rsp,%rsi,8), %r10
-; AVX512VL-NEXT: movq %r10, %rdi
-; AVX512VL-NEXT: shrdq %cl, %r9, %rdi
-; AVX512VL-NEXT: movq -72(%rsp,%rsi,8), %rsi
-; AVX512VL-NEXT: shrxq %rcx, %r8, %r8
-; AVX512VL-NEXT: # kill: def $cl killed $cl killed $rcx
-; AVX512VL-NEXT: shrdq %cl, %r10, %rsi
-; AVX512VL-NEXT: jmp .LBB16_3
-; AVX512VL-NEXT: .LBB16_1:
-; AVX512VL-NEXT: xorl %esi, %esi
+; AVX512VL-NEXT: movq -72(%rsp,%rdx,8), %rdx
+; AVX512VL-NEXT: shrdq %cl, %r9, %rdx
; AVX512VL-NEXT: xorl %edi, %edi
-; AVX512VL-NEXT: xorl %edx, %edx
-; AVX512VL-NEXT: xorl %r8d, %r8d
-; AVX512VL-NEXT: .LBB16_3: # %select.end
-; AVX512VL-NEXT: movq %rsi, (%rax)
-; AVX512VL-NEXT: movq %rdi, 8(%rax)
-; AVX512VL-NEXT: movq %rdx, 16(%rax)
-; AVX512VL-NEXT: movq %r8, 24(%rax)
+; AVX512VL-NEXT: vptest %ymm0, %ymm0
+; AVX512VL-NEXT: shrxq %rcx, %rsi, %rcx
+; AVX512VL-NEXT: cmoveq %rdi, %r10
+; AVX512VL-NEXT: cmoveq %rdi, %r8
+; AVX512VL-NEXT: cmoveq %rdi, %rdx
+; AVX512VL-NEXT: cmoveq %rdi, %rcx
+; AVX512VL-NEXT: movq %rcx, 24(%rax)
+; AVX512VL-NEXT: movq %r8, 16(%rax)
+; AVX512VL-NEXT: movq %r10, 8(%rax)
+; AVX512VL-NEXT: movq %rdx, (%rax)
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VBMI-LABEL: isolate_msb_i256_vector:
; AVX512VBMI: # %bb.0:
+; AVX512VBMI-NEXT: vpermq {{.*#+}} ymm1 = ymm0[3,2,1,0]
+; AVX512VBMI-NEXT: vptestmq %ymm1, %ymm1, %k1
+; AVX512VBMI-NEXT: vplzcntq %ymm1, %ymm1
+; AVX512VBMI-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,64,128,192]
+; AVX512VBMI-NEXT: vpcompressq %ymm1, %ymm1 {%k1}
+; AVX512VBMI-NEXT: vxorps %xmm2, %xmm2, %xmm2
+; AVX512VBMI-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: vmovaps {{.*#+}} ymm2 = [0,0,0,9223372036854775808]
+; AVX512VBMI-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: vmovd %xmm1, %ecx
+; AVX512VBMI-NEXT: movl %ecx, %eax
+; AVX512VBMI-NEXT: shrb $6, %al
+; AVX512VBMI-NEXT: movzbl %al, %edx
+; AVX512VBMI-NEXT: movq -48(%rsp,%rdx,8), %rsi
+; AVX512VBMI-NEXT: movq -56(%rsp,%rdx,8), %rax
+; AVX512VBMI-NEXT: movq %rax, %r8
+; AVX512VBMI-NEXT: shrdq %cl, %rsi, %r8
+; AVX512VBMI-NEXT: movq -64(%rsp,%rdx,8), %r9
+; AVX512VBMI-NEXT: movq %r9, %r10
+; AVX512VBMI-NEXT: shrdq %cl, %rax, %r10
; AVX512VBMI-NEXT: movq %rdi, %rax
-; AVX512VBMI-NEXT: vptest %ymm0, %ymm0
-; AVX512VBMI-NEXT: je .LBB16_1
-; AVX512VBMI-NEXT: # %bb.2: # %select.false.sink
-; AVX512VBMI-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX512VBMI-NEXT: vpextrq $1, %xmm1, %rdx
-; AVX512VBMI-NEXT: vpextrq $1, %xmm0, %rsi
-; AVX512VBMI-NEXT: vmovq %xmm1, %rdi
-; AVX512VBMI-NEXT: vmovq %xmm0, %rcx
-; AVX512VBMI-NEXT: lzcntq %rdx, %r8
-; AVX512VBMI-NEXT: lzcntq %rdi, %r9
-; AVX512VBMI-NEXT: addq $64, %r9
-; AVX512VBMI-NEXT: testq %rdx, %rdx
-; AVX512VBMI-NEXT: cmovneq %r8, %r9
-; AVX512VBMI-NEXT: lzcntq %rsi, %r8
-; AVX512VBMI-NEXT: lzcntq %rcx, %rcx
-; AVX512VBMI-NEXT: addq $64, %rcx
-; AVX512VBMI-NEXT: testq %rsi, %rsi
-; AVX512VBMI-NEXT: cmovneq %r8, %rcx
-; AVX512VBMI-NEXT: subq $-128, %rcx
-; AVX512VBMI-NEXT: orq %rdx, %rdi
-; AVX512VBMI-NEXT: cmovneq %r9, %rcx
-; AVX512VBMI-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX512VBMI-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: vmovdqa {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
-; AVX512VBMI-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: movl %ecx, %edx
-; AVX512VBMI-NEXT: shrb $6, %dl
-; AVX512VBMI-NEXT: movzbl %dl, %esi
-; AVX512VBMI-NEXT: movq -48(%rsp,%rsi,8), %r8
-; AVX512VBMI-NEXT: movq -56(%rsp,%rsi,8), %r9
-; AVX512VBMI-NEXT: movq %r9, %rdx
-; AVX512VBMI-NEXT: shrdq %cl, %r8, %rdx
-; AVX512VBMI-NEXT: movq -64(%rsp,%rsi,8), %r10
-; AVX512VBMI-NEXT: movq %r10, %rdi
-; AVX512VBMI-NEXT: shrdq %cl, %r9, %rdi
-; AVX512VBMI-NEXT: movq -72(%rsp,%rsi,8), %rsi
-; AVX512VBMI-NEXT: shrxq %rcx, %r8, %r8
-; AVX512VBMI-NEXT: # kill: def $cl killed $cl killed $rcx
-; AVX512VBMI-NEXT: shrdq %cl, %r10, %rsi
-; AVX512VBMI-NEXT: jmp .LBB16_3
-; AVX512VBMI-NEXT: .LBB16_1:
-; AVX512VBMI-NEXT: xorl %esi, %esi
+; AVX512VBMI-NEXT: movq -72(%rsp,%rdx,8), %rdx
+; AVX512VBMI-NEXT: shrdq %cl, %r9, %rdx
; AVX512VBMI-NEXT: xorl %edi, %edi
-; AVX512VBMI-NEXT: xorl %edx, %edx
-; AVX512VBMI-NEXT: xorl %r8d, %r8d
-; AVX512VBMI-NEXT: .LBB16_3: # %select.end
-; AVX512VBMI-NEXT: movq %rsi, (%rax)
-; AVX512VBMI-NEXT: movq %rdi, 8(%rax)
-; AVX512VBMI-NEXT: movq %rdx, 16(%rax)
-; AVX512VBMI-NEXT: movq %r8, 24(%rax)
+; AVX512VBMI-NEXT: vptest %ymm0, %ymm0
+; AVX512VBMI-NEXT: shrxq %rcx, %rsi, %rcx
+; AVX512VBMI-NEXT: cmoveq %rdi, %r10
+; AVX512VBMI-NEXT: cmoveq %rdi, %r8
+; AVX512VBMI-NEXT: cmoveq %rdi, %rdx
+; AVX512VBMI-NEXT: cmoveq %rdi, %rcx
+; AVX512VBMI-NEXT: movq %rcx, 24(%rax)
+; AVX512VBMI-NEXT: movq %r8, 16(%rax)
+; AVX512VBMI-NEXT: movq %r10, 8(%rax)
+; AVX512VBMI-NEXT: movq %rdx, (%rax)
; AVX512VBMI-NEXT: vzeroupper
; AVX512VBMI-NEXT: retq
%a0 = bitcast <4 x i64> %v0 to i256
@@ -3065,35 +2983,32 @@ define i256 @isolate_msb_i256_vector(<4 x i64> %v0, i256 %idx) nounwind {
define i256 @isolate_msb_i256_load(ptr %p0, i256 %idx) nounwind {
; SSE2-LABEL: isolate_msb_i256_load:
; SSE2: # %bb.0:
-; SSE2-NEXT: movq %rdi, %rax
+; SSE2-NEXT: movq 8(%rsi), %r9
+; SSE2-NEXT: movq 16(%rsi), %rdx
+; SSE2-NEXT: movq 24(%rsi), %r8
; SSE2-NEXT: movdqa (%rsi), %xmm1
; SSE2-NEXT: por 16(%rsi), %xmm1
; SSE2-NEXT: pxor %xmm0, %xmm0
; SSE2-NEXT: pcmpeqd %xmm0, %xmm1
-; SSE2-NEXT: movmskps %xmm1, %ecx
-; SSE2-NEXT: xorl $15, %ecx
-; SSE2-NEXT: je .LBB17_1
-; SSE2-NEXT: # %bb.2: # %select.false.sink
-; SSE2-NEXT: movq 24(%rsi), %rdx
-; SSE2-NEXT: movq 16(%rsi), %rdi
-; SSE2-NEXT: movq 8(%rsi), %r8
-; SSE2-NEXT: bsrq %rdx, %rcx
+; SSE2-NEXT: movmskps %xmm1, %eax
+; SSE2-NEXT: xorl $15, %eax
+; SSE2-NEXT: bsrq %r8, %rcx
; SSE2-NEXT: xorq $63, %rcx
-; SSE2-NEXT: bsrq %rdi, %r9
-; SSE2-NEXT: xorq $63, %r9
-; SSE2-NEXT: orq $64, %r9
-; SSE2-NEXT: testq %rdx, %rdx
-; SSE2-NEXT: cmovneq %rcx, %r9
-; SSE2-NEXT: bsrq %r8, %r10
+; SSE2-NEXT: bsrq %rdx, %r10
; SSE2-NEXT: xorq $63, %r10
+; SSE2-NEXT: orq $64, %r10
+; SSE2-NEXT: testq %r8, %r8
+; SSE2-NEXT: cmovneq %rcx, %r10
+; SSE2-NEXT: bsrq %r9, %r11
+; SSE2-NEXT: xorq $63, %r11
; SSE2-NEXT: bsrq (%rsi), %rcx
; SSE2-NEXT: xorq $63, %rcx
; SSE2-NEXT: orq $64, %rcx
-; SSE2-NEXT: testq %r8, %r8
-; SSE2-NEXT: cmovneq %r10, %rcx
+; SSE2-NEXT: testq %r9, %r9
+; SSE2-NEXT: cmovneq %r11, %rcx
; SSE2-NEXT: orq $128, %rcx
-; SSE2-NEXT: orq %rdx, %rdi
-; SSE2-NEXT: cmovneq %r9, %rcx
+; SSE2-NEXT: orq %r8, %rdx
+; SSE2-NEXT: cmovneq %r10, %rcx
; SSE2-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: movabsq $-9223372036854775808, %rdx # imm = 0x8000000000000000
@@ -3101,45 +3016,41 @@ define i256 @isolate_msb_i256_load(ptr %p0, i256 %idx) nounwind {
; SSE2-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: movl %ecx, %edx
; SSE2-NEXT: shrb $6, %dl
-; SSE2-NEXT: movzbl %dl, %edi
+; SSE2-NEXT: movzbl %dl, %esi
; SSE2-NEXT: movq $0, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movq -48(%rsp,%rdi,8), %rdx
-; SSE2-NEXT: movq -56(%rsp,%rdi,8), %r9
-; SSE2-NEXT: movq %r9, %rsi
-; SSE2-NEXT: shrdq %cl, %rdx, %rsi
-; SSE2-NEXT: movq -64(%rsp,%rdi,8), %r10
-; SSE2-NEXT: movq %r10, %r8
-; SSE2-NEXT: shrdq %cl, %r9, %r8
-; SSE2-NEXT: movq -72(%rsp,%rdi,8), %rdi
+; SSE2-NEXT: movq -48(%rsp,%rsi,8), %rdx
+; SSE2-NEXT: movq -56(%rsp,%rsi,8), %r8
+; SSE2-NEXT: movq %r8, %r9
+; SSE2-NEXT: shrdq %cl, %rdx, %r9
+; SSE2-NEXT: movq -64(%rsp,%rsi,8), %r10
+; SSE2-NEXT: movq %r10, %r11
+; SSE2-NEXT: shrdq %cl, %r8, %r11
+; SSE2-NEXT: movq -72(%rsp,%rsi,8), %rsi
; SSE2-NEXT: shrq %cl, %rdx
; SSE2-NEXT: # kill: def $cl killed $cl killed $rcx
-; SSE2-NEXT: shrdq %cl, %r10, %rdi
-; SSE2-NEXT: jmp .LBB17_3
-; SSE2-NEXT: .LBB17_1:
-; SSE2-NEXT: xorl %edi, %edi
-; SSE2-NEXT: xorl %r8d, %r8d
-; SSE2-NEXT: xorl %esi, %esi
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: .LBB17_3: # %select.end
-; SSE2-NEXT: movq %rdi, (%rax)
-; SSE2-NEXT: movq %r8, 8(%rax)
-; SSE2-NEXT: movq %rsi, 16(%rax)
-; SSE2-NEXT: movq %rdx, 24(%rax)
+; SSE2-NEXT: shrdq %cl, %r10, %rsi
+; SSE2-NEXT: xorl %ecx, %ecx
+; SSE2-NEXT: testl %eax, %eax
+; SSE2-NEXT: cmoveq %rcx, %r11
+; SSE2-NEXT: cmoveq %rcx, %r9
+; SSE2-NEXT: cmoveq %rcx, %rsi
+; SSE2-NEXT: movq %rdi, %rax
+; SSE2-NEXT: cmoveq %rcx, %rdx
+; SSE2-NEXT: movq %rdx, 24(%rdi)
+; SSE2-NEXT: movq %r9, 16(%rdi)
+; SSE2-NEXT: movq %r11, 8(%rdi)
+; SSE2-NEXT: movq %rsi, (%rdi)
; SSE2-NEXT: retq
;
; SSE42-LABEL: isolate_msb_i256_load:
; SSE42: # %bb.0:
-; SSE42-NEXT: movq %rdi, %rax
+; SSE42-NEXT: movq 16(%rsi), %rax
+; SSE42-NEXT: movq 24(%rsi), %rdx
; SSE42-NEXT: movdqa (%rsi), %xmm0
; SSE42-NEXT: por 16(%rsi), %xmm0
-; SSE42-NEXT: ptest %xmm0, %xmm0
-; SSE42-NEXT: je .LBB17_1
-; SSE42-NEXT: # %bb.2: # %select.false.sink
-; SSE42-NEXT: movq 24(%rsi), %rdx
-; SSE42-NEXT: movq 16(%rsi), %rdi
; SSE42-NEXT: bsrq %rdx, %rcx
; SSE42-NEXT: xorq $63, %rcx
-; SSE42-NEXT: bsrq %rdi, %r8
+; SSE42-NEXT: bsrq %rax, %r8
; SSE42-NEXT: xorq $63, %r8
; SSE42-NEXT: orq $64, %r8
; SSE42-NEXT: testq %rdx, %rdx
@@ -3153,259 +3064,210 @@ define i256 @isolate_msb_i256_load(ptr %p0, i256 %idx) nounwind {
; SSE42-NEXT: testq %r9, %r9
; SSE42-NEXT: cmovneq %r10, %rcx
; SSE42-NEXT: orq $128, %rcx
-; SSE42-NEXT: orq %rdx, %rdi
+; SSE42-NEXT: orq %rdx, %rax
; SSE42-NEXT: cmovneq %r8, %rcx
-; SSE42-NEXT: pxor %xmm0, %xmm0
-; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movabsq $-9223372036854775808, %rdx # imm = 0x8000000000000000
-; SSE42-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movl %ecx, %edx
-; SSE42-NEXT: shrb $6, %dl
-; SSE42-NEXT: movzbl %dl, %edi
+; SSE42-NEXT: xorps %xmm1, %xmm1
+; SSE42-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movabsq $-9223372036854775808, %rax # imm = 0x8000000000000000
+; SSE42-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movl %ecx, %eax
+; SSE42-NEXT: shrb $6, %al
+; SSE42-NEXT: movzbl %al, %eax
; SSE42-NEXT: movq $0, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movq -48(%rsp,%rdi,8), %rdx
-; SSE42-NEXT: movq -56(%rsp,%rdi,8), %r9
-; SSE42-NEXT: movq %r9, %rsi
-; SSE42-NEXT: shrdq %cl, %rdx, %rsi
-; SSE42-NEXT: movq -64(%rsp,%rdi,8), %r10
-; SSE42-NEXT: movq %r10, %r8
-; SSE42-NEXT: shrdq %cl, %r9, %r8
-; SSE42-NEXT: movq -72(%rsp,%rdi,8), %rdi
+; SSE42-NEXT: movq -48(%rsp,%rax,8), %rdx
+; SSE42-NEXT: movq -56(%rsp,%rax,8), %rsi
+; SSE42-NEXT: movq %rsi, %r8
+; SSE42-NEXT: shrdq %cl, %rdx, %r8
+; SSE42-NEXT: movq -64(%rsp,%rax,8), %r9
+; SSE42-NEXT: movq %r9, %r10
+; SSE42-NEXT: shrdq %cl, %rsi, %r10
+; SSE42-NEXT: movq -72(%rsp,%rax,8), %rsi
; SSE42-NEXT: shrq %cl, %rdx
; SSE42-NEXT: # kill: def $cl killed $cl killed $rcx
-; SSE42-NEXT: shrdq %cl, %r10, %rdi
-; SSE42-NEXT: jmp .LBB17_3
-; SSE42-NEXT: .LBB17_1:
-; SSE42-NEXT: xorl %edi, %edi
-; SSE42-NEXT: xorl %r8d, %r8d
-; SSE42-NEXT: xorl %esi, %esi
-; SSE42-NEXT: xorl %edx, %edx
-; SSE42-NEXT: .LBB17_3: # %select.end
-; SSE42-NEXT: movq %rdi, (%rax)
-; SSE42-NEXT: movq %r8, 8(%rax)
-; SSE42-NEXT: movq %rsi, 16(%rax)
-; SSE42-NEXT: movq %rdx, 24(%rax)
+; SSE42-NEXT: shrdq %cl, %r9, %rsi
+; SSE42-NEXT: xorl %ecx, %ecx
+; SSE42-NEXT: ptest %xmm0, %xmm0
+; SSE42-NEXT: cmoveq %rcx, %r10
+; SSE42-NEXT: cmoveq %rcx, %r8
+; SSE42-NEXT: cmoveq %rcx, %rsi
+; SSE42-NEXT: movq %rdi, %rax
+; SSE42-NEXT: cmoveq %rcx, %rdx
+; SSE42-NEXT: movq %rdx, 24(%rdi)
+; SSE42-NEXT: movq %r8, 16(%rdi)
+; SSE42-NEXT: movq %r10, 8(%rdi)
+; SSE42-NEXT: movq %rsi, (%rdi)
; SSE42-NEXT: retq
;
; AVX2-LABEL: isolate_msb_i256_load:
; AVX2: # %bb.0:
-; AVX2-NEXT: movq %rdi, %rax
-; AVX2-NEXT: vmovdqu (%rsi), %ymm0
-; AVX2-NEXT: vptest %ymm0, %ymm0
-; AVX2-NEXT: je .LBB17_1
-; AVX2-NEXT: # %bb.2: # %select.false.sink
+; AVX2-NEXT: movq 16(%rsi), %rax
; AVX2-NEXT: movq 24(%rsi), %rdx
-; AVX2-NEXT: movq 16(%rsi), %rdi
-; AVX2-NEXT: movq (%rsi), %rcx
-; AVX2-NEXT: movq 8(%rsi), %rsi
-; AVX2-NEXT: lzcntq %rdx, %r8
-; AVX2-NEXT: lzcntq %rdi, %r9
-; AVX2-NEXT: addq $64, %r9
+; AVX2-NEXT: lzcntq %rdx, %rcx
+; AVX2-NEXT: lzcntq %rax, %r8
+; AVX2-NEXT: addq $64, %r8
; AVX2-NEXT: testq %rdx, %rdx
-; AVX2-NEXT: cmovneq %r8, %r9
-; AVX2-NEXT: xorl %r8d, %r8d
-; AVX2-NEXT: lzcntq %rsi, %r8
-; AVX2-NEXT: lzcntq %rcx, %rcx
+; AVX2-NEXT: cmovneq %rcx, %r8
+; AVX2-NEXT: movq 8(%rsi), %r9
+; AVX2-NEXT: lzcntq %r9, %r10
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: lzcntq (%rsi), %rcx
; AVX2-NEXT: addq $64, %rcx
-; AVX2-NEXT: testq %rsi, %rsi
-; AVX2-NEXT: cmovneq %r8, %rcx
+; AVX2-NEXT: testq %r9, %r9
+; AVX2-NEXT: cmovneq %r10, %rcx
; AVX2-NEXT: subq $-128, %rcx
-; AVX2-NEXT: orq %rdx, %rdi
-; AVX2-NEXT: cmovneq %r9, %rcx
-; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vmovdqa {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
-; AVX2-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movl %ecx, %edx
-; AVX2-NEXT: shrb $6, %dl
-; AVX2-NEXT: movzbl %dl, %esi
-; AVX2-NEXT: movq -48(%rsp,%rsi,8), %r8
-; AVX2-NEXT: movq -56(%rsp,%rsi,8), %r9
-; AVX2-NEXT: movq %r9, %rdx
-; AVX2-NEXT: shrdq %cl, %r8, %rdx
-; AVX2-NEXT: movq -64(%rsp,%rsi,8), %r10
-; AVX2-NEXT: movq %r10, %rdi
-; AVX2-NEXT: shrdq %cl, %r9, %rdi
-; AVX2-NEXT: movq -72(%rsp,%rsi,8), %rsi
-; AVX2-NEXT: shrxq %rcx, %r8, %r8
-; AVX2-NEXT: # kill: def $cl killed $cl killed $rcx
-; AVX2-NEXT: shrdq %cl, %r10, %rsi
-; AVX2-NEXT: jmp .LBB17_3
-; AVX2-NEXT: .LBB17_1:
-; AVX2-NEXT: xorl %esi, %esi
-; AVX2-NEXT: xorl %edi, %edi
+; AVX2-NEXT: orq %rdx, %rax
+; AVX2-NEXT: cmovneq %r8, %rcx
+; AVX2-NEXT: vmovdqu (%rsi), %ymm0
+; AVX2-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovaps {{.*#+}} ymm1 = [0,0,0,9223372036854775808]
+; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movl %ecx, %eax
+; AVX2-NEXT: shrb $6, %al
+; AVX2-NEXT: movzbl %al, %edx
+; AVX2-NEXT: movq -48(%rsp,%rdx,8), %rsi
+; AVX2-NEXT: movq -56(%rsp,%rdx,8), %r8
+; AVX2-NEXT: movq %r8, %r9
+; AVX2-NEXT: shrdq %cl, %rsi, %r9
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: movq -72(%rsp,%rdx,8), %rdi
+; AVX2-NEXT: movq -64(%rsp,%rdx,8), %rdx
+; AVX2-NEXT: movq %rdx, %r10
+; AVX2-NEXT: shrdq %cl, %r8, %r10
+; AVX2-NEXT: shrdq %cl, %rdx, %rdi
; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: xorl %r8d, %r8d
-; AVX2-NEXT: .LBB17_3: # %select.end
-; AVX2-NEXT: movq %rsi, (%rax)
-; AVX2-NEXT: movq %rdi, 8(%rax)
-; AVX2-NEXT: movq %rdx, 16(%rax)
-; AVX2-NEXT: movq %r8, 24(%rax)
+; AVX2-NEXT: vptest %ymm0, %ymm0
+; AVX2-NEXT: shrxq %rcx, %rsi, %rcx
+; AVX2-NEXT: cmoveq %rdx, %r10
+; AVX2-NEXT: cmoveq %rdx, %r9
+; AVX2-NEXT: cmoveq %rdx, %rdi
+; AVX2-NEXT: cmoveq %rdx, %rcx
+; AVX2-NEXT: movq %rcx, 24(%rax)
+; AVX2-NEXT: movq %r9, 16(%rax)
+; AVX2-NEXT: movq %r10, 8(%rax)
+; AVX2-NEXT: movq %rdi, (%rax)
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512F-LABEL: isolate_msb_i256_load:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: movq %rdi, %rax
; AVX512F-NEXT: vmovdqu (%rsi), %ymm0
+; AVX512F-NEXT: vmovaps {{.*#+}} zmm1 = [0,0,0,9223372036854775808,0,0,0,0]
+; AVX512F-NEXT: vmovups %zmm1, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: vpermq {{.*#+}} ymm1 = ymm0[3,2,1,0]
+; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k0
+; AVX512F-NEXT: vplzcntq %zmm1, %zmm1
+; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,64,128,192]
+; AVX512F-NEXT: kshiftlw $12, %k0, %k0
+; AVX512F-NEXT: kshiftrw $12, %k0, %k1
+; AVX512F-NEXT: vpcompressq %zmm1, %zmm1 {%k1} {z}
+; AVX512F-NEXT: vmovd %xmm1, %ecx
+; AVX512F-NEXT: movl %ecx, %eax
+; AVX512F-NEXT: shrb $6, %al
+; AVX512F-NEXT: movzbl %al, %edx
+; AVX512F-NEXT: movq -48(%rsp,%rdx,8), %rsi
+; AVX512F-NEXT: movq -56(%rsp,%rdx,8), %r8
+; AVX512F-NEXT: movq %r8, %r9
+; AVX512F-NEXT: shrdq %cl, %rsi, %r9
+; AVX512F-NEXT: movq %rdi, %rax
+; AVX512F-NEXT: movq -72(%rsp,%rdx,8), %rdi
+; AVX512F-NEXT: movq -64(%rsp,%rdx,8), %rdx
+; AVX512F-NEXT: movq %rdx, %r10
+; AVX512F-NEXT: shrdq %cl, %r8, %r10
+; AVX512F-NEXT: shrdq %cl, %rdx, %rdi
; AVX512F-NEXT: vptestmd %zmm0, %zmm0, %k0
-; AVX512F-NEXT: kortestw %k0, %k0
-; AVX512F-NEXT: je .LBB17_1
-; AVX512F-NEXT: # %bb.2: # %select.false.sink
-; AVX512F-NEXT: movq 24(%rsi), %rdx
-; AVX512F-NEXT: movq 16(%rsi), %rdi
-; AVX512F-NEXT: lzcntq %rdx, %rcx
-; AVX512F-NEXT: lzcntq %rdi, %r8
-; AVX512F-NEXT: addq $64, %r8
-; AVX512F-NEXT: testq %rdx, %rdx
-; AVX512F-NEXT: cmovneq %rcx, %r8
-; AVX512F-NEXT: movq 8(%rsi), %r9
-; AVX512F-NEXT: lzcntq %r9, %r10
-; AVX512F-NEXT: lzcntq (%rsi), %rcx
-; AVX512F-NEXT: addq $64, %rcx
-; AVX512F-NEXT: testq %r9, %r9
-; AVX512F-NEXT: cmovneq %r10, %rcx
-; AVX512F-NEXT: subq $-128, %rcx
-; AVX512F-NEXT: orq %rdx, %rdi
-; AVX512F-NEXT: cmovneq %r8, %rcx
-; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm0 = [0,0,0,9223372036854775808,0,0,0,0]
-; AVX512F-NEXT: vmovdqu64 %zmm0, -{{[0-9]+}}(%rsp)
-; AVX512F-NEXT: movl %ecx, %edx
-; AVX512F-NEXT: shrb $6, %dl
-; AVX512F-NEXT: movzbl %dl, %esi
-; AVX512F-NEXT: movq -48(%rsp,%rsi,8), %r8
-; AVX512F-NEXT: movq -56(%rsp,%rsi,8), %r9
-; AVX512F-NEXT: movq %r9, %rdx
-; AVX512F-NEXT: shrdq %cl, %r8, %rdx
-; AVX512F-NEXT: movq -64(%rsp,%rsi,8), %r10
-; AVX512F-NEXT: movq %r10, %rdi
-; AVX512F-NEXT: shrdq %cl, %r9, %rdi
-; AVX512F-NEXT: movq -72(%rsp,%rsi,8), %rsi
-; AVX512F-NEXT: shrxq %rcx, %r8, %r8
-; AVX512F-NEXT: # kill: def $cl killed $cl killed $rcx
-; AVX512F-NEXT: shrdq %cl, %r10, %rsi
-; AVX512F-NEXT: jmp .LBB17_3
-; AVX512F-NEXT: .LBB17_1:
-; AVX512F-NEXT: xorl %esi, %esi
-; AVX512F-NEXT: xorl %edi, %edi
; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: xorl %r8d, %r8d
-; AVX512F-NEXT: .LBB17_3: # %select.end
-; AVX512F-NEXT: movq %rsi, (%rax)
-; AVX512F-NEXT: movq %rdi, 8(%rax)
-; AVX512F-NEXT: movq %rdx, 16(%rax)
-; AVX512F-NEXT: movq %r8, 24(%rax)
+; AVX512F-NEXT: kortestw %k0, %k0
+; AVX512F-NEXT: shrxq %rcx, %rsi, %rcx
+; AVX512F-NEXT: cmoveq %rdx, %r10
+; AVX512F-NEXT: cmoveq %rdx, %r9
+; AVX512F-NEXT: cmoveq %rdx, %rdi
+; AVX512F-NEXT: cmoveq %rdx, %rcx
+; AVX512F-NEXT: movq %rcx, 24(%rax)
+; AVX512F-NEXT: movq %r9, 16(%rax)
+; AVX512F-NEXT: movq %r10, 8(%rax)
+; AVX512F-NEXT: movq %rdi, (%rax)
; AVX512F-NEXT: retq
;
; AVX512VL-LABEL: isolate_msb_i256_load:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: movq %rdi, %rax
; AVX512VL-NEXT: vmovdqu (%rsi), %ymm0
-; AVX512VL-NEXT: vptest %ymm0, %ymm0
-; AVX512VL-NEXT: je .LBB17_1
-; AVX512VL-NEXT: # %bb.2: # %select.false.sink
-; AVX512VL-NEXT: movq 24(%rsi), %rdx
-; AVX512VL-NEXT: movq 16(%rsi), %rdi
-; AVX512VL-NEXT: movq 8(%rsi), %r8
-; AVX512VL-NEXT: lzcntq %rdx, %rcx
-; AVX512VL-NEXT: lzcntq %rdi, %r9
-; AVX512VL-NEXT: addq $64, %r9
-; AVX512VL-NEXT: testq %rdx, %rdx
-; AVX512VL-NEXT: cmovneq %rcx, %r9
-; AVX512VL-NEXT: lzcntq %r8, %r10
-; AVX512VL-NEXT: lzcntq (%rsi), %rcx
-; AVX512VL-NEXT: addq $64, %rcx
-; AVX512VL-NEXT: testq %r8, %r8
-; AVX512VL-NEXT: cmovneq %r10, %rcx
-; AVX512VL-NEXT: subq $-128, %rcx
-; AVX512VL-NEXT: orq %rdx, %rdi
-; AVX512VL-NEXT: cmovneq %r9, %rcx
-; AVX512VL-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX512VL-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: vmovdqa {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
-; AVX512VL-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: movl %ecx, %edx
-; AVX512VL-NEXT: shrb $6, %dl
-; AVX512VL-NEXT: movzbl %dl, %esi
-; AVX512VL-NEXT: movq -48(%rsp,%rsi,8), %r8
-; AVX512VL-NEXT: movq -56(%rsp,%rsi,8), %r9
-; AVX512VL-NEXT: movq %r9, %rdx
-; AVX512VL-NEXT: shrdq %cl, %r8, %rdx
-; AVX512VL-NEXT: movq -64(%rsp,%rsi,8), %r10
-; AVX512VL-NEXT: movq %r10, %rdi
-; AVX512VL-NEXT: shrdq %cl, %r9, %rdi
-; AVX512VL-NEXT: movq -72(%rsp,%rsi,8), %rsi
-; AVX512VL-NEXT: shrxq %rcx, %r8, %r8
-; AVX512VL-NEXT: # kill: def $cl killed $cl killed $rcx
-; AVX512VL-NEXT: shrdq %cl, %r10, %rsi
-; AVX512VL-NEXT: jmp .LBB17_3
-; AVX512VL-NEXT: .LBB17_1:
-; AVX512VL-NEXT: xorl %esi, %esi
+; AVX512VL-NEXT: vpermq {{.*#+}} ymm1 = ymm0[3,2,1,0]
+; AVX512VL-NEXT: vptestmq %ymm1, %ymm1, %k1
+; AVX512VL-NEXT: vplzcntq %ymm1, %ymm1
+; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,64,128,192]
+; AVX512VL-NEXT: vpcompressq %ymm1, %ymm1 {%k1} {z}
+; AVX512VL-NEXT: vxorps %xmm2, %xmm2, %xmm2
+; AVX512VL-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: vmovaps {{.*#+}} ymm2 = [0,0,0,9223372036854775808]
+; AVX512VL-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: vmovd %xmm1, %ecx
+; AVX512VL-NEXT: movl %ecx, %eax
+; AVX512VL-NEXT: shrb $6, %al
+; AVX512VL-NEXT: movzbl %al, %edx
+; AVX512VL-NEXT: movq -48(%rsp,%rdx,8), %rsi
+; AVX512VL-NEXT: movq -56(%rsp,%rdx,8), %rax
+; AVX512VL-NEXT: movq %rax, %r8
+; AVX512VL-NEXT: shrdq %cl, %rsi, %r8
+; AVX512VL-NEXT: movq -64(%rsp,%rdx,8), %r9
+; AVX512VL-NEXT: movq %r9, %r10
+; AVX512VL-NEXT: shrdq %cl, %rax, %r10
+; AVX512VL-NEXT: movq %rdi, %rax
+; AVX512VL-NEXT: movq -72(%rsp,%rdx,8), %rdx
+; AVX512VL-NEXT: shrdq %cl, %r9, %rdx
; AVX512VL-NEXT: xorl %edi, %edi
-; AVX512VL-NEXT: xorl %edx, %edx
-; AVX512VL-NEXT: xorl %r8d, %r8d
-; AVX512VL-NEXT: .LBB17_3: # %select.end
-; AVX512VL-NEXT: movq %rsi, (%rax)
-; AVX512VL-NEXT: movq %rdi, 8(%rax)
-; AVX512VL-NEXT: movq %rdx, 16(%rax)
-; AVX512VL-NEXT: movq %r8, 24(%rax)
+; AVX512VL-NEXT: vptest %ymm0, %ymm0
+; AVX512VL-NEXT: shrxq %rcx, %rsi, %rcx
+; AVX512VL-NEXT: cmoveq %rdi, %r10
+; AVX512VL-NEXT: cmoveq %rdi, %r8
+; AVX512VL-NEXT: cmoveq %rdi, %rdx
+; AVX512VL-NEXT: cmoveq %rdi, %rcx
+; AVX512VL-NEXT: movq %rcx, 24(%rax)
+; AVX512VL-NEXT: movq %r8, 16(%rax)
+; AVX512VL-NEXT: movq %r10, 8(%rax)
+; AVX512VL-NEXT: movq %rdx, (%rax)
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VBMI-LABEL: isolate_msb_i256_load:
; AVX512VBMI: # %bb.0:
-; AVX512VBMI-NEXT: movq %rdi, %rax
; AVX512VBMI-NEXT: vmovdqu (%rsi), %ymm0
-; AVX512VBMI-NEXT: vptest %ymm0, %ymm0
-; AVX512VBMI-NEXT: je .LBB17_1
-; AVX512VBMI-NEXT: # %bb.2: # %select.false.sink
-; AVX512VBMI-NEXT: movq 24(%rsi), %rdx
-; AVX512VBMI-NEXT: movq 16(%rsi), %rdi
-; AVX512VBMI-NEXT: movq 8(%rsi), %r8
-; AVX512VBMI-NEXT: lzcntq %rdx, %rcx
-; AVX512VBMI-NEXT: lzcntq %rdi, %r9
-; AVX512VBMI-NEXT: addq $64, %r9
-; AVX512VBMI-NEXT: testq %rdx, %rdx
-; AVX512VBMI-NEXT: cmovneq %rcx, %r9
-; AVX512VBMI-NEXT: lzcntq %r8, %r10
-; AVX512VBMI-NEXT: lzcntq (%rsi), %rcx
-; AVX512VBMI-NEXT: addq $64, %rcx
-; AVX512VBMI-NEXT: testq %r8, %r8
-; AVX512VBMI-NEXT: cmovneq %r10, %rcx
-; AVX512VBMI-NEXT: subq $-128, %rcx
-; AVX512VBMI-NEXT: orq %rdx, %rdi
-; AVX512VBMI-NEXT: cmovneq %r9, %rcx
-; AVX512VBMI-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX512VBMI-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: vmovdqa {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
-; AVX512VBMI-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: movl %ecx, %edx
-; AVX512VBMI-NEXT: shrb $6, %dl
-; AVX512VBMI-NEXT: movzbl %dl, %esi
-; AVX512VBMI-NEXT: movq -48(%rsp,%rsi,8), %r8
-; AVX512VBMI-NEXT: movq -56(%rsp,%rsi,8), %r9
-; AVX512VBMI-NEXT: movq %r9, %rdx
-; AVX512VBMI-NEXT: shrdq %cl, %r8, %rdx
-; AVX512VBMI-NEXT: movq -64(%rsp,%rsi,8), %r10
-; AVX512VBMI-NEXT: movq %r10, %rdi
-; AVX512VBMI-NEXT: shrdq %cl, %r9, %rdi
-; AVX512VBMI-NEXT: movq -72(%rsp,%rsi,8), %rsi
-; AVX512VBMI-NEXT: shrxq %rcx, %r8, %r8
-; AVX512VBMI-NEXT: # kill: def $cl killed $cl killed $rcx
-; AVX512VBMI-NEXT: shrdq %cl, %r10, %rsi
-; AVX512VBMI-NEXT: jmp .LBB17_3
-; AVX512VBMI-NEXT: .LBB17_1:
-; AVX512VBMI-NEXT: xorl %esi, %esi
+; AVX512VBMI-NEXT: vpermq {{.*#+}} ymm1 = ymm0[3,2,1,0]
+; AVX512VBMI-NEXT: vptestmq %ymm1, %ymm1, %k1
+; AVX512VBMI-NEXT: vplzcntq %ymm1, %ymm1
+; AVX512VBMI-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,64,128,192]
+; AVX512VBMI-NEXT: vpcompressq %ymm1, %ymm1 {%k1} {z}
+; AVX512VBMI-NEXT: vxorps %xmm2, %xmm2, %xmm2
+; AVX512VBMI-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: vmovaps {{.*#+}} ymm2 = [0,0,0,9223372036854775808]
+; AVX512VBMI-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: vmovd %xmm1, %ecx
+; AVX512VBMI-NEXT: movl %ecx, %eax
+; AVX512VBMI-NEXT: shrb $6, %al
+; AVX512VBMI-NEXT: movzbl %al, %edx
+; AVX512VBMI-NEXT: movq -48(%rsp,%rdx,8), %rsi
+; AVX512VBMI-NEXT: movq -56(%rsp,%rdx,8), %rax
+; AVX512VBMI-NEXT: movq %rax, %r8
+; AVX512VBMI-NEXT: shrdq %cl, %rsi, %r8
+; AVX512VBMI-NEXT: movq -64(%rsp,%rdx,8), %r9
+; AVX512VBMI-NEXT: movq %r9, %r10
+; AVX512VBMI-NEXT: shrdq %cl, %rax, %r10
+; AVX512VBMI-NEXT: movq %rdi, %rax
+; AVX512VBMI-NEXT: movq -72(%rsp,%rdx,8), %rdx
+; AVX512VBMI-NEXT: shrdq %cl, %r9, %rdx
; AVX512VBMI-NEXT: xorl %edi, %edi
-; AVX512VBMI-NEXT: xorl %edx, %edx
-; AVX512VBMI-NEXT: xorl %r8d, %r8d
-; AVX512VBMI-NEXT: .LBB17_3: # %select.end
-; AVX512VBMI-NEXT: movq %rsi, (%rax)
-; AVX512VBMI-NEXT: movq %rdi, 8(%rax)
-; AVX512VBMI-NEXT: movq %rdx, 16(%rax)
-; AVX512VBMI-NEXT: movq %r8, 24(%rax)
+; AVX512VBMI-NEXT: vptest %ymm0, %ymm0
+; AVX512VBMI-NEXT: shrxq %rcx, %rsi, %rcx
+; AVX512VBMI-NEXT: cmoveq %rdi, %r10
+; AVX512VBMI-NEXT: cmoveq %rdi, %r8
+; AVX512VBMI-NEXT: cmoveq %rdi, %rdx
+; AVX512VBMI-NEXT: cmoveq %rdi, %rcx
+; AVX512VBMI-NEXT: movq %rcx, 24(%rax)
+; AVX512VBMI-NEXT: movq %r8, 16(%rax)
+; AVX512VBMI-NEXT: movq %r10, 8(%rax)
+; AVX512VBMI-NEXT: movq %rdx, (%rax)
; AVX512VBMI-NEXT: vzeroupper
; AVX512VBMI-NEXT: retq
%a0 = load i256, ptr %p0
diff --git a/llvm/test/CodeGen/X86/bit-manip-i512.ll b/llvm/test/CodeGen/X86/bit-manip-i512.ll
index 81694e88b66b0..5b62070e1d01d 100644
--- a/llvm/test/CodeGen/X86/bit-manip-i512.ll
+++ b/llvm/test/CodeGen/X86/bit-manip-i512.ll
@@ -4101,64 +4101,62 @@ define i512 @isolate_msb_i512(i512 %a0, i512 %idx) nounwind {
; SSE: # %bb.0:
; SSE-NEXT: pushq %r15
; SSE-NEXT: pushq %r14
+; SSE-NEXT: pushq %r13
+; SSE-NEXT: pushq %r12
; SSE-NEXT: pushq %rbx
-; SSE-NEXT: movq %rdi, %rax
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r10
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r11
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rdi
-; SSE-NEXT: movq %r8, %rbx
-; SSE-NEXT: orq %r10, %rbx
-; SSE-NEXT: movq %rdx, %r14
-; SSE-NEXT: orq %rdi, %r14
-; SSE-NEXT: orq %rbx, %r14
-; SSE-NEXT: movq %rcx, %rbx
-; SSE-NEXT: orq %r11, %rbx
-; SSE-NEXT: movq %rsi, %r15
-; SSE-NEXT: orq %r9, %r15
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r14
+; SSE-NEXT: movq %r8, %r10
+; SSE-NEXT: orq %r14, %r10
+; SSE-NEXT: movq %rdx, %rax
+; SSE-NEXT: orq %r11, %rax
+; SSE-NEXT: orq %r10, %rax
+; SSE-NEXT: movq %rcx, %r15
; SSE-NEXT: orq %rbx, %r15
-; SSE-NEXT: orq %r14, %r15
-; SSE-NEXT: je .LBB15_1
-; SSE-NEXT: # %bb.2: # %select.false.sink
-; SSE-NEXT: bsrq %r10, %rbx
-; SSE-NEXT: xorq $63, %rbx
-; SSE-NEXT: bsrq %r11, %r14
-; SSE-NEXT: xorq $63, %r14
-; SSE-NEXT: orq $64, %r14
-; SSE-NEXT: testq %r10, %r10
-; SSE-NEXT: cmovneq %rbx, %r14
-; SSE-NEXT: bsrq %rdi, %r15
+; SSE-NEXT: movq %rsi, %r10
+; SSE-NEXT: orq %r9, %r10
+; SSE-NEXT: orq %r15, %r10
+; SSE-NEXT: bsrq %r14, %r15
; SSE-NEXT: xorq $63, %r15
-; SSE-NEXT: bsrq %r9, %rbx
-; SSE-NEXT: xorq $63, %rbx
-; SSE-NEXT: orq $64, %rbx
-; SSE-NEXT: testq %rdi, %rdi
-; SSE-NEXT: cmovneq %r15, %rbx
-; SSE-NEXT: orq $128, %rbx
-; SSE-NEXT: movq %r11, %r15
-; SSE-NEXT: orq %r10, %r15
-; SSE-NEXT: cmovneq %r14, %rbx
-; SSE-NEXT: bsrq %r8, %r14
-; SSE-NEXT: xorq $63, %r14
-; SSE-NEXT: bsrq %rcx, %r15
+; SSE-NEXT: bsrq %rbx, %r12
+; SSE-NEXT: xorq $63, %r12
+; SSE-NEXT: orq $64, %r12
+; SSE-NEXT: testq %r14, %r14
+; SSE-NEXT: cmovneq %r15, %r12
+; SSE-NEXT: bsrq %r11, %r13
+; SSE-NEXT: xorq $63, %r13
+; SSE-NEXT: bsrq %r9, %r15
; SSE-NEXT: xorq $63, %r15
; SSE-NEXT: orq $64, %r15
+; SSE-NEXT: testq %r11, %r11
+; SSE-NEXT: cmovneq %r13, %r15
+; SSE-NEXT: orq $128, %r15
+; SSE-NEXT: movq %rbx, %r13
+; SSE-NEXT: orq %r14, %r13
+; SSE-NEXT: cmovneq %r12, %r15
+; SSE-NEXT: bsrq %r8, %r12
+; SSE-NEXT: xorq $63, %r12
+; SSE-NEXT: bsrq %rcx, %r13
+; SSE-NEXT: xorq $63, %r13
+; SSE-NEXT: orq $64, %r13
; SSE-NEXT: testq %r8, %r8
-; SSE-NEXT: cmovneq %r14, %r15
-; SSE-NEXT: bsrq %rdx, %r14
-; SSE-NEXT: xorq $63, %r14
+; SSE-NEXT: cmovneq %r12, %r13
+; SSE-NEXT: bsrq %rdx, %r12
+; SSE-NEXT: xorq $63, %r12
; SSE-NEXT: bsrq %rsi, %rsi
; SSE-NEXT: xorq $63, %rsi
; SSE-NEXT: orq $64, %rsi
; SSE-NEXT: testq %rdx, %rdx
-; SSE-NEXT: cmovneq %r14, %rsi
+; SSE-NEXT: cmovneq %r12, %rsi
; SSE-NEXT: orq $128, %rsi
; SSE-NEXT: orq %r8, %rcx
-; SSE-NEXT: cmovneq %r15, %rsi
+; SSE-NEXT: cmovneq %r13, %rsi
; SSE-NEXT: orq $256, %rsi # imm = 0x100
+; SSE-NEXT: orq %r14, %r11
+; SSE-NEXT: orq %rbx, %r9
; SSE-NEXT: orq %r11, %r9
-; SSE-NEXT: orq %r10, %rdi
-; SSE-NEXT: orq %r9, %rdi
-; SSE-NEXT: cmovneq %rbx, %rsi
+; SSE-NEXT: cmovneq %r15, %rsi
; SSE-NEXT: xorps %xmm0, %xmm0
; SSE-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
; SSE-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
@@ -4175,288 +4173,244 @@ define i512 @isolate_msb_i512(i512 %a0, i512 %idx) nounwind {
; SSE-NEXT: andl $56, %esi
; SSE-NEXT: movq $0, -{{[0-9]+}}(%rsp)
; SSE-NEXT: movq -72(%rsp,%rsi), %rdx
-; SSE-NEXT: movq -80(%rsp,%rsi), %r9
-; SSE-NEXT: movq %r9, %rdi
-; SSE-NEXT: shrdq %cl, %rdx, %rdi
-; SSE-NEXT: movq -88(%rsp,%rsi), %r10
-; SSE-NEXT: movq %r10, %r8
-; SSE-NEXT: shrdq %cl, %r9, %r8
-; SSE-NEXT: movq -96(%rsp,%rsi), %r11
-; SSE-NEXT: movq %r11, %r9
-; SSE-NEXT: shrdq %cl, %r10, %r9
-; SSE-NEXT: movq -104(%rsp,%rsi), %rbx
-; SSE-NEXT: movq %rbx, %r10
-; SSE-NEXT: shrdq %cl, %r11, %r10
-; SSE-NEXT: movq -112(%rsp,%rsi), %r14
+; SSE-NEXT: movq -80(%rsp,%rsi), %r11
+; SSE-NEXT: movq %r11, %r8
+; SSE-NEXT: shrdq %cl, %rdx, %r8
+; SSE-NEXT: movq -88(%rsp,%rsi), %rbx
+; SSE-NEXT: movq %rbx, %r9
+; SSE-NEXT: shrdq %cl, %r11, %r9
+; SSE-NEXT: movq -96(%rsp,%rsi), %r14
; SSE-NEXT: movq %r14, %r11
; SSE-NEXT: shrdq %cl, %rbx, %r11
-; SSE-NEXT: movq -120(%rsp,%rsi), %r15
+; SSE-NEXT: movq -104(%rsp,%rsi), %r15
; SSE-NEXT: movq %r15, %rbx
; SSE-NEXT: shrdq %cl, %r14, %rbx
+; SSE-NEXT: movq -112(%rsp,%rsi), %r12
+; SSE-NEXT: movq %r12, %r14
+; SSE-NEXT: shrdq %cl, %r15, %r14
+; SSE-NEXT: movq -120(%rsp,%rsi), %r13
+; SSE-NEXT: movq %r13, %r15
+; SSE-NEXT: shrdq %cl, %r12, %r15
; SSE-NEXT: movq -128(%rsp,%rsi), %rsi
; SSE-NEXT: shrq %cl, %rdx
; SSE-NEXT: # kill: def $cl killed $cl killed $ecx
-; SSE-NEXT: shrdq %cl, %r15, %rsi
-; SSE-NEXT: jmp .LBB15_3
-; SSE-NEXT: .LBB15_1:
-; SSE-NEXT: xorl %esi, %esi
-; SSE-NEXT: xorl %ebx, %ebx
-; SSE-NEXT: xorl %r11d, %r11d
-; SSE-NEXT: xorl %r10d, %r10d
-; SSE-NEXT: xorl %r9d, %r9d
-; SSE-NEXT: xorl %r8d, %r8d
-; SSE-NEXT: xorl %edi, %edi
-; SSE-NEXT: xorl %edx, %edx
-; SSE-NEXT: .LBB15_3: # %select.end
-; SSE-NEXT: movq %rsi, (%rax)
-; SSE-NEXT: movq %rbx, 8(%rax)
-; SSE-NEXT: movq %r11, 16(%rax)
-; SSE-NEXT: movq %r10, 24(%rax)
-; SSE-NEXT: movq %r9, 32(%rax)
-; SSE-NEXT: movq %r8, 40(%rax)
-; SSE-NEXT: movq %rdi, 48(%rax)
-; SSE-NEXT: movq %rdx, 56(%rax)
+; SSE-NEXT: shrdq %cl, %r13, %rsi
+; SSE-NEXT: xorl %ecx, %ecx
+; SSE-NEXT: orq %rax, %r10
+; SSE-NEXT: cmoveq %rcx, %r15
+; SSE-NEXT: cmoveq %rcx, %r14
+; SSE-NEXT: cmoveq %rcx, %rbx
+; SSE-NEXT: cmoveq %rcx, %r11
+; SSE-NEXT: cmoveq %rcx, %r9
+; SSE-NEXT: cmoveq %rcx, %r8
+; SSE-NEXT: cmoveq %rcx, %rsi
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: cmoveq %rcx, %rdx
+; SSE-NEXT: movq %rdx, 56(%rdi)
+; SSE-NEXT: movq %r8, 48(%rdi)
+; SSE-NEXT: movq %r9, 40(%rdi)
+; SSE-NEXT: movq %r11, 32(%rdi)
+; SSE-NEXT: movq %rbx, 24(%rdi)
+; SSE-NEXT: movq %r14, 16(%rdi)
+; SSE-NEXT: movq %r15, 8(%rdi)
+; SSE-NEXT: movq %rsi, (%rdi)
; SSE-NEXT: popq %rbx
+; SSE-NEXT: popq %r12
+; SSE-NEXT: popq %r13
; SSE-NEXT: popq %r14
; SSE-NEXT: popq %r15
; SSE-NEXT: retq
;
; AVX2-LABEL: isolate_msb_i512:
; AVX2: # %bb.0:
+; AVX2-NEXT: pushq %rbp
; AVX2-NEXT: pushq %r15
; AVX2-NEXT: pushq %r14
+; AVX2-NEXT: pushq %r13
+; AVX2-NEXT: pushq %r12
; AVX2-NEXT: pushq %rbx
-; AVX2-NEXT: movq %rdi, %rax
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r11
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rdi
-; AVX2-NEXT: movq %r8, %rbx
-; AVX2-NEXT: orq %r10, %rbx
-; AVX2-NEXT: movq %rdx, %r14
-; AVX2-NEXT: orq %rdi, %r14
-; AVX2-NEXT: orq %rbx, %r14
-; AVX2-NEXT: movq %rcx, %rbx
-; AVX2-NEXT: orq %r11, %rbx
-; AVX2-NEXT: movq %rsi, %r15
-; AVX2-NEXT: orq %r9, %r15
-; AVX2-NEXT: orq %rbx, %r15
-; AVX2-NEXT: orq %r14, %r15
-; AVX2-NEXT: je .LBB15_1
-; AVX2-NEXT: # %bb.2: # %select.false.sink
-; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: lzcntq %r10, %rbx
-; AVX2-NEXT: xorl %r14d, %r14d
-; AVX2-NEXT: lzcntq %r11, %r14
-; AVX2-NEXT: addq $64, %r14
-; AVX2-NEXT: testq %r10, %r10
-; AVX2-NEXT: cmovneq %rbx, %r14
-; AVX2-NEXT: xorl %r15d, %r15d
-; AVX2-NEXT: lzcntq %rdi, %r15
-; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: lzcntq %r9, %rbx
-; AVX2-NEXT: addq $64, %rbx
-; AVX2-NEXT: testq %rdi, %rdi
-; AVX2-NEXT: cmovneq %r15, %rbx
-; AVX2-NEXT: subq $-128, %rbx
-; AVX2-NEXT: movq %r11, %r15
-; AVX2-NEXT: orq %r10, %r15
-; AVX2-NEXT: cmovneq %r14, %rbx
-; AVX2-NEXT: xorl %r14d, %r14d
-; AVX2-NEXT: lzcntq %r8, %r14
-; AVX2-NEXT: xorl %r15d, %r15d
-; AVX2-NEXT: lzcntq %rcx, %r15
-; AVX2-NEXT: addq $64, %r15
+; AVX2-NEXT: pushq %rax
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r14
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r15
+; AVX2-NEXT: movq %r8, %rax
+; AVX2-NEXT: orq %r15, %rax
+; AVX2-NEXT: movq %rdx, %r10
+; AVX2-NEXT: orq %rbx, %r10
+; AVX2-NEXT: orq %rax, %r10
+; AVX2-NEXT: movq %rcx, %rax
+; AVX2-NEXT: orq %r14, %rax
+; AVX2-NEXT: movq %rsi, %r11
+; AVX2-NEXT: orq %r9, %r11
+; AVX2-NEXT: orq %rax, %r11
+; AVX2-NEXT: xorl %eax, %eax
+; AVX2-NEXT: lzcntq %r15, %rax
+; AVX2-NEXT: xorl %r13d, %r13d
+; AVX2-NEXT: lzcntq %r14, %r13
+; AVX2-NEXT: addq $64, %r13
+; AVX2-NEXT: testq %r15, %r15
+; AVX2-NEXT: cmovneq %rax, %r13
+; AVX2-NEXT: xorl %eax, %eax
+; AVX2-NEXT: lzcntq %rbx, %rax
+; AVX2-NEXT: xorl %r12d, %r12d
+; AVX2-NEXT: lzcntq %r9, %r12
+; AVX2-NEXT: addq $64, %r12
+; AVX2-NEXT: testq %rbx, %rbx
+; AVX2-NEXT: cmovneq %rax, %r12
+; AVX2-NEXT: subq $-128, %r12
+; AVX2-NEXT: movq %r14, %rax
+; AVX2-NEXT: orq %r15, %rax
+; AVX2-NEXT: cmovneq %r13, %r12
+; AVX2-NEXT: xorl %eax, %eax
+; AVX2-NEXT: lzcntq %r8, %rax
+; AVX2-NEXT: xorl %r13d, %r13d
+; AVX2-NEXT: lzcntq %rcx, %r13
+; AVX2-NEXT: addq $64, %r13
; AVX2-NEXT: testq %r8, %r8
-; AVX2-NEXT: cmovneq %r14, %r15
-; AVX2-NEXT: xorl %r14d, %r14d
-; AVX2-NEXT: lzcntq %rdx, %r14
-; AVX2-NEXT: lzcntq %rsi, %rsi
-; AVX2-NEXT: addq $64, %rsi
+; AVX2-NEXT: cmovneq %rax, %r13
+; AVX2-NEXT: xorl %ebp, %ebp
+; AVX2-NEXT: lzcntq %rdx, %rbp
+; AVX2-NEXT: xorl %eax, %eax
+; AVX2-NEXT: lzcntq %rsi, %rax
+; AVX2-NEXT: addq $64, %rax
; AVX2-NEXT: testq %rdx, %rdx
-; AVX2-NEXT: cmovneq %r14, %rsi
-; AVX2-NEXT: subq $-128, %rsi
+; AVX2-NEXT: cmovneq %rbp, %rax
+; AVX2-NEXT: subq $-128, %rax
; AVX2-NEXT: orq %r8, %rcx
-; AVX2-NEXT: cmovneq %r15, %rsi
-; AVX2-NEXT: addq $256, %rsi # imm = 0x100
-; AVX2-NEXT: orq %r11, %r9
-; AVX2-NEXT: orq %r10, %rdi
-; AVX2-NEXT: orq %r9, %rdi
-; AVX2-NEXT: cmovneq %rbx, %rsi
+; AVX2-NEXT: cmovneq %r13, %rax
+; AVX2-NEXT: addq $256, %rax # imm = 0x100
+; AVX2-NEXT: orq %r15, %rbx
+; AVX2-NEXT: orq %r14, %r9
+; AVX2-NEXT: orq %rbx, %r9
+; AVX2-NEXT: cmovneq %r12, %rax
; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movl %esi, %ecx
+; AVX2-NEXT: movl %eax, %ecx
; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: shrl $3, %esi
-; AVX2-NEXT: andl $56, %esi
-; AVX2-NEXT: movq -72(%rsp,%rsi), %rbx
-; AVX2-NEXT: movq -80(%rsp,%rsi), %r8
+; AVX2-NEXT: shrl $3, %eax
+; AVX2-NEXT: andl $56, %eax
+; AVX2-NEXT: movq -72(%rsp,%rax), %rbx
+; AVX2-NEXT: movq -80(%rsp,%rax), %r8
; AVX2-NEXT: movq %r8, %rdx
; AVX2-NEXT: shrdq %cl, %rbx, %rdx
-; AVX2-NEXT: movq -88(%rsp,%rsi), %r9
-; AVX2-NEXT: movq %r9, %rdi
-; AVX2-NEXT: shrdq %cl, %r8, %rdi
-; AVX2-NEXT: movq -96(%rsp,%rsi), %r10
-; AVX2-NEXT: movq %r10, %r8
+; AVX2-NEXT: movq -88(%rsp,%rax), %r9
+; AVX2-NEXT: movq %r9, %rsi
+; AVX2-NEXT: shrdq %cl, %r8, %rsi
+; AVX2-NEXT: movq -96(%rsp,%rax), %r14
+; AVX2-NEXT: movq %r14, %r8
; AVX2-NEXT: shrdq %cl, %r9, %r8
-; AVX2-NEXT: movq -104(%rsp,%rsi), %r11
-; AVX2-NEXT: movq %r11, %r9
-; AVX2-NEXT: shrdq %cl, %r10, %r9
-; AVX2-NEXT: movq -112(%rsp,%rsi), %r14
-; AVX2-NEXT: movq %r14, %r10
-; AVX2-NEXT: shrdq %cl, %r11, %r10
-; AVX2-NEXT: movq -120(%rsp,%rsi), %r15
-; AVX2-NEXT: movq %r15, %r11
-; AVX2-NEXT: shrdq %cl, %r14, %r11
-; AVX2-NEXT: movq -128(%rsp,%rsi), %rsi
-; AVX2-NEXT: shrxq %rcx, %rbx, %rbx
-; AVX2-NEXT: # kill: def $cl killed $cl killed $rcx
-; AVX2-NEXT: shrdq %cl, %r15, %rsi
-; AVX2-NEXT: jmp .LBB15_3
-; AVX2-NEXT: .LBB15_1:
-; AVX2-NEXT: xorl %esi, %esi
-; AVX2-NEXT: xorl %r11d, %r11d
-; AVX2-NEXT: xorl %r10d, %r10d
-; AVX2-NEXT: xorl %r9d, %r9d
-; AVX2-NEXT: xorl %r8d, %r8d
+; AVX2-NEXT: movq -104(%rsp,%rax), %r15
+; AVX2-NEXT: movq %r15, %r9
+; AVX2-NEXT: shrdq %cl, %r14, %r9
+; AVX2-NEXT: movq -112(%rsp,%rax), %r13
+; AVX2-NEXT: movq %r13, %r14
+; AVX2-NEXT: shrdq %cl, %r15, %r14
+; AVX2-NEXT: movq -128(%rsp,%rax), %r15
+; AVX2-NEXT: movq -120(%rsp,%rax), %rax
+; AVX2-NEXT: movq %rax, %r12
+; AVX2-NEXT: shrdq %cl, %r13, %r12
+; AVX2-NEXT: shrdq %cl, %rax, %r15
+; AVX2-NEXT: movq %rdi, %rax
; AVX2-NEXT: xorl %edi, %edi
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: .LBB15_3: # %select.end
-; AVX2-NEXT: movq %rsi, (%rax)
-; AVX2-NEXT: movq %r11, 8(%rax)
-; AVX2-NEXT: movq %r10, 16(%rax)
-; AVX2-NEXT: movq %r9, 24(%rax)
-; AVX2-NEXT: movq %r8, 32(%rax)
-; AVX2-NEXT: movq %rdi, 40(%rax)
+; AVX2-NEXT: orq %r10, %r11
+; AVX2-NEXT: shrxq %rcx, %rbx, %rcx
+; AVX2-NEXT: cmoveq %rdi, %r12
+; AVX2-NEXT: cmoveq %rdi, %r14
+; AVX2-NEXT: cmoveq %rdi, %r9
+; AVX2-NEXT: cmoveq %rdi, %r8
+; AVX2-NEXT: cmoveq %rdi, %rsi
+; AVX2-NEXT: cmoveq %rdi, %rdx
+; AVX2-NEXT: cmoveq %rdi, %r15
+; AVX2-NEXT: cmoveq %rdi, %rcx
+; AVX2-NEXT: movq %rcx, 56(%rax)
; AVX2-NEXT: movq %rdx, 48(%rax)
-; AVX2-NEXT: movq %rbx, 56(%rax)
+; AVX2-NEXT: movq %rsi, 40(%rax)
+; AVX2-NEXT: movq %r8, 32(%rax)
+; AVX2-NEXT: movq %r9, 24(%rax)
+; AVX2-NEXT: movq %r14, 16(%rax)
+; AVX2-NEXT: movq %r12, 8(%rax)
+; AVX2-NEXT: movq %r15, (%rax)
+; AVX2-NEXT: addq $8, %rsp
; AVX2-NEXT: popq %rbx
+; AVX2-NEXT: popq %r12
+; AVX2-NEXT: popq %r13
; AVX2-NEXT: popq %r14
; AVX2-NEXT: popq %r15
+; AVX2-NEXT: popq %rbp
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
; AVX512F-LABEL: isolate_msb_i512:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: pushq %r15
-; AVX512F-NEXT: pushq %r14
-; AVX512F-NEXT: pushq %rbx
; AVX512F-NEXT: movq %rdi, %rax
; AVX512F-NEXT: movq {{[0-9]+}}(%rsp), %rdi
-; AVX512F-NEXT: movq {{[0-9]+}}(%rsp), %r10
-; AVX512F-NEXT: movq {{[0-9]+}}(%rsp), %r11
-; AVX512F-NEXT: movq %r8, %rbx
-; AVX512F-NEXT: orq %rdi, %rbx
-; AVX512F-NEXT: movq %rdx, %r14
-; AVX512F-NEXT: orq %r11, %r14
-; AVX512F-NEXT: orq %rbx, %r14
-; AVX512F-NEXT: movq %rcx, %rbx
-; AVX512F-NEXT: orq %r10, %rbx
-; AVX512F-NEXT: movq %rsi, %r15
-; AVX512F-NEXT: orq %r9, %r15
-; AVX512F-NEXT: orq %rbx, %r15
-; AVX512F-NEXT: orq %r14, %r15
-; AVX512F-NEXT: je .LBB15_1
-; AVX512F-NEXT: # %bb.2: # %select.false.sink
-; AVX512F-NEXT: vmovq %rsi, %xmm0
-; AVX512F-NEXT: vmovq %rdx, %xmm1
-; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX512F-NEXT: vmovq %rcx, %xmm1
-; AVX512F-NEXT: vmovq %r8, %xmm2
-; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX512F-NEXT: vmovq %rdx, %xmm0
+; AVX512F-NEXT: orq %rdi, %rdx
+; AVX512F-NEXT: vmovq %r8, %xmm1
+; AVX512F-NEXT: orq {{[0-9]+}}(%rsp), %r8
+; AVX512F-NEXT: orq %rdx, %r8
+; AVX512F-NEXT: vmovq %rsi, %xmm2
+; AVX512F-NEXT: vmovq %rcx, %xmm3
+; AVX512F-NEXT: orq {{[0-9]+}}(%rsp), %rcx
+; AVX512F-NEXT: orq %r9, %rsi
+; AVX512F-NEXT: orq %rsi, %rcx
+; AVX512F-NEXT: xorl %edx, %edx
+; AVX512F-NEXT: orq %r8, %rcx
+; AVX512F-NEXT: sete %dl
+; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
; AVX512F-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
-; AVX512F-NEXT: vmovq %r9, %xmm1
-; AVX512F-NEXT: vmovq %r11, %xmm2
-; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX512F-NEXT: vmovq %r10, %xmm2
+; AVX512F-NEXT: vpshufd {{.*#+}} xmm1 = mem[2,3,0,1]
+; AVX512F-NEXT: vmovq %r9, %xmm2
; AVX512F-NEXT: vmovq %rdi, %xmm3
; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
-; AVX512F-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
+; AVX512F-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
; AVX512F-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
-; AVX512F-NEXT: vplzcntq %zmm0, %zmm1
-; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1 # [0,64,128,192,256,320,384,448]
; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
-; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
+; AVX512F-NEXT: vplzcntq %zmm0, %zmm0
+; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,64,128,192,256,320,384,448]
+; AVX512F-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
; AVX512F-NEXT: vmovq %xmm0, %rcx
-; AVX512F-NEXT: movabsq $-9223372036854775808, %rdx # imm = 0x8000000000000000
-; AVX512F-NEXT: shrxq %rcx, %rdx, %rdx
+; AVX512F-NEXT: movabsq $-9223372036854775808, %rsi # imm = 0x8000000000000000
+; AVX512F-NEXT: shrxq %rcx, %rsi, %rsi
; AVX512F-NEXT: shrl $6, %ecx
-; AVX512F-NEXT: movl $128, %esi
-; AVX512F-NEXT: shrxq %rcx, %rsi, %rcx
+; AVX512F-NEXT: movl $128, %edi
+; AVX512F-NEXT: shrxq %rcx, %rdi, %rcx
; AVX512F-NEXT: kmovw %ecx, %k1
-; AVX512F-NEXT: vpbroadcastq %rdx, %zmm0 {%k1} {z}
-; AVX512F-NEXT: vextracti32x4 $3, %zmm0, %xmm1
-; AVX512F-NEXT: vpextrq $1, %xmm1, %rcx
-; AVX512F-NEXT: vmovq %xmm1, %rdx
-; AVX512F-NEXT: vextracti32x4 $2, %zmm0, %xmm1
-; AVX512F-NEXT: vpextrq $1, %xmm1, %rsi
-; AVX512F-NEXT: vmovq %xmm1, %r8
-; AVX512F-NEXT: vpextrq $1, %xmm0, %r9
-; AVX512F-NEXT: vmovq %xmm0, %rdi
-; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm0
-; AVX512F-NEXT: vpextrq $1, %xmm0, %r10
-; AVX512F-NEXT: vmovq %xmm0, %r11
-; AVX512F-NEXT: jmp .LBB15_3
-; AVX512F-NEXT: .LBB15_1:
-; AVX512F-NEXT: xorl %edi, %edi
-; AVX512F-NEXT: xorl %r9d, %r9d
-; AVX512F-NEXT: xorl %r11d, %r11d
-; AVX512F-NEXT: xorl %r10d, %r10d
-; AVX512F-NEXT: xorl %r8d, %r8d
-; AVX512F-NEXT: xorl %esi, %esi
-; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: xorl %ecx, %ecx
-; AVX512F-NEXT: .LBB15_3: # %select.end
-; AVX512F-NEXT: movq %rdi, (%rax)
-; AVX512F-NEXT: movq %r9, 8(%rax)
-; AVX512F-NEXT: movq %r11, 16(%rax)
-; AVX512F-NEXT: movq %r10, 24(%rax)
-; AVX512F-NEXT: movq %r8, 32(%rax)
-; AVX512F-NEXT: movq %rsi, 40(%rax)
-; AVX512F-NEXT: movq %rdx, 48(%rax)
-; AVX512F-NEXT: movq %rcx, 56(%rax)
-; AVX512F-NEXT: popq %rbx
-; AVX512F-NEXT: popq %r14
-; AVX512F-NEXT: popq %r15
+; AVX512F-NEXT: vpbroadcastq %rsi, %zmm0 {%k1} {z}
+; AVX512F-NEXT: negl %edx
+; AVX512F-NEXT: kmovw %edx, %k0
+; AVX512F-NEXT: knotw %k0, %k1
+; AVX512F-NEXT: vmovdqa32 %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT: vmovdqu64 %zmm0, (%rax)
; AVX512F-NEXT: retq
;
; AVX512VL-LABEL: isolate_msb_i512:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: pushq %r15
-; AVX512VL-NEXT: pushq %r14
-; AVX512VL-NEXT: pushq %rbx
; AVX512VL-NEXT: movq %rdi, %rax
; AVX512VL-NEXT: movq {{[0-9]+}}(%rsp), %rdi
-; AVX512VL-NEXT: movq {{[0-9]+}}(%rsp), %r10
-; AVX512VL-NEXT: movq {{[0-9]+}}(%rsp), %r11
-; AVX512VL-NEXT: movq %r8, %rbx
-; AVX512VL-NEXT: orq %rdi, %rbx
-; AVX512VL-NEXT: movq %rdx, %r14
-; AVX512VL-NEXT: orq %r11, %r14
-; AVX512VL-NEXT: orq %rbx, %r14
-; AVX512VL-NEXT: movq %rcx, %rbx
-; AVX512VL-NEXT: orq %r10, %rbx
-; AVX512VL-NEXT: movq %rsi, %r15
-; AVX512VL-NEXT: orq %r9, %r15
-; AVX512VL-NEXT: orq %rbx, %r15
-; AVX512VL-NEXT: orq %r14, %r15
-; AVX512VL-NEXT: je .LBB15_1
-; AVX512VL-NEXT: # %bb.2: # %select.false.sink
-; AVX512VL-NEXT: vmovq %rsi, %xmm0
-; AVX512VL-NEXT: vmovq %rdx, %xmm1
-; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX512VL-NEXT: vmovq %rcx, %xmm1
-; AVX512VL-NEXT: vmovq %r8, %xmm2
-; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX512VL-NEXT: vmovq %rdx, %xmm0
+; AVX512VL-NEXT: orq %rdi, %rdx
+; AVX512VL-NEXT: vmovq %r8, %xmm1
+; AVX512VL-NEXT: orq {{[0-9]+}}(%rsp), %r8
+; AVX512VL-NEXT: orq %rdx, %r8
+; AVX512VL-NEXT: vmovq %rsi, %xmm2
+; AVX512VL-NEXT: orq %r9, %rsi
+; AVX512VL-NEXT: vmovq %rcx, %xmm3
+; AVX512VL-NEXT: orq {{[0-9]+}}(%rsp), %rcx
+; AVX512VL-NEXT: orq %rsi, %rcx
+; AVX512VL-NEXT: xorl %edx, %edx
+; AVX512VL-NEXT: orq %r8, %rcx
+; AVX512VL-NEXT: sete %dl
+; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; AVX512VL-NEXT: vpshufd {{.*#+}} xmm2 = mem[2,3,0,1]
; AVX512VL-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
; AVX512VL-NEXT: vmovq %r9, %xmm1
-; AVX512VL-NEXT: vmovq %r11, %xmm2
-; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX512VL-NEXT: vmovq %r10, %xmm2
; AVX512VL-NEXT: vmovq %rdi, %xmm3
-; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
+; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm3[0],xmm1[0]
; AVX512VL-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
; AVX512VL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
@@ -4464,84 +4418,45 @@ define i512 @isolate_msb_i512(i512 %a0, i512 %idx) nounwind {
; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,64,128,192,256,320,384,448]
; AVX512VL-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
; AVX512VL-NEXT: vmovq %xmm0, %rcx
-; AVX512VL-NEXT: movabsq $-9223372036854775808, %rdx # imm = 0x8000000000000000
-; AVX512VL-NEXT: shrxq %rcx, %rdx, %rdx
+; AVX512VL-NEXT: movabsq $-9223372036854775808, %rsi # imm = 0x8000000000000000
+; AVX512VL-NEXT: shrxq %rcx, %rsi, %rsi
; AVX512VL-NEXT: shrl $6, %ecx
-; AVX512VL-NEXT: movl $128, %esi
-; AVX512VL-NEXT: shrxq %rcx, %rsi, %rcx
+; AVX512VL-NEXT: movl $128, %edi
+; AVX512VL-NEXT: shrxq %rcx, %rdi, %rcx
; AVX512VL-NEXT: kmovd %ecx, %k1
-; AVX512VL-NEXT: vpbroadcastq %rdx, %zmm0 {%k1} {z}
-; AVX512VL-NEXT: vextracti32x4 $3, %zmm0, %xmm1
-; AVX512VL-NEXT: vpextrq $1, %xmm1, %rcx
-; AVX512VL-NEXT: vmovq %xmm1, %rdx
-; AVX512VL-NEXT: vextracti32x4 $2, %zmm0, %xmm1
-; AVX512VL-NEXT: vpextrq $1, %xmm1, %rsi
-; AVX512VL-NEXT: vpextrq $1, %xmm0, %r8
-; AVX512VL-NEXT: vmovq %xmm1, %r9
-; AVX512VL-NEXT: vmovq %xmm0, %rdi
-; AVX512VL-NEXT: vextracti128 $1, %ymm0, %xmm0
-; AVX512VL-NEXT: vpextrq $1, %xmm0, %r10
-; AVX512VL-NEXT: vmovq %xmm0, %r11
-; AVX512VL-NEXT: jmp .LBB15_3
-; AVX512VL-NEXT: .LBB15_1:
-; AVX512VL-NEXT: xorl %edi, %edi
-; AVX512VL-NEXT: xorl %r8d, %r8d
-; AVX512VL-NEXT: xorl %r11d, %r11d
-; AVX512VL-NEXT: xorl %r10d, %r10d
-; AVX512VL-NEXT: xorl %r9d, %r9d
-; AVX512VL-NEXT: xorl %esi, %esi
-; AVX512VL-NEXT: xorl %edx, %edx
-; AVX512VL-NEXT: xorl %ecx, %ecx
-; AVX512VL-NEXT: .LBB15_3: # %select.end
-; AVX512VL-NEXT: movq %rdi, (%rax)
-; AVX512VL-NEXT: movq %r8, 8(%rax)
-; AVX512VL-NEXT: movq %r11, 16(%rax)
-; AVX512VL-NEXT: movq %r10, 24(%rax)
-; AVX512VL-NEXT: movq %r9, 32(%rax)
-; AVX512VL-NEXT: movq %rsi, 40(%rax)
-; AVX512VL-NEXT: movq %rdx, 48(%rax)
-; AVX512VL-NEXT: movq %rcx, 56(%rax)
-; AVX512VL-NEXT: popq %rbx
-; AVX512VL-NEXT: popq %r14
-; AVX512VL-NEXT: popq %r15
+; AVX512VL-NEXT: vpbroadcastq %rsi, %zmm0 {%k1} {z}
+; AVX512VL-NEXT: negl %edx
+; AVX512VL-NEXT: kmovd %edx, %k0
+; AVX512VL-NEXT: knotw %k0, %k1
+; AVX512VL-NEXT: vmovdqa32 %zmm0, %zmm0 {%k1} {z}
+; AVX512VL-NEXT: vmovdqu64 %zmm0, (%rax)
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VBMI-LABEL: isolate_msb_i512:
; AVX512VBMI: # %bb.0:
-; AVX512VBMI-NEXT: pushq %r15
-; AVX512VBMI-NEXT: pushq %r14
-; AVX512VBMI-NEXT: pushq %rbx
; AVX512VBMI-NEXT: movq %rdi, %rax
; AVX512VBMI-NEXT: movq {{[0-9]+}}(%rsp), %rdi
-; AVX512VBMI-NEXT: movq {{[0-9]+}}(%rsp), %r10
-; AVX512VBMI-NEXT: movq {{[0-9]+}}(%rsp), %r11
-; AVX512VBMI-NEXT: movq %r8, %rbx
-; AVX512VBMI-NEXT: orq %rdi, %rbx
-; AVX512VBMI-NEXT: movq %rdx, %r14
-; AVX512VBMI-NEXT: orq %r11, %r14
-; AVX512VBMI-NEXT: orq %rbx, %r14
-; AVX512VBMI-NEXT: movq %rcx, %rbx
-; AVX512VBMI-NEXT: orq %r10, %rbx
-; AVX512VBMI-NEXT: movq %rsi, %r15
-; AVX512VBMI-NEXT: orq %r9, %r15
-; AVX512VBMI-NEXT: orq %rbx, %r15
-; AVX512VBMI-NEXT: orq %r14, %r15
-; AVX512VBMI-NEXT: je .LBB15_1
-; AVX512VBMI-NEXT: # %bb.2: # %select.false.sink
-; AVX512VBMI-NEXT: vmovq %rsi, %xmm0
-; AVX512VBMI-NEXT: vmovq %rdx, %xmm1
-; AVX512VBMI-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX512VBMI-NEXT: vmovq %rcx, %xmm1
-; AVX512VBMI-NEXT: vmovq %r8, %xmm2
-; AVX512VBMI-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX512VBMI-NEXT: vmovq %rdx, %xmm0
+; AVX512VBMI-NEXT: orq %rdi, %rdx
+; AVX512VBMI-NEXT: vmovq %r8, %xmm1
+; AVX512VBMI-NEXT: orq {{[0-9]+}}(%rsp), %r8
+; AVX512VBMI-NEXT: orq %rdx, %r8
+; AVX512VBMI-NEXT: vmovq %rsi, %xmm2
+; AVX512VBMI-NEXT: orq %r9, %rsi
+; AVX512VBMI-NEXT: vmovq %rcx, %xmm3
+; AVX512VBMI-NEXT: orq {{[0-9]+}}(%rsp), %rcx
+; AVX512VBMI-NEXT: orq %rsi, %rcx
+; AVX512VBMI-NEXT: xorl %edx, %edx
+; AVX512VBMI-NEXT: orq %r8, %rcx
+; AVX512VBMI-NEXT: sete %dl
+; AVX512VBMI-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; AVX512VBMI-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; AVX512VBMI-NEXT: vpshufd {{.*#+}} xmm2 = mem[2,3,0,1]
; AVX512VBMI-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
; AVX512VBMI-NEXT: vmovq %r9, %xmm1
-; AVX512VBMI-NEXT: vmovq %r11, %xmm2
-; AVX512VBMI-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX512VBMI-NEXT: vmovq %r10, %xmm2
; AVX512VBMI-NEXT: vmovq %rdi, %xmm3
-; AVX512VBMI-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
+; AVX512VBMI-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm3[0],xmm1[0]
; AVX512VBMI-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
; AVX512VBMI-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
; AVX512VBMI-NEXT: vptestmq %zmm0, %zmm0, %k1
@@ -4549,46 +4464,18 @@ define i512 @isolate_msb_i512(i512 %a0, i512 %idx) nounwind {
; AVX512VBMI-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,64,128,192,256,320,384,448]
; AVX512VBMI-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
; AVX512VBMI-NEXT: vmovq %xmm0, %rcx
-; AVX512VBMI-NEXT: movabsq $-9223372036854775808, %rdx # imm = 0x8000000000000000
-; AVX512VBMI-NEXT: shrxq %rcx, %rdx, %rdx
+; AVX512VBMI-NEXT: movabsq $-9223372036854775808, %rsi # imm = 0x8000000000000000
+; AVX512VBMI-NEXT: shrxq %rcx, %rsi, %rsi
; AVX512VBMI-NEXT: shrl $6, %ecx
-; AVX512VBMI-NEXT: movl $128, %esi
-; AVX512VBMI-NEXT: shrxq %rcx, %rsi, %rcx
+; AVX512VBMI-NEXT: movl $128, %edi
+; AVX512VBMI-NEXT: shrxq %rcx, %rdi, %rcx
; AVX512VBMI-NEXT: kmovd %ecx, %k1
-; AVX512VBMI-NEXT: vpbroadcastq %rdx, %zmm0 {%k1} {z}
-; AVX512VBMI-NEXT: vextracti32x4 $3, %zmm0, %xmm1
-; AVX512VBMI-NEXT: vpextrq $1, %xmm1, %rcx
-; AVX512VBMI-NEXT: vmovq %xmm1, %rdx
-; AVX512VBMI-NEXT: vextracti32x4 $2, %zmm0, %xmm1
-; AVX512VBMI-NEXT: vpextrq $1, %xmm1, %rsi
-; AVX512VBMI-NEXT: vpextrq $1, %xmm0, %r8
-; AVX512VBMI-NEXT: vmovq %xmm1, %r9
-; AVX512VBMI-NEXT: vmovq %xmm0, %rdi
-; AVX512VBMI-NEXT: vextracti128 $1, %ymm0, %xmm0
-; AVX512VBMI-NEXT: vpextrq $1, %xmm0, %r10
-; AVX512VBMI-NEXT: vmovq %xmm0, %r11
-; AVX512VBMI-NEXT: jmp .LBB15_3
-; AVX512VBMI-NEXT: .LBB15_1:
-; AVX512VBMI-NEXT: xorl %edi, %edi
-; AVX512VBMI-NEXT: xorl %r8d, %r8d
-; AVX512VBMI-NEXT: xorl %r11d, %r11d
-; AVX512VBMI-NEXT: xorl %r10d, %r10d
-; AVX512VBMI-NEXT: xorl %r9d, %r9d
-; AVX512VBMI-NEXT: xorl %esi, %esi
-; AVX512VBMI-NEXT: xorl %edx, %edx
-; AVX512VBMI-NEXT: xorl %ecx, %ecx
-; AVX512VBMI-NEXT: .LBB15_3: # %select.end
-; AVX512VBMI-NEXT: movq %rdi, (%rax)
-; AVX512VBMI-NEXT: movq %r8, 8(%rax)
-; AVX512VBMI-NEXT: movq %r11, 16(%rax)
-; AVX512VBMI-NEXT: movq %r10, 24(%rax)
-; AVX512VBMI-NEXT: movq %r9, 32(%rax)
-; AVX512VBMI-NEXT: movq %rsi, 40(%rax)
-; AVX512VBMI-NEXT: movq %rdx, 48(%rax)
-; AVX512VBMI-NEXT: movq %rcx, 56(%rax)
-; AVX512VBMI-NEXT: popq %rbx
-; AVX512VBMI-NEXT: popq %r14
-; AVX512VBMI-NEXT: popq %r15
+; AVX512VBMI-NEXT: vpbroadcastq %rsi, %zmm0 {%k1} {z}
+; AVX512VBMI-NEXT: negl %edx
+; AVX512VBMI-NEXT: kmovd %edx, %k0
+; AVX512VBMI-NEXT: knotw %k0, %k1
+; AVX512VBMI-NEXT: vmovdqa32 %zmm0, %zmm0 {%k1} {z}
+; AVX512VBMI-NEXT: vmovdqu64 %zmm0, (%rax)
; AVX512VBMI-NEXT: vzeroupper
; AVX512VBMI-NEXT: retq
%eqz = icmp eq i512 %a0, 0
@@ -4604,71 +4491,73 @@ define i512 @isolate_msb_i512_vector(<8 x i64> %v0, i512 %idx) nounwind {
; SSE2: # %bb.0:
; SSE2-NEXT: pushq %r15
; SSE2-NEXT: pushq %r14
+; SSE2-NEXT: pushq %r12
; SSE2-NEXT: pushq %rbx
-; SSE2-NEXT: movq %rdi, %rax
+; SSE2-NEXT: pushq %rax
+; SSE2-NEXT: movq %xmm0, %rdx
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; SSE2-NEXT: movq %xmm4, %rcx
+; SSE2-NEXT: movq %xmm1, %r8
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
+; SSE2-NEXT: movq %xmm4, %rsi
+; SSE2-NEXT: movq %xmm2, %r9
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]
+; SSE2-NEXT: movq %xmm4, %r11
+; SSE2-NEXT: movq %xmm3, %r10
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
+; SSE2-NEXT: movq %xmm4, %rbx
; SSE2-NEXT: movdqa %xmm1, %xmm4
; SSE2-NEXT: por %xmm3, %xmm4
-; SSE2-NEXT: movdqa %xmm0, %xmm5
-; SSE2-NEXT: por %xmm2, %xmm5
-; SSE2-NEXT: por %xmm4, %xmm5
+; SSE2-NEXT: por %xmm2, %xmm0
+; SSE2-NEXT: por %xmm4, %xmm0
; SSE2-NEXT: pxor %xmm4, %xmm4
-; SSE2-NEXT: pcmpeqd %xmm4, %xmm5
-; SSE2-NEXT: movmskps %xmm5, %ecx
-; SSE2-NEXT: xorl $15, %ecx
-; SSE2-NEXT: je .LBB16_1
-; SSE2-NEXT: # %bb.2: # %select.false.sink
-; SSE2-NEXT: movq %xmm3, %rsi
-; SSE2-NEXT: movq %xmm2, %rcx
-; SSE2-NEXT: movq %xmm1, %r9
-; SSE2-NEXT: movq %xmm0, %rdx
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
-; SSE2-NEXT: movq %xmm3, %r8
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
-; SSE2-NEXT: movq %xmm2, %rdi
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
-; SSE2-NEXT: movq %xmm1, %r10
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
-; SSE2-NEXT: movq %xmm0, %rbx
-; SSE2-NEXT: bsrq %r8, %r11
-; SSE2-NEXT: xorq $63, %r11
-; SSE2-NEXT: bsrq %rsi, %r14
-; SSE2-NEXT: xorq $63, %r14
-; SSE2-NEXT: orq $64, %r14
-; SSE2-NEXT: testq %r8, %r8
-; SSE2-NEXT: cmovneq %r11, %r14
-; SSE2-NEXT: bsrq %rdi, %r15
-; SSE2-NEXT: xorq $63, %r15
-; SSE2-NEXT: bsrq %rcx, %r11
-; SSE2-NEXT: xorq $63, %r11
-; SSE2-NEXT: orq $64, %r11
-; SSE2-NEXT: testq %rdi, %rdi
-; SSE2-NEXT: cmovneq %r15, %r11
-; SSE2-NEXT: orq $128, %r11
-; SSE2-NEXT: movq %rsi, %r15
-; SSE2-NEXT: orq %r8, %r15
-; SSE2-NEXT: cmovneq %r14, %r11
-; SSE2-NEXT: bsrq %r10, %r14
-; SSE2-NEXT: xorq $63, %r14
-; SSE2-NEXT: bsrq %r9, %r15
-; SSE2-NEXT: xorq $63, %r15
-; SSE2-NEXT: orq $64, %r15
-; SSE2-NEXT: testq %r10, %r10
-; SSE2-NEXT: cmovneq %r14, %r15
+; SSE2-NEXT: pcmpeqd %xmm4, %xmm0
+; SSE2-NEXT: movmskps %xmm0, %eax
+; SSE2-NEXT: xorl $15, %eax
; SSE2-NEXT: bsrq %rbx, %r14
; SSE2-NEXT: xorq $63, %r14
+; SSE2-NEXT: bsrq %r10, %r10
+; SSE2-NEXT: xorq $63, %r10
+; SSE2-NEXT: orq $64, %r10
+; SSE2-NEXT: testq %rbx, %rbx
+; SSE2-NEXT: cmovneq %r14, %r10
+; SSE2-NEXT: bsrq %r11, %rbx
+; SSE2-NEXT: xorq $63, %rbx
+; SSE2-NEXT: bsrq %r9, %r9
+; SSE2-NEXT: xorq $63, %r9
+; SSE2-NEXT: orq $64, %r9
+; SSE2-NEXT: testq %r11, %r11
+; SSE2-NEXT: cmovneq %rbx, %r9
+; SSE2-NEXT: orq $128, %r9
+; SSE2-NEXT: por %xmm3, %xmm2
+; SSE2-NEXT: pcmpeqd %xmm4, %xmm3
+; SSE2-NEXT: movmskps %xmm3, %r11d
+; SSE2-NEXT: xorl $15, %r11d
+; SSE2-NEXT: cmovneq %r10, %r9
+; SSE2-NEXT: bsrq %rsi, %r10
+; SSE2-NEXT: xorq $63, %r10
+; SSE2-NEXT: bsrq %r8, %r8
+; SSE2-NEXT: xorq $63, %r8
+; SSE2-NEXT: orq $64, %r8
+; SSE2-NEXT: testq %rsi, %rsi
+; SSE2-NEXT: cmovneq %r10, %r8
+; SSE2-NEXT: bsrq %rcx, %rsi
+; SSE2-NEXT: xorq $63, %rsi
; SSE2-NEXT: bsrq %rdx, %rdx
; SSE2-NEXT: xorq $63, %rdx
; SSE2-NEXT: orq $64, %rdx
-; SSE2-NEXT: testq %rbx, %rbx
-; SSE2-NEXT: cmovneq %r14, %rdx
+; SSE2-NEXT: testq %rcx, %rcx
+; SSE2-NEXT: cmovneq %rsi, %rdx
; SSE2-NEXT: orq $128, %rdx
-; SSE2-NEXT: orq %r10, %r9
-; SSE2-NEXT: cmovneq %r15, %rdx
+; SSE2-NEXT: pcmpeqd %xmm4, %xmm1
+; SSE2-NEXT: movmskps %xmm1, %ecx
+; SSE2-NEXT: xorl $15, %ecx
+; SSE2-NEXT: cmovneq %r8, %rdx
; SSE2-NEXT: orq $256, %rdx # imm = 0x100
-; SSE2-NEXT: orq %rsi, %rcx
-; SSE2-NEXT: orq %r8, %rdi
-; SSE2-NEXT: orq %rcx, %rdi
-; SSE2-NEXT: cmovneq %r11, %rdx
+; SSE2-NEXT: pcmpeqd %xmm4, %xmm2
+; SSE2-NEXT: movmskps %xmm2, %ecx
+; SSE2-NEXT: xorl $15, %ecx
+; SSE2-NEXT: cmovneq %r9, %rdx
; SSE2-NEXT: movdqa %xmm4, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: movdqa %xmm4, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: movdqa %xmm4, -{{[0-9]+}}(%rsp)
@@ -4684,48 +4573,50 @@ define i512 @isolate_msb_i512_vector(<8 x i64> %v0, i512 %idx) nounwind {
; SSE2-NEXT: andl $56, %edx
; SSE2-NEXT: movq $0, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: movq -72(%rsp,%rdx), %rsi
-; SSE2-NEXT: movq -80(%rsp,%rdx), %r9
-; SSE2-NEXT: movq %r9, %rdi
-; SSE2-NEXT: shrdq %cl, %rsi, %rdi
-; SSE2-NEXT: movq -88(%rsp,%rdx), %r10
+; SSE2-NEXT: movq -80(%rsp,%rdx), %r10
; SSE2-NEXT: movq %r10, %r8
-; SSE2-NEXT: shrdq %cl, %r9, %r8
-; SSE2-NEXT: movq -96(%rsp,%rdx), %r11
+; SSE2-NEXT: shrdq %cl, %rsi, %r8
+; SSE2-NEXT: movq -88(%rsp,%rdx), %r11
; SSE2-NEXT: movq %r11, %r9
; SSE2-NEXT: shrdq %cl, %r10, %r9
-; SSE2-NEXT: movq -104(%rsp,%rdx), %rbx
+; SSE2-NEXT: movq -96(%rsp,%rdx), %rbx
; SSE2-NEXT: movq %rbx, %r10
; SSE2-NEXT: shrdq %cl, %r11, %r10
-; SSE2-NEXT: movq -112(%rsp,%rdx), %r14
+; SSE2-NEXT: movq -104(%rsp,%rdx), %r14
; SSE2-NEXT: movq %r14, %r11
; SSE2-NEXT: shrdq %cl, %rbx, %r11
-; SSE2-NEXT: movq -120(%rsp,%rdx), %r15
+; SSE2-NEXT: movq -112(%rsp,%rdx), %r15
; SSE2-NEXT: movq %r15, %rbx
; SSE2-NEXT: shrdq %cl, %r14, %rbx
+; SSE2-NEXT: movq -120(%rsp,%rdx), %r12
+; SSE2-NEXT: movq %r12, %r14
+; SSE2-NEXT: shrdq %cl, %r15, %r14
; SSE2-NEXT: movq -128(%rsp,%rdx), %rdx
; SSE2-NEXT: shrq %cl, %rsi
; SSE2-NEXT: # kill: def $cl killed $cl killed $ecx
-; SSE2-NEXT: shrdq %cl, %r15, %rdx
-; SSE2-NEXT: jmp .LBB16_3
-; SSE2-NEXT: .LBB16_1:
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: xorl %ebx, %ebx
-; SSE2-NEXT: xorl %r11d, %r11d
-; SSE2-NEXT: xorl %r10d, %r10d
-; SSE2-NEXT: xorl %r9d, %r9d
-; SSE2-NEXT: xorl %r8d, %r8d
-; SSE2-NEXT: xorl %edi, %edi
-; SSE2-NEXT: xorl %esi, %esi
-; SSE2-NEXT: .LBB16_3: # %select.end
-; SSE2-NEXT: movq %rdx, (%rax)
-; SSE2-NEXT: movq %rbx, 8(%rax)
-; SSE2-NEXT: movq %r11, 16(%rax)
-; SSE2-NEXT: movq %r10, 24(%rax)
-; SSE2-NEXT: movq %r9, 32(%rax)
-; SSE2-NEXT: movq %r8, 40(%rax)
-; SSE2-NEXT: movq %rdi, 48(%rax)
-; SSE2-NEXT: movq %rsi, 56(%rax)
+; SSE2-NEXT: shrdq %cl, %r12, %rdx
+; SSE2-NEXT: xorl %ecx, %ecx
+; SSE2-NEXT: testl %eax, %eax
+; SSE2-NEXT: cmoveq %rcx, %r14
+; SSE2-NEXT: cmoveq %rcx, %rbx
+; SSE2-NEXT: cmoveq %rcx, %r11
+; SSE2-NEXT: cmoveq %rcx, %r10
+; SSE2-NEXT: cmoveq %rcx, %r9
+; SSE2-NEXT: cmoveq %rcx, %r8
+; SSE2-NEXT: cmoveq %rcx, %rdx
+; SSE2-NEXT: movq %rdi, %rax
+; SSE2-NEXT: cmoveq %rcx, %rsi
+; SSE2-NEXT: movq %rsi, 56(%rdi)
+; SSE2-NEXT: movq %r8, 48(%rdi)
+; SSE2-NEXT: movq %r9, 40(%rdi)
+; SSE2-NEXT: movq %r10, 32(%rdi)
+; SSE2-NEXT: movq %r11, 24(%rdi)
+; SSE2-NEXT: movq %rbx, 16(%rdi)
+; SSE2-NEXT: movq %r14, 8(%rdi)
+; SSE2-NEXT: movq %rdx, (%rdi)
+; SSE2-NEXT: addq $8, %rsp
; SSE2-NEXT: popq %rbx
+; SSE2-NEXT: popq %r12
; SSE2-NEXT: popq %r14
; SSE2-NEXT: popq %r15
; SSE2-NEXT: retq
@@ -4735,120 +4626,112 @@ define i512 @isolate_msb_i512_vector(<8 x i64> %v0, i512 %idx) nounwind {
; SSE42-NEXT: pushq %r15
; SSE42-NEXT: pushq %r14
; SSE42-NEXT: pushq %rbx
-; SSE42-NEXT: movq %rdi, %rax
-; SSE42-NEXT: movdqa %xmm1, %xmm4
-; SSE42-NEXT: por %xmm3, %xmm4
-; SSE42-NEXT: movdqa %xmm0, %xmm5
-; SSE42-NEXT: por %xmm2, %xmm5
-; SSE42-NEXT: por %xmm4, %xmm5
-; SSE42-NEXT: ptest %xmm5, %xmm5
-; SSE42-NEXT: je .LBB16_1
-; SSE42-NEXT: # %bb.2: # %select.false.sink
-; SSE42-NEXT: pextrq $1, %xmm3, %rsi
-; SSE42-NEXT: movq %xmm3, %rdi
-; SSE42-NEXT: pextrq $1, %xmm2, %rcx
-; SSE42-NEXT: pextrq $1, %xmm1, %r9
-; SSE42-NEXT: movq %xmm2, %r8
-; SSE42-NEXT: movq %xmm1, %r10
-; SSE42-NEXT: pextrq $1, %xmm0, %rbx
-; SSE42-NEXT: bsrq %rsi, %rdx
-; SSE42-NEXT: xorq $63, %rdx
-; SSE42-NEXT: bsrq %rdi, %r14
-; SSE42-NEXT: xorq $63, %r14
-; SSE42-NEXT: orq $64, %r14
-; SSE42-NEXT: testq %rsi, %rsi
-; SSE42-NEXT: cmovneq %rdx, %r14
-; SSE42-NEXT: bsrq %rcx, %rdx
-; SSE42-NEXT: xorq $63, %rdx
-; SSE42-NEXT: bsrq %r8, %r11
+; SSE42-NEXT: movq %xmm0, %rax
+; SSE42-NEXT: pextrq $1, %xmm0, %rcx
+; SSE42-NEXT: pextrq $1, %xmm1, %rdx
+; SSE42-NEXT: movq %xmm1, %r8
+; SSE42-NEXT: movq %xmm2, %rsi
+; SSE42-NEXT: pextrq $1, %xmm2, %r9
+; SSE42-NEXT: movq %xmm3, %r10
+; SSE42-NEXT: pextrq $1, %xmm3, %r11
+; SSE42-NEXT: bsrq %r11, %rbx
+; SSE42-NEXT: xorq $63, %rbx
+; SSE42-NEXT: bsrq %r10, %r10
+; SSE42-NEXT: xorq $63, %r10
+; SSE42-NEXT: orq $64, %r10
+; SSE42-NEXT: testq %r11, %r11
+; SSE42-NEXT: cmovneq %rbx, %r10
+; SSE42-NEXT: bsrq %r9, %r11
; SSE42-NEXT: xorq $63, %r11
-; SSE42-NEXT: orq $64, %r11
-; SSE42-NEXT: testq %rcx, %rcx
-; SSE42-NEXT: cmovneq %rdx, %r11
-; SSE42-NEXT: movq %xmm0, %rdx
-; SSE42-NEXT: orq $128, %r11
-; SSE42-NEXT: movq %rdi, %r15
-; SSE42-NEXT: orq %rsi, %r15
-; SSE42-NEXT: cmovneq %r14, %r11
-; SSE42-NEXT: bsrq %r9, %r14
-; SSE42-NEXT: xorq $63, %r14
-; SSE42-NEXT: bsrq %r10, %r15
-; SSE42-NEXT: xorq $63, %r15
-; SSE42-NEXT: orq $64, %r15
+; SSE42-NEXT: bsrq %rsi, %rsi
+; SSE42-NEXT: xorq $63, %rsi
+; SSE42-NEXT: orq $64, %rsi
; SSE42-NEXT: testq %r9, %r9
-; SSE42-NEXT: cmovneq %r14, %r15
-; SSE42-NEXT: bsrq %rbx, %r14
-; SSE42-NEXT: xorq $63, %r14
-; SSE42-NEXT: bsrq %rdx, %rdx
+; SSE42-NEXT: cmovneq %r11, %rsi
+; SSE42-NEXT: orq $128, %rsi
+; SSE42-NEXT: ptest %xmm3, %xmm3
+; SSE42-NEXT: cmovneq %r10, %rsi
+; SSE42-NEXT: bsrq %rdx, %r9
+; SSE42-NEXT: xorq $63, %r9
+; SSE42-NEXT: bsrq %r8, %r8
+; SSE42-NEXT: xorq $63, %r8
+; SSE42-NEXT: orq $64, %r8
+; SSE42-NEXT: testq %rdx, %rdx
+; SSE42-NEXT: cmovneq %r9, %r8
+; SSE42-NEXT: bsrq %rcx, %rdx
; SSE42-NEXT: xorq $63, %rdx
-; SSE42-NEXT: orq $64, %rdx
-; SSE42-NEXT: testq %rbx, %rbx
-; SSE42-NEXT: cmovneq %r14, %rdx
-; SSE42-NEXT: orq $128, %rdx
-; SSE42-NEXT: orq %r9, %r10
-; SSE42-NEXT: cmovneq %r15, %rdx
-; SSE42-NEXT: orq $256, %rdx # imm = 0x100
-; SSE42-NEXT: orq %rdi, %r8
-; SSE42-NEXT: orq %rsi, %rcx
-; SSE42-NEXT: orq %r8, %rcx
-; SSE42-NEXT: cmovneq %r11, %rdx
-; SSE42-NEXT: pxor %xmm0, %xmm0
-; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: bsrq %rax, %rax
+; SSE42-NEXT: xorq $63, %rax
+; SSE42-NEXT: orq $64, %rax
+; SSE42-NEXT: testq %rcx, %rcx
+; SSE42-NEXT: cmovneq %rdx, %rax
+; SSE42-NEXT: por %xmm2, %xmm0
+; SSE42-NEXT: orq $128, %rax
+; SSE42-NEXT: ptest %xmm1, %xmm1
+; SSE42-NEXT: cmovneq %r8, %rax
+; SSE42-NEXT: orq $256, %rax # imm = 0x100
+; SSE42-NEXT: por %xmm3, %xmm2
+; SSE42-NEXT: ptest %xmm2, %xmm2
+; SSE42-NEXT: cmovneq %rsi, %rax
+; SSE42-NEXT: pxor %xmm2, %xmm2
+; SSE42-NEXT: movdqa %xmm2, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movdqa %xmm2, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movdqa %xmm2, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movdqa %xmm2, -{{[0-9]+}}(%rsp)
; SSE42-NEXT: movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000
; SSE42-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movl %edx, %ecx
+; SSE42-NEXT: movdqa %xmm2, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movdqa %xmm2, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movdqa %xmm2, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movl %eax, %ecx
; SSE42-NEXT: andl $63, %ecx
-; SSE42-NEXT: shrl $3, %edx
-; SSE42-NEXT: andl $56, %edx
+; SSE42-NEXT: shrl $3, %eax
+; SSE42-NEXT: andl $56, %eax
; SSE42-NEXT: movq $0, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movq -72(%rsp,%rdx), %rsi
-; SSE42-NEXT: movq -80(%rsp,%rdx), %r9
-; SSE42-NEXT: movq %r9, %rdi
-; SSE42-NEXT: shrdq %cl, %rsi, %rdi
-; SSE42-NEXT: movq -88(%rsp,%rdx), %r10
+; SSE42-NEXT: movq -72(%rsp,%rax), %rdx
+; SSE42-NEXT: movq -80(%rsp,%rax), %r9
+; SSE42-NEXT: movq %r9, %rsi
+; SSE42-NEXT: shrdq %cl, %rdx, %rsi
+; SSE42-NEXT: movq -88(%rsp,%rax), %r10
; SSE42-NEXT: movq %r10, %r8
; SSE42-NEXT: shrdq %cl, %r9, %r8
-; SSE42-NEXT: movq -96(%rsp,%rdx), %r11
+; SSE42-NEXT: movq -96(%rsp,%rax), %r11
; SSE42-NEXT: movq %r11, %r9
; SSE42-NEXT: shrdq %cl, %r10, %r9
-; SSE42-NEXT: movq -104(%rsp,%rdx), %rbx
+; SSE42-NEXT: movq -104(%rsp,%rax), %rbx
; SSE42-NEXT: movq %rbx, %r10
; SSE42-NEXT: shrdq %cl, %r11, %r10
-; SSE42-NEXT: movq -112(%rsp,%rdx), %r14
+; SSE42-NEXT: movq -112(%rsp,%rax), %r14
; SSE42-NEXT: movq %r14, %r11
; SSE42-NEXT: shrdq %cl, %rbx, %r11
-; SSE42-NEXT: movq -120(%rsp,%rdx), %r15
+; SSE42-NEXT: movq -120(%rsp,%rax), %r15
; SSE42-NEXT: movq %r15, %rbx
; SSE42-NEXT: shrdq %cl, %r14, %rbx
-; SSE42-NEXT: movq -128(%rsp,%rdx), %rdx
-; SSE42-NEXT: shrq %cl, %rsi
+; SSE42-NEXT: movq -128(%rsp,%rax), %r14
+; SSE42-NEXT: shrq %cl, %rdx
; SSE42-NEXT: # kill: def $cl killed $cl killed $ecx
-; SSE42-NEXT: shrdq %cl, %r15, %rdx
-; SSE42-NEXT: jmp .LBB16_3
-; SSE42-NEXT: .LBB16_1:
-; SSE42-NEXT: xorl %edx, %edx
-; SSE42-NEXT: xorl %ebx, %ebx
-; SSE42-NEXT: xorl %r11d, %r11d
-; SSE42-NEXT: xorl %r10d, %r10d
-; SSE42-NEXT: xorl %r9d, %r9d
-; SSE42-NEXT: xorl %r8d, %r8d
-; SSE42-NEXT: xorl %edi, %edi
-; SSE42-NEXT: xorl %esi, %esi
-; SSE42-NEXT: .LBB16_3: # %select.end
-; SSE42-NEXT: movq %rdx, (%rax)
-; SSE42-NEXT: movq %rbx, 8(%rax)
-; SSE42-NEXT: movq %r11, 16(%rax)
-; SSE42-NEXT: movq %r10, 24(%rax)
-; SSE42-NEXT: movq %r9, 32(%rax)
-; SSE42-NEXT: movq %r8, 40(%rax)
-; SSE42-NEXT: movq %rdi, 48(%rax)
-; SSE42-NEXT: movq %rsi, 56(%rax)
+; SSE42-NEXT: shrdq %cl, %r15, %r14
+; SSE42-NEXT: por %xmm3, %xmm1
+; SSE42-NEXT: por %xmm1, %xmm0
+; SSE42-NEXT: xorl %ecx, %ecx
+; SSE42-NEXT: ptest %xmm0, %xmm0
+; SSE42-NEXT: cmoveq %rcx, %rbx
+; SSE42-NEXT: cmoveq %rcx, %r11
+; SSE42-NEXT: cmoveq %rcx, %r10
+; SSE42-NEXT: cmoveq %rcx, %r9
+; SSE42-NEXT: cmoveq %rcx, %r8
+; SSE42-NEXT: cmoveq %rcx, %rsi
+; SSE42-NEXT: cmoveq %rcx, %r14
+; SSE42-NEXT: movq %rdi, %rax
+; SSE42-NEXT: cmoveq %rcx, %rdx
+; SSE42-NEXT: movq %rdx, 56(%rdi)
+; SSE42-NEXT: movq %rsi, 48(%rdi)
+; SSE42-NEXT: movq %r8, 40(%rdi)
+; SSE42-NEXT: movq %r9, 32(%rdi)
+; SSE42-NEXT: movq %r10, 24(%rdi)
+; SSE42-NEXT: movq %r11, 16(%rdi)
+; SSE42-NEXT: movq %rbx, 8(%rdi)
+; SSE42-NEXT: movq %r14, (%rdi)
; SSE42-NEXT: popq %rbx
; SSE42-NEXT: popq %r14
; SSE42-NEXT: popq %r15
@@ -4859,111 +4742,103 @@ define i512 @isolate_msb_i512_vector(<8 x i64> %v0, i512 %idx) nounwind {
; AVX2-NEXT: pushq %r15
; AVX2-NEXT: pushq %r14
; AVX2-NEXT: pushq %rbx
-; AVX2-NEXT: movq %rdi, %rax
-; AVX2-NEXT: vpor %ymm1, %ymm0, %ymm2
-; AVX2-NEXT: vptest %ymm2, %ymm2
-; AVX2-NEXT: je .LBB16_1
-; AVX2-NEXT: # %bb.2: # %select.false.sink
-; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT: vmovq %xmm2, %rax
; AVX2-NEXT: vpextrq $1, %xmm2, %rcx
-; AVX2-NEXT: vmovq %xmm2, %r8
-; AVX2-NEXT: vpextrq $1, %xmm1, %rsi
-; AVX2-NEXT: vmovq %xmm1, %rdi
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT: vpextrq $1, %xmm1, %r9
+; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
+; AVX2-NEXT: vmovq %xmm2, %rdx
+; AVX2-NEXT: vpextrq $1, %xmm2, %rsi
+; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT: vmovdqu %ymm2, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovaps {{.*#+}} ymm3 = [0,0,0,9223372036854775808]
+; AVX2-NEXT: vmovups %ymm3, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vpextrq $1, %xmm0, %r9
+; AVX2-NEXT: vmovq %xmm0, %r8
; AVX2-NEXT: vmovq %xmm1, %r10
-; AVX2-NEXT: vpextrq $1, %xmm0, %rbx
-; AVX2-NEXT: vmovq %xmm0, %rdx
-; AVX2-NEXT: lzcntq %rcx, %r11
+; AVX2-NEXT: vpextrq $1, %xmm1, %r11
+; AVX2-NEXT: xorl %ebx, %ebx
+; AVX2-NEXT: lzcntq %rsi, %rbx
; AVX2-NEXT: xorl %r14d, %r14d
-; AVX2-NEXT: lzcntq %r8, %r14
+; AVX2-NEXT: lzcntq %rdx, %r14
; AVX2-NEXT: addq $64, %r14
-; AVX2-NEXT: testq %rcx, %rcx
-; AVX2-NEXT: cmovneq %r11, %r14
-; AVX2-NEXT: xorl %r15d, %r15d
-; AVX2-NEXT: lzcntq %rsi, %r15
-; AVX2-NEXT: xorl %r11d, %r11d
-; AVX2-NEXT: lzcntq %rdi, %r11
-; AVX2-NEXT: addq $64, %r11
; AVX2-NEXT: testq %rsi, %rsi
-; AVX2-NEXT: cmovneq %r15, %r11
-; AVX2-NEXT: subq $-128, %r11
-; AVX2-NEXT: movq %r8, %r15
-; AVX2-NEXT: orq %rcx, %r15
-; AVX2-NEXT: cmovneq %r14, %r11
-; AVX2-NEXT: xorl %r14d, %r14d
-; AVX2-NEXT: lzcntq %r9, %r14
-; AVX2-NEXT: xorl %r15d, %r15d
-; AVX2-NEXT: lzcntq %r10, %r15
-; AVX2-NEXT: addq $64, %r15
+; AVX2-NEXT: cmovneq %rbx, %r14
+; AVX2-NEXT: xorl %ebx, %ebx
+; AVX2-NEXT: lzcntq %r11, %rbx
+; AVX2-NEXT: lzcntq %r10, %r10
+; AVX2-NEXT: addq $64, %r10
+; AVX2-NEXT: testq %r11, %r11
+; AVX2-NEXT: cmovneq %rbx, %r10
+; AVX2-NEXT: subq $-128, %r10
+; AVX2-NEXT: orq %rsi, %rdx
+; AVX2-NEXT: cmovneq %r14, %r10
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: lzcntq %rcx, %rdx
+; AVX2-NEXT: xorl %esi, %esi
+; AVX2-NEXT: lzcntq %rax, %rsi
+; AVX2-NEXT: addq $64, %rsi
+; AVX2-NEXT: testq %rcx, %rcx
+; AVX2-NEXT: cmovneq %rdx, %rsi
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: lzcntq %r9, %rdx
+; AVX2-NEXT: lzcntq %r8, %r8
+; AVX2-NEXT: addq $64, %r8
; AVX2-NEXT: testq %r9, %r9
-; AVX2-NEXT: cmovneq %r14, %r15
-; AVX2-NEXT: xorl %r14d, %r14d
-; AVX2-NEXT: lzcntq %rbx, %r14
-; AVX2-NEXT: lzcntq %rdx, %rdx
-; AVX2-NEXT: addq $64, %rdx
-; AVX2-NEXT: testq %rbx, %rbx
-; AVX2-NEXT: cmovneq %r14, %rdx
-; AVX2-NEXT: subq $-128, %rdx
-; AVX2-NEXT: orq %r9, %r10
-; AVX2-NEXT: cmovneq %r15, %rdx
-; AVX2-NEXT: addq $256, %rdx # imm = 0x100
-; AVX2-NEXT: orq %r8, %rdi
-; AVX2-NEXT: orq %rcx, %rsi
-; AVX2-NEXT: orq %rdi, %rsi
-; AVX2-NEXT: cmovneq %r11, %rdx
-; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vmovdqa {{.*#+}} ymm1 = [0,0,0,9223372036854775808]
-; AVX2-NEXT: vmovdqu %ymm1, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movl %edx, %ecx
+; AVX2-NEXT: cmovneq %rdx, %r8
+; AVX2-NEXT: subq $-128, %r8
+; AVX2-NEXT: orq %rcx, %rax
+; AVX2-NEXT: cmovneq %rsi, %r8
+; AVX2-NEXT: addq $256, %r8 # imm = 0x100
+; AVX2-NEXT: vptest %ymm1, %ymm1
+; AVX2-NEXT: cmovneq %r10, %r8
+; AVX2-NEXT: vmovdqu %ymm2, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovdqu %ymm2, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movl %r8d, %ecx
; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: shrl $3, %edx
-; AVX2-NEXT: andl $56, %edx
-; AVX2-NEXT: movq -72(%rsp,%rdx), %rbx
-; AVX2-NEXT: movq -80(%rsp,%rdx), %r8
-; AVX2-NEXT: movq %r8, %rsi
-; AVX2-NEXT: shrdq %cl, %rbx, %rsi
-; AVX2-NEXT: movq -88(%rsp,%rdx), %r9
-; AVX2-NEXT: movq %r9, %rdi
-; AVX2-NEXT: shrdq %cl, %r8, %rdi
-; AVX2-NEXT: movq -96(%rsp,%rdx), %r10
-; AVX2-NEXT: movq %r10, %r8
-; AVX2-NEXT: shrdq %cl, %r9, %r8
-; AVX2-NEXT: movq -104(%rsp,%rdx), %r11
-; AVX2-NEXT: movq %r11, %r9
+; AVX2-NEXT: shrl $3, %r8d
+; AVX2-NEXT: andl $56, %r8d
+; AVX2-NEXT: movq -72(%rsp,%r8), %r11
+; AVX2-NEXT: movq -80(%rsp,%r8), %rax
+; AVX2-NEXT: movq %rax, %rdx
+; AVX2-NEXT: shrdq %cl, %r11, %rdx
+; AVX2-NEXT: movq -88(%rsp,%r8), %r10
+; AVX2-NEXT: movq %r10, %rsi
+; AVX2-NEXT: shrdq %cl, %rax, %rsi
+; AVX2-NEXT: movq -96(%rsp,%r8), %rax
+; AVX2-NEXT: movq %rax, %r9
; AVX2-NEXT: shrdq %cl, %r10, %r9
-; AVX2-NEXT: movq -112(%rsp,%rdx), %r14
+; AVX2-NEXT: movq -104(%rsp,%r8), %r14
; AVX2-NEXT: movq %r14, %r10
-; AVX2-NEXT: shrdq %cl, %r11, %r10
-; AVX2-NEXT: movq -120(%rsp,%rdx), %r15
-; AVX2-NEXT: movq %r15, %r11
-; AVX2-NEXT: shrdq %cl, %r14, %r11
-; AVX2-NEXT: movq -128(%rsp,%rdx), %rdx
-; AVX2-NEXT: shrxq %rcx, %rbx, %rbx
-; AVX2-NEXT: # kill: def $cl killed $cl killed $rcx
-; AVX2-NEXT: shrdq %cl, %r15, %rdx
-; AVX2-NEXT: jmp .LBB16_3
-; AVX2-NEXT: .LBB16_1:
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: xorl %r11d, %r11d
-; AVX2-NEXT: xorl %r10d, %r10d
-; AVX2-NEXT: xorl %r9d, %r9d
-; AVX2-NEXT: xorl %r8d, %r8d
-; AVX2-NEXT: xorl %edi, %edi
-; AVX2-NEXT: xorl %esi, %esi
-; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: .LBB16_3: # %select.end
-; AVX2-NEXT: movq %rdx, (%rax)
-; AVX2-NEXT: movq %r11, 8(%rax)
-; AVX2-NEXT: movq %r10, 16(%rax)
-; AVX2-NEXT: movq %r9, 24(%rax)
-; AVX2-NEXT: movq %r8, 32(%rax)
-; AVX2-NEXT: movq %rdi, 40(%rax)
-; AVX2-NEXT: movq %rsi, 48(%rax)
-; AVX2-NEXT: movq %rbx, 56(%rax)
+; AVX2-NEXT: shrdq %cl, %rax, %r10
+; AVX2-NEXT: movq -112(%rsp,%r8), %r15
+; AVX2-NEXT: movq %r15, %rbx
+; AVX2-NEXT: shrdq %cl, %r14, %rbx
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: vpor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: movq -128(%rsp,%r8), %rdi
+; AVX2-NEXT: movq -120(%rsp,%r8), %r14
+; AVX2-NEXT: movq %r14, %r8
+; AVX2-NEXT: shrdq %cl, %r15, %r8
+; AVX2-NEXT: shrdq %cl, %r14, %rdi
+; AVX2-NEXT: xorl %r14d, %r14d
+; AVX2-NEXT: vptest %ymm0, %ymm0
+; AVX2-NEXT: shrxq %rcx, %r11, %rcx
+; AVX2-NEXT: cmoveq %r14, %r8
+; AVX2-NEXT: cmoveq %r14, %rbx
+; AVX2-NEXT: cmoveq %r14, %r10
+; AVX2-NEXT: cmoveq %r14, %r9
+; AVX2-NEXT: cmoveq %r14, %rsi
+; AVX2-NEXT: cmoveq %r14, %rdx
+; AVX2-NEXT: cmoveq %r14, %rdi
+; AVX2-NEXT: cmoveq %r14, %rcx
+; AVX2-NEXT: movq %rcx, 56(%rax)
+; AVX2-NEXT: movq %rdx, 48(%rax)
+; AVX2-NEXT: movq %rsi, 40(%rax)
+; AVX2-NEXT: movq %r9, 32(%rax)
+; AVX2-NEXT: movq %r10, 24(%rax)
+; AVX2-NEXT: movq %rbx, 16(%rax)
+; AVX2-NEXT: movq %r8, 8(%rax)
+; AVX2-NEXT: movq %rdi, (%rax)
; AVX2-NEXT: popq %rbx
; AVX2-NEXT: popq %r14
; AVX2-NEXT: popq %r15
@@ -4974,154 +4849,56 @@ define i512 @isolate_msb_i512_vector(<8 x i64> %v0, i512 %idx) nounwind {
; AVX512F: # %bb.0:
; AVX512F-NEXT: movq %rdi, %rax
; AVX512F-NEXT: vptestmd %zmm0, %zmm0, %k0
+; AVX512F-NEXT: xorl %ecx, %ecx
; AVX512F-NEXT: kortestw %k0, %k0
-; AVX512F-NEXT: je .LBB16_1
-; AVX512F-NEXT: # %bb.2: # %select.false.sink
-; AVX512F-NEXT: vextracti32x4 $3, %zmm0, %xmm1
-; AVX512F-NEXT: vpextrq $1, %xmm1, %rcx
-; AVX512F-NEXT: vmovq %xmm1, %rdx
-; AVX512F-NEXT: vextracti32x4 $2, %zmm0, %xmm1
-; AVX512F-NEXT: vpextrq $1, %xmm1, %rsi
-; AVX512F-NEXT: vpextrq $1, %xmm0, %rdi
-; AVX512F-NEXT: vmovq %xmm1, %r8
-; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX512F-NEXT: vpextrq $1, %xmm1, %r9
-; AVX512F-NEXT: vmovq %xmm0, %r10
-; AVX512F-NEXT: vmovq %xmm1, %r11
-; AVX512F-NEXT: vmovq %r10, %xmm0
-; AVX512F-NEXT: vmovq %rdi, %xmm1
-; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX512F-NEXT: vmovq %r11, %xmm1
-; AVX512F-NEXT: vmovq %r9, %xmm2
-; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX512F-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
-; AVX512F-NEXT: vmovq %r8, %xmm1
-; AVX512F-NEXT: vmovq %rsi, %xmm2
-; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX512F-NEXT: vmovq %rdx, %xmm2
-; AVX512F-NEXT: vmovq %rcx, %xmm3
-; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
-; AVX512F-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
-; AVX512F-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
-; AVX512F-NEXT: vplzcntq %zmm0, %zmm1
-; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1 # [0,64,128,192,256,320,384,448]
+; AVX512F-NEXT: sete %cl
+; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
+; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0
; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
-; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
-; AVX512F-NEXT: vmovq %xmm0, %rcx
-; AVX512F-NEXT: movabsq $-9223372036854775808, %rdx # imm = 0x8000000000000000
-; AVX512F-NEXT: shrxq %rcx, %rdx, %rdx
-; AVX512F-NEXT: shrl $6, %ecx
-; AVX512F-NEXT: movl $128, %esi
-; AVX512F-NEXT: shrxq %rcx, %rsi, %rcx
-; AVX512F-NEXT: kmovw %ecx, %k1
-; AVX512F-NEXT: vpbroadcastq %rdx, %zmm0 {%k1} {z}
-; AVX512F-NEXT: vextracti32x4 $3, %zmm0, %xmm1
-; AVX512F-NEXT: vpextrq $1, %xmm1, %rcx
-; AVX512F-NEXT: vmovq %xmm1, %rdx
-; AVX512F-NEXT: vextracti32x4 $2, %zmm0, %xmm1
-; AVX512F-NEXT: vpextrq $1, %xmm1, %rsi
-; AVX512F-NEXT: vmovq %xmm1, %r8
-; AVX512F-NEXT: vpextrq $1, %xmm0, %r9
-; AVX512F-NEXT: vmovq %xmm0, %rdi
-; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm0
-; AVX512F-NEXT: vpextrq $1, %xmm0, %r10
-; AVX512F-NEXT: vmovq %xmm0, %r11
-; AVX512F-NEXT: jmp .LBB16_3
-; AVX512F-NEXT: .LBB16_1:
-; AVX512F-NEXT: xorl %edi, %edi
-; AVX512F-NEXT: xorl %r9d, %r9d
-; AVX512F-NEXT: xorl %r11d, %r11d
-; AVX512F-NEXT: xorl %r10d, %r10d
-; AVX512F-NEXT: xorl %r8d, %r8d
-; AVX512F-NEXT: xorl %esi, %esi
-; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: xorl %ecx, %ecx
-; AVX512F-NEXT: .LBB16_3: # %select.end
-; AVX512F-NEXT: movq %rdi, (%rax)
-; AVX512F-NEXT: movq %r9, 8(%rax)
-; AVX512F-NEXT: movq %r11, 16(%rax)
-; AVX512F-NEXT: movq %r10, 24(%rax)
-; AVX512F-NEXT: movq %r8, 32(%rax)
-; AVX512F-NEXT: movq %rsi, 40(%rax)
-; AVX512F-NEXT: movq %rdx, 48(%rax)
-; AVX512F-NEXT: movq %rcx, 56(%rax)
+; AVX512F-NEXT: vplzcntq %zmm0, %zmm0
+; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,64,128,192,256,320,384,448]
+; AVX512F-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT: vmovq %xmm0, %rdx
+; AVX512F-NEXT: movabsq $-9223372036854775808, %rsi # imm = 0x8000000000000000
+; AVX512F-NEXT: shrxq %rdx, %rsi, %rsi
+; AVX512F-NEXT: shrl $6, %edx
+; AVX512F-NEXT: movl $128, %edi
+; AVX512F-NEXT: shrxq %rdx, %rdi, %rdx
+; AVX512F-NEXT: kmovw %edx, %k1
+; AVX512F-NEXT: vpbroadcastq %rsi, %zmm0 {%k1} {z}
+; AVX512F-NEXT: negl %ecx
+; AVX512F-NEXT: kmovw %ecx, %k0
+; AVX512F-NEXT: knotw %k0, %k1
+; AVX512F-NEXT: vmovdqa32 %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT: vmovdqu64 %zmm0, (%rax)
; AVX512F-NEXT: retq
;
; AVX512VL-LABEL: isolate_msb_i512_vector:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: movq %rdi, %rax
; AVX512VL-NEXT: vptestmd %zmm0, %zmm0, %k0
+; AVX512VL-NEXT: xorl %ecx, %ecx
; AVX512VL-NEXT: kortestw %k0, %k0
-; AVX512VL-NEXT: je .LBB16_1
-; AVX512VL-NEXT: # %bb.2: # %select.false.sink
-; AVX512VL-NEXT: vextracti32x4 $3, %zmm0, %xmm1
-; AVX512VL-NEXT: vpextrq $1, %xmm1, %rcx
-; AVX512VL-NEXT: vmovq %xmm1, %rdx
-; AVX512VL-NEXT: vextracti32x4 $2, %zmm0, %xmm1
-; AVX512VL-NEXT: vpextrq $1, %xmm1, %rsi
-; AVX512VL-NEXT: vmovq %xmm1, %rdi
-; AVX512VL-NEXT: vpextrq $1, %xmm0, %r8
-; AVX512VL-NEXT: vmovq %xmm0, %r9
-; AVX512VL-NEXT: vextracti128 $1, %ymm0, %xmm0
-; AVX512VL-NEXT: vpextrq $1, %xmm0, %r10
-; AVX512VL-NEXT: vmovq %xmm0, %r11
-; AVX512VL-NEXT: vmovq %r9, %xmm0
-; AVX512VL-NEXT: vmovq %r8, %xmm1
-; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX512VL-NEXT: vmovq %r11, %xmm1
-; AVX512VL-NEXT: vmovq %r10, %xmm2
-; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX512VL-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
-; AVX512VL-NEXT: vmovq %rdi, %xmm1
-; AVX512VL-NEXT: vmovq %rsi, %xmm2
-; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX512VL-NEXT: vmovq %rdx, %xmm2
-; AVX512VL-NEXT: vmovq %rcx, %xmm3
-; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
-; AVX512VL-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
-; AVX512VL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512VL-NEXT: sete %cl
+; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
+; AVX512VL-NEXT: vpermq %zmm0, %zmm1, %zmm0
; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VL-NEXT: vplzcntq %zmm0, %zmm0
; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,64,128,192,256,320,384,448]
; AVX512VL-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
-; AVX512VL-NEXT: vmovq %xmm0, %rcx
-; AVX512VL-NEXT: movabsq $-9223372036854775808, %rdx # imm = 0x8000000000000000
-; AVX512VL-NEXT: shrxq %rcx, %rdx, %rdx
-; AVX512VL-NEXT: shrl $6, %ecx
-; AVX512VL-NEXT: movl $128, %esi
-; AVX512VL-NEXT: shrxq %rcx, %rsi, %rcx
-; AVX512VL-NEXT: kmovd %ecx, %k1
-; AVX512VL-NEXT: vpbroadcastq %rdx, %zmm0 {%k1} {z}
-; AVX512VL-NEXT: vextracti32x4 $3, %zmm0, %xmm1
-; AVX512VL-NEXT: vpextrq $1, %xmm1, %rcx
-; AVX512VL-NEXT: vmovq %xmm1, %rdx
-; AVX512VL-NEXT: vextracti32x4 $2, %zmm0, %xmm1
-; AVX512VL-NEXT: vpextrq $1, %xmm1, %rsi
-; AVX512VL-NEXT: vpextrq $1, %xmm0, %r8
-; AVX512VL-NEXT: vmovq %xmm1, %r9
-; AVX512VL-NEXT: vmovq %xmm0, %rdi
-; AVX512VL-NEXT: vextracti128 $1, %ymm0, %xmm0
-; AVX512VL-NEXT: vpextrq $1, %xmm0, %r10
-; AVX512VL-NEXT: vmovq %xmm0, %r11
-; AVX512VL-NEXT: jmp .LBB16_3
-; AVX512VL-NEXT: .LBB16_1:
-; AVX512VL-NEXT: xorl %edi, %edi
-; AVX512VL-NEXT: xorl %r8d, %r8d
-; AVX512VL-NEXT: xorl %r11d, %r11d
-; AVX512VL-NEXT: xorl %r10d, %r10d
-; AVX512VL-NEXT: xorl %r9d, %r9d
-; AVX512VL-NEXT: xorl %esi, %esi
-; AVX512VL-NEXT: xorl %edx, %edx
-; AVX512VL-NEXT: xorl %ecx, %ecx
-; AVX512VL-NEXT: .LBB16_3: # %select.end
-; AVX512VL-NEXT: movq %rdi, (%rax)
-; AVX512VL-NEXT: movq %r8, 8(%rax)
-; AVX512VL-NEXT: movq %r11, 16(%rax)
-; AVX512VL-NEXT: movq %r10, 24(%rax)
-; AVX512VL-NEXT: movq %r9, 32(%rax)
-; AVX512VL-NEXT: movq %rsi, 40(%rax)
-; AVX512VL-NEXT: movq %rdx, 48(%rax)
-; AVX512VL-NEXT: movq %rcx, 56(%rax)
+; AVX512VL-NEXT: vmovq %xmm0, %rdx
+; AVX512VL-NEXT: movabsq $-9223372036854775808, %rsi # imm = 0x8000000000000000
+; AVX512VL-NEXT: shrxq %rdx, %rsi, %rsi
+; AVX512VL-NEXT: shrl $6, %edx
+; AVX512VL-NEXT: movl $128, %edi
+; AVX512VL-NEXT: shrxq %rdx, %rdi, %rdx
+; AVX512VL-NEXT: kmovd %edx, %k1
+; AVX512VL-NEXT: vpbroadcastq %rsi, %zmm0 {%k1} {z}
+; AVX512VL-NEXT: negl %ecx
+; AVX512VL-NEXT: kmovd %ecx, %k0
+; AVX512VL-NEXT: knotw %k0, %k1
+; AVX512VL-NEXT: vmovdqa32 %zmm0, %zmm0 {%k1} {z}
+; AVX512VL-NEXT: vmovdqu64 %zmm0, (%rax)
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
@@ -5129,77 +4906,28 @@ define i512 @isolate_msb_i512_vector(<8 x i64> %v0, i512 %idx) nounwind {
; AVX512VBMI: # %bb.0:
; AVX512VBMI-NEXT: movq %rdi, %rax
; AVX512VBMI-NEXT: vptestmd %zmm0, %zmm0, %k0
+; AVX512VBMI-NEXT: xorl %ecx, %ecx
; AVX512VBMI-NEXT: kortestw %k0, %k0
-; AVX512VBMI-NEXT: je .LBB16_1
-; AVX512VBMI-NEXT: # %bb.2: # %select.false.sink
-; AVX512VBMI-NEXT: vextracti32x4 $3, %zmm0, %xmm1
-; AVX512VBMI-NEXT: vpextrq $1, %xmm1, %rcx
-; AVX512VBMI-NEXT: vmovq %xmm1, %rdx
-; AVX512VBMI-NEXT: vextracti32x4 $2, %zmm0, %xmm1
-; AVX512VBMI-NEXT: vpextrq $1, %xmm1, %rsi
-; AVX512VBMI-NEXT: vmovq %xmm1, %rdi
-; AVX512VBMI-NEXT: vpextrq $1, %xmm0, %r8
-; AVX512VBMI-NEXT: vmovq %xmm0, %r9
-; AVX512VBMI-NEXT: vextracti128 $1, %ymm0, %xmm0
-; AVX512VBMI-NEXT: vpextrq $1, %xmm0, %r10
-; AVX512VBMI-NEXT: vmovq %xmm0, %r11
-; AVX512VBMI-NEXT: vmovq %r9, %xmm0
-; AVX512VBMI-NEXT: vmovq %r8, %xmm1
-; AVX512VBMI-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX512VBMI-NEXT: vmovq %r11, %xmm1
-; AVX512VBMI-NEXT: vmovq %r10, %xmm2
-; AVX512VBMI-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX512VBMI-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
-; AVX512VBMI-NEXT: vmovq %rdi, %xmm1
-; AVX512VBMI-NEXT: vmovq %rsi, %xmm2
-; AVX512VBMI-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX512VBMI-NEXT: vmovq %rdx, %xmm2
-; AVX512VBMI-NEXT: vmovq %rcx, %xmm3
-; AVX512VBMI-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
-; AVX512VBMI-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
-; AVX512VBMI-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512VBMI-NEXT: sete %cl
+; AVX512VBMI-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
+; AVX512VBMI-NEXT: vpermq %zmm0, %zmm1, %zmm0
; AVX512VBMI-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VBMI-NEXT: vplzcntq %zmm0, %zmm0
; AVX512VBMI-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,64,128,192,256,320,384,448]
; AVX512VBMI-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
-; AVX512VBMI-NEXT: vmovq %xmm0, %rcx
-; AVX512VBMI-NEXT: movabsq $-9223372036854775808, %rdx # imm = 0x8000000000000000
-; AVX512VBMI-NEXT: shrxq %rcx, %rdx, %rdx
-; AVX512VBMI-NEXT: shrl $6, %ecx
-; AVX512VBMI-NEXT: movl $128, %esi
-; AVX512VBMI-NEXT: shrxq %rcx, %rsi, %rcx
-; AVX512VBMI-NEXT: kmovd %ecx, %k1
-; AVX512VBMI-NEXT: vpbroadcastq %rdx, %zmm0 {%k1} {z}
-; AVX512VBMI-NEXT: vextracti32x4 $3, %zmm0, %xmm1
-; AVX512VBMI-NEXT: vpextrq $1, %xmm1, %rcx
-; AVX512VBMI-NEXT: vmovq %xmm1, %rdx
-; AVX512VBMI-NEXT: vextracti32x4 $2, %zmm0, %xmm1
-; AVX512VBMI-NEXT: vpextrq $1, %xmm1, %rsi
-; AVX512VBMI-NEXT: vpextrq $1, %xmm0, %r8
-; AVX512VBMI-NEXT: vmovq %xmm1, %r9
-; AVX512VBMI-NEXT: vmovq %xmm0, %rdi
-; AVX512VBMI-NEXT: vextracti128 $1, %ymm0, %xmm0
-; AVX512VBMI-NEXT: vpextrq $1, %xmm0, %r10
-; AVX512VBMI-NEXT: vmovq %xmm0, %r11
-; AVX512VBMI-NEXT: jmp .LBB16_3
-; AVX512VBMI-NEXT: .LBB16_1:
-; AVX512VBMI-NEXT: xorl %edi, %edi
-; AVX512VBMI-NEXT: xorl %r8d, %r8d
-; AVX512VBMI-NEXT: xorl %r11d, %r11d
-; AVX512VBMI-NEXT: xorl %r10d, %r10d
-; AVX512VBMI-NEXT: xorl %r9d, %r9d
-; AVX512VBMI-NEXT: xorl %esi, %esi
-; AVX512VBMI-NEXT: xorl %edx, %edx
-; AVX512VBMI-NEXT: xorl %ecx, %ecx
-; AVX512VBMI-NEXT: .LBB16_3: # %select.end
-; AVX512VBMI-NEXT: movq %rdi, (%rax)
-; AVX512VBMI-NEXT: movq %r8, 8(%rax)
-; AVX512VBMI-NEXT: movq %r11, 16(%rax)
-; AVX512VBMI-NEXT: movq %r10, 24(%rax)
-; AVX512VBMI-NEXT: movq %r9, 32(%rax)
-; AVX512VBMI-NEXT: movq %rsi, 40(%rax)
-; AVX512VBMI-NEXT: movq %rdx, 48(%rax)
-; AVX512VBMI-NEXT: movq %rcx, 56(%rax)
+; AVX512VBMI-NEXT: vmovq %xmm0, %rdx
+; AVX512VBMI-NEXT: movabsq $-9223372036854775808, %rsi # imm = 0x8000000000000000
+; AVX512VBMI-NEXT: shrxq %rdx, %rsi, %rsi
+; AVX512VBMI-NEXT: shrl $6, %edx
+; AVX512VBMI-NEXT: movl $128, %edi
+; AVX512VBMI-NEXT: shrxq %rdx, %rdi, %rdx
+; AVX512VBMI-NEXT: kmovd %edx, %k1
+; AVX512VBMI-NEXT: vpbroadcastq %rsi, %zmm0 {%k1} {z}
+; AVX512VBMI-NEXT: negl %ecx
+; AVX512VBMI-NEXT: kmovd %ecx, %k0
+; AVX512VBMI-NEXT: knotw %k0, %k1
+; AVX512VBMI-NEXT: vmovdqa32 %zmm0, %zmm0 {%k1} {z}
+; AVX512VBMI-NEXT: vmovdqu64 %zmm0, (%rax)
; AVX512VBMI-NEXT: vzeroupper
; AVX512VBMI-NEXT: retq
%a0 = bitcast <8 x i64> %v0 to i512
@@ -5219,65 +4947,63 @@ define i512 @isolate_msb_i512_load(ptr %p0, i512 %idx) nounwind {
; SSE2-NEXT: pushq %r12
; SSE2-NEXT: pushq %rbx
; SSE2-NEXT: pushq %rax
-; SSE2-NEXT: movq %rdi, %rax
+; SSE2-NEXT: movq 8(%rsi), %r9
+; SSE2-NEXT: movq 16(%rsi), %rcx
+; SSE2-NEXT: movq 24(%rsi), %r8
+; SSE2-NEXT: movq 48(%rsi), %rdx
+; SSE2-NEXT: movq 56(%rsi), %r11
+; SSE2-NEXT: movdqa 32(%rsi), %xmm1
+; SSE2-NEXT: movdqa 48(%rsi), %xmm2
; SSE2-NEXT: movdqa 16(%rsi), %xmm0
-; SSE2-NEXT: por 48(%rsi), %xmm0
-; SSE2-NEXT: movdqa (%rsi), %xmm1
-; SSE2-NEXT: por 32(%rsi), %xmm1
-; SSE2-NEXT: por %xmm0, %xmm1
+; SSE2-NEXT: por %xmm2, %xmm0
+; SSE2-NEXT: movdqa (%rsi), %xmm3
+; SSE2-NEXT: por %xmm1, %xmm3
+; SSE2-NEXT: por %xmm0, %xmm3
; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: pcmpeqd %xmm0, %xmm1
-; SSE2-NEXT: movmskps %xmm1, %ecx
-; SSE2-NEXT: xorl $15, %ecx
-; SSE2-NEXT: je .LBB17_1
-; SSE2-NEXT: # %bb.2: # %select.false.sink
-; SSE2-NEXT: movq 56(%rsi), %rdi
-; SSE2-NEXT: movq 48(%rsi), %r9
-; SSE2-NEXT: movq 40(%rsi), %rcx
-; SSE2-NEXT: movq 32(%rsi), %r8
-; SSE2-NEXT: movq 24(%rsi), %r10
-; SSE2-NEXT: movq 16(%rsi), %r11
-; SSE2-NEXT: movq 8(%rsi), %r14
-; SSE2-NEXT: bsrq %rdi, %rdx
-; SSE2-NEXT: xorq $63, %rdx
-; SSE2-NEXT: bsrq %r9, %r15
-; SSE2-NEXT: xorq $63, %r15
-; SSE2-NEXT: orq $64, %r15
-; SSE2-NEXT: testq %rdi, %rdi
-; SSE2-NEXT: cmovneq %rdx, %r15
-; SSE2-NEXT: bsrq %rcx, %rdx
-; SSE2-NEXT: xorq $63, %rdx
-; SSE2-NEXT: bsrq %r8, %rbx
+; SSE2-NEXT: pcmpeqd %xmm0, %xmm3
+; SSE2-NEXT: movmskps %xmm3, %eax
+; SSE2-NEXT: xorl $15, %eax
+; SSE2-NEXT: bsrq %r11, %r10
+; SSE2-NEXT: xorq $63, %r10
+; SSE2-NEXT: bsrq %rdx, %rbx
; SSE2-NEXT: xorq $63, %rbx
; SSE2-NEXT: orq $64, %rbx
-; SSE2-NEXT: testq %rcx, %rcx
-; SSE2-NEXT: cmovneq %rdx, %rbx
-; SSE2-NEXT: orq $128, %rbx
-; SSE2-NEXT: movq %r9, %rdx
-; SSE2-NEXT: orq %rdi, %rdx
-; SSE2-NEXT: cmovneq %r15, %rbx
-; SSE2-NEXT: bsrq %r10, %rdx
-; SSE2-NEXT: xorq $63, %rdx
-; SSE2-NEXT: bsrq %r11, %r15
+; SSE2-NEXT: testq %r11, %r11
+; SSE2-NEXT: cmovneq %r10, %rbx
+; SSE2-NEXT: movq 40(%rsi), %r14
+; SSE2-NEXT: bsrq %r14, %r15
+; SSE2-NEXT: bsrq 32(%rsi), %r10
; SSE2-NEXT: xorq $63, %r15
-; SSE2-NEXT: orq $64, %r15
-; SSE2-NEXT: testq %r10, %r10
-; SSE2-NEXT: cmovneq %rdx, %r15
-; SSE2-NEXT: bsrq %r14, %r12
-; SSE2-NEXT: xorq $63, %r12
+; SSE2-NEXT: xorq $63, %r10
+; SSE2-NEXT: orq $64, %r10
+; SSE2-NEXT: testq %r14, %r14
+; SSE2-NEXT: cmovneq %r15, %r10
+; SSE2-NEXT: orq $128, %r10
+; SSE2-NEXT: orq %r11, %rdx
+; SSE2-NEXT: cmovneq %rbx, %r10
+; SSE2-NEXT: bsrq %r8, %rdx
+; SSE2-NEXT: xorq $63, %rdx
+; SSE2-NEXT: bsrq %rcx, %r11
+; SSE2-NEXT: xorq $63, %r11
+; SSE2-NEXT: orq $64, %r11
+; SSE2-NEXT: testq %r8, %r8
+; SSE2-NEXT: cmovneq %rdx, %r11
+; SSE2-NEXT: bsrq %r9, %rbx
+; SSE2-NEXT: xorq $63, %rbx
; SSE2-NEXT: bsrq (%rsi), %rdx
; SSE2-NEXT: xorq $63, %rdx
; SSE2-NEXT: orq $64, %rdx
-; SSE2-NEXT: testq %r14, %r14
-; SSE2-NEXT: cmovneq %r12, %rdx
+; SSE2-NEXT: testq %r9, %r9
+; SSE2-NEXT: cmovneq %rbx, %rdx
; SSE2-NEXT: orq $128, %rdx
-; SSE2-NEXT: orq %r10, %r11
-; SSE2-NEXT: cmovneq %r15, %rdx
-; SSE2-NEXT: orq $256, %rdx # imm = 0x100
-; SSE2-NEXT: orq %r9, %r8
-; SSE2-NEXT: orq %rdi, %rcx
; SSE2-NEXT: orq %r8, %rcx
-; SSE2-NEXT: cmovneq %rbx, %rdx
+; SSE2-NEXT: cmovneq %r11, %rdx
+; SSE2-NEXT: orq $256, %rdx # imm = 0x100
+; SSE2-NEXT: por %xmm2, %xmm1
+; SSE2-NEXT: pcmpeqd %xmm0, %xmm1
+; SSE2-NEXT: movmskps %xmm1, %ecx
+; SSE2-NEXT: xorl $15, %ecx
+; SSE2-NEXT: cmovneq %r10, %rdx
; SSE2-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
@@ -5293,47 +5019,47 @@ define i512 @isolate_msb_i512_load(ptr %p0, i512 %idx) nounwind {
; SSE2-NEXT: andl $56, %edx
; SSE2-NEXT: movq $0, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: movq -72(%rsp,%rdx), %rsi
-; SSE2-NEXT: movq -80(%rsp,%rdx), %r9
-; SSE2-NEXT: movq %r9, %rdi
-; SSE2-NEXT: shrdq %cl, %rsi, %rdi
-; SSE2-NEXT: movq -88(%rsp,%rdx), %r10
+; SSE2-NEXT: movq -80(%rsp,%rdx), %r10
; SSE2-NEXT: movq %r10, %r8
-; SSE2-NEXT: shrdq %cl, %r9, %r8
-; SSE2-NEXT: movq -96(%rsp,%rdx), %r11
+; SSE2-NEXT: shrdq %cl, %rsi, %r8
+; SSE2-NEXT: movq -88(%rsp,%rdx), %r11
; SSE2-NEXT: movq %r11, %r9
; SSE2-NEXT: shrdq %cl, %r10, %r9
-; SSE2-NEXT: movq -104(%rsp,%rdx), %rbx
+; SSE2-NEXT: movq -96(%rsp,%rdx), %rbx
; SSE2-NEXT: movq %rbx, %r10
; SSE2-NEXT: shrdq %cl, %r11, %r10
-; SSE2-NEXT: movq -112(%rsp,%rdx), %r14
+; SSE2-NEXT: movq -104(%rsp,%rdx), %r14
; SSE2-NEXT: movq %r14, %r11
; SSE2-NEXT: shrdq %cl, %rbx, %r11
-; SSE2-NEXT: movq -120(%rsp,%rdx), %r15
+; SSE2-NEXT: movq -112(%rsp,%rdx), %r15
; SSE2-NEXT: movq %r15, %rbx
; SSE2-NEXT: shrdq %cl, %r14, %rbx
+; SSE2-NEXT: movq -120(%rsp,%rdx), %r12
+; SSE2-NEXT: movq %r12, %r14
+; SSE2-NEXT: shrdq %cl, %r15, %r14
; SSE2-NEXT: movq -128(%rsp,%rdx), %rdx
; SSE2-NEXT: shrq %cl, %rsi
; SSE2-NEXT: # kill: def $cl killed $cl killed $ecx
-; SSE2-NEXT: shrdq %cl, %r15, %rdx
-; SSE2-NEXT: jmp .LBB17_3
-; SSE2-NEXT: .LBB17_1:
-; SSE2-NEXT: xorl %edx, %edx
-; SSE2-NEXT: xorl %ebx, %ebx
-; SSE2-NEXT: xorl %r11d, %r11d
-; SSE2-NEXT: xorl %r10d, %r10d
-; SSE2-NEXT: xorl %r9d, %r9d
-; SSE2-NEXT: xorl %r8d, %r8d
-; SSE2-NEXT: xorl %edi, %edi
-; SSE2-NEXT: xorl %esi, %esi
-; SSE2-NEXT: .LBB17_3: # %select.end
-; SSE2-NEXT: movq %rdx, (%rax)
-; SSE2-NEXT: movq %rbx, 8(%rax)
-; SSE2-NEXT: movq %r11, 16(%rax)
-; SSE2-NEXT: movq %r10, 24(%rax)
-; SSE2-NEXT: movq %r9, 32(%rax)
-; SSE2-NEXT: movq %r8, 40(%rax)
-; SSE2-NEXT: movq %rdi, 48(%rax)
-; SSE2-NEXT: movq %rsi, 56(%rax)
+; SSE2-NEXT: shrdq %cl, %r12, %rdx
+; SSE2-NEXT: xorl %ecx, %ecx
+; SSE2-NEXT: testl %eax, %eax
+; SSE2-NEXT: cmoveq %rcx, %r14
+; SSE2-NEXT: cmoveq %rcx, %rbx
+; SSE2-NEXT: cmoveq %rcx, %r11
+; SSE2-NEXT: cmoveq %rcx, %r10
+; SSE2-NEXT: cmoveq %rcx, %r9
+; SSE2-NEXT: cmoveq %rcx, %r8
+; SSE2-NEXT: cmoveq %rcx, %rdx
+; SSE2-NEXT: movq %rdi, %rax
+; SSE2-NEXT: cmoveq %rcx, %rsi
+; SSE2-NEXT: movq %rsi, 56(%rdi)
+; SSE2-NEXT: movq %r8, 48(%rdi)
+; SSE2-NEXT: movq %r9, 40(%rdi)
+; SSE2-NEXT: movq %r10, 32(%rdi)
+; SSE2-NEXT: movq %r11, 24(%rdi)
+; SSE2-NEXT: movq %rbx, 16(%rdi)
+; SSE2-NEXT: movq %r14, 8(%rdi)
+; SSE2-NEXT: movq %rdx, (%rdi)
; SSE2-NEXT: addq $8, %rsp
; SSE2-NEXT: popq %rbx
; SSE2-NEXT: popq %r12
@@ -5345,125 +5071,116 @@ define i512 @isolate_msb_i512_load(ptr %p0, i512 %idx) nounwind {
; SSE42: # %bb.0:
; SSE42-NEXT: pushq %r15
; SSE42-NEXT: pushq %r14
-; SSE42-NEXT: pushq %r12
; SSE42-NEXT: pushq %rbx
-; SSE42-NEXT: pushq %rax
-; SSE42-NEXT: movdqa (%rsi), %xmm0
-; SSE42-NEXT: movdqa 16(%rsi), %xmm1
-; SSE42-NEXT: por 48(%rsi), %xmm1
-; SSE42-NEXT: por 32(%rsi), %xmm0
-; SSE42-NEXT: movq %rdi, %rax
-; SSE42-NEXT: por %xmm1, %xmm0
-; SSE42-NEXT: ptest %xmm0, %xmm0
-; SSE42-NEXT: je .LBB17_1
-; SSE42-NEXT: # %bb.2: # %select.false.sink
-; SSE42-NEXT: movq 56(%rsi), %rdi
-; SSE42-NEXT: movq 48(%rsi), %r9
-; SSE42-NEXT: movq 40(%rsi), %rcx
-; SSE42-NEXT: movq 32(%rsi), %r8
-; SSE42-NEXT: movq 24(%rsi), %r10
-; SSE42-NEXT: movq 16(%rsi), %r11
-; SSE42-NEXT: bsrq %rdi, %rdx
-; SSE42-NEXT: xorq $63, %rdx
-; SSE42-NEXT: bsrq %r9, %r14
-; SSE42-NEXT: xorq $63, %r14
-; SSE42-NEXT: orq $64, %r14
-; SSE42-NEXT: testq %rdi, %rdi
-; SSE42-NEXT: cmovneq %rdx, %r14
-; SSE42-NEXT: bsrq %rcx, %rdx
-; SSE42-NEXT: xorq $63, %rdx
-; SSE42-NEXT: bsrq %r8, %rbx
+; SSE42-NEXT: movq 8(%rsi), %r8
+; SSE42-NEXT: movq 16(%rsi), %rcx
+; SSE42-NEXT: movq 24(%rsi), %rdx
+; SSE42-NEXT: movq 40(%rsi), %r11
+; SSE42-NEXT: movq 48(%rsi), %rax
+; SSE42-NEXT: movq 56(%rsi), %r10
+; SSE42-NEXT: movdqa 32(%rsi), %xmm2
+; SSE42-NEXT: movdqa 48(%rsi), %xmm0
+; SSE42-NEXT: movdqa (%rsi), %xmm1
+; SSE42-NEXT: bsrq %r10, %r9
+; SSE42-NEXT: xorq $63, %r9
+; SSE42-NEXT: bsrq %rax, %rbx
; SSE42-NEXT: xorq $63, %rbx
; SSE42-NEXT: orq $64, %rbx
-; SSE42-NEXT: testq %rcx, %rcx
-; SSE42-NEXT: cmovneq %rdx, %rbx
-; SSE42-NEXT: orq $128, %rbx
-; SSE42-NEXT: movq %r9, %rdx
-; SSE42-NEXT: orq %rdi, %rdx
-; SSE42-NEXT: cmovneq %r14, %rbx
-; SSE42-NEXT: bsrq %r10, %rdx
-; SSE42-NEXT: xorq $63, %rdx
+; SSE42-NEXT: testq %r10, %r10
+; SSE42-NEXT: cmovneq %r9, %rbx
; SSE42-NEXT: bsrq %r11, %r14
; SSE42-NEXT: xorq $63, %r14
-; SSE42-NEXT: orq $64, %r14
-; SSE42-NEXT: testq %r10, %r10
-; SSE42-NEXT: cmovneq %rdx, %r14
-; SSE42-NEXT: movq 8(%rsi), %r15
-; SSE42-NEXT: bsrq %r15, %r12
-; SSE42-NEXT: bsrq (%rsi), %rdx
-; SSE42-NEXT: xorq $63, %r12
-; SSE42-NEXT: xorq $63, %rdx
-; SSE42-NEXT: orq $64, %rdx
-; SSE42-NEXT: testq %r15, %r15
-; SSE42-NEXT: cmovneq %r12, %rdx
-; SSE42-NEXT: orq $128, %rdx
-; SSE42-NEXT: orq %r10, %r11
-; SSE42-NEXT: cmovneq %r14, %rdx
-; SSE42-NEXT: orq $256, %rdx # imm = 0x100
-; SSE42-NEXT: orq %r9, %r8
-; SSE42-NEXT: orq %rdi, %rcx
-; SSE42-NEXT: orq %r8, %rcx
-; SSE42-NEXT: cmovneq %rbx, %rdx
-; SSE42-NEXT: pxor %xmm0, %xmm0
-; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: bsrq 32(%rsi), %r9
+; SSE42-NEXT: xorq $63, %r9
+; SSE42-NEXT: orq $64, %r9
+; SSE42-NEXT: testq %r11, %r11
+; SSE42-NEXT: cmovneq %r14, %r9
+; SSE42-NEXT: orq $128, %r9
+; SSE42-NEXT: orq %r10, %rax
+; SSE42-NEXT: cmovneq %rbx, %r9
+; SSE42-NEXT: bsrq %rdx, %rax
+; SSE42-NEXT: xorq $63, %rax
+; SSE42-NEXT: bsrq %rcx, %r10
+; SSE42-NEXT: xorq $63, %r10
+; SSE42-NEXT: orq $64, %r10
+; SSE42-NEXT: testq %rdx, %rdx
+; SSE42-NEXT: cmovneq %rax, %r10
+; SSE42-NEXT: por %xmm2, %xmm1
+; SSE42-NEXT: bsrq %r8, %r11
+; SSE42-NEXT: bsrq (%rsi), %rax
+; SSE42-NEXT: xorq $63, %r11
+; SSE42-NEXT: xorq $63, %rax
+; SSE42-NEXT: orq $64, %rax
+; SSE42-NEXT: testq %r8, %r8
+; SSE42-NEXT: cmovneq %r11, %rax
+; SSE42-NEXT: orq $128, %rax
+; SSE42-NEXT: orq %rdx, %rcx
+; SSE42-NEXT: cmovneq %r10, %rax
+; SSE42-NEXT: orq $256, %rax # imm = 0x100
+; SSE42-NEXT: por %xmm0, %xmm2
+; SSE42-NEXT: ptest %xmm2, %xmm2
+; SSE42-NEXT: cmovneq %r9, %rax
+; SSE42-NEXT: movdqa 16(%rsi), %xmm2
+; SSE42-NEXT: xorps %xmm3, %xmm3
+; SSE42-NEXT: movaps %xmm3, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movaps %xmm3, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movaps %xmm3, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movaps %xmm3, -{{[0-9]+}}(%rsp)
; SSE42-NEXT: movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000
; SSE42-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movl %edx, %ecx
+; SSE42-NEXT: movaps %xmm3, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movaps %xmm3, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movaps %xmm3, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movl %eax, %ecx
; SSE42-NEXT: andl $63, %ecx
-; SSE42-NEXT: shrl $3, %edx
-; SSE42-NEXT: andl $56, %edx
+; SSE42-NEXT: shrl $3, %eax
+; SSE42-NEXT: andl $56, %eax
; SSE42-NEXT: movq $0, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movq -72(%rsp,%rdx), %rsi
-; SSE42-NEXT: movq -80(%rsp,%rdx), %r9
-; SSE42-NEXT: movq %r9, %rdi
-; SSE42-NEXT: shrdq %cl, %rsi, %rdi
-; SSE42-NEXT: movq -88(%rsp,%rdx), %r10
+; SSE42-NEXT: movq -72(%rsp,%rax), %rdx
+; SSE42-NEXT: movq -80(%rsp,%rax), %r9
+; SSE42-NEXT: movq %r9, %rsi
+; SSE42-NEXT: shrdq %cl, %rdx, %rsi
+; SSE42-NEXT: movq -88(%rsp,%rax), %r10
; SSE42-NEXT: movq %r10, %r8
; SSE42-NEXT: shrdq %cl, %r9, %r8
-; SSE42-NEXT: movq -96(%rsp,%rdx), %r11
+; SSE42-NEXT: movq -96(%rsp,%rax), %r11
; SSE42-NEXT: movq %r11, %r9
; SSE42-NEXT: shrdq %cl, %r10, %r9
-; SSE42-NEXT: movq -104(%rsp,%rdx), %rbx
+; SSE42-NEXT: movq -104(%rsp,%rax), %rbx
; SSE42-NEXT: movq %rbx, %r10
; SSE42-NEXT: shrdq %cl, %r11, %r10
-; SSE42-NEXT: movq -112(%rsp,%rdx), %r14
+; SSE42-NEXT: movq -112(%rsp,%rax), %r14
; SSE42-NEXT: movq %r14, %r11
; SSE42-NEXT: shrdq %cl, %rbx, %r11
-; SSE42-NEXT: movq -120(%rsp,%rdx), %r15
+; SSE42-NEXT: movq -120(%rsp,%rax), %r15
; SSE42-NEXT: movq %r15, %rbx
; SSE42-NEXT: shrdq %cl, %r14, %rbx
-; SSE42-NEXT: movq -128(%rsp,%rdx), %rdx
-; SSE42-NEXT: shrq %cl, %rsi
+; SSE42-NEXT: movq -128(%rsp,%rax), %r14
+; SSE42-NEXT: shrq %cl, %rdx
; SSE42-NEXT: # kill: def $cl killed $cl killed $ecx
-; SSE42-NEXT: shrdq %cl, %r15, %rdx
-; SSE42-NEXT: jmp .LBB17_3
-; SSE42-NEXT: .LBB17_1:
-; SSE42-NEXT: xorl %edx, %edx
-; SSE42-NEXT: xorl %ebx, %ebx
-; SSE42-NEXT: xorl %r11d, %r11d
-; SSE42-NEXT: xorl %r10d, %r10d
-; SSE42-NEXT: xorl %r9d, %r9d
-; SSE42-NEXT: xorl %r8d, %r8d
-; SSE42-NEXT: xorl %edi, %edi
-; SSE42-NEXT: xorl %esi, %esi
-; SSE42-NEXT: .LBB17_3: # %select.end
-; SSE42-NEXT: movq %rdx, (%rax)
-; SSE42-NEXT: movq %rbx, 8(%rax)
-; SSE42-NEXT: movq %r11, 16(%rax)
-; SSE42-NEXT: movq %r10, 24(%rax)
-; SSE42-NEXT: movq %r9, 32(%rax)
-; SSE42-NEXT: movq %r8, 40(%rax)
-; SSE42-NEXT: movq %rdi, 48(%rax)
-; SSE42-NEXT: movq %rsi, 56(%rax)
-; SSE42-NEXT: addq $8, %rsp
+; SSE42-NEXT: shrdq %cl, %r15, %r14
+; SSE42-NEXT: por %xmm0, %xmm2
+; SSE42-NEXT: por %xmm2, %xmm1
+; SSE42-NEXT: xorl %ecx, %ecx
+; SSE42-NEXT: ptest %xmm1, %xmm1
+; SSE42-NEXT: cmoveq %rcx, %rbx
+; SSE42-NEXT: cmoveq %rcx, %r11
+; SSE42-NEXT: cmoveq %rcx, %r10
+; SSE42-NEXT: cmoveq %rcx, %r9
+; SSE42-NEXT: cmoveq %rcx, %r8
+; SSE42-NEXT: cmoveq %rcx, %rsi
+; SSE42-NEXT: cmoveq %rcx, %r14
+; SSE42-NEXT: movq %rdi, %rax
+; SSE42-NEXT: cmoveq %rcx, %rdx
+; SSE42-NEXT: movq %rdx, 56(%rdi)
+; SSE42-NEXT: movq %rsi, 48(%rdi)
+; SSE42-NEXT: movq %r8, 40(%rdi)
+; SSE42-NEXT: movq %r9, 32(%rdi)
+; SSE42-NEXT: movq %r10, 24(%rdi)
+; SSE42-NEXT: movq %r11, 16(%rdi)
+; SSE42-NEXT: movq %rbx, 8(%rdi)
+; SSE42-NEXT: movq %r14, (%rdi)
; SSE42-NEXT: popq %rbx
-; SSE42-NEXT: popq %r12
; SSE42-NEXT: popq %r14
; SSE42-NEXT: popq %r15
; SSE42-NEXT: retq
@@ -5473,111 +5190,102 @@ define i512 @isolate_msb_i512_load(ptr %p0, i512 %idx) nounwind {
; AVX2-NEXT: pushq %r15
; AVX2-NEXT: pushq %r14
; AVX2-NEXT: pushq %rbx
-; AVX2-NEXT: movq %rdi, %rax
-; AVX2-NEXT: vmovdqu (%rsi), %ymm0
-; AVX2-NEXT: vpor 32(%rsi), %ymm0, %ymm0
-; AVX2-NEXT: vptest %ymm0, %ymm0
-; AVX2-NEXT: je .LBB17_1
-; AVX2-NEXT: # %bb.2: # %select.false.sink
-; AVX2-NEXT: movq 56(%rsi), %rdi
-; AVX2-NEXT: movq 48(%rsi), %r9
-; AVX2-NEXT: movq 40(%rsi), %rcx
-; AVX2-NEXT: movq 32(%rsi), %r8
-; AVX2-NEXT: movq 24(%rsi), %r10
-; AVX2-NEXT: movq 16(%rsi), %r11
-; AVX2-NEXT: movq (%rsi), %rdx
-; AVX2-NEXT: movq 8(%rsi), %rbx
-; AVX2-NEXT: xorl %esi, %esi
-; AVX2-NEXT: lzcntq %rdi, %rsi
+; AVX2-NEXT: vmovdqu 32(%rsi), %ymm1
+; AVX2-NEXT: movq 8(%rsi), %r8
+; AVX2-NEXT: movq 16(%rsi), %rax
+; AVX2-NEXT: movq 24(%rsi), %rcx
+; AVX2-NEXT: movq 40(%rsi), %rdx
+; AVX2-NEXT: movq 48(%rsi), %r10
+; AVX2-NEXT: vpor (%rsi), %ymm1, %ymm0
+; AVX2-NEXT: movq 56(%rsi), %r11
+; AVX2-NEXT: lzcntq %r11, %r9
+; AVX2-NEXT: xorl %ebx, %ebx
+; AVX2-NEXT: lzcntq %r10, %rbx
+; AVX2-NEXT: addq $64, %rbx
+; AVX2-NEXT: testq %r11, %r11
+; AVX2-NEXT: cmovneq %r9, %rbx
; AVX2-NEXT: xorl %r14d, %r14d
-; AVX2-NEXT: lzcntq %r9, %r14
-; AVX2-NEXT: addq $64, %r14
-; AVX2-NEXT: testq %rdi, %rdi
-; AVX2-NEXT: cmovneq %rsi, %r14
-; AVX2-NEXT: xorl %r15d, %r15d
-; AVX2-NEXT: lzcntq %rcx, %r15
-; AVX2-NEXT: xorl %esi, %esi
-; AVX2-NEXT: lzcntq %r8, %rsi
-; AVX2-NEXT: addq $64, %rsi
+; AVX2-NEXT: lzcntq %rdx, %r14
+; AVX2-NEXT: xorl %r9d, %r9d
+; AVX2-NEXT: lzcntq 32(%rsi), %r9
+; AVX2-NEXT: addq $64, %r9
+; AVX2-NEXT: testq %rdx, %rdx
+; AVX2-NEXT: cmovneq %r14, %r9
+; AVX2-NEXT: subq $-128, %r9
+; AVX2-NEXT: orq %r11, %r10
+; AVX2-NEXT: cmovneq %rbx, %r9
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: lzcntq %rcx, %rdx
+; AVX2-NEXT: xorl %r10d, %r10d
+; AVX2-NEXT: lzcntq %rax, %r10
+; AVX2-NEXT: addq $64, %r10
; AVX2-NEXT: testq %rcx, %rcx
-; AVX2-NEXT: cmovneq %r15, %rsi
-; AVX2-NEXT: subq $-128, %rsi
-; AVX2-NEXT: movq %r9, %r15
-; AVX2-NEXT: orq %rdi, %r15
-; AVX2-NEXT: cmovneq %r14, %rsi
-; AVX2-NEXT: xorl %r14d, %r14d
-; AVX2-NEXT: lzcntq %r10, %r14
-; AVX2-NEXT: xorl %r15d, %r15d
-; AVX2-NEXT: lzcntq %r11, %r15
-; AVX2-NEXT: addq $64, %r15
-; AVX2-NEXT: testq %r10, %r10
-; AVX2-NEXT: cmovneq %r14, %r15
-; AVX2-NEXT: xorl %r14d, %r14d
-; AVX2-NEXT: lzcntq %rbx, %r14
-; AVX2-NEXT: lzcntq %rdx, %rdx
+; AVX2-NEXT: cmovneq %rdx, %r10
+; AVX2-NEXT: xorl %r11d, %r11d
+; AVX2-NEXT: lzcntq %r8, %r11
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: lzcntq (%rsi), %rdx
; AVX2-NEXT: addq $64, %rdx
-; AVX2-NEXT: testq %rbx, %rbx
-; AVX2-NEXT: cmovneq %r14, %rdx
+; AVX2-NEXT: testq %r8, %r8
+; AVX2-NEXT: cmovneq %r11, %rdx
; AVX2-NEXT: subq $-128, %rdx
-; AVX2-NEXT: orq %r10, %r11
-; AVX2-NEXT: cmovneq %r15, %rdx
+; AVX2-NEXT: orq %rcx, %rax
+; AVX2-NEXT: cmovneq %r10, %rdx
; AVX2-NEXT: addq $256, %rdx # imm = 0x100
-; AVX2-NEXT: orq %r9, %r8
-; AVX2-NEXT: orq %rdi, %rcx
-; AVX2-NEXT: orq %r8, %rcx
-; AVX2-NEXT: cmovneq %rsi, %rdx
-; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vmovaps {{.*#+}} ymm1 = [0,0,0,9223372036854775808]
-; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vpor 48(%rsi), %xmm1, %xmm1
+; AVX2-NEXT: vptest %xmm1, %xmm1
+; AVX2-NEXT: cmovneq %r9, %rdx
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vmovdqu %ymm1, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovdqu %ymm1, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovaps {{.*#+}} ymm2 = [0,0,0,9223372036854775808]
+; AVX2-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovdqu %ymm1, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movl %edx, %ecx
; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: shrl $3, %edx
; AVX2-NEXT: andl $56, %edx
-; AVX2-NEXT: movq -72(%rsp,%rdx), %rbx
-; AVX2-NEXT: movq -80(%rsp,%rdx), %r8
-; AVX2-NEXT: movq %r8, %rsi
-; AVX2-NEXT: shrdq %cl, %rbx, %rsi
-; AVX2-NEXT: movq -88(%rsp,%rdx), %r9
-; AVX2-NEXT: movq %r9, %rdi
-; AVX2-NEXT: shrdq %cl, %r8, %rdi
-; AVX2-NEXT: movq -96(%rsp,%rdx), %r10
+; AVX2-NEXT: movq -72(%rsp,%rdx), %r11
+; AVX2-NEXT: movq -80(%rsp,%rdx), %rax
+; AVX2-NEXT: movq %rax, %rsi
+; AVX2-NEXT: shrdq %cl, %r11, %rsi
+; AVX2-NEXT: movq -88(%rsp,%rdx), %r10
; AVX2-NEXT: movq %r10, %r8
-; AVX2-NEXT: shrdq %cl, %r9, %r8
-; AVX2-NEXT: movq -104(%rsp,%rdx), %r11
-; AVX2-NEXT: movq %r11, %r9
+; AVX2-NEXT: shrdq %cl, %rax, %r8
+; AVX2-NEXT: movq -96(%rsp,%rdx), %rax
+; AVX2-NEXT: movq %rax, %r9
; AVX2-NEXT: shrdq %cl, %r10, %r9
-; AVX2-NEXT: movq -112(%rsp,%rdx), %r14
+; AVX2-NEXT: movq -104(%rsp,%rdx), %r14
; AVX2-NEXT: movq %r14, %r10
-; AVX2-NEXT: shrdq %cl, %r11, %r10
-; AVX2-NEXT: movq -120(%rsp,%rdx), %r15
-; AVX2-NEXT: movq %r15, %r11
-; AVX2-NEXT: shrdq %cl, %r14, %r11
-; AVX2-NEXT: movq -128(%rsp,%rdx), %rdx
-; AVX2-NEXT: shrxq %rcx, %rbx, %rbx
-; AVX2-NEXT: # kill: def $cl killed $cl killed $rcx
+; AVX2-NEXT: shrdq %cl, %rax, %r10
+; AVX2-NEXT: movq -112(%rsp,%rdx), %r15
+; AVX2-NEXT: movq %r15, %rbx
+; AVX2-NEXT: shrdq %cl, %r14, %rbx
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: movq -128(%rsp,%rdx), %rdi
+; AVX2-NEXT: movq -120(%rsp,%rdx), %r14
+; AVX2-NEXT: movq %r14, %rdx
; AVX2-NEXT: shrdq %cl, %r15, %rdx
-; AVX2-NEXT: jmp .LBB17_3
-; AVX2-NEXT: .LBB17_1:
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: xorl %r11d, %r11d
-; AVX2-NEXT: xorl %r10d, %r10d
-; AVX2-NEXT: xorl %r9d, %r9d
-; AVX2-NEXT: xorl %r8d, %r8d
-; AVX2-NEXT: xorl %edi, %edi
-; AVX2-NEXT: xorl %esi, %esi
-; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: .LBB17_3: # %select.end
-; AVX2-NEXT: movq %rdx, (%rax)
-; AVX2-NEXT: movq %r11, 8(%rax)
-; AVX2-NEXT: movq %r10, 16(%rax)
-; AVX2-NEXT: movq %r9, 24(%rax)
-; AVX2-NEXT: movq %r8, 32(%rax)
-; AVX2-NEXT: movq %rdi, 40(%rax)
+; AVX2-NEXT: shrdq %cl, %r14, %rdi
+; AVX2-NEXT: xorl %r14d, %r14d
+; AVX2-NEXT: vptest %ymm0, %ymm0
+; AVX2-NEXT: shrxq %rcx, %r11, %rcx
+; AVX2-NEXT: cmoveq %r14, %rdx
+; AVX2-NEXT: cmoveq %r14, %rbx
+; AVX2-NEXT: cmoveq %r14, %r10
+; AVX2-NEXT: cmoveq %r14, %r9
+; AVX2-NEXT: cmoveq %r14, %r8
+; AVX2-NEXT: cmoveq %r14, %rsi
+; AVX2-NEXT: cmoveq %r14, %rdi
+; AVX2-NEXT: cmoveq %r14, %rcx
+; AVX2-NEXT: movq %rcx, 56(%rax)
; AVX2-NEXT: movq %rsi, 48(%rax)
-; AVX2-NEXT: movq %rbx, 56(%rax)
+; AVX2-NEXT: movq %r8, 40(%rax)
+; AVX2-NEXT: movq %r9, 32(%rax)
+; AVX2-NEXT: movq %r10, 24(%rax)
+; AVX2-NEXT: movq %rbx, 16(%rax)
+; AVX2-NEXT: movq %rdx, 8(%rax)
+; AVX2-NEXT: movq %rdi, (%rax)
; AVX2-NEXT: popq %rbx
; AVX2-NEXT: popq %r14
; AVX2-NEXT: popq %r15
@@ -5589,66 +5297,28 @@ define i512 @isolate_msb_i512_load(ptr %p0, i512 %idx) nounwind {
; AVX512F-NEXT: movq %rdi, %rax
; AVX512F-NEXT: vmovdqu64 (%rsi), %zmm0
; AVX512F-NEXT: vptestmd %zmm0, %zmm0, %k0
+; AVX512F-NEXT: xorl %ecx, %ecx
; AVX512F-NEXT: kortestw %k0, %k0
-; AVX512F-NEXT: je .LBB17_1
-; AVX512F-NEXT: # %bb.2: # %select.false.sink
-; AVX512F-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX512F-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX512F-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX512F-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero
-; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX512F-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
-; AVX512F-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX512F-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero
-; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX512F-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero
-; AVX512F-NEXT: vmovq {{.*#+}} xmm3 = mem[0],zero
-; AVX512F-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
-; AVX512F-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
-; AVX512F-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
-; AVX512F-NEXT: vplzcntq %zmm0, %zmm1
-; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1 # [0,64,128,192,256,320,384,448]
+; AVX512F-NEXT: sete %cl
+; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
+; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0
; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
-; AVX512F-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
-; AVX512F-NEXT: vmovq %xmm0, %rcx
-; AVX512F-NEXT: movabsq $-9223372036854775808, %rdx # imm = 0x8000000000000000
-; AVX512F-NEXT: shrxq %rcx, %rdx, %rdx
-; AVX512F-NEXT: shrl $6, %ecx
-; AVX512F-NEXT: movl $128, %esi
-; AVX512F-NEXT: shrxq %rcx, %rsi, %rcx
-; AVX512F-NEXT: kmovw %ecx, %k1
-; AVX512F-NEXT: vpbroadcastq %rdx, %zmm0 {%k1} {z}
-; AVX512F-NEXT: vextracti32x4 $3, %zmm0, %xmm1
-; AVX512F-NEXT: vpextrq $1, %xmm1, %rcx
-; AVX512F-NEXT: vmovq %xmm1, %rdx
-; AVX512F-NEXT: vextracti32x4 $2, %zmm0, %xmm1
-; AVX512F-NEXT: vpextrq $1, %xmm1, %rsi
-; AVX512F-NEXT: vmovq %xmm1, %r8
-; AVX512F-NEXT: vpextrq $1, %xmm0, %r9
-; AVX512F-NEXT: vmovq %xmm0, %rdi
-; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm0
-; AVX512F-NEXT: vpextrq $1, %xmm0, %r10
-; AVX512F-NEXT: vmovq %xmm0, %r11
-; AVX512F-NEXT: jmp .LBB17_3
-; AVX512F-NEXT: .LBB17_1:
-; AVX512F-NEXT: xorl %edi, %edi
-; AVX512F-NEXT: xorl %r9d, %r9d
-; AVX512F-NEXT: xorl %r11d, %r11d
-; AVX512F-NEXT: xorl %r10d, %r10d
-; AVX512F-NEXT: xorl %r8d, %r8d
-; AVX512F-NEXT: xorl %esi, %esi
-; AVX512F-NEXT: xorl %edx, %edx
-; AVX512F-NEXT: xorl %ecx, %ecx
-; AVX512F-NEXT: .LBB17_3: # %select.end
-; AVX512F-NEXT: movq %rdi, (%rax)
-; AVX512F-NEXT: movq %r9, 8(%rax)
-; AVX512F-NEXT: movq %r11, 16(%rax)
-; AVX512F-NEXT: movq %r10, 24(%rax)
-; AVX512F-NEXT: movq %r8, 32(%rax)
-; AVX512F-NEXT: movq %rsi, 40(%rax)
-; AVX512F-NEXT: movq %rdx, 48(%rax)
-; AVX512F-NEXT: movq %rcx, 56(%rax)
+; AVX512F-NEXT: vplzcntq %zmm0, %zmm0
+; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,64,128,192,256,320,384,448]
+; AVX512F-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT: vmovq %xmm0, %rdx
+; AVX512F-NEXT: movabsq $-9223372036854775808, %rsi # imm = 0x8000000000000000
+; AVX512F-NEXT: shrxq %rdx, %rsi, %rsi
+; AVX512F-NEXT: shrl $6, %edx
+; AVX512F-NEXT: movl $128, %edi
+; AVX512F-NEXT: shrxq %rdx, %rdi, %rdx
+; AVX512F-NEXT: kmovw %edx, %k1
+; AVX512F-NEXT: vpbroadcastq %rsi, %zmm0 {%k1} {z}
+; AVX512F-NEXT: negl %ecx
+; AVX512F-NEXT: kmovw %ecx, %k0
+; AVX512F-NEXT: knotw %k0, %k1
+; AVX512F-NEXT: vmovdqa32 %zmm0, %zmm0 {%k1} {z}
+; AVX512F-NEXT: vmovdqu64 %zmm0, (%rax)
; AVX512F-NEXT: retq
;
; AVX512VL-LABEL: isolate_msb_i512_load:
@@ -5656,66 +5326,28 @@ define i512 @isolate_msb_i512_load(ptr %p0, i512 %idx) nounwind {
; AVX512VL-NEXT: movq %rdi, %rax
; AVX512VL-NEXT: vmovdqu64 (%rsi), %zmm0
; AVX512VL-NEXT: vptestmd %zmm0, %zmm0, %k0
+; AVX512VL-NEXT: xorl %ecx, %ecx
; AVX512VL-NEXT: kortestw %k0, %k0
-; AVX512VL-NEXT: je .LBB17_1
-; AVX512VL-NEXT: # %bb.2: # %select.false.sink
-; AVX512VL-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX512VL-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX512VL-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX512VL-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero
-; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX512VL-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
-; AVX512VL-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX512VL-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero
-; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX512VL-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero
-; AVX512VL-NEXT: vmovq {{.*#+}} xmm3 = mem[0],zero
-; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
-; AVX512VL-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
-; AVX512VL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512VL-NEXT: sete %cl
+; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
+; AVX512VL-NEXT: vpermq %zmm0, %zmm1, %zmm0
; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VL-NEXT: vplzcntq %zmm0, %zmm0
; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,64,128,192,256,320,384,448]
; AVX512VL-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
-; AVX512VL-NEXT: vmovq %xmm0, %rcx
-; AVX512VL-NEXT: movabsq $-9223372036854775808, %rdx # imm = 0x8000000000000000
-; AVX512VL-NEXT: shrxq %rcx, %rdx, %rdx
-; AVX512VL-NEXT: shrl $6, %ecx
-; AVX512VL-NEXT: movl $128, %esi
-; AVX512VL-NEXT: shrxq %rcx, %rsi, %rcx
-; AVX512VL-NEXT: kmovd %ecx, %k1
-; AVX512VL-NEXT: vpbroadcastq %rdx, %zmm0 {%k1} {z}
-; AVX512VL-NEXT: vextracti32x4 $3, %zmm0, %xmm1
-; AVX512VL-NEXT: vpextrq $1, %xmm1, %rcx
-; AVX512VL-NEXT: vmovq %xmm1, %rdx
-; AVX512VL-NEXT: vextracti32x4 $2, %zmm0, %xmm1
-; AVX512VL-NEXT: vpextrq $1, %xmm1, %rsi
-; AVX512VL-NEXT: vpextrq $1, %xmm0, %r8
-; AVX512VL-NEXT: vmovq %xmm1, %r9
-; AVX512VL-NEXT: vmovq %xmm0, %rdi
-; AVX512VL-NEXT: vextracti128 $1, %ymm0, %xmm0
-; AVX512VL-NEXT: vpextrq $1, %xmm0, %r10
-; AVX512VL-NEXT: vmovq %xmm0, %r11
-; AVX512VL-NEXT: jmp .LBB17_3
-; AVX512VL-NEXT: .LBB17_1:
-; AVX512VL-NEXT: xorl %edi, %edi
-; AVX512VL-NEXT: xorl %r8d, %r8d
-; AVX512VL-NEXT: xorl %r11d, %r11d
-; AVX512VL-NEXT: xorl %r10d, %r10d
-; AVX512VL-NEXT: xorl %r9d, %r9d
-; AVX512VL-NEXT: xorl %esi, %esi
-; AVX512VL-NEXT: xorl %edx, %edx
-; AVX512VL-NEXT: xorl %ecx, %ecx
-; AVX512VL-NEXT: .LBB17_3: # %select.end
-; AVX512VL-NEXT: movq %rdi, (%rax)
-; AVX512VL-NEXT: movq %r8, 8(%rax)
-; AVX512VL-NEXT: movq %r11, 16(%rax)
-; AVX512VL-NEXT: movq %r10, 24(%rax)
-; AVX512VL-NEXT: movq %r9, 32(%rax)
-; AVX512VL-NEXT: movq %rsi, 40(%rax)
-; AVX512VL-NEXT: movq %rdx, 48(%rax)
-; AVX512VL-NEXT: movq %rcx, 56(%rax)
+; AVX512VL-NEXT: vmovq %xmm0, %rdx
+; AVX512VL-NEXT: movabsq $-9223372036854775808, %rsi # imm = 0x8000000000000000
+; AVX512VL-NEXT: shrxq %rdx, %rsi, %rsi
+; AVX512VL-NEXT: shrl $6, %edx
+; AVX512VL-NEXT: movl $128, %edi
+; AVX512VL-NEXT: shrxq %rdx, %rdi, %rdx
+; AVX512VL-NEXT: kmovd %edx, %k1
+; AVX512VL-NEXT: vpbroadcastq %rsi, %zmm0 {%k1} {z}
+; AVX512VL-NEXT: negl %ecx
+; AVX512VL-NEXT: kmovd %ecx, %k0
+; AVX512VL-NEXT: knotw %k0, %k1
+; AVX512VL-NEXT: vmovdqa32 %zmm0, %zmm0 {%k1} {z}
+; AVX512VL-NEXT: vmovdqu64 %zmm0, (%rax)
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
@@ -5724,66 +5356,28 @@ define i512 @isolate_msb_i512_load(ptr %p0, i512 %idx) nounwind {
; AVX512VBMI-NEXT: movq %rdi, %rax
; AVX512VBMI-NEXT: vmovdqu64 (%rsi), %zmm0
; AVX512VBMI-NEXT: vptestmd %zmm0, %zmm0, %k0
+; AVX512VBMI-NEXT: xorl %ecx, %ecx
; AVX512VBMI-NEXT: kortestw %k0, %k0
-; AVX512VBMI-NEXT: je .LBB17_1
-; AVX512VBMI-NEXT: # %bb.2: # %select.false.sink
-; AVX512VBMI-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
-; AVX512VBMI-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX512VBMI-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
-; AVX512VBMI-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX512VBMI-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero
-; AVX512VBMI-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX512VBMI-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
-; AVX512VBMI-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX512VBMI-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero
-; AVX512VBMI-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; AVX512VBMI-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero
-; AVX512VBMI-NEXT: vmovq {{.*#+}} xmm3 = mem[0],zero
-; AVX512VBMI-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm3[0],xmm2[0]
-; AVX512VBMI-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
-; AVX512VBMI-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512VBMI-NEXT: sete %cl
+; AVX512VBMI-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
+; AVX512VBMI-NEXT: vpermq %zmm0, %zmm1, %zmm0
; AVX512VBMI-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VBMI-NEXT: vplzcntq %zmm0, %zmm0
; AVX512VBMI-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,64,128,192,256,320,384,448]
; AVX512VBMI-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
-; AVX512VBMI-NEXT: vmovq %xmm0, %rcx
-; AVX512VBMI-NEXT: movabsq $-9223372036854775808, %rdx # imm = 0x8000000000000000
-; AVX512VBMI-NEXT: shrxq %rcx, %rdx, %rdx
-; AVX512VBMI-NEXT: shrl $6, %ecx
-; AVX512VBMI-NEXT: movl $128, %esi
-; AVX512VBMI-NEXT: shrxq %rcx, %rsi, %rcx
-; AVX512VBMI-NEXT: kmovd %ecx, %k1
-; AVX512VBMI-NEXT: vpbroadcastq %rdx, %zmm0 {%k1} {z}
-; AVX512VBMI-NEXT: vextracti32x4 $3, %zmm0, %xmm1
-; AVX512VBMI-NEXT: vpextrq $1, %xmm1, %rcx
-; AVX512VBMI-NEXT: vmovq %xmm1, %rdx
-; AVX512VBMI-NEXT: vextracti32x4 $2, %zmm0, %xmm1
-; AVX512VBMI-NEXT: vpextrq $1, %xmm1, %rsi
-; AVX512VBMI-NEXT: vpextrq $1, %xmm0, %r8
-; AVX512VBMI-NEXT: vmovq %xmm1, %r9
-; AVX512VBMI-NEXT: vmovq %xmm0, %rdi
-; AVX512VBMI-NEXT: vextracti128 $1, %ymm0, %xmm0
-; AVX512VBMI-NEXT: vpextrq $1, %xmm0, %r10
-; AVX512VBMI-NEXT: vmovq %xmm0, %r11
-; AVX512VBMI-NEXT: jmp .LBB17_3
-; AVX512VBMI-NEXT: .LBB17_1:
-; AVX512VBMI-NEXT: xorl %edi, %edi
-; AVX512VBMI-NEXT: xorl %r8d, %r8d
-; AVX512VBMI-NEXT: xorl %r11d, %r11d
-; AVX512VBMI-NEXT: xorl %r10d, %r10d
-; AVX512VBMI-NEXT: xorl %r9d, %r9d
-; AVX512VBMI-NEXT: xorl %esi, %esi
-; AVX512VBMI-NEXT: xorl %edx, %edx
-; AVX512VBMI-NEXT: xorl %ecx, %ecx
-; AVX512VBMI-NEXT: .LBB17_3: # %select.end
-; AVX512VBMI-NEXT: movq %rdi, (%rax)
-; AVX512VBMI-NEXT: movq %r8, 8(%rax)
-; AVX512VBMI-NEXT: movq %r11, 16(%rax)
-; AVX512VBMI-NEXT: movq %r10, 24(%rax)
-; AVX512VBMI-NEXT: movq %r9, 32(%rax)
-; AVX512VBMI-NEXT: movq %rsi, 40(%rax)
-; AVX512VBMI-NEXT: movq %rdx, 48(%rax)
-; AVX512VBMI-NEXT: movq %rcx, 56(%rax)
+; AVX512VBMI-NEXT: vmovq %xmm0, %rdx
+; AVX512VBMI-NEXT: movabsq $-9223372036854775808, %rsi # imm = 0x8000000000000000
+; AVX512VBMI-NEXT: shrxq %rdx, %rsi, %rsi
+; AVX512VBMI-NEXT: shrl $6, %edx
+; AVX512VBMI-NEXT: movl $128, %edi
+; AVX512VBMI-NEXT: shrxq %rdx, %rdi, %rdx
+; AVX512VBMI-NEXT: kmovd %edx, %k1
+; AVX512VBMI-NEXT: vpbroadcastq %rsi, %zmm0 {%k1} {z}
+; AVX512VBMI-NEXT: negl %ecx
+; AVX512VBMI-NEXT: kmovd %ecx, %k0
+; AVX512VBMI-NEXT: knotw %k0, %k1
+; AVX512VBMI-NEXT: vmovdqa32 %zmm0, %zmm0 {%k1} {z}
+; AVX512VBMI-NEXT: vmovdqu64 %zmm0, (%rax)
; AVX512VBMI-NEXT: vzeroupper
; AVX512VBMI-NEXT: retq
%a0 = load i512, ptr %p0
diff --git a/llvm/test/CodeGen/X86/bsr.ll b/llvm/test/CodeGen/X86/bsr.ll
index 44e9f60bd49eb..affacc5ee6487 100644
--- a/llvm/test/CodeGen/X86/bsr.ll
+++ b/llvm/test/CodeGen/X86/bsr.ll
@@ -248,27 +248,26 @@ define i64 @cmov_bsr64(i64 %x, i64 %y) nounwind {
define i64 @cmov_bsr64_undef(i64 %x, i64 %y) nounwind {
; X86-LABEL: cmov_bsr64_undef:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ecx, %edx
-; X86-NEXT: orl %eax, %edx
-; X86-NEXT: je .LBB7_1
-; X86-NEXT: # %bb.2: # %select.false.sink
-; X86-NEXT: testl %eax, %eax
-; X86-NEXT: jne .LBB7_3
-; X86-NEXT: # %bb.4: # %select.false.sink
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: testl %edx, %edx
+; X86-NEXT: jne .LBB7_1
+; X86-NEXT: # %bb.2:
; X86-NEXT: bsrl %ecx, %eax
; X86-NEXT: xorl $31, %eax
; X86-NEXT: orl $32, %eax
-; X86-NEXT: jmp .LBB7_5
-; X86-NEXT: .LBB7_1:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: orl %edx, %ecx
+; X86-NEXT: jne .LBB7_5
+; X86-NEXT: .LBB7_4:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-NEXT: retl
-; X86-NEXT: .LBB7_3:
-; X86-NEXT: bsrl %eax, %eax
+; X86-NEXT: .LBB7_1:
+; X86-NEXT: bsrl %edx, %eax
; X86-NEXT: xorl $31, %eax
-; X86-NEXT: .LBB7_5: # %select.false.sink
+; X86-NEXT: orl %edx, %ecx
+; X86-NEXT: je .LBB7_4
+; X86-NEXT: .LBB7_5:
; X86-NEXT: xorl $63, %eax
; X86-NEXT: xorl %edx, %edx
; X86-NEXT: retl
@@ -373,23 +372,20 @@ define i128 @cmov_bsr128(i128 %x, i128 %y) nounwind {
;
; X64-LABEL: cmov_bsr128:
; X64: # %bb.0:
-; X64-NEXT: movq %rdx, %rax
-; X64-NEXT: movq %rdi, %rdx
-; X64-NEXT: orq %rsi, %rdx
-; X64-NEXT: je .LBB8_2
-; X64-NEXT: # %bb.1: # %select.false.sink
-; X64-NEXT: bsrq %rsi, %rcx
-; X64-NEXT: xorq $63, %rcx
+; X64-NEXT: bsrq %rsi, %r8
+; X64-NEXT: xorq $63, %r8
; X64-NEXT: movl $127, %eax
; X64-NEXT: bsrq %rdi, %rax
; X64-NEXT: xorq $63, %rax
; X64-NEXT: addq $64, %rax
; X64-NEXT: testq %rsi, %rsi
-; X64-NEXT: cmovneq %rcx, %rax
+; X64-NEXT: cmovneq %r8, %rax
; X64-NEXT: xorq $127, %rax
-; X64-NEXT: xorl %ecx, %ecx
-; X64-NEXT: .LBB8_2: # %select.end
-; X64-NEXT: movq %rcx, %rdx
+; X64-NEXT: xorl %r8d, %r8d
+; X64-NEXT: orq %rsi, %rdi
+; X64-NEXT: cmoveq %rdx, %rax
+; X64-NEXT: cmoveq %rcx, %r8
+; X64-NEXT: movq %r8, %rdx
; X64-NEXT: retq
%1 = tail call i128 @llvm.ctlz.i128(i128 %x, i1 false)
%2 = xor i128 %1, 127
@@ -408,62 +404,60 @@ define i128 @cmov_bsr128_undef(i128 %x, i128 %y) nounwind {
; X86-NEXT: pushl %esi
; X86-NEXT: andl $-16, %esp
; X86-NEXT: subl $16, %esp
-; X86-NEXT: movl 36(%ebp), %edx
-; X86-NEXT: movl 32(%ebp), %esi
-; X86-NEXT: movl 28(%ebp), %edi
+; X86-NEXT: movl 28(%ebp), %edx
+; X86-NEXT: movl 32(%ebp), %edi
+; X86-NEXT: movl 36(%ebp), %eax
+; X86-NEXT: testl %eax, %eax
+; X86-NEXT: jne .LBB9_1
+; X86-NEXT: # %bb.2:
+; X86-NEXT: bsrl %edi, %esi
+; X86-NEXT: xorl $31, %esi
+; X86-NEXT: orl $32, %esi
+; X86-NEXT: jmp .LBB9_3
+; X86-NEXT: .LBB9_1:
+; X86-NEXT: bsrl %eax, %esi
+; X86-NEXT: xorl $31, %esi
+; X86-NEXT: .LBB9_3:
; X86-NEXT: movl 24(%ebp), %ebx
-; X86-NEXT: movl %edi, %eax
-; X86-NEXT: orl %edx, %eax
-; X86-NEXT: movl %ebx, %ecx
-; X86-NEXT: orl %esi, %ecx
-; X86-NEXT: orl %eax, %ecx
-; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: je .LBB9_1
-; X86-NEXT: # %bb.2: # %select.true.sink
; X86-NEXT: testl %edx, %edx
-; X86-NEXT: jne .LBB9_3
-; X86-NEXT: # %bb.4: # %select.true.sink
-; X86-NEXT: bsrl %esi, %ecx
+; X86-NEXT: jne .LBB9_4
+; X86-NEXT: # %bb.5:
+; X86-NEXT: bsrl %ebx, %ecx
; X86-NEXT: xorl $31, %ecx
; X86-NEXT: orl $32, %ecx
-; X86-NEXT: testl %edi, %edi
+; X86-NEXT: orl %eax, %edi
; X86-NEXT: je .LBB9_7
-; X86-NEXT: .LBB9_6:
-; X86-NEXT: bsrl %edi, %edi
-; X86-NEXT: xorl $31, %edi
-; X86-NEXT: orl %edx, %esi
-; X86-NEXT: je .LBB9_9
-; X86-NEXT: jmp .LBB9_10
-; X86-NEXT: .LBB9_1:
-; X86-NEXT: movl 52(%ebp), %esi
-; X86-NEXT: movl 48(%ebp), %edi
-; X86-NEXT: movl 44(%ebp), %edx
-; X86-NEXT: movl 40(%ebp), %ecx
-; X86-NEXT: jmp .LBB9_11
-; X86-NEXT: .LBB9_3:
+; X86-NEXT: jmp .LBB9_8
+; X86-NEXT: .LBB9_4:
; X86-NEXT: bsrl %edx, %ecx
; X86-NEXT: xorl $31, %ecx
-; X86-NEXT: testl %edi, %edi
-; X86-NEXT: jne .LBB9_6
-; X86-NEXT: .LBB9_7: # %select.true.sink
-; X86-NEXT: bsrl %ebx, %edi
-; X86-NEXT: xorl $31, %edi
-; X86-NEXT: orl $32, %edi
-; X86-NEXT: orl %edx, %esi
-; X86-NEXT: jne .LBB9_10
-; X86-NEXT: .LBB9_9: # %select.true.sink
-; X86-NEXT: orl $64, %edi
-; X86-NEXT: movl %edi, %ecx
-; X86-NEXT: .LBB9_10: # %select.true.sink
-; X86-NEXT: xorl $127, %ecx
+; X86-NEXT: orl %eax, %edi
+; X86-NEXT: jne .LBB9_8
+; X86-NEXT: .LBB9_7:
+; X86-NEXT: orl $64, %ecx
+; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: .LBB9_8:
+; X86-NEXT: orl %eax, %edx
+; X86-NEXT: orl 32(%ebp), %ebx
+; X86-NEXT: orl %edx, %ebx
+; X86-NEXT: jne .LBB9_9
+; X86-NEXT: # %bb.10:
+; X86-NEXT: movl 48(%ebp), %edx
+; X86-NEXT: movl 52(%ebp), %edi
+; X86-NEXT: movl 40(%ebp), %esi
+; X86-NEXT: movl 44(%ebp), %ecx
+; X86-NEXT: jmp .LBB9_11
+; X86-NEXT: .LBB9_9:
+; X86-NEXT: xorl $127, %esi
+; X86-NEXT: xorl %ecx, %ecx
; X86-NEXT: xorl %edx, %edx
; X86-NEXT: xorl %edi, %edi
-; X86-NEXT: xorl %esi, %esi
-; X86-NEXT: .LBB9_11: # %select.end
-; X86-NEXT: movl %ecx, (%eax)
-; X86-NEXT: movl %edx, 4(%eax)
-; X86-NEXT: movl %edi, 8(%eax)
-; X86-NEXT: movl %esi, 12(%eax)
+; X86-NEXT: .LBB9_11:
+; X86-NEXT: movl 8(%ebp), %eax
+; X86-NEXT: movl %edi, 12(%eax)
+; X86-NEXT: movl %edx, 8(%eax)
+; X86-NEXT: movl %ecx, 4(%eax)
+; X86-NEXT: movl %esi, (%eax)
; X86-NEXT: leal -12(%ebp), %esp
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
@@ -473,22 +467,19 @@ define i128 @cmov_bsr128_undef(i128 %x, i128 %y) nounwind {
;
; X64-LABEL: cmov_bsr128_undef:
; X64: # %bb.0:
-; X64-NEXT: movq %rdx, %rax
-; X64-NEXT: movq %rdi, %rdx
-; X64-NEXT: orq %rsi, %rdx
-; X64-NEXT: je .LBB9_2
-; X64-NEXT: # %bb.1: # %select.true.sink
-; X64-NEXT: bsrq %rsi, %rcx
-; X64-NEXT: xorq $63, %rcx
+; X64-NEXT: bsrq %rsi, %r8
+; X64-NEXT: xorq $63, %r8
; X64-NEXT: bsrq %rdi, %rax
; X64-NEXT: xorq $63, %rax
; X64-NEXT: orq $64, %rax
; X64-NEXT: testq %rsi, %rsi
-; X64-NEXT: cmovneq %rcx, %rax
+; X64-NEXT: cmovneq %r8, %rax
; X64-NEXT: xorq $127, %rax
-; X64-NEXT: xorl %ecx, %ecx
-; X64-NEXT: .LBB9_2: # %select.end
-; X64-NEXT: movq %rcx, %rdx
+; X64-NEXT: xorl %r8d, %r8d
+; X64-NEXT: orq %rsi, %rdi
+; X64-NEXT: cmoveq %rdx, %rax
+; X64-NEXT: cmoveq %rcx, %r8
+; X64-NEXT: movq %r8, %rdx
; X64-NEXT: retq
%1 = tail call i128 @llvm.ctlz.i128(i128 %x, i1 true)
%2 = xor i128 %1, 127
diff --git a/llvm/test/CodeGen/X86/freeze-unary.ll b/llvm/test/CodeGen/X86/freeze-unary.ll
index ad653d3f5134e..222ec1eb3b9de 100644
--- a/llvm/test/CodeGen/X86/freeze-unary.ll
+++ b/llvm/test/CodeGen/X86/freeze-unary.ll
@@ -209,18 +209,11 @@ declare <4 x i32> @llvm.bitreverse.v4i32(<4 x i32>)
define i32 @freeze_ctlz(i32 %a0) nounwind {
; X86-LABEL: freeze_ctlz:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: testl %eax, %eax
-; X86-NEXT: je .LBB12_1
-; X86-NEXT: # %bb.2: # %select.false.sink
-; X86-NEXT: bsrl %eax, %ecx
+; X86-NEXT: bsrl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl $63, %eax
; X86-NEXT: cmovnel %ecx, %eax
; X86-NEXT: xorl $31, %eax
; X86-NEXT: retl
-; X86-NEXT: .LBB12_1:
-; X86-NEXT: movl $32, %eax
-; X86-NEXT: retl
;
; X64-LABEL: freeze_ctlz:
; X64: # %bb.0:
diff --git a/llvm/test/CodeGen/X86/pseudo_cmov_lower.ll b/llvm/test/CodeGen/X86/pseudo_cmov_lower.ll
index 956b8811cf283..7d7d72e4c89f6 100644
--- a/llvm/test/CodeGen/X86/pseudo_cmov_lower.ll
+++ b/llvm/test/CodeGen/X86/pseudo_cmov_lower.ll
@@ -119,13 +119,13 @@ entry:
ret <2 x double> %sub
}
-; This test checks that only a single conditional jump gets generated in the
-; final code for lowering the CMOV pseudos that get created for this IR. This
-; combines all the supported types together into one long string of selects
-; based on the same condition.
+; This test checks that only a single ja gets generated in the final code
+; for lowering the CMOV pseudos that get created for this IR. This combines
+; all the supported types together into one long string of selects based
+; on the same condition.
; CHECK-LABEL: foo8:
-; CHECK: jbe
-; CHECK-NOT: jbe
+; CHECK: ja
+; CHECK-NOT: ja
define void @foo8(i32 %v1,
i8 %v2, i8 %v3,
i16 %v12, i16 %v13,
@@ -200,8 +200,8 @@ entry:
ret void
}
-; This test checks that only a single conditional jump gets generated in the
-; final code for lowering the CMOV pseudos that get created for this IR.
+; This test checks that only a single ja gets generated in the final code
+; for lowering the CMOV pseudos that get created for this IR.
; on the same condition.
; Contrary to my expectations, this doesn't exercise the code for
; CMOV_V8I1, CMOV_V16I1, CMOV_V32I1, or CMOV_V64I1. Instead the selects all
@@ -213,8 +213,8 @@ entry:
; tests those opcodes.
;
; CHECK-LABEL: foo9:
-; CHECK: jae
-; CHECK-NOT: jae
+; CHECK: ja
+; CHECK-NOT: ja
define void @foo9(i32 %v1,
<8 x i1> %v12, <8 x i1> %v13,
<16 x i1> %v22, <16 x i1> %v23,
diff --git a/llvm/test/CodeGen/X86/pseudo_cmov_lower2.ll b/llvm/test/CodeGen/X86/pseudo_cmov_lower2.ll
index 9ecdb23723faa..193a07c8ad3b4 100644
--- a/llvm/test/CodeGen/X86/pseudo_cmov_lower2.ll
+++ b/llvm/test/CodeGen/X86/pseudo_cmov_lower2.ll
@@ -123,16 +123,16 @@ define double @foo5(float %p1, double %p2, double %p3) nounwind {
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: xorps %xmm3, %xmm3
; CHECK-NEXT: ucomiss %xmm3, %xmm0
-; CHECK-NEXT: jae .LBB4_1
-; CHECK-NEXT: # %bb.2: # %select.false.sink
; CHECK-NEXT: movsd {{.*#+}} xmm0 = [1.25E+0,0.0E+0]
+; CHECK-NEXT: jae .LBB4_1
+; CHECK-NEXT: # %bb.2: # %select.false
; CHECK-NEXT: addsd %xmm2, %xmm0
; CHECK-NEXT: .LBB4_3: # %select.end
; CHECK-NEXT: subsd %xmm1, %xmm0
; CHECK-NEXT: addsd %xmm2, %xmm0
; CHECK-NEXT: retq
-; CHECK-NEXT: .LBB4_1: # %select.true.sink
-; CHECK-NEXT: addsd {{.*}}, %xmm1
+; CHECK-NEXT: .LBB4_1:
+; CHECK-NEXT: addsd %xmm0, %xmm1
; CHECK-NEXT: movapd %xmm1, %xmm0
; CHECK-NEXT: movapd %xmm1, %xmm2
; CHECK-NEXT: jmp .LBB4_3
@@ -157,28 +157,27 @@ define double @foo6(float %p1, double %p2, double %p3) nounwind {
; CHECK-NEXT: movaps %xmm0, %xmm3
; CHECK-NEXT: xorps %xmm0, %xmm0
; CHECK-NEXT: ucomiss %xmm0, %xmm3
-; CHECK-NEXT: jae .LBB5_1
-; CHECK-NEXT: # %bb.2: # %select.false.sink
; CHECK-NEXT: movsd {{.*#+}} xmm0 = [1.25E+0,0.0E+0]
+; CHECK-NEXT: jae .LBB5_1
+; CHECK-NEXT: # %bb.2: # %select.false
; CHECK-NEXT: addsd %xmm2, %xmm0
; CHECK-NEXT: .LBB5_3: # %select.end
; CHECK-NEXT: ucomiss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
; CHECK-NEXT: movapd %xmm0, %xmm4
; CHECK-NEXT: jae .LBB5_5
-; CHECK-NEXT: # %bb.4: # %select.false
+; CHECK-NEXT: # %bb.4: # %select.false2
; CHECK-NEXT: movapd %xmm1, %xmm4
; CHECK-NEXT: .LBB5_5: # %select.end1
; CHECK-NEXT: ucomiss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3
; CHECK-NEXT: movapd %xmm4, %xmm1
; CHECK-NEXT: jae .LBB5_7
-; CHECK-NEXT: # %bb.6: # %select.false3
+; CHECK-NEXT: # %bb.6: # %select.false4
; CHECK-NEXT: movapd %xmm2, %xmm1
-; CHECK-NEXT: .LBB5_7: # %select.end2
+; CHECK-NEXT: .LBB5_7: # %select.end3
; CHECK-NEXT: subsd %xmm4, %xmm0
; CHECK-NEXT: addsd %xmm1, %xmm0
; CHECK-NEXT: retq
-; CHECK-NEXT: .LBB5_1: # %select.true.sink
-; CHECK-NEXT: movsd {{.*#+}} xmm0 = [1.25E+0,0.0E+0]
+; CHECK-NEXT: .LBB5_1:
; CHECK-NEXT: addsd %xmm1, %xmm0
; CHECK-NEXT: jmp .LBB5_3
entry:
diff --git a/llvm/test/CodeGen/X86/shrink-wrap-chkstk-x86_64.ll b/llvm/test/CodeGen/X86/shrink-wrap-chkstk-x86_64.ll
index b0c07debbeda2..0793c33d2e8eb 100644
--- a/llvm/test/CodeGen/X86/shrink-wrap-chkstk-x86_64.ll
+++ b/llvm/test/CodeGen/X86/shrink-wrap-chkstk-x86_64.ll
@@ -9,19 +9,24 @@
define void @fn1() nounwind uwtable {
; CHECK-LABEL: fn1:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: movl $4136, %eax # imm = 0x1028
+; CHECK-NEXT: movl a(%rip), %eax
+; CHECK-NEXT: testl %eax, %eax
+; CHECK-NEXT: jne .LBB0_2
+; CHECK-NEXT: # %bb.1: # %select.true.sink
+; CHECK-NEXT: cltq
+; CHECK-NEXT: imulq $715827883, %rax, %rax # imm = 0x2AAAAAAB
+; CHECK-NEXT: movq %rax, %rcx
+; CHECK-NEXT: shrq $63, %rcx
+; CHECK-NEXT: shrq $32, %rax
+; CHECK-NEXT: addl %ecx, %eax
+; CHECK-NEXT: .LBB0_2: # %select.end
+; CHECK-NEXT: pushq %rax
+; CHECK-NEXT: movl $4128, %eax # imm = 0x1020
; CHECK-NEXT: callq ___chkstk_ms
; CHECK-NEXT: subq %rax, %rsp
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rax
; CHECK-NEXT: .cfi_def_cfa_offset 4144
-; CHECK-NEXT: movslq a(%rip), %rax
-; CHECK-NEXT: imulq $715827883, %rax, %rcx # imm = 0x2AAAAAAB
-; CHECK-NEXT: movq %rcx, %rdx
-; CHECK-NEXT: shrq $63, %rdx
-; CHECK-NEXT: shrq $32, %rcx
-; CHECK-NEXT: addl %edx, %ecx
-; CHECK-NEXT: testq %rax, %rax
-; CHECK-NEXT: cmovnel %eax, %ecx
-; CHECK-NEXT: movl %ecx, b(%rip)
+; CHECK-NEXT: movl %eax, b(%rip)
; CHECK-NEXT: leaq {{[0-9]+}}(%rsp), %rcx
; CHECK-NEXT: # kill: def $ecx killed $ecx killed $rcx
; CHECK-NEXT: callq fn2
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/optimizeSelect-DT.ll b/llvm/test/Transforms/CodeGenPrepare/X86/optimizeSelect-DT.ll
index 1f82e2a4da57b..aaf3df0934681 100644
--- a/llvm/test/Transforms/CodeGenPrepare/X86/optimizeSelect-DT.ll
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/optimizeSelect-DT.ll
@@ -4,18 +4,21 @@
target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
target triple = "x86_64-unknown-linux-gnu"
-; srem by constant 2 is not expensive (lowers to cheap bit ops), so no sinking.
define i1 @PR41004(i32 %x, i32 %y, i32 %t1) {
; CHECK-LABEL: @PR41004(
; CHECK-NEXT: entry:
+; CHECK-NEXT: [[MUL_FR:%.*]] = freeze i32 [[Y:%.*]]
+; CHECK-NEXT: [[T0:%.*]] = icmp eq i32 [[MUL_FR]], 1
+; CHECK-NEXT: br i1 [[T0]], label [[SELECT_TRUE_SINK:%.*]], label [[SELECT_END:%.*]]
+; CHECK: select.true.sink:
; CHECK-NEXT: [[REM:%.*]] = srem i32 [[X:%.*]], 2
-; CHECK-NEXT: [[T0:%.*]] = icmp eq i32 [[Y:%.*]], 1
-; CHECK-NEXT: [[MUL:%.*]] = select i1 [[T0]], i32 [[REM]], i32 0
-; CHECK-NEXT: [[NEG:%.*]] = add i32 [[T1:%.*]], -1
+; CHECK-NEXT: br label [[SELECT_END]]
+; CHECK: select.end:
+; CHECK-NEXT: [[MUL:%.*]] = phi i32 [ [[REM]], [[SELECT_TRUE_SINK]] ], [ 0, [[ENTRY:%.*]] ]
+; CHECK-NEXT: [[USUB:%.*]] = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 [[T1:%.*]], i32 1)
+; CHECK-NEXT: [[NEG:%.*]] = extractvalue { i32, i1 } [[USUB]], 0
+; CHECK-NEXT: [[TOBOOL:%.*]] = extractvalue { i32, i1 } [[USUB]], 1
; CHECK-NEXT: [[ADD:%.*]] = add i32 [[NEG]], [[MUL]]
-; CHECK-NEXT: br label [[IF:%.*]]
-; CHECK: if:
-; CHECK-NEXT: [[TOBOOL:%.*]] = icmp eq i32 [[T1]], 0
; CHECK-NEXT: ret i1 [[TOBOOL]]
;
entry:
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/select.ll b/llvm/test/Transforms/CodeGenPrepare/X86/select.ll
index eb2e16a4e269c..1dafde445a3e2 100644
--- a/llvm/test/Transforms/CodeGenPrepare/X86/select.ll
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/select.ll
@@ -200,23 +200,31 @@ define float @fadd_no_sink(float %a, float %b) {
ret float %sel
}
-; Recursive sinking: the fdiv is expensive, so sink the entire chain
-; (fdiv + fadd) into the taken branch.
+; Possible enhancement: sinkability is only calculated with the direct
+; operand of the select, so we don't try to sink this. The fdiv cost is not
+; taken into account.
-define float @fdiv_chain_sink(float %a, float %b) {
-; CHECK-LABEL: @fdiv_chain_sink(
+define float @fdiv_no_sink(float %a, float %b) {
+; CHECK-LABEL: @fdiv_no_sink(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[SEL_FROZEN:%.*]] = freeze float [[A:%.*]]
-; CHECK-NEXT: [[CMP:%.*]] = fcmp ogt float [[SEL_FROZEN]], 1.000000e+00
-; CHECK-NEXT: br i1 [[CMP]], label [[SELECT_TRUE_SINK:%.*]], label [[SELECT_END:%.*]]
-; CHECK: select.true.sink:
-; CHECK-NEXT: [[DIV:%.*]] = fdiv float [[A]], [[B:%.*]]
+; CHECK-NEXT: [[DIV:%.*]] = fdiv float [[A:%.*]], [[B:%.*]]
; CHECK-NEXT: [[ADD:%.*]] = fadd float [[DIV]], [[B]]
-; CHECK-NEXT: br label [[SELECT_END]]
-; CHECK: select.end:
-; CHECK-NEXT: [[SEL:%.*]] = phi float [ [[ADD]], [[SELECT_TRUE_SINK]] ], [ 8.000000e+00, [[ENTRY:%.*]] ]
+; CHECK-NEXT: [[CMP:%.*]] = fcmp ogt float [[A]], 1.000000e+00
+; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], float [[ADD]], float 8.000000e+00
; CHECK-NEXT: ret float [[SEL]]
;
+; DEBUG-LABEL: @fdiv_no_sink(
+; DEBUG-NEXT: entry:
+; DEBUG-NEXT: [[DIV:%.*]] = fdiv float [[A:%.*]], [[B:%.*]], !dbg [[DBG72:![0-9]+]]
+; DEBUG-NEXT: #dbg_value(float [[DIV]], [[META68:![0-9]+]], !DIExpression(), [[DBG72]])
+; DEBUG-NEXT: [[ADD:%.*]] = fadd float [[DIV]], [[B]], !dbg [[DBG73:![0-9]+]]
+; DEBUG-NEXT: #dbg_value(float [[ADD]], [[META69:![0-9]+]], !DIExpression(), [[DBG73]])
+; DEBUG-NEXT: [[CMP:%.*]] = fcmp ogt float [[A]], 1.000000e+00, !dbg [[DBG74:![0-9]+]]
+; DEBUG-NEXT: #dbg_value(i1 [[CMP]], [[META70:![0-9]+]], !DIExpression(), [[DBG74]])
+; DEBUG-NEXT: [[SEL:%.*]] = select i1 [[CMP]], float [[ADD]], float 8.000000e+00, !dbg [[DBG75:![0-9]+]]
+; DEBUG-NEXT: #dbg_value(float [[SEL]], [[META71:![0-9]+]], !DIExpression(), [[DBG75]])
+; DEBUG-NEXT: ret float [[SEL]], !dbg [[DBG76:![0-9]+]]
+;
entry:
%div = fdiv float %a, %b
%add = fadd float %div, %b
>From 9f0144b0aa4ead2de4fedac851ee1167e30057da Mon Sep 17 00:00:00 2001
From: Ayush Rai <ayrai at amd.com>
Date: Wed, 22 Jul 2026 15:18:50 +0530
Subject: [PATCH 7/8] [CodeGenPrepare] Restrict select sink latency heuristic
to X86
---
llvm/lib/CodeGen/CodeGenPrepare.cpp | 41 +++++++++++++++++------------
1 file changed, 24 insertions(+), 17 deletions(-)
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index 58a0849f60145..1f2d9b762a237 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -92,6 +92,7 @@
#include "llvm/Support/raw_ostream.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Target/TargetOptions.h"
+#include "llvm/TargetParser/Triple.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include "llvm/Transforms/Utils/BypassSlowDivision.h"
#include "llvm/Transforms/Utils/Local.h"
@@ -7623,7 +7624,11 @@ bool CodeGenPrepare::optimizeLoadExt(LoadInst *Load) {
}
static bool isLatencyExpensiveToSpeculate(const TargetTransformInfo *TTI,
- const Instruction *I) {
+ const Instruction *I,
+ bool UseLatencySelectSink) {
+ if (!UseLatencySelectSink)
+ return TTI->isExpensiveToSpeculativelyExecute(I);
+
InstructionCost InstLat =
TTI->getInstructionCost(I, TargetTransformInfo::TCK_Latency);
InstructionCost BrPenalty = TTI->getBranchMispredictPenalty();
@@ -7634,12 +7639,12 @@ static bool isLatencyExpensiveToSpeculate(const TargetTransformInfo *TTI,
return InstLat >= BrPenalty;
}
-static constexpr int SinkSelectChainMaxDepth = 8;
+enum { SinkSelectChainMaxDepth = 8 };
static bool collectSinkableChain(const TargetTransformInfo *TTI, Value *V,
BasicBlock *StartBlock, Value *SharedCond,
SmallVectorImpl<Instruction *> &Chain,
- int Depth) {
+ int Depth, bool UseLatencySelectSink) {
auto *I = dyn_cast<Instruction>(V);
if (!I || Depth <= 0)
return false;
@@ -7653,9 +7658,11 @@ static bool collectSinkableChain(const TargetTransformInfo *TTI, Value *V,
if (!isSafeToSpeculativelyExecute(I))
return false;
- bool HasExpensive = isLatencyExpensiveToSpeculate(TTI, I);
+ bool HasExpensive =
+ isLatencyExpensiveToSpeculate(TTI, I, UseLatencySelectSink);
for (Value *Op : I->operands()) {
- if (collectSinkableChain(TTI, Op, StartBlock, SharedCond, Chain, Depth - 1))
+ if (collectSinkableChain(TTI, Op, StartBlock, SharedCond, Chain, Depth - 1,
+ UseLatencySelectSink))
HasExpensive = true;
}
@@ -7669,32 +7676,31 @@ static bool collectSinkableChain(const TargetTransformInfo *TTI, Value *V,
struct SelectSinkPlan {
SmallVector<Instruction *, 4> TrueChain;
SmallVector<Instruction *, 4> FalseChain;
- bool HasExpensive = false;
+ bool HasExpensiveOp = false;
};
static SelectSinkPlan planSelectSink(const TargetTransformInfo *TTI,
- ArrayRef<SelectInst *> ASI) {
+ ArrayRef<SelectInst *> ASI,
+ bool UseLatencySelectSink) {
SelectSinkPlan Plan;
BasicBlock *StartBlock = ASI.front()->getParent();
Value *SharedCond = ASI.front()->getCondition();
- InstructionCost BrPenalty = TTI->getBranchMispredictPenalty();
- int MaxDepth = (BrPenalty.isValid() && BrPenalty.getValue() != 0)
- ? SinkSelectChainMaxDepth
- : 1;
+ int MaxDepth = UseLatencySelectSink ? SinkSelectChainMaxDepth : 1;
for (SelectInst *SI : ASI) {
if (collectSinkableChain(TTI, SI->getTrueValue(), StartBlock, SharedCond,
- Plan.TrueChain, MaxDepth))
- Plan.HasExpensive = true;
+ Plan.TrueChain, MaxDepth, UseLatencySelectSink))
+ Plan.HasExpensiveOp = true;
if (collectSinkableChain(TTI, SI->getFalseValue(), StartBlock, SharedCond,
- Plan.FalseChain, MaxDepth))
- Plan.HasExpensive = true;
+ Plan.FalseChain, MaxDepth, UseLatencySelectSink))
+ Plan.HasExpensiveOp = true;
}
return Plan;
}
+/// Returns true if a SelectInst should be turned into an explicit branch.
static bool isFormingBranchFromSelectProfitable(const TargetTransformInfo *TTI,
const TargetLowering *TLI,
ArrayRef<SelectInst *> ASI,
@@ -7733,7 +7739,7 @@ static bool isFormingBranchFromSelectProfitable(const TargetTransformInfo *TTI,
return false;
}
- if (Plan.HasExpensive) {
+ if (Plan.HasExpensiveOp) {
LLVM_DEBUG(dbgs() << "CGP: Forming branch from select due to expensive "
"operand in group of "
<< ASI.size() << " selects\n");
@@ -7876,7 +7882,8 @@ bool CodeGenPrepare::optimizeSelectInst(SelectInst *SI) {
else
SelectKind = TargetLowering::ScalarValSelect;
- SelectSinkPlan Plan = planSelectSink(TTI, ASI);
+ bool UseLatencySelectSink = TM && TM->getTargetTriple().isX86();
+ SelectSinkPlan Plan = planSelectSink(TTI, ASI, UseLatencySelectSink);
if (TLI->isSelectSupported(SelectKind) &&
(!isFormingBranchFromSelectProfitable(TTI, TLI, ASI, Plan) ||
>From 08efc721a56c3f0191dea0b315c76fd3fbc65b1b Mon Sep 17 00:00:00 2001
From: Ayush Rai <ayrai at amd.com>
Date: Wed, 22 Jul 2026 17:45:45 +0530
Subject: [PATCH 8/8] [X86] Update select sinking test expectations
---
llvm/lib/CodeGen/CodeGenPrepare.cpp | 8 +-
llvm/test/CodeGen/X86/abs.ll | 30 +-
llvm/test/CodeGen/X86/bit-manip-i256.ll | 36 +-
llvm/test/CodeGen/X86/bit-manip-i512.ll | 1062 ++++---
llvm/test/CodeGen/X86/bsf.ll | 162 +-
.../X86/div-rem-pair-recomposition-signed.ll | 399 ++-
.../div-rem-pair-recomposition-unsigned.ll | 221 +-
llvm/test/CodeGen/X86/i128-udiv.ll | 2821 +++++++++--------
llvm/test/CodeGen/X86/pr38539.ll | 167 +-
llvm/test/CodeGen/X86/pseudo_cmov_lower.ll | 1708 +++++++++-
.../CodeGen/X86/shrink-wrap-chkstk-x86_64.ll | 25 +-
.../CodeGenPrepare/X86/optimizeSelect-DT.ll | 16 +-
.../Transforms/CodeGenPrepare/X86/select.ll | 34 +-
.../X86/sink-addrmode-reg-does-not-geps.ll | 13 +-
14 files changed, 4192 insertions(+), 2510 deletions(-)
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index 1f2d9b762a237..eae27ba85c5e8 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -7658,19 +7658,19 @@ static bool collectSinkableChain(const TargetTransformInfo *TTI, Value *V,
if (!isSafeToSpeculativelyExecute(I))
return false;
- bool HasExpensive =
+ bool HasExpensiveOp =
isLatencyExpensiveToSpeculate(TTI, I, UseLatencySelectSink);
for (Value *Op : I->operands()) {
if (collectSinkableChain(TTI, Op, StartBlock, SharedCond, Chain, Depth - 1,
UseLatencySelectSink))
- HasExpensive = true;
+ HasExpensiveOp = true;
}
- if (HasExpensive) {
+ if (HasExpensiveOp) {
Chain.push_back(I);
LLVM_DEBUG(dbgs() << "CGP: Collected instruction to sink: " << *I << "\n");
}
- return HasExpensive;
+ return HasExpensiveOp;
}
struct SelectSinkPlan {
diff --git a/llvm/test/CodeGen/X86/abs.ll b/llvm/test/CodeGen/X86/abs.ll
index e252d5953e60e..9aa06f3261913 100644
--- a/llvm/test/CodeGen/X86/abs.ll
+++ b/llvm/test/CodeGen/X86/abs.ll
@@ -820,22 +820,26 @@ define i64 @test_minsigned_i64(i64 %a0, i64 %a1) nounwind {
;
; X86-LABEL: test_minsigned_i64:
; X86: # %bb.0:
+; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: leal -2147483648(%edx), %esi
+; X86-NEXT: movl %edx, %edi
+; X86-NEXT: sarl $31, %edi
+; X86-NEXT: xorl %edi, %edx
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: xorl %edi, %eax
+; X86-NEXT: subl %edi, %eax
+; X86-NEXT: sbbl %edi, %edx
+; X86-NEXT: orl %ecx, %esi
+; X86-NEXT: jne .LBB20_2
+; X86-NEXT: # %bb.1:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: leal -2147483648(%edx), %ecx
-; X86-NEXT: orl %eax, %ecx
-; X86-NEXT: jne .LBB20_1
-; X86-NEXT: # %bb.2: # %select.end
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: retl
-; X86-NEXT: .LBB20_1: # %select.false.sink
-; X86-NEXT: movl %edx, %ecx
-; X86-NEXT: sarl $31, %ecx
-; X86-NEXT: xorl %ecx, %edx
-; X86-NEXT: xorl %ecx, %eax
-; X86-NEXT: subl %ecx, %eax
-; X86-NEXT: sbbl %ecx, %edx
+; X86-NEXT: .LBB20_2:
+; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
; X86-NEXT: retl
%lim = icmp eq i64 %a0, -9223372036854775808
%abs = tail call i64 @llvm.abs.i64(i64 %a0, i1 false)
diff --git a/llvm/test/CodeGen/X86/bit-manip-i256.ll b/llvm/test/CodeGen/X86/bit-manip-i256.ll
index e4f5421c6fba9..042b7d428ba5c 100644
--- a/llvm/test/CodeGen/X86/bit-manip-i256.ll
+++ b/llvm/test/CodeGen/X86/bit-manip-i256.ll
@@ -2486,10 +2486,10 @@ define i256 @isolate_msb_i256(i256 %a0, i256 %idx) nounwind {
; AVX2-NEXT: subq $-128, %rcx
; AVX2-NEXT: orq %r8, %rax
; AVX2-NEXT: cmovneq %r11, %rcx
-; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovaps {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movl %ecx, %eax
; AVX2-NEXT: shrb $6, %al
; AVX2-NEXT: movzbl %al, %eax
@@ -2586,11 +2586,11 @@ define i256 @isolate_msb_i256(i256 %a0, i256 %idx) nounwind {
; AVX512VL-NEXT: cmovneq %r10, %rcx
; AVX512VL-NEXT: subq $-128, %rcx
; AVX512VL-NEXT: orq %r8, %rax
+; AVX512VL-NEXT: vmovaps {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
+; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: cmovneq %r11, %rcx
; AVX512VL-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: vmovaps {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
-; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movl %ecx, %eax
; AVX512VL-NEXT: shrb $6, %al
; AVX512VL-NEXT: movzbl %al, %eax
@@ -2638,11 +2638,11 @@ define i256 @isolate_msb_i256(i256 %a0, i256 %idx) nounwind {
; AVX512VBMI-NEXT: cmovneq %r10, %rcx
; AVX512VBMI-NEXT: subq $-128, %rcx
; AVX512VBMI-NEXT: orq %r8, %rax
+; AVX512VBMI-NEXT: vmovaps {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
+; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: cmovneq %r11, %rcx
; AVX512VBMI-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: vmovaps {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
-; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movl %ecx, %eax
; AVX512VBMI-NEXT: shrb $6, %al
; AVX512VBMI-NEXT: movzbl %al, %eax
@@ -2823,10 +2823,10 @@ define i256 @isolate_msb_i256_vector(<4 x i64> %v0, i256 %idx) nounwind {
; AVX2-NEXT: subq $-128, %rcx
; AVX2-NEXT: orq %r8, %rsi
; AVX2-NEXT: cmovneq %r9, %rcx
-; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT: vmovdqu %ymm1, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovaps {{.*#+}} ymm1 = [0,0,0,9223372036854775808]
; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movl %ecx, %eax
; AVX2-NEXT: shrb $6, %al
; AVX2-NEXT: movzbl %al, %edx
@@ -2901,10 +2901,10 @@ define i256 @isolate_msb_i256_vector(<4 x i64> %v0, i256 %idx) nounwind {
; AVX512VL-NEXT: vplzcntq %ymm1, %ymm1
; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,64,128,192]
; AVX512VL-NEXT: vpcompressq %ymm1, %ymm1 {%k1}
-; AVX512VL-NEXT: vxorps %xmm2, %xmm2, %xmm2
-; AVX512VL-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: vmovaps {{.*#+}} ymm2 = [0,0,0,9223372036854775808]
; AVX512VL-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: vxorps %xmm2, %xmm2, %xmm2
+; AVX512VL-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: vmovd %xmm1, %ecx
; AVX512VL-NEXT: movl %ecx, %eax
; AVX512VL-NEXT: shrb $6, %al
@@ -2940,10 +2940,10 @@ define i256 @isolate_msb_i256_vector(<4 x i64> %v0, i256 %idx) nounwind {
; AVX512VBMI-NEXT: vplzcntq %ymm1, %ymm1
; AVX512VBMI-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,64,128,192]
; AVX512VBMI-NEXT: vpcompressq %ymm1, %ymm1 {%k1}
-; AVX512VBMI-NEXT: vxorps %xmm2, %xmm2, %xmm2
-; AVX512VBMI-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: vmovaps {{.*#+}} ymm2 = [0,0,0,9223372036854775808]
; AVX512VBMI-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: vxorps %xmm2, %xmm2, %xmm2
+; AVX512VBMI-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: vmovd %xmm1, %ecx
; AVX512VBMI-NEXT: movl %ecx, %eax
; AVX512VBMI-NEXT: shrb $6, %al
@@ -3120,10 +3120,10 @@ define i256 @isolate_msb_i256_load(ptr %p0, i256 %idx) nounwind {
; AVX2-NEXT: orq %rdx, %rax
; AVX2-NEXT: cmovneq %r8, %rcx
; AVX2-NEXT: vmovdqu (%rsi), %ymm0
-; AVX2-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovaps {{.*#+}} ymm1 = [0,0,0,9223372036854775808]
; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movl %ecx, %eax
; AVX2-NEXT: shrb $6, %al
; AVX2-NEXT: movzbl %al, %edx
@@ -3199,10 +3199,10 @@ define i256 @isolate_msb_i256_load(ptr %p0, i256 %idx) nounwind {
; AVX512VL-NEXT: vplzcntq %ymm1, %ymm1
; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,64,128,192]
; AVX512VL-NEXT: vpcompressq %ymm1, %ymm1 {%k1} {z}
-; AVX512VL-NEXT: vxorps %xmm2, %xmm2, %xmm2
-; AVX512VL-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: vmovaps {{.*#+}} ymm2 = [0,0,0,9223372036854775808]
; AVX512VL-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: vxorps %xmm2, %xmm2, %xmm2
+; AVX512VL-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: vmovd %xmm1, %ecx
; AVX512VL-NEXT: movl %ecx, %eax
; AVX512VL-NEXT: shrb $6, %al
@@ -3239,10 +3239,10 @@ define i256 @isolate_msb_i256_load(ptr %p0, i256 %idx) nounwind {
; AVX512VBMI-NEXT: vplzcntq %ymm1, %ymm1
; AVX512VBMI-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,64,128,192]
; AVX512VBMI-NEXT: vpcompressq %ymm1, %ymm1 {%k1} {z}
-; AVX512VBMI-NEXT: vxorps %xmm2, %xmm2, %xmm2
-; AVX512VBMI-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: vmovaps {{.*#+}} ymm2 = [0,0,0,9223372036854775808]
; AVX512VBMI-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: vxorps %xmm2, %xmm2, %xmm2
+; AVX512VBMI-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: vmovd %xmm1, %ecx
; AVX512VBMI-NEXT: movl %ecx, %eax
; AVX512VBMI-NEXT: shrb $6, %al
diff --git a/llvm/test/CodeGen/X86/bit-manip-i512.ll b/llvm/test/CodeGen/X86/bit-manip-i512.ll
index 5b62070e1d01d..0f221c29b9d28 100644
--- a/llvm/test/CodeGen/X86/bit-manip-i512.ll
+++ b/llvm/test/CodeGen/X86/bit-manip-i512.ll
@@ -4101,122 +4101,126 @@ define i512 @isolate_msb_i512(i512 %a0, i512 %idx) nounwind {
; SSE: # %bb.0:
; SSE-NEXT: pushq %r15
; SSE-NEXT: pushq %r14
-; SSE-NEXT: pushq %r13
; SSE-NEXT: pushq %r12
; SSE-NEXT: pushq %rbx
+; SSE-NEXT: pushq %rax
+; SSE-NEXT: movq %rdi, %rax
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r10
; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r11
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rbx
-; SSE-NEXT: movq {{[0-9]+}}(%rsp), %r14
-; SSE-NEXT: movq %r8, %r10
-; SSE-NEXT: orq %r14, %r10
-; SSE-NEXT: movq %rdx, %rax
-; SSE-NEXT: orq %r11, %rax
-; SSE-NEXT: orq %r10, %rax
-; SSE-NEXT: movq %rcx, %r15
+; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rdi
+; SSE-NEXT: movq %r8, %rbx
+; SSE-NEXT: orq %r10, %rbx
+; SSE-NEXT: movq %rdx, %r14
+; SSE-NEXT: orq %rdi, %r14
+; SSE-NEXT: orq %rbx, %r14
+; SSE-NEXT: movq %rcx, %rbx
+; SSE-NEXT: orq %r11, %rbx
+; SSE-NEXT: movq %rsi, %r15
+; SSE-NEXT: orq %r9, %r15
; SSE-NEXT: orq %rbx, %r15
-; SSE-NEXT: movq %rsi, %r10
-; SSE-NEXT: orq %r9, %r10
-; SSE-NEXT: orq %r15, %r10
-; SSE-NEXT: bsrq %r14, %r15
+; SSE-NEXT: orq %r14, %r15
+; SSE-NEXT: je .LBB15_1
+; SSE-NEXT: # %bb.2: # %select.false.sink
+; SSE-NEXT: movabsq $-9223372036854775808, %rbx # imm = 0x8000000000000000
+; SSE-NEXT: bsrq %r10, %r14
+; SSE-NEXT: xorq $63, %r14
+; SSE-NEXT: bsrq %r11, %r15
; SSE-NEXT: xorq $63, %r15
-; SSE-NEXT: bsrq %rbx, %r12
+; SSE-NEXT: orq $64, %r15
+; SSE-NEXT: testq %r10, %r10
+; SSE-NEXT: cmovneq %r14, %r15
+; SSE-NEXT: bsrq %rdi, %r12
; SSE-NEXT: xorq $63, %r12
-; SSE-NEXT: orq $64, %r12
-; SSE-NEXT: testq %r14, %r14
-; SSE-NEXT: cmovneq %r15, %r12
-; SSE-NEXT: bsrq %r11, %r13
-; SSE-NEXT: xorq $63, %r13
-; SSE-NEXT: bsrq %r9, %r15
+; SSE-NEXT: bsrq %r9, %r14
+; SSE-NEXT: xorq $63, %r14
+; SSE-NEXT: orq $64, %r14
+; SSE-NEXT: testq %rdi, %rdi
+; SSE-NEXT: cmovneq %r12, %r14
+; SSE-NEXT: orq $128, %r14
+; SSE-NEXT: movq %r11, %r12
+; SSE-NEXT: orq %r10, %r12
+; SSE-NEXT: cmovneq %r15, %r14
+; SSE-NEXT: bsrq %r8, %r15
; SSE-NEXT: xorq $63, %r15
-; SSE-NEXT: orq $64, %r15
-; SSE-NEXT: testq %r11, %r11
-; SSE-NEXT: cmovneq %r13, %r15
-; SSE-NEXT: orq $128, %r15
-; SSE-NEXT: movq %rbx, %r13
-; SSE-NEXT: orq %r14, %r13
-; SSE-NEXT: cmovneq %r12, %r15
-; SSE-NEXT: bsrq %r8, %r12
+; SSE-NEXT: bsrq %rcx, %r12
; SSE-NEXT: xorq $63, %r12
-; SSE-NEXT: bsrq %rcx, %r13
-; SSE-NEXT: xorq $63, %r13
-; SSE-NEXT: orq $64, %r13
+; SSE-NEXT: orq $64, %r12
; SSE-NEXT: testq %r8, %r8
-; SSE-NEXT: cmovneq %r12, %r13
-; SSE-NEXT: bsrq %rdx, %r12
-; SSE-NEXT: xorq $63, %r12
+; SSE-NEXT: cmovneq %r15, %r12
+; SSE-NEXT: bsrq %rdx, %r15
+; SSE-NEXT: xorq $63, %r15
; SSE-NEXT: bsrq %rsi, %rsi
; SSE-NEXT: xorq $63, %rsi
; SSE-NEXT: orq $64, %rsi
; SSE-NEXT: testq %rdx, %rdx
-; SSE-NEXT: cmovneq %r12, %rsi
+; SSE-NEXT: cmovneq %r15, %rsi
; SSE-NEXT: orq $128, %rsi
; SSE-NEXT: orq %r8, %rcx
-; SSE-NEXT: cmovneq %r13, %rsi
+; SSE-NEXT: cmovneq %r12, %rsi
; SSE-NEXT: orq $256, %rsi # imm = 0x100
-; SSE-NEXT: orq %r14, %r11
-; SSE-NEXT: orq %rbx, %r9
; SSE-NEXT: orq %r11, %r9
-; SSE-NEXT: cmovneq %r15, %rsi
+; SSE-NEXT: orq %r10, %rdi
+; SSE-NEXT: orq %r9, %rdi
+; SSE-NEXT: cmovneq %r14, %rsi
; SSE-NEXT: xorps %xmm0, %xmm0
; SSE-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
; SSE-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
; SSE-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
; SSE-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; SSE-NEXT: movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000
-; SSE-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
; SSE-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
; SSE-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
; SSE-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; SSE-NEXT: movq %rbx, -{{[0-9]+}}(%rsp)
; SSE-NEXT: movl %esi, %ecx
; SSE-NEXT: andl $63, %ecx
; SSE-NEXT: shrl $3, %esi
; SSE-NEXT: andl $56, %esi
; SSE-NEXT: movq $0, -{{[0-9]+}}(%rsp)
; SSE-NEXT: movq -72(%rsp,%rsi), %rdx
-; SSE-NEXT: movq -80(%rsp,%rsi), %r11
-; SSE-NEXT: movq %r11, %r8
-; SSE-NEXT: shrdq %cl, %rdx, %r8
-; SSE-NEXT: movq -88(%rsp,%rsi), %rbx
-; SSE-NEXT: movq %rbx, %r9
-; SSE-NEXT: shrdq %cl, %r11, %r9
-; SSE-NEXT: movq -96(%rsp,%rsi), %r14
+; SSE-NEXT: movq -80(%rsp,%rsi), %r9
+; SSE-NEXT: movq %r9, %rdi
+; SSE-NEXT: shrdq %cl, %rdx, %rdi
+; SSE-NEXT: movq -88(%rsp,%rsi), %r10
+; SSE-NEXT: movq %r10, %r8
+; SSE-NEXT: shrdq %cl, %r9, %r8
+; SSE-NEXT: movq -96(%rsp,%rsi), %r11
+; SSE-NEXT: movq %r11, %r9
+; SSE-NEXT: shrdq %cl, %r10, %r9
+; SSE-NEXT: movq -104(%rsp,%rsi), %rbx
+; SSE-NEXT: movq %rbx, %r10
+; SSE-NEXT: shrdq %cl, %r11, %r10
+; SSE-NEXT: movq -112(%rsp,%rsi), %r14
; SSE-NEXT: movq %r14, %r11
; SSE-NEXT: shrdq %cl, %rbx, %r11
-; SSE-NEXT: movq -104(%rsp,%rsi), %r15
+; SSE-NEXT: movq -120(%rsp,%rsi), %r15
; SSE-NEXT: movq %r15, %rbx
; SSE-NEXT: shrdq %cl, %r14, %rbx
-; SSE-NEXT: movq -112(%rsp,%rsi), %r12
-; SSE-NEXT: movq %r12, %r14
-; SSE-NEXT: shrdq %cl, %r15, %r14
-; SSE-NEXT: movq -120(%rsp,%rsi), %r13
-; SSE-NEXT: movq %r13, %r15
-; SSE-NEXT: shrdq %cl, %r12, %r15
; SSE-NEXT: movq -128(%rsp,%rsi), %rsi
; SSE-NEXT: shrq %cl, %rdx
; SSE-NEXT: # kill: def $cl killed $cl killed $ecx
-; SSE-NEXT: shrdq %cl, %r13, %rsi
-; SSE-NEXT: xorl %ecx, %ecx
-; SSE-NEXT: orq %rax, %r10
-; SSE-NEXT: cmoveq %rcx, %r15
-; SSE-NEXT: cmoveq %rcx, %r14
-; SSE-NEXT: cmoveq %rcx, %rbx
-; SSE-NEXT: cmoveq %rcx, %r11
-; SSE-NEXT: cmoveq %rcx, %r9
-; SSE-NEXT: cmoveq %rcx, %r8
-; SSE-NEXT: cmoveq %rcx, %rsi
-; SSE-NEXT: movq %rdi, %rax
-; SSE-NEXT: cmoveq %rcx, %rdx
-; SSE-NEXT: movq %rdx, 56(%rdi)
-; SSE-NEXT: movq %r8, 48(%rdi)
-; SSE-NEXT: movq %r9, 40(%rdi)
-; SSE-NEXT: movq %r11, 32(%rdi)
-; SSE-NEXT: movq %rbx, 24(%rdi)
-; SSE-NEXT: movq %r14, 16(%rdi)
-; SSE-NEXT: movq %r15, 8(%rdi)
-; SSE-NEXT: movq %rsi, (%rdi)
+; SSE-NEXT: shrdq %cl, %r15, %rsi
+; SSE-NEXT: jmp .LBB15_3
+; SSE-NEXT: .LBB15_1:
+; SSE-NEXT: xorl %esi, %esi
+; SSE-NEXT: xorl %ebx, %ebx
+; SSE-NEXT: xorl %r11d, %r11d
+; SSE-NEXT: xorl %r10d, %r10d
+; SSE-NEXT: xorl %r9d, %r9d
+; SSE-NEXT: xorl %r8d, %r8d
+; SSE-NEXT: xorl %edi, %edi
+; SSE-NEXT: xorl %edx, %edx
+; SSE-NEXT: .LBB15_3: # %select.end
+; SSE-NEXT: movq %rsi, (%rax)
+; SSE-NEXT: movq %rbx, 8(%rax)
+; SSE-NEXT: movq %r11, 16(%rax)
+; SSE-NEXT: movq %r10, 24(%rax)
+; SSE-NEXT: movq %r9, 32(%rax)
+; SSE-NEXT: movq %r8, 40(%rax)
+; SSE-NEXT: movq %rdi, 48(%rax)
+; SSE-NEXT: movq %rdx, 56(%rax)
+; SSE-NEXT: addq $8, %rsp
; SSE-NEXT: popq %rbx
; SSE-NEXT: popq %r12
-; SSE-NEXT: popq %r13
; SSE-NEXT: popq %r14
; SSE-NEXT: popq %r15
; SSE-NEXT: retq
@@ -4283,6 +4287,8 @@ define i512 @isolate_msb_i512(i512 %a0, i512 %idx) nounwind {
; AVX2-NEXT: orq %r14, %r9
; AVX2-NEXT: orq %rbx, %r9
; AVX2-NEXT: cmovneq %r12, %rax
+; AVX2-NEXT: vmovaps {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
+; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
@@ -4491,132 +4497,128 @@ define i512 @isolate_msb_i512_vector(<8 x i64> %v0, i512 %idx) nounwind {
; SSE2: # %bb.0:
; SSE2-NEXT: pushq %r15
; SSE2-NEXT: pushq %r14
-; SSE2-NEXT: pushq %r12
; SSE2-NEXT: pushq %rbx
-; SSE2-NEXT: pushq %rax
-; SSE2-NEXT: movq %xmm0, %rdx
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
-; SSE2-NEXT: movq %xmm4, %rcx
-; SSE2-NEXT: movq %xmm1, %r8
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
-; SSE2-NEXT: movq %xmm4, %rsi
-; SSE2-NEXT: movq %xmm2, %r9
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]
-; SSE2-NEXT: movq %xmm4, %r11
-; SSE2-NEXT: movq %xmm3, %r10
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
-; SSE2-NEXT: movq %xmm4, %rbx
+; SSE2-NEXT: movq %rdi, %rax
; SSE2-NEXT: movdqa %xmm1, %xmm4
; SSE2-NEXT: por %xmm3, %xmm4
-; SSE2-NEXT: por %xmm2, %xmm0
-; SSE2-NEXT: por %xmm4, %xmm0
+; SSE2-NEXT: movdqa %xmm0, %xmm5
+; SSE2-NEXT: por %xmm2, %xmm5
+; SSE2-NEXT: por %xmm4, %xmm5
; SSE2-NEXT: pxor %xmm4, %xmm4
-; SSE2-NEXT: pcmpeqd %xmm4, %xmm0
-; SSE2-NEXT: movmskps %xmm0, %eax
-; SSE2-NEXT: xorl $15, %eax
+; SSE2-NEXT: pcmpeqd %xmm4, %xmm5
+; SSE2-NEXT: movmskps %xmm5, %ecx
+; SSE2-NEXT: xorl $15, %ecx
+; SSE2-NEXT: je .LBB16_1
+; SSE2-NEXT: # %bb.2: # %select.false.sink
+; SSE2-NEXT: movq %xmm3, %rsi
+; SSE2-NEXT: movq %xmm2, %rcx
+; SSE2-NEXT: movq %xmm1, %r9
+; SSE2-NEXT: movq %xmm0, %rdx
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
+; SSE2-NEXT: movq %xmm3, %r8
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
+; SSE2-NEXT: movq %xmm2, %rdi
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; SSE2-NEXT: movq %xmm1, %r10
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; SSE2-NEXT: movq %xmm0, %rbx
+; SSE2-NEXT: bsrq %r8, %r11
+; SSE2-NEXT: xorq $63, %r11
+; SSE2-NEXT: bsrq %rsi, %r14
+; SSE2-NEXT: xorq $63, %r14
+; SSE2-NEXT: orq $64, %r14
+; SSE2-NEXT: testq %r8, %r8
+; SSE2-NEXT: cmovneq %r11, %r14
+; SSE2-NEXT: bsrq %rdi, %r15
+; SSE2-NEXT: xorq $63, %r15
+; SSE2-NEXT: bsrq %rcx, %r11
+; SSE2-NEXT: xorq $63, %r11
+; SSE2-NEXT: orq $64, %r11
+; SSE2-NEXT: testq %rdi, %rdi
+; SSE2-NEXT: cmovneq %r15, %r11
+; SSE2-NEXT: orq $128, %r11
+; SSE2-NEXT: movq %rsi, %r15
+; SSE2-NEXT: orq %r8, %r15
+; SSE2-NEXT: cmovneq %r14, %r11
+; SSE2-NEXT: bsrq %r10, %r14
+; SSE2-NEXT: xorq $63, %r14
+; SSE2-NEXT: bsrq %r9, %r15
+; SSE2-NEXT: xorq $63, %r15
+; SSE2-NEXT: orq $64, %r15
+; SSE2-NEXT: testq %r10, %r10
+; SSE2-NEXT: cmovneq %r14, %r15
; SSE2-NEXT: bsrq %rbx, %r14
; SSE2-NEXT: xorq $63, %r14
-; SSE2-NEXT: bsrq %r10, %r10
-; SSE2-NEXT: xorq $63, %r10
-; SSE2-NEXT: orq $64, %r10
-; SSE2-NEXT: testq %rbx, %rbx
-; SSE2-NEXT: cmovneq %r14, %r10
-; SSE2-NEXT: bsrq %r11, %rbx
-; SSE2-NEXT: xorq $63, %rbx
-; SSE2-NEXT: bsrq %r9, %r9
-; SSE2-NEXT: xorq $63, %r9
-; SSE2-NEXT: orq $64, %r9
-; SSE2-NEXT: testq %r11, %r11
-; SSE2-NEXT: cmovneq %rbx, %r9
-; SSE2-NEXT: orq $128, %r9
-; SSE2-NEXT: por %xmm3, %xmm2
-; SSE2-NEXT: pcmpeqd %xmm4, %xmm3
-; SSE2-NEXT: movmskps %xmm3, %r11d
-; SSE2-NEXT: xorl $15, %r11d
-; SSE2-NEXT: cmovneq %r10, %r9
-; SSE2-NEXT: bsrq %rsi, %r10
-; SSE2-NEXT: xorq $63, %r10
-; SSE2-NEXT: bsrq %r8, %r8
-; SSE2-NEXT: xorq $63, %r8
-; SSE2-NEXT: orq $64, %r8
-; SSE2-NEXT: testq %rsi, %rsi
-; SSE2-NEXT: cmovneq %r10, %r8
-; SSE2-NEXT: bsrq %rcx, %rsi
-; SSE2-NEXT: xorq $63, %rsi
; SSE2-NEXT: bsrq %rdx, %rdx
; SSE2-NEXT: xorq $63, %rdx
; SSE2-NEXT: orq $64, %rdx
-; SSE2-NEXT: testq %rcx, %rcx
-; SSE2-NEXT: cmovneq %rsi, %rdx
+; SSE2-NEXT: testq %rbx, %rbx
+; SSE2-NEXT: cmovneq %r14, %rdx
; SSE2-NEXT: orq $128, %rdx
-; SSE2-NEXT: pcmpeqd %xmm4, %xmm1
-; SSE2-NEXT: movmskps %xmm1, %ecx
-; SSE2-NEXT: xorl $15, %ecx
-; SSE2-NEXT: cmovneq %r8, %rdx
+; SSE2-NEXT: orq %r10, %r9
+; SSE2-NEXT: cmovneq %r15, %rdx
+; SSE2-NEXT: movabsq $-9223372036854775808, %r9 # imm = 0x8000000000000000
; SSE2-NEXT: orq $256, %rdx # imm = 0x100
-; SSE2-NEXT: pcmpeqd %xmm4, %xmm2
-; SSE2-NEXT: movmskps %xmm2, %ecx
-; SSE2-NEXT: xorl $15, %ecx
-; SSE2-NEXT: cmovneq %r9, %rdx
+; SSE2-NEXT: orq %rsi, %rcx
+; SSE2-NEXT: orq %r8, %rdi
+; SSE2-NEXT: orq %rcx, %rdi
+; SSE2-NEXT: cmovneq %r11, %rdx
; SSE2-NEXT: movdqa %xmm4, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: movdqa %xmm4, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: movdqa %xmm4, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: movdqa %xmm4, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000
-; SSE2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: movdqa %xmm4, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: movdqa %xmm4, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: movdqa %xmm4, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: movl %edx, %ecx
; SSE2-NEXT: andl $63, %ecx
; SSE2-NEXT: shrl $3, %edx
; SSE2-NEXT: andl $56, %edx
; SSE2-NEXT: movq $0, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: movq -72(%rsp,%rdx), %rsi
-; SSE2-NEXT: movq -80(%rsp,%rdx), %r10
+; SSE2-NEXT: movq -80(%rsp,%rdx), %r9
+; SSE2-NEXT: movq %r9, %rdi
+; SSE2-NEXT: shrdq %cl, %rsi, %rdi
+; SSE2-NEXT: movq -88(%rsp,%rdx), %r10
; SSE2-NEXT: movq %r10, %r8
-; SSE2-NEXT: shrdq %cl, %rsi, %r8
-; SSE2-NEXT: movq -88(%rsp,%rdx), %r11
+; SSE2-NEXT: shrdq %cl, %r9, %r8
+; SSE2-NEXT: movq -96(%rsp,%rdx), %r11
; SSE2-NEXT: movq %r11, %r9
; SSE2-NEXT: shrdq %cl, %r10, %r9
-; SSE2-NEXT: movq -96(%rsp,%rdx), %rbx
+; SSE2-NEXT: movq -104(%rsp,%rdx), %rbx
; SSE2-NEXT: movq %rbx, %r10
; SSE2-NEXT: shrdq %cl, %r11, %r10
-; SSE2-NEXT: movq -104(%rsp,%rdx), %r14
+; SSE2-NEXT: movq -112(%rsp,%rdx), %r14
; SSE2-NEXT: movq %r14, %r11
; SSE2-NEXT: shrdq %cl, %rbx, %r11
-; SSE2-NEXT: movq -112(%rsp,%rdx), %r15
+; SSE2-NEXT: movq -120(%rsp,%rdx), %r15
; SSE2-NEXT: movq %r15, %rbx
; SSE2-NEXT: shrdq %cl, %r14, %rbx
-; SSE2-NEXT: movq -120(%rsp,%rdx), %r12
-; SSE2-NEXT: movq %r12, %r14
-; SSE2-NEXT: shrdq %cl, %r15, %r14
; SSE2-NEXT: movq -128(%rsp,%rdx), %rdx
; SSE2-NEXT: shrq %cl, %rsi
; SSE2-NEXT: # kill: def $cl killed $cl killed $ecx
-; SSE2-NEXT: shrdq %cl, %r12, %rdx
-; SSE2-NEXT: xorl %ecx, %ecx
-; SSE2-NEXT: testl %eax, %eax
-; SSE2-NEXT: cmoveq %rcx, %r14
-; SSE2-NEXT: cmoveq %rcx, %rbx
-; SSE2-NEXT: cmoveq %rcx, %r11
-; SSE2-NEXT: cmoveq %rcx, %r10
-; SSE2-NEXT: cmoveq %rcx, %r9
-; SSE2-NEXT: cmoveq %rcx, %r8
-; SSE2-NEXT: cmoveq %rcx, %rdx
-; SSE2-NEXT: movq %rdi, %rax
-; SSE2-NEXT: cmoveq %rcx, %rsi
-; SSE2-NEXT: movq %rsi, 56(%rdi)
-; SSE2-NEXT: movq %r8, 48(%rdi)
-; SSE2-NEXT: movq %r9, 40(%rdi)
-; SSE2-NEXT: movq %r10, 32(%rdi)
-; SSE2-NEXT: movq %r11, 24(%rdi)
-; SSE2-NEXT: movq %rbx, 16(%rdi)
-; SSE2-NEXT: movq %r14, 8(%rdi)
-; SSE2-NEXT: movq %rdx, (%rdi)
-; SSE2-NEXT: addq $8, %rsp
+; SSE2-NEXT: shrdq %cl, %r15, %rdx
+; SSE2-NEXT: jmp .LBB16_3
+; SSE2-NEXT: .LBB16_1:
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: xorl %ebx, %ebx
+; SSE2-NEXT: xorl %r11d, %r11d
+; SSE2-NEXT: xorl %r10d, %r10d
+; SSE2-NEXT: xorl %r9d, %r9d
+; SSE2-NEXT: xorl %r8d, %r8d
+; SSE2-NEXT: xorl %edi, %edi
+; SSE2-NEXT: xorl %esi, %esi
+; SSE2-NEXT: .LBB16_3: # %select.end
+; SSE2-NEXT: movq %rdx, (%rax)
+; SSE2-NEXT: movq %rbx, 8(%rax)
+; SSE2-NEXT: movq %r11, 16(%rax)
+; SSE2-NEXT: movq %r10, 24(%rax)
+; SSE2-NEXT: movq %r9, 32(%rax)
+; SSE2-NEXT: movq %r8, 40(%rax)
+; SSE2-NEXT: movq %rdi, 48(%rax)
+; SSE2-NEXT: movq %rsi, 56(%rax)
; SSE2-NEXT: popq %rbx
-; SSE2-NEXT: popq %r12
; SSE2-NEXT: popq %r14
; SSE2-NEXT: popq %r15
; SSE2-NEXT: retq
@@ -4625,114 +4627,126 @@ define i512 @isolate_msb_i512_vector(<8 x i64> %v0, i512 %idx) nounwind {
; SSE42: # %bb.0:
; SSE42-NEXT: pushq %r15
; SSE42-NEXT: pushq %r14
+; SSE42-NEXT: pushq %r12
; SSE42-NEXT: pushq %rbx
-; SSE42-NEXT: movq %xmm0, %rax
-; SSE42-NEXT: pextrq $1, %xmm0, %rcx
-; SSE42-NEXT: pextrq $1, %xmm1, %rdx
-; SSE42-NEXT: movq %xmm1, %r8
-; SSE42-NEXT: movq %xmm2, %rsi
-; SSE42-NEXT: pextrq $1, %xmm2, %r9
-; SSE42-NEXT: movq %xmm3, %r10
-; SSE42-NEXT: pextrq $1, %xmm3, %r11
-; SSE42-NEXT: bsrq %r11, %rbx
-; SSE42-NEXT: xorq $63, %rbx
-; SSE42-NEXT: bsrq %r10, %r10
-; SSE42-NEXT: xorq $63, %r10
-; SSE42-NEXT: orq $64, %r10
-; SSE42-NEXT: testq %r11, %r11
-; SSE42-NEXT: cmovneq %rbx, %r10
-; SSE42-NEXT: bsrq %r9, %r11
-; SSE42-NEXT: xorq $63, %r11
-; SSE42-NEXT: bsrq %rsi, %rsi
-; SSE42-NEXT: xorq $63, %rsi
-; SSE42-NEXT: orq $64, %rsi
-; SSE42-NEXT: testq %r9, %r9
-; SSE42-NEXT: cmovneq %r11, %rsi
-; SSE42-NEXT: orq $128, %rsi
-; SSE42-NEXT: ptest %xmm3, %xmm3
-; SSE42-NEXT: cmovneq %r10, %rsi
-; SSE42-NEXT: bsrq %rdx, %r9
-; SSE42-NEXT: xorq $63, %r9
-; SSE42-NEXT: bsrq %r8, %r8
-; SSE42-NEXT: xorq $63, %r8
-; SSE42-NEXT: orq $64, %r8
-; SSE42-NEXT: testq %rdx, %rdx
-; SSE42-NEXT: cmovneq %r9, %r8
-; SSE42-NEXT: bsrq %rcx, %rdx
-; SSE42-NEXT: xorq $63, %rdx
-; SSE42-NEXT: bsrq %rax, %rax
-; SSE42-NEXT: xorq $63, %rax
-; SSE42-NEXT: orq $64, %rax
-; SSE42-NEXT: testq %rcx, %rcx
-; SSE42-NEXT: cmovneq %rdx, %rax
-; SSE42-NEXT: por %xmm2, %xmm0
-; SSE42-NEXT: orq $128, %rax
-; SSE42-NEXT: ptest %xmm1, %xmm1
-; SSE42-NEXT: cmovneq %r8, %rax
-; SSE42-NEXT: orq $256, %rax # imm = 0x100
-; SSE42-NEXT: por %xmm3, %xmm2
-; SSE42-NEXT: ptest %xmm2, %xmm2
-; SSE42-NEXT: cmovneq %rsi, %rax
-; SSE42-NEXT: pxor %xmm2, %xmm2
-; SSE42-NEXT: movdqa %xmm2, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movdqa %xmm2, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movdqa %xmm2, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movdqa %xmm2, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: pushq %rax
+; SSE42-NEXT: movq %rdi, %rax
+; SSE42-NEXT: movdqa %xmm1, %xmm4
+; SSE42-NEXT: por %xmm3, %xmm4
+; SSE42-NEXT: movdqa %xmm0, %xmm5
+; SSE42-NEXT: por %xmm2, %xmm5
+; SSE42-NEXT: por %xmm4, %xmm5
+; SSE42-NEXT: ptest %xmm5, %xmm5
+; SSE42-NEXT: je .LBB16_1
+; SSE42-NEXT: # %bb.2: # %select.false.sink
+; SSE42-NEXT: pextrq $1, %xmm3, %rdi
+; SSE42-NEXT: movq %xmm3, %r9
+; SSE42-NEXT: pextrq $1, %xmm2, %rsi
+; SSE42-NEXT: movq %xmm2, %r8
+; SSE42-NEXT: pextrq $1, %xmm1, %r10
+; SSE42-NEXT: pextrq $1, %xmm0, %rbx
+; SSE42-NEXT: movq %xmm1, %r11
+; SSE42-NEXT: movq %xmm0, %rdx
; SSE42-NEXT: movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000
+; SSE42-NEXT: bsrq %rdi, %r14
+; SSE42-NEXT: xorq $63, %r14
+; SSE42-NEXT: bsrq %r9, %r15
+; SSE42-NEXT: xorq $63, %r15
+; SSE42-NEXT: orq $64, %r15
+; SSE42-NEXT: testq %rdi, %rdi
+; SSE42-NEXT: cmovneq %r14, %r15
+; SSE42-NEXT: bsrq %rsi, %r12
+; SSE42-NEXT: xorq $63, %r12
+; SSE42-NEXT: bsrq %r8, %r14
+; SSE42-NEXT: xorq $63, %r14
+; SSE42-NEXT: orq $64, %r14
+; SSE42-NEXT: testq %rsi, %rsi
+; SSE42-NEXT: cmovneq %r12, %r14
+; SSE42-NEXT: orq $128, %r14
+; SSE42-NEXT: movq %r9, %r12
+; SSE42-NEXT: orq %rdi, %r12
+; SSE42-NEXT: cmovneq %r15, %r14
+; SSE42-NEXT: bsrq %r10, %r15
+; SSE42-NEXT: xorq $63, %r15
+; SSE42-NEXT: bsrq %r11, %r12
+; SSE42-NEXT: xorq $63, %r12
+; SSE42-NEXT: orq $64, %r12
+; SSE42-NEXT: testq %r10, %r10
+; SSE42-NEXT: cmovneq %r15, %r12
+; SSE42-NEXT: bsrq %rbx, %r15
+; SSE42-NEXT: xorq $63, %r15
+; SSE42-NEXT: bsrq %rdx, %rdx
+; SSE42-NEXT: xorq $63, %rdx
+; SSE42-NEXT: orq $64, %rdx
+; SSE42-NEXT: testq %rbx, %rbx
+; SSE42-NEXT: cmovneq %r15, %rdx
+; SSE42-NEXT: orq $128, %rdx
+; SSE42-NEXT: orq %r10, %r11
+; SSE42-NEXT: cmovneq %r12, %rdx
+; SSE42-NEXT: orq $256, %rdx # imm = 0x100
+; SSE42-NEXT: orq %r9, %r8
+; SSE42-NEXT: orq %rdi, %rsi
+; SSE42-NEXT: orq %r8, %rsi
+; SSE42-NEXT: cmovneq %r14, %rdx
+; SSE42-NEXT: pxor %xmm0, %xmm0
+; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
; SSE42-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movdqa %xmm2, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movdqa %xmm2, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movdqa %xmm2, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movl %eax, %ecx
+; SSE42-NEXT: movl %edx, %ecx
; SSE42-NEXT: andl $63, %ecx
-; SSE42-NEXT: shrl $3, %eax
-; SSE42-NEXT: andl $56, %eax
+; SSE42-NEXT: shrl $3, %edx
+; SSE42-NEXT: andl $56, %edx
; SSE42-NEXT: movq $0, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movq -72(%rsp,%rax), %rdx
-; SSE42-NEXT: movq -80(%rsp,%rax), %r9
-; SSE42-NEXT: movq %r9, %rsi
-; SSE42-NEXT: shrdq %cl, %rdx, %rsi
-; SSE42-NEXT: movq -88(%rsp,%rax), %r10
+; SSE42-NEXT: movq -72(%rsp,%rdx), %rsi
+; SSE42-NEXT: movq -80(%rsp,%rdx), %r9
+; SSE42-NEXT: movq %r9, %rdi
+; SSE42-NEXT: shrdq %cl, %rsi, %rdi
+; SSE42-NEXT: movq -88(%rsp,%rdx), %r10
; SSE42-NEXT: movq %r10, %r8
; SSE42-NEXT: shrdq %cl, %r9, %r8
-; SSE42-NEXT: movq -96(%rsp,%rax), %r11
+; SSE42-NEXT: movq -96(%rsp,%rdx), %r11
; SSE42-NEXT: movq %r11, %r9
; SSE42-NEXT: shrdq %cl, %r10, %r9
-; SSE42-NEXT: movq -104(%rsp,%rax), %rbx
+; SSE42-NEXT: movq -104(%rsp,%rdx), %rbx
; SSE42-NEXT: movq %rbx, %r10
; SSE42-NEXT: shrdq %cl, %r11, %r10
-; SSE42-NEXT: movq -112(%rsp,%rax), %r14
+; SSE42-NEXT: movq -112(%rsp,%rdx), %r14
; SSE42-NEXT: movq %r14, %r11
; SSE42-NEXT: shrdq %cl, %rbx, %r11
-; SSE42-NEXT: movq -120(%rsp,%rax), %r15
+; SSE42-NEXT: movq -120(%rsp,%rdx), %r15
; SSE42-NEXT: movq %r15, %rbx
; SSE42-NEXT: shrdq %cl, %r14, %rbx
-; SSE42-NEXT: movq -128(%rsp,%rax), %r14
-; SSE42-NEXT: shrq %cl, %rdx
+; SSE42-NEXT: movq -128(%rsp,%rdx), %rdx
+; SSE42-NEXT: shrq %cl, %rsi
; SSE42-NEXT: # kill: def $cl killed $cl killed $ecx
-; SSE42-NEXT: shrdq %cl, %r15, %r14
-; SSE42-NEXT: por %xmm3, %xmm1
-; SSE42-NEXT: por %xmm1, %xmm0
-; SSE42-NEXT: xorl %ecx, %ecx
-; SSE42-NEXT: ptest %xmm0, %xmm0
-; SSE42-NEXT: cmoveq %rcx, %rbx
-; SSE42-NEXT: cmoveq %rcx, %r11
-; SSE42-NEXT: cmoveq %rcx, %r10
-; SSE42-NEXT: cmoveq %rcx, %r9
-; SSE42-NEXT: cmoveq %rcx, %r8
-; SSE42-NEXT: cmoveq %rcx, %rsi
-; SSE42-NEXT: cmoveq %rcx, %r14
-; SSE42-NEXT: movq %rdi, %rax
-; SSE42-NEXT: cmoveq %rcx, %rdx
-; SSE42-NEXT: movq %rdx, 56(%rdi)
-; SSE42-NEXT: movq %rsi, 48(%rdi)
-; SSE42-NEXT: movq %r8, 40(%rdi)
-; SSE42-NEXT: movq %r9, 32(%rdi)
-; SSE42-NEXT: movq %r10, 24(%rdi)
-; SSE42-NEXT: movq %r11, 16(%rdi)
-; SSE42-NEXT: movq %rbx, 8(%rdi)
-; SSE42-NEXT: movq %r14, (%rdi)
+; SSE42-NEXT: shrdq %cl, %r15, %rdx
+; SSE42-NEXT: jmp .LBB16_3
+; SSE42-NEXT: .LBB16_1:
+; SSE42-NEXT: xorl %edx, %edx
+; SSE42-NEXT: xorl %ebx, %ebx
+; SSE42-NEXT: xorl %r11d, %r11d
+; SSE42-NEXT: xorl %r10d, %r10d
+; SSE42-NEXT: xorl %r9d, %r9d
+; SSE42-NEXT: xorl %r8d, %r8d
+; SSE42-NEXT: xorl %edi, %edi
+; SSE42-NEXT: xorl %esi, %esi
+; SSE42-NEXT: .LBB16_3: # %select.end
+; SSE42-NEXT: movq %rdx, (%rax)
+; SSE42-NEXT: movq %rbx, 8(%rax)
+; SSE42-NEXT: movq %r11, 16(%rax)
+; SSE42-NEXT: movq %r10, 24(%rax)
+; SSE42-NEXT: movq %r9, 32(%rax)
+; SSE42-NEXT: movq %r8, 40(%rax)
+; SSE42-NEXT: movq %rdi, 48(%rax)
+; SSE42-NEXT: movq %rsi, 56(%rax)
+; SSE42-NEXT: addq $8, %rsp
; SSE42-NEXT: popq %rbx
+; SSE42-NEXT: popq %r12
; SSE42-NEXT: popq %r14
; SSE42-NEXT: popq %r15
; SSE42-NEXT: retq
@@ -4742,102 +4756,102 @@ define i512 @isolate_msb_i512_vector(<8 x i64> %v0, i512 %idx) nounwind {
; AVX2-NEXT: pushq %r15
; AVX2-NEXT: pushq %r14
; AVX2-NEXT: pushq %rbx
+; AVX2-NEXT: vpextrq $1, %xmm0, %rsi
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2
; AVX2-NEXT: vmovq %xmm2, %rax
; AVX2-NEXT: vpextrq $1, %xmm2, %rcx
+; AVX2-NEXT: vmovq %xmm1, %rdx
+; AVX2-NEXT: vpextrq $1, %xmm1, %r9
; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
-; AVX2-NEXT: vmovq %xmm2, %rdx
-; AVX2-NEXT: vpextrq $1, %xmm2, %rsi
-; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT: vmovdqu %ymm2, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vmovaps {{.*#+}} ymm3 = [0,0,0,9223372036854775808]
-; AVX2-NEXT: vmovups %ymm3, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vpextrq $1, %xmm0, %r9
-; AVX2-NEXT: vmovq %xmm0, %r8
-; AVX2-NEXT: vmovq %xmm1, %r10
-; AVX2-NEXT: vpextrq $1, %xmm1, %r11
+; AVX2-NEXT: vmovq %xmm2, %r10
+; AVX2-NEXT: vpextrq $1, %xmm2, %r11
+; AVX2-NEXT: lzcntq %r11, %r8
; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: lzcntq %rsi, %rbx
-; AVX2-NEXT: xorl %r14d, %r14d
-; AVX2-NEXT: lzcntq %rdx, %r14
-; AVX2-NEXT: addq $64, %r14
-; AVX2-NEXT: testq %rsi, %rsi
-; AVX2-NEXT: cmovneq %rbx, %r14
-; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: lzcntq %r11, %rbx
-; AVX2-NEXT: lzcntq %r10, %r10
-; AVX2-NEXT: addq $64, %r10
+; AVX2-NEXT: lzcntq %r10, %rbx
+; AVX2-NEXT: addq $64, %rbx
; AVX2-NEXT: testq %r11, %r11
-; AVX2-NEXT: cmovneq %rbx, %r10
-; AVX2-NEXT: subq $-128, %r10
-; AVX2-NEXT: orq %rsi, %rdx
-; AVX2-NEXT: cmovneq %r14, %r10
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: lzcntq %rcx, %rdx
-; AVX2-NEXT: xorl %esi, %esi
-; AVX2-NEXT: lzcntq %rax, %rsi
-; AVX2-NEXT: addq $64, %rsi
-; AVX2-NEXT: testq %rcx, %rcx
-; AVX2-NEXT: cmovneq %rdx, %rsi
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: lzcntq %r9, %rdx
-; AVX2-NEXT: lzcntq %r8, %r8
+; AVX2-NEXT: cmovneq %r8, %rbx
+; AVX2-NEXT: xorl %r14d, %r14d
+; AVX2-NEXT: lzcntq %r9, %r14
+; AVX2-NEXT: xorl %r8d, %r8d
+; AVX2-NEXT: lzcntq %rdx, %r8
; AVX2-NEXT: addq $64, %r8
; AVX2-NEXT: testq %r9, %r9
-; AVX2-NEXT: cmovneq %rdx, %r8
+; AVX2-NEXT: cmovneq %r14, %r8
; AVX2-NEXT: subq $-128, %r8
+; AVX2-NEXT: orq %r11, %r10
+; AVX2-NEXT: cmovneq %rbx, %r8
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: lzcntq %rcx, %rdx
+; AVX2-NEXT: xorl %r9d, %r9d
+; AVX2-NEXT: lzcntq %rax, %r9
+; AVX2-NEXT: addq $64, %r9
+; AVX2-NEXT: testq %rcx, %rcx
+; AVX2-NEXT: cmovneq %rdx, %r9
+; AVX2-NEXT: vmovq %xmm0, %rdx
+; AVX2-NEXT: xorl %r10d, %r10d
+; AVX2-NEXT: lzcntq %rsi, %r10
+; AVX2-NEXT: lzcntq %rdx, %rdx
+; AVX2-NEXT: addq $64, %rdx
+; AVX2-NEXT: testq %rsi, %rsi
+; AVX2-NEXT: cmovneq %r10, %rdx
+; AVX2-NEXT: subq $-128, %rdx
; AVX2-NEXT: orq %rcx, %rax
-; AVX2-NEXT: cmovneq %rsi, %r8
-; AVX2-NEXT: addq $256, %r8 # imm = 0x100
+; AVX2-NEXT: cmovneq %r9, %rdx
+; AVX2-NEXT: addq $256, %rdx # imm = 0x100
; AVX2-NEXT: vptest %ymm1, %ymm1
-; AVX2-NEXT: cmovneq %r10, %r8
-; AVX2-NEXT: vmovdqu %ymm2, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vmovdqu %ymm2, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movl %r8d, %ecx
+; AVX2-NEXT: cmovneq %r8, %rdx
+; AVX2-NEXT: vmovaps {{.*#+}} ymm2 = [0,0,0,9223372036854775808]
+; AVX2-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vxorps %xmm2, %xmm2, %xmm2
+; AVX2-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movl %edx, %ecx
; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: shrl $3, %r8d
-; AVX2-NEXT: andl $56, %r8d
-; AVX2-NEXT: movq -72(%rsp,%r8), %r11
-; AVX2-NEXT: movq -80(%rsp,%r8), %rax
-; AVX2-NEXT: movq %rax, %rdx
-; AVX2-NEXT: shrdq %cl, %r11, %rdx
-; AVX2-NEXT: movq -88(%rsp,%r8), %r10
-; AVX2-NEXT: movq %r10, %rsi
-; AVX2-NEXT: shrdq %cl, %rax, %rsi
-; AVX2-NEXT: movq -96(%rsp,%r8), %rax
+; AVX2-NEXT: shrl $3, %edx
+; AVX2-NEXT: andl $56, %edx
+; AVX2-NEXT: movq -72(%rsp,%rdx), %r11
+; AVX2-NEXT: movq -80(%rsp,%rdx), %rax
+; AVX2-NEXT: movq %rax, %rsi
+; AVX2-NEXT: shrdq %cl, %r11, %rsi
+; AVX2-NEXT: movq -88(%rsp,%rdx), %r10
+; AVX2-NEXT: movq %r10, %r8
+; AVX2-NEXT: shrdq %cl, %rax, %r8
+; AVX2-NEXT: movq -96(%rsp,%rdx), %rax
; AVX2-NEXT: movq %rax, %r9
; AVX2-NEXT: shrdq %cl, %r10, %r9
-; AVX2-NEXT: movq -104(%rsp,%r8), %r14
+; AVX2-NEXT: movq -104(%rsp,%rdx), %r14
; AVX2-NEXT: movq %r14, %r10
; AVX2-NEXT: shrdq %cl, %rax, %r10
-; AVX2-NEXT: movq -112(%rsp,%r8), %r15
+; AVX2-NEXT: movq -112(%rsp,%rdx), %r15
; AVX2-NEXT: movq %r15, %rbx
; AVX2-NEXT: shrdq %cl, %r14, %rbx
; AVX2-NEXT: movq %rdi, %rax
; AVX2-NEXT: vpor %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: movq -128(%rsp,%r8), %rdi
-; AVX2-NEXT: movq -120(%rsp,%r8), %r14
-; AVX2-NEXT: movq %r14, %r8
-; AVX2-NEXT: shrdq %cl, %r15, %r8
+; AVX2-NEXT: movq -128(%rsp,%rdx), %rdi
+; AVX2-NEXT: movq -120(%rsp,%rdx), %r14
+; AVX2-NEXT: movq %r14, %rdx
+; AVX2-NEXT: shrdq %cl, %r15, %rdx
; AVX2-NEXT: shrdq %cl, %r14, %rdi
; AVX2-NEXT: xorl %r14d, %r14d
; AVX2-NEXT: vptest %ymm0, %ymm0
; AVX2-NEXT: shrxq %rcx, %r11, %rcx
-; AVX2-NEXT: cmoveq %r14, %r8
+; AVX2-NEXT: cmoveq %r14, %rdx
; AVX2-NEXT: cmoveq %r14, %rbx
; AVX2-NEXT: cmoveq %r14, %r10
; AVX2-NEXT: cmoveq %r14, %r9
+; AVX2-NEXT: cmoveq %r14, %r8
; AVX2-NEXT: cmoveq %r14, %rsi
-; AVX2-NEXT: cmoveq %r14, %rdx
; AVX2-NEXT: cmoveq %r14, %rdi
; AVX2-NEXT: cmoveq %r14, %rcx
; AVX2-NEXT: movq %rcx, 56(%rax)
-; AVX2-NEXT: movq %rdx, 48(%rax)
-; AVX2-NEXT: movq %rsi, 40(%rax)
+; AVX2-NEXT: movq %rsi, 48(%rax)
+; AVX2-NEXT: movq %r8, 40(%rax)
; AVX2-NEXT: movq %r9, 32(%rax)
; AVX2-NEXT: movq %r10, 24(%rax)
; AVX2-NEXT: movq %rbx, 16(%rax)
-; AVX2-NEXT: movq %r8, 8(%rax)
+; AVX2-NEXT: movq %rdx, 8(%rax)
; AVX2-NEXT: movq %rdi, (%rax)
; AVX2-NEXT: popq %rbx
; AVX2-NEXT: popq %r14
@@ -4947,119 +4961,121 @@ define i512 @isolate_msb_i512_load(ptr %p0, i512 %idx) nounwind {
; SSE2-NEXT: pushq %r12
; SSE2-NEXT: pushq %rbx
; SSE2-NEXT: pushq %rax
-; SSE2-NEXT: movq 8(%rsi), %r9
-; SSE2-NEXT: movq 16(%rsi), %rcx
-; SSE2-NEXT: movq 24(%rsi), %r8
-; SSE2-NEXT: movq 48(%rsi), %rdx
-; SSE2-NEXT: movq 56(%rsi), %r11
-; SSE2-NEXT: movdqa 32(%rsi), %xmm1
-; SSE2-NEXT: movdqa 48(%rsi), %xmm2
+; SSE2-NEXT: movq %rdi, %rax
; SSE2-NEXT: movdqa 16(%rsi), %xmm0
-; SSE2-NEXT: por %xmm2, %xmm0
-; SSE2-NEXT: movdqa (%rsi), %xmm3
-; SSE2-NEXT: por %xmm1, %xmm3
-; SSE2-NEXT: por %xmm0, %xmm3
+; SSE2-NEXT: por 48(%rsi), %xmm0
+; SSE2-NEXT: movdqa (%rsi), %xmm1
+; SSE2-NEXT: por 32(%rsi), %xmm1
+; SSE2-NEXT: por %xmm0, %xmm1
; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: pcmpeqd %xmm0, %xmm3
-; SSE2-NEXT: movmskps %xmm3, %eax
-; SSE2-NEXT: xorl $15, %eax
-; SSE2-NEXT: bsrq %r11, %r10
-; SSE2-NEXT: xorq $63, %r10
-; SSE2-NEXT: bsrq %rdx, %rbx
-; SSE2-NEXT: xorq $63, %rbx
-; SSE2-NEXT: orq $64, %rbx
-; SSE2-NEXT: testq %r11, %r11
-; SSE2-NEXT: cmovneq %r10, %rbx
-; SSE2-NEXT: movq 40(%rsi), %r14
-; SSE2-NEXT: bsrq %r14, %r15
-; SSE2-NEXT: bsrq 32(%rsi), %r10
+; SSE2-NEXT: pcmpeqd %xmm0, %xmm1
+; SSE2-NEXT: movmskps %xmm1, %ecx
+; SSE2-NEXT: xorl $15, %ecx
+; SSE2-NEXT: je .LBB17_1
+; SSE2-NEXT: # %bb.2: # %select.false.sink
+; SSE2-NEXT: movq 56(%rsi), %rdi
+; SSE2-NEXT: movq 48(%rsi), %r9
+; SSE2-NEXT: movq 40(%rsi), %rcx
+; SSE2-NEXT: movq 32(%rsi), %r8
+; SSE2-NEXT: movq 24(%rsi), %r10
+; SSE2-NEXT: movq 16(%rsi), %r11
+; SSE2-NEXT: movq 8(%rsi), %r14
+; SSE2-NEXT: bsrq %rdi, %rdx
+; SSE2-NEXT: xorq $63, %rdx
+; SSE2-NEXT: bsrq %r9, %r15
; SSE2-NEXT: xorq $63, %r15
-; SSE2-NEXT: xorq $63, %r10
-; SSE2-NEXT: orq $64, %r10
-; SSE2-NEXT: testq %r14, %r14
-; SSE2-NEXT: cmovneq %r15, %r10
-; SSE2-NEXT: orq $128, %r10
-; SSE2-NEXT: orq %r11, %rdx
-; SSE2-NEXT: cmovneq %rbx, %r10
-; SSE2-NEXT: bsrq %r8, %rdx
+; SSE2-NEXT: orq $64, %r15
+; SSE2-NEXT: testq %rdi, %rdi
+; SSE2-NEXT: cmovneq %rdx, %r15
+; SSE2-NEXT: bsrq %rcx, %rdx
; SSE2-NEXT: xorq $63, %rdx
-; SSE2-NEXT: bsrq %rcx, %r11
-; SSE2-NEXT: xorq $63, %r11
-; SSE2-NEXT: orq $64, %r11
-; SSE2-NEXT: testq %r8, %r8
-; SSE2-NEXT: cmovneq %rdx, %r11
-; SSE2-NEXT: bsrq %r9, %rbx
+; SSE2-NEXT: bsrq %r8, %rbx
; SSE2-NEXT: xorq $63, %rbx
+; SSE2-NEXT: orq $64, %rbx
+; SSE2-NEXT: testq %rcx, %rcx
+; SSE2-NEXT: cmovneq %rdx, %rbx
+; SSE2-NEXT: orq $128, %rbx
+; SSE2-NEXT: movq %r9, %rdx
+; SSE2-NEXT: orq %rdi, %rdx
+; SSE2-NEXT: cmovneq %r15, %rbx
+; SSE2-NEXT: bsrq %r10, %rdx
+; SSE2-NEXT: xorq $63, %rdx
+; SSE2-NEXT: bsrq %r11, %r15
+; SSE2-NEXT: xorq $63, %r15
+; SSE2-NEXT: orq $64, %r15
+; SSE2-NEXT: testq %r10, %r10
+; SSE2-NEXT: cmovneq %rdx, %r15
+; SSE2-NEXT: bsrq %r14, %r12
+; SSE2-NEXT: xorq $63, %r12
; SSE2-NEXT: bsrq (%rsi), %rdx
; SSE2-NEXT: xorq $63, %rdx
; SSE2-NEXT: orq $64, %rdx
-; SSE2-NEXT: testq %r9, %r9
-; SSE2-NEXT: cmovneq %rbx, %rdx
+; SSE2-NEXT: testq %r14, %r14
+; SSE2-NEXT: cmovneq %r12, %rdx
; SSE2-NEXT: orq $128, %rdx
-; SSE2-NEXT: orq %r8, %rcx
-; SSE2-NEXT: cmovneq %r11, %rdx
+; SSE2-NEXT: orq %r10, %r11
+; SSE2-NEXT: cmovneq %r15, %rdx
+; SSE2-NEXT: movabsq $-9223372036854775808, %rsi # imm = 0x8000000000000000
; SSE2-NEXT: orq $256, %rdx # imm = 0x100
-; SSE2-NEXT: por %xmm2, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm0, %xmm1
-; SSE2-NEXT: movmskps %xmm1, %ecx
-; SSE2-NEXT: xorl $15, %ecx
-; SSE2-NEXT: cmovneq %r10, %rdx
+; SSE2-NEXT: orq %r9, %r8
+; SSE2-NEXT: orq %rdi, %rcx
+; SSE2-NEXT: orq %r8, %rcx
+; SSE2-NEXT: cmovneq %rbx, %rdx
; SSE2-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000
-; SSE2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE2-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: movl %edx, %ecx
; SSE2-NEXT: andl $63, %ecx
; SSE2-NEXT: shrl $3, %edx
; SSE2-NEXT: andl $56, %edx
; SSE2-NEXT: movq $0, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: movq -72(%rsp,%rdx), %rsi
-; SSE2-NEXT: movq -80(%rsp,%rdx), %r10
+; SSE2-NEXT: movq -80(%rsp,%rdx), %r9
+; SSE2-NEXT: movq %r9, %rdi
+; SSE2-NEXT: shrdq %cl, %rsi, %rdi
+; SSE2-NEXT: movq -88(%rsp,%rdx), %r10
; SSE2-NEXT: movq %r10, %r8
-; SSE2-NEXT: shrdq %cl, %rsi, %r8
-; SSE2-NEXT: movq -88(%rsp,%rdx), %r11
+; SSE2-NEXT: shrdq %cl, %r9, %r8
+; SSE2-NEXT: movq -96(%rsp,%rdx), %r11
; SSE2-NEXT: movq %r11, %r9
; SSE2-NEXT: shrdq %cl, %r10, %r9
-; SSE2-NEXT: movq -96(%rsp,%rdx), %rbx
+; SSE2-NEXT: movq -104(%rsp,%rdx), %rbx
; SSE2-NEXT: movq %rbx, %r10
; SSE2-NEXT: shrdq %cl, %r11, %r10
-; SSE2-NEXT: movq -104(%rsp,%rdx), %r14
+; SSE2-NEXT: movq -112(%rsp,%rdx), %r14
; SSE2-NEXT: movq %r14, %r11
; SSE2-NEXT: shrdq %cl, %rbx, %r11
-; SSE2-NEXT: movq -112(%rsp,%rdx), %r15
+; SSE2-NEXT: movq -120(%rsp,%rdx), %r15
; SSE2-NEXT: movq %r15, %rbx
; SSE2-NEXT: shrdq %cl, %r14, %rbx
-; SSE2-NEXT: movq -120(%rsp,%rdx), %r12
-; SSE2-NEXT: movq %r12, %r14
-; SSE2-NEXT: shrdq %cl, %r15, %r14
; SSE2-NEXT: movq -128(%rsp,%rdx), %rdx
; SSE2-NEXT: shrq %cl, %rsi
; SSE2-NEXT: # kill: def $cl killed $cl killed $ecx
-; SSE2-NEXT: shrdq %cl, %r12, %rdx
-; SSE2-NEXT: xorl %ecx, %ecx
-; SSE2-NEXT: testl %eax, %eax
-; SSE2-NEXT: cmoveq %rcx, %r14
-; SSE2-NEXT: cmoveq %rcx, %rbx
-; SSE2-NEXT: cmoveq %rcx, %r11
-; SSE2-NEXT: cmoveq %rcx, %r10
-; SSE2-NEXT: cmoveq %rcx, %r9
-; SSE2-NEXT: cmoveq %rcx, %r8
-; SSE2-NEXT: cmoveq %rcx, %rdx
-; SSE2-NEXT: movq %rdi, %rax
-; SSE2-NEXT: cmoveq %rcx, %rsi
-; SSE2-NEXT: movq %rsi, 56(%rdi)
-; SSE2-NEXT: movq %r8, 48(%rdi)
-; SSE2-NEXT: movq %r9, 40(%rdi)
-; SSE2-NEXT: movq %r10, 32(%rdi)
-; SSE2-NEXT: movq %r11, 24(%rdi)
-; SSE2-NEXT: movq %rbx, 16(%rdi)
-; SSE2-NEXT: movq %r14, 8(%rdi)
-; SSE2-NEXT: movq %rdx, (%rdi)
+; SSE2-NEXT: shrdq %cl, %r15, %rdx
+; SSE2-NEXT: jmp .LBB17_3
+; SSE2-NEXT: .LBB17_1:
+; SSE2-NEXT: xorl %edx, %edx
+; SSE2-NEXT: xorl %ebx, %ebx
+; SSE2-NEXT: xorl %r11d, %r11d
+; SSE2-NEXT: xorl %r10d, %r10d
+; SSE2-NEXT: xorl %r9d, %r9d
+; SSE2-NEXT: xorl %r8d, %r8d
+; SSE2-NEXT: xorl %edi, %edi
+; SSE2-NEXT: xorl %esi, %esi
+; SSE2-NEXT: .LBB17_3: # %select.end
+; SSE2-NEXT: movq %rdx, (%rax)
+; SSE2-NEXT: movq %rbx, 8(%rax)
+; SSE2-NEXT: movq %r11, 16(%rax)
+; SSE2-NEXT: movq %r10, 24(%rax)
+; SSE2-NEXT: movq %r9, 32(%rax)
+; SSE2-NEXT: movq %r8, 40(%rax)
+; SSE2-NEXT: movq %rdi, 48(%rax)
+; SSE2-NEXT: movq %rsi, 56(%rax)
; SSE2-NEXT: addq $8, %rsp
; SSE2-NEXT: popq %rbx
; SSE2-NEXT: popq %r12
@@ -5071,116 +5087,125 @@ define i512 @isolate_msb_i512_load(ptr %p0, i512 %idx) nounwind {
; SSE42: # %bb.0:
; SSE42-NEXT: pushq %r15
; SSE42-NEXT: pushq %r14
+; SSE42-NEXT: pushq %r13
+; SSE42-NEXT: pushq %r12
; SSE42-NEXT: pushq %rbx
-; SSE42-NEXT: movq 8(%rsi), %r8
-; SSE42-NEXT: movq 16(%rsi), %rcx
-; SSE42-NEXT: movq 24(%rsi), %rdx
-; SSE42-NEXT: movq 40(%rsi), %r11
-; SSE42-NEXT: movq 48(%rsi), %rax
-; SSE42-NEXT: movq 56(%rsi), %r10
-; SSE42-NEXT: movdqa 32(%rsi), %xmm2
-; SSE42-NEXT: movdqa 48(%rsi), %xmm0
-; SSE42-NEXT: movdqa (%rsi), %xmm1
-; SSE42-NEXT: bsrq %r10, %r9
-; SSE42-NEXT: xorq $63, %r9
-; SSE42-NEXT: bsrq %rax, %rbx
-; SSE42-NEXT: xorq $63, %rbx
-; SSE42-NEXT: orq $64, %rbx
-; SSE42-NEXT: testq %r10, %r10
-; SSE42-NEXT: cmovneq %r9, %rbx
-; SSE42-NEXT: bsrq %r11, %r14
+; SSE42-NEXT: movdqa (%rsi), %xmm0
+; SSE42-NEXT: movdqa 16(%rsi), %xmm1
+; SSE42-NEXT: por 48(%rsi), %xmm1
+; SSE42-NEXT: por 32(%rsi), %xmm0
+; SSE42-NEXT: movq %rdi, %rax
+; SSE42-NEXT: por %xmm1, %xmm0
+; SSE42-NEXT: ptest %xmm0, %xmm0
+; SSE42-NEXT: je .LBB17_1
+; SSE42-NEXT: # %bb.2: # %select.false.sink
+; SSE42-NEXT: movq 56(%rsi), %r8
+; SSE42-NEXT: movq 48(%rsi), %r10
+; SSE42-NEXT: movq 40(%rsi), %rdi
+; SSE42-NEXT: movq 32(%rsi), %r9
+; SSE42-NEXT: movq 24(%rsi), %r11
+; SSE42-NEXT: movq 16(%rsi), %rbx
+; SSE42-NEXT: movq 8(%rsi), %r15
+; SSE42-NEXT: movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000
+; SSE42-NEXT: bsrq %r8, %rdx
+; SSE42-NEXT: xorq $63, %rdx
+; SSE42-NEXT: bsrq %r10, %r12
+; SSE42-NEXT: xorq $63, %r12
+; SSE42-NEXT: orq $64, %r12
+; SSE42-NEXT: testq %r8, %r8
+; SSE42-NEXT: cmovneq %rdx, %r12
+; SSE42-NEXT: bsrq %rdi, %rdx
+; SSE42-NEXT: xorq $63, %rdx
+; SSE42-NEXT: bsrq %r9, %r14
; SSE42-NEXT: xorq $63, %r14
-; SSE42-NEXT: bsrq 32(%rsi), %r9
-; SSE42-NEXT: xorq $63, %r9
-; SSE42-NEXT: orq $64, %r9
+; SSE42-NEXT: orq $64, %r14
+; SSE42-NEXT: testq %rdi, %rdi
+; SSE42-NEXT: cmovneq %rdx, %r14
+; SSE42-NEXT: orq $128, %r14
+; SSE42-NEXT: movq %r10, %rdx
+; SSE42-NEXT: orq %r8, %rdx
+; SSE42-NEXT: cmovneq %r12, %r14
+; SSE42-NEXT: bsrq %r11, %rdx
+; SSE42-NEXT: xorq $63, %rdx
+; SSE42-NEXT: bsrq %rbx, %r12
+; SSE42-NEXT: xorq $63, %r12
+; SSE42-NEXT: orq $64, %r12
; SSE42-NEXT: testq %r11, %r11
-; SSE42-NEXT: cmovneq %r14, %r9
-; SSE42-NEXT: orq $128, %r9
-; SSE42-NEXT: orq %r10, %rax
-; SSE42-NEXT: cmovneq %rbx, %r9
-; SSE42-NEXT: bsrq %rdx, %rax
-; SSE42-NEXT: xorq $63, %rax
-; SSE42-NEXT: bsrq %rcx, %r10
-; SSE42-NEXT: xorq $63, %r10
-; SSE42-NEXT: orq $64, %r10
-; SSE42-NEXT: testq %rdx, %rdx
-; SSE42-NEXT: cmovneq %rax, %r10
-; SSE42-NEXT: por %xmm2, %xmm1
-; SSE42-NEXT: bsrq %r8, %r11
-; SSE42-NEXT: bsrq (%rsi), %rax
-; SSE42-NEXT: xorq $63, %r11
-; SSE42-NEXT: xorq $63, %rax
-; SSE42-NEXT: orq $64, %rax
-; SSE42-NEXT: testq %r8, %r8
-; SSE42-NEXT: cmovneq %r11, %rax
-; SSE42-NEXT: orq $128, %rax
-; SSE42-NEXT: orq %rdx, %rcx
-; SSE42-NEXT: cmovneq %r10, %rax
-; SSE42-NEXT: orq $256, %rax # imm = 0x100
-; SSE42-NEXT: por %xmm0, %xmm2
-; SSE42-NEXT: ptest %xmm2, %xmm2
-; SSE42-NEXT: cmovneq %r9, %rax
-; SSE42-NEXT: movdqa 16(%rsi), %xmm2
-; SSE42-NEXT: xorps %xmm3, %xmm3
-; SSE42-NEXT: movaps %xmm3, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movaps %xmm3, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movaps %xmm3, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movaps %xmm3, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000
+; SSE42-NEXT: cmovneq %rdx, %r12
+; SSE42-NEXT: bsrq %r15, %r13
+; SSE42-NEXT: xorq $63, %r13
+; SSE42-NEXT: bsrq (%rsi), %rdx
+; SSE42-NEXT: xorq $63, %rdx
+; SSE42-NEXT: orq $64, %rdx
+; SSE42-NEXT: testq %r15, %r15
+; SSE42-NEXT: cmovneq %r13, %rdx
+; SSE42-NEXT: orq $128, %rdx
+; SSE42-NEXT: orq %r11, %rbx
+; SSE42-NEXT: cmovneq %r12, %rdx
+; SSE42-NEXT: orq $256, %rdx # imm = 0x100
+; SSE42-NEXT: orq %r10, %r9
+; SSE42-NEXT: orq %r8, %rdi
+; SSE42-NEXT: orq %r9, %rdi
+; SSE42-NEXT: cmovneq %r14, %rdx
+; SSE42-NEXT: pxor %xmm0, %xmm0
+; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
; SSE42-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movaps %xmm3, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movaps %xmm3, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movaps %xmm3, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movl %eax, %ecx
+; SSE42-NEXT: movl %edx, %ecx
; SSE42-NEXT: andl $63, %ecx
-; SSE42-NEXT: shrl $3, %eax
-; SSE42-NEXT: andl $56, %eax
+; SSE42-NEXT: shrl $3, %edx
+; SSE42-NEXT: andl $56, %edx
; SSE42-NEXT: movq $0, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movq -72(%rsp,%rax), %rdx
-; SSE42-NEXT: movq -80(%rsp,%rax), %r9
-; SSE42-NEXT: movq %r9, %rsi
-; SSE42-NEXT: shrdq %cl, %rdx, %rsi
-; SSE42-NEXT: movq -88(%rsp,%rax), %r10
+; SSE42-NEXT: movq -72(%rsp,%rdx), %rsi
+; SSE42-NEXT: movq -80(%rsp,%rdx), %r9
+; SSE42-NEXT: movq %r9, %rdi
+; SSE42-NEXT: shrdq %cl, %rsi, %rdi
+; SSE42-NEXT: movq -88(%rsp,%rdx), %r10
; SSE42-NEXT: movq %r10, %r8
; SSE42-NEXT: shrdq %cl, %r9, %r8
-; SSE42-NEXT: movq -96(%rsp,%rax), %r11
+; SSE42-NEXT: movq -96(%rsp,%rdx), %r11
; SSE42-NEXT: movq %r11, %r9
; SSE42-NEXT: shrdq %cl, %r10, %r9
-; SSE42-NEXT: movq -104(%rsp,%rax), %rbx
+; SSE42-NEXT: movq -104(%rsp,%rdx), %rbx
; SSE42-NEXT: movq %rbx, %r10
; SSE42-NEXT: shrdq %cl, %r11, %r10
-; SSE42-NEXT: movq -112(%rsp,%rax), %r14
+; SSE42-NEXT: movq -112(%rsp,%rdx), %r14
; SSE42-NEXT: movq %r14, %r11
; SSE42-NEXT: shrdq %cl, %rbx, %r11
-; SSE42-NEXT: movq -120(%rsp,%rax), %r15
+; SSE42-NEXT: movq -120(%rsp,%rdx), %r15
; SSE42-NEXT: movq %r15, %rbx
; SSE42-NEXT: shrdq %cl, %r14, %rbx
-; SSE42-NEXT: movq -128(%rsp,%rax), %r14
-; SSE42-NEXT: shrq %cl, %rdx
+; SSE42-NEXT: movq -128(%rsp,%rdx), %rdx
+; SSE42-NEXT: shrq %cl, %rsi
; SSE42-NEXT: # kill: def $cl killed $cl killed $ecx
-; SSE42-NEXT: shrdq %cl, %r15, %r14
-; SSE42-NEXT: por %xmm0, %xmm2
-; SSE42-NEXT: por %xmm2, %xmm1
-; SSE42-NEXT: xorl %ecx, %ecx
-; SSE42-NEXT: ptest %xmm1, %xmm1
-; SSE42-NEXT: cmoveq %rcx, %rbx
-; SSE42-NEXT: cmoveq %rcx, %r11
-; SSE42-NEXT: cmoveq %rcx, %r10
-; SSE42-NEXT: cmoveq %rcx, %r9
-; SSE42-NEXT: cmoveq %rcx, %r8
-; SSE42-NEXT: cmoveq %rcx, %rsi
-; SSE42-NEXT: cmoveq %rcx, %r14
-; SSE42-NEXT: movq %rdi, %rax
-; SSE42-NEXT: cmoveq %rcx, %rdx
-; SSE42-NEXT: movq %rdx, 56(%rdi)
-; SSE42-NEXT: movq %rsi, 48(%rdi)
-; SSE42-NEXT: movq %r8, 40(%rdi)
-; SSE42-NEXT: movq %r9, 32(%rdi)
-; SSE42-NEXT: movq %r10, 24(%rdi)
-; SSE42-NEXT: movq %r11, 16(%rdi)
-; SSE42-NEXT: movq %rbx, 8(%rdi)
-; SSE42-NEXT: movq %r14, (%rdi)
+; SSE42-NEXT: shrdq %cl, %r15, %rdx
+; SSE42-NEXT: jmp .LBB17_3
+; SSE42-NEXT: .LBB17_1:
+; SSE42-NEXT: xorl %edx, %edx
+; SSE42-NEXT: xorl %ebx, %ebx
+; SSE42-NEXT: xorl %r11d, %r11d
+; SSE42-NEXT: xorl %r10d, %r10d
+; SSE42-NEXT: xorl %r9d, %r9d
+; SSE42-NEXT: xorl %r8d, %r8d
+; SSE42-NEXT: xorl %edi, %edi
+; SSE42-NEXT: xorl %esi, %esi
+; SSE42-NEXT: .LBB17_3: # %select.end
+; SSE42-NEXT: movq %rdx, (%rax)
+; SSE42-NEXT: movq %rbx, 8(%rax)
+; SSE42-NEXT: movq %r11, 16(%rax)
+; SSE42-NEXT: movq %r10, 24(%rax)
+; SSE42-NEXT: movq %r9, 32(%rax)
+; SSE42-NEXT: movq %r8, 40(%rax)
+; SSE42-NEXT: movq %rdi, 48(%rax)
+; SSE42-NEXT: movq %rsi, 56(%rax)
; SSE42-NEXT: popq %rbx
+; SSE42-NEXT: popq %r12
+; SSE42-NEXT: popq %r13
; SSE42-NEXT: popq %r14
; SSE42-NEXT: popq %r15
; SSE42-NEXT: retq
@@ -5191,56 +5216,55 @@ define i512 @isolate_msb_i512_load(ptr %p0, i512 %idx) nounwind {
; AVX2-NEXT: pushq %r14
; AVX2-NEXT: pushq %rbx
; AVX2-NEXT: vmovdqu 32(%rsi), %ymm1
-; AVX2-NEXT: movq 8(%rsi), %r8
; AVX2-NEXT: movq 16(%rsi), %rax
; AVX2-NEXT: movq 24(%rsi), %rcx
; AVX2-NEXT: movq 40(%rsi), %rdx
-; AVX2-NEXT: movq 48(%rsi), %r10
+; AVX2-NEXT: movq 48(%rsi), %r9
; AVX2-NEXT: vpor (%rsi), %ymm1, %ymm0
-; AVX2-NEXT: movq 56(%rsi), %r11
-; AVX2-NEXT: lzcntq %r11, %r9
+; AVX2-NEXT: movq 56(%rsi), %r10
+; AVX2-NEXT: lzcntq %r10, %r8
+; AVX2-NEXT: lzcntq %r9, %r11
+; AVX2-NEXT: addq $64, %r11
+; AVX2-NEXT: testq %r10, %r10
+; AVX2-NEXT: cmovneq %r8, %r11
; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: lzcntq %r10, %rbx
-; AVX2-NEXT: addq $64, %rbx
-; AVX2-NEXT: testq %r11, %r11
-; AVX2-NEXT: cmovneq %r9, %rbx
-; AVX2-NEXT: xorl %r14d, %r14d
-; AVX2-NEXT: lzcntq %rdx, %r14
-; AVX2-NEXT: xorl %r9d, %r9d
-; AVX2-NEXT: lzcntq 32(%rsi), %r9
-; AVX2-NEXT: addq $64, %r9
+; AVX2-NEXT: lzcntq %rdx, %rbx
+; AVX2-NEXT: xorl %r8d, %r8d
+; AVX2-NEXT: lzcntq 32(%rsi), %r8
+; AVX2-NEXT: addq $64, %r8
; AVX2-NEXT: testq %rdx, %rdx
-; AVX2-NEXT: cmovneq %r14, %r9
-; AVX2-NEXT: subq $-128, %r9
-; AVX2-NEXT: orq %r11, %r10
-; AVX2-NEXT: cmovneq %rbx, %r9
+; AVX2-NEXT: cmovneq %rbx, %r8
+; AVX2-NEXT: subq $-128, %r8
+; AVX2-NEXT: orq %r10, %r9
+; AVX2-NEXT: cmovneq %r11, %r8
; AVX2-NEXT: xorl %edx, %edx
; AVX2-NEXT: lzcntq %rcx, %rdx
-; AVX2-NEXT: xorl %r10d, %r10d
-; AVX2-NEXT: lzcntq %rax, %r10
-; AVX2-NEXT: addq $64, %r10
+; AVX2-NEXT: xorl %r9d, %r9d
+; AVX2-NEXT: lzcntq %rax, %r9
+; AVX2-NEXT: addq $64, %r9
; AVX2-NEXT: testq %rcx, %rcx
-; AVX2-NEXT: cmovneq %rdx, %r10
+; AVX2-NEXT: cmovneq %rdx, %r9
+; AVX2-NEXT: movq 8(%rsi), %r10
; AVX2-NEXT: xorl %r11d, %r11d
-; AVX2-NEXT: lzcntq %r8, %r11
+; AVX2-NEXT: lzcntq %r10, %r11
; AVX2-NEXT: xorl %edx, %edx
; AVX2-NEXT: lzcntq (%rsi), %rdx
; AVX2-NEXT: addq $64, %rdx
-; AVX2-NEXT: testq %r8, %r8
+; AVX2-NEXT: testq %r10, %r10
; AVX2-NEXT: cmovneq %r11, %rdx
; AVX2-NEXT: subq $-128, %rdx
; AVX2-NEXT: orq %rcx, %rax
-; AVX2-NEXT: cmovneq %r10, %rdx
+; AVX2-NEXT: cmovneq %r9, %rdx
; AVX2-NEXT: addq $256, %rdx # imm = 0x100
; AVX2-NEXT: vpor 48(%rsi), %xmm1, %xmm1
; AVX2-NEXT: vptest %xmm1, %xmm1
-; AVX2-NEXT: cmovneq %r9, %rdx
-; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT: vmovdqu %ymm1, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vmovdqu %ymm1, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vmovaps {{.*#+}} ymm2 = [0,0,0,9223372036854775808]
-; AVX2-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vmovdqu %ymm1, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: cmovneq %r8, %rdx
+; AVX2-NEXT: vmovaps {{.*#+}} ymm1 = [0,0,0,9223372036854775808]
+; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movl %edx, %ecx
; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: shrl $3, %edx
diff --git a/llvm/test/CodeGen/X86/bsf.ll b/llvm/test/CodeGen/X86/bsf.ll
index f780f0172647f..c889fb8d6154a 100644
--- a/llvm/test/CodeGen/X86/bsf.ll
+++ b/llvm/test/CodeGen/X86/bsf.ll
@@ -223,25 +223,24 @@ define i64 @cmov_bsf64(i64 %x, i64 %y) nounwind {
define i64 @cmov_bsf64_undef(i64 %x, i64 %y) nounwind {
; X86-LABEL: cmov_bsf64_undef:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ecx, %edx
-; X86-NEXT: orl %eax, %edx
-; X86-NEXT: je .LBB7_5
-; X86-NEXT: # %bb.1: # %select.false.sink
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-NEXT: testl %ecx, %ecx
-; X86-NEXT: jne .LBB7_2
-; X86-NEXT: # %bb.3: # %select.false.sink
-; X86-NEXT: rep bsfl %eax, %eax
+; X86-NEXT: jne .LBB7_1
+; X86-NEXT: # %bb.2:
+; X86-NEXT: rep bsfl %edx, %eax
; X86-NEXT: addl $32, %eax
-; X86-NEXT: xorl %edx, %edx
-; X86-NEXT: retl
-; X86-NEXT: .LBB7_5: # %select.end
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: orl %edx, %ecx
+; X86-NEXT: jne .LBB7_5
+; X86-NEXT: .LBB7_4:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-NEXT: retl
-; X86-NEXT: .LBB7_2:
+; X86-NEXT: .LBB7_1:
; X86-NEXT: rep bsfl %ecx, %eax
+; X86-NEXT: orl %edx, %ecx
+; X86-NEXT: je .LBB7_4
+; X86-NEXT: .LBB7_5:
; X86-NEXT: xorl %edx, %edx
; X86-NEXT: retl
;
@@ -335,21 +334,17 @@ define i128 @cmov_bsf128(i128 %x, i128 %y) nounwind {
;
; X64-LABEL: cmov_bsf128:
; X64: # %bb.0:
-; X64-NEXT: movq %rdi, %rax
-; X64-NEXT: orq %rsi, %rax
-; X64-NEXT: je .LBB8_2
-; X64-NEXT: # %bb.1: # %select.false.sink
-; X64-NEXT: rep bsfq %rdi, %rcx
+; X64-NEXT: rep bsfq %rdi, %r8
; X64-NEXT: movl $64, %eax
; X64-NEXT: rep bsfq %rsi, %rax
; X64-NEXT: addq $64, %rax
; X64-NEXT: testq %rdi, %rdi
-; X64-NEXT: cmovneq %rcx, %rax
-; X64-NEXT: xorl %edx, %edx
-; X64-NEXT: retq
-; X64-NEXT: .LBB8_2: # %select.end
-; X64-NEXT: movq %rdx, %rax
-; X64-NEXT: movq %rcx, %rdx
+; X64-NEXT: cmovneq %r8, %rax
+; X64-NEXT: xorl %r8d, %r8d
+; X64-NEXT: orq %rsi, %rdi
+; X64-NEXT: cmoveq %rdx, %rax
+; X64-NEXT: cmoveq %rcx, %r8
+; X64-NEXT: movq %r8, %rdx
; X64-NEXT: retq
%1 = tail call i128 @llvm.cttz.i128(i128 %x, i1 false)
%2 = icmp eq i128 %x, 0
@@ -367,58 +362,59 @@ define i128 @cmov_bsf128_undef(i128 %x, i128 %y) nounwind {
; X86-NEXT: pushl %esi
; X86-NEXT: andl $-16, %esp
; X86-NEXT: subl $16, %esp
-; X86-NEXT: movl 36(%ebp), %esi
-; X86-NEXT: movl 32(%ebp), %edi
-; X86-NEXT: movl 28(%ebp), %ecx
-; X86-NEXT: movl 24(%ebp), %edx
-; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: orl %esi, %eax
-; X86-NEXT: movl %edx, %ebx
-; X86-NEXT: orl %edi, %ebx
+; X86-NEXT: movl 32(%ebp), %edx
+; X86-NEXT: movl 24(%ebp), %ebx
+; X86-NEXT: movl 28(%ebp), %esi
+; X86-NEXT: testl %ebx, %ebx
+; X86-NEXT: jne .LBB9_1
+; X86-NEXT: # %bb.2:
+; X86-NEXT: rep bsfl %esi, %eax
+; X86-NEXT: addl $32, %eax
+; X86-NEXT: jmp .LBB9_3
+; X86-NEXT: .LBB9_1:
+; X86-NEXT: rep bsfl %ebx, %eax
+; X86-NEXT: .LBB9_3:
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 36(%ebp), %ecx
+; X86-NEXT: testl %edx, %edx
+; X86-NEXT: jne .LBB9_4
+; X86-NEXT: # %bb.5:
+; X86-NEXT: movl %edx, %eax
+; X86-NEXT: movl %ecx, %edi
+; X86-NEXT: rep bsfl %ecx, %ecx
+; X86-NEXT: addl $32, %ecx
+; X86-NEXT: jmp .LBB9_6
+; X86-NEXT: .LBB9_4:
+; X86-NEXT: movl %ecx, %edi
+; X86-NEXT: movl %edx, %eax
+; X86-NEXT: rep bsfl %edx, %ecx
+; X86-NEXT: .LBB9_6:
+; X86-NEXT: movl %ebx, %edx
+; X86-NEXT: orl %esi, %edx
+; X86-NEXT: jne .LBB9_8
+; X86-NEXT: # %bb.7:
+; X86-NEXT: addl $64, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: .LBB9_8:
+; X86-NEXT: orl %edi, %esi
; X86-NEXT: orl %eax, %ebx
+; X86-NEXT: xorl %edi, %edi
+; X86-NEXT: orl %esi, %ebx
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: movl $0, %edx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: jne .LBB9_10
+; X86-NEXT: # %bb.9:
+; X86-NEXT: movl 48(%ebp), %ecx
+; X86-NEXT: movl 52(%ebp), %edx
+; X86-NEXT: movl 40(%ebp), %esi
+; X86-NEXT: movl 44(%ebp), %edi
+; X86-NEXT: .LBB9_10:
; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: je .LBB9_11
-; X86-NEXT: # %bb.1: # %select.true.sink
-; X86-NEXT: testl %edx, %edx
-; X86-NEXT: jne .LBB9_2
-; X86-NEXT: # %bb.3: # %select.true.sink
-; X86-NEXT: rep bsfl %ecx, %ebx
-; X86-NEXT: addl $32, %ebx
-; X86-NEXT: testl %edi, %edi
-; X86-NEXT: je .LBB9_6
-; X86-NEXT: .LBB9_5:
-; X86-NEXT: rep bsfl %edi, %esi
-; X86-NEXT: orl %ecx, %edx
-; X86-NEXT: je .LBB9_8
-; X86-NEXT: jmp .LBB9_9
-; X86-NEXT: .LBB9_11: # %select.end
-; X86-NEXT: movl 52(%ebp), %ecx
-; X86-NEXT: movl 48(%ebp), %edx
-; X86-NEXT: movl 44(%ebp), %esi
-; X86-NEXT: movl 40(%ebp), %edi
-; X86-NEXT: movl %edi, (%eax)
-; X86-NEXT: movl %esi, 4(%eax)
-; X86-NEXT: movl %edx, 8(%eax)
-; X86-NEXT: movl %ecx, 12(%eax)
-; X86-NEXT: jmp .LBB9_10
-; X86-NEXT: .LBB9_2:
-; X86-NEXT: rep bsfl %edx, %ebx
-; X86-NEXT: testl %edi, %edi
-; X86-NEXT: jne .LBB9_5
-; X86-NEXT: .LBB9_6: # %select.true.sink
-; X86-NEXT: rep bsfl %esi, %esi
-; X86-NEXT: addl $32, %esi
-; X86-NEXT: orl %ecx, %edx
-; X86-NEXT: jne .LBB9_9
-; X86-NEXT: .LBB9_8: # %select.true.sink
-; X86-NEXT: addl $64, %esi
-; X86-NEXT: movl %esi, %ebx
-; X86-NEXT: .LBB9_9: # %select.true.sink
-; X86-NEXT: movl %ebx, (%eax)
-; X86-NEXT: movl $0, 12(%eax)
-; X86-NEXT: movl $0, 8(%eax)
-; X86-NEXT: movl $0, 4(%eax)
-; X86-NEXT: .LBB9_10: # %select.true.sink
+; X86-NEXT: movl %edx, 12(%eax)
+; X86-NEXT: movl %ecx, 8(%eax)
+; X86-NEXT: movl %edi, 4(%eax)
+; X86-NEXT: movl %esi, (%eax)
; X86-NEXT: leal -12(%ebp), %esp
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
@@ -428,20 +424,16 @@ define i128 @cmov_bsf128_undef(i128 %x, i128 %y) nounwind {
;
; X64-LABEL: cmov_bsf128_undef:
; X64: # %bb.0:
-; X64-NEXT: movq %rdi, %rax
-; X64-NEXT: orq %rsi, %rax
-; X64-NEXT: je .LBB9_2
-; X64-NEXT: # %bb.1: # %select.true.sink
-; X64-NEXT: rep bsfq %rdi, %rcx
+; X64-NEXT: rep bsfq %rdi, %r8
; X64-NEXT: rep bsfq %rsi, %rax
; X64-NEXT: addq $64, %rax
; X64-NEXT: testq %rdi, %rdi
-; X64-NEXT: cmovneq %rcx, %rax
-; X64-NEXT: xorl %edx, %edx
-; X64-NEXT: retq
-; X64-NEXT: .LBB9_2: # %select.end
-; X64-NEXT: movq %rdx, %rax
-; X64-NEXT: movq %rcx, %rdx
+; X64-NEXT: cmovneq %r8, %rax
+; X64-NEXT: xorl %r8d, %r8d
+; X64-NEXT: orq %rsi, %rdi
+; X64-NEXT: cmoveq %rdx, %rax
+; X64-NEXT: cmoveq %rcx, %r8
+; X64-NEXT: movq %r8, %rdx
; X64-NEXT: retq
%1 = tail call i128 @llvm.cttz.i128(i128 %x, i1 true)
%2 = icmp ne i128 %x, 0
diff --git a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
index 5cfdbb7af38ad..96d0c32ce9e96 100644
--- a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
+++ b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-signed.ll
@@ -175,23 +175,23 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: movl 52(%ebp), %ecx
; X86-NEXT: movl %ecx, %edx
; X86-NEXT: sarl $31, %edx
-; X86-NEXT: movl %ecx, %ebx
-; X86-NEXT: xorl %edx, %ebx
+; X86-NEXT: movl %ecx, %edi
+; X86-NEXT: xorl %edx, %edi
; X86-NEXT: movl 48(%ebp), %esi
; X86-NEXT: xorl %edx, %esi
-; X86-NEXT: movl 44(%ebp), %eax
+; X86-NEXT: movl 44(%ebp), %ebx
+; X86-NEXT: xorl %edx, %ebx
+; X86-NEXT: movl 40(%ebp), %eax
; X86-NEXT: xorl %edx, %eax
-; X86-NEXT: movl 40(%ebp), %edi
-; X86-NEXT: xorl %edx, %edi
-; X86-NEXT: subl %edx, %edi
-; X86-NEXT: sbbl %edx, %eax
+; X86-NEXT: subl %edx, %eax
+; X86-NEXT: sbbl %edx, %ebx
; X86-NEXT: sbbl %edx, %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: sbbl %edx, %ebx
-; X86-NEXT: movl %eax, %ecx
-; X86-NEXT: orl %ebx, %ecx
-; X86-NEXT: movl %edi, %edx
+; X86-NEXT: sbbl %edx, %edi
+; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: orl %edi, %ecx
+; X86-NEXT: movl %eax, %edx
; X86-NEXT: orl %esi, %edx
; X86-NEXT: orl %ecx, %edx
; X86-NEXT: sete %dl
@@ -203,34 +203,35 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: sete %cl
; X86-NEXT: orb %dl, %cl
; X86-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X86-NEXT: bsrl %ebx, %esi
+; X86-NEXT: bsrl %edi, %esi
; X86-NEXT: xorl $31, %esi
-; X86-NEXT: bsrl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: bsrl %ecx, %edx
; X86-NEXT: xorl $31, %edx
; X86-NEXT: orl $32, %edx
-; X86-NEXT: testl %ebx, %ebx
+; X86-NEXT: testl %edi, %edi
; X86-NEXT: cmovnel %esi, %edx
-; X86-NEXT: bsrl %eax, %esi
+; X86-NEXT: bsrl %ebx, %esi
; X86-NEXT: xorl $31, %esi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: bsrl %edi, %ecx
-; X86-NEXT: xorl $31, %ecx
-; X86-NEXT: orl $32, %ecx
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: testl %eax, %eax
-; X86-NEXT: cmovnel %esi, %ecx
-; X86-NEXT: orl $64, %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: bsrl %eax, %eax
+; X86-NEXT: xorl $31, %eax
+; X86-NEXT: orl $32, %eax
; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %ebx, %esi
-; X86-NEXT: cmovnel %edx, %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: bsrl %eax, %esi
+; X86-NEXT: testl %ebx, %ebx
+; X86-NEXT: cmovnel %esi, %eax
+; X86-NEXT: orl $64, %eax
+; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %edi, %esi
+; X86-NEXT: cmovnel %edx, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: bsrl %ecx, %esi
; X86-NEXT: xorl $31, %esi
; X86-NEXT: bsrl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
; X86-NEXT: xorl $31, %edx
; X86-NEXT: orl $32, %edx
-; X86-NEXT: testl %eax, %eax
+; X86-NEXT: testl %ecx, %ecx
; X86-NEXT: cmovnel %esi, %edx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NEXT: bsrl %ebx, %edi
@@ -242,58 +243,54 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: cmovnel %edi, %esi
; X86-NEXT: orl $64, %esi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: orl %eax, %edi
+; X86-NEXT: orl %ecx, %edi
; X86-NEXT: cmovnel %edx, %esi
-; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: xorl %ebx, %ebx
+; X86-NEXT: xorl %edi, %edi
; X86-NEXT: subl %esi, %eax
-; X86-NEXT: movl $0, %edx
-; X86-NEXT: sbbl %edx, %edx
; X86-NEXT: movl $0, %esi
; X86-NEXT: sbbl %esi, %esi
-; X86-NEXT: movl $0, %edi
-; X86-NEXT: sbbl %edi, %edi
+; X86-NEXT: movl $0, %ebx
+; X86-NEXT: sbbl %ebx, %ebx
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: sbbl %ecx, %ecx
+; X86-NEXT: movl $127, %edx
+; X86-NEXT: cmpl %eax, %edx
+; X86-NEXT: movl $0, %edx
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: movl $0, %edx
+; X86-NEXT: sbbl %ebx, %edx
+; X86-NEXT: movl $0, %edx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %ecx, %edx
+; X86-NEXT: setb %dl
; X86-NEXT: cmpb $0, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Reload
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ebx, %ecx
; X86-NEXT: jne .LBB4_1
-; X86-NEXT: # %bb.2: # %select.false.sink
-; X86-NEXT: movl $127, %ecx
-; X86-NEXT: cmpl %eax, %ecx
-; X86-NEXT: movl $0, %ecx
-; X86-NEXT: sbbl %edx, %ecx
-; X86-NEXT: movl $0, %ecx
-; X86-NEXT: sbbl %esi, %ecx
-; X86-NEXT: movl $0, %ecx
-; X86-NEXT: sbbl %edi, %ecx
-; X86-NEXT: setb %cl
-; X86-NEXT: .LBB4_3: # %select.end
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X86-NEXT: testb %cl, %cl
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: cmovnel %ebx, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $0, %edi
+; X86-NEXT: .LBB4_2: # %select.end
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: cmovnel %edi, %ebx
+; X86-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X86-NEXT: testb %dl, %dl
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NEXT: cmovnel %edi, %esi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: cmovnel %edi, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: cmovnel %edi, %edx
; X86-NEXT: cmovel {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X86-NEXT: jne .LBB4_4
-; X86-NEXT: # %bb.10: # %select.end
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: jne .LBB4_8
+; X86-NEXT: # %bb.3: # %select.end
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: xorl $127, %eax
-; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %ecx, %eax
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
; X86-NEXT: orl %eax, %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: je .LBB4_11
-; X86-NEXT: # %bb.8: # %udiv-bb1
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: je .LBB4_8
+; X86-NEXT: # %bb.4: # %udiv-bb1
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: xorps %xmm0, %xmm0
@@ -302,8 +299,8 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NEXT: movl %ebx, %ecx
; X86-NEXT: xorb $127, %cl
@@ -312,195 +309,193 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: andb $12, %al
; X86-NEXT: negb %al
; X86-NEXT: movsbl %al, %eax
-; X86-NEXT: movl 152(%esp,%eax), %edx
-; X86-NEXT: movl 156(%esp,%eax), %esi
-; X86-NEXT: shldl %cl, %edx, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 144(%esp,%eax), %esi
-; X86-NEXT: movl 148(%esp,%eax), %eax
-; X86-NEXT: shldl %cl, %eax, %edx
+; X86-NEXT: movl 152(%esp,%eax), %edi
+; X86-NEXT: movl 156(%esp,%eax), %edx
+; X86-NEXT: shldl %cl, %edi, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shldl %cl, %esi, %eax
+; X86-NEXT: movl 144(%esp,%eax), %edx
+; X86-NEXT: movl 148(%esp,%eax), %eax
+; X86-NEXT: shldl %cl, %eax, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shldl %cl, %edx, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shll %cl, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shll %cl, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: addl $1, %ebx
; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: jb .LBB4_9
; X86-NEXT: # %bb.5: # %udiv-preheader
-; X86-NEXT: movl %ebx, %ecx
; X86-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %ebx, %ecx
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: shrb $3, %al
; X86-NEXT: andb $12, %al
; X86-NEXT: movzbl %al, %eax
; X86-NEXT: movl 108(%esp,%eax), %ebx
-; X86-NEXT: movl 104(%esp,%eax), %edx
-; X86-NEXT: movl %edx, %esi
-; X86-NEXT: shrdl %cl, %ebx, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 96(%esp,%eax), %esi
+; X86-NEXT: movl 104(%esp,%eax), %esi
+; X86-NEXT: movl %esi, %edx
+; X86-NEXT: shrdl %cl, %ebx, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 96(%esp,%eax), %edx
; X86-NEXT: movl 100(%esp,%eax), %edi
; X86-NEXT: movl %edi, %eax
-; X86-NEXT: shrdl %cl, %edx, %eax
+; X86-NEXT: shrdl %cl, %esi, %eax
+; X86-NEXT: movl %eax, %esi
; X86-NEXT: shrl %cl, %ebx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NEXT: shrdl %cl, %edi, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: addl $-1, %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: adcl $-1, %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: adcl $-1, %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: adcl $-1, %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shrdl %cl, %edi, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: addl $-1, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: adcl $-1, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: adcl $-1, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: adcl $-1, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: .p2align 4
; X86-NEXT: .LBB4_6: # %udiv-do-while
; X86-NEXT: # =>This Inner Loop Header: Depth=1
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: shldl $1, %edx, %ebx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %eax, %esi
-; X86-NEXT: shldl $1, %eax, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shldl $1, %eax, %esi
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: shldl $1, %ebx, %eax
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-NEXT: shldl $1, %edi, %ebx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: orl %edx, %ebx
; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shldl $1, %ecx, %edi
-; X86-NEXT: orl %edx, %edi
+; X86-NEXT: movl %esi, %ecx
+; X86-NEXT: shldl $1, %esi, %edi
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: shldl $1, %esi, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: shldl $1, %edi, %ecx
-; X86-NEXT: orl %edx, %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: addl %edi, %edi
-; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X86-NEXT: shldl $1, %edi, %esi
+; X86-NEXT: shldl $1, %edx, %edi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: orl %ebx, %edi
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: cmpl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: sbbl %esi, %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: shldl $1, %eax, %edx
+; X86-NEXT: orl %ebx, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: sbbl %edx, %ecx
-; X86-NEXT: sarl $31, %ecx
-; X86-NEXT: movl %ecx, %edi
-; X86-NEXT: andl $1, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %ecx, %edi
-; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %ecx, %ebx
+; X86-NEXT: shldl $1, %edx, %eax
+; X86-NEXT: orl %ebx, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: addl %edx, %edx
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: cmpl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: sbbl %ecx, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X86-NEXT: sarl $31, %edx
+; X86-NEXT: movl %edx, %eax
+; X86-NEXT: andl $1, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %edx, %eax
+; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NEXT: movl %edx, %ebx
; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X86-NEXT: movl %ecx, %edi
+; X86-NEXT: movl %edx, %edi
; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: subl %ecx, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: sbbl %edi, %esi
-; X86-NEXT: movl %esi, %eax
+; X86-NEXT: subl %edx, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %edi, %ecx
+; X86-NEXT: movl %ecx, %esi
; X86-NEXT: sbbl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: movl %edx, %ebx
-; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: addl $-1, %edx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: adcl $-1, %ecx
-; X86-NEXT: adcl $-1, %esi
+; X86-NEXT: sbbl %eax, %ebx
+; X86-NEXT: addl $-1, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: adcl $-1, %eax
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-NEXT: adcl $-1, %edi
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %edi, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: adcl $-1, %edx
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %esi, %edx
-; X86-NEXT: orl %ecx, %edx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: orl %edx, %eax
+; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %edi, %edx
+; X86-NEXT: orl %eax, %edx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: jne .LBB4_6
; X86-NEXT: .LBB4_7: # %udiv-loop-exit
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shldl $1, %esi, %eax
-; X86-NEXT: shldl $1, %ecx, %esi
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: shldl $1, %edx, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: leal (%edi,%edx,2), %ebx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: movl %eax, %edi
-; X86-NEXT: .LBB4_11: # %udiv-end
-; X86-NEXT: xorl %edx, %edi
-; X86-NEXT: xorl %edx, %esi
-; X86-NEXT: xorl %edx, %ecx
-; X86-NEXT: xorl %edx, %ebx
-; X86-NEXT: subl %edx, %ebx
-; X86-NEXT: sbbl %edx, %ecx
+; X86-NEXT: shldl $1, %edx, %edi
+; X86-NEXT: shldl $1, %eax, %edx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: shldl $1, %ecx, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: leal (%esi,%ecx,2), %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: sbbl %edx, %esi
-; X86-NEXT: sbbl %edx, %edi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: movl %eax, %esi
+; X86-NEXT: .LBB4_8: # %udiv-end
+; X86-NEXT: xorl %ebx, %edi
+; X86-NEXT: xorl %ebx, %edx
+; X86-NEXT: xorl %ebx, %esi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: xorl %ebx, %ecx
+; X86-NEXT: subl %ebx, %ecx
+; X86-NEXT: sbbl %ebx, %esi
+; X86-NEXT: sbbl %ebx, %edx
+; X86-NEXT: sbbl %ebx, %edi
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl 56(%ebp), %eax
-; X86-NEXT: movl %ebx, (%eax)
-; X86-NEXT: movl %ecx, 4(%eax)
-; X86-NEXT: movl %esi, 8(%eax)
+; X86-NEXT: movl %ecx, (%eax)
+; X86-NEXT: movl %ecx, %ebx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %esi, 4(%eax)
+; X86-NEXT: movl %edx, 8(%eax)
; X86-NEXT: movl %edi, 12(%eax)
-; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: movl %esi, %eax
; X86-NEXT: movl 40(%ebp), %ecx
-; X86-NEXT: mull %ecx
; X86-NEXT: movl %edx, %edi
+; X86-NEXT: mull %ecx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %ebx, %eax
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: mull %ecx
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %edx, %ecx
; X86-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X86-NEXT: adcl $0, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: movl %ebx, %eax
-; X86-NEXT: movl %esi, %edi
+; X86-NEXT: movl %esi, %ebx
; X86-NEXT: movl 44(%ebp), %esi
; X86-NEXT: mull %esi
; X86-NEXT: addl %ecx, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
; X86-NEXT: movl %edx, %ecx
-; X86-NEXT: setb %bl
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: movl %esi, %eax
-; X86-NEXT: mull 44(%ebp)
+; X86-NEXT: setb {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
+; X86-NEXT: movl %ebx, %eax
+; X86-NEXT: mull %esi
; X86-NEXT: addl %ecx, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movzbl %bl, %eax
+; X86-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
; X86-NEXT: adcl %eax, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
@@ -508,12 +503,12 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: imull %eax, %ecx
; X86-NEXT: mull %edi
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: imull 44(%ebp), %edi
+; X86-NEXT: imull %esi, %edi
; X86-NEXT: addl %edx, %edi
; X86-NEXT: addl %ecx, %edi
; X86-NEXT: movl 48(%ebp), %eax
; X86-NEXT: movl %eax, %ecx
-; X86-NEXT: imull %esi, %ecx
+; X86-NEXT: imull %ebx, %ecx
; X86-NEXT: movl 52(%ebp), %esi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NEXT: imull %edx, %esi
@@ -544,15 +539,13 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: popl %ebp
; X86-NEXT: retl $4
; X86-NEXT: .LBB4_1:
-; X86-NEXT: movb $1, %cl
-; X86-NEXT: jmp .LBB4_3
+; X86-NEXT: movb $1, %dl
+; X86-NEXT: jmp .LBB4_2
; X86-NEXT: .LBB4_9:
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: jmp .LBB4_7
-; X86-NEXT: .LBB4_4:
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: jmp .LBB4_11
;
; X64-LABEL: scalar_i128:
; X64: # %bb.0:
diff --git a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll
index 488a6d5860c40..81f11a651396d 100644
--- a/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll
+++ b/llvm/test/CodeGen/X86/div-rem-pair-recomposition-unsigned.ll
@@ -209,64 +209,62 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: movl %eax, %esi
; X86-NEXT: orl %edi, %esi
; X86-NEXT: cmovnel %ecx, %edx
-; X86-NEXT: xorl %ecx, %ecx
+; X86-NEXT: xorl %eax, %eax
; X86-NEXT: subl %edx, %ebx
-; X86-NEXT: movl %ebx, %edx
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: sbbl %ecx, %ecx
; X86-NEXT: movl $0, %esi
; X86-NEXT: sbbl %esi, %esi
-; X86-NEXT: movl $0, %ebx
-; X86-NEXT: sbbl %ebx, %ebx
; X86-NEXT: movl $0, %edi
; X86-NEXT: sbbl %edi, %edi
+; X86-NEXT: movl $127, %edx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: cmpl %ebx, %edx
+; X86-NEXT: movl $0, %edx
+; X86-NEXT: sbbl %ecx, %edx
+; X86-NEXT: movl $0, %edx
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: movl $0, %edx
+; X86-NEXT: sbbl %edi, %edx
+; X86-NEXT: setb %dl
; X86-NEXT: cmpb $0, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Reload
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: jne .LBB4_1
-; X86-NEXT: # %bb.2: # %select.false.sink
-; X86-NEXT: movl $127, %eax
-; X86-NEXT: cmpl %edx, %eax
-; X86-NEXT: movl $0, %eax
-; X86-NEXT: sbbl %esi, %eax
-; X86-NEXT: movl $0, %eax
-; X86-NEXT: sbbl %ebx, %eax
-; X86-NEXT: movl $0, %eax
-; X86-NEXT: sbbl %edi, %eax
-; X86-NEXT: setb %al
-; X86-NEXT: .LBB4_3: # %select.end
-; X86-NEXT: testb %al, %al
+; X86-NEXT: .LBB4_2: # %select.end
+; X86-NEXT: testb %dl, %dl
; X86-NEXT: movl 28(%ebp), %edx
-; X86-NEXT: cmovnel %ecx, %edx
+; X86-NEXT: cmovnel %eax, %edx
+; X86-NEXT: movl $0, %edi
; X86-NEXT: movl 24(%ebp), %eax
-; X86-NEXT: cmovnel %ecx, %eax
+; X86-NEXT: cmovnel %edi, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 32(%ebp), %edi
-; X86-NEXT: cmovnel %ecx, %edi
-; X86-NEXT: movl 36(%ebp), %ebx
-; X86-NEXT: cmovnel %ecx, %ebx
-; X86-NEXT: jne .LBB4_9
-; X86-NEXT: # %bb.4: # %select.end
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: movl 32(%ebp), %ebx
+; X86-NEXT: cmovnel %edi, %ebx
+; X86-NEXT: movl 36(%ebp), %esi
; X86-NEXT: movl %esi, %eax
+; X86-NEXT: cmovnel %edi, %eax
+; X86-NEXT: movl %eax, %edi
+; X86-NEXT: jne .LBB4_8
+; X86-NEXT: # %bb.3: # %select.end
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: xorl $127, %eax
; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
; X86-NEXT: orl %eax, %ecx
-; X86-NEXT: movl 28(%ebp), %ecx
-; X86-NEXT: je .LBB4_9
-; X86-NEXT: # %bb.5: # %udiv-bb1
+; X86-NEXT: movl 28(%ebp), %eax
+; X86-NEXT: je .LBB4_8
+; X86-NEXT: # %bb.4: # %udiv-bb1
; X86-NEXT: movl 24(%ebp), %edi
; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X86-NEXT: xorps %xmm0, %xmm0
; X86-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
-; X86-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 32(%ebp), %edx
-; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 36(%ebp), %ecx
-; X86-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; X86-NEXT: movl %esi, %edx
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 32(%ebp), %edi
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NEXT: movl %edx, %ecx
; X86-NEXT: xorb $127, %cl
; X86-NEXT: movl %ecx, %eax
@@ -278,26 +276,26 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: movl 140(%esp,%eax), %ebx
; X86-NEXT: shldl %cl, %esi, %ebx
; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 128(%esp,%eax), %edi
-; X86-NEXT: movl 132(%esp,%eax), %ebx
-; X86-NEXT: shldl %cl, %ebx, %esi
+; X86-NEXT: movl 128(%esp,%eax), %ebx
+; X86-NEXT: movl 132(%esp,%eax), %eax
+; X86-NEXT: shldl %cl, %eax, %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shldl %cl, %edi, %ebx
-; X86-NEXT: shll %cl, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shldl %cl, %ebx, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shll %cl, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: addl $1, %edx
; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: jb .LBB4_10
-; X86-NEXT: # %bb.6: # %udiv-preheader
+; X86-NEXT: jb .LBB4_9
+; X86-NEXT: # %bb.5: # %udiv-preheader
; X86-NEXT: movaps %xmm0, {{[0-9]+}}(%esp)
; X86-NEXT: movl 24(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: movl 28(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 32(%ebp), %eax
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X86-NEXT: movl 36(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: movl %edx, %eax
@@ -305,23 +303,20 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: andb $12, %al
; X86-NEXT: movzbl %al, %eax
; X86-NEXT: movl 92(%esp,%eax), %esi
-; X86-NEXT: movl 88(%esp,%eax), %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 88(%esp,%eax), %ebx
+; X86-NEXT: movl %ebx, %edi
; X86-NEXT: movl %edx, %ecx
; X86-NEXT: shrdl %cl, %esi, %edi
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 80(%esp,%eax), %edx
+; X86-NEXT: movl 80(%esp,%eax), %edi
; X86-NEXT: movl 84(%esp,%eax), %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: shrdl %cl, %edi, %eax
+; X86-NEXT: shrdl %cl, %ebx, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: shrl %cl, %esi
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: # kill: def $cl killed $cl killed $ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shrdl %cl, %eax, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shrdl %cl, %eax, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl 40(%ebp), %ecx
; X86-NEXT: addl $-1, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
@@ -335,13 +330,15 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: adcl $-1, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: movl %edx, %ebx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: .p2align 4
-; X86-NEXT: .LBB4_7: # %udiv-do-while
+; X86-NEXT: .LBB4_6: # %udiv-do-while
; X86-NEXT: # =>This Inner Loop Header: Depth=1
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: shldl $1, %eax, %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
@@ -352,20 +349,21 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: shldl $1, %eax, %edi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NEXT: shldl $1, %esi, %eax
-; X86-NEXT: shldl $1, %ecx, %esi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: shldl $1, %ebx, %esi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NEXT: orl %edx, %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shldl $1, %ebx, %ecx
-; X86-NEXT: orl %edx, %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: shldl $1, %ecx, %ebx
; X86-NEXT: orl %edx, %ebx
; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: addl %ecx, %ecx
-; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: shldl $1, %esi, %ecx
+; X86-NEXT: orl %edx, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: addl %esi, %esi
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: cmpl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: sbbl %edi, %ecx
@@ -396,92 +394,89 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-NEXT: movl %edx, %esi
; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: addl $-1, %edx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: adcl $-1, %ecx
+; X86-NEXT: addl $-1, %ebx
+; X86-NEXT: adcl $-1, %edi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: adcl $-1, %eax
-; X86-NEXT: adcl $-1, %edi
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %edi, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: adcl $-1, %edx
+; X86-NEXT: movl %edi, %ecx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %edx, %ecx
+; X86-NEXT: movl %ebx, %edx
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: orl %eax, %edx
; X86-NEXT: orl %ecx, %edx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: jne .LBB4_7
-; X86-NEXT: .LBB4_8: # %udiv-loop-exit
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shldl $1, %ecx, %eax
-; X86-NEXT: shldl $1, %ebx, %ecx
+; X86-NEXT: jne .LBB4_6
+; X86-NEXT: .LBB4_7: # %udiv-loop-exit
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: shldl $1, %ebx, %edi
+; X86-NEXT: shldl $1, %ecx, %ebx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: shldl $1, %edx, %ebx
+; X86-NEXT: shldl $1, %edx, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NEXT: leal (%esi,%edx,2), %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %ecx, %edi
-; X86-NEXT: movl %ebx, %edx
-; X86-NEXT: movl %eax, %ebx
-; X86-NEXT: .LBB4_9: # %udiv-end
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 56(%ebp), %eax
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: movl %esi, (%eax)
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: .LBB4_8: # %udiv-end
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 56(%ebp), %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %ecx, (%eax)
; X86-NEXT: movl %edx, 4(%eax)
-; X86-NEXT: movl %edi, 8(%eax)
-; X86-NEXT: movl %ebx, 12(%eax)
+; X86-NEXT: movl %ebx, 8(%eax)
+; X86-NEXT: movl %edi, 12(%eax)
; X86-NEXT: movl 48(%ebp), %eax
-; X86-NEXT: movl %eax, %ecx
-; X86-NEXT: imull %edx, %ecx
-; X86-NEXT: mull %esi
+; X86-NEXT: movl %edi, %esi
+; X86-NEXT: movl %eax, %edi
+; X86-NEXT: imull %edx, %edi
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: mull %ecx
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: addl %ecx, %edx
+; X86-NEXT: addl %edi, %edx
; X86-NEXT: movl 52(%ebp), %edi
-; X86-NEXT: imull %esi, %edi
+; X86-NEXT: imull %ecx, %edi
; X86-NEXT: addl %edx, %edi
; X86-NEXT: movl 40(%ebp), %eax
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: mull %esi
+; X86-NEXT: mull %ebx
; X86-NEXT: movl %eax, %ecx
-; X86-NEXT: imull 40(%ebp), %ebx
-; X86-NEXT: addl %edx, %ebx
+; X86-NEXT: imull 40(%ebp), %esi
+; X86-NEXT: addl %edx, %esi
; X86-NEXT: movl 44(%ebp), %eax
-; X86-NEXT: imull %eax, %esi
-; X86-NEXT: addl %ebx, %esi
+; X86-NEXT: imull %eax, %ebx
+; X86-NEXT: addl %esi, %ebx
; X86-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: adcl %edi, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: adcl %edi, %ebx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-NEXT: movl %edi, %eax
; X86-NEXT: movl 40(%ebp), %ecx
; X86-NEXT: mull %ecx
-; X86-NEXT: movl %edx, %esi
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: movl %ebx, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: movl %esi, %eax
; X86-NEXT: mull %ecx
; X86-NEXT: movl %eax, %ecx
-; X86-NEXT: addl %esi, %ecx
+; X86-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
; X86-NEXT: adcl $0, %edx
-; X86-NEXT: movl %edx, %esi
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %edi, %eax
; X86-NEXT: mull 44(%ebp)
; X86-NEXT: movl %edx, %edi
; X86-NEXT: addl %ecx, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: adcl %esi, %edi
+; X86-NEXT: adcl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
; X86-NEXT: setb %cl
-; X86-NEXT: movl %ebx, %eax
+; X86-NEXT: movl %esi, %eax
; X86-NEXT: mull 44(%ebp)
; X86-NEXT: addl %edi, %eax
; X86-NEXT: movzbl %cl, %ecx
; X86-NEXT: adcl %ecx, %edx
; X86-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NEXT: adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X86-NEXT: adcl %ebx, %edx
; X86-NEXT: movl 24(%ebp), %edi
; X86-NEXT: subl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
; X86-NEXT: movl 28(%ebp), %ebx
@@ -502,12 +497,12 @@ define i128 @scalar_i128(i128 %x, i128 %y, ptr %divdst) nounwind {
; X86-NEXT: popl %ebp
; X86-NEXT: retl $4
; X86-NEXT: .LBB4_1:
-; X86-NEXT: movb $1, %al
-; X86-NEXT: jmp .LBB4_3
-; X86-NEXT: .LBB4_10:
+; X86-NEXT: movb $1, %dl
+; X86-NEXT: jmp .LBB4_2
+; X86-NEXT: .LBB4_9:
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: jmp .LBB4_8
+; X86-NEXT: jmp .LBB4_7
;
; X64-LABEL: scalar_i128:
; X64: # %bb.0:
diff --git a/llvm/test/CodeGen/X86/i128-udiv.ll b/llvm/test/CodeGen/X86/i128-udiv.ll
index e11725e426602..6bff99f4bd436 100644
--- a/llvm/test/CodeGen/X86/i128-udiv.ll
+++ b/llvm/test/CodeGen/X86/i128-udiv.ll
@@ -44,99 +44,98 @@ define i128 @test2(i128 %x) nounwind {
; X86-NEXT: pushl %esi
; X86-NEXT: andl $-16, %esp
; X86-NEXT: subl $144, %esp
-; X86-NEXT: movl 32(%ebp), %esi
-; X86-NEXT: movl 36(%ebp), %edi
-; X86-NEXT: movl 28(%ebp), %ecx
-; X86-NEXT: testl %edi, %edi
+; X86-NEXT: movl 32(%ebp), %edi
+; X86-NEXT: movl 36(%ebp), %ebx
+; X86-NEXT: movl 28(%ebp), %eax
+; X86-NEXT: testl %ebx, %ebx
; X86-NEXT: jne .LBB1_1
; X86-NEXT: # %bb.2: # %_udiv-special-cases
-; X86-NEXT: bsrl %esi, %ebx
-; X86-NEXT: xorl $31, %ebx
-; X86-NEXT: orl $32, %ebx
+; X86-NEXT: bsrl %edi, %ecx
+; X86-NEXT: xorl $31, %ecx
+; X86-NEXT: orl $32, %ecx
; X86-NEXT: jmp .LBB1_3
; X86-NEXT: .LBB1_1:
-; X86-NEXT: bsrl %edi, %ebx
-; X86-NEXT: xorl $31, %ebx
+; X86-NEXT: bsrl %ebx, %ecx
+; X86-NEXT: xorl $31, %ecx
; X86-NEXT: .LBB1_3: # %_udiv-special-cases
; X86-NEXT: movl 24(%ebp), %edx
-; X86-NEXT: testl %ecx, %ecx
+; X86-NEXT: testl %eax, %eax
; X86-NEXT: jne .LBB1_4
; X86-NEXT: # %bb.5: # %_udiv-special-cases
-; X86-NEXT: bsrl %edx, %eax
-; X86-NEXT: xorl $31, %eax
-; X86-NEXT: orl $32, %eax
+; X86-NEXT: bsrl %edx, %esi
+; X86-NEXT: xorl $31, %esi
+; X86-NEXT: orl $32, %esi
; X86-NEXT: jmp .LBB1_6
; X86-NEXT: .LBB1_4:
-; X86-NEXT: bsrl %ecx, %eax
-; X86-NEXT: xorl $31, %eax
+; X86-NEXT: bsrl %eax, %esi
+; X86-NEXT: xorl $31, %esi
; X86-NEXT: .LBB1_6: # %_udiv-special-cases
-; X86-NEXT: orl %edi, %ecx
-; X86-NEXT: orl %esi, %edx
-; X86-NEXT: orl %edi, %esi
+; X86-NEXT: movl $0, (%esp) # 4-byte Folded Spill
+; X86-NEXT: orl %ebx, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %edi, %edx
+; X86-NEXT: orl %ebx, %edi
; X86-NEXT: jne .LBB1_8
; X86-NEXT: # %bb.7: # %_udiv-special-cases
-; X86-NEXT: orl $64, %eax
-; X86-NEXT: movl %eax, %ebx
+; X86-NEXT: orl $64, %esi
+; X86-NEXT: movl %esi, %ecx
; X86-NEXT: .LBB1_8: # %_udiv-special-cases
-; X86-NEXT: negl %ebx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $0, %esi
-; X86-NEXT: sbbl %esi, %esi
+; X86-NEXT: negl %ecx
+; X86-NEXT: movl $0, %eax
+; X86-NEXT: sbbl %eax, %eax
; X86-NEXT: movl $0, %ebx
; X86-NEXT: sbbl %ebx, %ebx
; X86-NEXT: movl $0, %edi
; X86-NEXT: sbbl %edi, %edi
-; X86-NEXT: orl %ecx, %edx
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: je .LBB1_9
-; X86-NEXT: # %bb.10: # %select.false.sink
-; X86-NEXT: xorl %eax, %eax
-; X86-NEXT: movl $127, %ecx
-; X86-NEXT: cmpl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X86-NEXT: movl $0, %ecx
-; X86-NEXT: sbbl %esi, %ecx
-; X86-NEXT: movl $0, %ecx
-; X86-NEXT: sbbl %ebx, %ecx
-; X86-NEXT: sbbl %edi, %eax
+; X86-NEXT: movl $127, %esi
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: cmpl %ecx, %esi
+; X86-NEXT: movl $0, %esi
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %eax, %esi
+; X86-NEXT: movl $0, %esi
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %ebx, %esi
+; X86-NEXT: sbbl %edi, (%esp) # 4-byte Folded Spill
; X86-NEXT: setb %cl
-; X86-NEXT: .LBB1_11: # %select.end
-; X86-NEXT: movl %ebx, %esi
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X86-NEXT: je .LBB1_9
+; X86-NEXT: .LBB1_10: # %select.end
; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: movl $0, (%esp) # 4-byte Folded Spill
+; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: testb %cl, %cl
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl $0, %edi
; X86-NEXT: movl $0, %ecx
-; X86-NEXT: jne .LBB1_13
-; X86-NEXT: # %bb.12: # %select.end
-; X86-NEXT: movl 28(%ebp), %ebx
-; X86-NEXT: movl %ebx, (%esp) # 4-byte Spill
+; X86-NEXT: movl $0, (%esp) # 4-byte Folded Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: movl 28(%ebp), %esi
+; X86-NEXT: jne .LBB1_12
+; X86-NEXT: # %bb.11: # %select.end
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl 24(%ebp), %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 32(%ebp), %edi
-; X86-NEXT: movl 36(%ebp), %ecx
-; X86-NEXT: .LBB1_13: # %select.end
+; X86-NEXT: movl 32(%ebp), %ecx
+; X86-NEXT: movl 36(%ebp), %ebx
+; X86-NEXT: movl %ebx, (%esp) # 4-byte Spill
+; X86-NEXT: .LBB1_12: # %select.end
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: jne .LBB1_14
-; X86-NEXT: # %bb.20: # %select.end
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: jne .LBB1_13
+; X86-NEXT: # %bb.19: # %select.end
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl %ecx, %ebx
; X86-NEXT: xorl $127, %ecx
-; X86-NEXT: orl %esi, %ecx
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X86-NEXT: orl %edx, %ecx
+; X86-NEXT: movl %ebx, %edx
+; X86-NEXT: orl %edi, %edx
; X86-NEXT: orl %ecx, %edx
-; X86-NEXT: movl (%esp), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: je .LBB1_21
-; X86-NEXT: # %bb.18: # %udiv-bb1
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: movl (%esp), %edx # 4-byte Reload
+; X86-NEXT: je .LBB1_20
+; X86-NEXT: # %bb.17: # %udiv-bb1
; X86-NEXT: movl 24(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 28(%ebp), %eax
-; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
; X86-NEXT: movl 32(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: movl 36(%ebp), %eax
@@ -145,7 +144,8 @@ define i128 @test2(i128 %x) nounwind {
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
-; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl %eax, %ecx
; X86-NEXT: xorb $127, %cl
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: shrb $3, %al
@@ -153,24 +153,25 @@ define i128 @test2(i128 %x) nounwind {
; X86-NEXT: negb %al
; X86-NEXT: movsbl %al, %eax
; X86-NEXT: movl 120(%esp,%eax), %edx
-; X86-NEXT: movl 124(%esp,%eax), %edi
-; X86-NEXT: shldl %cl, %edx, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 112(%esp,%eax), %edi
+; X86-NEXT: movl 124(%esp,%eax), %esi
+; X86-NEXT: shldl %cl, %edx, %esi
+; X86-NEXT: movl %esi, (%esp) # 4-byte Spill
+; X86-NEXT: movl 112(%esp,%eax), %ebx
; X86-NEXT: movl 116(%esp,%eax), %eax
; X86-NEXT: shldl %cl, %eax, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shldl %cl, %edi, %eax
-; X86-NEXT: movl %eax, (%esp) # 4-byte Spill
-; X86-NEXT: shll %cl, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: addl $1, %ebx
+; X86-NEXT: shldl %cl, %ebx, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shll %cl, %ebx
; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: addl $1, %ecx
; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: jb .LBB1_19
-; X86-NEXT: # %bb.15: # %udiv-preheader
+; X86-NEXT: adcl $0, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: jb .LBB1_18
+; X86-NEXT: # %bb.14: # %udiv-preheader
; X86-NEXT: movl 24(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: movl 28(%ebp), %eax
@@ -183,25 +184,23 @@ define i128 @test2(i128 %x) nounwind {
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: shrb $3, %al
; X86-NEXT: andb $12, %al
-; X86-NEXT: movzbl %al, %esi
-; X86-NEXT: movl 76(%esp,%esi), %eax
-; X86-NEXT: movl 72(%esp,%esi), %edi
-; X86-NEXT: movl %edi, %edx
+; X86-NEXT: movzbl %al, %edi
+; X86-NEXT: movl 76(%esp,%edi), %eax
+; X86-NEXT: movl 72(%esp,%edi), %esi
+; X86-NEXT: movl %esi, %edx
; X86-NEXT: shrdl %cl, %eax, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 64(%esp,%esi), %edx
-; X86-NEXT: movl 68(%esp,%esi), %esi
-; X86-NEXT: movl %esi, %ebx
-; X86-NEXT: shrdl %cl, %edi, %ebx
+; X86-NEXT: movl 64(%esp,%edi), %edx
+; X86-NEXT: movl 68(%esp,%edi), %edi
+; X86-NEXT: movl %edi, %ebx
+; X86-NEXT: shrdl %cl, %esi, %ebx
; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: shrl %cl, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NEXT: shrdl %cl, %esi, %edx
+; X86-NEXT: shrdl %cl, %edi, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $-4, %eax
; X86-NEXT: addl $-1, %eax
@@ -210,38 +209,37 @@ define i128 @test2(i128 %x) nounwind {
; X86-NEXT: adcl $-1, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: movl (%esp), %edx # 4-byte Reload
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: .p2align 4
-; X86-NEXT: .LBB1_16: # %udiv-do-while
+; X86-NEXT: .LBB1_15: # %udiv-do-while
; X86-NEXT: # =>This Inner Loop Header: Depth=1
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: shldl $1, %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: shldl $1, %esi, %ebx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: shldl $1, %ecx, %esi
+; X86-NEXT: shldl $1, %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: shldl $1, %edi, %ecx
+; X86-NEXT: shldl $1, %edi, %esi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: shldl $1, %ecx, %edi
+; X86-NEXT: shldl $1, %edx, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shldl $1, %eax, %edi
+; X86-NEXT: shldl $1, %eax, %edx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: orl %ebx, %edx
+; X86-NEXT: movl %edx, (%esp) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: orl %edx, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl (%esp), %edi # 4-byte Reload
-; X86-NEXT: shldl $1, %edi, %eax
-; X86-NEXT: orl %edx, %eax
+; X86-NEXT: shldl $1, %edx, %eax
+; X86-NEXT: orl %ebx, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shldl $1, %eax, %edi
-; X86-NEXT: orl %edx, %edi
-; X86-NEXT: movl %edi, (%esp) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: shldl $1, %eax, %edx
+; X86-NEXT: orl %ebx, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl (%esp), %edx # 4-byte Reload
; X86-NEXT: addl %eax, %eax
; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
@@ -249,56 +247,59 @@ define i128 @test2(i128 %x) nounwind {
; X86-NEXT: movl $-1, %eax
; X86-NEXT: cmpl %ecx, %eax
; X86-NEXT: movl $-1, %ecx
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %edi, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: sbbl %esi, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NEXT: sbbl %ebx, %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: sbbl %edx, %ecx
; X86-NEXT: sarl $31, %ecx
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: andl $1, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: movl %ecx, %edi
; X86-NEXT: movl $-1, %eax
-; X86-NEXT: andl %eax, %esi
+; X86-NEXT: andl %eax, %edi
; X86-NEXT: movl $-4, %eax
; X86-NEXT: andl %eax, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: subl %ecx, %ebx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %edx, %ecx
-; X86-NEXT: sbbl %esi, %ecx
+; X86-NEXT: subl %ecx, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: sbbl %edi, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: addl $-1, %esi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: addl $-1, %ecx
; X86-NEXT: adcl $-1, %eax
-; X86-NEXT: adcl $-1, %edx
-; X86-NEXT: adcl $-1, %edi
-; X86-NEXT: movl %eax, %ecx
-; X86-NEXT: orl %edi, %ecx
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %edx, %esi
-; X86-NEXT: orl %ecx, %esi
-; X86-NEXT: jne .LBB1_16
-; X86-NEXT: .LBB1_17: # %udiv-loop-exit
+; X86-NEXT: adcl $-1, %ebx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: shldl $1, %edi, %edx
-; X86-NEXT: movl (%esp), %ecx # 4-byte Reload
-; X86-NEXT: shldl $1, %ecx, %edi
+; X86-NEXT: adcl $-1, %edi
+; X86-NEXT: movl %eax, %esi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %edi, %esi
+; X86-NEXT: movl %ecx, %edi
+; X86-NEXT: orl %ebx, %edi
+; X86-NEXT: orl %esi, %edi
+; X86-NEXT: jne .LBB1_15
+; X86-NEXT: .LBB1_16: # %udiv-loop-exit
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: shldl $1, %ecx, %edx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: shldl $1, %ebx, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shldl $1, %eax, %ecx
+; X86-NEXT: shldl $1, %eax, %ebx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: leal (%esi,%eax,2), %esi
+; X86-NEXT: leal (%esi,%eax,2), %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: .LBB1_21: # %udiv-end
+; X86-NEXT: .LBB1_20: # %udiv-end
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NEXT: movl %esi, (%eax)
-; X86-NEXT: movl %ecx, 4(%eax)
-; X86-NEXT: movl %edi, 8(%eax)
+; X86-NEXT: movl %ebx, 4(%eax)
+; X86-NEXT: movl %ecx, 8(%eax)
; X86-NEXT: movl %edx, 12(%eax)
; X86-NEXT: leal -12(%ebp), %esp
; X86-NEXT: popl %esi
@@ -308,15 +309,16 @@ define i128 @test2(i128 %x) nounwind {
; X86-NEXT: retl $4
; X86-NEXT: .LBB1_9:
; X86-NEXT: movb $1, %cl
-; X86-NEXT: jmp .LBB1_11
-; X86-NEXT: .LBB1_19:
+; X86-NEXT: jmp .LBB1_10
+; X86-NEXT: .LBB1_18:
+; X86-NEXT: movl (%esp), %edx # 4-byte Reload
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: jmp .LBB1_17
-; X86-NEXT: .LBB1_14:
-; X86-NEXT: movl (%esp), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: jmp .LBB1_21
+; X86-NEXT: jmp .LBB1_16
+; X86-NEXT: .LBB1_13:
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: movl (%esp), %edx # 4-byte Reload
+; X86-NEXT: jmp .LBB1_20
;
; X64-LABEL: test2:
; X64: # %bb.0:
@@ -360,9 +362,9 @@ define i128 @test3(i128 %x) nounwind {
; X86-NEXT: andl $-16, %esp
; X86-NEXT: subl $160, %esp
; X86-NEXT: movl 32(%ebp), %edi
-; X86-NEXT: movl 36(%ebp), %edx
-; X86-NEXT: movl 28(%ebp), %esi
-; X86-NEXT: testl %edx, %edx
+; X86-NEXT: movl 36(%ebp), %eax
+; X86-NEXT: movl 28(%ebp), %ecx
+; X86-NEXT: testl %eax, %eax
; X86-NEXT: jne .LBB2_1
; X86-NEXT: # %bb.2: # %_udiv-special-cases
; X86-NEXT: bsrl %edi, %ebx
@@ -370,96 +372,95 @@ define i128 @test3(i128 %x) nounwind {
; X86-NEXT: orl $32, %ebx
; X86-NEXT: jmp .LBB2_3
; X86-NEXT: .LBB2_1:
-; X86-NEXT: bsrl %edx, %ebx
+; X86-NEXT: bsrl %eax, %ebx
; X86-NEXT: xorl $31, %ebx
; X86-NEXT: .LBB2_3: # %_udiv-special-cases
-; X86-NEXT: movl 24(%ebp), %eax
-; X86-NEXT: testl %esi, %esi
+; X86-NEXT: movl 24(%ebp), %esi
+; X86-NEXT: testl %ecx, %ecx
; X86-NEXT: jne .LBB2_4
; X86-NEXT: # %bb.5: # %_udiv-special-cases
-; X86-NEXT: bsrl %eax, %ecx
-; X86-NEXT: xorl $31, %ecx
-; X86-NEXT: orl $32, %ecx
+; X86-NEXT: bsrl %esi, %edx
+; X86-NEXT: xorl $31, %edx
+; X86-NEXT: orl $32, %edx
; X86-NEXT: jmp .LBB2_6
; X86-NEXT: .LBB2_4:
-; X86-NEXT: bsrl %esi, %ecx
-; X86-NEXT: xorl $31, %ecx
+; X86-NEXT: bsrl %ecx, %edx
+; X86-NEXT: xorl $31, %edx
; X86-NEXT: .LBB2_6: # %_udiv-special-cases
-; X86-NEXT: orl %edx, %esi
-; X86-NEXT: orl %edi, %eax
-; X86-NEXT: orl %edx, %edi
+; X86-NEXT: orl %eax, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %esi, %ecx
+; X86-NEXT: orl %edi, %ecx
+; X86-NEXT: orl %eax, %edi
; X86-NEXT: jne .LBB2_8
; X86-NEXT: # %bb.7: # %_udiv-special-cases
-; X86-NEXT: orl $64, %ecx
-; X86-NEXT: movl %ecx, %ebx
+; X86-NEXT: orl $64, %edx
+; X86-NEXT: movl %edx, %ebx
; X86-NEXT: .LBB2_8: # %_udiv-special-cases
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: negl %ebx
-; X86-NEXT: movl $0, %ecx
-; X86-NEXT: sbbl %ecx, %ecx
+; X86-NEXT: movl $0, %edi
+; X86-NEXT: sbbl %edi, %edi
+; X86-NEXT: movl $0, %esi
+; X86-NEXT: sbbl %esi, %esi
+; X86-NEXT: movl $0, %eax
+; X86-NEXT: sbbl %eax, %eax
+; X86-NEXT: movl $127, %edx
+; X86-NEXT: cmpl %ebx, %edx
; X86-NEXT: movl $0, %edx
-; X86-NEXT: sbbl %edx, %edx
-; X86-NEXT: movl %ebx, %edi
-; X86-NEXT: movl $0, %ebx
-; X86-NEXT: sbbl %ebx, %ebx
-; X86-NEXT: orl %esi, %eax
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %edi, %edx
+; X86-NEXT: movl $0, %edx
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: movl $0, %edx
+; X86-NEXT: sbbl %eax, %edx
+; X86-NEXT: setb %dl
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
; X86-NEXT: je .LBB2_9
-; X86-NEXT: # %bb.10: # %select.false.sink
-; X86-NEXT: movl $127, %eax
-; X86-NEXT: cmpl %edi, %eax
-; X86-NEXT: movl $0, %eax
-; X86-NEXT: movl %ecx, %edi
-; X86-NEXT: sbbl %ecx, %eax
-; X86-NEXT: movl $0, %eax
-; X86-NEXT: sbbl %edx, %eax
-; X86-NEXT: movl $0, %eax
-; X86-NEXT: sbbl %ebx, %eax
-; X86-NEXT: setb %cl
-; X86-NEXT: .LBB2_11: # %select.end
-; X86-NEXT: movl %ebx, %esi
+; X86-NEXT: .LBB2_10: # %select.end
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: testb %cl, %cl
-; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: testb %dl, %dl
+; X86-NEXT: movl $0, %esi
; X86-NEXT: movl $0, %ecx
-; X86-NEXT: movl $0, %ebx
-; X86-NEXT: jne .LBB2_13
-; X86-NEXT: # %bb.12: # %select.end
+; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: jne .LBB2_12
+; X86-NEXT: # %bb.11: # %select.end
; X86-NEXT: movl 28(%ebp), %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 24(%ebp), %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 24(%ebp), %esi
; X86-NEXT: movl 32(%ebp), %ecx
-; X86-NEXT: movl 36(%ebp), %ebx
-; X86-NEXT: .LBB2_13: # %select.end
+; X86-NEXT: movl 36(%ebp), %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: .LBB2_12: # %select.end
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: jne .LBB2_14
-; X86-NEXT: # %bb.20: # %select.end
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: jne .LBB2_13
+; X86-NEXT: # %bb.19: # %select.end
+; X86-NEXT: movl %ebx, %ecx
; X86-NEXT: xorl $127, %ecx
-; X86-NEXT: orl %edx, %ecx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %esi, %ecx
; X86-NEXT: movl %edi, %edx
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %esi, %edx
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
; X86-NEXT: orl %ecx, %edx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: je .LBB2_21
-; X86-NEXT: # %bb.18: # %udiv-bb1
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: je .LBB2_20
+; X86-NEXT: # %bb.17: # %udiv-bb1
; X86-NEXT: movl 24(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: movl 28(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 32(%ebp), %ebx
-; X86-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 36(%ebp), %ebx
-; X86-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 32(%ebp), %eax
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 36(%ebp), %eax
+; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NEXT: movl %ebx, %ecx
; X86-NEXT: xorb $127, %cl
; X86-NEXT: movl %ecx, %eax
@@ -484,9 +485,8 @@ define i128 @test3(i128 %x) nounwind {
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: jb .LBB2_19
-; X86-NEXT: # %bb.15: # %udiv-preheader
-; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: jb .LBB2_18
+; X86-NEXT: # %bb.14: # %udiv-preheader
; X86-NEXT: movl 24(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: movl 28(%ebp), %eax
@@ -499,133 +499,137 @@ define i128 @test3(i128 %x) nounwind {
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
-; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: movl %ebx, %eax
; X86-NEXT: shrb $3, %al
; X86-NEXT: andb $12, %al
; X86-NEXT: movzbl %al, %edx
; X86-NEXT: movl 92(%esp,%edx), %edi
; X86-NEXT: movl 88(%esp,%edx), %esi
; X86-NEXT: movl %esi, %eax
+; X86-NEXT: movl %ebx, %ecx
; X86-NEXT: shrdl %cl, %edi, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 80(%esp,%edx), %eax
-; X86-NEXT: movl 84(%esp,%edx), %ebx
-; X86-NEXT: movl %ebx, %edx
-; X86-NEXT: shrdl %cl, %esi, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: movl 80(%esp,%edx), %ebx
+; X86-NEXT: movl 84(%esp,%edx), %edx
+; X86-NEXT: movl %edx, %eax
+; X86-NEXT: shrdl %cl, %esi, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: shrl %cl, %edi
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NEXT: shrdl %cl, %ebx, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $-3, %eax
-; X86-NEXT: addl $-1, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $-1, %eax
-; X86-NEXT: adcl $-1, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $-5, %eax
-; X86-NEXT: adcl $-1, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $-1, %eax
-; X86-NEXT: adcl $-1, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shrdl %cl, %edx, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $-3, %ecx
+; X86-NEXT: addl $-1, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $-1, %ecx
+; X86-NEXT: adcl $-1, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $-5, %ecx
+; X86-NEXT: adcl $-1, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $-1, %ecx
+; X86-NEXT: adcl $-1, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: .p2align 4
-; X86-NEXT: .LBB2_16: # %udiv-do-while
+; X86-NEXT: .LBB2_15: # %udiv-do-while
; X86-NEXT: # =>This Inner Loop Header: Depth=1
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: shldl $1, %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: shldl $1, %edi, %eax
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: shldl $1, %edx, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: shldl $1, %ebx, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: shldl $1, %eax, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: shldl $1, %edi, %ebx
+; X86-NEXT: shldl $1, %edi, %eax
; X86-NEXT: shldl $1, %ecx, %edi
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: orl %eax, %edi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: orl %edx, %edi
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: shldl $1, %esi, %ecx
-; X86-NEXT: orl %eax, %ecx
+; X86-NEXT: orl %edx, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shldl $1, %edx, %esi
-; X86-NEXT: orl %eax, %esi
+; X86-NEXT: shldl $1, %ebx, %esi
+; X86-NEXT: orl %edx, %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: addl %edx, %edx
-; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: cmpl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; X86-NEXT: addl %ebx, %ebx
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: cmpl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: sbbl %eax, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: sbbl %edx, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
; X86-NEXT: sarl $31, %ecx
-; X86-NEXT: movl %ecx, %edx
-; X86-NEXT: andl $1, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: andl $1, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: movl %ecx, %edi
; X86-NEXT: movl $-1, %esi
; X86-NEXT: andl %esi, %edi
; X86-NEXT: movl %ecx, %esi
-; X86-NEXT: movl $-5, %edx
-; X86-NEXT: andl %edx, %esi
-; X86-NEXT: movl $-3, %edx
-; X86-NEXT: andl %edx, %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: subl %ecx, %ebx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $-5, %ebx
+; X86-NEXT: andl %ebx, %esi
+; X86-NEXT: movl $-3, %ebx
+; X86-NEXT: andl %ebx, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: subl %ecx, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %edx, %eax
; X86-NEXT: sbbl %edi, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NEXT: sbbl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: sbbl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NEXT: addl $-1, %esi
+; X86-NEXT: adcl $-1, %edx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: adcl $-1, %eax
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: adcl $-1, %ebx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-NEXT: adcl $-1, %edi
-; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: movl %edx, %ecx
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: orl %edi, %ecx
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %ebx, %esi
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %eax, %esi
; X86-NEXT: orl %ecx, %esi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: jne .LBB2_16
-; X86-NEXT: .LBB2_17: # %udiv-loop-exit
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: shldl $1, %ecx, %ebx
+; X86-NEXT: jne .LBB2_15
+; X86-NEXT: .LBB2_16: # %udiv-loop-exit
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: shldl $1, %ecx, %edi
; X86-NEXT: shldl $1, %esi, %ecx
-; X86-NEXT: shldl $1, %edx, %esi
+; X86-NEXT: shldl $1, %ebx, %esi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: leal (%eax,%edx,2), %eax
+; X86-NEXT: leal (%eax,%ebx,2), %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl 8(%ebp), %eax
; X86-NEXT: movl %ecx, %edx
-; X86-NEXT: .LBB2_21: # %udiv-end
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl %ecx, (%eax)
+; X86-NEXT: movl %edi, %ecx
+; X86-NEXT: .LBB2_20: # %udiv-end
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: movl %edi, (%eax)
; X86-NEXT: movl %esi, 4(%eax)
; X86-NEXT: movl %edx, 8(%eax)
-; X86-NEXT: movl %ebx, 12(%eax)
+; X86-NEXT: movl %ecx, 12(%eax)
; X86-NEXT: leal -12(%ebp), %esp
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
@@ -633,19 +637,19 @@ define i128 @test3(i128 %x) nounwind {
; X86-NEXT: popl %ebp
; X86-NEXT: retl $4
; X86-NEXT: .LBB2_9:
-; X86-NEXT: movl %ecx, %edi
-; X86-NEXT: movb $1, %cl
-; X86-NEXT: jmp .LBB2_11
-; X86-NEXT: .LBB2_19:
+; X86-NEXT: movb $1, %dl
+; X86-NEXT: jmp .LBB2_10
+; X86-NEXT: .LBB2_18:
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: jmp .LBB2_17
-; X86-NEXT: .LBB2_14:
+; X86-NEXT: jmp .LBB2_16
+; X86-NEXT: .LBB2_13:
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: jmp .LBB2_21
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: jmp .LBB2_20
;
; X64-LABEL: test3:
; X64: # %bb.0:
@@ -689,97 +693,98 @@ define i128 @div_by_7(i128 %x) nounwind {
; X86-NEXT: pushl %esi
; X86-NEXT: andl $-16, %esp
; X86-NEXT: subl $160, %esp
-; X86-NEXT: movl 32(%ebp), %edi
-; X86-NEXT: movl 36(%ebp), %ebx
-; X86-NEXT: movl 28(%ebp), %edx
-; X86-NEXT: testl %ebx, %ebx
+; X86-NEXT: movl 32(%ebp), %ebx
+; X86-NEXT: movl 36(%ebp), %edx
+; X86-NEXT: movl 28(%ebp), %ecx
+; X86-NEXT: testl %edx, %edx
; X86-NEXT: jne .LBB3_1
; X86-NEXT: # %bb.2: # %entry_udiv-special-cases
-; X86-NEXT: bsrl %edi, %esi
-; X86-NEXT: xorl $31, %esi
-; X86-NEXT: orl $32, %esi
+; X86-NEXT: bsrl %ebx, %eax
+; X86-NEXT: xorl $31, %eax
+; X86-NEXT: orl $32, %eax
; X86-NEXT: jmp .LBB3_3
; X86-NEXT: .LBB3_1:
-; X86-NEXT: bsrl %ebx, %esi
-; X86-NEXT: xorl $31, %esi
+; X86-NEXT: bsrl %edx, %eax
+; X86-NEXT: xorl $31, %eax
; X86-NEXT: .LBB3_3: # %entry_udiv-special-cases
-; X86-NEXT: movl 24(%ebp), %eax
-; X86-NEXT: testl %edx, %edx
+; X86-NEXT: movl 24(%ebp), %esi
+; X86-NEXT: testl %ecx, %ecx
; X86-NEXT: jne .LBB3_4
; X86-NEXT: # %bb.5: # %entry_udiv-special-cases
-; X86-NEXT: bsrl %eax, %ecx
-; X86-NEXT: xorl $31, %ecx
-; X86-NEXT: orl $32, %ecx
+; X86-NEXT: bsrl %esi, %edi
+; X86-NEXT: xorl $31, %edi
+; X86-NEXT: orl $32, %edi
; X86-NEXT: jmp .LBB3_6
; X86-NEXT: .LBB3_4:
-; X86-NEXT: bsrl %edx, %ecx
-; X86-NEXT: xorl $31, %ecx
+; X86-NEXT: bsrl %ecx, %edi
+; X86-NEXT: xorl $31, %edi
; X86-NEXT: .LBB3_6: # %entry_udiv-special-cases
-; X86-NEXT: orl %ebx, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %edi, %eax
-; X86-NEXT: orl %ebx, %edi
+; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: orl %edx, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %ebx, %esi
+; X86-NEXT: orl %edx, %ebx
; X86-NEXT: jne .LBB3_8
; X86-NEXT: # %bb.7: # %entry_udiv-special-cases
-; X86-NEXT: orl $64, %ecx
-; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: orl $64, %edi
+; X86-NEXT: movl %edi, %eax
; X86-NEXT: .LBB3_8: # %entry_udiv-special-cases
; X86-NEXT: movl $125, %ebx
-; X86-NEXT: subl %esi, %ebx
+; X86-NEXT: subl %eax, %ebx
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: sbbl %ecx, %ecx
; X86-NEXT: movl $0, %edx
; X86-NEXT: sbbl %edx, %edx
-; X86-NEXT: movl $0, %esi
-; X86-NEXT: sbbl %esi, %esi
; X86-NEXT: movl $0, %edi
; X86-NEXT: sbbl %edi, %edi
-; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $127, %eax
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: cmpl %ebx, %eax
+; X86-NEXT: movl $0, %eax
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %ecx, %eax
+; X86-NEXT: movl $0, %eax
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %edx, %eax
+; X86-NEXT: sbbl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: setb %dl
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: je .LBB3_9
-; X86-NEXT: # %bb.10: # %select.false.sink
-; X86-NEXT: xorl %eax, %eax
-; X86-NEXT: movl $127, %ecx
-; X86-NEXT: cmpl %ebx, %ecx
-; X86-NEXT: movl $0, %ecx
-; X86-NEXT: sbbl %edx, %ecx
-; X86-NEXT: movl $0, %ecx
-; X86-NEXT: sbbl %esi, %ecx
-; X86-NEXT: sbbl %edi, %eax
-; X86-NEXT: setb %cl
-; X86-NEXT: .LBB3_11: # %select.end
+; X86-NEXT: .LBB3_10: # %select.false
+; X86-NEXT: movl 32(%ebp), %ebx
+; X86-NEXT: movl 24(%ebp), %esi
; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: xorl %edx, %edx
-; X86-NEXT: testb %cl, %cl
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl $0, %ecx
+; X86-NEXT: testb %dl, %dl
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: jne .LBB3_13
-; X86-NEXT: # %bb.12: # %select.end
-; X86-NEXT: movl 28(%ebp), %edx
-; X86-NEXT: movl 24(%ebp), %ecx
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: movl $0, %edx
+; X86-NEXT: jne .LBB3_12
+; X86-NEXT: # %bb.11: # %select.end
+; X86-NEXT: movl 28(%ebp), %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 32(%ebp), %ecx
-; X86-NEXT: movl 36(%ebp), %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: .LBB3_13: # %select.end
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: movl 36(%ebp), %edx
+; X86-NEXT: .LBB3_12: # %select.end
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: jne .LBB3_14
-; X86-NEXT: # %bb.20: # %select.end
-; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: jne .LBB3_13
+; X86-NEXT: # %bb.19: # %select.end
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: xorl $127, %ecx
; X86-NEXT: orl %esi, %ecx
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: movl %esi, %edx
-; X86-NEXT: orl %edi, %edx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: movl %edi, %edx
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
; X86-NEXT: orl %ecx, %edx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: je .LBB3_21
-; X86-NEXT: # %bb.18: # %udiv-bb1
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: je .LBB3_20
+; X86-NEXT: # %bb.17: # %udiv-bb1
; X86-NEXT: movl 24(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: movl 28(%ebp), %eax
@@ -792,7 +797,9 @@ define i128 @div_by_7(i128 %x) nounwind {
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
-; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: movl %eax, %esi
; X86-NEXT: xorb $127, %cl
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: shrb $3, %al
@@ -800,25 +807,24 @@ define i128 @div_by_7(i128 %x) nounwind {
; X86-NEXT: negb %al
; X86-NEXT: movsbl %al, %eax
; X86-NEXT: movl 136(%esp,%eax), %edx
-; X86-NEXT: movl 140(%esp,%eax), %edi
-; X86-NEXT: shldl %cl, %edx, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 128(%esp,%eax), %edi
+; X86-NEXT: movl 140(%esp,%eax), %ebx
+; X86-NEXT: shldl %cl, %edx, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 128(%esp,%eax), %ebx
; X86-NEXT: movl 132(%esp,%eax), %eax
; X86-NEXT: shldl %cl, %eax, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shldl %cl, %edi, %eax
+; X86-NEXT: shldl %cl, %ebx, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shll %cl, %edi
+; X86-NEXT: shll %cl, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: addl $1, %esi
+; X86-NEXT: adcl $0, %edi
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: addl $1, %ebx
-; X86-NEXT: adcl $0, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: jb .LBB3_19
-; X86-NEXT: # %bb.15: # %udiv-preheader
-; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: jb .LBB3_18
+; X86-NEXT: # %bb.14: # %udiv-preheader
; X86-NEXT: movl 24(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: movl 28(%ebp), %eax
@@ -831,123 +837,128 @@ define i128 @div_by_7(i128 %x) nounwind {
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %esi, %ecx
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: shrb $3, %al
; X86-NEXT: andb $12, %al
-; X86-NEXT: movzbl %al, %edi
-; X86-NEXT: movl 92(%esp,%edi), %eax
-; X86-NEXT: movl 88(%esp,%edi), %ebx
-; X86-NEXT: movl %ebx, %esi
-; X86-NEXT: shrdl %cl, %eax, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 80(%esp,%edi), %esi
-; X86-NEXT: movl 84(%esp,%edi), %edi
-; X86-NEXT: movl %edi, %edx
-; X86-NEXT: shrdl %cl, %ebx, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shrl %cl, %eax
+; X86-NEXT: movzbl %al, %esi
+; X86-NEXT: movl 92(%esp,%esi), %ebx
+; X86-NEXT: movl 88(%esp,%esi), %edx
+; X86-NEXT: movl %edx, %edi
+; X86-NEXT: shrdl %cl, %ebx, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 80(%esp,%esi), %edi
+; X86-NEXT: movl 84(%esp,%esi), %esi
+; X86-NEXT: movl %esi, %eax
+; X86-NEXT: shrdl %cl, %edx, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shrl %cl, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NEXT: shrdl %cl, %edi, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $7, %eax
-; X86-NEXT: addl $-1, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $0, %eax
-; X86-NEXT: adcl $-1, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $0, %eax
-; X86-NEXT: adcl $-1, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $0, %eax
-; X86-NEXT: adcl $-1, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shrdl %cl, %esi, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $7, %ecx
+; X86-NEXT: addl $-1, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: adcl $-1, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: adcl $-1, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: adcl $-1, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: .p2align 4
-; X86-NEXT: .LBB3_16: # %udiv-do-while
+; X86-NEXT: .LBB3_15: # %udiv-do-while
; X86-NEXT: # =>This Inner Loop Header: Depth=1
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: shldl $1, %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: shldl $1, %edi, %ebx
+; X86-NEXT: shldl $1, %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NEXT: shldl $1, %edx, %edi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: shldl $1, %eax, %edx
-; X86-NEXT: shldl $1, %ecx, %eax
+; X86-NEXT: shldl $1, %ebx, %eax
+; X86-NEXT: shldl $1, %ecx, %ebx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: orl %esi, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shldl $1, %eax, %ecx
+; X86-NEXT: orl %esi, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: shldl $1, %ebx, %ecx
; X86-NEXT: orl %esi, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: shldl $1, %ecx, %eax
-; X86-NEXT: orl %esi, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shldl $1, %ecx, %ebx
+; X86-NEXT: orl %esi, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: addl %ecx, %ecx
; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: cmpl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; X86-NEXT: cmpl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: sbbl %edi, %ecx
+; X86-NEXT: sbbl %edx, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: sbbl %ebx, %ecx
+; X86-NEXT: sbbl %edi, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: sbbl %esi, %ecx
; X86-NEXT: sarl $31, %ecx
-; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: andl $1, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ecx, %ebx
+; X86-NEXT: andl $1, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl $7, %eax
-; X86-NEXT: andl %eax, %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: subl %ecx, %edx
+; X86-NEXT: movl $7, %ebx
+; X86-NEXT: andl %ebx, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: subl %ecx, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl $0, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: sbbl $0, %edi
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: sbbl $0, %ebx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %esi, %eax
+; X86-NEXT: sbbl $0, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: sbbl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: addl $-1, %ebx
+; X86-NEXT: addl $-1, %esi
; X86-NEXT: adcl $-1, %edi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NEXT: adcl $-1, %edx
-; X86-NEXT: adcl $-1, %esi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: adcl $-1, %eax
; X86-NEXT: movl %edi, %ecx
-; X86-NEXT: orl %esi, %ecx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %eax, %ecx
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %edx, %ebx
-; X86-NEXT: orl %ecx, %ebx
+; X86-NEXT: orl %edx, %esi
+; X86-NEXT: orl %ecx, %esi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: jne .LBB3_16
-; X86-NEXT: .LBB3_17: # %udiv-loop-exit
-; X86-NEXT: shldl $1, %ecx, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: shldl $1, %edx, %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shldl $1, %eax, %edx
+; X86-NEXT: jne .LBB3_15
+; X86-NEXT: .LBB3_16: # %udiv-loop-exit
+; X86-NEXT: shldl $1, %ecx, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: leal (%esi,%eax,2), %edi
+; X86-NEXT: shldl $1, %esi, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: shldl $1, %eax, %esi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: leal (%edx,%eax,2), %ebx
; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: .LBB3_21: # %udiv-end
-; X86-NEXT: movl %edi, (%eax)
-; X86-NEXT: movl %edx, 4(%eax)
-; X86-NEXT: movl %ecx, 8(%eax)
+; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: movl %esi, %ecx
+; X86-NEXT: .LBB3_20: # %udiv-end
+; X86-NEXT: movl %ebx, (%eax)
+; X86-NEXT: movl %ecx, 4(%eax)
+; X86-NEXT: movl %edx, 8(%eax)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: movl %ecx, 12(%eax)
; X86-NEXT: leal -12(%ebp), %esp
@@ -957,18 +968,18 @@ define i128 @div_by_7(i128 %x) nounwind {
; X86-NEXT: popl %ebp
; X86-NEXT: retl $4
; X86-NEXT: .LBB3_9:
-; X86-NEXT: movb $1, %cl
-; X86-NEXT: jmp .LBB3_11
-; X86-NEXT: .LBB3_19:
+; X86-NEXT: movb $1, %dl
+; X86-NEXT: jmp .LBB3_10
+; X86-NEXT: .LBB3_18:
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: jmp .LBB3_17
-; X86-NEXT: .LBB3_14:
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: jmp .LBB3_16
+; X86-NEXT: .LBB3_13:
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: jmp .LBB3_21
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: jmp .LBB3_20
;
; X64-LABEL: div_by_7:
; X64: # %bb.0: # %entry
@@ -1015,97 +1026,98 @@ define i128 @div_by_11(i128 %x) nounwind {
; X86-NEXT: pushl %esi
; X86-NEXT: andl $-16, %esp
; X86-NEXT: subl $160, %esp
-; X86-NEXT: movl 32(%ebp), %edi
-; X86-NEXT: movl 36(%ebp), %ebx
-; X86-NEXT: movl 28(%ebp), %edx
-; X86-NEXT: testl %ebx, %ebx
+; X86-NEXT: movl 32(%ebp), %ebx
+; X86-NEXT: movl 36(%ebp), %edx
+; X86-NEXT: movl 28(%ebp), %ecx
+; X86-NEXT: testl %edx, %edx
; X86-NEXT: jne .LBB4_1
; X86-NEXT: # %bb.2: # %_udiv-special-cases
-; X86-NEXT: bsrl %edi, %esi
-; X86-NEXT: xorl $31, %esi
-; X86-NEXT: orl $32, %esi
+; X86-NEXT: bsrl %ebx, %eax
+; X86-NEXT: xorl $31, %eax
+; X86-NEXT: orl $32, %eax
; X86-NEXT: jmp .LBB4_3
; X86-NEXT: .LBB4_1:
-; X86-NEXT: bsrl %ebx, %esi
-; X86-NEXT: xorl $31, %esi
+; X86-NEXT: bsrl %edx, %eax
+; X86-NEXT: xorl $31, %eax
; X86-NEXT: .LBB4_3: # %_udiv-special-cases
-; X86-NEXT: movl 24(%ebp), %eax
-; X86-NEXT: testl %edx, %edx
+; X86-NEXT: movl 24(%ebp), %esi
+; X86-NEXT: testl %ecx, %ecx
; X86-NEXT: jne .LBB4_4
; X86-NEXT: # %bb.5: # %_udiv-special-cases
-; X86-NEXT: bsrl %eax, %ecx
-; X86-NEXT: xorl $31, %ecx
-; X86-NEXT: orl $32, %ecx
+; X86-NEXT: bsrl %esi, %edi
+; X86-NEXT: xorl $31, %edi
+; X86-NEXT: orl $32, %edi
; X86-NEXT: jmp .LBB4_6
; X86-NEXT: .LBB4_4:
-; X86-NEXT: bsrl %edx, %ecx
-; X86-NEXT: xorl $31, %ecx
+; X86-NEXT: bsrl %ecx, %edi
+; X86-NEXT: xorl $31, %edi
; X86-NEXT: .LBB4_6: # %_udiv-special-cases
-; X86-NEXT: orl %ebx, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %edi, %eax
-; X86-NEXT: orl %ebx, %edi
+; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: orl %edx, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %ebx, %esi
+; X86-NEXT: orl %edx, %ebx
; X86-NEXT: jne .LBB4_8
; X86-NEXT: # %bb.7: # %_udiv-special-cases
-; X86-NEXT: orl $64, %ecx
-; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: orl $64, %edi
+; X86-NEXT: movl %edi, %eax
; X86-NEXT: .LBB4_8: # %_udiv-special-cases
; X86-NEXT: movl $124, %ebx
-; X86-NEXT: subl %esi, %ebx
+; X86-NEXT: subl %eax, %ebx
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: sbbl %ecx, %ecx
; X86-NEXT: movl $0, %edx
; X86-NEXT: sbbl %edx, %edx
-; X86-NEXT: movl $0, %esi
-; X86-NEXT: sbbl %esi, %esi
; X86-NEXT: movl $0, %edi
; X86-NEXT: sbbl %edi, %edi
-; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $127, %eax
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: cmpl %ebx, %eax
+; X86-NEXT: movl $0, %eax
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %ecx, %eax
+; X86-NEXT: movl $0, %eax
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %edx, %eax
+; X86-NEXT: sbbl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: setb %dl
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: je .LBB4_9
-; X86-NEXT: # %bb.10: # %select.false.sink
-; X86-NEXT: xorl %eax, %eax
-; X86-NEXT: movl $127, %ecx
-; X86-NEXT: cmpl %ebx, %ecx
-; X86-NEXT: movl $0, %ecx
-; X86-NEXT: sbbl %edx, %ecx
-; X86-NEXT: movl $0, %ecx
-; X86-NEXT: sbbl %esi, %ecx
-; X86-NEXT: sbbl %edi, %eax
-; X86-NEXT: setb %cl
-; X86-NEXT: .LBB4_11: # %select.end
+; X86-NEXT: .LBB4_10: # %select.false
+; X86-NEXT: movl 32(%ebp), %ebx
+; X86-NEXT: movl 24(%ebp), %esi
; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: xorl %edx, %edx
-; X86-NEXT: testb %cl, %cl
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl $0, %ecx
+; X86-NEXT: testb %dl, %dl
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: jne .LBB4_13
-; X86-NEXT: # %bb.12: # %select.end
-; X86-NEXT: movl 28(%ebp), %edx
-; X86-NEXT: movl 24(%ebp), %ecx
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: movl $0, %edx
+; X86-NEXT: jne .LBB4_12
+; X86-NEXT: # %bb.11: # %select.end
+; X86-NEXT: movl 28(%ebp), %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 32(%ebp), %ecx
-; X86-NEXT: movl 36(%ebp), %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: .LBB4_13: # %select.end
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: movl 36(%ebp), %edx
+; X86-NEXT: .LBB4_12: # %select.end
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: jne .LBB4_14
-; X86-NEXT: # %bb.20: # %select.end
-; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: jne .LBB4_13
+; X86-NEXT: # %bb.19: # %select.end
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: xorl $127, %ecx
; X86-NEXT: orl %esi, %ecx
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: movl %esi, %edx
-; X86-NEXT: orl %edi, %edx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: movl %edi, %edx
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
; X86-NEXT: orl %ecx, %edx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: je .LBB4_21
-; X86-NEXT: # %bb.18: # %udiv-bb1
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: je .LBB4_20
+; X86-NEXT: # %bb.17: # %udiv-bb1
; X86-NEXT: movl 24(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: movl 28(%ebp), %eax
@@ -1118,7 +1130,9 @@ define i128 @div_by_11(i128 %x) nounwind {
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
-; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: movl %eax, %esi
; X86-NEXT: xorb $127, %cl
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: shrb $3, %al
@@ -1126,25 +1140,24 @@ define i128 @div_by_11(i128 %x) nounwind {
; X86-NEXT: negb %al
; X86-NEXT: movsbl %al, %eax
; X86-NEXT: movl 136(%esp,%eax), %edx
-; X86-NEXT: movl 140(%esp,%eax), %edi
-; X86-NEXT: shldl %cl, %edx, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 128(%esp,%eax), %edi
+; X86-NEXT: movl 140(%esp,%eax), %ebx
+; X86-NEXT: shldl %cl, %edx, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 128(%esp,%eax), %ebx
; X86-NEXT: movl 132(%esp,%eax), %eax
; X86-NEXT: shldl %cl, %eax, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shldl %cl, %edi, %eax
+; X86-NEXT: shldl %cl, %ebx, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shll %cl, %edi
+; X86-NEXT: shll %cl, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: addl $1, %esi
+; X86-NEXT: adcl $0, %edi
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: addl $1, %ebx
-; X86-NEXT: adcl $0, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: jb .LBB4_19
-; X86-NEXT: # %bb.15: # %udiv-preheader
-; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: jb .LBB4_18
+; X86-NEXT: # %bb.14: # %udiv-preheader
; X86-NEXT: movl 24(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: movl 28(%ebp), %eax
@@ -1157,123 +1170,128 @@ define i128 @div_by_11(i128 %x) nounwind {
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %esi, %ecx
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: shrb $3, %al
; X86-NEXT: andb $12, %al
-; X86-NEXT: movzbl %al, %edi
-; X86-NEXT: movl 92(%esp,%edi), %eax
-; X86-NEXT: movl 88(%esp,%edi), %ebx
-; X86-NEXT: movl %ebx, %esi
-; X86-NEXT: shrdl %cl, %eax, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 80(%esp,%edi), %esi
-; X86-NEXT: movl 84(%esp,%edi), %edi
-; X86-NEXT: movl %edi, %edx
-; X86-NEXT: shrdl %cl, %ebx, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shrl %cl, %eax
+; X86-NEXT: movzbl %al, %esi
+; X86-NEXT: movl 92(%esp,%esi), %ebx
+; X86-NEXT: movl 88(%esp,%esi), %edx
+; X86-NEXT: movl %edx, %edi
+; X86-NEXT: shrdl %cl, %ebx, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 80(%esp,%esi), %edi
+; X86-NEXT: movl 84(%esp,%esi), %esi
+; X86-NEXT: movl %esi, %eax
+; X86-NEXT: shrdl %cl, %edx, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shrl %cl, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NEXT: shrdl %cl, %edi, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $11, %eax
-; X86-NEXT: addl $-1, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $0, %eax
-; X86-NEXT: adcl $-1, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $0, %eax
-; X86-NEXT: adcl $-1, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $0, %eax
-; X86-NEXT: adcl $-1, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shrdl %cl, %esi, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $11, %ecx
+; X86-NEXT: addl $-1, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: adcl $-1, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: adcl $-1, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: adcl $-1, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: .p2align 4
-; X86-NEXT: .LBB4_16: # %udiv-do-while
+; X86-NEXT: .LBB4_15: # %udiv-do-while
; X86-NEXT: # =>This Inner Loop Header: Depth=1
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: shldl $1, %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: shldl $1, %edi, %ebx
+; X86-NEXT: shldl $1, %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NEXT: shldl $1, %edx, %edi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: shldl $1, %eax, %edx
-; X86-NEXT: shldl $1, %ecx, %eax
+; X86-NEXT: shldl $1, %ebx, %eax
+; X86-NEXT: shldl $1, %ecx, %ebx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: orl %esi, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shldl $1, %eax, %ecx
+; X86-NEXT: orl %esi, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: shldl $1, %ebx, %ecx
; X86-NEXT: orl %esi, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: shldl $1, %ecx, %eax
-; X86-NEXT: orl %esi, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shldl $1, %ecx, %ebx
+; X86-NEXT: orl %esi, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: addl %ecx, %ecx
; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: cmpl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; X86-NEXT: cmpl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: sbbl %edi, %ecx
+; X86-NEXT: sbbl %edx, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: sbbl %ebx, %ecx
+; X86-NEXT: sbbl %edi, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: sbbl %esi, %ecx
; X86-NEXT: sarl $31, %ecx
-; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: andl $1, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ecx, %ebx
+; X86-NEXT: andl $1, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl $11, %eax
-; X86-NEXT: andl %eax, %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: subl %ecx, %edx
+; X86-NEXT: movl $11, %ebx
+; X86-NEXT: andl %ebx, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: subl %ecx, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl $0, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: sbbl $0, %edi
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: sbbl $0, %ebx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %esi, %eax
+; X86-NEXT: sbbl $0, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: sbbl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: addl $-1, %ebx
+; X86-NEXT: addl $-1, %esi
; X86-NEXT: adcl $-1, %edi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NEXT: adcl $-1, %edx
-; X86-NEXT: adcl $-1, %esi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: adcl $-1, %eax
; X86-NEXT: movl %edi, %ecx
-; X86-NEXT: orl %esi, %ecx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %eax, %ecx
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %edx, %ebx
-; X86-NEXT: orl %ecx, %ebx
+; X86-NEXT: orl %edx, %esi
+; X86-NEXT: orl %ecx, %esi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: jne .LBB4_16
-; X86-NEXT: .LBB4_17: # %udiv-loop-exit
-; X86-NEXT: shldl $1, %ecx, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: shldl $1, %edx, %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shldl $1, %eax, %edx
+; X86-NEXT: jne .LBB4_15
+; X86-NEXT: .LBB4_16: # %udiv-loop-exit
+; X86-NEXT: shldl $1, %ecx, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: leal (%esi,%eax,2), %edi
+; X86-NEXT: shldl $1, %esi, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: shldl $1, %eax, %esi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: leal (%edx,%eax,2), %ebx
; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: .LBB4_21: # %udiv-end
-; X86-NEXT: movl %edi, (%eax)
-; X86-NEXT: movl %edx, 4(%eax)
-; X86-NEXT: movl %ecx, 8(%eax)
+; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: movl %esi, %ecx
+; X86-NEXT: .LBB4_20: # %udiv-end
+; X86-NEXT: movl %ebx, (%eax)
+; X86-NEXT: movl %ecx, 4(%eax)
+; X86-NEXT: movl %edx, 8(%eax)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: movl %ecx, 12(%eax)
; X86-NEXT: leal -12(%ebp), %esp
@@ -1283,18 +1301,18 @@ define i128 @div_by_11(i128 %x) nounwind {
; X86-NEXT: popl %ebp
; X86-NEXT: retl $4
; X86-NEXT: .LBB4_9:
-; X86-NEXT: movb $1, %cl
-; X86-NEXT: jmp .LBB4_11
-; X86-NEXT: .LBB4_19:
+; X86-NEXT: movb $1, %dl
+; X86-NEXT: jmp .LBB4_10
+; X86-NEXT: .LBB4_18:
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: jmp .LBB4_17
-; X86-NEXT: .LBB4_14:
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: jmp .LBB4_16
+; X86-NEXT: .LBB4_13:
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: jmp .LBB4_21
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: jmp .LBB4_20
;
; X64-LABEL: div_by_11:
; X64: # %bb.0:
@@ -1339,97 +1357,98 @@ define i128 @div_by_22(i128 %x) nounwind {
; X86-NEXT: pushl %esi
; X86-NEXT: andl $-16, %esp
; X86-NEXT: subl $160, %esp
-; X86-NEXT: movl 32(%ebp), %edi
-; X86-NEXT: movl 36(%ebp), %ebx
-; X86-NEXT: movl 28(%ebp), %edx
-; X86-NEXT: testl %ebx, %ebx
+; X86-NEXT: movl 32(%ebp), %ebx
+; X86-NEXT: movl 36(%ebp), %edx
+; X86-NEXT: movl 28(%ebp), %ecx
+; X86-NEXT: testl %edx, %edx
; X86-NEXT: jne .LBB5_1
; X86-NEXT: # %bb.2: # %entry_udiv-special-cases
-; X86-NEXT: bsrl %edi, %esi
-; X86-NEXT: xorl $31, %esi
-; X86-NEXT: orl $32, %esi
+; X86-NEXT: bsrl %ebx, %eax
+; X86-NEXT: xorl $31, %eax
+; X86-NEXT: orl $32, %eax
; X86-NEXT: jmp .LBB5_3
; X86-NEXT: .LBB5_1:
-; X86-NEXT: bsrl %ebx, %esi
-; X86-NEXT: xorl $31, %esi
+; X86-NEXT: bsrl %edx, %eax
+; X86-NEXT: xorl $31, %eax
; X86-NEXT: .LBB5_3: # %entry_udiv-special-cases
-; X86-NEXT: movl 24(%ebp), %eax
-; X86-NEXT: testl %edx, %edx
+; X86-NEXT: movl 24(%ebp), %esi
+; X86-NEXT: testl %ecx, %ecx
; X86-NEXT: jne .LBB5_4
; X86-NEXT: # %bb.5: # %entry_udiv-special-cases
-; X86-NEXT: bsrl %eax, %ecx
-; X86-NEXT: xorl $31, %ecx
-; X86-NEXT: orl $32, %ecx
+; X86-NEXT: bsrl %esi, %edi
+; X86-NEXT: xorl $31, %edi
+; X86-NEXT: orl $32, %edi
; X86-NEXT: jmp .LBB5_6
; X86-NEXT: .LBB5_4:
-; X86-NEXT: bsrl %edx, %ecx
-; X86-NEXT: xorl $31, %ecx
+; X86-NEXT: bsrl %ecx, %edi
+; X86-NEXT: xorl $31, %edi
; X86-NEXT: .LBB5_6: # %entry_udiv-special-cases
-; X86-NEXT: orl %ebx, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %edi, %eax
-; X86-NEXT: orl %ebx, %edi
+; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: orl %edx, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %ebx, %esi
+; X86-NEXT: orl %edx, %ebx
; X86-NEXT: jne .LBB5_8
; X86-NEXT: # %bb.7: # %entry_udiv-special-cases
-; X86-NEXT: orl $64, %ecx
-; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: orl $64, %edi
+; X86-NEXT: movl %edi, %eax
; X86-NEXT: .LBB5_8: # %entry_udiv-special-cases
; X86-NEXT: movl $123, %ebx
-; X86-NEXT: subl %esi, %ebx
+; X86-NEXT: subl %eax, %ebx
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: sbbl %ecx, %ecx
; X86-NEXT: movl $0, %edx
; X86-NEXT: sbbl %edx, %edx
-; X86-NEXT: movl $0, %esi
-; X86-NEXT: sbbl %esi, %esi
; X86-NEXT: movl $0, %edi
; X86-NEXT: sbbl %edi, %edi
-; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $127, %eax
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: cmpl %ebx, %eax
+; X86-NEXT: movl $0, %eax
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %ecx, %eax
+; X86-NEXT: movl $0, %eax
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %edx, %eax
+; X86-NEXT: sbbl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: setb %dl
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: je .LBB5_9
-; X86-NEXT: # %bb.10: # %select.false.sink
-; X86-NEXT: xorl %eax, %eax
-; X86-NEXT: movl $127, %ecx
-; X86-NEXT: cmpl %ebx, %ecx
-; X86-NEXT: movl $0, %ecx
-; X86-NEXT: sbbl %edx, %ecx
-; X86-NEXT: movl $0, %ecx
-; X86-NEXT: sbbl %esi, %ecx
-; X86-NEXT: sbbl %edi, %eax
-; X86-NEXT: setb %cl
-; X86-NEXT: .LBB5_11: # %select.end
+; X86-NEXT: .LBB5_10: # %select.false
+; X86-NEXT: movl 32(%ebp), %ebx
+; X86-NEXT: movl 24(%ebp), %esi
; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: xorl %edx, %edx
-; X86-NEXT: testb %cl, %cl
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl $0, %ecx
+; X86-NEXT: testb %dl, %dl
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: jne .LBB5_13
-; X86-NEXT: # %bb.12: # %select.end
-; X86-NEXT: movl 28(%ebp), %edx
-; X86-NEXT: movl 24(%ebp), %ecx
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: movl $0, %edx
+; X86-NEXT: jne .LBB5_12
+; X86-NEXT: # %bb.11: # %select.end
+; X86-NEXT: movl 28(%ebp), %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 32(%ebp), %ecx
-; X86-NEXT: movl 36(%ebp), %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: .LBB5_13: # %select.end
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: movl 36(%ebp), %edx
+; X86-NEXT: .LBB5_12: # %select.end
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: jne .LBB5_14
-; X86-NEXT: # %bb.20: # %select.end
-; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: jne .LBB5_13
+; X86-NEXT: # %bb.19: # %select.end
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: xorl $127, %ecx
; X86-NEXT: orl %esi, %ecx
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: movl %esi, %edx
-; X86-NEXT: orl %edi, %edx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: movl %edi, %edx
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
; X86-NEXT: orl %ecx, %edx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: je .LBB5_21
-; X86-NEXT: # %bb.18: # %udiv-bb1
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: je .LBB5_20
+; X86-NEXT: # %bb.17: # %udiv-bb1
; X86-NEXT: movl 24(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: movl 28(%ebp), %eax
@@ -1442,7 +1461,9 @@ define i128 @div_by_22(i128 %x) nounwind {
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
-; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: movl %eax, %esi
; X86-NEXT: xorb $127, %cl
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: shrb $3, %al
@@ -1450,25 +1471,24 @@ define i128 @div_by_22(i128 %x) nounwind {
; X86-NEXT: negb %al
; X86-NEXT: movsbl %al, %eax
; X86-NEXT: movl 136(%esp,%eax), %edx
-; X86-NEXT: movl 140(%esp,%eax), %edi
-; X86-NEXT: shldl %cl, %edx, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 128(%esp,%eax), %edi
+; X86-NEXT: movl 140(%esp,%eax), %ebx
+; X86-NEXT: shldl %cl, %edx, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 128(%esp,%eax), %ebx
; X86-NEXT: movl 132(%esp,%eax), %eax
; X86-NEXT: shldl %cl, %eax, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shldl %cl, %edi, %eax
+; X86-NEXT: shldl %cl, %ebx, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shll %cl, %edi
+; X86-NEXT: shll %cl, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: addl $1, %esi
+; X86-NEXT: adcl $0, %edi
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: addl $1, %ebx
-; X86-NEXT: adcl $0, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: jb .LBB5_19
-; X86-NEXT: # %bb.15: # %udiv-preheader
-; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: jb .LBB5_18
+; X86-NEXT: # %bb.14: # %udiv-preheader
; X86-NEXT: movl 24(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: movl 28(%ebp), %eax
@@ -1481,123 +1501,128 @@ define i128 @div_by_22(i128 %x) nounwind {
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %esi, %ecx
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: shrb $3, %al
; X86-NEXT: andb $12, %al
-; X86-NEXT: movzbl %al, %edi
-; X86-NEXT: movl 92(%esp,%edi), %eax
-; X86-NEXT: movl 88(%esp,%edi), %ebx
-; X86-NEXT: movl %ebx, %esi
-; X86-NEXT: shrdl %cl, %eax, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 80(%esp,%edi), %esi
-; X86-NEXT: movl 84(%esp,%edi), %edi
-; X86-NEXT: movl %edi, %edx
-; X86-NEXT: shrdl %cl, %ebx, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shrl %cl, %eax
+; X86-NEXT: movzbl %al, %esi
+; X86-NEXT: movl 92(%esp,%esi), %ebx
+; X86-NEXT: movl 88(%esp,%esi), %edx
+; X86-NEXT: movl %edx, %edi
+; X86-NEXT: shrdl %cl, %ebx, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 80(%esp,%esi), %edi
+; X86-NEXT: movl 84(%esp,%esi), %esi
+; X86-NEXT: movl %esi, %eax
+; X86-NEXT: shrdl %cl, %edx, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shrl %cl, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NEXT: shrdl %cl, %edi, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $22, %eax
-; X86-NEXT: addl $-1, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $0, %eax
-; X86-NEXT: adcl $-1, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $0, %eax
-; X86-NEXT: adcl $-1, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $0, %eax
-; X86-NEXT: adcl $-1, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shrdl %cl, %esi, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $22, %ecx
+; X86-NEXT: addl $-1, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: adcl $-1, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: adcl $-1, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: adcl $-1, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: .p2align 4
-; X86-NEXT: .LBB5_16: # %udiv-do-while
+; X86-NEXT: .LBB5_15: # %udiv-do-while
; X86-NEXT: # =>This Inner Loop Header: Depth=1
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: shldl $1, %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: shldl $1, %edi, %ebx
+; X86-NEXT: shldl $1, %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NEXT: shldl $1, %edx, %edi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: shldl $1, %eax, %edx
-; X86-NEXT: shldl $1, %ecx, %eax
+; X86-NEXT: shldl $1, %ebx, %eax
+; X86-NEXT: shldl $1, %ecx, %ebx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: orl %esi, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shldl $1, %eax, %ecx
+; X86-NEXT: orl %esi, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: shldl $1, %ebx, %ecx
; X86-NEXT: orl %esi, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: shldl $1, %ecx, %eax
-; X86-NEXT: orl %esi, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shldl $1, %ecx, %ebx
+; X86-NEXT: orl %esi, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: addl %ecx, %ecx
; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: cmpl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; X86-NEXT: cmpl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: sbbl %edi, %ecx
+; X86-NEXT: sbbl %edx, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: sbbl %ebx, %ecx
+; X86-NEXT: sbbl %edi, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: sbbl %esi, %ecx
; X86-NEXT: sarl $31, %ecx
-; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: andl $1, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ecx, %ebx
+; X86-NEXT: andl $1, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl $22, %eax
-; X86-NEXT: andl %eax, %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: subl %ecx, %edx
+; X86-NEXT: movl $22, %ebx
+; X86-NEXT: andl %ebx, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: subl %ecx, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl $0, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: sbbl $0, %edi
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: sbbl $0, %ebx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %esi, %eax
+; X86-NEXT: sbbl $0, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: sbbl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: addl $-1, %ebx
+; X86-NEXT: addl $-1, %esi
; X86-NEXT: adcl $-1, %edi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NEXT: adcl $-1, %edx
-; X86-NEXT: adcl $-1, %esi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: adcl $-1, %eax
; X86-NEXT: movl %edi, %ecx
-; X86-NEXT: orl %esi, %ecx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %eax, %ecx
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %edx, %ebx
-; X86-NEXT: orl %ecx, %ebx
+; X86-NEXT: orl %edx, %esi
+; X86-NEXT: orl %ecx, %esi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: jne .LBB5_16
-; X86-NEXT: .LBB5_17: # %udiv-loop-exit
-; X86-NEXT: shldl $1, %ecx, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: shldl $1, %edx, %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shldl $1, %eax, %edx
+; X86-NEXT: jne .LBB5_15
+; X86-NEXT: .LBB5_16: # %udiv-loop-exit
+; X86-NEXT: shldl $1, %ecx, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: leal (%esi,%eax,2), %edi
+; X86-NEXT: shldl $1, %esi, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: shldl $1, %eax, %esi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: leal (%edx,%eax,2), %ebx
; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: .LBB5_21: # %udiv-end
-; X86-NEXT: movl %edi, (%eax)
-; X86-NEXT: movl %edx, 4(%eax)
-; X86-NEXT: movl %ecx, 8(%eax)
+; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: movl %esi, %ecx
+; X86-NEXT: .LBB5_20: # %udiv-end
+; X86-NEXT: movl %ebx, (%eax)
+; X86-NEXT: movl %ecx, 4(%eax)
+; X86-NEXT: movl %edx, 8(%eax)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: movl %ecx, 12(%eax)
; X86-NEXT: leal -12(%ebp), %esp
@@ -1607,18 +1632,18 @@ define i128 @div_by_22(i128 %x) nounwind {
; X86-NEXT: popl %ebp
; X86-NEXT: retl $4
; X86-NEXT: .LBB5_9:
-; X86-NEXT: movb $1, %cl
-; X86-NEXT: jmp .LBB5_11
-; X86-NEXT: .LBB5_19:
+; X86-NEXT: movb $1, %dl
+; X86-NEXT: jmp .LBB5_10
+; X86-NEXT: .LBB5_18:
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: jmp .LBB5_17
-; X86-NEXT: .LBB5_14:
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: jmp .LBB5_16
+; X86-NEXT: .LBB5_13:
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: jmp .LBB5_21
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: jmp .LBB5_20
;
; X64-LABEL: div_by_22:
; X64: # %bb.0: # %entry
@@ -1665,97 +1690,98 @@ define i128 @div_by_56(i128 %x) nounwind {
; X86-NEXT: pushl %esi
; X86-NEXT: andl $-16, %esp
; X86-NEXT: subl $160, %esp
-; X86-NEXT: movl 32(%ebp), %edi
-; X86-NEXT: movl 36(%ebp), %ebx
-; X86-NEXT: movl 28(%ebp), %edx
-; X86-NEXT: testl %ebx, %ebx
+; X86-NEXT: movl 32(%ebp), %ebx
+; X86-NEXT: movl 36(%ebp), %edx
+; X86-NEXT: movl 28(%ebp), %ecx
+; X86-NEXT: testl %edx, %edx
; X86-NEXT: jne .LBB6_1
; X86-NEXT: # %bb.2: # %_udiv-special-cases
-; X86-NEXT: bsrl %edi, %esi
-; X86-NEXT: xorl $31, %esi
-; X86-NEXT: orl $32, %esi
+; X86-NEXT: bsrl %ebx, %eax
+; X86-NEXT: xorl $31, %eax
+; X86-NEXT: orl $32, %eax
; X86-NEXT: jmp .LBB6_3
; X86-NEXT: .LBB6_1:
-; X86-NEXT: bsrl %ebx, %esi
-; X86-NEXT: xorl $31, %esi
+; X86-NEXT: bsrl %edx, %eax
+; X86-NEXT: xorl $31, %eax
; X86-NEXT: .LBB6_3: # %_udiv-special-cases
-; X86-NEXT: movl 24(%ebp), %eax
-; X86-NEXT: testl %edx, %edx
+; X86-NEXT: movl 24(%ebp), %esi
+; X86-NEXT: testl %ecx, %ecx
; X86-NEXT: jne .LBB6_4
; X86-NEXT: # %bb.5: # %_udiv-special-cases
-; X86-NEXT: bsrl %eax, %ecx
-; X86-NEXT: xorl $31, %ecx
-; X86-NEXT: orl $32, %ecx
+; X86-NEXT: bsrl %esi, %edi
+; X86-NEXT: xorl $31, %edi
+; X86-NEXT: orl $32, %edi
; X86-NEXT: jmp .LBB6_6
; X86-NEXT: .LBB6_4:
-; X86-NEXT: bsrl %edx, %ecx
-; X86-NEXT: xorl $31, %ecx
+; X86-NEXT: bsrl %ecx, %edi
+; X86-NEXT: xorl $31, %edi
; X86-NEXT: .LBB6_6: # %_udiv-special-cases
-; X86-NEXT: orl %ebx, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %edi, %eax
-; X86-NEXT: orl %ebx, %edi
+; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: orl %edx, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %ebx, %esi
+; X86-NEXT: orl %edx, %ebx
; X86-NEXT: jne .LBB6_8
; X86-NEXT: # %bb.7: # %_udiv-special-cases
-; X86-NEXT: orl $64, %ecx
-; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: orl $64, %edi
+; X86-NEXT: movl %edi, %eax
; X86-NEXT: .LBB6_8: # %_udiv-special-cases
; X86-NEXT: movl $122, %ebx
-; X86-NEXT: subl %esi, %ebx
+; X86-NEXT: subl %eax, %ebx
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: sbbl %ecx, %ecx
; X86-NEXT: movl $0, %edx
; X86-NEXT: sbbl %edx, %edx
-; X86-NEXT: movl $0, %esi
-; X86-NEXT: sbbl %esi, %esi
; X86-NEXT: movl $0, %edi
; X86-NEXT: sbbl %edi, %edi
-; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $127, %eax
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: cmpl %ebx, %eax
+; X86-NEXT: movl $0, %eax
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %ecx, %eax
+; X86-NEXT: movl $0, %eax
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %edx, %eax
+; X86-NEXT: sbbl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: setb %dl
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: je .LBB6_9
-; X86-NEXT: # %bb.10: # %select.false.sink
-; X86-NEXT: xorl %eax, %eax
-; X86-NEXT: movl $127, %ecx
-; X86-NEXT: cmpl %ebx, %ecx
-; X86-NEXT: movl $0, %ecx
-; X86-NEXT: sbbl %edx, %ecx
-; X86-NEXT: movl $0, %ecx
-; X86-NEXT: sbbl %esi, %ecx
-; X86-NEXT: sbbl %edi, %eax
-; X86-NEXT: setb %cl
-; X86-NEXT: .LBB6_11: # %select.end
+; X86-NEXT: .LBB6_10: # %select.false
+; X86-NEXT: movl 32(%ebp), %ebx
+; X86-NEXT: movl 24(%ebp), %esi
; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: xorl %edx, %edx
-; X86-NEXT: testb %cl, %cl
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl $0, %ecx
+; X86-NEXT: testb %dl, %dl
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: jne .LBB6_13
-; X86-NEXT: # %bb.12: # %select.end
-; X86-NEXT: movl 28(%ebp), %edx
-; X86-NEXT: movl 24(%ebp), %ecx
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 32(%ebp), %ecx
-; X86-NEXT: movl 36(%ebp), %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: .LBB6_13: # %select.end
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: movl $0, %edx
+; X86-NEXT: jne .LBB6_12
+; X86-NEXT: # %bb.11: # %select.end
+; X86-NEXT: movl 28(%ebp), %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: jne .LBB6_14
-; X86-NEXT: # %bb.20: # %select.end
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: movl 36(%ebp), %edx
+; X86-NEXT: .LBB6_12: # %select.end
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: jne .LBB6_13
+; X86-NEXT: # %bb.19: # %select.end
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: xorl $127, %ecx
; X86-NEXT: orl %esi, %ecx
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: movl %esi, %edx
-; X86-NEXT: orl %edi, %edx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: movl %edi, %edx
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
; X86-NEXT: orl %ecx, %edx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: je .LBB6_21
-; X86-NEXT: # %bb.18: # %udiv-bb1
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: je .LBB6_20
+; X86-NEXT: # %bb.17: # %udiv-bb1
; X86-NEXT: movl 24(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: movl 28(%ebp), %eax
@@ -1768,7 +1794,9 @@ define i128 @div_by_56(i128 %x) nounwind {
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
-; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: movl %eax, %esi
; X86-NEXT: xorb $127, %cl
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: shrb $3, %al
@@ -1776,25 +1804,24 @@ define i128 @div_by_56(i128 %x) nounwind {
; X86-NEXT: negb %al
; X86-NEXT: movsbl %al, %eax
; X86-NEXT: movl 136(%esp,%eax), %edx
-; X86-NEXT: movl 140(%esp,%eax), %edi
-; X86-NEXT: shldl %cl, %edx, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 128(%esp,%eax), %edi
+; X86-NEXT: movl 140(%esp,%eax), %ebx
+; X86-NEXT: shldl %cl, %edx, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 128(%esp,%eax), %ebx
; X86-NEXT: movl 132(%esp,%eax), %eax
; X86-NEXT: shldl %cl, %eax, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shldl %cl, %edi, %eax
+; X86-NEXT: shldl %cl, %ebx, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shll %cl, %edi
+; X86-NEXT: shll %cl, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: addl $1, %esi
+; X86-NEXT: adcl $0, %edi
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: addl $1, %ebx
-; X86-NEXT: adcl $0, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: jb .LBB6_19
-; X86-NEXT: # %bb.15: # %udiv-preheader
-; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: jb .LBB6_18
+; X86-NEXT: # %bb.14: # %udiv-preheader
; X86-NEXT: movl 24(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: movl 28(%ebp), %eax
@@ -1807,123 +1834,128 @@ define i128 @div_by_56(i128 %x) nounwind {
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %esi, %ecx
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: shrb $3, %al
; X86-NEXT: andb $12, %al
-; X86-NEXT: movzbl %al, %edi
-; X86-NEXT: movl 92(%esp,%edi), %eax
-; X86-NEXT: movl 88(%esp,%edi), %ebx
-; X86-NEXT: movl %ebx, %esi
-; X86-NEXT: shrdl %cl, %eax, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 80(%esp,%edi), %esi
-; X86-NEXT: movl 84(%esp,%edi), %edi
-; X86-NEXT: movl %edi, %edx
-; X86-NEXT: shrdl %cl, %ebx, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shrl %cl, %eax
+; X86-NEXT: movzbl %al, %esi
+; X86-NEXT: movl 92(%esp,%esi), %ebx
+; X86-NEXT: movl 88(%esp,%esi), %edx
+; X86-NEXT: movl %edx, %edi
+; X86-NEXT: shrdl %cl, %ebx, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 80(%esp,%esi), %edi
+; X86-NEXT: movl 84(%esp,%esi), %esi
+; X86-NEXT: movl %esi, %eax
+; X86-NEXT: shrdl %cl, %edx, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shrl %cl, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NEXT: shrdl %cl, %edi, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $56, %eax
-; X86-NEXT: addl $-1, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $0, %eax
-; X86-NEXT: adcl $-1, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $0, %eax
-; X86-NEXT: adcl $-1, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $0, %eax
-; X86-NEXT: adcl $-1, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shrdl %cl, %esi, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $56, %ecx
+; X86-NEXT: addl $-1, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: adcl $-1, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: adcl $-1, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: adcl $-1, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: .p2align 4
-; X86-NEXT: .LBB6_16: # %udiv-do-while
+; X86-NEXT: .LBB6_15: # %udiv-do-while
; X86-NEXT: # =>This Inner Loop Header: Depth=1
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: shldl $1, %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: shldl $1, %edi, %ebx
+; X86-NEXT: shldl $1, %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NEXT: shldl $1, %edx, %edi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: shldl $1, %eax, %edx
-; X86-NEXT: shldl $1, %ecx, %eax
+; X86-NEXT: shldl $1, %ebx, %eax
+; X86-NEXT: shldl $1, %ecx, %ebx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: orl %esi, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shldl $1, %eax, %ecx
+; X86-NEXT: orl %esi, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: shldl $1, %ebx, %ecx
; X86-NEXT: orl %esi, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: shldl $1, %ecx, %eax
-; X86-NEXT: orl %esi, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shldl $1, %ecx, %ebx
+; X86-NEXT: orl %esi, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: addl %ecx, %ecx
; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: cmpl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; X86-NEXT: cmpl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: sbbl %edi, %ecx
+; X86-NEXT: sbbl %edx, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: sbbl %ebx, %ecx
+; X86-NEXT: sbbl %edi, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: sbbl %esi, %ecx
; X86-NEXT: sarl $31, %ecx
-; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: andl $1, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ecx, %ebx
+; X86-NEXT: andl $1, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl $56, %eax
-; X86-NEXT: andl %eax, %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: subl %ecx, %edx
+; X86-NEXT: movl $56, %ebx
+; X86-NEXT: andl %ebx, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: subl %ecx, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl $0, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: sbbl $0, %edi
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: sbbl $0, %ebx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %esi, %eax
+; X86-NEXT: sbbl $0, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: sbbl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: addl $-1, %ebx
+; X86-NEXT: addl $-1, %esi
; X86-NEXT: adcl $-1, %edi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NEXT: adcl $-1, %edx
-; X86-NEXT: adcl $-1, %esi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: adcl $-1, %eax
; X86-NEXT: movl %edi, %ecx
-; X86-NEXT: orl %esi, %ecx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %eax, %ecx
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %edx, %ebx
-; X86-NEXT: orl %ecx, %ebx
+; X86-NEXT: orl %edx, %esi
+; X86-NEXT: orl %ecx, %esi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: jne .LBB6_16
-; X86-NEXT: .LBB6_17: # %udiv-loop-exit
-; X86-NEXT: shldl $1, %ecx, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: shldl $1, %edx, %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shldl $1, %eax, %edx
+; X86-NEXT: jne .LBB6_15
+; X86-NEXT: .LBB6_16: # %udiv-loop-exit
+; X86-NEXT: shldl $1, %ecx, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: leal (%esi,%eax,2), %edi
+; X86-NEXT: shldl $1, %esi, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: shldl $1, %eax, %esi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: leal (%edx,%eax,2), %ebx
; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: .LBB6_21: # %udiv-end
-; X86-NEXT: movl %edi, (%eax)
-; X86-NEXT: movl %edx, 4(%eax)
-; X86-NEXT: movl %ecx, 8(%eax)
+; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: movl %esi, %ecx
+; X86-NEXT: .LBB6_20: # %udiv-end
+; X86-NEXT: movl %ebx, (%eax)
+; X86-NEXT: movl %ecx, 4(%eax)
+; X86-NEXT: movl %edx, 8(%eax)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: movl %ecx, 12(%eax)
; X86-NEXT: leal -12(%ebp), %esp
@@ -1933,18 +1965,18 @@ define i128 @div_by_56(i128 %x) nounwind {
; X86-NEXT: popl %ebp
; X86-NEXT: retl $4
; X86-NEXT: .LBB6_9:
-; X86-NEXT: movb $1, %cl
-; X86-NEXT: jmp .LBB6_11
-; X86-NEXT: .LBB6_19:
+; X86-NEXT: movb $1, %dl
+; X86-NEXT: jmp .LBB6_10
+; X86-NEXT: .LBB6_18:
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: jmp .LBB6_17
-; X86-NEXT: .LBB6_14:
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: jmp .LBB6_16
+; X86-NEXT: .LBB6_13:
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: jmp .LBB6_21
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: jmp .LBB6_20
;
; X64-LABEL: div_by_56:
; X64: # %bb.0:
@@ -1991,110 +2023,107 @@ define i128 @rem_by_7(i128 %x) nounwind {
; X86-NEXT: pushl %esi
; X86-NEXT: andl $-16, %esp
; X86-NEXT: subl $160, %esp
-; X86-NEXT: movl 32(%ebp), %edi
+; X86-NEXT: movl 32(%ebp), %edx
; X86-NEXT: movl 36(%ebp), %ebx
-; X86-NEXT: movl 28(%ebp), %eax
+; X86-NEXT: movl 28(%ebp), %ecx
; X86-NEXT: testl %ebx, %ebx
; X86-NEXT: jne .LBB7_1
; X86-NEXT: # %bb.2: # %_udiv-special-cases
-; X86-NEXT: bsrl %edi, %edx
-; X86-NEXT: xorl $31, %edx
-; X86-NEXT: orl $32, %edx
+; X86-NEXT: bsrl %edx, %eax
+; X86-NEXT: xorl $31, %eax
+; X86-NEXT: orl $32, %eax
; X86-NEXT: jmp .LBB7_3
; X86-NEXT: .LBB7_1:
-; X86-NEXT: bsrl %ebx, %edx
-; X86-NEXT: xorl $31, %edx
+; X86-NEXT: bsrl %ebx, %eax
+; X86-NEXT: xorl $31, %eax
; X86-NEXT: .LBB7_3: # %_udiv-special-cases
; X86-NEXT: movl 24(%ebp), %esi
-; X86-NEXT: testl %eax, %eax
+; X86-NEXT: testl %ecx, %ecx
; X86-NEXT: jne .LBB7_4
; X86-NEXT: # %bb.5: # %_udiv-special-cases
-; X86-NEXT: bsrl %esi, %ecx
-; X86-NEXT: xorl $31, %ecx
-; X86-NEXT: orl $32, %ecx
+; X86-NEXT: bsrl %esi, %edi
+; X86-NEXT: xorl $31, %edi
+; X86-NEXT: orl $32, %edi
; X86-NEXT: jmp .LBB7_6
; X86-NEXT: .LBB7_4:
-; X86-NEXT: bsrl %eax, %ecx
-; X86-NEXT: xorl $31, %ecx
+; X86-NEXT: bsrl %ecx, %edi
+; X86-NEXT: xorl $31, %edi
; X86-NEXT: .LBB7_6: # %_udiv-special-cases
-; X86-NEXT: orl %ebx, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %edi, %esi
-; X86-NEXT: orl %ebx, %edi
+; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: orl %ebx, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: orl %edx, %esi
+; X86-NEXT: movl %edx, %ebx
+; X86-NEXT: orl %ecx, %ebx
; X86-NEXT: jne .LBB7_8
; X86-NEXT: # %bb.7: # %_udiv-special-cases
-; X86-NEXT: orl $64, %ecx
-; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: orl $64, %edi
+; X86-NEXT: movl %edi, %eax
; X86-NEXT: .LBB7_8: # %_udiv-special-cases
-; X86-NEXT: movl $125, %ebx
-; X86-NEXT: subl %edx, %ebx
+; X86-NEXT: movl $125, %ecx
+; X86-NEXT: subl %eax, %ecx
+; X86-NEXT: movl $0, %edi
+; X86-NEXT: sbbl %edi, %edi
; X86-NEXT: movl $0, %edx
; X86-NEXT: sbbl %edx, %edx
+; X86-NEXT: movl $0, %ebx
+; X86-NEXT: sbbl %ebx, %ebx
+; X86-NEXT: movl $127, %eax
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: cmpl %ecx, %eax
; X86-NEXT: movl $0, %eax
-; X86-NEXT: sbbl %eax, %eax
-; X86-NEXT: movl $0, %edi
-; X86-NEXT: sbbl %edi, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %edi, %eax
+; X86-NEXT: movl $0, %eax
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %edx, %eax
+; X86-NEXT: sbbl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: setb %al
; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: je .LBB7_9
-; X86-NEXT: # %bb.10: # %select.false.sink
-; X86-NEXT: xorl %esi, %esi
-; X86-NEXT: movl $127, %ecx
-; X86-NEXT: cmpl %ebx, %ecx
-; X86-NEXT: movl $0, %ecx
-; X86-NEXT: sbbl %edx, %ecx
-; X86-NEXT: movl $0, %ecx
-; X86-NEXT: sbbl %eax, %ecx
-; X86-NEXT: sbbl %edi, %esi
-; X86-NEXT: setb %al
-; X86-NEXT: .LBB7_11: # %select.end
-; X86-NEXT: movl 32(%ebp), %edi
-; X86-NEXT: movl %edx, %ebx
-; X86-NEXT: xorl %edx, %edx
-; X86-NEXT: testb %al, %al
+; X86-NEXT: .LBB7_10: # %select.false
+; X86-NEXT: movl 32(%ebp), %ecx
+; X86-NEXT: movl 28(%ebp), %edi
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl $0, %eax
+; X86-NEXT: testb %al, %al
+; X86-NEXT: movl $0, %ebx
+; X86-NEXT: movl $0, %edx
; X86-NEXT: movl $0, %esi
-; X86-NEXT: jne .LBB7_13
-; X86-NEXT: # %bb.12: # %select.end
-; X86-NEXT: movl 28(%ebp), %edx
-; X86-NEXT: movl 24(%ebp), %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %edi, %eax
+; X86-NEXT: jne .LBB7_12
+; X86-NEXT: # %bb.11: # %select.end
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 24(%ebp), %ebx
+; X86-NEXT: movl %ecx, %edx
; X86-NEXT: movl 36(%ebp), %esi
-; X86-NEXT: .LBB7_13: # %select.end
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: .LBB7_12: # %select.end
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: jne .LBB7_14
-; X86-NEXT: # %bb.20: # %select.end
+; X86-NEXT: jne .LBB7_18
+; X86-NEXT: # %bb.13: # %select.end
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: xorl $127, %eax
; X86-NEXT: orl %ecx, %eax
-; X86-NEXT: movl %ebx, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: orl %edi, %ecx
+; X86-NEXT: movl %edi, %ecx
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
; X86-NEXT: orl %eax, %ecx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: je .LBB7_21
-; X86-NEXT: # %bb.18: # %udiv-bb1
+; X86-NEXT: je .LBB7_18
+; X86-NEXT: # %bb.14: # %udiv-bb1
; X86-NEXT: movl 24(%ebp), %ebx
; X86-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 28(%ebp), %esi
-; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 32(%ebp), %eax
+; X86-NEXT: movl 28(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 36(%ebp), %edx
+; X86-NEXT: movl 32(%ebp), %edx
; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 36(%ebp), %esi
+; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl %eax, %ecx
; X86-NEXT: xorb $127, %cl
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: shrb $3, %al
@@ -2105,19 +2134,20 @@ define i128 @rem_by_7(i128 %x) nounwind {
; X86-NEXT: movl 140(%esp,%eax), %edx
; X86-NEXT: shldl %cl, %esi, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 128(%esp,%eax), %edx
+; X86-NEXT: movl 128(%esp,%eax), %ebx
; X86-NEXT: movl 132(%esp,%eax), %eax
; X86-NEXT: shldl %cl, %eax, %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shldl %cl, %edx, %eax
+; X86-NEXT: shldl %cl, %ebx, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shll %cl, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: addl $1, %ebx
-; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: shll %cl, %ebx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: addl $1, %ecx
; X86-NEXT: adcl $0, %edi
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: jb .LBB7_19
; X86-NEXT: # %bb.15: # %udiv-preheader
; X86-NEXT: movl 24(%ebp), %eax
@@ -2132,145 +2162,147 @@ define i128 @rem_by_7(i128 %x) nounwind {
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
-; X86-NEXT: movl %ebx, %ecx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: shrb $3, %al
; X86-NEXT: andb $12, %al
; X86-NEXT: movzbl %al, %eax
; X86-NEXT: movl 92(%esp,%eax), %esi
-; X86-NEXT: movl 88(%esp,%eax), %edx
-; X86-NEXT: movl %edx, %edi
+; X86-NEXT: movl 88(%esp,%eax), %ebx
+; X86-NEXT: movl %ebx, %edi
; X86-NEXT: shrdl %cl, %esi, %edi
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl 80(%esp,%eax), %edi
-; X86-NEXT: movl 84(%esp,%eax), %ebx
-; X86-NEXT: movl %ebx, %eax
-; X86-NEXT: shrdl %cl, %edx, %eax
+; X86-NEXT: movl 84(%esp,%eax), %eax
+; X86-NEXT: movl %eax, %edx
+; X86-NEXT: shrdl %cl, %ebx, %edx
; X86-NEXT: shrl %cl, %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NEXT: shrdl %cl, %ebx, %edi
+; X86-NEXT: shrdl %cl, %eax, %edi
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $7, %edx
-; X86-NEXT: addl $-1, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $0, %edx
-; X86-NEXT: adcl $-1, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $0, %edx
-; X86-NEXT: adcl $-1, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $0, %edx
-; X86-NEXT: adcl $-1, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: xorl %edx, %edx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: movl $7, %eax
+; X86-NEXT: addl $-1, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, %eax
+; X86-NEXT: adcl $-1, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, %eax
+; X86-NEXT: adcl $-1, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, %eax
+; X86-NEXT: adcl $-1, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: movl %ecx, %ebx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: .p2align 4
; X86-NEXT: .LBB7_16: # %udiv-do-while
; X86-NEXT: # =>This Inner Loop Header: Depth=1
; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: shldl $1, %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: movl %edx, %edi
+; X86-NEXT: shldl $1, %edx, %esi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: shldl $1, %edx, %edi
+; X86-NEXT: shldl $1, %ecx, %edx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: shldl $1, %ebx, %esi
-; X86-NEXT: shldl $1, %eax, %ebx
+; X86-NEXT: shldl $1, %ebx, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shldl $1, %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: shldl $1, %ecx, %eax
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: shldl $1, %edi, %ecx
-; X86-NEXT: orl %edx, %ecx
+; X86-NEXT: orl %eax, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: shldl $1, %ecx, %edi
-; X86-NEXT: orl %edx, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: shldl $1, %edi, %ecx
-; X86-NEXT: orl %edx, %ecx
+; X86-NEXT: shldl $1, %ecx, %ebx
+; X86-NEXT: orl %eax, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: shldl $1, %ebx, %ecx
+; X86-NEXT: orl %eax, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: addl %edi, %edi
-; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X86-NEXT: cmpl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: addl %ebx, %ebx
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: cmpl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: sbbl %ebx, %ecx
+; X86-NEXT: sbbl %edi, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: sbbl %esi, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
; X86-NEXT: sarl $31, %ecx
-; X86-NEXT: movl %ecx, %edx
-; X86-NEXT: andl $1, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $7, %edx
-; X86-NEXT: andl %edx, %ecx
-; X86-NEXT: subl %ecx, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: sbbl $0, %eax
-; X86-NEXT: sbbl $0, %ebx
+; X86-NEXT: movl %ecx, %ebx
+; X86-NEXT: andl $1, %ebx
; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: movl $7, %ebx
+; X86-NEXT: andl %ebx, %ecx
+; X86-NEXT: subl %ecx, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl $0, %edi
+; X86-NEXT: movl %edi, %edx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-NEXT: sbbl $0, %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: sbbl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NEXT: addl $-1, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: adcl $-1, %ecx
-; X86-NEXT: adcl $-1, %ebx
+; X86-NEXT: adcl $-1, %edi
+; X86-NEXT: adcl $-1, %eax
; X86-NEXT: adcl $-1, %esi
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %edi, %ecx
; X86-NEXT: orl %esi, %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: orl %ebx, %edx
-; X86-NEXT: orl %ecx, %edx
-; X86-NEXT: movl $0, %edx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: orl %eax, %ebx
+; X86-NEXT: orl %ecx, %ebx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NEXT: jne .LBB7_16
; X86-NEXT: .LBB7_17: # %udiv-loop-exit
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shldl $1, %eax, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: shldl $1, %edx, %eax
-; X86-NEXT: shldl $1, %edi, %edx
+; X86-NEXT: shldl $1, %edx, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: shldl $1, %eax, %edx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: leal (%esi,%edi,2), %ebx
+; X86-NEXT: shldl $1, %esi, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: leal (%edi,%esi,2), %ebx
; X86-NEXT: movl %ecx, %esi
-; X86-NEXT: .LBB7_21: # %udiv-end
+; X86-NEXT: .LBB7_18: # %udiv-end
; X86-NEXT: movl $7, %ecx
; X86-NEXT: imull %ecx, %esi
-; X86-NEXT: movl %edx, %edi
+; X86-NEXT: movl %edx, %eax
; X86-NEXT: mull %ecx
; X86-NEXT: movl %edx, %ecx
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %eax, %edi
; X86-NEXT: movl $7, %eax
; X86-NEXT: mull %ebx
; X86-NEXT: movl %edx, %ebx
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $7, %eax
-; X86-NEXT: mull %edi
+; X86-NEXT: mull {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
; X86-NEXT: addl %ebx, %eax
-; X86-NEXT: adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X86-NEXT: adcl %edi, %edx
; X86-NEXT: adcl %esi, %ecx
-; X86-NEXT: movl 24(%ebp), %ebx
-; X86-NEXT: subl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X86-NEXT: movl 28(%ebp), %esi
-; X86-NEXT: sbbl %eax, %esi
-; X86-NEXT: movl 32(%ebp), %edi
-; X86-NEXT: sbbl %edx, %edi
+; X86-NEXT: movl 24(%ebp), %esi
+; X86-NEXT: subl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NEXT: movl 28(%ebp), %edi
+; X86-NEXT: sbbl %eax, %edi
+; X86-NEXT: movl 32(%ebp), %ebx
+; X86-NEXT: sbbl %edx, %ebx
; X86-NEXT: movl 36(%ebp), %edx
; X86-NEXT: sbbl %ecx, %edx
; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: movl %ebx, (%eax)
-; X86-NEXT: movl %esi, 4(%eax)
-; X86-NEXT: movl %edi, 8(%eax)
+; X86-NEXT: movl %esi, (%eax)
+; X86-NEXT: movl %edi, 4(%eax)
+; X86-NEXT: movl %ebx, 8(%eax)
; X86-NEXT: movl %edx, 12(%eax)
; X86-NEXT: leal -12(%ebp), %esp
; X86-NEXT: popl %esi
@@ -2280,16 +2312,11 @@ define i128 @rem_by_7(i128 %x) nounwind {
; X86-NEXT: retl $4
; X86-NEXT: .LBB7_9:
; X86-NEXT: movb $1, %al
-; X86-NEXT: jmp .LBB7_11
+; X86-NEXT: jmp .LBB7_10
; X86-NEXT: .LBB7_19:
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: jmp .LBB7_17
-; X86-NEXT: .LBB7_14:
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: jmp .LBB7_21
;
; X64-LABEL: rem_by_7:
; X64: # %bb.0:
@@ -2325,110 +2352,107 @@ define i128 @rem_by_14(i128 %x) nounwind {
; X86-NEXT: pushl %esi
; X86-NEXT: andl $-16, %esp
; X86-NEXT: subl $160, %esp
-; X86-NEXT: movl 32(%ebp), %edi
+; X86-NEXT: movl 32(%ebp), %edx
; X86-NEXT: movl 36(%ebp), %ebx
-; X86-NEXT: movl 28(%ebp), %eax
+; X86-NEXT: movl 28(%ebp), %ecx
; X86-NEXT: testl %ebx, %ebx
; X86-NEXT: jne .LBB8_1
; X86-NEXT: # %bb.2: # %_udiv-special-cases
-; X86-NEXT: bsrl %edi, %edx
-; X86-NEXT: xorl $31, %edx
-; X86-NEXT: orl $32, %edx
+; X86-NEXT: bsrl %edx, %eax
+; X86-NEXT: xorl $31, %eax
+; X86-NEXT: orl $32, %eax
; X86-NEXT: jmp .LBB8_3
; X86-NEXT: .LBB8_1:
-; X86-NEXT: bsrl %ebx, %edx
-; X86-NEXT: xorl $31, %edx
+; X86-NEXT: bsrl %ebx, %eax
+; X86-NEXT: xorl $31, %eax
; X86-NEXT: .LBB8_3: # %_udiv-special-cases
; X86-NEXT: movl 24(%ebp), %esi
-; X86-NEXT: testl %eax, %eax
+; X86-NEXT: testl %ecx, %ecx
; X86-NEXT: jne .LBB8_4
; X86-NEXT: # %bb.5: # %_udiv-special-cases
-; X86-NEXT: bsrl %esi, %ecx
-; X86-NEXT: xorl $31, %ecx
-; X86-NEXT: orl $32, %ecx
+; X86-NEXT: bsrl %esi, %edi
+; X86-NEXT: xorl $31, %edi
+; X86-NEXT: orl $32, %edi
; X86-NEXT: jmp .LBB8_6
; X86-NEXT: .LBB8_4:
-; X86-NEXT: bsrl %eax, %ecx
-; X86-NEXT: xorl $31, %ecx
+; X86-NEXT: bsrl %ecx, %edi
+; X86-NEXT: xorl $31, %edi
; X86-NEXT: .LBB8_6: # %_udiv-special-cases
-; X86-NEXT: orl %ebx, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %edi, %esi
-; X86-NEXT: orl %ebx, %edi
+; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: orl %ebx, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: orl %edx, %esi
+; X86-NEXT: movl %edx, %ebx
+; X86-NEXT: orl %ecx, %ebx
; X86-NEXT: jne .LBB8_8
; X86-NEXT: # %bb.7: # %_udiv-special-cases
-; X86-NEXT: orl $64, %ecx
-; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: orl $64, %edi
+; X86-NEXT: movl %edi, %eax
; X86-NEXT: .LBB8_8: # %_udiv-special-cases
-; X86-NEXT: movl $124, %ebx
-; X86-NEXT: subl %edx, %ebx
+; X86-NEXT: movl $124, %ecx
+; X86-NEXT: subl %eax, %ecx
+; X86-NEXT: movl $0, %edi
+; X86-NEXT: sbbl %edi, %edi
; X86-NEXT: movl $0, %edx
; X86-NEXT: sbbl %edx, %edx
+; X86-NEXT: movl $0, %ebx
+; X86-NEXT: sbbl %ebx, %ebx
+; X86-NEXT: movl $127, %eax
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: cmpl %ecx, %eax
; X86-NEXT: movl $0, %eax
-; X86-NEXT: sbbl %eax, %eax
-; X86-NEXT: movl $0, %edi
-; X86-NEXT: sbbl %edi, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %edi, %eax
+; X86-NEXT: movl $0, %eax
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %edx, %eax
+; X86-NEXT: sbbl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: setb %al
; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: je .LBB8_9
-; X86-NEXT: # %bb.10: # %select.false.sink
-; X86-NEXT: xorl %esi, %esi
-; X86-NEXT: movl $127, %ecx
-; X86-NEXT: cmpl %ebx, %ecx
-; X86-NEXT: movl $0, %ecx
-; X86-NEXT: sbbl %edx, %ecx
-; X86-NEXT: movl $0, %ecx
-; X86-NEXT: sbbl %eax, %ecx
-; X86-NEXT: sbbl %edi, %esi
-; X86-NEXT: setb %al
-; X86-NEXT: .LBB8_11: # %select.end
-; X86-NEXT: movl 32(%ebp), %edi
-; X86-NEXT: movl %edx, %ebx
-; X86-NEXT: xorl %edx, %edx
-; X86-NEXT: testb %al, %al
+; X86-NEXT: .LBB8_10: # %select.false
+; X86-NEXT: movl 32(%ebp), %ecx
+; X86-NEXT: movl 28(%ebp), %edi
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl $0, %eax
+; X86-NEXT: testb %al, %al
+; X86-NEXT: movl $0, %ebx
+; X86-NEXT: movl $0, %edx
; X86-NEXT: movl $0, %esi
-; X86-NEXT: jne .LBB8_13
-; X86-NEXT: # %bb.12: # %select.end
-; X86-NEXT: movl 28(%ebp), %edx
-; X86-NEXT: movl 24(%ebp), %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %edi, %eax
+; X86-NEXT: jne .LBB8_12
+; X86-NEXT: # %bb.11: # %select.end
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 24(%ebp), %ebx
+; X86-NEXT: movl %ecx, %edx
; X86-NEXT: movl 36(%ebp), %esi
-; X86-NEXT: .LBB8_13: # %select.end
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: .LBB8_12: # %select.end
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: jne .LBB8_14
-; X86-NEXT: # %bb.20: # %select.end
+; X86-NEXT: jne .LBB8_18
+; X86-NEXT: # %bb.13: # %select.end
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: xorl $127, %eax
; X86-NEXT: orl %ecx, %eax
-; X86-NEXT: movl %ebx, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: orl %edi, %ecx
+; X86-NEXT: movl %edi, %ecx
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
; X86-NEXT: orl %eax, %ecx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: je .LBB8_21
-; X86-NEXT: # %bb.18: # %udiv-bb1
+; X86-NEXT: je .LBB8_18
+; X86-NEXT: # %bb.14: # %udiv-bb1
; X86-NEXT: movl 24(%ebp), %ebx
; X86-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 28(%ebp), %esi
-; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 32(%ebp), %eax
+; X86-NEXT: movl 28(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X86-NEXT: movl 36(%ebp), %edx
+; X86-NEXT: movl 32(%ebp), %edx
; X86-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X86-NEXT: movl 36(%ebp), %esi
+; X86-NEXT: movl %esi, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl %eax, %ecx
; X86-NEXT: xorb $127, %cl
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: shrb $3, %al
@@ -2439,19 +2463,20 @@ define i128 @rem_by_14(i128 %x) nounwind {
; X86-NEXT: movl 140(%esp,%eax), %edx
; X86-NEXT: shldl %cl, %esi, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 128(%esp,%eax), %edx
+; X86-NEXT: movl 128(%esp,%eax), %ebx
; X86-NEXT: movl 132(%esp,%eax), %eax
; X86-NEXT: shldl %cl, %eax, %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shldl %cl, %edx, %eax
+; X86-NEXT: shldl %cl, %ebx, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shll %cl, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: addl $1, %ebx
-; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: shll %cl, %ebx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: addl $1, %ecx
; X86-NEXT: adcl $0, %edi
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: jb .LBB8_19
; X86-NEXT: # %bb.15: # %udiv-preheader
; X86-NEXT: movl 24(%ebp), %eax
@@ -2466,145 +2491,147 @@ define i128 @rem_by_14(i128 %x) nounwind {
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
-; X86-NEXT: movl %ebx, %ecx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: shrb $3, %al
; X86-NEXT: andb $12, %al
; X86-NEXT: movzbl %al, %eax
; X86-NEXT: movl 92(%esp,%eax), %esi
-; X86-NEXT: movl 88(%esp,%eax), %edx
-; X86-NEXT: movl %edx, %edi
+; X86-NEXT: movl 88(%esp,%eax), %ebx
+; X86-NEXT: movl %ebx, %edi
; X86-NEXT: shrdl %cl, %esi, %edi
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl 80(%esp,%eax), %edi
-; X86-NEXT: movl 84(%esp,%eax), %ebx
-; X86-NEXT: movl %ebx, %eax
-; X86-NEXT: shrdl %cl, %edx, %eax
+; X86-NEXT: movl 84(%esp,%eax), %eax
+; X86-NEXT: movl %eax, %edx
+; X86-NEXT: shrdl %cl, %ebx, %edx
; X86-NEXT: shrl %cl, %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NEXT: shrdl %cl, %ebx, %edi
+; X86-NEXT: shrdl %cl, %eax, %edi
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $14, %edx
-; X86-NEXT: addl $-1, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $0, %edx
-; X86-NEXT: adcl $-1, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $0, %edx
-; X86-NEXT: adcl $-1, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $0, %edx
-; X86-NEXT: adcl $-1, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: xorl %edx, %edx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: movl $14, %eax
+; X86-NEXT: addl $-1, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, %eax
+; X86-NEXT: adcl $-1, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, %eax
+; X86-NEXT: adcl $-1, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, %eax
+; X86-NEXT: adcl $-1, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: movl %ecx, %ebx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: .p2align 4
; X86-NEXT: .LBB8_16: # %udiv-do-while
; X86-NEXT: # =>This Inner Loop Header: Depth=1
; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: shldl $1, %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: movl %edx, %edi
+; X86-NEXT: shldl $1, %edx, %esi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: shldl $1, %edx, %edi
+; X86-NEXT: shldl $1, %ecx, %edx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: shldl $1, %ebx, %esi
-; X86-NEXT: shldl $1, %eax, %ebx
+; X86-NEXT: shldl $1, %ebx, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shldl $1, %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: shldl $1, %ecx, %eax
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: shldl $1, %edi, %ecx
-; X86-NEXT: orl %edx, %ecx
+; X86-NEXT: orl %eax, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: shldl $1, %ecx, %edi
-; X86-NEXT: orl %edx, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: shldl $1, %edi, %ecx
-; X86-NEXT: orl %edx, %ecx
+; X86-NEXT: shldl $1, %ecx, %ebx
+; X86-NEXT: orl %eax, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: shldl $1, %ebx, %ecx
+; X86-NEXT: orl %eax, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: addl %edi, %edi
-; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X86-NEXT: cmpl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: addl %ebx, %ebx
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: cmpl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: sbbl %ebx, %ecx
+; X86-NEXT: sbbl %edi, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: sbbl %esi, %ecx
-; X86-NEXT: sarl $31, %ecx
-; X86-NEXT: movl %ecx, %edx
-; X86-NEXT: andl $1, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $14, %edx
-; X86-NEXT: andl %edx, %ecx
-; X86-NEXT: subl %ecx, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: sbbl $0, %eax
-; X86-NEXT: sbbl $0, %ebx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: sarl $31, %ecx
+; X86-NEXT: movl %ecx, %ebx
+; X86-NEXT: andl $1, %ebx
; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: movl $14, %ebx
+; X86-NEXT: andl %ebx, %ecx
+; X86-NEXT: subl %ecx, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl $0, %edi
+; X86-NEXT: movl %edi, %edx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-NEXT: sbbl $0, %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: sbbl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NEXT: addl $-1, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: adcl $-1, %ecx
-; X86-NEXT: adcl $-1, %ebx
+; X86-NEXT: adcl $-1, %edi
+; X86-NEXT: adcl $-1, %eax
; X86-NEXT: adcl $-1, %esi
-; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %edi, %ecx
; X86-NEXT: orl %esi, %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: orl %ebx, %edx
-; X86-NEXT: orl %ecx, %edx
-; X86-NEXT: movl $0, %edx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: orl %eax, %ebx
+; X86-NEXT: orl %ecx, %ebx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NEXT: jne .LBB8_16
; X86-NEXT: .LBB8_17: # %udiv-loop-exit
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shldl $1, %eax, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: shldl $1, %edx, %eax
-; X86-NEXT: shldl $1, %edi, %edx
+; X86-NEXT: shldl $1, %edx, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: shldl $1, %eax, %edx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: leal (%esi,%edi,2), %ebx
+; X86-NEXT: shldl $1, %esi, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: leal (%edi,%esi,2), %ebx
; X86-NEXT: movl %ecx, %esi
-; X86-NEXT: .LBB8_21: # %udiv-end
+; X86-NEXT: .LBB8_18: # %udiv-end
; X86-NEXT: movl $14, %ecx
; X86-NEXT: imull %ecx, %esi
-; X86-NEXT: movl %edx, %edi
+; X86-NEXT: movl %edx, %eax
; X86-NEXT: mull %ecx
; X86-NEXT: movl %edx, %ecx
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %eax, %edi
; X86-NEXT: movl $14, %eax
; X86-NEXT: mull %ebx
; X86-NEXT: movl %edx, %ebx
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $14, %eax
-; X86-NEXT: mull %edi
+; X86-NEXT: mull {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
; X86-NEXT: addl %ebx, %eax
-; X86-NEXT: adcl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X86-NEXT: adcl %edi, %edx
; X86-NEXT: adcl %esi, %ecx
-; X86-NEXT: movl 24(%ebp), %ebx
-; X86-NEXT: subl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X86-NEXT: movl 28(%ebp), %esi
-; X86-NEXT: sbbl %eax, %esi
-; X86-NEXT: movl 32(%ebp), %edi
-; X86-NEXT: sbbl %edx, %edi
+; X86-NEXT: movl 24(%ebp), %esi
+; X86-NEXT: subl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NEXT: movl 28(%ebp), %edi
+; X86-NEXT: sbbl %eax, %edi
+; X86-NEXT: movl 32(%ebp), %ebx
+; X86-NEXT: sbbl %edx, %ebx
; X86-NEXT: movl 36(%ebp), %edx
; X86-NEXT: sbbl %ecx, %edx
; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: movl %ebx, (%eax)
-; X86-NEXT: movl %esi, 4(%eax)
-; X86-NEXT: movl %edi, 8(%eax)
+; X86-NEXT: movl %esi, (%eax)
+; X86-NEXT: movl %edi, 4(%eax)
+; X86-NEXT: movl %ebx, 8(%eax)
; X86-NEXT: movl %edx, 12(%eax)
; X86-NEXT: leal -12(%ebp), %esp
; X86-NEXT: popl %esi
@@ -2614,16 +2641,11 @@ define i128 @rem_by_14(i128 %x) nounwind {
; X86-NEXT: retl $4
; X86-NEXT: .LBB8_9:
; X86-NEXT: movb $1, %al
-; X86-NEXT: jmp .LBB8_11
+; X86-NEXT: jmp .LBB8_10
; X86-NEXT: .LBB8_19:
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: jmp .LBB8_17
-; X86-NEXT: .LBB8_14:
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: jmp .LBB8_21
;
; X64-LABEL: rem_by_14:
; X64: # %bb.0:
@@ -2662,97 +2684,98 @@ define i128 @div_by_67(i128 %x) nounwind {
; X86-NEXT: pushl %esi
; X86-NEXT: andl $-16, %esp
; X86-NEXT: subl $160, %esp
-; X86-NEXT: movl 32(%ebp), %edi
-; X86-NEXT: movl 36(%ebp), %ebx
-; X86-NEXT: movl 28(%ebp), %edx
-; X86-NEXT: testl %ebx, %ebx
+; X86-NEXT: movl 32(%ebp), %ebx
+; X86-NEXT: movl 36(%ebp), %edx
+; X86-NEXT: movl 28(%ebp), %ecx
+; X86-NEXT: testl %edx, %edx
; X86-NEXT: jne .LBB9_1
; X86-NEXT: # %bb.2: # %entry_udiv-special-cases
-; X86-NEXT: bsrl %edi, %esi
-; X86-NEXT: xorl $31, %esi
-; X86-NEXT: orl $32, %esi
+; X86-NEXT: bsrl %ebx, %eax
+; X86-NEXT: xorl $31, %eax
+; X86-NEXT: orl $32, %eax
; X86-NEXT: jmp .LBB9_3
; X86-NEXT: .LBB9_1:
-; X86-NEXT: bsrl %ebx, %esi
-; X86-NEXT: xorl $31, %esi
+; X86-NEXT: bsrl %edx, %eax
+; X86-NEXT: xorl $31, %eax
; X86-NEXT: .LBB9_3: # %entry_udiv-special-cases
-; X86-NEXT: movl 24(%ebp), %eax
-; X86-NEXT: testl %edx, %edx
+; X86-NEXT: movl 24(%ebp), %esi
+; X86-NEXT: testl %ecx, %ecx
; X86-NEXT: jne .LBB9_4
; X86-NEXT: # %bb.5: # %entry_udiv-special-cases
-; X86-NEXT: bsrl %eax, %ecx
-; X86-NEXT: xorl $31, %ecx
-; X86-NEXT: orl $32, %ecx
+; X86-NEXT: bsrl %esi, %edi
+; X86-NEXT: xorl $31, %edi
+; X86-NEXT: orl $32, %edi
; X86-NEXT: jmp .LBB9_6
; X86-NEXT: .LBB9_4:
-; X86-NEXT: bsrl %edx, %ecx
-; X86-NEXT: xorl $31, %ecx
+; X86-NEXT: bsrl %ecx, %edi
+; X86-NEXT: xorl $31, %edi
; X86-NEXT: .LBB9_6: # %entry_udiv-special-cases
-; X86-NEXT: orl %ebx, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %edi, %eax
-; X86-NEXT: orl %ebx, %edi
+; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: orl %edx, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %ebx, %esi
+; X86-NEXT: orl %edx, %ebx
; X86-NEXT: jne .LBB9_8
; X86-NEXT: # %bb.7: # %entry_udiv-special-cases
-; X86-NEXT: orl $64, %ecx
-; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: orl $64, %edi
+; X86-NEXT: movl %edi, %eax
; X86-NEXT: .LBB9_8: # %entry_udiv-special-cases
; X86-NEXT: movl $121, %ebx
-; X86-NEXT: subl %esi, %ebx
+; X86-NEXT: subl %eax, %ebx
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: sbbl %ecx, %ecx
; X86-NEXT: movl $0, %edx
; X86-NEXT: sbbl %edx, %edx
-; X86-NEXT: movl $0, %esi
-; X86-NEXT: sbbl %esi, %esi
; X86-NEXT: movl $0, %edi
; X86-NEXT: sbbl %edi, %edi
-; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $127, %eax
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: cmpl %ebx, %eax
+; X86-NEXT: movl $0, %eax
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %ecx, %eax
+; X86-NEXT: movl $0, %eax
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %edx, %eax
+; X86-NEXT: sbbl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: setb %dl
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: je .LBB9_9
-; X86-NEXT: # %bb.10: # %select.false.sink
-; X86-NEXT: xorl %eax, %eax
-; X86-NEXT: movl $127, %ecx
-; X86-NEXT: cmpl %ebx, %ecx
-; X86-NEXT: movl $0, %ecx
-; X86-NEXT: sbbl %edx, %ecx
-; X86-NEXT: movl $0, %ecx
-; X86-NEXT: sbbl %esi, %ecx
-; X86-NEXT: sbbl %edi, %eax
-; X86-NEXT: setb %cl
-; X86-NEXT: .LBB9_11: # %select.end
+; X86-NEXT: .LBB9_10: # %select.false
+; X86-NEXT: movl 32(%ebp), %ebx
+; X86-NEXT: movl 24(%ebp), %esi
; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: xorl %edx, %edx
-; X86-NEXT: testb %cl, %cl
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl $0, %ecx
+; X86-NEXT: testb %dl, %dl
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: jne .LBB9_13
-; X86-NEXT: # %bb.12: # %select.end
-; X86-NEXT: movl 28(%ebp), %edx
-; X86-NEXT: movl 24(%ebp), %ecx
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: movl $0, %edx
+; X86-NEXT: jne .LBB9_12
+; X86-NEXT: # %bb.11: # %select.end
+; X86-NEXT: movl 28(%ebp), %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 32(%ebp), %ecx
-; X86-NEXT: movl 36(%ebp), %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: .LBB9_13: # %select.end
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: movl 36(%ebp), %edx
+; X86-NEXT: .LBB9_12: # %select.end
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: jne .LBB9_14
-; X86-NEXT: # %bb.20: # %select.end
-; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: jne .LBB9_13
+; X86-NEXT: # %bb.19: # %select.end
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: xorl $127, %ecx
; X86-NEXT: orl %esi, %ecx
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: movl %esi, %edx
-; X86-NEXT: orl %edi, %edx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: movl %edi, %edx
+; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
; X86-NEXT: orl %ecx, %edx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: je .LBB9_21
-; X86-NEXT: # %bb.18: # %udiv-bb1
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: je .LBB9_20
+; X86-NEXT: # %bb.17: # %udiv-bb1
; X86-NEXT: movl 24(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: movl 28(%ebp), %eax
@@ -2765,7 +2788,9 @@ define i128 @div_by_67(i128 %x) nounwind {
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
-; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: movl %eax, %esi
; X86-NEXT: xorb $127, %cl
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: shrb $3, %al
@@ -2773,25 +2798,24 @@ define i128 @div_by_67(i128 %x) nounwind {
; X86-NEXT: negb %al
; X86-NEXT: movsbl %al, %eax
; X86-NEXT: movl 136(%esp,%eax), %edx
-; X86-NEXT: movl 140(%esp,%eax), %edi
-; X86-NEXT: shldl %cl, %edx, %edi
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 128(%esp,%eax), %edi
+; X86-NEXT: movl 140(%esp,%eax), %ebx
+; X86-NEXT: shldl %cl, %edx, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 128(%esp,%eax), %ebx
; X86-NEXT: movl 132(%esp,%eax), %eax
; X86-NEXT: shldl %cl, %eax, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shldl %cl, %edi, %eax
+; X86-NEXT: shldl %cl, %ebx, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shll %cl, %edi
+; X86-NEXT: shll %cl, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: addl $1, %esi
+; X86-NEXT: adcl $0, %edi
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: addl $1, %ebx
-; X86-NEXT: adcl $0, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: adcl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: jb .LBB9_19
-; X86-NEXT: # %bb.15: # %udiv-preheader
-; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: jb .LBB9_18
+; X86-NEXT: # %bb.14: # %udiv-preheader
; X86-NEXT: movl 24(%ebp), %eax
; X86-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X86-NEXT: movl 28(%ebp), %eax
@@ -2804,123 +2828,128 @@ define i128 @div_by_67(i128 %x) nounwind {
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
; X86-NEXT: movl $0, {{[0-9]+}}(%esp)
+; X86-NEXT: movl %esi, %ecx
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: shrb $3, %al
; X86-NEXT: andb $12, %al
-; X86-NEXT: movzbl %al, %edi
-; X86-NEXT: movl 92(%esp,%edi), %eax
-; X86-NEXT: movl 88(%esp,%edi), %ebx
-; X86-NEXT: movl %ebx, %esi
-; X86-NEXT: shrdl %cl, %eax, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 80(%esp,%edi), %esi
-; X86-NEXT: movl 84(%esp,%edi), %edi
-; X86-NEXT: movl %edi, %edx
-; X86-NEXT: shrdl %cl, %ebx, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shrl %cl, %eax
+; X86-NEXT: movzbl %al, %esi
+; X86-NEXT: movl 92(%esp,%esi), %ebx
+; X86-NEXT: movl 88(%esp,%esi), %edx
+; X86-NEXT: movl %edx, %edi
+; X86-NEXT: shrdl %cl, %ebx, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 80(%esp,%esi), %edi
+; X86-NEXT: movl 84(%esp,%esi), %esi
+; X86-NEXT: movl %esi, %eax
+; X86-NEXT: shrdl %cl, %edx, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shrl %cl, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: # kill: def $cl killed $cl killed $ecx
-; X86-NEXT: shrdl %cl, %edi, %esi
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $67, %eax
-; X86-NEXT: addl $-1, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $0, %eax
-; X86-NEXT: adcl $-1, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $0, %eax
-; X86-NEXT: adcl $-1, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl $0, %eax
-; X86-NEXT: adcl $-1, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shrdl %cl, %esi, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $67, %ecx
+; X86-NEXT: addl $-1, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: adcl $-1, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: adcl $-1, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: adcl $-1, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: .p2align 4
-; X86-NEXT: .LBB9_16: # %udiv-do-while
+; X86-NEXT: .LBB9_15: # %udiv-do-while
; X86-NEXT: # =>This Inner Loop Header: Depth=1
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: shldl $1, %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: shldl $1, %edi, %ebx
+; X86-NEXT: shldl $1, %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NEXT: shldl $1, %edx, %edi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: shldl $1, %eax, %edx
-; X86-NEXT: shldl $1, %ecx, %eax
+; X86-NEXT: shldl $1, %ebx, %eax
+; X86-NEXT: shldl $1, %ecx, %ebx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: orl %esi, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shldl $1, %eax, %ecx
+; X86-NEXT: orl %esi, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: shldl $1, %ebx, %ecx
; X86-NEXT: orl %esi, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: shldl $1, %ecx, %eax
-; X86-NEXT: orl %esi, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: shldl $1, %ecx, %ebx
+; X86-NEXT: orl %esi, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: addl %ecx, %ecx
; X86-NEXT: orl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: cmpl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; X86-NEXT: cmpl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: sbbl %edi, %ecx
+; X86-NEXT: sbbl %edx, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: sbbl %ebx, %ecx
+; X86-NEXT: sbbl %edi, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: sbbl %esi, %ecx
; X86-NEXT: sarl $31, %ecx
-; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: andl $1, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ecx, %ebx
+; X86-NEXT: andl $1, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl $67, %eax
-; X86-NEXT: andl %eax, %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: subl %ecx, %edx
+; X86-NEXT: movl $67, %ebx
+; X86-NEXT: andl %ebx, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: subl %ecx, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl $0, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: sbbl $0, %edi
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: sbbl $0, %ebx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %esi, %eax
+; X86-NEXT: sbbl $0, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: sbbl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: addl $-1, %ebx
+; X86-NEXT: addl $-1, %esi
; X86-NEXT: adcl $-1, %edi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NEXT: adcl $-1, %edx
-; X86-NEXT: adcl $-1, %esi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: adcl $-1, %eax
; X86-NEXT: movl %edi, %ecx
-; X86-NEXT: orl %esi, %ecx
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %eax, %ecx
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: orl %edx, %ebx
-; X86-NEXT: orl %ecx, %ebx
+; X86-NEXT: orl %edx, %esi
+; X86-NEXT: orl %ecx, %esi
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: jne .LBB9_16
-; X86-NEXT: .LBB9_17: # %udiv-loop-exit
-; X86-NEXT: shldl $1, %ecx, %eax
-; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: shldl $1, %edx, %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-NEXT: shldl $1, %eax, %edx
+; X86-NEXT: jne .LBB9_15
+; X86-NEXT: .LBB9_16: # %udiv-loop-exit
+; X86-NEXT: shldl $1, %ecx, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: leal (%esi,%eax,2), %edi
+; X86-NEXT: shldl $1, %esi, %ecx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: shldl $1, %eax, %esi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: leal (%edx,%eax,2), %ebx
; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: .LBB9_21: # %udiv-end
-; X86-NEXT: movl %edi, (%eax)
-; X86-NEXT: movl %edx, 4(%eax)
-; X86-NEXT: movl %ecx, 8(%eax)
+; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: movl %esi, %ecx
+; X86-NEXT: .LBB9_20: # %udiv-end
+; X86-NEXT: movl %ebx, (%eax)
+; X86-NEXT: movl %ecx, 4(%eax)
+; X86-NEXT: movl %edx, 8(%eax)
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: movl %ecx, 12(%eax)
; X86-NEXT: leal -12(%ebp), %esp
@@ -2930,18 +2959,18 @@ define i128 @div_by_67(i128 %x) nounwind {
; X86-NEXT: popl %ebp
; X86-NEXT: retl $4
; X86-NEXT: .LBB9_9:
-; X86-NEXT: movb $1, %cl
-; X86-NEXT: jmp .LBB9_11
-; X86-NEXT: .LBB9_19:
+; X86-NEXT: movb $1, %dl
+; X86-NEXT: jmp .LBB9_10
+; X86-NEXT: .LBB9_18:
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: jmp .LBB9_17
-; X86-NEXT: .LBB9_14:
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: jmp .LBB9_16
+; X86-NEXT: .LBB9_13:
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X86-NEXT: jmp .LBB9_21
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: jmp .LBB9_20
;
; X64-LABEL: div_by_67:
; X64: # %bb.0: # %entry
diff --git a/llvm/test/CodeGen/X86/pr38539.ll b/llvm/test/CodeGen/X86/pr38539.ll
index e0e6f6bec4876..a9640081badfe 100644
--- a/llvm/test/CodeGen/X86/pr38539.ll
+++ b/llvm/test/CodeGen/X86/pr38539.ll
@@ -24,8 +24,8 @@ define void @f() nounwind {
; X86-NEXT: andl $-16, %esp
; X86-NEXT: subl $96, %esp
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx
; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
; X86-NEXT: movzbl (%eax), %eax
; X86-NEXT: movzbl (%eax), %ecx
; X86-NEXT: movzbl %al, %eax
@@ -37,93 +37,97 @@ define void @f() nounwind {
; X86-NEXT: sarl $30, %ecx
; X86-NEXT: sarl $31, %eax
; X86-NEXT: xorl %eax, %edx
-; X86-NEXT: xorl %eax, %esi
+; X86-NEXT: xorl %eax, %edi
; X86-NEXT: shrdl $1, %eax, %ecx
-; X86-NEXT: xorl %ecx, %ebx
-; X86-NEXT: subl %ecx, %ebx
-; X86-NEXT: sbbl %eax, %esi
+; X86-NEXT: xorl %ecx, %esi
+; X86-NEXT: subl %ecx, %esi
+; X86-NEXT: sbbl %eax, %edi
; X86-NEXT: sbbl %eax, %edx
; X86-NEXT: movl %edx, %eax
-; X86-NEXT: shldl $30, %esi, %eax
-; X86-NEXT: movl %ebx, %edi
-; X86-NEXT: movl %ebx, %ecx
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shrdl $2, %esi, %ecx
+; X86-NEXT: shldl $30, %edi, %eax
+; X86-NEXT: movl %esi, %ecx
+; X86-NEXT: shrdl $2, %edi, %ecx
; X86-NEXT: testl %eax, %eax
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: jne .LBB0_1
; X86-NEXT: # %bb.2: # %BB_udiv-special-cases
-; X86-NEXT: bsrl %ecx, %esi
-; X86-NEXT: xorl $31, %esi
-; X86-NEXT: orl $32, %esi
+; X86-NEXT: bsrl %ecx, %edi
+; X86-NEXT: xorl $31, %edi
+; X86-NEXT: orl $32, %edi
; X86-NEXT: jmp .LBB0_3
; X86-NEXT: .LBB0_1:
-; X86-NEXT: bsrl %eax, %esi
-; X86-NEXT: xorl $31, %esi
+; X86-NEXT: bsrl %eax, %edi
+; X86-NEXT: xorl $31, %edi
; X86-NEXT: .LBB0_3: # %BB_udiv-special-cases
-; X86-NEXT: movl %edi, %edx
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %esi, %edx
; X86-NEXT: shll $30, %edx
; X86-NEXT: jne .LBB0_4
; X86-NEXT: # %bb.5: # %BB_udiv-special-cases
-; X86-NEXT: movl $64, %edx
-; X86-NEXT: orl %eax, %ecx
-; X86-NEXT: je .LBB0_7
-; X86-NEXT: jmp .LBB0_8
+; X86-NEXT: movl $64, %esi
+; X86-NEXT: jmp .LBB0_6
; X86-NEXT: .LBB0_4:
-; X86-NEXT: bsrl %edx, %edx
-; X86-NEXT: xorl $31, %edx
+; X86-NEXT: bsrl %edx, %esi
+; X86-NEXT: xorl $31, %esi
+; X86-NEXT: .LBB0_6: # %BB_udiv-special-cases
+; X86-NEXT: xorl %edx, %edx
; X86-NEXT: orl %eax, %ecx
; X86-NEXT: jne .LBB0_8
-; X86-NEXT: .LBB0_7: # %BB_udiv-special-cases
-; X86-NEXT: addl $64, %edx
-; X86-NEXT: movl %edx, %esi
+; X86-NEXT: # %bb.7: # %BB_udiv-special-cases
+; X86-NEXT: addl $64, %esi
+; X86-NEXT: movl %esi, %edi
; X86-NEXT: .LBB0_8: # %BB_udiv-special-cases
-; X86-NEXT: movl %esi, %edx
-; X86-NEXT: addl $-66, %edx
+; X86-NEXT: movl %edi, %esi
+; X86-NEXT: addl $-66, %esi
; X86-NEXT: movl $0, %ebx
; X86-NEXT: adcl $-1, %ebx
-; X86-NEXT: movl $0, %esi
-; X86-NEXT: adcl $3, %esi
-; X86-NEXT: andl $3, %esi
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: adcl $3, %ecx
+; X86-NEXT: andl $3, %ecx
+; X86-NEXT: movl $65, %eax
+; X86-NEXT: cmpl %esi, %eax
+; X86-NEXT: movl $0, %eax
+; X86-NEXT: movl %ebx, %edi
+; X86-NEXT: sbbl %ebx, %eax
+; X86-NEXT: movl $0, %eax
+; X86-NEXT: movl %ecx, %ebx
+; X86-NEXT: sbbl %ecx, %eax
+; X86-NEXT: sbbl %edx, %edx
+; X86-NEXT: setb %cl
; X86-NEXT: movb $1, %al
; X86-NEXT: testb %al, %al
; X86-NEXT: jne .LBB0_10
-; X86-NEXT: # %bb.9: # %select.false.sink
-; X86-NEXT: xorl %eax, %eax
-; X86-NEXT: movl $65, %ecx
-; X86-NEXT: cmpl %edx, %ecx
-; X86-NEXT: movl $0, %ecx
-; X86-NEXT: sbbl %ebx, %ecx
-; X86-NEXT: movl $0, %ecx
-; X86-NEXT: sbbl %esi, %ecx
-; X86-NEXT: sbbl %eax, %eax
-; X86-NEXT: setb %al
+; X86-NEXT: # %bb.9: # %select.false
+; X86-NEXT: movl %ecx, %eax
; X86-NEXT: .LBB0_10: # %select.end
; X86-NEXT: testb %al, %al
; X86-NEXT: jne .LBB0_15
; X86-NEXT: # %bb.11: # %select.end
-; X86-NEXT: movl %edx, %eax
+; X86-NEXT: movl %esi, %eax
; X86-NEXT: xorl $65, %eax
-; X86-NEXT: orl %esi, %eax
; X86-NEXT: orl %ebx, %eax
+; X86-NEXT: orl %edi, %eax
; X86-NEXT: je .LBB0_15
; X86-NEXT: # %bb.12: # %udiv-bb1
-; X86-NEXT: addl $1, %edx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: adcl $0, %ebx
-; X86-NEXT: adcl $0, %esi
-; X86-NEXT: andl $3, %esi
+; X86-NEXT: addl $1, %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %edx, %eax
-; X86-NEXT: orl %esi, %eax
+; X86-NEXT: movl %edi, %ecx
+; X86-NEXT: adcl $0, %ecx
+; X86-NEXT: adcl $0, %ebx
+; X86-NEXT: andl $3, %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %esi, %eax
; X86-NEXT: orl %ebx, %eax
+; X86-NEXT: orl %ecx, %eax
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: je .LBB0_15
; X86-NEXT: # %bb.13: # %udiv-preheader
+; X86-NEXT: movl %ecx, %edi
; X86-NEXT: andl $3, %eax
; X86-NEXT: xorl %ecx, %ecx
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl %edi, %edx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NEXT: addl $-1, %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
@@ -134,20 +138,19 @@ define void @f() nounwind {
; X86-NEXT: andl $3, %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: xorl %esi, %esi
-; X86-NEXT: xorl %eax, %eax
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: xorl %eax, %eax
+; X86-NEXT: xorl %ebx, %ebx
; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: .p2align 4
; X86-NEXT: .LBB0_14: # %udiv-do-while
; X86-NEXT: # =>This Inner Loop Header: Depth=1
-; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: shldl $1, %esi, %eax
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: shldl $1, %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: shldl $1, %edx, %eax
+; X86-NEXT: shldl $1, %esi, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %ecx, %edx
; X86-NEXT: andl $2, %edx
; X86-NEXT: shrl %edx
@@ -156,7 +159,6 @@ define void @f() nounwind {
; X86-NEXT: shldl $1, %edi, %ecx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NEXT: orl %edx, %ecx
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
; X86-NEXT: shldl $1, %ebx, %edi
; X86-NEXT: orl %edx, %edi
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
@@ -167,37 +169,40 @@ define void @f() nounwind {
; X86-NEXT: cmpl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: sbbl %eax, %edx
-; X86-NEXT: shll $30, %edx
-; X86-NEXT: movl %edx, %ebx
-; X86-NEXT: sarl $31, %ebx
-; X86-NEXT: sarl $30, %edx
-; X86-NEXT: shrdl $1, %ebx, %edx
-; X86-NEXT: movl %edx, %edi
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: sbbl %eax, %ebx
+; X86-NEXT: shll $30, %ebx
+; X86-NEXT: movl %ebx, %edx
+; X86-NEXT: sarl $31, %edx
+; X86-NEXT: sarl $30, %ebx
+; X86-NEXT: shrdl $1, %edx, %ebx
+; X86-NEXT: movl %ebx, %edi
; X86-NEXT: andl $1, %edi
; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X86-NEXT: movl %ebx, %edi
-; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X86-NEXT: subl %edx, %esi
+; X86-NEXT: movl %edx, %edi
+; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X86-NEXT: andl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X86-NEXT: subl %ebx, %esi
; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X86-NEXT: sbbl %ebx, %esi
+; X86-NEXT: sbbl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: sbbl %edi, %eax
-; X86-NEXT: andl $3, %eax
-; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X86-NEXT: addl $-1, %edx
-; X86-NEXT: adcl $-1, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: adcl $3, %ebx
-; X86-NEXT: andl $3, %ebx
-; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: andl $3, %eax
+; X86-NEXT: addl $-1, %esi
+; X86-NEXT: adcl $-1, %ebx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: adcl $3, %edi
+; X86-NEXT: andl $3, %edi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %esi, %edx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: orl %edi, %edx
+; X86-NEXT: movl %ebx, %edi
; X86-NEXT: orl %ebx, %edx
; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X86-NEXT: orl %ebx, %edx
; X86-NEXT: jne .LBB0_14
; X86-NEXT: .LBB0_15: # %udiv-end
; X86-NEXT: cmpb $0, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Reload
diff --git a/llvm/test/CodeGen/X86/pseudo_cmov_lower.ll b/llvm/test/CodeGen/X86/pseudo_cmov_lower.ll
index 7d7d72e4c89f6..e2696c3d407ef 100644
--- a/llvm/test/CodeGen/X86/pseudo_cmov_lower.ll
+++ b/llvm/test/CodeGen/X86/pseudo_cmov_lower.ll
@@ -1,11 +1,24 @@
-; RUN: llc < %s -mtriple=i386-linux-gnu -o - | FileCheck %s
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=i386-linux-gnu -o - | FileCheck %s
; This test checks that only a single js gets generated in the final code
; for lowering the CMOV pseudos that get created for this IR.
-; CHECK-LABEL: foo1:
-; CHECK: js
-; CHECK-NOT: js
define i32 @foo1(i32 %v1, i32 %v2, i32 %v3) nounwind {
+; CHECK-LABEL: foo1:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movl {{[0-9]+}}(%esp), %edx
+; CHECK-NEXT: testl %edx, %edx
+; CHECK-NEXT: js .LBB0_1
+; CHECK-NEXT: # %bb.2: # %entry
+; CHECK-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT: subl %ecx, %eax
+; CHECK-NEXT: retl
+; CHECK-NEXT: .LBB0_1:
+; CHECK-NEXT: movl %eax, %ecx
+; CHECK-NEXT: movl %edx, %eax
+; CHECK-NEXT: subl %ecx, %eax
+; CHECK-NEXT: retl
entry:
%cmp = icmp slt i32 %v1, 0
%v2.v3 = select i1 %cmp, i32 %v2, i32 %v3
@@ -17,11 +30,21 @@ entry:
; This test checks that only a single js gets generated in the final code
; for lowering the CMOV pseudos that get created for this IR. This makes
; sure the code for the lowering for opposite conditions gets tested.
-; CHECK-LABEL: foo11:
-; CHECK: js
-; CHECK-NOT: js
-; CHECK-NOT: jns
define i32 @foo11(i32 %v1, i32 %v2, i32 %v3) nounwind {
+; CHECK-LABEL: foo11:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: testl %eax, %eax
+; CHECK-NEXT: js .LBB1_1
+; CHECK-NEXT: # %bb.2: # %entry
+; CHECK-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT: subl %ecx, %eax
+; CHECK-NEXT: retl
+; CHECK-NEXT: .LBB1_1:
+; CHECK-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT: movl %ecx, %eax
+; CHECK-NEXT: subl %ecx, %eax
+; CHECK-NEXT: retl
entry:
%cmp1 = icmp slt i32 %v1, 0
%v2.v3 = select i1 %cmp1, i32 %v2, i32 %v3
@@ -33,10 +56,24 @@ entry:
; This test checks that only a single js gets generated in the final code
; for lowering the CMOV pseudos that get created for this IR.
-; CHECK-LABEL: foo2:
-; CHECK: js
-; CHECK-NOT: js
define i32 @foo2(i8 %v1, i8 %v2, i8 %v3) nounwind {
+; CHECK-LABEL: foo2:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %edx
+; CHECK-NEXT: testb %dl, %dl
+; CHECK-NEXT: js .LBB2_1
+; CHECK-NEXT: # %bb.2: # %entry
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: jmp .LBB2_3
+; CHECK-NEXT: .LBB2_1:
+; CHECK-NEXT: movl %ecx, %eax
+; CHECK-NEXT: movl %edx, %ecx
+; CHECK-NEXT: .LBB2_3: # %entry
+; CHECK-NEXT: movsbl %al, %eax
+; CHECK-NEXT: movsbl %cl, %ecx
+; CHECK-NEXT: subl %ecx, %eax
+; CHECK-NEXT: retl
entry:
%cmp = icmp slt i8 %v1, 0
%v2.v3 = select i1 %cmp, i8 %v2, i8 %v3
@@ -49,10 +86,22 @@ entry:
; This test checks that only a single js gets generated in the final code
; for lowering the CMOV pseudos that get created for this IR.
-; CHECK-LABEL: foo3:
-; CHECK: js
-; CHECK-NOT: js
define i32 @foo3(i16 %v1, i16 %v2, i16 %v3) nounwind {
+; CHECK-LABEL: foo3:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT: testw %cx, %cx
+; CHECK-NEXT: movl %eax, %edx
+; CHECK-NEXT: js .LBB3_2
+; CHECK-NEXT: # %bb.1: # %entry
+; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %edx
+; CHECK-NEXT: movl %eax, %ecx
+; CHECK-NEXT: .LBB3_2: # %entry
+; CHECK-NEXT: movswl %dx, %eax
+; CHECK-NEXT: movswl %cx, %ecx
+; CHECK-NEXT: subl %ecx, %eax
+; CHECK-NEXT: retl
entry:
%cmp = icmp slt i16 %v1, 0
%v2.v3 = select i1 %cmp, i16 %v2, i16 %v3
@@ -65,10 +114,22 @@ entry:
; This test checks that only a single js gets generated in the final code
; for lowering the CMOV pseudos that get created for this IR.
-; CHECK-LABEL: foo4:
-; CHECK: js
-; CHECK-NOT: js
define float @foo4(i32 %v1, float %v2, float %v3, float %v4) nounwind {
+; CHECK-LABEL: foo4:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; CHECK-NEXT: js .LBB4_1
+; CHECK-NEXT: # %bb.2: # %entry
+; CHECK-NEXT: fstp %st(1)
+; CHECK-NEXT: fsubp %st, %st(1)
+; CHECK-NEXT: retl
+; CHECK-NEXT: .LBB4_1:
+; CHECK-NEXT: fstp %st(2)
+; CHECK-NEXT: fsubp %st, %st(1)
+; CHECK-NEXT: retl
entry:
%cmp = icmp slt i32 %v1, 0
%t1 = select i1 %cmp, float %v2, float %v3
@@ -79,10 +140,22 @@ entry:
; This test checks that only a single je gets generated in the final code
; for lowering the CMOV pseudos that get created for this IR.
-; CHECK-LABEL: foo5:
-; CHECK: je
-; CHECK-NOT: je
define double @foo5(i32 %v1, double %v2, double %v3, double %v4) nounwind {
+; CHECK-LABEL: foo5:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT: fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT: fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; CHECK-NEXT: je .LBB5_1
+; CHECK-NEXT: # %bb.2: # %entry
+; CHECK-NEXT: fstp %st(1)
+; CHECK-NEXT: fsubp %st, %st(1)
+; CHECK-NEXT: retl
+; CHECK-NEXT: .LBB5_1:
+; CHECK-NEXT: fstp %st(2)
+; CHECK-NEXT: fsubp %st, %st(1)
+; CHECK-NEXT: retl
entry:
%cmp = icmp eq i32 %v1, 0
%t1 = select i1 %cmp, double %v2, double %v3
@@ -93,10 +166,67 @@ entry:
; This test checks that only a single je gets generated in the final code
; for lowering the CMOV pseudos that get created for this IR.
-; CHECK-LABEL: foo6:
-; CHECK: je
-; CHECK-NOT: je
define <4 x float> @foo6(i32 %v1, <4 x float> %v2, <4 x float> %v3, <4 x float> %v4) nounwind {
+; CHECK-LABEL: foo6:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: subl $28, %esp
+; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: je .LBB6_1
+; CHECK-NEXT: # %bb.2: # %entry
+; CHECK-NEXT: fstp %st(3)
+; CHECK-NEXT: fstp %st(4)
+; CHECK-NEXT: fstp %st(3)
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: fxch %st(1)
+; CHECK-NEXT: fstps (%esp) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: jmp .LBB6_3
+; CHECK-NEXT: .LBB6_1:
+; CHECK-NEXT: fxch %st(3)
+; CHECK-NEXT: fstps (%esp) # 4-byte Folded Spill
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fxch %st(5)
+; CHECK-NEXT: fxch %st(3)
+; CHECK-NEXT: fxch %st(2)
+; CHECK-NEXT: .LBB6_3: # %entry
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fsubp %st, %st(1)
+; CHECK-NEXT: flds (%esp) # 4-byte Folded Reload
+; CHECK-NEXT: fsubp %st, %st(2)
+; CHECK-NEXT: fxch %st(4)
+; CHECK-NEXT: fsubp %st, %st(2)
+; CHECK-NEXT: fxch %st(4)
+; CHECK-NEXT: fsubp %st, %st(2)
+; CHECK-NEXT: fxch %st(2)
+; CHECK-NEXT: fstps 12(%eax)
+; CHECK-NEXT: fxch %st(2)
+; CHECK-NEXT: fstps 8(%eax)
+; CHECK-NEXT: fstps 4(%eax)
+; CHECK-NEXT: fstps (%eax)
+; CHECK-NEXT: addl $28, %esp
+; CHECK-NEXT: retl $4
entry:
%cmp = icmp eq i32 %v1, 0
%t1 = select i1 %cmp, <4 x float> %v2, <4 x float> %v3
@@ -107,10 +237,32 @@ entry:
; This test checks that only a single je gets generated in the final code
; for lowering the CMOV pseudos that get created for this IR.
-; CHECK-LABEL: foo7:
-; CHECK: je
-; CHECK-NOT: je
define <2 x double> @foo7(i32 %v1, <2 x double> %v2, <2 x double> %v3, <2 x double> %v4) nounwind {
+; CHECK-LABEL: foo7:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT: fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT: fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT: fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT: fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT: fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; CHECK-NEXT: je .LBB7_1
+; CHECK-NEXT: # %bb.2: # %entry
+; CHECK-NEXT: fstp %st(3)
+; CHECK-NEXT: fstp %st(0)
+; CHECK-NEXT: jmp .LBB7_3
+; CHECK-NEXT: .LBB7_1:
+; CHECK-NEXT: fstp %st(5)
+; CHECK-NEXT: fstp %st(3)
+; CHECK-NEXT: fxch %st(3)
+; CHECK-NEXT: fxch %st(2)
+; CHECK-NEXT: fxch %st(1)
+; CHECK-NEXT: .LBB7_3: # %entry
+; CHECK-NEXT: fxch %st(1)
+; CHECK-NEXT: fsubp %st, %st(2)
+; CHECK-NEXT: fsubp %st, %st(2)
+; CHECK-NEXT: retl
entry:
%cmp = icmp eq i32 %v1, 0
%t1 = select i1 %cmp, <2 x double> %v2, <2 x double> %v3
@@ -123,10 +275,413 @@ entry:
; for lowering the CMOV pseudos that get created for this IR. This combines
; all the supported types together into one long string of selects based
; on the same condition.
-; CHECK-LABEL: foo8:
-; CHECK: ja
-; CHECK-NOT: ja
define void @foo8(i32 %v1,
+; CHECK-LABEL: foo8:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: subl $408, %esp # imm = 0x198
+; CHECK-NEXT: fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstpl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT: fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstpl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT: fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstpl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT: fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstpl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT: fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstpl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT: fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstpl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: fsubrs {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstp %st(0)
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: fsubrs {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstp %st(0)
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: fsubrs {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstp %st(0)
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: fsubrs {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstp %st(0)
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: fsubrs {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstp %st(0)
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: fsubrs {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstp %st(0)
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: fsubrs {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstp %st(0)
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: fsubrs {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstp %st(0)
+; CHECK-NEXT: fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstpl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT: movl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: cmpl $31, {{[0-9]+}}(%esp)
+; CHECK-NEXT: fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstpl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT: fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstpl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT: fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstpl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT: fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstpl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT: fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstpl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT: fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstpl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT: fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstpl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT: fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstpl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT: fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstpl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstps (%esp) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstpl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT: fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstpl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT: fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstpl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT: fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstpl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT: fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstpl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT: fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstpl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT: fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstpl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT: fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstpl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT: fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT: fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstpl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: fldl {{[0-9]+}}(%esp)
+; CHECK-NEXT: flds {{[0-9]+}}(%esp)
+; CHECK-NEXT: fsts {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: fadds {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: fstl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT: faddl {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstpl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT: fxch %st(3)
+; CHECK-NEXT: fsts {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: fadds {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: fxch %st(1)
+; CHECK-NEXT: fsts {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: fadds {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: fsts {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: fadds {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: fsts {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: fadds {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: fstl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT: faddl {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstpl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload
+; CHECK-NEXT: faddl {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstpl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fsubrs {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fsubrs {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: flds (%esp) # 4-byte Folded Reload
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fsubrs {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fsubrs {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fsubrs {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fld %st(0)
+; CHECK-NEXT: fsubrs {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: fxch %st(5)
+; CHECK-NEXT: fsubrs {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: fxch %st(3)
+; CHECK-NEXT: fsubrs {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fxch %st(2)
+; CHECK-NEXT: fsubrs {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: fxch %st(2)
+; CHECK-NEXT: fsubrs {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: fxch %st(1)
+; CHECK-NEXT: fsubrs {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fsubrs {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fld %st(0)
+; CHECK-NEXT: fsubrs {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fsubrs {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fsubrs {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: fxch %st(1)
+; CHECK-NEXT: fsubrs {{[0-9]+}}(%esp)
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: jbe .LBB8_2
+; CHECK-NEXT: # %bb.1:
+; CHECK-NEXT: movl {{[0-9]+}}(%esp), %edx
+; CHECK-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT: flds (%esp) # 4-byte Folded Reload
+; CHECK-NEXT: fstp %st(0)
+; CHECK-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload
+; CHECK-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload
+; CHECK-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstp %st(0)
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstp %st(0)
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstp %st(0)
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstp %st(0)
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstp %st(0)
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstp %st(0)
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstp %st(0)
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstp %st(0)
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstp %st(0)
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstp %st(0)
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstp %st(0)
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstp %st(0)
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: jmp .LBB8_3
+; CHECK-NEXT: .LBB8_2: # %select.false.sink
+; CHECK-NEXT: fstp %st(1)
+; CHECK-NEXT: fstp %st(0)
+; CHECK-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload
+; CHECK-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload
+; CHECK-NEXT: fsubrp %st, %st(1)
+; CHECK-NEXT: fstp %st(0)
+; CHECK-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload
+; CHECK-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload
+; CHECK-NEXT: fsubrp %st, %st(1)
+; CHECK-NEXT: fstp %st(0)
+; CHECK-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload
+; CHECK-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload
+; CHECK-NEXT: fsubrp %st, %st(1)
+; CHECK-NEXT: fstp %st(0)
+; CHECK-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload
+; CHECK-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload
+; CHECK-NEXT: fsubrp %st, %st(1)
+; CHECK-NEXT: fstp %st(0)
+; CHECK-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload
+; CHECK-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload
+; CHECK-NEXT: fsubp %st, %st(1)
+; CHECK-NEXT: fstpl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload
+; CHECK-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload
+; CHECK-NEXT: fsubp %st, %st(1)
+; CHECK-NEXT: fstpl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload
+; CHECK-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload
+; CHECK-NEXT: fsubp %st, %st(1)
+; CHECK-NEXT: fstpl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload
+; CHECK-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload
+; CHECK-NEXT: fsubrp %st, %st(1)
+; CHECK-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload
+; CHECK-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload
+; CHECK-NEXT: fsubrp %st, %st(1)
+; CHECK-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload
+; CHECK-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload
+; CHECK-NEXT: fsubrp %st, %st(1)
+; CHECK-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload
+; CHECK-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload
+; CHECK-NEXT: fsubrp %st, %st(1)
+; CHECK-NEXT: fstpl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload
+; CHECK-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload
+; CHECK-NEXT: fsubrp %st, %st(1)
+; CHECK-NEXT: fstpl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; CHECK-NEXT: xorl $11, %ecx
+; CHECK-NEXT: movl $1234, %edx # imm = 0x4D2
+; CHECK-NEXT: xorl {{[0-9]+}}(%esp), %edx
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload
+; CHECK-NEXT: fstpl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload
+; CHECK-NEXT: fstpl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload
+; CHECK-NEXT: fstpl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Spill
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstps (%esp) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstps {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fxch %st(1)
+; CHECK-NEXT: fxch %st(5)
+; CHECK-NEXT: fxch %st(1)
+; CHECK-NEXT: fxch %st(4)
+; CHECK-NEXT: fxch %st(2)
+; CHECK-NEXT: .LBB8_3: # %select.end
+; CHECK-NEXT: movw %cx, 2(%eax)
+; CHECK-NEXT: movl %edx, 4(%eax)
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstps 8(%eax)
+; CHECK-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload
+; CHECK-NEXT: fstpl 16(%eax)
+; CHECK-NEXT: fstps 32(%eax)
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstps 36(%eax)
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstps 40(%eax)
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstps 44(%eax)
+; CHECK-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload
+; CHECK-NEXT: fstpl 48(%eax)
+; CHECK-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload
+; CHECK-NEXT: fstpl 56(%eax)
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstps 64(%eax)
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstps 68(%eax)
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstps 72(%eax)
+; CHECK-NEXT: fxch %st(3)
+; CHECK-NEXT: fstps 76(%eax)
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstps 80(%eax)
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstps 84(%eax)
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstps 88(%eax)
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstps 92(%eax)
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstps 96(%eax)
+; CHECK-NEXT: flds (%esp) # 4-byte Folded Reload
+; CHECK-NEXT: fstps 100(%eax)
+; CHECK-NEXT: fxch %st(3)
+; CHECK-NEXT: fstps 104(%eax)
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstps 108(%eax)
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstps 112(%eax)
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstps 116(%eax)
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstps 120(%eax)
+; CHECK-NEXT: flds {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; CHECK-NEXT: fstps 124(%eax)
+; CHECK-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload
+; CHECK-NEXT: fstpl 128(%eax)
+; CHECK-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload
+; CHECK-NEXT: fstpl 136(%eax)
+; CHECK-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload
+; CHECK-NEXT: fstpl 144(%eax)
+; CHECK-NEXT: fxch %st(1)
+; CHECK-NEXT: fstpl 152(%eax)
+; CHECK-NEXT: fxch %st(1)
+; CHECK-NEXT: fstpl 160(%eax)
+; CHECK-NEXT: fstpl 168(%eax)
+; CHECK-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload
+; CHECK-NEXT: fstpl 176(%eax)
+; CHECK-NEXT: fldl {{[-0-9]+}}(%e{{[sb]}}p) # 8-byte Folded Reload
+; CHECK-NEXT: fstpl 184(%eax)
+; CHECK-NEXT: addl $408, %esp # imm = 0x198
+; CHECK-NEXT: retl
i8 %v2, i8 %v3,
i16 %v12, i16 %v13,
i32 %v22, i32 %v23,
@@ -212,10 +767,1099 @@ entry:
; pseudo-opcodes to be generated, this test should be replaced with one that
; tests those opcodes.
;
-; CHECK-LABEL: foo9:
-; CHECK: ja
-; CHECK-NOT: ja
define void @foo9(i32 %v1,
+; CHECK-LABEL: foo9:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: pushl %ebp
+; CHECK-NEXT: pushl %ebx
+; CHECK-NEXT: pushl %edi
+; CHECK-NEXT: pushl %esi
+; CHECK-NEXT: subl $144, %esp
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: cmpl $32, {{[0-9]+}}(%esp)
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[0-9]+}}(%esp), %ah
+; CHECK-NEXT: movb {{[0-9]+}}(%esp), %al
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[0-9]+}}(%esp), %al
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[0-9]+}}(%esp), %al
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[0-9]+}}(%esp), %ch
+; CHECK-NEXT: movb {{[0-9]+}}(%esp), %al
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[0-9]+}}(%esp), %al
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[0-9]+}}(%esp), %al
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[0-9]+}}(%esp), %dh
+; CHECK-NEXT: movb {{[0-9]+}}(%esp), %al
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[0-9]+}}(%esp), %al
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[0-9]+}}(%esp), %al
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movzbl {{[0-9]+}}(%esp), %ebx
+; CHECK-NEXT: movb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb {{[0-9]+}}(%esp), %al
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[0-9]+}}(%esp), %dl
+; CHECK-NEXT: movb {{[0-9]+}}(%esp), %al
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[0-9]+}}(%esp), %al
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[0-9]+}}(%esp), %bh
+; CHECK-NEXT: jae .LBB9_1
+; CHECK-NEXT: # %bb.2: # %entry
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %bh
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %al # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %al
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb %bh, %al
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %bh # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %bh
+; CHECK-NEXT: movb %bh, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %dl
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %bh # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %bh
+; CHECK-NEXT: movb %bh, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb %al, %bh
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %bl
+; CHECK-NEXT: movb %bl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %dh
+; CHECK-NEXT: movb %dh, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %ch
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %ah
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[0-9]+}}(%esp), %al
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %al
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[0-9]+}}(%esp), %al
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %al
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %al # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %al
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %al # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %al
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %al # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %al
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %al # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %al
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %al # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %al
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %al # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %al
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: xorb {{[0-9]+}}(%esp), %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %dh # 1-byte Reload
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %bl # 1-byte Reload
+; CHECK-NEXT: jmp .LBB9_3
+; CHECK-NEXT: .LBB9_1:
+; CHECK-NEXT: movb {{[0-9]+}}(%esp), %al
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[0-9]+}}(%esp), %al
+; CHECK-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: .LBB9_3: # %entry
+; CHECK-NEXT: shlb $3, %bh
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %al # 1-byte Reload
+; CHECK-NEXT: andb $1, %al
+; CHECK-NEXT: shlb $2, %al
+; CHECK-NEXT: orb %bh, %al
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %bh # 1-byte Reload
+; CHECK-NEXT: addb %bh, %bh
+; CHECK-NEXT: andb $1, %dl
+; CHECK-NEXT: orb %bh, %dl
+; CHECK-NEXT: andb $3, %dl
+; CHECK-NEXT: orb %al, %dl
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %al # 1-byte Reload
+; CHECK-NEXT: shlb $3, %al
+; CHECK-NEXT: andb $1, %cl
+; CHECK-NEXT: shlb $2, %cl
+; CHECK-NEXT: orb %al, %cl
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %al # 1-byte Reload
+; CHECK-NEXT: addb %al, %al
+; CHECK-NEXT: movb %cl, %bh
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: andb $1, %cl
+; CHECK-NEXT: orb %al, %cl
+; CHECK-NEXT: andb $3, %cl
+; CHECK-NEXT: orb %bh, %cl
+; CHECK-NEXT: shlb $4, %dl
+; CHECK-NEXT: andb $15, %cl
+; CHECK-NEXT: orb %dl, %cl
+; CHECK-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %dl # 1-byte Reload
+; CHECK-NEXT: shlb $3, %dl
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: andb $1, %cl
+; CHECK-NEXT: shlb $2, %cl
+; CHECK-NEXT: orb %dl, %cl
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %dl # 1-byte Reload
+; CHECK-NEXT: addb %dl, %dl
+; CHECK-NEXT: andb $1, %bl
+; CHECK-NEXT: orb %dl, %bl
+; CHECK-NEXT: andb $3, %bl
+; CHECK-NEXT: orb %cl, %bl
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %dl # 1-byte Reload
+; CHECK-NEXT: shlb $3, %dl
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: andb $1, %cl
+; CHECK-NEXT: shlb $2, %cl
+; CHECK-NEXT: orb %dl, %cl
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %dl # 1-byte Reload
+; CHECK-NEXT: addb %dl, %dl
+; CHECK-NEXT: andb $1, %dh
+; CHECK-NEXT: orb %dl, %dh
+; CHECK-NEXT: andb $3, %dh
+; CHECK-NEXT: orb %cl, %dh
+; CHECK-NEXT: shlb $4, %bl
+; CHECK-NEXT: andb $15, %dh
+; CHECK-NEXT: orb %bl, %dh
+; CHECK-NEXT: movzbl %dh, %edx
+; CHECK-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: shlb $3, %cl
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %al # 1-byte Reload
+; CHECK-NEXT: andb $1, %al
+; CHECK-NEXT: shlb $2, %al
+; CHECK-NEXT: orb %cl, %al
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: addb %cl, %cl
+; CHECK-NEXT: andb $1, %ch
+; CHECK-NEXT: orb %cl, %ch
+; CHECK-NEXT: andb $3, %ch
+; CHECK-NEXT: orb %al, %ch
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: shlb $3, %cl
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %al # 1-byte Reload
+; CHECK-NEXT: andb $1, %al
+; CHECK-NEXT: shlb $2, %al
+; CHECK-NEXT: orb %cl, %al
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: addb %cl, %cl
+; CHECK-NEXT: andb $1, %ah
+; CHECK-NEXT: orb %cl, %ah
+; CHECK-NEXT: andb $3, %ah
+; CHECK-NEXT: orb %al, %ah
+; CHECK-NEXT: shlb $4, %ch
+; CHECK-NEXT: andb $15, %ah
+; CHECK-NEXT: orb %ch, %ah
+; CHECK-NEXT: movzbl %ah, %eax
+; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; CHECK-NEXT: shlb $3, %al
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; CHECK-NEXT: andb $1, %cl
+; CHECK-NEXT: shlb $2, %cl
+; CHECK-NEXT: orb %al, %cl
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 1-byte Folded Reload
+; CHECK-NEXT: addb %dl, %dl
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; CHECK-NEXT: andb $1, %al
+; CHECK-NEXT: orb %dl, %al
+; CHECK-NEXT: andb $3, %al
+; CHECK-NEXT: orb %cl, %al
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 1-byte Folded Reload
+; CHECK-NEXT: shlb $3, %dl
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; CHECK-NEXT: andb $1, %cl
+; CHECK-NEXT: shlb $2, %cl
+; CHECK-NEXT: orb %dl, %cl
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 1-byte Folded Reload
+; CHECK-NEXT: addb %dl, %dl
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %ah # 1-byte Reload
+; CHECK-NEXT: andb $1, %ah
+; CHECK-NEXT: orb %dl, %ah
+; CHECK-NEXT: andb $3, %ah
+; CHECK-NEXT: orb %cl, %ah
+; CHECK-NEXT: shlb $4, %al
+; CHECK-NEXT: andb $15, %ah
+; CHECK-NEXT: orb %al, %ah
+; CHECK-NEXT: movzbl %ah, %eax
+; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; CHECK-NEXT: shlb $3, %al
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; CHECK-NEXT: andb $1, %cl
+; CHECK-NEXT: shlb $2, %cl
+; CHECK-NEXT: orb %al, %cl
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 1-byte Folded Reload
+; CHECK-NEXT: addb %dl, %dl
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; CHECK-NEXT: andb $1, %al
+; CHECK-NEXT: orb %dl, %al
+; CHECK-NEXT: andb $3, %al
+; CHECK-NEXT: orb %cl, %al
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 1-byte Folded Reload
+; CHECK-NEXT: shlb $3, %dl
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; CHECK-NEXT: andb $1, %cl
+; CHECK-NEXT: shlb $2, %cl
+; CHECK-NEXT: orb %dl, %cl
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 1-byte Folded Reload
+; CHECK-NEXT: addb %dl, %dl
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %ah # 1-byte Reload
+; CHECK-NEXT: andb $1, %ah
+; CHECK-NEXT: orb %dl, %ah
+; CHECK-NEXT: andb $3, %ah
+; CHECK-NEXT: orb %cl, %ah
+; CHECK-NEXT: shlb $4, %al
+; CHECK-NEXT: andb $15, %ah
+; CHECK-NEXT: orb %al, %ah
+; CHECK-NEXT: movzbl %ah, %esi
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; CHECK-NEXT: shlb $3, %cl
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; CHECK-NEXT: andb $1, %al
+; CHECK-NEXT: shlb $2, %al
+; CHECK-NEXT: orb %cl, %al
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 1-byte Folded Reload
+; CHECK-NEXT: addb %dl, %dl
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; CHECK-NEXT: andb $1, %cl
+; CHECK-NEXT: orb %dl, %cl
+; CHECK-NEXT: andb $3, %cl
+; CHECK-NEXT: orb %al, %cl
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; CHECK-NEXT: shlb $3, %al
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 1-byte Folded Reload
+; CHECK-NEXT: andb $1, %dl
+; CHECK-NEXT: shlb $2, %dl
+; CHECK-NEXT: orb %al, %dl
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %ah # 1-byte Reload
+; CHECK-NEXT: addb %ah, %ah
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %al # 1-byte Reload
+; CHECK-NEXT: andb $1, %al
+; CHECK-NEXT: orb %ah, %al
+; CHECK-NEXT: andb $3, %al
+; CHECK-NEXT: orb %dl, %al
+; CHECK-NEXT: shlb $4, %cl
+; CHECK-NEXT: andb $15, %al
+; CHECK-NEXT: orb %cl, %al
+; CHECK-NEXT: movzbl %al, %eax
+; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; CHECK-NEXT: shlb $3, %cl
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; CHECK-NEXT: andb $1, %al
+; CHECK-NEXT: shlb $2, %al
+; CHECK-NEXT: orb %cl, %al
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 1-byte Folded Reload
+; CHECK-NEXT: addb %dl, %dl
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; CHECK-NEXT: andb $1, %cl
+; CHECK-NEXT: orb %dl, %cl
+; CHECK-NEXT: andb $3, %cl
+; CHECK-NEXT: orb %al, %cl
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; CHECK-NEXT: shlb $3, %al
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 1-byte Folded Reload
+; CHECK-NEXT: andb $1, %dl
+; CHECK-NEXT: shlb $2, %dl
+; CHECK-NEXT: orb %al, %dl
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %ah # 1-byte Reload
+; CHECK-NEXT: addb %ah, %ah
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %al # 1-byte Reload
+; CHECK-NEXT: andb $1, %al
+; CHECK-NEXT: orb %ah, %al
+; CHECK-NEXT: andb $3, %al
+; CHECK-NEXT: orb %dl, %al
+; CHECK-NEXT: shlb $4, %cl
+; CHECK-NEXT: andb $15, %al
+; CHECK-NEXT: orb %cl, %al
+; CHECK-NEXT: movzbl %al, %ebp
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; CHECK-NEXT: shlb $3, %cl
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; CHECK-NEXT: andb $1, %al
+; CHECK-NEXT: shlb $2, %al
+; CHECK-NEXT: orb %cl, %al
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 1-byte Folded Reload
+; CHECK-NEXT: addb %dl, %dl
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; CHECK-NEXT: andb $1, %cl
+; CHECK-NEXT: orb %dl, %cl
+; CHECK-NEXT: andb $3, %cl
+; CHECK-NEXT: orb %al, %cl
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; CHECK-NEXT: shlb $3, %al
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 1-byte Folded Reload
+; CHECK-NEXT: andb $1, %dl
+; CHECK-NEXT: shlb $2, %dl
+; CHECK-NEXT: orb %al, %dl
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %ah # 1-byte Reload
+; CHECK-NEXT: addb %ah, %ah
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %al # 1-byte Reload
+; CHECK-NEXT: andb $1, %al
+; CHECK-NEXT: orb %ah, %al
+; CHECK-NEXT: andb $3, %al
+; CHECK-NEXT: orb %dl, %al
+; CHECK-NEXT: shlb $4, %cl
+; CHECK-NEXT: andb $15, %al
+; CHECK-NEXT: orb %cl, %al
+; CHECK-NEXT: movzbl %al, %eax
+; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; CHECK-NEXT: shlb $3, %cl
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; CHECK-NEXT: andb $1, %al
+; CHECK-NEXT: shlb $2, %al
+; CHECK-NEXT: orb %cl, %al
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 1-byte Folded Reload
+; CHECK-NEXT: addb %dl, %dl
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; CHECK-NEXT: andb $1, %cl
+; CHECK-NEXT: orb %dl, %cl
+; CHECK-NEXT: andb $3, %cl
+; CHECK-NEXT: orb %al, %cl
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; CHECK-NEXT: shlb $3, %al
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 1-byte Folded Reload
+; CHECK-NEXT: andb $1, %dl
+; CHECK-NEXT: shlb $2, %dl
+; CHECK-NEXT: orb %al, %dl
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %ah # 1-byte Reload
+; CHECK-NEXT: addb %ah, %ah
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %al # 1-byte Reload
+; CHECK-NEXT: andb $1, %al
+; CHECK-NEXT: orb %ah, %al
+; CHECK-NEXT: andb $3, %al
+; CHECK-NEXT: orb %dl, %al
+; CHECK-NEXT: shlb $4, %cl
+; CHECK-NEXT: andb $15, %al
+; CHECK-NEXT: orb %cl, %al
+; CHECK-NEXT: movzbl %al, %edx
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; CHECK-NEXT: shlb $3, %cl
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; CHECK-NEXT: andb $1, %al
+; CHECK-NEXT: shlb $2, %al
+; CHECK-NEXT: orb %cl, %al
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %ah # 1-byte Reload
+; CHECK-NEXT: addb %ah, %ah
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; CHECK-NEXT: andb $1, %cl
+; CHECK-NEXT: orb %ah, %cl
+; CHECK-NEXT: andb $3, %cl
+; CHECK-NEXT: orb %al, %cl
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; CHECK-NEXT: shlb $3, %al
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %ah # 1-byte Reload
+; CHECK-NEXT: andb $1, %ah
+; CHECK-NEXT: shlb $2, %ah
+; CHECK-NEXT: orb %al, %ah
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %ch # 1-byte Reload
+; CHECK-NEXT: addb %ch, %ch
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %al # 1-byte Reload
+; CHECK-NEXT: andb $1, %al
+; CHECK-NEXT: orb %ch, %al
+; CHECK-NEXT: andb $3, %al
+; CHECK-NEXT: orb %ah, %al
+; CHECK-NEXT: shlb $4, %cl
+; CHECK-NEXT: andb $15, %al
+; CHECK-NEXT: orb %cl, %al
+; CHECK-NEXT: movzbl %al, %eax
+; CHECK-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; CHECK-NEXT: shlb $3, %cl
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; CHECK-NEXT: andb $1, %al
+; CHECK-NEXT: shlb $2, %al
+; CHECK-NEXT: orb %cl, %al
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %ah # 1-byte Reload
+; CHECK-NEXT: addb %ah, %ah
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; CHECK-NEXT: andb $1, %cl
+; CHECK-NEXT: orb %ah, %cl
+; CHECK-NEXT: andb $3, %cl
+; CHECK-NEXT: orb %al, %cl
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; CHECK-NEXT: shlb $3, %al
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %ah # 1-byte Reload
+; CHECK-NEXT: andb $1, %ah
+; CHECK-NEXT: shlb $2, %ah
+; CHECK-NEXT: orb %al, %ah
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %ch # 1-byte Reload
+; CHECK-NEXT: addb %ch, %ch
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %al # 1-byte Reload
+; CHECK-NEXT: andb $1, %al
+; CHECK-NEXT: orb %ch, %al
+; CHECK-NEXT: andb $3, %al
+; CHECK-NEXT: orb %ah, %al
+; CHECK-NEXT: shlb $4, %cl
+; CHECK-NEXT: andb $15, %al
+; CHECK-NEXT: orb %cl, %al
+; CHECK-NEXT: movzbl %al, %eax
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %ch # 1-byte Reload
+; CHECK-NEXT: shlb $3, %ch
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: andb $1, %cl
+; CHECK-NEXT: shlb $2, %cl
+; CHECK-NEXT: orb %ch, %cl
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 1-byte Folded Reload
+; CHECK-NEXT: addb %bl, %bl
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %ch # 1-byte Reload
+; CHECK-NEXT: andb $1, %ch
+; CHECK-NEXT: orb %bl, %ch
+; CHECK-NEXT: andb $3, %ch
+; CHECK-NEXT: orb %cl, %ch
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: shlb $3, %cl
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 1-byte Folded Reload
+; CHECK-NEXT: andb $1, %bl
+; CHECK-NEXT: shlb $2, %bl
+; CHECK-NEXT: orb %cl, %bl
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %bh # 1-byte Reload
+; CHECK-NEXT: addb %bh, %bh
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: andb $1, %cl
+; CHECK-NEXT: orb %bh, %cl
+; CHECK-NEXT: andb $3, %cl
+; CHECK-NEXT: orb %bl, %cl
+; CHECK-NEXT: shlb $4, %ch
+; CHECK-NEXT: andb $15, %cl
+; CHECK-NEXT: orb %ch, %cl
+; CHECK-NEXT: movzbl %cl, %ecx
+; CHECK-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %ch # 1-byte Reload
+; CHECK-NEXT: shlb $3, %ch
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: andb $1, %cl
+; CHECK-NEXT: shlb $2, %cl
+; CHECK-NEXT: orb %ch, %cl
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 1-byte Folded Reload
+; CHECK-NEXT: addb %bl, %bl
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %ch # 1-byte Reload
+; CHECK-NEXT: andb $1, %ch
+; CHECK-NEXT: orb %bl, %ch
+; CHECK-NEXT: andb $3, %ch
+; CHECK-NEXT: orb %cl, %ch
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: shlb $3, %cl
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 1-byte Folded Reload
+; CHECK-NEXT: andb $1, %bl
+; CHECK-NEXT: shlb $2, %bl
+; CHECK-NEXT: orb %cl, %bl
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %bh # 1-byte Reload
+; CHECK-NEXT: addb %bh, %bh
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: andb $1, %cl
+; CHECK-NEXT: orb %bh, %cl
+; CHECK-NEXT: andb $3, %cl
+; CHECK-NEXT: orb %bl, %cl
+; CHECK-NEXT: shlb $4, %ch
+; CHECK-NEXT: andb $15, %cl
+; CHECK-NEXT: orb %ch, %cl
+; CHECK-NEXT: movzbl %cl, %edi
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %bh # 1-byte Reload
+; CHECK-NEXT: shlb $3, %bh
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %bl # 1-byte Reload
+; CHECK-NEXT: andb $1, %bl
+; CHECK-NEXT: shlb $2, %bl
+; CHECK-NEXT: orb %bh, %bl
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %bh # 1-byte Reload
+; CHECK-NEXT: addb %bh, %bh
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %ch # 1-byte Reload
+; CHECK-NEXT: andb $1, %ch
+; CHECK-NEXT: orb %bh, %ch
+; CHECK-NEXT: andb $3, %ch
+; CHECK-NEXT: orb %bl, %ch
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: shlb $3, %cl
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %bh # 1-byte Reload
+; CHECK-NEXT: andb $1, %bh
+; CHECK-NEXT: shlb $2, %bh
+; CHECK-NEXT: orb %cl, %bh
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %bl # 1-byte Reload
+; CHECK-NEXT: addb %bl, %bl
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %cl # 1-byte Reload
+; CHECK-NEXT: andb $1, %cl
+; CHECK-NEXT: orb %bl, %cl
+; CHECK-NEXT: andb $3, %cl
+; CHECK-NEXT: orb %bh, %cl
+; CHECK-NEXT: shlb $4, %ch
+; CHECK-NEXT: andb $15, %cl
+; CHECK-NEXT: orb %ch, %cl
+; CHECK-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; CHECK-NEXT: shll $8, %ebx
+; CHECK-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; CHECK-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT: shll $8, %esi
+; CHECK-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; CHECK-NEXT: shll $8, %ebp
+; CHECK-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; CHECK-NEXT: shll $16, %esi
+; CHECK-NEXT: movzwl %bp, %ebx
+; CHECK-NEXT: orl %esi, %ebx
+; CHECK-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; CHECK-NEXT: shll $8, %edx
+; CHECK-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; CHECK-NEXT: shll $8, %eax
+; CHECK-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; CHECK-NEXT: shll $16, %edx
+; CHECK-NEXT: movzwl %ax, %ebp
+; CHECK-NEXT: orl %edx, %ebp
+; CHECK-NEXT: shll $8, %edi
+; CHECK-NEXT: addl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; CHECK-NEXT: shlb $3, %al
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 1-byte Folded Reload
+; CHECK-NEXT: andb $1, %dl
+; CHECK-NEXT: shlb $2, %dl
+; CHECK-NEXT: orb %al, %dl
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; CHECK-NEXT: addb %al, %al
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %ch # 1-byte Reload
+; CHECK-NEXT: andb $1, %ch
+; CHECK-NEXT: orb %al, %ch
+; CHECK-NEXT: andb $3, %ch
+; CHECK-NEXT: orb %dl, %ch
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 1-byte Folded Reload
+; CHECK-NEXT: shlb $3, %al
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 1-byte Folded Reload
+; CHECK-NEXT: andb $1, %dl
+; CHECK-NEXT: shlb $2, %dl
+; CHECK-NEXT: orb %al, %dl
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %ah # 1-byte Reload
+; CHECK-NEXT: addb %ah, %ah
+; CHECK-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %al # 1-byte Reload
+; CHECK-NEXT: andb $1, %al
+; CHECK-NEXT: orb %ah, %al
+; CHECK-NEXT: andb $3, %al
+; CHECK-NEXT: orb %dl, %al
+; CHECK-NEXT: shlb $4, %ch
+; CHECK-NEXT: andb $15, %al
+; CHECK-NEXT: orb %ch, %al
+; CHECK-NEXT: movzbl %cl, %edx
+; CHECK-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; CHECK-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; CHECK-NEXT: movb %cl, (%ebx)
+; CHECK-NEXT: movzbl %al, %esi
+; CHECK-NEXT: shll $8, %esi
+; CHECK-NEXT: orl %edx, %esi
+; CHECK-NEXT: shll $16, %edi
+; CHECK-NEXT: movzwl %si, %edx
+; CHECK-NEXT: orl %edi, %edx
+; CHECK-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; CHECK-NEXT: movw %ax, 4(%ebx)
+; CHECK-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; CHECK-NEXT: movl %eax, 8(%ebx)
+; CHECK-NEXT: movl %edx, 20(%ebx)
+; CHECK-NEXT: movl %ebp, 16(%ebx)
+; CHECK-NEXT: addl $144, %esp
+; CHECK-NEXT: popl %esi
+; CHECK-NEXT: popl %edi
+; CHECK-NEXT: popl %ebx
+; CHECK-NEXT: popl %ebp
+; CHECK-NEXT: retl
<8 x i1> %v12, <8 x i1> %v13,
<16 x i1> %v22, <16 x i1> %v23,
<32 x i1> %v32, <32 x i1> %v33,
diff --git a/llvm/test/CodeGen/X86/shrink-wrap-chkstk-x86_64.ll b/llvm/test/CodeGen/X86/shrink-wrap-chkstk-x86_64.ll
index 0793c33d2e8eb..b0c07debbeda2 100644
--- a/llvm/test/CodeGen/X86/shrink-wrap-chkstk-x86_64.ll
+++ b/llvm/test/CodeGen/X86/shrink-wrap-chkstk-x86_64.ll
@@ -9,24 +9,19 @@
define void @fn1() nounwind uwtable {
; CHECK-LABEL: fn1:
; CHECK: # %bb.0: # %entry
-; CHECK-NEXT: movl a(%rip), %eax
-; CHECK-NEXT: testl %eax, %eax
-; CHECK-NEXT: jne .LBB0_2
-; CHECK-NEXT: # %bb.1: # %select.true.sink
-; CHECK-NEXT: cltq
-; CHECK-NEXT: imulq $715827883, %rax, %rax # imm = 0x2AAAAAAB
-; CHECK-NEXT: movq %rax, %rcx
-; CHECK-NEXT: shrq $63, %rcx
-; CHECK-NEXT: shrq $32, %rax
-; CHECK-NEXT: addl %ecx, %eax
-; CHECK-NEXT: .LBB0_2: # %select.end
-; CHECK-NEXT: pushq %rax
-; CHECK-NEXT: movl $4128, %eax # imm = 0x1020
+; CHECK-NEXT: movl $4136, %eax # imm = 0x1028
; CHECK-NEXT: callq ___chkstk_ms
; CHECK-NEXT: subq %rax, %rsp
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rax
; CHECK-NEXT: .cfi_def_cfa_offset 4144
-; CHECK-NEXT: movl %eax, b(%rip)
+; CHECK-NEXT: movslq a(%rip), %rax
+; CHECK-NEXT: imulq $715827883, %rax, %rcx # imm = 0x2AAAAAAB
+; CHECK-NEXT: movq %rcx, %rdx
+; CHECK-NEXT: shrq $63, %rdx
+; CHECK-NEXT: shrq $32, %rcx
+; CHECK-NEXT: addl %edx, %ecx
+; CHECK-NEXT: testq %rax, %rax
+; CHECK-NEXT: cmovnel %eax, %ecx
+; CHECK-NEXT: movl %ecx, b(%rip)
; CHECK-NEXT: leaq {{[0-9]+}}(%rsp), %rcx
; CHECK-NEXT: # kill: def $ecx killed $ecx killed $rcx
; CHECK-NEXT: callq fn2
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/optimizeSelect-DT.ll b/llvm/test/Transforms/CodeGenPrepare/X86/optimizeSelect-DT.ll
index aaf3df0934681..e18ead311cb51 100644
--- a/llvm/test/Transforms/CodeGenPrepare/X86/optimizeSelect-DT.ll
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/optimizeSelect-DT.ll
@@ -7,18 +7,14 @@ target triple = "x86_64-unknown-linux-gnu"
define i1 @PR41004(i32 %x, i32 %y, i32 %t1) {
; CHECK-LABEL: @PR41004(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[MUL_FR:%.*]] = freeze i32 [[Y:%.*]]
-; CHECK-NEXT: [[T0:%.*]] = icmp eq i32 [[MUL_FR]], 1
-; CHECK-NEXT: br i1 [[T0]], label [[SELECT_TRUE_SINK:%.*]], label [[SELECT_END:%.*]]
-; CHECK: select.true.sink:
; CHECK-NEXT: [[REM:%.*]] = srem i32 [[X:%.*]], 2
-; CHECK-NEXT: br label [[SELECT_END]]
-; CHECK: select.end:
-; CHECK-NEXT: [[MUL:%.*]] = phi i32 [ [[REM]], [[SELECT_TRUE_SINK]] ], [ 0, [[ENTRY:%.*]] ]
-; CHECK-NEXT: [[USUB:%.*]] = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 [[T1:%.*]], i32 1)
-; CHECK-NEXT: [[NEG:%.*]] = extractvalue { i32, i1 } [[USUB]], 0
-; CHECK-NEXT: [[TOBOOL:%.*]] = extractvalue { i32, i1 } [[USUB]], 1
+; CHECK-NEXT: [[T0:%.*]] = icmp eq i32 [[Y:%.*]], 1
+; CHECK-NEXT: [[MUL:%.*]] = select i1 [[T0]], i32 [[REM]], i32 0
+; CHECK-NEXT: [[NEG:%.*]] = add i32 [[T1:%.*]], -1
; CHECK-NEXT: [[ADD:%.*]] = add i32 [[NEG]], [[MUL]]
+; CHECK-NEXT: br label [[IF:%.*]]
+; CHECK: if:
+; CHECK-NEXT: [[TOBOOL:%.*]] = icmp eq i32 [[T1]], 0
; CHECK-NEXT: ret i1 [[TOBOOL]]
;
entry:
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/select.ll b/llvm/test/Transforms/CodeGenPrepare/X86/select.ll
index 1dafde445a3e2..f1a599508a4cb 100644
--- a/llvm/test/Transforms/CodeGenPrepare/X86/select.ll
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/select.ll
@@ -207,23 +207,33 @@ define float @fadd_no_sink(float %a, float %b) {
define float @fdiv_no_sink(float %a, float %b) {
; CHECK-LABEL: @fdiv_no_sink(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[DIV:%.*]] = fdiv float [[A:%.*]], [[B:%.*]]
+; CHECK-NEXT: [[SEL_FROZEN:%.*]] = freeze float [[A:%.*]]
+; CHECK-NEXT: [[CMP:%.*]] = fcmp ogt float [[SEL_FROZEN]], 1.000000e+00
+; CHECK-NEXT: br i1 [[CMP]], label [[SELECT_TRUE_SINK:%.*]], label [[SELECT_END:%.*]]
+; CHECK: select.true.sink:
+; CHECK-NEXT: [[DIV:%.*]] = fdiv float [[A]], [[B:%.*]]
; CHECK-NEXT: [[ADD:%.*]] = fadd float [[DIV]], [[B]]
-; CHECK-NEXT: [[CMP:%.*]] = fcmp ogt float [[A]], 1.000000e+00
-; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], float [[ADD]], float 8.000000e+00
+; CHECK-NEXT: br label [[SELECT_END]]
+; CHECK: select.end:
+; CHECK-NEXT: [[SEL:%.*]] = phi float [ [[ADD]], [[SELECT_TRUE_SINK]] ], [ 8.000000e+00, [[ENTRY:%.*]] ]
; CHECK-NEXT: ret float [[SEL]]
;
; DEBUG-LABEL: @fdiv_no_sink(
; DEBUG-NEXT: entry:
-; DEBUG-NEXT: [[DIV:%.*]] = fdiv float [[A:%.*]], [[B:%.*]], !dbg [[DBG72:![0-9]+]]
-; DEBUG-NEXT: #dbg_value(float [[DIV]], [[META68:![0-9]+]], !DIExpression(), [[DBG72]])
-; DEBUG-NEXT: [[ADD:%.*]] = fadd float [[DIV]], [[B]], !dbg [[DBG73:![0-9]+]]
-; DEBUG-NEXT: #dbg_value(float [[ADD]], [[META69:![0-9]+]], !DIExpression(), [[DBG73]])
-; DEBUG-NEXT: [[CMP:%.*]] = fcmp ogt float [[A]], 1.000000e+00, !dbg [[DBG74:![0-9]+]]
-; DEBUG-NEXT: #dbg_value(i1 [[CMP]], [[META70:![0-9]+]], !DIExpression(), [[DBG74]])
-; DEBUG-NEXT: [[SEL:%.*]] = select i1 [[CMP]], float [[ADD]], float 8.000000e+00, !dbg [[DBG75:![0-9]+]]
-; DEBUG-NEXT: #dbg_value(float [[SEL]], [[META71:![0-9]+]], !DIExpression(), [[DBG75]])
-; DEBUG-NEXT: ret float [[SEL]], !dbg [[DBG76:![0-9]+]]
+; DEBUG-NEXT: [[SEL_FROZEN:%.*]] = freeze float [[A:%.*]]
+; DEBUG-NEXT: [[CMP:%.*]] = fcmp ogt float [[SEL_FROZEN]], 1.000000e+00, !dbg [[DBG72:![0-9]+]]
+; DEBUG-NEXT: #dbg_value(i1 [[CMP]], [[META70:![0-9]+]], !DIExpression(), [[DBG72]])
+; DEBUG-NEXT: br i1 [[CMP]], label [[SELECT_TRUE_SINK:%.*]], label [[SELECT_END:%.*]], !dbg [[DBG73:![0-9]+]]
+; DEBUG: select.true.sink:
+; DEBUG-NEXT: [[DIV:%.*]] = fdiv float [[A]], [[B:%.*]], !dbg [[DBG74:![0-9]+]]
+; DEBUG-NEXT: #dbg_value(float [[DIV]], [[META68:![0-9]+]], !DIExpression(), [[DBG74]])
+; DEBUG-NEXT: [[ADD:%.*]] = fadd float [[DIV]], [[B]], !dbg [[DBG75:![0-9]+]]
+; DEBUG-NEXT: #dbg_value(float [[ADD]], [[META69:![0-9]+]], !DIExpression(), [[DBG75]])
+; DEBUG-NEXT: br label [[SELECT_END]], !dbg [[DBG76:![0-9]+]]
+; DEBUG: select.end:
+; DEBUG-NEXT: [[SEL:%.*]] = phi float [ [[ADD]], [[SELECT_TRUE_SINK]] ], [ 8.000000e+00, [[ENTRY:%.*]] ], !dbg [[DBG73]]
+; DEBUG-NEXT: #dbg_value(float [[SEL]], [[META71:![0-9]+]], !DIExpression(), [[DBG73]])
+; DEBUG-NEXT: ret float [[SEL]], !dbg [[DBG76]]
;
entry:
%div = fdiv float %a, %b
diff --git a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-reg-does-not-geps.ll b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-reg-does-not-geps.ll
index 1640bafbd0bf9..b7fab08e5bd1a 100644
--- a/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-reg-does-not-geps.ll
+++ b/llvm/test/Transforms/CodeGenPrepare/X86/sink-addrmode-reg-does-not-geps.ll
@@ -49,17 +49,12 @@ exit:
define void @check_dt_after_modifying_cfg(ptr %dst, i64 %x, i8 %y, i8 %z) {
; CHECK-LABEL: define void @check_dt_after_modifying_cfg(
; CHECK-SAME: ptr [[DST:%.*]], i64 [[X:%.*]], i8 [[Y:%.*]], i8 [[Z:%.*]]) {
-; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[OFFSET:%.*]] = lshr i64 [[X]], 2
-; CHECK-NEXT: [[SEL_FROZEN:%.*]] = freeze i8 [[Z]]
-; CHECK-NEXT: [[CMP:%.*]] = icmp slt i8 [[SEL_FROZEN]], 0
-; CHECK-NEXT: br i1 [[CMP]], label %[[SELECT_END:.*]], label %[[SELECT_FALSE_SINK:.*]]
-; CHECK: [[SELECT_FALSE_SINK]]:
-; CHECK-NEXT: [[SMIN:%.*]] = tail call i8 @llvm.smin.i8(i8 [[Y]], i8 0)
-; CHECK-NEXT: br label %[[SELECT_END]]
-; CHECK: [[SELECT_END]]:
-; CHECK-NEXT: [[SEL:%.*]] = phi i8 [ 0, %[[ENTRY]] ], [ [[SMIN]], %[[SELECT_FALSE_SINK]] ]
; CHECK-NEXT: [[SUNKADDR:%.*]] = getelementptr i8, ptr [[DST]], i64 [[OFFSET]]
+; CHECK-NEXT: [[SMIN:%.*]] = tail call i8 @llvm.smin.i8(i8 [[Y]], i8 0)
+; CHECK-NEXT: [[CMP:%.*]] = icmp slt i8 [[Z]], 0
+; CHECK-NEXT: [[SEL:%.*]] = select i1 [[CMP]], i8 0, i8 [[SMIN]]
; CHECK-NEXT: store i8 [[SEL]], ptr [[SUNKADDR]], align 1
; CHECK-NEXT: ret void
;
More information about the llvm-commits
mailing list