[llvm] [X86] Truncates i64 add arithmetic to i32 when known zeroes in upper 33 bits (PR #191619)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Apr 13 03:31:08 PDT 2026
https://github.com/2elliti updated https://github.com/llvm/llvm-project/pull/191619
>From 5b648c57e1ba538d0852b0242de80ad9ec60463d Mon Sep 17 00:00:00 2001
From: 2elliti <forstoic724321 at gmail.com>
Date: Sat, 11 Apr 2026 16:35:22 +0530
Subject: [PATCH 1/4] [X86] Truncates i64 add arithmetic to i32 when known
zeroes in upper 33 bits
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 25 +++
llvm/test/CodeGen/X86/truncate-i64-add.ll | 178 ++++++++++++++++++++++
2 files changed, 203 insertions(+)
create mode 100644 llvm/test/CodeGen/X86/truncate-i64-add.ll
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 7ccfc412ff184..e2e6a5ad1d5e9 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -59402,6 +59402,31 @@ static SDValue combineAdd(SDNode *N, SelectionDAG &DAG,
if (SDValue IFMA52 = matchVPMADD52(N, DAG, DL, VT, Subtarget))
return IFMA52;
+ // If upper 33 bits of operands are 0, truncates opcode from i64 to i32.
+ if (VT == MVT::i64) {
+ APInt mask = APInt::getHighBitsSet(64, 33);
+ if (DAG.MaskedValueIsZero(Op0, mask) && DAG.MaskedValueIsZero(Op1, mask)) {
+ // Truncate operands MVT::i64 -> MVT::i32
+ SDValue X = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Op0);
+ SDValue Y = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Op1);
+
+ // now check for NUW and NSW
+ SDNodeFlags flags;
+ // No unsigned wrap, both operands has their upper 33bits 0, making their
+ // sum lower then max unsigned int32.
+ flags.setNoUnsignedWrap(true);
+ // Now check id node had NSW set true or false.
+ bool isNSW = N->getFlags().hasNoSignedWrap();
+
+ // Verify if new nodes has NSW.
+ isNSW = isNSW & DAG.willNotOverflowAdd(true, X, Y);
+ flags.setNoSignedWrap(isNSW);
+ auto addl = DAG.getNode(ISD::ADD, DL, MVT::i32, X, Y, flags);
+
+ return DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, addl);
+ }
+ }
+
return combineAddOrSubToADCOrSBB(N, DL, DAG);
}
diff --git a/llvm/test/CodeGen/X86/truncate-i64-add.ll b/llvm/test/CodeGen/X86/truncate-i64-add.ll
new file mode 100644
index 0000000000000..b51e0d954163d
--- /dev/null
+++ b/llvm/test/CodeGen/X86/truncate-i64-add.ll
@@ -0,0 +1,178 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=i686-unknown-unknown | FileCheck %s --check-prefix=X86
+; RUN: llc < %s -mtriple=x86_64-linux | FileCheck %s --check-prefix=X64-LINUX
+; RUN: llc < %s -mtriple=x86_64-win32 | FileCheck %s --check-prefix=X64-WIN32
+
+; Both operands upper 33 bits are zero
+define i64 @test_add_i64_narrow(i64 %a, i64 %b) {
+; X86-LABEL: test_add_i64_narrow:
+; X86: # %bb.0:
+; X86-NEXT: movl $2147483647, %eax # imm = 0x7FFFFFFF
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: andl %eax, %ecx
+; X86-NEXT: andl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: addl %ecx, %eax
+; X86-NEXT: xorl %edx, %edx
+; X86-NEXT: retl
+;
+; X64-LINUX-LABEL: test_add_i64_narrow:
+; X64-LINUX: # %bb.0:
+; X64-LINUX-NEXT: andl $2147483647, %edi # imm = 0x7FFFFFFF
+; X64-LINUX-NEXT: andl $2147483647, %esi # imm = 0x7FFFFFFF
+; X64-LINUX-NEXT: leal (%rsi,%rdi), %eax
+; X64-LINUX-NEXT: retq
+;
+; X64-WIN32-LABEL: test_add_i64_narrow:
+; X64-WIN32: # %bb.0:
+; X64-WIN32-NEXT: andl $2147483647, %ecx # imm = 0x7FFFFFFF
+; X64-WIN32-NEXT: andl $2147483647, %edx # imm = 0x7FFFFFFF
+; X64-WIN32-NEXT: leal (%rdx,%rcx), %eax
+; X64-WIN32-NEXT: retq
+ %a_masked = and i64 %a, 2147483647
+ %b_masked = and i64 %b, 2147483647
+ %res = add i64 %a_masked, %b_masked
+ ret i64 %res
+}
+
+; Both operands zero-extended from i16, should truncate to 32-bit add
+define i64 @add_zext_i16(i16 %a, i16 %b) nounwind {
+; X86-LABEL: add_zext_i16:
+; X86: # %bb.0:
+; X86-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: addl %ecx, %eax
+; X86-NEXT: xorl %edx, %edx
+; X86-NEXT: retl
+;
+; X64-LINUX-LABEL: add_zext_i16:
+; X64-LINUX: # %bb.0:
+; X64-LINUX-NEXT: movzwl %si, %ecx
+; X64-LINUX-NEXT: movzwl %di, %eax
+; X64-LINUX-NEXT: addl %ecx, %eax
+; X64-LINUX-NEXT: retq
+;
+; X64-WIN32-LABEL: add_zext_i16:
+; X64-WIN32: # %bb.0:
+; X64-WIN32-NEXT: movzwl %dx, %edx
+; X64-WIN32-NEXT: movzwl %cx, %eax
+; X64-WIN32-NEXT: addl %edx, %eax
+; X64-WIN32-NEXT: retq
+ %za = zext i16 %a to i64
+ %zb = zext i16 %b to i64
+ %sum = add nuw nsw i64 %za, %zb
+ ret i64 %sum
+}
+
+; One operand is a small constant
+define i64 @add_zext_i16_const(i16 %a) nounwind {
+; X86-LABEL: add_zext_i16_const:
+; X86: # %bb.0:
+; X86-NEXT: movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: addl $42, %eax
+; X86-NEXT: xorl %edx, %edx
+; X86-NEXT: retl
+;
+; X64-LINUX-LABEL: add_zext_i16_const:
+; X64-LINUX: # %bb.0:
+; X64-LINUX-NEXT: movzwl %di, %eax
+; X64-LINUX-NEXT: addl $42, %eax
+; X64-LINUX-NEXT: retq
+;
+; X64-WIN32-LABEL: add_zext_i16_const:
+; X64-WIN32: # %bb.0:
+; X64-WIN32-NEXT: movzwl %cx, %eax
+; X64-WIN32-NEXT: addl $42, %eax
+; X64-WIN32-NEXT: retq
+ %za = zext i16 %a to i64
+ %sum = add nuw nsw i64 %za, 42
+ ret i64 %sum
+}
+
+; Bit 32 set — must NOT truncate
+define i64 @add_bit32_set(i16 %a) nounwind {
+; X86-LABEL: add_bit32_set:
+; X86: # %bb.0:
+; X86-NEXT: movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: addl $42, %eax
+; X86-NEXT: movl $1, %edx
+; X86-NEXT: retl
+;
+; X64-LINUX-LABEL: add_bit32_set:
+; X64-LINUX: # %bb.0:
+; X64-LINUX-NEXT: movzwl %di, %ecx
+; X64-LINUX-NEXT: movabsq $4294967338, %rax # imm = 0x10000002A
+; X64-LINUX-NEXT: addq %rcx, %rax
+; X64-LINUX-NEXT: retq
+;
+; X64-WIN32-LABEL: add_bit32_set:
+; X64-WIN32: # %bb.0:
+; X64-WIN32-NEXT: movzwl %cx, %ecx
+; X64-WIN32-NEXT: movabsq $4294967338, %rax # imm = 0x10000002A
+; X64-WIN32-NEXT: addq %rcx, %rax
+; X64-WIN32-NEXT: retq
+ %za = zext i16 %a to i64
+ %or = or i64 %za, 4294967296
+ %sum = add nuw nsw i64 %or, 42
+ ret i64 %sum
+}
+
+; Sign-extended — must NOT truncate
+define i64 @add_sext_i16(i16 %a, i16 %b) nounwind {
+; X86-LABEL: add_sext_i16:
+; X86: # %bb.0:
+; X86-NEXT: movswl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: sarl $31, %edx
+; X86-NEXT: movzwl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: addl %ecx, %eax
+; X86-NEXT: adcl $0, %edx
+; X86-NEXT: retl
+;
+; X64-LINUX-LABEL: add_sext_i16:
+; X64-LINUX: # %bb.0:
+; X64-LINUX-NEXT: # kill: def $edi killed $edi def $rdi
+; X64-LINUX-NEXT: movswq %di, %rcx
+; X64-LINUX-NEXT: movzwl %si, %eax
+; X64-LINUX-NEXT: addq %rcx, %rax
+; X64-LINUX-NEXT: retq
+;
+; X64-WIN32-LABEL: add_sext_i16:
+; X64-WIN32: # %bb.0:
+; X64-WIN32-NEXT: movswq %cx, %rcx
+; X64-WIN32-NEXT: movzwl %dx, %eax
+; X64-WIN32-NEXT: addq %rcx, %rax
+; X64-WIN32-NEXT: retq
+ %sa = sext i16 %a to i64
+ %zb = zext i16 %b to i64
+ %sum = add nuw nsw i64 %sa, %zb
+ ret i64 %sum
+}
+
+; i8 source
+define i64 @test_i8_add(i8 %a, i8 %b) nounwind {
+; X86-LABEL: test_i8_add:
+; X86: # %bb.0:
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: addl %ecx, %eax
+; X86-NEXT: xorl %edx, %edx
+; X86-NEXT: retl
+;
+; X64-LINUX-LABEL: test_i8_add:
+; X64-LINUX: # %bb.0:
+; X64-LINUX-NEXT: movzbl %sil, %ecx
+; X64-LINUX-NEXT: movzbl %dil, %eax
+; X64-LINUX-NEXT: addl %ecx, %eax
+; X64-LINUX-NEXT: retq
+;
+; X64-WIN32-LABEL: test_i8_add:
+; X64-WIN32: # %bb.0:
+; X64-WIN32-NEXT: movzbl %dl, %edx
+; X64-WIN32-NEXT: movzbl %cl, %eax
+; X64-WIN32-NEXT: addl %edx, %eax
+; X64-WIN32-NEXT: retq
+ %za = zext i8 %a to i64
+ %zb = zext i8 %b to i64
+ %sum = add i64 %za, %zb ; no nuw/nsw — should still fire
+ ret i64 %sum
+}
>From 70f3aaa3c5e31bd2f4c6d9678e76b4ddf6eaf42d Mon Sep 17 00:00:00 2001
From: 2elliti <forstoic724321 at gmail.com>
Date: Mon, 13 Apr 2026 00:44:38 +0530
Subject: [PATCH 2/4] Address review feedback: simplify wrap flags and update
tests
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 14 +--
llvm/test/CodeGen/X86/truncate-i64-add.ll | 137 +++++++++-------------
2 files changed, 63 insertions(+), 88 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index e2e6a5ad1d5e9..15c7264f61249 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -59411,19 +59411,15 @@ static SDValue combineAdd(SDNode *N, SelectionDAG &DAG,
SDValue Y = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Op1);
// now check for NUW and NSW
- SDNodeFlags flags;
+ SDNodeFlags Flags;
// No unsigned wrap, both operands has their upper 33bits 0, making their
// sum lower then max unsigned int32.
- flags.setNoUnsignedWrap(true);
- // Now check id node had NSW set true or false.
- bool isNSW = N->getFlags().hasNoSignedWrap();
+ Flags.setNoUnsignedWrap(true);
+ Flags.setNoSignedWrap(DAG.willNotOverflowAdd(true, X, Y));
- // Verify if new nodes has NSW.
- isNSW = isNSW & DAG.willNotOverflowAdd(true, X, Y);
- flags.setNoSignedWrap(isNSW);
- auto addl = DAG.getNode(ISD::ADD, DL, MVT::i32, X, Y, flags);
+ SDValue Add32 = DAG.getNode(ISD::ADD, DL, MVT::i32, X, Y, Flags);
- return DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, addl);
+ return DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Add32);
}
}
diff --git a/llvm/test/CodeGen/X86/truncate-i64-add.ll b/llvm/test/CodeGen/X86/truncate-i64-add.ll
index b51e0d954163d..1ea5374b91694 100644
--- a/llvm/test/CodeGen/X86/truncate-i64-add.ll
+++ b/llvm/test/CodeGen/X86/truncate-i64-add.ll
@@ -1,7 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc < %s -mtriple=i686-unknown-unknown | FileCheck %s --check-prefix=X86
-; RUN: llc < %s -mtriple=x86_64-linux | FileCheck %s --check-prefix=X64-LINUX
-; RUN: llc < %s -mtriple=x86_64-win32 | FileCheck %s --check-prefix=X64-WIN32
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefix=X64
; Both operands upper 33 bits are zero
define i64 @test_add_i64_narrow(i64 %a, i64 %b) {
@@ -15,19 +14,12 @@ define i64 @test_add_i64_narrow(i64 %a, i64 %b) {
; X86-NEXT: xorl %edx, %edx
; X86-NEXT: retl
;
-; X64-LINUX-LABEL: test_add_i64_narrow:
-; X64-LINUX: # %bb.0:
-; X64-LINUX-NEXT: andl $2147483647, %edi # imm = 0x7FFFFFFF
-; X64-LINUX-NEXT: andl $2147483647, %esi # imm = 0x7FFFFFFF
-; X64-LINUX-NEXT: leal (%rsi,%rdi), %eax
-; X64-LINUX-NEXT: retq
-;
-; X64-WIN32-LABEL: test_add_i64_narrow:
-; X64-WIN32: # %bb.0:
-; X64-WIN32-NEXT: andl $2147483647, %ecx # imm = 0x7FFFFFFF
-; X64-WIN32-NEXT: andl $2147483647, %edx # imm = 0x7FFFFFFF
-; X64-WIN32-NEXT: leal (%rdx,%rcx), %eax
-; X64-WIN32-NEXT: retq
+; X64-LABEL: test_add_i64_narrow:
+; X64: # %bb.0:
+; X64-NEXT: andl $2147483647, %edi # imm = 0x7FFFFFFF
+; X64-NEXT: andl $2147483647, %esi # imm = 0x7FFFFFFF
+; X64-NEXT: leal (%rsi,%rdi), %eax
+; X64-NEXT: retq
%a_masked = and i64 %a, 2147483647
%b_masked = and i64 %b, 2147483647
%res = add i64 %a_masked, %b_masked
@@ -44,19 +36,12 @@ define i64 @add_zext_i16(i16 %a, i16 %b) nounwind {
; X86-NEXT: xorl %edx, %edx
; X86-NEXT: retl
;
-; X64-LINUX-LABEL: add_zext_i16:
-; X64-LINUX: # %bb.0:
-; X64-LINUX-NEXT: movzwl %si, %ecx
-; X64-LINUX-NEXT: movzwl %di, %eax
-; X64-LINUX-NEXT: addl %ecx, %eax
-; X64-LINUX-NEXT: retq
-;
-; X64-WIN32-LABEL: add_zext_i16:
-; X64-WIN32: # %bb.0:
-; X64-WIN32-NEXT: movzwl %dx, %edx
-; X64-WIN32-NEXT: movzwl %cx, %eax
-; X64-WIN32-NEXT: addl %edx, %eax
-; X64-WIN32-NEXT: retq
+; X64-LABEL: add_zext_i16:
+; X64: # %bb.0:
+; X64-NEXT: movzwl %si, %ecx
+; X64-NEXT: movzwl %di, %eax
+; X64-NEXT: addl %ecx, %eax
+; X64-NEXT: retq
%za = zext i16 %a to i64
%zb = zext i16 %b to i64
%sum = add nuw nsw i64 %za, %zb
@@ -72,17 +57,11 @@ define i64 @add_zext_i16_const(i16 %a) nounwind {
; X86-NEXT: xorl %edx, %edx
; X86-NEXT: retl
;
-; X64-LINUX-LABEL: add_zext_i16_const:
-; X64-LINUX: # %bb.0:
-; X64-LINUX-NEXT: movzwl %di, %eax
-; X64-LINUX-NEXT: addl $42, %eax
-; X64-LINUX-NEXT: retq
-;
-; X64-WIN32-LABEL: add_zext_i16_const:
-; X64-WIN32: # %bb.0:
-; X64-WIN32-NEXT: movzwl %cx, %eax
-; X64-WIN32-NEXT: addl $42, %eax
-; X64-WIN32-NEXT: retq
+; X64-LABEL: add_zext_i16_const:
+; X64: # %bb.0:
+; X64-NEXT: movzwl %di, %eax
+; X64-NEXT: addl $42, %eax
+; X64-NEXT: retq
%za = zext i16 %a to i64
%sum = add nuw nsw i64 %za, 42
ret i64 %sum
@@ -97,19 +76,12 @@ define i64 @add_bit32_set(i16 %a) nounwind {
; X86-NEXT: movl $1, %edx
; X86-NEXT: retl
;
-; X64-LINUX-LABEL: add_bit32_set:
-; X64-LINUX: # %bb.0:
-; X64-LINUX-NEXT: movzwl %di, %ecx
-; X64-LINUX-NEXT: movabsq $4294967338, %rax # imm = 0x10000002A
-; X64-LINUX-NEXT: addq %rcx, %rax
-; X64-LINUX-NEXT: retq
-;
-; X64-WIN32-LABEL: add_bit32_set:
-; X64-WIN32: # %bb.0:
-; X64-WIN32-NEXT: movzwl %cx, %ecx
-; X64-WIN32-NEXT: movabsq $4294967338, %rax # imm = 0x10000002A
-; X64-WIN32-NEXT: addq %rcx, %rax
-; X64-WIN32-NEXT: retq
+; X64-LABEL: add_bit32_set:
+; X64: # %bb.0:
+; X64-NEXT: movzwl %di, %ecx
+; X64-NEXT: movabsq $4294967338, %rax # imm = 0x10000002A
+; X64-NEXT: addq %rcx, %rax
+; X64-NEXT: retq
%za = zext i16 %a to i64
%or = or i64 %za, 4294967296
%sum = add nuw nsw i64 %or, 42
@@ -128,20 +100,13 @@ define i64 @add_sext_i16(i16 %a, i16 %b) nounwind {
; X86-NEXT: adcl $0, %edx
; X86-NEXT: retl
;
-; X64-LINUX-LABEL: add_sext_i16:
-; X64-LINUX: # %bb.0:
-; X64-LINUX-NEXT: # kill: def $edi killed $edi def $rdi
-; X64-LINUX-NEXT: movswq %di, %rcx
-; X64-LINUX-NEXT: movzwl %si, %eax
-; X64-LINUX-NEXT: addq %rcx, %rax
-; X64-LINUX-NEXT: retq
-;
-; X64-WIN32-LABEL: add_sext_i16:
-; X64-WIN32: # %bb.0:
-; X64-WIN32-NEXT: movswq %cx, %rcx
-; X64-WIN32-NEXT: movzwl %dx, %eax
-; X64-WIN32-NEXT: addq %rcx, %rax
-; X64-WIN32-NEXT: retq
+; X64-LABEL: add_sext_i16:
+; X64: # %bb.0:
+; X64-NEXT: # kill: def $edi killed $edi def $rdi
+; X64-NEXT: movswq %di, %rcx
+; X64-NEXT: movzwl %si, %eax
+; X64-NEXT: addq %rcx, %rax
+; X64-NEXT: retq
%sa = sext i16 %a to i64
%zb = zext i16 %b to i64
%sum = add nuw nsw i64 %sa, %zb
@@ -158,21 +123,35 @@ define i64 @test_i8_add(i8 %a, i8 %b) nounwind {
; X86-NEXT: xorl %edx, %edx
; X86-NEXT: retl
;
-; X64-LINUX-LABEL: test_i8_add:
-; X64-LINUX: # %bb.0:
-; X64-LINUX-NEXT: movzbl %sil, %ecx
-; X64-LINUX-NEXT: movzbl %dil, %eax
-; X64-LINUX-NEXT: addl %ecx, %eax
-; X64-LINUX-NEXT: retq
-;
-; X64-WIN32-LABEL: test_i8_add:
-; X64-WIN32: # %bb.0:
-; X64-WIN32-NEXT: movzbl %dl, %edx
-; X64-WIN32-NEXT: movzbl %cl, %eax
-; X64-WIN32-NEXT: addl %edx, %eax
-; X64-WIN32-NEXT: retq
+; X64-LABEL: test_i8_add:
+; X64: # %bb.0:
+; X64-NEXT: movzbl %sil, %ecx
+; X64-NEXT: movzbl %dil, %eax
+; X64-NEXT: addl %ecx, %eax
+; X64-NEXT: retq
%za = zext i8 %a to i64
%zb = zext i8 %b to i64
%sum = add i64 %za, %zb ; no nuw/nsw — should still fire
ret i64 %sum
}
+
+define i64 @add_zext_i32(i32 %a, i32 %b) {
+; X86-LABEL: add_zext_i32:
+; X86: # %bb.0:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: xorl %edx, %edx
+; X86-NEXT: addl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: setb %dl
+; X86-NEXT: retl
+;
+; X64-LABEL: add_zext_i32:
+; X64: # %bb.0:
+; X64-NEXT: movl %edi, %ecx
+; X64-NEXT: movl %esi, %eax
+; X64-NEXT: addq %rcx, %rax
+; X64-NEXT: retq
+ %za = zext i32 %a to i64
+ %zb = zext i32 %b to i64
+ %sum = add i64 %za, %zb
+ ret i64 %sum
+}
>From b1083407b9fc2752e469d9560fd91955bbad70ad Mon Sep 17 00:00:00 2001
From: 2elliti <forstoic724321 at gmail.com>
Date: Mon, 13 Apr 2026 15:04:59 +0530
Subject: [PATCH 3/4] update tests
---
llvm/test/CodeGen/X86/bit-manip-i256.ll | 163 +++---
llvm/test/CodeGen/X86/bit-manip-i512.ll | 549 +++++++++---------
llvm/test/CodeGen/X86/bitcnt-big-integer.ll | 455 +++++++--------
llvm/test/CodeGen/X86/bittest-big-integer.ll | 92 +--
llvm/test/CodeGen/X86/bsf.ll | 4 +-
llvm/test/CodeGen/X86/bsr.ll | 8 +-
.../CodeGen/X86/dag-update-nodetomatch.ll | 4 +-
llvm/test/CodeGen/X86/dagcombine-select.ll | 28 +-
llvm/test/CodeGen/X86/dagcombine-shifts.ll | 3 +-
llvm/test/CodeGen/X86/divrem8_ext.ll | 2 +-
.../X86/fold-int-pow2-with-fmul-or-fdiv.ll | 28 +-
llvm/test/CodeGen/X86/h-registers-1.ll | 28 +-
llvm/test/CodeGen/X86/narrow-add-i64.ll | 8 +-
llvm/test/CodeGen/X86/popcnt.ll | 168 +++---
llvm/test/CodeGen/X86/pr173924.ll | 105 +++-
.../CodeGen/X86/scheduler-backtracking.ll | 284 ++++-----
llvm/test/CodeGen/X86/vector-compress.ll | 414 +++++++------
llvm/test/CodeGen/X86/xaluo.ll | 2 +-
18 files changed, 1170 insertions(+), 1175 deletions(-)
diff --git a/llvm/test/CodeGen/X86/bit-manip-i256.ll b/llvm/test/CodeGen/X86/bit-manip-i256.ll
index 9bebab1834e41..af1327a367f3d 100644
--- a/llvm/test/CodeGen/X86/bit-manip-i256.ll
+++ b/llvm/test/CodeGen/X86/bit-manip-i256.ll
@@ -2413,22 +2413,23 @@ define i256 @isolate_msb_i256(i256 %a0, i256 %idx) nounwind {
; SSE-NEXT: movq %rcx, %rax
; SSE-NEXT: movq %rdx, %r9
; SSE-NEXT: orq %r8, %r9
-; SSE-NEXT: bsrq %rsi, %rcx
-; SSE-NEXT: orq %rax, %rsi
-; SSE-NEXT: bsrq %r8, %r10
-; SSE-NEXT: xorq $63, %r10
+; SSE-NEXT: movq %rsi, %r10
+; SSE-NEXT: orq %rcx, %r10
+; SSE-NEXT: bsrq %r8, %rcx
+; SSE-NEXT: xorq $63, %rcx
; SSE-NEXT: bsrq %rax, %r11
-; SSE-NEXT: xorq $63, %r11
-; SSE-NEXT: orq $64, %r11
+; SSE-NEXT: xorl $63, %r11d
+; SSE-NEXT: orl $64, %r11d
; SSE-NEXT: testq %r8, %r8
-; SSE-NEXT: cmovneq %r10, %r11
-; SSE-NEXT: bsrq %rdx, %r10
-; SSE-NEXT: xorq $63, %r10
-; SSE-NEXT: xorq $63, %rcx
-; SSE-NEXT: orq $64, %rcx
+; SSE-NEXT: cmovneq %rcx, %r11
+; SSE-NEXT: bsrq %rdx, %rbx
+; SSE-NEXT: xorl $63, %ebx
+; SSE-NEXT: bsrq %rsi, %rcx
+; SSE-NEXT: xorl $63, %ecx
+; SSE-NEXT: orl $64, %ecx
; SSE-NEXT: testq %rdx, %rdx
-; SSE-NEXT: cmovneq %r10, %rcx
-; SSE-NEXT: orq $128, %rcx
+; SSE-NEXT: cmovnel %ebx, %ecx
+; SSE-NEXT: subl $-128, %ecx
; SSE-NEXT: orq %r8, %rax
; SSE-NEXT: cmovneq %r11, %rcx
; SSE-NEXT: xorps %xmm0, %xmm0
@@ -2442,27 +2443,27 @@ define i256 @isolate_msb_i256(i256 %a0, i256 %idx) nounwind {
; SSE-NEXT: movzbl %al, %eax
; SSE-NEXT: movq $0, -{{[0-9]+}}(%rsp)
; SSE-NEXT: movq -40(%rsp,%rax,8), %rdx
-; SSE-NEXT: movq -48(%rsp,%rax,8), %r8
-; SSE-NEXT: movq %r8, %r10
-; SSE-NEXT: shrdq %cl, %rdx, %r10
+; SSE-NEXT: movq -48(%rsp,%rax,8), %rsi
+; SSE-NEXT: movq %rsi, %r8
+; SSE-NEXT: shrdq %cl, %rdx, %r8
; SSE-NEXT: movq -56(%rsp,%rax,8), %r11
; SSE-NEXT: movq %r11, %rbx
-; SSE-NEXT: shrdq %cl, %r8, %rbx
-; SSE-NEXT: movq -64(%rsp,%rax,8), %r8
+; SSE-NEXT: shrdq %cl, %rsi, %rbx
+; SSE-NEXT: movq -64(%rsp,%rax,8), %rsi
; SSE-NEXT: shrq %cl, %rdx
; SSE-NEXT: # kill: def $cl killed $cl killed $rcx
-; SSE-NEXT: shrdq %cl, %r11, %r8
+; SSE-NEXT: shrdq %cl, %r11, %rsi
; SSE-NEXT: xorl %ecx, %ecx
-; SSE-NEXT: orq %r9, %rsi
+; SSE-NEXT: orq %r9, %r10
; SSE-NEXT: cmoveq %rcx, %rbx
-; SSE-NEXT: cmoveq %rcx, %r10
; SSE-NEXT: cmoveq %rcx, %r8
+; SSE-NEXT: cmoveq %rcx, %rsi
; SSE-NEXT: movq %rdi, %rax
; SSE-NEXT: cmoveq %rcx, %rdx
; SSE-NEXT: movq %rdx, 24(%rdi)
-; SSE-NEXT: movq %r10, 16(%rdi)
+; SSE-NEXT: movq %r8, 16(%rdi)
; SSE-NEXT: movq %rbx, 8(%rdi)
-; SSE-NEXT: movq %r8, (%rdi)
+; SSE-NEXT: movq %rsi, (%rdi)
; SSE-NEXT: popq %rbx
; SSE-NEXT: retq
;
@@ -2477,15 +2478,15 @@ define i256 @isolate_msb_i256(i256 %a0, i256 %idx) nounwind {
; AVX2-NEXT: orq %rax, %rsi
; AVX2-NEXT: lzcntq %r8, %r10
; AVX2-NEXT: lzcntq %rax, %r11
-; AVX2-NEXT: addq $64, %r11
+; AVX2-NEXT: addl $64, %r11d
; AVX2-NEXT: testq %r8, %r8
; AVX2-NEXT: cmovneq %r10, %r11
; AVX2-NEXT: xorl %r10d, %r10d
; AVX2-NEXT: lzcntq %rdx, %r10
-; AVX2-NEXT: addq $64, %rcx
+; AVX2-NEXT: addl $64, %ecx
; AVX2-NEXT: testq %rdx, %rdx
-; AVX2-NEXT: cmovneq %r10, %rcx
-; AVX2-NEXT: subq $-128, %rcx
+; AVX2-NEXT: cmovnel %r10d, %ecx
+; AVX2-NEXT: subl $-128, %ecx
; AVX2-NEXT: orq %r8, %rax
; AVX2-NEXT: cmovneq %r11, %rcx
; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
@@ -2530,14 +2531,14 @@ define i256 @isolate_msb_i256(i256 %a0, i256 %idx) nounwind {
; AVX512F-NEXT: orq %rax, %rsi
; AVX512F-NEXT: lzcntq %r8, %r10
; AVX512F-NEXT: lzcntq %rax, %r11
-; AVX512F-NEXT: addq $64, %r11
+; AVX512F-NEXT: addl $64, %r11d
; AVX512F-NEXT: testq %r8, %r8
; AVX512F-NEXT: cmovneq %r10, %r11
; AVX512F-NEXT: lzcntq %rdx, %r10
-; AVX512F-NEXT: addq $64, %rcx
+; AVX512F-NEXT: addl $64, %ecx
; AVX512F-NEXT: testq %rdx, %rdx
-; AVX512F-NEXT: cmovneq %r10, %rcx
-; AVX512F-NEXT: subq $-128, %rcx
+; AVX512F-NEXT: cmovnel %r10d, %ecx
+; AVX512F-NEXT: subl $-128, %ecx
; AVX512F-NEXT: orq %r8, %rax
; AVX512F-NEXT: cmovneq %r11, %rcx
; AVX512F-NEXT: vmovaps {{.*#+}} zmm0 = [0,0,0,9223372036854775808,0,0,0,0]
@@ -2579,14 +2580,14 @@ define i256 @isolate_msb_i256(i256 %a0, i256 %idx) nounwind {
; AVX512VL-NEXT: orq %rax, %rsi
; AVX512VL-NEXT: lzcntq %r8, %r10
; AVX512VL-NEXT: lzcntq %rax, %r11
-; AVX512VL-NEXT: addq $64, %r11
+; AVX512VL-NEXT: addl $64, %r11d
; AVX512VL-NEXT: testq %r8, %r8
; AVX512VL-NEXT: cmovneq %r10, %r11
; AVX512VL-NEXT: lzcntq %rdx, %r10
-; AVX512VL-NEXT: addq $64, %rcx
+; AVX512VL-NEXT: addl $64, %ecx
; AVX512VL-NEXT: testq %rdx, %rdx
-; AVX512VL-NEXT: cmovneq %r10, %rcx
-; AVX512VL-NEXT: subq $-128, %rcx
+; AVX512VL-NEXT: cmovnel %r10d, %ecx
+; AVX512VL-NEXT: subl $-128, %ecx
; AVX512VL-NEXT: orq %r8, %rax
; AVX512VL-NEXT: cmovneq %r11, %rcx
; AVX512VL-NEXT: vxorps %xmm0, %xmm0, %xmm0
@@ -2631,14 +2632,14 @@ define i256 @isolate_msb_i256(i256 %a0, i256 %idx) nounwind {
; AVX512VBMI-NEXT: orq %rax, %rsi
; AVX512VBMI-NEXT: lzcntq %r8, %r10
; AVX512VBMI-NEXT: lzcntq %rax, %r11
-; AVX512VBMI-NEXT: addq $64, %r11
+; AVX512VBMI-NEXT: addl $64, %r11d
; AVX512VBMI-NEXT: testq %r8, %r8
; AVX512VBMI-NEXT: cmovneq %r10, %r11
; AVX512VBMI-NEXT: lzcntq %rdx, %r10
-; AVX512VBMI-NEXT: addq $64, %rcx
+; AVX512VBMI-NEXT: addl $64, %ecx
; AVX512VBMI-NEXT: testq %rdx, %rdx
-; AVX512VBMI-NEXT: cmovneq %r10, %rcx
-; AVX512VBMI-NEXT: subq $-128, %rcx
+; AVX512VBMI-NEXT: cmovnel %r10d, %ecx
+; AVX512VBMI-NEXT: subl $-128, %ecx
; AVX512VBMI-NEXT: orq %r8, %rax
; AVX512VBMI-NEXT: cmovneq %r11, %rcx
; AVX512VBMI-NEXT: vxorps %xmm0, %xmm0, %xmm0
@@ -2697,18 +2698,18 @@ define i256 @isolate_msb_i256_vector(<4 x i64> %v0, i256 %idx) nounwind {
; SSE2-NEXT: bsrq %r8, %r9
; SSE2-NEXT: xorq $63, %r9
; SSE2-NEXT: bsrq %rsi, %rsi
-; SSE2-NEXT: xorq $63, %rsi
-; SSE2-NEXT: orq $64, %rsi
+; SSE2-NEXT: xorl $63, %esi
+; SSE2-NEXT: orl $64, %esi
; SSE2-NEXT: testq %r8, %r8
; SSE2-NEXT: cmovneq %r9, %rsi
; SSE2-NEXT: bsrq %rdx, %r8
-; SSE2-NEXT: xorq $63, %r8
+; SSE2-NEXT: xorl $63, %r8d
; SSE2-NEXT: bsrq %rcx, %rcx
-; SSE2-NEXT: xorq $63, %rcx
-; SSE2-NEXT: orq $64, %rcx
+; SSE2-NEXT: xorl $63, %ecx
+; SSE2-NEXT: orl $64, %ecx
; SSE2-NEXT: testq %rdx, %rdx
-; SSE2-NEXT: cmovneq %r8, %rcx
-; SSE2-NEXT: orq $128, %rcx
+; SSE2-NEXT: cmovnel %r8d, %ecx
+; SSE2-NEXT: subl $-128, %ecx
; SSE2-NEXT: pcmpeqd %xmm2, %xmm1
; SSE2-NEXT: movmskps %xmm1, %edx
; SSE2-NEXT: xorl $15, %edx
@@ -2755,22 +2756,22 @@ define i256 @isolate_msb_i256_vector(<4 x i64> %v0, i256 %idx) nounwind {
; SSE42-NEXT: bsrq %rsi, %r8
; SSE42-NEXT: xorq $63, %r8
; SSE42-NEXT: bsrq %rax, %rax
-; SSE42-NEXT: xorq $63, %rax
-; SSE42-NEXT: orq $64, %rax
+; SSE42-NEXT: xorl $63, %eax
+; SSE42-NEXT: orl $64, %eax
; SSE42-NEXT: testq %rsi, %rsi
; SSE42-NEXT: cmovneq %r8, %rax
; SSE42-NEXT: bsrq %rdx, %rsi
-; SSE42-NEXT: xorq $63, %rsi
+; SSE42-NEXT: xorl $63, %esi
; SSE42-NEXT: bsrq %rcx, %rcx
-; SSE42-NEXT: xorq $63, %rcx
-; SSE42-NEXT: orq $64, %rcx
+; SSE42-NEXT: xorl $63, %ecx
+; SSE42-NEXT: orl $64, %ecx
; SSE42-NEXT: testq %rdx, %rdx
-; SSE42-NEXT: cmovneq %rsi, %rcx
+; SSE42-NEXT: cmovnel %esi, %ecx
; SSE42-NEXT: xorps %xmm2, %xmm2
; SSE42-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
; SSE42-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
; SSE42-NEXT: movaps %xmm2, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: orq $128, %rcx
+; SSE42-NEXT: subl $-128, %ecx
; SSE42-NEXT: ptest %xmm1, %xmm1
; SSE42-NEXT: cmovneq %rax, %rcx
; SSE42-NEXT: movabsq $-9223372036854775808, %rax # imm = 0x8000000000000000
@@ -2813,16 +2814,16 @@ define i256 @isolate_msb_i256_vector(<4 x i64> %v0, i256 %idx) nounwind {
; AVX2-NEXT: vpextrq $1, %xmm1, %r8
; AVX2-NEXT: lzcntq %r8, %rcx
; AVX2-NEXT: lzcntq %rsi, %r9
-; AVX2-NEXT: addq $64, %r9
+; AVX2-NEXT: addl $64, %r9d
; AVX2-NEXT: testq %r8, %r8
; AVX2-NEXT: cmovneq %rcx, %r9
; AVX2-NEXT: lzcntq %rdx, %r10
; AVX2-NEXT: xorl %ecx, %ecx
; AVX2-NEXT: lzcntq %rax, %rcx
-; AVX2-NEXT: addq $64, %rcx
+; AVX2-NEXT: addl $64, %ecx
; AVX2-NEXT: testq %rdx, %rdx
-; AVX2-NEXT: cmovneq %r10, %rcx
-; AVX2-NEXT: subq $-128, %rcx
+; AVX2-NEXT: cmovnel %r10d, %ecx
+; AVX2-NEXT: subl $-128, %ecx
; AVX2-NEXT: orq %r8, %rsi
; AVX2-NEXT: cmovneq %r9, %rcx
; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
@@ -2863,7 +2864,7 @@ define i256 @isolate_msb_i256_vector(<4 x i64> %v0, i256 %idx) nounwind {
; AVX512F-NEXT: vpermq {{.*#+}} ymm1 = ymm0[3,2,1,0]
; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k0
; AVX512F-NEXT: vplzcntq %zmm1, %zmm1
-; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,64,128,192]
; AVX512F-NEXT: kshiftlw $12, %k0, %k0
; AVX512F-NEXT: kshiftrw $12, %k0, %k1
; AVX512F-NEXT: vpcompressq %zmm1, %zmm1 {%k1}
@@ -2901,7 +2902,7 @@ define i256 @isolate_msb_i256_vector(<4 x i64> %v0, i256 %idx) nounwind {
; AVX512VL-NEXT: vpermq {{.*#+}} ymm1 = ymm0[3,2,1,0]
; AVX512VL-NEXT: vptestmq %ymm1, %ymm1, %k1
; AVX512VL-NEXT: vplzcntq %ymm1, %ymm1
-; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,64,128,192]
; AVX512VL-NEXT: vpcompressq %ymm1, %ymm1 {%k1}
; AVX512VL-NEXT: vxorps %xmm2, %xmm2, %xmm2
; AVX512VL-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
@@ -2940,7 +2941,7 @@ define i256 @isolate_msb_i256_vector(<4 x i64> %v0, i256 %idx) nounwind {
; AVX512VBMI-NEXT: vpermq {{.*#+}} ymm1 = ymm0[3,2,1,0]
; AVX512VBMI-NEXT: vptestmq %ymm1, %ymm1, %k1
; AVX512VBMI-NEXT: vplzcntq %ymm1, %ymm1
-; AVX512VBMI-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512VBMI-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,64,128,192]
; AVX512VBMI-NEXT: vpcompressq %ymm1, %ymm1 {%k1}
; AVX512VBMI-NEXT: vxorps %xmm2, %xmm2, %xmm2
; AVX512VBMI-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
@@ -2997,18 +2998,18 @@ define i256 @isolate_msb_i256_load(ptr %p0, i256 %idx) nounwind {
; SSE2-NEXT: bsrq %r8, %rcx
; SSE2-NEXT: xorq $63, %rcx
; SSE2-NEXT: bsrq %rdx, %r10
-; SSE2-NEXT: xorq $63, %r10
-; SSE2-NEXT: orq $64, %r10
+; SSE2-NEXT: xorl $63, %r10d
+; SSE2-NEXT: orl $64, %r10d
; SSE2-NEXT: testq %r8, %r8
; SSE2-NEXT: cmovneq %rcx, %r10
; SSE2-NEXT: bsrq %r9, %r11
-; SSE2-NEXT: xorq $63, %r11
+; SSE2-NEXT: xorl $63, %r11d
; SSE2-NEXT: bsrq (%rsi), %rcx
-; SSE2-NEXT: xorq $63, %rcx
-; SSE2-NEXT: orq $64, %rcx
+; SSE2-NEXT: xorl $63, %ecx
+; SSE2-NEXT: orl $64, %ecx
; SSE2-NEXT: testq %r9, %r9
-; SSE2-NEXT: cmovneq %r11, %rcx
-; SSE2-NEXT: orq $128, %rcx
+; SSE2-NEXT: cmovnel %r11d, %ecx
+; SSE2-NEXT: subl $-128, %ecx
; SSE2-NEXT: orq %r8, %rdx
; SSE2-NEXT: cmovneq %r10, %rcx
; SSE2-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
@@ -3053,19 +3054,19 @@ define i256 @isolate_msb_i256_load(ptr %p0, i256 %idx) nounwind {
; SSE42-NEXT: bsrq %rdx, %rcx
; SSE42-NEXT: xorq $63, %rcx
; SSE42-NEXT: bsrq %rax, %r8
-; SSE42-NEXT: xorq $63, %r8
-; SSE42-NEXT: orq $64, %r8
+; SSE42-NEXT: xorl $63, %r8d
+; SSE42-NEXT: orl $64, %r8d
; SSE42-NEXT: testq %rdx, %rdx
; SSE42-NEXT: cmovneq %rcx, %r8
; SSE42-NEXT: movq 8(%rsi), %r9
; SSE42-NEXT: bsrq %r9, %r10
; SSE42-NEXT: bsrq (%rsi), %rcx
-; SSE42-NEXT: xorq $63, %r10
-; SSE42-NEXT: xorq $63, %rcx
-; SSE42-NEXT: orq $64, %rcx
+; SSE42-NEXT: xorl $63, %r10d
+; SSE42-NEXT: xorl $63, %ecx
+; SSE42-NEXT: orl $64, %ecx
; SSE42-NEXT: testq %r9, %r9
-; SSE42-NEXT: cmovneq %r10, %rcx
-; SSE42-NEXT: orq $128, %rcx
+; SSE42-NEXT: cmovnel %r10d, %ecx
+; SSE42-NEXT: subl $-128, %ecx
; SSE42-NEXT: orq %rdx, %rax
; SSE42-NEXT: cmovneq %r8, %rcx
; SSE42-NEXT: xorps %xmm1, %xmm1
@@ -3108,17 +3109,17 @@ define i256 @isolate_msb_i256_load(ptr %p0, i256 %idx) nounwind {
; AVX2-NEXT: movq 24(%rsi), %rdx
; AVX2-NEXT: lzcntq %rdx, %rcx
; AVX2-NEXT: lzcntq %rax, %r8
-; AVX2-NEXT: addq $64, %r8
+; AVX2-NEXT: addl $64, %r8d
; AVX2-NEXT: testq %rdx, %rdx
; AVX2-NEXT: cmovneq %rcx, %r8
; AVX2-NEXT: movq 8(%rsi), %r9
; AVX2-NEXT: lzcntq %r9, %r10
; AVX2-NEXT: xorl %ecx, %ecx
; AVX2-NEXT: lzcntq (%rsi), %rcx
-; AVX2-NEXT: addq $64, %rcx
+; AVX2-NEXT: addl $64, %ecx
; AVX2-NEXT: testq %r9, %r9
-; AVX2-NEXT: cmovneq %r10, %rcx
-; AVX2-NEXT: subq $-128, %rcx
+; AVX2-NEXT: cmovnel %r10d, %ecx
+; AVX2-NEXT: subl $-128, %ecx
; AVX2-NEXT: orq %rdx, %rax
; AVX2-NEXT: cmovneq %r8, %rcx
; AVX2-NEXT: vmovdqu (%rsi), %ymm0
@@ -3161,7 +3162,7 @@ define i256 @isolate_msb_i256_load(ptr %p0, i256 %idx) nounwind {
; AVX512F-NEXT: vpermq {{.*#+}} ymm1 = ymm0[3,2,1,0]
; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k0
; AVX512F-NEXT: vplzcntq %zmm1, %zmm1
-; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,64,128,192]
; AVX512F-NEXT: kshiftlw $12, %k0, %k0
; AVX512F-NEXT: kshiftrw $12, %k0, %k1
; AVX512F-NEXT: vpcompressq %zmm1, %zmm1 {%k1} {z}
@@ -3199,7 +3200,7 @@ define i256 @isolate_msb_i256_load(ptr %p0, i256 %idx) nounwind {
; AVX512VL-NEXT: vpermq {{.*#+}} ymm1 = ymm0[3,2,1,0]
; AVX512VL-NEXT: vptestmq %ymm1, %ymm1, %k1
; AVX512VL-NEXT: vplzcntq %ymm1, %ymm1
-; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,64,128,192]
; AVX512VL-NEXT: vpcompressq %ymm1, %ymm1 {%k1} {z}
; AVX512VL-NEXT: vxorps %xmm2, %xmm2, %xmm2
; AVX512VL-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
@@ -3239,7 +3240,7 @@ define i256 @isolate_msb_i256_load(ptr %p0, i256 %idx) nounwind {
; AVX512VBMI-NEXT: vpermq {{.*#+}} ymm1 = ymm0[3,2,1,0]
; AVX512VBMI-NEXT: vptestmq %ymm1, %ymm1, %k1
; AVX512VBMI-NEXT: vplzcntq %ymm1, %ymm1
-; AVX512VBMI-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512VBMI-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,64,128,192]
; AVX512VBMI-NEXT: vpcompressq %ymm1, %ymm1 {%k1} {z}
; AVX512VBMI-NEXT: vxorps %xmm2, %xmm2, %xmm2
; AVX512VBMI-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
diff --git a/llvm/test/CodeGen/X86/bit-manip-i512.ll b/llvm/test/CodeGen/X86/bit-manip-i512.ll
index 1688e97d26272..c06dc1bcfae8d 100644
--- a/llvm/test/CodeGen/X86/bit-manip-i512.ll
+++ b/llvm/test/CodeGen/X86/bit-manip-i512.ll
@@ -4127,39 +4127,39 @@ define i512 @isolate_msb_i512(i512 %a0, i512 %idx) nounwind {
; SSE-NEXT: bsrq %r14, %r15
; SSE-NEXT: xorq $63, %r15
; SSE-NEXT: bsrq %rbx, %r12
-; SSE-NEXT: xorq $63, %r12
-; SSE-NEXT: orq $64, %r12
+; SSE-NEXT: xorl $63, %r12d
+; SSE-NEXT: orl $64, %r12d
; SSE-NEXT: testq %r14, %r14
; SSE-NEXT: cmovneq %r15, %r12
; SSE-NEXT: bsrq %r11, %r13
-; SSE-NEXT: xorq $63, %r13
+; SSE-NEXT: xorl $63, %r13d
; SSE-NEXT: bsrq %r9, %r15
-; SSE-NEXT: xorq $63, %r15
-; SSE-NEXT: orq $64, %r15
+; SSE-NEXT: xorl $63, %r15d
+; SSE-NEXT: orl $64, %r15d
; SSE-NEXT: testq %r11, %r11
-; SSE-NEXT: cmovneq %r13, %r15
-; SSE-NEXT: orq $128, %r15
+; SSE-NEXT: cmovnel %r13d, %r15d
+; SSE-NEXT: subl $-128, %r15d
; SSE-NEXT: movq %rbx, %r13
; SSE-NEXT: orq %r14, %r13
; SSE-NEXT: cmovneq %r12, %r15
; SSE-NEXT: bsrq %r8, %r12
-; SSE-NEXT: xorq $63, %r12
+; SSE-NEXT: xorl $63, %r12d
; SSE-NEXT: bsrq %rcx, %r13
-; SSE-NEXT: xorq $63, %r13
-; SSE-NEXT: orq $64, %r13
+; SSE-NEXT: xorl $63, %r13d
+; SSE-NEXT: orl $64, %r13d
; SSE-NEXT: testq %r8, %r8
-; SSE-NEXT: cmovneq %r12, %r13
+; SSE-NEXT: cmovnel %r12d, %r13d
; SSE-NEXT: bsrq %rdx, %r12
-; SSE-NEXT: xorq $63, %r12
+; SSE-NEXT: xorl $63, %r12d
; SSE-NEXT: bsrq %rsi, %rsi
-; SSE-NEXT: xorq $63, %rsi
-; SSE-NEXT: orq $64, %rsi
+; SSE-NEXT: xorl $63, %esi
+; SSE-NEXT: orl $64, %esi
; SSE-NEXT: testq %rdx, %rdx
-; SSE-NEXT: cmovneq %r12, %rsi
-; SSE-NEXT: orq $128, %rsi
+; SSE-NEXT: cmovnel %r12d, %esi
+; SSE-NEXT: subl $-128, %esi
; SSE-NEXT: orq %r8, %rcx
-; SSE-NEXT: cmovneq %r13, %rsi
-; SSE-NEXT: orq $256, %rsi # imm = 0x100
+; SSE-NEXT: cmovnel %r13d, %esi
+; SSE-NEXT: addl $256, %esi # imm = 0x100
; SSE-NEXT: orq %r14, %r11
; SSE-NEXT: orq %rbx, %r9
; SSE-NEXT: orq %r11, %r9
@@ -4230,124 +4230,119 @@ define i512 @isolate_msb_i512(i512 %a0, i512 %idx) nounwind {
;
; AVX2-LABEL: isolate_msb_i512:
; AVX2: # %bb.0:
-; AVX2-NEXT: pushq %rbp
; AVX2-NEXT: pushq %r15
; AVX2-NEXT: pushq %r14
; AVX2-NEXT: pushq %r13
; AVX2-NEXT: pushq %r12
; AVX2-NEXT: pushq %rbx
-; AVX2-NEXT: pushq %rax
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rbx
; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r14
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r15
-; AVX2-NEXT: movq %r8, %rax
-; AVX2-NEXT: orq %r15, %rax
+; AVX2-NEXT: movq %r8, %r11
+; AVX2-NEXT: orq %r14, %r11
; AVX2-NEXT: movq %rdx, %r10
-; AVX2-NEXT: orq %rbx, %r10
; AVX2-NEXT: orq %rax, %r10
-; AVX2-NEXT: movq %rcx, %rax
-; AVX2-NEXT: orq %r14, %rax
+; AVX2-NEXT: orq %r11, %r10
+; AVX2-NEXT: movq %rcx, %r15
+; AVX2-NEXT: orq %rbx, %r15
; AVX2-NEXT: movq %rsi, %r11
; AVX2-NEXT: orq %r9, %r11
-; AVX2-NEXT: orq %rax, %r11
-; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: lzcntq %r15, %rax
+; AVX2-NEXT: orq %r15, %r11
+; AVX2-NEXT: xorl %r15d, %r15d
+; AVX2-NEXT: lzcntq %r14, %r15
+; AVX2-NEXT: xorl %r12d, %r12d
+; AVX2-NEXT: lzcntq %rbx, %r12
+; AVX2-NEXT: addl $64, %r12d
+; AVX2-NEXT: testq %r14, %r14
+; AVX2-NEXT: cmovneq %r15, %r12
; AVX2-NEXT: xorl %r13d, %r13d
-; AVX2-NEXT: lzcntq %r14, %r13
-; AVX2-NEXT: addq $64, %r13
-; AVX2-NEXT: testq %r15, %r15
-; AVX2-NEXT: cmovneq %rax, %r13
-; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: lzcntq %rbx, %rax
+; AVX2-NEXT: lzcntq %rax, %r13
+; AVX2-NEXT: xorl %r15d, %r15d
+; AVX2-NEXT: lzcntq %r9, %r15
+; AVX2-NEXT: addl $64, %r15d
+; AVX2-NEXT: testq %rax, %rax
+; AVX2-NEXT: cmovnel %r13d, %r15d
+; AVX2-NEXT: subl $-128, %r15d
+; AVX2-NEXT: movq %rbx, %r13
+; AVX2-NEXT: orq %r14, %r13
+; AVX2-NEXT: cmovneq %r12, %r15
; AVX2-NEXT: xorl %r12d, %r12d
-; AVX2-NEXT: lzcntq %r9, %r12
-; AVX2-NEXT: addq $64, %r12
-; AVX2-NEXT: testq %rbx, %rbx
-; AVX2-NEXT: cmovneq %rax, %r12
-; AVX2-NEXT: subq $-128, %r12
-; AVX2-NEXT: movq %r14, %rax
-; AVX2-NEXT: orq %r15, %rax
-; AVX2-NEXT: cmovneq %r13, %r12
-; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: lzcntq %r8, %rax
+; AVX2-NEXT: lzcntq %r8, %r12
; AVX2-NEXT: xorl %r13d, %r13d
; AVX2-NEXT: lzcntq %rcx, %r13
-; AVX2-NEXT: addq $64, %r13
+; AVX2-NEXT: addl $64, %r13d
; AVX2-NEXT: testq %r8, %r8
-; AVX2-NEXT: cmovneq %rax, %r13
-; AVX2-NEXT: xorl %ebp, %ebp
-; AVX2-NEXT: lzcntq %rdx, %rbp
-; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: lzcntq %rsi, %rax
-; AVX2-NEXT: addq $64, %rax
+; AVX2-NEXT: cmovnel %r12d, %r13d
+; AVX2-NEXT: xorl %r12d, %r12d
+; AVX2-NEXT: lzcntq %rdx, %r12
+; AVX2-NEXT: lzcntq %rsi, %rsi
+; AVX2-NEXT: addl $64, %esi
; AVX2-NEXT: testq %rdx, %rdx
-; AVX2-NEXT: cmovneq %rbp, %rax
-; AVX2-NEXT: subq $-128, %rax
+; AVX2-NEXT: cmovnel %r12d, %esi
+; AVX2-NEXT: subl $-128, %esi
; AVX2-NEXT: orq %r8, %rcx
-; AVX2-NEXT: cmovneq %r13, %rax
-; AVX2-NEXT: addq $256, %rax # imm = 0x100
-; AVX2-NEXT: orq %r15, %rbx
-; AVX2-NEXT: orq %r14, %r9
+; AVX2-NEXT: cmovnel %r13d, %esi
+; AVX2-NEXT: addl $256, %esi # imm = 0x100
+; AVX2-NEXT: orq %r14, %rax
; AVX2-NEXT: orq %rbx, %r9
-; AVX2-NEXT: cmovneq %r12, %rax
+; AVX2-NEXT: orq %rax, %r9
+; AVX2-NEXT: cmovneq %r15, %rsi
; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovaps {{.*#+}} ymm1 = [0,0,0,9223372036854775808]
; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movl %eax, %ecx
+; AVX2-NEXT: movl %esi, %ecx
; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: shrl $3, %eax
-; AVX2-NEXT: andl $56, %eax
-; AVX2-NEXT: movq -72(%rsp,%rax), %rbx
-; AVX2-NEXT: movq -80(%rsp,%rax), %r8
-; AVX2-NEXT: movq %r8, %rdx
-; AVX2-NEXT: shrdq %cl, %rbx, %rdx
-; AVX2-NEXT: movq -88(%rsp,%rax), %r9
-; AVX2-NEXT: movq %r9, %rsi
-; AVX2-NEXT: shrdq %cl, %r8, %rsi
-; AVX2-NEXT: movq -96(%rsp,%rax), %r14
-; AVX2-NEXT: movq %r14, %r8
-; AVX2-NEXT: shrdq %cl, %r9, %r8
-; AVX2-NEXT: movq -104(%rsp,%rax), %r15
-; AVX2-NEXT: movq %r15, %r9
-; AVX2-NEXT: shrdq %cl, %r14, %r9
-; AVX2-NEXT: movq -112(%rsp,%rax), %r13
-; AVX2-NEXT: movq %r13, %r14
-; AVX2-NEXT: shrdq %cl, %r15, %r14
-; AVX2-NEXT: movq -128(%rsp,%rax), %r15
-; AVX2-NEXT: movq -120(%rsp,%rax), %rax
-; AVX2-NEXT: movq %rax, %r12
+; AVX2-NEXT: shrl $3, %esi
+; AVX2-NEXT: andl $56, %esi
+; AVX2-NEXT: movq -72(%rsp,%rsi), %r14
+; AVX2-NEXT: movq -80(%rsp,%rsi), %rax
+; AVX2-NEXT: movq %rax, %rdx
+; AVX2-NEXT: shrdq %cl, %r14, %rdx
+; AVX2-NEXT: movq -88(%rsp,%rsi), %rbx
+; AVX2-NEXT: movq %rbx, %r8
+; AVX2-NEXT: shrdq %cl, %rax, %r8
+; AVX2-NEXT: movq -96(%rsp,%rsi), %rax
+; AVX2-NEXT: movq %rax, %r9
+; AVX2-NEXT: shrdq %cl, %rbx, %r9
+; AVX2-NEXT: movq -104(%rsp,%rsi), %r12
+; AVX2-NEXT: movq %r12, %rbx
+; AVX2-NEXT: shrdq %cl, %rax, %rbx
+; AVX2-NEXT: movq -112(%rsp,%rsi), %rax
+; AVX2-NEXT: movq %rax, %r15
+; AVX2-NEXT: shrdq %cl, %r12, %r15
+; AVX2-NEXT: movq -128(%rsp,%rsi), %r12
+; AVX2-NEXT: movq -120(%rsp,%rsi), %r13
+; AVX2-NEXT: movq %r13, %rsi
+; AVX2-NEXT: shrdq %cl, %rax, %rsi
; AVX2-NEXT: shrdq %cl, %r13, %r12
-; AVX2-NEXT: shrdq %cl, %rax, %r15
; AVX2-NEXT: movq %rdi, %rax
; AVX2-NEXT: xorl %edi, %edi
; AVX2-NEXT: orq %r10, %r11
-; AVX2-NEXT: shrxq %rcx, %rbx, %rcx
-; AVX2-NEXT: cmoveq %rdi, %r12
-; AVX2-NEXT: cmoveq %rdi, %r14
+; AVX2-NEXT: shrxq %rcx, %r14, %rcx
+; AVX2-NEXT: cmoveq %rdi, %rsi
+; AVX2-NEXT: cmoveq %rdi, %r15
+; AVX2-NEXT: cmoveq %rdi, %rbx
; AVX2-NEXT: cmoveq %rdi, %r9
; AVX2-NEXT: cmoveq %rdi, %r8
-; AVX2-NEXT: cmoveq %rdi, %rsi
; AVX2-NEXT: cmoveq %rdi, %rdx
-; AVX2-NEXT: cmoveq %rdi, %r15
+; AVX2-NEXT: cmoveq %rdi, %r12
; AVX2-NEXT: cmoveq %rdi, %rcx
; AVX2-NEXT: movq %rcx, 56(%rax)
; AVX2-NEXT: movq %rdx, 48(%rax)
-; AVX2-NEXT: movq %rsi, 40(%rax)
-; AVX2-NEXT: movq %r8, 32(%rax)
-; AVX2-NEXT: movq %r9, 24(%rax)
-; AVX2-NEXT: movq %r14, 16(%rax)
-; AVX2-NEXT: movq %r12, 8(%rax)
-; AVX2-NEXT: movq %r15, (%rax)
-; AVX2-NEXT: addq $8, %rsp
+; AVX2-NEXT: movq %r8, 40(%rax)
+; AVX2-NEXT: movq %r9, 32(%rax)
+; AVX2-NEXT: movq %rbx, 24(%rax)
+; AVX2-NEXT: movq %r15, 16(%rax)
+; AVX2-NEXT: movq %rsi, 8(%rax)
+; AVX2-NEXT: movq %r12, (%rax)
; AVX2-NEXT: popq %rbx
; AVX2-NEXT: popq %r12
; AVX2-NEXT: popq %r13
; AVX2-NEXT: popq %r14
; AVX2-NEXT: popq %r15
-; AVX2-NEXT: popq %rbp
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
@@ -4379,7 +4374,7 @@ define i512 @isolate_msb_i512(i512 %a0, i512 %idx) nounwind {
; AVX512F-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512F-NEXT: vplzcntq %zmm0, %zmm0
-; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
+; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,64,128,192,256,320,384,448]
; AVX512F-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
; AVX512F-NEXT: vmovq %xmm0, %rcx
; AVX512F-NEXT: movabsq $-9223372036854775808, %rsi # imm = 0x8000000000000000
@@ -4424,7 +4419,7 @@ define i512 @isolate_msb_i512(i512 %a0, i512 %idx) nounwind {
; AVX512VL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VL-NEXT: vplzcntq %zmm0, %zmm0
-; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
+; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,64,128,192,256,320,384,448]
; AVX512VL-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
; AVX512VL-NEXT: vmovq %xmm0, %rcx
; AVX512VL-NEXT: movabsq $-9223372036854775808, %rsi # imm = 0x8000000000000000
@@ -4470,7 +4465,7 @@ define i512 @isolate_msb_i512(i512 %a0, i512 %idx) nounwind {
; AVX512VBMI-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
; AVX512VBMI-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VBMI-NEXT: vplzcntq %zmm0, %zmm0
-; AVX512VBMI-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
+; AVX512VBMI-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,64,128,192,256,320,384,448]
; AVX512VBMI-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
; AVX512VBMI-NEXT: vmovq %xmm0, %rcx
; AVX512VBMI-NEXT: movabsq $-9223372036854775808, %rsi # imm = 0x8000000000000000
@@ -4526,43 +4521,43 @@ define i512 @isolate_msb_i512_vector(<8 x i64> %v0, i512 %idx) nounwind {
; SSE2-NEXT: bsrq %rbx, %r14
; SSE2-NEXT: xorq $63, %r14
; SSE2-NEXT: bsrq %r10, %r10
-; SSE2-NEXT: xorq $63, %r10
-; SSE2-NEXT: orq $64, %r10
+; SSE2-NEXT: xorl $63, %r10d
+; SSE2-NEXT: orl $64, %r10d
; SSE2-NEXT: testq %rbx, %rbx
; SSE2-NEXT: cmovneq %r14, %r10
; SSE2-NEXT: bsrq %r11, %rbx
-; SSE2-NEXT: xorq $63, %rbx
+; SSE2-NEXT: xorl $63, %ebx
; SSE2-NEXT: bsrq %r9, %r9
-; SSE2-NEXT: xorq $63, %r9
-; SSE2-NEXT: orq $64, %r9
+; SSE2-NEXT: xorl $63, %r9d
+; SSE2-NEXT: orl $64, %r9d
; SSE2-NEXT: testq %r11, %r11
-; SSE2-NEXT: cmovneq %rbx, %r9
-; SSE2-NEXT: orq $128, %r9
+; SSE2-NEXT: cmovnel %ebx, %r9d
+; SSE2-NEXT: subl $-128, %r9d
; SSE2-NEXT: por %xmm3, %xmm2
; SSE2-NEXT: pcmpeqd %xmm4, %xmm3
; SSE2-NEXT: movmskps %xmm3, %r11d
; SSE2-NEXT: xorl $15, %r11d
; SSE2-NEXT: cmovneq %r10, %r9
; SSE2-NEXT: bsrq %rsi, %r10
-; SSE2-NEXT: xorq $63, %r10
+; SSE2-NEXT: xorl $63, %r10d
; SSE2-NEXT: bsrq %r8, %r8
-; SSE2-NEXT: xorq $63, %r8
-; SSE2-NEXT: orq $64, %r8
+; SSE2-NEXT: xorl $63, %r8d
+; SSE2-NEXT: orl $64, %r8d
; SSE2-NEXT: testq %rsi, %rsi
-; SSE2-NEXT: cmovneq %r10, %r8
+; SSE2-NEXT: cmovnel %r10d, %r8d
; SSE2-NEXT: bsrq %rcx, %rsi
-; SSE2-NEXT: xorq $63, %rsi
+; SSE2-NEXT: xorl $63, %esi
; SSE2-NEXT: bsrq %rdx, %rdx
-; SSE2-NEXT: xorq $63, %rdx
-; SSE2-NEXT: orq $64, %rdx
+; SSE2-NEXT: xorl $63, %edx
+; SSE2-NEXT: orl $64, %edx
; SSE2-NEXT: testq %rcx, %rcx
-; SSE2-NEXT: cmovneq %rsi, %rdx
-; SSE2-NEXT: orq $128, %rdx
+; SSE2-NEXT: cmovnel %esi, %edx
+; SSE2-NEXT: subl $-128, %edx
; SSE2-NEXT: pcmpeqd %xmm4, %xmm1
; SSE2-NEXT: movmskps %xmm1, %ecx
; SSE2-NEXT: xorl $15, %ecx
-; SSE2-NEXT: cmovneq %r8, %rdx
-; SSE2-NEXT: orq $256, %rdx # imm = 0x100
+; SSE2-NEXT: cmovnel %r8d, %edx
+; SSE2-NEXT: addl $256, %edx # imm = 0x100
; SSE2-NEXT: pcmpeqd %xmm4, %xmm2
; SSE2-NEXT: movmskps %xmm2, %ecx
; SSE2-NEXT: xorl $15, %ecx
@@ -4646,39 +4641,39 @@ define i512 @isolate_msb_i512_vector(<8 x i64> %v0, i512 %idx) nounwind {
; SSE42-NEXT: bsrq %r11, %rbx
; SSE42-NEXT: xorq $63, %rbx
; SSE42-NEXT: bsrq %r10, %r10
-; SSE42-NEXT: xorq $63, %r10
-; SSE42-NEXT: orq $64, %r10
+; SSE42-NEXT: xorl $63, %r10d
+; SSE42-NEXT: orl $64, %r10d
; SSE42-NEXT: testq %r11, %r11
; SSE42-NEXT: cmovneq %rbx, %r10
; SSE42-NEXT: bsrq %r9, %r11
-; SSE42-NEXT: xorq $63, %r11
+; SSE42-NEXT: xorl $63, %r11d
; SSE42-NEXT: bsrq %rsi, %rsi
-; SSE42-NEXT: xorq $63, %rsi
-; SSE42-NEXT: orq $64, %rsi
+; SSE42-NEXT: xorl $63, %esi
+; SSE42-NEXT: orl $64, %esi
; SSE42-NEXT: testq %r9, %r9
-; SSE42-NEXT: cmovneq %r11, %rsi
-; SSE42-NEXT: orq $128, %rsi
+; SSE42-NEXT: cmovnel %r11d, %esi
+; SSE42-NEXT: subl $-128, %esi
; SSE42-NEXT: ptest %xmm3, %xmm3
; SSE42-NEXT: cmovneq %r10, %rsi
; SSE42-NEXT: bsrq %rdx, %r9
-; SSE42-NEXT: xorq $63, %r9
+; SSE42-NEXT: xorl $63, %r9d
; SSE42-NEXT: bsrq %r8, %r8
-; SSE42-NEXT: xorq $63, %r8
-; SSE42-NEXT: orq $64, %r8
+; SSE42-NEXT: xorl $63, %r8d
+; SSE42-NEXT: orl $64, %r8d
; SSE42-NEXT: testq %rdx, %rdx
-; SSE42-NEXT: cmovneq %r9, %r8
+; SSE42-NEXT: cmovnel %r9d, %r8d
; SSE42-NEXT: bsrq %rcx, %rdx
-; SSE42-NEXT: xorq $63, %rdx
+; SSE42-NEXT: xorl $63, %edx
; SSE42-NEXT: bsrq %rax, %rax
-; SSE42-NEXT: xorq $63, %rax
-; SSE42-NEXT: orq $64, %rax
+; SSE42-NEXT: xorl $63, %eax
+; SSE42-NEXT: orl $64, %eax
; SSE42-NEXT: testq %rcx, %rcx
-; SSE42-NEXT: cmovneq %rdx, %rax
+; SSE42-NEXT: cmovnel %edx, %eax
; SSE42-NEXT: por %xmm2, %xmm0
-; SSE42-NEXT: orq $128, %rax
+; SSE42-NEXT: subl $-128, %eax
; SSE42-NEXT: ptest %xmm1, %xmm1
-; SSE42-NEXT: cmovneq %r8, %rax
-; SSE42-NEXT: orq $256, %rax # imm = 0x100
+; SSE42-NEXT: cmovnel %r8d, %eax
+; SSE42-NEXT: addl $256, %eax # imm = 0x100
; SSE42-NEXT: por %xmm3, %xmm2
; SSE42-NEXT: ptest %xmm2, %xmm2
; SSE42-NEXT: cmovneq %rsi, %rax
@@ -4769,35 +4764,35 @@ define i512 @isolate_msb_i512_vector(<8 x i64> %v0, i512 %idx) nounwind {
; AVX2-NEXT: lzcntq %rsi, %rbx
; AVX2-NEXT: xorl %r14d, %r14d
; AVX2-NEXT: lzcntq %rdx, %r14
-; AVX2-NEXT: addq $64, %r14
+; AVX2-NEXT: addl $64, %r14d
; AVX2-NEXT: testq %rsi, %rsi
; AVX2-NEXT: cmovneq %rbx, %r14
; AVX2-NEXT: xorl %ebx, %ebx
; AVX2-NEXT: lzcntq %r11, %rbx
; AVX2-NEXT: lzcntq %r10, %r10
-; AVX2-NEXT: addq $64, %r10
+; AVX2-NEXT: addl $64, %r10d
; AVX2-NEXT: testq %r11, %r11
-; AVX2-NEXT: cmovneq %rbx, %r10
-; AVX2-NEXT: subq $-128, %r10
+; AVX2-NEXT: cmovnel %ebx, %r10d
+; AVX2-NEXT: subl $-128, %r10d
; AVX2-NEXT: orq %rsi, %rdx
; AVX2-NEXT: cmovneq %r14, %r10
; AVX2-NEXT: xorl %edx, %edx
; AVX2-NEXT: lzcntq %rcx, %rdx
; AVX2-NEXT: xorl %esi, %esi
; AVX2-NEXT: lzcntq %rax, %rsi
-; AVX2-NEXT: addq $64, %rsi
+; AVX2-NEXT: addl $64, %esi
; AVX2-NEXT: testq %rcx, %rcx
-; AVX2-NEXT: cmovneq %rdx, %rsi
+; AVX2-NEXT: cmovnel %edx, %esi
; AVX2-NEXT: xorl %edx, %edx
; AVX2-NEXT: lzcntq %r9, %rdx
; AVX2-NEXT: lzcntq %r8, %r8
-; AVX2-NEXT: addq $64, %r8
+; AVX2-NEXT: addl $64, %r8d
; AVX2-NEXT: testq %r9, %r9
-; AVX2-NEXT: cmovneq %rdx, %r8
-; AVX2-NEXT: subq $-128, %r8
+; AVX2-NEXT: cmovnel %edx, %r8d
+; AVX2-NEXT: subl $-128, %r8d
; AVX2-NEXT: orq %rcx, %rax
-; AVX2-NEXT: cmovneq %rsi, %r8
-; AVX2-NEXT: addq $256, %r8 # imm = 0x100
+; AVX2-NEXT: cmovnel %esi, %r8d
+; AVX2-NEXT: addl $256, %r8d # imm = 0x100
; AVX2-NEXT: vptest %ymm1, %ymm1
; AVX2-NEXT: cmovneq %r10, %r8
; AVX2-NEXT: vmovdqu %ymm2, -{{[0-9]+}}(%rsp)
@@ -4865,7 +4860,7 @@ define i512 @isolate_msb_i512_vector(<8 x i64> %v0, i512 %idx) nounwind {
; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0
; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512F-NEXT: vplzcntq %zmm0, %zmm0
-; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
+; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,64,128,192,256,320,384,448]
; AVX512F-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
; AVX512F-NEXT: vmovq %xmm0, %rdx
; AVX512F-NEXT: movabsq $-9223372036854775808, %rsi # imm = 0x8000000000000000
@@ -4893,7 +4888,7 @@ define i512 @isolate_msb_i512_vector(<8 x i64> %v0, i512 %idx) nounwind {
; AVX512VL-NEXT: vpermq %zmm0, %zmm1, %zmm0
; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VL-NEXT: vplzcntq %zmm0, %zmm0
-; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
+; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,64,128,192,256,320,384,448]
; AVX512VL-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
; AVX512VL-NEXT: vmovq %xmm0, %rdx
; AVX512VL-NEXT: movabsq $-9223372036854775808, %rsi # imm = 0x8000000000000000
@@ -4922,7 +4917,7 @@ define i512 @isolate_msb_i512_vector(<8 x i64> %v0, i512 %idx) nounwind {
; AVX512VBMI-NEXT: vpermq %zmm0, %zmm1, %zmm0
; AVX512VBMI-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VBMI-NEXT: vplzcntq %zmm0, %zmm0
-; AVX512VBMI-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
+; AVX512VBMI-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,64,128,192,256,320,384,448]
; AVX512VBMI-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
; AVX512VBMI-NEXT: vmovq %xmm0, %rdx
; AVX512VBMI-NEXT: movabsq $-9223372036854775808, %rsi # imm = 0x8000000000000000
@@ -4956,10 +4951,10 @@ define i512 @isolate_msb_i512_load(ptr %p0, i512 %idx) nounwind {
; SSE2-NEXT: pushq %r12
; SSE2-NEXT: pushq %rbx
; SSE2-NEXT: pushq %rax
-; SSE2-NEXT: movq 8(%rsi), %r9
+; SSE2-NEXT: movq 8(%rsi), %r8
; SSE2-NEXT: movq 16(%rsi), %rcx
-; SSE2-NEXT: movq 24(%rsi), %r8
-; SSE2-NEXT: movq 48(%rsi), %rdx
+; SSE2-NEXT: movq 24(%rsi), %rdx
+; SSE2-NEXT: movq 48(%rsi), %r10
; SSE2-NEXT: movq 56(%rsi), %r11
; SSE2-NEXT: movdqa 32(%rsi), %xmm1
; SSE2-NEXT: movdqa 48(%rsi), %xmm2
@@ -4972,47 +4967,47 @@ define i512 @isolate_msb_i512_load(ptr %p0, i512 %idx) nounwind {
; SSE2-NEXT: pcmpeqd %xmm0, %xmm3
; SSE2-NEXT: movmskps %xmm3, %eax
; SSE2-NEXT: xorl $15, %eax
-; SSE2-NEXT: bsrq %r11, %r10
-; SSE2-NEXT: xorq $63, %r10
-; SSE2-NEXT: bsrq %rdx, %rbx
-; SSE2-NEXT: xorq $63, %rbx
-; SSE2-NEXT: orq $64, %rbx
+; SSE2-NEXT: bsrq %r11, %r9
+; SSE2-NEXT: xorq $63, %r9
+; SSE2-NEXT: bsrq %r10, %rbx
+; SSE2-NEXT: xorl $63, %ebx
+; SSE2-NEXT: orl $64, %ebx
; SSE2-NEXT: testq %r11, %r11
-; SSE2-NEXT: cmovneq %r10, %rbx
+; SSE2-NEXT: cmovneq %r9, %rbx
; SSE2-NEXT: movq 40(%rsi), %r14
; SSE2-NEXT: bsrq %r14, %r15
-; SSE2-NEXT: bsrq 32(%rsi), %r10
-; SSE2-NEXT: xorq $63, %r15
-; SSE2-NEXT: xorq $63, %r10
-; SSE2-NEXT: orq $64, %r10
+; SSE2-NEXT: bsrq 32(%rsi), %r9
+; SSE2-NEXT: xorl $63, %r15d
+; SSE2-NEXT: xorl $63, %r9d
+; SSE2-NEXT: orl $64, %r9d
; SSE2-NEXT: testq %r14, %r14
-; SSE2-NEXT: cmovneq %r15, %r10
-; SSE2-NEXT: orq $128, %r10
-; SSE2-NEXT: orq %r11, %rdx
-; SSE2-NEXT: cmovneq %rbx, %r10
-; SSE2-NEXT: bsrq %r8, %rdx
-; SSE2-NEXT: xorq $63, %rdx
+; SSE2-NEXT: cmovnel %r15d, %r9d
+; SSE2-NEXT: subl $-128, %r9d
+; SSE2-NEXT: orq %r11, %r10
+; SSE2-NEXT: cmovneq %rbx, %r9
+; SSE2-NEXT: bsrq %rdx, %r10
+; SSE2-NEXT: xorl $63, %r10d
; SSE2-NEXT: bsrq %rcx, %r11
-; SSE2-NEXT: xorq $63, %r11
-; SSE2-NEXT: orq $64, %r11
+; SSE2-NEXT: xorl $63, %r11d
+; SSE2-NEXT: orl $64, %r11d
+; SSE2-NEXT: testq %rdx, %rdx
+; SSE2-NEXT: cmovnel %r10d, %r11d
+; SSE2-NEXT: bsrq %r8, %r10
+; SSE2-NEXT: xorl $63, %r10d
+; SSE2-NEXT: bsrq (%rsi), %rsi
+; SSE2-NEXT: xorl $63, %esi
+; SSE2-NEXT: orl $64, %esi
; SSE2-NEXT: testq %r8, %r8
-; SSE2-NEXT: cmovneq %rdx, %r11
-; SSE2-NEXT: bsrq %r9, %rbx
-; SSE2-NEXT: xorq $63, %rbx
-; SSE2-NEXT: bsrq (%rsi), %rdx
-; SSE2-NEXT: xorq $63, %rdx
-; SSE2-NEXT: orq $64, %rdx
-; SSE2-NEXT: testq %r9, %r9
-; SSE2-NEXT: cmovneq %rbx, %rdx
-; SSE2-NEXT: orq $128, %rdx
-; SSE2-NEXT: orq %r8, %rcx
-; SSE2-NEXT: cmovneq %r11, %rdx
-; SSE2-NEXT: orq $256, %rdx # imm = 0x100
+; SSE2-NEXT: cmovnel %r10d, %esi
+; SSE2-NEXT: subl $-128, %esi
+; SSE2-NEXT: orq %rdx, %rcx
+; SSE2-NEXT: cmovnel %r11d, %esi
+; SSE2-NEXT: addl $256, %esi # imm = 0x100
; SSE2-NEXT: por %xmm2, %xmm1
; SSE2-NEXT: pcmpeqd %xmm0, %xmm1
; SSE2-NEXT: movmskps %xmm1, %ecx
; SSE2-NEXT: xorl $15, %ecx
-; SSE2-NEXT: cmovneq %r10, %rdx
+; SSE2-NEXT: cmovneq %r9, %rsi
; SSE2-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
@@ -5022,34 +5017,34 @@ define i512 @isolate_msb_i512_load(ptr %p0, i512 %idx) nounwind {
; SSE2-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
; SSE2-NEXT: movdqa %xmm0, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movl %edx, %ecx
+; SSE2-NEXT: movl %esi, %ecx
; SSE2-NEXT: andl $63, %ecx
-; SSE2-NEXT: shrl $3, %edx
-; SSE2-NEXT: andl $56, %edx
+; SSE2-NEXT: shrl $3, %esi
+; SSE2-NEXT: andl $56, %esi
; SSE2-NEXT: movq $0, -{{[0-9]+}}(%rsp)
-; SSE2-NEXT: movq -72(%rsp,%rdx), %rsi
-; SSE2-NEXT: movq -80(%rsp,%rdx), %r10
+; SSE2-NEXT: movq -72(%rsp,%rsi), %rdx
+; SSE2-NEXT: movq -80(%rsp,%rsi), %r10
; SSE2-NEXT: movq %r10, %r8
-; SSE2-NEXT: shrdq %cl, %rsi, %r8
-; SSE2-NEXT: movq -88(%rsp,%rdx), %r11
+; SSE2-NEXT: shrdq %cl, %rdx, %r8
+; SSE2-NEXT: movq -88(%rsp,%rsi), %r11
; SSE2-NEXT: movq %r11, %r9
; SSE2-NEXT: shrdq %cl, %r10, %r9
-; SSE2-NEXT: movq -96(%rsp,%rdx), %rbx
+; SSE2-NEXT: movq -96(%rsp,%rsi), %rbx
; SSE2-NEXT: movq %rbx, %r10
; SSE2-NEXT: shrdq %cl, %r11, %r10
-; SSE2-NEXT: movq -104(%rsp,%rdx), %r14
+; SSE2-NEXT: movq -104(%rsp,%rsi), %r14
; SSE2-NEXT: movq %r14, %r11
; SSE2-NEXT: shrdq %cl, %rbx, %r11
-; SSE2-NEXT: movq -112(%rsp,%rdx), %r15
+; SSE2-NEXT: movq -112(%rsp,%rsi), %r15
; SSE2-NEXT: movq %r15, %rbx
; SSE2-NEXT: shrdq %cl, %r14, %rbx
-; SSE2-NEXT: movq -120(%rsp,%rdx), %r12
+; SSE2-NEXT: movq -120(%rsp,%rsi), %r12
; SSE2-NEXT: movq %r12, %r14
; SSE2-NEXT: shrdq %cl, %r15, %r14
-; SSE2-NEXT: movq -128(%rsp,%rdx), %rdx
-; SSE2-NEXT: shrq %cl, %rsi
+; SSE2-NEXT: movq -128(%rsp,%rsi), %rsi
+; SSE2-NEXT: shrq %cl, %rdx
; SSE2-NEXT: # kill: def $cl killed $cl killed $ecx
-; SSE2-NEXT: shrdq %cl, %r12, %rdx
+; SSE2-NEXT: shrdq %cl, %r12, %rsi
; SSE2-NEXT: xorl %ecx, %ecx
; SSE2-NEXT: testl %eax, %eax
; SSE2-NEXT: cmoveq %rcx, %r14
@@ -5058,17 +5053,17 @@ define i512 @isolate_msb_i512_load(ptr %p0, i512 %idx) nounwind {
; SSE2-NEXT: cmoveq %rcx, %r10
; SSE2-NEXT: cmoveq %rcx, %r9
; SSE2-NEXT: cmoveq %rcx, %r8
-; SSE2-NEXT: cmoveq %rcx, %rdx
-; SSE2-NEXT: movq %rdi, %rax
; SSE2-NEXT: cmoveq %rcx, %rsi
-; SSE2-NEXT: movq %rsi, 56(%rdi)
+; SSE2-NEXT: movq %rdi, %rax
+; SSE2-NEXT: cmoveq %rcx, %rdx
+; SSE2-NEXT: movq %rdx, 56(%rdi)
; SSE2-NEXT: movq %r8, 48(%rdi)
; SSE2-NEXT: movq %r9, 40(%rdi)
; SSE2-NEXT: movq %r10, 32(%rdi)
; SSE2-NEXT: movq %r11, 24(%rdi)
; SSE2-NEXT: movq %rbx, 16(%rdi)
; SSE2-NEXT: movq %r14, 8(%rdi)
-; SSE2-NEXT: movq %rdx, (%rdi)
+; SSE2-NEXT: movq %rsi, (%rdi)
; SSE2-NEXT: addq $8, %rsp
; SSE2-NEXT: popq %rbx
; SSE2-NEXT: popq %r12
@@ -5082,113 +5077,113 @@ define i512 @isolate_msb_i512_load(ptr %p0, i512 %idx) nounwind {
; SSE42-NEXT: pushq %r14
; SSE42-NEXT: pushq %rbx
; SSE42-NEXT: movq 8(%rsi), %r8
-; SSE42-NEXT: movq 16(%rsi), %rcx
-; SSE42-NEXT: movq 24(%rsi), %rdx
+; SSE42-NEXT: movq 16(%rsi), %rax
+; SSE42-NEXT: movq 24(%rsi), %rcx
; SSE42-NEXT: movq 40(%rsi), %r11
-; SSE42-NEXT: movq 48(%rsi), %rax
+; SSE42-NEXT: movq 48(%rsi), %rdx
; SSE42-NEXT: movq 56(%rsi), %r10
; SSE42-NEXT: movdqa 32(%rsi), %xmm2
; SSE42-NEXT: movdqa 48(%rsi), %xmm0
; SSE42-NEXT: movdqa (%rsi), %xmm1
; SSE42-NEXT: bsrq %r10, %r9
; SSE42-NEXT: xorq $63, %r9
-; SSE42-NEXT: bsrq %rax, %rbx
-; SSE42-NEXT: xorq $63, %rbx
-; SSE42-NEXT: orq $64, %rbx
+; SSE42-NEXT: bsrq %rdx, %rbx
+; SSE42-NEXT: xorl $63, %ebx
+; SSE42-NEXT: orl $64, %ebx
; SSE42-NEXT: testq %r10, %r10
; SSE42-NEXT: cmovneq %r9, %rbx
; SSE42-NEXT: bsrq %r11, %r14
-; SSE42-NEXT: xorq $63, %r14
+; SSE42-NEXT: xorl $63, %r14d
; SSE42-NEXT: bsrq 32(%rsi), %r9
-; SSE42-NEXT: xorq $63, %r9
-; SSE42-NEXT: orq $64, %r9
+; SSE42-NEXT: xorl $63, %r9d
+; SSE42-NEXT: orl $64, %r9d
; SSE42-NEXT: testq %r11, %r11
-; SSE42-NEXT: cmovneq %r14, %r9
-; SSE42-NEXT: orq $128, %r9
-; SSE42-NEXT: orq %r10, %rax
+; SSE42-NEXT: cmovnel %r14d, %r9d
+; SSE42-NEXT: subl $-128, %r9d
+; SSE42-NEXT: orq %r10, %rdx
; SSE42-NEXT: cmovneq %rbx, %r9
-; SSE42-NEXT: bsrq %rdx, %rax
-; SSE42-NEXT: xorq $63, %rax
-; SSE42-NEXT: bsrq %rcx, %r10
-; SSE42-NEXT: xorq $63, %r10
-; SSE42-NEXT: orq $64, %r10
-; SSE42-NEXT: testq %rdx, %rdx
-; SSE42-NEXT: cmovneq %rax, %r10
+; SSE42-NEXT: bsrq %rcx, %rdx
+; SSE42-NEXT: xorl $63, %edx
+; SSE42-NEXT: bsrq %rax, %r10
+; SSE42-NEXT: xorl $63, %r10d
+; SSE42-NEXT: orl $64, %r10d
+; SSE42-NEXT: testq %rcx, %rcx
+; SSE42-NEXT: cmovnel %edx, %r10d
; SSE42-NEXT: por %xmm2, %xmm1
; SSE42-NEXT: bsrq %r8, %r11
-; SSE42-NEXT: bsrq (%rsi), %rax
-; SSE42-NEXT: xorq $63, %r11
-; SSE42-NEXT: xorq $63, %rax
-; SSE42-NEXT: orq $64, %rax
+; SSE42-NEXT: bsrq (%rsi), %rdx
+; SSE42-NEXT: xorl $63, %r11d
+; SSE42-NEXT: xorl $63, %edx
+; SSE42-NEXT: orl $64, %edx
; SSE42-NEXT: testq %r8, %r8
-; SSE42-NEXT: cmovneq %r11, %rax
-; SSE42-NEXT: orq $128, %rax
-; SSE42-NEXT: orq %rdx, %rcx
-; SSE42-NEXT: cmovneq %r10, %rax
-; SSE42-NEXT: orq $256, %rax # imm = 0x100
+; SSE42-NEXT: cmovnel %r11d, %edx
+; SSE42-NEXT: subl $-128, %edx
+; SSE42-NEXT: orq %rcx, %rax
+; SSE42-NEXT: cmovnel %r10d, %edx
+; SSE42-NEXT: addl $256, %edx # imm = 0x100
; SSE42-NEXT: por %xmm0, %xmm2
; SSE42-NEXT: ptest %xmm2, %xmm2
-; SSE42-NEXT: cmovneq %r9, %rax
+; SSE42-NEXT: cmovneq %r9, %rdx
; SSE42-NEXT: movdqa 16(%rsi), %xmm2
; SSE42-NEXT: xorps %xmm3, %xmm3
; SSE42-NEXT: movaps %xmm3, -{{[0-9]+}}(%rsp)
; SSE42-NEXT: movaps %xmm3, -{{[0-9]+}}(%rsp)
; SSE42-NEXT: movaps %xmm3, -{{[0-9]+}}(%rsp)
; SSE42-NEXT: movaps %xmm3, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movabsq $-9223372036854775808, %rcx # imm = 0x8000000000000000
-; SSE42-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
+; SSE42-NEXT: movabsq $-9223372036854775808, %rax # imm = 0x8000000000000000
+; SSE42-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
; SSE42-NEXT: movaps %xmm3, -{{[0-9]+}}(%rsp)
; SSE42-NEXT: movaps %xmm3, -{{[0-9]+}}(%rsp)
; SSE42-NEXT: movaps %xmm3, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movl %eax, %ecx
+; SSE42-NEXT: movl %edx, %ecx
; SSE42-NEXT: andl $63, %ecx
-; SSE42-NEXT: shrl $3, %eax
-; SSE42-NEXT: andl $56, %eax
+; SSE42-NEXT: shrl $3, %edx
+; SSE42-NEXT: andl $56, %edx
; SSE42-NEXT: movq $0, -{{[0-9]+}}(%rsp)
-; SSE42-NEXT: movq -72(%rsp,%rax), %rdx
-; SSE42-NEXT: movq -80(%rsp,%rax), %r9
-; SSE42-NEXT: movq %r9, %rsi
-; SSE42-NEXT: shrdq %cl, %rdx, %rsi
-; SSE42-NEXT: movq -88(%rsp,%rax), %r10
-; SSE42-NEXT: movq %r10, %r8
-; SSE42-NEXT: shrdq %cl, %r9, %r8
-; SSE42-NEXT: movq -96(%rsp,%rax), %r11
+; SSE42-NEXT: movq -72(%rsp,%rdx), %rsi
+; SSE42-NEXT: movq -80(%rsp,%rdx), %rax
+; SSE42-NEXT: movq %rax, %r8
+; SSE42-NEXT: shrdq %cl, %rsi, %r8
+; SSE42-NEXT: movq -88(%rsp,%rdx), %r11
; SSE42-NEXT: movq %r11, %r9
-; SSE42-NEXT: shrdq %cl, %r10, %r9
-; SSE42-NEXT: movq -104(%rsp,%rax), %rbx
-; SSE42-NEXT: movq %rbx, %r10
+; SSE42-NEXT: shrdq %cl, %rax, %r9
+; SSE42-NEXT: movq -96(%rsp,%rdx), %rax
+; SSE42-NEXT: movq %rax, %r10
; SSE42-NEXT: shrdq %cl, %r11, %r10
-; SSE42-NEXT: movq -112(%rsp,%rax), %r14
+; SSE42-NEXT: movq -104(%rsp,%rdx), %r14
; SSE42-NEXT: movq %r14, %r11
-; SSE42-NEXT: shrdq %cl, %rbx, %r11
-; SSE42-NEXT: movq -120(%rsp,%rax), %r15
-; SSE42-NEXT: movq %r15, %rbx
+; SSE42-NEXT: shrdq %cl, %rax, %r11
+; SSE42-NEXT: movq -112(%rsp,%rdx), %rax
+; SSE42-NEXT: movq %rax, %rbx
; SSE42-NEXT: shrdq %cl, %r14, %rbx
-; SSE42-NEXT: movq -128(%rsp,%rax), %r14
-; SSE42-NEXT: shrq %cl, %rdx
+; SSE42-NEXT: movq -120(%rsp,%rdx), %r15
+; SSE42-NEXT: movq %r15, %r14
+; SSE42-NEXT: shrdq %cl, %rax, %r14
+; SSE42-NEXT: movq -128(%rsp,%rdx), %rdx
+; SSE42-NEXT: shrq %cl, %rsi
; SSE42-NEXT: # kill: def $cl killed $cl killed $ecx
-; SSE42-NEXT: shrdq %cl, %r15, %r14
+; SSE42-NEXT: shrdq %cl, %r15, %rdx
; SSE42-NEXT: por %xmm0, %xmm2
; SSE42-NEXT: por %xmm2, %xmm1
; SSE42-NEXT: xorl %ecx, %ecx
; SSE42-NEXT: ptest %xmm1, %xmm1
+; SSE42-NEXT: cmoveq %rcx, %r14
; SSE42-NEXT: cmoveq %rcx, %rbx
; SSE42-NEXT: cmoveq %rcx, %r11
; SSE42-NEXT: cmoveq %rcx, %r10
; SSE42-NEXT: cmoveq %rcx, %r9
; SSE42-NEXT: cmoveq %rcx, %r8
-; SSE42-NEXT: cmoveq %rcx, %rsi
-; SSE42-NEXT: cmoveq %rcx, %r14
-; SSE42-NEXT: movq %rdi, %rax
; SSE42-NEXT: cmoveq %rcx, %rdx
-; SSE42-NEXT: movq %rdx, 56(%rdi)
-; SSE42-NEXT: movq %rsi, 48(%rdi)
-; SSE42-NEXT: movq %r8, 40(%rdi)
-; SSE42-NEXT: movq %r9, 32(%rdi)
-; SSE42-NEXT: movq %r10, 24(%rdi)
-; SSE42-NEXT: movq %r11, 16(%rdi)
-; SSE42-NEXT: movq %rbx, 8(%rdi)
-; SSE42-NEXT: movq %r14, (%rdi)
+; SSE42-NEXT: movq %rdi, %rax
+; SSE42-NEXT: cmoveq %rcx, %rsi
+; SSE42-NEXT: movq %rsi, 56(%rdi)
+; SSE42-NEXT: movq %r8, 48(%rdi)
+; SSE42-NEXT: movq %r9, 40(%rdi)
+; SSE42-NEXT: movq %r10, 32(%rdi)
+; SSE42-NEXT: movq %r11, 24(%rdi)
+; SSE42-NEXT: movq %rbx, 16(%rdi)
+; SSE42-NEXT: movq %r14, 8(%rdi)
+; SSE42-NEXT: movq %rdx, (%rdi)
; SSE42-NEXT: popq %rbx
; SSE42-NEXT: popq %r14
; SSE42-NEXT: popq %r15
@@ -5210,37 +5205,37 @@ define i512 @isolate_msb_i512_load(ptr %p0, i512 %idx) nounwind {
; AVX2-NEXT: lzcntq %r11, %r9
; AVX2-NEXT: xorl %ebx, %ebx
; AVX2-NEXT: lzcntq %r10, %rbx
-; AVX2-NEXT: addq $64, %rbx
+; AVX2-NEXT: addl $64, %ebx
; AVX2-NEXT: testq %r11, %r11
; AVX2-NEXT: cmovneq %r9, %rbx
; AVX2-NEXT: xorl %r14d, %r14d
; AVX2-NEXT: lzcntq %rdx, %r14
; AVX2-NEXT: xorl %r9d, %r9d
; AVX2-NEXT: lzcntq 32(%rsi), %r9
-; AVX2-NEXT: addq $64, %r9
+; AVX2-NEXT: addl $64, %r9d
; AVX2-NEXT: testq %rdx, %rdx
-; AVX2-NEXT: cmovneq %r14, %r9
-; AVX2-NEXT: subq $-128, %r9
+; AVX2-NEXT: cmovnel %r14d, %r9d
+; AVX2-NEXT: subl $-128, %r9d
; AVX2-NEXT: orq %r11, %r10
; AVX2-NEXT: cmovneq %rbx, %r9
; AVX2-NEXT: xorl %edx, %edx
; AVX2-NEXT: lzcntq %rcx, %rdx
; AVX2-NEXT: xorl %r10d, %r10d
; AVX2-NEXT: lzcntq %rax, %r10
-; AVX2-NEXT: addq $64, %r10
+; AVX2-NEXT: addl $64, %r10d
; AVX2-NEXT: testq %rcx, %rcx
-; AVX2-NEXT: cmovneq %rdx, %r10
+; AVX2-NEXT: cmovnel %edx, %r10d
; AVX2-NEXT: xorl %r11d, %r11d
; AVX2-NEXT: lzcntq %r8, %r11
; AVX2-NEXT: xorl %edx, %edx
; AVX2-NEXT: lzcntq (%rsi), %rdx
-; AVX2-NEXT: addq $64, %rdx
+; AVX2-NEXT: addl $64, %edx
; AVX2-NEXT: testq %r8, %r8
-; AVX2-NEXT: cmovneq %r11, %rdx
-; AVX2-NEXT: subq $-128, %rdx
+; AVX2-NEXT: cmovnel %r11d, %edx
+; AVX2-NEXT: subl $-128, %edx
; AVX2-NEXT: orq %rcx, %rax
-; AVX2-NEXT: cmovneq %r10, %rdx
-; AVX2-NEXT: addq $256, %rdx # imm = 0x100
+; AVX2-NEXT: cmovnel %r10d, %edx
+; AVX2-NEXT: addl $256, %edx # imm = 0x100
; AVX2-NEXT: vpor 48(%rsi), %xmm1, %xmm1
; AVX2-NEXT: vptest %xmm1, %xmm1
; AVX2-NEXT: cmovneq %r9, %rdx
@@ -5313,7 +5308,7 @@ define i512 @isolate_msb_i512_load(ptr %p0, i512 %idx) nounwind {
; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0
; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512F-NEXT: vplzcntq %zmm0, %zmm0
-; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
+; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,64,128,192,256,320,384,448]
; AVX512F-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
; AVX512F-NEXT: vmovq %xmm0, %rdx
; AVX512F-NEXT: movabsq $-9223372036854775808, %rsi # imm = 0x8000000000000000
@@ -5342,7 +5337,7 @@ define i512 @isolate_msb_i512_load(ptr %p0, i512 %idx) nounwind {
; AVX512VL-NEXT: vpermq %zmm0, %zmm1, %zmm0
; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VL-NEXT: vplzcntq %zmm0, %zmm0
-; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
+; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,64,128,192,256,320,384,448]
; AVX512VL-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
; AVX512VL-NEXT: vmovq %xmm0, %rdx
; AVX512VL-NEXT: movabsq $-9223372036854775808, %rsi # imm = 0x8000000000000000
@@ -5372,7 +5367,7 @@ define i512 @isolate_msb_i512_load(ptr %p0, i512 %idx) nounwind {
; AVX512VBMI-NEXT: vpermq %zmm0, %zmm1, %zmm0
; AVX512VBMI-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VBMI-NEXT: vplzcntq %zmm0, %zmm0
-; AVX512VBMI-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
+; AVX512VBMI-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,64,128,192,256,320,384,448]
; AVX512VBMI-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
; AVX512VBMI-NEXT: vmovq %xmm0, %rdx
; AVX512VBMI-NEXT: movabsq $-9223372036854775808, %rsi # imm = 0x8000000000000000
diff --git a/llvm/test/CodeGen/X86/bitcnt-big-integer.ll b/llvm/test/CodeGen/X86/bitcnt-big-integer.ll
index 5c9a9338bc32f..a0f83d8ac048a 100644
--- a/llvm/test/CodeGen/X86/bitcnt-big-integer.ll
+++ b/llvm/test/CodeGen/X86/bitcnt-big-integer.ll
@@ -123,84 +123,78 @@ define i32 @vector_ctpop_i128(<4 x i32> %v0) nounwind {
define <2 x i32> @load_ctpop_v2i128(ptr %p0) nounwind {
; SSE-LABEL: load_ctpop_v2i128:
; SSE: # %bb.0:
-; SSE-NEXT: popcntq 8(%rdi), %rax
-; SSE-NEXT: popcntq (%rdi), %rcx
-; SSE-NEXT: popcntq 24(%rdi), %rdx
+; SSE-NEXT: popcntq 24(%rdi), %rax
+; SSE-NEXT: popcntq 16(%rdi), %rcx
+; SSE-NEXT: popcntq 8(%rdi), %rdx
; SSE-NEXT: addl %eax, %ecx
; SSE-NEXT: xorl %eax, %eax
-; SSE-NEXT: popcntq 16(%rdi), %rax
+; SSE-NEXT: popcntq (%rdi), %rax
; SSE-NEXT: addl %edx, %eax
-; SSE-NEXT: movd %eax, %xmm1
-; SSE-NEXT: movd %ecx, %xmm0
-; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE-NEXT: movd %eax, %xmm0
+; SSE-NEXT: pinsrd $1, %ecx, %xmm0
; SSE-NEXT: retq
;
; AVX2-LABEL: load_ctpop_v2i128:
; AVX2: # %bb.0:
-; AVX2-NEXT: popcntq 8(%rdi), %rax
-; AVX2-NEXT: popcntq (%rdi), %rcx
+; AVX2-NEXT: popcntq 24(%rdi), %rax
+; AVX2-NEXT: popcntq 16(%rdi), %rcx
; AVX2-NEXT: addl %eax, %ecx
; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: popcntq 24(%rdi), %rax
-; AVX2-NEXT: popcntq 16(%rdi), %rdx
+; AVX2-NEXT: popcntq 8(%rdi), %rax
+; AVX2-NEXT: popcntq (%rdi), %rdx
; AVX2-NEXT: addl %eax, %edx
; AVX2-NEXT: vmovd %edx, %xmm0
-; AVX2-NEXT: vmovd %ecx, %xmm1
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; AVX2-NEXT: vpinsrd $1, %ecx, %xmm0, %xmm0
; AVX2-NEXT: retq
;
; AVX512F-LABEL: load_ctpop_v2i128:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: popcntq 8(%rdi), %rax
-; AVX512F-NEXT: popcntq (%rdi), %rcx
-; AVX512F-NEXT: addl %eax, %ecx
; AVX512F-NEXT: popcntq 24(%rdi), %rax
-; AVX512F-NEXT: popcntq 16(%rdi), %rdx
+; AVX512F-NEXT: popcntq 16(%rdi), %rcx
+; AVX512F-NEXT: addl %eax, %ecx
+; AVX512F-NEXT: popcntq 8(%rdi), %rax
+; AVX512F-NEXT: popcntq (%rdi), %rdx
; AVX512F-NEXT: addl %eax, %edx
; AVX512F-NEXT: vmovd %edx, %xmm0
-; AVX512F-NEXT: vmovd %ecx, %xmm1
-; AVX512F-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; AVX512F-NEXT: vpinsrd $1, %ecx, %xmm0, %xmm0
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: load_ctpop_v2i128:
; AVX512POPCNT: # %bb.0:
-; AVX512POPCNT-NEXT: popcntq 8(%rdi), %rax
-; AVX512POPCNT-NEXT: popcntq (%rdi), %rcx
-; AVX512POPCNT-NEXT: addl %eax, %ecx
; AVX512POPCNT-NEXT: popcntq 24(%rdi), %rax
-; AVX512POPCNT-NEXT: popcntq 16(%rdi), %rdx
+; AVX512POPCNT-NEXT: popcntq 16(%rdi), %rcx
+; AVX512POPCNT-NEXT: addl %eax, %ecx
+; AVX512POPCNT-NEXT: popcntq 8(%rdi), %rax
+; AVX512POPCNT-NEXT: popcntq (%rdi), %rdx
; AVX512POPCNT-NEXT: addl %eax, %edx
; AVX512POPCNT-NEXT: vmovd %edx, %xmm0
-; AVX512POPCNT-NEXT: vmovd %ecx, %xmm1
-; AVX512POPCNT-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; AVX512POPCNT-NEXT: vpinsrd $1, %ecx, %xmm0, %xmm0
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: load_ctpop_v2i128:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: popcntq 8(%rdi), %rax
-; AVX512VL-NEXT: popcntq (%rdi), %rcx
+; AVX512VL-NEXT: popcntq 24(%rdi), %rax
+; AVX512VL-NEXT: popcntq 16(%rdi), %rcx
; AVX512VL-NEXT: addl %eax, %ecx
; AVX512VL-NEXT: xorl %eax, %eax
-; AVX512VL-NEXT: popcntq 24(%rdi), %rax
-; AVX512VL-NEXT: popcntq 16(%rdi), %rdx
+; AVX512VL-NEXT: popcntq 8(%rdi), %rax
+; AVX512VL-NEXT: popcntq (%rdi), %rdx
; AVX512VL-NEXT: addl %eax, %edx
; AVX512VL-NEXT: vmovd %edx, %xmm0
-; AVX512VL-NEXT: vmovd %ecx, %xmm1
-; AVX512VL-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; AVX512VL-NEXT: vpinsrd $1, %ecx, %xmm0, %xmm0
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: load_ctpop_v2i128:
; AVX512VLPOPCNT: # %bb.0:
-; AVX512VLPOPCNT-NEXT: popcntq 8(%rdi), %rax
-; AVX512VLPOPCNT-NEXT: popcntq (%rdi), %rcx
+; AVX512VLPOPCNT-NEXT: popcntq 24(%rdi), %rax
+; AVX512VLPOPCNT-NEXT: popcntq 16(%rdi), %rcx
; AVX512VLPOPCNT-NEXT: addl %eax, %ecx
; AVX512VLPOPCNT-NEXT: xorl %eax, %eax
-; AVX512VLPOPCNT-NEXT: popcntq 24(%rdi), %rax
-; AVX512VLPOPCNT-NEXT: popcntq 16(%rdi), %rdx
+; AVX512VLPOPCNT-NEXT: popcntq 8(%rdi), %rax
+; AVX512VLPOPCNT-NEXT: popcntq (%rdi), %rdx
; AVX512VLPOPCNT-NEXT: addl %eax, %edx
; AVX512VLPOPCNT-NEXT: vmovd %edx, %xmm0
-; AVX512VLPOPCNT-NEXT: vmovd %ecx, %xmm1
-; AVX512VLPOPCNT-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; AVX512VLPOPCNT-NEXT: vpinsrd $1, %ecx, %xmm0, %xmm0
; AVX512VLPOPCNT-NEXT: retq
%a0 = load <2 x i128>, ptr %p0
%cnt = call <2 x i128> @llvm.ctpop.v2i128(<2 x i128> %a0)
@@ -211,28 +205,25 @@ define <2 x i32> @load_ctpop_v2i128(ptr %p0) nounwind {
define <4 x i32> @load_ctpop_v4i128(ptr %p0) nounwind {
; SSE-LABEL: load_ctpop_v4i128:
; SSE: # %bb.0:
-; SSE-NEXT: popcntq 8(%rdi), %rax
-; SSE-NEXT: popcntq (%rdi), %rcx
-; SSE-NEXT: popcntq 24(%rdi), %rdx
-; SSE-NEXT: popcntq 16(%rdi), %rsi
-; SSE-NEXT: addl %eax, %ecx
-; SSE-NEXT: addl %edx, %esi
-; SSE-NEXT: xorl %eax, %eax
-; SSE-NEXT: popcntq 40(%rdi), %rax
-; SSE-NEXT: xorl %edx, %edx
+; SSE-NEXT: popcntq 56(%rdi), %rax
+; SSE-NEXT: popcntq 40(%rdi), %rcx
; SSE-NEXT: popcntq 32(%rdi), %rdx
-; SSE-NEXT: popcntq 56(%rdi), %r8
-; SSE-NEXT: addl %eax, %edx
-; SSE-NEXT: xorl %eax, %eax
-; SSE-NEXT: popcntq 48(%rdi), %rax
-; SSE-NEXT: addl %r8d, %eax
-; SSE-NEXT: movd %eax, %xmm0
-; SSE-NEXT: movd %edx, %xmm1
-; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; SSE-NEXT: movd %esi, %xmm2
-; SSE-NEXT: movd %ecx, %xmm0
-; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSE-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSE-NEXT: popcntq 24(%rdi), %rsi
+; SSE-NEXT: addl %ecx, %edx
+; SSE-NEXT: xorl %ecx, %ecx
+; SSE-NEXT: popcntq 16(%rdi), %rcx
+; SSE-NEXT: addl %esi, %ecx
+; SSE-NEXT: xorl %esi, %esi
+; SSE-NEXT: popcntq 8(%rdi), %rsi
+; SSE-NEXT: popcntq (%rdi), %r8
+; SSE-NEXT: addl %esi, %r8d
+; SSE-NEXT: xorl %esi, %esi
+; SSE-NEXT: popcntq 48(%rdi), %rsi
+; SSE-NEXT: movd %r8d, %xmm0
+; SSE-NEXT: pinsrd $1, %ecx, %xmm0
+; SSE-NEXT: pinsrd $2, %edx, %xmm0
+; SSE-NEXT: addl %eax, %esi
+; SSE-NEXT: pinsrd $3, %esi, %xmm0
; SSE-NEXT: retq
;
; AVX2-LABEL: load_ctpop_v4i128:
@@ -243,137 +234,105 @@ define <4 x i32> @load_ctpop_v4i128(ptr %p0) nounwind {
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: popcntq 40(%rdi), %rax
; AVX2-NEXT: popcntq 32(%rdi), %rdx
-; AVX2-NEXT: popcntq 8(%rdi), %rsi
-; AVX2-NEXT: popcntq (%rdi), %r8
+; AVX2-NEXT: popcntq 24(%rdi), %rsi
; AVX2-NEXT: addl %eax, %edx
-; AVX2-NEXT: addl %esi, %r8d
; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: popcntq 24(%rdi), %rax
+; AVX2-NEXT: popcntq 16(%rdi), %rax
+; AVX2-NEXT: addl %esi, %eax
; AVX2-NEXT: xorl %esi, %esi
-; AVX2-NEXT: popcntq 16(%rdi), %rsi
-; AVX2-NEXT: addl %eax, %esi
-; AVX2-NEXT: vmovd %esi, %xmm0
-; AVX2-NEXT: vmovd %r8d, %xmm1
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; AVX2-NEXT: vmovd %edx, %xmm1
-; AVX2-NEXT: vmovd %ecx, %xmm2
-; AVX2-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
-; AVX2-NEXT: vpbroadcastq {{.*#+}} xmm2 = [0,4,0,4]
-; AVX2-NEXT: vpermd %ymm1, %ymm2, %ymm1
-; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
-; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: popcntq 8(%rdi), %rsi
+; AVX2-NEXT: popcntq (%rdi), %rdi
+; AVX2-NEXT: addl %esi, %edi
+; AVX2-NEXT: vmovd %edi, %xmm0
+; AVX2-NEXT: vpinsrd $1, %eax, %xmm0, %xmm0
+; AVX2-NEXT: vpinsrd $2, %edx, %xmm0, %xmm0
+; AVX2-NEXT: vpinsrd $3, %ecx, %xmm0, %xmm0
; AVX2-NEXT: retq
;
; AVX512F-LABEL: load_ctpop_v4i128:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: popcntq 24(%rdi), %rcx
+; AVX512F-NEXT: popcntq 56(%rdi), %rax
+; AVX512F-NEXT: popcntq 48(%rdi), %rcx
+; AVX512F-NEXT: addl %eax, %ecx
+; AVX512F-NEXT: popcntq 40(%rdi), %rax
+; AVX512F-NEXT: popcntq 32(%rdi), %rdx
+; AVX512F-NEXT: popcntq 24(%rdi), %rsi
+; AVX512F-NEXT: addl %eax, %edx
; AVX512F-NEXT: popcntq 16(%rdi), %rax
-; AVX512F-NEXT: addl %ecx, %eax
-; AVX512F-NEXT: popcntq 8(%rdi), %rcx
-; AVX512F-NEXT: popcntq (%rdi), %rdx
-; AVX512F-NEXT: popcntq 40(%rdi), %rsi
-; AVX512F-NEXT: popcntq 32(%rdi), %r8
-; AVX512F-NEXT: addl %ecx, %edx
-; AVX512F-NEXT: addl %esi, %r8d
-; AVX512F-NEXT: popcntq 56(%rdi), %rcx
-; AVX512F-NEXT: popcntq 48(%rdi), %rsi
-; AVX512F-NEXT: addl %ecx, %esi
-; AVX512F-NEXT: vpbroadcastq {{.*#+}} xmm0 = [0,16,0,16]
-; AVX512F-NEXT: vmovd %esi, %xmm1
-; AVX512F-NEXT: vmovd %r8d, %xmm2
-; AVX512F-NEXT: vpermt2d %zmm1, %zmm0, %zmm2
-; AVX512F-NEXT: vmovd %edx, %xmm0
-; AVX512F-NEXT: vmovd %eax, %xmm1
-; AVX512F-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512F-NEXT: vmovdqa {{.*#+}} xmm0 = [0,4,18,19]
-; AVX512F-NEXT: vpermi2d %zmm2, %zmm1, %zmm0
-; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512F-NEXT: addl %esi, %eax
+; AVX512F-NEXT: popcntq 8(%rdi), %rsi
+; AVX512F-NEXT: popcntq (%rdi), %rdi
+; AVX512F-NEXT: addl %esi, %edi
+; AVX512F-NEXT: vmovd %edi, %xmm0
+; AVX512F-NEXT: vpinsrd $1, %eax, %xmm0, %xmm0
+; AVX512F-NEXT: vpinsrd $2, %edx, %xmm0, %xmm0
+; AVX512F-NEXT: vpinsrd $3, %ecx, %xmm0, %xmm0
; AVX512F-NEXT: retq
;
; AVX512POPCNT-LABEL: load_ctpop_v4i128:
; AVX512POPCNT: # %bb.0:
-; AVX512POPCNT-NEXT: popcntq 24(%rdi), %rcx
+; AVX512POPCNT-NEXT: popcntq 56(%rdi), %rax
+; AVX512POPCNT-NEXT: popcntq 48(%rdi), %rcx
+; AVX512POPCNT-NEXT: addl %eax, %ecx
+; AVX512POPCNT-NEXT: popcntq 40(%rdi), %rax
+; AVX512POPCNT-NEXT: popcntq 32(%rdi), %rdx
+; AVX512POPCNT-NEXT: popcntq 24(%rdi), %rsi
+; AVX512POPCNT-NEXT: addl %eax, %edx
; AVX512POPCNT-NEXT: popcntq 16(%rdi), %rax
-; AVX512POPCNT-NEXT: addl %ecx, %eax
-; AVX512POPCNT-NEXT: popcntq 8(%rdi), %rcx
-; AVX512POPCNT-NEXT: popcntq (%rdi), %rdx
-; AVX512POPCNT-NEXT: popcntq 40(%rdi), %rsi
-; AVX512POPCNT-NEXT: popcntq 32(%rdi), %r8
-; AVX512POPCNT-NEXT: addl %ecx, %edx
-; AVX512POPCNT-NEXT: addl %esi, %r8d
-; AVX512POPCNT-NEXT: popcntq 56(%rdi), %rcx
-; AVX512POPCNT-NEXT: popcntq 48(%rdi), %rsi
-; AVX512POPCNT-NEXT: addl %ecx, %esi
-; AVX512POPCNT-NEXT: vpbroadcastq {{.*#+}} xmm0 = [0,16,0,16]
-; AVX512POPCNT-NEXT: vmovd %esi, %xmm1
-; AVX512POPCNT-NEXT: vmovd %r8d, %xmm2
-; AVX512POPCNT-NEXT: vpermt2d %zmm1, %zmm0, %zmm2
-; AVX512POPCNT-NEXT: vmovd %edx, %xmm0
-; AVX512POPCNT-NEXT: vmovd %eax, %xmm1
-; AVX512POPCNT-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512POPCNT-NEXT: vmovdqa {{.*#+}} xmm0 = [0,4,18,19]
-; AVX512POPCNT-NEXT: vpermi2d %zmm2, %zmm1, %zmm0
-; AVX512POPCNT-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0
+; AVX512POPCNT-NEXT: addl %esi, %eax
+; AVX512POPCNT-NEXT: popcntq 8(%rdi), %rsi
+; AVX512POPCNT-NEXT: popcntq (%rdi), %rdi
+; AVX512POPCNT-NEXT: addl %esi, %edi
+; AVX512POPCNT-NEXT: vmovd %edi, %xmm0
+; AVX512POPCNT-NEXT: vpinsrd $1, %eax, %xmm0, %xmm0
+; AVX512POPCNT-NEXT: vpinsrd $2, %edx, %xmm0, %xmm0
+; AVX512POPCNT-NEXT: vpinsrd $3, %ecx, %xmm0, %xmm0
; AVX512POPCNT-NEXT: retq
;
; AVX512VL-LABEL: load_ctpop_v4i128:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: popcntq 24(%rdi), %rcx
-; AVX512VL-NEXT: popcntq 16(%rdi), %rax
-; AVX512VL-NEXT: addl %ecx, %eax
-; AVX512VL-NEXT: xorl %ecx, %ecx
-; AVX512VL-NEXT: popcntq 8(%rdi), %rcx
-; AVX512VL-NEXT: popcntq (%rdi), %rdx
-; AVX512VL-NEXT: popcntq 40(%rdi), %rsi
-; AVX512VL-NEXT: popcntq 32(%rdi), %r8
-; AVX512VL-NEXT: addl %ecx, %edx
-; AVX512VL-NEXT: addl %esi, %r8d
-; AVX512VL-NEXT: xorl %ecx, %ecx
-; AVX512VL-NEXT: popcntq 56(%rdi), %rcx
-; AVX512VL-NEXT: xorl %esi, %esi
-; AVX512VL-NEXT: popcntq 48(%rdi), %rsi
-; AVX512VL-NEXT: addl %ecx, %esi
-; AVX512VL-NEXT: vmovd %esi, %xmm0
-; AVX512VL-NEXT: vmovd %r8d, %xmm1
-; AVX512VL-NEXT: vpbroadcastq {{.*#+}} xmm2 = [0,4,0,4]
-; AVX512VL-NEXT: vpermi2d %xmm0, %xmm1, %xmm2
-; AVX512VL-NEXT: vmovd %edx, %xmm0
-; AVX512VL-NEXT: vmovd %eax, %xmm1
-; AVX512VL-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512VL-NEXT: vmovdqa {{.*#+}} xmm0 = [0,4,10,11]
-; AVX512VL-NEXT: vpermi2d %ymm2, %ymm1, %ymm0
-; AVX512VL-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX512VL-NEXT: vzeroupper
+; AVX512VL-NEXT: popcntq 56(%rdi), %rax
+; AVX512VL-NEXT: popcntq 48(%rdi), %rcx
+; AVX512VL-NEXT: addl %eax, %ecx
+; AVX512VL-NEXT: xorl %eax, %eax
+; AVX512VL-NEXT: popcntq 40(%rdi), %rax
+; AVX512VL-NEXT: popcntq 32(%rdi), %rdx
+; AVX512VL-NEXT: addl %eax, %edx
+; AVX512VL-NEXT: xorl %eax, %eax
+; AVX512VL-NEXT: popcntq 24(%rdi), %rax
+; AVX512VL-NEXT: popcntq 16(%rdi), %rsi
+; AVX512VL-NEXT: addl %eax, %esi
+; AVX512VL-NEXT: xorl %eax, %eax
+; AVX512VL-NEXT: popcntq 8(%rdi), %rax
+; AVX512VL-NEXT: popcntq (%rdi), %rdi
+; AVX512VL-NEXT: addl %eax, %edi
+; AVX512VL-NEXT: vmovd %edi, %xmm0
+; AVX512VL-NEXT: vpinsrd $1, %esi, %xmm0, %xmm0
+; AVX512VL-NEXT: vpinsrd $2, %edx, %xmm0, %xmm0
+; AVX512VL-NEXT: vpinsrd $3, %ecx, %xmm0, %xmm0
; AVX512VL-NEXT: retq
;
; AVX512VLPOPCNT-LABEL: load_ctpop_v4i128:
; AVX512VLPOPCNT: # %bb.0:
-; AVX512VLPOPCNT-NEXT: popcntq 24(%rdi), %rcx
-; AVX512VLPOPCNT-NEXT: popcntq 16(%rdi), %rax
-; AVX512VLPOPCNT-NEXT: addl %ecx, %eax
-; AVX512VLPOPCNT-NEXT: xorl %ecx, %ecx
-; AVX512VLPOPCNT-NEXT: popcntq 8(%rdi), %rcx
-; AVX512VLPOPCNT-NEXT: popcntq (%rdi), %rdx
-; AVX512VLPOPCNT-NEXT: popcntq 40(%rdi), %rsi
-; AVX512VLPOPCNT-NEXT: popcntq 32(%rdi), %r8
-; AVX512VLPOPCNT-NEXT: addl %ecx, %edx
-; AVX512VLPOPCNT-NEXT: addl %esi, %r8d
-; AVX512VLPOPCNT-NEXT: xorl %ecx, %ecx
-; AVX512VLPOPCNT-NEXT: popcntq 56(%rdi), %rcx
-; AVX512VLPOPCNT-NEXT: xorl %esi, %esi
-; AVX512VLPOPCNT-NEXT: popcntq 48(%rdi), %rsi
-; AVX512VLPOPCNT-NEXT: addl %ecx, %esi
-; AVX512VLPOPCNT-NEXT: vmovd %esi, %xmm0
-; AVX512VLPOPCNT-NEXT: vmovd %r8d, %xmm1
-; AVX512VLPOPCNT-NEXT: vpbroadcastq {{.*#+}} xmm2 = [0,4,0,4]
-; AVX512VLPOPCNT-NEXT: vpermi2d %xmm0, %xmm1, %xmm2
-; AVX512VLPOPCNT-NEXT: vmovd %edx, %xmm0
-; AVX512VLPOPCNT-NEXT: vmovd %eax, %xmm1
-; AVX512VLPOPCNT-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; AVX512VLPOPCNT-NEXT: vmovdqa {{.*#+}} xmm0 = [0,4,10,11]
-; AVX512VLPOPCNT-NEXT: vpermi2d %ymm2, %ymm1, %ymm0
-; AVX512VLPOPCNT-NEXT: # kill: def $xmm0 killed $xmm0 killed $ymm0
-; AVX512VLPOPCNT-NEXT: vzeroupper
+; AVX512VLPOPCNT-NEXT: popcntq 56(%rdi), %rax
+; AVX512VLPOPCNT-NEXT: popcntq 48(%rdi), %rcx
+; AVX512VLPOPCNT-NEXT: addl %eax, %ecx
+; AVX512VLPOPCNT-NEXT: xorl %eax, %eax
+; AVX512VLPOPCNT-NEXT: popcntq 40(%rdi), %rax
+; AVX512VLPOPCNT-NEXT: popcntq 32(%rdi), %rdx
+; AVX512VLPOPCNT-NEXT: addl %eax, %edx
+; AVX512VLPOPCNT-NEXT: xorl %eax, %eax
+; AVX512VLPOPCNT-NEXT: popcntq 24(%rdi), %rax
+; AVX512VLPOPCNT-NEXT: popcntq 16(%rdi), %rsi
+; AVX512VLPOPCNT-NEXT: addl %eax, %esi
+; AVX512VLPOPCNT-NEXT: xorl %eax, %eax
+; AVX512VLPOPCNT-NEXT: popcntq 8(%rdi), %rax
+; AVX512VLPOPCNT-NEXT: popcntq (%rdi), %rdi
+; AVX512VLPOPCNT-NEXT: addl %eax, %edi
+; AVX512VLPOPCNT-NEXT: vmovd %edi, %xmm0
+; AVX512VLPOPCNT-NEXT: vpinsrd $1, %esi, %xmm0, %xmm0
+; AVX512VLPOPCNT-NEXT: vpinsrd $2, %edx, %xmm0, %xmm0
+; AVX512VLPOPCNT-NEXT: vpinsrd $3, %ecx, %xmm0, %xmm0
; AVX512VLPOPCNT-NEXT: retq
%a0 = load <4 x i128>, ptr %p0
%cnt = call <4 x i128> @llvm.ctpop.v4i128(<4 x i128> %a0)
@@ -635,52 +594,50 @@ define i32 @vector_ctpop_i256(<8 x i32> %v0) nounwind {
define <2 x i32> @load_ctpop_v2i256_v2i32(ptr %p0) nounwind {
; SSE-LABEL: load_ctpop_v2i256_v2i32:
; SSE: # %bb.0:
-; SSE-NEXT: popcntq 24(%rdi), %rax
-; SSE-NEXT: popcntq 16(%rdi), %rcx
-; SSE-NEXT: popcntq 8(%rdi), %rdx
+; SSE-NEXT: popcntq 56(%rdi), %rax
+; SSE-NEXT: popcntq 48(%rdi), %rcx
+; SSE-NEXT: popcntq 40(%rdi), %rdx
; SSE-NEXT: addl %eax, %ecx
; SSE-NEXT: xorl %eax, %eax
-; SSE-NEXT: popcntq (%rdi), %rax
+; SSE-NEXT: popcntq 32(%rdi), %rax
; SSE-NEXT: addl %edx, %eax
; SSE-NEXT: xorl %edx, %edx
-; SSE-NEXT: popcntq 56(%rdi), %rdx
-; SSE-NEXT: popcntq 48(%rdi), %rsi
+; SSE-NEXT: popcntq 24(%rdi), %rdx
+; SSE-NEXT: popcntq 16(%rdi), %rsi
; SSE-NEXT: addl %ecx, %eax
; SSE-NEXT: addl %edx, %esi
; SSE-NEXT: xorl %ecx, %ecx
-; SSE-NEXT: popcntq 40(%rdi), %rcx
+; SSE-NEXT: popcntq 8(%rdi), %rcx
; SSE-NEXT: xorl %edx, %edx
-; SSE-NEXT: popcntq 32(%rdi), %rdx
+; SSE-NEXT: popcntq (%rdi), %rdx
; SSE-NEXT: addl %ecx, %edx
; SSE-NEXT: addl %esi, %edx
-; SSE-NEXT: movd %edx, %xmm1
-; SSE-NEXT: movd %eax, %xmm0
-; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE-NEXT: movd %edx, %xmm0
+; SSE-NEXT: pinsrd $1, %eax, %xmm0
; SSE-NEXT: retq
;
; AVX2-LABEL: load_ctpop_v2i256_v2i32:
; AVX2: # %bb.0:
-; AVX2-NEXT: popcntq 24(%rdi), %rax
-; AVX2-NEXT: popcntq 16(%rdi), %rcx
+; AVX2-NEXT: popcntq 56(%rdi), %rax
+; AVX2-NEXT: popcntq 48(%rdi), %rcx
; AVX2-NEXT: addl %eax, %ecx
; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: popcntq 8(%rdi), %rax
-; AVX2-NEXT: popcntq (%rdi), %rdx
+; AVX2-NEXT: popcntq 40(%rdi), %rax
+; AVX2-NEXT: popcntq 32(%rdi), %rdx
; AVX2-NEXT: addl %eax, %edx
; AVX2-NEXT: addl %ecx, %edx
; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: popcntq 56(%rdi), %rax
+; AVX2-NEXT: popcntq 24(%rdi), %rax
; AVX2-NEXT: xorl %ecx, %ecx
-; AVX2-NEXT: popcntq 48(%rdi), %rcx
-; AVX2-NEXT: popcntq 40(%rdi), %rsi
+; AVX2-NEXT: popcntq 16(%rdi), %rcx
+; AVX2-NEXT: popcntq 8(%rdi), %rsi
; AVX2-NEXT: addl %eax, %ecx
; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: popcntq 32(%rdi), %rax
+; AVX2-NEXT: popcntq (%rdi), %rax
; AVX2-NEXT: addl %esi, %eax
; AVX2-NEXT: addl %ecx, %eax
; AVX2-NEXT: vmovd %eax, %xmm0
-; AVX2-NEXT: vmovd %edx, %xmm1
-; AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; AVX2-NEXT: vpinsrd $1, %edx, %xmm0, %xmm0
; AVX2-NEXT: retq
;
; AVX512F-LABEL: load_ctpop_v2i256_v2i32:
@@ -2018,7 +1975,7 @@ define i32 @load_ctlz_i256(ptr %p0) nounwind {
; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm0 = [256,256,256,256]
; AVX512F-NEXT: vpermq {{.*#+}} ymm1 = mem[3,2,1,0]
; AVX512F-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
+; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2 # [0,64,128,192]
; AVX512F-NEXT: vptestmq %zmm1, %zmm1, %k0
; AVX512F-NEXT: kshiftlw $12, %k0, %k0
; AVX512F-NEXT: kshiftrw $12, %k0, %k1
@@ -2031,7 +1988,7 @@ define i32 @load_ctlz_i256(ptr %p0) nounwind {
; AVX512POPCNT-NEXT: vpbroadcastq {{.*#+}} ymm0 = [256,256,256,256]
; AVX512POPCNT-NEXT: vpermq {{.*#+}} ymm1 = mem[3,2,1,0]
; AVX512POPCNT-NEXT: vplzcntq %zmm1, %zmm2
-; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
+; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2 # [0,64,128,192]
; AVX512POPCNT-NEXT: vptestmq %zmm1, %zmm1, %k0
; AVX512POPCNT-NEXT: kshiftlw $12, %k0, %k0
; AVX512POPCNT-NEXT: kshiftrw $12, %k0, %k1
@@ -2043,7 +2000,7 @@ define i32 @load_ctlz_i256(ptr %p0) nounwind {
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: vpermq {{.*#+}} ymm0 = mem[3,2,1,0]
; AVX512VL-NEXT: vplzcntq %ymm0, %ymm1
-; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,64,128,192]
; AVX512VL-NEXT: vptestmq %ymm0, %ymm0, %k1
; AVX512VL-NEXT: vpbroadcastq {{.*#+}} ymm0 = [256,256,256,256]
; AVX512VL-NEXT: vpcompressq %ymm1, %ymm0 {%k1}
@@ -2055,7 +2012,7 @@ define i32 @load_ctlz_i256(ptr %p0) nounwind {
; AVX512VLPOPCNT: # %bb.0:
; AVX512VLPOPCNT-NEXT: vpermq {{.*#+}} ymm0 = mem[3,2,1,0]
; AVX512VLPOPCNT-NEXT: vplzcntq %ymm0, %ymm1
-; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,64,128,192]
; AVX512VLPOPCNT-NEXT: vptestmq %ymm0, %ymm0, %k1
; AVX512VLPOPCNT-NEXT: vpbroadcastq {{.*#+}} ymm0 = [256,256,256,256]
; AVX512VLPOPCNT-NEXT: vpcompressq %ymm1, %ymm0 {%k1}
@@ -2126,7 +2083,7 @@ define i32 @vector_ctlz_i256(<8 x i32> %v0) nounwind {
; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm1 = [256,256,256,256]
; AVX512F-NEXT: vpermq {{.*#+}} ymm0 = ymm0[3,2,1,0]
; AVX512F-NEXT: vplzcntq %zmm0, %zmm2
-; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
+; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2 # [0,64,128,192]
; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k0
; AVX512F-NEXT: kshiftlw $12, %k0, %k0
; AVX512F-NEXT: kshiftrw $12, %k0, %k1
@@ -2139,7 +2096,7 @@ define i32 @vector_ctlz_i256(<8 x i32> %v0) nounwind {
; AVX512POPCNT-NEXT: vpbroadcastq {{.*#+}} ymm1 = [256,256,256,256]
; AVX512POPCNT-NEXT: vpermq {{.*#+}} ymm0 = ymm0[3,2,1,0]
; AVX512POPCNT-NEXT: vplzcntq %zmm0, %zmm2
-; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
+; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2 # [0,64,128,192]
; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k0
; AVX512POPCNT-NEXT: kshiftlw $12, %k0, %k0
; AVX512POPCNT-NEXT: kshiftrw $12, %k0, %k1
@@ -2151,7 +2108,7 @@ define i32 @vector_ctlz_i256(<8 x i32> %v0) nounwind {
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[3,2,1,0]
; AVX512VL-NEXT: vplzcntq %ymm0, %ymm1
-; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,64,128,192]
; AVX512VL-NEXT: vptestmq %ymm0, %ymm0, %k1
; AVX512VL-NEXT: vpbroadcastq {{.*#+}} ymm0 = [256,256,256,256]
; AVX512VL-NEXT: vpcompressq %ymm1, %ymm0 {%k1}
@@ -2163,7 +2120,7 @@ define i32 @vector_ctlz_i256(<8 x i32> %v0) nounwind {
; AVX512VLPOPCNT: # %bb.0:
; AVX512VLPOPCNT-NEXT: vpermq {{.*#+}} ymm0 = ymm0[3,2,1,0]
; AVX512VLPOPCNT-NEXT: vplzcntq %ymm0, %ymm1
-; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,64,128,192]
; AVX512VLPOPCNT-NEXT: vptestmq %ymm0, %ymm0, %k1
; AVX512VLPOPCNT-NEXT: vpbroadcastq {{.*#+}} ymm0 = [256,256,256,256]
; AVX512VLPOPCNT-NEXT: vpcompressq %ymm1, %ymm0 {%k1}
@@ -2300,7 +2257,7 @@ define i32 @test_ctlz_i512(i512 %a0) nounwind {
; AVX512F-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512F-NEXT: vplzcntq %zmm0, %zmm0
-; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
+; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,64,128,192,256,320,384,448]
; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
; AVX512F-NEXT: vpcompressq %zmm0, %zmm1 {%k1}
; AVX512F-NEXT: vmovd %xmm1, %eax
@@ -2323,7 +2280,7 @@ define i32 @test_ctlz_i512(i512 %a0) nounwind {
; AVX512POPCNT-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512POPCNT-NEXT: vplzcntq %zmm0, %zmm0
-; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
+; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,64,128,192,256,320,384,448]
; AVX512POPCNT-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
; AVX512POPCNT-NEXT: vpcompressq %zmm0, %zmm1 {%k1}
; AVX512POPCNT-NEXT: vmovd %xmm1, %eax
@@ -2345,7 +2302,7 @@ define i32 @test_ctlz_i512(i512 %a0) nounwind {
; AVX512VL-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
; AVX512VL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
; AVX512VL-NEXT: vplzcntq %zmm0, %zmm1
-; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
+; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1 # [0,64,128,192,256,320,384,448]
; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VL-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
; AVX512VL-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
@@ -2369,7 +2326,7 @@ define i32 @test_ctlz_i512(i512 %a0) nounwind {
; AVX512VLPOPCNT-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
; AVX512VLPOPCNT-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
; AVX512VLPOPCNT-NEXT: vplzcntq %zmm0, %zmm1
-; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
+; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1 # [0,64,128,192,256,320,384,448]
; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VLPOPCNT-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
; AVX512VLPOPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
@@ -2486,7 +2443,7 @@ define i32 @load_ctlz_i512(ptr %p0) nounwind {
; AVX512F-NEXT: vpermq (%rdi), %zmm0, %zmm0
; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512F-NEXT: vplzcntq %zmm0, %zmm0
-; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
+; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,64,128,192,256,320,384,448]
; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
; AVX512F-NEXT: vpcompressq %zmm0, %zmm1 {%k1}
; AVX512F-NEXT: vmovd %xmm1, %eax
@@ -2498,7 +2455,7 @@ define i32 @load_ctlz_i512(ptr %p0) nounwind {
; AVX512POPCNT-NEXT: vpermq (%rdi), %zmm0, %zmm0
; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512POPCNT-NEXT: vplzcntq %zmm0, %zmm0
-; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
+; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,64,128,192,256,320,384,448]
; AVX512POPCNT-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
; AVX512POPCNT-NEXT: vpcompressq %zmm0, %zmm1 {%k1}
; AVX512POPCNT-NEXT: vmovd %xmm1, %eax
@@ -2509,7 +2466,7 @@ define i32 @load_ctlz_i512(ptr %p0) nounwind {
; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]
; AVX512VL-NEXT: vpermq (%rdi), %zmm0, %zmm0
; AVX512VL-NEXT: vplzcntq %zmm0, %zmm1
-; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
+; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1 # [0,64,128,192,256,320,384,448]
; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VL-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
; AVX512VL-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
@@ -2522,7 +2479,7 @@ define i32 @load_ctlz_i512(ptr %p0) nounwind {
; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm0 = [7,6,5,4,3,2,1,0]
; AVX512VLPOPCNT-NEXT: vpermq (%rdi), %zmm0, %zmm0
; AVX512VLPOPCNT-NEXT: vplzcntq %zmm0, %zmm1
-; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
+; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1 # [0,64,128,192,256,320,384,448]
; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VLPOPCNT-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
; AVX512VLPOPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
@@ -2643,7 +2600,7 @@ define i32 @vector_ctlz_i512(<16 x i32> %v0) nounwind {
; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0
; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512F-NEXT: vplzcntq %zmm0, %zmm0
-; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
+; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,64,128,192,256,320,384,448]
; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
; AVX512F-NEXT: vpcompressq %zmm0, %zmm1 {%k1}
; AVX512F-NEXT: vmovd %xmm1, %eax
@@ -2655,7 +2612,7 @@ define i32 @vector_ctlz_i512(<16 x i32> %v0) nounwind {
; AVX512POPCNT-NEXT: vpermq %zmm0, %zmm1, %zmm0
; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512POPCNT-NEXT: vplzcntq %zmm0, %zmm0
-; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
+; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,64,128,192,256,320,384,448]
; AVX512POPCNT-NEXT: vpbroadcastq {{.*#+}} zmm1 = [512,512,512,512,512,512,512,512]
; AVX512POPCNT-NEXT: vpcompressq %zmm0, %zmm1 {%k1}
; AVX512POPCNT-NEXT: vmovd %xmm1, %eax
@@ -2666,7 +2623,7 @@ define i32 @vector_ctlz_i512(<16 x i32> %v0) nounwind {
; AVX512VL-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
; AVX512VL-NEXT: vpermq %zmm0, %zmm1, %zmm0
; AVX512VL-NEXT: vplzcntq %zmm0, %zmm1
-; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
+; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1 # [0,64,128,192,256,320,384,448]
; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VL-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
; AVX512VL-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
@@ -2679,7 +2636,7 @@ define i32 @vector_ctlz_i512(<16 x i32> %v0) nounwind {
; AVX512VLPOPCNT-NEXT: vmovdqa64 {{.*#+}} zmm1 = [7,6,5,4,3,2,1,0]
; AVX512VLPOPCNT-NEXT: vpermq %zmm0, %zmm1, %zmm0
; AVX512VLPOPCNT-NEXT: vplzcntq %zmm0, %zmm1
-; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
+; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1 # [0,64,128,192,256,320,384,448]
; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VLPOPCNT-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
; AVX512VLPOPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
@@ -3643,7 +3600,7 @@ define i32 @load_ctlz_undef_i256(ptr %p0) nounwind {
; AVX512F: # %bb.0:
; AVX512F-NEXT: vpermq {{.*#+}} ymm0 = mem[3,2,1,0]
; AVX512F-NEXT: vplzcntq %zmm0, %zmm1
-; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,64,128,192]
; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k0
; AVX512F-NEXT: kshiftlw $12, %k0, %k0
; AVX512F-NEXT: kshiftrw $12, %k0, %k1
@@ -3655,7 +3612,7 @@ define i32 @load_ctlz_undef_i256(ptr %p0) nounwind {
; AVX512POPCNT: # %bb.0:
; AVX512POPCNT-NEXT: vpermq {{.*#+}} ymm0 = mem[3,2,1,0]
; AVX512POPCNT-NEXT: vplzcntq %zmm0, %zmm1
-; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,64,128,192]
; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k0
; AVX512POPCNT-NEXT: kshiftlw $12, %k0, %k0
; AVX512POPCNT-NEXT: kshiftrw $12, %k0, %k1
@@ -3668,7 +3625,7 @@ define i32 @load_ctlz_undef_i256(ptr %p0) nounwind {
; AVX512VL-NEXT: vpermq {{.*#+}} ymm0 = mem[3,2,1,0]
; AVX512VL-NEXT: vptestmq %ymm0, %ymm0, %k1
; AVX512VL-NEXT: vplzcntq %ymm0, %ymm0
-; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # [0,64,128,192]
; AVX512VL-NEXT: vpcompressq %ymm0, %ymm0 {%k1} {z}
; AVX512VL-NEXT: vmovd %xmm0, %eax
; AVX512VL-NEXT: vzeroupper
@@ -3679,7 +3636,7 @@ define i32 @load_ctlz_undef_i256(ptr %p0) nounwind {
; AVX512VLPOPCNT-NEXT: vpermq {{.*#+}} ymm0 = mem[3,2,1,0]
; AVX512VLPOPCNT-NEXT: vptestmq %ymm0, %ymm0, %k1
; AVX512VLPOPCNT-NEXT: vplzcntq %ymm0, %ymm0
-; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # [0,64,128,192]
; AVX512VLPOPCNT-NEXT: vpcompressq %ymm0, %ymm0 {%k1} {z}
; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
@@ -3746,7 +3703,7 @@ define i32 @vector_ctlz_undef_i256(<8 x i32> %v0) nounwind {
; AVX512F: # %bb.0:
; AVX512F-NEXT: vpermq {{.*#+}} ymm0 = ymm0[3,2,1,0]
; AVX512F-NEXT: vplzcntq %zmm0, %zmm1
-; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,64,128,192]
; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k0
; AVX512F-NEXT: kshiftlw $12, %k0, %k0
; AVX512F-NEXT: kshiftrw $12, %k0, %k1
@@ -3758,7 +3715,7 @@ define i32 @vector_ctlz_undef_i256(<8 x i32> %v0) nounwind {
; AVX512POPCNT: # %bb.0:
; AVX512POPCNT-NEXT: vpermq {{.*#+}} ymm0 = ymm0[3,2,1,0]
; AVX512POPCNT-NEXT: vplzcntq %zmm0, %zmm1
-; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,64,128,192]
; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k0
; AVX512POPCNT-NEXT: kshiftlw $12, %k0, %k0
; AVX512POPCNT-NEXT: kshiftrw $12, %k0, %k1
@@ -3771,7 +3728,7 @@ define i32 @vector_ctlz_undef_i256(<8 x i32> %v0) nounwind {
; AVX512VL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[3,2,1,0]
; AVX512VL-NEXT: vptestmq %ymm0, %ymm0, %k1
; AVX512VL-NEXT: vplzcntq %ymm0, %ymm0
-; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # [0,64,128,192]
; AVX512VL-NEXT: vpcompressq %ymm0, %ymm0 {%k1} {z}
; AVX512VL-NEXT: vmovd %xmm0, %eax
; AVX512VL-NEXT: vzeroupper
@@ -3782,7 +3739,7 @@ define i32 @vector_ctlz_undef_i256(<8 x i32> %v0) nounwind {
; AVX512VLPOPCNT-NEXT: vpermq {{.*#+}} ymm0 = ymm0[3,2,1,0]
; AVX512VLPOPCNT-NEXT: vptestmq %ymm0, %ymm0, %k1
; AVX512VLPOPCNT-NEXT: vplzcntq %ymm0, %ymm0
-; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 # [0,64,128,192]
; AVX512VLPOPCNT-NEXT: vpcompressq %ymm0, %ymm0 {%k1} {z}
; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
@@ -3916,7 +3873,7 @@ define i32 @test_ctlz_undef_i512(i512 %a0) nounwind {
; AVX512F-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512F-NEXT: vplzcntq %zmm0, %zmm0
-; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
+; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,64,128,192,256,320,384,448]
; AVX512F-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
; AVX512F-NEXT: vmovd %xmm0, %eax
; AVX512F-NEXT: retq
@@ -3938,7 +3895,7 @@ define i32 @test_ctlz_undef_i512(i512 %a0) nounwind {
; AVX512POPCNT-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512POPCNT-NEXT: vplzcntq %zmm0, %zmm0
-; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
+; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,64,128,192,256,320,384,448]
; AVX512POPCNT-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
; AVX512POPCNT-NEXT: vmovd %xmm0, %eax
; AVX512POPCNT-NEXT: retq
@@ -3960,7 +3917,7 @@ define i32 @test_ctlz_undef_i512(i512 %a0) nounwind {
; AVX512VL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VL-NEXT: vplzcntq %zmm0, %zmm0
-; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
+; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,64,128,192,256,320,384,448]
; AVX512VL-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
; AVX512VL-NEXT: vmovd %xmm0, %eax
; AVX512VL-NEXT: vzeroupper
@@ -3983,7 +3940,7 @@ define i32 @test_ctlz_undef_i512(i512 %a0) nounwind {
; AVX512VLPOPCNT-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VLPOPCNT-NEXT: vplzcntq %zmm0, %zmm0
-; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
+; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,64,128,192,256,320,384,448]
; AVX512VLPOPCNT-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
@@ -4099,7 +4056,7 @@ define i32 @load_ctlz_undef_i512(ptr %p0) nounwind {
; AVX512F-NEXT: vpermq (%rdi), %zmm0, %zmm0
; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512F-NEXT: vplzcntq %zmm0, %zmm0
-; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
+; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,64,128,192,256,320,384,448]
; AVX512F-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
; AVX512F-NEXT: vmovd %xmm0, %eax
; AVX512F-NEXT: retq
@@ -4110,7 +4067,7 @@ define i32 @load_ctlz_undef_i512(ptr %p0) nounwind {
; AVX512POPCNT-NEXT: vpermq (%rdi), %zmm0, %zmm0
; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512POPCNT-NEXT: vplzcntq %zmm0, %zmm0
-; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
+; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,64,128,192,256,320,384,448]
; AVX512POPCNT-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
; AVX512POPCNT-NEXT: vmovd %xmm0, %eax
; AVX512POPCNT-NEXT: retq
@@ -4121,7 +4078,7 @@ define i32 @load_ctlz_undef_i512(ptr %p0) nounwind {
; AVX512VL-NEXT: vpermq (%rdi), %zmm0, %zmm0
; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VL-NEXT: vplzcntq %zmm0, %zmm0
-; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
+; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,64,128,192,256,320,384,448]
; AVX512VL-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
; AVX512VL-NEXT: vmovd %xmm0, %eax
; AVX512VL-NEXT: vzeroupper
@@ -4133,7 +4090,7 @@ define i32 @load_ctlz_undef_i512(ptr %p0) nounwind {
; AVX512VLPOPCNT-NEXT: vpermq (%rdi), %zmm0, %zmm0
; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VLPOPCNT-NEXT: vplzcntq %zmm0, %zmm0
-; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
+; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,64,128,192,256,320,384,448]
; AVX512VLPOPCNT-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
@@ -4251,7 +4208,7 @@ define i32 @vector_ctlz_undef_i512(<16 x i32> %v0) nounwind {
; AVX512F-NEXT: vpermq %zmm0, %zmm1, %zmm0
; AVX512F-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512F-NEXT: vplzcntq %zmm0, %zmm0
-; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
+; AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,64,128,192,256,320,384,448]
; AVX512F-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
; AVX512F-NEXT: vmovd %xmm0, %eax
; AVX512F-NEXT: retq
@@ -4262,7 +4219,7 @@ define i32 @vector_ctlz_undef_i512(<16 x i32> %v0) nounwind {
; AVX512POPCNT-NEXT: vpermq %zmm0, %zmm1, %zmm0
; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512POPCNT-NEXT: vplzcntq %zmm0, %zmm0
-; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
+; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,64,128,192,256,320,384,448]
; AVX512POPCNT-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
; AVX512POPCNT-NEXT: vmovd %xmm0, %eax
; AVX512POPCNT-NEXT: retq
@@ -4273,7 +4230,7 @@ define i32 @vector_ctlz_undef_i512(<16 x i32> %v0) nounwind {
; AVX512VL-NEXT: vpermq %zmm0, %zmm1, %zmm0
; AVX512VL-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VL-NEXT: vplzcntq %zmm0, %zmm0
-; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
+; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,64,128,192,256,320,384,448]
; AVX512VL-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
; AVX512VL-NEXT: vmovd %xmm0, %eax
; AVX512VL-NEXT: vzeroupper
@@ -4285,7 +4242,7 @@ define i32 @vector_ctlz_undef_i512(<16 x i32> %v0) nounwind {
; AVX512VLPOPCNT-NEXT: vpermq %zmm0, %zmm1, %zmm0
; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VLPOPCNT-NEXT: vplzcntq %zmm0, %zmm0
-; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
+; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,64,128,192,256,320,384,448]
; AVX512VLPOPCNT-NEXT: vpcompressq %zmm0, %zmm0 {%k1} {z}
; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
; AVX512VLPOPCNT-NEXT: vzeroupper
@@ -5211,7 +5168,7 @@ define i32 @load_cttz_i256(ptr %p0) nounwind {
; AVX512POPCNT-NEXT: vpaddq %ymm2, %ymm0, %ymm2
; AVX512POPCNT-NEXT: vpandn %ymm2, %ymm0, %ymm2
; AVX512POPCNT-NEXT: vpopcntq %zmm2, %zmm2
-; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
+; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2 # [0,64,128,192]
; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k0
; AVX512POPCNT-NEXT: kshiftlw $12, %k0, %k0
; AVX512POPCNT-NEXT: kshiftrw $12, %k0, %k1
@@ -5242,7 +5199,7 @@ define i32 @load_cttz_i256(ptr %p0) nounwind {
; AVX512VLPOPCNT-NEXT: vpaddq %ymm1, %ymm0, %ymm1
; AVX512VLPOPCNT-NEXT: vpandn %ymm1, %ymm0, %ymm1
; AVX512VLPOPCNT-NEXT: vpopcntq %ymm1, %ymm1
-; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,64,128,192]
; AVX512VLPOPCNT-NEXT: vptestmq %ymm0, %ymm0, %k1
; AVX512VLPOPCNT-NEXT: vpbroadcastq {{.*#+}} ymm0 = [256,256,256,256]
; AVX512VLPOPCNT-NEXT: vpcompressq %ymm1, %ymm0 {%k1}
@@ -5329,7 +5286,7 @@ define i32 @vector_cttz_i256(<8 x i32> %v0) nounwind {
; AVX512POPCNT-NEXT: vpaddq %ymm2, %ymm0, %ymm2
; AVX512POPCNT-NEXT: vpandn %ymm2, %ymm0, %ymm2
; AVX512POPCNT-NEXT: vpopcntq %zmm2, %zmm2
-; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
+; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2 # [0,64,128,192]
; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k0
; AVX512POPCNT-NEXT: kshiftlw $12, %k0, %k0
; AVX512POPCNT-NEXT: kshiftrw $12, %k0, %k1
@@ -5358,7 +5315,7 @@ define i32 @vector_cttz_i256(<8 x i32> %v0) nounwind {
; AVX512VLPOPCNT-NEXT: vpaddq %ymm1, %ymm0, %ymm1
; AVX512VLPOPCNT-NEXT: vpandn %ymm1, %ymm0, %ymm1
; AVX512VLPOPCNT-NEXT: vpopcntq %ymm1, %ymm1
-; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,64,128,192]
; AVX512VLPOPCNT-NEXT: vptestmq %ymm0, %ymm0, %k1
; AVX512VLPOPCNT-NEXT: vpbroadcastq {{.*#+}} ymm0 = [256,256,256,256]
; AVX512VLPOPCNT-NEXT: vpcompressq %ymm1, %ymm0 {%k1}
@@ -5506,7 +5463,7 @@ define i32 @test_cttz_i512(i512 %a0) nounwind {
; AVX512POPCNT-NEXT: vpaddq %zmm1, %zmm0, %zmm1
; AVX512POPCNT-NEXT: vpandnq %zmm1, %zmm0, %zmm1
; AVX512POPCNT-NEXT: vpopcntq %zmm1, %zmm1
-; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
+; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1 # [0,64,128,192,256,320,384,448]
; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512POPCNT-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
; AVX512POPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
@@ -5558,7 +5515,7 @@ define i32 @test_cttz_i512(i512 %a0) nounwind {
; AVX512VLPOPCNT-NEXT: vpaddq %zmm1, %zmm0, %zmm1
; AVX512VLPOPCNT-NEXT: vpandnq %zmm1, %zmm0, %zmm1
; AVX512VLPOPCNT-NEXT: vpopcntq %zmm1, %zmm1
-; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
+; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1 # [0,64,128,192,256,320,384,448]
; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VLPOPCNT-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
; AVX512VLPOPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
@@ -5683,7 +5640,7 @@ define i32 @load_cttz_i512(ptr %p0) nounwind {
; AVX512POPCNT-NEXT: vpaddq %zmm1, %zmm0, %zmm1
; AVX512POPCNT-NEXT: vpandnq %zmm1, %zmm0, %zmm1
; AVX512POPCNT-NEXT: vpopcntq %zmm1, %zmm1
-; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
+; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1 # [0,64,128,192,256,320,384,448]
; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512POPCNT-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
; AVX512POPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
@@ -5713,7 +5670,7 @@ define i32 @load_cttz_i512(ptr %p0) nounwind {
; AVX512VLPOPCNT-NEXT: vpaddq %zmm1, %zmm0, %zmm1
; AVX512VLPOPCNT-NEXT: vpandnq %zmm1, %zmm0, %zmm1
; AVX512VLPOPCNT-NEXT: vpopcntq %zmm1, %zmm1
-; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
+; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1 # [0,64,128,192,256,320,384,448]
; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VLPOPCNT-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
; AVX512VLPOPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
@@ -5840,7 +5797,7 @@ define i32 @vector_cttz_i512(<16 x i32> %v0) nounwind {
; AVX512POPCNT-NEXT: vpaddq %zmm1, %zmm0, %zmm1
; AVX512POPCNT-NEXT: vpandnq %zmm1, %zmm0, %zmm1
; AVX512POPCNT-NEXT: vpopcntq %zmm1, %zmm1
-; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
+; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1 # [0,64,128,192,256,320,384,448]
; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512POPCNT-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
; AVX512POPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
@@ -5868,7 +5825,7 @@ define i32 @vector_cttz_i512(<16 x i32> %v0) nounwind {
; AVX512VLPOPCNT-NEXT: vpaddq %zmm1, %zmm0, %zmm1
; AVX512VLPOPCNT-NEXT: vpandnq %zmm1, %zmm0, %zmm1
; AVX512VLPOPCNT-NEXT: vpopcntq %zmm1, %zmm1
-; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
+; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1 # [0,64,128,192,256,320,384,448]
; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VLPOPCNT-NEXT: vpbroadcastq {{.*#+}} zmm0 = [512,512,512,512,512,512,512,512]
; AVX512VLPOPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1}
@@ -6834,7 +6791,7 @@ define i32 @load_cttz_undef_i256(ptr %p0) nounwind {
; AVX512POPCNT-NEXT: vpaddq %ymm1, %ymm0, %ymm1
; AVX512POPCNT-NEXT: vpandn %ymm1, %ymm0, %ymm1
; AVX512POPCNT-NEXT: vpopcntq %zmm1, %zmm1
-; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,64,128,192]
; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k0
; AVX512POPCNT-NEXT: kshiftlw $12, %k0, %k0
; AVX512POPCNT-NEXT: kshiftrw $12, %k0, %k1
@@ -6864,7 +6821,7 @@ define i32 @load_cttz_undef_i256(ptr %p0) nounwind {
; AVX512VLPOPCNT-NEXT: vpaddq %ymm1, %ymm0, %ymm1
; AVX512VLPOPCNT-NEXT: vpandn %ymm1, %ymm0, %ymm1
; AVX512VLPOPCNT-NEXT: vpopcntq %ymm1, %ymm1
-; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,64,128,192]
; AVX512VLPOPCNT-NEXT: vptestmq %ymm0, %ymm0, %k1
; AVX512VLPOPCNT-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
@@ -6947,7 +6904,7 @@ define i32 @vector_cttz_undef_i256(<8 x i32> %v0) nounwind {
; AVX512POPCNT-NEXT: vpaddq %ymm1, %ymm0, %ymm1
; AVX512POPCNT-NEXT: vpandn %ymm1, %ymm0, %ymm1
; AVX512POPCNT-NEXT: vpopcntq %zmm1, %zmm1
-; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,64,128,192]
; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k0
; AVX512POPCNT-NEXT: kshiftlw $12, %k0, %k0
; AVX512POPCNT-NEXT: kshiftrw $12, %k0, %k1
@@ -6975,7 +6932,7 @@ define i32 @vector_cttz_undef_i256(<8 x i32> %v0) nounwind {
; AVX512VLPOPCNT-NEXT: vpaddq %ymm1, %ymm0, %ymm1
; AVX512VLPOPCNT-NEXT: vpandn %ymm1, %ymm0, %ymm1
; AVX512VLPOPCNT-NEXT: vpopcntq %ymm1, %ymm1
-; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,64,128,192]
; AVX512VLPOPCNT-NEXT: vptestmq %ymm0, %ymm0, %k1
; AVX512VLPOPCNT-NEXT: vpcompressq %ymm1, %ymm0 {%k1} {z}
; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
@@ -7120,7 +7077,7 @@ define i32 @test_cttz_undef_i512(i512 %a0) nounwind {
; AVX512POPCNT-NEXT: vpaddq %zmm1, %zmm0, %zmm1
; AVX512POPCNT-NEXT: vpandnq %zmm1, %zmm0, %zmm1
; AVX512POPCNT-NEXT: vpopcntq %zmm1, %zmm1
-; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
+; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1 # [0,64,128,192,256,320,384,448]
; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512POPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
; AVX512POPCNT-NEXT: vmovd %xmm0, %eax
@@ -7170,7 +7127,7 @@ define i32 @test_cttz_undef_i512(i512 %a0) nounwind {
; AVX512VLPOPCNT-NEXT: vpaddq %zmm1, %zmm0, %zmm1
; AVX512VLPOPCNT-NEXT: vpandnq %zmm1, %zmm0, %zmm1
; AVX512VLPOPCNT-NEXT: vpopcntq %zmm1, %zmm1
-; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
+; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1 # [0,64,128,192,256,320,384,448]
; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VLPOPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
@@ -7294,7 +7251,7 @@ define i32 @load_cttz_undef_i512(ptr %p0) nounwind {
; AVX512POPCNT-NEXT: vpaddq %zmm1, %zmm0, %zmm1
; AVX512POPCNT-NEXT: vpandnq %zmm1, %zmm0, %zmm1
; AVX512POPCNT-NEXT: vpopcntq %zmm1, %zmm1
-; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
+; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1 # [0,64,128,192,256,320,384,448]
; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512POPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
; AVX512POPCNT-NEXT: vmovd %xmm0, %eax
@@ -7322,7 +7279,7 @@ define i32 @load_cttz_undef_i512(ptr %p0) nounwind {
; AVX512VLPOPCNT-NEXT: vpaddq %zmm1, %zmm0, %zmm1
; AVX512VLPOPCNT-NEXT: vpandnq %zmm1, %zmm0, %zmm1
; AVX512VLPOPCNT-NEXT: vpopcntq %zmm1, %zmm1
-; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
+; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1 # [0,64,128,192,256,320,384,448]
; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VLPOPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
@@ -7446,7 +7403,7 @@ define i32 @vector_cttz_undef_i512(<16 x i32> %v0) nounwind {
; AVX512POPCNT-NEXT: vpaddq %zmm1, %zmm0, %zmm1
; AVX512POPCNT-NEXT: vpandnq %zmm1, %zmm0, %zmm1
; AVX512POPCNT-NEXT: vpopcntq %zmm1, %zmm1
-; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
+; AVX512POPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1 # [0,64,128,192,256,320,384,448]
; AVX512POPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512POPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
; AVX512POPCNT-NEXT: vmovd %xmm0, %eax
@@ -7472,7 +7429,7 @@ define i32 @vector_cttz_undef_i512(<16 x i32> %v0) nounwind {
; AVX512VLPOPCNT-NEXT: vpaddq %zmm1, %zmm0, %zmm1
; AVX512VLPOPCNT-NEXT: vpandnq %zmm1, %zmm0, %zmm1
; AVX512VLPOPCNT-NEXT: vpopcntq %zmm1, %zmm1
-; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1
+; AVX512VLPOPCNT-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm1 # [0,64,128,192,256,320,384,448]
; AVX512VLPOPCNT-NEXT: vptestmq %zmm0, %zmm0, %k1
; AVX512VLPOPCNT-NEXT: vpcompressq %zmm1, %zmm0 {%k1} {z}
; AVX512VLPOPCNT-NEXT: vmovd %xmm0, %eax
diff --git a/llvm/test/CodeGen/X86/bittest-big-integer.ll b/llvm/test/CodeGen/X86/bittest-big-integer.ll
index 96ccc7b0f7527..03bc9046fe9b8 100644
--- a/llvm/test/CodeGen/X86/bittest-big-integer.ll
+++ b/llvm/test/CodeGen/X86/bittest-big-integer.ll
@@ -1897,45 +1897,45 @@ define i32 @blsr_u512(ptr %word) nounwind {
; SSE2-NEXT: pushq %rbx
; SSE2-NEXT: movdqa (%rdi), %xmm0
; SSE2-NEXT: movq 48(%rdi), %r8
-; SSE2-NEXT: movq 40(%rdi), %rdx
-; SSE2-NEXT: movq 32(%rdi), %rsi
+; SSE2-NEXT: movq 40(%rdi), %rcx
+; SSE2-NEXT: movq 32(%rdi), %rdx
; SSE2-NEXT: movq (%rdi), %rax
; SSE2-NEXT: movq 8(%rdi), %r9
-; SSE2-NEXT: rep bsfq %rax, %rcx
+; SSE2-NEXT: rep bsfq %rax, %rsi
; SSE2-NEXT: rep bsfq %r9, %r10
-; SSE2-NEXT: addq $64, %r10
+; SSE2-NEXT: addl $64, %r10d
; SSE2-NEXT: testq %rax, %rax
-; SSE2-NEXT: cmovneq %rcx, %r10
+; SSE2-NEXT: cmovneq %rsi, %r10
; SSE2-NEXT: movq 16(%rdi), %r11
; SSE2-NEXT: rep bsfq %r11, %rbx
-; SSE2-NEXT: rep bsfq 24(%rdi), %rcx
-; SSE2-NEXT: addq $64, %rcx
+; SSE2-NEXT: rep bsfq 24(%rdi), %rsi
+; SSE2-NEXT: addl $64, %esi
; SSE2-NEXT: testq %r11, %r11
-; SSE2-NEXT: cmovneq %rbx, %rcx
-; SSE2-NEXT: subq $-128, %rcx
+; SSE2-NEXT: cmovnel %ebx, %esi
+; SSE2-NEXT: subl $-128, %esi
; SSE2-NEXT: orq %r9, %rax
-; SSE2-NEXT: cmovneq %r10, %rcx
-; SSE2-NEXT: rep bsfq %rsi, %rax
-; SSE2-NEXT: rep bsfq %rdx, %r9
-; SSE2-NEXT: addq $64, %r9
-; SSE2-NEXT: testq %rsi, %rsi
-; SSE2-NEXT: cmovneq %rax, %r9
+; SSE2-NEXT: cmovneq %r10, %rsi
+; SSE2-NEXT: rep bsfq %rdx, %rax
+; SSE2-NEXT: rep bsfq %rcx, %r9
+; SSE2-NEXT: addl $64, %r9d
+; SSE2-NEXT: testq %rdx, %rdx
+; SSE2-NEXT: cmovnel %eax, %r9d
; SSE2-NEXT: rep bsfq %r8, %r10
; SSE2-NEXT: movl $64, %eax
; SSE2-NEXT: rep bsfq 56(%rdi), %rax
-; SSE2-NEXT: addq $64, %rax
+; SSE2-NEXT: addl $64, %eax
; SSE2-NEXT: testq %r8, %r8
-; SSE2-NEXT: cmovneq %r10, %rax
-; SSE2-NEXT: subq $-128, %rax
-; SSE2-NEXT: orq %rdx, %rsi
-; SSE2-NEXT: cmovneq %r9, %rax
+; SSE2-NEXT: cmovnel %r10d, %eax
+; SSE2-NEXT: subl $-128, %eax
+; SSE2-NEXT: orq %rcx, %rdx
+; SSE2-NEXT: cmovnel %r9d, %eax
; SSE2-NEXT: por 16(%rdi), %xmm0
-; SSE2-NEXT: addq $256, %rax # imm = 0x100
+; SSE2-NEXT: addl $256, %eax # imm = 0x100
; SSE2-NEXT: pxor %xmm1, %xmm1
; SSE2-NEXT: pcmpeqd %xmm0, %xmm1
-; SSE2-NEXT: movmskps %xmm1, %edx
-; SSE2-NEXT: xorl $15, %edx
-; SSE2-NEXT: cmovneq %rcx, %rax
+; SSE2-NEXT: movmskps %xmm1, %ecx
+; SSE2-NEXT: xorl $15, %ecx
+; SSE2-NEXT: cmovneq %rsi, %rax
; SSE2-NEXT: movl $-2, %edx
; SSE2-NEXT: movl %eax, %ecx
; SSE2-NEXT: roll %cl, %edx
@@ -1956,34 +1956,34 @@ define i32 @blsr_u512(ptr %word) nounwind {
; SSE4-NEXT: movq 8(%rdi), %r8
; SSE4-NEXT: rep bsfq %rax, %rsi
; SSE4-NEXT: rep bsfq %r8, %r9
-; SSE4-NEXT: addq $64, %r9
+; SSE4-NEXT: addl $64, %r9d
; SSE4-NEXT: testq %rax, %rax
; SSE4-NEXT: cmovneq %rsi, %r9
; SSE4-NEXT: movq 16(%rdi), %r10
; SSE4-NEXT: rep bsfq %r10, %r11
; SSE4-NEXT: rep bsfq 24(%rdi), %rsi
-; SSE4-NEXT: addq $64, %rsi
+; SSE4-NEXT: addl $64, %esi
; SSE4-NEXT: testq %r10, %r10
-; SSE4-NEXT: cmovneq %r11, %rsi
-; SSE4-NEXT: subq $-128, %rsi
+; SSE4-NEXT: cmovnel %r11d, %esi
+; SSE4-NEXT: subl $-128, %esi
; SSE4-NEXT: orq %r8, %rax
; SSE4-NEXT: cmovneq %r9, %rsi
; SSE4-NEXT: movq 32(%rdi), %r8
; SSE4-NEXT: rep bsfq %r8, %rax
; SSE4-NEXT: rep bsfq %rcx, %r9
-; SSE4-NEXT: addq $64, %r9
+; SSE4-NEXT: addl $64, %r9d
; SSE4-NEXT: testq %r8, %r8
-; SSE4-NEXT: cmovneq %rax, %r9
+; SSE4-NEXT: cmovnel %eax, %r9d
; SSE4-NEXT: rep bsfq %rdx, %r10
; SSE4-NEXT: movl $64, %eax
; SSE4-NEXT: rep bsfq 56(%rdi), %rax
-; SSE4-NEXT: addq $64, %rax
+; SSE4-NEXT: addl $64, %eax
; SSE4-NEXT: testq %rdx, %rdx
-; SSE4-NEXT: cmovneq %r10, %rax
-; SSE4-NEXT: subq $-128, %rax
+; SSE4-NEXT: cmovnel %r10d, %eax
+; SSE4-NEXT: subl $-128, %eax
; SSE4-NEXT: orq %rcx, %r8
-; SSE4-NEXT: cmovneq %r9, %rax
-; SSE4-NEXT: addq $256, %rax # imm = 0x100
+; SSE4-NEXT: cmovnel %r9d, %eax
+; SSE4-NEXT: addl $256, %eax # imm = 0x100
; SSE4-NEXT: por 16(%rdi), %xmm0
; SSE4-NEXT: ptest %xmm0, %xmm0
; SSE4-NEXT: cmovneq %rsi, %rax
@@ -2009,37 +2009,37 @@ define i32 @blsr_u512(ptr %word) nounwind {
; AVX2-NEXT: movq 8(%rdi), %r10
; AVX2-NEXT: tzcntq %r9, %r8
; AVX2-NEXT: tzcntq %r10, %r11
-; AVX2-NEXT: addq $64, %r11
+; AVX2-NEXT: addl $64, %r11d
; AVX2-NEXT: testq %r9, %r9
; AVX2-NEXT: cmovneq %r8, %r11
; AVX2-NEXT: xorl %ebx, %ebx
; AVX2-NEXT: tzcntq %rax, %rbx
; AVX2-NEXT: xorl %r8d, %r8d
; AVX2-NEXT: tzcntq 24(%rdi), %r8
-; AVX2-NEXT: addq $64, %r8
+; AVX2-NEXT: addl $64, %r8d
; AVX2-NEXT: testq %rax, %rax
-; AVX2-NEXT: cmovneq %rbx, %r8
-; AVX2-NEXT: subq $-128, %r8
+; AVX2-NEXT: cmovnel %ebx, %r8d
+; AVX2-NEXT: subl $-128, %r8d
; AVX2-NEXT: orq %r10, %r9
; AVX2-NEXT: cmovneq %r11, %r8
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq %rdx, %rax
; AVX2-NEXT: xorl %r9d, %r9d
; AVX2-NEXT: tzcntq %rcx, %r9
-; AVX2-NEXT: addq $64, %r9
+; AVX2-NEXT: addl $64, %r9d
; AVX2-NEXT: testq %rdx, %rdx
-; AVX2-NEXT: cmovneq %rax, %r9
+; AVX2-NEXT: cmovnel %eax, %r9d
; AVX2-NEXT: xorl %r10d, %r10d
; AVX2-NEXT: tzcntq %rsi, %r10
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: tzcntq 56(%rdi), %rax
-; AVX2-NEXT: addq $64, %rax
+; AVX2-NEXT: addl $64, %eax
; AVX2-NEXT: testq %rsi, %rsi
-; AVX2-NEXT: cmovneq %r10, %rax
-; AVX2-NEXT: subq $-128, %rax
+; AVX2-NEXT: cmovnel %r10d, %eax
+; AVX2-NEXT: subl $-128, %eax
; AVX2-NEXT: orq %rcx, %rdx
-; AVX2-NEXT: cmovneq %r9, %rax
-; AVX2-NEXT: addq $256, %rax # imm = 0x100
+; AVX2-NEXT: cmovnel %r9d, %eax
+; AVX2-NEXT: addl $256, %eax # imm = 0x100
; AVX2-NEXT: vpor 16(%rdi), %xmm0, %xmm0
; AVX2-NEXT: vptest %xmm0, %xmm0
; AVX2-NEXT: cmovneq %r8, %rax
diff --git a/llvm/test/CodeGen/X86/bsf.ll b/llvm/test/CodeGen/X86/bsf.ll
index f780f0172647f..8109bedd988af 100644
--- a/llvm/test/CodeGen/X86/bsf.ll
+++ b/llvm/test/CodeGen/X86/bsf.ll
@@ -342,7 +342,7 @@ define i128 @cmov_bsf128(i128 %x, i128 %y) nounwind {
; X64-NEXT: rep bsfq %rdi, %rcx
; X64-NEXT: movl $64, %eax
; X64-NEXT: rep bsfq %rsi, %rax
-; X64-NEXT: addq $64, %rax
+; X64-NEXT: addl $64, %eax
; X64-NEXT: testq %rdi, %rdi
; X64-NEXT: cmovneq %rcx, %rax
; X64-NEXT: xorl %edx, %edx
@@ -434,7 +434,7 @@ define i128 @cmov_bsf128_undef(i128 %x, i128 %y) nounwind {
; X64-NEXT: # %bb.1: # %select.true.sink
; X64-NEXT: rep bsfq %rdi, %rcx
; X64-NEXT: rep bsfq %rsi, %rax
-; X64-NEXT: addq $64, %rax
+; X64-NEXT: addl $64, %eax
; X64-NEXT: testq %rdi, %rdi
; X64-NEXT: cmovneq %rcx, %rax
; X64-NEXT: xorl %edx, %edx
diff --git a/llvm/test/CodeGen/X86/bsr.ll b/llvm/test/CodeGen/X86/bsr.ll
index affacc5ee6487..0a8d71e7e7738 100644
--- a/llvm/test/CodeGen/X86/bsr.ll
+++ b/llvm/test/CodeGen/X86/bsr.ll
@@ -376,8 +376,8 @@ define i128 @cmov_bsr128(i128 %x, i128 %y) nounwind {
; X64-NEXT: xorq $63, %r8
; X64-NEXT: movl $127, %eax
; X64-NEXT: bsrq %rdi, %rax
-; X64-NEXT: xorq $63, %rax
-; X64-NEXT: addq $64, %rax
+; X64-NEXT: xorl $63, %eax
+; X64-NEXT: addl $64, %eax
; X64-NEXT: testq %rsi, %rsi
; X64-NEXT: cmovneq %r8, %rax
; X64-NEXT: xorq $127, %rax
@@ -470,8 +470,8 @@ define i128 @cmov_bsr128_undef(i128 %x, i128 %y) nounwind {
; X64-NEXT: bsrq %rsi, %r8
; X64-NEXT: xorq $63, %r8
; X64-NEXT: bsrq %rdi, %rax
-; X64-NEXT: xorq $63, %rax
-; X64-NEXT: orq $64, %rax
+; X64-NEXT: xorl $63, %eax
+; X64-NEXT: orl $64, %eax
; X64-NEXT: testq %rsi, %rsi
; X64-NEXT: cmovneq %r8, %rax
; X64-NEXT: xorq $127, %rax
diff --git a/llvm/test/CodeGen/X86/dag-update-nodetomatch.ll b/llvm/test/CodeGen/X86/dag-update-nodetomatch.ll
index 71ad598abe683..2d66c0efc995b 100644
--- a/llvm/test/CodeGen/X86/dag-update-nodetomatch.ll
+++ b/llvm/test/CodeGen/X86/dag-update-nodetomatch.ll
@@ -99,7 +99,7 @@ define void @_Z2x6v() local_unnamed_addr {
; CHECK-NEXT: movq x1 at GOTPCREL(%rip), %rax
; CHECK-NEXT: movl (%rax), %edx
; CHECK-NEXT: andl $511, %edx # imm = 0x1FF
-; CHECK-NEXT: leaq 1(%rdx), %rax
+; CHECK-NEXT: leal 1(%rdx), %eax
; CHECK-NEXT: movq x4 at GOTPCREL(%rip), %rcx
; CHECK-NEXT: movl %eax, (%rcx)
; CHECK-NEXT: movq x3 at GOTPCREL(%rip), %rcx
@@ -137,7 +137,7 @@ define void @_Z2x6v() local_unnamed_addr {
; CHECK-NEXT: leal 8(,%rdx,8), %eax
; CHECK-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; CHECK-NEXT: leaq 32(%rsi), %rbx
-; CHECK-NEXT: leaq 8(,%rdx,8), %r14
+; CHECK-NEXT: leal 8(,%rdx,8), %r14d
; CHECK-NEXT: xorl %r15d, %r15d
; CHECK-NEXT: movq x0 at GOTPCREL(%rip), %r12
; CHECK-NEXT: movq %rsi, %r13
diff --git a/llvm/test/CodeGen/X86/dagcombine-select.ll b/llvm/test/CodeGen/X86/dagcombine-select.ll
index 1380c02663ee0..58e6c46327e9f 100644
--- a/llvm/test/CodeGen/X86/dagcombine-select.ll
+++ b/llvm/test/CodeGen/X86/dagcombine-select.ll
@@ -325,18 +325,20 @@ declare i64 @llvm.cttz.i64(i64, i1)
define i64 @cttz_64_eq_select(i64 %v) nounwind {
; NOBMI-LABEL: cttz_64_eq_select:
; NOBMI: # %bb.0:
-; NOBMI-NEXT: bsfq %rdi, %rcx
-; NOBMI-NEXT: movq $-1, %rax
+; NOBMI-NEXT: rep bsfq %rdi, %rcx
+; NOBMI-NEXT: addl $6, %ecx
+; NOBMI-NEXT: testq %rdi, %rdi
+; NOBMI-NEXT: movl $5, %eax
; NOBMI-NEXT: cmovneq %rcx, %rax
-; NOBMI-NEXT: addq $6, %rax
; NOBMI-NEXT: retq
;
; BMI-LABEL: cttz_64_eq_select:
; BMI: # %bb.0:
; BMI-NEXT: tzcntq %rdi, %rcx
-; BMI-NEXT: movq $-1, %rax
-; BMI-NEXT: cmovaeq %rcx, %rax
-; BMI-NEXT: addq $6, %rax
+; BMI-NEXT: addl $6, %ecx
+; BMI-NEXT: testq %rdi, %rdi
+; BMI-NEXT: movl $5, %eax
+; BMI-NEXT: cmovneq %rcx, %rax
; BMI-NEXT: retq
%cnt = tail call i64 @llvm.cttz.i64(i64 %v, i1 true)
@@ -349,18 +351,20 @@ define i64 @cttz_64_eq_select(i64 %v) nounwind {
define i64 @cttz_64_ne_select(i64 %v) nounwind {
; NOBMI-LABEL: cttz_64_ne_select:
; NOBMI: # %bb.0:
-; NOBMI-NEXT: bsfq %rdi, %rcx
-; NOBMI-NEXT: movq $-1, %rax
+; NOBMI-NEXT: rep bsfq %rdi, %rcx
+; NOBMI-NEXT: addl $6, %ecx
+; NOBMI-NEXT: testq %rdi, %rdi
+; NOBMI-NEXT: movl $5, %eax
; NOBMI-NEXT: cmovneq %rcx, %rax
-; NOBMI-NEXT: addq $6, %rax
; NOBMI-NEXT: retq
;
; BMI-LABEL: cttz_64_ne_select:
; BMI: # %bb.0:
; BMI-NEXT: tzcntq %rdi, %rcx
-; BMI-NEXT: movq $-1, %rax
-; BMI-NEXT: cmovaeq %rcx, %rax
-; BMI-NEXT: addq $6, %rax
+; BMI-NEXT: addl $6, %ecx
+; BMI-NEXT: testq %rdi, %rdi
+; BMI-NEXT: movl $5, %eax
+; BMI-NEXT: cmovneq %rcx, %rax
; BMI-NEXT: retq
%cnt = tail call i64 @llvm.cttz.i64(i64 %v, i1 true)
diff --git a/llvm/test/CodeGen/X86/dagcombine-shifts.ll b/llvm/test/CodeGen/X86/dagcombine-shifts.ll
index e9a1e8ed728a4..e8a1da31b3f82 100644
--- a/llvm/test/CodeGen/X86/dagcombine-shifts.ll
+++ b/llvm/test/CodeGen/X86/dagcombine-shifts.ll
@@ -243,11 +243,10 @@ define i64 @fun11(i16 zeroext %v) {
;
; X64-LABEL: fun11:
; X64: # %bb.0: # %entry
-; X64-NEXT: # kill: def $edi killed $edi def $rdi
; X64-NEXT: movl %edi, %eax
; X64-NEXT: shrl $4, %eax
; X64-NEXT: andl $-16, %edi
-; X64-NEXT: addq %rdi, %rax
+; X64-NEXT: addl %edi, %eax
; X64-NEXT: retq
entry:
%shr = lshr i16 %v, 4
diff --git a/llvm/test/CodeGen/X86/divrem8_ext.ll b/llvm/test/CodeGen/X86/divrem8_ext.ll
index bfc982d2def5d..6fb25d8a2443d 100644
--- a/llvm/test/CodeGen/X86/divrem8_ext.ll
+++ b/llvm/test/CodeGen/X86/divrem8_ext.ll
@@ -197,7 +197,7 @@ define i64 @pr25754(i8 %a, i8 %c) {
; X64-NEXT: divb %sil
; X64-NEXT: movzbl %ah, %ecx
; X64-NEXT: movzbl %al, %eax
-; X64-NEXT: addq %rcx, %rax
+; X64-NEXT: addl %ecx, %eax
; X64-NEXT: retq
%r1 = urem i8 %a, %c
%d1 = udiv i8 %a, %c
diff --git a/llvm/test/CodeGen/X86/fold-int-pow2-with-fmul-or-fdiv.ll b/llvm/test/CodeGen/X86/fold-int-pow2-with-fmul-or-fdiv.ll
index a8a6786d97ea0..58bd4e4ff5abb 100644
--- a/llvm/test/CodeGen/X86/fold-int-pow2-with-fmul-or-fdiv.ll
+++ b/llvm/test/CodeGen/X86/fold-int-pow2-with-fmul-or-fdiv.ll
@@ -13,7 +13,7 @@ define <4 x float> @fmul_pow2_4xfloat(<4 x i32> %i) {
; CHECK-SSE-LABEL: fmul_pow2_4xfloat:
; CHECK-SSE: # %bb.0:
; CHECK-SSE-NEXT: pslld $23, %xmm0
-; CHECK-SSE-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; CHECK-SSE-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [1091567616,1091567616,1091567616,1091567616]
; CHECK-SSE-NEXT: retq
;
; CHECK-AVX2-LABEL: fmul_pow2_4xfloat:
@@ -772,7 +772,8 @@ define double @fmul_pow_mul_max_pow2(i16 %cnt) nounwind {
; CHECK-SSE-LABEL: fmul_pow_mul_max_pow2:
; CHECK-SSE: # %bb.0:
; CHECK-SSE-NEXT: movzbl %dil, %eax
-; CHECK-SSE-NEXT: leaq 1(%rax), %rcx
+; CHECK-SSE-NEXT: movl %eax, %ecx
+; CHECK-SSE-NEXT: incl %ecx
; CHECK-SSE-NEXT: cmpq %rcx, %rax
; CHECK-SSE-NEXT: cmovaq %rax, %rcx
; CHECK-SSE-NEXT: shlq $52, %rcx
@@ -784,7 +785,8 @@ define double @fmul_pow_mul_max_pow2(i16 %cnt) nounwind {
; CHECK-AVX-LABEL: fmul_pow_mul_max_pow2:
; CHECK-AVX: # %bb.0:
; CHECK-AVX-NEXT: movzbl %dil, %eax
-; CHECK-AVX-NEXT: leaq 1(%rax), %rcx
+; CHECK-AVX-NEXT: movl %eax, %ecx
+; CHECK-AVX-NEXT: incl %ecx
; CHECK-AVX-NEXT: cmpq %rcx, %rax
; CHECK-AVX-NEXT: cmovaq %rax, %rcx
; CHECK-AVX-NEXT: shlq $52, %rcx
@@ -911,19 +913,19 @@ define <2 x double> @fmul_pow_shl_cnt_vec(<2 x i64> %cnt) nounwind {
; CHECK-SSE-LABEL: fmul_pow_shl_cnt_vec:
; CHECK-SSE: # %bb.0:
; CHECK-SSE-NEXT: psllq $52, %xmm0
-; CHECK-SSE-NEXT: paddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; CHECK-SSE-NEXT: paddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [4629137466983448576,4629137466983448576]
; CHECK-SSE-NEXT: retq
;
; CHECK-AVX2-LABEL: fmul_pow_shl_cnt_vec:
; CHECK-AVX2: # %bb.0:
; CHECK-AVX2-NEXT: vpsllq $52, %xmm0, %xmm0
-; CHECK-AVX2-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; CHECK-AVX2-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [4629137466983448576,4629137466983448576]
; CHECK-AVX2-NEXT: retq
;
; CHECK-ONLY-AVX512F-LABEL: fmul_pow_shl_cnt_vec:
; CHECK-ONLY-AVX512F: # %bb.0:
; CHECK-ONLY-AVX512F-NEXT: vpsllq $52, %xmm0, %xmm0
-; CHECK-ONLY-AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; CHECK-ONLY-AVX512F-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [4629137466983448576,4629137466983448576]
; CHECK-ONLY-AVX512F-NEXT: retq
;
; CHECK-SKX-LABEL: fmul_pow_shl_cnt_vec:
@@ -941,7 +943,7 @@ define <4 x float> @fmul_pow_shl_cnt_vec_preserve_fma(<4 x i32> %cnt, <4 x float
; CHECK-SSE-LABEL: fmul_pow_shl_cnt_vec_preserve_fma:
; CHECK-SSE: # %bb.0:
; CHECK-SSE-NEXT: pslld $23, %xmm0
-; CHECK-SSE-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; CHECK-SSE-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [1092616192,1092616192,1092616192,1092616192]
; CHECK-SSE-NEXT: addps %xmm1, %xmm0
; CHECK-SSE-NEXT: retq
;
@@ -980,7 +982,7 @@ define <4 x float> @fmul_pow_shl_cnt_vec_no_fma(<4 x i32> %cnt, <4 x float> %add
; CHECK-SSE-LABEL: fmul_pow_shl_cnt_vec_no_fma:
; CHECK-SSE: # %bb.0:
; CHECK-SSE-NEXT: pslld $23, %xmm0
-; CHECK-SSE-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; CHECK-SSE-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [1092616192,1092616192,1092616192,1092616192]
; CHECK-SSE-NEXT: addps %xmm1, %xmm0
; CHECK-SSE-NEXT: retq
;
@@ -1017,13 +1019,13 @@ define <2 x double> @fmul_pow_shl_cnt_vec_non_splat_todo(<2 x i64> %cnt) nounwin
; CHECK-SSE-LABEL: fmul_pow_shl_cnt_vec_non_splat_todo:
; CHECK-SSE: # %bb.0:
; CHECK-SSE-NEXT: psllq $52, %xmm0
-; CHECK-SSE-NEXT: paddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; CHECK-SSE-NEXT: paddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [4629137466983448576,4628574517030027264]
; CHECK-SSE-NEXT: retq
;
; CHECK-AVX-LABEL: fmul_pow_shl_cnt_vec_non_splat_todo:
; CHECK-AVX: # %bb.0:
; CHECK-AVX-NEXT: vpsllq $52, %xmm0, %xmm0
-; CHECK-AVX-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; CHECK-AVX-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [4629137466983448576,4628574517030027264]
; CHECK-AVX-NEXT: retq
%shl = shl nsw nuw <2 x i64> <i64 2, i64 2>, %cnt
%conv = uitofp <2 x i64> %shl to <2 x double>
@@ -1035,13 +1037,13 @@ define <2 x double> @fmul_pow_shl_cnt_vec_non_splat2_todo(<2 x i64> %cnt) nounwi
; CHECK-SSE-LABEL: fmul_pow_shl_cnt_vec_non_splat2_todo:
; CHECK-SSE: # %bb.0:
; CHECK-SSE-NEXT: psllq $52, %xmm0
-; CHECK-SSE-NEXT: paddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; CHECK-SSE-NEXT: paddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [4629137466983448576,4624633867356078080]
; CHECK-SSE-NEXT: retq
;
; CHECK-AVX-LABEL: fmul_pow_shl_cnt_vec_non_splat2_todo:
; CHECK-AVX: # %bb.0:
; CHECK-AVX-NEXT: vpsllq $52, %xmm0, %xmm0
-; CHECK-AVX-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; CHECK-AVX-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [4629137466983448576,4624633867356078080]
; CHECK-AVX-NEXT: retq
%shl = shl nsw nuw <2 x i64> <i64 2, i64 1>, %cnt
%conv = uitofp <2 x i64> %shl to <2 x double>
@@ -1056,7 +1058,7 @@ define <2 x half> @fmul_pow_shl_cnt_vec_fail_to_large(<2 x i16> %cnt) nounwind {
; CHECK-SSE-NEXT: pxor %xmm1, %xmm1
; CHECK-SSE-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
; CHECK-SSE-NEXT: pslld $23, %xmm0
-; CHECK-SSE-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; CHECK-SSE-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [1065353216,1065353216,1065353216,1065353216]
; CHECK-SSE-NEXT: cvttps2dq %xmm0, %xmm0
; CHECK-SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,2,2,3,4,5,6,7]
; CHECK-SSE-NEXT: pmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [2,2,u,u,u,u,u,u]
diff --git a/llvm/test/CodeGen/X86/h-registers-1.ll b/llvm/test/CodeGen/X86/h-registers-1.ll
index 07d85d260a37a..db8fb569f3fcd 100644
--- a/llvm/test/CodeGen/X86/h-registers-1.ll
+++ b/llvm/test/CodeGen/X86/h-registers-1.ll
@@ -28,13 +28,13 @@ define i64 @foo(i64 %a, i64 %b, i64 %c, i64 %d, i64 %e, i64 %f, i64 %g, i64 %h)
; CHECK-NEXT: movzbl %ah, %ebx
; CHECK-NEXT: movzbl {{[0-9]+}}(%rsp), %eax
; CHECK-NEXT: movzbl {{[0-9]+}}(%rsp), %r8d
-; CHECK-NEXT: addq %rdi, %rsi
-; CHECK-NEXT: addq %rbp, %rdx
-; CHECK-NEXT: addq %rsi, %rdx
-; CHECK-NEXT: addq %rbx, %rcx
-; CHECK-NEXT: addq %r8, %rax
-; CHECK-NEXT: addq %rcx, %rax
-; CHECK-NEXT: addq %rdx, %rax
+; CHECK-NEXT: addl %edi, %esi
+; CHECK-NEXT: addl %ebp, %edx
+; CHECK-NEXT: addl %esi, %edx
+; CHECK-NEXT: addl %ebx, %ecx
+; CHECK-NEXT: addl %r8d, %eax
+; CHECK-NEXT: addl %ecx, %eax
+; CHECK-NEXT: addl %edx, %eax
; CHECK-NEXT: popq %rbx
; CHECK-NEXT: .cfi_def_cfa_offset 16
; CHECK-NEXT: popq %rbp
@@ -61,13 +61,13 @@ define i64 @foo(i64 %a, i64 %b, i64 %c, i64 %d, i64 %e, i64 %f, i64 %g, i64 %h)
; GNUX32-NEXT: movzbl %ah, %ebx
; GNUX32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
; GNUX32-NEXT: movzbl {{[0-9]+}}(%esp), %r8d
-; GNUX32-NEXT: addq %rdi, %rsi
-; GNUX32-NEXT: addq %rbp, %rdx
-; GNUX32-NEXT: addq %rsi, %rdx
-; GNUX32-NEXT: addq %rbx, %rcx
-; GNUX32-NEXT: addq %r8, %rax
-; GNUX32-NEXT: addq %rcx, %rax
-; GNUX32-NEXT: addq %rdx, %rax
+; GNUX32-NEXT: addl %edi, %esi
+; GNUX32-NEXT: addl %ebp, %edx
+; GNUX32-NEXT: addl %esi, %edx
+; GNUX32-NEXT: addl %ebx, %ecx
+; GNUX32-NEXT: addl %r8d, %eax
+; GNUX32-NEXT: addl %ecx, %eax
+; GNUX32-NEXT: addl %edx, %eax
; GNUX32-NEXT: popq %rbx
; GNUX32-NEXT: .cfi_def_cfa_offset 16
; GNUX32-NEXT: popq %rbp
diff --git a/llvm/test/CodeGen/X86/narrow-add-i64.ll b/llvm/test/CodeGen/X86/narrow-add-i64.ll
index a7a54fd57413b..3d05395085d7f 100644
--- a/llvm/test/CodeGen/X86/narrow-add-i64.ll
+++ b/llvm/test/CodeGen/X86/narrow-add-i64.ll
@@ -13,7 +13,7 @@ define i64 @test_add_i64_i16_const(i16 %a) nounwind {
; X64-LABEL: test_add_i64_i16_const:
; X64: # %bb.0:
; X64-NEXT: movzwl %di, %eax
-; X64-NEXT: addq $42, %rax
+; X64-NEXT: addl $42, %eax
; X64-NEXT: retq
%zext_a = zext i16 %a to i64
%sum = add nuw nsw i64 %zext_a, 42
@@ -32,9 +32,9 @@ define i64 @test_add_i64_i16_zext(i16 %a, i16 %b) nounwind {
;
; X64-LABEL: test_add_i64_i16_zext:
; X64: # %bb.0:
-; X64-NEXT: movzwl %di, %ecx
-; X64-NEXT: movzwl %si, %eax
-; X64-NEXT: addq %rcx, %rax
+; X64-NEXT: movzwl %si, %ecx
+; X64-NEXT: movzwl %di, %eax
+; X64-NEXT: addl %ecx, %eax
; X64-NEXT: retq
%zext_a = zext i16 %a to i64
%zext_b = zext i16 %b to i64
diff --git a/llvm/test/CodeGen/X86/popcnt.ll b/llvm/test/CodeGen/X86/popcnt.ll
index 3004b8b72fcc5..e9398b788a7b6 100644
--- a/llvm/test/CodeGen/X86/popcnt.ll
+++ b/llvm/test/CodeGen/X86/popcnt.ll
@@ -427,39 +427,39 @@ define i128 @cnt128(i128 %x) nounwind readnone {
; X64-BASE: # %bb.0:
; X64-BASE-NEXT: movq %rsi, %rax
; X64-BASE-NEXT: shrq %rax
-; X64-BASE-NEXT: movabsq $6148914691236517205, %r8 # imm = 0x5555555555555555
-; X64-BASE-NEXT: andq %r8, %rax
+; X64-BASE-NEXT: movabsq $6148914691236517205, %rdx # imm = 0x5555555555555555
+; X64-BASE-NEXT: andq %rdx, %rax
; X64-BASE-NEXT: subq %rax, %rsi
-; X64-BASE-NEXT: movabsq $3689348814741910323, %rcx # imm = 0x3333333333333333
-; X64-BASE-NEXT: movq %rsi, %rax
-; X64-BASE-NEXT: andq %rcx, %rax
+; X64-BASE-NEXT: movabsq $3689348814741910323, %rax # imm = 0x3333333333333333
+; X64-BASE-NEXT: movq %rsi, %r8
+; X64-BASE-NEXT: andq %rax, %r8
; X64-BASE-NEXT: shrq $2, %rsi
-; X64-BASE-NEXT: andq %rcx, %rsi
-; X64-BASE-NEXT: addq %rsi, %rax
-; X64-BASE-NEXT: movq %rax, %rdx
-; X64-BASE-NEXT: shrq $4, %rdx
-; X64-BASE-NEXT: addq %rax, %rdx
+; X64-BASE-NEXT: andq %rax, %rsi
+; X64-BASE-NEXT: addq %rsi, %r8
+; X64-BASE-NEXT: movq %r8, %rcx
+; X64-BASE-NEXT: shrq $4, %rcx
+; X64-BASE-NEXT: addq %r8, %rcx
; X64-BASE-NEXT: movabsq $1085102592571150095, %rsi # imm = 0xF0F0F0F0F0F0F0F
-; X64-BASE-NEXT: andq %rsi, %rdx
-; X64-BASE-NEXT: movabsq $72340172838076673, %r9 # imm = 0x101010101010101
-; X64-BASE-NEXT: imulq %r9, %rdx
-; X64-BASE-NEXT: shrq $56, %rdx
-; X64-BASE-NEXT: movq %rdi, %rax
-; X64-BASE-NEXT: shrq %rax
-; X64-BASE-NEXT: andq %r8, %rax
-; X64-BASE-NEXT: subq %rax, %rdi
-; X64-BASE-NEXT: movq %rdi, %rax
-; X64-BASE-NEXT: andq %rcx, %rax
+; X64-BASE-NEXT: andq %rsi, %rcx
+; X64-BASE-NEXT: movabsq $72340172838076673, %r8 # imm = 0x101010101010101
+; X64-BASE-NEXT: imulq %r8, %rcx
+; X64-BASE-NEXT: shrq $56, %rcx
+; X64-BASE-NEXT: movq %rdi, %r9
+; X64-BASE-NEXT: shrq %r9
+; X64-BASE-NEXT: andq %rdx, %r9
+; X64-BASE-NEXT: subq %r9, %rdi
+; X64-BASE-NEXT: movq %rdi, %rdx
+; X64-BASE-NEXT: andq %rax, %rdx
; X64-BASE-NEXT: shrq $2, %rdi
-; X64-BASE-NEXT: andq %rdi, %rcx
-; X64-BASE-NEXT: addq %rax, %rcx
-; X64-BASE-NEXT: movq %rcx, %rax
+; X64-BASE-NEXT: andq %rax, %rdi
+; X64-BASE-NEXT: addq %rdx, %rdi
+; X64-BASE-NEXT: movq %rdi, %rax
; X64-BASE-NEXT: shrq $4, %rax
-; X64-BASE-NEXT: addq %rcx, %rax
+; X64-BASE-NEXT: addq %rdi, %rax
; X64-BASE-NEXT: andq %rsi, %rax
-; X64-BASE-NEXT: imulq %r9, %rax
+; X64-BASE-NEXT: imulq %r8, %rax
; X64-BASE-NEXT: shrq $56, %rax
-; X64-BASE-NEXT: addq %rdx, %rax
+; X64-BASE-NEXT: addl %ecx, %eax
; X64-BASE-NEXT: xorl %edx, %edx
; X64-BASE-NEXT: retq
;
@@ -491,7 +491,7 @@ define i128 @cnt128(i128 %x) nounwind readnone {
; X64-POPCNT: # %bb.0:
; X64-POPCNT-NEXT: popcntq %rsi, %rcx
; X64-POPCNT-NEXT: popcntq %rdi, %rax
-; X64-POPCNT-NEXT: addq %rcx, %rax
+; X64-POPCNT-NEXT: addl %ecx, %eax
; X64-POPCNT-NEXT: xorl %edx, %edx
; X64-POPCNT-NEXT: retq
;
@@ -525,7 +525,7 @@ define i128 @cnt128(i128 %x) nounwind readnone {
; X64-NDD-NEXT: andq %rsi, %rax
; X64-NDD-NEXT: imulq %r8, %rax
; X64-NDD-NEXT: shrq $56, %rax
-; X64-NDD-NEXT: addq %rdx, %rax
+; X64-NDD-NEXT: addl %edx, %eax
; X64-NDD-NEXT: xorl %edx, %edx
; X64-NDD-NEXT: retq
;
@@ -1045,39 +1045,39 @@ define i128 @cnt128_optsize(i128 %x) nounwind readnone optsize {
; X64-BASE: # %bb.0:
; X64-BASE-NEXT: movq %rsi, %rax
; X64-BASE-NEXT: shrq %rax
-; X64-BASE-NEXT: movabsq $6148914691236517205, %r8 # imm = 0x5555555555555555
-; X64-BASE-NEXT: andq %r8, %rax
+; X64-BASE-NEXT: movabsq $6148914691236517205, %rdx # imm = 0x5555555555555555
+; X64-BASE-NEXT: andq %rdx, %rax
; X64-BASE-NEXT: subq %rax, %rsi
-; X64-BASE-NEXT: movabsq $3689348814741910323, %rcx # imm = 0x3333333333333333
-; X64-BASE-NEXT: movq %rsi, %rax
-; X64-BASE-NEXT: andq %rcx, %rax
+; X64-BASE-NEXT: movabsq $3689348814741910323, %rax # imm = 0x3333333333333333
+; X64-BASE-NEXT: movq %rsi, %r8
+; X64-BASE-NEXT: andq %rax, %r8
; X64-BASE-NEXT: shrq $2, %rsi
-; X64-BASE-NEXT: andq %rcx, %rsi
-; X64-BASE-NEXT: addq %rsi, %rax
-; X64-BASE-NEXT: movq %rax, %rdx
-; X64-BASE-NEXT: shrq $4, %rdx
-; X64-BASE-NEXT: addq %rax, %rdx
+; X64-BASE-NEXT: andq %rax, %rsi
+; X64-BASE-NEXT: addq %rsi, %r8
+; X64-BASE-NEXT: movq %r8, %rcx
+; X64-BASE-NEXT: shrq $4, %rcx
+; X64-BASE-NEXT: addq %r8, %rcx
; X64-BASE-NEXT: movabsq $1085102592571150095, %rsi # imm = 0xF0F0F0F0F0F0F0F
-; X64-BASE-NEXT: andq %rsi, %rdx
-; X64-BASE-NEXT: movabsq $72340172838076673, %r9 # imm = 0x101010101010101
-; X64-BASE-NEXT: imulq %r9, %rdx
-; X64-BASE-NEXT: shrq $56, %rdx
-; X64-BASE-NEXT: movq %rdi, %rax
-; X64-BASE-NEXT: shrq %rax
-; X64-BASE-NEXT: andq %r8, %rax
-; X64-BASE-NEXT: subq %rax, %rdi
-; X64-BASE-NEXT: movq %rdi, %rax
-; X64-BASE-NEXT: andq %rcx, %rax
+; X64-BASE-NEXT: andq %rsi, %rcx
+; X64-BASE-NEXT: movabsq $72340172838076673, %r8 # imm = 0x101010101010101
+; X64-BASE-NEXT: imulq %r8, %rcx
+; X64-BASE-NEXT: shrq $56, %rcx
+; X64-BASE-NEXT: movq %rdi, %r9
+; X64-BASE-NEXT: shrq %r9
+; X64-BASE-NEXT: andq %rdx, %r9
+; X64-BASE-NEXT: subq %r9, %rdi
+; X64-BASE-NEXT: movq %rdi, %rdx
+; X64-BASE-NEXT: andq %rax, %rdx
; X64-BASE-NEXT: shrq $2, %rdi
-; X64-BASE-NEXT: andq %rdi, %rcx
-; X64-BASE-NEXT: addq %rax, %rcx
-; X64-BASE-NEXT: movq %rcx, %rax
+; X64-BASE-NEXT: andq %rax, %rdi
+; X64-BASE-NEXT: addq %rdx, %rdi
+; X64-BASE-NEXT: movq %rdi, %rax
; X64-BASE-NEXT: shrq $4, %rax
-; X64-BASE-NEXT: addq %rcx, %rax
+; X64-BASE-NEXT: addq %rdi, %rax
; X64-BASE-NEXT: andq %rsi, %rax
-; X64-BASE-NEXT: imulq %r9, %rax
+; X64-BASE-NEXT: imulq %r8, %rax
; X64-BASE-NEXT: shrq $56, %rax
-; X64-BASE-NEXT: addq %rdx, %rax
+; X64-BASE-NEXT: addl %ecx, %eax
; X64-BASE-NEXT: xorl %edx, %edx
; X64-BASE-NEXT: retq
;
@@ -1110,7 +1110,7 @@ define i128 @cnt128_optsize(i128 %x) nounwind readnone optsize {
; X64-POPCNT: # %bb.0:
; X64-POPCNT-NEXT: popcntq %rsi, %rcx
; X64-POPCNT-NEXT: popcntq %rdi, %rax
-; X64-POPCNT-NEXT: addq %rcx, %rax
+; X64-POPCNT-NEXT: addl %ecx, %eax
; X64-POPCNT-NEXT: xorl %edx, %edx
; X64-POPCNT-NEXT: retq
;
@@ -1144,7 +1144,7 @@ define i128 @cnt128_optsize(i128 %x) nounwind readnone optsize {
; X64-NDD-NEXT: andq %rsi, %rax
; X64-NDD-NEXT: imulq %r8, %rax
; X64-NDD-NEXT: shrq $56, %rax
-; X64-NDD-NEXT: addq %rdx, %rax
+; X64-NDD-NEXT: addl %edx, %eax
; X64-NDD-NEXT: xorl %edx, %edx
; X64-NDD-NEXT: retq
;
@@ -1547,39 +1547,39 @@ define i128 @cnt128_pgso(i128 %x) nounwind readnone !prof !14 {
; X64-BASE: # %bb.0:
; X64-BASE-NEXT: movq %rsi, %rax
; X64-BASE-NEXT: shrq %rax
-; X64-BASE-NEXT: movabsq $6148914691236517205, %r8 # imm = 0x5555555555555555
-; X64-BASE-NEXT: andq %r8, %rax
+; X64-BASE-NEXT: movabsq $6148914691236517205, %rdx # imm = 0x5555555555555555
+; X64-BASE-NEXT: andq %rdx, %rax
; X64-BASE-NEXT: subq %rax, %rsi
-; X64-BASE-NEXT: movabsq $3689348814741910323, %rcx # imm = 0x3333333333333333
-; X64-BASE-NEXT: movq %rsi, %rax
-; X64-BASE-NEXT: andq %rcx, %rax
+; X64-BASE-NEXT: movabsq $3689348814741910323, %rax # imm = 0x3333333333333333
+; X64-BASE-NEXT: movq %rsi, %r8
+; X64-BASE-NEXT: andq %rax, %r8
; X64-BASE-NEXT: shrq $2, %rsi
-; X64-BASE-NEXT: andq %rcx, %rsi
-; X64-BASE-NEXT: addq %rsi, %rax
-; X64-BASE-NEXT: movq %rax, %rdx
-; X64-BASE-NEXT: shrq $4, %rdx
-; X64-BASE-NEXT: addq %rax, %rdx
+; X64-BASE-NEXT: andq %rax, %rsi
+; X64-BASE-NEXT: addq %rsi, %r8
+; X64-BASE-NEXT: movq %r8, %rcx
+; X64-BASE-NEXT: shrq $4, %rcx
+; X64-BASE-NEXT: addq %r8, %rcx
; X64-BASE-NEXT: movabsq $1085102592571150095, %rsi # imm = 0xF0F0F0F0F0F0F0F
-; X64-BASE-NEXT: andq %rsi, %rdx
-; X64-BASE-NEXT: movabsq $72340172838076673, %r9 # imm = 0x101010101010101
-; X64-BASE-NEXT: imulq %r9, %rdx
-; X64-BASE-NEXT: shrq $56, %rdx
-; X64-BASE-NEXT: movq %rdi, %rax
-; X64-BASE-NEXT: shrq %rax
-; X64-BASE-NEXT: andq %r8, %rax
-; X64-BASE-NEXT: subq %rax, %rdi
-; X64-BASE-NEXT: movq %rdi, %rax
-; X64-BASE-NEXT: andq %rcx, %rax
+; X64-BASE-NEXT: andq %rsi, %rcx
+; X64-BASE-NEXT: movabsq $72340172838076673, %r8 # imm = 0x101010101010101
+; X64-BASE-NEXT: imulq %r8, %rcx
+; X64-BASE-NEXT: shrq $56, %rcx
+; X64-BASE-NEXT: movq %rdi, %r9
+; X64-BASE-NEXT: shrq %r9
+; X64-BASE-NEXT: andq %rdx, %r9
+; X64-BASE-NEXT: subq %r9, %rdi
+; X64-BASE-NEXT: movq %rdi, %rdx
+; X64-BASE-NEXT: andq %rax, %rdx
; X64-BASE-NEXT: shrq $2, %rdi
-; X64-BASE-NEXT: andq %rdi, %rcx
-; X64-BASE-NEXT: addq %rax, %rcx
-; X64-BASE-NEXT: movq %rcx, %rax
+; X64-BASE-NEXT: andq %rax, %rdi
+; X64-BASE-NEXT: addq %rdx, %rdi
+; X64-BASE-NEXT: movq %rdi, %rax
; X64-BASE-NEXT: shrq $4, %rax
-; X64-BASE-NEXT: addq %rcx, %rax
+; X64-BASE-NEXT: addq %rdi, %rax
; X64-BASE-NEXT: andq %rsi, %rax
-; X64-BASE-NEXT: imulq %r9, %rax
+; X64-BASE-NEXT: imulq %r8, %rax
; X64-BASE-NEXT: shrq $56, %rax
-; X64-BASE-NEXT: addq %rdx, %rax
+; X64-BASE-NEXT: addl %ecx, %eax
; X64-BASE-NEXT: xorl %edx, %edx
; X64-BASE-NEXT: retq
;
@@ -1612,7 +1612,7 @@ define i128 @cnt128_pgso(i128 %x) nounwind readnone !prof !14 {
; X64-POPCNT: # %bb.0:
; X64-POPCNT-NEXT: popcntq %rsi, %rcx
; X64-POPCNT-NEXT: popcntq %rdi, %rax
-; X64-POPCNT-NEXT: addq %rcx, %rax
+; X64-POPCNT-NEXT: addl %ecx, %eax
; X64-POPCNT-NEXT: xorl %edx, %edx
; X64-POPCNT-NEXT: retq
;
@@ -1646,7 +1646,7 @@ define i128 @cnt128_pgso(i128 %x) nounwind readnone !prof !14 {
; X64-NDD-NEXT: andq %rsi, %rax
; X64-NDD-NEXT: imulq %r8, %rax
; X64-NDD-NEXT: shrq $56, %rax
-; X64-NDD-NEXT: addq %rdx, %rax
+; X64-NDD-NEXT: addl %edx, %eax
; X64-NDD-NEXT: xorl %edx, %edx
; X64-NDD-NEXT: retq
;
diff --git a/llvm/test/CodeGen/X86/pr173924.ll b/llvm/test/CodeGen/X86/pr173924.ll
index a25f62a0ab071..60e5b03d3e6dd 100644
--- a/llvm/test/CodeGen/X86/pr173924.ll
+++ b/llvm/test/CodeGen/X86/pr173924.ll
@@ -1,38 +1,81 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=CHECK,V3
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=CHECK,V4
define i256 @PR173924(<8 x i256> %a0) {
-; CHECK-LABEL: PR173924:
-; CHECK: # %bb.0:
-; CHECK-NEXT: movq %rdi, %rax
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rdi
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rdx
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rcx
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r8
-; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r10
-; CHECK-NEXT: vmovd {{.*#+}} xmm0 = [1,0,0,0]
-; CHECK-NEXT: vpand {{[0-9]+}}(%rsp), %ymm0, %ymm0
-; CHECK-NEXT: vmovq %xmm0, %r11
-; CHECK-NEXT: andl $1, %r10d
-; CHECK-NEXT: andl $1, %esi
-; CHECK-NEXT: addq %r10, %rsi
-; CHECK-NEXT: andl $1, %r8d
-; CHECK-NEXT: andl $1, %ecx
-; CHECK-NEXT: addq %r8, %rcx
-; CHECK-NEXT: addq %rsi, %rcx
-; CHECK-NEXT: andl $1, %edx
-; CHECK-NEXT: addq %r11, %rdx
-; CHECK-NEXT: andl $1, %edi
-; CHECK-NEXT: andl $1, %r9d
-; CHECK-NEXT: addq %rdi, %r9
-; CHECK-NEXT: addq %rdx, %r9
-; CHECK-NEXT: addq %rcx, %r9
-; CHECK-NEXT: vmovq %r9, %xmm0
-; CHECK-NEXT: vmovdqu %ymm0, (%rax)
-; CHECK-NEXT: vzeroupper
-; CHECK-NEXT: retq
+; V3-LABEL: PR173924:
+; V3: # %bb.0:
+; V3-NEXT: movq %rdi, %rax
+; V3-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
+; V3-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
+; V3-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero
+; V3-NEXT: vmovq {{.*#+}} xmm3 = mem[0],zero
+; V3-NEXT: vmovq {{.*#+}} xmm4 = mem[0],zero
+; V3-NEXT: vmovq {{.*#+}} xmm5 = mem[0],zero
+; V3-NEXT: andl $1, %r9d
+; V3-NEXT: andl $1, %esi
+; V3-NEXT: vmovd {{.*#+}} xmm6 = [1,0,0,0]
+; V3-NEXT: vpand %xmm6, %xmm5, %xmm5
+; V3-NEXT: vmovd %xmm5, %ecx
+; V3-NEXT: vpand %xmm6, %xmm4, %xmm4
+; V3-NEXT: vmovd %xmm4, %edx
+; V3-NEXT: addl %ecx, %edx
+; V3-NEXT: vpand %xmm6, %xmm3, %xmm3
+; V3-NEXT: vmovd %xmm3, %ecx
+; V3-NEXT: vpand %xmm6, %xmm2, %xmm2
+; V3-NEXT: vmovd %xmm2, %edi
+; V3-NEXT: addl %ecx, %edi
+; V3-NEXT: vpand %xmm6, %xmm1, %xmm1
+; V3-NEXT: vmovd %xmm1, %ecx
+; V3-NEXT: addl %esi, %ecx
+; V3-NEXT: addl %edx, %ecx
+; V3-NEXT: vpand %xmm6, %xmm0, %xmm0
+; V3-NEXT: vmovd %xmm0, %edx
+; V3-NEXT: addl %r9d, %edx
+; V3-NEXT: addl %edi, %edx
+; V3-NEXT: addl %ecx, %edx
+; V3-NEXT: vmovd %edx, %xmm0
+; V3-NEXT: vmovdqu %ymm0, (%rax)
+; V3-NEXT: vzeroupper
+; V3-NEXT: retq
+;
+; V4-LABEL: PR173924:
+; V4: # %bb.0:
+; V4-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
+; V4-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
+; V4-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero
+; V4-NEXT: vmovq {{.*#+}} xmm3 = mem[0],zero
+; V4-NEXT: vmovq {{.*#+}} xmm4 = mem[0],zero
+; V4-NEXT: vmovd {{.*#+}} xmm5 = [1,0,0,0]
+; V4-NEXT: vpand {{[0-9]+}}(%rsp), %ymm5, %ymm6
+; V4-NEXT: movq %rdi, %rax
+; V4-NEXT: andl $1, %esi
+; V4-NEXT: andl $1, %r9d
+; V4-NEXT: vpand %xmm5, %xmm4, %xmm4
+; V4-NEXT: vmovd %xmm4, %ecx
+; V4-NEXT: vmovd %xmm6, %edx
+; V4-NEXT: addl %ecx, %edx
+; V4-NEXT: vpand %xmm5, %xmm3, %xmm3
+; V4-NEXT: vmovd %xmm3, %ecx
+; V4-NEXT: vpand %xmm5, %xmm2, %xmm2
+; V4-NEXT: vmovd %xmm2, %edi
+; V4-NEXT: addl %ecx, %edi
+; V4-NEXT: vpand %xmm5, %xmm1, %xmm1
+; V4-NEXT: vmovd %xmm1, %ecx
+; V4-NEXT: addl %r9d, %ecx
+; V4-NEXT: addl %edx, %ecx
+; V4-NEXT: vpand %xmm5, %xmm0, %xmm0
+; V4-NEXT: vmovd %xmm0, %edx
+; V4-NEXT: addl %esi, %edx
+; V4-NEXT: addl %edi, %edx
+; V4-NEXT: addl %ecx, %edx
+; V4-NEXT: vmovd %edx, %xmm0
+; V4-NEXT: vmovdqu %ymm0, (%rax)
+; V4-NEXT: vzeroupper
+; V4-NEXT: retq
%m = and <8 x i256> %a0, splat (i256 1)
%r = call i256 @llvm.vector.reduce.add.v8i256(<8 x i256> %m)
ret i256 %r
}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
diff --git a/llvm/test/CodeGen/X86/scheduler-backtracking.ll b/llvm/test/CodeGen/X86/scheduler-backtracking.ll
index 28029793211f0..fade8cbd21685 100644
--- a/llvm/test/CodeGen/X86/scheduler-backtracking.ll
+++ b/llvm/test/CodeGen/X86/scheduler-backtracking.ll
@@ -220,30 +220,30 @@ define i256 @test2(i256 %a) nounwind {
; ILP-NEXT: movl $0, %r9d
; ILP-NEXT: sbbq %rcx, %r9
; ILP-NEXT: sbbq %r8, %rdi
+; ILP-NEXT: andq %rcx, %r9
; ILP-NEXT: andq %r8, %rdi
; ILP-NEXT: bsrq %rdi, %r8
; ILP-NEXT: andq %rdx, %r10
; ILP-NEXT: bsrq %r10, %rdx
; ILP-NEXT: xorq $63, %r8
-; ILP-NEXT: andq %rcx, %r9
; ILP-NEXT: bsrq %r9, %rcx
-; ILP-NEXT: xorq $63, %rcx
-; ILP-NEXT: orq $64, %rcx
+; ILP-NEXT: xorl $63, %ecx
+; ILP-NEXT: orl $64, %ecx
; ILP-NEXT: testq %rdi, %rdi
+; ILP-NEXT: movq $0, 8(%rax)
; ILP-NEXT: cmovneq %r8, %rcx
-; ILP-NEXT: xorq $63, %rdx
+; ILP-NEXT: xorl $63, %edx
; ILP-NEXT: andq %rsi, %r11
; ILP-NEXT: movl $127, %esi
; ILP-NEXT: bsrq %r11, %rsi
-; ILP-NEXT: xorq $63, %rsi
-; ILP-NEXT: addq $64, %rsi
+; ILP-NEXT: xorl $63, %esi
+; ILP-NEXT: addl $64, %esi
; ILP-NEXT: testq %r10, %r10
-; ILP-NEXT: cmovneq %rdx, %rsi
-; ILP-NEXT: subq $-128, %rsi
-; ILP-NEXT: orq %rdi, %r9
+; ILP-NEXT: cmovnel %edx, %esi
+; ILP-NEXT: subl $-128, %esi
+; ILP-NEXT: orq %r9, %rdi
; ILP-NEXT: cmovneq %rcx, %rsi
; ILP-NEXT: movq %rsi, (%rax)
-; ILP-NEXT: movq $0, 8(%rax)
; ILP-NEXT: retq
;
; HYBRID-LABEL: test2:
@@ -264,21 +264,21 @@ define i256 @test2(i256 %a) nounwind {
; HYBRID-NEXT: xorq $63, %r8
; HYBRID-NEXT: andq %rcx, %r9
; HYBRID-NEXT: bsrq %r9, %rcx
-; HYBRID-NEXT: xorq $63, %rcx
-; HYBRID-NEXT: orq $64, %rcx
+; HYBRID-NEXT: xorl $63, %ecx
+; HYBRID-NEXT: orl $64, %ecx
; HYBRID-NEXT: testq %rdi, %rdi
; HYBRID-NEXT: cmovneq %r8, %rcx
; HYBRID-NEXT: andq %rdx, %r10
; HYBRID-NEXT: bsrq %r10, %rdx
-; HYBRID-NEXT: xorq $63, %rdx
+; HYBRID-NEXT: xorl $63, %edx
; HYBRID-NEXT: andq %rsi, %r11
; HYBRID-NEXT: movl $127, %esi
; HYBRID-NEXT: bsrq %r11, %rsi
-; HYBRID-NEXT: xorq $63, %rsi
-; HYBRID-NEXT: addq $64, %rsi
+; HYBRID-NEXT: xorl $63, %esi
+; HYBRID-NEXT: addl $64, %esi
; HYBRID-NEXT: testq %r10, %r10
-; HYBRID-NEXT: cmovneq %rdx, %rsi
-; HYBRID-NEXT: subq $-128, %rsi
+; HYBRID-NEXT: cmovnel %edx, %esi
+; HYBRID-NEXT: subl $-128, %esi
; HYBRID-NEXT: orq %rdi, %r9
; HYBRID-NEXT: cmovneq %rcx, %rsi
; HYBRID-NEXT: movq %rsi, (%rax)
@@ -303,21 +303,21 @@ define i256 @test2(i256 %a) nounwind {
; BURR-NEXT: xorq $63, %r8
; BURR-NEXT: andq %rcx, %r9
; BURR-NEXT: bsrq %r9, %rcx
-; BURR-NEXT: xorq $63, %rcx
-; BURR-NEXT: orq $64, %rcx
+; BURR-NEXT: xorl $63, %ecx
+; BURR-NEXT: orl $64, %ecx
; BURR-NEXT: testq %rdi, %rdi
; BURR-NEXT: cmovneq %r8, %rcx
; BURR-NEXT: andq %rdx, %r10
; BURR-NEXT: bsrq %r10, %rdx
-; BURR-NEXT: xorq $63, %rdx
+; BURR-NEXT: xorl $63, %edx
; BURR-NEXT: andq %rsi, %r11
; BURR-NEXT: movl $127, %esi
; BURR-NEXT: bsrq %r11, %rsi
-; BURR-NEXT: xorq $63, %rsi
-; BURR-NEXT: addq $64, %rsi
+; BURR-NEXT: xorl $63, %esi
+; BURR-NEXT: addl $64, %esi
; BURR-NEXT: testq %r10, %r10
-; BURR-NEXT: cmovneq %rdx, %rsi
-; BURR-NEXT: subq $-128, %rsi
+; BURR-NEXT: cmovnel %edx, %esi
+; BURR-NEXT: subl $-128, %esi
; BURR-NEXT: orq %rdi, %r9
; BURR-NEXT: cmovneq %rcx, %rsi
; BURR-NEXT: movq %rsi, (%rax)
@@ -339,24 +339,24 @@ define i256 @test2(i256 %a) nounwind {
; SRC-NEXT: andq %rcx, %r9
; SRC-NEXT: andq %r8, %rdi
; SRC-NEXT: andq %rsi, %r11
-; SRC-NEXT: bsrq %rdi, %rcx
-; SRC-NEXT: xorq $63, %rcx
-; SRC-NEXT: bsrq %r9, %rdx
+; SRC-NEXT: bsrq %rdi, %rdx
; SRC-NEXT: xorq $63, %rdx
-; SRC-NEXT: orq $64, %rdx
+; SRC-NEXT: bsrq %r9, %rcx
+; SRC-NEXT: xorl $63, %ecx
+; SRC-NEXT: orl $64, %ecx
; SRC-NEXT: testq %rdi, %rdi
-; SRC-NEXT: cmovneq %rcx, %rdx
-; SRC-NEXT: bsrq %r10, %rcx
-; SRC-NEXT: xorq $63, %rcx
+; SRC-NEXT: cmovneq %rdx, %rcx
+; SRC-NEXT: bsrq %r10, %rdx
+; SRC-NEXT: xorl $63, %edx
; SRC-NEXT: movl $127, %esi
; SRC-NEXT: bsrq %r11, %rsi
-; SRC-NEXT: xorq $63, %rsi
-; SRC-NEXT: addq $64, %rsi
+; SRC-NEXT: xorl $63, %esi
+; SRC-NEXT: addl $64, %esi
; SRC-NEXT: testq %r10, %r10
-; SRC-NEXT: cmovneq %rcx, %rsi
-; SRC-NEXT: subq $-128, %rsi
+; SRC-NEXT: cmovnel %edx, %esi
+; SRC-NEXT: subl $-128, %esi
; SRC-NEXT: orq %r9, %rdi
-; SRC-NEXT: cmovneq %rdx, %rsi
+; SRC-NEXT: cmovneq %rcx, %rsi
; SRC-NEXT: xorps %xmm0, %xmm0
; SRC-NEXT: movaps %xmm0, 16(%rax)
; SRC-NEXT: movq %rsi, (%rax)
@@ -373,23 +373,23 @@ define i256 @test2(i256 %a) nounwind {
; LIN-NEXT: negq %r9
; LIN-NEXT: andq %rsi, %r9
; LIN-NEXT: bsrq %r9, %rdi
-; LIN-NEXT: xorq $63, %rdi
-; LIN-NEXT: addq $64, %rdi
+; LIN-NEXT: xorl $63, %edi
+; LIN-NEXT: addl $64, %edi
; LIN-NEXT: xorl %esi, %esi
; LIN-NEXT: movl $0, %r9d
; LIN-NEXT: sbbq %rdx, %r9
; LIN-NEXT: andq %rdx, %r9
; LIN-NEXT: bsrq %r9, %rdx
-; LIN-NEXT: xorq $63, %rdx
+; LIN-NEXT: xorl $63, %edx
; LIN-NEXT: testq %r9, %r9
-; LIN-NEXT: cmoveq %rdi, %rdx
-; LIN-NEXT: subq $-128, %rdx
+; LIN-NEXT: cmovel %edi, %edx
+; LIN-NEXT: subl $-128, %edx
; LIN-NEXT: movl $0, %edi
; LIN-NEXT: sbbq %rcx, %rdi
; LIN-NEXT: andq %rcx, %rdi
; LIN-NEXT: bsrq %rdi, %rcx
-; LIN-NEXT: xorq $63, %rcx
-; LIN-NEXT: orq $64, %rcx
+; LIN-NEXT: xorl $63, %ecx
+; LIN-NEXT: orl $64, %ecx
; LIN-NEXT: sbbq %r8, %rsi
; LIN-NEXT: andq %r8, %rsi
; LIN-NEXT: bsrq %rsi, %r8
@@ -413,42 +413,42 @@ define i256 @test3(i256 %n) nounwind {
; ILP-NEXT: movq %rdi, %rax
; ILP-NEXT: xorps %xmm0, %xmm0
; ILP-NEXT: movaps %xmm0, 16(%rdi)
-; ILP-NEXT: xorl %r9d, %r9d
-; ILP-NEXT: movq %rsi, %rdi
-; ILP-NEXT: negq %rdi
+; ILP-NEXT: xorl %edi, %edi
+; ILP-NEXT: movq %rsi, %r9
+; ILP-NEXT: negq %r9
; ILP-NEXT: movl $0, %r10d
; ILP-NEXT: sbbq %rdx, %r10
; ILP-NEXT: movl $0, %r11d
; ILP-NEXT: sbbq %rcx, %r11
-; ILP-NEXT: sbbq %r8, %r9
+; ILP-NEXT: sbbq %r8, %rdi
+; ILP-NEXT: notq %rcx
+; ILP-NEXT: andq %r11, %rcx
; ILP-NEXT: notq %r8
-; ILP-NEXT: andq %r9, %r8
-; ILP-NEXT: bsrq %r8, %r9
+; ILP-NEXT: andq %rdi, %r8
+; ILP-NEXT: bsrq %rcx, %rdi
+; ILP-NEXT: bsrq %r8, %r11
; ILP-NEXT: notq %rdx
; ILP-NEXT: andq %r10, %rdx
; ILP-NEXT: bsrq %rdx, %r10
-; ILP-NEXT: xorq $63, %r9
-; ILP-NEXT: notq %rcx
-; ILP-NEXT: andq %r11, %rcx
-; ILP-NEXT: bsrq %rcx, %r11
; ILP-NEXT: xorq $63, %r11
-; ILP-NEXT: orq $64, %r11
-; ILP-NEXT: testq %r8, %r8
-; ILP-NEXT: cmovneq %r9, %r11
-; ILP-NEXT: xorq $63, %r10
+; ILP-NEXT: xorl $63, %edi
; ILP-NEXT: notq %rsi
-; ILP-NEXT: andq %rdi, %rsi
-; ILP-NEXT: movl $127, %edi
-; ILP-NEXT: bsrq %rsi, %rdi
-; ILP-NEXT: xorq $63, %rdi
-; ILP-NEXT: addq $64, %rdi
-; ILP-NEXT: testq %rdx, %rdx
-; ILP-NEXT: cmovneq %r10, %rdi
-; ILP-NEXT: subq $-128, %rdi
-; ILP-NEXT: orq %r8, %rcx
-; ILP-NEXT: cmovneq %r11, %rdi
-; ILP-NEXT: movq %rdi, (%rax)
+; ILP-NEXT: orl $64, %edi
+; ILP-NEXT: testq %r8, %r8
; ILP-NEXT: movq $0, 8(%rax)
+; ILP-NEXT: cmovneq %r11, %rdi
+; ILP-NEXT: xorl $63, %r10d
+; ILP-NEXT: andq %r9, %rsi
+; ILP-NEXT: movl $127, %r9d
+; ILP-NEXT: bsrq %rsi, %r9
+; ILP-NEXT: xorl $63, %r9d
+; ILP-NEXT: addl $64, %r9d
+; ILP-NEXT: testq %rdx, %rdx
+; ILP-NEXT: cmovnel %r10d, %r9d
+; ILP-NEXT: subl $-128, %r9d
+; ILP-NEXT: orq %rcx, %r8
+; ILP-NEXT: cmovneq %rdi, %r9
+; ILP-NEXT: movq %r9, (%rax)
; ILP-NEXT: retq
;
; HYBRID-LABEL: test3:
@@ -472,23 +472,23 @@ define i256 @test3(i256 %n) nounwind {
; HYBRID-NEXT: notq %rcx
; HYBRID-NEXT: andq %r11, %rcx
; HYBRID-NEXT: bsrq %rcx, %r9
-; HYBRID-NEXT: xorq $63, %r9
-; HYBRID-NEXT: orq $64, %r9
+; HYBRID-NEXT: xorl $63, %r9d
+; HYBRID-NEXT: orl $64, %r9d
; HYBRID-NEXT: testq %r8, %r8
; HYBRID-NEXT: cmovneq %rbx, %r9
; HYBRID-NEXT: notq %rdx
; HYBRID-NEXT: andq %r10, %rdx
; HYBRID-NEXT: bsrq %rdx, %r10
-; HYBRID-NEXT: xorq $63, %r10
+; HYBRID-NEXT: xorl $63, %r10d
; HYBRID-NEXT: notq %rsi
; HYBRID-NEXT: andq %rdi, %rsi
; HYBRID-NEXT: movl $127, %edi
; HYBRID-NEXT: bsrq %rsi, %rdi
-; HYBRID-NEXT: xorq $63, %rdi
-; HYBRID-NEXT: addq $64, %rdi
+; HYBRID-NEXT: xorl $63, %edi
+; HYBRID-NEXT: addl $64, %edi
; HYBRID-NEXT: testq %rdx, %rdx
-; HYBRID-NEXT: cmovneq %r10, %rdi
-; HYBRID-NEXT: subq $-128, %rdi
+; HYBRID-NEXT: cmovnel %r10d, %edi
+; HYBRID-NEXT: subl $-128, %edi
; HYBRID-NEXT: orq %r8, %rcx
; HYBRID-NEXT: cmovneq %r9, %rdi
; HYBRID-NEXT: movq %rdi, (%rax)
@@ -517,23 +517,23 @@ define i256 @test3(i256 %n) nounwind {
; BURR-NEXT: notq %rcx
; BURR-NEXT: andq %r11, %rcx
; BURR-NEXT: bsrq %rcx, %r9
-; BURR-NEXT: xorq $63, %r9
-; BURR-NEXT: orq $64, %r9
+; BURR-NEXT: xorl $63, %r9d
+; BURR-NEXT: orl $64, %r9d
; BURR-NEXT: testq %r8, %r8
; BURR-NEXT: cmovneq %rbx, %r9
; BURR-NEXT: notq %rdx
; BURR-NEXT: andq %r10, %rdx
; BURR-NEXT: bsrq %rdx, %r10
-; BURR-NEXT: xorq $63, %r10
+; BURR-NEXT: xorl $63, %r10d
; BURR-NEXT: notq %rsi
; BURR-NEXT: andq %rdi, %rsi
; BURR-NEXT: movl $127, %edi
; BURR-NEXT: bsrq %rsi, %rdi
-; BURR-NEXT: xorq $63, %rdi
-; BURR-NEXT: addq $64, %rdi
+; BURR-NEXT: xorl $63, %edi
+; BURR-NEXT: addl $64, %edi
; BURR-NEXT: testq %rdx, %rdx
-; BURR-NEXT: cmovneq %r10, %rdi
-; BURR-NEXT: subq $-128, %rdi
+; BURR-NEXT: cmovnel %r10d, %edi
+; BURR-NEXT: subl $-128, %edi
; BURR-NEXT: orq %r8, %rcx
; BURR-NEXT: cmovneq %r9, %rdi
; BURR-NEXT: movq %rdi, (%rax)
@@ -560,24 +560,24 @@ define i256 @test3(i256 %n) nounwind {
; SRC-NEXT: andq %r11, %rcx
; SRC-NEXT: andq %r9, %r8
; SRC-NEXT: andq %rdi, %rsi
-; SRC-NEXT: bsrq %r8, %rdi
-; SRC-NEXT: xorq $63, %rdi
-; SRC-NEXT: bsrq %rcx, %r9
+; SRC-NEXT: bsrq %r8, %r9
; SRC-NEXT: xorq $63, %r9
-; SRC-NEXT: orq $64, %r9
+; SRC-NEXT: bsrq %rcx, %rdi
+; SRC-NEXT: xorl $63, %edi
+; SRC-NEXT: orl $64, %edi
; SRC-NEXT: testq %r8, %r8
-; SRC-NEXT: cmovneq %rdi, %r9
-; SRC-NEXT: bsrq %rdx, %rdi
-; SRC-NEXT: xorq $63, %rdi
+; SRC-NEXT: cmovneq %r9, %rdi
+; SRC-NEXT: bsrq %rdx, %r9
+; SRC-NEXT: xorl $63, %r9d
; SRC-NEXT: movl $127, %r10d
; SRC-NEXT: bsrq %rsi, %r10
-; SRC-NEXT: xorq $63, %r10
-; SRC-NEXT: addq $64, %r10
+; SRC-NEXT: xorl $63, %r10d
+; SRC-NEXT: addl $64, %r10d
; SRC-NEXT: testq %rdx, %rdx
-; SRC-NEXT: cmovneq %rdi, %r10
-; SRC-NEXT: subq $-128, %r10
+; SRC-NEXT: cmovnel %r9d, %r10d
+; SRC-NEXT: subl $-128, %r10d
; SRC-NEXT: orq %rcx, %r8
-; SRC-NEXT: cmovneq %r9, %r10
+; SRC-NEXT: cmovneq %rdi, %r10
; SRC-NEXT: xorps %xmm0, %xmm0
; SRC-NEXT: movaps %xmm0, 16(%rax)
; SRC-NEXT: movq %r10, (%rax)
@@ -595,25 +595,25 @@ define i256 @test3(i256 %n) nounwind {
; LIN-NEXT: notq %rsi
; LIN-NEXT: andq %rdi, %rsi
; LIN-NEXT: bsrq %rsi, %r9
-; LIN-NEXT: xorq $63, %r9
-; LIN-NEXT: addq $64, %r9
+; LIN-NEXT: xorl $63, %r9d
+; LIN-NEXT: addl $64, %r9d
; LIN-NEXT: xorl %edi, %edi
; LIN-NEXT: movl $0, %esi
; LIN-NEXT: sbbq %rdx, %rsi
; LIN-NEXT: notq %rdx
; LIN-NEXT: andq %rsi, %rdx
; LIN-NEXT: bsrq %rdx, %rsi
-; LIN-NEXT: xorq $63, %rsi
+; LIN-NEXT: xorl $63, %esi
; LIN-NEXT: testq %rdx, %rdx
-; LIN-NEXT: cmoveq %r9, %rsi
-; LIN-NEXT: subq $-128, %rsi
+; LIN-NEXT: cmovel %r9d, %esi
+; LIN-NEXT: subl $-128, %esi
; LIN-NEXT: movl $0, %edx
; LIN-NEXT: sbbq %rcx, %rdx
; LIN-NEXT: notq %rcx
; LIN-NEXT: andq %rdx, %rcx
; LIN-NEXT: bsrq %rcx, %rdx
-; LIN-NEXT: xorq $63, %rdx
-; LIN-NEXT: orq $64, %rdx
+; LIN-NEXT: xorl $63, %edx
+; LIN-NEXT: orl $64, %edx
; LIN-NEXT: sbbq %r8, %rdi
; LIN-NEXT: notq %r8
; LIN-NEXT: andq %rdi, %r8
@@ -738,22 +738,22 @@ define i256 @PR25498(i256 %a) nounwind {
; ILP-NEXT: je .LBB4_1
; ILP-NEXT: # %bb.2: # %cond.false
; ILP-NEXT: bsrq %r10, %rdx
-; ILP-NEXT: bsrq %rdi, %rcx
-; ILP-NEXT: xorq $63, %rcx
-; ILP-NEXT: bsrq %r9, %rsi
-; ILP-NEXT: xorq $63, %rsi
-; ILP-NEXT: orq $64, %rsi
-; ILP-NEXT: testq %rdi, %rdi
-; ILP-NEXT: cmovneq %rcx, %rsi
-; ILP-NEXT: xorq $63, %rdx
+; ILP-NEXT: bsrq %rdi, %rsi
+; ILP-NEXT: xorl $63, %edx
; ILP-NEXT: bsrq %r11, %rcx
-; ILP-NEXT: xorq $63, %rcx
-; ILP-NEXT: orq $64, %rcx
+; ILP-NEXT: xorl $63, %ecx
+; ILP-NEXT: orl $64, %ecx
; ILP-NEXT: testq %r10, %r10
-; ILP-NEXT: cmovneq %rdx, %rcx
-; ILP-NEXT: orq $128, %rcx
+; ILP-NEXT: cmovnel %edx, %ecx
+; ILP-NEXT: xorq $63, %rsi
+; ILP-NEXT: bsrq %r9, %rdx
+; ILP-NEXT: xorl $63, %edx
+; ILP-NEXT: orl $64, %edx
+; ILP-NEXT: testq %rdi, %rdi
+; ILP-NEXT: cmovneq %rsi, %rdx
+; ILP-NEXT: subl $-128, %ecx
; ILP-NEXT: orq %rdi, %r9
-; ILP-NEXT: cmovneq %rsi, %rcx
+; ILP-NEXT: cmovneq %rdx, %rcx
; ILP-NEXT: jmp .LBB4_3
; ILP-NEXT: .LBB4_1:
; ILP-NEXT: movl $256, %ecx # imm = 0x100
@@ -783,18 +783,18 @@ define i256 @PR25498(i256 %a) nounwind {
; HYBRID-NEXT: bsrq %rdi, %rcx
; HYBRID-NEXT: xorq $63, %rcx
; HYBRID-NEXT: bsrq %r9, %rdx
-; HYBRID-NEXT: xorq $63, %rdx
-; HYBRID-NEXT: orq $64, %rdx
+; HYBRID-NEXT: xorl $63, %edx
+; HYBRID-NEXT: orl $64, %edx
; HYBRID-NEXT: testq %rdi, %rdi
; HYBRID-NEXT: cmovneq %rcx, %rdx
; HYBRID-NEXT: bsrq %r10, %rsi
-; HYBRID-NEXT: xorq $63, %rsi
+; HYBRID-NEXT: xorl $63, %esi
; HYBRID-NEXT: bsrq %r11, %rcx
-; HYBRID-NEXT: xorq $63, %rcx
-; HYBRID-NEXT: orq $64, %rcx
+; HYBRID-NEXT: xorl $63, %ecx
+; HYBRID-NEXT: orl $64, %ecx
; HYBRID-NEXT: testq %r10, %r10
-; HYBRID-NEXT: cmovneq %rsi, %rcx
-; HYBRID-NEXT: orq $128, %rcx
+; HYBRID-NEXT: cmovnel %esi, %ecx
+; HYBRID-NEXT: subl $-128, %ecx
; HYBRID-NEXT: orq %rdi, %r9
; HYBRID-NEXT: cmovneq %rdx, %rcx
; HYBRID-NEXT: jmp .LBB4_3
@@ -826,18 +826,18 @@ define i256 @PR25498(i256 %a) nounwind {
; BURR-NEXT: bsrq %rdi, %rcx
; BURR-NEXT: xorq $63, %rcx
; BURR-NEXT: bsrq %r9, %rdx
-; BURR-NEXT: xorq $63, %rdx
-; BURR-NEXT: orq $64, %rdx
+; BURR-NEXT: xorl $63, %edx
+; BURR-NEXT: orl $64, %edx
; BURR-NEXT: testq %rdi, %rdi
; BURR-NEXT: cmovneq %rcx, %rdx
; BURR-NEXT: bsrq %r10, %rsi
-; BURR-NEXT: xorq $63, %rsi
+; BURR-NEXT: xorl $63, %esi
; BURR-NEXT: bsrq %r11, %rcx
-; BURR-NEXT: xorq $63, %rcx
-; BURR-NEXT: orq $64, %rcx
+; BURR-NEXT: xorl $63, %ecx
+; BURR-NEXT: orl $64, %ecx
; BURR-NEXT: testq %r10, %r10
-; BURR-NEXT: cmovneq %rsi, %rcx
-; BURR-NEXT: orq $128, %rcx
+; BURR-NEXT: cmovnel %esi, %ecx
+; BURR-NEXT: subl $-128, %ecx
; BURR-NEXT: orq %rdi, %r9
; BURR-NEXT: cmovneq %rdx, %rcx
; BURR-NEXT: jmp .LBB4_3
@@ -869,18 +869,18 @@ define i256 @PR25498(i256 %a) nounwind {
; SRC-NEXT: bsrq %rdi, %rcx
; SRC-NEXT: xorq $63, %rcx
; SRC-NEXT: bsrq %r9, %rdx
-; SRC-NEXT: xorq $63, %rdx
-; SRC-NEXT: orq $64, %rdx
+; SRC-NEXT: xorl $63, %edx
+; SRC-NEXT: orl $64, %edx
; SRC-NEXT: testq %rdi, %rdi
; SRC-NEXT: cmovneq %rcx, %rdx
; SRC-NEXT: bsrq %r10, %rsi
-; SRC-NEXT: xorq $63, %rsi
+; SRC-NEXT: xorl $63, %esi
; SRC-NEXT: bsrq %r11, %rcx
-; SRC-NEXT: xorq $63, %rcx
-; SRC-NEXT: orq $64, %rcx
+; SRC-NEXT: xorl $63, %ecx
+; SRC-NEXT: orl $64, %ecx
; SRC-NEXT: testq %r10, %r10
-; SRC-NEXT: cmovneq %rsi, %rcx
-; SRC-NEXT: orq $128, %rcx
+; SRC-NEXT: cmovnel %esi, %ecx
+; SRC-NEXT: subl $-128, %ecx
; SRC-NEXT: orq %rdi, %r9
; SRC-NEXT: cmovneq %rdx, %rcx
; SRC-NEXT: jmp .LBB4_3
@@ -910,16 +910,16 @@ define i256 @PR25498(i256 %a) nounwind {
; LIN-NEXT: je .LBB4_1
; LIN-NEXT: # %bb.2: # %cond.false
; LIN-NEXT: bsrq %r11, %rcx
-; LIN-NEXT: xorq $63, %rcx
-; LIN-NEXT: orq $64, %rcx
+; LIN-NEXT: xorl $63, %ecx
+; LIN-NEXT: orl $64, %ecx
; LIN-NEXT: bsrq %r10, %rdx
-; LIN-NEXT: xorq $63, %rdx
+; LIN-NEXT: xorl $63, %edx
; LIN-NEXT: testq %r10, %r10
-; LIN-NEXT: cmoveq %rcx, %rdx
-; LIN-NEXT: orq $128, %rdx
+; LIN-NEXT: cmovel %ecx, %edx
+; LIN-NEXT: subl $-128, %edx
; LIN-NEXT: bsrq %r9, %rsi
-; LIN-NEXT: xorq $63, %rsi
-; LIN-NEXT: orq $64, %rsi
+; LIN-NEXT: xorl $63, %esi
+; LIN-NEXT: orl $64, %esi
; LIN-NEXT: bsrq %rdi, %rcx
; LIN-NEXT: xorq $63, %rcx
; LIN-NEXT: testq %rdi, %rdi
diff --git a/llvm/test/CodeGen/X86/vector-compress.ll b/llvm/test/CodeGen/X86/vector-compress.ll
index 01bdf0a098e7a..15c27e5d02783 100644
--- a/llvm/test/CodeGen/X86/vector-compress.ll
+++ b/llvm/test/CodeGen/X86/vector-compress.ll
@@ -10,32 +10,30 @@ define <4 x i32> @test_compress_v4i32(<4 x i32> %vec, <4 x i1> %mask, <4 x i32>
; AVX2-NEXT: vpsrad $31, %xmm1, %xmm1
; AVX2-NEXT: vmovaps %xmm2, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vpextrd $1, %xmm1, %eax
-; AVX2-NEXT: vmovd %xmm1, %esi
-; AVX2-NEXT: andl $1, %esi
+; AVX2-NEXT: vmovd %xmm1, %ecx
+; AVX2-NEXT: andl $1, %ecx
+; AVX2-NEXT: movl %ecx, %edx
+; AVX2-NEXT: subl %eax, %edx
+; AVX2-NEXT: vpextrd $2, %xmm1, %eax
+; AVX2-NEXT: movl %edx, %esi
+; AVX2-NEXT: subl %eax, %esi
+; AVX2-NEXT: vpextrd $3, %xmm1, %eax
; AVX2-NEXT: movl %esi, %edi
; AVX2-NEXT: subl %eax, %edi
-; AVX2-NEXT: vpextrd $2, %xmm1, %edx
-; AVX2-NEXT: subl %edx, %edi
-; AVX2-NEXT: vpextrd $3, %xmm1, %ecx
-; AVX2-NEXT: subl %ecx, %edi
-; AVX2-NEXT: andl $3, %edi
-; AVX2-NEXT: andl $1, %eax
-; AVX2-NEXT: addq %rsi, %rax
-; AVX2-NEXT: andl $1, %edx
-; AVX2-NEXT: addq %rax, %rdx
-; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rdx, %rcx
+; AVX2-NEXT: movl %edi, %eax
+; AVX2-NEXT: andl $3, %eax
; AVX2-NEXT: vextractps $3, %xmm0, %r8d
-; AVX2-NEXT: cmpq $4, %rcx
-; AVX2-NEXT: cmovbl -24(%rsp,%rdi,4), %r8d
+; AVX2-NEXT: cmpq $4, %rdi
+; AVX2-NEXT: cmovbl -24(%rsp,%rax,4), %r8d
; AVX2-NEXT: vmovss %xmm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vextractps $1, %xmm0, -24(%rsp,%rsi,4)
-; AVX2-NEXT: vextractps $2, %xmm0, -24(%rsp,%rax,4)
-; AVX2-NEXT: andl $3, %edx
-; AVX2-NEXT: vextractps $3, %xmm0, -24(%rsp,%rdx,4)
-; AVX2-NEXT: cmpq $3, %rcx
+; AVX2-NEXT: vextractps $1, %xmm0, -24(%rsp,%rcx,4)
+; AVX2-NEXT: shll $2, %edx
+; AVX2-NEXT: vextractps $2, %xmm0, -24(%rsp,%rdx)
+; AVX2-NEXT: andl $3, %esi
+; AVX2-NEXT: vextractps $3, %xmm0, -24(%rsp,%rsi,4)
+; AVX2-NEXT: cmpq $3, %rdi
; AVX2-NEXT: movl $3, %eax
-; AVX2-NEXT: cmovbq %rcx, %rax
+; AVX2-NEXT: cmovbq %rdi, %rax
; AVX2-NEXT: movl %r8d, -24(%rsp,%rax,4)
; AVX2-NEXT: vmovaps -{{[0-9]+}}(%rsp), %xmm0
; AVX2-NEXT: retq
@@ -70,38 +68,35 @@ define <4 x float> @test_compress_v4f32(<4 x float> %vec, <4 x i1> %mask, <4 x f
; AVX2-NEXT: vpslld $31, %xmm1, %xmm1
; AVX2-NEXT: vpsrad $31, %xmm1, %xmm1
; AVX2-NEXT: vmovaps %xmm2, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vpextrd $1, %xmm1, %edx
-; AVX2-NEXT: vmovd %xmm1, %esi
-; AVX2-NEXT: andl $1, %esi
-; AVX2-NEXT: movl %esi, %edi
-; AVX2-NEXT: subl %edx, %edi
-; AVX2-NEXT: vpextrd $2, %xmm1, %ecx
-; AVX2-NEXT: subl %ecx, %edi
+; AVX2-NEXT: vpextrd $1, %xmm1, %eax
+; AVX2-NEXT: vmovd %xmm1, %ecx
+; AVX2-NEXT: andl $1, %ecx
+; AVX2-NEXT: movl %ecx, %edx
+; AVX2-NEXT: subl %eax, %edx
+; AVX2-NEXT: vpextrd $2, %xmm1, %eax
+; AVX2-NEXT: movl %edx, %esi
+; AVX2-NEXT: subl %eax, %esi
; AVX2-NEXT: vpextrd $3, %xmm1, %eax
+; AVX2-NEXT: movl %esi, %edi
; AVX2-NEXT: subl %eax, %edi
-; AVX2-NEXT: andl $3, %edi
+; AVX2-NEXT: movl %edi, %eax
+; AVX2-NEXT: andl $3, %eax
; AVX2-NEXT: vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
; AVX2-NEXT: vmovss %xmm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vextractps $1, %xmm0, -24(%rsp,%rsi,4)
-; AVX2-NEXT: andl $1, %edx
-; AVX2-NEXT: addq %rsi, %rdx
-; AVX2-NEXT: vextractps $2, %xmm0, -24(%rsp,%rdx,4)
-; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rdx, %rcx
-; AVX2-NEXT: andl $1, %eax
-; AVX2-NEXT: addq %rcx, %rax
-; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
-; AVX2-NEXT: andl $3, %ecx
+; AVX2-NEXT: vextractps $1, %xmm0, -24(%rsp,%rcx,4)
+; AVX2-NEXT: shll $2, %edx
+; AVX2-NEXT: vextractps $2, %xmm0, -24(%rsp,%rdx)
+; AVX2-NEXT: andl $3, %esi
; AVX2-NEXT: vshufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; AVX2-NEXT: vmovss %xmm0, -24(%rsp,%rcx,4)
-; AVX2-NEXT: cmpq $3, %rax
-; AVX2-NEXT: movl $3, %ecx
-; AVX2-NEXT: cmovbq %rax, %rcx
+; AVX2-NEXT: vmovss %xmm0, -24(%rsp,%rsi,4)
+; AVX2-NEXT: cmpq $3, %rdi
+; AVX2-NEXT: movl $3, %eax
+; AVX2-NEXT: cmovbq %rdi, %rax
; AVX2-NEXT: ja .LBB1_2
; AVX2-NEXT: # %bb.1:
; AVX2-NEXT: vmovaps %xmm1, %xmm0
; AVX2-NEXT: .LBB1_2:
-; AVX2-NEXT: vmovss %xmm0, -24(%rsp,%rcx,4)
+; AVX2-NEXT: vmovss %xmm0, -24(%rsp,%rax,4)
; AVX2-NEXT: vmovaps -{{[0-9]+}}(%rsp), %xmm0
; AVX2-NEXT: retq
;
@@ -265,19 +260,19 @@ define <8 x i32> @test_compress_v8i32(<8 x i32> %vec, <8 x i1> %mask, <8 x i32>
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: vmovd %xmm3, %edx
; AVX2-NEXT: andl $1, %edx
-; AVX2-NEXT: addq %rdx, %rcx
+; AVX2-NEXT: addl %edx, %ecx
; AVX2-NEXT: vpextrd $2, %xmm3, %esi
; AVX2-NEXT: andl $1, %esi
-; AVX2-NEXT: addq %rcx, %rsi
+; AVX2-NEXT: addl %ecx, %esi
; AVX2-NEXT: vpextrd $3, %xmm3, %edi
; AVX2-NEXT: andl $1, %edi
-; AVX2-NEXT: addq %rsi, %rdi
+; AVX2-NEXT: addl %esi, %edi
; AVX2-NEXT: vmovd %xmm1, %r8d
; AVX2-NEXT: andl $1, %r8d
-; AVX2-NEXT: addq %rdi, %r8
+; AVX2-NEXT: addl %edi, %r8d
; AVX2-NEXT: vpextrd $1, %xmm1, %r9d
; AVX2-NEXT: andl $1, %r9d
-; AVX2-NEXT: addq %r8, %r9
+; AVX2-NEXT: addl %r8d, %r9d
; AVX2-NEXT: vpextrd $2, %xmm1, %r10d
; AVX2-NEXT: andl $1, %r10d
; AVX2-NEXT: addq %r9, %r10
@@ -366,26 +361,26 @@ define <8 x float> @test_compress_v8f32(<8 x float> %vec, <8 x i1> %mask, <8 x f
; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%rax,4)
; AVX2-NEXT: vpextrd $1, %xmm3, %ecx
; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rax, %rcx
+; AVX2-NEXT: addl %eax, %ecx
; AVX2-NEXT: vextractps $2, %xmm0, (%rsp,%rcx,4)
; AVX2-NEXT: vpextrd $2, %xmm3, %eax
; AVX2-NEXT: andl $1, %eax
-; AVX2-NEXT: addq %rcx, %rax
+; AVX2-NEXT: addl %ecx, %eax
; AVX2-NEXT: vextractps $3, %xmm0, (%rsp,%rax,4)
; AVX2-NEXT: vpextrd $3, %xmm3, %ecx
; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rax, %rcx
+; AVX2-NEXT: addl %eax, %ecx
; AVX2-NEXT: vmovd %xmm1, %eax
; AVX2-NEXT: andl $1, %eax
-; AVX2-NEXT: addq %rcx, %rax
-; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
+; AVX2-NEXT: addl %ecx, %eax
+; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
; AVX2-NEXT: andl $7, %ecx
; AVX2-NEXT: vextractf128 $1, %ymm0, %xmm0
; AVX2-NEXT: vmovss %xmm0, (%rsp,%rcx,4)
; AVX2-NEXT: vpextrd $1, %xmm1, %ecx
; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rax, %rcx
-; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
+; AVX2-NEXT: addl %eax, %ecx
+; AVX2-NEXT: # kill: def $eax killed $eax def $rax
; AVX2-NEXT: andl $7, %eax
; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%rax,4)
; AVX2-NEXT: vpextrd $2, %xmm1, %edx
@@ -610,35 +605,34 @@ define <16 x i32> @test_compress_v16i32(<16 x i32> %vec, <16 x i1> %mask, <16 x
; AVX2-NEXT: vmovd %xmm3, %ecx
; AVX2-NEXT: addl %eax, %ecx
; AVX2-NEXT: vpextrd $2, %xmm3, %eax
-; AVX2-NEXT: vpextrd $3, %xmm3, %edx
-; AVX2-NEXT: addl %eax, %edx
-; AVX2-NEXT: addl %ecx, %edx
-; AVX2-NEXT: andl $15, %edx
-; AVX2-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: vpextrd $3, %xmm3, %esi
+; AVX2-NEXT: addl %eax, %esi
+; AVX2-NEXT: addl %ecx, %esi
+; AVX2-NEXT: andl $15, %esi
; AVX2-NEXT: vpextrb $1, %xmm2, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: vmovd %xmm2, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: addq %rcx, %rax
+; AVX2-NEXT: addl %ecx, %eax
; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: vpextrb $2, %xmm2, %ecx
; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rax, %rcx
+; AVX2-NEXT: addl %eax, %ecx
; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: vpextrb $3, %xmm2, %eax
; AVX2-NEXT: andl $1, %eax
-; AVX2-NEXT: addq %rcx, %rax
+; AVX2-NEXT: addl %ecx, %eax
; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: vpextrb $4, %xmm2, %r8d
; AVX2-NEXT: andl $1, %r8d
-; AVX2-NEXT: addq %rax, %r8
+; AVX2-NEXT: addl %eax, %r8d
; AVX2-NEXT: vpextrb $5, %xmm2, %r9d
; AVX2-NEXT: andl $1, %r9d
-; AVX2-NEXT: addq %r8, %r9
+; AVX2-NEXT: addl %r8d, %r9d
; AVX2-NEXT: vpextrb $6, %xmm2, %r10d
; AVX2-NEXT: andl $1, %r10d
-; AVX2-NEXT: addq %r9, %r10
+; AVX2-NEXT: addl %r9d, %r10d
; AVX2-NEXT: vpextrb $7, %xmm2, %r11d
; AVX2-NEXT: andl $1, %r11d
; AVX2-NEXT: addq %r10, %r11
@@ -668,10 +662,8 @@ define <16 x i32> @test_compress_v16i32(<16 x i32> %vec, <16 x i1> %mask, <16 x
; AVX2-NEXT: addq %rax, %rdx
; AVX2-NEXT: vextractf128 $1, %ymm1, %xmm2
; AVX2-NEXT: cmpq $16, %rdx
-; AVX2-NEXT: vextractps $3, %xmm2, %esi
-; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
-; AVX2-NEXT: cmovbl (%rsp,%rdi,4), %esi
-; AVX2-NEXT: movl %esi, %edi
+; AVX2-NEXT: vextractps $3, %xmm2, %edi
+; AVX2-NEXT: cmovbl (%rsp,%rsi,4), %edi
; AVX2-NEXT: vmovss %xmm0, (%rsp)
; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%rsi,4)
@@ -771,30 +763,30 @@ define <16 x float> @test_compress_v16f32(<16 x float> %vec, <16 x i1> %mask, <1
; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%rax,4)
; AVX2-NEXT: vpextrb $1, %xmm2, %ecx
; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rax, %rcx
+; AVX2-NEXT: addl %eax, %ecx
; AVX2-NEXT: vextractps $2, %xmm0, (%rsp,%rcx,4)
; AVX2-NEXT: vpextrb $2, %xmm2, %eax
; AVX2-NEXT: andl $1, %eax
-; AVX2-NEXT: addq %rcx, %rax
+; AVX2-NEXT: addl %ecx, %eax
; AVX2-NEXT: vextractps $3, %xmm0, (%rsp,%rax,4)
; AVX2-NEXT: vpextrb $3, %xmm2, %ecx
; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rax, %rcx
+; AVX2-NEXT: addl %eax, %ecx
; AVX2-NEXT: vextractf128 $1, %ymm0, %xmm0
; AVX2-NEXT: vmovss %xmm0, (%rsp,%rcx,4)
; AVX2-NEXT: vpextrb $4, %xmm2, %eax
; AVX2-NEXT: andl $1, %eax
-; AVX2-NEXT: addq %rcx, %rax
+; AVX2-NEXT: addl %ecx, %eax
; AVX2-NEXT: vpextrb $5, %xmm2, %ecx
; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rax, %rcx
-; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
+; AVX2-NEXT: addl %eax, %ecx
+; AVX2-NEXT: # kill: def $eax killed $eax def $rax
; AVX2-NEXT: andl $15, %eax
; AVX2-NEXT: vextractps $1, %xmm0, (%rsp,%rax,4)
; AVX2-NEXT: vpextrb $6, %xmm2, %eax
; AVX2-NEXT: andl $1, %eax
-; AVX2-NEXT: addq %rcx, %rax
-; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
+; AVX2-NEXT: addl %ecx, %eax
+; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
; AVX2-NEXT: andl $15, %ecx
; AVX2-NEXT: vextractps $2, %xmm0, (%rsp,%rcx,4)
; AVX2-NEXT: vpextrb $7, %xmm2, %ecx
@@ -914,22 +906,22 @@ define <8 x i64> @test_compress_v8i64(<8 x i64> %vec, <8 x i1> %mask, <8 x i64>
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: vmovd %xmm2, %edx
; AVX2-NEXT: andl $1, %edx
-; AVX2-NEXT: addq %rdx, %rcx
+; AVX2-NEXT: addl %edx, %ecx
; AVX2-NEXT: vpextrw $2, %xmm2, %esi
; AVX2-NEXT: andl $1, %esi
-; AVX2-NEXT: addq %rcx, %rsi
+; AVX2-NEXT: addl %ecx, %esi
; AVX2-NEXT: vpextrw $3, %xmm2, %edi
; AVX2-NEXT: andl $1, %edi
-; AVX2-NEXT: addq %rsi, %rdi
+; AVX2-NEXT: addl %esi, %edi
; AVX2-NEXT: vpextrw $4, %xmm2, %r8d
; AVX2-NEXT: andl $1, %r8d
-; AVX2-NEXT: addq %rdi, %r8
+; AVX2-NEXT: addl %edi, %r8d
; AVX2-NEXT: vpextrw $5, %xmm2, %r9d
; AVX2-NEXT: andl $1, %r9d
-; AVX2-NEXT: addq %r8, %r9
+; AVX2-NEXT: addl %r8d, %r9d
; AVX2-NEXT: vpextrw $6, %xmm2, %r10d
; AVX2-NEXT: andl $1, %r10d
-; AVX2-NEXT: addq %r9, %r10
+; AVX2-NEXT: addl %r9d, %r10d
; AVX2-NEXT: vpextrw $7, %xmm2, %r11d
; AVX2-NEXT: andl $1, %r11d
; AVX2-NEXT: addq %r10, %r11
@@ -1009,35 +1001,35 @@ define <8 x double> @test_compress_v8f64(<8 x double> %vec, <8 x i1> %mask, <8 x
; AVX2-NEXT: vmovhps %xmm0, (%rsp,%rax,8)
; AVX2-NEXT: vpextrw $1, %xmm2, %ecx
; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rax, %rcx
+; AVX2-NEXT: addl %eax, %ecx
; AVX2-NEXT: vextractf128 $1, %ymm0, %xmm0
; AVX2-NEXT: vmovlps %xmm0, (%rsp,%rcx,8)
; AVX2-NEXT: vpextrw $2, %xmm2, %eax
; AVX2-NEXT: andl $1, %eax
-; AVX2-NEXT: addq %rcx, %rax
+; AVX2-NEXT: addl %ecx, %eax
; AVX2-NEXT: vmovhps %xmm0, (%rsp,%rax,8)
; AVX2-NEXT: vpextrw $3, %xmm2, %ecx
; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rax, %rcx
+; AVX2-NEXT: addl %eax, %ecx
; AVX2-NEXT: vpextrw $4, %xmm2, %eax
; AVX2-NEXT: andl $1, %eax
-; AVX2-NEXT: addq %rcx, %rax
-; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
+; AVX2-NEXT: addl %ecx, %eax
+; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
; AVX2-NEXT: andl $7, %ecx
; AVX2-NEXT: vmovlpd %xmm1, (%rsp,%rcx,8)
; AVX2-NEXT: vpextrw $5, %xmm2, %ecx
; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rax, %rcx
-; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
+; AVX2-NEXT: addl %eax, %ecx
+; AVX2-NEXT: # kill: def $eax killed $eax def $rax
; AVX2-NEXT: andl $7, %eax
; AVX2-NEXT: vmovhpd %xmm1, (%rsp,%rax,8)
; AVX2-NEXT: vpextrw $6, %xmm2, %edx
; AVX2-NEXT: andl $1, %edx
-; AVX2-NEXT: addq %rcx, %rdx
-; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
-; AVX2-NEXT: andl $7, %ecx
+; AVX2-NEXT: addl %ecx, %edx
+; AVX2-NEXT: movl %ecx, %eax
+; AVX2-NEXT: andl $7, %eax
; AVX2-NEXT: vextractf128 $1, %ymm1, %xmm0
-; AVX2-NEXT: vmovlpd %xmm0, (%rsp,%rcx,8)
+; AVX2-NEXT: vmovlpd %xmm0, (%rsp,%rax,8)
; AVX2-NEXT: vpextrw $7, %xmm2, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rdx, %rax
@@ -1093,19 +1085,19 @@ define <16 x i8> @test_compress_v16i8(<16 x i8> %vec, <16 x i1> %mask, <16 x i8>
; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
; AVX2-NEXT: vpcmpgtb %xmm1, %xmm3, %xmm1
; AVX2-NEXT: vmovaps %xmm2, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vpextrb $1, %xmm1, %r13d
+; AVX2-NEXT: vpextrb $1, %xmm1, %r12d
; AVX2-NEXT: vmovd %xmm1, %esi
; AVX2-NEXT: movl %esi, %eax
; AVX2-NEXT: andb $1, %al
-; AVX2-NEXT: subb %r13b, %al
+; AVX2-NEXT: subb %r12b, %al
; AVX2-NEXT: vpextrb $2, %xmm1, %edx
; AVX2-NEXT: subb %dl, %al
-; AVX2-NEXT: vpextrb $3, %xmm1, %ebp
-; AVX2-NEXT: subb %bpl, %al
-; AVX2-NEXT: vpextrb $4, %xmm1, %r12d
-; AVX2-NEXT: subb %r12b, %al
-; AVX2-NEXT: vpextrb $5, %xmm1, %r15d
+; AVX2-NEXT: vpextrb $3, %xmm1, %r13d
+; AVX2-NEXT: subb %r13b, %al
+; AVX2-NEXT: vpextrb $4, %xmm1, %r15d
; AVX2-NEXT: subb %r15b, %al
+; AVX2-NEXT: vpextrb $5, %xmm1, %ebp
+; AVX2-NEXT: subb %bpl, %al
; AVX2-NEXT: vpextrb $6, %xmm1, %r14d
; AVX2-NEXT: subb %r14b, %al
; AVX2-NEXT: vpextrb $7, %xmm1, %ebx
@@ -1135,27 +1127,27 @@ define <16 x i8> @test_compress_v16i8(<16 x i8> %vec, <16 x i1> %mask, <16 x i8>
; AVX2-NEXT: vpextrb $0, %xmm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: andl $1, %esi
; AVX2-NEXT: vpextrb $1, %xmm0, -40(%rsp,%rsi)
-; AVX2-NEXT: andl $1, %r13d
-; AVX2-NEXT: addq %rsi, %r13
-; AVX2-NEXT: vpextrb $2, %xmm0, -40(%rsp,%r13)
+; AVX2-NEXT: andl $1, %r12d
+; AVX2-NEXT: addl %esi, %r12d
+; AVX2-NEXT: vpextrb $2, %xmm0, -40(%rsp,%r12)
; AVX2-NEXT: andl $1, %edx
-; AVX2-NEXT: addq %r13, %rdx
+; AVX2-NEXT: addl %r12d, %edx
; AVX2-NEXT: vpextrb $3, %xmm0, -40(%rsp,%rdx)
-; AVX2-NEXT: andl $1, %ebp
-; AVX2-NEXT: addq %rdx, %rbp
-; AVX2-NEXT: vpextrb $4, %xmm0, -40(%rsp,%rbp)
-; AVX2-NEXT: andl $1, %r12d
-; AVX2-NEXT: addq %rbp, %r12
+; AVX2-NEXT: andl $1, %r13d
+; AVX2-NEXT: addl %edx, %r13d
+; AVX2-NEXT: vpextrb $4, %xmm0, -40(%rsp,%r13)
; AVX2-NEXT: andl $1, %r15d
-; AVX2-NEXT: addq %r12, %r15
-; AVX2-NEXT: # kill: def $r12d killed $r12d killed $r12 def $r12
-; AVX2-NEXT: andl $15, %r12d
-; AVX2-NEXT: vpextrb $5, %xmm0, -40(%rsp,%r12)
+; AVX2-NEXT: addl %r13d, %r15d
+; AVX2-NEXT: andl $1, %ebp
+; AVX2-NEXT: addl %r15d, %ebp
+; AVX2-NEXT: movl %r15d, %eax
+; AVX2-NEXT: andl $15, %eax
+; AVX2-NEXT: vpextrb $5, %xmm0, -40(%rsp,%rax)
; AVX2-NEXT: andl $1, %r14d
-; AVX2-NEXT: addq %r15, %r14
-; AVX2-NEXT: # kill: def $r15d killed $r15d killed $r15 def $r15
-; AVX2-NEXT: andl $15, %r15d
-; AVX2-NEXT: vpextrb $6, %xmm0, -40(%rsp,%r15)
+; AVX2-NEXT: addl %ebp, %r14d
+; AVX2-NEXT: movl %ebp, %eax
+; AVX2-NEXT: andl $15, %eax
+; AVX2-NEXT: vpextrb $6, %xmm0, -40(%rsp,%rax)
; AVX2-NEXT: andl $1, %ebx
; AVX2-NEXT: addq %r14, %rbx
; AVX2-NEXT: # kill: def $r14d killed $r14d killed $r14 def $r14
@@ -1252,53 +1244,47 @@ define <8 x i16> @test_compress_v8i16(<8 x i16> %vec, <8 x i1> %mask, <8 x i16>
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: vmovd %xmm1, %ecx
; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: leal (%rcx,%rax), %esi
-; AVX2-NEXT: vpextrw $2, %xmm1, %edi
-; AVX2-NEXT: andl $1, %edi
-; AVX2-NEXT: vpextrw $3, %xmm1, %edx
+; AVX2-NEXT: addl %ecx, %eax
+; AVX2-NEXT: vpextrw $2, %xmm1, %edx
; AVX2-NEXT: andl $1, %edx
-; AVX2-NEXT: leal (%rdi,%rdx), %r10d
-; AVX2-NEXT: addl %esi, %r10d
-; AVX2-NEXT: vpextrw $4, %xmm1, %r9d
-; AVX2-NEXT: andl $1, %r9d
-; AVX2-NEXT: vpextrw $5, %xmm1, %esi
+; AVX2-NEXT: addl %eax, %edx
+; AVX2-NEXT: vpextrw $3, %xmm1, %esi
; AVX2-NEXT: andl $1, %esi
-; AVX2-NEXT: leal (%r9,%rsi), %r11d
-; AVX2-NEXT: vpextrw $6, %xmm1, %r8d
+; AVX2-NEXT: addl %edx, %esi
+; AVX2-NEXT: vpextrw $4, %xmm1, %edi
+; AVX2-NEXT: andl $1, %edi
+; AVX2-NEXT: addl %esi, %edi
+; AVX2-NEXT: vpextrw $5, %xmm1, %r8d
; AVX2-NEXT: andl $1, %r8d
-; AVX2-NEXT: addl %r8d, %r11d
-; AVX2-NEXT: addl %r10d, %r11d
+; AVX2-NEXT: addl %edi, %r8d
+; AVX2-NEXT: vpextrw $6, %xmm1, %r9d
+; AVX2-NEXT: andl $1, %r9d
+; AVX2-NEXT: addl %r8d, %r9d
; AVX2-NEXT: vpextrw $7, %xmm1, %r10d
; AVX2-NEXT: andl $1, %r10d
-; AVX2-NEXT: addl %r10d, %r11d
-; AVX2-NEXT: andl $7, %r11d
-; AVX2-NEXT: addq %rcx, %rax
-; AVX2-NEXT: addq %rax, %rdi
-; AVX2-NEXT: addq %rdi, %rdx
-; AVX2-NEXT: addq %rdx, %r9
-; AVX2-NEXT: addq %r9, %rsi
-; AVX2-NEXT: addq %rsi, %r8
-; AVX2-NEXT: addq %r8, %r10
-; AVX2-NEXT: vpextrw $7, %xmm0, %ebx
+; AVX2-NEXT: leal (%r9,%r10), %ebx
+; AVX2-NEXT: andl $7, %ebx
+; AVX2-NEXT: addq %r9, %r10
+; AVX2-NEXT: vpextrw $7, %xmm0, %r11d
; AVX2-NEXT: cmpq $8, %r10
-; AVX2-NEXT: cmovbw -16(%rsp,%r11,2), %bx
+; AVX2-NEXT: cmovbw -16(%rsp,%rbx,2), %r11w
; AVX2-NEXT: vpextrw $0, %xmm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vpextrw $1, %xmm0, -16(%rsp,%rcx,2)
; AVX2-NEXT: vpextrw $2, %xmm0, -16(%rsp,%rax,2)
-; AVX2-NEXT: vpextrw $3, %xmm0, -16(%rsp,%rdi,2)
-; AVX2-NEXT: andl $7, %edx
-; AVX2-NEXT: vpextrw $4, %xmm0, -16(%rsp,%rdx,2)
-; AVX2-NEXT: andl $7, %r9d
-; AVX2-NEXT: vpextrw $5, %xmm0, -16(%rsp,%r9,2)
+; AVX2-NEXT: vpextrw $3, %xmm0, -16(%rsp,%rdx,2)
; AVX2-NEXT: andl $7, %esi
-; AVX2-NEXT: vpextrw $6, %xmm0, -16(%rsp,%rsi,2)
+; AVX2-NEXT: vpextrw $4, %xmm0, -16(%rsp,%rsi,2)
+; AVX2-NEXT: andl $7, %edi
+; AVX2-NEXT: vpextrw $5, %xmm0, -16(%rsp,%rdi,2)
; AVX2-NEXT: andl $7, %r8d
-; AVX2-NEXT: vpextrw $7, %xmm0, -16(%rsp,%r8,2)
+; AVX2-NEXT: vpextrw $6, %xmm0, -16(%rsp,%r8,2)
+; AVX2-NEXT: andl $7, %r9d
+; AVX2-NEXT: vpextrw $7, %xmm0, -16(%rsp,%r9,2)
; AVX2-NEXT: cmpq $7, %r10
; AVX2-NEXT: movl $7, %eax
; AVX2-NEXT: cmovbq %r10, %rax
; AVX2-NEXT: movl %eax, %eax
-; AVX2-NEXT: movw %bx, -16(%rsp,%rax,2)
+; AVX2-NEXT: movw %r11w, -16(%rsp,%rax,2)
; AVX2-NEXT: vmovaps -{{[0-9]+}}(%rsp), %xmm0
; AVX2-NEXT: popq %rbx
; AVX2-NEXT: retq
@@ -1382,27 +1368,27 @@ define <32 x i8> @test_compress_v32i8(<32 x i8> %vec, <32 x i1> %mask, <32 x i8>
; AVX2-NEXT: vpextrb $1, %xmm0, (%rsp,%rcx)
; AVX2-NEXT: vpextrb $1, %xmm3, %edx
; AVX2-NEXT: andl $1, %edx
-; AVX2-NEXT: addq %rcx, %rdx
+; AVX2-NEXT: addl %ecx, %edx
; AVX2-NEXT: vpextrb $2, %xmm0, (%rsp,%rdx)
; AVX2-NEXT: vpextrb $2, %xmm3, %ecx
; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rdx, %rcx
+; AVX2-NEXT: addl %edx, %ecx
; AVX2-NEXT: vpextrb $3, %xmm0, (%rsp,%rcx)
; AVX2-NEXT: vpextrb $3, %xmm3, %edx
; AVX2-NEXT: andl $1, %edx
-; AVX2-NEXT: addq %rcx, %rdx
+; AVX2-NEXT: addl %ecx, %edx
; AVX2-NEXT: vpextrb $4, %xmm0, (%rsp,%rdx)
; AVX2-NEXT: vpextrb $4, %xmm3, %ecx
; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rdx, %rcx
+; AVX2-NEXT: addl %edx, %ecx
; AVX2-NEXT: vpextrb $5, %xmm0, (%rsp,%rcx)
; AVX2-NEXT: vpextrb $5, %xmm3, %edx
; AVX2-NEXT: andl $1, %edx
-; AVX2-NEXT: addq %rcx, %rdx
+; AVX2-NEXT: addl %ecx, %edx
; AVX2-NEXT: vpextrb $6, %xmm3, %ecx
; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rdx, %rcx
-; AVX2-NEXT: # kill: def $edx killed $edx killed $rdx def $rdx
+; AVX2-NEXT: addl %edx, %ecx
+; AVX2-NEXT: # kill: def $edx killed $edx def $rdx
; AVX2-NEXT: andl $31, %edx
; AVX2-NEXT: vpextrb $6, %xmm0, (%rsp,%rdx)
; AVX2-NEXT: vpextrb $7, %xmm3, %edx
@@ -1659,25 +1645,25 @@ define <16 x i16> @test_compress_v16i16(<16 x i16> %vec, <16 x i1> %mask, <16 x
; AVX2-NEXT: vmovd %xmm1, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: addq %rcx, %rax
+; AVX2-NEXT: addl %ecx, %eax
; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: vpextrw $2, %xmm1, %ecx
; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rax, %rcx
+; AVX2-NEXT: addl %eax, %ecx
; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: vpextrw $3, %xmm1, %eax
; AVX2-NEXT: andl $1, %eax
-; AVX2-NEXT: addq %rcx, %rax
+; AVX2-NEXT: addl %ecx, %eax
; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: vpextrw $4, %xmm1, %r8d
; AVX2-NEXT: andl $1, %r8d
-; AVX2-NEXT: addq %rax, %r8
+; AVX2-NEXT: addl %eax, %r8d
; AVX2-NEXT: vpextrw $5, %xmm1, %r9d
; AVX2-NEXT: andl $1, %r9d
-; AVX2-NEXT: addq %r8, %r9
+; AVX2-NEXT: addl %r8d, %r9d
; AVX2-NEXT: vpextrw $6, %xmm1, %r10d
; AVX2-NEXT: andl $1, %r10d
-; AVX2-NEXT: addq %r9, %r10
+; AVX2-NEXT: addl %r9d, %r10d
; AVX2-NEXT: vpextrw $7, %xmm1, %r11d
; AVX2-NEXT: andl $1, %r11d
; AVX2-NEXT: addq %r10, %r11
@@ -1969,20 +1955,19 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX2-NEXT: andl $1, %edi
; AVX2-NEXT: vpextrb $1, %xmm0, (%rsp,%rdi)
; AVX2-NEXT: andl $1, %esi
-; AVX2-NEXT: addq %rdi, %rsi
+; AVX2-NEXT: addl %edi, %esi
; AVX2-NEXT: vpextrb $2, %xmm0, (%rsp,%rsi)
; AVX2-NEXT: andl $1, %r8d
-; AVX2-NEXT: addq %rsi, %r8
+; AVX2-NEXT: addl %esi, %r8d
; AVX2-NEXT: vpextrb $3, %xmm0, (%rsp,%r8)
; AVX2-NEXT: andl $1, %r9d
-; AVX2-NEXT: addq %r8, %r9
+; AVX2-NEXT: addl %r8d, %r9d
; AVX2-NEXT: vpextrb $4, %xmm0, (%rsp,%r9)
; AVX2-NEXT: andl $1, %r10d
-; AVX2-NEXT: addq %r9, %r10
-; AVX2-NEXT: movl %r10d, %eax
-; AVX2-NEXT: vpextrb $5, %xmm0, (%rsp,%rax)
+; AVX2-NEXT: addl %r9d, %r10d
+; AVX2-NEXT: vpextrb $5, %xmm0, (%rsp,%r10)
; AVX2-NEXT: andl $1, %r11d
-; AVX2-NEXT: addq %r10, %r11
+; AVX2-NEXT: addl %r10d, %r11d
; AVX2-NEXT: movzbl %bl, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %r11, %rax
@@ -2972,27 +2957,27 @@ define <32 x i16> @test_compress_v32i16(<32 x i16> %vec, <32 x i1> %mask, <32 x
; AVX2-NEXT: vmovd %xmm2, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: addq %rcx, %rax
+; AVX2-NEXT: addl %ecx, %eax
; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: vpextrb $2, %xmm2, %ecx
; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rax, %rcx
+; AVX2-NEXT: addl %eax, %ecx
; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: vpextrb $3, %xmm2, %eax
; AVX2-NEXT: andl $1, %eax
-; AVX2-NEXT: addq %rcx, %rax
+; AVX2-NEXT: addl %ecx, %eax
; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: vpextrb $4, %xmm2, %ecx
; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rax, %rcx
+; AVX2-NEXT: addl %eax, %ecx
; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: vpextrb $5, %xmm2, %eax
; AVX2-NEXT: andl $1, %eax
-; AVX2-NEXT: addq %rcx, %rax
+; AVX2-NEXT: addl %ecx, %eax
; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: vpextrb $6, %xmm2, %ecx
; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rax, %rcx
+; AVX2-NEXT: addl %eax, %ecx
; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: vpextrb $7, %xmm2, %eax
; AVX2-NEXT: andl $1, %eax
@@ -4317,22 +4302,22 @@ define <8 x i64> @test_compress_knownbits_zext_v8i16_8i64(<8 x i16> %vec, <8 x
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: vmovd %xmm1, %edx
; AVX2-NEXT: andl $1, %edx
-; AVX2-NEXT: addq %rdx, %rcx
+; AVX2-NEXT: addl %edx, %ecx
; AVX2-NEXT: vpextrw $2, %xmm1, %esi
; AVX2-NEXT: andl $1, %esi
-; AVX2-NEXT: addq %rcx, %rsi
+; AVX2-NEXT: addl %ecx, %esi
; AVX2-NEXT: vpextrw $3, %xmm1, %edi
; AVX2-NEXT: andl $1, %edi
-; AVX2-NEXT: addq %rsi, %rdi
+; AVX2-NEXT: addl %esi, %edi
; AVX2-NEXT: vpextrw $4, %xmm1, %r8d
; AVX2-NEXT: andl $1, %r8d
-; AVX2-NEXT: addq %rdi, %r8
+; AVX2-NEXT: addl %edi, %r8d
; AVX2-NEXT: vpextrw $5, %xmm1, %r9d
; AVX2-NEXT: andl $1, %r9d
-; AVX2-NEXT: addq %r8, %r9
+; AVX2-NEXT: addl %r8d, %r9d
; AVX2-NEXT: vpextrw $6, %xmm1, %r10d
; AVX2-NEXT: andl $1, %r10d
-; AVX2-NEXT: addq %r9, %r10
+; AVX2-NEXT: addl %r9d, %r10d
; AVX2-NEXT: vpextrw $7, %xmm1, %r11d
; AVX2-NEXT: andl $1, %r11d
; AVX2-NEXT: addq %r10, %r11
@@ -4421,22 +4406,22 @@ define <8 x i64> @test_compress_knownbits_sext_v8i16_8i64(<8 x i16> %vec, <8 x i
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: vmovd %xmm1, %edx
; AVX2-NEXT: andl $1, %edx
-; AVX2-NEXT: addq %rdx, %rcx
+; AVX2-NEXT: addl %edx, %ecx
; AVX2-NEXT: vpextrw $2, %xmm1, %esi
; AVX2-NEXT: andl $1, %esi
-; AVX2-NEXT: addq %rcx, %rsi
+; AVX2-NEXT: addl %ecx, %esi
; AVX2-NEXT: vpextrw $3, %xmm1, %edi
; AVX2-NEXT: andl $1, %edi
-; AVX2-NEXT: addq %rsi, %rdi
+; AVX2-NEXT: addl %esi, %edi
; AVX2-NEXT: vpextrw $4, %xmm1, %r8d
; AVX2-NEXT: andl $1, %r8d
-; AVX2-NEXT: addq %rdi, %r8
+; AVX2-NEXT: addl %edi, %r8d
; AVX2-NEXT: vpextrw $5, %xmm1, %r9d
; AVX2-NEXT: andl $1, %r9d
-; AVX2-NEXT: addq %r8, %r9
+; AVX2-NEXT: addl %r8d, %r9d
; AVX2-NEXT: vpextrw $6, %xmm1, %r10d
; AVX2-NEXT: andl $1, %r10d
-; AVX2-NEXT: addq %r9, %r10
+; AVX2-NEXT: addl %r9d, %r10d
; AVX2-NEXT: vpextrw $7, %xmm1, %r11d
; AVX2-NEXT: andl $1, %r11d
; AVX2-NEXT: addq %r10, %r11
@@ -4600,15 +4585,26 @@ define <4 x i8> @test_compress_small(<4 x i8> %vec, <4 x i1> %mask) nounwind {
; AVX2-NEXT: vpextrb $1, %xmm0, -24(%rsp,%rax)
; AVX2-NEXT: vpextrb $1, %xmm1, %ecx
; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rax, %rcx
+; AVX2-NEXT: addl %eax, %ecx
; AVX2-NEXT: vpextrb $2, %xmm0, -24(%rsp,%rcx)
; AVX2-NEXT: vpextrb $2, %xmm1, %eax
; AVX2-NEXT: andl $1, %eax
-; AVX2-NEXT: addq %rcx, %rax
+; AVX2-NEXT: addl %ecx, %eax
; AVX2-NEXT: vpextrb $3, %xmm0, -24(%rsp,%rax)
; AVX2-NEXT: vpextrb $3, %xmm1, %ecx
; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rax, %rcx
+; AVX2-NEXT: addl %eax, %ecx
+; AVX2-NEXT: vpextrb $4, %xmm0, -24(%rsp,%rcx)
+; AVX2-NEXT: vpextrb $5, %xmm0, -24(%rsp,%rcx)
+; AVX2-NEXT: vpextrb $6, %xmm0, -24(%rsp,%rcx)
+; AVX2-NEXT: vpextrb $7, %xmm0, -24(%rsp,%rcx)
+; AVX2-NEXT: vpextrb $8, %xmm0, -24(%rsp,%rcx)
+; AVX2-NEXT: vpextrb $9, %xmm0, -24(%rsp,%rcx)
+; AVX2-NEXT: vpextrb $10, %xmm0, -24(%rsp,%rcx)
+; AVX2-NEXT: vpextrb $11, %xmm0, -24(%rsp,%rcx)
+; AVX2-NEXT: vpextrb $12, %xmm0, -24(%rsp,%rcx)
+; AVX2-NEXT: vpextrb $13, %xmm0, -24(%rsp,%rcx)
+; AVX2-NEXT: vpextrb $14, %xmm0, -24(%rsp,%rcx)
; AVX2-NEXT: vpextrb $15, %xmm0, -24(%rsp,%rcx)
; AVX2-NEXT: vmovaps -{{[0-9]+}}(%rsp), %xmm0
; AVX2-NEXT: retq
@@ -4852,25 +4848,23 @@ define <4 x i32> @test_compress_v4i32_zero_passthru(<4 x i32> %vec, <4 x i1> %ma
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: vextractps $1, %xmm0, -24(%rsp,%rax,4)
; AVX2-NEXT: vpextrd $1, %xmm1, %ecx
-; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rax, %rcx
-; AVX2-NEXT: vextractps $2, %xmm0, -24(%rsp,%rcx,4)
-; AVX2-NEXT: vpextrd $2, %xmm1, %eax
-; AVX2-NEXT: andl $1, %eax
-; AVX2-NEXT: addq %rcx, %rax
+; AVX2-NEXT: subl %ecx, %eax
+; AVX2-NEXT: leal (,%rax,4), %ecx
+; AVX2-NEXT: vextractps $2, %xmm0, -24(%rsp,%rcx)
+; AVX2-NEXT: vpextrd $2, %xmm1, %ecx
+; AVX2-NEXT: subl %ecx, %eax
+; AVX2-NEXT: movl %eax, %ecx
+; AVX2-NEXT: andl $3, %ecx
+; AVX2-NEXT: vextractps $3, %xmm0, -24(%rsp,%rcx,4)
; AVX2-NEXT: vpextrd $3, %xmm1, %ecx
-; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rax, %rcx
-; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
-; AVX2-NEXT: andl $3, %eax
-; AVX2-NEXT: vextractps $3, %xmm0, -24(%rsp,%rax,4)
-; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: cmpq $3, %rcx
+; AVX2-NEXT: subl %ecx, %eax
+; AVX2-NEXT: xorl %ecx, %ecx
+; AVX2-NEXT: cmpq $3, %rax
; AVX2-NEXT: movl $3, %edx
-; AVX2-NEXT: cmovbq %rcx, %rdx
-; AVX2-NEXT: vextractps $3, %xmm0, %ecx
-; AVX2-NEXT: cmovbel %eax, %ecx
-; AVX2-NEXT: movl %ecx, -24(%rsp,%rdx,4)
+; AVX2-NEXT: cmovbq %rax, %rdx
+; AVX2-NEXT: vextractps $3, %xmm0, %eax
+; AVX2-NEXT: cmovbel %ecx, %eax
+; AVX2-NEXT: movl %eax, -24(%rsp,%rdx,4)
; AVX2-NEXT: vmovaps -{{[0-9]+}}(%rsp), %xmm0
; AVX2-NEXT: retq
;
diff --git a/llvm/test/CodeGen/X86/xaluo.ll b/llvm/test/CodeGen/X86/xaluo.ll
index c2a8002c949ce..fb6f4590d9736 100644
--- a/llvm/test/CodeGen/X86/xaluo.ll
+++ b/llvm/test/CodeGen/X86/xaluo.ll
@@ -966,7 +966,7 @@ define {i64, i1} @uaddoovf(i64 %a, i64 %b) {
; CHECK: ## %bb.0:
; CHECK-NEXT: movzbl %dil, %ecx
; CHECK-NEXT: movzbl %sil, %eax
-; CHECK-NEXT: addq %rcx, %rax
+; CHECK-NEXT: addl %ecx, %eax
; CHECK-NEXT: xorl %edx, %edx
; CHECK-NEXT: retq
%1 = and i64 %a, 255
>From 5c3ba9de432d47e9ce6a4b83dd28863661ac791a Mon Sep 17 00:00:00 2001
From: 2elliti <forstoic724321 at gmail.com>
Date: Mon, 13 Apr 2026 15:49:37 +0530
Subject: [PATCH 4/4] Address review feedback: capitalize variable name
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 15c7264f61249..3bc873cd61ff9 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -59404,15 +59404,15 @@ static SDValue combineAdd(SDNode *N, SelectionDAG &DAG,
// If upper 33 bits of operands are 0, truncates opcode from i64 to i32.
if (VT == MVT::i64) {
- APInt mask = APInt::getHighBitsSet(64, 33);
- if (DAG.MaskedValueIsZero(Op0, mask) && DAG.MaskedValueIsZero(Op1, mask)) {
+ APInt Mask = APInt::getHighBitsSet(64, 33);
+ if (DAG.MaskedValueIsZero(Op0, Mask) && DAG.MaskedValueIsZero(Op1, Mask)) {
// Truncate operands MVT::i64 -> MVT::i32
SDValue X = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Op0);
SDValue Y = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Op1);
// now check for NUW and NSW
SDNodeFlags Flags;
- // No unsigned wrap, both operands has their upper 33bits 0, making their
+ // No unsigned wrap, both operands have their upper 33bits 0, making their
// sum lower then max unsigned int32.
Flags.setNoUnsignedWrap(true);
Flags.setNoSignedWrap(DAG.willNotOverflowAdd(true, X, Y));
More information about the llvm-commits
mailing list