[llvm] [SelectionDAG] Preserver poison for abs INT_MIN lowering (PR #183851)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Mar 2 12:47:51 PST 2026
https://github.com/aryanmagoon updated https://github.com/llvm/llvm-project/pull/183851
>From 08abcc0b137d4afbaca77a178d8785c6f06618af Mon Sep 17 00:00:00 2001
From: aryanmagoon <amagoon at nvidia.com>
Date: Fri, 27 Feb 2026 19:13:40 +0000
Subject: [PATCH] SelectionDAG: Preserve poison for abs INT_MIN lowering.
Update affected CodeGen lit tests and added a new lit test for this behavior.
---
.../SelectionDAG/SelectionDAGBuilder.cpp | 14 +-
llvm/test/CodeGen/AArch64/abd-combine.ll | 103 +-
llvm/test/CodeGen/AArch64/abds-neg.ll | 135 +-
llvm/test/CodeGen/AArch64/abds.ll | 138 ++-
llvm/test/CodeGen/AArch64/abdu-neg.ll | 82 +-
llvm/test/CodeGen/AArch64/abdu.ll | 87 +-
llvm/test/CodeGen/AArch64/abs.ll | 342 ++++--
.../CodeGen/AArch64/extract-vector-elt.ll | 3 +-
llvm/test/CodeGen/AArch64/neon-sad.ll | 26 +-
llvm/test/CodeGen/AArch64/sched-movprfx.ll | 5 +-
.../AArch64/sve-fixed-length-int-arith.ll | 192 ++-
.../sve-fixed-length-optimize-ptrue.ll | 8 +-
llvm/test/CodeGen/AArch64/sve-int-arith.ll | 45 +-
.../test/CodeGen/AArch64/sve-merging-unary.ll | 28 +-
...e-streaming-mode-fixed-length-int-arith.ll | 762 ++++++++----
...eaming-mode-fixed-length-optimize-ptrue.ll | 154 ++-
.../test/CodeGen/AArch64/sve-unary-movprfx.ll | 36 +-
llvm/test/CodeGen/ARM/iabs.ll | 8 +-
llvm/test/CodeGen/NVPTX/int-abs.ll | 120 ++
llvm/test/CodeGen/X86/abds-neg.ll | 587 ++++++---
llvm/test/CodeGen/X86/abds-vector-128.ll | 1084 +++++++++++++++--
llvm/test/CodeGen/X86/abds-vector-256.ll | 870 +++++++++++--
llvm/test/CodeGen/X86/abds-vector-512.ll | 398 +++++-
llvm/test/CodeGen/X86/abds.ll | 562 ++++++---
llvm/test/CodeGen/X86/abdu-neg.ll | 363 ++++--
llvm/test/CodeGen/X86/abdu-vector-128.ll | 791 ++++++++++--
llvm/test/CodeGen/X86/abdu-vector-256.ll | 716 +++++++++--
llvm/test/CodeGen/X86/abdu-vector-512.ll | 313 ++++-
llvm/test/CodeGen/X86/abdu.ll | 341 ++++--
llvm/test/CodeGen/X86/abs.ll | 707 ++++++-----
llvm/test/CodeGen/X86/apx/no-rex2-general.ll | 21 +-
.../CodeGen/X86/avx2-intrinsics-fast-isel.ll | 12 +-
.../X86/avx2-intrinsics-x86-upgrade.ll | 12 +-
.../CodeGen/X86/avx512-intrinsics-upgrade.ll | 24 +-
.../X86/avx512bw-intrinsics-upgrade.ll | 24 +-
.../X86/avx512bwvl-intrinsics-upgrade.ll | 48 +-
.../X86/avx512vl-intrinsics-upgrade.ll | 48 +-
llvm/test/CodeGen/X86/build-vector-128.ll | 38 +-
llvm/test/CodeGen/X86/combine-abs.ll | 164 ++-
.../CodeGen/X86/expand-vp-int-intrinsics.ll | 20 +-
llvm/test/CodeGen/X86/freeze-unary.ll | 41 +-
llvm/test/CodeGen/X86/neg-abs.ll | 187 +--
llvm/test/CodeGen/X86/pr55271.ll | 4 +-
.../CodeGen/X86/ssse3-intrinsics-fast-isel.ll | 32 +-
.../X86/ssse3-intrinsics-x86-upgrade.ll | 44 +-
45 files changed, 7559 insertions(+), 2180 deletions(-)
create mode 100644 llvm/test/CodeGen/NVPTX/int-abs.ll
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 392e53b99c64e..212aeb9c678e2 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -7427,9 +7427,19 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
return;
}
case Intrinsic::abs: {
- // TODO: Preserve "int min is poison" arg in SDAG?
+ //Preserve "int min is poison" arg in SDAG
SDValue Op1 = getValue(I.getArgOperand(0));
- setValue(&I, DAG.getNode(ISD::ABS, sdl, Op1.getValueType(), Op1));
+ bool IntMinIsPoison = cast<ConstantInt>(I.getArgOperand(1))->isOne();
+ EVT VT = Op1.getValueType();
+ if (IntMinIsPoison) {
+ setValue(&I, DAG.getNode(ISD::ABS, sdl, VT, Op1));
+ } else {
+
+ Op1 = DAG.getFreeze(Op1);
+ SDValue Neg = DAG.getNode(ISD::SUB, sdl, VT,
+ DAG.getConstant(0, sdl, VT), Op1);
+ setValue(&I, DAG.getNode(ISD::SMAX, sdl, VT, Op1, Neg));
+ }
return;
}
case Intrinsic::scmp: {
diff --git a/llvm/test/CodeGen/AArch64/abd-combine.ll b/llvm/test/CodeGen/AArch64/abd-combine.ll
index cdb40ceb46b1e..b43957e6c5afe 100644
--- a/llvm/test/CodeGen/AArch64/abd-combine.ll
+++ b/llvm/test/CodeGen/AArch64/abd-combine.ll
@@ -4,7 +4,13 @@
define <8 x i16> @abdu_base(<8 x i16> %src1, <8 x i16> %src2) {
; CHECK-LABEL: abdu_base:
; CHECK: // %bb.0:
-; CHECK-NEXT: uabd v0.8h, v0.8h, v1.8h
+; CHECK-NEXT: usubl v2.4s, v0.4h, v1.4h
+; CHECK-NEXT: usubl2 v0.4s, v0.8h, v1.8h
+; CHECK-NEXT: neg v1.4s, v0.4s
+; CHECK-NEXT: neg v3.4s, v2.4s
+; CHECK-NEXT: smax v0.4s, v0.4s, v1.4s
+; CHECK-NEXT: smax v1.4s, v2.4s, v3.4s
+; CHECK-NEXT: uzp1 v0.8h, v1.8h, v0.8h
; CHECK-NEXT: ret
%zextsrc1 = zext <8 x i16> %src1 to <8 x i32>
%zextsrc2 = zext <8 x i16> %src2 to <8 x i32>
@@ -17,9 +23,16 @@ define <8 x i16> @abdu_base(<8 x i16> %src1, <8 x i16> %src2) {
define <8 x i16> @abdu_const(<8 x i16> %src1) {
; CHECK-LABEL: abdu_const:
; CHECK: // %bb.0:
-; CHECK-NEXT: movi v1.4h, #1
-; CHECK-NEXT: mov v1.d[1], v1.d[0]
-; CHECK-NEXT: uabd v0.8h, v0.8h, v1.8h
+; CHECK-NEXT: movi v1.4s, #1
+; CHECK-NEXT: ushll2 v2.4s, v0.8h, #0
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NEXT: sub v0.4s, v0.4s, v1.4s
+; CHECK-NEXT: sub v1.4s, v2.4s, v1.4s
+; CHECK-NEXT: neg v2.4s, v1.4s
+; CHECK-NEXT: neg v3.4s, v0.4s
+; CHECK-NEXT: smax v1.4s, v1.4s, v2.4s
+; CHECK-NEXT: smax v0.4s, v0.4s, v3.4s
+; CHECK-NEXT: uzp1 v0.8h, v0.8h, v1.8h
; CHECK-NEXT: ret
%zextsrc1 = zext <8 x i16> %src1 to <8 x i32>
%sub = sub <8 x i32> %zextsrc1, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
@@ -31,9 +44,14 @@ define <8 x i16> @abdu_const(<8 x i16> %src1) {
define <8 x i16> @abdu_const_lhs(<8 x i16> %src1) {
; CHECK-LABEL: abdu_const_lhs:
; CHECK: // %bb.0:
-; CHECK-NEXT: movi v1.4h, #1
-; CHECK-NEXT: mov v1.d[1], v1.d[0]
-; CHECK-NEXT: uabd v0.8h, v0.8h, v1.8h
+; CHECK-NEXT: movi v1.4s, #1
+; CHECK-NEXT: usubw v2.4s, v1.4s, v0.4h
+; CHECK-NEXT: usubw2 v0.4s, v1.4s, v0.8h
+; CHECK-NEXT: neg v1.4s, v0.4s
+; CHECK-NEXT: neg v3.4s, v2.4s
+; CHECK-NEXT: smax v0.4s, v0.4s, v1.4s
+; CHECK-NEXT: smax v1.4s, v2.4s, v3.4s
+; CHECK-NEXT: uzp1 v0.8h, v1.8h, v0.8h
; CHECK-NEXT: ret
%zextsrc1 = zext <8 x i16> %src1 to <8 x i32>
%sub = sub <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>, %zextsrc1
@@ -45,6 +63,14 @@ define <8 x i16> @abdu_const_lhs(<8 x i16> %src1) {
define <8 x i16> @abdu_const_zero(<8 x i16> %src1) {
; CHECK-LABEL: abdu_const_zero:
; CHECK: // %bb.0:
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: ushll v2.4s, v0.4h, #0
+; CHECK-NEXT: ushll2 v3.4s, v0.8h, #0
+; CHECK-NEXT: usubw2 v0.4s, v1.4s, v0.8h
+; CHECK-NEXT: neg v1.4s, v2.4s
+; CHECK-NEXT: smax v0.4s, v0.4s, v3.4s
+; CHECK-NEXT: smax v1.4s, v1.4s, v2.4s
+; CHECK-NEXT: uzp1 v0.8h, v1.8h, v0.8h
; CHECK-NEXT: ret
%zextsrc1 = zext <8 x i16> %src1 to <8 x i32>
%sub = sub <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>, %zextsrc1
@@ -80,6 +106,14 @@ define <8 x i16> @abdu_const_bothhigh() {
define <8 x i16> @abdu_undef(<8 x i16> %src1) {
; CHECK-LABEL: abdu_undef:
; CHECK: // %bb.0:
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: ushll v2.4s, v0.4h, #0
+; CHECK-NEXT: ushll2 v3.4s, v0.8h, #0
+; CHECK-NEXT: usubw2 v0.4s, v1.4s, v0.8h
+; CHECK-NEXT: neg v1.4s, v2.4s
+; CHECK-NEXT: smax v0.4s, v3.4s, v0.4s
+; CHECK-NEXT: smax v1.4s, v2.4s, v1.4s
+; CHECK-NEXT: uzp1 v0.8h, v1.8h, v0.8h
; CHECK-NEXT: ret
%zextsrc1 = zext <8 x i16> %src1 to <8 x i32>
%zextsrc2 = zext <8 x i16> undef to <8 x i32>
@@ -299,7 +333,13 @@ define <8 x i16> @abdu_i_reassoc(<8 x i16> %src1) {
define <8 x i16> @abds_base(<8 x i16> %src1, <8 x i16> %src2) {
; CHECK-LABEL: abds_base:
; CHECK: // %bb.0:
-; CHECK-NEXT: sabd v0.8h, v0.8h, v1.8h
+; CHECK-NEXT: ssubl v2.4s, v0.4h, v1.4h
+; CHECK-NEXT: ssubl2 v0.4s, v0.8h, v1.8h
+; CHECK-NEXT: neg v1.4s, v0.4s
+; CHECK-NEXT: neg v3.4s, v2.4s
+; CHECK-NEXT: smax v0.4s, v0.4s, v1.4s
+; CHECK-NEXT: smax v1.4s, v2.4s, v3.4s
+; CHECK-NEXT: uzp1 v0.8h, v1.8h, v0.8h
; CHECK-NEXT: ret
%zextsrc1 = sext <8 x i16> %src1 to <8 x i32>
%zextsrc2 = sext <8 x i16> %src2 to <8 x i32>
@@ -312,9 +352,16 @@ define <8 x i16> @abds_base(<8 x i16> %src1, <8 x i16> %src2) {
define <8 x i16> @abds_const(<8 x i16> %src1) {
; CHECK-LABEL: abds_const:
; CHECK: // %bb.0:
-; CHECK-NEXT: movi v1.4h, #1
-; CHECK-NEXT: mov v1.d[1], v1.d[0]
-; CHECK-NEXT: sabd v0.8h, v0.8h, v1.8h
+; CHECK-NEXT: movi v1.4s, #1
+; CHECK-NEXT: sshll2 v2.4s, v0.8h, #0
+; CHECK-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-NEXT: sub v0.4s, v0.4s, v1.4s
+; CHECK-NEXT: sub v1.4s, v2.4s, v1.4s
+; CHECK-NEXT: neg v2.4s, v1.4s
+; CHECK-NEXT: neg v3.4s, v0.4s
+; CHECK-NEXT: smax v1.4s, v1.4s, v2.4s
+; CHECK-NEXT: smax v0.4s, v0.4s, v3.4s
+; CHECK-NEXT: uzp1 v0.8h, v0.8h, v1.8h
; CHECK-NEXT: ret
%zextsrc1 = sext <8 x i16> %src1 to <8 x i32>
%sub = sub <8 x i32> %zextsrc1, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
@@ -326,9 +373,14 @@ define <8 x i16> @abds_const(<8 x i16> %src1) {
define <8 x i16> @abds_const_lhs(<8 x i16> %src1) {
; CHECK-LABEL: abds_const_lhs:
; CHECK: // %bb.0:
-; CHECK-NEXT: movi v1.4h, #1
-; CHECK-NEXT: mov v1.d[1], v1.d[0]
-; CHECK-NEXT: sabd v0.8h, v0.8h, v1.8h
+; CHECK-NEXT: movi v1.4s, #1
+; CHECK-NEXT: ssubw v2.4s, v1.4s, v0.4h
+; CHECK-NEXT: ssubw2 v0.4s, v1.4s, v0.8h
+; CHECK-NEXT: neg v1.4s, v0.4s
+; CHECK-NEXT: neg v3.4s, v2.4s
+; CHECK-NEXT: smax v0.4s, v0.4s, v1.4s
+; CHECK-NEXT: smax v1.4s, v2.4s, v3.4s
+; CHECK-NEXT: uzp1 v0.8h, v1.8h, v0.8h
; CHECK-NEXT: ret
%zextsrc1 = sext <8 x i16> %src1 to <8 x i32>
%sub = sub <8 x i32> <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>, %zextsrc1
@@ -340,10 +392,14 @@ define <8 x i16> @abds_const_lhs(<8 x i16> %src1) {
define <8 x i16> @abds_const_zero(<8 x i16> %src1) {
; CHECK-LABEL: abds_const_zero:
; CHECK: // %bb.0:
-; CHECK-NEXT: ext v1.16b, v0.16b, v0.16b, #8
-; CHECK-NEXT: abs v0.4h, v0.4h
-; CHECK-NEXT: abs v1.4h, v1.4h
-; CHECK-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: sshll v2.4s, v0.4h, #0
+; CHECK-NEXT: sshll2 v3.4s, v0.8h, #0
+; CHECK-NEXT: ssubw2 v0.4s, v1.4s, v0.8h
+; CHECK-NEXT: neg v1.4s, v2.4s
+; CHECK-NEXT: smax v0.4s, v0.4s, v3.4s
+; CHECK-NEXT: smax v1.4s, v1.4s, v2.4s
+; CHECK-NEXT: uzp1 v0.8h, v1.8h, v0.8h
; CHECK-NEXT: ret
%zextsrc1 = sext <8 x i16> %src1 to <8 x i32>
%sub = sub <8 x i32> <i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0, i32 0>, %zextsrc1
@@ -379,10 +435,13 @@ define <8 x i16> @abds_const_bothhigh() {
define <8 x i16> @abds_undef(<8 x i16> %src1) {
; CHECK-LABEL: abds_undef:
; CHECK: // %bb.0:
-; CHECK-NEXT: sshll v1.4s, v0.4h, #0
-; CHECK-NEXT: sshll2 v0.4s, v0.8h, #0
-; CHECK-NEXT: abs v0.4s, v0.4s
-; CHECK-NEXT: abs v1.4s, v1.4s
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: sshll v2.4s, v0.4h, #0
+; CHECK-NEXT: sshll2 v3.4s, v0.8h, #0
+; CHECK-NEXT: ssubw2 v0.4s, v1.4s, v0.8h
+; CHECK-NEXT: neg v1.4s, v2.4s
+; CHECK-NEXT: smax v0.4s, v3.4s, v0.4s
+; CHECK-NEXT: smax v1.4s, v2.4s, v1.4s
; CHECK-NEXT: uzp1 v0.8h, v1.8h, v0.8h
; CHECK-NEXT: ret
%zextsrc1 = sext <8 x i16> %src1 to <8 x i32>
diff --git a/llvm/test/CodeGen/AArch64/abds-neg.ll b/llvm/test/CodeGen/AArch64/abds-neg.ll
index 37319642f5b34..a2496e1194335 100644
--- a/llvm/test/CodeGen/AArch64/abds-neg.ll
+++ b/llvm/test/CodeGen/AArch64/abds-neg.ll
@@ -8,9 +8,13 @@
define i8 @abd_ext_i8(i8 %a, i8 %b) nounwind {
; CHECK-LABEL: abd_ext_i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: sxtb w8, w0
-; CHECK-NEXT: subs w8, w8, w1, sxtb
-; CHECK-NEXT: cneg w0, w8, pl
+; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT: sxtb x8, w0
+; CHECK-NEXT: sub x8, x8, w1, sxtb
+; CHECK-NEXT: neg x9, x8
+; CHECK-NEXT: cmp x8, x9
+; CHECK-NEXT: cneg x8, x8, le
+; CHECK-NEXT: neg w0, w8
; CHECK-NEXT: ret
%aext = sext i8 %a to i64
%bext = sext i8 %b to i64
@@ -24,9 +28,13 @@ define i8 @abd_ext_i8(i8 %a, i8 %b) nounwind {
define i8 @abd_ext_i8_i16(i8 %a, i16 %b) nounwind {
; CHECK-LABEL: abd_ext_i8_i16:
; CHECK: // %bb.0:
-; CHECK-NEXT: sxtb w8, w0
-; CHECK-NEXT: subs w8, w8, w1, sxth
-; CHECK-NEXT: cneg w0, w8, pl
+; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT: sxtb x8, w0
+; CHECK-NEXT: sub x8, x8, w1, sxth
+; CHECK-NEXT: neg x9, x8
+; CHECK-NEXT: cmp x8, x9
+; CHECK-NEXT: cneg x8, x8, le
+; CHECK-NEXT: neg w0, w8
; CHECK-NEXT: ret
%aext = sext i8 %a to i64
%bext = sext i16 %b to i64
@@ -56,9 +64,13 @@ define i8 @abd_ext_i8_undef(i8 %a, i8 %b) nounwind {
define i16 @abd_ext_i16(i16 %a, i16 %b) nounwind {
; CHECK-LABEL: abd_ext_i16:
; CHECK: // %bb.0:
-; CHECK-NEXT: sxth w8, w0
-; CHECK-NEXT: subs w8, w8, w1, sxth
-; CHECK-NEXT: cneg w0, w8, pl
+; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT: sxth x8, w0
+; CHECK-NEXT: sub x8, x8, w1, sxth
+; CHECK-NEXT: neg x9, x8
+; CHECK-NEXT: cmp x8, x9
+; CHECK-NEXT: cneg x8, x8, le
+; CHECK-NEXT: neg w0, w8
; CHECK-NEXT: ret
%aext = sext i16 %a to i64
%bext = sext i16 %b to i64
@@ -72,9 +84,13 @@ define i16 @abd_ext_i16(i16 %a, i16 %b) nounwind {
define i16 @abd_ext_i16_i32(i16 %a, i32 %b) nounwind {
; CHECK-LABEL: abd_ext_i16_i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: sxth w8, w0
-; CHECK-NEXT: subs w8, w8, w1
-; CHECK-NEXT: cneg w0, w8, gt
+; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT: sxth x8, w0
+; CHECK-NEXT: sub x8, x8, w1, sxtw
+; CHECK-NEXT: neg x9, x8
+; CHECK-NEXT: cmp x8, x9
+; CHECK-NEXT: cneg x8, x8, le
+; CHECK-NEXT: neg w0, w8
; CHECK-NEXT: ret
%aext = sext i16 %a to i64
%bext = sext i32 %b to i64
@@ -104,8 +120,13 @@ define i16 @abd_ext_i16_undef(i16 %a, i16 %b) nounwind {
define i32 @abd_ext_i32(i32 %a, i32 %b) nounwind {
; CHECK-LABEL: abd_ext_i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: subs w8, w0, w1
-; CHECK-NEXT: cneg w0, w8, gt
+; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT: sxtw x8, w0
+; CHECK-NEXT: sub x8, x8, w1, sxtw
+; CHECK-NEXT: neg x9, x8
+; CHECK-NEXT: cmp x8, x9
+; CHECK-NEXT: cneg x8, x8, le
+; CHECK-NEXT: neg w0, w8
; CHECK-NEXT: ret
%aext = sext i32 %a to i64
%bext = sext i32 %b to i64
@@ -119,8 +140,13 @@ define i32 @abd_ext_i32(i32 %a, i32 %b) nounwind {
define i32 @abd_ext_i32_i16(i32 %a, i16 %b) nounwind {
; CHECK-LABEL: abd_ext_i32_i16:
; CHECK: // %bb.0:
-; CHECK-NEXT: subs w8, w0, w1, sxth
-; CHECK-NEXT: cneg w0, w8, gt
+; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT: sxtw x8, w0
+; CHECK-NEXT: sub x8, x8, w1, sxth
+; CHECK-NEXT: neg x9, x8
+; CHECK-NEXT: cmp x8, x9
+; CHECK-NEXT: cneg x8, x8, le
+; CHECK-NEXT: neg w0, w8
; CHECK-NEXT: ret
%aext = sext i32 %a to i64
%bext = sext i16 %b to i64
@@ -149,8 +175,16 @@ define i32 @abd_ext_i32_undef(i32 %a, i32 %b) nounwind {
define i64 @abd_ext_i64(i64 %a, i64 %b) nounwind {
; CHECK-LABEL: abd_ext_i64:
; CHECK: // %bb.0:
-; CHECK-NEXT: subs x8, x0, x1
-; CHECK-NEXT: cneg x0, x8, gt
+; CHECK-NEXT: asr x8, x0, #63
+; CHECK-NEXT: asr x9, x1, #63
+; CHECK-NEXT: subs x10, x0, x1
+; CHECK-NEXT: sbc x8, x8, x9
+; CHECK-NEXT: negs x9, x10
+; CHECK-NEXT: ngc x11, x8
+; CHECK-NEXT: cmp x9, x10
+; CHECK-NEXT: sbcs xzr, x11, x8
+; CHECK-NEXT: csel x8, x10, x9, lt
+; CHECK-NEXT: neg x0, x8
; CHECK-NEXT: ret
%aext = sext i64 %a to i128
%bext = sext i64 %b to i128
@@ -179,12 +213,22 @@ define i64 @abd_ext_i64_undef(i64 %a, i64 %b) nounwind {
define i128 @abd_ext_i128(i128 %a, i128 %b) nounwind {
; CHECK-LABEL: abd_ext_i128:
; CHECK: // %bb.0:
-; CHECK-NEXT: subs x8, x0, x2
-; CHECK-NEXT: sbc x9, x1, x3
-; CHECK-NEXT: subs x10, x2, x0
-; CHECK-NEXT: sbcs x11, x3, x1
-; CHECK-NEXT: csel x8, x8, x10, lt
-; CHECK-NEXT: csel x9, x9, x11, lt
+; CHECK-NEXT: asr x8, x1, #63
+; CHECK-NEXT: asr x9, x3, #63
+; CHECK-NEXT: subs x10, x0, x2
+; CHECK-NEXT: sbcs x11, x1, x3
+; CHECK-NEXT: sbcs x12, x8, x9
+; CHECK-NEXT: sbc x8, x8, x9
+; CHECK-NEXT: negs x9, x10
+; CHECK-NEXT: ngcs x13, x11
+; CHECK-NEXT: ngcs x14, x12
+; CHECK-NEXT: ngc x15, x8
+; CHECK-NEXT: cmp x9, x10
+; CHECK-NEXT: sbcs xzr, x13, x11
+; CHECK-NEXT: sbcs xzr, x14, x12
+; CHECK-NEXT: sbcs xzr, x15, x8
+; CHECK-NEXT: csel x8, x10, x9, lt
+; CHECK-NEXT: csel x9, x11, x13, lt
; CHECK-NEXT: negs x0, x8
; CHECK-NEXT: ngc x1, x9
; CHECK-NEXT: ret
@@ -389,9 +433,12 @@ define i8 @abd_subnsw_i8(i8 %a, i8 %b) nounwind {
; CHECK-LABEL: abd_subnsw_i8:
; CHECK: // %bb.0:
; CHECK-NEXT: sub w8, w0, w1
-; CHECK-NEXT: sbfx w9, w8, #7, #1
-; CHECK-NEXT: eor w8, w8, w9
-; CHECK-NEXT: sub w0, w9, w8
+; CHECK-NEXT: neg w9, w8
+; CHECK-NEXT: sxtb w8, w8
+; CHECK-NEXT: sxtb w9, w9
+; CHECK-NEXT: cmp w8, w9
+; CHECK-NEXT: csel w8, w8, w9, gt
+; CHECK-NEXT: neg w0, w8
; CHECK-NEXT: ret
%sub = sub nsw i8 %a, %b
%abs = call i8 @llvm.abs.i8(i8 %sub, i1 false)
@@ -417,9 +464,12 @@ define i16 @abd_subnsw_i16(i16 %a, i16 %b) nounwind {
; CHECK-LABEL: abd_subnsw_i16:
; CHECK: // %bb.0:
; CHECK-NEXT: sub w8, w0, w1
-; CHECK-NEXT: sbfx w9, w8, #15, #1
-; CHECK-NEXT: eor w8, w8, w9
-; CHECK-NEXT: sub w0, w9, w8
+; CHECK-NEXT: neg w9, w8
+; CHECK-NEXT: sxth w8, w8
+; CHECK-NEXT: sxth w9, w9
+; CHECK-NEXT: cmp w8, w9
+; CHECK-NEXT: csel w8, w8, w9, gt
+; CHECK-NEXT: neg w0, w8
; CHECK-NEXT: ret
%sub = sub nsw i16 %a, %b
%abs = call i16 @llvm.abs.i16(i16 %sub, i1 false)
@@ -444,8 +494,10 @@ define i16 @abd_subnsw_i16_undef(i16 %a, i16 %b) nounwind {
define i32 @abd_subnsw_i32(i32 %a, i32 %b) nounwind {
; CHECK-LABEL: abd_subnsw_i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: subs w8, w0, w1
-; CHECK-NEXT: cneg w0, w8, pl
+; CHECK-NEXT: sub w8, w0, w1
+; CHECK-NEXT: neg w9, w8
+; CHECK-NEXT: cmp w8, w9
+; CHECK-NEXT: cneg w0, w8, gt
; CHECK-NEXT: ret
%sub = sub nsw i32 %a, %b
%abs = call i32 @llvm.abs.i32(i32 %sub, i1 false)
@@ -468,8 +520,10 @@ define i32 @abd_subnsw_i32_undef(i32 %a, i32 %b) nounwind {
define i64 @abd_subnsw_i64(i64 %a, i64 %b) nounwind {
; CHECK-LABEL: abd_subnsw_i64:
; CHECK: // %bb.0:
-; CHECK-NEXT: subs x8, x0, x1
-; CHECK-NEXT: cneg x0, x8, pl
+; CHECK-NEXT: sub x8, x0, x1
+; CHECK-NEXT: neg x9, x8
+; CHECK-NEXT: cmp x8, x9
+; CHECK-NEXT: cneg x0, x8, gt
; CHECK-NEXT: ret
%sub = sub nsw i64 %a, %b
%abs = call i64 @llvm.abs.i64(i64 %sub, i1 false)
@@ -494,11 +548,14 @@ define i128 @abd_subnsw_i128(i128 %a, i128 %b) nounwind {
; CHECK: // %bb.0:
; CHECK-NEXT: subs x8, x0, x2
; CHECK-NEXT: sbc x9, x1, x3
-; CHECK-NEXT: asr x10, x9, #63
-; CHECK-NEXT: eor x8, x8, x10
-; CHECK-NEXT: eor x9, x9, x10
-; CHECK-NEXT: subs x0, x10, x8
-; CHECK-NEXT: sbc x1, x10, x9
+; CHECK-NEXT: negs x10, x8
+; CHECK-NEXT: ngc x11, x9
+; CHECK-NEXT: cmp x10, x8
+; CHECK-NEXT: sbcs xzr, x11, x9
+; CHECK-NEXT: csel x8, x8, x10, lt
+; CHECK-NEXT: csel x9, x9, x11, lt
+; CHECK-NEXT: negs x0, x8
+; CHECK-NEXT: ngc x1, x9
; CHECK-NEXT: ret
%sub = sub nsw i128 %a, %b
%abs = call i128 @llvm.abs.i128(i128 %sub, i1 false)
diff --git a/llvm/test/CodeGen/AArch64/abds.ll b/llvm/test/CodeGen/AArch64/abds.ll
index d0c390c2f0632..2a4f5b2f32027 100644
--- a/llvm/test/CodeGen/AArch64/abds.ll
+++ b/llvm/test/CodeGen/AArch64/abds.ll
@@ -8,9 +8,13 @@
define i8 @abd_ext_i8(i8 %a, i8 %b) nounwind {
; CHECK-LABEL: abd_ext_i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: sxtb w8, w0
-; CHECK-NEXT: subs w8, w8, w1, sxtb
-; CHECK-NEXT: cneg w0, w8, mi
+; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT: sxtb x8, w0
+; CHECK-NEXT: sub x8, x8, w1, sxtb
+; CHECK-NEXT: neg x9, x8
+; CHECK-NEXT: cmp x8, x9
+; CHECK-NEXT: cneg x0, x8, le
+; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-NEXT: ret
%aext = sext i8 %a to i64
%bext = sext i8 %b to i64
@@ -23,9 +27,13 @@ define i8 @abd_ext_i8(i8 %a, i8 %b) nounwind {
define i8 @abd_ext_i8_i16(i8 %a, i16 %b) nounwind {
; CHECK-LABEL: abd_ext_i8_i16:
; CHECK: // %bb.0:
-; CHECK-NEXT: sxtb w8, w0
-; CHECK-NEXT: subs w8, w8, w1, sxth
-; CHECK-NEXT: cneg w0, w8, mi
+; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT: sxtb x8, w0
+; CHECK-NEXT: sub x8, x8, w1, sxth
+; CHECK-NEXT: neg x9, x8
+; CHECK-NEXT: cmp x8, x9
+; CHECK-NEXT: cneg x0, x8, le
+; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-NEXT: ret
%aext = sext i8 %a to i64
%bext = sext i16 %b to i64
@@ -53,9 +61,13 @@ define i8 @abd_ext_i8_undef(i8 %a, i8 %b) nounwind {
define i16 @abd_ext_i16(i16 %a, i16 %b) nounwind {
; CHECK-LABEL: abd_ext_i16:
; CHECK: // %bb.0:
-; CHECK-NEXT: sxth w8, w0
-; CHECK-NEXT: subs w8, w8, w1, sxth
-; CHECK-NEXT: cneg w0, w8, mi
+; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT: sxth x8, w0
+; CHECK-NEXT: sub x8, x8, w1, sxth
+; CHECK-NEXT: neg x9, x8
+; CHECK-NEXT: cmp x8, x9
+; CHECK-NEXT: cneg x0, x8, le
+; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-NEXT: ret
%aext = sext i16 %a to i64
%bext = sext i16 %b to i64
@@ -68,9 +80,13 @@ define i16 @abd_ext_i16(i16 %a, i16 %b) nounwind {
define i16 @abd_ext_i16_i32(i16 %a, i32 %b) nounwind {
; CHECK-LABEL: abd_ext_i16_i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: sxth w8, w0
-; CHECK-NEXT: subs w8, w8, w1
-; CHECK-NEXT: cneg w0, w8, le
+; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT: sxth x8, w0
+; CHECK-NEXT: sub x8, x8, w1, sxtw
+; CHECK-NEXT: neg x9, x8
+; CHECK-NEXT: cmp x8, x9
+; CHECK-NEXT: cneg x0, x8, le
+; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-NEXT: ret
%aext = sext i16 %a to i64
%bext = sext i32 %b to i64
@@ -98,8 +114,13 @@ define i16 @abd_ext_i16_undef(i16 %a, i16 %b) nounwind {
define i32 @abd_ext_i32(i32 %a, i32 %b) nounwind {
; CHECK-LABEL: abd_ext_i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: subs w8, w0, w1
-; CHECK-NEXT: cneg w0, w8, le
+; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT: sxtw x8, w0
+; CHECK-NEXT: sub x8, x8, w1, sxtw
+; CHECK-NEXT: neg x9, x8
+; CHECK-NEXT: cmp x8, x9
+; CHECK-NEXT: cneg x0, x8, le
+; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-NEXT: ret
%aext = sext i32 %a to i64
%bext = sext i32 %b to i64
@@ -112,8 +133,13 @@ define i32 @abd_ext_i32(i32 %a, i32 %b) nounwind {
define i32 @abd_ext_i32_i16(i32 %a, i16 %b) nounwind {
; CHECK-LABEL: abd_ext_i32_i16:
; CHECK: // %bb.0:
-; CHECK-NEXT: subs w8, w0, w1, sxth
-; CHECK-NEXT: cneg w0, w8, le
+; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT: sxtw x8, w0
+; CHECK-NEXT: sub x8, x8, w1, sxth
+; CHECK-NEXT: neg x9, x8
+; CHECK-NEXT: cmp x8, x9
+; CHECK-NEXT: cneg x0, x8, le
+; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-NEXT: ret
%aext = sext i32 %a to i64
%bext = sext i16 %b to i64
@@ -140,8 +166,15 @@ define i32 @abd_ext_i32_undef(i32 %a, i32 %b) nounwind {
define i64 @abd_ext_i64(i64 %a, i64 %b) nounwind {
; CHECK-LABEL: abd_ext_i64:
; CHECK: // %bb.0:
-; CHECK-NEXT: subs x8, x0, x1
-; CHECK-NEXT: cneg x0, x8, le
+; CHECK-NEXT: asr x8, x0, #63
+; CHECK-NEXT: asr x9, x1, #63
+; CHECK-NEXT: subs x10, x0, x1
+; CHECK-NEXT: sbc x8, x8, x9
+; CHECK-NEXT: negs x9, x10
+; CHECK-NEXT: ngc x11, x8
+; CHECK-NEXT: cmp x9, x10
+; CHECK-NEXT: sbcs xzr, x11, x8
+; CHECK-NEXT: csel x0, x10, x9, lt
; CHECK-NEXT: ret
%aext = sext i64 %a to i128
%bext = sext i64 %b to i128
@@ -168,12 +201,22 @@ define i64 @abd_ext_i64_undef(i64 %a, i64 %b) nounwind {
define i128 @abd_ext_i128(i128 %a, i128 %b) nounwind {
; CHECK-LABEL: abd_ext_i128:
; CHECK: // %bb.0:
-; CHECK-NEXT: subs x8, x0, x2
-; CHECK-NEXT: sbc x9, x1, x3
-; CHECK-NEXT: subs x10, x2, x0
-; CHECK-NEXT: sbcs x11, x3, x1
-; CHECK-NEXT: csel x0, x8, x10, lt
-; CHECK-NEXT: csel x1, x9, x11, lt
+; CHECK-NEXT: asr x8, x1, #63
+; CHECK-NEXT: asr x9, x3, #63
+; CHECK-NEXT: subs x10, x0, x2
+; CHECK-NEXT: sbcs x11, x1, x3
+; CHECK-NEXT: sbcs x12, x8, x9
+; CHECK-NEXT: sbc x8, x8, x9
+; CHECK-NEXT: negs x9, x10
+; CHECK-NEXT: ngcs x13, x11
+; CHECK-NEXT: ngcs x14, x12
+; CHECK-NEXT: ngc x15, x8
+; CHECK-NEXT: cmp x9, x10
+; CHECK-NEXT: sbcs xzr, x13, x11
+; CHECK-NEXT: sbcs xzr, x14, x12
+; CHECK-NEXT: sbcs xzr, x15, x8
+; CHECK-NEXT: csel x0, x10, x9, lt
+; CHECK-NEXT: csel x1, x11, x13, lt
; CHECK-NEXT: ret
%aext = sext i128 %a to i256
%bext = sext i128 %b to i256
@@ -422,9 +465,11 @@ define i8 @abd_subnsw_i8(i8 %a, i8 %b) nounwind {
; CHECK-LABEL: abd_subnsw_i8:
; CHECK: // %bb.0:
; CHECK-NEXT: sub w8, w0, w1
+; CHECK-NEXT: neg w9, w8
; CHECK-NEXT: sxtb w8, w8
-; CHECK-NEXT: cmp w8, #0
-; CHECK-NEXT: cneg w0, w8, mi
+; CHECK-NEXT: sxtb w9, w9
+; CHECK-NEXT: cmp w8, w9
+; CHECK-NEXT: csel w0, w8, w9, gt
; CHECK-NEXT: ret
%sub = sub nsw i8 %a, %b
%abs = call i8 @llvm.abs.i8(i8 %sub, i1 false)
@@ -448,9 +493,11 @@ define i16 @abd_subnsw_i16(i16 %a, i16 %b) nounwind {
; CHECK-LABEL: abd_subnsw_i16:
; CHECK: // %bb.0:
; CHECK-NEXT: sub w8, w0, w1
+; CHECK-NEXT: neg w9, w8
; CHECK-NEXT: sxth w8, w8
-; CHECK-NEXT: cmp w8, #0
-; CHECK-NEXT: cneg w0, w8, mi
+; CHECK-NEXT: sxth w9, w9
+; CHECK-NEXT: cmp w8, w9
+; CHECK-NEXT: csel w0, w8, w9, gt
; CHECK-NEXT: ret
%sub = sub nsw i16 %a, %b
%abs = call i16 @llvm.abs.i16(i16 %sub, i1 false)
@@ -473,8 +520,10 @@ define i16 @abd_subnsw_i16_undef(i16 %a, i16 %b) nounwind {
define i32 @abd_subnsw_i32(i32 %a, i32 %b) nounwind {
; CHECK-LABEL: abd_subnsw_i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: subs w8, w0, w1
-; CHECK-NEXT: cneg w0, w8, mi
+; CHECK-NEXT: sub w8, w0, w1
+; CHECK-NEXT: neg w9, w8
+; CHECK-NEXT: cmp w8, w9
+; CHECK-NEXT: cneg w0, w8, le
; CHECK-NEXT: ret
%sub = sub nsw i32 %a, %b
%abs = call i32 @llvm.abs.i32(i32 %sub, i1 false)
@@ -495,8 +544,10 @@ define i32 @abd_subnsw_i32_undef(i32 %a, i32 %b) nounwind {
define i64 @abd_subnsw_i64(i64 %a, i64 %b) nounwind {
; CHECK-LABEL: abd_subnsw_i64:
; CHECK: // %bb.0:
-; CHECK-NEXT: subs x8, x0, x1
-; CHECK-NEXT: cneg x0, x8, mi
+; CHECK-NEXT: sub x8, x0, x1
+; CHECK-NEXT: neg x9, x8
+; CHECK-NEXT: cmp x8, x9
+; CHECK-NEXT: cneg x0, x8, le
; CHECK-NEXT: ret
%sub = sub nsw i64 %a, %b
%abs = call i64 @llvm.abs.i64(i64 %sub, i1 false)
@@ -519,11 +570,12 @@ define i128 @abd_subnsw_i128(i128 %a, i128 %b) nounwind {
; CHECK: // %bb.0:
; CHECK-NEXT: subs x8, x0, x2
; CHECK-NEXT: sbc x9, x1, x3
-; CHECK-NEXT: asr x10, x9, #63
-; CHECK-NEXT: eor x8, x8, x10
-; CHECK-NEXT: eor x9, x9, x10
-; CHECK-NEXT: subs x0, x8, x10
-; CHECK-NEXT: sbc x1, x9, x10
+; CHECK-NEXT: negs x10, x8
+; CHECK-NEXT: ngc x11, x9
+; CHECK-NEXT: cmp x10, x8
+; CHECK-NEXT: sbcs xzr, x11, x9
+; CHECK-NEXT: csel x0, x8, x10, lt
+; CHECK-NEXT: csel x1, x9, x11, lt
; CHECK-NEXT: ret
%sub = sub nsw i128 %a, %b
%abs = call i128 @llvm.abs.i128(i128 %sub, i1 false)
@@ -553,8 +605,10 @@ define i128 @abd_subnsw_i128_undef(i128 %a, i128 %b) nounwind {
define i32 @abd_sub_i32(i32 %a, i32 %b) nounwind {
; CHECK-LABEL: abd_sub_i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: subs w8, w0, w1
-; CHECK-NEXT: cneg w0, w8, mi
+; CHECK-NEXT: sub w8, w0, w1
+; CHECK-NEXT: neg w9, w8
+; CHECK-NEXT: cmp w8, w9
+; CHECK-NEXT: cneg w0, w8, le
; CHECK-NEXT: ret
%sub = sub i32 %a, %b
%abs = call i32 @llvm.abs.i32(i32 %sub, i1 false)
@@ -565,8 +619,10 @@ define i64 @vector_legalized(i16 %a, i16 %b) {
; CHECK-LABEL: vector_legalized:
; CHECK: // %bb.0:
; CHECK-NEXT: sxth w8, w0
-; CHECK-NEXT: subs w8, w8, w1, sxth
-; CHECK-NEXT: cneg w0, w8, mi
+; CHECK-NEXT: sub w8, w8, w1, sxth
+; CHECK-NEXT: neg w9, w8
+; CHECK-NEXT: cmp w8, w9
+; CHECK-NEXT: cneg w0, w8, le
; CHECK-NEXT: ret
%ea = sext i16 %a to i32
%eb = sext i16 %b to i32
diff --git a/llvm/test/CodeGen/AArch64/abdu-neg.ll b/llvm/test/CodeGen/AArch64/abdu-neg.ll
index 269cbf03f32a0..0f2b4a0868d0d 100644
--- a/llvm/test/CodeGen/AArch64/abdu-neg.ll
+++ b/llvm/test/CodeGen/AArch64/abdu-neg.ll
@@ -8,9 +8,13 @@
define i8 @abd_ext_i8(i8 %a, i8 %b) nounwind {
; CHECK-LABEL: abd_ext_i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: and w8, w0, #0xff
-; CHECK-NEXT: subs w8, w8, w1, uxtb
-; CHECK-NEXT: cneg w0, w8, pl
+; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT: and x8, x0, #0xff
+; CHECK-NEXT: sub x8, x8, w1, uxtb
+; CHECK-NEXT: neg x9, x8
+; CHECK-NEXT: cmp x8, x9
+; CHECK-NEXT: cneg x8, x8, le
+; CHECK-NEXT: neg w0, w8
; CHECK-NEXT: ret
%aext = zext i8 %a to i64
%bext = zext i8 %b to i64
@@ -24,9 +28,13 @@ define i8 @abd_ext_i8(i8 %a, i8 %b) nounwind {
define i8 @abd_ext_i8_i16(i8 %a, i16 %b) nounwind {
; CHECK-LABEL: abd_ext_i8_i16:
; CHECK: // %bb.0:
-; CHECK-NEXT: and w8, w0, #0xff
-; CHECK-NEXT: subs w8, w8, w1, uxth
-; CHECK-NEXT: cneg w0, w8, pl
+; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT: and x8, x0, #0xff
+; CHECK-NEXT: sub x8, x8, w1, uxth
+; CHECK-NEXT: neg x9, x8
+; CHECK-NEXT: cmp x8, x9
+; CHECK-NEXT: cneg x8, x8, le
+; CHECK-NEXT: neg w0, w8
; CHECK-NEXT: ret
%aext = zext i8 %a to i64
%bext = zext i16 %b to i64
@@ -56,9 +64,13 @@ define i8 @abd_ext_i8_undef(i8 %a, i8 %b) nounwind {
define i16 @abd_ext_i16(i16 %a, i16 %b) nounwind {
; CHECK-LABEL: abd_ext_i16:
; CHECK: // %bb.0:
-; CHECK-NEXT: and w8, w0, #0xffff
-; CHECK-NEXT: subs w8, w8, w1, uxth
-; CHECK-NEXT: cneg w0, w8, pl
+; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT: and x8, x0, #0xffff
+; CHECK-NEXT: sub x8, x8, w1, uxth
+; CHECK-NEXT: neg x9, x8
+; CHECK-NEXT: cmp x8, x9
+; CHECK-NEXT: cneg x8, x8, le
+; CHECK-NEXT: neg w0, w8
; CHECK-NEXT: ret
%aext = zext i16 %a to i64
%bext = zext i16 %b to i64
@@ -72,9 +84,13 @@ define i16 @abd_ext_i16(i16 %a, i16 %b) nounwind {
define i16 @abd_ext_i16_i32(i16 %a, i32 %b) nounwind {
; CHECK-LABEL: abd_ext_i16_i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: and w8, w0, #0xffff
-; CHECK-NEXT: subs w8, w8, w1
-; CHECK-NEXT: cneg w0, w8, hi
+; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT: and x8, x0, #0xffff
+; CHECK-NEXT: sub x8, x8, w1, uxtw
+; CHECK-NEXT: neg x9, x8
+; CHECK-NEXT: cmp x8, x9
+; CHECK-NEXT: cneg x8, x8, le
+; CHECK-NEXT: neg w0, w8
; CHECK-NEXT: ret
%aext = zext i16 %a to i64
%bext = zext i32 %b to i64
@@ -104,8 +120,12 @@ define i16 @abd_ext_i16_undef(i16 %a, i16 %b) nounwind {
define i32 @abd_ext_i32(i32 %a, i32 %b) nounwind {
; CHECK-LABEL: abd_ext_i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: subs w8, w0, w1
-; CHECK-NEXT: cneg w0, w8, hi
+; CHECK-NEXT: mov w8, w0
+; CHECK-NEXT: sub x8, x8, w1, uxtw
+; CHECK-NEXT: neg x9, x8
+; CHECK-NEXT: cmp x8, x9
+; CHECK-NEXT: cneg x8, x8, le
+; CHECK-NEXT: neg w0, w8
; CHECK-NEXT: ret
%aext = zext i32 %a to i64
%bext = zext i32 %b to i64
@@ -119,8 +139,12 @@ define i32 @abd_ext_i32(i32 %a, i32 %b) nounwind {
define i32 @abd_ext_i32_i16(i32 %a, i16 %b) nounwind {
; CHECK-LABEL: abd_ext_i32_i16:
; CHECK: // %bb.0:
-; CHECK-NEXT: subs w8, w0, w1, uxth
-; CHECK-NEXT: cneg w0, w8, hi
+; CHECK-NEXT: mov w8, w0
+; CHECK-NEXT: sub x8, x8, w1, uxth
+; CHECK-NEXT: neg x9, x8
+; CHECK-NEXT: cmp x8, x9
+; CHECK-NEXT: cneg x8, x8, le
+; CHECK-NEXT: neg w0, w8
; CHECK-NEXT: ret
%aext = zext i32 %a to i64
%bext = zext i16 %b to i64
@@ -150,7 +174,13 @@ define i64 @abd_ext_i64(i64 %a, i64 %b) nounwind {
; CHECK-LABEL: abd_ext_i64:
; CHECK: // %bb.0:
; CHECK-NEXT: subs x8, x0, x1
-; CHECK-NEXT: cneg x0, x8, hi
+; CHECK-NEXT: ngc x9, xzr
+; CHECK-NEXT: negs x10, x8
+; CHECK-NEXT: ngc x11, x9
+; CHECK-NEXT: cmp x10, x8
+; CHECK-NEXT: sbcs xzr, x11, x9
+; CHECK-NEXT: csel x8, x8, x10, lt
+; CHECK-NEXT: neg x0, x8
; CHECK-NEXT: ret
%aext = zext i64 %a to i128
%bext = zext i64 %b to i128
@@ -180,11 +210,19 @@ define i128 @abd_ext_i128(i128 %a, i128 %b) nounwind {
; CHECK-LABEL: abd_ext_i128:
; CHECK: // %bb.0:
; CHECK-NEXT: subs x8, x0, x2
-; CHECK-NEXT: sbc x9, x1, x3
-; CHECK-NEXT: subs x10, x2, x0
-; CHECK-NEXT: sbcs x11, x3, x1
-; CHECK-NEXT: csel x8, x8, x10, lo
-; CHECK-NEXT: csel x9, x9, x11, lo
+; CHECK-NEXT: sbcs x9, x1, x3
+; CHECK-NEXT: ngcs x10, xzr
+; CHECK-NEXT: ngc x11, xzr
+; CHECK-NEXT: negs x12, x8
+; CHECK-NEXT: ngcs x13, x9
+; CHECK-NEXT: ngcs x14, x10
+; CHECK-NEXT: ngc x15, x11
+; CHECK-NEXT: cmp x12, x8
+; CHECK-NEXT: sbcs xzr, x13, x9
+; CHECK-NEXT: sbcs xzr, x14, x10
+; CHECK-NEXT: sbcs xzr, x15, x11
+; CHECK-NEXT: csel x8, x8, x12, lt
+; CHECK-NEXT: csel x9, x9, x13, lt
; CHECK-NEXT: negs x0, x8
; CHECK-NEXT: ngc x1, x9
; CHECK-NEXT: ret
diff --git a/llvm/test/CodeGen/AArch64/abdu.ll b/llvm/test/CodeGen/AArch64/abdu.ll
index 3cbe648788a84..9fc96a5ae39b3 100644
--- a/llvm/test/CodeGen/AArch64/abdu.ll
+++ b/llvm/test/CodeGen/AArch64/abdu.ll
@@ -8,9 +8,13 @@
define i8 @abd_ext_i8(i8 %a, i8 %b) nounwind {
; CHECK-LABEL: abd_ext_i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: and w8, w0, #0xff
-; CHECK-NEXT: subs w8, w8, w1, uxtb
-; CHECK-NEXT: cneg w0, w8, mi
+; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT: and x8, x0, #0xff
+; CHECK-NEXT: sub x8, x8, w1, uxtb
+; CHECK-NEXT: neg x9, x8
+; CHECK-NEXT: cmp x8, x9
+; CHECK-NEXT: cneg x0, x8, le
+; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-NEXT: ret
%aext = zext i8 %a to i64
%bext = zext i8 %b to i64
@@ -23,9 +27,13 @@ define i8 @abd_ext_i8(i8 %a, i8 %b) nounwind {
define i8 @abd_ext_i8_i16(i8 %a, i16 %b) nounwind {
; CHECK-LABEL: abd_ext_i8_i16:
; CHECK: // %bb.0:
-; CHECK-NEXT: and w8, w0, #0xff
-; CHECK-NEXT: subs w8, w8, w1, uxth
-; CHECK-NEXT: cneg w0, w8, mi
+; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT: and x8, x0, #0xff
+; CHECK-NEXT: sub x8, x8, w1, uxth
+; CHECK-NEXT: neg x9, x8
+; CHECK-NEXT: cmp x8, x9
+; CHECK-NEXT: cneg x0, x8, le
+; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-NEXT: ret
%aext = zext i8 %a to i64
%bext = zext i16 %b to i64
@@ -53,9 +61,13 @@ define i8 @abd_ext_i8_undef(i8 %a, i8 %b) nounwind {
define i16 @abd_ext_i16(i16 %a, i16 %b) nounwind {
; CHECK-LABEL: abd_ext_i16:
; CHECK: // %bb.0:
-; CHECK-NEXT: and w8, w0, #0xffff
-; CHECK-NEXT: subs w8, w8, w1, uxth
-; CHECK-NEXT: cneg w0, w8, mi
+; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT: and x8, x0, #0xffff
+; CHECK-NEXT: sub x8, x8, w1, uxth
+; CHECK-NEXT: neg x9, x8
+; CHECK-NEXT: cmp x8, x9
+; CHECK-NEXT: cneg x0, x8, le
+; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-NEXT: ret
%aext = zext i16 %a to i64
%bext = zext i16 %b to i64
@@ -68,9 +80,13 @@ define i16 @abd_ext_i16(i16 %a, i16 %b) nounwind {
define i16 @abd_ext_i16_i32(i16 %a, i32 %b) nounwind {
; CHECK-LABEL: abd_ext_i16_i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: and w8, w0, #0xffff
-; CHECK-NEXT: subs w8, w8, w1
-; CHECK-NEXT: cneg w0, w8, ls
+; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0
+; CHECK-NEXT: and x8, x0, #0xffff
+; CHECK-NEXT: sub x8, x8, w1, uxtw
+; CHECK-NEXT: neg x9, x8
+; CHECK-NEXT: cmp x8, x9
+; CHECK-NEXT: cneg x0, x8, le
+; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-NEXT: ret
%aext = zext i16 %a to i64
%bext = zext i32 %b to i64
@@ -98,8 +114,12 @@ define i16 @abd_ext_i16_undef(i16 %a, i16 %b) nounwind {
define i32 @abd_ext_i32(i32 %a, i32 %b) nounwind {
; CHECK-LABEL: abd_ext_i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: subs w8, w0, w1
-; CHECK-NEXT: cneg w0, w8, ls
+; CHECK-NEXT: mov w8, w0
+; CHECK-NEXT: sub x8, x8, w1, uxtw
+; CHECK-NEXT: neg x9, x8
+; CHECK-NEXT: cmp x8, x9
+; CHECK-NEXT: cneg x0, x8, le
+; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-NEXT: ret
%aext = zext i32 %a to i64
%bext = zext i32 %b to i64
@@ -112,8 +132,12 @@ define i32 @abd_ext_i32(i32 %a, i32 %b) nounwind {
define i32 @abd_ext_i32_i16(i32 %a, i16 %b) nounwind {
; CHECK-LABEL: abd_ext_i32_i16:
; CHECK: // %bb.0:
-; CHECK-NEXT: subs w8, w0, w1, uxth
-; CHECK-NEXT: cneg w0, w8, ls
+; CHECK-NEXT: mov w8, w0
+; CHECK-NEXT: sub x8, x8, w1, uxth
+; CHECK-NEXT: neg x9, x8
+; CHECK-NEXT: cmp x8, x9
+; CHECK-NEXT: cneg x0, x8, le
+; CHECK-NEXT: // kill: def $w0 killed $w0 killed $x0
; CHECK-NEXT: ret
%aext = zext i32 %a to i64
%bext = zext i16 %b to i64
@@ -141,7 +165,12 @@ define i64 @abd_ext_i64(i64 %a, i64 %b) nounwind {
; CHECK-LABEL: abd_ext_i64:
; CHECK: // %bb.0:
; CHECK-NEXT: subs x8, x0, x1
-; CHECK-NEXT: cneg x0, x8, ls
+; CHECK-NEXT: ngc x9, xzr
+; CHECK-NEXT: negs x10, x8
+; CHECK-NEXT: ngc x11, x9
+; CHECK-NEXT: cmp x10, x8
+; CHECK-NEXT: sbcs xzr, x11, x9
+; CHECK-NEXT: csel x0, x8, x10, lt
; CHECK-NEXT: ret
%aext = zext i64 %a to i128
%bext = zext i64 %b to i128
@@ -169,11 +198,19 @@ define i128 @abd_ext_i128(i128 %a, i128 %b) nounwind {
; CHECK-LABEL: abd_ext_i128:
; CHECK: // %bb.0:
; CHECK-NEXT: subs x8, x0, x2
-; CHECK-NEXT: sbc x9, x1, x3
-; CHECK-NEXT: subs x10, x2, x0
-; CHECK-NEXT: sbcs x11, x3, x1
-; CHECK-NEXT: csel x0, x8, x10, lo
-; CHECK-NEXT: csel x1, x9, x11, lo
+; CHECK-NEXT: sbcs x9, x1, x3
+; CHECK-NEXT: ngcs x10, xzr
+; CHECK-NEXT: ngc x11, xzr
+; CHECK-NEXT: negs x12, x8
+; CHECK-NEXT: ngcs x13, x9
+; CHECK-NEXT: ngcs x14, x10
+; CHECK-NEXT: ngc x15, x11
+; CHECK-NEXT: cmp x12, x8
+; CHECK-NEXT: sbcs xzr, x13, x9
+; CHECK-NEXT: sbcs xzr, x14, x10
+; CHECK-NEXT: sbcs xzr, x15, x11
+; CHECK-NEXT: csel x0, x8, x12, lt
+; CHECK-NEXT: csel x1, x9, x13, lt
; CHECK-NEXT: ret
%aext = zext i128 %a to i256
%bext = zext i128 %b to i256
@@ -354,8 +391,10 @@ define i64 @vector_legalized(i16 %a, i16 %b) {
; CHECK-LABEL: vector_legalized:
; CHECK: // %bb.0:
; CHECK-NEXT: and w8, w0, #0xffff
-; CHECK-NEXT: subs w8, w8, w1, uxth
-; CHECK-NEXT: cneg w0, w8, mi
+; CHECK-NEXT: sub w8, w8, w1, uxth
+; CHECK-NEXT: neg w9, w8
+; CHECK-NEXT: cmp w8, w9
+; CHECK-NEXT: cneg w0, w8, le
; CHECK-NEXT: ret
%ea = zext i16 %a to i32
%eb = zext i16 %b to i32
diff --git a/llvm/test/CodeGen/AArch64/abs.ll b/llvm/test/CodeGen/AArch64/abs.ll
index 0f56d25a47b2a..5a1c69b8fc75d 100644
--- a/llvm/test/CodeGen/AArch64/abs.ll
+++ b/llvm/test/CodeGen/AArch64/abs.ll
@@ -7,9 +7,11 @@
define i8 @abs_i8(i8 %a){
; CHECK-SD-LABEL: abs_i8:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: sxtb w8, w0
-; CHECK-SD-NEXT: cmp w8, #0
-; CHECK-SD-NEXT: cneg w0, w8, mi
+; CHECK-SD-NEXT: neg w8, w0
+; CHECK-SD-NEXT: sxtb w9, w0
+; CHECK-SD-NEXT: sxtb w8, w8
+; CHECK-SD-NEXT: cmp w9, w8
+; CHECK-SD-NEXT: csel w0, w9, w8, gt
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: abs_i8:
@@ -27,9 +29,11 @@ declare i8 @llvm.abs.i8(i8, i1)
define i16 @abs_i16(i16 %a){
; CHECK-SD-LABEL: abs_i16:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: sxth w8, w0
-; CHECK-SD-NEXT: cmp w8, #0
-; CHECK-SD-NEXT: cneg w0, w8, mi
+; CHECK-SD-NEXT: neg w8, w0
+; CHECK-SD-NEXT: sxth w9, w0
+; CHECK-SD-NEXT: sxth w8, w8
+; CHECK-SD-NEXT: cmp w9, w8
+; CHECK-SD-NEXT: csel w0, w9, w8, gt
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: abs_i16:
@@ -47,8 +51,9 @@ declare i16 @llvm.abs.i16(i16, i1)
define i32 @abs_i32(i32 %a){
; CHECK-SD-LABEL: abs_i32:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: cmp w0, #0
-; CHECK-SD-NEXT: cneg w0, w0, mi
+; CHECK-SD-NEXT: neg w8, w0
+; CHECK-SD-NEXT: cmp w0, w8
+; CHECK-SD-NEXT: cneg w0, w0, le
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: abs_i32:
@@ -65,8 +70,9 @@ declare i32 @llvm.abs.i32(i32, i1)
define i64 @abs_i64(i64 %a){
; CHECK-SD-LABEL: abs_i64:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: cmp x0, #0
-; CHECK-SD-NEXT: cneg x0, x0, mi
+; CHECK-SD-NEXT: neg x8, x0
+; CHECK-SD-NEXT: cmp x0, x8
+; CHECK-SD-NEXT: cneg x0, x0, le
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: abs_i64:
@@ -83,11 +89,12 @@ declare i64 @llvm.abs.i64(i64, i1)
define i128 @abs_i128(i128 %a){
; CHECK-SD-LABEL: abs_i128:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: asr x8, x1, #63
-; CHECK-SD-NEXT: eor x9, x0, x8
-; CHECK-SD-NEXT: eor x10, x1, x8
-; CHECK-SD-NEXT: subs x0, x9, x8
-; CHECK-SD-NEXT: sbc x1, x10, x8
+; CHECK-SD-NEXT: negs x8, x0
+; CHECK-SD-NEXT: ngc x9, x1
+; CHECK-SD-NEXT: cmp x8, x0
+; CHECK-SD-NEXT: sbcs xzr, x9, x1
+; CHECK-SD-NEXT: csel x0, x0, x8, lt
+; CHECK-SD-NEXT: csel x1, x1, x9, lt
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: abs_i128:
@@ -107,10 +114,16 @@ declare i128 @llvm.abs.i128(i128, i1)
; ===== Legal Vector Types =====
define <8 x i8> @abs_v8i8(<8 x i8> %a){
-; CHECK-LABEL: abs_v8i8:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: abs v0.8b, v0.8b
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: abs_v8i8:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: neg v1.8b, v0.8b
+; CHECK-SD-NEXT: smax v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: abs_v8i8:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: abs v0.8b, v0.8b
+; CHECK-GI-NEXT: ret
entry:
%res = call <8 x i8> @llvm.abs.v8i8(<8 x i8> %a, i1 0)
ret <8 x i8> %res
@@ -118,10 +131,16 @@ entry:
declare <8 x i8> @llvm.abs.v8i8(<8 x i8>, i1)
define <16 x i8> @abs_v16i8(<16 x i8> %a){
-; CHECK-LABEL: abs_v16i8:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: abs v0.16b, v0.16b
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: abs_v16i8:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: neg v1.16b, v0.16b
+; CHECK-SD-NEXT: smax v0.16b, v0.16b, v1.16b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: abs_v16i8:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: abs v0.16b, v0.16b
+; CHECK-GI-NEXT: ret
entry:
%res = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a, i1 0)
ret <16 x i8> %res
@@ -129,10 +148,16 @@ entry:
declare <16 x i8> @llvm.abs.v16i8(<16 x i8>, i1)
define <4 x i16> @abs_v4i16(<4 x i16> %a){
-; CHECK-LABEL: abs_v4i16:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: abs v0.4h, v0.4h
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: abs_v4i16:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: neg v1.4h, v0.4h
+; CHECK-SD-NEXT: smax v0.4h, v0.4h, v1.4h
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: abs_v4i16:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: abs v0.4h, v0.4h
+; CHECK-GI-NEXT: ret
entry:
%res = call <4 x i16> @llvm.abs.v4i16(<4 x i16> %a, i1 0)
ret <4 x i16> %res
@@ -140,10 +165,16 @@ entry:
declare <4 x i16> @llvm.abs.v4i16(<4 x i16>, i1)
define <8 x i16> @abs_v8i16(<8 x i16> %a){
-; CHECK-LABEL: abs_v8i16:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: abs v0.8h, v0.8h
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: abs_v8i16:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: neg v1.8h, v0.8h
+; CHECK-SD-NEXT: smax v0.8h, v0.8h, v1.8h
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: abs_v8i16:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: abs v0.8h, v0.8h
+; CHECK-GI-NEXT: ret
entry:
%res = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a, i1 0)
ret <8 x i16> %res
@@ -151,10 +182,16 @@ entry:
declare <8 x i16> @llvm.abs.v8i16(<8 x i16>, i1)
define <2 x i32> @abs_v2i32(<2 x i32> %a){
-; CHECK-LABEL: abs_v2i32:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: abs v0.2s, v0.2s
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: abs_v2i32:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: neg v1.2s, v0.2s
+; CHECK-SD-NEXT: smax v0.2s, v0.2s, v1.2s
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: abs_v2i32:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: abs v0.2s, v0.2s
+; CHECK-GI-NEXT: ret
entry:
%res = call <2 x i32> @llvm.abs.v2i32(<2 x i32> %a, i1 0)
ret <2 x i32> %res
@@ -162,10 +199,16 @@ entry:
declare <2 x i32> @llvm.abs.v2i32(<2 x i32>, i1)
define <4 x i32> @abs_v4i32(<4 x i32> %a){
-; CHECK-LABEL: abs_v4i32:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: abs v0.4s, v0.4s
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: abs_v4i32:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: neg v1.4s, v0.4s
+; CHECK-SD-NEXT: smax v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: abs_v4i32:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: abs v0.4s, v0.4s
+; CHECK-GI-NEXT: ret
entry:
%res = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a, i1 0)
ret <4 x i32> %res
@@ -173,10 +216,17 @@ entry:
declare <4 x i32> @llvm.abs.v4i32(<4 x i32>, i1)
define <2 x i64> @abs_v2i64(<2 x i64> %a){
-; CHECK-LABEL: abs_v2i64:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: abs v0.2d, v0.2d
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: abs_v2i64:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: neg v1.2d, v0.2d
+; CHECK-SD-NEXT: cmgt v2.2d, v0.2d, v1.2d
+; CHECK-SD-NEXT: bif v0.16b, v1.16b, v2.16b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: abs_v2i64:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: abs v0.2d, v0.2d
+; CHECK-GI-NEXT: ret
entry:
%res = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %a, i1 0)
ret <2 x i64> %res
@@ -186,12 +236,22 @@ declare <2 x i64> @llvm.abs.v2i64(<2 x i64>, i1)
; ===== Smaller/Larger Width Vectors with Legal Element Sizes =====
define <4 x i8> @abs_v4i8(<4 x i8> %a){
-; CHECK-LABEL: abs_v4i8:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: shl v0.4h, v0.4h, #8
-; CHECK-NEXT: sshr v0.4h, v0.4h, #8
-; CHECK-NEXT: abs v0.4h, v0.4h
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: abs_v4i8:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: neg v1.4h, v0.4h
+; CHECK-SD-NEXT: shl v0.4h, v0.4h, #8
+; CHECK-SD-NEXT: shl v1.4h, v1.4h, #8
+; CHECK-SD-NEXT: sshr v0.4h, v0.4h, #8
+; CHECK-SD-NEXT: sshr v1.4h, v1.4h, #8
+; CHECK-SD-NEXT: smax v0.4h, v0.4h, v1.4h
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: abs_v4i8:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: shl v0.4h, v0.4h, #8
+; CHECK-GI-NEXT: sshr v0.4h, v0.4h, #8
+; CHECK-GI-NEXT: abs v0.4h, v0.4h
+; CHECK-GI-NEXT: ret
entry:
%res = call <4 x i8> @llvm.abs.v4i8(<4 x i8> %a, i1 0)
ret <4 x i8> %res
@@ -199,11 +259,19 @@ entry:
declare <4 x i8> @llvm.abs.v4i8(<4 x i8>, i1)
define <32 x i8> @abs_v32i8(<32 x i8> %a){
-; CHECK-LABEL: abs_v32i8:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: abs v0.16b, v0.16b
-; CHECK-NEXT: abs v1.16b, v1.16b
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: abs_v32i8:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: neg v2.16b, v0.16b
+; CHECK-SD-NEXT: neg v3.16b, v1.16b
+; CHECK-SD-NEXT: smax v0.16b, v0.16b, v2.16b
+; CHECK-SD-NEXT: smax v1.16b, v1.16b, v3.16b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: abs_v32i8:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: abs v0.16b, v0.16b
+; CHECK-GI-NEXT: abs v1.16b, v1.16b
+; CHECK-GI-NEXT: ret
entry:
%res = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %a, i1 0)
ret <32 x i8> %res
@@ -211,12 +279,22 @@ entry:
declare <32 x i8> @llvm.abs.v32i8(<32 x i8>, i1)
define <2 x i16> @abs_v2i16(<2 x i16> %a){
-; CHECK-LABEL: abs_v2i16:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: shl v0.2s, v0.2s, #16
-; CHECK-NEXT: sshr v0.2s, v0.2s, #16
-; CHECK-NEXT: abs v0.2s, v0.2s
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: abs_v2i16:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: neg v1.2s, v0.2s
+; CHECK-SD-NEXT: shl v0.2s, v0.2s, #16
+; CHECK-SD-NEXT: shl v1.2s, v1.2s, #16
+; CHECK-SD-NEXT: sshr v0.2s, v0.2s, #16
+; CHECK-SD-NEXT: sshr v1.2s, v1.2s, #16
+; CHECK-SD-NEXT: smax v0.2s, v0.2s, v1.2s
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: abs_v2i16:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: shl v0.2s, v0.2s, #16
+; CHECK-GI-NEXT: sshr v0.2s, v0.2s, #16
+; CHECK-GI-NEXT: abs v0.2s, v0.2s
+; CHECK-GI-NEXT: ret
entry:
%res = call <2 x i16> @llvm.abs.v2i16(<2 x i16> %a, i1 0)
ret <2 x i16> %res
@@ -224,11 +302,19 @@ entry:
declare <2 x i16> @llvm.abs.v2i16(<2 x i16>, i1)
define <16 x i16> @abs_v16i16(<16 x i16> %a){
-; CHECK-LABEL: abs_v16i16:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: abs v0.8h, v0.8h
-; CHECK-NEXT: abs v1.8h, v1.8h
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: abs_v16i16:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: neg v2.8h, v0.8h
+; CHECK-SD-NEXT: neg v3.8h, v1.8h
+; CHECK-SD-NEXT: smax v0.8h, v0.8h, v2.8h
+; CHECK-SD-NEXT: smax v1.8h, v1.8h, v3.8h
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: abs_v16i16:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: abs v0.8h, v0.8h
+; CHECK-GI-NEXT: abs v1.8h, v1.8h
+; CHECK-GI-NEXT: ret
entry:
%res = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %a, i1 0)
ret <16 x i16> %res
@@ -238,7 +324,8 @@ declare <16 x i16> @llvm.abs.v16i16(<16 x i16>, i1)
define <1 x i32> @abs_v1i32(<1 x i32> %a){
; CHECK-SD-LABEL: abs_v1i32:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: abs v0.2s, v0.2s
+; CHECK-SD-NEXT: neg v1.2s, v0.2s
+; CHECK-SD-NEXT: smax v0.2s, v0.2s, v1.2s
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: abs_v1i32:
@@ -256,11 +343,19 @@ entry:
declare <1 x i32> @llvm.abs.v1i32(<1 x i32>, i1)
define <8 x i32> @abs_v8i32(<8 x i32> %a){
-; CHECK-LABEL: abs_v8i32:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: abs v0.4s, v0.4s
-; CHECK-NEXT: abs v1.4s, v1.4s
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: abs_v8i32:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: neg v2.4s, v0.4s
+; CHECK-SD-NEXT: neg v3.4s, v1.4s
+; CHECK-SD-NEXT: smax v0.4s, v0.4s, v2.4s
+; CHECK-SD-NEXT: smax v1.4s, v1.4s, v3.4s
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: abs_v8i32:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: abs v0.4s, v0.4s
+; CHECK-GI-NEXT: abs v1.4s, v1.4s
+; CHECK-GI-NEXT: ret
entry:
%res = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %a, i1 0)
ret <8 x i32> %res
@@ -268,11 +363,21 @@ entry:
declare <8 x i32> @llvm.abs.v8i32(<8 x i32>, i1)
define <4 x i64> @abs_v4i64(<4 x i64> %a){
-; CHECK-LABEL: abs_v4i64:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: abs v0.2d, v0.2d
-; CHECK-NEXT: abs v1.2d, v1.2d
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: abs_v4i64:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: neg v2.2d, v0.2d
+; CHECK-SD-NEXT: neg v3.2d, v1.2d
+; CHECK-SD-NEXT: cmgt v4.2d, v0.2d, v2.2d
+; CHECK-SD-NEXT: cmgt v5.2d, v1.2d, v3.2d
+; CHECK-SD-NEXT: bif v0.16b, v2.16b, v4.16b
+; CHECK-SD-NEXT: bif v1.16b, v3.16b, v5.16b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: abs_v4i64:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: abs v0.2d, v0.2d
+; CHECK-GI-NEXT: abs v1.2d, v1.2d
+; CHECK-GI-NEXT: ret
entry:
%res = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %a, i1 0)
ret <4 x i64> %res
@@ -282,16 +387,18 @@ declare <4 x i64> @llvm.abs.v4i64(<4 x i64>, i1)
define <2 x i128> @abs_v4i128(<2 x i128> %a){
; CHECK-SD-LABEL: abs_v4i128:
; CHECK-SD: // %bb.0: // %entry
-; CHECK-SD-NEXT: asr x8, x1, #63
-; CHECK-SD-NEXT: asr x9, x3, #63
-; CHECK-SD-NEXT: eor x10, x0, x8
-; CHECK-SD-NEXT: eor x11, x1, x8
-; CHECK-SD-NEXT: subs x0, x10, x8
-; CHECK-SD-NEXT: eor x10, x2, x9
-; CHECK-SD-NEXT: sbc x1, x11, x8
-; CHECK-SD-NEXT: eor x8, x3, x9
-; CHECK-SD-NEXT: subs x2, x10, x9
-; CHECK-SD-NEXT: sbc x3, x8, x9
+; CHECK-SD-NEXT: negs x8, x0
+; CHECK-SD-NEXT: ngc x9, x1
+; CHECK-SD-NEXT: cmp x8, x0
+; CHECK-SD-NEXT: sbcs xzr, x9, x1
+; CHECK-SD-NEXT: csel x0, x0, x8, lt
+; CHECK-SD-NEXT: csel x1, x1, x9, lt
+; CHECK-SD-NEXT: negs x8, x2
+; CHECK-SD-NEXT: ngc x9, x3
+; CHECK-SD-NEXT: cmp x8, x2
+; CHECK-SD-NEXT: sbcs xzr, x9, x3
+; CHECK-SD-NEXT: csel x2, x2, x8, lt
+; CHECK-SD-NEXT: csel x3, x3, x9, lt
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: abs_v4i128:
@@ -321,9 +428,12 @@ define <3 x i8> @abs_v3i8(<3 x i8> %a){
; CHECK-SD-NEXT: fmov s0, w0
; CHECK-SD-NEXT: mov v0.h[1], w1
; CHECK-SD-NEXT: mov v0.h[2], w2
+; CHECK-SD-NEXT: neg v1.4h, v0.4h
; CHECK-SD-NEXT: shl v0.4h, v0.4h, #8
+; CHECK-SD-NEXT: shl v1.4h, v1.4h, #8
; CHECK-SD-NEXT: sshr v0.4h, v0.4h, #8
-; CHECK-SD-NEXT: abs v0.4h, v0.4h
+; CHECK-SD-NEXT: sshr v1.4h, v1.4h, #8
+; CHECK-SD-NEXT: smax v0.4h, v0.4h, v1.4h
; CHECK-SD-NEXT: umov w0, v0.h[0]
; CHECK-SD-NEXT: umov w1, v0.h[1]
; CHECK-SD-NEXT: umov w2, v0.h[2]
@@ -346,10 +456,16 @@ entry:
declare <3 x i8> @llvm.abs.v3i8(<3 x i8>, i1)
define <7 x i8> @abs_v7i8(<7 x i8> %a){
-; CHECK-LABEL: abs_v7i8:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: abs v0.8b, v0.8b
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: abs_v7i8:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: neg v1.8b, v0.8b
+; CHECK-SD-NEXT: smax v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: abs_v7i8:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: abs v0.8b, v0.8b
+; CHECK-GI-NEXT: ret
entry:
%res = call <7 x i8> @llvm.abs.v7i8(<7 x i8> %a, i1 0)
ret <7 x i8> %res
@@ -357,10 +473,16 @@ entry:
declare <7 x i8> @llvm.abs.v7i8(<7 x i8>, i1)
define <3 x i16> @abs_v3i16(<3 x i16> %a){
-; CHECK-LABEL: abs_v3i16:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: abs v0.4h, v0.4h
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: abs_v3i16:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: neg v1.4h, v0.4h
+; CHECK-SD-NEXT: smax v0.4h, v0.4h, v1.4h
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: abs_v3i16:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: abs v0.4h, v0.4h
+; CHECK-GI-NEXT: ret
entry:
%res = call <3 x i16> @llvm.abs.v3i16(<3 x i16> %a, i1 0)
ret <3 x i16> %res
@@ -368,10 +490,16 @@ entry:
declare <3 x i16> @llvm.abs.v3i16(<3 x i16>, i1)
define <7 x i16> @abs_v7i16(<7 x i16> %a){
-; CHECK-LABEL: abs_v7i16:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: abs v0.8h, v0.8h
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: abs_v7i16:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: neg v1.8h, v0.8h
+; CHECK-SD-NEXT: smax v0.8h, v0.8h, v1.8h
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: abs_v7i16:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: abs v0.8h, v0.8h
+; CHECK-GI-NEXT: ret
entry:
%res = call <7 x i16> @llvm.abs.v7i16(<7 x i16> %a, i1 0)
ret <7 x i16> %res
@@ -379,12 +507,20 @@ entry:
declare <7 x i16> @llvm.abs.v7i16(<7 x i16>, i1)
define <3 x i32> @abs_v3i32(<3 x i32> %a){
-; CHECK-LABEL: abs_v3i32:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: abs v0.4s, v0.4s
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: abs_v3i32:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: neg v1.4s, v0.4s
+; CHECK-SD-NEXT: smax v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: abs_v3i32:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: abs v0.4s, v0.4s
+; CHECK-GI-NEXT: ret
entry:
%res = call <3 x i32> @llvm.abs.v3i32(<3 x i32> %a, i1 0)
ret <3 x i32> %res
}
declare <3 x i32> @llvm.abs.v3i32(<3 x i32>, i1)
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/extract-vector-elt.ll b/llvm/test/CodeGen/AArch64/extract-vector-elt.ll
index 23bc6e0fb04ea..31603fb346c71 100644
--- a/llvm/test/CodeGen/AArch64/extract-vector-elt.ll
+++ b/llvm/test/CodeGen/AArch64/extract-vector-elt.ll
@@ -657,7 +657,8 @@ define i32 @extract_v4i32_abs(<4 x float> %a, i32 %c) {
; CHECK-SD-NEXT: frintm v0.4s, v0.4s
; CHECK-SD-NEXT: fabs v0.4s, v0.4s
; CHECK-SD-NEXT: fcvtzs v0.4s, v0.4s
-; CHECK-SD-NEXT: abs v0.4s, v0.4s
+; CHECK-SD-NEXT: neg v1.4s, v0.4s
+; CHECK-SD-NEXT: smax v0.4s, v0.4s, v1.4s
; CHECK-SD-NEXT: str q0, [sp]
; CHECK-SD-NEXT: ldr w0, [x8]
; CHECK-SD-NEXT: add sp, sp, #16
diff --git a/llvm/test/CodeGen/AArch64/neon-sad.ll b/llvm/test/CodeGen/AArch64/neon-sad.ll
index f21fafea50451..f957fee239eeb 100644
--- a/llvm/test/CodeGen/AArch64/neon-sad.ll
+++ b/llvm/test/CodeGen/AArch64/neon-sad.ll
@@ -51,9 +51,16 @@ define i32 @test_sad_v16i8_two_step_zext(ptr noundef readonly %a, ptr noundef re
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: ldr q0, [x0]
; CHECK-NEXT: ldr q1, [x1]
-; CHECK-NEXT: uabdl v2.8h, v1.8b, v0.8b
-; CHECK-NEXT: uabal2 v2.8h, v1.16b, v0.16b
-; CHECK-NEXT: uaddlv s0, v2.8h
+; CHECK-NEXT: usubl2 v2.8h, v1.16b, v0.16b
+; CHECK-NEXT: usubl v0.8h, v1.8b, v0.8b
+; CHECK-NEXT: neg v1.8h, v0.8h
+; CHECK-NEXT: neg v3.8h, v2.8h
+; CHECK-NEXT: smax v0.8h, v0.8h, v1.8h
+; CHECK-NEXT: smax v1.8h, v2.8h, v3.8h
+; CHECK-NEXT: uaddl2 v2.4s, v0.8h, v1.8h
+; CHECK-NEXT: uaddl v0.4s, v0.4h, v1.4h
+; CHECK-NEXT: add v0.4s, v0.4s, v2.4s
+; CHECK-NEXT: addv s0, v0.4s
; CHECK-NEXT: fmov w0, s0
; CHECK-NEXT: ret
entry:
@@ -73,9 +80,16 @@ define i32 @test_sad_v16i8_two_step_sext(ptr noundef readonly %a, ptr noundef re
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: ldr q0, [x0]
; CHECK-NEXT: ldr q1, [x1]
-; CHECK-NEXT: sabdl v2.8h, v1.8b, v0.8b
-; CHECK-NEXT: sabal2 v2.8h, v1.16b, v0.16b
-; CHECK-NEXT: uaddlv s0, v2.8h
+; CHECK-NEXT: ssubl2 v2.8h, v1.16b, v0.16b
+; CHECK-NEXT: ssubl v0.8h, v1.8b, v0.8b
+; CHECK-NEXT: neg v1.8h, v0.8h
+; CHECK-NEXT: neg v3.8h, v2.8h
+; CHECK-NEXT: smax v0.8h, v0.8h, v1.8h
+; CHECK-NEXT: smax v1.8h, v2.8h, v3.8h
+; CHECK-NEXT: uaddl2 v2.4s, v0.8h, v1.8h
+; CHECK-NEXT: uaddl v0.4s, v0.4h, v1.4h
+; CHECK-NEXT: add v0.4s, v0.4s, v2.4s
+; CHECK-NEXT: addv s0, v0.4s
; CHECK-NEXT: fmov w0, s0
; CHECK-NEXT: ret
entry:
diff --git a/llvm/test/CodeGen/AArch64/sched-movprfx.ll b/llvm/test/CodeGen/AArch64/sched-movprfx.ll
index cf1c1f45f700c..b1fb5cae933a1 100644
--- a/llvm/test/CodeGen/AArch64/sched-movprfx.ll
+++ b/llvm/test/CodeGen/AArch64/sched-movprfx.ll
@@ -11,9 +11,10 @@ define <vscale x 2 x i64> @and_i64_zero(<vscale x 2 x i1> %pg, <vscale x 2 x i64
; CHECK-LABEL: and_i64_zero:
; CHECK: // %bb.0:
; CHECK-NEXT: ld1d { z1.d }, p0/z, [x0]
+; CHECK-NEXT: mov z0.d, z2.d
; CHECK-NEXT: ptrue p1.d
-; CHECK-NEXT: movprfx z0, z2
-; CHECK-NEXT: abs z0.d, p1/m, z2.d
+; CHECK-NEXT: subr z0.d, z0.d, #0 // =0x0
+; CHECK-NEXT: smax z0.d, p1/m, z0.d, z2.d
; CHECK-NEXT: sel z1.d, p0, z1.d, z2.d
; CHECK-NEXT: add z0.d, z0.d, z1.d
; CHECK-NEXT: ret
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-arith.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-arith.ll
index ace0422bba09a..0f65fff17ed1f 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-int-arith.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-int-arith.ll
@@ -1213,7 +1213,8 @@ define void @sub_v32i64(ptr %a, ptr %b) vscale_range(16,0) #0 {
define <8 x i8> @abs_v8i8(<8 x i8> %op1) vscale_range(2,0) #0 {
; CHECK-LABEL: abs_v8i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: abs v0.8b, v0.8b
+; CHECK-NEXT: neg v1.8b, v0.8b
+; CHECK-NEXT: smax v0.8b, v0.8b, v1.8b
; CHECK-NEXT: ret
%res = call <8 x i8> @llvm.abs.v8i8(<8 x i8> %op1, i1 false)
ret <8 x i8> %res
@@ -1223,7 +1224,8 @@ define <8 x i8> @abs_v8i8(<8 x i8> %op1) vscale_range(2,0) #0 {
define <16 x i8> @abs_v16i8(<16 x i8> %op1) vscale_range(2,0) #0 {
; CHECK-LABEL: abs_v16i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: abs v0.16b, v0.16b
+; CHECK-NEXT: neg v1.16b, v0.16b
+; CHECK-NEXT: smax v0.16b, v0.16b, v1.16b
; CHECK-NEXT: ret
%res = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %op1, i1 false)
ret <16 x i8> %res
@@ -1234,7 +1236,9 @@ define void @abs_v32i8(ptr %a) vscale_range(2,0) #0 {
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.b, vl32
; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]
-; CHECK-NEXT: abs z0.b, p0/m, z0.b
+; CHECK-NEXT: mov z1.d, z0.d
+; CHECK-NEXT: subr z1.b, z1.b, #0 // =0x0
+; CHECK-NEXT: smax z0.b, p0/m, z0.b, z1.b
; CHECK-NEXT: st1b { z0.b }, p0, [x0]
; CHECK-NEXT: ret
%op1 = load <32 x i8>, ptr %a
@@ -1250,8 +1254,12 @@ define void @abs_v64i8(ptr %a) #0 {
; VBITS_GE_256-NEXT: mov w8, #32 // =0x20
; VBITS_GE_256-NEXT: ld1b { z0.b }, p0/z, [x0, x8]
; VBITS_GE_256-NEXT: ld1b { z1.b }, p0/z, [x0]
-; VBITS_GE_256-NEXT: abs z0.b, p0/m, z0.b
-; VBITS_GE_256-NEXT: abs z1.b, p0/m, z1.b
+; VBITS_GE_256-NEXT: mov z2.d, z0.d
+; VBITS_GE_256-NEXT: mov z3.d, z1.d
+; VBITS_GE_256-NEXT: subr z2.b, z2.b, #0 // =0x0
+; VBITS_GE_256-NEXT: subr z3.b, z3.b, #0 // =0x0
+; VBITS_GE_256-NEXT: smax z0.b, p0/m, z0.b, z2.b
+; VBITS_GE_256-NEXT: smax z1.b, p0/m, z1.b, z3.b
; VBITS_GE_256-NEXT: st1b { z0.b }, p0, [x0, x8]
; VBITS_GE_256-NEXT: st1b { z1.b }, p0, [x0]
; VBITS_GE_256-NEXT: ret
@@ -1260,7 +1268,9 @@ define void @abs_v64i8(ptr %a) #0 {
; VBITS_GE_512: // %bb.0:
; VBITS_GE_512-NEXT: ptrue p0.b, vl64
; VBITS_GE_512-NEXT: ld1b { z0.b }, p0/z, [x0]
-; VBITS_GE_512-NEXT: abs z0.b, p0/m, z0.b
+; VBITS_GE_512-NEXT: mov z1.d, z0.d
+; VBITS_GE_512-NEXT: subr z1.b, z1.b, #0 // =0x0
+; VBITS_GE_512-NEXT: smax z0.b, p0/m, z0.b, z1.b
; VBITS_GE_512-NEXT: st1b { z0.b }, p0, [x0]
; VBITS_GE_512-NEXT: ret
%op1 = load <64 x i8>, ptr %a
@@ -1274,7 +1284,9 @@ define void @abs_v128i8(ptr %a) vscale_range(8,0) #0 {
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.b, vl128
; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]
-; CHECK-NEXT: abs z0.b, p0/m, z0.b
+; CHECK-NEXT: mov z1.d, z0.d
+; CHECK-NEXT: subr z1.b, z1.b, #0 // =0x0
+; CHECK-NEXT: smax z0.b, p0/m, z0.b, z1.b
; CHECK-NEXT: st1b { z0.b }, p0, [x0]
; CHECK-NEXT: ret
%op1 = load <128 x i8>, ptr %a
@@ -1288,7 +1300,9 @@ define void @abs_v256i8(ptr %a) vscale_range(16,0) #0 {
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.b, vl256
; CHECK-NEXT: ld1b { z0.b }, p0/z, [x0]
-; CHECK-NEXT: abs z0.b, p0/m, z0.b
+; CHECK-NEXT: mov z1.d, z0.d
+; CHECK-NEXT: subr z1.b, z1.b, #0 // =0x0
+; CHECK-NEXT: smax z0.b, p0/m, z0.b, z1.b
; CHECK-NEXT: st1b { z0.b }, p0, [x0]
; CHECK-NEXT: ret
%op1 = load <256 x i8>, ptr %a
@@ -1301,7 +1315,8 @@ define void @abs_v256i8(ptr %a) vscale_range(16,0) #0 {
define <4 x i16> @abs_v4i16(<4 x i16> %op1) vscale_range(2,0) #0 {
; CHECK-LABEL: abs_v4i16:
; CHECK: // %bb.0:
-; CHECK-NEXT: abs v0.4h, v0.4h
+; CHECK-NEXT: neg v1.4h, v0.4h
+; CHECK-NEXT: smax v0.4h, v0.4h, v1.4h
; CHECK-NEXT: ret
%res = call <4 x i16> @llvm.abs.v4i16(<4 x i16> %op1, i1 false)
ret <4 x i16> %res
@@ -1311,7 +1326,8 @@ define <4 x i16> @abs_v4i16(<4 x i16> %op1) vscale_range(2,0) #0 {
define <8 x i16> @abs_v8i16(<8 x i16> %op1) vscale_range(2,0) #0 {
; CHECK-LABEL: abs_v8i16:
; CHECK: // %bb.0:
-; CHECK-NEXT: abs v0.8h, v0.8h
+; CHECK-NEXT: neg v1.8h, v0.8h
+; CHECK-NEXT: smax v0.8h, v0.8h, v1.8h
; CHECK-NEXT: ret
%res = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %op1, i1 false)
ret <8 x i16> %res
@@ -1322,7 +1338,9 @@ define void @abs_v16i16(ptr %a) vscale_range(2,0) #0 {
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.h, vl16
; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]
-; CHECK-NEXT: abs z0.h, p0/m, z0.h
+; CHECK-NEXT: mov z1.d, z0.d
+; CHECK-NEXT: subr z1.h, z1.h, #0 // =0x0
+; CHECK-NEXT: smax z0.h, p0/m, z0.h, z1.h
; CHECK-NEXT: st1h { z0.h }, p0, [x0]
; CHECK-NEXT: ret
%op1 = load <16 x i16>, ptr %a
@@ -1338,8 +1356,12 @@ define void @abs_v32i16(ptr %a) vscale_range(2,0) #0 {
; CHECK-NEXT: mov x8, #16 // =0x10
; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1]
; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0]
-; CHECK-NEXT: abs z0.h, p0/m, z0.h
-; CHECK-NEXT: abs z1.h, p0/m, z1.h
+; CHECK-NEXT: mov z2.d, z0.d
+; CHECK-NEXT: mov z3.d, z1.d
+; CHECK-NEXT: subr z2.h, z2.h, #0 // =0x0
+; CHECK-NEXT: subr z3.h, z3.h, #0 // =0x0
+; CHECK-NEXT: smax z0.h, p0/m, z0.h, z2.h
+; CHECK-NEXT: smax z1.h, p0/m, z1.h, z3.h
; CHECK-NEXT: st1h { z0.h }, p0, [x0, x8, lsl #1]
; CHECK-NEXT: st1h { z1.h }, p0, [x0]
; CHECK-NEXT: ret
@@ -1360,10 +1382,18 @@ define void @abs_v64i16(ptr %a) vscale_range(2,0) #0 {
; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0, x9, lsl #1]
; CHECK-NEXT: ld1h { z2.h }, p0/z, [x0, x10, lsl #1]
; CHECK-NEXT: ld1h { z3.h }, p0/z, [x0]
-; CHECK-NEXT: abs z0.h, p0/m, z0.h
-; CHECK-NEXT: abs z1.h, p0/m, z1.h
-; CHECK-NEXT: abs z2.h, p0/m, z2.h
-; CHECK-NEXT: abs z3.h, p0/m, z3.h
+; CHECK-NEXT: mov z4.d, z0.d
+; CHECK-NEXT: mov z5.d, z1.d
+; CHECK-NEXT: mov z6.d, z2.d
+; CHECK-NEXT: mov z7.d, z3.d
+; CHECK-NEXT: subr z4.h, z4.h, #0 // =0x0
+; CHECK-NEXT: subr z5.h, z5.h, #0 // =0x0
+; CHECK-NEXT: subr z6.h, z6.h, #0 // =0x0
+; CHECK-NEXT: subr z7.h, z7.h, #0 // =0x0
+; CHECK-NEXT: smax z0.h, p0/m, z0.h, z4.h
+; CHECK-NEXT: smax z1.h, p0/m, z1.h, z5.h
+; CHECK-NEXT: smax z2.h, p0/m, z2.h, z6.h
+; CHECK-NEXT: smax z3.h, p0/m, z3.h, z7.h
; CHECK-NEXT: st1h { z0.h }, p0, [x0, x8, lsl #1]
; CHECK-NEXT: st1h { z1.h }, p0, [x0, x9, lsl #1]
; CHECK-NEXT: st1h { z2.h }, p0, [x0, x10, lsl #1]
@@ -1379,37 +1409,53 @@ define void @abs_v128i16(ptr %a) vscale_range(2,0) #0 {
; CHECK-LABEL: abs_v128i16:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.h, vl16
-; CHECK-NEXT: mov x8, #96 // =0x60
-; CHECK-NEXT: mov x9, #112 // =0x70
-; CHECK-NEXT: mov x10, #64 // =0x40
-; CHECK-NEXT: mov x11, #80 // =0x50
+; CHECK-NEXT: mov x8, #80 // =0x50
+; CHECK-NEXT: mov x9, #96 // =0x60
+; CHECK-NEXT: mov x10, #112 // =0x70
+; CHECK-NEXT: mov x11, #64 // =0x40
; CHECK-NEXT: mov x12, #32 // =0x20
; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0, x8, lsl #1]
-; CHECK-NEXT: mov x13, #48 // =0x30
-; CHECK-NEXT: mov x14, #16 // =0x10
; CHECK-NEXT: ld1h { z1.h }, p0/z, [x0, x9, lsl #1]
; CHECK-NEXT: ld1h { z2.h }, p0/z, [x0, x10, lsl #1]
; CHECK-NEXT: ld1h { z3.h }, p0/z, [x0, x11, lsl #1]
-; CHECK-NEXT: ld1h { z4.h }, p0/z, [x0, x12, lsl #1]
-; CHECK-NEXT: ld1h { z5.h }, p0/z, [x0, x13, lsl #1]
-; CHECK-NEXT: ld1h { z6.h }, p0/z, [x0, x14, lsl #1]
-; CHECK-NEXT: abs z0.h, p0/m, z0.h
-; CHECK-NEXT: abs z1.h, p0/m, z1.h
-; CHECK-NEXT: abs z2.h, p0/m, z2.h
-; CHECK-NEXT: abs z3.h, p0/m, z3.h
-; CHECK-NEXT: abs z4.h, p0/m, z4.h
-; CHECK-NEXT: abs z5.h, p0/m, z5.h
-; CHECK-NEXT: abs z6.h, p0/m, z6.h
-; CHECK-NEXT: st1h { z0.h }, p0, [x0, x8, lsl #1]
-; CHECK-NEXT: ld1h { z0.h }, p0/z, [x0]
+; CHECK-NEXT: mov x13, #48 // =0x30
+; CHECK-NEXT: mov x14, #16 // =0x10
+; CHECK-NEXT: ld1h { z6.h }, p0/z, [x0, x12, lsl #1]
+; CHECK-NEXT: ld1h { z16.h }, p0/z, [x0, x13, lsl #1]
+; CHECK-NEXT: ld1h { z18.h }, p0/z, [x0, x14, lsl #1]
+; CHECK-NEXT: mov z4.d, z0.d
+; CHECK-NEXT: mov z5.d, z1.d
+; CHECK-NEXT: mov z7.d, z2.d
+; CHECK-NEXT: mov z17.d, z3.d
+; CHECK-NEXT: subr z4.h, z4.h, #0 // =0x0
+; CHECK-NEXT: subr z5.h, z5.h, #0 // =0x0
+; CHECK-NEXT: subr z7.h, z7.h, #0 // =0x0
+; CHECK-NEXT: subr z17.h, z17.h, #0 // =0x0
+; CHECK-NEXT: smax z0.h, p0/m, z0.h, z4.h
+; CHECK-NEXT: ld1h { z4.h }, p0/z, [x0]
+; CHECK-NEXT: smax z1.h, p0/m, z1.h, z5.h
+; CHECK-NEXT: mov z5.d, z6.d
+; CHECK-NEXT: smax z2.h, p0/m, z2.h, z7.h
+; CHECK-NEXT: mov z7.d, z16.d
+; CHECK-NEXT: smax z3.h, p0/m, z3.h, z17.h
+; CHECK-NEXT: mov z17.d, z18.d
; CHECK-NEXT: st1h { z1.h }, p0, [x0, x9, lsl #1]
+; CHECK-NEXT: mov z1.d, z4.d
+; CHECK-NEXT: subr z5.h, z5.h, #0 // =0x0
+; CHECK-NEXT: subr z7.h, z7.h, #0 // =0x0
; CHECK-NEXT: st1h { z2.h }, p0, [x0, x10, lsl #1]
-; CHECK-NEXT: abs z0.h, p0/m, z0.h
+; CHECK-NEXT: subr z17.h, z17.h, #0 // =0x0
; CHECK-NEXT: st1h { z3.h }, p0, [x0, x11, lsl #1]
-; CHECK-NEXT: st1h { z4.h }, p0, [x0, x12, lsl #1]
-; CHECK-NEXT: st1h { z5.h }, p0, [x0, x13, lsl #1]
-; CHECK-NEXT: st1h { z6.h }, p0, [x0, x14, lsl #1]
-; CHECK-NEXT: st1h { z0.h }, p0, [x0]
+; CHECK-NEXT: subr z1.h, z1.h, #0 // =0x0
+; CHECK-NEXT: st1h { z0.h }, p0, [x0, x8, lsl #1]
+; CHECK-NEXT: smax z5.h, p0/m, z5.h, z6.h
+; CHECK-NEXT: smax z7.h, p0/m, z7.h, z16.h
+; CHECK-NEXT: smax z17.h, p0/m, z17.h, z18.h
+; CHECK-NEXT: smax z1.h, p0/m, z1.h, z4.h
+; CHECK-NEXT: st1h { z5.h }, p0, [x0, x12, lsl #1]
+; CHECK-NEXT: st1h { z7.h }, p0, [x0, x13, lsl #1]
+; CHECK-NEXT: st1h { z17.h }, p0, [x0, x14, lsl #1]
+; CHECK-NEXT: st1h { z1.h }, p0, [x0]
; CHECK-NEXT: ret
%op1 = load <128 x i16>, ptr %a
%res = call <128 x i16> @llvm.abs.v128i16(<128 x i16> %op1, i1 false)
@@ -1421,7 +1467,8 @@ define void @abs_v128i16(ptr %a) vscale_range(2,0) #0 {
define <2 x i32> @abs_v2i32(<2 x i32> %op1) vscale_range(2,0) #0 {
; CHECK-LABEL: abs_v2i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: abs v0.2s, v0.2s
+; CHECK-NEXT: neg v1.2s, v0.2s
+; CHECK-NEXT: smax v0.2s, v0.2s, v1.2s
; CHECK-NEXT: ret
%res = call <2 x i32> @llvm.abs.v2i32(<2 x i32> %op1, i1 false)
ret <2 x i32> %res
@@ -1431,7 +1478,8 @@ define <2 x i32> @abs_v2i32(<2 x i32> %op1) vscale_range(2,0) #0 {
define <4 x i32> @abs_v4i32(<4 x i32> %op1) vscale_range(2,0) #0 {
; CHECK-LABEL: abs_v4i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: abs v0.4s, v0.4s
+; CHECK-NEXT: neg v1.4s, v0.4s
+; CHECK-NEXT: smax v0.4s, v0.4s, v1.4s
; CHECK-NEXT: ret
%res = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %op1, i1 false)
ret <4 x i32> %res
@@ -1442,7 +1490,9 @@ define void @abs_v8i32(ptr %a) vscale_range(2,0) #0 {
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.s, vl8
; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]
-; CHECK-NEXT: abs z0.s, p0/m, z0.s
+; CHECK-NEXT: mov z1.d, z0.d
+; CHECK-NEXT: subr z1.s, z1.s, #0 // =0x0
+; CHECK-NEXT: smax z0.s, p0/m, z0.s, z1.s
; CHECK-NEXT: st1w { z0.s }, p0, [x0]
; CHECK-NEXT: ret
%op1 = load <8 x i32>, ptr %a
@@ -1458,8 +1508,12 @@ define void @abs_v16i32(ptr %a) #0 {
; VBITS_GE_256-NEXT: mov x8, #8 // =0x8
; VBITS_GE_256-NEXT: ld1w { z0.s }, p0/z, [x0, x8, lsl #2]
; VBITS_GE_256-NEXT: ld1w { z1.s }, p0/z, [x0]
-; VBITS_GE_256-NEXT: abs z0.s, p0/m, z0.s
-; VBITS_GE_256-NEXT: abs z1.s, p0/m, z1.s
+; VBITS_GE_256-NEXT: mov z2.d, z0.d
+; VBITS_GE_256-NEXT: mov z3.d, z1.d
+; VBITS_GE_256-NEXT: subr z2.s, z2.s, #0 // =0x0
+; VBITS_GE_256-NEXT: subr z3.s, z3.s, #0 // =0x0
+; VBITS_GE_256-NEXT: smax z0.s, p0/m, z0.s, z2.s
+; VBITS_GE_256-NEXT: smax z1.s, p0/m, z1.s, z3.s
; VBITS_GE_256-NEXT: st1w { z0.s }, p0, [x0, x8, lsl #2]
; VBITS_GE_256-NEXT: st1w { z1.s }, p0, [x0]
; VBITS_GE_256-NEXT: ret
@@ -1468,7 +1522,9 @@ define void @abs_v16i32(ptr %a) #0 {
; VBITS_GE_512: // %bb.0:
; VBITS_GE_512-NEXT: ptrue p0.s, vl16
; VBITS_GE_512-NEXT: ld1w { z0.s }, p0/z, [x0]
-; VBITS_GE_512-NEXT: abs z0.s, p0/m, z0.s
+; VBITS_GE_512-NEXT: mov z1.d, z0.d
+; VBITS_GE_512-NEXT: subr z1.s, z1.s, #0 // =0x0
+; VBITS_GE_512-NEXT: smax z0.s, p0/m, z0.s, z1.s
; VBITS_GE_512-NEXT: st1w { z0.s }, p0, [x0]
; VBITS_GE_512-NEXT: ret
%op1 = load <16 x i32>, ptr %a
@@ -1482,7 +1538,9 @@ define void @abs_v32i32(ptr %a) vscale_range(8,0) #0 {
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.s, vl32
; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]
-; CHECK-NEXT: abs z0.s, p0/m, z0.s
+; CHECK-NEXT: mov z1.d, z0.d
+; CHECK-NEXT: subr z1.s, z1.s, #0 // =0x0
+; CHECK-NEXT: smax z0.s, p0/m, z0.s, z1.s
; CHECK-NEXT: st1w { z0.s }, p0, [x0]
; CHECK-NEXT: ret
%op1 = load <32 x i32>, ptr %a
@@ -1496,7 +1554,9 @@ define void @abs_v64i32(ptr %a) vscale_range(16,0) #0 {
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.s, vl64
; CHECK-NEXT: ld1w { z0.s }, p0/z, [x0]
-; CHECK-NEXT: abs z0.s, p0/m, z0.s
+; CHECK-NEXT: mov z1.d, z0.d
+; CHECK-NEXT: subr z1.s, z1.s, #0 // =0x0
+; CHECK-NEXT: smax z0.s, p0/m, z0.s, z1.s
; CHECK-NEXT: st1w { z0.s }, p0, [x0]
; CHECK-NEXT: ret
%op1 = load <64 x i32>, ptr %a
@@ -1509,7 +1569,11 @@ define void @abs_v64i32(ptr %a) vscale_range(16,0) #0 {
define <1 x i64> @abs_v1i64(<1 x i64> %op1) vscale_range(2,0) #0 {
; CHECK-LABEL: abs_v1i64:
; CHECK: // %bb.0:
-; CHECK-NEXT: abs d0, d0
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $z0
+; CHECK-NEXT: neg d1, d0
+; CHECK-NEXT: ptrue p0.d, vl1
+; CHECK-NEXT: smax z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $z0
; CHECK-NEXT: ret
%res = call <1 x i64> @llvm.abs.v1i64(<1 x i64> %op1, i1 false)
ret <1 x i64> %res
@@ -1519,7 +1583,11 @@ define <1 x i64> @abs_v1i64(<1 x i64> %op1) vscale_range(2,0) #0 {
define <2 x i64> @abs_v2i64(<2 x i64> %op1) vscale_range(2,0) #0 {
; CHECK-LABEL: abs_v2i64:
; CHECK: // %bb.0:
-; CHECK-NEXT: abs v0.2d, v0.2d
+; CHECK-NEXT: // kill: def $q0 killed $q0 def $z0
+; CHECK-NEXT: neg v1.2d, v0.2d
+; CHECK-NEXT: ptrue p0.d, vl2
+; CHECK-NEXT: smax z0.d, p0/m, z0.d, z1.d
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0
; CHECK-NEXT: ret
%res = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %op1, i1 false)
ret <2 x i64> %res
@@ -1530,7 +1598,9 @@ define void @abs_v4i64(ptr %a) vscale_range(2,0) #0 {
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d, vl4
; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
-; CHECK-NEXT: abs z0.d, p0/m, z0.d
+; CHECK-NEXT: mov z1.d, z0.d
+; CHECK-NEXT: subr z1.d, z1.d, #0 // =0x0
+; CHECK-NEXT: smax z0.d, p0/m, z0.d, z1.d
; CHECK-NEXT: st1d { z0.d }, p0, [x0]
; CHECK-NEXT: ret
%op1 = load <4 x i64>, ptr %a
@@ -1546,8 +1616,12 @@ define void @abs_v8i64(ptr %a) #0 {
; VBITS_GE_256-NEXT: mov x8, #4 // =0x4
; VBITS_GE_256-NEXT: ld1d { z0.d }, p0/z, [x0, x8, lsl #3]
; VBITS_GE_256-NEXT: ld1d { z1.d }, p0/z, [x0]
-; VBITS_GE_256-NEXT: abs z0.d, p0/m, z0.d
-; VBITS_GE_256-NEXT: abs z1.d, p0/m, z1.d
+; VBITS_GE_256-NEXT: mov z2.d, z0.d
+; VBITS_GE_256-NEXT: mov z3.d, z1.d
+; VBITS_GE_256-NEXT: subr z2.d, z2.d, #0 // =0x0
+; VBITS_GE_256-NEXT: subr z3.d, z3.d, #0 // =0x0
+; VBITS_GE_256-NEXT: smax z0.d, p0/m, z0.d, z2.d
+; VBITS_GE_256-NEXT: smax z1.d, p0/m, z1.d, z3.d
; VBITS_GE_256-NEXT: st1d { z0.d }, p0, [x0, x8, lsl #3]
; VBITS_GE_256-NEXT: st1d { z1.d }, p0, [x0]
; VBITS_GE_256-NEXT: ret
@@ -1556,7 +1630,9 @@ define void @abs_v8i64(ptr %a) #0 {
; VBITS_GE_512: // %bb.0:
; VBITS_GE_512-NEXT: ptrue p0.d, vl8
; VBITS_GE_512-NEXT: ld1d { z0.d }, p0/z, [x0]
-; VBITS_GE_512-NEXT: abs z0.d, p0/m, z0.d
+; VBITS_GE_512-NEXT: mov z1.d, z0.d
+; VBITS_GE_512-NEXT: subr z1.d, z1.d, #0 // =0x0
+; VBITS_GE_512-NEXT: smax z0.d, p0/m, z0.d, z1.d
; VBITS_GE_512-NEXT: st1d { z0.d }, p0, [x0]
; VBITS_GE_512-NEXT: ret
%op1 = load <8 x i64>, ptr %a
@@ -1570,7 +1646,9 @@ define void @abs_v16i64(ptr %a) vscale_range(8,0) #0 {
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d, vl16
; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
-; CHECK-NEXT: abs z0.d, p0/m, z0.d
+; CHECK-NEXT: mov z1.d, z0.d
+; CHECK-NEXT: subr z1.d, z1.d, #0 // =0x0
+; CHECK-NEXT: smax z0.d, p0/m, z0.d, z1.d
; CHECK-NEXT: st1d { z0.d }, p0, [x0]
; CHECK-NEXT: ret
%op1 = load <16 x i64>, ptr %a
@@ -1584,7 +1662,9 @@ define void @abs_v32i64(ptr %a) vscale_range(16,0) #0 {
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.d, vl32
; CHECK-NEXT: ld1d { z0.d }, p0/z, [x0]
-; CHECK-NEXT: abs z0.d, p0/m, z0.d
+; CHECK-NEXT: mov z1.d, z0.d
+; CHECK-NEXT: subr z1.d, z1.d, #0 // =0x0
+; CHECK-NEXT: smax z0.d, p0/m, z0.d, z1.d
; CHECK-NEXT: st1d { z0.d }, p0, [x0]
; CHECK-NEXT: ret
%op1 = load <32 x i64>, ptr %a
diff --git a/llvm/test/CodeGen/AArch64/sve-fixed-length-optimize-ptrue.ll b/llvm/test/CodeGen/AArch64/sve-fixed-length-optimize-ptrue.ll
index d5aad7670cf7a..4d359ae8a1f40 100644
--- a/llvm/test/CodeGen/AArch64/sve-fixed-length-optimize-ptrue.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fixed-length-optimize-ptrue.ll
@@ -38,7 +38,9 @@ define void @abs_v16i32(ptr %a) #0 {
; CHECK: // %bb.0:
; CHECK-NEXT: ldr z0, [x0]
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: abs z0.s, p0/m, z0.s
+; CHECK-NEXT: mov z1.d, z0.d
+; CHECK-NEXT: subr z1.s, z1.s, #0 // =0x0
+; CHECK-NEXT: smax z0.s, p0/m, z0.s, z1.s
; CHECK-NEXT: str z0, [x0]
; CHECK-NEXT: ret
%op1 = load <16 x i32>, ptr %a
@@ -52,7 +54,9 @@ define void @abs_v8i64(ptr %a) #0 {
; CHECK: // %bb.0:
; CHECK-NEXT: ldr z0, [x0]
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: abs z0.d, p0/m, z0.d
+; CHECK-NEXT: mov z1.d, z0.d
+; CHECK-NEXT: subr z1.d, z1.d, #0 // =0x0
+; CHECK-NEXT: smax z0.d, p0/m, z0.d, z1.d
; CHECK-NEXT: str z0, [x0]
; CHECK-NEXT: ret
%op1 = load <8 x i64>, ptr %a
diff --git a/llvm/test/CodeGen/AArch64/sve-int-arith.ll b/llvm/test/CodeGen/AArch64/sve-int-arith.ll
index c59b1d430ff4f..31d8839a12fc1 100644
--- a/llvm/test/CodeGen/AArch64/sve-int-arith.ll
+++ b/llvm/test/CodeGen/AArch64/sve-int-arith.ll
@@ -102,8 +102,10 @@ define <vscale x 16 x i8> @sub_i8_zero(<vscale x 16 x i8> %a) {
define <vscale x 16 x i8> @abs_nxv16i8(<vscale x 16 x i8> %a) {
; CHECK-LABEL: abs_nxv16i8:
; CHECK: // %bb.0:
+; CHECK-NEXT: mov z1.d, z0.d
; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: abs z0.b, p0/m, z0.b
+; CHECK-NEXT: subr z1.b, z1.b, #0 // =0x0
+; CHECK-NEXT: smax z0.b, p0/m, z0.b, z1.b
; CHECK-NEXT: ret
%res = call <vscale x 16 x i8> @llvm.abs.nxv16i8(<vscale x 16 x i8> %a, i1 false)
ret <vscale x 16 x i8> %res
@@ -112,8 +114,10 @@ define <vscale x 16 x i8> @abs_nxv16i8(<vscale x 16 x i8> %a) {
define <vscale x 8 x i16> @abs_nxv8i16(<vscale x 8 x i16> %a) {
; CHECK-LABEL: abs_nxv8i16:
; CHECK: // %bb.0:
+; CHECK-NEXT: mov z1.d, z0.d
; CHECK-NEXT: ptrue p0.h
-; CHECK-NEXT: abs z0.h, p0/m, z0.h
+; CHECK-NEXT: subr z1.h, z1.h, #0 // =0x0
+; CHECK-NEXT: smax z0.h, p0/m, z0.h, z1.h
; CHECK-NEXT: ret
%res = call <vscale x 8 x i16> @llvm.abs.nxv8i16(<vscale x 8 x i16> %a, i1 false)
ret <vscale x 8 x i16> %res
@@ -122,8 +126,10 @@ define <vscale x 8 x i16> @abs_nxv8i16(<vscale x 8 x i16> %a) {
define <vscale x 4 x i32> @abs_nxv4i32(<vscale x 4 x i32> %a) {
; CHECK-LABEL: abs_nxv4i32:
; CHECK: // %bb.0:
+; CHECK-NEXT: mov z1.d, z0.d
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: abs z0.s, p0/m, z0.s
+; CHECK-NEXT: subr z1.s, z1.s, #0 // =0x0
+; CHECK-NEXT: smax z0.s, p0/m, z0.s, z1.s
; CHECK-NEXT: ret
%res = call <vscale x 4 x i32> @llvm.abs.nxv4i32(<vscale x 4 x i32> %a, i1 false)
ret <vscale x 4 x i32> %res
@@ -132,8 +138,10 @@ define <vscale x 4 x i32> @abs_nxv4i32(<vscale x 4 x i32> %a) {
define <vscale x 2 x i64> @abs_nxv2i64(<vscale x 2 x i64> %a) {
; CHECK-LABEL: abs_nxv2i64:
; CHECK: // %bb.0:
+; CHECK-NEXT: mov z1.d, z0.d
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: abs z0.d, p0/m, z0.d
+; CHECK-NEXT: subr z1.d, z1.d, #0 // =0x0
+; CHECK-NEXT: smax z0.d, p0/m, z0.d, z1.d
; CHECK-NEXT: ret
%res = call <vscale x 2 x i64> @llvm.abs.nxv2i64(<vscale x 2 x i64> %a, i1 false)
ret <vscale x 2 x i64> %res
@@ -143,8 +151,11 @@ define <vscale x 4 x i16> @abs_nxv4i16(<vscale x 4 x i16> %a) {
; CHECK-LABEL: abs_nxv4i16:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: movprfx z1, z0
+; CHECK-NEXT: sxth z1.s, p0/m, z0.s
+; CHECK-NEXT: subr z0.s, z0.s, #0 // =0x0
; CHECK-NEXT: sxth z0.s, p0/m, z0.s
-; CHECK-NEXT: abs z0.s, p0/m, z0.s
+; CHECK-NEXT: smax z0.s, p0/m, z0.s, z1.s
; CHECK-NEXT: ret
%res = call <vscale x 4 x i16> @llvm.abs.nxv4i16(<vscale x 4 x i16> %a, i1 false)
ret <vscale x 4 x i16> %res
@@ -153,9 +164,13 @@ define <vscale x 4 x i16> @abs_nxv4i16(<vscale x 4 x i16> %a) {
define <vscale x 32 x i8> @abs_nxv32i8(<vscale x 32 x i8> %a) {
; CHECK-LABEL: abs_nxv32i8:
; CHECK: // %bb.0:
+; CHECK-NEXT: mov z2.d, z0.d
+; CHECK-NEXT: mov z3.d, z1.d
; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: abs z0.b, p0/m, z0.b
-; CHECK-NEXT: abs z1.b, p0/m, z1.b
+; CHECK-NEXT: subr z2.b, z2.b, #0 // =0x0
+; CHECK-NEXT: subr z3.b, z3.b, #0 // =0x0
+; CHECK-NEXT: smax z0.b, p0/m, z0.b, z2.b
+; CHECK-NEXT: smax z1.b, p0/m, z1.b, z3.b
; CHECK-NEXT: ret
%res = call <vscale x 32 x i8> @llvm.abs.nxv32i8(<vscale x 32 x i8> %a, i1 false)
ret <vscale x 32 x i8> %res
@@ -164,11 +179,19 @@ define <vscale x 32 x i8> @abs_nxv32i8(<vscale x 32 x i8> %a) {
define <vscale x 8 x i64> @abs_nxv8i64(<vscale x 8 x i64> %a) {
; CHECK-LABEL: abs_nxv8i64:
; CHECK: // %bb.0:
+; CHECK-NEXT: mov z4.d, z0.d
+; CHECK-NEXT: mov z5.d, z1.d
+; CHECK-NEXT: mov z6.d, z2.d
+; CHECK-NEXT: mov z7.d, z3.d
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: abs z0.d, p0/m, z0.d
-; CHECK-NEXT: abs z1.d, p0/m, z1.d
-; CHECK-NEXT: abs z2.d, p0/m, z2.d
-; CHECK-NEXT: abs z3.d, p0/m, z3.d
+; CHECK-NEXT: subr z4.d, z4.d, #0 // =0x0
+; CHECK-NEXT: subr z5.d, z5.d, #0 // =0x0
+; CHECK-NEXT: subr z6.d, z6.d, #0 // =0x0
+; CHECK-NEXT: subr z7.d, z7.d, #0 // =0x0
+; CHECK-NEXT: smax z0.d, p0/m, z0.d, z4.d
+; CHECK-NEXT: smax z1.d, p0/m, z1.d, z5.d
+; CHECK-NEXT: smax z2.d, p0/m, z2.d, z6.d
+; CHECK-NEXT: smax z3.d, p0/m, z3.d, z7.d
; CHECK-NEXT: ret
%res = call <vscale x 8 x i64> @llvm.abs.nxv8i64(<vscale x 8 x i64> %a, i1 false)
ret <vscale x 8 x i64> %res
diff --git a/llvm/test/CodeGen/AArch64/sve-merging-unary.ll b/llvm/test/CodeGen/AArch64/sve-merging-unary.ll
index 02c815844f1d6..5816895746775 100644
--- a/llvm/test/CodeGen/AArch64/sve-merging-unary.ll
+++ b/llvm/test/CodeGen/AArch64/sve-merging-unary.ll
@@ -7,7 +7,11 @@ target triple = "aarch64-unknown-linux-gnu"
define <vscale x 16 x i8> @abs_nxv16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
; CHECK-LABEL: abs_nxv16i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: abs z0.b, p0/m, z1.b
+; CHECK-NEXT: mov z2.d, z1.d
+; CHECK-NEXT: ptrue p1.b
+; CHECK-NEXT: subr z2.b, z2.b, #0 // =0x0
+; CHECK-NEXT: smax z1.b, p1/m, z1.b, z2.b
+; CHECK-NEXT: mov z0.b, p0/m, z1.b
; CHECK-NEXT: ret
%b.op = call <vscale x 16 x i8> @llvm.abs.nxv16i8(<vscale x 16 x i8> %b, i1 0)
%res = select <vscale x 16 x i1> %pg, <vscale x 16 x i8> %b.op, <vscale x 16 x i8> %a
@@ -17,7 +21,11 @@ define <vscale x 16 x i8> @abs_nxv16i8(<vscale x 16 x i1> %pg, <vscale x 16 x i8
define <vscale x 8 x i16> @abs_nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16> %a, <vscale x 8 x i16> %b) {
; CHECK-LABEL: abs_nxv8i16:
; CHECK: // %bb.0:
-; CHECK-NEXT: abs z0.h, p0/m, z1.h
+; CHECK-NEXT: mov z2.d, z1.d
+; CHECK-NEXT: ptrue p1.h
+; CHECK-NEXT: subr z2.h, z2.h, #0 // =0x0
+; CHECK-NEXT: smax z1.h, p1/m, z1.h, z2.h
+; CHECK-NEXT: mov z0.h, p0/m, z1.h
; CHECK-NEXT: ret
%b.op = call <vscale x 8 x i16> @llvm.abs.nxv8i16(<vscale x 8 x i16> %b, i1 0)
%res = select <vscale x 8 x i1> %pg, <vscale x 8 x i16> %b.op, <vscale x 8 x i16> %a
@@ -27,7 +35,11 @@ define <vscale x 8 x i16> @abs_nxv8i16(<vscale x 8 x i1> %pg, <vscale x 8 x i16>
define <vscale x 4 x i32> @abs_nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32> %a, <vscale x 4 x i32> %b) {
; CHECK-LABEL: abs_nxv4i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: abs z0.s, p0/m, z1.s
+; CHECK-NEXT: mov z2.d, z1.d
+; CHECK-NEXT: ptrue p1.s
+; CHECK-NEXT: subr z2.s, z2.s, #0 // =0x0
+; CHECK-NEXT: smax z1.s, p1/m, z1.s, z2.s
+; CHECK-NEXT: mov z0.s, p0/m, z1.s
; CHECK-NEXT: ret
%b.op = call <vscale x 4 x i32> @llvm.abs.nxv4i32(<vscale x 4 x i32> %b, i1 0)
%res = select <vscale x 4 x i1> %pg, <vscale x 4 x i32> %b.op, <vscale x 4 x i32> %a
@@ -37,7 +49,11 @@ define <vscale x 4 x i32> @abs_nxv4i32(<vscale x 4 x i1> %pg, <vscale x 4 x i32>
define <vscale x 2 x i64> @abs_nxv2i64(<vscale x 2 x i1> %pg, <vscale x 2 x i64> %a, <vscale x 2 x i64> %b) {
; CHECK-LABEL: abs_nxv2i64:
; CHECK: // %bb.0:
-; CHECK-NEXT: abs z0.d, p0/m, z1.d
+; CHECK-NEXT: mov z2.d, z1.d
+; CHECK-NEXT: ptrue p1.d
+; CHECK-NEXT: subr z2.d, z2.d, #0 // =0x0
+; CHECK-NEXT: smax z1.d, p1/m, z1.d, z2.d
+; CHECK-NEXT: mov z0.d, p0/m, z1.d
; CHECK-NEXT: ret
%b.op = call <vscale x 2 x i64> @llvm.abs.nxv2i64(<vscale x 2 x i64> %b, i1 0)
%res = select <vscale x 2 x i1> %pg, <vscale x 2 x i64> %b.op, <vscale x 2 x i64> %a
@@ -1456,8 +1472,10 @@ declare void @use(<vscale x 16 x i8>,<vscale x 16 x i8>)
define void @abs_nxv16i8_multi_use(<vscale x 16 x i1> %pg, <vscale x 16 x i8> %a, <vscale x 16 x i8> %b) {
; CHECK-LABEL: abs_nxv16i8_multi_use:
; CHECK: // %bb.0:
+; CHECK-NEXT: mov z2.d, z1.d
; CHECK-NEXT: ptrue p1.b
-; CHECK-NEXT: abs z1.b, p1/m, z1.b
+; CHECK-NEXT: subr z2.b, z2.b, #0 // =0x0
+; CHECK-NEXT: smax z1.b, p1/m, z1.b, z2.b
; CHECK-NEXT: mov z0.b, p0/m, z1.b
; CHECK-NEXT: b use
%b.op = tail call <vscale x 16 x i8> @llvm.abs.nxv16i8(<vscale x 16 x i8> %b, i1 0)
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-arith.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-arith.ll
index 28980d3f08579..c835e35d7345a 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-arith.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-arith.ll
@@ -2283,29 +2283,44 @@ define <4 x i8> @abs_v4i8(<4 x i8> %op1) {
; CHECK-LABEL: abs_v4i8:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.h, vl4
+; CHECK-NEXT: movprfx z1, z0
+; CHECK-NEXT: sxtb z1.h, p0/m, z0.h
+; CHECK-NEXT: subr z0.h, z0.h, #0 // =0x0
; CHECK-NEXT: sxtb z0.h, p0/m, z0.h
-; CHECK-NEXT: abs z0.h, p0/m, z0.h
+; CHECK-NEXT: smax z0.h, p0/m, z0.h, z1.h
; CHECK-NEXT: ret
;
; NONEON-NOSVE-LABEL: abs_v4i8:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: str d0, [sp, #-16]!
; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #6]
-; NONEON-NOSVE-NEXT: ldrsb w9, [sp, #4]
-; NONEON-NOSVE-NEXT: ldrsb w10, [sp, #2]
-; NONEON-NOSVE-NEXT: ldrsb w11, [sp]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
-; NONEON-NOSVE-NEXT: cmp w9, #0
-; NONEON-NOSVE-NEXT: strh w8, [sp, #14]
-; NONEON-NOSVE-NEXT: cneg w8, w9, mi
-; NONEON-NOSVE-NEXT: cmp w10, #0
-; NONEON-NOSVE-NEXT: strh w8, [sp, #12]
-; NONEON-NOSVE-NEXT: cneg w8, w10, mi
-; NONEON-NOSVE-NEXT: cmp w11, #0
-; NONEON-NOSVE-NEXT: strh w8, [sp, #10]
-; NONEON-NOSVE-NEXT: cneg w8, w11, mi
+; NONEON-NOSVE-NEXT: ldrh w10, [sp, #6]
+; NONEON-NOSVE-NEXT: ldrh w11, [sp, #4]
+; NONEON-NOSVE-NEXT: ldrh w9, [sp, #2]
+; NONEON-NOSVE-NEXT: ldrh w8, [sp]
+; NONEON-NOSVE-NEXT: neg w14, w10
+; NONEON-NOSVE-NEXT: sxtb w10, w10
+; NONEON-NOSVE-NEXT: neg w15, w11
+; NONEON-NOSVE-NEXT: sxtb w14, w14
+; NONEON-NOSVE-NEXT: sxtb w13, w9
+; NONEON-NOSVE-NEXT: sxtb w11, w11
+; NONEON-NOSVE-NEXT: sxtb w15, w15
+; NONEON-NOSVE-NEXT: neg w9, w9
+; NONEON-NOSVE-NEXT: sxtb w12, w8
+; NONEON-NOSVE-NEXT: cmp w10, w14
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: neg w8, w8
+; NONEON-NOSVE-NEXT: csel w10, w10, w14, gt
+; NONEON-NOSVE-NEXT: cmp w11, w15
+; NONEON-NOSVE-NEXT: sxtb w8, w8
+; NONEON-NOSVE-NEXT: strh w10, [sp, #14]
+; NONEON-NOSVE-NEXT: csel w10, w11, w15, gt
+; NONEON-NOSVE-NEXT: cmp w13, w9
+; NONEON-NOSVE-NEXT: csel w9, w13, w9, gt
+; NONEON-NOSVE-NEXT: cmp w12, w8
+; NONEON-NOSVE-NEXT: strh w10, [sp, #12]
+; NONEON-NOSVE-NEXT: csel w8, w12, w8, gt
+; NONEON-NOSVE-NEXT: strh w9, [sp, #10]
; NONEON-NOSVE-NEXT: strh w8, [sp, #8]
; NONEON-NOSVE-NEXT: ldr d0, [sp, #8]
; NONEON-NOSVE-NEXT: add sp, sp, #16
@@ -2317,8 +2332,10 @@ define <4 x i8> @abs_v4i8(<4 x i8> %op1) {
define <8 x i8> @abs_v8i8(<8 x i8> %op1) {
; CHECK-LABEL: abs_v8i8:
; CHECK: // %bb.0:
+; CHECK-NEXT: mov z1.d, z0.d
; CHECK-NEXT: ptrue p0.b, vl8
-; CHECK-NEXT: abs z0.b, p0/m, z0.b
+; CHECK-NEXT: subr z1.b, z1.b, #0 // =0x0
+; CHECK-NEXT: smax z0.b, p0/m, z0.b, z1.b
; CHECK-NEXT: ret
;
; NONEON-NOSVE-LABEL: abs_v8i8:
@@ -2326,36 +2343,52 @@ define <8 x i8> @abs_v8i8(<8 x i8> %op1) {
; NONEON-NOSVE-NEXT: str d0, [sp, #-16]!
; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 16
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #7]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #15]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #6]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #14]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #5]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #13]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #4]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #12]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #3]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #11]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #2]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #10]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #1]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #9]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #8]
; NONEON-NOSVE-NEXT: ldr d0, [sp, #8]
; NONEON-NOSVE-NEXT: add sp, sp, #16
@@ -2367,8 +2400,10 @@ define <8 x i8> @abs_v8i8(<8 x i8> %op1) {
define <16 x i8> @abs_v16i8(<16 x i8> %op1) {
; CHECK-LABEL: abs_v16i8:
; CHECK: // %bb.0:
+; CHECK-NEXT: mov z1.d, z0.d
; CHECK-NEXT: ptrue p0.b, vl16
-; CHECK-NEXT: abs z0.b, p0/m, z0.b
+; CHECK-NEXT: subr z1.b, z1.b, #0 // =0x0
+; CHECK-NEXT: smax z0.b, p0/m, z0.b, z1.b
; CHECK-NEXT: ret
;
; NONEON-NOSVE-LABEL: abs_v16i8:
@@ -2376,68 +2411,100 @@ define <16 x i8> @abs_v16i8(<16 x i8> %op1) {
; NONEON-NOSVE-NEXT: str q0, [sp, #-32]!
; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 32
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #15]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #31]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #14]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #30]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #13]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #29]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #12]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #28]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #11]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #27]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #10]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #26]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #9]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #25]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #8]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #24]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #7]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #23]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #6]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #22]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #5]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #21]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #4]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #20]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #3]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #19]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #2]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #18]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #1]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #17]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #16]
; NONEON-NOSVE-NEXT: ldr q0, [sp, #16]
; NONEON-NOSVE-NEXT: add sp, sp, #32
@@ -2451,8 +2518,12 @@ define void @abs_v32i8(ptr %a) {
; CHECK: // %bb.0:
; CHECK-NEXT: ldp q0, q1, [x0]
; CHECK-NEXT: ptrue p0.b, vl16
-; CHECK-NEXT: abs z0.b, p0/m, z0.b
-; CHECK-NEXT: abs z1.b, p0/m, z1.b
+; CHECK-NEXT: mov z2.d, z0.d
+; CHECK-NEXT: mov z3.d, z1.d
+; CHECK-NEXT: subr z2.b, z2.b, #0 // =0x0
+; CHECK-NEXT: subr z3.b, z3.b, #0 // =0x0
+; CHECK-NEXT: smax z0.b, p0/m, z0.b, z2.b
+; CHECK-NEXT: smax z1.b, p0/m, z1.b, z3.b
; CHECK-NEXT: stp q0, q1, [x0]
; CHECK-NEXT: ret
;
@@ -2462,132 +2533,196 @@ define void @abs_v32i8(ptr %a) {
; NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #31]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #63]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #30]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #62]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #29]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #61]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #28]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #60]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #27]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #59]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #26]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #58]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #25]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #57]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #24]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #56]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #23]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #55]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #22]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #54]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #21]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #53]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #20]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #52]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #19]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #51]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #18]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #50]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #17]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #49]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #16]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #48]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #15]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #47]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #14]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #46]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #13]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #45]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #12]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #44]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #11]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #43]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #10]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #42]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #9]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #41]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #8]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #40]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #7]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #39]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #6]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #38]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #5]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #37]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #4]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #36]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #3]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #35]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #2]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #34]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp, #1]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #33]
; NONEON-NOSVE-NEXT: ldrsb w8, [sp]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxtb w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strb w8, [sp, #32]
; NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
; NONEON-NOSVE-NEXT: stp q0, q1, [x0]
@@ -2603,20 +2738,28 @@ define <2 x i16> @abs_v2i16(<2 x i16> %op1) {
; CHECK-LABEL: abs_v2i16:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.s, vl2
+; CHECK-NEXT: movprfx z1, z0
+; CHECK-NEXT: sxth z1.s, p0/m, z0.s
+; CHECK-NEXT: subr z0.s, z0.s, #0 // =0x0
; CHECK-NEXT: sxth z0.s, p0/m, z0.s
-; CHECK-NEXT: abs z0.s, p0/m, z0.s
+; CHECK-NEXT: smax z0.s, p0/m, z0.s, z1.s
; CHECK-NEXT: ret
;
; NONEON-NOSVE-LABEL: abs_v2i16:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: str d0, [sp, #-16]!
; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT: ldrsh w8, [sp, #4]
-; NONEON-NOSVE-NEXT: ldrsh w9, [sp]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
-; NONEON-NOSVE-NEXT: cmp w9, #0
-; NONEON-NOSVE-NEXT: cneg w9, w9, mi
+; NONEON-NOSVE-NEXT: ldp w9, w8, [sp]
+; NONEON-NOSVE-NEXT: neg w10, w8
+; NONEON-NOSVE-NEXT: sxth w8, w8
+; NONEON-NOSVE-NEXT: neg w11, w9
+; NONEON-NOSVE-NEXT: sxth w10, w10
+; NONEON-NOSVE-NEXT: sxth w9, w9
+; NONEON-NOSVE-NEXT: sxth w11, w11
+; NONEON-NOSVE-NEXT: cmp w8, w10
+; NONEON-NOSVE-NEXT: csel w8, w8, w10, gt
+; NONEON-NOSVE-NEXT: cmp w9, w11
+; NONEON-NOSVE-NEXT: csel w9, w9, w11, gt
; NONEON-NOSVE-NEXT: stp w9, w8, [sp, #8]
; NONEON-NOSVE-NEXT: ldr d0, [sp, #8]
; NONEON-NOSVE-NEXT: add sp, sp, #16
@@ -2628,8 +2771,10 @@ define <2 x i16> @abs_v2i16(<2 x i16> %op1) {
define <4 x i16> @abs_v4i16(<4 x i16> %op1) {
; CHECK-LABEL: abs_v4i16:
; CHECK: // %bb.0:
+; CHECK-NEXT: mov z1.d, z0.d
; CHECK-NEXT: ptrue p0.h, vl4
-; CHECK-NEXT: abs z0.h, p0/m, z0.h
+; CHECK-NEXT: subr z1.h, z1.h, #0 // =0x0
+; CHECK-NEXT: smax z0.h, p0/m, z0.h, z1.h
; CHECK-NEXT: ret
;
; NONEON-NOSVE-LABEL: abs_v4i16:
@@ -2637,20 +2782,28 @@ define <4 x i16> @abs_v4i16(<4 x i16> %op1) {
; NONEON-NOSVE-NEXT: str d0, [sp, #-16]!
; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 16
; NONEON-NOSVE-NEXT: ldrsh w8, [sp, #6]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxth w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strh w8, [sp, #14]
; NONEON-NOSVE-NEXT: ldrsh w8, [sp, #4]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxth w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strh w8, [sp, #12]
; NONEON-NOSVE-NEXT: ldrsh w8, [sp, #2]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxth w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strh w8, [sp, #10]
; NONEON-NOSVE-NEXT: ldrsh w8, [sp]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxth w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strh w8, [sp, #8]
; NONEON-NOSVE-NEXT: ldr d0, [sp, #8]
; NONEON-NOSVE-NEXT: add sp, sp, #16
@@ -2662,8 +2815,10 @@ define <4 x i16> @abs_v4i16(<4 x i16> %op1) {
define <8 x i16> @abs_v8i16(<8 x i16> %op1) {
; CHECK-LABEL: abs_v8i16:
; CHECK: // %bb.0:
+; CHECK-NEXT: mov z1.d, z0.d
; CHECK-NEXT: ptrue p0.h, vl8
-; CHECK-NEXT: abs z0.h, p0/m, z0.h
+; CHECK-NEXT: subr z1.h, z1.h, #0 // =0x0
+; CHECK-NEXT: smax z0.h, p0/m, z0.h, z1.h
; CHECK-NEXT: ret
;
; NONEON-NOSVE-LABEL: abs_v8i16:
@@ -2671,36 +2826,52 @@ define <8 x i16> @abs_v8i16(<8 x i16> %op1) {
; NONEON-NOSVE-NEXT: str q0, [sp, #-32]!
; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 32
; NONEON-NOSVE-NEXT: ldrsh w8, [sp, #14]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxth w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strh w8, [sp, #30]
; NONEON-NOSVE-NEXT: ldrsh w8, [sp, #12]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxth w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strh w8, [sp, #28]
; NONEON-NOSVE-NEXT: ldrsh w8, [sp, #10]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxth w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strh w8, [sp, #26]
; NONEON-NOSVE-NEXT: ldrsh w8, [sp, #8]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxth w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strh w8, [sp, #24]
; NONEON-NOSVE-NEXT: ldrsh w8, [sp, #6]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxth w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strh w8, [sp, #22]
; NONEON-NOSVE-NEXT: ldrsh w8, [sp, #4]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxth w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strh w8, [sp, #20]
; NONEON-NOSVE-NEXT: ldrsh w8, [sp, #2]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxth w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strh w8, [sp, #18]
; NONEON-NOSVE-NEXT: ldrsh w8, [sp]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxth w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strh w8, [sp, #16]
; NONEON-NOSVE-NEXT: ldr q0, [sp, #16]
; NONEON-NOSVE-NEXT: add sp, sp, #32
@@ -2714,8 +2885,12 @@ define void @abs_v16i16(ptr %a) {
; CHECK: // %bb.0:
; CHECK-NEXT: ldp q0, q1, [x0]
; CHECK-NEXT: ptrue p0.h, vl8
-; CHECK-NEXT: abs z0.h, p0/m, z0.h
-; CHECK-NEXT: abs z1.h, p0/m, z1.h
+; CHECK-NEXT: mov z2.d, z0.d
+; CHECK-NEXT: mov z3.d, z1.d
+; CHECK-NEXT: subr z2.h, z2.h, #0 // =0x0
+; CHECK-NEXT: subr z3.h, z3.h, #0 // =0x0
+; CHECK-NEXT: smax z0.h, p0/m, z0.h, z2.h
+; CHECK-NEXT: smax z1.h, p0/m, z1.h, z3.h
; CHECK-NEXT: stp q0, q1, [x0]
; CHECK-NEXT: ret
;
@@ -2725,68 +2900,100 @@ define void @abs_v16i16(ptr %a) {
; NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
; NONEON-NOSVE-NEXT: ldrsh w8, [sp, #30]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxth w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strh w8, [sp, #62]
; NONEON-NOSVE-NEXT: ldrsh w8, [sp, #28]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxth w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strh w8, [sp, #60]
; NONEON-NOSVE-NEXT: ldrsh w8, [sp, #26]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxth w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strh w8, [sp, #58]
; NONEON-NOSVE-NEXT: ldrsh w8, [sp, #24]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxth w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strh w8, [sp, #56]
; NONEON-NOSVE-NEXT: ldrsh w8, [sp, #22]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxth w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strh w8, [sp, #54]
; NONEON-NOSVE-NEXT: ldrsh w8, [sp, #20]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxth w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strh w8, [sp, #52]
; NONEON-NOSVE-NEXT: ldrsh w8, [sp, #18]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxth w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strh w8, [sp, #50]
; NONEON-NOSVE-NEXT: ldrsh w8, [sp, #16]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxth w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strh w8, [sp, #48]
; NONEON-NOSVE-NEXT: ldrsh w8, [sp, #14]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxth w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strh w8, [sp, #46]
; NONEON-NOSVE-NEXT: ldrsh w8, [sp, #12]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxth w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strh w8, [sp, #44]
; NONEON-NOSVE-NEXT: ldrsh w8, [sp, #10]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxth w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strh w8, [sp, #42]
; NONEON-NOSVE-NEXT: ldrsh w8, [sp, #8]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxth w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strh w8, [sp, #40]
; NONEON-NOSVE-NEXT: ldrsh w8, [sp, #6]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxth w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strh w8, [sp, #38]
; NONEON-NOSVE-NEXT: ldrsh w8, [sp, #4]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxth w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strh w8, [sp, #36]
; NONEON-NOSVE-NEXT: ldrsh w8, [sp, #2]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxth w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strh w8, [sp, #34]
; NONEON-NOSVE-NEXT: ldrsh w8, [sp]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: sxth w9, w9
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: csel w8, w8, w9, gt
; NONEON-NOSVE-NEXT: strh w8, [sp, #32]
; NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
; NONEON-NOSVE-NEXT: stp q0, q1, [x0]
@@ -2801,8 +3008,10 @@ define void @abs_v16i16(ptr %a) {
define <2 x i32> @abs_v2i32(<2 x i32> %op1) {
; CHECK-LABEL: abs_v2i32:
; CHECK: // %bb.0:
+; CHECK-NEXT: mov z1.d, z0.d
; CHECK-NEXT: ptrue p0.s, vl2
-; CHECK-NEXT: abs z0.s, p0/m, z0.s
+; CHECK-NEXT: subr z1.s, z1.s, #0 // =0x0
+; CHECK-NEXT: smax z0.s, p0/m, z0.s, z1.s
; CHECK-NEXT: ret
;
; NONEON-NOSVE-LABEL: abs_v2i32:
@@ -2810,12 +3019,14 @@ define <2 x i32> @abs_v2i32(<2 x i32> %op1) {
; NONEON-NOSVE-NEXT: str d0, [sp, #-16]!
; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 16
; NONEON-NOSVE-NEXT: ldr w8, [sp, #4]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w9, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: cneg w10, w8, le
; NONEON-NOSVE-NEXT: ldr w8, [sp]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #8]
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: cneg w8, w8, le
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #8]
; NONEON-NOSVE-NEXT: ldr d0, [sp, #8]
; NONEON-NOSVE-NEXT: add sp, sp, #16
; NONEON-NOSVE-NEXT: ret
@@ -2826,8 +3037,10 @@ define <2 x i32> @abs_v2i32(<2 x i32> %op1) {
define <4 x i32> @abs_v4i32(<4 x i32> %op1) {
; CHECK-LABEL: abs_v4i32:
; CHECK: // %bb.0:
+; CHECK-NEXT: mov z1.d, z0.d
; CHECK-NEXT: ptrue p0.s, vl4
-; CHECK-NEXT: abs z0.s, p0/m, z0.s
+; CHECK-NEXT: subr z1.s, z1.s, #0 // =0x0
+; CHECK-NEXT: smax z0.s, p0/m, z0.s, z1.s
; CHECK-NEXT: ret
;
; NONEON-NOSVE-LABEL: abs_v4i32:
@@ -2835,19 +3048,23 @@ define <4 x i32> @abs_v4i32(<4 x i32> %op1) {
; NONEON-NOSVE-NEXT: str q0, [sp, #-32]!
; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 32
; NONEON-NOSVE-NEXT: ldr w8, [sp, #12]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w9, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: cneg w10, w8, le
; NONEON-NOSVE-NEXT: ldr w8, [sp, #8]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #24]
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: cneg w8, w8, le
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #24]
; NONEON-NOSVE-NEXT: ldr w8, [sp, #4]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w9, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: cneg w10, w8, le
; NONEON-NOSVE-NEXT: ldr w8, [sp]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #16]
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: cneg w8, w8, le
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #16]
; NONEON-NOSVE-NEXT: ldr q0, [sp, #16]
; NONEON-NOSVE-NEXT: add sp, sp, #32
; NONEON-NOSVE-NEXT: ret
@@ -2860,8 +3077,12 @@ define void @abs_v8i32(ptr %a) {
; CHECK: // %bb.0:
; CHECK-NEXT: ldp q0, q1, [x0]
; CHECK-NEXT: ptrue p0.s, vl4
-; CHECK-NEXT: abs z0.s, p0/m, z0.s
-; CHECK-NEXT: abs z1.s, p0/m, z1.s
+; CHECK-NEXT: mov z2.d, z0.d
+; CHECK-NEXT: mov z3.d, z1.d
+; CHECK-NEXT: subr z2.s, z2.s, #0 // =0x0
+; CHECK-NEXT: subr z3.s, z3.s, #0 // =0x0
+; CHECK-NEXT: smax z0.s, p0/m, z0.s, z2.s
+; CHECK-NEXT: smax z1.s, p0/m, z1.s, z3.s
; CHECK-NEXT: stp q0, q1, [x0]
; CHECK-NEXT: ret
;
@@ -2871,33 +3092,41 @@ define void @abs_v8i32(ptr %a) {
; NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
; NONEON-NOSVE-NEXT: ldr w8, [sp, #28]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w9, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: cneg w10, w8, le
; NONEON-NOSVE-NEXT: ldr w8, [sp, #24]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #56]
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: cneg w8, w8, le
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #56]
; NONEON-NOSVE-NEXT: ldr w8, [sp, #20]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w9, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: cneg w10, w8, le
; NONEON-NOSVE-NEXT: ldr w8, [sp, #16]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #48]
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: cneg w8, w8, le
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #48]
; NONEON-NOSVE-NEXT: ldr w8, [sp, #12]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w9, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: cneg w10, w8, le
; NONEON-NOSVE-NEXT: ldr w8, [sp, #8]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #40]
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: cneg w8, w8, le
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #40]
; NONEON-NOSVE-NEXT: ldr w8, [sp, #4]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w9, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: cneg w10, w8, le
; NONEON-NOSVE-NEXT: ldr w8, [sp]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #32]
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: cneg w8, w8, le
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #32]
; NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
; NONEON-NOSVE-NEXT: stp q0, q1, [x0]
; NONEON-NOSVE-NEXT: add sp, sp, #64
@@ -2911,8 +3140,10 @@ define void @abs_v8i32(ptr %a) {
define <1 x i64> @abs_v1i64(<1 x i64> %op1) {
; CHECK-LABEL: abs_v1i64:
; CHECK: // %bb.0:
+; CHECK-NEXT: mov z1.d, z0.d
; CHECK-NEXT: ptrue p0.d, vl1
-; CHECK-NEXT: abs z0.d, p0/m, z0.d
+; CHECK-NEXT: subr z1.d, z1.d, #0 // =0x0
+; CHECK-NEXT: smax z0.d, p0/m, z0.d, z1.d
; CHECK-NEXT: ret
;
; NONEON-NOSVE-LABEL: abs_v1i64:
@@ -2920,8 +3151,9 @@ define <1 x i64> @abs_v1i64(<1 x i64> %op1) {
; NONEON-NOSVE-NEXT: sub sp, sp, #16
; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 16
; NONEON-NOSVE-NEXT: fmov x8, d0
-; NONEON-NOSVE-NEXT: cmp x8, #0
-; NONEON-NOSVE-NEXT: cneg x8, x8, mi
+; NONEON-NOSVE-NEXT: neg x9, x8
+; NONEON-NOSVE-NEXT: cmp x8, x9
+; NONEON-NOSVE-NEXT: cneg x8, x8, le
; NONEON-NOSVE-NEXT: fmov d0, x8
; NONEON-NOSVE-NEXT: add sp, sp, #16
; NONEON-NOSVE-NEXT: ret
@@ -2932,8 +3164,10 @@ define <1 x i64> @abs_v1i64(<1 x i64> %op1) {
define <2 x i64> @abs_v2i64(<2 x i64> %op1) {
; CHECK-LABEL: abs_v2i64:
; CHECK: // %bb.0:
+; CHECK-NEXT: mov z1.d, z0.d
; CHECK-NEXT: ptrue p0.d, vl2
-; CHECK-NEXT: abs z0.d, p0/m, z0.d
+; CHECK-NEXT: subr z1.d, z1.d, #0 // =0x0
+; CHECK-NEXT: smax z0.d, p0/m, z0.d, z1.d
; CHECK-NEXT: ret
;
; NONEON-NOSVE-LABEL: abs_v2i64:
@@ -2941,12 +3175,14 @@ define <2 x i64> @abs_v2i64(<2 x i64> %op1) {
; NONEON-NOSVE-NEXT: str q0, [sp, #-32]!
; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 32
; NONEON-NOSVE-NEXT: ldr x8, [sp, #8]
-; NONEON-NOSVE-NEXT: cmp x8, #0
-; NONEON-NOSVE-NEXT: cneg x9, x8, mi
+; NONEON-NOSVE-NEXT: neg x9, x8
+; NONEON-NOSVE-NEXT: cmp x8, x9
+; NONEON-NOSVE-NEXT: cneg x10, x8, le
; NONEON-NOSVE-NEXT: ldr x8, [sp]
-; NONEON-NOSVE-NEXT: cmp x8, #0
-; NONEON-NOSVE-NEXT: cneg x8, x8, mi
-; NONEON-NOSVE-NEXT: stp x8, x9, [sp, #16]
+; NONEON-NOSVE-NEXT: neg x9, x8
+; NONEON-NOSVE-NEXT: cmp x8, x9
+; NONEON-NOSVE-NEXT: cneg x8, x8, le
+; NONEON-NOSVE-NEXT: stp x8, x10, [sp, #16]
; NONEON-NOSVE-NEXT: ldr q0, [sp, #16]
; NONEON-NOSVE-NEXT: add sp, sp, #32
; NONEON-NOSVE-NEXT: ret
@@ -2959,8 +3195,12 @@ define void @abs_v4i64(ptr %a) {
; CHECK: // %bb.0:
; CHECK-NEXT: ldp q0, q1, [x0]
; CHECK-NEXT: ptrue p0.d, vl2
-; CHECK-NEXT: abs z0.d, p0/m, z0.d
-; CHECK-NEXT: abs z1.d, p0/m, z1.d
+; CHECK-NEXT: mov z2.d, z0.d
+; CHECK-NEXT: mov z3.d, z1.d
+; CHECK-NEXT: subr z2.d, z2.d, #0 // =0x0
+; CHECK-NEXT: subr z3.d, z3.d, #0 // =0x0
+; CHECK-NEXT: smax z0.d, p0/m, z0.d, z2.d
+; CHECK-NEXT: smax z1.d, p0/m, z1.d, z3.d
; CHECK-NEXT: stp q0, q1, [x0]
; CHECK-NEXT: ret
;
@@ -2970,19 +3210,23 @@ define void @abs_v4i64(ptr %a) {
; NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
; NONEON-NOSVE-NEXT: ldr x8, [sp, #24]
-; NONEON-NOSVE-NEXT: cmp x8, #0
-; NONEON-NOSVE-NEXT: cneg x9, x8, mi
+; NONEON-NOSVE-NEXT: neg x9, x8
+; NONEON-NOSVE-NEXT: cmp x8, x9
+; NONEON-NOSVE-NEXT: cneg x10, x8, le
; NONEON-NOSVE-NEXT: ldr x8, [sp, #16]
-; NONEON-NOSVE-NEXT: cmp x8, #0
-; NONEON-NOSVE-NEXT: cneg x8, x8, mi
-; NONEON-NOSVE-NEXT: stp x8, x9, [sp, #48]
+; NONEON-NOSVE-NEXT: neg x9, x8
+; NONEON-NOSVE-NEXT: cmp x8, x9
+; NONEON-NOSVE-NEXT: cneg x8, x8, le
+; NONEON-NOSVE-NEXT: stp x8, x10, [sp, #48]
; NONEON-NOSVE-NEXT: ldr x8, [sp, #8]
-; NONEON-NOSVE-NEXT: cmp x8, #0
-; NONEON-NOSVE-NEXT: cneg x9, x8, mi
+; NONEON-NOSVE-NEXT: neg x9, x8
+; NONEON-NOSVE-NEXT: cmp x8, x9
+; NONEON-NOSVE-NEXT: cneg x10, x8, le
; NONEON-NOSVE-NEXT: ldr x8, [sp]
-; NONEON-NOSVE-NEXT: cmp x8, #0
-; NONEON-NOSVE-NEXT: cneg x8, x8, mi
-; NONEON-NOSVE-NEXT: stp x8, x9, [sp, #32]
+; NONEON-NOSVE-NEXT: neg x9, x8
+; NONEON-NOSVE-NEXT: cmp x8, x9
+; NONEON-NOSVE-NEXT: cneg x8, x8, le
+; NONEON-NOSVE-NEXT: stp x8, x10, [sp, #32]
; NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
; NONEON-NOSVE-NEXT: stp q0, q1, [x0]
; NONEON-NOSVE-NEXT: add sp, sp, #64
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-optimize-ptrue.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-optimize-ptrue.ll
index 4e72615a9446b..743e73b09f439 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-optimize-ptrue.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-optimize-ptrue.ll
@@ -573,9 +573,11 @@ define void @add_v16i16(ptr %a, ptr %b, ptr %c) {
define void @abs_v2i32(ptr %a) {
; CHECK-LABEL: abs_v2i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.s, vl2
; CHECK-NEXT: ldr d0, [x0]
-; CHECK-NEXT: abs z0.s, p0/m, z0.s
+; CHECK-NEXT: ptrue p0.s, vl2
+; CHECK-NEXT: mov z1.d, z0.d
+; CHECK-NEXT: subr z1.s, z1.s, #0 // =0x0
+; CHECK-NEXT: smax z0.s, p0/m, z0.s, z1.s
; CHECK-NEXT: str d0, [x0]
; CHECK-NEXT: ret
;
@@ -585,12 +587,14 @@ define void @abs_v2i32(ptr %a) {
; NONEON-NOSVE-NEXT: str d0, [sp, #-16]!
; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 16
; NONEON-NOSVE-NEXT: ldr w8, [sp, #4]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w9, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: cneg w10, w8, le
; NONEON-NOSVE-NEXT: ldr w8, [sp]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #8]
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: cneg w8, w8, le
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #8]
; NONEON-NOSVE-NEXT: ldr d0, [sp, #8]
; NONEON-NOSVE-NEXT: str d0, [x0]
; NONEON-NOSVE-NEXT: add sp, sp, #16
@@ -604,9 +608,11 @@ define void @abs_v2i32(ptr %a) {
define void @abs_v4i32(ptr %a) {
; CHECK-LABEL: abs_v4i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.s, vl4
; CHECK-NEXT: ldr q0, [x0]
-; CHECK-NEXT: abs z0.s, p0/m, z0.s
+; CHECK-NEXT: ptrue p0.s, vl4
+; CHECK-NEXT: mov z1.d, z0.d
+; CHECK-NEXT: subr z1.s, z1.s, #0 // =0x0
+; CHECK-NEXT: smax z0.s, p0/m, z0.s, z1.s
; CHECK-NEXT: str q0, [x0]
; CHECK-NEXT: ret
;
@@ -616,19 +622,23 @@ define void @abs_v4i32(ptr %a) {
; NONEON-NOSVE-NEXT: str q0, [sp, #-32]!
; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 32
; NONEON-NOSVE-NEXT: ldr w8, [sp, #12]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w9, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: cneg w10, w8, le
; NONEON-NOSVE-NEXT: ldr w8, [sp, #8]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #24]
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: cneg w8, w8, le
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #24]
; NONEON-NOSVE-NEXT: ldr w8, [sp, #4]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w9, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: cneg w10, w8, le
; NONEON-NOSVE-NEXT: ldr w8, [sp]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #16]
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: cneg w8, w8, le
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #16]
; NONEON-NOSVE-NEXT: ldr q0, [sp, #16]
; NONEON-NOSVE-NEXT: str q0, [x0]
; NONEON-NOSVE-NEXT: add sp, sp, #32
@@ -644,8 +654,12 @@ define void @abs_v8i32(ptr %a) {
; CHECK: // %bb.0:
; CHECK-NEXT: ldp q0, q1, [x0]
; CHECK-NEXT: ptrue p0.s, vl4
-; CHECK-NEXT: abs z0.s, p0/m, z0.s
-; CHECK-NEXT: abs z1.s, p0/m, z1.s
+; CHECK-NEXT: mov z2.d, z0.d
+; CHECK-NEXT: mov z3.d, z1.d
+; CHECK-NEXT: subr z2.s, z2.s, #0 // =0x0
+; CHECK-NEXT: subr z3.s, z3.s, #0 // =0x0
+; CHECK-NEXT: smax z0.s, p0/m, z0.s, z2.s
+; CHECK-NEXT: smax z1.s, p0/m, z1.s, z3.s
; CHECK-NEXT: stp q0, q1, [x0]
; CHECK-NEXT: ret
;
@@ -655,33 +669,41 @@ define void @abs_v8i32(ptr %a) {
; NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
; NONEON-NOSVE-NEXT: ldr w8, [sp, #28]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w9, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: cneg w10, w8, le
; NONEON-NOSVE-NEXT: ldr w8, [sp, #24]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #56]
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: cneg w8, w8, le
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #56]
; NONEON-NOSVE-NEXT: ldr w8, [sp, #20]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w9, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: cneg w10, w8, le
; NONEON-NOSVE-NEXT: ldr w8, [sp, #16]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #48]
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: cneg w8, w8, le
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #48]
; NONEON-NOSVE-NEXT: ldr w8, [sp, #12]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w9, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: cneg w10, w8, le
; NONEON-NOSVE-NEXT: ldr w8, [sp, #8]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #40]
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: cneg w8, w8, le
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #40]
; NONEON-NOSVE-NEXT: ldr w8, [sp, #4]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w9, w8, mi
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: cneg w10, w8, le
; NONEON-NOSVE-NEXT: ldr w8, [sp]
-; NONEON-NOSVE-NEXT: cmp w8, #0
-; NONEON-NOSVE-NEXT: cneg w8, w8, mi
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #32]
+; NONEON-NOSVE-NEXT: neg w9, w8
+; NONEON-NOSVE-NEXT: cmp w8, w9
+; NONEON-NOSVE-NEXT: cneg w8, w8, le
+; NONEON-NOSVE-NEXT: stp w8, w10, [sp, #32]
; NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
; NONEON-NOSVE-NEXT: stp q0, q1, [x0]
; NONEON-NOSVE-NEXT: add sp, sp, #64
@@ -695,9 +717,11 @@ define void @abs_v8i32(ptr %a) {
define void @abs_v2i64(ptr %a) {
; CHECK-LABEL: abs_v2i64:
; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.d, vl2
; CHECK-NEXT: ldr q0, [x0]
-; CHECK-NEXT: abs z0.d, p0/m, z0.d
+; CHECK-NEXT: ptrue p0.d, vl2
+; CHECK-NEXT: mov z1.d, z0.d
+; CHECK-NEXT: subr z1.d, z1.d, #0 // =0x0
+; CHECK-NEXT: smax z0.d, p0/m, z0.d, z1.d
; CHECK-NEXT: str q0, [x0]
; CHECK-NEXT: ret
;
@@ -707,12 +731,14 @@ define void @abs_v2i64(ptr %a) {
; NONEON-NOSVE-NEXT: str q0, [sp, #-32]!
; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 32
; NONEON-NOSVE-NEXT: ldr x8, [sp, #8]
-; NONEON-NOSVE-NEXT: cmp x8, #0
-; NONEON-NOSVE-NEXT: cneg x9, x8, mi
+; NONEON-NOSVE-NEXT: neg x9, x8
+; NONEON-NOSVE-NEXT: cmp x8, x9
+; NONEON-NOSVE-NEXT: cneg x10, x8, le
; NONEON-NOSVE-NEXT: ldr x8, [sp]
-; NONEON-NOSVE-NEXT: cmp x8, #0
-; NONEON-NOSVE-NEXT: cneg x8, x8, mi
-; NONEON-NOSVE-NEXT: stp x8, x9, [sp, #16]
+; NONEON-NOSVE-NEXT: neg x9, x8
+; NONEON-NOSVE-NEXT: cmp x8, x9
+; NONEON-NOSVE-NEXT: cneg x8, x8, le
+; NONEON-NOSVE-NEXT: stp x8, x10, [sp, #16]
; NONEON-NOSVE-NEXT: ldr q0, [sp, #16]
; NONEON-NOSVE-NEXT: str q0, [x0]
; NONEON-NOSVE-NEXT: add sp, sp, #32
@@ -728,8 +754,12 @@ define void @abs_v4i64(ptr %a) {
; CHECK: // %bb.0:
; CHECK-NEXT: ldp q0, q1, [x0]
; CHECK-NEXT: ptrue p0.d, vl2
-; CHECK-NEXT: abs z0.d, p0/m, z0.d
-; CHECK-NEXT: abs z1.d, p0/m, z1.d
+; CHECK-NEXT: mov z2.d, z0.d
+; CHECK-NEXT: mov z3.d, z1.d
+; CHECK-NEXT: subr z2.d, z2.d, #0 // =0x0
+; CHECK-NEXT: subr z3.d, z3.d, #0 // =0x0
+; CHECK-NEXT: smax z0.d, p0/m, z0.d, z2.d
+; CHECK-NEXT: smax z1.d, p0/m, z1.d, z3.d
; CHECK-NEXT: stp q0, q1, [x0]
; CHECK-NEXT: ret
;
@@ -739,19 +769,23 @@ define void @abs_v4i64(ptr %a) {
; NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
; NONEON-NOSVE-NEXT: ldr x8, [sp, #24]
-; NONEON-NOSVE-NEXT: cmp x8, #0
-; NONEON-NOSVE-NEXT: cneg x9, x8, mi
+; NONEON-NOSVE-NEXT: neg x9, x8
+; NONEON-NOSVE-NEXT: cmp x8, x9
+; NONEON-NOSVE-NEXT: cneg x10, x8, le
; NONEON-NOSVE-NEXT: ldr x8, [sp, #16]
-; NONEON-NOSVE-NEXT: cmp x8, #0
-; NONEON-NOSVE-NEXT: cneg x8, x8, mi
-; NONEON-NOSVE-NEXT: stp x8, x9, [sp, #48]
+; NONEON-NOSVE-NEXT: neg x9, x8
+; NONEON-NOSVE-NEXT: cmp x8, x9
+; NONEON-NOSVE-NEXT: cneg x8, x8, le
+; NONEON-NOSVE-NEXT: stp x8, x10, [sp, #48]
; NONEON-NOSVE-NEXT: ldr x8, [sp, #8]
-; NONEON-NOSVE-NEXT: cmp x8, #0
-; NONEON-NOSVE-NEXT: cneg x9, x8, mi
+; NONEON-NOSVE-NEXT: neg x9, x8
+; NONEON-NOSVE-NEXT: cmp x8, x9
+; NONEON-NOSVE-NEXT: cneg x10, x8, le
; NONEON-NOSVE-NEXT: ldr x8, [sp]
-; NONEON-NOSVE-NEXT: cmp x8, #0
-; NONEON-NOSVE-NEXT: cneg x8, x8, mi
-; NONEON-NOSVE-NEXT: stp x8, x9, [sp, #32]
+; NONEON-NOSVE-NEXT: neg x9, x8
+; NONEON-NOSVE-NEXT: cmp x8, x9
+; NONEON-NOSVE-NEXT: cneg x8, x8, le
+; NONEON-NOSVE-NEXT: stp x8, x10, [sp, #32]
; NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
; NONEON-NOSVE-NEXT: stp q0, q1, [x0]
; NONEON-NOSVE-NEXT: add sp, sp, #64
diff --git a/llvm/test/CodeGen/AArch64/sve-unary-movprfx.ll b/llvm/test/CodeGen/AArch64/sve-unary-movprfx.ll
index a6c5abef19ab8..6ef2f93432987 100644
--- a/llvm/test/CodeGen/AArch64/sve-unary-movprfx.ll
+++ b/llvm/test/CodeGen/AArch64/sve-unary-movprfx.ll
@@ -11,9 +11,10 @@ target triple = "aarch64-unknown-linux-gnu"
define <vscale x 16 x i8> @abs_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b) #0 {
; CHECK-LABEL: abs_i8:
; CHECK: // %bb.0:
+; CHECK-NEXT: mov z0.d, z1.d
; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: movprfx z0, z1
-; CHECK-NEXT: abs z0.b, p0/m, z1.b
+; CHECK-NEXT: subr z0.b, z0.b, #0 // =0x0
+; CHECK-NEXT: smax z0.b, p0/m, z0.b, z1.b
; CHECK-NEXT: ret
%ret = tail call <vscale x 16 x i8> @llvm.abs.nxv16i8(<vscale x 16 x i8> %b, i1 0)
ret <vscale x 16 x i8> %ret
@@ -23,8 +24,10 @@ define <vscale x 16 x i8> @abs_i8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
define <vscale x 16 x i8> @abs_i8_dupreg(<vscale x 16 x i8> %a) #0 {
; CHECK-LABEL: abs_i8_dupreg:
; CHECK: // %bb.0:
+; CHECK-NEXT: mov z1.d, z0.d
; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: abs z0.b, p0/m, z0.b
+; CHECK-NEXT: subr z1.b, z1.b, #0 // =0x0
+; CHECK-NEXT: smax z0.b, p0/m, z0.b, z1.b
; CHECK-NEXT: ret
%ret = tail call <vscale x 16 x i8> @llvm.abs.nxv16i8(<vscale x 16 x i8> %a, i1 0)
ret <vscale x 16 x i8> %ret
@@ -72,9 +75,10 @@ define <vscale x 16 x i8> @abs_i8_not_active(<vscale x 16 x i8> %a, <vscale x 16
define <vscale x 8 x i16> @abs_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b) #0 {
; CHECK-LABEL: abs_i16:
; CHECK: // %bb.0:
+; CHECK-NEXT: mov z0.d, z1.d
; CHECK-NEXT: ptrue p0.h
-; CHECK-NEXT: movprfx z0, z1
-; CHECK-NEXT: abs z0.h, p0/m, z1.h
+; CHECK-NEXT: subr z0.h, z0.h, #0 // =0x0
+; CHECK-NEXT: smax z0.h, p0/m, z0.h, z1.h
; CHECK-NEXT: ret
%ret = tail call <vscale x 8 x i16> @llvm.abs.nxv8i16(<vscale x 8 x i16> %b, i1 0)
ret <vscale x 8 x i16> %ret
@@ -83,8 +87,10 @@ define <vscale x 8 x i16> @abs_i16(<vscale x 8 x i16> %a, <vscale x 8 x i16> %b)
define <vscale x 8 x i16> @abs_i16_dupreg(<vscale x 8 x i16> %a) #0 {
; CHECK-LABEL: abs_i16_dupreg:
; CHECK: // %bb.0:
+; CHECK-NEXT: mov z1.d, z0.d
; CHECK-NEXT: ptrue p0.h
-; CHECK-NEXT: abs z0.h, p0/m, z0.h
+; CHECK-NEXT: subr z1.h, z1.h, #0 // =0x0
+; CHECK-NEXT: smax z0.h, p0/m, z0.h, z1.h
; CHECK-NEXT: ret
%ret = tail call <vscale x 8 x i16> @llvm.abs.nxv8i16(<vscale x 8 x i16> %a, i1 0)
ret <vscale x 8 x i16> %ret
@@ -130,9 +136,10 @@ define <vscale x 8 x i16> @abs_i16_not_active(<vscale x 8 x i16> %a, <vscale x 8
define <vscale x 4 x i32> @abs_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b) #0 {
; CHECK-LABEL: abs_i32:
; CHECK: // %bb.0:
+; CHECK-NEXT: mov z0.d, z1.d
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: movprfx z0, z1
-; CHECK-NEXT: abs z0.s, p0/m, z1.s
+; CHECK-NEXT: subr z0.s, z0.s, #0 // =0x0
+; CHECK-NEXT: smax z0.s, p0/m, z0.s, z1.s
; CHECK-NEXT: ret
%ret = tail call <vscale x 4 x i32> @llvm.abs.nxv4i32(<vscale x 4 x i32> %b, i1 0)
ret <vscale x 4 x i32> %ret
@@ -141,8 +148,10 @@ define <vscale x 4 x i32> @abs_i32(<vscale x 4 x i32> %a, <vscale x 4 x i32> %b)
define <vscale x 4 x i32> @abs_i32_dupreg(<vscale x 4 x i32> %a) #0 {
; CHECK-LABEL: abs_i32_dupreg:
; CHECK: // %bb.0:
+; CHECK-NEXT: mov z1.d, z0.d
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: abs z0.s, p0/m, z0.s
+; CHECK-NEXT: subr z1.s, z1.s, #0 // =0x0
+; CHECK-NEXT: smax z0.s, p0/m, z0.s, z1.s
; CHECK-NEXT: ret
%ret = tail call <vscale x 4 x i32> @llvm.abs.nxv4i32(<vscale x 4 x i32> %a, i1 0)
ret <vscale x 4 x i32> %ret
@@ -188,9 +197,10 @@ define <vscale x 4 x i32> @abs_i32_not_active(<vscale x 4 x i32> %a, <vscale x 4
define <vscale x 2 x i64> @abs_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b) #0 {
; CHECK-LABEL: abs_i64:
; CHECK: // %bb.0:
+; CHECK-NEXT: mov z0.d, z1.d
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: movprfx z0, z1
-; CHECK-NEXT: abs z0.d, p0/m, z1.d
+; CHECK-NEXT: subr z0.d, z0.d, #0 // =0x0
+; CHECK-NEXT: smax z0.d, p0/m, z0.d, z1.d
; CHECK-NEXT: ret
%ret = tail call <vscale x 2 x i64> @llvm.abs.nxv2i64(<vscale x 2 x i64> %b, i1 0)
ret <vscale x 2 x i64> %ret
@@ -199,8 +209,10 @@ define <vscale x 2 x i64> @abs_i64(<vscale x 2 x i64> %a, <vscale x 2 x i64> %b)
define <vscale x 2 x i64> @abs_i64_dupreg(<vscale x 2 x i64> %a) #0 {
; CHECK-LABEL: abs_i64_dupreg:
; CHECK: // %bb.0:
+; CHECK-NEXT: mov z1.d, z0.d
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: abs z0.d, p0/m, z0.d
+; CHECK-NEXT: subr z1.d, z1.d, #0 // =0x0
+; CHECK-NEXT: smax z0.d, p0/m, z0.d, z1.d
; CHECK-NEXT: ret
%ret = tail call <vscale x 2 x i64> @llvm.abs.nxv2i64(<vscale x 2 x i64> %a, i1 0)
ret <vscale x 2 x i64> %ret
diff --git a/llvm/test/CodeGen/ARM/iabs.ll b/llvm/test/CodeGen/ARM/iabs.ll
index 758fe7507c0b2..de021e1590da9 100644
--- a/llvm/test/CodeGen/ARM/iabs.ll
+++ b/llvm/test/CodeGen/ARM/iabs.ll
@@ -60,12 +60,14 @@ define void @testcallframe(i32 %a) {
; CHECK-NEXT: push {r11, lr}
; CHECK-NEXT: .pad #8
; CHECK-NEXT: sub sp, sp, #8
-; CHECK-NEXT: cmp r0, #0
; CHECK-NEXT: mov r1, #0
-; CHECK-NEXT: rsbmi r0, r0, #0
; CHECK-NEXT: mov r2, #0
-; CHECK-NEXT: mov r3, #0
; CHECK-NEXT: str r1, [sp]
+; CHECK-NEXT: rsb r1, r0, #0
+; CHECK-NEXT: cmp r0, r1
+; CHECK-NEXT: mov r3, #0
+; CHECK-NEXT: movle r0, r1
+; CHECK-NEXT: mov r1, #0
; CHECK-NEXT: bl callee
; CHECK-NEXT: add sp, sp, #8
; CHECK-NEXT: pop {r11, lr}
diff --git a/llvm/test/CodeGen/NVPTX/int-abs.ll b/llvm/test/CodeGen/NVPTX/int-abs.ll
new file mode 100644
index 0000000000000..1ee9396b66984
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/int-abs.ll
@@ -0,0 +1,120 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_50 | FileCheck %s
+
+target triple = "nvptx64-nvidia-cuda"
+
+
+define i32 @abs_i32_poison(i32 %x) {
+; CHECK-LABEL: abs_i32_poison(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [abs_i32_poison_param_0];
+; CHECK-NEXT: abs.s32 %r2, %r1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r2;
+; CHECK-NEXT: ret;
+ %r = call i32 @llvm.abs.i32(i32 %x, i1 true)
+ ret i32 %r
+}
+
+
+define i32 @abs_i32_no_poison(i32 %x) {
+; CHECK-LABEL: abs_i32_no_poison(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [abs_i32_no_poison_param_0];
+; CHECK-NEXT: neg.s32 %r2, %r1;
+; CHECK-NEXT: max.s32 %r3, %r1, %r2;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %r = call i32 @llvm.abs.i32(i32 %x, i1 false)
+ ret i32 %r
+}
+
+define i16 @abs_i16_poison(i16 %x) {
+; CHECK-LABEL: abs_i16_poison(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<3>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [abs_i16_poison_param_0];
+; CHECK-NEXT: abs.s16 %rs2, %rs1;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs2;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %r = call i16 @llvm.abs.i16(i16 %x, i1 true)
+ ret i16 %r
+}
+
+define i16 @abs_i16_no_poison(i16 %x) {
+; CHECK-LABEL: abs_i16_no_poison(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<4>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [abs_i16_no_poison_param_0];
+; CHECK-NEXT: neg.s16 %rs2, %rs1;
+; CHECK-NEXT: max.s16 %rs3, %rs1, %rs2;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs3;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %r = call i16 @llvm.abs.i16(i16 %x, i1 false)
+ ret i16 %r
+}
+
+define i64 @abs_i64_poison(i64 %x) {
+; CHECK-LABEL: abs_i64_poison(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [abs_i64_poison_param_0];
+; CHECK-NEXT: abs.s64 %rd2, %rd1;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %r = call i64 @llvm.abs.i64(i64 %x, i1 true)
+ ret i64 %r
+}
+
+define i64 @abs_i64_no_poison(i64 %x) {
+; CHECK-LABEL: abs_i64_no_poison(
+; CHECK: {
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b64 %rd1, [abs_i64_no_poison_param_0];
+; CHECK-NEXT: neg.s64 %rd2, %rd1;
+; CHECK-NEXT: max.s64 %rd3, %rd1, %rd2;
+; CHECK-NEXT: st.param.b64 [func_retval0], %rd3;
+; CHECK-NEXT: ret;
+ %r = call i64 @llvm.abs.i64(i64 %x, i1 false)
+ ret i64 %r
+}
+
+define i32 @abs_i32_intmin_no_poison() {
+; CHECK-LABEL: abs_i32_intmin_no_poison(
+; CHECK: {
+; CHECK-EMPTY:
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: st.param.b32 [func_retval0], -2147483648;
+; CHECK-NEXT: ret;
+ %r = call i32 @llvm.abs.i32(i32 -2147483648, i1 false)
+ ret i32 %r
+}
+
+define i32 @abs_i32_intmin_poison() {
+; CHECK-LABEL: abs_i32_intmin_poison(
+; CHECK: {
+; CHECK-EMPTY:
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ret;
+ %r = call i32 @llvm.abs.i32(i32 -2147483648, i1 true)
+ ret i32 %r
+}
\ No newline at end of file
diff --git a/llvm/test/CodeGen/X86/abds-neg.ll b/llvm/test/CodeGen/X86/abds-neg.ll
index d9064c684cb20..4b001dab6f8e9 100644
--- a/llvm/test/CodeGen/X86/abds-neg.ll
+++ b/llvm/test/CodeGen/X86/abds-neg.ll
@@ -9,26 +9,40 @@
define i8 @abd_ext_i8(i8 %a, i8 %b) nounwind {
; X86-LABEL: abd_ext_i8:
; X86: # %bb.0:
+; X86-NEXT: pushl %esi
; X86-NEXT: movsbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %eax, %edx
+; X86-NEXT: sarl $31, %edx
; X86-NEXT: movsbl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ecx, %edx
-; X86-NEXT: subl %eax, %edx
-; X86-NEXT: subl %ecx, %eax
-; X86-NEXT: cmovll %edx, %eax
+; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: sarl $31, %esi
+; X86-NEXT: subl %eax, %ecx
+; X86-NEXT: sbbl %edx, %esi
+; X86-NEXT: xorl %edx, %edx
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: negl %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmpl %ecx, %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmovll %ecx, %eax
; X86-NEXT: negb %al
; X86-NEXT: # kill: def $al killed $al killed $eax
+; X86-NEXT: popl %esi
; X86-NEXT: retl
;
; X64-LABEL: abd_ext_i8:
; X64: # %bb.0:
-; X64-NEXT: movsbl %sil, %eax
-; X64-NEXT: movsbl %dil, %ecx
-; X64-NEXT: movl %ecx, %edx
-; X64-NEXT: subl %eax, %edx
-; X64-NEXT: subl %ecx, %eax
-; X64-NEXT: cmovll %edx, %eax
+; X64-NEXT: # kill: def $esi killed $esi def $rsi
+; X64-NEXT: # kill: def $edi killed $edi def $rdi
+; X64-NEXT: movsbq %dil, %rcx
+; X64-NEXT: movsbq %sil, %rax
+; X64-NEXT: subq %rax, %rcx
+; X64-NEXT: movq %rcx, %rax
+; X64-NEXT: negq %rax
+; X64-NEXT: cmpq %rax, %rcx
+; X64-NEXT: cmovgq %rcx, %rax
; X64-NEXT: negb %al
-; X64-NEXT: # kill: def $al killed $al killed $eax
+; X64-NEXT: # kill: def $al killed $al killed $rax
; X64-NEXT: retq
%aext = sext i8 %a to i64
%bext = sext i8 %b to i64
@@ -42,25 +56,40 @@ define i8 @abd_ext_i8(i8 %a, i8 %b) nounwind {
define i8 @abd_ext_i8_i16(i8 %a, i16 %b) nounwind {
; X86-LABEL: abd_ext_i8_i16:
; X86: # %bb.0:
+; X86-NEXT: pushl %esi
; X86-NEXT: movswl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %eax, %edx
+; X86-NEXT: sarl $31, %edx
; X86-NEXT: movsbl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ecx, %edx
-; X86-NEXT: subl %eax, %edx
-; X86-NEXT: subl %ecx, %eax
-; X86-NEXT: cmovll %edx, %eax
+; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: sarl $31, %esi
+; X86-NEXT: subl %eax, %ecx
+; X86-NEXT: sbbl %edx, %esi
+; X86-NEXT: xorl %edx, %edx
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: negl %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmpl %ecx, %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmovll %ecx, %eax
; X86-NEXT: negb %al
; X86-NEXT: # kill: def $al killed $al killed $eax
+; X86-NEXT: popl %esi
; X86-NEXT: retl
;
; X64-LABEL: abd_ext_i8_i16:
; X64: # %bb.0:
-; X64-NEXT: movsbl %dil, %ecx
-; X64-NEXT: subl %esi, %edi
-; X64-NEXT: movswl %si, %eax
-; X64-NEXT: subl %ecx, %eax
-; X64-NEXT: cmovll %edi, %eax
+; X64-NEXT: # kill: def $esi killed $esi def $rsi
+; X64-NEXT: # kill: def $edi killed $edi def $rdi
+; X64-NEXT: movsbq %dil, %rcx
+; X64-NEXT: movswq %si, %rax
+; X64-NEXT: subq %rax, %rcx
+; X64-NEXT: movq %rcx, %rax
+; X64-NEXT: negq %rax
+; X64-NEXT: cmpq %rax, %rcx
+; X64-NEXT: cmovgq %rcx, %rax
; X64-NEXT: negb %al
-; X64-NEXT: # kill: def $al killed $al killed $eax
+; X64-NEXT: # kill: def $al killed $al killed $rax
; X64-NEXT: retq
%aext = sext i8 %a to i64
%bext = sext i16 %b to i64
@@ -107,24 +136,40 @@ define i8 @abd_ext_i8_undef(i8 %a, i8 %b) nounwind {
define i16 @abd_ext_i16(i16 %a, i16 %b) nounwind {
; X86-LABEL: abd_ext_i16:
; X86: # %bb.0:
+; X86-NEXT: pushl %esi
; X86-NEXT: movswl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %eax, %edx
+; X86-NEXT: sarl $31, %edx
; X86-NEXT: movswl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ecx, %edx
-; X86-NEXT: subl %eax, %edx
-; X86-NEXT: subl %ecx, %eax
-; X86-NEXT: cmovgel %edx, %eax
+; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: sarl $31, %esi
+; X86-NEXT: subl %eax, %ecx
+; X86-NEXT: sbbl %edx, %esi
+; X86-NEXT: xorl %edx, %edx
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: negl %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmpl %ecx, %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmovll %ecx, %eax
+; X86-NEXT: negl %eax
; X86-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-NEXT: popl %esi
; X86-NEXT: retl
;
; X64-LABEL: abd_ext_i16:
; X64: # %bb.0:
-; X64-NEXT: movswl %di, %ecx
-; X64-NEXT: subl %esi, %edi
-; X64-NEXT: movswl %si, %eax
-; X64-NEXT: subl %ecx, %eax
-; X64-NEXT: cmovll %edi, %eax
+; X64-NEXT: # kill: def $esi killed $esi def $rsi
+; X64-NEXT: # kill: def $edi killed $edi def $rdi
+; X64-NEXT: movswq %di, %rcx
+; X64-NEXT: movswq %si, %rax
+; X64-NEXT: subq %rax, %rcx
+; X64-NEXT: movq %rcx, %rax
+; X64-NEXT: negq %rax
+; X64-NEXT: cmpq %rax, %rcx
+; X64-NEXT: cmovgq %rcx, %rax
; X64-NEXT: negl %eax
-; X64-NEXT: # kill: def $ax killed $ax killed $eax
+; X64-NEXT: # kill: def $ax killed $ax killed $rax
; X64-NEXT: retq
%aext = sext i16 %a to i64
%bext = sext i16 %b to i64
@@ -138,24 +183,39 @@ define i16 @abd_ext_i16(i16 %a, i16 %b) nounwind {
define i16 @abd_ext_i16_i32(i16 %a, i32 %b) nounwind {
; X86-LABEL: abd_ext_i16_i32:
; X86: # %bb.0:
+; X86-NEXT: pushl %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %eax, %edx
+; X86-NEXT: sarl $31, %edx
; X86-NEXT: movswl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ecx, %edx
-; X86-NEXT: subl %eax, %edx
-; X86-NEXT: subl %ecx, %eax
-; X86-NEXT: cmovgel %edx, %eax
+; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: sarl $31, %esi
+; X86-NEXT: subl %eax, %ecx
+; X86-NEXT: sbbl %edx, %esi
+; X86-NEXT: xorl %edx, %edx
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: negl %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmpl %ecx, %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmovll %ecx, %eax
+; X86-NEXT: negl %eax
; X86-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-NEXT: popl %esi
; X86-NEXT: retl
;
; X64-LABEL: abd_ext_i16_i32:
; X64: # %bb.0:
-; X64-NEXT: movswl %di, %ecx
-; X64-NEXT: movl %edi, %eax
-; X64-NEXT: subl %esi, %eax
-; X64-NEXT: subl %ecx, %esi
-; X64-NEXT: cmovgel %esi, %eax
+; X64-NEXT: # kill: def $edi killed $edi def $rdi
+; X64-NEXT: movswq %di, %rcx
+; X64-NEXT: movslq %esi, %rax
+; X64-NEXT: subq %rax, %rcx
+; X64-NEXT: movq %rcx, %rax
+; X64-NEXT: negq %rax
+; X64-NEXT: cmpq %rax, %rcx
+; X64-NEXT: cmovgq %rcx, %rax
; X64-NEXT: negl %eax
-; X64-NEXT: # kill: def $ax killed $ax killed $eax
+; X64-NEXT: # kill: def $ax killed $ax killed $rax
; X64-NEXT: retq
%aext = sext i16 %a to i64
%bext = sext i32 %b to i64
@@ -200,20 +260,37 @@ define i16 @abd_ext_i16_undef(i16 %a, i16 %b) nounwind {
define i32 @abd_ext_i32(i32 %a, i32 %b) nounwind {
; X86-LABEL: abd_ext_i32:
; X86: # %bb.0:
+; X86-NEXT: pushl %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %eax, %edx
+; X86-NEXT: sarl $31, %edx
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ecx, %edx
-; X86-NEXT: subl %eax, %edx
-; X86-NEXT: subl %ecx, %eax
-; X86-NEXT: cmovgel %edx, %eax
+; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: sarl $31, %esi
+; X86-NEXT: subl %eax, %ecx
+; X86-NEXT: sbbl %edx, %esi
+; X86-NEXT: xorl %edx, %edx
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: negl %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmpl %ecx, %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmovll %ecx, %eax
+; X86-NEXT: negl %eax
+; X86-NEXT: popl %esi
; X86-NEXT: retl
;
; X64-LABEL: abd_ext_i32:
; X64: # %bb.0:
-; X64-NEXT: movl %edi, %eax
-; X64-NEXT: subl %esi, %eax
-; X64-NEXT: subl %edi, %esi
-; X64-NEXT: cmovll %esi, %eax
+; X64-NEXT: movslq %edi, %rcx
+; X64-NEXT: movslq %esi, %rax
+; X64-NEXT: subq %rax, %rcx
+; X64-NEXT: movq %rcx, %rax
+; X64-NEXT: negq %rax
+; X64-NEXT: cmpq %rax, %rcx
+; X64-NEXT: cmovgq %rcx, %rax
+; X64-NEXT: negl %eax
+; X64-NEXT: # kill: def $eax killed $eax killed $rax
; X64-NEXT: retq
%aext = sext i32 %a to i64
%bext = sext i32 %b to i64
@@ -227,21 +304,38 @@ define i32 @abd_ext_i32(i32 %a, i32 %b) nounwind {
define i32 @abd_ext_i32_i16(i32 %a, i16 %b) nounwind {
; X86-LABEL: abd_ext_i32_i16:
; X86: # %bb.0:
+; X86-NEXT: pushl %esi
; X86-NEXT: movswl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %eax, %edx
+; X86-NEXT: sarl $31, %edx
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ecx, %edx
-; X86-NEXT: subl %eax, %edx
-; X86-NEXT: subl %ecx, %eax
-; X86-NEXT: cmovgel %edx, %eax
+; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: sarl $31, %esi
+; X86-NEXT: subl %eax, %ecx
+; X86-NEXT: sbbl %edx, %esi
+; X86-NEXT: xorl %edx, %edx
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: negl %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmpl %ecx, %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmovll %ecx, %eax
+; X86-NEXT: negl %eax
+; X86-NEXT: popl %esi
; X86-NEXT: retl
;
; X64-LABEL: abd_ext_i32_i16:
; X64: # %bb.0:
-; X64-NEXT: movswl %si, %eax
-; X64-NEXT: movl %edi, %ecx
-; X64-NEXT: subl %eax, %ecx
-; X64-NEXT: subl %edi, %eax
-; X64-NEXT: cmovgel %ecx, %eax
+; X64-NEXT: # kill: def $esi killed $esi def $rsi
+; X64-NEXT: movslq %edi, %rcx
+; X64-NEXT: movswq %si, %rax
+; X64-NEXT: subq %rax, %rcx
+; X64-NEXT: movq %rcx, %rax
+; X64-NEXT: negq %rax
+; X64-NEXT: cmpq %rax, %rcx
+; X64-NEXT: cmovgq %rcx, %rax
+; X64-NEXT: negl %eax
+; X64-NEXT: # kill: def $eax killed $eax killed $rax
; X64-NEXT: retq
%aext = sext i32 %a to i64
%bext = sext i16 %b to i64
@@ -282,35 +376,66 @@ define i32 @abd_ext_i32_undef(i32 %a, i32 %b) nounwind {
define i64 @abd_ext_i64(i64 %a, i64 %b) nounwind {
; X86-LABEL: abd_ext_i64:
; X86: # %bb.0:
+; X86-NEXT: pushl %ebp
; X86-NEXT: pushl %ebx
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
+; X86-NEXT: subl $8, %esp
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT: movl %edx, %edi
-; X86-NEXT: subl %eax, %edi
-; X86-NEXT: movl %esi, %ebx
-; X86-NEXT: sbbl %ecx, %ebx
-; X86-NEXT: subl %edx, %eax
-; X86-NEXT: sbbl %esi, %ecx
-; X86-NEXT: cmovll %ebx, %ecx
-; X86-NEXT: cmovll %edi, %eax
-; X86-NEXT: xorl %edx, %edx
+; X86-NEXT: movl %eax, %edx
+; X86-NEXT: sarl $31, %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT: movl %ebp, %edi
+; X86-NEXT: sarl $31, %edi
+; X86-NEXT: subl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %eax, %ebp
+; X86-NEXT: movl %edi, %eax
+; X86-NEXT: sbbl %edx, %eax
+; X86-NEXT: movl %eax, (%esp) # 4-byte Spill
+; X86-NEXT: sbbl %edx, %edi
+; X86-NEXT: movl %ebx, %eax
+; X86-NEXT: negl %eax
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: sbbl %ebp, %ecx
+; X86-NEXT: movl $0, %esi
+; X86-NEXT: sbbl (%esp), %esi # 4-byte Folded Reload
+; X86-NEXT: movl $0, %edx
+; X86-NEXT: sbbl %edi, %edx
+; X86-NEXT: cmpl %ebx, %eax
+; X86-NEXT: movl %ecx, %ebx
+; X86-NEXT: sbbl %ebp, %ebx
+; X86-NEXT: sbbl (%esp), %esi # 4-byte Folded Reload
+; X86-NEXT: sbbl %edi, %edx
+; X86-NEXT: cmovll %ebp, %ecx
+; X86-NEXT: cmovll {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
; X86-NEXT: negl %eax
+; X86-NEXT: movl $0, %edx
; X86-NEXT: sbbl %ecx, %edx
+; X86-NEXT: addl $8, %esp
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
; X86-NEXT: popl %ebx
+; X86-NEXT: popl %ebp
; X86-NEXT: retl
;
; X64-LABEL: abd_ext_i64:
; X64: # %bb.0:
+; X64-NEXT: movq %rdi, %rcx
+; X64-NEXT: sarq $63, %rcx
+; X64-NEXT: movq %rsi, %rax
+; X64-NEXT: sarq $63, %rax
+; X64-NEXT: subq %rsi, %rdi
+; X64-NEXT: sbbq %rax, %rcx
+; X64-NEXT: xorl %edx, %edx
; X64-NEXT: movq %rdi, %rax
-; X64-NEXT: subq %rsi, %rax
-; X64-NEXT: subq %rdi, %rsi
-; X64-NEXT: cmovlq %rsi, %rax
+; X64-NEXT: negq %rax
+; X64-NEXT: sbbq %rcx, %rdx
+; X64-NEXT: cmpq %rdi, %rax
+; X64-NEXT: sbbq %rcx, %rdx
+; X64-NEXT: cmovlq %rdi, %rax
+; X64-NEXT: negq %rax
; X64-NEXT: retq
%aext = sext i64 %a to i128
%bext = sext i64 %b to i128
@@ -372,43 +497,88 @@ define i128 @abd_ext_i128(i128 %a, i128 %b) nounwind {
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
; X86-NEXT: andl $-16, %esp
-; X86-NEXT: subl $16, %esp
-; X86-NEXT: movl 40(%ebp), %ecx
-; X86-NEXT: movl 44(%ebp), %eax
+; X86-NEXT: subl $48, %esp
; X86-NEXT: movl 24(%ebp), %edx
-; X86-NEXT: movl 28(%ebp), %esi
-; X86-NEXT: subl %ecx, %edx
+; X86-NEXT: movl 52(%ebp), %ebx
+; X86-NEXT: movl 36(%ebp), %eax
+; X86-NEXT: movl %eax, %edi
+; X86-NEXT: movl %eax, %esi
+; X86-NEXT: sarl $31, %edi
+; X86-NEXT: movl %ebx, %eax
+; X86-NEXT: sarl $31, %eax
+; X86-NEXT: subl 40(%ebp), %edx
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %esi, %edx
-; X86-NEXT: sbbl %eax, %edx
+; X86-NEXT: movl 28(%ebp), %ecx
+; X86-NEXT: sbbl 44(%ebp), %ecx
+; X86-NEXT: movl %ecx, (%esp) # 4-byte Spill
+; X86-NEXT: movl 32(%ebp), %ecx
+; X86-NEXT: sbbl 48(%ebp), %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %ebx, %esi
+; X86-NEXT: movl %esi, %ebx
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %edi, %esi
+; X86-NEXT: sbbl %eax, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %edi, %esi
+; X86-NEXT: sbbl %eax, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %edi, %esi
+; X86-NEXT: sbbl %eax, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %eax, %edi
+; X86-NEXT: movl %edx, %eax
+; X86-NEXT: negl %eax
+; X86-NEXT: movl $0, %edx
+; X86-NEXT: sbbl (%esp), %edx # 4-byte Folded Reload
; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 48(%ebp), %edx
-; X86-NEXT: movl 32(%ebp), %ebx
-; X86-NEXT: sbbl %edx, %ebx
-; X86-NEXT: movl 52(%ebp), %esi
-; X86-NEXT: movl 36(%ebp), %edi
-; X86-NEXT: sbbl %esi, %edi
-; X86-NEXT: subl 24(%ebp), %ecx
-; X86-NEXT: sbbl 28(%ebp), %eax
-; X86-NEXT: sbbl 32(%ebp), %edx
-; X86-NEXT: sbbl 36(%ebp), %esi
-; X86-NEXT: cmovll %edi, %esi
-; X86-NEXT: cmovll %ebx, %edx
-; X86-NEXT: cmovll {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X86-NEXT: cmovll {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X86-NEXT: xorl %edi, %edi
-; X86-NEXT: negl %ecx
+; X86-NEXT: movl $0, %edx
+; X86-NEXT: sbbl %ecx, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: sbbl %ebx, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, %edx
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X86-NEXT: movl $0, %esi
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
; X86-NEXT: movl $0, %ebx
-; X86-NEXT: sbbl %eax, %ebx
-; X86-NEXT: movl $0, %eax
-; X86-NEXT: sbbl %edx, %eax
-; X86-NEXT: sbbl %esi, %edi
-; X86-NEXT: movl 8(%ebp), %edx
-; X86-NEXT: movl %ecx, (%edx)
-; X86-NEXT: movl %ebx, 4(%edx)
-; X86-NEXT: movl %eax, 8(%edx)
-; X86-NEXT: movl %edi, 12(%edx)
-; X86-NEXT: movl %edx, %eax
+; X86-NEXT: sbbl %edi, %ebx
+; X86-NEXT: cmpl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: sbbl (%esp), %ecx # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NEXT: sbbl %edi, %ebx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: cmovll {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: cmovll {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: cmovll (%esp), %esi # 4-byte Folded Reload
+; X86-NEXT: cmovll {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NEXT: negl %eax
+; X86-NEXT: movl $0, %edx
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: movl $0, %esi
+; X86-NEXT: sbbl %edi, %esi
+; X86-NEXT: movl $0, %edi
+; X86-NEXT: sbbl %ecx, %edi
+; X86-NEXT: movl 8(%ebp), %ecx
+; X86-NEXT: movl %eax, (%ecx)
+; X86-NEXT: movl %edx, 4(%ecx)
+; X86-NEXT: movl %esi, 8(%ecx)
+; X86-NEXT: movl %edi, 12(%ecx)
+; X86-NEXT: movl %ecx, %eax
; X86-NEXT: leal -12(%ebp), %esp
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
@@ -418,17 +588,35 @@ define i128 @abd_ext_i128(i128 %a, i128 %b) nounwind {
;
; X64-LABEL: abd_ext_i128:
; X64: # %bb.0:
-; X64-NEXT: movq %rdi, %rax
-; X64-NEXT: subq %rdx, %rax
+; X64-NEXT: pushq %rbx
; X64-NEXT: movq %rsi, %r8
-; X64-NEXT: sbbq %rcx, %r8
-; X64-NEXT: subq %rdi, %rdx
-; X64-NEXT: sbbq %rsi, %rcx
-; X64-NEXT: cmovlq %r8, %rcx
-; X64-NEXT: cmovgeq %rdx, %rax
+; X64-NEXT: sarq $63, %r8
+; X64-NEXT: movq %rcx, %rax
+; X64-NEXT: sarq $63, %rax
+; X64-NEXT: subq %rdx, %rdi
+; X64-NEXT: sbbq %rcx, %rsi
+; X64-NEXT: movq %r8, %rcx
+; X64-NEXT: sbbq %rax, %rcx
+; X64-NEXT: sbbq %rax, %r8
; X64-NEXT: xorl %edx, %edx
+; X64-NEXT: movq %rdi, %rax
; X64-NEXT: negq %rax
-; X64-NEXT: sbbq %rcx, %rdx
+; X64-NEXT: movl $0, %r9d
+; X64-NEXT: sbbq %rsi, %r9
+; X64-NEXT: movl $0, %r10d
+; X64-NEXT: sbbq %rcx, %r10
+; X64-NEXT: movl $0, %r11d
+; X64-NEXT: sbbq %r8, %r11
+; X64-NEXT: cmpq %rdi, %rax
+; X64-NEXT: movq %r9, %rbx
+; X64-NEXT: sbbq %rsi, %rbx
+; X64-NEXT: sbbq %rcx, %r10
+; X64-NEXT: sbbq %r8, %r11
+; X64-NEXT: cmovlq %rsi, %r9
+; X64-NEXT: cmovlq %rdi, %rax
+; X64-NEXT: negq %rax
+; X64-NEXT: sbbq %r9, %rdx
+; X64-NEXT: popq %rbx
; X64-NEXT: retq
%aext = sext i128 %a to i256
%bext = sext i128 %b to i256
@@ -931,19 +1119,27 @@ define i8 @abd_subnsw_i8(i8 %a, i8 %b) nounwind {
; X86: # %bb.0:
; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: subb {{[0-9]+}}(%esp), %cl
-; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: sarb $7, %al
-; X86-NEXT: xorb %al, %cl
-; X86-NEXT: subb %cl, %al
+; X86-NEXT: movzbl %cl, %edx
+; X86-NEXT: movb %cl, %ch
+; X86-NEXT: negb %ch
+; X86-NEXT: movzbl %ch, %eax
+; X86-NEXT: cmpb %ch, %cl
+; X86-NEXT: cmovgl %edx, %eax
+; X86-NEXT: negb %al
+; X86-NEXT: # kill: def $al killed $al killed $eax
; X86-NEXT: retl
;
; X64-LABEL: abd_subnsw_i8:
; X64: # %bb.0:
; X64-NEXT: subb %sil, %dil
-; X64-NEXT: movl %edi, %eax
-; X64-NEXT: sarb $7, %al
-; X64-NEXT: xorb %al, %dil
-; X64-NEXT: subb %dil, %al
+; X64-NEXT: movzbl %dil, %ecx
+; X64-NEXT: movl %ecx, %eax
+; X64-NEXT: negb %al
+; X64-NEXT: movzbl %al, %eax
+; X64-NEXT: cmpb %al, %cl
+; X64-NEXT: cmovgl %ecx, %eax
+; X64-NEXT: negb %al
+; X64-NEXT: # kill: def $al killed $al killed $eax
; X64-NEXT: retq
%sub = sub nsw i8 %a, %b
%abs = call i8 @llvm.abs.i8(i8 %sub, i1 false)
@@ -982,16 +1178,22 @@ define i16 @abd_subnsw_i16(i16 %a, i16 %b) nounwind {
; X86-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: subw {{[0-9]+}}(%esp), %cx
; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: negw %ax
-; X86-NEXT: cmovnsw %cx, %ax
+; X86-NEXT: negl %eax
+; X86-NEXT: cmpw %ax, %cx
+; X86-NEXT: cmovgl %ecx, %eax
+; X86-NEXT: negl %eax
+; X86-NEXT: # kill: def $ax killed $ax killed $eax
; X86-NEXT: retl
;
; X64-LABEL: abd_subnsw_i16:
; X64: # %bb.0:
; X64-NEXT: subl %esi, %edi
; X64-NEXT: movl %edi, %eax
-; X64-NEXT: negw %ax
-; X64-NEXT: cmovnsw %di, %ax
+; X64-NEXT: negl %eax
+; X64-NEXT: cmpw %ax, %di
+; X64-NEXT: cmovgl %edi, %eax
+; X64-NEXT: negl %eax
+; X64-NEXT: # kill: def $ax killed $ax killed $eax
; X64-NEXT: retq
%sub = sub nsw i16 %a, %b
%abs = call i16 @llvm.abs.i16(i16 %sub, i1 false)
@@ -1025,20 +1227,23 @@ define i16 @abd_subnsw_i16_undef(i16 %a, i16 %b) nounwind {
define i32 @abd_subnsw_i32(i32 %a, i32 %b) nounwind {
; X86-LABEL: abd_subnsw_i32:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ecx, %edx
-; X86-NEXT: subl %eax, %edx
-; X86-NEXT: subl %ecx, %eax
-; X86-NEXT: cmovgel %edx, %eax
+; X86-NEXT: subl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: negl %eax
+; X86-NEXT: cmpl %eax, %ecx
+; X86-NEXT: cmovgl %ecx, %eax
+; X86-NEXT: negl %eax
; X86-NEXT: retl
;
; X64-LABEL: abd_subnsw_i32:
; X64: # %bb.0:
+; X64-NEXT: subl %esi, %edi
; X64-NEXT: movl %edi, %eax
-; X64-NEXT: subl %esi, %eax
-; X64-NEXT: subl %edi, %esi
-; X64-NEXT: cmovll %esi, %eax
+; X64-NEXT: negl %eax
+; X64-NEXT: cmpl %eax, %edi
+; X64-NEXT: cmovgl %edi, %eax
+; X64-NEXT: negl %eax
; X64-NEXT: retq
%sub = sub nsw i32 %a, %b
%abs = call i32 @llvm.abs.i32(i32 %sub, i1 false)
@@ -1073,27 +1278,38 @@ define i32 @abd_subnsw_i32_undef(i32 %a, i32 %b) nounwind {
define i64 @abd_subnsw_i64(i64 %a, i64 %b) nounwind {
; X86-LABEL: abd_subnsw_i64:
; X86: # %bb.0:
+; X86-NEXT: pushl %ebx
+; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
; X86-NEXT: subl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: sbbl {{[0-9]+}}(%esp), %esi
-; X86-NEXT: movl %esi, %edx
-; X86-NEXT: sarl $31, %edx
-; X86-NEXT: xorl %edx, %esi
-; X86-NEXT: xorl %edx, %ecx
-; X86-NEXT: movl %edx, %eax
-; X86-NEXT: subl %ecx, %eax
-; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: xorl %edx, %edx
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: negl %eax
+; X86-NEXT: movl $0, %edi
+; X86-NEXT: sbbl %esi, %edi
+; X86-NEXT: cmpl %ecx, %eax
+; X86-NEXT: movl %edi, %ebx
+; X86-NEXT: sbbl %esi, %ebx
+; X86-NEXT: cmovll %esi, %edi
+; X86-NEXT: cmovll %ecx, %eax
+; X86-NEXT: negl %eax
+; X86-NEXT: sbbl %edi, %edx
; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
+; X86-NEXT: popl %ebx
; X86-NEXT: retl
;
; X64-LABEL: abd_subnsw_i64:
; X64: # %bb.0:
+; X64-NEXT: subq %rsi, %rdi
; X64-NEXT: movq %rdi, %rax
-; X64-NEXT: subq %rsi, %rax
-; X64-NEXT: subq %rdi, %rsi
-; X64-NEXT: cmovlq %rsi, %rax
+; X64-NEXT: negq %rax
+; X64-NEXT: cmpq %rax, %rdi
+; X64-NEXT: cmovgq %rdi, %rax
+; X64-NEXT: negq %rax
; X64-NEXT: retq
%sub = sub nsw i64 %a, %b
%abs = call i64 @llvm.abs.i64(i64 %sub, i1 false)
@@ -1142,32 +1358,48 @@ define i128 @abd_subnsw_i128(i128 %a, i128 %b) nounwind {
; X86-NEXT: pushl %esi
; X86-NEXT: andl $-16, %esp
; X86-NEXT: subl $16, %esp
-; X86-NEXT: movl 32(%ebp), %ecx
-; X86-NEXT: movl 36(%ebp), %eax
-; X86-NEXT: movl 24(%ebp), %edi
-; X86-NEXT: movl 28(%ebp), %edx
-; X86-NEXT: subl 40(%ebp), %edi
-; X86-NEXT: sbbl 44(%ebp), %edx
-; X86-NEXT: sbbl 48(%ebp), %ecx
-; X86-NEXT: sbbl 52(%ebp), %eax
-; X86-NEXT: movl %eax, %esi
-; X86-NEXT: sarl $31, %esi
-; X86-NEXT: xorl %esi, %eax
-; X86-NEXT: xorl %esi, %ecx
-; X86-NEXT: xorl %esi, %edx
-; X86-NEXT: xorl %esi, %edi
-; X86-NEXT: movl %esi, %ebx
-; X86-NEXT: subl %edi, %ebx
-; X86-NEXT: movl %esi, %edi
-; X86-NEXT: sbbl %edx, %edi
+; X86-NEXT: movl 32(%ebp), %eax
+; X86-NEXT: movl 36(%ebp), %ebx
+; X86-NEXT: movl 24(%ebp), %esi
+; X86-NEXT: movl 28(%ebp), %ecx
+; X86-NEXT: subl 40(%ebp), %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl 44(%ebp), %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl 48(%ebp), %eax
+; X86-NEXT: movl %eax, (%esp) # 4-byte Spill
+; X86-NEXT: sbbl 52(%ebp), %ebx
; X86-NEXT: movl %esi, %edx
-; X86-NEXT: sbbl %ecx, %edx
-; X86-NEXT: sbbl %eax, %esi
+; X86-NEXT: negl %edx
+; X86-NEXT: movl $0, %edi
+; X86-NEXT: sbbl %ecx, %edi
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: sbbl %eax, %ecx
+; X86-NEXT: movl $0, %eax
+; X86-NEXT: sbbl %ebx, %eax
+; X86-NEXT: cmpl %esi, %edx
+; X86-NEXT: movl %edi, %esi
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: sbbl (%esp), %esi # 4-byte Folded Reload
+; X86-NEXT: movl %eax, %esi
+; X86-NEXT: sbbl %ebx, %esi
+; X86-NEXT: cmovll %ebx, %eax
+; X86-NEXT: cmovll (%esp), %ecx # 4-byte Folded Reload
+; X86-NEXT: cmovll {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X86-NEXT: cmovll {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X86-NEXT: negl %edx
+; X86-NEXT: movl $0, %esi
+; X86-NEXT: sbbl %edi, %esi
+; X86-NEXT: movl $0, %edi
+; X86-NEXT: sbbl %ecx, %edi
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: sbbl %eax, %ecx
; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: movl %ebx, (%eax)
-; X86-NEXT: movl %edi, 4(%eax)
-; X86-NEXT: movl %edx, 8(%eax)
-; X86-NEXT: movl %esi, 12(%eax)
+; X86-NEXT: movl %edx, (%eax)
+; X86-NEXT: movl %esi, 4(%eax)
+; X86-NEXT: movl %edi, 8(%eax)
+; X86-NEXT: movl %ecx, 12(%eax)
; X86-NEXT: leal -12(%ebp), %esp
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
@@ -1179,13 +1411,18 @@ define i128 @abd_subnsw_i128(i128 %a, i128 %b) nounwind {
; X64: # %bb.0:
; X64-NEXT: subq %rdx, %rdi
; X64-NEXT: sbbq %rcx, %rsi
-; X64-NEXT: movq %rsi, %rdx
-; X64-NEXT: sarq $63, %rdx
-; X64-NEXT: xorq %rdx, %rsi
-; X64-NEXT: xorq %rdx, %rdi
-; X64-NEXT: movq %rdx, %rax
-; X64-NEXT: subq %rdi, %rax
-; X64-NEXT: sbbq %rsi, %rdx
+; X64-NEXT: xorl %edx, %edx
+; X64-NEXT: movq %rdi, %rax
+; X64-NEXT: negq %rax
+; X64-NEXT: movl $0, %ecx
+; X64-NEXT: sbbq %rsi, %rcx
+; X64-NEXT: cmpq %rdi, %rax
+; X64-NEXT: movq %rcx, %r8
+; X64-NEXT: sbbq %rsi, %r8
+; X64-NEXT: cmovlq %rsi, %rcx
+; X64-NEXT: cmovlq %rdi, %rax
+; X64-NEXT: negq %rax
+; X64-NEXT: sbbq %rcx, %rdx
; X64-NEXT: retq
%sub = sub nsw i128 %a, %b
%abs = call i128 @llvm.abs.i128(i128 %sub, i1 false)
diff --git a/llvm/test/CodeGen/X86/abds-vector-128.ll b/llvm/test/CodeGen/X86/abds-vector-128.ll
index 148be83892b72..d16f8f3bd29e9 100644
--- a/llvm/test/CodeGen/X86/abds-vector-128.ll
+++ b/llvm/test/CodeGen/X86/abds-vector-128.ll
@@ -12,28 +12,464 @@
define <16 x i8> @abd_ext_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
; SSE2-LABEL: abd_ext_v16i8:
; SSE2: # %bb.0:
-; SSE2-NEXT: movdqa %xmm0, %xmm2
-; SSE2-NEXT: pcmpgtb %xmm1, %xmm2
-; SSE2-NEXT: psubb %xmm1, %xmm0
-; SSE2-NEXT: pxor %xmm2, %xmm0
-; SSE2-NEXT: psubb %xmm0, %xmm2
-; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm9 = xmm9[0],xmm3[0],xmm9[1],xmm3[1],xmm9[2],xmm3[2],xmm9[3],xmm3[3]
+; SSE2-NEXT: psrad $24, %xmm9
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: pxor %xmm4, %xmm4
+; SSE2-NEXT: pcmpgtd %xmm9, %xmm4
+; SSE2-NEXT: movdqa %xmm9, %xmm11
+; SSE2-NEXT: punpckldq {{.*#+}} xmm11 = xmm11[0],xmm4[0],xmm11[1],xmm4[1]
+; SSE2-NEXT: punpckhdq {{.*#+}} xmm9 = xmm9[2],xmm4[2],xmm9[3],xmm4[3]
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm7 = xmm7[4],xmm3[4],xmm7[5],xmm3[5],xmm7[6],xmm3[6],xmm7[7],xmm3[7]
+; SSE2-NEXT: psrad $24, %xmm7
+; SSE2-NEXT: pxor %xmm3, %xmm3
+; SSE2-NEXT: pcmpgtd %xmm7, %xmm3
+; SSE2-NEXT: movdqa %xmm7, %xmm10
+; SSE2-NEXT: punpckldq {{.*#+}} xmm10 = xmm10[0],xmm3[0],xmm10[1],xmm3[1]
+; SSE2-NEXT: punpckhdq {{.*#+}} xmm7 = xmm7[2],xmm3[2],xmm7[3],xmm3[3]
+; SSE2-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3]
+; SSE2-NEXT: psrad $24, %xmm5
+; SSE2-NEXT: pxor %xmm3, %xmm3
+; SSE2-NEXT: pcmpgtd %xmm5, %xmm3
+; SSE2-NEXT: movdqa %xmm5, %xmm6
+; SSE2-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm3[0],xmm6[1],xmm3[1]
+; SSE2-NEXT: punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm3[2],xmm5[3],xmm3[3]
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT: psrad $24, %xmm3
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: pcmpgtd %xmm3, %xmm0
+; SSE2-NEXT: movdqa %xmm3, %xmm4
+; SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1]
+; SSE2-NEXT: punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm0[2],xmm3[3],xmm0[3]
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm8 = xmm8[0],xmm0[0],xmm8[1],xmm0[1],xmm8[2],xmm0[2],xmm8[3],xmm0[3]
+; SSE2-NEXT: psrad $24, %xmm8
+; SSE2-NEXT: pxor %xmm12, %xmm12
+; SSE2-NEXT: pcmpgtd %xmm8, %xmm12
+; SSE2-NEXT: movdqa %xmm8, %xmm13
+; SSE2-NEXT: punpckldq {{.*#+}} xmm13 = xmm13[0],xmm12[0],xmm13[1],xmm12[1]
+; SSE2-NEXT: psubq %xmm13, %xmm11
+; SSE2-NEXT: punpckhdq {{.*#+}} xmm8 = xmm8[2],xmm12[2],xmm8[3],xmm12[3]
+; SSE2-NEXT: psubq %xmm8, %xmm9
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
+; SSE2-NEXT: psrad $24, %xmm0
+; SSE2-NEXT: pxor %xmm8, %xmm8
+; SSE2-NEXT: pcmpgtd %xmm0, %xmm8
+; SSE2-NEXT: movdqa %xmm0, %xmm12
+; SSE2-NEXT: punpckldq {{.*#+}} xmm12 = xmm12[0],xmm8[0],xmm12[1],xmm8[1]
+; SSE2-NEXT: psubq %xmm12, %xmm10
+; SSE2-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm8[2],xmm0[3],xmm8[3]
+; SSE2-NEXT: psubq %xmm0, %xmm7
+; SSE2-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3]
+; SSE2-NEXT: psrad $24, %xmm1
+; SSE2-NEXT: pxor %xmm8, %xmm8
+; SSE2-NEXT: pcmpgtd %xmm1, %xmm8
+; SSE2-NEXT: movdqa %xmm1, %xmm12
+; SSE2-NEXT: punpckldq {{.*#+}} xmm12 = xmm12[0],xmm8[0],xmm12[1],xmm8[1]
+; SSE2-NEXT: psubq %xmm12, %xmm6
+; SSE2-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm8[2],xmm1[3],xmm8[3]
+; SSE2-NEXT: psubq %xmm1, %xmm5
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
+; SSE2-NEXT: psrad $24, %xmm0
+; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: pcmpgtd %xmm0, %xmm1
+; SSE2-NEXT: movdqa %xmm0, %xmm8
+; SSE2-NEXT: punpckldq {{.*#+}} xmm8 = xmm8[0],xmm1[0],xmm8[1],xmm1[1]
+; SSE2-NEXT: psubq %xmm8, %xmm4
+; SSE2-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE2-NEXT: psubq %xmm0, %xmm3
+; SSE2-NEXT: movdqa {{.*#+}} xmm8 = [2147483648,2147483648]
+; SSE2-NEXT: movdqa %xmm11, %xmm0
+; SSE2-NEXT: pxor %xmm8, %xmm0
+; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: psubq %xmm11, %xmm1
+; SSE2-NEXT: movdqa %xmm1, %xmm12
+; SSE2-NEXT: pxor %xmm8, %xmm12
+; SSE2-NEXT: movdqa %xmm0, %xmm13
+; SSE2-NEXT: pcmpgtd %xmm12, %xmm13
+; SSE2-NEXT: pshufd {{.*#+}} xmm14 = xmm13[0,0,2,2]
+; SSE2-NEXT: pcmpeqd %xmm0, %xmm12
+; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm12[1,1,3,3]
+; SSE2-NEXT: pand %xmm14, %xmm12
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm13[1,1,3,3]
+; SSE2-NEXT: por %xmm12, %xmm0
+; SSE2-NEXT: pand %xmm0, %xmm11
+; SSE2-NEXT: pandn %xmm1, %xmm0
+; SSE2-NEXT: por %xmm11, %xmm0
+; SSE2-NEXT: movdqa %xmm9, %xmm1
+; SSE2-NEXT: pxor %xmm8, %xmm1
+; SSE2-NEXT: pxor %xmm11, %xmm11
+; SSE2-NEXT: psubq %xmm9, %xmm11
+; SSE2-NEXT: movdqa %xmm11, %xmm12
+; SSE2-NEXT: pxor %xmm8, %xmm12
+; SSE2-NEXT: movdqa %xmm1, %xmm13
+; SSE2-NEXT: pcmpgtd %xmm12, %xmm13
+; SSE2-NEXT: pshufd {{.*#+}} xmm14 = xmm13[0,0,2,2]
+; SSE2-NEXT: pcmpeqd %xmm1, %xmm12
+; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm12[1,1,3,3]
+; SSE2-NEXT: pand %xmm14, %xmm12
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm13[1,1,3,3]
+; SSE2-NEXT: por %xmm12, %xmm1
+; SSE2-NEXT: pand %xmm1, %xmm9
+; SSE2-NEXT: pandn %xmm11, %xmm1
+; SSE2-NEXT: por %xmm9, %xmm1
+; SSE2-NEXT: movdqa %xmm10, %xmm9
+; SSE2-NEXT: pxor %xmm8, %xmm9
+; SSE2-NEXT: pxor %xmm11, %xmm11
+; SSE2-NEXT: psubq %xmm10, %xmm11
+; SSE2-NEXT: movdqa %xmm11, %xmm12
+; SSE2-NEXT: pxor %xmm8, %xmm12
+; SSE2-NEXT: movdqa %xmm9, %xmm13
+; SSE2-NEXT: pcmpgtd %xmm12, %xmm13
+; SSE2-NEXT: pshufd {{.*#+}} xmm14 = xmm13[0,0,2,2]
+; SSE2-NEXT: pcmpeqd %xmm9, %xmm12
+; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm12[1,1,3,3]
+; SSE2-NEXT: pand %xmm14, %xmm12
+; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm13[1,1,3,3]
+; SSE2-NEXT: por %xmm12, %xmm9
+; SSE2-NEXT: pand %xmm9, %xmm10
+; SSE2-NEXT: pandn %xmm11, %xmm9
+; SSE2-NEXT: por %xmm10, %xmm9
+; SSE2-NEXT: movdqa %xmm7, %xmm10
+; SSE2-NEXT: pxor %xmm8, %xmm10
+; SSE2-NEXT: pxor %xmm11, %xmm11
+; SSE2-NEXT: psubq %xmm7, %xmm11
+; SSE2-NEXT: movdqa %xmm11, %xmm12
+; SSE2-NEXT: pxor %xmm8, %xmm12
+; SSE2-NEXT: movdqa %xmm10, %xmm13
+; SSE2-NEXT: pcmpgtd %xmm12, %xmm13
+; SSE2-NEXT: pshufd {{.*#+}} xmm14 = xmm13[0,0,2,2]
+; SSE2-NEXT: pcmpeqd %xmm10, %xmm12
+; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm12[1,1,3,3]
+; SSE2-NEXT: pand %xmm14, %xmm12
+; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm13[1,1,3,3]
+; SSE2-NEXT: por %xmm12, %xmm10
+; SSE2-NEXT: pand %xmm10, %xmm7
+; SSE2-NEXT: pandn %xmm11, %xmm10
+; SSE2-NEXT: por %xmm7, %xmm10
+; SSE2-NEXT: movdqa %xmm6, %xmm7
+; SSE2-NEXT: pxor %xmm8, %xmm7
+; SSE2-NEXT: pxor %xmm11, %xmm11
+; SSE2-NEXT: psubq %xmm6, %xmm11
+; SSE2-NEXT: movdqa %xmm11, %xmm12
+; SSE2-NEXT: pxor %xmm8, %xmm12
+; SSE2-NEXT: movdqa %xmm7, %xmm13
+; SSE2-NEXT: pcmpgtd %xmm12, %xmm13
+; SSE2-NEXT: pshufd {{.*#+}} xmm14 = xmm13[0,0,2,2]
+; SSE2-NEXT: pcmpeqd %xmm7, %xmm12
+; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm12[1,1,3,3]
+; SSE2-NEXT: pand %xmm14, %xmm12
+; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm13[1,1,3,3]
+; SSE2-NEXT: por %xmm12, %xmm7
+; SSE2-NEXT: pand %xmm7, %xmm6
+; SSE2-NEXT: pandn %xmm11, %xmm7
+; SSE2-NEXT: por %xmm6, %xmm7
+; SSE2-NEXT: movdqa %xmm5, %xmm6
+; SSE2-NEXT: pxor %xmm8, %xmm6
+; SSE2-NEXT: pxor %xmm11, %xmm11
+; SSE2-NEXT: psubq %xmm5, %xmm11
+; SSE2-NEXT: movdqa %xmm11, %xmm12
+; SSE2-NEXT: pxor %xmm8, %xmm12
+; SSE2-NEXT: movdqa %xmm6, %xmm13
+; SSE2-NEXT: pcmpgtd %xmm12, %xmm13
+; SSE2-NEXT: pshufd {{.*#+}} xmm14 = xmm13[0,0,2,2]
+; SSE2-NEXT: pcmpeqd %xmm6, %xmm12
+; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm12[1,1,3,3]
+; SSE2-NEXT: pand %xmm14, %xmm12
+; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm13[1,1,3,3]
+; SSE2-NEXT: por %xmm12, %xmm6
+; SSE2-NEXT: pand %xmm6, %xmm5
+; SSE2-NEXT: pandn %xmm11, %xmm6
+; SSE2-NEXT: por %xmm5, %xmm6
+; SSE2-NEXT: movdqa %xmm4, %xmm5
+; SSE2-NEXT: pxor %xmm8, %xmm5
+; SSE2-NEXT: pxor %xmm11, %xmm11
+; SSE2-NEXT: psubq %xmm4, %xmm11
+; SSE2-NEXT: movdqa %xmm11, %xmm12
+; SSE2-NEXT: pxor %xmm8, %xmm12
+; SSE2-NEXT: movdqa %xmm5, %xmm13
+; SSE2-NEXT: pcmpgtd %xmm12, %xmm13
+; SSE2-NEXT: pshufd {{.*#+}} xmm14 = xmm13[0,0,2,2]
+; SSE2-NEXT: pcmpeqd %xmm5, %xmm12
+; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm12[1,1,3,3]
+; SSE2-NEXT: pand %xmm14, %xmm12
+; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm13[1,1,3,3]
+; SSE2-NEXT: por %xmm12, %xmm5
+; SSE2-NEXT: pand %xmm5, %xmm4
+; SSE2-NEXT: pandn %xmm11, %xmm5
+; SSE2-NEXT: por %xmm4, %xmm5
+; SSE2-NEXT: movdqa %xmm3, %xmm4
+; SSE2-NEXT: pxor %xmm8, %xmm4
+; SSE2-NEXT: psubq %xmm3, %xmm2
+; SSE2-NEXT: pxor %xmm2, %xmm8
+; SSE2-NEXT: movdqa %xmm4, %xmm11
+; SSE2-NEXT: pcmpgtd %xmm8, %xmm11
+; SSE2-NEXT: pshufd {{.*#+}} xmm12 = xmm11[0,0,2,2]
+; SSE2-NEXT: pcmpeqd %xmm4, %xmm8
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm8[1,1,3,3]
+; SSE2-NEXT: pand %xmm12, %xmm4
+; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm11[1,1,3,3]
+; SSE2-NEXT: por %xmm4, %xmm8
+; SSE2-NEXT: pand %xmm8, %xmm3
+; SSE2-NEXT: pandn %xmm2, %xmm8
+; SSE2-NEXT: por %xmm3, %xmm8
+; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0]
+; SSE2-NEXT: pand %xmm2, %xmm8
+; SSE2-NEXT: pand %xmm2, %xmm5
+; SSE2-NEXT: packuswb %xmm8, %xmm5
+; SSE2-NEXT: pand %xmm2, %xmm6
+; SSE2-NEXT: pand %xmm2, %xmm7
+; SSE2-NEXT: packuswb %xmm6, %xmm7
+; SSE2-NEXT: packuswb %xmm5, %xmm7
+; SSE2-NEXT: pand %xmm2, %xmm10
+; SSE2-NEXT: pand %xmm2, %xmm9
+; SSE2-NEXT: packuswb %xmm10, %xmm9
+; SSE2-NEXT: pand %xmm2, %xmm1
+; SSE2-NEXT: pand %xmm2, %xmm0
+; SSE2-NEXT: packuswb %xmm1, %xmm0
+; SSE2-NEXT: packuswb %xmm9, %xmm0
+; SSE2-NEXT: packuswb %xmm7, %xmm0
; SSE2-NEXT: retq
;
; SSE42-LABEL: abd_ext_v16i8:
; SSE42: # %bb.0:
; SSE42-NEXT: movdqa %xmm0, %xmm2
-; SSE42-NEXT: pminsb %xmm1, %xmm2
-; SSE42-NEXT: pmaxsb %xmm1, %xmm0
-; SSE42-NEXT: psubb %xmm2, %xmm0
+; SSE42-NEXT: psrld $16, %xmm2
+; SSE42-NEXT: pmovsxbq %xmm2, %xmm9
+; SSE42-NEXT: pmovsxbq %xmm0, %xmm8
+; SSE42-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,1,1]
+; SSE42-NEXT: pmovsxbq %xmm2, %xmm7
+; SSE42-NEXT: movdqa %xmm0, %xmm2
+; SSE42-NEXT: psrlq $48, %xmm2
+; SSE42-NEXT: pmovsxbq %xmm2, %xmm6
+; SSE42-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
+; SSE42-NEXT: pmovsxbq %xmm2, %xmm5
+; SSE42-NEXT: movdqa %xmm0, %xmm2
+; SSE42-NEXT: psrldq {{.*#+}} xmm2 = xmm2[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE42-NEXT: pmovsxbq %xmm2, %xmm4
+; SSE42-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
+; SSE42-NEXT: pmovsxbq %xmm2, %xmm3
+; SSE42-NEXT: psrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE42-NEXT: pmovsxbq %xmm0, %xmm2
+; SSE42-NEXT: movdqa %xmm1, %xmm0
+; SSE42-NEXT: psrld $16, %xmm0
+; SSE42-NEXT: pmovsxbq %xmm0, %xmm0
+; SSE42-NEXT: psubq %xmm0, %xmm9
+; SSE42-NEXT: pmovsxbq %xmm1, %xmm0
+; SSE42-NEXT: psubq %xmm0, %xmm8
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE42-NEXT: pmovsxbq %xmm0, %xmm0
+; SSE42-NEXT: psubq %xmm0, %xmm7
+; SSE42-NEXT: movdqa %xmm1, %xmm0
+; SSE42-NEXT: psrlq $48, %xmm0
+; SSE42-NEXT: pmovsxbq %xmm0, %xmm0
+; SSE42-NEXT: psubq %xmm0, %xmm6
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; SSE42-NEXT: pmovsxbq %xmm0, %xmm0
+; SSE42-NEXT: psubq %xmm0, %xmm5
+; SSE42-NEXT: movdqa %xmm1, %xmm0
+; SSE42-NEXT: psrldq {{.*#+}} xmm0 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE42-NEXT: pmovsxbq %xmm0, %xmm0
+; SSE42-NEXT: psubq %xmm0, %xmm4
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm1[3,3,3,3]
+; SSE42-NEXT: pmovsxbq %xmm0, %xmm0
+; SSE42-NEXT: psubq %xmm0, %xmm3
+; SSE42-NEXT: psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE42-NEXT: pmovsxbq %xmm1, %xmm0
+; SSE42-NEXT: psubq %xmm0, %xmm2
+; SSE42-NEXT: pxor %xmm10, %xmm10
+; SSE42-NEXT: pxor %xmm1, %xmm1
+; SSE42-NEXT: psubq %xmm9, %xmm1
+; SSE42-NEXT: pmaxsw %xmm9, %xmm1
+; SSE42-NEXT: pxor %xmm0, %xmm0
+; SSE42-NEXT: psubq %xmm8, %xmm0
+; SSE42-NEXT: pmaxsw %xmm8, %xmm0
+; SSE42-NEXT: pxor %xmm8, %xmm8
+; SSE42-NEXT: psubq %xmm7, %xmm8
+; SSE42-NEXT: pmaxsw %xmm7, %xmm8
+; SSE42-NEXT: pxor %xmm7, %xmm7
+; SSE42-NEXT: psubq %xmm6, %xmm7
+; SSE42-NEXT: pmaxsw %xmm6, %xmm7
+; SSE42-NEXT: pxor %xmm6, %xmm6
+; SSE42-NEXT: psubq %xmm5, %xmm6
+; SSE42-NEXT: pmaxsw %xmm5, %xmm6
+; SSE42-NEXT: pxor %xmm5, %xmm5
+; SSE42-NEXT: psubq %xmm4, %xmm5
+; SSE42-NEXT: pmaxsw %xmm4, %xmm5
+; SSE42-NEXT: pxor %xmm4, %xmm4
+; SSE42-NEXT: psubq %xmm3, %xmm4
+; SSE42-NEXT: pmaxsw %xmm3, %xmm4
+; SSE42-NEXT: psubq %xmm2, %xmm10
+; SSE42-NEXT: pmaxsw %xmm2, %xmm10
+; SSE42-NEXT: pmovzxbq {{.*#+}} xmm2 = [255,255]
+; SSE42-NEXT: pand %xmm2, %xmm10
+; SSE42-NEXT: pand %xmm2, %xmm4
+; SSE42-NEXT: packusdw %xmm10, %xmm4
+; SSE42-NEXT: pand %xmm2, %xmm5
+; SSE42-NEXT: pand %xmm2, %xmm6
+; SSE42-NEXT: packusdw %xmm5, %xmm6
+; SSE42-NEXT: packusdw %xmm4, %xmm6
+; SSE42-NEXT: pand %xmm2, %xmm7
+; SSE42-NEXT: pand %xmm2, %xmm8
+; SSE42-NEXT: packusdw %xmm7, %xmm8
+; SSE42-NEXT: pand %xmm2, %xmm0
+; SSE42-NEXT: pand %xmm2, %xmm1
+; SSE42-NEXT: packusdw %xmm1, %xmm0
+; SSE42-NEXT: packusdw %xmm8, %xmm0
+; SSE42-NEXT: packuswb %xmm6, %xmm0
; SSE42-NEXT: retq
;
-; AVX-LABEL: abd_ext_v16i8:
-; AVX: # %bb.0:
-; AVX-NEXT: vpminsb %xmm1, %xmm0, %xmm2
-; AVX-NEXT: vpmaxsb %xmm1, %xmm0, %xmm0
-; AVX-NEXT: vpsubb %xmm2, %xmm0, %xmm0
-; AVX-NEXT: retq
+; AVX1-LABEL: abd_ext_v16i8:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
+; AVX1-NEXT: vpmovsxbw %xmm2, %xmm3
+; AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[1,1,1,1]
+; AVX1-NEXT: vpmovsxwq %xmm4, %xmm4
+; AVX1-NEXT: vpmovsxbq %xmm2, %xmm2
+; AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm3[2,3,2,3]
+; AVX1-NEXT: vpmovsxwq %xmm5, %xmm5
+; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
+; AVX1-NEXT: vpmovsxwq %xmm3, %xmm3
+; AVX1-NEXT: vpmovsxbw %xmm0, %xmm6
+; AVX1-NEXT: vpshufd {{.*#+}} xmm7 = xmm6[2,3,2,3]
+; AVX1-NEXT: vpmovsxwq %xmm7, %xmm7
+; AVX1-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[3,3,3,3]
+; AVX1-NEXT: vpmovsxwq %xmm6, %xmm6
+; AVX1-NEXT: vpsrld $16, %xmm0, %xmm8
+; AVX1-NEXT: vpmovsxbq %xmm8, %xmm8
+; AVX1-NEXT: vpmovsxbq %xmm0, %xmm0
+; AVX1-NEXT: vpshufd {{.*#+}} xmm9 = xmm1[2,3,2,3]
+; AVX1-NEXT: vpmovsxbw %xmm9, %xmm10
+; AVX1-NEXT: vpshufd {{.*#+}} xmm11 = xmm10[1,1,1,1]
+; AVX1-NEXT: vpmovsxwq %xmm11, %xmm11
+; AVX1-NEXT: vpsubq %xmm11, %xmm4, %xmm4
+; AVX1-NEXT: vpmovsxbq %xmm9, %xmm9
+; AVX1-NEXT: vpsubq %xmm9, %xmm2, %xmm2
+; AVX1-NEXT: vpshufd {{.*#+}} xmm9 = xmm10[2,3,2,3]
+; AVX1-NEXT: vpmovsxwq %xmm9, %xmm9
+; AVX1-NEXT: vpsubq %xmm9, %xmm5, %xmm5
+; AVX1-NEXT: vpshufd {{.*#+}} xmm9 = xmm10[3,3,3,3]
+; AVX1-NEXT: vpmovsxwq %xmm9, %xmm9
+; AVX1-NEXT: vpsubq %xmm9, %xmm3, %xmm3
+; AVX1-NEXT: vpmovsxbw %xmm1, %xmm9
+; AVX1-NEXT: vpshufd {{.*#+}} xmm10 = xmm9[2,3,2,3]
+; AVX1-NEXT: vpmovsxwq %xmm10, %xmm10
+; AVX1-NEXT: vpsubq %xmm10, %xmm7, %xmm7
+; AVX1-NEXT: vpshufd {{.*#+}} xmm9 = xmm9[3,3,3,3]
+; AVX1-NEXT: vpmovsxwq %xmm9, %xmm9
+; AVX1-NEXT: vpsubq %xmm9, %xmm6, %xmm6
+; AVX1-NEXT: vpsrld $16, %xmm1, %xmm9
+; AVX1-NEXT: vpmovsxbq %xmm9, %xmm9
+; AVX1-NEXT: vpsubq %xmm9, %xmm8, %xmm8
+; AVX1-NEXT: vpmovsxbq %xmm1, %xmm1
+; AVX1-NEXT: vpsubq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT: vpsubq %xmm4, %xmm1, %xmm9
+; AVX1-NEXT: vpmaxsd %xmm9, %xmm4, %xmm4
+; AVX1-NEXT: vpsubq %xmm2, %xmm1, %xmm9
+; AVX1-NEXT: vpmaxsw %xmm2, %xmm9, %xmm2
+; AVX1-NEXT: vpsubq %xmm5, %xmm1, %xmm9
+; AVX1-NEXT: vpmaxsd %xmm9, %xmm5, %xmm5
+; AVX1-NEXT: vpsubq %xmm3, %xmm1, %xmm9
+; AVX1-NEXT: vpmaxsd %xmm9, %xmm3, %xmm3
+; AVX1-NEXT: vpsubq %xmm7, %xmm1, %xmm9
+; AVX1-NEXT: vpmaxsd %xmm9, %xmm7, %xmm7
+; AVX1-NEXT: vpsubq %xmm6, %xmm1, %xmm9
+; AVX1-NEXT: vpmaxsd %xmm9, %xmm6, %xmm6
+; AVX1-NEXT: vpsubq %xmm8, %xmm1, %xmm9
+; AVX1-NEXT: vpmaxsw %xmm9, %xmm8, %xmm8
+; AVX1-NEXT: vpsubq %xmm0, %xmm1, %xmm1
+; AVX1-NEXT: vpmaxsw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpmovzxbq {{.*#+}} xmm1 = [255,255]
+; AVX1-NEXT: vpand %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpand %xmm1, %xmm8, %xmm8
+; AVX1-NEXT: vpackusdw %xmm8, %xmm0, %xmm0
+; AVX1-NEXT: vpand %xmm1, %xmm6, %xmm6
+; AVX1-NEXT: vpand %xmm1, %xmm7, %xmm7
+; AVX1-NEXT: vpackusdw %xmm6, %xmm7, %xmm6
+; AVX1-NEXT: vpackusdw %xmm6, %xmm0, %xmm0
+; AVX1-NEXT: vpand %xmm1, %xmm3, %xmm3
+; AVX1-NEXT: vpand %xmm1, %xmm5, %xmm5
+; AVX1-NEXT: vpackusdw %xmm3, %xmm5, %xmm3
+; AVX1-NEXT: vpand %xmm1, %xmm2, %xmm2
+; AVX1-NEXT: vpand %xmm1, %xmm4, %xmm1
+; AVX1-NEXT: vpackusdw %xmm1, %xmm2, %xmm1
+; AVX1-NEXT: vpackusdw %xmm3, %xmm1, %xmm1
+; AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: retq
+;
+; AVX2-LABEL: abd_ext_v16i8:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpmovsxbw %xmm0, %ymm2
+; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
+; AVX2-NEXT: vpmovsxwq %xmm3, %ymm3
+; AVX2-NEXT: vpmovsxbq %xmm0, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm2
+; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]
+; AVX2-NEXT: vpmovsxwq %xmm4, %ymm4
+; AVX2-NEXT: vpmovsxwq %xmm2, %ymm2
+; AVX2-NEXT: vpmovsxbw %xmm1, %ymm5
+; AVX2-NEXT: vpshufd {{.*#+}} xmm6 = xmm5[2,3,2,3]
+; AVX2-NEXT: vpmovsxwq %xmm6, %ymm6
+; AVX2-NEXT: vpsubq %ymm6, %ymm3, %ymm3
+; AVX2-NEXT: vpmovsxbq %xmm1, %ymm1
+; AVX2-NEXT: vpsubq %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm5, %xmm1
+; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[2,3,2,3]
+; AVX2-NEXT: vpmovsxwq %xmm5, %ymm5
+; AVX2-NEXT: vpsubq %ymm5, %ymm4, %ymm4
+; AVX2-NEXT: vpmovsxwq %xmm1, %ymm1
+; AVX2-NEXT: vpsubq %ymm1, %ymm2, %ymm1
+; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT: vpsubq %ymm3, %ymm2, %ymm5
+; AVX2-NEXT: vpmaxsw %ymm5, %ymm3, %ymm3
+; AVX2-NEXT: vpsubq %ymm0, %ymm2, %ymm5
+; AVX2-NEXT: vpmaxsw %ymm5, %ymm0, %ymm0
+; AVX2-NEXT: vpsubq %ymm4, %ymm2, %ymm5
+; AVX2-NEXT: vpmaxsw %ymm5, %ymm4, %ymm4
+; AVX2-NEXT: vpsubq %ymm1, %ymm2, %ymm2
+; AVX2-NEXT: vpmaxsw %ymm2, %ymm1, %ymm1
+; AVX2-NEXT: vpbroadcastq {{.*#+}} ymm2 = [255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0]
+; AVX2-NEXT: vpand %ymm2, %ymm1, %ymm1
+; AVX2-NEXT: vpand %ymm2, %ymm4, %ymm4
+; AVX2-NEXT: vpackusdw %ymm4, %ymm1, %ymm1
+; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
+; AVX2-NEXT: vpand %ymm2, %ymm0, %ymm0
+; AVX2-NEXT: vpand %ymm2, %ymm3, %ymm2
+; AVX2-NEXT: vpackusdw %ymm2, %ymm0, %ymm0
+; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT: vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: abd_ext_v16i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpmovsxbw %xmm0, %ymm0
+; AVX512-NEXT: vpmovsxwq %xmm0, %zmm2
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX512-NEXT: vpmovsxwq %xmm0, %zmm0
+; AVX512-NEXT: vpmovsxbw %xmm1, %ymm1
+; AVX512-NEXT: vpmovsxwq %xmm1, %zmm3
+; AVX512-NEXT: vpsubq %zmm3, %zmm2, %zmm2
+; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm1
+; AVX512-NEXT: vpmovsxwq %xmm1, %zmm1
+; AVX512-NEXT: vpsubq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpsubq %zmm2, %zmm1, %zmm3
+; AVX512-NEXT: vpmaxsq %zmm3, %zmm2, %zmm2
+; AVX512-NEXT: vpsubq %zmm0, %zmm1, %zmm1
+; AVX512-NEXT: vpmaxsq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpmovqb %zmm0, %xmm0
+; AVX512-NEXT: vpmovqb %zmm2, %xmm1
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%aext = sext <16 x i8> %a to <16 x i64>
%bext = sext <16 x i8> %b to <16 x i64>
%sub = sub <16 x i64> %aext, %bext
@@ -76,20 +512,232 @@ define <16 x i8> @abd_ext_v16i8_undef(<16 x i8> %a, <16 x i8> %b) nounwind {
}
define <8 x i16> @abd_ext_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
-; SSE-LABEL: abd_ext_v8i16:
-; SSE: # %bb.0:
-; SSE-NEXT: movdqa %xmm0, %xmm2
-; SSE-NEXT: pminsw %xmm1, %xmm2
-; SSE-NEXT: pmaxsw %xmm1, %xmm0
-; SSE-NEXT: psubw %xmm2, %xmm0
-; SSE-NEXT: retq
+; SSE2-LABEL: abd_ext_v8i16:
+; SSE2: # %bb.0:
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3]
+; SSE2-NEXT: psrad $16, %xmm5
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: pxor %xmm3, %xmm3
+; SSE2-NEXT: pcmpgtd %xmm5, %xmm3
+; SSE2-NEXT: movdqa %xmm5, %xmm6
+; SSE2-NEXT: punpckhdq {{.*#+}} xmm6 = xmm6[2],xmm3[2],xmm6[3],xmm3[3]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
+; SSE2-NEXT: psrad $16, %xmm3
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: pcmpgtd %xmm3, %xmm0
+; SSE2-NEXT: movdqa %xmm3, %xmm4
+; SSE2-NEXT: punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm0[2],xmm4[3],xmm0[3]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE2-NEXT: psrad $16, %xmm0
+; SSE2-NEXT: pxor %xmm7, %xmm7
+; SSE2-NEXT: pcmpgtd %xmm0, %xmm7
+; SSE2-NEXT: movdqa %xmm0, %xmm8
+; SSE2-NEXT: punpckhdq {{.*#+}} xmm8 = xmm8[2],xmm7[2],xmm8[3],xmm7[3]
+; SSE2-NEXT: psubq %xmm8, %xmm6
+; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm7[0],xmm0[1],xmm7[1]
+; SSE2-NEXT: psubq %xmm0, %xmm5
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE2-NEXT: psrad $16, %xmm0
+; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: pcmpgtd %xmm0, %xmm1
+; SSE2-NEXT: movdqa %xmm0, %xmm7
+; SSE2-NEXT: punpckhdq {{.*#+}} xmm7 = xmm7[2],xmm1[2],xmm7[3],xmm1[3]
+; SSE2-NEXT: psubq %xmm7, %xmm4
+; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT: psubq %xmm0, %xmm3
+; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [2147483648,2147483648]
+; SSE2-NEXT: movdqa %xmm6, %xmm0
+; SSE2-NEXT: pxor %xmm1, %xmm0
+; SSE2-NEXT: pxor %xmm8, %xmm8
+; SSE2-NEXT: psubq %xmm6, %xmm8
+; SSE2-NEXT: movdqa %xmm8, %xmm7
+; SSE2-NEXT: pxor %xmm1, %xmm7
+; SSE2-NEXT: movdqa %xmm0, %xmm9
+; SSE2-NEXT: pcmpgtd %xmm7, %xmm9
+; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm9[0,0,2,2]
+; SSE2-NEXT: pcmpeqd %xmm0, %xmm7
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm7[1,1,3,3]
+; SSE2-NEXT: pand %xmm10, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm9[1,1,3,3]
+; SSE2-NEXT: por %xmm0, %xmm7
+; SSE2-NEXT: pand %xmm7, %xmm6
+; SSE2-NEXT: pandn %xmm8, %xmm7
+; SSE2-NEXT: por %xmm6, %xmm7
+; SSE2-NEXT: movdqa %xmm5, %xmm0
+; SSE2-NEXT: pxor %xmm1, %xmm0
+; SSE2-NEXT: pxor %xmm6, %xmm6
+; SSE2-NEXT: psubq %xmm5, %xmm6
+; SSE2-NEXT: movdqa %xmm6, %xmm8
+; SSE2-NEXT: pxor %xmm1, %xmm8
+; SSE2-NEXT: movdqa %xmm0, %xmm9
+; SSE2-NEXT: pcmpgtd %xmm8, %xmm9
+; SSE2-NEXT: pshufd {{.*#+}} xmm10 = xmm9[0,0,2,2]
+; SSE2-NEXT: pcmpeqd %xmm0, %xmm8
+; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[1,1,3,3]
+; SSE2-NEXT: pand %xmm10, %xmm8
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm9[1,1,3,3]
+; SSE2-NEXT: por %xmm8, %xmm0
+; SSE2-NEXT: pand %xmm0, %xmm5
+; SSE2-NEXT: pandn %xmm6, %xmm0
+; SSE2-NEXT: por %xmm5, %xmm0
+; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,2],xmm7[0,2]
+; SSE2-NEXT: movdqa %xmm4, %xmm5
+; SSE2-NEXT: pxor %xmm1, %xmm5
+; SSE2-NEXT: pxor %xmm6, %xmm6
+; SSE2-NEXT: psubq %xmm4, %xmm6
+; SSE2-NEXT: movdqa %xmm6, %xmm7
+; SSE2-NEXT: pxor %xmm1, %xmm7
+; SSE2-NEXT: movdqa %xmm5, %xmm8
+; SSE2-NEXT: pcmpgtd %xmm7, %xmm8
+; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm8[0,0,2,2]
+; SSE2-NEXT: pcmpeqd %xmm5, %xmm7
+; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
+; SSE2-NEXT: pand %xmm9, %xmm7
+; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm8[1,1,3,3]
+; SSE2-NEXT: por %xmm7, %xmm5
+; SSE2-NEXT: pand %xmm5, %xmm4
+; SSE2-NEXT: pandn %xmm6, %xmm5
+; SSE2-NEXT: por %xmm4, %xmm5
+; SSE2-NEXT: movdqa %xmm3, %xmm4
+; SSE2-NEXT: pxor %xmm1, %xmm4
+; SSE2-NEXT: psubq %xmm3, %xmm2
+; SSE2-NEXT: pxor %xmm2, %xmm1
+; SSE2-NEXT: movdqa %xmm4, %xmm6
+; SSE2-NEXT: pcmpgtd %xmm1, %xmm6
+; SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm6[0,0,2,2]
+; SSE2-NEXT: pcmpeqd %xmm4, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT: pand %xmm7, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm6[1,1,3,3]
+; SSE2-NEXT: por %xmm1, %xmm4
+; SSE2-NEXT: pand %xmm4, %xmm3
+; SSE2-NEXT: pandn %xmm2, %xmm4
+; SSE2-NEXT: por %xmm3, %xmm4
+; SSE2-NEXT: shufps {{.*#+}} xmm4 = xmm4[0,2],xmm5[0,2]
+; SSE2-NEXT: pslld $16, %xmm4
+; SSE2-NEXT: psrad $16, %xmm4
+; SSE2-NEXT: pslld $16, %xmm0
+; SSE2-NEXT: psrad $16, %xmm0
+; SSE2-NEXT: packssdw %xmm4, %xmm0
+; SSE2-NEXT: retq
;
-; AVX-LABEL: abd_ext_v8i16:
-; AVX: # %bb.0:
-; AVX-NEXT: vpminsw %xmm1, %xmm0, %xmm2
-; AVX-NEXT: vpmaxsw %xmm1, %xmm0, %xmm0
-; AVX-NEXT: vpsubw %xmm2, %xmm0, %xmm0
-; AVX-NEXT: retq
+; SSE42-LABEL: abd_ext_v8i16:
+; SSE42: # %bb.0:
+; SSE42-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,1,1]
+; SSE42-NEXT: pmovsxwq %xmm2, %xmm2
+; SSE42-NEXT: pmovsxwq %xmm0, %xmm3
+; SSE42-NEXT: pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; SSE42-NEXT: pmovsxwq %xmm4, %xmm4
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; SSE42-NEXT: pmovsxwq %xmm0, %xmm5
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE42-NEXT: pmovsxwq %xmm0, %xmm0
+; SSE42-NEXT: psubq %xmm0, %xmm2
+; SSE42-NEXT: pmovsxwq %xmm1, %xmm0
+; SSE42-NEXT: psubq %xmm0, %xmm3
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; SSE42-NEXT: pmovsxwq %xmm0, %xmm0
+; SSE42-NEXT: psubq %xmm0, %xmm4
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm1[3,3,3,3]
+; SSE42-NEXT: pmovsxwq %xmm0, %xmm0
+; SSE42-NEXT: psubq %xmm0, %xmm5
+; SSE42-NEXT: pxor %xmm1, %xmm1
+; SSE42-NEXT: pxor %xmm6, %xmm6
+; SSE42-NEXT: psubq %xmm2, %xmm6
+; SSE42-NEXT: pmaxsd %xmm2, %xmm6
+; SSE42-NEXT: pxor %xmm0, %xmm0
+; SSE42-NEXT: psubq %xmm3, %xmm0
+; SSE42-NEXT: pmaxsd %xmm3, %xmm0
+; SSE42-NEXT: pxor %xmm2, %xmm2
+; SSE42-NEXT: psubq %xmm4, %xmm2
+; SSE42-NEXT: pmaxsd %xmm4, %xmm2
+; SSE42-NEXT: pxor %xmm3, %xmm3
+; SSE42-NEXT: psubq %xmm5, %xmm3
+; SSE42-NEXT: pmaxsd %xmm5, %xmm3
+; SSE42-NEXT: pblendw {{.*#+}} xmm3 = xmm3[0],xmm1[1,2,3],xmm3[4],xmm1[5,6,7]
+; SSE42-NEXT: pblendw {{.*#+}} xmm2 = xmm2[0],xmm1[1,2,3],xmm2[4],xmm1[5,6,7]
+; SSE42-NEXT: packusdw %xmm3, %xmm2
+; SSE42-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3],xmm0[4],xmm1[5,6,7]
+; SSE42-NEXT: pblendw {{.*#+}} xmm6 = xmm6[0],xmm1[1,2,3],xmm6[4],xmm1[5,6,7]
+; SSE42-NEXT: packusdw %xmm6, %xmm0
+; SSE42-NEXT: packusdw %xmm2, %xmm0
+; SSE42-NEXT: retq
+;
+; AVX1-LABEL: abd_ext_v8i16:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[1,1,1,1]
+; AVX1-NEXT: vpmovsxwq %xmm2, %xmm2
+; AVX1-NEXT: vpmovsxwq %xmm0, %xmm3
+; AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; AVX1-NEXT: vpmovsxwq %xmm4, %xmm4
+; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; AVX1-NEXT: vpmovsxwq %xmm0, %xmm0
+; AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[1,1,1,1]
+; AVX1-NEXT: vpmovsxwq %xmm5, %xmm5
+; AVX1-NEXT: vpsubq %xmm5, %xmm2, %xmm2
+; AVX1-NEXT: vpmovsxwq %xmm1, %xmm5
+; AVX1-NEXT: vpsubq %xmm5, %xmm3, %xmm3
+; AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[2,3,2,3]
+; AVX1-NEXT: vpmovsxwq %xmm5, %xmm5
+; AVX1-NEXT: vpsubq %xmm5, %xmm4, %xmm4
+; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
+; AVX1-NEXT: vpmovsxwq %xmm1, %xmm1
+; AVX1-NEXT: vpsubq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT: vpsubq %xmm2, %xmm1, %xmm5
+; AVX1-NEXT: vpmaxsd %xmm5, %xmm2, %xmm2
+; AVX1-NEXT: vpsubq %xmm3, %xmm1, %xmm5
+; AVX1-NEXT: vpmaxsd %xmm5, %xmm3, %xmm3
+; AVX1-NEXT: vpsubq %xmm4, %xmm1, %xmm5
+; AVX1-NEXT: vpmaxsd %xmm5, %xmm4, %xmm4
+; AVX1-NEXT: vpsubq %xmm0, %xmm1, %xmm5
+; AVX1-NEXT: vpmaxsd %xmm5, %xmm0, %xmm0
+; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3],xmm0[4],xmm1[5,6,7]
+; AVX1-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0],xmm1[1,2,3],xmm4[4],xmm1[5,6,7]
+; AVX1-NEXT: vpackusdw %xmm0, %xmm4, %xmm0
+; AVX1-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0],xmm1[1,2,3],xmm3[4],xmm1[5,6,7]
+; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3],xmm2[4],xmm1[5,6,7]
+; AVX1-NEXT: vpackusdw %xmm1, %xmm3, %xmm1
+; AVX1-NEXT: vpackusdw %xmm0, %xmm1, %xmm0
+; AVX1-NEXT: retq
+;
+; AVX2-LABEL: abd_ext_v8i16:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
+; AVX2-NEXT: vpmovsxwq %xmm2, %ymm2
+; AVX2-NEXT: vpmovsxwq %xmm0, %ymm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; AVX2-NEXT: vpmovsxwq %xmm3, %ymm3
+; AVX2-NEXT: vpsubq %ymm3, %ymm2, %ymm2
+; AVX2-NEXT: vpmovsxwq %xmm1, %ymm1
+; AVX2-NEXT: vpsubq %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpsubq %ymm2, %ymm1, %ymm3
+; AVX2-NEXT: vpmaxsd %ymm3, %ymm2, %ymm2
+; AVX2-NEXT: vpsubq %ymm0, %ymm1, %ymm3
+; AVX2-NEXT: vpmaxsd %ymm3, %ymm0, %ymm0
+; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7],ymm0[8],ymm1[9,10,11],ymm0[12],ymm1[13,14,15]
+; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1,2,3],ymm2[4],ymm1[5,6,7],ymm2[8],ymm1[9,10,11],ymm2[12],ymm1[13,14,15]
+; AVX2-NEXT: vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: abd_ext_v8i16:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpmovsxwq %xmm0, %zmm0
+; AVX512-NEXT: vpmovsxwq %xmm1, %zmm1
+; AVX512-NEXT: vpsubq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpsubq %zmm0, %zmm1, %zmm1
+; AVX512-NEXT: vpmaxsq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpmovqw %zmm0, %xmm0
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%aext = sext <8 x i16> %a to <8 x i64>
%bext = sext <8 x i16> %b to <8 x i64>
%sub = sub <8 x i64> %aext, %bext
@@ -124,28 +772,132 @@ define <8 x i16> @abd_ext_v8i16_undef(<8 x i16> %a, <8 x i16> %b) nounwind {
define <4 x i32> @abd_ext_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
; SSE2-LABEL: abd_ext_v4i32:
; SSE2: # %bb.0:
-; SSE2-NEXT: movdqa %xmm0, %xmm2
-; SSE2-NEXT: pcmpgtd %xmm1, %xmm2
-; SSE2-NEXT: psubd %xmm1, %xmm0
-; SSE2-NEXT: pxor %xmm2, %xmm0
-; SSE2-NEXT: psubd %xmm0, %xmm2
-; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: pxor %xmm4, %xmm4
+; SSE2-NEXT: pcmpgtd %xmm0, %xmm4
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
+; SSE2-NEXT: pxor %xmm4, %xmm4
+; SSE2-NEXT: pcmpgtd %xmm3, %xmm4
+; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
+; SSE2-NEXT: pxor %xmm4, %xmm4
+; SSE2-NEXT: pcmpgtd %xmm1, %xmm4
+; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm1[2,3,2,3]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
+; SSE2-NEXT: psubq %xmm1, %xmm0
+; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: pcmpgtd %xmm5, %xmm1
+; SSE2-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1]
+; SSE2-NEXT: psubq %xmm5, %xmm3
+; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [2147483648,2147483648]
+; SSE2-NEXT: movdqa %xmm0, %xmm4
+; SSE2-NEXT: pxor %xmm1, %xmm4
+; SSE2-NEXT: pxor %xmm5, %xmm5
+; SSE2-NEXT: psubq %xmm0, %xmm5
+; SSE2-NEXT: movdqa %xmm5, %xmm6
+; SSE2-NEXT: pxor %xmm1, %xmm6
+; SSE2-NEXT: movdqa %xmm4, %xmm7
+; SSE2-NEXT: pcmpgtd %xmm6, %xmm7
+; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
+; SSE2-NEXT: pcmpeqd %xmm4, %xmm6
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm6[1,1,3,3]
+; SSE2-NEXT: pand %xmm8, %xmm4
+; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm7[1,1,3,3]
+; SSE2-NEXT: por %xmm4, %xmm6
+; SSE2-NEXT: pand %xmm6, %xmm0
+; SSE2-NEXT: pandn %xmm5, %xmm6
+; SSE2-NEXT: por %xmm6, %xmm0
+; SSE2-NEXT: movdqa %xmm3, %xmm4
+; SSE2-NEXT: pxor %xmm1, %xmm4
+; SSE2-NEXT: psubq %xmm3, %xmm2
+; SSE2-NEXT: pxor %xmm2, %xmm1
+; SSE2-NEXT: movdqa %xmm4, %xmm5
+; SSE2-NEXT: pcmpgtd %xmm1, %xmm5
+; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSE2-NEXT: pcmpeqd %xmm4, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT: pand %xmm6, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm5[1,1,3,3]
+; SSE2-NEXT: por %xmm1, %xmm4
+; SSE2-NEXT: pand %xmm4, %xmm3
+; SSE2-NEXT: pandn %xmm2, %xmm4
+; SSE2-NEXT: por %xmm3, %xmm4
+; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,2],xmm4[0,2]
; SSE2-NEXT: retq
;
; SSE42-LABEL: abd_ext_v4i32:
; SSE42: # %bb.0:
-; SSE42-NEXT: movdqa %xmm0, %xmm2
-; SSE42-NEXT: pminsd %xmm1, %xmm2
-; SSE42-NEXT: pmaxsd %xmm1, %xmm0
-; SSE42-NEXT: psubd %xmm2, %xmm0
+; SSE42-NEXT: pmovsxdq %xmm0, %xmm2
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; SSE42-NEXT: pmovsxdq %xmm0, %xmm3
+; SSE42-NEXT: pmovsxdq %xmm1, %xmm0
+; SSE42-NEXT: psubq %xmm0, %xmm2
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; SSE42-NEXT: pmovsxdq %xmm0, %xmm0
+; SSE42-NEXT: psubq %xmm0, %xmm3
+; SSE42-NEXT: pxor %xmm1, %xmm1
+; SSE42-NEXT: pxor %xmm4, %xmm4
+; SSE42-NEXT: psubq %xmm2, %xmm4
+; SSE42-NEXT: movdqa %xmm2, %xmm0
+; SSE42-NEXT: pcmpgtq %xmm4, %xmm0
+; SSE42-NEXT: pxor %xmm0, %xmm2
+; SSE42-NEXT: psubq %xmm2, %xmm0
+; SSE42-NEXT: psubq %xmm3, %xmm1
+; SSE42-NEXT: movdqa %xmm3, %xmm2
+; SSE42-NEXT: pcmpgtq %xmm1, %xmm2
+; SSE42-NEXT: pxor %xmm2, %xmm3
+; SSE42-NEXT: psubq %xmm3, %xmm2
+; SSE42-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,2],xmm2[0,2]
; SSE42-NEXT: retq
;
-; AVX-LABEL: abd_ext_v4i32:
-; AVX: # %bb.0:
-; AVX-NEXT: vpminsd %xmm1, %xmm0, %xmm2
-; AVX-NEXT: vpmaxsd %xmm1, %xmm0, %xmm0
-; AVX-NEXT: vpsubd %xmm2, %xmm0, %xmm0
-; AVX-NEXT: retq
+; AVX1-LABEL: abd_ext_v4i32:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpmovsxdq %xmm0, %xmm2
+; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; AVX1-NEXT: vpmovsxdq %xmm0, %xmm0
+; AVX1-NEXT: vpmovsxdq %xmm1, %xmm3
+; AVX1-NEXT: vpsubq %xmm3, %xmm2, %xmm2
+; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; AVX1-NEXT: vpmovsxdq %xmm1, %xmm1
+; AVX1-NEXT: vpsubq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT: vpsubq %xmm2, %xmm1, %xmm3
+; AVX1-NEXT: vpcmpgtq %xmm3, %xmm2, %xmm3
+; AVX1-NEXT: vpxor %xmm3, %xmm2, %xmm2
+; AVX1-NEXT: vpsubq %xmm2, %xmm3, %xmm2
+; AVX1-NEXT: vpsubq %xmm0, %xmm1, %xmm1
+; AVX1-NEXT: vpcmpgtq %xmm1, %xmm0, %xmm1
+; AVX1-NEXT: vpxor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpsubq %xmm0, %xmm1, %xmm0
+; AVX1-NEXT: vshufps {{.*#+}} xmm0 = xmm2[0,2],xmm0[0,2]
+; AVX1-NEXT: retq
+;
+; AVX2-LABEL: abd_ext_v4i32:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpmovsxdq %xmm0, %ymm0
+; AVX2-NEXT: vpmovsxdq %xmm1, %ymm1
+; AVX2-NEXT: vpsubq %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpsubq %ymm0, %ymm1, %ymm1
+; AVX2-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm1
+; AVX2-NEXT: vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpsubq %ymm0, %ymm1, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: abd_ext_v4i32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpmovsxdq %xmm0, %ymm0
+; AVX512-NEXT: vpmovsxdq %xmm1, %ymm1
+; AVX512-NEXT: vpsubq %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpsubq %ymm0, %ymm1, %ymm1
+; AVX512-NEXT: vpmaxsq %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpmovqd %ymm0, %xmm0
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%aext = sext <4 x i32> %a to <4 x i64>
%bext = sext <4 x i32> %b to <4 x i64>
%sub = sub <4 x i64> %aext, %bext
@@ -190,56 +942,116 @@ define <4 x i32> @abd_ext_v4i32_undef(<4 x i32> %a, <4 x i32> %b) nounwind {
define <2 x i64> @abd_ext_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
; SSE2-LABEL: abd_ext_v2i64:
; SSE2: # %bb.0:
-; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648]
-; SSE2-NEXT: movdqa %xmm1, %xmm3
-; SSE2-NEXT: pxor %xmm2, %xmm3
-; SSE2-NEXT: pxor %xmm0, %xmm2
-; SSE2-NEXT: movdqa %xmm2, %xmm4
-; SSE2-NEXT: pcmpgtd %xmm3, %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm3, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
-; SSE2-NEXT: pand %xmm5, %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
-; SSE2-NEXT: por %xmm3, %xmm2
-; SSE2-NEXT: psubq %xmm1, %xmm0
-; SSE2-NEXT: pxor %xmm2, %xmm0
-; SSE2-NEXT: psubq %xmm0, %xmm2
-; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
+; SSE2-NEXT: movq %xmm2, %rdx
+; SSE2-NEXT: movq %rdx, %rsi
+; SSE2-NEXT: sarq $63, %rsi
+; SSE2-NEXT: movq %xmm0, %rax
+; SSE2-NEXT: movq %rax, %rcx
+; SSE2-NEXT: sarq $63, %rcx
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; SSE2-NEXT: movq %xmm0, %rdi
+; SSE2-NEXT: movq %rdi, %r8
+; SSE2-NEXT: sarq $63, %r8
+; SSE2-NEXT: movq %xmm1, %r9
+; SSE2-NEXT: movq %r9, %r10
+; SSE2-NEXT: sarq $63, %r10
+; SSE2-NEXT: subq %r9, %rax
+; SSE2-NEXT: sbbq %r10, %rcx
+; SSE2-NEXT: subq %rdi, %rdx
+; SSE2-NEXT: sbbq %r8, %rsi
+; SSE2-NEXT: xorl %edi, %edi
+; SSE2-NEXT: movq %rdx, %r8
+; SSE2-NEXT: negq %r8
+; SSE2-NEXT: movl $0, %r9d
+; SSE2-NEXT: sbbq %rsi, %r9
+; SSE2-NEXT: cmpq %rdx, %r8
+; SSE2-NEXT: sbbq %rsi, %r9
+; SSE2-NEXT: cmovlq %rdx, %r8
+; SSE2-NEXT: movq %rax, %rdx
+; SSE2-NEXT: negq %rdx
+; SSE2-NEXT: sbbq %rcx, %rdi
+; SSE2-NEXT: cmpq %rax, %rdx
+; SSE2-NEXT: sbbq %rcx, %rdi
+; SSE2-NEXT: cmovlq %rax, %rdx
+; SSE2-NEXT: movq %rdx, %xmm0
+; SSE2-NEXT: movq %r8, %xmm1
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; SSE2-NEXT: retq
;
; SSE42-LABEL: abd_ext_v2i64:
; SSE42: # %bb.0:
-; SSE42-NEXT: movdqa %xmm0, %xmm2
-; SSE42-NEXT: pcmpgtq %xmm1, %xmm2
-; SSE42-NEXT: psubq %xmm1, %xmm0
-; SSE42-NEXT: pxor %xmm2, %xmm0
-; SSE42-NEXT: psubq %xmm0, %xmm2
-; SSE42-NEXT: movdqa %xmm2, %xmm0
+; SSE42-NEXT: movq %xmm0, %rdx
+; SSE42-NEXT: movq %rdx, %rsi
+; SSE42-NEXT: sarq $63, %rsi
+; SSE42-NEXT: pextrq $1, %xmm0, %rax
+; SSE42-NEXT: movq %rax, %rcx
+; SSE42-NEXT: sarq $63, %rcx
+; SSE42-NEXT: movq %xmm1, %rdi
+; SSE42-NEXT: movq %rdi, %r8
+; SSE42-NEXT: sarq $63, %r8
+; SSE42-NEXT: pextrq $1, %xmm1, %r9
+; SSE42-NEXT: movq %r9, %r10
+; SSE42-NEXT: sarq $63, %r10
+; SSE42-NEXT: subq %r9, %rax
+; SSE42-NEXT: sbbq %r10, %rcx
+; SSE42-NEXT: subq %rdi, %rdx
+; SSE42-NEXT: sbbq %r8, %rsi
+; SSE42-NEXT: xorl %edi, %edi
+; SSE42-NEXT: movq %rdx, %r8
+; SSE42-NEXT: negq %r8
+; SSE42-NEXT: movl $0, %r9d
+; SSE42-NEXT: sbbq %rsi, %r9
+; SSE42-NEXT: cmpq %rdx, %r8
+; SSE42-NEXT: sbbq %rsi, %r9
+; SSE42-NEXT: cmovlq %rdx, %r8
+; SSE42-NEXT: movq %rax, %rdx
+; SSE42-NEXT: negq %rdx
+; SSE42-NEXT: sbbq %rcx, %rdi
+; SSE42-NEXT: cmpq %rax, %rdx
+; SSE42-NEXT: sbbq %rcx, %rdi
+; SSE42-NEXT: cmovlq %rax, %rdx
+; SSE42-NEXT: movq %rdx, %xmm1
+; SSE42-NEXT: movq %r8, %xmm0
+; SSE42-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; SSE42-NEXT: retq
;
-; AVX1-LABEL: abd_ext_v2i64:
-; AVX1: # %bb.0:
-; AVX1-NEXT: vpcmpgtq %xmm1, %xmm0, %xmm2
-; AVX1-NEXT: vpsubq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpxor %xmm2, %xmm0, %xmm0
-; AVX1-NEXT: vpsubq %xmm0, %xmm2, %xmm0
-; AVX1-NEXT: retq
-;
-; AVX2-LABEL: abd_ext_v2i64:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpgtq %xmm1, %xmm0, %xmm2
-; AVX2-NEXT: vpsubq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpxor %xmm2, %xmm0, %xmm0
-; AVX2-NEXT: vpsubq %xmm0, %xmm2, %xmm0
-; AVX2-NEXT: retq
-;
-; AVX512-LABEL: abd_ext_v2i64:
-; AVX512: # %bb.0:
-; AVX512-NEXT: vpminsq %xmm1, %xmm0, %xmm2
-; AVX512-NEXT: vpmaxsq %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vpsubq %xmm2, %xmm0, %xmm0
-; AVX512-NEXT: retq
+; AVX-LABEL: abd_ext_v2i64:
+; AVX: # %bb.0:
+; AVX-NEXT: vmovq %xmm0, %rdx
+; AVX-NEXT: movq %rdx, %rsi
+; AVX-NEXT: sarq $63, %rsi
+; AVX-NEXT: vpextrq $1, %xmm0, %rax
+; AVX-NEXT: movq %rax, %rcx
+; AVX-NEXT: sarq $63, %rcx
+; AVX-NEXT: vmovq %xmm1, %rdi
+; AVX-NEXT: movq %rdi, %r8
+; AVX-NEXT: sarq $63, %r8
+; AVX-NEXT: vpextrq $1, %xmm1, %r9
+; AVX-NEXT: movq %r9, %r10
+; AVX-NEXT: sarq $63, %r10
+; AVX-NEXT: subq %r9, %rax
+; AVX-NEXT: sbbq %r10, %rcx
+; AVX-NEXT: subq %rdi, %rdx
+; AVX-NEXT: sbbq %r8, %rsi
+; AVX-NEXT: xorl %edi, %edi
+; AVX-NEXT: movq %rdx, %r8
+; AVX-NEXT: negq %r8
+; AVX-NEXT: movl $0, %r9d
+; AVX-NEXT: sbbq %rsi, %r9
+; AVX-NEXT: cmpq %rdx, %r8
+; AVX-NEXT: sbbq %rsi, %r9
+; AVX-NEXT: cmovlq %rdx, %r8
+; AVX-NEXT: movq %rax, %rdx
+; AVX-NEXT: negq %rdx
+; AVX-NEXT: sbbq %rcx, %rdi
+; AVX-NEXT: cmpq %rax, %rdx
+; AVX-NEXT: sbbq %rcx, %rdi
+; AVX-NEXT: cmovlq %rax, %rdx
+; AVX-NEXT: vmovq %rdx, %xmm0
+; AVX-NEXT: vmovq %r8, %xmm1
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX-NEXT: retq
%aext = sext <2 x i64> %a to <2 x i128>
%bext = sext <2 x i64> %b to <2 x i128>
%sub = sub <2 x i128> %aext, %bext
@@ -613,21 +1425,29 @@ define <16 x i8> @abd_subnsw_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
; SSE2-LABEL: abd_subnsw_v16i8:
; SSE2: # %bb.0:
; SSE2-NEXT: psubb %xmm1, %xmm0
-; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: psubb %xmm0, %xmm2
+; SSE2-NEXT: movdqa %xmm0, %xmm1
+; SSE2-NEXT: pcmpgtb %xmm2, %xmm1
+; SSE2-NEXT: pxor %xmm1, %xmm0
; SSE2-NEXT: psubb %xmm0, %xmm1
-; SSE2-NEXT: pminub %xmm1, %xmm0
+; SSE2-NEXT: movdqa %xmm1, %xmm0
; SSE2-NEXT: retq
;
; SSE42-LABEL: abd_subnsw_v16i8:
; SSE42: # %bb.0:
; SSE42-NEXT: psubb %xmm1, %xmm0
-; SSE42-NEXT: pabsb %xmm0, %xmm0
+; SSE42-NEXT: pxor %xmm1, %xmm1
+; SSE42-NEXT: psubb %xmm0, %xmm1
+; SSE42-NEXT: pmaxsb %xmm1, %xmm0
; SSE42-NEXT: retq
;
; AVX-LABEL: abd_subnsw_v16i8:
; AVX: # %bb.0:
; AVX-NEXT: vpsubb %xmm1, %xmm0, %xmm0
-; AVX-NEXT: vpabsb %xmm0, %xmm0
+; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX-NEXT: vpsubb %xmm0, %xmm1, %xmm1
+; AVX-NEXT: vpmaxsb %xmm1, %xmm0, %xmm0
; AVX-NEXT: retq
%sub = sub nsw <16 x i8> %a, %b
%abs = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %sub, i1 false)
@@ -635,24 +1455,20 @@ define <16 x i8> @abd_subnsw_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
}
define <8 x i16> @abd_subnsw_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
-; SSE2-LABEL: abd_subnsw_v8i16:
-; SSE2: # %bb.0:
-; SSE2-NEXT: psubw %xmm1, %xmm0
-; SSE2-NEXT: pxor %xmm1, %xmm1
-; SSE2-NEXT: psubw %xmm0, %xmm1
-; SSE2-NEXT: pmaxsw %xmm1, %xmm0
-; SSE2-NEXT: retq
-;
-; SSE42-LABEL: abd_subnsw_v8i16:
-; SSE42: # %bb.0:
-; SSE42-NEXT: psubw %xmm1, %xmm0
-; SSE42-NEXT: pabsw %xmm0, %xmm0
-; SSE42-NEXT: retq
+; SSE-LABEL: abd_subnsw_v8i16:
+; SSE: # %bb.0:
+; SSE-NEXT: psubw %xmm1, %xmm0
+; SSE-NEXT: pxor %xmm1, %xmm1
+; SSE-NEXT: psubw %xmm0, %xmm1
+; SSE-NEXT: pmaxsw %xmm1, %xmm0
+; SSE-NEXT: retq
;
; AVX-LABEL: abd_subnsw_v8i16:
; AVX: # %bb.0:
; AVX-NEXT: vpsubw %xmm1, %xmm0, %xmm0
-; AVX-NEXT: vpabsw %xmm0, %xmm0
+; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX-NEXT: vpsubw %xmm0, %xmm1, %xmm1
+; AVX-NEXT: vpmaxsw %xmm1, %xmm0, %xmm0
; AVX-NEXT: retq
%sub = sub nsw <8 x i16> %a, %b
%abs = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %sub, i1 false)
@@ -663,22 +1479,29 @@ define <4 x i32> @abd_subnsw_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
; SSE2-LABEL: abd_subnsw_v4i32:
; SSE2: # %bb.0:
; SSE2-NEXT: psubd %xmm1, %xmm0
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: psubd %xmm0, %xmm2
; SSE2-NEXT: movdqa %xmm0, %xmm1
-; SSE2-NEXT: psrad $31, %xmm1
+; SSE2-NEXT: pcmpgtd %xmm2, %xmm1
; SSE2-NEXT: pxor %xmm1, %xmm0
-; SSE2-NEXT: psubd %xmm1, %xmm0
+; SSE2-NEXT: psubd %xmm0, %xmm1
+; SSE2-NEXT: movdqa %xmm1, %xmm0
; SSE2-NEXT: retq
;
; SSE42-LABEL: abd_subnsw_v4i32:
; SSE42: # %bb.0:
; SSE42-NEXT: psubd %xmm1, %xmm0
-; SSE42-NEXT: pabsd %xmm0, %xmm0
+; SSE42-NEXT: pxor %xmm1, %xmm1
+; SSE42-NEXT: psubd %xmm0, %xmm1
+; SSE42-NEXT: pmaxsd %xmm1, %xmm0
; SSE42-NEXT: retq
;
; AVX-LABEL: abd_subnsw_v4i32:
; AVX: # %bb.0:
; AVX-NEXT: vpsubd %xmm1, %xmm0, %xmm0
-; AVX-NEXT: vpabsd %xmm0, %xmm0
+; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX-NEXT: vpsubd %xmm0, %xmm1, %xmm1
+; AVX-NEXT: vpmaxsd %xmm1, %xmm0, %xmm0
; AVX-NEXT: retq
%sub = sub nsw <4 x i32> %a, %b
%abs = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %sub, i1 false)
@@ -689,18 +1512,35 @@ define <2 x i64> @abd_subnsw_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
; SSE2-LABEL: abd_subnsw_v2i64:
; SSE2: # %bb.0:
; SSE2-NEXT: psubq %xmm1, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,3,3]
-; SSE2-NEXT: psrad $31, %xmm1
-; SSE2-NEXT: pxor %xmm1, %xmm0
-; SSE2-NEXT: psubq %xmm1, %xmm0
+; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [2147483648,2147483648]
+; SSE2-NEXT: movdqa %xmm0, %xmm2
+; SSE2-NEXT: pxor %xmm1, %xmm2
+; SSE2-NEXT: pxor %xmm3, %xmm3
+; SSE2-NEXT: psubq %xmm0, %xmm3
+; SSE2-NEXT: pxor %xmm3, %xmm1
+; SSE2-NEXT: movdqa %xmm2, %xmm4
+; SSE2-NEXT: pcmpgtd %xmm1, %xmm4
+; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
+; SSE2-NEXT: pcmpeqd %xmm2, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT: pand %xmm5, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
+; SSE2-NEXT: por %xmm1, %xmm2
+; SSE2-NEXT: pand %xmm2, %xmm0
+; SSE2-NEXT: pandn %xmm3, %xmm2
+; SSE2-NEXT: por %xmm2, %xmm0
; SSE2-NEXT: retq
;
; SSE42-LABEL: abd_subnsw_v2i64:
; SSE42: # %bb.0:
; SSE42-NEXT: psubq %xmm1, %xmm0
-; SSE42-NEXT: pxor %xmm1, %xmm1
+; SSE42-NEXT: pxor %xmm2, %xmm2
+; SSE42-NEXT: psubq %xmm0, %xmm2
+; SSE42-NEXT: movdqa %xmm0, %xmm1
+; SSE42-NEXT: pcmpgtq %xmm2, %xmm1
+; SSE42-NEXT: pxor %xmm1, %xmm0
; SSE42-NEXT: psubq %xmm0, %xmm1
-; SSE42-NEXT: blendvpd %xmm0, %xmm1, %xmm0
+; SSE42-NEXT: movdqa %xmm1, %xmm0
; SSE42-NEXT: retq
;
; AVX1-LABEL: abd_subnsw_v2i64:
@@ -708,7 +1548,9 @@ define <2 x i64> @abd_subnsw_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
; AVX1-NEXT: vpsubq %xmm1, %xmm0, %xmm0
; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX1-NEXT: vpsubq %xmm0, %xmm1, %xmm1
-; AVX1-NEXT: vblendvpd %xmm0, %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpcmpgtq %xmm1, %xmm0, %xmm1
+; AVX1-NEXT: vpxor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpsubq %xmm0, %xmm1, %xmm0
; AVX1-NEXT: retq
;
; AVX2-LABEL: abd_subnsw_v2i64:
@@ -716,13 +1558,17 @@ define <2 x i64> @abd_subnsw_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
; AVX2-NEXT: vpsubq %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX2-NEXT: vpsubq %xmm0, %xmm1, %xmm1
-; AVX2-NEXT: vblendvpd %xmm0, %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpcmpgtq %xmm1, %xmm0, %xmm1
+; AVX2-NEXT: vpxor %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpsubq %xmm0, %xmm1, %xmm0
; AVX2-NEXT: retq
;
; AVX512-LABEL: abd_subnsw_v2i64:
; AVX512: # %bb.0:
; AVX512-NEXT: vpsubq %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vpabsq %xmm0, %xmm0
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpsubq %xmm0, %xmm1, %xmm1
+; AVX512-NEXT: vpmaxsq %xmm1, %xmm0, %xmm0
; AVX512-NEXT: retq
%sub = sub nsw <2 x i64> %a, %b
%abs = call <2 x i64> @llvm.abs.v2i64(<2 x i64> %sub, i1 false)
diff --git a/llvm/test/CodeGen/X86/abds-vector-256.ll b/llvm/test/CodeGen/X86/abds-vector-256.ll
index cc63ad04c08a6..fe16232a6bb7b 100644
--- a/llvm/test/CodeGen/X86/abds-vector-256.ll
+++ b/llvm/test/CodeGen/X86/abds-vector-256.ll
@@ -10,29 +10,294 @@
define <32 x i8> @abd_ext_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
; AVX1-LABEL: abd_ext_v32i8:
; AVX1: # %bb.0:
-; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
-; AVX1-NEXT: vpminsb %xmm2, %xmm3, %xmm4
-; AVX1-NEXT: vpmaxsb %xmm2, %xmm3, %xmm2
-; AVX1-NEXT: vpsubb %xmm4, %xmm2, %xmm2
-; AVX1-NEXT: vpminsb %xmm1, %xmm0, %xmm3
-; AVX1-NEXT: vpmaxsb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpsubb %xmm3, %xmm0, %xmm0
-; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
+; AVX1-NEXT: vpmovsxbw %xmm2, %xmm5
+; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm5[1,1,1,1]
+; AVX1-NEXT: vpmovsxwq %xmm3, %xmm3
+; AVX1-NEXT: vpmovsxbq %xmm2, %xmm13
+; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm5[2,3,2,3]
+; AVX1-NEXT: vpmovsxwq %xmm2, %xmm4
+; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm5[3,3,3,3]
+; AVX1-NEXT: vpmovsxwq %xmm2, %xmm5
+; AVX1-NEXT: vpmovsxbw %xmm0, %xmm2
+; AVX1-NEXT: vpshufd {{.*#+}} xmm6 = xmm2[2,3,2,3]
+; AVX1-NEXT: vpmovsxwq %xmm6, %xmm6
+; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
+; AVX1-NEXT: vpmovsxwq %xmm2, %xmm7
+; AVX1-NEXT: vpsrld $16, %xmm0, %xmm2
+; AVX1-NEXT: vpmovsxbq %xmm2, %xmm9
+; AVX1-NEXT: vpmovsxbq %xmm0, %xmm8
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
+; AVX1-NEXT: vpmovsxbw %xmm2, %xmm11
+; AVX1-NEXT: vpshufd {{.*#+}} xmm10 = xmm11[1,1,1,1]
+; AVX1-NEXT: vpmovsxwq %xmm10, %xmm10
+; AVX1-NEXT: vpmovsxbq %xmm2, %xmm12
+; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; AVX1-NEXT: vpmovsxbw %xmm2, %xmm14
+; AVX1-NEXT: vpshufd {{.*#+}} xmm15 = xmm14[1,1,1,1]
+; AVX1-NEXT: vpmovsxwq %xmm15, %xmm15
+; AVX1-NEXT: vpsubq %xmm15, %xmm3, %xmm3
+; AVX1-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT: vpmovsxbq %xmm2, %xmm2
+; AVX1-NEXT: vpsubq %xmm2, %xmm13, %xmm2
+; AVX1-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm14[2,3,2,3]
+; AVX1-NEXT: vpmovsxwq %xmm2, %xmm2
+; AVX1-NEXT: vpsubq %xmm2, %xmm4, %xmm2
+; AVX1-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm14[3,3,3,3]
+; AVX1-NEXT: vpmovsxwq %xmm2, %xmm2
+; AVX1-NEXT: vpsubq %xmm2, %xmm5, %xmm4
+; AVX1-NEXT: vpmovsxbw %xmm1, %xmm2
+; AVX1-NEXT: vpshufd {{.*#+}} xmm13 = xmm2[2,3,2,3]
+; AVX1-NEXT: vpmovsxwq %xmm13, %xmm13
+; AVX1-NEXT: vpsubq %xmm13, %xmm6, %xmm5
+; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
+; AVX1-NEXT: vpmovsxwq %xmm2, %xmm2
+; AVX1-NEXT: vpsubq %xmm2, %xmm7, %xmm6
+; AVX1-NEXT: vpsrld $16, %xmm1, %xmm2
+; AVX1-NEXT: vpmovsxbq %xmm2, %xmm2
+; AVX1-NEXT: vpsubq %xmm2, %xmm9, %xmm7
+; AVX1-NEXT: vpmovsxbq %xmm1, %xmm2
+; AVX1-NEXT: vpsubq %xmm2, %xmm8, %xmm8
+; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
+; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; AVX1-NEXT: vpmovsxbw %xmm2, %xmm13
+; AVX1-NEXT: vpshufd {{.*#+}} xmm14 = xmm13[1,1,1,1]
+; AVX1-NEXT: vpmovsxwq %xmm14, %xmm14
+; AVX1-NEXT: vpsubq %xmm14, %xmm10, %xmm9
+; AVX1-NEXT: vpshufd {{.*#+}} xmm14 = xmm11[2,3,2,3]
+; AVX1-NEXT: vpmovsxwq %xmm14, %xmm14
+; AVX1-NEXT: vpshufd {{.*#+}} xmm11 = xmm11[3,3,3,3]
+; AVX1-NEXT: vpmovsxwq %xmm11, %xmm15
+; AVX1-NEXT: vpmovsxbq %xmm2, %xmm2
+; AVX1-NEXT: vpsubq %xmm2, %xmm12, %xmm10
+; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm13[2,3,2,3]
+; AVX1-NEXT: vpmovsxwq %xmm2, %xmm2
+; AVX1-NEXT: vpsubq %xmm2, %xmm14, %xmm11
+; AVX1-NEXT: vpmovsxbw %xmm0, %xmm2
+; AVX1-NEXT: vpshufd {{.*#+}} xmm13 = xmm13[3,3,3,3]
+; AVX1-NEXT: vpmovsxwq %xmm13, %xmm13
+; AVX1-NEXT: vpsubq %xmm13, %xmm15, %xmm12
+; AVX1-NEXT: vpshufd {{.*#+}} xmm14 = xmm2[2,3,2,3]
+; AVX1-NEXT: vpmovsxwq %xmm14, %xmm14
+; AVX1-NEXT: vpmovsxbw %xmm1, %xmm15
+; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm15[2,3,2,3]
+; AVX1-NEXT: vpmovsxwq %xmm3, %xmm3
+; AVX1-NEXT: vpsubq %xmm3, %xmm14, %xmm13
+; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
+; AVX1-NEXT: vpmovsxwq %xmm2, %xmm2
+; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm15[3,3,3,3]
+; AVX1-NEXT: vpmovsxwq %xmm3, %xmm3
+; AVX1-NEXT: vpsubq %xmm3, %xmm2, %xmm14
+; AVX1-NEXT: vpsrld $16, %xmm0, %xmm2
+; AVX1-NEXT: vpmovsxbq %xmm2, %xmm2
+; AVX1-NEXT: vpsrld $16, %xmm1, %xmm3
+; AVX1-NEXT: vpmovsxbq %xmm3, %xmm3
+; AVX1-NEXT: vpsubq %xmm3, %xmm2, %xmm2
+; AVX1-NEXT: vpmovsxbq %xmm0, %xmm0
+; AVX1-NEXT: vpmovsxbq %xmm1, %xmm1
+; AVX1-NEXT: vpsubq %xmm1, %xmm0, %xmm15
+; AVX1-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; AVX1-NEXT: vpsubq %xmm1, %xmm0, %xmm3
+; AVX1-NEXT: vpmaxsd %xmm3, %xmm1, %xmm1
+; AVX1-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; AVX1-NEXT: vpsubq %xmm1, %xmm0, %xmm3
+; AVX1-NEXT: vpmaxsw %xmm3, %xmm1, %xmm1
+; AVX1-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; AVX1-NEXT: vpsubq %xmm1, %xmm0, %xmm3
+; AVX1-NEXT: vpmaxsd %xmm3, %xmm1, %xmm1
+; AVX1-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT: vpsubq %xmm4, %xmm0, %xmm3
+; AVX1-NEXT: vpmaxsd %xmm3, %xmm4, %xmm3
+; AVX1-NEXT: vpsubq %xmm5, %xmm0, %xmm4
+; AVX1-NEXT: vpmaxsd %xmm4, %xmm5, %xmm4
+; AVX1-NEXT: vpsubq %xmm6, %xmm0, %xmm5
+; AVX1-NEXT: vpmaxsd %xmm5, %xmm6, %xmm5
+; AVX1-NEXT: vpsubq %xmm7, %xmm0, %xmm6
+; AVX1-NEXT: vpmaxsw %xmm6, %xmm7, %xmm6
+; AVX1-NEXT: vpsubq %xmm8, %xmm0, %xmm7
+; AVX1-NEXT: vpmaxsw %xmm7, %xmm8, %xmm7
+; AVX1-NEXT: vpsubq %xmm9, %xmm0, %xmm8
+; AVX1-NEXT: vpmaxsd %xmm8, %xmm9, %xmm8
+; AVX1-NEXT: vpsubq %xmm10, %xmm0, %xmm9
+; AVX1-NEXT: vpmaxsw %xmm9, %xmm10, %xmm9
+; AVX1-NEXT: vpsubq %xmm11, %xmm0, %xmm10
+; AVX1-NEXT: vpmaxsd %xmm10, %xmm11, %xmm10
+; AVX1-NEXT: vpsubq %xmm12, %xmm0, %xmm11
+; AVX1-NEXT: vpmaxsd %xmm11, %xmm12, %xmm11
+; AVX1-NEXT: vpsubq %xmm13, %xmm0, %xmm12
+; AVX1-NEXT: vpmaxsd %xmm12, %xmm13, %xmm12
+; AVX1-NEXT: vpsubq %xmm14, %xmm0, %xmm13
+; AVX1-NEXT: vpmaxsd %xmm13, %xmm14, %xmm13
+; AVX1-NEXT: vpsubq %xmm2, %xmm0, %xmm14
+; AVX1-NEXT: vpmaxsw %xmm2, %xmm14, %xmm2
+; AVX1-NEXT: vpsubq %xmm15, %xmm0, %xmm0
+; AVX1-NEXT: vpmaxsw %xmm0, %xmm15, %xmm0
+; AVX1-NEXT: vpmovzxbq {{.*#+}} xmm1 = [255,255]
+; AVX1-NEXT: vpand %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpand %xmm1, %xmm2, %xmm2
+; AVX1-NEXT: vpackusdw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT: vpand %xmm1, %xmm13, %xmm2
+; AVX1-NEXT: vpand %xmm1, %xmm12, %xmm12
+; AVX1-NEXT: vpackusdw %xmm2, %xmm12, %xmm2
+; AVX1-NEXT: vpackusdw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT: vpand %xmm1, %xmm11, %xmm2
+; AVX1-NEXT: vpand %xmm1, %xmm10, %xmm10
+; AVX1-NEXT: vpackusdw %xmm2, %xmm10, %xmm2
+; AVX1-NEXT: vpand %xmm1, %xmm9, %xmm9
+; AVX1-NEXT: vpand %xmm1, %xmm8, %xmm8
+; AVX1-NEXT: vpackusdw %xmm8, %xmm9, %xmm8
+; AVX1-NEXT: vpackusdw %xmm2, %xmm8, %xmm2
+; AVX1-NEXT: vpackuswb %xmm2, %xmm0, %xmm0
+; AVX1-NEXT: vpand %xmm1, %xmm7, %xmm2
+; AVX1-NEXT: vpand %xmm1, %xmm6, %xmm6
+; AVX1-NEXT: vpackusdw %xmm6, %xmm2, %xmm2
+; AVX1-NEXT: vpand %xmm1, %xmm5, %xmm5
+; AVX1-NEXT: vpand %xmm1, %xmm4, %xmm4
+; AVX1-NEXT: vpackusdw %xmm5, %xmm4, %xmm4
+; AVX1-NEXT: vpackusdw %xmm4, %xmm2, %xmm2
+; AVX1-NEXT: vpand %xmm1, %xmm3, %xmm3
+; AVX1-NEXT: vpand {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm4 # 16-byte Folded Reload
+; AVX1-NEXT: vpackusdw %xmm3, %xmm4, %xmm3
+; AVX1-NEXT: vpand {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm4 # 16-byte Folded Reload
+; AVX1-NEXT: vpand {{[-0-9]+}}(%r{{[sb]}}p), %xmm1, %xmm1 # 16-byte Folded Reload
+; AVX1-NEXT: vpackusdw %xmm1, %xmm4, %xmm1
+; AVX1-NEXT: vpackusdw %xmm3, %xmm1, %xmm1
+; AVX1-NEXT: vpackuswb %xmm1, %xmm2, %xmm1
+; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0
; AVX1-NEXT: retq
;
; AVX2-LABEL: abd_ext_v32i8:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpminsb %ymm1, %ymm0, %ymm2
-; AVX2-NEXT: vpmaxsb %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: vpsubb %ymm2, %ymm0, %ymm0
+; AVX2-NEXT: vpmovsxbw %xmm0, %ymm2
+; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
+; AVX2-NEXT: vpmovsxwq %xmm3, %ymm3
+; AVX2-NEXT: vpmovsxbq %xmm0, %ymm4
+; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm2
+; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm2[2,3,2,3]
+; AVX2-NEXT: vpmovsxwq %xmm5, %ymm5
+; AVX2-NEXT: vpmovsxwq %xmm2, %ymm2
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX2-NEXT: vpmovsxbw %xmm0, %ymm6
+; AVX2-NEXT: vpshufd {{.*#+}} xmm7 = xmm6[2,3,2,3]
+; AVX2-NEXT: vpmovsxwq %xmm7, %ymm7
+; AVX2-NEXT: vpmovsxbq %xmm0, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm6, %xmm6
+; AVX2-NEXT: vpshufd {{.*#+}} xmm8 = xmm6[2,3,2,3]
+; AVX2-NEXT: vpmovsxwq %xmm8, %ymm8
+; AVX2-NEXT: vpmovsxwq %xmm6, %ymm6
+; AVX2-NEXT: vpmovsxbw %xmm1, %ymm9
+; AVX2-NEXT: vpshufd {{.*#+}} xmm10 = xmm9[2,3,2,3]
+; AVX2-NEXT: vpmovsxwq %xmm10, %ymm10
+; AVX2-NEXT: vpsubq %ymm10, %ymm3, %ymm3
+; AVX2-NEXT: vpmovsxbq %xmm1, %ymm10
+; AVX2-NEXT: vpsubq %ymm10, %ymm4, %ymm4
+; AVX2-NEXT: vextracti128 $1, %ymm9, %xmm9
+; AVX2-NEXT: vpshufd {{.*#+}} xmm10 = xmm9[2,3,2,3]
+; AVX2-NEXT: vpmovsxwq %xmm10, %ymm10
+; AVX2-NEXT: vpsubq %ymm10, %ymm5, %ymm5
+; AVX2-NEXT: vpmovsxwq %xmm9, %ymm9
+; AVX2-NEXT: vpsubq %ymm9, %ymm2, %ymm2
+; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm1
+; AVX2-NEXT: vpmovsxbw %xmm1, %ymm9
+; AVX2-NEXT: vpshufd {{.*#+}} xmm10 = xmm9[2,3,2,3]
+; AVX2-NEXT: vpmovsxwq %xmm10, %ymm10
+; AVX2-NEXT: vpsubq %ymm10, %ymm7, %ymm7
+; AVX2-NEXT: vpmovsxbq %xmm1, %ymm1
+; AVX2-NEXT: vpsubq %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm9, %xmm1
+; AVX2-NEXT: vpshufd {{.*#+}} xmm9 = xmm1[2,3,2,3]
+; AVX2-NEXT: vpmovsxwq %xmm9, %ymm9
+; AVX2-NEXT: vpsubq %ymm9, %ymm8, %ymm8
+; AVX2-NEXT: vpmovsxwq %xmm1, %ymm1
+; AVX2-NEXT: vpsubq %ymm1, %ymm6, %ymm1
+; AVX2-NEXT: vpxor %xmm6, %xmm6, %xmm6
+; AVX2-NEXT: vpsubq %ymm3, %ymm6, %ymm9
+; AVX2-NEXT: vpmaxsw %ymm3, %ymm9, %ymm3
+; AVX2-NEXT: vpsubq %ymm4, %ymm6, %ymm9
+; AVX2-NEXT: vpmaxsw %ymm4, %ymm9, %ymm4
+; AVX2-NEXT: vpsubq %ymm5, %ymm6, %ymm9
+; AVX2-NEXT: vpmaxsw %ymm5, %ymm9, %ymm5
+; AVX2-NEXT: vpsubq %ymm2, %ymm6, %ymm9
+; AVX2-NEXT: vpmaxsw %ymm2, %ymm9, %ymm2
+; AVX2-NEXT: vpsubq %ymm7, %ymm6, %ymm9
+; AVX2-NEXT: vpmaxsw %ymm7, %ymm9, %ymm7
+; AVX2-NEXT: vpsubq %ymm0, %ymm6, %ymm9
+; AVX2-NEXT: vpmaxsw %ymm0, %ymm9, %ymm0
+; AVX2-NEXT: vpsubq %ymm8, %ymm6, %ymm9
+; AVX2-NEXT: vpmaxsw %ymm9, %ymm8, %ymm8
+; AVX2-NEXT: vpsubq %ymm1, %ymm6, %ymm6
+; AVX2-NEXT: vpmaxsw %ymm6, %ymm1, %ymm1
+; AVX2-NEXT: vpbroadcastq {{.*#+}} ymm6 = [255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0]
+; AVX2-NEXT: vpand %ymm6, %ymm1, %ymm1
+; AVX2-NEXT: vpand %ymm6, %ymm8, %ymm8
+; AVX2-NEXT: vpackusdw %ymm8, %ymm1, %ymm1
+; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
+; AVX2-NEXT: vpand %ymm6, %ymm0, %ymm0
+; AVX2-NEXT: vpand %ymm6, %ymm7, %ymm7
+; AVX2-NEXT: vpackusdw %ymm7, %ymm0, %ymm0
+; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT: vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT: vpand %ymm6, %ymm2, %ymm1
+; AVX2-NEXT: vpand %ymm6, %ymm5, %ymm2
+; AVX2-NEXT: vpackusdw %ymm2, %ymm1, %ymm1
+; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
+; AVX2-NEXT: vpand %ymm6, %ymm4, %ymm2
+; AVX2-NEXT: vpand %ymm6, %ymm3, %ymm3
+; AVX2-NEXT: vpackusdw %ymm3, %ymm2, %ymm2
+; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
+; AVX2-NEXT: vpackusdw %ymm1, %ymm2, %ymm1
+; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
+; AVX2-NEXT: vpackuswb %ymm0, %ymm1, %ymm0
+; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
; AVX2-NEXT: retq
;
; AVX512-LABEL: abd_ext_v32i8:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpminsb %ymm1, %ymm0, %ymm2
-; AVX512-NEXT: vpmaxsb %ymm1, %ymm0, %ymm0
-; AVX512-NEXT: vpsubb %ymm2, %ymm0, %ymm0
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm2
+; AVX512-NEXT: vpmovsxbw %xmm2, %ymm2
+; AVX512-NEXT: vpmovsxwq %xmm2, %zmm3
+; AVX512-NEXT: vpmovsxbw %xmm0, %ymm0
+; AVX512-NEXT: vpmovsxwq %xmm0, %zmm4
+; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm2
+; AVX512-NEXT: vpmovsxwq %xmm2, %zmm2
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX512-NEXT: vpmovsxwq %xmm0, %zmm0
+; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm5
+; AVX512-NEXT: vpmovsxbw %xmm5, %ymm5
+; AVX512-NEXT: vpmovsxwq %xmm5, %zmm6
+; AVX512-NEXT: vpsubq %zmm6, %zmm3, %zmm3
+; AVX512-NEXT: vpmovsxbw %xmm1, %ymm1
+; AVX512-NEXT: vpmovsxwq %xmm1, %zmm6
+; AVX512-NEXT: vpsubq %zmm6, %zmm4, %zmm4
+; AVX512-NEXT: vextracti128 $1, %ymm5, %xmm5
+; AVX512-NEXT: vpmovsxwq %xmm5, %zmm5
+; AVX512-NEXT: vpsubq %zmm5, %zmm2, %zmm2
+; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm1
+; AVX512-NEXT: vpmovsxwq %xmm1, %zmm1
+; AVX512-NEXT: vpsubq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpsubq %zmm3, %zmm1, %zmm5
+; AVX512-NEXT: vpmaxsq %zmm5, %zmm3, %zmm3
+; AVX512-NEXT: vpsubq %zmm4, %zmm1, %zmm5
+; AVX512-NEXT: vpmaxsq %zmm5, %zmm4, %zmm4
+; AVX512-NEXT: vpsubq %zmm2, %zmm1, %zmm5
+; AVX512-NEXT: vpmaxsq %zmm5, %zmm2, %zmm2
+; AVX512-NEXT: vpsubq %zmm0, %zmm1, %zmm1
+; AVX512-NEXT: vpmaxsq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpmovqb %zmm0, %xmm0
+; AVX512-NEXT: vpmovqb %zmm2, %xmm1
+; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512-NEXT: vpmovqb %zmm4, %xmm1
+; AVX512-NEXT: vpmovqb %zmm3, %xmm2
+; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
; AVX512-NEXT: retq
%aext = sext <32 x i8> %a to <32 x i64>
%bext = sext <32 x i8> %b to <32 x i64>
@@ -80,29 +345,137 @@ define <32 x i8> @abd_ext_v32i8_undef(<32 x i8> %a, <32 x i8> %b) nounwind {
define <16 x i16> @abd_ext_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind {
; AVX1-LABEL: abd_ext_v16i16:
; AVX1: # %bb.0:
-; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
-; AVX1-NEXT: vpminsw %xmm2, %xmm3, %xmm4
-; AVX1-NEXT: vpmaxsw %xmm2, %xmm3, %xmm2
-; AVX1-NEXT: vpsubw %xmm4, %xmm2, %xmm2
-; AVX1-NEXT: vpminsw %xmm1, %xmm0, %xmm3
-; AVX1-NEXT: vpmaxsw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpsubw %xmm3, %xmm0, %xmm0
-; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[1,1,1,1]
+; AVX1-NEXT: vpmovsxwq %xmm2, %xmm2
+; AVX1-NEXT: vpmovsxwq %xmm0, %xmm3
+; AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; AVX1-NEXT: vpmovsxwq %xmm4, %xmm4
+; AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm0[3,3,3,3]
+; AVX1-NEXT: vpmovsxwq %xmm5, %xmm5
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT: vpshufd {{.*#+}} xmm6 = xmm0[1,1,1,1]
+; AVX1-NEXT: vpmovsxwq %xmm6, %xmm6
+; AVX1-NEXT: vpmovsxwq %xmm0, %xmm7
+; AVX1-NEXT: vpshufd {{.*#+}} xmm8 = xmm0[2,3,2,3]
+; AVX1-NEXT: vpmovsxwq %xmm8, %xmm8
+; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; AVX1-NEXT: vpmovsxwq %xmm0, %xmm0
+; AVX1-NEXT: vpshufd {{.*#+}} xmm9 = xmm1[1,1,1,1]
+; AVX1-NEXT: vpmovsxwq %xmm9, %xmm9
+; AVX1-NEXT: vpsubq %xmm9, %xmm2, %xmm2
+; AVX1-NEXT: vpmovsxwq %xmm1, %xmm9
+; AVX1-NEXT: vpsubq %xmm9, %xmm3, %xmm3
+; AVX1-NEXT: vpshufd {{.*#+}} xmm9 = xmm1[2,3,2,3]
+; AVX1-NEXT: vpmovsxwq %xmm9, %xmm9
+; AVX1-NEXT: vpsubq %xmm9, %xmm4, %xmm4
+; AVX1-NEXT: vpshufd {{.*#+}} xmm9 = xmm1[3,3,3,3]
+; AVX1-NEXT: vpmovsxwq %xmm9, %xmm9
+; AVX1-NEXT: vpsubq %xmm9, %xmm5, %xmm5
+; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
+; AVX1-NEXT: vpshufd {{.*#+}} xmm9 = xmm1[1,1,1,1]
+; AVX1-NEXT: vpmovsxwq %xmm9, %xmm9
+; AVX1-NEXT: vpsubq %xmm9, %xmm6, %xmm6
+; AVX1-NEXT: vpmovsxwq %xmm1, %xmm9
+; AVX1-NEXT: vpsubq %xmm9, %xmm7, %xmm7
+; AVX1-NEXT: vpshufd {{.*#+}} xmm9 = xmm1[2,3,2,3]
+; AVX1-NEXT: vpmovsxwq %xmm9, %xmm9
+; AVX1-NEXT: vpsubq %xmm9, %xmm8, %xmm8
+; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
+; AVX1-NEXT: vpmovsxwq %xmm1, %xmm1
+; AVX1-NEXT: vpsubq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT: vpsubq %xmm2, %xmm1, %xmm9
+; AVX1-NEXT: vpmaxsd %xmm9, %xmm2, %xmm2
+; AVX1-NEXT: vpsubq %xmm3, %xmm1, %xmm9
+; AVX1-NEXT: vpmaxsd %xmm9, %xmm3, %xmm3
+; AVX1-NEXT: vpsubq %xmm4, %xmm1, %xmm9
+; AVX1-NEXT: vpmaxsd %xmm9, %xmm4, %xmm4
+; AVX1-NEXT: vpsubq %xmm5, %xmm1, %xmm9
+; AVX1-NEXT: vpmaxsd %xmm9, %xmm5, %xmm5
+; AVX1-NEXT: vpsubq %xmm6, %xmm1, %xmm9
+; AVX1-NEXT: vpmaxsd %xmm9, %xmm6, %xmm6
+; AVX1-NEXT: vpsubq %xmm7, %xmm1, %xmm9
+; AVX1-NEXT: vpmaxsd %xmm9, %xmm7, %xmm7
+; AVX1-NEXT: vpsubq %xmm8, %xmm1, %xmm9
+; AVX1-NEXT: vpmaxsd %xmm9, %xmm8, %xmm8
+; AVX1-NEXT: vpsubq %xmm0, %xmm1, %xmm9
+; AVX1-NEXT: vpmaxsd %xmm9, %xmm0, %xmm0
+; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3],xmm0[4],xmm1[5,6,7]
+; AVX1-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0],xmm1[1,2,3],xmm8[4],xmm1[5,6,7]
+; AVX1-NEXT: vpackusdw %xmm0, %xmm8, %xmm0
+; AVX1-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0],xmm1[1,2,3],xmm7[4],xmm1[5,6,7]
+; AVX1-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0],xmm1[1,2,3],xmm6[4],xmm1[5,6,7]
+; AVX1-NEXT: vpackusdw %xmm6, %xmm7, %xmm6
+; AVX1-NEXT: vpackusdw %xmm0, %xmm6, %xmm0
+; AVX1-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0],xmm1[1,2,3],xmm5[4],xmm1[5,6,7]
+; AVX1-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0],xmm1[1,2,3],xmm4[4],xmm1[5,6,7]
+; AVX1-NEXT: vpackusdw %xmm5, %xmm4, %xmm4
+; AVX1-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0],xmm1[1,2,3],xmm3[4],xmm1[5,6,7]
+; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3],xmm2[4],xmm1[5,6,7]
+; AVX1-NEXT: vpackusdw %xmm1, %xmm3, %xmm1
+; AVX1-NEXT: vpackusdw %xmm4, %xmm1, %xmm1
+; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0
; AVX1-NEXT: retq
;
; AVX2-LABEL: abd_ext_v16i16:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpminsw %ymm1, %ymm0, %ymm2
-; AVX2-NEXT: vpmaxsw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: vpsubw %ymm2, %ymm0, %ymm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
+; AVX2-NEXT: vpmovsxwq %xmm2, %ymm2
+; AVX2-NEXT: vpmovsxwq %xmm0, %ymm3
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; AVX2-NEXT: vpmovsxwq %xmm4, %ymm4
+; AVX2-NEXT: vpmovsxwq %xmm0, %ymm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[2,3,2,3]
+; AVX2-NEXT: vpmovsxwq %xmm5, %ymm5
+; AVX2-NEXT: vpsubq %ymm5, %ymm2, %ymm2
+; AVX2-NEXT: vpmovsxwq %xmm1, %ymm5
+; AVX2-NEXT: vpsubq %ymm5, %ymm3, %ymm3
+; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm1
+; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[2,3,2,3]
+; AVX2-NEXT: vpmovsxwq %xmm5, %ymm5
+; AVX2-NEXT: vpsubq %ymm5, %ymm4, %ymm4
+; AVX2-NEXT: vpmovsxwq %xmm1, %ymm1
+; AVX2-NEXT: vpsubq %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpsubq %ymm2, %ymm1, %ymm5
+; AVX2-NEXT: vpmaxsd %ymm5, %ymm2, %ymm2
+; AVX2-NEXT: vpsubq %ymm3, %ymm1, %ymm5
+; AVX2-NEXT: vpmaxsd %ymm5, %ymm3, %ymm3
+; AVX2-NEXT: vpsubq %ymm4, %ymm1, %ymm5
+; AVX2-NEXT: vpmaxsd %ymm5, %ymm4, %ymm4
+; AVX2-NEXT: vpsubq %ymm0, %ymm1, %ymm5
+; AVX2-NEXT: vpmaxsd %ymm5, %ymm0, %ymm0
+; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7],ymm0[8],ymm1[9,10,11],ymm0[12],ymm1[13,14,15]
+; AVX2-NEXT: vpblendw {{.*#+}} ymm4 = ymm4[0],ymm1[1,2,3],ymm4[4],ymm1[5,6,7],ymm4[8],ymm1[9,10,11],ymm4[12],ymm1[13,14,15]
+; AVX2-NEXT: vpackusdw %ymm4, %ymm0, %ymm0
+; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT: vpblendw {{.*#+}} ymm3 = ymm3[0],ymm1[1,2,3],ymm3[4],ymm1[5,6,7],ymm3[8],ymm1[9,10,11],ymm3[12],ymm1[13,14,15]
+; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1,2,3],ymm2[4],ymm1[5,6,7],ymm2[8],ymm1[9,10,11],ymm2[12],ymm1[13,14,15]
+; AVX2-NEXT: vpackusdw %ymm1, %ymm3, %ymm1
+; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
+; AVX2-NEXT: vpackusdw %ymm0, %ymm1, %ymm0
+; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
; AVX2-NEXT: retq
;
; AVX512-LABEL: abd_ext_v16i16:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpminsw %ymm1, %ymm0, %ymm2
-; AVX512-NEXT: vpmaxsw %ymm1, %ymm0, %ymm0
-; AVX512-NEXT: vpsubw %ymm2, %ymm0, %ymm0
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm2
+; AVX512-NEXT: vpmovsxwq %xmm2, %zmm2
+; AVX512-NEXT: vpmovsxwq %xmm0, %zmm0
+; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm3
+; AVX512-NEXT: vpmovsxwq %xmm3, %zmm3
+; AVX512-NEXT: vpsubq %zmm3, %zmm2, %zmm2
+; AVX512-NEXT: vpmovsxwq %xmm1, %zmm1
+; AVX512-NEXT: vpsubq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpsubq %zmm2, %zmm1, %zmm3
+; AVX512-NEXT: vpmaxsq %zmm3, %zmm2, %zmm2
+; AVX512-NEXT: vpsubq %zmm0, %zmm1, %zmm1
+; AVX512-NEXT: vpmaxsq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpmovqw %zmm0, %xmm0
+; AVX512-NEXT: vpmovqw %zmm2, %xmm1
+; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
; AVX512-NEXT: retq
%aext = sext <16 x i16> %a to <16 x i64>
%bext = sext <16 x i16> %b to <16 x i64>
@@ -150,29 +523,78 @@ define <16 x i16> @abd_ext_v16i16_undef(<16 x i16> %a, <16 x i16> %b) nounwind {
define <8 x i32> @abd_ext_v8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
; AVX1-LABEL: abd_ext_v8i32:
; AVX1: # %bb.0:
-; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
-; AVX1-NEXT: vpminsd %xmm2, %xmm3, %xmm4
-; AVX1-NEXT: vpmaxsd %xmm2, %xmm3, %xmm2
-; AVX1-NEXT: vpsubd %xmm4, %xmm2, %xmm2
-; AVX1-NEXT: vpminsd %xmm1, %xmm0, %xmm3
-; AVX1-NEXT: vpmaxsd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpsubd %xmm3, %xmm0, %xmm0
-; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT: vpmovsxdq %xmm0, %xmm2
+; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
+; AVX1-NEXT: vpmovsxdq %xmm3, %xmm3
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT: vpmovsxdq %xmm0, %xmm4
+; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; AVX1-NEXT: vpmovsxdq %xmm0, %xmm0
+; AVX1-NEXT: vpmovsxdq %xmm1, %xmm5
+; AVX1-NEXT: vpsubq %xmm5, %xmm2, %xmm2
+; AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[2,3,2,3]
+; AVX1-NEXT: vpmovsxdq %xmm5, %xmm5
+; AVX1-NEXT: vpsubq %xmm5, %xmm3, %xmm3
+; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
+; AVX1-NEXT: vpmovsxdq %xmm1, %xmm5
+; AVX1-NEXT: vpsubq %xmm5, %xmm4, %xmm4
+; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; AVX1-NEXT: vpmovsxdq %xmm1, %xmm1
+; AVX1-NEXT: vpsubq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT: vpsubq %xmm2, %xmm1, %xmm5
+; AVX1-NEXT: vpcmpgtq %xmm5, %xmm2, %xmm5
+; AVX1-NEXT: vpxor %xmm5, %xmm2, %xmm2
+; AVX1-NEXT: vpsubq %xmm2, %xmm5, %xmm2
+; AVX1-NEXT: vpsubq %xmm3, %xmm1, %xmm5
+; AVX1-NEXT: vpcmpgtq %xmm5, %xmm3, %xmm5
+; AVX1-NEXT: vpxor %xmm5, %xmm3, %xmm3
+; AVX1-NEXT: vpsubq %xmm3, %xmm5, %xmm3
+; AVX1-NEXT: vshufps {{.*#+}} xmm2 = xmm2[0,2],xmm3[0,2]
+; AVX1-NEXT: vpsubq %xmm4, %xmm1, %xmm3
+; AVX1-NEXT: vpcmpgtq %xmm3, %xmm4, %xmm3
+; AVX1-NEXT: vpxor %xmm3, %xmm4, %xmm4
+; AVX1-NEXT: vpsubq %xmm4, %xmm3, %xmm3
+; AVX1-NEXT: vpsubq %xmm0, %xmm1, %xmm1
+; AVX1-NEXT: vpcmpgtq %xmm1, %xmm0, %xmm1
+; AVX1-NEXT: vpxor %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpsubq %xmm0, %xmm1, %xmm0
+; AVX1-NEXT: vshufps {{.*#+}} xmm0 = xmm3[0,2],xmm0[0,2]
+; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm2, %ymm0
; AVX1-NEXT: retq
;
; AVX2-LABEL: abd_ext_v8i32:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpminsd %ymm1, %ymm0, %ymm2
-; AVX2-NEXT: vpmaxsd %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: vpsubd %ymm2, %ymm0, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT: vpmovsxdq %xmm2, %ymm2
+; AVX2-NEXT: vpmovsxdq %xmm0, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm3
+; AVX2-NEXT: vpmovsxdq %xmm3, %ymm3
+; AVX2-NEXT: vpsubq %ymm3, %ymm2, %ymm2
+; AVX2-NEXT: vpmovsxdq %xmm1, %ymm1
+; AVX2-NEXT: vpsubq %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpsubq %ymm2, %ymm1, %ymm3
+; AVX2-NEXT: vpcmpgtq %ymm3, %ymm2, %ymm3
+; AVX2-NEXT: vpxor %ymm3, %ymm2, %ymm2
+; AVX2-NEXT: vpsubq %ymm2, %ymm3, %ymm2
+; AVX2-NEXT: vpsubq %ymm0, %ymm1, %ymm1
+; AVX2-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm1
+; AVX2-NEXT: vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpsubq %ymm0, %ymm1, %ymm0
+; AVX2-NEXT: vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm2[0,2],ymm0[4,6],ymm2[4,6]
+; AVX2-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
; AVX2-NEXT: retq
;
; AVX512-LABEL: abd_ext_v8i32:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpminsd %ymm1, %ymm0, %ymm2
-; AVX512-NEXT: vpmaxsd %ymm1, %ymm0, %ymm0
-; AVX512-NEXT: vpsubd %ymm2, %ymm0, %ymm0
+; AVX512-NEXT: vpmovsxdq %ymm0, %zmm0
+; AVX512-NEXT: vpmovsxdq %ymm1, %zmm1
+; AVX512-NEXT: vpsubq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpsubq %zmm0, %zmm1, %zmm1
+; AVX512-NEXT: vpmaxsq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpmovqd %zmm0, %ymm0
; AVX512-NEXT: retq
%aext = sext <8 x i32> %a to <8 x i64>
%bext = sext <8 x i32> %b to <8 x i64>
@@ -220,32 +642,257 @@ define <8 x i32> @abd_ext_v8i32_undef(<8 x i32> %a, <8 x i32> %b) nounwind {
define <4 x i64> @abd_ext_v4i64(<4 x i64> %a, <4 x i64> %b) nounwind {
; AVX1-LABEL: abd_ext_v4i64:
; AVX1: # %bb.0:
-; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
-; AVX1-NEXT: vpcmpgtq %xmm2, %xmm3, %xmm4
-; AVX1-NEXT: vpsubq %xmm2, %xmm3, %xmm2
-; AVX1-NEXT: vpxor %xmm4, %xmm2, %xmm2
-; AVX1-NEXT: vpsubq %xmm2, %xmm4, %xmm2
-; AVX1-NEXT: vpcmpgtq %xmm1, %xmm0, %xmm3
-; AVX1-NEXT: vpsubq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpxor %xmm3, %xmm0, %xmm0
-; AVX1-NEXT: vpsubq %xmm0, %xmm3, %xmm0
-; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT: pushq %rbp
+; AVX1-NEXT: pushq %r15
+; AVX1-NEXT: pushq %r14
+; AVX1-NEXT: pushq %r13
+; AVX1-NEXT: pushq %r12
+; AVX1-NEXT: pushq %rbx
+; AVX1-NEXT: vmovq %xmm0, %r9
+; AVX1-NEXT: movq %r9, %r10
+; AVX1-NEXT: sarq $63, %r10
+; AVX1-NEXT: vpextrq $1, %xmm0, %rdi
+; AVX1-NEXT: movq %rdi, %r8
+; AVX1-NEXT: sarq $63, %r8
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT: vmovq %xmm0, %rdx
+; AVX1-NEXT: movq %rdx, %rsi
+; AVX1-NEXT: sarq $63, %rsi
+; AVX1-NEXT: vpextrq $1, %xmm0, %r14
+; AVX1-NEXT: movq %r14, %rcx
+; AVX1-NEXT: sarq $63, %rcx
+; AVX1-NEXT: vmovq %xmm1, %rbx
+; AVX1-NEXT: movq %rbx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX1-NEXT: sarq $63, %rbx
+; AVX1-NEXT: vpextrq $1, %xmm1, %r11
+; AVX1-NEXT: movq %r11, %r15
+; AVX1-NEXT: sarq $63, %r15
+; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm0
+; AVX1-NEXT: vmovq %xmm0, %r12
+; AVX1-NEXT: movq %r12, %r13
+; AVX1-NEXT: sarq $63, %r13
+; AVX1-NEXT: vpextrq $1, %xmm0, %rax
+; AVX1-NEXT: movq %rax, %rbp
+; AVX1-NEXT: sarq $63, %rbp
+; AVX1-NEXT: subq %rax, %r14
+; AVX1-NEXT: sbbq %rbp, %rcx
+; AVX1-NEXT: subq %r12, %rdx
+; AVX1-NEXT: sbbq %r13, %rsi
+; AVX1-NEXT: subq %r11, %rdi
+; AVX1-NEXT: sbbq %r15, %r8
+; AVX1-NEXT: subq {{[-0-9]+}}(%r{{[sb]}}p), %r9 # 8-byte Folded Reload
+; AVX1-NEXT: sbbq %rbx, %r10
+; AVX1-NEXT: xorl %ebx, %ebx
+; AVX1-NEXT: movq %r9, %r11
+; AVX1-NEXT: negq %r11
+; AVX1-NEXT: movl $0, %eax
+; AVX1-NEXT: sbbq %r10, %rax
+; AVX1-NEXT: cmpq %r9, %r11
+; AVX1-NEXT: sbbq %r10, %rax
+; AVX1-NEXT: cmovlq %r9, %r11
+; AVX1-NEXT: movq %rdi, %rax
+; AVX1-NEXT: negq %rax
+; AVX1-NEXT: movl $0, %r9d
+; AVX1-NEXT: sbbq %r8, %r9
+; AVX1-NEXT: cmpq %rdi, %rax
+; AVX1-NEXT: sbbq %r8, %r9
+; AVX1-NEXT: cmovlq %rdi, %rax
+; AVX1-NEXT: movq %rdx, %rdi
+; AVX1-NEXT: negq %rdi
+; AVX1-NEXT: movl $0, %r8d
+; AVX1-NEXT: sbbq %rsi, %r8
+; AVX1-NEXT: cmpq %rdx, %rdi
+; AVX1-NEXT: sbbq %rsi, %r8
+; AVX1-NEXT: cmovlq %rdx, %rdi
+; AVX1-NEXT: movq %r14, %rdx
+; AVX1-NEXT: negq %rdx
+; AVX1-NEXT: sbbq %rcx, %rbx
+; AVX1-NEXT: cmpq %r14, %rdx
+; AVX1-NEXT: sbbq %rcx, %rbx
+; AVX1-NEXT: cmovlq %r14, %rdx
+; AVX1-NEXT: vmovq %rdx, %xmm0
+; AVX1-NEXT: vmovq %rdi, %xmm1
+; AVX1-NEXT: vmovq %rax, %xmm2
+; AVX1-NEXT: vmovq %r11, %xmm3
+; AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm3[0],xmm2[0]
+; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT: popq %rbx
+; AVX1-NEXT: popq %r12
+; AVX1-NEXT: popq %r13
+; AVX1-NEXT: popq %r14
+; AVX1-NEXT: popq %r15
+; AVX1-NEXT: popq %rbp
; AVX1-NEXT: retq
;
; AVX2-LABEL: abd_ext_v4i64:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm2
-; AVX2-NEXT: vpsubq %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: vpxor %ymm2, %ymm0, %ymm0
-; AVX2-NEXT: vpsubq %ymm0, %ymm2, %ymm0
+; AVX2-NEXT: pushq %rbp
+; AVX2-NEXT: pushq %r15
+; AVX2-NEXT: pushq %r14
+; AVX2-NEXT: pushq %r13
+; AVX2-NEXT: pushq %r12
+; AVX2-NEXT: pushq %rbx
+; AVX2-NEXT: vmovq %xmm0, %r9
+; AVX2-NEXT: movq %r9, %r10
+; AVX2-NEXT: sarq $63, %r10
+; AVX2-NEXT: vpextrq $1, %xmm0, %rdi
+; AVX2-NEXT: movq %rdi, %r8
+; AVX2-NEXT: sarq $63, %r8
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX2-NEXT: vmovq %xmm0, %rdx
+; AVX2-NEXT: movq %rdx, %rsi
+; AVX2-NEXT: sarq $63, %rsi
+; AVX2-NEXT: vpextrq $1, %xmm0, %r14
+; AVX2-NEXT: movq %r14, %rcx
+; AVX2-NEXT: sarq $63, %rcx
+; AVX2-NEXT: vmovq %xmm1, %rbx
+; AVX2-NEXT: movq %rbx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: sarq $63, %rbx
+; AVX2-NEXT: vpextrq $1, %xmm1, %r11
+; AVX2-NEXT: movq %r11, %r15
+; AVX2-NEXT: sarq $63, %r15
+; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm0
+; AVX2-NEXT: vmovq %xmm0, %r12
+; AVX2-NEXT: movq %r12, %r13
+; AVX2-NEXT: sarq $63, %r13
+; AVX2-NEXT: vpextrq $1, %xmm0, %rax
+; AVX2-NEXT: movq %rax, %rbp
+; AVX2-NEXT: sarq $63, %rbp
+; AVX2-NEXT: subq %rax, %r14
+; AVX2-NEXT: sbbq %rbp, %rcx
+; AVX2-NEXT: subq %r12, %rdx
+; AVX2-NEXT: sbbq %r13, %rsi
+; AVX2-NEXT: subq %r11, %rdi
+; AVX2-NEXT: sbbq %r15, %r8
+; AVX2-NEXT: subq {{[-0-9]+}}(%r{{[sb]}}p), %r9 # 8-byte Folded Reload
+; AVX2-NEXT: sbbq %rbx, %r10
+; AVX2-NEXT: xorl %ebx, %ebx
+; AVX2-NEXT: movq %r9, %r11
+; AVX2-NEXT: negq %r11
+; AVX2-NEXT: movl $0, %eax
+; AVX2-NEXT: sbbq %r10, %rax
+; AVX2-NEXT: cmpq %r9, %r11
+; AVX2-NEXT: sbbq %r10, %rax
+; AVX2-NEXT: cmovlq %r9, %r11
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: negq %rax
+; AVX2-NEXT: movl $0, %r9d
+; AVX2-NEXT: sbbq %r8, %r9
+; AVX2-NEXT: cmpq %rdi, %rax
+; AVX2-NEXT: sbbq %r8, %r9
+; AVX2-NEXT: cmovlq %rdi, %rax
+; AVX2-NEXT: movq %rdx, %rdi
+; AVX2-NEXT: negq %rdi
+; AVX2-NEXT: movl $0, %r8d
+; AVX2-NEXT: sbbq %rsi, %r8
+; AVX2-NEXT: cmpq %rdx, %rdi
+; AVX2-NEXT: sbbq %rsi, %r8
+; AVX2-NEXT: cmovlq %rdx, %rdi
+; AVX2-NEXT: movq %r14, %rdx
+; AVX2-NEXT: negq %rdx
+; AVX2-NEXT: sbbq %rcx, %rbx
+; AVX2-NEXT: cmpq %r14, %rdx
+; AVX2-NEXT: sbbq %rcx, %rbx
+; AVX2-NEXT: cmovlq %r14, %rdx
+; AVX2-NEXT: vmovq %rdx, %xmm0
+; AVX2-NEXT: vmovq %rdi, %xmm1
+; AVX2-NEXT: vmovq %rax, %xmm2
+; AVX2-NEXT: vmovq %r11, %xmm3
+; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm3[0],xmm2[0]
+; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX2-NEXT: popq %rbx
+; AVX2-NEXT: popq %r12
+; AVX2-NEXT: popq %r13
+; AVX2-NEXT: popq %r14
+; AVX2-NEXT: popq %r15
+; AVX2-NEXT: popq %rbp
; AVX2-NEXT: retq
;
; AVX512-LABEL: abd_ext_v4i64:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpminsq %ymm1, %ymm0, %ymm2
-; AVX512-NEXT: vpmaxsq %ymm1, %ymm0, %ymm0
-; AVX512-NEXT: vpsubq %ymm2, %ymm0, %ymm0
+; AVX512-NEXT: pushq %rbp
+; AVX512-NEXT: pushq %r15
+; AVX512-NEXT: pushq %r14
+; AVX512-NEXT: pushq %r13
+; AVX512-NEXT: pushq %r12
+; AVX512-NEXT: pushq %rbx
+; AVX512-NEXT: vmovq %xmm0, %r9
+; AVX512-NEXT: movq %r9, %r10
+; AVX512-NEXT: sarq $63, %r10
+; AVX512-NEXT: vpextrq $1, %xmm0, %rdi
+; AVX512-NEXT: movq %rdi, %r8
+; AVX512-NEXT: sarq $63, %r8
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX512-NEXT: vmovq %xmm0, %rdx
+; AVX512-NEXT: movq %rdx, %rsi
+; AVX512-NEXT: sarq $63, %rsi
+; AVX512-NEXT: vpextrq $1, %xmm0, %r14
+; AVX512-NEXT: movq %r14, %rcx
+; AVX512-NEXT: sarq $63, %rcx
+; AVX512-NEXT: vmovq %xmm1, %rbx
+; AVX512-NEXT: movq %rbx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT: sarq $63, %rbx
+; AVX512-NEXT: vpextrq $1, %xmm1, %r11
+; AVX512-NEXT: movq %r11, %r15
+; AVX512-NEXT: sarq $63, %r15
+; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm0
+; AVX512-NEXT: vmovq %xmm0, %r12
+; AVX512-NEXT: movq %r12, %r13
+; AVX512-NEXT: sarq $63, %r13
+; AVX512-NEXT: vpextrq $1, %xmm0, %rax
+; AVX512-NEXT: movq %rax, %rbp
+; AVX512-NEXT: sarq $63, %rbp
+; AVX512-NEXT: subq %rax, %r14
+; AVX512-NEXT: sbbq %rbp, %rcx
+; AVX512-NEXT: subq %r12, %rdx
+; AVX512-NEXT: sbbq %r13, %rsi
+; AVX512-NEXT: subq %r11, %rdi
+; AVX512-NEXT: sbbq %r15, %r8
+; AVX512-NEXT: subq {{[-0-9]+}}(%r{{[sb]}}p), %r9 # 8-byte Folded Reload
+; AVX512-NEXT: sbbq %rbx, %r10
+; AVX512-NEXT: xorl %ebx, %ebx
+; AVX512-NEXT: movq %r9, %r11
+; AVX512-NEXT: negq %r11
+; AVX512-NEXT: movl $0, %eax
+; AVX512-NEXT: sbbq %r10, %rax
+; AVX512-NEXT: cmpq %r9, %r11
+; AVX512-NEXT: sbbq %r10, %rax
+; AVX512-NEXT: cmovlq %r9, %r11
+; AVX512-NEXT: movq %rdi, %rax
+; AVX512-NEXT: negq %rax
+; AVX512-NEXT: movl $0, %r9d
+; AVX512-NEXT: sbbq %r8, %r9
+; AVX512-NEXT: cmpq %rdi, %rax
+; AVX512-NEXT: sbbq %r8, %r9
+; AVX512-NEXT: cmovlq %rdi, %rax
+; AVX512-NEXT: movq %rdx, %rdi
+; AVX512-NEXT: negq %rdi
+; AVX512-NEXT: movl $0, %r8d
+; AVX512-NEXT: sbbq %rsi, %r8
+; AVX512-NEXT: cmpq %rdx, %rdi
+; AVX512-NEXT: sbbq %rsi, %r8
+; AVX512-NEXT: cmovlq %rdx, %rdi
+; AVX512-NEXT: movq %r14, %rdx
+; AVX512-NEXT: negq %rdx
+; AVX512-NEXT: sbbq %rcx, %rbx
+; AVX512-NEXT: cmpq %r14, %rdx
+; AVX512-NEXT: sbbq %rcx, %rbx
+; AVX512-NEXT: cmovlq %r14, %rdx
+; AVX512-NEXT: vmovq %rdx, %xmm0
+; AVX512-NEXT: vmovq %rdi, %xmm1
+; AVX512-NEXT: vmovq %rax, %xmm2
+; AVX512-NEXT: vmovq %r11, %xmm3
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm3[0],xmm2[0]
+; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512-NEXT: popq %rbx
+; AVX512-NEXT: popq %r12
+; AVX512-NEXT: popq %r13
+; AVX512-NEXT: popq %r14
+; AVX512-NEXT: popq %r15
+; AVX512-NEXT: popq %rbp
; AVX512-NEXT: retq
%aext = sext <4 x i64> %a to <4 x i128>
%bext = sext <4 x i64> %b to <4 x i128>
@@ -582,25 +1229,32 @@ define <4 x i64> @abd_cmp_v4i64(<4 x i64> %a, <4 x i64> %b) nounwind {
define <32 x i8> @abd_subnsw_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
; AVX1-LABEL: abd_subnsw_v32i8:
; AVX1: # %bb.0:
-; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
-; AVX1-NEXT: vpsubb %xmm2, %xmm3, %xmm2
+; AVX1-NEXT: vpsubb %xmm1, %xmm0, %xmm2
+; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
; AVX1-NEXT: vpsubb %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpabsb %xmm0, %xmm0
-; AVX1-NEXT: vpabsb %xmm2, %xmm1
-; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT: vpsubb %xmm0, %xmm1, %xmm3
+; AVX1-NEXT: vpmaxsb %xmm3, %xmm0, %xmm0
+; AVX1-NEXT: vpsubb %xmm2, %xmm1, %xmm1
+; AVX1-NEXT: vpmaxsb %xmm1, %xmm2, %xmm1
+; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0
; AVX1-NEXT: retq
;
; AVX2-LABEL: abd_subnsw_v32i8:
; AVX2: # %bb.0:
; AVX2-NEXT: vpsubb %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: vpabsb %ymm0, %ymm0
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpsubb %ymm0, %ymm1, %ymm1
+; AVX2-NEXT: vpmaxsb %ymm1, %ymm0, %ymm0
; AVX2-NEXT: retq
;
; AVX512-LABEL: abd_subnsw_v32i8:
; AVX512: # %bb.0:
; AVX512-NEXT: vpsubb %ymm1, %ymm0, %ymm0
-; AVX512-NEXT: vpabsb %ymm0, %ymm0
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpsubb %ymm0, %ymm1, %ymm1
+; AVX512-NEXT: vpmaxsb %ymm1, %ymm0, %ymm0
; AVX512-NEXT: retq
%sub = sub nsw <32 x i8> %a, %b
%abs = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %sub, i1 false)
@@ -610,25 +1264,32 @@ define <32 x i8> @abd_subnsw_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
define <16 x i16> @abd_subnsw_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind {
; AVX1-LABEL: abd_subnsw_v16i16:
; AVX1: # %bb.0:
-; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
-; AVX1-NEXT: vpsubw %xmm2, %xmm3, %xmm2
+; AVX1-NEXT: vpsubw %xmm1, %xmm0, %xmm2
+; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
; AVX1-NEXT: vpsubw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpabsw %xmm0, %xmm0
-; AVX1-NEXT: vpabsw %xmm2, %xmm1
-; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT: vpsubw %xmm0, %xmm1, %xmm3
+; AVX1-NEXT: vpmaxsw %xmm3, %xmm0, %xmm0
+; AVX1-NEXT: vpsubw %xmm2, %xmm1, %xmm1
+; AVX1-NEXT: vpmaxsw %xmm1, %xmm2, %xmm1
+; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0
; AVX1-NEXT: retq
;
; AVX2-LABEL: abd_subnsw_v16i16:
; AVX2: # %bb.0:
; AVX2-NEXT: vpsubw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: vpabsw %ymm0, %ymm0
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpsubw %ymm0, %ymm1, %ymm1
+; AVX2-NEXT: vpmaxsw %ymm1, %ymm0, %ymm0
; AVX2-NEXT: retq
;
; AVX512-LABEL: abd_subnsw_v16i16:
; AVX512: # %bb.0:
; AVX512-NEXT: vpsubw %ymm1, %ymm0, %ymm0
-; AVX512-NEXT: vpabsw %ymm0, %ymm0
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpsubw %ymm0, %ymm1, %ymm1
+; AVX512-NEXT: vpmaxsw %ymm1, %ymm0, %ymm0
; AVX512-NEXT: retq
%sub = sub nsw <16 x i16> %a, %b
%abs = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %sub, i1 false)
@@ -638,25 +1299,32 @@ define <16 x i16> @abd_subnsw_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind {
define <8 x i32> @abd_subnsw_v8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
; AVX1-LABEL: abd_subnsw_v8i32:
; AVX1: # %bb.0:
-; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
-; AVX1-NEXT: vpsubd %xmm2, %xmm3, %xmm2
+; AVX1-NEXT: vpsubd %xmm1, %xmm0, %xmm2
+; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
; AVX1-NEXT: vpsubd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpabsd %xmm0, %xmm0
-; AVX1-NEXT: vpabsd %xmm2, %xmm1
-; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
+; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT: vpsubd %xmm0, %xmm1, %xmm3
+; AVX1-NEXT: vpmaxsd %xmm3, %xmm0, %xmm0
+; AVX1-NEXT: vpsubd %xmm2, %xmm1, %xmm1
+; AVX1-NEXT: vpmaxsd %xmm1, %xmm2, %xmm1
+; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0
; AVX1-NEXT: retq
;
; AVX2-LABEL: abd_subnsw_v8i32:
; AVX2: # %bb.0:
; AVX2-NEXT: vpsubd %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: vpabsd %ymm0, %ymm0
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpsubd %ymm0, %ymm1, %ymm1
+; AVX2-NEXT: vpmaxsd %ymm1, %ymm0, %ymm0
; AVX2-NEXT: retq
;
; AVX512-LABEL: abd_subnsw_v8i32:
; AVX512: # %bb.0:
; AVX512-NEXT: vpsubd %ymm1, %ymm0, %ymm0
-; AVX512-NEXT: vpabsd %ymm0, %ymm0
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpsubd %ymm0, %ymm1, %ymm1
+; AVX512-NEXT: vpmaxsd %ymm1, %ymm0, %ymm0
; AVX512-NEXT: retq
%sub = sub nsw <8 x i32> %a, %b
%abs = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %sub, i1 false)
@@ -666,16 +1334,20 @@ define <8 x i32> @abd_subnsw_v8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
define <4 x i64> @abd_subnsw_v4i64(<4 x i64> %a, <4 x i64> %b) nounwind {
; AVX1-LABEL: abd_subnsw_v4i64:
; AVX1: # %bb.0:
-; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
-; AVX1-NEXT: vpsubq %xmm2, %xmm3, %xmm2
+; AVX1-NEXT: vpsubq %xmm1, %xmm0, %xmm2
+; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
; AVX1-NEXT: vpsubq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm1
-; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT: vpsubq %xmm2, %xmm3, %xmm2
+; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT: vpsubq %xmm0, %xmm1, %xmm3
+; AVX1-NEXT: vpcmpgtq %xmm3, %xmm0, %xmm3
+; AVX1-NEXT: vpxor %xmm3, %xmm0, %xmm0
; AVX1-NEXT: vpsubq %xmm0, %xmm3, %xmm0
-; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; AVX1-NEXT: vblendvpd %ymm1, %ymm0, %ymm1, %ymm0
+; AVX1-NEXT: vpsubq %xmm2, %xmm1, %xmm1
+; AVX1-NEXT: vpcmpgtq %xmm1, %xmm2, %xmm1
+; AVX1-NEXT: vpxor %xmm1, %xmm2, %xmm2
+; AVX1-NEXT: vpsubq %xmm2, %xmm1, %xmm1
+; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0
; AVX1-NEXT: retq
;
; AVX2-LABEL: abd_subnsw_v4i64:
@@ -683,13 +1355,17 @@ define <4 x i64> @abd_subnsw_v4i64(<4 x i64> %a, <4 x i64> %b) nounwind {
; AVX2-NEXT: vpsubq %ymm1, %ymm0, %ymm0
; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX2-NEXT: vpsubq %ymm0, %ymm1, %ymm1
-; AVX2-NEXT: vblendvpd %ymm0, %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm1
+; AVX2-NEXT: vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpsubq %ymm0, %ymm1, %ymm0
; AVX2-NEXT: retq
;
; AVX512-LABEL: abd_subnsw_v4i64:
; AVX512: # %bb.0:
; AVX512-NEXT: vpsubq %ymm1, %ymm0, %ymm0
-; AVX512-NEXT: vpabsq %ymm0, %ymm0
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpsubq %ymm0, %ymm1, %ymm1
+; AVX512-NEXT: vpmaxsq %ymm1, %ymm0, %ymm0
; AVX512-NEXT: retq
%sub = sub nsw <4 x i64> %a, %b
%abs = call <4 x i64> @llvm.abs.v4i64(<4 x i64> %sub, i1 false)
diff --git a/llvm/test/CodeGen/X86/abds-vector-512.ll b/llvm/test/CodeGen/X86/abds-vector-512.ll
index 359b962a152b2..2a142a78fa7ac 100644
--- a/llvm/test/CodeGen/X86/abds-vector-512.ll
+++ b/llvm/test/CodeGen/X86/abds-vector-512.ll
@@ -7,25 +7,87 @@
;
define <64 x i8> @abd_ext_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
-; AVX512BW-LABEL: abd_ext_v64i8:
-; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpminsb %zmm1, %zmm0, %zmm2
-; AVX512BW-NEXT: vpmaxsb %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT: vpsubb %zmm2, %zmm0, %zmm0
-; AVX512BW-NEXT: retq
-;
-; AVX512DQ-LABEL: abd_ext_v64i8:
-; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vextracti64x4 $1, %zmm1, %ymm2
-; AVX512DQ-NEXT: vextracti64x4 $1, %zmm0, %ymm3
-; AVX512DQ-NEXT: vpminsb %ymm2, %ymm3, %ymm4
-; AVX512DQ-NEXT: vpmaxsb %ymm2, %ymm3, %ymm2
-; AVX512DQ-NEXT: vpsubb %ymm4, %ymm2, %ymm2
-; AVX512DQ-NEXT: vpminsb %ymm1, %ymm0, %ymm3
-; AVX512DQ-NEXT: vpmaxsb %ymm1, %ymm0, %ymm0
-; AVX512DQ-NEXT: vpsubb %ymm3, %ymm0, %ymm0
-; AVX512DQ-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX512DQ-NEXT: retq
+; AVX512-LABEL: abd_ext_v64i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm2
+; AVX512-NEXT: vpmovsxbw %xmm2, %ymm2
+; AVX512-NEXT: vpmovsxwq %xmm2, %zmm3
+; AVX512-NEXT: vpmovsxbw %xmm0, %ymm4
+; AVX512-NEXT: vpmovsxwq %xmm4, %zmm5
+; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm0
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm6
+; AVX512-NEXT: vpmovsxbw %xmm6, %ymm6
+; AVX512-NEXT: vpmovsxwq %xmm6, %zmm7
+; AVX512-NEXT: vpmovsxbw %xmm0, %ymm0
+; AVX512-NEXT: vpmovsxwq %xmm0, %zmm8
+; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm2
+; AVX512-NEXT: vpmovsxwq %xmm2, %zmm2
+; AVX512-NEXT: vextracti128 $1, %ymm4, %xmm4
+; AVX512-NEXT: vpmovsxwq %xmm4, %zmm4
+; AVX512-NEXT: vextracti128 $1, %ymm6, %xmm6
+; AVX512-NEXT: vpmovsxwq %xmm6, %zmm6
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX512-NEXT: vpmovsxwq %xmm0, %zmm0
+; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm9
+; AVX512-NEXT: vpmovsxbw %xmm9, %ymm9
+; AVX512-NEXT: vpmovsxwq %xmm9, %zmm10
+; AVX512-NEXT: vpsubq %zmm10, %zmm3, %zmm3
+; AVX512-NEXT: vpmovsxbw %xmm1, %ymm10
+; AVX512-NEXT: vpmovsxwq %xmm10, %zmm11
+; AVX512-NEXT: vpsubq %zmm11, %zmm5, %zmm5
+; AVX512-NEXT: vextracti64x4 $1, %zmm1, %ymm1
+; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm11
+; AVX512-NEXT: vpmovsxbw %xmm11, %ymm11
+; AVX512-NEXT: vpmovsxwq %xmm11, %zmm12
+; AVX512-NEXT: vpsubq %zmm12, %zmm7, %zmm7
+; AVX512-NEXT: vpmovsxbw %xmm1, %ymm1
+; AVX512-NEXT: vpmovsxwq %xmm1, %zmm12
+; AVX512-NEXT: vpsubq %zmm12, %zmm8, %zmm8
+; AVX512-NEXT: vextracti128 $1, %ymm9, %xmm9
+; AVX512-NEXT: vpmovsxwq %xmm9, %zmm9
+; AVX512-NEXT: vpsubq %zmm9, %zmm2, %zmm2
+; AVX512-NEXT: vextracti128 $1, %ymm10, %xmm9
+; AVX512-NEXT: vpmovsxwq %xmm9, %zmm9
+; AVX512-NEXT: vpsubq %zmm9, %zmm4, %zmm4
+; AVX512-NEXT: vextracti128 $1, %ymm11, %xmm9
+; AVX512-NEXT: vpmovsxwq %xmm9, %zmm9
+; AVX512-NEXT: vpsubq %zmm9, %zmm6, %zmm6
+; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm1
+; AVX512-NEXT: vpmovsxwq %xmm1, %zmm1
+; AVX512-NEXT: vpsubq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpsubq %zmm3, %zmm1, %zmm9
+; AVX512-NEXT: vpmaxsq %zmm9, %zmm3, %zmm3
+; AVX512-NEXT: vpsubq %zmm5, %zmm1, %zmm9
+; AVX512-NEXT: vpmaxsq %zmm9, %zmm5, %zmm5
+; AVX512-NEXT: vpsubq %zmm7, %zmm1, %zmm9
+; AVX512-NEXT: vpmaxsq %zmm9, %zmm7, %zmm7
+; AVX512-NEXT: vpsubq %zmm8, %zmm1, %zmm9
+; AVX512-NEXT: vpmaxsq %zmm9, %zmm8, %zmm8
+; AVX512-NEXT: vpsubq %zmm2, %zmm1, %zmm9
+; AVX512-NEXT: vpmaxsq %zmm9, %zmm2, %zmm2
+; AVX512-NEXT: vpsubq %zmm4, %zmm1, %zmm9
+; AVX512-NEXT: vpmaxsq %zmm9, %zmm4, %zmm4
+; AVX512-NEXT: vpsubq %zmm6, %zmm1, %zmm9
+; AVX512-NEXT: vpmaxsq %zmm9, %zmm6, %zmm6
+; AVX512-NEXT: vpsubq %zmm0, %zmm1, %zmm1
+; AVX512-NEXT: vpmaxsq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpmovqb %zmm0, %xmm0
+; AVX512-NEXT: vpmovqb %zmm6, %xmm1
+; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512-NEXT: vpmovqb %zmm4, %xmm1
+; AVX512-NEXT: vpmovqb %zmm2, %xmm2
+; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512-NEXT: vpmovqb %zmm8, %xmm1
+; AVX512-NEXT: vpmovqb %zmm7, %xmm2
+; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512-NEXT: vpmovqb %zmm5, %xmm2
+; AVX512-NEXT: vpmovqb %zmm3, %xmm3
+; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
+; AVX512-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} zmm0 = zmm1[0],zmm0[0],zmm1[2],zmm0[2],zmm1[4],zmm0[4],zmm1[6],zmm0[6]
+; AVX512-NEXT: retq
%aext = sext <64 x i8> %a to <64 x i64>
%bext = sext <64 x i8> %b to <64 x i64>
%sub = sub <64 x i64> %aext, %bext
@@ -63,25 +125,43 @@ define <64 x i8> @abd_ext_v64i8_undef(<64 x i8> %a, <64 x i8> %b) nounwind {
}
define <32 x i16> @abd_ext_v32i16(<32 x i16> %a, <32 x i16> %b) nounwind {
-; AVX512BW-LABEL: abd_ext_v32i16:
-; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpminsw %zmm1, %zmm0, %zmm2
-; AVX512BW-NEXT: vpmaxsw %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT: vpsubw %zmm2, %zmm0, %zmm0
-; AVX512BW-NEXT: retq
-;
-; AVX512DQ-LABEL: abd_ext_v32i16:
-; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vextracti64x4 $1, %zmm1, %ymm2
-; AVX512DQ-NEXT: vextracti64x4 $1, %zmm0, %ymm3
-; AVX512DQ-NEXT: vpminsw %ymm2, %ymm3, %ymm4
-; AVX512DQ-NEXT: vpmaxsw %ymm2, %ymm3, %ymm2
-; AVX512DQ-NEXT: vpsubw %ymm4, %ymm2, %ymm2
-; AVX512DQ-NEXT: vpminsw %ymm1, %ymm0, %ymm3
-; AVX512DQ-NEXT: vpmaxsw %ymm1, %ymm0, %ymm0
-; AVX512DQ-NEXT: vpsubw %ymm3, %ymm0, %ymm0
-; AVX512DQ-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX512DQ-NEXT: retq
+; AVX512-LABEL: abd_ext_v32i16:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm2
+; AVX512-NEXT: vpmovsxwq %xmm2, %zmm2
+; AVX512-NEXT: vpmovsxwq %xmm0, %zmm3
+; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm0
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm4
+; AVX512-NEXT: vpmovsxwq %xmm4, %zmm4
+; AVX512-NEXT: vpmovsxwq %xmm0, %zmm0
+; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm5
+; AVX512-NEXT: vpmovsxwq %xmm5, %zmm5
+; AVX512-NEXT: vpsubq %zmm5, %zmm2, %zmm2
+; AVX512-NEXT: vpmovsxwq %xmm1, %zmm5
+; AVX512-NEXT: vpsubq %zmm5, %zmm3, %zmm3
+; AVX512-NEXT: vextracti64x4 $1, %zmm1, %ymm1
+; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm5
+; AVX512-NEXT: vpmovsxwq %xmm5, %zmm5
+; AVX512-NEXT: vpsubq %zmm5, %zmm4, %zmm4
+; AVX512-NEXT: vpmovsxwq %xmm1, %zmm1
+; AVX512-NEXT: vpsubq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpsubq %zmm2, %zmm1, %zmm5
+; AVX512-NEXT: vpmaxsq %zmm5, %zmm2, %zmm2
+; AVX512-NEXT: vpsubq %zmm3, %zmm1, %zmm5
+; AVX512-NEXT: vpmaxsq %zmm5, %zmm3, %zmm3
+; AVX512-NEXT: vpsubq %zmm4, %zmm1, %zmm5
+; AVX512-NEXT: vpmaxsq %zmm5, %zmm4, %zmm4
+; AVX512-NEXT: vpsubq %zmm0, %zmm1, %zmm1
+; AVX512-NEXT: vpmaxsq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpmovqw %zmm0, %xmm0
+; AVX512-NEXT: vpmovqw %zmm4, %xmm1
+; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512-NEXT: vpmovqw %zmm3, %xmm1
+; AVX512-NEXT: vpmovqw %zmm2, %xmm2
+; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512-NEXT: retq
%aext = sext <32 x i16> %a to <32 x i64>
%bext = sext <32 x i16> %b to <32 x i64>
%sub = sub <32 x i64> %aext, %bext
@@ -121,9 +201,22 @@ define <32 x i16> @abd_ext_v32i16_undef(<32 x i16> %a, <32 x i16> %b) nounwind {
define <16 x i32> @abd_ext_v16i32(<16 x i32> %a, <16 x i32> %b) nounwind {
; AVX512-LABEL: abd_ext_v16i32:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpminsd %zmm1, %zmm0, %zmm2
-; AVX512-NEXT: vpmaxsd %zmm1, %zmm0, %zmm0
-; AVX512-NEXT: vpsubd %zmm2, %zmm0, %zmm0
+; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm2
+; AVX512-NEXT: vpmovsxdq %ymm2, %zmm2
+; AVX512-NEXT: vpmovsxdq %ymm0, %zmm0
+; AVX512-NEXT: vextracti64x4 $1, %zmm1, %ymm3
+; AVX512-NEXT: vpmovsxdq %ymm3, %zmm3
+; AVX512-NEXT: vpsubq %zmm3, %zmm2, %zmm2
+; AVX512-NEXT: vpmovsxdq %ymm1, %zmm1
+; AVX512-NEXT: vpsubq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpsubq %zmm2, %zmm1, %zmm3
+; AVX512-NEXT: vpmaxsq %zmm3, %zmm2, %zmm2
+; AVX512-NEXT: vpsubq %zmm0, %zmm1, %zmm1
+; AVX512-NEXT: vpmaxsq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpmovqd %zmm0, %ymm0
+; AVX512-NEXT: vpmovqd %zmm2, %ymm1
+; AVX512-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
; AVX512-NEXT: retq
%aext = sext <16 x i32> %a to <16 x i64>
%bext = sext <16 x i32> %b to <16 x i64>
@@ -151,9 +244,180 @@ define <16 x i32> @abd_ext_v16i32_undef(<16 x i32> %a, <16 x i32> %b) nounwind {
define <8 x i64> @abd_ext_v8i64(<8 x i64> %a, <8 x i64> %b) nounwind {
; AVX512-LABEL: abd_ext_v8i64:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpminsq %zmm1, %zmm0, %zmm2
-; AVX512-NEXT: vpmaxsq %zmm1, %zmm0, %zmm0
-; AVX512-NEXT: vpsubq %zmm2, %zmm0, %zmm0
+; AVX512-NEXT: pushq %rbp
+; AVX512-NEXT: pushq %r15
+; AVX512-NEXT: pushq %r14
+; AVX512-NEXT: pushq %r13
+; AVX512-NEXT: pushq %r12
+; AVX512-NEXT: pushq %rbx
+; AVX512-NEXT: pushq %rax
+; AVX512-NEXT: vmovq %xmm0, %r14
+; AVX512-NEXT: movq %r14, %rbp
+; AVX512-NEXT: sarq $63, %rbp
+; AVX512-NEXT: vpextrq $1, %xmm0, %r12
+; AVX512-NEXT: movq %r12, %r13
+; AVX512-NEXT: sarq $63, %r13
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm2
+; AVX512-NEXT: vmovq %xmm2, %rcx
+; AVX512-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT: sarq $63, %rcx
+; AVX512-NEXT: movq %rcx, (%rsp) # 8-byte Spill
+; AVX512-NEXT: vpextrq $1, %xmm2, %rax
+; AVX512-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT: sarq $63, %rax
+; AVX512-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm0
+; AVX512-NEXT: vmovq %xmm0, %rax
+; AVX512-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT: sarq $63, %rax
+; AVX512-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT: vpextrq $1, %xmm0, %rax
+; AVX512-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT: sarq $63, %rax
+; AVX512-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX512-NEXT: vmovq %xmm0, %rax
+; AVX512-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT: sarq $63, %rax
+; AVX512-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT: vpextrq $1, %xmm0, %rax
+; AVX512-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT: sarq $63, %rax
+; AVX512-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT: vmovq %xmm1, %rax
+; AVX512-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT: sarq $63, %rax
+; AVX512-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT: vpextrq $1, %xmm1, %r15
+; AVX512-NEXT: movq %r15, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT: sarq $63, %r15
+; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm0
+; AVX512-NEXT: vmovq %xmm0, %r11
+; AVX512-NEXT: movq %r11, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT: sarq $63, %r11
+; AVX512-NEXT: vpextrq $1, %xmm0, %r10
+; AVX512-NEXT: movq %r10, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT: sarq $63, %r10
+; AVX512-NEXT: vextracti64x4 $1, %zmm1, %ymm0
+; AVX512-NEXT: vmovq %xmm0, %rbx
+; AVX512-NEXT: movq %rbx, %r8
+; AVX512-NEXT: sarq $63, %r8
+; AVX512-NEXT: vpextrq $1, %xmm0, %r9
+; AVX512-NEXT: movq %r9, %rsi
+; AVX512-NEXT: sarq $63, %rsi
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX512-NEXT: vmovq %xmm0, %rdi
+; AVX512-NEXT: movq %rdi, %rcx
+; AVX512-NEXT: sarq $63, %rcx
+; AVX512-NEXT: vpextrq $1, %xmm0, %rdx
+; AVX512-NEXT: movq %rdx, %rax
+; AVX512-NEXT: sarq $63, %rax
+; AVX512-NEXT: subq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; AVX512-NEXT: sbbq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; AVX512-NEXT: subq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; AVX512-NEXT: sbbq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; AVX512-NEXT: subq %r9, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; AVX512-NEXT: sbbq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; AVX512-NEXT: subq %rbx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Folded Spill
+; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r9 # 8-byte Reload
+; AVX512-NEXT: sbbq %r8, %r9
+; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Reload
+; AVX512-NEXT: subq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Folded Reload
+; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
+; AVX512-NEXT: sbbq %r10, %r8
+; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Reload
+; AVX512-NEXT: subq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Folded Reload
+; AVX512-NEXT: movq (%rsp), %r10 # 8-byte Reload
+; AVX512-NEXT: sbbq %r11, %r10
+; AVX512-NEXT: subq {{[-0-9]+}}(%r{{[sb]}}p), %r12 # 8-byte Folded Reload
+; AVX512-NEXT: sbbq %r15, %r13
+; AVX512-NEXT: subq {{[-0-9]+}}(%r{{[sb]}}p), %r14 # 8-byte Folded Reload
+; AVX512-NEXT: sbbq {{[-0-9]+}}(%r{{[sb]}}p), %rbp # 8-byte Folded Reload
+; AVX512-NEXT: xorl %edx, %edx
+; AVX512-NEXT: movq %r14, %rax
+; AVX512-NEXT: negq %rax
+; AVX512-NEXT: movl $0, %ecx
+; AVX512-NEXT: sbbq %rbp, %rcx
+; AVX512-NEXT: cmpq %r14, %rax
+; AVX512-NEXT: sbbq %rbp, %rcx
+; AVX512-NEXT: cmovlq %r14, %rax
+; AVX512-NEXT: movq %r12, %rcx
+; AVX512-NEXT: negq %rcx
+; AVX512-NEXT: movl $0, %esi
+; AVX512-NEXT: sbbq %r13, %rsi
+; AVX512-NEXT: cmpq %r12, %rcx
+; AVX512-NEXT: sbbq %r13, %rsi
+; AVX512-NEXT: cmovlq %r12, %rcx
+; AVX512-NEXT: movq %rdi, %r12
+; AVX512-NEXT: negq %r12
+; AVX512-NEXT: movl $0, %esi
+; AVX512-NEXT: sbbq %r10, %rsi
+; AVX512-NEXT: cmpq %rdi, %r12
+; AVX512-NEXT: sbbq %r10, %rsi
+; AVX512-NEXT: cmovlq %rdi, %r12
+; AVX512-NEXT: movq %rbx, %r14
+; AVX512-NEXT: negq %r14
+; AVX512-NEXT: movl $0, %esi
+; AVX512-NEXT: sbbq %r8, %rsi
+; AVX512-NEXT: cmpq %rbx, %r14
+; AVX512-NEXT: sbbq %r8, %rsi
+; AVX512-NEXT: cmovlq %rbx, %r14
+; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r8 # 8-byte Reload
+; AVX512-NEXT: movq %r8, %rsi
+; AVX512-NEXT: negq %rsi
+; AVX512-NEXT: movl $0, %edi
+; AVX512-NEXT: sbbq %r9, %rdi
+; AVX512-NEXT: cmpq %r8, %rsi
+; AVX512-NEXT: sbbq %r9, %rdi
+; AVX512-NEXT: cmovlq %r8, %rsi
+; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r9 # 8-byte Reload
+; AVX512-NEXT: movq %r9, %rdi
+; AVX512-NEXT: negq %rdi
+; AVX512-NEXT: movl $0, %r8d
+; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Reload
+; AVX512-NEXT: sbbq %r10, %r8
+; AVX512-NEXT: cmpq %r9, %rdi
+; AVX512-NEXT: sbbq %r10, %r8
+; AVX512-NEXT: cmovlq %r9, %rdi
+; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Reload
+; AVX512-NEXT: movq %r10, %r8
+; AVX512-NEXT: negq %r8
+; AVX512-NEXT: movl $0, %r9d
+; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Reload
+; AVX512-NEXT: sbbq %r11, %r9
+; AVX512-NEXT: cmpq %r10, %r8
+; AVX512-NEXT: sbbq %r11, %r9
+; AVX512-NEXT: cmovlq %r10, %r8
+; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r10 # 8-byte Reload
+; AVX512-NEXT: movq %r10, %r9
+; AVX512-NEXT: negq %r9
+; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %r11 # 8-byte Reload
+; AVX512-NEXT: sbbq %r11, %rdx
+; AVX512-NEXT: cmpq %r10, %r9
+; AVX512-NEXT: sbbq %r11, %rdx
+; AVX512-NEXT: cmovlq %r10, %r9
+; AVX512-NEXT: vmovq %r9, %xmm0
+; AVX512-NEXT: vmovq %r8, %xmm1
+; AVX512-NEXT: vmovq %rdi, %xmm2
+; AVX512-NEXT: vmovq %rsi, %xmm3
+; AVX512-NEXT: vmovq %r14, %xmm4
+; AVX512-NEXT: vmovq %r12, %xmm5
+; AVX512-NEXT: vmovq %rcx, %xmm6
+; AVX512-NEXT: vmovq %rax, %xmm7
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm3[0],xmm2[0]
+; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm5[0],xmm4[0]
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm7[0],xmm6[0]
+; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
+; AVX512-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512-NEXT: addq $8, %rsp
+; AVX512-NEXT: popq %rbx
+; AVX512-NEXT: popq %r12
+; AVX512-NEXT: popq %r13
+; AVX512-NEXT: popq %r14
+; AVX512-NEXT: popq %r15
+; AVX512-NEXT: popq %rbp
; AVX512-NEXT: retq
%aext = sext <8 x i64> %a to <8 x i128>
%bext = sext <8 x i64> %b to <8 x i128>
@@ -354,18 +618,23 @@ define <64 x i8> @abd_subnsw_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
; AVX512BW-LABEL: abd_subnsw_v64i8:
; AVX512BW: # %bb.0:
; AVX512BW-NEXT: vpsubb %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT: vpabsb %zmm0, %zmm0
+; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT: vpsubb %zmm0, %zmm1, %zmm1
+; AVX512BW-NEXT: vpmaxsb %zmm1, %zmm0, %zmm0
; AVX512BW-NEXT: retq
;
; AVX512DQ-LABEL: abd_subnsw_v64i8:
; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vextracti64x4 $1, %zmm1, %ymm2
-; AVX512DQ-NEXT: vextracti64x4 $1, %zmm0, %ymm3
-; AVX512DQ-NEXT: vpsubb %ymm2, %ymm3, %ymm2
+; AVX512DQ-NEXT: vpsubb %ymm1, %ymm0, %ymm2
+; AVX512DQ-NEXT: vextracti64x4 $1, %zmm1, %ymm1
+; AVX512DQ-NEXT: vextracti64x4 $1, %zmm0, %ymm0
; AVX512DQ-NEXT: vpsubb %ymm1, %ymm0, %ymm0
-; AVX512DQ-NEXT: vpabsb %ymm0, %ymm0
-; AVX512DQ-NEXT: vpabsb %ymm2, %ymm1
-; AVX512DQ-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512DQ-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512DQ-NEXT: vpsubb %ymm0, %ymm1, %ymm3
+; AVX512DQ-NEXT: vpmaxsb %ymm3, %ymm0, %ymm0
+; AVX512DQ-NEXT: vpsubb %ymm2, %ymm1, %ymm1
+; AVX512DQ-NEXT: vpmaxsb %ymm1, %ymm2, %ymm1
+; AVX512DQ-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
; AVX512DQ-NEXT: retq
%sub = sub nsw <64 x i8> %a, %b
%abs = call <64 x i8> @llvm.abs.v64i8(<64 x i8> %sub, i1 false)
@@ -376,18 +645,23 @@ define <32 x i16> @abd_subnsw_v32i16(<32 x i16> %a, <32 x i16> %b) nounwind {
; AVX512BW-LABEL: abd_subnsw_v32i16:
; AVX512BW: # %bb.0:
; AVX512BW-NEXT: vpsubw %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT: vpabsw %zmm0, %zmm0
+; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT: vpsubw %zmm0, %zmm1, %zmm1
+; AVX512BW-NEXT: vpmaxsw %zmm1, %zmm0, %zmm0
; AVX512BW-NEXT: retq
;
; AVX512DQ-LABEL: abd_subnsw_v32i16:
; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vextracti64x4 $1, %zmm1, %ymm2
-; AVX512DQ-NEXT: vextracti64x4 $1, %zmm0, %ymm3
-; AVX512DQ-NEXT: vpsubw %ymm2, %ymm3, %ymm2
+; AVX512DQ-NEXT: vpsubw %ymm1, %ymm0, %ymm2
+; AVX512DQ-NEXT: vextracti64x4 $1, %zmm1, %ymm1
+; AVX512DQ-NEXT: vextracti64x4 $1, %zmm0, %ymm0
; AVX512DQ-NEXT: vpsubw %ymm1, %ymm0, %ymm0
-; AVX512DQ-NEXT: vpabsw %ymm0, %ymm0
-; AVX512DQ-NEXT: vpabsw %ymm2, %ymm1
-; AVX512DQ-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512DQ-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512DQ-NEXT: vpsubw %ymm0, %ymm1, %ymm3
+; AVX512DQ-NEXT: vpmaxsw %ymm3, %ymm0, %ymm0
+; AVX512DQ-NEXT: vpsubw %ymm2, %ymm1, %ymm1
+; AVX512DQ-NEXT: vpmaxsw %ymm1, %ymm2, %ymm1
+; AVX512DQ-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
; AVX512DQ-NEXT: retq
%sub = sub nsw <32 x i16> %a, %b
%abs = call <32 x i16> @llvm.abs.v32i16(<32 x i16> %sub, i1 false)
@@ -398,7 +672,9 @@ define <16 x i32> @abd_subnsw_v16i32(<16 x i32> %a, <16 x i32> %b) nounwind {
; AVX512-LABEL: abd_subnsw_v16i32:
; AVX512: # %bb.0:
; AVX512-NEXT: vpsubd %zmm1, %zmm0, %zmm0
-; AVX512-NEXT: vpabsd %zmm0, %zmm0
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpsubd %zmm0, %zmm1, %zmm1
+; AVX512-NEXT: vpmaxsd %zmm1, %zmm0, %zmm0
; AVX512-NEXT: retq
%sub = sub nsw <16 x i32> %a, %b
%abs = call <16 x i32> @llvm.abs.v16i32(<16 x i32> %sub, i1 false)
@@ -409,7 +685,9 @@ define <8 x i64> @abd_subnsw_v8i64(<8 x i64> %a, <8 x i64> %b) nounwind {
; AVX512-LABEL: abd_subnsw_v8i64:
; AVX512: # %bb.0:
; AVX512-NEXT: vpsubq %zmm1, %zmm0, %zmm0
-; AVX512-NEXT: vpabsq %zmm0, %zmm0
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpsubq %zmm0, %zmm1, %zmm1
+; AVX512-NEXT: vpmaxsq %zmm1, %zmm0, %zmm0
; AVX512-NEXT: retq
%sub = sub nsw <8 x i64> %a, %b
%abs = call <8 x i64> @llvm.abs.v8i64(<8 x i64> %sub, i1 false)
diff --git a/llvm/test/CodeGen/X86/abds.ll b/llvm/test/CodeGen/X86/abds.ll
index a1a4ba81ae493..99906020a3e50 100644
--- a/llvm/test/CodeGen/X86/abds.ll
+++ b/llvm/test/CodeGen/X86/abds.ll
@@ -9,24 +9,38 @@
define i8 @abd_ext_i8(i8 %a, i8 %b) nounwind {
; X86-LABEL: abd_ext_i8:
; X86: # %bb.0:
+; X86-NEXT: pushl %esi
; X86-NEXT: movsbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %eax, %edx
+; X86-NEXT: sarl $31, %edx
; X86-NEXT: movsbl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ecx, %edx
-; X86-NEXT: subl %eax, %edx
-; X86-NEXT: subl %ecx, %eax
-; X86-NEXT: cmovll %edx, %eax
+; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: sarl $31, %esi
+; X86-NEXT: subl %eax, %ecx
+; X86-NEXT: sbbl %edx, %esi
+; X86-NEXT: xorl %edx, %edx
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: negl %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmpl %ecx, %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmovll %ecx, %eax
; X86-NEXT: # kill: def $al killed $al killed $eax
+; X86-NEXT: popl %esi
; X86-NEXT: retl
;
; X64-LABEL: abd_ext_i8:
; X64: # %bb.0:
-; X64-NEXT: movsbl %sil, %eax
-; X64-NEXT: movsbl %dil, %ecx
-; X64-NEXT: movl %ecx, %edx
-; X64-NEXT: subl %eax, %edx
-; X64-NEXT: subl %ecx, %eax
-; X64-NEXT: cmovll %edx, %eax
-; X64-NEXT: # kill: def $al killed $al killed $eax
+; X64-NEXT: # kill: def $esi killed $esi def $rsi
+; X64-NEXT: # kill: def $edi killed $edi def $rdi
+; X64-NEXT: movsbq %dil, %rcx
+; X64-NEXT: movsbq %sil, %rax
+; X64-NEXT: subq %rax, %rcx
+; X64-NEXT: movq %rcx, %rax
+; X64-NEXT: negq %rax
+; X64-NEXT: cmpq %rax, %rcx
+; X64-NEXT: cmovgq %rcx, %rax
+; X64-NEXT: # kill: def $al killed $al killed $rax
; X64-NEXT: retq
%aext = sext i8 %a to i64
%bext = sext i8 %b to i64
@@ -39,23 +53,38 @@ define i8 @abd_ext_i8(i8 %a, i8 %b) nounwind {
define i8 @abd_ext_i8_i16(i8 %a, i16 %b) nounwind {
; X86-LABEL: abd_ext_i8_i16:
; X86: # %bb.0:
+; X86-NEXT: pushl %esi
; X86-NEXT: movswl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %eax, %edx
+; X86-NEXT: sarl $31, %edx
; X86-NEXT: movsbl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ecx, %edx
-; X86-NEXT: subl %eax, %edx
-; X86-NEXT: subl %ecx, %eax
-; X86-NEXT: cmovll %edx, %eax
+; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: sarl $31, %esi
+; X86-NEXT: subl %eax, %ecx
+; X86-NEXT: sbbl %edx, %esi
+; X86-NEXT: xorl %edx, %edx
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: negl %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmpl %ecx, %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmovll %ecx, %eax
; X86-NEXT: # kill: def $al killed $al killed $eax
+; X86-NEXT: popl %esi
; X86-NEXT: retl
;
; X64-LABEL: abd_ext_i8_i16:
; X64: # %bb.0:
-; X64-NEXT: movsbl %dil, %ecx
-; X64-NEXT: subl %esi, %edi
-; X64-NEXT: movswl %si, %eax
-; X64-NEXT: subl %ecx, %eax
-; X64-NEXT: cmovll %edi, %eax
-; X64-NEXT: # kill: def $al killed $al killed $eax
+; X64-NEXT: # kill: def $esi killed $esi def $rsi
+; X64-NEXT: # kill: def $edi killed $edi def $rdi
+; X64-NEXT: movsbq %dil, %rcx
+; X64-NEXT: movswq %si, %rax
+; X64-NEXT: subq %rax, %rcx
+; X64-NEXT: movq %rcx, %rax
+; X64-NEXT: negq %rax
+; X64-NEXT: cmpq %rax, %rcx
+; X64-NEXT: cmovgq %rcx, %rax
+; X64-NEXT: # kill: def $al killed $al killed $rax
; X64-NEXT: retq
%aext = sext i8 %a to i64
%bext = sext i16 %b to i64
@@ -98,24 +127,38 @@ define i8 @abd_ext_i8_undef(i8 %a, i8 %b) nounwind {
define i16 @abd_ext_i16(i16 %a, i16 %b) nounwind {
; X86-LABEL: abd_ext_i16:
; X86: # %bb.0:
+; X86-NEXT: pushl %esi
; X86-NEXT: movswl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %eax, %edx
+; X86-NEXT: sarl $31, %edx
; X86-NEXT: movswl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ecx, %edx
-; X86-NEXT: subl %eax, %edx
-; X86-NEXT: subl %ecx, %eax
-; X86-NEXT: cmovll %edx, %eax
+; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: sarl $31, %esi
+; X86-NEXT: subl %eax, %ecx
+; X86-NEXT: sbbl %edx, %esi
+; X86-NEXT: xorl %edx, %edx
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: negl %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmpl %ecx, %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmovll %ecx, %eax
; X86-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-NEXT: popl %esi
; X86-NEXT: retl
;
; X64-LABEL: abd_ext_i16:
; X64: # %bb.0:
-; X64-NEXT: movswl %si, %eax
-; X64-NEXT: movswl %di, %ecx
-; X64-NEXT: movl %ecx, %edx
-; X64-NEXT: subl %eax, %edx
-; X64-NEXT: subl %ecx, %eax
-; X64-NEXT: cmovll %edx, %eax
-; X64-NEXT: # kill: def $ax killed $ax killed $eax
+; X64-NEXT: # kill: def $esi killed $esi def $rsi
+; X64-NEXT: # kill: def $edi killed $edi def $rdi
+; X64-NEXT: movswq %di, %rcx
+; X64-NEXT: movswq %si, %rax
+; X64-NEXT: subq %rax, %rcx
+; X64-NEXT: movq %rcx, %rax
+; X64-NEXT: negq %rax
+; X64-NEXT: cmpq %rax, %rcx
+; X64-NEXT: cmovgq %rcx, %rax
+; X64-NEXT: # kill: def $ax killed $ax killed $rax
; X64-NEXT: retq
%aext = sext i16 %a to i64
%bext = sext i16 %b to i64
@@ -128,23 +171,37 @@ define i16 @abd_ext_i16(i16 %a, i16 %b) nounwind {
define i16 @abd_ext_i16_i32(i16 %a, i32 %b) nounwind {
; X86-LABEL: abd_ext_i16_i32:
; X86: # %bb.0:
+; X86-NEXT: pushl %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %eax, %edx
+; X86-NEXT: sarl $31, %edx
; X86-NEXT: movswl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ecx, %edx
-; X86-NEXT: subl %eax, %edx
-; X86-NEXT: subl %ecx, %eax
-; X86-NEXT: cmovll %edx, %eax
+; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: sarl $31, %esi
+; X86-NEXT: subl %eax, %ecx
+; X86-NEXT: sbbl %edx, %esi
+; X86-NEXT: xorl %edx, %edx
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: negl %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmpl %ecx, %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmovll %ecx, %eax
; X86-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-NEXT: popl %esi
; X86-NEXT: retl
;
; X64-LABEL: abd_ext_i16_i32:
; X64: # %bb.0:
-; X64-NEXT: movswl %di, %ecx
-; X64-NEXT: movl %edi, %eax
-; X64-NEXT: subl %esi, %eax
-; X64-NEXT: subl %ecx, %esi
-; X64-NEXT: cmovgel %esi, %eax
-; X64-NEXT: # kill: def $ax killed $ax killed $eax
+; X64-NEXT: # kill: def $edi killed $edi def $rdi
+; X64-NEXT: movswq %di, %rcx
+; X64-NEXT: movslq %esi, %rax
+; X64-NEXT: subq %rax, %rcx
+; X64-NEXT: movq %rcx, %rax
+; X64-NEXT: negq %rax
+; X64-NEXT: cmpq %rax, %rcx
+; X64-NEXT: cmovgq %rcx, %rax
+; X64-NEXT: # kill: def $ax killed $ax killed $rax
; X64-NEXT: retq
%aext = sext i16 %a to i64
%bext = sext i32 %b to i64
@@ -187,20 +244,35 @@ define i16 @abd_ext_i16_undef(i16 %a, i16 %b) nounwind {
define i32 @abd_ext_i32(i32 %a, i32 %b) nounwind {
; X86-LABEL: abd_ext_i32:
; X86: # %bb.0:
+; X86-NEXT: pushl %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %eax, %edx
+; X86-NEXT: sarl $31, %edx
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ecx, %edx
-; X86-NEXT: subl %eax, %edx
-; X86-NEXT: subl %ecx, %eax
-; X86-NEXT: cmovll %edx, %eax
+; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: sarl $31, %esi
+; X86-NEXT: subl %eax, %ecx
+; X86-NEXT: sbbl %edx, %esi
+; X86-NEXT: xorl %edx, %edx
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: negl %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmpl %ecx, %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmovll %ecx, %eax
+; X86-NEXT: popl %esi
; X86-NEXT: retl
;
; X64-LABEL: abd_ext_i32:
; X64: # %bb.0:
-; X64-NEXT: movl %edi, %eax
-; X64-NEXT: subl %esi, %eax
-; X64-NEXT: subl %edi, %esi
-; X64-NEXT: cmovgel %esi, %eax
+; X64-NEXT: movslq %edi, %rcx
+; X64-NEXT: movslq %esi, %rax
+; X64-NEXT: subq %rax, %rcx
+; X64-NEXT: movq %rcx, %rax
+; X64-NEXT: negq %rax
+; X64-NEXT: cmpq %rax, %rcx
+; X64-NEXT: cmovgq %rcx, %rax
+; X64-NEXT: # kill: def $eax killed $eax killed $rax
; X64-NEXT: retq
%aext = sext i32 %a to i64
%bext = sext i32 %b to i64
@@ -213,21 +285,36 @@ define i32 @abd_ext_i32(i32 %a, i32 %b) nounwind {
define i32 @abd_ext_i32_i16(i32 %a, i16 %b) nounwind {
; X86-LABEL: abd_ext_i32_i16:
; X86: # %bb.0:
+; X86-NEXT: pushl %esi
; X86-NEXT: movswl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %eax, %edx
+; X86-NEXT: sarl $31, %edx
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ecx, %edx
-; X86-NEXT: subl %eax, %edx
-; X86-NEXT: subl %ecx, %eax
-; X86-NEXT: cmovll %edx, %eax
+; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: sarl $31, %esi
+; X86-NEXT: subl %eax, %ecx
+; X86-NEXT: sbbl %edx, %esi
+; X86-NEXT: xorl %edx, %edx
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: negl %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmpl %ecx, %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmovll %ecx, %eax
+; X86-NEXT: popl %esi
; X86-NEXT: retl
;
; X64-LABEL: abd_ext_i32_i16:
; X64: # %bb.0:
-; X64-NEXT: movswl %si, %eax
-; X64-NEXT: movl %edi, %ecx
-; X64-NEXT: subl %eax, %ecx
-; X64-NEXT: subl %edi, %eax
-; X64-NEXT: cmovll %ecx, %eax
+; X64-NEXT: # kill: def $esi killed $esi def $rsi
+; X64-NEXT: movslq %edi, %rcx
+; X64-NEXT: movswq %si, %rax
+; X64-NEXT: subq %rax, %rcx
+; X64-NEXT: movq %rcx, %rax
+; X64-NEXT: negq %rax
+; X64-NEXT: cmpq %rax, %rcx
+; X64-NEXT: cmovgq %rcx, %rax
+; X64-NEXT: # kill: def $eax killed $eax killed $rax
; X64-NEXT: retq
%aext = sext i32 %a to i64
%bext = sext i16 %b to i64
@@ -266,32 +353,64 @@ define i32 @abd_ext_i32_undef(i32 %a, i32 %b) nounwind {
define i64 @abd_ext_i64(i64 %a, i64 %b) nounwind {
; X86-LABEL: abd_ext_i64:
; X86: # %bb.0:
+; X86-NEXT: pushl %ebp
; X86-NEXT: pushl %ebx
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
+; X86-NEXT: subl $8, %esp
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movl %eax, %edx
+; X86-NEXT: sarl $31, %edx
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT: movl %ecx, %edi
-; X86-NEXT: subl %eax, %edi
-; X86-NEXT: movl %esi, %ebx
-; X86-NEXT: sbbl %edx, %ebx
-; X86-NEXT: subl %ecx, %eax
-; X86-NEXT: sbbl %esi, %edx
-; X86-NEXT: cmovll %edi, %eax
-; X86-NEXT: cmovll %ebx, %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT: movl %ebx, %edi
+; X86-NEXT: sarl $31, %edi
+; X86-NEXT: subl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: sbbl %eax, %ebx
+; X86-NEXT: movl %edi, %eax
+; X86-NEXT: sbbl %edx, %eax
+; X86-NEXT: movl %eax, (%esp) # 4-byte Spill
+; X86-NEXT: sbbl %edx, %edi
+; X86-NEXT: xorl %ebp, %ebp
+; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: negl %eax
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: sbbl %ebx, %ecx
+; X86-NEXT: movl $0, %esi
+; X86-NEXT: sbbl (%esp), %esi # 4-byte Folded Reload
+; X86-NEXT: sbbl %edi, %ebp
+; X86-NEXT: cmpl %edx, %eax
+; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: sbbl %ebx, %edx
+; X86-NEXT: sbbl (%esp), %esi # 4-byte Folded Reload
+; X86-NEXT: sbbl %edi, %ebp
+; X86-NEXT: cmovll {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NEXT: cmovll %ebx, %ecx
+; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: addl $8, %esp
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
; X86-NEXT: popl %ebx
+; X86-NEXT: popl %ebp
; X86-NEXT: retl
;
; X64-LABEL: abd_ext_i64:
; X64: # %bb.0:
+; X64-NEXT: movq %rdi, %rcx
+; X64-NEXT: sarq $63, %rcx
+; X64-NEXT: movq %rsi, %rax
+; X64-NEXT: sarq $63, %rax
+; X64-NEXT: subq %rsi, %rdi
+; X64-NEXT: sbbq %rax, %rcx
+; X64-NEXT: xorl %edx, %edx
; X64-NEXT: movq %rdi, %rax
-; X64-NEXT: subq %rsi, %rax
-; X64-NEXT: subq %rdi, %rsi
-; X64-NEXT: cmovgeq %rsi, %rax
+; X64-NEXT: negq %rax
+; X64-NEXT: sbbq %rcx, %rdx
+; X64-NEXT: cmpq %rdi, %rax
+; X64-NEXT: sbbq %rcx, %rdx
+; X64-NEXT: cmovlq %rdi, %rax
; X64-NEXT: retq
%aext = sext i64 %a to i128
%bext = sext i64 %b to i128
@@ -348,35 +467,79 @@ define i128 @abd_ext_i128(i128 %a, i128 %b) nounwind {
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
; X86-NEXT: andl $-16, %esp
-; X86-NEXT: subl $16, %esp
-; X86-NEXT: movl 40(%ebp), %ecx
-; X86-NEXT: movl 44(%ebp), %edx
-; X86-NEXT: movl 24(%ebp), %eax
-; X86-NEXT: movl 28(%ebp), %esi
-; X86-NEXT: subl %ecx, %eax
+; X86-NEXT: subl $48, %esp
+; X86-NEXT: movl 24(%ebp), %edi
+; X86-NEXT: movl 52(%ebp), %edx
+; X86-NEXT: movl 36(%ebp), %eax
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: movl %eax, %esi
+; X86-NEXT: sarl $31, %ecx
+; X86-NEXT: movl %edx, %eax
+; X86-NEXT: sarl $31, %eax
+; X86-NEXT: subl 40(%ebp), %edi
+; X86-NEXT: movl %edi, %ebx
+; X86-NEXT: movl 28(%ebp), %edx
+; X86-NEXT: sbbl 44(%ebp), %edx
+; X86-NEXT: movl %edx, (%esp) # 4-byte Spill
+; X86-NEXT: movl 32(%ebp), %edi
+; X86-NEXT: sbbl 48(%ebp), %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl 52(%ebp), %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: sbbl %eax, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: sbbl %eax, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: sbbl %eax, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl %eax, %ecx
+; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: movl %ebx, %eax
+; X86-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: negl %eax
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl %esi, %eax
-; X86-NEXT: sbbl %edx, %eax
+; X86-NEXT: movl $0, %eax
+; X86-NEXT: sbbl (%esp), %eax # 4-byte Folded Reload
; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NEXT: movl 48(%ebp), %esi
-; X86-NEXT: movl 32(%ebp), %edi
+; X86-NEXT: movl $0, %eax
+; X86-NEXT: sbbl %edi, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, %edi
; X86-NEXT: sbbl %esi, %edi
-; X86-NEXT: movl 52(%ebp), %ebx
-; X86-NEXT: movl 36(%ebp), %eax
-; X86-NEXT: sbbl %ebx, %eax
-; X86-NEXT: subl 24(%ebp), %ecx
-; X86-NEXT: sbbl 28(%ebp), %edx
-; X86-NEXT: sbbl 32(%ebp), %esi
-; X86-NEXT: sbbl 36(%ebp), %ebx
-; X86-NEXT: cmovll {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X86-NEXT: cmovll {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X86-NEXT: cmovll %edi, %esi
-; X86-NEXT: cmovll %eax, %ebx
-; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: movl %ebx, 12(%eax)
-; X86-NEXT: movl %esi, 8(%eax)
-; X86-NEXT: movl %edx, 4(%eax)
-; X86-NEXT: movl %ecx, (%eax)
+; X86-NEXT: movl $0, %eax
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NEXT: movl $0, %edx
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X86-NEXT: movl $0, %esi
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NEXT: sbbl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: cmpl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: sbbl (%esp), %ebx # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X86-NEXT: movl %edi, %ebx
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NEXT: sbbl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: cmovll {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X86-NEXT: cmovll (%esp), %ecx # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: cmovll {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NEXT: cmovll {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X86-NEXT: movl 8(%ebp), %edx
+; X86-NEXT: movl %edi, 12(%edx)
+; X86-NEXT: movl %esi, 8(%edx)
+; X86-NEXT: movl %ecx, 4(%edx)
+; X86-NEXT: movl %eax, (%edx)
+; X86-NEXT: movl %edx, %eax
; X86-NEXT: leal -12(%ebp), %esp
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
@@ -386,15 +549,30 @@ define i128 @abd_ext_i128(i128 %a, i128 %b) nounwind {
;
; X64-LABEL: abd_ext_i128:
; X64: # %bb.0:
-; X64-NEXT: movq %rdi, %rax
-; X64-NEXT: subq %rdx, %rax
; X64-NEXT: movq %rsi, %r8
-; X64-NEXT: sbbq %rcx, %r8
-; X64-NEXT: subq %rdi, %rdx
-; X64-NEXT: sbbq %rsi, %rcx
-; X64-NEXT: cmovgeq %rdx, %rax
-; X64-NEXT: cmovgeq %rcx, %r8
-; X64-NEXT: movq %r8, %rdx
+; X64-NEXT: sarq $63, %r8
+; X64-NEXT: movq %rcx, %rax
+; X64-NEXT: sarq $63, %rax
+; X64-NEXT: subq %rdx, %rdi
+; X64-NEXT: sbbq %rcx, %rsi
+; X64-NEXT: movq %r8, %rcx
+; X64-NEXT: sbbq %rax, %rcx
+; X64-NEXT: sbbq %rax, %r8
+; X64-NEXT: xorl %r9d, %r9d
+; X64-NEXT: movq %rdi, %rax
+; X64-NEXT: negq %rax
+; X64-NEXT: movl $0, %edx
+; X64-NEXT: sbbq %rsi, %rdx
+; X64-NEXT: movl $0, %r10d
+; X64-NEXT: sbbq %rcx, %r10
+; X64-NEXT: sbbq %r8, %r9
+; X64-NEXT: cmpq %rdi, %rax
+; X64-NEXT: movq %rdx, %r11
+; X64-NEXT: sbbq %rsi, %r11
+; X64-NEXT: sbbq %rcx, %r10
+; X64-NEXT: sbbq %r8, %r9
+; X64-NEXT: cmovlq %rdi, %rax
+; X64-NEXT: cmovlq %rsi, %rdx
; X64-NEXT: retq
%aext = sext i128 %a to i256
%bext = sext i128 %b to i256
@@ -847,22 +1025,26 @@ define i128 @abd_cmp_i128(i128 %a, i128 %b) nounwind {
define i8 @abd_subnsw_i8(i8 %a, i8 %b) nounwind {
; X86-LABEL: abd_subnsw_i8:
; X86: # %bb.0:
-; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: subb {{[0-9]+}}(%esp), %al
-; X86-NEXT: movl %eax, %ecx
-; X86-NEXT: sarb $7, %cl
-; X86-NEXT: xorb %cl, %al
-; X86-NEXT: subb %cl, %al
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: subb {{[0-9]+}}(%esp), %cl
+; X86-NEXT: movzbl %cl, %edx
+; X86-NEXT: movb %cl, %ch
+; X86-NEXT: negb %ch
+; X86-NEXT: movzbl %ch, %eax
+; X86-NEXT: cmpb %ch, %cl
+; X86-NEXT: cmovgl %edx, %eax
+; X86-NEXT: # kill: def $al killed $al killed $eax
; X86-NEXT: retl
;
; X64-LABEL: abd_subnsw_i8:
; X64: # %bb.0:
-; X64-NEXT: movl %edi, %eax
-; X64-NEXT: subb %sil, %al
-; X64-NEXT: movl %eax, %ecx
-; X64-NEXT: sarb $7, %cl
-; X64-NEXT: xorb %cl, %al
-; X64-NEXT: subb %cl, %al
+; X64-NEXT: subb %sil, %dil
+; X64-NEXT: movzbl %dil, %ecx
+; X64-NEXT: movl %ecx, %eax
+; X64-NEXT: negb %al
+; X64-NEXT: movzbl %al, %eax
+; X64-NEXT: cmpb %al, %cl
+; X64-NEXT: cmovgl %ecx, %eax
; X64-NEXT: # kill: def $al killed $al killed $eax
; X64-NEXT: retq
%sub = sub nsw i8 %a, %b
@@ -902,16 +1084,20 @@ define i16 @abd_subnsw_i16(i16 %a, i16 %b) nounwind {
; X86-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: subw {{[0-9]+}}(%esp), %cx
; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: negw %ax
-; X86-NEXT: cmovsw %cx, %ax
+; X86-NEXT: negl %eax
+; X86-NEXT: cmpw %ax, %cx
+; X86-NEXT: cmovgl %ecx, %eax
+; X86-NEXT: # kill: def $ax killed $ax killed $eax
; X86-NEXT: retl
;
; X64-LABEL: abd_subnsw_i16:
; X64: # %bb.0:
; X64-NEXT: subl %esi, %edi
; X64-NEXT: movl %edi, %eax
-; X64-NEXT: negw %ax
-; X64-NEXT: cmovsw %di, %ax
+; X64-NEXT: negl %eax
+; X64-NEXT: cmpw %ax, %di
+; X64-NEXT: cmovgl %edi, %eax
+; X64-NEXT: # kill: def $ax killed $ax killed $eax
; X64-NEXT: retq
%sub = sub nsw i16 %a, %b
%abs = call i16 @llvm.abs.i16(i16 %sub, i1 false)
@@ -943,20 +1129,21 @@ define i16 @abd_subnsw_i16_undef(i16 %a, i16 %b) nounwind {
define i32 @abd_subnsw_i32(i32 %a, i32 %b) nounwind {
; X86-LABEL: abd_subnsw_i32:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ecx, %edx
-; X86-NEXT: subl %eax, %edx
-; X86-NEXT: subl %ecx, %eax
-; X86-NEXT: cmovll %edx, %eax
+; X86-NEXT: subl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: negl %eax
+; X86-NEXT: cmpl %eax, %ecx
+; X86-NEXT: cmovgl %ecx, %eax
; X86-NEXT: retl
;
; X64-LABEL: abd_subnsw_i32:
; X64: # %bb.0:
+; X64-NEXT: subl %esi, %edi
; X64-NEXT: movl %edi, %eax
-; X64-NEXT: subl %esi, %eax
-; X64-NEXT: subl %edi, %esi
-; X64-NEXT: cmovgel %esi, %eax
+; X64-NEXT: negl %eax
+; X64-NEXT: cmpl %eax, %edi
+; X64-NEXT: cmovgl %edi, %eax
; X64-NEXT: retq
%sub = sub nsw i32 %a, %b
%abs = call i32 @llvm.abs.i32(i32 %sub, i1 false)
@@ -989,24 +1176,32 @@ define i32 @abd_subnsw_i32_undef(i32 %a, i32 %b) nounwind {
define i64 @abd_subnsw_i64(i64 %a, i64 %b) nounwind {
; X86-LABEL: abd_subnsw_i64:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: subl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: sbbl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl %edx, %ecx
-; X86-NEXT: sarl $31, %ecx
-; X86-NEXT: xorl %ecx, %edx
-; X86-NEXT: xorl %ecx, %eax
-; X86-NEXT: subl %ecx, %eax
+; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: subl {{[0-9]+}}(%esp), %esi
+; X86-NEXT: sbbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: xorl %edx, %edx
+; X86-NEXT: movl %esi, %eax
+; X86-NEXT: negl %eax
; X86-NEXT: sbbl %ecx, %edx
+; X86-NEXT: cmpl %esi, %eax
+; X86-NEXT: movl %edx, %edi
+; X86-NEXT: sbbl %ecx, %edi
+; X86-NEXT: cmovll %esi, %eax
+; X86-NEXT: cmovll %ecx, %edx
+; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
; X86-NEXT: retl
;
; X64-LABEL: abd_subnsw_i64:
; X64: # %bb.0:
+; X64-NEXT: subq %rsi, %rdi
; X64-NEXT: movq %rdi, %rax
-; X64-NEXT: subq %rsi, %rax
-; X64-NEXT: subq %rdi, %rsi
-; X64-NEXT: cmovgeq %rsi, %rax
+; X64-NEXT: negq %rax
+; X64-NEXT: cmpq %rax, %rdi
+; X64-NEXT: cmovgq %rdi, %rax
; X64-NEXT: retq
%sub = sub nsw i64 %a, %b
%abs = call i64 @llvm.abs.i64(i64 %sub, i1 false)
@@ -1045,50 +1240,67 @@ define i128 @abd_subnsw_i128(i128 %a, i128 %b) nounwind {
; X86: # %bb.0:
; X86-NEXT: pushl %ebp
; X86-NEXT: movl %esp, %ebp
+; X86-NEXT: pushl %ebx
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
; X86-NEXT: andl $-16, %esp
-; X86-NEXT: movl 32(%ebp), %edx
+; X86-NEXT: subl $16, %esp
+; X86-NEXT: movl 32(%ebp), %eax
; X86-NEXT: movl 36(%ebp), %ecx
-; X86-NEXT: movl 24(%ebp), %edi
-; X86-NEXT: movl 28(%ebp), %esi
-; X86-NEXT: subl 40(%ebp), %edi
-; X86-NEXT: sbbl 44(%ebp), %esi
-; X86-NEXT: sbbl 48(%ebp), %edx
+; X86-NEXT: movl 24(%ebp), %esi
+; X86-NEXT: movl 28(%ebp), %edx
+; X86-NEXT: subl 40(%ebp), %esi
+; X86-NEXT: sbbl 44(%ebp), %edx
+; X86-NEXT: movl %edx, (%esp) # 4-byte Spill
+; X86-NEXT: sbbl 48(%ebp), %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: sbbl 52(%ebp), %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: xorl %ebx, %ebx
+; X86-NEXT: movl %esi, %edi
+; X86-NEXT: negl %edi
+; X86-NEXT: movl $0, %edx
+; X86-NEXT: sbbl (%esp), %edx # 4-byte Folded Reload
+; X86-NEXT: movl $0, %eax
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NEXT: sbbl %ecx, %ebx
+; X86-NEXT: cmpl %esi, %edi
+; X86-NEXT: movl %edx, %ecx
+; X86-NEXT: sbbl (%esp), %ecx # 4-byte Folded Reload
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: movl %ebx, %ecx
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: cmovll %esi, %edi
+; X86-NEXT: cmovll (%esp), %edx # 4-byte Folded Reload
+; X86-NEXT: cmovll {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NEXT: cmovll {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X86-NEXT: movl 8(%ebp), %ecx
+; X86-NEXT: movl %ebx, 12(%ecx)
+; X86-NEXT: movl %eax, 8(%ecx)
+; X86-NEXT: movl %edx, 4(%ecx)
+; X86-NEXT: movl %edi, (%ecx)
; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: sarl $31, %eax
-; X86-NEXT: xorl %eax, %ecx
-; X86-NEXT: xorl %eax, %edx
-; X86-NEXT: xorl %eax, %esi
-; X86-NEXT: xorl %eax, %edi
-; X86-NEXT: subl %eax, %edi
-; X86-NEXT: sbbl %eax, %esi
-; X86-NEXT: sbbl %eax, %edx
-; X86-NEXT: sbbl %eax, %ecx
-; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: movl %edi, (%eax)
-; X86-NEXT: movl %esi, 4(%eax)
-; X86-NEXT: movl %edx, 8(%eax)
-; X86-NEXT: movl %ecx, 12(%eax)
-; X86-NEXT: leal -8(%ebp), %esp
+; X86-NEXT: leal -12(%ebp), %esp
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
+; X86-NEXT: popl %ebx
; X86-NEXT: popl %ebp
; X86-NEXT: retl $4
;
; X64-LABEL: abd_subnsw_i128:
; X64: # %bb.0:
-; X64-NEXT: movq %rdi, %rax
-; X64-NEXT: subq %rdx, %rax
-; X64-NEXT: sbbq %rcx, %rsi
-; X64-NEXT: movq %rsi, %rcx
-; X64-NEXT: sarq $63, %rcx
-; X64-NEXT: xorq %rcx, %rsi
-; X64-NEXT: xorq %rcx, %rax
-; X64-NEXT: subq %rcx, %rax
+; X64-NEXT: subq %rdx, %rdi
; X64-NEXT: sbbq %rcx, %rsi
-; X64-NEXT: movq %rsi, %rdx
+; X64-NEXT: xorl %edx, %edx
+; X64-NEXT: movq %rdi, %rax
+; X64-NEXT: negq %rax
+; X64-NEXT: sbbq %rsi, %rdx
+; X64-NEXT: cmpq %rdi, %rax
+; X64-NEXT: movq %rdx, %rcx
+; X64-NEXT: sbbq %rsi, %rcx
+; X64-NEXT: cmovlq %rdi, %rax
+; X64-NEXT: cmovlq %rsi, %rdx
; X64-NEXT: retq
%sub = sub nsw i128 %a, %b
%abs = call i128 @llvm.abs.i128(i128 %sub, i1 false)
@@ -1161,7 +1373,8 @@ define i32 @abd_sub_i32(i32 %a, i32 %b) nounwind {
; X86-NEXT: subl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: negl %eax
-; X86-NEXT: cmovsl %ecx, %eax
+; X86-NEXT: cmpl %eax, %ecx
+; X86-NEXT: cmovgl %ecx, %eax
; X86-NEXT: retl
;
; X64-LABEL: abd_sub_i32:
@@ -1169,7 +1382,8 @@ define i32 @abd_sub_i32(i32 %a, i32 %b) nounwind {
; X64-NEXT: subl %esi, %edi
; X64-NEXT: movl %edi, %eax
; X64-NEXT: negl %eax
-; X64-NEXT: cmovsl %edi, %eax
+; X64-NEXT: cmpl %eax, %edi
+; X64-NEXT: cmovgl %edi, %eax
; X64-NEXT: retq
%sub = sub i32 %a, %b
%abs = call i32 @llvm.abs.i32(i32 %sub, i1 false)
diff --git a/llvm/test/CodeGen/X86/abdu-neg.ll b/llvm/test/CodeGen/X86/abdu-neg.ll
index b7c34070f1af6..2fda3e227989f 100644
--- a/llvm/test/CodeGen/X86/abdu-neg.ll
+++ b/llvm/test/CodeGen/X86/abdu-neg.ll
@@ -9,26 +9,35 @@
define i8 @abd_ext_i8(i8 %a, i8 %b) nounwind {
; X86-LABEL: abd_ext_i8:
; X86: # %bb.0:
+; X86-NEXT: pushl %esi
; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ecx, %edx
-; X86-NEXT: subl %eax, %edx
-; X86-NEXT: subl %ecx, %eax
-; X86-NEXT: cmovbl %edx, %eax
+; X86-NEXT: xorl %edx, %edx
+; X86-NEXT: subl %eax, %ecx
+; X86-NEXT: movl $0, %esi
+; X86-NEXT: sbbl %esi, %esi
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: negl %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmpl %ecx, %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmovll %ecx, %eax
; X86-NEXT: negb %al
; X86-NEXT: # kill: def $al killed $al killed $eax
+; X86-NEXT: popl %esi
; X86-NEXT: retl
;
; X64-LABEL: abd_ext_i8:
; X64: # %bb.0:
-; X64-NEXT: movzbl %sil, %eax
; X64-NEXT: movzbl %dil, %ecx
-; X64-NEXT: movl %ecx, %edx
-; X64-NEXT: subl %eax, %edx
-; X64-NEXT: subl %ecx, %eax
-; X64-NEXT: cmovbl %edx, %eax
+; X64-NEXT: movzbl %sil, %eax
+; X64-NEXT: subq %rax, %rcx
+; X64-NEXT: movq %rcx, %rax
+; X64-NEXT: negq %rax
+; X64-NEXT: cmpq %rax, %rcx
+; X64-NEXT: cmovgq %rcx, %rax
; X64-NEXT: negb %al
-; X64-NEXT: # kill: def $al killed $al killed $eax
+; X64-NEXT: # kill: def $al killed $al killed $rax
; X64-NEXT: retq
%aext = zext i8 %a to i64
%bext = zext i8 %b to i64
@@ -42,25 +51,35 @@ define i8 @abd_ext_i8(i8 %a, i8 %b) nounwind {
define i8 @abd_ext_i8_i16(i8 %a, i16 %b) nounwind {
; X86-LABEL: abd_ext_i8_i16:
; X86: # %bb.0:
+; X86-NEXT: pushl %esi
; X86-NEXT: movzwl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ecx, %edx
-; X86-NEXT: subl %eax, %edx
-; X86-NEXT: subl %ecx, %eax
-; X86-NEXT: cmovbl %edx, %eax
+; X86-NEXT: xorl %edx, %edx
+; X86-NEXT: subl %eax, %ecx
+; X86-NEXT: movl $0, %esi
+; X86-NEXT: sbbl %esi, %esi
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: negl %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmpl %ecx, %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmovll %ecx, %eax
; X86-NEXT: negb %al
; X86-NEXT: # kill: def $al killed $al killed $eax
+; X86-NEXT: popl %esi
; X86-NEXT: retl
;
; X64-LABEL: abd_ext_i8_i16:
; X64: # %bb.0:
; X64-NEXT: movzbl %dil, %ecx
-; X64-NEXT: subl %esi, %edi
; X64-NEXT: movzwl %si, %eax
-; X64-NEXT: subl %ecx, %eax
-; X64-NEXT: cmovbl %edi, %eax
+; X64-NEXT: subq %rax, %rcx
+; X64-NEXT: movq %rcx, %rax
+; X64-NEXT: negq %rax
+; X64-NEXT: cmpq %rax, %rcx
+; X64-NEXT: cmovgq %rcx, %rax
; X64-NEXT: negb %al
-; X64-NEXT: # kill: def $al killed $al killed $eax
+; X64-NEXT: # kill: def $al killed $al killed $rax
; X64-NEXT: retq
%aext = zext i8 %a to i64
%bext = zext i16 %b to i64
@@ -107,24 +126,35 @@ define i8 @abd_ext_i8_undef(i8 %a, i8 %b) nounwind {
define i16 @abd_ext_i16(i16 %a, i16 %b) nounwind {
; X86-LABEL: abd_ext_i16:
; X86: # %bb.0:
+; X86-NEXT: pushl %esi
; X86-NEXT: movzwl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ecx, %edx
-; X86-NEXT: subl %eax, %edx
-; X86-NEXT: subl %ecx, %eax
-; X86-NEXT: cmovael %edx, %eax
+; X86-NEXT: xorl %edx, %edx
+; X86-NEXT: subl %eax, %ecx
+; X86-NEXT: movl $0, %esi
+; X86-NEXT: sbbl %esi, %esi
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: negl %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmpl %ecx, %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmovll %ecx, %eax
+; X86-NEXT: negl %eax
; X86-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-NEXT: popl %esi
; X86-NEXT: retl
;
; X64-LABEL: abd_ext_i16:
; X64: # %bb.0:
; X64-NEXT: movzwl %di, %ecx
-; X64-NEXT: subl %esi, %edi
; X64-NEXT: movzwl %si, %eax
-; X64-NEXT: subl %ecx, %eax
-; X64-NEXT: cmovbl %edi, %eax
+; X64-NEXT: subq %rax, %rcx
+; X64-NEXT: movq %rcx, %rax
+; X64-NEXT: negq %rax
+; X64-NEXT: cmpq %rax, %rcx
+; X64-NEXT: cmovgq %rcx, %rax
; X64-NEXT: negl %eax
-; X64-NEXT: # kill: def $ax killed $ax killed $eax
+; X64-NEXT: # kill: def $ax killed $ax killed $rax
; X64-NEXT: retq
%aext = zext i16 %a to i64
%bext = zext i16 %b to i64
@@ -138,24 +168,34 @@ define i16 @abd_ext_i16(i16 %a, i16 %b) nounwind {
define i16 @abd_ext_i16_i32(i16 %a, i32 %b) nounwind {
; X86-LABEL: abd_ext_i16_i32:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: pushl %esi
; X86-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ecx, %edx
-; X86-NEXT: subl %eax, %edx
-; X86-NEXT: subl %ecx, %eax
-; X86-NEXT: cmovael %edx, %eax
+; X86-NEXT: xorl %edx, %edx
+; X86-NEXT: subl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl $0, %esi
+; X86-NEXT: sbbl %esi, %esi
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: negl %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmpl %ecx, %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmovll %ecx, %eax
+; X86-NEXT: negl %eax
; X86-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-NEXT: popl %esi
; X86-NEXT: retl
;
; X64-LABEL: abd_ext_i16_i32:
; X64: # %bb.0:
; X64-NEXT: movzwl %di, %ecx
-; X64-NEXT: movl %edi, %eax
-; X64-NEXT: subl %esi, %eax
-; X64-NEXT: subl %ecx, %esi
-; X64-NEXT: cmovael %esi, %eax
+; X64-NEXT: movl %esi, %eax
+; X64-NEXT: subq %rax, %rcx
+; X64-NEXT: movq %rcx, %rax
+; X64-NEXT: negq %rax
+; X64-NEXT: cmpq %rax, %rcx
+; X64-NEXT: cmovgq %rcx, %rax
; X64-NEXT: negl %eax
-; X64-NEXT: # kill: def $ax killed $ax killed $eax
+; X64-NEXT: # kill: def $ax killed $ax killed $rax
; X64-NEXT: retq
%aext = zext i16 %a to i64
%bext = zext i32 %b to i64
@@ -200,20 +240,33 @@ define i16 @abd_ext_i16_undef(i16 %a, i16 %b) nounwind {
define i32 @abd_ext_i32(i32 %a, i32 %b) nounwind {
; X86-LABEL: abd_ext_i32:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: pushl %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ecx, %edx
-; X86-NEXT: subl %eax, %edx
-; X86-NEXT: subl %ecx, %eax
-; X86-NEXT: cmovael %edx, %eax
+; X86-NEXT: xorl %edx, %edx
+; X86-NEXT: subl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl $0, %esi
+; X86-NEXT: sbbl %esi, %esi
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: negl %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmpl %ecx, %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmovll %ecx, %eax
+; X86-NEXT: negl %eax
+; X86-NEXT: popl %esi
; X86-NEXT: retl
;
; X64-LABEL: abd_ext_i32:
; X64: # %bb.0:
-; X64-NEXT: movl %edi, %eax
-; X64-NEXT: subl %esi, %eax
-; X64-NEXT: subl %edi, %esi
-; X64-NEXT: cmovbl %esi, %eax
+; X64-NEXT: movl %edi, %ecx
+; X64-NEXT: movl %esi, %eax
+; X64-NEXT: subq %rax, %rcx
+; X64-NEXT: movq %rcx, %rax
+; X64-NEXT: negq %rax
+; X64-NEXT: cmpq %rax, %rcx
+; X64-NEXT: cmovgq %rcx, %rax
+; X64-NEXT: negl %eax
+; X64-NEXT: # kill: def $eax killed $eax killed $rax
; X64-NEXT: retq
%aext = zext i32 %a to i64
%bext = zext i32 %b to i64
@@ -227,21 +280,34 @@ define i32 @abd_ext_i32(i32 %a, i32 %b) nounwind {
define i32 @abd_ext_i32_i16(i32 %a, i16 %b) nounwind {
; X86-LABEL: abd_ext_i32_i16:
; X86: # %bb.0:
+; X86-NEXT: pushl %esi
; X86-NEXT: movzwl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ecx, %edx
-; X86-NEXT: subl %eax, %edx
-; X86-NEXT: subl %ecx, %eax
-; X86-NEXT: cmovael %edx, %eax
+; X86-NEXT: xorl %edx, %edx
+; X86-NEXT: subl %eax, %ecx
+; X86-NEXT: movl $0, %esi
+; X86-NEXT: sbbl %esi, %esi
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: negl %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmpl %ecx, %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmovll %ecx, %eax
+; X86-NEXT: negl %eax
+; X86-NEXT: popl %esi
; X86-NEXT: retl
;
; X64-LABEL: abd_ext_i32_i16:
; X64: # %bb.0:
-; X64-NEXT: movzwl %si, %eax
; X64-NEXT: movl %edi, %ecx
-; X64-NEXT: subl %eax, %ecx
-; X64-NEXT: subl %edi, %eax
-; X64-NEXT: cmovael %ecx, %eax
+; X64-NEXT: movzwl %si, %eax
+; X64-NEXT: subq %rax, %rcx
+; X64-NEXT: movq %rcx, %rax
+; X64-NEXT: negq %rax
+; X64-NEXT: cmpq %rax, %rcx
+; X64-NEXT: cmovgq %rcx, %rax
+; X64-NEXT: negl %eax
+; X64-NEXT: # kill: def $eax killed $eax killed $rax
; X64-NEXT: retq
%aext = zext i32 %a to i64
%bext = zext i16 %b to i64
@@ -282,29 +348,59 @@ define i32 @abd_ext_i32_undef(i32 %a, i32 %b) nounwind {
define i64 @abd_ext_i64(i64 %a, i64 %b) nounwind {
; X86-LABEL: abd_ext_i64:
; X86: # %bb.0:
+; X86-NEXT: pushl %ebp
+; X86-NEXT: pushl %ebx
+; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: xorl %edx, %edx
-; X86-NEXT: subl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: sbbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: subl $8, %esp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: subl {{[0-9]+}}(%esp), %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movl %edx, (%esp) # 4-byte Spill
+; X86-NEXT: movl $0, %ebx
+; X86-NEXT: sbbl %ebx, %ebx
+; X86-NEXT: movl $0, %ebp
+; X86-NEXT: sbbl %ebp, %ebp
+; X86-NEXT: movl %edi, %eax
+; X86-NEXT: negl %eax
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: sbbl %edx, %ecx
; X86-NEXT: movl $0, %esi
-; X86-NEXT: sbbl %esi, %esi
-; X86-NEXT: xorl %esi, %ecx
-; X86-NEXT: xorl %esi, %eax
-; X86-NEXT: subl %esi, %eax
-; X86-NEXT: sbbl %esi, %ecx
+; X86-NEXT: sbbl %ebx, %esi
+; X86-NEXT: movl $0, %edx
+; X86-NEXT: sbbl %ebp, %edx
+; X86-NEXT: cmpl %edi, %eax
+; X86-NEXT: movl %ecx, %edi
+; X86-NEXT: sbbl (%esp), %edi # 4-byte Folded Reload
+; X86-NEXT: sbbl %ebx, %esi
+; X86-NEXT: sbbl %ebp, %edx
+; X86-NEXT: cmovll (%esp), %ecx # 4-byte Folded Reload
+; X86-NEXT: cmovll {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
; X86-NEXT: negl %eax
+; X86-NEXT: movl $0, %edx
; X86-NEXT: sbbl %ecx, %edx
+; X86-NEXT: addl $8, %esp
; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
+; X86-NEXT: popl %ebx
+; X86-NEXT: popl %ebp
; X86-NEXT: retl
;
; X64-LABEL: abd_ext_i64:
; X64: # %bb.0:
+; X64-NEXT: xorl %ecx, %ecx
+; X64-NEXT: subq %rsi, %rdi
+; X64-NEXT: movl $0, %edx
+; X64-NEXT: sbbq %rdx, %rdx
; X64-NEXT: movq %rdi, %rax
-; X64-NEXT: subq %rsi, %rax
-; X64-NEXT: subq %rdi, %rsi
-; X64-NEXT: cmovbq %rsi, %rax
+; X64-NEXT: negq %rax
+; X64-NEXT: sbbq %rdx, %rcx
+; X64-NEXT: cmpq %rdi, %rax
+; X64-NEXT: sbbq %rdx, %rcx
+; X64-NEXT: cmovlq %rdi, %rax
+; X64-NEXT: negq %rax
; X64-NEXT: retq
%aext = zext i64 %a to i128
%bext = zext i64 %b to i128
@@ -360,37 +456,84 @@ define i128 @abd_ext_i128(i128 %a, i128 %b) nounwind {
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
; X86-NEXT: andl $-16, %esp
-; X86-NEXT: subl $16, %esp
-; X86-NEXT: movl 32(%ebp), %esi
-; X86-NEXT: movl 36(%ebp), %eax
-; X86-NEXT: movl 24(%ebp), %ecx
-; X86-NEXT: movl 28(%ebp), %edi
-; X86-NEXT: xorl %edx, %edx
-; X86-NEXT: subl 40(%ebp), %ecx
-; X86-NEXT: sbbl 44(%ebp), %edi
-; X86-NEXT: sbbl 48(%ebp), %esi
-; X86-NEXT: sbbl 52(%ebp), %eax
-; X86-NEXT: movl $0, %ebx
-; X86-NEXT: sbbl %ebx, %ebx
-; X86-NEXT: xorl %ebx, %eax
-; X86-NEXT: xorl %ebx, %esi
-; X86-NEXT: xorl %ebx, %edi
-; X86-NEXT: xorl %ebx, %ecx
-; X86-NEXT: subl %ebx, %ecx
-; X86-NEXT: sbbl %ebx, %edi
-; X86-NEXT: sbbl %ebx, %esi
-; X86-NEXT: sbbl %ebx, %eax
-; X86-NEXT: negl %ecx
-; X86-NEXT: movl $0, %ebx
-; X86-NEXT: sbbl %edi, %ebx
+; X86-NEXT: subl $48, %esp
+; X86-NEXT: movl 32(%ebp), %ecx
+; X86-NEXT: movl 36(%ebp), %edi
+; X86-NEXT: movl 24(%ebp), %eax
+; X86-NEXT: movl 28(%ebp), %edx
+; X86-NEXT: subl 40(%ebp), %eax
+; X86-NEXT: movl %eax, %esi
+; X86-NEXT: sbbl 44(%ebp), %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: sbbl 48(%ebp), %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %edi, %ecx
+; X86-NEXT: sbbl 52(%ebp), %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl $0, %edi
-; X86-NEXT: sbbl %esi, %edi
+; X86-NEXT: sbbl %edi, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, %edi
+; X86-NEXT: sbbl %edi, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, %edi
+; X86-NEXT: sbbl %edi, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, %edi
+; X86-NEXT: sbbl %edi, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl %esi, %ebx
+; X86-NEXT: movl %esi, %edi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: negl %ebx
+; X86-NEXT: movl $0, %esi
+; X86-NEXT: sbbl %edx, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, %edx
; X86-NEXT: sbbl %eax, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, %eax
+; X86-NEXT: sbbl %ecx, %eax
+; X86-NEXT: movl %eax, (%esp) # 4-byte Spill
+; X86-NEXT: movl $0, %eax
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: movl $0, %edx
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X86-NEXT: movl $0, %esi
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NEXT: cmpl %edi, %ebx
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X86-NEXT: movl (%esp), %edi # 4-byte Reload
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NEXT: movl (%esp), %eax # 4-byte Reload
+; X86-NEXT: cmovll {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X86-NEXT: cmovll {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: cmovll {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X86-NEXT: cmovll {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X86-NEXT: negl %ebx
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: sbbl %edx, %ecx
+; X86-NEXT: movl $0, %edx
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: movl $0, %esi
+; X86-NEXT: sbbl %eax, %esi
; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: movl %ecx, (%eax)
-; X86-NEXT: movl %ebx, 4(%eax)
-; X86-NEXT: movl %edi, 8(%eax)
-; X86-NEXT: movl %edx, 12(%eax)
+; X86-NEXT: movl %ebx, (%eax)
+; X86-NEXT: movl %ecx, 4(%eax)
+; X86-NEXT: movl %edx, 8(%eax)
+; X86-NEXT: movl %esi, 12(%eax)
; X86-NEXT: leal -12(%ebp), %esp
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
@@ -400,19 +543,33 @@ define i128 @abd_ext_i128(i128 %a, i128 %b) nounwind {
;
; X64-LABEL: abd_ext_i128:
; X64: # %bb.0:
-; X64-NEXT: movq %rdi, %rax
-; X64-NEXT: xorl %edi, %edi
-; X64-NEXT: subq %rdx, %rax
+; X64-NEXT: pushq %rbx
+; X64-NEXT: movq %rdx, %rax
+; X64-NEXT: xorl %edx, %edx
+; X64-NEXT: subq %rax, %rdi
; X64-NEXT: sbbq %rcx, %rsi
; X64-NEXT: movl $0, %ecx
; X64-NEXT: sbbq %rcx, %rcx
-; X64-NEXT: xorq %rcx, %rsi
-; X64-NEXT: xorq %rcx, %rax
-; X64-NEXT: subq %rcx, %rax
-; X64-NEXT: sbbq %rcx, %rsi
+; X64-NEXT: movl $0, %r8d
+; X64-NEXT: sbbq %r8, %r8
+; X64-NEXT: movq %rdi, %rax
; X64-NEXT: negq %rax
-; X64-NEXT: sbbq %rsi, %rdi
-; X64-NEXT: movq %rdi, %rdx
+; X64-NEXT: movl $0, %r9d
+; X64-NEXT: sbbq %rsi, %r9
+; X64-NEXT: movl $0, %r10d
+; X64-NEXT: sbbq %rcx, %r10
+; X64-NEXT: movl $0, %r11d
+; X64-NEXT: sbbq %r8, %r11
+; X64-NEXT: cmpq %rdi, %rax
+; X64-NEXT: movq %r9, %rbx
+; X64-NEXT: sbbq %rsi, %rbx
+; X64-NEXT: sbbq %rcx, %r10
+; X64-NEXT: sbbq %r8, %r11
+; X64-NEXT: cmovlq %rsi, %r9
+; X64-NEXT: cmovlq %rdi, %rax
+; X64-NEXT: negq %rax
+; X64-NEXT: sbbq %r9, %rdx
+; X64-NEXT: popq %rbx
; X64-NEXT: retq
%aext = zext i128 %a to i256
%bext = zext i128 %b to i256
diff --git a/llvm/test/CodeGen/X86/abdu-vector-128.ll b/llvm/test/CodeGen/X86/abdu-vector-128.ll
index 78b315a3773ec..286ea8bc03f5c 100644
--- a/llvm/test/CodeGen/X86/abdu-vector-128.ll
+++ b/llvm/test/CodeGen/X86/abdu-vector-128.ll
@@ -10,20 +10,335 @@
;
define <16 x i8> @abd_ext_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind {
-; SSE-LABEL: abd_ext_v16i8:
-; SSE: # %bb.0:
-; SSE-NEXT: movdqa %xmm0, %xmm2
-; SSE-NEXT: pminub %xmm1, %xmm2
-; SSE-NEXT: pmaxub %xmm1, %xmm0
-; SSE-NEXT: psubb %xmm2, %xmm0
-; SSE-NEXT: retq
+; SSE2-LABEL: abd_ext_v16i8:
+; SSE2: # %bb.0:
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: movdqa %xmm0, %xmm3
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; SSE2-NEXT: movdqa %xmm3, %xmm5
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; SSE2-NEXT: movdqa %xmm5, %xmm9
+; SSE2-NEXT: punpckldq {{.*#+}} xmm9 = xmm9[0],xmm2[0],xmm9[1],xmm2[1]
+; SSE2-NEXT: punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; SSE2-NEXT: movdqa %xmm3, %xmm8
+; SSE2-NEXT: punpckldq {{.*#+}} xmm8 = xmm8[0],xmm2[0],xmm8[1],xmm2[1]
+; SSE2-NEXT: punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE2-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
+; SSE2-NEXT: movdqa %xmm0, %xmm4
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
+; SSE2-NEXT: movdqa %xmm4, %xmm7
+; SSE2-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1]
+; SSE2-NEXT: punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm2[2],xmm4[3],xmm2[3]
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE2-NEXT: movdqa %xmm0, %xmm6
+; SSE2-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1]
+; SSE2-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE2-NEXT: movdqa %xmm1, %xmm10
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm10 = xmm10[0],xmm2[0],xmm10[1],xmm2[1],xmm10[2],xmm2[2],xmm10[3],xmm2[3],xmm10[4],xmm2[4],xmm10[5],xmm2[5],xmm10[6],xmm2[6],xmm10[7],xmm2[7]
+; SSE2-NEXT: movdqa %xmm10, %xmm11
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm11 = xmm11[0],xmm2[0],xmm11[1],xmm2[1],xmm11[2],xmm2[2],xmm11[3],xmm2[3]
+; SSE2-NEXT: movdqa %xmm11, %xmm12
+; SSE2-NEXT: punpckldq {{.*#+}} xmm12 = xmm12[0],xmm2[0],xmm12[1],xmm2[1]
+; SSE2-NEXT: psubq %xmm12, %xmm9
+; SSE2-NEXT: punpckhdq {{.*#+}} xmm11 = xmm11[2],xmm2[2],xmm11[3],xmm2[3]
+; SSE2-NEXT: psubq %xmm11, %xmm5
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm10 = xmm10[4],xmm2[4],xmm10[5],xmm2[5],xmm10[6],xmm2[6],xmm10[7],xmm2[7]
+; SSE2-NEXT: movdqa %xmm10, %xmm11
+; SSE2-NEXT: punpckldq {{.*#+}} xmm11 = xmm11[0],xmm2[0],xmm11[1],xmm2[1]
+; SSE2-NEXT: psubq %xmm11, %xmm8
+; SSE2-NEXT: punpckhdq {{.*#+}} xmm10 = xmm10[2],xmm2[2],xmm10[3],xmm2[3]
+; SSE2-NEXT: psubq %xmm10, %xmm3
+; SSE2-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
+; SSE2-NEXT: movdqa %xmm1, %xmm10
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm10 = xmm10[0],xmm2[0],xmm10[1],xmm2[1],xmm10[2],xmm2[2],xmm10[3],xmm2[3]
+; SSE2-NEXT: movdqa %xmm10, %xmm11
+; SSE2-NEXT: punpckldq {{.*#+}} xmm11 = xmm11[0],xmm2[0],xmm11[1],xmm2[1]
+; SSE2-NEXT: psubq %xmm11, %xmm7
+; SSE2-NEXT: punpckhdq {{.*#+}} xmm10 = xmm10[2],xmm2[2],xmm10[3],xmm2[3]
+; SSE2-NEXT: psubq %xmm10, %xmm4
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE2-NEXT: movdqa %xmm1, %xmm10
+; SSE2-NEXT: punpckldq {{.*#+}} xmm10 = xmm10[0],xmm2[0],xmm10[1],xmm2[1]
+; SSE2-NEXT: psubq %xmm10, %xmm6
+; SSE2-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSE2-NEXT: psubq %xmm1, %xmm0
+; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: psubq %xmm9, %xmm1
+; SSE2-NEXT: pmaxsw %xmm9, %xmm1
+; SSE2-NEXT: pxor %xmm9, %xmm9
+; SSE2-NEXT: psubq %xmm5, %xmm9
+; SSE2-NEXT: pmaxsw %xmm5, %xmm9
+; SSE2-NEXT: pxor %xmm5, %xmm5
+; SSE2-NEXT: psubq %xmm8, %xmm5
+; SSE2-NEXT: pmaxsw %xmm8, %xmm5
+; SSE2-NEXT: pxor %xmm8, %xmm8
+; SSE2-NEXT: psubq %xmm3, %xmm8
+; SSE2-NEXT: pmaxsw %xmm3, %xmm8
+; SSE2-NEXT: pxor %xmm3, %xmm3
+; SSE2-NEXT: psubq %xmm7, %xmm3
+; SSE2-NEXT: pmaxsw %xmm7, %xmm3
+; SSE2-NEXT: pxor %xmm7, %xmm7
+; SSE2-NEXT: psubq %xmm4, %xmm7
+; SSE2-NEXT: pmaxsw %xmm4, %xmm7
+; SSE2-NEXT: pxor %xmm4, %xmm4
+; SSE2-NEXT: psubq %xmm6, %xmm4
+; SSE2-NEXT: pmaxsw %xmm6, %xmm4
+; SSE2-NEXT: psubq %xmm0, %xmm2
+; SSE2-NEXT: pmaxsw %xmm0, %xmm2
+; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0]
+; SSE2-NEXT: pand %xmm0, %xmm2
+; SSE2-NEXT: pand %xmm0, %xmm4
+; SSE2-NEXT: packuswb %xmm2, %xmm4
+; SSE2-NEXT: pand %xmm0, %xmm7
+; SSE2-NEXT: pand %xmm0, %xmm3
+; SSE2-NEXT: packuswb %xmm7, %xmm3
+; SSE2-NEXT: packuswb %xmm4, %xmm3
+; SSE2-NEXT: pand %xmm0, %xmm8
+; SSE2-NEXT: pand %xmm0, %xmm5
+; SSE2-NEXT: packuswb %xmm8, %xmm5
+; SSE2-NEXT: pand %xmm0, %xmm9
+; SSE2-NEXT: pand %xmm0, %xmm1
+; SSE2-NEXT: packuswb %xmm9, %xmm1
+; SSE2-NEXT: packuswb %xmm5, %xmm1
+; SSE2-NEXT: packuswb %xmm3, %xmm1
+; SSE2-NEXT: movdqa %xmm1, %xmm0
+; SSE2-NEXT: retq
;
-; AVX-LABEL: abd_ext_v16i8:
-; AVX: # %bb.0:
-; AVX-NEXT: vpminub %xmm1, %xmm0, %xmm2
-; AVX-NEXT: vpmaxub %xmm1, %xmm0, %xmm0
-; AVX-NEXT: vpsubb %xmm2, %xmm0, %xmm0
-; AVX-NEXT: retq
+; SSE42-LABEL: abd_ext_v16i8:
+; SSE42: # %bb.0:
+; SSE42-NEXT: movdqa %xmm0, %xmm2
+; SSE42-NEXT: psrld $16, %xmm2
+; SSE42-NEXT: pmovzxbq {{.*#+}} xmm9 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; SSE42-NEXT: pmovzxbq {{.*#+}} xmm8 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
+; SSE42-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,1,1]
+; SSE42-NEXT: pmovzxbq {{.*#+}} xmm7 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; SSE42-NEXT: movdqa %xmm0, %xmm2
+; SSE42-NEXT: psrlq $48, %xmm2
+; SSE42-NEXT: pmovzxbq {{.*#+}} xmm6 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; SSE42-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
+; SSE42-NEXT: pmovzxbq {{.*#+}} xmm5 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; SSE42-NEXT: movdqa %xmm0, %xmm2
+; SSE42-NEXT: psrldq {{.*#+}} xmm2 = xmm2[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE42-NEXT: pmovzxbq {{.*#+}} xmm4 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; SSE42-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
+; SSE42-NEXT: pmovzxbq {{.*#+}} xmm3 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; SSE42-NEXT: psrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE42-NEXT: pmovzxbq {{.*#+}} xmm2 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
+; SSE42-NEXT: movdqa %xmm1, %xmm0
+; SSE42-NEXT: psrld $16, %xmm0
+; SSE42-NEXT: pmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
+; SSE42-NEXT: psubq %xmm0, %xmm9
+; SSE42-NEXT: pmovzxbq {{.*#+}} xmm0 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; SSE42-NEXT: psubq %xmm0, %xmm8
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE42-NEXT: pmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
+; SSE42-NEXT: psubq %xmm0, %xmm7
+; SSE42-NEXT: movdqa %xmm1, %xmm0
+; SSE42-NEXT: psrlq $48, %xmm0
+; SSE42-NEXT: pmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
+; SSE42-NEXT: psubq %xmm0, %xmm6
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; SSE42-NEXT: pmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
+; SSE42-NEXT: psubq %xmm0, %xmm5
+; SSE42-NEXT: movdqa %xmm1, %xmm0
+; SSE42-NEXT: psrldq {{.*#+}} xmm0 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE42-NEXT: pmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
+; SSE42-NEXT: psubq %xmm0, %xmm4
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm1[3,3,3,3]
+; SSE42-NEXT: pmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
+; SSE42-NEXT: psubq %xmm0, %xmm3
+; SSE42-NEXT: psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE42-NEXT: pmovzxbq {{.*#+}} xmm0 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; SSE42-NEXT: psubq %xmm0, %xmm2
+; SSE42-NEXT: pxor %xmm10, %xmm10
+; SSE42-NEXT: pxor %xmm1, %xmm1
+; SSE42-NEXT: psubq %xmm9, %xmm1
+; SSE42-NEXT: pmaxsw %xmm9, %xmm1
+; SSE42-NEXT: pxor %xmm0, %xmm0
+; SSE42-NEXT: psubq %xmm8, %xmm0
+; SSE42-NEXT: pmaxsw %xmm8, %xmm0
+; SSE42-NEXT: pxor %xmm8, %xmm8
+; SSE42-NEXT: psubq %xmm7, %xmm8
+; SSE42-NEXT: pmaxsw %xmm7, %xmm8
+; SSE42-NEXT: pxor %xmm7, %xmm7
+; SSE42-NEXT: psubq %xmm6, %xmm7
+; SSE42-NEXT: pmaxsw %xmm6, %xmm7
+; SSE42-NEXT: pxor %xmm6, %xmm6
+; SSE42-NEXT: psubq %xmm5, %xmm6
+; SSE42-NEXT: pmaxsw %xmm5, %xmm6
+; SSE42-NEXT: pxor %xmm5, %xmm5
+; SSE42-NEXT: psubq %xmm4, %xmm5
+; SSE42-NEXT: pmaxsw %xmm4, %xmm5
+; SSE42-NEXT: pxor %xmm4, %xmm4
+; SSE42-NEXT: psubq %xmm3, %xmm4
+; SSE42-NEXT: pmaxsw %xmm3, %xmm4
+; SSE42-NEXT: psubq %xmm2, %xmm10
+; SSE42-NEXT: pmaxsw %xmm2, %xmm10
+; SSE42-NEXT: pmovzxbq {{.*#+}} xmm2 = [255,255]
+; SSE42-NEXT: pand %xmm2, %xmm10
+; SSE42-NEXT: pand %xmm2, %xmm4
+; SSE42-NEXT: packusdw %xmm10, %xmm4
+; SSE42-NEXT: pand %xmm2, %xmm5
+; SSE42-NEXT: pand %xmm2, %xmm6
+; SSE42-NEXT: packusdw %xmm5, %xmm6
+; SSE42-NEXT: packusdw %xmm4, %xmm6
+; SSE42-NEXT: pand %xmm2, %xmm7
+; SSE42-NEXT: pand %xmm2, %xmm8
+; SSE42-NEXT: packusdw %xmm7, %xmm8
+; SSE42-NEXT: pand %xmm2, %xmm0
+; SSE42-NEXT: pand %xmm2, %xmm1
+; SSE42-NEXT: packusdw %xmm1, %xmm0
+; SSE42-NEXT: packusdw %xmm8, %xmm0
+; SSE42-NEXT: packuswb %xmm6, %xmm0
+; SSE42-NEXT: retq
+;
+; AVX1-LABEL: abd_ext_v16i8:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[1,1,1,1]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm5 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
+; AVX1-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[2,3,2,3]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero
+; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
+; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm7 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX1-NEXT: vpshufd {{.*#+}} xmm8 = xmm7[2,3,2,3]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm8 = xmm8[0],zero,zero,zero,xmm8[1],zero,zero,zero
+; AVX1-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[3,3,3,3]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero
+; AVX1-NEXT: vpsrld $16, %xmm0, %xmm9
+; AVX1-NEXT: vpmovzxbq {{.*#+}} xmm9 = xmm9[0],zero,zero,zero,zero,zero,zero,zero,xmm9[1],zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT: vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm10 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm11 = xmm10[1,1,1,1]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm11 = xmm11[0],zero,zero,zero,xmm11[1],zero,zero,zero
+; AVX1-NEXT: vpsubq %xmm11, %xmm4, %xmm4
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm11 = xmm10[0],zero,zero,zero,xmm10[1],zero,zero,zero
+; AVX1-NEXT: vpsubq %xmm11, %xmm5, %xmm5
+; AVX1-NEXT: vpshufd {{.*#+}} xmm11 = xmm10[2,3,2,3]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm11 = xmm11[0],zero,zero,zero,xmm11[1],zero,zero,zero
+; AVX1-NEXT: vpsubq %xmm11, %xmm6, %xmm6
+; AVX1-NEXT: vpshufd {{.*#+}} xmm10 = xmm10[3,3,3,3]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm10 = xmm10[0],zero,zero,zero,xmm10[1],zero,zero,zero
+; AVX1-NEXT: vpsubq %xmm10, %xmm3, %xmm3
+; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm10 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX1-NEXT: vpshufd {{.*#+}} xmm11 = xmm10[2,3,2,3]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm11 = xmm11[0],zero,zero,zero,xmm11[1],zero,zero,zero
+; AVX1-NEXT: vpsubq %xmm11, %xmm8, %xmm8
+; AVX1-NEXT: vpshufd {{.*#+}} xmm10 = xmm10[3,3,3,3]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm10 = xmm10[0],zero,zero,zero,xmm10[1],zero,zero,zero
+; AVX1-NEXT: vpsubq %xmm10, %xmm7, %xmm7
+; AVX1-NEXT: vpsrld $16, %xmm1, %xmm10
+; AVX1-NEXT: vpmovzxbq {{.*#+}} xmm10 = xmm10[0],zero,zero,zero,zero,zero,zero,zero,xmm10[1],zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT: vpsubq %xmm10, %xmm9, %xmm9
+; AVX1-NEXT: vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT: vpsubq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpsubq %xmm4, %xmm2, %xmm1
+; AVX1-NEXT: vpmaxsd %xmm1, %xmm4, %xmm1
+; AVX1-NEXT: vpsubq %xmm5, %xmm2, %xmm4
+; AVX1-NEXT: vpmaxsd %xmm4, %xmm5, %xmm4
+; AVX1-NEXT: vpsubq %xmm6, %xmm2, %xmm5
+; AVX1-NEXT: vpmaxsd %xmm5, %xmm6, %xmm5
+; AVX1-NEXT: vpsubq %xmm3, %xmm2, %xmm6
+; AVX1-NEXT: vpmaxsd %xmm6, %xmm3, %xmm3
+; AVX1-NEXT: vpsubq %xmm8, %xmm2, %xmm6
+; AVX1-NEXT: vpmaxsd %xmm6, %xmm8, %xmm6
+; AVX1-NEXT: vpsubq %xmm7, %xmm2, %xmm8
+; AVX1-NEXT: vpmaxsd %xmm8, %xmm7, %xmm7
+; AVX1-NEXT: vpsubq %xmm9, %xmm2, %xmm8
+; AVX1-NEXT: vpmaxsw %xmm8, %xmm9, %xmm8
+; AVX1-NEXT: vpsubq %xmm0, %xmm2, %xmm2
+; AVX1-NEXT: vpmaxsw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT: vpmovzxbq {{.*#+}} xmm2 = [255,255]
+; AVX1-NEXT: vpand %xmm2, %xmm3, %xmm3
+; AVX1-NEXT: vpand %xmm2, %xmm5, %xmm5
+; AVX1-NEXT: vpackusdw %xmm3, %xmm5, %xmm3
+; AVX1-NEXT: vpand %xmm2, %xmm4, %xmm4
+; AVX1-NEXT: vpand %xmm2, %xmm1, %xmm1
+; AVX1-NEXT: vpackusdw %xmm1, %xmm4, %xmm1
+; AVX1-NEXT: vpackusdw %xmm3, %xmm1, %xmm1
+; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm0
+; AVX1-NEXT: vpand %xmm2, %xmm8, %xmm3
+; AVX1-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
+; AVX1-NEXT: vpand %xmm2, %xmm7, %xmm3
+; AVX1-NEXT: vpand %xmm2, %xmm6, %xmm2
+; AVX1-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
+; AVX1-NEXT: vpackusdw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: retq
+;
+; AVX2-LABEL: abd_ext_v16i8:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
+; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; AVX2-NEXT: vpmovzxbq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero
+; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm2
+; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]
+; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
+; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
+; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm5 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX2-NEXT: vpshufd {{.*#+}} xmm6 = xmm5[2,3,2,3]
+; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero
+; AVX2-NEXT: vpsubq %ymm6, %ymm3, %ymm3
+; AVX2-NEXT: vpmovzxbq {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero,xmm1[2],zero,zero,zero,zero,zero,zero,zero,xmm1[3],zero,zero,zero,zero,zero,zero,zero
+; AVX2-NEXT: vpsubq %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm5, %xmm1
+; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[2,3,2,3]
+; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero
+; AVX2-NEXT: vpsubq %ymm5, %ymm4, %ymm4
+; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; AVX2-NEXT: vpsubq %ymm1, %ymm2, %ymm1
+; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT: vpsubq %ymm3, %ymm2, %ymm5
+; AVX2-NEXT: vpmaxsw %ymm5, %ymm3, %ymm3
+; AVX2-NEXT: vpsubq %ymm0, %ymm2, %ymm5
+; AVX2-NEXT: vpmaxsw %ymm5, %ymm0, %ymm0
+; AVX2-NEXT: vpsubq %ymm4, %ymm2, %ymm5
+; AVX2-NEXT: vpmaxsw %ymm5, %ymm4, %ymm4
+; AVX2-NEXT: vpsubq %ymm1, %ymm2, %ymm2
+; AVX2-NEXT: vpmaxsw %ymm2, %ymm1, %ymm1
+; AVX2-NEXT: vpbroadcastq {{.*#+}} ymm2 = [255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0]
+; AVX2-NEXT: vpand %ymm2, %ymm1, %ymm1
+; AVX2-NEXT: vpand %ymm2, %ymm4, %ymm4
+; AVX2-NEXT: vpackusdw %ymm4, %ymm1, %ymm1
+; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
+; AVX2-NEXT: vpand %ymm2, %ymm0, %ymm0
+; AVX2-NEXT: vpand %ymm2, %ymm3, %ymm2
+; AVX2-NEXT: vpackusdw %ymm2, %ymm0, %ymm0
+; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT: vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: abd_ext_v16i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512-NEXT: vpmovzxwq {{.*#+}} zmm2 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX512-NEXT: vpmovzxwq {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; AVX512-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512-NEXT: vpmovzxwq {{.*#+}} zmm3 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero
+; AVX512-NEXT: vpsubq %zmm3, %zmm2, %zmm2
+; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm1
+; AVX512-NEXT: vpmovzxwq {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero
+; AVX512-NEXT: vpsubq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpsubq %zmm2, %zmm1, %zmm3
+; AVX512-NEXT: vpmaxsq %zmm3, %zmm2, %zmm2
+; AVX512-NEXT: vpsubq %zmm0, %zmm1, %zmm1
+; AVX512-NEXT: vpmaxsq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpmovqb %zmm0, %xmm0
+; AVX512-NEXT: vpmovqb %zmm2, %xmm1
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%aext = zext <16 x i8> %a to <16 x i64>
%bext = zext <16 x i8> %b to <16 x i64>
%sub = sub <16 x i64> %aext, %bext
@@ -58,26 +373,181 @@ define <16 x i8> @abd_ext_v16i8_undef(<16 x i8> %a, <16 x i8> %b) nounwind {
define <8 x i16> @abd_ext_v8i16(<8 x i16> %a, <8 x i16> %b) nounwind {
; SSE2-LABEL: abd_ext_v8i16:
; SSE2: # %bb.0:
-; SSE2-NEXT: movdqa %xmm1, %xmm2
-; SSE2-NEXT: psubusw %xmm0, %xmm2
-; SSE2-NEXT: psubusw %xmm1, %xmm0
-; SSE2-NEXT: por %xmm2, %xmm0
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: movdqa %xmm0, %xmm4
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
+; SSE2-NEXT: movdqa %xmm4, %xmm5
+; SSE2-NEXT: punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; SSE2-NEXT: movdqa %xmm0, %xmm3
+; SSE2-NEXT: punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSE2-NEXT: movdqa %xmm1, %xmm6
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
+; SSE2-NEXT: movdqa %xmm6, %xmm7
+; SSE2-NEXT: punpckhdq {{.*#+}} xmm7 = xmm7[2],xmm2[2],xmm7[3],xmm2[3]
+; SSE2-NEXT: psubq %xmm7, %xmm5
+; SSE2-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1]
+; SSE2-NEXT: psubq %xmm6, %xmm4
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; SSE2-NEXT: movdqa %xmm1, %xmm6
+; SSE2-NEXT: punpckhdq {{.*#+}} xmm6 = xmm6[2],xmm2[2],xmm6[3],xmm2[3]
+; SSE2-NEXT: psubq %xmm6, %xmm3
+; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSE2-NEXT: psubq %xmm1, %xmm0
+; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: psubq %xmm5, %xmm1
+; SSE2-NEXT: movdqa %xmm5, %xmm6
+; SSE2-NEXT: pcmpgtd %xmm1, %xmm6
+; SSE2-NEXT: pand %xmm6, %xmm5
+; SSE2-NEXT: pandn %xmm1, %xmm6
+; SSE2-NEXT: por %xmm5, %xmm6
+; SSE2-NEXT: pxor %xmm5, %xmm5
+; SSE2-NEXT: psubq %xmm4, %xmm5
+; SSE2-NEXT: movdqa %xmm4, %xmm1
+; SSE2-NEXT: pcmpgtd %xmm5, %xmm1
+; SSE2-NEXT: pand %xmm1, %xmm4
+; SSE2-NEXT: pandn %xmm5, %xmm1
+; SSE2-NEXT: por %xmm4, %xmm1
+; SSE2-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,2],xmm6[0,2]
+; SSE2-NEXT: pxor %xmm4, %xmm4
+; SSE2-NEXT: psubq %xmm3, %xmm4
+; SSE2-NEXT: movdqa %xmm3, %xmm5
+; SSE2-NEXT: pcmpgtd %xmm4, %xmm5
+; SSE2-NEXT: pand %xmm5, %xmm3
+; SSE2-NEXT: pandn %xmm4, %xmm5
+; SSE2-NEXT: por %xmm3, %xmm5
+; SSE2-NEXT: psubq %xmm0, %xmm2
+; SSE2-NEXT: movdqa %xmm0, %xmm3
+; SSE2-NEXT: pcmpgtd %xmm2, %xmm3
+; SSE2-NEXT: pand %xmm3, %xmm0
+; SSE2-NEXT: pandn %xmm2, %xmm3
+; SSE2-NEXT: por %xmm0, %xmm3
+; SSE2-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,2],xmm5[0,2]
+; SSE2-NEXT: pslld $16, %xmm3
+; SSE2-NEXT: psrad $16, %xmm3
+; SSE2-NEXT: pslld $16, %xmm1
+; SSE2-NEXT: psrad $16, %xmm1
+; SSE2-NEXT: packssdw %xmm3, %xmm1
+; SSE2-NEXT: movdqa %xmm1, %xmm0
; SSE2-NEXT: retq
;
; SSE42-LABEL: abd_ext_v8i16:
; SSE42: # %bb.0:
-; SSE42-NEXT: movdqa %xmm0, %xmm2
-; SSE42-NEXT: pminuw %xmm1, %xmm2
-; SSE42-NEXT: pmaxuw %xmm1, %xmm0
-; SSE42-NEXT: psubw %xmm2, %xmm0
+; SSE42-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,1,1]
+; SSE42-NEXT: pmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
+; SSE42-NEXT: pmovzxwq {{.*#+}} xmm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
+; SSE42-NEXT: pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; SSE42-NEXT: pmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; SSE42-NEXT: pmovzxwq {{.*#+}} xmm5 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE42-NEXT: pmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
+; SSE42-NEXT: psubq %xmm0, %xmm2
+; SSE42-NEXT: pmovzxwq {{.*#+}} xmm0 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; SSE42-NEXT: psubq %xmm0, %xmm3
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; SSE42-NEXT: pmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
+; SSE42-NEXT: psubq %xmm0, %xmm4
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm1[3,3,3,3]
+; SSE42-NEXT: pmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
+; SSE42-NEXT: psubq %xmm0, %xmm5
+; SSE42-NEXT: pxor %xmm1, %xmm1
+; SSE42-NEXT: pxor %xmm6, %xmm6
+; SSE42-NEXT: psubq %xmm2, %xmm6
+; SSE42-NEXT: pmaxsd %xmm2, %xmm6
+; SSE42-NEXT: pxor %xmm0, %xmm0
+; SSE42-NEXT: psubq %xmm3, %xmm0
+; SSE42-NEXT: pmaxsd %xmm3, %xmm0
+; SSE42-NEXT: pxor %xmm2, %xmm2
+; SSE42-NEXT: psubq %xmm4, %xmm2
+; SSE42-NEXT: pmaxsd %xmm4, %xmm2
+; SSE42-NEXT: pxor %xmm3, %xmm3
+; SSE42-NEXT: psubq %xmm5, %xmm3
+; SSE42-NEXT: pmaxsd %xmm5, %xmm3
+; SSE42-NEXT: pblendw {{.*#+}} xmm3 = xmm3[0],xmm1[1,2,3],xmm3[4],xmm1[5,6,7]
+; SSE42-NEXT: pblendw {{.*#+}} xmm2 = xmm2[0],xmm1[1,2,3],xmm2[4],xmm1[5,6,7]
+; SSE42-NEXT: packusdw %xmm3, %xmm2
+; SSE42-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3],xmm0[4],xmm1[5,6,7]
+; SSE42-NEXT: pblendw {{.*#+}} xmm6 = xmm6[0],xmm1[1,2,3],xmm6[4],xmm1[5,6,7]
+; SSE42-NEXT: packusdw %xmm6, %xmm0
+; SSE42-NEXT: packusdw %xmm2, %xmm0
; SSE42-NEXT: retq
;
-; AVX-LABEL: abd_ext_v8i16:
-; AVX: # %bb.0:
-; AVX-NEXT: vpminuw %xmm1, %xmm0, %xmm2
-; AVX-NEXT: vpmaxuw %xmm1, %xmm0, %xmm0
-; AVX-NEXT: vpsubw %xmm2, %xmm0, %xmm0
-; AVX-NEXT: retq
+; AVX1-LABEL: abd_ext_v8i16:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[1,1,1,1]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
+; AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
+; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
+; AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[1,1,1,1]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero
+; AVX1-NEXT: vpsubq %xmm5, %xmm2, %xmm2
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm5 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX1-NEXT: vpsubq %xmm5, %xmm3, %xmm3
+; AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[2,3,2,3]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero
+; AVX1-NEXT: vpsubq %xmm5, %xmm4, %xmm4
+; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX1-NEXT: vpsubq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT: vpsubq %xmm2, %xmm1, %xmm5
+; AVX1-NEXT: vpmaxsd %xmm5, %xmm2, %xmm2
+; AVX1-NEXT: vpsubq %xmm3, %xmm1, %xmm5
+; AVX1-NEXT: vpmaxsd %xmm5, %xmm3, %xmm3
+; AVX1-NEXT: vpsubq %xmm4, %xmm1, %xmm5
+; AVX1-NEXT: vpmaxsd %xmm5, %xmm4, %xmm4
+; AVX1-NEXT: vpsubq %xmm0, %xmm1, %xmm5
+; AVX1-NEXT: vpmaxsd %xmm5, %xmm0, %xmm0
+; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3],xmm0[4],xmm1[5,6,7]
+; AVX1-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0],xmm1[1,2,3],xmm4[4],xmm1[5,6,7]
+; AVX1-NEXT: vpackusdw %xmm0, %xmm4, %xmm0
+; AVX1-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0],xmm1[1,2,3],xmm3[4],xmm1[5,6,7]
+; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3],xmm2[4],xmm1[5,6,7]
+; AVX1-NEXT: vpackusdw %xmm1, %xmm3, %xmm1
+; AVX1-NEXT: vpackusdw %xmm0, %xmm1, %xmm0
+; AVX1-NEXT: retq
+;
+; AVX2-LABEL: abd_ext_v8i16:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
+; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
+; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; AVX2-NEXT: vpsubq %ymm3, %ymm2, %ymm2
+; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; AVX2-NEXT: vpsubq %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpsubq %ymm2, %ymm1, %ymm3
+; AVX2-NEXT: vpmaxsd %ymm3, %ymm2, %ymm2
+; AVX2-NEXT: vpsubq %ymm0, %ymm1, %ymm3
+; AVX2-NEXT: vpmaxsd %ymm3, %ymm0, %ymm0
+; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7],ymm0[8],ymm1[9,10,11],ymm0[12],ymm1[13,14,15]
+; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1,2,3],ymm2[4],ymm1[5,6,7],ymm2[8],ymm1[9,10,11],ymm2[12],ymm1[13,14,15]
+; AVX2-NEXT: vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: abd_ext_v8i16:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpmovzxwq {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; AVX512-NEXT: vpmovzxwq {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero
+; AVX512-NEXT: vpsubq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpsubq %zmm0, %zmm1, %zmm1
+; AVX512-NEXT: vpmaxsq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpmovqw %zmm0, %xmm0
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%aext = zext <8 x i16> %a to <8 x i64>
%bext = zext <8 x i16> %b to <8 x i64>
%sub = sub <8 x i64> %aext, %bext
@@ -120,31 +590,120 @@ define <8 x i16> @abd_ext_v8i16_undef(<8 x i16> %a, <8 x i16> %b) nounwind {
define <4 x i32> @abd_ext_v4i32(<4 x i32> %a, <4 x i32> %b) nounwind {
; SSE2-LABEL: abd_ext_v4i32:
; SSE2: # %bb.0:
-; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [2147483648,2147483648,2147483648,2147483648]
-; SSE2-NEXT: movdqa %xmm1, %xmm3
-; SSE2-NEXT: pxor %xmm2, %xmm3
-; SSE2-NEXT: pxor %xmm0, %xmm2
-; SSE2-NEXT: pcmpgtd %xmm3, %xmm2
-; SSE2-NEXT: psubd %xmm1, %xmm0
-; SSE2-NEXT: pxor %xmm2, %xmm0
-; SSE2-NEXT: psubd %xmm0, %xmm2
-; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: movdqa %xmm0, %xmm3
+; SSE2-NEXT: punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSE2-NEXT: movdqa %xmm1, %xmm4
+; SSE2-NEXT: punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm2[2],xmm4[3],xmm2[3]
+; SSE2-NEXT: psubq %xmm4, %xmm3
+; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSE2-NEXT: psubq %xmm1, %xmm0
+; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [2147483648,2147483648]
+; SSE2-NEXT: movdqa %xmm3, %xmm4
+; SSE2-NEXT: pxor %xmm1, %xmm4
+; SSE2-NEXT: pxor %xmm5, %xmm5
+; SSE2-NEXT: psubq %xmm3, %xmm5
+; SSE2-NEXT: movdqa %xmm5, %xmm6
+; SSE2-NEXT: pxor %xmm1, %xmm6
+; SSE2-NEXT: movdqa %xmm4, %xmm7
+; SSE2-NEXT: pcmpgtd %xmm6, %xmm7
+; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,0,2,2]
+; SSE2-NEXT: pcmpeqd %xmm4, %xmm6
+; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; SSE2-NEXT: pand %xmm8, %xmm6
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm7[1,1,3,3]
+; SSE2-NEXT: por %xmm6, %xmm4
+; SSE2-NEXT: pand %xmm4, %xmm3
+; SSE2-NEXT: pandn %xmm5, %xmm4
+; SSE2-NEXT: por %xmm3, %xmm4
+; SSE2-NEXT: movdqa %xmm0, %xmm3
+; SSE2-NEXT: pxor %xmm1, %xmm3
+; SSE2-NEXT: psubq %xmm0, %xmm2
+; SSE2-NEXT: pxor %xmm2, %xmm1
+; SSE2-NEXT: movdqa %xmm3, %xmm5
+; SSE2-NEXT: pcmpgtd %xmm1, %xmm5
+; SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm5[0,0,2,2]
+; SSE2-NEXT: pcmpeqd %xmm3, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; SSE2-NEXT: pand %xmm6, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm5[1,1,3,3]
+; SSE2-NEXT: por %xmm1, %xmm3
+; SSE2-NEXT: pand %xmm3, %xmm0
+; SSE2-NEXT: pandn %xmm2, %xmm3
+; SSE2-NEXT: por %xmm3, %xmm0
+; SSE2-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,2],xmm4[0,2]
; SSE2-NEXT: retq
;
; SSE42-LABEL: abd_ext_v4i32:
; SSE42: # %bb.0:
-; SSE42-NEXT: movdqa %xmm0, %xmm2
-; SSE42-NEXT: pminud %xmm1, %xmm2
-; SSE42-NEXT: pmaxud %xmm1, %xmm0
-; SSE42-NEXT: psubd %xmm2, %xmm0
+; SSE42-NEXT: pxor %xmm2, %xmm2
+; SSE42-NEXT: pmovzxdq {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero
+; SSE42-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE42-NEXT: pmovzxdq {{.*#+}} xmm4 = xmm1[0],zero,xmm1[1],zero
+; SSE42-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; SSE42-NEXT: psubq %xmm1, %xmm0
+; SSE42-NEXT: psubq %xmm4, %xmm3
+; SSE42-NEXT: pxor %xmm1, %xmm1
+; SSE42-NEXT: psubq %xmm0, %xmm1
+; SSE42-NEXT: movdqa %xmm0, %xmm4
+; SSE42-NEXT: pcmpgtq %xmm1, %xmm4
+; SSE42-NEXT: pxor %xmm4, %xmm0
+; SSE42-NEXT: psubq %xmm0, %xmm4
+; SSE42-NEXT: psubq %xmm3, %xmm2
+; SSE42-NEXT: movdqa %xmm3, %xmm0
+; SSE42-NEXT: pcmpgtq %xmm2, %xmm0
+; SSE42-NEXT: pxor %xmm0, %xmm3
+; SSE42-NEXT: psubq %xmm3, %xmm0
+; SSE42-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,2],xmm4[0,2]
; SSE42-NEXT: retq
;
-; AVX-LABEL: abd_ext_v4i32:
-; AVX: # %bb.0:
-; AVX-NEXT: vpminud %xmm1, %xmm0, %xmm2
-; AVX-NEXT: vpmaxud %xmm1, %xmm0, %xmm0
-; AVX-NEXT: vpsubd %xmm2, %xmm0, %xmm0
-; AVX-NEXT: retq
+; AVX1-LABEL: abd_ext_v4i32:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm3 = xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm4 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; AVX1-NEXT: vpsubq %xmm4, %xmm3, %xmm3
+; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
+; AVX1-NEXT: vpsubq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpsubq %xmm3, %xmm2, %xmm1
+; AVX1-NEXT: vpcmpgtq %xmm1, %xmm3, %xmm1
+; AVX1-NEXT: vpxor %xmm1, %xmm3, %xmm3
+; AVX1-NEXT: vpsubq %xmm3, %xmm1, %xmm1
+; AVX1-NEXT: vpsubq %xmm0, %xmm2, %xmm2
+; AVX1-NEXT: vpcmpgtq %xmm2, %xmm0, %xmm2
+; AVX1-NEXT: vpxor %xmm2, %xmm0, %xmm0
+; AVX1-NEXT: vpsubq %xmm0, %xmm2, %xmm0
+; AVX1-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; AVX1-NEXT: retq
+;
+; AVX2-LABEL: abd_ext_v4i32:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX2-NEXT: vpsubq %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpsubq %ymm0, %ymm1, %ymm1
+; AVX2-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm1
+; AVX2-NEXT: vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpsubq %ymm0, %ymm1, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: abd_ext_v4i32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX512-NEXT: vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX512-NEXT: vpsubq %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpsubq %ymm0, %ymm1, %ymm1
+; AVX512-NEXT: vpmaxsq %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpmovqd %ymm0, %xmm0
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%aext = zext <4 x i32> %a to <4 x i64>
%bext = zext <4 x i32> %b to <4 x i64>
%sub = sub <4 x i64> %aext, %bext
@@ -192,66 +751,98 @@ define <4 x i32> @abd_ext_v4i32_undef(<4 x i32> %a, <4 x i32> %b) nounwind {
define <2 x i64> @abd_ext_v2i64(<2 x i64> %a, <2 x i64> %b) nounwind {
; SSE2-LABEL: abd_ext_v2i64:
; SSE2: # %bb.0:
-; SSE2-NEXT: movdqa {{.*#+}} xmm2 = [9223372039002259456,9223372039002259456]
-; SSE2-NEXT: movdqa %xmm1, %xmm3
-; SSE2-NEXT: pxor %xmm2, %xmm3
-; SSE2-NEXT: pxor %xmm0, %xmm2
-; SSE2-NEXT: movdqa %xmm2, %xmm4
-; SSE2-NEXT: pcmpgtd %xmm3, %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,0,2,2]
-; SSE2-NEXT: pcmpeqd %xmm3, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
-; SSE2-NEXT: pand %xmm5, %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
-; SSE2-NEXT: por %xmm3, %xmm2
-; SSE2-NEXT: psubq %xmm1, %xmm0
-; SSE2-NEXT: pxor %xmm2, %xmm0
-; SSE2-NEXT: psubq %xmm0, %xmm2
-; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
+; SSE2-NEXT: movq %xmm2, %rcx
+; SSE2-NEXT: movq %xmm0, %rax
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; SSE2-NEXT: movq %xmm0, %rdx
+; SSE2-NEXT: movq %xmm1, %rsi
+; SSE2-NEXT: xorl %edi, %edi
+; SSE2-NEXT: subq %rsi, %rax
+; SSE2-NEXT: movl $0, %esi
+; SSE2-NEXT: sbbq %rsi, %rsi
+; SSE2-NEXT: subq %rdx, %rcx
+; SSE2-NEXT: movl $0, %edx
+; SSE2-NEXT: sbbq %rdx, %rdx
+; SSE2-NEXT: movq %rcx, %r8
+; SSE2-NEXT: negq %r8
+; SSE2-NEXT: movl $0, %r9d
+; SSE2-NEXT: sbbq %rdx, %r9
+; SSE2-NEXT: cmpq %rcx, %r8
+; SSE2-NEXT: sbbq %rdx, %r9
+; SSE2-NEXT: cmovlq %rcx, %r8
+; SSE2-NEXT: movq %rax, %rcx
+; SSE2-NEXT: negq %rcx
+; SSE2-NEXT: sbbq %rsi, %rdi
+; SSE2-NEXT: cmpq %rax, %rcx
+; SSE2-NEXT: sbbq %rsi, %rdi
+; SSE2-NEXT: cmovlq %rax, %rcx
+; SSE2-NEXT: movq %rcx, %xmm0
+; SSE2-NEXT: movq %r8, %xmm1
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; SSE2-NEXT: retq
;
; SSE42-LABEL: abd_ext_v2i64:
; SSE42: # %bb.0:
-; SSE42-NEXT: movdqa {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
-; SSE42-NEXT: movdqa %xmm1, %xmm3
-; SSE42-NEXT: pxor %xmm2, %xmm3
-; SSE42-NEXT: pxor %xmm0, %xmm2
-; SSE42-NEXT: pcmpgtq %xmm3, %xmm2
-; SSE42-NEXT: psubq %xmm1, %xmm0
-; SSE42-NEXT: pxor %xmm2, %xmm0
-; SSE42-NEXT: psubq %xmm0, %xmm2
-; SSE42-NEXT: movdqa %xmm2, %xmm0
+; SSE42-NEXT: movq %xmm0, %rax
+; SSE42-NEXT: pextrq $1, %xmm0, %rcx
+; SSE42-NEXT: movq %xmm1, %rdx
+; SSE42-NEXT: pextrq $1, %xmm1, %rsi
+; SSE42-NEXT: xorl %edi, %edi
+; SSE42-NEXT: subq %rsi, %rcx
+; SSE42-NEXT: movl $0, %esi
+; SSE42-NEXT: sbbq %rsi, %rsi
+; SSE42-NEXT: subq %rdx, %rax
+; SSE42-NEXT: movl $0, %edx
+; SSE42-NEXT: sbbq %rdx, %rdx
+; SSE42-NEXT: movq %rax, %r8
+; SSE42-NEXT: negq %r8
+; SSE42-NEXT: movl $0, %r9d
+; SSE42-NEXT: sbbq %rdx, %r9
+; SSE42-NEXT: cmpq %rax, %r8
+; SSE42-NEXT: sbbq %rdx, %r9
+; SSE42-NEXT: cmovlq %rax, %r8
+; SSE42-NEXT: movq %rcx, %rax
+; SSE42-NEXT: negq %rax
+; SSE42-NEXT: sbbq %rsi, %rdi
+; SSE42-NEXT: cmpq %rcx, %rax
+; SSE42-NEXT: sbbq %rsi, %rdi
+; SSE42-NEXT: cmovlq %rcx, %rax
+; SSE42-NEXT: movq %rax, %xmm1
+; SSE42-NEXT: movq %r8, %xmm0
+; SSE42-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; SSE42-NEXT: retq
;
-; AVX1-LABEL: abd_ext_v2i64:
-; AVX1: # %bb.0:
-; AVX1-NEXT: vmovddup {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
-; AVX1-NEXT: # xmm2 = mem[0,0]
-; AVX1-NEXT: vpxor %xmm2, %xmm1, %xmm3
-; AVX1-NEXT: vpxor %xmm2, %xmm0, %xmm2
-; AVX1-NEXT: vpcmpgtq %xmm3, %xmm2, %xmm2
-; AVX1-NEXT: vpsubq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpxor %xmm2, %xmm0, %xmm0
-; AVX1-NEXT: vpsubq %xmm0, %xmm2, %xmm0
-; AVX1-NEXT: retq
-;
-; AVX2-LABEL: abd_ext_v2i64:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpbroadcastq {{.*#+}} xmm2 = [9223372036854775808,9223372036854775808]
-; AVX2-NEXT: vpxor %xmm2, %xmm1, %xmm3
-; AVX2-NEXT: vpxor %xmm2, %xmm0, %xmm2
-; AVX2-NEXT: vpcmpgtq %xmm3, %xmm2, %xmm2
-; AVX2-NEXT: vpsubq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpxor %xmm2, %xmm0, %xmm0
-; AVX2-NEXT: vpsubq %xmm0, %xmm2, %xmm0
-; AVX2-NEXT: retq
-;
-; AVX512-LABEL: abd_ext_v2i64:
-; AVX512: # %bb.0:
-; AVX512-NEXT: vpminuq %xmm1, %xmm0, %xmm2
-; AVX512-NEXT: vpmaxuq %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vpsubq %xmm2, %xmm0, %xmm0
-; AVX512-NEXT: retq
+; AVX-LABEL: abd_ext_v2i64:
+; AVX: # %bb.0:
+; AVX-NEXT: vmovq %xmm0, %rax
+; AVX-NEXT: vpextrq $1, %xmm0, %rcx
+; AVX-NEXT: vmovq %xmm1, %rdx
+; AVX-NEXT: vpextrq $1, %xmm1, %rsi
+; AVX-NEXT: xorl %edi, %edi
+; AVX-NEXT: subq %rsi, %rcx
+; AVX-NEXT: movl $0, %esi
+; AVX-NEXT: sbbq %rsi, %rsi
+; AVX-NEXT: subq %rdx, %rax
+; AVX-NEXT: movl $0, %edx
+; AVX-NEXT: sbbq %rdx, %rdx
+; AVX-NEXT: movq %rax, %r8
+; AVX-NEXT: negq %r8
+; AVX-NEXT: movl $0, %r9d
+; AVX-NEXT: sbbq %rdx, %r9
+; AVX-NEXT: cmpq %rax, %r8
+; AVX-NEXT: sbbq %rdx, %r9
+; AVX-NEXT: cmovlq %rax, %r8
+; AVX-NEXT: movq %rcx, %rax
+; AVX-NEXT: negq %rax
+; AVX-NEXT: sbbq %rsi, %rdi
+; AVX-NEXT: cmpq %rcx, %rax
+; AVX-NEXT: sbbq %rsi, %rdi
+; AVX-NEXT: cmovlq %rcx, %rax
+; AVX-NEXT: vmovq %rax, %xmm0
+; AVX-NEXT: vmovq %r8, %xmm1
+; AVX-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX-NEXT: retq
%aext = zext <2 x i64> %a to <2 x i128>
%bext = zext <2 x i64> %b to <2 x i128>
%sub = sub <2 x i128> %aext, %bext
diff --git a/llvm/test/CodeGen/X86/abdu-vector-256.ll b/llvm/test/CodeGen/X86/abdu-vector-256.ll
index 080fb779fecb2..5f7c8b3cc3835 100644
--- a/llvm/test/CodeGen/X86/abdu-vector-256.ll
+++ b/llvm/test/CodeGen/X86/abdu-vector-256.ll
@@ -10,29 +10,292 @@
define <32 x i8> @abd_ext_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
; AVX1-LABEL: abd_ext_v32i8:
; AVX1: # %bb.0:
-; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
-; AVX1-NEXT: vpminub %xmm2, %xmm3, %xmm4
-; AVX1-NEXT: vpmaxub %xmm2, %xmm3, %xmm2
-; AVX1-NEXT: vpsubb %xmm4, %xmm2, %xmm2
-; AVX1-NEXT: vpminub %xmm1, %xmm0, %xmm3
-; AVX1-NEXT: vpmaxub %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpsubb %xmm3, %xmm0, %xmm0
-; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm4[1,1,1,1]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm5 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm3 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
+; AVX1-NEXT: vpshufd {{.*#+}} xmm6 = xmm4[2,3,2,3]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero
+; AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[3,3,3,3]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm7 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm11 = xmm4[8],xmm2[8],xmm4[9],xmm2[9],xmm4[10],xmm2[10],xmm4[11],xmm2[11],xmm4[12],xmm2[12],xmm4[13],xmm2[13],xmm4[14],xmm2[14],xmm4[15],xmm2[15]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm8 = xmm11[1,1,1,1]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm9 = xmm8[0],zero,zero,zero,xmm8[1],zero,zero,zero
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm8 = xmm11[0],zero,zero,zero,xmm11[1],zero,zero,zero
+; AVX1-NEXT: vpshufd {{.*#+}} xmm10 = xmm11[2,3,2,3]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm10 = xmm10[0],zero,zero,zero,xmm10[1],zero,zero,zero
+; AVX1-NEXT: vpshufd {{.*#+}} xmm11 = xmm11[3,3,3,3]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm11 = xmm11[0],zero,zero,zero,xmm11[1],zero,zero,zero
+; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm14 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm12 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm13 = xmm12[1,1,1,1]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm13 = xmm13[0],zero,zero,zero,xmm13[1],zero,zero,zero
+; AVX1-NEXT: vpsubq %xmm13, %xmm5, %xmm5
+; AVX1-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm5 = xmm12[0],zero,zero,zero,xmm12[1],zero,zero,zero
+; AVX1-NEXT: vpsubq %xmm5, %xmm3, %xmm3
+; AVX1-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm12[2,3,2,3]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
+; AVX1-NEXT: vpsubq %xmm3, %xmm6, %xmm3
+; AVX1-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm12[3,3,3,3]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
+; AVX1-NEXT: vpsubq %xmm3, %xmm7, %xmm3
+; AVX1-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm13
+; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm13[8],xmm2[8],xmm13[9],xmm2[9],xmm13[10],xmm2[10],xmm13[11],xmm2[11],xmm13[12],xmm2[12],xmm13[13],xmm2[13],xmm13[14],xmm2[14],xmm13[15],xmm2[15]
+; AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm3[1,1,1,1]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero
+; AVX1-NEXT: vpsubq %xmm5, %xmm9, %xmm9
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm5 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
+; AVX1-NEXT: vpsubq %xmm5, %xmm8, %xmm8
+; AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm3[2,3,2,3]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero
+; AVX1-NEXT: vpsubq %xmm5, %xmm10, %xmm10
+; AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm14[2,3,2,3]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero
+; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
+; AVX1-NEXT: vpsubq %xmm3, %xmm11, %xmm11
+; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX1-NEXT: vpshufd {{.*#+}} xmm12 = xmm3[2,3,2,3]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm12 = xmm12[0],zero,zero,zero,xmm12[1],zero,zero,zero
+; AVX1-NEXT: vpsubq %xmm12, %xmm5, %xmm12
+; AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm14[3,3,3,3]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero
+; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
+; AVX1-NEXT: vpsubq %xmm3, %xmm5, %xmm14
+; AVX1-NEXT: vpsrld $16, %xmm0, %xmm3
+; AVX1-NEXT: vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT: vpsrld $16, %xmm1, %xmm5
+; AVX1-NEXT: vpmovzxbq {{.*#+}} xmm5 = xmm5[0],zero,zero,zero,zero,zero,zero,zero,xmm5[1],zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT: vpsubq %xmm5, %xmm3, %xmm15
+; AVX1-NEXT: vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT: vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT: vpsubq %xmm1, %xmm0, %xmm7
+; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
+; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm3 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm13[0],zero,xmm13[1],zero,xmm13[2],zero,xmm13[3],zero,xmm13[4],zero,xmm13[5],zero,xmm13[6],zero,xmm13[7],zero
+; AVX1-NEXT: vpshufd {{.*#+}} xmm6 = xmm1[2,3,2,3]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero
+; AVX1-NEXT: vpsubq %xmm6, %xmm3, %xmm5
+; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
+; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX1-NEXT: vpsubq %xmm1, %xmm0, %xmm3
+; AVX1-NEXT: vpsrld $16, %xmm4, %xmm0
+; AVX1-NEXT: vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT: vpsrld $16, %xmm13, %xmm6
+; AVX1-NEXT: vpmovzxbq {{.*#+}} xmm6 = xmm6[0],zero,zero,zero,zero,zero,zero,zero,xmm6[1],zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT: vpsubq %xmm6, %xmm0, %xmm1
+; AVX1-NEXT: vpmovzxbq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,zero,zero,zero,zero,xmm4[1],zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT: vpmovzxbq {{.*#+}} xmm6 = xmm13[0],zero,zero,zero,zero,zero,zero,zero,xmm13[1],zero,zero,zero,zero,zero,zero,zero
+; AVX1-NEXT: vpsubq %xmm6, %xmm4, %xmm0
+; AVX1-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
+; AVX1-NEXT: vpsubq %xmm6, %xmm2, %xmm4
+; AVX1-NEXT: vpmaxsd %xmm4, %xmm6, %xmm4
+; AVX1-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
+; AVX1-NEXT: vpsubq %xmm6, %xmm2, %xmm4
+; AVX1-NEXT: vpmaxsd %xmm4, %xmm6, %xmm4
+; AVX1-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
+; AVX1-NEXT: vpsubq %xmm6, %xmm2, %xmm4
+; AVX1-NEXT: vpmaxsd %xmm4, %xmm6, %xmm4
+; AVX1-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX1-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; AVX1-NEXT: vpsubq %xmm4, %xmm2, %xmm6
+; AVX1-NEXT: vpmaxsd %xmm6, %xmm4, %xmm6
+; AVX1-NEXT: vpsubq %xmm9, %xmm2, %xmm13
+; AVX1-NEXT: vpmaxsd %xmm13, %xmm9, %xmm13
+; AVX1-NEXT: vpsubq %xmm8, %xmm2, %xmm9
+; AVX1-NEXT: vpmaxsd %xmm9, %xmm8, %xmm4
+; AVX1-NEXT: vpsubq %xmm10, %xmm2, %xmm8
+; AVX1-NEXT: vpmaxsd %xmm8, %xmm10, %xmm9
+; AVX1-NEXT: vpsubq %xmm11, %xmm2, %xmm8
+; AVX1-NEXT: vpmaxsd %xmm8, %xmm11, %xmm11
+; AVX1-NEXT: vpsubq %xmm12, %xmm2, %xmm8
+; AVX1-NEXT: vpmaxsd %xmm8, %xmm12, %xmm12
+; AVX1-NEXT: vpsubq %xmm14, %xmm2, %xmm8
+; AVX1-NEXT: vpmaxsd %xmm8, %xmm14, %xmm10
+; AVX1-NEXT: vpsubq %xmm15, %xmm2, %xmm8
+; AVX1-NEXT: vpmaxsw %xmm8, %xmm15, %xmm8
+; AVX1-NEXT: vpsubq %xmm7, %xmm2, %xmm14
+; AVX1-NEXT: vpmaxsw %xmm7, %xmm14, %xmm7
+; AVX1-NEXT: vpsubq %xmm5, %xmm2, %xmm14
+; AVX1-NEXT: vpmaxsd %xmm14, %xmm5, %xmm5
+; AVX1-NEXT: vpsubq %xmm3, %xmm2, %xmm14
+; AVX1-NEXT: vpmaxsd %xmm14, %xmm3, %xmm3
+; AVX1-NEXT: vpsubq %xmm1, %xmm2, %xmm14
+; AVX1-NEXT: vpmaxsw %xmm1, %xmm14, %xmm1
+; AVX1-NEXT: vpsubq %xmm0, %xmm2, %xmm2
+; AVX1-NEXT: vpmaxsw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT: vpmovzxbq {{.*#+}} xmm2 = [255,255]
+; AVX1-NEXT: vpand %xmm2, %xmm11, %xmm11
+; AVX1-NEXT: vpand %xmm2, %xmm9, %xmm9
+; AVX1-NEXT: vpackusdw %xmm11, %xmm9, %xmm9
+; AVX1-NEXT: vpand %xmm2, %xmm4, %xmm4
+; AVX1-NEXT: vpand %xmm2, %xmm13, %xmm11
+; AVX1-NEXT: vpackusdw %xmm11, %xmm4, %xmm4
+; AVX1-NEXT: vpackusdw %xmm9, %xmm4, %xmm4
+; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm0
+; AVX1-NEXT: vpand %xmm2, %xmm1, %xmm1
+; AVX1-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpand %xmm2, %xmm3, %xmm1
+; AVX1-NEXT: vpand %xmm2, %xmm5, %xmm3
+; AVX1-NEXT: vpackusdw %xmm1, %xmm3, %xmm1
+; AVX1-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpackuswb %xmm4, %xmm0, %xmm0
+; AVX1-NEXT: vpand %xmm2, %xmm6, %xmm1
+; AVX1-NEXT: vpand {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm3 # 16-byte Folded Reload
+; AVX1-NEXT: vpackusdw %xmm1, %xmm3, %xmm1
+; AVX1-NEXT: vpand {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm3 # 16-byte Folded Reload
+; AVX1-NEXT: vpand {{[-0-9]+}}(%r{{[sb]}}p), %xmm2, %xmm4 # 16-byte Folded Reload
+; AVX1-NEXT: vpackusdw %xmm4, %xmm3, %xmm3
+; AVX1-NEXT: vpackusdw %xmm1, %xmm3, %xmm1
+; AVX1-NEXT: vpand %xmm2, %xmm7, %xmm3
+; AVX1-NEXT: vpand %xmm2, %xmm8, %xmm4
+; AVX1-NEXT: vpackusdw %xmm4, %xmm3, %xmm3
+; AVX1-NEXT: vpand %xmm2, %xmm10, %xmm4
+; AVX1-NEXT: vpand %xmm2, %xmm12, %xmm2
+; AVX1-NEXT: vpackusdw %xmm4, %xmm2, %xmm2
+; AVX1-NEXT: vpackusdw %xmm2, %xmm3, %xmm2
+; AVX1-NEXT: vpackuswb %xmm1, %xmm2, %xmm1
+; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0
; AVX1-NEXT: retq
;
; AVX2-LABEL: abd_ext_v32i8:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpminub %ymm1, %ymm0, %ymm2
-; AVX2-NEXT: vpmaxub %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: vpsubb %ymm2, %ymm0, %ymm0
+; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
+; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; AVX2-NEXT: vpmovzxbq {{.*#+}} ymm4 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero
+; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm2
+; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm2[2,3,2,3]
+; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero
+; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm6 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX2-NEXT: vpshufd {{.*#+}} xmm7 = xmm6[2,3,2,3]
+; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm7 = xmm7[0],zero,zero,zero,xmm7[1],zero,zero,zero,xmm7[2],zero,zero,zero,xmm7[3],zero,zero,zero
+; AVX2-NEXT: vpmovzxbq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero
+; AVX2-NEXT: vextracti128 $1, %ymm6, %xmm6
+; AVX2-NEXT: vpshufd {{.*#+}} xmm8 = xmm6[2,3,2,3]
+; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm8 = xmm8[0],zero,zero,zero,xmm8[1],zero,zero,zero,xmm8[2],zero,zero,zero,xmm8[3],zero,zero,zero
+; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero
+; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm9 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX2-NEXT: vpshufd {{.*#+}} xmm10 = xmm9[2,3,2,3]
+; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm10 = xmm10[0],zero,zero,zero,xmm10[1],zero,zero,zero,xmm10[2],zero,zero,zero,xmm10[3],zero,zero,zero
+; AVX2-NEXT: vpsubq %ymm10, %ymm3, %ymm3
+; AVX2-NEXT: vpmovzxbq {{.*#+}} ymm10 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero,xmm1[2],zero,zero,zero,zero,zero,zero,zero,xmm1[3],zero,zero,zero,zero,zero,zero,zero
+; AVX2-NEXT: vpsubq %ymm10, %ymm4, %ymm4
+; AVX2-NEXT: vextracti128 $1, %ymm9, %xmm9
+; AVX2-NEXT: vpshufd {{.*#+}} xmm10 = xmm9[2,3,2,3]
+; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm10 = xmm10[0],zero,zero,zero,xmm10[1],zero,zero,zero,xmm10[2],zero,zero,zero,xmm10[3],zero,zero,zero
+; AVX2-NEXT: vpsubq %ymm10, %ymm5, %ymm5
+; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm9 = xmm9[0],zero,zero,zero,xmm9[1],zero,zero,zero,xmm9[2],zero,zero,zero,xmm9[3],zero,zero,zero
+; AVX2-NEXT: vpsubq %ymm9, %ymm2, %ymm2
+; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm1
+; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm9 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX2-NEXT: vpshufd {{.*#+}} xmm10 = xmm9[2,3,2,3]
+; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm10 = xmm10[0],zero,zero,zero,xmm10[1],zero,zero,zero,xmm10[2],zero,zero,zero,xmm10[3],zero,zero,zero
+; AVX2-NEXT: vpsubq %ymm10, %ymm7, %ymm7
+; AVX2-NEXT: vpmovzxbq {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero,xmm1[2],zero,zero,zero,zero,zero,zero,zero,xmm1[3],zero,zero,zero,zero,zero,zero,zero
+; AVX2-NEXT: vpsubq %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm9, %xmm1
+; AVX2-NEXT: vpshufd {{.*#+}} xmm9 = xmm1[2,3,2,3]
+; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm9 = xmm9[0],zero,zero,zero,xmm9[1],zero,zero,zero,xmm9[2],zero,zero,zero,xmm9[3],zero,zero,zero
+; AVX2-NEXT: vpsubq %ymm9, %ymm8, %ymm8
+; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; AVX2-NEXT: vpsubq %ymm1, %ymm6, %ymm1
+; AVX2-NEXT: vpxor %xmm6, %xmm6, %xmm6
+; AVX2-NEXT: vpsubq %ymm3, %ymm6, %ymm9
+; AVX2-NEXT: vpmaxsw %ymm3, %ymm9, %ymm3
+; AVX2-NEXT: vpsubq %ymm4, %ymm6, %ymm9
+; AVX2-NEXT: vpmaxsw %ymm4, %ymm9, %ymm4
+; AVX2-NEXT: vpsubq %ymm5, %ymm6, %ymm9
+; AVX2-NEXT: vpmaxsw %ymm5, %ymm9, %ymm5
+; AVX2-NEXT: vpsubq %ymm2, %ymm6, %ymm9
+; AVX2-NEXT: vpmaxsw %ymm2, %ymm9, %ymm2
+; AVX2-NEXT: vpsubq %ymm7, %ymm6, %ymm9
+; AVX2-NEXT: vpmaxsw %ymm7, %ymm9, %ymm7
+; AVX2-NEXT: vpsubq %ymm0, %ymm6, %ymm9
+; AVX2-NEXT: vpmaxsw %ymm0, %ymm9, %ymm0
+; AVX2-NEXT: vpsubq %ymm8, %ymm6, %ymm9
+; AVX2-NEXT: vpmaxsw %ymm9, %ymm8, %ymm8
+; AVX2-NEXT: vpsubq %ymm1, %ymm6, %ymm6
+; AVX2-NEXT: vpmaxsw %ymm6, %ymm1, %ymm1
+; AVX2-NEXT: vpbroadcastq {{.*#+}} ymm6 = [255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0]
+; AVX2-NEXT: vpand %ymm6, %ymm1, %ymm1
+; AVX2-NEXT: vpand %ymm6, %ymm8, %ymm8
+; AVX2-NEXT: vpackusdw %ymm8, %ymm1, %ymm1
+; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
+; AVX2-NEXT: vpand %ymm6, %ymm0, %ymm0
+; AVX2-NEXT: vpand %ymm6, %ymm7, %ymm7
+; AVX2-NEXT: vpackusdw %ymm7, %ymm0, %ymm0
+; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT: vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT: vpand %ymm6, %ymm2, %ymm1
+; AVX2-NEXT: vpand %ymm6, %ymm5, %ymm2
+; AVX2-NEXT: vpackusdw %ymm2, %ymm1, %ymm1
+; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
+; AVX2-NEXT: vpand %ymm6, %ymm4, %ymm2
+; AVX2-NEXT: vpand %ymm6, %ymm3, %ymm3
+; AVX2-NEXT: vpackusdw %ymm3, %ymm2, %ymm2
+; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
+; AVX2-NEXT: vpackusdw %ymm1, %ymm2, %ymm1
+; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
+; AVX2-NEXT: vpackuswb %ymm0, %ymm1, %ymm0
+; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
; AVX2-NEXT: retq
;
; AVX512-LABEL: abd_ext_v32i8:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpminub %ymm1, %ymm0, %ymm2
-; AVX512-NEXT: vpmaxub %ymm1, %ymm0, %ymm0
-; AVX512-NEXT: vpsubb %ymm2, %ymm0, %ymm0
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm2
+; AVX512-NEXT: vpmovzxbw {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
+; AVX512-NEXT: vpmovzxwq {{.*#+}} zmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero
+; AVX512-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512-NEXT: vpmovzxwq {{.*#+}} zmm4 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm2
+; AVX512-NEXT: vpmovzxwq {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX512-NEXT: vpmovzxwq {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm5
+; AVX512-NEXT: vpmovzxbw {{.*#+}} ymm5 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero,xmm5[8],zero,xmm5[9],zero,xmm5[10],zero,xmm5[11],zero,xmm5[12],zero,xmm5[13],zero,xmm5[14],zero,xmm5[15],zero
+; AVX512-NEXT: vpmovzxwq {{.*#+}} zmm6 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero,xmm5[4],zero,zero,zero,xmm5[5],zero,zero,zero,xmm5[6],zero,zero,zero,xmm5[7],zero,zero,zero
+; AVX512-NEXT: vpsubq %zmm6, %zmm3, %zmm3
+; AVX512-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512-NEXT: vpmovzxwq {{.*#+}} zmm6 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero
+; AVX512-NEXT: vpsubq %zmm6, %zmm4, %zmm4
+; AVX512-NEXT: vextracti128 $1, %ymm5, %xmm5
+; AVX512-NEXT: vpmovzxwq {{.*#+}} zmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero,xmm5[4],zero,zero,zero,xmm5[5],zero,zero,zero,xmm5[6],zero,zero,zero,xmm5[7],zero,zero,zero
+; AVX512-NEXT: vpsubq %zmm5, %zmm2, %zmm2
+; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm1
+; AVX512-NEXT: vpmovzxwq {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero
+; AVX512-NEXT: vpsubq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpsubq %zmm3, %zmm1, %zmm5
+; AVX512-NEXT: vpmaxsq %zmm5, %zmm3, %zmm3
+; AVX512-NEXT: vpsubq %zmm4, %zmm1, %zmm5
+; AVX512-NEXT: vpmaxsq %zmm5, %zmm4, %zmm4
+; AVX512-NEXT: vpsubq %zmm2, %zmm1, %zmm5
+; AVX512-NEXT: vpmaxsq %zmm5, %zmm2, %zmm2
+; AVX512-NEXT: vpsubq %zmm0, %zmm1, %zmm1
+; AVX512-NEXT: vpmaxsq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpmovqb %zmm0, %xmm0
+; AVX512-NEXT: vpmovqb %zmm2, %xmm1
+; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512-NEXT: vpmovqb %zmm4, %xmm1
+; AVX512-NEXT: vpmovqb %zmm3, %xmm2
+; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
; AVX512-NEXT: retq
%aext = zext <32 x i8> %a to <32 x i64>
%bext = zext <32 x i8> %b to <32 x i64>
@@ -80,29 +343,137 @@ define <32 x i8> @abd_ext_v32i8_undef(<32 x i8> %a, <32 x i8> %b) nounwind {
define <16 x i16> @abd_ext_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind {
; AVX1-LABEL: abd_ext_v16i16:
; AVX1: # %bb.0:
-; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
-; AVX1-NEXT: vpminuw %xmm2, %xmm3, %xmm4
-; AVX1-NEXT: vpmaxuw %xmm2, %xmm3, %xmm2
-; AVX1-NEXT: vpsubw %xmm4, %xmm2, %xmm2
-; AVX1-NEXT: vpminuw %xmm1, %xmm0, %xmm3
-; AVX1-NEXT: vpmaxuw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpsubw %xmm3, %xmm0, %xmm0
-; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[1,1,1,1]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
+; AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
+; AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm0[3,3,3,3]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT: vpshufd {{.*#+}} xmm6 = xmm0[1,1,1,1]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm7 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
+; AVX1-NEXT: vpshufd {{.*#+}} xmm8 = xmm0[2,3,2,3]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm8 = xmm8[0],zero,zero,zero,xmm8[1],zero,zero,zero
+; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
+; AVX1-NEXT: vpshufd {{.*#+}} xmm9 = xmm1[1,1,1,1]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm9 = xmm9[0],zero,zero,zero,xmm9[1],zero,zero,zero
+; AVX1-NEXT: vpsubq %xmm9, %xmm2, %xmm2
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm9 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX1-NEXT: vpsubq %xmm9, %xmm3, %xmm3
+; AVX1-NEXT: vpshufd {{.*#+}} xmm9 = xmm1[2,3,2,3]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm9 = xmm9[0],zero,zero,zero,xmm9[1],zero,zero,zero
+; AVX1-NEXT: vpsubq %xmm9, %xmm4, %xmm4
+; AVX1-NEXT: vpshufd {{.*#+}} xmm9 = xmm1[3,3,3,3]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm9 = xmm9[0],zero,zero,zero,xmm9[1],zero,zero,zero
+; AVX1-NEXT: vpsubq %xmm9, %xmm5, %xmm5
+; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
+; AVX1-NEXT: vpshufd {{.*#+}} xmm9 = xmm1[1,1,1,1]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm9 = xmm9[0],zero,zero,zero,xmm9[1],zero,zero,zero
+; AVX1-NEXT: vpsubq %xmm9, %xmm6, %xmm6
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm9 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX1-NEXT: vpsubq %xmm9, %xmm7, %xmm7
+; AVX1-NEXT: vpshufd {{.*#+}} xmm9 = xmm1[2,3,2,3]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm9 = xmm9[0],zero,zero,zero,xmm9[1],zero,zero,zero
+; AVX1-NEXT: vpsubq %xmm9, %xmm8, %xmm8
+; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
+; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; AVX1-NEXT: vpsubq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT: vpsubq %xmm2, %xmm1, %xmm9
+; AVX1-NEXT: vpmaxsd %xmm9, %xmm2, %xmm2
+; AVX1-NEXT: vpsubq %xmm3, %xmm1, %xmm9
+; AVX1-NEXT: vpmaxsd %xmm9, %xmm3, %xmm3
+; AVX1-NEXT: vpsubq %xmm4, %xmm1, %xmm9
+; AVX1-NEXT: vpmaxsd %xmm9, %xmm4, %xmm4
+; AVX1-NEXT: vpsubq %xmm5, %xmm1, %xmm9
+; AVX1-NEXT: vpmaxsd %xmm9, %xmm5, %xmm5
+; AVX1-NEXT: vpsubq %xmm6, %xmm1, %xmm9
+; AVX1-NEXT: vpmaxsd %xmm9, %xmm6, %xmm6
+; AVX1-NEXT: vpsubq %xmm7, %xmm1, %xmm9
+; AVX1-NEXT: vpmaxsd %xmm9, %xmm7, %xmm7
+; AVX1-NEXT: vpsubq %xmm8, %xmm1, %xmm9
+; AVX1-NEXT: vpmaxsd %xmm9, %xmm8, %xmm8
+; AVX1-NEXT: vpsubq %xmm0, %xmm1, %xmm9
+; AVX1-NEXT: vpmaxsd %xmm9, %xmm0, %xmm0
+; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3],xmm0[4],xmm1[5,6,7]
+; AVX1-NEXT: vpblendw {{.*#+}} xmm8 = xmm8[0],xmm1[1,2,3],xmm8[4],xmm1[5,6,7]
+; AVX1-NEXT: vpackusdw %xmm0, %xmm8, %xmm0
+; AVX1-NEXT: vpblendw {{.*#+}} xmm7 = xmm7[0],xmm1[1,2,3],xmm7[4],xmm1[5,6,7]
+; AVX1-NEXT: vpblendw {{.*#+}} xmm6 = xmm6[0],xmm1[1,2,3],xmm6[4],xmm1[5,6,7]
+; AVX1-NEXT: vpackusdw %xmm6, %xmm7, %xmm6
+; AVX1-NEXT: vpackusdw %xmm0, %xmm6, %xmm0
+; AVX1-NEXT: vpblendw {{.*#+}} xmm5 = xmm5[0],xmm1[1,2,3],xmm5[4],xmm1[5,6,7]
+; AVX1-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0],xmm1[1,2,3],xmm4[4],xmm1[5,6,7]
+; AVX1-NEXT: vpackusdw %xmm5, %xmm4, %xmm4
+; AVX1-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0],xmm1[1,2,3],xmm3[4],xmm1[5,6,7]
+; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3],xmm2[4],xmm1[5,6,7]
+; AVX1-NEXT: vpackusdw %xmm1, %xmm3, %xmm1
+; AVX1-NEXT: vpackusdw %xmm4, %xmm1, %xmm1
+; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0
; AVX1-NEXT: retq
;
; AVX2-LABEL: abd_ext_v16i16:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpminuw %ymm1, %ymm0, %ymm2
-; AVX2-NEXT: vpmaxuw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: vpsubw %ymm2, %ymm0, %ymm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
+; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
+; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
+; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[2,3,2,3]
+; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero
+; AVX2-NEXT: vpsubq %ymm5, %ymm2, %ymm2
+; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm5 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; AVX2-NEXT: vpsubq %ymm5, %ymm3, %ymm3
+; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm1
+; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[2,3,2,3]
+; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero
+; AVX2-NEXT: vpsubq %ymm5, %ymm4, %ymm4
+; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; AVX2-NEXT: vpsubq %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpsubq %ymm2, %ymm1, %ymm5
+; AVX2-NEXT: vpmaxsd %ymm5, %ymm2, %ymm2
+; AVX2-NEXT: vpsubq %ymm3, %ymm1, %ymm5
+; AVX2-NEXT: vpmaxsd %ymm5, %ymm3, %ymm3
+; AVX2-NEXT: vpsubq %ymm4, %ymm1, %ymm5
+; AVX2-NEXT: vpmaxsd %ymm5, %ymm4, %ymm4
+; AVX2-NEXT: vpsubq %ymm0, %ymm1, %ymm5
+; AVX2-NEXT: vpmaxsd %ymm5, %ymm0, %ymm0
+; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7],ymm0[8],ymm1[9,10,11],ymm0[12],ymm1[13,14,15]
+; AVX2-NEXT: vpblendw {{.*#+}} ymm4 = ymm4[0],ymm1[1,2,3],ymm4[4],ymm1[5,6,7],ymm4[8],ymm1[9,10,11],ymm4[12],ymm1[13,14,15]
+; AVX2-NEXT: vpackusdw %ymm4, %ymm0, %ymm0
+; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; AVX2-NEXT: vpblendw {{.*#+}} ymm3 = ymm3[0],ymm1[1,2,3],ymm3[4],ymm1[5,6,7],ymm3[8],ymm1[9,10,11],ymm3[12],ymm1[13,14,15]
+; AVX2-NEXT: vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1,2,3],ymm2[4],ymm1[5,6,7],ymm2[8],ymm1[9,10,11],ymm2[12],ymm1[13,14,15]
+; AVX2-NEXT: vpackusdw %ymm1, %ymm3, %ymm1
+; AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
+; AVX2-NEXT: vpackusdw %ymm0, %ymm1, %ymm0
+; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
; AVX2-NEXT: retq
;
; AVX512-LABEL: abd_ext_v16i16:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpminuw %ymm1, %ymm0, %ymm2
-; AVX512-NEXT: vpmaxuw %ymm1, %ymm0, %ymm0
-; AVX512-NEXT: vpsubw %ymm2, %ymm0, %ymm0
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm2
+; AVX512-NEXT: vpmovzxwq {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero
+; AVX512-NEXT: vpmovzxwq {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm3
+; AVX512-NEXT: vpmovzxwq {{.*#+}} zmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero,xmm3[4],zero,zero,zero,xmm3[5],zero,zero,zero,xmm3[6],zero,zero,zero,xmm3[7],zero,zero,zero
+; AVX512-NEXT: vpsubq %zmm3, %zmm2, %zmm2
+; AVX512-NEXT: vpmovzxwq {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero
+; AVX512-NEXT: vpsubq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpsubq %zmm2, %zmm1, %zmm3
+; AVX512-NEXT: vpmaxsq %zmm3, %zmm2, %zmm2
+; AVX512-NEXT: vpsubq %zmm0, %zmm1, %zmm1
+; AVX512-NEXT: vpmaxsq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpmovqw %zmm0, %xmm0
+; AVX512-NEXT: vpmovqw %zmm2, %xmm1
+; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
; AVX512-NEXT: retq
%aext = zext <16 x i16> %a to <16 x i64>
%bext = zext <16 x i16> %b to <16 x i64>
@@ -150,29 +521,74 @@ define <16 x i16> @abd_ext_v16i16_undef(<16 x i16> %a, <16 x i16> %b) nounwind {
define <8 x i32> @abd_ext_v8i32(<8 x i32> %a, <8 x i32> %b) nounwind {
; AVX1-LABEL: abd_ext_v8i32:
; AVX1: # %bb.0:
-; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
-; AVX1-NEXT: vpminud %xmm2, %xmm3, %xmm4
-; AVX1-NEXT: vpmaxud %xmm2, %xmm3, %xmm2
-; AVX1-NEXT: vpsubd %xmm4, %xmm2, %xmm2
-; AVX1-NEXT: vpminud %xmm1, %xmm0, %xmm3
-; AVX1-NEXT: vpmaxud %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpsubd %xmm3, %xmm0, %xmm0
-; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm3 = xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm4[2],xmm2[2],xmm4[3],xmm2[3]
+; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
+; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm6 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; AVX1-NEXT: vpsubq %xmm6, %xmm3, %xmm3
+; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm6
+; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm7 = xmm6[2],xmm2[2],xmm6[3],xmm2[3]
+; AVX1-NEXT: vpsubq %xmm7, %xmm5, %xmm5
+; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
+; AVX1-NEXT: vpsubq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm1 = xmm6[0],zero,xmm6[1],zero
+; AVX1-NEXT: vpsubq %xmm1, %xmm4, %xmm1
+; AVX1-NEXT: vpsubq %xmm3, %xmm2, %xmm4
+; AVX1-NEXT: vpcmpgtq %xmm4, %xmm3, %xmm4
+; AVX1-NEXT: vpxor %xmm4, %xmm3, %xmm3
+; AVX1-NEXT: vpsubq %xmm3, %xmm4, %xmm3
+; AVX1-NEXT: vpsubq %xmm5, %xmm2, %xmm4
+; AVX1-NEXT: vpcmpgtq %xmm4, %xmm5, %xmm4
+; AVX1-NEXT: vpxor %xmm4, %xmm5, %xmm5
+; AVX1-NEXT: vpsubq %xmm5, %xmm4, %xmm4
+; AVX1-NEXT: vpsubq %xmm0, %xmm2, %xmm5
+; AVX1-NEXT: vpcmpgtq %xmm5, %xmm0, %xmm5
+; AVX1-NEXT: vpxor %xmm5, %xmm0, %xmm0
+; AVX1-NEXT: vpsubq %xmm0, %xmm5, %xmm0
+; AVX1-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm3[0,2]
+; AVX1-NEXT: vpsubq %xmm1, %xmm2, %xmm2
+; AVX1-NEXT: vpcmpgtq %xmm2, %xmm1, %xmm2
+; AVX1-NEXT: vpxor %xmm2, %xmm1, %xmm1
+; AVX1-NEXT: vpsubq %xmm1, %xmm2, %xmm1
+; AVX1-NEXT: vshufps {{.*#+}} xmm1 = xmm1[0,2],xmm4[0,2]
+; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
; AVX1-NEXT: retq
;
; AVX2-LABEL: abd_ext_v8i32:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpminud %ymm1, %ymm0, %ymm2
-; AVX2-NEXT: vpmaxud %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: vpsubd %ymm2, %ymm0, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero
+; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm3
+; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm3 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero
+; AVX2-NEXT: vpsubq %ymm3, %ymm2, %ymm2
+; AVX2-NEXT: vpmovzxdq {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero
+; AVX2-NEXT: vpsubq %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpsubq %ymm2, %ymm1, %ymm3
+; AVX2-NEXT: vpcmpgtq %ymm3, %ymm2, %ymm3
+; AVX2-NEXT: vpxor %ymm3, %ymm2, %ymm2
+; AVX2-NEXT: vpsubq %ymm2, %ymm3, %ymm2
+; AVX2-NEXT: vpsubq %ymm0, %ymm1, %ymm1
+; AVX2-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm1
+; AVX2-NEXT: vpxor %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpsubq %ymm0, %ymm1, %ymm0
+; AVX2-NEXT: vshufps {{.*#+}} ymm0 = ymm0[0,2],ymm2[0,2],ymm0[4,6],ymm2[4,6]
+; AVX2-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,2,1,3]
; AVX2-NEXT: retq
;
; AVX512-LABEL: abd_ext_v8i32:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpminud %ymm1, %ymm0, %ymm2
-; AVX512-NEXT: vpmaxud %ymm1, %ymm0, %ymm0
-; AVX512-NEXT: vpsubd %ymm2, %ymm0, %ymm0
+; AVX512-NEXT: vpmovzxdq {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero
+; AVX512-NEXT: vpmovzxdq {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero
+; AVX512-NEXT: vpsubq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpsubq %zmm0, %zmm1, %zmm1
+; AVX512-NEXT: vpmaxsq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpmovqd %zmm0, %ymm0
; AVX512-NEXT: retq
%aext = zext <8 x i32> %a to <8 x i64>
%bext = zext <8 x i32> %b to <8 x i64>
@@ -220,41 +636,197 @@ define <8 x i32> @abd_ext_v8i32_undef(<8 x i32> %a, <8 x i32> %b) nounwind {
define <4 x i64> @abd_ext_v4i64(<4 x i64> %a, <4 x i64> %b) nounwind {
; AVX1-LABEL: abd_ext_v4i64:
; AVX1: # %bb.0:
-; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT: vmovddup {{.*#+}} xmm3 = [9223372036854775808,9223372036854775808]
-; AVX1-NEXT: # xmm3 = mem[0,0]
-; AVX1-NEXT: vpxor %xmm3, %xmm2, %xmm4
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5
-; AVX1-NEXT: vpxor %xmm3, %xmm5, %xmm6
-; AVX1-NEXT: vpcmpgtq %xmm4, %xmm6, %xmm4
-; AVX1-NEXT: vpsubq %xmm2, %xmm5, %xmm2
-; AVX1-NEXT: vpxor %xmm4, %xmm2, %xmm2
-; AVX1-NEXT: vpsubq %xmm2, %xmm4, %xmm2
-; AVX1-NEXT: vpxor %xmm3, %xmm1, %xmm4
-; AVX1-NEXT: vpxor %xmm3, %xmm0, %xmm3
-; AVX1-NEXT: vpcmpgtq %xmm4, %xmm3, %xmm3
-; AVX1-NEXT: vpsubq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpxor %xmm3, %xmm0, %xmm0
-; AVX1-NEXT: vpsubq %xmm0, %xmm3, %xmm0
-; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-NEXT: pushq %r14
+; AVX1-NEXT: pushq %rbx
+; AVX1-NEXT: vmovq %xmm0, %r9
+; AVX1-NEXT: vpextrq $1, %xmm0, %rdi
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
+; AVX1-NEXT: vmovq %xmm0, %rcx
+; AVX1-NEXT: vpextrq $1, %xmm0, %rax
+; AVX1-NEXT: vmovq %xmm1, %r8
+; AVX1-NEXT: vpextrq $1, %xmm1, %r11
+; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm0
+; AVX1-NEXT: vmovq %xmm0, %r10
+; AVX1-NEXT: vpextrq $1, %xmm0, %rsi
+; AVX1-NEXT: xorl %edx, %edx
+; AVX1-NEXT: subq %rsi, %rax
+; AVX1-NEXT: movl $0, %esi
+; AVX1-NEXT: sbbq %rsi, %rsi
+; AVX1-NEXT: subq %r10, %rcx
+; AVX1-NEXT: movl $0, %r10d
+; AVX1-NEXT: sbbq %r10, %r10
+; AVX1-NEXT: subq %r11, %rdi
+; AVX1-NEXT: movl $0, %r11d
+; AVX1-NEXT: sbbq %r11, %r11
+; AVX1-NEXT: subq %r8, %r9
+; AVX1-NEXT: movl $0, %ebx
+; AVX1-NEXT: sbbq %rbx, %rbx
+; AVX1-NEXT: movq %r9, %r8
+; AVX1-NEXT: negq %r8
+; AVX1-NEXT: movl $0, %r14d
+; AVX1-NEXT: sbbq %rbx, %r14
+; AVX1-NEXT: cmpq %r9, %r8
+; AVX1-NEXT: sbbq %rbx, %r14
+; AVX1-NEXT: cmovlq %r9, %r8
+; AVX1-NEXT: movq %rdi, %r9
+; AVX1-NEXT: negq %r9
+; AVX1-NEXT: movl $0, %ebx
+; AVX1-NEXT: sbbq %r11, %rbx
+; AVX1-NEXT: cmpq %rdi, %r9
+; AVX1-NEXT: sbbq %r11, %rbx
+; AVX1-NEXT: cmovlq %rdi, %r9
+; AVX1-NEXT: movq %rcx, %rdi
+; AVX1-NEXT: negq %rdi
+; AVX1-NEXT: movl $0, %r11d
+; AVX1-NEXT: sbbq %r10, %r11
+; AVX1-NEXT: cmpq %rcx, %rdi
+; AVX1-NEXT: sbbq %r10, %r11
+; AVX1-NEXT: cmovlq %rcx, %rdi
+; AVX1-NEXT: movq %rax, %rcx
+; AVX1-NEXT: negq %rcx
+; AVX1-NEXT: sbbq %rsi, %rdx
+; AVX1-NEXT: cmpq %rax, %rcx
+; AVX1-NEXT: sbbq %rsi, %rdx
+; AVX1-NEXT: cmovlq %rax, %rcx
+; AVX1-NEXT: vmovq %rcx, %xmm0
+; AVX1-NEXT: vmovq %rdi, %xmm1
+; AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX1-NEXT: vmovq %r9, %xmm1
+; AVX1-NEXT: vmovq %r8, %xmm2
+; AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT: popq %rbx
+; AVX1-NEXT: popq %r14
; AVX1-NEXT: retq
;
; AVX2-LABEL: abd_ext_v4i64:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpbroadcastq {{.*#+}} ymm2 = [9223372036854775808,9223372036854775808,9223372036854775808,9223372036854775808]
-; AVX2-NEXT: vpxor %ymm2, %ymm1, %ymm3
-; AVX2-NEXT: vpxor %ymm2, %ymm0, %ymm2
-; AVX2-NEXT: vpcmpgtq %ymm3, %ymm2, %ymm2
-; AVX2-NEXT: vpsubq %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: vpxor %ymm2, %ymm0, %ymm0
-; AVX2-NEXT: vpsubq %ymm0, %ymm2, %ymm0
+; AVX2-NEXT: pushq %r14
+; AVX2-NEXT: pushq %rbx
+; AVX2-NEXT: vmovq %xmm0, %r9
+; AVX2-NEXT: vpextrq $1, %xmm0, %rdi
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX2-NEXT: vmovq %xmm0, %rcx
+; AVX2-NEXT: vpextrq $1, %xmm0, %rax
+; AVX2-NEXT: vmovq %xmm1, %r8
+; AVX2-NEXT: vpextrq $1, %xmm1, %r11
+; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm0
+; AVX2-NEXT: vmovq %xmm0, %r10
+; AVX2-NEXT: vpextrq $1, %xmm0, %rsi
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: subq %rsi, %rax
+; AVX2-NEXT: movl $0, %esi
+; AVX2-NEXT: sbbq %rsi, %rsi
+; AVX2-NEXT: subq %r10, %rcx
+; AVX2-NEXT: movl $0, %r10d
+; AVX2-NEXT: sbbq %r10, %r10
+; AVX2-NEXT: subq %r11, %rdi
+; AVX2-NEXT: movl $0, %r11d
+; AVX2-NEXT: sbbq %r11, %r11
+; AVX2-NEXT: subq %r8, %r9
+; AVX2-NEXT: movl $0, %ebx
+; AVX2-NEXT: sbbq %rbx, %rbx
+; AVX2-NEXT: movq %r9, %r8
+; AVX2-NEXT: negq %r8
+; AVX2-NEXT: movl $0, %r14d
+; AVX2-NEXT: sbbq %rbx, %r14
+; AVX2-NEXT: cmpq %r9, %r8
+; AVX2-NEXT: sbbq %rbx, %r14
+; AVX2-NEXT: cmovlq %r9, %r8
+; AVX2-NEXT: movq %rdi, %r9
+; AVX2-NEXT: negq %r9
+; AVX2-NEXT: movl $0, %ebx
+; AVX2-NEXT: sbbq %r11, %rbx
+; AVX2-NEXT: cmpq %rdi, %r9
+; AVX2-NEXT: sbbq %r11, %rbx
+; AVX2-NEXT: cmovlq %rdi, %r9
+; AVX2-NEXT: movq %rcx, %rdi
+; AVX2-NEXT: negq %rdi
+; AVX2-NEXT: movl $0, %r11d
+; AVX2-NEXT: sbbq %r10, %r11
+; AVX2-NEXT: cmpq %rcx, %rdi
+; AVX2-NEXT: sbbq %r10, %r11
+; AVX2-NEXT: cmovlq %rcx, %rdi
+; AVX2-NEXT: movq %rax, %rcx
+; AVX2-NEXT: negq %rcx
+; AVX2-NEXT: sbbq %rsi, %rdx
+; AVX2-NEXT: cmpq %rax, %rcx
+; AVX2-NEXT: sbbq %rsi, %rdx
+; AVX2-NEXT: cmovlq %rax, %rcx
+; AVX2-NEXT: vmovq %rcx, %xmm0
+; AVX2-NEXT: vmovq %rdi, %xmm1
+; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX2-NEXT: vmovq %r9, %xmm1
+; AVX2-NEXT: vmovq %r8, %xmm2
+; AVX2-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX2-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX2-NEXT: popq %rbx
+; AVX2-NEXT: popq %r14
; AVX2-NEXT: retq
;
; AVX512-LABEL: abd_ext_v4i64:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpminuq %ymm1, %ymm0, %ymm2
-; AVX512-NEXT: vpmaxuq %ymm1, %ymm0, %ymm0
-; AVX512-NEXT: vpsubq %ymm2, %ymm0, %ymm0
+; AVX512-NEXT: pushq %r14
+; AVX512-NEXT: pushq %rbx
+; AVX512-NEXT: vmovq %xmm0, %r9
+; AVX512-NEXT: vpextrq $1, %xmm0, %rdi
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX512-NEXT: vmovq %xmm0, %rcx
+; AVX512-NEXT: vpextrq $1, %xmm0, %rax
+; AVX512-NEXT: vmovq %xmm1, %r8
+; AVX512-NEXT: vpextrq $1, %xmm1, %r11
+; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm0
+; AVX512-NEXT: vmovq %xmm0, %r10
+; AVX512-NEXT: vpextrq $1, %xmm0, %rsi
+; AVX512-NEXT: xorl %edx, %edx
+; AVX512-NEXT: subq %rsi, %rax
+; AVX512-NEXT: movl $0, %esi
+; AVX512-NEXT: sbbq %rsi, %rsi
+; AVX512-NEXT: subq %r10, %rcx
+; AVX512-NEXT: movl $0, %r10d
+; AVX512-NEXT: sbbq %r10, %r10
+; AVX512-NEXT: subq %r11, %rdi
+; AVX512-NEXT: movl $0, %r11d
+; AVX512-NEXT: sbbq %r11, %r11
+; AVX512-NEXT: subq %r8, %r9
+; AVX512-NEXT: movl $0, %ebx
+; AVX512-NEXT: sbbq %rbx, %rbx
+; AVX512-NEXT: movq %r9, %r8
+; AVX512-NEXT: negq %r8
+; AVX512-NEXT: movl $0, %r14d
+; AVX512-NEXT: sbbq %rbx, %r14
+; AVX512-NEXT: cmpq %r9, %r8
+; AVX512-NEXT: sbbq %rbx, %r14
+; AVX512-NEXT: cmovlq %r9, %r8
+; AVX512-NEXT: movq %rdi, %r9
+; AVX512-NEXT: negq %r9
+; AVX512-NEXT: movl $0, %ebx
+; AVX512-NEXT: sbbq %r11, %rbx
+; AVX512-NEXT: cmpq %rdi, %r9
+; AVX512-NEXT: sbbq %r11, %rbx
+; AVX512-NEXT: cmovlq %rdi, %r9
+; AVX512-NEXT: movq %rcx, %rdi
+; AVX512-NEXT: negq %rdi
+; AVX512-NEXT: movl $0, %r11d
+; AVX512-NEXT: sbbq %r10, %r11
+; AVX512-NEXT: cmpq %rcx, %rdi
+; AVX512-NEXT: sbbq %r10, %r11
+; AVX512-NEXT: cmovlq %rcx, %rdi
+; AVX512-NEXT: movq %rax, %rcx
+; AVX512-NEXT: negq %rcx
+; AVX512-NEXT: sbbq %rsi, %rdx
+; AVX512-NEXT: cmpq %rax, %rcx
+; AVX512-NEXT: sbbq %rsi, %rdx
+; AVX512-NEXT: cmovlq %rax, %rcx
+; AVX512-NEXT: vmovq %rcx, %xmm0
+; AVX512-NEXT: vmovq %rdi, %xmm1
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512-NEXT: vmovq %r9, %xmm1
+; AVX512-NEXT: vmovq %r8, %xmm2
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
+; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512-NEXT: popq %rbx
+; AVX512-NEXT: popq %r14
; AVX512-NEXT: retq
%aext = zext <4 x i64> %a to <4 x i128>
%bext = zext <4 x i64> %b to <4 x i128>
diff --git a/llvm/test/CodeGen/X86/abdu-vector-512.ll b/llvm/test/CodeGen/X86/abdu-vector-512.ll
index a855bea61e530..38209153b4096 100644
--- a/llvm/test/CodeGen/X86/abdu-vector-512.ll
+++ b/llvm/test/CodeGen/X86/abdu-vector-512.ll
@@ -7,25 +7,87 @@
;
define <64 x i8> @abd_ext_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
-; AVX512BW-LABEL: abd_ext_v64i8:
-; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpminub %zmm1, %zmm0, %zmm2
-; AVX512BW-NEXT: vpmaxub %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT: vpsubb %zmm2, %zmm0, %zmm0
-; AVX512BW-NEXT: retq
-;
-; AVX512DQ-LABEL: abd_ext_v64i8:
-; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vextracti64x4 $1, %zmm1, %ymm2
-; AVX512DQ-NEXT: vextracti64x4 $1, %zmm0, %ymm3
-; AVX512DQ-NEXT: vpminub %ymm2, %ymm3, %ymm4
-; AVX512DQ-NEXT: vpmaxub %ymm2, %ymm3, %ymm2
-; AVX512DQ-NEXT: vpsubb %ymm4, %ymm2, %ymm2
-; AVX512DQ-NEXT: vpminub %ymm1, %ymm0, %ymm3
-; AVX512DQ-NEXT: vpmaxub %ymm1, %ymm0, %ymm0
-; AVX512DQ-NEXT: vpsubb %ymm3, %ymm0, %ymm0
-; AVX512DQ-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX512DQ-NEXT: retq
+; AVX512-LABEL: abd_ext_v64i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm2
+; AVX512-NEXT: vpmovzxbw {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
+; AVX512-NEXT: vpmovzxwq {{.*#+}} zmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero
+; AVX512-NEXT: vpmovzxbw {{.*#+}} ymm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512-NEXT: vpmovzxwq {{.*#+}} zmm5 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero,xmm4[4],zero,zero,zero,xmm4[5],zero,zero,zero,xmm4[6],zero,zero,zero,xmm4[7],zero,zero,zero
+; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm0
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm6
+; AVX512-NEXT: vpmovzxbw {{.*#+}} ymm6 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero,xmm6[8],zero,xmm6[9],zero,xmm6[10],zero,xmm6[11],zero,xmm6[12],zero,xmm6[13],zero,xmm6[14],zero,xmm6[15],zero
+; AVX512-NEXT: vpmovzxwq {{.*#+}} zmm7 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero,xmm6[4],zero,zero,zero,xmm6[5],zero,zero,zero,xmm6[6],zero,zero,zero,xmm6[7],zero,zero,zero
+; AVX512-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512-NEXT: vpmovzxwq {{.*#+}} zmm8 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm2
+; AVX512-NEXT: vpmovzxwq {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero
+; AVX512-NEXT: vextracti128 $1, %ymm4, %xmm4
+; AVX512-NEXT: vpmovzxwq {{.*#+}} zmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero,xmm4[4],zero,zero,zero,xmm4[5],zero,zero,zero,xmm4[6],zero,zero,zero,xmm4[7],zero,zero,zero
+; AVX512-NEXT: vextracti128 $1, %ymm6, %xmm6
+; AVX512-NEXT: vpmovzxwq {{.*#+}} zmm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero,xmm6[2],zero,zero,zero,xmm6[3],zero,zero,zero,xmm6[4],zero,zero,zero,xmm6[5],zero,zero,zero,xmm6[6],zero,zero,zero,xmm6[7],zero,zero,zero
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX512-NEXT: vpmovzxwq {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm9
+; AVX512-NEXT: vpmovzxbw {{.*#+}} ymm9 = xmm9[0],zero,xmm9[1],zero,xmm9[2],zero,xmm9[3],zero,xmm9[4],zero,xmm9[5],zero,xmm9[6],zero,xmm9[7],zero,xmm9[8],zero,xmm9[9],zero,xmm9[10],zero,xmm9[11],zero,xmm9[12],zero,xmm9[13],zero,xmm9[14],zero,xmm9[15],zero
+; AVX512-NEXT: vpmovzxwq {{.*#+}} zmm10 = xmm9[0],zero,zero,zero,xmm9[1],zero,zero,zero,xmm9[2],zero,zero,zero,xmm9[3],zero,zero,zero,xmm9[4],zero,zero,zero,xmm9[5],zero,zero,zero,xmm9[6],zero,zero,zero,xmm9[7],zero,zero,zero
+; AVX512-NEXT: vpsubq %zmm10, %zmm3, %zmm3
+; AVX512-NEXT: vpmovzxbw {{.*#+}} ymm10 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512-NEXT: vpmovzxwq {{.*#+}} zmm11 = xmm10[0],zero,zero,zero,xmm10[1],zero,zero,zero,xmm10[2],zero,zero,zero,xmm10[3],zero,zero,zero,xmm10[4],zero,zero,zero,xmm10[5],zero,zero,zero,xmm10[6],zero,zero,zero,xmm10[7],zero,zero,zero
+; AVX512-NEXT: vpsubq %zmm11, %zmm5, %zmm5
+; AVX512-NEXT: vextracti64x4 $1, %zmm1, %ymm1
+; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm11
+; AVX512-NEXT: vpmovzxbw {{.*#+}} ymm11 = xmm11[0],zero,xmm11[1],zero,xmm11[2],zero,xmm11[3],zero,xmm11[4],zero,xmm11[5],zero,xmm11[6],zero,xmm11[7],zero,xmm11[8],zero,xmm11[9],zero,xmm11[10],zero,xmm11[11],zero,xmm11[12],zero,xmm11[13],zero,xmm11[14],zero,xmm11[15],zero
+; AVX512-NEXT: vpmovzxwq {{.*#+}} zmm12 = xmm11[0],zero,zero,zero,xmm11[1],zero,zero,zero,xmm11[2],zero,zero,zero,xmm11[3],zero,zero,zero,xmm11[4],zero,zero,zero,xmm11[5],zero,zero,zero,xmm11[6],zero,zero,zero,xmm11[7],zero,zero,zero
+; AVX512-NEXT: vpsubq %zmm12, %zmm7, %zmm7
+; AVX512-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX512-NEXT: vpmovzxwq {{.*#+}} zmm12 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero
+; AVX512-NEXT: vpsubq %zmm12, %zmm8, %zmm8
+; AVX512-NEXT: vextracti128 $1, %ymm9, %xmm9
+; AVX512-NEXT: vpmovzxwq {{.*#+}} zmm9 = xmm9[0],zero,zero,zero,xmm9[1],zero,zero,zero,xmm9[2],zero,zero,zero,xmm9[3],zero,zero,zero,xmm9[4],zero,zero,zero,xmm9[5],zero,zero,zero,xmm9[6],zero,zero,zero,xmm9[7],zero,zero,zero
+; AVX512-NEXT: vpsubq %zmm9, %zmm2, %zmm2
+; AVX512-NEXT: vextracti128 $1, %ymm10, %xmm9
+; AVX512-NEXT: vpmovzxwq {{.*#+}} zmm9 = xmm9[0],zero,zero,zero,xmm9[1],zero,zero,zero,xmm9[2],zero,zero,zero,xmm9[3],zero,zero,zero,xmm9[4],zero,zero,zero,xmm9[5],zero,zero,zero,xmm9[6],zero,zero,zero,xmm9[7],zero,zero,zero
+; AVX512-NEXT: vpsubq %zmm9, %zmm4, %zmm4
+; AVX512-NEXT: vextracti128 $1, %ymm11, %xmm9
+; AVX512-NEXT: vpmovzxwq {{.*#+}} zmm9 = xmm9[0],zero,zero,zero,xmm9[1],zero,zero,zero,xmm9[2],zero,zero,zero,xmm9[3],zero,zero,zero,xmm9[4],zero,zero,zero,xmm9[5],zero,zero,zero,xmm9[6],zero,zero,zero,xmm9[7],zero,zero,zero
+; AVX512-NEXT: vpsubq %zmm9, %zmm6, %zmm6
+; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm1
+; AVX512-NEXT: vpmovzxwq {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero
+; AVX512-NEXT: vpsubq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpsubq %zmm3, %zmm1, %zmm9
+; AVX512-NEXT: vpmaxsq %zmm9, %zmm3, %zmm3
+; AVX512-NEXT: vpsubq %zmm5, %zmm1, %zmm9
+; AVX512-NEXT: vpmaxsq %zmm9, %zmm5, %zmm5
+; AVX512-NEXT: vpsubq %zmm7, %zmm1, %zmm9
+; AVX512-NEXT: vpmaxsq %zmm9, %zmm7, %zmm7
+; AVX512-NEXT: vpsubq %zmm8, %zmm1, %zmm9
+; AVX512-NEXT: vpmaxsq %zmm9, %zmm8, %zmm8
+; AVX512-NEXT: vpsubq %zmm2, %zmm1, %zmm9
+; AVX512-NEXT: vpmaxsq %zmm9, %zmm2, %zmm2
+; AVX512-NEXT: vpsubq %zmm4, %zmm1, %zmm9
+; AVX512-NEXT: vpmaxsq %zmm9, %zmm4, %zmm4
+; AVX512-NEXT: vpsubq %zmm6, %zmm1, %zmm9
+; AVX512-NEXT: vpmaxsq %zmm9, %zmm6, %zmm6
+; AVX512-NEXT: vpsubq %zmm0, %zmm1, %zmm1
+; AVX512-NEXT: vpmaxsq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpmovqb %zmm0, %xmm0
+; AVX512-NEXT: vpmovqb %zmm6, %xmm1
+; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512-NEXT: vpmovqb %zmm4, %xmm1
+; AVX512-NEXT: vpmovqb %zmm2, %xmm2
+; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512-NEXT: vpmovqb %zmm8, %xmm1
+; AVX512-NEXT: vpmovqb %zmm7, %xmm2
+; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512-NEXT: vpmovqb %zmm5, %xmm2
+; AVX512-NEXT: vpmovqb %zmm3, %xmm3
+; AVX512-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
+; AVX512-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} zmm0 = zmm1[0],zmm0[0],zmm1[2],zmm0[2],zmm1[4],zmm0[4],zmm1[6],zmm0[6]
+; AVX512-NEXT: retq
%aext = zext <64 x i8> %a to <64 x i64>
%bext = zext <64 x i8> %b to <64 x i64>
%sub = sub <64 x i64> %aext, %bext
@@ -63,25 +125,43 @@ define <64 x i8> @abd_ext_v64i8_undef(<64 x i8> %a, <64 x i8> %b) nounwind {
}
define <32 x i16> @abd_ext_v32i16(<32 x i16> %a, <32 x i16> %b) nounwind {
-; AVX512BW-LABEL: abd_ext_v32i16:
-; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpminuw %zmm1, %zmm0, %zmm2
-; AVX512BW-NEXT: vpmaxuw %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT: vpsubw %zmm2, %zmm0, %zmm0
-; AVX512BW-NEXT: retq
-;
-; AVX512DQ-LABEL: abd_ext_v32i16:
-; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vextracti64x4 $1, %zmm1, %ymm2
-; AVX512DQ-NEXT: vextracti64x4 $1, %zmm0, %ymm3
-; AVX512DQ-NEXT: vpminuw %ymm2, %ymm3, %ymm4
-; AVX512DQ-NEXT: vpmaxuw %ymm2, %ymm3, %ymm2
-; AVX512DQ-NEXT: vpsubw %ymm4, %ymm2, %ymm2
-; AVX512DQ-NEXT: vpminuw %ymm1, %ymm0, %ymm3
-; AVX512DQ-NEXT: vpmaxuw %ymm1, %ymm0, %ymm0
-; AVX512DQ-NEXT: vpsubw %ymm3, %ymm0, %ymm0
-; AVX512DQ-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX512DQ-NEXT: retq
+; AVX512-LABEL: abd_ext_v32i16:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm2
+; AVX512-NEXT: vpmovzxwq {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero
+; AVX512-NEXT: vpmovzxwq {{.*#+}} zmm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm0
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm4
+; AVX512-NEXT: vpmovzxwq {{.*#+}} zmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero,xmm4[4],zero,zero,zero,xmm4[5],zero,zero,zero,xmm4[6],zero,zero,zero,xmm4[7],zero,zero,zero
+; AVX512-NEXT: vpmovzxwq {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm5
+; AVX512-NEXT: vpmovzxwq {{.*#+}} zmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero,xmm5[4],zero,zero,zero,xmm5[5],zero,zero,zero,xmm5[6],zero,zero,zero,xmm5[7],zero,zero,zero
+; AVX512-NEXT: vpsubq %zmm5, %zmm2, %zmm2
+; AVX512-NEXT: vpmovzxwq {{.*#+}} zmm5 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero
+; AVX512-NEXT: vpsubq %zmm5, %zmm3, %zmm3
+; AVX512-NEXT: vextracti64x4 $1, %zmm1, %ymm1
+; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm5
+; AVX512-NEXT: vpmovzxwq {{.*#+}} zmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero,xmm5[4],zero,zero,zero,xmm5[5],zero,zero,zero,xmm5[6],zero,zero,zero,xmm5[7],zero,zero,zero
+; AVX512-NEXT: vpsubq %zmm5, %zmm4, %zmm4
+; AVX512-NEXT: vpmovzxwq {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero
+; AVX512-NEXT: vpsubq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpsubq %zmm2, %zmm1, %zmm5
+; AVX512-NEXT: vpmaxsq %zmm5, %zmm2, %zmm2
+; AVX512-NEXT: vpsubq %zmm3, %zmm1, %zmm5
+; AVX512-NEXT: vpmaxsq %zmm5, %zmm3, %zmm3
+; AVX512-NEXT: vpsubq %zmm4, %zmm1, %zmm5
+; AVX512-NEXT: vpmaxsq %zmm5, %zmm4, %zmm4
+; AVX512-NEXT: vpsubq %zmm0, %zmm1, %zmm1
+; AVX512-NEXT: vpmaxsq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpmovqw %zmm0, %xmm0
+; AVX512-NEXT: vpmovqw %zmm4, %xmm1
+; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512-NEXT: vpmovqw %zmm3, %xmm1
+; AVX512-NEXT: vpmovqw %zmm2, %xmm2
+; AVX512-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
+; AVX512-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512-NEXT: retq
%aext = zext <32 x i16> %a to <32 x i64>
%bext = zext <32 x i16> %b to <32 x i64>
%sub = sub <32 x i64> %aext, %bext
@@ -121,9 +201,22 @@ define <32 x i16> @abd_ext_v32i16_undef(<32 x i16> %a, <32 x i16> %b) nounwind {
define <16 x i32> @abd_ext_v16i32(<16 x i32> %a, <16 x i32> %b) nounwind {
; AVX512-LABEL: abd_ext_v16i32:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpminud %zmm1, %zmm0, %zmm2
-; AVX512-NEXT: vpmaxud %zmm1, %zmm0, %zmm0
-; AVX512-NEXT: vpsubd %zmm2, %zmm0, %zmm0
+; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm2
+; AVX512-NEXT: vpmovzxdq {{.*#+}} zmm2 = ymm2[0],zero,ymm2[1],zero,ymm2[2],zero,ymm2[3],zero,ymm2[4],zero,ymm2[5],zero,ymm2[6],zero,ymm2[7],zero
+; AVX512-NEXT: vpmovzxdq {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero
+; AVX512-NEXT: vextracti64x4 $1, %zmm1, %ymm3
+; AVX512-NEXT: vpmovzxdq {{.*#+}} zmm3 = ymm3[0],zero,ymm3[1],zero,ymm3[2],zero,ymm3[3],zero,ymm3[4],zero,ymm3[5],zero,ymm3[6],zero,ymm3[7],zero
+; AVX512-NEXT: vpsubq %zmm3, %zmm2, %zmm2
+; AVX512-NEXT: vpmovzxdq {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero
+; AVX512-NEXT: vpsubq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpsubq %zmm2, %zmm1, %zmm3
+; AVX512-NEXT: vpmaxsq %zmm3, %zmm2, %zmm2
+; AVX512-NEXT: vpsubq %zmm0, %zmm1, %zmm1
+; AVX512-NEXT: vpmaxsq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpmovqd %zmm0, %ymm0
+; AVX512-NEXT: vpmovqd %zmm2, %ymm1
+; AVX512-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
; AVX512-NEXT: retq
%aext = zext <16 x i32> %a to <16 x i64>
%bext = zext <16 x i32> %b to <16 x i64>
@@ -151,9 +244,141 @@ define <16 x i32> @abd_ext_v16i32_undef(<16 x i32> %a, <16 x i32> %b) nounwind {
define <8 x i64> @abd_ext_v8i64(<8 x i64> %a, <8 x i64> %b) nounwind {
; AVX512-LABEL: abd_ext_v8i64:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpminuq %zmm1, %zmm0, %zmm2
-; AVX512-NEXT: vpmaxuq %zmm1, %zmm0, %zmm0
-; AVX512-NEXT: vpsubq %zmm2, %zmm0, %zmm0
+; AVX512-NEXT: pushq %rbp
+; AVX512-NEXT: pushq %r15
+; AVX512-NEXT: pushq %r14
+; AVX512-NEXT: pushq %r13
+; AVX512-NEXT: pushq %r12
+; AVX512-NEXT: pushq %rbx
+; AVX512-NEXT: vpextrq $1, %xmm0, %r14
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm2
+; AVX512-NEXT: vmovq %xmm2, %r11
+; AVX512-NEXT: vpextrq $1, %xmm2, %r9
+; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm2
+; AVX512-NEXT: vmovq %xmm2, %r8
+; AVX512-NEXT: vpextrq $1, %xmm2, %rsi
+; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm2
+; AVX512-NEXT: vmovq %xmm2, %rdx
+; AVX512-NEXT: vpextrq $1, %xmm2, %rcx
+; AVX512-NEXT: vpextrq $1, %xmm1, %rbx
+; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm2
+; AVX512-NEXT: vmovq %xmm2, %rax
+; AVX512-NEXT: vpextrq $1, %xmm2, %r12
+; AVX512-NEXT: vextracti64x4 $1, %zmm1, %ymm2
+; AVX512-NEXT: vmovq %xmm2, %r13
+; AVX512-NEXT: vpextrq $1, %xmm2, %r15
+; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm2
+; AVX512-NEXT: vmovq %xmm2, %r10
+; AVX512-NEXT: vpextrq $1, %xmm2, %rdi
+; AVX512-NEXT: subq %rdi, %rcx
+; AVX512-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT: movl $0, %edi
+; AVX512-NEXT: sbbq %rdi, %rdi
+; AVX512-NEXT: subq %r10, %rdx
+; AVX512-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT: movl $0, %r10d
+; AVX512-NEXT: sbbq %r10, %r10
+; AVX512-NEXT: subq %r15, %rsi
+; AVX512-NEXT: movq %rsi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX512-NEXT: movl $0, %r15d
+; AVX512-NEXT: sbbq %r15, %r15
+; AVX512-NEXT: subq %r13, %r8
+; AVX512-NEXT: movl $0, %r13d
+; AVX512-NEXT: sbbq %r13, %r13
+; AVX512-NEXT: subq %r12, %r9
+; AVX512-NEXT: movl $0, %ebp
+; AVX512-NEXT: sbbq %rbp, %rbp
+; AVX512-NEXT: subq %rax, %r11
+; AVX512-NEXT: movl $0, %esi
+; AVX512-NEXT: sbbq %rsi, %rsi
+; AVX512-NEXT: subq %rbx, %r14
+; AVX512-NEXT: movl $0, %edx
+; AVX512-NEXT: sbbq %rdx, %rdx
+; AVX512-NEXT: vmovq %xmm0, %r12
+; AVX512-NEXT: vmovq %xmm1, %rbx
+; AVX512-NEXT: subq %rbx, %r12
+; AVX512-NEXT: movl $0, %ecx
+; AVX512-NEXT: sbbq %rcx, %rcx
+; AVX512-NEXT: movq %r12, %rbx
+; AVX512-NEXT: negq %rbx
+; AVX512-NEXT: movl $0, %eax
+; AVX512-NEXT: sbbq %rcx, %rax
+; AVX512-NEXT: cmpq %r12, %rbx
+; AVX512-NEXT: sbbq %rcx, %rax
+; AVX512-NEXT: cmovlq %r12, %rbx
+; AVX512-NEXT: movq %r14, %r12
+; AVX512-NEXT: negq %r12
+; AVX512-NEXT: movl $0, %eax
+; AVX512-NEXT: sbbq %rdx, %rax
+; AVX512-NEXT: cmpq %r14, %r12
+; AVX512-NEXT: sbbq %rdx, %rax
+; AVX512-NEXT: cmovlq %r14, %r12
+; AVX512-NEXT: movq %r11, %r14
+; AVX512-NEXT: negq %r14
+; AVX512-NEXT: movl $0, %eax
+; AVX512-NEXT: sbbq %rsi, %rax
+; AVX512-NEXT: cmpq %r11, %r14
+; AVX512-NEXT: sbbq %rsi, %rax
+; AVX512-NEXT: cmovlq %r11, %r14
+; AVX512-NEXT: movq %r9, %r11
+; AVX512-NEXT: negq %r11
+; AVX512-NEXT: movl $0, %eax
+; AVX512-NEXT: sbbq %rbp, %rax
+; AVX512-NEXT: cmpq %r9, %r11
+; AVX512-NEXT: sbbq %rbp, %rax
+; AVX512-NEXT: cmovlq %r9, %r11
+; AVX512-NEXT: movq %r8, %r9
+; AVX512-NEXT: negq %r9
+; AVX512-NEXT: movl $0, %eax
+; AVX512-NEXT: sbbq %r13, %rax
+; AVX512-NEXT: cmpq %r8, %r9
+; AVX512-NEXT: sbbq %r13, %rax
+; AVX512-NEXT: cmovlq %r8, %r9
+; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
+; AVX512-NEXT: movq %rdx, %rax
+; AVX512-NEXT: negq %rax
+; AVX512-NEXT: movl $0, %ecx
+; AVX512-NEXT: sbbq %r15, %rcx
+; AVX512-NEXT: cmpq %rdx, %rax
+; AVX512-NEXT: sbbq %r15, %rcx
+; AVX512-NEXT: cmovlq %rdx, %rax
+; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
+; AVX512-NEXT: movq %rsi, %rcx
+; AVX512-NEXT: negq %rcx
+; AVX512-NEXT: movl $0, %edx
+; AVX512-NEXT: sbbq %r10, %rdx
+; AVX512-NEXT: cmpq %rsi, %rcx
+; AVX512-NEXT: sbbq %r10, %rdx
+; AVX512-NEXT: cmovlq %rsi, %rcx
+; AVX512-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
+; AVX512-NEXT: movq %rsi, %rdx
+; AVX512-NEXT: negq %rdx
+; AVX512-NEXT: movl $0, %r8d
+; AVX512-NEXT: sbbq %rdi, %r8
+; AVX512-NEXT: cmpq %rsi, %rdx
+; AVX512-NEXT: sbbq %rdi, %r8
+; AVX512-NEXT: cmovlq %rsi, %rdx
+; AVX512-NEXT: vmovq %rdx, %xmm0
+; AVX512-NEXT: vmovq %rcx, %xmm1
+; AVX512-NEXT: vmovq %rax, %xmm2
+; AVX512-NEXT: vmovq %r9, %xmm3
+; AVX512-NEXT: vmovq %r11, %xmm4
+; AVX512-NEXT: vmovq %r14, %xmm5
+; AVX512-NEXT: vmovq %r12, %xmm6
+; AVX512-NEXT: vmovq %rbx, %xmm7
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm1[0],xmm0[0]
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm3[0],xmm2[0]
+; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm0
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm5[0],xmm4[0]
+; AVX512-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm7[0],xmm6[0]
+; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm2, %ymm1
+; AVX512-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
+; AVX512-NEXT: popq %rbx
+; AVX512-NEXT: popq %r12
+; AVX512-NEXT: popq %r13
+; AVX512-NEXT: popq %r14
+; AVX512-NEXT: popq %r15
+; AVX512-NEXT: popq %rbp
; AVX512-NEXT: retq
%aext = zext <8 x i64> %a to <8 x i128>
%bext = zext <8 x i64> %b to <8 x i128>
diff --git a/llvm/test/CodeGen/X86/abdu.ll b/llvm/test/CodeGen/X86/abdu.ll
index 043c9155f52f9..a6cc900ebeca0 100644
--- a/llvm/test/CodeGen/X86/abdu.ll
+++ b/llvm/test/CodeGen/X86/abdu.ll
@@ -9,24 +9,33 @@
define i8 @abd_ext_i8(i8 %a, i8 %b) nounwind {
; X86-LABEL: abd_ext_i8:
; X86: # %bb.0:
+; X86-NEXT: pushl %esi
; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ecx, %edx
-; X86-NEXT: subl %eax, %edx
-; X86-NEXT: subl %ecx, %eax
-; X86-NEXT: cmovbl %edx, %eax
+; X86-NEXT: xorl %edx, %edx
+; X86-NEXT: subl %eax, %ecx
+; X86-NEXT: movl $0, %esi
+; X86-NEXT: sbbl %esi, %esi
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: negl %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmpl %ecx, %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmovll %ecx, %eax
; X86-NEXT: # kill: def $al killed $al killed $eax
+; X86-NEXT: popl %esi
; X86-NEXT: retl
;
; X64-LABEL: abd_ext_i8:
; X64: # %bb.0:
-; X64-NEXT: movzbl %sil, %eax
; X64-NEXT: movzbl %dil, %ecx
-; X64-NEXT: movl %ecx, %edx
-; X64-NEXT: subl %eax, %edx
-; X64-NEXT: subl %ecx, %eax
-; X64-NEXT: cmovbl %edx, %eax
-; X64-NEXT: # kill: def $al killed $al killed $eax
+; X64-NEXT: movzbl %sil, %eax
+; X64-NEXT: subq %rax, %rcx
+; X64-NEXT: movq %rcx, %rax
+; X64-NEXT: negq %rax
+; X64-NEXT: cmpq %rax, %rcx
+; X64-NEXT: cmovgq %rcx, %rax
+; X64-NEXT: # kill: def $al killed $al killed $rax
; X64-NEXT: retq
%aext = zext i8 %a to i64
%bext = zext i8 %b to i64
@@ -39,23 +48,33 @@ define i8 @abd_ext_i8(i8 %a, i8 %b) nounwind {
define i8 @abd_ext_i8_i16(i8 %a, i16 %b) nounwind {
; X86-LABEL: abd_ext_i8_i16:
; X86: # %bb.0:
+; X86-NEXT: pushl %esi
; X86-NEXT: movzwl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ecx, %edx
-; X86-NEXT: subl %eax, %edx
-; X86-NEXT: subl %ecx, %eax
-; X86-NEXT: cmovbl %edx, %eax
+; X86-NEXT: xorl %edx, %edx
+; X86-NEXT: subl %eax, %ecx
+; X86-NEXT: movl $0, %esi
+; X86-NEXT: sbbl %esi, %esi
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: negl %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmpl %ecx, %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmovll %ecx, %eax
; X86-NEXT: # kill: def $al killed $al killed $eax
+; X86-NEXT: popl %esi
; X86-NEXT: retl
;
; X64-LABEL: abd_ext_i8_i16:
; X64: # %bb.0:
; X64-NEXT: movzbl %dil, %ecx
-; X64-NEXT: subl %esi, %edi
; X64-NEXT: movzwl %si, %eax
-; X64-NEXT: subl %ecx, %eax
-; X64-NEXT: cmovbl %edi, %eax
-; X64-NEXT: # kill: def $al killed $al killed $eax
+; X64-NEXT: subq %rax, %rcx
+; X64-NEXT: movq %rcx, %rax
+; X64-NEXT: negq %rax
+; X64-NEXT: cmpq %rax, %rcx
+; X64-NEXT: cmovgq %rcx, %rax
+; X64-NEXT: # kill: def $al killed $al killed $rax
; X64-NEXT: retq
%aext = zext i8 %a to i64
%bext = zext i16 %b to i64
@@ -98,24 +117,33 @@ define i8 @abd_ext_i8_undef(i8 %a, i8 %b) nounwind {
define i16 @abd_ext_i16(i16 %a, i16 %b) nounwind {
; X86-LABEL: abd_ext_i16:
; X86: # %bb.0:
+; X86-NEXT: pushl %esi
; X86-NEXT: movzwl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ecx, %edx
-; X86-NEXT: subl %eax, %edx
-; X86-NEXT: subl %ecx, %eax
-; X86-NEXT: cmovbl %edx, %eax
+; X86-NEXT: xorl %edx, %edx
+; X86-NEXT: subl %eax, %ecx
+; X86-NEXT: movl $0, %esi
+; X86-NEXT: sbbl %esi, %esi
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: negl %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmpl %ecx, %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmovll %ecx, %eax
; X86-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-NEXT: popl %esi
; X86-NEXT: retl
;
; X64-LABEL: abd_ext_i16:
; X64: # %bb.0:
-; X64-NEXT: movzwl %si, %eax
; X64-NEXT: movzwl %di, %ecx
-; X64-NEXT: movl %ecx, %edx
-; X64-NEXT: subl %eax, %edx
-; X64-NEXT: subl %ecx, %eax
-; X64-NEXT: cmovbl %edx, %eax
-; X64-NEXT: # kill: def $ax killed $ax killed $eax
+; X64-NEXT: movzwl %si, %eax
+; X64-NEXT: subq %rax, %rcx
+; X64-NEXT: movq %rcx, %rax
+; X64-NEXT: negq %rax
+; X64-NEXT: cmpq %rax, %rcx
+; X64-NEXT: cmovgq %rcx, %rax
+; X64-NEXT: # kill: def $ax killed $ax killed $rax
; X64-NEXT: retq
%aext = zext i16 %a to i64
%bext = zext i16 %b to i64
@@ -128,23 +156,32 @@ define i16 @abd_ext_i16(i16 %a, i16 %b) nounwind {
define i16 @abd_ext_i16_i32(i16 %a, i32 %b) nounwind {
; X86-LABEL: abd_ext_i16_i32:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: pushl %esi
; X86-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ecx, %edx
-; X86-NEXT: subl %eax, %edx
-; X86-NEXT: subl %ecx, %eax
-; X86-NEXT: cmovbl %edx, %eax
+; X86-NEXT: xorl %edx, %edx
+; X86-NEXT: subl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl $0, %esi
+; X86-NEXT: sbbl %esi, %esi
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: negl %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmpl %ecx, %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmovll %ecx, %eax
; X86-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-NEXT: popl %esi
; X86-NEXT: retl
;
; X64-LABEL: abd_ext_i16_i32:
; X64: # %bb.0:
; X64-NEXT: movzwl %di, %ecx
-; X64-NEXT: movl %edi, %eax
-; X64-NEXT: subl %esi, %eax
-; X64-NEXT: subl %ecx, %esi
-; X64-NEXT: cmovael %esi, %eax
-; X64-NEXT: # kill: def $ax killed $ax killed $eax
+; X64-NEXT: movl %esi, %eax
+; X64-NEXT: subq %rax, %rcx
+; X64-NEXT: movq %rcx, %rax
+; X64-NEXT: negq %rax
+; X64-NEXT: cmpq %rax, %rcx
+; X64-NEXT: cmovgq %rcx, %rax
+; X64-NEXT: # kill: def $ax killed $ax killed $rax
; X64-NEXT: retq
%aext = zext i16 %a to i64
%bext = zext i32 %b to i64
@@ -187,20 +224,31 @@ define i16 @abd_ext_i16_undef(i16 %a, i16 %b) nounwind {
define i32 @abd_ext_i32(i32 %a, i32 %b) nounwind {
; X86-LABEL: abd_ext_i32:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: pushl %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ecx, %edx
-; X86-NEXT: subl %eax, %edx
-; X86-NEXT: subl %ecx, %eax
-; X86-NEXT: cmovbl %edx, %eax
+; X86-NEXT: xorl %edx, %edx
+; X86-NEXT: subl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl $0, %esi
+; X86-NEXT: sbbl %esi, %esi
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: negl %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmpl %ecx, %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmovll %ecx, %eax
+; X86-NEXT: popl %esi
; X86-NEXT: retl
;
; X64-LABEL: abd_ext_i32:
; X64: # %bb.0:
-; X64-NEXT: movl %edi, %eax
-; X64-NEXT: subl %esi, %eax
-; X64-NEXT: subl %edi, %esi
-; X64-NEXT: cmovael %esi, %eax
+; X64-NEXT: movl %edi, %ecx
+; X64-NEXT: movl %esi, %eax
+; X64-NEXT: subq %rax, %rcx
+; X64-NEXT: movq %rcx, %rax
+; X64-NEXT: negq %rax
+; X64-NEXT: cmpq %rax, %rcx
+; X64-NEXT: cmovgq %rcx, %rax
+; X64-NEXT: # kill: def $eax killed $eax killed $rax
; X64-NEXT: retq
%aext = zext i32 %a to i64
%bext = zext i32 %b to i64
@@ -213,21 +261,32 @@ define i32 @abd_ext_i32(i32 %a, i32 %b) nounwind {
define i32 @abd_ext_i32_i16(i32 %a, i16 %b) nounwind {
; X86-LABEL: abd_ext_i32_i16:
; X86: # %bb.0:
+; X86-NEXT: pushl %esi
; X86-NEXT: movzwl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ecx, %edx
-; X86-NEXT: subl %eax, %edx
-; X86-NEXT: subl %ecx, %eax
-; X86-NEXT: cmovbl %edx, %eax
+; X86-NEXT: xorl %edx, %edx
+; X86-NEXT: subl %eax, %ecx
+; X86-NEXT: movl $0, %esi
+; X86-NEXT: sbbl %esi, %esi
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: negl %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmpl %ecx, %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmovll %ecx, %eax
+; X86-NEXT: popl %esi
; X86-NEXT: retl
;
; X64-LABEL: abd_ext_i32_i16:
; X64: # %bb.0:
-; X64-NEXT: movzwl %si, %eax
; X64-NEXT: movl %edi, %ecx
-; X64-NEXT: subl %eax, %ecx
-; X64-NEXT: subl %edi, %eax
-; X64-NEXT: cmovbl %ecx, %eax
+; X64-NEXT: movzwl %si, %eax
+; X64-NEXT: subq %rax, %rcx
+; X64-NEXT: movq %rcx, %rax
+; X64-NEXT: negq %rax
+; X64-NEXT: cmpq %rax, %rcx
+; X64-NEXT: cmovgq %rcx, %rax
+; X64-NEXT: # kill: def $eax killed $eax killed $rax
; X64-NEXT: retq
%aext = zext i32 %a to i64
%bext = zext i16 %b to i64
@@ -266,24 +325,55 @@ define i32 @abd_ext_i32_undef(i32 %a, i32 %b) nounwind {
define i64 @abd_ext_i64(i64 %a, i64 %b) nounwind {
; X86-LABEL: abd_ext_i64:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: pushl %ebp
+; X86-NEXT: pushl %ebx
+; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %esi
+; X86-NEXT: subl $8, %esp
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: xorl %ecx, %ecx
-; X86-NEXT: subl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: subl {{[0-9]+}}(%esp), %edi
; X86-NEXT: sbbl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: sbbl %ecx, %ecx
-; X86-NEXT: xorl %ecx, %edx
-; X86-NEXT: xorl %ecx, %eax
-; X86-NEXT: subl %ecx, %eax
-; X86-NEXT: sbbl %ecx, %edx
+; X86-NEXT: movl %edx, (%esp) # 4-byte Spill
+; X86-NEXT: movl $0, %ebx
+; X86-NEXT: sbbl %ebx, %ebx
+; X86-NEXT: movl $0, %ebp
+; X86-NEXT: sbbl %ebp, %ebp
+; X86-NEXT: movl %edi, %eax
+; X86-NEXT: negl %eax
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: sbbl %edx, %ecx
+; X86-NEXT: movl $0, %esi
+; X86-NEXT: sbbl %ebx, %esi
+; X86-NEXT: sbbl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: cmpl %edi, %eax
+; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: sbbl (%esp), %edx # 4-byte Folded Reload
+; X86-NEXT: sbbl %ebx, %esi
+; X86-NEXT: sbbl %ebp, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: cmovll %edi, %eax
+; X86-NEXT: cmovll (%esp), %ecx # 4-byte Folded Reload
+; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: addl $8, %esp
+; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
+; X86-NEXT: popl %ebx
+; X86-NEXT: popl %ebp
; X86-NEXT: retl
;
; X64-LABEL: abd_ext_i64:
; X64: # %bb.0:
+; X64-NEXT: xorl %ecx, %ecx
+; X64-NEXT: subq %rsi, %rdi
+; X64-NEXT: movl $0, %edx
+; X64-NEXT: sbbq %rdx, %rdx
; X64-NEXT: movq %rdi, %rax
-; X64-NEXT: subq %rsi, %rax
-; X64-NEXT: subq %rdi, %rsi
-; X64-NEXT: cmovaeq %rsi, %rax
+; X64-NEXT: negq %rax
+; X64-NEXT: sbbq %rdx, %rcx
+; X64-NEXT: cmpq %rdi, %rax
+; X64-NEXT: sbbq %rdx, %rcx
+; X64-NEXT: cmovlq %rdi, %rax
; X64-NEXT: retq
%aext = zext i64 %a to i128
%bext = zext i64 %b to i128
@@ -328,50 +418,107 @@ define i128 @abd_ext_i128(i128 %a, i128 %b) nounwind {
; X86: # %bb.0:
; X86-NEXT: pushl %ebp
; X86-NEXT: movl %esp, %ebp
+; X86-NEXT: pushl %ebx
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
; X86-NEXT: andl $-16, %esp
-; X86-NEXT: movl 32(%ebp), %edx
-; X86-NEXT: movl 36(%ebp), %ecx
-; X86-NEXT: movl 24(%ebp), %edi
-; X86-NEXT: movl 28(%ebp), %esi
-; X86-NEXT: xorl %eax, %eax
-; X86-NEXT: subl 40(%ebp), %edi
-; X86-NEXT: sbbl 44(%ebp), %esi
-; X86-NEXT: sbbl 48(%ebp), %edx
-; X86-NEXT: sbbl 52(%ebp), %ecx
-; X86-NEXT: sbbl %eax, %eax
-; X86-NEXT: xorl %eax, %ecx
-; X86-NEXT: xorl %eax, %edx
-; X86-NEXT: xorl %eax, %esi
-; X86-NEXT: xorl %eax, %edi
-; X86-NEXT: subl %eax, %edi
+; X86-NEXT: subl $48, %esp
+; X86-NEXT: movl 32(%ebp), %ecx
+; X86-NEXT: movl 36(%ebp), %eax
+; X86-NEXT: movl 24(%ebp), %esi
+; X86-NEXT: movl 28(%ebp), %edx
+; X86-NEXT: movl $0, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: subl 40(%ebp), %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl 44(%ebp), %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl 48(%ebp), %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: sbbl 52(%ebp), %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, %edi
+; X86-NEXT: sbbl %edi, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, %edi
+; X86-NEXT: sbbl %edi, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, %edi
+; X86-NEXT: sbbl %edi, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, %edi
+; X86-NEXT: sbbl %edi, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: negl %esi
+; X86-NEXT: movl %esi, (%esp) # 4-byte Spill
+; X86-NEXT: movl $0, %esi
+; X86-NEXT: sbbl %edx, %esi
+; X86-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl $0, %edi
+; X86-NEXT: sbbl %ecx, %edi
+; X86-NEXT: movl $0, %esi
; X86-NEXT: sbbl %eax, %esi
-; X86-NEXT: sbbl %eax, %edx
-; X86-NEXT: sbbl %eax, %ecx
+; X86-NEXT: movl $0, %eax
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NEXT: movl $0, %ecx
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: movl $0, %edx
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: sbbl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: movl (%esp), %ebx # 4-byte Reload
+; X86-NEXT: cmpl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X86-NEXT: movl %edi, %ebx
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X86-NEXT: movl %esi, %ebx
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X86-NEXT: sbbl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X86-NEXT: movl (%esp), %ecx # 4-byte Reload
+; X86-NEXT: cmovll {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X86-NEXT: cmovll {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X86-NEXT: cmovll {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X86-NEXT: cmovll {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: movl %edi, (%eax)
-; X86-NEXT: movl %esi, 4(%eax)
-; X86-NEXT: movl %edx, 8(%eax)
-; X86-NEXT: movl %ecx, 12(%eax)
-; X86-NEXT: leal -8(%ebp), %esp
+; X86-NEXT: movl %esi, 12(%eax)
+; X86-NEXT: movl %edi, 8(%eax)
+; X86-NEXT: movl %edx, 4(%eax)
+; X86-NEXT: movl %ecx, (%eax)
+; X86-NEXT: leal -12(%ebp), %esp
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
+; X86-NEXT: popl %ebx
; X86-NEXT: popl %ebp
; X86-NEXT: retl $4
;
; X64-LABEL: abd_ext_i128:
; X64: # %bb.0:
-; X64-NEXT: movq %rdi, %rax
-; X64-NEXT: xorl %edi, %edi
-; X64-NEXT: subq %rdx, %rax
+; X64-NEXT: xorl %r8d, %r8d
+; X64-NEXT: subq %rdx, %rdi
; X64-NEXT: sbbq %rcx, %rsi
-; X64-NEXT: sbbq %rdi, %rdi
-; X64-NEXT: xorq %rdi, %rsi
-; X64-NEXT: xorq %rdi, %rax
-; X64-NEXT: subq %rdi, %rax
-; X64-NEXT: sbbq %rdi, %rsi
-; X64-NEXT: movq %rsi, %rdx
+; X64-NEXT: movl $0, %ecx
+; X64-NEXT: sbbq %rcx, %rcx
+; X64-NEXT: movl $0, %r9d
+; X64-NEXT: sbbq %r9, %r9
+; X64-NEXT: movq %rdi, %rax
+; X64-NEXT: negq %rax
+; X64-NEXT: movl $0, %edx
+; X64-NEXT: sbbq %rsi, %rdx
+; X64-NEXT: movl $0, %r10d
+; X64-NEXT: sbbq %rcx, %r10
+; X64-NEXT: sbbq %r9, %r8
+; X64-NEXT: cmpq %rdi, %rax
+; X64-NEXT: movq %rdx, %r11
+; X64-NEXT: sbbq %rsi, %r11
+; X64-NEXT: sbbq %rcx, %r10
+; X64-NEXT: sbbq %r9, %r8
+; X64-NEXT: cmovlq %rdi, %rax
+; X64-NEXT: cmovlq %rsi, %rdx
; X64-NEXT: retq
%aext = zext i128 %a to i256
%bext = zext i128 %b to i256
diff --git a/llvm/test/CodeGen/X86/abs.ll b/llvm/test/CodeGen/X86/abs.ll
index e252d5953e60e..817cfb649a255 100644
--- a/llvm/test/CodeGen/X86/abs.ll
+++ b/llvm/test/CodeGen/X86/abs.ll
@@ -26,20 +26,23 @@ define i8 @test_i8(i8 %a) nounwind {
; X64-LABEL: test_i8:
; X64: # %bb.0:
; X64-NEXT: movl %edi, %eax
-; X64-NEXT: movl %eax, %ecx
-; X64-NEXT: sarb $7, %cl
-; X64-NEXT: xorb %cl, %al
-; X64-NEXT: subb %cl, %al
+; X64-NEXT: negb %al
+; X64-NEXT: movzbl %al, %eax
+; X64-NEXT: cmpb %al, %dil
+; X64-NEXT: cmovgl %edi, %eax
; X64-NEXT: # kill: def $al killed $al killed $eax
; X64-NEXT: retq
;
; X86-LABEL: test_i8:
; X86: # %bb.0:
-; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl %eax, %ecx
-; X86-NEXT: sarb $7, %cl
-; X86-NEXT: xorb %cl, %al
-; X86-NEXT: subb %cl, %al
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movzbl %cl, %edx
+; X86-NEXT: movb %cl, %ch
+; X86-NEXT: negb %ch
+; X86-NEXT: movzbl %ch, %eax
+; X86-NEXT: cmpb %ch, %cl
+; X86-NEXT: cmovgl %edx, %eax
+; X86-NEXT: # kill: def $al killed $al killed $eax
; X86-NEXT: retl
%r = call i8 @llvm.abs.i8(i8 %a, i1 false)
ret i8 %r
@@ -49,16 +52,20 @@ define i16 @test_i16(i16 %a) nounwind {
; X64-LABEL: test_i16:
; X64: # %bb.0:
; X64-NEXT: movl %edi, %eax
-; X64-NEXT: negw %ax
-; X64-NEXT: cmovsw %di, %ax
+; X64-NEXT: negl %eax
+; X64-NEXT: cmpw %ax, %di
+; X64-NEXT: cmovgl %edi, %eax
+; X64-NEXT: # kill: def $ax killed $ax killed $eax
; X64-NEXT: retq
;
; X86-LABEL: test_i16:
; X86: # %bb.0:
; X86-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: negw %ax
-; X86-NEXT: cmovsw %cx, %ax
+; X86-NEXT: negl %eax
+; X86-NEXT: cmpw %ax, %cx
+; X86-NEXT: cmovgl %ecx, %eax
+; X86-NEXT: # kill: def $ax killed $ax killed $eax
; X86-NEXT: retl
%r = call i16 @llvm.abs.i16(i16 %a, i1 false)
ret i16 %r
@@ -67,21 +74,27 @@ define i16 @test_i16(i16 %a) nounwind {
define i24 @test_i24(i24 %a) nounwind {
; X64-LABEL: test_i24:
; X64: # %bb.0:
+; X64-NEXT: movl %edi, %eax
+; X64-NEXT: shll $8, %eax
+; X64-NEXT: sarl $8, %eax
+; X64-NEXT: negl %edi
; X64-NEXT: shll $8, %edi
; X64-NEXT: sarl $8, %edi
-; X64-NEXT: movl %edi, %eax
-; X64-NEXT: negl %eax
-; X64-NEXT: cmovsl %edi, %eax
+; X64-NEXT: cmpl %edi, %eax
+; X64-NEXT: cmovlel %edi, %eax
; X64-NEXT: retq
;
; X86-LABEL: test_i24:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %eax, %ecx
; X86-NEXT: shll $8, %ecx
; X86-NEXT: sarl $8, %ecx
-; X86-NEXT: movl %ecx, %eax
; X86-NEXT: negl %eax
-; X86-NEXT: cmovsl %ecx, %eax
+; X86-NEXT: shll $8, %eax
+; X86-NEXT: sarl $8, %eax
+; X86-NEXT: cmpl %eax, %ecx
+; X86-NEXT: cmovgl %ecx, %eax
; X86-NEXT: retl
%r = call i24 @llvm.abs.i24(i24 %a, i1 false)
ret i24 %r
@@ -92,7 +105,8 @@ define i32 @test_i32(i32 %a) nounwind {
; X64: # %bb.0:
; X64-NEXT: movl %edi, %eax
; X64-NEXT: negl %eax
-; X64-NEXT: cmovsl %edi, %eax
+; X64-NEXT: cmpl %eax, %edi
+; X64-NEXT: cmovgl %edi, %eax
; X64-NEXT: retq
;
; X86-LABEL: test_i32:
@@ -100,7 +114,8 @@ define i32 @test_i32(i32 %a) nounwind {
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: negl %eax
-; X86-NEXT: cmovsl %ecx, %eax
+; X86-NEXT: cmpl %eax, %ecx
+; X86-NEXT: cmovgl %ecx, %eax
; X86-NEXT: retl
%r = call i32 @llvm.abs.i32(i32 %a, i1 false)
ret i32 %r
@@ -111,19 +126,27 @@ define i64 @test_i64(i64 %a) nounwind {
; X64: # %bb.0:
; X64-NEXT: movq %rdi, %rax
; X64-NEXT: negq %rax
-; X64-NEXT: cmovsq %rdi, %rax
+; X64-NEXT: cmpq %rax, %rdi
+; X64-NEXT: cmovgq %rdi, %rax
; X64-NEXT: retq
;
; X86-LABEL: test_i64:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl %edx, %ecx
-; X86-NEXT: sarl $31, %ecx
-; X86-NEXT: xorl %ecx, %edx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: xorl %ecx, %eax
-; X86-NEXT: subl %ecx, %eax
+; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT: xorl %edx, %edx
+; X86-NEXT: movl %esi, %eax
+; X86-NEXT: negl %eax
; X86-NEXT: sbbl %ecx, %edx
+; X86-NEXT: cmpl %esi, %eax
+; X86-NEXT: movl %edx, %edi
+; X86-NEXT: sbbl %ecx, %edi
+; X86-NEXT: cmovll %ecx, %edx
+; X86-NEXT: cmovll %esi, %eax
+; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
; X86-NEXT: retl
%r = call i64 @llvm.abs.i64(i64 %a, i1 false)
ret i64 %r
@@ -132,45 +155,56 @@ define i64 @test_i64(i64 %a) nounwind {
define i128 @test_i128(i128 %a) nounwind {
; X64-LABEL: test_i128:
; X64: # %bb.0:
-; X64-NEXT: movq %rsi, %rdx
+; X64-NEXT: xorl %edx, %edx
; X64-NEXT: movq %rdi, %rax
-; X64-NEXT: movq %rsi, %rcx
-; X64-NEXT: sarq $63, %rcx
-; X64-NEXT: xorq %rcx, %rdx
-; X64-NEXT: xorq %rcx, %rax
-; X64-NEXT: subq %rcx, %rax
-; X64-NEXT: sbbq %rcx, %rdx
+; X64-NEXT: negq %rax
+; X64-NEXT: sbbq %rsi, %rdx
+; X64-NEXT: cmpq %rdi, %rax
+; X64-NEXT: movq %rdx, %rcx
+; X64-NEXT: sbbq %rsi, %rcx
+; X64-NEXT: cmovlq %rsi, %rdx
+; X64-NEXT: cmovlq %rdi, %rax
; X64-NEXT: retq
;
; X86-LABEL: test_i128:
; X86: # %bb.0:
; X86-NEXT: pushl %ebp
; X86-NEXT: movl %esp, %ebp
+; X86-NEXT: pushl %ebx
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
; X86-NEXT: andl $-16, %esp
-; X86-NEXT: movl 36(%ebp), %ecx
-; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: sarl $31, %eax
-; X86-NEXT: xorl %eax, %ecx
-; X86-NEXT: movl 32(%ebp), %edx
-; X86-NEXT: xorl %eax, %edx
-; X86-NEXT: movl 28(%ebp), %esi
-; X86-NEXT: xorl %eax, %esi
-; X86-NEXT: movl 24(%ebp), %edi
-; X86-NEXT: xorl %eax, %edi
-; X86-NEXT: subl %eax, %edi
-; X86-NEXT: sbbl %eax, %esi
-; X86-NEXT: sbbl %eax, %edx
-; X86-NEXT: sbbl %eax, %ecx
-; X86-NEXT: movl 8(%ebp), %eax
-; X86-NEXT: movl %edi, (%eax)
-; X86-NEXT: movl %esi, 4(%eax)
-; X86-NEXT: movl %edx, 8(%eax)
-; X86-NEXT: movl %ecx, 12(%eax)
-; X86-NEXT: leal -8(%ebp), %esp
+; X86-NEXT: subl $16, %esp
+; X86-NEXT: movl 24(%ebp), %ecx
+; X86-NEXT: xorl %edx, %edx
+; X86-NEXT: negl %ecx
+; X86-NEXT: movl $0, %eax
+; X86-NEXT: sbbl 28(%ebp), %eax
+; X86-NEXT: movl $0, %ebx
+; X86-NEXT: sbbl 32(%ebp), %ebx
+; X86-NEXT: movl 36(%ebp), %edi
+; X86-NEXT: sbbl %edi, %edx
+; X86-NEXT: cmpl 24(%ebp), %ecx
+; X86-NEXT: movl %eax, %esi
+; X86-NEXT: sbbl 28(%ebp), %esi
+; X86-NEXT: movl %ebx, %esi
+; X86-NEXT: sbbl 32(%ebp), %esi
+; X86-NEXT: movl %edx, %esi
+; X86-NEXT: sbbl %edi, %esi
+; X86-NEXT: cmovll 24(%ebp), %ecx
+; X86-NEXT: cmovll 28(%ebp), %eax
+; X86-NEXT: cmovll 32(%ebp), %ebx
+; X86-NEXT: cmovll %edi, %edx
+; X86-NEXT: movl 8(%ebp), %esi
+; X86-NEXT: movl %edx, 12(%esi)
+; X86-NEXT: movl %ebx, 8(%esi)
+; X86-NEXT: movl %eax, 4(%esi)
+; X86-NEXT: movl %ecx, (%esi)
+; X86-NEXT: movl %esi, %eax
+; X86-NEXT: leal -12(%ebp), %esp
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
+; X86-NEXT: popl %ebx
; X86-NEXT: popl %ebp
; X86-NEXT: retl $4
%r = call i128 @llvm.abs.i128(i128 %a, i1 false)
@@ -182,7 +216,8 @@ define <1 x i32> @test_v1i32(<1 x i32> %a) nounwind {
; X64: # %bb.0:
; X64-NEXT: movl %edi, %eax
; X64-NEXT: negl %eax
-; X64-NEXT: cmovsl %edi, %eax
+; X64-NEXT: cmpl %eax, %edi
+; X64-NEXT: cmovgl %edi, %eax
; X64-NEXT: retq
;
; X86-LABEL: test_v1i32:
@@ -190,7 +225,8 @@ define <1 x i32> @test_v1i32(<1 x i32> %a) nounwind {
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: negl %eax
-; X86-NEXT: cmovsl %ecx, %eax
+; X86-NEXT: cmpl %eax, %ecx
+; X86-NEXT: cmovgl %ecx, %eax
; X86-NEXT: retl
%r = call <1 x i32> @llvm.abs.v1i32(<1 x i32> %a, i1 false)
ret <1 x i32> %r
@@ -199,15 +235,20 @@ define <1 x i32> @test_v1i32(<1 x i32> %a) nounwind {
define <2 x i32> @test_v2i32(<2 x i32> %a) nounwind {
; SSE-LABEL: test_v2i32:
; SSE: # %bb.0:
-; SSE-NEXT: movdqa %xmm0, %xmm1
-; SSE-NEXT: psrad $31, %xmm1
-; SSE-NEXT: pxor %xmm1, %xmm0
-; SSE-NEXT: psubd %xmm1, %xmm0
+; SSE-NEXT: pxor %xmm1, %xmm1
+; SSE-NEXT: psubd %xmm0, %xmm1
+; SSE-NEXT: movdqa %xmm0, %xmm2
+; SSE-NEXT: pcmpgtd %xmm1, %xmm2
+; SSE-NEXT: pand %xmm2, %xmm0
+; SSE-NEXT: pandn %xmm1, %xmm2
+; SSE-NEXT: por %xmm2, %xmm0
; SSE-NEXT: retq
;
; AVX-LABEL: test_v2i32:
; AVX: # %bb.0:
-; AVX-NEXT: vpabsd %xmm0, %xmm0
+; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX-NEXT: vpsubd %xmm0, %xmm1, %xmm1
+; AVX-NEXT: vpmaxsd %xmm1, %xmm0, %xmm0
; AVX-NEXT: retq
;
; X86-LABEL: test_v2i32:
@@ -216,10 +257,12 @@ define <2 x i32> @test_v2i32(<2 x i32> %a) nounwind {
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-NEXT: movl %edx, %eax
; X86-NEXT: negl %eax
-; X86-NEXT: cmovsl %edx, %eax
+; X86-NEXT: cmpl %eax, %edx
+; X86-NEXT: cmovgl %edx, %eax
; X86-NEXT: movl %ecx, %edx
; X86-NEXT: negl %edx
-; X86-NEXT: cmovsl %ecx, %edx
+; X86-NEXT: cmpl %edx, %ecx
+; X86-NEXT: cmovgl %ecx, %edx
; X86-NEXT: retl
%r = call <2 x i32> @llvm.abs.v2i32(<2 x i32> %a, i1 false)
ret <2 x i32> %r
@@ -228,15 +271,20 @@ define <2 x i32> @test_v2i32(<2 x i32> %a) nounwind {
define <3 x i32> @test_v3i32(<3 x i32> %a) nounwind {
; SSE-LABEL: test_v3i32:
; SSE: # %bb.0:
-; SSE-NEXT: movdqa %xmm0, %xmm1
-; SSE-NEXT: psrad $31, %xmm1
-; SSE-NEXT: pxor %xmm1, %xmm0
-; SSE-NEXT: psubd %xmm1, %xmm0
+; SSE-NEXT: pxor %xmm1, %xmm1
+; SSE-NEXT: psubd %xmm0, %xmm1
+; SSE-NEXT: movdqa %xmm0, %xmm2
+; SSE-NEXT: pcmpgtd %xmm1, %xmm2
+; SSE-NEXT: pand %xmm2, %xmm0
+; SSE-NEXT: pandn %xmm1, %xmm2
+; SSE-NEXT: por %xmm2, %xmm0
; SSE-NEXT: retq
;
; AVX-LABEL: test_v3i32:
; AVX: # %bb.0:
-; AVX-NEXT: vpabsd %xmm0, %xmm0
+; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX-NEXT: vpsubd %xmm0, %xmm1, %xmm1
+; AVX-NEXT: vpmaxsd %xmm1, %xmm0, %xmm0
; AVX-NEXT: retq
;
; X86-LABEL: test_v3i32:
@@ -247,13 +295,16 @@ define <3 x i32> @test_v3i32(<3 x i32> %a) nounwind {
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-NEXT: movl %edx, %eax
; X86-NEXT: negl %eax
-; X86-NEXT: cmovsl %edx, %eax
+; X86-NEXT: cmpl %eax, %edx
+; X86-NEXT: cmovgl %edx, %eax
; X86-NEXT: movl %ecx, %edx
; X86-NEXT: negl %edx
-; X86-NEXT: cmovsl %ecx, %edx
+; X86-NEXT: cmpl %edx, %ecx
+; X86-NEXT: cmovgl %ecx, %edx
; X86-NEXT: movl %esi, %ecx
; X86-NEXT: negl %ecx
-; X86-NEXT: cmovsl %esi, %ecx
+; X86-NEXT: cmpl %ecx, %esi
+; X86-NEXT: cmovgl %esi, %ecx
; X86-NEXT: popl %esi
; X86-NEXT: retl
%r = call <3 x i32> @llvm.abs.v3i32(<3 x i32> %a, i1 false)
@@ -263,15 +314,20 @@ define <3 x i32> @test_v3i32(<3 x i32> %a) nounwind {
define <4 x i32> @test_v4i32(<4 x i32> %a) nounwind {
; SSE-LABEL: test_v4i32:
; SSE: # %bb.0:
+; SSE-NEXT: pxor %xmm2, %xmm2
+; SSE-NEXT: psubd %xmm0, %xmm2
; SSE-NEXT: movdqa %xmm0, %xmm1
-; SSE-NEXT: psrad $31, %xmm1
+; SSE-NEXT: pcmpgtd %xmm2, %xmm1
; SSE-NEXT: pxor %xmm1, %xmm0
-; SSE-NEXT: psubd %xmm1, %xmm0
+; SSE-NEXT: psubd %xmm0, %xmm1
+; SSE-NEXT: movdqa %xmm1, %xmm0
; SSE-NEXT: retq
;
; AVX-LABEL: test_v4i32:
; AVX: # %bb.0:
-; AVX-NEXT: vpabsd %xmm0, %xmm0
+; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX-NEXT: vpsubd %xmm0, %xmm1, %xmm1
+; AVX-NEXT: vpmaxsd %xmm1, %xmm0, %xmm0
; AVX-NEXT: retq
;
; X86-LABEL: test_v4i32:
@@ -279,26 +335,30 @@ define <4 x i32> @test_v4i32(<4 x i32> %a) nounwind {
; X86-NEXT: pushl %ebx
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X86-NEXT: movl %ebx, %edx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %edi, %edx
; X86-NEXT: negl %edx
-; X86-NEXT: cmovsl %ebx, %edx
-; X86-NEXT: movl %edi, %ebx
-; X86-NEXT: negl %ebx
-; X86-NEXT: cmovsl %edi, %ebx
-; X86-NEXT: movl %esi, %edi
+; X86-NEXT: cmpl %edx, %edi
+; X86-NEXT: cmovgl %edi, %edx
+; X86-NEXT: movl %ebx, %edi
; X86-NEXT: negl %edi
-; X86-NEXT: cmovsl %esi, %edi
+; X86-NEXT: cmpl %edi, %ebx
+; X86-NEXT: cmovgl %ebx, %edi
+; X86-NEXT: movl %esi, %ebx
+; X86-NEXT: negl %ebx
+; X86-NEXT: cmpl %ebx, %esi
+; X86-NEXT: cmovgl %esi, %ebx
; X86-NEXT: movl %ecx, %esi
; X86-NEXT: negl %esi
-; X86-NEXT: cmovsl %ecx, %esi
+; X86-NEXT: cmpl %esi, %ecx
+; X86-NEXT: cmovgl %ecx, %esi
; X86-NEXT: movl %esi, 12(%eax)
-; X86-NEXT: movl %edi, 8(%eax)
-; X86-NEXT: movl %ebx, 4(%eax)
+; X86-NEXT: movl %ebx, 8(%eax)
+; X86-NEXT: movl %edi, 4(%eax)
; X86-NEXT: movl %edx, (%eax)
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
@@ -311,27 +371,38 @@ define <4 x i32> @test_v4i32(<4 x i32> %a) nounwind {
define <8 x i32> @test_v8i32(<8 x i32> %a) nounwind {
; SSE-LABEL: test_v8i32:
; SSE: # %bb.0:
+; SSE-NEXT: pxor %xmm4, %xmm4
+; SSE-NEXT: pxor %xmm3, %xmm3
+; SSE-NEXT: psubd %xmm0, %xmm3
; SSE-NEXT: movdqa %xmm0, %xmm2
-; SSE-NEXT: psrad $31, %xmm2
+; SSE-NEXT: pcmpgtd %xmm3, %xmm2
; SSE-NEXT: pxor %xmm2, %xmm0
-; SSE-NEXT: psubd %xmm2, %xmm0
-; SSE-NEXT: movdqa %xmm1, %xmm2
-; SSE-NEXT: psrad $31, %xmm2
-; SSE-NEXT: pxor %xmm2, %xmm1
-; SSE-NEXT: psubd %xmm2, %xmm1
+; SSE-NEXT: psubd %xmm0, %xmm2
+; SSE-NEXT: psubd %xmm1, %xmm4
+; SSE-NEXT: movdqa %xmm1, %xmm3
+; SSE-NEXT: pcmpgtd %xmm4, %xmm3
+; SSE-NEXT: pxor %xmm3, %xmm1
+; SSE-NEXT: psubd %xmm1, %xmm3
+; SSE-NEXT: movdqa %xmm2, %xmm0
+; SSE-NEXT: movdqa %xmm3, %xmm1
; SSE-NEXT: retq
;
; AVX1-LABEL: test_v8i32:
; AVX1: # %bb.0:
-; AVX1-NEXT: vpabsd %xmm0, %xmm1
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT: vpabsd %xmm0, %xmm0
-; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
+; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX1-NEXT: vpsubd %xmm1, %xmm2, %xmm3
+; AVX1-NEXT: vpmaxsd %xmm3, %xmm1, %xmm1
+; AVX1-NEXT: vpsubd %xmm0, %xmm2, %xmm2
+; AVX1-NEXT: vpmaxsd %xmm2, %xmm0, %xmm0
+; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
; AVX1-NEXT: retq
;
; AVX2-LABEL: test_v8i32:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpabsd %ymm0, %ymm0
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpsubd %ymm0, %ymm1, %ymm1
+; AVX2-NEXT: vpmaxsd %ymm1, %ymm0, %ymm0
; AVX2-NEXT: retq
;
; X86-LABEL: test_v8i32:
@@ -349,32 +420,40 @@ define <8 x i32> @test_v8i32(<8 x i32> %a) nounwind {
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-NEXT: movl %edx, %ecx
; X86-NEXT: negl %ecx
-; X86-NEXT: cmovsl %edx, %ecx
+; X86-NEXT: cmpl %ecx, %edx
+; X86-NEXT: cmovgl %edx, %ecx
; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %esi, %ecx
; X86-NEXT: negl %ecx
-; X86-NEXT: cmovsl %esi, %ecx
+; X86-NEXT: cmpl %ecx, %esi
+; X86-NEXT: cmovgl %esi, %ecx
; X86-NEXT: movl %ecx, (%esp) # 4-byte Spill
; X86-NEXT: movl %edi, %esi
; X86-NEXT: negl %esi
-; X86-NEXT: cmovsl %edi, %esi
+; X86-NEXT: cmpl %esi, %edi
+; X86-NEXT: cmovgl %edi, %esi
; X86-NEXT: movl %ebp, %edi
; X86-NEXT: negl %edi
-; X86-NEXT: cmovsl %ebp, %edi
+; X86-NEXT: cmpl %edi, %ebp
+; X86-NEXT: cmovgl %ebp, %edi
; X86-NEXT: movl %ebx, %ebp
; X86-NEXT: negl %ebp
-; X86-NEXT: cmovsl %ebx, %ebp
+; X86-NEXT: cmpl %ebp, %ebx
+; X86-NEXT: cmovgl %ebx, %ebp
; X86-NEXT: movl %eax, %ebx
; X86-NEXT: negl %ebx
-; X86-NEXT: cmovsl %eax, %ebx
+; X86-NEXT: cmpl %ebx, %eax
+; X86-NEXT: cmovgl %eax, %ebx
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: negl %eax
-; X86-NEXT: cmovsl %ecx, %eax
+; X86-NEXT: cmpl %eax, %ecx
+; X86-NEXT: cmovgl %ecx, %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-NEXT: movl %edx, %ecx
; X86-NEXT: negl %ecx
-; X86-NEXT: cmovsl %edx, %ecx
+; X86-NEXT: cmpl %ecx, %edx
+; X86-NEXT: cmovgl %edx, %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-NEXT: movl %ecx, 28(%edx)
; X86-NEXT: movl %eax, 24(%edx)
@@ -407,7 +486,9 @@ define <8 x i16> @test_v8i16(<8 x i16> %a) nounwind {
;
; AVX-LABEL: test_v8i16:
; AVX: # %bb.0:
-; AVX-NEXT: vpabsw %xmm0, %xmm0
+; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX-NEXT: vpsubw %xmm0, %xmm1, %xmm1
+; AVX-NEXT: vpmaxsw %xmm1, %xmm0, %xmm0
; AVX-NEXT: retq
;
; X86-LABEL: test_v8i16:
@@ -416,41 +497,49 @@ define <8 x i16> @test_v8i16(<8 x i16> %a) nounwind {
; X86-NEXT: pushl %ebx
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
-; X86-NEXT: pushl %eax
+; X86-NEXT: subl $8, %esp
; X86-NEXT: movzwl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movzwl {{[0-9]+}}(%esp), %ebx
; X86-NEXT: movzwl {{[0-9]+}}(%esp), %ebp
; X86-NEXT: movzwl {{[0-9]+}}(%esp), %edi
; X86-NEXT: movzwl {{[0-9]+}}(%esp), %esi
-; X86-NEXT: movzwl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl %edx, %ecx
-; X86-NEXT: negw %cx
-; X86-NEXT: cmovsw %dx, %cx
-; X86-NEXT: movw %cx, {{[-0-9]+}}(%e{{[sb]}}p) # 2-byte Spill
+; X86-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: negl %edx
+; X86-NEXT: cmpw %dx, %cx
+; X86-NEXT: cmovgl %ecx, %edx
+; X86-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NEXT: movl %esi, %ecx
-; X86-NEXT: negw %cx
-; X86-NEXT: cmovsw %si, %cx
-; X86-NEXT: movw %cx, (%esp) # 2-byte Spill
+; X86-NEXT: negl %ecx
+; X86-NEXT: cmpw %cx, %si
+; X86-NEXT: cmovgl %esi, %ecx
+; X86-NEXT: movl %ecx, (%esp) # 4-byte Spill
; X86-NEXT: movl %edi, %esi
-; X86-NEXT: negw %si
-; X86-NEXT: cmovsw %di, %si
+; X86-NEXT: negl %esi
+; X86-NEXT: cmpw %si, %di
+; X86-NEXT: cmovgl %edi, %esi
; X86-NEXT: movl %ebp, %edi
-; X86-NEXT: negw %di
-; X86-NEXT: cmovsw %bp, %di
+; X86-NEXT: negl %edi
+; X86-NEXT: cmpw %di, %bp
+; X86-NEXT: cmovgl %ebp, %edi
; X86-NEXT: movl %ebx, %ebp
-; X86-NEXT: negw %bp
-; X86-NEXT: cmovsw %bx, %bp
+; X86-NEXT: negl %ebp
+; X86-NEXT: cmpw %bp, %bx
+; X86-NEXT: cmovgl %ebx, %ebp
; X86-NEXT: movl %eax, %ebx
-; X86-NEXT: negw %bx
-; X86-NEXT: cmovsw %ax, %bx
+; X86-NEXT: negl %ebx
+; X86-NEXT: cmpw %bx, %ax
+; X86-NEXT: cmovgl %eax, %ebx
; X86-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: negw %ax
-; X86-NEXT: cmovsw %cx, %ax
+; X86-NEXT: negl %eax
+; X86-NEXT: cmpw %ax, %cx
+; X86-NEXT: cmovgl %ecx, %eax
; X86-NEXT: movzwl {{[0-9]+}}(%esp), %edx
; X86-NEXT: movl %edx, %ecx
-; X86-NEXT: negw %cx
-; X86-NEXT: cmovsw %dx, %cx
+; X86-NEXT: negl %ecx
+; X86-NEXT: cmpw %cx, %dx
+; X86-NEXT: cmovgl %edx, %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-NEXT: movw %cx, 14(%edx)
; X86-NEXT: movw %ax, 12(%edx)
@@ -458,12 +547,12 @@ define <8 x i16> @test_v8i16(<8 x i16> %a) nounwind {
; X86-NEXT: movw %bp, 8(%edx)
; X86-NEXT: movw %di, 6(%edx)
; X86-NEXT: movw %si, 4(%edx)
-; X86-NEXT: movzwl (%esp), %eax # 2-byte Folded Reload
+; X86-NEXT: movl (%esp), %eax # 4-byte Reload
; X86-NEXT: movw %ax, 2(%edx)
-; X86-NEXT: movzwl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 2-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-NEXT: movw %ax, (%edx)
; X86-NEXT: movl %edx, %eax
-; X86-NEXT: addl $4, %esp
+; X86-NEXT: addl $8, %esp
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
; X86-NEXT: popl %ebx
@@ -476,139 +565,187 @@ define <8 x i16> @test_v8i16(<8 x i16> %a) nounwind {
define <16 x i8> @test_v16i8(<16 x i8> %a) nounwind {
; SSE-LABEL: test_v16i8:
; SSE: # %bb.0:
-; SSE-NEXT: pxor %xmm1, %xmm1
+; SSE-NEXT: pxor %xmm2, %xmm2
+; SSE-NEXT: psubb %xmm0, %xmm2
+; SSE-NEXT: movdqa %xmm0, %xmm1
+; SSE-NEXT: pcmpgtb %xmm2, %xmm1
+; SSE-NEXT: pxor %xmm1, %xmm0
; SSE-NEXT: psubb %xmm0, %xmm1
-; SSE-NEXT: pminub %xmm1, %xmm0
+; SSE-NEXT: movdqa %xmm1, %xmm0
; SSE-NEXT: retq
;
; AVX-LABEL: test_v16i8:
; AVX: # %bb.0:
-; AVX-NEXT: vpabsb %xmm0, %xmm0
+; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX-NEXT: vpsubb %xmm0, %xmm1, %xmm1
+; AVX-NEXT: vpmaxsb %xmm1, %xmm0, %xmm0
; AVX-NEXT: retq
;
; X86-LABEL: test_v16i8:
; X86: # %bb.0:
+; X86-NEXT: pushl %ebp
; X86-NEXT: pushl %ebx
-; X86-NEXT: subl $12, %esp
-; X86-NEXT: movb {{[0-9]+}}(%esp), %bh
-; X86-NEXT: movb {{[0-9]+}}(%esp), %bl
-; X86-NEXT: movb {{[0-9]+}}(%esp), %dh
-; X86-NEXT: movb {{[0-9]+}}(%esp), %ch
-; X86-NEXT: movb {{[0-9]+}}(%esp), %ah
-; X86-NEXT: movb {{[0-9]+}}(%esp), %dl
-; X86-NEXT: movb {{[0-9]+}}(%esp), %cl
-; X86-NEXT: movb %cl, %al
-; X86-NEXT: sarb $7, %al
-; X86-NEXT: xorb %al, %cl
-; X86-NEXT: subb %al, %cl
-; X86-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X86-NEXT: movb %dl, %al
-; X86-NEXT: sarb $7, %al
-; X86-NEXT: xorb %al, %dl
-; X86-NEXT: subb %al, %dl
-; X86-NEXT: movb %dl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X86-NEXT: movb %ah, %al
-; X86-NEXT: sarb $7, %al
-; X86-NEXT: xorb %al, %ah
-; X86-NEXT: subb %al, %ah
-; X86-NEXT: movb %ah, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X86-NEXT: movb %ch, %al
-; X86-NEXT: sarb $7, %al
-; X86-NEXT: xorb %al, %ch
-; X86-NEXT: subb %al, %ch
-; X86-NEXT: movb %ch, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X86-NEXT: movb %dh, %al
-; X86-NEXT: sarb $7, %al
-; X86-NEXT: xorb %al, %dh
-; X86-NEXT: subb %al, %dh
-; X86-NEXT: movb %dh, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X86-NEXT: movl %ebx, %eax
-; X86-NEXT: sarb $7, %al
-; X86-NEXT: xorb %al, %bl
-; X86-NEXT: subb %al, %bl
-; X86-NEXT: movb %bl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X86-NEXT: movb %bh, %al
-; X86-NEXT: sarb $7, %al
-; X86-NEXT: xorb %al, %bh
-; X86-NEXT: subb %al, %bh
-; X86-NEXT: movb %bh, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: sarb $7, %al
-; X86-NEXT: xorb %al, %cl
-; X86-NEXT: subb %al, %cl
-; X86-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: sarb $7, %al
-; X86-NEXT: xorb %al, %cl
-; X86-NEXT: subb %al, %cl
-; X86-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
+; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %esi
+; X86-NEXT: subl $44, %esp
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: sarb $7, %al
-; X86-NEXT: xorb %al, %cl
-; X86-NEXT: subb %al, %cl
-; X86-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X86-NEXT: movb {{[0-9]+}}(%esp), %bh
-; X86-NEXT: movb %bh, %al
-; X86-NEXT: sarb $7, %al
-; X86-NEXT: xorb %al, %bh
-; X86-NEXT: subb %al, %bh
-; X86-NEXT: movb {{[0-9]+}}(%esp), %bl
-; X86-NEXT: movl %ebx, %eax
-; X86-NEXT: sarb $7, %al
-; X86-NEXT: xorb %al, %bl
-; X86-NEXT: subb %al, %bl
-; X86-NEXT: movb {{[0-9]+}}(%esp), %dh
-; X86-NEXT: movb %dh, %al
-; X86-NEXT: sarb $7, %al
-; X86-NEXT: xorb %al, %dh
-; X86-NEXT: subb %al, %dh
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %edx
+; X86-NEXT: movb {{[0-9]+}}(%esp), %ah
; X86-NEXT: movb {{[0-9]+}}(%esp), %ch
-; X86-NEXT: movb %ch, %al
-; X86-NEXT: sarb $7, %al
-; X86-NEXT: xorb %al, %ch
-; X86-NEXT: subb %al, %ch
-; X86-NEXT: movb {{[0-9]+}}(%esp), %dl
-; X86-NEXT: movl %edx, %eax
-; X86-NEXT: sarb $7, %al
-; X86-NEXT: xorb %al, %dl
-; X86-NEXT: subb %al, %dl
-; X86-NEXT: movb {{[0-9]+}}(%esp), %cl
-; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: sarb $7, %al
-; X86-NEXT: xorb %al, %cl
-; X86-NEXT: subb %al, %cl
+; X86-NEXT: movzbl %ch, %esi
+; X86-NEXT: movb %ch, %dh
+; X86-NEXT: negb %dh
+; X86-NEXT: movzbl %dh, %edi
+; X86-NEXT: cmpb %dh, %ch
+; X86-NEXT: cmovgl %esi, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movzbl %ah, %esi
+; X86-NEXT: movb %ah, %ch
+; X86-NEXT: negb %ch
+; X86-NEXT: movzbl %ch, %edi
+; X86-NEXT: cmpb %ch, %ah
+; X86-NEXT: cmovgl %esi, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movzbl %dl, %esi
+; X86-NEXT: movb %dl, %ah
+; X86-NEXT: negb %ah
+; X86-NEXT: movzbl %ah, %edi
+; X86-NEXT: cmpb %ah, %dl
+; X86-NEXT: cmovgl %esi, %edi
+; X86-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movzbl %cl, %edx
+; X86-NEXT: movb %cl, %ah
+; X86-NEXT: negb %ah
+; X86-NEXT: cmpb %ah, %cl
+; X86-NEXT: movzbl %ah, %ecx
+; X86-NEXT: cmovgl %edx, %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movzbl %al, %ecx
+; X86-NEXT: movl %eax, %edx
+; X86-NEXT: negb %dl
+; X86-NEXT: cmpb %dl, %al
+; X86-NEXT: movzbl %dl, %eax
+; X86-NEXT: cmovgl %ecx, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movzbl %al, %ecx
+; X86-NEXT: movl %eax, %edx
+; X86-NEXT: negb %dl
+; X86-NEXT: cmpb %dl, %al
+; X86-NEXT: movzbl %dl, %eax
+; X86-NEXT: cmovgl %ecx, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: negb %cl
+; X86-NEXT: movzbl %al, %edx
+; X86-NEXT: cmpb %cl, %al
+; X86-NEXT: movzbl %cl, %eax
+; X86-NEXT: cmovgl %edx, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: negb %cl
+; X86-NEXT: movzbl %al, %edx
+; X86-NEXT: cmpb %cl, %al
+; X86-NEXT: movzbl %cl, %eax
+; X86-NEXT: cmovgl %edx, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: negb %cl
+; X86-NEXT: movzbl %al, %edx
+; X86-NEXT: cmpb %cl, %al
+; X86-NEXT: movzbl %cl, %eax
+; X86-NEXT: cmovgl %edx, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: negb %cl
+; X86-NEXT: movzbl %al, %edx
+; X86-NEXT: cmpb %cl, %al
+; X86-NEXT: movzbl %cl, %eax
+; X86-NEXT: cmovgl %edx, %eax
+; X86-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: negb %cl
+; X86-NEXT: movzbl %al, %edx
+; X86-NEXT: cmpb %cl, %al
+; X86-NEXT: movzbl %cl, %eax
+; X86-NEXT: cmovgl %edx, %eax
+; X86-NEXT: movl %eax, (%esp) # 4-byte Spill
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: negb %cl
+; X86-NEXT: movzbl %al, %edx
+; X86-NEXT: cmpb %cl, %al
+; X86-NEXT: movzbl %cl, %ebp
+; X86-NEXT: cmovgl %edx, %ebp
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: negb %cl
+; X86-NEXT: movzbl %al, %edx
+; X86-NEXT: cmpb %cl, %al
+; X86-NEXT: movzbl %cl, %edi
+; X86-NEXT: cmovgl %edx, %edi
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: negb %cl
+; X86-NEXT: movzbl %al, %esi
+; X86-NEXT: cmpb %cl, %al
+; X86-NEXT: movzbl %cl, %ebx
+; X86-NEXT: cmovgl %esi, %ebx
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: negb %cl
+; X86-NEXT: movzbl %al, %esi
+; X86-NEXT: cmpb %cl, %al
+; X86-NEXT: movzbl %cl, %edx
+; X86-NEXT: cmovgl %esi, %edx
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movb %al, %ah
+; X86-NEXT: negb %ah
+; X86-NEXT: movzbl %al, %esi
+; X86-NEXT: cmpb %ah, %al
+; X86-NEXT: movzbl %ah, %ecx
+; X86-NEXT: cmovgl %esi, %ecx
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movb %cl, 15(%eax)
; X86-NEXT: movb %dl, 14(%eax)
-; X86-NEXT: movb %ch, 13(%eax)
-; X86-NEXT: movb %dh, 12(%eax)
-; X86-NEXT: movb %bl, 11(%eax)
-; X86-NEXT: movb %bh, 10(%eax)
-; X86-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X86-NEXT: movb %bl, 13(%eax)
+; X86-NEXT: movl %edi, %ecx
+; X86-NEXT: movb %cl, 12(%eax)
+; X86-NEXT: movl %ebp, %ecx
+; X86-NEXT: movb %cl, 11(%eax)
+; X86-NEXT: movl (%esp), %ecx # 4-byte Reload
+; X86-NEXT: movb %cl, 10(%eax)
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: movb %cl, 9(%eax)
-; X86-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: movb %cl, 8(%eax)
-; X86-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: movb %cl, 7(%eax)
-; X86-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: movb %cl, 6(%eax)
-; X86-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: movb %cl, 5(%eax)
-; X86-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: movb %cl, 4(%eax)
-; X86-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: movb %cl, 3(%eax)
-; X86-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: movb %cl, 2(%eax)
-; X86-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: movb %cl, 1(%eax)
-; X86-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
+; X86-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
; X86-NEXT: movb %cl, (%eax)
-; X86-NEXT: addl $12, %esp
+; X86-NEXT: addl $44, %esp
+; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
; X86-NEXT: popl %ebx
+; X86-NEXT: popl %ebp
; X86-NEXT: retl $4
%r = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %a, i1 false)
ret <16 x i8> %r
@@ -723,31 +860,29 @@ define i8 @test_minsigned_i8(i8 %a0, i8 %a1) nounwind {
; X64-LABEL: test_minsigned_i8:
; X64: # %bb.0:
; X64-NEXT: movl %edi, %eax
-; X64-NEXT: sarb $7, %al
-; X64-NEXT: movl %edi, %ecx
-; X64-NEXT: xorb %al, %cl
-; X64-NEXT: subb %al, %cl
+; X64-NEXT: negb %al
+; X64-NEXT: movzbl %al, %eax
+; X64-NEXT: cmpb %al, %dil
+; X64-NEXT: cmovgl %edi, %eax
; X64-NEXT: cmpb $-128, %dil
-; X64-NEXT: movzbl %cl, %eax
; X64-NEXT: cmovel %esi, %eax
; X64-NEXT: # kill: def $al killed $al killed $eax
; X64-NEXT: retq
;
; X86-LABEL: test_minsigned_i8:
; X86: # %bb.0:
-; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: cmpb $-128, %al
-; X86-NEXT: jne .LBB17_1
-; X86-NEXT: # %bb.2:
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movzbl %cl, %edx
+; X86-NEXT: movb %cl, %ch
+; X86-NEXT: negb %ch
+; X86-NEXT: movzbl %ch, %eax
+; X86-NEXT: cmpb %ch, %cl
+; X86-NEXT: cmovgl %edx, %eax
+; X86-NEXT: cmpb $-128, %cl
+; X86-NEXT: jne .LBB17_2
+; X86-NEXT: # %bb.1:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: # kill: def $al killed $al killed $eax
-; X86-NEXT: retl
-; X86-NEXT: .LBB17_1:
-; X86-NEXT: movl %eax, %ecx
-; X86-NEXT: sarb $7, %cl
-; X86-NEXT: xorb %cl, %al
-; X86-NEXT: subb %cl, %al
-; X86-NEXT: movzbl %al, %eax
+; X86-NEXT: .LBB17_2:
; X86-NEXT: # kill: def $al killed $al killed $eax
; X86-NEXT: retl
%lim = icmp eq i8 %a0, -128
@@ -760,9 +895,10 @@ define i16 @test_minsigned_i16(i16 %a0, i16 %a1) nounwind {
; X64-LABEL: test_minsigned_i16:
; X64: # %bb.0:
; X64-NEXT: movzwl %di, %ecx
-; X64-NEXT: movl %ecx, %eax
-; X64-NEXT: negw %ax
-; X64-NEXT: cmovsw %cx, %ax
+; X64-NEXT: movl %edi, %eax
+; X64-NEXT: negl %eax
+; X64-NEXT: cmpw %ax, %cx
+; X64-NEXT: cmovgl %edi, %eax
; X64-NEXT: cmpl $32768, %ecx # imm = 0x8000
; X64-NEXT: cmovel %esi, %eax
; X64-NEXT: # kill: def $ax killed $ax killed $eax
@@ -772,13 +908,15 @@ define i16 @test_minsigned_i16(i16 %a0, i16 %a1) nounwind {
; X86: # %bb.0:
; X86-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: negw %ax
-; X86-NEXT: cmovsw %cx, %ax
+; X86-NEXT: negl %eax
+; X86-NEXT: cmpw %ax, %cx
+; X86-NEXT: cmovgl %ecx, %eax
; X86-NEXT: cmpl $32768, %ecx # imm = 0x8000
; X86-NEXT: jne .LBB18_2
; X86-NEXT: # %bb.1:
; X86-NEXT: movzwl {{[0-9]+}}(%esp), %eax
; X86-NEXT: .LBB18_2:
+; X86-NEXT: # kill: def $ax killed $ax killed $eax
; X86-NEXT: retl
%lim = icmp eq i16 %a0, -32768
%abs = tail call i16 @llvm.abs.i16(i16 %a0, i1 false)
@@ -791,7 +929,9 @@ define i32 @test_minsigned_i32(i32 %a0, i32 %a1) nounwind {
; X64: # %bb.0:
; X64-NEXT: movl %edi, %eax
; X64-NEXT: negl %eax
-; X64-NEXT: cmovsl %edi, %eax
+; X64-NEXT: cmpl %eax, %edi
+; X64-NEXT: cmovgl %edi, %eax
+; X64-NEXT: negl %edi
; X64-NEXT: cmovol %esi, %eax
; X64-NEXT: retq
;
@@ -800,8 +940,13 @@ define i32 @test_minsigned_i32(i32 %a0, i32 %a1) nounwind {
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: negl %eax
-; X86-NEXT: cmovsl %ecx, %eax
-; X86-NEXT: cmovol {{[0-9]+}}(%esp), %eax
+; X86-NEXT: cmpl %eax, %ecx
+; X86-NEXT: cmovgl %ecx, %eax
+; X86-NEXT: negl %ecx
+; X86-NEXT: jno .LBB19_2
+; X86-NEXT: # %bb.1:
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: .LBB19_2:
; X86-NEXT: retl
%lim = icmp eq i32 %a0, -2147483648
%abs = tail call i32 @llvm.abs.i32(i32 %a0, i1 false)
@@ -814,28 +959,38 @@ define i64 @test_minsigned_i64(i64 %a0, i64 %a1) nounwind {
; X64: # %bb.0:
; X64-NEXT: movq %rdi, %rax
; X64-NEXT: negq %rax
-; X64-NEXT: cmovsq %rdi, %rax
+; X64-NEXT: cmpq %rax, %rdi
+; X64-NEXT: cmovgq %rdi, %rax
+; X64-NEXT: negq %rdi
; X64-NEXT: cmovoq %rsi, %rax
; X64-NEXT: retq
;
; X86-LABEL: test_minsigned_i64:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: leal -2147483648(%edx), %ecx
-; X86-NEXT: orl %eax, %ecx
+; X86-NEXT: pushl %edi
+; X86-NEXT: pushl %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: leal -2147483648(%esi), %eax
+; X86-NEXT: orl %ecx, %eax
; X86-NEXT: jne .LBB20_1
; X86-NEXT: # %bb.2: # %select.end
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: retl
+; X86-NEXT: jmp .LBB20_3
; X86-NEXT: .LBB20_1: # %select.false.sink
-; X86-NEXT: movl %edx, %ecx
-; X86-NEXT: sarl $31, %ecx
-; X86-NEXT: xorl %ecx, %edx
-; X86-NEXT: xorl %ecx, %eax
-; X86-NEXT: subl %ecx, %eax
-; X86-NEXT: sbbl %ecx, %edx
+; X86-NEXT: xorl %edx, %edx
+; X86-NEXT: movl %ecx, %eax
+; X86-NEXT: negl %eax
+; X86-NEXT: sbbl %esi, %edx
+; X86-NEXT: cmpl %ecx, %eax
+; X86-NEXT: movl %edx, %edi
+; X86-NEXT: sbbl %esi, %edi
+; X86-NEXT: cmovll %esi, %edx
+; X86-NEXT: cmovll %ecx, %eax
+; X86-NEXT: .LBB20_3: # %select.end
+; X86-NEXT: popl %esi
+; X86-NEXT: popl %edi
; X86-NEXT: retl
%lim = icmp eq i64 %a0, -9223372036854775808
%abs = tail call i64 @llvm.abs.i64(i64 %a0, i1 false)
diff --git a/llvm/test/CodeGen/X86/apx/no-rex2-general.ll b/llvm/test/CodeGen/X86/apx/no-rex2-general.ll
index 6f31aef9aee98..762ad34acad83 100644
--- a/llvm/test/CodeGen/X86/apx/no-rex2-general.ll
+++ b/llvm/test/CodeGen/X86/apx/no-rex2-general.ll
@@ -48,16 +48,18 @@ entry:
define <2 x i64> @map2_or_vex(ptr nocapture noundef readonly %b, i64 noundef %c) nounwind {
; SSE-LABEL: map2_or_vex:
; SSE: # %bb.0: # %entry
-; SSE-NEXT: pushq %r14 # encoding: [0x41,0x56]
-; SSE-NEXT: pushq %rbx # encoding: [0x53]
-; SSE-NEXT: movq %rsi, %rbx # encoding: [0x48,0x89,0xf3]
-; SSE-NEXT: movq %rdi, %r14 # encoding: [0x49,0x89,0xfe]
+; SSE-NEXT: movq %rsi, %r16 # encoding: [0xd5,0x18,0x89,0xf0]
+; SSE-NEXT: movq %rdi, %r17 # encoding: [0xd5,0x18,0x89,0xf9]
; SSE-NEXT: #APP
; SSE-NEXT: nop # encoding: [0x90]
; SSE-NEXT: #NO_APP
-; SSE-NEXT: pabsb (%r14,%rbx,4), %xmm0 # encoding: [0x66,0x41,0x0f,0x38,0x1c,0x04,0x9e]
-; SSE-NEXT: popq %rbx # encoding: [0x5b]
-; SSE-NEXT: popq %r14 # encoding: [0x41,0x5e]
+; SSE-NEXT: movdqa (%r17,%r16,4), %xmm1 # encoding: [0x66,0xd5,0xb0,0x6f,0x0c,0x81]
+; SSE-NEXT: pxor %xmm2, %xmm2 # encoding: [0x66,0x0f,0xef,0xd2]
+; SSE-NEXT: psubb %xmm1, %xmm2 # encoding: [0x66,0x0f,0xf8,0xd1]
+; SSE-NEXT: movdqa %xmm1, %xmm0 # encoding: [0x66,0x0f,0x6f,0xc1]
+; SSE-NEXT: pcmpgtb %xmm2, %xmm0 # encoding: [0x66,0x0f,0x64,0xc2]
+; SSE-NEXT: pxor %xmm0, %xmm1 # encoding: [0x66,0x0f,0xef,0xc8]
+; SSE-NEXT: psubb %xmm1, %xmm0 # encoding: [0x66,0x0f,0xf8,0xc1]
; SSE-NEXT: retq # encoding: [0xc3]
;
; AVX-LABEL: map2_or_vex:
@@ -69,7 +71,10 @@ define <2 x i64> @map2_or_vex(ptr nocapture noundef readonly %b, i64 noundef %c)
; AVX-NEXT: #APP
; AVX-NEXT: nop # encoding: [0x90]
; AVX-NEXT: #NO_APP
-; AVX-NEXT: vpabsb (%r14,%rbx,4), %xmm0 # encoding: [0xc4,0xc2,0x79,0x1c,0x04,0x9e]
+; AVX-NEXT: vmovdqa (%r14,%rbx,4), %xmm0 # encoding: [0xc4,0xc1,0x79,0x6f,0x04,0x9e]
+; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xef,0xc9]
+; AVX-NEXT: vpsubb %xmm0, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xf8,0xc8]
+; AVX-NEXT: vpmaxsb %xmm1, %xmm0, %xmm0 # encoding: [0xc4,0xe2,0x79,0x3c,0xc1]
; AVX-NEXT: popq %rbx # encoding: [0x5b]
; AVX-NEXT: popq %r14 # encoding: [0x41,0x5e]
; AVX-NEXT: retq # encoding: [0xc3]
diff --git a/llvm/test/CodeGen/X86/avx2-intrinsics-fast-isel.ll b/llvm/test/CodeGen/X86/avx2-intrinsics-fast-isel.ll
index a93ac67905979..76ce9e92e8494 100644
--- a/llvm/test/CodeGen/X86/avx2-intrinsics-fast-isel.ll
+++ b/llvm/test/CodeGen/X86/avx2-intrinsics-fast-isel.ll
@@ -7,7 +7,9 @@
define <4 x i64> @test_mm256_abs_epi8(<4 x i64> %a0) {
; CHECK-LABEL: test_mm256_abs_epi8:
; CHECK: # %bb.0:
-; CHECK-NEXT: vpabsb %ymm0, %ymm0
+; CHECK-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; CHECK-NEXT: vpsubb %ymm0, %ymm1, %ymm1
+; CHECK-NEXT: vpmaxsb %ymm1, %ymm0, %ymm0
; CHECK-NEXT: ret{{[l|q]}}
%arg = bitcast <4 x i64> %a0 to <32 x i8>
%abs = call <32 x i8> @llvm.abs.v32i8(<32 x i8> %arg, i1 false)
@@ -19,7 +21,9 @@ declare <32 x i8> @llvm.abs.v32i8(<32 x i8>, i1) nounwind readnone
define <4 x i64> @test_mm256_abs_epi16(<4 x i64> %a0) {
; CHECK-LABEL: test_mm256_abs_epi16:
; CHECK: # %bb.0:
-; CHECK-NEXT: vpabsw %ymm0, %ymm0
+; CHECK-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; CHECK-NEXT: vpsubw %ymm0, %ymm1, %ymm1
+; CHECK-NEXT: vpmaxsw %ymm1, %ymm0, %ymm0
; CHECK-NEXT: ret{{[l|q]}}
%arg = bitcast <4 x i64> %a0 to <16 x i16>
%abs = call <16 x i16> @llvm.abs.v16i16(<16 x i16> %arg, i1 false)
@@ -31,7 +35,9 @@ declare <16 x i16> @llvm.abs.v16i16(<16 x i16>, i1) nounwind readnone
define <4 x i64> @test_mm256_abs_epi32(<4 x i64> %a0) {
; CHECK-LABEL: test_mm256_abs_epi32:
; CHECK: # %bb.0:
-; CHECK-NEXT: vpabsd %ymm0, %ymm0
+; CHECK-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; CHECK-NEXT: vpsubd %ymm0, %ymm1, %ymm1
+; CHECK-NEXT: vpmaxsd %ymm1, %ymm0, %ymm0
; CHECK-NEXT: ret{{[l|q]}}
%arg = bitcast <4 x i64> %a0 to <8 x i32>
%abs = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %arg, i1 false)
diff --git a/llvm/test/CodeGen/X86/avx2-intrinsics-x86-upgrade.ll b/llvm/test/CodeGen/X86/avx2-intrinsics-x86-upgrade.ll
index 0cbaa4f63a74c..1396f53ed12f2 100644
--- a/llvm/test/CodeGen/X86/avx2-intrinsics-x86-upgrade.ll
+++ b/llvm/test/CodeGen/X86/avx2-intrinsics-x86-upgrade.ll
@@ -532,7 +532,9 @@ declare <8 x i32> @llvm.x86.avx2.pminu.d(<8 x i32>, <8 x i32>) nounwind readnone
define <32 x i8> @test_x86_avx2_pabs_b(<32 x i8> %a0) {
; CHECK-LABEL: test_x86_avx2_pabs_b:
; CHECK: ## %bb.0:
-; CHECK-NEXT: vpabsb %ymm0, %ymm0
+; CHECK-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; CHECK-NEXT: vpsubb %ymm0, %ymm1, %ymm1
+; CHECK-NEXT: vpmaxsb %ymm1, %ymm0, %ymm0
; CHECK-NEXT: ret{{[l|q]}}
%res = call <32 x i8> @llvm.x86.avx2.pabs.b(<32 x i8> %a0) ; <<32 x i8>> [#uses=1]
ret <32 x i8> %res
@@ -542,7 +544,9 @@ declare <32 x i8> @llvm.x86.avx2.pabs.b(<32 x i8>) nounwind readnone
define <8 x i32> @test_x86_avx2_pabs_d(<8 x i32> %a0) {
; CHECK-LABEL: test_x86_avx2_pabs_d:
; CHECK: ## %bb.0:
-; CHECK-NEXT: vpabsd %ymm0, %ymm0
+; CHECK-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; CHECK-NEXT: vpsubd %ymm0, %ymm1, %ymm1
+; CHECK-NEXT: vpmaxsd %ymm1, %ymm0, %ymm0
; CHECK-NEXT: ret{{[l|q]}}
%res = call <8 x i32> @llvm.x86.avx2.pabs.d(<8 x i32> %a0) ; <<8 x i32>> [#uses=1]
ret <8 x i32> %res
@@ -553,7 +557,9 @@ declare <8 x i32> @llvm.x86.avx2.pabs.d(<8 x i32>) nounwind readnone
define <16 x i16> @test_x86_avx2_pabs_w(<16 x i16> %a0) {
; CHECK-LABEL: test_x86_avx2_pabs_w:
; CHECK: ## %bb.0:
-; CHECK-NEXT: vpabsw %ymm0, %ymm0
+; CHECK-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; CHECK-NEXT: vpsubw %ymm0, %ymm1, %ymm1
+; CHECK-NEXT: vpmaxsw %ymm1, %ymm0, %ymm0
; CHECK-NEXT: ret{{[l|q]}}
%res = call <16 x i16> @llvm.x86.avx2.pabs.w(<16 x i16> %a0) ; <<16 x i16>> [#uses=1]
ret <16 x i16> %res
diff --git a/llvm/test/CodeGen/X86/avx512-intrinsics-upgrade.ll b/llvm/test/CodeGen/X86/avx512-intrinsics-upgrade.ll
index 5a0a9b7c3c25c..874a89f133c39 100644
--- a/llvm/test/CodeGen/X86/avx512-intrinsics-upgrade.ll
+++ b/llvm/test/CodeGen/X86/avx512-intrinsics-upgrade.ll
@@ -6721,7 +6721,9 @@ declare <16 x i32> @llvm.x86.avx512.mask.pabs.d.512(<16 x i32>, <16 x i32>, i16)
define <16 x i32>@test_int_x86_avx512_pabs_d_512(<16 x i32> %x0, <16 x i32> %x1) {
; CHECK-LABEL: test_int_x86_avx512_pabs_d_512:
; CHECK: ## %bb.0:
-; CHECK-NEXT: vpabsd %zmm0, %zmm0 ## encoding: [0x62,0xf2,0x7d,0x48,0x1e,0xc0]
+; CHECK-NEXT: vpxor %xmm1, %xmm1, %xmm1 ## encoding: [0xc5,0xf1,0xef,0xc9]
+; CHECK-NEXT: vpsubd %zmm0, %zmm1, %zmm1 ## encoding: [0x62,0xf1,0x75,0x48,0xfa,0xc8]
+; CHECK-NEXT: vpmaxsd %zmm1, %zmm0, %zmm0 ## encoding: [0x62,0xf2,0x7d,0x48,0x3d,0xc1]
; CHECK-NEXT: ret{{[l|q]}} ## encoding: [0xc3]
%res = call <16 x i32> @llvm.x86.avx512.mask.pabs.d.512(<16 x i32> %x0, <16 x i32> %x1, i16 -1)
ret <16 x i32> %res
@@ -6730,15 +6732,19 @@ define <16 x i32>@test_int_x86_avx512_pabs_d_512(<16 x i32> %x0, <16 x i32> %x1)
define <16 x i32>@test_int_x86_avx512_mask_pabs_d_512(<16 x i32> %x0, <16 x i32> %x1, i16 %x2) {
; X86-LABEL: test_int_x86_avx512_mask_pabs_d_512:
; X86: ## %bb.0:
+; X86-NEXT: vpxor %xmm2, %xmm2, %xmm2 ## encoding: [0xc5,0xe9,0xef,0xd2]
+; X86-NEXT: vpsubd %zmm0, %zmm2, %zmm2 ## encoding: [0x62,0xf1,0x6d,0x48,0xfa,0xd0]
; X86-NEXT: kmovw {{[0-9]+}}(%esp), %k1 ## encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
-; X86-NEXT: vpabsd %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf2,0x7d,0x49,0x1e,0xc8]
+; X86-NEXT: vpmaxsd %zmm2, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf2,0x7d,0x49,0x3d,0xca]
; X86-NEXT: vmovdqa64 %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
; X86-NEXT: retl ## encoding: [0xc3]
;
; X64-LABEL: test_int_x86_avx512_mask_pabs_d_512:
; X64: ## %bb.0:
+; X64-NEXT: vpxor %xmm2, %xmm2, %xmm2 ## encoding: [0xc5,0xe9,0xef,0xd2]
+; X64-NEXT: vpsubd %zmm0, %zmm2, %zmm2 ## encoding: [0x62,0xf1,0x6d,0x48,0xfa,0xd0]
; X64-NEXT: kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT: vpabsd %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf2,0x7d,0x49,0x1e,0xc8]
+; X64-NEXT: vpmaxsd %zmm2, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf2,0x7d,0x49,0x3d,0xca]
; X64-NEXT: vmovdqa64 %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
; X64-NEXT: retq ## encoding: [0xc3]
%res = call <16 x i32> @llvm.x86.avx512.mask.pabs.d.512(<16 x i32> %x0, <16 x i32> %x1, i16 %x2)
@@ -6750,7 +6756,9 @@ declare <8 x i64> @llvm.x86.avx512.mask.pabs.q.512(<8 x i64>, <8 x i64>, i8)
define <8 x i64>@test_int_x86_avx512_pabs_q_512(<8 x i64> %x0, <8 x i64> %x1, i8 %x2) {
; CHECK-LABEL: test_int_x86_avx512_pabs_q_512:
; CHECK: ## %bb.0:
-; CHECK-NEXT: vpabsq %zmm0, %zmm0 ## encoding: [0x62,0xf2,0xfd,0x48,0x1f,0xc0]
+; CHECK-NEXT: vpxor %xmm1, %xmm1, %xmm1 ## encoding: [0xc5,0xf1,0xef,0xc9]
+; CHECK-NEXT: vpsubq %zmm0, %zmm1, %zmm1 ## encoding: [0x62,0xf1,0xf5,0x48,0xfb,0xc8]
+; CHECK-NEXT: vpmaxsq %zmm1, %zmm0, %zmm0 ## encoding: [0x62,0xf2,0xfd,0x48,0x3d,0xc1]
; CHECK-NEXT: ret{{[l|q]}} ## encoding: [0xc3]
%res = call <8 x i64> @llvm.x86.avx512.mask.pabs.q.512(<8 x i64> %x0, <8 x i64> %x1, i8 -1)
ret <8 x i64> %res
@@ -6759,16 +6767,20 @@ define <8 x i64>@test_int_x86_avx512_pabs_q_512(<8 x i64> %x0, <8 x i64> %x1, i8
define <8 x i64>@test_int_x86_avx512_mask_pabs_q_512(<8 x i64> %x0, <8 x i64> %x1, i8 %x2) {
; X86-LABEL: test_int_x86_avx512_mask_pabs_q_512:
; X86: ## %bb.0:
+; X86-NEXT: vpxor %xmm2, %xmm2, %xmm2 ## encoding: [0xc5,0xe9,0xef,0xd2]
+; X86-NEXT: vpsubq %zmm0, %zmm2, %zmm2 ## encoding: [0x62,0xf1,0xed,0x48,0xfb,0xd0]
; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax ## encoding: [0x0f,0xb6,0x44,0x24,0x04]
; X86-NEXT: kmovw %eax, %k1 ## encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT: vpabsq %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf2,0xfd,0x49,0x1f,0xc8]
+; X86-NEXT: vpmaxsq %zmm2, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf2,0xfd,0x49,0x3d,0xca]
; X86-NEXT: vmovdqa64 %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
; X86-NEXT: retl ## encoding: [0xc3]
;
; X64-LABEL: test_int_x86_avx512_mask_pabs_q_512:
; X64: ## %bb.0:
+; X64-NEXT: vpxor %xmm2, %xmm2, %xmm2 ## encoding: [0xc5,0xe9,0xef,0xd2]
+; X64-NEXT: vpsubq %zmm0, %zmm2, %zmm2 ## encoding: [0x62,0xf1,0xed,0x48,0xfb,0xd0]
; X64-NEXT: kmovw %edi, %k1 ## encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT: vpabsq %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf2,0xfd,0x49,0x1f,0xc8]
+; X64-NEXT: vpmaxsq %zmm2, %zmm0, %zmm1 {%k1} ## encoding: [0x62,0xf2,0xfd,0x49,0x3d,0xca]
; X64-NEXT: vmovdqa64 %zmm1, %zmm0 ## encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
; X64-NEXT: retq ## encoding: [0xc3]
%res = call <8 x i64> @llvm.x86.avx512.mask.pabs.q.512(<8 x i64> %x0, <8 x i64> %x1, i8 %x2)
diff --git a/llvm/test/CodeGen/X86/avx512bw-intrinsics-upgrade.ll b/llvm/test/CodeGen/X86/avx512bw-intrinsics-upgrade.ll
index 51ffeca52a665..5dd95e21accf5 100644
--- a/llvm/test/CodeGen/X86/avx512bw-intrinsics-upgrade.ll
+++ b/llvm/test/CodeGen/X86/avx512bw-intrinsics-upgrade.ll
@@ -2533,7 +2533,9 @@ declare <32 x i16> @llvm.x86.avx512.mask.pabs.w.512(<32 x i16>, <32 x i16>, i32)
define <32 x i16> @test_int_x86_avx512_pabs_w_512(<32 x i16> %x0, <32 x i16> %x1) nounwind {
; CHECK-LABEL: test_int_x86_avx512_pabs_w_512:
; CHECK: # %bb.0:
-; CHECK-NEXT: vpabsw %zmm0, %zmm0 # encoding: [0x62,0xf2,0x7d,0x48,0x1d,0xc0]
+; CHECK-NEXT: vpxor %xmm1, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xef,0xc9]
+; CHECK-NEXT: vpsubw %zmm0, %zmm1, %zmm1 # encoding: [0x62,0xf1,0x75,0x48,0xf9,0xc8]
+; CHECK-NEXT: vpmaxsw %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf1,0x7d,0x48,0xee,0xc1]
; CHECK-NEXT: ret{{[l|q]}} # encoding: [0xc3]
%res = call <32 x i16> @llvm.x86.avx512.mask.pabs.w.512(<32 x i16> %x0, <32 x i16> %x1, i32 -1)
ret <32 x i16> %res
@@ -2542,15 +2544,19 @@ define <32 x i16> @test_int_x86_avx512_pabs_w_512(<32 x i16> %x0, <32 x i16> %x1
define <32 x i16> @test_int_x86_avx512_mask_pabs_w_512(<32 x i16> %x0, <32 x i16> %x1, i32 %x2) nounwind {
; X86-LABEL: test_int_x86_avx512_mask_pabs_w_512:
; X86: # %bb.0:
+; X86-NEXT: vpxor %xmm2, %xmm2, %xmm2 # encoding: [0xc5,0xe9,0xef,0xd2]
+; X86-NEXT: vpsubw %zmm0, %zmm2, %zmm2 # encoding: [0x62,0xf1,0x6d,0x48,0xf9,0xd0]
; X86-NEXT: kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
-; X86-NEXT: vpabsw %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x49,0x1d,0xc8]
+; X86-NEXT: vpmaxsw %zmm2, %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x49,0xee,0xca]
; X86-NEXT: vmovdqa64 %zmm1, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
; X86-NEXT: retl # encoding: [0xc3]
;
; X64-LABEL: test_int_x86_avx512_mask_pabs_w_512:
; X64: # %bb.0:
+; X64-NEXT: vpxor %xmm2, %xmm2, %xmm2 # encoding: [0xc5,0xe9,0xef,0xd2]
+; X64-NEXT: vpsubw %zmm0, %zmm2, %zmm2 # encoding: [0x62,0xf1,0x6d,0x48,0xf9,0xd0]
; X64-NEXT: kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT: vpabsw %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x49,0x1d,0xc8]
+; X64-NEXT: vpmaxsw %zmm2, %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x49,0xee,0xca]
; X64-NEXT: vmovdqa64 %zmm1, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
; X64-NEXT: retq # encoding: [0xc3]
%res = call <32 x i16> @llvm.x86.avx512.mask.pabs.w.512(<32 x i16> %x0, <32 x i16> %x1, i32 %x2)
@@ -2562,7 +2568,9 @@ declare <64 x i8> @llvm.x86.avx512.mask.pabs.b.512(<64 x i8>, <64 x i8>, i64)
define <64 x i8> @test_int_x86_avx512_pabs_b_512(<64 x i8> %x0, <64 x i8> %x1) nounwind {
; CHECK-LABEL: test_int_x86_avx512_pabs_b_512:
; CHECK: # %bb.0:
-; CHECK-NEXT: vpabsb %zmm0, %zmm0 # encoding: [0x62,0xf2,0x7d,0x48,0x1c,0xc0]
+; CHECK-NEXT: vpxor %xmm1, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xef,0xc9]
+; CHECK-NEXT: vpsubb %zmm0, %zmm1, %zmm1 # encoding: [0x62,0xf1,0x75,0x48,0xf8,0xc8]
+; CHECK-NEXT: vpmaxsb %zmm1, %zmm0, %zmm0 # encoding: [0x62,0xf2,0x7d,0x48,0x3c,0xc1]
; CHECK-NEXT: ret{{[l|q]}} # encoding: [0xc3]
%res = call <64 x i8> @llvm.x86.avx512.mask.pabs.b.512(<64 x i8> %x0, <64 x i8> %x1, i64 -1)
ret <64 x i8> %res
@@ -2572,14 +2580,18 @@ define <64 x i8> @test_int_x86_avx512_mask_pabs_b_512(<64 x i8> %x0, <64 x i8> %
; X86-LABEL: test_int_x86_avx512_mask_pabs_b_512:
; X86: # %bb.0:
; X86-NEXT: kmovq {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf8,0x90,0x4c,0x24,0x04]
-; X86-NEXT: vpabsb %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x49,0x1c,0xc8]
+; X86-NEXT: vpxor %xmm2, %xmm2, %xmm2 # encoding: [0xc5,0xe9,0xef,0xd2]
+; X86-NEXT: vpsubb %zmm0, %zmm2, %zmm2 # encoding: [0x62,0xf1,0x6d,0x48,0xf8,0xd0]
+; X86-NEXT: vpmaxsb %zmm2, %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x49,0x3c,0xca]
; X86-NEXT: vmovdqa64 %zmm1, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
; X86-NEXT: retl # encoding: [0xc3]
;
; X64-LABEL: test_int_x86_avx512_mask_pabs_b_512:
; X64: # %bb.0:
+; X64-NEXT: vpxor %xmm2, %xmm2, %xmm2 # encoding: [0xc5,0xe9,0xef,0xd2]
+; X64-NEXT: vpsubb %zmm0, %zmm2, %zmm2 # encoding: [0x62,0xf1,0x6d,0x48,0xf8,0xd0]
; X64-NEXT: kmovq %rdi, %k1 # encoding: [0xc4,0xe1,0xfb,0x92,0xcf]
-; X64-NEXT: vpabsb %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x49,0x1c,0xc8]
+; X64-NEXT: vpmaxsb %zmm2, %zmm0, %zmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x49,0x3c,0xca]
; X64-NEXT: vmovdqa64 %zmm1, %zmm0 # encoding: [0x62,0xf1,0xfd,0x48,0x6f,0xc1]
; X64-NEXT: retq # encoding: [0xc3]
%res = call <64 x i8> @llvm.x86.avx512.mask.pabs.b.512(<64 x i8> %x0, <64 x i8> %x1, i64 %x2)
diff --git a/llvm/test/CodeGen/X86/avx512bwvl-intrinsics-upgrade.ll b/llvm/test/CodeGen/X86/avx512bwvl-intrinsics-upgrade.ll
index ae710cc40a522..ced70a56a8ed6 100644
--- a/llvm/test/CodeGen/X86/avx512bwvl-intrinsics-upgrade.ll
+++ b/llvm/test/CodeGen/X86/avx512bwvl-intrinsics-upgrade.ll
@@ -5911,7 +5911,9 @@ declare <16 x i8> @llvm.x86.avx512.mask.pabs.b.128(<16 x i8>, <16 x i8>, i16)
define <16 x i8>@test_int_x86_avx512_pabs_b_128(<16 x i8> %x0, <16 x i8> %x1) {
; CHECK-LABEL: test_int_x86_avx512_pabs_b_128:
; CHECK: # %bb.0:
-; CHECK-NEXT: vpabsb %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x1c,0xc0]
+; CHECK-NEXT: vpxor %xmm1, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xef,0xc9]
+; CHECK-NEXT: vpsubb %xmm0, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xf8,0xc8]
+; CHECK-NEXT: vpmaxsb %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x3c,0xc1]
; CHECK-NEXT: ret{{[l|q]}} # encoding: [0xc3]
%res = call <16 x i8> @llvm.x86.avx512.mask.pabs.b.128(<16 x i8> %x0, <16 x i8> %x1, i16 -1)
ret <16 x i8> %res
@@ -5920,15 +5922,19 @@ define <16 x i8>@test_int_x86_avx512_pabs_b_128(<16 x i8> %x0, <16 x i8> %x1) {
define <16 x i8>@test_int_x86_avx512_mask_pabs_b_128(<16 x i8> %x0, <16 x i8> %x1, i16 %x2) {
; X86-LABEL: test_int_x86_avx512_mask_pabs_b_128:
; X86: # %bb.0:
+; X86-NEXT: vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
+; X86-NEXT: vpsubb %xmm0, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xf8,0xd0]
; X86-NEXT: kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
-; X86-NEXT: vpabsb %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x1c,0xc8]
+; X86-NEXT: vpmaxsb %xmm2, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x3c,0xca]
; X86-NEXT: vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
; X86-NEXT: retl # encoding: [0xc3]
;
; X64-LABEL: test_int_x86_avx512_mask_pabs_b_128:
; X64: # %bb.0:
+; X64-NEXT: vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
+; X64-NEXT: vpsubb %xmm0, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xf8,0xd0]
; X64-NEXT: kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT: vpabsb %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x1c,0xc8]
+; X64-NEXT: vpmaxsb %xmm2, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x3c,0xca]
; X64-NEXT: vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
; X64-NEXT: retq # encoding: [0xc3]
%res = call <16 x i8> @llvm.x86.avx512.mask.pabs.b.128(<16 x i8> %x0, <16 x i8> %x1, i16 %x2)
@@ -5969,7 +5975,9 @@ declare <32 x i8> @llvm.x86.avx512.mask.pabs.b.256(<32 x i8>, <32 x i8>, i32)
define <32 x i8>@test_int_x86_avx512_pabs_b_256(<32 x i8> %x0, <32 x i8> %x1) {
; CHECK-LABEL: test_int_x86_avx512_pabs_b_256:
; CHECK: # %bb.0:
-; CHECK-NEXT: vpabsb %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x1c,0xc0]
+; CHECK-NEXT: vpxor %xmm1, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xef,0xc9]
+; CHECK-NEXT: vpsubb %ymm0, %ymm1, %ymm1 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xf8,0xc8]
+; CHECK-NEXT: vpmaxsb %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x3c,0xc1]
; CHECK-NEXT: ret{{[l|q]}} # encoding: [0xc3]
%res = call <32 x i8> @llvm.x86.avx512.mask.pabs.b.256(<32 x i8> %x0, <32 x i8> %x1, i32 -1)
ret <32 x i8> %res
@@ -5978,15 +5986,19 @@ define <32 x i8>@test_int_x86_avx512_pabs_b_256(<32 x i8> %x0, <32 x i8> %x1) {
define <32 x i8>@test_int_x86_avx512_mask_pabs_b_256(<32 x i8> %x0, <32 x i8> %x1, i32 %x2) {
; X86-LABEL: test_int_x86_avx512_mask_pabs_b_256:
; X86: # %bb.0:
+; X86-NEXT: vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
+; X86-NEXT: vpsubb %ymm0, %ymm2, %ymm2 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xf8,0xd0]
; X86-NEXT: kmovd {{[0-9]+}}(%esp), %k1 # encoding: [0xc4,0xe1,0xf9,0x90,0x4c,0x24,0x04]
-; X86-NEXT: vpabsb %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x1c,0xc8]
+; X86-NEXT: vpmaxsb %ymm2, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x3c,0xca]
; X86-NEXT: vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
; X86-NEXT: retl # encoding: [0xc3]
;
; X64-LABEL: test_int_x86_avx512_mask_pabs_b_256:
; X64: # %bb.0:
+; X64-NEXT: vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
+; X64-NEXT: vpsubb %ymm0, %ymm2, %ymm2 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xf8,0xd0]
; X64-NEXT: kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT: vpabsb %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x1c,0xc8]
+; X64-NEXT: vpmaxsb %ymm2, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x3c,0xca]
; X64-NEXT: vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
; X64-NEXT: retq # encoding: [0xc3]
%res = call <32 x i8> @llvm.x86.avx512.mask.pabs.b.256(<32 x i8> %x0, <32 x i8> %x1, i32 %x2)
@@ -6028,7 +6040,9 @@ declare <8 x i16> @llvm.x86.avx512.mask.pabs.w.128(<8 x i16>, <8 x i16>, i8)
define <8 x i16>@test_int_x86_avx512_pabs_w_128(<8 x i16> %x0, <8 x i16> %x1) {
; CHECK-LABEL: test_int_x86_avx512_pabs_w_128:
; CHECK: # %bb.0:
-; CHECK-NEXT: vpabsw %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x1d,0xc0]
+; CHECK-NEXT: vpxor %xmm1, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xef,0xc9]
+; CHECK-NEXT: vpsubw %xmm0, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xf9,0xc8]
+; CHECK-NEXT: vpmaxsw %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0xee,0xc1]
; CHECK-NEXT: ret{{[l|q]}} # encoding: [0xc3]
%res = call <8 x i16> @llvm.x86.avx512.mask.pabs.w.128(<8 x i16> %x0, <8 x i16> %x1, i8 -1)
ret <8 x i16> %res
@@ -6037,16 +6051,20 @@ define <8 x i16>@test_int_x86_avx512_pabs_w_128(<8 x i16> %x0, <8 x i16> %x1) {
define <8 x i16>@test_int_x86_avx512_mask_pabs_w_128(<8 x i16> %x0, <8 x i16> %x1, i8 %x2) {
; X86-LABEL: test_int_x86_avx512_mask_pabs_w_128:
; X86: # %bb.0:
+; X86-NEXT: vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
+; X86-NEXT: vpsubw %xmm0, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xf9,0xd0]
; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
; X86-NEXT: kmovd %eax, %k1 # encoding: [0xc5,0xfb,0x92,0xc8]
-; X86-NEXT: vpabsw %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x1d,0xc8]
+; X86-NEXT: vpmaxsw %xmm2, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xee,0xca]
; X86-NEXT: vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
; X86-NEXT: retl # encoding: [0xc3]
;
; X64-LABEL: test_int_x86_avx512_mask_pabs_w_128:
; X64: # %bb.0:
+; X64-NEXT: vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
+; X64-NEXT: vpsubw %xmm0, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xf9,0xd0]
; X64-NEXT: kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT: vpabsw %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x1d,0xc8]
+; X64-NEXT: vpmaxsw %xmm2, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x09,0xee,0xca]
; X64-NEXT: vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
; X64-NEXT: retq # encoding: [0xc3]
%res = call <8 x i16> @llvm.x86.avx512.mask.pabs.w.128(<8 x i16> %x0, <8 x i16> %x1, i8 %x2)
@@ -6087,7 +6105,9 @@ declare <16 x i16> @llvm.x86.avx512.mask.pabs.w.256(<16 x i16>, <16 x i16>, i16)
define <16 x i16>@test_int_x86_avx512_pabs_w_256(<16 x i16> %x0, <16 x i16> %x1) {
; CHECK-LABEL: test_int_x86_avx512_pabs_w_256:
; CHECK: # %bb.0:
-; CHECK-NEXT: vpabsw %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x1d,0xc0]
+; CHECK-NEXT: vpxor %xmm1, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xef,0xc9]
+; CHECK-NEXT: vpsubw %ymm0, %ymm1, %ymm1 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xf9,0xc8]
+; CHECK-NEXT: vpmaxsw %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0xee,0xc1]
; CHECK-NEXT: ret{{[l|q]}} # encoding: [0xc3]
%res = call <16 x i16> @llvm.x86.avx512.mask.pabs.w.256(<16 x i16> %x0, <16 x i16> %x1, i16 -1)
ret <16 x i16> %res
@@ -6096,15 +6116,19 @@ define <16 x i16>@test_int_x86_avx512_pabs_w_256(<16 x i16> %x0, <16 x i16> %x1)
define <16 x i16>@test_int_x86_avx512_mask_pabs_w_256(<16 x i16> %x0, <16 x i16> %x1, i16 %x2) {
; X86-LABEL: test_int_x86_avx512_mask_pabs_w_256:
; X86: # %bb.0:
+; X86-NEXT: vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
+; X86-NEXT: vpsubw %ymm0, %ymm2, %ymm2 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xf9,0xd0]
; X86-NEXT: kmovw {{[0-9]+}}(%esp), %k1 # encoding: [0xc5,0xf8,0x90,0x4c,0x24,0x04]
-; X86-NEXT: vpabsw %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x1d,0xc8]
+; X86-NEXT: vpmaxsw %ymm2, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xee,0xca]
; X86-NEXT: vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
; X86-NEXT: retl # encoding: [0xc3]
;
; X64-LABEL: test_int_x86_avx512_mask_pabs_w_256:
; X64: # %bb.0:
+; X64-NEXT: vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
+; X64-NEXT: vpsubw %ymm0, %ymm2, %ymm2 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xf9,0xd0]
; X64-NEXT: kmovd %edi, %k1 # encoding: [0xc5,0xfb,0x92,0xcf]
-; X64-NEXT: vpabsw %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x1d,0xc8]
+; X64-NEXT: vpmaxsw %ymm2, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf1,0x7d,0x29,0xee,0xca]
; X64-NEXT: vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
; X64-NEXT: retq # encoding: [0xc3]
%res = call <16 x i16> @llvm.x86.avx512.mask.pabs.w.256(<16 x i16> %x0, <16 x i16> %x1, i16 %x2)
diff --git a/llvm/test/CodeGen/X86/avx512vl-intrinsics-upgrade.ll b/llvm/test/CodeGen/X86/avx512vl-intrinsics-upgrade.ll
index c8504ed7151fb..5320cc477c72c 100644
--- a/llvm/test/CodeGen/X86/avx512vl-intrinsics-upgrade.ll
+++ b/llvm/test/CodeGen/X86/avx512vl-intrinsics-upgrade.ll
@@ -10600,7 +10600,9 @@ declare <2 x i64> @llvm.x86.avx512.mask.pabs.q.128(<2 x i64>, <2 x i64>, i8)
define <2 x i64>@test_int_x86_avx512_pabs_q_128(<2 x i64> %x0, <2 x i64> %x1) {
; CHECK-LABEL: test_int_x86_avx512_pabs_q_128:
; CHECK: # %bb.0:
-; CHECK-NEXT: vpabsq %xmm0, %xmm0 # encoding: [0x62,0xf2,0xfd,0x08,0x1f,0xc0]
+; CHECK-NEXT: vpxor %xmm1, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xef,0xc9]
+; CHECK-NEXT: vpsubq %xmm0, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfb,0xc8]
+; CHECK-NEXT: vpmaxsq %xmm1, %xmm0, %xmm0 # encoding: [0x62,0xf2,0xfd,0x08,0x3d,0xc1]
; CHECK-NEXT: ret{{[l|q]}} # encoding: [0xc3]
%res = call <2 x i64> @llvm.x86.avx512.mask.pabs.q.128(<2 x i64> %x0, <2 x i64> %x1, i8 -1)
ret <2 x i64> %res
@@ -10609,16 +10611,20 @@ define <2 x i64>@test_int_x86_avx512_pabs_q_128(<2 x i64> %x0, <2 x i64> %x1) {
define <2 x i64>@test_int_x86_avx512_mask_pabs_q_128(<2 x i64> %x0, <2 x i64> %x1, i8 %x2) {
; X86-LABEL: test_int_x86_avx512_mask_pabs_q_128:
; X86: # %bb.0:
+; X86-NEXT: vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
+; X86-NEXT: vpsubq %xmm0, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfb,0xd0]
; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
; X86-NEXT: kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT: vpabsq %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x1f,0xc8]
+; X86-NEXT: vpmaxsq %xmm2, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x3d,0xca]
; X86-NEXT: vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
; X86-NEXT: retl # encoding: [0xc3]
;
; X64-LABEL: test_int_x86_avx512_mask_pabs_q_128:
; X64: # %bb.0:
+; X64-NEXT: vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
+; X64-NEXT: vpsubq %xmm0, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfb,0xd0]
; X64-NEXT: kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT: vpabsq %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x1f,0xc8]
+; X64-NEXT: vpmaxsq %xmm2, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x09,0x3d,0xca]
; X64-NEXT: vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
; X64-NEXT: retq # encoding: [0xc3]
%res = call <2 x i64> @llvm.x86.avx512.mask.pabs.q.128(<2 x i64> %x0, <2 x i64> %x1, i8 %x2)
@@ -10630,7 +10636,9 @@ declare <4 x i64> @llvm.x86.avx512.mask.pabs.q.256(<4 x i64>, <4 x i64>, i8)
define <4 x i64>@test_int_x86_avx512_pabs_q_256(<4 x i64> %x0, <4 x i64> %x1) {
; CHECK-LABEL: test_int_x86_avx512_pabs_q_256:
; CHECK: # %bb.0:
-; CHECK-NEXT: vpabsq %ymm0, %ymm0 # encoding: [0x62,0xf2,0xfd,0x28,0x1f,0xc0]
+; CHECK-NEXT: vpxor %xmm1, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xef,0xc9]
+; CHECK-NEXT: vpsubq %ymm0, %ymm1, %ymm1 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xfb,0xc8]
+; CHECK-NEXT: vpmaxsq %ymm1, %ymm0, %ymm0 # encoding: [0x62,0xf2,0xfd,0x28,0x3d,0xc1]
; CHECK-NEXT: ret{{[l|q]}} # encoding: [0xc3]
%res = call <4 x i64> @llvm.x86.avx512.mask.pabs.q.256(<4 x i64> %x0, <4 x i64> %x1, i8 -1)
ret <4 x i64> %res
@@ -10639,16 +10647,20 @@ define <4 x i64>@test_int_x86_avx512_pabs_q_256(<4 x i64> %x0, <4 x i64> %x1) {
define <4 x i64>@test_int_x86_avx512_mask_pabs_q_256(<4 x i64> %x0, <4 x i64> %x1, i8 %x2) {
; X86-LABEL: test_int_x86_avx512_mask_pabs_q_256:
; X86: # %bb.0:
+; X86-NEXT: vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
+; X86-NEXT: vpsubq %ymm0, %ymm2, %ymm2 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfb,0xd0]
; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
; X86-NEXT: kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT: vpabsq %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x1f,0xc8]
+; X86-NEXT: vpmaxsq %ymm2, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x3d,0xca]
; X86-NEXT: vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
; X86-NEXT: retl # encoding: [0xc3]
;
; X64-LABEL: test_int_x86_avx512_mask_pabs_q_256:
; X64: # %bb.0:
+; X64-NEXT: vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
+; X64-NEXT: vpsubq %ymm0, %ymm2, %ymm2 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfb,0xd0]
; X64-NEXT: kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT: vpabsq %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x1f,0xc8]
+; X64-NEXT: vpmaxsq %ymm2, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0xfd,0x29,0x3d,0xca]
; X64-NEXT: vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
; X64-NEXT: retq # encoding: [0xc3]
%res = call <4 x i64> @llvm.x86.avx512.mask.pabs.q.256(<4 x i64> %x0, <4 x i64> %x1, i8 %x2)
@@ -10660,7 +10672,9 @@ declare <4 x i32> @llvm.x86.avx512.mask.pabs.d.128(<4 x i32>, <4 x i32>, i8)
define <4 x i32>@test_int_x86_avx512_pabs_d_128(<4 x i32> %x0, <4 x i32> %x1) {
; CHECK-LABEL: test_int_x86_avx512_pabs_d_128:
; CHECK: # %bb.0:
-; CHECK-NEXT: vpabsd %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x1e,0xc0]
+; CHECK-NEXT: vpxor %xmm1, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xef,0xc9]
+; CHECK-NEXT: vpsubd %xmm0, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfa,0xc8]
+; CHECK-NEXT: vpmaxsd %xmm1, %xmm0, %xmm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x3d,0xc1]
; CHECK-NEXT: ret{{[l|q]}} # encoding: [0xc3]
%res = call <4 x i32> @llvm.x86.avx512.mask.pabs.d.128(<4 x i32> %x0, <4 x i32> %x1, i8 -1)
ret <4 x i32> %res
@@ -10669,16 +10683,20 @@ define <4 x i32>@test_int_x86_avx512_pabs_d_128(<4 x i32> %x0, <4 x i32> %x1) {
define <4 x i32>@test_int_x86_avx512_mask_pabs_d_128(<4 x i32> %x0, <4 x i32> %x1, i8 %x2) {
; X86-LABEL: test_int_x86_avx512_mask_pabs_d_128:
; X86: # %bb.0:
+; X86-NEXT: vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
+; X86-NEXT: vpsubd %xmm0, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfa,0xd0]
; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
; X86-NEXT: kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT: vpabsd %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x1e,0xc8]
+; X86-NEXT: vpmaxsd %xmm2, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x3d,0xca]
; X86-NEXT: vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
; X86-NEXT: retl # encoding: [0xc3]
;
; X64-LABEL: test_int_x86_avx512_mask_pabs_d_128:
; X64: # %bb.0:
+; X64-NEXT: vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
+; X64-NEXT: vpsubd %xmm0, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xfa,0xd0]
; X64-NEXT: kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT: vpabsd %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x1e,0xc8]
+; X64-NEXT: vpmaxsd %xmm2, %xmm0, %xmm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x09,0x3d,0xca]
; X64-NEXT: vmovdqa %xmm1, %xmm0 # EVEX TO VEX Compression encoding: [0xc5,0xf9,0x6f,0xc1]
; X64-NEXT: retq # encoding: [0xc3]
%res = call <4 x i32> @llvm.x86.avx512.mask.pabs.d.128(<4 x i32> %x0, <4 x i32> %x1, i8 %x2)
@@ -10690,7 +10708,9 @@ declare <8 x i32> @llvm.x86.avx512.mask.pabs.d.256(<8 x i32>, <8 x i32>, i8)
define <8 x i32>@test_int_x86_avx512_pabs_d_256(<8 x i32> %x0, <8 x i32> %x1) {
; CHECK-LABEL: test_int_x86_avx512_pabs_d_256:
; CHECK: # %bb.0:
-; CHECK-NEXT: vpabsd %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x1e,0xc0]
+; CHECK-NEXT: vpxor %xmm1, %xmm1, %xmm1 # EVEX TO VEX Compression encoding: [0xc5,0xf1,0xef,0xc9]
+; CHECK-NEXT: vpsubd %ymm0, %ymm1, %ymm1 # EVEX TO VEX Compression encoding: [0xc5,0xf5,0xfa,0xc8]
+; CHECK-NEXT: vpmaxsd %ymm1, %ymm0, %ymm0 # EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x3d,0xc1]
; CHECK-NEXT: ret{{[l|q]}} # encoding: [0xc3]
%res = call <8 x i32> @llvm.x86.avx512.mask.pabs.d.256(<8 x i32> %x0, <8 x i32> %x1, i8 -1)
ret <8 x i32> %res
@@ -10699,16 +10719,20 @@ define <8 x i32>@test_int_x86_avx512_pabs_d_256(<8 x i32> %x0, <8 x i32> %x1) {
define <8 x i32>@test_int_x86_avx512_mask_pabs_d_256(<8 x i32> %x0, <8 x i32> %x1, i8 %x2) {
; X86-LABEL: test_int_x86_avx512_mask_pabs_d_256:
; X86: # %bb.0:
+; X86-NEXT: vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
+; X86-NEXT: vpsubd %ymm0, %ymm2, %ymm2 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfa,0xd0]
; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax # encoding: [0x0f,0xb6,0x44,0x24,0x04]
; X86-NEXT: kmovw %eax, %k1 # encoding: [0xc5,0xf8,0x92,0xc8]
-; X86-NEXT: vpabsd %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x1e,0xc8]
+; X86-NEXT: vpmaxsd %ymm2, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x3d,0xca]
; X86-NEXT: vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
; X86-NEXT: retl # encoding: [0xc3]
;
; X64-LABEL: test_int_x86_avx512_mask_pabs_d_256:
; X64: # %bb.0:
+; X64-NEXT: vpxor %xmm2, %xmm2, %xmm2 # EVEX TO VEX Compression encoding: [0xc5,0xe9,0xef,0xd2]
+; X64-NEXT: vpsubd %ymm0, %ymm2, %ymm2 # EVEX TO VEX Compression encoding: [0xc5,0xed,0xfa,0xd0]
; X64-NEXT: kmovw %edi, %k1 # encoding: [0xc5,0xf8,0x92,0xcf]
-; X64-NEXT: vpabsd %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x1e,0xc8]
+; X64-NEXT: vpmaxsd %ymm2, %ymm0, %ymm1 {%k1} # encoding: [0x62,0xf2,0x7d,0x29,0x3d,0xca]
; X64-NEXT: vmovdqa %ymm1, %ymm0 # EVEX TO VEX Compression encoding: [0xc5,0xfd,0x6f,0xc1]
; X64-NEXT: retq # encoding: [0xc3]
%res = call <8 x i32> @llvm.x86.avx512.mask.pabs.d.256(<8 x i32> %x0, <8 x i32> %x1, i8 %x2)
diff --git a/llvm/test/CodeGen/X86/build-vector-128.ll b/llvm/test/CodeGen/X86/build-vector-128.ll
index 51d3a65be5236..3a8599610d73c 100644
--- a/llvm/test/CodeGen/X86/build-vector-128.ll
+++ b/llvm/test/CodeGen/X86/build-vector-128.ll
@@ -1414,11 +1414,12 @@ define void @pr60168_buildvector_of_zeros_and_undef(<2 x i32> %x, ptr %out) {
; SSE2-32-NEXT: paddd %xmm0, %xmm0
; SSE2-32-NEXT: pxor %xmm1, %xmm1
; SSE2-32-NEXT: psubd %xmm0, %xmm1
-; SSE2-32-NEXT: movdqa %xmm1, %xmm0
-; SSE2-32-NEXT: psrad $31, %xmm0
-; SSE2-32-NEXT: pxor %xmm0, %xmm1
-; SSE2-32-NEXT: psubd %xmm0, %xmm1
-; SSE2-32-NEXT: movq %xmm1, (%eax)
+; SSE2-32-NEXT: movdqa %xmm1, %xmm2
+; SSE2-32-NEXT: pcmpgtd %xmm0, %xmm2
+; SSE2-32-NEXT: pand %xmm2, %xmm1
+; SSE2-32-NEXT: pandn %xmm0, %xmm2
+; SSE2-32-NEXT: por %xmm1, %xmm2
+; SSE2-32-NEXT: movq %xmm2, (%eax)
; SSE2-32-NEXT: retl
;
; SSE2-64-LABEL: pr60168_buildvector_of_zeros_and_undef:
@@ -1426,11 +1427,12 @@ define void @pr60168_buildvector_of_zeros_and_undef(<2 x i32> %x, ptr %out) {
; SSE2-64-NEXT: paddd %xmm0, %xmm0
; SSE2-64-NEXT: pxor %xmm1, %xmm1
; SSE2-64-NEXT: psubd %xmm0, %xmm1
-; SSE2-64-NEXT: movdqa %xmm1, %xmm0
-; SSE2-64-NEXT: psrad $31, %xmm0
-; SSE2-64-NEXT: pxor %xmm0, %xmm1
-; SSE2-64-NEXT: psubd %xmm0, %xmm1
-; SSE2-64-NEXT: movq %xmm1, (%rdi)
+; SSE2-64-NEXT: movdqa %xmm1, %xmm2
+; SSE2-64-NEXT: pcmpgtd %xmm0, %xmm2
+; SSE2-64-NEXT: pand %xmm2, %xmm1
+; SSE2-64-NEXT: pandn %xmm0, %xmm2
+; SSE2-64-NEXT: por %xmm1, %xmm2
+; SSE2-64-NEXT: movq %xmm2, (%rdi)
; SSE2-64-NEXT: retq
;
; SSE41-32-LABEL: pr60168_buildvector_of_zeros_and_undef:
@@ -1439,8 +1441,8 @@ define void @pr60168_buildvector_of_zeros_and_undef(<2 x i32> %x, ptr %out) {
; SSE41-32-NEXT: paddd %xmm0, %xmm0
; SSE41-32-NEXT: pxor %xmm1, %xmm1
; SSE41-32-NEXT: psubd %xmm0, %xmm1
-; SSE41-32-NEXT: pabsd %xmm1, %xmm0
-; SSE41-32-NEXT: movq %xmm0, (%eax)
+; SSE41-32-NEXT: pmaxsd %xmm0, %xmm1
+; SSE41-32-NEXT: movq %xmm1, (%eax)
; SSE41-32-NEXT: retl
;
; SSE41-64-LABEL: pr60168_buildvector_of_zeros_and_undef:
@@ -1448,8 +1450,8 @@ define void @pr60168_buildvector_of_zeros_and_undef(<2 x i32> %x, ptr %out) {
; SSE41-64-NEXT: paddd %xmm0, %xmm0
; SSE41-64-NEXT: pxor %xmm1, %xmm1
; SSE41-64-NEXT: psubd %xmm0, %xmm1
-; SSE41-64-NEXT: pabsd %xmm1, %xmm0
-; SSE41-64-NEXT: movq %xmm0, (%rdi)
+; SSE41-64-NEXT: pmaxsd %xmm0, %xmm1
+; SSE41-64-NEXT: movq %xmm1, (%rdi)
; SSE41-64-NEXT: retq
;
; AVX-32-LABEL: pr60168_buildvector_of_zeros_and_undef:
@@ -1457,8 +1459,8 @@ define void @pr60168_buildvector_of_zeros_and_undef(<2 x i32> %x, ptr %out) {
; AVX-32-NEXT: movl {{[0-9]+}}(%esp), %eax
; AVX-32-NEXT: vpaddd %xmm0, %xmm0, %xmm0
; AVX-32-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX-32-NEXT: vpsubd %xmm0, %xmm1, %xmm0
-; AVX-32-NEXT: vpabsd %xmm0, %xmm0
+; AVX-32-NEXT: vpsubd %xmm0, %xmm1, %xmm1
+; AVX-32-NEXT: vpmaxsd %xmm0, %xmm1, %xmm0
; AVX-32-NEXT: vmovq %xmm0, (%eax)
; AVX-32-NEXT: retl
;
@@ -1466,8 +1468,8 @@ define void @pr60168_buildvector_of_zeros_and_undef(<2 x i32> %x, ptr %out) {
; AVX-64: # %bb.0:
; AVX-64-NEXT: vpaddd %xmm0, %xmm0, %xmm0
; AVX-64-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX-64-NEXT: vpsubd %xmm0, %xmm1, %xmm0
-; AVX-64-NEXT: vpabsd %xmm0, %xmm0
+; AVX-64-NEXT: vpsubd %xmm0, %xmm1, %xmm1
+; AVX-64-NEXT: vpmaxsd %xmm0, %xmm1, %xmm0
; AVX-64-NEXT: vmovq %xmm0, (%rdi)
; AVX-64-NEXT: retq
%i2 = mul <2 x i32> %x, <i32 -2, i32 -2>
diff --git a/llvm/test/CodeGen/X86/combine-abs.ll b/llvm/test/CodeGen/X86/combine-abs.ll
index 76ee02e798707..c5ace08dc2141 100644
--- a/llvm/test/CodeGen/X86/combine-abs.ll
+++ b/llvm/test/CodeGen/X86/combine-abs.ll
@@ -56,17 +56,26 @@ define <8 x i16> @combine_v8i16_abs_abs(<8 x i16> %a) {
; SSE2-LABEL: combine_v8i16_abs_abs:
; SSE2: # %bb.0:
; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: psubw %xmm0, %xmm2
+; SSE2-NEXT: pmaxsw %xmm2, %xmm0
; SSE2-NEXT: psubw %xmm0, %xmm1
; SSE2-NEXT: pmaxsw %xmm1, %xmm0
; SSE2-NEXT: retq
;
; SSE42-LABEL: combine_v8i16_abs_abs:
; SSE42: # %bb.0:
-; SSE42-NEXT: pabsw %xmm0, %xmm0
+; SSE42-NEXT: pxor %xmm1, %xmm1
+; SSE42-NEXT: psubw %xmm0, %xmm1
+; SSE42-NEXT: pmaxsw %xmm0, %xmm1
+; SSE42-NEXT: pabsw %xmm1, %xmm0
; SSE42-NEXT: retq
;
; AVX-LABEL: combine_v8i16_abs_abs:
; AVX: # %bb.0:
+; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX-NEXT: vpsubw %xmm0, %xmm1, %xmm1
+; AVX-NEXT: vpmaxsw %xmm1, %xmm0, %xmm0
; AVX-NEXT: vpabsw %xmm0, %xmm0
; AVX-NEXT: retq
%a1 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %a, i1 false)
@@ -81,21 +90,43 @@ define <32 x i8> @combine_v32i8_abs_abs(<32 x i8> %a) {
; SSE2: # %bb.0:
; SSE2-NEXT: pxor %xmm2, %xmm2
; SSE2-NEXT: pxor %xmm3, %xmm3
-; SSE2-NEXT: psubb %xmm0, %xmm3
-; SSE2-NEXT: pminub %xmm3, %xmm0
-; SSE2-NEXT: psubb %xmm1, %xmm2
-; SSE2-NEXT: pminub %xmm2, %xmm1
+; SSE2-NEXT: psubb %xmm1, %xmm3
+; SSE2-NEXT: pminub %xmm1, %xmm3
+; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: psubb %xmm0, %xmm1
+; SSE2-NEXT: pminub %xmm0, %xmm1
+; SSE2-NEXT: pxor %xmm4, %xmm4
+; SSE2-NEXT: psubb %xmm1, %xmm4
+; SSE2-NEXT: movdqa %xmm1, %xmm0
+; SSE2-NEXT: pcmpgtb %xmm4, %xmm0
+; SSE2-NEXT: pxor %xmm0, %xmm1
+; SSE2-NEXT: psubb %xmm1, %xmm0
+; SSE2-NEXT: psubb %xmm3, %xmm2
+; SSE2-NEXT: movdqa %xmm3, %xmm1
+; SSE2-NEXT: pcmpgtb %xmm2, %xmm1
+; SSE2-NEXT: pxor %xmm1, %xmm3
+; SSE2-NEXT: psubb %xmm3, %xmm1
; SSE2-NEXT: retq
;
; SSE42-LABEL: combine_v32i8_abs_abs:
; SSE42: # %bb.0:
-; SSE42-NEXT: pabsb %xmm0, %xmm0
+; SSE42-NEXT: pxor %xmm2, %xmm2
; SSE42-NEXT: pabsb %xmm1, %xmm1
+; SSE42-NEXT: pabsb %xmm0, %xmm3
+; SSE42-NEXT: pxor %xmm0, %xmm0
+; SSE42-NEXT: psubb %xmm3, %xmm0
+; SSE42-NEXT: pmaxsb %xmm3, %xmm0
+; SSE42-NEXT: psubb %xmm1, %xmm2
+; SSE42-NEXT: pmaxsb %xmm1, %xmm2
+; SSE42-NEXT: movdqa %xmm2, %xmm1
; SSE42-NEXT: retq
;
; AVX-LABEL: combine_v32i8_abs_abs:
; AVX: # %bb.0:
+; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX-NEXT: vpabsb %ymm0, %ymm0
+; AVX-NEXT: vpsubb %ymm0, %ymm1, %ymm1
+; AVX-NEXT: vpmaxsb %ymm1, %ymm0, %ymm0
; AVX-NEXT: retq
%n1 = sub <32 x i8> zeroinitializer, %a
%b1 = icmp slt <32 x i8> %a, zeroinitializer
@@ -159,24 +190,48 @@ define <4 x i64> @combine_v4i64_abs_abs(<4 x i64> %a) {
; fold (abs x) -> x iff not-negative
define <16 x i8> @combine_v16i8_abs_constant(<16 x i8> %a) {
-; SSE-LABEL: combine_v16i8_abs_constant:
-; SSE: # %bb.0:
-; SSE-NEXT: andps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE-NEXT: retq
+; SSE2-LABEL: combine_v16i8_abs_constant:
+; SSE2: # %bb.0:
+; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: psubb %xmm0, %xmm2
+; SSE2-NEXT: movdqa %xmm0, %xmm1
+; SSE2-NEXT: pcmpgtb %xmm2, %xmm1
+; SSE2-NEXT: pxor %xmm1, %xmm0
+; SSE2-NEXT: psubb %xmm0, %xmm1
+; SSE2-NEXT: movdqa %xmm1, %xmm0
+; SSE2-NEXT: retq
+;
+; SSE42-LABEL: combine_v16i8_abs_constant:
+; SSE42: # %bb.0:
+; SSE42-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE42-NEXT: pxor %xmm1, %xmm1
+; SSE42-NEXT: psubb %xmm0, %xmm1
+; SSE42-NEXT: pmaxsb %xmm1, %xmm0
+; SSE42-NEXT: retq
;
; AVX2-LABEL: combine_v16i8_abs_constant:
; AVX2: # %bb.0:
-; AVX2-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpsubb %xmm0, %xmm1, %xmm1
+; AVX2-NEXT: vpmaxsb %xmm1, %xmm0, %xmm0
; AVX2-NEXT: retq
;
; AVX512F-LABEL: combine_v16i8_abs_constant:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT: vpsubb %xmm0, %xmm1, %xmm1
+; AVX512F-NEXT: vpmaxsb %xmm1, %xmm0, %xmm0
; AVX512F-NEXT: retq
;
; AVX512VL-LABEL: combine_v16i8_abs_constant:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm0
+; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512VL-NEXT: vpsubb %xmm0, %xmm1, %xmm1
+; AVX512VL-NEXT: vpmaxsb %xmm1, %xmm0, %xmm0
; AVX512VL-NEXT: retq
%1 = insertelement <16 x i8> undef, i8 15, i32 0
%2 = shufflevector <16 x i8> %1, <16 x i8> undef, <16 x i32> zeroinitializer
@@ -186,15 +241,44 @@ define <16 x i8> @combine_v16i8_abs_constant(<16 x i8> %a) {
}
define <8 x i32> @combine_v8i32_abs_pos(<8 x i32> %a) {
-; SSE-LABEL: combine_v8i32_abs_pos:
-; SSE: # %bb.0:
-; SSE-NEXT: psrld $1, %xmm0
-; SSE-NEXT: psrld $1, %xmm1
-; SSE-NEXT: retq
+; SSE2-LABEL: combine_v8i32_abs_pos:
+; SSE2: # %bb.0:
+; SSE2-NEXT: psrld $1, %xmm1
+; SSE2-NEXT: psrld $1, %xmm0
+; SSE2-NEXT: pxor %xmm4, %xmm4
+; SSE2-NEXT: pxor %xmm3, %xmm3
+; SSE2-NEXT: psubd %xmm0, %xmm3
+; SSE2-NEXT: movdqa %xmm0, %xmm2
+; SSE2-NEXT: pcmpgtd %xmm3, %xmm2
+; SSE2-NEXT: pxor %xmm2, %xmm0
+; SSE2-NEXT: psubd %xmm0, %xmm2
+; SSE2-NEXT: psubd %xmm1, %xmm4
+; SSE2-NEXT: movdqa %xmm1, %xmm3
+; SSE2-NEXT: pcmpgtd %xmm4, %xmm3
+; SSE2-NEXT: pxor %xmm3, %xmm1
+; SSE2-NEXT: psubd %xmm1, %xmm3
+; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: movdqa %xmm3, %xmm1
+; SSE2-NEXT: retq
+;
+; SSE42-LABEL: combine_v8i32_abs_pos:
+; SSE42: # %bb.0:
+; SSE42-NEXT: psrld $1, %xmm1
+; SSE42-NEXT: psrld $1, %xmm0
+; SSE42-NEXT: pxor %xmm2, %xmm2
+; SSE42-NEXT: pxor %xmm3, %xmm3
+; SSE42-NEXT: psubd %xmm0, %xmm3
+; SSE42-NEXT: pmaxsd %xmm3, %xmm0
+; SSE42-NEXT: psubd %xmm1, %xmm2
+; SSE42-NEXT: pmaxsd %xmm2, %xmm1
+; SSE42-NEXT: retq
;
; AVX-LABEL: combine_v8i32_abs_pos:
; AVX: # %bb.0:
; AVX-NEXT: vpsrld $1, %ymm0, %ymm0
+; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX-NEXT: vpsubd %ymm0, %ymm1, %ymm1
+; AVX-NEXT: vpmaxsd %ymm1, %ymm0, %ymm0
; AVX-NEXT: retq
%1 = lshr <8 x i32> %a, <i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1, i32 1>
%2 = call <8 x i32> @llvm.abs.v8i32(<8 x i32> %1, i1 false)
@@ -205,7 +289,11 @@ define <8 x i32> @combine_v8i32_abs_pos(<8 x i32> %a) {
define i32 @combine_i32_abs_zerosign(i32 %a) {
; CHECK-LABEL: combine_i32_abs_zerosign:
; CHECK: # %bb.0:
-; CHECK-NEXT: xorl %eax, %eax
+; CHECK-NEXT: sarl $15, %edi
+; CHECK-NEXT: movl %edi, %eax
+; CHECK-NEXT: negl %eax
+; CHECK-NEXT: andl %edi, %eax
+; CHECK-NEXT: andl $-524288, %eax # imm = 0xFFF80000
; CHECK-NEXT: retq
%1 = ashr i32 %a, 15
%2 = call i32 @llvm.abs.i32(i32 %1, i1 false)
@@ -216,13 +304,38 @@ define i32 @combine_i32_abs_zerosign(i32 %a) {
define <8 x i16> @combine_v8i16_abs_zerosign(<8 x i16> %a) {
; SSE-LABEL: combine_v8i16_abs_zerosign:
; SSE: # %bb.0:
-; SSE-NEXT: xorps %xmm0, %xmm0
+; SSE-NEXT: pmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [512,256,128,64,32,16,8,4]
+; SSE-NEXT: pxor %xmm1, %xmm1
+; SSE-NEXT: psubw %xmm0, %xmm1
+; SSE-NEXT: pand %xmm1, %xmm0
+; SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
; SSE-NEXT: retq
;
-; AVX-LABEL: combine_v8i16_abs_zerosign:
-; AVX: # %bb.0:
-; AVX-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; AVX-NEXT: retq
+; AVX2-LABEL: combine_v8i16_abs_zerosign:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [512,256,128,64,32,16,8,4]
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpsubw %xmm0, %xmm1, %xmm1
+; AVX2-NEXT: vpand %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT: retq
+;
+; AVX512F-LABEL: combine_v8i16_abs_zerosign:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: vpmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [512,256,128,64,32,16,8,4]
+; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT: vpsubw %xmm0, %xmm1, %xmm1
+; AVX512F-NEXT: vpand %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: combine_v8i16_abs_zerosign:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: vpmulhw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1 # [512,256,128,64,32,16,8,4]
+; AVX512VL-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512VL-NEXT: vpsubw %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT: vpternlogd {{.*#+}} xmm0 = xmm0 & xmm1 & m32bcst
+; AVX512VL-NEXT: retq
%1 = ashr <8 x i16> %a, <i16 7, i16 8, i16 9, i16 10, i16 11, i16 12, i16 13, i16 14>
%2 = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %1, i1 false)
%3 = and <8 x i16> %2, <i16 32768, i16 32768, i16 32768, i16 32768, i16 32768, i16 32768, i16 32768, i16 32768>
@@ -236,8 +349,9 @@ define i32 @combine_i32_abs_zerosign_negative(i32 %a) {
; CHECK-NEXT: sarl $3, %edi
; CHECK-NEXT: movl %edi, %eax
; CHECK-NEXT: negl %eax
-; CHECK-NEXT: cmovsl %edi, %eax
-; CHECK-NEXT: andl $536346624, %eax # imm = 0x1FF80000
+; CHECK-NEXT: cmpl %eax, %edi
+; CHECK-NEXT: cmovgl %edi, %eax
+; CHECK-NEXT: andl $-524288, %eax # imm = 0xFFF80000
; CHECK-NEXT: retq
%1 = ashr i32 %a, 3
%2 = call i32 @llvm.abs.i32(i32 %1, i1 false)
diff --git a/llvm/test/CodeGen/X86/expand-vp-int-intrinsics.ll b/llvm/test/CodeGen/X86/expand-vp-int-intrinsics.ll
index dbfa69d497698..e75c153f5c631 100644
--- a/llvm/test/CodeGen/X86/expand-vp-int-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/expand-vp-int-intrinsics.ll
@@ -1079,22 +1079,28 @@ define void @vp_abs_v4i32(<4 x i32> %a0, ptr %out, i32 %vp) nounwind {
; X86-LABEL: vp_abs_v4i32:
; X86: # %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: vpabsd %xmm0, %xmm0
+; X86-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-NEXT: vpsubd %xmm0, %xmm1, %xmm1
+; X86-NEXT: vpmaxsd %xmm1, %xmm0, %xmm0
; X86-NEXT: vmovdqa %xmm0, (%eax)
; X86-NEXT: retl
;
; SSE-LABEL: vp_abs_v4i32:
; SSE: # %bb.0:
-; SSE-NEXT: movdqa %xmm0, %xmm1
-; SSE-NEXT: psrad $31, %xmm1
-; SSE-NEXT: pxor %xmm1, %xmm0
-; SSE-NEXT: psubd %xmm1, %xmm0
-; SSE-NEXT: movdqa %xmm0, (%rdi)
+; SSE-NEXT: pxor %xmm1, %xmm1
+; SSE-NEXT: psubd %xmm0, %xmm1
+; SSE-NEXT: movdqa %xmm0, %xmm2
+; SSE-NEXT: pcmpgtd %xmm1, %xmm2
+; SSE-NEXT: pxor %xmm2, %xmm0
+; SSE-NEXT: psubd %xmm0, %xmm2
+; SSE-NEXT: movdqa %xmm2, (%rdi)
; SSE-NEXT: retq
;
; AVX-LABEL: vp_abs_v4i32:
; AVX: # %bb.0:
-; AVX-NEXT: vpabsd %xmm0, %xmm0
+; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX-NEXT: vpsubd %xmm0, %xmm1, %xmm1
+; AVX-NEXT: vpmaxsd %xmm1, %xmm0, %xmm0
; AVX-NEXT: vmovdqa %xmm0, (%rdi)
; AVX-NEXT: retq
%res = call <4 x i32> @llvm.vp.abs.v4i32(<4 x i32> %a0, i1 false, <4 x i1> <i1 -1, i1 -1, i1 -1, i1 -1>, i32 %vp)
diff --git a/llvm/test/CodeGen/X86/freeze-unary.ll b/llvm/test/CodeGen/X86/freeze-unary.ll
index bc9e29957c74a..46a29b8392192 100644
--- a/llvm/test/CodeGen/X86/freeze-unary.ll
+++ b/llvm/test/CodeGen/X86/freeze-unary.ll
@@ -73,17 +73,27 @@ define <2 x i64> @freeze_zext_vec(<2 x i16> %a0) nounwind {
define i32 @freeze_abs(i32 %a0) nounwind {
; X86-LABEL: freeze_abs:
; X86: # %bb.0:
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %eax, %ecx
+; X86-NEXT: negl %ecx
+; X86-NEXT: cmpl %ecx, %eax
+; X86-NEXT: cmovgl %eax, %ecx
; X86-NEXT: movl %ecx, %eax
; X86-NEXT: negl %eax
-; X86-NEXT: cmovsl %ecx, %eax
+; X86-NEXT: cmpl %eax, %ecx
+; X86-NEXT: cmovgl %ecx, %eax
; X86-NEXT: retl
;
; X64-LABEL: freeze_abs:
; X64: # %bb.0:
-; X64-NEXT: movl %edi, %eax
+; X64-NEXT: movl %edi, %ecx
+; X64-NEXT: negl %ecx
+; X64-NEXT: cmpl %ecx, %edi
+; X64-NEXT: cmovgl %edi, %ecx
+; X64-NEXT: movl %ecx, %eax
; X64-NEXT: negl %eax
-; X64-NEXT: cmovsl %edi, %eax
+; X64-NEXT: cmpl %eax, %ecx
+; X64-NEXT: cmovgl %ecx, %eax
; X64-NEXT: retq
%x = call i32 @llvm.abs.i32(i32 %a0, i1 0)
%f = freeze i32 %x
@@ -94,15 +104,28 @@ define i32 @freeze_abs(i32 %a0) nounwind {
define <4 x i32> @freeze_abs_vec(<4 x i32> %a0) nounwind {
; X86-LABEL: freeze_abs_vec:
; X86: # %bb.0:
-; X86-NEXT: movdqa %xmm0, %xmm1
-; X86-NEXT: psrad $31, %xmm1
-; X86-NEXT: pxor %xmm1, %xmm0
-; X86-NEXT: psubd %xmm1, %xmm0
+; X86-NEXT: pxor %xmm1, %xmm1
+; X86-NEXT: pxor %xmm2, %xmm2
+; X86-NEXT: psubd %xmm0, %xmm2
+; X86-NEXT: movdqa %xmm0, %xmm3
+; X86-NEXT: pcmpgtd %xmm2, %xmm3
+; X86-NEXT: pxor %xmm3, %xmm0
+; X86-NEXT: psubd %xmm0, %xmm3
+; X86-NEXT: psubd %xmm3, %xmm1
+; X86-NEXT: movdqa %xmm3, %xmm0
+; X86-NEXT: pcmpgtd %xmm1, %xmm0
+; X86-NEXT: pxor %xmm0, %xmm3
+; X86-NEXT: psubd %xmm3, %xmm0
; X86-NEXT: retl
;
; X64-LABEL: freeze_abs_vec:
; X64: # %bb.0:
-; X64-NEXT: pabsd %xmm0, %xmm0
+; X64-NEXT: pxor %xmm1, %xmm1
+; X64-NEXT: pxor %xmm2, %xmm2
+; X64-NEXT: psubd %xmm0, %xmm2
+; X64-NEXT: pmaxsd %xmm2, %xmm0
+; X64-NEXT: psubd %xmm0, %xmm1
+; X64-NEXT: pmaxsd %xmm1, %xmm0
; X64-NEXT: retq
%x = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %a0, i1 0)
%f = freeze <4 x i32> %x
diff --git a/llvm/test/CodeGen/X86/neg-abs.ll b/llvm/test/CodeGen/X86/neg-abs.ll
index 050610426e6bd..3e81f610ea566 100644
--- a/llvm/test/CodeGen/X86/neg-abs.ll
+++ b/llvm/test/CodeGen/X86/neg-abs.ll
@@ -159,20 +159,27 @@ define i8 @sub_abs_i8(i8 %x, i8 %y) nounwind {
; X86-LABEL: sub_abs_i8:
; X86: # %bb.0:
; X86-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: sarb $7, %al
-; X86-NEXT: xorb %al, %cl
+; X86-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: negb %dl
+; X86-NEXT: cmpb %dl, %cl
+; X86-NEXT: jg .LBB5_2
+; X86-NEXT: # %bb.1:
+; X86-NEXT: movl %edx, %ecx
+; X86-NEXT: .LBB5_2:
; X86-NEXT: subb %cl, %al
-; X86-NEXT: addb {{[0-9]+}}(%esp), %al
; X86-NEXT: retl
;
; X64-LABEL: sub_abs_i8:
; X64: # %bb.0:
-; X64-NEXT: movl %edi, %eax
-; X64-NEXT: sarb $7, %al
-; X64-NEXT: xorb %al, %dil
-; X64-NEXT: subb %dil, %al
-; X64-NEXT: addb %sil, %al
+; X64-NEXT: movl %esi, %eax
+; X64-NEXT: movl %edi, %ecx
+; X64-NEXT: negb %cl
+; X64-NEXT: movzbl %cl, %ecx
+; X64-NEXT: cmpb %cl, %dil
+; X64-NEXT: cmovgl %edi, %ecx
+; X64-NEXT: subb %cl, %al
+; X64-NEXT: # kill: def $al killed $al killed $eax
; X64-NEXT: retq
%abs = tail call i8 @llvm.abs.i8(i8 %x, i1 false)
%neg = sub nsw i8 %y, %abs
@@ -182,21 +189,27 @@ define i8 @sub_abs_i8(i8 %x, i8 %y) nounwind {
define i16 @sub_abs_i16(i16 %x, i16 %y) nounwind {
; X86-LABEL: sub_abs_i16:
; X86: # %bb.0:
-; X86-NEXT: movswl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: sarl $15, %eax
-; X86-NEXT: xorl %eax, %ecx
+; X86-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: negl %edx
+; X86-NEXT: cmpw %dx, %cx
+; X86-NEXT: jg .LBB6_2
+; X86-NEXT: # %bb.1:
+; X86-NEXT: movl %edx, %ecx
+; X86-NEXT: .LBB6_2:
; X86-NEXT: subl %ecx, %eax
-; X86-NEXT: addl {{[0-9]+}}(%esp), %eax
; X86-NEXT: # kill: def $ax killed $ax killed $eax
; X86-NEXT: retl
;
; X64-LABEL: sub_abs_i16:
; X64: # %bb.0:
-; X64-NEXT: movl %edi, %eax
-; X64-NEXT: negw %ax
-; X64-NEXT: cmovnsw %di, %ax
-; X64-NEXT: addl %esi, %eax
+; X64-NEXT: movl %esi, %eax
+; X64-NEXT: movl %edi, %ecx
+; X64-NEXT: negl %ecx
+; X64-NEXT: cmpw %cx, %di
+; X64-NEXT: cmovgl %edi, %ecx
+; X64-NEXT: subl %ecx, %eax
; X64-NEXT: # kill: def $ax killed $ax killed $eax
; X64-NEXT: retq
%abs = tail call i16 @llvm.abs.i16(i16 %x, i1 false)
@@ -208,19 +221,25 @@ define i32 @sub_abs_i32(i32 %x, i32 %y) nounwind {
; X86-LABEL: sub_abs_i32:
; X86: # %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ecx, %eax
-; X86-NEXT: sarl $31, %eax
-; X86-NEXT: xorl %eax, %ecx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl %ecx, %edx
+; X86-NEXT: negl %edx
+; X86-NEXT: cmpl %edx, %ecx
+; X86-NEXT: jg .LBB7_2
+; X86-NEXT: # %bb.1:
+; X86-NEXT: movl %edx, %ecx
+; X86-NEXT: .LBB7_2:
; X86-NEXT: subl %ecx, %eax
-; X86-NEXT: addl {{[0-9]+}}(%esp), %eax
; X86-NEXT: retl
;
; X64-LABEL: sub_abs_i32:
; X64: # %bb.0:
-; X64-NEXT: movl %edi, %eax
-; X64-NEXT: negl %eax
-; X64-NEXT: cmovnsl %edi, %eax
-; X64-NEXT: addl %esi, %eax
+; X64-NEXT: movl %esi, %eax
+; X64-NEXT: movl %edi, %ecx
+; X64-NEXT: negl %ecx
+; X64-NEXT: cmpl %ecx, %edi
+; X64-NEXT: cmovgl %edi, %ecx
+; X64-NEXT: subl %ecx, %eax
; X64-NEXT: retq
%abs = tail call i32 @llvm.abs.i32(i32 %x, i1 false)
%neg = sub i32 %y, %abs
@@ -230,30 +249,42 @@ define i32 @sub_abs_i32(i32 %x, i32 %y) nounwind {
define i64 @sub_abs_i64(i64 %x, i64 %y) nounwind {
; X86-LABEL: sub_abs_i64:
; X86: # %bb.0:
+; X86-NEXT: pushl %ebp
+; X86-NEXT: pushl %ebx
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ecx, %esi
-; X86-NEXT: sarl $31, %esi
-; X86-NEXT: xorl %esi, %ecx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X86-NEXT: xorl %esi, %edi
-; X86-NEXT: subl %esi, %edi
-; X86-NEXT: sbbl %esi, %ecx
-; X86-NEXT: subl %edi, %eax
+; X86-NEXT: xorl %ecx, %ecx
+; X86-NEXT: movl %ebx, %esi
+; X86-NEXT: negl %esi
+; X86-NEXT: sbbl %edi, %ecx
+; X86-NEXT: cmpl %ebx, %esi
+; X86-NEXT: movl %ecx, %ebp
+; X86-NEXT: sbbl %edi, %ebp
+; X86-NEXT: jge .LBB8_2
+; X86-NEXT: # %bb.1:
+; X86-NEXT: movl %edi, %ecx
+; X86-NEXT: movl %ebx, %esi
+; X86-NEXT: .LBB8_2:
+; X86-NEXT: subl %esi, %eax
; X86-NEXT: sbbl %ecx, %edx
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
+; X86-NEXT: popl %ebx
+; X86-NEXT: popl %ebp
; X86-NEXT: retl
;
; X64-LABEL: sub_abs_i64:
; X64: # %bb.0:
-; X64-NEXT: movq %rdi, %rax
-; X64-NEXT: negq %rax
-; X64-NEXT: cmovnsq %rdi, %rax
-; X64-NEXT: addq %rsi, %rax
+; X64-NEXT: movq %rsi, %rax
+; X64-NEXT: movq %rdi, %rcx
+; X64-NEXT: negq %rcx
+; X64-NEXT: cmpq %rcx, %rdi
+; X64-NEXT: cmovgq %rdi, %rcx
+; X64-NEXT: subq %rcx, %rax
; X64-NEXT: retq
%abs = tail call i64 @llvm.abs.i64(i64 %x, i1 false)
%neg = sub i64 %y, %abs
@@ -265,53 +296,73 @@ define i128 @sub_abs_i128(i128 %x, i128 %y) nounwind {
; X86: # %bb.0:
; X86-NEXT: pushl %ebp
; X86-NEXT: movl %esp, %ebp
+; X86-NEXT: pushl %ebx
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
; X86-NEXT: andl $-16, %esp
-; X86-NEXT: movl 36(%ebp), %eax
-; X86-NEXT: movl %eax, %edx
-; X86-NEXT: sarl $31, %edx
-; X86-NEXT: xorl %edx, %eax
-; X86-NEXT: movl 32(%ebp), %ecx
-; X86-NEXT: xorl %edx, %ecx
-; X86-NEXT: movl 28(%ebp), %esi
-; X86-NEXT: xorl %edx, %esi
-; X86-NEXT: movl 24(%ebp), %edi
-; X86-NEXT: xorl %edx, %edi
-; X86-NEXT: subl %edx, %edi
-; X86-NEXT: sbbl %edx, %esi
+; X86-NEXT: subl $16, %esp
+; X86-NEXT: movl 24(%ebp), %eax
+; X86-NEXT: xorl %ecx, %ecx
+; X86-NEXT: movl %eax, %esi
+; X86-NEXT: negl %esi
+; X86-NEXT: movl $0, %ebx
+; X86-NEXT: sbbl 28(%ebp), %ebx
+; X86-NEXT: movl $0, %edi
+; X86-NEXT: sbbl 32(%ebp), %edi
+; X86-NEXT: movl 36(%ebp), %edx
; X86-NEXT: sbbl %edx, %ecx
+; X86-NEXT: cmpl %eax, %esi
+; X86-NEXT: movl %ebx, %eax
+; X86-NEXT: sbbl 28(%ebp), %eax
+; X86-NEXT: movl %edi, %eax
+; X86-NEXT: sbbl 32(%ebp), %eax
+; X86-NEXT: movl %ecx, %eax
; X86-NEXT: sbbl %edx, %eax
; X86-NEXT: movl 40(%ebp), %edx
-; X86-NEXT: subl %edi, %edx
-; X86-NEXT: movl 44(%ebp), %edi
-; X86-NEXT: sbbl %esi, %edi
+; X86-NEXT: jl .LBB9_1
+; X86-NEXT: # %bb.2:
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: jmp .LBB9_3
+; X86-NEXT: .LBB9_1:
+; X86-NEXT: movl 36(%ebp), %ecx
+; X86-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NEXT: movl 32(%ebp), %edi
+; X86-NEXT: movl 28(%ebp), %ebx
+; X86-NEXT: movl 24(%ebp), %esi
+; X86-NEXT: .LBB9_3:
+; X86-NEXT: subl %esi, %edx
+; X86-NEXT: movl 44(%ebp), %ecx
+; X86-NEXT: sbbl %ebx, %ecx
; X86-NEXT: movl 48(%ebp), %esi
-; X86-NEXT: sbbl %ecx, %esi
-; X86-NEXT: movl 52(%ebp), %ecx
-; X86-NEXT: sbbl %eax, %ecx
+; X86-NEXT: sbbl %edi, %esi
+; X86-NEXT: movl 52(%ebp), %edi
+; X86-NEXT: sbbl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
; X86-NEXT: movl 8(%ebp), %eax
; X86-NEXT: movl %edx, (%eax)
-; X86-NEXT: movl %edi, 4(%eax)
+; X86-NEXT: movl %ecx, 4(%eax)
; X86-NEXT: movl %esi, 8(%eax)
-; X86-NEXT: movl %ecx, 12(%eax)
-; X86-NEXT: leal -8(%ebp), %esp
+; X86-NEXT: movl %edi, 12(%eax)
+; X86-NEXT: leal -12(%ebp), %esp
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
+; X86-NEXT: popl %ebx
; X86-NEXT: popl %ebp
; X86-NEXT: retl $4
;
; X64-LABEL: sub_abs_i128:
; X64: # %bb.0:
; X64-NEXT: movq %rdx, %rax
-; X64-NEXT: movq %rsi, %rdx
-; X64-NEXT: sarq $63, %rdx
-; X64-NEXT: xorq %rdx, %rsi
-; X64-NEXT: xorq %rdx, %rdi
-; X64-NEXT: subq %rdx, %rdi
-; X64-NEXT: sbbq %rdx, %rsi
-; X64-NEXT: subq %rdi, %rax
-; X64-NEXT: sbbq %rsi, %rcx
+; X64-NEXT: xorl %edx, %edx
+; X64-NEXT: movq %rdi, %r8
+; X64-NEXT: negq %r8
+; X64-NEXT: sbbq %rsi, %rdx
+; X64-NEXT: cmpq %rdi, %r8
+; X64-NEXT: movq %rdx, %r9
+; X64-NEXT: sbbq %rsi, %r9
+; X64-NEXT: cmovlq %rsi, %rdx
+; X64-NEXT: cmovlq %rdi, %r8
+; X64-NEXT: subq %r8, %rax
+; X64-NEXT: sbbq %rdx, %rcx
; X64-NEXT: movq %rcx, %rdx
; X64-NEXT: retq
%abs = tail call i128 @llvm.abs.i128(i128 %x, i1 false)
diff --git a/llvm/test/CodeGen/X86/pr55271.ll b/llvm/test/CodeGen/X86/pr55271.ll
index 3ffa6cf4acb80..8ee9a2a498c0c 100644
--- a/llvm/test/CodeGen/X86/pr55271.ll
+++ b/llvm/test/CodeGen/X86/pr55271.ll
@@ -8,7 +8,9 @@ declare i32 @llvm.abs.i32(i32, i1 immarg) #0
define i32 @abs(i32 %0) {
; CHECK-LABEL: abs:
; CHECK: # %bb.0:
-; CHECK-NEXT: xorl %eax, %eax
+; CHECK-NEXT: negl %eax
+; CHECK-NEXT: cmpl %eax, %eax
+; CHECK-NEXT: cmovgl %eax, %eax
; CHECK-NEXT: retq
%2 = call i32 @llvm.abs.i32(i32 undef, i1 false)
ret i32 %2
diff --git a/llvm/test/CodeGen/X86/ssse3-intrinsics-fast-isel.ll b/llvm/test/CodeGen/X86/ssse3-intrinsics-fast-isel.ll
index 1dd21c644140a..641156d702077 100644
--- a/llvm/test/CodeGen/X86/ssse3-intrinsics-fast-isel.ll
+++ b/llvm/test/CodeGen/X86/ssse3-intrinsics-fast-isel.ll
@@ -11,12 +11,20 @@
define <2 x i64> @test_mm_abs_epi8(<2 x i64> %a0) {
; SSE-LABEL: test_mm_abs_epi8:
; SSE: # %bb.0:
-; SSE-NEXT: pabsb %xmm0, %xmm0
+; SSE-NEXT: pxor %xmm2, %xmm2
+; SSE-NEXT: psubb %xmm0, %xmm2
+; SSE-NEXT: movdqa %xmm0, %xmm1
+; SSE-NEXT: pcmpgtb %xmm2, %xmm1
+; SSE-NEXT: pxor %xmm1, %xmm0
+; SSE-NEXT: psubb %xmm0, %xmm1
+; SSE-NEXT: movdqa %xmm1, %xmm0
; SSE-NEXT: ret{{[l|q]}}
;
; AVX-LABEL: test_mm_abs_epi8:
; AVX: # %bb.0:
-; AVX-NEXT: vpabsb %xmm0, %xmm0
+; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX-NEXT: vpsubb %xmm0, %xmm1, %xmm1
+; AVX-NEXT: vpmaxsb %xmm1, %xmm0, %xmm0
; AVX-NEXT: ret{{[l|q]}}
%arg = bitcast <2 x i64> %a0 to <16 x i8>
%abs = call <16 x i8> @llvm.abs.v16i8(<16 x i8> %arg, i1 false)
@@ -28,12 +36,16 @@ declare <16 x i8> @llvm.abs.v16i8(<16 x i8>, i1) nounwind readnone
define <2 x i64> @test_mm_abs_epi16(<2 x i64> %a0) {
; SSE-LABEL: test_mm_abs_epi16:
; SSE: # %bb.0:
-; SSE-NEXT: pabsw %xmm0, %xmm0
+; SSE-NEXT: pxor %xmm1, %xmm1
+; SSE-NEXT: psubw %xmm0, %xmm1
+; SSE-NEXT: pmaxsw %xmm1, %xmm0
; SSE-NEXT: ret{{[l|q]}}
;
; AVX-LABEL: test_mm_abs_epi16:
; AVX: # %bb.0:
-; AVX-NEXT: vpabsw %xmm0, %xmm0
+; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX-NEXT: vpsubw %xmm0, %xmm1, %xmm1
+; AVX-NEXT: vpmaxsw %xmm1, %xmm0, %xmm0
; AVX-NEXT: ret{{[l|q]}}
%arg = bitcast <2 x i64> %a0 to <8 x i16>
%abs = call <8 x i16> @llvm.abs.v8i16(<8 x i16> %arg, i1 false)
@@ -45,12 +57,20 @@ declare <8 x i16> @llvm.abs.v8i16(<8 x i16>, i1) nounwind readnone
define <2 x i64> @test_mm_abs_epi32(<2 x i64> %a0) {
; SSE-LABEL: test_mm_abs_epi32:
; SSE: # %bb.0:
-; SSE-NEXT: pabsd %xmm0, %xmm0
+; SSE-NEXT: pxor %xmm2, %xmm2
+; SSE-NEXT: psubd %xmm0, %xmm2
+; SSE-NEXT: movdqa %xmm0, %xmm1
+; SSE-NEXT: pcmpgtd %xmm2, %xmm1
+; SSE-NEXT: pxor %xmm1, %xmm0
+; SSE-NEXT: psubd %xmm0, %xmm1
+; SSE-NEXT: movdqa %xmm1, %xmm0
; SSE-NEXT: ret{{[l|q]}}
;
; AVX-LABEL: test_mm_abs_epi32:
; AVX: # %bb.0:
-; AVX-NEXT: vpabsd %xmm0, %xmm0
+; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX-NEXT: vpsubd %xmm0, %xmm1, %xmm1
+; AVX-NEXT: vpmaxsd %xmm1, %xmm0, %xmm0
; AVX-NEXT: ret{{[l|q]}}
%arg = bitcast <2 x i64> %a0 to <4 x i32>
%abs = call <4 x i32> @llvm.abs.v4i32(<4 x i32> %arg, i1 false)
diff --git a/llvm/test/CodeGen/X86/ssse3-intrinsics-x86-upgrade.ll b/llvm/test/CodeGen/X86/ssse3-intrinsics-x86-upgrade.ll
index 20d7ee573a7d4..7c010bf6a8720 100644
--- a/llvm/test/CodeGen/X86/ssse3-intrinsics-x86-upgrade.ll
+++ b/llvm/test/CodeGen/X86/ssse3-intrinsics-x86-upgrade.ll
@@ -9,17 +9,27 @@
define <16 x i8> @test_x86_ssse3_pabs_b_128(<16 x i8> %a0) {
; SSE-LABEL: test_x86_ssse3_pabs_b_128:
; SSE: ## %bb.0:
-; SSE-NEXT: pabsb %xmm0, %xmm0 ## encoding: [0x66,0x0f,0x38,0x1c,0xc0]
+; SSE-NEXT: pxor %xmm2, %xmm2 ## encoding: [0x66,0x0f,0xef,0xd2]
+; SSE-NEXT: psubb %xmm0, %xmm2 ## encoding: [0x66,0x0f,0xf8,0xd0]
+; SSE-NEXT: movdqa %xmm0, %xmm1 ## encoding: [0x66,0x0f,0x6f,0xc8]
+; SSE-NEXT: pcmpgtb %xmm2, %xmm1 ## encoding: [0x66,0x0f,0x64,0xca]
+; SSE-NEXT: pxor %xmm1, %xmm0 ## encoding: [0x66,0x0f,0xef,0xc1]
+; SSE-NEXT: psubb %xmm0, %xmm1 ## encoding: [0x66,0x0f,0xf8,0xc8]
+; SSE-NEXT: movdqa %xmm1, %xmm0 ## encoding: [0x66,0x0f,0x6f,0xc1]
; SSE-NEXT: ret{{[l|q]}} ## encoding: [0xc3]
;
; AVX1-LABEL: test_x86_ssse3_pabs_b_128:
; AVX1: ## %bb.0:
-; AVX1-NEXT: vpabsb %xmm0, %xmm0 ## encoding: [0xc4,0xe2,0x79,0x1c,0xc0]
+; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 ## encoding: [0xc5,0xf1,0xef,0xc9]
+; AVX1-NEXT: vpsubb %xmm0, %xmm1, %xmm1 ## encoding: [0xc5,0xf1,0xf8,0xc8]
+; AVX1-NEXT: vpmaxsb %xmm1, %xmm0, %xmm0 ## encoding: [0xc4,0xe2,0x79,0x3c,0xc1]
; AVX1-NEXT: ret{{[l|q]}} ## encoding: [0xc3]
;
; AVX512-LABEL: test_x86_ssse3_pabs_b_128:
; AVX512: ## %bb.0:
-; AVX512-NEXT: vpabsb %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x1c,0xc0]
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1 ## EVEX TO VEX Compression encoding: [0xc5,0xf1,0xef,0xc9]
+; AVX512-NEXT: vpsubb %xmm0, %xmm1, %xmm1 ## EVEX TO VEX Compression encoding: [0xc5,0xf1,0xf8,0xc8]
+; AVX512-NEXT: vpmaxsb %xmm1, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x3c,0xc1]
; AVX512-NEXT: ret{{[l|q]}} ## encoding: [0xc3]
%res = call <16 x i8> @llvm.x86.ssse3.pabs.b.128(<16 x i8> %a0) ; <<16 x i8>> [#uses=1]
ret <16 x i8> %res
@@ -30,17 +40,27 @@ declare <16 x i8> @llvm.x86.ssse3.pabs.b.128(<16 x i8>) nounwind readnone
define <4 x i32> @test_x86_ssse3_pabs_d_128(<4 x i32> %a0) {
; SSE-LABEL: test_x86_ssse3_pabs_d_128:
; SSE: ## %bb.0:
-; SSE-NEXT: pabsd %xmm0, %xmm0 ## encoding: [0x66,0x0f,0x38,0x1e,0xc0]
+; SSE-NEXT: pxor %xmm2, %xmm2 ## encoding: [0x66,0x0f,0xef,0xd2]
+; SSE-NEXT: psubd %xmm0, %xmm2 ## encoding: [0x66,0x0f,0xfa,0xd0]
+; SSE-NEXT: movdqa %xmm0, %xmm1 ## encoding: [0x66,0x0f,0x6f,0xc8]
+; SSE-NEXT: pcmpgtd %xmm2, %xmm1 ## encoding: [0x66,0x0f,0x66,0xca]
+; SSE-NEXT: pxor %xmm1, %xmm0 ## encoding: [0x66,0x0f,0xef,0xc1]
+; SSE-NEXT: psubd %xmm0, %xmm1 ## encoding: [0x66,0x0f,0xfa,0xc8]
+; SSE-NEXT: movdqa %xmm1, %xmm0 ## encoding: [0x66,0x0f,0x6f,0xc1]
; SSE-NEXT: ret{{[l|q]}} ## encoding: [0xc3]
;
; AVX1-LABEL: test_x86_ssse3_pabs_d_128:
; AVX1: ## %bb.0:
-; AVX1-NEXT: vpabsd %xmm0, %xmm0 ## encoding: [0xc4,0xe2,0x79,0x1e,0xc0]
+; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 ## encoding: [0xc5,0xf1,0xef,0xc9]
+; AVX1-NEXT: vpsubd %xmm0, %xmm1, %xmm1 ## encoding: [0xc5,0xf1,0xfa,0xc8]
+; AVX1-NEXT: vpmaxsd %xmm1, %xmm0, %xmm0 ## encoding: [0xc4,0xe2,0x79,0x3d,0xc1]
; AVX1-NEXT: ret{{[l|q]}} ## encoding: [0xc3]
;
; AVX512-LABEL: test_x86_ssse3_pabs_d_128:
; AVX512: ## %bb.0:
-; AVX512-NEXT: vpabsd %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x1e,0xc0]
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1 ## EVEX TO VEX Compression encoding: [0xc5,0xf1,0xef,0xc9]
+; AVX512-NEXT: vpsubd %xmm0, %xmm1, %xmm1 ## EVEX TO VEX Compression encoding: [0xc5,0xf1,0xfa,0xc8]
+; AVX512-NEXT: vpmaxsd %xmm1, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x3d,0xc1]
; AVX512-NEXT: ret{{[l|q]}} ## encoding: [0xc3]
%res = call <4 x i32> @llvm.x86.ssse3.pabs.d.128(<4 x i32> %a0) ; <<4 x i32>> [#uses=1]
ret <4 x i32> %res
@@ -51,17 +71,23 @@ declare <4 x i32> @llvm.x86.ssse3.pabs.d.128(<4 x i32>) nounwind readnone
define <8 x i16> @test_x86_ssse3_pabs_w_128(<8 x i16> %a0) {
; SSE-LABEL: test_x86_ssse3_pabs_w_128:
; SSE: ## %bb.0:
-; SSE-NEXT: pabsw %xmm0, %xmm0 ## encoding: [0x66,0x0f,0x38,0x1d,0xc0]
+; SSE-NEXT: pxor %xmm1, %xmm1 ## encoding: [0x66,0x0f,0xef,0xc9]
+; SSE-NEXT: psubw %xmm0, %xmm1 ## encoding: [0x66,0x0f,0xf9,0xc8]
+; SSE-NEXT: pmaxsw %xmm1, %xmm0 ## encoding: [0x66,0x0f,0xee,0xc1]
; SSE-NEXT: ret{{[l|q]}} ## encoding: [0xc3]
;
; AVX1-LABEL: test_x86_ssse3_pabs_w_128:
; AVX1: ## %bb.0:
-; AVX1-NEXT: vpabsw %xmm0, %xmm0 ## encoding: [0xc4,0xe2,0x79,0x1d,0xc0]
+; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 ## encoding: [0xc5,0xf1,0xef,0xc9]
+; AVX1-NEXT: vpsubw %xmm0, %xmm1, %xmm1 ## encoding: [0xc5,0xf1,0xf9,0xc8]
+; AVX1-NEXT: vpmaxsw %xmm1, %xmm0, %xmm0 ## encoding: [0xc5,0xf9,0xee,0xc1]
; AVX1-NEXT: ret{{[l|q]}} ## encoding: [0xc3]
;
; AVX512-LABEL: test_x86_ssse3_pabs_w_128:
; AVX512: ## %bb.0:
-; AVX512-NEXT: vpabsw %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x79,0x1d,0xc0]
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1 ## EVEX TO VEX Compression encoding: [0xc5,0xf1,0xef,0xc9]
+; AVX512-NEXT: vpsubw %xmm0, %xmm1, %xmm1 ## EVEX TO VEX Compression encoding: [0xc5,0xf1,0xf9,0xc8]
+; AVX512-NEXT: vpmaxsw %xmm1, %xmm0, %xmm0 ## EVEX TO VEX Compression encoding: [0xc5,0xf9,0xee,0xc1]
; AVX512-NEXT: ret{{[l|q]}} ## encoding: [0xc3]
%res = call <8 x i16> @llvm.x86.ssse3.pabs.w.128(<8 x i16> %a0) ; <<8 x i16>> [#uses=1]
ret <8 x i16> %res
More information about the llvm-commits
mailing list