[llvm] [AArch64] Select sqdmulh for smul.fix.sat with scale == eltbits-1 (PR #211574)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 23 08:09:17 PDT 2026
https://github.com/guyfischman created https://github.com/llvm/llvm-project/pull/211574
Follow-up to #209351 (which vectorized the generic expansion): on AArch64, smul.fix.sat with scale == eltbits-1 is exactly sqdmulh, so select it directly instead of the generic clamp. Split out per @davemgreen's request.
I used AI to split this out of #209351, and reviewed it.
>From 38276130b10d365d7e3005f61f5c1807c7995f4c Mon Sep 17 00:00:00 2001
From: Guy Fischman <gfischman at gmail.com>
Date: Thu, 23 Jul 2026 11:59:15 +0200
Subject: [PATCH 1/2] [SelectionDAG][AArch64][X86] Expand saturating
fixed-point multiplies for vector types
---
.../SelectionDAG/LegalizeVectorOps.cpp | 9 +-
llvm/test/CodeGen/AArch64/smul_fix_sat.ll | 892 +++++++-----------
llvm/test/CodeGen/AArch64/umul_fix_sat.ll | 731 ++++++--------
llvm/test/CodeGen/X86/smul_fix_sat.ll | 185 ++--
llvm/test/CodeGen/X86/umul_fix_sat.ll | 110 +--
.../CodeGen/X86/vector-mulfix-legalize.ll | 214 +++--
6 files changed, 882 insertions(+), 1259 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
index f4cb60736ef7a..7b97c41ac2dbc 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
@@ -1278,18 +1278,13 @@ void VectorLegalizer::Expand(SDNode *Node, SmallVectorImpl<SDValue> &Results) {
break;
case ISD::SMULFIX:
case ISD::UMULFIX:
+ case ISD::SMULFIXSAT:
+ case ISD::UMULFIXSAT:
if (SDValue Expanded = TLI.expandFixedPointMul(Node, DAG)) {
Results.push_back(Expanded);
return;
}
break;
- case ISD::SMULFIXSAT:
- case ISD::UMULFIXSAT:
- // FIXME: We do not expand SMULFIXSAT/UMULFIXSAT here yet, not sure exactly
- // why. Maybe it results in worse codegen compared to the unroll for some
- // targets? This should probably be investigated. And if we still prefer to
- // unroll an explanation could be helpful.
- break;
case ISD::SDIVFIX:
case ISD::UDIVFIX:
ExpandFixedPointDiv(Node, Results);
diff --git a/llvm/test/CodeGen/AArch64/smul_fix_sat.ll b/llvm/test/CodeGen/AArch64/smul_fix_sat.ll
index 4cef8160da145..b5d81af9f2dfe 100644
--- a/llvm/test/CodeGen/AArch64/smul_fix_sat.ll
+++ b/llvm/test/CodeGen/AArch64/smul_fix_sat.ll
@@ -165,107 +165,70 @@ define i64 @func8(i64 %x, i64 %y) {
define <8 x i8> @vec_v8i8(<8 x i8> %x, <8 x i8> %y) {
; CHECK-LABEL: vec_v8i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT: umov w8, v0.b[0]
-; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1
-; CHECK-NEXT: smov x9, v1.b[0]
-; CHECK-NEXT: umov w10, v0.b[1]
-; CHECK-NEXT: smov x11, v1.b[1]
-; CHECK-NEXT: umov w12, v0.b[2]
-; CHECK-NEXT: smov x14, v1.b[2]
-; CHECK-NEXT: umov w15, v0.b[3]
-; CHECK-NEXT: smov x16, v1.b[3]
-; CHECK-NEXT: umov w17, v0.b[4]
-; CHECK-NEXT: lsl w8, w8, #24
-; CHECK-NEXT: lsl w10, w10, #24
-; CHECK-NEXT: smull x13, w9, w8
-; CHECK-NEXT: mov w9, #2147483647 // =0x7fffffff
-; CHECK-NEXT: mov w8, #-2147483648 // =0x80000000
-; CHECK-NEXT: smull x10, w11, w10
-; CHECK-NEXT: lsl w11, w12, #24
-; CHECK-NEXT: lsl w15, w15, #24
-; CHECK-NEXT: lsl w17, w17, #24
-; CHECK-NEXT: lsr x12, x13, #32
-; CHECK-NEXT: smull x11, w14, w11
-; CHECK-NEXT: lsr x14, x10, #32
-; CHECK-NEXT: smull x15, w16, w15
-; CHECK-NEXT: extr w13, w12, w13, #2
-; CHECK-NEXT: cmp w12, #1
-; CHECK-NEXT: extr w10, w14, w10, #2
-; CHECK-NEXT: lsr x16, x11, #32
-; CHECK-NEXT: csel w13, w9, w13, gt
-; CHECK-NEXT: cmn w12, #2
-; CHECK-NEXT: smov x12, v1.b[4]
-; CHECK-NEXT: csel w13, w8, w13, lt
-; CHECK-NEXT: cmp w14, #1
-; CHECK-NEXT: extr w11, w16, w11, #2
-; CHECK-NEXT: asr w13, w13, #24
-; CHECK-NEXT: csel w10, w9, w10, gt
-; CHECK-NEXT: cmn w14, #2
-; CHECK-NEXT: umov w14, v0.b[5]
-; CHECK-NEXT: csel w10, w8, w10, lt
-; CHECK-NEXT: cmp w16, #1
-; CHECK-NEXT: fmov s2, w13
-; CHECK-NEXT: asr w10, w10, #24
-; CHECK-NEXT: smov x13, v1.b[5]
-; CHECK-NEXT: smull x12, w12, w17
-; CHECK-NEXT: csel w11, w9, w11, gt
-; CHECK-NEXT: cmn w16, #2
-; CHECK-NEXT: lsr x16, x15, #32
-; CHECK-NEXT: csel w11, w8, w11, lt
-; CHECK-NEXT: mov v2.b[1], w10
-; CHECK-NEXT: lsl w10, w14, #24
-; CHECK-NEXT: asr w11, w11, #24
-; CHECK-NEXT: extr w15, w16, w15, #2
-; CHECK-NEXT: umov w14, v0.b[6]
-; CHECK-NEXT: cmp w16, #1
-; CHECK-NEXT: smull x10, w13, w10
-; CHECK-NEXT: lsr x13, x12, #32
-; CHECK-NEXT: csel w15, w9, w15, gt
-; CHECK-NEXT: cmn w16, #2
-; CHECK-NEXT: mov v2.b[2], w11
-; CHECK-NEXT: smov x11, v1.b[6]
-; CHECK-NEXT: extr w12, w13, w12, #2
-; CHECK-NEXT: csel w15, w8, w15, lt
-; CHECK-NEXT: cmp w13, #1
-; CHECK-NEXT: lsl w14, w14, #24
-; CHECK-NEXT: asr w15, w15, #24
-; CHECK-NEXT: csel w12, w9, w12, gt
-; CHECK-NEXT: cmn w13, #2
-; CHECK-NEXT: umov w13, v0.b[7]
-; CHECK-NEXT: lsr x16, x10, #32
-; CHECK-NEXT: csel w12, w8, w12, lt
-; CHECK-NEXT: mov v2.b[3], w15
-; CHECK-NEXT: smull x11, w11, w14
-; CHECK-NEXT: smov x14, v1.b[7]
-; CHECK-NEXT: extr w10, w16, w10, #2
-; CHECK-NEXT: asr w12, w12, #24
-; CHECK-NEXT: cmp w16, #1
-; CHECK-NEXT: lsl w13, w13, #24
-; CHECK-NEXT: csel w10, w9, w10, gt
-; CHECK-NEXT: cmn w16, #2
-; CHECK-NEXT: mov v2.b[4], w12
-; CHECK-NEXT: lsr x12, x11, #32
-; CHECK-NEXT: csel w10, w8, w10, lt
-; CHECK-NEXT: smull x13, w14, w13
-; CHECK-NEXT: asr w10, w10, #24
-; CHECK-NEXT: extr w11, w12, w11, #2
+; CHECK-NEXT: smull v0.8h, v0.8b, v1.8b
+; CHECK-NEXT: mov w8, #127 // =0x7f
+; CHECK-NEXT: shrn v2.8b, v0.8h, #8
+; CHECK-NEXT: xtn v0.8b, v0.8h
+; CHECK-NEXT: shl v1.8b, v2.8b, #6
+; CHECK-NEXT: smov w9, v2.b[1]
+; CHECK-NEXT: smov w11, v2.b[0]
+; CHECK-NEXT: usra v1.8b, v0.8b, #2
+; CHECK-NEXT: cmp w9, #1
+; CHECK-NEXT: umov w10, v1.b[1]
+; CHECK-NEXT: umov w12, v1.b[0]
+; CHECK-NEXT: umov w13, v1.b[2]
+; CHECK-NEXT: csel w10, w8, w10, gt
+; CHECK-NEXT: cmn w9, #2
+; CHECK-NEXT: mov w9, #-128 // =0xffffff80
+; CHECK-NEXT: csel w10, w9, w10, lt
+; CHECK-NEXT: cmp w11, #1
+; CHECK-NEXT: csel w12, w8, w12, gt
+; CHECK-NEXT: cmn w11, #2
+; CHECK-NEXT: smov w11, v2.b[2]
+; CHECK-NEXT: csel w12, w9, w12, lt
+; CHECK-NEXT: fmov s0, w12
+; CHECK-NEXT: cmp w11, #1
+; CHECK-NEXT: mov v0.b[1], w10
+; CHECK-NEXT: smov w10, v2.b[3]
+; CHECK-NEXT: csel w12, w8, w13, gt
+; CHECK-NEXT: cmn w11, #2
+; CHECK-NEXT: umov w11, v1.b[3]
+; CHECK-NEXT: csel w12, w9, w12, lt
+; CHECK-NEXT: mov v0.b[2], w12
+; CHECK-NEXT: cmp w10, #1
+; CHECK-NEXT: smov w12, v2.b[4]
+; CHECK-NEXT: csel w11, w8, w11, gt
+; CHECK-NEXT: cmn w10, #2
+; CHECK-NEXT: umov w10, v1.b[4]
+; CHECK-NEXT: csel w11, w9, w11, lt
+; CHECK-NEXT: mov v0.b[3], w11
; CHECK-NEXT: cmp w12, #1
-; CHECK-NEXT: mov v2.b[5], w10
-; CHECK-NEXT: csel w10, w9, w11, gt
+; CHECK-NEXT: smov w11, v2.b[5]
+; CHECK-NEXT: csel w10, w8, w10, gt
; CHECK-NEXT: cmn w12, #2
-; CHECK-NEXT: lsr x11, x13, #32
-; CHECK-NEXT: csel w10, w8, w10, lt
-; CHECK-NEXT: asr w10, w10, #24
-; CHECK-NEXT: extr w12, w11, w13, #2
+; CHECK-NEXT: umov w12, v1.b[5]
+; CHECK-NEXT: csel w10, w9, w10, lt
+; CHECK-NEXT: mov v0.b[4], w10
; CHECK-NEXT: cmp w11, #1
-; CHECK-NEXT: mov v2.b[6], w10
-; CHECK-NEXT: csel w9, w9, w12, gt
+; CHECK-NEXT: smov w10, v2.b[6]
+; CHECK-NEXT: csel w12, w8, w12, gt
; CHECK-NEXT: cmn w11, #2
-; CHECK-NEXT: csel w8, w8, w9, lt
-; CHECK-NEXT: asr w8, w8, #24
-; CHECK-NEXT: mov v2.b[7], w8
-; CHECK-NEXT: fmov d0, d2
+; CHECK-NEXT: umov w11, v1.b[6]
+; CHECK-NEXT: csel w12, w9, w12, lt
+; CHECK-NEXT: mov v0.b[5], w12
+; CHECK-NEXT: cmp w10, #1
+; CHECK-NEXT: smov w12, v2.b[7]
+; CHECK-NEXT: csel w11, w8, w11, gt
+; CHECK-NEXT: cmn w10, #2
+; CHECK-NEXT: umov w10, v1.b[7]
+; CHECK-NEXT: csel w11, w9, w11, lt
+; CHECK-NEXT: mov v0.b[6], w11
+; CHECK-NEXT: cmp w12, #1
+; CHECK-NEXT: csel w8, w8, w10, gt
+; CHECK-NEXT: cmn w12, #2
+; CHECK-NEXT: csel w8, w9, w8, lt
+; CHECK-NEXT: mov v0.b[7], w8
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-NEXT: ret
%tmp = call <8 x i8> @llvm.smul.fix.sat.v8i8(<8 x i8> %x, <8 x i8> %y, i32 2)
ret <8 x i8> %tmp
@@ -274,201 +237,126 @@ define <8 x i8> @vec_v8i8(<8 x i8> %x, <8 x i8> %y) {
define <16 x i8> @vec_v16i8(<16 x i8> %x, <16 x i8> %y) {
; CHECK-LABEL: vec_v16i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: umov w9, v0.b[1]
-; CHECK-NEXT: smov x10, v1.b[1]
-; CHECK-NEXT: mov w8, #2147483647 // =0x7fffffff
-; CHECK-NEXT: umov w11, v0.b[0]
-; CHECK-NEXT: smov x12, v1.b[0]
-; CHECK-NEXT: umov w13, v0.b[2]
-; CHECK-NEXT: umov w15, v0.b[3]
-; CHECK-NEXT: umov w16, v0.b[4]
-; CHECK-NEXT: smov x17, v1.b[3]
-; CHECK-NEXT: lsl w9, w9, #24
-; CHECK-NEXT: lsl w13, w13, #24
-; CHECK-NEXT: smull x10, w10, w9
-; CHECK-NEXT: lsl w9, w11, #24
-; CHECK-NEXT: smov x11, v1.b[2]
-; CHECK-NEXT: lsl w15, w15, #24
-; CHECK-NEXT: lsl w16, w16, #24
-; CHECK-NEXT: smull x12, w12, w9
-; CHECK-NEXT: mov w9, #-2147483648 // =0x80000000
-; CHECK-NEXT: lsr x14, x10, #32
-; CHECK-NEXT: smull x15, w17, w15
-; CHECK-NEXT: umov w17, v0.b[5]
-; CHECK-NEXT: smull x11, w11, w13
-; CHECK-NEXT: extr w10, w14, w10, #2
-; CHECK-NEXT: lsr x13, x12, #32
-; CHECK-NEXT: cmp w14, #1
+; CHECK-NEXT: smull2 v2.8h, v0.16b, v1.16b
+; CHECK-NEXT: smull v3.8h, v0.8b, v1.8b
+; CHECK-NEXT: mov w8, #127 // =0x7f
+; CHECK-NEXT: mul v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: uzp2 v2.16b, v3.16b, v2.16b
+; CHECK-NEXT: shl v1.16b, v2.16b, #6
+; CHECK-NEXT: smov w9, v2.b[1]
+; CHECK-NEXT: smov w11, v2.b[0]
+; CHECK-NEXT: usra v1.16b, v0.16b, #2
+; CHECK-NEXT: cmp w9, #1
+; CHECK-NEXT: umov w10, v1.b[1]
+; CHECK-NEXT: umov w12, v1.b[0]
+; CHECK-NEXT: umov w13, v1.b[2]
; CHECK-NEXT: csel w10, w8, w10, gt
-; CHECK-NEXT: cmn w14, #2
-; CHECK-NEXT: extr w12, w13, w12, #2
+; CHECK-NEXT: cmn w9, #2
+; CHECK-NEXT: mov w9, #-128 // =0xffffff80
; CHECK-NEXT: csel w10, w9, w10, lt
-; CHECK-NEXT: cmp w13, #1
-; CHECK-NEXT: smov x14, v1.b[4]
+; CHECK-NEXT: cmp w11, #1
; CHECK-NEXT: csel w12, w8, w12, gt
-; CHECK-NEXT: cmn w13, #2
-; CHECK-NEXT: lsr x13, x11, #32
+; CHECK-NEXT: cmn w11, #2
+; CHECK-NEXT: smov w11, v2.b[2]
+; CHECK-NEXT: csel w12, w9, w12, lt
+; CHECK-NEXT: fmov s0, w12
+; CHECK-NEXT: cmp w11, #1
+; CHECK-NEXT: mov v0.b[1], w10
+; CHECK-NEXT: smov w10, v2.b[3]
+; CHECK-NEXT: csel w12, w8, w13, gt
+; CHECK-NEXT: cmn w11, #2
+; CHECK-NEXT: umov w11, v1.b[3]
; CHECK-NEXT: csel w12, w9, w12, lt
-; CHECK-NEXT: asr w10, w10, #24
-; CHECK-NEXT: asr w12, w12, #24
-; CHECK-NEXT: extr w11, w13, w11, #2
-; CHECK-NEXT: cmp w13, #1
-; CHECK-NEXT: smull x14, w14, w16
-; CHECK-NEXT: smov x16, v1.b[5]
-; CHECK-NEXT: fmov s2, w12
-; CHECK-NEXT: lsr x12, x15, #32
+; CHECK-NEXT: mov v0.b[2], w12
+; CHECK-NEXT: cmp w10, #1
+; CHECK-NEXT: smov w12, v2.b[4]
; CHECK-NEXT: csel w11, w8, w11, gt
-; CHECK-NEXT: cmn w13, #2
-; CHECK-NEXT: umov w13, v0.b[6]
+; CHECK-NEXT: cmn w10, #2
+; CHECK-NEXT: umov w10, v1.b[4]
; CHECK-NEXT: csel w11, w9, w11, lt
-; CHECK-NEXT: extr w15, w12, w15, #2
+; CHECK-NEXT: mov v0.b[3], w11
; CHECK-NEXT: cmp w12, #1
-; CHECK-NEXT: mov v2.b[1], w10
-; CHECK-NEXT: lsl w10, w17, #24
-; CHECK-NEXT: asr w11, w11, #24
-; CHECK-NEXT: smov x17, v1.b[6]
-; CHECK-NEXT: csel w15, w8, w15, gt
-; CHECK-NEXT: cmn w12, #2
-; CHECK-NEXT: smull x10, w16, w10
-; CHECK-NEXT: lsr x16, x14, #32
-; CHECK-NEXT: umov w12, v0.b[7]
-; CHECK-NEXT: mov v2.b[2], w11
-; CHECK-NEXT: lsl w11, w13, #24
-; CHECK-NEXT: extr w13, w16, w14, #2
-; CHECK-NEXT: csel w14, w9, w15, lt
-; CHECK-NEXT: cmp w16, #1
-; CHECK-NEXT: smov x15, v1.b[7]
-; CHECK-NEXT: asr w14, w14, #24
-; CHECK-NEXT: smull x11, w17, w11
-; CHECK-NEXT: lsr x17, x10, #32
-; CHECK-NEXT: csel w13, w8, w13, gt
-; CHECK-NEXT: cmn w16, #2
-; CHECK-NEXT: umov w16, v0.b[8]
-; CHECK-NEXT: mov v2.b[3], w14
-; CHECK-NEXT: csel w13, w9, w13, lt
-; CHECK-NEXT: lsl w12, w12, #24
-; CHECK-NEXT: extr w10, w17, w10, #2
-; CHECK-NEXT: asr w13, w13, #24
-; CHECK-NEXT: cmp w17, #1
-; CHECK-NEXT: smov x14, v1.b[8]
-; CHECK-NEXT: smull x12, w15, w12
-; CHECK-NEXT: lsr x15, x11, #32
+; CHECK-NEXT: smov w11, v2.b[5]
; CHECK-NEXT: csel w10, w8, w10, gt
-; CHECK-NEXT: cmn w17, #2
-; CHECK-NEXT: umov w17, v0.b[9]
-; CHECK-NEXT: mov v2.b[4], w13
+; CHECK-NEXT: cmn w12, #2
+; CHECK-NEXT: umov w12, v1.b[5]
; CHECK-NEXT: csel w10, w9, w10, lt
-; CHECK-NEXT: lsl w13, w16, #24
-; CHECK-NEXT: extr w11, w15, w11, #2
-; CHECK-NEXT: asr w10, w10, #24
-; CHECK-NEXT: cmp w15, #1
-; CHECK-NEXT: smov x16, v1.b[9]
-; CHECK-NEXT: smull x13, w14, w13
-; CHECK-NEXT: lsr x14, x12, #32
-; CHECK-NEXT: csel w11, w8, w11, gt
-; CHECK-NEXT: cmn w15, #2
-; CHECK-NEXT: umov w15, v0.b[10]
-; CHECK-NEXT: mov v2.b[5], w10
-; CHECK-NEXT: csel w11, w9, w11, lt
-; CHECK-NEXT: lsl w10, w17, #24
-; CHECK-NEXT: extr w12, w14, w12, #2
-; CHECK-NEXT: asr w11, w11, #24
-; CHECK-NEXT: cmp w14, #1
-; CHECK-NEXT: smov x17, v1.b[10]
-; CHECK-NEXT: smull x10, w16, w10
-; CHECK-NEXT: lsr x16, x13, #32
+; CHECK-NEXT: mov v0.b[4], w10
+; CHECK-NEXT: cmp w11, #1
+; CHECK-NEXT: smov w10, v2.b[6]
; CHECK-NEXT: csel w12, w8, w12, gt
-; CHECK-NEXT: cmn w14, #2
-; CHECK-NEXT: umov w14, v0.b[11]
-; CHECK-NEXT: mov v2.b[6], w11
+; CHECK-NEXT: cmn w11, #2
+; CHECK-NEXT: umov w11, v1.b[6]
; CHECK-NEXT: csel w12, w9, w12, lt
-; CHECK-NEXT: lsl w11, w15, #24
-; CHECK-NEXT: extr w13, w16, w13, #2
-; CHECK-NEXT: asr w12, w12, #24
-; CHECK-NEXT: cmp w16, #1
-; CHECK-NEXT: smov x15, v1.b[11]
-; CHECK-NEXT: smull x11, w17, w11
-; CHECK-NEXT: lsr x17, x10, #32
-; CHECK-NEXT: csel w13, w8, w13, gt
-; CHECK-NEXT: cmn w16, #2
-; CHECK-NEXT: umov w16, v0.b[12]
-; CHECK-NEXT: mov v2.b[7], w12
-; CHECK-NEXT: csel w13, w9, w13, lt
-; CHECK-NEXT: lsl w12, w14, #24
-; CHECK-NEXT: extr w10, w17, w10, #2
-; CHECK-NEXT: asr w13, w13, #24
-; CHECK-NEXT: cmp w17, #1
-; CHECK-NEXT: smull x12, w15, w12
-; CHECK-NEXT: lsr x15, x11, #32
-; CHECK-NEXT: smov x14, v1.b[12]
+; CHECK-NEXT: mov v0.b[5], w12
+; CHECK-NEXT: cmp w10, #1
+; CHECK-NEXT: smov w12, v2.b[7]
+; CHECK-NEXT: csel w11, w8, w11, gt
+; CHECK-NEXT: cmn w10, #2
+; CHECK-NEXT: umov w10, v1.b[7]
+; CHECK-NEXT: csel w11, w9, w11, lt
+; CHECK-NEXT: mov v0.b[6], w11
+; CHECK-NEXT: cmp w12, #1
+; CHECK-NEXT: smov w11, v2.b[8]
; CHECK-NEXT: csel w10, w8, w10, gt
-; CHECK-NEXT: cmn w17, #2
-; CHECK-NEXT: lsl w16, w16, #24
-; CHECK-NEXT: mov v2.b[8], w13
+; CHECK-NEXT: cmn w12, #2
+; CHECK-NEXT: umov w12, v1.b[8]
; CHECK-NEXT: csel w10, w9, w10, lt
-; CHECK-NEXT: extr w11, w15, w11, #2
-; CHECK-NEXT: asr w10, w10, #24
-; CHECK-NEXT: umov w13, v0.b[13]
-; CHECK-NEXT: cmp w15, #1
+; CHECK-NEXT: mov v0.b[7], w10
+; CHECK-NEXT: cmp w11, #1
+; CHECK-NEXT: smov w10, v2.b[9]
+; CHECK-NEXT: csel w12, w8, w12, gt
+; CHECK-NEXT: cmn w11, #2
+; CHECK-NEXT: umov w11, v1.b[9]
+; CHECK-NEXT: csel w12, w9, w12, lt
+; CHECK-NEXT: mov v0.b[8], w12
+; CHECK-NEXT: cmp w10, #1
+; CHECK-NEXT: smov w12, v2.b[10]
; CHECK-NEXT: csel w11, w8, w11, gt
-; CHECK-NEXT: cmn w15, #2
-; CHECK-NEXT: lsr x15, x12, #32
-; CHECK-NEXT: smull x14, w14, w16
-; CHECK-NEXT: smov x16, v1.b[13]
+; CHECK-NEXT: cmn w10, #2
+; CHECK-NEXT: umov w10, v1.b[10]
; CHECK-NEXT: csel w11, w9, w11, lt
-; CHECK-NEXT: mov v2.b[9], w10
-; CHECK-NEXT: extr w12, w15, w12, #2
-; CHECK-NEXT: asr w11, w11, #24
-; CHECK-NEXT: lsl w10, w13, #24
-; CHECK-NEXT: cmp w15, #1
-; CHECK-NEXT: umov w13, v0.b[14]
+; CHECK-NEXT: mov v0.b[9], w11
+; CHECK-NEXT: cmp w12, #1
+; CHECK-NEXT: smov w11, v2.b[11]
+; CHECK-NEXT: csel w10, w8, w10, gt
+; CHECK-NEXT: cmn w12, #2
+; CHECK-NEXT: umov w12, v1.b[11]
+; CHECK-NEXT: csel w10, w9, w10, lt
+; CHECK-NEXT: mov v0.b[10], w10
+; CHECK-NEXT: cmp w11, #1
+; CHECK-NEXT: smov w10, v2.b[12]
; CHECK-NEXT: csel w12, w8, w12, gt
-; CHECK-NEXT: cmn w15, #2
-; CHECK-NEXT: smull x10, w16, w10
-; CHECK-NEXT: lsr x16, x14, #32
+; CHECK-NEXT: cmn w11, #2
+; CHECK-NEXT: umov w11, v1.b[12]
; CHECK-NEXT: csel w12, w9, w12, lt
-; CHECK-NEXT: mov v2.b[10], w11
-; CHECK-NEXT: smov x11, v1.b[14]
-; CHECK-NEXT: asr w12, w12, #24
-; CHECK-NEXT: extr w14, w16, w14, #2
-; CHECK-NEXT: cmp w16, #1
-; CHECK-NEXT: lsl w13, w13, #24
-; CHECK-NEXT: lsr x15, x10, #32
-; CHECK-NEXT: csel w14, w8, w14, gt
-; CHECK-NEXT: cmn w16, #2
-; CHECK-NEXT: mov v2.b[11], w12
-; CHECK-NEXT: umov w12, v0.b[15]
-; CHECK-NEXT: smull x11, w11, w13
-; CHECK-NEXT: csel w13, w9, w14, lt
-; CHECK-NEXT: smov x14, v1.b[15]
-; CHECK-NEXT: extr w10, w15, w10, #2
-; CHECK-NEXT: asr w13, w13, #24
-; CHECK-NEXT: cmp w15, #1
+; CHECK-NEXT: mov v0.b[11], w12
+; CHECK-NEXT: cmp w10, #1
+; CHECK-NEXT: smov w12, v2.b[13]
+; CHECK-NEXT: csel w11, w8, w11, gt
+; CHECK-NEXT: cmn w10, #2
+; CHECK-NEXT: umov w10, v1.b[13]
+; CHECK-NEXT: csel w11, w9, w11, lt
+; CHECK-NEXT: mov v0.b[12], w11
+; CHECK-NEXT: cmp w12, #1
+; CHECK-NEXT: smov w11, v2.b[14]
; CHECK-NEXT: csel w10, w8, w10, gt
-; CHECK-NEXT: cmn w15, #2
-; CHECK-NEXT: lsl w12, w12, #24
-; CHECK-NEXT: mov v2.b[12], w13
-; CHECK-NEXT: lsr x13, x11, #32
-; CHECK-NEXT: csel w10, w9, w10, lt
-; CHECK-NEXT: smull x12, w14, w12
-; CHECK-NEXT: asr w10, w10, #24
-; CHECK-NEXT: extr w11, w13, w11, #2
-; CHECK-NEXT: cmp w13, #1
-; CHECK-NEXT: mov v2.b[13], w10
-; CHECK-NEXT: csel w10, w8, w11, gt
-; CHECK-NEXT: cmn w13, #2
-; CHECK-NEXT: lsr x11, x12, #32
+; CHECK-NEXT: cmn w12, #2
+; CHECK-NEXT: umov w12, v1.b[14]
; CHECK-NEXT: csel w10, w9, w10, lt
-; CHECK-NEXT: asr w10, w10, #24
-; CHECK-NEXT: extr w12, w11, w12, #2
+; CHECK-NEXT: mov v0.b[13], w10
; CHECK-NEXT: cmp w11, #1
-; CHECK-NEXT: mov v2.b[14], w10
-; CHECK-NEXT: csel w8, w8, w12, gt
+; CHECK-NEXT: smov w10, v2.b[15]
+; CHECK-NEXT: csel w12, w8, w12, gt
; CHECK-NEXT: cmn w11, #2
+; CHECK-NEXT: umov w11, v1.b[15]
+; CHECK-NEXT: csel w12, w9, w12, lt
+; CHECK-NEXT: mov v0.b[14], w12
+; CHECK-NEXT: cmp w10, #1
+; CHECK-NEXT: csel w8, w8, w11, gt
+; CHECK-NEXT: cmn w10, #2
; CHECK-NEXT: csel w8, w9, w8, lt
-; CHECK-NEXT: asr w8, w8, #24
-; CHECK-NEXT: mov v2.b[15], w8
-; CHECK-NEXT: mov v0.16b, v2.16b
+; CHECK-NEXT: mov v0.b[15], w8
; CHECK-NEXT: ret
%tmp = call <16 x i8> @llvm.smul.fix.sat.v16i8(<16 x i8> %x, <16 x i8> %y, i32 2)
ret <16 x i8> %tmp
@@ -477,57 +365,40 @@ define <16 x i8> @vec_v16i8(<16 x i8> %x, <16 x i8> %y) {
define <4 x i16> @vec_v4i16(<4 x i16> %x, <4 x i16> %y) {
; CHECK-LABEL: vec_v4i16:
; CHECK: // %bb.0:
-; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT: umov w9, v0.h[0]
-; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1
-; CHECK-NEXT: smov x10, v1.h[0]
-; CHECK-NEXT: mov w8, #2147483647 // =0x7fffffff
-; CHECK-NEXT: umov w11, v0.h[1]
-; CHECK-NEXT: smov x12, v1.h[1]
-; CHECK-NEXT: smov x13, v1.h[2]
-; CHECK-NEXT: umov w14, v0.h[3]
-; CHECK-NEXT: lsl w9, w9, #16
-; CHECK-NEXT: lsl w11, w11, #16
-; CHECK-NEXT: smull x9, w10, w9
-; CHECK-NEXT: umov w10, v0.h[2]
-; CHECK-NEXT: lsl w14, w14, #16
-; CHECK-NEXT: smull x11, w12, w11
-; CHECK-NEXT: lsr x12, x9, #32
-; CHECK-NEXT: lsl w10, w10, #16
-; CHECK-NEXT: lsr x15, x11, #32
-; CHECK-NEXT: extr w9, w12, w9, #10
-; CHECK-NEXT: cmp w12, #511
-; CHECK-NEXT: smull x10, w13, w10
-; CHECK-NEXT: mov w13, #-2147483648 // =0x80000000
-; CHECK-NEXT: extr w11, w15, w11, #10
-; CHECK-NEXT: csel w9, w8, w9, gt
-; CHECK-NEXT: cmn w12, #512
-; CHECK-NEXT: smov x12, v1.h[3]
-; CHECK-NEXT: csel w9, w13, w9, lt
-; CHECK-NEXT: cmp w15, #511
-; CHECK-NEXT: asr w9, w9, #16
-; CHECK-NEXT: csel w11, w8, w11, gt
-; CHECK-NEXT: cmn w15, #512
-; CHECK-NEXT: lsr x15, x10, #32
-; CHECK-NEXT: csel w11, w13, w11, lt
-; CHECK-NEXT: smull x12, w12, w14
-; CHECK-NEXT: fmov s0, w9
-; CHECK-NEXT: asr w11, w11, #16
-; CHECK-NEXT: extr w9, w15, w10, #10
-; CHECK-NEXT: cmp w15, #511
-; CHECK-NEXT: mov v0.h[1], w11
-; CHECK-NEXT: csel w9, w8, w9, gt
-; CHECK-NEXT: cmn w15, #512
-; CHECK-NEXT: lsr x10, x12, #32
-; CHECK-NEXT: csel w9, w13, w9, lt
-; CHECK-NEXT: asr w9, w9, #16
-; CHECK-NEXT: extr w11, w10, w12, #10
+; CHECK-NEXT: smull v0.4s, v0.4h, v1.4h
+; CHECK-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-NEXT: shrn v1.4h, v0.4s, #16
+; CHECK-NEXT: xtn v0.4h, v0.4s
+; CHECK-NEXT: shl v2.4h, v1.4h, #6
+; CHECK-NEXT: smov w9, v1.h[1]
+; CHECK-NEXT: smov w11, v1.h[0]
+; CHECK-NEXT: usra v2.4h, v0.4h, #10
+; CHECK-NEXT: cmp w9, #511
+; CHECK-NEXT: umov w10, v2.h[1]
+; CHECK-NEXT: umov w12, v2.h[0]
+; CHECK-NEXT: umov w13, v2.h[2]
+; CHECK-NEXT: csel w10, w8, w10, gt
+; CHECK-NEXT: cmn w9, #512
+; CHECK-NEXT: mov w9, #-32768 // =0xffff8000
+; CHECK-NEXT: csel w10, w9, w10, lt
+; CHECK-NEXT: cmp w11, #511
+; CHECK-NEXT: csel w12, w8, w12, gt
+; CHECK-NEXT: cmn w11, #512
+; CHECK-NEXT: smov w11, v1.h[2]
+; CHECK-NEXT: csel w12, w9, w12, lt
+; CHECK-NEXT: fmov s0, w12
+; CHECK-NEXT: cmp w11, #511
+; CHECK-NEXT: mov v0.h[1], w10
+; CHECK-NEXT: smov w10, v1.h[3]
+; CHECK-NEXT: csel w12, w8, w13, gt
+; CHECK-NEXT: cmn w11, #512
+; CHECK-NEXT: umov w11, v2.h[3]
+; CHECK-NEXT: csel w12, w9, w12, lt
+; CHECK-NEXT: mov v0.h[2], w12
; CHECK-NEXT: cmp w10, #511
-; CHECK-NEXT: mov v0.h[2], w9
; CHECK-NEXT: csel w8, w8, w11, gt
; CHECK-NEXT: cmn w10, #512
-; CHECK-NEXT: csel w8, w13, w8, lt
-; CHECK-NEXT: asr w8, w8, #16
+; CHECK-NEXT: csel w8, w9, w8, lt
; CHECK-NEXT: mov v0.h[3], w8
; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-NEXT: ret
@@ -538,105 +409,70 @@ define <4 x i16> @vec_v4i16(<4 x i16> %x, <4 x i16> %y) {
define <8 x i16> @vec_v8i16(<8 x i16> %x, <8 x i16> %y) {
; CHECK-LABEL: vec_v8i16:
; CHECK: // %bb.0:
-; CHECK-NEXT: umov w8, v0.h[1]
-; CHECK-NEXT: smov x9, v1.h[1]
-; CHECK-NEXT: umov w10, v0.h[0]
-; CHECK-NEXT: smov x11, v1.h[0]
-; CHECK-NEXT: umov w13, v0.h[2]
-; CHECK-NEXT: umov w15, v0.h[3]
-; CHECK-NEXT: smov x16, v1.h[3]
-; CHECK-NEXT: lsl w8, w8, #16
-; CHECK-NEXT: lsl w13, w13, #16
-; CHECK-NEXT: lsl w15, w15, #16
-; CHECK-NEXT: smull x12, w9, w8
-; CHECK-NEXT: lsl w8, w10, #16
-; CHECK-NEXT: smov x10, v1.h[2]
-; CHECK-NEXT: mov w9, #2147483647 // =0x7fffffff
-; CHECK-NEXT: smull x15, w16, w15
-; CHECK-NEXT: umov w16, v0.h[5]
-; CHECK-NEXT: smull x11, w11, w8
-; CHECK-NEXT: mov w8, #-2147483648 // =0x80000000
-; CHECK-NEXT: lsr x14, x12, #32
-; CHECK-NEXT: smull x10, w10, w13
-; CHECK-NEXT: umov w13, v0.h[4]
-; CHECK-NEXT: extr w12, w14, w12, #2
-; CHECK-NEXT: lsr x17, x11, #32
-; CHECK-NEXT: cmp w14, #1
-; CHECK-NEXT: csel w12, w9, w12, gt
-; CHECK-NEXT: cmn w14, #2
-; CHECK-NEXT: extr w11, w17, w11, #2
-; CHECK-NEXT: csel w12, w8, w12, lt
-; CHECK-NEXT: cmp w17, #1
-; CHECK-NEXT: smov x14, v1.h[4]
-; CHECK-NEXT: csel w11, w9, w11, gt
-; CHECK-NEXT: cmn w17, #2
-; CHECK-NEXT: lsr x17, x10, #32
-; CHECK-NEXT: csel w11, w8, w11, lt
-; CHECK-NEXT: lsl w13, w13, #16
-; CHECK-NEXT: asr w12, w12, #16
-; CHECK-NEXT: asr w11, w11, #16
-; CHECK-NEXT: extr w10, w17, w10, #2
-; CHECK-NEXT: cmp w17, #1
-; CHECK-NEXT: smull x13, w14, w13
-; CHECK-NEXT: lsr x14, x15, #32
-; CHECK-NEXT: fmov s2, w11
-; CHECK-NEXT: smov x11, v1.h[5]
-; CHECK-NEXT: csel w10, w9, w10, gt
-; CHECK-NEXT: cmn w17, #2
-; CHECK-NEXT: extr w15, w14, w15, #2
-; CHECK-NEXT: csel w10, w8, w10, lt
-; CHECK-NEXT: cmp w14, #1
-; CHECK-NEXT: mov v2.h[1], w12
-; CHECK-NEXT: lsl w12, w16, #16
-; CHECK-NEXT: asr w10, w10, #16
-; CHECK-NEXT: umov w16, v0.h[6]
-; CHECK-NEXT: csel w15, w9, w15, gt
-; CHECK-NEXT: cmn w14, #2
-; CHECK-NEXT: smull x11, w11, w12
-; CHECK-NEXT: lsr x12, x13, #32
-; CHECK-NEXT: csel w14, w8, w15, lt
-; CHECK-NEXT: asr w14, w14, #16
-; CHECK-NEXT: mov v2.h[2], w10
-; CHECK-NEXT: smov x10, v1.h[6]
-; CHECK-NEXT: extr w13, w12, w13, #2
+; CHECK-NEXT: smull2 v2.4s, v0.8h, v1.8h
+; CHECK-NEXT: smull v3.4s, v0.4h, v1.4h
+; CHECK-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-NEXT: mul v0.8h, v0.8h, v1.8h
+; CHECK-NEXT: uzp2 v2.8h, v3.8h, v2.8h
+; CHECK-NEXT: shl v1.8h, v2.8h, #14
+; CHECK-NEXT: smov w9, v2.h[1]
+; CHECK-NEXT: smov w11, v2.h[0]
+; CHECK-NEXT: usra v1.8h, v0.8h, #2
+; CHECK-NEXT: cmp w9, #1
+; CHECK-NEXT: umov w10, v1.h[1]
+; CHECK-NEXT: umov w12, v1.h[0]
+; CHECK-NEXT: umov w13, v1.h[2]
+; CHECK-NEXT: csel w10, w8, w10, gt
+; CHECK-NEXT: cmn w9, #2
+; CHECK-NEXT: mov w9, #-32768 // =0xffff8000
+; CHECK-NEXT: csel w10, w9, w10, lt
+; CHECK-NEXT: cmp w11, #1
+; CHECK-NEXT: csel w12, w8, w12, gt
+; CHECK-NEXT: cmn w11, #2
+; CHECK-NEXT: smov w11, v2.h[2]
+; CHECK-NEXT: csel w12, w9, w12, lt
+; CHECK-NEXT: fmov s0, w12
+; CHECK-NEXT: cmp w11, #1
+; CHECK-NEXT: mov v0.h[1], w10
+; CHECK-NEXT: smov w10, v2.h[3]
+; CHECK-NEXT: csel w12, w8, w13, gt
+; CHECK-NEXT: cmn w11, #2
+; CHECK-NEXT: umov w11, v1.h[3]
+; CHECK-NEXT: csel w12, w9, w12, lt
+; CHECK-NEXT: mov v0.h[2], w12
+; CHECK-NEXT: cmp w10, #1
+; CHECK-NEXT: smov w12, v2.h[4]
+; CHECK-NEXT: csel w11, w8, w11, gt
+; CHECK-NEXT: cmn w10, #2
+; CHECK-NEXT: umov w10, v1.h[4]
+; CHECK-NEXT: csel w11, w9, w11, lt
+; CHECK-NEXT: mov v0.h[3], w11
; CHECK-NEXT: cmp w12, #1
-; CHECK-NEXT: lsl w15, w16, #16
-; CHECK-NEXT: lsr x16, x11, #32
-; CHECK-NEXT: csel w13, w9, w13, gt
+; CHECK-NEXT: smov w11, v2.h[5]
+; CHECK-NEXT: csel w10, w8, w10, gt
; CHECK-NEXT: cmn w12, #2
-; CHECK-NEXT: umov w12, v0.h[7]
-; CHECK-NEXT: csel w13, w8, w13, lt
-; CHECK-NEXT: extr w11, w16, w11, #2
-; CHECK-NEXT: cmp w16, #1
-; CHECK-NEXT: mov v2.h[3], w14
-; CHECK-NEXT: smull x10, w10, w15
-; CHECK-NEXT: smov x14, v1.h[7]
-; CHECK-NEXT: asr w13, w13, #16
-; CHECK-NEXT: csel w11, w9, w11, gt
-; CHECK-NEXT: cmn w16, #2
-; CHECK-NEXT: lsl w12, w12, #16
-; CHECK-NEXT: csel w11, w8, w11, lt
-; CHECK-NEXT: asr w11, w11, #16
-; CHECK-NEXT: mov v2.h[4], w13
-; CHECK-NEXT: lsr x13, x10, #32
-; CHECK-NEXT: smull x12, w14, w12
-; CHECK-NEXT: extr w10, w13, w10, #2
-; CHECK-NEXT: cmp w13, #1
-; CHECK-NEXT: mov v2.h[5], w11
-; CHECK-NEXT: csel w10, w9, w10, gt
-; CHECK-NEXT: cmn w13, #2
-; CHECK-NEXT: lsr x11, x12, #32
-; CHECK-NEXT: csel w10, w8, w10, lt
-; CHECK-NEXT: asr w10, w10, #16
-; CHECK-NEXT: extr w12, w11, w12, #2
+; CHECK-NEXT: umov w12, v1.h[5]
+; CHECK-NEXT: csel w10, w9, w10, lt
+; CHECK-NEXT: mov v0.h[4], w10
; CHECK-NEXT: cmp w11, #1
-; CHECK-NEXT: mov v2.h[6], w10
-; CHECK-NEXT: csel w9, w9, w12, gt
+; CHECK-NEXT: smov w10, v2.h[6]
+; CHECK-NEXT: csel w12, w8, w12, gt
; CHECK-NEXT: cmn w11, #2
-; CHECK-NEXT: csel w8, w8, w9, lt
-; CHECK-NEXT: asr w8, w8, #16
-; CHECK-NEXT: mov v2.h[7], w8
-; CHECK-NEXT: mov v0.16b, v2.16b
+; CHECK-NEXT: umov w11, v1.h[6]
+; CHECK-NEXT: csel w12, w9, w12, lt
+; CHECK-NEXT: mov v0.h[5], w12
+; CHECK-NEXT: cmp w10, #1
+; CHECK-NEXT: smov w12, v2.h[7]
+; CHECK-NEXT: csel w11, w8, w11, gt
+; CHECK-NEXT: cmn w10, #2
+; CHECK-NEXT: umov w10, v1.h[7]
+; CHECK-NEXT: csel w11, w9, w11, lt
+; CHECK-NEXT: mov v0.h[6], w11
+; CHECK-NEXT: cmp w12, #1
+; CHECK-NEXT: csel w8, w8, w10, gt
+; CHECK-NEXT: cmn w12, #2
+; CHECK-NEXT: csel w8, w9, w8, lt
+; CHECK-NEXT: mov v0.h[7], w8
; CHECK-NEXT: ret
%tmp = call <8 x i16> @llvm.smul.fix.sat.v8i16(<8 x i16> %x, <8 x i16> %y, i32 2)
ret <8 x i16> %tmp
@@ -645,28 +481,23 @@ define <8 x i16> @vec_v8i16(<8 x i16> %x, <8 x i16> %y) {
define <2 x i32> @vec_v2i32(<2 x i32> %x, <2 x i32> %y) {
; CHECK-LABEL: vec_v2i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1
-; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT: mov w9, v1.s[1]
-; CHECK-NEXT: mov w10, v0.s[1]
-; CHECK-NEXT: mov w8, #-2147483648 // =0x80000000
-; CHECK-NEXT: fmov w12, s0
-; CHECK-NEXT: smull x11, w10, w9
-; CHECK-NEXT: eor w9, w10, w9
-; CHECK-NEXT: fmov w10, s1
-; CHECK-NEXT: cmp w9, #0
-; CHECK-NEXT: smull x9, w12, w10
-; CHECK-NEXT: eor w10, w12, w10
-; CHECK-NEXT: cinv w12, w8, pl
-; CHECK-NEXT: cmp x11, w11, sxtw
-; CHECK-NEXT: csel w11, w12, w11, ne
-; CHECK-NEXT: cmp w10, #0
-; CHECK-NEXT: cinv w8, w8, pl
-; CHECK-NEXT: cmp x9, w9, sxtw
-; CHECK-NEXT: csel w8, w8, w9, ne
-; CHECK-NEXT: fmov s0, w8
-; CHECK-NEXT: mov v0.s[1], w11
-; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT: smull v0.2d, v0.2s, v1.2s
+; CHECK-NEXT: shrn v1.2s, v0.2d, #32
+; CHECK-NEXT: xtn v0.2s, v0.2d
+; CHECK-NEXT: fmov w9, s1
+; CHECK-NEXT: mov w8, v1.s[1]
+; CHECK-NEXT: add v2.2s, v1.2s, v1.2s
+; CHECK-NEXT: cmlt v3.2s, v0.2s, #0
+; CHECK-NEXT: asr w9, w9, #31
+; CHECK-NEXT: shl v2.2s, v2.2s, #31
+; CHECK-NEXT: asr w8, w8, #31
+; CHECK-NEXT: cmeq v1.2s, v1.2s, v3.2s
+; CHECK-NEXT: eor w9, w9, #0x7fffffff
+; CHECK-NEXT: fmov s4, w9
+; CHECK-NEXT: eor w8, w8, #0x7fffffff
+; CHECK-NEXT: orr v0.8b, v2.8b, v0.8b
+; CHECK-NEXT: mov v4.s[1], w8
+; CHECK-NEXT: bif v0.8b, v4.8b, v1.8b
; CHECK-NEXT: ret
%tmp = call <2 x i32> @llvm.smul.fix.sat.v2i32(<2 x i32> %x, <2 x i32> %y, i32 0)
ret <2 x i32> %tmp
@@ -675,47 +506,41 @@ define <2 x i32> @vec_v2i32(<2 x i32> %x, <2 x i32> %y) {
define <4 x i32> @vec_v4i32(<4 x i32> %x, <4 x i32> %y) {
; CHECK-LABEL: vec_v4i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov w9, v1.s[1]
-; CHECK-NEXT: mov w10, v0.s[1]
+; CHECK-NEXT: smull2 v2.2d, v0.4s, v1.4s
+; CHECK-NEXT: smull v3.2d, v0.2s, v1.2s
; CHECK-NEXT: mov w8, #2147483647 // =0x7fffffff
-; CHECK-NEXT: fmov w11, s0
-; CHECK-NEXT: mov w12, v1.s[2]
-; CHECK-NEXT: mov w13, v0.s[2]
-; CHECK-NEXT: mov w14, v1.s[3]
-; CHECK-NEXT: mov w16, v0.s[3]
-; CHECK-NEXT: smull x9, w10, w9
-; CHECK-NEXT: fmov w10, s1
-; CHECK-NEXT: smull x10, w11, w10
-; CHECK-NEXT: lsr x11, x9, #32
-; CHECK-NEXT: extr w9, w11, w9, #15
+; CHECK-NEXT: mul v0.4s, v0.4s, v1.4s
+; CHECK-NEXT: uzp2 v2.4s, v3.4s, v2.4s
+; CHECK-NEXT: shl v1.4s, v2.4s, #17
+; CHECK-NEXT: mov w9, v2.s[1]
+; CHECK-NEXT: fmov w11, s2
+; CHECK-NEXT: mov w13, v2.s[2]
+; CHECK-NEXT: usra v1.4s, v0.4s, #15
+; CHECK-NEXT: cmp w9, #4, lsl #12 // =16384
+; CHECK-NEXT: mov w10, v1.s[1]
+; CHECK-NEXT: fmov w12, s1
+; CHECK-NEXT: csel w10, w8, w10, ge
+; CHECK-NEXT: cmn w9, #4, lsl #12 // =16384
+; CHECK-NEXT: mov w9, #-2147483648 // =0x80000000
+; CHECK-NEXT: csel w10, w9, w10, lt
; CHECK-NEXT: cmp w11, #4, lsl #12 // =16384
-; CHECK-NEXT: lsr x15, x10, #32
-; CHECK-NEXT: csel w9, w8, w9, ge
+; CHECK-NEXT: csel w12, w8, w12, ge
; CHECK-NEXT: cmn w11, #4, lsl #12 // =16384
-; CHECK-NEXT: smull x11, w13, w12
-; CHECK-NEXT: mov w12, #-2147483648 // =0x80000000
-; CHECK-NEXT: extr w10, w15, w10, #15
-; CHECK-NEXT: csel w9, w12, w9, lt
-; CHECK-NEXT: cmp w15, #4, lsl #12 // =16384
-; CHECK-NEXT: csel w10, w8, w10, ge
-; CHECK-NEXT: cmn w15, #4, lsl #12 // =16384
-; CHECK-NEXT: lsr x13, x11, #32
-; CHECK-NEXT: csel w10, w12, w10, lt
-; CHECK-NEXT: fmov s0, w10
-; CHECK-NEXT: smull x10, w16, w14
+; CHECK-NEXT: mov w11, v1.s[2]
+; CHECK-NEXT: csel w12, w9, w12, lt
; CHECK-NEXT: cmp w13, #4, lsl #12 // =16384
-; CHECK-NEXT: mov v0.s[1], w9
-; CHECK-NEXT: extr w9, w13, w11, #15
-; CHECK-NEXT: lsr x11, x10, #32
-; CHECK-NEXT: csel w9, w8, w9, ge
+; CHECK-NEXT: fmov s0, w12
+; CHECK-NEXT: mov w12, v1.s[3]
+; CHECK-NEXT: csel w11, w8, w11, ge
; CHECK-NEXT: cmn w13, #4, lsl #12 // =16384
-; CHECK-NEXT: csel w9, w12, w9, lt
-; CHECK-NEXT: cmp w11, #4, lsl #12 // =16384
-; CHECK-NEXT: mov v0.s[2], w9
-; CHECK-NEXT: extr w9, w11, w10, #15
-; CHECK-NEXT: csel w8, w8, w9, ge
-; CHECK-NEXT: cmn w11, #4, lsl #12 // =16384
-; CHECK-NEXT: csel w8, w12, w8, lt
+; CHECK-NEXT: mov v0.s[1], w10
+; CHECK-NEXT: mov w10, v2.s[3]
+; CHECK-NEXT: csel w11, w9, w11, lt
+; CHECK-NEXT: mov v0.s[2], w11
+; CHECK-NEXT: cmp w10, #4, lsl #12 // =16384
+; CHECK-NEXT: csel w8, w8, w12, ge
+; CHECK-NEXT: cmn w10, #4, lsl #12 // =16384
+; CHECK-NEXT: csel w8, w9, w8, lt
; CHECK-NEXT: mov v0.s[3], w8
; CHECK-NEXT: ret
%tmp = call <4 x i32> @llvm.smul.fix.sat.v4i32(<4 x i32> %x, <4 x i32> %y, i32 15)
@@ -725,79 +550,58 @@ define <4 x i32> @vec_v4i32(<4 x i32> %x, <4 x i32> %y) {
define <8 x i32> @vec_v8i32(<8 x i32> %x, <8 x i32> %y) {
; CHECK-LABEL: vec_v8i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov w9, v2.s[1]
-; CHECK-NEXT: mov w10, v0.s[1]
-; CHECK-NEXT: mov w8, #-2147483648 // =0x80000000
-; CHECK-NEXT: fmov w13, s2
-; CHECK-NEXT: fmov w16, s0
-; CHECK-NEXT: mov w11, v2.s[2]
-; CHECK-NEXT: mov w14, v0.s[2]
-; CHECK-NEXT: mov w12, v2.s[3]
-; CHECK-NEXT: mov w15, v0.s[3]
-; CHECK-NEXT: mov w18, v3.s[1]
-; CHECK-NEXT: mov w0, v1.s[1]
-; CHECK-NEXT: smull x17, w10, w9
-; CHECK-NEXT: eor w9, w10, w9
-; CHECK-NEXT: mov w10, v3.s[2]
-; CHECK-NEXT: cmp w9, #0
-; CHECK-NEXT: smull x9, w16, w13
-; CHECK-NEXT: eor w13, w16, w13
-; CHECK-NEXT: cinv w16, w8, pl
-; CHECK-NEXT: cmp x17, w17, sxtw
-; CHECK-NEXT: csel w16, w16, w17, ne
-; CHECK-NEXT: smull x17, w14, w11
-; CHECK-NEXT: cmp w13, #0
-; CHECK-NEXT: eor w11, w14, w11
-; CHECK-NEXT: cinv w13, w8, pl
-; CHECK-NEXT: cmp x9, w9, sxtw
-; CHECK-NEXT: csel w9, w13, w9, ne
-; CHECK-NEXT: cmp w11, #0
-; CHECK-NEXT: smull x14, w15, w12
-; CHECK-NEXT: eor w11, w15, w12
-; CHECK-NEXT: cinv w12, w8, pl
-; CHECK-NEXT: cmp x17, w17, sxtw
-; CHECK-NEXT: eor w13, w0, w18
-; CHECK-NEXT: csel w12, w12, w17, ne
-; CHECK-NEXT: cmp w11, #0
-; CHECK-NEXT: smull x11, w0, w18
-; CHECK-NEXT: fmov w17, s3
-; CHECK-NEXT: fmov w0, s1
-; CHECK-NEXT: cinv w15, w8, pl
-; CHECK-NEXT: cmp x14, w14, sxtw
-; CHECK-NEXT: mov w18, v1.s[2]
-; CHECK-NEXT: csel w14, w15, w14, ne
-; CHECK-NEXT: cmp w13, #0
-; CHECK-NEXT: fmov s0, w9
-; CHECK-NEXT: smull x13, w0, w17
-; CHECK-NEXT: cinv w15, w8, pl
-; CHECK-NEXT: cmp x11, w11, sxtw
-; CHECK-NEXT: eor w17, w0, w17
-; CHECK-NEXT: csel w11, w15, w11, ne
-; CHECK-NEXT: mov w0, v1.s[3]
-; CHECK-NEXT: cmp w17, #0
-; CHECK-NEXT: mov w17, v3.s[3]
-; CHECK-NEXT: smull x9, w18, w10
-; CHECK-NEXT: cinv w15, w8, pl
-; CHECK-NEXT: cmp x13, w13, sxtw
-; CHECK-NEXT: mov v0.s[1], w16
-; CHECK-NEXT: csel w13, w15, w13, ne
-; CHECK-NEXT: eor w10, w18, w10
-; CHECK-NEXT: fmov s1, w13
-; CHECK-NEXT: cmp w10, #0
-; CHECK-NEXT: cinv w10, w8, pl
-; CHECK-NEXT: cmp x9, w9, sxtw
-; CHECK-NEXT: csel w9, w10, w9, ne
-; CHECK-NEXT: smull x10, w0, w17
-; CHECK-NEXT: mov v0.s[2], w12
-; CHECK-NEXT: mov v1.s[1], w11
-; CHECK-NEXT: mov v0.s[3], w14
-; CHECK-NEXT: mov v1.s[2], w9
-; CHECK-NEXT: eor w9, w0, w17
-; CHECK-NEXT: cmp w9, #0
-; CHECK-NEXT: cinv w8, w8, pl
-; CHECK-NEXT: cmp x10, w10, sxtw
-; CHECK-NEXT: csel w8, w8, w10, ne
-; CHECK-NEXT: mov v1.s[3], w8
+; CHECK-NEXT: smull2 v5.2d, v0.4s, v2.4s
+; CHECK-NEXT: smull v6.2d, v0.2s, v2.2s
+; CHECK-NEXT: smull2 v4.2d, v1.4s, v3.4s
+; CHECK-NEXT: smull v7.2d, v1.2s, v3.2s
+; CHECK-NEXT: mul v0.4s, v0.4s, v2.4s
+; CHECK-NEXT: mul v1.4s, v1.4s, v3.4s
+; CHECK-NEXT: uzp2 v5.4s, v6.4s, v5.4s
+; CHECK-NEXT: uzp2 v4.4s, v7.4s, v4.4s
+; CHECK-NEXT: cmlt v16.4s, v0.4s, #0
+; CHECK-NEXT: cmlt v17.4s, v1.4s, #0
+; CHECK-NEXT: fmov w10, s5
+; CHECK-NEXT: mov w8, v5.s[1]
+; CHECK-NEXT: mov w12, v5.s[2]
+; CHECK-NEXT: fmov w11, s4
+; CHECK-NEXT: mov w9, v4.s[1]
+; CHECK-NEXT: mov w13, v4.s[2]
+; CHECK-NEXT: mov w14, v5.s[3]
+; CHECK-NEXT: add v2.4s, v5.4s, v5.4s
+; CHECK-NEXT: add v3.4s, v4.4s, v4.4s
+; CHECK-NEXT: asr w10, w10, #31
+; CHECK-NEXT: cmeq v5.4s, v5.4s, v16.4s
+; CHECK-NEXT: asr w11, w11, #31
+; CHECK-NEXT: asr w8, w8, #31
+; CHECK-NEXT: eor w10, w10, #0x7fffffff
+; CHECK-NEXT: asr w9, w9, #31
+; CHECK-NEXT: shl v2.4s, v2.4s, #31
+; CHECK-NEXT: fmov s6, w10
+; CHECK-NEXT: eor w10, w11, #0x7fffffff
+; CHECK-NEXT: eor w8, w8, #0x7fffffff
+; CHECK-NEXT: fmov s7, w10
+; CHECK-NEXT: eor w9, w9, #0x7fffffff
+; CHECK-NEXT: mov w10, v4.s[3]
+; CHECK-NEXT: shl v3.4s, v3.4s, #31
+; CHECK-NEXT: cmeq v4.4s, v4.4s, v17.4s
+; CHECK-NEXT: orr v0.16b, v2.16b, v0.16b
+; CHECK-NEXT: mov v6.s[1], w8
+; CHECK-NEXT: asr w8, w12, #31
+; CHECK-NEXT: mov v7.s[1], w9
+; CHECK-NEXT: asr w9, w13, #31
+; CHECK-NEXT: eor w8, w8, #0x7fffffff
+; CHECK-NEXT: orr v1.16b, v3.16b, v1.16b
+; CHECK-NEXT: eor w9, w9, #0x7fffffff
+; CHECK-NEXT: mov v6.s[2], w8
+; CHECK-NEXT: asr w8, w14, #31
+; CHECK-NEXT: mov v7.s[2], w9
+; CHECK-NEXT: asr w9, w10, #31
+; CHECK-NEXT: eor w8, w8, #0x7fffffff
+; CHECK-NEXT: eor w9, w9, #0x7fffffff
+; CHECK-NEXT: mov v6.s[3], w8
+; CHECK-NEXT: mov v7.s[3], w9
+; CHECK-NEXT: bif v0.16b, v6.16b, v5.16b
+; CHECK-NEXT: bif v1.16b, v7.16b, v4.16b
; CHECK-NEXT: ret
%tmp = call <8 x i32> @llvm.smul.fix.sat.v8i32(<8 x i32> %x, <8 x i32> %y, i32 0)
ret <8 x i32> %tmp
diff --git a/llvm/test/CodeGen/AArch64/umul_fix_sat.ll b/llvm/test/CodeGen/AArch64/umul_fix_sat.ll
index 3b319ca5a9676..9668a866697e7 100644
--- a/llvm/test/CodeGen/AArch64/umul_fix_sat.ll
+++ b/llvm/test/CodeGen/AArch64/umul_fix_sat.ll
@@ -131,89 +131,52 @@ define i64 @func8(i64 %x, i64 %y) {
define <8 x i8> @vec_v8i8(<8 x i8> %x, <8 x i8> %y) {
; CHECK-LABEL: vec_v8i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT: umov w8, v0.b[0]
-; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1
-; CHECK-NEXT: umov w10, v1.b[0]
-; CHECK-NEXT: umov w9, v0.b[1]
-; CHECK-NEXT: umov w11, v1.b[1]
-; CHECK-NEXT: umov w12, v0.b[2]
-; CHECK-NEXT: lsl w8, w8, #24
-; CHECK-NEXT: lsl w9, w9, #24
-; CHECK-NEXT: lsl w12, w12, #24
-; CHECK-NEXT: umull x8, w8, w10
-; CHECK-NEXT: umov w10, v1.b[2]
-; CHECK-NEXT: umull x9, w9, w11
-; CHECK-NEXT: umov w11, v0.b[3]
-; CHECK-NEXT: lsr x13, x8, #32
-; CHECK-NEXT: lsr x14, x9, #32
-; CHECK-NEXT: umull x10, w12, w10
-; CHECK-NEXT: umov w12, v1.b[3]
-; CHECK-NEXT: extr w8, w13, w8, #2
-; CHECK-NEXT: cmp w13, #3
-; CHECK-NEXT: umov w13, v0.b[4]
-; CHECK-NEXT: extr w9, w14, w9, #2
-; CHECK-NEXT: lsl w11, w11, #24
-; CHECK-NEXT: csinv w8, w8, wzr, ls
-; CHECK-NEXT: cmp w14, #3
-; CHECK-NEXT: lsr x14, x10, #32
-; CHECK-NEXT: lsr w8, w8, #24
-; CHECK-NEXT: csinv w9, w9, wzr, ls
-; CHECK-NEXT: umull x11, w11, w12
-; CHECK-NEXT: lsr w9, w9, #24
-; CHECK-NEXT: extr w10, w14, w10, #2
-; CHECK-NEXT: cmp w14, #3
-; CHECK-NEXT: fmov s2, w8
-; CHECK-NEXT: umov w8, v1.b[4]
-; CHECK-NEXT: umov w12, v0.b[5]
-; CHECK-NEXT: csinv w10, w10, wzr, ls
-; CHECK-NEXT: lsr w10, w10, #24
-; CHECK-NEXT: mov v2.b[1], w9
-; CHECK-NEXT: lsl w9, w13, #24
-; CHECK-NEXT: lsr x13, x11, #32
-; CHECK-NEXT: lsl w12, w12, #24
-; CHECK-NEXT: umull x8, w9, w8
-; CHECK-NEXT: umov w9, v1.b[5]
-; CHECK-NEXT: extr w11, w13, w11, #2
-; CHECK-NEXT: cmp w13, #3
-; CHECK-NEXT: mov v2.b[2], w10
-; CHECK-NEXT: umov w10, v0.b[6]
-; CHECK-NEXT: csinv w11, w11, wzr, ls
-; CHECK-NEXT: lsr x13, x8, #32
-; CHECK-NEXT: lsr w11, w11, #24
-; CHECK-NEXT: umull x9, w12, w9
-; CHECK-NEXT: umov w12, v1.b[6]
-; CHECK-NEXT: extr w8, w13, w8, #2
-; CHECK-NEXT: cmp w13, #3
-; CHECK-NEXT: mov v2.b[3], w11
-; CHECK-NEXT: lsl w10, w10, #24
-; CHECK-NEXT: umov w11, v0.b[7]
-; CHECK-NEXT: csinv w8, w8, wzr, ls
-; CHECK-NEXT: lsr x13, x9, #32
-; CHECK-NEXT: lsr w8, w8, #24
-; CHECK-NEXT: umull x10, w10, w12
-; CHECK-NEXT: umov w12, v1.b[7]
-; CHECK-NEXT: extr w9, w13, w9, #2
-; CHECK-NEXT: cmp w13, #3
-; CHECK-NEXT: mov v2.b[4], w8
-; CHECK-NEXT: lsl w8, w11, #24
-; CHECK-NEXT: csinv w9, w9, wzr, ls
-; CHECK-NEXT: lsr x11, x10, #32
-; CHECK-NEXT: umull x8, w8, w12
-; CHECK-NEXT: lsr w9, w9, #24
-; CHECK-NEXT: extr w10, w11, w10, #2
-; CHECK-NEXT: cmp w11, #3
-; CHECK-NEXT: mov v2.b[5], w9
-; CHECK-NEXT: csinv w9, w10, wzr, ls
-; CHECK-NEXT: lsr x10, x8, #32
-; CHECK-NEXT: lsr w9, w9, #24
-; CHECK-NEXT: extr w8, w10, w8, #2
-; CHECK-NEXT: cmp w10, #3
-; CHECK-NEXT: mov v2.b[6], w9
-; CHECK-NEXT: csinv w8, w8, wzr, ls
-; CHECK-NEXT: lsr w8, w8, #24
-; CHECK-NEXT: mov v2.b[7], w8
-; CHECK-NEXT: fmov d0, d2
+; CHECK-NEXT: umull v0.8h, v0.8b, v1.8b
+; CHECK-NEXT: shrn v2.8b, v0.8h, #8
+; CHECK-NEXT: xtn v0.8b, v0.8h
+; CHECK-NEXT: shl v1.8b, v2.8b, #6
+; CHECK-NEXT: umov w8, v2.b[1]
+; CHECK-NEXT: umov w10, v2.b[0]
+; CHECK-NEXT: usra v1.8b, v0.8b, #2
+; CHECK-NEXT: tst w8, #0xfc
+; CHECK-NEXT: umov w9, v1.b[1]
+; CHECK-NEXT: umov w11, v1.b[0]
+; CHECK-NEXT: csinv w8, w9, wzr, eq
+; CHECK-NEXT: tst w10, #0xfc
+; CHECK-NEXT: umov w10, v2.b[2]
+; CHECK-NEXT: csinv w9, w11, wzr, eq
+; CHECK-NEXT: fmov s0, w9
+; CHECK-NEXT: umov w9, v1.b[2]
+; CHECK-NEXT: tst w10, #0xfc
+; CHECK-NEXT: umov w10, v1.b[3]
+; CHECK-NEXT: mov v0.b[1], w8
+; CHECK-NEXT: umov w8, v2.b[3]
+; CHECK-NEXT: csinv w9, w9, wzr, eq
+; CHECK-NEXT: mov v0.b[2], w9
+; CHECK-NEXT: tst w8, #0xfc
+; CHECK-NEXT: umov w8, v2.b[4]
+; CHECK-NEXT: csinv w9, w10, wzr, eq
+; CHECK-NEXT: umov w10, v1.b[4]
+; CHECK-NEXT: mov v0.b[3], w9
+; CHECK-NEXT: tst w8, #0xfc
+; CHECK-NEXT: umov w8, v2.b[5]
+; CHECK-NEXT: csinv w9, w10, wzr, eq
+; CHECK-NEXT: umov w10, v1.b[5]
+; CHECK-NEXT: mov v0.b[4], w9
+; CHECK-NEXT: tst w8, #0xfc
+; CHECK-NEXT: umov w8, v2.b[6]
+; CHECK-NEXT: csinv w9, w10, wzr, eq
+; CHECK-NEXT: umov w10, v1.b[6]
+; CHECK-NEXT: mov v0.b[5], w9
+; CHECK-NEXT: tst w8, #0xfc
+; CHECK-NEXT: umov w8, v2.b[7]
+; CHECK-NEXT: csinv w9, w10, wzr, eq
+; CHECK-NEXT: umov w10, v1.b[7]
+; CHECK-NEXT: mov v0.b[6], w9
+; CHECK-NEXT: tst w8, #0xfc
+; CHECK-NEXT: csinv w8, w10, wzr, eq
+; CHECK-NEXT: mov v0.b[7], w8
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-NEXT: ret
%tmp = call <8 x i8> @llvm.umul.fix.sat.v8i8(<8 x i8> %x, <8 x i8> %y, i32 2)
ret <8 x i8> %tmp
@@ -222,167 +185,92 @@ define <8 x i8> @vec_v8i8(<8 x i8> %x, <8 x i8> %y) {
define <16 x i8> @vec_v16i8(<16 x i8> %x, <16 x i8> %y) {
; CHECK-LABEL: vec_v16i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: umov w8, v0.b[1]
-; CHECK-NEXT: umov w9, v0.b[0]
-; CHECK-NEXT: umov w10, v1.b[1]
+; CHECK-NEXT: umull2 v2.8h, v0.16b, v1.16b
+; CHECK-NEXT: umull v3.8h, v0.8b, v1.8b
+; CHECK-NEXT: mul v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: uzp2 v2.16b, v3.16b, v2.16b
+; CHECK-NEXT: shl v1.16b, v2.16b, #6
+; CHECK-NEXT: umov w8, v2.b[1]
+; CHECK-NEXT: umov w10, v2.b[0]
+; CHECK-NEXT: usra v1.16b, v0.16b, #2
+; CHECK-NEXT: tst w8, #0xfc
+; CHECK-NEXT: umov w9, v1.b[1]
; CHECK-NEXT: umov w11, v1.b[0]
-; CHECK-NEXT: umov w12, v0.b[2]
-; CHECK-NEXT: umov w13, v0.b[3]
-; CHECK-NEXT: lsl w8, w8, #24
-; CHECK-NEXT: lsl w9, w9, #24
-; CHECK-NEXT: lsl w13, w13, #24
-; CHECK-NEXT: umull x8, w8, w10
-; CHECK-NEXT: umov w10, v1.b[2]
-; CHECK-NEXT: umull x9, w9, w11
-; CHECK-NEXT: lsl w11, w12, #24
-; CHECK-NEXT: lsr x12, x8, #32
-; CHECK-NEXT: lsr x14, x9, #32
-; CHECK-NEXT: umull x10, w11, w10
-; CHECK-NEXT: umov w11, v1.b[3]
-; CHECK-NEXT: extr w8, w12, w8, #2
-; CHECK-NEXT: cmp w12, #3
-; CHECK-NEXT: umov w12, v0.b[4]
-; CHECK-NEXT: extr w9, w14, w9, #2
-; CHECK-NEXT: csinv w8, w8, wzr, ls
-; CHECK-NEXT: cmp w14, #3
-; CHECK-NEXT: lsr x14, x10, #32
-; CHECK-NEXT: csinv w9, w9, wzr, ls
-; CHECK-NEXT: umull x11, w13, w11
-; CHECK-NEXT: umov w13, v1.b[4]
-; CHECK-NEXT: lsr w9, w9, #24
-; CHECK-NEXT: lsr w8, w8, #24
-; CHECK-NEXT: extr w10, w14, w10, #2
-; CHECK-NEXT: cmp w14, #3
-; CHECK-NEXT: lsl w12, w12, #24
-; CHECK-NEXT: fmov s2, w9
-; CHECK-NEXT: umov w9, v0.b[5]
-; CHECK-NEXT: csinv w10, w10, wzr, ls
-; CHECK-NEXT: umull x12, w12, w13
-; CHECK-NEXT: umov w13, v1.b[5]
-; CHECK-NEXT: lsr w10, w10, #24
-; CHECK-NEXT: mov v2.b[1], w8
-; CHECK-NEXT: lsr x8, x11, #32
-; CHECK-NEXT: lsl w9, w9, #24
-; CHECK-NEXT: extr w11, w8, w11, #2
-; CHECK-NEXT: cmp w8, #3
-; CHECK-NEXT: umov w8, v0.b[6]
-; CHECK-NEXT: umull x9, w9, w13
-; CHECK-NEXT: umov w13, v1.b[6]
-; CHECK-NEXT: mov v2.b[2], w10
-; CHECK-NEXT: lsr x10, x12, #32
-; CHECK-NEXT: csinv w11, w11, wzr, ls
-; CHECK-NEXT: lsr w11, w11, #24
-; CHECK-NEXT: extr w12, w10, w12, #2
-; CHECK-NEXT: cmp w10, #3
-; CHECK-NEXT: umov w10, v0.b[7]
-; CHECK-NEXT: lsl w8, w8, #24
-; CHECK-NEXT: mov v2.b[3], w11
-; CHECK-NEXT: lsr x11, x9, #32
-; CHECK-NEXT: csinv w12, w12, wzr, ls
-; CHECK-NEXT: umull x8, w8, w13
-; CHECK-NEXT: umov w13, v1.b[7]
-; CHECK-NEXT: lsr w12, w12, #24
-; CHECK-NEXT: extr w9, w11, w9, #2
-; CHECK-NEXT: cmp w11, #3
-; CHECK-NEXT: umov w11, v0.b[8]
-; CHECK-NEXT: lsl w10, w10, #24
-; CHECK-NEXT: mov v2.b[4], w12
-; CHECK-NEXT: lsr x12, x8, #32
-; CHECK-NEXT: csinv w9, w9, wzr, ls
-; CHECK-NEXT: umull x10, w10, w13
-; CHECK-NEXT: umov w13, v1.b[8]
-; CHECK-NEXT: lsr w9, w9, #24
-; CHECK-NEXT: extr w8, w12, w8, #2
-; CHECK-NEXT: cmp w12, #3
-; CHECK-NEXT: umov w12, v0.b[9]
-; CHECK-NEXT: lsl w11, w11, #24
-; CHECK-NEXT: mov v2.b[5], w9
-; CHECK-NEXT: lsr x9, x10, #32
-; CHECK-NEXT: csinv w8, w8, wzr, ls
-; CHECK-NEXT: umull x11, w11, w13
-; CHECK-NEXT: umov w13, v1.b[9]
-; CHECK-NEXT: lsr w8, w8, #24
-; CHECK-NEXT: extr w10, w9, w10, #2
-; CHECK-NEXT: cmp w9, #3
-; CHECK-NEXT: umov w9, v0.b[10]
-; CHECK-NEXT: lsl w12, w12, #24
-; CHECK-NEXT: mov v2.b[6], w8
-; CHECK-NEXT: lsr x8, x11, #32
-; CHECK-NEXT: csinv w10, w10, wzr, ls
-; CHECK-NEXT: umull x12, w12, w13
-; CHECK-NEXT: umov w13, v1.b[10]
-; CHECK-NEXT: lsr w10, w10, #24
-; CHECK-NEXT: extr w11, w8, w11, #2
-; CHECK-NEXT: cmp w8, #3
-; CHECK-NEXT: umov w8, v0.b[11]
-; CHECK-NEXT: lsl w9, w9, #24
-; CHECK-NEXT: mov v2.b[7], w10
-; CHECK-NEXT: lsr x10, x12, #32
-; CHECK-NEXT: csinv w11, w11, wzr, ls
-; CHECK-NEXT: umull x9, w9, w13
-; CHECK-NEXT: umov w13, v1.b[11]
-; CHECK-NEXT: lsr w11, w11, #24
-; CHECK-NEXT: extr w12, w10, w12, #2
-; CHECK-NEXT: lsl w8, w8, #24
-; CHECK-NEXT: cmp w10, #3
-; CHECK-NEXT: umov w10, v0.b[12]
-; CHECK-NEXT: mov v2.b[8], w11
-; CHECK-NEXT: lsr x11, x9, #32
-; CHECK-NEXT: csinv w12, w12, wzr, ls
-; CHECK-NEXT: umull x8, w8, w13
-; CHECK-NEXT: lsr w12, w12, #24
-; CHECK-NEXT: umov w13, v1.b[12]
-; CHECK-NEXT: extr w9, w11, w9, #2
-; CHECK-NEXT: cmp w11, #3
-; CHECK-NEXT: umov w11, v0.b[13]
-; CHECK-NEXT: lsl w10, w10, #24
-; CHECK-NEXT: mov v2.b[9], w12
-; CHECK-NEXT: lsr x12, x8, #32
-; CHECK-NEXT: csinv w9, w9, wzr, ls
-; CHECK-NEXT: lsr w9, w9, #24
-; CHECK-NEXT: umull x10, w10, w13
-; CHECK-NEXT: umov w13, v1.b[13]
-; CHECK-NEXT: extr w8, w12, w8, #2
-; CHECK-NEXT: cmp w12, #3
-; CHECK-NEXT: lsl w11, w11, #24
-; CHECK-NEXT: mov v2.b[10], w9
-; CHECK-NEXT: umov w9, v0.b[14]
-; CHECK-NEXT: csinv w8, w8, wzr, ls
-; CHECK-NEXT: lsr x12, x10, #32
-; CHECK-NEXT: lsr w8, w8, #24
-; CHECK-NEXT: umull x11, w11, w13
-; CHECK-NEXT: umov w13, v1.b[14]
-; CHECK-NEXT: extr w10, w12, w10, #2
-; CHECK-NEXT: cmp w12, #3
-; CHECK-NEXT: mov v2.b[11], w8
-; CHECK-NEXT: lsl w8, w9, #24
-; CHECK-NEXT: umov w9, v0.b[15]
-; CHECK-NEXT: csinv w10, w10, wzr, ls
-; CHECK-NEXT: lsr x12, x11, #32
-; CHECK-NEXT: lsr w10, w10, #24
-; CHECK-NEXT: umull x8, w8, w13
-; CHECK-NEXT: umov w13, v1.b[15]
-; CHECK-NEXT: extr w11, w12, w11, #2
-; CHECK-NEXT: cmp w12, #3
-; CHECK-NEXT: mov v2.b[12], w10
-; CHECK-NEXT: lsl w9, w9, #24
-; CHECK-NEXT: csinv w10, w11, wzr, ls
-; CHECK-NEXT: lsr x11, x8, #32
-; CHECK-NEXT: umull x9, w9, w13
-; CHECK-NEXT: lsr w10, w10, #24
-; CHECK-NEXT: extr w8, w11, w8, #2
-; CHECK-NEXT: cmp w11, #3
-; CHECK-NEXT: mov v2.b[13], w10
-; CHECK-NEXT: csinv w8, w8, wzr, ls
-; CHECK-NEXT: lsr x10, x9, #32
-; CHECK-NEXT: lsr w8, w8, #24
-; CHECK-NEXT: extr w9, w10, w9, #2
-; CHECK-NEXT: cmp w10, #3
-; CHECK-NEXT: mov v2.b[14], w8
-; CHECK-NEXT: csinv w8, w9, wzr, ls
-; CHECK-NEXT: lsr w8, w8, #24
-; CHECK-NEXT: mov v2.b[15], w8
-; CHECK-NEXT: mov v0.16b, v2.16b
+; CHECK-NEXT: csinv w8, w9, wzr, eq
+; CHECK-NEXT: tst w10, #0xfc
+; CHECK-NEXT: umov w10, v2.b[2]
+; CHECK-NEXT: csinv w9, w11, wzr, eq
+; CHECK-NEXT: fmov s0, w9
+; CHECK-NEXT: umov w9, v1.b[2]
+; CHECK-NEXT: tst w10, #0xfc
+; CHECK-NEXT: umov w10, v1.b[3]
+; CHECK-NEXT: mov v0.b[1], w8
+; CHECK-NEXT: umov w8, v2.b[3]
+; CHECK-NEXT: csinv w9, w9, wzr, eq
+; CHECK-NEXT: mov v0.b[2], w9
+; CHECK-NEXT: tst w8, #0xfc
+; CHECK-NEXT: umov w8, v2.b[4]
+; CHECK-NEXT: csinv w9, w10, wzr, eq
+; CHECK-NEXT: umov w10, v1.b[4]
+; CHECK-NEXT: mov v0.b[3], w9
+; CHECK-NEXT: tst w8, #0xfc
+; CHECK-NEXT: umov w8, v2.b[5]
+; CHECK-NEXT: csinv w9, w10, wzr, eq
+; CHECK-NEXT: umov w10, v1.b[5]
+; CHECK-NEXT: mov v0.b[4], w9
+; CHECK-NEXT: tst w8, #0xfc
+; CHECK-NEXT: umov w8, v2.b[6]
+; CHECK-NEXT: csinv w9, w10, wzr, eq
+; CHECK-NEXT: umov w10, v1.b[6]
+; CHECK-NEXT: mov v0.b[5], w9
+; CHECK-NEXT: tst w8, #0xfc
+; CHECK-NEXT: umov w8, v2.b[7]
+; CHECK-NEXT: csinv w9, w10, wzr, eq
+; CHECK-NEXT: umov w10, v1.b[7]
+; CHECK-NEXT: mov v0.b[6], w9
+; CHECK-NEXT: tst w8, #0xfc
+; CHECK-NEXT: umov w8, v2.b[8]
+; CHECK-NEXT: csinv w9, w10, wzr, eq
+; CHECK-NEXT: umov w10, v1.b[8]
+; CHECK-NEXT: mov v0.b[7], w9
+; CHECK-NEXT: tst w8, #0xfc
+; CHECK-NEXT: umov w8, v2.b[9]
+; CHECK-NEXT: csinv w9, w10, wzr, eq
+; CHECK-NEXT: umov w10, v1.b[9]
+; CHECK-NEXT: mov v0.b[8], w9
+; CHECK-NEXT: tst w8, #0xfc
+; CHECK-NEXT: umov w8, v2.b[10]
+; CHECK-NEXT: csinv w9, w10, wzr, eq
+; CHECK-NEXT: umov w10, v1.b[10]
+; CHECK-NEXT: mov v0.b[9], w9
+; CHECK-NEXT: tst w8, #0xfc
+; CHECK-NEXT: umov w8, v2.b[11]
+; CHECK-NEXT: csinv w9, w10, wzr, eq
+; CHECK-NEXT: umov w10, v1.b[11]
+; CHECK-NEXT: mov v0.b[10], w9
+; CHECK-NEXT: tst w8, #0xfc
+; CHECK-NEXT: umov w8, v2.b[12]
+; CHECK-NEXT: csinv w9, w10, wzr, eq
+; CHECK-NEXT: umov w10, v1.b[12]
+; CHECK-NEXT: mov v0.b[11], w9
+; CHECK-NEXT: tst w8, #0xfc
+; CHECK-NEXT: umov w8, v2.b[13]
+; CHECK-NEXT: csinv w9, w10, wzr, eq
+; CHECK-NEXT: umov w10, v1.b[13]
+; CHECK-NEXT: mov v0.b[12], w9
+; CHECK-NEXT: tst w8, #0xfc
+; CHECK-NEXT: umov w8, v2.b[14]
+; CHECK-NEXT: csinv w9, w10, wzr, eq
+; CHECK-NEXT: umov w10, v1.b[14]
+; CHECK-NEXT: mov v0.b[13], w9
+; CHECK-NEXT: tst w8, #0xfc
+; CHECK-NEXT: umov w8, v2.b[15]
+; CHECK-NEXT: csinv w9, w10, wzr, eq
+; CHECK-NEXT: umov w10, v1.b[15]
+; CHECK-NEXT: mov v0.b[14], w9
+; CHECK-NEXT: tst w8, #0xfc
+; CHECK-NEXT: csinv w8, w10, wzr, eq
+; CHECK-NEXT: mov v0.b[15], w8
; CHECK-NEXT: ret
%tmp = call <16 x i8> @llvm.umul.fix.sat.v16i8(<16 x i8> %x, <16 x i8> %y, i32 2)
ret <16 x i8> %tmp
@@ -391,47 +279,30 @@ define <16 x i8> @vec_v16i8(<16 x i8> %x, <16 x i8> %y) {
define <4 x i16> @vec_v4i16(<4 x i16> %x, <4 x i16> %y) {
; CHECK-LABEL: vec_v4i16:
; CHECK: // %bb.0:
-; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT: umov w8, v0.h[0]
-; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT: umull v0.4s, v0.4h, v1.4h
+; CHECK-NEXT: shrn v1.4h, v0.4s, #16
+; CHECK-NEXT: xtn v0.4h, v0.4s
+; CHECK-NEXT: shl v2.4h, v1.4h, #6
+; CHECK-NEXT: umov w8, v1.h[1]
; CHECK-NEXT: umov w10, v1.h[0]
-; CHECK-NEXT: umov w9, v0.h[1]
-; CHECK-NEXT: umov w11, v1.h[1]
-; CHECK-NEXT: umov w12, v0.h[2]
-; CHECK-NEXT: umov w13, v0.h[3]
-; CHECK-NEXT: lsl w8, w8, #16
-; CHECK-NEXT: lsl w9, w9, #16
-; CHECK-NEXT: lsl w12, w12, #16
-; CHECK-NEXT: umull x8, w8, w10
+; CHECK-NEXT: usra v2.4h, v0.4h, #10
+; CHECK-NEXT: tst w8, #0xfc00
+; CHECK-NEXT: umov w9, v2.h[1]
+; CHECK-NEXT: umov w11, v2.h[0]
+; CHECK-NEXT: csinv w8, w9, wzr, eq
+; CHECK-NEXT: tst w10, #0xfc00
; CHECK-NEXT: umov w10, v1.h[2]
-; CHECK-NEXT: umull x9, w9, w11
-; CHECK-NEXT: lsr x11, x8, #32
-; CHECK-NEXT: lsr x14, x9, #32
-; CHECK-NEXT: umull x10, w12, w10
-; CHECK-NEXT: umov w12, v1.h[3]
-; CHECK-NEXT: extr w8, w11, w8, #10
-; CHECK-NEXT: cmp w11, #1023
-; CHECK-NEXT: lsl w11, w13, #16
-; CHECK-NEXT: extr w9, w14, w9, #10
-; CHECK-NEXT: csinv w8, w8, wzr, ls
-; CHECK-NEXT: cmp w14, #1023
-; CHECK-NEXT: lsr x13, x10, #32
-; CHECK-NEXT: lsr w8, w8, #16
-; CHECK-NEXT: csinv w9, w9, wzr, ls
-; CHECK-NEXT: umull x11, w11, w12
-; CHECK-NEXT: lsr w9, w9, #16
-; CHECK-NEXT: cmp w13, #1023
-; CHECK-NEXT: fmov s0, w8
-; CHECK-NEXT: extr w8, w13, w10, #10
-; CHECK-NEXT: csinv w8, w8, wzr, ls
-; CHECK-NEXT: mov v0.h[1], w9
-; CHECK-NEXT: lsr x9, x11, #32
-; CHECK-NEXT: lsr w8, w8, #16
-; CHECK-NEXT: extr w10, w9, w11, #10
-; CHECK-NEXT: cmp w9, #1023
-; CHECK-NEXT: mov v0.h[2], w8
-; CHECK-NEXT: csinv w8, w10, wzr, ls
-; CHECK-NEXT: lsr w8, w8, #16
+; CHECK-NEXT: csinv w9, w11, wzr, eq
+; CHECK-NEXT: fmov s0, w9
+; CHECK-NEXT: umov w9, v2.h[2]
+; CHECK-NEXT: tst w10, #0xfc00
+; CHECK-NEXT: umov w10, v2.h[3]
+; CHECK-NEXT: mov v0.h[1], w8
+; CHECK-NEXT: umov w8, v1.h[3]
+; CHECK-NEXT: csinv w9, w9, wzr, eq
+; CHECK-NEXT: mov v0.h[2], w9
+; CHECK-NEXT: tst w8, #0xfc00
+; CHECK-NEXT: csinv w8, w10, wzr, eq
; CHECK-NEXT: mov v0.h[3], w8
; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-NEXT: ret
@@ -442,87 +313,52 @@ define <4 x i16> @vec_v4i16(<4 x i16> %x, <4 x i16> %y) {
define <8 x i16> @vec_v8i16(<8 x i16> %x, <8 x i16> %y) {
; CHECK-LABEL: vec_v8i16:
; CHECK: // %bb.0:
-; CHECK-NEXT: umov w8, v0.h[1]
-; CHECK-NEXT: umov w9, v0.h[0]
-; CHECK-NEXT: umov w10, v1.h[1]
+; CHECK-NEXT: umull2 v2.4s, v0.8h, v1.8h
+; CHECK-NEXT: umull v3.4s, v0.4h, v1.4h
+; CHECK-NEXT: mul v0.8h, v0.8h, v1.8h
+; CHECK-NEXT: uzp2 v2.8h, v3.8h, v2.8h
+; CHECK-NEXT: shl v1.8h, v2.8h, #14
+; CHECK-NEXT: umov w8, v2.h[1]
+; CHECK-NEXT: umov w10, v2.h[0]
+; CHECK-NEXT: usra v1.8h, v0.8h, #2
+; CHECK-NEXT: tst w8, #0xfffc
+; CHECK-NEXT: umov w9, v1.h[1]
; CHECK-NEXT: umov w11, v1.h[0]
-; CHECK-NEXT: umov w12, v0.h[2]
-; CHECK-NEXT: umov w13, v0.h[3]
-; CHECK-NEXT: lsl w8, w8, #16
-; CHECK-NEXT: lsl w9, w9, #16
-; CHECK-NEXT: umull x8, w8, w10
-; CHECK-NEXT: umov w10, v1.h[2]
-; CHECK-NEXT: umull x9, w9, w11
-; CHECK-NEXT: lsl w11, w12, #16
-; CHECK-NEXT: lsr x12, x8, #32
-; CHECK-NEXT: lsr x14, x9, #32
-; CHECK-NEXT: umull x10, w11, w10
-; CHECK-NEXT: umov w11, v1.h[3]
-; CHECK-NEXT: extr w8, w12, w8, #2
-; CHECK-NEXT: cmp w12, #3
-; CHECK-NEXT: lsl w12, w13, #16
-; CHECK-NEXT: extr w9, w14, w9, #2
-; CHECK-NEXT: umov w13, v0.h[4]
-; CHECK-NEXT: csinv w8, w8, wzr, ls
-; CHECK-NEXT: cmp w14, #3
-; CHECK-NEXT: lsr x14, x10, #32
-; CHECK-NEXT: csinv w9, w9, wzr, ls
-; CHECK-NEXT: umull x11, w12, w11
-; CHECK-NEXT: lsr w8, w8, #16
-; CHECK-NEXT: lsr w9, w9, #16
-; CHECK-NEXT: extr w10, w14, w10, #2
-; CHECK-NEXT: cmp w14, #3
-; CHECK-NEXT: umov w12, v0.h[5]
-; CHECK-NEXT: fmov s2, w9
-; CHECK-NEXT: umov w9, v1.h[4]
-; CHECK-NEXT: csinv w10, w10, wzr, ls
-; CHECK-NEXT: lsr w10, w10, #16
-; CHECK-NEXT: mov v2.h[1], w8
-; CHECK-NEXT: lsl w8, w13, #16
-; CHECK-NEXT: lsr x13, x11, #32
-; CHECK-NEXT: lsl w12, w12, #16
-; CHECK-NEXT: umull x8, w8, w9
-; CHECK-NEXT: umov w9, v1.h[5]
-; CHECK-NEXT: extr w11, w13, w11, #2
-; CHECK-NEXT: cmp w13, #3
-; CHECK-NEXT: mov v2.h[2], w10
-; CHECK-NEXT: umov w10, v0.h[6]
-; CHECK-NEXT: csinv w11, w11, wzr, ls
-; CHECK-NEXT: lsr x13, x8, #32
-; CHECK-NEXT: lsr w11, w11, #16
-; CHECK-NEXT: umull x9, w12, w9
-; CHECK-NEXT: umov w12, v1.h[6]
-; CHECK-NEXT: extr w8, w13, w8, #2
-; CHECK-NEXT: cmp w13, #3
-; CHECK-NEXT: mov v2.h[3], w11
-; CHECK-NEXT: lsl w10, w10, #16
-; CHECK-NEXT: umov w11, v0.h[7]
-; CHECK-NEXT: csinv w8, w8, wzr, ls
-; CHECK-NEXT: lsr x13, x9, #32
-; CHECK-NEXT: lsr w8, w8, #16
-; CHECK-NEXT: umull x10, w10, w12
-; CHECK-NEXT: umov w12, v1.h[7]
-; CHECK-NEXT: extr w9, w13, w9, #2
-; CHECK-NEXT: cmp w13, #3
-; CHECK-NEXT: mov v2.h[4], w8
-; CHECK-NEXT: lsl w8, w11, #16
-; CHECK-NEXT: csinv w9, w9, wzr, ls
-; CHECK-NEXT: lsr x11, x10, #32
-; CHECK-NEXT: umull x8, w8, w12
-; CHECK-NEXT: lsr w9, w9, #16
-; CHECK-NEXT: extr w10, w11, w10, #2
-; CHECK-NEXT: cmp w11, #3
-; CHECK-NEXT: mov v2.h[5], w9
-; CHECK-NEXT: csinv w9, w10, wzr, ls
-; CHECK-NEXT: lsr x10, x8, #32
-; CHECK-NEXT: lsr w9, w9, #16
-; CHECK-NEXT: extr w8, w10, w8, #2
-; CHECK-NEXT: cmp w10, #3
-; CHECK-NEXT: mov v2.h[6], w9
-; CHECK-NEXT: csinv w8, w8, wzr, ls
-; CHECK-NEXT: lsr w8, w8, #16
-; CHECK-NEXT: mov v2.h[7], w8
-; CHECK-NEXT: mov v0.16b, v2.16b
+; CHECK-NEXT: csinv w8, w9, wzr, eq
+; CHECK-NEXT: tst w10, #0xfffc
+; CHECK-NEXT: umov w10, v2.h[2]
+; CHECK-NEXT: csinv w9, w11, wzr, eq
+; CHECK-NEXT: fmov s0, w9
+; CHECK-NEXT: umov w9, v1.h[2]
+; CHECK-NEXT: tst w10, #0xfffc
+; CHECK-NEXT: umov w10, v1.h[3]
+; CHECK-NEXT: mov v0.h[1], w8
+; CHECK-NEXT: umov w8, v2.h[3]
+; CHECK-NEXT: csinv w9, w9, wzr, eq
+; CHECK-NEXT: mov v0.h[2], w9
+; CHECK-NEXT: tst w8, #0xfffc
+; CHECK-NEXT: umov w8, v2.h[4]
+; CHECK-NEXT: csinv w9, w10, wzr, eq
+; CHECK-NEXT: umov w10, v1.h[4]
+; CHECK-NEXT: mov v0.h[3], w9
+; CHECK-NEXT: tst w8, #0xfffc
+; CHECK-NEXT: umov w8, v2.h[5]
+; CHECK-NEXT: csinv w9, w10, wzr, eq
+; CHECK-NEXT: umov w10, v1.h[5]
+; CHECK-NEXT: mov v0.h[4], w9
+; CHECK-NEXT: tst w8, #0xfffc
+; CHECK-NEXT: umov w8, v2.h[6]
+; CHECK-NEXT: csinv w9, w10, wzr, eq
+; CHECK-NEXT: umov w10, v1.h[6]
+; CHECK-NEXT: mov v0.h[5], w9
+; CHECK-NEXT: tst w8, #0xfffc
+; CHECK-NEXT: umov w8, v2.h[7]
+; CHECK-NEXT: csinv w9, w10, wzr, eq
+; CHECK-NEXT: umov w10, v1.h[7]
+; CHECK-NEXT: mov v0.h[6], w9
+; CHECK-NEXT: tst w8, #0xfffc
+; CHECK-NEXT: csinv w8, w10, wzr, eq
+; CHECK-NEXT: mov v0.h[7], w8
; CHECK-NEXT: ret
%tmp = call <8 x i16> @llvm.umul.fix.sat.v8i16(<8 x i16> %x, <8 x i16> %y, i32 2)
ret <8 x i16> %tmp
@@ -531,20 +367,19 @@ define <8 x i16> @vec_v8i16(<8 x i16> %x, <8 x i16> %y) {
define <2 x i32> @vec_v2i32(<2 x i32> %x, <2 x i32> %y) {
; CHECK-LABEL: vec_v2i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1
-; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT: mov w8, v1.s[1]
-; CHECK-NEXT: mov w9, v0.s[1]
-; CHECK-NEXT: fmov w10, s0
-; CHECK-NEXT: umull x8, w9, w8
-; CHECK-NEXT: fmov w9, s1
-; CHECK-NEXT: umull x9, w10, w9
-; CHECK-NEXT: tst x8, #0xffffffff00000000
-; CHECK-NEXT: csinv w8, w8, wzr, eq
-; CHECK-NEXT: tst x9, #0xffffffff00000000
+; CHECK-NEXT: umull v0.2d, v0.2s, v1.2s
+; CHECK-NEXT: xtn v1.2s, v0.2d
+; CHECK-NEXT: shrn v0.2s, v0.2d, #32
+; CHECK-NEXT: mov w8, v0.s[1]
+; CHECK-NEXT: mov w9, v1.s[1]
+; CHECK-NEXT: fmov w10, s1
+; CHECK-NEXT: cmp w8, #0
+; CHECK-NEXT: fmov w8, s0
; CHECK-NEXT: csinv w9, w9, wzr, eq
-; CHECK-NEXT: fmov s0, w9
-; CHECK-NEXT: mov v0.s[1], w8
+; CHECK-NEXT: cmp w8, #0
+; CHECK-NEXT: csinv w8, w10, wzr, eq
+; CHECK-NEXT: fmov s0, w8
+; CHECK-NEXT: mov v0.s[1], w9
; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
; CHECK-NEXT: ret
%tmp = call <2 x i32> @llvm.umul.fix.sat.v2i32(<2 x i32> %x, <2 x i32> %y, i32 0)
@@ -554,37 +389,31 @@ define <2 x i32> @vec_v2i32(<2 x i32> %x, <2 x i32> %y) {
define <4 x i32> @vec_v4i32(<4 x i32> %x, <4 x i32> %y) {
; CHECK-LABEL: vec_v4i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov w8, v1.s[1]
-; CHECK-NEXT: mov w9, v0.s[1]
-; CHECK-NEXT: fmov w10, s0
-; CHECK-NEXT: mov w11, v0.s[2]
-; CHECK-NEXT: mov w14, v1.s[3]
-; CHECK-NEXT: mov w15, v0.s[3]
-; CHECK-NEXT: umull x8, w9, w8
-; CHECK-NEXT: fmov w9, s1
-; CHECK-NEXT: umull x9, w10, w9
-; CHECK-NEXT: lsr x12, x8, #32
-; CHECK-NEXT: mov w10, v1.s[2]
-; CHECK-NEXT: extr w8, w12, w8, #15
-; CHECK-NEXT: cmp w12, #8, lsl #12 // =32768
-; CHECK-NEXT: lsr x13, x9, #32
-; CHECK-NEXT: csinv w8, w8, wzr, lo
-; CHECK-NEXT: umull x10, w11, w10
-; CHECK-NEXT: extr w9, w13, w9, #15
-; CHECK-NEXT: cmp w13, #8, lsl #12 // =32768
-; CHECK-NEXT: csinv w9, w9, wzr, lo
-; CHECK-NEXT: fmov s0, w9
-; CHECK-NEXT: lsr x9, x10, #32
-; CHECK-NEXT: extr w10, w9, w10, #15
-; CHECK-NEXT: cmp w9, #8, lsl #12 // =32768
-; CHECK-NEXT: mov v0.s[1], w8
-; CHECK-NEXT: umull x8, w15, w14
-; CHECK-NEXT: csinv w9, w10, wzr, lo
-; CHECK-NEXT: lsr x10, x8, #32
-; CHECK-NEXT: mov v0.s[2], w9
-; CHECK-NEXT: extr w8, w10, w8, #15
-; CHECK-NEXT: cmp w10, #8, lsl #12 // =32768
-; CHECK-NEXT: csinv w8, w8, wzr, lo
+; CHECK-NEXT: umull2 v2.2d, v0.4s, v1.4s
+; CHECK-NEXT: umull v3.2d, v0.2s, v1.2s
+; CHECK-NEXT: mul v0.4s, v0.4s, v1.4s
+; CHECK-NEXT: uzp2 v2.4s, v3.4s, v2.4s
+; CHECK-NEXT: shl v1.4s, v2.4s, #17
+; CHECK-NEXT: mov w8, v2.s[1]
+; CHECK-NEXT: usra v1.4s, v0.4s, #15
+; CHECK-NEXT: cmp wzr, w8, lsr #15
+; CHECK-NEXT: fmov w8, s2
+; CHECK-NEXT: mov w9, v1.s[1]
+; CHECK-NEXT: fmov w10, s1
+; CHECK-NEXT: mov w11, v1.s[2]
+; CHECK-NEXT: csinv w9, w9, wzr, eq
+; CHECK-NEXT: cmp wzr, w8, lsr #15
+; CHECK-NEXT: mov w8, v2.s[2]
+; CHECK-NEXT: csinv w10, w10, wzr, eq
+; CHECK-NEXT: fmov s0, w10
+; CHECK-NEXT: cmp wzr, w8, lsr #15
+; CHECK-NEXT: mov w8, v1.s[3]
+; CHECK-NEXT: mov v0.s[1], w9
+; CHECK-NEXT: mov w9, v2.s[3]
+; CHECK-NEXT: csinv w10, w11, wzr, eq
+; CHECK-NEXT: mov v0.s[2], w10
+; CHECK-NEXT: cmp wzr, w9, lsr #15
+; CHECK-NEXT: csinv w8, w8, wzr, eq
; CHECK-NEXT: mov v0.s[3], w8
; CHECK-NEXT: ret
%tmp = call <4 x i32> @llvm.umul.fix.sat.v4i32(<4 x i32> %x, <4 x i32> %y, i32 15)
@@ -594,53 +423,59 @@ define <4 x i32> @vec_v4i32(<4 x i32> %x, <4 x i32> %y) {
define <8 x i32> @vec_v8i32(<8 x i32> %x, <8 x i32> %y) {
; CHECK-LABEL: vec_v8i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov w8, v2.s[1]
+; CHECK-NEXT: umull2 v4.2d, v0.4s, v2.4s
+; CHECK-NEXT: umull v5.2d, v0.2s, v2.2s
+; CHECK-NEXT: umull v6.2d, v1.2s, v3.2s
+; CHECK-NEXT: mul v0.4s, v0.4s, v2.4s
+; CHECK-NEXT: uzp2 v4.4s, v5.4s, v4.4s
+; CHECK-NEXT: umull2 v5.2d, v1.4s, v3.4s
+; CHECK-NEXT: mul v1.4s, v1.4s, v3.4s
+; CHECK-NEXT: add v7.4s, v4.4s, v4.4s
+; CHECK-NEXT: uzp2 v5.4s, v6.4s, v5.4s
+; CHECK-NEXT: mov w8, v4.s[1]
+; CHECK-NEXT: mov w10, v4.s[2]
+; CHECK-NEXT: fmov w12, s4
+; CHECK-NEXT: mov w11, v4.s[3]
+; CHECK-NEXT: shl v2.4s, v7.4s, #31
+; CHECK-NEXT: add v6.4s, v5.4s, v5.4s
+; CHECK-NEXT: cmp w8, #0
+; CHECK-NEXT: mov w13, v5.s[1]
+; CHECK-NEXT: orr v0.16b, v2.16b, v0.16b
+; CHECK-NEXT: shl v2.4s, v6.4s, #31
; CHECK-NEXT: mov w9, v0.s[1]
-; CHECK-NEXT: mov w10, v2.s[2]
-; CHECK-NEXT: mov w11, v0.s[2]
-; CHECK-NEXT: mov w12, v2.s[3]
-; CHECK-NEXT: fmov w13, s2
-; CHECK-NEXT: mov w14, v0.s[3]
+; CHECK-NEXT: mov w8, v0.s[2]
; CHECK-NEXT: fmov w15, s0
-; CHECK-NEXT: mov w16, v3.s[1]
-; CHECK-NEXT: mov w17, v1.s[1]
-; CHECK-NEXT: umull x8, w9, w8
-; CHECK-NEXT: umull x9, w15, w13
-; CHECK-NEXT: fmov w15, s1
-; CHECK-NEXT: mov w13, v3.s[2]
-; CHECK-NEXT: umull x10, w11, w10
-; CHECK-NEXT: umull x11, w14, w12
-; CHECK-NEXT: fmov w14, s3
-; CHECK-NEXT: tst x8, #0xffffffff00000000
-; CHECK-NEXT: umull x12, w17, w16
-; CHECK-NEXT: csinv w8, w8, wzr, eq
-; CHECK-NEXT: tst x9, #0xffffffff00000000
+; CHECK-NEXT: orr v2.16b, v2.16b, v1.16b
+; CHECK-NEXT: mov w14, v0.s[3]
; CHECK-NEXT: csinv w9, w9, wzr, eq
-; CHECK-NEXT: tst x10, #0xffffffff00000000
-; CHECK-NEXT: mov w17, v1.s[3]
-; CHECK-NEXT: umull x14, w15, w14
-; CHECK-NEXT: csinv w10, w10, wzr, eq
-; CHECK-NEXT: tst x11, #0xffffffff00000000
-; CHECK-NEXT: csinv w11, w11, wzr, eq
-; CHECK-NEXT: tst x12, #0xffffffff00000000
-; CHECK-NEXT: mov w15, v1.s[2]
+; CHECK-NEXT: cmp w12, #0
+; CHECK-NEXT: mov w12, v2.s[1]
+; CHECK-NEXT: csinv w15, w15, wzr, eq
+; CHECK-NEXT: cmp w10, #0
+; CHECK-NEXT: csinv w8, w8, wzr, eq
+; CHECK-NEXT: cmp w11, #0
+; CHECK-NEXT: fmov w11, s5
+; CHECK-NEXT: csinv w10, w14, wzr, eq
+; CHECK-NEXT: cmp w13, #0
+; CHECK-NEXT: fmov w13, s2
; CHECK-NEXT: csinv w12, w12, wzr, eq
-; CHECK-NEXT: fmov s0, w9
-; CHECK-NEXT: mov w16, v3.s[3]
-; CHECK-NEXT: tst x14, #0xffffffff00000000
-; CHECK-NEXT: csinv w14, w14, wzr, eq
-; CHECK-NEXT: fmov s1, w14
-; CHECK-NEXT: umull x9, w15, w13
-; CHECK-NEXT: mov v0.s[1], w8
-; CHECK-NEXT: umull x8, w17, w16
+; CHECK-NEXT: fmov s0, w15
+; CHECK-NEXT: mov w14, v2.s[2]
+; CHECK-NEXT: cmp w11, #0
+; CHECK-NEXT: mov w11, v5.s[2]
+; CHECK-NEXT: csinv w13, w13, wzr, eq
+; CHECK-NEXT: fmov s1, w13
+; CHECK-NEXT: mov v0.s[1], w9
+; CHECK-NEXT: mov w9, v5.s[3]
+; CHECK-NEXT: cmp w11, #0
+; CHECK-NEXT: mov w11, v2.s[3]
; CHECK-NEXT: mov v1.s[1], w12
-; CHECK-NEXT: tst x9, #0xffffffff00000000
-; CHECK-NEXT: csinv w9, w9, wzr, eq
-; CHECK-NEXT: mov v0.s[2], w10
-; CHECK-NEXT: tst x8, #0xffffffff00000000
-; CHECK-NEXT: csinv w8, w8, wzr, eq
-; CHECK-NEXT: mov v1.s[2], w9
-; CHECK-NEXT: mov v0.s[3], w11
+; CHECK-NEXT: csinv w12, w14, wzr, eq
+; CHECK-NEXT: mov v0.s[2], w8
+; CHECK-NEXT: cmp w9, #0
+; CHECK-NEXT: csinv w8, w11, wzr, eq
+; CHECK-NEXT: mov v1.s[2], w12
+; CHECK-NEXT: mov v0.s[3], w10
; CHECK-NEXT: mov v1.s[3], w8
; CHECK-NEXT: ret
%tmp = call <8 x i32> @llvm.umul.fix.sat.v8i32(<8 x i32> %x, <8 x i32> %y, i32 0)
diff --git a/llvm/test/CodeGen/X86/smul_fix_sat.ll b/llvm/test/CodeGen/X86/smul_fix_sat.ll
index e68b6e328b723..2a863e32cab63 100644
--- a/llvm/test/CodeGen/X86/smul_fix_sat.ll
+++ b/llvm/test/CodeGen/X86/smul_fix_sat.ll
@@ -193,70 +193,66 @@ define i4 @func3(i4 %x, i4 %y) nounwind {
define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
; X64-LABEL: vec:
; X64: # %bb.0:
-; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
+; X64-NEXT: pxor %xmm3, %xmm3
+; X64-NEXT: pxor %xmm2, %xmm2
+; X64-NEXT: pcmpgtd %xmm1, %xmm2
+; X64-NEXT: pand %xmm0, %xmm2
+; X64-NEXT: pcmpgtd %xmm0, %xmm3
+; X64-NEXT: pand %xmm1, %xmm3
+; X64-NEXT: paddd %xmm2, %xmm3
+; X64-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
+; X64-NEXT: pmuludq %xmm1, %xmm0
+; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,3,2,3]
+; X64-NEXT: pshufd {{.*#+}} xmm5 = xmm1[1,1,3,3]
+; X64-NEXT: pmuludq %xmm4, %xmm5
+; X64-NEXT: pshufd {{.*#+}} xmm1 = xmm5[1,3,2,3]
+; X64-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; X64-NEXT: psubd %xmm3, %xmm2
+; X64-NEXT: pshufd {{.*#+}} xmm1 = xmm0[0,2,2,3]
+; X64-NEXT: pshufd {{.*#+}} xmm0 = xmm5[0,2,2,3]
+; X64-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; X64-NEXT: pshufd {{.*#+}} xmm0 = xmm2[3,3,3,3]
+; X64-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
; X64-NEXT: movd %xmm2, %eax
-; X64-NEXT: cltq
-; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
+; X64-NEXT: pshufd {{.*#+}} xmm4 = xmm2[1,1,1,1]
+; X64-NEXT: pslld $30, %xmm2
+; X64-NEXT: psrld $2, %xmm1
+; X64-NEXT: por %xmm2, %xmm1
+; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
; X64-NEXT: movd %xmm2, %ecx
-; X64-NEXT: movslq %ecx, %rdx
-; X64-NEXT: imulq %rax, %rdx
-; X64-NEXT: movq %rdx, %rcx
-; X64-NEXT: shrq $32, %rcx
-; X64-NEXT: shrdl $2, %ecx, %edx
-; X64-NEXT: cmpl $2, %ecx
-; X64-NEXT: movl $2147483647, %eax # imm = 0x7FFFFFFF
-; X64-NEXT: cmovgel %eax, %edx
-; X64-NEXT: cmpl $-2, %ecx
-; X64-NEXT: movl $-2147483648, %ecx # imm = 0x80000000
-; X64-NEXT: cmovll %ecx, %edx
-; X64-NEXT: movd %edx, %xmm2
-; X64-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
-; X64-NEXT: movd %xmm3, %edx
-; X64-NEXT: movslq %edx, %rdx
-; X64-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
-; X64-NEXT: movd %xmm3, %esi
-; X64-NEXT: movslq %esi, %rsi
-; X64-NEXT: imulq %rdx, %rsi
-; X64-NEXT: movq %rsi, %rdx
-; X64-NEXT: shrq $32, %rdx
-; X64-NEXT: shrdl $2, %edx, %esi
+; X64-NEXT: movd %xmm0, %edx
; X64-NEXT: cmpl $2, %edx
-; X64-NEXT: cmovgel %eax, %esi
+; X64-NEXT: movl $2147483647, %esi # imm = 0x7FFFFFFF
+; X64-NEXT: cmovgel %esi, %ecx
; X64-NEXT: cmpl $-2, %edx
-; X64-NEXT: cmovll %ecx, %esi
-; X64-NEXT: movd %esi, %xmm3
-; X64-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; X64-NEXT: movd %xmm1, %edx
-; X64-NEXT: movslq %edx, %rdx
-; X64-NEXT: movd %xmm0, %esi
-; X64-NEXT: movslq %esi, %rsi
-; X64-NEXT: imulq %rdx, %rsi
-; X64-NEXT: movq %rsi, %rdx
-; X64-NEXT: shrq $32, %rdx
-; X64-NEXT: shrdl $2, %edx, %esi
-; X64-NEXT: cmpl $2, %edx
-; X64-NEXT: cmovgel %eax, %esi
-; X64-NEXT: cmpl $-2, %edx
-; X64-NEXT: cmovll %ecx, %esi
-; X64-NEXT: movd %esi, %xmm2
-; X64-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; X64-NEXT: movd %xmm1, %edx
-; X64-NEXT: movslq %edx, %rdx
-; X64-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; X64-NEXT: movd %xmm0, %esi
-; X64-NEXT: movslq %esi, %rsi
-; X64-NEXT: imulq %rdx, %rsi
-; X64-NEXT: movq %rsi, %rdx
-; X64-NEXT: shrq $32, %rdx
-; X64-NEXT: shrdl $2, %edx, %esi
-; X64-NEXT: cmpl $2, %edx
-; X64-NEXT: cmovgel %eax, %esi
-; X64-NEXT: cmpl $-2, %edx
-; X64-NEXT: cmovll %ecx, %esi
-; X64-NEXT: movd %esi, %xmm0
+; X64-NEXT: movl $-2147483648, %edx # imm = 0x80000000
+; X64-NEXT: cmovll %edx, %ecx
+; X64-NEXT: movd %ecx, %xmm0
+; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; X64-NEXT: movd %xmm2, %ecx
+; X64-NEXT: movd %xmm3, %edi
+; X64-NEXT: cmpl $2, %edi
+; X64-NEXT: cmovgel %esi, %ecx
+; X64-NEXT: cmpl $-2, %edi
+; X64-NEXT: cmovll %edx, %ecx
+; X64-NEXT: movd %ecx, %xmm2
; X64-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; X64-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; X64-NEXT: movdqa %xmm2, %xmm0
+; X64-NEXT: movd %xmm1, %ecx
+; X64-NEXT: cmpl $2, %eax
+; X64-NEXT: cmovgel %esi, %ecx
+; X64-NEXT: cmpl $-2, %eax
+; X64-NEXT: cmovll %edx, %ecx
+; X64-NEXT: movd %ecx, %xmm0
+; X64-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
+; X64-NEXT: movd %xmm1, %eax
+; X64-NEXT: movd %xmm4, %ecx
+; X64-NEXT: cmpl $2, %ecx
+; X64-NEXT: cmovgel %esi, %eax
+; X64-NEXT: cmpl $-2, %ecx
+; X64-NEXT: cmovll %edx, %eax
+; X64-NEXT: movd %eax, %xmm1
+; X64-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X64-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
; X64-NEXT: retq
;
; X86-LABEL: vec:
@@ -485,56 +481,31 @@ define i4 @func6(i4 %x, i4 %y) nounwind {
define <4 x i32> @vec2(<4 x i32> %x, <4 x i32> %y) nounwind {
; X64-LABEL: vec2:
; X64: # %bb.0:
-; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; X64-NEXT: movd %xmm2, %eax
-; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; X64-NEXT: movd %xmm2, %ecx
-; X64-NEXT: xorl %edx, %edx
-; X64-NEXT: movl %ecx, %esi
-; X64-NEXT: xorl %eax, %esi
-; X64-NEXT: sets %dl
-; X64-NEXT: addl $2147483647, %edx # imm = 0x7FFFFFFF
-; X64-NEXT: imull %eax, %ecx
-; X64-NEXT: cmovol %edx, %ecx
-; X64-NEXT: movd %ecx, %xmm2
-; X64-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
-; X64-NEXT: movd %xmm3, %eax
-; X64-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
-; X64-NEXT: movd %xmm3, %ecx
-; X64-NEXT: xorl %edx, %edx
-; X64-NEXT: movl %ecx, %esi
-; X64-NEXT: xorl %eax, %esi
-; X64-NEXT: sets %dl
-; X64-NEXT: addl $2147483647, %edx # imm = 0x7FFFFFFF
-; X64-NEXT: imull %eax, %ecx
-; X64-NEXT: cmovol %edx, %ecx
-; X64-NEXT: movd %ecx, %xmm3
-; X64-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; X64-NEXT: movd %xmm1, %eax
-; X64-NEXT: movd %xmm0, %ecx
-; X64-NEXT: xorl %edx, %edx
-; X64-NEXT: movl %ecx, %esi
-; X64-NEXT: xorl %eax, %esi
-; X64-NEXT: sets %dl
-; X64-NEXT: addl $2147483647, %edx # imm = 0x7FFFFFFF
-; X64-NEXT: imull %eax, %ecx
-; X64-NEXT: cmovol %edx, %ecx
-; X64-NEXT: movd %ecx, %xmm2
-; X64-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; X64-NEXT: movd %xmm1, %eax
-; X64-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; X64-NEXT: movd %xmm0, %ecx
-; X64-NEXT: xorl %edx, %edx
-; X64-NEXT: movl %ecx, %esi
-; X64-NEXT: xorl %eax, %esi
-; X64-NEXT: sets %dl
-; X64-NEXT: addl $2147483647, %edx # imm = 0x7FFFFFFF
-; X64-NEXT: imull %eax, %ecx
-; X64-NEXT: cmovol %edx, %ecx
-; X64-NEXT: movd %ecx, %xmm0
+; X64-NEXT: pxor %xmm2, %xmm2
+; X64-NEXT: pxor %xmm3, %xmm3
+; X64-NEXT: pcmpgtd %xmm1, %xmm3
+; X64-NEXT: pand %xmm0, %xmm3
+; X64-NEXT: pcmpgtd %xmm0, %xmm2
+; X64-NEXT: pand %xmm1, %xmm2
+; X64-NEXT: paddd %xmm3, %xmm2
+; X64-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; X64-NEXT: pmuludq %xmm1, %xmm0
+; X64-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,3,2,3]
+; X64-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; X64-NEXT: pmuludq %xmm3, %xmm1
+; X64-NEXT: pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
+; X64-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
+; X64-NEXT: psubd %xmm2, %xmm4
+; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,2,2,3]
+; X64-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
; X64-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; X64-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; X64-NEXT: psrad $31, %xmm2
+; X64-NEXT: pcmpeqd %xmm4, %xmm2
+; X64-NEXT: psrad $31, %xmm4
+; X64-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4
; X64-NEXT: movdqa %xmm2, %xmm0
+; X64-NEXT: pandn %xmm4, %xmm0
+; X64-NEXT: por %xmm2, %xmm0
; X64-NEXT: retq
;
; X86-LABEL: vec2:
diff --git a/llvm/test/CodeGen/X86/umul_fix_sat.ll b/llvm/test/CodeGen/X86/umul_fix_sat.ll
index 8c7078c726328..e08919c81b53a 100644
--- a/llvm/test/CodeGen/X86/umul_fix_sat.ll
+++ b/llvm/test/CodeGen/X86/umul_fix_sat.ll
@@ -138,53 +138,49 @@ define i4 @func3(i4 %x, i4 %y) nounwind {
define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
; X64-LABEL: vec:
; X64: # %bb.0:
-; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; X64-NEXT: movd %xmm2, %eax
-; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
+; X64-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; X64-NEXT: pmuludq %xmm1, %xmm0
+; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,2,2,3]
+; X64-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; X64-NEXT: pmuludq %xmm3, %xmm1
+; X64-NEXT: pshufd {{.*#+}} xmm3 = xmm1[0,2,2,3]
+; X64-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; X64-NEXT: psrld $2, %xmm2
+; X64-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
+; X64-NEXT: pshufd {{.*#+}} xmm4 = xmm1[1,3,2,3]
+; X64-NEXT: movd %xmm3, %eax
+; X64-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
+; X64-NEXT: pslld $30, %xmm3
+; X64-NEXT: por %xmm2, %xmm3
+; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm3[3,3,3,3]
; X64-NEXT: movd %xmm2, %ecx
-; X64-NEXT: imulq %rax, %rcx
-; X64-NEXT: movq %rcx, %rax
-; X64-NEXT: shrq $32, %rax
-; X64-NEXT: shrdl $2, %eax, %ecx
-; X64-NEXT: cmpl $4, %eax
-; X64-NEXT: movl $-1, %eax
-; X64-NEXT: cmovael %eax, %ecx
+; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
+; X64-NEXT: movd %xmm2, %edx
+; X64-NEXT: cmpl $4, %edx
+; X64-NEXT: movl $-1, %edx
+; X64-NEXT: cmovael %edx, %ecx
; X64-NEXT: movd %ecx, %xmm2
-; X64-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
-; X64-NEXT: movd %xmm3, %ecx
-; X64-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
-; X64-NEXT: movd %xmm3, %edx
-; X64-NEXT: imulq %rcx, %rdx
-; X64-NEXT: movq %rdx, %rcx
-; X64-NEXT: shrq $32, %rcx
-; X64-NEXT: shrdl $2, %ecx, %edx
-; X64-NEXT: cmpl $4, %ecx
-; X64-NEXT: cmovael %eax, %edx
-; X64-NEXT: movd %edx, %xmm3
-; X64-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; X64-NEXT: movd %xmm1, %ecx
-; X64-NEXT: movd %xmm0, %edx
-; X64-NEXT: imulq %rcx, %rdx
-; X64-NEXT: movq %rdx, %rcx
-; X64-NEXT: shrq $32, %rcx
-; X64-NEXT: shrdl $2, %ecx, %edx
+; X64-NEXT: pshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
+; X64-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; X64-NEXT: movd %xmm0, %ecx
; X64-NEXT: cmpl $4, %ecx
-; X64-NEXT: cmovael %eax, %edx
-; X64-NEXT: movd %edx, %xmm2
+; X64-NEXT: movd %xmm4, %ecx
+; X64-NEXT: cmovael %edx, %ecx
+; X64-NEXT: movd %ecx, %xmm4
+; X64-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; X64-NEXT: movd %xmm3, %ecx
+; X64-NEXT: cmpl $4, %eax
+; X64-NEXT: cmovael %edx, %ecx
+; X64-NEXT: movd %ecx, %xmm0
+; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm3[1,1,1,1]
+; X64-NEXT: movd %xmm2, %eax
; X64-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
; X64-NEXT: movd %xmm1, %ecx
-; X64-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; X64-NEXT: movd %xmm0, %edx
-; X64-NEXT: imulq %rcx, %rdx
-; X64-NEXT: movq %rdx, %rcx
-; X64-NEXT: shrq $32, %rcx
-; X64-NEXT: shrdl $2, %ecx, %edx
; X64-NEXT: cmpl $4, %ecx
-; X64-NEXT: cmovael %eax, %edx
-; X64-NEXT: movd %edx, %xmm0
-; X64-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; X64-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; X64-NEXT: movdqa %xmm2, %xmm0
+; X64-NEXT: cmovael %edx, %eax
+; X64-NEXT: movd %eax, %xmm1
+; X64-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X64-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
; X64-NEXT: retq
;
; X86-LABEL: vec:
@@ -351,37 +347,7 @@ define i4 @func6(i4 %x, i4 %y) nounwind {
define <4 x i32> @vec2(<4 x i32> %x, <4 x i32> %y) nounwind {
; X64-LABEL: vec2:
; X64: # %bb.0:
-; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; X64-NEXT: movd %xmm2, %eax
-; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; X64-NEXT: movd %xmm2, %ecx
-; X64-NEXT: mull %ecx
-; X64-NEXT: movl $-1, %ecx
-; X64-NEXT: cmovol %ecx, %eax
-; X64-NEXT: movd %eax, %xmm2
-; X64-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
-; X64-NEXT: movd %xmm3, %eax
-; X64-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
-; X64-NEXT: movd %xmm3, %edx
-; X64-NEXT: mull %edx
-; X64-NEXT: cmovol %ecx, %eax
-; X64-NEXT: movd %eax, %xmm3
-; X64-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; X64-NEXT: movd %xmm0, %eax
-; X64-NEXT: movd %xmm1, %edx
-; X64-NEXT: mull %edx
-; X64-NEXT: cmovol %ecx, %eax
-; X64-NEXT: movd %eax, %xmm2
-; X64-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; X64-NEXT: movd %xmm0, %eax
-; X64-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
-; X64-NEXT: movd %xmm0, %edx
-; X64-NEXT: mull %edx
-; X64-NEXT: cmovol %ecx, %eax
-; X64-NEXT: movd %eax, %xmm0
-; X64-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; X64-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; X64-NEXT: movdqa %xmm2, %xmm0
+; X64-NEXT: pcmpeqd %xmm0, %xmm0
; X64-NEXT: retq
;
; X86-LABEL: vec2:
diff --git a/llvm/test/CodeGen/X86/vector-mulfix-legalize.ll b/llvm/test/CodeGen/X86/vector-mulfix-legalize.ll
index 324fe12de9400..64a098a282779 100644
--- a/llvm/test/CodeGen/X86/vector-mulfix-legalize.ll
+++ b/llvm/test/CodeGen/X86/vector-mulfix-legalize.ll
@@ -43,57 +43,91 @@ define <4 x i16> @umulfix(<4 x i16> %a) {
define <4 x i16> @smulfixsat(<4 x i16> %a) {
; CHECK-LABEL: smulfixsat:
; CHECK: # %bb.0:
-; CHECK-NEXT: pextrw $2, %xmm0, %eax
-; CHECK-NEXT: cwtl
-; CHECK-NEXT: leal (%rax,%rax,2), %ecx
-; CHECK-NEXT: movl %ecx, %edx
-; CHECK-NEXT: shrl $16, %edx
-; CHECK-NEXT: shldw $1, %cx, %dx
-; CHECK-NEXT: sarl $16, %ecx
-; CHECK-NEXT: cmpl $16384, %ecx # imm = 0x4000
+; CHECK-NEXT: pushq %rbx
+; CHECK-NEXT: .cfi_def_cfa_offset 16
+; CHECK-NEXT: .cfi_offset %rbx, -16
+; CHECK-NEXT: movq {{.*#+}} xmm2 = [1,2,3,4,0,0,0,0]
+; CHECK-NEXT: movdqa %xmm0, %xmm1
+; CHECK-NEXT: pmullw %xmm2, %xmm1
+; CHECK-NEXT: psrlw $15, %xmm1
+; CHECK-NEXT: pmulhw %xmm2, %xmm0
+; CHECK-NEXT: pextrw $7, %xmm0, %eax
+; CHECK-NEXT: pextrw $6, %xmm0, %ecx
+; CHECK-NEXT: pextrw $5, %xmm0, %edx
+; CHECK-NEXT: pextrw $4, %xmm0, %esi
+; CHECK-NEXT: pextrw $3, %xmm0, %edi
+; CHECK-NEXT: pextrw $2, %xmm0, %r8d
+; CHECK-NEXT: pextrw $1, %xmm0, %r9d
+; CHECK-NEXT: movd %xmm0, %r10d
+; CHECK-NEXT: paddw %xmm0, %xmm0
+; CHECK-NEXT: por %xmm1, %xmm0
+; CHECK-NEXT: pextrw $7, %xmm0, %ebx
+; CHECK-NEXT: movswl %ax, %r11d
+; CHECK-NEXT: cmpl $16384, %r11d # imm = 0x4000
; CHECK-NEXT: movl $32767, %eax # imm = 0x7FFF
-; CHECK-NEXT: cmovgel %eax, %edx
+; CHECK-NEXT: cmovgel %eax, %ebx
+; CHECK-NEXT: cmpl $-16384, %r11d # imm = 0xC000
+; CHECK-NEXT: movl $32768, %r11d # imm = 0x8000
+; CHECK-NEXT: cmovll %r11d, %ebx
+; CHECK-NEXT: movd %ebx, %xmm3
+; CHECK-NEXT: pextrw $6, %xmm0, %ebx
+; CHECK-NEXT: movswl %cx, %ecx
+; CHECK-NEXT: cmpl $16384, %ecx # imm = 0x4000
+; CHECK-NEXT: cmovgel %eax, %ebx
; CHECK-NEXT: cmpl $-16384, %ecx # imm = 0xC000
-; CHECK-NEXT: movl $32768, %ecx # imm = 0x8000
-; CHECK-NEXT: cmovll %ecx, %edx
-; CHECK-NEXT: pextrw $1, %xmm0, %esi
-; CHECK-NEXT: leal (%rsi,%rsi), %edi
-; CHECK-NEXT: movswl %si, %r8d
-; CHECK-NEXT: movl %r8d, %esi
-; CHECK-NEXT: shrl $16, %esi
-; CHECK-NEXT: shldw $1, %di, %si
-; CHECK-NEXT: sarl $16, %r8d
-; CHECK-NEXT: cmpl $16384, %r8d # imm = 0x4000
-; CHECK-NEXT: cmovgel %eax, %esi
-; CHECK-NEXT: cmpl $-16384, %r8d # imm = 0xC000
-; CHECK-NEXT: cmovll %ecx, %esi
-; CHECK-NEXT: movd %xmm0, %edi
-; CHECK-NEXT: movswl %di, %edi
-; CHECK-NEXT: movl %edi, %r8d
-; CHECK-NEXT: shrl $16, %r8d
-; CHECK-NEXT: shldw $1, %di, %r8w
-; CHECK-NEXT: sarl $16, %edi
-; CHECK-NEXT: cmpl $16384, %edi # imm = 0x4000
-; CHECK-NEXT: cmovgel %eax, %r8d
-; CHECK-NEXT: cmpl $-16384, %edi # imm = 0xC000
-; CHECK-NEXT: cmovll %ecx, %r8d
-; CHECK-NEXT: movzwl %r8w, %edi
-; CHECK-NEXT: movd %edi, %xmm1
-; CHECK-NEXT: pinsrw $1, %esi, %xmm1
-; CHECK-NEXT: pinsrw $2, %edx, %xmm1
-; CHECK-NEXT: pextrw $3, %xmm0, %edx
+; CHECK-NEXT: cmovll %r11d, %ebx
+; CHECK-NEXT: movd %ebx, %xmm2
+; CHECK-NEXT: pextrw $5, %xmm0, %ecx
; CHECK-NEXT: movswl %dx, %edx
-; CHECK-NEXT: leal (,%rdx,4), %esi
-; CHECK-NEXT: movl %esi, %edi
-; CHECK-NEXT: shrl $16, %esi
-; CHECK-NEXT: shldw $1, %di, %si
-; CHECK-NEXT: sarl $14, %edx
; CHECK-NEXT: cmpl $16384, %edx # imm = 0x4000
-; CHECK-NEXT: cmovgel %eax, %esi
+; CHECK-NEXT: cmovgel %eax, %ecx
+; CHECK-NEXT: cmpl $-16384, %edx # imm = 0xC000
+; CHECK-NEXT: cmovll %r11d, %ecx
+; CHECK-NEXT: movd %ecx, %xmm4
+; CHECK-NEXT: pextrw $4, %xmm0, %ecx
+; CHECK-NEXT: movswl %si, %edx
+; CHECK-NEXT: cmpl $16384, %edx # imm = 0x4000
+; CHECK-NEXT: cmovgel %eax, %ecx
+; CHECK-NEXT: cmpl $-16384, %edx # imm = 0xC000
+; CHECK-NEXT: cmovll %r11d, %ecx
+; CHECK-NEXT: movd %ecx, %xmm1
+; CHECK-NEXT: pextrw $3, %xmm0, %ecx
+; CHECK-NEXT: movswl %di, %edx
+; CHECK-NEXT: cmpl $16384, %edx # imm = 0x4000
+; CHECK-NEXT: cmovgel %eax, %ecx
; CHECK-NEXT: cmpl $-16384, %edx # imm = 0xC000
-; CHECK-NEXT: cmovll %ecx, %esi
-; CHECK-NEXT: pinsrw $3, %esi, %xmm1
-; CHECK-NEXT: movdqa %xmm1, %xmm0
+; CHECK-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; CHECK-NEXT: cmovll %r11d, %ecx
+; CHECK-NEXT: movd %ecx, %xmm3
+; CHECK-NEXT: pextrw $2, %xmm0, %ecx
+; CHECK-NEXT: movswl %r8w, %edx
+; CHECK-NEXT: cmpl $16384, %edx # imm = 0x4000
+; CHECK-NEXT: cmovgel %eax, %ecx
+; CHECK-NEXT: cmpl $-16384, %edx # imm = 0xC000
+; CHECK-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
+; CHECK-NEXT: cmovll %r11d, %ecx
+; CHECK-NEXT: movd %ecx, %xmm4
+; CHECK-NEXT: pextrw $1, %xmm0, %ecx
+; CHECK-NEXT: movswl %r9w, %edx
+; CHECK-NEXT: cmpl $16384, %edx # imm = 0x4000
+; CHECK-NEXT: cmovgel %eax, %ecx
+; CHECK-NEXT: cmpl $-16384, %edx # imm = 0xC000
+; CHECK-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; CHECK-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; CHECK-NEXT: cmovll %r11d, %ecx
+; CHECK-NEXT: movd %ecx, %xmm2
+; CHECK-NEXT: movd %xmm0, %ecx
+; CHECK-NEXT: movswl %r10w, %edx
+; CHECK-NEXT: cmpl $16384, %edx # imm = 0x4000
+; CHECK-NEXT: cmovgel %eax, %ecx
+; CHECK-NEXT: cmpl $-16384, %edx # imm = 0xC000
+; CHECK-NEXT: cmovll %r11d, %ecx
+; CHECK-NEXT: movd %ecx, %xmm0
+; CHECK-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; CHECK-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
+; CHECK-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; CHECK-NEXT: popq %rbx
+; CHECK-NEXT: .cfi_def_cfa_offset 8
; CHECK-NEXT: retq
%t = call <4 x i16> @llvm.smul.fix.sat.v4i16(<4 x i16> <i16 1, i16 2, i16 3, i16 4>, <4 x i16> %a, i32 15)
ret <4 x i16> %t
@@ -103,43 +137,61 @@ define <4 x i16> @smulfixsat(<4 x i16> %a) {
define <4 x i16> @umulfixsat(<4 x i16> %a) {
; CHECK-LABEL: umulfixsat:
; CHECK: # %bb.0:
-; CHECK-NEXT: pextrw $2, %xmm0, %eax
-; CHECK-NEXT: leal (%rax,%rax,2), %eax
-; CHECK-NEXT: movl %eax, %edx
-; CHECK-NEXT: shrl $16, %eax
-; CHECK-NEXT: movl %eax, %ecx
-; CHECK-NEXT: shldw $1, %dx, %cx
-; CHECK-NEXT: cmpl $32768, %eax # imm = 0x8000
+; CHECK-NEXT: movq {{.*#+}} xmm2 = [1,2,3,4,0,0,0,0]
+; CHECK-NEXT: movdqa %xmm0, %xmm1
+; CHECK-NEXT: pmullw %xmm2, %xmm1
+; CHECK-NEXT: psrlw $15, %xmm1
+; CHECK-NEXT: pmulhuw %xmm2, %xmm0
+; CHECK-NEXT: pextrw $7, %xmm0, %eax
+; CHECK-NEXT: pextrw $6, %xmm0, %ecx
+; CHECK-NEXT: pextrw $5, %xmm0, %edx
+; CHECK-NEXT: pextrw $4, %xmm0, %esi
+; CHECK-NEXT: pextrw $3, %xmm0, %edi
+; CHECK-NEXT: pextrw $2, %xmm0, %r8d
+; CHECK-NEXT: pextrw $1, %xmm0, %r9d
+; CHECK-NEXT: paddw %xmm0, %xmm0
+; CHECK-NEXT: por %xmm1, %xmm0
+; CHECK-NEXT: pextrw $7, %xmm0, %r10d
+; CHECK-NEXT: testw %ax, %ax
; CHECK-NEXT: movl $65535, %eax # imm = 0xFFFF
-; CHECK-NEXT: cmovael %eax, %ecx
-; CHECK-NEXT: pextrw $1, %xmm0, %edx
-; CHECK-NEXT: addl %edx, %edx
-; CHECK-NEXT: movl %edx, %esi
-; CHECK-NEXT: shrl $16, %edx
-; CHECK-NEXT: movl %edx, %edi
-; CHECK-NEXT: shldw $1, %si, %di
-; CHECK-NEXT: cmpl $32768, %edx # imm = 0x8000
-; CHECK-NEXT: cmovael %eax, %edi
-; CHECK-NEXT: movd %xmm0, %edx
-; CHECK-NEXT: xorl %esi, %esi
-; CHECK-NEXT: shldw $1, %dx, %si
-; CHECK-NEXT: movl $32768, %edx # imm = 0x8000
-; CHECK-NEXT: negl %edx
-; CHECK-NEXT: cmovael %eax, %esi
-; CHECK-NEXT: movzwl %si, %edx
-; CHECK-NEXT: movd %edx, %xmm1
-; CHECK-NEXT: pinsrw $1, %edi, %xmm1
-; CHECK-NEXT: pinsrw $2, %ecx, %xmm1
+; CHECK-NEXT: cmovsl %eax, %r10d
+; CHECK-NEXT: movd %r10d, %xmm1
+; CHECK-NEXT: testw %cx, %cx
+; CHECK-NEXT: pextrw $6, %xmm0, %ecx
+; CHECK-NEXT: cmovsl %eax, %ecx
+; CHECK-NEXT: movd %ecx, %xmm2
+; CHECK-NEXT: testw %dx, %dx
+; CHECK-NEXT: pextrw $5, %xmm0, %ecx
+; CHECK-NEXT: cmovsl %eax, %ecx
+; CHECK-NEXT: movd %ecx, %xmm3
+; CHECK-NEXT: testw %si, %si
+; CHECK-NEXT: pextrw $4, %xmm0, %ecx
+; CHECK-NEXT: cmovsl %eax, %ecx
+; CHECK-NEXT: movd %ecx, %xmm4
+; CHECK-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; CHECK-NEXT: testw %di, %di
; CHECK-NEXT: pextrw $3, %xmm0, %ecx
-; CHECK-NEXT: shll $2, %ecx
-; CHECK-NEXT: movl %ecx, %edx
-; CHECK-NEXT: shrl $16, %ecx
-; CHECK-NEXT: movl %ecx, %esi
-; CHECK-NEXT: shldw $1, %dx, %si
-; CHECK-NEXT: cmpl $32768, %ecx # imm = 0x8000
-; CHECK-NEXT: cmovael %eax, %esi
-; CHECK-NEXT: pinsrw $3, %esi, %xmm1
-; CHECK-NEXT: movdqa %xmm1, %xmm0
+; CHECK-NEXT: cmovsl %eax, %ecx
+; CHECK-NEXT: movd %ecx, %xmm1
+; CHECK-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
+; CHECK-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; CHECK-NEXT: testw %r8w, %r8w
+; CHECK-NEXT: pextrw $2, %xmm0, %ecx
+; CHECK-NEXT: cmovsl %eax, %ecx
+; CHECK-NEXT: movd %ecx, %xmm2
+; CHECK-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; CHECK-NEXT: pextrw $1, %xmm0, %ecx
+; CHECK-NEXT: testw %r9w, %r9w
+; CHECK-NEXT: cmovsl %eax, %ecx
+; CHECK-NEXT: movd %ecx, %xmm1
+; CHECK-NEXT: movd %xmm0, %ecx
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: testw %dx, %dx
+; CHECK-NEXT: cmovsl %eax, %ecx
+; CHECK-NEXT: movd %ecx, %xmm0
+; CHECK-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; CHECK-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; CHECK-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
; CHECK-NEXT: retq
%t = call <4 x i16> @llvm.umul.fix.sat.v4i16(<4 x i16> <i16 1, i16 2, i16 3, i16 4>, <4 x i16> %a, i32 15)
ret <4 x i16> %t
>From 0e0e7805676f3633dff32049c57e4ef98c75081e Mon Sep 17 00:00:00 2001
From: Guy Fischman <gfischman at gmail.com>
Date: Thu, 23 Jul 2026 11:59:17 +0200
Subject: [PATCH 2/2] [SelectionDAG][AArch64][X86] Vectorize the fixed-point
saturation clamp
---
.../CodeGen/SelectionDAG/TargetLowering.cpp | 38 +-
llvm/test/CodeGen/AArch64/smul_fix_sat.ll | 456 ++++--------------
llvm/test/CodeGen/AArch64/umul_fix_sat.ll | 335 +++----------
llvm/test/CodeGen/X86/smul_fix_sat.ll | 121 ++---
llvm/test/CodeGen/X86/umul_fix_sat.ll | 54 +--
.../CodeGen/X86/vector-mulfix-legalize.ll | 169 ++-----
6 files changed, 258 insertions(+), 915 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index ae0aa07443845..b82f42493f57d 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -12900,6 +12900,16 @@ TargetLowering::expandFixedPointMul(SDNode *Node, SelectionDAG &DAG) const {
assert(LHS.getValueType() == RHS.getValueType() &&
"Expected both operands to be the same type");
+ // Select the saturated value when Cond0 <CC> Cond1, keeping it vectorized:
+ // SELECT_CC is scalarized for vector types, so build SETCC + VSELECT there.
+ auto getSaturatingSelect = [&](SDValue Cond0, SDValue Cond1, SDValue Sat,
+ SDValue Val, ISD::CondCode CC) {
+ if (VT.isVector())
+ return DAG.getSelect(dl, VT, DAG.getSetCC(dl, BoolVT, Cond0, Cond1, CC),
+ Sat, Val);
+ return DAG.getSelectCC(dl, Cond0, Cond1, Sat, Val, CC);
+ };
+
// Get the upper and lower bits of the result.
SDValue Lo, Hi;
unsigned LoHiOp = Signed ? ISD::SMUL_LOHI : ISD::UMUL_LOHI;
@@ -12950,13 +12960,10 @@ TargetLowering::expandFixedPointMul(SDNode *Node, SelectionDAG &DAG) const {
// Saturate to max if ((Hi >> Scale) != 0),
// which is the same as if (Hi > ((1 << Scale) - 1))
APInt MaxVal = APInt::getMaxValue(VTSize);
- SDValue LowMask = DAG.getConstant(APInt::getLowBitsSet(VTSize, Scale),
- dl, VT);
- Result = DAG.getSelectCC(dl, Hi, LowMask,
- DAG.getConstant(MaxVal, dl, VT), Result,
- ISD::SETUGT);
-
- return Result;
+ SDValue LowMask =
+ DAG.getConstant(APInt::getLowBitsSet(VTSize, Scale), dl, VT);
+ return getSaturatingSelect(Hi, LowMask, DAG.getConstant(MaxVal, dl, VT),
+ Result, ISD::SETUGT);
}
// Signed overflow happened if the upper (VTSize - Scale + 1) bits (of the
@@ -12972,8 +12979,8 @@ TargetLowering::expandFixedPointMul(SDNode *Node, SelectionDAG &DAG) const {
// Saturated to SatMin if wide product is negative, and SatMax if wide
// product is positive ...
SDValue Zero = DAG.getConstant(0, dl, VT);
- SDValue ResultIfOverflow = DAG.getSelectCC(dl, Hi, Zero, SatMin, SatMax,
- ISD::SETLT);
+ SDValue ResultIfOverflow =
+ getSaturatingSelect(Hi, Zero, SatMin, SatMax, ISD::SETLT);
// ... but only if we overflowed.
return DAG.getSelect(dl, VT, Overflow, ResultIfOverflow, Result);
}
@@ -12982,15 +12989,14 @@ TargetLowering::expandFixedPointMul(SDNode *Node, SelectionDAG &DAG) const {
// Saturate to max if ((Hi >> (Scale - 1)) > 0),
// which is the same as if (Hi > (1 << (Scale - 1)) - 1)
- SDValue LowMask = DAG.getConstant(APInt::getLowBitsSet(VTSize, Scale - 1),
- dl, VT);
- Result = DAG.getSelectCC(dl, Hi, LowMask, SatMax, Result, ISD::SETGT);
+ SDValue LowMask =
+ DAG.getConstant(APInt::getLowBitsSet(VTSize, Scale - 1), dl, VT);
// Saturate to min if (Hi >> (Scale - 1)) < -1),
// which is the same as if (HI < (-1 << (Scale - 1))
- SDValue HighMask =
- DAG.getConstant(APInt::getHighBitsSet(VTSize, VTSize - Scale + 1),
- dl, VT);
- Result = DAG.getSelectCC(dl, Hi, HighMask, SatMin, Result, ISD::SETLT);
+ SDValue HighMask = DAG.getConstant(
+ APInt::getHighBitsSet(VTSize, VTSize - Scale + 1), dl, VT);
+ Result = getSaturatingSelect(Hi, LowMask, SatMax, Result, ISD::SETGT);
+ Result = getSaturatingSelect(Hi, HighMask, SatMin, Result, ISD::SETLT);
return Result;
}
diff --git a/llvm/test/CodeGen/AArch64/smul_fix_sat.ll b/llvm/test/CodeGen/AArch64/smul_fix_sat.ll
index b5d81af9f2dfe..1d85853f7fa8d 100644
--- a/llvm/test/CodeGen/AArch64/smul_fix_sat.ll
+++ b/llvm/test/CodeGen/AArch64/smul_fix_sat.ll
@@ -166,69 +166,18 @@ define <8 x i8> @vec_v8i8(<8 x i8> %x, <8 x i8> %y) {
; CHECK-LABEL: vec_v8i8:
; CHECK: // %bb.0:
; CHECK-NEXT: smull v0.8h, v0.8b, v1.8b
-; CHECK-NEXT: mov w8, #127 // =0x7f
-; CHECK-NEXT: shrn v2.8b, v0.8h, #8
+; CHECK-NEXT: movi v2.8b, #1
+; CHECK-NEXT: movi v4.8b, #127
+; CHECK-NEXT: shrn v1.8b, v0.8h, #8
; CHECK-NEXT: xtn v0.8b, v0.8h
-; CHECK-NEXT: shl v1.8b, v2.8b, #6
-; CHECK-NEXT: smov w9, v2.b[1]
-; CHECK-NEXT: smov w11, v2.b[0]
-; CHECK-NEXT: usra v1.8b, v0.8b, #2
-; CHECK-NEXT: cmp w9, #1
-; CHECK-NEXT: umov w10, v1.b[1]
-; CHECK-NEXT: umov w12, v1.b[0]
-; CHECK-NEXT: umov w13, v1.b[2]
-; CHECK-NEXT: csel w10, w8, w10, gt
-; CHECK-NEXT: cmn w9, #2
-; CHECK-NEXT: mov w9, #-128 // =0xffffff80
-; CHECK-NEXT: csel w10, w9, w10, lt
-; CHECK-NEXT: cmp w11, #1
-; CHECK-NEXT: csel w12, w8, w12, gt
-; CHECK-NEXT: cmn w11, #2
-; CHECK-NEXT: smov w11, v2.b[2]
-; CHECK-NEXT: csel w12, w9, w12, lt
-; CHECK-NEXT: fmov s0, w12
-; CHECK-NEXT: cmp w11, #1
-; CHECK-NEXT: mov v0.b[1], w10
-; CHECK-NEXT: smov w10, v2.b[3]
-; CHECK-NEXT: csel w12, w8, w13, gt
-; CHECK-NEXT: cmn w11, #2
-; CHECK-NEXT: umov w11, v1.b[3]
-; CHECK-NEXT: csel w12, w9, w12, lt
-; CHECK-NEXT: mov v0.b[2], w12
-; CHECK-NEXT: cmp w10, #1
-; CHECK-NEXT: smov w12, v2.b[4]
-; CHECK-NEXT: csel w11, w8, w11, gt
-; CHECK-NEXT: cmn w10, #2
-; CHECK-NEXT: umov w10, v1.b[4]
-; CHECK-NEXT: csel w11, w9, w11, lt
-; CHECK-NEXT: mov v0.b[3], w11
-; CHECK-NEXT: cmp w12, #1
-; CHECK-NEXT: smov w11, v2.b[5]
-; CHECK-NEXT: csel w10, w8, w10, gt
-; CHECK-NEXT: cmn w12, #2
-; CHECK-NEXT: umov w12, v1.b[5]
-; CHECK-NEXT: csel w10, w9, w10, lt
-; CHECK-NEXT: mov v0.b[4], w10
-; CHECK-NEXT: cmp w11, #1
-; CHECK-NEXT: smov w10, v2.b[6]
-; CHECK-NEXT: csel w12, w8, w12, gt
-; CHECK-NEXT: cmn w11, #2
-; CHECK-NEXT: umov w11, v1.b[6]
-; CHECK-NEXT: csel w12, w9, w12, lt
-; CHECK-NEXT: mov v0.b[5], w12
-; CHECK-NEXT: cmp w10, #1
-; CHECK-NEXT: smov w12, v2.b[7]
-; CHECK-NEXT: csel w11, w8, w11, gt
-; CHECK-NEXT: cmn w10, #2
-; CHECK-NEXT: umov w10, v1.b[7]
-; CHECK-NEXT: csel w11, w9, w11, lt
-; CHECK-NEXT: mov v0.b[6], w11
-; CHECK-NEXT: cmp w12, #1
-; CHECK-NEXT: csel w8, w8, w10, gt
-; CHECK-NEXT: cmn w12, #2
-; CHECK-NEXT: csel w8, w9, w8, lt
-; CHECK-NEXT: mov v0.b[7], w8
-; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT: shl v3.8b, v1.8b, #6
+; CHECK-NEXT: usra v3.8b, v0.8b, #2
+; CHECK-NEXT: cmgt v0.8b, v1.8b, v2.8b
+; CHECK-NEXT: movi v2.8b, #254
+; CHECK-NEXT: bsl v0.8b, v4.8b, v3.8b
+; CHECK-NEXT: movi v3.8b, #128
+; CHECK-NEXT: cmgt v1.8b, v2.8b, v1.8b
+; CHECK-NEXT: bit v0.8b, v3.8b, v1.8b
; CHECK-NEXT: ret
%tmp = call <8 x i8> @llvm.smul.fix.sat.v8i8(<8 x i8> %x, <8 x i8> %y, i32 2)
ret <8 x i8> %tmp
@@ -237,126 +186,20 @@ define <8 x i8> @vec_v8i8(<8 x i8> %x, <8 x i8> %y) {
define <16 x i8> @vec_v16i8(<16 x i8> %x, <16 x i8> %y) {
; CHECK-LABEL: vec_v16i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: smull2 v2.8h, v0.16b, v1.16b
-; CHECK-NEXT: smull v3.8h, v0.8b, v1.8b
-; CHECK-NEXT: mov w8, #127 // =0x7f
+; CHECK-NEXT: smull2 v3.8h, v0.16b, v1.16b
+; CHECK-NEXT: smull v4.8h, v0.8b, v1.8b
+; CHECK-NEXT: movi v2.16b, #1
; CHECK-NEXT: mul v0.16b, v0.16b, v1.16b
-; CHECK-NEXT: uzp2 v2.16b, v3.16b, v2.16b
-; CHECK-NEXT: shl v1.16b, v2.16b, #6
-; CHECK-NEXT: smov w9, v2.b[1]
-; CHECK-NEXT: smov w11, v2.b[0]
+; CHECK-NEXT: uzp2 v3.16b, v4.16b, v3.16b
+; CHECK-NEXT: movi v4.16b, #127
+; CHECK-NEXT: shl v1.16b, v3.16b, #6
+; CHECK-NEXT: cmgt v2.16b, v3.16b, v2.16b
; CHECK-NEXT: usra v1.16b, v0.16b, #2
-; CHECK-NEXT: cmp w9, #1
-; CHECK-NEXT: umov w10, v1.b[1]
-; CHECK-NEXT: umov w12, v1.b[0]
-; CHECK-NEXT: umov w13, v1.b[2]
-; CHECK-NEXT: csel w10, w8, w10, gt
-; CHECK-NEXT: cmn w9, #2
-; CHECK-NEXT: mov w9, #-128 // =0xffffff80
-; CHECK-NEXT: csel w10, w9, w10, lt
-; CHECK-NEXT: cmp w11, #1
-; CHECK-NEXT: csel w12, w8, w12, gt
-; CHECK-NEXT: cmn w11, #2
-; CHECK-NEXT: smov w11, v2.b[2]
-; CHECK-NEXT: csel w12, w9, w12, lt
-; CHECK-NEXT: fmov s0, w12
-; CHECK-NEXT: cmp w11, #1
-; CHECK-NEXT: mov v0.b[1], w10
-; CHECK-NEXT: smov w10, v2.b[3]
-; CHECK-NEXT: csel w12, w8, w13, gt
-; CHECK-NEXT: cmn w11, #2
-; CHECK-NEXT: umov w11, v1.b[3]
-; CHECK-NEXT: csel w12, w9, w12, lt
-; CHECK-NEXT: mov v0.b[2], w12
-; CHECK-NEXT: cmp w10, #1
-; CHECK-NEXT: smov w12, v2.b[4]
-; CHECK-NEXT: csel w11, w8, w11, gt
-; CHECK-NEXT: cmn w10, #2
-; CHECK-NEXT: umov w10, v1.b[4]
-; CHECK-NEXT: csel w11, w9, w11, lt
-; CHECK-NEXT: mov v0.b[3], w11
-; CHECK-NEXT: cmp w12, #1
-; CHECK-NEXT: smov w11, v2.b[5]
-; CHECK-NEXT: csel w10, w8, w10, gt
-; CHECK-NEXT: cmn w12, #2
-; CHECK-NEXT: umov w12, v1.b[5]
-; CHECK-NEXT: csel w10, w9, w10, lt
-; CHECK-NEXT: mov v0.b[4], w10
-; CHECK-NEXT: cmp w11, #1
-; CHECK-NEXT: smov w10, v2.b[6]
-; CHECK-NEXT: csel w12, w8, w12, gt
-; CHECK-NEXT: cmn w11, #2
-; CHECK-NEXT: umov w11, v1.b[6]
-; CHECK-NEXT: csel w12, w9, w12, lt
-; CHECK-NEXT: mov v0.b[5], w12
-; CHECK-NEXT: cmp w10, #1
-; CHECK-NEXT: smov w12, v2.b[7]
-; CHECK-NEXT: csel w11, w8, w11, gt
-; CHECK-NEXT: cmn w10, #2
-; CHECK-NEXT: umov w10, v1.b[7]
-; CHECK-NEXT: csel w11, w9, w11, lt
-; CHECK-NEXT: mov v0.b[6], w11
-; CHECK-NEXT: cmp w12, #1
-; CHECK-NEXT: smov w11, v2.b[8]
-; CHECK-NEXT: csel w10, w8, w10, gt
-; CHECK-NEXT: cmn w12, #2
-; CHECK-NEXT: umov w12, v1.b[8]
-; CHECK-NEXT: csel w10, w9, w10, lt
-; CHECK-NEXT: mov v0.b[7], w10
-; CHECK-NEXT: cmp w11, #1
-; CHECK-NEXT: smov w10, v2.b[9]
-; CHECK-NEXT: csel w12, w8, w12, gt
-; CHECK-NEXT: cmn w11, #2
-; CHECK-NEXT: umov w11, v1.b[9]
-; CHECK-NEXT: csel w12, w9, w12, lt
-; CHECK-NEXT: mov v0.b[8], w12
-; CHECK-NEXT: cmp w10, #1
-; CHECK-NEXT: smov w12, v2.b[10]
-; CHECK-NEXT: csel w11, w8, w11, gt
-; CHECK-NEXT: cmn w10, #2
-; CHECK-NEXT: umov w10, v1.b[10]
-; CHECK-NEXT: csel w11, w9, w11, lt
-; CHECK-NEXT: mov v0.b[9], w11
-; CHECK-NEXT: cmp w12, #1
-; CHECK-NEXT: smov w11, v2.b[11]
-; CHECK-NEXT: csel w10, w8, w10, gt
-; CHECK-NEXT: cmn w12, #2
-; CHECK-NEXT: umov w12, v1.b[11]
-; CHECK-NEXT: csel w10, w9, w10, lt
-; CHECK-NEXT: mov v0.b[10], w10
-; CHECK-NEXT: cmp w11, #1
-; CHECK-NEXT: smov w10, v2.b[12]
-; CHECK-NEXT: csel w12, w8, w12, gt
-; CHECK-NEXT: cmn w11, #2
-; CHECK-NEXT: umov w11, v1.b[12]
-; CHECK-NEXT: csel w12, w9, w12, lt
-; CHECK-NEXT: mov v0.b[11], w12
-; CHECK-NEXT: cmp w10, #1
-; CHECK-NEXT: smov w12, v2.b[13]
-; CHECK-NEXT: csel w11, w8, w11, gt
-; CHECK-NEXT: cmn w10, #2
-; CHECK-NEXT: umov w10, v1.b[13]
-; CHECK-NEXT: csel w11, w9, w11, lt
-; CHECK-NEXT: mov v0.b[12], w11
-; CHECK-NEXT: cmp w12, #1
-; CHECK-NEXT: smov w11, v2.b[14]
-; CHECK-NEXT: csel w10, w8, w10, gt
-; CHECK-NEXT: cmn w12, #2
-; CHECK-NEXT: umov w12, v1.b[14]
-; CHECK-NEXT: csel w10, w9, w10, lt
-; CHECK-NEXT: mov v0.b[13], w10
-; CHECK-NEXT: cmp w11, #1
-; CHECK-NEXT: smov w10, v2.b[15]
-; CHECK-NEXT: csel w12, w8, w12, gt
-; CHECK-NEXT: cmn w11, #2
-; CHECK-NEXT: umov w11, v1.b[15]
-; CHECK-NEXT: csel w12, w9, w12, lt
-; CHECK-NEXT: mov v0.b[14], w12
-; CHECK-NEXT: cmp w10, #1
-; CHECK-NEXT: csel w8, w8, w11, gt
-; CHECK-NEXT: cmn w10, #2
-; CHECK-NEXT: csel w8, w9, w8, lt
-; CHECK-NEXT: mov v0.b[15], w8
+; CHECK-NEXT: movi v0.16b, #254
+; CHECK-NEXT: bit v1.16b, v4.16b, v2.16b
+; CHECK-NEXT: movi v2.16b, #128
+; CHECK-NEXT: cmgt v0.16b, v0.16b, v3.16b
+; CHECK-NEXT: bsl v0.16b, v2.16b, v1.16b
; CHECK-NEXT: ret
%tmp = call <16 x i8> @llvm.smul.fix.sat.v16i8(<16 x i8> %x, <16 x i8> %y, i32 2)
ret <16 x i8> %tmp
@@ -366,41 +209,19 @@ define <4 x i16> @vec_v4i16(<4 x i16> %x, <4 x i16> %y) {
; CHECK-LABEL: vec_v4i16:
; CHECK: // %bb.0:
; CHECK-NEXT: smull v0.4s, v0.4h, v1.4h
-; CHECK-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-NEXT: mvni v2.4h, #254, lsl #8
+; CHECK-NEXT: movi v4.4h, #128, lsl #8
; CHECK-NEXT: shrn v1.4h, v0.4s, #16
; CHECK-NEXT: xtn v0.4h, v0.4s
-; CHECK-NEXT: shl v2.4h, v1.4h, #6
-; CHECK-NEXT: smov w9, v1.h[1]
-; CHECK-NEXT: smov w11, v1.h[0]
-; CHECK-NEXT: usra v2.4h, v0.4h, #10
-; CHECK-NEXT: cmp w9, #511
-; CHECK-NEXT: umov w10, v2.h[1]
-; CHECK-NEXT: umov w12, v2.h[0]
-; CHECK-NEXT: umov w13, v2.h[2]
-; CHECK-NEXT: csel w10, w8, w10, gt
-; CHECK-NEXT: cmn w9, #512
-; CHECK-NEXT: mov w9, #-32768 // =0xffff8000
-; CHECK-NEXT: csel w10, w9, w10, lt
-; CHECK-NEXT: cmp w11, #511
-; CHECK-NEXT: csel w12, w8, w12, gt
-; CHECK-NEXT: cmn w11, #512
-; CHECK-NEXT: smov w11, v1.h[2]
-; CHECK-NEXT: csel w12, w9, w12, lt
-; CHECK-NEXT: fmov s0, w12
-; CHECK-NEXT: cmp w11, #511
-; CHECK-NEXT: mov v0.h[1], w10
-; CHECK-NEXT: smov w10, v1.h[3]
-; CHECK-NEXT: csel w12, w8, w13, gt
-; CHECK-NEXT: cmn w11, #512
-; CHECK-NEXT: umov w11, v2.h[3]
-; CHECK-NEXT: csel w12, w9, w12, lt
-; CHECK-NEXT: mov v0.h[2], w12
-; CHECK-NEXT: cmp w10, #511
-; CHECK-NEXT: csel w8, w8, w11, gt
-; CHECK-NEXT: cmn w10, #512
-; CHECK-NEXT: csel w8, w9, w8, lt
-; CHECK-NEXT: mov v0.h[3], w8
-; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT: shl v3.4h, v1.4h, #6
+; CHECK-NEXT: usra v3.4h, v0.4h, #10
+; CHECK-NEXT: cmgt v0.4h, v1.4h, v2.4h
+; CHECK-NEXT: movi v2.4h, #254, lsl #8
+; CHECK-NEXT: bic v3.8b, v3.8b, v0.8b
+; CHECK-NEXT: bic v0.4h, #128, lsl #8
+; CHECK-NEXT: cmgt v1.4h, v2.4h, v1.4h
+; CHECK-NEXT: orr v0.8b, v0.8b, v3.8b
+; CHECK-NEXT: bit v0.8b, v4.8b, v1.8b
; CHECK-NEXT: ret
%tmp = call <4 x i16> @llvm.smul.fix.sat.v4i16(<4 x i16> %x, <4 x i16> %y, i32 10)
ret <4 x i16> %tmp
@@ -409,70 +230,21 @@ define <4 x i16> @vec_v4i16(<4 x i16> %x, <4 x i16> %y) {
define <8 x i16> @vec_v8i16(<8 x i16> %x, <8 x i16> %y) {
; CHECK-LABEL: vec_v8i16:
; CHECK: // %bb.0:
-; CHECK-NEXT: smull2 v2.4s, v0.8h, v1.8h
-; CHECK-NEXT: smull v3.4s, v0.4h, v1.4h
-; CHECK-NEXT: mov w8, #32767 // =0x7fff
+; CHECK-NEXT: smull2 v3.4s, v0.8h, v1.8h
+; CHECK-NEXT: smull v4.4s, v0.4h, v1.4h
+; CHECK-NEXT: movi v2.8h, #1
; CHECK-NEXT: mul v0.8h, v0.8h, v1.8h
-; CHECK-NEXT: uzp2 v2.8h, v3.8h, v2.8h
-; CHECK-NEXT: shl v1.8h, v2.8h, #14
-; CHECK-NEXT: smov w9, v2.h[1]
-; CHECK-NEXT: smov w11, v2.h[0]
+; CHECK-NEXT: uzp2 v3.8h, v4.8h, v3.8h
+; CHECK-NEXT: movi v4.8h, #128, lsl #8
+; CHECK-NEXT: shl v1.8h, v3.8h, #14
+; CHECK-NEXT: cmgt v2.8h, v3.8h, v2.8h
; CHECK-NEXT: usra v1.8h, v0.8h, #2
-; CHECK-NEXT: cmp w9, #1
-; CHECK-NEXT: umov w10, v1.h[1]
-; CHECK-NEXT: umov w12, v1.h[0]
-; CHECK-NEXT: umov w13, v1.h[2]
-; CHECK-NEXT: csel w10, w8, w10, gt
-; CHECK-NEXT: cmn w9, #2
-; CHECK-NEXT: mov w9, #-32768 // =0xffff8000
-; CHECK-NEXT: csel w10, w9, w10, lt
-; CHECK-NEXT: cmp w11, #1
-; CHECK-NEXT: csel w12, w8, w12, gt
-; CHECK-NEXT: cmn w11, #2
-; CHECK-NEXT: smov w11, v2.h[2]
-; CHECK-NEXT: csel w12, w9, w12, lt
-; CHECK-NEXT: fmov s0, w12
-; CHECK-NEXT: cmp w11, #1
-; CHECK-NEXT: mov v0.h[1], w10
-; CHECK-NEXT: smov w10, v2.h[3]
-; CHECK-NEXT: csel w12, w8, w13, gt
-; CHECK-NEXT: cmn w11, #2
-; CHECK-NEXT: umov w11, v1.h[3]
-; CHECK-NEXT: csel w12, w9, w12, lt
-; CHECK-NEXT: mov v0.h[2], w12
-; CHECK-NEXT: cmp w10, #1
-; CHECK-NEXT: smov w12, v2.h[4]
-; CHECK-NEXT: csel w11, w8, w11, gt
-; CHECK-NEXT: cmn w10, #2
-; CHECK-NEXT: umov w10, v1.h[4]
-; CHECK-NEXT: csel w11, w9, w11, lt
-; CHECK-NEXT: mov v0.h[3], w11
-; CHECK-NEXT: cmp w12, #1
-; CHECK-NEXT: smov w11, v2.h[5]
-; CHECK-NEXT: csel w10, w8, w10, gt
-; CHECK-NEXT: cmn w12, #2
-; CHECK-NEXT: umov w12, v1.h[5]
-; CHECK-NEXT: csel w10, w9, w10, lt
-; CHECK-NEXT: mov v0.h[4], w10
-; CHECK-NEXT: cmp w11, #1
-; CHECK-NEXT: smov w10, v2.h[6]
-; CHECK-NEXT: csel w12, w8, w12, gt
-; CHECK-NEXT: cmn w11, #2
-; CHECK-NEXT: umov w11, v1.h[6]
-; CHECK-NEXT: csel w12, w9, w12, lt
-; CHECK-NEXT: mov v0.h[5], w12
-; CHECK-NEXT: cmp w10, #1
-; CHECK-NEXT: smov w12, v2.h[7]
-; CHECK-NEXT: csel w11, w8, w11, gt
-; CHECK-NEXT: cmn w10, #2
-; CHECK-NEXT: umov w10, v1.h[7]
-; CHECK-NEXT: csel w11, w9, w11, lt
-; CHECK-NEXT: mov v0.h[6], w11
-; CHECK-NEXT: cmp w12, #1
-; CHECK-NEXT: csel w8, w8, w10, gt
-; CHECK-NEXT: cmn w12, #2
-; CHECK-NEXT: csel w8, w9, w8, lt
-; CHECK-NEXT: mov v0.h[7], w8
+; CHECK-NEXT: bic v0.16b, v1.16b, v2.16b
+; CHECK-NEXT: bic v2.8h, #128, lsl #8
+; CHECK-NEXT: mvni v1.8h, #1
+; CHECK-NEXT: orr v0.16b, v2.16b, v0.16b
+; CHECK-NEXT: cmgt v1.8h, v1.8h, v3.8h
+; CHECK-NEXT: bit v0.16b, v4.16b, v1.16b
; CHECK-NEXT: ret
%tmp = call <8 x i16> @llvm.smul.fix.sat.v8i16(<8 x i16> %x, <8 x i16> %y, i32 2)
ret <8 x i16> %tmp
@@ -482,22 +254,18 @@ define <2 x i32> @vec_v2i32(<2 x i32> %x, <2 x i32> %y) {
; CHECK-LABEL: vec_v2i32:
; CHECK: // %bb.0:
; CHECK-NEXT: smull v0.2d, v0.2s, v1.2s
+; CHECK-NEXT: movi v2.2s, #128, lsl #24
; CHECK-NEXT: shrn v1.2s, v0.2d, #32
; CHECK-NEXT: xtn v0.2s, v0.2d
-; CHECK-NEXT: fmov w9, s1
-; CHECK-NEXT: mov w8, v1.s[1]
-; CHECK-NEXT: add v2.2s, v1.2s, v1.2s
-; CHECK-NEXT: cmlt v3.2s, v0.2s, #0
-; CHECK-NEXT: asr w9, w9, #31
-; CHECK-NEXT: shl v2.2s, v2.2s, #31
-; CHECK-NEXT: asr w8, w8, #31
-; CHECK-NEXT: cmeq v1.2s, v1.2s, v3.2s
-; CHECK-NEXT: eor w9, w9, #0x7fffffff
-; CHECK-NEXT: fmov s4, w9
-; CHECK-NEXT: eor w8, w8, #0x7fffffff
-; CHECK-NEXT: orr v0.8b, v2.8b, v0.8b
-; CHECK-NEXT: mov v4.s[1], w8
-; CHECK-NEXT: bif v0.8b, v4.8b, v1.8b
+; CHECK-NEXT: cmlt v3.2s, v1.2s, #0
+; CHECK-NEXT: add v4.2s, v1.2s, v1.2s
+; CHECK-NEXT: cmlt v6.2s, v0.2s, #0
+; CHECK-NEXT: mvn v5.8b, v3.8b
+; CHECK-NEXT: shl v4.2s, v4.2s, #31
+; CHECK-NEXT: cmeq v1.2s, v1.2s, v6.2s
+; CHECK-NEXT: bsl v2.8b, v3.8b, v5.8b
+; CHECK-NEXT: orr v0.8b, v4.8b, v0.8b
+; CHECK-NEXT: bif v0.8b, v2.8b, v1.8b
; CHECK-NEXT: ret
%tmp = call <2 x i32> @llvm.smul.fix.sat.v2i32(<2 x i32> %x, <2 x i32> %y, i32 0)
ret <2 x i32> %tmp
@@ -506,42 +274,21 @@ define <2 x i32> @vec_v2i32(<2 x i32> %x, <2 x i32> %y) {
define <4 x i32> @vec_v4i32(<4 x i32> %x, <4 x i32> %y) {
; CHECK-LABEL: vec_v4i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: smull2 v2.2d, v0.4s, v1.4s
-; CHECK-NEXT: smull v3.2d, v0.2s, v1.2s
-; CHECK-NEXT: mov w8, #2147483647 // =0x7fffffff
+; CHECK-NEXT: smull2 v3.2d, v0.4s, v1.4s
+; CHECK-NEXT: smull v4.2d, v0.2s, v1.2s
+; CHECK-NEXT: movi v2.4s, #63, msl #8
; CHECK-NEXT: mul v0.4s, v0.4s, v1.4s
-; CHECK-NEXT: uzp2 v2.4s, v3.4s, v2.4s
-; CHECK-NEXT: shl v1.4s, v2.4s, #17
-; CHECK-NEXT: mov w9, v2.s[1]
-; CHECK-NEXT: fmov w11, s2
-; CHECK-NEXT: mov w13, v2.s[2]
+; CHECK-NEXT: uzp2 v3.4s, v4.4s, v3.4s
+; CHECK-NEXT: movi v4.4s, #128, lsl #24
+; CHECK-NEXT: shl v1.4s, v3.4s, #17
+; CHECK-NEXT: cmgt v2.4s, v3.4s, v2.4s
; CHECK-NEXT: usra v1.4s, v0.4s, #15
-; CHECK-NEXT: cmp w9, #4, lsl #12 // =16384
-; CHECK-NEXT: mov w10, v1.s[1]
-; CHECK-NEXT: fmov w12, s1
-; CHECK-NEXT: csel w10, w8, w10, ge
-; CHECK-NEXT: cmn w9, #4, lsl #12 // =16384
-; CHECK-NEXT: mov w9, #-2147483648 // =0x80000000
-; CHECK-NEXT: csel w10, w9, w10, lt
-; CHECK-NEXT: cmp w11, #4, lsl #12 // =16384
-; CHECK-NEXT: csel w12, w8, w12, ge
-; CHECK-NEXT: cmn w11, #4, lsl #12 // =16384
-; CHECK-NEXT: mov w11, v1.s[2]
-; CHECK-NEXT: csel w12, w9, w12, lt
-; CHECK-NEXT: cmp w13, #4, lsl #12 // =16384
-; CHECK-NEXT: fmov s0, w12
-; CHECK-NEXT: mov w12, v1.s[3]
-; CHECK-NEXT: csel w11, w8, w11, ge
-; CHECK-NEXT: cmn w13, #4, lsl #12 // =16384
-; CHECK-NEXT: mov v0.s[1], w10
-; CHECK-NEXT: mov w10, v2.s[3]
-; CHECK-NEXT: csel w11, w9, w11, lt
-; CHECK-NEXT: mov v0.s[2], w11
-; CHECK-NEXT: cmp w10, #4, lsl #12 // =16384
-; CHECK-NEXT: csel w8, w8, w12, ge
-; CHECK-NEXT: cmn w10, #4, lsl #12 // =16384
-; CHECK-NEXT: csel w8, w9, w8, lt
-; CHECK-NEXT: mov v0.s[3], w8
+; CHECK-NEXT: bic v0.16b, v1.16b, v2.16b
+; CHECK-NEXT: bic v2.4s, #128, lsl #24
+; CHECK-NEXT: mvni v1.4s, #63, msl #8
+; CHECK-NEXT: orr v0.16b, v2.16b, v0.16b
+; CHECK-NEXT: cmgt v1.4s, v1.4s, v3.4s
+; CHECK-NEXT: bit v0.16b, v4.16b, v1.16b
; CHECK-NEXT: ret
%tmp = call <4 x i32> @llvm.smul.fix.sat.v4i32(<4 x i32> %x, <4 x i32> %y, i32 15)
ret <4 x i32> %tmp
@@ -550,58 +297,33 @@ define <4 x i32> @vec_v4i32(<4 x i32> %x, <4 x i32> %y) {
define <8 x i32> @vec_v8i32(<8 x i32> %x, <8 x i32> %y) {
; CHECK-LABEL: vec_v8i32:
; CHECK: // %bb.0:
+; CHECK-NEXT: smull2 v4.2d, v1.4s, v3.4s
; CHECK-NEXT: smull2 v5.2d, v0.4s, v2.4s
; CHECK-NEXT: smull v6.2d, v0.2s, v2.2s
-; CHECK-NEXT: smull2 v4.2d, v1.4s, v3.4s
; CHECK-NEXT: smull v7.2d, v1.2s, v3.2s
; CHECK-NEXT: mul v0.4s, v0.4s, v2.4s
; CHECK-NEXT: mul v1.4s, v1.4s, v3.4s
+; CHECK-NEXT: movi v16.4s, #128, lsl #24
; CHECK-NEXT: uzp2 v5.4s, v6.4s, v5.4s
; CHECK-NEXT: uzp2 v4.4s, v7.4s, v4.4s
-; CHECK-NEXT: cmlt v16.4s, v0.4s, #0
-; CHECK-NEXT: cmlt v17.4s, v1.4s, #0
-; CHECK-NEXT: fmov w10, s5
-; CHECK-NEXT: mov w8, v5.s[1]
-; CHECK-NEXT: mov w12, v5.s[2]
-; CHECK-NEXT: fmov w11, s4
-; CHECK-NEXT: mov w9, v4.s[1]
-; CHECK-NEXT: mov w13, v4.s[2]
-; CHECK-NEXT: mov w14, v5.s[3]
-; CHECK-NEXT: add v2.4s, v5.4s, v5.4s
-; CHECK-NEXT: add v3.4s, v4.4s, v4.4s
-; CHECK-NEXT: asr w10, w10, #31
-; CHECK-NEXT: cmeq v5.4s, v5.4s, v16.4s
-; CHECK-NEXT: asr w11, w11, #31
-; CHECK-NEXT: asr w8, w8, #31
-; CHECK-NEXT: eor w10, w10, #0x7fffffff
-; CHECK-NEXT: asr w9, w9, #31
-; CHECK-NEXT: shl v2.4s, v2.4s, #31
-; CHECK-NEXT: fmov s6, w10
-; CHECK-NEXT: eor w10, w11, #0x7fffffff
-; CHECK-NEXT: eor w8, w8, #0x7fffffff
-; CHECK-NEXT: fmov s7, w10
-; CHECK-NEXT: eor w9, w9, #0x7fffffff
-; CHECK-NEXT: mov w10, v4.s[3]
+; CHECK-NEXT: cmlt v18.4s, v0.4s, #0
+; CHECK-NEXT: cmlt v20.4s, v1.4s, #0
+; CHECK-NEXT: cmlt v2.4s, v5.4s, #0
+; CHECK-NEXT: add v3.4s, v5.4s, v5.4s
+; CHECK-NEXT: cmeq v5.4s, v5.4s, v18.4s
+; CHECK-NEXT: cmlt v6.4s, v4.4s, #0
+; CHECK-NEXT: add v7.4s, v4.4s, v4.4s
+; CHECK-NEXT: mvn v17.16b, v2.16b
; CHECK-NEXT: shl v3.4s, v3.4s, #31
-; CHECK-NEXT: cmeq v4.4s, v4.4s, v17.4s
-; CHECK-NEXT: orr v0.16b, v2.16b, v0.16b
-; CHECK-NEXT: mov v6.s[1], w8
-; CHECK-NEXT: asr w8, w12, #31
-; CHECK-NEXT: mov v7.s[1], w9
-; CHECK-NEXT: asr w9, w13, #31
-; CHECK-NEXT: eor w8, w8, #0x7fffffff
-; CHECK-NEXT: orr v1.16b, v3.16b, v1.16b
-; CHECK-NEXT: eor w9, w9, #0x7fffffff
-; CHECK-NEXT: mov v6.s[2], w8
-; CHECK-NEXT: asr w8, w14, #31
-; CHECK-NEXT: mov v7.s[2], w9
-; CHECK-NEXT: asr w9, w10, #31
-; CHECK-NEXT: eor w8, w8, #0x7fffffff
-; CHECK-NEXT: eor w9, w9, #0x7fffffff
-; CHECK-NEXT: mov v6.s[3], w8
-; CHECK-NEXT: mov v7.s[3], w9
-; CHECK-NEXT: bif v0.16b, v6.16b, v5.16b
-; CHECK-NEXT: bif v1.16b, v7.16b, v4.16b
+; CHECK-NEXT: mvn v19.16b, v6.16b
+; CHECK-NEXT: shl v7.4s, v7.4s, #31
+; CHECK-NEXT: bif v2.16b, v17.16b, v16.16b
+; CHECK-NEXT: orr v0.16b, v3.16b, v0.16b
+; CHECK-NEXT: cmeq v3.4s, v4.4s, v20.4s
+; CHECK-NEXT: bif v6.16b, v19.16b, v16.16b
+; CHECK-NEXT: orr v1.16b, v7.16b, v1.16b
+; CHECK-NEXT: bif v0.16b, v2.16b, v5.16b
+; CHECK-NEXT: bif v1.16b, v6.16b, v3.16b
; CHECK-NEXT: ret
%tmp = call <8 x i32> @llvm.smul.fix.sat.v8i32(<8 x i32> %x, <8 x i32> %y, i32 0)
ret <8 x i32> %tmp
diff --git a/llvm/test/CodeGen/AArch64/umul_fix_sat.ll b/llvm/test/CodeGen/AArch64/umul_fix_sat.ll
index 9668a866697e7..18a94a3c38711 100644
--- a/llvm/test/CodeGen/AArch64/umul_fix_sat.ll
+++ b/llvm/test/CodeGen/AArch64/umul_fix_sat.ll
@@ -132,51 +132,13 @@ define <8 x i8> @vec_v8i8(<8 x i8> %x, <8 x i8> %y) {
; CHECK-LABEL: vec_v8i8:
; CHECK: // %bb.0:
; CHECK-NEXT: umull v0.8h, v0.8b, v1.8b
-; CHECK-NEXT: shrn v2.8b, v0.8h, #8
+; CHECK-NEXT: movi v2.8b, #3
+; CHECK-NEXT: shrn v1.8b, v0.8h, #8
; CHECK-NEXT: xtn v0.8b, v0.8h
-; CHECK-NEXT: shl v1.8b, v2.8b, #6
-; CHECK-NEXT: umov w8, v2.b[1]
-; CHECK-NEXT: umov w10, v2.b[0]
-; CHECK-NEXT: usra v1.8b, v0.8b, #2
-; CHECK-NEXT: tst w8, #0xfc
-; CHECK-NEXT: umov w9, v1.b[1]
-; CHECK-NEXT: umov w11, v1.b[0]
-; CHECK-NEXT: csinv w8, w9, wzr, eq
-; CHECK-NEXT: tst w10, #0xfc
-; CHECK-NEXT: umov w10, v2.b[2]
-; CHECK-NEXT: csinv w9, w11, wzr, eq
-; CHECK-NEXT: fmov s0, w9
-; CHECK-NEXT: umov w9, v1.b[2]
-; CHECK-NEXT: tst w10, #0xfc
-; CHECK-NEXT: umov w10, v1.b[3]
-; CHECK-NEXT: mov v0.b[1], w8
-; CHECK-NEXT: umov w8, v2.b[3]
-; CHECK-NEXT: csinv w9, w9, wzr, eq
-; CHECK-NEXT: mov v0.b[2], w9
-; CHECK-NEXT: tst w8, #0xfc
-; CHECK-NEXT: umov w8, v2.b[4]
-; CHECK-NEXT: csinv w9, w10, wzr, eq
-; CHECK-NEXT: umov w10, v1.b[4]
-; CHECK-NEXT: mov v0.b[3], w9
-; CHECK-NEXT: tst w8, #0xfc
-; CHECK-NEXT: umov w8, v2.b[5]
-; CHECK-NEXT: csinv w9, w10, wzr, eq
-; CHECK-NEXT: umov w10, v1.b[5]
-; CHECK-NEXT: mov v0.b[4], w9
-; CHECK-NEXT: tst w8, #0xfc
-; CHECK-NEXT: umov w8, v2.b[6]
-; CHECK-NEXT: csinv w9, w10, wzr, eq
-; CHECK-NEXT: umov w10, v1.b[6]
-; CHECK-NEXT: mov v0.b[5], w9
-; CHECK-NEXT: tst w8, #0xfc
-; CHECK-NEXT: umov w8, v2.b[7]
-; CHECK-NEXT: csinv w9, w10, wzr, eq
-; CHECK-NEXT: umov w10, v1.b[7]
-; CHECK-NEXT: mov v0.b[6], w9
-; CHECK-NEXT: tst w8, #0xfc
-; CHECK-NEXT: csinv w8, w10, wzr, eq
-; CHECK-NEXT: mov v0.b[7], w8
-; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT: shl v3.8b, v1.8b, #6
+; CHECK-NEXT: usra v3.8b, v0.8b, #2
+; CHECK-NEXT: cmhi v0.8b, v1.8b, v2.8b
+; CHECK-NEXT: orr v0.8b, v3.8b, v0.8b
; CHECK-NEXT: ret
%tmp = call <8 x i8> @llvm.umul.fix.sat.v8i8(<8 x i8> %x, <8 x i8> %y, i32 2)
ret <8 x i8> %tmp
@@ -185,92 +147,15 @@ define <8 x i8> @vec_v8i8(<8 x i8> %x, <8 x i8> %y) {
define <16 x i8> @vec_v16i8(<16 x i8> %x, <16 x i8> %y) {
; CHECK-LABEL: vec_v16i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: umull2 v2.8h, v0.16b, v1.16b
-; CHECK-NEXT: umull v3.8h, v0.8b, v1.8b
+; CHECK-NEXT: umull2 v3.8h, v0.16b, v1.16b
+; CHECK-NEXT: umull v4.8h, v0.8b, v1.8b
+; CHECK-NEXT: movi v2.16b, #3
; CHECK-NEXT: mul v0.16b, v0.16b, v1.16b
-; CHECK-NEXT: uzp2 v2.16b, v3.16b, v2.16b
-; CHECK-NEXT: shl v1.16b, v2.16b, #6
-; CHECK-NEXT: umov w8, v2.b[1]
-; CHECK-NEXT: umov w10, v2.b[0]
+; CHECK-NEXT: uzp2 v3.16b, v4.16b, v3.16b
+; CHECK-NEXT: shl v1.16b, v3.16b, #6
+; CHECK-NEXT: cmhi v2.16b, v3.16b, v2.16b
; CHECK-NEXT: usra v1.16b, v0.16b, #2
-; CHECK-NEXT: tst w8, #0xfc
-; CHECK-NEXT: umov w9, v1.b[1]
-; CHECK-NEXT: umov w11, v1.b[0]
-; CHECK-NEXT: csinv w8, w9, wzr, eq
-; CHECK-NEXT: tst w10, #0xfc
-; CHECK-NEXT: umov w10, v2.b[2]
-; CHECK-NEXT: csinv w9, w11, wzr, eq
-; CHECK-NEXT: fmov s0, w9
-; CHECK-NEXT: umov w9, v1.b[2]
-; CHECK-NEXT: tst w10, #0xfc
-; CHECK-NEXT: umov w10, v1.b[3]
-; CHECK-NEXT: mov v0.b[1], w8
-; CHECK-NEXT: umov w8, v2.b[3]
-; CHECK-NEXT: csinv w9, w9, wzr, eq
-; CHECK-NEXT: mov v0.b[2], w9
-; CHECK-NEXT: tst w8, #0xfc
-; CHECK-NEXT: umov w8, v2.b[4]
-; CHECK-NEXT: csinv w9, w10, wzr, eq
-; CHECK-NEXT: umov w10, v1.b[4]
-; CHECK-NEXT: mov v0.b[3], w9
-; CHECK-NEXT: tst w8, #0xfc
-; CHECK-NEXT: umov w8, v2.b[5]
-; CHECK-NEXT: csinv w9, w10, wzr, eq
-; CHECK-NEXT: umov w10, v1.b[5]
-; CHECK-NEXT: mov v0.b[4], w9
-; CHECK-NEXT: tst w8, #0xfc
-; CHECK-NEXT: umov w8, v2.b[6]
-; CHECK-NEXT: csinv w9, w10, wzr, eq
-; CHECK-NEXT: umov w10, v1.b[6]
-; CHECK-NEXT: mov v0.b[5], w9
-; CHECK-NEXT: tst w8, #0xfc
-; CHECK-NEXT: umov w8, v2.b[7]
-; CHECK-NEXT: csinv w9, w10, wzr, eq
-; CHECK-NEXT: umov w10, v1.b[7]
-; CHECK-NEXT: mov v0.b[6], w9
-; CHECK-NEXT: tst w8, #0xfc
-; CHECK-NEXT: umov w8, v2.b[8]
-; CHECK-NEXT: csinv w9, w10, wzr, eq
-; CHECK-NEXT: umov w10, v1.b[8]
-; CHECK-NEXT: mov v0.b[7], w9
-; CHECK-NEXT: tst w8, #0xfc
-; CHECK-NEXT: umov w8, v2.b[9]
-; CHECK-NEXT: csinv w9, w10, wzr, eq
-; CHECK-NEXT: umov w10, v1.b[9]
-; CHECK-NEXT: mov v0.b[8], w9
-; CHECK-NEXT: tst w8, #0xfc
-; CHECK-NEXT: umov w8, v2.b[10]
-; CHECK-NEXT: csinv w9, w10, wzr, eq
-; CHECK-NEXT: umov w10, v1.b[10]
-; CHECK-NEXT: mov v0.b[9], w9
-; CHECK-NEXT: tst w8, #0xfc
-; CHECK-NEXT: umov w8, v2.b[11]
-; CHECK-NEXT: csinv w9, w10, wzr, eq
-; CHECK-NEXT: umov w10, v1.b[11]
-; CHECK-NEXT: mov v0.b[10], w9
-; CHECK-NEXT: tst w8, #0xfc
-; CHECK-NEXT: umov w8, v2.b[12]
-; CHECK-NEXT: csinv w9, w10, wzr, eq
-; CHECK-NEXT: umov w10, v1.b[12]
-; CHECK-NEXT: mov v0.b[11], w9
-; CHECK-NEXT: tst w8, #0xfc
-; CHECK-NEXT: umov w8, v2.b[13]
-; CHECK-NEXT: csinv w9, w10, wzr, eq
-; CHECK-NEXT: umov w10, v1.b[13]
-; CHECK-NEXT: mov v0.b[12], w9
-; CHECK-NEXT: tst w8, #0xfc
-; CHECK-NEXT: umov w8, v2.b[14]
-; CHECK-NEXT: csinv w9, w10, wzr, eq
-; CHECK-NEXT: umov w10, v1.b[14]
-; CHECK-NEXT: mov v0.b[13], w9
-; CHECK-NEXT: tst w8, #0xfc
-; CHECK-NEXT: umov w8, v2.b[15]
-; CHECK-NEXT: csinv w9, w10, wzr, eq
-; CHECK-NEXT: umov w10, v1.b[15]
-; CHECK-NEXT: mov v0.b[14], w9
-; CHECK-NEXT: tst w8, #0xfc
-; CHECK-NEXT: csinv w8, w10, wzr, eq
-; CHECK-NEXT: mov v0.b[15], w8
+; CHECK-NEXT: orr v0.16b, v1.16b, v2.16b
; CHECK-NEXT: ret
%tmp = call <16 x i8> @llvm.umul.fix.sat.v16i8(<16 x i8> %x, <16 x i8> %y, i32 2)
ret <16 x i8> %tmp
@@ -280,31 +165,13 @@ define <4 x i16> @vec_v4i16(<4 x i16> %x, <4 x i16> %y) {
; CHECK-LABEL: vec_v4i16:
; CHECK: // %bb.0:
; CHECK-NEXT: umull v0.4s, v0.4h, v1.4h
+; CHECK-NEXT: mvni v2.4h, #252, lsl #8
; CHECK-NEXT: shrn v1.4h, v0.4s, #16
; CHECK-NEXT: xtn v0.4h, v0.4s
-; CHECK-NEXT: shl v2.4h, v1.4h, #6
-; CHECK-NEXT: umov w8, v1.h[1]
-; CHECK-NEXT: umov w10, v1.h[0]
-; CHECK-NEXT: usra v2.4h, v0.4h, #10
-; CHECK-NEXT: tst w8, #0xfc00
-; CHECK-NEXT: umov w9, v2.h[1]
-; CHECK-NEXT: umov w11, v2.h[0]
-; CHECK-NEXT: csinv w8, w9, wzr, eq
-; CHECK-NEXT: tst w10, #0xfc00
-; CHECK-NEXT: umov w10, v1.h[2]
-; CHECK-NEXT: csinv w9, w11, wzr, eq
-; CHECK-NEXT: fmov s0, w9
-; CHECK-NEXT: umov w9, v2.h[2]
-; CHECK-NEXT: tst w10, #0xfc00
-; CHECK-NEXT: umov w10, v2.h[3]
-; CHECK-NEXT: mov v0.h[1], w8
-; CHECK-NEXT: umov w8, v1.h[3]
-; CHECK-NEXT: csinv w9, w9, wzr, eq
-; CHECK-NEXT: mov v0.h[2], w9
-; CHECK-NEXT: tst w8, #0xfc00
-; CHECK-NEXT: csinv w8, w10, wzr, eq
-; CHECK-NEXT: mov v0.h[3], w8
-; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT: shl v3.4h, v1.4h, #6
+; CHECK-NEXT: usra v3.4h, v0.4h, #10
+; CHECK-NEXT: cmhi v0.4h, v1.4h, v2.4h
+; CHECK-NEXT: orr v0.8b, v3.8b, v0.8b
; CHECK-NEXT: ret
%tmp = call <4 x i16> @llvm.umul.fix.sat.v4i16(<4 x i16> %x, <4 x i16> %y, i32 10)
ret <4 x i16> %tmp
@@ -313,52 +180,15 @@ define <4 x i16> @vec_v4i16(<4 x i16> %x, <4 x i16> %y) {
define <8 x i16> @vec_v8i16(<8 x i16> %x, <8 x i16> %y) {
; CHECK-LABEL: vec_v8i16:
; CHECK: // %bb.0:
-; CHECK-NEXT: umull2 v2.4s, v0.8h, v1.8h
-; CHECK-NEXT: umull v3.4s, v0.4h, v1.4h
+; CHECK-NEXT: umull2 v3.4s, v0.8h, v1.8h
+; CHECK-NEXT: umull v4.4s, v0.4h, v1.4h
+; CHECK-NEXT: movi v2.8h, #3
; CHECK-NEXT: mul v0.8h, v0.8h, v1.8h
-; CHECK-NEXT: uzp2 v2.8h, v3.8h, v2.8h
-; CHECK-NEXT: shl v1.8h, v2.8h, #14
-; CHECK-NEXT: umov w8, v2.h[1]
-; CHECK-NEXT: umov w10, v2.h[0]
+; CHECK-NEXT: uzp2 v3.8h, v4.8h, v3.8h
+; CHECK-NEXT: shl v1.8h, v3.8h, #14
+; CHECK-NEXT: cmhi v2.8h, v3.8h, v2.8h
; CHECK-NEXT: usra v1.8h, v0.8h, #2
-; CHECK-NEXT: tst w8, #0xfffc
-; CHECK-NEXT: umov w9, v1.h[1]
-; CHECK-NEXT: umov w11, v1.h[0]
-; CHECK-NEXT: csinv w8, w9, wzr, eq
-; CHECK-NEXT: tst w10, #0xfffc
-; CHECK-NEXT: umov w10, v2.h[2]
-; CHECK-NEXT: csinv w9, w11, wzr, eq
-; CHECK-NEXT: fmov s0, w9
-; CHECK-NEXT: umov w9, v1.h[2]
-; CHECK-NEXT: tst w10, #0xfffc
-; CHECK-NEXT: umov w10, v1.h[3]
-; CHECK-NEXT: mov v0.h[1], w8
-; CHECK-NEXT: umov w8, v2.h[3]
-; CHECK-NEXT: csinv w9, w9, wzr, eq
-; CHECK-NEXT: mov v0.h[2], w9
-; CHECK-NEXT: tst w8, #0xfffc
-; CHECK-NEXT: umov w8, v2.h[4]
-; CHECK-NEXT: csinv w9, w10, wzr, eq
-; CHECK-NEXT: umov w10, v1.h[4]
-; CHECK-NEXT: mov v0.h[3], w9
-; CHECK-NEXT: tst w8, #0xfffc
-; CHECK-NEXT: umov w8, v2.h[5]
-; CHECK-NEXT: csinv w9, w10, wzr, eq
-; CHECK-NEXT: umov w10, v1.h[5]
-; CHECK-NEXT: mov v0.h[4], w9
-; CHECK-NEXT: tst w8, #0xfffc
-; CHECK-NEXT: umov w8, v2.h[6]
-; CHECK-NEXT: csinv w9, w10, wzr, eq
-; CHECK-NEXT: umov w10, v1.h[6]
-; CHECK-NEXT: mov v0.h[5], w9
-; CHECK-NEXT: tst w8, #0xfffc
-; CHECK-NEXT: umov w8, v2.h[7]
-; CHECK-NEXT: csinv w9, w10, wzr, eq
-; CHECK-NEXT: umov w10, v1.h[7]
-; CHECK-NEXT: mov v0.h[6], w9
-; CHECK-NEXT: tst w8, #0xfffc
-; CHECK-NEXT: csinv w8, w10, wzr, eq
-; CHECK-NEXT: mov v0.h[7], w8
+; CHECK-NEXT: orr v0.16b, v1.16b, v2.16b
; CHECK-NEXT: ret
%tmp = call <8 x i16> @llvm.umul.fix.sat.v8i16(<8 x i16> %x, <8 x i16> %y, i32 2)
ret <8 x i16> %tmp
@@ -368,19 +198,14 @@ define <2 x i32> @vec_v2i32(<2 x i32> %x, <2 x i32> %y) {
; CHECK-LABEL: vec_v2i32:
; CHECK: // %bb.0:
; CHECK-NEXT: umull v0.2d, v0.2s, v1.2s
-; CHECK-NEXT: xtn v1.2s, v0.2d
-; CHECK-NEXT: shrn v0.2s, v0.2d, #32
-; CHECK-NEXT: mov w8, v0.s[1]
-; CHECK-NEXT: mov w9, v1.s[1]
-; CHECK-NEXT: fmov w10, s1
-; CHECK-NEXT: cmp w8, #0
-; CHECK-NEXT: fmov w8, s0
-; CHECK-NEXT: csinv w9, w9, wzr, eq
-; CHECK-NEXT: cmp w8, #0
-; CHECK-NEXT: csinv w8, w10, wzr, eq
-; CHECK-NEXT: fmov s0, w8
-; CHECK-NEXT: mov v0.s[1], w9
-; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT: movi v2.2d, #0000000000000000
+; CHECK-NEXT: shrn v1.2s, v0.2d, #32
+; CHECK-NEXT: xtn v0.2s, v0.2d
+; CHECK-NEXT: add v3.2s, v1.2s, v1.2s
+; CHECK-NEXT: cmhi v1.2s, v1.2s, v2.2s
+; CHECK-NEXT: shl v2.2s, v3.2s, #31
+; CHECK-NEXT: orr v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: orr v0.8b, v2.8b, v0.8b
; CHECK-NEXT: ret
%tmp = call <2 x i32> @llvm.umul.fix.sat.v2i32(<2 x i32> %x, <2 x i32> %y, i32 0)
ret <2 x i32> %tmp
@@ -389,32 +214,15 @@ define <2 x i32> @vec_v2i32(<2 x i32> %x, <2 x i32> %y) {
define <4 x i32> @vec_v4i32(<4 x i32> %x, <4 x i32> %y) {
; CHECK-LABEL: vec_v4i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: umull2 v2.2d, v0.4s, v1.4s
-; CHECK-NEXT: umull v3.2d, v0.2s, v1.2s
+; CHECK-NEXT: umull2 v3.2d, v0.4s, v1.4s
+; CHECK-NEXT: umull v4.2d, v0.2s, v1.2s
+; CHECK-NEXT: movi v2.4s, #127, msl #8
; CHECK-NEXT: mul v0.4s, v0.4s, v1.4s
-; CHECK-NEXT: uzp2 v2.4s, v3.4s, v2.4s
-; CHECK-NEXT: shl v1.4s, v2.4s, #17
-; CHECK-NEXT: mov w8, v2.s[1]
+; CHECK-NEXT: uzp2 v3.4s, v4.4s, v3.4s
+; CHECK-NEXT: shl v1.4s, v3.4s, #17
+; CHECK-NEXT: cmhi v2.4s, v3.4s, v2.4s
; CHECK-NEXT: usra v1.4s, v0.4s, #15
-; CHECK-NEXT: cmp wzr, w8, lsr #15
-; CHECK-NEXT: fmov w8, s2
-; CHECK-NEXT: mov w9, v1.s[1]
-; CHECK-NEXT: fmov w10, s1
-; CHECK-NEXT: mov w11, v1.s[2]
-; CHECK-NEXT: csinv w9, w9, wzr, eq
-; CHECK-NEXT: cmp wzr, w8, lsr #15
-; CHECK-NEXT: mov w8, v2.s[2]
-; CHECK-NEXT: csinv w10, w10, wzr, eq
-; CHECK-NEXT: fmov s0, w10
-; CHECK-NEXT: cmp wzr, w8, lsr #15
-; CHECK-NEXT: mov w8, v1.s[3]
-; CHECK-NEXT: mov v0.s[1], w9
-; CHECK-NEXT: mov w9, v2.s[3]
-; CHECK-NEXT: csinv w10, w11, wzr, eq
-; CHECK-NEXT: mov v0.s[2], w10
-; CHECK-NEXT: cmp wzr, w9, lsr #15
-; CHECK-NEXT: csinv w8, w8, wzr, eq
-; CHECK-NEXT: mov v0.s[3], w8
+; CHECK-NEXT: orr v0.16b, v1.16b, v2.16b
; CHECK-NEXT: ret
%tmp = call <4 x i32> @llvm.umul.fix.sat.v4i32(<4 x i32> %x, <4 x i32> %y, i32 15)
ret <4 x i32> %tmp
@@ -425,58 +233,23 @@ define <8 x i32> @vec_v8i32(<8 x i32> %x, <8 x i32> %y) {
; CHECK: // %bb.0:
; CHECK-NEXT: umull2 v4.2d, v0.4s, v2.4s
; CHECK-NEXT: umull v5.2d, v0.2s, v2.2s
-; CHECK-NEXT: umull v6.2d, v1.2s, v3.2s
+; CHECK-NEXT: umull2 v6.2d, v1.4s, v3.4s
+; CHECK-NEXT: umull v7.2d, v1.2s, v3.2s
; CHECK-NEXT: mul v0.4s, v0.4s, v2.4s
-; CHECK-NEXT: uzp2 v4.4s, v5.4s, v4.4s
-; CHECK-NEXT: umull2 v5.2d, v1.4s, v3.4s
; CHECK-NEXT: mul v1.4s, v1.4s, v3.4s
-; CHECK-NEXT: add v7.4s, v4.4s, v4.4s
-; CHECK-NEXT: uzp2 v5.4s, v6.4s, v5.4s
-; CHECK-NEXT: mov w8, v4.s[1]
-; CHECK-NEXT: mov w10, v4.s[2]
-; CHECK-NEXT: fmov w12, s4
-; CHECK-NEXT: mov w11, v4.s[3]
-; CHECK-NEXT: shl v2.4s, v7.4s, #31
-; CHECK-NEXT: add v6.4s, v5.4s, v5.4s
-; CHECK-NEXT: cmp w8, #0
-; CHECK-NEXT: mov w13, v5.s[1]
+; CHECK-NEXT: uzp2 v4.4s, v5.4s, v4.4s
+; CHECK-NEXT: uzp2 v5.4s, v7.4s, v6.4s
+; CHECK-NEXT: movi v6.2d, #0000000000000000
+; CHECK-NEXT: add v2.4s, v4.4s, v4.4s
+; CHECK-NEXT: cmhi v3.4s, v4.4s, v6.4s
+; CHECK-NEXT: cmhi v4.4s, v5.4s, v6.4s
+; CHECK-NEXT: add v5.4s, v5.4s, v5.4s
+; CHECK-NEXT: shl v2.4s, v2.4s, #31
+; CHECK-NEXT: orr v0.16b, v0.16b, v3.16b
+; CHECK-NEXT: shl v3.4s, v5.4s, #31
+; CHECK-NEXT: orr v1.16b, v1.16b, v4.16b
; CHECK-NEXT: orr v0.16b, v2.16b, v0.16b
-; CHECK-NEXT: shl v2.4s, v6.4s, #31
-; CHECK-NEXT: mov w9, v0.s[1]
-; CHECK-NEXT: mov w8, v0.s[2]
-; CHECK-NEXT: fmov w15, s0
-; CHECK-NEXT: orr v2.16b, v2.16b, v1.16b
-; CHECK-NEXT: mov w14, v0.s[3]
-; CHECK-NEXT: csinv w9, w9, wzr, eq
-; CHECK-NEXT: cmp w12, #0
-; CHECK-NEXT: mov w12, v2.s[1]
-; CHECK-NEXT: csinv w15, w15, wzr, eq
-; CHECK-NEXT: cmp w10, #0
-; CHECK-NEXT: csinv w8, w8, wzr, eq
-; CHECK-NEXT: cmp w11, #0
-; CHECK-NEXT: fmov w11, s5
-; CHECK-NEXT: csinv w10, w14, wzr, eq
-; CHECK-NEXT: cmp w13, #0
-; CHECK-NEXT: fmov w13, s2
-; CHECK-NEXT: csinv w12, w12, wzr, eq
-; CHECK-NEXT: fmov s0, w15
-; CHECK-NEXT: mov w14, v2.s[2]
-; CHECK-NEXT: cmp w11, #0
-; CHECK-NEXT: mov w11, v5.s[2]
-; CHECK-NEXT: csinv w13, w13, wzr, eq
-; CHECK-NEXT: fmov s1, w13
-; CHECK-NEXT: mov v0.s[1], w9
-; CHECK-NEXT: mov w9, v5.s[3]
-; CHECK-NEXT: cmp w11, #0
-; CHECK-NEXT: mov w11, v2.s[3]
-; CHECK-NEXT: mov v1.s[1], w12
-; CHECK-NEXT: csinv w12, w14, wzr, eq
-; CHECK-NEXT: mov v0.s[2], w8
-; CHECK-NEXT: cmp w9, #0
-; CHECK-NEXT: csinv w8, w11, wzr, eq
-; CHECK-NEXT: mov v1.s[2], w12
-; CHECK-NEXT: mov v0.s[3], w10
-; CHECK-NEXT: mov v1.s[3], w8
+; CHECK-NEXT: orr v1.16b, v3.16b, v1.16b
; CHECK-NEXT: ret
%tmp = call <8 x i32> @llvm.umul.fix.sat.v8i32(<8 x i32> %x, <8 x i32> %y, i32 0)
ret <8 x i32> %tmp
diff --git a/llvm/test/CodeGen/X86/smul_fix_sat.ll b/llvm/test/CodeGen/X86/smul_fix_sat.ll
index 2a863e32cab63..69d5769709372 100644
--- a/llvm/test/CodeGen/X86/smul_fix_sat.ll
+++ b/llvm/test/CodeGen/X86/smul_fix_sat.ll
@@ -193,66 +193,39 @@ define i4 @func3(i4 %x, i4 %y) nounwind {
define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
; X64-LABEL: vec:
; X64: # %bb.0:
+; X64-NEXT: movdqa %xmm0, %xmm2
; X64-NEXT: pxor %xmm3, %xmm3
-; X64-NEXT: pxor %xmm2, %xmm2
-; X64-NEXT: pcmpgtd %xmm1, %xmm2
-; X64-NEXT: pand %xmm0, %xmm2
-; X64-NEXT: pcmpgtd %xmm0, %xmm3
+; X64-NEXT: pxor %xmm0, %xmm0
+; X64-NEXT: pcmpgtd %xmm1, %xmm0
+; X64-NEXT: pand %xmm2, %xmm0
+; X64-NEXT: pcmpgtd %xmm2, %xmm3
; X64-NEXT: pand %xmm1, %xmm3
-; X64-NEXT: paddd %xmm2, %xmm3
-; X64-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
-; X64-NEXT: pmuludq %xmm1, %xmm0
-; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,3,2,3]
-; X64-NEXT: pshufd {{.*#+}} xmm5 = xmm1[1,1,3,3]
-; X64-NEXT: pmuludq %xmm4, %xmm5
-; X64-NEXT: pshufd {{.*#+}} xmm1 = xmm5[1,3,2,3]
+; X64-NEXT: paddd %xmm0, %xmm3
+; X64-NEXT: pshufd {{.*#+}} xmm4 = xmm2[1,1,3,3]
+; X64-NEXT: pmuludq %xmm1, %xmm2
+; X64-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,3,2,3]
+; X64-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
+; X64-NEXT: pmuludq %xmm4, %xmm1
+; X64-NEXT: pshufd {{.*#+}} xmm4 = xmm1[1,3,2,3]
+; X64-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
+; X64-NEXT: psubd %xmm3, %xmm0
+; X64-NEXT: movdqa %xmm0, %xmm3
+; X64-NEXT: pslld $30, %xmm3
+; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; X64-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
; X64-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; X64-NEXT: psubd %xmm3, %xmm2
-; X64-NEXT: pshufd {{.*#+}} xmm1 = xmm0[0,2,2,3]
-; X64-NEXT: pshufd {{.*#+}} xmm0 = xmm5[0,2,2,3]
-; X64-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; X64-NEXT: pshufd {{.*#+}} xmm0 = xmm2[3,3,3,3]
-; X64-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
-; X64-NEXT: movd %xmm2, %eax
-; X64-NEXT: pshufd {{.*#+}} xmm4 = xmm2[1,1,1,1]
-; X64-NEXT: pslld $30, %xmm2
-; X64-NEXT: psrld $2, %xmm1
-; X64-NEXT: por %xmm2, %xmm1
-; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; X64-NEXT: movd %xmm2, %ecx
-; X64-NEXT: movd %xmm0, %edx
-; X64-NEXT: cmpl $2, %edx
-; X64-NEXT: movl $2147483647, %esi # imm = 0x7FFFFFFF
-; X64-NEXT: cmovgel %esi, %ecx
-; X64-NEXT: cmpl $-2, %edx
-; X64-NEXT: movl $-2147483648, %edx # imm = 0x80000000
-; X64-NEXT: cmovll %edx, %ecx
-; X64-NEXT: movd %ecx, %xmm0
-; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
-; X64-NEXT: movd %xmm2, %ecx
-; X64-NEXT: movd %xmm3, %edi
-; X64-NEXT: cmpl $2, %edi
-; X64-NEXT: cmovgel %esi, %ecx
-; X64-NEXT: cmpl $-2, %edi
-; X64-NEXT: cmovll %edx, %ecx
-; X64-NEXT: movd %ecx, %xmm2
-; X64-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; X64-NEXT: movd %xmm1, %ecx
-; X64-NEXT: cmpl $2, %eax
-; X64-NEXT: cmovgel %esi, %ecx
-; X64-NEXT: cmpl $-2, %eax
-; X64-NEXT: cmovll %edx, %ecx
-; X64-NEXT: movd %ecx, %xmm0
-; X64-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; X64-NEXT: movd %xmm1, %eax
-; X64-NEXT: movd %xmm4, %ecx
-; X64-NEXT: cmpl $2, %ecx
-; X64-NEXT: cmovgel %esi, %eax
-; X64-NEXT: cmpl $-2, %ecx
-; X64-NEXT: cmovll %edx, %eax
-; X64-NEXT: movd %eax, %xmm1
-; X64-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X64-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; X64-NEXT: psrld $2, %xmm2
+; X64-NEXT: por %xmm3, %xmm2
+; X64-NEXT: movdqa %xmm0, %xmm1
+; X64-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; X64-NEXT: movdqa %xmm1, %xmm3
+; X64-NEXT: pandn %xmm2, %xmm3
+; X64-NEXT: psrld $1, %xmm1
+; X64-NEXT: por %xmm3, %xmm1
+; X64-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; X64-NEXT: pand %xmm0, %xmm1
+; X64-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; X64-NEXT: por %xmm1, %xmm0
; X64-NEXT: retq
;
; X86-LABEL: vec:
@@ -485,27 +458,31 @@ define <4 x i32> @vec2(<4 x i32> %x, <4 x i32> %y) nounwind {
; X64-NEXT: pxor %xmm3, %xmm3
; X64-NEXT: pcmpgtd %xmm1, %xmm3
; X64-NEXT: pand %xmm0, %xmm3
-; X64-NEXT: pcmpgtd %xmm0, %xmm2
-; X64-NEXT: pand %xmm1, %xmm2
-; X64-NEXT: paddd %xmm3, %xmm2
+; X64-NEXT: pxor %xmm4, %xmm4
+; X64-NEXT: pcmpgtd %xmm0, %xmm4
+; X64-NEXT: pand %xmm1, %xmm4
+; X64-NEXT: paddd %xmm3, %xmm4
; X64-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
; X64-NEXT: pmuludq %xmm1, %xmm0
-; X64-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,3,2,3]
+; X64-NEXT: pshufd {{.*#+}} xmm5 = xmm0[1,3,2,3]
; X64-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
; X64-NEXT: pmuludq %xmm3, %xmm1
; X64-NEXT: pshufd {{.*#+}} xmm3 = xmm1[1,3,2,3]
-; X64-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
-; X64-NEXT: psubd %xmm2, %xmm4
-; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,2,2,3]
+; X64-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
+; X64-NEXT: psubd %xmm4, %xmm5
+; X64-NEXT: movdqa {{.*#+}} xmm3 = [2147483648,2147483648,2147483648,2147483648]
+; X64-NEXT: pand %xmm5, %xmm3
+; X64-NEXT: pcmpgtd %xmm5, %xmm2
+; X64-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
+; X64-NEXT: por %xmm3, %xmm2
+; X64-NEXT: pshufd {{.*#+}} xmm3 = xmm0[0,2,2,3]
; X64-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
-; X64-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; X64-NEXT: psrad $31, %xmm2
-; X64-NEXT: pcmpeqd %xmm4, %xmm2
-; X64-NEXT: psrad $31, %xmm4
-; X64-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4
-; X64-NEXT: movdqa %xmm2, %xmm0
-; X64-NEXT: pandn %xmm4, %xmm0
-; X64-NEXT: por %xmm2, %xmm0
+; X64-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]
+; X64-NEXT: psrad $31, %xmm3
+; X64-NEXT: pcmpeqd %xmm5, %xmm3
+; X64-NEXT: movdqa %xmm3, %xmm0
+; X64-NEXT: pandn %xmm2, %xmm0
+; X64-NEXT: por %xmm3, %xmm0
; X64-NEXT: retq
;
; X86-LABEL: vec2:
diff --git a/llvm/test/CodeGen/X86/umul_fix_sat.ll b/llvm/test/CodeGen/X86/umul_fix_sat.ll
index e08919c81b53a..9aff6a392d65d 100644
--- a/llvm/test/CodeGen/X86/umul_fix_sat.ll
+++ b/llvm/test/CodeGen/X86/umul_fix_sat.ll
@@ -138,49 +138,23 @@ define i4 @func3(i4 %x, i4 %y) nounwind {
define <4 x i32> @vec(<4 x i32> %x, <4 x i32> %y) nounwind {
; X64-LABEL: vec:
; X64: # %bb.0:
-; X64-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
; X64-NEXT: pmuludq %xmm1, %xmm0
-; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,2,2,3]
+; X64-NEXT: pshufd {{.*#+}} xmm3 = xmm0[0,2,2,3]
; X64-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; X64-NEXT: pmuludq %xmm3, %xmm1
-; X64-NEXT: pshufd {{.*#+}} xmm3 = xmm1[0,2,2,3]
-; X64-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; X64-NEXT: psrld $2, %xmm2
-; X64-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,3,2,3]
-; X64-NEXT: pshufd {{.*#+}} xmm4 = xmm1[1,3,2,3]
-; X64-NEXT: movd %xmm3, %eax
-; X64-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
-; X64-NEXT: pslld $30, %xmm3
-; X64-NEXT: por %xmm2, %xmm3
-; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm3[3,3,3,3]
-; X64-NEXT: movd %xmm2, %ecx
-; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; X64-NEXT: movd %xmm2, %edx
-; X64-NEXT: cmpl $4, %edx
-; X64-NEXT: movl $-1, %edx
-; X64-NEXT: cmovael %edx, %ecx
-; X64-NEXT: movd %ecx, %xmm2
-; X64-NEXT: pshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
-; X64-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; X64-NEXT: movd %xmm0, %ecx
-; X64-NEXT: cmpl $4, %ecx
-; X64-NEXT: movd %xmm4, %ecx
-; X64-NEXT: cmovael %edx, %ecx
-; X64-NEXT: movd %ecx, %xmm4
-; X64-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
-; X64-NEXT: movd %xmm3, %ecx
-; X64-NEXT: cmpl $4, %eax
-; X64-NEXT: cmovael %edx, %ecx
-; X64-NEXT: movd %ecx, %xmm0
-; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm3[1,1,1,1]
-; X64-NEXT: movd %xmm2, %eax
-; X64-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; X64-NEXT: movd %xmm1, %ecx
-; X64-NEXT: cmpl $4, %ecx
-; X64-NEXT: cmovael %edx, %eax
-; X64-NEXT: movd %eax, %xmm1
+; X64-NEXT: pmuludq %xmm2, %xmm1
+; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm1[0,2,2,3]
+; X64-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; X64-NEXT: psrld $2, %xmm3
+; X64-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,3,2,3]
+; X64-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,3,2,3]
; X64-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X64-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; X64-NEXT: movdqa %xmm0, %xmm1
+; X64-NEXT: pslld $30, %xmm1
+; X64-NEXT: por %xmm3, %xmm1
+; X64-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; X64-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; X64-NEXT: por %xmm1, %xmm0
; X64-NEXT: retq
;
; X86-LABEL: vec:
diff --git a/llvm/test/CodeGen/X86/vector-mulfix-legalize.ll b/llvm/test/CodeGen/X86/vector-mulfix-legalize.ll
index 64a098a282779..32b5b8fa404d2 100644
--- a/llvm/test/CodeGen/X86/vector-mulfix-legalize.ll
+++ b/llvm/test/CodeGen/X86/vector-mulfix-legalize.ll
@@ -43,91 +43,24 @@ define <4 x i16> @umulfix(<4 x i16> %a) {
define <4 x i16> @smulfixsat(<4 x i16> %a) {
; CHECK-LABEL: smulfixsat:
; CHECK: # %bb.0:
-; CHECK-NEXT: pushq %rbx
-; CHECK-NEXT: .cfi_def_cfa_offset 16
-; CHECK-NEXT: .cfi_offset %rbx, -16
-; CHECK-NEXT: movq {{.*#+}} xmm2 = [1,2,3,4,0,0,0,0]
+; CHECK-NEXT: movq {{.*#+}} xmm1 = [1,2,3,4,0,0,0,0]
+; CHECK-NEXT: movdqa %xmm0, %xmm2
+; CHECK-NEXT: pmullw %xmm1, %xmm2
+; CHECK-NEXT: psrlw $15, %xmm2
+; CHECK-NEXT: pmulhw %xmm1, %xmm0
; CHECK-NEXT: movdqa %xmm0, %xmm1
-; CHECK-NEXT: pmullw %xmm2, %xmm1
-; CHECK-NEXT: psrlw $15, %xmm1
-; CHECK-NEXT: pmulhw %xmm2, %xmm0
-; CHECK-NEXT: pextrw $7, %xmm0, %eax
-; CHECK-NEXT: pextrw $6, %xmm0, %ecx
-; CHECK-NEXT: pextrw $5, %xmm0, %edx
-; CHECK-NEXT: pextrw $4, %xmm0, %esi
-; CHECK-NEXT: pextrw $3, %xmm0, %edi
-; CHECK-NEXT: pextrw $2, %xmm0, %r8d
-; CHECK-NEXT: pextrw $1, %xmm0, %r9d
-; CHECK-NEXT: movd %xmm0, %r10d
-; CHECK-NEXT: paddw %xmm0, %xmm0
-; CHECK-NEXT: por %xmm1, %xmm0
-; CHECK-NEXT: pextrw $7, %xmm0, %ebx
-; CHECK-NEXT: movswl %ax, %r11d
-; CHECK-NEXT: cmpl $16384, %r11d # imm = 0x4000
-; CHECK-NEXT: movl $32767, %eax # imm = 0x7FFF
-; CHECK-NEXT: cmovgel %eax, %ebx
-; CHECK-NEXT: cmpl $-16384, %r11d # imm = 0xC000
-; CHECK-NEXT: movl $32768, %r11d # imm = 0x8000
-; CHECK-NEXT: cmovll %r11d, %ebx
-; CHECK-NEXT: movd %ebx, %xmm3
-; CHECK-NEXT: pextrw $6, %xmm0, %ebx
-; CHECK-NEXT: movswl %cx, %ecx
-; CHECK-NEXT: cmpl $16384, %ecx # imm = 0x4000
-; CHECK-NEXT: cmovgel %eax, %ebx
-; CHECK-NEXT: cmpl $-16384, %ecx # imm = 0xC000
-; CHECK-NEXT: cmovll %r11d, %ebx
-; CHECK-NEXT: movd %ebx, %xmm2
-; CHECK-NEXT: pextrw $5, %xmm0, %ecx
-; CHECK-NEXT: movswl %dx, %edx
-; CHECK-NEXT: cmpl $16384, %edx # imm = 0x4000
-; CHECK-NEXT: cmovgel %eax, %ecx
-; CHECK-NEXT: cmpl $-16384, %edx # imm = 0xC000
-; CHECK-NEXT: cmovll %r11d, %ecx
-; CHECK-NEXT: movd %ecx, %xmm4
-; CHECK-NEXT: pextrw $4, %xmm0, %ecx
-; CHECK-NEXT: movswl %si, %edx
-; CHECK-NEXT: cmpl $16384, %edx # imm = 0x4000
-; CHECK-NEXT: cmovgel %eax, %ecx
-; CHECK-NEXT: cmpl $-16384, %edx # imm = 0xC000
-; CHECK-NEXT: cmovll %r11d, %ecx
-; CHECK-NEXT: movd %ecx, %xmm1
-; CHECK-NEXT: pextrw $3, %xmm0, %ecx
-; CHECK-NEXT: movswl %di, %edx
-; CHECK-NEXT: cmpl $16384, %edx # imm = 0x4000
-; CHECK-NEXT: cmovgel %eax, %ecx
-; CHECK-NEXT: cmpl $-16384, %edx # imm = 0xC000
-; CHECK-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; CHECK-NEXT: cmovll %r11d, %ecx
-; CHECK-NEXT: movd %ecx, %xmm3
-; CHECK-NEXT: pextrw $2, %xmm0, %ecx
-; CHECK-NEXT: movswl %r8w, %edx
-; CHECK-NEXT: cmpl $16384, %edx # imm = 0x4000
-; CHECK-NEXT: cmovgel %eax, %ecx
-; CHECK-NEXT: cmpl $-16384, %edx # imm = 0xC000
-; CHECK-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
-; CHECK-NEXT: cmovll %r11d, %ecx
-; CHECK-NEXT: movd %ecx, %xmm4
-; CHECK-NEXT: pextrw $1, %xmm0, %ecx
-; CHECK-NEXT: movswl %r9w, %edx
-; CHECK-NEXT: cmpl $16384, %edx # imm = 0x4000
-; CHECK-NEXT: cmovgel %eax, %ecx
-; CHECK-NEXT: cmpl $-16384, %edx # imm = 0xC000
-; CHECK-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; CHECK-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; CHECK-NEXT: cmovll %r11d, %ecx
-; CHECK-NEXT: movd %ecx, %xmm2
-; CHECK-NEXT: movd %xmm0, %ecx
-; CHECK-NEXT: movswl %r10w, %edx
-; CHECK-NEXT: cmpl $16384, %edx # imm = 0x4000
-; CHECK-NEXT: cmovgel %eax, %ecx
-; CHECK-NEXT: cmpl $-16384, %edx # imm = 0xC000
-; CHECK-NEXT: cmovll %r11d, %ecx
-; CHECK-NEXT: movd %ecx, %xmm0
-; CHECK-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
-; CHECK-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
-; CHECK-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; CHECK-NEXT: popq %rbx
-; CHECK-NEXT: .cfi_def_cfa_offset 8
+; CHECK-NEXT: paddw %xmm1, %xmm1
+; CHECK-NEXT: por %xmm2, %xmm1
+; CHECK-NEXT: movdqa %xmm0, %xmm2
+; CHECK-NEXT: pcmpgtw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
+; CHECK-NEXT: movdqa %xmm2, %xmm3
+; CHECK-NEXT: pandn %xmm1, %xmm3
+; CHECK-NEXT: psrlw $1, %xmm2
+; CHECK-NEXT: por %xmm3, %xmm2
+; CHECK-NEXT: pcmpgtw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; CHECK-NEXT: pand %xmm0, %xmm2
+; CHECK-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; CHECK-NEXT: por %xmm2, %xmm0
; CHECK-NEXT: retq
%t = call <4 x i16> @llvm.smul.fix.sat.v4i16(<4 x i16> <i16 1, i16 2, i16 3, i16 4>, <4 x i16> %a, i32 15)
ret <4 x i16> %t
@@ -137,61 +70,19 @@ define <4 x i16> @smulfixsat(<4 x i16> %a) {
define <4 x i16> @umulfixsat(<4 x i16> %a) {
; CHECK-LABEL: umulfixsat:
; CHECK: # %bb.0:
-; CHECK-NEXT: movq {{.*#+}} xmm2 = [1,2,3,4,0,0,0,0]
-; CHECK-NEXT: movdqa %xmm0, %xmm1
-; CHECK-NEXT: pmullw %xmm2, %xmm1
-; CHECK-NEXT: psrlw $15, %xmm1
-; CHECK-NEXT: pmulhuw %xmm2, %xmm0
-; CHECK-NEXT: pextrw $7, %xmm0, %eax
-; CHECK-NEXT: pextrw $6, %xmm0, %ecx
-; CHECK-NEXT: pextrw $5, %xmm0, %edx
-; CHECK-NEXT: pextrw $4, %xmm0, %esi
-; CHECK-NEXT: pextrw $3, %xmm0, %edi
-; CHECK-NEXT: pextrw $2, %xmm0, %r8d
-; CHECK-NEXT: pextrw $1, %xmm0, %r9d
-; CHECK-NEXT: paddw %xmm0, %xmm0
-; CHECK-NEXT: por %xmm1, %xmm0
-; CHECK-NEXT: pextrw $7, %xmm0, %r10d
-; CHECK-NEXT: testw %ax, %ax
-; CHECK-NEXT: movl $65535, %eax # imm = 0xFFFF
-; CHECK-NEXT: cmovsl %eax, %r10d
-; CHECK-NEXT: movd %r10d, %xmm1
-; CHECK-NEXT: testw %cx, %cx
-; CHECK-NEXT: pextrw $6, %xmm0, %ecx
-; CHECK-NEXT: cmovsl %eax, %ecx
-; CHECK-NEXT: movd %ecx, %xmm2
-; CHECK-NEXT: testw %dx, %dx
-; CHECK-NEXT: pextrw $5, %xmm0, %ecx
-; CHECK-NEXT: cmovsl %eax, %ecx
-; CHECK-NEXT: movd %ecx, %xmm3
-; CHECK-NEXT: testw %si, %si
-; CHECK-NEXT: pextrw $4, %xmm0, %ecx
-; CHECK-NEXT: cmovsl %eax, %ecx
-; CHECK-NEXT: movd %ecx, %xmm4
-; CHECK-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; CHECK-NEXT: testw %di, %di
-; CHECK-NEXT: pextrw $3, %xmm0, %ecx
-; CHECK-NEXT: cmovsl %eax, %ecx
-; CHECK-NEXT: movd %ecx, %xmm1
-; CHECK-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; CHECK-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
-; CHECK-NEXT: testw %r8w, %r8w
-; CHECK-NEXT: pextrw $2, %xmm0, %ecx
-; CHECK-NEXT: cmovsl %eax, %ecx
-; CHECK-NEXT: movd %ecx, %xmm2
-; CHECK-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
-; CHECK-NEXT: pextrw $1, %xmm0, %ecx
-; CHECK-NEXT: testw %r9w, %r9w
-; CHECK-NEXT: cmovsl %eax, %ecx
-; CHECK-NEXT: movd %ecx, %xmm1
-; CHECK-NEXT: movd %xmm0, %ecx
-; CHECK-NEXT: xorl %edx, %edx
-; CHECK-NEXT: testw %dx, %dx
-; CHECK-NEXT: cmovsl %eax, %ecx
-; CHECK-NEXT: movd %ecx, %xmm0
-; CHECK-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; CHECK-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; CHECK-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm4[0]
+; CHECK-NEXT: movq {{.*#+}} xmm1 = [1,2,3,4,0,0,0,0]
+; CHECK-NEXT: movdqa %xmm0, %xmm2
+; CHECK-NEXT: pmullw %xmm1, %xmm2
+; CHECK-NEXT: psrlw $15, %xmm2
+; CHECK-NEXT: pmulhuw %xmm1, %xmm0
+; CHECK-NEXT: movdqa {{.*#+}} xmm1 = [32768,32768,32768,32768,32768,32768,32768,32768]
+; CHECK-NEXT: psubusw %xmm0, %xmm1
+; CHECK-NEXT: movdqa %xmm0, %xmm3
+; CHECK-NEXT: paddw %xmm3, %xmm3
+; CHECK-NEXT: por %xmm2, %xmm3
+; CHECK-NEXT: pxor %xmm0, %xmm0
+; CHECK-NEXT: pcmpeqw %xmm1, %xmm0
+; CHECK-NEXT: por %xmm3, %xmm0
; CHECK-NEXT: retq
%t = call <4 x i16> @llvm.umul.fix.sat.v4i16(<4 x i16> <i16 1, i16 2, i16 3, i16 4>, <4 x i16> %a, i32 15)
ret <4 x i16> %t
More information about the llvm-commits
mailing list