[llvm] [AArch64][GlobalISel] Add tablegen patterns for smull+sqshrn -> sqdmulh. (PR #217318)
David Green via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 19 05:17:19 PDT 2026
https://github.com/davemgreen created https://github.com/llvm/llvm-project/pull/217318
We do this in parts in globalisel, recognizing the smull and trunc_sat, which when combined with the correct shift can fold into a single sqdmulh.
>From 1bf5596558b24fa9ec6a164a859fc8d66b805bf6 Mon Sep 17 00:00:00 2001
From: David Green <david.green at arm.com>
Date: Wed, 19 Aug 2026 10:32:49 +0100
Subject: [PATCH 1/6] [AArch64] Add tests for trunc nsw smin combine. NFC
The qmov.ll test was cleaned up a little, and the saturating-vec-smull.ll was
updated with flags expected to be generated from the -O2 pipeline.
---
llvm/test/CodeGen/AArch64/qmovn.ll | 100 +++++++++++++++---
.../CodeGen/AArch64/saturating-vec-smull.ll | 52 ++++-----
2 files changed, 113 insertions(+), 39 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/qmovn.ll b/llvm/test/CodeGen/AArch64/qmovn.ll
index 5d534e09cdbd3..2fb4e2e467c7d 100644
--- a/llvm/test/CodeGen/AArch64/qmovn.ll
+++ b/llvm/test/CodeGen/AArch64/qmovn.ll
@@ -206,7 +206,7 @@ entry:
ret <2 x i32> %t
}
-; Test the (concat_vectors (X), (trunc(smin(smax(Y, -2^n), 2^n-1))) pattern.
+; Test the (concat_vectors (X), (trunc(smin(smax(Y, -2^n), 2^n-1)))) pattern.
define <16 x i8> @signed_minmax_v8i16_to_v16i8(<8 x i8> %x, <8 x i16> %y) {
; CHECK-LABEL: signed_minmax_v8i16_to_v16i8:
@@ -250,7 +250,7 @@ entry:
ret <4 x i32> %shuffle
}
-; Test the (concat_vectors (X), (trunc(smax(smin(Y, 2^n-1), -2^n))) pattern.
+; Test the (concat_vectors (X), (trunc(smax(smin(Y, 2^n-1), -2^n)))) pattern.
define <16 x i8> @signed_maxmin_v8i16_to_v16i8(<8 x i8> %x, <8 x i16> %y) {
; CHECK-LABEL: signed_maxmin_v8i16_to_v16i8:
@@ -335,7 +335,7 @@ entry:
ret <4 x i32> %shuffle
}
-; Test the (concat_vectors (X), (trunc(umin(smax(Y, 0), 2^n))))) pattern.
+; Test the (concat_vectors (X), (trunc(umin(smax(Y, 0), 2^n)))) pattern.
define <16 x i8> @us_maxmin_v8i16_to_v16i8(<8 x i8> %x, <8 x i16> %y) {
; CHECK-LABEL: us_maxmin_v8i16_to_v16i8:
@@ -379,7 +379,7 @@ entry:
ret <4 x i32> %shuffle
}
-; Test the (concat_vectors (X), (trunc(smin(smax(Y, 0), 2^n))))) pattern.
+; Test the (concat_vectors (X), (trunc(smin(smax(Y, 0), 2^n)))) pattern.
define <16 x i8> @sminsmax_range_unsigned_i16_to_i8(<8 x i8> %x, <8 x i16> %y) {
; CHECK-LABEL: sminsmax_range_unsigned_i16_to_i8:
@@ -425,7 +425,7 @@ entry:
-; Test the (concat_vectors (X), (trunc(smin(smax(Y, -2^n), 2^n-1))) pattern.
+; Test the larger (trunc(smin(smax(Y, -2^n), 2^n-1))) pattern.
define <16 x i8> @signed_minmax_v16i16_to_v16i8(<16 x i16> %y) {
; CHECK-LABEL: signed_minmax_v16i16_to_v16i8:
@@ -466,7 +466,7 @@ entry:
ret <4 x i32> %trunc
}
-; Test the (concat_vectors (X), (trunc(smax(smin(Y, 2^n-1), -2^n))) pattern.
+; Test the larger (trunc(smax(smin(Y, 2^n-1), -2^n))) pattern.
define <16 x i8> @signed_maxmin_v16i16_to_v16i8(<16 x i16> %y) {
; CHECK-LABEL: signed_maxmin_v16i16_to_v16i8:
@@ -507,7 +507,7 @@ entry:
ret <4 x i32> %trunc
}
-; Test the (concat_vectors (X), (trunc(umin(Y, 2^n)))) pattern.
+; Test the larger (trunc(umin(Y, 2^n))) pattern.
define <16 x i8> @unsigned_v16i16_to_v16i8(<16 x i16> %y) {
; CHECK-LABEL: unsigned_v16i16_to_v16i8:
@@ -545,7 +545,7 @@ entry:
ret <4 x i32> %trunc
}
-; Test the (concat_vectors (X), (trunc(umin(smax(Y, 0), 2^n))))) pattern.
+; Test the larger (trunc(umin(smax(Y, 0), 2^n)))) pattern.
define <16 x i8> @us_maxmin_v16i16_to_v16i8(<16 x i16> %y) {
; CHECK-LABEL: us_maxmin_v16i16_to_v16i8:
@@ -586,7 +586,7 @@ entry:
ret <4 x i32> %trunc
}
-; Test the (concat_vectors (X), (trunc(smin(smax(Y, 0), 2^n))))) pattern.
+; Test the larger (trunc(smin(smax(Y, 0), 2^n))) pattern.
define <16 x i8> @sminsmax_range_unsig16ed_i16_to_i8(<16 x i16> %y) {
; CHECK-LABEL: sminsmax_range_unsig16ed_i16_to_i8:
@@ -627,6 +627,80 @@ entry:
ret <4 x i32> %trunc
}
+; Test the trunc nsw trunc(smin(smax(Y, -2^n), 2^n-1)) pattern.
+
+define <16 x i8> @signed_minnsw_v16i16_to_v16i8(<16 x i16> %y) {
+; CHECK-SD-LABEL: signed_minnsw_v16i16_to_v16i8:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: movi v2.8h, #127
+; CHECK-SD-NEXT: smin v1.8h, v1.8h, v2.8h
+; CHECK-SD-NEXT: smin v0.8h, v0.8h, v2.8h
+; CHECK-SD-NEXT: uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: signed_minnsw_v16i16_to_v16i8:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: movi v2.8h, #127
+; CHECK-GI-NEXT: smin v0.8h, v0.8h, v2.8h
+; CHECK-GI-NEXT: smin v1.8h, v1.8h, v2.8h
+; CHECK-GI-NEXT: uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT: ret
+entry:
+ %min = call <16 x i16> @llvm.smin.v16i16(<16 x i16> %y, <16 x i16> <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>)
+ %trunc = trunc nsw <16 x i16> %min to <16 x i8>
+ ret <16 x i8> %trunc
+}
+
+define <8 x i16> @signed_minnsw_v8i32_to_v8i16(<8 x i32> %y) {
+; CHECK-SD-LABEL: signed_minnsw_v8i32_to_v8i16:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: movi v2.4s, #127, msl #8
+; CHECK-SD-NEXT: smin v1.4s, v1.4s, v2.4s
+; CHECK-SD-NEXT: smin v0.4s, v0.4s, v2.4s
+; CHECK-SD-NEXT: uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: signed_minnsw_v8i32_to_v8i16:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: movi v2.4s, #127, msl #8
+; CHECK-GI-NEXT: smin v0.4s, v0.4s, v2.4s
+; CHECK-GI-NEXT: smin v1.4s, v1.4s, v2.4s
+; CHECK-GI-NEXT: uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-GI-NEXT: ret
+entry:
+ %min = call <8 x i32> @llvm.smin.v8i32(<8 x i32> %y, <8 x i32> <i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767>)
+ %trunc = trunc nsw <8 x i32> %min to <8 x i16>
+ ret <8 x i16> %trunc
+}
+
+define <4 x i32> @signed_minnsw_v4i64_to_v4i32(<4 x i64> %y) {
+; CHECK-SD-LABEL: signed_minnsw_v4i64_to_v4i32:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: mov w8, #2147483647 // =0x7fffffff
+; CHECK-SD-NEXT: dup v2.2d, x8
+; CHECK-SD-NEXT: cmgt v3.2d, v2.2d, v1.2d
+; CHECK-SD-NEXT: cmgt v4.2d, v2.2d, v0.2d
+; CHECK-SD-NEXT: bif v1.16b, v2.16b, v3.16b
+; CHECK-SD-NEXT: bif v0.16b, v2.16b, v4.16b
+; CHECK-SD-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: signed_minnsw_v4i64_to_v4i32:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: adrp x8, .LCPI47_0
+; CHECK-GI-NEXT: ldr q2, [x8, :lo12:.LCPI47_0]
+; CHECK-GI-NEXT: cmgt v3.2d, v2.2d, v0.2d
+; CHECK-GI-NEXT: cmgt v4.2d, v2.2d, v1.2d
+; CHECK-GI-NEXT: bif v0.16b, v2.16b, v3.16b
+; CHECK-GI-NEXT: bif v1.16b, v2.16b, v4.16b
+; CHECK-GI-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: ret
+entry:
+ %min = call <4 x i64> @llvm.smin.v4i64(<4 x i64> %y, <4 x i64> <i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647>)
+ %trunc = trunc nsw <4 x i64> %min to <4 x i32>
+ ret <4 x i32> %trunc
+}
+
; Type support varification - not supported with saturated value
@@ -681,13 +755,13 @@ define <4 x i16> @sminsmax_range_signed_i64_to_i16(<2 x i16> %x, <2 x i64> %y) {
;
; CHECK-GI-LABEL: sminsmax_range_signed_i64_to_i16:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: adrp x8, .LCPI46_1
+; CHECK-GI-NEXT: adrp x8, .LCPI49_1
; CHECK-GI-NEXT: uzp1 v0.4h, v0.4h, v0.4h
-; CHECK-GI-NEXT: ldr q2, [x8, :lo12:.LCPI46_1]
-; CHECK-GI-NEXT: adrp x8, .LCPI46_0
+; CHECK-GI-NEXT: ldr q2, [x8, :lo12:.LCPI49_1]
+; CHECK-GI-NEXT: adrp x8, .LCPI49_0
; CHECK-GI-NEXT: cmgt v3.2d, v1.2d, v2.2d
; CHECK-GI-NEXT: bif v1.16b, v2.16b, v3.16b
-; CHECK-GI-NEXT: ldr q2, [x8, :lo12:.LCPI46_0]
+; CHECK-GI-NEXT: ldr q2, [x8, :lo12:.LCPI49_0]
; CHECK-GI-NEXT: cmgt v3.2d, v2.2d, v1.2d
; CHECK-GI-NEXT: bif v1.16b, v2.16b, v3.16b
; CHECK-GI-NEXT: xtn v1.2s, v1.2d
diff --git a/llvm/test/CodeGen/AArch64/saturating-vec-smull.ll b/llvm/test/CodeGen/AArch64/saturating-vec-smull.ll
index 24f89fa0e85c3..fe9b060be6ae2 100644
--- a/llvm/test/CodeGen/AArch64/saturating-vec-smull.ll
+++ b/llvm/test/CodeGen/AArch64/saturating-vec-smull.ll
@@ -27,10 +27,10 @@ define <2 x i16> @saturating_2xi16(<2 x i16> %a, <2 x i16> %b) {
; CHECK-GI-NEXT: ret
%as = sext <2 x i16> %a to <2 x i32>
%bs = sext <2 x i16> %b to <2 x i32>
- %m = mul <2 x i32> %bs, %as
+ %m = mul nsw <2 x i32> %bs, %as
%sh = ashr <2 x i32> %m, splat (i32 15)
- %ma = tail call <2 x i32> @llvm.smin.v4i32(<2 x i32> %sh, <2 x i32> splat (i32 32767))
- %t = trunc <2 x i32> %ma to <2 x i16>
+ %ma = tail call <2 x i32> @llvm.smin.v2i32(<2 x i32> %sh, <2 x i32> splat (i32 32767))
+ %t = trunc nsw <2 x i32> %ma to <2 x i16>
ret <2 x i16> %t
}
@@ -50,10 +50,10 @@ define <4 x i16> @saturating_4xi16(<4 x i16> %a, <4 x i16> %b) {
; CHECK-GI-NEXT: ret
%as = sext <4 x i16> %a to <4 x i32>
%bs = sext <4 x i16> %b to <4 x i32>
- %m = mul <4 x i32> %bs, %as
+ %m = mul nsw <4 x i32> %bs, %as
%sh = ashr <4 x i32> %m, splat (i32 15)
%ma = tail call <4 x i32> @llvm.smin.v4i32(<4 x i32> %sh, <4 x i32> splat (i32 32767))
- %t = trunc <4 x i32> %ma to <4 x i16>
+ %t = trunc nsw <4 x i32> %ma to <4 x i16>
ret <4 x i16> %t
}
@@ -76,10 +76,10 @@ define <8 x i16> @saturating_8xi16(<8 x i16> %a, <8 x i16> %b) {
; CHECK-GI-NEXT: ret
%as = sext <8 x i16> %a to <8 x i32>
%bs = sext <8 x i16> %b to <8 x i32>
- %m = mul <8 x i32> %bs, %as
+ %m = mul nsw <8 x i32> %bs, %as
%sh = ashr <8 x i32> %m, splat (i32 15)
%ma = tail call <8 x i32> @llvm.smin.v8i32(<8 x i32> %sh, <8 x i32> splat (i32 32767))
- %t = trunc <8 x i32> %ma to <8 x i16>
+ %t = trunc nsw <8 x i32> %ma to <8 x i16>
ret <8 x i16> %t
}
@@ -101,10 +101,10 @@ define <2 x i32> @saturating_2xi32(<2 x i32> %a, <2 x i32> %b) {
; CHECK-GI-NEXT: ret
%as = sext <2 x i32> %a to <2 x i64>
%bs = sext <2 x i32> %b to <2 x i64>
- %m = mul <2 x i64> %bs, %as
+ %m = mul nsw <2 x i64> %bs, %as
%sh = ashr <2 x i64> %m, splat (i64 31)
- %ma = tail call <2 x i64> @llvm.smin.v8i64(<2 x i64> %sh, <2 x i64> splat (i64 2147483647))
- %t = trunc <2 x i64> %ma to <2 x i32>
+ %ma = tail call <2 x i64> @llvm.smin.v2i64(<2 x i64> %sh, <2 x i64> splat (i64 2147483647))
+ %t = trunc nsw <2 x i64> %ma to <2 x i32>
ret <2 x i32> %t
}
@@ -130,10 +130,10 @@ define <4 x i32> @saturating_4xi32(<4 x i32> %a, <4 x i32> %b) {
; CHECK-GI-NEXT: ret
%as = sext <4 x i32> %a to <4 x i64>
%bs = sext <4 x i32> %b to <4 x i64>
- %m = mul <4 x i64> %bs, %as
+ %m = mul nsw <4 x i64> %bs, %as
%sh = ashr <4 x i64> %m, splat (i64 31)
%ma = tail call <4 x i64> @llvm.smin.v4i64(<4 x i64> %sh, <4 x i64> splat (i64 2147483647))
- %t = trunc <4 x i64> %ma to <4 x i32>
+ %t = trunc nsw <4 x i64> %ma to <4 x i32>
ret <4 x i32> %t
}
@@ -169,10 +169,10 @@ define <8 x i32> @saturating_8xi32(<8 x i32> %a, <8 x i32> %b) {
; CHECK-GI-NEXT: ret
%as = sext <8 x i32> %a to <8 x i64>
%bs = sext <8 x i32> %b to <8 x i64>
- %m = mul <8 x i64> %bs, %as
+ %m = mul nsw <8 x i64> %bs, %as
%sh = ashr <8 x i64> %m, splat (i64 31)
%ma = tail call <8 x i64> @llvm.smin.v8i64(<8 x i64> %sh, <8 x i64> splat (i64 2147483647))
- %t = trunc <8 x i64> %ma to <8 x i32>
+ %t = trunc nsw <8 x i64> %ma to <8 x i32>
ret <8 x i32> %t
}
@@ -194,9 +194,9 @@ define <2 x i64> @saturating_2xi32_2xi64(<2 x i32> %a, <2 x i32> %b) {
; CHECK-GI-NEXT: ret
%as = sext <2 x i32> %a to <2 x i64>
%bs = sext <2 x i32> %b to <2 x i64>
- %m = mul <2 x i64> %bs, %as
+ %m = mul nsw <2 x i64> %bs, %as
%sh = ashr <2 x i64> %m, splat (i64 31)
- %ma = tail call <2 x i64> @llvm.smin.v8i64(<2 x i64> %sh, <2 x i64> splat (i64 2147483647))
+ %ma = tail call <2 x i64> @llvm.smin.v2i64(<2 x i64> %sh, <2 x i64> splat (i64 2147483647))
ret <2 x i64> %ma
}
@@ -253,10 +253,10 @@ define <6 x i16> @saturating_6xi16(<6 x i16> %a, <6 x i16> %b) {
; CHECK-GI-NEXT: ret
%as = sext <6 x i16> %a to <6 x i32>
%bs = sext <6 x i16> %b to <6 x i32>
- %m = mul <6 x i32> %bs, %as
+ %m = mul nsw <6 x i32> %bs, %as
%sh = ashr <6 x i32> %m, splat (i32 15)
%ma = tail call <6 x i32> @llvm.smin.v6i32(<6 x i32> %sh, <6 x i32> splat (i32 32767))
- %t = trunc <6 x i32> %ma to <6 x i16>
+ %t = trunc nsw <6 x i32> %ma to <6 x i16>
ret <6 x i16> %t
}
@@ -271,10 +271,10 @@ define <4 x i16> @unsupported_saturation_value_v4i16(<4 x i16> %a, <4 x i16> %b)
; CHECK-NEXT: ret
%as = sext <4 x i16> %a to <4 x i32>
%bs = sext <4 x i16> %b to <4 x i32>
- %m = mul <4 x i32> %bs, %as
+ %m = mul nsw <4 x i32> %bs, %as
%sh = ashr <4 x i32> %m, splat (i32 15)
%ma = tail call <4 x i32> @llvm.smin.v4i32(<4 x i32> %sh, <4 x i32> splat (i32 42))
- %t = trunc <4 x i32> %ma to <4 x i16>
+ %t = trunc nsw <4 x i32> %ma to <4 x i16>
ret <4 x i16> %t
}
@@ -289,7 +289,7 @@ define <4 x i16> @unsupported_shift_value_v4i16(<4 x i16> %a, <4 x i16> %b) {
; CHECK-NEXT: ret
%as = sext <4 x i16> %a to <4 x i32>
%bs = sext <4 x i16> %b to <4 x i32>
- %m = mul <4 x i32> %bs, %as
+ %m = mul nsw <4 x i32> %bs, %as
%sh = ashr <4 x i32> %m, splat (i32 3)
%ma = tail call <4 x i32> @llvm.smin.v4i32(<4 x i32> %sh, <4 x i32> splat (i32 32767))
%t = trunc <4 x i32> %ma to <4 x i16>
@@ -307,10 +307,10 @@ define <2 x i16> @extend_to_illegal_type(<2 x i16> %a, <2 x i16> %b) {
; CHECK-NEXT: ret
%as = sext <2 x i16> %a to <2 x i48>
%bs = sext <2 x i16> %b to <2 x i48>
- %m = mul <2 x i48> %bs, %as
+ %m = mul nsw <2 x i48> %bs, %as
%sh = ashr <2 x i48> %m, splat (i48 15)
- %ma = tail call <2 x i48> @llvm.smin.v4i32(<2 x i48> %sh, <2 x i48> splat (i48 32767))
- %t = trunc <2 x i48> %ma to <2 x i16>
+ %ma = tail call <2 x i48> @llvm.smin.v2i48(<2 x i48> %sh, <2 x i48> splat (i48 32767))
+ %t = trunc nsw <2 x i48> %ma to <2 x i16>
ret <2 x i16> %t
}
@@ -365,9 +365,9 @@ define <1 x i16> @saturating_1xi16(<1 x i16> %a, <1 x i16> %b) {
; CHECK-GI-NEXT: ret
%as = sext <1 x i16> %a to <1 x i32>
%bs = sext <1 x i16> %b to <1 x i32>
- %m = mul <1 x i32> %bs, %as
+ %m = mul nsw <1 x i32> %bs, %as
%sh = ashr <1 x i32> %m, splat (i32 15)
%ma = tail call <1 x i32> @llvm.smin.v1i32(<1 x i32> %sh, <1 x i32> splat (i32 32767))
- %t = trunc <1 x i32> %ma to <1 x i16>
+ %t = trunc nsw <1 x i32> %ma to <1 x i16>
ret <1 x i16> %t
}
>From 5426b65b44c5374cd82077e3b142054c004569dc Mon Sep 17 00:00:00 2001
From: David Green <david.green at arm.com>
Date: Wed, 19 Aug 2026 07:22:35 +0100
Subject: [PATCH 2/6] [AArch64][GlobalISel] Add lowering for s/umul.fix.sat
This extends the existing s/umul.fix lowering to handle the saturating
versions of the instructions. They use a TruncSSatS or TruncUSatU instead of
the trunc, keeping the implementation simple. That does require truncsat
lowering to be added to allow those operations to subsequently lower if needed.
---
.../llvm/CodeGen/GlobalISel/LegalizerHelper.h | 1 +
.../CodeGen/GlobalISel/LegalizerHelper.cpp | 67 +-
.../AArch64/GISel/AArch64LegalizerInfo.cpp | 7 +-
.../GlobalISel/legalizer-info-validation.mir | 22 +-
llvm/test/CodeGen/AArch64/smul_fix_sat.ll | 1089 +++++++++++------
llvm/test/CodeGen/AArch64/umul_fix_sat.ll | 688 +++++++----
6 files changed, 1277 insertions(+), 597 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
index ce7bf3c49712f..de00cf8966970 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
@@ -575,6 +575,7 @@ class LegalizerHelper {
LLVM_ABI LegalizeResult lowerAddSubSatToMinMax(MachineInstr &MI);
LLVM_ABI LegalizeResult lowerAddSubSatToAddoSubo(MachineInstr &MI);
LLVM_ABI LegalizeResult lowerShlSat(MachineInstr &MI);
+ LLVM_ABI LegalizeResult lowerTruncSat(MachineInstr &MI);
LLVM_ABI LegalizeResult lowerBswap(MachineInstr &MI);
LLVM_ABI LegalizeResult lowerBitreverse(MachineInstr &MI);
LLVM_ABI LegalizeResult lowerReadWriteRegister(MachineInstr &MI);
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 6f85390fd4ede..3a9de29c55224 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -5058,6 +5058,10 @@ LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT LowerHintTy) {
case G_SSHLSAT:
case G_USHLSAT:
return lowerShlSat(MI);
+ case G_TRUNC_SSAT_S:
+ case G_TRUNC_USAT_U:
+ case G_TRUNC_SSAT_U:
+ return lowerTruncSat(MI);
case G_ABS:
return lowerAbsToAddXor(MI);
case G_ABDS:
@@ -5113,6 +5117,8 @@ LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT LowerHintTy) {
}
case G_SMULFIX:
case G_UMULFIX:
+ case G_SMULFIXSAT:
+ case G_UMULFIXSAT:
return lowerMulfix(MI);
}
}
@@ -10297,6 +10303,39 @@ LegalizerHelper::lowerShlSat(MachineInstr &MI) {
return Legalized;
}
+LegalizerHelper::LegalizeResult
+LegalizerHelper::lowerTruncSat(MachineInstr &MI) {
+ unsigned Opc = MI.getOpcode();
+ auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
+ unsigned DstSize = DstTy.getScalarSizeInBits();
+ unsigned SrcSize = SrcTy.getScalarSizeInBits();
+
+ if (Opc == TargetOpcode::G_TRUNC_SSAT_S) {
+ auto Max = MIRBuilder.buildConstant(
+ SrcTy, APInt::getSignedMaxValue(DstSize).sext(SrcSize));
+ Src = MIRBuilder.buildSMin(SrcTy, Src, Max).getReg(0);
+ auto Min = MIRBuilder.buildConstant(
+ SrcTy, APInt::getSignedMinValue(DstSize).sext(SrcSize));
+ Src = MIRBuilder.buildSMax(SrcTy, Src, Min).getReg(0);
+ } else if (Opc == TargetOpcode::G_TRUNC_USAT_U) {
+ auto Max = MIRBuilder.buildConstant(
+ SrcTy, APInt::getAllOnes(DstSize).zext(SrcSize));
+ Src = MIRBuilder.buildUMin(SrcTy, Src, Max).getReg(0);
+ } else if (Opc == TargetOpcode::G_TRUNC_SSAT_U) {
+ auto Max = MIRBuilder.buildConstant(
+ SrcTy, APInt::getAllOnes(DstSize).zext(SrcSize));
+ Src = MIRBuilder.buildSMin(SrcTy, Src, Max).getReg(0);
+ auto Min = MIRBuilder.buildConstant(SrcTy, APInt::getZero(SrcSize));
+ Src = MIRBuilder.buildSMax(SrcTy, Src, Min).getReg(0);
+ } else {
+ llvm_unreachable("Expected truncsat opcode!");
+ }
+
+ MIRBuilder.buildTrunc(Dst, Src);
+ MI.eraseFromParent();
+ return Legalized;
+}
+
LegalizerHelper::LegalizeResult LegalizerHelper::lowerBswap(MachineInstr &MI) {
auto [Dst, Src] = MI.getFirst2Regs();
const LLT Ty = MRI.getType(Src);
@@ -10933,25 +10972,32 @@ LegalizerHelper::LegalizeResult LegalizerHelper::lowerVAArg(MachineInstr &MI) {
}
LegalizerHelper::LegalizeResult LegalizerHelper::lowerMulfix(MachineInstr &MI) {
- [[maybe_unused]] unsigned OpCode = MI.getOpcode();
+ unsigned OpCode = MI.getOpcode();
assert((OpCode == TargetOpcode::G_SMULFIX ||
- OpCode == TargetOpcode::G_UMULFIX) &&
- "Operator must be either G_SMULFIX or G_UMULFIX!");
+ OpCode == TargetOpcode::G_UMULFIX ||
+ OpCode == TargetOpcode::G_SMULFIXSAT ||
+ OpCode == TargetOpcode::G_UMULFIXSAT) &&
+ "Operator must be either G_SMULFIX[SAT] or G_UMULFIX[SAT]!");
auto [Dst, LHS, RHS] = MI.getFirst3Regs();
LLT Ty = MRI.getType(Dst);
unsigned Scale = MI.getOperand(3).getImm();
- if (Scale == 0) {
+ bool Saturating = (OpCode == TargetOpcode::G_SMULFIXSAT ||
+ OpCode == TargetOpcode::G_UMULFIXSAT);
+ bool IsSigned = (OpCode == TargetOpcode::G_SMULFIX ||
+ OpCode == TargetOpcode::G_SMULFIXSAT);
+
+ if (!Saturating && Scale == 0) {
MIRBuilder.buildMul(Dst, LHS, RHS);
MI.eraseFromParent();
return Legalized;
}
- // TODO: Port other lowerng paths from SelectionDAG.
+ // TODO: Port other lowering paths from SelectionDAG.
LLT WideTy = Ty.changeElementSize(Ty.getScalarSizeInBits() * 2);
auto ShiftAmt = MIRBuilder.buildConstant(WideTy, Scale);
MachineInstrBuilder ExtLHS{}, ExtRHS{}, Shift{};
- if (MI.getOpcode() == TargetOpcode::G_SMULFIX) {
+ if (IsSigned) {
ExtLHS = MIRBuilder.buildSExt(WideTy, LHS);
ExtRHS = MIRBuilder.buildSExt(WideTy, RHS);
} else {
@@ -10960,12 +11006,17 @@ LegalizerHelper::LegalizeResult LegalizerHelper::lowerMulfix(MachineInstr &MI) {
}
auto Mul = MIRBuilder.buildMul(WideTy, ExtLHS, ExtRHS);
- if (MI.getOpcode() == TargetOpcode::G_SMULFIX)
+ if (IsSigned)
Shift = MIRBuilder.buildAShr(WideTy, Mul, ShiftAmt);
else
Shift = MIRBuilder.buildLShr(WideTy, Mul, ShiftAmt);
- MIRBuilder.buildTrunc(Dst, Shift);
+ if (!Saturating)
+ MIRBuilder.buildTrunc(Dst, Shift);
+ else if (IsSigned)
+ MIRBuilder.buildTruncSSatS(Dst, Shift);
+ else
+ MIRBuilder.buildTruncUSatU(Dst, Shift);
MI.eraseFromParent();
return Legalized;
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index 1274aab686bae..91ad1911818f6 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -296,7 +296,9 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
.legalFor({i64, v16i8, v8i16, v4i32})
.lower();
- getActionDefinitionsBuilder({G_SMULFIX, G_UMULFIX}).lower();
+ getActionDefinitionsBuilder(
+ {G_SMULFIX, G_UMULFIX, G_SMULFIXSAT, G_UMULFIXSAT})
+ .lower();
getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX})
.legalFor({v8i8, v16i8, v4i16, v8i16, v2i32, v4i32})
@@ -858,7 +860,8 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
getActionDefinitionsBuilder({G_TRUNC_SSAT_S, G_TRUNC_SSAT_U, G_TRUNC_USAT_U})
.legalFor({{v8i8, v8i16}, {v4i16, v4i32}, {v2i32, v2i64}})
- .clampNumElements(0, v2s32, v2s32);
+ .clampNumElements(0, v2s32, v2s32)
+ .lower();
getActionDefinitionsBuilder(G_SEXT_INREG)
.legalFor({i32, i64, v8i8, v16i8, v4i16, v8i16, v2i32, v4i32, v2i64})
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
index 4a88a30d56e01..de55cc6cc5393 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
@@ -354,16 +354,16 @@
# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_TRUNC_SSAT_S (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
-# DEBUG-NEXT: .. the first uncovered type index: 2, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_TRUNC_SSAT_U (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
-# DEBUG-NEXT: .. the first uncovered type index: 2, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_TRUNC_USAT_U (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
-# DEBUG-NEXT: .. the first uncovered type index: 2, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_CONSTANT (opcode {{[0-9]+}}): 1 type index, 0 imm indices
# DEBUG-NEXT: .. the first uncovered type index: 1, OK
# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
@@ -500,11 +500,13 @@
# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_SMULFIXSAT (opcode {{[0-9]+}}): 1 type index, 1 imm index
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_UMULFIXSAT (opcode {{[0-9]+}}): 1 type index, 1 imm index
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_SDIVFIX (opcode {{[0-9]+}}): 1 type index, 1 imm index
# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
diff --git a/llvm/test/CodeGen/AArch64/smul_fix_sat.ll b/llvm/test/CodeGen/AArch64/smul_fix_sat.ll
index 0a087c949d4d3..865b11100c4f5 100644
--- a/llvm/test/CodeGen/AArch64/smul_fix_sat.ll
+++ b/llvm/test/CodeGen/AArch64/smul_fix_sat.ll
@@ -2,455 +2,856 @@
; RUN: llc < %s -mtriple=aarch64-linux-gnu -global-isel=0 | FileCheck %s --check-prefixes=CHECK,CHECK-SD
; RUN: llc < %s -mtriple=aarch64-linux-gnu -global-isel=1 -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
-; CHECK-GI: warning: Instruction selection used fallback path for func
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for func2
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for func3
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for func4
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for func5
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for func6
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for func7
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for func8
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vec_v8i8
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vec_v16i8
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vec_v4i16
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vec_v8i16
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vec_v2i32
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vec_v4i32
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vec_v8i32
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vec_v2i64
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vec_v4i64
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vec_sqdmulh_v8i16
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vec_sqdmulh_v4i16
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vec_sqdmulh_v4i32
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vec_sqdmulh_v2i32
define i32 @func(i32 %x, i32 %y) {
-; CHECK-LABEL: func:
-; CHECK: // %bb.0:
-; CHECK-NEXT: smull x9, w0, w1
-; CHECK-NEXT: mov w8, #2147483647 // =0x7fffffff
-; CHECK-NEXT: lsr x10, x9, #32
-; CHECK-NEXT: extr w9, w10, w9, #2
-; CHECK-NEXT: cmp w10, #1
-; CHECK-NEXT: csel w8, w8, w9, gt
-; CHECK-NEXT: cmn w10, #2
-; CHECK-NEXT: mov w9, #-2147483648 // =0x80000000
-; CHECK-NEXT: csel w0, w9, w8, lt
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: func:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: smull x9, w0, w1
+; CHECK-SD-NEXT: mov w8, #2147483647 // =0x7fffffff
+; CHECK-SD-NEXT: lsr x10, x9, #32
+; CHECK-SD-NEXT: extr w9, w10, w9, #2
+; CHECK-SD-NEXT: cmp w10, #1
+; CHECK-SD-NEXT: csel w8, w8, w9, gt
+; CHECK-SD-NEXT: cmn w10, #2
+; CHECK-SD-NEXT: mov w9, #-2147483648 // =0x80000000
+; CHECK-SD-NEXT: csel w0, w9, w8, lt
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: func:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: smull x8, w0, w1
+; CHECK-GI-NEXT: mov w9, #2147483647 // =0x7fffffff
+; CHECK-GI-NEXT: asr x8, x8, #2
+; CHECK-GI-NEXT: cmp x8, x9
+; CHECK-GI-NEXT: csel x8, x8, x9, lt
+; CHECK-GI-NEXT: mov x9, #-2147483648 // =0xffffffff80000000
+; CHECK-GI-NEXT: cmp x8, x9
+; CHECK-GI-NEXT: csel x0, x8, x9, gt
+; CHECK-GI-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-GI-NEXT: ret
%tmp = call i32 @llvm.smul.fix.sat.i32(i32 %x, i32 %y, i32 2)
ret i32 %tmp
}
define i64 @func2(i64 %x, i64 %y) {
-; CHECK-LABEL: func2:
-; CHECK: // %bb.0:
-; CHECK-NEXT: mul x9, x0, x1
-; CHECK-NEXT: mov x8, #9223372036854775807 // =0x7fffffffffffffff
-; CHECK-NEXT: smulh x10, x0, x1
-; CHECK-NEXT: extr x9, x10, x9, #2
-; CHECK-NEXT: cmp x10, #1
-; CHECK-NEXT: csel x8, x8, x9, gt
-; CHECK-NEXT: cmn x10, #2
-; CHECK-NEXT: mov x9, #-9223372036854775808 // =0x8000000000000000
-; CHECK-NEXT: csel x0, x9, x8, lt
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: func2:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: mul x9, x0, x1
+; CHECK-SD-NEXT: mov x8, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-SD-NEXT: smulh x10, x0, x1
+; CHECK-SD-NEXT: extr x9, x10, x9, #2
+; CHECK-SD-NEXT: cmp x10, #1
+; CHECK-SD-NEXT: csel x8, x8, x9, gt
+; CHECK-SD-NEXT: cmn x10, #2
+; CHECK-SD-NEXT: mov x9, #-9223372036854775808 // =0x8000000000000000
+; CHECK-SD-NEXT: csel x0, x9, x8, lt
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: func2:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: umulh x9, x0, x1
+; CHECK-GI-NEXT: asr x10, x1, #63
+; CHECK-GI-NEXT: mov x8, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-GI-NEXT: mul x11, x0, x1
+; CHECK-GI-NEXT: madd x9, x0, x10, x9
+; CHECK-GI-NEXT: asr x10, x0, #63
+; CHECK-GI-NEXT: madd x9, x10, x1, x9
+; CHECK-GI-NEXT: extr x10, x9, x11, #2
+; CHECK-GI-NEXT: asr x9, x9, #2
+; CHECK-GI-NEXT: cmp x10, x8
+; CHECK-GI-NEXT: cset w11, lo
+; CHECK-GI-NEXT: cmp x9, #0
+; CHECK-GI-NEXT: cset w12, mi
+; CHECK-GI-NEXT: csel w11, w11, w12, eq
+; CHECK-GI-NEXT: tst w11, #0x1
+; CHECK-GI-NEXT: csel x8, x10, x8, ne
+; CHECK-GI-NEXT: mov x10, #-9223372036854775808 // =0x8000000000000000
+; CHECK-GI-NEXT: csel x9, x9, xzr, ne
+; CHECK-GI-NEXT: cmp x8, x10
+; CHECK-GI-NEXT: cset w11, hi
+; CHECK-GI-NEXT: cmn x9, #1
+; CHECK-GI-NEXT: cset w9, gt
+; CHECK-GI-NEXT: csel w9, w11, w9, eq
+; CHECK-GI-NEXT: tst w9, #0x1
+; CHECK-GI-NEXT: csel x0, x8, x10, ne
+; CHECK-GI-NEXT: ret
%tmp = call i64 @llvm.smul.fix.sat.i64(i64 %x, i64 %y, i32 2)
ret i64 %tmp
}
define i4 @func3(i4 %x, i4 %y) {
-; CHECK-LABEL: func3:
-; CHECK: // %bb.0:
-; CHECK-NEXT: sbfx w9, w1, #0, #4
-; CHECK-NEXT: lsl w10, w0, #28
-; CHECK-NEXT: mov w8, #2147483647 // =0x7fffffff
-; CHECK-NEXT: smull x9, w10, w9
-; CHECK-NEXT: lsr x10, x9, #32
-; CHECK-NEXT: extr w9, w10, w9, #2
-; CHECK-NEXT: cmp w10, #1
-; CHECK-NEXT: csel w8, w8, w9, gt
-; CHECK-NEXT: cmn w10, #2
-; CHECK-NEXT: mov w9, #-2147483648 // =0x80000000
-; CHECK-NEXT: csel w8, w9, w8, lt
-; CHECK-NEXT: asr w0, w8, #28
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: func3:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: sbfx w9, w1, #0, #4
+; CHECK-SD-NEXT: lsl w10, w0, #28
+; CHECK-SD-NEXT: mov w8, #2147483647 // =0x7fffffff
+; CHECK-SD-NEXT: smull x9, w10, w9
+; CHECK-SD-NEXT: lsr x10, x9, #32
+; CHECK-SD-NEXT: extr w9, w10, w9, #2
+; CHECK-SD-NEXT: cmp w10, #1
+; CHECK-SD-NEXT: csel w8, w8, w9, gt
+; CHECK-SD-NEXT: cmn w10, #2
+; CHECK-SD-NEXT: mov w9, #-2147483648 // =0x80000000
+; CHECK-SD-NEXT: csel w8, w9, w8, lt
+; CHECK-SD-NEXT: asr w0, w8, #28
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: func3:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: sbfx w9, w0, #0, #4
+; CHECK-GI-NEXT: sbfx w10, w1, #0, #4
+; CHECK-GI-NEXT: mov w8, #7 // =0x7
+; CHECK-GI-NEXT: mul w9, w9, w10
+; CHECK-GI-NEXT: sbfx w9, w9, #2, #6
+; CHECK-GI-NEXT: cmp w9, #7
+; CHECK-GI-NEXT: csel w8, w9, w8, lt
+; CHECK-GI-NEXT: mov w9, #-8 // =0xfffffff8
+; CHECK-GI-NEXT: cmn w8, #8
+; CHECK-GI-NEXT: csel w0, w8, w9, gt
+; CHECK-GI-NEXT: ret
%tmp = call i4 @llvm.smul.fix.sat.i4(i4 %x, i4 %y, i32 2)
ret i4 %tmp
}
;; These result in regular integer multiplication with a saturation check.
define i32 @func4(i32 %x, i32 %y) {
-; CHECK-LABEL: func4:
-; CHECK: // %bb.0:
-; CHECK-NEXT: smull x9, w0, w1
-; CHECK-NEXT: eor w10, w0, w1
-; CHECK-NEXT: mov w8, #-2147483648 // =0x80000000
-; CHECK-NEXT: cmp w10, #0
-; CHECK-NEXT: cinv w8, w8, pl
-; CHECK-NEXT: cmp x9, w9, sxtw
-; CHECK-NEXT: csel w0, w8, w9, ne
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: func4:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: smull x9, w0, w1
+; CHECK-SD-NEXT: eor w10, w0, w1
+; CHECK-SD-NEXT: mov w8, #-2147483648 // =0x80000000
+; CHECK-SD-NEXT: cmp w10, #0
+; CHECK-SD-NEXT: cinv w8, w8, pl
+; CHECK-SD-NEXT: cmp x9, w9, sxtw
+; CHECK-SD-NEXT: csel w0, w8, w9, ne
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: func4:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: smull x8, w0, w1
+; CHECK-GI-NEXT: mov w9, #2147483647 // =0x7fffffff
+; CHECK-GI-NEXT: cmp x8, x9
+; CHECK-GI-NEXT: csel x8, x8, x9, lt
+; CHECK-GI-NEXT: mov x9, #-2147483648 // =0xffffffff80000000
+; CHECK-GI-NEXT: cmp x8, x9
+; CHECK-GI-NEXT: csel x0, x8, x9, gt
+; CHECK-GI-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-GI-NEXT: ret
%tmp = call i32 @llvm.smul.fix.sat.i32(i32 %x, i32 %y, i32 0)
ret i32 %tmp
}
define i64 @func5(i64 %x, i64 %y) {
-; CHECK-LABEL: func5:
-; CHECK: // %bb.0:
-; CHECK-NEXT: mul x9, x0, x1
-; CHECK-NEXT: eor x11, x0, x1
-; CHECK-NEXT: mov x8, #-9223372036854775808 // =0x8000000000000000
-; CHECK-NEXT: cmp x11, #0
-; CHECK-NEXT: smulh x10, x0, x1
-; CHECK-NEXT: cinv x8, x8, pl
-; CHECK-NEXT: cmp x10, x9, asr #63
-; CHECK-NEXT: csel x0, x8, x9, ne
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: func5:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: mul x9, x0, x1
+; CHECK-SD-NEXT: eor x11, x0, x1
+; CHECK-SD-NEXT: mov x8, #-9223372036854775808 // =0x8000000000000000
+; CHECK-SD-NEXT: cmp x11, #0
+; CHECK-SD-NEXT: smulh x10, x0, x1
+; CHECK-SD-NEXT: cinv x8, x8, pl
+; CHECK-SD-NEXT: cmp x10, x9, asr #63
+; CHECK-SD-NEXT: csel x0, x8, x9, ne
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: func5:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: umulh x8, x0, x1
+; CHECK-GI-NEXT: asr x9, x1, #63
+; CHECK-GI-NEXT: mul x10, x0, x1
+; CHECK-GI-NEXT: madd x8, x0, x9, x8
+; CHECK-GI-NEXT: asr x9, x0, #63
+; CHECK-GI-NEXT: madd x8, x9, x1, x8
+; CHECK-GI-NEXT: mov x9, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-GI-NEXT: cmp x10, x9
+; CHECK-GI-NEXT: cset w11, lo
+; CHECK-GI-NEXT: cmp x8, #0
+; CHECK-GI-NEXT: cset w12, mi
+; CHECK-GI-NEXT: csel w11, w11, w12, eq
+; CHECK-GI-NEXT: tst w11, #0x1
+; CHECK-GI-NEXT: csel x9, x10, x9, ne
+; CHECK-GI-NEXT: mov x10, #-9223372036854775808 // =0x8000000000000000
+; CHECK-GI-NEXT: csel x8, x8, xzr, ne
+; CHECK-GI-NEXT: cmp x9, x10
+; CHECK-GI-NEXT: cset w11, hi
+; CHECK-GI-NEXT: cmn x8, #1
+; CHECK-GI-NEXT: cset w8, gt
+; CHECK-GI-NEXT: csel w8, w11, w8, eq
+; CHECK-GI-NEXT: tst w8, #0x1
+; CHECK-GI-NEXT: csel x0, x9, x10, ne
+; CHECK-GI-NEXT: ret
%tmp = call i64 @llvm.smul.fix.sat.i64(i64 %x, i64 %y, i32 0)
ret i64 %tmp
}
define i4 @func6(i4 %x, i4 %y) {
-; CHECK-LABEL: func6:
-; CHECK: // %bb.0:
-; CHECK-NEXT: sbfx w9, w1, #0, #4
-; CHECK-NEXT: lsl w10, w0, #28
-; CHECK-NEXT: mov w8, #-2147483648 // =0x80000000
-; CHECK-NEXT: smull x11, w10, w9
-; CHECK-NEXT: eor w9, w10, w9
-; CHECK-NEXT: cmp w9, #0
-; CHECK-NEXT: cinv w8, w8, pl
-; CHECK-NEXT: cmp x11, w11, sxtw
-; CHECK-NEXT: csel w8, w8, w11, ne
-; CHECK-NEXT: asr w0, w8, #28
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: func6:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: sbfx w9, w1, #0, #4
+; CHECK-SD-NEXT: lsl w10, w0, #28
+; CHECK-SD-NEXT: mov w8, #-2147483648 // =0x80000000
+; CHECK-SD-NEXT: smull x11, w10, w9
+; CHECK-SD-NEXT: eor w9, w10, w9
+; CHECK-SD-NEXT: cmp w9, #0
+; CHECK-SD-NEXT: cinv w8, w8, pl
+; CHECK-SD-NEXT: cmp x11, w11, sxtw
+; CHECK-SD-NEXT: csel w8, w8, w11, ne
+; CHECK-SD-NEXT: asr w0, w8, #28
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: func6:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: sbfx w9, w0, #0, #4
+; CHECK-GI-NEXT: sbfx w10, w1, #0, #4
+; CHECK-GI-NEXT: mov w8, #7 // =0x7
+; CHECK-GI-NEXT: mul w9, w9, w10
+; CHECK-GI-NEXT: sxtb w9, w9
+; CHECK-GI-NEXT: cmp w9, #7
+; CHECK-GI-NEXT: csel w8, w9, w8, lt
+; CHECK-GI-NEXT: mov w9, #-8 // =0xfffffff8
+; CHECK-GI-NEXT: cmn w8, #8
+; CHECK-GI-NEXT: csel w0, w8, w9, gt
+; CHECK-GI-NEXT: ret
%tmp = call i4 @llvm.smul.fix.sat.i4(i4 %x, i4 %y, i32 0)
ret i4 %tmp
}
define i64 @func7(i64 %x, i64 %y) {
-; CHECK-LABEL: func7:
-; CHECK: // %bb.0:
-; CHECK-NEXT: mul x9, x0, x1
-; CHECK-NEXT: mov w8, #2147483647 // =0x7fffffff
-; CHECK-NEXT: mov x11, #-2147483648 // =0xffffffff80000000
-; CHECK-NEXT: smulh x10, x0, x1
-; CHECK-NEXT: extr x9, x10, x9, #32
-; CHECK-NEXT: cmp x10, x8
-; CHECK-NEXT: mov x8, #9223372036854775807 // =0x7fffffffffffffff
-; CHECK-NEXT: csel x8, x8, x9, gt
-; CHECK-NEXT: cmp x10, x11
-; CHECK-NEXT: mov x9, #-9223372036854775808 // =0x8000000000000000
-; CHECK-NEXT: csel x0, x9, x8, lt
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: func7:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: mul x9, x0, x1
+; CHECK-SD-NEXT: mov w8, #2147483647 // =0x7fffffff
+; CHECK-SD-NEXT: mov x11, #-2147483648 // =0xffffffff80000000
+; CHECK-SD-NEXT: smulh x10, x0, x1
+; CHECK-SD-NEXT: extr x9, x10, x9, #32
+; CHECK-SD-NEXT: cmp x10, x8
+; CHECK-SD-NEXT: mov x8, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-SD-NEXT: csel x8, x8, x9, gt
+; CHECK-SD-NEXT: cmp x10, x11
+; CHECK-SD-NEXT: mov x9, #-9223372036854775808 // =0x8000000000000000
+; CHECK-SD-NEXT: csel x0, x9, x8, lt
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: func7:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: umulh x9, x0, x1
+; CHECK-GI-NEXT: asr x10, x1, #63
+; CHECK-GI-NEXT: mov x8, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-GI-NEXT: mul x11, x0, x1
+; CHECK-GI-NEXT: madd x9, x0, x10, x9
+; CHECK-GI-NEXT: asr x10, x0, #63
+; CHECK-GI-NEXT: madd x9, x10, x1, x9
+; CHECK-GI-NEXT: extr x10, x9, x11, #32
+; CHECK-GI-NEXT: asr x9, x9, #32
+; CHECK-GI-NEXT: cmp x10, x8
+; CHECK-GI-NEXT: cset w11, lo
+; CHECK-GI-NEXT: cmp x9, #0
+; CHECK-GI-NEXT: cset w12, mi
+; CHECK-GI-NEXT: csel w11, w11, w12, eq
+; CHECK-GI-NEXT: tst w11, #0x1
+; CHECK-GI-NEXT: csel x8, x10, x8, ne
+; CHECK-GI-NEXT: mov x10, #-9223372036854775808 // =0x8000000000000000
+; CHECK-GI-NEXT: csel x9, x9, xzr, ne
+; CHECK-GI-NEXT: cmp x8, x10
+; CHECK-GI-NEXT: cset w11, hi
+; CHECK-GI-NEXT: cmn x9, #1
+; CHECK-GI-NEXT: cset w9, gt
+; CHECK-GI-NEXT: csel w9, w11, w9, eq
+; CHECK-GI-NEXT: tst w9, #0x1
+; CHECK-GI-NEXT: csel x0, x8, x10, ne
+; CHECK-GI-NEXT: ret
%tmp = call i64 @llvm.smul.fix.sat.i64(i64 %x, i64 %y, i32 32)
ret i64 %tmp
}
define i64 @func8(i64 %x, i64 %y) {
-; CHECK-LABEL: func8:
-; CHECK: // %bb.0:
-; CHECK-NEXT: mul x9, x0, x1
-; CHECK-NEXT: mov x8, #4611686018427387903 // =0x3fffffffffffffff
-; CHECK-NEXT: mov x11, #-4611686018427387904 // =0xc000000000000000
-; CHECK-NEXT: smulh x10, x0, x1
-; CHECK-NEXT: extr x9, x10, x9, #63
-; CHECK-NEXT: cmp x10, x8
-; CHECK-NEXT: mov x8, #9223372036854775807 // =0x7fffffffffffffff
-; CHECK-NEXT: csel x8, x8, x9, gt
-; CHECK-NEXT: cmp x10, x11
-; CHECK-NEXT: mov x9, #-9223372036854775808 // =0x8000000000000000
-; CHECK-NEXT: csel x0, x9, x8, lt
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: func8:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: mul x9, x0, x1
+; CHECK-SD-NEXT: mov x8, #4611686018427387903 // =0x3fffffffffffffff
+; CHECK-SD-NEXT: mov x11, #-4611686018427387904 // =0xc000000000000000
+; CHECK-SD-NEXT: smulh x10, x0, x1
+; CHECK-SD-NEXT: extr x9, x10, x9, #63
+; CHECK-SD-NEXT: cmp x10, x8
+; CHECK-SD-NEXT: mov x8, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-SD-NEXT: csel x8, x8, x9, gt
+; CHECK-SD-NEXT: cmp x10, x11
+; CHECK-SD-NEXT: mov x9, #-9223372036854775808 // =0x8000000000000000
+; CHECK-SD-NEXT: csel x0, x9, x8, lt
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: func8:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: umulh x9, x0, x1
+; CHECK-GI-NEXT: asr x10, x1, #63
+; CHECK-GI-NEXT: mov x8, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-GI-NEXT: mul x11, x0, x1
+; CHECK-GI-NEXT: madd x9, x0, x10, x9
+; CHECK-GI-NEXT: asr x10, x0, #63
+; CHECK-GI-NEXT: madd x9, x10, x1, x9
+; CHECK-GI-NEXT: extr x10, x9, x11, #63
+; CHECK-GI-NEXT: asr x9, x9, #63
+; CHECK-GI-NEXT: cmp x10, x8
+; CHECK-GI-NEXT: cset w11, lo
+; CHECK-GI-NEXT: cmp x9, #0
+; CHECK-GI-NEXT: cset w12, mi
+; CHECK-GI-NEXT: csel w11, w11, w12, eq
+; CHECK-GI-NEXT: tst w11, #0x1
+; CHECK-GI-NEXT: csel x8, x10, x8, ne
+; CHECK-GI-NEXT: mov x10, #-9223372036854775808 // =0x8000000000000000
+; CHECK-GI-NEXT: csel x9, x9, xzr, ne
+; CHECK-GI-NEXT: cmp x8, x10
+; CHECK-GI-NEXT: cset w11, hi
+; CHECK-GI-NEXT: cmn x9, #1
+; CHECK-GI-NEXT: cset w9, gt
+; CHECK-GI-NEXT: csel w9, w11, w9, eq
+; CHECK-GI-NEXT: tst w9, #0x1
+; CHECK-GI-NEXT: csel x0, x8, x10, ne
+; CHECK-GI-NEXT: ret
%tmp = call i64 @llvm.smul.fix.sat.i64(i64 %x, i64 %y, i32 63)
ret i64 %tmp
}
define <8 x i8> @vec_v8i8(<8 x i8> %x, <8 x i8> %y) {
-; CHECK-LABEL: vec_v8i8:
-; CHECK: // %bb.0:
-; CHECK-NEXT: smull v0.8h, v0.8b, v1.8b
-; CHECK-NEXT: movi v2.8b, #1
-; CHECK-NEXT: movi v4.8b, #127
-; CHECK-NEXT: shrn v1.8b, v0.8h, #8
-; CHECK-NEXT: xtn v0.8b, v0.8h
-; CHECK-NEXT: shl v3.8b, v1.8b, #6
-; CHECK-NEXT: usra v3.8b, v0.8b, #2
-; CHECK-NEXT: cmgt v0.8b, v1.8b, v2.8b
-; CHECK-NEXT: movi v2.8b, #254
-; CHECK-NEXT: bsl v0.8b, v4.8b, v3.8b
-; CHECK-NEXT: movi v3.8b, #128
-; CHECK-NEXT: cmgt v1.8b, v2.8b, v1.8b
-; CHECK-NEXT: bit v0.8b, v3.8b, v1.8b
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vec_v8i8:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: smull v0.8h, v0.8b, v1.8b
+; CHECK-SD-NEXT: movi v2.8b, #1
+; CHECK-SD-NEXT: movi v4.8b, #127
+; CHECK-SD-NEXT: shrn v1.8b, v0.8h, #8
+; CHECK-SD-NEXT: xtn v0.8b, v0.8h
+; CHECK-SD-NEXT: shl v3.8b, v1.8b, #6
+; CHECK-SD-NEXT: usra v3.8b, v0.8b, #2
+; CHECK-SD-NEXT: cmgt v0.8b, v1.8b, v2.8b
+; CHECK-SD-NEXT: movi v2.8b, #254
+; CHECK-SD-NEXT: bsl v0.8b, v4.8b, v3.8b
+; CHECK-SD-NEXT: movi v3.8b, #128
+; CHECK-SD-NEXT: cmgt v1.8b, v2.8b, v1.8b
+; CHECK-SD-NEXT: bit v0.8b, v3.8b, v1.8b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vec_v8i8:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: smull v0.8h, v0.8b, v1.8b
+; CHECK-GI-NEXT: sqshrn v0.8b, v0.8h, #2
+; CHECK-GI-NEXT: ret
%tmp = call <8 x i8> @llvm.smul.fix.sat.v8i8(<8 x i8> %x, <8 x i8> %y, i32 2)
ret <8 x i8> %tmp
}
define <16 x i8> @vec_v16i8(<16 x i8> %x, <16 x i8> %y) {
-; CHECK-LABEL: vec_v16i8:
-; CHECK: // %bb.0:
-; CHECK-NEXT: smull2 v3.8h, v0.16b, v1.16b
-; CHECK-NEXT: smull v4.8h, v0.8b, v1.8b
-; CHECK-NEXT: movi v2.16b, #1
-; CHECK-NEXT: mul v0.16b, v0.16b, v1.16b
-; CHECK-NEXT: uzp2 v3.16b, v4.16b, v3.16b
-; CHECK-NEXT: movi v4.16b, #127
-; CHECK-NEXT: shl v1.16b, v3.16b, #6
-; CHECK-NEXT: cmgt v2.16b, v3.16b, v2.16b
-; CHECK-NEXT: usra v1.16b, v0.16b, #2
-; CHECK-NEXT: movi v0.16b, #254
-; CHECK-NEXT: bit v1.16b, v4.16b, v2.16b
-; CHECK-NEXT: movi v2.16b, #128
-; CHECK-NEXT: cmgt v0.16b, v0.16b, v3.16b
-; CHECK-NEXT: bsl v0.16b, v2.16b, v1.16b
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vec_v16i8:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: smull2 v3.8h, v0.16b, v1.16b
+; CHECK-SD-NEXT: smull v4.8h, v0.8b, v1.8b
+; CHECK-SD-NEXT: movi v2.16b, #1
+; CHECK-SD-NEXT: mul v0.16b, v0.16b, v1.16b
+; CHECK-SD-NEXT: uzp2 v3.16b, v4.16b, v3.16b
+; CHECK-SD-NEXT: movi v4.16b, #127
+; CHECK-SD-NEXT: shl v1.16b, v3.16b, #6
+; CHECK-SD-NEXT: cmgt v2.16b, v3.16b, v2.16b
+; CHECK-SD-NEXT: usra v1.16b, v0.16b, #2
+; CHECK-SD-NEXT: movi v0.16b, #254
+; CHECK-SD-NEXT: bit v1.16b, v4.16b, v2.16b
+; CHECK-SD-NEXT: movi v2.16b, #128
+; CHECK-SD-NEXT: cmgt v0.16b, v0.16b, v3.16b
+; CHECK-SD-NEXT: bsl v0.16b, v2.16b, v1.16b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vec_v16i8:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: smull v2.8h, v0.8b, v1.8b
+; CHECK-GI-NEXT: smull2 v1.8h, v0.16b, v1.16b
+; CHECK-GI-NEXT: sqshrn v0.8b, v2.8h, #2
+; CHECK-GI-NEXT: sqshrn2 v0.16b, v1.8h, #2
+; CHECK-GI-NEXT: ret
%tmp = call <16 x i8> @llvm.smul.fix.sat.v16i8(<16 x i8> %x, <16 x i8> %y, i32 2)
ret <16 x i8> %tmp
}
define <4 x i16> @vec_v4i16(<4 x i16> %x, <4 x i16> %y) {
-; CHECK-LABEL: vec_v4i16:
-; CHECK: // %bb.0:
-; CHECK-NEXT: smull v0.4s, v0.4h, v1.4h
-; CHECK-NEXT: mvni v2.4h, #254, lsl #8
-; CHECK-NEXT: movi v4.4h, #128, lsl #8
-; CHECK-NEXT: shrn v1.4h, v0.4s, #16
-; CHECK-NEXT: xtn v0.4h, v0.4s
-; CHECK-NEXT: shl v3.4h, v1.4h, #6
-; CHECK-NEXT: usra v3.4h, v0.4h, #10
-; CHECK-NEXT: cmgt v0.4h, v1.4h, v2.4h
-; CHECK-NEXT: movi v2.4h, #254, lsl #8
-; CHECK-NEXT: bic v3.8b, v3.8b, v0.8b
-; CHECK-NEXT: bic v0.4h, #128, lsl #8
-; CHECK-NEXT: cmgt v1.4h, v2.4h, v1.4h
-; CHECK-NEXT: orr v0.8b, v0.8b, v3.8b
-; CHECK-NEXT: bit v0.8b, v4.8b, v1.8b
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vec_v4i16:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: smull v0.4s, v0.4h, v1.4h
+; CHECK-SD-NEXT: mvni v2.4h, #254, lsl #8
+; CHECK-SD-NEXT: movi v4.4h, #128, lsl #8
+; CHECK-SD-NEXT: shrn v1.4h, v0.4s, #16
+; CHECK-SD-NEXT: xtn v0.4h, v0.4s
+; CHECK-SD-NEXT: shl v3.4h, v1.4h, #6
+; CHECK-SD-NEXT: usra v3.4h, v0.4h, #10
+; CHECK-SD-NEXT: cmgt v0.4h, v1.4h, v2.4h
+; CHECK-SD-NEXT: movi v2.4h, #254, lsl #8
+; CHECK-SD-NEXT: bic v3.8b, v3.8b, v0.8b
+; CHECK-SD-NEXT: bic v0.4h, #128, lsl #8
+; CHECK-SD-NEXT: cmgt v1.4h, v2.4h, v1.4h
+; CHECK-SD-NEXT: orr v0.8b, v0.8b, v3.8b
+; CHECK-SD-NEXT: bit v0.8b, v4.8b, v1.8b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vec_v4i16:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: smull v0.4s, v0.4h, v1.4h
+; CHECK-GI-NEXT: sqshrn v0.4h, v0.4s, #10
+; CHECK-GI-NEXT: ret
%tmp = call <4 x i16> @llvm.smul.fix.sat.v4i16(<4 x i16> %x, <4 x i16> %y, i32 10)
ret <4 x i16> %tmp
}
define <8 x i16> @vec_v8i16(<8 x i16> %x, <8 x i16> %y) {
-; CHECK-LABEL: vec_v8i16:
-; CHECK: // %bb.0:
-; CHECK-NEXT: smull2 v3.4s, v0.8h, v1.8h
-; CHECK-NEXT: smull v4.4s, v0.4h, v1.4h
-; CHECK-NEXT: movi v2.8h, #1
-; CHECK-NEXT: mul v0.8h, v0.8h, v1.8h
-; CHECK-NEXT: uzp2 v3.8h, v4.8h, v3.8h
-; CHECK-NEXT: movi v4.8h, #128, lsl #8
-; CHECK-NEXT: shl v1.8h, v3.8h, #14
-; CHECK-NEXT: cmgt v2.8h, v3.8h, v2.8h
-; CHECK-NEXT: usra v1.8h, v0.8h, #2
-; CHECK-NEXT: bic v0.16b, v1.16b, v2.16b
-; CHECK-NEXT: bic v2.8h, #128, lsl #8
-; CHECK-NEXT: mvni v1.8h, #1
-; CHECK-NEXT: orr v0.16b, v2.16b, v0.16b
-; CHECK-NEXT: cmgt v1.8h, v1.8h, v3.8h
-; CHECK-NEXT: bit v0.16b, v4.16b, v1.16b
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vec_v8i16:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: smull2 v3.4s, v0.8h, v1.8h
+; CHECK-SD-NEXT: smull v4.4s, v0.4h, v1.4h
+; CHECK-SD-NEXT: movi v2.8h, #1
+; CHECK-SD-NEXT: mul v0.8h, v0.8h, v1.8h
+; CHECK-SD-NEXT: uzp2 v3.8h, v4.8h, v3.8h
+; CHECK-SD-NEXT: movi v4.8h, #128, lsl #8
+; CHECK-SD-NEXT: shl v1.8h, v3.8h, #14
+; CHECK-SD-NEXT: cmgt v2.8h, v3.8h, v2.8h
+; CHECK-SD-NEXT: usra v1.8h, v0.8h, #2
+; CHECK-SD-NEXT: bic v0.16b, v1.16b, v2.16b
+; CHECK-SD-NEXT: bic v2.8h, #128, lsl #8
+; CHECK-SD-NEXT: mvni v1.8h, #1
+; CHECK-SD-NEXT: orr v0.16b, v2.16b, v0.16b
+; CHECK-SD-NEXT: cmgt v1.8h, v1.8h, v3.8h
+; CHECK-SD-NEXT: bit v0.16b, v4.16b, v1.16b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vec_v8i16:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: smull v2.4s, v0.4h, v1.4h
+; CHECK-GI-NEXT: smull2 v1.4s, v0.8h, v1.8h
+; CHECK-GI-NEXT: sqshrn v0.4h, v2.4s, #2
+; CHECK-GI-NEXT: sqshrn2 v0.8h, v1.4s, #2
+; CHECK-GI-NEXT: ret
%tmp = call <8 x i16> @llvm.smul.fix.sat.v8i16(<8 x i16> %x, <8 x i16> %y, i32 2)
ret <8 x i16> %tmp
}
define <2 x i32> @vec_v2i32(<2 x i32> %x, <2 x i32> %y) {
-; CHECK-LABEL: vec_v2i32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: smull v0.2d, v0.2s, v1.2s
-; CHECK-NEXT: movi v2.2s, #128, lsl #24
-; CHECK-NEXT: shrn v1.2s, v0.2d, #32
-; CHECK-NEXT: xtn v0.2s, v0.2d
-; CHECK-NEXT: cmlt v3.2s, v1.2s, #0
-; CHECK-NEXT: add v4.2s, v1.2s, v1.2s
-; CHECK-NEXT: cmlt v6.2s, v0.2s, #0
-; CHECK-NEXT: mvn v5.8b, v3.8b
-; CHECK-NEXT: shl v4.2s, v4.2s, #31
-; CHECK-NEXT: cmeq v1.2s, v1.2s, v6.2s
-; CHECK-NEXT: bsl v2.8b, v3.8b, v5.8b
-; CHECK-NEXT: orr v0.8b, v4.8b, v0.8b
-; CHECK-NEXT: bif v0.8b, v2.8b, v1.8b
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vec_v2i32:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: smull v0.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT: movi v2.2s, #128, lsl #24
+; CHECK-SD-NEXT: shrn v1.2s, v0.2d, #32
+; CHECK-SD-NEXT: xtn v0.2s, v0.2d
+; CHECK-SD-NEXT: cmlt v3.2s, v1.2s, #0
+; CHECK-SD-NEXT: add v4.2s, v1.2s, v1.2s
+; CHECK-SD-NEXT: cmlt v6.2s, v0.2s, #0
+; CHECK-SD-NEXT: mvn v5.8b, v3.8b
+; CHECK-SD-NEXT: shl v4.2s, v4.2s, #31
+; CHECK-SD-NEXT: cmeq v1.2s, v1.2s, v6.2s
+; CHECK-SD-NEXT: bsl v2.8b, v3.8b, v5.8b
+; CHECK-SD-NEXT: orr v0.8b, v4.8b, v0.8b
+; CHECK-SD-NEXT: bif v0.8b, v2.8b, v1.8b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vec_v2i32:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: smull v0.2d, v0.2s, v1.2s
+; CHECK-GI-NEXT: sqxtn v0.2s, v0.2d
+; CHECK-GI-NEXT: ret
%tmp = call <2 x i32> @llvm.smul.fix.sat.v2i32(<2 x i32> %x, <2 x i32> %y, i32 0)
ret <2 x i32> %tmp
}
define <4 x i32> @vec_v4i32(<4 x i32> %x, <4 x i32> %y) {
-; CHECK-LABEL: vec_v4i32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: smull2 v3.2d, v0.4s, v1.4s
-; CHECK-NEXT: smull v4.2d, v0.2s, v1.2s
-; CHECK-NEXT: movi v2.4s, #63, msl #8
-; CHECK-NEXT: mul v0.4s, v0.4s, v1.4s
-; CHECK-NEXT: uzp2 v3.4s, v4.4s, v3.4s
-; CHECK-NEXT: movi v4.4s, #128, lsl #24
-; CHECK-NEXT: shl v1.4s, v3.4s, #17
-; CHECK-NEXT: cmgt v2.4s, v3.4s, v2.4s
-; CHECK-NEXT: usra v1.4s, v0.4s, #15
-; CHECK-NEXT: bic v0.16b, v1.16b, v2.16b
-; CHECK-NEXT: bic v2.4s, #128, lsl #24
-; CHECK-NEXT: mvni v1.4s, #63, msl #8
-; CHECK-NEXT: orr v0.16b, v2.16b, v0.16b
-; CHECK-NEXT: cmgt v1.4s, v1.4s, v3.4s
-; CHECK-NEXT: bit v0.16b, v4.16b, v1.16b
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vec_v4i32:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: smull2 v3.2d, v0.4s, v1.4s
+; CHECK-SD-NEXT: smull v4.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT: movi v2.4s, #63, msl #8
+; CHECK-SD-NEXT: mul v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT: uzp2 v3.4s, v4.4s, v3.4s
+; CHECK-SD-NEXT: movi v4.4s, #128, lsl #24
+; CHECK-SD-NEXT: shl v1.4s, v3.4s, #17
+; CHECK-SD-NEXT: cmgt v2.4s, v3.4s, v2.4s
+; CHECK-SD-NEXT: usra v1.4s, v0.4s, #15
+; CHECK-SD-NEXT: bic v0.16b, v1.16b, v2.16b
+; CHECK-SD-NEXT: bic v2.4s, #128, lsl #24
+; CHECK-SD-NEXT: mvni v1.4s, #63, msl #8
+; CHECK-SD-NEXT: orr v0.16b, v2.16b, v0.16b
+; CHECK-SD-NEXT: cmgt v1.4s, v1.4s, v3.4s
+; CHECK-SD-NEXT: bit v0.16b, v4.16b, v1.16b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vec_v4i32:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: smull v2.2d, v0.2s, v1.2s
+; CHECK-GI-NEXT: smull2 v1.2d, v0.4s, v1.4s
+; CHECK-GI-NEXT: sqshrn v0.2s, v2.2d, #15
+; CHECK-GI-NEXT: sqshrn2 v0.4s, v1.2d, #15
+; CHECK-GI-NEXT: ret
%tmp = call <4 x i32> @llvm.smul.fix.sat.v4i32(<4 x i32> %x, <4 x i32> %y, i32 15)
ret <4 x i32> %tmp
}
define <8 x i32> @vec_v8i32(<8 x i32> %x, <8 x i32> %y) {
-; CHECK-LABEL: vec_v8i32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: smull2 v4.2d, v1.4s, v3.4s
-; CHECK-NEXT: smull2 v5.2d, v0.4s, v2.4s
-; CHECK-NEXT: smull v6.2d, v0.2s, v2.2s
-; CHECK-NEXT: smull v7.2d, v1.2s, v3.2s
-; CHECK-NEXT: mul v0.4s, v0.4s, v2.4s
-; CHECK-NEXT: mul v1.4s, v1.4s, v3.4s
-; CHECK-NEXT: movi v16.4s, #128, lsl #24
-; CHECK-NEXT: uzp2 v5.4s, v6.4s, v5.4s
-; CHECK-NEXT: uzp2 v4.4s, v7.4s, v4.4s
-; CHECK-NEXT: cmlt v18.4s, v0.4s, #0
-; CHECK-NEXT: cmlt v20.4s, v1.4s, #0
-; CHECK-NEXT: cmlt v2.4s, v5.4s, #0
-; CHECK-NEXT: add v3.4s, v5.4s, v5.4s
-; CHECK-NEXT: cmeq v5.4s, v5.4s, v18.4s
-; CHECK-NEXT: cmlt v6.4s, v4.4s, #0
-; CHECK-NEXT: add v7.4s, v4.4s, v4.4s
-; CHECK-NEXT: mvn v17.16b, v2.16b
-; CHECK-NEXT: shl v3.4s, v3.4s, #31
-; CHECK-NEXT: mvn v19.16b, v6.16b
-; CHECK-NEXT: shl v7.4s, v7.4s, #31
-; CHECK-NEXT: bif v2.16b, v17.16b, v16.16b
-; CHECK-NEXT: orr v0.16b, v3.16b, v0.16b
-; CHECK-NEXT: cmeq v3.4s, v4.4s, v20.4s
-; CHECK-NEXT: bif v6.16b, v19.16b, v16.16b
-; CHECK-NEXT: orr v1.16b, v7.16b, v1.16b
-; CHECK-NEXT: bif v0.16b, v2.16b, v5.16b
-; CHECK-NEXT: bif v1.16b, v6.16b, v3.16b
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vec_v8i32:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: smull2 v4.2d, v1.4s, v3.4s
+; CHECK-SD-NEXT: smull2 v5.2d, v0.4s, v2.4s
+; CHECK-SD-NEXT: smull v6.2d, v0.2s, v2.2s
+; CHECK-SD-NEXT: smull v7.2d, v1.2s, v3.2s
+; CHECK-SD-NEXT: mul v0.4s, v0.4s, v2.4s
+; CHECK-SD-NEXT: mul v1.4s, v1.4s, v3.4s
+; CHECK-SD-NEXT: movi v16.4s, #128, lsl #24
+; CHECK-SD-NEXT: uzp2 v5.4s, v6.4s, v5.4s
+; CHECK-SD-NEXT: uzp2 v4.4s, v7.4s, v4.4s
+; CHECK-SD-NEXT: cmlt v18.4s, v0.4s, #0
+; CHECK-SD-NEXT: cmlt v20.4s, v1.4s, #0
+; CHECK-SD-NEXT: cmlt v2.4s, v5.4s, #0
+; CHECK-SD-NEXT: add v3.4s, v5.4s, v5.4s
+; CHECK-SD-NEXT: cmeq v5.4s, v5.4s, v18.4s
+; CHECK-SD-NEXT: cmlt v6.4s, v4.4s, #0
+; CHECK-SD-NEXT: add v7.4s, v4.4s, v4.4s
+; CHECK-SD-NEXT: mvn v17.16b, v2.16b
+; CHECK-SD-NEXT: shl v3.4s, v3.4s, #31
+; CHECK-SD-NEXT: mvn v19.16b, v6.16b
+; CHECK-SD-NEXT: shl v7.4s, v7.4s, #31
+; CHECK-SD-NEXT: bif v2.16b, v17.16b, v16.16b
+; CHECK-SD-NEXT: orr v0.16b, v3.16b, v0.16b
+; CHECK-SD-NEXT: cmeq v3.4s, v4.4s, v20.4s
+; CHECK-SD-NEXT: bif v6.16b, v19.16b, v16.16b
+; CHECK-SD-NEXT: orr v1.16b, v7.16b, v1.16b
+; CHECK-SD-NEXT: bif v0.16b, v2.16b, v5.16b
+; CHECK-SD-NEXT: bif v1.16b, v6.16b, v3.16b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vec_v8i32:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: smull v4.2d, v0.2s, v2.2s
+; CHECK-GI-NEXT: smull v5.2d, v1.2s, v3.2s
+; CHECK-GI-NEXT: smull2 v2.2d, v0.4s, v2.4s
+; CHECK-GI-NEXT: smull2 v3.2d, v1.4s, v3.4s
+; CHECK-GI-NEXT: sqxtn v0.2s, v4.2d
+; CHECK-GI-NEXT: sqxtn v1.2s, v5.2d
+; CHECK-GI-NEXT: sqxtn2 v0.4s, v2.2d
+; CHECK-GI-NEXT: sqxtn2 v1.4s, v3.2d
+; CHECK-GI-NEXT: ret
%tmp = call <8 x i32> @llvm.smul.fix.sat.v8i32(<8 x i32> %x, <8 x i32> %y, i32 0)
ret <8 x i32> %tmp
}
define <2 x i64> @vec_v2i64(<2 x i64> %x, <2 x i64> %y) {
-; CHECK-LABEL: vec_v2i64:
-; CHECK: // %bb.0:
-; CHECK-NEXT: mov x9, v1.d[1]
-; CHECK-NEXT: mov x10, v0.d[1]
-; CHECK-NEXT: mov x8, #9223372036854775807 // =0x7fffffffffffffff
-; CHECK-NEXT: fmov x12, d0
-; CHECK-NEXT: mul x11, x10, x9
-; CHECK-NEXT: smulh x9, x10, x9
-; CHECK-NEXT: fmov x10, d1
-; CHECK-NEXT: mul x13, x12, x10
-; CHECK-NEXT: smulh x10, x12, x10
-; CHECK-NEXT: extr x11, x9, x11, #15
-; CHECK-NEXT: cmp x9, #4, lsl #12 // =16384
-; CHECK-NEXT: csel x11, x8, x11, ge
-; CHECK-NEXT: cmn x9, #4, lsl #12 // =16384
-; CHECK-NEXT: mov x9, #-9223372036854775808 // =0x8000000000000000
-; CHECK-NEXT: csel x11, x9, x11, lt
-; CHECK-NEXT: extr x12, x10, x13, #15
-; CHECK-NEXT: cmp x10, #4, lsl #12 // =16384
-; CHECK-NEXT: csel x8, x8, x12, ge
-; CHECK-NEXT: cmn x10, #4, lsl #12 // =16384
-; CHECK-NEXT: csel x8, x9, x8, lt
-; CHECK-NEXT: fmov d0, x8
-; CHECK-NEXT: mov v0.d[1], x11
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vec_v2i64:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: mov x9, v1.d[1]
+; CHECK-SD-NEXT: mov x10, v0.d[1]
+; CHECK-SD-NEXT: mov x8, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-SD-NEXT: fmov x12, d0
+; CHECK-SD-NEXT: mul x11, x10, x9
+; CHECK-SD-NEXT: smulh x9, x10, x9
+; CHECK-SD-NEXT: fmov x10, d1
+; CHECK-SD-NEXT: mul x13, x12, x10
+; CHECK-SD-NEXT: smulh x10, x12, x10
+; CHECK-SD-NEXT: extr x11, x9, x11, #15
+; CHECK-SD-NEXT: cmp x9, #4, lsl #12 // =16384
+; CHECK-SD-NEXT: csel x11, x8, x11, ge
+; CHECK-SD-NEXT: cmn x9, #4, lsl #12 // =16384
+; CHECK-SD-NEXT: mov x9, #-9223372036854775808 // =0x8000000000000000
+; CHECK-SD-NEXT: csel x11, x9, x11, lt
+; CHECK-SD-NEXT: extr x12, x10, x13, #15
+; CHECK-SD-NEXT: cmp x10, #4, lsl #12 // =16384
+; CHECK-SD-NEXT: csel x8, x8, x12, ge
+; CHECK-SD-NEXT: cmn x10, #4, lsl #12 // =16384
+; CHECK-SD-NEXT: csel x8, x9, x8, lt
+; CHECK-SD-NEXT: fmov d0, x8
+; CHECK-SD-NEXT: mov v0.d[1], x11
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vec_v2i64:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: fmov x9, d0
+; CHECK-GI-NEXT: fmov x10, d1
+; CHECK-GI-NEXT: mov x8, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-GI-NEXT: mov d0, v0.d[1]
+; CHECK-GI-NEXT: mov d1, v1.d[1]
+; CHECK-GI-NEXT: umulh x11, x9, x10
+; CHECK-GI-NEXT: asr x12, x10, #63
+; CHECK-GI-NEXT: fmov x14, d1
+; CHECK-GI-NEXT: mul x13, x9, x10
+; CHECK-GI-NEXT: madd x11, x9, x12, x11
+; CHECK-GI-NEXT: fmov x12, d0
+; CHECK-GI-NEXT: asr x9, x9, #63
+; CHECK-GI-NEXT: asr x16, x14, #63
+; CHECK-GI-NEXT: umulh x15, x12, x14
+; CHECK-GI-NEXT: madd x9, x9, x10, x11
+; CHECK-GI-NEXT: asr x11, x12, #63
+; CHECK-GI-NEXT: madd x10, x12, x16, x15
+; CHECK-GI-NEXT: mul x12, x12, x14
+; CHECK-GI-NEXT: madd x10, x11, x14, x10
+; CHECK-GI-NEXT: extr x11, x9, x13, #15
+; CHECK-GI-NEXT: asr x9, x9, #15
+; CHECK-GI-NEXT: cmp x11, x8
+; CHECK-GI-NEXT: cset w13, lo
+; CHECK-GI-NEXT: cmp x9, #0
+; CHECK-GI-NEXT: cset w14, mi
+; CHECK-GI-NEXT: extr x12, x10, x12, #15
+; CHECK-GI-NEXT: asr x10, x10, #15
+; CHECK-GI-NEXT: csel w13, w13, w14, eq
+; CHECK-GI-NEXT: cmp x12, x8
+; CHECK-GI-NEXT: cset w14, lo
+; CHECK-GI-NEXT: cmp x10, #0
+; CHECK-GI-NEXT: cset w15, mi
+; CHECK-GI-NEXT: csel w14, w14, w15, eq
+; CHECK-GI-NEXT: tst w13, #0x1
+; CHECK-GI-NEXT: mov x13, #-9223372036854775808 // =0x8000000000000000
+; CHECK-GI-NEXT: csel x11, x11, x8, ne
+; CHECK-GI-NEXT: csel x9, x9, xzr, ne
+; CHECK-GI-NEXT: tst w14, #0x1
+; CHECK-GI-NEXT: csel x8, x12, x8, ne
+; CHECK-GI-NEXT: csel x10, x10, xzr, ne
+; CHECK-GI-NEXT: cmp x11, x13
+; CHECK-GI-NEXT: cset w12, hi
+; CHECK-GI-NEXT: cmn x9, #1
+; CHECK-GI-NEXT: cset w9, gt
+; CHECK-GI-NEXT: csel w9, w12, w9, eq
+; CHECK-GI-NEXT: cmp x8, x13
+; CHECK-GI-NEXT: cset w12, hi
+; CHECK-GI-NEXT: cmn x10, #1
+; CHECK-GI-NEXT: cset w10, gt
+; CHECK-GI-NEXT: csel w10, w12, w10, eq
+; CHECK-GI-NEXT: tst w9, #0x1
+; CHECK-GI-NEXT: csel x9, x11, x13, ne
+; CHECK-GI-NEXT: tst w10, #0x1
+; CHECK-GI-NEXT: fmov d0, x9
+; CHECK-GI-NEXT: csel x8, x8, x13, ne
+; CHECK-GI-NEXT: mov v0.d[1], x8
+; CHECK-GI-NEXT: ret
%tmp = call <2 x i64> @llvm.smul.fix.sat.v2i64(<2 x i64> %x, <2 x i64> %y, i32 15)
ret <2 x i64> %tmp
}
define <4 x i64> @vec_v4i64(<4 x i64> %x, <4 x i64> %y) {
-; CHECK-LABEL: vec_v4i64:
-; CHECK: // %bb.0:
-; CHECK-NEXT: mov x8, v2.d[1]
-; CHECK-NEXT: mov x9, v0.d[1]
-; CHECK-NEXT: mov w16, #2147483647 // =0x7fffffff
-; CHECK-NEXT: fmov x10, d2
-; CHECK-NEXT: fmov x11, d0
-; CHECK-NEXT: mov x18, #9223372036854775807 // =0x7fffffffffffffff
-; CHECK-NEXT: mov x14, v3.d[1]
-; CHECK-NEXT: mov x15, v1.d[1]
-; CHECK-NEXT: mul x13, x9, x8
-; CHECK-NEXT: smulh x8, x9, x8
-; CHECK-NEXT: mul x12, x11, x10
-; CHECK-NEXT: smulh x9, x11, x10
-; CHECK-NEXT: extr x13, x8, x13, #32
-; CHECK-NEXT: cmp x8, x16
-; CHECK-NEXT: mul x10, x15, x14
-; CHECK-NEXT: csel x13, x18, x13, gt
-; CHECK-NEXT: smulh x11, x15, x14
-; CHECK-NEXT: fmov x14, d3
-; CHECK-NEXT: fmov x15, d1
-; CHECK-NEXT: extr x12, x9, x12, #32
-; CHECK-NEXT: mul x17, x15, x14
-; CHECK-NEXT: smulh x14, x15, x14
-; CHECK-NEXT: mov x15, #-2147483648 // =0xffffffff80000000
-; CHECK-NEXT: cmp x8, x15
-; CHECK-NEXT: mov x8, #-9223372036854775808 // =0x8000000000000000
-; CHECK-NEXT: csel x13, x8, x13, lt
-; CHECK-NEXT: cmp x9, x16
-; CHECK-NEXT: csel x12, x18, x12, gt
-; CHECK-NEXT: cmp x9, x15
-; CHECK-NEXT: extr x9, x11, x10, #32
-; CHECK-NEXT: csel x10, x8, x12, lt
-; CHECK-NEXT: cmp x11, x16
-; CHECK-NEXT: csel x9, x18, x9, gt
-; CHECK-NEXT: cmp x11, x15
-; CHECK-NEXT: extr x11, x14, x17, #32
-; CHECK-NEXT: csel x9, x8, x9, lt
-; CHECK-NEXT: cmp x14, x16
-; CHECK-NEXT: fmov d0, x10
-; CHECK-NEXT: csel x11, x18, x11, gt
-; CHECK-NEXT: cmp x14, x15
-; CHECK-NEXT: csel x8, x8, x11, lt
-; CHECK-NEXT: fmov d1, x8
-; CHECK-NEXT: mov v0.d[1], x13
-; CHECK-NEXT: mov v1.d[1], x9
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vec_v4i64:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: mov x8, v2.d[1]
+; CHECK-SD-NEXT: mov x9, v0.d[1]
+; CHECK-SD-NEXT: mov w16, #2147483647 // =0x7fffffff
+; CHECK-SD-NEXT: fmov x10, d2
+; CHECK-SD-NEXT: fmov x11, d0
+; CHECK-SD-NEXT: mov x18, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-SD-NEXT: mov x14, v3.d[1]
+; CHECK-SD-NEXT: mov x15, v1.d[1]
+; CHECK-SD-NEXT: mul x13, x9, x8
+; CHECK-SD-NEXT: smulh x8, x9, x8
+; CHECK-SD-NEXT: mul x12, x11, x10
+; CHECK-SD-NEXT: smulh x9, x11, x10
+; CHECK-SD-NEXT: extr x13, x8, x13, #32
+; CHECK-SD-NEXT: cmp x8, x16
+; CHECK-SD-NEXT: mul x10, x15, x14
+; CHECK-SD-NEXT: csel x13, x18, x13, gt
+; CHECK-SD-NEXT: smulh x11, x15, x14
+; CHECK-SD-NEXT: fmov x14, d3
+; CHECK-SD-NEXT: fmov x15, d1
+; CHECK-SD-NEXT: extr x12, x9, x12, #32
+; CHECK-SD-NEXT: mul x17, x15, x14
+; CHECK-SD-NEXT: smulh x14, x15, x14
+; CHECK-SD-NEXT: mov x15, #-2147483648 // =0xffffffff80000000
+; CHECK-SD-NEXT: cmp x8, x15
+; CHECK-SD-NEXT: mov x8, #-9223372036854775808 // =0x8000000000000000
+; CHECK-SD-NEXT: csel x13, x8, x13, lt
+; CHECK-SD-NEXT: cmp x9, x16
+; CHECK-SD-NEXT: csel x12, x18, x12, gt
+; CHECK-SD-NEXT: cmp x9, x15
+; CHECK-SD-NEXT: extr x9, x11, x10, #32
+; CHECK-SD-NEXT: csel x10, x8, x12, lt
+; CHECK-SD-NEXT: cmp x11, x16
+; CHECK-SD-NEXT: csel x9, x18, x9, gt
+; CHECK-SD-NEXT: cmp x11, x15
+; CHECK-SD-NEXT: extr x11, x14, x17, #32
+; CHECK-SD-NEXT: csel x9, x8, x9, lt
+; CHECK-SD-NEXT: cmp x14, x16
+; CHECK-SD-NEXT: fmov d0, x10
+; CHECK-SD-NEXT: csel x11, x18, x11, gt
+; CHECK-SD-NEXT: cmp x14, x15
+; CHECK-SD-NEXT: csel x8, x8, x11, lt
+; CHECK-SD-NEXT: fmov d1, x8
+; CHECK-SD-NEXT: mov v0.d[1], x13
+; CHECK-SD-NEXT: mov v1.d[1], x9
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vec_v4i64:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: fmov x8, d0
+; CHECK-GI-NEXT: fmov x9, d2
+; CHECK-GI-NEXT: mov d0, v0.d[1]
+; CHECK-GI-NEXT: mov d2, v2.d[1]
+; CHECK-GI-NEXT: umulh x10, x8, x9
+; CHECK-GI-NEXT: asr x11, x9, #63
+; CHECK-GI-NEXT: fmov x13, d2
+; CHECK-GI-NEXT: mul x12, x8, x9
+; CHECK-GI-NEXT: madd x10, x8, x11, x10
+; CHECK-GI-NEXT: fmov x11, d0
+; CHECK-GI-NEXT: asr x8, x8, #63
+; CHECK-GI-NEXT: asr x15, x13, #63
+; CHECK-GI-NEXT: mov d0, v1.d[1]
+; CHECK-GI-NEXT: umulh x14, x11, x13
+; CHECK-GI-NEXT: madd x8, x8, x9, x10
+; CHECK-GI-NEXT: asr x10, x11, #63
+; CHECK-GI-NEXT: madd x9, x11, x15, x14
+; CHECK-GI-NEXT: mul x11, x11, x13
+; CHECK-GI-NEXT: extr x12, x8, x12, #32
+; CHECK-GI-NEXT: asr x8, x8, #32
+; CHECK-GI-NEXT: madd x9, x10, x13, x9
+; CHECK-GI-NEXT: fmov x10, d1
+; CHECK-GI-NEXT: fmov x13, d3
+; CHECK-GI-NEXT: mov d1, v3.d[1]
+; CHECK-GI-NEXT: umulh x14, x10, x13
+; CHECK-GI-NEXT: asr x15, x13, #63
+; CHECK-GI-NEXT: fmov x17, d1
+; CHECK-GI-NEXT: mul x16, x10, x13
+; CHECK-GI-NEXT: extr x11, x9, x11, #32
+; CHECK-GI-NEXT: asr x9, x9, #32
+; CHECK-GI-NEXT: madd x14, x10, x15, x14
+; CHECK-GI-NEXT: fmov x15, d0
+; CHECK-GI-NEXT: asr x10, x10, #63
+; CHECK-GI-NEXT: asr x0, x17, #63
+; CHECK-GI-NEXT: umulh x18, x15, x17
+; CHECK-GI-NEXT: madd x10, x10, x13, x14
+; CHECK-GI-NEXT: mov x14, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-GI-NEXT: cmp x12, x14
+; CHECK-GI-NEXT: madd x13, x15, x0, x18
+; CHECK-GI-NEXT: asr x18, x15, #63
+; CHECK-GI-NEXT: cset w0, lo
+; CHECK-GI-NEXT: cmp x8, #0
+; CHECK-GI-NEXT: mul x15, x15, x17
+; CHECK-GI-NEXT: cset w1, mi
+; CHECK-GI-NEXT: extr x16, x10, x16, #32
+; CHECK-GI-NEXT: asr x10, x10, #32
+; CHECK-GI-NEXT: madd x13, x18, x17, x13
+; CHECK-GI-NEXT: csel w17, w0, w1, eq
+; CHECK-GI-NEXT: cmp x11, x14
+; CHECK-GI-NEXT: cset w18, lo
+; CHECK-GI-NEXT: cmp x9, #0
+; CHECK-GI-NEXT: cset w0, mi
+; CHECK-GI-NEXT: csel w18, w18, w0, eq
+; CHECK-GI-NEXT: cmp x16, x14
+; CHECK-GI-NEXT: cset w0, lo
+; CHECK-GI-NEXT: cmp x10, #0
+; CHECK-GI-NEXT: extr x15, x13, x15, #32
+; CHECK-GI-NEXT: asr x13, x13, #32
+; CHECK-GI-NEXT: cset w1, mi
+; CHECK-GI-NEXT: csel w0, w0, w1, eq
+; CHECK-GI-NEXT: cmp x15, x14
+; CHECK-GI-NEXT: cset w1, lo
+; CHECK-GI-NEXT: cmp x13, #0
+; CHECK-GI-NEXT: cset w2, mi
+; CHECK-GI-NEXT: csel w1, w1, w2, eq
+; CHECK-GI-NEXT: tst w17, #0x1
+; CHECK-GI-NEXT: mov x17, #-9223372036854775808 // =0x8000000000000000
+; CHECK-GI-NEXT: csel x12, x12, x14, ne
+; CHECK-GI-NEXT: csel x8, x8, xzr, ne
+; CHECK-GI-NEXT: tst w18, #0x1
+; CHECK-GI-NEXT: csel x11, x11, x14, ne
+; CHECK-GI-NEXT: csel x9, x9, xzr, ne
+; CHECK-GI-NEXT: tst w0, #0x1
+; CHECK-GI-NEXT: csel x16, x16, x14, ne
+; CHECK-GI-NEXT: csel x10, x10, xzr, ne
+; CHECK-GI-NEXT: tst w1, #0x1
+; CHECK-GI-NEXT: csel x14, x15, x14, ne
+; CHECK-GI-NEXT: csel x13, x13, xzr, ne
+; CHECK-GI-NEXT: cmp x12, x17
+; CHECK-GI-NEXT: cset w15, hi
+; CHECK-GI-NEXT: cmn x8, #1
+; CHECK-GI-NEXT: cset w8, gt
+; CHECK-GI-NEXT: csel w8, w15, w8, eq
+; CHECK-GI-NEXT: cmp x11, x17
+; CHECK-GI-NEXT: cset w15, hi
+; CHECK-GI-NEXT: cmn x9, #1
+; CHECK-GI-NEXT: cset w9, gt
+; CHECK-GI-NEXT: csel w9, w15, w9, eq
+; CHECK-GI-NEXT: cmp x16, x17
+; CHECK-GI-NEXT: cset w15, hi
+; CHECK-GI-NEXT: cmn x10, #1
+; CHECK-GI-NEXT: cset w10, gt
+; CHECK-GI-NEXT: csel w10, w15, w10, eq
+; CHECK-GI-NEXT: cmp x14, x17
+; CHECK-GI-NEXT: cset w15, hi
+; CHECK-GI-NEXT: cmn x13, #1
+; CHECK-GI-NEXT: cset w13, gt
+; CHECK-GI-NEXT: csel w13, w15, w13, eq
+; CHECK-GI-NEXT: tst w8, #0x1
+; CHECK-GI-NEXT: csel x8, x12, x17, ne
+; CHECK-GI-NEXT: tst w9, #0x1
+; CHECK-GI-NEXT: csel x9, x11, x17, ne
+; CHECK-GI-NEXT: tst w10, #0x1
+; CHECK-GI-NEXT: fmov d0, x8
+; CHECK-GI-NEXT: csel x10, x16, x17, ne
+; CHECK-GI-NEXT: tst w13, #0x1
+; CHECK-GI-NEXT: fmov d1, x10
+; CHECK-GI-NEXT: csel x11, x14, x17, ne
+; CHECK-GI-NEXT: mov v0.d[1], x9
+; CHECK-GI-NEXT: mov v1.d[1], x11
+; CHECK-GI-NEXT: ret
%tmp = call <4 x i64> @llvm.smul.fix.sat.v4i64(<4 x i64> %x, <4 x i64> %y, i32 32)
ret <4 x i64> %tmp
}
define <8 x i16> @vec_sqdmulh_v8i16(<8 x i16> %x, <8 x i16> %y) {
-; CHECK-LABEL: vec_sqdmulh_v8i16:
-; CHECK: // %bb.0:
-; CHECK-NEXT: sqdmulh v0.8h, v0.8h, v1.8h
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vec_sqdmulh_v8i16:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: sqdmulh v0.8h, v0.8h, v1.8h
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vec_sqdmulh_v8i16:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: smull v2.4s, v0.4h, v1.4h
+; CHECK-GI-NEXT: smull2 v1.4s, v0.8h, v1.8h
+; CHECK-GI-NEXT: sqshrn v0.4h, v2.4s, #15
+; CHECK-GI-NEXT: sqshrn2 v0.8h, v1.4s, #15
+; CHECK-GI-NEXT: ret
%tmp = call <8 x i16> @llvm.smul.fix.sat.v8i16(<8 x i16> %x, <8 x i16> %y, i32 15)
ret <8 x i16> %tmp
}
define <4 x i16> @vec_sqdmulh_v4i16(<4 x i16> %x, <4 x i16> %y) {
-; CHECK-LABEL: vec_sqdmulh_v4i16:
-; CHECK: // %bb.0:
-; CHECK-NEXT: sqdmulh v0.4h, v0.4h, v1.4h
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vec_sqdmulh_v4i16:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: sqdmulh v0.4h, v0.4h, v1.4h
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vec_sqdmulh_v4i16:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: smull v0.4s, v0.4h, v1.4h
+; CHECK-GI-NEXT: sqshrn v0.4h, v0.4s, #15
+; CHECK-GI-NEXT: ret
%tmp = call <4 x i16> @llvm.smul.fix.sat.v4i16(<4 x i16> %x, <4 x i16> %y, i32 15)
ret <4 x i16> %tmp
}
define <4 x i32> @vec_sqdmulh_v4i32(<4 x i32> %x, <4 x i32> %y) {
-; CHECK-LABEL: vec_sqdmulh_v4i32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: sqdmulh v0.4s, v0.4s, v1.4s
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vec_sqdmulh_v4i32:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: sqdmulh v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vec_sqdmulh_v4i32:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: smull v2.2d, v0.2s, v1.2s
+; CHECK-GI-NEXT: smull2 v1.2d, v0.4s, v1.4s
+; CHECK-GI-NEXT: sqshrn v0.2s, v2.2d, #31
+; CHECK-GI-NEXT: sqshrn2 v0.4s, v1.2d, #31
+; CHECK-GI-NEXT: ret
%tmp = call <4 x i32> @llvm.smul.fix.sat.v4i32(<4 x i32> %x, <4 x i32> %y, i32 31)
ret <4 x i32> %tmp
}
define <2 x i32> @vec_sqdmulh_v2i32(<2 x i32> %x, <2 x i32> %y) {
-; CHECK-LABEL: vec_sqdmulh_v2i32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: sqdmulh v0.2s, v0.2s, v1.2s
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vec_sqdmulh_v2i32:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: sqdmulh v0.2s, v0.2s, v1.2s
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vec_sqdmulh_v2i32:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: smull v0.2d, v0.2s, v1.2s
+; CHECK-GI-NEXT: sqshrn v0.2s, v0.2d, #31
+; CHECK-GI-NEXT: ret
%tmp = call <2 x i32> @llvm.smul.fix.sat.v2i32(<2 x i32> %x, <2 x i32> %y, i32 31)
ret <2 x i32> %tmp
}
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; CHECK-GI: {{.*}}
-; CHECK-SD: {{.*}}
+; CHECK: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/umul_fix_sat.ll b/llvm/test/CodeGen/AArch64/umul_fix_sat.ll
index 18a94a3c38711..f7ffb9ec61abf 100644
--- a/llvm/test/CodeGen/AArch64/umul_fix_sat.ll
+++ b/llvm/test/CodeGen/AArch64/umul_fix_sat.ll
@@ -2,324 +2,546 @@
; RUN: llc < %s -mtriple=aarch64-linux-gnu -global-isel=0 | FileCheck %s --check-prefixes=CHECK,CHECK-SD
; RUN: llc < %s -mtriple=aarch64-linux-gnu -global-isel=1 -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
-; CHECK-GI: warning: Instruction selection used fallback path for func
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for func2
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for func3
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for func4
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for func5
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for func6
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for func7
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for func8
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vec_v8i8
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vec_v16i8
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vec_v4i16
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vec_v8i16
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vec_v2i32
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vec_v4i32
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vec_v8i32
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vec_v2i64
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vec_v4i64
define i32 @func(i32 %x, i32 %y) {
-; CHECK-LABEL: func:
-; CHECK: // %bb.0:
-; CHECK-NEXT: umull x8, w0, w1
-; CHECK-NEXT: lsr x9, x8, #32
-; CHECK-NEXT: extr w8, w9, w8, #2
-; CHECK-NEXT: cmp w9, #3
-; CHECK-NEXT: csinv w0, w8, wzr, ls
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: func:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: umull x8, w0, w1
+; CHECK-SD-NEXT: lsr x9, x8, #32
+; CHECK-SD-NEXT: extr w8, w9, w8, #2
+; CHECK-SD-NEXT: cmp w9, #3
+; CHECK-SD-NEXT: csinv w0, w8, wzr, ls
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: func:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: umull x8, w0, w1
+; CHECK-GI-NEXT: mov w9, #-1 // =0xffffffff
+; CHECK-GI-NEXT: lsr x8, x8, #2
+; CHECK-GI-NEXT: cmp x8, x9
+; CHECK-GI-NEXT: csel x0, x8, x9, lo
+; CHECK-GI-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-GI-NEXT: ret
%tmp = call i32 @llvm.umul.fix.sat.i32(i32 %x, i32 %y, i32 2)
ret i32 %tmp
}
define i64 @func2(i64 %x, i64 %y) {
-; CHECK-LABEL: func2:
-; CHECK: // %bb.0:
-; CHECK-NEXT: mul x8, x0, x1
-; CHECK-NEXT: umulh x9, x0, x1
-; CHECK-NEXT: extr x8, x9, x8, #2
-; CHECK-NEXT: cmp x9, #3
-; CHECK-NEXT: csinv x0, x8, xzr, ls
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: func2:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: mul x8, x0, x1
+; CHECK-SD-NEXT: umulh x9, x0, x1
+; CHECK-SD-NEXT: extr x8, x9, x8, #2
+; CHECK-SD-NEXT: cmp x9, #3
+; CHECK-SD-NEXT: csinv x0, x8, xzr, ls
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: func2:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mul x8, x0, x1
+; CHECK-GI-NEXT: umulh x9, x0, x1
+; CHECK-GI-NEXT: extr x8, x9, x8, #2
+; CHECK-GI-NEXT: lsr x9, x9, #2
+; CHECK-GI-NEXT: cmn x8, #1
+; CHECK-GI-NEXT: cset w10, lo
+; CHECK-GI-NEXT: cmp x9, #0
+; CHECK-GI-NEXT: csel w9, w10, wzr, eq
+; CHECK-GI-NEXT: tst w9, #0x1
+; CHECK-GI-NEXT: csinv x0, x8, xzr, ne
+; CHECK-GI-NEXT: ret
%tmp = call i64 @llvm.umul.fix.sat.i64(i64 %x, i64 %y, i32 2)
ret i64 %tmp
}
define i4 @func3(i4 %x, i4 %y) {
-; CHECK-LABEL: func3:
-; CHECK: // %bb.0:
-; CHECK-NEXT: lsl w8, w0, #28
-; CHECK-NEXT: and w9, w1, #0xf
-; CHECK-NEXT: umull x8, w8, w9
-; CHECK-NEXT: lsr x9, x8, #32
-; CHECK-NEXT: extr w8, w9, w8, #2
-; CHECK-NEXT: cmp w9, #3
-; CHECK-NEXT: csinv w8, w8, wzr, ls
-; CHECK-NEXT: lsr w0, w8, #28
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: func3:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: lsl w8, w0, #28
+; CHECK-SD-NEXT: and w9, w1, #0xf
+; CHECK-SD-NEXT: umull x8, w8, w9
+; CHECK-SD-NEXT: lsr x9, x8, #32
+; CHECK-SD-NEXT: extr w8, w9, w8, #2
+; CHECK-SD-NEXT: cmp w9, #3
+; CHECK-SD-NEXT: csinv w8, w8, wzr, ls
+; CHECK-SD-NEXT: lsr w0, w8, #28
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: func3:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: and w9, w0, #0xf
+; CHECK-GI-NEXT: and w10, w1, #0xf
+; CHECK-GI-NEXT: mov w8, #15 // =0xf
+; CHECK-GI-NEXT: mul w9, w9, w10
+; CHECK-GI-NEXT: lsr w9, w9, #2
+; CHECK-GI-NEXT: cmp w9, #15
+; CHECK-GI-NEXT: csel w0, w9, w8, lo
+; CHECK-GI-NEXT: ret
%tmp = call i4 @llvm.umul.fix.sat.i4(i4 %x, i4 %y, i32 2)
ret i4 %tmp
}
;; These result in regular integer multiplication with a saturation check.
define i32 @func4(i32 %x, i32 %y) {
-; CHECK-LABEL: func4:
-; CHECK: // %bb.0:
-; CHECK-NEXT: umull x8, w0, w1
-; CHECK-NEXT: tst x8, #0xffffffff00000000
-; CHECK-NEXT: csinv w0, w8, wzr, eq
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: func4:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: umull x8, w0, w1
+; CHECK-SD-NEXT: tst x8, #0xffffffff00000000
+; CHECK-SD-NEXT: csinv w0, w8, wzr, eq
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: func4:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: umull x8, w0, w1
+; CHECK-GI-NEXT: mov w9, #-1 // =0xffffffff
+; CHECK-GI-NEXT: cmp x8, x9
+; CHECK-GI-NEXT: csel x0, x8, x9, lo
+; CHECK-GI-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-GI-NEXT: ret
%tmp = call i32 @llvm.umul.fix.sat.i32(i32 %x, i32 %y, i32 0)
ret i32 %tmp
}
define i64 @func5(i64 %x, i64 %y) {
-; CHECK-LABEL: func5:
-; CHECK: // %bb.0:
-; CHECK-NEXT: umulh x8, x0, x1
-; CHECK-NEXT: mul x9, x0, x1
-; CHECK-NEXT: cmp xzr, x8
-; CHECK-NEXT: csinv x0, x9, xzr, eq
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: func5:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: umulh x8, x0, x1
+; CHECK-SD-NEXT: mul x9, x0, x1
+; CHECK-SD-NEXT: cmp xzr, x8
+; CHECK-SD-NEXT: csinv x0, x9, xzr, eq
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: func5:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mul x8, x0, x1
+; CHECK-GI-NEXT: umulh x9, x0, x1
+; CHECK-GI-NEXT: cmn x8, #1
+; CHECK-GI-NEXT: cset w10, lo
+; CHECK-GI-NEXT: cmp x9, #0
+; CHECK-GI-NEXT: csel w9, w10, wzr, eq
+; CHECK-GI-NEXT: tst w9, #0x1
+; CHECK-GI-NEXT: csinv x0, x8, xzr, ne
+; CHECK-GI-NEXT: ret
%tmp = call i64 @llvm.umul.fix.sat.i64(i64 %x, i64 %y, i32 0)
ret i64 %tmp
}
define i4 @func6(i4 %x, i4 %y) {
-; CHECK-LABEL: func6:
-; CHECK: // %bb.0:
-; CHECK-NEXT: lsl w8, w0, #28
-; CHECK-NEXT: and w9, w1, #0xf
-; CHECK-NEXT: umull x8, w8, w9
-; CHECK-NEXT: tst x8, #0xffffffff00000000
-; CHECK-NEXT: csinv w8, w8, wzr, eq
-; CHECK-NEXT: lsr w0, w8, #28
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: func6:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: lsl w8, w0, #28
+; CHECK-SD-NEXT: and w9, w1, #0xf
+; CHECK-SD-NEXT: umull x8, w8, w9
+; CHECK-SD-NEXT: tst x8, #0xffffffff00000000
+; CHECK-SD-NEXT: csinv w8, w8, wzr, eq
+; CHECK-SD-NEXT: lsr w0, w8, #28
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: func6:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: and w9, w0, #0xf
+; CHECK-GI-NEXT: and w10, w1, #0xf
+; CHECK-GI-NEXT: mov w8, #15 // =0xf
+; CHECK-GI-NEXT: mul w9, w9, w10
+; CHECK-GI-NEXT: cmp w9, #15
+; CHECK-GI-NEXT: csel w0, w9, w8, lo
+; CHECK-GI-NEXT: ret
%tmp = call i4 @llvm.umul.fix.sat.i4(i4 %x, i4 %y, i32 0)
ret i4 %tmp
}
define i64 @func7(i64 %x, i64 %y) {
-; CHECK-LABEL: func7:
-; CHECK: // %bb.0:
-; CHECK-NEXT: mul x9, x0, x1
-; CHECK-NEXT: mov w8, #-1 // =0xffffffff
-; CHECK-NEXT: umulh x10, x0, x1
-; CHECK-NEXT: extr x9, x10, x9, #32
-; CHECK-NEXT: cmp x10, x8
-; CHECK-NEXT: csinv x0, x9, xzr, ls
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: func7:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: mul x9, x0, x1
+; CHECK-SD-NEXT: mov w8, #-1 // =0xffffffff
+; CHECK-SD-NEXT: umulh x10, x0, x1
+; CHECK-SD-NEXT: extr x9, x10, x9, #32
+; CHECK-SD-NEXT: cmp x10, x8
+; CHECK-SD-NEXT: csinv x0, x9, xzr, ls
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: func7:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mul x8, x0, x1
+; CHECK-GI-NEXT: umulh x9, x0, x1
+; CHECK-GI-NEXT: extr x8, x9, x8, #32
+; CHECK-GI-NEXT: lsr x9, x9, #32
+; CHECK-GI-NEXT: cmn x8, #1
+; CHECK-GI-NEXT: cset w10, lo
+; CHECK-GI-NEXT: cmp x9, #0
+; CHECK-GI-NEXT: csel w9, w10, wzr, eq
+; CHECK-GI-NEXT: tst w9, #0x1
+; CHECK-GI-NEXT: csinv x0, x8, xzr, ne
+; CHECK-GI-NEXT: ret
%tmp = call i64 @llvm.umul.fix.sat.i64(i64 %x, i64 %y, i32 32)
ret i64 %tmp
}
define i64 @func8(i64 %x, i64 %y) {
-; CHECK-LABEL: func8:
-; CHECK: // %bb.0:
-; CHECK-NEXT: mul x9, x0, x1
-; CHECK-NEXT: mov x8, #9223372036854775807 // =0x7fffffffffffffff
-; CHECK-NEXT: umulh x10, x0, x1
-; CHECK-NEXT: extr x9, x10, x9, #63
-; CHECK-NEXT: cmp x10, x8
-; CHECK-NEXT: csinv x0, x9, xzr, ls
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: func8:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: mul x9, x0, x1
+; CHECK-SD-NEXT: mov x8, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-SD-NEXT: umulh x10, x0, x1
+; CHECK-SD-NEXT: extr x9, x10, x9, #63
+; CHECK-SD-NEXT: cmp x10, x8
+; CHECK-SD-NEXT: csinv x0, x9, xzr, ls
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: func8:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mul x8, x0, x1
+; CHECK-GI-NEXT: umulh x9, x0, x1
+; CHECK-GI-NEXT: extr x8, x9, x8, #63
+; CHECK-GI-NEXT: lsr x9, x9, #63
+; CHECK-GI-NEXT: cmn x8, #1
+; CHECK-GI-NEXT: cset w10, lo
+; CHECK-GI-NEXT: cmp x9, #0
+; CHECK-GI-NEXT: csel w9, w10, wzr, eq
+; CHECK-GI-NEXT: tst w9, #0x1
+; CHECK-GI-NEXT: csinv x0, x8, xzr, ne
+; CHECK-GI-NEXT: ret
%tmp = call i64 @llvm.umul.fix.sat.i64(i64 %x, i64 %y, i32 63)
ret i64 %tmp
}
define <8 x i8> @vec_v8i8(<8 x i8> %x, <8 x i8> %y) {
-; CHECK-LABEL: vec_v8i8:
-; CHECK: // %bb.0:
-; CHECK-NEXT: umull v0.8h, v0.8b, v1.8b
-; CHECK-NEXT: movi v2.8b, #3
-; CHECK-NEXT: shrn v1.8b, v0.8h, #8
-; CHECK-NEXT: xtn v0.8b, v0.8h
-; CHECK-NEXT: shl v3.8b, v1.8b, #6
-; CHECK-NEXT: usra v3.8b, v0.8b, #2
-; CHECK-NEXT: cmhi v0.8b, v1.8b, v2.8b
-; CHECK-NEXT: orr v0.8b, v3.8b, v0.8b
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vec_v8i8:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: umull v0.8h, v0.8b, v1.8b
+; CHECK-SD-NEXT: movi v2.8b, #3
+; CHECK-SD-NEXT: shrn v1.8b, v0.8h, #8
+; CHECK-SD-NEXT: xtn v0.8b, v0.8h
+; CHECK-SD-NEXT: shl v3.8b, v1.8b, #6
+; CHECK-SD-NEXT: usra v3.8b, v0.8b, #2
+; CHECK-SD-NEXT: cmhi v0.8b, v1.8b, v2.8b
+; CHECK-SD-NEXT: orr v0.8b, v3.8b, v0.8b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vec_v8i8:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: umull v0.8h, v0.8b, v1.8b
+; CHECK-GI-NEXT: uqshrn v0.8b, v0.8h, #2
+; CHECK-GI-NEXT: ret
%tmp = call <8 x i8> @llvm.umul.fix.sat.v8i8(<8 x i8> %x, <8 x i8> %y, i32 2)
ret <8 x i8> %tmp
}
define <16 x i8> @vec_v16i8(<16 x i8> %x, <16 x i8> %y) {
-; CHECK-LABEL: vec_v16i8:
-; CHECK: // %bb.0:
-; CHECK-NEXT: umull2 v3.8h, v0.16b, v1.16b
-; CHECK-NEXT: umull v4.8h, v0.8b, v1.8b
-; CHECK-NEXT: movi v2.16b, #3
-; CHECK-NEXT: mul v0.16b, v0.16b, v1.16b
-; CHECK-NEXT: uzp2 v3.16b, v4.16b, v3.16b
-; CHECK-NEXT: shl v1.16b, v3.16b, #6
-; CHECK-NEXT: cmhi v2.16b, v3.16b, v2.16b
-; CHECK-NEXT: usra v1.16b, v0.16b, #2
-; CHECK-NEXT: orr v0.16b, v1.16b, v2.16b
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vec_v16i8:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: umull2 v3.8h, v0.16b, v1.16b
+; CHECK-SD-NEXT: umull v4.8h, v0.8b, v1.8b
+; CHECK-SD-NEXT: movi v2.16b, #3
+; CHECK-SD-NEXT: mul v0.16b, v0.16b, v1.16b
+; CHECK-SD-NEXT: uzp2 v3.16b, v4.16b, v3.16b
+; CHECK-SD-NEXT: shl v1.16b, v3.16b, #6
+; CHECK-SD-NEXT: cmhi v2.16b, v3.16b, v2.16b
+; CHECK-SD-NEXT: usra v1.16b, v0.16b, #2
+; CHECK-SD-NEXT: orr v0.16b, v1.16b, v2.16b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vec_v16i8:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: umull v2.8h, v0.8b, v1.8b
+; CHECK-GI-NEXT: umull2 v1.8h, v0.16b, v1.16b
+; CHECK-GI-NEXT: uqshrn v0.8b, v2.8h, #2
+; CHECK-GI-NEXT: uqshrn2 v0.16b, v1.8h, #2
+; CHECK-GI-NEXT: ret
%tmp = call <16 x i8> @llvm.umul.fix.sat.v16i8(<16 x i8> %x, <16 x i8> %y, i32 2)
ret <16 x i8> %tmp
}
define <4 x i16> @vec_v4i16(<4 x i16> %x, <4 x i16> %y) {
-; CHECK-LABEL: vec_v4i16:
-; CHECK: // %bb.0:
-; CHECK-NEXT: umull v0.4s, v0.4h, v1.4h
-; CHECK-NEXT: mvni v2.4h, #252, lsl #8
-; CHECK-NEXT: shrn v1.4h, v0.4s, #16
-; CHECK-NEXT: xtn v0.4h, v0.4s
-; CHECK-NEXT: shl v3.4h, v1.4h, #6
-; CHECK-NEXT: usra v3.4h, v0.4h, #10
-; CHECK-NEXT: cmhi v0.4h, v1.4h, v2.4h
-; CHECK-NEXT: orr v0.8b, v3.8b, v0.8b
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vec_v4i16:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: umull v0.4s, v0.4h, v1.4h
+; CHECK-SD-NEXT: mvni v2.4h, #252, lsl #8
+; CHECK-SD-NEXT: shrn v1.4h, v0.4s, #16
+; CHECK-SD-NEXT: xtn v0.4h, v0.4s
+; CHECK-SD-NEXT: shl v3.4h, v1.4h, #6
+; CHECK-SD-NEXT: usra v3.4h, v0.4h, #10
+; CHECK-SD-NEXT: cmhi v0.4h, v1.4h, v2.4h
+; CHECK-SD-NEXT: orr v0.8b, v3.8b, v0.8b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vec_v4i16:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: umull v0.4s, v0.4h, v1.4h
+; CHECK-GI-NEXT: uqshrn v0.4h, v0.4s, #10
+; CHECK-GI-NEXT: ret
%tmp = call <4 x i16> @llvm.umul.fix.sat.v4i16(<4 x i16> %x, <4 x i16> %y, i32 10)
ret <4 x i16> %tmp
}
define <8 x i16> @vec_v8i16(<8 x i16> %x, <8 x i16> %y) {
-; CHECK-LABEL: vec_v8i16:
-; CHECK: // %bb.0:
-; CHECK-NEXT: umull2 v3.4s, v0.8h, v1.8h
-; CHECK-NEXT: umull v4.4s, v0.4h, v1.4h
-; CHECK-NEXT: movi v2.8h, #3
-; CHECK-NEXT: mul v0.8h, v0.8h, v1.8h
-; CHECK-NEXT: uzp2 v3.8h, v4.8h, v3.8h
-; CHECK-NEXT: shl v1.8h, v3.8h, #14
-; CHECK-NEXT: cmhi v2.8h, v3.8h, v2.8h
-; CHECK-NEXT: usra v1.8h, v0.8h, #2
-; CHECK-NEXT: orr v0.16b, v1.16b, v2.16b
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vec_v8i16:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: umull2 v3.4s, v0.8h, v1.8h
+; CHECK-SD-NEXT: umull v4.4s, v0.4h, v1.4h
+; CHECK-SD-NEXT: movi v2.8h, #3
+; CHECK-SD-NEXT: mul v0.8h, v0.8h, v1.8h
+; CHECK-SD-NEXT: uzp2 v3.8h, v4.8h, v3.8h
+; CHECK-SD-NEXT: shl v1.8h, v3.8h, #14
+; CHECK-SD-NEXT: cmhi v2.8h, v3.8h, v2.8h
+; CHECK-SD-NEXT: usra v1.8h, v0.8h, #2
+; CHECK-SD-NEXT: orr v0.16b, v1.16b, v2.16b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vec_v8i16:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: umull v2.4s, v0.4h, v1.4h
+; CHECK-GI-NEXT: umull2 v1.4s, v0.8h, v1.8h
+; CHECK-GI-NEXT: uqshrn v0.4h, v2.4s, #2
+; CHECK-GI-NEXT: uqshrn2 v0.8h, v1.4s, #2
+; CHECK-GI-NEXT: ret
%tmp = call <8 x i16> @llvm.umul.fix.sat.v8i16(<8 x i16> %x, <8 x i16> %y, i32 2)
ret <8 x i16> %tmp
}
define <2 x i32> @vec_v2i32(<2 x i32> %x, <2 x i32> %y) {
-; CHECK-LABEL: vec_v2i32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: umull v0.2d, v0.2s, v1.2s
-; CHECK-NEXT: movi v2.2d, #0000000000000000
-; CHECK-NEXT: shrn v1.2s, v0.2d, #32
-; CHECK-NEXT: xtn v0.2s, v0.2d
-; CHECK-NEXT: add v3.2s, v1.2s, v1.2s
-; CHECK-NEXT: cmhi v1.2s, v1.2s, v2.2s
-; CHECK-NEXT: shl v2.2s, v3.2s, #31
-; CHECK-NEXT: orr v0.8b, v0.8b, v1.8b
-; CHECK-NEXT: orr v0.8b, v2.8b, v0.8b
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vec_v2i32:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: umull v0.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT: movi v2.2d, #0000000000000000
+; CHECK-SD-NEXT: shrn v1.2s, v0.2d, #32
+; CHECK-SD-NEXT: xtn v0.2s, v0.2d
+; CHECK-SD-NEXT: add v3.2s, v1.2s, v1.2s
+; CHECK-SD-NEXT: cmhi v1.2s, v1.2s, v2.2s
+; CHECK-SD-NEXT: shl v2.2s, v3.2s, #31
+; CHECK-SD-NEXT: orr v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT: orr v0.8b, v2.8b, v0.8b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vec_v2i32:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: umull v0.2d, v0.2s, v1.2s
+; CHECK-GI-NEXT: uqxtn v0.2s, v0.2d
+; CHECK-GI-NEXT: ret
%tmp = call <2 x i32> @llvm.umul.fix.sat.v2i32(<2 x i32> %x, <2 x i32> %y, i32 0)
ret <2 x i32> %tmp
}
define <4 x i32> @vec_v4i32(<4 x i32> %x, <4 x i32> %y) {
-; CHECK-LABEL: vec_v4i32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: umull2 v3.2d, v0.4s, v1.4s
-; CHECK-NEXT: umull v4.2d, v0.2s, v1.2s
-; CHECK-NEXT: movi v2.4s, #127, msl #8
-; CHECK-NEXT: mul v0.4s, v0.4s, v1.4s
-; CHECK-NEXT: uzp2 v3.4s, v4.4s, v3.4s
-; CHECK-NEXT: shl v1.4s, v3.4s, #17
-; CHECK-NEXT: cmhi v2.4s, v3.4s, v2.4s
-; CHECK-NEXT: usra v1.4s, v0.4s, #15
-; CHECK-NEXT: orr v0.16b, v1.16b, v2.16b
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vec_v4i32:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: umull2 v3.2d, v0.4s, v1.4s
+; CHECK-SD-NEXT: umull v4.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT: movi v2.4s, #127, msl #8
+; CHECK-SD-NEXT: mul v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT: uzp2 v3.4s, v4.4s, v3.4s
+; CHECK-SD-NEXT: shl v1.4s, v3.4s, #17
+; CHECK-SD-NEXT: cmhi v2.4s, v3.4s, v2.4s
+; CHECK-SD-NEXT: usra v1.4s, v0.4s, #15
+; CHECK-SD-NEXT: orr v0.16b, v1.16b, v2.16b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vec_v4i32:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: umull v2.2d, v0.2s, v1.2s
+; CHECK-GI-NEXT: umull2 v1.2d, v0.4s, v1.4s
+; CHECK-GI-NEXT: uqshrn v0.2s, v2.2d, #15
+; CHECK-GI-NEXT: uqshrn2 v0.4s, v1.2d, #15
+; CHECK-GI-NEXT: ret
%tmp = call <4 x i32> @llvm.umul.fix.sat.v4i32(<4 x i32> %x, <4 x i32> %y, i32 15)
ret <4 x i32> %tmp
}
define <8 x i32> @vec_v8i32(<8 x i32> %x, <8 x i32> %y) {
-; CHECK-LABEL: vec_v8i32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: umull2 v4.2d, v0.4s, v2.4s
-; CHECK-NEXT: umull v5.2d, v0.2s, v2.2s
-; CHECK-NEXT: umull2 v6.2d, v1.4s, v3.4s
-; CHECK-NEXT: umull v7.2d, v1.2s, v3.2s
-; CHECK-NEXT: mul v0.4s, v0.4s, v2.4s
-; CHECK-NEXT: mul v1.4s, v1.4s, v3.4s
-; CHECK-NEXT: uzp2 v4.4s, v5.4s, v4.4s
-; CHECK-NEXT: uzp2 v5.4s, v7.4s, v6.4s
-; CHECK-NEXT: movi v6.2d, #0000000000000000
-; CHECK-NEXT: add v2.4s, v4.4s, v4.4s
-; CHECK-NEXT: cmhi v3.4s, v4.4s, v6.4s
-; CHECK-NEXT: cmhi v4.4s, v5.4s, v6.4s
-; CHECK-NEXT: add v5.4s, v5.4s, v5.4s
-; CHECK-NEXT: shl v2.4s, v2.4s, #31
-; CHECK-NEXT: orr v0.16b, v0.16b, v3.16b
-; CHECK-NEXT: shl v3.4s, v5.4s, #31
-; CHECK-NEXT: orr v1.16b, v1.16b, v4.16b
-; CHECK-NEXT: orr v0.16b, v2.16b, v0.16b
-; CHECK-NEXT: orr v1.16b, v3.16b, v1.16b
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vec_v8i32:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: umull2 v4.2d, v0.4s, v2.4s
+; CHECK-SD-NEXT: umull v5.2d, v0.2s, v2.2s
+; CHECK-SD-NEXT: umull2 v6.2d, v1.4s, v3.4s
+; CHECK-SD-NEXT: umull v7.2d, v1.2s, v3.2s
+; CHECK-SD-NEXT: mul v0.4s, v0.4s, v2.4s
+; CHECK-SD-NEXT: mul v1.4s, v1.4s, v3.4s
+; CHECK-SD-NEXT: uzp2 v4.4s, v5.4s, v4.4s
+; CHECK-SD-NEXT: uzp2 v5.4s, v7.4s, v6.4s
+; CHECK-SD-NEXT: movi v6.2d, #0000000000000000
+; CHECK-SD-NEXT: add v2.4s, v4.4s, v4.4s
+; CHECK-SD-NEXT: cmhi v3.4s, v4.4s, v6.4s
+; CHECK-SD-NEXT: cmhi v4.4s, v5.4s, v6.4s
+; CHECK-SD-NEXT: add v5.4s, v5.4s, v5.4s
+; CHECK-SD-NEXT: shl v2.4s, v2.4s, #31
+; CHECK-SD-NEXT: orr v0.16b, v0.16b, v3.16b
+; CHECK-SD-NEXT: shl v3.4s, v5.4s, #31
+; CHECK-SD-NEXT: orr v1.16b, v1.16b, v4.16b
+; CHECK-SD-NEXT: orr v0.16b, v2.16b, v0.16b
+; CHECK-SD-NEXT: orr v1.16b, v3.16b, v1.16b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vec_v8i32:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: umull v4.2d, v0.2s, v2.2s
+; CHECK-GI-NEXT: umull v5.2d, v1.2s, v3.2s
+; CHECK-GI-NEXT: umull2 v2.2d, v0.4s, v2.4s
+; CHECK-GI-NEXT: umull2 v3.2d, v1.4s, v3.4s
+; CHECK-GI-NEXT: uqxtn v0.2s, v4.2d
+; CHECK-GI-NEXT: uqxtn v1.2s, v5.2d
+; CHECK-GI-NEXT: uqxtn2 v0.4s, v2.2d
+; CHECK-GI-NEXT: uqxtn2 v1.4s, v3.2d
+; CHECK-GI-NEXT: ret
%tmp = call <8 x i32> @llvm.umul.fix.sat.v8i32(<8 x i32> %x, <8 x i32> %y, i32 0)
ret <8 x i32> %tmp
}
define <2 x i64> @vec_v2i64(<2 x i64> %x, <2 x i64> %y) {
-; CHECK-LABEL: vec_v2i64:
-; CHECK: // %bb.0:
-; CHECK-NEXT: mov x8, v1.d[1]
-; CHECK-NEXT: mov x9, v0.d[1]
-; CHECK-NEXT: fmov x11, d0
-; CHECK-NEXT: mul x10, x9, x8
-; CHECK-NEXT: umulh x8, x9, x8
-; CHECK-NEXT: fmov x9, d1
-; CHECK-NEXT: mul x12, x11, x9
-; CHECK-NEXT: umulh x9, x11, x9
-; CHECK-NEXT: extr x10, x8, x10, #15
-; CHECK-NEXT: cmp x8, #8, lsl #12 // =32768
-; CHECK-NEXT: csinv x10, x10, xzr, lo
-; CHECK-NEXT: extr x8, x9, x12, #15
-; CHECK-NEXT: cmp x9, #8, lsl #12 // =32768
-; CHECK-NEXT: csinv x8, x8, xzr, lo
-; CHECK-NEXT: fmov d0, x8
-; CHECK-NEXT: mov v0.d[1], x10
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vec_v2i64:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: mov x8, v1.d[1]
+; CHECK-SD-NEXT: mov x9, v0.d[1]
+; CHECK-SD-NEXT: fmov x11, d0
+; CHECK-SD-NEXT: mul x10, x9, x8
+; CHECK-SD-NEXT: umulh x8, x9, x8
+; CHECK-SD-NEXT: fmov x9, d1
+; CHECK-SD-NEXT: mul x12, x11, x9
+; CHECK-SD-NEXT: umulh x9, x11, x9
+; CHECK-SD-NEXT: extr x10, x8, x10, #15
+; CHECK-SD-NEXT: cmp x8, #8, lsl #12 // =32768
+; CHECK-SD-NEXT: csinv x10, x10, xzr, lo
+; CHECK-SD-NEXT: extr x8, x9, x12, #15
+; CHECK-SD-NEXT: cmp x9, #8, lsl #12 // =32768
+; CHECK-SD-NEXT: csinv x8, x8, xzr, lo
+; CHECK-SD-NEXT: fmov d0, x8
+; CHECK-SD-NEXT: mov v0.d[1], x10
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vec_v2i64:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mov d2, v0.d[1]
+; CHECK-GI-NEXT: mov d3, v1.d[1]
+; CHECK-GI-NEXT: fmov x8, d0
+; CHECK-GI-NEXT: fmov x9, d1
+; CHECK-GI-NEXT: mul x10, x8, x9
+; CHECK-GI-NEXT: fmov x11, d3
+; CHECK-GI-NEXT: umulh x8, x8, x9
+; CHECK-GI-NEXT: fmov x9, d2
+; CHECK-GI-NEXT: mul x12, x9, x11
+; CHECK-GI-NEXT: umulh x9, x9, x11
+; CHECK-GI-NEXT: extr x10, x8, x10, #15
+; CHECK-GI-NEXT: lsr x8, x8, #15
+; CHECK-GI-NEXT: cmn x10, #1
+; CHECK-GI-NEXT: extr x11, x9, x12, #15
+; CHECK-GI-NEXT: lsr x9, x9, #15
+; CHECK-GI-NEXT: cset w12, lo
+; CHECK-GI-NEXT: cmp x8, #0
+; CHECK-GI-NEXT: csel w8, w12, wzr, eq
+; CHECK-GI-NEXT: cmn x11, #1
+; CHECK-GI-NEXT: cset w12, lo
+; CHECK-GI-NEXT: cmp x9, #0
+; CHECK-GI-NEXT: csel w9, w12, wzr, eq
+; CHECK-GI-NEXT: tst w8, #0x1
+; CHECK-GI-NEXT: csinv x8, x10, xzr, ne
+; CHECK-GI-NEXT: tst w9, #0x1
+; CHECK-GI-NEXT: fmov d0, x8
+; CHECK-GI-NEXT: csinv x9, x11, xzr, ne
+; CHECK-GI-NEXT: mov v0.d[1], x9
+; CHECK-GI-NEXT: ret
%tmp = call <2 x i64> @llvm.umul.fix.sat.v2i64(<2 x i64> %x, <2 x i64> %y, i32 15)
ret <2 x i64> %tmp
}
define <4 x i64> @vec_v4i64(<4 x i64> %x, <4 x i64> %y) {
-; CHECK-LABEL: vec_v4i64:
-; CHECK: // %bb.0:
-; CHECK-NEXT: mov x8, v2.d[1]
-; CHECK-NEXT: mov x9, v0.d[1]
-; CHECK-NEXT: mov x14, v3.d[1]
-; CHECK-NEXT: mov x15, v1.d[1]
-; CHECK-NEXT: fmov x10, d2
-; CHECK-NEXT: fmov x11, d0
-; CHECK-NEXT: mul x12, x11, x10
-; CHECK-NEXT: mul x13, x9, x8
-; CHECK-NEXT: umulh x8, x9, x8
-; CHECK-NEXT: umulh x9, x11, x10
-; CHECK-NEXT: mul x10, x15, x14
-; CHECK-NEXT: extr x13, x8, x13, #32
-; CHECK-NEXT: umulh x11, x15, x14
-; CHECK-NEXT: fmov x14, d3
-; CHECK-NEXT: fmov x15, d1
-; CHECK-NEXT: mul x16, x15, x14
-; CHECK-NEXT: umulh x14, x15, x14
-; CHECK-NEXT: mov w15, #-1 // =0xffffffff
-; CHECK-NEXT: cmp x8, x15
-; CHECK-NEXT: extr x8, x9, x12, #32
-; CHECK-NEXT: csinv x12, x13, xzr, ls
-; CHECK-NEXT: cmp x9, x15
-; CHECK-NEXT: extr x9, x11, x10, #32
-; CHECK-NEXT: csinv x8, x8, xzr, ls
-; CHECK-NEXT: cmp x11, x15
-; CHECK-NEXT: csinv x9, x9, xzr, ls
-; CHECK-NEXT: fmov d0, x8
-; CHECK-NEXT: extr x10, x14, x16, #32
-; CHECK-NEXT: cmp x14, x15
-; CHECK-NEXT: csinv x10, x10, xzr, ls
-; CHECK-NEXT: mov v0.d[1], x12
-; CHECK-NEXT: fmov d1, x10
-; CHECK-NEXT: mov v1.d[1], x9
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vec_v4i64:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: mov x8, v2.d[1]
+; CHECK-SD-NEXT: mov x9, v0.d[1]
+; CHECK-SD-NEXT: mov x14, v3.d[1]
+; CHECK-SD-NEXT: mov x15, v1.d[1]
+; CHECK-SD-NEXT: fmov x10, d2
+; CHECK-SD-NEXT: fmov x11, d0
+; CHECK-SD-NEXT: mul x12, x11, x10
+; CHECK-SD-NEXT: mul x13, x9, x8
+; CHECK-SD-NEXT: umulh x8, x9, x8
+; CHECK-SD-NEXT: umulh x9, x11, x10
+; CHECK-SD-NEXT: mul x10, x15, x14
+; CHECK-SD-NEXT: extr x13, x8, x13, #32
+; CHECK-SD-NEXT: umulh x11, x15, x14
+; CHECK-SD-NEXT: fmov x14, d3
+; CHECK-SD-NEXT: fmov x15, d1
+; CHECK-SD-NEXT: mul x16, x15, x14
+; CHECK-SD-NEXT: umulh x14, x15, x14
+; CHECK-SD-NEXT: mov w15, #-1 // =0xffffffff
+; CHECK-SD-NEXT: cmp x8, x15
+; CHECK-SD-NEXT: extr x8, x9, x12, #32
+; CHECK-SD-NEXT: csinv x12, x13, xzr, ls
+; CHECK-SD-NEXT: cmp x9, x15
+; CHECK-SD-NEXT: extr x9, x11, x10, #32
+; CHECK-SD-NEXT: csinv x8, x8, xzr, ls
+; CHECK-SD-NEXT: cmp x11, x15
+; CHECK-SD-NEXT: csinv x9, x9, xzr, ls
+; CHECK-SD-NEXT: fmov d0, x8
+; CHECK-SD-NEXT: extr x10, x14, x16, #32
+; CHECK-SD-NEXT: cmp x14, x15
+; CHECK-SD-NEXT: csinv x10, x10, xzr, ls
+; CHECK-SD-NEXT: mov v0.d[1], x12
+; CHECK-SD-NEXT: fmov d1, x10
+; CHECK-SD-NEXT: mov v1.d[1], x9
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vec_v4i64:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mov d4, v0.d[1]
+; CHECK-GI-NEXT: mov d5, v2.d[1]
+; CHECK-GI-NEXT: fmov x8, d0
+; CHECK-GI-NEXT: fmov x9, d2
+; CHECK-GI-NEXT: mov d0, v1.d[1]
+; CHECK-GI-NEXT: mov d2, v3.d[1]
+; CHECK-GI-NEXT: fmov x13, d3
+; CHECK-GI-NEXT: mul x10, x8, x9
+; CHECK-GI-NEXT: fmov x11, d5
+; CHECK-GI-NEXT: fmov x15, d2
+; CHECK-GI-NEXT: umulh x8, x8, x9
+; CHECK-GI-NEXT: fmov x9, d4
+; CHECK-GI-NEXT: mul x12, x9, x11
+; CHECK-GI-NEXT: umulh x9, x9, x11
+; CHECK-GI-NEXT: fmov x11, d1
+; CHECK-GI-NEXT: extr x10, x8, x10, #32
+; CHECK-GI-NEXT: lsr x8, x8, #32
+; CHECK-GI-NEXT: cmn x10, #1
+; CHECK-GI-NEXT: mul x14, x11, x13
+; CHECK-GI-NEXT: umulh x11, x11, x13
+; CHECK-GI-NEXT: fmov x13, d0
+; CHECK-GI-NEXT: extr x12, x9, x12, #32
+; CHECK-GI-NEXT: lsr x9, x9, #32
+; CHECK-GI-NEXT: mul x16, x13, x15
+; CHECK-GI-NEXT: umulh x13, x13, x15
+; CHECK-GI-NEXT: cset w15, lo
+; CHECK-GI-NEXT: cmp x8, #0
+; CHECK-GI-NEXT: extr x8, x11, x14, #32
+; CHECK-GI-NEXT: csel w14, w15, wzr, eq
+; CHECK-GI-NEXT: cmn x12, #1
+; CHECK-GI-NEXT: lsr x11, x11, #32
+; CHECK-GI-NEXT: cset w15, lo
+; CHECK-GI-NEXT: cmp x9, #0
+; CHECK-GI-NEXT: csel w15, w15, wzr, eq
+; CHECK-GI-NEXT: cmn x8, #1
+; CHECK-GI-NEXT: extr x9, x13, x16, #32
+; CHECK-GI-NEXT: lsr x13, x13, #32
+; CHECK-GI-NEXT: cset w16, lo
+; CHECK-GI-NEXT: cmp x11, #0
+; CHECK-GI-NEXT: csel w11, w16, wzr, eq
+; CHECK-GI-NEXT: cmn x9, #1
+; CHECK-GI-NEXT: cset w16, lo
+; CHECK-GI-NEXT: cmp x13, #0
+; CHECK-GI-NEXT: csel w13, w16, wzr, eq
+; CHECK-GI-NEXT: tst w14, #0x1
+; CHECK-GI-NEXT: csinv x10, x10, xzr, ne
+; CHECK-GI-NEXT: tst w15, #0x1
+; CHECK-GI-NEXT: csinv x12, x12, xzr, ne
+; CHECK-GI-NEXT: tst w11, #0x1
+; CHECK-GI-NEXT: fmov d0, x10
+; CHECK-GI-NEXT: csinv x8, x8, xzr, ne
+; CHECK-GI-NEXT: tst w13, #0x1
+; CHECK-GI-NEXT: fmov d1, x8
+; CHECK-GI-NEXT: csinv x9, x9, xzr, ne
+; CHECK-GI-NEXT: mov v0.d[1], x12
+; CHECK-GI-NEXT: mov v1.d[1], x9
+; CHECK-GI-NEXT: ret
%tmp = call <4 x i64> @llvm.umul.fix.sat.v4i64(<4 x i64> %x, <4 x i64> %y, i32 32)
ret <4 x i64> %tmp
}
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; CHECK-GI: {{.*}}
-; CHECK-SD: {{.*}}
+; CHECK: {{.*}}
>From b2edebcf996a9ed7eb67a13bbb787b713d76047d Mon Sep 17 00:00:00 2001
From: David Green <david.green at arm.com>
Date: Wed, 19 Aug 2026 10:08:36 +0100
Subject: [PATCH 3/6] [AArch64][GlobalISel] Clamp TruncSat num elements.
Just like v2i32, we can clamp v8i8 and v4i16 vectors to make use of truncsat in
more places and split larger vectors with fewerElements.
---
.../AArch64/GISel/AArch64LegalizerInfo.cpp | 2 +
llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll | 38 ++++---------------
2 files changed, 10 insertions(+), 30 deletions(-)
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index 91ad1911818f6..5c1d43139f0a4 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -860,6 +860,8 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
getActionDefinitionsBuilder({G_TRUNC_SSAT_S, G_TRUNC_SSAT_U, G_TRUNC_USAT_U})
.legalFor({{v8i8, v8i16}, {v4i16, v4i32}, {v2i32, v2i64}})
+ .clampNumElements(0, v8s8, v8s8)
+ .clampNumElements(0, v4s16, v4s16)
.clampNumElements(0, v2s32, v2s32)
.lower();
diff --git a/llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll b/llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll
index dcb0051bd4c3c..b406a8c43b12d 100644
--- a/llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll
+++ b/llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll
@@ -459,20 +459,10 @@ define <8 x i16> @utest_f16i16(<8 x half> %x) {
; CHECK-CVT-NEXT: uqxtn2 v0.8h, v2.4s
; CHECK-CVT-NEXT: ret
;
-; CHECK-FP16-SD-LABEL: utest_f16i16:
-; CHECK-FP16-SD: // %bb.0: // %entry
-; CHECK-FP16-SD-NEXT: fcvtzu v0.8h, v0.8h
-; CHECK-FP16-SD-NEXT: ret
-;
-; CHECK-FP16-GI-LABEL: utest_f16i16:
-; CHECK-FP16-GI: // %bb.0: // %entry
-; CHECK-FP16-GI-NEXT: fcvtl v1.4s, v0.4h
-; CHECK-FP16-GI-NEXT: fcvtl2 v0.4s, v0.8h
-; CHECK-FP16-GI-NEXT: fcvtzu v1.4s, v1.4s
-; CHECK-FP16-GI-NEXT: fcvtzu v2.4s, v0.4s
-; CHECK-FP16-GI-NEXT: uqxtn v0.4h, v1.4s
-; CHECK-FP16-GI-NEXT: uqxtn2 v0.8h, v2.4s
-; CHECK-FP16-GI-NEXT: ret
+; CHECK-FP16-LABEL: utest_f16i16:
+; CHECK-FP16: // %bb.0: // %entry
+; CHECK-FP16-NEXT: fcvtzu v0.8h, v0.8h
+; CHECK-FP16-NEXT: ret
entry:
%conv = fptoui <8 x half> %x to <8 x i32>
%0 = icmp ult <8 x i32> %conv, <i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535>
@@ -2252,20 +2242,10 @@ define <8 x i16> @utest_f16i16_mm(<8 x half> %x) {
; CHECK-CVT-NEXT: uqxtn2 v0.8h, v2.4s
; CHECK-CVT-NEXT: ret
;
-; CHECK-FP16-SD-LABEL: utest_f16i16_mm:
-; CHECK-FP16-SD: // %bb.0: // %entry
-; CHECK-FP16-SD-NEXT: fcvtzu v0.8h, v0.8h
-; CHECK-FP16-SD-NEXT: ret
-;
-; CHECK-FP16-GI-LABEL: utest_f16i16_mm:
-; CHECK-FP16-GI: // %bb.0: // %entry
-; CHECK-FP16-GI-NEXT: fcvtl v1.4s, v0.4h
-; CHECK-FP16-GI-NEXT: fcvtl2 v0.4s, v0.8h
-; CHECK-FP16-GI-NEXT: fcvtzu v1.4s, v1.4s
-; CHECK-FP16-GI-NEXT: fcvtzu v2.4s, v0.4s
-; CHECK-FP16-GI-NEXT: uqxtn v0.4h, v1.4s
-; CHECK-FP16-GI-NEXT: uqxtn2 v0.8h, v2.4s
-; CHECK-FP16-GI-NEXT: ret
+; CHECK-FP16-LABEL: utest_f16i16_mm:
+; CHECK-FP16: // %bb.0: // %entry
+; CHECK-FP16-NEXT: fcvtzu v0.8h, v0.8h
+; CHECK-FP16-NEXT: ret
entry:
%conv = fptoui <8 x half> %x to <8 x i32>
%spec.store.select = call <8 x i32> @llvm.umin.v8i32(<8 x i32> %conv, <8 x i32> <i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535>)
@@ -3648,5 +3628,3 @@ declare <4 x i64> @llvm.umin.v4i64(<4 x i64>, <4 x i64>)
declare <2 x i128> @llvm.smin.v2i128(<2 x i128>, <2 x i128>)
declare <2 x i128> @llvm.smax.v2i128(<2 x i128>, <2 x i128>)
declare <2 x i128> @llvm.umin.v2i128(<2 x i128>, <2 x i128>)
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; CHECK-FP16: {{.*}}
>From 31a730b2d103b19a658c310ac9ac760299b5ef29 Mon Sep 17 00:00:00 2001
From: David Green <david.green at arm.com>
Date: Wed, 19 Aug 2026 10:36:57 +0100
Subject: [PATCH 4/6] [AArch64][GlobalISel] Combine trunc_nsw(smin) to truncsat
The midend of LLVM, in CVP, will often transform the canonical pattern for a
saturate trunc(smin(smax)) into trunc nsw (smin), as the range analysis proves
that the smax is outside of range. This adds a fold back, converting the trunc
with nsw + smin into G_TRUNC_SSAT_S.
https://alive2.llvm.org/ce/z/xfPEXE
---
.../lib/CodeGen/GlobalISel/CombinerHelper.cpp | 26 ++--
llvm/test/CodeGen/AArch64/qmovn.ll | 21 +--
.../CodeGen/AArch64/saturating-vec-smull.ll | 125 ++++--------------
3 files changed, 51 insertions(+), 121 deletions(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index 55c4339edc39e..75917a76d6bc7 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -6290,14 +6290,24 @@ bool CombinerHelper::matchTruncSSatS(MachineInstr &MI,
APInt SignedMax = APInt::getSignedMaxValue(NumDstBits).sext(NumSrcBits);
APInt SignedMin = APInt::getSignedMinValue(NumDstBits).sext(NumSrcBits);
- return mi_match(Src, MRI,
- m_GSMin(m_GSMax(m_Reg(MatchInfo),
- m_SpecificICstOrSplat(SignedMin)),
- m_SpecificICstOrSplat(SignedMax))) ||
- mi_match(Src, MRI,
- m_GSMax(m_GSMin(m_Reg(MatchInfo),
- m_SpecificICstOrSplat(SignedMax)),
- m_SpecificICstOrSplat(SignedMin)));
+ if (mi_match(
+ Src, MRI,
+ m_GSMin(m_GSMax(m_Reg(MatchInfo), m_SpecificICstOrSplat(SignedMin)),
+ m_SpecificICstOrSplat(SignedMax))) ||
+ mi_match(
+ Src, MRI,
+ m_GSMax(m_GSMin(m_Reg(MatchInfo), m_SpecificICstOrSplat(SignedMax)),
+ m_SpecificICstOrSplat(SignedMin))))
+ return true;
+
+ // CVP in the midend will often transform trunc(smin(smax(..)) into
+ // trunc nsw(smin(..)) as the smax against INT_MIN never saturates.
+ if (MI.getFlag(MachineInstr::MIFlag::NoSWrap) &&
+ mi_match(Src, MRI,
+ m_GSMin(m_Reg(MatchInfo), m_SpecificICstOrSplat(SignedMax))))
+ return true;
+
+ return false;
}
void CombinerHelper::applyTruncSSatS(MachineInstr &MI,
diff --git a/llvm/test/CodeGen/AArch64/qmovn.ll b/llvm/test/CodeGen/AArch64/qmovn.ll
index 2fb4e2e467c7d..75bf1c2206422 100644
--- a/llvm/test/CodeGen/AArch64/qmovn.ll
+++ b/llvm/test/CodeGen/AArch64/qmovn.ll
@@ -640,10 +640,8 @@ define <16 x i8> @signed_minnsw_v16i16_to_v16i8(<16 x i16> %y) {
;
; CHECK-GI-LABEL: signed_minnsw_v16i16_to_v16i8:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: movi v2.8h, #127
-; CHECK-GI-NEXT: smin v0.8h, v0.8h, v2.8h
-; CHECK-GI-NEXT: smin v1.8h, v1.8h, v2.8h
-; CHECK-GI-NEXT: uzp1 v0.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT: sqxtn v0.8b, v0.8h
+; CHECK-GI-NEXT: sqxtn2 v0.16b, v1.8h
; CHECK-GI-NEXT: ret
entry:
%min = call <16 x i16> @llvm.smin.v16i16(<16 x i16> %y, <16 x i16> <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>)
@@ -662,10 +660,8 @@ define <8 x i16> @signed_minnsw_v8i32_to_v8i16(<8 x i32> %y) {
;
; CHECK-GI-LABEL: signed_minnsw_v8i32_to_v8i16:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: movi v2.4s, #127, msl #8
-; CHECK-GI-NEXT: smin v0.4s, v0.4s, v2.4s
-; CHECK-GI-NEXT: smin v1.4s, v1.4s, v2.4s
-; CHECK-GI-NEXT: uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-GI-NEXT: sqxtn v0.4h, v0.4s
+; CHECK-GI-NEXT: sqxtn2 v0.8h, v1.4s
; CHECK-GI-NEXT: ret
entry:
%min = call <8 x i32> @llvm.smin.v8i32(<8 x i32> %y, <8 x i32> <i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767, i32 32767>)
@@ -687,13 +683,8 @@ define <4 x i32> @signed_minnsw_v4i64_to_v4i32(<4 x i64> %y) {
;
; CHECK-GI-LABEL: signed_minnsw_v4i64_to_v4i32:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: adrp x8, .LCPI47_0
-; CHECK-GI-NEXT: ldr q2, [x8, :lo12:.LCPI47_0]
-; CHECK-GI-NEXT: cmgt v3.2d, v2.2d, v0.2d
-; CHECK-GI-NEXT: cmgt v4.2d, v2.2d, v1.2d
-; CHECK-GI-NEXT: bif v0.16b, v2.16b, v3.16b
-; CHECK-GI-NEXT: bif v1.16b, v2.16b, v4.16b
-; CHECK-GI-NEXT: uzp1 v0.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT: sqxtn v0.2s, v0.2d
+; CHECK-GI-NEXT: sqxtn2 v0.4s, v1.2d
; CHECK-GI-NEXT: ret
entry:
%min = call <4 x i64> @llvm.smin.v4i64(<4 x i64> %y, <4 x i64> <i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647>)
diff --git a/llvm/test/CodeGen/AArch64/saturating-vec-smull.ll b/llvm/test/CodeGen/AArch64/saturating-vec-smull.ll
index fe9b060be6ae2..f4585a5cd4dca 100644
--- a/llvm/test/CodeGen/AArch64/saturating-vec-smull.ll
+++ b/llvm/test/CodeGen/AArch64/saturating-vec-smull.ll
@@ -2,7 +2,8 @@
; RUN: llc -mtriple=aarch64-none-elf < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SD
; RUN: llc -mtriple=aarch64-none-elf -global-isel -global-isel-abort=2 2>&1 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-GI
-; CHECK-GI: warning: Instruction selection used fallback path for extend_to_illegal_type
+; CHECK-GI: warning: Instruction selection used fallback path for saturating_6xi16
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for extend_to_illegal_type
define <2 x i16> @saturating_2xi16(<2 x i16> %a, <2 x i16> %b) {
; CHECK-SD-LABEL: saturating_2xi16:
@@ -43,10 +44,7 @@ define <4 x i16> @saturating_4xi16(<4 x i16> %a, <4 x i16> %b) {
; CHECK-GI-LABEL: saturating_4xi16:
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: smull v0.4s, v1.4h, v0.4h
-; CHECK-GI-NEXT: movi v1.4s, #127, msl #8
-; CHECK-GI-NEXT: sshr v0.4s, v0.4s, #15
-; CHECK-GI-NEXT: smin v0.4s, v0.4s, v1.4s
-; CHECK-GI-NEXT: xtn v0.4h, v0.4s
+; CHECK-GI-NEXT: sqshrn v0.4h, v0.4s, #15
; CHECK-GI-NEXT: ret
%as = sext <4 x i16> %a to <4 x i32>
%bs = sext <4 x i16> %b to <4 x i32>
@@ -65,14 +63,10 @@ define <8 x i16> @saturating_8xi16(<8 x i16> %a, <8 x i16> %b) {
;
; CHECK-GI-LABEL: saturating_8xi16:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: smull v3.4s, v1.4h, v0.4h
-; CHECK-GI-NEXT: smull2 v0.4s, v1.8h, v0.8h
-; CHECK-GI-NEXT: movi v2.4s, #127, msl #8
-; CHECK-GI-NEXT: sshr v1.4s, v3.4s, #15
-; CHECK-GI-NEXT: sshr v0.4s, v0.4s, #15
-; CHECK-GI-NEXT: smin v1.4s, v1.4s, v2.4s
-; CHECK-GI-NEXT: smin v0.4s, v0.4s, v2.4s
-; CHECK-GI-NEXT: uzp1 v0.8h, v1.8h, v0.8h
+; CHECK-GI-NEXT: smull v2.4s, v1.4h, v0.4h
+; CHECK-GI-NEXT: smull2 v1.4s, v1.8h, v0.8h
+; CHECK-GI-NEXT: sqshrn v0.4h, v2.4s, #15
+; CHECK-GI-NEXT: sqshrn2 v0.8h, v1.4s, #15
; CHECK-GI-NEXT: ret
%as = sext <8 x i16> %a to <8 x i32>
%bs = sext <8 x i16> %b to <8 x i32>
@@ -92,12 +86,7 @@ define <2 x i32> @saturating_2xi32(<2 x i32> %a, <2 x i32> %b) {
; CHECK-GI-LABEL: saturating_2xi32:
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: smull v0.2d, v1.2s, v0.2s
-; CHECK-GI-NEXT: adrp x8, .LCPI3_0
-; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI3_0]
-; CHECK-GI-NEXT: sshr v0.2d, v0.2d, #31
-; CHECK-GI-NEXT: cmgt v2.2d, v1.2d, v0.2d
-; CHECK-GI-NEXT: bif v0.16b, v1.16b, v2.16b
-; CHECK-GI-NEXT: xtn v0.2s, v0.2d
+; CHECK-GI-NEXT: sqshrn v0.2s, v0.2d, #31
; CHECK-GI-NEXT: ret
%as = sext <2 x i32> %a to <2 x i64>
%bs = sext <2 x i32> %b to <2 x i64>
@@ -117,16 +106,9 @@ define <4 x i32> @saturating_4xi32(<4 x i32> %a, <4 x i32> %b) {
; CHECK-GI-LABEL: saturating_4xi32:
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: smull v2.2d, v1.2s, v0.2s
-; CHECK-GI-NEXT: smull2 v0.2d, v1.4s, v0.4s
-; CHECK-GI-NEXT: adrp x8, .LCPI4_0
-; CHECK-GI-NEXT: sshr v1.2d, v2.2d, #31
-; CHECK-GI-NEXT: sshr v0.2d, v0.2d, #31
-; CHECK-GI-NEXT: ldr q2, [x8, :lo12:.LCPI4_0]
-; CHECK-GI-NEXT: cmgt v3.2d, v2.2d, v1.2d
-; CHECK-GI-NEXT: cmgt v4.2d, v2.2d, v0.2d
-; CHECK-GI-NEXT: bif v1.16b, v2.16b, v3.16b
-; CHECK-GI-NEXT: bif v0.16b, v2.16b, v4.16b
-; CHECK-GI-NEXT: uzp1 v0.4s, v1.4s, v0.4s
+; CHECK-GI-NEXT: smull2 v1.2d, v1.4s, v0.4s
+; CHECK-GI-NEXT: sqshrn v0.2s, v2.2d, #31
+; CHECK-GI-NEXT: sqshrn2 v0.4s, v1.2d, #31
; CHECK-GI-NEXT: ret
%as = sext <4 x i32> %a to <4 x i64>
%bs = sext <4 x i32> %b to <4 x i64>
@@ -147,25 +129,13 @@ define <8 x i32> @saturating_8xi32(<8 x i32> %a, <8 x i32> %b) {
; CHECK-GI-LABEL: saturating_8xi32:
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: smull v4.2d, v2.2s, v0.2s
-; CHECK-GI-NEXT: smull2 v0.2d, v2.4s, v0.4s
-; CHECK-GI-NEXT: adrp x8, .LCPI5_0
-; CHECK-GI-NEXT: smull v2.2d, v3.2s, v1.2s
-; CHECK-GI-NEXT: smull2 v1.2d, v3.4s, v1.4s
-; CHECK-GI-NEXT: ldr q3, [x8, :lo12:.LCPI5_0]
-; CHECK-GI-NEXT: sshr v4.2d, v4.2d, #31
-; CHECK-GI-NEXT: sshr v0.2d, v0.2d, #31
-; CHECK-GI-NEXT: sshr v2.2d, v2.2d, #31
-; CHECK-GI-NEXT: sshr v1.2d, v1.2d, #31
-; CHECK-GI-NEXT: cmgt v5.2d, v3.2d, v4.2d
-; CHECK-GI-NEXT: cmgt v6.2d, v3.2d, v0.2d
-; CHECK-GI-NEXT: cmgt v7.2d, v3.2d, v2.2d
-; CHECK-GI-NEXT: cmgt v16.2d, v3.2d, v1.2d
-; CHECK-GI-NEXT: bif v4.16b, v3.16b, v5.16b
-; CHECK-GI-NEXT: bif v0.16b, v3.16b, v6.16b
-; CHECK-GI-NEXT: bif v2.16b, v3.16b, v7.16b
-; CHECK-GI-NEXT: bif v1.16b, v3.16b, v16.16b
-; CHECK-GI-NEXT: uzp1 v0.4s, v4.4s, v0.4s
-; CHECK-GI-NEXT: uzp1 v1.4s, v2.4s, v1.4s
+; CHECK-GI-NEXT: smull v5.2d, v3.2s, v1.2s
+; CHECK-GI-NEXT: smull2 v2.2d, v2.4s, v0.4s
+; CHECK-GI-NEXT: smull2 v3.2d, v3.4s, v1.4s
+; CHECK-GI-NEXT: sqshrn v0.2s, v4.2d, #31
+; CHECK-GI-NEXT: sqshrn v1.2s, v5.2d, #31
+; CHECK-GI-NEXT: sqshrn2 v0.4s, v2.2d, #31
+; CHECK-GI-NEXT: sqshrn2 v1.4s, v3.2d, #31
; CHECK-GI-NEXT: ret
%as = sext <8 x i32> %a to <8 x i64>
%bs = sext <8 x i32> %b to <8 x i64>
@@ -201,56 +171,15 @@ define <2 x i64> @saturating_2xi32_2xi64(<2 x i32> %a, <2 x i32> %b) {
}
define <6 x i16> @saturating_6xi16(<6 x i16> %a, <6 x i16> %b) {
-; CHECK-SD-LABEL: saturating_6xi16:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: smull2 v2.4s, v1.8h, v0.8h
-; CHECK-SD-NEXT: movi v3.4s, #127, msl #8
-; CHECK-SD-NEXT: sqdmulh v0.4h, v1.4h, v0.4h
-; CHECK-SD-NEXT: sshr v2.4s, v2.4s, #15
-; CHECK-SD-NEXT: smin v1.4s, v2.4s, v3.4s
-; CHECK-SD-NEXT: xtn2 v0.8h, v1.4s
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: saturating_6xi16:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: smov w8, v1.h[0]
-; CHECK-GI-NEXT: smov w9, v0.h[0]
-; CHECK-GI-NEXT: smov w10, v1.h[1]
-; CHECK-GI-NEXT: smov w11, v0.h[1]
-; CHECK-GI-NEXT: smov w12, v1.h[2]
-; CHECK-GI-NEXT: movi v2.4s, #127, msl #8
-; CHECK-GI-NEXT: fmov s3, w8
-; CHECK-GI-NEXT: fmov s4, w9
-; CHECK-GI-NEXT: smov w8, v0.h[2]
-; CHECK-GI-NEXT: smov w9, v1.h[3]
-; CHECK-GI-NEXT: sshll2 v1.4s, v1.8h, #0
-; CHECK-GI-NEXT: mov v3.s[1], w10
-; CHECK-GI-NEXT: mov v4.s[1], w11
-; CHECK-GI-NEXT: smov w10, v0.h[3]
-; CHECK-GI-NEXT: sshll2 v0.4s, v0.8h, #0
-; CHECK-GI-NEXT: mov v3.s[2], w12
-; CHECK-GI-NEXT: mov v4.s[2], w8
-; CHECK-GI-NEXT: mul v0.2s, v1.2s, v0.2s
-; CHECK-GI-NEXT: movi v1.2s, #127, msl #8
-; CHECK-GI-NEXT: mov v3.s[3], w9
-; CHECK-GI-NEXT: mov v4.s[3], w10
-; CHECK-GI-NEXT: sshr v0.2s, v0.2s, #15
-; CHECK-GI-NEXT: smin v0.2s, v0.2s, v1.2s
-; CHECK-GI-NEXT: mul v3.4s, v3.4s, v4.4s
-; CHECK-GI-NEXT: sshr v3.4s, v3.4s, #15
-; CHECK-GI-NEXT: smin v2.4s, v3.4s, v2.4s
-; CHECK-GI-NEXT: mov w8, v2.s[1]
-; CHECK-GI-NEXT: mov w9, v2.s[2]
-; CHECK-GI-NEXT: mov w10, v2.s[3]
-; CHECK-GI-NEXT: mov v2.h[1], w8
-; CHECK-GI-NEXT: fmov w8, s0
-; CHECK-GI-NEXT: mov v2.h[2], w9
-; CHECK-GI-NEXT: mov w9, v0.s[1]
-; CHECK-GI-NEXT: mov v2.h[3], w10
-; CHECK-GI-NEXT: mov v2.h[4], w8
-; CHECK-GI-NEXT: mov v2.h[5], w9
-; CHECK-GI-NEXT: mov v0.16b, v2.16b
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: saturating_6xi16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: smull2 v2.4s, v1.8h, v0.8h
+; CHECK-NEXT: movi v3.4s, #127, msl #8
+; CHECK-NEXT: sqdmulh v0.4h, v1.4h, v0.4h
+; CHECK-NEXT: sshr v2.4s, v2.4s, #15
+; CHECK-NEXT: smin v1.4s, v2.4s, v3.4s
+; CHECK-NEXT: xtn2 v0.8h, v1.4s
+; CHECK-NEXT: ret
%as = sext <6 x i16> %a to <6 x i32>
%bs = sext <6 x i16> %b to <6 x i32>
%m = mul nsw <6 x i32> %bs, %as
>From 356375db6e6e1a04801e357fb5ce19c7bf79887b Mon Sep 17 00:00:00 2001
From: David Green <david.green at arm.com>
Date: Wed, 19 Aug 2026 10:54:17 +0100
Subject: [PATCH 5/6] [AArch64][GlobalISel] G_TRUNC_SSAT_S moreElements
handling
These operations can be widened in the same way as a standard trunc, by using
buildPadVectorWithUndefElements.
---
.../CodeGen/GlobalISel/LegalizerHelper.cpp | 5 +-
.../CodeGen/AArch64/saturating-vec-smull.ll | 55 +++++++++++++++----
2 files changed, 48 insertions(+), 12 deletions(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 3a9de29c55224..8117e47042af0 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -7074,7 +7074,10 @@ LegalizerHelper::moreElementsVector(MachineInstr &MI, unsigned TypeIdx,
case TargetOpcode::G_FPTOSI_SAT:
case TargetOpcode::G_FPTOUI_SAT:
case TargetOpcode::G_SITOFP:
- case TargetOpcode::G_UITOFP: {
+ case TargetOpcode::G_UITOFP:
+ case TargetOpcode::G_TRUNC_SSAT_S:
+ case TargetOpcode::G_TRUNC_SSAT_U:
+ case TargetOpcode::G_TRUNC_USAT_U: {
Observer.changingInstr(MI);
LLT SrcExtTy;
LLT DstExtTy;
diff --git a/llvm/test/CodeGen/AArch64/saturating-vec-smull.ll b/llvm/test/CodeGen/AArch64/saturating-vec-smull.ll
index f4585a5cd4dca..a16eb65b32a71 100644
--- a/llvm/test/CodeGen/AArch64/saturating-vec-smull.ll
+++ b/llvm/test/CodeGen/AArch64/saturating-vec-smull.ll
@@ -2,8 +2,7 @@
; RUN: llc -mtriple=aarch64-none-elf < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SD
; RUN: llc -mtriple=aarch64-none-elf -global-isel -global-isel-abort=2 2>&1 < %s | FileCheck %s --check-prefixes=CHECK,CHECK-GI
-; CHECK-GI: warning: Instruction selection used fallback path for saturating_6xi16
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for extend_to_illegal_type
+; CHECK-GI: warning: Instruction selection used fallback path for extend_to_illegal_type
define <2 x i16> @saturating_2xi16(<2 x i16> %a, <2 x i16> %b) {
; CHECK-SD-LABEL: saturating_2xi16:
@@ -171,15 +170,49 @@ define <2 x i64> @saturating_2xi32_2xi64(<2 x i32> %a, <2 x i32> %b) {
}
define <6 x i16> @saturating_6xi16(<6 x i16> %a, <6 x i16> %b) {
-; CHECK-LABEL: saturating_6xi16:
-; CHECK: // %bb.0:
-; CHECK-NEXT: smull2 v2.4s, v1.8h, v0.8h
-; CHECK-NEXT: movi v3.4s, #127, msl #8
-; CHECK-NEXT: sqdmulh v0.4h, v1.4h, v0.4h
-; CHECK-NEXT: sshr v2.4s, v2.4s, #15
-; CHECK-NEXT: smin v1.4s, v2.4s, v3.4s
-; CHECK-NEXT: xtn2 v0.8h, v1.4s
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: saturating_6xi16:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: smull2 v2.4s, v1.8h, v0.8h
+; CHECK-SD-NEXT: movi v3.4s, #127, msl #8
+; CHECK-SD-NEXT: sqdmulh v0.4h, v1.4h, v0.4h
+; CHECK-SD-NEXT: sshr v2.4s, v2.4s, #15
+; CHECK-SD-NEXT: smin v1.4s, v2.4s, v3.4s
+; CHECK-SD-NEXT: xtn2 v0.8h, v1.4s
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: saturating_6xi16:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: smov w8, v1.h[0]
+; CHECK-GI-NEXT: smov w9, v0.h[0]
+; CHECK-GI-NEXT: smov w10, v1.h[1]
+; CHECK-GI-NEXT: smov w11, v0.h[1]
+; CHECK-GI-NEXT: fmov s2, w8
+; CHECK-GI-NEXT: fmov s3, w9
+; CHECK-GI-NEXT: smov w8, v1.h[2]
+; CHECK-GI-NEXT: smov w9, v0.h[2]
+; CHECK-GI-NEXT: mov v2.s[1], w10
+; CHECK-GI-NEXT: mov v3.s[1], w11
+; CHECK-GI-NEXT: smov w10, v1.h[3]
+; CHECK-GI-NEXT: sshll2 v1.4s, v1.8h, #0
+; CHECK-GI-NEXT: mov v2.s[2], w8
+; CHECK-GI-NEXT: smov w8, v0.h[3]
+; CHECK-GI-NEXT: mov v3.s[2], w9
+; CHECK-GI-NEXT: sshll2 v0.4s, v0.8h, #0
+; CHECK-GI-NEXT: mov v2.s[3], w10
+; CHECK-GI-NEXT: mov v3.s[3], w8
+; CHECK-GI-NEXT: mul v0.2s, v1.2s, v0.2s
+; CHECK-GI-NEXT: mul v2.4s, v2.4s, v3.4s
+; CHECK-GI-NEXT: sshr v0.2s, v0.2s, #15
+; CHECK-GI-NEXT: sqxtn v0.4h, v0.4s
+; CHECK-GI-NEXT: sqshrn v3.4h, v2.4s, #15
+; CHECK-GI-NEXT: mov v2.h[0], v3.h[0]
+; CHECK-GI-NEXT: mov v2.h[1], v3.h[1]
+; CHECK-GI-NEXT: mov v2.h[2], v3.h[2]
+; CHECK-GI-NEXT: mov v2.h[3], v3.h[3]
+; CHECK-GI-NEXT: mov v2.h[4], v0.h[0]
+; CHECK-GI-NEXT: mov v2.h[5], v0.h[1]
+; CHECK-GI-NEXT: mov v0.16b, v2.16b
+; CHECK-GI-NEXT: ret
%as = sext <6 x i16> %a to <6 x i32>
%bs = sext <6 x i16> %b to <6 x i32>
%m = mul nsw <6 x i32> %bs, %as
>From 29c8fd83ef68bf15a1900229679516b2e365fb07 Mon Sep 17 00:00:00 2001
From: David Green <david.green at arm.com>
Date: Wed, 19 Aug 2026 12:59:46 +0100
Subject: [PATCH 6/6] [AArch64][GlobalISel] Add tablegen patterns for
smull+sqshrn -> sqdmulh.
We do this in parts in globalisel, recognizing the smull and trunc_sat, which
when combined with the correct shift can fold into a single sqdmulh.
---
llvm/lib/Target/AArch64/AArch64InstrInfo.td | 21 ++++++
llvm/test/CodeGen/AArch64/arm64-tbl.ll | 2 +-
llvm/test/CodeGen/AArch64/clmul-fixed.ll | 8 +--
llvm/test/CodeGen/AArch64/qmovn.ll | 6 +-
llvm/test/CodeGen/AArch64/qshrn.ll | 20 +++---
llvm/test/CodeGen/AArch64/rqshrn.ll | 20 +++---
.../CodeGen/AArch64/saturating-vec-smull.ll | 70 +++++--------------
llvm/test/CodeGen/AArch64/smul_fix_sat.ll | 62 +++++-----------
llvm/test/CodeGen/AArch64/vector-minmax.ll | 2 +-
llvm/test/CodeGen/AArch64/xar.ll | 11 +--
.../MemorySanitizer/AArch64/arm64-tbl.ll | 2 +-
11 files changed, 88 insertions(+), 136 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index da6b90e9636f8..a9240defae6e0 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -10452,6 +10452,27 @@ def : Pat<(v8i16 (AArch64sqdmulh (v8i16 V128:$Rn), (v8i16 V128:$Rm))),
def : Pat<(v4i32 (AArch64sqdmulh (v4i32 V128:$Rn), (v4i32 V128:$Rm))),
(SQDMULHv4i32 V128:$Rn, V128:$Rm)>;
+def AArch64sqdmulhPat : PatFrag<(ops node:$Rn, node:$Rm, node:$offset),
+ (truncssat_s (AArch64vashr (AArch64smull node:$Rn, node:$Rm), node:$offset))>;
+def : Pat<(v4i16 (AArch64sqdmulhPat (v4i16 V64:$Rn), (v4i16 V64:$Rm), (i32 15))),
+ (SQDMULHv4i16 V64:$Rn, V64:$Rm)>;
+def : Pat<(v2i32 (AArch64sqdmulhPat (v2i32 V64:$Rn), (v2i32 V64:$Rm), (i32 31))),
+ (SQDMULHv2i32 V64:$Rn, V64:$Rm)>;
+def : Pat<(v8i16 (concat_vectors (v4i16 (AArch64sqdmulhPat (v4i16 (extract_subvector V128:$Rn, (i64 0))),
+ (v4i16 (extract_subvector V128:$Rm, (i64 0))),
+ (i32 15))),
+ (v4i16 (AArch64sqdmulhPat (v4i16 (extract_subvector V128:$Rn, (i64 4))),
+ (v4i16 (extract_subvector V128:$Rm, (i64 4))),
+ (i32 15))))),
+ (SQDMULHv8i16 V128:$Rn, V128:$Rm)>;
+def : Pat<(v4i32 (concat_vectors (v2i32 (AArch64sqdmulhPat (v2i32 (extract_subvector V128:$Rn, (i64 0))),
+ (v2i32 (extract_subvector V128:$Rm, (i64 0))),
+ (i32 31))),
+ (v2i32 (AArch64sqdmulhPat (v2i32 (extract_subvector V128:$Rn, (i64 2))),
+ (v2i32 (extract_subvector V128:$Rm, (i64 2))),
+ (i32 31))))),
+ (SQDMULHv4i32 V128:$Rn, V128:$Rm)>;
+
// Conversions within AdvSIMD types in the same register size are free.
// But because we need a consistent lane ordering, in big endian many
// conversions require one or more REV instructions.
diff --git a/llvm/test/CodeGen/AArch64/arm64-tbl.ll b/llvm/test/CodeGen/AArch64/arm64-tbl.ll
index 27694bd08526b..200fc65f01828 100644
--- a/llvm/test/CodeGen/AArch64/arm64-tbl.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-tbl.ll
@@ -1177,7 +1177,7 @@ define <8 x i8> @tbx3_8b(<8 x i8> %A, <16 x i8> %B, <16 x i8> %C, <16 x i8> %D,
; CHECK-GI-NEXT: // kill: def $q3 killed $q3 killed $q1_q2_q3 def $q1_q2_q3
; CHECK-GI-NEXT: tbx.8b v0, { v1, v2, v3 }, v4
; CHECK-GI-NEXT: ret
- %tmp3 = call <8 x i8> @llvm.aarch64.neon.tbx3.v8i8(< 8 x i8> %A, <16 x i8> %B, <16 x i8> %C, <16 x i8> %D, <8 x i8> %E)
+ %tmp3 = call <8 x i8> @llvm.aarch64.neon.tbx3.v8i8(<8 x i8> %A, <16 x i8> %B, <16 x i8> %C, <16 x i8> %D, <8 x i8> %E)
ret <8 x i8> %tmp3
}
diff --git a/llvm/test/CodeGen/AArch64/clmul-fixed.ll b/llvm/test/CodeGen/AArch64/clmul-fixed.ll
index a924bf4cdf07a..4e5e4c07e281f 100644
--- a/llvm/test/CodeGen/AArch64/clmul-fixed.ll
+++ b/llvm/test/CodeGen/AArch64/clmul-fixed.ll
@@ -1507,7 +1507,7 @@ define <1 x i128> @clmul_v1i128_neon_zext(<1 x i64> %x, <1 x i64> %y) {
; CHECK-AES-NEXT: ret
%zextx = zext <1 x i64> %x to <1 x i128>
%zexty = zext <1 x i64> %y to <1 x i128>
- %a = call <1 x i128> @llvm.clmul.v2i128(<1 x i128> %zextx, <1 x i128> %zexty)
+ %a = call <1 x i128> @llvm.clmul.v1i128(<1 x i128> %zextx, <1 x i128> %zexty)
ret <1 x i128> %a
}
@@ -1841,7 +1841,7 @@ define <8 x i8> @clmulr_v8i8_neon(<8 x i8> %a, <8 x i8> %b) nounwind {
; CHECK-NEXT: ret
%a.ext = zext <8 x i8> %a to <8 x i16>
%b.ext = zext <8 x i8> %b to <8 x i16>
- %clmul = call <8 x i16> @llvm.clmul.v16i16(<8 x i16> %a.ext, <8 x i16> %b.ext)
+ %clmul = call <8 x i16> @llvm.clmul.v8i16(<8 x i16> %a.ext, <8 x i16> %b.ext)
%res.ext = lshr <8 x i16> %clmul, splat (i16 7)
%res = trunc <8 x i16> %res.ext to <8 x i8>
ret <8 x i8> %res
@@ -2403,7 +2403,7 @@ define <1 x i64> @clmulr_v1i64_neon(<1 x i64> %a, <1 x i64> %b) nounwind {
; CHECK-AES-NEXT: ret
%a.ext = zext <1 x i64> %a to <1 x i128>
%b.ext = zext <1 x i64> %b to <1 x i128>
- %clmul = call <1 x i128> @llvm.clmul.v2i128(<1 x i128> %a.ext, <1 x i128> %b.ext)
+ %clmul = call <1 x i128> @llvm.clmul.v1i128(<1 x i128> %a.ext, <1 x i128> %b.ext)
%res.ext = lshr <1 x i128> %clmul, splat (i128 63)
%res = trunc <1 x i128> %res.ext to <1 x i64>
ret <1 x i64> %res
@@ -2437,7 +2437,7 @@ define <8 x i8> @clmulh_v8i8_neon(<8 x i8> %a, <8 x i8> %b) nounwind {
; CHECK-NEXT: ret
%a.ext = zext <8 x i8> %a to <8 x i16>
%b.ext = zext <8 x i8> %b to <8 x i16>
- %clmul = call <8 x i16> @llvm.clmul.v16i16(<8 x i16> %a.ext, <8 x i16> %b.ext)
+ %clmul = call <8 x i16> @llvm.clmul.v8i16(<8 x i16> %a.ext, <8 x i16> %b.ext)
%res.ext = lshr <8 x i16> %clmul, splat (i16 8)
%res = trunc <8 x i16> %res.ext to <8 x i8>
ret <8 x i8> %res
diff --git a/llvm/test/CodeGen/AArch64/qmovn.ll b/llvm/test/CodeGen/AArch64/qmovn.ll
index 75bf1c2206422..c1b8c1c221614 100644
--- a/llvm/test/CodeGen/AArch64/qmovn.ll
+++ b/llvm/test/CodeGen/AArch64/qmovn.ll
@@ -516,7 +516,7 @@ define <16 x i8> @unsigned_v16i16_to_v16i8(<16 x i16> %y) {
; CHECK-NEXT: uqxtn2 v0.16b, v1.8h
; CHECK-NEXT: ret
entry:
- %min = call <16 x i16> @llvm.umin.v8i16(<16 x i16> %y, <16 x i16> <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>)
+ %min = call <16 x i16> @llvm.umin.v16i16(<16 x i16> %y, <16 x i16> <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>)
%trunc = trunc <16 x i16> %min to <16 x i8>
ret <16 x i8> %trunc
}
@@ -528,7 +528,7 @@ define <8 x i16> @unsigned_v8i32_to_v8i16(<8 x i32> %y) {
; CHECK-NEXT: uqxtn2 v0.8h, v1.4s
; CHECK-NEXT: ret
entry:
- %min = call <8 x i32> @llvm.umin.v4i32(<8 x i32> %y, <8 x i32> <i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535>)
+ %min = call <8 x i32> @llvm.umin.v8i32(<8 x i32> %y, <8 x i32> <i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535>)
%trunc = trunc <8 x i32> %min to <8 x i16>
ret <8 x i16> %trunc
}
@@ -540,7 +540,7 @@ define <4 x i32> @unsigned_v4i64_to_v4i32(<4 x i64> %y) {
; CHECK-NEXT: uqxtn2 v0.4s, v1.2d
; CHECK-NEXT: ret
entry:
- %min = call <4 x i64> @llvm.umin.v2i64(<4 x i64> %y, <4 x i64> <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>)
+ %min = call <4 x i64> @llvm.umin.v4i64(<4 x i64> %y, <4 x i64> <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>)
%trunc = trunc <4 x i64> %min to <4 x i32>
ret <4 x i32> %trunc
}
diff --git a/llvm/test/CodeGen/AArch64/qshrn.ll b/llvm/test/CodeGen/AArch64/qshrn.ll
index a3ed524007335..630a9bf39ddad 100644
--- a/llvm/test/CodeGen/AArch64/qshrn.ll
+++ b/llvm/test/CodeGen/AArch64/qshrn.ll
@@ -236,8 +236,8 @@ define <16 x i8> @signed_minmax_v8i16_to_v16i8(<16 x i16> %x) {
; CHECK-NEXT: ret
entry:
%s = ashr <16 x i16> %x, <i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5>
- %min = call <16 x i16> @llvm.smin.v8i16(<16 x i16> %s, <16 x i16> <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>)
- %max = call <16 x i16> @llvm.smax.v8i16(<16 x i16> %min, <16 x i16> <i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128>)
+ %min = call <16 x i16> @llvm.smin.v16i16(<16 x i16> %s, <16 x i16> <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>)
+ %max = call <16 x i16> @llvm.smax.v16i16(<16 x i16> %min, <16 x i16> <i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128>)
%trunc = trunc <16 x i16> %max to <16 x i8>
ret <16 x i8> %trunc
}
@@ -250,7 +250,7 @@ define <16 x i8> @unsigned_minmax_v8i16_to_v16i8(<16 x i16> %x) {
; CHECK-NEXT: ret
entry:
%s = lshr <16 x i16> %x, <i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5>
- %min = call <16 x i16> @llvm.umin.v8i16(<16 x i16> %s, <16 x i16> <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>)
+ %min = call <16 x i16> @llvm.umin.v16i16(<16 x i16> %s, <16 x i16> <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>)
%trunc = trunc <16 x i16> %min to <16 x i8>
ret <16 x i8> %trunc
}
@@ -263,8 +263,8 @@ define <16 x i8> @unsigned_signed_minmax_v8i16_to_v16i8(<16 x i16> %x) {
; CHECK-NEXT: ret
entry:
%s = ashr <16 x i16> %x, <i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5, i16 5>
- %max = call <16 x i16> @llvm.smax.v8i16(<16 x i16> %s, <16 x i16> <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>)
- %min = call <16 x i16> @llvm.umin.v8i16(<16 x i16> %max, <16 x i16> <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>)
+ %max = call <16 x i16> @llvm.smax.v16i16(<16 x i16> %s, <16 x i16> <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>)
+ %min = call <16 x i16> @llvm.umin.v16i16(<16 x i16> %max, <16 x i16> <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>)
%trunc = trunc <16 x i16> %min to <16 x i8>
ret <16 x i8> %trunc
}
@@ -320,8 +320,8 @@ define <4 x i32> @signed_minmax_v4i64_to_v8i32(<4 x i64> %x) {
; CHECK-NEXT: ret
entry:
%s = ashr <4 x i64> %x, <i64 5, i64 5, i64 5, i64 5>
- %min = call <4 x i64> @llvm.smin.v8i64(<4 x i64> %s, <4 x i64> <i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647>)
- %max = call <4 x i64> @llvm.smax.v8i64(<4 x i64> %min, <4 x i64> <i64 -2147483648, i64 -2147483648, i64 -2147483648, i64 -2147483648>)
+ %min = call <4 x i64> @llvm.smin.v4i64(<4 x i64> %s, <4 x i64> <i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647>)
+ %max = call <4 x i64> @llvm.smax.v4i64(<4 x i64> %min, <4 x i64> <i64 -2147483648, i64 -2147483648, i64 -2147483648, i64 -2147483648>)
%trunc = trunc <4 x i64> %max to <4 x i32>
ret <4 x i32> %trunc
}
@@ -334,7 +334,7 @@ define <4 x i32> @unsigned_minmax_v4i64_to_v8i32(<4 x i64> %x) {
; CHECK-NEXT: ret
entry:
%s = lshr <4 x i64> %x, <i64 5, i64 5, i64 5, i64 5>
- %min = call <4 x i64> @llvm.umin.v8i64(<4 x i64> %s, <4 x i64> <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>)
+ %min = call <4 x i64> @llvm.umin.v4i64(<4 x i64> %s, <4 x i64> <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>)
%trunc = trunc <4 x i64> %min to <4 x i32>
ret <4 x i32> %trunc
}
@@ -347,8 +347,8 @@ define <4 x i32> @unsigned_signed_minmax_v4i64_to_v8i32(<4 x i64> %x) {
; CHECK-NEXT: ret
entry:
%s = ashr <4 x i64> %x, <i64 5, i64 5, i64 5, i64 5>
- %max = call <4 x i64> @llvm.smax.v8i64(<4 x i64> %s, <4 x i64> <i64 0, i64 0, i64 0, i64 0>)
- %min = call <4 x i64> @llvm.umin.v8i64(<4 x i64> %max, <4 x i64> <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>)
+ %max = call <4 x i64> @llvm.smax.v4i64(<4 x i64> %s, <4 x i64> <i64 0, i64 0, i64 0, i64 0>)
+ %min = call <4 x i64> @llvm.umin.v4i64(<4 x i64> %max, <4 x i64> <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>)
%trunc = trunc <4 x i64> %min to <4 x i32>
ret <4 x i32> %trunc
}
diff --git a/llvm/test/CodeGen/AArch64/rqshrn.ll b/llvm/test/CodeGen/AArch64/rqshrn.ll
index e7522f5cd3ab0..f31f13783ac71 100644
--- a/llvm/test/CodeGen/AArch64/rqshrn.ll
+++ b/llvm/test/CodeGen/AArch64/rqshrn.ll
@@ -237,8 +237,8 @@ entry:
%l = call <8 x i16> @llvm.aarch64.neon.srshl(<8 x i16> %x, <8 x i16> <i16 -5, i16 -5, i16 -5, i16 -5, i16 -5, i16 -5, i16 -5, i16 -5>)
%h = call <8 x i16> @llvm.aarch64.neon.srshl(<8 x i16> %y, <8 x i16> <i16 -5, i16 -5, i16 -5, i16 -5, i16 -5, i16 -5, i16 -5, i16 -5>)
%s = shufflevector <8 x i16> %l, <8 x i16> %h, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
- %min = call <16 x i16> @llvm.smin.v8i16(<16 x i16> %s, <16 x i16> <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>)
- %max = call <16 x i16> @llvm.smax.v8i16(<16 x i16> %min, <16 x i16> <i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128>)
+ %min = call <16 x i16> @llvm.smin.v16i16(<16 x i16> %s, <16 x i16> <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>)
+ %max = call <16 x i16> @llvm.smax.v16i16(<16 x i16> %min, <16 x i16> <i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128, i16 -128>)
%trunc = trunc <16 x i16> %max to <16 x i8>
ret <16 x i8> %trunc
}
@@ -253,7 +253,7 @@ entry:
%l = call <8 x i16> @llvm.aarch64.neon.urshl(<8 x i16> %x, <8 x i16> <i16 -5, i16 -5, i16 -5, i16 -5, i16 -5, i16 -5, i16 -5, i16 -5>)
%h = call <8 x i16> @llvm.aarch64.neon.urshl(<8 x i16> %y, <8 x i16> <i16 -5, i16 -5, i16 -5, i16 -5, i16 -5, i16 -5, i16 -5, i16 -5>)
%s = shufflevector <8 x i16> %l, <8 x i16> %h, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
- %min = call <16 x i16> @llvm.umin.v8i16(<16 x i16> %s, <16 x i16> <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>)
+ %min = call <16 x i16> @llvm.umin.v16i16(<16 x i16> %s, <16 x i16> <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>)
%trunc = trunc <16 x i16> %min to <16 x i8>
ret <16 x i8> %trunc
}
@@ -268,8 +268,8 @@ entry:
%l = call <8 x i16> @llvm.aarch64.neon.srshl(<8 x i16> %x, <8 x i16> <i16 -5, i16 -5, i16 -5, i16 -5, i16 -5, i16 -5, i16 -5, i16 -5>)
%h = call <8 x i16> @llvm.aarch64.neon.srshl(<8 x i16> %y, <8 x i16> <i16 -5, i16 -5, i16 -5, i16 -5, i16 -5, i16 -5, i16 -5, i16 -5>)
%s = shufflevector <8 x i16> %l, <8 x i16> %h, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
- %max = call <16 x i16> @llvm.smax.v8i16(<16 x i16> %s, <16 x i16> <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>)
- %min = call <16 x i16> @llvm.umin.v8i16(<16 x i16> %max, <16 x i16> <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>)
+ %max = call <16 x i16> @llvm.smax.v16i16(<16 x i16> %s, <16 x i16> <i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0, i16 0>)
+ %min = call <16 x i16> @llvm.umin.v16i16(<16 x i16> %max, <16 x i16> <i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255, i16 255>)
%trunc = trunc <16 x i16> %min to <16 x i8>
ret <16 x i8> %trunc
}
@@ -333,8 +333,8 @@ entry:
%l = call <2 x i64> @llvm.aarch64.neon.srshl.v2i64(<2 x i64> %x, <2 x i64> <i64 -5, i64 -5>)
%h = call <2 x i64> @llvm.aarch64.neon.srshl.v2i64(<2 x i64> %y, <2 x i64> <i64 -5, i64 -5>)
%s = shufflevector <2 x i64> %l, <2 x i64> %h, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
- %min = call <4 x i64> @llvm.smin.v8i64(<4 x i64> %s, <4 x i64> <i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647>)
- %max = call <4 x i64> @llvm.smax.v8i64(<4 x i64> %min, <4 x i64> <i64 -2147483648, i64 -2147483648, i64 -2147483648, i64 -2147483648>)
+ %min = call <4 x i64> @llvm.smin.v4i64(<4 x i64> %s, <4 x i64> <i64 2147483647, i64 2147483647, i64 2147483647, i64 2147483647>)
+ %max = call <4 x i64> @llvm.smax.v4i64(<4 x i64> %min, <4 x i64> <i64 -2147483648, i64 -2147483648, i64 -2147483648, i64 -2147483648>)
%trunc = trunc <4 x i64> %max to <4 x i32>
ret <4 x i32> %trunc
}
@@ -349,7 +349,7 @@ entry:
%l = call <2 x i64> @llvm.aarch64.neon.urshl.v2i64(<2 x i64> %x, <2 x i64> <i64 -5, i64 -5>)
%h = call <2 x i64> @llvm.aarch64.neon.urshl.v2i64(<2 x i64> %y, <2 x i64> <i64 -5, i64 -5>)
%s = shufflevector <2 x i64> %l, <2 x i64> %h, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
- %min = call <4 x i64> @llvm.umin.v8i64(<4 x i64> %s, <4 x i64> <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>)
+ %min = call <4 x i64> @llvm.umin.v4i64(<4 x i64> %s, <4 x i64> <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>)
%trunc = trunc <4 x i64> %min to <4 x i32>
ret <4 x i32> %trunc
}
@@ -364,8 +364,8 @@ entry:
%l = call <2 x i64> @llvm.aarch64.neon.srshl.v2i64(<2 x i64> %x, <2 x i64> <i64 -5, i64 -5>)
%h = call <2 x i64> @llvm.aarch64.neon.srshl.v2i64(<2 x i64> %y, <2 x i64> <i64 -5, i64 -5>)
%s = shufflevector <2 x i64> %l, <2 x i64> %h, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
- %max = call <4 x i64> @llvm.smax.v8i64(<4 x i64> %s, <4 x i64> <i64 0, i64 0, i64 0, i64 0>)
- %min = call <4 x i64> @llvm.umin.v8i64(<4 x i64> %max, <4 x i64> <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>)
+ %max = call <4 x i64> @llvm.smax.v4i64(<4 x i64> %s, <4 x i64> <i64 0, i64 0, i64 0, i64 0>)
+ %min = call <4 x i64> @llvm.umin.v4i64(<4 x i64> %max, <4 x i64> <i64 4294967295, i64 4294967295, i64 4294967295, i64 4294967295>)
%trunc = trunc <4 x i64> %min to <4 x i32>
ret <4 x i32> %trunc
}
diff --git a/llvm/test/CodeGen/AArch64/saturating-vec-smull.ll b/llvm/test/CodeGen/AArch64/saturating-vec-smull.ll
index a16eb65b32a71..c92ad65c5c810 100644
--- a/llvm/test/CodeGen/AArch64/saturating-vec-smull.ll
+++ b/llvm/test/CodeGen/AArch64/saturating-vec-smull.ll
@@ -35,16 +35,10 @@ define <2 x i16> @saturating_2xi16(<2 x i16> %a, <2 x i16> %b) {
}
define <4 x i16> @saturating_4xi16(<4 x i16> %a, <4 x i16> %b) {
-; CHECK-SD-LABEL: saturating_4xi16:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: sqdmulh v0.4h, v1.4h, v0.4h
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: saturating_4xi16:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: smull v0.4s, v1.4h, v0.4h
-; CHECK-GI-NEXT: sqshrn v0.4h, v0.4s, #15
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: saturating_4xi16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sqdmulh v0.4h, v1.4h, v0.4h
+; CHECK-NEXT: ret
%as = sext <4 x i16> %a to <4 x i32>
%bs = sext <4 x i16> %b to <4 x i32>
%m = mul nsw <4 x i32> %bs, %as
@@ -55,18 +49,10 @@ define <4 x i16> @saturating_4xi16(<4 x i16> %a, <4 x i16> %b) {
}
define <8 x i16> @saturating_8xi16(<8 x i16> %a, <8 x i16> %b) {
-; CHECK-SD-LABEL: saturating_8xi16:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: sqdmulh v0.8h, v1.8h, v0.8h
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: saturating_8xi16:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: smull v2.4s, v1.4h, v0.4h
-; CHECK-GI-NEXT: smull2 v1.4s, v1.8h, v0.8h
-; CHECK-GI-NEXT: sqshrn v0.4h, v2.4s, #15
-; CHECK-GI-NEXT: sqshrn2 v0.8h, v1.4s, #15
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: saturating_8xi16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sqdmulh v0.8h, v1.8h, v0.8h
+; CHECK-NEXT: ret
%as = sext <8 x i16> %a to <8 x i32>
%bs = sext <8 x i16> %b to <8 x i32>
%m = mul nsw <8 x i32> %bs, %as
@@ -77,16 +63,10 @@ define <8 x i16> @saturating_8xi16(<8 x i16> %a, <8 x i16> %b) {
}
define <2 x i32> @saturating_2xi32(<2 x i32> %a, <2 x i32> %b) {
-; CHECK-SD-LABEL: saturating_2xi32:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: sqdmulh v0.2s, v1.2s, v0.2s
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: saturating_2xi32:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: smull v0.2d, v1.2s, v0.2s
-; CHECK-GI-NEXT: sqshrn v0.2s, v0.2d, #31
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: saturating_2xi32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sqdmulh v0.2s, v1.2s, v0.2s
+; CHECK-NEXT: ret
%as = sext <2 x i32> %a to <2 x i64>
%bs = sext <2 x i32> %b to <2 x i64>
%m = mul nsw <2 x i64> %bs, %as
@@ -97,18 +77,10 @@ define <2 x i32> @saturating_2xi32(<2 x i32> %a, <2 x i32> %b) {
}
define <4 x i32> @saturating_4xi32(<4 x i32> %a, <4 x i32> %b) {
-; CHECK-SD-LABEL: saturating_4xi32:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: sqdmulh v0.4s, v1.4s, v0.4s
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: saturating_4xi32:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: smull v2.2d, v1.2s, v0.2s
-; CHECK-GI-NEXT: smull2 v1.2d, v1.4s, v0.4s
-; CHECK-GI-NEXT: sqshrn v0.2s, v2.2d, #31
-; CHECK-GI-NEXT: sqshrn2 v0.4s, v1.2d, #31
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: saturating_4xi32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sqdmulh v0.4s, v1.4s, v0.4s
+; CHECK-NEXT: ret
%as = sext <4 x i32> %a to <4 x i64>
%bs = sext <4 x i32> %b to <4 x i64>
%m = mul nsw <4 x i64> %bs, %as
@@ -127,14 +99,8 @@ define <8 x i32> @saturating_8xi32(<8 x i32> %a, <8 x i32> %b) {
;
; CHECK-GI-LABEL: saturating_8xi32:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: smull v4.2d, v2.2s, v0.2s
-; CHECK-GI-NEXT: smull v5.2d, v3.2s, v1.2s
-; CHECK-GI-NEXT: smull2 v2.2d, v2.4s, v0.4s
-; CHECK-GI-NEXT: smull2 v3.2d, v3.4s, v1.4s
-; CHECK-GI-NEXT: sqshrn v0.2s, v4.2d, #31
-; CHECK-GI-NEXT: sqshrn v1.2s, v5.2d, #31
-; CHECK-GI-NEXT: sqshrn2 v0.4s, v2.2d, #31
-; CHECK-GI-NEXT: sqshrn2 v1.4s, v3.2d, #31
+; CHECK-GI-NEXT: sqdmulh v0.4s, v2.4s, v0.4s
+; CHECK-GI-NEXT: sqdmulh v1.4s, v3.4s, v1.4s
; CHECK-GI-NEXT: ret
%as = sext <8 x i32> %a to <8 x i64>
%bs = sext <8 x i32> %b to <8 x i64>
diff --git a/llvm/test/CodeGen/AArch64/smul_fix_sat.ll b/llvm/test/CodeGen/AArch64/smul_fix_sat.ll
index 865b11100c4f5..744d1be126ff1 100644
--- a/llvm/test/CodeGen/AArch64/smul_fix_sat.ll
+++ b/llvm/test/CodeGen/AArch64/smul_fix_sat.ll
@@ -790,68 +790,38 @@ define <4 x i64> @vec_v4i64(<4 x i64> %x, <4 x i64> %y) {
}
define <8 x i16> @vec_sqdmulh_v8i16(<8 x i16> %x, <8 x i16> %y) {
-; CHECK-SD-LABEL: vec_sqdmulh_v8i16:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: sqdmulh v0.8h, v0.8h, v1.8h
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: vec_sqdmulh_v8i16:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: smull v2.4s, v0.4h, v1.4h
-; CHECK-GI-NEXT: smull2 v1.4s, v0.8h, v1.8h
-; CHECK-GI-NEXT: sqshrn v0.4h, v2.4s, #15
-; CHECK-GI-NEXT: sqshrn2 v0.8h, v1.4s, #15
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: vec_sqdmulh_v8i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sqdmulh v0.8h, v0.8h, v1.8h
+; CHECK-NEXT: ret
%tmp = call <8 x i16> @llvm.smul.fix.sat.v8i16(<8 x i16> %x, <8 x i16> %y, i32 15)
ret <8 x i16> %tmp
}
define <4 x i16> @vec_sqdmulh_v4i16(<4 x i16> %x, <4 x i16> %y) {
-; CHECK-SD-LABEL: vec_sqdmulh_v4i16:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: sqdmulh v0.4h, v0.4h, v1.4h
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: vec_sqdmulh_v4i16:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: smull v0.4s, v0.4h, v1.4h
-; CHECK-GI-NEXT: sqshrn v0.4h, v0.4s, #15
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: vec_sqdmulh_v4i16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sqdmulh v0.4h, v0.4h, v1.4h
+; CHECK-NEXT: ret
%tmp = call <4 x i16> @llvm.smul.fix.sat.v4i16(<4 x i16> %x, <4 x i16> %y, i32 15)
ret <4 x i16> %tmp
}
define <4 x i32> @vec_sqdmulh_v4i32(<4 x i32> %x, <4 x i32> %y) {
-; CHECK-SD-LABEL: vec_sqdmulh_v4i32:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: sqdmulh v0.4s, v0.4s, v1.4s
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: vec_sqdmulh_v4i32:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: smull v2.2d, v0.2s, v1.2s
-; CHECK-GI-NEXT: smull2 v1.2d, v0.4s, v1.4s
-; CHECK-GI-NEXT: sqshrn v0.2s, v2.2d, #31
-; CHECK-GI-NEXT: sqshrn2 v0.4s, v1.2d, #31
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: vec_sqdmulh_v4i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sqdmulh v0.4s, v0.4s, v1.4s
+; CHECK-NEXT: ret
%tmp = call <4 x i32> @llvm.smul.fix.sat.v4i32(<4 x i32> %x, <4 x i32> %y, i32 31)
ret <4 x i32> %tmp
}
define <2 x i32> @vec_sqdmulh_v2i32(<2 x i32> %x, <2 x i32> %y) {
-; CHECK-SD-LABEL: vec_sqdmulh_v2i32:
-; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: sqdmulh v0.2s, v0.2s, v1.2s
-; CHECK-SD-NEXT: ret
-;
-; CHECK-GI-LABEL: vec_sqdmulh_v2i32:
-; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: smull v0.2d, v0.2s, v1.2s
-; CHECK-GI-NEXT: sqshrn v0.2s, v0.2d, #31
-; CHECK-GI-NEXT: ret
+; CHECK-LABEL: vec_sqdmulh_v2i32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sqdmulh v0.2s, v0.2s, v1.2s
+; CHECK-NEXT: ret
%tmp = call <2 x i32> @llvm.smul.fix.sat.v2i32(<2 x i32> %x, <2 x i32> %y, i32 31)
ret <2 x i32> %tmp
}
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; CHECK: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/vector-minmax.ll b/llvm/test/CodeGen/AArch64/vector-minmax.ll
index 6696f94d404c5..a026d23e19cc6 100644
--- a/llvm/test/CodeGen/AArch64/vector-minmax.ll
+++ b/llvm/test/CodeGen/AArch64/vector-minmax.ll
@@ -98,7 +98,7 @@ define <1 x i64> @smax_v1i64(<1 x i64> %a, <1 x i64> %b){
; CHECK-SVE-NEXT: // kill: def $d0 killed $d0 killed $z0
; CHECK-SVE-NEXT: ret
entry:
- %0 = call <1 x i64> @llvm.smax.v2i64(<1 x i64> %a, <1 x i64> %b)
+ %0 = call <1 x i64> @llvm.smax.v1i64(<1 x i64> %a, <1 x i64> %b)
ret <1 x i64> %0
}
diff --git a/llvm/test/CodeGen/AArch64/xar.ll b/llvm/test/CodeGen/AArch64/xar.ll
index 4f940a39dfe39..8399a7ad914a6 100644
--- a/llvm/test/CodeGen/AArch64/xar.ll
+++ b/llvm/test/CodeGen/AArch64/xar.ll
@@ -134,7 +134,7 @@ define <4 x i32> @xar_instead_of_or_v4i32(<4 x i32> %r) {
; SVE2-NEXT: // kill: def $q0 killed $q0 killed $z0
; SVE2-NEXT: ret
entry:
- %or = call <4 x i32> @llvm.fshl.v2i32(<4 x i32> %r, <4 x i32> %r, <4 x i32> splat (i32 25))
+ %or = call <4 x i32> @llvm.fshl.v4i32(<4 x i32> %r, <4 x i32> %r, <4 x i32> splat (i32 25))
ret <4 x i32> %or
}
@@ -161,7 +161,7 @@ define <8 x i16> @xar_instead_of_or_v8i16(<8 x i16> %r) {
; SVE2-NEXT: // kill: def $q0 killed $q0 killed $z0
; SVE2-NEXT: ret
entry:
- %or = call <8 x i16> @llvm.fshl.v2i16(<8 x i16> %r, <8 x i16> %r, <8 x i16> splat (i16 25))
+ %or = call <8 x i16> @llvm.fshl.v8i16(<8 x i16> %r, <8 x i16> %r, <8 x i16> splat (i16 25))
ret <8 x i16> %or
}
@@ -188,7 +188,7 @@ define <16 x i8> @xar_instead_of_or_v16i8(<16 x i8> %r) {
; SVE2-NEXT: // kill: def $q0 killed $q0 killed $z0
; SVE2-NEXT: ret
entry:
- %or = call <16 x i8> @llvm.fshl.v2i8(<16 x i8> %r, <16 x i8> %r, <16 x i8> splat (i8 25))
+ %or = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %r, <16 x i8> %r, <16 x i8> splat (i8 25))
ret <16 x i8> %or
}
@@ -448,8 +448,3 @@ define <4 x i16> @rev16_v4i16_shifts_r(<4 x i16> %a) {
%or = or <4 x i16> %shr, %shl
ret <4 x i16> %or
}
-
-declare <2 x i64> @llvm.fshl.v2i64(<2 x i64>, <2 x i64>, <2 x i64>)
-declare <4 x i32> @llvm.fshl.v4i32(<4 x i32>, <4 x i32>, <4 x i32>)
-declare <8 x i16> @llvm.fshl.v8i16(<8 x i16>, <8 x i16>, <8 x i16>)
-declare <16 x i8> @llvm.fshl.v16i8(<16 x i8>, <16 x i8>, <16 x i8>)
diff --git a/llvm/test/Instrumentation/MemorySanitizer/AArch64/arm64-tbl.ll b/llvm/test/Instrumentation/MemorySanitizer/AArch64/arm64-tbl.ll
index 3d6e7fa9ed4b8..b1db5e4ff8a1e 100644
--- a/llvm/test/Instrumentation/MemorySanitizer/AArch64/arm64-tbl.ll
+++ b/llvm/test/Instrumentation/MemorySanitizer/AArch64/arm64-tbl.ll
@@ -669,7 +669,7 @@ define <8 x i8> @tbx3_8b(<8 x i8> %A, <16 x i8> %B, <16 x i8> %C, <16 x i8> %D,
; CHECK-NEXT: store <8 x i8> [[_MSPROP]], ptr @__msan_retval_tls, align 8
; CHECK-NEXT: ret <8 x i8> [[OUT]]
;
- %out = call <8 x i8> @llvm.aarch64.neon.tbx3.v8i8(< 8 x i8> %A, <16 x i8> %B, <16 x i8> %C, <16 x i8> %D, <8 x i8> %E)
+ %out = call <8 x i8> @llvm.aarch64.neon.tbx3.v8i8(<8 x i8> %A, <16 x i8> %B, <16 x i8> %C, <16 x i8> %D, <8 x i8> %E)
ret <8 x i8> %out
}
More information about the llvm-commits
mailing list