[llvm] [GlobalISel] Add G_UDIV/G_SDIV computeKnownBits (PR #181307)
Alex Wang via llvm-commits
llvm-commits at lists.llvm.org
Thu Feb 12 21:51:55 PST 2026
https://github.com/aeft updated https://github.com/llvm/llvm-project/pull/181307
>From d55c40a09af1f146aef7662946b10465d5b6d5a4 Mon Sep 17 00:00:00 2001
From: Alex Wang <yesterda9 at gmail.com>
Date: Thu, 12 Feb 2026 20:39:36 -0800
Subject: [PATCH 1/2] [GlobalISel] Add G_UDIV/G_SDIV computeKnownBits
Code ported from SelectionDAG::computeKnownBits.
---
.../CodeGen/GlobalISel/GISelValueTracking.cpp | 18 +++++
.../AArch64/GlobalISel/knownbits-sdiv.mir | 67 +++++++++++++++++++
.../AArch64/GlobalISel/knownbits-udiv.mir | 55 +++++++++++++++
3 files changed, 140 insertions(+)
create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/knownbits-sdiv.mir
create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/knownbits-udiv.mir
diff --git a/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp b/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp
index fce06dfde0edc..6fe91406ce958 100644
--- a/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp
@@ -368,6 +368,24 @@ void GISelValueTracking::computeKnownBitsImpl(Register R, KnownBits &Known,
Known = KnownBits::mulhs(Known, Known2);
break;
}
+ case TargetOpcode::G_UDIV: {
+ computeKnownBitsImpl(MI.getOperand(1).getReg(), Known, DemandedElts,
+ Depth + 1);
+ computeKnownBitsImpl(MI.getOperand(2).getReg(), Known2, DemandedElts,
+ Depth + 1);
+ Known = KnownBits::udiv(Known, Known2,
+ MI.getFlag(MachineInstr::MIFlag::IsExact));
+ break;
+ }
+ case TargetOpcode::G_SDIV: {
+ computeKnownBitsImpl(MI.getOperand(1).getReg(), Known, DemandedElts,
+ Depth + 1);
+ computeKnownBitsImpl(MI.getOperand(2).getReg(), Known2, DemandedElts,
+ Depth + 1);
+ Known = KnownBits::sdiv(Known, Known2,
+ MI.getFlag(MachineInstr::MIFlag::IsExact));
+ break;
+ }
case TargetOpcode::G_SELECT: {
computeKnownBitsMin(MI.getOperand(2).getReg(), MI.getOperand(3).getReg(),
Known, DemandedElts, Depth + 1);
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/knownbits-sdiv.mir b/llvm/test/CodeGen/AArch64/GlobalISel/knownbits-sdiv.mir
new file mode 100644
index 0000000000000..7b315205bc2e8
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/knownbits-sdiv.mir
@@ -0,0 +1,67 @@
+# NOTE: Assertions have been autogenerated by utils/update_givaluetracking_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=aarch64 -passes='print<gisel-value-tracking>' -filetype=null %s 2>&1 | FileCheck %s
+
+---
+name: Cst
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: @Cst
+ ; CHECK-NEXT: %0:_ KnownBits:01100100 SignBits:1
+ ; CHECK-NEXT: %1:_ KnownBits:00000100 SignBits:5
+ ; CHECK-NEXT: %2:_ KnownBits:000????? SignBits:3
+ %0:_(s8) = G_CONSTANT i8 100
+ %1:_(s8) = G_CONSTANT i8 4
+ %2:_(s8) = G_SDIV %0, %1
+...
+---
+name: PartialKnown
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: @PartialKnown
+ ; CHECK-NEXT: %0:_ KnownBits:???????? SignBits:1
+ ; CHECK-NEXT: %1:_ KnownBits:00001111 SignBits:4
+ ; CHECK-NEXT: %2:_ KnownBits:0000???? SignBits:4
+ ; CHECK-NEXT: %3:_ KnownBits:00000100 SignBits:5
+ ; CHECK-NEXT: %4:_ KnownBits:000000?? SignBits:6
+ %0:_(s8) = G_IMPLICIT_DEF
+ %1:_(s8) = G_CONSTANT i8 15
+ %2:_(s8) = G_AND %0, %1
+ %3:_(s8) = G_CONSTANT i8 4
+ %4:_(s8) = G_SDIV %2, %3
+...
+---
+name: Exact
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: @Exact
+ ; CHECK-NEXT: %0:_ KnownBits:00001100 SignBits:4
+ ; CHECK-NEXT: %1:_ KnownBits:00000100 SignBits:5
+ ; CHECK-NEXT: %2:_ KnownBits:000000?1 SignBits:6
+ %0:_(s8) = G_CONSTANT i8 12
+ %1:_(s8) = G_CONSTANT i8 4
+ %2:_(s8) = exact G_SDIV %0, %1
+...
+---
+name: NegDividend
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: @NegDividend
+ ; CHECK-NEXT: %0:_ KnownBits:11110000 SignBits:4
+ ; CHECK-NEXT: %1:_ KnownBits:00000100 SignBits:5
+ ; CHECK-NEXT: %2:_ KnownBits:111111?? SignBits:6
+ %0:_(s8) = G_CONSTANT i8 -16
+ %1:_(s8) = G_CONSTANT i8 4
+ %2:_(s8) = G_SDIV %0, %1
+...
+---
+name: Unknown
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: @Unknown
+ ; CHECK-NEXT: %0:_ KnownBits:???????? SignBits:1
+ ; CHECK-NEXT: %1:_ KnownBits:???????? SignBits:1
+ ; CHECK-NEXT: %2:_ KnownBits:???????? SignBits:1
+ %0:_(s8) = COPY $b0
+ %1:_(s8) = COPY $b1
+ %2:_(s8) = G_SDIV %0, %1
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/knownbits-udiv.mir b/llvm/test/CodeGen/AArch64/GlobalISel/knownbits-udiv.mir
new file mode 100644
index 0000000000000..dfbcd3f3c6188
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/knownbits-udiv.mir
@@ -0,0 +1,55 @@
+# NOTE: Assertions have been autogenerated by utils/update_givaluetracking_test_checks.py UTC_ARGS: --version 6
+# RUN: llc -mtriple=aarch64 -passes='print<gisel-value-tracking>' -filetype=null %s 2>&1 | FileCheck %s
+
+---
+name: Cst
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: @Cst
+ ; CHECK-NEXT: %0:_ KnownBits:01100100 SignBits:1
+ ; CHECK-NEXT: %1:_ KnownBits:00000100 SignBits:5
+ ; CHECK-NEXT: %2:_ KnownBits:000????? SignBits:3
+ %0:_(s8) = G_CONSTANT i8 100
+ %1:_(s8) = G_CONSTANT i8 4
+ %2:_(s8) = G_UDIV %0, %1
+...
+---
+name: PartialKnown
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: @PartialKnown
+ ; CHECK-NEXT: %0:_ KnownBits:???????? SignBits:1
+ ; CHECK-NEXT: %1:_ KnownBits:00001111 SignBits:4
+ ; CHECK-NEXT: %2:_ KnownBits:0000???? SignBits:4
+ ; CHECK-NEXT: %3:_ KnownBits:00000100 SignBits:5
+ ; CHECK-NEXT: %4:_ KnownBits:000000?? SignBits:6
+ %0:_(s8) = G_IMPLICIT_DEF
+ %1:_(s8) = G_CONSTANT i8 15
+ %2:_(s8) = G_AND %0, %1
+ %3:_(s8) = G_CONSTANT i8 4
+ %4:_(s8) = G_UDIV %2, %3
+...
+---
+name: Exact
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: @Exact
+ ; CHECK-NEXT: %0:_ KnownBits:00001100 SignBits:4
+ ; CHECK-NEXT: %1:_ KnownBits:00000100 SignBits:5
+ ; CHECK-NEXT: %2:_ KnownBits:000000?1 SignBits:6
+ %0:_(s8) = G_CONSTANT i8 12
+ %1:_(s8) = G_CONSTANT i8 4
+ %2:_(s8) = exact G_UDIV %0, %1
+...
+---
+name: Unknown
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: @Unknown
+ ; CHECK-NEXT: %0:_ KnownBits:???????? SignBits:1
+ ; CHECK-NEXT: %1:_ KnownBits:???????? SignBits:1
+ ; CHECK-NEXT: %2:_ KnownBits:???????? SignBits:1
+ %0:_(s8) = COPY $b0
+ %1:_(s8) = COPY $b1
+ %2:_(s8) = G_UDIV %0, %1
+...
>From 7eab4f3f600d7e2ed2465f7a119c7c99a5d9c0fd Mon Sep 17 00:00:00 2001
From: Alex Wang <yesterda9 at gmail.com>
Date: Thu, 12 Feb 2026 21:51:41 -0800
Subject: [PATCH 2/2] fixup! [GlobalISel] Add G_UDIV/G_SDIV computeKnownBits
---
.../CodeGen/AArch64/arm64-neon-mul-div.ll | 304 ++++----
llvm/test/CodeGen/AArch64/funnel-shift.ll | 4 +-
llvm/test/CodeGen/AArch64/rem.ll | 720 +++++++++---------
3 files changed, 515 insertions(+), 513 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/arm64-neon-mul-div.ll b/llvm/test/CodeGen/AArch64/arm64-neon-mul-div.ll
index 27a116d64be64..cd101afe57c40 100644
--- a/llvm/test/CodeGen/AArch64/arm64-neon-mul-div.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-neon-mul-div.ll
@@ -2009,47 +2009,48 @@ define <8 x i8> @urem8x8(<8 x i8> %A, <8 x i8> %B) {
;
; CHECK-GI-LABEL: urem8x8:
; CHECK-GI: // %bb.0:
-; CHECK-GI-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT: ushll v2.8h, v0.8b, #0
; CHECK-GI-NEXT: ushll v1.8h, v1.8b, #0
-; CHECK-GI-NEXT: ushll v2.4s, v0.4h, #0
+; CHECK-GI-NEXT: ushll v0.4s, v2.4h, #0
; CHECK-GI-NEXT: ushll v3.4s, v1.4h, #0
-; CHECK-GI-NEXT: ushll2 v0.4s, v0.8h, #0
-; CHECK-GI-NEXT: ushll2 v1.4s, v1.8h, #0
-; CHECK-GI-NEXT: fmov w8, s2
+; CHECK-GI-NEXT: ushll2 v2.4s, v2.8h, #0
+; CHECK-GI-NEXT: mov d5, v1.d[1]
+; CHECK-GI-NEXT: fmov w8, s0
; CHECK-GI-NEXT: fmov w9, s3
-; CHECK-GI-NEXT: mov w10, v3.s[1]
-; CHECK-GI-NEXT: mov w11, v3.s[2]
-; CHECK-GI-NEXT: mov w12, v3.s[3]
-; CHECK-GI-NEXT: fmov w13, s1
-; CHECK-GI-NEXT: mov w14, v1.s[1]
-; CHECK-GI-NEXT: mov w15, v1.s[2]
-; CHECK-GI-NEXT: mov w16, v1.s[3]
+; CHECK-GI-NEXT: mov w10, v0.s[1]
+; CHECK-GI-NEXT: mov w11, v0.s[2]
+; CHECK-GI-NEXT: mov w12, v0.s[3]
+; CHECK-GI-NEXT: mov w14, v2.s[1]
+; CHECK-GI-NEXT: mov w15, v2.s[2]
+; CHECK-GI-NEXT: mov w16, v2.s[3]
; CHECK-GI-NEXT: udiv w8, w8, w9
-; CHECK-GI-NEXT: mov w9, v2.s[1]
-; CHECK-GI-NEXT: udiv w9, w9, w10
-; CHECK-GI-NEXT: mov w10, v2.s[2]
-; CHECK-GI-NEXT: fmov s4, w8
-; CHECK-GI-NEXT: udiv w10, w10, w11
-; CHECK-GI-NEXT: mov w11, v2.s[3]
-; CHECK-GI-NEXT: mov v4.s[1], w9
-; CHECK-GI-NEXT: udiv w11, w11, w12
-; CHECK-GI-NEXT: fmov w12, s0
-; CHECK-GI-NEXT: mov v4.s[2], w10
+; CHECK-GI-NEXT: mov w9, v3.s[1]
+; CHECK-GI-NEXT: udiv w9, w10, w9
+; CHECK-GI-NEXT: mov w10, v3.s[2]
+; CHECK-GI-NEXT: udiv w10, w11, w10
+; CHECK-GI-NEXT: mov w11, v3.s[3]
+; CHECK-GI-NEXT: ushll2 v3.4s, v1.8h, #0
+; CHECK-GI-NEXT: fmov w13, s3
+; CHECK-GI-NEXT: udiv w11, w12, w11
+; CHECK-GI-NEXT: fmov w12, s2
; CHECK-GI-NEXT: udiv w12, w12, w13
-; CHECK-GI-NEXT: mov w13, v0.s[1]
-; CHECK-GI-NEXT: mov v4.s[3], w11
-; CHECK-GI-NEXT: mls v2.4s, v4.4s, v3.4s
-; CHECK-GI-NEXT: udiv w13, w13, w14
-; CHECK-GI-NEXT: mov w14, v0.s[2]
-; CHECK-GI-NEXT: fmov s5, w12
-; CHECK-GI-NEXT: udiv w14, w14, w15
-; CHECK-GI-NEXT: mov w15, v0.s[3]
-; CHECK-GI-NEXT: mov v5.s[1], w13
-; CHECK-GI-NEXT: udiv w8, w15, w16
-; CHECK-GI-NEXT: mov v5.s[2], w14
-; CHECK-GI-NEXT: mov v5.s[3], w8
-; CHECK-GI-NEXT: mls v0.4s, v5.4s, v1.4s
-; CHECK-GI-NEXT: uzp1 v0.8h, v2.8h, v0.8h
+; CHECK-GI-NEXT: mov w13, v3.s[1]
+; CHECK-GI-NEXT: udiv w13, w14, w13
+; CHECK-GI-NEXT: mov w14, v3.s[2]
+; CHECK-GI-NEXT: fmov s4, w12
+; CHECK-GI-NEXT: udiv w14, w15, w14
+; CHECK-GI-NEXT: mov w15, v3.s[3]
+; CHECK-GI-NEXT: fmov s3, w8
+; CHECK-GI-NEXT: mov v4.h[1], w13
+; CHECK-GI-NEXT: mov v3.h[1], w9
+; CHECK-GI-NEXT: mov v3.h[2], w10
+; CHECK-GI-NEXT: udiv w8, w16, w15
+; CHECK-GI-NEXT: mov v4.h[2], w14
+; CHECK-GI-NEXT: mov v3.h[3], w11
+; CHECK-GI-NEXT: umlsl v0.4s, v3.4h, v1.4h
+; CHECK-GI-NEXT: mov v4.h[3], w8
+; CHECK-GI-NEXT: umlsl v2.4s, v4.4h, v5.4h
+; CHECK-GI-NEXT: uzp1 v0.8h, v0.8h, v2.8h
; CHECK-GI-NEXT: xtn v0.8b, v0.8h
; CHECK-GI-NEXT: ret
%tmp3 = urem <8 x i8> %A, %B;
@@ -2166,87 +2167,89 @@ define <16 x i8> @urem16x8(<16 x i8> %A, <16 x i8> %B) {
; CHECK-GI-LABEL: urem16x8:
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: ushll v4.8h, v0.8b, #0
-; CHECK-GI-NEXT: ushll v5.8h, v1.8b, #0
-; CHECK-GI-NEXT: ushll2 v6.8h, v0.16b, #0
-; CHECK-GI-NEXT: ushll2 v7.8h, v1.16b, #0
+; CHECK-GI-NEXT: ushll v3.8h, v1.8b, #0
+; CHECK-GI-NEXT: ushll2 v1.8h, v1.16b, #0
; CHECK-GI-NEXT: ushll v2.4s, v4.4h, #0
-; CHECK-GI-NEXT: ushll v3.4s, v5.4h, #0
+; CHECK-GI-NEXT: ushll v5.4s, v3.4h, #0
; CHECK-GI-NEXT: ushll2 v4.4s, v4.8h, #0
-; CHECK-GI-NEXT: ushll2 v5.4s, v5.8h, #0
-; CHECK-GI-NEXT: ushll v0.4s, v6.4h, #0
-; CHECK-GI-NEXT: ushll v1.4s, v7.4h, #0
-; CHECK-GI-NEXT: ushll2 v6.4s, v6.8h, #0
-; CHECK-GI-NEXT: ushll2 v7.4s, v7.8h, #0
+; CHECK-GI-NEXT: ushll v6.4s, v1.4h, #0
+; CHECK-GI-NEXT: mov d18, v3.d[1]
+; CHECK-GI-NEXT: mov d19, v1.d[1]
; CHECK-GI-NEXT: fmov w8, s2
-; CHECK-GI-NEXT: fmov w9, s3
-; CHECK-GI-NEXT: mov w12, v3.s[3]
-; CHECK-GI-NEXT: fmov w13, s5
-; CHECK-GI-NEXT: mov w16, v5.s[3]
-; CHECK-GI-NEXT: fmov w17, s1
-; CHECK-GI-NEXT: mov w18, v1.s[1]
-; CHECK-GI-NEXT: mov w0, v1.s[2]
-; CHECK-GI-NEXT: mov w1, v1.s[3]
+; CHECK-GI-NEXT: fmov w9, s5
+; CHECK-GI-NEXT: mov w12, v2.s[3]
+; CHECK-GI-NEXT: mov w16, v4.s[3]
+; CHECK-GI-NEXT: fmov w17, s6
; CHECK-GI-NEXT: udiv w10, w8, w9
-; CHECK-GI-NEXT: mov w8, v2.s[1]
-; CHECK-GI-NEXT: mov w9, v3.s[1]
-; CHECK-GI-NEXT: fmov w2, s7
-; CHECK-GI-NEXT: mov w3, v7.s[1]
-; CHECK-GI-NEXT: mov w4, v7.s[2]
-; CHECK-GI-NEXT: mov w5, v7.s[3]
-; CHECK-GI-NEXT: udiv w11, w8, w9
-; CHECK-GI-NEXT: mov w8, v2.s[2]
-; CHECK-GI-NEXT: mov w9, v3.s[2]
-; CHECK-GI-NEXT: fmov s16, w10
-; CHECK-GI-NEXT: udiv w9, w8, w9
-; CHECK-GI-NEXT: mov w8, v2.s[3]
-; CHECK-GI-NEXT: mov v16.s[1], w11
-; CHECK-GI-NEXT: udiv w8, w8, w12
+; CHECK-GI-NEXT: mov w8, v5.s[1]
+; CHECK-GI-NEXT: mov w9, v2.s[1]
+; CHECK-GI-NEXT: udiv w11, w9, w8
+; CHECK-GI-NEXT: mov w8, v5.s[2]
+; CHECK-GI-NEXT: mov w9, v2.s[2]
+; CHECK-GI-NEXT: udiv w9, w9, w8
+; CHECK-GI-NEXT: mov w8, v5.s[3]
+; CHECK-GI-NEXT: ushll2 v5.4s, v3.8h, #0
+; CHECK-GI-NEXT: fmov w13, s5
+; CHECK-GI-NEXT: udiv w8, w12, w8
; CHECK-GI-NEXT: fmov w12, s4
-; CHECK-GI-NEXT: mov v16.s[2], w9
; CHECK-GI-NEXT: udiv w14, w12, w13
-; CHECK-GI-NEXT: mov w12, v4.s[1]
-; CHECK-GI-NEXT: mov w13, v5.s[1]
-; CHECK-GI-NEXT: mov v16.s[3], w8
-; CHECK-GI-NEXT: mls v2.4s, v16.4s, v3.4s
-; CHECK-GI-NEXT: udiv w15, w12, w13
-; CHECK-GI-NEXT: mov w12, v4.s[2]
-; CHECK-GI-NEXT: mov w13, v5.s[2]
-; CHECK-GI-NEXT: fmov s17, w14
-; CHECK-GI-NEXT: udiv w13, w12, w13
-; CHECK-GI-NEXT: mov w12, v4.s[3]
-; CHECK-GI-NEXT: mov v17.s[1], w15
-; CHECK-GI-NEXT: udiv w12, w12, w16
+; CHECK-GI-NEXT: mov w12, v5.s[1]
+; CHECK-GI-NEXT: mov w13, v4.s[1]
+; CHECK-GI-NEXT: udiv w15, w13, w12
+; CHECK-GI-NEXT: mov w12, v5.s[2]
+; CHECK-GI-NEXT: mov w13, v4.s[2]
+; CHECK-GI-NEXT: fmov s7, w14
+; CHECK-GI-NEXT: udiv w13, w13, w12
+; CHECK-GI-NEXT: mov w12, v5.s[3]
+; CHECK-GI-NEXT: ushll2 v5.8h, v0.16b, #0
+; CHECK-GI-NEXT: mov v7.h[1], w15
+; CHECK-GI-NEXT: ushll v0.4s, v5.4h, #0
+; CHECK-GI-NEXT: ushll2 v5.4s, v5.8h, #0
+; CHECK-GI-NEXT: mov w18, v0.s[1]
+; CHECK-GI-NEXT: mov w0, v0.s[2]
+; CHECK-GI-NEXT: mov w1, v0.s[3]
+; CHECK-GI-NEXT: mov w3, v5.s[1]
+; CHECK-GI-NEXT: mov w4, v5.s[2]
+; CHECK-GI-NEXT: mov w5, v5.s[3]
+; CHECK-GI-NEXT: udiv w12, w16, w12
; CHECK-GI-NEXT: fmov w16, s0
-; CHECK-GI-NEXT: mov v17.s[2], w13
+; CHECK-GI-NEXT: mov v7.h[2], w13
; CHECK-GI-NEXT: udiv w16, w16, w17
-; CHECK-GI-NEXT: mov w17, v0.s[1]
-; CHECK-GI-NEXT: mov v17.s[3], w12
-; CHECK-GI-NEXT: mls v4.4s, v17.4s, v5.4s
-; CHECK-GI-NEXT: udiv w17, w17, w18
-; CHECK-GI-NEXT: mov w18, v0.s[2]
-; CHECK-GI-NEXT: fmov s18, w16
-; CHECK-GI-NEXT: udiv w18, w18, w0
-; CHECK-GI-NEXT: mov w0, v0.s[3]
-; CHECK-GI-NEXT: mov v18.s[1], w17
-; CHECK-GI-NEXT: udiv w0, w0, w1
-; CHECK-GI-NEXT: fmov w1, s6
-; CHECK-GI-NEXT: mov v18.s[2], w18
+; CHECK-GI-NEXT: mov w17, v6.s[1]
+; CHECK-GI-NEXT: mov v7.h[3], w12
+; CHECK-GI-NEXT: umlsl v4.4s, v7.4h, v18.4h
+; CHECK-GI-NEXT: udiv w17, w18, w17
+; CHECK-GI-NEXT: mov w18, v6.s[2]
+; CHECK-GI-NEXT: fmov s16, w16
+; CHECK-GI-NEXT: udiv w18, w0, w18
+; CHECK-GI-NEXT: mov w0, v6.s[3]
+; CHECK-GI-NEXT: ushll2 v6.4s, v1.8h, #0
+; CHECK-GI-NEXT: mov v16.h[1], w17
+; CHECK-GI-NEXT: fmov w2, s6
+; CHECK-GI-NEXT: udiv w0, w1, w0
+; CHECK-GI-NEXT: fmov w1, s5
+; CHECK-GI-NEXT: mov v16.h[2], w18
; CHECK-GI-NEXT: udiv w1, w1, w2
; CHECK-GI-NEXT: mov w2, v6.s[1]
-; CHECK-GI-NEXT: mov v18.s[3], w0
-; CHECK-GI-NEXT: mls v0.4s, v18.4s, v1.4s
-; CHECK-GI-NEXT: uzp1 v1.8h, v2.8h, v4.8h
-; CHECK-GI-NEXT: udiv w2, w2, w3
+; CHECK-GI-NEXT: mov v16.h[3], w0
+; CHECK-GI-NEXT: umlsl v0.4s, v16.4h, v1.4h
+; CHECK-GI-NEXT: udiv w2, w3, w2
; CHECK-GI-NEXT: mov w3, v6.s[2]
-; CHECK-GI-NEXT: fmov s19, w1
-; CHECK-GI-NEXT: udiv w3, w3, w4
+; CHECK-GI-NEXT: fmov s17, w1
+; CHECK-GI-NEXT: udiv w3, w4, w3
; CHECK-GI-NEXT: mov w4, v6.s[3]
-; CHECK-GI-NEXT: mov v19.s[1], w2
-; CHECK-GI-NEXT: udiv w10, w4, w5
-; CHECK-GI-NEXT: mov v19.s[2], w3
-; CHECK-GI-NEXT: mov v19.s[3], w10
-; CHECK-GI-NEXT: mls v6.4s, v19.4s, v7.4s
-; CHECK-GI-NEXT: uzp1 v0.8h, v0.8h, v6.8h
+; CHECK-GI-NEXT: fmov s6, w10
+; CHECK-GI-NEXT: mov v17.h[1], w2
+; CHECK-GI-NEXT: mov v6.h[1], w11
+; CHECK-GI-NEXT: mov v6.h[2], w9
+; CHECK-GI-NEXT: udiv w10, w5, w4
+; CHECK-GI-NEXT: mov v17.h[2], w3
+; CHECK-GI-NEXT: mov v6.h[3], w8
+; CHECK-GI-NEXT: umlsl v2.4s, v6.4h, v3.4h
+; CHECK-GI-NEXT: mov v17.h[3], w10
+; CHECK-GI-NEXT: uzp1 v1.8h, v2.8h, v4.8h
+; CHECK-GI-NEXT: umlsl v5.4s, v17.4h, v19.4h
+; CHECK-GI-NEXT: uzp1 v0.8h, v0.8h, v5.8h
; CHECK-GI-NEXT: uzp1 v0.16b, v1.16b, v0.16b
; CHECK-GI-NEXT: ret
%tmp3 = urem <16 x i8> %A, %B;
@@ -2312,24 +2315,24 @@ define <4 x i16> @urem4x16(<4 x i16> %A, <4 x i16> %B) {
; CHECK-GI-LABEL: urem4x16:
; CHECK-GI: // %bb.0:
; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0
-; CHECK-GI-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT: ushll v2.4s, v1.4h, #0
; CHECK-GI-NEXT: fmov w8, s0
-; CHECK-GI-NEXT: fmov w9, s1
-; CHECK-GI-NEXT: mov w10, v1.s[1]
-; CHECK-GI-NEXT: mov w11, v1.s[2]
-; CHECK-GI-NEXT: mov w12, v1.s[3]
+; CHECK-GI-NEXT: fmov w9, s2
+; CHECK-GI-NEXT: mov w10, v0.s[1]
+; CHECK-GI-NEXT: mov w11, v0.s[2]
+; CHECK-GI-NEXT: mov w12, v0.s[3]
; CHECK-GI-NEXT: udiv w8, w8, w9
-; CHECK-GI-NEXT: mov w9, v0.s[1]
-; CHECK-GI-NEXT: udiv w9, w9, w10
-; CHECK-GI-NEXT: mov w10, v0.s[2]
+; CHECK-GI-NEXT: mov w9, v2.s[1]
+; CHECK-GI-NEXT: udiv w9, w10, w9
+; CHECK-GI-NEXT: mov w10, v2.s[2]
+; CHECK-GI-NEXT: udiv w10, w11, w10
+; CHECK-GI-NEXT: mov w11, v2.s[3]
; CHECK-GI-NEXT: fmov s2, w8
-; CHECK-GI-NEXT: udiv w10, w10, w11
-; CHECK-GI-NEXT: mov w11, v0.s[3]
-; CHECK-GI-NEXT: mov v2.s[1], w9
-; CHECK-GI-NEXT: udiv w8, w11, w12
-; CHECK-GI-NEXT: mov v2.s[2], w10
-; CHECK-GI-NEXT: mov v2.s[3], w8
-; CHECK-GI-NEXT: mls v0.4s, v2.4s, v1.4s
+; CHECK-GI-NEXT: mov v2.h[1], w9
+; CHECK-GI-NEXT: udiv w8, w12, w11
+; CHECK-GI-NEXT: mov v2.h[2], w10
+; CHECK-GI-NEXT: mov v2.h[3], w8
+; CHECK-GI-NEXT: umlsl v0.4s, v2.4h, v1.4h
; CHECK-GI-NEXT: xtn v0.4h, v0.4s
; CHECK-GI-NEXT: ret
%tmp3 = urem <4 x i16> %A, %B;
@@ -2387,41 +2390,42 @@ define <8 x i16> @urem8x16(<8 x i16> %A, <8 x i16> %B) {
; CHECK-GI-NEXT: ushll v2.4s, v0.4h, #0
; CHECK-GI-NEXT: ushll v3.4s, v1.4h, #0
; CHECK-GI-NEXT: ushll2 v0.4s, v0.8h, #0
-; CHECK-GI-NEXT: ushll2 v1.4s, v1.8h, #0
+; CHECK-GI-NEXT: mov d5, v1.d[1]
; CHECK-GI-NEXT: fmov w8, s2
; CHECK-GI-NEXT: fmov w9, s3
-; CHECK-GI-NEXT: mov w10, v3.s[1]
-; CHECK-GI-NEXT: mov w11, v3.s[2]
-; CHECK-GI-NEXT: mov w12, v3.s[3]
-; CHECK-GI-NEXT: fmov w13, s1
-; CHECK-GI-NEXT: mov w14, v1.s[1]
-; CHECK-GI-NEXT: mov w15, v1.s[2]
-; CHECK-GI-NEXT: mov w16, v1.s[3]
+; CHECK-GI-NEXT: mov w10, v2.s[1]
+; CHECK-GI-NEXT: mov w11, v2.s[2]
+; CHECK-GI-NEXT: mov w12, v2.s[3]
+; CHECK-GI-NEXT: mov w14, v0.s[1]
+; CHECK-GI-NEXT: mov w15, v0.s[2]
+; CHECK-GI-NEXT: mov w16, v0.s[3]
; CHECK-GI-NEXT: udiv w8, w8, w9
-; CHECK-GI-NEXT: mov w9, v2.s[1]
-; CHECK-GI-NEXT: udiv w9, w9, w10
-; CHECK-GI-NEXT: mov w10, v2.s[2]
-; CHECK-GI-NEXT: fmov s4, w8
-; CHECK-GI-NEXT: udiv w10, w10, w11
-; CHECK-GI-NEXT: mov w11, v2.s[3]
-; CHECK-GI-NEXT: mov v4.s[1], w9
-; CHECK-GI-NEXT: udiv w11, w11, w12
+; CHECK-GI-NEXT: mov w9, v3.s[1]
+; CHECK-GI-NEXT: udiv w9, w10, w9
+; CHECK-GI-NEXT: mov w10, v3.s[2]
+; CHECK-GI-NEXT: udiv w10, w11, w10
+; CHECK-GI-NEXT: mov w11, v3.s[3]
+; CHECK-GI-NEXT: ushll2 v3.4s, v1.8h, #0
+; CHECK-GI-NEXT: fmov w13, s3
+; CHECK-GI-NEXT: udiv w11, w12, w11
; CHECK-GI-NEXT: fmov w12, s0
-; CHECK-GI-NEXT: mov v4.s[2], w10
; CHECK-GI-NEXT: udiv w12, w12, w13
-; CHECK-GI-NEXT: mov w13, v0.s[1]
-; CHECK-GI-NEXT: mov v4.s[3], w11
-; CHECK-GI-NEXT: mls v2.4s, v4.4s, v3.4s
-; CHECK-GI-NEXT: udiv w13, w13, w14
-; CHECK-GI-NEXT: mov w14, v0.s[2]
-; CHECK-GI-NEXT: fmov s5, w12
-; CHECK-GI-NEXT: udiv w14, w14, w15
-; CHECK-GI-NEXT: mov w15, v0.s[3]
-; CHECK-GI-NEXT: mov v5.s[1], w13
-; CHECK-GI-NEXT: udiv w8, w15, w16
-; CHECK-GI-NEXT: mov v5.s[2], w14
-; CHECK-GI-NEXT: mov v5.s[3], w8
-; CHECK-GI-NEXT: mls v0.4s, v5.4s, v1.4s
+; CHECK-GI-NEXT: mov w13, v3.s[1]
+; CHECK-GI-NEXT: udiv w13, w14, w13
+; CHECK-GI-NEXT: mov w14, v3.s[2]
+; CHECK-GI-NEXT: fmov s4, w12
+; CHECK-GI-NEXT: udiv w14, w15, w14
+; CHECK-GI-NEXT: mov w15, v3.s[3]
+; CHECK-GI-NEXT: fmov s3, w8
+; CHECK-GI-NEXT: mov v4.h[1], w13
+; CHECK-GI-NEXT: mov v3.h[1], w9
+; CHECK-GI-NEXT: mov v3.h[2], w10
+; CHECK-GI-NEXT: udiv w8, w16, w15
+; CHECK-GI-NEXT: mov v4.h[2], w14
+; CHECK-GI-NEXT: mov v3.h[3], w11
+; CHECK-GI-NEXT: umlsl v2.4s, v3.4h, v1.4h
+; CHECK-GI-NEXT: mov v4.h[3], w8
+; CHECK-GI-NEXT: umlsl v0.4s, v4.4h, v5.4h
; CHECK-GI-NEXT: uzp1 v0.8h, v2.8h, v0.8h
; CHECK-GI-NEXT: ret
%tmp3 = urem <8 x i16> %A, %B;
diff --git a/llvm/test/CodeGen/AArch64/funnel-shift.ll b/llvm/test/CodeGen/AArch64/funnel-shift.ll
index e0bbfc620e2f8..97fb0df34d190 100644
--- a/llvm/test/CodeGen/AArch64/funnel-shift.ll
+++ b/llvm/test/CodeGen/AArch64/funnel-shift.ll
@@ -149,7 +149,7 @@ define i37 @fshl_i37(i37 %x, i37 %y, i37 %z) {
; CHECK-GI-NEXT: mov w8, #37 // =0x25
; CHECK-GI-NEXT: and x9, x2, #0x1fffffffff
; CHECK-GI-NEXT: udiv x10, x9, x8
-; CHECK-GI-NEXT: msub x8, x10, x8, x9
+; CHECK-GI-NEXT: umsubl x8, w10, w8, x9
; CHECK-GI-NEXT: mov w9, #36 // =0x24
; CHECK-GI-NEXT: ubfx x10, x1, #1, #36
; CHECK-GI-NEXT: sub x9, x9, x8
@@ -364,7 +364,7 @@ define i37 @fshr_i37(i37 %x, i37 %y, i37 %z) {
; CHECK-GI-NEXT: and x9, x2, #0x1fffffffff
; CHECK-GI-NEXT: and x11, x1, #0x1fffffffff
; CHECK-GI-NEXT: udiv x10, x9, x8
-; CHECK-GI-NEXT: msub x8, x10, x8, x9
+; CHECK-GI-NEXT: umsubl x8, w10, w8, x9
; CHECK-GI-NEXT: mov w9, #36 // =0x24
; CHECK-GI-NEXT: lsl x10, x0, #1
; CHECK-GI-NEXT: sub x9, x9, x8
diff --git a/llvm/test/CodeGen/AArch64/rem.ll b/llvm/test/CodeGen/AArch64/rem.ll
index b557ba34107b7..5dc5aa52b9155 100644
--- a/llvm/test/CodeGen/AArch64/rem.ll
+++ b/llvm/test/CodeGen/AArch64/rem.ll
@@ -1271,47 +1271,48 @@ define <8 x i8> @uv8i8(<8 x i8> %d, <8 x i8> %e) {
;
; CHECK-GI-LABEL: uv8i8:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT: ushll v2.8h, v0.8b, #0
; CHECK-GI-NEXT: ushll v1.8h, v1.8b, #0
-; CHECK-GI-NEXT: ushll v2.4s, v0.4h, #0
+; CHECK-GI-NEXT: ushll v0.4s, v2.4h, #0
; CHECK-GI-NEXT: ushll v3.4s, v1.4h, #0
-; CHECK-GI-NEXT: ushll2 v0.4s, v0.8h, #0
-; CHECK-GI-NEXT: ushll2 v1.4s, v1.8h, #0
-; CHECK-GI-NEXT: fmov w8, s2
+; CHECK-GI-NEXT: ushll2 v2.4s, v2.8h, #0
+; CHECK-GI-NEXT: mov d5, v1.d[1]
+; CHECK-GI-NEXT: fmov w8, s0
; CHECK-GI-NEXT: fmov w9, s3
-; CHECK-GI-NEXT: mov w10, v3.s[1]
-; CHECK-GI-NEXT: mov w11, v3.s[2]
-; CHECK-GI-NEXT: mov w12, v3.s[3]
-; CHECK-GI-NEXT: fmov w13, s1
-; CHECK-GI-NEXT: mov w14, v1.s[1]
-; CHECK-GI-NEXT: mov w15, v1.s[2]
-; CHECK-GI-NEXT: mov w16, v1.s[3]
+; CHECK-GI-NEXT: mov w10, v0.s[1]
+; CHECK-GI-NEXT: mov w11, v0.s[2]
+; CHECK-GI-NEXT: mov w12, v0.s[3]
+; CHECK-GI-NEXT: mov w14, v2.s[1]
+; CHECK-GI-NEXT: mov w15, v2.s[2]
+; CHECK-GI-NEXT: mov w16, v2.s[3]
; CHECK-GI-NEXT: udiv w8, w8, w9
-; CHECK-GI-NEXT: mov w9, v2.s[1]
-; CHECK-GI-NEXT: udiv w9, w9, w10
-; CHECK-GI-NEXT: mov w10, v2.s[2]
-; CHECK-GI-NEXT: fmov s4, w8
-; CHECK-GI-NEXT: udiv w10, w10, w11
-; CHECK-GI-NEXT: mov w11, v2.s[3]
-; CHECK-GI-NEXT: mov v4.s[1], w9
-; CHECK-GI-NEXT: udiv w11, w11, w12
-; CHECK-GI-NEXT: fmov w12, s0
-; CHECK-GI-NEXT: mov v4.s[2], w10
+; CHECK-GI-NEXT: mov w9, v3.s[1]
+; CHECK-GI-NEXT: udiv w9, w10, w9
+; CHECK-GI-NEXT: mov w10, v3.s[2]
+; CHECK-GI-NEXT: udiv w10, w11, w10
+; CHECK-GI-NEXT: mov w11, v3.s[3]
+; CHECK-GI-NEXT: ushll2 v3.4s, v1.8h, #0
+; CHECK-GI-NEXT: fmov w13, s3
+; CHECK-GI-NEXT: udiv w11, w12, w11
+; CHECK-GI-NEXT: fmov w12, s2
; CHECK-GI-NEXT: udiv w12, w12, w13
-; CHECK-GI-NEXT: mov w13, v0.s[1]
-; CHECK-GI-NEXT: mov v4.s[3], w11
-; CHECK-GI-NEXT: mls v2.4s, v4.4s, v3.4s
-; CHECK-GI-NEXT: udiv w13, w13, w14
-; CHECK-GI-NEXT: mov w14, v0.s[2]
-; CHECK-GI-NEXT: fmov s5, w12
-; CHECK-GI-NEXT: udiv w14, w14, w15
-; CHECK-GI-NEXT: mov w15, v0.s[3]
-; CHECK-GI-NEXT: mov v5.s[1], w13
-; CHECK-GI-NEXT: udiv w8, w15, w16
-; CHECK-GI-NEXT: mov v5.s[2], w14
-; CHECK-GI-NEXT: mov v5.s[3], w8
-; CHECK-GI-NEXT: mls v0.4s, v5.4s, v1.4s
-; CHECK-GI-NEXT: uzp1 v0.8h, v2.8h, v0.8h
+; CHECK-GI-NEXT: mov w13, v3.s[1]
+; CHECK-GI-NEXT: udiv w13, w14, w13
+; CHECK-GI-NEXT: mov w14, v3.s[2]
+; CHECK-GI-NEXT: fmov s4, w12
+; CHECK-GI-NEXT: udiv w14, w15, w14
+; CHECK-GI-NEXT: mov w15, v3.s[3]
+; CHECK-GI-NEXT: fmov s3, w8
+; CHECK-GI-NEXT: mov v4.h[1], w13
+; CHECK-GI-NEXT: mov v3.h[1], w9
+; CHECK-GI-NEXT: mov v3.h[2], w10
+; CHECK-GI-NEXT: udiv w8, w16, w15
+; CHECK-GI-NEXT: mov v4.h[2], w14
+; CHECK-GI-NEXT: mov v3.h[3], w11
+; CHECK-GI-NEXT: umlsl v0.4s, v3.4h, v1.4h
+; CHECK-GI-NEXT: mov v4.h[3], w8
+; CHECK-GI-NEXT: umlsl v2.4s, v4.4h, v5.4h
+; CHECK-GI-NEXT: uzp1 v0.8h, v0.8h, v2.8h
; CHECK-GI-NEXT: xtn v0.8b, v0.8h
; CHECK-GI-NEXT: ret
entry:
@@ -1429,87 +1430,89 @@ define <16 x i8> @uv16i8(<16 x i8> %d, <16 x i8> %e) {
; CHECK-GI-LABEL: uv16i8:
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: ushll v4.8h, v0.8b, #0
-; CHECK-GI-NEXT: ushll v5.8h, v1.8b, #0
-; CHECK-GI-NEXT: ushll2 v6.8h, v0.16b, #0
-; CHECK-GI-NEXT: ushll2 v7.8h, v1.16b, #0
+; CHECK-GI-NEXT: ushll v3.8h, v1.8b, #0
+; CHECK-GI-NEXT: ushll2 v1.8h, v1.16b, #0
; CHECK-GI-NEXT: ushll v2.4s, v4.4h, #0
-; CHECK-GI-NEXT: ushll v3.4s, v5.4h, #0
+; CHECK-GI-NEXT: ushll v5.4s, v3.4h, #0
; CHECK-GI-NEXT: ushll2 v4.4s, v4.8h, #0
-; CHECK-GI-NEXT: ushll2 v5.4s, v5.8h, #0
-; CHECK-GI-NEXT: ushll v0.4s, v6.4h, #0
-; CHECK-GI-NEXT: ushll v1.4s, v7.4h, #0
-; CHECK-GI-NEXT: ushll2 v6.4s, v6.8h, #0
-; CHECK-GI-NEXT: ushll2 v7.4s, v7.8h, #0
+; CHECK-GI-NEXT: ushll v6.4s, v1.4h, #0
+; CHECK-GI-NEXT: mov d18, v3.d[1]
+; CHECK-GI-NEXT: mov d19, v1.d[1]
; CHECK-GI-NEXT: fmov w8, s2
-; CHECK-GI-NEXT: fmov w9, s3
-; CHECK-GI-NEXT: mov w12, v3.s[3]
-; CHECK-GI-NEXT: fmov w13, s5
-; CHECK-GI-NEXT: mov w16, v5.s[3]
-; CHECK-GI-NEXT: fmov w17, s1
-; CHECK-GI-NEXT: mov w18, v1.s[1]
-; CHECK-GI-NEXT: mov w0, v1.s[2]
-; CHECK-GI-NEXT: mov w1, v1.s[3]
+; CHECK-GI-NEXT: fmov w9, s5
+; CHECK-GI-NEXT: mov w12, v2.s[3]
+; CHECK-GI-NEXT: mov w16, v4.s[3]
+; CHECK-GI-NEXT: fmov w17, s6
; CHECK-GI-NEXT: udiv w10, w8, w9
-; CHECK-GI-NEXT: mov w8, v2.s[1]
-; CHECK-GI-NEXT: mov w9, v3.s[1]
-; CHECK-GI-NEXT: fmov w2, s7
-; CHECK-GI-NEXT: mov w3, v7.s[1]
-; CHECK-GI-NEXT: mov w4, v7.s[2]
-; CHECK-GI-NEXT: mov w5, v7.s[3]
-; CHECK-GI-NEXT: udiv w11, w8, w9
-; CHECK-GI-NEXT: mov w8, v2.s[2]
-; CHECK-GI-NEXT: mov w9, v3.s[2]
-; CHECK-GI-NEXT: fmov s16, w10
-; CHECK-GI-NEXT: udiv w9, w8, w9
-; CHECK-GI-NEXT: mov w8, v2.s[3]
-; CHECK-GI-NEXT: mov v16.s[1], w11
-; CHECK-GI-NEXT: udiv w8, w8, w12
+; CHECK-GI-NEXT: mov w8, v5.s[1]
+; CHECK-GI-NEXT: mov w9, v2.s[1]
+; CHECK-GI-NEXT: udiv w11, w9, w8
+; CHECK-GI-NEXT: mov w8, v5.s[2]
+; CHECK-GI-NEXT: mov w9, v2.s[2]
+; CHECK-GI-NEXT: udiv w9, w9, w8
+; CHECK-GI-NEXT: mov w8, v5.s[3]
+; CHECK-GI-NEXT: ushll2 v5.4s, v3.8h, #0
+; CHECK-GI-NEXT: fmov w13, s5
+; CHECK-GI-NEXT: udiv w8, w12, w8
; CHECK-GI-NEXT: fmov w12, s4
-; CHECK-GI-NEXT: mov v16.s[2], w9
; CHECK-GI-NEXT: udiv w14, w12, w13
-; CHECK-GI-NEXT: mov w12, v4.s[1]
-; CHECK-GI-NEXT: mov w13, v5.s[1]
-; CHECK-GI-NEXT: mov v16.s[3], w8
-; CHECK-GI-NEXT: mls v2.4s, v16.4s, v3.4s
-; CHECK-GI-NEXT: udiv w15, w12, w13
-; CHECK-GI-NEXT: mov w12, v4.s[2]
-; CHECK-GI-NEXT: mov w13, v5.s[2]
-; CHECK-GI-NEXT: fmov s17, w14
-; CHECK-GI-NEXT: udiv w13, w12, w13
-; CHECK-GI-NEXT: mov w12, v4.s[3]
-; CHECK-GI-NEXT: mov v17.s[1], w15
-; CHECK-GI-NEXT: udiv w12, w12, w16
+; CHECK-GI-NEXT: mov w12, v5.s[1]
+; CHECK-GI-NEXT: mov w13, v4.s[1]
+; CHECK-GI-NEXT: udiv w15, w13, w12
+; CHECK-GI-NEXT: mov w12, v5.s[2]
+; CHECK-GI-NEXT: mov w13, v4.s[2]
+; CHECK-GI-NEXT: fmov s7, w14
+; CHECK-GI-NEXT: udiv w13, w13, w12
+; CHECK-GI-NEXT: mov w12, v5.s[3]
+; CHECK-GI-NEXT: ushll2 v5.8h, v0.16b, #0
+; CHECK-GI-NEXT: mov v7.h[1], w15
+; CHECK-GI-NEXT: ushll v0.4s, v5.4h, #0
+; CHECK-GI-NEXT: ushll2 v5.4s, v5.8h, #0
+; CHECK-GI-NEXT: mov w18, v0.s[1]
+; CHECK-GI-NEXT: mov w0, v0.s[2]
+; CHECK-GI-NEXT: mov w1, v0.s[3]
+; CHECK-GI-NEXT: mov w3, v5.s[1]
+; CHECK-GI-NEXT: mov w4, v5.s[2]
+; CHECK-GI-NEXT: mov w5, v5.s[3]
+; CHECK-GI-NEXT: udiv w12, w16, w12
; CHECK-GI-NEXT: fmov w16, s0
-; CHECK-GI-NEXT: mov v17.s[2], w13
+; CHECK-GI-NEXT: mov v7.h[2], w13
; CHECK-GI-NEXT: udiv w16, w16, w17
-; CHECK-GI-NEXT: mov w17, v0.s[1]
-; CHECK-GI-NEXT: mov v17.s[3], w12
-; CHECK-GI-NEXT: mls v4.4s, v17.4s, v5.4s
-; CHECK-GI-NEXT: udiv w17, w17, w18
-; CHECK-GI-NEXT: mov w18, v0.s[2]
-; CHECK-GI-NEXT: fmov s18, w16
-; CHECK-GI-NEXT: udiv w18, w18, w0
-; CHECK-GI-NEXT: mov w0, v0.s[3]
-; CHECK-GI-NEXT: mov v18.s[1], w17
-; CHECK-GI-NEXT: udiv w0, w0, w1
-; CHECK-GI-NEXT: fmov w1, s6
-; CHECK-GI-NEXT: mov v18.s[2], w18
+; CHECK-GI-NEXT: mov w17, v6.s[1]
+; CHECK-GI-NEXT: mov v7.h[3], w12
+; CHECK-GI-NEXT: umlsl v4.4s, v7.4h, v18.4h
+; CHECK-GI-NEXT: udiv w17, w18, w17
+; CHECK-GI-NEXT: mov w18, v6.s[2]
+; CHECK-GI-NEXT: fmov s16, w16
+; CHECK-GI-NEXT: udiv w18, w0, w18
+; CHECK-GI-NEXT: mov w0, v6.s[3]
+; CHECK-GI-NEXT: ushll2 v6.4s, v1.8h, #0
+; CHECK-GI-NEXT: mov v16.h[1], w17
+; CHECK-GI-NEXT: fmov w2, s6
+; CHECK-GI-NEXT: udiv w0, w1, w0
+; CHECK-GI-NEXT: fmov w1, s5
+; CHECK-GI-NEXT: mov v16.h[2], w18
; CHECK-GI-NEXT: udiv w1, w1, w2
; CHECK-GI-NEXT: mov w2, v6.s[1]
-; CHECK-GI-NEXT: mov v18.s[3], w0
-; CHECK-GI-NEXT: mls v0.4s, v18.4s, v1.4s
-; CHECK-GI-NEXT: uzp1 v1.8h, v2.8h, v4.8h
-; CHECK-GI-NEXT: udiv w2, w2, w3
+; CHECK-GI-NEXT: mov v16.h[3], w0
+; CHECK-GI-NEXT: umlsl v0.4s, v16.4h, v1.4h
+; CHECK-GI-NEXT: udiv w2, w3, w2
; CHECK-GI-NEXT: mov w3, v6.s[2]
-; CHECK-GI-NEXT: fmov s19, w1
-; CHECK-GI-NEXT: udiv w3, w3, w4
+; CHECK-GI-NEXT: fmov s17, w1
+; CHECK-GI-NEXT: udiv w3, w4, w3
; CHECK-GI-NEXT: mov w4, v6.s[3]
-; CHECK-GI-NEXT: mov v19.s[1], w2
-; CHECK-GI-NEXT: udiv w10, w4, w5
-; CHECK-GI-NEXT: mov v19.s[2], w3
-; CHECK-GI-NEXT: mov v19.s[3], w10
-; CHECK-GI-NEXT: mls v6.4s, v19.4s, v7.4s
-; CHECK-GI-NEXT: uzp1 v0.8h, v0.8h, v6.8h
+; CHECK-GI-NEXT: fmov s6, w10
+; CHECK-GI-NEXT: mov v17.h[1], w2
+; CHECK-GI-NEXT: mov v6.h[1], w11
+; CHECK-GI-NEXT: mov v6.h[2], w9
+; CHECK-GI-NEXT: udiv w10, w5, w4
+; CHECK-GI-NEXT: mov v17.h[2], w3
+; CHECK-GI-NEXT: mov v6.h[3], w8
+; CHECK-GI-NEXT: umlsl v2.4s, v6.4h, v3.4h
+; CHECK-GI-NEXT: mov v17.h[3], w10
+; CHECK-GI-NEXT: uzp1 v1.8h, v2.8h, v4.8h
+; CHECK-GI-NEXT: umlsl v5.4s, v17.4h, v19.4h
+; CHECK-GI-NEXT: uzp1 v0.8h, v0.8h, v5.8h
; CHECK-GI-NEXT: uzp1 v0.16b, v1.16b, v0.16b
; CHECK-GI-NEXT: ret
entry:
@@ -1788,14 +1791,13 @@ define <32 x i8> @uv32i8(<32 x i8> %d, <32 x i8> %e) {
;
; CHECK-GI-LABEL: uv32i8:
; CHECK-GI: // %bb.0: // %entry
-; CHECK-GI-NEXT: sub sp, sp, #112
-; CHECK-GI-NEXT: stp x29, x30, [sp, #16] // 16-byte Folded Spill
-; CHECK-GI-NEXT: stp x28, x27, [sp, #32] // 16-byte Folded Spill
-; CHECK-GI-NEXT: stp x26, x25, [sp, #48] // 16-byte Folded Spill
-; CHECK-GI-NEXT: stp x24, x23, [sp, #64] // 16-byte Folded Spill
-; CHECK-GI-NEXT: stp x22, x21, [sp, #80] // 16-byte Folded Spill
-; CHECK-GI-NEXT: stp x20, x19, [sp, #96] // 16-byte Folded Spill
-; CHECK-GI-NEXT: .cfi_def_cfa_offset 112
+; CHECK-GI-NEXT: stp x29, x30, [sp, #-96]! // 16-byte Folded Spill
+; CHECK-GI-NEXT: stp x28, x27, [sp, #16] // 16-byte Folded Spill
+; CHECK-GI-NEXT: stp x26, x25, [sp, #32] // 16-byte Folded Spill
+; CHECK-GI-NEXT: stp x24, x23, [sp, #48] // 16-byte Folded Spill
+; CHECK-GI-NEXT: stp x22, x21, [sp, #64] // 16-byte Folded Spill
+; CHECK-GI-NEXT: stp x20, x19, [sp, #80] // 16-byte Folded Spill
+; CHECK-GI-NEXT: .cfi_def_cfa_offset 96
; CHECK-GI-NEXT: .cfi_offset w19, -8
; CHECK-GI-NEXT: .cfi_offset w20, -16
; CHECK-GI-NEXT: .cfi_offset w21, -24
@@ -1810,195 +1812,190 @@ define <32 x i8> @uv32i8(<32 x i8> %d, <32 x i8> %e) {
; CHECK-GI-NEXT: .cfi_offset w29, -96
; CHECK-GI-NEXT: ushll v4.8h, v0.8b, #0
; CHECK-GI-NEXT: ushll v5.8h, v2.8b, #0
-; CHECK-GI-NEXT: ushll v16.8h, v1.8b, #0
-; CHECK-GI-NEXT: ushll v17.8h, v3.8b, #0
+; CHECK-GI-NEXT: ushll v16.8h, v3.8b, #0
; CHECK-GI-NEXT: ushll v6.4s, v4.4h, #0
; CHECK-GI-NEXT: ushll v7.4s, v5.4h, #0
; CHECK-GI-NEXT: ushll2 v4.4s, v4.8h, #0
; CHECK-GI-NEXT: ushll2 v5.4s, v5.8h, #0
; CHECK-GI-NEXT: ushll v18.4s, v16.4h, #0
-; CHECK-GI-NEXT: ushll v19.4s, v17.4h, #0
; CHECK-GI-NEXT: ushll2 v16.4s, v16.8h, #0
-; CHECK-GI-NEXT: ushll2 v17.4s, v17.8h, #0
; CHECK-GI-NEXT: fmov w8, s6
; CHECK-GI-NEXT: fmov w9, s7
+; CHECK-GI-NEXT: mov w10, v7.s[1]
; CHECK-GI-NEXT: mov w12, v7.s[3]
; CHECK-GI-NEXT: fmov w13, s5
-; CHECK-GI-NEXT: mov w14, v5.s[1]
; CHECK-GI-NEXT: mov w16, v5.s[3]
-; CHECK-GI-NEXT: fmov w6, s19
-; CHECK-GI-NEXT: mov w7, v19.s[3]
-; CHECK-GI-NEXT: fmov w21, s17
-; CHECK-GI-NEXT: udiv w10, w8, w9
+; CHECK-GI-NEXT: fmov w6, s18
+; CHECK-GI-NEXT: mov w7, v18.s[3]
+; CHECK-GI-NEXT: fmov w21, s16
+; CHECK-GI-NEXT: udiv w9, w8, w9
; CHECK-GI-NEXT: mov w8, v6.s[1]
-; CHECK-GI-NEXT: mov w9, v7.s[1]
-; CHECK-GI-NEXT: mov w22, v17.s[3]
-; CHECK-GI-NEXT: udiv w11, w8, w9
+; CHECK-GI-NEXT: mov w24, v16.s[3]
+; CHECK-GI-NEXT: udiv w11, w8, w10
; CHECK-GI-NEXT: mov w8, v6.s[2]
-; CHECK-GI-NEXT: mov w9, v7.s[2]
-; CHECK-GI-NEXT: fmov s20, w10
-; CHECK-GI-NEXT: udiv w9, w8, w9
+; CHECK-GI-NEXT: mov w10, v7.s[2]
+; CHECK-GI-NEXT: fmov s19, w9
+; CHECK-GI-NEXT: udiv w10, w8, w10
; CHECK-GI-NEXT: mov w8, v6.s[3]
-; CHECK-GI-NEXT: ushll2 v6.8h, v0.16b, #0
-; CHECK-GI-NEXT: mov v20.s[1], w11
-; CHECK-GI-NEXT: ushll v0.8h, v0.8b, #0
-; CHECK-GI-NEXT: ushll v28.4s, v0.4h, #0
-; CHECK-GI-NEXT: ushll2 v0.4s, v0.8h, #0
+; CHECK-GI-NEXT: mov v19.h[1], w11
; CHECK-GI-NEXT: udiv w8, w8, w12
; CHECK-GI-NEXT: fmov w12, s4
-; CHECK-GI-NEXT: mov v20.s[2], w9
-; CHECK-GI-NEXT: udiv w13, w12, w13
+; CHECK-GI-NEXT: mov v19.h[2], w10
+; CHECK-GI-NEXT: udiv w14, w12, w13
; CHECK-GI-NEXT: mov w12, v4.s[1]
-; CHECK-GI-NEXT: str w8, [sp, #12] // 4-byte Spill
-; CHECK-GI-NEXT: ldr w11, [sp, #12] // 4-byte Reload
-; CHECK-GI-NEXT: mov v20.s[3], w11
-; CHECK-GI-NEXT: udiv w15, w12, w14
+; CHECK-GI-NEXT: mov w13, v5.s[1]
+; CHECK-GI-NEXT: mov v19.h[3], w8
+; CHECK-GI-NEXT: udiv w15, w12, w13
; CHECK-GI-NEXT: mov w12, v4.s[2]
-; CHECK-GI-NEXT: mov w14, v5.s[2]
-; CHECK-GI-NEXT: ushll v5.4s, v6.4h, #0
-; CHECK-GI-NEXT: fmov s21, w13
-; CHECK-GI-NEXT: udiv w14, w12, w14
+; CHECK-GI-NEXT: mov w13, v5.s[2]
+; CHECK-GI-NEXT: ushll2 v5.8h, v0.16b, #0
+; CHECK-GI-NEXT: fmov s20, w14
+; CHECK-GI-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-GI-NEXT: ushll v6.4s, v5.4h, #0
+; CHECK-GI-NEXT: ushll v26.4s, v0.4h, #0
+; CHECK-GI-NEXT: ushll2 v0.4s, v0.8h, #0
+; CHECK-GI-NEXT: udiv w13, w12, w13
; CHECK-GI-NEXT: mov w12, v4.s[3]
; CHECK-GI-NEXT: ushll2 v4.8h, v2.16b, #0
-; CHECK-GI-NEXT: mov v21.s[1], w15
+; CHECK-GI-NEXT: mov v20.h[1], w15
; CHECK-GI-NEXT: ushll v2.8h, v2.8b, #0
; CHECK-GI-NEXT: ushll v7.4s, v4.4h, #0
-; CHECK-GI-NEXT: ushll v30.4s, v2.4h, #0
-; CHECK-GI-NEXT: ushll2 v2.4s, v2.8h, #0
+; CHECK-GI-NEXT: mov d29, v4.d[1]
+; CHECK-GI-NEXT: mov d28, v2.d[1]
+; CHECK-GI-NEXT: umlsl v26.4s, v19.4h, v2.4h
; CHECK-GI-NEXT: fmov w17, s7
-; CHECK-GI-NEXT: mls v28.4s, v20.4s, v30.4s
; CHECK-GI-NEXT: udiv w12, w12, w16
-; CHECK-GI-NEXT: fmov w16, s5
-; CHECK-GI-NEXT: mov v21.s[2], w14
-; CHECK-GI-NEXT: udiv w18, w16, w17
-; CHECK-GI-NEXT: mov w16, v5.s[1]
+; CHECK-GI-NEXT: fmov w16, s6
+; CHECK-GI-NEXT: mov v20.h[2], w13
+; CHECK-GI-NEXT: udiv w0, w16, w17
+; CHECK-GI-NEXT: mov w16, v6.s[1]
; CHECK-GI-NEXT: mov w17, v7.s[1]
-; CHECK-GI-NEXT: mov v21.s[3], w12
-; CHECK-GI-NEXT: mls v0.4s, v21.4s, v2.4s
+; CHECK-GI-NEXT: mov v20.h[3], w12
+; CHECK-GI-NEXT: umlsl v0.4s, v20.4h, v28.4h
; CHECK-GI-NEXT: udiv w1, w16, w17
-; CHECK-GI-NEXT: mov w16, v5.s[2]
+; CHECK-GI-NEXT: mov w16, v6.s[2]
; CHECK-GI-NEXT: mov w17, v7.s[2]
-; CHECK-GI-NEXT: fmov s22, w18
-; CHECK-GI-NEXT: uzp1 v0.8h, v28.8h, v0.8h
-; CHECK-GI-NEXT: udiv w0, w16, w17
-; CHECK-GI-NEXT: mov w16, v5.s[3]
+; CHECK-GI-NEXT: fmov s21, w0
+; CHECK-GI-NEXT: uzp1 v0.8h, v26.8h, v0.8h
+; CHECK-GI-NEXT: udiv w18, w16, w17
+; CHECK-GI-NEXT: mov w16, v6.s[3]
; CHECK-GI-NEXT: mov w17, v7.s[3]
-; CHECK-GI-NEXT: ushll2 v5.4s, v6.8h, #0
+; CHECK-GI-NEXT: ushll2 v6.4s, v5.8h, #0
; CHECK-GI-NEXT: ushll2 v7.4s, v4.8h, #0
-; CHECK-GI-NEXT: mov v22.s[1], w1
-; CHECK-GI-NEXT: ushll v6.4s, v6.4h, #0
-; CHECK-GI-NEXT: ushll v4.4s, v4.4h, #0
+; CHECK-GI-NEXT: mov v21.h[1], w1
+; CHECK-GI-NEXT: ushll v5.4s, v5.4h, #0
; CHECK-GI-NEXT: fmov w2, s7
-; CHECK-GI-NEXT: mov w3, v7.s[3]
+; CHECK-GI-NEXT: mov w5, v6.s[3]
; CHECK-GI-NEXT: udiv w16, w16, w17
-; CHECK-GI-NEXT: fmov w17, s5
-; CHECK-GI-NEXT: mov v22.s[2], w0
-; CHECK-GI-NEXT: udiv w5, w17, w2
-; CHECK-GI-NEXT: mov w17, v5.s[1]
-; CHECK-GI-NEXT: mov w2, v7.s[1]
-; CHECK-GI-NEXT: mov v22.s[3], w16
-; CHECK-GI-NEXT: mls v6.4s, v22.4s, v4.4s
-; CHECK-GI-NEXT: udiv w4, w17, w2
-; CHECK-GI-NEXT: mov w17, v5.s[2]
-; CHECK-GI-NEXT: mov w2, v7.s[2]
-; CHECK-GI-NEXT: fmov s23, w5
-; CHECK-GI-NEXT: udiv w2, w17, w2
-; CHECK-GI-NEXT: mov w17, v5.s[3]
-; CHECK-GI-NEXT: mov v23.s[1], w4
-; CHECK-GI-NEXT: udiv w17, w17, w3
-; CHECK-GI-NEXT: fmov w3, s18
-; CHECK-GI-NEXT: mov v23.s[2], w2
-; CHECK-GI-NEXT: udiv w20, w3, w6
-; CHECK-GI-NEXT: mov w3, v18.s[1]
-; CHECK-GI-NEXT: mov w6, v19.s[1]
-; CHECK-GI-NEXT: mov v23.s[3], w17
-; CHECK-GI-NEXT: mls v5.4s, v23.4s, v7.4s
-; CHECK-GI-NEXT: udiv w19, w3, w6
-; CHECK-GI-NEXT: mov w3, v18.s[2]
-; CHECK-GI-NEXT: mov w6, v19.s[2]
-; CHECK-GI-NEXT: fmov s24, w20
-; CHECK-GI-NEXT: uzp1 v2.8h, v6.8h, v5.8h
-; CHECK-GI-NEXT: uzp1 v0.16b, v0.16b, v2.16b
-; CHECK-GI-NEXT: udiv w6, w3, w6
-; CHECK-GI-NEXT: mov w3, v18.s[3]
-; CHECK-GI-NEXT: mov v24.s[1], w19
-; CHECK-GI-NEXT: ldp x20, x19, [sp, #96] // 16-byte Folded Reload
-; CHECK-GI-NEXT: udiv w3, w3, w7
-; CHECK-GI-NEXT: fmov w7, s16
-; CHECK-GI-NEXT: mov v24.s[2], w6
+; CHECK-GI-NEXT: fmov w17, s6
+; CHECK-GI-NEXT: mov v21.h[2], w18
+; CHECK-GI-NEXT: udiv w3, w17, w2
+; CHECK-GI-NEXT: mov w17, v7.s[1]
+; CHECK-GI-NEXT: mov w2, v6.s[1]
+; CHECK-GI-NEXT: mov v21.h[3], w16
+; CHECK-GI-NEXT: umlsl v5.4s, v21.4h, v4.4h
+; CHECK-GI-NEXT: udiv w4, w2, w17
+; CHECK-GI-NEXT: mov w17, v7.s[2]
+; CHECK-GI-NEXT: mov w2, v6.s[2]
+; CHECK-GI-NEXT: udiv w2, w2, w17
+; CHECK-GI-NEXT: mov w17, v7.s[3]
+; CHECK-GI-NEXT: ushll v7.8h, v1.8b, #0
+; CHECK-GI-NEXT: ushll v17.4s, v7.4h, #0
+; CHECK-GI-NEXT: ushll2 v7.4s, v7.8h, #0
+; CHECK-GI-NEXT: udiv w17, w5, w17
+; CHECK-GI-NEXT: fmov w5, s17
+; CHECK-GI-NEXT: udiv w19, w5, w6
+; CHECK-GI-NEXT: mov w5, v17.s[1]
+; CHECK-GI-NEXT: mov w6, v18.s[1]
+; CHECK-GI-NEXT: udiv w20, w5, w6
+; CHECK-GI-NEXT: mov w5, v17.s[2]
+; CHECK-GI-NEXT: mov w6, v18.s[2]
+; CHECK-GI-NEXT: fmov s22, w19
+; CHECK-GI-NEXT: udiv w6, w5, w6
+; CHECK-GI-NEXT: mov w5, v17.s[3]
+; CHECK-GI-NEXT: mov v22.h[1], w20
+; CHECK-GI-NEXT: ldp x20, x19, [sp, #80] // 16-byte Folded Reload
+; CHECK-GI-NEXT: udiv w5, w5, w7
+; CHECK-GI-NEXT: fmov w7, s7
+; CHECK-GI-NEXT: mov v22.h[2], w6
+; CHECK-GI-NEXT: udiv w22, w7, w21
+; CHECK-GI-NEXT: mov w7, v7.s[1]
+; CHECK-GI-NEXT: mov w21, v16.s[1]
+; CHECK-GI-NEXT: mov v22.h[3], w5
; CHECK-GI-NEXT: udiv w23, w7, w21
-; CHECK-GI-NEXT: mov w7, v16.s[1]
-; CHECK-GI-NEXT: mov w21, v17.s[1]
-; CHECK-GI-NEXT: mov v24.s[3], w3
-; CHECK-GI-NEXT: udiv w24, w7, w21
-; CHECK-GI-NEXT: mov w7, v16.s[2]
-; CHECK-GI-NEXT: mov w21, v17.s[2]
-; CHECK-GI-NEXT: ushll2 v17.8h, v1.16b, #0
-; CHECK-GI-NEXT: fmov s25, w23
+; CHECK-GI-NEXT: mov w7, v7.s[2]
+; CHECK-GI-NEXT: mov w21, v16.s[2]
+; CHECK-GI-NEXT: ushll2 v16.8h, v1.16b, #0
+; CHECK-GI-NEXT: fmov s23, w22
; CHECK-GI-NEXT: ushll v1.8h, v1.8b, #0
-; CHECK-GI-NEXT: ushll v18.4s, v17.4h, #0
-; CHECK-GI-NEXT: ushll v29.4s, v1.4h, #0
+; CHECK-GI-NEXT: ushll v17.4s, v16.4h, #0
+; CHECK-GI-NEXT: ushll v27.4s, v1.4h, #0
; CHECK-GI-NEXT: ushll2 v1.4s, v1.8h, #0
; CHECK-GI-NEXT: udiv w21, w7, w21
-; CHECK-GI-NEXT: mov w7, v16.s[3]
-; CHECK-GI-NEXT: ushll2 v16.8h, v3.16b, #0
-; CHECK-GI-NEXT: mov v25.s[1], w24
+; CHECK-GI-NEXT: mov w7, v7.s[3]
+; CHECK-GI-NEXT: ushll2 v7.8h, v3.16b, #0
+; CHECK-GI-NEXT: mov v23.h[1], w23
; CHECK-GI-NEXT: ushll v3.8h, v3.8b, #0
-; CHECK-GI-NEXT: ldp x24, x23, [sp, #64] // 16-byte Folded Reload
-; CHECK-GI-NEXT: ushll v19.4s, v16.4h, #0
-; CHECK-GI-NEXT: ushll v31.4s, v3.4h, #0
-; CHECK-GI-NEXT: ushll2 v3.4s, v3.8h, #0
-; CHECK-GI-NEXT: fmov w25, s19
-; CHECK-GI-NEXT: mov w26, v19.s[1]
-; CHECK-GI-NEXT: mov w27, v19.s[2]
-; CHECK-GI-NEXT: mov w28, v19.s[3]
-; CHECK-GI-NEXT: ushll2 v19.4s, v16.8h, #0
+; CHECK-GI-NEXT: ushll v18.4s, v7.4h, #0
+; CHECK-GI-NEXT: mov d31, v7.d[1]
+; CHECK-GI-NEXT: mov d30, v3.d[1]
+; CHECK-GI-NEXT: umlsl v27.4s, v22.4h, v3.4h
+; CHECK-GI-NEXT: fmov w25, s18
+; CHECK-GI-NEXT: mov w28, v18.s[3]
+; CHECK-GI-NEXT: udiv w7, w7, w24
+; CHECK-GI-NEXT: fmov w24, s17
+; CHECK-GI-NEXT: mov v23.h[2], w21
+; CHECK-GI-NEXT: ldp x22, x21, [sp, #64] // 16-byte Folded Reload
+; CHECK-GI-NEXT: udiv w26, w24, w25
+; CHECK-GI-NEXT: mov w24, v17.s[1]
+; CHECK-GI-NEXT: mov w25, v18.s[1]
+; CHECK-GI-NEXT: mov v23.h[3], w7
+; CHECK-GI-NEXT: umlsl v1.4s, v23.4h, v30.4h
+; CHECK-GI-NEXT: udiv w27, w24, w25
+; CHECK-GI-NEXT: mov w24, v17.s[2]
+; CHECK-GI-NEXT: mov w25, v18.s[2]
+; CHECK-GI-NEXT: ushll2 v18.4s, v7.8h, #0
+; CHECK-GI-NEXT: fmov s24, w26
+; CHECK-GI-NEXT: uzp1 v1.8h, v27.8h, v1.8h
+; CHECK-GI-NEXT: fmov w29, s18
+; CHECK-GI-NEXT: mov w9, v18.s[2]
+; CHECK-GI-NEXT: mov w10, v18.s[3]
+; CHECK-GI-NEXT: udiv w25, w24, w25
+; CHECK-GI-NEXT: mov w24, v17.s[3]
+; CHECK-GI-NEXT: ushll2 v17.4s, v16.8h, #0
+; CHECK-GI-NEXT: mov v24.h[1], w27
; CHECK-GI-NEXT: ushll v16.4s, v16.4h, #0
-; CHECK-GI-NEXT: udiv w7, w7, w22
-; CHECK-GI-NEXT: fmov w22, s18
-; CHECK-GI-NEXT: mov v25.s[2], w21
-; CHECK-GI-NEXT: mls v29.4s, v24.4s, v31.4s
-; CHECK-GI-NEXT: fmov w29, s19
-; CHECK-GI-NEXT: mov w30, v19.s[1]
-; CHECK-GI-NEXT: mov w8, v19.s[2]
-; CHECK-GI-NEXT: mov w10, v19.s[3]
-; CHECK-GI-NEXT: udiv w25, w22, w25
-; CHECK-GI-NEXT: mov w22, v18.s[1]
-; CHECK-GI-NEXT: mov v25.s[3], w7
-; CHECK-GI-NEXT: mls v1.4s, v25.4s, v3.4s
-; CHECK-GI-NEXT: udiv w26, w22, w26
-; CHECK-GI-NEXT: mov w22, v18.s[2]
-; CHECK-GI-NEXT: fmov s26, w25
-; CHECK-GI-NEXT: uzp1 v1.8h, v29.8h, v1.8h
-; CHECK-GI-NEXT: udiv w27, w22, w27
-; CHECK-GI-NEXT: mov w22, v18.s[3]
-; CHECK-GI-NEXT: ushll2 v18.4s, v17.8h, #0
-; CHECK-GI-NEXT: mov v26.s[1], w26
-; CHECK-GI-NEXT: ushll v17.4s, v17.4h, #0
-; CHECK-GI-NEXT: ldp x26, x25, [sp, #48] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mov w9, v18.s[3]
-; CHECK-GI-NEXT: udiv w22, w22, w28
-; CHECK-GI-NEXT: fmov w28, s18
-; CHECK-GI-NEXT: mov v26.s[2], w27
+; CHECK-GI-NEXT: mov w30, v17.s[1]
+; CHECK-GI-NEXT: mov w11, v17.s[2]
+; CHECK-GI-NEXT: mov w8, v17.s[3]
+; CHECK-GI-NEXT: udiv w24, w24, w28
+; CHECK-GI-NEXT: fmov w28, s17
+; CHECK-GI-NEXT: mov v24.h[2], w25
+; CHECK-GI-NEXT: ldp x26, x25, [sp, #32] // 16-byte Folded Reload
; CHECK-GI-NEXT: udiv w28, w28, w29
; CHECK-GI-NEXT: mov w29, v18.s[1]
-; CHECK-GI-NEXT: mov v26.s[3], w22
-; CHECK-GI-NEXT: ldp x22, x21, [sp, #80] // 16-byte Folded Reload
-; CHECK-GI-NEXT: mls v17.4s, v26.4s, v16.4s
-; CHECK-GI-NEXT: udiv w29, w29, w30
-; CHECK-GI-NEXT: mov w30, v18.s[2]
-; CHECK-GI-NEXT: fmov s27, w28
-; CHECK-GI-NEXT: ldp x28, x27, [sp, #32] // 16-byte Folded Reload
-; CHECK-GI-NEXT: udiv w8, w30, w8
-; CHECK-GI-NEXT: mov v27.s[1], w29
-; CHECK-GI-NEXT: ldp x29, x30, [sp, #16] // 16-byte Folded Reload
-; CHECK-GI-NEXT: udiv w9, w9, w10
-; CHECK-GI-NEXT: mov v27.s[2], w8
-; CHECK-GI-NEXT: mov v27.s[3], w9
-; CHECK-GI-NEXT: mls v18.4s, v27.4s, v19.4s
-; CHECK-GI-NEXT: uzp1 v3.8h, v17.8h, v18.8h
+; CHECK-GI-NEXT: fmov s18, w3
+; CHECK-GI-NEXT: mov v24.h[3], w24
+; CHECK-GI-NEXT: ldp x24, x23, [sp, #48] // 16-byte Folded Reload
+; CHECK-GI-NEXT: mov v18.h[1], w4
+; CHECK-GI-NEXT: umlsl v16.4s, v24.4h, v7.4h
+; CHECK-GI-NEXT: mov v18.h[2], w2
+; CHECK-GI-NEXT: udiv w29, w30, w29
+; CHECK-GI-NEXT: fmov s25, w28
+; CHECK-GI-NEXT: ldp x28, x27, [sp, #16] // 16-byte Folded Reload
+; CHECK-GI-NEXT: mov v18.h[3], w17
+; CHECK-GI-NEXT: umlsl v6.4s, v18.4h, v29.4h
+; CHECK-GI-NEXT: udiv w9, w11, w9
+; CHECK-GI-NEXT: mov v25.h[1], w29
+; CHECK-GI-NEXT: uzp1 v2.8h, v5.8h, v6.8h
+; CHECK-GI-NEXT: uzp1 v0.16b, v0.16b, v2.16b
+; CHECK-GI-NEXT: udiv w8, w8, w10
+; CHECK-GI-NEXT: mov v25.h[2], w9
+; CHECK-GI-NEXT: mov v25.h[3], w8
+; CHECK-GI-NEXT: umlsl v17.4s, v25.4h, v31.4h
+; CHECK-GI-NEXT: uzp1 v3.8h, v16.8h, v17.8h
; CHECK-GI-NEXT: uzp1 v1.16b, v1.16b, v3.16b
-; CHECK-GI-NEXT: add sp, sp, #112
+; CHECK-GI-NEXT: ldp x29, x30, [sp], #96 // 16-byte Folded Reload
; CHECK-GI-NEXT: ret
entry:
%s = urem <32 x i8> %d, %e
@@ -2595,24 +2592,24 @@ define <4 x i16> @uv4i16(<4 x i16> %d, <4 x i16> %e) {
; CHECK-GI-LABEL: uv4i16:
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0
-; CHECK-GI-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT: ushll v2.4s, v1.4h, #0
; CHECK-GI-NEXT: fmov w8, s0
-; CHECK-GI-NEXT: fmov w9, s1
-; CHECK-GI-NEXT: mov w10, v1.s[1]
-; CHECK-GI-NEXT: mov w11, v1.s[2]
-; CHECK-GI-NEXT: mov w12, v1.s[3]
+; CHECK-GI-NEXT: fmov w9, s2
+; CHECK-GI-NEXT: mov w10, v0.s[1]
+; CHECK-GI-NEXT: mov w11, v0.s[2]
+; CHECK-GI-NEXT: mov w12, v0.s[3]
; CHECK-GI-NEXT: udiv w8, w8, w9
-; CHECK-GI-NEXT: mov w9, v0.s[1]
-; CHECK-GI-NEXT: udiv w9, w9, w10
-; CHECK-GI-NEXT: mov w10, v0.s[2]
+; CHECK-GI-NEXT: mov w9, v2.s[1]
+; CHECK-GI-NEXT: udiv w9, w10, w9
+; CHECK-GI-NEXT: mov w10, v2.s[2]
+; CHECK-GI-NEXT: udiv w10, w11, w10
+; CHECK-GI-NEXT: mov w11, v2.s[3]
; CHECK-GI-NEXT: fmov s2, w8
-; CHECK-GI-NEXT: udiv w10, w10, w11
-; CHECK-GI-NEXT: mov w11, v0.s[3]
-; CHECK-GI-NEXT: mov v2.s[1], w9
-; CHECK-GI-NEXT: udiv w8, w11, w12
-; CHECK-GI-NEXT: mov v2.s[2], w10
-; CHECK-GI-NEXT: mov v2.s[3], w8
-; CHECK-GI-NEXT: mls v0.4s, v2.4s, v1.4s
+; CHECK-GI-NEXT: mov v2.h[1], w9
+; CHECK-GI-NEXT: udiv w8, w12, w11
+; CHECK-GI-NEXT: mov v2.h[2], w10
+; CHECK-GI-NEXT: mov v2.h[3], w8
+; CHECK-GI-NEXT: umlsl v0.4s, v2.4h, v1.4h
; CHECK-GI-NEXT: xtn v0.4h, v0.4s
; CHECK-GI-NEXT: ret
entry:
@@ -2671,41 +2668,42 @@ define <8 x i16> @uv8i16(<8 x i16> %d, <8 x i16> %e) {
; CHECK-GI-NEXT: ushll v2.4s, v0.4h, #0
; CHECK-GI-NEXT: ushll v3.4s, v1.4h, #0
; CHECK-GI-NEXT: ushll2 v0.4s, v0.8h, #0
-; CHECK-GI-NEXT: ushll2 v1.4s, v1.8h, #0
+; CHECK-GI-NEXT: mov d5, v1.d[1]
; CHECK-GI-NEXT: fmov w8, s2
; CHECK-GI-NEXT: fmov w9, s3
-; CHECK-GI-NEXT: mov w10, v3.s[1]
-; CHECK-GI-NEXT: mov w11, v3.s[2]
-; CHECK-GI-NEXT: mov w12, v3.s[3]
-; CHECK-GI-NEXT: fmov w13, s1
-; CHECK-GI-NEXT: mov w14, v1.s[1]
-; CHECK-GI-NEXT: mov w15, v1.s[2]
-; CHECK-GI-NEXT: mov w16, v1.s[3]
+; CHECK-GI-NEXT: mov w10, v2.s[1]
+; CHECK-GI-NEXT: mov w11, v2.s[2]
+; CHECK-GI-NEXT: mov w12, v2.s[3]
+; CHECK-GI-NEXT: mov w14, v0.s[1]
+; CHECK-GI-NEXT: mov w15, v0.s[2]
+; CHECK-GI-NEXT: mov w16, v0.s[3]
; CHECK-GI-NEXT: udiv w8, w8, w9
-; CHECK-GI-NEXT: mov w9, v2.s[1]
-; CHECK-GI-NEXT: udiv w9, w9, w10
-; CHECK-GI-NEXT: mov w10, v2.s[2]
-; CHECK-GI-NEXT: fmov s4, w8
-; CHECK-GI-NEXT: udiv w10, w10, w11
-; CHECK-GI-NEXT: mov w11, v2.s[3]
-; CHECK-GI-NEXT: mov v4.s[1], w9
-; CHECK-GI-NEXT: udiv w11, w11, w12
+; CHECK-GI-NEXT: mov w9, v3.s[1]
+; CHECK-GI-NEXT: udiv w9, w10, w9
+; CHECK-GI-NEXT: mov w10, v3.s[2]
+; CHECK-GI-NEXT: udiv w10, w11, w10
+; CHECK-GI-NEXT: mov w11, v3.s[3]
+; CHECK-GI-NEXT: ushll2 v3.4s, v1.8h, #0
+; CHECK-GI-NEXT: fmov w13, s3
+; CHECK-GI-NEXT: udiv w11, w12, w11
; CHECK-GI-NEXT: fmov w12, s0
-; CHECK-GI-NEXT: mov v4.s[2], w10
; CHECK-GI-NEXT: udiv w12, w12, w13
-; CHECK-GI-NEXT: mov w13, v0.s[1]
-; CHECK-GI-NEXT: mov v4.s[3], w11
-; CHECK-GI-NEXT: mls v2.4s, v4.4s, v3.4s
-; CHECK-GI-NEXT: udiv w13, w13, w14
-; CHECK-GI-NEXT: mov w14, v0.s[2]
-; CHECK-GI-NEXT: fmov s5, w12
-; CHECK-GI-NEXT: udiv w14, w14, w15
-; CHECK-GI-NEXT: mov w15, v0.s[3]
-; CHECK-GI-NEXT: mov v5.s[1], w13
-; CHECK-GI-NEXT: udiv w8, w15, w16
-; CHECK-GI-NEXT: mov v5.s[2], w14
-; CHECK-GI-NEXT: mov v5.s[3], w8
-; CHECK-GI-NEXT: mls v0.4s, v5.4s, v1.4s
+; CHECK-GI-NEXT: mov w13, v3.s[1]
+; CHECK-GI-NEXT: udiv w13, w14, w13
+; CHECK-GI-NEXT: mov w14, v3.s[2]
+; CHECK-GI-NEXT: fmov s4, w12
+; CHECK-GI-NEXT: udiv w14, w15, w14
+; CHECK-GI-NEXT: mov w15, v3.s[3]
+; CHECK-GI-NEXT: fmov s3, w8
+; CHECK-GI-NEXT: mov v4.h[1], w13
+; CHECK-GI-NEXT: mov v3.h[1], w9
+; CHECK-GI-NEXT: mov v3.h[2], w10
+; CHECK-GI-NEXT: udiv w8, w16, w15
+; CHECK-GI-NEXT: mov v4.h[2], w14
+; CHECK-GI-NEXT: mov v3.h[3], w11
+; CHECK-GI-NEXT: umlsl v2.4s, v3.4h, v1.4h
+; CHECK-GI-NEXT: mov v4.h[3], w8
+; CHECK-GI-NEXT: umlsl v0.4s, v4.4h, v5.4h
; CHECK-GI-NEXT: uzp1 v0.8h, v2.8h, v0.8h
; CHECK-GI-NEXT: ret
entry:
@@ -2852,86 +2850,86 @@ define <16 x i16> @uv16i16(<16 x i16> %d, <16 x i16> %e) {
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: ushll v4.4s, v0.4h, #0
; CHECK-GI-NEXT: ushll v5.4s, v2.4h, #0
-; CHECK-GI-NEXT: ushll v6.4s, v1.4h, #0
-; CHECK-GI-NEXT: ushll v7.4s, v3.4h, #0
+; CHECK-GI-NEXT: ushll v6.4s, v3.4h, #0
+; CHECK-GI-NEXT: mov d18, v2.d[1]
+; CHECK-GI-NEXT: mov d19, v3.d[1]
; CHECK-GI-NEXT: fmov w8, s4
; CHECK-GI-NEXT: fmov w9, s5
; CHECK-GI-NEXT: mov w12, v5.s[3]
-; CHECK-GI-NEXT: fmov w17, s7
-; CHECK-GI-NEXT: mov w18, v7.s[1]
-; CHECK-GI-NEXT: mov w0, v7.s[2]
-; CHECK-GI-NEXT: mov w1, v7.s[3]
-; CHECK-GI-NEXT: ushll2 v7.4s, v3.8h, #0
-; CHECK-GI-NEXT: ushll v3.4s, v3.4h, #0
+; CHECK-GI-NEXT: fmov w17, s6
+; CHECK-GI-NEXT: mov w18, v6.s[1]
+; CHECK-GI-NEXT: mov w0, v6.s[2]
+; CHECK-GI-NEXT: mov w1, v6.s[3]
+; CHECK-GI-NEXT: ushll2 v6.4s, v3.8h, #0
; CHECK-GI-NEXT: udiv w10, w8, w9
; CHECK-GI-NEXT: mov w8, v4.s[1]
; CHECK-GI-NEXT: mov w9, v5.s[1]
-; CHECK-GI-NEXT: fmov w2, s7
-; CHECK-GI-NEXT: mov w3, v7.s[1]
-; CHECK-GI-NEXT: mov w4, v7.s[2]
-; CHECK-GI-NEXT: mov w5, v7.s[3]
+; CHECK-GI-NEXT: fmov w2, s6
; CHECK-GI-NEXT: udiv w11, w8, w9
; CHECK-GI-NEXT: mov w8, v4.s[2]
; CHECK-GI-NEXT: mov w9, v5.s[2]
; CHECK-GI-NEXT: ushll2 v5.4s, v2.8h, #0
-; CHECK-GI-NEXT: fmov s16, w10
-; CHECK-GI-NEXT: ushll v2.4s, v2.4h, #0
; CHECK-GI-NEXT: fmov w13, s5
-; CHECK-GI-NEXT: mov w14, v5.s[1]
-; CHECK-GI-NEXT: mov w15, v5.s[2]
-; CHECK-GI-NEXT: mov w16, v5.s[3]
; CHECK-GI-NEXT: udiv w9, w8, w9
; CHECK-GI-NEXT: mov w8, v4.s[3]
; CHECK-GI-NEXT: ushll2 v4.4s, v0.8h, #0
-; CHECK-GI-NEXT: mov v16.s[1], w11
; CHECK-GI-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-GI-NEXT: mov w16, v4.s[3]
; CHECK-GI-NEXT: udiv w8, w8, w12
; CHECK-GI-NEXT: fmov w12, s4
-; CHECK-GI-NEXT: mov v16.s[2], w9
-; CHECK-GI-NEXT: udiv w13, w12, w13
-; CHECK-GI-NEXT: mov w12, v4.s[1]
-; CHECK-GI-NEXT: mov v16.s[3], w8
-; CHECK-GI-NEXT: mls v0.4s, v16.4s, v2.4s
-; CHECK-GI-NEXT: udiv w14, w12, w14
-; CHECK-GI-NEXT: mov w12, v4.s[2]
-; CHECK-GI-NEXT: fmov s17, w13
-; CHECK-GI-NEXT: udiv w15, w12, w15
-; CHECK-GI-NEXT: mov w12, v4.s[3]
-; CHECK-GI-NEXT: mov v17.s[1], w14
-; CHECK-GI-NEXT: udiv w12, w12, w16
-; CHECK-GI-NEXT: fmov w16, s6
-; CHECK-GI-NEXT: mov v17.s[2], w15
+; CHECK-GI-NEXT: udiv w14, w12, w13
+; CHECK-GI-NEXT: mov w12, v5.s[1]
+; CHECK-GI-NEXT: mov w13, v4.s[1]
+; CHECK-GI-NEXT: udiv w15, w13, w12
+; CHECK-GI-NEXT: mov w12, v5.s[2]
+; CHECK-GI-NEXT: mov w13, v4.s[2]
+; CHECK-GI-NEXT: fmov s7, w14
+; CHECK-GI-NEXT: udiv w13, w13, w12
+; CHECK-GI-NEXT: mov w12, v5.s[3]
+; CHECK-GI-NEXT: ushll v5.4s, v1.4h, #0
+; CHECK-GI-NEXT: mov v7.h[1], w15
+; CHECK-GI-NEXT: udiv w12, w16, w12
+; CHECK-GI-NEXT: fmov w16, s5
+; CHECK-GI-NEXT: mov v7.h[2], w13
; CHECK-GI-NEXT: udiv w16, w16, w17
-; CHECK-GI-NEXT: mov w17, v6.s[1]
-; CHECK-GI-NEXT: mov v17.s[3], w12
-; CHECK-GI-NEXT: mls v4.4s, v17.4s, v5.4s
+; CHECK-GI-NEXT: mov w17, v5.s[1]
+; CHECK-GI-NEXT: mov v7.h[3], w12
+; CHECK-GI-NEXT: umlsl v4.4s, v7.4h, v18.4h
; CHECK-GI-NEXT: udiv w17, w17, w18
-; CHECK-GI-NEXT: mov w18, v6.s[2]
-; CHECK-GI-NEXT: fmov s18, w16
-; CHECK-GI-NEXT: uzp1 v0.8h, v0.8h, v4.8h
+; CHECK-GI-NEXT: mov w18, v5.s[2]
+; CHECK-GI-NEXT: fmov s16, w16
; CHECK-GI-NEXT: udiv w18, w18, w0
-; CHECK-GI-NEXT: mov w0, v6.s[3]
-; CHECK-GI-NEXT: ushll2 v6.4s, v1.8h, #0
-; CHECK-GI-NEXT: mov v18.s[1], w17
+; CHECK-GI-NEXT: mov w0, v5.s[3]
+; CHECK-GI-NEXT: ushll2 v5.4s, v1.8h, #0
+; CHECK-GI-NEXT: mov v16.h[1], w17
; CHECK-GI-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-GI-NEXT: mov w3, v5.s[1]
+; CHECK-GI-NEXT: mov w4, v5.s[2]
+; CHECK-GI-NEXT: mov w5, v5.s[3]
; CHECK-GI-NEXT: udiv w0, w0, w1
-; CHECK-GI-NEXT: fmov w1, s6
-; CHECK-GI-NEXT: mov v18.s[2], w18
+; CHECK-GI-NEXT: fmov w1, s5
+; CHECK-GI-NEXT: mov v16.h[2], w18
; CHECK-GI-NEXT: udiv w1, w1, w2
; CHECK-GI-NEXT: mov w2, v6.s[1]
-; CHECK-GI-NEXT: mov v18.s[3], w0
-; CHECK-GI-NEXT: mls v1.4s, v18.4s, v3.4s
-; CHECK-GI-NEXT: udiv w2, w2, w3
+; CHECK-GI-NEXT: mov v16.h[3], w0
+; CHECK-GI-NEXT: umlsl v1.4s, v16.4h, v3.4h
+; CHECK-GI-NEXT: udiv w2, w3, w2
; CHECK-GI-NEXT: mov w3, v6.s[2]
-; CHECK-GI-NEXT: fmov s19, w1
-; CHECK-GI-NEXT: udiv w3, w3, w4
+; CHECK-GI-NEXT: fmov s17, w1
+; CHECK-GI-NEXT: udiv w3, w4, w3
; CHECK-GI-NEXT: mov w4, v6.s[3]
-; CHECK-GI-NEXT: mov v19.s[1], w2
-; CHECK-GI-NEXT: udiv w10, w4, w5
-; CHECK-GI-NEXT: mov v19.s[2], w3
-; CHECK-GI-NEXT: mov v19.s[3], w10
-; CHECK-GI-NEXT: mls v6.4s, v19.4s, v7.4s
-; CHECK-GI-NEXT: uzp1 v1.8h, v1.8h, v6.8h
+; CHECK-GI-NEXT: fmov s6, w10
+; CHECK-GI-NEXT: mov v17.h[1], w2
+; CHECK-GI-NEXT: mov v6.h[1], w11
+; CHECK-GI-NEXT: mov v6.h[2], w9
+; CHECK-GI-NEXT: udiv w10, w5, w4
+; CHECK-GI-NEXT: mov v17.h[2], w3
+; CHECK-GI-NEXT: mov v6.h[3], w8
+; CHECK-GI-NEXT: umlsl v0.4s, v6.4h, v2.4h
+; CHECK-GI-NEXT: mov v17.h[3], w10
+; CHECK-GI-NEXT: uzp1 v0.8h, v0.8h, v4.8h
+; CHECK-GI-NEXT: umlsl v5.4s, v17.4h, v19.4h
+; CHECK-GI-NEXT: uzp1 v1.8h, v1.8h, v5.8h
; CHECK-GI-NEXT: ret
entry:
%s = urem <16 x i16> %d, %e
More information about the llvm-commits
mailing list