[llvm] [AArch64][SVE][SelectionDAG] Improve codegen for SVE patterns sext(icmp) and zext(icmp) (PR #192052)
Sushant Gokhale via llvm-commits
llvm-commits at lists.llvm.org
Wed Apr 15 02:41:28 PDT 2026
https://github.com/sushgokh updated https://github.com/llvm/llvm-project/pull/192052
>From 89e6808cbb99b0fa95f317e6e143c1a1d81513a8 Mon Sep 17 00:00:00 2001
From: sgokhale <sgokhale at nvidia.com>
Date: Tue, 14 Apr 2026 05:16:37 -0700
Subject: [PATCH 1/4] [NFC][AArch64] Add test cases for SVE sext(icmp) and
zext(icmp)
In some scenarios, these patterns can be replaced with better patterns. Future patch will improve these.
---
.../CodeGen/AArch64/sve-icmp-sext-zext.ll | 565 ++++++++++++++++++
1 file changed, 565 insertions(+)
create mode 100644 llvm/test/CodeGen/AArch64/sve-icmp-sext-zext.ll
diff --git a/llvm/test/CodeGen/AArch64/sve-icmp-sext-zext.ll b/llvm/test/CodeGen/AArch64/sve-icmp-sext-zext.ll
new file mode 100644
index 0000000000000..89437ddb82fa3
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-icmp-sext-zext.ll
@@ -0,0 +1,565 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=aarch64 -mattr=+dotprod,+sve2p1 --aarch64-sve-vector-bits-min=256 -o - < %s | FileCheck %s
+
+define <vscale x 4 x i32> @test1(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
+; CHECK-LABEL: test1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: sdot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+{
+ %1 = icmp ult <vscale x 16 x i8> %a, %b
+ %2 = sext <vscale x 16 x i1> %1 to <vscale x 16 x i8>
+ %3 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sdot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %2, <vscale x 16 x i8> %2)
+ ret <vscale x 4 x i32> %3
+}
+
+define <vscale x 4 x i32> @test2(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
+; CHECK-LABEL: test2:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #1 // =0x1
+; CHECK-NEXT: sdot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+{
+ %1 = icmp ult <vscale x 16 x i8> %a, %b
+ %2 = zext <vscale x 16 x i1> %1 to <vscale x 16 x i8>
+ %3 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sdot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %2, <vscale x 16 x i8> %2)
+ ret <vscale x 4 x i32> %3
+}
+
+define <vscale x 4 x i32> @test3(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
+; CHECK-LABEL: test3:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #1 // =0x1
+; CHECK-NEXT: sdot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+{
+ %1 = icmp ult <vscale x 16 x i8> %a, %b
+ %2 = zext <vscale x 16 x i1> %1 to <vscale x 16 x i8>
+ %3 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sdot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %2, <vscale x 16 x i8> %2)
+ ret <vscale x 4 x i32> %3
+}
+
+define <vscale x 4 x i32> @test4(<32 x i8> %a, <32 x i8> %b) {
+; CHECK-LABEL: test4:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: ptrue p0.b, vl16
+; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
+; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: sdot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+ %1 = icmp ult <32 x i8> %a, %b
+ %2 = sext <32 x i1> %1 to <32 x i8>
+ %3 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v32i8(<vscale x 16 x i8> poison, <32 x i8> %2, i64 0)
+ %4 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sdot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %3, <vscale x 16 x i8> %3)
+ ret <vscale x 4 x i32> %4
+}
+
+define <vscale x 4 x i32> @test5(<32 x i8> %a, <32 x i8> %b) {
+; CHECK-LABEL: test5:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: ptrue p0.b, vl16
+; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
+; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: and z1.b, z1.b, #0x1
+; CHECK-NEXT: sdot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+ %1 = icmp ult <32 x i8> %a, %b
+ %2 = zext <32 x i1> %1 to <32 x i8>
+ %3 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v32i8(<vscale x 16 x i8> poison, <32 x i8> %2, i64 0)
+ %4 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sdot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %3, <vscale x 16 x i8> %3)
+ ret <vscale x 4 x i32> %4
+}
+
+define <vscale x 4 x i32> @test6(<32 x i8> %a, <32 x i8> %b) {
+; CHECK-LABEL: test6:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: ptrue p0.b, vl16
+; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
+; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: and z1.b, z1.b, #0x1
+; CHECK-NEXT: sdot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+ %1 = icmp ult <32 x i8> %a, %b
+ %2 = zext <32 x i1> %1 to <32 x i8>
+ %3 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v32i8(<vscale x 16 x i8> poison, <32 x i8> %2, i64 0)
+ %4 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sdot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %3, <vscale x 16 x i8> %3)
+ ret <vscale x 4 x i32> %4
+}
+
+define <vscale x 4 x i32> @test7(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
+; CHECK-LABEL: test7:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: sdot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+{
+ %1 = icmp ugt <vscale x 16 x i8> %a, %b
+ %2 = sext <vscale x 16 x i1> %1 to <vscale x 16 x i8>
+ %3 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sdot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %2, <vscale x 16 x i8> %2)
+ ret <vscale x 4 x i32> %3
+}
+
+define <vscale x 4 x i32> @test8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
+; CHECK-LABEL: test8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #1 // =0x1
+; CHECK-NEXT: sdot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+{
+ %1 = icmp ugt <vscale x 16 x i8> %a, %b
+ %2 = zext <vscale x 16 x i1> %1 to <vscale x 16 x i8>
+ %3 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sdot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %2, <vscale x 16 x i8> %2)
+ ret <vscale x 4 x i32> %3
+}
+
+define <vscale x 4 x i32> @test9(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
+; CHECK-LABEL: test9:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #1 // =0x1
+; CHECK-NEXT: sdot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+{
+ %1 = icmp ugt <vscale x 16 x i8> %a, %b
+ %2 = zext <vscale x 16 x i1> %1 to <vscale x 16 x i8>
+ %3 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sdot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %2, <vscale x 16 x i8> %2)
+ ret <vscale x 4 x i32> %3
+}
+
+define <vscale x 4 x i32> @test10(<32 x i8> %a, <32 x i8> %b) {
+; CHECK-LABEL: test10:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: ptrue p0.b, vl16
+; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: splice z2.b, p0, { z2.b, z3.b }
+; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z2.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: sdot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+ %1 = icmp ugt <32 x i8> %a, %b
+ %2 = sext <32 x i1> %1 to <32 x i8>
+ %3 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v32i8(<vscale x 16 x i8> poison, <32 x i8> %2, i64 0)
+ %4 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sdot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %3, <vscale x 16 x i8> %3)
+ ret <vscale x 4 x i32> %4
+}
+
+define <vscale x 4 x i32> @test11(<32 x i8> %a, <32 x i8> %b) {
+; CHECK-LABEL: test11:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: ptrue p0.b, vl16
+; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: splice z2.b, p0, { z2.b, z3.b }
+; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z2.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: and z1.b, z1.b, #0x1
+; CHECK-NEXT: sdot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+ %1 = icmp ugt <32 x i8> %a, %b
+ %2 = zext <32 x i1> %1 to <32 x i8>
+ %3 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v32i8(<vscale x 16 x i8> poison, <32 x i8> %2, i64 0)
+ %4 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sdot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %3, <vscale x 16 x i8> %3)
+ ret <vscale x 4 x i32> %4
+}
+
+define <vscale x 4 x i32> @test12(<32 x i8> %a, <32 x i8> %b) {
+; CHECK-LABEL: test12:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: ptrue p0.b, vl16
+; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: splice z2.b, p0, { z2.b, z3.b }
+; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z2.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: and z1.b, z1.b, #0x1
+; CHECK-NEXT: sdot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+ %1 = icmp ugt <32 x i8> %a, %b
+ %2 = zext <32 x i1> %1 to <32 x i8>
+ %3 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v32i8(<vscale x 16 x i8> poison, <32 x i8> %2, i64 0)
+ %4 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sdot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %3, <vscale x 16 x i8> %3)
+ ret <vscale x 4 x i32> %4
+}
+
+define <vscale x 16 x i8> @test13(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1) {
+; CHECK-LABEL: test13:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: cmpgt p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: mov z0.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: ret
+entry:
+ %2 = icmp slt <vscale x 16 x i8> %0, %1
+ %3 = sext <vscale x 16 x i1> %2 to <vscale x 16 x i8>
+ ret <vscale x 16 x i8> %3
+}
+
+define <vscale x 16 x i8> @test14(<32 x i8> %0, <32 x i8> %1) {
+; CHECK-LABEL: test14:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: ptrue p0.b, vl16
+; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
+; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: cmpgt p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: mov z0.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: ret
+entry:
+ %2 = icmp slt <32 x i8> %0, %1
+ %3 = sext <32 x i1> %2 to <32 x i8>
+ %4 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v32i8(<vscale x 16 x i8> poison, <32 x i8> %3, i64 0)
+ ret <vscale x 16 x i8> %4
+}
+
+; FIXME: Lower this sequence to sqsub/sclamp
+define <vscale x 16 x i8> @test15(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1) {
+; CHECK-LABEL: test15:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: cmpgt p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: mov z1.b, #1 // =0x1
+; CHECK-NEXT: mov z0.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: add z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT: ret
+entry:
+ %2 = icmp slt <vscale x 16 x i8> %0, %1
+ %3 = sext <vscale x 16 x i1> %2 to <vscale x 16 x i8>
+ %4 = zext <vscale x 16 x i1> %2 to <vscale x 16 x i8>
+ %5 = add <vscale x 16 x i8> %3, %4
+ ret <vscale x 16 x i8> %5
+}
+
+define <vscale x 16 x i8> @test16(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1) {
+; CHECK-LABEL: test16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: cmpgt p0.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: mov z0.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: ret
+entry:
+ %2 = icmp sgt <vscale x 16 x i8> %0, %1
+ %3 = sext <vscale x 16 x i1> %2 to <vscale x 16 x i8>
+ ret <vscale x 16 x i8> %3
+}
+
+define <vscale x 16 x i8> @test17(<32 x i8> %0, <32 x i8> %1) {
+; CHECK-LABEL: test17:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: ptrue p0.b, vl16
+; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: splice z2.b, p0, { z2.b, z3.b }
+; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: cmpgt p0.b, p0/z, z0.b, z2.b
+; CHECK-NEXT: mov z0.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: ret
+entry:
+ %2 = icmp sgt <32 x i8> %0, %1
+ %3 = sext <32 x i1> %2 to <32 x i8>
+ %4 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v32i8(<vscale x 16 x i8> poison, <32 x i8> %3, i64 0)
+ ret <vscale x 16 x i8> %4
+}
+
+define <vscale x 4 x i32> @test18(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
+; CHECK-LABEL: test18:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: udot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+{
+ %1 = icmp ult <vscale x 16 x i8> %a, %b
+ %2 = sext <vscale x 16 x i1> %1 to <vscale x 16 x i8>
+ %3 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.udot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %2, <vscale x 16 x i8> %2)
+ ret <vscale x 4 x i32> %3
+}
+
+define <vscale x 4 x i32> @test19(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
+; CHECK-LABEL: test19:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #1 // =0x1
+; CHECK-NEXT: udot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+{
+ %1 = icmp ult <vscale x 16 x i8> %a, %b
+ %2 = zext <vscale x 16 x i1> %1 to <vscale x 16 x i8>
+ %3 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.udot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %2, <vscale x 16 x i8> %2)
+ ret <vscale x 4 x i32> %3
+}
+
+
+define <vscale x 4 x i32> @test20(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
+; CHECK-LABEL: test20:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #1 // =0x1
+; CHECK-NEXT: udot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+{
+ %1 = icmp ult <vscale x 16 x i8> %a, %b
+ %2 = zext <vscale x 16 x i1> %1 to <vscale x 16 x i8>
+ %3 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.udot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %2, <vscale x 16 x i8> %2)
+ ret <vscale x 4 x i32> %3
+}
+
+define <vscale x 4 x i32> @test21(<32 x i8> %a, <32 x i8> %b) {
+; CHECK-LABEL: test21:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: ptrue p0.b, vl16
+; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
+; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: udot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+ %1 = icmp ult <32 x i8> %a, %b
+ %2 = sext <32 x i1> %1 to <32 x i8>
+ %3 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v32i8(<vscale x 16 x i8> poison, <32 x i8> %2, i64 0)
+ %4 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.udot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %3, <vscale x 16 x i8> %3)
+ ret <vscale x 4 x i32> %4
+}
+
+define <vscale x 4 x i32> @test22(<32 x i8> %a, <32 x i8> %b) {
+; CHECK-LABEL: test22:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: ptrue p0.b, vl16
+; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
+; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: and z1.b, z1.b, #0x1
+; CHECK-NEXT: udot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+ %1 = icmp ult <32 x i8> %a, %b
+ %2 = zext <32 x i1> %1 to <32 x i8>
+ %3 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v32i8(<vscale x 16 x i8> poison, <32 x i8> %2, i64 0)
+ %4 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.udot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %3, <vscale x 16 x i8> %3)
+ ret <vscale x 4 x i32> %4
+}
+
+define <vscale x 4 x i32> @test23(<32 x i8> %a, <32 x i8> %b) {
+; CHECK-LABEL: test23:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: ptrue p0.b, vl16
+; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
+; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: and z1.b, z1.b, #0x1
+; CHECK-NEXT: udot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+ %1 = icmp ult <32 x i8> %a, %b
+ %2 = zext <32 x i1> %1 to <32 x i8>
+ %3 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v32i8(<vscale x 16 x i8> poison, <32 x i8> %2, i64 0)
+ %4 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.udot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %3, <vscale x 16 x i8> %3)
+ ret <vscale x 4 x i32> %4
+}
+
+define <vscale x 4 x i32> @test24(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
+; CHECK-LABEL: test24:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: udot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+{
+ %1 = icmp ugt <vscale x 16 x i8> %a, %b
+ %2 = sext <vscale x 16 x i1> %1 to <vscale x 16 x i8>
+ %3 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.udot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %2, <vscale x 16 x i8> %2)
+ ret <vscale x 4 x i32> %3
+}
+
+define <vscale x 4 x i32> @test25(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
+; CHECK-LABEL: test25:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #1 // =0x1
+; CHECK-NEXT: udot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+{
+ %1 = icmp ugt <vscale x 16 x i8> %a, %b
+ %2 = zext <vscale x 16 x i1> %1 to <vscale x 16 x i8>
+ %3 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.udot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %2, <vscale x 16 x i8> %2)
+ ret <vscale x 4 x i32> %3
+}
+
+define <vscale x 4 x i32> @test26(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
+; CHECK-LABEL: test26:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #1 // =0x1
+; CHECK-NEXT: udot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+{
+ %1 = icmp ugt <vscale x 16 x i8> %a, %b
+ %2 = zext <vscale x 16 x i1> %1 to <vscale x 16 x i8>
+ %3 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.udot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %2, <vscale x 16 x i8> %2)
+ ret <vscale x 4 x i32> %3
+}
+
+define <vscale x 4 x i32> @test27(<32 x i8> %a, <32 x i8> %b) {
+; CHECK-LABEL: test27:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: ptrue p0.b, vl16
+; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: splice z2.b, p0, { z2.b, z3.b }
+; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z2.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: udot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+ %1 = icmp ugt <32 x i8> %a, %b
+ %2 = sext <32 x i1> %1 to <32 x i8>
+ %3 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v32i8(<vscale x 16 x i8> poison, <32 x i8> %2, i64 0)
+ %4 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.udot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %3, <vscale x 16 x i8> %3)
+ ret <vscale x 4 x i32> %4
+}
+
+define <vscale x 4 x i32> @test28(<32 x i8> %a, <32 x i8> %b) {
+; CHECK-LABEL: test28:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: ptrue p0.b, vl16
+; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: splice z2.b, p0, { z2.b, z3.b }
+; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z2.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: and z1.b, z1.b, #0x1
+; CHECK-NEXT: udot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+ %1 = icmp ugt <32 x i8> %a, %b
+ %2 = zext <32 x i1> %1 to <32 x i8>
+ %3 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v32i8(<vscale x 16 x i8> poison, <32 x i8> %2, i64 0)
+ %4 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.udot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %3, <vscale x 16 x i8> %3)
+ ret <vscale x 4 x i32> %4
+}
+
+define <vscale x 4 x i32> @test29(<32 x i8> %a, <32 x i8> %b) {
+; CHECK-LABEL: test29:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: ptrue p0.b, vl16
+; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: splice z2.b, p0, { z2.b, z3.b }
+; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z2.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: and z1.b, z1.b, #0x1
+; CHECK-NEXT: udot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+ %1 = icmp ugt <32 x i8> %a, %b
+ %2 = zext <32 x i1> %1 to <32 x i8>
+ %3 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v32i8(<vscale x 16 x i8> poison, <32 x i8> %2, i64 0)
+ %4 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.udot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %3, <vscale x 16 x i8> %3)
+ ret <vscale x 4 x i32> %4
+}
>From 1c09c2f490f2c8b0fa00ac92852779bf1856207a Mon Sep 17 00:00:00 2001
From: sgokhale <sgokhale at nvidia.com>
Date: Tue, 14 Apr 2026 05:40:37 -0700
Subject: [PATCH 2/4] [AArch64][SVE][SelectionDAG] Improve codegen for SVE
patterns sext(icmp) and zext(icmp)
For SVE vectors:
1. sext(icmp slt) --> max(min(a-b, 0), -1) --> sclamp(sqsub(a, b), -1, 0)
2. sext(icmp slt) --> min(max(b-a, 0), 1) --> sclamp(sqsub(b, a), 0, 1)
3. zext(icmp ult) --> min(max(b-a, 0), 1) --> umin(uqsub(b, a), 1)
4. Let,
Op = sext(icmp ult(a,b))
NewOp = umin(uqsub(b, a), 1)
Then,
sdot(acc, Op, Op) --> sdot(acc, NewOp, NewOp)
The other changes follow:
1. icmp sgt is just inverse of the icmp slt.
2. udot is similar to sdot
---
.../Target/AArch64/AArch64ISelLowering.cpp | 97 +++++++++-
.../CodeGen/AArch64/sve-icmp-sext-zext.ll | 174 ++++++++----------
2 files changed, 169 insertions(+), 102 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index ece24767bdbb9..1f83b8cb705ea 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -20941,6 +20941,22 @@ static SDValue performSVEAndCombine(SDNode *N,
SDValue Src = N->getOperand(0);
unsigned Opc = Src->getOpcode();
+ // and(splat(1), sext(setcc_merge_zero)) -> zext(setcc_merge_zero)
+ SDLoc DL(N);
+ SDValue Op0 = N->getOperand(0);
+ SDValue Op1 = N->getOperand(1);
+ if (Op0.getOpcode() == ISD::SPLAT_VECTOR ||
+ Op1.getOpcode() == ISD::SPLAT_VECTOR) {
+ SDValue NonSplatOp = (Op0.getOpcode() == ISD::SPLAT_VECTOR ? Op1 : Op0);
+ SDValue SplatOp = (Op0.getOpcode() == ISD::SPLAT_VECTOR ? Op0 : Op1);
+ if (NonSplatOp.getOpcode() == ISD::SIGN_EXTEND) {
+ SDValue Compare = NonSplatOp.getOperand(0);
+ if (Compare.getOpcode() == AArch64ISD::SETCC_MERGE_ZERO) {
+ return DAG.getNode(ISD::ZERO_EXTEND, DL, N->getValueType(0), Compare);
+ }
+ }
+ }
+
// Zero/any extend of an unsigned unpack
if (Opc == AArch64ISD::UUNPKHI || Opc == AArch64ISD::UUNPKLO) {
SDValue UnpkOp = Src->getOperand(0);
@@ -20949,7 +20965,6 @@ static SDValue performSVEAndCombine(SDNode *N,
if (Dup.getOpcode() != ISD::SPLAT_VECTOR)
return SDValue();
- SDLoc DL(N);
ConstantSDNode *C = dyn_cast<ConstantSDNode>(Dup->getOperand(0));
if (!C)
return SDValue();
@@ -24091,6 +24106,37 @@ static SDValue performIntrinsicCombine(SDNode *N,
case Intrinsic::aarch64_sve_bsl2n:
case Intrinsic::aarch64_sve_nbsl:
return combineSVEBitSel(IID, N, DAG);
+ case Intrinsic::aarch64_sve_udot:
+ case Intrinsic::aarch64_sve_sdot: {
+ // sdot(acc, extend(icmp_ult(A, B)), extend(icmp_ult(A, B)))
+ // -> sdot(acc, umin(usubsat(B, A)), umin(usubsat(B, A)))
+
+ // sdot(acc, extend(icmp_ugt(A, B)), extend(icmp_ugt(A, B)))
+ // -> sdot(acc, umin(usubsat(A, B)), umin(usubsat(A, B)))
+ SDValue Extend1 = N->getOperand(2);
+ SDValue Extend2 = N->getOperand(3);
+ if (Extend1 == Extend2 && (Extend1.getOpcode() == ISD::SIGN_EXTEND ||
+ Extend1.getOpcode() == ISD::ZERO_EXTEND)) {
+ SDValue Compare = Extend1.getOperand(0);
+ if (Compare.getOpcode() != AArch64ISD::SETCC_MERGE_ZERO)
+ break;
+ ISD::CondCode CC = cast<CondCodeSDNode>(Compare.getOperand(3))->get();
+ if (CC != ISD::SETULT && CC != ISD::SETUGT)
+ break;
+ SDLoc DL(N);
+ EVT ResVT = Extend1.getValueType();
+ SDValue A = Compare.getOperand(CC == ISD::SETULT ? 1 : 2);
+ SDValue B = Compare.getOperand(CC == ISD::SETULT ? 2 : 1);
+ SDValue Sub = DAG.getNode(ISD::USUBSAT, DL, ResVT, B, A);
+ SDValue One = DAG.getConstant(1, DL, MVT::i32);
+ SDValue SplatOne = DAG.getSplatVector(ResVT, DL, One);
+ SDValue Min = DAG.getNode(ISD::UMIN, DL, ResVT, Sub, SplatOne);
+ unsigned Opcode = (IID == Intrinsic::aarch64_sve_udot ? AArch64ISD::UDOT
+ : AArch64ISD::SDOT);
+ return DAG.getNode(Opcode, DL, N->getValueType(0), N->getOperand(1), Min,
+ Min);
+ }
+ }
}
return SDValue();
}
@@ -24422,6 +24468,55 @@ static SDValue performExtendCombine(SDNode *N,
TargetLowering::DAGCombinerInfo &DCI,
SelectionDAG &DAG,
const AArch64Subtarget *Subtarget) {
+ // sext(icmp slt(a, b)) -> sclamp(sqsub(a, b), -1, 0)
+ // sext(icmp sgt(a, b)) -> sclamp(sqsub(b, a), -1, 0)
+ if (N->getOpcode() == ISD::SIGN_EXTEND) {
+ SDValue Compare = N->getOperand(0);
+ if (Compare.getOpcode() == AArch64ISD::SETCC_MERGE_ZERO) {
+ ISD::CondCode CC = cast<CondCodeSDNode>(Compare.getOperand(3))->get();
+ if (Compare.hasOneUse() && Subtarget->hasSVE2p1() &&
+ (CC == ISD::SETLT || CC == ISD::SETGT)) {
+ SDLoc DL(N);
+ SDValue A = Compare.getOperand(CC == ISD::SETLT ? 1 : 2);
+ SDValue B = Compare.getOperand(CC == ISD::SETLT ? 2 : 1);
+ SDValue Sub = DAG.getNode(ISD::SSUBSAT, DL, N->getValueType(0), A, B);
+ SDValue MinusOne = DAG.getSplatVector(
+ N->getValueType(0), DL, DAG.getSignedConstant(-1, DL, MVT::i32));
+ SDValue Zero = DAG.getSplatVector(N->getValueType(0), DL,
+ DAG.getConstant(0, DL, MVT::i32));
+ SDValue Sclamp = DAG.getNode(
+ ISD::INTRINSIC_WO_CHAIN, DL, N->getValueType(0),
+ DAG.getConstant(Intrinsic::aarch64_sve_sclamp, DL, MVT::i64), Sub,
+ MinusOne, Zero);
+ return Sclamp;
+ }
+ }
+ }
+ // zext(icmp slt(a, b)) -> sclamp(sqsub(b,a), 0, 1)
+ // zext(icmp sgt(a, b)) -> sclamp(sqsub(a,b), 0, 1)
+ if (N->getOpcode() == ISD::ZERO_EXTEND) {
+ SDValue Compare = N->getOperand(0);
+ if (Compare.getOpcode() == AArch64ISD::SETCC_MERGE_ZERO) {
+ ISD::CondCode CC = cast<CondCodeSDNode>(Compare.getOperand(3))->get();
+ if (Compare.hasOneUse() && Subtarget->hasSVE2p1() &&
+ (CC == ISD::SETLT || CC == ISD::SETGT)) {
+ SDLoc DL(N);
+ SDValue A = Compare.getOperand(CC == ISD::SETLT ? 1 : 2);
+ SDValue B = Compare.getOperand(CC == ISD::SETLT ? 2 : 1);
+ SDValue Sub = DAG.getNode(ISD::SSUBSAT, DL, N->getValueType(0), B, A);
+ SDValue One = DAG.getSplatVector(N->getValueType(0), DL,
+ DAG.getConstant(1, DL, MVT::i32));
+ SDValue Zero = DAG.getSplatVector(N->getValueType(0), DL,
+ DAG.getConstant(0, DL, MVT::i32));
+ SDValue Sclamp = DAG.getNode(
+ ISD::INTRINSIC_WO_CHAIN, DL, N->getValueType(0),
+ DAG.getConstant(Intrinsic::aarch64_sve_sclamp, DL, MVT::i64), Sub,
+ Zero, One);
+ return Sclamp;
+ }
+ }
+ }
+
// If we see something like (zext (sabd (extract_high ...), (DUP ...))) then
// we can convert that DUP into another extract_high (of a bigger DUP), which
// helps the backend to decide that an sabdl2 would be useful, saving a real
diff --git a/llvm/test/CodeGen/AArch64/sve-icmp-sext-zext.ll b/llvm/test/CodeGen/AArch64/sve-icmp-sext-zext.ll
index 89437ddb82fa3..b53d36e0ee13d 100644
--- a/llvm/test/CodeGen/AArch64/sve-icmp-sext-zext.ll
+++ b/llvm/test/CodeGen/AArch64/sve-icmp-sext-zext.ll
@@ -4,10 +4,9 @@
define <vscale x 4 x i32> @test1(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
; CHECK-LABEL: test1:
; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: uqsub z1.b, z1.b, z0.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: sdot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
{
@@ -20,10 +19,9 @@ define <vscale x 4 x i32> @test1(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
define <vscale x 4 x i32> @test2(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
; CHECK-LABEL: test2:
; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: uqsub z1.b, z1.b, z0.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #1 // =0x1
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: sdot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
{
@@ -36,10 +34,9 @@ define <vscale x 4 x i32> @test2(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
define <vscale x 4 x i32> @test3(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
; CHECK-LABEL: test3:
; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: uqsub z1.b, z1.b, z0.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #1 // =0x1
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: sdot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
{
@@ -59,10 +56,9 @@ define <vscale x 4 x i32> @test4(<32 x i8> %a, <32 x i8> %b) {
; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
-; CHECK-NEXT: ptrue p0.b, vl32
-; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: uqsub z1.b, z1.b, z0.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: sdot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
%1 = icmp ult <32 x i8> %a, %b
@@ -82,11 +78,9 @@ define <vscale x 4 x i32> @test5(<32 x i8> %a, <32 x i8> %b) {
; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
-; CHECK-NEXT: ptrue p0.b, vl32
-; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: uqsub z1.b, z1.b, z0.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: and z1.b, z1.b, #0x1
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: sdot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
%1 = icmp ult <32 x i8> %a, %b
@@ -106,11 +100,9 @@ define <vscale x 4 x i32> @test6(<32 x i8> %a, <32 x i8> %b) {
; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
-; CHECK-NEXT: ptrue p0.b, vl32
-; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: uqsub z1.b, z1.b, z0.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: and z1.b, z1.b, #0x1
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: sdot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
%1 = icmp ult <32 x i8> %a, %b
@@ -123,10 +115,9 @@ define <vscale x 4 x i32> @test6(<32 x i8> %a, <32 x i8> %b) {
define <vscale x 4 x i32> @test7(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
; CHECK-LABEL: test7:
; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: uqsub z1.b, z0.b, z1.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: sdot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
{
@@ -139,10 +130,9 @@ define <vscale x 4 x i32> @test7(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
define <vscale x 4 x i32> @test8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
; CHECK-LABEL: test8:
; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: uqsub z1.b, z0.b, z1.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #1 // =0x1
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: sdot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
{
@@ -155,10 +145,9 @@ define <vscale x 4 x i32> @test8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
define <vscale x 4 x i32> @test9(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
; CHECK-LABEL: test9:
; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: uqsub z1.b, z0.b, z1.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #1 // =0x1
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: sdot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
{
@@ -178,10 +167,9 @@ define <vscale x 4 x i32> @test10(<32 x i8> %a, <32 x i8> %b) {
; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
; CHECK-NEXT: splice z2.b, p0, { z2.b, z3.b }
; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
-; CHECK-NEXT: ptrue p0.b, vl32
-; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z2.b
+; CHECK-NEXT: uqsub z1.b, z0.b, z2.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: sdot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
%1 = icmp ugt <32 x i8> %a, %b
@@ -201,11 +189,9 @@ define <vscale x 4 x i32> @test11(<32 x i8> %a, <32 x i8> %b) {
; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
; CHECK-NEXT: splice z2.b, p0, { z2.b, z3.b }
; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
-; CHECK-NEXT: ptrue p0.b, vl32
-; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z2.b
+; CHECK-NEXT: uqsub z1.b, z0.b, z2.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: and z1.b, z1.b, #0x1
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: sdot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
%1 = icmp ugt <32 x i8> %a, %b
@@ -225,11 +211,9 @@ define <vscale x 4 x i32> @test12(<32 x i8> %a, <32 x i8> %b) {
; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
; CHECK-NEXT: splice z2.b, p0, { z2.b, z3.b }
; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
-; CHECK-NEXT: ptrue p0.b, vl32
-; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z2.b
+; CHECK-NEXT: uqsub z1.b, z0.b, z2.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: and z1.b, z1.b, #0x1
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: sdot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
%1 = icmp ugt <32 x i8> %a, %b
@@ -242,9 +226,10 @@ define <vscale x 4 x i32> @test12(<32 x i8> %a, <32 x i8> %b) {
define <vscale x 16 x i8> @test13(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1) {
; CHECK-LABEL: test13:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: cmpgt p0.b, p0/z, z1.b, z0.b
-; CHECK-NEXT: mov z0.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: movi v2.2d, #0000000000000000
+; CHECK-NEXT: sqsub z0.b, z0.b, z1.b
+; CHECK-NEXT: mov z1.b, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: sclamp z0.b, z1.b, z2.b
; CHECK-NEXT: ret
entry:
%2 = icmp slt <vscale x 16 x i8> %0, %1
@@ -255,16 +240,17 @@ entry:
define <vscale x 16 x i8> @test14(<32 x i8> %0, <32 x i8> %1) {
; CHECK-LABEL: test14:
; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
; CHECK-NEXT: ptrue p0.b, vl16
-; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
-; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: splice z2.b, p0, { z2.b, z3.b }
; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
-; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
-; CHECK-NEXT: ptrue p0.b, vl32
-; CHECK-NEXT: cmpgt p0.b, p0/z, z1.b, z0.b
-; CHECK-NEXT: mov z0.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: sqsub z0.b, z0.b, z2.b
+; CHECK-NEXT: mov z2.b, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: sclamp z0.b, z2.b, z1.b
; CHECK-NEXT: ret
entry:
%2 = icmp slt <32 x i8> %0, %1
@@ -278,10 +264,10 @@ define <vscale x 16 x i8> @test15(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1)
; CHECK-LABEL: test15:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: cmpgt p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: cmpgt p1.b, p0/z, z1.b, z0.b
; CHECK-NEXT: mov z1.b, #1 // =0x1
-; CHECK-NEXT: mov z0.b, p0/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: add z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: add z0.b, p1/m, z0.b, z1.b
; CHECK-NEXT: ret
entry:
%2 = icmp slt <vscale x 16 x i8> %0, %1
@@ -294,9 +280,10 @@ entry:
define <vscale x 16 x i8> @test16(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1) {
; CHECK-LABEL: test16:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: cmpgt p0.b, p0/z, z0.b, z1.b
-; CHECK-NEXT: mov z0.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: movi v2.2d, #0000000000000000
+; CHECK-NEXT: sqsub z0.b, z1.b, z0.b
+; CHECK-NEXT: mov z1.b, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: sclamp z0.b, z1.b, z2.b
; CHECK-NEXT: ret
entry:
%2 = icmp sgt <vscale x 16 x i8> %0, %1
@@ -307,16 +294,17 @@ entry:
define <vscale x 16 x i8> @test17(<32 x i8> %0, <32 x i8> %1) {
; CHECK-LABEL: test17:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
; CHECK-NEXT: ptrue p0.b, vl16
-; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
-; CHECK-NEXT: splice z2.b, p0, { z2.b, z3.b }
+; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
-; CHECK-NEXT: ptrue p0.b, vl32
-; CHECK-NEXT: cmpgt p0.b, p0/z, z0.b, z2.b
-; CHECK-NEXT: mov z0.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
+; CHECK-NEXT: mov z2.b, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: sqsub z0.b, z1.b, z0.b
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: sclamp z0.b, z2.b, z1.b
; CHECK-NEXT: ret
entry:
%2 = icmp sgt <32 x i8> %0, %1
@@ -328,10 +316,9 @@ entry:
define <vscale x 4 x i32> @test18(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
; CHECK-LABEL: test18:
; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: uqsub z1.b, z1.b, z0.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: udot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
{
@@ -344,10 +331,9 @@ define <vscale x 4 x i32> @test18(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
define <vscale x 4 x i32> @test19(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
; CHECK-LABEL: test19:
; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: uqsub z1.b, z1.b, z0.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #1 // =0x1
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: udot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
{
@@ -361,10 +347,9 @@ define <vscale x 4 x i32> @test19(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
define <vscale x 4 x i32> @test20(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
; CHECK-LABEL: test20:
; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: uqsub z1.b, z1.b, z0.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #1 // =0x1
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: udot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
{
@@ -384,10 +369,9 @@ define <vscale x 4 x i32> @test21(<32 x i8> %a, <32 x i8> %b) {
; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
-; CHECK-NEXT: ptrue p0.b, vl32
-; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: uqsub z1.b, z1.b, z0.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: udot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
%1 = icmp ult <32 x i8> %a, %b
@@ -407,11 +391,9 @@ define <vscale x 4 x i32> @test22(<32 x i8> %a, <32 x i8> %b) {
; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
-; CHECK-NEXT: ptrue p0.b, vl32
-; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: uqsub z1.b, z1.b, z0.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: and z1.b, z1.b, #0x1
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: udot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
%1 = icmp ult <32 x i8> %a, %b
@@ -431,11 +413,9 @@ define <vscale x 4 x i32> @test23(<32 x i8> %a, <32 x i8> %b) {
; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
-; CHECK-NEXT: ptrue p0.b, vl32
-; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: uqsub z1.b, z1.b, z0.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: and z1.b, z1.b, #0x1
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: udot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
%1 = icmp ult <32 x i8> %a, %b
@@ -448,10 +428,9 @@ define <vscale x 4 x i32> @test23(<32 x i8> %a, <32 x i8> %b) {
define <vscale x 4 x i32> @test24(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
; CHECK-LABEL: test24:
; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: uqsub z1.b, z0.b, z1.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: udot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
{
@@ -464,10 +443,9 @@ define <vscale x 4 x i32> @test24(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
define <vscale x 4 x i32> @test25(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
; CHECK-LABEL: test25:
; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: uqsub z1.b, z0.b, z1.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #1 // =0x1
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: udot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
{
@@ -480,10 +458,9 @@ define <vscale x 4 x i32> @test25(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
define <vscale x 4 x i32> @test26(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
; CHECK-LABEL: test26:
; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: uqsub z1.b, z0.b, z1.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #1 // =0x1
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: udot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
{
@@ -503,10 +480,9 @@ define <vscale x 4 x i32> @test27(<32 x i8> %a, <32 x i8> %b) {
; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
; CHECK-NEXT: splice z2.b, p0, { z2.b, z3.b }
; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
-; CHECK-NEXT: ptrue p0.b, vl32
-; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z2.b
+; CHECK-NEXT: uqsub z1.b, z0.b, z2.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: udot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
%1 = icmp ugt <32 x i8> %a, %b
@@ -526,11 +502,9 @@ define <vscale x 4 x i32> @test28(<32 x i8> %a, <32 x i8> %b) {
; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
; CHECK-NEXT: splice z2.b, p0, { z2.b, z3.b }
; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
-; CHECK-NEXT: ptrue p0.b, vl32
-; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z2.b
+; CHECK-NEXT: uqsub z1.b, z0.b, z2.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: and z1.b, z1.b, #0x1
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: udot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
%1 = icmp ugt <32 x i8> %a, %b
@@ -550,11 +524,9 @@ define <vscale x 4 x i32> @test29(<32 x i8> %a, <32 x i8> %b) {
; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
; CHECK-NEXT: splice z2.b, p0, { z2.b, z3.b }
; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
-; CHECK-NEXT: ptrue p0.b, vl32
-; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z2.b
+; CHECK-NEXT: uqsub z1.b, z0.b, z2.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: and z1.b, z1.b, #0x1
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: udot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
%1 = icmp ugt <32 x i8> %a, %b
>From aa4af1d162f79e9bd263fa2ce6d983a68b4009d6 Mon Sep 17 00:00:00 2001
From: sgokhale <sgokhale at nvidia.com>
Date: Tue, 14 Apr 2026 23:07:32 -0700
Subject: [PATCH 3/4] [AArch64][NFC] Correct the test cases
The basic patterns are
sext(slt)
sext(ult)
zext(slt)
zext(ult)
sext(sgt)
sext(ugt)
zext(sgt)
zext(ugt)
---
.../CodeGen/AArch64/sve-icmp-sext-zext.ll | 551 +++++-------------
1 file changed, 146 insertions(+), 405 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/sve-icmp-sext-zext.ll b/llvm/test/CodeGen/AArch64/sve-icmp-sext-zext.ll
index b53d36e0ee13d..7b585a2b713d4 100644
--- a/llvm/test/CodeGen/AArch64/sve-icmp-sext-zext.ll
+++ b/llvm/test/CodeGen/AArch64/sve-icmp-sext-zext.ll
@@ -1,53 +1,21 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -mtriple=aarch64 -mattr=+dotprod,+sve2p1 --aarch64-sve-vector-bits-min=256 -o - < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64 -mattr=+sve2 --aarch64-sve-vector-bits-min=256 -o - < %s | FileCheck %s
-define <vscale x 4 x i32> @test1(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
-; CHECK-LABEL: test1:
+define <vscale x 16 x i8> @sext_slt_test1(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
+; CHECK-LABEL: sext_slt_test1:
; CHECK: // %bb.0:
-; CHECK-NEXT: uqsub z1.b, z1.b, z0.b
-; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: umin z1.b, z1.b, #1
-; CHECK-NEXT: sdot z0.s, z1.b, z1.b
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: cmpgt p1.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT: ret
{
- %1 = icmp ult <vscale x 16 x i8> %a, %b
+ %1 = icmp slt <vscale x 16 x i8> %a, %b
%2 = sext <vscale x 16 x i1> %1 to <vscale x 16 x i8>
- %3 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sdot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %2, <vscale x 16 x i8> %2)
- ret <vscale x 4 x i32> %3
-}
-
-define <vscale x 4 x i32> @test2(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
-; CHECK-LABEL: test2:
-; CHECK: // %bb.0:
-; CHECK-NEXT: uqsub z1.b, z1.b, z0.b
-; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: umin z1.b, z1.b, #1
-; CHECK-NEXT: sdot z0.s, z1.b, z1.b
-; CHECK-NEXT: ret
-{
- %1 = icmp ult <vscale x 16 x i8> %a, %b
- %2 = zext <vscale x 16 x i1> %1 to <vscale x 16 x i8>
- %3 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sdot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %2, <vscale x 16 x i8> %2)
- ret <vscale x 4 x i32> %3
-}
-
-define <vscale x 4 x i32> @test3(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
-; CHECK-LABEL: test3:
-; CHECK: // %bb.0:
-; CHECK-NEXT: uqsub z1.b, z1.b, z0.b
-; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: umin z1.b, z1.b, #1
-; CHECK-NEXT: sdot z0.s, z1.b, z1.b
-; CHECK-NEXT: ret
-{
- %1 = icmp ult <vscale x 16 x i8> %a, %b
- %2 = zext <vscale x 16 x i1> %1 to <vscale x 16 x i8>
- %3 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sdot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %2, <vscale x 16 x i8> %2)
- ret <vscale x 4 x i32> %3
+ ret <vscale x 16 x i8> %2
}
-define <vscale x 4 x i32> @test4(<32 x i8> %a, <32 x i8> %b) {
-; CHECK-LABEL: test4:
+define <vscale x 16 x i8> @sext_slt_test2(<32 x i8> %a, <32 x i8> %b)
+; CHECK-LABEL: sext_slt_test2:
; CHECK: // %bb.0:
; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
; CHECK-NEXT: ptrue p0.b, vl16
@@ -56,42 +24,32 @@ define <vscale x 4 x i32> @test4(<32 x i8> %a, <32 x i8> %b) {
; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
-; CHECK-NEXT: uqsub z1.b, z1.b, z0.b
-; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: umin z1.b, z1.b, #1
-; CHECK-NEXT: sdot z0.s, z1.b, z1.b
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: cmpgt p1.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT: ret
- %1 = icmp ult <32 x i8> %a, %b
+{
+ %1 = icmp slt <32 x i8> %a, %b
%2 = sext <32 x i1> %1 to <32 x i8>
- %3 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v32i8(<vscale x 16 x i8> poison, <32 x i8> %2, i64 0)
- %4 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sdot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %3, <vscale x 16 x i8> %3)
- ret <vscale x 4 x i32> %4
+ %3 = tail call <vscale x 16 x i8> @llvm.vector.insert(<vscale x 16 x i8> poison, <32 x i8> %2, i64 0)
+ ret <vscale x 16 x i8> %3
}
-define <vscale x 4 x i32> @test5(<32 x i8> %a, <32 x i8> %b) {
-; CHECK-LABEL: test5:
+define <vscale x 16 x i8> @zext_slt_test1(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
+; CHECK-LABEL: zext_slt_test1:
; CHECK: // %bb.0:
-; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
-; CHECK-NEXT: ptrue p0.b, vl16
-; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
-; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
-; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
-; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
-; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
-; CHECK-NEXT: uqsub z1.b, z1.b, z0.b
-; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: umin z1.b, z1.b, #1
-; CHECK-NEXT: sdot z0.s, z1.b, z1.b
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: cmpgt p1.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: mov z0.b, p1/z, #1 // =0x1
; CHECK-NEXT: ret
- %1 = icmp ult <32 x i8> %a, %b
- %2 = zext <32 x i1> %1 to <32 x i8>
- %3 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v32i8(<vscale x 16 x i8> poison, <32 x i8> %2, i64 0)
- %4 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sdot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %3, <vscale x 16 x i8> %3)
- ret <vscale x 4 x i32> %4
+{
+ %1 = icmp slt <vscale x 16 x i8> %a, %b
+ %2 = zext <vscale x 16 x i1> %1 to <vscale x 16 x i8>
+ ret <vscale x 16 x i8> %2
}
-define <vscale x 4 x i32> @test6(<32 x i8> %a, <32 x i8> %b) {
-; CHECK-LABEL: test6:
+define <vscale x 16 x i8> @zext_slt_test2(<32 x i8> %a, <32 x i8> %b)
+; CHECK-LABEL: zext_slt_test2:
; CHECK: // %bb.0:
; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
; CHECK-NEXT: ptrue p0.b, vl16
@@ -100,146 +58,103 @@ define <vscale x 4 x i32> @test6(<32 x i8> %a, <32 x i8> %b) {
; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
-; CHECK-NEXT: uqsub z1.b, z1.b, z0.b
-; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: umin z1.b, z1.b, #1
-; CHECK-NEXT: sdot z0.s, z1.b, z1.b
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: cmpgt p1.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: and z0.b, z0.b, #0x1
; CHECK-NEXT: ret
- %1 = icmp ult <32 x i8> %a, %b
+{
+ %1 = icmp slt <32 x i8> %a, %b
%2 = zext <32 x i1> %1 to <32 x i8>
- %3 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v32i8(<vscale x 16 x i8> poison, <32 x i8> %2, i64 0)
- %4 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sdot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %3, <vscale x 16 x i8> %3)
- ret <vscale x 4 x i32> %4
+ %3 = tail call <vscale x 16 x i8> @llvm.vector.insert(<vscale x 16 x i8> poison, <32 x i8> %2, i64 0)
+ ret <vscale x 16 x i8> %3
}
-define <vscale x 4 x i32> @test7(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
-; CHECK-LABEL: test7:
+define <vscale x 16 x i8> @sext_ult_test1(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
+; CHECK-LABEL: sext_ult_test1:
; CHECK: // %bb.0:
-; CHECK-NEXT: uqsub z1.b, z0.b, z1.b
-; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: umin z1.b, z1.b, #1
-; CHECK-NEXT: sdot z0.s, z1.b, z1.b
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: cmphi p1.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT: ret
{
- %1 = icmp ugt <vscale x 16 x i8> %a, %b
+ %1 = icmp ult <vscale x 16 x i8> %a, %b
%2 = sext <vscale x 16 x i1> %1 to <vscale x 16 x i8>
- %3 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sdot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %2, <vscale x 16 x i8> %2)
- ret <vscale x 4 x i32> %3
-}
-
-define <vscale x 4 x i32> @test8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
-; CHECK-LABEL: test8:
-; CHECK: // %bb.0:
-; CHECK-NEXT: uqsub z1.b, z0.b, z1.b
-; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: umin z1.b, z1.b, #1
-; CHECK-NEXT: sdot z0.s, z1.b, z1.b
-; CHECK-NEXT: ret
-{
- %1 = icmp ugt <vscale x 16 x i8> %a, %b
- %2 = zext <vscale x 16 x i1> %1 to <vscale x 16 x i8>
- %3 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sdot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %2, <vscale x 16 x i8> %2)
- ret <vscale x 4 x i32> %3
+ ret <vscale x 16 x i8> %2
}
-define <vscale x 4 x i32> @test9(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
-; CHECK-LABEL: test9:
+define <vscale x 16 x i8> @sext_ult_test2(<32 x i8> %a, <32 x i8> %b)
+; CHECK-LABEL: sext_ult_test2:
; CHECK: // %bb.0:
-; CHECK-NEXT: uqsub z1.b, z0.b, z1.b
-; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: umin z1.b, z1.b, #1
-; CHECK-NEXT: sdot z0.s, z1.b, z1.b
-; CHECK-NEXT: ret
-{
- %1 = icmp ugt <vscale x 16 x i8> %a, %b
- %2 = zext <vscale x 16 x i1> %1 to <vscale x 16 x i8>
- %3 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sdot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %2, <vscale x 16 x i8> %2)
- ret <vscale x 4 x i32> %3
-}
-
-define <vscale x 4 x i32> @test10(<32 x i8> %a, <32 x i8> %b) {
-; CHECK-LABEL: test10:
-; CHECK: // %bb.0:
-; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
; CHECK-NEXT: ptrue p0.b, vl16
-; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
-; CHECK-NEXT: splice z2.b, p0, { z2.b, z3.b }
+; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
-; CHECK-NEXT: uqsub z1.b, z0.b, z2.b
-; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: umin z1.b, z1.b, #1
-; CHECK-NEXT: sdot z0.s, z1.b, z1.b
+; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: cmphi p1.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT: ret
- %1 = icmp ugt <32 x i8> %a, %b
+{
+ %1 = icmp ult <32 x i8> %a, %b
%2 = sext <32 x i1> %1 to <32 x i8>
- %3 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v32i8(<vscale x 16 x i8> poison, <32 x i8> %2, i64 0)
- %4 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sdot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %3, <vscale x 16 x i8> %3)
- ret <vscale x 4 x i32> %4
+ %3 = tail call <vscale x 16 x i8> @llvm.vector.insert(<vscale x 16 x i8> poison, <32 x i8> %2, i64 0)
+ ret <vscale x 16 x i8> %3
}
-define <vscale x 4 x i32> @test11(<32 x i8> %a, <32 x i8> %b) {
-; CHECK-LABEL: test11:
+define <vscale x 16 x i8> @zext_ult_test1(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
+; CHECK-LABEL: zext_ult_test1:
; CHECK: // %bb.0:
-; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
-; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
-; CHECK-NEXT: ptrue p0.b, vl16
-; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
-; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
-; CHECK-NEXT: splice z2.b, p0, { z2.b, z3.b }
-; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
-; CHECK-NEXT: uqsub z1.b, z0.b, z2.b
-; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: umin z1.b, z1.b, #1
-; CHECK-NEXT: sdot z0.s, z1.b, z1.b
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: cmphi p1.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: mov z0.b, p1/z, #1 // =0x1
; CHECK-NEXT: ret
- %1 = icmp ugt <32 x i8> %a, %b
- %2 = zext <32 x i1> %1 to <32 x i8>
- %3 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v32i8(<vscale x 16 x i8> poison, <32 x i8> %2, i64 0)
- %4 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sdot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %3, <vscale x 16 x i8> %3)
- ret <vscale x 4 x i32> %4
+{
+ %1 = icmp ult <vscale x 16 x i8> %a, %b
+ %2 = zext <vscale x 16 x i1> %1 to <vscale x 16 x i8>
+ ret <vscale x 16 x i8> %2
}
-define <vscale x 4 x i32> @test12(<32 x i8> %a, <32 x i8> %b) {
-; CHECK-LABEL: test12:
+define <vscale x 16 x i8> @zext_ult_test2(<32 x i8> %a, <32 x i8> %b)
+; CHECK-LABEL: zext_ult_test2:
; CHECK: // %bb.0:
-; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
; CHECK-NEXT: ptrue p0.b, vl16
-; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
-; CHECK-NEXT: splice z2.b, p0, { z2.b, z3.b }
+; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
-; CHECK-NEXT: uqsub z1.b, z0.b, z2.b
-; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: umin z1.b, z1.b, #1
-; CHECK-NEXT: sdot z0.s, z1.b, z1.b
+; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: cmphi p1.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: and z0.b, z0.b, #0x1
; CHECK-NEXT: ret
- %1 = icmp ugt <32 x i8> %a, %b
+{
+ %1 = icmp ult <32 x i8> %a, %b
%2 = zext <32 x i1> %1 to <32 x i8>
- %3 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v32i8(<vscale x 16 x i8> poison, <32 x i8> %2, i64 0)
- %4 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sdot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %3, <vscale x 16 x i8> %3)
- ret <vscale x 4 x i32> %4
+ %3 = tail call <vscale x 16 x i8> @llvm.vector.insert(<vscale x 16 x i8> poison, <32 x i8> %2, i64 0)
+ ret <vscale x 16 x i8> %3
}
-define <vscale x 16 x i8> @test13(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1) {
-; CHECK-LABEL: test13:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: movi v2.2d, #0000000000000000
-; CHECK-NEXT: sqsub z0.b, z0.b, z1.b
-; CHECK-NEXT: mov z1.b, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: sclamp z0.b, z1.b, z2.b
+define <vscale x 16 x i8> @sext_sgt_test1(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
+; CHECK-LABEL: sext_sgt_test1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: cmpgt p1.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT: ret
-entry:
- %2 = icmp slt <vscale x 16 x i8> %0, %1
- %3 = sext <vscale x 16 x i1> %2 to <vscale x 16 x i8>
- ret <vscale x 16 x i8> %3
+{
+ %1 = icmp sgt <vscale x 16 x i8> %a, %b
+ %2 = sext <vscale x 16 x i1> %1 to <vscale x 16 x i8>
+ ret <vscale x 16 x i8> %2
}
-define <vscale x 16 x i8> @test14(<32 x i8> %0, <32 x i8> %1) {
-; CHECK-LABEL: test14:
-; CHECK: // %bb.0: // %entry
+define <vscale x 16 x i8> @sext_sgt_test2(<32 x i8> %a, <32 x i8> %b)
+; CHECK-LABEL: sext_sgt_test2:
+; CHECK: // %bb.0:
; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
; CHECK-NEXT: ptrue p0.b, vl16
@@ -247,231 +162,67 @@ define <vscale x 16 x i8> @test14(<32 x i8> %0, <32 x i8> %1) {
; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
; CHECK-NEXT: splice z2.b, p0, { z2.b, z3.b }
; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
-; CHECK-NEXT: movi v1.2d, #0000000000000000
-; CHECK-NEXT: sqsub z0.b, z0.b, z2.b
-; CHECK-NEXT: mov z2.b, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: sclamp z0.b, z2.b, z1.b
-; CHECK-NEXT: ret
-entry:
- %2 = icmp slt <32 x i8> %0, %1
- %3 = sext <32 x i1> %2 to <32 x i8>
- %4 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v32i8(<vscale x 16 x i8> poison, <32 x i8> %3, i64 0)
- ret <vscale x 16 x i8> %4
-}
-
-; FIXME: Lower this sequence to sqsub/sclamp
-define <vscale x 16 x i8> @test15(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1) {
-; CHECK-LABEL: test15:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: cmpgt p1.b, p0/z, z1.b, z0.b
-; CHECK-NEXT: mov z1.b, #1 // =0x1
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: cmpgt p1.b, p0/z, z0.b, z2.b
; CHECK-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: add z0.b, p1/m, z0.b, z1.b
-; CHECK-NEXT: ret
-entry:
- %2 = icmp slt <vscale x 16 x i8> %0, %1
- %3 = sext <vscale x 16 x i1> %2 to <vscale x 16 x i8>
- %4 = zext <vscale x 16 x i1> %2 to <vscale x 16 x i8>
- %5 = add <vscale x 16 x i8> %3, %4
- ret <vscale x 16 x i8> %5
-}
-
-define <vscale x 16 x i8> @test16(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1) {
-; CHECK-LABEL: test16:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: movi v2.2d, #0000000000000000
-; CHECK-NEXT: sqsub z0.b, z1.b, z0.b
-; CHECK-NEXT: mov z1.b, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: sclamp z0.b, z1.b, z2.b
-; CHECK-NEXT: ret
-entry:
- %2 = icmp sgt <vscale x 16 x i8> %0, %1
- %3 = sext <vscale x 16 x i1> %2 to <vscale x 16 x i8>
- ret <vscale x 16 x i8> %3
-}
-
-define <vscale x 16 x i8> @test17(<32 x i8> %0, <32 x i8> %1) {
-; CHECK-LABEL: test17:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
-; CHECK-NEXT: ptrue p0.b, vl16
-; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
-; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
-; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
-; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
-; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
-; CHECK-NEXT: mov z2.b, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: sqsub z0.b, z1.b, z0.b
-; CHECK-NEXT: movi v1.2d, #0000000000000000
-; CHECK-NEXT: sclamp z0.b, z2.b, z1.b
-; CHECK-NEXT: ret
-entry:
- %2 = icmp sgt <32 x i8> %0, %1
- %3 = sext <32 x i1> %2 to <32 x i8>
- %4 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v32i8(<vscale x 16 x i8> poison, <32 x i8> %3, i64 0)
- ret <vscale x 16 x i8> %4
-}
-
-define <vscale x 4 x i32> @test18(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
-; CHECK-LABEL: test18:
-; CHECK: // %bb.0:
-; CHECK-NEXT: uqsub z1.b, z1.b, z0.b
-; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: umin z1.b, z1.b, #1
-; CHECK-NEXT: udot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
{
- %1 = icmp ult <vscale x 16 x i8> %a, %b
- %2 = sext <vscale x 16 x i1> %1 to <vscale x 16 x i8>
- %3 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.udot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %2, <vscale x 16 x i8> %2)
- ret <vscale x 4 x i32> %3
-}
-
-define <vscale x 4 x i32> @test19(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
-; CHECK-LABEL: test19:
-; CHECK: // %bb.0:
-; CHECK-NEXT: uqsub z1.b, z1.b, z0.b
-; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: umin z1.b, z1.b, #1
-; CHECK-NEXT: udot z0.s, z1.b, z1.b
-; CHECK-NEXT: ret
-{
- %1 = icmp ult <vscale x 16 x i8> %a, %b
- %2 = zext <vscale x 16 x i1> %1 to <vscale x 16 x i8>
- %3 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.udot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %2, <vscale x 16 x i8> %2)
- ret <vscale x 4 x i32> %3
+ %1 = icmp sgt <32 x i8> %a, %b
+ %2 = sext <32 x i1> %1 to <32 x i8>
+ %3 = tail call <vscale x 16 x i8> @llvm.vector.insert(<vscale x 16 x i8> poison, <32 x i8> %2, i64 0)
+ ret <vscale x 16 x i8> %3
}
-
-define <vscale x 4 x i32> @test20(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
-; CHECK-LABEL: test20:
+define <vscale x 16 x i8> @zext_sgt_test1(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
+; CHECK-LABEL: zext_sgt_test1:
; CHECK: // %bb.0:
-; CHECK-NEXT: uqsub z1.b, z1.b, z0.b
-; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: umin z1.b, z1.b, #1
-; CHECK-NEXT: udot z0.s, z1.b, z1.b
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: cmpgt p1.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: mov z0.b, p1/z, #1 // =0x1
; CHECK-NEXT: ret
{
- %1 = icmp ult <vscale x 16 x i8> %a, %b
+ %1 = icmp sgt <vscale x 16 x i8> %a, %b
%2 = zext <vscale x 16 x i1> %1 to <vscale x 16 x i8>
- %3 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.udot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %2, <vscale x 16 x i8> %2)
- ret <vscale x 4 x i32> %3
+ ret <vscale x 16 x i8> %2
}
-define <vscale x 4 x i32> @test21(<32 x i8> %a, <32 x i8> %b) {
-; CHECK-LABEL: test21:
+define <vscale x 16 x i8> @zext_sgt_test2(<32 x i8> %a, <32 x i8> %b)
+; CHECK-LABEL: zext_sgt_test2:
; CHECK: // %bb.0:
-; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
-; CHECK-NEXT: ptrue p0.b, vl16
; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
-; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
-; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
-; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
-; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
-; CHECK-NEXT: uqsub z1.b, z1.b, z0.b
-; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: umin z1.b, z1.b, #1
-; CHECK-NEXT: udot z0.s, z1.b, z1.b
-; CHECK-NEXT: ret
- %1 = icmp ult <32 x i8> %a, %b
- %2 = sext <32 x i1> %1 to <32 x i8>
- %3 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v32i8(<vscale x 16 x i8> poison, <32 x i8> %2, i64 0)
- %4 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.udot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %3, <vscale x 16 x i8> %3)
- ret <vscale x 4 x i32> %4
-}
-
-define <vscale x 4 x i32> @test22(<32 x i8> %a, <32 x i8> %b) {
-; CHECK-LABEL: test22:
-; CHECK: // %bb.0:
; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
; CHECK-NEXT: ptrue p0.b, vl16
-; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
-; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
-; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
-; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
-; CHECK-NEXT: uqsub z1.b, z1.b, z0.b
-; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: umin z1.b, z1.b, #1
-; CHECK-NEXT: udot z0.s, z1.b, z1.b
-; CHECK-NEXT: ret
- %1 = icmp ult <32 x i8> %a, %b
- %2 = zext <32 x i1> %1 to <32 x i8>
- %3 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v32i8(<vscale x 16 x i8> poison, <32 x i8> %2, i64 0)
- %4 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.udot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %3, <vscale x 16 x i8> %3)
- ret <vscale x 4 x i32> %4
-}
-
-define <vscale x 4 x i32> @test23(<32 x i8> %a, <32 x i8> %b) {
-; CHECK-LABEL: test23:
-; CHECK: // %bb.0:
-; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
-; CHECK-NEXT: ptrue p0.b, vl16
-; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
-; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: splice z2.b, p0, { z2.b, z3.b }
; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
-; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
-; CHECK-NEXT: uqsub z1.b, z1.b, z0.b
-; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: umin z1.b, z1.b, #1
-; CHECK-NEXT: udot z0.s, z1.b, z1.b
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: cmpgt p1.b, p0/z, z0.b, z2.b
+; CHECK-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: and z0.b, z0.b, #0x1
; CHECK-NEXT: ret
- %1 = icmp ult <32 x i8> %a, %b
+{
+ %1 = icmp sgt <32 x i8> %a, %b
%2 = zext <32 x i1> %1 to <32 x i8>
- %3 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v32i8(<vscale x 16 x i8> poison, <32 x i8> %2, i64 0)
- %4 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.udot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %3, <vscale x 16 x i8> %3)
- ret <vscale x 4 x i32> %4
+ %3 = tail call <vscale x 16 x i8> @llvm.vector.insert(<vscale x 16 x i8> poison, <32 x i8> %2, i64 0)
+ ret <vscale x 16 x i8> %3
}
-define <vscale x 4 x i32> @test24(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
-; CHECK-LABEL: test24:
+define <vscale x 16 x i8> @sext_ugt_test1(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
+; CHECK-LABEL: sext_ugt_test1:
; CHECK: // %bb.0:
-; CHECK-NEXT: uqsub z1.b, z0.b, z1.b
-; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: umin z1.b, z1.b, #1
-; CHECK-NEXT: udot z0.s, z1.b, z1.b
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: cmphi p1.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT: ret
{
%1 = icmp ugt <vscale x 16 x i8> %a, %b
%2 = sext <vscale x 16 x i1> %1 to <vscale x 16 x i8>
- %3 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.udot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %2, <vscale x 16 x i8> %2)
- ret <vscale x 4 x i32> %3
+ ret <vscale x 16 x i8> %2
}
-define <vscale x 4 x i32> @test25(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
-; CHECK-LABEL: test25:
-; CHECK: // %bb.0:
-; CHECK-NEXT: uqsub z1.b, z0.b, z1.b
-; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: umin z1.b, z1.b, #1
-; CHECK-NEXT: udot z0.s, z1.b, z1.b
-; CHECK-NEXT: ret
-{
- %1 = icmp ugt <vscale x 16 x i8> %a, %b
- %2 = zext <vscale x 16 x i1> %1 to <vscale x 16 x i8>
- %3 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.udot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %2, <vscale x 16 x i8> %2)
- ret <vscale x 4 x i32> %3
-}
-
-define <vscale x 4 x i32> @test26(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
-; CHECK-LABEL: test26:
-; CHECK: // %bb.0:
-; CHECK-NEXT: uqsub z1.b, z0.b, z1.b
-; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: umin z1.b, z1.b, #1
-; CHECK-NEXT: udot z0.s, z1.b, z1.b
-; CHECK-NEXT: ret
-{
- %1 = icmp ugt <vscale x 16 x i8> %a, %b
- %2 = zext <vscale x 16 x i1> %1 to <vscale x 16 x i8>
- %3 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.udot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %2, <vscale x 16 x i8> %2)
- ret <vscale x 4 x i32> %3
-}
-
-define <vscale x 4 x i32> @test27(<32 x i8> %a, <32 x i8> %b) {
-; CHECK-LABEL: test27:
+define <vscale x 16 x i8> @sext_ugt_test2(<32 x i8> %a, <32 x i8> %b)
+; CHECK-LABEL: sext_ugt_test2:
; CHECK: // %bb.0:
; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
@@ -480,42 +231,32 @@ define <vscale x 4 x i32> @test27(<32 x i8> %a, <32 x i8> %b) {
; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
; CHECK-NEXT: splice z2.b, p0, { z2.b, z3.b }
; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
-; CHECK-NEXT: uqsub z1.b, z0.b, z2.b
-; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: umin z1.b, z1.b, #1
-; CHECK-NEXT: udot z0.s, z1.b, z1.b
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: cmphi p1.b, p0/z, z0.b, z2.b
+; CHECK-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff
; CHECK-NEXT: ret
+{
%1 = icmp ugt <32 x i8> %a, %b
%2 = sext <32 x i1> %1 to <32 x i8>
- %3 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v32i8(<vscale x 16 x i8> poison, <32 x i8> %2, i64 0)
- %4 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.udot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %3, <vscale x 16 x i8> %3)
- ret <vscale x 4 x i32> %4
+ %3 = tail call <vscale x 16 x i8> @llvm.vector.insert(<vscale x 16 x i8> poison, <32 x i8> %2, i64 0)
+ ret <vscale x 16 x i8> %3
}
-define <vscale x 4 x i32> @test28(<32 x i8> %a, <32 x i8> %b) {
-; CHECK-LABEL: test28:
+define <vscale x 16 x i8> @zext_ugt_test1(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
+; CHECK-LABEL: zext_ugt_test1:
; CHECK: // %bb.0:
-; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
-; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
-; CHECK-NEXT: ptrue p0.b, vl16
-; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
-; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
-; CHECK-NEXT: splice z2.b, p0, { z2.b, z3.b }
-; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
-; CHECK-NEXT: uqsub z1.b, z0.b, z2.b
-; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: umin z1.b, z1.b, #1
-; CHECK-NEXT: udot z0.s, z1.b, z1.b
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: cmphi p1.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: mov z0.b, p1/z, #1 // =0x1
; CHECK-NEXT: ret
- %1 = icmp ugt <32 x i8> %a, %b
- %2 = zext <32 x i1> %1 to <32 x i8>
- %3 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v32i8(<vscale x 16 x i8> poison, <32 x i8> %2, i64 0)
- %4 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.udot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %3, <vscale x 16 x i8> %3)
- ret <vscale x 4 x i32> %4
+{
+ %1 = icmp ugt <vscale x 16 x i8> %a, %b
+ %2 = zext <vscale x 16 x i1> %1 to <vscale x 16 x i8>
+ ret <vscale x 16 x i8> %2
}
-define <vscale x 4 x i32> @test29(<32 x i8> %a, <32 x i8> %b) {
-; CHECK-LABEL: test29:
+define <vscale x 16 x i8> @zext_ugt_test2(<32 x i8> %a, <32 x i8> %b)
+; CHECK-LABEL: zext_ugt_test2:
; CHECK: // %bb.0:
; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
@@ -524,14 +265,14 @@ define <vscale x 4 x i32> @test29(<32 x i8> %a, <32 x i8> %b) {
; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
; CHECK-NEXT: splice z2.b, p0, { z2.b, z3.b }
; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
-; CHECK-NEXT: uqsub z1.b, z0.b, z2.b
-; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: umin z1.b, z1.b, #1
-; CHECK-NEXT: udot z0.s, z1.b, z1.b
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: cmphi p1.b, p0/z, z0.b, z2.b
+; CHECK-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: and z0.b, z0.b, #0x1
; CHECK-NEXT: ret
+{
%1 = icmp ugt <32 x i8> %a, %b
%2 = zext <32 x i1> %1 to <32 x i8>
- %3 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v32i8(<vscale x 16 x i8> poison, <32 x i8> %2, i64 0)
- %4 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.udot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %3, <vscale x 16 x i8> %3)
- ret <vscale x 4 x i32> %4
+ %3 = tail call <vscale x 16 x i8> @llvm.vector.insert(<vscale x 16 x i8> poison, <32 x i8> %2, i64 0)
+ ret <vscale x 16 x i8> %3
}
>From 89a7f81fc214c58810332dfc299336e2836ce9e6 Mon Sep 17 00:00:00 2001
From: sgokhale <sgokhale at nvidia.com>
Date: Wed, 15 Apr 2026 02:35:30 -0700
Subject: [PATCH 4/4] [AArch64][SelectionDAG] Fold following extend(icmp)
patterns
sext(icmp slt a, b) --> asr (shsub a, b), bitwidth-1
sext(icmp ult a, b) --> asr (uhsub a, b), bitwidth-1
zext(icmp slt a, b) --> lsr (shsub a, b), bitwidth-1
zext(icmp ult a, b) --> lsr (uhsub a, b), bitwidth-1
sext(icmp sgt a, b) --> asr (shsub b, a), bitwidth-1
sext(icmp ugt a, b) --> asr (uhsub b, a), bitwidth-1
zext(icmp sgt a, b) --> lsr (shsub b, a), bitwidth-1
zext(icmp ugt a, b) --> lsr (uhsub b, a), bitwidth-1
---
.../Target/AArch64/AArch64ISelLowering.cpp | 163 +-
llvm/test/CodeGen/AArch64/active_lane_mask.ll | 24 +-
.../CodeGen/AArch64/sve-icmp-sext-zext.ll | 116 +-
...treaming-mode-fixed-length-int-compares.ll | 566 +-
...eaming-mode-fixed-length-int-immediates.ll | 4600 +++++++++++++++--
5 files changed, 4736 insertions(+), 733 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 1f83b8cb705ea..767b63dc01501 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -24106,37 +24106,6 @@ static SDValue performIntrinsicCombine(SDNode *N,
case Intrinsic::aarch64_sve_bsl2n:
case Intrinsic::aarch64_sve_nbsl:
return combineSVEBitSel(IID, N, DAG);
- case Intrinsic::aarch64_sve_udot:
- case Intrinsic::aarch64_sve_sdot: {
- // sdot(acc, extend(icmp_ult(A, B)), extend(icmp_ult(A, B)))
- // -> sdot(acc, umin(usubsat(B, A)), umin(usubsat(B, A)))
-
- // sdot(acc, extend(icmp_ugt(A, B)), extend(icmp_ugt(A, B)))
- // -> sdot(acc, umin(usubsat(A, B)), umin(usubsat(A, B)))
- SDValue Extend1 = N->getOperand(2);
- SDValue Extend2 = N->getOperand(3);
- if (Extend1 == Extend2 && (Extend1.getOpcode() == ISD::SIGN_EXTEND ||
- Extend1.getOpcode() == ISD::ZERO_EXTEND)) {
- SDValue Compare = Extend1.getOperand(0);
- if (Compare.getOpcode() != AArch64ISD::SETCC_MERGE_ZERO)
- break;
- ISD::CondCode CC = cast<CondCodeSDNode>(Compare.getOperand(3))->get();
- if (CC != ISD::SETULT && CC != ISD::SETUGT)
- break;
- SDLoc DL(N);
- EVT ResVT = Extend1.getValueType();
- SDValue A = Compare.getOperand(CC == ISD::SETULT ? 1 : 2);
- SDValue B = Compare.getOperand(CC == ISD::SETULT ? 2 : 1);
- SDValue Sub = DAG.getNode(ISD::USUBSAT, DL, ResVT, B, A);
- SDValue One = DAG.getConstant(1, DL, MVT::i32);
- SDValue SplatOne = DAG.getSplatVector(ResVT, DL, One);
- SDValue Min = DAG.getNode(ISD::UMIN, DL, ResVT, Sub, SplatOne);
- unsigned Opcode = (IID == Intrinsic::aarch64_sve_udot ? AArch64ISD::UDOT
- : AArch64ISD::SDOT);
- return DAG.getNode(Opcode, DL, N->getValueType(0), N->getOperand(1), Min,
- Min);
- }
- }
}
return SDValue();
}
@@ -24468,51 +24437,101 @@ static SDValue performExtendCombine(SDNode *N,
TargetLowering::DAGCombinerInfo &DCI,
SelectionDAG &DAG,
const AArch64Subtarget *Subtarget) {
- // sext(icmp slt(a, b)) -> sclamp(sqsub(a, b), -1, 0)
- // sext(icmp sgt(a, b)) -> sclamp(sqsub(b, a), -1, 0)
- if (N->getOpcode() == ISD::SIGN_EXTEND) {
- SDValue Compare = N->getOperand(0);
- if (Compare.getOpcode() == AArch64ISD::SETCC_MERGE_ZERO) {
- ISD::CondCode CC = cast<CondCodeSDNode>(Compare.getOperand(3))->get();
- if (Compare.hasOneUse() && Subtarget->hasSVE2p1() &&
- (CC == ISD::SETLT || CC == ISD::SETGT)) {
- SDLoc DL(N);
- SDValue A = Compare.getOperand(CC == ISD::SETLT ? 1 : 2);
- SDValue B = Compare.getOperand(CC == ISD::SETLT ? 2 : 1);
- SDValue Sub = DAG.getNode(ISD::SSUBSAT, DL, N->getValueType(0), A, B);
- SDValue MinusOne = DAG.getSplatVector(
- N->getValueType(0), DL, DAG.getSignedConstant(-1, DL, MVT::i32));
- SDValue Zero = DAG.getSplatVector(N->getValueType(0), DL,
- DAG.getConstant(0, DL, MVT::i32));
- SDValue Sclamp = DAG.getNode(
+ // sext(icmp slt a, b) --> asr (shsub a, b), bitwidth-1
+ // sext(icmp ult a, b) --> asr (uhsub a, b), bitwidth-1
+ // zext(icmp slt a, b) --> lsr (shsub a, b), bitwidth-1
+ // zext(icmp ult a, b) --> lsr (uhsub a, b), bitwidth-1
+ // sext(icmp sgt a, b) --> asr (shsub b, a), bitwidth-1
+ // sext(icmp ugt a, b) --> asr (uhsub b, a), bitwidth-1
+ // zext(icmp sgt a, b) --> lsr (shsub b, a), bitwidth-1
+ // zext(icmp ugt a, b) --> lsr (uhsub b, a), bitwidth-1
+ SDValue Compare = N->getOperand(0);
+ SDLoc DL(N);
+ if (Compare.getOpcode() == AArch64ISD::SETCC_MERGE_ZERO &&
+ N->getValueType(0).isScalableVector() &&
+ Compare.getOperand(1).getValueType().getScalarType().isInteger() &&
+ (Subtarget->hasSVE2() || Subtarget->isStreamingSVEAvailable())) {
+ SDValue ComparePred = Compare.getOperand(0);
+ unsigned ExtendTy = N->getOpcode(); // sext or zext
+ ISD::CondCode CC = cast<CondCodeSDNode>(Compare.getOperand(3))->get();
+ SDValue A = Compare.getOperand(1);
+ SDValue B = Compare.getOperand(2);
+ unsigned BitwidthMinusOne = N->getValueType(0).getScalarSizeInBits() - 1;
+ EVT PredVT =
+ N->getValueType(0).changeElementType(*DAG.getContext(), MVT::i1);
+ //SDValue AllTruePred = getPTrue(DAG, DL, PredVT, AArch64SVEPredPattern::all);
+ SDValue AllTruePred = ComparePred ;
+ EVT EltVT = N->getValueType(0).getVectorElementType();
+ MVT ConstantTy = EltVT.bitsGT(MVT::i32) ? MVT::i64 : MVT::i32;
+ SDValue ShiftAmt =
+ DAG.getSplatVector(N->getValueType(0), DL,
+ DAG.getConstant(BitwidthMinusOne, DL, ConstantTy));
+ if (ExtendTy == ISD::SIGN_EXTEND) {
+ if (CC == ISD::SETLT) {
+ // sext(icmp slt a, b) --> asr (shsub a, b), bitwidth-1
+ SDValue Sub = DAG.getNode(
+ ISD::INTRINSIC_WO_CHAIN, DL, N->getValueType(0),
+ DAG.getConstant(Intrinsic::aarch64_sve_shsub_u, DL, MVT::i64),
+ AllTruePred, A, B);
+ return DAG.getNode(ISD::SRA, DL, N->getValueType(0), Sub, ShiftAmt);
+ }
+ if (CC == ISD::SETULT) {
+ // sext(icmp ult a, b) --> asr (uhsub a, b), bitwidth-1
+ SDValue Sub = DAG.getNode(
+ ISD::INTRINSIC_WO_CHAIN, DL, N->getValueType(0),
+ DAG.getConstant(Intrinsic::aarch64_sve_uhsub_u, DL, MVT::i64),
+ AllTruePred, A, B);
+ return DAG.getNode(ISD::SRA, DL, N->getValueType(0), Sub, ShiftAmt);
+ }
+ if (CC == ISD::SETGT) {
+ // sext(icmp sgt a, b) --> asr (shsub b, a), bitwidth-1
+ SDValue Sub = DAG.getNode(
+ ISD::INTRINSIC_WO_CHAIN, DL, N->getValueType(0),
+ DAG.getConstant(Intrinsic::aarch64_sve_shsub_u, DL, MVT::i64),
+ AllTruePred, B, A);
+ return DAG.getNode(ISD::SRA, DL, N->getValueType(0), Sub, ShiftAmt);
+ }
+ if (CC == ISD::SETUGT) {
+ // sext(icmp ugt a, b) --> asr (uhsub b, a), bitwidth-1
+ SDValue Sub = DAG.getNode(
ISD::INTRINSIC_WO_CHAIN, DL, N->getValueType(0),
- DAG.getConstant(Intrinsic::aarch64_sve_sclamp, DL, MVT::i64), Sub,
- MinusOne, Zero);
- return Sclamp;
+ DAG.getConstant(Intrinsic::aarch64_sve_uhsub_u, DL, MVT::i64),
+ AllTruePred, B, A);
+ return DAG.getNode(ISD::SRA, DL, N->getValueType(0), Sub, ShiftAmt);
}
}
- }
- // zext(icmp slt(a, b)) -> sclamp(sqsub(b,a), 0, 1)
- // zext(icmp sgt(a, b)) -> sclamp(sqsub(a,b), 0, 1)
- if (N->getOpcode() == ISD::ZERO_EXTEND) {
- SDValue Compare = N->getOperand(0);
- if (Compare.getOpcode() == AArch64ISD::SETCC_MERGE_ZERO) {
- ISD::CondCode CC = cast<CondCodeSDNode>(Compare.getOperand(3))->get();
- if (Compare.hasOneUse() && Subtarget->hasSVE2p1() &&
- (CC == ISD::SETLT || CC == ISD::SETGT)) {
- SDLoc DL(N);
- SDValue A = Compare.getOperand(CC == ISD::SETLT ? 1 : 2);
- SDValue B = Compare.getOperand(CC == ISD::SETLT ? 2 : 1);
- SDValue Sub = DAG.getNode(ISD::SSUBSAT, DL, N->getValueType(0), B, A);
- SDValue One = DAG.getSplatVector(N->getValueType(0), DL,
- DAG.getConstant(1, DL, MVT::i32));
- SDValue Zero = DAG.getSplatVector(N->getValueType(0), DL,
- DAG.getConstant(0, DL, MVT::i32));
- SDValue Sclamp = DAG.getNode(
+ if (ExtendTy == ISD::ZERO_EXTEND) {
+ if (CC == ISD::SETLT) {
+ // zext(icmp slt a, b) --> lsr (shsub a, b), bitwidth-1
+ SDValue Sub = DAG.getNode(
+ ISD::INTRINSIC_WO_CHAIN, DL, N->getValueType(0),
+ DAG.getConstant(Intrinsic::aarch64_sve_shsub_u, DL, MVT::i64),
+ AllTruePred, A, B);
+ return DAG.getNode(ISD::SRL, DL, N->getValueType(0), Sub, ShiftAmt);
+ }
+ if (CC == ISD::SETULT) {
+ // zext(icmp ult a, b) --> lsr (uhsub a, b), bitwidth-1
+ SDValue Sub = DAG.getNode(
+ ISD::INTRINSIC_WO_CHAIN, DL, N->getValueType(0),
+ DAG.getConstant(Intrinsic::aarch64_sve_uhsub_u, DL, MVT::i64),
+ AllTruePred, A, B);
+ return DAG.getNode(ISD::SRL, DL, N->getValueType(0), Sub, ShiftAmt);
+ }
+ if (CC == ISD::SETGT) {
+ // zext(icmp sgt a, b) --> lsr (shsub b, a), bitwidth-1
+ SDValue Sub = DAG.getNode(
+ ISD::INTRINSIC_WO_CHAIN, DL, N->getValueType(0),
+ DAG.getConstant(Intrinsic::aarch64_sve_shsub_u, DL, MVT::i64),
+ AllTruePred, B, A);
+ return DAG.getNode(ISD::SRL, DL, N->getValueType(0), Sub, ShiftAmt);
+ }
+ if (CC == ISD::SETUGT) {
+ // zext(icmp ugt a, b) --> lsr (uhsub b, a), bitwidth-1
+ SDValue Sub = DAG.getNode(
ISD::INTRINSIC_WO_CHAIN, DL, N->getValueType(0),
- DAG.getConstant(Intrinsic::aarch64_sve_sclamp, DL, MVT::i64), Sub,
- Zero, One);
- return Sclamp;
+ DAG.getConstant(Intrinsic::aarch64_sve_uhsub_u, DL, MVT::i64),
+ AllTruePred, B, A);
+ return DAG.getNode(ISD::SRL, DL, N->getValueType(0), Sub, ShiftAmt);
}
}
}
diff --git a/llvm/test/CodeGen/AArch64/active_lane_mask.ll b/llvm/test/CodeGen/AArch64/active_lane_mask.ll
index 778be79038a78..b94cfe3847074 100644
--- a/llvm/test/CodeGen/AArch64/active_lane_mask.ll
+++ b/llvm/test/CodeGen/AArch64/active_lane_mask.ll
@@ -299,12 +299,12 @@ define <16 x i1> @lane_mask_v16i1_i8(i8 %index, i8 %TC) {
; CHECK-STREAMING-NEXT: index z0.b, w0, #1
; CHECK-STREAMING-NEXT: mov z1.b, w0
; CHECK-STREAMING-NEXT: ptrue p0.b, vl16
-; CHECK-STREAMING-NEXT: cmphi p1.b, p0/z, z1.b, z0.b
-; CHECK-STREAMING-NEXT: mov z1.b, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-STREAMING-NEXT: uhsubr z1.b, p0/m, z1.b, z0.b
+; CHECK-STREAMING-NEXT: asr z1.b, z1.b, #7
; CHECK-STREAMING-NEXT: orr z0.d, z0.d, z1.d
; CHECK-STREAMING-NEXT: mov z1.b, w1
-; CHECK-STREAMING-NEXT: cmphi p1.b, p0/z, z1.b, z0.b
-; CHECK-STREAMING-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-STREAMING-NEXT: uhsub z0.b, p0/m, z0.b, z1.b
+; CHECK-STREAMING-NEXT: asr z0.b, z0.b, #7
; CHECK-STREAMING-NEXT: ret
%active.lane.mask = call <16 x i1> @llvm.get.active.lane.mask.v16i1.i8(i8 %index, i8 %TC)
ret <16 x i1> %active.lane.mask
@@ -325,12 +325,12 @@ define <8 x i1> @lane_mask_v8i1_i8(i8 %index, i8 %TC) {
; CHECK-STREAMING-NEXT: index z0.b, w0, #1
; CHECK-STREAMING-NEXT: mov z1.b, w0
; CHECK-STREAMING-NEXT: ptrue p0.b, vl8
-; CHECK-STREAMING-NEXT: cmphi p1.b, p0/z, z1.b, z0.b
-; CHECK-STREAMING-NEXT: mov z1.b, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-STREAMING-NEXT: uhsubr z1.b, p0/m, z1.b, z0.b
+; CHECK-STREAMING-NEXT: asr z1.b, z1.b, #7
; CHECK-STREAMING-NEXT: orr z0.d, z0.d, z1.d
; CHECK-STREAMING-NEXT: mov z1.b, w1
-; CHECK-STREAMING-NEXT: cmphi p1.b, p0/z, z1.b, z0.b
-; CHECK-STREAMING-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-STREAMING-NEXT: uhsub z0.b, p0/m, z0.b, z1.b
+; CHECK-STREAMING-NEXT: asr z0.b, z0.b, #7
; CHECK-STREAMING-NEXT: ret
%active.lane.mask = call <8 x i1> @llvm.get.active.lane.mask.v8i1.i8(i8 %index, i8 %TC)
ret <8 x i1> %active.lane.mask
@@ -360,8 +360,8 @@ define <4 x i1> @lane_mask_v4i1_i8(i8 %index, i8 %TC) {
; CHECK-STREAMING-NEXT: mov z1.h, w1
; CHECK-STREAMING-NEXT: umin z0.h, z0.h, #255
; CHECK-STREAMING-NEXT: and z1.h, z1.h, #0xff
-; CHECK-STREAMING-NEXT: cmphi p1.h, p0/z, z1.h, z0.h
-; CHECK-STREAMING-NEXT: mov z0.h, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-STREAMING-NEXT: uhsub z0.h, p0/m, z0.h, z1.h
+; CHECK-STREAMING-NEXT: asr z0.h, z0.h, #15
; CHECK-STREAMING-NEXT: ret
%active.lane.mask = call <4 x i1> @llvm.get.active.lane.mask.v4i1.i8(i8 %index, i8 %TC)
ret <4 x i1> %active.lane.mask
@@ -389,8 +389,8 @@ define <2 x i1> @lane_mask_v2i1_i8(i8 %index, i8 %TC) {
; CHECK-STREAMING-NEXT: and w8, w1, #0xff
; CHECK-STREAMING-NEXT: mov z1.s, w8
; CHECK-STREAMING-NEXT: umin z0.s, z0.s, #255
-; CHECK-STREAMING-NEXT: cmphi p1.s, p0/z, z1.s, z0.s
-; CHECK-STREAMING-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-STREAMING-NEXT: uhsub z0.s, p0/m, z0.s, z1.s
+; CHECK-STREAMING-NEXT: asr z0.s, z0.s, #31
; CHECK-STREAMING-NEXT: ret
%active.lane.mask = call <2 x i1> @llvm.get.active.lane.mask.v2i1.i8(i8 %index, i8 %TC)
ret <2 x i1> %active.lane.mask
diff --git a/llvm/test/CodeGen/AArch64/sve-icmp-sext-zext.ll b/llvm/test/CodeGen/AArch64/sve-icmp-sext-zext.ll
index 7b585a2b713d4..634f48d372f66 100644
--- a/llvm/test/CodeGen/AArch64/sve-icmp-sext-zext.ll
+++ b/llvm/test/CodeGen/AArch64/sve-icmp-sext-zext.ll
@@ -5,8 +5,8 @@ define <vscale x 16 x i8> @sext_slt_test1(<vscale x 16 x i8> %a, <vscale x 16 x
; CHECK-LABEL: sext_slt_test1:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: cmpgt p1.b, p0/z, z1.b, z0.b
-; CHECK-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: shsub z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT: asr z0.b, z0.b, #7
; CHECK-NEXT: ret
{
%1 = icmp slt <vscale x 16 x i8> %a, %b
@@ -17,16 +17,16 @@ define <vscale x 16 x i8> @sext_slt_test1(<vscale x 16 x i8> %a, <vscale x 16 x
define <vscale x 16 x i8> @sext_slt_test2(<32 x i8> %a, <32 x i8> %b)
; CHECK-LABEL: sext_slt_test2:
; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
; CHECK-NEXT: ptrue p0.b, vl16
-; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
-; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: splice z2.b, p0, { z2.b, z3.b }
; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
-; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
; CHECK-NEXT: ptrue p0.b, vl32
-; CHECK-NEXT: cmpgt p1.b, p0/z, z1.b, z0.b
-; CHECK-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: shsub z0.b, p0/m, z0.b, z2.b
+; CHECK-NEXT: asr z0.b, z0.b, #7
; CHECK-NEXT: ret
{
%1 = icmp slt <32 x i8> %a, %b
@@ -39,8 +39,8 @@ define <vscale x 16 x i8> @zext_slt_test1(<vscale x 16 x i8> %a, <vscale x 16 x
; CHECK-LABEL: zext_slt_test1:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: cmpgt p1.b, p0/z, z1.b, z0.b
-; CHECK-NEXT: mov z0.b, p1/z, #1 // =0x1
+; CHECK-NEXT: shsub z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT: lsr z0.b, z0.b, #7
; CHECK-NEXT: ret
{
%1 = icmp slt <vscale x 16 x i8> %a, %b
@@ -51,17 +51,16 @@ define <vscale x 16 x i8> @zext_slt_test1(<vscale x 16 x i8> %a, <vscale x 16 x
define <vscale x 16 x i8> @zext_slt_test2(<32 x i8> %a, <32 x i8> %b)
; CHECK-LABEL: zext_slt_test2:
; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
; CHECK-NEXT: ptrue p0.b, vl16
-; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
-; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: splice z2.b, p0, { z2.b, z3.b }
; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
-; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
; CHECK-NEXT: ptrue p0.b, vl32
-; CHECK-NEXT: cmpgt p1.b, p0/z, z1.b, z0.b
-; CHECK-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: and z0.b, z0.b, #0x1
+; CHECK-NEXT: shsub z0.b, p0/m, z0.b, z2.b
+; CHECK-NEXT: lsr z0.b, z0.b, #7
; CHECK-NEXT: ret
{
%1 = icmp slt <32 x i8> %a, %b
@@ -74,8 +73,8 @@ define <vscale x 16 x i8> @sext_ult_test1(<vscale x 16 x i8> %a, <vscale x 16 x
; CHECK-LABEL: sext_ult_test1:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: cmphi p1.b, p0/z, z1.b, z0.b
-; CHECK-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: uhsub z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT: asr z0.b, z0.b, #7
; CHECK-NEXT: ret
{
%1 = icmp ult <vscale x 16 x i8> %a, %b
@@ -86,16 +85,16 @@ define <vscale x 16 x i8> @sext_ult_test1(<vscale x 16 x i8> %a, <vscale x 16 x
define <vscale x 16 x i8> @sext_ult_test2(<32 x i8> %a, <32 x i8> %b)
; CHECK-LABEL: sext_ult_test2:
; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
; CHECK-NEXT: ptrue p0.b, vl16
-; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
-; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: splice z2.b, p0, { z2.b, z3.b }
; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
-; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
; CHECK-NEXT: ptrue p0.b, vl32
-; CHECK-NEXT: cmphi p1.b, p0/z, z1.b, z0.b
-; CHECK-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: uhsub z0.b, p0/m, z0.b, z2.b
+; CHECK-NEXT: asr z0.b, z0.b, #7
; CHECK-NEXT: ret
{
%1 = icmp ult <32 x i8> %a, %b
@@ -108,8 +107,8 @@ define <vscale x 16 x i8> @zext_ult_test1(<vscale x 16 x i8> %a, <vscale x 16 x
; CHECK-LABEL: zext_ult_test1:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: cmphi p1.b, p0/z, z1.b, z0.b
-; CHECK-NEXT: mov z0.b, p1/z, #1 // =0x1
+; CHECK-NEXT: uhsub z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT: lsr z0.b, z0.b, #7
; CHECK-NEXT: ret
{
%1 = icmp ult <vscale x 16 x i8> %a, %b
@@ -120,17 +119,16 @@ define <vscale x 16 x i8> @zext_ult_test1(<vscale x 16 x i8> %a, <vscale x 16 x
define <vscale x 16 x i8> @zext_ult_test2(<32 x i8> %a, <32 x i8> %b)
; CHECK-LABEL: zext_ult_test2:
; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
; CHECK-NEXT: ptrue p0.b, vl16
-; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
-; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: splice z2.b, p0, { z2.b, z3.b }
; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
-; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
; CHECK-NEXT: ptrue p0.b, vl32
-; CHECK-NEXT: cmphi p1.b, p0/z, z1.b, z0.b
-; CHECK-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: and z0.b, z0.b, #0x1
+; CHECK-NEXT: uhsub z0.b, p0/m, z0.b, z2.b
+; CHECK-NEXT: lsr z0.b, z0.b, #7
; CHECK-NEXT: ret
{
%1 = icmp ult <32 x i8> %a, %b
@@ -143,8 +141,8 @@ define <vscale x 16 x i8> @sext_sgt_test1(<vscale x 16 x i8> %a, <vscale x 16 x
; CHECK-LABEL: sext_sgt_test1:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: cmpgt p1.b, p0/z, z0.b, z1.b
-; CHECK-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: shsubr z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT: asr z0.b, z0.b, #7
; CHECK-NEXT: ret
{
%1 = icmp sgt <vscale x 16 x i8> %a, %b
@@ -155,16 +153,16 @@ define <vscale x 16 x i8> @sext_sgt_test1(<vscale x 16 x i8> %a, <vscale x 16 x
define <vscale x 16 x i8> @sext_sgt_test2(<32 x i8> %a, <32 x i8> %b)
; CHECK-LABEL: sext_sgt_test2:
; CHECK: // %bb.0:
-; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
; CHECK-NEXT: ptrue p0.b, vl16
-; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
-; CHECK-NEXT: splice z2.b, p0, { z2.b, z3.b }
+; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
+; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
; CHECK-NEXT: ptrue p0.b, vl32
-; CHECK-NEXT: cmpgt p1.b, p0/z, z0.b, z2.b
-; CHECK-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: shsubr z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT: asr z0.b, z0.b, #7
; CHECK-NEXT: ret
{
%1 = icmp sgt <32 x i8> %a, %b
@@ -177,8 +175,8 @@ define <vscale x 16 x i8> @zext_sgt_test1(<vscale x 16 x i8> %a, <vscale x 16 x
; CHECK-LABEL: zext_sgt_test1:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: cmpgt p1.b, p0/z, z0.b, z1.b
-; CHECK-NEXT: mov z0.b, p1/z, #1 // =0x1
+; CHECK-NEXT: shsubr z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT: lsr z0.b, z0.b, #7
; CHECK-NEXT: ret
{
%1 = icmp sgt <vscale x 16 x i8> %a, %b
@@ -189,17 +187,16 @@ define <vscale x 16 x i8> @zext_sgt_test1(<vscale x 16 x i8> %a, <vscale x 16 x
define <vscale x 16 x i8> @zext_sgt_test2(<32 x i8> %a, <32 x i8> %b)
; CHECK-LABEL: zext_sgt_test2:
; CHECK: // %bb.0:
-; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
; CHECK-NEXT: ptrue p0.b, vl16
-; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
-; CHECK-NEXT: splice z2.b, p0, { z2.b, z3.b }
+; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
+; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
; CHECK-NEXT: ptrue p0.b, vl32
-; CHECK-NEXT: cmpgt p1.b, p0/z, z0.b, z2.b
-; CHECK-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: and z0.b, z0.b, #0x1
+; CHECK-NEXT: shsubr z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT: lsr z0.b, z0.b, #7
; CHECK-NEXT: ret
{
%1 = icmp sgt <32 x i8> %a, %b
@@ -212,8 +209,8 @@ define <vscale x 16 x i8> @sext_ugt_test1(<vscale x 16 x i8> %a, <vscale x 16 x
; CHECK-LABEL: sext_ugt_test1:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: cmphi p1.b, p0/z, z0.b, z1.b
-; CHECK-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: uhsubr z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT: asr z0.b, z0.b, #7
; CHECK-NEXT: ret
{
%1 = icmp ugt <vscale x 16 x i8> %a, %b
@@ -224,16 +221,16 @@ define <vscale x 16 x i8> @sext_ugt_test1(<vscale x 16 x i8> %a, <vscale x 16 x
define <vscale x 16 x i8> @sext_ugt_test2(<32 x i8> %a, <32 x i8> %b)
; CHECK-LABEL: sext_ugt_test2:
; CHECK: // %bb.0:
-; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
; CHECK-NEXT: ptrue p0.b, vl16
-; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
-; CHECK-NEXT: splice z2.b, p0, { z2.b, z3.b }
+; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
+; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
; CHECK-NEXT: ptrue p0.b, vl32
-; CHECK-NEXT: cmphi p1.b, p0/z, z0.b, z2.b
-; CHECK-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: uhsubr z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT: asr z0.b, z0.b, #7
; CHECK-NEXT: ret
{
%1 = icmp ugt <32 x i8> %a, %b
@@ -246,8 +243,8 @@ define <vscale x 16 x i8> @zext_ugt_test1(<vscale x 16 x i8> %a, <vscale x 16 x
; CHECK-LABEL: zext_ugt_test1:
; CHECK: // %bb.0:
; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: cmphi p1.b, p0/z, z0.b, z1.b
-; CHECK-NEXT: mov z0.b, p1/z, #1 // =0x1
+; CHECK-NEXT: uhsubr z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT: lsr z0.b, z0.b, #7
; CHECK-NEXT: ret
{
%1 = icmp ugt <vscale x 16 x i8> %a, %b
@@ -258,17 +255,16 @@ define <vscale x 16 x i8> @zext_ugt_test1(<vscale x 16 x i8> %a, <vscale x 16 x
define <vscale x 16 x i8> @zext_ugt_test2(<32 x i8> %a, <32 x i8> %b)
; CHECK-LABEL: zext_ugt_test2:
; CHECK: // %bb.0:
-; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
; CHECK-NEXT: ptrue p0.b, vl16
-; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
-; CHECK-NEXT: splice z2.b, p0, { z2.b, z3.b }
+; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
+; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
; CHECK-NEXT: ptrue p0.b, vl32
-; CHECK-NEXT: cmphi p1.b, p0/z, z0.b, z2.b
-; CHECK-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: and z0.b, z0.b, #0x1
+; CHECK-NEXT: uhsubr z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT: lsr z0.b, z0.b, #7
; CHECK-NEXT: ret
{
%1 = icmp ugt <32 x i8> %a, %b
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-compares.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-compares.ll
index 72c4fe528db32..291e1fe711978 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-compares.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-compares.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -enable-subreg-liveness -mattr=+sve -force-streaming-compatible < %s | FileCheck %s
-; RUN: llc -enable-subreg-liveness -mattr=+sme -force-streaming < %s | FileCheck %s
-; RUN: llc -enable-subreg-liveness -force-streaming-compatible < %s | FileCheck %s --check-prefix=NONEON-NOSVE
+; RUN: llc -enable-subreg-liveness -mattr=+sve -force-streaming-compatible < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SVE
+; RUN: llc -enable-subreg-liveness -mattr=+sme -force-streaming < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SME
+; RUN: llc -enable-subreg-liveness -force-streaming-compatible < %s | FileCheck %s --check-prefixes=CHECK,NONEON-NOSVE
target triple = "aarch64-unknown-linux-gnu"
@@ -10,12 +10,19 @@ target triple = "aarch64-unknown-linux-gnu"
;
define <8 x i8> @icmp_eq_v8i8(<8 x i8> %op1, <8 x i8> %op2) {
-; CHECK-LABEL: icmp_eq_v8i8:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.b, vl8
-; CHECK-NEXT: cmpeq p1.b, p0/z, z0.b, z1.b
-; CHECK-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: icmp_eq_v8i8:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ptrue p0.b, vl8
+; CHECK-SVE-NEXT: cmpeq p1.b, p0/z, z0.b, z1.b
+; CHECK-SVE-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: icmp_eq_v8i8:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ptrue p0.b, vl8
+; CHECK-SME-NEXT: cmpeq p1.b, p0/z, z0.b, z1.b
+; CHECK-SME-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-SME-NEXT: ret
;
; NONEON-NOSVE-LABEL: icmp_eq_v8i8:
; NONEON-NOSVE: // %bb.0:
@@ -71,12 +78,19 @@ define <8 x i8> @icmp_eq_v8i8(<8 x i8> %op1, <8 x i8> %op2) {
}
define <16 x i8> @icmp_eq_v16i8(<16 x i8> %op1, <16 x i8> %op2) {
-; CHECK-LABEL: icmp_eq_v16i8:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.b, vl16
-; CHECK-NEXT: cmpeq p1.b, p0/z, z0.b, z1.b
-; CHECK-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: icmp_eq_v16i8:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ptrue p0.b, vl16
+; CHECK-SVE-NEXT: cmpeq p1.b, p0/z, z0.b, z1.b
+; CHECK-SVE-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: icmp_eq_v16i8:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ptrue p0.b, vl16
+; CHECK-SME-NEXT: cmpeq p1.b, p0/z, z0.b, z1.b
+; CHECK-SME-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-SME-NEXT: ret
;
; NONEON-NOSVE-LABEL: icmp_eq_v16i8:
; NONEON-NOSVE: // %bb.0:
@@ -171,17 +185,29 @@ define <16 x i8> @icmp_eq_v16i8(<16 x i8> %op1, <16 x i8> %op2) {
}
define void @icmp_eq_v32i8(ptr %a, ptr %b) {
-; CHECK-LABEL: icmp_eq_v32i8:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q3, [x1]
-; CHECK-NEXT: ptrue p0.b, vl16
-; CHECK-NEXT: ldp q1, q2, [x0]
-; CHECK-NEXT: cmpeq p1.b, p0/z, z1.b, z0.b
-; CHECK-NEXT: cmpeq p2.b, p0/z, z2.b, z3.b
-; CHECK-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: mov z1.b, p2/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: icmp_eq_v32i8:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q3, [x1]
+; CHECK-SVE-NEXT: ptrue p0.b, vl16
+; CHECK-SVE-NEXT: ldp q1, q2, [x0]
+; CHECK-SVE-NEXT: cmpeq p1.b, p0/z, z1.b, z0.b
+; CHECK-SVE-NEXT: cmpeq p2.b, p0/z, z2.b, z3.b
+; CHECK-SVE-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-SVE-NEXT: mov z1.b, p2/z, #-1 // =0xffffffffffffffff
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: icmp_eq_v32i8:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q3, [x1]
+; CHECK-SME-NEXT: ptrue p0.b, vl16
+; CHECK-SME-NEXT: ldp q1, q2, [x0]
+; CHECK-SME-NEXT: cmpeq p1.b, p0/z, z1.b, z0.b
+; CHECK-SME-NEXT: cmpeq p2.b, p0/z, z2.b, z3.b
+; CHECK-SME-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-SME-NEXT: mov z1.b, p2/z, #-1 // =0xffffffffffffffff
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
;
; NONEON-NOSVE-LABEL: icmp_eq_v32i8:
; NONEON-NOSVE: // %bb.0:
@@ -364,12 +390,19 @@ define void @icmp_eq_v32i8(ptr %a, ptr %b) {
}
define <4 x i16> @icmp_eq_v4i16(<4 x i16> %op1, <4 x i16> %op2) {
-; CHECK-LABEL: icmp_eq_v4i16:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.h, vl4
-; CHECK-NEXT: cmpeq p1.h, p0/z, z0.h, z1.h
-; CHECK-NEXT: mov z0.h, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: icmp_eq_v4i16:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ptrue p0.h, vl4
+; CHECK-SVE-NEXT: cmpeq p1.h, p0/z, z0.h, z1.h
+; CHECK-SVE-NEXT: mov z0.h, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: icmp_eq_v4i16:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ptrue p0.h, vl4
+; CHECK-SME-NEXT: cmpeq p1.h, p0/z, z0.h, z1.h
+; CHECK-SME-NEXT: mov z0.h, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-SME-NEXT: ret
;
; NONEON-NOSVE-LABEL: icmp_eq_v4i16:
; NONEON-NOSVE: // %bb.0:
@@ -405,12 +438,19 @@ define <4 x i16> @icmp_eq_v4i16(<4 x i16> %op1, <4 x i16> %op2) {
}
define <8 x i16> @icmp_eq_v8i16(<8 x i16> %op1, <8 x i16> %op2) {
-; CHECK-LABEL: icmp_eq_v8i16:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.h, vl8
-; CHECK-NEXT: cmpeq p1.h, p0/z, z0.h, z1.h
-; CHECK-NEXT: mov z0.h, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: icmp_eq_v8i16:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ptrue p0.h, vl8
+; CHECK-SVE-NEXT: cmpeq p1.h, p0/z, z0.h, z1.h
+; CHECK-SVE-NEXT: mov z0.h, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: icmp_eq_v8i16:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ptrue p0.h, vl8
+; CHECK-SME-NEXT: cmpeq p1.h, p0/z, z0.h, z1.h
+; CHECK-SME-NEXT: mov z0.h, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-SME-NEXT: ret
;
; NONEON-NOSVE-LABEL: icmp_eq_v8i16:
; NONEON-NOSVE: // %bb.0:
@@ -465,17 +505,29 @@ define <8 x i16> @icmp_eq_v8i16(<8 x i16> %op1, <8 x i16> %op2) {
}
define void @icmp_eq_v16i16(ptr %a, ptr %b) {
-; CHECK-LABEL: icmp_eq_v16i16:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q3, [x1]
-; CHECK-NEXT: ptrue p0.h, vl8
-; CHECK-NEXT: ldp q1, q2, [x0]
-; CHECK-NEXT: cmpeq p1.h, p0/z, z1.h, z0.h
-; CHECK-NEXT: cmpeq p2.h, p0/z, z2.h, z3.h
-; CHECK-NEXT: mov z0.h, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: mov z1.h, p2/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: icmp_eq_v16i16:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q3, [x1]
+; CHECK-SVE-NEXT: ptrue p0.h, vl8
+; CHECK-SVE-NEXT: ldp q1, q2, [x0]
+; CHECK-SVE-NEXT: cmpeq p1.h, p0/z, z1.h, z0.h
+; CHECK-SVE-NEXT: cmpeq p2.h, p0/z, z2.h, z3.h
+; CHECK-SVE-NEXT: mov z0.h, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-SVE-NEXT: mov z1.h, p2/z, #-1 // =0xffffffffffffffff
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: icmp_eq_v16i16:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q3, [x1]
+; CHECK-SME-NEXT: ptrue p0.h, vl8
+; CHECK-SME-NEXT: ldp q1, q2, [x0]
+; CHECK-SME-NEXT: cmpeq p1.h, p0/z, z1.h, z0.h
+; CHECK-SME-NEXT: cmpeq p2.h, p0/z, z2.h, z3.h
+; CHECK-SME-NEXT: mov z0.h, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-SME-NEXT: mov z1.h, p2/z, #-1 // =0xffffffffffffffff
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
;
; NONEON-NOSVE-LABEL: icmp_eq_v16i16:
; NONEON-NOSVE: // %bb.0:
@@ -578,12 +630,19 @@ define void @icmp_eq_v16i16(ptr %a, ptr %b) {
}
define <2 x i32> @icmp_eq_v2i32(<2 x i32> %op1, <2 x i32> %op2) {
-; CHECK-LABEL: icmp_eq_v2i32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.s, vl2
-; CHECK-NEXT: cmpeq p1.s, p0/z, z0.s, z1.s
-; CHECK-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: icmp_eq_v2i32:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ptrue p0.s, vl2
+; CHECK-SVE-NEXT: cmpeq p1.s, p0/z, z0.s, z1.s
+; CHECK-SVE-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: icmp_eq_v2i32:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ptrue p0.s, vl2
+; CHECK-SME-NEXT: cmpeq p1.s, p0/z, z0.s, z1.s
+; CHECK-SME-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-SME-NEXT: ret
;
; NONEON-NOSVE-LABEL: icmp_eq_v2i32:
; NONEON-NOSVE: // %bb.0:
@@ -607,12 +666,19 @@ define <2 x i32> @icmp_eq_v2i32(<2 x i32> %op1, <2 x i32> %op2) {
}
define <4 x i32> @icmp_eq_v4i32(<4 x i32> %op1, <4 x i32> %op2) {
-; CHECK-LABEL: icmp_eq_v4i32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.s, vl4
-; CHECK-NEXT: cmpeq p1.s, p0/z, z0.s, z1.s
-; CHECK-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: icmp_eq_v4i32:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ptrue p0.s, vl4
+; CHECK-SVE-NEXT: cmpeq p1.s, p0/z, z0.s, z1.s
+; CHECK-SVE-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: icmp_eq_v4i32:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ptrue p0.s, vl4
+; CHECK-SME-NEXT: cmpeq p1.s, p0/z, z0.s, z1.s
+; CHECK-SME-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-SME-NEXT: ret
;
; NONEON-NOSVE-LABEL: icmp_eq_v4i32:
; NONEON-NOSVE: // %bb.0:
@@ -643,17 +709,29 @@ define <4 x i32> @icmp_eq_v4i32(<4 x i32> %op1, <4 x i32> %op2) {
}
define void @icmp_eq_v8i32(ptr %a, ptr %b) {
-; CHECK-LABEL: icmp_eq_v8i32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q3, [x1]
-; CHECK-NEXT: ptrue p0.s, vl4
-; CHECK-NEXT: ldp q1, q2, [x0]
-; CHECK-NEXT: cmpeq p1.s, p0/z, z1.s, z0.s
-; CHECK-NEXT: cmpeq p2.s, p0/z, z2.s, z3.s
-; CHECK-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: mov z1.s, p2/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: icmp_eq_v8i32:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q3, [x1]
+; CHECK-SVE-NEXT: ptrue p0.s, vl4
+; CHECK-SVE-NEXT: ldp q1, q2, [x0]
+; CHECK-SVE-NEXT: cmpeq p1.s, p0/z, z1.s, z0.s
+; CHECK-SVE-NEXT: cmpeq p2.s, p0/z, z2.s, z3.s
+; CHECK-SVE-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-SVE-NEXT: mov z1.s, p2/z, #-1 // =0xffffffffffffffff
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: icmp_eq_v8i32:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q3, [x1]
+; CHECK-SME-NEXT: ptrue p0.s, vl4
+; CHECK-SME-NEXT: ldp q1, q2, [x0]
+; CHECK-SME-NEXT: cmpeq p1.s, p0/z, z1.s, z0.s
+; CHECK-SME-NEXT: cmpeq p2.s, p0/z, z2.s, z3.s
+; CHECK-SME-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-SME-NEXT: mov z1.s, p2/z, #-1 // =0xffffffffffffffff
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
;
; NONEON-NOSVE-LABEL: icmp_eq_v8i32:
; NONEON-NOSVE: // %bb.0:
@@ -708,12 +786,19 @@ define void @icmp_eq_v8i32(ptr %a, ptr %b) {
}
define <1 x i64> @icmp_eq_v1i64(<1 x i64> %op1, <1 x i64> %op2) {
-; CHECK-LABEL: icmp_eq_v1i64:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.d, vl1
-; CHECK-NEXT: cmpeq p1.d, p0/z, z0.d, z1.d
-; CHECK-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: icmp_eq_v1i64:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ptrue p0.d, vl1
+; CHECK-SVE-NEXT: cmpeq p1.d, p0/z, z0.d, z1.d
+; CHECK-SVE-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: icmp_eq_v1i64:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ptrue p0.d, vl1
+; CHECK-SME-NEXT: cmpeq p1.d, p0/z, z0.d, z1.d
+; CHECK-SME-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-SME-NEXT: ret
;
; NONEON-NOSVE-LABEL: icmp_eq_v1i64:
; NONEON-NOSVE: // %bb.0:
@@ -732,12 +817,19 @@ define <1 x i64> @icmp_eq_v1i64(<1 x i64> %op1, <1 x i64> %op2) {
}
define <2 x i64> @icmp_eq_v2i64(<2 x i64> %op1, <2 x i64> %op2) {
-; CHECK-LABEL: icmp_eq_v2i64:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.d, vl2
-; CHECK-NEXT: cmpeq p1.d, p0/z, z0.d, z1.d
-; CHECK-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: icmp_eq_v2i64:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ptrue p0.d, vl2
+; CHECK-SVE-NEXT: cmpeq p1.d, p0/z, z0.d, z1.d
+; CHECK-SVE-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: icmp_eq_v2i64:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ptrue p0.d, vl2
+; CHECK-SME-NEXT: cmpeq p1.d, p0/z, z0.d, z1.d
+; CHECK-SME-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-SME-NEXT: ret
;
; NONEON-NOSVE-LABEL: icmp_eq_v2i64:
; NONEON-NOSVE: // %bb.0:
@@ -760,17 +852,29 @@ define <2 x i64> @icmp_eq_v2i64(<2 x i64> %op1, <2 x i64> %op2) {
}
define void @icmp_eq_v4i64(ptr %a, ptr %b) {
-; CHECK-LABEL: icmp_eq_v4i64:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q3, [x1]
-; CHECK-NEXT: ptrue p0.d, vl2
-; CHECK-NEXT: ldp q1, q2, [x0]
-; CHECK-NEXT: cmpeq p1.d, p0/z, z1.d, z0.d
-; CHECK-NEXT: cmpeq p2.d, p0/z, z2.d, z3.d
-; CHECK-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: mov z1.d, p2/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: icmp_eq_v4i64:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q3, [x1]
+; CHECK-SVE-NEXT: ptrue p0.d, vl2
+; CHECK-SVE-NEXT: ldp q1, q2, [x0]
+; CHECK-SVE-NEXT: cmpeq p1.d, p0/z, z1.d, z0.d
+; CHECK-SVE-NEXT: cmpeq p2.d, p0/z, z2.d, z3.d
+; CHECK-SVE-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-SVE-NEXT: mov z1.d, p2/z, #-1 // =0xffffffffffffffff
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: icmp_eq_v4i64:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q3, [x1]
+; CHECK-SME-NEXT: ptrue p0.d, vl2
+; CHECK-SME-NEXT: ldp q1, q2, [x0]
+; CHECK-SME-NEXT: cmpeq p1.d, p0/z, z1.d, z0.d
+; CHECK-SME-NEXT: cmpeq p2.d, p0/z, z2.d, z3.d
+; CHECK-SME-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-SME-NEXT: mov z1.d, p2/z, #-1 // =0xffffffffffffffff
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
;
; NONEON-NOSVE-LABEL: icmp_eq_v4i64:
; NONEON-NOSVE: // %bb.0:
@@ -813,17 +917,29 @@ define void @icmp_eq_v4i64(ptr %a, ptr %b) {
;
define void @icmp_ne_v32i8(ptr %a, ptr %b) {
-; CHECK-LABEL: icmp_ne_v32i8:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q3, [x1]
-; CHECK-NEXT: ptrue p0.b, vl16
-; CHECK-NEXT: ldp q1, q2, [x0]
-; CHECK-NEXT: cmpne p1.b, p0/z, z1.b, z0.b
-; CHECK-NEXT: cmpne p2.b, p0/z, z2.b, z3.b
-; CHECK-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: mov z1.b, p2/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: icmp_ne_v32i8:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q3, [x1]
+; CHECK-SVE-NEXT: ptrue p0.b, vl16
+; CHECK-SVE-NEXT: ldp q1, q2, [x0]
+; CHECK-SVE-NEXT: cmpne p1.b, p0/z, z1.b, z0.b
+; CHECK-SVE-NEXT: cmpne p2.b, p0/z, z2.b, z3.b
+; CHECK-SVE-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-SVE-NEXT: mov z1.b, p2/z, #-1 // =0xffffffffffffffff
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: icmp_ne_v32i8:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q3, [x1]
+; CHECK-SME-NEXT: ptrue p0.b, vl16
+; CHECK-SME-NEXT: ldp q1, q2, [x0]
+; CHECK-SME-NEXT: cmpne p1.b, p0/z, z1.b, z0.b
+; CHECK-SME-NEXT: cmpne p2.b, p0/z, z2.b, z3.b
+; CHECK-SME-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-SME-NEXT: mov z1.b, p2/z, #-1 // =0xffffffffffffffff
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
;
; NONEON-NOSVE-LABEL: icmp_ne_v32i8:
; NONEON-NOSVE: // %bb.0:
@@ -1010,15 +1126,25 @@ define void @icmp_ne_v32i8(ptr %a, ptr %b) {
;
define void @icmp_sge_v8i16(ptr %a, ptr %b) {
-; CHECK-LABEL: icmp_sge_v8i16:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.h, vl8
-; CHECK-NEXT: ldr q0, [x0]
-; CHECK-NEXT: ldr q1, [x1]
-; CHECK-NEXT: cmpge p1.h, p0/z, z0.h, z1.h
-; CHECK-NEXT: mov z0.h, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: str q0, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: icmp_sge_v8i16:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ptrue p0.h, vl8
+; CHECK-SVE-NEXT: ldr q0, [x0]
+; CHECK-SVE-NEXT: ldr q1, [x1]
+; CHECK-SVE-NEXT: cmpge p1.h, p0/z, z0.h, z1.h
+; CHECK-SVE-NEXT: mov z0.h, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-SVE-NEXT: str q0, [x0]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: icmp_sge_v8i16:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ptrue p0.h, vl8
+; CHECK-SME-NEXT: ldr q0, [x0]
+; CHECK-SME-NEXT: ldr q1, [x1]
+; CHECK-SME-NEXT: cmpge p1.h, p0/z, z0.h, z1.h
+; CHECK-SME-NEXT: mov z0.h, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-SME-NEXT: str q0, [x0]
+; CHECK-SME-NEXT: ret
;
; NONEON-NOSVE-LABEL: icmp_sge_v8i16:
; NONEON-NOSVE: // %bb.0:
@@ -1083,17 +1209,29 @@ define void @icmp_sge_v8i16(ptr %a, ptr %b) {
;
define void @icmp_sgt_v16i16(ptr %a, ptr %b) {
-; CHECK-LABEL: icmp_sgt_v16i16:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q3, [x1]
-; CHECK-NEXT: ptrue p0.h, vl8
-; CHECK-NEXT: ldp q1, q2, [x0]
-; CHECK-NEXT: cmpgt p1.h, p0/z, z1.h, z0.h
-; CHECK-NEXT: cmpgt p2.h, p0/z, z2.h, z3.h
-; CHECK-NEXT: mov z0.h, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: mov z1.h, p2/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: icmp_sgt_v16i16:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q3, [x1]
+; CHECK-SVE-NEXT: ptrue p0.h, vl8
+; CHECK-SVE-NEXT: ldp q1, q2, [x0]
+; CHECK-SVE-NEXT: cmpgt p1.h, p0/z, z1.h, z0.h
+; CHECK-SVE-NEXT: cmpgt p2.h, p0/z, z2.h, z3.h
+; CHECK-SVE-NEXT: mov z0.h, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-SVE-NEXT: mov z1.h, p2/z, #-1 // =0xffffffffffffffff
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: icmp_sgt_v16i16:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q3, [x1]
+; CHECK-SME-NEXT: ptrue p0.h, vl8
+; CHECK-SME-NEXT: ldp q1, q2, [x0]
+; CHECK-SME-NEXT: shsub z0.h, p0/m, z0.h, z1.h
+; CHECK-SME-NEXT: shsubr z2.h, p0/m, z2.h, z3.h
+; CHECK-SME-NEXT: asr z0.h, z0.h, #15
+; CHECK-SME-NEXT: asr z1.h, z2.h, #15
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
;
; NONEON-NOSVE-LABEL: icmp_sgt_v16i16:
; NONEON-NOSVE: // %bb.0:
@@ -1200,15 +1338,25 @@ define void @icmp_sgt_v16i16(ptr %a, ptr %b) {
;
define void @icmp_sle_v4i32(ptr %a, ptr %b) {
-; CHECK-LABEL: icmp_sle_v4i32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.s, vl4
-; CHECK-NEXT: ldr q0, [x0]
-; CHECK-NEXT: ldr q1, [x1]
-; CHECK-NEXT: cmpge p1.s, p0/z, z1.s, z0.s
-; CHECK-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: str q0, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: icmp_sle_v4i32:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ptrue p0.s, vl4
+; CHECK-SVE-NEXT: ldr q0, [x0]
+; CHECK-SVE-NEXT: ldr q1, [x1]
+; CHECK-SVE-NEXT: cmpge p1.s, p0/z, z1.s, z0.s
+; CHECK-SVE-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-SVE-NEXT: str q0, [x0]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: icmp_sle_v4i32:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ptrue p0.s, vl4
+; CHECK-SME-NEXT: ldr q0, [x0]
+; CHECK-SME-NEXT: ldr q1, [x1]
+; CHECK-SME-NEXT: cmpge p1.s, p0/z, z1.s, z0.s
+; CHECK-SME-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-SME-NEXT: str q0, [x0]
+; CHECK-SME-NEXT: ret
;
; NONEON-NOSVE-LABEL: icmp_sle_v4i32:
; NONEON-NOSVE: // %bb.0:
@@ -1249,17 +1397,29 @@ define void @icmp_sle_v4i32(ptr %a, ptr %b) {
;
define void @icmp_slt_v8i32(ptr %a, ptr %b) {
-; CHECK-LABEL: icmp_slt_v8i32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q3, [x1]
-; CHECK-NEXT: ptrue p0.s, vl4
-; CHECK-NEXT: ldp q1, q2, [x0]
-; CHECK-NEXT: cmpgt p1.s, p0/z, z0.s, z1.s
-; CHECK-NEXT: cmpgt p2.s, p0/z, z3.s, z2.s
-; CHECK-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: mov z1.s, p2/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: icmp_slt_v8i32:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q3, [x1]
+; CHECK-SVE-NEXT: ptrue p0.s, vl4
+; CHECK-SVE-NEXT: ldp q1, q2, [x0]
+; CHECK-SVE-NEXT: cmpgt p1.s, p0/z, z0.s, z1.s
+; CHECK-SVE-NEXT: cmpgt p2.s, p0/z, z3.s, z2.s
+; CHECK-SVE-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-SVE-NEXT: mov z1.s, p2/z, #-1 // =0xffffffffffffffff
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: icmp_slt_v8i32:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q3, [x1]
+; CHECK-SME-NEXT: ptrue p0.s, vl4
+; CHECK-SME-NEXT: ldp q1, q2, [x0]
+; CHECK-SME-NEXT: shsubr z0.s, p0/m, z0.s, z1.s
+; CHECK-SME-NEXT: shsub z2.s, p0/m, z2.s, z3.s
+; CHECK-SME-NEXT: asr z0.s, z0.s, #31
+; CHECK-SME-NEXT: asr z1.s, z2.s, #31
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
;
; NONEON-NOSVE-LABEL: icmp_slt_v8i32:
; NONEON-NOSVE: // %bb.0:
@@ -1318,15 +1478,25 @@ define void @icmp_slt_v8i32(ptr %a, ptr %b) {
;
define void @icmp_uge_v2i64(ptr %a, ptr %b) {
-; CHECK-LABEL: icmp_uge_v2i64:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.d, vl2
-; CHECK-NEXT: ldr q0, [x0]
-; CHECK-NEXT: ldr q1, [x1]
-; CHECK-NEXT: cmphs p1.d, p0/z, z0.d, z1.d
-; CHECK-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: str q0, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: icmp_uge_v2i64:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ptrue p0.d, vl2
+; CHECK-SVE-NEXT: ldr q0, [x0]
+; CHECK-SVE-NEXT: ldr q1, [x1]
+; CHECK-SVE-NEXT: cmphs p1.d, p0/z, z0.d, z1.d
+; CHECK-SVE-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-SVE-NEXT: str q0, [x0]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: icmp_uge_v2i64:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ptrue p0.d, vl2
+; CHECK-SME-NEXT: ldr q0, [x0]
+; CHECK-SME-NEXT: ldr q1, [x1]
+; CHECK-SME-NEXT: cmphs p1.d, p0/z, z0.d, z1.d
+; CHECK-SME-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-SME-NEXT: str q0, [x0]
+; CHECK-SME-NEXT: ret
;
; NONEON-NOSVE-LABEL: icmp_uge_v2i64:
; NONEON-NOSVE: // %bb.0:
@@ -1359,15 +1529,25 @@ define void @icmp_uge_v2i64(ptr %a, ptr %b) {
;
define void @icmp_ugt_v2i64(ptr %a, ptr %b) {
-; CHECK-LABEL: icmp_ugt_v2i64:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.d, vl2
-; CHECK-NEXT: ldr q0, [x0]
-; CHECK-NEXT: ldr q1, [x1]
-; CHECK-NEXT: cmphi p1.d, p0/z, z0.d, z1.d
-; CHECK-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: str q0, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: icmp_ugt_v2i64:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ptrue p0.d, vl2
+; CHECK-SVE-NEXT: ldr q0, [x0]
+; CHECK-SVE-NEXT: ldr q1, [x1]
+; CHECK-SVE-NEXT: cmphi p1.d, p0/z, z0.d, z1.d
+; CHECK-SVE-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-SVE-NEXT: str q0, [x0]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: icmp_ugt_v2i64:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ptrue p0.d, vl2
+; CHECK-SME-NEXT: ldr q0, [x0]
+; CHECK-SME-NEXT: ldr q1, [x1]
+; CHECK-SME-NEXT: uhsubr z0.d, p0/m, z0.d, z1.d
+; CHECK-SME-NEXT: asr z0.d, z0.d, #63
+; CHECK-SME-NEXT: str q0, [x0]
+; CHECK-SME-NEXT: ret
;
; NONEON-NOSVE-LABEL: icmp_ugt_v2i64:
; NONEON-NOSVE: // %bb.0:
@@ -1400,15 +1580,25 @@ define void @icmp_ugt_v2i64(ptr %a, ptr %b) {
;
define void @icmp_ule_v2i64(ptr %a, ptr %b) {
-; CHECK-LABEL: icmp_ule_v2i64:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.d, vl2
-; CHECK-NEXT: ldr q0, [x0]
-; CHECK-NEXT: ldr q1, [x1]
-; CHECK-NEXT: cmphs p1.d, p0/z, z1.d, z0.d
-; CHECK-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: str q0, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: icmp_ule_v2i64:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ptrue p0.d, vl2
+; CHECK-SVE-NEXT: ldr q0, [x0]
+; CHECK-SVE-NEXT: ldr q1, [x1]
+; CHECK-SVE-NEXT: cmphs p1.d, p0/z, z1.d, z0.d
+; CHECK-SVE-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-SVE-NEXT: str q0, [x0]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: icmp_ule_v2i64:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ptrue p0.d, vl2
+; CHECK-SME-NEXT: ldr q0, [x0]
+; CHECK-SME-NEXT: ldr q1, [x1]
+; CHECK-SME-NEXT: cmphs p1.d, p0/z, z1.d, z0.d
+; CHECK-SME-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-SME-NEXT: str q0, [x0]
+; CHECK-SME-NEXT: ret
;
; NONEON-NOSVE-LABEL: icmp_ule_v2i64:
; NONEON-NOSVE: // %bb.0:
@@ -1441,15 +1631,25 @@ define void @icmp_ule_v2i64(ptr %a, ptr %b) {
;
define void @icmp_ult_v2i64(ptr %a, ptr %b) {
-; CHECK-LABEL: icmp_ult_v2i64:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.d, vl2
-; CHECK-NEXT: ldr q0, [x0]
-; CHECK-NEXT: ldr q1, [x1]
-; CHECK-NEXT: cmphi p1.d, p0/z, z1.d, z0.d
-; CHECK-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: str q0, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: icmp_ult_v2i64:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ptrue p0.d, vl2
+; CHECK-SVE-NEXT: ldr q0, [x0]
+; CHECK-SVE-NEXT: ldr q1, [x1]
+; CHECK-SVE-NEXT: cmphi p1.d, p0/z, z1.d, z0.d
+; CHECK-SVE-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-SVE-NEXT: str q0, [x0]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: icmp_ult_v2i64:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ptrue p0.d, vl2
+; CHECK-SME-NEXT: ldr q0, [x0]
+; CHECK-SME-NEXT: ldr q1, [x1]
+; CHECK-SME-NEXT: uhsub z0.d, p0/m, z0.d, z1.d
+; CHECK-SME-NEXT: asr z0.d, z0.d, #63
+; CHECK-SME-NEXT: str q0, [x0]
+; CHECK-SME-NEXT: ret
;
; NONEON-NOSVE-LABEL: icmp_ult_v2i64:
; NONEON-NOSVE: // %bb.0:
@@ -1476,3 +1676,5 @@ define void @icmp_ult_v2i64(ptr %a, ptr %b) {
store <2 x i64> %sext, ptr %a
ret void
}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-immediates.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-immediates.ll
index fd5a618b0d151..b4acc7ff9171c 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-immediates.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-immediates.ll
@@ -1,7 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mattr=+sve -force-streaming-compatible < %s | FileCheck %s
-; RUN: llc -mattr=+sme -force-streaming < %s | FileCheck %s
-; RUN: llc -force-streaming-compatible < %s | FileCheck %s --check-prefix=NONEON-NOSVE
+; RUN: llc -mattr=+sve -force-streaming-compatible < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SVE
+; RUN: llc -mattr=+sme -force-streaming < %s | FileCheck %s --check-prefixes=CHECK,CHECK-SME
+; RUN: llc -force-streaming-compatible < %s | FileCheck %s --check-prefixes=CHECK,CHECK-NONEON-NOSVE
target triple = "aarch64-unknown-linux-gnu"
@@ -16,14 +16,127 @@ target triple = "aarch64-unknown-linux-gnu"
;
define void @add_v32i8(ptr %a) {
-; CHECK-LABEL: add_v32i8:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: add z0.b, z0.b, #7 // =0x7
-; CHECK-NEXT: add z1.b, z1.b, #7 // =0x7
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: add_v32i8:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: add z0.b, z0.b, #7 // =0x7
+; CHECK-SVE-NEXT: add z1.b, z1.b, #7 // =0x7
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
;
+; CHECK-SME-LABEL: add_v32i8:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: add z0.b, z0.b, #7 // =0x7
+; CHECK-SME-NEXT: add z1.b, z1.b, #7 // =0x7
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
+;
+; CHECK-NONEON-NOSVE-LABEL: add_v32i8:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #31]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #63]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #30]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #62]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #29]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #61]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #28]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #60]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #27]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #59]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #26]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #58]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #25]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #57]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #56]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #23]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #55]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #22]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #54]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #21]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #53]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #20]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #52]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #19]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #51]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #18]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #50]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #17]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #49]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #15]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #47]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #14]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #46]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #13]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #45]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #12]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #44]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #11]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #43]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #10]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #42]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #9]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #41]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #40]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #7]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #39]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #6]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #38]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #5]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #37]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #4]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #36]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #3]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #35]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #2]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #34]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #1]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #33]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: add_v32i8:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -136,14 +249,79 @@ define void @add_v32i8(ptr %a) {
}
define void @add_v16i16(ptr %a) {
-; CHECK-LABEL: add_v16i16:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: add z0.h, z0.h, #15 // =0xf
-; CHECK-NEXT: add z1.h, z1.h, #15 // =0xf
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: add_v16i16:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: add z0.h, z0.h, #15 // =0xf
+; CHECK-SVE-NEXT: add z1.h, z1.h, #15 // =0xf
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: add_v16i16:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: add z0.h, z0.h, #15 // =0xf
+; CHECK-SME-NEXT: add z1.h, z1.h, #15 // =0xf
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
;
+; CHECK-NONEON-NOSVE-LABEL: add_v16i16:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #30]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #62]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #28]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #60]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #26]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #58]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #56]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #22]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #54]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #20]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #52]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #18]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #50]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #14]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #46]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #12]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #44]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #10]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #42]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #40]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #6]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #38]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #4]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #36]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #2]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #34]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: add_v16i16:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -208,14 +386,51 @@ define void @add_v16i16(ptr %a) {
}
define void @add_v8i32(ptr %a) {
-; CHECK-LABEL: add_v8i32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: add z0.s, z0.s, #31 // =0x1f
-; CHECK-NEXT: add z1.s, z1.s, #31 // =0x1f
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: add_v8i32:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: add z0.s, z0.s, #31 // =0x1f
+; CHECK-SVE-NEXT: add z1.s, z1.s, #31 // =0x1f
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
;
+; CHECK-SME-LABEL: add_v8i32:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: add z0.s, z0.s, #31 // =0x1f
+; CHECK-SME-NEXT: add z1.s, z1.s, #31 // =0x1f
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
+;
+; CHECK-NONEON-NOSVE-LABEL: add_v8i32:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #28]
+; CHECK-NONEON-NOSVE-NEXT: add w9, w8, #31
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #31
+; CHECK-NONEON-NOSVE-NEXT: stp w8, w9, [sp, #56]
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #20]
+; CHECK-NONEON-NOSVE-NEXT: add w9, w8, #31
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #31
+; CHECK-NONEON-NOSVE-NEXT: stp w8, w9, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #12]
+; CHECK-NONEON-NOSVE-NEXT: add w9, w8, #31
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #31
+; CHECK-NONEON-NOSVE-NEXT: stp w8, w9, [sp, #40]
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #4]
+; CHECK-NONEON-NOSVE-NEXT: add w9, w8, #31
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp]
+; CHECK-NONEON-NOSVE-NEXT: add w8, w8, #31
+; CHECK-NONEON-NOSVE-NEXT: stp w8, w9, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: add_v8i32:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -252,14 +467,41 @@ define void @add_v8i32(ptr %a) {
}
define void @add_v4i64(ptr %a) {
-; CHECK-LABEL: add_v4i64:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: add z0.d, z0.d, #63 // =0x3f
-; CHECK-NEXT: add z1.d, z1.d, #63 // =0x3f
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: add_v4i64:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: add z0.d, z0.d, #63 // =0x3f
+; CHECK-SVE-NEXT: add z1.d, z1.d, #63 // =0x3f
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: add_v4i64:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: add z0.d, z0.d, #63 // =0x3f
+; CHECK-SME-NEXT: add z1.d, z1.d, #63 // =0x3f
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
;
+; CHECK-NONEON-NOSVE-LABEL: add_v4i64:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldr x8, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: add x9, x8, #63
+; CHECK-NONEON-NOSVE-NEXT: ldr x8, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: add x8, x8, #63
+; CHECK-NONEON-NOSVE-NEXT: stp x8, x9, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldr x8, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: add x9, x8, #63
+; CHECK-NONEON-NOSVE-NEXT: ldr x8, [sp]
+; CHECK-NONEON-NOSVE-NEXT: add x8, x8, #63
+; CHECK-NONEON-NOSVE-NEXT: stp x8, x9, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: add_v4i64:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -290,14 +532,127 @@ define void @add_v4i64(ptr %a) {
;
define void @and_v32i8(ptr %a) {
-; CHECK-LABEL: and_v32i8:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: and z0.b, z0.b, #0x7
-; CHECK-NEXT: and z1.b, z1.b, #0x7
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: and_v32i8:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: and z0.b, z0.b, #0x7
+; CHECK-SVE-NEXT: and z1.b, z1.b, #0x7
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
;
+; CHECK-SME-LABEL: and_v32i8:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: and z0.b, z0.b, #0x7
+; CHECK-SME-NEXT: and z1.b, z1.b, #0x7
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
+;
+; CHECK-NONEON-NOSVE-LABEL: and_v32i8:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #31]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #63]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #30]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #62]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #29]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #61]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #28]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #60]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #27]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #59]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #26]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #58]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #25]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #57]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #56]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #23]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #55]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #22]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #54]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #21]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #53]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #20]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #52]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #19]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #51]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #18]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #50]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #17]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #49]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #15]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #47]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #14]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #46]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #13]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #45]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #12]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #44]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #11]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #43]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #10]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #42]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #9]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #41]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #40]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #7]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #39]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #6]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #38]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #5]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #37]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #4]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #36]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #3]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #35]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #2]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #34]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #1]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #33]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: and_v32i8:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -410,14 +765,79 @@ define void @and_v32i8(ptr %a) {
}
define void @and_v16i16(ptr %a) {
-; CHECK-LABEL: and_v16i16:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: and z0.h, z0.h, #0xf
-; CHECK-NEXT: and z1.h, z1.h, #0xf
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: and_v16i16:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: and z0.h, z0.h, #0xf
+; CHECK-SVE-NEXT: and z1.h, z1.h, #0xf
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: and_v16i16:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: and z0.h, z0.h, #0xf
+; CHECK-SME-NEXT: and z1.h, z1.h, #0xf
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
;
+; CHECK-NONEON-NOSVE-LABEL: and_v16i16:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #30]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #62]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #28]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #60]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #26]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #58]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #56]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #22]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #54]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #20]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #52]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #18]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #50]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #14]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #46]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #12]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #44]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #10]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #42]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #40]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #6]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #38]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #4]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #36]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #2]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #34]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: and_v16i16:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -482,14 +902,51 @@ define void @and_v16i16(ptr %a) {
}
define void @and_v8i32(ptr %a) {
-; CHECK-LABEL: and_v8i32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: and z0.s, z0.s, #0x1f
-; CHECK-NEXT: and z1.s, z1.s, #0x1f
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: and_v8i32:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: and z0.s, z0.s, #0x1f
+; CHECK-SVE-NEXT: and z1.s, z1.s, #0x1f
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: and_v8i32:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: and z0.s, z0.s, #0x1f
+; CHECK-SME-NEXT: and z1.s, z1.s, #0x1f
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
;
+; CHECK-NONEON-NOSVE-LABEL: and_v8i32:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #28]
+; CHECK-NONEON-NOSVE-NEXT: and w9, w8, #0x1f
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0x1f
+; CHECK-NONEON-NOSVE-NEXT: stp w8, w9, [sp, #56]
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #20]
+; CHECK-NONEON-NOSVE-NEXT: and w9, w8, #0x1f
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0x1f
+; CHECK-NONEON-NOSVE-NEXT: stp w8, w9, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #12]
+; CHECK-NONEON-NOSVE-NEXT: and w9, w8, #0x1f
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0x1f
+; CHECK-NONEON-NOSVE-NEXT: stp w8, w9, [sp, #40]
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #4]
+; CHECK-NONEON-NOSVE-NEXT: and w9, w8, #0x1f
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp]
+; CHECK-NONEON-NOSVE-NEXT: and w8, w8, #0x1f
+; CHECK-NONEON-NOSVE-NEXT: stp w8, w9, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: and_v8i32:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -526,14 +983,41 @@ define void @and_v8i32(ptr %a) {
}
define void @and_v4i64(ptr %a) {
-; CHECK-LABEL: and_v4i64:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: and z0.d, z0.d, #0x3f
-; CHECK-NEXT: and z1.d, z1.d, #0x3f
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: and_v4i64:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: and z0.d, z0.d, #0x3f
+; CHECK-SVE-NEXT: and z1.d, z1.d, #0x3f
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
;
+; CHECK-SME-LABEL: and_v4i64:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: and z0.d, z0.d, #0x3f
+; CHECK-SME-NEXT: and z1.d, z1.d, #0x3f
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
+;
+; CHECK-NONEON-NOSVE-LABEL: and_v4i64:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldr x8, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: and x9, x8, #0x3f
+; CHECK-NONEON-NOSVE-NEXT: ldr x8, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: and x8, x8, #0x3f
+; CHECK-NONEON-NOSVE-NEXT: stp x8, x9, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldr x8, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: and x9, x8, #0x3f
+; CHECK-NONEON-NOSVE-NEXT: ldr x8, [sp]
+; CHECK-NONEON-NOSVE-NEXT: and x8, x8, #0x3f
+; CHECK-NONEON-NOSVE-NEXT: stp x8, x9, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: and_v4i64:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -564,14 +1048,127 @@ define void @and_v4i64(ptr %a) {
;
define void @ashr_v32i8(ptr %a) {
-; CHECK-LABEL: ashr_v32i8:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: asr z0.b, z0.b, #7
-; CHECK-NEXT: asr z1.b, z1.b, #7
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: ashr_v32i8:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: asr z0.b, z0.b, #7
+; CHECK-SVE-NEXT: asr z1.b, z1.b, #7
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: ashr_v32i8:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: asr z0.b, z0.b, #7
+; CHECK-SME-NEXT: asr z1.b, z1.b, #7
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
;
+; CHECK-NONEON-NOSVE-LABEL: ashr_v32i8:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w8, [sp, #31]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #63]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w8, [sp, #30]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #62]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w8, [sp, #29]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #61]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w8, [sp, #28]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #60]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w8, [sp, #27]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #59]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w8, [sp, #26]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #58]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w8, [sp, #25]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #57]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w8, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #56]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w8, [sp, #23]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #55]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w8, [sp, #22]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #54]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w8, [sp, #21]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #53]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w8, [sp, #20]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #52]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w8, [sp, #19]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #51]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w8, [sp, #18]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #50]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w8, [sp, #17]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #49]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w8, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w8, [sp, #15]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #47]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w8, [sp, #14]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #46]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w8, [sp, #13]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #45]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w8, [sp, #12]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #44]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w8, [sp, #11]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #43]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w8, [sp, #10]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #42]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w8, [sp, #9]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #41]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w8, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #40]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w8, [sp, #7]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #39]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w8, [sp, #6]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #38]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w8, [sp, #5]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #37]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w8, [sp, #4]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #36]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w8, [sp, #3]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #35]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w8, [sp, #2]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #34]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w8, [sp, #1]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #33]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w8, [sp]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: ashr_v32i8:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -684,14 +1281,79 @@ define void @ashr_v32i8(ptr %a) {
}
define void @ashr_v16i16(ptr %a) {
-; CHECK-LABEL: ashr_v16i16:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: asr z0.h, z0.h, #15
-; CHECK-NEXT: asr z1.h, z1.h, #15
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: ashr_v16i16:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: asr z0.h, z0.h, #15
+; CHECK-SVE-NEXT: asr z1.h, z1.h, #15
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
;
+; CHECK-SME-LABEL: ashr_v16i16:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: asr z0.h, z0.h, #15
+; CHECK-SME-NEXT: asr z1.h, z1.h, #15
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
+;
+; CHECK-NONEON-NOSVE-LABEL: ashr_v16i16:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w8, [sp, #30]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #62]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w8, [sp, #28]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #60]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w8, [sp, #26]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #58]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w8, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #56]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w8, [sp, #22]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #54]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w8, [sp, #20]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #52]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w8, [sp, #18]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #50]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w8, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w8, [sp, #14]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #46]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w8, [sp, #12]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #44]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w8, [sp, #10]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #42]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w8, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #40]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w8, [sp, #6]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #38]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w8, [sp, #4]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #36]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w8, [sp, #2]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #34]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w8, [sp]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: ashr_v16i16:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -756,14 +1418,51 @@ define void @ashr_v16i16(ptr %a) {
}
define void @ashr_v8i32(ptr %a) {
-; CHECK-LABEL: ashr_v8i32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: asr z0.s, z0.s, #31
-; CHECK-NEXT: asr z1.s, z1.s, #31
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: ashr_v8i32:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: asr z0.s, z0.s, #31
+; CHECK-SVE-NEXT: asr z1.s, z1.s, #31
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: ashr_v8i32:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: asr z0.s, z0.s, #31
+; CHECK-SME-NEXT: asr z1.s, z1.s, #31
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
;
+; CHECK-NONEON-NOSVE-LABEL: ashr_v8i32:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #28]
+; CHECK-NONEON-NOSVE-NEXT: asr w9, w8, #31
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: asr w8, w8, #31
+; CHECK-NONEON-NOSVE-NEXT: stp w8, w9, [sp, #56]
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #20]
+; CHECK-NONEON-NOSVE-NEXT: asr w9, w8, #31
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: asr w8, w8, #31
+; CHECK-NONEON-NOSVE-NEXT: stp w8, w9, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #12]
+; CHECK-NONEON-NOSVE-NEXT: asr w9, w8, #31
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: asr w8, w8, #31
+; CHECK-NONEON-NOSVE-NEXT: stp w8, w9, [sp, #40]
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #4]
+; CHECK-NONEON-NOSVE-NEXT: asr w9, w8, #31
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp]
+; CHECK-NONEON-NOSVE-NEXT: asr w8, w8, #31
+; CHECK-NONEON-NOSVE-NEXT: stp w8, w9, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: ashr_v8i32:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -800,14 +1499,41 @@ define void @ashr_v8i32(ptr %a) {
}
define void @ashr_v4i64(ptr %a) {
-; CHECK-LABEL: ashr_v4i64:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: asr z0.d, z0.d, #63
-; CHECK-NEXT: asr z1.d, z1.d, #63
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: ashr_v4i64:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: asr z0.d, z0.d, #63
+; CHECK-SVE-NEXT: asr z1.d, z1.d, #63
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
;
+; CHECK-SME-LABEL: ashr_v4i64:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: asr z0.d, z0.d, #63
+; CHECK-SME-NEXT: asr z1.d, z1.d, #63
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
+;
+; CHECK-NONEON-NOSVE-LABEL: ashr_v4i64:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldr x8, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: asr x9, x8, #63
+; CHECK-NONEON-NOSVE-NEXT: ldr x8, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: asr x8, x8, #63
+; CHECK-NONEON-NOSVE-NEXT: stp x8, x9, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldr x8, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: asr x9, x8, #63
+; CHECK-NONEON-NOSVE-NEXT: ldr x8, [sp]
+; CHECK-NONEON-NOSVE-NEXT: asr x8, x8, #63
+; CHECK-NONEON-NOSVE-NEXT: stp x8, x9, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: ashr_v4i64:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -838,17 +1564,165 @@ define void @ashr_v4i64(ptr %a) {
;
define void @icmp_eq_v32i8(ptr %a) {
-; CHECK-LABEL: icmp_eq_v32i8:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: ptrue p0.b, vl16
-; CHECK-NEXT: cmpeq p1.b, p0/z, z0.b, #7
-; CHECK-NEXT: cmpeq p2.b, p0/z, z1.b, #7
-; CHECK-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: mov z1.b, p2/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: icmp_eq_v32i8:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: ptrue p0.b, vl16
+; CHECK-SVE-NEXT: cmpeq p1.b, p0/z, z0.b, #7
+; CHECK-SVE-NEXT: cmpeq p2.b, p0/z, z1.b, #7
+; CHECK-SVE-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-SVE-NEXT: mov z1.b, p2/z, #-1 // =0xffffffffffffffff
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: icmp_eq_v32i8:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: ptrue p0.b, vl16
+; CHECK-SME-NEXT: cmpeq p1.b, p0/z, z0.b, #7
+; CHECK-SME-NEXT: cmpeq p2.b, p0/z, z1.b, #7
+; CHECK-SME-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-SME-NEXT: mov z1.b, p2/z, #-1 // =0xffffffffffffffff
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
;
+; CHECK-NONEON-NOSVE-LABEL: icmp_eq_v32i8:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #31]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #7
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, eq
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #63]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #30]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #7
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, eq
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #62]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #29]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #7
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, eq
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #61]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #28]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #7
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, eq
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #60]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #27]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #7
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, eq
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #59]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #26]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #7
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, eq
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #58]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #25]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #7
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, eq
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #57]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #7
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, eq
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #56]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #23]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #7
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, eq
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #55]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #22]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #7
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, eq
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #54]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #21]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #7
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, eq
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #53]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #20]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #7
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, eq
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #52]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #19]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #7
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, eq
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #51]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #18]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #7
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, eq
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #50]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #17]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #7
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, eq
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #49]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #7
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, eq
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #15]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #7
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, eq
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #47]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #14]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #7
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, eq
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #46]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #13]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #7
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, eq
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #45]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #12]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #7
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, eq
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #44]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #11]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #7
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, eq
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #43]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #10]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #7
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, eq
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #42]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #9]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #7
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, eq
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #41]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #7
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, eq
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #40]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #7]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #7
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, eq
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #39]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #6]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #7
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, eq
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #38]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #5]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #7
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, eq
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #37]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #4]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #7
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, eq
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #36]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #3]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #7
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, eq
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #35]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #2]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #7
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, eq
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #34]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #1]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #7
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, eq
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #33]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #7
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, eq
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: icmp_eq_v32i8:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -994,17 +1868,101 @@ define void @icmp_eq_v32i8(ptr %a) {
}
define void @icmp_sge_v16i16(ptr %a) {
-; CHECK-LABEL: icmp_sge_v16i16:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: ptrue p0.h, vl8
-; CHECK-NEXT: cmpge p1.h, p0/z, z0.h, #15
-; CHECK-NEXT: cmpge p2.h, p0/z, z1.h, #15
-; CHECK-NEXT: mov z0.h, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: mov z1.h, p2/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: icmp_sge_v16i16:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: ptrue p0.h, vl8
+; CHECK-SVE-NEXT: cmpge p1.h, p0/z, z0.h, #15
+; CHECK-SVE-NEXT: cmpge p2.h, p0/z, z1.h, #15
+; CHECK-SVE-NEXT: mov z0.h, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-SVE-NEXT: mov z1.h, p2/z, #-1 // =0xffffffffffffffff
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: icmp_sge_v16i16:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: ptrue p0.h, vl8
+; CHECK-SME-NEXT: cmpge p1.h, p0/z, z0.h, #15
+; CHECK-SME-NEXT: cmpge p2.h, p0/z, z1.h, #15
+; CHECK-SME-NEXT: mov z0.h, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-SME-NEXT: mov z1.h, p2/z, #-1 // =0xffffffffffffffff
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
;
+; CHECK-NONEON-NOSVE-LABEL: icmp_sge_v16i16:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w8, [sp, #30]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #14
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #62]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w8, [sp, #28]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #14
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #60]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w8, [sp, #26]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #14
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #58]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w8, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #14
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #56]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w8, [sp, #22]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #14
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #54]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w8, [sp, #20]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #14
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #52]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w8, [sp, #18]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #14
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #50]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w8, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #14
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w8, [sp, #14]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #14
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #46]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w8, [sp, #12]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #14
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #44]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w8, [sp, #10]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #14
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #42]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w8, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #14
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #40]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w8, [sp, #6]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #14
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #38]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w8, [sp, #4]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #14
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #36]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w8, [sp, #2]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #14
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #34]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w8, [sp]
+; CHECK-NONEON-NOSVE-NEXT: cmp w8, #14
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: icmp_sge_v16i16:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -1086,17 +2044,66 @@ define void @icmp_sge_v16i16(ptr %a) {
}
define void @icmp_sgt_v8i32(ptr %a) {
-; CHECK-LABEL: icmp_sgt_v8i32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: ptrue p0.s, vl4
-; CHECK-NEXT: cmpgt p1.s, p0/z, z0.s, #-8
-; CHECK-NEXT: cmpgt p2.s, p0/z, z1.s, #-8
-; CHECK-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: mov z1.s, p2/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: icmp_sgt_v8i32:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: ptrue p0.s, vl4
+; CHECK-SVE-NEXT: cmpgt p1.s, p0/z, z0.s, #-8
+; CHECK-SVE-NEXT: cmpgt p2.s, p0/z, z1.s, #-8
+; CHECK-SVE-NEXT: mov z0.s, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-SVE-NEXT: mov z1.s, p2/z, #-1 // =0xffffffffffffffff
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
;
+; CHECK-SME-LABEL: icmp_sgt_v8i32:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: mov z0.s, #-8 // =0xfffffffffffffff8
+; CHECK-SME-NEXT: ldp q1, q2, [x0]
+; CHECK-SME-NEXT: ptrue p0.s, vl4
+; CHECK-SME-NEXT: shsubr z1.s, p0/m, z1.s, z0.s
+; CHECK-SME-NEXT: shsub z0.s, p0/m, z0.s, z2.s
+; CHECK-SME-NEXT: asr z1.s, z1.s, #31
+; CHECK-SME-NEXT: asr z0.s, z0.s, #31
+; CHECK-SME-NEXT: stp q1, q0, [x0]
+; CHECK-SME-NEXT: ret
+;
+; CHECK-NONEON-NOSVE-LABEL: icmp_sgt_v8i32:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #28]
+; CHECK-NONEON-NOSVE-NEXT: cmn w8, #8
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: csetm w9, gt
+; CHECK-NONEON-NOSVE-NEXT: cmn w8, #8
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, gt
+; CHECK-NONEON-NOSVE-NEXT: stp w8, w9, [sp, #56]
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #20]
+; CHECK-NONEON-NOSVE-NEXT: cmn w8, #8
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: csetm w9, gt
+; CHECK-NONEON-NOSVE-NEXT: cmn w8, #8
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, gt
+; CHECK-NONEON-NOSVE-NEXT: stp w8, w9, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #12]
+; CHECK-NONEON-NOSVE-NEXT: cmn w8, #8
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: csetm w9, gt
+; CHECK-NONEON-NOSVE-NEXT: cmn w8, #8
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, gt
+; CHECK-NONEON-NOSVE-NEXT: stp w8, w9, [sp, #40]
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #4]
+; CHECK-NONEON-NOSVE-NEXT: cmn w8, #8
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp]
+; CHECK-NONEON-NOSVE-NEXT: csetm w9, gt
+; CHECK-NONEON-NOSVE-NEXT: cmn w8, #8
+; CHECK-NONEON-NOSVE-NEXT: csetm w8, gt
+; CHECK-NONEON-NOSVE-NEXT: stp w8, w9, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: icmp_sgt_v8i32:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -1142,17 +2149,52 @@ define void @icmp_sgt_v8i32(ptr %a) {
}
define void @icmp_ult_v4i64(ptr %a) {
-; CHECK-LABEL: icmp_ult_v4i64:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: ptrue p0.d, vl2
-; CHECK-NEXT: cmplo p1.d, p0/z, z0.d, #63
-; CHECK-NEXT: cmplo p2.d, p0/z, z1.d, #63
-; CHECK-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: mov z1.d, p2/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: icmp_ult_v4i64:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: ptrue p0.d, vl2
+; CHECK-SVE-NEXT: cmplo p1.d, p0/z, z0.d, #63
+; CHECK-SVE-NEXT: cmplo p2.d, p0/z, z1.d, #63
+; CHECK-SVE-NEXT: mov z0.d, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-SVE-NEXT: mov z1.d, p2/z, #-1 // =0xffffffffffffffff
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: icmp_ult_v4i64:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: mov z0.d, #63 // =0x3f
+; CHECK-SME-NEXT: ldp q1, q2, [x0]
+; CHECK-SME-NEXT: ptrue p0.d, vl2
+; CHECK-SME-NEXT: uhsub z1.d, p0/m, z1.d, z0.d
+; CHECK-SME-NEXT: uhsubr z0.d, p0/m, z0.d, z2.d
+; CHECK-SME-NEXT: asr z1.d, z1.d, #63
+; CHECK-SME-NEXT: asr z0.d, z0.d, #63
+; CHECK-SME-NEXT: stp q1, q0, [x0]
+; CHECK-SME-NEXT: ret
;
+; CHECK-NONEON-NOSVE-LABEL: icmp_ult_v4i64:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldr x8, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: cmp x8, #63
+; CHECK-NONEON-NOSVE-NEXT: ldr x8, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: csetm x9, lo
+; CHECK-NONEON-NOSVE-NEXT: cmp x8, #63
+; CHECK-NONEON-NOSVE-NEXT: csetm x8, lo
+; CHECK-NONEON-NOSVE-NEXT: stp x8, x9, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldr x8, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: cmp x8, #63
+; CHECK-NONEON-NOSVE-NEXT: ldr x8, [sp]
+; CHECK-NONEON-NOSVE-NEXT: csetm x9, lo
+; CHECK-NONEON-NOSVE-NEXT: cmp x8, #63
+; CHECK-NONEON-NOSVE-NEXT: csetm x8, lo
+; CHECK-NONEON-NOSVE-NEXT: stp x8, x9, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: icmp_ult_v4i64:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -1188,14 +2230,127 @@ define void @icmp_ult_v4i64(ptr %a) {
;
define void @lshr_v32i8(ptr %a) {
-; CHECK-LABEL: lshr_v32i8:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: lsr z0.b, z0.b, #7
-; CHECK-NEXT: lsr z1.b, z1.b, #7
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: lshr_v32i8:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: lsr z0.b, z0.b, #7
+; CHECK-SVE-NEXT: lsr z1.b, z1.b, #7
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
;
+; CHECK-SME-LABEL: lshr_v32i8:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: lsr z0.b, z0.b, #7
+; CHECK-SME-NEXT: lsr z1.b, z1.b, #7
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
+;
+; CHECK-NONEON-NOSVE-LABEL: lshr_v32i8:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #31]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #7, #1
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #63]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #30]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #7, #1
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #62]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #29]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #7, #1
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #61]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #28]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #7, #1
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #60]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #27]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #7, #1
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #59]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #26]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #7, #1
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #58]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #25]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #7, #1
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #57]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #7, #1
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #56]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #23]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #7, #1
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #55]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #22]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #7, #1
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #54]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #21]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #7, #1
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #53]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #20]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #7, #1
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #52]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #19]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #7, #1
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #51]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #18]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #7, #1
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #50]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #17]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #7, #1
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #49]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #7, #1
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #15]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #7, #1
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #47]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #14]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #7, #1
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #46]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #13]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #7, #1
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #45]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #12]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #7, #1
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #44]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #11]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #7, #1
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #43]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #10]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #7, #1
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #42]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #9]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #7, #1
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #41]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #7, #1
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #40]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #7]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #7, #1
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #39]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #6]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #7, #1
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #38]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #5]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #7, #1
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #37]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #4]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #7, #1
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #36]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #3]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #7, #1
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #35]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #2]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #7, #1
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #34]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #1]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #7, #1
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #33]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #7, #1
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: lshr_v32i8:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -1308,14 +2463,79 @@ define void @lshr_v32i8(ptr %a) {
}
define void @lshr_v16i16(ptr %a) {
-; CHECK-LABEL: lshr_v16i16:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: lsr z0.h, z0.h, #15
-; CHECK-NEXT: lsr z1.h, z1.h, #15
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: lshr_v16i16:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: lsr z0.h, z0.h, #15
+; CHECK-SVE-NEXT: lsr z1.h, z1.h, #15
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: lshr_v16i16:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: lsr z0.h, z0.h, #15
+; CHECK-SME-NEXT: lsr z1.h, z1.h, #15
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
;
+; CHECK-NONEON-NOSVE-LABEL: lshr_v16i16:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #30]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #15, #1
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #62]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #28]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #15, #1
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #60]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #26]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #15, #1
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #58]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #15, #1
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #56]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #22]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #15, #1
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #54]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #20]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #15, #1
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #52]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #18]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #15, #1
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #50]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #15, #1
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #14]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #15, #1
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #46]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #12]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #15, #1
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #44]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #10]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #15, #1
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #42]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #15, #1
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #40]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #6]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #15, #1
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #38]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #4]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #15, #1
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #36]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #2]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #15, #1
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #34]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp]
+; CHECK-NONEON-NOSVE-NEXT: ubfx w8, w8, #15, #1
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: lshr_v16i16:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -1380,14 +2600,51 @@ define void @lshr_v16i16(ptr %a) {
}
define void @lshr_v8i32(ptr %a) {
-; CHECK-LABEL: lshr_v8i32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: lsr z0.s, z0.s, #31
-; CHECK-NEXT: lsr z1.s, z1.s, #31
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: lshr_v8i32:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: lsr z0.s, z0.s, #31
+; CHECK-SVE-NEXT: lsr z1.s, z1.s, #31
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
;
+; CHECK-SME-LABEL: lshr_v8i32:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: lsr z0.s, z0.s, #31
+; CHECK-SME-NEXT: lsr z1.s, z1.s, #31
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
+;
+; CHECK-NONEON-NOSVE-LABEL: lshr_v8i32:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #28]
+; CHECK-NONEON-NOSVE-NEXT: lsr w9, w8, #31
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #31
+; CHECK-NONEON-NOSVE-NEXT: stp w8, w9, [sp, #56]
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #20]
+; CHECK-NONEON-NOSVE-NEXT: lsr w9, w8, #31
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #31
+; CHECK-NONEON-NOSVE-NEXT: stp w8, w9, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #12]
+; CHECK-NONEON-NOSVE-NEXT: lsr w9, w8, #31
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #31
+; CHECK-NONEON-NOSVE-NEXT: stp w8, w9, [sp, #40]
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #4]
+; CHECK-NONEON-NOSVE-NEXT: lsr w9, w8, #31
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp]
+; CHECK-NONEON-NOSVE-NEXT: lsr w8, w8, #31
+; CHECK-NONEON-NOSVE-NEXT: stp w8, w9, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: lshr_v8i32:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -1424,14 +2681,41 @@ define void @lshr_v8i32(ptr %a) {
}
define void @lshr_v4i64(ptr %a) {
-; CHECK-LABEL: lshr_v4i64:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: lsr z0.d, z0.d, #63
-; CHECK-NEXT: lsr z1.d, z1.d, #63
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: lshr_v4i64:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: lsr z0.d, z0.d, #63
+; CHECK-SVE-NEXT: lsr z1.d, z1.d, #63
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: lshr_v4i64:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: lsr z0.d, z0.d, #63
+; CHECK-SME-NEXT: lsr z1.d, z1.d, #63
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
;
+; CHECK-NONEON-NOSVE-LABEL: lshr_v4i64:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldr x8, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: lsr x9, x8, #63
+; CHECK-NONEON-NOSVE-NEXT: ldr x8, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: lsr x8, x8, #63
+; CHECK-NONEON-NOSVE-NEXT: stp x8, x9, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldr x8, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: lsr x9, x8, #63
+; CHECK-NONEON-NOSVE-NEXT: ldr x8, [sp]
+; CHECK-NONEON-NOSVE-NEXT: lsr x8, x8, #63
+; CHECK-NONEON-NOSVE-NEXT: stp x8, x9, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: lshr_v4i64:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -1462,14 +2746,159 @@ define void @lshr_v4i64(ptr %a) {
;
define void @mul_v32i8(ptr %a) {
-; CHECK-LABEL: mul_v32i8:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: mul z0.b, z0.b, #7
-; CHECK-NEXT: mul z1.b, z1.b, #7
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: mul_v32i8:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: mul z0.b, z0.b, #7
+; CHECK-SVE-NEXT: mul z1.b, z1.b, #7
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: mul_v32i8:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: mul z0.b, z0.b, #7
+; CHECK-SME-NEXT: mul z1.b, z1.b, #7
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
;
+; CHECK-NONEON-NOSVE-LABEL: mul_v32i8:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #31]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #3
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #63]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #30]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #3
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #62]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #29]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #3
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #61]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #28]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #3
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #60]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #27]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #3
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #59]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #26]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #3
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #58]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #25]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #3
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #57]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #3
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #56]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #23]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #3
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #55]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #22]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #3
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #54]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #21]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #3
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #53]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #20]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #3
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #52]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #19]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #3
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #51]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #18]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #3
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #50]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #17]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #3
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #49]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #3
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #15]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #3
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #47]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #14]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #3
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #46]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #13]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #3
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #45]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #12]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #3
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #44]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #11]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #3
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #43]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #10]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #3
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #42]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #9]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #3
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #41]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #3
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #40]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #7]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #3
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #39]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #6]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #3
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #38]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #5]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #3
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #37]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #4]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #3
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #36]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #3]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #3
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #35]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #2]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #3
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #34]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #1]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #3
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #33]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #3
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: mul_v32i8:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -1614,14 +3043,95 @@ define void @mul_v32i8(ptr %a) {
}
define void @mul_v16i16(ptr %a) {
-; CHECK-LABEL: mul_v16i16:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: mul z0.h, z0.h, #15
-; CHECK-NEXT: mul z1.h, z1.h, #15
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: mul_v16i16:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: mul z0.h, z0.h, #15
+; CHECK-SVE-NEXT: mul z1.h, z1.h, #15
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
;
+; CHECK-SME-LABEL: mul_v16i16:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: mul z0.h, z0.h, #15
+; CHECK-SME-NEXT: mul z1.h, z1.h, #15
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
+;
+; CHECK-NONEON-NOSVE-LABEL: mul_v16i16:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #30]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #4
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #62]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #28]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #4
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #60]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #26]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #4
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #58]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #4
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #56]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #22]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #4
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #54]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #20]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #4
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #52]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #18]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #4
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #50]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #4
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #14]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #4
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #46]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #12]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #4
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #44]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #10]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #4
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #42]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #4
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #40]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #6]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #4
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #38]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #4]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #4
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #36]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #2]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #4
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #34]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #4
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: mul_v16i16:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -1702,14 +3212,63 @@ define void @mul_v16i16(ptr %a) {
}
define void @mul_v8i32(ptr %a) {
-; CHECK-LABEL: mul_v8i32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: mul z0.s, z0.s, #31
-; CHECK-NEXT: mul z1.s, z1.s, #31
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: mul_v8i32:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: mul z0.s, z0.s, #31
+; CHECK-SVE-NEXT: mul z1.s, z1.s, #31
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: mul_v8i32:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: mul z0.s, z0.s, #31
+; CHECK-SME-NEXT: mul z1.s, z1.s, #31
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
;
+; CHECK-NONEON-NOSVE-LABEL: mul_v8i32:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #28]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #5
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: str w8, [sp, #60]
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #5
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: str w8, [sp, #56]
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #20]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #5
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: str w8, [sp, #52]
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #5
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: str w8, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #12]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #5
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: str w8, [sp, #44]
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #5
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: str w8, [sp, #40]
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #4]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #5
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: str w8, [sp, #36]
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #5
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w9, w8
+; CHECK-NONEON-NOSVE-NEXT: str w8, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: mul_v8i32:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -1758,14 +3317,47 @@ define void @mul_v8i32(ptr %a) {
}
define void @mul_v4i64(ptr %a) {
-; CHECK-LABEL: mul_v4i64:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: mul z0.d, z0.d, #63
-; CHECK-NEXT: mul z1.d, z1.d, #63
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: mul_v4i64:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: mul z0.d, z0.d, #63
+; CHECK-SVE-NEXT: mul z1.d, z1.d, #63
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
;
+; CHECK-SME-LABEL: mul_v4i64:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: mul z0.d, z0.d, #63
+; CHECK-SME-NEXT: mul z1.d, z1.d, #63
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
+;
+; CHECK-NONEON-NOSVE-LABEL: mul_v4i64:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldr x8, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: lsl x9, x8, #6
+; CHECK-NONEON-NOSVE-NEXT: sub x8, x9, x8
+; CHECK-NONEON-NOSVE-NEXT: str x8, [sp, #56]
+; CHECK-NONEON-NOSVE-NEXT: ldr x8, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: lsl x9, x8, #6
+; CHECK-NONEON-NOSVE-NEXT: sub x8, x9, x8
+; CHECK-NONEON-NOSVE-NEXT: str x8, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldr x8, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: lsl x9, x8, #6
+; CHECK-NONEON-NOSVE-NEXT: sub x8, x9, x8
+; CHECK-NONEON-NOSVE-NEXT: str x8, [sp, #40]
+; CHECK-NONEON-NOSVE-NEXT: ldr x8, [sp]
+; CHECK-NONEON-NOSVE-NEXT: lsl x9, x8, #6
+; CHECK-NONEON-NOSVE-NEXT: sub x8, x9, x8
+; CHECK-NONEON-NOSVE-NEXT: str x8, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: mul_v4i64:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -1802,14 +3394,127 @@ define void @mul_v4i64(ptr %a) {
;
define void @or_v32i8(ptr %a) {
-; CHECK-LABEL: or_v32i8:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: orr z0.b, z0.b, #0x7
-; CHECK-NEXT: orr z1.b, z1.b, #0x7
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: or_v32i8:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: orr z0.b, z0.b, #0x7
+; CHECK-SVE-NEXT: orr z1.b, z1.b, #0x7
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: or_v32i8:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: orr z0.b, z0.b, #0x7
+; CHECK-SME-NEXT: orr z1.b, z1.b, #0x7
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
;
+; CHECK-NONEON-NOSVE-LABEL: or_v32i8:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #31]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #63]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #30]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #62]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #29]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #61]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #28]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #60]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #27]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #59]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #26]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #58]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #25]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #57]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #56]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #23]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #55]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #22]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #54]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #21]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #53]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #20]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #52]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #19]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #51]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #18]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #50]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #17]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #49]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #15]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #47]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #14]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #46]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #13]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #45]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #12]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #44]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #11]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #43]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #10]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #42]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #9]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #41]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #40]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #7]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #39]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #6]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #38]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #5]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #37]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #4]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #36]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #3]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #35]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #2]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #34]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #1]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #33]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: or_v32i8:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -1922,14 +3627,79 @@ define void @or_v32i8(ptr %a) {
}
define void @or_v16i16(ptr %a) {
-; CHECK-LABEL: or_v16i16:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: orr z0.h, z0.h, #0xf
-; CHECK-NEXT: orr z1.h, z1.h, #0xf
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: or_v16i16:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: orr z0.h, z0.h, #0xf
+; CHECK-SVE-NEXT: orr z1.h, z1.h, #0xf
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
;
+; CHECK-SME-LABEL: or_v16i16:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: orr z0.h, z0.h, #0xf
+; CHECK-SME-NEXT: orr z1.h, z1.h, #0xf
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
+;
+; CHECK-NONEON-NOSVE-LABEL: or_v16i16:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #30]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #62]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #28]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #60]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #26]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #58]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #56]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #22]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #54]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #20]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #52]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #18]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #50]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #14]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #46]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #12]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #44]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #10]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #42]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #40]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #6]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #38]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #4]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #36]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #2]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #34]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: or_v16i16:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -1994,14 +3764,51 @@ define void @or_v16i16(ptr %a) {
}
define void @or_v8i32(ptr %a) {
-; CHECK-LABEL: or_v8i32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: orr z0.s, z0.s, #0x1f
-; CHECK-NEXT: orr z1.s, z1.s, #0x1f
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: or_v8i32:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: orr z0.s, z0.s, #0x1f
+; CHECK-SVE-NEXT: orr z1.s, z1.s, #0x1f
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: or_v8i32:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: orr z0.s, z0.s, #0x1f
+; CHECK-SME-NEXT: orr z1.s, z1.s, #0x1f
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
;
+; CHECK-NONEON-NOSVE-LABEL: or_v8i32:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #28]
+; CHECK-NONEON-NOSVE-NEXT: orr w9, w8, #0x1f
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0x1f
+; CHECK-NONEON-NOSVE-NEXT: stp w8, w9, [sp, #56]
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #20]
+; CHECK-NONEON-NOSVE-NEXT: orr w9, w8, #0x1f
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0x1f
+; CHECK-NONEON-NOSVE-NEXT: stp w8, w9, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #12]
+; CHECK-NONEON-NOSVE-NEXT: orr w9, w8, #0x1f
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0x1f
+; CHECK-NONEON-NOSVE-NEXT: stp w8, w9, [sp, #40]
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #4]
+; CHECK-NONEON-NOSVE-NEXT: orr w9, w8, #0x1f
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp]
+; CHECK-NONEON-NOSVE-NEXT: orr w8, w8, #0x1f
+; CHECK-NONEON-NOSVE-NEXT: stp w8, w9, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: or_v8i32:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -2038,14 +3845,41 @@ define void @or_v8i32(ptr %a) {
}
define void @or_v4i64(ptr %a) {
-; CHECK-LABEL: or_v4i64:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: orr z0.d, z0.d, #0x3f
-; CHECK-NEXT: orr z1.d, z1.d, #0x3f
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: or_v4i64:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: orr z0.d, z0.d, #0x3f
+; CHECK-SVE-NEXT: orr z1.d, z1.d, #0x3f
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: or_v4i64:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: orr z0.d, z0.d, #0x3f
+; CHECK-SME-NEXT: orr z1.d, z1.d, #0x3f
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
;
+; CHECK-NONEON-NOSVE-LABEL: or_v4i64:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldr x8, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: orr x9, x8, #0x3f
+; CHECK-NONEON-NOSVE-NEXT: ldr x8, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: orr x8, x8, #0x3f
+; CHECK-NONEON-NOSVE-NEXT: stp x8, x9, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldr x8, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: orr x9, x8, #0x3f
+; CHECK-NONEON-NOSVE-NEXT: ldr x8, [sp]
+; CHECK-NONEON-NOSVE-NEXT: orr x8, x8, #0x3f
+; CHECK-NONEON-NOSVE-NEXT: stp x8, x9, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: or_v4i64:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -2076,14 +3910,127 @@ define void @or_v4i64(ptr %a) {
;
define void @shl_v32i8(ptr %a) {
-; CHECK-LABEL: shl_v32i8:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: lsl z0.b, z0.b, #7
-; CHECK-NEXT: lsl z1.b, z1.b, #7
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: shl_v32i8:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: lsl z0.b, z0.b, #7
+; CHECK-SVE-NEXT: lsl z1.b, z1.b, #7
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
;
+; CHECK-SME-LABEL: shl_v32i8:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: lsl z0.b, z0.b, #7
+; CHECK-SME-NEXT: lsl z1.b, z1.b, #7
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
+;
+; CHECK-NONEON-NOSVE-LABEL: shl_v32i8:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #31]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #63]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #30]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #62]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #29]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #61]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #28]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #60]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #27]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #59]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #26]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #58]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #25]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #57]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #56]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #23]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #55]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #22]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #54]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #21]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #53]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #20]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #52]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #19]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #51]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #18]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #50]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #17]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #49]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #15]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #47]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #14]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #46]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #13]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #45]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #12]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #44]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #11]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #43]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #10]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #42]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #9]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #41]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #40]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #7]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #39]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #6]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #38]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #5]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #37]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #4]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #36]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #3]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #35]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #2]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #34]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #1]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #33]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: shl_v32i8:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -2196,14 +4143,79 @@ define void @shl_v32i8(ptr %a) {
}
define void @shl_v16i16(ptr %a) {
-; CHECK-LABEL: shl_v16i16:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: lsl z0.h, z0.h, #15
-; CHECK-NEXT: lsl z1.h, z1.h, #15
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: shl_v16i16:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: lsl z0.h, z0.h, #15
+; CHECK-SVE-NEXT: lsl z1.h, z1.h, #15
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: shl_v16i16:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: lsl z0.h, z0.h, #15
+; CHECK-SME-NEXT: lsl z1.h, z1.h, #15
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
;
+; CHECK-NONEON-NOSVE-LABEL: shl_v16i16:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #30]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #62]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #28]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #60]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #26]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #58]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #56]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #22]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #54]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #20]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #52]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #18]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #50]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #14]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #46]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #12]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #44]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #10]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #42]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #40]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #6]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #38]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #4]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #36]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #2]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #34]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: shl_v16i16:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -2268,14 +4280,51 @@ define void @shl_v16i16(ptr %a) {
}
define void @shl_v8i32(ptr %a) {
-; CHECK-LABEL: shl_v8i32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: lsl z0.s, z0.s, #31
-; CHECK-NEXT: lsl z1.s, z1.s, #31
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: shl_v8i32:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: lsl z0.s, z0.s, #31
+; CHECK-SVE-NEXT: lsl z1.s, z1.s, #31
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
;
+; CHECK-SME-LABEL: shl_v8i32:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: lsl z0.s, z0.s, #31
+; CHECK-SME-NEXT: lsl z1.s, z1.s, #31
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
+;
+; CHECK-NONEON-NOSVE-LABEL: shl_v8i32:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #28]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #31
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #31
+; CHECK-NONEON-NOSVE-NEXT: stp w8, w9, [sp, #56]
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #20]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #31
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #31
+; CHECK-NONEON-NOSVE-NEXT: stp w8, w9, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #12]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #31
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #31
+; CHECK-NONEON-NOSVE-NEXT: stp w8, w9, [sp, #40]
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #4]
+; CHECK-NONEON-NOSVE-NEXT: lsl w9, w8, #31
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp]
+; CHECK-NONEON-NOSVE-NEXT: lsl w8, w8, #31
+; CHECK-NONEON-NOSVE-NEXT: stp w8, w9, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: shl_v8i32:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -2312,14 +4361,41 @@ define void @shl_v8i32(ptr %a) {
}
define void @shl_v4i64(ptr %a) {
-; CHECK-LABEL: shl_v4i64:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: lsl z0.d, z0.d, #63
-; CHECK-NEXT: lsl z1.d, z1.d, #63
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: shl_v4i64:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: lsl z0.d, z0.d, #63
+; CHECK-SVE-NEXT: lsl z1.d, z1.d, #63
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: shl_v4i64:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: lsl z0.d, z0.d, #63
+; CHECK-SME-NEXT: lsl z1.d, z1.d, #63
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
;
+; CHECK-NONEON-NOSVE-LABEL: shl_v4i64:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldr x8, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: lsl x9, x8, #63
+; CHECK-NONEON-NOSVE-NEXT: ldr x8, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: lsl x8, x8, #63
+; CHECK-NONEON-NOSVE-NEXT: stp x8, x9, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldr x8, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: lsl x9, x8, #63
+; CHECK-NONEON-NOSVE-NEXT: ldr x8, [sp]
+; CHECK-NONEON-NOSVE-NEXT: lsl x8, x8, #63
+; CHECK-NONEON-NOSVE-NEXT: stp x8, x9, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: shl_v4i64:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -2350,14 +4426,160 @@ define void @shl_v4i64(ptr %a) {
;
define void @smax_v32i8(ptr %a) {
-; CHECK-LABEL: smax_v32i8:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: smax z0.b, z0.b, #7
-; CHECK-NEXT: smax z1.b, z1.b, #7
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: smax_v32i8:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: smax z0.b, z0.b, #7
+; CHECK-SVE-NEXT: smax z1.b, z1.b, #7
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
;
+; CHECK-SME-LABEL: smax_v32i8:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: smax z0.b, z0.b, #7
+; CHECK-SME-NEXT: smax z1.b, z1.b, #7
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
+;
+; CHECK-NONEON-NOSVE-LABEL: smax_v32i8:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: mov w8, #7 // =0x7
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #31]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #63]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #30]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #62]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #29]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #61]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #28]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #60]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #27]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #59]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #26]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #58]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #25]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #57]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #56]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #23]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #55]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #22]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #54]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #21]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #53]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #20]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #52]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #19]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #51]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #18]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #50]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #17]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #49]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #15]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #47]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #14]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #46]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #13]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #45]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #12]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #44]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #11]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #43]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #10]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #42]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #9]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #41]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #40]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #7]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #39]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #6]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #38]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #5]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #37]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #4]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #36]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #3]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #35]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #2]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #34]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #1]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #33]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w8, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: smax_v32i8:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -2503,14 +4725,96 @@ define void @smax_v32i8(ptr %a) {
}
define void @smax_v16i16(ptr %a) {
-; CHECK-LABEL: smax_v16i16:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: smax z0.h, z0.h, #15
-; CHECK-NEXT: smax z1.h, z1.h, #15
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: smax_v16i16:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: smax z0.h, z0.h, #15
+; CHECK-SVE-NEXT: smax z1.h, z1.h, #15
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: smax_v16i16:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: smax z0.h, z0.h, #15
+; CHECK-SME-NEXT: smax z1.h, z1.h, #15
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
;
+; CHECK-NONEON-NOSVE-LABEL: smax_v16i16:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: mov w8, #15 // =0xf
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w9, [sp, #30]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #62]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w9, [sp, #28]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #60]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w9, [sp, #26]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #58]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w9, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #56]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w9, [sp, #22]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #54]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w9, [sp, #20]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #52]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w9, [sp, #18]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #50]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w9, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w9, [sp, #14]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #46]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w9, [sp, #12]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #44]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w9, [sp, #10]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #42]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w9, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #40]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w9, [sp, #6]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #38]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w9, [sp, #4]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #36]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w9, [sp, #2]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #34]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w9, [sp]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w8, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: smax_v16i16:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -2592,14 +4896,60 @@ define void @smax_v16i16(ptr %a) {
}
define void @smax_v8i32(ptr %a) {
-; CHECK-LABEL: smax_v8i32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: smax z0.s, z0.s, #31
-; CHECK-NEXT: smax z1.s, z1.s, #31
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: smax_v8i32:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: smax z0.s, z0.s, #31
+; CHECK-SVE-NEXT: smax z1.s, z1.s, #31
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: smax_v8i32:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: smax z0.s, z0.s, #31
+; CHECK-SME-NEXT: smax z1.s, z1.s, #31
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
;
+; CHECK-NONEON-NOSVE-LABEL: smax_v8i32:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: mov w8, #31 // =0x1f
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldr w9, [sp, #28]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #31
+; CHECK-NONEON-NOSVE-NEXT: csel w10, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: ldr w9, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #31
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: stp w9, w10, [sp, #56]
+; CHECK-NONEON-NOSVE-NEXT: ldr w9, [sp, #20]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #31
+; CHECK-NONEON-NOSVE-NEXT: csel w10, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: ldr w9, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #31
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: stp w9, w10, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldr w9, [sp, #12]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #31
+; CHECK-NONEON-NOSVE-NEXT: csel w10, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: ldr w9, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #31
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: stp w9, w10, [sp, #40]
+; CHECK-NONEON-NOSVE-NEXT: ldr w9, [sp, #4]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #31
+; CHECK-NONEON-NOSVE-NEXT: csel w10, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: ldr w9, [sp]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #31
+; CHECK-NONEON-NOSVE-NEXT: csel w8, w9, w8, gt
+; CHECK-NONEON-NOSVE-NEXT: stp w8, w10, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: smax_v8i32:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -2645,14 +4995,46 @@ define void @smax_v8i32(ptr %a) {
}
define void @smax_v4i64(ptr %a) {
-; CHECK-LABEL: smax_v4i64:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: smax z0.d, z0.d, #63
-; CHECK-NEXT: smax z1.d, z1.d, #63
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: smax_v4i64:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: smax z0.d, z0.d, #63
+; CHECK-SVE-NEXT: smax z1.d, z1.d, #63
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
;
+; CHECK-SME-LABEL: smax_v4i64:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: smax z0.d, z0.d, #63
+; CHECK-SME-NEXT: smax z1.d, z1.d, #63
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
+;
+; CHECK-NONEON-NOSVE-LABEL: smax_v4i64:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: mov w8, #63 // =0x3f
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldr x9, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: cmp x9, #63
+; CHECK-NONEON-NOSVE-NEXT: csel x10, x9, x8, gt
+; CHECK-NONEON-NOSVE-NEXT: ldr x9, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: cmp x9, #63
+; CHECK-NONEON-NOSVE-NEXT: csel x9, x9, x8, gt
+; CHECK-NONEON-NOSVE-NEXT: stp x9, x10, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldr x9, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: cmp x9, #63
+; CHECK-NONEON-NOSVE-NEXT: csel x10, x9, x8, gt
+; CHECK-NONEON-NOSVE-NEXT: ldr x9, [sp]
+; CHECK-NONEON-NOSVE-NEXT: cmp x9, #63
+; CHECK-NONEON-NOSVE-NEXT: csel x8, x9, x8, gt
+; CHECK-NONEON-NOSVE-NEXT: stp x8, x10, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: smax_v4i64:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -2688,14 +5070,160 @@ define void @smax_v4i64(ptr %a) {
;
define void @smin_v32i8(ptr %a) {
-; CHECK-LABEL: smin_v32i8:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: smin z0.b, z0.b, #7
-; CHECK-NEXT: smin z1.b, z1.b, #7
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: smin_v32i8:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: smin z0.b, z0.b, #7
+; CHECK-SVE-NEXT: smin z1.b, z1.b, #7
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: smin_v32i8:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: smin z0.b, z0.b, #7
+; CHECK-SME-NEXT: smin z1.b, z1.b, #7
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
;
+; CHECK-NONEON-NOSVE-LABEL: smin_v32i8:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: mov w8, #7 // =0x7
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #31]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #63]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #30]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #62]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #29]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #61]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #28]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #60]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #27]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #59]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #26]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #58]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #25]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #57]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #56]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #23]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #55]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #22]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #54]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #21]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #53]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #20]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #52]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #19]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #51]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #18]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #50]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #17]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #49]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #15]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #47]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #14]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #46]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #13]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #45]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #12]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #44]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #11]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #43]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #10]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #42]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #9]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #41]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #40]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #7]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #39]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #6]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #38]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #5]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #37]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #4]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #36]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #3]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #35]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #2]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #34]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp, #1]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #33]
+; CHECK-NONEON-NOSVE-NEXT: ldrsb w9, [sp]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w8, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: smin_v32i8:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -2841,14 +5369,96 @@ define void @smin_v32i8(ptr %a) {
}
define void @smin_v16i16(ptr %a) {
-; CHECK-LABEL: smin_v16i16:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: smin z0.h, z0.h, #15
-; CHECK-NEXT: smin z1.h, z1.h, #15
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: smin_v16i16:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: smin z0.h, z0.h, #15
+; CHECK-SVE-NEXT: smin z1.h, z1.h, #15
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
;
+; CHECK-SME-LABEL: smin_v16i16:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: smin z0.h, z0.h, #15
+; CHECK-SME-NEXT: smin z1.h, z1.h, #15
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
+;
+; CHECK-NONEON-NOSVE-LABEL: smin_v16i16:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: mov w8, #15 // =0xf
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w9, [sp, #30]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #62]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w9, [sp, #28]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #60]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w9, [sp, #26]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #58]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w9, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #56]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w9, [sp, #22]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #54]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w9, [sp, #20]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #52]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w9, [sp, #18]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #50]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w9, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w9, [sp, #14]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #46]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w9, [sp, #12]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #44]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w9, [sp, #10]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #42]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w9, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #40]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w9, [sp, #6]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #38]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w9, [sp, #4]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #36]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w9, [sp, #2]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #34]
+; CHECK-NONEON-NOSVE-NEXT: ldrsh w9, [sp]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w8, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: smin_v16i16:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -2930,14 +5540,60 @@ define void @smin_v16i16(ptr %a) {
}
define void @smin_v8i32(ptr %a) {
-; CHECK-LABEL: smin_v8i32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: smin z0.s, z0.s, #31
-; CHECK-NEXT: smin z1.s, z1.s, #31
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: smin_v8i32:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: smin z0.s, z0.s, #31
+; CHECK-SVE-NEXT: smin z1.s, z1.s, #31
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: smin_v8i32:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: smin z0.s, z0.s, #31
+; CHECK-SME-NEXT: smin z1.s, z1.s, #31
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
;
+; CHECK-NONEON-NOSVE-LABEL: smin_v8i32:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: mov w8, #31 // =0x1f
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldr w9, [sp, #28]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #31
+; CHECK-NONEON-NOSVE-NEXT: csel w10, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: ldr w9, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #31
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: stp w9, w10, [sp, #56]
+; CHECK-NONEON-NOSVE-NEXT: ldr w9, [sp, #20]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #31
+; CHECK-NONEON-NOSVE-NEXT: csel w10, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: ldr w9, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #31
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: stp w9, w10, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldr w9, [sp, #12]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #31
+; CHECK-NONEON-NOSVE-NEXT: csel w10, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: ldr w9, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #31
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: stp w9, w10, [sp, #40]
+; CHECK-NONEON-NOSVE-NEXT: ldr w9, [sp, #4]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #31
+; CHECK-NONEON-NOSVE-NEXT: csel w10, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: ldr w9, [sp]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #31
+; CHECK-NONEON-NOSVE-NEXT: csel w8, w9, w8, lt
+; CHECK-NONEON-NOSVE-NEXT: stp w8, w10, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: smin_v8i32:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -2983,14 +5639,46 @@ define void @smin_v8i32(ptr %a) {
}
define void @smin_v4i64(ptr %a) {
-; CHECK-LABEL: smin_v4i64:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: smin z0.d, z0.d, #63
-; CHECK-NEXT: smin z1.d, z1.d, #63
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: smin_v4i64:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: smin z0.d, z0.d, #63
+; CHECK-SVE-NEXT: smin z1.d, z1.d, #63
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
;
+; CHECK-SME-LABEL: smin_v4i64:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: smin z0.d, z0.d, #63
+; CHECK-SME-NEXT: smin z1.d, z1.d, #63
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
+;
+; CHECK-NONEON-NOSVE-LABEL: smin_v4i64:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: mov w8, #63 // =0x3f
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldr x9, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: cmp x9, #63
+; CHECK-NONEON-NOSVE-NEXT: csel x10, x9, x8, lt
+; CHECK-NONEON-NOSVE-NEXT: ldr x9, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: cmp x9, #63
+; CHECK-NONEON-NOSVE-NEXT: csel x9, x9, x8, lt
+; CHECK-NONEON-NOSVE-NEXT: stp x9, x10, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldr x9, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: cmp x9, #63
+; CHECK-NONEON-NOSVE-NEXT: csel x10, x9, x8, lt
+; CHECK-NONEON-NOSVE-NEXT: ldr x9, [sp]
+; CHECK-NONEON-NOSVE-NEXT: cmp x9, #63
+; CHECK-NONEON-NOSVE-NEXT: csel x8, x9, x8, lt
+; CHECK-NONEON-NOSVE-NEXT: stp x8, x10, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: smin_v4i64:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -3026,14 +5714,127 @@ define void @smin_v4i64(ptr %a) {
;
define void @sub_v32i8(ptr %a) {
-; CHECK-LABEL: sub_v32i8:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: sub z0.b, z0.b, #7 // =0x7
-; CHECK-NEXT: sub z1.b, z1.b, #7 // =0x7
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: sub_v32i8:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: sub z0.b, z0.b, #7 // =0x7
+; CHECK-SVE-NEXT: sub z1.b, z1.b, #7 // =0x7
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: sub_v32i8:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: sub z0.b, z0.b, #7 // =0x7
+; CHECK-SME-NEXT: sub z1.b, z1.b, #7 // =0x7
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
;
+; CHECK-NONEON-NOSVE-LABEL: sub_v32i8:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #31]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #63]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #30]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #62]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #29]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #61]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #28]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #60]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #27]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #59]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #26]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #58]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #25]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #57]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #56]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #23]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #55]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #22]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #54]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #21]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #53]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #20]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #52]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #19]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #51]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #18]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #50]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #17]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #49]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #15]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #47]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #14]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #46]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #13]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #45]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #12]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #44]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #11]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #43]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #10]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #42]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #9]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #41]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #40]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #7]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #39]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #6]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #38]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #5]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #37]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #4]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #36]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #3]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #35]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #2]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #34]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #1]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #33]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: sub_v32i8:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -3146,14 +5947,79 @@ define void @sub_v32i8(ptr %a) {
}
define void @sub_v16i16(ptr %a) {
-; CHECK-LABEL: sub_v16i16:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: sub z0.h, z0.h, #15 // =0xf
-; CHECK-NEXT: sub z1.h, z1.h, #15 // =0xf
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: sub_v16i16:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: sub z0.h, z0.h, #15 // =0xf
+; CHECK-SVE-NEXT: sub z1.h, z1.h, #15 // =0xf
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: sub_v16i16:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: sub z0.h, z0.h, #15 // =0xf
+; CHECK-SME-NEXT: sub z1.h, z1.h, #15 // =0xf
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
;
+; CHECK-NONEON-NOSVE-LABEL: sub_v16i16:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #30]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #62]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #28]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #60]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #26]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #58]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #56]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #22]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #54]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #20]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #52]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #18]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #50]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #14]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #46]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #12]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #44]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #10]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #42]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #40]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #6]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #38]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #4]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #36]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #2]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #34]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #15
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: sub_v16i16:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -3218,14 +6084,51 @@ define void @sub_v16i16(ptr %a) {
}
define void @sub_v8i32(ptr %a) {
-; CHECK-LABEL: sub_v8i32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: sub z0.s, z0.s, #31 // =0x1f
-; CHECK-NEXT: sub z1.s, z1.s, #31 // =0x1f
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: sub_v8i32:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: sub z0.s, z0.s, #31 // =0x1f
+; CHECK-SVE-NEXT: sub z1.s, z1.s, #31 // =0x1f
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
;
+; CHECK-SME-LABEL: sub_v8i32:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: sub z0.s, z0.s, #31 // =0x1f
+; CHECK-SME-NEXT: sub z1.s, z1.s, #31 // =0x1f
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
+;
+; CHECK-NONEON-NOSVE-LABEL: sub_v8i32:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #28]
+; CHECK-NONEON-NOSVE-NEXT: sub w9, w8, #31
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #31
+; CHECK-NONEON-NOSVE-NEXT: stp w8, w9, [sp, #56]
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #20]
+; CHECK-NONEON-NOSVE-NEXT: sub w9, w8, #31
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #31
+; CHECK-NONEON-NOSVE-NEXT: stp w8, w9, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #12]
+; CHECK-NONEON-NOSVE-NEXT: sub w9, w8, #31
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #31
+; CHECK-NONEON-NOSVE-NEXT: stp w8, w9, [sp, #40]
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #4]
+; CHECK-NONEON-NOSVE-NEXT: sub w9, w8, #31
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp]
+; CHECK-NONEON-NOSVE-NEXT: sub w8, w8, #31
+; CHECK-NONEON-NOSVE-NEXT: stp w8, w9, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: sub_v8i32:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -3262,14 +6165,41 @@ define void @sub_v8i32(ptr %a) {
}
define void @sub_v4i64(ptr %a) {
-; CHECK-LABEL: sub_v4i64:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: sub z0.d, z0.d, #63 // =0x3f
-; CHECK-NEXT: sub z1.d, z1.d, #63 // =0x3f
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: sub_v4i64:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: sub z0.d, z0.d, #63 // =0x3f
+; CHECK-SVE-NEXT: sub z1.d, z1.d, #63 // =0x3f
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: sub_v4i64:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: sub z0.d, z0.d, #63 // =0x3f
+; CHECK-SME-NEXT: sub z1.d, z1.d, #63 // =0x3f
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
;
+; CHECK-NONEON-NOSVE-LABEL: sub_v4i64:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldr x8, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: sub x9, x8, #63
+; CHECK-NONEON-NOSVE-NEXT: ldr x8, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: sub x8, x8, #63
+; CHECK-NONEON-NOSVE-NEXT: stp x8, x9, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldr x8, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: sub x9, x8, #63
+; CHECK-NONEON-NOSVE-NEXT: ldr x8, [sp]
+; CHECK-NONEON-NOSVE-NEXT: sub x8, x8, #63
+; CHECK-NONEON-NOSVE-NEXT: stp x8, x9, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: sub_v4i64:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -3300,14 +6230,160 @@ define void @sub_v4i64(ptr %a) {
;
define void @umax_v32i8(ptr %a) {
-; CHECK-LABEL: umax_v32i8:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: umax z0.b, z0.b, #7
-; CHECK-NEXT: umax z1.b, z1.b, #7
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: umax_v32i8:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: umax z0.b, z0.b, #7
+; CHECK-SVE-NEXT: umax z1.b, z1.b, #7
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
;
+; CHECK-SME-LABEL: umax_v32i8:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: umax z0.b, z0.b, #7
+; CHECK-SME-NEXT: umax z1.b, z1.b, #7
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
+;
+; CHECK-NONEON-NOSVE-LABEL: umax_v32i8:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: mov w8, #7 // =0x7
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #31]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xf8
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #63]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #30]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xf8
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #62]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #29]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xf8
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #61]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #28]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xf8
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #60]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #27]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xf8
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #59]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #26]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xf8
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #58]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #25]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xf8
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #57]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xf8
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #56]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #23]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xf8
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #55]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #22]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xf8
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #54]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #21]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xf8
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #53]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #20]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xf8
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #52]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #19]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xf8
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #51]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #18]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xf8
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #50]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #17]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xf8
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #49]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xf8
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #15]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xf8
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #47]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #14]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xf8
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #46]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #13]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xf8
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #45]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #12]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xf8
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #44]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #11]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xf8
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #43]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #10]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xf8
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #42]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #9]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xf8
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #41]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xf8
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #40]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #7]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xf8
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #39]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #6]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xf8
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #38]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #5]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xf8
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #37]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #4]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xf8
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #36]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #3]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xf8
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #35]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #2]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xf8
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #34]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #1]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xf8
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #33]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xf8
+; CHECK-NONEON-NOSVE-NEXT: csel w8, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: umax_v32i8:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -3453,14 +6529,96 @@ define void @umax_v32i8(ptr %a) {
}
define void @umax_v16i16(ptr %a) {
-; CHECK-LABEL: umax_v16i16:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: umax z0.h, z0.h, #15
-; CHECK-NEXT: umax z1.h, z1.h, #15
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: umax_v16i16:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: umax z0.h, z0.h, #15
+; CHECK-SVE-NEXT: umax z1.h, z1.h, #15
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: umax_v16i16:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: umax z0.h, z0.h, #15
+; CHECK-SME-NEXT: umax z1.h, z1.h, #15
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
;
+; CHECK-NONEON-NOSVE-LABEL: umax_v16i16:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: mov w8, #15 // =0xf
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldrh w9, [sp, #30]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xfff0
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #62]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w9, [sp, #28]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xfff0
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #60]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w9, [sp, #26]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xfff0
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #58]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w9, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xfff0
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #56]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w9, [sp, #22]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xfff0
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #54]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w9, [sp, #20]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xfff0
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #52]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w9, [sp, #18]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xfff0
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #50]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w9, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xfff0
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w9, [sp, #14]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xfff0
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #46]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w9, [sp, #12]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xfff0
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #44]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w9, [sp, #10]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xfff0
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #42]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w9, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xfff0
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #40]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w9, [sp, #6]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xfff0
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #38]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w9, [sp, #4]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xfff0
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #36]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w9, [sp, #2]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xfff0
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #34]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w9, [sp]
+; CHECK-NONEON-NOSVE-NEXT: tst w9, #0xfff0
+; CHECK-NONEON-NOSVE-NEXT: csel w8, w9, w8, ne
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: umax_v16i16:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -3542,14 +6700,60 @@ define void @umax_v16i16(ptr %a) {
}
define void @umax_v8i32(ptr %a) {
-; CHECK-LABEL: umax_v8i32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: umax z0.s, z0.s, #31
-; CHECK-NEXT: umax z1.s, z1.s, #31
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: umax_v8i32:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: umax z0.s, z0.s, #31
+; CHECK-SVE-NEXT: umax z1.s, z1.s, #31
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
;
+; CHECK-SME-LABEL: umax_v8i32:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: umax z0.s, z0.s, #31
+; CHECK-SME-NEXT: umax z1.s, z1.s, #31
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
+;
+; CHECK-NONEON-NOSVE-LABEL: umax_v8i32:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: mov w8, #31 // =0x1f
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldr w9, [sp, #28]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #31
+; CHECK-NONEON-NOSVE-NEXT: csel w10, w9, w8, hi
+; CHECK-NONEON-NOSVE-NEXT: ldr w9, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #31
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, hi
+; CHECK-NONEON-NOSVE-NEXT: stp w9, w10, [sp, #56]
+; CHECK-NONEON-NOSVE-NEXT: ldr w9, [sp, #20]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #31
+; CHECK-NONEON-NOSVE-NEXT: csel w10, w9, w8, hi
+; CHECK-NONEON-NOSVE-NEXT: ldr w9, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #31
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, hi
+; CHECK-NONEON-NOSVE-NEXT: stp w9, w10, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldr w9, [sp, #12]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #31
+; CHECK-NONEON-NOSVE-NEXT: csel w10, w9, w8, hi
+; CHECK-NONEON-NOSVE-NEXT: ldr w9, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #31
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, hi
+; CHECK-NONEON-NOSVE-NEXT: stp w9, w10, [sp, #40]
+; CHECK-NONEON-NOSVE-NEXT: ldr w9, [sp, #4]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #31
+; CHECK-NONEON-NOSVE-NEXT: csel w10, w9, w8, hi
+; CHECK-NONEON-NOSVE-NEXT: ldr w9, [sp]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #31
+; CHECK-NONEON-NOSVE-NEXT: csel w8, w9, w8, hi
+; CHECK-NONEON-NOSVE-NEXT: stp w8, w10, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: umax_v8i32:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -3595,14 +6799,46 @@ define void @umax_v8i32(ptr %a) {
}
define void @umax_v4i64(ptr %a) {
-; CHECK-LABEL: umax_v4i64:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: umax z0.d, z0.d, #63
-; CHECK-NEXT: umax z1.d, z1.d, #63
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: umax_v4i64:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: umax z0.d, z0.d, #63
+; CHECK-SVE-NEXT: umax z1.d, z1.d, #63
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: umax_v4i64:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: umax z0.d, z0.d, #63
+; CHECK-SME-NEXT: umax z1.d, z1.d, #63
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
;
+; CHECK-NONEON-NOSVE-LABEL: umax_v4i64:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: mov w8, #63 // =0x3f
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldr x9, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: cmp x9, #63
+; CHECK-NONEON-NOSVE-NEXT: csel x10, x9, x8, hi
+; CHECK-NONEON-NOSVE-NEXT: ldr x9, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: cmp x9, #63
+; CHECK-NONEON-NOSVE-NEXT: csel x9, x9, x8, hi
+; CHECK-NONEON-NOSVE-NEXT: stp x9, x10, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldr x9, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: cmp x9, #63
+; CHECK-NONEON-NOSVE-NEXT: csel x10, x9, x8, hi
+; CHECK-NONEON-NOSVE-NEXT: ldr x9, [sp]
+; CHECK-NONEON-NOSVE-NEXT: cmp x9, #63
+; CHECK-NONEON-NOSVE-NEXT: csel x8, x9, x8, hi
+; CHECK-NONEON-NOSVE-NEXT: stp x8, x10, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: umax_v4i64:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -3638,14 +6874,160 @@ define void @umax_v4i64(ptr %a) {
;
define void @umin_v32i8(ptr %a) {
-; CHECK-LABEL: umin_v32i8:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: umin z0.b, z0.b, #7
-; CHECK-NEXT: umin z1.b, z1.b, #7
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: umin_v32i8:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: umin z0.b, z0.b, #7
+; CHECK-SVE-NEXT: umin z1.b, z1.b, #7
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: umin_v32i8:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: umin z0.b, z0.b, #7
+; CHECK-SME-NEXT: umin z1.b, z1.b, #7
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
;
+; CHECK-NONEON-NOSVE-LABEL: umin_v32i8:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: mov w8, #7 // =0x7
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #31]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #63]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #30]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #62]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #29]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #61]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #28]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #60]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #27]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #59]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #26]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #58]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #25]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #57]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #56]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #23]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #55]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #22]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #54]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #21]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #53]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #20]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #52]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #19]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #51]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #18]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #50]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #17]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #49]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #15]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #47]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #14]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #46]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #13]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #45]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #12]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #44]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #11]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #43]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #10]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #42]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #9]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #41]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #40]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #7]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #39]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #6]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #38]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #5]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #37]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #4]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #36]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #3]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #35]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #2]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #34]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp, #1]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strb w9, [sp, #33]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w9, [sp]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #7
+; CHECK-NONEON-NOSVE-NEXT: csel w8, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: umin_v32i8:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -3791,14 +7173,96 @@ define void @umin_v32i8(ptr %a) {
}
define void @umin_v16i16(ptr %a) {
-; CHECK-LABEL: umin_v16i16:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: umin z0.h, z0.h, #15
-; CHECK-NEXT: umin z1.h, z1.h, #15
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: umin_v16i16:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: umin z0.h, z0.h, #15
+; CHECK-SVE-NEXT: umin z1.h, z1.h, #15
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
;
+; CHECK-SME-LABEL: umin_v16i16:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: umin z0.h, z0.h, #15
+; CHECK-SME-NEXT: umin z1.h, z1.h, #15
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
+;
+; CHECK-NONEON-NOSVE-LABEL: umin_v16i16:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: mov w8, #15 // =0xf
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldrh w9, [sp, #30]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #62]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w9, [sp, #28]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #60]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w9, [sp, #26]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #58]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w9, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #56]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w9, [sp, #22]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #54]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w9, [sp, #20]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #52]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w9, [sp, #18]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #50]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w9, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w9, [sp, #14]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #46]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w9, [sp, #12]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #44]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w9, [sp, #10]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #42]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w9, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #40]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w9, [sp, #6]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #38]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w9, [sp, #4]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #36]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w9, [sp, #2]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strh w9, [sp, #34]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w9, [sp]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #15
+; CHECK-NONEON-NOSVE-NEXT: csel w8, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: umin_v16i16:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -3880,14 +7344,60 @@ define void @umin_v16i16(ptr %a) {
}
define void @umin_v8i32(ptr %a) {
-; CHECK-LABEL: umin_v8i32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: umin z0.s, z0.s, #31
-; CHECK-NEXT: umin z1.s, z1.s, #31
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: umin_v8i32:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: umin z0.s, z0.s, #31
+; CHECK-SVE-NEXT: umin z1.s, z1.s, #31
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: umin_v8i32:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: umin z0.s, z0.s, #31
+; CHECK-SME-NEXT: umin z1.s, z1.s, #31
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
;
+; CHECK-NONEON-NOSVE-LABEL: umin_v8i32:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: mov w8, #31 // =0x1f
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldr w9, [sp, #28]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #31
+; CHECK-NONEON-NOSVE-NEXT: csel w10, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: ldr w9, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #31
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: stp w9, w10, [sp, #56]
+; CHECK-NONEON-NOSVE-NEXT: ldr w9, [sp, #20]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #31
+; CHECK-NONEON-NOSVE-NEXT: csel w10, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: ldr w9, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #31
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: stp w9, w10, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldr w9, [sp, #12]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #31
+; CHECK-NONEON-NOSVE-NEXT: csel w10, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: ldr w9, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #31
+; CHECK-NONEON-NOSVE-NEXT: csel w9, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: stp w9, w10, [sp, #40]
+; CHECK-NONEON-NOSVE-NEXT: ldr w9, [sp, #4]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #31
+; CHECK-NONEON-NOSVE-NEXT: csel w10, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: ldr w9, [sp]
+; CHECK-NONEON-NOSVE-NEXT: cmp w9, #31
+; CHECK-NONEON-NOSVE-NEXT: csel w8, w9, w8, lo
+; CHECK-NONEON-NOSVE-NEXT: stp w8, w10, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: umin_v8i32:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -3933,14 +7443,46 @@ define void @umin_v8i32(ptr %a) {
}
define void @umin_v4i64(ptr %a) {
-; CHECK-LABEL: umin_v4i64:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: umin z0.d, z0.d, #63
-; CHECK-NEXT: umin z1.d, z1.d, #63
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: umin_v4i64:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: umin z0.d, z0.d, #63
+; CHECK-SVE-NEXT: umin z1.d, z1.d, #63
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
;
+; CHECK-SME-LABEL: umin_v4i64:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: umin z0.d, z0.d, #63
+; CHECK-SME-NEXT: umin z1.d, z1.d, #63
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
+;
+; CHECK-NONEON-NOSVE-LABEL: umin_v4i64:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: mov w8, #63 // =0x3f
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldr x9, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: cmp x9, #63
+; CHECK-NONEON-NOSVE-NEXT: csel x10, x9, x8, lo
+; CHECK-NONEON-NOSVE-NEXT: ldr x9, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: cmp x9, #63
+; CHECK-NONEON-NOSVE-NEXT: csel x9, x9, x8, lo
+; CHECK-NONEON-NOSVE-NEXT: stp x9, x10, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldr x9, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: cmp x9, #63
+; CHECK-NONEON-NOSVE-NEXT: csel x10, x9, x8, lo
+; CHECK-NONEON-NOSVE-NEXT: ldr x9, [sp]
+; CHECK-NONEON-NOSVE-NEXT: cmp x9, #63
+; CHECK-NONEON-NOSVE-NEXT: csel x8, x9, x8, lo
+; CHECK-NONEON-NOSVE-NEXT: stp x8, x10, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: umin_v4i64:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -3976,14 +7518,127 @@ define void @umin_v4i64(ptr %a) {
;
define void @xor_v32i8(ptr %a) {
-; CHECK-LABEL: xor_v32i8:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: eor z0.b, z0.b, #0x7
-; CHECK-NEXT: eor z1.b, z1.b, #0x7
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: xor_v32i8:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: eor z0.b, z0.b, #0x7
+; CHECK-SVE-NEXT: eor z1.b, z1.b, #0x7
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: xor_v32i8:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: eor z0.b, z0.b, #0x7
+; CHECK-SME-NEXT: eor z1.b, z1.b, #0x7
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
;
+; CHECK-NONEON-NOSVE-LABEL: xor_v32i8:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #31]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #63]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #30]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #62]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #29]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #61]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #28]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #60]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #27]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #59]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #26]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #58]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #25]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #57]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #56]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #23]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #55]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #22]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #54]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #21]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #53]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #20]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #52]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #19]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #51]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #18]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #50]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #17]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #49]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #15]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #47]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #14]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #46]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #13]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #45]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #12]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #44]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #11]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #43]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #10]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #42]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #9]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #41]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #40]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #7]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #39]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #6]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #38]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #5]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #37]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #4]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #36]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #3]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #35]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #2]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #34]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp, #1]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #33]
+; CHECK-NONEON-NOSVE-NEXT: ldrb w8, [sp]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0x7
+; CHECK-NONEON-NOSVE-NEXT: strb w8, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: xor_v32i8:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -4096,14 +7751,79 @@ define void @xor_v32i8(ptr %a) {
}
define void @xor_v16i16(ptr %a) {
-; CHECK-LABEL: xor_v16i16:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: eor z0.h, z0.h, #0xf
-; CHECK-NEXT: eor z1.h, z1.h, #0xf
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: xor_v16i16:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: eor z0.h, z0.h, #0xf
+; CHECK-SVE-NEXT: eor z1.h, z1.h, #0xf
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
;
+; CHECK-SME-LABEL: xor_v16i16:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: eor z0.h, z0.h, #0xf
+; CHECK-SME-NEXT: eor z1.h, z1.h, #0xf
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
+;
+; CHECK-NONEON-NOSVE-LABEL: xor_v16i16:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #30]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #62]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #28]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #60]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #26]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #58]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #56]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #22]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #54]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #20]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #52]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #18]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #50]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #14]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #46]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #12]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #44]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #10]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #42]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #40]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #6]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #38]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #4]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #36]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp, #2]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #34]
+; CHECK-NONEON-NOSVE-NEXT: ldrh w8, [sp]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0xf
+; CHECK-NONEON-NOSVE-NEXT: strh w8, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: xor_v16i16:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -4168,14 +7888,51 @@ define void @xor_v16i16(ptr %a) {
}
define void @xor_v8i32(ptr %a) {
-; CHECK-LABEL: xor_v8i32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: eor z0.s, z0.s, #0x1f
-; CHECK-NEXT: eor z1.s, z1.s, #0x1f
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: xor_v8i32:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: eor z0.s, z0.s, #0x1f
+; CHECK-SVE-NEXT: eor z1.s, z1.s, #0x1f
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: xor_v8i32:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: eor z0.s, z0.s, #0x1f
+; CHECK-SME-NEXT: eor z1.s, z1.s, #0x1f
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
;
+; CHECK-NONEON-NOSVE-LABEL: xor_v8i32:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #28]
+; CHECK-NONEON-NOSVE-NEXT: eor w9, w8, #0x1f
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0x1f
+; CHECK-NONEON-NOSVE-NEXT: stp w8, w9, [sp, #56]
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #20]
+; CHECK-NONEON-NOSVE-NEXT: eor w9, w8, #0x1f
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0x1f
+; CHECK-NONEON-NOSVE-NEXT: stp w8, w9, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #12]
+; CHECK-NONEON-NOSVE-NEXT: eor w9, w8, #0x1f
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0x1f
+; CHECK-NONEON-NOSVE-NEXT: stp w8, w9, [sp, #40]
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp, #4]
+; CHECK-NONEON-NOSVE-NEXT: eor w9, w8, #0x1f
+; CHECK-NONEON-NOSVE-NEXT: ldr w8, [sp]
+; CHECK-NONEON-NOSVE-NEXT: eor w8, w8, #0x1f
+; CHECK-NONEON-NOSVE-NEXT: stp w8, w9, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: xor_v8i32:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -4212,14 +7969,41 @@ define void @xor_v8i32(ptr %a) {
}
define void @xor_v4i64(ptr %a) {
-; CHECK-LABEL: xor_v4i64:
-; CHECK: // %bb.0:
-; CHECK-NEXT: ldp q0, q1, [x0]
-; CHECK-NEXT: eor z0.d, z0.d, #0x3f
-; CHECK-NEXT: eor z1.d, z1.d, #0x3f
-; CHECK-NEXT: stp q0, q1, [x0]
-; CHECK-NEXT: ret
+; CHECK-SVE-LABEL: xor_v4i64:
+; CHECK-SVE: // %bb.0:
+; CHECK-SVE-NEXT: ldp q0, q1, [x0]
+; CHECK-SVE-NEXT: eor z0.d, z0.d, #0x3f
+; CHECK-SVE-NEXT: eor z1.d, z1.d, #0x3f
+; CHECK-SVE-NEXT: stp q0, q1, [x0]
+; CHECK-SVE-NEXT: ret
+;
+; CHECK-SME-LABEL: xor_v4i64:
+; CHECK-SME: // %bb.0:
+; CHECK-SME-NEXT: ldp q0, q1, [x0]
+; CHECK-SME-NEXT: eor z0.d, z0.d, #0x3f
+; CHECK-SME-NEXT: eor z1.d, z1.d, #0x3f
+; CHECK-SME-NEXT: stp q0, q1, [x0]
+; CHECK-SME-NEXT: ret
;
+; CHECK-NONEON-NOSVE-LABEL: xor_v4i64:
+; CHECK-NONEON-NOSVE: // %bb.0:
+; CHECK-NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
+; CHECK-NONEON-NOSVE-NEXT: stp q1, q0, [sp, #-64]!
+; CHECK-NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
+; CHECK-NONEON-NOSVE-NEXT: ldr x8, [sp, #24]
+; CHECK-NONEON-NOSVE-NEXT: eor x9, x8, #0x3f
+; CHECK-NONEON-NOSVE-NEXT: ldr x8, [sp, #16]
+; CHECK-NONEON-NOSVE-NEXT: eor x8, x8, #0x3f
+; CHECK-NONEON-NOSVE-NEXT: stp x8, x9, [sp, #48]
+; CHECK-NONEON-NOSVE-NEXT: ldr x8, [sp, #8]
+; CHECK-NONEON-NOSVE-NEXT: eor x9, x8, #0x3f
+; CHECK-NONEON-NOSVE-NEXT: ldr x8, [sp]
+; CHECK-NONEON-NOSVE-NEXT: eor x8, x8, #0x3f
+; CHECK-NONEON-NOSVE-NEXT: stp x8, x9, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #32]
+; CHECK-NONEON-NOSVE-NEXT: stp q0, q1, [x0]
+; CHECK-NONEON-NOSVE-NEXT: add sp, sp, #64
+; CHECK-NONEON-NOSVE-NEXT: ret
; NONEON-NOSVE-LABEL: xor_v4i64:
; NONEON-NOSVE: // %bb.0:
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
@@ -4264,3 +8048,5 @@ declare <32 x i8> @llvm.umin.v32i8(<32 x i8>, <32 x i8>)
declare <16 x i16> @llvm.umin.v16i16(<16 x i16>, <16 x i16>)
declare <8 x i32> @llvm.umin.v8i32(<8 x i32>, <8 x i32>)
declare <4 x i64> @llvm.umin.v4i64(<4 x i64>, <4 x i64>)
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
More information about the llvm-commits
mailing list