[llvm] [AArch64][SVE][SelectionDAG] Improve codegen for SVE patterns sext(icmp) and zext(icmp) (PR #192052)
Sushant Gokhale via llvm-commits
llvm-commits at lists.llvm.org
Tue Apr 14 06:06:08 PDT 2026
https://github.com/sushgokh created https://github.com/llvm/llvm-project/pull/192052
For SVE vectors:
1. sext(icmp slt) --> max(min(a-b, 0), -1) --> sclamp(sqsub(a, b), -1, 0)
2. sext(icmp slt) --> min(max(b-a, 0), 1) --> sclamp(sqsub(b, a), 0, 1)
3. zext(icmp ult) --> min(max(b-a, 0), 1) --> umin(uqsub(b, a), 1)
4. Let,
Op = sext(icmp ult(a,b))
NewOp = umin(uqsub(b, a), 1)
Then,
sdot(acc, Op, Op) --> sdot(acc, NewOp, NewOp)
The other changes follow:
1. icmp sgt is just inverse of the icmp slt.
2. udot is similar to sdot
>From 89e6808cbb99b0fa95f317e6e143c1a1d81513a8 Mon Sep 17 00:00:00 2001
From: sgokhale <sgokhale at nvidia.com>
Date: Tue, 14 Apr 2026 05:16:37 -0700
Subject: [PATCH 1/2] [NFC][AArch64] Add test cases for SVE sext(icmp) and
zext(icmp)
In some scenarios, these patterns can be replaced with better patterns. Future patch will improve these.
---
.../CodeGen/AArch64/sve-icmp-sext-zext.ll | 565 ++++++++++++++++++
1 file changed, 565 insertions(+)
create mode 100644 llvm/test/CodeGen/AArch64/sve-icmp-sext-zext.ll
diff --git a/llvm/test/CodeGen/AArch64/sve-icmp-sext-zext.ll b/llvm/test/CodeGen/AArch64/sve-icmp-sext-zext.ll
new file mode 100644
index 0000000000000..89437ddb82fa3
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-icmp-sext-zext.ll
@@ -0,0 +1,565 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=aarch64 -mattr=+dotprod,+sve2p1 --aarch64-sve-vector-bits-min=256 -o - < %s | FileCheck %s
+
+define <vscale x 4 x i32> @test1(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
+; CHECK-LABEL: test1:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: sdot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+{
+ %1 = icmp ult <vscale x 16 x i8> %a, %b
+ %2 = sext <vscale x 16 x i1> %1 to <vscale x 16 x i8>
+ %3 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sdot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %2, <vscale x 16 x i8> %2)
+ ret <vscale x 4 x i32> %3
+}
+
+define <vscale x 4 x i32> @test2(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
+; CHECK-LABEL: test2:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #1 // =0x1
+; CHECK-NEXT: sdot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+{
+ %1 = icmp ult <vscale x 16 x i8> %a, %b
+ %2 = zext <vscale x 16 x i1> %1 to <vscale x 16 x i8>
+ %3 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sdot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %2, <vscale x 16 x i8> %2)
+ ret <vscale x 4 x i32> %3
+}
+
+define <vscale x 4 x i32> @test3(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
+; CHECK-LABEL: test3:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #1 // =0x1
+; CHECK-NEXT: sdot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+{
+ %1 = icmp ult <vscale x 16 x i8> %a, %b
+ %2 = zext <vscale x 16 x i1> %1 to <vscale x 16 x i8>
+ %3 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sdot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %2, <vscale x 16 x i8> %2)
+ ret <vscale x 4 x i32> %3
+}
+
+define <vscale x 4 x i32> @test4(<32 x i8> %a, <32 x i8> %b) {
+; CHECK-LABEL: test4:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: ptrue p0.b, vl16
+; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
+; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: sdot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+ %1 = icmp ult <32 x i8> %a, %b
+ %2 = sext <32 x i1> %1 to <32 x i8>
+ %3 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v32i8(<vscale x 16 x i8> poison, <32 x i8> %2, i64 0)
+ %4 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sdot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %3, <vscale x 16 x i8> %3)
+ ret <vscale x 4 x i32> %4
+}
+
+define <vscale x 4 x i32> @test5(<32 x i8> %a, <32 x i8> %b) {
+; CHECK-LABEL: test5:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: ptrue p0.b, vl16
+; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
+; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: and z1.b, z1.b, #0x1
+; CHECK-NEXT: sdot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+ %1 = icmp ult <32 x i8> %a, %b
+ %2 = zext <32 x i1> %1 to <32 x i8>
+ %3 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v32i8(<vscale x 16 x i8> poison, <32 x i8> %2, i64 0)
+ %4 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sdot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %3, <vscale x 16 x i8> %3)
+ ret <vscale x 4 x i32> %4
+}
+
+define <vscale x 4 x i32> @test6(<32 x i8> %a, <32 x i8> %b) {
+; CHECK-LABEL: test6:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: ptrue p0.b, vl16
+; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
+; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: and z1.b, z1.b, #0x1
+; CHECK-NEXT: sdot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+ %1 = icmp ult <32 x i8> %a, %b
+ %2 = zext <32 x i1> %1 to <32 x i8>
+ %3 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v32i8(<vscale x 16 x i8> poison, <32 x i8> %2, i64 0)
+ %4 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sdot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %3, <vscale x 16 x i8> %3)
+ ret <vscale x 4 x i32> %4
+}
+
+define <vscale x 4 x i32> @test7(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
+; CHECK-LABEL: test7:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: sdot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+{
+ %1 = icmp ugt <vscale x 16 x i8> %a, %b
+ %2 = sext <vscale x 16 x i1> %1 to <vscale x 16 x i8>
+ %3 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sdot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %2, <vscale x 16 x i8> %2)
+ ret <vscale x 4 x i32> %3
+}
+
+define <vscale x 4 x i32> @test8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
+; CHECK-LABEL: test8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #1 // =0x1
+; CHECK-NEXT: sdot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+{
+ %1 = icmp ugt <vscale x 16 x i8> %a, %b
+ %2 = zext <vscale x 16 x i1> %1 to <vscale x 16 x i8>
+ %3 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sdot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %2, <vscale x 16 x i8> %2)
+ ret <vscale x 4 x i32> %3
+}
+
+define <vscale x 4 x i32> @test9(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
+; CHECK-LABEL: test9:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #1 // =0x1
+; CHECK-NEXT: sdot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+{
+ %1 = icmp ugt <vscale x 16 x i8> %a, %b
+ %2 = zext <vscale x 16 x i1> %1 to <vscale x 16 x i8>
+ %3 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sdot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %2, <vscale x 16 x i8> %2)
+ ret <vscale x 4 x i32> %3
+}
+
+define <vscale x 4 x i32> @test10(<32 x i8> %a, <32 x i8> %b) {
+; CHECK-LABEL: test10:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: ptrue p0.b, vl16
+; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: splice z2.b, p0, { z2.b, z3.b }
+; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z2.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: sdot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+ %1 = icmp ugt <32 x i8> %a, %b
+ %2 = sext <32 x i1> %1 to <32 x i8>
+ %3 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v32i8(<vscale x 16 x i8> poison, <32 x i8> %2, i64 0)
+ %4 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sdot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %3, <vscale x 16 x i8> %3)
+ ret <vscale x 4 x i32> %4
+}
+
+define <vscale x 4 x i32> @test11(<32 x i8> %a, <32 x i8> %b) {
+; CHECK-LABEL: test11:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: ptrue p0.b, vl16
+; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: splice z2.b, p0, { z2.b, z3.b }
+; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z2.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: and z1.b, z1.b, #0x1
+; CHECK-NEXT: sdot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+ %1 = icmp ugt <32 x i8> %a, %b
+ %2 = zext <32 x i1> %1 to <32 x i8>
+ %3 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v32i8(<vscale x 16 x i8> poison, <32 x i8> %2, i64 0)
+ %4 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sdot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %3, <vscale x 16 x i8> %3)
+ ret <vscale x 4 x i32> %4
+}
+
+define <vscale x 4 x i32> @test12(<32 x i8> %a, <32 x i8> %b) {
+; CHECK-LABEL: test12:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: ptrue p0.b, vl16
+; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: splice z2.b, p0, { z2.b, z3.b }
+; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z2.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: and z1.b, z1.b, #0x1
+; CHECK-NEXT: sdot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+ %1 = icmp ugt <32 x i8> %a, %b
+ %2 = zext <32 x i1> %1 to <32 x i8>
+ %3 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v32i8(<vscale x 16 x i8> poison, <32 x i8> %2, i64 0)
+ %4 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.sdot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %3, <vscale x 16 x i8> %3)
+ ret <vscale x 4 x i32> %4
+}
+
+define <vscale x 16 x i8> @test13(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1) {
+; CHECK-LABEL: test13:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: cmpgt p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: mov z0.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: ret
+entry:
+ %2 = icmp slt <vscale x 16 x i8> %0, %1
+ %3 = sext <vscale x 16 x i1> %2 to <vscale x 16 x i8>
+ ret <vscale x 16 x i8> %3
+}
+
+define <vscale x 16 x i8> @test14(<32 x i8> %0, <32 x i8> %1) {
+; CHECK-LABEL: test14:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: ptrue p0.b, vl16
+; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
+; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: cmpgt p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: mov z0.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: ret
+entry:
+ %2 = icmp slt <32 x i8> %0, %1
+ %3 = sext <32 x i1> %2 to <32 x i8>
+ %4 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v32i8(<vscale x 16 x i8> poison, <32 x i8> %3, i64 0)
+ ret <vscale x 16 x i8> %4
+}
+
+; FIXME: Lower this sequence to sqsub/sclamp
+define <vscale x 16 x i8> @test15(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1) {
+; CHECK-LABEL: test15:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: cmpgt p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: mov z1.b, #1 // =0x1
+; CHECK-NEXT: mov z0.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: add z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT: ret
+entry:
+ %2 = icmp slt <vscale x 16 x i8> %0, %1
+ %3 = sext <vscale x 16 x i1> %2 to <vscale x 16 x i8>
+ %4 = zext <vscale x 16 x i1> %2 to <vscale x 16 x i8>
+ %5 = add <vscale x 16 x i8> %3, %4
+ ret <vscale x 16 x i8> %5
+}
+
+define <vscale x 16 x i8> @test16(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1) {
+; CHECK-LABEL: test16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: cmpgt p0.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: mov z0.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: ret
+entry:
+ %2 = icmp sgt <vscale x 16 x i8> %0, %1
+ %3 = sext <vscale x 16 x i1> %2 to <vscale x 16 x i8>
+ ret <vscale x 16 x i8> %3
+}
+
+define <vscale x 16 x i8> @test17(<32 x i8> %0, <32 x i8> %1) {
+; CHECK-LABEL: test17:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: ptrue p0.b, vl16
+; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: splice z2.b, p0, { z2.b, z3.b }
+; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: cmpgt p0.b, p0/z, z0.b, z2.b
+; CHECK-NEXT: mov z0.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: ret
+entry:
+ %2 = icmp sgt <32 x i8> %0, %1
+ %3 = sext <32 x i1> %2 to <32 x i8>
+ %4 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v32i8(<vscale x 16 x i8> poison, <32 x i8> %3, i64 0)
+ ret <vscale x 16 x i8> %4
+}
+
+define <vscale x 4 x i32> @test18(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
+; CHECK-LABEL: test18:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: udot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+{
+ %1 = icmp ult <vscale x 16 x i8> %a, %b
+ %2 = sext <vscale x 16 x i1> %1 to <vscale x 16 x i8>
+ %3 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.udot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %2, <vscale x 16 x i8> %2)
+ ret <vscale x 4 x i32> %3
+}
+
+define <vscale x 4 x i32> @test19(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
+; CHECK-LABEL: test19:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #1 // =0x1
+; CHECK-NEXT: udot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+{
+ %1 = icmp ult <vscale x 16 x i8> %a, %b
+ %2 = zext <vscale x 16 x i1> %1 to <vscale x 16 x i8>
+ %3 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.udot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %2, <vscale x 16 x i8> %2)
+ ret <vscale x 4 x i32> %3
+}
+
+
+define <vscale x 4 x i32> @test20(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
+; CHECK-LABEL: test20:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #1 // =0x1
+; CHECK-NEXT: udot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+{
+ %1 = icmp ult <vscale x 16 x i8> %a, %b
+ %2 = zext <vscale x 16 x i1> %1 to <vscale x 16 x i8>
+ %3 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.udot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %2, <vscale x 16 x i8> %2)
+ ret <vscale x 4 x i32> %3
+}
+
+define <vscale x 4 x i32> @test21(<32 x i8> %a, <32 x i8> %b) {
+; CHECK-LABEL: test21:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: ptrue p0.b, vl16
+; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
+; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: udot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+ %1 = icmp ult <32 x i8> %a, %b
+ %2 = sext <32 x i1> %1 to <32 x i8>
+ %3 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v32i8(<vscale x 16 x i8> poison, <32 x i8> %2, i64 0)
+ %4 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.udot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %3, <vscale x 16 x i8> %3)
+ ret <vscale x 4 x i32> %4
+}
+
+define <vscale x 4 x i32> @test22(<32 x i8> %a, <32 x i8> %b) {
+; CHECK-LABEL: test22:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: ptrue p0.b, vl16
+; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
+; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: and z1.b, z1.b, #0x1
+; CHECK-NEXT: udot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+ %1 = icmp ult <32 x i8> %a, %b
+ %2 = zext <32 x i1> %1 to <32 x i8>
+ %3 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v32i8(<vscale x 16 x i8> poison, <32 x i8> %2, i64 0)
+ %4 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.udot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %3, <vscale x 16 x i8> %3)
+ ret <vscale x 4 x i32> %4
+}
+
+define <vscale x 4 x i32> @test23(<32 x i8> %a, <32 x i8> %b) {
+; CHECK-LABEL: test23:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: ptrue p0.b, vl16
+; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
+; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: and z1.b, z1.b, #0x1
+; CHECK-NEXT: udot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+ %1 = icmp ult <32 x i8> %a, %b
+ %2 = zext <32 x i1> %1 to <32 x i8>
+ %3 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v32i8(<vscale x 16 x i8> poison, <32 x i8> %2, i64 0)
+ %4 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.udot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %3, <vscale x 16 x i8> %3)
+ ret <vscale x 4 x i32> %4
+}
+
+define <vscale x 4 x i32> @test24(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
+; CHECK-LABEL: test24:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: udot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+{
+ %1 = icmp ugt <vscale x 16 x i8> %a, %b
+ %2 = sext <vscale x 16 x i1> %1 to <vscale x 16 x i8>
+ %3 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.udot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %2, <vscale x 16 x i8> %2)
+ ret <vscale x 4 x i32> %3
+}
+
+define <vscale x 4 x i32> @test25(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
+; CHECK-LABEL: test25:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #1 // =0x1
+; CHECK-NEXT: udot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+{
+ %1 = icmp ugt <vscale x 16 x i8> %a, %b
+ %2 = zext <vscale x 16 x i1> %1 to <vscale x 16 x i8>
+ %3 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.udot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %2, <vscale x 16 x i8> %2)
+ ret <vscale x 4 x i32> %3
+}
+
+define <vscale x 4 x i32> @test26(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
+; CHECK-LABEL: test26:
+; CHECK: // %bb.0:
+; CHECK-NEXT: ptrue p0.b
+; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #1 // =0x1
+; CHECK-NEXT: udot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+{
+ %1 = icmp ugt <vscale x 16 x i8> %a, %b
+ %2 = zext <vscale x 16 x i1> %1 to <vscale x 16 x i8>
+ %3 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.udot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %2, <vscale x 16 x i8> %2)
+ ret <vscale x 4 x i32> %3
+}
+
+define <vscale x 4 x i32> @test27(<32 x i8> %a, <32 x i8> %b) {
+; CHECK-LABEL: test27:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: ptrue p0.b, vl16
+; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: splice z2.b, p0, { z2.b, z3.b }
+; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z2.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: udot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+ %1 = icmp ugt <32 x i8> %a, %b
+ %2 = sext <32 x i1> %1 to <32 x i8>
+ %3 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v32i8(<vscale x 16 x i8> poison, <32 x i8> %2, i64 0)
+ %4 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.udot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %3, <vscale x 16 x i8> %3)
+ ret <vscale x 4 x i32> %4
+}
+
+define <vscale x 4 x i32> @test28(<32 x i8> %a, <32 x i8> %b) {
+; CHECK-LABEL: test28:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: ptrue p0.b, vl16
+; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: splice z2.b, p0, { z2.b, z3.b }
+; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z2.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: and z1.b, z1.b, #0x1
+; CHECK-NEXT: udot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+ %1 = icmp ugt <32 x i8> %a, %b
+ %2 = zext <32 x i1> %1 to <32 x i8>
+ %3 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v32i8(<vscale x 16 x i8> poison, <32 x i8> %2, i64 0)
+ %4 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.udot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %3, <vscale x 16 x i8> %3)
+ ret <vscale x 4 x i32> %4
+}
+
+define <vscale x 4 x i32> @test29(<32 x i8> %a, <32 x i8> %b) {
+; CHECK-LABEL: test29:
+; CHECK: // %bb.0:
+; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: ptrue p0.b, vl16
+; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: splice z2.b, p0, { z2.b, z3.b }
+; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
+; CHECK-NEXT: ptrue p0.b, vl32
+; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z2.b
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: and z1.b, z1.b, #0x1
+; CHECK-NEXT: udot z0.s, z1.b, z1.b
+; CHECK-NEXT: ret
+ %1 = icmp ugt <32 x i8> %a, %b
+ %2 = zext <32 x i1> %1 to <32 x i8>
+ %3 = tail call <vscale x 16 x i8> @llvm.vector.insert.nxv16i8.v32i8(<vscale x 16 x i8> poison, <32 x i8> %2, i64 0)
+ %4 = tail call <vscale x 4 x i32> @llvm.aarch64.sve.udot.nxv4i32(<vscale x 4 x i32> zeroinitializer, <vscale x 16 x i8> %3, <vscale x 16 x i8> %3)
+ ret <vscale x 4 x i32> %4
+}
>From 1c09c2f490f2c8b0fa00ac92852779bf1856207a Mon Sep 17 00:00:00 2001
From: sgokhale <sgokhale at nvidia.com>
Date: Tue, 14 Apr 2026 05:40:37 -0700
Subject: [PATCH 2/2] [AArch64][SVE][SelectionDAG] Improve codegen for SVE
patterns sext(icmp) and zext(icmp)
For SVE vectors:
1. sext(icmp slt) --> max(min(a-b, 0), -1) --> sclamp(sqsub(a, b), -1, 0)
2. sext(icmp slt) --> min(max(b-a, 0), 1) --> sclamp(sqsub(b, a), 0, 1)
3. zext(icmp ult) --> min(max(b-a, 0), 1) --> umin(uqsub(b, a), 1)
4. Let,
Op = sext(icmp ult(a,b))
NewOp = umin(uqsub(b, a), 1)
Then,
sdot(acc, Op, Op) --> sdot(acc, NewOp, NewOp)
The other changes follow:
1. icmp sgt is just inverse of the icmp slt.
2. udot is similar to sdot
---
.../Target/AArch64/AArch64ISelLowering.cpp | 97 +++++++++-
.../CodeGen/AArch64/sve-icmp-sext-zext.ll | 174 ++++++++----------
2 files changed, 169 insertions(+), 102 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index ece24767bdbb9..1f83b8cb705ea 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -20941,6 +20941,22 @@ static SDValue performSVEAndCombine(SDNode *N,
SDValue Src = N->getOperand(0);
unsigned Opc = Src->getOpcode();
+ // and(splat(1), sext(setcc_merge_zero)) -> zext(setcc_merge_zero)
+ SDLoc DL(N);
+ SDValue Op0 = N->getOperand(0);
+ SDValue Op1 = N->getOperand(1);
+ if (Op0.getOpcode() == ISD::SPLAT_VECTOR ||
+ Op1.getOpcode() == ISD::SPLAT_VECTOR) {
+ SDValue NonSplatOp = (Op0.getOpcode() == ISD::SPLAT_VECTOR ? Op1 : Op0);
+ SDValue SplatOp = (Op0.getOpcode() == ISD::SPLAT_VECTOR ? Op0 : Op1);
+ if (NonSplatOp.getOpcode() == ISD::SIGN_EXTEND) {
+ SDValue Compare = NonSplatOp.getOperand(0);
+ if (Compare.getOpcode() == AArch64ISD::SETCC_MERGE_ZERO) {
+ return DAG.getNode(ISD::ZERO_EXTEND, DL, N->getValueType(0), Compare);
+ }
+ }
+ }
+
// Zero/any extend of an unsigned unpack
if (Opc == AArch64ISD::UUNPKHI || Opc == AArch64ISD::UUNPKLO) {
SDValue UnpkOp = Src->getOperand(0);
@@ -20949,7 +20965,6 @@ static SDValue performSVEAndCombine(SDNode *N,
if (Dup.getOpcode() != ISD::SPLAT_VECTOR)
return SDValue();
- SDLoc DL(N);
ConstantSDNode *C = dyn_cast<ConstantSDNode>(Dup->getOperand(0));
if (!C)
return SDValue();
@@ -24091,6 +24106,37 @@ static SDValue performIntrinsicCombine(SDNode *N,
case Intrinsic::aarch64_sve_bsl2n:
case Intrinsic::aarch64_sve_nbsl:
return combineSVEBitSel(IID, N, DAG);
+ case Intrinsic::aarch64_sve_udot:
+ case Intrinsic::aarch64_sve_sdot: {
+ // sdot(acc, extend(icmp_ult(A, B)), extend(icmp_ult(A, B)))
+ // -> sdot(acc, umin(usubsat(B, A)), umin(usubsat(B, A)))
+
+ // sdot(acc, extend(icmp_ugt(A, B)), extend(icmp_ugt(A, B)))
+ // -> sdot(acc, umin(usubsat(A, B)), umin(usubsat(A, B)))
+ SDValue Extend1 = N->getOperand(2);
+ SDValue Extend2 = N->getOperand(3);
+ if (Extend1 == Extend2 && (Extend1.getOpcode() == ISD::SIGN_EXTEND ||
+ Extend1.getOpcode() == ISD::ZERO_EXTEND)) {
+ SDValue Compare = Extend1.getOperand(0);
+ if (Compare.getOpcode() != AArch64ISD::SETCC_MERGE_ZERO)
+ break;
+ ISD::CondCode CC = cast<CondCodeSDNode>(Compare.getOperand(3))->get();
+ if (CC != ISD::SETULT && CC != ISD::SETUGT)
+ break;
+ SDLoc DL(N);
+ EVT ResVT = Extend1.getValueType();
+ SDValue A = Compare.getOperand(CC == ISD::SETULT ? 1 : 2);
+ SDValue B = Compare.getOperand(CC == ISD::SETULT ? 2 : 1);
+ SDValue Sub = DAG.getNode(ISD::USUBSAT, DL, ResVT, B, A);
+ SDValue One = DAG.getConstant(1, DL, MVT::i32);
+ SDValue SplatOne = DAG.getSplatVector(ResVT, DL, One);
+ SDValue Min = DAG.getNode(ISD::UMIN, DL, ResVT, Sub, SplatOne);
+ unsigned Opcode = (IID == Intrinsic::aarch64_sve_udot ? AArch64ISD::UDOT
+ : AArch64ISD::SDOT);
+ return DAG.getNode(Opcode, DL, N->getValueType(0), N->getOperand(1), Min,
+ Min);
+ }
+ }
}
return SDValue();
}
@@ -24422,6 +24468,55 @@ static SDValue performExtendCombine(SDNode *N,
TargetLowering::DAGCombinerInfo &DCI,
SelectionDAG &DAG,
const AArch64Subtarget *Subtarget) {
+ // sext(icmp slt(a, b)) -> sclamp(sqsub(a, b), -1, 0)
+ // sext(icmp sgt(a, b)) -> sclamp(sqsub(b, a), -1, 0)
+ if (N->getOpcode() == ISD::SIGN_EXTEND) {
+ SDValue Compare = N->getOperand(0);
+ if (Compare.getOpcode() == AArch64ISD::SETCC_MERGE_ZERO) {
+ ISD::CondCode CC = cast<CondCodeSDNode>(Compare.getOperand(3))->get();
+ if (Compare.hasOneUse() && Subtarget->hasSVE2p1() &&
+ (CC == ISD::SETLT || CC == ISD::SETGT)) {
+ SDLoc DL(N);
+ SDValue A = Compare.getOperand(CC == ISD::SETLT ? 1 : 2);
+ SDValue B = Compare.getOperand(CC == ISD::SETLT ? 2 : 1);
+ SDValue Sub = DAG.getNode(ISD::SSUBSAT, DL, N->getValueType(0), A, B);
+ SDValue MinusOne = DAG.getSplatVector(
+ N->getValueType(0), DL, DAG.getSignedConstant(-1, DL, MVT::i32));
+ SDValue Zero = DAG.getSplatVector(N->getValueType(0), DL,
+ DAG.getConstant(0, DL, MVT::i32));
+ SDValue Sclamp = DAG.getNode(
+ ISD::INTRINSIC_WO_CHAIN, DL, N->getValueType(0),
+ DAG.getConstant(Intrinsic::aarch64_sve_sclamp, DL, MVT::i64), Sub,
+ MinusOne, Zero);
+ return Sclamp;
+ }
+ }
+ }
+ // zext(icmp slt(a, b)) -> sclamp(sqsub(b,a), 0, 1)
+ // zext(icmp sgt(a, b)) -> sclamp(sqsub(a,b), 0, 1)
+ if (N->getOpcode() == ISD::ZERO_EXTEND) {
+ SDValue Compare = N->getOperand(0);
+ if (Compare.getOpcode() == AArch64ISD::SETCC_MERGE_ZERO) {
+ ISD::CondCode CC = cast<CondCodeSDNode>(Compare.getOperand(3))->get();
+ if (Compare.hasOneUse() && Subtarget->hasSVE2p1() &&
+ (CC == ISD::SETLT || CC == ISD::SETGT)) {
+ SDLoc DL(N);
+ SDValue A = Compare.getOperand(CC == ISD::SETLT ? 1 : 2);
+ SDValue B = Compare.getOperand(CC == ISD::SETLT ? 2 : 1);
+ SDValue Sub = DAG.getNode(ISD::SSUBSAT, DL, N->getValueType(0), B, A);
+ SDValue One = DAG.getSplatVector(N->getValueType(0), DL,
+ DAG.getConstant(1, DL, MVT::i32));
+ SDValue Zero = DAG.getSplatVector(N->getValueType(0), DL,
+ DAG.getConstant(0, DL, MVT::i32));
+ SDValue Sclamp = DAG.getNode(
+ ISD::INTRINSIC_WO_CHAIN, DL, N->getValueType(0),
+ DAG.getConstant(Intrinsic::aarch64_sve_sclamp, DL, MVT::i64), Sub,
+ Zero, One);
+ return Sclamp;
+ }
+ }
+ }
+
// If we see something like (zext (sabd (extract_high ...), (DUP ...))) then
// we can convert that DUP into another extract_high (of a bigger DUP), which
// helps the backend to decide that an sabdl2 would be useful, saving a real
diff --git a/llvm/test/CodeGen/AArch64/sve-icmp-sext-zext.ll b/llvm/test/CodeGen/AArch64/sve-icmp-sext-zext.ll
index 89437ddb82fa3..b53d36e0ee13d 100644
--- a/llvm/test/CodeGen/AArch64/sve-icmp-sext-zext.ll
+++ b/llvm/test/CodeGen/AArch64/sve-icmp-sext-zext.ll
@@ -4,10 +4,9 @@
define <vscale x 4 x i32> @test1(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
; CHECK-LABEL: test1:
; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: uqsub z1.b, z1.b, z0.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: sdot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
{
@@ -20,10 +19,9 @@ define <vscale x 4 x i32> @test1(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
define <vscale x 4 x i32> @test2(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
; CHECK-LABEL: test2:
; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: uqsub z1.b, z1.b, z0.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #1 // =0x1
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: sdot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
{
@@ -36,10 +34,9 @@ define <vscale x 4 x i32> @test2(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
define <vscale x 4 x i32> @test3(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
; CHECK-LABEL: test3:
; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: uqsub z1.b, z1.b, z0.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #1 // =0x1
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: sdot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
{
@@ -59,10 +56,9 @@ define <vscale x 4 x i32> @test4(<32 x i8> %a, <32 x i8> %b) {
; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
-; CHECK-NEXT: ptrue p0.b, vl32
-; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: uqsub z1.b, z1.b, z0.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: sdot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
%1 = icmp ult <32 x i8> %a, %b
@@ -82,11 +78,9 @@ define <vscale x 4 x i32> @test5(<32 x i8> %a, <32 x i8> %b) {
; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
-; CHECK-NEXT: ptrue p0.b, vl32
-; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: uqsub z1.b, z1.b, z0.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: and z1.b, z1.b, #0x1
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: sdot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
%1 = icmp ult <32 x i8> %a, %b
@@ -106,11 +100,9 @@ define <vscale x 4 x i32> @test6(<32 x i8> %a, <32 x i8> %b) {
; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
-; CHECK-NEXT: ptrue p0.b, vl32
-; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: uqsub z1.b, z1.b, z0.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: and z1.b, z1.b, #0x1
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: sdot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
%1 = icmp ult <32 x i8> %a, %b
@@ -123,10 +115,9 @@ define <vscale x 4 x i32> @test6(<32 x i8> %a, <32 x i8> %b) {
define <vscale x 4 x i32> @test7(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
; CHECK-LABEL: test7:
; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: uqsub z1.b, z0.b, z1.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: sdot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
{
@@ -139,10 +130,9 @@ define <vscale x 4 x i32> @test7(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
define <vscale x 4 x i32> @test8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
; CHECK-LABEL: test8:
; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: uqsub z1.b, z0.b, z1.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #1 // =0x1
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: sdot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
{
@@ -155,10 +145,9 @@ define <vscale x 4 x i32> @test8(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
define <vscale x 4 x i32> @test9(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
; CHECK-LABEL: test9:
; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: uqsub z1.b, z0.b, z1.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #1 // =0x1
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: sdot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
{
@@ -178,10 +167,9 @@ define <vscale x 4 x i32> @test10(<32 x i8> %a, <32 x i8> %b) {
; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
; CHECK-NEXT: splice z2.b, p0, { z2.b, z3.b }
; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
-; CHECK-NEXT: ptrue p0.b, vl32
-; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z2.b
+; CHECK-NEXT: uqsub z1.b, z0.b, z2.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: sdot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
%1 = icmp ugt <32 x i8> %a, %b
@@ -201,11 +189,9 @@ define <vscale x 4 x i32> @test11(<32 x i8> %a, <32 x i8> %b) {
; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
; CHECK-NEXT: splice z2.b, p0, { z2.b, z3.b }
; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
-; CHECK-NEXT: ptrue p0.b, vl32
-; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z2.b
+; CHECK-NEXT: uqsub z1.b, z0.b, z2.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: and z1.b, z1.b, #0x1
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: sdot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
%1 = icmp ugt <32 x i8> %a, %b
@@ -225,11 +211,9 @@ define <vscale x 4 x i32> @test12(<32 x i8> %a, <32 x i8> %b) {
; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
; CHECK-NEXT: splice z2.b, p0, { z2.b, z3.b }
; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
-; CHECK-NEXT: ptrue p0.b, vl32
-; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z2.b
+; CHECK-NEXT: uqsub z1.b, z0.b, z2.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: and z1.b, z1.b, #0x1
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: sdot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
%1 = icmp ugt <32 x i8> %a, %b
@@ -242,9 +226,10 @@ define <vscale x 4 x i32> @test12(<32 x i8> %a, <32 x i8> %b) {
define <vscale x 16 x i8> @test13(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1) {
; CHECK-LABEL: test13:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: cmpgt p0.b, p0/z, z1.b, z0.b
-; CHECK-NEXT: mov z0.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: movi v2.2d, #0000000000000000
+; CHECK-NEXT: sqsub z0.b, z0.b, z1.b
+; CHECK-NEXT: mov z1.b, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: sclamp z0.b, z1.b, z2.b
; CHECK-NEXT: ret
entry:
%2 = icmp slt <vscale x 16 x i8> %0, %1
@@ -255,16 +240,17 @@ entry:
define <vscale x 16 x i8> @test14(<32 x i8> %0, <32 x i8> %1) {
; CHECK-LABEL: test14:
; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
; CHECK-NEXT: ptrue p0.b, vl16
-; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
-; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: splice z2.b, p0, { z2.b, z3.b }
; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
-; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
-; CHECK-NEXT: ptrue p0.b, vl32
-; CHECK-NEXT: cmpgt p0.b, p0/z, z1.b, z0.b
-; CHECK-NEXT: mov z0.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: sqsub z0.b, z0.b, z2.b
+; CHECK-NEXT: mov z2.b, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: sclamp z0.b, z2.b, z1.b
; CHECK-NEXT: ret
entry:
%2 = icmp slt <32 x i8> %0, %1
@@ -278,10 +264,10 @@ define <vscale x 16 x i8> @test15(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1)
; CHECK-LABEL: test15:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: cmpgt p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: cmpgt p1.b, p0/z, z1.b, z0.b
; CHECK-NEXT: mov z1.b, #1 // =0x1
-; CHECK-NEXT: mov z0.b, p0/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: add z0.b, p0/m, z0.b, z1.b
+; CHECK-NEXT: mov z0.b, p1/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: add z0.b, p1/m, z0.b, z1.b
; CHECK-NEXT: ret
entry:
%2 = icmp slt <vscale x 16 x i8> %0, %1
@@ -294,9 +280,10 @@ entry:
define <vscale x 16 x i8> @test16(<vscale x 16 x i8> %0, <vscale x 16 x i8> %1) {
; CHECK-LABEL: test16:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: cmpgt p0.b, p0/z, z0.b, z1.b
-; CHECK-NEXT: mov z0.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: movi v2.2d, #0000000000000000
+; CHECK-NEXT: sqsub z0.b, z1.b, z0.b
+; CHECK-NEXT: mov z1.b, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: sclamp z0.b, z1.b, z2.b
; CHECK-NEXT: ret
entry:
%2 = icmp sgt <vscale x 16 x i8> %0, %1
@@ -307,16 +294,17 @@ entry:
define <vscale x 16 x i8> @test17(<32 x i8> %0, <32 x i8> %1) {
; CHECK-LABEL: test17:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
; CHECK-NEXT: // kill: def $q1 killed $q1 killed $z0_z1 def $z0_z1
; CHECK-NEXT: ptrue p0.b, vl16
-; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
+; CHECK-NEXT: // kill: def $q3 killed $q3 killed $z2_z3 def $z2_z3
; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
-; CHECK-NEXT: splice z2.b, p0, { z2.b, z3.b }
+; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
-; CHECK-NEXT: ptrue p0.b, vl32
-; CHECK-NEXT: cmpgt p0.b, p0/z, z0.b, z2.b
-; CHECK-NEXT: mov z0.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
+; CHECK-NEXT: mov z2.b, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: sqsub z0.b, z1.b, z0.b
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: sclamp z0.b, z2.b, z1.b
; CHECK-NEXT: ret
entry:
%2 = icmp sgt <32 x i8> %0, %1
@@ -328,10 +316,9 @@ entry:
define <vscale x 4 x i32> @test18(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
; CHECK-LABEL: test18:
; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: uqsub z1.b, z1.b, z0.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: udot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
{
@@ -344,10 +331,9 @@ define <vscale x 4 x i32> @test18(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
define <vscale x 4 x i32> @test19(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
; CHECK-LABEL: test19:
; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: uqsub z1.b, z1.b, z0.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #1 // =0x1
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: udot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
{
@@ -361,10 +347,9 @@ define <vscale x 4 x i32> @test19(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
define <vscale x 4 x i32> @test20(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
; CHECK-LABEL: test20:
; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: uqsub z1.b, z1.b, z0.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #1 // =0x1
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: udot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
{
@@ -384,10 +369,9 @@ define <vscale x 4 x i32> @test21(<32 x i8> %a, <32 x i8> %b) {
; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
-; CHECK-NEXT: ptrue p0.b, vl32
-; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: uqsub z1.b, z1.b, z0.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: udot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
%1 = icmp ult <32 x i8> %a, %b
@@ -407,11 +391,9 @@ define <vscale x 4 x i32> @test22(<32 x i8> %a, <32 x i8> %b) {
; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
-; CHECK-NEXT: ptrue p0.b, vl32
-; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: uqsub z1.b, z1.b, z0.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: and z1.b, z1.b, #0x1
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: udot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
%1 = icmp ult <32 x i8> %a, %b
@@ -431,11 +413,9 @@ define <vscale x 4 x i32> @test23(<32 x i8> %a, <32 x i8> %b) {
; CHECK-NEXT: // kill: def $q2 killed $q2 killed $z2_z3 def $z2_z3
; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
; CHECK-NEXT: splice z1.b, p0, { z2.b, z3.b }
-; CHECK-NEXT: ptrue p0.b, vl32
-; CHECK-NEXT: cmphi p0.b, p0/z, z1.b, z0.b
+; CHECK-NEXT: uqsub z1.b, z1.b, z0.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: and z1.b, z1.b, #0x1
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: udot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
%1 = icmp ult <32 x i8> %a, %b
@@ -448,10 +428,9 @@ define <vscale x 4 x i32> @test23(<32 x i8> %a, <32 x i8> %b) {
define <vscale x 4 x i32> @test24(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
; CHECK-LABEL: test24:
; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: uqsub z1.b, z0.b, z1.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: udot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
{
@@ -464,10 +443,9 @@ define <vscale x 4 x i32> @test24(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
define <vscale x 4 x i32> @test25(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
; CHECK-LABEL: test25:
; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: uqsub z1.b, z0.b, z1.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #1 // =0x1
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: udot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
{
@@ -480,10 +458,9 @@ define <vscale x 4 x i32> @test25(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
define <vscale x 4 x i32> @test26(<vscale x 16 x i8> %a, <vscale x 16 x i8> %b)
; CHECK-LABEL: test26:
; CHECK: // %bb.0:
-; CHECK-NEXT: ptrue p0.b
-; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z1.b
+; CHECK-NEXT: uqsub z1.b, z0.b, z1.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #1 // =0x1
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: udot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
{
@@ -503,10 +480,9 @@ define <vscale x 4 x i32> @test27(<32 x i8> %a, <32 x i8> %b) {
; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
; CHECK-NEXT: splice z2.b, p0, { z2.b, z3.b }
; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
-; CHECK-NEXT: ptrue p0.b, vl32
-; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z2.b
+; CHECK-NEXT: uqsub z1.b, z0.b, z2.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: udot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
%1 = icmp ugt <32 x i8> %a, %b
@@ -526,11 +502,9 @@ define <vscale x 4 x i32> @test28(<32 x i8> %a, <32 x i8> %b) {
; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
; CHECK-NEXT: splice z2.b, p0, { z2.b, z3.b }
; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
-; CHECK-NEXT: ptrue p0.b, vl32
-; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z2.b
+; CHECK-NEXT: uqsub z1.b, z0.b, z2.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: and z1.b, z1.b, #0x1
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: udot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
%1 = icmp ugt <32 x i8> %a, %b
@@ -550,11 +524,9 @@ define <vscale x 4 x i32> @test29(<32 x i8> %a, <32 x i8> %b) {
; CHECK-NEXT: // kill: def $q0 killed $q0 killed $z0_z1 def $z0_z1
; CHECK-NEXT: splice z2.b, p0, { z2.b, z3.b }
; CHECK-NEXT: splice z0.b, p0, { z0.b, z1.b }
-; CHECK-NEXT: ptrue p0.b, vl32
-; CHECK-NEXT: cmphi p0.b, p0/z, z0.b, z2.b
+; CHECK-NEXT: uqsub z1.b, z0.b, z2.b
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: mov z1.b, p0/z, #-1 // =0xffffffffffffffff
-; CHECK-NEXT: and z1.b, z1.b, #0x1
+; CHECK-NEXT: umin z1.b, z1.b, #1
; CHECK-NEXT: udot z0.s, z1.b, z1.b
; CHECK-NEXT: ret
%1 = icmp ugt <32 x i8> %a, %b
More information about the llvm-commits
mailing list