[llvm] 160abfb - [LLVM][CodeGen][SVE] Add ISel for bfloat scalable vector compares. (#138707)
via llvm-commits
llvm-commits at lists.llvm.org
Thu May 8 02:57:18 PDT 2025
Author: Paul Walker
Date: 2025-05-08T10:57:15+01:00
New Revision: 160abfb5e623a67014de311a6b50f12e4104f7c9
URL: https://github.com/llvm/llvm-project/commit/160abfb5e623a67014de311a6b50f12e4104f7c9
DIFF: https://github.com/llvm/llvm-project/commit/160abfb5e623a67014de311a6b50f12e4104f7c9.diff
LOG: [LLVM][CodeGen][SVE] Add ISel for bfloat scalable vector compares. (#138707)
Added:
llvm/test/CodeGen/AArch64/sve-bf16-compares.ll
Modified:
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 6e7f13c20db68..9c25ff0d6afc0 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1758,10 +1758,10 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
for (auto Opcode :
{ISD::FCEIL, ISD::FDIV, ISD::FFLOOR, ISD::FNEARBYINT, ISD::FRINT,
- ISD::FROUND, ISD::FROUNDEVEN, ISD::FSQRT, ISD::FTRUNC}) {
+ ISD::FROUND, ISD::FROUNDEVEN, ISD::FSQRT, ISD::FTRUNC, ISD::SETCC}) {
setOperationPromotedToType(Opcode, MVT::nxv2bf16, MVT::nxv2f32);
setOperationPromotedToType(Opcode, MVT::nxv4bf16, MVT::nxv4f32);
- setOperationAction(Opcode, MVT::nxv8bf16, Expand);
+ setOperationPromotedToType(Opcode, MVT::nxv8bf16, MVT::nxv8f32);
}
if (!Subtarget->hasSVEB16B16()) {
@@ -1769,7 +1769,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
ISD::FMINIMUM, ISD::FMINNUM, ISD::FMUL, ISD::FSUB}) {
setOperationPromotedToType(Opcode, MVT::nxv2bf16, MVT::nxv2f32);
setOperationPromotedToType(Opcode, MVT::nxv4bf16, MVT::nxv4f32);
- setOperationAction(Opcode, MVT::nxv8bf16, Expand);
+ setOperationPromotedToType(Opcode, MVT::nxv8bf16, MVT::nxv8f32);
}
}
diff --git a/llvm/test/CodeGen/AArch64/sve-bf16-compares.ll b/llvm/test/CodeGen/AArch64/sve-bf16-compares.ll
new file mode 100644
index 0000000000000..e0bf755c851b7
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-bf16-compares.ll
@@ -0,0 +1,1001 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --extra_scrub
+; RUN: llc -mattr=+sve < %s | FileCheck %s
+; RUN: llc -mattr=+sme -force-streaming < %s | FileCheck %s
+
+target triple = "aarch64-unknown-linux-gnu"
+
+;
+; OEQ
+;
+
+define <vscale x 2 x i1> @fcmp_oeq_nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) {
+; CHECK-LABEL: fcmp_oeq_nxv2bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: fcmeq p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: ret
+ %res = fcmp oeq <vscale x 2 x bfloat> %a, %b
+ ret <vscale x 2 x i1> %res
+}
+
+define <vscale x 4 x i1> @fcmp_oeq_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) {
+; CHECK-LABEL: fcmp_oeq_nxv4bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: fcmeq p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: ret
+ %res = fcmp oeq <vscale x 4 x bfloat> %a, %b
+ ret <vscale x 4 x i1> %res
+}
+
+define <vscale x 8 x i1> @fcmp_oeq_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
+; CHECK-LABEL: fcmp_oeq_nxv8bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uunpkhi z2.s, z1.h
+; CHECK-NEXT: uunpkhi z3.s, z0.h
+; CHECK-NEXT: uunpklo z1.s, z1.h
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: lsl z2.s, z2.s, #16
+; CHECK-NEXT: lsl z3.s, z3.s, #16
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: fcmeq p1.s, p0/z, z3.s, z2.s
+; CHECK-NEXT: fcmeq p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h
+; CHECK-NEXT: ret
+ %res = fcmp oeq <vscale x 8 x bfloat> %a, %b
+ ret <vscale x 8 x i1> %res
+}
+
+;
+; OGT
+;
+
+define <vscale x 2 x i1> @fcmp_ogt_nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) {
+; CHECK-LABEL: fcmp_ogt_nxv2bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: fcmgt p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: ret
+ %res = fcmp ogt <vscale x 2 x bfloat> %a, %b
+ ret <vscale x 2 x i1> %res
+}
+
+define <vscale x 4 x i1> @fcmp_ogt_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) {
+; CHECK-LABEL: fcmp_ogt_nxv4bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: fcmgt p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: ret
+ %res = fcmp ogt <vscale x 4 x bfloat> %a, %b
+ ret <vscale x 4 x i1> %res
+}
+
+define <vscale x 8 x i1> @fcmp_ogt_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
+; CHECK-LABEL: fcmp_ogt_nxv8bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uunpkhi z2.s, z1.h
+; CHECK-NEXT: uunpkhi z3.s, z0.h
+; CHECK-NEXT: uunpklo z1.s, z1.h
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: lsl z2.s, z2.s, #16
+; CHECK-NEXT: lsl z3.s, z3.s, #16
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: fcmgt p1.s, p0/z, z3.s, z2.s
+; CHECK-NEXT: fcmgt p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h
+; CHECK-NEXT: ret
+ %res = fcmp ogt <vscale x 8 x bfloat> %a, %b
+ ret <vscale x 8 x i1> %res
+}
+
+;
+; OGE
+;
+
+define <vscale x 2 x i1> @fcmp_oge_nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) {
+; CHECK-LABEL: fcmp_oge_nxv2bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: fcmge p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: ret
+ %res = fcmp oge <vscale x 2 x bfloat> %a, %b
+ ret <vscale x 2 x i1> %res
+}
+
+define <vscale x 4 x i1> @fcmp_oge_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) {
+; CHECK-LABEL: fcmp_oge_nxv4bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: fcmge p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: ret
+ %res = fcmp oge <vscale x 4 x bfloat> %a, %b
+ ret <vscale x 4 x i1> %res
+}
+
+define <vscale x 8 x i1> @fcmp_oge_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
+; CHECK-LABEL: fcmp_oge_nxv8bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uunpkhi z2.s, z1.h
+; CHECK-NEXT: uunpkhi z3.s, z0.h
+; CHECK-NEXT: uunpklo z1.s, z1.h
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: lsl z2.s, z2.s, #16
+; CHECK-NEXT: lsl z3.s, z3.s, #16
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: fcmge p1.s, p0/z, z3.s, z2.s
+; CHECK-NEXT: fcmge p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h
+; CHECK-NEXT: ret
+ %res = fcmp oge <vscale x 8 x bfloat> %a, %b
+ ret <vscale x 8 x i1> %res
+}
+
+;
+; OLT
+;
+
+define <vscale x 2 x i1> @fcmp_olt_nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) {
+; CHECK-LABEL: fcmp_olt_nxv2bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: fcmgt p0.s, p0/z, z1.s, z0.s
+; CHECK-NEXT: ret
+ %res = fcmp olt <vscale x 2 x bfloat> %a, %b
+ ret <vscale x 2 x i1> %res
+}
+
+define <vscale x 4 x i1> @fcmp_olt_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) {
+; CHECK-LABEL: fcmp_olt_nxv4bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: fcmgt p0.s, p0/z, z1.s, z0.s
+; CHECK-NEXT: ret
+ %res = fcmp olt <vscale x 4 x bfloat> %a, %b
+ ret <vscale x 4 x i1> %res
+}
+
+define <vscale x 8 x i1> @fcmp_olt_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
+; CHECK-LABEL: fcmp_olt_nxv8bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uunpkhi z2.s, z0.h
+; CHECK-NEXT: uunpkhi z3.s, z1.h
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: uunpklo z1.s, z1.h
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: lsl z2.s, z2.s, #16
+; CHECK-NEXT: lsl z3.s, z3.s, #16
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: fcmgt p1.s, p0/z, z3.s, z2.s
+; CHECK-NEXT: fcmgt p0.s, p0/z, z1.s, z0.s
+; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h
+; CHECK-NEXT: ret
+ %res = fcmp olt <vscale x 8 x bfloat> %a, %b
+ ret <vscale x 8 x i1> %res
+}
+
+;
+; OLE
+;
+
+define <vscale x 2 x i1> @fcmp_ole_nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) {
+; CHECK-LABEL: fcmp_ole_nxv2bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: fcmge p0.s, p0/z, z1.s, z0.s
+; CHECK-NEXT: ret
+ %res = fcmp ole <vscale x 2 x bfloat> %a, %b
+ ret <vscale x 2 x i1> %res
+}
+
+define <vscale x 4 x i1> @fcmp_ole_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) {
+; CHECK-LABEL: fcmp_ole_nxv4bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: fcmge p0.s, p0/z, z1.s, z0.s
+; CHECK-NEXT: ret
+ %res = fcmp ole <vscale x 4 x bfloat> %a, %b
+ ret <vscale x 4 x i1> %res
+}
+
+define <vscale x 8 x i1> @fcmp_ole_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
+; CHECK-LABEL: fcmp_ole_nxv8bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uunpkhi z2.s, z0.h
+; CHECK-NEXT: uunpkhi z3.s, z1.h
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: uunpklo z1.s, z1.h
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: lsl z2.s, z2.s, #16
+; CHECK-NEXT: lsl z3.s, z3.s, #16
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: fcmge p1.s, p0/z, z3.s, z2.s
+; CHECK-NEXT: fcmge p0.s, p0/z, z1.s, z0.s
+; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h
+; CHECK-NEXT: ret
+ %res = fcmp ole <vscale x 8 x bfloat> %a, %b
+ ret <vscale x 8 x i1> %res
+}
+
+;
+; ONE
+;
+
+define <vscale x 2 x i1> @fcmp_one_nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) {
+; CHECK-LABEL: fcmp_one_nxv2bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: fcmgt p1.s, p0/z, z1.s, z0.s
+; CHECK-NEXT: fcmgt p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: sel p0.b, p0, p0.b, p1.b
+; CHECK-NEXT: ret
+ %res = fcmp one <vscale x 2 x bfloat> %a, %b
+ ret <vscale x 2 x i1> %res
+}
+
+define <vscale x 4 x i1> @fcmp_one_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) {
+; CHECK-LABEL: fcmp_one_nxv4bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: fcmgt p1.s, p0/z, z1.s, z0.s
+; CHECK-NEXT: fcmgt p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: sel p0.b, p0, p0.b, p1.b
+; CHECK-NEXT: ret
+ %res = fcmp one <vscale x 4 x bfloat> %a, %b
+ ret <vscale x 4 x i1> %res
+}
+
+define <vscale x 8 x i1> @fcmp_one_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
+; CHECK-LABEL: fcmp_one_nxv8bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uunpkhi z2.s, z0.h
+; CHECK-NEXT: uunpkhi z3.s, z1.h
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: uunpklo z1.s, z1.h
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: lsl z2.s, z2.s, #16
+; CHECK-NEXT: lsl z3.s, z3.s, #16
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: fcmgt p1.s, p0/z, z3.s, z2.s
+; CHECK-NEXT: fcmgt p2.s, p0/z, z2.s, z3.s
+; CHECK-NEXT: fcmgt p3.s, p0/z, z1.s, z0.s
+; CHECK-NEXT: fcmgt p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: mov p1.b, p2/m, p2.b
+; CHECK-NEXT: sel p0.b, p0, p0.b, p3.b
+; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h
+; CHECK-NEXT: ret
+ %res = fcmp one <vscale x 8 x bfloat> %a, %b
+ ret <vscale x 8 x i1> %res
+}
+
+;
+; ORD
+;
+
+define <vscale x 2 x i1> @fcmp_ord_nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) {
+; CHECK-LABEL: fcmp_ord_nxv2bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: fcmuo p1.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: not p0.b, p0/z, p1.b
+; CHECK-NEXT: ret
+ %res = fcmp ord <vscale x 2 x bfloat> %a, %b
+ ret <vscale x 2 x i1> %res
+}
+
+define <vscale x 4 x i1> @fcmp_ord_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) {
+; CHECK-LABEL: fcmp_ord_nxv4bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: fcmuo p1.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: not p0.b, p0/z, p1.b
+; CHECK-NEXT: ret
+ %res = fcmp ord <vscale x 4 x bfloat> %a, %b
+ ret <vscale x 4 x i1> %res
+}
+
+define <vscale x 8 x i1> @fcmp_ord_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
+; CHECK-LABEL: fcmp_ord_nxv8bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uunpkhi z2.s, z1.h
+; CHECK-NEXT: uunpkhi z3.s, z0.h
+; CHECK-NEXT: uunpklo z1.s, z1.h
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: lsl z2.s, z2.s, #16
+; CHECK-NEXT: lsl z3.s, z3.s, #16
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: fcmuo p1.s, p0/z, z3.s, z2.s
+; CHECK-NEXT: fcmuo p2.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: not p1.b, p0/z, p1.b
+; CHECK-NEXT: not p0.b, p0/z, p2.b
+; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h
+; CHECK-NEXT: ret
+ %res = fcmp ord <vscale x 8 x bfloat> %a, %b
+ ret <vscale x 8 x i1> %res
+}
+
+;
+; UEQ
+;
+
+define <vscale x 2 x i1> @fcmp_ueq_nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) {
+; CHECK-LABEL: fcmp_ueq_nxv2bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: fcmuo p1.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: fcmeq p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: sel p0.b, p0, p0.b, p1.b
+; CHECK-NEXT: ret
+ %res = fcmp ueq <vscale x 2 x bfloat> %a, %b
+ ret <vscale x 2 x i1> %res
+}
+
+define <vscale x 4 x i1> @fcmp_ueq_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) {
+; CHECK-LABEL: fcmp_ueq_nxv4bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: fcmuo p1.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: fcmeq p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: sel p0.b, p0, p0.b, p1.b
+; CHECK-NEXT: ret
+ %res = fcmp ueq <vscale x 4 x bfloat> %a, %b
+ ret <vscale x 4 x i1> %res
+}
+
+define <vscale x 8 x i1> @fcmp_ueq_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
+; CHECK-LABEL: fcmp_ueq_nxv8bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uunpkhi z2.s, z1.h
+; CHECK-NEXT: uunpkhi z3.s, z0.h
+; CHECK-NEXT: uunpklo z1.s, z1.h
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: lsl z2.s, z2.s, #16
+; CHECK-NEXT: lsl z3.s, z3.s, #16
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: fcmuo p1.s, p0/z, z3.s, z2.s
+; CHECK-NEXT: fcmeq p2.s, p0/z, z3.s, z2.s
+; CHECK-NEXT: fcmuo p3.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: fcmeq p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: mov p1.b, p2/m, p2.b
+; CHECK-NEXT: sel p0.b, p0, p0.b, p3.b
+; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h
+; CHECK-NEXT: ret
+ %res = fcmp ueq <vscale x 8 x bfloat> %a, %b
+ ret <vscale x 8 x i1> %res
+}
+
+;
+; UGT
+;
+
+define <vscale x 2 x i1> @fcmp_ugt_nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) {
+; CHECK-LABEL: fcmp_ugt_nxv2bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: fcmge p1.s, p0/z, z1.s, z0.s
+; CHECK-NEXT: not p0.b, p0/z, p1.b
+; CHECK-NEXT: ret
+ %res = fcmp ugt <vscale x 2 x bfloat> %a, %b
+ ret <vscale x 2 x i1> %res
+}
+
+define <vscale x 4 x i1> @fcmp_ugt_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) {
+; CHECK-LABEL: fcmp_ugt_nxv4bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: fcmge p1.s, p0/z, z1.s, z0.s
+; CHECK-NEXT: not p0.b, p0/z, p1.b
+; CHECK-NEXT: ret
+ %res = fcmp ugt <vscale x 4 x bfloat> %a, %b
+ ret <vscale x 4 x i1> %res
+}
+
+define <vscale x 8 x i1> @fcmp_ugt_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
+; CHECK-LABEL: fcmp_ugt_nxv8bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uunpkhi z2.s, z0.h
+; CHECK-NEXT: uunpkhi z3.s, z1.h
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: uunpklo z1.s, z1.h
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: lsl z2.s, z2.s, #16
+; CHECK-NEXT: lsl z3.s, z3.s, #16
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: fcmge p1.s, p0/z, z3.s, z2.s
+; CHECK-NEXT: fcmge p2.s, p0/z, z1.s, z0.s
+; CHECK-NEXT: not p1.b, p0/z, p1.b
+; CHECK-NEXT: not p0.b, p0/z, p2.b
+; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h
+; CHECK-NEXT: ret
+ %res = fcmp ugt <vscale x 8 x bfloat> %a, %b
+ ret <vscale x 8 x i1> %res
+}
+
+;
+; UGE
+;
+
+define <vscale x 2 x i1> @fcmp_uge_nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) {
+; CHECK-LABEL: fcmp_uge_nxv2bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: fcmgt p1.s, p0/z, z1.s, z0.s
+; CHECK-NEXT: not p0.b, p0/z, p1.b
+; CHECK-NEXT: ret
+ %res = fcmp uge <vscale x 2 x bfloat> %a, %b
+ ret <vscale x 2 x i1> %res
+}
+
+define <vscale x 4 x i1> @fcmp_uge_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) {
+; CHECK-LABEL: fcmp_uge_nxv4bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: fcmgt p1.s, p0/z, z1.s, z0.s
+; CHECK-NEXT: not p0.b, p0/z, p1.b
+; CHECK-NEXT: ret
+ %res = fcmp uge <vscale x 4 x bfloat> %a, %b
+ ret <vscale x 4 x i1> %res
+}
+
+define <vscale x 8 x i1> @fcmp_uge_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
+; CHECK-LABEL: fcmp_uge_nxv8bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uunpkhi z2.s, z0.h
+; CHECK-NEXT: uunpkhi z3.s, z1.h
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: uunpklo z1.s, z1.h
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: lsl z2.s, z2.s, #16
+; CHECK-NEXT: lsl z3.s, z3.s, #16
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: fcmgt p1.s, p0/z, z3.s, z2.s
+; CHECK-NEXT: fcmgt p2.s, p0/z, z1.s, z0.s
+; CHECK-NEXT: not p1.b, p0/z, p1.b
+; CHECK-NEXT: not p0.b, p0/z, p2.b
+; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h
+; CHECK-NEXT: ret
+ %res = fcmp uge <vscale x 8 x bfloat> %a, %b
+ ret <vscale x 8 x i1> %res
+}
+
+;
+; ULT
+;
+
+define <vscale x 2 x i1> @fcmp_ult_nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) {
+; CHECK-LABEL: fcmp_ult_nxv2bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: fcmge p1.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: not p0.b, p0/z, p1.b
+; CHECK-NEXT: ret
+ %res = fcmp ult <vscale x 2 x bfloat> %a, %b
+ ret <vscale x 2 x i1> %res
+}
+
+define <vscale x 4 x i1> @fcmp_ult_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) {
+; CHECK-LABEL: fcmp_ult_nxv4bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: fcmge p1.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: not p0.b, p0/z, p1.b
+; CHECK-NEXT: ret
+ %res = fcmp ult <vscale x 4 x bfloat> %a, %b
+ ret <vscale x 4 x i1> %res
+}
+
+define <vscale x 8 x i1> @fcmp_ult_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
+; CHECK-LABEL: fcmp_ult_nxv8bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uunpkhi z2.s, z1.h
+; CHECK-NEXT: uunpkhi z3.s, z0.h
+; CHECK-NEXT: uunpklo z1.s, z1.h
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: lsl z2.s, z2.s, #16
+; CHECK-NEXT: lsl z3.s, z3.s, #16
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: fcmge p1.s, p0/z, z3.s, z2.s
+; CHECK-NEXT: fcmge p2.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: not p1.b, p0/z, p1.b
+; CHECK-NEXT: not p0.b, p0/z, p2.b
+; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h
+; CHECK-NEXT: ret
+ %res = fcmp ult <vscale x 8 x bfloat> %a, %b
+ ret <vscale x 8 x i1> %res
+}
+
+;
+; ULE
+;
+
+define <vscale x 2 x i1> @fcmp_ule_nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) {
+; CHECK-LABEL: fcmp_ule_nxv2bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: fcmgt p1.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: not p0.b, p0/z, p1.b
+; CHECK-NEXT: ret
+ %res = fcmp ule <vscale x 2 x bfloat> %a, %b
+ ret <vscale x 2 x i1> %res
+}
+
+define <vscale x 4 x i1> @fcmp_ule_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) {
+; CHECK-LABEL: fcmp_ule_nxv4bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: fcmgt p1.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: not p0.b, p0/z, p1.b
+; CHECK-NEXT: ret
+ %res = fcmp ule <vscale x 4 x bfloat> %a, %b
+ ret <vscale x 4 x i1> %res
+}
+
+define <vscale x 8 x i1> @fcmp_ule_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
+; CHECK-LABEL: fcmp_ule_nxv8bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uunpkhi z2.s, z1.h
+; CHECK-NEXT: uunpkhi z3.s, z0.h
+; CHECK-NEXT: uunpklo z1.s, z1.h
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: lsl z2.s, z2.s, #16
+; CHECK-NEXT: lsl z3.s, z3.s, #16
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: fcmgt p1.s, p0/z, z3.s, z2.s
+; CHECK-NEXT: fcmgt p2.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: not p1.b, p0/z, p1.b
+; CHECK-NEXT: not p0.b, p0/z, p2.b
+; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h
+; CHECK-NEXT: ret
+ %res = fcmp ule <vscale x 8 x bfloat> %a, %b
+ ret <vscale x 8 x i1> %res
+}
+
+;
+; UNE
+;
+
+define <vscale x 2 x i1> @fcmp_une_nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) {
+; CHECK-LABEL: fcmp_une_nxv2bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: fcmne p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: ret
+ %res = fcmp une <vscale x 2 x bfloat> %a, %b
+ ret <vscale x 2 x i1> %res
+}
+
+define <vscale x 4 x i1> @fcmp_une_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) {
+; CHECK-LABEL: fcmp_une_nxv4bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: fcmne p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: ret
+ %res = fcmp une <vscale x 4 x bfloat> %a, %b
+ ret <vscale x 4 x i1> %res
+}
+
+define <vscale x 8 x i1> @fcmp_une_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
+; CHECK-LABEL: fcmp_une_nxv8bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uunpkhi z2.s, z1.h
+; CHECK-NEXT: uunpkhi z3.s, z0.h
+; CHECK-NEXT: uunpklo z1.s, z1.h
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: lsl z2.s, z2.s, #16
+; CHECK-NEXT: lsl z3.s, z3.s, #16
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: fcmne p1.s, p0/z, z3.s, z2.s
+; CHECK-NEXT: fcmne p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h
+; CHECK-NEXT: ret
+ %res = fcmp une <vscale x 8 x bfloat> %a, %b
+ ret <vscale x 8 x i1> %res
+}
+
+;
+; UNO
+;
+
+define <vscale x 2 x i1> @fcmp_uno_nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) {
+; CHECK-LABEL: fcmp_uno_nxv2bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: fcmuo p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: ret
+ %res = fcmp uno <vscale x 2 x bfloat> %a, %b
+ ret <vscale x 2 x i1> %res
+}
+
+define <vscale x 4 x i1> @fcmp_uno_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) {
+; CHECK-LABEL: fcmp_uno_nxv4bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: fcmuo p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: ret
+ %res = fcmp uno <vscale x 4 x bfloat> %a, %b
+ ret <vscale x 4 x i1> %res
+}
+
+define <vscale x 8 x i1> @fcmp_uno_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
+; CHECK-LABEL: fcmp_uno_nxv8bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uunpkhi z2.s, z1.h
+; CHECK-NEXT: uunpkhi z3.s, z0.h
+; CHECK-NEXT: uunpklo z1.s, z1.h
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: lsl z2.s, z2.s, #16
+; CHECK-NEXT: lsl z3.s, z3.s, #16
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: fcmuo p1.s, p0/z, z3.s, z2.s
+; CHECK-NEXT: fcmuo p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h
+; CHECK-NEXT: ret
+ %res = fcmp uno <vscale x 8 x bfloat> %a, %b
+ ret <vscale x 8 x i1> %res
+}
+
+;
+; EQ
+;
+
+define <vscale x 2 x i1> @fcmp_eq_nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) {
+; CHECK-LABEL: fcmp_eq_nxv2bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: fcmeq p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: ret
+ %res = fcmp fast oeq <vscale x 2 x bfloat> %a, %b
+ ret <vscale x 2 x i1> %res
+}
+
+define <vscale x 4 x i1> @fcmp_eq_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) {
+; CHECK-LABEL: fcmp_eq_nxv4bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: fcmeq p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: ret
+ %res = fcmp fast oeq <vscale x 4 x bfloat> %a, %b
+ ret <vscale x 4 x i1> %res
+}
+
+define <vscale x 8 x i1> @fcmp_eq_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
+; CHECK-LABEL: fcmp_eq_nxv8bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uunpkhi z2.s, z1.h
+; CHECK-NEXT: uunpkhi z3.s, z0.h
+; CHECK-NEXT: uunpklo z1.s, z1.h
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: lsl z2.s, z2.s, #16
+; CHECK-NEXT: lsl z3.s, z3.s, #16
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: fcmeq p1.s, p0/z, z3.s, z2.s
+; CHECK-NEXT: fcmeq p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h
+; CHECK-NEXT: ret
+ %res = fcmp fast oeq <vscale x 8 x bfloat> %a, %b
+ ret <vscale x 8 x i1> %res
+}
+
+;
+; GT
+;
+
+define <vscale x 2 x i1> @fcmp_gt_nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) {
+; CHECK-LABEL: fcmp_gt_nxv2bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: fcmgt p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: ret
+ %res = fcmp fast ogt <vscale x 2 x bfloat> %a, %b
+ ret <vscale x 2 x i1> %res
+}
+
+define <vscale x 4 x i1> @fcmp_gt_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) {
+; CHECK-LABEL: fcmp_gt_nxv4bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: fcmgt p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: ret
+ %res = fcmp fast ogt <vscale x 4 x bfloat> %a, %b
+ ret <vscale x 4 x i1> %res
+}
+
+define <vscale x 8 x i1> @fcmp_gt_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
+; CHECK-LABEL: fcmp_gt_nxv8bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uunpkhi z2.s, z1.h
+; CHECK-NEXT: uunpkhi z3.s, z0.h
+; CHECK-NEXT: uunpklo z1.s, z1.h
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: lsl z2.s, z2.s, #16
+; CHECK-NEXT: lsl z3.s, z3.s, #16
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: fcmgt p1.s, p0/z, z3.s, z2.s
+; CHECK-NEXT: fcmgt p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h
+; CHECK-NEXT: ret
+ %res = fcmp fast ogt <vscale x 8 x bfloat> %a, %b
+ ret <vscale x 8 x i1> %res
+}
+
+;
+; GE
+;
+
+define <vscale x 2 x i1> @fcmp_ge_nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) {
+; CHECK-LABEL: fcmp_ge_nxv2bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: fcmge p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: ret
+ %res = fcmp fast oge <vscale x 2 x bfloat> %a, %b
+ ret <vscale x 2 x i1> %res
+}
+
+define <vscale x 4 x i1> @fcmp_ge_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) {
+; CHECK-LABEL: fcmp_ge_nxv4bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: fcmge p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: ret
+ %res = fcmp fast oge <vscale x 4 x bfloat> %a, %b
+ ret <vscale x 4 x i1> %res
+}
+
+define <vscale x 8 x i1> @fcmp_ge_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
+; CHECK-LABEL: fcmp_ge_nxv8bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uunpkhi z2.s, z1.h
+; CHECK-NEXT: uunpkhi z3.s, z0.h
+; CHECK-NEXT: uunpklo z1.s, z1.h
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: lsl z2.s, z2.s, #16
+; CHECK-NEXT: lsl z3.s, z3.s, #16
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: fcmge p1.s, p0/z, z3.s, z2.s
+; CHECK-NEXT: fcmge p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h
+; CHECK-NEXT: ret
+ %res = fcmp fast oge <vscale x 8 x bfloat> %a, %b
+ ret <vscale x 8 x i1> %res
+}
+
+;
+; LT
+;
+
+define <vscale x 2 x i1> @fcmp_lt_nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) {
+; CHECK-LABEL: fcmp_lt_nxv2bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: fcmgt p0.s, p0/z, z1.s, z0.s
+; CHECK-NEXT: ret
+ %res = fcmp fast olt <vscale x 2 x bfloat> %a, %b
+ ret <vscale x 2 x i1> %res
+}
+
+define <vscale x 4 x i1> @fcmp_lt_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) {
+; CHECK-LABEL: fcmp_lt_nxv4bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: fcmgt p0.s, p0/z, z1.s, z0.s
+; CHECK-NEXT: ret
+ %res = fcmp fast olt <vscale x 4 x bfloat> %a, %b
+ ret <vscale x 4 x i1> %res
+}
+
+define <vscale x 8 x i1> @fcmp_lt_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
+; CHECK-LABEL: fcmp_lt_nxv8bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uunpkhi z2.s, z0.h
+; CHECK-NEXT: uunpkhi z3.s, z1.h
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: uunpklo z1.s, z1.h
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: lsl z2.s, z2.s, #16
+; CHECK-NEXT: lsl z3.s, z3.s, #16
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: fcmgt p1.s, p0/z, z3.s, z2.s
+; CHECK-NEXT: fcmgt p0.s, p0/z, z1.s, z0.s
+; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h
+; CHECK-NEXT: ret
+ %res = fcmp fast olt <vscale x 8 x bfloat> %a, %b
+ ret <vscale x 8 x i1> %res
+}
+
+;
+; LE
+;
+
+define <vscale x 2 x i1> @fcmp_le_nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) {
+; CHECK-LABEL: fcmp_le_nxv2bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: fcmge p0.s, p0/z, z1.s, z0.s
+; CHECK-NEXT: ret
+ %res = fcmp fast ole <vscale x 2 x bfloat> %a, %b
+ ret <vscale x 2 x i1> %res
+}
+
+define <vscale x 4 x i1> @fcmp_le_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) {
+; CHECK-LABEL: fcmp_le_nxv4bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: fcmge p0.s, p0/z, z1.s, z0.s
+; CHECK-NEXT: ret
+ %res = fcmp fast ole <vscale x 4 x bfloat> %a, %b
+ ret <vscale x 4 x i1> %res
+}
+
+define <vscale x 8 x i1> @fcmp_le_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
+; CHECK-LABEL: fcmp_le_nxv8bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uunpkhi z2.s, z0.h
+; CHECK-NEXT: uunpkhi z3.s, z1.h
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: uunpklo z1.s, z1.h
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: lsl z2.s, z2.s, #16
+; CHECK-NEXT: lsl z3.s, z3.s, #16
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: fcmge p1.s, p0/z, z3.s, z2.s
+; CHECK-NEXT: fcmge p0.s, p0/z, z1.s, z0.s
+; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h
+; CHECK-NEXT: ret
+ %res = fcmp fast ole <vscale x 8 x bfloat> %a, %b
+ ret <vscale x 8 x i1> %res
+}
+
+;
+; NE
+;
+
+define <vscale x 2 x i1> @fcmp_ne_nxv2bf16(<vscale x 2 x bfloat> %a, <vscale x 2 x bfloat> %b) {
+; CHECK-LABEL: fcmp_ne_nxv2bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: ptrue p0.d
+; CHECK-NEXT: fcmne p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: ret
+ %res = fcmp fast one <vscale x 2 x bfloat> %a, %b
+ ret <vscale x 2 x i1> %res
+}
+
+define <vscale x 4 x i1> @fcmp_ne_nxv4bf16(<vscale x 4 x bfloat> %a, <vscale x 4 x bfloat> %b) {
+; CHECK-LABEL: fcmp_ne_nxv4bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: fcmne p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: ret
+ %res = fcmp fast one <vscale x 4 x bfloat> %a, %b
+ ret <vscale x 4 x i1> %res
+}
+
+define <vscale x 8 x i1> @fcmp_ne_nxv8bf16(<vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) {
+; CHECK-LABEL: fcmp_ne_nxv8bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uunpkhi z2.s, z1.h
+; CHECK-NEXT: uunpkhi z3.s, z0.h
+; CHECK-NEXT: uunpklo z1.s, z1.h
+; CHECK-NEXT: uunpklo z0.s, z0.h
+; CHECK-NEXT: ptrue p0.s
+; CHECK-NEXT: lsl z2.s, z2.s, #16
+; CHECK-NEXT: lsl z3.s, z3.s, #16
+; CHECK-NEXT: lsl z1.s, z1.s, #16
+; CHECK-NEXT: lsl z0.s, z0.s, #16
+; CHECK-NEXT: fcmne p1.s, p0/z, z3.s, z2.s
+; CHECK-NEXT: fcmne p0.s, p0/z, z0.s, z1.s
+; CHECK-NEXT: uzp1 p0.h, p0.h, p1.h
+; CHECK-NEXT: ret
+ %res = fcmp fast one <vscale x 8 x bfloat> %a, %b
+ ret <vscale x 8 x i1> %res
+}
More information about the llvm-commits
mailing list