[clang] [llvm] [AArch64][clang][llvm] Add structured sparsity outer product (TMOP) intrinsics (PR #135145)
Jonathan Thackray via cfe-commits
cfe-commits at lists.llvm.org
Fri Apr 11 15:47:18 PDT 2025
https://github.com/jthackray updated https://github.com/llvm/llvm-project/pull/135145
>From c0bff6fe1d1630ba93361ac29bca798810a64178 Mon Sep 17 00:00:00 2001
From: Jonathan Thackray <jonathan.thackray at arm.com>
Date: Mon, 3 Mar 2025 16:16:46 +0000
Subject: [PATCH 1/5] [AArch64][clang][llvm] Add structured sparsity outer
product (TMOP) instrinics
Implement all {BF/F/S/U/SU/US}TMOP intrinsics in clang and llvm
following the ACLE in https://github.com/ARM-software/acle/pull/380/files
---
clang/include/clang/Basic/arm_sme.td | 42 +++++
.../AArch64/sme2-intrinsics/acle_sme2_tmop.c | 112 +++++++++++
.../acle_sme2_tmop.cpp | 176 ++++++++++++++++++
llvm/include/llvm/IR/IntrinsicsAArch64.td | 17 ++
.../lib/Target/AArch64/AArch64RegisterInfo.td | 6 +-
.../lib/Target/AArch64/AArch64SMEInstrInfo.td | 42 ++---
llvm/lib/Target/AArch64/SMEInstrFormats.td | 66 ++++---
.../AArch64/GlobalISel/regbank-inlineasm.mir | 6 +-
.../emit_fneg_with_non_register_operand.mir | 4 +-
.../CodeGen/AArch64/peephole-insvigpr.mir | 4 +-
.../CodeGen/AArch64/sme2-intrinsics-tmop.ll | 138 ++++++++++++++
11 files changed, 552 insertions(+), 61 deletions(-)
create mode 100644 clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_tmop.c
create mode 100644 clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_tmop.cpp
create mode 100644 llvm/test/CodeGen/AArch64/sme2-intrinsics-tmop.ll
diff --git a/clang/include/clang/Basic/arm_sme.td b/clang/include/clang/Basic/arm_sme.td
index 1bfcf4c31d552..d425c2c1e38bb 100644
--- a/clang/include/clang/Basic/arm_sme.td
+++ b/clang/include/clang/Basic/arm_sme.td
@@ -907,6 +907,48 @@ let SMETargetGuard = "sme-f16f16" in {
}
+////////////////////////////////////////////////////////////////////////////////
+// SME2 - TMOP, SUTMOP, USTMOP
+
+multiclass USTMOP<string za, string t, list<ImmCheck> checks> {
+ def _ : SInst<"svtmopa_lane_" # za # "[_{d}_{3}]",
+ "vi2x[i", t, MergeNone, "aarch64_sme_ustmopa",
+ [IsStreaming, IsInOutZA],
+ checks>;
+}
+
+multiclass SUTMOP<string za, string t, list<ImmCheck> checks> {
+ def _ : SInst<"svtmopa_lane_" # za # "[_{d}_{3}]",
+ "vi2u[i", t, MergeNone, "aarch64_sme_sutmopa",
+ [IsStreaming, IsInOutZA],
+ checks>;
+}
+
+let SMETargetGuard = "sme2,sme-tmop" in {
+ def SVTMOPA_ZA32 : Inst<"svtmopa_lane_za32[_{d}_{d}]", "vi2d[i", "hbf", MergeNone, "aarch64_sme_tmopa", [IsStreaming, IsInOutZA], [ImmCheck<0, ImmCheck0_3>, ImmCheck<4, ImmCheck0_3>]>;
+ def SVSTMOPA_ZA32 : Inst<"svtmopa_lane_za32[_{d}_{d}]", "vi2d[i", "cs", MergeNone, "aarch64_sme_stmopa", [IsStreaming, IsInOutZA], [ImmCheck<0, ImmCheck0_3>, ImmCheck<4, ImmCheck0_3>]>;
+ def SVUTMOPA_ZA32 : Inst<"svtmopa_lane_za32[_{d}_{d}]", "vi2d[i", "UcUs", MergeNone, "aarch64_sme_utmopa", [IsStreaming, IsInOutZA], [ImmCheck<0, ImmCheck0_3>, ImmCheck<4, ImmCheck0_3>]>;
+
+ defm SVSUTMOPA_S : SUTMOP<"za32", "c", [ImmCheck<0, ImmCheck0_3>, ImmCheck<4, ImmCheck0_3>]>;
+ defm SVUSTMOPA_S : USTMOP<"za32", "Uc", [ImmCheck<0, ImmCheck0_3>, ImmCheck<4, ImmCheck0_3>]>;
+}
+
+let SMETargetGuard = "sme2,sme-tmop,sme-f16f16" in {
+ def SVTMOPA_F16 : Inst<"svtmopa_lane_za16[_{d}_{d}]", "vi2d[i", "hb", MergeNone, "aarch64_sme_tmopa", [IsStreaming, IsInOutZA], [ImmCheck<0, ImmCheck0_3>, ImmCheck<4, ImmCheck0_3>]>;
+}
+
+let SMETargetGuard = "sme2,sme-tmop,sme-b16b16" in {
+ def SVTMOPA_BF16 : Inst<"svtmopa_lane_za16[_{d}_{d}]", "vi2d[i", "U", MergeNone, "aarch64_sme_tmopa", [IsStreaming, IsInOutZA], [ImmCheck<0, ImmCheck0_3>, ImmCheck<4, ImmCheck0_3>]>;
+}
+
+let SMETargetGuard = "sme2,sme-tmop,sme-f8f16" in {
+ def SVTMOPA_ZA16_FPM : Inst<"svtmopa_lane_za16[_{d}_{d}]", "vi2.MM[i>", "m", MergeNone, "aarch64_sme_tmopa", [IsStreaming, IsInOutZA], [ImmCheck<0, ImmCheck0_3>, ImmCheck<4, ImmCheck0_3>]>;
+}
+
+let SMETargetGuard = "sme2,sme-tmop,sme-f8f32" in {
+ def SVTMOPA_ZA32_FPM : Inst<"svtmopa_lane_za32[_{d}_{d}]", "vi2.MM[i>", "m", MergeNone, "aarch64_sme_tmopa", [IsStreaming, IsInOutZA], [ImmCheck<0, ImmCheck0_3>, ImmCheck<4, ImmCheck0_3>]>;
+}
+
multiclass ZAReadz<string n_suffix, string vg_num, string t, string i_prefix, list<ImmCheck> ch> {
let SMETargetGuard = "sme2p1" in {
def NAME # _H : SInst<"svreadz_hor_" # n_suffix # "_{d}_vg" # vg_num, vg_num # "im", t,
diff --git a/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_tmop.c b/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_tmop.c
new file mode 100644
index 0000000000000..a1456275dd810
--- /dev/null
+++ b/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_tmop.c
@@ -0,0 +1,112 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
+
+// REQUIRES: aarch64-registered-target
+// RUN: %clang_cc1 -triple aarch64 -target-feature +bf16 -target-feature +sme-tmop -target-feature +sme-f16f16 -target-feature +sme-b16b16 -target-feature +sme-f8f16 -target-feature +sme-f8f32 -target-feature +sme -target-feature +sme2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s
+// RUN: %clang_cc1 -DSME_OVERLOADED_FORMS -triple aarch64 -target-feature +bf16 -target-feature +sme-tmop -target-feature +sme-f16f16 -target-feature +sme-b16b16 -target-feature +sme-f8f16 -target-feature +sme-f8f32 -target-feature +sme -target-feature +sme2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64 -target-feature +bf16 -target-feature +sme-tmop -target-feature +sme-f16f16 -target-feature +sme-b16b16 -target-feature +sme-f8f16 -target-feature +sme-f8f32 -target-feature +sme -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
+
+#include <arm_sme.h>
+
+#ifdef SME_OVERLOADED_FORMS
+#define SME_ACLE_FUNC(A1,A2_UNUSED,A3) A1##A3
+#else
+#define SME_ACLE_FUNC(A1,A2,A3) A1##A2##A3
+#endif
+
+// CHECK-LABEL: @test_svtmopa_lane_za32_s8_s8(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.aarch64.sme.stmopa.nxv16i8(i32 1, <vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CHECK-NEXT: ret void
+//
+void test_svtmopa_lane_za32_s8_s8(svint8x2_t zn, svint8_t zm, svuint8_t zk) __arm_streaming __arm_inout("za") {
+ SME_ACLE_FUNC(svtmopa_lane_za32,_s8_s8,)(1, zn, zm, zk, 3);
+}
+
+// CHECK-LABEL: @test_svtmopa_lane_za32_u8_u8(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.aarch64.sme.utmopa.nxv16i8(i32 1, <vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CHECK-NEXT: ret void
+//
+void test_svtmopa_lane_za32_u8_u8(svuint8x2_t zn, svuint8_t zm, svuint8_t zk) __arm_streaming __arm_inout("za") {
+ SME_ACLE_FUNC(svtmopa_lane_za32,_u8_u8,)(1, zn, zm, zk, 3);
+}
+
+// CHECK-LABEL: @test_svtmopa_lane_za32_s8_u8(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.aarch64.sme.sutmopa.nxv16i8(i32 1, <vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CHECK-NEXT: ret void
+//
+void test_svtmopa_lane_za32_s8_u8(svint8x2_t zn, svuint8_t zm, svuint8_t zk) __arm_streaming __arm_inout("za") {
+ SME_ACLE_FUNC(svtmopa_lane_za32,_s8_u8,)(1, zn, zm, zk, 3);
+}
+
+// CHECK-LABEL: @test_svtmopa_lane_za32_u8_s8(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.aarch64.sme.ustmopa.nxv16i8(i32 1, <vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CHECK-NEXT: ret void
+//
+void test_svtmopa_lane_za32_u8_s8(svuint8x2_t zn, svint8_t zm, svuint8_t zk) __arm_streaming __arm_inout("za") {
+ SME_ACLE_FUNC(svtmopa_lane_za32,_u8_s8,)(1, zn, zm, zk, 3);
+}
+
+// CHECK-LABEL: @test_svtmopa_lane_za32_s16_s16(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.aarch64.sme.stmopa.nxv8i16(i32 1, <vscale x 8 x i16> [[ZN_COERCE0:%.*]], <vscale x 8 x i16> [[ZN_COERCE1:%.*]], <vscale x 8 x i16> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CHECK-NEXT: ret void
+//
+void test_svtmopa_lane_za32_s16_s16(svint16x2_t zn, svint16_t zm, svuint8_t zk) __arm_streaming __arm_inout("za") {
+ SME_ACLE_FUNC(svtmopa_lane_za32,_s16_s16,)(1, zn, zm, zk, 3);
+}
+
+// CHECK-LABEL: @test_svtmopa_lane_za32_u16_u16(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.aarch64.sme.utmopa.nxv8i16(i32 1, <vscale x 8 x i16> [[ZN_COERCE0:%.*]], <vscale x 8 x i16> [[ZN_COERCE1:%.*]], <vscale x 8 x i16> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CHECK-NEXT: ret void
+//
+void test_svtmopa_lane_za32_u16_u16(svuint16x2_t zn, svuint16_t zm, svuint8_t zk) __arm_streaming __arm_inout("za") {
+ SME_ACLE_FUNC(svtmopa_lane_za32,_u16_u16,)(1, zn, zm, zk, 3);
+}
+
+// CHECK-LABEL: @test_svtmopa_lane_za32_f16_f16(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.aarch64.sme.tmopa.nxv8f16(i32 1, <vscale x 8 x half> [[ZN_COERCE0:%.*]], <vscale x 8 x half> [[ZN_COERCE1:%.*]], <vscale x 8 x half> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CHECK-NEXT: ret void
+//
+void test_svtmopa_lane_za32_f16_f16(svfloat16x2_t zn, svfloat16_t zm, svuint8_t zk) __arm_streaming __arm_inout("za") {
+ SME_ACLE_FUNC(svtmopa_lane_za32,_f16_f16,)(1, zn, zm, zk, 3);
+}
+
+// CHECK-LABEL: @test_svtmopa_lane_za32_bf16_bf16(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.aarch64.sme.tmopa.nxv8bf16(i32 1, <vscale x 8 x bfloat> [[ZN_COERCE0:%.*]], <vscale x 8 x bfloat> [[ZN_COERCE1:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CHECK-NEXT: ret void
+//
+void test_svtmopa_lane_za32_bf16_bf16(svbfloat16x2_t zn, svbfloat16_t zm, svuint8_t zk) __arm_streaming __arm_inout("za") {
+ SME_ACLE_FUNC(svtmopa_lane_za32,_bf16_bf16,)(1, zn, zm, zk, 3);
+}
+
+// CHECK-LABEL: @test_svtmopa_lane_za16_f16_f16(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.aarch64.sme.tmopa.nxv8f16(i32 1, <vscale x 8 x half> [[ZN_COERCE0:%.*]], <vscale x 8 x half> [[ZN_COERCE1:%.*]], <vscale x 8 x half> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CHECK-NEXT: ret void
+//
+void test_svtmopa_lane_za16_f16_f16(svfloat16x2_t zn, svfloat16_t zm, svuint8_t zk) __arm_streaming __arm_inout("za") {
+ SME_ACLE_FUNC(svtmopa_lane_za16,_f16_f16,)(1, zn, zm, zk, 3);
+}
+
+// CHECK-LABEL: @test_svtmopa_lane_za16_bf16_bf16(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.aarch64.sme.tmopa.nxv8bf16(i32 1, <vscale x 8 x bfloat> [[ZN_COERCE0:%.*]], <vscale x 8 x bfloat> [[ZN_COERCE1:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CHECK-NEXT: ret void
+//
+void test_svtmopa_lane_za16_bf16_bf16(svbfloat16x2_t zn, svbfloat16_t zm, svuint8_t zk) __arm_streaming __arm_inout("za") {
+ SME_ACLE_FUNC(svtmopa_lane_za16,_bf16_bf16,)(1, zn, zm, zk, 3);
+}
+
+//void test_svtmopa_lane_za16_mf8_mf8_fpm(svfloat32x2_t zn, svfloat32_t zm, svuint8_t zk, fpm_t fpmr) __arm_streaming __arm_inout("za") {
+// SME_ACLE_FUNC(svtmopa_lane_za16,_mf8_mf8,_fpm)(1, zn, zm, zk, 3, fpmr);
+//}
+
+//void test_svtmopa_lane_za32_mf8_mf8_fpm(svfloat32x2_t zn, svfloat32_t zm, svuint8_t zk, fpm_t fpmr) __arm_streaming __arm_inout("za") {
+// SME_ACLE_FUNC(svtmopa_lane_za32,_mf8_mf8,_fpm)(1, zn, zm, zk, 3, fpmr);
+//}
diff --git a/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_tmop.cpp b/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_tmop.cpp
new file mode 100644
index 0000000000000..6ccd3acaf749f
--- /dev/null
+++ b/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_tmop.cpp
@@ -0,0 +1,176 @@
+// RUN: %clang_cc1 -triple aarch64 -target-feature +sme -target-feature +sme2 -target-feature +bf16 -target-feature +sme-f16f16 -target-feature +sme-b16b16 -verify -emit-llvm -o - %s
+
+// REQUIRES: aarch64-registered-target
+
+#include <arm_sme.h>
+
+void test_features() __arm_streaming __arm_inout("za") {
+ svuint8x2_t zn_u8;
+ svint8x2_t zn_s8;
+ svuint8_t zm_u8;
+ svint8_t zm_s8;
+ svuint16x2_t zn_u16;
+ svint16x2_t zn_s16;
+ svuint16_t zm_u16;
+ svint16_t zm_s16;
+ svbfloat16x2_t zn_bf16;
+ svfloat16x2_t zn_f16;
+ svbfloat16_t zm_bf16;
+ svfloat16_t zm_f16;
+ svfloat32x2_t zn_f32;
+ svfloat32_t zm_f32;
+ fpm_t fpm = 0;
+ svuint8_t zk;
+
+// expected-error at +1 {{'svtmopa_lane_za32_s8_s8' needs target feature sme,sme2,sme-tmop}}
+ svtmopa_lane_za32_s8_s8(0, zn_s8, zm_s8, zk, 0);
+// expected-error at +1 {{'svtmopa_lane_za32_u8_u8' needs target feature sme,sme2,sme-tmop}}
+ svtmopa_lane_za32_u8_u8(0, zn_u8, zm_u8, zk, 0);
+// expected-error at +1 {{'svtmopa_lane_za32_s8_u8' needs target feature sme,sme2,sme-tmop}}
+ svtmopa_lane_za32_s8_u8(0, zn_s8, zm_u8, zk, 0);
+// expected-error at +1 {{'svtmopa_lane_za32_u8_s8' needs target feature sme,sme2,sme-tmop}}
+ svtmopa_lane_za32_u8_s8(0, zn_u8, zm_s8, zk, 0);
+// expected-error at +1 {{'svtmopa_lane_za32_s16_s16' needs target feature sme,sme2,sme-tmop}}
+ svtmopa_lane_za32_s16_s16(0, zn_s16, zm_s16, zk, 0);
+// expected-error at +1 {{'svtmopa_lane_za32_u16_u16' needs target feature sme,sme2,sme-tmop}}
+ svtmopa_lane_za32_u16_u16(0, zn_u16, zm_u16, zk, 0);
+// expected-error at +1 {{'svtmopa_lane_za32_f16_f16' needs target feature sme,sme2,sme-tmop}}
+ svtmopa_lane_za32_f16_f16(0, zn_f16, zm_f16, zk, 0);
+// expected-error at +1 {{'svtmopa_lane_za32_bf16_bf16' needs target feature sme,sme2,sme-tmop}}
+ svtmopa_lane_za32_bf16_bf16(0, zn_bf16, zm_bf16, zk, 0);
+// expected-error at +1 {{'svtmopa_lane_za16_f16_f16' needs target feature sme,sme2,sme-tmop,sme-f16f16}}
+ svtmopa_lane_za16_f16_f16(0, zn_f16, zm_f16, zk, 0);
+// expected-error at +1 {{'svtmopa_lane_za16_bf16_bf16' needs target feature sme,sme2,sme-tmop,sme-f16f16}}
+ svtmopa_lane_za16_bf16_bf16(0, zn_bf16, zm_bf16, zk, 0);
+// expected-error at +1 {{'svtmopa_lane_za16_mf8_mf8_fpm' needs target feature sme,sme2,sme-tmop,sme-f8f16}}
+ svtmopa_lane_za16_mf8_mf8_fpm(0, zn_f32, zm_f32, zk, 0, fpm);
+// expected-error at +1 {{'svtmopa_lane_za32_mf8_mf8_fpm' needs target feature sme,sme2,sme-tmop,sme-f8f32}}
+ svtmopa_lane_za32_mf8_mf8_fpm(0, zn_f32, zm_f32, zk, 0, fpm);
+}
+
+void test_imm() __arm_streaming __arm_inout("za") {
+ svuint8x2_t zn_u8;
+ svint8x2_t zn_s8;
+ svuint8_t zm_u8;
+ svint8_t zm_s8;
+ svuint16x2_t zn_u16;
+ svint16x2_t zn_s16;
+ svuint16_t zm_u16;
+ svint16_t zm_s16;
+ svbfloat16x2_t zn_bf16;
+ svfloat16x2_t zn_f16;
+ svbfloat16_t zm_bf16;
+ svfloat16_t zm_f16;
+ svfloat32x2_t zn_f32;
+ svfloat32_t zm_f32;
+ fpm_t fpm;
+ svuint8_t zk;
+
+// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
+ svtmopa_lane_za32_s8_s8(3, zn_s8, zm_s8, zk, 4);
+// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
+ svtmopa_lane_za32_s8_s8(4, zn_s8, zm_s8, zk, 3);
+// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
+ svtmopa_lane_za32_s8_s8(0, zn_s8, zm_s8, zk, -1);
+// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
+ svtmopa_lane_za32_s8_s8(-1, zn_s8, zm_s8, zk, 0);
+
+// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
+ svtmopa_lane_za32_u8_u8(3, zn_u8, zm_u8, zk, 4);
+// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
+ svtmopa_lane_za32_u8_u8(4, zn_u8, zm_u8, zk, 3);
+// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
+ svtmopa_lane_za32_u8_u8(0, zn_u8, zm_u8, zk, -1);
+// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
+ svtmopa_lane_za32_u8_u8(-1, zn_u8, zm_u8, zk, 0);
+
+// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
+ svtmopa_lane_za32_s8_u8(3, zn_s8, zm_u8, zk, 4);
+// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
+ svtmopa_lane_za32_s8_u8(4, zn_s8, zm_u8, zk, 3);
+// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
+ svtmopa_lane_za32_s8_u8(0, zn_s8, zm_u8, zk, -1);
+// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
+ svtmopa_lane_za32_s8_u8(-1, zn_s8, zm_u8, zk, 0);
+
+// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
+ svtmopa_lane_za32_u8_s8(3, zn_u8, zm_s8, zk, 4);
+// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
+ svtmopa_lane_za32_u8_s8(4, zn_u8, zm_s8, zk, 3);
+// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
+ svtmopa_lane_za32_u8_s8(0, zn_u8, zm_s8, zk, -1);
+// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
+ svtmopa_lane_za32_u8_s8(-1, zn_u8, zm_s8, zk, 0);
+
+// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
+ svtmopa_lane_za32_s16_s16(3, zn_s16, zm_s16, zk, 4);
+// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
+ svtmopa_lane_za32_s16_s16(4, zn_s16, zm_s16, zk, 3);
+// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
+ svtmopa_lane_za32_s16_s16(0, zn_s16, zm_s16, zk, -1);
+// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
+ svtmopa_lane_za32_s16_s16(-1, zn_s16, zm_s16, zk, 0);
+
+// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
+ svtmopa_lane_za32_u16_u16(3, zn_u16, zm_u16, zk, 4);
+// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
+ svtmopa_lane_za32_u16_u16(4, zn_u16, zm_u16, zk, 3);
+// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
+ svtmopa_lane_za32_u16_u16(0, zn_u16, zm_u16, zk, -1);
+// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
+ svtmopa_lane_za32_u16_u16(-1, zn_u16, zm_u16, zk, 0);
+
+// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
+ svtmopa_lane_za32_f16_f16(3, zn_f16, zm_f16, zk, 4);
+// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
+ svtmopa_lane_za32_f16_f16(4, zn_f16, zm_f16, zk, 3);
+// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
+ svtmopa_lane_za32_f16_f16(0, zn_f16, zm_f16, zk, -1);
+// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
+ svtmopa_lane_za32_f16_f16(-1, zn_f16, zm_f16, zk, 0);
+
+// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
+ svtmopa_lane_za32_bf16_bf16(3, zn_bf16, zm_bf16, zk, 4);
+// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
+ svtmopa_lane_za32_bf16_bf16(4, zn_bf16, zm_bf16, zk, 3);
+// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
+ svtmopa_lane_za32_bf16_bf16(0, zn_bf16, zm_bf16, zk, -1);
+// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
+ svtmopa_lane_za32_bf16_bf16(-1, zn_bf16, zm_bf16, zk, 0);
+
+// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
+ svtmopa_lane_za16_f16_f16(3, zn_f16, zm_f16, zk, 4);
+// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
+ svtmopa_lane_za16_f16_f16(4, zn_f16, zm_f16, zk, 3);
+// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
+ svtmopa_lane_za16_f16_f16(0, zn_f16, zm_f16, zk, -1);
+// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
+ svtmopa_lane_za16_f16_f16(-1, zn_f16, zm_f16, zk, 0);
+
+// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
+ svtmopa_lane_za16_bf16_bf16(3, zn_bf16, zm_bf16, zk, 4);
+// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
+ svtmopa_lane_za16_bf16_bf16(4, zn_bf16, zm_bf16, zk, 3);
+// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
+ svtmopa_lane_za16_bf16_bf16(0, zn_bf16, zm_bf16, zk, -1);
+// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
+ svtmopa_lane_za16_bf16_bf16(-1, zn_bf16, zm_bf16, zk, 0);
+
+// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
+ svtmopa_lane_za16_mf8_mf8_fpm(3, zn_f32, zm_f32, zk, 4, fpm);
+// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
+ svtmopa_lane_za16_mf8_mf8_fpm(4, zn_f32, zm_f32, zk, 3, fpm);
+// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
+ svtmopa_lane_za16_mf8_mf8_fpm(0, zn_f32, zm_f32, zk, -1, fpm);
+// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
+ svtmopa_lane_za16_mf8_mf8_fpm(-1, zn_f32, zm_f32, zk, 0, fpm);
+
+// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
+ svtmopa_lane_za32_mf8_mf8_fpm(3, zn_f32, zm_f32, zk, 4, fpm);
+// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
+ svtmopa_lane_za32_mf8_mf8_fpm(4, zn_f32, zm_f32, zk, 3, fpm);
+// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
+ svtmopa_lane_za32_mf8_mf8_fpm(0, zn_f32, zm_f32, zk, -1, fpm);
+// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
+ svtmopa_lane_za32_mf8_mf8_fpm(-1, zn_f32, zm_f32, zk, 0, fpm);
+}
diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td
index 77ea0bcaa4b5f..349d05628261a 100644
--- a/llvm/include/llvm/IR/IntrinsicsAArch64.td
+++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td
@@ -3107,6 +3107,23 @@ let TargetPrefix = "aarch64" in {
}
}
+ class SME_OuterProduct_TMOP_Intrinsic
+ : DefaultAttrsIntrinsic<[],
+ [llvm_i32_ty,
+ llvm_anyvector_ty,
+ LLVMMatchType<0>,
+ LLVMMatchType<0>,
+ llvm_nxv16i8_ty,
+ llvm_i32_ty],
+ [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<5>>,
+ IntrNoMem, IntrHasSideEffects]>;
+
+ def int_aarch64_sme_tmopa : SME_OuterProduct_TMOP_Intrinsic;
+ def int_aarch64_sme_stmopa : SME_OuterProduct_TMOP_Intrinsic;
+ def int_aarch64_sme_utmopa : SME_OuterProduct_TMOP_Intrinsic;
+ def int_aarch64_sme_sutmopa : SME_OuterProduct_TMOP_Intrinsic;
+ def int_aarch64_sme_ustmopa : SME_OuterProduct_TMOP_Intrinsic;
+
class SME_AddVectorToTile_Intrinsic
: DefaultAttrsIntrinsic<[],
[llvm_i32_ty,
diff --git a/llvm/lib/Target/AArch64/AArch64RegisterInfo.td b/llvm/lib/Target/AArch64/AArch64RegisterInfo.td
index fed9b7b173e9c..40553aff04919 100644
--- a/llvm/lib/Target/AArch64/AArch64RegisterInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64RegisterInfo.td
@@ -1269,8 +1269,10 @@ def ZPRMul2AsmOp32_Hi : ZPRAsmOperand<"VectorS_Hi", 32, "Mul2_Hi">;
def ZPRMul2AsmOp64_Lo : ZPRAsmOperand<"VectorD_Lo", 64, "Mul2_Lo">;
def ZPRMul2AsmOp64_Hi : ZPRAsmOperand<"VectorD_Hi", 64, "Mul2_Hi">;
-def ZPR_K : RegisterClass<"AArch64", [untyped], 128,
- (add Z20, Z21, Z22, Z23, Z28, Z29, Z30, Z31)>;
+def ZPR_K : RegisterClass<"AArch64", [nxv16i8], 128,
+ (add Z20, Z21, Z22, Z23, Z28, Z29, Z30, Z31)>{
+ let Size = 128;
+}
def ZK : RegisterOperand<ZPR_K, "printSVERegOp<>">{
let EncoderMethod = "EncodeZK";
diff --git a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
index f992f73171e0e..330dce018daa4 100644
--- a/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SMEInstrInfo.td
@@ -175,12 +175,12 @@ let Predicates = [HasSME_MOP4, HasSMEI16I64] in {
}
let Predicates = [HasSME_TMOP] in {
-def STMOPA_M2ZZZI_BtoS : sme_int_sparse_outer_product_i32<0b00100, ZZ_b_mul_r, ZPR8, "stmopa">;
-def STMOPA_M2ZZZI_HtoS : sme_int_sparse_outer_product_i32<0b00101, ZZ_h_mul_r, ZPR16, "stmopa">;
-def UTMOPA_M2ZZZI_BtoS : sme_int_sparse_outer_product_i32<0b11100, ZZ_b_mul_r, ZPR8, "utmopa">;
-def UTMOPA_M2ZZZI_HtoS : sme_int_sparse_outer_product_i32<0b10101, ZZ_h_mul_r, ZPR16, "utmopa">;
-def SUTMOPA_M2ZZZI_BtoS : sme_int_sparse_outer_product_i32<0b01100, ZZ_b_mul_r, ZPR8, "sutmopa">;
-def USTMOPA_M2ZZZI_BtoS : sme_int_sparse_outer_product_i32<0b10100, ZZ_b_mul_r, ZPR8, "ustmopa">;
+ defm STMOPA_M2ZZZI_BtoS : sme_tmopa_32b<0b00100, ZZ_b_mul_r, ZPR8, nxv16i8, "stmopa", "int_aarch64_sme_stmopa">;
+ defm STMOPA_M2ZZZI_HtoS : sme_tmopa_32b<0b00101, ZZ_h_mul_r, ZPR16, nxv8i16, "stmopa", "int_aarch64_sme_stmopa">;
+ defm UTMOPA_M2ZZZI_BtoS : sme_tmopa_32b<0b11100, ZZ_b_mul_r, ZPR8, nxv16i8, "utmopa", "int_aarch64_sme_utmopa">;
+ defm UTMOPA_M2ZZZI_HtoS : sme_tmopa_32b<0b10101, ZZ_h_mul_r, ZPR16, nxv8i16, "utmopa", "int_aarch64_sme_utmopa">;
+ defm SUTMOPA_M2ZZZI_BtoS : sme_tmopa_32b<0b01100, ZZ_b_mul_r, ZPR8, nxv16i8, "sutmopa", "int_aarch64_sme_sutmopa">;
+ defm USTMOPA_M2ZZZI_BtoS : sme_tmopa_32b<0b10100, ZZ_b_mul_r, ZPR8, nxv16i8, "ustmopa", "int_aarch64_sme_ustmopa">;
}
let Predicates = [HasSME] in {
@@ -1064,12 +1064,6 @@ let Predicates = [HasSME_MOP4] in {
defm FMOP4S : sme2_fmop4as_fp32_non_widening<1, "fmop4s", "int_aarch64_sme_mop4s">;
}
-let Predicates = [HasSME_TMOP] in {
- def FTMOPA_M2ZZZI_HtoS : sme_tmopa_32b<0b11000, ZZ_h_mul_r, ZPR16, "ftmopa">;
- def FTMOPA_M2ZZZI_StoS : sme_tmopa_32b<0b00000, ZZ_s_mul_r, ZPR32, "ftmopa">;
- def BFTMOPA_M2ZZZI_HtoS : sme_tmopa_32b<0b10000, ZZ_h_mul_r, ZPR16, "bftmopa">;
-}
-
let Predicates = [HasSME2p2] in {
defm FMUL_2ZZ : sme2_multi2_fmul_sm<"fmul">;
defm FMUL_2Z2Z : sme2_multi2_fmul_mm< "fmul">;
@@ -1078,26 +1072,32 @@ let Predicates = [HasSME2p2] in {
} // [HasSME2p2]
+let Predicates = [HasSME_TMOP] in {
+ defm FTMOPA_M2ZZZI_HtoS : sme_tmopa_32b<0b11000, ZZ_h_mul_r, ZPR16, nxv8f16, "ftmopa", "int_aarch64_sme_tmopa">;
+ defm FTMOPA_M2ZZZI_StoS : sme_tmopa_32b<0b00000, ZZ_s_mul_r, ZPR32, nxv4f32, "ftmopa", "int_aarch64_sme_tmopa">;
+ defm BFTMOPA_M2ZZZI_HtoS : sme_tmopa_32b<0b10000, ZZ_h_mul_r, ZPR16, nxv8bf16, "bftmopa", "int_aarch64_sme_tmopa">;
+}
+
+let Predicates = [HasSME_TMOP, HasSMEF16F16] in {
+ defm FTMOPA_M2ZZZI_HtoH : sme_tmopa_16b<0b10001, ZZ_h_mul_r, ZPR16, nxv8f16, "ftmopa", "int_aarch64_sme_tmopa">;
+}
+
let Predicates = [HasSME_TMOP, HasSMEB16B16] in {
- def BFTMOPA_M2ZZZI_HtoH : sme_tmopa_16b<0b11001, ZZ_h_mul_r, ZPR16, "bftmopa">;
+ defm BFTMOPA_M2ZZZI_HtoH : sme_tmopa_16b<0b11001, ZZ_h_mul_r, ZPR16, nxv8bf16, "bftmopa", "int_aarch64_sme_tmopa">;
}
-let Predicates = [HasSME_TMOP, HasSMEF8F32], Uses = [FPMR, FPCR] in {
- def FTMOPA_M2ZZZI_BtoS : sme_tmopa_32b<0b01000, ZZ_b_mul_r, ZPR8, "ftmopa">;
+let Predicates = [HasSME_TMOP, HasSMEF8F32] in {
+ defm FTMOPA_M2ZZZI_BtoS : sme_tmopa_32b<0b01000, ZZ_b_mul_r, ZPR8, nxv16i8, "ftmopa", "int_aarch64_sme_tmopa">;
}
-let Predicates = [HasSME_TMOP, HasSMEF8F16], Uses = [FPMR, FPCR] in {
- def FTMOPA_M2ZZZI_BtoH : sme_tmopa_16b<0b01001, ZZ_b_mul_r, ZPR8, "ftmopa">;
+let Predicates = [HasSME_TMOP, HasSMEF8F16] in {
+ defm FTMOPA_M2ZZZI_BtoH : sme_tmopa_16b<0b01001, ZZ_b_mul_r, ZPR8, nxv16i8, "ftmopa", "int_aarch64_sme_tmopa">;
}
let Predicates = [HasSME_MOP4, HasSMEF8F16], Uses = [FPMR, FPCR] in {
defm FMOP4A : sme2_fmop4a_fp8_fp16_2way<"fmop4a">;
}
-let Predicates = [HasSME_TMOP, HasSMEF16F16] in {
- def FTMOPA_M2ZZZI_HtoH : sme_tmopa_16b<0b10001, ZZ_h_mul_r, ZPR16, "ftmopa">;
-}
-
let Predicates = [HasSME_MOP4, HasSMEF16F16] in {
defm FMOP4A : sme2_fmop4as_fp16_non_widening<0, "fmop4a", "int_aarch64_sme_mop4a">;
defm FMOP4S : sme2_fmop4as_fp16_non_widening<1, "fmop4s", "int_aarch64_sme_mop4s">;
diff --git a/llvm/lib/Target/AArch64/SMEInstrFormats.td b/llvm/lib/Target/AArch64/SMEInstrFormats.td
index c008cda21cf05..c42299b050eb6 100644
--- a/llvm/lib/Target/AArch64/SMEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SMEInstrFormats.td
@@ -104,6 +104,14 @@ class sme_outer_product_pseudo<ZPRRegOp zpr_ty, SMEMatrixTypeEnum za_flag>
let usesCustomInserter = 1;
}
+class sme_sparse_outer_product_pseudo<RegisterOperand zn_ty, RegisterOperand zm_ty, SMEMatrixTypeEnum za_flag>
+ : Pseudo<(outs), (ins i32imm:$tile, zn_ty:$zn, zm_ty:$zm, ZK:$zk, i32imm:$idx), []>,
+ Sched<[]> {
+ // Translated to the actual instructions in AArch64ISelLowering.cpp
+ let SMEMatrixType = za_flag;
+ let usesCustomInserter = 1;
+}
+
class sme2_quarter_tile_outer_product_pseudo<RegisterOperand zn_ty, RegisterOperand zm_ty, SMEMatrixTypeEnum za_flag>
: Pseudo<(outs), (ins i32imm:$tile,
zn_ty:$zn, zm_ty:$zm), []>,
@@ -281,6 +289,12 @@ class SME2_ZA_Tile_Vec_Multi_Single_Pat<string name, SDPatternOperator intrinsic
class SME2_ZA_Tile_Vec_Multi_Multi_Pat<string name, SDPatternOperator intrinsic, Operand imm_ty, ValueType vt>
: Pat<(intrinsic imm_ty:$tile, vt:$Zn1, vt:$Zn2, vt:$Zm1, vt:$Zm2),
(!cast<Instruction>(name # _PSEUDO) $tile, (REG_SEQUENCE ZPR2Mul2, vt:$Zn1, zsub0, vt:$Zn2, zsub1), (REG_SEQUENCE ZPR2Mul2, vt:$Zm1, zsub0, vt:$Zm2, zsub1))>;
+
+class SME2_ZA_TMOP_Pat<string name, SDPatternOperator intrinsic, Operand imm_ty, ValueType vt>
+ : Pat<(intrinsic imm_ty:$tile, vt:$Zn1, vt:$Zn2, vt:$Zm, (nxv16i8 ZK:$Zk), imm_ty:$idx),
+ (!cast<Instruction>(name # _PSEUDO) $tile, (REG_SEQUENCE ZPR2Mul2, vt:$Zn1, zsub0, vt:$Zn2, zsub1), $Zm, $Zk, $idx)>;
+
+
//===----------------------------------------------------------------------===//
// SME pattern match helpers.
//===----------------------------------------------------------------------===//
@@ -474,35 +488,6 @@ multiclass sme_int_outer_product_i64<bits<3> opc, string mnemonic,
def : SME_ZA_Tile_TwoPred_TwoVec_Pat<NAME, op, timm32_0_7, nxv8i1, nxv8i16>;
}
-class sme_int_sparse_outer_product_i32<bits<5> opc, RegisterOperand zn_ty, RegisterOperand zm_ty, string mnemonic>
- : I<(outs TileOp32:$ZAda),
- (ins TileOp32:$_ZAda, zn_ty:$Zn, zm_ty:$Zm, ZK:$Zk, VectorIndexS32b:$imm),
- mnemonic, "\t$ZAda, $Zn, $Zm, $Zk$imm",
- "", []>,
- Sched<[]> {
- bits<2> ZAda;
- bits<4> Zn;
- bits<5> Zm;
- bits<3> Zk;
- bits<2> imm;
- let Inst{31-25} = 0b1000000;
- let Inst{24} = opc{4};
- let Inst{23-22} = 0b01;
- let Inst{21} = opc{3};
- let Inst{20-16} = Zm;
- let Inst{15} = opc{2};
- let Inst{14} = 0b0;
- let Inst{13} = opc{1};
- let Inst{12-10} = Zk;
- let Inst{9-6} = Zn;
- let Inst{5-4} = imm;
- let Inst{3} = opc{0};
- let Inst{2} = 0b0;
- let Inst{1-0} = ZAda;
-
- let Constraints = "$ZAda = $_ZAda";
-}
-
class sme_outer_product_widening_inst<bits<3> opc, ZPRRegOp zpr_ty, string mnemonic>
: I<(outs TileOp32:$ZAda),
(ins TileOp32:$_ZAda, PPR3bAny:$Pn, PPR3bAny:$Pm, zpr_ty:$Zn, zpr_ty:$Zm),
@@ -3535,7 +3520,7 @@ multiclass sme2_int_bmopx_tile<string mnemonic, bits<3> op, SDPatternOperator i
//===----------------------------------------------------------------------===//
// SME2 Sparse Outer Product and Accumulate
-class sme_tmopa_16b<bits<5> opc, RegisterOperand zn_ty, RegisterOperand zm_ty, string mnemonic>
+class sme_int_sparse_outer_product_i16<bits<5> opc, RegisterOperand zn_ty, RegisterOperand zm_ty, string mnemonic>
: I<(outs TileOp16:$ZAda),
(ins TileOp16:$_ZAda, zn_ty:$Zn, zm_ty:$Zm, ZK:$Zk, VectorIndexS32b:$imm),
mnemonic, "\t$ZAda, $Zn, $Zm, $Zk$imm",
@@ -3564,7 +3549,7 @@ class sme_tmopa_16b<bits<5> opc, RegisterOperand zn_ty, RegisterOperand zm_ty, s
let Constraints = "$ZAda = $_ZAda";
}
-class sme_tmopa_32b<bits<5> opc, RegisterOperand zn_ty, RegisterOperand zm_ty, string mnemonic>
+class sme_int_sparse_outer_product_i32<bits<5> opc, RegisterOperand zn_ty, RegisterOperand zm_ty, string mnemonic>
: I<(outs TileOp32:$ZAda),
(ins TileOp32:$_ZAda, zn_ty:$Zn, zm_ty:$Zm, ZK:$Zk, VectorIndexS32b:$imm),
mnemonic, "\t$ZAda, $Zn, $Zm, $Zk$imm",
@@ -3593,6 +3578,25 @@ class sme_tmopa_32b<bits<5> opc, RegisterOperand zn_ty, RegisterOperand zm_ty, s
let Constraints = "$ZAda = $_ZAda";
}
+multiclass sme_tmopa_16b<bits<5> opc, RegisterOperand zn_ty, RegisterOperand zm_ty, ValueType vt, string mnemonic, string intrinsic> {
+ def NAME : sme_int_sparse_outer_product_i16<opc, zn_ty, zm_ty, mnemonic>, SMEPseudo2Instr<NAME, 1> {
+ let Uses = [FPMR, FPCR];
+ }
+
+ def NAME # _PSEUDO : sme_sparse_outer_product_pseudo<zn_ty, zm_ty, SMEMatrixTileH>, SMEPseudo2Instr<NAME, 0>;
+
+ def _ : SME2_ZA_TMOP_Pat<NAME, !cast<SDPatternOperator>(intrinsic), timm32_0_3, vt>;
+}
+
+multiclass sme_tmopa_32b<bits<5> opc, RegisterOperand zn_ty, RegisterOperand zm_ty, ValueType vt, string mnemonic, string intrinsic> {
+ def NAME : sme_int_sparse_outer_product_i32<opc, zn_ty, zm_ty, mnemonic>, SMEPseudo2Instr<NAME, 1> {
+ let Uses = [FPMR, FPCR];
+ }
+
+ def NAME # _PSEUDO : sme_sparse_outer_product_pseudo<zn_ty, zm_ty, SMEMatrixTileS>, SMEPseudo2Instr<NAME, 0>;
+
+ def : SME2_ZA_TMOP_Pat<NAME, !cast<SDPatternOperator>(intrinsic), timm32_0_3, vt>;
+}
//===----------------------------------------------------------------------===///
// SME2 Zero Lookup Table.
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/regbank-inlineasm.mir b/llvm/test/CodeGen/AArch64/GlobalISel/regbank-inlineasm.mir
index 216f94b2b51e3..12c015f30ec33 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/regbank-inlineasm.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/regbank-inlineasm.mir
@@ -57,11 +57,11 @@ tracksRegLiveness: true
body: |
bb.1:
; CHECK-LABEL: name: inlineasm_virt_reg_output
- ; CHECK: INLINEASM &"mov ${0:w}, 7", 0 /* attdialect */, 2883594 /* regdef:GPR32common */, def %0
+ ; CHECK: INLINEASM &"mov ${0:w}, 7", 0 /* attdialect */, 2818058 /* regdef:GPR32common */, def %0
; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr(s32) = COPY %0
; CHECK-NEXT: $w0 = COPY [[COPY]](s32)
; CHECK-NEXT: RET_ReallyLR implicit $w0
- INLINEASM &"mov ${0:w}, 7", 0 /* attdialect */, 2883594 /* regdef:GPR32common */, def %0:gpr32common
+ INLINEASM &"mov ${0:w}, 7", 0 /* attdialect */, 2818058 /* regdef:GPR32common */, def %0:gpr32common
%1:_(s32) = COPY %0
$w0 = COPY %1(s32)
RET_ReallyLR implicit $w0
@@ -75,7 +75,7 @@ tracksRegLiveness: true
body: |
bb.1:
; CHECK-LABEL: name: inlineasm_virt_mixed_types
- ; CHECK: INLINEASM &"mov $0, #0; mov $1, #0", 0 /* attdialect */, 2883594 /* regdef:GPR32common */, def %0, 3735562 /* regdef:FPR64 */, def %1
+ ; CHECK: INLINEASM &"mov $0, #0; mov $1, #0", 0 /* attdialect */, 2883594 /* regdef:FPR32_with_hsub_in_FPR16_lo */, def %0, 3735562 /* regdef:GPR64 */, def %1
; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr(s32) = COPY %0
; CHECK-NEXT: [[COPY1:%[0-9]+]]:fpr(s64) = COPY %1
; CHECK-NEXT: $d0 = COPY [[COPY1]](s64)
diff --git a/llvm/test/CodeGen/AArch64/emit_fneg_with_non_register_operand.mir b/llvm/test/CodeGen/AArch64/emit_fneg_with_non_register_operand.mir
index 253e6a9c076c6..251828b0b4f9b 100644
--- a/llvm/test/CodeGen/AArch64/emit_fneg_with_non_register_operand.mir
+++ b/llvm/test/CodeGen/AArch64/emit_fneg_with_non_register_operand.mir
@@ -91,10 +91,10 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[LOADgot:%[0-9]+]]:gpr64common = LOADgot target-flags(aarch64-got) @c
; CHECK-NEXT: [[LDRDui:%[0-9]+]]:fpr64 = LDRDui [[LOADgot]], 0 :: (dereferenceable load (s64) from @c)
- ; CHECK-NEXT: INLINEASM &"", 1 /* sideeffect attdialect */, 3735562 /* regdef:FPR64 */, def %2, 2147483657 /* reguse tiedto:$0 */, [[LDRDui]](tied-def 3)
+ ; CHECK-NEXT: INLINEASM &"", 1 /* sideeffect attdialect */, 3735562 /* regdef:GPR64 */, def %2, 2147483657 /* reguse tiedto:$0 */, [[LDRDui]](tied-def 3)
; CHECK-NEXT: [[COPY:%[0-9]+]]:fpr64 = COPY %2
; CHECK-NEXT: [[LDRDui1:%[0-9]+]]:fpr64 = LDRDui [[LOADgot]], 0 :: (dereferenceable load (s64) from @c)
- ; CHECK-NEXT: INLINEASM &"", 1 /* sideeffect attdialect */, 3735562 /* regdef:FPR64 */, def %4, 2147483657 /* reguse tiedto:$0 */, [[LDRDui1]](tied-def 3)
+ ; CHECK-NEXT: INLINEASM &"", 1 /* sideeffect attdialect */, 3735562 /* regdef:GPR64 */, def %4, 2147483657 /* reguse tiedto:$0 */, [[LDRDui1]](tied-def 3)
; CHECK-NEXT: [[FNEGDr:%[0-9]+]]:fpr64 = FNEGDr %2
; CHECK-NEXT: nofpexcept FCMPDrr %4, killed [[FNEGDr]], implicit-def $nzcv, implicit $fpcr
; CHECK-NEXT: Bcc 1, %bb.2, implicit $nzcv
diff --git a/llvm/test/CodeGen/AArch64/peephole-insvigpr.mir b/llvm/test/CodeGen/AArch64/peephole-insvigpr.mir
index 3174d3c8c1a73..aef01e42ed7cc 100644
--- a/llvm/test/CodeGen/AArch64/peephole-insvigpr.mir
+++ b/llvm/test/CodeGen/AArch64/peephole-insvigpr.mir
@@ -487,7 +487,7 @@ body: |
; CHECK-NEXT: [[COPY:%[0-9]+]]:gpr64common = COPY $x0
; CHECK-NEXT: [[DEF:%[0-9]+]]:gpr64all = IMPLICIT_DEF
; CHECK-NEXT: [[COPY1:%[0-9]+]]:gpr64sp = COPY [[DEF]]
- ; CHECK-NEXT: INLINEASM &"ldr ${0:s}, $1", 8 /* mayload attdialect */, 3735562 /* regdef:FPR64 */, def %1, 262158 /* mem:m */, killed [[COPY1]]
+ ; CHECK-NEXT: INLINEASM &"ldr ${0:s}, $1", 8 /* mayload attdialect */, 3670026 /* regdef:FPR64 */, def %1, 262158 /* mem:m */, killed [[COPY1]]
; CHECK-NEXT: [[MOVIv2d_ns:%[0-9]+]]:fpr128 = MOVIv2d_ns 0
; CHECK-NEXT: [[COPY2:%[0-9]+]]:fpr64 = COPY [[MOVIv2d_ns]].dsub
; CHECK-NEXT: [[DEF1:%[0-9]+]]:fpr128 = IMPLICIT_DEF
@@ -505,7 +505,7 @@ body: |
%0:gpr64common = COPY $x0
%2:gpr64all = IMPLICIT_DEF
%3:gpr64sp = COPY %2
- INLINEASM &"ldr ${0:s}, $1", 8 /* mayload attdialect */, 3735562 /* regdef:FPR64 */, def %1, 262158 /* mem:m */, killed %3
+ INLINEASM &"ldr ${0:s}, $1", 8 /* mayload attdialect */, 3670026 /* regdef:FPR64 */, def %1, 262158 /* mem:m */, killed %3
%4:fpr128 = MOVIv2d_ns 0
%5:fpr64 = COPY %4.dsub
%7:fpr128 = IMPLICIT_DEF
diff --git a/llvm/test/CodeGen/AArch64/sme2-intrinsics-tmop.ll b/llvm/test/CodeGen/AArch64/sme2-intrinsics-tmop.ll
new file mode 100644
index 0000000000000..856b1780c508c
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sme2-intrinsics-tmop.ll
@@ -0,0 +1,138 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc -force-streaming -verify-machineinstrs < %s | FileCheck %s
+
+target triple = "aarch64-linux"
+
+define void @tmopa_za32_s8(<vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zm, <vscale x 16 x i8> %zk) #0 {
+; CHECK-LABEL: tmopa_za32_s8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov z28.d, z3.d
+; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: stmopa za0.s, { z0.b, z1.b }, z2.b, z28[0]
+; CHECK-NEXT: ret
+ call void @llvm.aarch64.sme.stmopa.nxv16i8(i32 0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zm, <vscale x 16 x i8> %zk, i32 0)
+ ret void
+}
+
+define void @tmopa_za32_u8(<vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zm, <vscale x 16 x i8> %zk) #0 {
+; CHECK-LABEL: tmopa_za32_u8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov z28.d, z3.d
+; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: utmopa za0.s, { z0.b, z1.b }, z2.b, z28[0]
+; CHECK-NEXT: ret
+ call void @llvm.aarch64.sme.utmopa.nxv16i8(i32 0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zm, <vscale x 16 x i8> %zk, i32 0)
+ ret void
+}
+
+define void @tmopa_za32_u8_s8(<vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zm, <vscale x 16 x i8> %zk) #0 {
+; CHECK-LABEL: tmopa_za32_u8_s8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov z28.d, z3.d
+; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: ustmopa za0.s, { z0.b, z1.b }, z2.b, z28[0]
+; CHECK-NEXT: ret
+ call void @llvm.aarch64.sme.ustmopa.nxv16i8(i32 0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zm, <vscale x 16 x i8> %zk, i32 0)
+ ret void
+}
+
+define void @tmopa_za32_s8_u8(<vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zm, <vscale x 16 x i8> %zk) #0 {
+; CHECK-LABEL: tmopa_za32_s8_u8:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov z28.d, z3.d
+; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: sutmopa za0.s, { z0.b, z1.b }, z2.b, z28[0]
+; CHECK-NEXT: ret
+ call void @llvm.aarch64.sme.sutmopa.nxv16i8(i32 0, <vscale x 16 x i8> %zn1, <vscale x 16 x i8> %zn2, <vscale x 16 x i8> %zm, <vscale x 16 x i8> %zk, i32 0)
+ ret void
+}
+
+define void @tmopa_za32_s16(<vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zm, <vscale x 16 x i8> %zk) #0 {
+; CHECK-LABEL: tmopa_za32_s16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov z28.d, z3.d
+; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: stmopa za0.s, { z0.h, z1.h }, z2.h, z28[0]
+; CHECK-NEXT: ret
+ call void @llvm.aarch64.sme.stmopa.nxv8i16(i32 0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zm, <vscale x 16 x i8> %zk, i32 0)
+ ret void
+}
+
+define void @tmopa_za32_u16(<vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zm, <vscale x 16 x i8> %zk) #0 {
+; CHECK-LABEL: tmopa_za32_u16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov z28.d, z3.d
+; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: utmopa za0.s, { z0.h, z1.h }, z2.h, z28[0]
+; CHECK-NEXT: ret
+ call void @llvm.aarch64.sme.utmopa.nxv8i16(i32 0, <vscale x 8 x i16> %zn1, <vscale x 8 x i16> %zn2, <vscale x 8 x i16> %zm, <vscale x 16 x i8> %zk, i32 0)
+ ret void
+}
+
+define void @tmopa_za32_f16(<vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2, <vscale x 8 x half> %zm, <vscale x 16 x i8> %zk) #0 {
+; CHECK-LABEL: tmopa_za32_f16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov z28.d, z3.d
+; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: ftmopa za0.s, { z0.h, z1.h }, z2.h, z28[0]
+; CHECK-NEXT: ret
+ call void @llvm.aarch64.sme.tmopa.nxv8f16(i32 0, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2, <vscale x 8 x half> %zm, <vscale x 16 x i8> %zk, i32 0)
+ ret void
+}
+
+define void @tmopa_za32_bf16(<vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zm, <vscale x 16 x i8> %zk) #0 {
+; CHECK-LABEL: tmopa_za32_bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov z28.d, z3.d
+; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: bftmopa za0.s, { z0.h, z1.h }, z2.h, z28[0]
+; CHECK-NEXT: ret
+ call void @llvm.aarch64.sme.tmopa.nxv8bf16(i32 0, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zm, <vscale x 16 x i8> %zk, i32 0)
+ ret void
+}
+
+define void @tmopa_za32_f32(<vscale x 4 x float> %zn1, <vscale x 4 x float> %zn2, <vscale x 4 x float> %zm, <vscale x 16 x i8> %zk) #0 {
+; CHECK-LABEL: tmopa_za32_f32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov z28.d, z3.d
+; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: ftmopa za0.s, { z0.s, z1.s }, z2.s, z28[0]
+; CHECK-NEXT: ret
+ call void @llvm.aarch64.sme.tmopa.nxv4f32(i32 0, <vscale x 4 x float> %zn1, <vscale x 4 x float> %zn2, <vscale x 4 x float> %zm, <vscale x 16 x i8> %zk, i32 0)
+ ret void
+}
+
+define void @tmopa_za16_f16(<vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2, <vscale x 8 x half> %zm, <vscale x 16 x i8> %zk) #0 {
+; CHECK-LABEL: tmopa_za16_f16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov z28.d, z3.d
+; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: ftmopa za0.s, { z0.h, z1.h }, z2.h, z28[0]
+; CHECK-NEXT: ret
+ call void @llvm.aarch64.sme.tmopa.nxv8f16(i32 0, <vscale x 8 x half> %zn1, <vscale x 8 x half> %zn2, <vscale x 8 x half> %zm, <vscale x 16 x i8> %zk, i32 0)
+ ret void
+}
+
+define void @tmopa_za16_bf16(<vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zm, <vscale x 16 x i8> %zk) #0 {
+; CHECK-LABEL: tmopa_za16_bf16:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov z28.d, z3.d
+; CHECK-NEXT: // kill: def $z1 killed $z1 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: // kill: def $z0 killed $z0 killed $z0_z1 def $z0_z1
+; CHECK-NEXT: bftmopa za0.s, { z0.h, z1.h }, z2.h, z28[0]
+; CHECK-NEXT: ret
+ call void @llvm.aarch64.sme.tmopa.nxv8bf16(i32 0, <vscale x 8 x bfloat> %zn1, <vscale x 8 x bfloat> %zn2, <vscale x 8 x bfloat> %zm, <vscale x 16 x i8> %zk, i32 0)
+ ret void
+}
+
+attributes #0 = {nounwind "target-features" = "+sme2,+sme-tmop,+sme-f16f16,+sme-b16b16,+sme-f8f16,+sme-f8f32,+sme2p1,+bf16" }
>From 2624dc6222b93efb603163575d547893c0995876 Mon Sep 17 00:00:00 2001
From: Jonathan Thackray <jonathan.thackray at arm.com>
Date: Fri, 11 Apr 2025 09:44:22 +0000
Subject: [PATCH 2/5] fixup! Fixup code review issues for TMOP
---
clang/include/clang/Basic/arm_sme.td | 8 +-
.../AArch64/sme2-intrinsics/acle_sme2_tmop.c | 33 +++++--
.../acle_sme2_tmop.cpp | 87 +++++++++++--------
llvm/lib/Target/AArch64/SMEInstrFormats.td | 2 +-
4 files changed, 83 insertions(+), 47 deletions(-)
diff --git a/clang/include/clang/Basic/arm_sme.td b/clang/include/clang/Basic/arm_sme.td
index d425c2c1e38bb..a0d66bb038108 100644
--- a/clang/include/clang/Basic/arm_sme.td
+++ b/clang/include/clang/Basic/arm_sme.td
@@ -934,19 +934,19 @@ let SMETargetGuard = "sme2,sme-tmop" in {
}
let SMETargetGuard = "sme2,sme-tmop,sme-f16f16" in {
- def SVTMOPA_F16 : Inst<"svtmopa_lane_za16[_{d}_{d}]", "vi2d[i", "hb", MergeNone, "aarch64_sme_tmopa", [IsStreaming, IsInOutZA], [ImmCheck<0, ImmCheck0_3>, ImmCheck<4, ImmCheck0_3>]>;
+ def SVTMOPA_F16 : Inst<"svtmopa_lane_za16[_{d}_{d}]", "vi2d[i", "hb", MergeNone, "aarch64_sme_tmopa", [IsStreaming, IsInOutZA], [ImmCheck<0, ImmCheck0_1>, ImmCheck<4, ImmCheck0_3>]>;
}
let SMETargetGuard = "sme2,sme-tmop,sme-b16b16" in {
- def SVTMOPA_BF16 : Inst<"svtmopa_lane_za16[_{d}_{d}]", "vi2d[i", "U", MergeNone, "aarch64_sme_tmopa", [IsStreaming, IsInOutZA], [ImmCheck<0, ImmCheck0_3>, ImmCheck<4, ImmCheck0_3>]>;
+ def SVTMOPA_BF16 : Inst<"svtmopa_lane_za16[_{d}_{d}]", "vi2d[i", "U", MergeNone, "aarch64_sme_tmopa", [IsStreaming, IsInOutZA], [ImmCheck<0, ImmCheck0_1>, ImmCheck<4, ImmCheck0_3>]>;
}
let SMETargetGuard = "sme2,sme-tmop,sme-f8f16" in {
- def SVTMOPA_ZA16_FPM : Inst<"svtmopa_lane_za16[_{d}_{d}]", "vi2.MM[i>", "m", MergeNone, "aarch64_sme_tmopa", [IsStreaming, IsInOutZA], [ImmCheck<0, ImmCheck0_3>, ImmCheck<4, ImmCheck0_3>]>;
+ def SVTMOPA_ZA16_FPM : Inst<"svtmopa_lane_za16[_{d}_{d}]", "vi2.~~[i>", "m", MergeNone, "aarch64_sme_tmopa", [IsStreaming, IsInOutZA], [ImmCheck<0, ImmCheck0_1>, ImmCheck<4, ImmCheck0_3>]>;
}
let SMETargetGuard = "sme2,sme-tmop,sme-f8f32" in {
- def SVTMOPA_ZA32_FPM : Inst<"svtmopa_lane_za32[_{d}_{d}]", "vi2.MM[i>", "m", MergeNone, "aarch64_sme_tmopa", [IsStreaming, IsInOutZA], [ImmCheck<0, ImmCheck0_3>, ImmCheck<4, ImmCheck0_3>]>;
+ def SVTMOPA_ZA32_FPM : Inst<"svtmopa_lane_za32[_{d}_{d}]", "vi2.~~[i>", "m", MergeNone, "aarch64_sme_tmopa", [IsStreaming, IsInOutZA], [ImmCheck<0, ImmCheck0_3>, ImmCheck<4, ImmCheck0_3>]>;
}
multiclass ZAReadz<string n_suffix, string vg_num, string t, string i_prefix, list<ImmCheck> ch> {
diff --git a/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_tmop.c b/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_tmop.c
index a1456275dd810..85c24c0843409 100644
--- a/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_tmop.c
+++ b/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_tmop.c
@@ -76,6 +76,15 @@ void test_svtmopa_lane_za32_f16_f16(svfloat16x2_t zn, svfloat16_t zm, svuint8_t
SME_ACLE_FUNC(svtmopa_lane_za32,_f16_f16,)(1, zn, zm, zk, 3);
}
+// CHECK-LABEL: @test_svtmopa_lane_za32_f32_f32(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.aarch64.sme.tmopa.nxv4f32(i32 1, <vscale x 4 x float> [[ZN_COERCE0:%.*]], <vscale x 4 x float> [[ZN_COERCE1:%.*]], <vscale x 4 x float> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CHECK-NEXT: ret void
+//
+void test_svtmopa_lane_za32_f32_f32(svfloat32x2_t zn, svfloat32_t zm, svuint8_t zk) __arm_streaming __arm_inout("za") {
+ SME_ACLE_FUNC(svtmopa_lane_za32,_f32_f32,)(1, zn, zm, zk, 3);
+}
+
// CHECK-LABEL: @test_svtmopa_lane_za32_bf16_bf16(
// CHECK-NEXT: entry:
// CHECK-NEXT: tail call void @llvm.aarch64.sme.tmopa.nxv8bf16(i32 1, <vscale x 8 x bfloat> [[ZN_COERCE0:%.*]], <vscale x 8 x bfloat> [[ZN_COERCE1:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
@@ -103,10 +112,22 @@ void test_svtmopa_lane_za16_bf16_bf16(svbfloat16x2_t zn, svbfloat16_t zm, svuint
SME_ACLE_FUNC(svtmopa_lane_za16,_bf16_bf16,)(1, zn, zm, zk, 3);
}
-//void test_svtmopa_lane_za16_mf8_mf8_fpm(svfloat32x2_t zn, svfloat32_t zm, svuint8_t zk, fpm_t fpmr) __arm_streaming __arm_inout("za") {
-// SME_ACLE_FUNC(svtmopa_lane_za16,_mf8_mf8,_fpm)(1, zn, zm, zk, 3, fpmr);
-//}
+// CHECK-LABEL: @test_svtmopa_lane_za16_mf8_mf8_fpm(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.aarch64.set.fpmr(i64 [[FPMR:%.*]])
+// CHECK-NEXT: tail call void @llvm.aarch64.sme.tmopa.nxv16i8(i32 1, <vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CHECK-NEXT: ret void
+//
+void test_svtmopa_lane_za16_mf8_mf8_fpm(svmfloat8x2_t zn, svmfloat8_t zm, svuint8_t zk, fpm_t fpmr) __arm_streaming __arm_inout("za") {
+ SME_ACLE_FUNC(svtmopa_lane_za16,_mf8_mf8,_fpm)(1, zn, zm, zk, 3, fpmr);
+}
-//void test_svtmopa_lane_za32_mf8_mf8_fpm(svfloat32x2_t zn, svfloat32_t zm, svuint8_t zk, fpm_t fpmr) __arm_streaming __arm_inout("za") {
-// SME_ACLE_FUNC(svtmopa_lane_za32,_mf8_mf8,_fpm)(1, zn, zm, zk, 3, fpmr);
-//}
+// CHECK-LABEL: @test_svtmopa_lane_za32_mf8_mf8_fpm(
+// CHECK-NEXT: entry:
+// CHECK-NEXT: tail call void @llvm.aarch64.set.fpmr(i64 [[FPMR:%.*]])
+// CHECK-NEXT: tail call void @llvm.aarch64.sme.tmopa.nxv16i8(i32 1, <vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CHECK-NEXT: ret void
+//
+void test_svtmopa_lane_za32_mf8_mf8_fpm(svmfloat8x2_t zn, svmfloat8_t zm, svuint8_t zk, fpm_t fpmr) __arm_streaming __arm_inout("za") {
+ SME_ACLE_FUNC(svtmopa_lane_za32,_mf8_mf8,_fpm)(1, zn, zm, zk, 3, fpmr);
+}
diff --git a/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_tmop.cpp b/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_tmop.cpp
index 6ccd3acaf749f..afa650950d61e 100644
--- a/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_tmop.cpp
+++ b/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_tmop.cpp
@@ -19,6 +19,8 @@ void test_features() __arm_streaming __arm_inout("za") {
svfloat16_t zm_f16;
svfloat32x2_t zn_f32;
svfloat32_t zm_f32;
+ svmfloat8x2_t zn_f8;
+ svmfloat8_t zm_f8;
fpm_t fpm = 0;
svuint8_t zk;
@@ -36,6 +38,8 @@ void test_features() __arm_streaming __arm_inout("za") {
svtmopa_lane_za32_u16_u16(0, zn_u16, zm_u16, zk, 0);
// expected-error at +1 {{'svtmopa_lane_za32_f16_f16' needs target feature sme,sme2,sme-tmop}}
svtmopa_lane_za32_f16_f16(0, zn_f16, zm_f16, zk, 0);
+// expected-error at +1 {{'svtmopa_lane_za32_f32_f32' needs target feature sme,sme2,sme-tmop}}
+ svtmopa_lane_za32_f32_f32(0, zn_f32, zm_f32, zk, 0);
// expected-error at +1 {{'svtmopa_lane_za32_bf16_bf16' needs target feature sme,sme2,sme-tmop}}
svtmopa_lane_za32_bf16_bf16(0, zn_bf16, zm_bf16, zk, 0);
// expected-error at +1 {{'svtmopa_lane_za16_f16_f16' needs target feature sme,sme2,sme-tmop,sme-f16f16}}
@@ -43,9 +47,9 @@ void test_features() __arm_streaming __arm_inout("za") {
// expected-error at +1 {{'svtmopa_lane_za16_bf16_bf16' needs target feature sme,sme2,sme-tmop,sme-f16f16}}
svtmopa_lane_za16_bf16_bf16(0, zn_bf16, zm_bf16, zk, 0);
// expected-error at +1 {{'svtmopa_lane_za16_mf8_mf8_fpm' needs target feature sme,sme2,sme-tmop,sme-f8f16}}
- svtmopa_lane_za16_mf8_mf8_fpm(0, zn_f32, zm_f32, zk, 0, fpm);
+ svtmopa_lane_za16_mf8_mf8_fpm(0, zn_f8, zm_f8, zk, 0, fpm);
// expected-error at +1 {{'svtmopa_lane_za32_mf8_mf8_fpm' needs target feature sme,sme2,sme-tmop,sme-f8f32}}
- svtmopa_lane_za32_mf8_mf8_fpm(0, zn_f32, zm_f32, zk, 0, fpm);
+ svtmopa_lane_za32_mf8_mf8_fpm(0, zn_f8, zm_f8, zk, 0, fpm);
}
void test_imm() __arm_streaming __arm_inout("za") {
@@ -63,114 +67,125 @@ void test_imm() __arm_streaming __arm_inout("za") {
svfloat16_t zm_f16;
svfloat32x2_t zn_f32;
svfloat32_t zm_f32;
+ svmfloat8x2_t zn_f8;
+ svmfloat8_t zm_f8;
fpm_t fpm;
svuint8_t zk;
// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
- svtmopa_lane_za32_s8_s8(3, zn_s8, zm_s8, zk, 4);
+ svtmopa_lane_za32_s8_s8(0, zn_s8, zm_s8, zk, 4);
// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
- svtmopa_lane_za32_s8_s8(4, zn_s8, zm_s8, zk, 3);
+ svtmopa_lane_za32_s8_s8(4, zn_s8, zm_s8, zk, 0);
// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
svtmopa_lane_za32_s8_s8(0, zn_s8, zm_s8, zk, -1);
// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
svtmopa_lane_za32_s8_s8(-1, zn_s8, zm_s8, zk, 0);
// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
- svtmopa_lane_za32_u8_u8(3, zn_u8, zm_u8, zk, 4);
+ svtmopa_lane_za32_u8_u8(0, zn_u8, zm_u8, zk, 4);
// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
- svtmopa_lane_za32_u8_u8(4, zn_u8, zm_u8, zk, 3);
+ svtmopa_lane_za32_u8_u8(4, zn_u8, zm_u8, zk, 0);
// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
svtmopa_lane_za32_u8_u8(0, zn_u8, zm_u8, zk, -1);
// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
svtmopa_lane_za32_u8_u8(-1, zn_u8, zm_u8, zk, 0);
// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
- svtmopa_lane_za32_s8_u8(3, zn_s8, zm_u8, zk, 4);
+ svtmopa_lane_za32_s8_u8(0, zn_s8, zm_u8, zk, 4);
// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
- svtmopa_lane_za32_s8_u8(4, zn_s8, zm_u8, zk, 3);
+ svtmopa_lane_za32_s8_u8(4, zn_s8, zm_u8, zk, 0);
// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
svtmopa_lane_za32_s8_u8(0, zn_s8, zm_u8, zk, -1);
// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
svtmopa_lane_za32_s8_u8(-1, zn_s8, zm_u8, zk, 0);
// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
- svtmopa_lane_za32_u8_s8(3, zn_u8, zm_s8, zk, 4);
+ svtmopa_lane_za32_u8_s8(0, zn_u8, zm_s8, zk, 4);
// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
- svtmopa_lane_za32_u8_s8(4, zn_u8, zm_s8, zk, 3);
+ svtmopa_lane_za32_u8_s8(4, zn_u8, zm_s8, zk, 0);
// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
svtmopa_lane_za32_u8_s8(0, zn_u8, zm_s8, zk, -1);
// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
svtmopa_lane_za32_u8_s8(-1, zn_u8, zm_s8, zk, 0);
// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
- svtmopa_lane_za32_s16_s16(3, zn_s16, zm_s16, zk, 4);
+ svtmopa_lane_za32_s16_s16(0, zn_s16, zm_s16, zk, 4);
// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
- svtmopa_lane_za32_s16_s16(4, zn_s16, zm_s16, zk, 3);
+ svtmopa_lane_za32_s16_s16(4, zn_s16, zm_s16, zk, 0);
// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
svtmopa_lane_za32_s16_s16(0, zn_s16, zm_s16, zk, -1);
// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
svtmopa_lane_za32_s16_s16(-1, zn_s16, zm_s16, zk, 0);
// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
- svtmopa_lane_za32_u16_u16(3, zn_u16, zm_u16, zk, 4);
+ svtmopa_lane_za32_u16_u16(0, zn_u16, zm_u16, zk, 4);
// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
- svtmopa_lane_za32_u16_u16(4, zn_u16, zm_u16, zk, 3);
+ svtmopa_lane_za32_u16_u16(4, zn_u16, zm_u16, zk, 0);
// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
svtmopa_lane_za32_u16_u16(0, zn_u16, zm_u16, zk, -1);
// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
svtmopa_lane_za32_u16_u16(-1, zn_u16, zm_u16, zk, 0);
// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
- svtmopa_lane_za32_f16_f16(3, zn_f16, zm_f16, zk, 4);
+ svtmopa_lane_za32_f16_f16(0, zn_f16, zm_f16, zk, 4);
// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
- svtmopa_lane_za32_f16_f16(4, zn_f16, zm_f16, zk, 3);
+ svtmopa_lane_za32_f16_f16(4, zn_f16, zm_f16, zk, 0);
// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
svtmopa_lane_za32_f16_f16(0, zn_f16, zm_f16, zk, -1);
// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
svtmopa_lane_za32_f16_f16(-1, zn_f16, zm_f16, zk, 0);
// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
- svtmopa_lane_za32_bf16_bf16(3, zn_bf16, zm_bf16, zk, 4);
+ svtmopa_lane_za32_f32_f32(0, zn_f32, zm_f32, zk, 4);
// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
- svtmopa_lane_za32_bf16_bf16(4, zn_bf16, zm_bf16, zk, 3);
+ svtmopa_lane_za32_f32_f32(4, zn_f32, zm_f32, zk, 0);
+// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
+ svtmopa_lane_za32_f32_f32(0, zn_f32, zm_f32, zk, -1);
+// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
+ svtmopa_lane_za32_f32_f32(-1, zn_f32, zm_f32, zk, 0);
+
+// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
+ svtmopa_lane_za32_bf16_bf16(0, zn_bf16, zm_bf16, zk, 4);
+// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
+ svtmopa_lane_za32_bf16_bf16(4, zn_bf16, zm_bf16, zk, 0);
// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
svtmopa_lane_za32_bf16_bf16(0, zn_bf16, zm_bf16, zk, -1);
// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
svtmopa_lane_za32_bf16_bf16(-1, zn_bf16, zm_bf16, zk, 0);
// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
- svtmopa_lane_za16_f16_f16(3, zn_f16, zm_f16, zk, 4);
-// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
- svtmopa_lane_za16_f16_f16(4, zn_f16, zm_f16, zk, 3);
+ svtmopa_lane_za16_f16_f16(0, zn_f16, zm_f16, zk, 4);
+// expected-error at +1 {{argument value 2 is outside the valid range [0, 1]}}
+ svtmopa_lane_za16_f16_f16(2, zn_f16, zm_f16, zk, 0);
// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
svtmopa_lane_za16_f16_f16(0, zn_f16, zm_f16, zk, -1);
-// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
+// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 1]}}
svtmopa_lane_za16_f16_f16(-1, zn_f16, zm_f16, zk, 0);
// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
- svtmopa_lane_za16_bf16_bf16(3, zn_bf16, zm_bf16, zk, 4);
-// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
- svtmopa_lane_za16_bf16_bf16(4, zn_bf16, zm_bf16, zk, 3);
+ svtmopa_lane_za16_bf16_bf16(0, zn_bf16, zm_bf16, zk, 4);
+// expected-error at +1 {{argument value 2 is outside the valid range [0, 1]}}
+ svtmopa_lane_za16_bf16_bf16(2, zn_bf16, zm_bf16, zk, 0);
// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
svtmopa_lane_za16_bf16_bf16(0, zn_bf16, zm_bf16, zk, -1);
-// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
+// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 1]}}
svtmopa_lane_za16_bf16_bf16(-1, zn_bf16, zm_bf16, zk, 0);
// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
- svtmopa_lane_za16_mf8_mf8_fpm(3, zn_f32, zm_f32, zk, 4, fpm);
-// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
- svtmopa_lane_za16_mf8_mf8_fpm(4, zn_f32, zm_f32, zk, 3, fpm);
-// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
- svtmopa_lane_za16_mf8_mf8_fpm(0, zn_f32, zm_f32, zk, -1, fpm);
+ svtmopa_lane_za16_mf8_mf8_fpm(0, zn_f8, zm_f8, zk, 4, fpm);
+// expected-error at +1 {{argument value 2 is outside the valid range [0, 1]}}
+ svtmopa_lane_za16_mf8_mf8_fpm(2, zn_f8, zm_f8, zk, 0, fpm);
// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
- svtmopa_lane_za16_mf8_mf8_fpm(-1, zn_f32, zm_f32, zk, 0, fpm);
+ svtmopa_lane_za16_mf8_mf8_fpm(0, zn_f8, zm_f8, zk, -1, fpm);
+// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 1]}}
+ svtmopa_lane_za16_mf8_mf8_fpm(-1, zn_f8, zm_f8, zk, 0, fpm);
// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
- svtmopa_lane_za32_mf8_mf8_fpm(3, zn_f32, zm_f32, zk, 4, fpm);
+ svtmopa_lane_za32_mf8_mf8_fpm(0, zn_f8, zm_f8, zk, 4, fpm);
// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
- svtmopa_lane_za32_mf8_mf8_fpm(4, zn_f32, zm_f32, zk, 3, fpm);
+ svtmopa_lane_za32_mf8_mf8_fpm(4, zn_f8, zm_f8, zk, 0, fpm);
// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
- svtmopa_lane_za32_mf8_mf8_fpm(0, zn_f32, zm_f32, zk, -1, fpm);
+ svtmopa_lane_za32_mf8_mf8_fpm(0, zn_f8, zm_f8, zk, -1, fpm);
// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
- svtmopa_lane_za32_mf8_mf8_fpm(-1, zn_f32, zm_f32, zk, 0, fpm);
+ svtmopa_lane_za32_mf8_mf8_fpm(-1, zn_f8, zm_f8, zk, 0, fpm);
}
diff --git a/llvm/lib/Target/AArch64/SMEInstrFormats.td b/llvm/lib/Target/AArch64/SMEInstrFormats.td
index c42299b050eb6..a85bc344220f2 100644
--- a/llvm/lib/Target/AArch64/SMEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SMEInstrFormats.td
@@ -3585,7 +3585,7 @@ multiclass sme_tmopa_16b<bits<5> opc, RegisterOperand zn_ty, RegisterOperand zm_
def NAME # _PSEUDO : sme_sparse_outer_product_pseudo<zn_ty, zm_ty, SMEMatrixTileH>, SMEPseudo2Instr<NAME, 0>;
- def _ : SME2_ZA_TMOP_Pat<NAME, !cast<SDPatternOperator>(intrinsic), timm32_0_3, vt>;
+ def : SME2_ZA_TMOP_Pat<NAME, !cast<SDPatternOperator>(intrinsic), timm32_0_3, vt>;
}
multiclass sme_tmopa_32b<bits<5> opc, RegisterOperand zn_ty, RegisterOperand zm_ty, ValueType vt, string mnemonic, string intrinsic> {
>From bf4bb103febe23b2d94397f1d1ab5c19cbcfc1e9 Mon Sep 17 00:00:00 2001
From: Jonathan Thackray <jonathan.thackray at arm.com>
Date: Fri, 11 Apr 2025 10:54:57 +0000
Subject: [PATCH 3/5] fixup! Add extra parameter to SME2_ZA_TMOP_Pat for
TileOp16 and VectorIndexS32b
---
llvm/lib/Target/AArch64/SMEInstrFormats.td | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Target/AArch64/SMEInstrFormats.td b/llvm/lib/Target/AArch64/SMEInstrFormats.td
index a85bc344220f2..20f40b54a3122 100644
--- a/llvm/lib/Target/AArch64/SMEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SMEInstrFormats.td
@@ -290,8 +290,8 @@ class SME2_ZA_Tile_Vec_Multi_Multi_Pat<string name, SDPatternOperator intrinsic,
: Pat<(intrinsic imm_ty:$tile, vt:$Zn1, vt:$Zn2, vt:$Zm1, vt:$Zm2),
(!cast<Instruction>(name # _PSEUDO) $tile, (REG_SEQUENCE ZPR2Mul2, vt:$Zn1, zsub0, vt:$Zn2, zsub1), (REG_SEQUENCE ZPR2Mul2, vt:$Zm1, zsub0, vt:$Zm2, zsub1))>;
-class SME2_ZA_TMOP_Pat<string name, SDPatternOperator intrinsic, Operand imm_ty, ValueType vt>
- : Pat<(intrinsic imm_ty:$tile, vt:$Zn1, vt:$Zn2, vt:$Zm, (nxv16i8 ZK:$Zk), imm_ty:$idx),
+class SME2_ZA_TMOP_Pat<string name, SDPatternOperator intrinsic, Operand tile_imm, Operand imm_ty, ValueType vt>
+ : Pat<(intrinsic tile_imm:$tile, vt:$Zn1, vt:$Zn2, vt:$Zm, (nxv16i8 ZK:$Zk), imm_ty:$idx),
(!cast<Instruction>(name # _PSEUDO) $tile, (REG_SEQUENCE ZPR2Mul2, vt:$Zn1, zsub0, vt:$Zn2, zsub1), $Zm, $Zk, $idx)>;
@@ -3585,7 +3585,7 @@ multiclass sme_tmopa_16b<bits<5> opc, RegisterOperand zn_ty, RegisterOperand zm_
def NAME # _PSEUDO : sme_sparse_outer_product_pseudo<zn_ty, zm_ty, SMEMatrixTileH>, SMEPseudo2Instr<NAME, 0>;
- def : SME2_ZA_TMOP_Pat<NAME, !cast<SDPatternOperator>(intrinsic), timm32_0_3, vt>;
+ def : SME2_ZA_TMOP_Pat<NAME, !cast<SDPatternOperator>(intrinsic), timm32_0_1, timm32_0_3, vt>;
}
multiclass sme_tmopa_32b<bits<5> opc, RegisterOperand zn_ty, RegisterOperand zm_ty, ValueType vt, string mnemonic, string intrinsic> {
@@ -3595,7 +3595,7 @@ multiclass sme_tmopa_32b<bits<5> opc, RegisterOperand zn_ty, RegisterOperand zm_
def NAME # _PSEUDO : sme_sparse_outer_product_pseudo<zn_ty, zm_ty, SMEMatrixTileS>, SMEPseudo2Instr<NAME, 0>;
- def : SME2_ZA_TMOP_Pat<NAME, !cast<SDPatternOperator>(intrinsic), timm32_0_3, vt>;
+ def : SME2_ZA_TMOP_Pat<NAME, !cast<SDPatternOperator>(intrinsic), timm32_0_1, timm32_0_3, vt>;
}
//===----------------------------------------------------------------------===///
>From 51462fa9deb4547e1528ba49cf4caa8a8a04314b Mon Sep 17 00:00:00 2001
From: Jonathan Thackray <jonathan.thackray at arm.com>
Date: Fri, 11 Apr 2025 21:51:28 +0000
Subject: [PATCH 4/5] fixup! Fix Marian's code review comments
---
clang/include/clang/Basic/arm_sme.td | 33 +++-----
.../AArch64/sme2-intrinsics/acle_sme2_tmop.c | 75 ++++++++++++++++++-
.../acle_sme2_tmop.cpp | 61 ++++++---------
llvm/lib/Target/AArch64/SMEInstrFormats.td | 2 +-
4 files changed, 103 insertions(+), 68 deletions(-)
diff --git a/clang/include/clang/Basic/arm_sme.td b/clang/include/clang/Basic/arm_sme.td
index a0d66bb038108..037a7eaa39a34 100644
--- a/clang/include/clang/Basic/arm_sme.td
+++ b/clang/include/clang/Basic/arm_sme.td
@@ -910,43 +910,28 @@ let SMETargetGuard = "sme-f16f16" in {
////////////////////////////////////////////////////////////////////////////////
// SME2 - TMOP, SUTMOP, USTMOP
-multiclass USTMOP<string za, string t, list<ImmCheck> checks> {
- def _ : SInst<"svtmopa_lane_" # za # "[_{d}_{3}]",
- "vi2x[i", t, MergeNone, "aarch64_sme_ustmopa",
- [IsStreaming, IsInOutZA],
- checks>;
-}
-
-multiclass SUTMOP<string za, string t, list<ImmCheck> checks> {
- def _ : SInst<"svtmopa_lane_" # za # "[_{d}_{3}]",
- "vi2u[i", t, MergeNone, "aarch64_sme_sutmopa",
- [IsStreaming, IsInOutZA],
- checks>;
-}
-
let SMETargetGuard = "sme2,sme-tmop" in {
- def SVTMOPA_ZA32 : Inst<"svtmopa_lane_za32[_{d}_{d}]", "vi2d[i", "hbf", MergeNone, "aarch64_sme_tmopa", [IsStreaming, IsInOutZA], [ImmCheck<0, ImmCheck0_3>, ImmCheck<4, ImmCheck0_3>]>;
- def SVSTMOPA_ZA32 : Inst<"svtmopa_lane_za32[_{d}_{d}]", "vi2d[i", "cs", MergeNone, "aarch64_sme_stmopa", [IsStreaming, IsInOutZA], [ImmCheck<0, ImmCheck0_3>, ImmCheck<4, ImmCheck0_3>]>;
- def SVUTMOPA_ZA32 : Inst<"svtmopa_lane_za32[_{d}_{d}]", "vi2d[i", "UcUs", MergeNone, "aarch64_sme_utmopa", [IsStreaming, IsInOutZA], [ImmCheck<0, ImmCheck0_3>, ImmCheck<4, ImmCheck0_3>]>;
-
- defm SVSUTMOPA_S : SUTMOP<"za32", "c", [ImmCheck<0, ImmCheck0_3>, ImmCheck<4, ImmCheck0_3>]>;
- defm SVUSTMOPA_S : USTMOP<"za32", "Uc", [ImmCheck<0, ImmCheck0_3>, ImmCheck<4, ImmCheck0_3>]>;
+ def SVTMOPA_ZA32 : Inst<"svtmopa_lane_za32[_{d}_{d}]", "vi2d[i", "hbf", MergeNone, "aarch64_sme_tmopa", [IsStreaming, IsInOutZA], [ImmCheck<0, ImmCheck0_3>, ImmCheck<4, ImmCheck0_3>]>;
+ def SVSTMOPA_ZA32 : Inst<"svtmopa_lane_za32[_{d}_{d}]", "vi2d[i", "cs", MergeNone, "aarch64_sme_stmopa", [IsStreaming, IsInOutZA], [ImmCheck<0, ImmCheck0_3>, ImmCheck<4, ImmCheck0_3>]>;
+ def SVUTMOPA_ZA32 : Inst<"svtmopa_lane_za32[_{d}_{d}]", "vi2d[i", "UcUs", MergeNone, "aarch64_sme_utmopa", [IsStreaming, IsInOutZA], [ImmCheck<0, ImmCheck0_3>, ImmCheck<4, ImmCheck0_3>]>;
+ def SVSUTMOPA_ZA32 : Inst<"svtmopa_lane_za32[_{d}_{3}]", "vi2u[i", "c", MergeNone, "aarch64_sme_sutmopa", [IsStreaming, IsInOutZA], [ImmCheck<0, ImmCheck0_3>, ImmCheck<4, ImmCheck0_3>]>;
+ def SVUSTMOPA_ZA32 : Inst<"svtmopa_lane_za32[_{d}_{3}]", "vi2x[i", "Uc", MergeNone, "aarch64_sme_ustmopa", [IsStreaming, IsInOutZA], [ImmCheck<0, ImmCheck0_3>, ImmCheck<4, ImmCheck0_3>]>;
}
let SMETargetGuard = "sme2,sme-tmop,sme-f16f16" in {
- def SVTMOPA_F16 : Inst<"svtmopa_lane_za16[_{d}_{d}]", "vi2d[i", "hb", MergeNone, "aarch64_sme_tmopa", [IsStreaming, IsInOutZA], [ImmCheck<0, ImmCheck0_1>, ImmCheck<4, ImmCheck0_3>]>;
+ def SVTMOPA_F16 : Inst<"svtmopa_lane_za16[_{d}_{d}]", "vi2d[i", "h", MergeNone, "aarch64_sme_tmopa", [IsStreaming, IsInOutZA], [ImmCheck<0, ImmCheck0_1>, ImmCheck<4, ImmCheck0_3>]>;
}
let SMETargetGuard = "sme2,sme-tmop,sme-b16b16" in {
- def SVTMOPA_BF16 : Inst<"svtmopa_lane_za16[_{d}_{d}]", "vi2d[i", "U", MergeNone, "aarch64_sme_tmopa", [IsStreaming, IsInOutZA], [ImmCheck<0, ImmCheck0_1>, ImmCheck<4, ImmCheck0_3>]>;
+ def SVTMOPA_BF16 : Inst<"svtmopa_lane_za16[_{d}_{d}]", "vi2d[i", "b", MergeNone, "aarch64_sme_tmopa", [IsStreaming, IsInOutZA], [ImmCheck<0, ImmCheck0_1>, ImmCheck<4, ImmCheck0_3>]>;
}
let SMETargetGuard = "sme2,sme-tmop,sme-f8f16" in {
- def SVTMOPA_ZA16_FPM : Inst<"svtmopa_lane_za16[_{d}_{d}]", "vi2.~~[i>", "m", MergeNone, "aarch64_sme_tmopa", [IsStreaming, IsInOutZA], [ImmCheck<0, ImmCheck0_1>, ImmCheck<4, ImmCheck0_3>]>;
+ def SVTMOPA_ZA16_FPM : Inst<"svtmopa_lane_za16[_{d}_{d}]", "vi2.dd[i>", "m", MergeNone, "aarch64_sme_tmopa", [IsStreaming, IsInOutZA], [ImmCheck<0, ImmCheck0_1>, ImmCheck<4, ImmCheck0_3>]>;
}
let SMETargetGuard = "sme2,sme-tmop,sme-f8f32" in {
- def SVTMOPA_ZA32_FPM : Inst<"svtmopa_lane_za32[_{d}_{d}]", "vi2.~~[i>", "m", MergeNone, "aarch64_sme_tmopa", [IsStreaming, IsInOutZA], [ImmCheck<0, ImmCheck0_3>, ImmCheck<4, ImmCheck0_3>]>;
+ def SVTMOPA_ZA32_FPM : Inst<"svtmopa_lane_za32[_{d}_{d}]", "vi2.dd[i>", "m", MergeNone, "aarch64_sme_tmopa", [IsStreaming, IsInOutZA], [ImmCheck<0, ImmCheck0_3>, ImmCheck<4, ImmCheck0_3>]>;
}
multiclass ZAReadz<string n_suffix, string vg_num, string t, string i_prefix, list<ImmCheck> ch> {
diff --git a/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_tmop.c b/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_tmop.c
index 85c24c0843409..37c24c1672911 100644
--- a/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_tmop.c
+++ b/clang/test/CodeGen/AArch64/sme2-intrinsics/acle_sme2_tmop.c
@@ -1,9 +1,11 @@
// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py
// REQUIRES: aarch64-registered-target
-// RUN: %clang_cc1 -triple aarch64 -target-feature +bf16 -target-feature +sme-tmop -target-feature +sme-f16f16 -target-feature +sme-b16b16 -target-feature +sme-f8f16 -target-feature +sme-f8f32 -target-feature +sme -target-feature +sme2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s
-// RUN: %clang_cc1 -DSME_OVERLOADED_FORMS -triple aarch64 -target-feature +bf16 -target-feature +sme-tmop -target-feature +sme-f16f16 -target-feature +sme-b16b16 -target-feature +sme-f8f16 -target-feature +sme-f8f32 -target-feature +sme -target-feature +sme2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s
-// RUN: %clang_cc1 -triple aarch64 -target-feature +bf16 -target-feature +sme-tmop -target-feature +sme-f16f16 -target-feature +sme-b16b16 -target-feature +sme-f8f16 -target-feature +sme-f8f32 -target-feature +sme -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
+// RUN: %clang_cc1 -triple aarch64 -target-feature +sme-tmop -target-feature +sme-f16f16 -target-feature +sme-f8f32 -target-feature +sme-b16b16 -target-feature +sme-f8f16 -target-feature +sme -target-feature +sme2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s
+// RUN: %clang_cc1 -triple aarch64 -target-feature +sme-tmop -target-feature +sme-f16f16 -target-feature +sme-f8f32 -target-feature +sme-b16b16 -target-feature +sme-f8f16 -target-feature +sme -target-feature +sme2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -DSME_OVERLOADED_FORMS -triple aarch64 -target-feature +sme-tmop -target-feature +sme-f16f16 -target-feature +sme-f8f32 -target-feature +sme-b16b16 -target-feature +sme-f8f16 -target-feature +sme -target-feature +sme2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s
+// RUN: %clang_cc1 -DSME_OVERLOADED_FORMS -triple aarch64 -target-feature +sme-tmop -target-feature +sme-f16f16 -target-feature +sme-f8f32 -target-feature +sme-b16b16 -target-feature +sme-f8f16 -target-feature +sme -target-feature +sme2 -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -triple aarch64 -target-feature +sme-tmop -target-feature +sme-f16f16 -target-feature +sme-f8f32 -target-feature +sme-b16b16 -target-feature +sme-f8f16 -target-feature +sme -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
#include <arm_sme.h>
@@ -18,6 +20,11 @@
// CHECK-NEXT: tail call void @llvm.aarch64.sme.stmopa.nxv16i8(i32 1, <vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
// CHECK-NEXT: ret void
//
+// CPP-CHECK-LABEL: @_Z28test_svtmopa_lane_za32_s8_s810svint8x2_tu10__SVInt8_tu11__SVUint8_t(
+// CPP-CHECK-NEXT: entry:
+// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.stmopa.nxv16i8(i32 1, <vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CPP-CHECK-NEXT: ret void
+//
void test_svtmopa_lane_za32_s8_s8(svint8x2_t zn, svint8_t zm, svuint8_t zk) __arm_streaming __arm_inout("za") {
SME_ACLE_FUNC(svtmopa_lane_za32,_s8_s8,)(1, zn, zm, zk, 3);
}
@@ -27,6 +34,11 @@ void test_svtmopa_lane_za32_s8_s8(svint8x2_t zn, svint8_t zm, svuint8_t zk) __ar
// CHECK-NEXT: tail call void @llvm.aarch64.sme.utmopa.nxv16i8(i32 1, <vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
// CHECK-NEXT: ret void
//
+// CPP-CHECK-LABEL: @_Z28test_svtmopa_lane_za32_u8_u811svuint8x2_tu11__SVUint8_tS0_(
+// CPP-CHECK-NEXT: entry:
+// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.utmopa.nxv16i8(i32 1, <vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CPP-CHECK-NEXT: ret void
+//
void test_svtmopa_lane_za32_u8_u8(svuint8x2_t zn, svuint8_t zm, svuint8_t zk) __arm_streaming __arm_inout("za") {
SME_ACLE_FUNC(svtmopa_lane_za32,_u8_u8,)(1, zn, zm, zk, 3);
}
@@ -36,6 +48,11 @@ void test_svtmopa_lane_za32_u8_u8(svuint8x2_t zn, svuint8_t zm, svuint8_t zk) __
// CHECK-NEXT: tail call void @llvm.aarch64.sme.sutmopa.nxv16i8(i32 1, <vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
// CHECK-NEXT: ret void
//
+// CPP-CHECK-LABEL: @_Z28test_svtmopa_lane_za32_s8_u810svint8x2_tu11__SVUint8_tS0_(
+// CPP-CHECK-NEXT: entry:
+// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.sutmopa.nxv16i8(i32 1, <vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CPP-CHECK-NEXT: ret void
+//
void test_svtmopa_lane_za32_s8_u8(svint8x2_t zn, svuint8_t zm, svuint8_t zk) __arm_streaming __arm_inout("za") {
SME_ACLE_FUNC(svtmopa_lane_za32,_s8_u8,)(1, zn, zm, zk, 3);
}
@@ -45,6 +62,11 @@ void test_svtmopa_lane_za32_s8_u8(svint8x2_t zn, svuint8_t zm, svuint8_t zk) __a
// CHECK-NEXT: tail call void @llvm.aarch64.sme.ustmopa.nxv16i8(i32 1, <vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
// CHECK-NEXT: ret void
//
+// CPP-CHECK-LABEL: @_Z28test_svtmopa_lane_za32_u8_s811svuint8x2_tu10__SVInt8_tu11__SVUint8_t(
+// CPP-CHECK-NEXT: entry:
+// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.ustmopa.nxv16i8(i32 1, <vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CPP-CHECK-NEXT: ret void
+//
void test_svtmopa_lane_za32_u8_s8(svuint8x2_t zn, svint8_t zm, svuint8_t zk) __arm_streaming __arm_inout("za") {
SME_ACLE_FUNC(svtmopa_lane_za32,_u8_s8,)(1, zn, zm, zk, 3);
}
@@ -54,6 +76,11 @@ void test_svtmopa_lane_za32_u8_s8(svuint8x2_t zn, svint8_t zm, svuint8_t zk) __a
// CHECK-NEXT: tail call void @llvm.aarch64.sme.stmopa.nxv8i16(i32 1, <vscale x 8 x i16> [[ZN_COERCE0:%.*]], <vscale x 8 x i16> [[ZN_COERCE1:%.*]], <vscale x 8 x i16> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
// CHECK-NEXT: ret void
//
+// CPP-CHECK-LABEL: @_Z30test_svtmopa_lane_za32_s16_s1611svint16x2_tu11__SVInt16_tu11__SVUint8_t(
+// CPP-CHECK-NEXT: entry:
+// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.stmopa.nxv8i16(i32 1, <vscale x 8 x i16> [[ZN_COERCE0:%.*]], <vscale x 8 x i16> [[ZN_COERCE1:%.*]], <vscale x 8 x i16> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CPP-CHECK-NEXT: ret void
+//
void test_svtmopa_lane_za32_s16_s16(svint16x2_t zn, svint16_t zm, svuint8_t zk) __arm_streaming __arm_inout("za") {
SME_ACLE_FUNC(svtmopa_lane_za32,_s16_s16,)(1, zn, zm, zk, 3);
}
@@ -63,6 +90,11 @@ void test_svtmopa_lane_za32_s16_s16(svint16x2_t zn, svint16_t zm, svuint8_t zk)
// CHECK-NEXT: tail call void @llvm.aarch64.sme.utmopa.nxv8i16(i32 1, <vscale x 8 x i16> [[ZN_COERCE0:%.*]], <vscale x 8 x i16> [[ZN_COERCE1:%.*]], <vscale x 8 x i16> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
// CHECK-NEXT: ret void
//
+// CPP-CHECK-LABEL: @_Z30test_svtmopa_lane_za32_u16_u1612svuint16x2_tu12__SVUint16_tu11__SVUint8_t(
+// CPP-CHECK-NEXT: entry:
+// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.utmopa.nxv8i16(i32 1, <vscale x 8 x i16> [[ZN_COERCE0:%.*]], <vscale x 8 x i16> [[ZN_COERCE1:%.*]], <vscale x 8 x i16> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CPP-CHECK-NEXT: ret void
+//
void test_svtmopa_lane_za32_u16_u16(svuint16x2_t zn, svuint16_t zm, svuint8_t zk) __arm_streaming __arm_inout("za") {
SME_ACLE_FUNC(svtmopa_lane_za32,_u16_u16,)(1, zn, zm, zk, 3);
}
@@ -72,6 +104,11 @@ void test_svtmopa_lane_za32_u16_u16(svuint16x2_t zn, svuint16_t zm, svuint8_t zk
// CHECK-NEXT: tail call void @llvm.aarch64.sme.tmopa.nxv8f16(i32 1, <vscale x 8 x half> [[ZN_COERCE0:%.*]], <vscale x 8 x half> [[ZN_COERCE1:%.*]], <vscale x 8 x half> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
// CHECK-NEXT: ret void
//
+// CPP-CHECK-LABEL: @_Z30test_svtmopa_lane_za32_f16_f1613svfloat16x2_tu13__SVFloat16_tu11__SVUint8_t(
+// CPP-CHECK-NEXT: entry:
+// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.tmopa.nxv8f16(i32 1, <vscale x 8 x half> [[ZN_COERCE0:%.*]], <vscale x 8 x half> [[ZN_COERCE1:%.*]], <vscale x 8 x half> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CPP-CHECK-NEXT: ret void
+//
void test_svtmopa_lane_za32_f16_f16(svfloat16x2_t zn, svfloat16_t zm, svuint8_t zk) __arm_streaming __arm_inout("za") {
SME_ACLE_FUNC(svtmopa_lane_za32,_f16_f16,)(1, zn, zm, zk, 3);
}
@@ -81,6 +118,11 @@ void test_svtmopa_lane_za32_f16_f16(svfloat16x2_t zn, svfloat16_t zm, svuint8_t
// CHECK-NEXT: tail call void @llvm.aarch64.sme.tmopa.nxv4f32(i32 1, <vscale x 4 x float> [[ZN_COERCE0:%.*]], <vscale x 4 x float> [[ZN_COERCE1:%.*]], <vscale x 4 x float> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
// CHECK-NEXT: ret void
//
+// CPP-CHECK-LABEL: @_Z30test_svtmopa_lane_za32_f32_f3213svfloat32x2_tu13__SVFloat32_tu11__SVUint8_t(
+// CPP-CHECK-NEXT: entry:
+// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.tmopa.nxv4f32(i32 1, <vscale x 4 x float> [[ZN_COERCE0:%.*]], <vscale x 4 x float> [[ZN_COERCE1:%.*]], <vscale x 4 x float> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CPP-CHECK-NEXT: ret void
+//
void test_svtmopa_lane_za32_f32_f32(svfloat32x2_t zn, svfloat32_t zm, svuint8_t zk) __arm_streaming __arm_inout("za") {
SME_ACLE_FUNC(svtmopa_lane_za32,_f32_f32,)(1, zn, zm, zk, 3);
}
@@ -90,6 +132,11 @@ void test_svtmopa_lane_za32_f32_f32(svfloat32x2_t zn, svfloat32_t zm, svuint8_t
// CHECK-NEXT: tail call void @llvm.aarch64.sme.tmopa.nxv8bf16(i32 1, <vscale x 8 x bfloat> [[ZN_COERCE0:%.*]], <vscale x 8 x bfloat> [[ZN_COERCE1:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
// CHECK-NEXT: ret void
//
+// CPP-CHECK-LABEL: @_Z32test_svtmopa_lane_za32_bf16_bf1614svbfloat16x2_tu14__SVBfloat16_tu11__SVUint8_t(
+// CPP-CHECK-NEXT: entry:
+// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.tmopa.nxv8bf16(i32 1, <vscale x 8 x bfloat> [[ZN_COERCE0:%.*]], <vscale x 8 x bfloat> [[ZN_COERCE1:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CPP-CHECK-NEXT: ret void
+//
void test_svtmopa_lane_za32_bf16_bf16(svbfloat16x2_t zn, svbfloat16_t zm, svuint8_t zk) __arm_streaming __arm_inout("za") {
SME_ACLE_FUNC(svtmopa_lane_za32,_bf16_bf16,)(1, zn, zm, zk, 3);
}
@@ -99,6 +146,11 @@ void test_svtmopa_lane_za32_bf16_bf16(svbfloat16x2_t zn, svbfloat16_t zm, svuint
// CHECK-NEXT: tail call void @llvm.aarch64.sme.tmopa.nxv8f16(i32 1, <vscale x 8 x half> [[ZN_COERCE0:%.*]], <vscale x 8 x half> [[ZN_COERCE1:%.*]], <vscale x 8 x half> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
// CHECK-NEXT: ret void
//
+// CPP-CHECK-LABEL: @_Z30test_svtmopa_lane_za16_f16_f1613svfloat16x2_tu13__SVFloat16_tu11__SVUint8_t(
+// CPP-CHECK-NEXT: entry:
+// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.tmopa.nxv8f16(i32 1, <vscale x 8 x half> [[ZN_COERCE0:%.*]], <vscale x 8 x half> [[ZN_COERCE1:%.*]], <vscale x 8 x half> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CPP-CHECK-NEXT: ret void
+//
void test_svtmopa_lane_za16_f16_f16(svfloat16x2_t zn, svfloat16_t zm, svuint8_t zk) __arm_streaming __arm_inout("za") {
SME_ACLE_FUNC(svtmopa_lane_za16,_f16_f16,)(1, zn, zm, zk, 3);
}
@@ -108,6 +160,11 @@ void test_svtmopa_lane_za16_f16_f16(svfloat16x2_t zn, svfloat16_t zm, svuint8_t
// CHECK-NEXT: tail call void @llvm.aarch64.sme.tmopa.nxv8bf16(i32 1, <vscale x 8 x bfloat> [[ZN_COERCE0:%.*]], <vscale x 8 x bfloat> [[ZN_COERCE1:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
// CHECK-NEXT: ret void
//
+// CPP-CHECK-LABEL: @_Z32test_svtmopa_lane_za16_bf16_bf1614svbfloat16x2_tu14__SVBfloat16_tu11__SVUint8_t(
+// CPP-CHECK-NEXT: entry:
+// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.tmopa.nxv8bf16(i32 1, <vscale x 8 x bfloat> [[ZN_COERCE0:%.*]], <vscale x 8 x bfloat> [[ZN_COERCE1:%.*]], <vscale x 8 x bfloat> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CPP-CHECK-NEXT: ret void
+//
void test_svtmopa_lane_za16_bf16_bf16(svbfloat16x2_t zn, svbfloat16_t zm, svuint8_t zk) __arm_streaming __arm_inout("za") {
SME_ACLE_FUNC(svtmopa_lane_za16,_bf16_bf16,)(1, zn, zm, zk, 3);
}
@@ -118,6 +175,12 @@ void test_svtmopa_lane_za16_bf16_bf16(svbfloat16x2_t zn, svbfloat16_t zm, svuint
// CHECK-NEXT: tail call void @llvm.aarch64.sme.tmopa.nxv16i8(i32 1, <vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
// CHECK-NEXT: ret void
//
+// CPP-CHECK-LABEL: @_Z34test_svtmopa_lane_za16_mf8_mf8_fpm13svmfloat8x2_tu13__SVMfloat8_tu11__SVUint8_tm(
+// CPP-CHECK-NEXT: entry:
+// CPP-CHECK-NEXT: tail call void @llvm.aarch64.set.fpmr(i64 [[FPMR:%.*]])
+// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.tmopa.nxv16i8(i32 1, <vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CPP-CHECK-NEXT: ret void
+//
void test_svtmopa_lane_za16_mf8_mf8_fpm(svmfloat8x2_t zn, svmfloat8_t zm, svuint8_t zk, fpm_t fpmr) __arm_streaming __arm_inout("za") {
SME_ACLE_FUNC(svtmopa_lane_za16,_mf8_mf8,_fpm)(1, zn, zm, zk, 3, fpmr);
}
@@ -128,6 +191,12 @@ void test_svtmopa_lane_za16_mf8_mf8_fpm(svmfloat8x2_t zn, svmfloat8_t zm, svuint
// CHECK-NEXT: tail call void @llvm.aarch64.sme.tmopa.nxv16i8(i32 1, <vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
// CHECK-NEXT: ret void
//
+// CPP-CHECK-LABEL: @_Z34test_svtmopa_lane_za32_mf8_mf8_fpm13svmfloat8x2_tu13__SVMfloat8_tu11__SVUint8_tm(
+// CPP-CHECK-NEXT: entry:
+// CPP-CHECK-NEXT: tail call void @llvm.aarch64.set.fpmr(i64 [[FPMR:%.*]])
+// CPP-CHECK-NEXT: tail call void @llvm.aarch64.sme.tmopa.nxv16i8(i32 1, <vscale x 16 x i8> [[ZN_COERCE0:%.*]], <vscale x 16 x i8> [[ZN_COERCE1:%.*]], <vscale x 16 x i8> [[ZM:%.*]], <vscale x 16 x i8> [[ZK:%.*]], i32 3)
+// CPP-CHECK-NEXT: ret void
+//
void test_svtmopa_lane_za32_mf8_mf8_fpm(svmfloat8x2_t zn, svmfloat8_t zm, svuint8_t zk, fpm_t fpmr) __arm_streaming __arm_inout("za") {
SME_ACLE_FUNC(svtmopa_lane_za32,_mf8_mf8,_fpm)(1, zn, zm, zk, 3, fpmr);
}
diff --git a/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_tmop.cpp b/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_tmop.cpp
index afa650950d61e..953169b0f55b9 100644
--- a/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_tmop.cpp
+++ b/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_tmop.cpp
@@ -1,28 +1,19 @@
-// RUN: %clang_cc1 -triple aarch64 -target-feature +sme -target-feature +sme2 -target-feature +bf16 -target-feature +sme-f16f16 -target-feature +sme-b16b16 -verify -emit-llvm -o - %s
+// RUN: %clang_cc1 -triple aarch64 \
+// RUN: -target-feature +sme -target-feature +sme2 -verify -emit-llvm -o - %s
// REQUIRES: aarch64-registered-target
#include <arm_sme.h>
-void test_features() __arm_streaming __arm_inout("za") {
- svuint8x2_t zn_u8;
- svint8x2_t zn_s8;
- svuint8_t zm_u8;
- svint8_t zm_s8;
- svuint16x2_t zn_u16;
- svint16x2_t zn_s16;
- svuint16_t zm_u16;
- svint16_t zm_s16;
- svbfloat16x2_t zn_bf16;
- svfloat16x2_t zn_f16;
- svbfloat16_t zm_bf16;
- svfloat16_t zm_f16;
- svfloat32x2_t zn_f32;
- svfloat32_t zm_f32;
- svmfloat8x2_t zn_f8;
- svmfloat8_t zm_f8;
- fpm_t fpm = 0;
- svuint8_t zk;
+void test_features(svuint8x2_t zn_u8, svuint8_t zm_u8,
+ svint8x2_t zn_s8, svint8_t zm_s8,
+ svint16x2_t zn_s16, svint16_t zm_s16,
+ svuint16x2_t zn_u16, svuint16_t zm_u16,
+ svfloat16x2_t zn_f16, svfloat16_t zm_f16,
+ svbfloat16x2_t zn_bf16, svbfloat16_t zm_bf16,
+ svfloat32x2_t zn_f32, svfloat32_t zm_f32,
+ svmfloat8x2_t zn_f8, svmfloat8_t zm_f8,
+ svuint8_t zk, fpm_t fpm) __arm_streaming __arm_inout("za") {
// expected-error at +1 {{'svtmopa_lane_za32_s8_s8' needs target feature sme,sme2,sme-tmop}}
svtmopa_lane_za32_s8_s8(0, zn_s8, zm_s8, zk, 0);
@@ -44,7 +35,7 @@ void test_features() __arm_streaming __arm_inout("za") {
svtmopa_lane_za32_bf16_bf16(0, zn_bf16, zm_bf16, zk, 0);
// expected-error at +1 {{'svtmopa_lane_za16_f16_f16' needs target feature sme,sme2,sme-tmop,sme-f16f16}}
svtmopa_lane_za16_f16_f16(0, zn_f16, zm_f16, zk, 0);
-// expected-error at +1 {{'svtmopa_lane_za16_bf16_bf16' needs target feature sme,sme2,sme-tmop,sme-f16f16}}
+// expected-error at +1 {{'svtmopa_lane_za16_bf16_bf16' needs target feature sme,sme2,sme-tmop,sme-b16b16}}
svtmopa_lane_za16_bf16_bf16(0, zn_bf16, zm_bf16, zk, 0);
// expected-error at +1 {{'svtmopa_lane_za16_mf8_mf8_fpm' needs target feature sme,sme2,sme-tmop,sme-f8f16}}
svtmopa_lane_za16_mf8_mf8_fpm(0, zn_f8, zm_f8, zk, 0, fpm);
@@ -52,25 +43,15 @@ void test_features() __arm_streaming __arm_inout("za") {
svtmopa_lane_za32_mf8_mf8_fpm(0, zn_f8, zm_f8, zk, 0, fpm);
}
-void test_imm() __arm_streaming __arm_inout("za") {
- svuint8x2_t zn_u8;
- svint8x2_t zn_s8;
- svuint8_t zm_u8;
- svint8_t zm_s8;
- svuint16x2_t zn_u16;
- svint16x2_t zn_s16;
- svuint16_t zm_u16;
- svint16_t zm_s16;
- svbfloat16x2_t zn_bf16;
- svfloat16x2_t zn_f16;
- svbfloat16_t zm_bf16;
- svfloat16_t zm_f16;
- svfloat32x2_t zn_f32;
- svfloat32_t zm_f32;
- svmfloat8x2_t zn_f8;
- svmfloat8_t zm_f8;
- fpm_t fpm;
- svuint8_t zk;
+void test_imm(svuint8x2_t zn_u8, svuint8_t zm_u8,
+ svint8x2_t zn_s8, svint8_t zm_s8,
+ svint16x2_t zn_s16, svint16_t zm_s16,
+ svuint16x2_t zn_u16, svuint16_t zm_u16,
+ svfloat16x2_t zn_f16, svfloat16_t zm_f16,
+ svbfloat16x2_t zn_bf16, svbfloat16_t zm_bf16,
+ svfloat32x2_t zn_f32, svfloat32_t zm_f32,
+ svmfloat8x2_t zn_f8, svmfloat8_t zm_f8,
+ svuint8_t zk, fpm_t fpm) __arm_streaming __arm_inout("za") {
// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
svtmopa_lane_za32_s8_s8(0, zn_s8, zm_s8, zk, 4);
diff --git a/llvm/lib/Target/AArch64/SMEInstrFormats.td b/llvm/lib/Target/AArch64/SMEInstrFormats.td
index 20f40b54a3122..86129eb53eb11 100644
--- a/llvm/lib/Target/AArch64/SMEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SMEInstrFormats.td
@@ -291,7 +291,7 @@ class SME2_ZA_Tile_Vec_Multi_Multi_Pat<string name, SDPatternOperator intrinsic,
(!cast<Instruction>(name # _PSEUDO) $tile, (REG_SEQUENCE ZPR2Mul2, vt:$Zn1, zsub0, vt:$Zn2, zsub1), (REG_SEQUENCE ZPR2Mul2, vt:$Zm1, zsub0, vt:$Zm2, zsub1))>;
class SME2_ZA_TMOP_Pat<string name, SDPatternOperator intrinsic, Operand tile_imm, Operand imm_ty, ValueType vt>
- : Pat<(intrinsic tile_imm:$tile, vt:$Zn1, vt:$Zn2, vt:$Zm, (nxv16i8 ZK:$Zk), imm_ty:$idx),
+ : Pat<(intrinsic tile_imm:$tile, vt:$Zn1, vt:$Zn2, vt:$Zm, nxv16i8:$Zk, imm_ty:$idx),
(!cast<Instruction>(name # _PSEUDO) $tile, (REG_SEQUENCE ZPR2Mul2, vt:$Zn1, zsub0, vt:$Zn2, zsub1), $Zm, $Zk, $idx)>;
>From 66c5ca0e0270a8148e484eb075bb40fd0b86e3fb Mon Sep 17 00:00:00 2001
From: Jonathan Thackray <jonathan.thackray at arm.com>
Date: Fri, 11 Apr 2025 22:24:38 +0000
Subject: [PATCH 5/5] fixup! Remove some range tests, and improve
SME2_ZA_TMOP_Pat
---
.../acle_sme2_tmop.cpp | 52 -------------------
llvm/lib/Target/AArch64/SMEInstrFormats.td | 8 +--
2 files changed, 4 insertions(+), 56 deletions(-)
diff --git a/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_tmop.cpp b/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_tmop.cpp
index 953169b0f55b9..5c229d0825366 100644
--- a/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_tmop.cpp
+++ b/clang/test/Sema/aarch64-sme2-intrinsics/acle_sme2_tmop.cpp
@@ -57,116 +57,64 @@ void test_imm(svuint8x2_t zn_u8, svuint8_t zm_u8,
svtmopa_lane_za32_s8_s8(0, zn_s8, zm_s8, zk, 4);
// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
svtmopa_lane_za32_s8_s8(4, zn_s8, zm_s8, zk, 0);
-// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
- svtmopa_lane_za32_s8_s8(0, zn_s8, zm_s8, zk, -1);
-// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
- svtmopa_lane_za32_s8_s8(-1, zn_s8, zm_s8, zk, 0);
// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
svtmopa_lane_za32_u8_u8(0, zn_u8, zm_u8, zk, 4);
// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
svtmopa_lane_za32_u8_u8(4, zn_u8, zm_u8, zk, 0);
-// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
- svtmopa_lane_za32_u8_u8(0, zn_u8, zm_u8, zk, -1);
-// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
- svtmopa_lane_za32_u8_u8(-1, zn_u8, zm_u8, zk, 0);
// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
svtmopa_lane_za32_s8_u8(0, zn_s8, zm_u8, zk, 4);
// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
svtmopa_lane_za32_s8_u8(4, zn_s8, zm_u8, zk, 0);
-// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
- svtmopa_lane_za32_s8_u8(0, zn_s8, zm_u8, zk, -1);
-// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
- svtmopa_lane_za32_s8_u8(-1, zn_s8, zm_u8, zk, 0);
// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
svtmopa_lane_za32_u8_s8(0, zn_u8, zm_s8, zk, 4);
// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
svtmopa_lane_za32_u8_s8(4, zn_u8, zm_s8, zk, 0);
-// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
- svtmopa_lane_za32_u8_s8(0, zn_u8, zm_s8, zk, -1);
-// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
- svtmopa_lane_za32_u8_s8(-1, zn_u8, zm_s8, zk, 0);
// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
svtmopa_lane_za32_s16_s16(0, zn_s16, zm_s16, zk, 4);
// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
svtmopa_lane_za32_s16_s16(4, zn_s16, zm_s16, zk, 0);
-// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
- svtmopa_lane_za32_s16_s16(0, zn_s16, zm_s16, zk, -1);
-// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
- svtmopa_lane_za32_s16_s16(-1, zn_s16, zm_s16, zk, 0);
// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
svtmopa_lane_za32_u16_u16(0, zn_u16, zm_u16, zk, 4);
// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
svtmopa_lane_za32_u16_u16(4, zn_u16, zm_u16, zk, 0);
-// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
- svtmopa_lane_za32_u16_u16(0, zn_u16, zm_u16, zk, -1);
-// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
- svtmopa_lane_za32_u16_u16(-1, zn_u16, zm_u16, zk, 0);
// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
svtmopa_lane_za32_f16_f16(0, zn_f16, zm_f16, zk, 4);
// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
svtmopa_lane_za32_f16_f16(4, zn_f16, zm_f16, zk, 0);
-// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
- svtmopa_lane_za32_f16_f16(0, zn_f16, zm_f16, zk, -1);
-// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
- svtmopa_lane_za32_f16_f16(-1, zn_f16, zm_f16, zk, 0);
// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
svtmopa_lane_za32_f32_f32(0, zn_f32, zm_f32, zk, 4);
// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
svtmopa_lane_za32_f32_f32(4, zn_f32, zm_f32, zk, 0);
-// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
- svtmopa_lane_za32_f32_f32(0, zn_f32, zm_f32, zk, -1);
-// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
- svtmopa_lane_za32_f32_f32(-1, zn_f32, zm_f32, zk, 0);
// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
svtmopa_lane_za32_bf16_bf16(0, zn_bf16, zm_bf16, zk, 4);
// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
svtmopa_lane_za32_bf16_bf16(4, zn_bf16, zm_bf16, zk, 0);
-// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
- svtmopa_lane_za32_bf16_bf16(0, zn_bf16, zm_bf16, zk, -1);
-// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
- svtmopa_lane_za32_bf16_bf16(-1, zn_bf16, zm_bf16, zk, 0);
// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
svtmopa_lane_za16_f16_f16(0, zn_f16, zm_f16, zk, 4);
// expected-error at +1 {{argument value 2 is outside the valid range [0, 1]}}
svtmopa_lane_za16_f16_f16(2, zn_f16, zm_f16, zk, 0);
-// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
- svtmopa_lane_za16_f16_f16(0, zn_f16, zm_f16, zk, -1);
-// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 1]}}
- svtmopa_lane_za16_f16_f16(-1, zn_f16, zm_f16, zk, 0);
// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
svtmopa_lane_za16_bf16_bf16(0, zn_bf16, zm_bf16, zk, 4);
// expected-error at +1 {{argument value 2 is outside the valid range [0, 1]}}
svtmopa_lane_za16_bf16_bf16(2, zn_bf16, zm_bf16, zk, 0);
-// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
- svtmopa_lane_za16_bf16_bf16(0, zn_bf16, zm_bf16, zk, -1);
-// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 1]}}
- svtmopa_lane_za16_bf16_bf16(-1, zn_bf16, zm_bf16, zk, 0);
// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
svtmopa_lane_za16_mf8_mf8_fpm(0, zn_f8, zm_f8, zk, 4, fpm);
// expected-error at +1 {{argument value 2 is outside the valid range [0, 1]}}
svtmopa_lane_za16_mf8_mf8_fpm(2, zn_f8, zm_f8, zk, 0, fpm);
-// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
- svtmopa_lane_za16_mf8_mf8_fpm(0, zn_f8, zm_f8, zk, -1, fpm);
-// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 1]}}
- svtmopa_lane_za16_mf8_mf8_fpm(-1, zn_f8, zm_f8, zk, 0, fpm);
// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
svtmopa_lane_za32_mf8_mf8_fpm(0, zn_f8, zm_f8, zk, 4, fpm);
// expected-error at +1 {{argument value 4 is outside the valid range [0, 3]}}
svtmopa_lane_za32_mf8_mf8_fpm(4, zn_f8, zm_f8, zk, 0, fpm);
-// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
- svtmopa_lane_za32_mf8_mf8_fpm(0, zn_f8, zm_f8, zk, -1, fpm);
-// expected-error at +1 {{argument value 18446744073709551615 is outside the valid range [0, 3]}}
- svtmopa_lane_za32_mf8_mf8_fpm(-1, zn_f8, zm_f8, zk, 0, fpm);
}
diff --git a/llvm/lib/Target/AArch64/SMEInstrFormats.td b/llvm/lib/Target/AArch64/SMEInstrFormats.td
index 86129eb53eb11..8777cdc9a86d1 100644
--- a/llvm/lib/Target/AArch64/SMEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SMEInstrFormats.td
@@ -290,8 +290,8 @@ class SME2_ZA_Tile_Vec_Multi_Multi_Pat<string name, SDPatternOperator intrinsic,
: Pat<(intrinsic imm_ty:$tile, vt:$Zn1, vt:$Zn2, vt:$Zm1, vt:$Zm2),
(!cast<Instruction>(name # _PSEUDO) $tile, (REG_SEQUENCE ZPR2Mul2, vt:$Zn1, zsub0, vt:$Zn2, zsub1), (REG_SEQUENCE ZPR2Mul2, vt:$Zm1, zsub0, vt:$Zm2, zsub1))>;
-class SME2_ZA_TMOP_Pat<string name, SDPatternOperator intrinsic, Operand tile_imm, Operand imm_ty, ValueType vt>
- : Pat<(intrinsic tile_imm:$tile, vt:$Zn1, vt:$Zn2, vt:$Zm, nxv16i8:$Zk, imm_ty:$idx),
+class SME2_ZA_TMOP_Pat<string name, SDPatternOperator intrinsic, Operand tile_imm, ValueType vt>
+ : Pat<(intrinsic tile_imm:$tile, vt:$Zn1, vt:$Zn2, vt:$Zm, nxv16i8:$Zk, timm32_0_3:$idx),
(!cast<Instruction>(name # _PSEUDO) $tile, (REG_SEQUENCE ZPR2Mul2, vt:$Zn1, zsub0, vt:$Zn2, zsub1), $Zm, $Zk, $idx)>;
@@ -3585,7 +3585,7 @@ multiclass sme_tmopa_16b<bits<5> opc, RegisterOperand zn_ty, RegisterOperand zm_
def NAME # _PSEUDO : sme_sparse_outer_product_pseudo<zn_ty, zm_ty, SMEMatrixTileH>, SMEPseudo2Instr<NAME, 0>;
- def : SME2_ZA_TMOP_Pat<NAME, !cast<SDPatternOperator>(intrinsic), timm32_0_1, timm32_0_3, vt>;
+ def : SME2_ZA_TMOP_Pat<NAME, !cast<SDPatternOperator>(intrinsic), timm32_0_1, vt>;
}
multiclass sme_tmopa_32b<bits<5> opc, RegisterOperand zn_ty, RegisterOperand zm_ty, ValueType vt, string mnemonic, string intrinsic> {
@@ -3595,7 +3595,7 @@ multiclass sme_tmopa_32b<bits<5> opc, RegisterOperand zn_ty, RegisterOperand zm_
def NAME # _PSEUDO : sme_sparse_outer_product_pseudo<zn_ty, zm_ty, SMEMatrixTileS>, SMEPseudo2Instr<NAME, 0>;
- def : SME2_ZA_TMOP_Pat<NAME, !cast<SDPatternOperator>(intrinsic), timm32_0_1, timm32_0_3, vt>;
+ def : SME2_ZA_TMOP_Pat<NAME, !cast<SDPatternOperator>(intrinsic), timm32_0_3, vt>;
}
//===----------------------------------------------------------------------===///
More information about the cfe-commits
mailing list