[clang] [llvm] [AArch64][clang][llvm] Add ACLE Armv9.7 matrix multiply-accumulate intrinsics (PR #193017)
Jonathan Thackray via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 23 06:22:52 PDT 2026
https://github.com/jthackray updated https://github.com/llvm/llvm-project/pull/193017
>From 7ad925dac41046ecfdbb6c2b112c615b1b9b6fd1 Mon Sep 17 00:00:00 2001
From: Jonathan Thackray <jonathan.thackray at arm.com>
Date: Mon, 20 Apr 2026 13:55:59 +0100
Subject: [PATCH 1/3] [AArch64][clang][llvm] Add ACLE Armv9.7 MMLA intrinsics
Implement new ACLE matrix multiply-accumulate intrinsics for Armv9.7:
```c
// 16-bit floating-point matrix multiply-accumulate.
// Only if __ARM_FEATURE_SVE_B16MM
// Variant also available for _f16 if (__ARM_FEATURE_SVE2p2 && __ARM_FEATURE_F16MM).
svbfloat16_t svmmla[_bf16](svbfloat16_t zda, svbfloat16_t zn, svbfloat16_t zm);
// Half-precision matrix multiply accumulating to single-precision
// instruction from Armv9.7-A. Requires the +f16f32mm architecture extension.
float32x4_t vmmlaq_f32_f16(float32x4_t r, float16x8_t a, float16x8_t b)
// Non-widening half-precision matrix multiply instruction. Requires the
// +f16mm architecture extension.
float16x8_t vmmlaq_f16_f16(float16x8_t r, float16x8_t a, float16x8_t b)
```
---
.../include/clang/Basic/AArch64CodeGenUtils.h | 2 +
clang/include/clang/Basic/arm_neon.td | 8 ++++
clang/include/clang/Basic/arm_sve.td | 8 ++++
clang/lib/CodeGen/TargetBuiltins/ARM.cpp | 9 ++++
.../sve-intrinsics/acle_sve_mmla-bf16.c | 32 +++++++++++++
.../sve-intrinsics/acle_sve_mmla-f16.c | 32 +++++++++++++
.../AArch64/v9.7a-neon-mmla-intrinsics.c | 47 +++++++++++++++++++
...sve_non_streaming_only_sve_AND_sve-b16mm.c | 32 +++++++++++++
..._streaming_only_sve_AND_sve2p2_AND_f16mm.c | 32 +++++++++++++
clang/test/Sema/aarch64-neon-target.c | 15 +++++-
.../aarch64-neon-without-target-feature.cpp | 4 +-
clang/utils/TableGen/NeonEmitter.cpp | 3 +-
llvm/include/llvm/IR/IntrinsicsAArch64.td | 5 ++
.../lib/Target/AArch64/AArch64InstrFormats.td | 8 ++--
llvm/lib/Target/AArch64/AArch64InstrInfo.td | 6 +--
.../lib/Target/AArch64/AArch64SVEInstrInfo.td | 4 +-
.../AArch64/aarch64-matmul-f16f32mm.ll | 15 ++++++
.../CodeGen/AArch64/aarch64-matmul-f16mm.ll | 15 ++++++
llvm/test/CodeGen/AArch64/sve-bfmmla-bf16.ll | 16 +++++++
llvm/test/CodeGen/AArch64/sve-fmmla-f16.ll | 16 +++++++
20 files changed, 297 insertions(+), 12 deletions(-)
create mode 100644 clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_mmla-bf16.c
create mode 100644 clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_mmla-f16.c
create mode 100644 clang/test/CodeGen/AArch64/v9.7a-neon-mmla-intrinsics.c
create mode 100644 clang/test/Sema/AArch64/arm_sve_non_streaming_only_sve_AND_sve-b16mm.c
create mode 100644 clang/test/Sema/AArch64/arm_sve_non_streaming_only_sve_AND_sve2p2_AND_f16mm.c
create mode 100644 llvm/test/CodeGen/AArch64/aarch64-matmul-f16f32mm.ll
create mode 100644 llvm/test/CodeGen/AArch64/aarch64-matmul-f16mm.ll
create mode 100644 llvm/test/CodeGen/AArch64/sve-bfmmla-bf16.ll
create mode 100644 llvm/test/CodeGen/AArch64/sve-fmmla-f16.ll
diff --git a/clang/include/clang/Basic/AArch64CodeGenUtils.h b/clang/include/clang/Basic/AArch64CodeGenUtils.h
index c0a4d8d6afb09..f64a41df63cf8 100644
--- a/clang/include/clang/Basic/AArch64CodeGenUtils.h
+++ b/clang/include/clang/Basic/AArch64CodeGenUtils.h
@@ -233,6 +233,8 @@ const inline ARMVectorIntrinsicInfo AArch64SIMDIntrinsicMap [] = {
NEONMAP1(vld1q_x2_v, aarch64_neon_ld1x2, 0),
NEONMAP1(vld1q_x3_v, aarch64_neon_ld1x3, 0),
NEONMAP1(vld1q_x4_v, aarch64_neon_ld1x4, 0),
+ NEONMAP1(vmmlaq_f16_f16, aarch64_neon_fmmla, 0),
+ NEONMAP1(vmmlaq_f32_f16, aarch64_neon_fmmla, 0),
NEONMAP1(vmmlaq_s32, aarch64_neon_smmla, 0),
NEONMAP1(vmmlaq_u32, aarch64_neon_ummla, 0),
NEONMAP0(vmovl_v),
diff --git a/clang/include/clang/Basic/arm_neon.td b/clang/include/clang/Basic/arm_neon.td
index e91d7ce975d31..f1c451792d6e6 100644
--- a/clang/include/clang/Basic/arm_neon.td
+++ b/clang/include/clang/Basic/arm_neon.td
@@ -1929,6 +1929,14 @@ let ArchGuard = "defined(__aarch64__)", TargetGuard = "f8f32mm,neon" in {
def VMMLA_F32_MF8 : VInst<"vmmla_f32_mf8_fpm", "(>>F)(>>F)..V", "Qm">;
}
+let ArchGuard = "defined(__aarch64__)", TargetGuard = "f16mm,neon" in {
+ def VMMLA_F16_F16 : SInst<"vmmlaq_f16_f16", "....", "Qh">;
+}
+
+let ArchGuard = "defined(__aarch64__)", TargetGuard = "f16f32mm,neon" in {
+ def VMMLA_F32_F16 : SInst<"vmmlaq_f32_f16", ">>..", "Qh">;
+}
+
let TargetGuard = "i8mm,neon" in {
def VMMLA : SInst<"vmmla", "..(<<)(<<)", "QUiQi">;
def VUSMMLA : SInst<"vusmmla", "..(<<U)(<<)", "Qi">;
diff --git a/clang/include/clang/Basic/arm_sve.td b/clang/include/clang/Basic/arm_sve.td
index 724802cce24f7..4441ecb1c4b0b 100644
--- a/clang/include/clang/Basic/arm_sve.td
+++ b/clang/include/clang/Basic/arm_sve.td
@@ -259,6 +259,10 @@ let SVETargetGuard = "bf16", SMETargetGuard = InvalidMode in {
def SVBFMMLA : SInst<"svbfmmla[_{0}]", "MMdd", "b", MergeNone, "aarch64_sve_bfmmla", [IsOverloadNone]>;
}
+let SVETargetGuard = "sve-b16mm", SMETargetGuard = InvalidMode in {
+ def SVMMLA_BF16 : SInst<"svmmla[_bf16]", "dddd", "b", MergeNone, "aarch64_sve_bfmmla_bf16", [IsOverloadNone]>;
+}
+
let SVETargetGuard = "bf16", SMETargetGuard = "bf16" in {
def SVBFDOT : SInst<"svbfdot[_{0}]", "MMdd", "b", MergeNone, "aarch64_sve_bfdot", [IsOverloadNone, VerifyRuntimeMode]>;
def SVBFMLALB : SInst<"svbfmlalb[_{0}]", "MMdd", "b", MergeNone, "aarch64_sve_bfmlalb", [IsOverloadNone, VerifyRuntimeMode]>;
@@ -1239,6 +1243,10 @@ let SVETargetGuard = "sve-f16f32mm", SMETargetGuard = InvalidMode in {
def SVMLLA_F32_F16 : SInst<"svmmla[_f32_f16]", "ddhh", "f", MergeNone, "aarch64_sve_fmmla", [IsOverloadFirstandLast]>;
}
+let SVETargetGuard = "sve2p2,f16mm", SMETargetGuard = InvalidMode in {
+ def SVMMLA_F16 : SInst<"svmmla[_f16]", "dddd", "h", MergeNone, "aarch64_sve_fmmla", [IsOverloadFirstandLast]>;
+}
+
let SVETargetGuard = "sve2,f8f32mm", SMETargetGuard = InvalidMode in {
def SVMLLA_F32_MF8 : SInst<"svmmla[_f32_mf8]", "dd~~>", "f", MergeNone, "aarch64_sve_fp8_fmmla">;
}
diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
index d897e52b711fd..15ce55da34288 100644
--- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
@@ -730,6 +730,8 @@ static const ARMVectorIntrinsicInfo ARMSIMDIntrinsicMap [] = {
NEONMAP1(vminnm_v, arm_neon_vminnm, Add1ArgType),
NEONMAP1(vminnmq_v, arm_neon_vminnm, Add1ArgType),
NEONMAP2(vminq_v, arm_neon_vminu, arm_neon_vmins, Add1ArgType | UnsignedAlts),
+ NEONMAP1(vmmlaq_f16_f16, aarch64_neon_fmmla, 0),
+ NEONMAP1(vmmlaq_f32_f16, aarch64_neon_fmmla, 0),
NEONMAP1(vmmlaq_s32, arm_neon_smmla, 0),
NEONMAP1(vmmlaq_u32, arm_neon_ummla, 0),
NEONMAP0(vmovl_v),
@@ -1854,6 +1856,13 @@ Value *CodeGenFunction::EmitCommonNeonBuiltinExpr(
llvm::Type *Tys[2] = { Ty, InputTy };
return EmitNeonCall(CGM.getIntrinsic(LLVMIntrinsic, Tys), Ops, "vmmla");
}
+ case NEON::BI__builtin_neon_vmmlaq_f16_f16:
+ case NEON::BI__builtin_neon_vmmlaq_f32_f16: {
+ auto *InputTy =
+ llvm::FixedVectorType::get(HalfTy, Ty->getPrimitiveSizeInBits() / 16);
+ llvm::Type *Tys[2] = {Ty, InputTy};
+ return EmitNeonCall(CGM.getIntrinsic(Int, Tys), Ops, "fmmla");
+ }
case NEON::BI__builtin_neon_vusmmlaq_s32: {
auto *InputTy =
llvm::FixedVectorType::get(Int8Ty, Ty->getPrimitiveSizeInBits() / 8);
diff --git a/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_mmla-bf16.c b/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_mmla-bf16.c
new file mode 100644
index 0000000000000..f717e2eb1d3de
--- /dev/null
+++ b/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_mmla-bf16.c
@@ -0,0 +1,32 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 6
+// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64 -target-feature +sve -target-feature +sve-b16mm -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s
+// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64 -target-feature +sve -target-feature +sve-b16mm -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -fclang-abi-compat=latest -DSVE_OVERLOADED_FORMS -triple aarch64 -target-feature +sve -target-feature +sve-b16mm -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s
+// RUN: %clang_cc1 -fclang-abi-compat=latest -DSVE_OVERLOADED_FORMS -triple aarch64 -target-feature +sve -target-feature +sve-b16mm -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -triple aarch64 -target-feature +sve -target-feature +sve-b16mm -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
+
+// REQUIRES: aarch64-registered-target
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+#define SVE_ACLE_FUNC(A1, A3) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1, A2) A1##A2
+#endif
+
+// CHECK-LABEL: define dso_local <vscale x 8 x bfloat> @test_bf16(
+// CHECK-SAME: <vscale x 8 x bfloat> [[ACC:%.*]], <vscale x 8 x bfloat> [[A:%.*]], <vscale x 8 x bfloat> [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.bfmmla.bf16(<vscale x 8 x bfloat> [[ACC]], <vscale x 8 x bfloat> [[A]], <vscale x 8 x bfloat> [[B]])
+// CHECK-NEXT: ret <vscale x 8 x bfloat> [[TMP0]]
+//
+// CPP-CHECK-LABEL: define dso_local <vscale x 8 x bfloat> @_Z9test_bf16u14__SVBfloat16_tS_S_(
+// CPP-CHECK-SAME: <vscale x 8 x bfloat> [[ACC:%.*]], <vscale x 8 x bfloat> [[A:%.*]], <vscale x 8 x bfloat> [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+// CPP-CHECK-NEXT: [[ENTRY:.*:]]
+// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.bfmmla.bf16(<vscale x 8 x bfloat> [[ACC]], <vscale x 8 x bfloat> [[A]], <vscale x 8 x bfloat> [[B]])
+// CPP-CHECK-NEXT: ret <vscale x 8 x bfloat> [[TMP0]]
+//
+svbfloat16_t test_bf16(svbfloat16_t acc, svbfloat16_t a, svbfloat16_t b) {
+ return SVE_ACLE_FUNC(svmmla, _bf16)(acc, a, b);
+}
diff --git a/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_mmla-f16.c b/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_mmla-f16.c
new file mode 100644
index 0000000000000..f79cfedaf76c2
--- /dev/null
+++ b/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_mmla-f16.c
@@ -0,0 +1,32 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 6
+// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64 -target-feature +sve -target-feature +sve2p2 -target-feature +f16mm -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s
+// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64 -target-feature +sve -target-feature +sve2p2 -target-feature +f16mm -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -fclang-abi-compat=latest -DSVE_OVERLOADED_FORMS -triple aarch64 -target-feature +sve -target-feature +sve2p2 -target-feature +f16mm -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s
+// RUN: %clang_cc1 -fclang-abi-compat=latest -DSVE_OVERLOADED_FORMS -triple aarch64 -target-feature +sve -target-feature +sve2p2 -target-feature +f16mm -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK
+// RUN: %clang_cc1 -triple aarch64 -target-feature +sve -target-feature +sve2p2 -target-feature +f16mm -S -disable-O0-optnone -Werror -Wall -o /dev/null %s
+
+// REQUIRES: aarch64-registered-target
+
+#include <arm_sve.h>
+
+#ifdef SVE_OVERLOADED_FORMS
+#define SVE_ACLE_FUNC(A1, A3) A1##A3
+#else
+#define SVE_ACLE_FUNC(A1, A2) A1##A2
+#endif
+
+// CHECK-LABEL: define dso_local <vscale x 8 x half> @test_f16(
+// CHECK-SAME: <vscale x 8 x half> [[ACC:%.*]], <vscale x 8 x half> [[A:%.*]], <vscale x 8 x half> [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sve.fmmla.nxv8f16.nxv8f16(<vscale x 8 x half> [[ACC]], <vscale x 8 x half> [[A]], <vscale x 8 x half> [[B]])
+// CHECK-NEXT: ret <vscale x 8 x half> [[TMP0]]
+//
+// CPP-CHECK-LABEL: define dso_local <vscale x 8 x half> @_Z8test_f16u13__SVFloat16_tS_S_(
+// CPP-CHECK-SAME: <vscale x 8 x half> [[ACC:%.*]], <vscale x 8 x half> [[A:%.*]], <vscale x 8 x half> [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+// CPP-CHECK-NEXT: [[ENTRY:.*:]]
+// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call <vscale x 8 x half> @llvm.aarch64.sve.fmmla.nxv8f16.nxv8f16(<vscale x 8 x half> [[ACC]], <vscale x 8 x half> [[A]], <vscale x 8 x half> [[B]])
+// CPP-CHECK-NEXT: ret <vscale x 8 x half> [[TMP0]]
+//
+svfloat16_t test_f16(svfloat16_t acc, svfloat16_t a, svfloat16_t b) {
+ return SVE_ACLE_FUNC(svmmla, _f16)(acc, a, b);
+}
diff --git a/clang/test/CodeGen/AArch64/v9.7a-neon-mmla-intrinsics.c b/clang/test/CodeGen/AArch64/v9.7a-neon-mmla-intrinsics.c
new file mode 100644
index 0000000000000..6bffc3e63f46b
--- /dev/null
+++ b/clang/test/CodeGen/AArch64/v9.7a-neon-mmla-intrinsics.c
@@ -0,0 +1,47 @@
+// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 6
+// RUN: %clang_cc1 -triple arm64-none-linux-gnu -target-feature +neon -target-feature +v9.7a -target-feature +f16mm -target-feature +f16f32mm \
+// RUN: -disable-O0-optnone -emit-llvm -o - %s \
+// RUN: | opt -S -passes=mem2reg,sroa \
+// RUN: | FileCheck %s
+
+// REQUIRES: aarch64-registered-target
+
+#include <arm_neon.h>
+
+// CHECK-LABEL: define dso_local <8 x half> @test_vmmlaq_f16_f16(
+// CHECK-SAME: <8 x half> noundef [[ACC:%.*]], <8 x half> noundef [[A:%.*]], <8 x half> noundef [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: [[TMP0:%.*]] = bitcast <8 x half> [[ACC]] to <8 x i16>
+// CHECK-NEXT: [[TMP1:%.*]] = bitcast <8 x half> [[A]] to <8 x i16>
+// CHECK-NEXT: [[TMP2:%.*]] = bitcast <8 x half> [[B]] to <8 x i16>
+// CHECK-NEXT: [[TMP3:%.*]] = bitcast <8 x i16> [[TMP0]] to <16 x i8>
+// CHECK-NEXT: [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to <16 x i8>
+// CHECK-NEXT: [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to <16 x i8>
+// CHECK-NEXT: [[FMMLA_I:%.*]] = bitcast <16 x i8> [[TMP3]] to <8 x half>
+// CHECK-NEXT: [[FMMLA1_I:%.*]] = bitcast <16 x i8> [[TMP4]] to <8 x half>
+// CHECK-NEXT: [[FMMLA2_I:%.*]] = bitcast <16 x i8> [[TMP5]] to <8 x half>
+// CHECK-NEXT: [[FMMLA3_I:%.*]] = call <8 x half> @llvm.aarch64.neon.fmmla.v8f16.v8f16(<8 x half> [[FMMLA_I]], <8 x half> [[FMMLA1_I]], <8 x half> [[FMMLA2_I]])
+// CHECK-NEXT: ret <8 x half> [[FMMLA3_I]]
+//
+float16x8_t test_vmmlaq_f16_f16(float16x8_t acc, float16x8_t a, float16x8_t b) {
+ return vmmlaq_f16_f16(acc, a, b);
+}
+
+// CHECK-LABEL: define dso_local <4 x float> @test_vmmlaq_f32_f16(
+// CHECK-SAME: <4 x float> noundef [[ACC:%.*]], <8 x half> noundef [[A:%.*]], <8 x half> noundef [[B:%.*]]) #[[ATTR0]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: [[TMP0:%.*]] = bitcast <4 x float> [[ACC]] to <4 x i32>
+// CHECK-NEXT: [[TMP1:%.*]] = bitcast <8 x half> [[A]] to <8 x i16>
+// CHECK-NEXT: [[TMP2:%.*]] = bitcast <8 x half> [[B]] to <8 x i16>
+// CHECK-NEXT: [[TMP3:%.*]] = bitcast <4 x i32> [[TMP0]] to <16 x i8>
+// CHECK-NEXT: [[TMP4:%.*]] = bitcast <8 x i16> [[TMP1]] to <16 x i8>
+// CHECK-NEXT: [[TMP5:%.*]] = bitcast <8 x i16> [[TMP2]] to <16 x i8>
+// CHECK-NEXT: [[FMMLA_I:%.*]] = bitcast <16 x i8> [[TMP3]] to <4 x float>
+// CHECK-NEXT: [[FMMLA1_I:%.*]] = bitcast <16 x i8> [[TMP4]] to <8 x half>
+// CHECK-NEXT: [[FMMLA2_I:%.*]] = bitcast <16 x i8> [[TMP5]] to <8 x half>
+// CHECK-NEXT: [[FMMLA3_I:%.*]] = call <4 x float> @llvm.aarch64.neon.fmmla.v4f32.v8f16(<4 x float> [[FMMLA_I]], <8 x half> [[FMMLA1_I]], <8 x half> [[FMMLA2_I]])
+// CHECK-NEXT: ret <4 x float> [[FMMLA3_I]]
+//
+float32x4_t test_vmmlaq_f32_f16(float32x4_t acc, float16x8_t a, float16x8_t b) {
+ return vmmlaq_f32_f16(acc, a, b);
+}
diff --git a/clang/test/Sema/AArch64/arm_sve_non_streaming_only_sve_AND_sve-b16mm.c b/clang/test/Sema/AArch64/arm_sve_non_streaming_only_sve_AND_sve-b16mm.c
new file mode 100644
index 0000000000000..77286ba40405b
--- /dev/null
+++ b/clang/test/Sema/AArch64/arm_sve_non_streaming_only_sve_AND_sve-b16mm.c
@@ -0,0 +1,32 @@
+// RUN: %clang_cc1 %s -fsyntax-only -triple aarch64-none-linux-gnu -target-feature +sme -target-feature +sve -target-feature +sve-b16mm -verify=guard
+
+// REQUIRES: aarch64-registered-target
+
+#include <arm_sve.h>
+
+// Properties: guard="sve,sve-b16mm" streaming_guard="" flags=""
+
+void test(void) {
+ svbfloat16_t svbfloat16_t_val;
+
+ svmmla(svbfloat16_t_val, svbfloat16_t_val, svbfloat16_t_val);
+ svmmla_bf16(svbfloat16_t_val, svbfloat16_t_val, svbfloat16_t_val);
+}
+
+void test_streaming(void) __arm_streaming{
+ svbfloat16_t svbfloat16_t_val;
+
+ // guard-error at +1 {{builtin can only be called from a non-streaming function}}
+ svmmla(svbfloat16_t_val, svbfloat16_t_val, svbfloat16_t_val);
+ // guard-error at +1 {{builtin can only be called from a non-streaming function}}
+ svmmla_bf16(svbfloat16_t_val, svbfloat16_t_val, svbfloat16_t_val);
+}
+
+void test_streaming_compatible(void) __arm_streaming_compatible{
+ svbfloat16_t svbfloat16_t_val;
+
+ // guard-error at +1 {{builtin can only be called from a non-streaming function}}
+ svmmla(svbfloat16_t_val, svbfloat16_t_val, svbfloat16_t_val);
+ // guard-error at +1 {{builtin can only be called from a non-streaming function}}
+ svmmla_bf16(svbfloat16_t_val, svbfloat16_t_val, svbfloat16_t_val);
+}
diff --git a/clang/test/Sema/AArch64/arm_sve_non_streaming_only_sve_AND_sve2p2_AND_f16mm.c b/clang/test/Sema/AArch64/arm_sve_non_streaming_only_sve_AND_sve2p2_AND_f16mm.c
new file mode 100644
index 0000000000000..5ecb5d1b135b9
--- /dev/null
+++ b/clang/test/Sema/AArch64/arm_sve_non_streaming_only_sve_AND_sve2p2_AND_f16mm.c
@@ -0,0 +1,32 @@
+// RUN: %clang_cc1 %s -fsyntax-only -triple aarch64-none-linux-gnu -target-feature +f16mm -target-feature +sme -target-feature +sve -target-feature +sve2p2 -verify=guard
+
+// REQUIRES: aarch64-registered-target
+
+#include <arm_sve.h>
+
+// Properties: guard="sve,sve2p2,f16mm" streaming_guard="" flags=""
+
+void test(void) {
+ svfloat16_t svfloat16_t_val;
+
+ svmmla(svfloat16_t_val, svfloat16_t_val, svfloat16_t_val);
+ svmmla_f16(svfloat16_t_val, svfloat16_t_val, svfloat16_t_val);
+}
+
+void test_streaming(void) __arm_streaming{
+ svfloat16_t svfloat16_t_val;
+
+ // guard-error at +1 {{builtin can only be called from a non-streaming function}}
+ svmmla(svfloat16_t_val, svfloat16_t_val, svfloat16_t_val);
+ // guard-error at +1 {{builtin can only be called from a non-streaming function}}
+ svmmla_f16(svfloat16_t_val, svfloat16_t_val, svfloat16_t_val);
+}
+
+void test_streaming_compatible(void) __arm_streaming_compatible{
+ svfloat16_t svfloat16_t_val;
+
+ // guard-error at +1 {{builtin can only be called from a non-streaming function}}
+ svmmla(svfloat16_t_val, svfloat16_t_val, svfloat16_t_val);
+ // guard-error at +1 {{builtin can only be called from a non-streaming function}}
+ svmmla_f16(svfloat16_t_val, svfloat16_t_val, svfloat16_t_val);
+}
diff --git a/clang/test/Sema/aarch64-neon-target.c b/clang/test/Sema/aarch64-neon-target.c
index 07d763ec84bd1..ff1928832862d 100644
--- a/clang/test/Sema/aarch64-neon-target.c
+++ b/clang/test/Sema/aarch64-neon-target.c
@@ -42,6 +42,16 @@ void bf16(uint32x2_t v2i32, uint32x4_t v4i32, uint16x8_t v8i16, uint8x16_t v16i8
vcvt_bf16_f32(v4f32);
}
+__attribute__((target("f16mm")))
+void f16mm(float16x8_t v8f16) {
+ vmmlaq_f16_f16(v8f16, v8f16, v8f16);
+}
+
+__attribute__((target("f16f32mm")))
+void f16f32mm(float32x4_t v4f32, float16x8_t v8f16) {
+ vmmlaq_f32_f16(v4f32, v8f16, v8f16);
+}
+
__attribute__((target("arch=armv8-a")))
uint64x2_t test_v8(uint64x2_t a, uint64x2_t b) {
return veorq_u64(a, b);
@@ -66,7 +76,7 @@ void test_v85(float32x4_t v4f32) {
vrnd32xq_f32(v4f32);
}
-void undefined(uint32x2_t v2i32, uint32x4_t v4i32, uint16x8_t v8i16, uint8x16_t v16i8, uint8x8_t v8i8, float32x2_t v2f32, float32x4_t v4f32, float16x4_t v4f16, float64x2_t v2f64, bfloat16x4_t v4bf16, __bf16 bf16, poly64_t poly64, poly64x2_t poly64x2) {
+void undefined(uint32x2_t v2i32, uint32x4_t v4i32, uint16x8_t v8i16, uint8x16_t v16i8, uint8x8_t v8i8, float32x2_t v2f32, float32x4_t v4f32, float16x4_t v4f16, float16x8_t v8f16, float64x2_t v2f64, bfloat16x4_t v4bf16, __bf16 bf16, poly64_t poly64, poly64x2_t poly64x2) {
// dotprod
vdot_u32(v2i32, v8i8, v8i8); // expected-error {{always_inline function 'vdot_u32' requires target feature 'dotprod'}}
vdot_laneq_u32(v2i32, v8i8, v16i8, 1); // expected-error {{always_inline function 'vdot_u32' requires target feature 'dotprod'}}
@@ -88,6 +98,9 @@ void undefined(uint32x2_t v2i32, uint32x4_t v4i32, uint16x8_t v8i16, uint8x16_t
vld1_bf16(0); // expected-error {{'__builtin_neon_vld1_bf16' needs target feature bf16}}
vcvt_f32_bf16(v4bf16); // expected-error {{always_inline function 'vcvt_f32_bf16' requires target feature 'bf16'}}
vcvt_bf16_f32(v4f32); // expected-error {{always_inline function 'vcvt_bf16_f32' requires target feature 'bf16'}}
+ // f16mm / f16f32mm
+ vmmlaq_f16_f16(v8f16, v8f16, v8f16); // expected-error {{always_inline function 'vmmlaq_f16_f16' requires target feature 'f16mm'}}
+ vmmlaq_f32_f16(v4f32, v8f16, v8f16); // expected-error {{always_inline function 'vmmlaq_f32_f16' requires target feature 'f16f32mm'}}
// v8.1 - qrdmla
vqrdmlahq_s32(v4i32, v4i32, v4i32); // expected-error {{always_inline function 'vqrdmlahq_s32' requires target feature 'v8.1a'}}
vqrdmlah_laneq_s32(v2i32, v2i32, v4i32, 1); // expected-error {{always_inline function 'vqrdmlah_s32' requires target feature 'v8.1a'}}
diff --git a/clang/test/Sema/aarch64-neon-without-target-feature.cpp b/clang/test/Sema/aarch64-neon-without-target-feature.cpp
index 0831eb7c754a7..86dbb343198c5 100644
--- a/clang/test/Sema/aarch64-neon-without-target-feature.cpp
+++ b/clang/test/Sema/aarch64-neon-without-target-feature.cpp
@@ -6,7 +6,7 @@
#include <arm_neon.h>
-void undefined(uint32x2_t v2i32, uint32x4_t v4i32, uint16x8_t v8i16, uint8x16_t v16i8, uint8x8_t v8i8, float32x2_t v2f32, float32x4_t v4f32, float16x4_t v4f16, float64x2_t v2f64, bfloat16x4_t v4bf16, __bf16 bf16, poly64_t poly64, poly64x2_t poly64x2) {
+void undefined(uint32x2_t v2i32, uint32x4_t v4i32, uint16x8_t v8i16, uint8x16_t v16i8, uint8x8_t v8i8, float32x2_t v2f32, float32x4_t v4f32, float16x4_t v4f16, float16x8_t v8f16, float64x2_t v2f64, bfloat16x4_t v4bf16, __bf16 bf16, poly64_t poly64, poly64x2_t poly64x2) {
// dotprod
vdot_u32(v2i32, v8i8, v8i8); // expected-error {{always_inline function 'vdot_u32' requires target feature 'neon'}}
vdot_laneq_u32(v2i32, v8i8, v16i8, 1); // expected-error {{always_inline function 'vdot_u32' requires target feature 'neon'}} expected-error {{'__builtin_neon_splat_laneq_v' needs target feature neon}}
@@ -28,6 +28,8 @@ void undefined(uint32x2_t v2i32, uint32x4_t v4i32, uint16x8_t v8i16, uint8x16_t
vld1_bf16(0); // expected-error {{'__builtin_neon_vld1_bf16' needs target feature bf16,neon}}
vcvt_f32_bf16(v4bf16); // expected-error {{always_inline function 'vcvt_f32_bf16' requires target feature 'neon'}}
vcvt_bf16_f32(v4f32); // expected-error {{always_inline function 'vcvt_bf16_f32' requires target feature 'neon'}}
+ vmmlaq_f16_f16(v8f16, v8f16, v8f16); // expected-error {{always_inline function 'vmmlaq_f16_f16' requires target feature 'neon'}}
+ vmmlaq_f32_f16(v4f32, v8f16, v8f16); // expected-error {{always_inline function 'vmmlaq_f32_f16' requires target feature 'neon'}}
vmull_p64(poly64, poly64); // expected-error {{always_inline function 'vmull_p64' requires target feature 'neon'}}
vmull_high_p64(poly64x2, poly64x2); // expected-error {{always_inline function 'vmull_high_p64' requires target feature 'neon'}}
vtrn1_s8(v8i8, v8i8); // expected-error {{always_inline function 'vtrn1_s8' requires target feature 'neon'}}
diff --git a/clang/utils/TableGen/NeonEmitter.cpp b/clang/utils/TableGen/NeonEmitter.cpp
index 51a90cb16751c..207fd5d89d673 100644
--- a/clang/utils/TableGen/NeonEmitter.cpp
+++ b/clang/utils/TableGen/NeonEmitter.cpp
@@ -1126,7 +1126,8 @@ std::string Intrinsic::mangleName(std::string Name, ClassKind LocalCK) const {
if (Name == "vcvt_f16_f32" || Name == "vcvt_f32_f16" ||
Name == "vcvt_f32_f64" || Name == "vcvt_f64_f32" ||
- Name == "vcvt_f32_bf16")
+ Name == "vcvt_f32_bf16" || Name == "vmmlaq_f16_f16" ||
+ Name == "vmmlaq_f32_f16")
return Name;
if (!typeCode.empty()) {
diff --git a/llvm/include/llvm/IR/IntrinsicsAArch64.td b/llvm/include/llvm/IR/IntrinsicsAArch64.td
index 578f54561910b..79b5ef8bf5926 100644
--- a/llvm/include/llvm/IR/IntrinsicsAArch64.td
+++ b/llvm/include/llvm/IR/IntrinsicsAArch64.td
@@ -2834,6 +2834,11 @@ def int_aarch64_sve_fp8_fmmla
: DefaultAttrsIntrinsic<[llvm_anyvector_ty],
[LLVMMatchType<0>, llvm_nxv16i8_ty, llvm_nxv16i8_ty],
[IntrReadMem, IntrInaccessibleMemOnly]>;
+def int_aarch64_sve_bfmmla_bf16
+ : DefaultAttrsIntrinsic<[llvm_nxv8bf16_ty],
+ [llvm_nxv8bf16_ty, llvm_nxv8bf16_ty,
+ llvm_nxv8bf16_ty],
+ [IntrNoMem]>;
//
// SVE ACLE: 7.2. BFloat16 extensions
diff --git a/llvm/lib/Target/AArch64/AArch64InstrFormats.td b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
index a1ad3196aae83..cd47ebb8b7610 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrFormats.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrFormats.td
@@ -6689,14 +6689,14 @@ multiclass SIMDThreeSameVectorMLAL<bit Q, bits<2> sz, string asm, SDPatternOpera
V128, v4f32, v16i8, op>;
}
-multiclass SIMDThreeSameVectorFMLA<string asm> {
+multiclass SIMDThreeSameVectorFMLA<string asm, SDPatternOperator OpNode = null_frag> {
def v8f16_v8f16 : BaseSIMDThreeSameVectorDot<0b1, 0b0, 0b11, 0b1101, asm, ".8h", ".8h",
- V128, v8f16, v8f16, null_frag>;
+ V128, v8f16, v8f16, OpNode>;
}
-multiclass SIMDThreeSameVectorFMLAWiden<string asm> {
+multiclass SIMDThreeSameVectorFMLAWiden<string asm, SDPatternOperator OpNode = null_frag> {
def v8f16_v4f32 : BaseSIMDThreeSameVectorDot<0b1, 0b0, 0b01, 0b1101, asm, ".4s", ".8h",
- V128, v4f32, v8f16, null_frag>;
+ V128, v4f32, v8f16, OpNode>;
}
multiclass SIMDThreeSameVectorFDot<string asm, SDPatternOperator OpNode = null_frag> {
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index ca377de8deb78..0431fbb4a2994 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -225,7 +225,7 @@ def HasSVE2p2 : Predicate<"Subtarget->hasSVE2p2()">,
AssemblerPredicateWithAll<(all_of FeatureSVE2p2), "sve2p2">;
def HasSVE_B16MM : Predicate<"Subtarget->isSVEAvailable() && Subtarget->hasSVE_B16MM()">,
AssemblerPredicateWithAll<(all_of FeatureSVE_B16MM), "sve-b16mm">;
-def HasF16MM : Predicate<"Subtarget->isSVEAvailable() && Subtarget->hasF16MM()">,
+def HasF16MM : Predicate<"Subtarget->hasF16MM()">,
AssemblerPredicateWithAll<(all_of FeatureF16MM), "f16mm">;
def HasSVE2p3 : Predicate<"Subtarget->hasSVE2p3()">,
AssemblerPredicateWithAll<(all_of FeatureSVE2p3), "sve2p3">;
@@ -11829,10 +11829,10 @@ let Predicates = [HasF16F32DOT] in {
}
let Predicates = [HasF16MM] in
- defm FMMLA : SIMDThreeSameVectorFMLA<"fmmla">;
+ defm FMMLA : SIMDThreeSameVectorFMLA<"fmmla", int_aarch64_neon_fmmla>;
let Predicates = [HasF16F32MM] in
- defm FMMLA : SIMDThreeSameVectorFMLAWiden<"fmmla">;
+ defm FMMLA : SIMDThreeSameVectorFMLAWiden<"fmmla", int_aarch64_neon_fmmla>;
let Uses = [FPMR, FPCR] in
defm FMMLA : SIMDThreeSameVectorFP8MatrixMul<"fmmla", int_aarch64_neon_fmmla>;
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 8ab4e7d33d41c..2a66740b156ec 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -4892,14 +4892,14 @@ let Predicates = [HasSVE2p3] in {
// SVE_B16MM Instructions
//===----------------------------------------------------------------------===//
let Predicates = [HasSVE_B16MM] in {
- def BFMMLA_ZZZ_H : sve_fp_matrix_mla<0b110, "bfmmla", ZPR16, ZPR16>;
+ defm BFMMLA_ZZZ_H : sve_fp_matrix_mla<0b110, "bfmmla", ZPR16, ZPR16, int_aarch64_sve_bfmmla_bf16, nxv8bf16, nxv8bf16>;
}
//===----------------------------------------------------------------------===//
// F16MM Instructions
//===----------------------------------------------------------------------===//
let Predicates = [HasSVE2p2, HasF16MM] in {
- def FMMLA_ZZZ_H : sve_fp_matrix_mla<0b100, "fmmla", ZPR16, ZPR16>;
+ defm FMMLA_ZZZ_H : sve_fp_matrix_mla<0b100, "fmmla", ZPR16, ZPR16, int_aarch64_sve_fmmla, nxv8f16, nxv8f16>;
}
//===----------------------------------------------------------------------===//
diff --git a/llvm/test/CodeGen/AArch64/aarch64-matmul-f16f32mm.ll b/llvm/test/CodeGen/AArch64/aarch64-matmul-f16f32mm.ll
new file mode 100644
index 0000000000000..8629aea639ae4
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/aarch64-matmul-f16f32mm.ll
@@ -0,0 +1,15 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple aarch64-none-linux-gnu -mattr=+neon,+f16f32mm < %s | FileCheck %s
+; RUN: llc -mtriple aarch64-none-linux-gnu -mattr=+neon,+f16f32mm -global-isel < %s | FileCheck %s
+
+define <4 x float> @fmmla.v4f32.v8f16(<4 x float> %acc, <8 x half> %a, <8 x half> %b) {
+; CHECK-LABEL: fmmla.v4f32.v8f16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: fmmla v0.4s, v1.8h, v2.8h
+; CHECK-NEXT: ret
+entry:
+ %out = tail call <4 x float> @llvm.aarch64.neon.fmmla.v4f32.v8f16(<4 x float> %acc, <8 x half> %a, <8 x half> %b)
+ ret <4 x float> %out
+}
+
+declare <4 x float> @llvm.aarch64.neon.fmmla.v4f32.v8f16(<4 x float>, <8 x half>, <8 x half>)
diff --git a/llvm/test/CodeGen/AArch64/aarch64-matmul-f16mm.ll b/llvm/test/CodeGen/AArch64/aarch64-matmul-f16mm.ll
new file mode 100644
index 0000000000000..69eb310ccd084
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/aarch64-matmul-f16mm.ll
@@ -0,0 +1,15 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple aarch64-none-linux-gnu -mattr=+neon,+f16mm < %s | FileCheck %s
+; RUN: llc -mtriple aarch64-none-linux-gnu -mattr=+neon,+f16mm -global-isel < %s | FileCheck %s
+
+define <8 x half> @fmmla.v8f16.v8f16(<8 x half> %acc, <8 x half> %a, <8 x half> %b) {
+; CHECK-LABEL: fmmla.v8f16.v8f16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: fmmla v0.8h, v1.8h, v2.8h
+; CHECK-NEXT: ret
+entry:
+ %out = tail call <8 x half> @llvm.aarch64.neon.fmmla.v8f16.v8f16(<8 x half> %acc, <8 x half> %a, <8 x half> %b)
+ ret <8 x half> %out
+}
+
+declare <8 x half> @llvm.aarch64.neon.fmmla.v8f16.v8f16(<8 x half>, <8 x half>, <8 x half>)
diff --git a/llvm/test/CodeGen/AArch64/sve-bfmmla-bf16.ll b/llvm/test/CodeGen/AArch64/sve-bfmmla-bf16.ll
new file mode 100644
index 0000000000000..a2948ba44b975
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-bfmmla-bf16.ll
@@ -0,0 +1,16 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+sve,+sve-b16mm < %s | FileCheck %s
+
+define <vscale x 8 x bfloat> @bfmmla_bf16(<vscale x 8 x bfloat> %acc, <vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b) #0 {
+; CHECK-LABEL: bfmmla_bf16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: bfmmla z0.h, z1.h, z2.h
+; CHECK-NEXT: ret
+entry:
+ %out = tail call <vscale x 8 x bfloat> @llvm.aarch64.sve.bfmmla.bf16(<vscale x 8 x bfloat> %acc, <vscale x 8 x bfloat> %a, <vscale x 8 x bfloat> %b)
+ ret <vscale x 8 x bfloat> %out
+}
+
+declare <vscale x 8 x bfloat> @llvm.aarch64.sve.bfmmla.bf16(<vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>)
+
+attributes #0 = { "target-features"="+sve,+sve-b16mm" }
diff --git a/llvm/test/CodeGen/AArch64/sve-fmmla-f16.ll b/llvm/test/CodeGen/AArch64/sve-fmmla-f16.ll
new file mode 100644
index 0000000000000..bc80f0bdf7e73
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/sve-fmmla-f16.ll
@@ -0,0 +1,16 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+sve2p2,+f16mm < %s | FileCheck %s
+
+define <vscale x 8 x half> @fmmla_f16(<vscale x 8 x half> %acc, <vscale x 8 x half> %a, <vscale x 8 x half> %b) #0 {
+; CHECK-LABEL: fmmla_f16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: fmmla z0.h, z1.h, z2.h
+; CHECK-NEXT: ret
+entry:
+ %out = tail call <vscale x 8 x half> @llvm.aarch64.sve.fmmla.nxv8f16.nxv8f16(<vscale x 8 x half> %acc, <vscale x 8 x half> %a, <vscale x 8 x half> %b)
+ ret <vscale x 8 x half> %out
+}
+
+declare <vscale x 8 x half> @llvm.aarch64.sve.fmmla.nxv8f16.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>)
+
+attributes #0 = { "target-features"="+sve,+sve2p2,+f16mm" }
>From a2e8fbbc0804f1e481f07b611dd47f0422035a89 Mon Sep 17 00:00:00 2001
From: Jonathan Thackray <jonathan.thackray at arm.com>
Date: Thu, 23 Apr 2026 14:14:26 +0100
Subject: [PATCH 2/3] fixup! Address Kerry's PR comments
---
clang/include/clang/Basic/arm_neon.td | 4 ++--
clang/lib/CodeGen/TargetBuiltins/ARM.cpp | 2 --
.../test/CodeGen/AArch64/sve-intrinsics/acle_sve_mmla-bf16.c | 2 +-
clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_mmla-f16.c | 2 +-
clang/test/CodeGen/AArch64/v9.7a-neon-mmla-intrinsics.c | 4 +---
clang/utils/TableGen/NeonEmitter.cpp | 3 +--
llvm/test/CodeGen/AArch64/sve-bfmmla-bf16.ll | 2 --
llvm/test/CodeGen/AArch64/sve-fmmla-f16.ll | 2 --
8 files changed, 6 insertions(+), 15 deletions(-)
diff --git a/clang/include/clang/Basic/arm_neon.td b/clang/include/clang/Basic/arm_neon.td
index f1c451792d6e6..9aad18c1b8750 100644
--- a/clang/include/clang/Basic/arm_neon.td
+++ b/clang/include/clang/Basic/arm_neon.td
@@ -1930,11 +1930,11 @@ let ArchGuard = "defined(__aarch64__)", TargetGuard = "f8f32mm,neon" in {
}
let ArchGuard = "defined(__aarch64__)", TargetGuard = "f16mm,neon" in {
- def VMMLA_F16_F16 : SInst<"vmmlaq_f16_f16", "....", "Qh">;
+ def VMMLA_F16_F16 : SInst<"vmmla_f16", "....", "Qh">;
}
let ArchGuard = "defined(__aarch64__)", TargetGuard = "f16f32mm,neon" in {
- def VMMLA_F32_F16 : SInst<"vmmlaq_f32_f16", ">>..", "Qh">;
+ def VMMLA_F32_F16 : SInst<"vmmla_f32", ">>..", "Qh">;
}
let TargetGuard = "i8mm,neon" in {
diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
index 15ce55da34288..f8d383b15313f 100644
--- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
@@ -730,8 +730,6 @@ static const ARMVectorIntrinsicInfo ARMSIMDIntrinsicMap [] = {
NEONMAP1(vminnm_v, arm_neon_vminnm, Add1ArgType),
NEONMAP1(vminnmq_v, arm_neon_vminnm, Add1ArgType),
NEONMAP2(vminq_v, arm_neon_vminu, arm_neon_vmins, Add1ArgType | UnsignedAlts),
- NEONMAP1(vmmlaq_f16_f16, aarch64_neon_fmmla, 0),
- NEONMAP1(vmmlaq_f32_f16, aarch64_neon_fmmla, 0),
NEONMAP1(vmmlaq_s32, arm_neon_smmla, 0),
NEONMAP1(vmmlaq_u32, arm_neon_ummla, 0),
NEONMAP0(vmovl_v),
diff --git a/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_mmla-bf16.c b/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_mmla-bf16.c
index f717e2eb1d3de..caeeb54d15c8a 100644
--- a/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_mmla-bf16.c
+++ b/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_mmla-bf16.c
@@ -10,7 +10,7 @@
#include <arm_sve.h>
#ifdef SVE_OVERLOADED_FORMS
-#define SVE_ACLE_FUNC(A1, A3) A1##A3
+#define SVE_ACLE_FUNC(A1, A2_UNUSED) A1
#else
#define SVE_ACLE_FUNC(A1, A2) A1##A2
#endif
diff --git a/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_mmla-f16.c b/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_mmla-f16.c
index f79cfedaf76c2..54d96a1964f38 100644
--- a/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_mmla-f16.c
+++ b/clang/test/CodeGen/AArch64/sve-intrinsics/acle_sve_mmla-f16.c
@@ -10,7 +10,7 @@
#include <arm_sve.h>
#ifdef SVE_OVERLOADED_FORMS
-#define SVE_ACLE_FUNC(A1, A3) A1##A3
+#define SVE_ACLE_FUNC(A1, A2_UNUSED) A1
#else
#define SVE_ACLE_FUNC(A1, A2) A1##A2
#endif
diff --git a/clang/test/CodeGen/AArch64/v9.7a-neon-mmla-intrinsics.c b/clang/test/CodeGen/AArch64/v9.7a-neon-mmla-intrinsics.c
index 6bffc3e63f46b..b6ccf5b772044 100644
--- a/clang/test/CodeGen/AArch64/v9.7a-neon-mmla-intrinsics.c
+++ b/clang/test/CodeGen/AArch64/v9.7a-neon-mmla-intrinsics.c
@@ -1,8 +1,6 @@
// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py UTC_ARGS: --version 6
// RUN: %clang_cc1 -triple arm64-none-linux-gnu -target-feature +neon -target-feature +v9.7a -target-feature +f16mm -target-feature +f16f32mm \
-// RUN: -disable-O0-optnone -emit-llvm -o - %s \
-// RUN: | opt -S -passes=mem2reg,sroa \
-// RUN: | FileCheck %s
+// RUN: -disable-O0-optnone -emit-llvm -o - %s | opt -S -passes=mem2reg,sroa | FileCheck %s
// REQUIRES: aarch64-registered-target
diff --git a/clang/utils/TableGen/NeonEmitter.cpp b/clang/utils/TableGen/NeonEmitter.cpp
index 207fd5d89d673..b961ce8a8cc92 100644
--- a/clang/utils/TableGen/NeonEmitter.cpp
+++ b/clang/utils/TableGen/NeonEmitter.cpp
@@ -1126,8 +1126,7 @@ std::string Intrinsic::mangleName(std::string Name, ClassKind LocalCK) const {
if (Name == "vcvt_f16_f32" || Name == "vcvt_f32_f16" ||
Name == "vcvt_f32_f64" || Name == "vcvt_f64_f32" ||
- Name == "vcvt_f32_bf16" || Name == "vmmlaq_f16_f16" ||
- Name == "vmmlaq_f32_f16")
+ Name == "vcvt_f32_bf16" )
return Name;
if (!typeCode.empty()) {
diff --git a/llvm/test/CodeGen/AArch64/sve-bfmmla-bf16.ll b/llvm/test/CodeGen/AArch64/sve-bfmmla-bf16.ll
index a2948ba44b975..982b8883b98b1 100644
--- a/llvm/test/CodeGen/AArch64/sve-bfmmla-bf16.ll
+++ b/llvm/test/CodeGen/AArch64/sve-bfmmla-bf16.ll
@@ -12,5 +12,3 @@ entry:
}
declare <vscale x 8 x bfloat> @llvm.aarch64.sve.bfmmla.bf16(<vscale x 8 x bfloat>, <vscale x 8 x bfloat>, <vscale x 8 x bfloat>)
-
-attributes #0 = { "target-features"="+sve,+sve-b16mm" }
diff --git a/llvm/test/CodeGen/AArch64/sve-fmmla-f16.ll b/llvm/test/CodeGen/AArch64/sve-fmmla-f16.ll
index bc80f0bdf7e73..bd9d70621f624 100644
--- a/llvm/test/CodeGen/AArch64/sve-fmmla-f16.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fmmla-f16.ll
@@ -12,5 +12,3 @@ entry:
}
declare <vscale x 8 x half> @llvm.aarch64.sve.fmmla.nxv8f16.nxv8f16(<vscale x 8 x half>, <vscale x 8 x half>, <vscale x 8 x half>)
-
-attributes #0 = { "target-features"="+sve,+sve2p2,+f16mm" }
>From 44b460d88e1a0f606a70c443198b5d06591d8c71 Mon Sep 17 00:00:00 2001
From: Jonathan Thackray <jonathan.thackray at arm.com>
Date: Thu, 23 Apr 2026 14:19:19 +0100
Subject: [PATCH 3/3] fixup! Rename files to be more logical
---
.../AArch64/{aarch64-matmul-f16mm.ll => neon-matmul-f16.ll} | 0
.../{aarch64-matmul-f16f32mm.ll => neon-matmul-f16f32mm.ll} | 0
.../AArch64/{sve-bfmmla-bf16.ll => sve-intrinsics-matmul-bf16.ll} | 0
.../AArch64/{sve-fmmla-f16.ll => sve-intrinsics-matmul-f16.ll} | 0
4 files changed, 0 insertions(+), 0 deletions(-)
rename llvm/test/CodeGen/AArch64/{aarch64-matmul-f16mm.ll => neon-matmul-f16.ll} (100%)
rename llvm/test/CodeGen/AArch64/{aarch64-matmul-f16f32mm.ll => neon-matmul-f16f32mm.ll} (100%)
rename llvm/test/CodeGen/AArch64/{sve-bfmmla-bf16.ll => sve-intrinsics-matmul-bf16.ll} (100%)
rename llvm/test/CodeGen/AArch64/{sve-fmmla-f16.ll => sve-intrinsics-matmul-f16.ll} (100%)
diff --git a/llvm/test/CodeGen/AArch64/aarch64-matmul-f16mm.ll b/llvm/test/CodeGen/AArch64/neon-matmul-f16.ll
similarity index 100%
rename from llvm/test/CodeGen/AArch64/aarch64-matmul-f16mm.ll
rename to llvm/test/CodeGen/AArch64/neon-matmul-f16.ll
diff --git a/llvm/test/CodeGen/AArch64/aarch64-matmul-f16f32mm.ll b/llvm/test/CodeGen/AArch64/neon-matmul-f16f32mm.ll
similarity index 100%
rename from llvm/test/CodeGen/AArch64/aarch64-matmul-f16f32mm.ll
rename to llvm/test/CodeGen/AArch64/neon-matmul-f16f32mm.ll
diff --git a/llvm/test/CodeGen/AArch64/sve-bfmmla-bf16.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-matmul-bf16.ll
similarity index 100%
rename from llvm/test/CodeGen/AArch64/sve-bfmmla-bf16.ll
rename to llvm/test/CodeGen/AArch64/sve-intrinsics-matmul-bf16.ll
diff --git a/llvm/test/CodeGen/AArch64/sve-fmmla-f16.ll b/llvm/test/CodeGen/AArch64/sve-intrinsics-matmul-f16.ll
similarity index 100%
rename from llvm/test/CodeGen/AArch64/sve-fmmla-f16.ll
rename to llvm/test/CodeGen/AArch64/sve-intrinsics-matmul-f16.ll
More information about the llvm-commits
mailing list