[clang] [compiler-rt] [llvm] [AArch64] Extend FMV support to more features (PR #229139)
via cfe-commits
cfe-commits at lists.llvm.org
Tue Oct 13 03:05:25 PDT 2026
https://github.com/Lukacma updated https://github.com/llvm/llvm-project/pull/229139
>From abb7c3403efa05f18a24147d5544e5ec04d4e554 Mon Sep 17 00:00:00 2001
From: Marian Lukac <Marian.Lukac at arm.com>
Date: Mon, 5 Oct 2026 15:27:19 +0000
Subject: [PATCH 1/4] [AArch64][FMV] Skip unreachable versions before emitting
feature checks
Skip unreachable function versions before building their resolver conditions. Update the resolver IR test to omit the unused feature checks.
---
clang/lib/CodeGen/CodeGenFunction.cpp | 8 ++++----
.../CodeGen/AArch64/fmv-unreachable-version.c | 18 +++---------------
2 files changed, 7 insertions(+), 19 deletions(-)
diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index 8f4d9aae1603e..8e47a0930098b 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -3198,6 +3198,10 @@ void CodeGenFunction::EmitAArch64MultiVersionResolver(
llvm::BasicBlock *CurBlock = createBasicBlock("resolver_entry", Resolver);
for (const FMVResolverOption &RO : Options) {
+ // Skip unreachable versions.
+ if (RO.Function == nullptr)
+ continue;
+
Builder.SetInsertPoint(CurBlock);
llvm::Value *Condition = FormAArch64ResolverCondition(RO);
@@ -3215,10 +3219,6 @@ void CodeGenFunction::EmitAArch64MultiVersionResolver(
Builder.SetInsertPoint(CurBlock);
}
- // Skip unreachable versions.
- if (RO.Function == nullptr)
- continue;
-
llvm::BasicBlock *RetBlock = createBasicBlock("resolver_return", Resolver);
CGBuilderTy RetBuilder(CGM, RetBlock);
CreateMultiVersionResolverReturn(CGM, Resolver, RetBuilder, RO.Function,
diff --git a/clang/test/CodeGen/AArch64/fmv-unreachable-version.c b/clang/test/CodeGen/AArch64/fmv-unreachable-version.c
index ffb3cb986913f..ca6ce76929713 100644
--- a/clang/test/CodeGen/AArch64/fmv-unreachable-version.c
+++ b/clang/test/CodeGen/AArch64/fmv-unreachable-version.c
@@ -56,25 +56,13 @@ __attribute__((target_clones("sve2;priority=4", "aes+sve2;priority=3", "lse;prio
// CHECK-NEXT: ret ptr @unreachable_versions._Msve
// CHECK: resolver_else:
// CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 69793284352
-// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 69793284352
+// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 128
+// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 128
// CHECK-NEXT: [[TMP7:%.*]] = and i1 true, [[TMP6]]
-// CHECK-NEXT: [[TMP8:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP9:%.*]] = and i64 [[TMP8]], 69793317632
-// CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 69793317632
-// CHECK-NEXT: [[TMP11:%.*]] = and i1 true, [[TMP10]]
-// CHECK-NEXT: [[TMP12:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP13:%.*]] = and i64 [[TMP12]], 128
-// CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 128
-// CHECK-NEXT: [[TMP15:%.*]] = and i1 true, [[TMP14]]
-// CHECK-NEXT: br i1 [[TMP15]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
+// CHECK-NEXT: br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
// CHECK: resolver_return1:
// CHECK-NEXT: ret ptr @unreachable_versions._Mlse
// CHECK: resolver_else2:
-// CHECK-NEXT: [[TMP16:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP17:%.*]] = and i64 [[TMP16]], 576460752303423616
-// CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[TMP17]], 576460752303423616
-// CHECK-NEXT: [[TMP19:%.*]] = and i1 true, [[TMP18]]
// CHECK-NEXT: ret ptr @unreachable_versions.default
//
//.
>From 8edb4b48cb93ad90d6eb34a726618d099068c05a Mon Sep 17 00:00:00 2001
From: Marian Lukac <Marian.Lukac at arm.com>
Date: Mon, 5 Oct 2026 16:12:32 +0000
Subject: [PATCH 2/4] [AArch64] Extend FMV support to more features
This patch implements FMV support for neq features
added to ACLE.
This required extending __aarch64_cpu_features by
another word and adjusting resolver CodeGen and
runtime accordingly.
---
clang/lib/CodeGen/CodeGenFunction.cpp | 13 +-
clang/lib/CodeGen/TargetBuiltins/ARM.cpp | 35 +-
clang/test/CodeGen/AArch64/cpu-supports.c | 32 +-
clang/test/CodeGen/AArch64/fmv-dependencies.c | 108 ++++++
clang/test/CodeGen/AArch64/fmv-features.c | 108 ++++++
clang/test/CodeGen/AArch64/fmv-priority.c | 363 +++++++++++++++++-
.../builtins/cpu_model/AArch64CPUFeatures.inc | 46 ++-
compiler-rt/lib/builtins/cpu_model/aarch64.c | 2 +-
.../builtins/cpu_model/aarch64/fmv/apple.inc | 11 +-
.../cpu_model/aarch64/fmv/baremetal.inc | 5 +-
.../cpu_model/aarch64/fmv/fuchsia.inc | 4 +-
.../builtins/cpu_model/aarch64/fmv/hwcap.inc | 65 +++-
.../cpu_model/aarch64/fmv/windows.inc | 60 ++-
.../lib/builtins/cpu_model/aarch64/hwcap.inc | 87 +++++
.../llvm/TargetParser/AArch64CPUFeatures.inc | 46 ++-
.../TargetParser/AArch64FeatPriorities.inc | 27 ++
.../llvm/TargetParser/AArch64TargetParser.h | 6 +-
llvm/lib/Target/AArch64/AArch64FMV.td | 27 ++
18 files changed, 982 insertions(+), 63 deletions(-)
diff --git a/clang/lib/CodeGen/CodeGenFunction.cpp b/clang/lib/CodeGen/CodeGenFunction.cpp
index 8e47a0930098b..1000440fb5e9a 100644
--- a/clang/lib/CodeGen/CodeGenFunction.cpp
+++ b/clang/lib/CodeGen/CodeGenFunction.cpp
@@ -3203,6 +3203,12 @@ void CodeGenFunction::EmitAArch64MultiVersionResolver(
continue;
Builder.SetInsertPoint(CurBlock);
+
+ if (!AArch64CpuInitialized && !RO.Features.empty()) {
+ EmitAArch64CpuInit();
+ AArch64CpuInitialized = true;
+ }
+
llvm::Value *Condition = FormAArch64ResolverCondition(RO);
// The 'default' or 'all features enabled' case.
@@ -3212,13 +3218,6 @@ void CodeGenFunction::EmitAArch64MultiVersionResolver(
return;
}
- if (!AArch64CpuInitialized) {
- Builder.SetInsertPoint(CurBlock, CurBlock->begin());
- EmitAArch64CpuInit();
- AArch64CpuInitialized = true;
- Builder.SetInsertPoint(CurBlock);
- }
-
llvm::BasicBlock *RetBlock = createBasicBlock("resolver_return", Resolver);
CGBuilderTy RetBuilder(CGM, RetBlock);
CreateMultiVersionResolverReturn(CGM, Resolver, RetBuilder, RO.Function,
diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
index e20fa60ee132b..fd0728b5cb657 100644
--- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
@@ -7389,6 +7389,7 @@ CodeGenFunction::EmitAArch64CpuSupports(ArrayRef<StringRef> FeaturesStrs) {
// Get features from structure in runtime library
// struct {
// unsigned long long features;
+ // unsigned long long features2;
// } __aarch64_cpu_features;
llvm::Type *STy = llvm::StructType::get(Int64Ty);
llvm::Constant *AArch64CPUFeatures =
@@ -7399,10 +7400,42 @@ CodeGenFunction::EmitAArch64CpuSupports(ArrayRef<StringRef> FeaturesStrs) {
{ConstantInt::get(Int32Ty, 0), ConstantInt::get(Int32Ty, 0)});
Value *Features = Builder.CreateAlignedLoad(Int64Ty, CpuFeatures,
CharUnits::fromQuantity(8));
- Value *Mask = Builder.getInt(FeaturesMask.trunc(64));
+ llvm::APInt LowerMask = FeaturesMask.trunc(64);
+ llvm::APInt UpperMask = FeaturesMask.lshr(64).trunc(64);
+ if (!UpperMask.isZero())
+ // Require the second feature word to be present.
+ LowerMask.setBit(llvm::AArch64::FEAT_EXT);
+
+ Value *Mask = Builder.getInt(LowerMask);
Value *Bitset = Builder.CreateAnd(Features, Mask);
Value *Cmp = Builder.CreateICmpEQ(Bitset, Mask);
Result = Builder.CreateAnd(Result, Cmp);
+
+ if (!UpperMask.isZero()) {
+ // Load the second feature word only if present.
+ llvm::BasicBlock *LowerBlock = Builder.GetInsertBlock();
+ llvm::BasicBlock *ExtensionBlock =
+ createBasicBlock("cpu_supports.extension", CurFn);
+ llvm::BasicBlock *EndBlock =
+ createBasicBlock("cpu_supports.end", CurFn);
+ Builder.CreateCondBr(Result, ExtensionBlock, EndBlock);
+
+ Builder.SetInsertPoint(ExtensionBlock);
+ Value *ExtendedFeatures = Builder.CreateGEP(
+ Int8Ty, AArch64CPUFeatures, ConstantInt::get(Int64Ty, 8));
+ Features = Builder.CreateAlignedLoad(Int64Ty, ExtendedFeatures,
+ CharUnits::fromQuantity(8));
+ Mask = Builder.getInt(UpperMask);
+ Bitset = Builder.CreateAnd(Features, Mask);
+ Cmp = Builder.CreateICmpEQ(Bitset, Mask);
+ Builder.CreateBr(EndBlock);
+
+ Builder.SetInsertPoint(EndBlock);
+ llvm::PHINode *ResultPhi = Builder.CreatePHI(Builder.getInt1Ty(), 2);
+ ResultPhi->addIncoming(Builder.getFalse(), LowerBlock);
+ ResultPhi->addIncoming(Cmp, ExtensionBlock);
+ Result = ResultPhi;
+ }
}
return Result;
}
diff --git a/clang/test/CodeGen/AArch64/cpu-supports.c b/clang/test/CodeGen/AArch64/cpu-supports.c
index 6a27849b824cb..a317d0d35d6af 100644
--- a/clang/test/CodeGen/AArch64/cpu-supports.c
+++ b/clang/test/CodeGen/AArch64/cpu-supports.c
@@ -40,11 +40,28 @@
// CHECK-NEXT: store i32 4, ptr [[RETVAL]], align 4
// CHECK-NEXT: br label [[RETURN]]
// CHECK: if.end6:
+// CHECK-NEXT: [[TMP12:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP13:%.*]] = and i64 [[TMP12]], 4611686088220672256
+// CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 4611686088220672256
+// CHECK-NEXT: [[TMP15:%.*]] = and i1 true, [[TMP14]]
+// CHECK-NEXT: br i1 [[TMP15]], label [[CPU_SUPPORTS_EXTENSION:%.*]], label [[CPU_SUPPORTS_END:%.*]]
+// CHECK: cpu_supports.extension:
+// CHECK-NEXT: [[TMP16:%.*]] = load i64, ptr getelementptr inbounds nuw (i8, ptr @__aarch64_cpu_features, i64 8), align 8
+// CHECK-NEXT: [[TMP17:%.*]] = and i64 [[TMP16]], 1
+// CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[TMP17]], 1
+// CHECK-NEXT: br label [[CPU_SUPPORTS_END]]
+// CHECK: cpu_supports.end:
+// CHECK-NEXT: [[TMP19:%.*]] = phi i1 [ false, [[IF_END6]] ], [ [[TMP18]], [[CPU_SUPPORTS_EXTENSION]] ]
+// CHECK-NEXT: br i1 [[TMP19]], label [[IF_THEN7:%.*]], label [[IF_END8:%.*]]
+// CHECK: if.then7:
+// CHECK-NEXT: store i32 5, ptr [[RETVAL]], align 4
+// CHECK-NEXT: br label [[RETURN]]
+// CHECK: if.end8:
// CHECK-NEXT: store i32 0, ptr [[RETVAL]], align 4
// CHECK-NEXT: br label [[RETURN]]
// CHECK: return:
-// CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[RETVAL]], align 4
-// CHECK-NEXT: ret i32 [[TMP12]]
+// CHECK-NEXT: [[TMP20:%.*]] = load i32, ptr [[RETVAL]], align 4
+// CHECK-NEXT: ret i32 [[TMP20]]
//
int main(void) {
if (__builtin_cpu_supports("sb"))
@@ -59,8 +76,17 @@ int main(void) {
if (__builtin_cpu_supports("avx2"))
return 4;
+ if (__builtin_cpu_supports("sve2p1"))
+ return 5;
+
return 0;
}
//.
-// CHECK: [[META0:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
+// CHECK: [[META0:![0-9]+]] = !{i32 1, !"ptrauth-elf-got", i32 0}
+// CHECK: [[META1:![0-9]+]] = !{i32 1, !"ptrauth-init-fini", i32 0}
+// CHECK: [[META2:![0-9]+]] = !{i32 1, !"ptrauth-init-fini-address-discrimination", i32 0}
+// CHECK: [[META3:![0-9]+]] = !{i32 1, !"ptrauth-sign-personality", i32 0}
+// CHECK: [[META4:![0-9]+]] = !{i32 1, !"aarch64-elf-pauthabi-platform", i32 268435458}
+// CHECK: [[META5:![0-9]+]] = !{i32 1, !"aarch64-elf-pauthabi-version", i32 0}
+// CHECK: [[META6:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
//.
diff --git a/clang/test/CodeGen/AArch64/fmv-dependencies.c b/clang/test/CodeGen/AArch64/fmv-dependencies.c
index 8f7f23f93dfea..b6e267346466b 100644
--- a/clang/test/CodeGen/AArch64/fmv-dependencies.c
+++ b/clang/test/CodeGen/AArch64/fmv-dependencies.c
@@ -138,6 +138,87 @@ __attribute__((target_version("wfxt"))) int fmv(void) { return 0; }
// CHECK: define dso_local i32 @fmv._Mcssc() #[[cssc:[0-9]+]] {
__attribute__((target_version("cssc"))) int fmv(void) { return 0; }
+// CHECK: define dso_local i32 @fmv._Mfp8() #[[fp8:[0-9]+]] {
+__attribute__((target_version("fp8"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mf8f32mm() #[[f8f32mm:[0-9]+]] {
+__attribute__((target_version("f8f32mm"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mfp8dot4() #[[fp8dot4:[0-9]+]] {
+__attribute__((target_version("fp8dot4"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mfp8dot2() #[[fp8dot2:[0-9]+]] {
+__attribute__((target_version("fp8dot2"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mfp8fma() #[[fp8fma:[0-9]+]] {
+__attribute__((target_version("fp8fma"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Msme-f8f32() #[[sme_f8f32:[0-9]+]] {
+__attribute__((target_version("sme-f8f32"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mssve-fp8dot4() #[[ssve_fp8dot4:[0-9]+]] {
+__attribute__((target_version("ssve-fp8dot4"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mssve-fp8fma() #[[ssve_fp8fma:[0-9]+]] {
+__attribute__((target_version("ssve-fp8fma"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mssve-bitperm() #[[ssve_bitperm:[0-9]+]] {
+__attribute__((target_version("ssve-bitperm"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mssve-fp8dot2() #[[ssve_fp8dot2:[0-9]+]] {
+__attribute__((target_version("ssve-fp8dot2"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mssve-aes() #[[ssve_aes:[0-9]+]] {
+__attribute__((target_version("ssve-aes"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mssve-fexpa() #[[ssve_fexpa:[0-9]+]] {
+__attribute__((target_version("ssve-fexpa"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mlut() #[[lut:[0-9]+]] {
+__attribute__((target_version("lut"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mfaminmax() #[[faminmax:[0-9]+]] {
+__attribute__((target_version("faminmax"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Msme-lutv2() #[[sme_lutv2:[0-9]+]] {
+__attribute__((target_version("sme-lutv2"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Msme2p1() #[[sme2p1:[0-9]+]] {
+__attribute__((target_version("sme2p1"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Msme2p2() #[[sme2p2:[0-9]+]] {
+__attribute__((target_version("sme2p2"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Msve2p1() #[[sve2p1:[0-9]+]] {
+__attribute__((target_version("sve2p1"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Msve2p2() #[[sve2p2:[0-9]+]] {
+__attribute__((target_version("sve2p2"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Msme-f16f16() #[[sme_f16f16:[0-9]+]] {
+__attribute__((target_version("sme-f16f16"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mgcs() #[[gcs:[0-9]+]] {
+__attribute__((target_version("gcs"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Msme-f8f16() #[[sme_f8f16:[0-9]+]] {
+__attribute__((target_version("sme-f8f16"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mf8f16mm() #[[f8f16mm:[0-9]+]] {
+__attribute__((target_version("f8f16mm"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Msve-aes2() #[[sve_aes2:[0-9]+]] {
+__attribute__((target_version("sve-aes2"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Msve-bfscale() #[[sve_bfscale:[0-9]+]] {
+__attribute__((target_version("sve-bfscale"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Msve-f16f32mm() #[[sve_f16f32mm:[0-9]+]] {
+__attribute__((target_version("sve-f16f32mm"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Msme-mop4() #[[sme_mop4:[0-9]+]] {
+__attribute__((target_version("sme-mop4"))) int fmv(void) { return 0; }
+
// CHECK-NOT: define dso_local i32 @fmv._M{{.*}}
__attribute__((target_version("non_existent_extension"))) int fmv(void);
@@ -192,3 +273,30 @@ int caller() {
// CHECK: attributes #[[sve2_sm4]] = { {{.*}} "target-features"="+fp-armv8,+fullfp16,+neon,+outline-atomics,+sm4,+sve,+sve-sm4,+sve2,+sve2-sm4,+v8a"
// CHECK: attributes #[[wfxt]] = { {{.*}} "target-features"="+fp-armv8,+neon,+outline-atomics,+v8a,+wfxt"
// CHECK: attributes #[[cssc]] = { {{.*}} "target-features"="+cssc,+fp-armv8,+neon,+outline-atomics,+v8a"
+// CHECK: attributes #[[fp8]] = { {{.*}} "target-features"="+fp-armv8,+fp8,+neon,+outline-atomics,+v8a"
+// CHECK: attributes #[[f8f32mm]] = { {{.*}} "target-features"="+f8f32mm,+fp-armv8,+fp8,+neon,+outline-atomics,+v8a"
+// CHECK: attributes #[[fp8dot4]] = { {{.*}} "target-features"="+fp-armv8,+fp8,+fp8dot4,+neon,+outline-atomics,+v8a"
+// CHECK: attributes #[[fp8dot2]] = { {{.*}} "target-features"="+fp-armv8,+fp8,+fp8dot2,+neon,+outline-atomics,+v8a"
+// CHECK: attributes #[[fp8fma]] = { {{.*}} "target-features"="+fp-armv8,+fp8,+fp8fma,+neon,+outline-atomics,+v8a"
+// CHECK: attributes #[[sme_f8f32]] = { {{.*}} "target-features"="+bf16,+fp-armv8,+fp8,+fullfp16,+neon,+outline-atomics,+sme,+sme-f8f32,+sme2,+v8a"
+// CHECK: attributes #[[ssve_fp8dot4]] = { {{.*}} "target-features"="+bf16,+fp-armv8,+fp8,+fullfp16,+neon,+outline-atomics,+sme,+sme2,+ssve-fp8dot4,+v8a"
+// CHECK: attributes #[[ssve_fp8fma]] = { {{.*}} "target-features"="+bf16,+fp-armv8,+fp8,+fullfp16,+neon,+outline-atomics,+sme,+sme2,+ssve-fp8fma,+v8a"
+// CHECK: attributes #[[ssve_bitperm]] = { {{.*}} "target-features"="+bf16,+fp-armv8,+fullfp16,+neon,+outline-atomics,+sme,+sme2,+ssve-bitperm,+sve-bitperm,+v8a"
+// CHECK: attributes #[[ssve_fp8dot2]] = { {{.*}} "target-features"="+bf16,+fp-armv8,+fp8,+fullfp16,+neon,+outline-atomics,+sme,+sme2,+ssve-fp8dot2,+v8a"
+// CHECK: attributes #[[ssve_aes]] = { {{.*}} "target-features"="+aes,+bf16,+fp-armv8,+fullfp16,+neon,+outline-atomics,+sme,+sme2,+ssve-aes,+sve-aes,+v8a"
+// CHECK: attributes #[[ssve_fexpa]] = { {{.*}} "target-features"="+bf16,+fp-armv8,+fullfp16,+neon,+outline-atomics,+sme,+sme2,+ssve-fexpa,+v8a"
+// CHECK: attributes #[[lut]] = { {{.*}} "target-features"="+fp-armv8,+lut,+neon,+outline-atomics,+v8a"
+// CHECK: attributes #[[faminmax]] = { {{.*}} "target-features"="+faminmax,+fp-armv8,+neon,+outline-atomics,+v8a"
+// CHECK: attributes #[[sme_lutv2]] = { {{.*}} "target-features"="+bf16,+fp-armv8,+fullfp16,+neon,+outline-atomics,+sme,+sme-lutv2,+sme2,+v8a"
+// CHECK: attributes #[[sme2p1]] = { {{.*}} "target-features"="+bf16,+fp-armv8,+fullfp16,+neon,+outline-atomics,+sme,+sme2,+sme2p1,+v8a"
+// CHECK: attributes #[[sme2p2]] = { {{.*}} "target-features"="+bf16,+fp-armv8,+fullfp16,+neon,+outline-atomics,+sme,+sme2,+sme2p1,+sme2p2,+v8a"
+// CHECK: attributes #[[sve2p1]] = { {{.*}} "target-features"="+fp-armv8,+fullfp16,+neon,+outline-atomics,+sve,+sve2,+sve2p1,+v8a"
+// CHECK: attributes #[[sve2p2]] = { {{.*}} "target-features"="+fp-armv8,+fullfp16,+neon,+outline-atomics,+sve,+sve2,+sve2p1,+sve2p2,+v8a"
+// CHECK: attributes #[[sme_f16f16]] = { {{.*}} "target-features"="+bf16,+fp-armv8,+fullfp16,+neon,+outline-atomics,+sme,+sme-f16f16,+sme2,+v8a"
+// CHECK: attributes #[[gcs]] = { {{.*}} "target-features"="+chk,+fp-armv8,+gcs,+neon,+outline-atomics,+v8a"
+// CHECK: attributes #[[sme_f8f16]] = { {{.*}} "target-features"="+bf16,+fp-armv8,+fp8,+fullfp16,+neon,+outline-atomics,+sme,+sme-f8f16,+sme2,+v8a"
+// CHECK: attributes #[[f8f16mm]] = { {{.*}} "target-features"="+f8f16mm,+fp-armv8,+fp8,+neon,+outline-atomics,+v8a"
+// CHECK: attributes #[[sve_aes2]] = { {{.*}} "target-features"="+fp-armv8,+neon,+outline-atomics,+sve-aes2,+v8a"
+// CHECK: attributes #[[sve_bfscale]] = { {{.*}} "target-features"="+fp-armv8,+neon,+outline-atomics,+sve-bfscale,+v8a"
+// CHECK: attributes #[[sve_f16f32mm]] = { {{.*}} "target-features"="+fp-armv8,+fullfp16,+neon,+outline-atomics,+sve,+sve-f16f32mm,+v8a"
+// CHECK: attributes #[[sme_mop4]] = { {{.*}} "target-features"="+bf16,+fp-armv8,+fullfp16,+neon,+outline-atomics,+sme,+sme-mop4,+sme2,+v8a"
diff --git a/clang/test/CodeGen/AArch64/fmv-features.c b/clang/test/CodeGen/AArch64/fmv-features.c
index f8b5a9b15456f..aa1e9da3201ed 100644
--- a/clang/test/CodeGen/AArch64/fmv-features.c
+++ b/clang/test/CodeGen/AArch64/fmv-features.c
@@ -133,6 +133,87 @@ __attribute__((target_version("sve2-sm4"))) int fmv(void) { return 0; }
// CHECK: define dso_local i32 @fmv._Mwfxt() #[[wfxt:[0-9]+]] {
__attribute__((target_version("wfxt"))) int fmv(void) { return 0; }
+// CHECK: define dso_local i32 @fmv._Mfp8() #[[fp8:[0-9]+]] {
+__attribute__((target_version("fp8"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mf8f32mm() #[[f8f32mm:[0-9]+]] {
+__attribute__((target_version("f8f32mm"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mfp8dot4() #[[fp8dot4:[0-9]+]] {
+__attribute__((target_version("fp8dot4"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mfp8dot2() #[[fp8dot2:[0-9]+]] {
+__attribute__((target_version("fp8dot2"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mfp8fma() #[[fp8fma:[0-9]+]] {
+__attribute__((target_version("fp8fma"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Msme-f8f32() #[[sme_f8f32:[0-9]+]] {
+__attribute__((target_version("sme-f8f32"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mssve-fp8dot4() #[[ssve_fp8dot4:[0-9]+]] {
+__attribute__((target_version("ssve-fp8dot4"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mssve-fp8fma() #[[ssve_fp8fma:[0-9]+]] {
+__attribute__((target_version("ssve-fp8fma"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mssve-bitperm() #[[ssve_bitperm:[0-9]+]] {
+__attribute__((target_version("ssve-bitperm"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mssve-fp8dot2() #[[ssve_fp8dot2:[0-9]+]] {
+__attribute__((target_version("ssve-fp8dot2"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mssve-aes() #[[ssve_aes:[0-9]+]] {
+__attribute__((target_version("ssve-aes"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mssve-fexpa() #[[ssve_fexpa:[0-9]+]] {
+__attribute__((target_version("ssve-fexpa"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mlut() #[[lut:[0-9]+]] {
+__attribute__((target_version("lut"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mfaminmax() #[[faminmax:[0-9]+]] {
+__attribute__((target_version("faminmax"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Msme-lutv2() #[[sme_lutv2:[0-9]+]] {
+__attribute__((target_version("sme-lutv2"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Msme2p1() #[[sme2p1:[0-9]+]] {
+__attribute__((target_version("sme2p1"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Msme2p2() #[[sme2p2:[0-9]+]] {
+__attribute__((target_version("sme2p2"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Msve2p1() #[[sve2p1:[0-9]+]] {
+__attribute__((target_version("sve2p1"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Msve2p2() #[[sve2p2:[0-9]+]] {
+__attribute__((target_version("sve2p2"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Msme-f16f16() #[[sme_f16f16:[0-9]+]] {
+__attribute__((target_version("sme-f16f16"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mgcs() #[[gcs:[0-9]+]] {
+__attribute__((target_version("gcs"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Msme-f8f16() #[[sme_f8f16:[0-9]+]] {
+__attribute__((target_version("sme-f8f16"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Mf8f16mm() #[[f8f16mm:[0-9]+]] {
+__attribute__((target_version("f8f16mm"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Msve-aes2() #[[sve_aes2:[0-9]+]] {
+__attribute__((target_version("sve-aes2"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Msve-bfscale() #[[sve_bfscale:[0-9]+]] {
+__attribute__((target_version("sve-bfscale"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Msve-f16f32mm() #[[sve_f16f32mm:[0-9]+]] {
+__attribute__((target_version("sve-f16f32mm"))) int fmv(void) { return 0; }
+
+// CHECK: define dso_local i32 @fmv._Msme-mop4() #[[sme_mop4:[0-9]+]] {
+__attribute__((target_version("sme-mop4"))) int fmv(void) { return 0; }
+
// CHECK: define dso_local i32 @fmv._MaesMbf16MbtiMcrc() #[[unordered_features_with_duplicates:[0-9]+]] {
__attribute__((target_version("crc+bti+bti+bti+aes+aes+bf16"))) int fmv(void) { return 0; }
@@ -190,5 +271,32 @@ int caller() {
// CHECK: attributes #[[sve2_sha3]] = {{.*}} "fmv-features"="sve2-sha3"
// CHECK: attributes #[[sve2_sm4]] = {{.*}} "fmv-features"="sve2-sm4"
// CHECK: attributes #[[wfxt]] = {{.*}} "fmv-features"="wfxt"
+// CHECK: attributes #[[fp8]] = {{.*}} "fmv-features"="fp8"
+// CHECK: attributes #[[f8f32mm]] = {{.*}} "fmv-features"="f8f32mm"
+// CHECK: attributes #[[fp8dot4]] = {{.*}} "fmv-features"="fp8dot4"
+// CHECK: attributes #[[fp8dot2]] = {{.*}} "fmv-features"="fp8dot2"
+// CHECK: attributes #[[fp8fma]] = {{.*}} "fmv-features"="fp8fma"
+// CHECK: attributes #[[sme_f8f32]] = {{.*}} "fmv-features"="sme-f8f32"
+// CHECK: attributes #[[ssve_fp8dot4]] = {{.*}} "fmv-features"="ssve-fp8dot4"
+// CHECK: attributes #[[ssve_fp8fma]] = {{.*}} "fmv-features"="ssve-fp8fma"
+// CHECK: attributes #[[ssve_bitperm]] = {{.*}} "fmv-features"="ssve-bitperm"
+// CHECK: attributes #[[ssve_fp8dot2]] = {{.*}} "fmv-features"="ssve-fp8dot2"
+// CHECK: attributes #[[ssve_aes]] = {{.*}} "fmv-features"="ssve-aes"
+// CHECK: attributes #[[ssve_fexpa]] = {{.*}} "fmv-features"="ssve-fexpa"
+// CHECK: attributes #[[lut]] = {{.*}} "fmv-features"="lut"
+// CHECK: attributes #[[faminmax]] = {{.*}} "fmv-features"="faminmax"
+// CHECK: attributes #[[sme_lutv2]] = {{.*}} "fmv-features"="sme-lutv2"
+// CHECK: attributes #[[sme2p1]] = {{.*}} "fmv-features"="sme2p1"
+// CHECK: attributes #[[sme2p2]] = {{.*}} "fmv-features"="sme2p2"
+// CHECK: attributes #[[sve2p1]] = {{.*}} "fmv-features"="sve2p1"
+// CHECK: attributes #[[sve2p2]] = {{.*}} "fmv-features"="sve2p2"
+// CHECK: attributes #[[sme_f16f16]] = {{.*}} "fmv-features"="sme-f16f16"
+// CHECK: attributes #[[gcs]] = {{.*}} "fmv-features"="gcs"
+// CHECK: attributes #[[sme_f8f16]] = {{.*}} "fmv-features"="sme-f8f16"
+// CHECK: attributes #[[f8f16mm]] = {{.*}} "fmv-features"="f8f16mm"
+// CHECK: attributes #[[sve_aes2]] = {{.*}} "fmv-features"="sve-aes2"
+// CHECK: attributes #[[sve_bfscale]] = {{.*}} "fmv-features"="sve-bfscale"
+// CHECK: attributes #[[sve_f16f32mm]] = {{.*}} "fmv-features"="sve-f16f32mm"
+// CHECK: attributes #[[sme_mop4]] = {{.*}} "fmv-features"="sme-mop4"
// CHECK: attributes #[[unordered_features_with_duplicates]] = {{.*}} "fmv-features"="aes,bf16,bti,crc"
// CHECK: attributes #[[default]] = {{.*}} "fmv-features"
diff --git a/clang/test/CodeGen/AArch64/fmv-priority.c b/clang/test/CodeGen/AArch64/fmv-priority.c
index 84c84df5a2fa0..2978458fbb952 100644
--- a/clang/test/CodeGen/AArch64/fmv-priority.c
+++ b/clang/test/CodeGen/AArch64/fmv-priority.c
@@ -16,6 +16,41 @@ __attribute__((target_version("default"))) int fn(void) { return 0; }
int call() { return fn(); }
+
+__attribute__((target_version("cssc"))) int priority(void);
+__attribute__((target_version("gcs"))) int priority(void);
+__attribute__((target_version("fp8"))) int priority(void);
+__attribute__((target_version("f8f32mm"))) int priority(void);
+__attribute__((target_version("fp8dot4"))) int priority(void);
+__attribute__((target_version("fp8dot2"))) int priority(void);
+__attribute__((target_version("fp8fma"))) int priority(void);
+__attribute__((target_version("lut"))) int priority(void);
+__attribute__((target_version("faminmax"))) int priority(void);
+__attribute__((target_version("f8f16mm"))) int priority(void);
+__attribute__((target_version("sve2p1"))) int priority(void);
+__attribute__((target_version("sve2p2"))) int priority(void);
+__attribute__((target_version("sve-aes2"))) int priority(void);
+__attribute__((target_version("sve-bfscale"))) int priority(void);
+__attribute__((target_version("sve-f16f32mm"))) int priority(void);
+__attribute__((target_version("ssve-fp8dot4"))) int priority(void);
+__attribute__((target_version("ssve-fp8dot2"))) int priority(void);
+__attribute__((target_version("ssve-fp8fma"))) int priority(void);
+__attribute__((target_version("ssve-bitperm"))) int priority(void);
+__attribute__((target_version("ssve-aes"))) int priority(void);
+__attribute__((target_version("ssve-fexpa"))) int priority(void);
+__attribute__((target_version("sme-f8f32"))) int priority(void);
+__attribute__((target_version("sme-lutv2"))) int priority(void);
+__attribute__((target_version("sme2p1"))) int priority(void);
+__attribute__((target_version("sme2p2"))) int priority(void);
+__attribute__((target_version("sme-f16f16"))) int priority(void);
+__attribute__((target_version("sme-f8f16"))) int priority(void);
+__attribute__((target_version("sme-mop4"))) int priority(void);
+__attribute__((target_version("default"))) int priority(void) { return 0; }
+
+int call_priority(void) {
+ return priority();
+}
+
// CHECK-LABEL: define dso_local i32 @fn.default(
// CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
// CHECK-NEXT: [[ENTRY:.*:]]
@@ -29,8 +64,21 @@ int call() { return fn(); }
// CHECK-NEXT: ret i32 [[CALL]]
//
//
-// CHECK-LABEL: define weak_odr ptr @fn.resolver()
-// CHECK-SAME: #[[ATTR_RESOLVER:[0-9]+]] comdat {
+// CHECK-LABEL: define dso_local i32 @priority.default(
+// CHECK-SAME: ) #[[ATTR0]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: ret i32 0
+//
+//
+// CHECK-LABEL: define dso_local i32 @call_priority(
+// CHECK-SAME: ) #[[ATTR1]] {
+// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: [[CALL:%.*]] = call i32 @priority()
+// CHECK-NEXT: ret i32 [[CALL]]
+//
+//
+// CHECK-LABEL: define weak_odr ptr @fn.resolver(
+// CHECK-SAME: ) #[[ATTR4:[0-9]+]] comdat {
// CHECK-NEXT: [[RESOLVER_ENTRY:.*:]]
// CHECK-NEXT: call void @__init_cpu_features_resolver()
// CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
@@ -51,3 +99,314 @@ int call() { return fn(); }
// CHECK: [[RESOLVER_ELSE2]]:
// CHECK-NEXT: ret ptr @fn.default
//
+//
+// CHECK-LABEL: define weak_odr ptr @priority.resolver(
+// CHECK-SAME: ) #[[ATTR4]] comdat {
+// CHECK-NEXT: [[RESOLVER_ENTRY:.*]]:
+// CHECK-NEXT: call void @__init_cpu_features_resolver()
+// CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 4755805604684038912
+// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 4755805604684038912
+// CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]]
+// CHECK-NEXT: br i1 [[TMP3]], label %[[CPU_SUPPORTS_EXTENSION:.*]], label %[[CPU_SUPPORTS_END:.*]]
+// CHECK: [[CPU_SUPPORTS_EXTENSION]]:
+// CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr getelementptr inbounds nuw (i8, ptr @__aarch64_cpu_features, i64 8), align 8
+// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 512
+// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 512
+// CHECK-NEXT: br label %[[CPU_SUPPORTS_END]]
+// CHECK: [[CPU_SUPPORTS_END]]:
+// CHECK-NEXT: [[TMP7:%.*]] = phi i1 [ false, %[[RESOLVER_ENTRY]] ], [ [[TMP6]], %[[CPU_SUPPORTS_EXTENSION]] ]
+// CHECK-NEXT: br i1 [[TMP7]], label %[[RESOLVER_RETURN:.*]], label %[[RESOLVER_ELSE:.*]]
+// CHECK: [[RESOLVER_RETURN]]:
+// CHECK-NEXT: ret ptr @priority._Msme-mop4
+// CHECK: [[RESOLVER_ELSE]]:
+// CHECK-NEXT: [[TMP8:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP9:%.*]] = and i64 [[TMP8]], 4755805604684055296
+// CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 4755805604684055296
+// CHECK-NEXT: [[TMP11:%.*]] = and i1 true, [[TMP10]]
+// CHECK-NEXT: br i1 [[TMP11]], label %[[CPU_SUPPORTS_EXTENSION1:.*]], label %[[CPU_SUPPORTS_END2:.*]]
+// CHECK: [[CPU_SUPPORTS_EXTENSION1]]:
+// CHECK-NEXT: [[TMP12:%.*]] = load i64, ptr getelementptr inbounds nuw (i8, ptr @__aarch64_cpu_features, i64 8), align 8
+// CHECK-NEXT: [[TMP13:%.*]] = and i64 [[TMP12]], 16
+// CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 16
+// CHECK-NEXT: br label %[[CPU_SUPPORTS_END2]]
+// CHECK: [[CPU_SUPPORTS_END2]]:
+// CHECK-NEXT: [[TMP15:%.*]] = phi i1 [ false, %[[RESOLVER_ELSE]] ], [ [[TMP14]], %[[CPU_SUPPORTS_EXTENSION1]] ]
+// CHECK-NEXT: br i1 [[TMP15]], label %[[RESOLVER_RETURN3:.*]], label %[[RESOLVER_ELSE4:.*]]
+// CHECK: [[RESOLVER_RETURN3]]:
+// CHECK-NEXT: ret ptr @priority._Msme-f8f16
+// CHECK: [[RESOLVER_ELSE4]]:
+// CHECK-NEXT: [[TMP16:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP17:%.*]] = and i64 [[TMP16]], 4755805604684038912
+// CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[TMP17]], 4755805604684038912
+// CHECK-NEXT: [[TMP19:%.*]] = and i1 true, [[TMP18]]
+// CHECK-NEXT: br i1 [[TMP19]], label %[[CPU_SUPPORTS_EXTENSION5:.*]], label %[[CPU_SUPPORTS_END6:.*]]
+// CHECK: [[CPU_SUPPORTS_EXTENSION5]]:
+// CHECK-NEXT: [[TMP20:%.*]] = load i64, ptr getelementptr inbounds nuw (i8, ptr @__aarch64_cpu_features, i64 8), align 8
+// CHECK-NEXT: [[TMP21:%.*]] = and i64 [[TMP20]], 4
+// CHECK-NEXT: [[TMP22:%.*]] = icmp eq i64 [[TMP21]], 4
+// CHECK-NEXT: br label %[[CPU_SUPPORTS_END6]]
+// CHECK: [[CPU_SUPPORTS_END6]]:
+// CHECK-NEXT: [[TMP23:%.*]] = phi i1 [ false, %[[RESOLVER_ELSE4]] ], [ [[TMP22]], %[[CPU_SUPPORTS_EXTENSION5]] ]
+// CHECK-NEXT: br i1 [[TMP23]], label %[[RESOLVER_RETURN7:.*]], label %[[RESOLVER_ELSE8:.*]]
+// CHECK: [[RESOLVER_RETURN7]]:
+// CHECK-NEXT: ret ptr @priority._Msme-f16f16
+// CHECK: [[RESOLVER_ELSE8]]:
+// CHECK-NEXT: [[TMP24:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP25:%.*]] = and i64 [[TMP24]], 3602884100077191936
+// CHECK-NEXT: [[TMP26:%.*]] = icmp eq i64 [[TMP25]], 3602884100077191936
+// CHECK-NEXT: [[TMP27:%.*]] = and i1 true, [[TMP26]]
+// CHECK-NEXT: br i1 [[TMP27]], label %[[RESOLVER_RETURN9:.*]], label %[[RESOLVER_ELSE10:.*]]
+// CHECK: [[RESOLVER_RETURN9]]:
+// CHECK-NEXT: ret ptr @priority._Msme2p2
+// CHECK: [[RESOLVER_ELSE10]]:
+// CHECK-NEXT: [[TMP28:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP29:%.*]] = and i64 [[TMP28]], 1297041090863497984
+// CHECK-NEXT: [[TMP30:%.*]] = icmp eq i64 [[TMP29]], 1297041090863497984
+// CHECK-NEXT: [[TMP31:%.*]] = and i1 true, [[TMP30]]
+// CHECK-NEXT: br i1 [[TMP31]], label %[[RESOLVER_RETURN11:.*]], label %[[RESOLVER_ELSE12:.*]]
+// CHECK: [[RESOLVER_RETURN11]]:
+// CHECK-NEXT: ret ptr @priority._Msme2p1
+// CHECK: [[RESOLVER_ELSE12]]:
+// CHECK-NEXT: [[TMP32:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP33:%.*]] = and i64 [[TMP32]], 153126785511392000
+// CHECK-NEXT: [[TMP34:%.*]] = icmp eq i64 [[TMP33]], 153126785511392000
+// CHECK-NEXT: [[TMP35:%.*]] = and i1 true, [[TMP34]]
+// CHECK-NEXT: br i1 [[TMP35]], label %[[RESOLVER_RETURN13:.*]], label %[[RESOLVER_ELSE14:.*]]
+// CHECK: [[RESOLVER_RETURN13]]:
+// CHECK-NEXT: ret ptr @priority._Msme-lutv2
+// CHECK: [[RESOLVER_ELSE14]]:
+// CHECK-NEXT: [[TMP36:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP37:%.*]] = and i64 [[TMP36]], 144119590551634688
+// CHECK-NEXT: [[TMP38:%.*]] = icmp eq i64 [[TMP37]], 144119590551634688
+// CHECK-NEXT: [[TMP39:%.*]] = and i1 true, [[TMP38]]
+// CHECK-NEXT: br i1 [[TMP39]], label %[[RESOLVER_RETURN15:.*]], label %[[RESOLVER_ELSE16:.*]]
+// CHECK: [[RESOLVER_RETURN15]]:
+// CHECK-NEXT: ret ptr @priority._Msme-f8f32
+// CHECK: [[RESOLVER_ELSE16]]:
+// CHECK-NEXT: [[TMP40:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP41:%.*]] = and i64 [[TMP40]], 144401061233361664
+// CHECK-NEXT: [[TMP42:%.*]] = icmp eq i64 [[TMP41]], 144401061233361664
+// CHECK-NEXT: [[TMP43:%.*]] = and i1 true, [[TMP42]]
+// CHECK-NEXT: br i1 [[TMP43]], label %[[RESOLVER_RETURN17:.*]], label %[[RESOLVER_ELSE18:.*]]
+// CHECK: [[RESOLVER_RETURN17]]:
+// CHECK-NEXT: ret ptr @priority._Mssve-fexpa
+// CHECK: [[RESOLVER_ELSE18]]:
+// CHECK-NEXT: [[TMP44:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP45:%.*]] = and i64 [[TMP44]], 144260323745039104
+// CHECK-NEXT: [[TMP46:%.*]] = icmp eq i64 [[TMP45]], 144260323745039104
+// CHECK-NEXT: [[TMP47:%.*]] = and i1 true, [[TMP46]]
+// CHECK-NEXT: br i1 [[TMP47]], label %[[RESOLVER_RETURN19:.*]], label %[[RESOLVER_ELSE20:.*]]
+// CHECK: [[RESOLVER_RETURN19]]:
+// CHECK-NEXT: ret ptr @priority._Mssve-aes
+// CHECK: [[RESOLVER_ELSE20]]:
+// CHECK-NEXT: [[TMP48:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP49:%.*]] = and i64 [[TMP48]], 144128382349673216
+// CHECK-NEXT: [[TMP50:%.*]] = icmp eq i64 [[TMP49]], 144128382349673216
+// CHECK-NEXT: [[TMP51:%.*]] = and i1 true, [[TMP50]]
+// CHECK-NEXT: br i1 [[TMP51]], label %[[RESOLVER_RETURN21:.*]], label %[[RESOLVER_ELSE22:.*]]
+// CHECK: [[RESOLVER_RETURN21]]:
+// CHECK-NEXT: ret ptr @priority._Mssve-bitperm
+// CHECK: [[RESOLVER_ELSE22]]:
+// CHECK-NEXT: [[TMP52:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP53:%.*]] = and i64 [[TMP52]], 144119723695620864
+// CHECK-NEXT: [[TMP54:%.*]] = icmp eq i64 [[TMP53]], 144119723695620864
+// CHECK-NEXT: [[TMP55:%.*]] = and i1 true, [[TMP54]]
+// CHECK-NEXT: br i1 [[TMP55]], label %[[RESOLVER_RETURN23:.*]], label %[[RESOLVER_ELSE24:.*]]
+// CHECK: [[RESOLVER_RETURN23]]:
+// CHECK-NEXT: ret ptr @priority._Mssve-fp8fma
+// CHECK: [[RESOLVER_ELSE24]]:
+// CHECK-NEXT: [[TMP56:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP57:%.*]] = and i64 [[TMP56]], 144154770628756224
+// CHECK-NEXT: [[TMP58:%.*]] = icmp eq i64 [[TMP57]], 144154770628756224
+// CHECK-NEXT: [[TMP59:%.*]] = and i1 true, [[TMP58]]
+// CHECK-NEXT: br i1 [[TMP59]], label %[[RESOLVER_RETURN25:.*]], label %[[RESOLVER_ELSE26:.*]]
+// CHECK: [[RESOLVER_RETURN25]]:
+// CHECK-NEXT: ret ptr @priority._Mssve-fp8dot2
+// CHECK: [[RESOLVER_ELSE26]]:
+// CHECK-NEXT: [[TMP60:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP61:%.*]] = and i64 [[TMP60]], 144119594846601984
+// CHECK-NEXT: [[TMP62:%.*]] = icmp eq i64 [[TMP61]], 144119594846601984
+// CHECK-NEXT: [[TMP63:%.*]] = and i1 true, [[TMP62]]
+// CHECK-NEXT: br i1 [[TMP63]], label %[[RESOLVER_RETURN27:.*]], label %[[RESOLVER_ELSE28:.*]]
+// CHECK: [[RESOLVER_RETURN27]]:
+// CHECK-NEXT: ret ptr @priority._Mssve-fp8dot4
+// CHECK: [[RESOLVER_ELSE28]]:
+// CHECK-NEXT: [[TMP64:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP65:%.*]] = and i64 [[TMP64]], 4611686019501195520
+// CHECK-NEXT: [[TMP66:%.*]] = icmp eq i64 [[TMP65]], 4611686019501195520
+// CHECK-NEXT: [[TMP67:%.*]] = and i1 true, [[TMP66]]
+// CHECK-NEXT: br i1 [[TMP67]], label %[[CPU_SUPPORTS_EXTENSION29:.*]], label %[[CPU_SUPPORTS_END30:.*]]
+// CHECK: [[CPU_SUPPORTS_EXTENSION29]]:
+// CHECK-NEXT: [[TMP68:%.*]] = load i64, ptr getelementptr inbounds nuw (i8, ptr @__aarch64_cpu_features, i64 8), align 8
+// CHECK-NEXT: [[TMP69:%.*]] = and i64 [[TMP68]], 256
+// CHECK-NEXT: [[TMP70:%.*]] = icmp eq i64 [[TMP69]], 256
+// CHECK-NEXT: br label %[[CPU_SUPPORTS_END30]]
+// CHECK: [[CPU_SUPPORTS_END30]]:
+// CHECK-NEXT: [[TMP71:%.*]] = phi i1 [ false, %[[RESOLVER_ELSE28]] ], [ [[TMP70]], %[[CPU_SUPPORTS_EXTENSION29]] ]
+// CHECK-NEXT: br i1 [[TMP71]], label %[[RESOLVER_RETURN31:.*]], label %[[RESOLVER_ELSE32:.*]]
+// CHECK: [[RESOLVER_RETURN31]]:
+// CHECK-NEXT: ret ptr @priority._Msve-f16f32mm
+// CHECK: [[RESOLVER_ELSE32]]:
+// CHECK-NEXT: [[TMP72:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP73:%.*]] = and i64 [[TMP72]], 4611686018427387904
+// CHECK-NEXT: [[TMP74:%.*]] = icmp eq i64 [[TMP73]], 4611686018427387904
+// CHECK-NEXT: [[TMP75:%.*]] = and i1 true, [[TMP74]]
+// CHECK-NEXT: br i1 [[TMP75]], label %[[CPU_SUPPORTS_EXTENSION33:.*]], label %[[CPU_SUPPORTS_END34:.*]]
+// CHECK: [[CPU_SUPPORTS_EXTENSION33]]:
+// CHECK-NEXT: [[TMP76:%.*]] = load i64, ptr getelementptr inbounds nuw (i8, ptr @__aarch64_cpu_features, i64 8), align 8
+// CHECK-NEXT: [[TMP77:%.*]] = and i64 [[TMP76]], 128
+// CHECK-NEXT: [[TMP78:%.*]] = icmp eq i64 [[TMP77]], 128
+// CHECK-NEXT: br label %[[CPU_SUPPORTS_END34]]
+// CHECK: [[CPU_SUPPORTS_END34]]:
+// CHECK-NEXT: [[TMP79:%.*]] = phi i1 [ false, %[[RESOLVER_ELSE32]] ], [ [[TMP78]], %[[CPU_SUPPORTS_EXTENSION33]] ]
+// CHECK-NEXT: br i1 [[TMP79]], label %[[RESOLVER_RETURN35:.*]], label %[[RESOLVER_ELSE36:.*]]
+// CHECK: [[RESOLVER_RETURN35]]:
+// CHECK-NEXT: ret ptr @priority._Msve-bfscale
+// CHECK: [[RESOLVER_ELSE36]]:
+// CHECK-NEXT: [[TMP80:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP81:%.*]] = and i64 [[TMP80]], 4611686018427387904
+// CHECK-NEXT: [[TMP82:%.*]] = icmp eq i64 [[TMP81]], 4611686018427387904
+// CHECK-NEXT: [[TMP83:%.*]] = and i1 true, [[TMP82]]
+// CHECK-NEXT: br i1 [[TMP83]], label %[[CPU_SUPPORTS_EXTENSION37:.*]], label %[[CPU_SUPPORTS_END38:.*]]
+// CHECK: [[CPU_SUPPORTS_EXTENSION37]]:
+// CHECK-NEXT: [[TMP84:%.*]] = load i64, ptr getelementptr inbounds nuw (i8, ptr @__aarch64_cpu_features, i64 8), align 8
+// CHECK-NEXT: [[TMP85:%.*]] = and i64 [[TMP84]], 64
+// CHECK-NEXT: [[TMP86:%.*]] = icmp eq i64 [[TMP85]], 64
+// CHECK-NEXT: br label %[[CPU_SUPPORTS_END38]]
+// CHECK: [[CPU_SUPPORTS_END38]]:
+// CHECK-NEXT: [[TMP87:%.*]] = phi i1 [ false, %[[RESOLVER_ELSE36]] ], [ [[TMP86]], %[[CPU_SUPPORTS_EXTENSION37]] ]
+// CHECK-NEXT: br i1 [[TMP87]], label %[[RESOLVER_RETURN39:.*]], label %[[RESOLVER_ELSE40:.*]]
+// CHECK: [[RESOLVER_RETURN39]]:
+// CHECK-NEXT: ret ptr @priority._Msve-aes2
+// CHECK: [[RESOLVER_ELSE40]]:
+// CHECK-NEXT: [[TMP88:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP89:%.*]] = and i64 [[TMP88]], 4611686088220672256
+// CHECK-NEXT: [[TMP90:%.*]] = icmp eq i64 [[TMP89]], 4611686088220672256
+// CHECK-NEXT: [[TMP91:%.*]] = and i1 true, [[TMP90]]
+// CHECK-NEXT: br i1 [[TMP91]], label %[[CPU_SUPPORTS_EXTENSION41:.*]], label %[[CPU_SUPPORTS_END42:.*]]
+// CHECK: [[CPU_SUPPORTS_EXTENSION41]]:
+// CHECK-NEXT: [[TMP92:%.*]] = load i64, ptr getelementptr inbounds nuw (i8, ptr @__aarch64_cpu_features, i64 8), align 8
+// CHECK-NEXT: [[TMP93:%.*]] = and i64 [[TMP92]], 3
+// CHECK-NEXT: [[TMP94:%.*]] = icmp eq i64 [[TMP93]], 3
+// CHECK-NEXT: br label %[[CPU_SUPPORTS_END42]]
+// CHECK: [[CPU_SUPPORTS_END42]]:
+// CHECK-NEXT: [[TMP95:%.*]] = phi i1 [ false, %[[RESOLVER_ELSE40]] ], [ [[TMP94]], %[[CPU_SUPPORTS_EXTENSION41]] ]
+// CHECK-NEXT: br i1 [[TMP95]], label %[[RESOLVER_RETURN43:.*]], label %[[RESOLVER_ELSE44:.*]]
+// CHECK: [[RESOLVER_RETURN43]]:
+// CHECK-NEXT: ret ptr @priority._Msve2p2
+// CHECK: [[RESOLVER_ELSE44]]:
+// CHECK-NEXT: [[TMP96:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP97:%.*]] = and i64 [[TMP96]], 4611686088220672256
+// CHECK-NEXT: [[TMP98:%.*]] = icmp eq i64 [[TMP97]], 4611686088220672256
+// CHECK-NEXT: [[TMP99:%.*]] = and i1 true, [[TMP98]]
+// CHECK-NEXT: br i1 [[TMP99]], label %[[CPU_SUPPORTS_EXTENSION45:.*]], label %[[CPU_SUPPORTS_END46:.*]]
+// CHECK: [[CPU_SUPPORTS_EXTENSION45]]:
+// CHECK-NEXT: [[TMP100:%.*]] = load i64, ptr getelementptr inbounds nuw (i8, ptr @__aarch64_cpu_features, i64 8), align 8
+// CHECK-NEXT: [[TMP101:%.*]] = and i64 [[TMP100]], 1
+// CHECK-NEXT: [[TMP102:%.*]] = icmp eq i64 [[TMP101]], 1
+// CHECK-NEXT: br label %[[CPU_SUPPORTS_END46]]
+// CHECK: [[CPU_SUPPORTS_END46]]:
+// CHECK-NEXT: [[TMP103:%.*]] = phi i1 [ false, %[[RESOLVER_ELSE44]] ], [ [[TMP102]], %[[CPU_SUPPORTS_EXTENSION45]] ]
+// CHECK-NEXT: br i1 [[TMP103]], label %[[RESOLVER_RETURN47:.*]], label %[[RESOLVER_ELSE48:.*]]
+// CHECK: [[RESOLVER_RETURN47]]:
+// CHECK-NEXT: ret ptr @priority._Msve2p1
+// CHECK: [[RESOLVER_ELSE48]]:
+// CHECK-NEXT: [[TMP104:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP105:%.*]] = and i64 [[TMP104]], 4611686018427405056
+// CHECK-NEXT: [[TMP106:%.*]] = icmp eq i64 [[TMP105]], 4611686018427405056
+// CHECK-NEXT: [[TMP107:%.*]] = and i1 true, [[TMP106]]
+// CHECK-NEXT: br i1 [[TMP107]], label %[[CPU_SUPPORTS_EXTENSION49:.*]], label %[[CPU_SUPPORTS_END50:.*]]
+// CHECK: [[CPU_SUPPORTS_EXTENSION49]]:
+// CHECK-NEXT: [[TMP108:%.*]] = load i64, ptr getelementptr inbounds nuw (i8, ptr @__aarch64_cpu_features, i64 8), align 8
+// CHECK-NEXT: [[TMP109:%.*]] = and i64 [[TMP108]], 32
+// CHECK-NEXT: [[TMP110:%.*]] = icmp eq i64 [[TMP109]], 32
+// CHECK-NEXT: br label %[[CPU_SUPPORTS_END50]]
+// CHECK: [[CPU_SUPPORTS_END50]]:
+// CHECK-NEXT: [[TMP111:%.*]] = phi i1 [ false, %[[RESOLVER_ELSE48]] ], [ [[TMP110]], %[[CPU_SUPPORTS_EXTENSION49]] ]
+// CHECK-NEXT: br i1 [[TMP111]], label %[[RESOLVER_RETURN51:.*]], label %[[RESOLVER_ELSE52:.*]]
+// CHECK: [[RESOLVER_RETURN51]]:
+// CHECK-NEXT: ret ptr @priority._Mf8f16mm
+// CHECK: [[RESOLVER_ELSE52]]:
+// CHECK-NEXT: [[TMP112:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP113:%.*]] = and i64 [[TMP112]], 4503599627371264
+// CHECK-NEXT: [[TMP114:%.*]] = icmp eq i64 [[TMP113]], 4503599627371264
+// CHECK-NEXT: [[TMP115:%.*]] = and i1 true, [[TMP114]]
+// CHECK-NEXT: br i1 [[TMP115]], label %[[RESOLVER_RETURN53:.*]], label %[[RESOLVER_ELSE54:.*]]
+// CHECK: [[RESOLVER_RETURN53]]:
+// CHECK-NEXT: ret ptr @priority._Mfaminmax
+// CHECK: [[RESOLVER_ELSE54]]:
+// CHECK-NEXT: [[TMP116:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP117:%.*]] = and i64 [[TMP116]], 2251799813686016
+// CHECK-NEXT: [[TMP118:%.*]] = icmp eq i64 [[TMP117]], 2251799813686016
+// CHECK-NEXT: [[TMP119:%.*]] = and i1 true, [[TMP118]]
+// CHECK-NEXT: br i1 [[TMP119]], label %[[RESOLVER_RETURN55:.*]], label %[[RESOLVER_ELSE56:.*]]
+// CHECK: [[RESOLVER_RETURN55]]:
+// CHECK-NEXT: ret ptr @priority._Mlut
+// CHECK: [[RESOLVER_ELSE56]]:
+// CHECK-NEXT: [[TMP120:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP121:%.*]] = and i64 [[TMP120]], 2147500800
+// CHECK-NEXT: [[TMP122:%.*]] = icmp eq i64 [[TMP121]], 2147500800
+// CHECK-NEXT: [[TMP123:%.*]] = and i1 true, [[TMP122]]
+// CHECK-NEXT: br i1 [[TMP123]], label %[[RESOLVER_RETURN57:.*]], label %[[RESOLVER_ELSE58:.*]]
+// CHECK: [[RESOLVER_RETURN57]]:
+// CHECK-NEXT: ret ptr @priority._Mfp8fma
+// CHECK: [[RESOLVER_ELSE58]]:
+// CHECK-NEXT: [[TMP124:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP125:%.*]] = and i64 [[TMP124]], 536888064
+// CHECK-NEXT: [[TMP126:%.*]] = icmp eq i64 [[TMP125]], 536888064
+// CHECK-NEXT: [[TMP127:%.*]] = and i1 true, [[TMP126]]
+// CHECK-NEXT: br i1 [[TMP127]], label %[[RESOLVER_RETURN59:.*]], label %[[RESOLVER_ELSE60:.*]]
+// CHECK: [[RESOLVER_RETURN59]]:
+// CHECK-NEXT: ret ptr @priority._Mfp8dot2
+// CHECK: [[RESOLVER_ELSE60]]:
+// CHECK-NEXT: [[TMP128:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP129:%.*]] = and i64 [[TMP128]], 268452608
+// CHECK-NEXT: [[TMP130:%.*]] = icmp eq i64 [[TMP129]], 268452608
+// CHECK-NEXT: [[TMP131:%.*]] = and i1 true, [[TMP130]]
+// CHECK-NEXT: br i1 [[TMP131]], label %[[RESOLVER_RETURN61:.*]], label %[[RESOLVER_ELSE62:.*]]
+// CHECK: [[RESOLVER_RETURN61]]:
+// CHECK-NEXT: ret ptr @priority._Mfp8dot4
+// CHECK: [[RESOLVER_ELSE62]]:
+// CHECK-NEXT: [[TMP132:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP133:%.*]] = and i64 [[TMP132]], 33571584
+// CHECK-NEXT: [[TMP134:%.*]] = icmp eq i64 [[TMP133]], 33571584
+// CHECK-NEXT: [[TMP135:%.*]] = and i1 true, [[TMP134]]
+// CHECK-NEXT: br i1 [[TMP135]], label %[[RESOLVER_RETURN63:.*]], label %[[RESOLVER_ELSE64:.*]]
+// CHECK: [[RESOLVER_RETURN63]]:
+// CHECK-NEXT: ret ptr @priority._Mf8f32mm
+// CHECK: [[RESOLVER_ELSE64]]:
+// CHECK-NEXT: [[TMP136:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP137:%.*]] = and i64 [[TMP136]], 17152
+// CHECK-NEXT: [[TMP138:%.*]] = icmp eq i64 [[TMP137]], 17152
+// CHECK-NEXT: [[TMP139:%.*]] = and i1 true, [[TMP138]]
+// CHECK-NEXT: br i1 [[TMP139]], label %[[RESOLVER_RETURN65:.*]], label %[[RESOLVER_ELSE66:.*]]
+// CHECK: [[RESOLVER_RETURN65]]:
+// CHECK-NEXT: ret ptr @priority._Mfp8
+// CHECK: [[RESOLVER_ELSE66]]:
+// CHECK-NEXT: [[TMP140:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP141:%.*]] = and i64 [[TMP140]], 4611686018427387904
+// CHECK-NEXT: [[TMP142:%.*]] = icmp eq i64 [[TMP141]], 4611686018427387904
+// CHECK-NEXT: [[TMP143:%.*]] = and i1 true, [[TMP142]]
+// CHECK-NEXT: br i1 [[TMP143]], label %[[CPU_SUPPORTS_EXTENSION67:.*]], label %[[CPU_SUPPORTS_END68:.*]]
+// CHECK: [[CPU_SUPPORTS_EXTENSION67]]:
+// CHECK-NEXT: [[TMP144:%.*]] = load i64, ptr getelementptr inbounds nuw (i8, ptr @__aarch64_cpu_features, i64 8), align 8
+// CHECK-NEXT: [[TMP145:%.*]] = and i64 [[TMP144]], 8
+// CHECK-NEXT: [[TMP146:%.*]] = icmp eq i64 [[TMP145]], 8
+// CHECK-NEXT: br label %[[CPU_SUPPORTS_END68]]
+// CHECK: [[CPU_SUPPORTS_END68]]:
+// CHECK-NEXT: [[TMP147:%.*]] = phi i1 [ false, %[[RESOLVER_ELSE66]] ], [ [[TMP146]], %[[CPU_SUPPORTS_EXTENSION67]] ]
+// CHECK-NEXT: br i1 [[TMP147]], label %[[RESOLVER_RETURN69:.*]], label %[[RESOLVER_ELSE70:.*]]
+// CHECK: [[RESOLVER_RETURN69]]:
+// CHECK-NEXT: ret ptr @priority._Mgcs
+// CHECK: [[RESOLVER_ELSE70]]:
+// CHECK-NEXT: [[TMP148:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP149:%.*]] = and i64 [[TMP148]], 2048
+// CHECK-NEXT: [[TMP150:%.*]] = icmp eq i64 [[TMP149]], 2048
+// CHECK-NEXT: [[TMP151:%.*]] = and i1 true, [[TMP150]]
+// CHECK-NEXT: br i1 [[TMP151]], label %[[RESOLVER_RETURN71:.*]], label %[[RESOLVER_ELSE72:.*]]
+// CHECK: [[RESOLVER_RETURN71]]:
+// CHECK-NEXT: ret ptr @priority._Mcssc
+// CHECK: [[RESOLVER_ELSE72]]:
+// CHECK-NEXT: ret ptr @priority.default
+//
diff --git a/compiler-rt/lib/builtins/cpu_model/AArch64CPUFeatures.inc b/compiler-rt/lib/builtins/cpu_model/AArch64CPUFeatures.inc
index 87a602aa5b271..e7d523cfbbad7 100644
--- a/compiler-rt/lib/builtins/cpu_model/AArch64CPUFeatures.inc
+++ b/compiler-rt/lib/builtins/cpu_model/AArch64CPUFeatures.inc
@@ -36,7 +36,7 @@ enum CPUFeatures {
FEAT_CSSC,
FEAT_SHA2,
FEAT_SHA3,
- RESERVED_FEAT_AES, // previously used and now ABI legacy
+ FEAT_FP8,
FEAT_PMULL,
FEAT_FP16,
FEAT_DIT,
@@ -47,45 +47,57 @@ enum CPUFeatures {
FEAT_RCPC,
FEAT_RCPC2,
FEAT_FRINTTS,
- RESERVED_FEAT_DGH, // previously used and now ABI legacy
+ FEAT_F8F32MM,
FEAT_I8MM,
FEAT_BF16,
- RESERVED_FEAT_EBF16, // previously used and now ABI legacy
- RESERVED_FEAT_RPRES, // previously used and now ABI legacy
+ FEAT_FP8DOT4,
+ FEAT_FP8DOT2,
FEAT_SVE,
- RESERVED_FEAT_SVE_BF16, // previously used and now ABI legacy
- RESERVED_FEAT_SVE_EBF16, // previously used and now ABI legacy
- RESERVED_FEAT_SVE_I8MM, // previously used and now ABI legacy
+ FEAT_FP8FMA,
+ FEAT_SME_F8F32,
+ FEAT_SSVE_FP8DOT4,
FEAT_SVE_F32MM,
FEAT_SVE_F64MM,
FEAT_SVE2,
- RESERVED_FEAT_SVE_AES, // previously used and now ABI legacy
+ FEAT_SSVE_FP8FMA,
FEAT_SVE_PMULL128,
FEAT_SVE_BITPERM,
FEAT_SVE_SHA3,
FEAT_SVE_SM4,
FEAT_SME,
- RESERVED_FEAT_MEMTAG, // previously used and now ABI legacy
+ FEAT_SSVE_BITPERM,
FEAT_MEMTAG2,
- RESERVED_FEAT_MEMTAG3, // previously used and now ABI legacy
+ FEAT_SSVE_FP8DOT2,
FEAT_SB,
- RESERVED_FEAT_PREDRES, // previously used and now ABI legacy
- RESERVED_FEAT_SSBS, // previously used and now ABI legacy
+ FEAT_SSVE_AES,
+ FEAT_SSVE_FEXPA,
FEAT_SSBS2,
FEAT_BTI,
- RESERVED_FEAT_LS64, // previously used and now ABI legacy
- RESERVED_FEAT_LS64_V, // previously used and now ABI legacy
- RESERVED_FEAT_LS64_ACCDATA, // previously used and now ABI legacy
+ FEAT_LUT,
+ FEAT_FAMINMAX,
+ FEAT_SME_LUTV2,
FEAT_WFXT,
FEAT_SME_F64,
FEAT_SME_I64,
FEAT_SME2,
FEAT_RCPC3,
FEAT_MOPS,
- FEAT_MAX,
+ FEAT_SME2P1,
+ FEAT_SME2P2,
FEAT_EXT = 62, // Reserved to indicate presence of additional features field
// in __aarch64_cpu_features
- FEAT_INIT // Used as flag of features initialization completion
+ FEAT_INIT, // Used as flag of features initialization completion
+ FEAT_SVE2P1,
+ FEAT_SVE2P2,
+ FEAT_SME_F16F16,
+ FEAT_GCS,
+ FEAT_SME_F8F16,
+ FEAT_F8F16MM,
+ FEAT_SVE_AES2,
+ FEAT_SVE_BFSCALE,
+ FEAT_SVE_F16F32MM,
+ FEAT_SME_MOP4,
+ FEAT_MAX
};
#endif
diff --git a/compiler-rt/lib/builtins/cpu_model/aarch64.c b/compiler-rt/lib/builtins/cpu_model/aarch64.c
index 347bb4e121c97..e2178edff476e 100644
--- a/compiler-rt/lib/builtins/cpu_model/aarch64.c
+++ b/compiler-rt/lib/builtins/cpu_model/aarch64.c
@@ -64,7 +64,7 @@ _Bool __aarch64_have_lse_atomics
// in Function Multi Versioning
struct {
unsigned long long features;
- // As features grows new fields could be added
+ unsigned long long features2;
} __aarch64_cpu_features __attribute__((visibility("hidden"), nocommon));
// The formatter wants to re-order these includes, but doing so is incorrect:
diff --git a/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/apple.inc b/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/apple.inc
index d5c85701ad1a0..74a317ba63555 100644
--- a/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/apple.inc
+++ b/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/apple.inc
@@ -34,6 +34,7 @@ static uint64_t deriveImplicitFeatures(uint64_t features) {
// FEAT_FP is always enabled
features |= (1ULL << FEAT_FP);
+ features |= (1ULL << FEAT_EXT);
features |= (1ULL << FEAT_INIT);
return features;
@@ -97,8 +98,9 @@ void __init_cpu_features_resolver(void) {
features = deriveImplicitFeatures(features);
- __atomic_store(&__aarch64_cpu_features.features, &features,
- __ATOMIC_RELAXED);
+ __atomic_store_n(&__aarch64_cpu_features.features2, 0, __ATOMIC_RELAXED);
+ __atomic_store_n(&__aarch64_cpu_features.features, features,
+ __ATOMIC_RELEASE);
return;
}
#endif
@@ -142,8 +144,9 @@ void __init_cpu_features_resolver(void) {
features = deriveImplicitFeatures(features);
- __atomic_store(&__aarch64_cpu_features.features, &features,
- __ATOMIC_RELAXED);
+ __atomic_store_n(&__aarch64_cpu_features.features2, 0, __ATOMIC_RELAXED);
+ __atomic_store_n(&__aarch64_cpu_features.features, features,
+ __ATOMIC_RELEASE);
}
#endif // TARGET_OS_OSX || TARGET_OS_IPHONE
diff --git a/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/baremetal.inc b/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/baremetal.inc
index e95ac9fac8709..7313cb79ca60e 100644
--- a/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/baremetal.inc
+++ b/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/baremetal.inc
@@ -32,5 +32,8 @@ void CONSTRUCTOR_ATTRIBUTE __init_cpu_features(void) {
if (has_sme)
feat |= 1ULL << FEAT_SME;
- __atomic_store_n(&__aarch64_cpu_features.features, feat, __ATOMIC_RELAXED);
+ feat |= 1ULL << FEAT_EXT;
+ feat |= 1ULL << FEAT_INIT;
+ __atomic_store_n(&__aarch64_cpu_features.features2, 0, __ATOMIC_RELAXED);
+ __atomic_store_n(&__aarch64_cpu_features.features, feat, __ATOMIC_RELEASE);
}
diff --git a/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/fuchsia.inc b/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/fuchsia.inc
index bb241f45fb141..115494f2596ed 100644
--- a/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/fuchsia.inc
+++ b/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/fuchsia.inc
@@ -44,6 +44,8 @@ void __init_cpu_features_resolver() {
setCPUFeature(FEAT_SVE);
setCPUFeature(FEAT_INIT);
+ setCPUFeature(FEAT_EXT);
- __atomic_store_n(&__aarch64_cpu_features.features, feat, __ATOMIC_RELAXED);
+ __atomic_store_n(&__aarch64_cpu_features.features2, 0, __ATOMIC_RELAXED);
+ __atomic_store_n(&__aarch64_cpu_features.features, feat, __ATOMIC_RELEASE);
}
diff --git a/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/hwcap.inc b/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/hwcap.inc
index 0f56cef97d4a3..29b0a570a1412 100644
--- a/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/hwcap.inc
+++ b/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/hwcap.inc
@@ -5,8 +5,8 @@
static void __init_cpu_features_constructor(unsigned long hwcap,
const __ifunc_arg_t *arg) {
- unsigned long long feat = 0;
-#define setCPUFeature(F) feat |= 1ULL << F
+ unsigned long long feat[2] = {0, 0};
+#define setCPUFeature(F) feat[(F) / 64] |= 1ULL << ((F) % 64)
unsigned long hwcap2 = 0;
if (hwcap & _IFUNC_ARG_HWCAP)
hwcap2 = arg->_hwcap2;
@@ -80,6 +80,61 @@ static void __init_cpu_features_constructor(unsigned long hwcap,
setCPUFeature(FEAT_MOPS);
if (hwcap2 & HWCAP2_CSSC)
setCPUFeature(FEAT_CSSC);
+ if ((hwcap2 & HWCAP2_F8CVT) && (hwcap2 & HWCAP2_F8E4M3) &&
+ (hwcap2 & HWCAP2_F8E5M2))
+ setCPUFeature(FEAT_FP8);
+ if (hwcap & HWCAP_F8MM8)
+ setCPUFeature(FEAT_F8F32MM);
+ if (hwcap2 & HWCAP2_F8DP4)
+ setCPUFeature(FEAT_FP8DOT4);
+ if (hwcap2 & HWCAP2_F8DP2)
+ setCPUFeature(FEAT_FP8DOT2);
+ if (hwcap2 & HWCAP2_F8FMA)
+ setCPUFeature(FEAT_FP8FMA);
+ if (hwcap2 & HWCAP2_SME_F8F32)
+ setCPUFeature(FEAT_SME_F8F32);
+ if (hwcap2 & HWCAP2_SME_SF8DP4)
+ setCPUFeature(FEAT_SSVE_FP8DOT4);
+ if (hwcap2 & HWCAP2_SME_SF8FMA)
+ setCPUFeature(FEAT_SSVE_FP8FMA);
+ if (hwcap & HWCAP_SME_SBITPERM)
+ setCPUFeature(FEAT_SSVE_BITPERM);
+ if (hwcap2 & HWCAP2_SME_SF8DP2)
+ setCPUFeature(FEAT_SSVE_FP8DOT2);
+ if (hwcap & HWCAP_SME_AES)
+ setCPUFeature(FEAT_SSVE_AES);
+ if (hwcap & HWCAP_SME_SFEXPA)
+ setCPUFeature(FEAT_SSVE_FEXPA);
+ if (hwcap2 & HWCAP2_LUT)
+ setCPUFeature(FEAT_LUT);
+ if (hwcap2 & HWCAP2_FAMINMAX)
+ setCPUFeature(FEAT_FAMINMAX);
+ if (hwcap2 & HWCAP2_SME_LUTV2)
+ setCPUFeature(FEAT_SME_LUTV2);
+ if (hwcap2 & HWCAP2_SME2P1)
+ setCPUFeature(FEAT_SME2P1);
+ if (hwcap & HWCAP_SME2P2)
+ setCPUFeature(FEAT_SME2P2);
+ if (hwcap2 & HWCAP2_SVE2P1)
+ setCPUFeature(FEAT_SVE2P1);
+ if (hwcap & HWCAP_SVE2P2)
+ setCPUFeature(FEAT_SVE2P2);
+ if (hwcap2 & HWCAP2_SME_F16F16)
+ setCPUFeature(FEAT_SME_F16F16);
+ if (hwcap & HWCAP_GCS)
+ setCPUFeature(FEAT_GCS);
+ if (hwcap2 & HWCAP2_SME_F8F16)
+ setCPUFeature(FEAT_SME_F8F16);
+ if (hwcap & HWCAP_F8MM4)
+ setCPUFeature(FEAT_F8F16MM);
+ if (hwcap & HWCAP_SVE_AES2)
+ setCPUFeature(FEAT_SVE_AES2);
+ if (hwcap & HWCAP_SVE_BFSCALE)
+ setCPUFeature(FEAT_SVE_BFSCALE);
+ if (hwcap & HWCAP_SVE_F16MM)
+ setCPUFeature(FEAT_SVE_F16F32MM);
+ if (hwcap & HWCAP_SME_SMOP4)
+ setCPUFeature(FEAT_SME_MOP4);
if (hwcap & HWCAP_FP) {
setCPUFeature(FEAT_FP);
// FP and AdvSIMD fields have the same value
@@ -101,7 +156,11 @@ static void __init_cpu_features_constructor(unsigned long hwcap,
setCPUFeature(FEAT_SVE2);
if (hwcap & HWCAP_SHA3)
setCPUFeature(FEAT_SHA3);
+ setCPUFeature(FEAT_EXT);
setCPUFeature(FEAT_INIT);
- __atomic_store_n(&__aarch64_cpu_features.features, feat, __ATOMIC_RELAXED);
+ __atomic_store_n(&__aarch64_cpu_features.features2, feat[1],
+ __ATOMIC_RELAXED);
+ __atomic_store_n(&__aarch64_cpu_features.features, feat[0],
+ __ATOMIC_RELEASE);
}
diff --git a/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/windows.inc b/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/windows.inc
index 7cbe0db7830c5..94e9cd19ab2ad 100644
--- a/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/windows.inc
+++ b/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/windows.inc
@@ -54,12 +54,45 @@
#ifndef PF_ARM_SME2_INSTRUCTIONS_AVAILABLE
#define PF_ARM_SME2_INSTRUCTIONS_AVAILABLE 71
#endif
+#ifndef PF_ARM_SME2_1_INSTRUCTIONS_AVAILABLE
+#define PF_ARM_SME2_1_INSTRUCTIONS_AVAILABLE 72
+#endif
+#ifndef PF_ARM_SME2_2_INSTRUCTIONS_AVAILABLE
+#define PF_ARM_SME2_2_INSTRUCTIONS_AVAILABLE 73
+#endif
+#ifndef PF_ARM_SME_AES_INSTRUCTIONS_AVAILABLE
+#define PF_ARM_SME_AES_INSTRUCTIONS_AVAILABLE 74
+#endif
+#ifndef PF_ARM_SME_SBITPERM_INSTRUCTIONS_AVAILABLE
+#define PF_ARM_SME_SBITPERM_INSTRUCTIONS_AVAILABLE 75
+#endif
+#ifndef PF_ARM_SME_SF8DP2_INSTRUCTIONS_AVAILABLE
+#define PF_ARM_SME_SF8DP2_INSTRUCTIONS_AVAILABLE 78
+#endif
+#ifndef PF_ARM_SME_SF8DP4_INSTRUCTIONS_AVAILABLE
+#define PF_ARM_SME_SF8DP4_INSTRUCTIONS_AVAILABLE 79
+#endif
+#ifndef PF_ARM_SME_SF8FMA_INSTRUCTIONS_AVAILABLE
+#define PF_ARM_SME_SF8FMA_INSTRUCTIONS_AVAILABLE 80
+#endif
+#ifndef PF_ARM_SME_F8F32_INSTRUCTIONS_AVAILABLE
+#define PF_ARM_SME_F8F32_INSTRUCTIONS_AVAILABLE 81
+#endif
+#ifndef PF_ARM_SME_F8F16_INSTRUCTIONS_AVAILABLE
+#define PF_ARM_SME_F8F16_INSTRUCTIONS_AVAILABLE 82
+#endif
+#ifndef PF_ARM_SME_F16F16_INSTRUCTIONS_AVAILABLE
+#define PF_ARM_SME_F16F16_INSTRUCTIONS_AVAILABLE 83
+#endif
#ifndef PF_ARM_SME_F64F64_INSTRUCTIONS_AVAILABLE
#define PF_ARM_SME_F64F64_INSTRUCTIONS_AVAILABLE 85
#endif
#ifndef PF_ARM_SME_I16I64_INSTRUCTIONS_AVAILABLE
#define PF_ARM_SME_I16I64_INSTRUCTIONS_AVAILABLE 86
#endif
+#ifndef PF_ARM_SME_LUTv2_INSTRUCTIONS_AVAILABLE
+#define PF_ARM_SME_LUTv2_INSTRUCTIONS_AVAILABLE 87
+#endif
void __init_cpu_features_resolver(unsigned long hwcap,
const __ifunc_arg_t *arg) {}
@@ -68,11 +101,12 @@ void CONSTRUCTOR_ATTRIBUTE __init_cpu_features(void) {
if (__atomic_load_n(&__aarch64_cpu_features.features, __ATOMIC_RELAXED))
return;
-#define setCPUFeature(F) features |= 1ULL << F
+#define setCPUFeature(F) features[(F) / 64] |= 1ULL << ((F) % 64)
- uint64_t features = 0;
+ uint64_t features[2] = {0, 0};
setCPUFeature(FEAT_INIT);
+ setCPUFeature(FEAT_EXT);
setCPUFeature(FEAT_FP);
// https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-isprocessorfeaturepresent
@@ -92,6 +126,7 @@ void CONSTRUCTOR_ATTRIBUTE __init_cpu_features(void) {
{PF_ARM_V83_LRCPC_INSTRUCTIONS_AVAILABLE, FEAT_RCPC},
{PF_ARM_SVE_INSTRUCTIONS_AVAILABLE, FEAT_SVE},
{PF_ARM_SVE2_INSTRUCTIONS_AVAILABLE, FEAT_SVE2},
+ {PF_ARM_SVE2_1_INSTRUCTIONS_AVAILABLE, FEAT_SVE2P1},
{PF_ARM_SVE_PMULL128_INSTRUCTIONS_AVAILABLE, FEAT_SVE_PMULL128},
{PF_ARM_SVE_BITPERM_INSTRUCTIONS_AVAILABLE, FEAT_SVE_BITPERM},
{PF_ARM_SVE_SHA3_INSTRUCTIONS_AVAILABLE, FEAT_SVE_SHA3},
@@ -103,8 +138,19 @@ void CONSTRUCTOR_ATTRIBUTE __init_cpu_features(void) {
{PF_ARM_V86_BF16_INSTRUCTIONS_AVAILABLE, FEAT_BF16},
{PF_ARM_SME_INSTRUCTIONS_AVAILABLE, FEAT_SME},
{PF_ARM_SME2_INSTRUCTIONS_AVAILABLE, FEAT_SME2},
+ {PF_ARM_SME2_1_INSTRUCTIONS_AVAILABLE, FEAT_SME2P1},
+ {PF_ARM_SME2_2_INSTRUCTIONS_AVAILABLE, FEAT_SME2P2},
+ {PF_ARM_SME_AES_INSTRUCTIONS_AVAILABLE, FEAT_SSVE_AES},
+ {PF_ARM_SME_SBITPERM_INSTRUCTIONS_AVAILABLE, FEAT_SSVE_BITPERM},
+ {PF_ARM_SME_SF8DP2_INSTRUCTIONS_AVAILABLE, FEAT_SSVE_FP8DOT2},
+ {PF_ARM_SME_SF8DP4_INSTRUCTIONS_AVAILABLE, FEAT_SSVE_FP8DOT4},
+ {PF_ARM_SME_SF8FMA_INSTRUCTIONS_AVAILABLE, FEAT_SSVE_FP8FMA},
+ {PF_ARM_SME_F8F32_INSTRUCTIONS_AVAILABLE, FEAT_SME_F8F32},
+ {PF_ARM_SME_F8F16_INSTRUCTIONS_AVAILABLE, FEAT_SME_F8F16},
+ {PF_ARM_SME_F16F16_INSTRUCTIONS_AVAILABLE, FEAT_SME_F16F16},
{PF_ARM_SME_I16I64_INSTRUCTIONS_AVAILABLE, FEAT_SME_I64},
{PF_ARM_SME_F64F64_INSTRUCTIONS_AVAILABLE, FEAT_SME_F64},
+ {PF_ARM_SME_LUTv2_INSTRUCTIONS_AVAILABLE, FEAT_SME_LUTV2},
};
// The following features are never detected because there is no known way
@@ -113,11 +159,17 @@ void CONSTRUCTOR_ATTRIBUTE __init_cpu_features(void) {
// FEAT_RNG, FEAT_FLAGM, FEAT_FLAGM2, FEAT_FP16FML, FEAT_RDM, FEAT_CSSC,
// FEAT_DIT, FEAT_DPB, FEAT_DPB2, FEAT_FCMA, FEAT_RCPC2, FEAT_FRINTTS,
// FEAT_MEMTAG2, FEAT_SB, FEAT_SSBS2, FEAT_BTI, FEAT_WFXT, FEAT_RCPC3,
- // FEAT_MOPS.
+ // FEAT_MOPS, FEAT_FP8, FEAT_F8F32MM, FEAT_FP8DOT4, FEAT_FP8DOT2, FEAT_FP8FMA,
+ // FEAT_SSVE_FEXPA, FEAT_LUT, FEAT_FAMINMAX, FEAT_SVE2P2, FEAT_GCS,
+ // FEAT_F8F16MM, FEAT_SVE_AES2, FEAT_SVE_BFSCALE,
+ // FEAT_SVE_F16F32MM, FEAT_SME_MOP4.
for (size_t I = 0, E = sizeof(FeatMap) / sizeof(FeatMap[0]); I != E; ++I)
if (IsProcessorFeaturePresent(FeatMap[I].WinApiFeature))
setCPUFeature(FeatMap[I].CPUFeature);
- __atomic_store(&__aarch64_cpu_features.features, &features, __ATOMIC_RELAXED);
+ __atomic_store_n(&__aarch64_cpu_features.features2, features[1],
+ __ATOMIC_RELAXED);
+ __atomic_store_n(&__aarch64_cpu_features.features, features[0],
+ __ATOMIC_RELEASE);
}
diff --git a/compiler-rt/lib/builtins/cpu_model/aarch64/hwcap.inc b/compiler-rt/lib/builtins/cpu_model/aarch64/hwcap.inc
index c70c73ccb2936..d4e19cf912862 100644
--- a/compiler-rt/lib/builtins/cpu_model/aarch64/hwcap.inc
+++ b/compiler-rt/lib/builtins/cpu_model/aarch64/hwcap.inc
@@ -88,6 +88,42 @@
#ifndef HWCAP_SB
#define HWCAP_SB (1 << 29)
#endif
+#ifndef HWCAP_GCS
+#define HWCAP_GCS (1UL << 32)
+#endif
+#ifndef HWCAP_F8MM8
+#define HWCAP_F8MM8 (1UL << 35)
+#endif
+#ifndef HWCAP_F8MM4
+#define HWCAP_F8MM4 (1UL << 36)
+#endif
+#ifndef HWCAP_SVE_F16MM
+#define HWCAP_SVE_F16MM (1UL << 37)
+#endif
+#ifndef HWCAP_SVE_AES2
+#define HWCAP_SVE_AES2 (1UL << 39)
+#endif
+#ifndef HWCAP_SVE_BFSCALE
+#define HWCAP_SVE_BFSCALE (1UL << 40)
+#endif
+#ifndef HWCAP_SVE2P2
+#define HWCAP_SVE2P2 (1UL << 41)
+#endif
+#ifndef HWCAP_SME2P2
+#define HWCAP_SME2P2 (1UL << 42)
+#endif
+#ifndef HWCAP_SME_SBITPERM
+#define HWCAP_SME_SBITPERM (1UL << 43)
+#endif
+#ifndef HWCAP_SME_AES
+#define HWCAP_SME_AES (1UL << 44)
+#endif
+#ifndef HWCAP_SME_SFEXPA
+#define HWCAP_SME_SFEXPA (1UL << 45)
+#endif
+#ifndef HWCAP_SME_SMOP4
+#define HWCAP_SME_SMOP4 (1UL << 47)
+#endif
#ifndef AT_HWCAP2
#define AT_HWCAP2 26
@@ -176,12 +212,63 @@
#ifndef HWCAP2_CSSC
#define HWCAP2_CSSC (1UL << 34)
#endif
+#ifndef HWCAP2_SVE2P1
+#define HWCAP2_SVE2P1 (1UL << 36)
+#endif
#ifndef HWCAP2_SME2
#define HWCAP2_SME2 (1UL << 37)
#endif
+#ifndef HWCAP2_SME2P1
+#define HWCAP2_SME2P1 (1UL << 38)
+#endif
+#ifndef HWCAP2_SME_F16F16
+#define HWCAP2_SME_F16F16 (1UL << 42)
+#endif
#ifndef HWCAP2_MOPS
#define HWCAP2_MOPS (1ULL << 43)
#endif
#ifndef HWCAP2_LRCPC3
#define HWCAP2_LRCPC3 (1UL << 46)
#endif
+#ifndef HWCAP2_LUT
+#define HWCAP2_LUT (1UL << 49)
+#endif
+#ifndef HWCAP2_FAMINMAX
+#define HWCAP2_FAMINMAX (1UL << 50)
+#endif
+#ifndef HWCAP2_F8CVT
+#define HWCAP2_F8CVT (1UL << 51)
+#endif
+#ifndef HWCAP2_F8FMA
+#define HWCAP2_F8FMA (1UL << 52)
+#endif
+#ifndef HWCAP2_F8DP4
+#define HWCAP2_F8DP4 (1UL << 53)
+#endif
+#ifndef HWCAP2_F8DP2
+#define HWCAP2_F8DP2 (1UL << 54)
+#endif
+#ifndef HWCAP2_F8E4M3
+#define HWCAP2_F8E4M3 (1UL << 55)
+#endif
+#ifndef HWCAP2_F8E5M2
+#define HWCAP2_F8E5M2 (1UL << 56)
+#endif
+#ifndef HWCAP2_SME_LUTV2
+#define HWCAP2_SME_LUTV2 (1UL << 57)
+#endif
+#ifndef HWCAP2_SME_F8F16
+#define HWCAP2_SME_F8F16 (1UL << 58)
+#endif
+#ifndef HWCAP2_SME_F8F32
+#define HWCAP2_SME_F8F32 (1UL << 59)
+#endif
+#ifndef HWCAP2_SME_SF8FMA
+#define HWCAP2_SME_SF8FMA (1UL << 60)
+#endif
+#ifndef HWCAP2_SME_SF8DP4
+#define HWCAP2_SME_SF8DP4 (1UL << 61)
+#endif
+#ifndef HWCAP2_SME_SF8DP2
+#define HWCAP2_SME_SF8DP2 (1UL << 62)
+#endif
diff --git a/llvm/include/llvm/TargetParser/AArch64CPUFeatures.inc b/llvm/include/llvm/TargetParser/AArch64CPUFeatures.inc
index 87a602aa5b271..e7d523cfbbad7 100644
--- a/llvm/include/llvm/TargetParser/AArch64CPUFeatures.inc
+++ b/llvm/include/llvm/TargetParser/AArch64CPUFeatures.inc
@@ -36,7 +36,7 @@ enum CPUFeatures {
FEAT_CSSC,
FEAT_SHA2,
FEAT_SHA3,
- RESERVED_FEAT_AES, // previously used and now ABI legacy
+ FEAT_FP8,
FEAT_PMULL,
FEAT_FP16,
FEAT_DIT,
@@ -47,45 +47,57 @@ enum CPUFeatures {
FEAT_RCPC,
FEAT_RCPC2,
FEAT_FRINTTS,
- RESERVED_FEAT_DGH, // previously used and now ABI legacy
+ FEAT_F8F32MM,
FEAT_I8MM,
FEAT_BF16,
- RESERVED_FEAT_EBF16, // previously used and now ABI legacy
- RESERVED_FEAT_RPRES, // previously used and now ABI legacy
+ FEAT_FP8DOT4,
+ FEAT_FP8DOT2,
FEAT_SVE,
- RESERVED_FEAT_SVE_BF16, // previously used and now ABI legacy
- RESERVED_FEAT_SVE_EBF16, // previously used and now ABI legacy
- RESERVED_FEAT_SVE_I8MM, // previously used and now ABI legacy
+ FEAT_FP8FMA,
+ FEAT_SME_F8F32,
+ FEAT_SSVE_FP8DOT4,
FEAT_SVE_F32MM,
FEAT_SVE_F64MM,
FEAT_SVE2,
- RESERVED_FEAT_SVE_AES, // previously used and now ABI legacy
+ FEAT_SSVE_FP8FMA,
FEAT_SVE_PMULL128,
FEAT_SVE_BITPERM,
FEAT_SVE_SHA3,
FEAT_SVE_SM4,
FEAT_SME,
- RESERVED_FEAT_MEMTAG, // previously used and now ABI legacy
+ FEAT_SSVE_BITPERM,
FEAT_MEMTAG2,
- RESERVED_FEAT_MEMTAG3, // previously used and now ABI legacy
+ FEAT_SSVE_FP8DOT2,
FEAT_SB,
- RESERVED_FEAT_PREDRES, // previously used and now ABI legacy
- RESERVED_FEAT_SSBS, // previously used and now ABI legacy
+ FEAT_SSVE_AES,
+ FEAT_SSVE_FEXPA,
FEAT_SSBS2,
FEAT_BTI,
- RESERVED_FEAT_LS64, // previously used and now ABI legacy
- RESERVED_FEAT_LS64_V, // previously used and now ABI legacy
- RESERVED_FEAT_LS64_ACCDATA, // previously used and now ABI legacy
+ FEAT_LUT,
+ FEAT_FAMINMAX,
+ FEAT_SME_LUTV2,
FEAT_WFXT,
FEAT_SME_F64,
FEAT_SME_I64,
FEAT_SME2,
FEAT_RCPC3,
FEAT_MOPS,
- FEAT_MAX,
+ FEAT_SME2P1,
+ FEAT_SME2P2,
FEAT_EXT = 62, // Reserved to indicate presence of additional features field
// in __aarch64_cpu_features
- FEAT_INIT // Used as flag of features initialization completion
+ FEAT_INIT, // Used as flag of features initialization completion
+ FEAT_SVE2P1,
+ FEAT_SVE2P2,
+ FEAT_SME_F16F16,
+ FEAT_GCS,
+ FEAT_SME_F8F16,
+ FEAT_F8F16MM,
+ FEAT_SVE_AES2,
+ FEAT_SVE_BFSCALE,
+ FEAT_SVE_F16F32MM,
+ FEAT_SME_MOP4,
+ FEAT_MAX
};
#endif
diff --git a/llvm/include/llvm/TargetParser/AArch64FeatPriorities.inc b/llvm/include/llvm/TargetParser/AArch64FeatPriorities.inc
index f0291926bbe75..57aa017d3cd25 100644
--- a/llvm/include/llvm/TargetParser/AArch64FeatPriorities.inc
+++ b/llvm/include/llvm/TargetParser/AArch64FeatPriorities.inc
@@ -60,6 +60,33 @@ enum FeatPriorities {
PRIOR_SME2,
PRIOR_MOPS,
PRIOR_CSSC,
+ PRIOR_GCS,
+ PRIOR_FP8,
+ PRIOR_F8F32MM,
+ PRIOR_FP8DOT4,
+ PRIOR_FP8DOT2,
+ PRIOR_FP8FMA,
+ PRIOR_LUT,
+ PRIOR_FAMINMAX,
+ PRIOR_F8F16MM,
+ PRIOR_SVE2P1,
+ PRIOR_SVE2P2,
+ PRIOR_SVE_AES2,
+ PRIOR_SVE_BFSCALE,
+ PRIOR_SVE_F16F32MM,
+ PRIOR_SSVE_FP8DOT4,
+ PRIOR_SSVE_FP8DOT2,
+ PRIOR_SSVE_FP8FMA,
+ PRIOR_SSVE_BITPERM,
+ PRIOR_SSVE_AES,
+ PRIOR_SSVE_FEXPA,
+ PRIOR_SME_F8F32,
+ PRIOR_SME_LUTV2,
+ PRIOR_SME2P1,
+ PRIOR_SME2P2,
+ PRIOR_SME_F16F16,
+ PRIOR_SME_F8F16,
+ PRIOR_SME_MOP4,
PRIOR_MAX,
PRIOR_P0 = 120,
PRIOR_P1,
diff --git a/llvm/include/llvm/TargetParser/AArch64TargetParser.h b/llvm/include/llvm/TargetParser/AArch64TargetParser.h
index 7dfd4f6678240..55637f3aa785c 100644
--- a/llvm/include/llvm/TargetParser/AArch64TargetParser.h
+++ b/llvm/include/llvm/TargetParser/AArch64TargetParser.h
@@ -39,8 +39,10 @@ struct CpuInfo;
#include "llvm/TargetParser/AArch64CPUFeatures.inc"
#include "llvm/TargetParser/AArch64FeatPriorities.inc"
-static_assert(FEAT_MAX < 62,
- "Number of features in CPUFeatures are limited to 62 entries");
+static_assert(FEAT_MAX <= 128,
+ "Number of features in CPUFeatures is limited to 128 entries");
+static_assert(FEAT_EXT == 62 && FEAT_INIT == 63,
+ "AArch64 CPU feature control bits are part of the runtime ABI");
static_assert(PRIOR_MAX < 120, "FeatPriorities is limited to 120 entries");
diff --git a/llvm/lib/Target/AArch64/AArch64FMV.td b/llvm/lib/Target/AArch64/AArch64FMV.td
index 12939997401ac..96e88456dffea 100644
--- a/llvm/lib/Target/AArch64/AArch64FMV.td
+++ b/llvm/lib/Target/AArch64/AArch64FMV.td
@@ -83,6 +83,33 @@ def : FMVExtension<"sve2-sha3", "SVE_SHA3">;
def : FMVExtension<"sve2-sm4", "SVE_SM4">;
def : FMVExtension<"wfxt", "WFXT">;
def : FMVExtension<"cssc", "CSSC">;
+def : FMVExtension<"gcs", "GCS">;
+def : FMVExtension<"fp8", "FP8">;
+def : FMVExtension<"f8f32mm", "F8F32MM">;
+def : FMVExtension<"fp8dot4", "FP8DOT4">;
+def : FMVExtension<"fp8dot2", "FP8DOT2">;
+def : FMVExtension<"fp8fma", "FP8FMA">;
+def : FMVExtension<"lut", "LUT">;
+def : FMVExtension<"faminmax", "FAMINMAX">;
+def : FMVExtension<"f8f16mm", "F8F16MM">;
+def : FMVExtension<"sve2p1", "SVE2P1">;
+def : FMVExtension<"sve2p2", "SVE2P2">;
+def : FMVExtension<"sve-aes2", "SVE_AES2">;
+def : FMVExtension<"sve-bfscale", "SVE_BFSCALE">;
+def : FMVExtension<"sve-f16f32mm", "SVE_F16F32MM">;
+def : FMVExtension<"ssve-fp8dot4", "SSVE_FP8DOT4">;
+def : FMVExtension<"ssve-fp8dot2", "SSVE_FP8DOT2">;
+def : FMVExtension<"ssve-fp8fma", "SSVE_FP8FMA">;
+def : FMVExtension<"ssve-bitperm", "SSVE_BITPERM">;
+def : FMVExtension<"ssve-aes", "SSVE_AES">;
+def : FMVExtension<"ssve-fexpa", "SSVE_FEXPA">;
+def : FMVExtension<"sme-f8f32", "SME_F8F32">;
+def : FMVExtension<"sme-lutv2", "SME_LUTV2">;
+def : FMVExtension<"sme2p1", "SME2P1">;
+def : FMVExtension<"sme2p2", "SME2P2">;
+def : FMVExtension<"sme-f16f16", "SME_F16F16">;
+def : FMVExtension<"sme-f8f16", "SME_F8F16">;
+def : FMVExtension<"sme-mop4", "SME_MOP4">;
// Extensions which allow the user to override version priority.
// 8-bits allow 256-1 priority levels (excluding all zeros).
>From 28ff38dbeac5d7b62643f580d776e60fd2bf7d90 Mon Sep 17 00:00:00 2001
From: Marian Lukac <Marian.Lukac at arm.com>
Date: Mon, 5 Oct 2026 16:42:30 +0000
Subject: [PATCH 3/4] Fix formatting
---
clang/lib/CodeGen/TargetBuiltins/ARM.cpp | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
index fd0728b5cb657..7f16ffd89ef70 100644
--- a/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
+++ b/clang/lib/CodeGen/TargetBuiltins/ARM.cpp
@@ -7416,13 +7416,12 @@ CodeGenFunction::EmitAArch64CpuSupports(ArrayRef<StringRef> FeaturesStrs) {
llvm::BasicBlock *LowerBlock = Builder.GetInsertBlock();
llvm::BasicBlock *ExtensionBlock =
createBasicBlock("cpu_supports.extension", CurFn);
- llvm::BasicBlock *EndBlock =
- createBasicBlock("cpu_supports.end", CurFn);
+ llvm::BasicBlock *EndBlock = createBasicBlock("cpu_supports.end", CurFn);
Builder.CreateCondBr(Result, ExtensionBlock, EndBlock);
Builder.SetInsertPoint(ExtensionBlock);
- Value *ExtendedFeatures = Builder.CreateGEP(
- Int8Ty, AArch64CPUFeatures, ConstantInt::get(Int64Ty, 8));
+ Value *ExtendedFeatures = Builder.CreateGEP(Int8Ty, AArch64CPUFeatures,
+ ConstantInt::get(Int64Ty, 8));
Features = Builder.CreateAlignedLoad(Int64Ty, ExtendedFeatures,
CharUnits::fromQuantity(8));
Mask = Builder.getInt(UpperMask);
>From 406f19a32bed0701966d01bbfc00c76c701de894 Mon Sep 17 00:00:00 2001
From: Marian Lukac <Marian.Lukac at arm.com>
Date: Tue, 13 Oct 2026 10:05:02 +0000
Subject: [PATCH 4/4] Address comments and added more tests
---
.../CodeGen/AArch64/cpu-supports-target.c | 150 ++-
clang/test/CodeGen/AArch64/cpu-supports.c | 1 +
clang/test/CodeGen/AArch64/fmv-detection.c | 940 ++++++++++++------
compiler-rt/lib/builtins/cpu_model/aarch64.c | 1 +
.../builtins/cpu_model/aarch64/fmv/apple.inc | 58 +-
.../builtins/cpu_model/aarch64/fmv/hwcap.inc | 3 +-
.../lib/builtins/cpu_model/aarch64/hwcap.inc | 1 +
7 files changed, 839 insertions(+), 315 deletions(-)
diff --git a/clang/test/CodeGen/AArch64/cpu-supports-target.c b/clang/test/CodeGen/AArch64/cpu-supports-target.c
index 1cd86210095b4..b570caad324d2 100644
--- a/clang/test/CodeGen/AArch64/cpu-supports-target.c
+++ b/clang/test/CodeGen/AArch64/cpu-supports-target.c
@@ -117,11 +117,115 @@
// CHECK-NEXT: store i32 12, ptr [[RETVAL]], align 4
// CHECK-NEXT: br label %[[RETURN]]
// CHECK: [[IF_ELSE22]]:
+// CHECK-NEXT: [[TMP48:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP49:%.*]] = and i64 [[TMP48]], 4611686018427387904
+// CHECK-NEXT: [[TMP50:%.*]] = icmp eq i64 [[TMP49]], 4611686018427387904
+// CHECK-NEXT: [[TMP51:%.*]] = and i1 true, [[TMP50]]
+// CHECK-NEXT: br i1 [[TMP51]], label %[[CPU_SUPPORTS_EXTENSION:.*]], label %[[CPU_SUPPORTS_END:.*]]
+// CHECK: [[CPU_SUPPORTS_EXTENSION]]:
+// CHECK-NEXT: [[TMP52:%.*]] = load i64, ptr getelementptr inbounds nuw (i8, ptr @__aarch64_cpu_features, i64 8), align 8
+// CHECK-NEXT: [[TMP53:%.*]] = and i64 [[TMP52]], 8
+// CHECK-NEXT: [[TMP54:%.*]] = icmp eq i64 [[TMP53]], 8
+// CHECK-NEXT: br label %[[CPU_SUPPORTS_END]]
+// CHECK: [[CPU_SUPPORTS_END]]:
+// CHECK-NEXT: [[TMP55:%.*]] = phi i1 [ false, %[[IF_ELSE22]] ], [ [[TMP54]], %[[CPU_SUPPORTS_EXTENSION]] ]
+// CHECK-NEXT: br i1 [[TMP55]], label %[[IF_THEN23:.*]], label %[[IF_ELSE24:.*]]
+// CHECK: [[IF_THEN23]]:
+// CHECK-NEXT: store i32 13, ptr [[RETVAL]], align 4
+// CHECK-NEXT: br label %[[RETURN]]
+// CHECK: [[IF_ELSE24]]:
+// CHECK-NEXT: [[TMP56:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP57:%.*]] = and i64 [[TMP56]], 2986361600
+// CHECK-NEXT: [[TMP58:%.*]] = icmp eq i64 [[TMP57]], 2986361600
+// CHECK-NEXT: [[TMP59:%.*]] = and i1 true, [[TMP58]]
+// CHECK-NEXT: br i1 [[TMP59]], label %[[IF_THEN25:.*]], label %[[IF_ELSE26:.*]]
+// CHECK: [[IF_THEN25]]:
+// CHECK-NEXT: store i32 14, ptr [[RETVAL]], align 4
+// CHECK-NEXT: br label %[[RETURN]]
+// CHECK: [[IF_ELSE26]]:
+// CHECK-NEXT: [[TMP60:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP61:%.*]] = and i64 [[TMP60]], 4618441417868460800
+// CHECK-NEXT: [[TMP62:%.*]] = icmp eq i64 [[TMP61]], 4618441417868460800
+// CHECK-NEXT: [[TMP63:%.*]] = and i1 true, [[TMP62]]
+// CHECK-NEXT: br i1 [[TMP63]], label %[[CPU_SUPPORTS_EXTENSION27:.*]], label %[[CPU_SUPPORTS_END28:.*]]
+// CHECK: [[CPU_SUPPORTS_EXTENSION27]]:
+// CHECK-NEXT: [[TMP64:%.*]] = load i64, ptr getelementptr inbounds nuw (i8, ptr @__aarch64_cpu_features, i64 8), align 8
+// CHECK-NEXT: [[TMP65:%.*]] = and i64 [[TMP64]], 32
+// CHECK-NEXT: [[TMP66:%.*]] = icmp eq i64 [[TMP65]], 32
+// CHECK-NEXT: br label %[[CPU_SUPPORTS_END28]]
+// CHECK: [[CPU_SUPPORTS_END28]]:
+// CHECK-NEXT: [[TMP67:%.*]] = phi i1 [ false, %[[IF_ELSE26]] ], [ [[TMP66]], %[[CPU_SUPPORTS_EXTENSION27]] ]
+// CHECK-NEXT: br i1 [[TMP67]], label %[[IF_THEN29:.*]], label %[[IF_ELSE30:.*]]
+// CHECK: [[IF_THEN29]]:
+// CHECK-NEXT: store i32 15, ptr [[RETVAL]], align 4
+// CHECK-NEXT: br label %[[RETURN]]
+// CHECK: [[IF_ELSE30]]:
+// CHECK-NEXT: [[TMP68:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP69:%.*]] = and i64 [[TMP68]], 4611686088220672256
+// CHECK-NEXT: [[TMP70:%.*]] = icmp eq i64 [[TMP69]], 4611686088220672256
+// CHECK-NEXT: [[TMP71:%.*]] = and i1 true, [[TMP70]]
+// CHECK-NEXT: br i1 [[TMP71]], label %[[CPU_SUPPORTS_EXTENSION31:.*]], label %[[CPU_SUPPORTS_END32:.*]]
+// CHECK: [[CPU_SUPPORTS_EXTENSION31]]:
+// CHECK-NEXT: [[TMP72:%.*]] = load i64, ptr getelementptr inbounds nuw (i8, ptr @__aarch64_cpu_features, i64 8), align 8
+// CHECK-NEXT: [[TMP73:%.*]] = and i64 [[TMP72]], 451
+// CHECK-NEXT: [[TMP74:%.*]] = icmp eq i64 [[TMP73]], 451
+// CHECK-NEXT: br label %[[CPU_SUPPORTS_END32]]
+// CHECK: [[CPU_SUPPORTS_END32]]:
+// CHECK-NEXT: [[TMP75:%.*]] = phi i1 [ false, %[[IF_ELSE30]] ], [ [[TMP74]], %[[CPU_SUPPORTS_EXTENSION31]] ]
+// CHECK-NEXT: br i1 [[TMP75]], label %[[IF_THEN33:.*]], label %[[IF_ELSE34:.*]]
+// CHECK: [[IF_THEN33]]:
+// CHECK-NEXT: store i32 16, ptr [[RETVAL]], align 4
+// CHECK-NEXT: br label %[[RETURN]]
+// CHECK: [[IF_ELSE34]]:
+// CHECK-NEXT: [[TMP76:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP77:%.*]] = and i64 [[TMP76]], 144154916657644288
+// CHECK-NEXT: [[TMP78:%.*]] = icmp eq i64 [[TMP77]], 144154916657644288
+// CHECK-NEXT: [[TMP79:%.*]] = and i1 true, [[TMP78]]
+// CHECK-NEXT: br i1 [[TMP79]], label %[[IF_THEN35:.*]], label %[[IF_ELSE36:.*]]
+// CHECK: [[IF_THEN35]]:
+// CHECK-NEXT: store i32 17, ptr [[RETVAL]], align 4
+// CHECK-NEXT: br label %[[RETURN]]
+// CHECK: [[IF_ELSE36]]:
+// CHECK-NEXT: [[TMP80:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP81:%.*]] = and i64 [[TMP80]], 144550594814771968
+// CHECK-NEXT: [[TMP82:%.*]] = icmp eq i64 [[TMP81]], 144550594814771968
+// CHECK-NEXT: [[TMP83:%.*]] = and i1 true, [[TMP82]]
+// CHECK-NEXT: br i1 [[TMP83]], label %[[IF_THEN37:.*]], label %[[IF_ELSE38:.*]]
+// CHECK: [[IF_THEN37]]:
+// CHECK-NEXT: store i32 18, ptr [[RETVAL]], align 4
+// CHECK-NEXT: br label %[[RETURN]]
+// CHECK: [[IF_ELSE38]]:
+// CHECK-NEXT: [[TMP84:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP85:%.*]] = and i64 [[TMP84]], 3611891303626916608
+// CHECK-NEXT: [[TMP86:%.*]] = icmp eq i64 [[TMP85]], 3611891303626916608
+// CHECK-NEXT: [[TMP87:%.*]] = and i1 true, [[TMP86]]
+// CHECK-NEXT: br i1 [[TMP87]], label %[[IF_THEN39:.*]], label %[[IF_ELSE40:.*]]
+// CHECK: [[IF_THEN39]]:
+// CHECK-NEXT: store i32 19, ptr [[RETVAL]], align 4
+// CHECK-NEXT: br label %[[RETURN]]
+// CHECK: [[IF_ELSE40]]:
+// CHECK-NEXT: [[TMP88:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP89:%.*]] = and i64 [[TMP88]], 4755805604684055296
+// CHECK-NEXT: [[TMP90:%.*]] = icmp eq i64 [[TMP89]], 4755805604684055296
+// CHECK-NEXT: [[TMP91:%.*]] = and i1 true, [[TMP90]]
+// CHECK-NEXT: br i1 [[TMP91]], label %[[CPU_SUPPORTS_EXTENSION41:.*]], label %[[CPU_SUPPORTS_END42:.*]]
+// CHECK: [[CPU_SUPPORTS_EXTENSION41]]:
+// CHECK-NEXT: [[TMP92:%.*]] = load i64, ptr getelementptr inbounds nuw (i8, ptr @__aarch64_cpu_features, i64 8), align 8
+// CHECK-NEXT: [[TMP93:%.*]] = and i64 [[TMP92]], 532
+// CHECK-NEXT: [[TMP94:%.*]] = icmp eq i64 [[TMP93]], 532
+// CHECK-NEXT: br label %[[CPU_SUPPORTS_END42]]
+// CHECK: [[CPU_SUPPORTS_END42]]:
+// CHECK-NEXT: [[TMP95:%.*]] = phi i1 [ false, %[[IF_ELSE40]] ], [ [[TMP94]], %[[CPU_SUPPORTS_EXTENSION41]] ]
+// CHECK-NEXT: br i1 [[TMP95]], label %[[IF_THEN43:.*]], label %[[IF_ELSE44:.*]]
+// CHECK: [[IF_THEN43]]:
+// CHECK-NEXT: store i32 20, ptr [[RETVAL]], align 4
+// CHECK-NEXT: br label %[[RETURN]]
+// CHECK: [[IF_ELSE44]]:
// CHECK-NEXT: store i32 0, ptr [[RETVAL]], align 4
// CHECK-NEXT: br label %[[RETURN]]
// CHECK: [[RETURN]]:
-// CHECK-NEXT: [[TMP48:%.*]] = load i32, ptr [[RETVAL]], align 4
-// CHECK-NEXT: ret i32 [[TMP48]]
+// CHECK-NEXT: [[TMP96:%.*]] = load i32, ptr [[RETVAL]], align 4
+// CHECK-NEXT: ret i32 [[TMP96]]
//
int check_all_features() {
if (__builtin_cpu_supports("rng+flagm+flagm2+fp16fml+dotprod+sm4"))
@@ -148,6 +252,22 @@ int check_all_features() {
return 11;
else if (__builtin_cpu_supports("sme-i16i64+sme2"))
return 12;
+ else if (__builtin_cpu_supports("gcs"))
+ return 13;
+ else if (__builtin_cpu_supports("fp8+f8f32mm+fp8dot4+fp8dot2+fp8fma"))
+ return 14;
+ else if (__builtin_cpu_supports("lut+faminmax+f8f16mm"))
+ return 15;
+ else if (__builtin_cpu_supports("sve2p1+sve2p2+sve-aes2+sve-bfscale+sve-f16f32mm"))
+ return 16;
+ else if (__builtin_cpu_supports("ssve-fp8dot4+ssve-fp8dot2+ssve-fp8fma"))
+ return 17;
+ else if (__builtin_cpu_supports("ssve-bitperm+ssve-aes+ssve-fexpa"))
+ return 18;
+ else if (__builtin_cpu_supports("sme-f8f32+sme-lutv2+sme2p1+sme2p2"))
+ return 19;
+ else if (__builtin_cpu_supports("sme-f16f16+sme-f8f16+sme-mop4"))
+ return 20;
else
return 0;
}
@@ -220,16 +340,24 @@ int test_versions() {
// CHECK: Function Attrs: noinline nounwind optnone
// CHECK-LABEL: define dso_local i32 @test_long(
// CHECK-SAME: ) #[[ATTR0]] {
-// CHECK-NEXT: [[ENTRY:.*:]]
+// CHECK-NEXT: [[ENTRY:.*]]:
// CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 577586773744664575
-// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 577586773744664575
+// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 8808385837051150335
+// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 8808385837051150335
// CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]]
-// CHECK-NEXT: [[CONV:%.*]] = zext i1 [[TMP3]] to i32
+// CHECK-NEXT: br i1 [[TMP3]], label %[[CPU_SUPPORTS_EXTENSION:.*]], label %[[CPU_SUPPORTS_END:.*]]
+// CHECK: [[CPU_SUPPORTS_EXTENSION]]:
+// CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr getelementptr inbounds nuw (i8, ptr @__aarch64_cpu_features, i64 8), align 8
+// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 1023
+// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 1023
+// CHECK-NEXT: br label %[[CPU_SUPPORTS_END]]
+// CHECK: [[CPU_SUPPORTS_END]]:
+// CHECK-NEXT: [[TMP7:%.*]] = phi i1 [ false, %[[ENTRY]] ], [ [[TMP6]], %[[CPU_SUPPORTS_EXTENSION]] ]
+// CHECK-NEXT: [[CONV:%.*]] = zext i1 [[TMP7]] to i32
// CHECK-NEXT: ret i32 [[CONV]]
//
int test_long(void) {
- return __builtin_cpu_supports("rng+flagm+flagm2+fp16fml+dotprod+sm4+rdm+lse+fp+simd+aes+bf16+bti+crc+cssc+dit+dotprod+f32mm+f64mm+flagm+fp16fml+fp16+i8mm+mops+sha2+sha3+sm4+sve2");
+ return __builtin_cpu_supports("rng+flagm+flagm2+fp16fml+dotprod+sm4+rdm+lse+fp+simd+aes+bf16+bti+crc+cssc+dit+dotprod+f32mm+f64mm+flagm+fp16fml+fp16+i8mm+mops+sha2+sha3+sm4+sve2+gcs+fp8+f8f32mm+fp8dot4+fp8dot2+fp8fma+lut+faminmax+f8f16mm+sve2p1+sve2p2+sve-aes2+sve-bfscale+sve-f16f32mm+ssve-fp8dot4+ssve-fp8dot2+ssve-fp8fma+ssve-bitperm+ssve-aes+ssve-fexpa+sme-f8f32+sme-lutv2+sme2p1+sme2p2+sme-f16f16+sme-f8f16+sme-mop4");
}
//.
@@ -237,5 +365,11 @@ int test_long(void) {
// CHECK: attributes #[[ATTR1]] = { noinline nounwind optnone "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+neon" }
// CHECK: attributes #[[ATTR2]] = { noinline nounwind optnone vscale_range(1,16) "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-features"="+fp-armv8,+fullfp16,+sve" }
//.
-// CHECK: [[META0:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
+// CHECK: [[META0:![0-9]+]] = !{i32 1, !"ptrauth-elf-got", i32 0}
+// CHECK: [[META1:![0-9]+]] = !{i32 1, !"ptrauth-init-fini", i32 0}
+// CHECK: [[META2:![0-9]+]] = !{i32 1, !"ptrauth-init-fini-address-discrimination", i32 0}
+// CHECK: [[META3:![0-9]+]] = !{i32 1, !"ptrauth-sign-personality", i32 0}
+// CHECK: [[META4:![0-9]+]] = !{i32 1, !"aarch64-elf-pauthabi-platform", i32 268435458}
+// CHECK: [[META5:![0-9]+]] = !{i32 1, !"aarch64-elf-pauthabi-version", i32 0}
+// CHECK: [[META6:![0-9]+]] = !{!"{{.*}}clang version {{.*}}"}
//.
diff --git a/clang/test/CodeGen/AArch64/cpu-supports.c b/clang/test/CodeGen/AArch64/cpu-supports.c
index a317d0d35d6af..4523c50f37675 100644
--- a/clang/test/CodeGen/AArch64/cpu-supports.c
+++ b/clang/test/CodeGen/AArch64/cpu-supports.c
@@ -76,6 +76,7 @@ int main(void) {
if (__builtin_cpu_supports("avx2"))
return 4;
+ //Test a feature in the second uint64_t bitfield.
if (__builtin_cpu_supports("sve2p1"))
return 5;
diff --git a/clang/test/CodeGen/AArch64/fmv-detection.c b/clang/test/CodeGen/AArch64/fmv-detection.c
index 344bde0af8c01..72a3b6367ea9c 100644
--- a/clang/test/CodeGen/AArch64/fmv-detection.c
+++ b/clang/test/CodeGen/AArch64/fmv-detection.c
@@ -95,6 +95,60 @@ __attribute__((target_version("sve2-sm4"))) int fmv(void) { return 0; }
__attribute__((target_version("wfxt"))) int fmv(void) { return 0; }
+__attribute__((target_version("gcs"))) int fmv(void);
+
+__attribute__((target_version("fp8"))) int fmv(void);
+
+__attribute__((target_version("f8f32mm"))) int fmv(void);
+
+__attribute__((target_version("fp8dot4"))) int fmv(void);
+
+__attribute__((target_version("fp8dot2"))) int fmv(void);
+
+__attribute__((target_version("fp8fma"))) int fmv(void);
+
+__attribute__((target_version("lut"))) int fmv(void);
+
+__attribute__((target_version("faminmax"))) int fmv(void);
+
+__attribute__((target_version("f8f16mm"))) int fmv(void);
+
+__attribute__((target_version("sve2p1"))) int fmv(void);
+
+__attribute__((target_version("sve2p2"))) int fmv(void);
+
+__attribute__((target_version("sve-aes2"))) int fmv(void);
+
+__attribute__((target_version("sve-bfscale"))) int fmv(void);
+
+__attribute__((target_version("sve-f16f32mm"))) int fmv(void);
+
+__attribute__((target_version("ssve-fp8dot4"))) int fmv(void);
+
+__attribute__((target_version("ssve-fp8dot2"))) int fmv(void);
+
+__attribute__((target_version("ssve-fp8fma"))) int fmv(void);
+
+__attribute__((target_version("ssve-bitperm"))) int fmv(void);
+
+__attribute__((target_version("ssve-aes"))) int fmv(void);
+
+__attribute__((target_version("ssve-fexpa"))) int fmv(void);
+
+__attribute__((target_version("sme-f8f32"))) int fmv(void);
+
+__attribute__((target_version("sme-lutv2"))) int fmv(void);
+
+__attribute__((target_version("sme2p1"))) int fmv(void);
+
+__attribute__((target_version("sme2p2"))) int fmv(void);
+
+__attribute__((target_version("sme-f16f16"))) int fmv(void);
+
+__attribute__((target_version("sme-f8f16"))) int fmv(void);
+
+__attribute__((target_version("sme-mop4"))) int fmv(void);
+
__attribute__((target_version("cssc+fp"))) int fmv(void);
__attribute__((target_version("default"))) int fmv(void) { return 0; }
@@ -440,376 +494,672 @@ int caller() {
// CHECK: Function Attrs: disable_sanitizer_instrumentation
// CHECK-LABEL: define {{[^@]+}}@fmv.resolver
// CHECK-SAME: () #[[ATTR48:[0-9]+]] comdat {
-// CHECK-NEXT: resolver_entry:
+// CHECK-NEXT: [[RESOLVER_ENTRY:.*]]:
// CHECK-NEXT: call void @__init_cpu_features_resolver()
// CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 2304
-// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 2304
+// CHECK-NEXT: [[TMP1:%.*]] = and i64 [[TMP0]], 4755805604684038912
+// CHECK-NEXT: [[TMP2:%.*]] = icmp eq i64 [[TMP1]], 4755805604684038912
// CHECK-NEXT: [[TMP3:%.*]] = and i1 true, [[TMP2]]
-// CHECK-NEXT: br i1 [[TMP3]], label [[RESOLVER_RETURN:%.*]], label [[RESOLVER_ELSE:%.*]]
-// CHECK: resolver_return:
-// CHECK-NEXT: ret ptr @fmv._McsscMfp
-// CHECK: resolver_else:
-// CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 2048
-// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 2048
-// CHECK-NEXT: [[TMP7:%.*]] = and i1 true, [[TMP6]]
-// CHECK-NEXT: br i1 [[TMP7]], label [[RESOLVER_RETURN1:%.*]], label [[RESOLVER_ELSE2:%.*]]
-// CHECK: resolver_return1:
-// CHECK-NEXT: ret ptr @fmv._Mcssc
-// CHECK: resolver_else2:
+// CHECK-NEXT: br i1 [[TMP3]], label %[[CPU_SUPPORTS_EXTENSION:.*]], label %[[CPU_SUPPORTS_END:.*]]
+// CHECK: [[CPU_SUPPORTS_EXTENSION]]:
+// CHECK-NEXT: [[TMP4:%.*]] = load i64, ptr getelementptr inbounds nuw (i8, ptr @__aarch64_cpu_features, i64 8), align 8
+// CHECK-NEXT: [[TMP5:%.*]] = and i64 [[TMP4]], 512
+// CHECK-NEXT: [[TMP6:%.*]] = icmp eq i64 [[TMP5]], 512
+// CHECK-NEXT: br label %[[CPU_SUPPORTS_END]]
+// CHECK: [[CPU_SUPPORTS_END]]:
+// CHECK-NEXT: [[TMP7:%.*]] = phi i1 [ false, %[[RESOLVER_ENTRY]] ], [ [[TMP6]], %[[CPU_SUPPORTS_EXTENSION]] ]
+// CHECK-NEXT: br i1 [[TMP7]], label %[[RESOLVER_RETURN:.*]], label %[[RESOLVER_ELSE:.*]]
+// CHECK: [[RESOLVER_RETURN]]:
+// CHECK-NEXT: ret ptr @fmv._Msme-mop4
+// CHECK: [[RESOLVER_ELSE]]:
// CHECK-NEXT: [[TMP8:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP9:%.*]] = and i64 [[TMP8]], 576460752303423488
-// CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 576460752303423488
+// CHECK-NEXT: [[TMP9:%.*]] = and i64 [[TMP8]], 4755805604684055296
+// CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[TMP9]], 4755805604684055296
// CHECK-NEXT: [[TMP11:%.*]] = and i1 true, [[TMP10]]
-// CHECK-NEXT: br i1 [[TMP11]], label [[RESOLVER_RETURN3:%.*]], label [[RESOLVER_ELSE4:%.*]]
-// CHECK: resolver_return3:
-// CHECK-NEXT: ret ptr @fmv._Mmops
-// CHECK: resolver_else4:
-// CHECK-NEXT: [[TMP12:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP13:%.*]] = and i64 [[TMP12]], 144119586256651008
-// CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 144119586256651008
-// CHECK-NEXT: [[TMP15:%.*]] = and i1 true, [[TMP14]]
-// CHECK-NEXT: br i1 [[TMP15]], label [[RESOLVER_RETURN5:%.*]], label [[RESOLVER_ELSE6:%.*]]
-// CHECK: resolver_return5:
-// CHECK-NEXT: ret ptr @fmv._Msme2
-// CHECK: resolver_else6:
+// CHECK-NEXT: br i1 [[TMP11]], label %[[CPU_SUPPORTS_EXTENSION1:.*]], label %[[CPU_SUPPORTS_END2:.*]]
+// CHECK: [[CPU_SUPPORTS_EXTENSION1]]:
+// CHECK-NEXT: [[TMP12:%.*]] = load i64, ptr getelementptr inbounds nuw (i8, ptr @__aarch64_cpu_features, i64 8), align 8
+// CHECK-NEXT: [[TMP13:%.*]] = and i64 [[TMP12]], 16
+// CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[TMP13]], 16
+// CHECK-NEXT: br label %[[CPU_SUPPORTS_END2]]
+// CHECK: [[CPU_SUPPORTS_END2]]:
+// CHECK-NEXT: [[TMP15:%.*]] = phi i1 [ false, %[[RESOLVER_ELSE]] ], [ [[TMP14]], %[[CPU_SUPPORTS_EXTENSION1]] ]
+// CHECK-NEXT: br i1 [[TMP15]], label %[[RESOLVER_RETURN3:.*]], label %[[RESOLVER_ELSE4:.*]]
+// CHECK: [[RESOLVER_RETURN3]]:
+// CHECK-NEXT: ret ptr @fmv._Msme-f8f16
+// CHECK: [[RESOLVER_ELSE4]]:
// CHECK-NEXT: [[TMP16:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP17:%.*]] = and i64 [[TMP16]], 72061992218723072
-// CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[TMP17]], 72061992218723072
+// CHECK-NEXT: [[TMP17:%.*]] = and i64 [[TMP16]], 4755805604684038912
+// CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[TMP17]], 4755805604684038912
// CHECK-NEXT: [[TMP19:%.*]] = and i1 true, [[TMP18]]
-// CHECK-NEXT: br i1 [[TMP19]], label [[RESOLVER_RETURN7:%.*]], label [[RESOLVER_ELSE8:%.*]]
-// CHECK: resolver_return7:
-// CHECK-NEXT: ret ptr @fmv._Msme-i16i64
-// CHECK: resolver_else8:
-// CHECK-NEXT: [[TMP20:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP21:%.*]] = and i64 [[TMP20]], 36033195199759104
-// CHECK-NEXT: [[TMP22:%.*]] = icmp eq i64 [[TMP21]], 36033195199759104
-// CHECK-NEXT: [[TMP23:%.*]] = and i1 true, [[TMP22]]
-// CHECK-NEXT: br i1 [[TMP23]], label [[RESOLVER_RETURN9:%.*]], label [[RESOLVER_ELSE10:%.*]]
-// CHECK: resolver_return9:
-// CHECK-NEXT: ret ptr @fmv._Msme-f64f64
-// CHECK: resolver_else10:
+// CHECK-NEXT: br i1 [[TMP19]], label %[[CPU_SUPPORTS_EXTENSION5:.*]], label %[[CPU_SUPPORTS_END6:.*]]
+// CHECK: [[CPU_SUPPORTS_EXTENSION5]]:
+// CHECK-NEXT: [[TMP20:%.*]] = load i64, ptr getelementptr inbounds nuw (i8, ptr @__aarch64_cpu_features, i64 8), align 8
+// CHECK-NEXT: [[TMP21:%.*]] = and i64 [[TMP20]], 4
+// CHECK-NEXT: [[TMP22:%.*]] = icmp eq i64 [[TMP21]], 4
+// CHECK-NEXT: br label %[[CPU_SUPPORTS_END6]]
+// CHECK: [[CPU_SUPPORTS_END6]]:
+// CHECK-NEXT: [[TMP23:%.*]] = phi i1 [ false, %[[RESOLVER_ELSE4]] ], [ [[TMP22]], %[[CPU_SUPPORTS_EXTENSION5]] ]
+// CHECK-NEXT: br i1 [[TMP23]], label %[[RESOLVER_RETURN7:.*]], label %[[RESOLVER_ELSE8:.*]]
+// CHECK: [[RESOLVER_RETURN7]]:
+// CHECK-NEXT: ret ptr @fmv._Msme-f16f16
+// CHECK: [[RESOLVER_ELSE8]]:
// CHECK-NEXT: [[TMP24:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP25:%.*]] = and i64 [[TMP24]], 18014398509481984
-// CHECK-NEXT: [[TMP26:%.*]] = icmp eq i64 [[TMP25]], 18014398509481984
+// CHECK-NEXT: [[TMP25:%.*]] = and i64 [[TMP24]], 3602884100077191936
+// CHECK-NEXT: [[TMP26:%.*]] = icmp eq i64 [[TMP25]], 3602884100077191936
// CHECK-NEXT: [[TMP27:%.*]] = and i1 true, [[TMP26]]
-// CHECK-NEXT: br i1 [[TMP27]], label [[RESOLVER_RETURN11:%.*]], label [[RESOLVER_ELSE12:%.*]]
-// CHECK: resolver_return11:
-// CHECK-NEXT: ret ptr @fmv._Mwfxt
-// CHECK: resolver_else12:
+// CHECK-NEXT: br i1 [[TMP27]], label %[[RESOLVER_RETURN9:.*]], label %[[RESOLVER_ELSE10:.*]]
+// CHECK: [[RESOLVER_RETURN9]]:
+// CHECK-NEXT: ret ptr @fmv._Msme2p2
+// CHECK: [[RESOLVER_ELSE10]]:
// CHECK-NEXT: [[TMP28:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP29:%.*]] = and i64 [[TMP28]], 1125899906842624
-// CHECK-NEXT: [[TMP30:%.*]] = icmp eq i64 [[TMP29]], 1125899906842624
+// CHECK-NEXT: [[TMP29:%.*]] = and i64 [[TMP28]], 1297041090863497984
+// CHECK-NEXT: [[TMP30:%.*]] = icmp eq i64 [[TMP29]], 1297041090863497984
// CHECK-NEXT: [[TMP31:%.*]] = and i1 true, [[TMP30]]
-// CHECK-NEXT: br i1 [[TMP31]], label [[RESOLVER_RETURN13:%.*]], label [[RESOLVER_ELSE14:%.*]]
-// CHECK: resolver_return13:
-// CHECK-NEXT: ret ptr @fmv._Mbti
-// CHECK: resolver_else14:
+// CHECK-NEXT: br i1 [[TMP31]], label %[[RESOLVER_RETURN11:.*]], label %[[RESOLVER_ELSE12:.*]]
+// CHECK: [[RESOLVER_RETURN11]]:
+// CHECK-NEXT: ret ptr @fmv._Msme2p1
+// CHECK: [[RESOLVER_ELSE12]]:
// CHECK-NEXT: [[TMP32:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP33:%.*]] = and i64 [[TMP32]], 562949953421312
-// CHECK-NEXT: [[TMP34:%.*]] = icmp eq i64 [[TMP33]], 562949953421312
+// CHECK-NEXT: [[TMP33:%.*]] = and i64 [[TMP32]], 153126785511392000
+// CHECK-NEXT: [[TMP34:%.*]] = icmp eq i64 [[TMP33]], 153126785511392000
// CHECK-NEXT: [[TMP35:%.*]] = and i1 true, [[TMP34]]
-// CHECK-NEXT: br i1 [[TMP35]], label [[RESOLVER_RETURN15:%.*]], label [[RESOLVER_ELSE16:%.*]]
-// CHECK: resolver_return15:
-// CHECK-NEXT: ret ptr @fmv._Mssbs
-// CHECK: resolver_else16:
+// CHECK-NEXT: br i1 [[TMP35]], label %[[RESOLVER_RETURN13:.*]], label %[[RESOLVER_ELSE14:.*]]
+// CHECK: [[RESOLVER_RETURN13]]:
+// CHECK-NEXT: ret ptr @fmv._Msme-lutv2
+// CHECK: [[RESOLVER_ELSE14]]:
// CHECK-NEXT: [[TMP36:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP37:%.*]] = and i64 [[TMP36]], 70368744177664
-// CHECK-NEXT: [[TMP38:%.*]] = icmp eq i64 [[TMP37]], 70368744177664
+// CHECK-NEXT: [[TMP37:%.*]] = and i64 [[TMP36]], 144119590551634688
+// CHECK-NEXT: [[TMP38:%.*]] = icmp eq i64 [[TMP37]], 144119590551634688
// CHECK-NEXT: [[TMP39:%.*]] = and i1 true, [[TMP38]]
-// CHECK-NEXT: br i1 [[TMP39]], label [[RESOLVER_RETURN17:%.*]], label [[RESOLVER_ELSE18:%.*]]
-// CHECK: resolver_return17:
-// CHECK-NEXT: ret ptr @fmv._Msb
-// CHECK: resolver_else18:
+// CHECK-NEXT: br i1 [[TMP39]], label %[[RESOLVER_RETURN15:.*]], label %[[RESOLVER_ELSE16:.*]]
+// CHECK: [[RESOLVER_RETURN15]]:
+// CHECK-NEXT: ret ptr @fmv._Msme-f8f32
+// CHECK: [[RESOLVER_ELSE16]]:
// CHECK-NEXT: [[TMP40:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP41:%.*]] = and i64 [[TMP40]], 17592186044416
-// CHECK-NEXT: [[TMP42:%.*]] = icmp eq i64 [[TMP41]], 17592186044416
+// CHECK-NEXT: [[TMP41:%.*]] = and i64 [[TMP40]], 144401061233361664
+// CHECK-NEXT: [[TMP42:%.*]] = icmp eq i64 [[TMP41]], 144401061233361664
// CHECK-NEXT: [[TMP43:%.*]] = and i1 true, [[TMP42]]
-// CHECK-NEXT: br i1 [[TMP43]], label [[RESOLVER_RETURN19:%.*]], label [[RESOLVER_ELSE20:%.*]]
-// CHECK: resolver_return19:
-// CHECK-NEXT: ret ptr @fmv._Mmemtag
-// CHECK: resolver_else20:
+// CHECK-NEXT: br i1 [[TMP43]], label %[[RESOLVER_RETURN17:.*]], label %[[RESOLVER_ELSE18:.*]]
+// CHECK: [[RESOLVER_RETURN17]]:
+// CHECK-NEXT: ret ptr @fmv._Mssve-fexpa
+// CHECK: [[RESOLVER_ELSE18]]:
// CHECK-NEXT: [[TMP44:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP45:%.*]] = and i64 [[TMP44]], 4398180795136
-// CHECK-NEXT: [[TMP46:%.*]] = icmp eq i64 [[TMP45]], 4398180795136
+// CHECK-NEXT: [[TMP45:%.*]] = and i64 [[TMP44]], 144260323745039104
+// CHECK-NEXT: [[TMP46:%.*]] = icmp eq i64 [[TMP45]], 144260323745039104
// CHECK-NEXT: [[TMP47:%.*]] = and i1 true, [[TMP46]]
-// CHECK-NEXT: br i1 [[TMP47]], label [[RESOLVER_RETURN21:%.*]], label [[RESOLVER_ELSE22:%.*]]
-// CHECK: resolver_return21:
-// CHECK-NEXT: ret ptr @fmv._Msme
-// CHECK: resolver_else22:
+// CHECK-NEXT: br i1 [[TMP47]], label %[[RESOLVER_RETURN19:.*]], label %[[RESOLVER_ELSE20:.*]]
+// CHECK: [[RESOLVER_RETURN19]]:
+// CHECK-NEXT: ret ptr @fmv._Mssve-aes
+// CHECK: [[RESOLVER_ELSE20]]:
// CHECK-NEXT: [[TMP48:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP49:%.*]] = and i64 [[TMP48]], 2268816540448
-// CHECK-NEXT: [[TMP50:%.*]] = icmp eq i64 [[TMP49]], 2268816540448
+// CHECK-NEXT: [[TMP49:%.*]] = and i64 [[TMP48]], 144128382349673216
+// CHECK-NEXT: [[TMP50:%.*]] = icmp eq i64 [[TMP49]], 144128382349673216
// CHECK-NEXT: [[TMP51:%.*]] = and i1 true, [[TMP50]]
-// CHECK-NEXT: br i1 [[TMP51]], label [[RESOLVER_RETURN23:%.*]], label [[RESOLVER_ELSE24:%.*]]
-// CHECK: resolver_return23:
-// CHECK-NEXT: ret ptr @fmv._Msve2-sm4
-// CHECK: resolver_else24:
+// CHECK-NEXT: br i1 [[TMP51]], label %[[RESOLVER_RETURN21:.*]], label %[[RESOLVER_ELSE22:.*]]
+// CHECK: [[RESOLVER_RETURN21]]:
+// CHECK-NEXT: ret ptr @fmv._Mssve-bitperm
+// CHECK: [[RESOLVER_ELSE22]]:
// CHECK-NEXT: [[TMP52:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP53:%.*]] = and i64 [[TMP52]], 1169304924928
-// CHECK-NEXT: [[TMP54:%.*]] = icmp eq i64 [[TMP53]], 1169304924928
+// CHECK-NEXT: [[TMP53:%.*]] = and i64 [[TMP52]], 144119723695620864
+// CHECK-NEXT: [[TMP54:%.*]] = icmp eq i64 [[TMP53]], 144119723695620864
// CHECK-NEXT: [[TMP55:%.*]] = and i1 true, [[TMP54]]
-// CHECK-NEXT: br i1 [[TMP55]], label [[RESOLVER_RETURN25:%.*]], label [[RESOLVER_ELSE26:%.*]]
-// CHECK: resolver_return25:
-// CHECK-NEXT: ret ptr @fmv._Msve2-sha3
-// CHECK: resolver_else26:
+// CHECK-NEXT: br i1 [[TMP55]], label %[[RESOLVER_RETURN23:.*]], label %[[RESOLVER_ELSE24:.*]]
+// CHECK: [[RESOLVER_RETURN23]]:
+// CHECK-NEXT: ret ptr @fmv._Mssve-fp8fma
+// CHECK: [[RESOLVER_ELSE24]]:
// CHECK-NEXT: [[TMP56:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP57:%.*]] = and i64 [[TMP56]], 619549098240
-// CHECK-NEXT: [[TMP58:%.*]] = icmp eq i64 [[TMP57]], 619549098240
+// CHECK-NEXT: [[TMP57:%.*]] = and i64 [[TMP56]], 144154770628756224
+// CHECK-NEXT: [[TMP58:%.*]] = icmp eq i64 [[TMP57]], 144154770628756224
// CHECK-NEXT: [[TMP59:%.*]] = and i1 true, [[TMP58]]
-// CHECK-NEXT: br i1 [[TMP59]], label [[RESOLVER_RETURN27:%.*]], label [[RESOLVER_ELSE28:%.*]]
-// CHECK: resolver_return27:
-// CHECK-NEXT: ret ptr @fmv._Msve2-bitperm
-// CHECK: resolver_else28:
+// CHECK-NEXT: br i1 [[TMP59]], label %[[RESOLVER_RETURN25:.*]], label %[[RESOLVER_ELSE26:.*]]
+// CHECK: [[RESOLVER_RETURN25]]:
+// CHECK-NEXT: ret ptr @fmv._Mssve-fp8dot2
+// CHECK: [[RESOLVER_ELSE26]]:
// CHECK-NEXT: [[TMP60:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP61:%.*]] = and i64 [[TMP60]], 344671224576
-// CHECK-NEXT: [[TMP62:%.*]] = icmp eq i64 [[TMP61]], 344671224576
+// CHECK-NEXT: [[TMP61:%.*]] = and i64 [[TMP60]], 144119594846601984
+// CHECK-NEXT: [[TMP62:%.*]] = icmp eq i64 [[TMP61]], 144119594846601984
// CHECK-NEXT: [[TMP63:%.*]] = and i1 true, [[TMP62]]
-// CHECK-NEXT: br i1 [[TMP63]], label [[RESOLVER_RETURN29:%.*]], label [[RESOLVER_ELSE30:%.*]]
-// CHECK: resolver_return29:
-// CHECK-NEXT: ret ptr @fmv._Msve2-aes
-// CHECK: resolver_else30:
+// CHECK-NEXT: br i1 [[TMP63]], label %[[RESOLVER_RETURN27:.*]], label %[[RESOLVER_ELSE28:.*]]
+// CHECK: [[RESOLVER_RETURN27]]:
+// CHECK-NEXT: ret ptr @fmv._Mssve-fp8dot4
+// CHECK: [[RESOLVER_ELSE28]]:
// CHECK-NEXT: [[TMP64:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP65:%.*]] = and i64 [[TMP64]], 69793284352
-// CHECK-NEXT: [[TMP66:%.*]] = icmp eq i64 [[TMP65]], 69793284352
+// CHECK-NEXT: [[TMP65:%.*]] = and i64 [[TMP64]], 4611686019501195520
+// CHECK-NEXT: [[TMP66:%.*]] = icmp eq i64 [[TMP65]], 4611686019501195520
// CHECK-NEXT: [[TMP67:%.*]] = and i1 true, [[TMP66]]
-// CHECK-NEXT: br i1 [[TMP67]], label [[RESOLVER_RETURN31:%.*]], label [[RESOLVER_ELSE32:%.*]]
-// CHECK: resolver_return31:
-// CHECK-NEXT: ret ptr @fmv._Msve2
-// CHECK: resolver_else32:
-// CHECK-NEXT: [[TMP68:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP69:%.*]] = and i64 [[TMP68]], 35433545984
-// CHECK-NEXT: [[TMP70:%.*]] = icmp eq i64 [[TMP69]], 35433545984
-// CHECK-NEXT: [[TMP71:%.*]] = and i1 true, [[TMP70]]
-// CHECK-NEXT: br i1 [[TMP71]], label [[RESOLVER_RETURN33:%.*]], label [[RESOLVER_ELSE34:%.*]]
-// CHECK: resolver_return33:
-// CHECK-NEXT: ret ptr @fmv._Mf64mm
-// CHECK: resolver_else34:
+// CHECK-NEXT: br i1 [[TMP67]], label %[[CPU_SUPPORTS_EXTENSION29:.*]], label %[[CPU_SUPPORTS_END30:.*]]
+// CHECK: [[CPU_SUPPORTS_EXTENSION29]]:
+// CHECK-NEXT: [[TMP68:%.*]] = load i64, ptr getelementptr inbounds nuw (i8, ptr @__aarch64_cpu_features, i64 8), align 8
+// CHECK-NEXT: [[TMP69:%.*]] = and i64 [[TMP68]], 256
+// CHECK-NEXT: [[TMP70:%.*]] = icmp eq i64 [[TMP69]], 256
+// CHECK-NEXT: br label %[[CPU_SUPPORTS_END30]]
+// CHECK: [[CPU_SUPPORTS_END30]]:
+// CHECK-NEXT: [[TMP71:%.*]] = phi i1 [ false, %[[RESOLVER_ELSE28]] ], [ [[TMP70]], %[[CPU_SUPPORTS_EXTENSION29]] ]
+// CHECK-NEXT: br i1 [[TMP71]], label %[[RESOLVER_RETURN31:.*]], label %[[RESOLVER_ELSE32:.*]]
+// CHECK: [[RESOLVER_RETURN31]]:
+// CHECK-NEXT: ret ptr @fmv._Msve-f16f32mm
+// CHECK: [[RESOLVER_ELSE32]]:
// CHECK-NEXT: [[TMP72:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP73:%.*]] = and i64 [[TMP72]], 18253676800
-// CHECK-NEXT: [[TMP74:%.*]] = icmp eq i64 [[TMP73]], 18253676800
+// CHECK-NEXT: [[TMP73:%.*]] = and i64 [[TMP72]], 4611686018427387904
+// CHECK-NEXT: [[TMP74:%.*]] = icmp eq i64 [[TMP73]], 4611686018427387904
// CHECK-NEXT: [[TMP75:%.*]] = and i1 true, [[TMP74]]
-// CHECK-NEXT: br i1 [[TMP75]], label [[RESOLVER_RETURN35:%.*]], label [[RESOLVER_ELSE36:%.*]]
-// CHECK: resolver_return35:
-// CHECK-NEXT: ret ptr @fmv._Mf32mm
-// CHECK: resolver_else36:
-// CHECK-NEXT: [[TMP76:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP77:%.*]] = and i64 [[TMP76]], 1073807616
-// CHECK-NEXT: [[TMP78:%.*]] = icmp eq i64 [[TMP77]], 1073807616
-// CHECK-NEXT: [[TMP79:%.*]] = and i1 true, [[TMP78]]
-// CHECK-NEXT: br i1 [[TMP79]], label [[RESOLVER_RETURN37:%.*]], label [[RESOLVER_ELSE38:%.*]]
-// CHECK: resolver_return37:
-// CHECK-NEXT: ret ptr @fmv._Msve
-// CHECK: resolver_else38:
+// CHECK-NEXT: br i1 [[TMP75]], label %[[CPU_SUPPORTS_EXTENSION33:.*]], label %[[CPU_SUPPORTS_END34:.*]]
+// CHECK: [[CPU_SUPPORTS_EXTENSION33]]:
+// CHECK-NEXT: [[TMP76:%.*]] = load i64, ptr getelementptr inbounds nuw (i8, ptr @__aarch64_cpu_features, i64 8), align 8
+// CHECK-NEXT: [[TMP77:%.*]] = and i64 [[TMP76]], 128
+// CHECK-NEXT: [[TMP78:%.*]] = icmp eq i64 [[TMP77]], 128
+// CHECK-NEXT: br label %[[CPU_SUPPORTS_END34]]
+// CHECK: [[CPU_SUPPORTS_END34]]:
+// CHECK-NEXT: [[TMP79:%.*]] = phi i1 [ false, %[[RESOLVER_ELSE32]] ], [ [[TMP78]], %[[CPU_SUPPORTS_EXTENSION33]] ]
+// CHECK-NEXT: br i1 [[TMP79]], label %[[RESOLVER_RETURN35:.*]], label %[[RESOLVER_ELSE36:.*]]
+// CHECK: [[RESOLVER_RETURN35]]:
+// CHECK-NEXT: ret ptr @fmv._Msve-bfscale
+// CHECK: [[RESOLVER_ELSE36]]:
// CHECK-NEXT: [[TMP80:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP81:%.*]] = and i64 [[TMP80]], 134218496
-// CHECK-NEXT: [[TMP82:%.*]] = icmp eq i64 [[TMP81]], 134218496
+// CHECK-NEXT: [[TMP81:%.*]] = and i64 [[TMP80]], 4611686018427387904
+// CHECK-NEXT: [[TMP82:%.*]] = icmp eq i64 [[TMP81]], 4611686018427387904
// CHECK-NEXT: [[TMP83:%.*]] = and i1 true, [[TMP82]]
-// CHECK-NEXT: br i1 [[TMP83]], label [[RESOLVER_RETURN39:%.*]], label [[RESOLVER_ELSE40:%.*]]
-// CHECK: resolver_return39:
-// CHECK-NEXT: ret ptr @fmv._Mbf16
-// CHECK: resolver_else40:
-// CHECK-NEXT: [[TMP84:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP85:%.*]] = and i64 [[TMP84]], 67109632
-// CHECK-NEXT: [[TMP86:%.*]] = icmp eq i64 [[TMP85]], 67109632
-// CHECK-NEXT: [[TMP87:%.*]] = and i1 true, [[TMP86]]
-// CHECK-NEXT: br i1 [[TMP87]], label [[RESOLVER_RETURN41:%.*]], label [[RESOLVER_ELSE42:%.*]]
-// CHECK: resolver_return41:
-// CHECK-NEXT: ret ptr @fmv._Mi8mm
-// CHECK: resolver_else42:
+// CHECK-NEXT: br i1 [[TMP83]], label %[[CPU_SUPPORTS_EXTENSION37:.*]], label %[[CPU_SUPPORTS_END38:.*]]
+// CHECK: [[CPU_SUPPORTS_EXTENSION37]]:
+// CHECK-NEXT: [[TMP84:%.*]] = load i64, ptr getelementptr inbounds nuw (i8, ptr @__aarch64_cpu_features, i64 8), align 8
+// CHECK-NEXT: [[TMP85:%.*]] = and i64 [[TMP84]], 64
+// CHECK-NEXT: [[TMP86:%.*]] = icmp eq i64 [[TMP85]], 64
+// CHECK-NEXT: br label %[[CPU_SUPPORTS_END38]]
+// CHECK: [[CPU_SUPPORTS_END38]]:
+// CHECK-NEXT: [[TMP87:%.*]] = phi i1 [ false, %[[RESOLVER_ELSE36]] ], [ [[TMP86]], %[[CPU_SUPPORTS_EXTENSION37]] ]
+// CHECK-NEXT: br i1 [[TMP87]], label %[[RESOLVER_RETURN39:.*]], label %[[RESOLVER_ELSE40:.*]]
+// CHECK: [[RESOLVER_RETURN39]]:
+// CHECK-NEXT: ret ptr @fmv._Msve-aes2
+// CHECK: [[RESOLVER_ELSE40]]:
// CHECK-NEXT: [[TMP88:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP89:%.*]] = and i64 [[TMP88]], 16777472
-// CHECK-NEXT: [[TMP90:%.*]] = icmp eq i64 [[TMP89]], 16777472
+// CHECK-NEXT: [[TMP89:%.*]] = and i64 [[TMP88]], 4611686088220672256
+// CHECK-NEXT: [[TMP90:%.*]] = icmp eq i64 [[TMP89]], 4611686088220672256
// CHECK-NEXT: [[TMP91:%.*]] = and i1 true, [[TMP90]]
-// CHECK-NEXT: br i1 [[TMP91]], label [[RESOLVER_RETURN43:%.*]], label [[RESOLVER_ELSE44:%.*]]
-// CHECK: resolver_return43:
-// CHECK-NEXT: ret ptr @fmv._Mfrintts
-// CHECK: resolver_else44:
-// CHECK-NEXT: [[TMP92:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP93:%.*]] = and i64 [[TMP92]], 288230376164294656
-// CHECK-NEXT: [[TMP94:%.*]] = icmp eq i64 [[TMP93]], 288230376164294656
-// CHECK-NEXT: [[TMP95:%.*]] = and i1 true, [[TMP94]]
-// CHECK-NEXT: br i1 [[TMP95]], label [[RESOLVER_RETURN45:%.*]], label [[RESOLVER_ELSE46:%.*]]
-// CHECK: resolver_return45:
-// CHECK-NEXT: ret ptr @fmv._Mrcpc3
-// CHECK: resolver_else46:
+// CHECK-NEXT: br i1 [[TMP91]], label %[[CPU_SUPPORTS_EXTENSION41:.*]], label %[[CPU_SUPPORTS_END42:.*]]
+// CHECK: [[CPU_SUPPORTS_EXTENSION41]]:
+// CHECK-NEXT: [[TMP92:%.*]] = load i64, ptr getelementptr inbounds nuw (i8, ptr @__aarch64_cpu_features, i64 8), align 8
+// CHECK-NEXT: [[TMP93:%.*]] = and i64 [[TMP92]], 3
+// CHECK-NEXT: [[TMP94:%.*]] = icmp eq i64 [[TMP93]], 3
+// CHECK-NEXT: br label %[[CPU_SUPPORTS_END42]]
+// CHECK: [[CPU_SUPPORTS_END42]]:
+// CHECK-NEXT: [[TMP95:%.*]] = phi i1 [ false, %[[RESOLVER_ELSE40]] ], [ [[TMP94]], %[[CPU_SUPPORTS_EXTENSION41]] ]
+// CHECK-NEXT: br i1 [[TMP95]], label %[[RESOLVER_RETURN43:.*]], label %[[RESOLVER_ELSE44:.*]]
+// CHECK: [[RESOLVER_RETURN43]]:
+// CHECK-NEXT: ret ptr @fmv._Msve2p2
+// CHECK: [[RESOLVER_ELSE44]]:
// CHECK-NEXT: [[TMP96:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP97:%.*]] = and i64 [[TMP96]], 12582912
-// CHECK-NEXT: [[TMP98:%.*]] = icmp eq i64 [[TMP97]], 12582912
+// CHECK-NEXT: [[TMP97:%.*]] = and i64 [[TMP96]], 4611686088220672256
+// CHECK-NEXT: [[TMP98:%.*]] = icmp eq i64 [[TMP97]], 4611686088220672256
// CHECK-NEXT: [[TMP99:%.*]] = and i1 true, [[TMP98]]
-// CHECK-NEXT: br i1 [[TMP99]], label [[RESOLVER_RETURN47:%.*]], label [[RESOLVER_ELSE48:%.*]]
-// CHECK: resolver_return47:
-// CHECK-NEXT: ret ptr @fmv._Mrcpc2
-// CHECK: resolver_else48:
-// CHECK-NEXT: [[TMP100:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP101:%.*]] = and i64 [[TMP100]], 4194304
-// CHECK-NEXT: [[TMP102:%.*]] = icmp eq i64 [[TMP101]], 4194304
-// CHECK-NEXT: [[TMP103:%.*]] = and i1 true, [[TMP102]]
-// CHECK-NEXT: br i1 [[TMP103]], label [[RESOLVER_RETURN49:%.*]], label [[RESOLVER_ELSE50:%.*]]
-// CHECK: resolver_return49:
-// CHECK-NEXT: ret ptr @fmv._Mrcpc
-// CHECK: resolver_else50:
+// CHECK-NEXT: br i1 [[TMP99]], label %[[CPU_SUPPORTS_EXTENSION45:.*]], label %[[CPU_SUPPORTS_END46:.*]]
+// CHECK: [[CPU_SUPPORTS_EXTENSION45]]:
+// CHECK-NEXT: [[TMP100:%.*]] = load i64, ptr getelementptr inbounds nuw (i8, ptr @__aarch64_cpu_features, i64 8), align 8
+// CHECK-NEXT: [[TMP101:%.*]] = and i64 [[TMP100]], 1
+// CHECK-NEXT: [[TMP102:%.*]] = icmp eq i64 [[TMP101]], 1
+// CHECK-NEXT: br label %[[CPU_SUPPORTS_END46]]
+// CHECK: [[CPU_SUPPORTS_END46]]:
+// CHECK-NEXT: [[TMP103:%.*]] = phi i1 [ false, %[[RESOLVER_ELSE44]] ], [ [[TMP102]], %[[CPU_SUPPORTS_EXTENSION45]] ]
+// CHECK-NEXT: br i1 [[TMP103]], label %[[RESOLVER_RETURN47:.*]], label %[[RESOLVER_ELSE48:.*]]
+// CHECK: [[RESOLVER_RETURN47]]:
+// CHECK-NEXT: ret ptr @fmv._Msve2p1
+// CHECK: [[RESOLVER_ELSE48]]:
// CHECK-NEXT: [[TMP104:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP105:%.*]] = and i64 [[TMP104]], 2097920
-// CHECK-NEXT: [[TMP106:%.*]] = icmp eq i64 [[TMP105]], 2097920
+// CHECK-NEXT: [[TMP105:%.*]] = and i64 [[TMP104]], 4611686018427405056
+// CHECK-NEXT: [[TMP106:%.*]] = icmp eq i64 [[TMP105]], 4611686018427405056
// CHECK-NEXT: [[TMP107:%.*]] = and i1 true, [[TMP106]]
-// CHECK-NEXT: br i1 [[TMP107]], label [[RESOLVER_RETURN51:%.*]], label [[RESOLVER_ELSE52:%.*]]
-// CHECK: resolver_return51:
-// CHECK-NEXT: ret ptr @fmv._Mfcma
-// CHECK: resolver_else52:
-// CHECK-NEXT: [[TMP108:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP109:%.*]] = and i64 [[TMP108]], 1048832
-// CHECK-NEXT: [[TMP110:%.*]] = icmp eq i64 [[TMP109]], 1048832
-// CHECK-NEXT: [[TMP111:%.*]] = and i1 true, [[TMP110]]
-// CHECK-NEXT: br i1 [[TMP111]], label [[RESOLVER_RETURN53:%.*]], label [[RESOLVER_ELSE54:%.*]]
-// CHECK: resolver_return53:
-// CHECK-NEXT: ret ptr @fmv._Mjscvt
-// CHECK: resolver_else54:
+// CHECK-NEXT: br i1 [[TMP107]], label %[[CPU_SUPPORTS_EXTENSION49:.*]], label %[[CPU_SUPPORTS_END50:.*]]
+// CHECK: [[CPU_SUPPORTS_EXTENSION49]]:
+// CHECK-NEXT: [[TMP108:%.*]] = load i64, ptr getelementptr inbounds nuw (i8, ptr @__aarch64_cpu_features, i64 8), align 8
+// CHECK-NEXT: [[TMP109:%.*]] = and i64 [[TMP108]], 32
+// CHECK-NEXT: [[TMP110:%.*]] = icmp eq i64 [[TMP109]], 32
+// CHECK-NEXT: br label %[[CPU_SUPPORTS_END50]]
+// CHECK: [[CPU_SUPPORTS_END50]]:
+// CHECK-NEXT: [[TMP111:%.*]] = phi i1 [ false, %[[RESOLVER_ELSE48]] ], [ [[TMP110]], %[[CPU_SUPPORTS_EXTENSION49]] ]
+// CHECK-NEXT: br i1 [[TMP111]], label %[[RESOLVER_RETURN51:.*]], label %[[RESOLVER_ELSE52:.*]]
+// CHECK: [[RESOLVER_RETURN51]]:
+// CHECK-NEXT: ret ptr @fmv._Mf8f16mm
+// CHECK: [[RESOLVER_ELSE52]]:
// CHECK-NEXT: [[TMP112:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP113:%.*]] = and i64 [[TMP112]], 786432
-// CHECK-NEXT: [[TMP114:%.*]] = icmp eq i64 [[TMP113]], 786432
+// CHECK-NEXT: [[TMP113:%.*]] = and i64 [[TMP112]], 4503599627371264
+// CHECK-NEXT: [[TMP114:%.*]] = icmp eq i64 [[TMP113]], 4503599627371264
// CHECK-NEXT: [[TMP115:%.*]] = and i1 true, [[TMP114]]
-// CHECK-NEXT: br i1 [[TMP115]], label [[RESOLVER_RETURN55:%.*]], label [[RESOLVER_ELSE56:%.*]]
-// CHECK: resolver_return55:
-// CHECK-NEXT: ret ptr @fmv._Mdpb2
-// CHECK: resolver_else56:
+// CHECK-NEXT: br i1 [[TMP115]], label %[[RESOLVER_RETURN53:.*]], label %[[RESOLVER_ELSE54:.*]]
+// CHECK: [[RESOLVER_RETURN53]]:
+// CHECK-NEXT: ret ptr @fmv._Mfaminmax
+// CHECK: [[RESOLVER_ELSE54]]:
// CHECK-NEXT: [[TMP116:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP117:%.*]] = and i64 [[TMP116]], 262144
-// CHECK-NEXT: [[TMP118:%.*]] = icmp eq i64 [[TMP117]], 262144
+// CHECK-NEXT: [[TMP117:%.*]] = and i64 [[TMP116]], 2251799813686016
+// CHECK-NEXT: [[TMP118:%.*]] = icmp eq i64 [[TMP117]], 2251799813686016
// CHECK-NEXT: [[TMP119:%.*]] = and i1 true, [[TMP118]]
-// CHECK-NEXT: br i1 [[TMP119]], label [[RESOLVER_RETURN57:%.*]], label [[RESOLVER_ELSE58:%.*]]
-// CHECK: resolver_return57:
-// CHECK-NEXT: ret ptr @fmv._Mdpb
-// CHECK: resolver_else58:
+// CHECK-NEXT: br i1 [[TMP119]], label %[[RESOLVER_RETURN55:.*]], label %[[RESOLVER_ELSE56:.*]]
+// CHECK: [[RESOLVER_RETURN55]]:
+// CHECK-NEXT: ret ptr @fmv._Mlut
+// CHECK: [[RESOLVER_ELSE56]]:
// CHECK-NEXT: [[TMP120:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP121:%.*]] = and i64 [[TMP120]], 131072
-// CHECK-NEXT: [[TMP122:%.*]] = icmp eq i64 [[TMP121]], 131072
+// CHECK-NEXT: [[TMP121:%.*]] = and i64 [[TMP120]], 2147500800
+// CHECK-NEXT: [[TMP122:%.*]] = icmp eq i64 [[TMP121]], 2147500800
// CHECK-NEXT: [[TMP123:%.*]] = and i1 true, [[TMP122]]
-// CHECK-NEXT: br i1 [[TMP123]], label [[RESOLVER_RETURN59:%.*]], label [[RESOLVER_ELSE60:%.*]]
-// CHECK: resolver_return59:
-// CHECK-NEXT: ret ptr @fmv._Mdit
-// CHECK: resolver_else60:
+// CHECK-NEXT: br i1 [[TMP123]], label %[[RESOLVER_RETURN57:.*]], label %[[RESOLVER_ELSE58:.*]]
+// CHECK: [[RESOLVER_RETURN57]]:
+// CHECK-NEXT: ret ptr @fmv._Mfp8fma
+// CHECK: [[RESOLVER_ELSE58]]:
// CHECK-NEXT: [[TMP124:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP125:%.*]] = and i64 [[TMP124]], 66312
-// CHECK-NEXT: [[TMP126:%.*]] = icmp eq i64 [[TMP125]], 66312
+// CHECK-NEXT: [[TMP125:%.*]] = and i64 [[TMP124]], 536888064
+// CHECK-NEXT: [[TMP126:%.*]] = icmp eq i64 [[TMP125]], 536888064
// CHECK-NEXT: [[TMP127:%.*]] = and i1 true, [[TMP126]]
-// CHECK-NEXT: br i1 [[TMP127]], label [[RESOLVER_RETURN61:%.*]], label [[RESOLVER_ELSE62:%.*]]
-// CHECK: resolver_return61:
-// CHECK-NEXT: ret ptr @fmv._Mfp16fml
-// CHECK: resolver_else62:
+// CHECK-NEXT: br i1 [[TMP127]], label %[[RESOLVER_RETURN59:.*]], label %[[RESOLVER_ELSE60:.*]]
+// CHECK: [[RESOLVER_RETURN59]]:
+// CHECK-NEXT: ret ptr @fmv._Mfp8dot2
+// CHECK: [[RESOLVER_ELSE60]]:
// CHECK-NEXT: [[TMP128:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP129:%.*]] = and i64 [[TMP128]], 65792
-// CHECK-NEXT: [[TMP130:%.*]] = icmp eq i64 [[TMP129]], 65792
+// CHECK-NEXT: [[TMP129:%.*]] = and i64 [[TMP128]], 268452608
+// CHECK-NEXT: [[TMP130:%.*]] = icmp eq i64 [[TMP129]], 268452608
// CHECK-NEXT: [[TMP131:%.*]] = and i1 true, [[TMP130]]
-// CHECK-NEXT: br i1 [[TMP131]], label [[RESOLVER_RETURN63:%.*]], label [[RESOLVER_ELSE64:%.*]]
-// CHECK: resolver_return63:
-// CHECK-NEXT: ret ptr @fmv._Mfp16
-// CHECK: resolver_else64:
+// CHECK-NEXT: br i1 [[TMP131]], label %[[RESOLVER_RETURN61:.*]], label %[[RESOLVER_ELSE62:.*]]
+// CHECK: [[RESOLVER_RETURN61]]:
+// CHECK-NEXT: ret ptr @fmv._Mfp8dot4
+// CHECK: [[RESOLVER_ELSE62]]:
// CHECK-NEXT: [[TMP132:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP133:%.*]] = and i64 [[TMP132]], 33536
-// CHECK-NEXT: [[TMP134:%.*]] = icmp eq i64 [[TMP133]], 33536
+// CHECK-NEXT: [[TMP133:%.*]] = and i64 [[TMP132]], 33571584
+// CHECK-NEXT: [[TMP134:%.*]] = icmp eq i64 [[TMP133]], 33571584
// CHECK-NEXT: [[TMP135:%.*]] = and i1 true, [[TMP134]]
-// CHECK-NEXT: br i1 [[TMP135]], label [[RESOLVER_RETURN65:%.*]], label [[RESOLVER_ELSE66:%.*]]
-// CHECK: resolver_return65:
-// CHECK-NEXT: ret ptr @fmv._Maes
-// CHECK: resolver_else66:
+// CHECK-NEXT: br i1 [[TMP135]], label %[[RESOLVER_RETURN63:.*]], label %[[RESOLVER_ELSE64:.*]]
+// CHECK: [[RESOLVER_RETURN63]]:
+// CHECK-NEXT: ret ptr @fmv._Mf8f32mm
+// CHECK: [[RESOLVER_ELSE64]]:
// CHECK-NEXT: [[TMP136:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP137:%.*]] = and i64 [[TMP136]], 13056
-// CHECK-NEXT: [[TMP138:%.*]] = icmp eq i64 [[TMP137]], 13056
+// CHECK-NEXT: [[TMP137:%.*]] = and i64 [[TMP136]], 17152
+// CHECK-NEXT: [[TMP138:%.*]] = icmp eq i64 [[TMP137]], 17152
// CHECK-NEXT: [[TMP139:%.*]] = and i1 true, [[TMP138]]
-// CHECK-NEXT: br i1 [[TMP139]], label [[RESOLVER_RETURN67:%.*]], label [[RESOLVER_ELSE68:%.*]]
-// CHECK: resolver_return67:
-// CHECK-NEXT: ret ptr @fmv._Msha3
-// CHECK: resolver_else68:
+// CHECK-NEXT: br i1 [[TMP139]], label %[[RESOLVER_RETURN65:.*]], label %[[RESOLVER_ELSE66:.*]]
+// CHECK: [[RESOLVER_RETURN65]]:
+// CHECK-NEXT: ret ptr @fmv._Mfp8
+// CHECK: [[RESOLVER_ELSE66]]:
// CHECK-NEXT: [[TMP140:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP141:%.*]] = and i64 [[TMP140]], 4864
-// CHECK-NEXT: [[TMP142:%.*]] = icmp eq i64 [[TMP141]], 4864
+// CHECK-NEXT: [[TMP141:%.*]] = and i64 [[TMP140]], 4611686018427387904
+// CHECK-NEXT: [[TMP142:%.*]] = icmp eq i64 [[TMP141]], 4611686018427387904
// CHECK-NEXT: [[TMP143:%.*]] = and i1 true, [[TMP142]]
-// CHECK-NEXT: br i1 [[TMP143]], label [[RESOLVER_RETURN69:%.*]], label [[RESOLVER_ELSE70:%.*]]
-// CHECK: resolver_return69:
-// CHECK-NEXT: ret ptr @fmv._Msha2
-// CHECK: resolver_else70:
-// CHECK-NEXT: [[TMP144:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP145:%.*]] = and i64 [[TMP144]], 1024
-// CHECK-NEXT: [[TMP146:%.*]] = icmp eq i64 [[TMP145]], 1024
-// CHECK-NEXT: [[TMP147:%.*]] = and i1 true, [[TMP146]]
-// CHECK-NEXT: br i1 [[TMP147]], label [[RESOLVER_RETURN71:%.*]], label [[RESOLVER_ELSE72:%.*]]
-// CHECK: resolver_return71:
-// CHECK-NEXT: ret ptr @fmv._Mcrc
-// CHECK: resolver_else72:
+// CHECK-NEXT: br i1 [[TMP143]], label %[[CPU_SUPPORTS_EXTENSION67:.*]], label %[[CPU_SUPPORTS_END68:.*]]
+// CHECK: [[CPU_SUPPORTS_EXTENSION67]]:
+// CHECK-NEXT: [[TMP144:%.*]] = load i64, ptr getelementptr inbounds nuw (i8, ptr @__aarch64_cpu_features, i64 8), align 8
+// CHECK-NEXT: [[TMP145:%.*]] = and i64 [[TMP144]], 8
+// CHECK-NEXT: [[TMP146:%.*]] = icmp eq i64 [[TMP145]], 8
+// CHECK-NEXT: br label %[[CPU_SUPPORTS_END68]]
+// CHECK: [[CPU_SUPPORTS_END68]]:
+// CHECK-NEXT: [[TMP147:%.*]] = phi i1 [ false, %[[RESOLVER_ELSE66]] ], [ [[TMP146]], %[[CPU_SUPPORTS_EXTENSION67]] ]
+// CHECK-NEXT: br i1 [[TMP147]], label %[[RESOLVER_RETURN69:.*]], label %[[RESOLVER_ELSE70:.*]]
+// CHECK: [[RESOLVER_RETURN69]]:
+// CHECK-NEXT: ret ptr @fmv._Mgcs
+// CHECK: [[RESOLVER_ELSE70]]:
// CHECK-NEXT: [[TMP148:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP149:%.*]] = and i64 [[TMP148]], 832
-// CHECK-NEXT: [[TMP150:%.*]] = icmp eq i64 [[TMP149]], 832
+// CHECK-NEXT: [[TMP149:%.*]] = and i64 [[TMP148]], 2304
+// CHECK-NEXT: [[TMP150:%.*]] = icmp eq i64 [[TMP149]], 2304
// CHECK-NEXT: [[TMP151:%.*]] = and i1 true, [[TMP150]]
-// CHECK-NEXT: br i1 [[TMP151]], label [[RESOLVER_RETURN73:%.*]], label [[RESOLVER_ELSE74:%.*]]
-// CHECK: resolver_return73:
-// CHECK-NEXT: ret ptr @fmv._Mrdm
-// CHECK: resolver_else74:
+// CHECK-NEXT: br i1 [[TMP151]], label [[RESOLVER_RETURN71:%.*]], label [[RESOLVER_ELSE72:%.*]]
+// CHECK: resolver_return71:
+// CHECK-NEXT: ret ptr @fmv._McsscMfp
+// CHECK: resolver_else72:
// CHECK-NEXT: [[TMP152:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP153:%.*]] = and i64 [[TMP152]], 800
-// CHECK-NEXT: [[TMP154:%.*]] = icmp eq i64 [[TMP153]], 800
+// CHECK-NEXT: [[TMP153:%.*]] = and i64 [[TMP152]], 2048
+// CHECK-NEXT: [[TMP154:%.*]] = icmp eq i64 [[TMP153]], 2048
// CHECK-NEXT: [[TMP155:%.*]] = and i1 true, [[TMP154]]
-// CHECK-NEXT: br i1 [[TMP155]], label [[RESOLVER_RETURN75:%.*]], label [[RESOLVER_ELSE76:%.*]]
-// CHECK: resolver_return75:
-// CHECK-NEXT: ret ptr @fmv._Msm4
-// CHECK: resolver_else76:
+// CHECK-NEXT: br i1 [[TMP155]], label [[RESOLVER_RETURN73:%.*]], label [[RESOLVER_ELSE74:%.*]]
+// CHECK: resolver_return73:
+// CHECK-NEXT: ret ptr @fmv._Mcssc
+// CHECK: resolver_else74:
// CHECK-NEXT: [[TMP156:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP157:%.*]] = and i64 [[TMP156]], 784
-// CHECK-NEXT: [[TMP158:%.*]] = icmp eq i64 [[TMP157]], 784
+// CHECK-NEXT: [[TMP157:%.*]] = and i64 [[TMP156]], 576460752303423488
+// CHECK-NEXT: [[TMP158:%.*]] = icmp eq i64 [[TMP157]], 576460752303423488
// CHECK-NEXT: [[TMP159:%.*]] = and i1 true, [[TMP158]]
-// CHECK-NEXT: br i1 [[TMP159]], label [[RESOLVER_RETURN77:%.*]], label [[RESOLVER_ELSE78:%.*]]
-// CHECK: resolver_return77:
-// CHECK-NEXT: ret ptr @fmv._Mdotprod
-// CHECK: resolver_else78:
+// CHECK-NEXT: br i1 [[TMP159]], label [[RESOLVER_RETURN75:%.*]], label [[RESOLVER_ELSE76:%.*]]
+// CHECK: resolver_return75:
+// CHECK-NEXT: ret ptr @fmv._Mmops
+// CHECK: resolver_else76:
// CHECK-NEXT: [[TMP160:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP161:%.*]] = and i64 [[TMP160]], 768
-// CHECK-NEXT: [[TMP162:%.*]] = icmp eq i64 [[TMP161]], 768
+// CHECK-NEXT: [[TMP161:%.*]] = and i64 [[TMP160]], 144119586256651008
+// CHECK-NEXT: [[TMP162:%.*]] = icmp eq i64 [[TMP161]], 144119586256651008
// CHECK-NEXT: [[TMP163:%.*]] = and i1 true, [[TMP162]]
-// CHECK-NEXT: br i1 [[TMP163]], label [[RESOLVER_RETURN79:%.*]], label [[RESOLVER_ELSE80:%.*]]
-// CHECK: resolver_return79:
-// CHECK-NEXT: ret ptr @fmv._Msimd
-// CHECK: resolver_else80:
+// CHECK-NEXT: br i1 [[TMP163]], label [[RESOLVER_RETURN77:%.*]], label [[RESOLVER_ELSE78:%.*]]
+// CHECK: resolver_return77:
+// CHECK-NEXT: ret ptr @fmv._Msme2
+// CHECK: resolver_else78:
// CHECK-NEXT: [[TMP164:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP165:%.*]] = and i64 [[TMP164]], 256
-// CHECK-NEXT: [[TMP166:%.*]] = icmp eq i64 [[TMP165]], 256
+// CHECK-NEXT: [[TMP165:%.*]] = and i64 [[TMP164]], 72061992218723072
+// CHECK-NEXT: [[TMP166:%.*]] = icmp eq i64 [[TMP165]], 72061992218723072
// CHECK-NEXT: [[TMP167:%.*]] = and i1 true, [[TMP166]]
-// CHECK-NEXT: br i1 [[TMP167]], label [[RESOLVER_RETURN81:%.*]], label [[RESOLVER_ELSE82:%.*]]
-// CHECK: resolver_return81:
-// CHECK-NEXT: ret ptr @fmv._Mfp
-// CHECK: resolver_else82:
+// CHECK-NEXT: br i1 [[TMP167]], label [[RESOLVER_RETURN79:%.*]], label [[RESOLVER_ELSE80:%.*]]
+// CHECK: resolver_return79:
+// CHECK-NEXT: ret ptr @fmv._Msme-i16i64
+// CHECK: resolver_else80:
// CHECK-NEXT: [[TMP168:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP169:%.*]] = and i64 [[TMP168]], 128
-// CHECK-NEXT: [[TMP170:%.*]] = icmp eq i64 [[TMP169]], 128
+// CHECK-NEXT: [[TMP169:%.*]] = and i64 [[TMP168]], 36033195199759104
+// CHECK-NEXT: [[TMP170:%.*]] = icmp eq i64 [[TMP169]], 36033195199759104
// CHECK-NEXT: [[TMP171:%.*]] = and i1 true, [[TMP170]]
-// CHECK-NEXT: br i1 [[TMP171]], label [[RESOLVER_RETURN83:%.*]], label [[RESOLVER_ELSE84:%.*]]
-// CHECK: resolver_return83:
-// CHECK-NEXT: ret ptr @fmv._Mlse
-// CHECK: resolver_else84:
+// CHECK-NEXT: br i1 [[TMP171]], label [[RESOLVER_RETURN81:%.*]], label [[RESOLVER_ELSE82:%.*]]
+// CHECK: resolver_return81:
+// CHECK-NEXT: ret ptr @fmv._Msme-f64f64
+// CHECK: resolver_else82:
// CHECK-NEXT: [[TMP172:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP173:%.*]] = and i64 [[TMP172]], 6
-// CHECK-NEXT: [[TMP174:%.*]] = icmp eq i64 [[TMP173]], 6
+// CHECK-NEXT: [[TMP173:%.*]] = and i64 [[TMP172]], 18014398509481984
+// CHECK-NEXT: [[TMP174:%.*]] = icmp eq i64 [[TMP173]], 18014398509481984
// CHECK-NEXT: [[TMP175:%.*]] = and i1 true, [[TMP174]]
-// CHECK-NEXT: br i1 [[TMP175]], label [[RESOLVER_RETURN85:%.*]], label [[RESOLVER_ELSE86:%.*]]
-// CHECK: resolver_return85:
-// CHECK-NEXT: ret ptr @fmv._Mflagm2
-// CHECK: resolver_else86:
+// CHECK-NEXT: br i1 [[TMP175]], label [[RESOLVER_RETURN83:%.*]], label [[RESOLVER_ELSE84:%.*]]
+// CHECK: resolver_return83:
+// CHECK-NEXT: ret ptr @fmv._Mwfxt
+// CHECK: resolver_else84:
// CHECK-NEXT: [[TMP176:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP177:%.*]] = and i64 [[TMP176]], 2
-// CHECK-NEXT: [[TMP178:%.*]] = icmp eq i64 [[TMP177]], 2
+// CHECK-NEXT: [[TMP177:%.*]] = and i64 [[TMP176]], 1125899906842624
+// CHECK-NEXT: [[TMP178:%.*]] = icmp eq i64 [[TMP177]], 1125899906842624
// CHECK-NEXT: [[TMP179:%.*]] = and i1 true, [[TMP178]]
-// CHECK-NEXT: br i1 [[TMP179]], label [[RESOLVER_RETURN87:%.*]], label [[RESOLVER_ELSE88:%.*]]
-// CHECK: resolver_return87:
-// CHECK-NEXT: ret ptr @fmv._Mflagm
-// CHECK: resolver_else88:
+// CHECK-NEXT: br i1 [[TMP179]], label [[RESOLVER_RETURN85:%.*]], label [[RESOLVER_ELSE86:%.*]]
+// CHECK: resolver_return85:
+// CHECK-NEXT: ret ptr @fmv._Mbti
+// CHECK: resolver_else86:
// CHECK-NEXT: [[TMP180:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
-// CHECK-NEXT: [[TMP181:%.*]] = and i64 [[TMP180]], 1
-// CHECK-NEXT: [[TMP182:%.*]] = icmp eq i64 [[TMP181]], 1
+// CHECK-NEXT: [[TMP181:%.*]] = and i64 [[TMP180]], 562949953421312
+// CHECK-NEXT: [[TMP182:%.*]] = icmp eq i64 [[TMP181]], 562949953421312
// CHECK-NEXT: [[TMP183:%.*]] = and i1 true, [[TMP182]]
-// CHECK-NEXT: br i1 [[TMP183]], label [[RESOLVER_RETURN89:%.*]], label [[RESOLVER_ELSE90:%.*]]
+// CHECK-NEXT: br i1 [[TMP183]], label [[RESOLVER_RETURN87:%.*]], label [[RESOLVER_ELSE88:%.*]]
+// CHECK: resolver_return87:
+// CHECK-NEXT: ret ptr @fmv._Mssbs
+// CHECK: resolver_else88:
+// CHECK-NEXT: [[TMP184:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP185:%.*]] = and i64 [[TMP184]], 70368744177664
+// CHECK-NEXT: [[TMP186:%.*]] = icmp eq i64 [[TMP185]], 70368744177664
+// CHECK-NEXT: [[TMP187:%.*]] = and i1 true, [[TMP186]]
+// CHECK-NEXT: br i1 [[TMP187]], label [[RESOLVER_RETURN89:%.*]], label [[RESOLVER_ELSE90:%.*]]
// CHECK: resolver_return89:
-// CHECK-NEXT: ret ptr @fmv._Mrng
+// CHECK-NEXT: ret ptr @fmv._Msb
// CHECK: resolver_else90:
+// CHECK-NEXT: [[TMP188:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP189:%.*]] = and i64 [[TMP188]], 17592186044416
+// CHECK-NEXT: [[TMP190:%.*]] = icmp eq i64 [[TMP189]], 17592186044416
+// CHECK-NEXT: [[TMP191:%.*]] = and i1 true, [[TMP190]]
+// CHECK-NEXT: br i1 [[TMP191]], label [[RESOLVER_RETURN91:%.*]], label [[RESOLVER_ELSE92:%.*]]
+// CHECK: resolver_return91:
+// CHECK-NEXT: ret ptr @fmv._Mmemtag
+// CHECK: resolver_else92:
+// CHECK-NEXT: [[TMP192:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP193:%.*]] = and i64 [[TMP192]], 4398180795136
+// CHECK-NEXT: [[TMP194:%.*]] = icmp eq i64 [[TMP193]], 4398180795136
+// CHECK-NEXT: [[TMP195:%.*]] = and i1 true, [[TMP194]]
+// CHECK-NEXT: br i1 [[TMP195]], label [[RESOLVER_RETURN93:%.*]], label [[RESOLVER_ELSE94:%.*]]
+// CHECK: resolver_return93:
+// CHECK-NEXT: ret ptr @fmv._Msme
+// CHECK: resolver_else94:
+// CHECK-NEXT: [[TMP196:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP197:%.*]] = and i64 [[TMP196]], 2268816540448
+// CHECK-NEXT: [[TMP198:%.*]] = icmp eq i64 [[TMP197]], 2268816540448
+// CHECK-NEXT: [[TMP199:%.*]] = and i1 true, [[TMP198]]
+// CHECK-NEXT: br i1 [[TMP199]], label [[RESOLVER_RETURN95:%.*]], label [[RESOLVER_ELSE96:%.*]]
+// CHECK: resolver_return95:
+// CHECK-NEXT: ret ptr @fmv._Msve2-sm4
+// CHECK: resolver_else96:
+// CHECK-NEXT: [[TMP200:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP201:%.*]] = and i64 [[TMP200]], 1169304924928
+// CHECK-NEXT: [[TMP202:%.*]] = icmp eq i64 [[TMP201]], 1169304924928
+// CHECK-NEXT: [[TMP203:%.*]] = and i1 true, [[TMP202]]
+// CHECK-NEXT: br i1 [[TMP203]], label [[RESOLVER_RETURN97:%.*]], label [[RESOLVER_ELSE98:%.*]]
+// CHECK: resolver_return97:
+// CHECK-NEXT: ret ptr @fmv._Msve2-sha3
+// CHECK: resolver_else98:
+// CHECK-NEXT: [[TMP204:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP205:%.*]] = and i64 [[TMP204]], 619549098240
+// CHECK-NEXT: [[TMP206:%.*]] = icmp eq i64 [[TMP205]], 619549098240
+// CHECK-NEXT: [[TMP207:%.*]] = and i1 true, [[TMP206]]
+// CHECK-NEXT: br i1 [[TMP207]], label [[RESOLVER_RETURN99:%.*]], label [[RESOLVER_ELSE100:%.*]]
+// CHECK: resolver_return99:
+// CHECK-NEXT: ret ptr @fmv._Msve2-bitperm
+// CHECK: resolver_else100:
+// CHECK-NEXT: [[TMP208:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP209:%.*]] = and i64 [[TMP208]], 344671224576
+// CHECK-NEXT: [[TMP210:%.*]] = icmp eq i64 [[TMP209]], 344671224576
+// CHECK-NEXT: [[TMP211:%.*]] = and i1 true, [[TMP210]]
+// CHECK-NEXT: br i1 [[TMP211]], label [[RESOLVER_RETURN101:%.*]], label [[RESOLVER_ELSE102:%.*]]
+// CHECK: resolver_return101:
+// CHECK-NEXT: ret ptr @fmv._Msve2-aes
+// CHECK: resolver_else102:
+// CHECK-NEXT: [[TMP212:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP213:%.*]] = and i64 [[TMP212]], 69793284352
+// CHECK-NEXT: [[TMP214:%.*]] = icmp eq i64 [[TMP213]], 69793284352
+// CHECK-NEXT: [[TMP215:%.*]] = and i1 true, [[TMP214]]
+// CHECK-NEXT: br i1 [[TMP215]], label [[RESOLVER_RETURN103:%.*]], label [[RESOLVER_ELSE104:%.*]]
+// CHECK: resolver_return103:
+// CHECK-NEXT: ret ptr @fmv._Msve2
+// CHECK: resolver_else104:
+// CHECK-NEXT: [[TMP216:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP217:%.*]] = and i64 [[TMP216]], 35433545984
+// CHECK-NEXT: [[TMP218:%.*]] = icmp eq i64 [[TMP217]], 35433545984
+// CHECK-NEXT: [[TMP219:%.*]] = and i1 true, [[TMP218]]
+// CHECK-NEXT: br i1 [[TMP219]], label [[RESOLVER_RETURN105:%.*]], label [[RESOLVER_ELSE106:%.*]]
+// CHECK: resolver_return105:
+// CHECK-NEXT: ret ptr @fmv._Mf64mm
+// CHECK: resolver_else106:
+// CHECK-NEXT: [[TMP220:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP221:%.*]] = and i64 [[TMP220]], 18253676800
+// CHECK-NEXT: [[TMP222:%.*]] = icmp eq i64 [[TMP221]], 18253676800
+// CHECK-NEXT: [[TMP223:%.*]] = and i1 true, [[TMP222]]
+// CHECK-NEXT: br i1 [[TMP223]], label [[RESOLVER_RETURN107:%.*]], label [[RESOLVER_ELSE108:%.*]]
+// CHECK: resolver_return107:
+// CHECK-NEXT: ret ptr @fmv._Mf32mm
+// CHECK: resolver_else108:
+// CHECK-NEXT: [[TMP224:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP225:%.*]] = and i64 [[TMP224]], 1073807616
+// CHECK-NEXT: [[TMP226:%.*]] = icmp eq i64 [[TMP225]], 1073807616
+// CHECK-NEXT: [[TMP227:%.*]] = and i1 true, [[TMP226]]
+// CHECK-NEXT: br i1 [[TMP227]], label [[RESOLVER_RETURN109:%.*]], label [[RESOLVER_ELSE110:%.*]]
+// CHECK: resolver_return109:
+// CHECK-NEXT: ret ptr @fmv._Msve
+// CHECK: resolver_else110:
+// CHECK-NEXT: [[TMP228:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP229:%.*]] = and i64 [[TMP228]], 134218496
+// CHECK-NEXT: [[TMP230:%.*]] = icmp eq i64 [[TMP229]], 134218496
+// CHECK-NEXT: [[TMP231:%.*]] = and i1 true, [[TMP230]]
+// CHECK-NEXT: br i1 [[TMP231]], label [[RESOLVER_RETURN111:%.*]], label [[RESOLVER_ELSE112:%.*]]
+// CHECK: resolver_return111:
+// CHECK-NEXT: ret ptr @fmv._Mbf16
+// CHECK: resolver_else112:
+// CHECK-NEXT: [[TMP232:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP233:%.*]] = and i64 [[TMP232]], 67109632
+// CHECK-NEXT: [[TMP234:%.*]] = icmp eq i64 [[TMP233]], 67109632
+// CHECK-NEXT: [[TMP235:%.*]] = and i1 true, [[TMP234]]
+// CHECK-NEXT: br i1 [[TMP235]], label [[RESOLVER_RETURN113:%.*]], label [[RESOLVER_ELSE114:%.*]]
+// CHECK: resolver_return113:
+// CHECK-NEXT: ret ptr @fmv._Mi8mm
+// CHECK: resolver_else114:
+// CHECK-NEXT: [[TMP236:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP237:%.*]] = and i64 [[TMP236]], 16777472
+// CHECK-NEXT: [[TMP238:%.*]] = icmp eq i64 [[TMP237]], 16777472
+// CHECK-NEXT: [[TMP239:%.*]] = and i1 true, [[TMP238]]
+// CHECK-NEXT: br i1 [[TMP239]], label [[RESOLVER_RETURN115:%.*]], label [[RESOLVER_ELSE116:%.*]]
+// CHECK: resolver_return115:
+// CHECK-NEXT: ret ptr @fmv._Mfrintts
+// CHECK: resolver_else116:
+// CHECK-NEXT: [[TMP240:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP241:%.*]] = and i64 [[TMP240]], 288230376164294656
+// CHECK-NEXT: [[TMP242:%.*]] = icmp eq i64 [[TMP241]], 288230376164294656
+// CHECK-NEXT: [[TMP243:%.*]] = and i1 true, [[TMP242]]
+// CHECK-NEXT: br i1 [[TMP243]], label [[RESOLVER_RETURN117:%.*]], label [[RESOLVER_ELSE118:%.*]]
+// CHECK: resolver_return117:
+// CHECK-NEXT: ret ptr @fmv._Mrcpc3
+// CHECK: resolver_else118:
+// CHECK-NEXT: [[TMP244:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP245:%.*]] = and i64 [[TMP244]], 12582912
+// CHECK-NEXT: [[TMP246:%.*]] = icmp eq i64 [[TMP245]], 12582912
+// CHECK-NEXT: [[TMP247:%.*]] = and i1 true, [[TMP246]]
+// CHECK-NEXT: br i1 [[TMP247]], label [[RESOLVER_RETURN119:%.*]], label [[RESOLVER_ELSE120:%.*]]
+// CHECK: resolver_return119:
+// CHECK-NEXT: ret ptr @fmv._Mrcpc2
+// CHECK: resolver_else120:
+// CHECK-NEXT: [[TMP248:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP249:%.*]] = and i64 [[TMP248]], 4194304
+// CHECK-NEXT: [[TMP250:%.*]] = icmp eq i64 [[TMP249]], 4194304
+// CHECK-NEXT: [[TMP251:%.*]] = and i1 true, [[TMP250]]
+// CHECK-NEXT: br i1 [[TMP251]], label [[RESOLVER_RETURN121:%.*]], label [[RESOLVER_ELSE122:%.*]]
+// CHECK: resolver_return121:
+// CHECK-NEXT: ret ptr @fmv._Mrcpc
+// CHECK: resolver_else122:
+// CHECK-NEXT: [[TMP252:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP253:%.*]] = and i64 [[TMP252]], 2097920
+// CHECK-NEXT: [[TMP254:%.*]] = icmp eq i64 [[TMP253]], 2097920
+// CHECK-NEXT: [[TMP255:%.*]] = and i1 true, [[TMP254]]
+// CHECK-NEXT: br i1 [[TMP255]], label [[RESOLVER_RETURN123:%.*]], label [[RESOLVER_ELSE124:%.*]]
+// CHECK: resolver_return123:
+// CHECK-NEXT: ret ptr @fmv._Mfcma
+// CHECK: resolver_else124:
+// CHECK-NEXT: [[TMP256:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP257:%.*]] = and i64 [[TMP256]], 1048832
+// CHECK-NEXT: [[TMP258:%.*]] = icmp eq i64 [[TMP257]], 1048832
+// CHECK-NEXT: [[TMP259:%.*]] = and i1 true, [[TMP258]]
+// CHECK-NEXT: br i1 [[TMP259]], label [[RESOLVER_RETURN125:%.*]], label [[RESOLVER_ELSE126:%.*]]
+// CHECK: resolver_return125:
+// CHECK-NEXT: ret ptr @fmv._Mjscvt
+// CHECK: resolver_else126:
+// CHECK-NEXT: [[TMP260:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP261:%.*]] = and i64 [[TMP260]], 786432
+// CHECK-NEXT: [[TMP262:%.*]] = icmp eq i64 [[TMP261]], 786432
+// CHECK-NEXT: [[TMP263:%.*]] = and i1 true, [[TMP262]]
+// CHECK-NEXT: br i1 [[TMP263]], label [[RESOLVER_RETURN127:%.*]], label [[RESOLVER_ELSE128:%.*]]
+// CHECK: resolver_return127:
+// CHECK-NEXT: ret ptr @fmv._Mdpb2
+// CHECK: resolver_else128:
+// CHECK-NEXT: [[TMP264:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP265:%.*]] = and i64 [[TMP264]], 262144
+// CHECK-NEXT: [[TMP266:%.*]] = icmp eq i64 [[TMP265]], 262144
+// CHECK-NEXT: [[TMP267:%.*]] = and i1 true, [[TMP266]]
+// CHECK-NEXT: br i1 [[TMP267]], label [[RESOLVER_RETURN129:%.*]], label [[RESOLVER_ELSE130:%.*]]
+// CHECK: resolver_return129:
+// CHECK-NEXT: ret ptr @fmv._Mdpb
+// CHECK: resolver_else130:
+// CHECK-NEXT: [[TMP268:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP269:%.*]] = and i64 [[TMP268]], 131072
+// CHECK-NEXT: [[TMP270:%.*]] = icmp eq i64 [[TMP269]], 131072
+// CHECK-NEXT: [[TMP271:%.*]] = and i1 true, [[TMP270]]
+// CHECK-NEXT: br i1 [[TMP271]], label [[RESOLVER_RETURN131:%.*]], label [[RESOLVER_ELSE132:%.*]]
+// CHECK: resolver_return131:
+// CHECK-NEXT: ret ptr @fmv._Mdit
+// CHECK: resolver_else132:
+// CHECK-NEXT: [[TMP272:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP273:%.*]] = and i64 [[TMP272]], 66312
+// CHECK-NEXT: [[TMP274:%.*]] = icmp eq i64 [[TMP273]], 66312
+// CHECK-NEXT: [[TMP275:%.*]] = and i1 true, [[TMP274]]
+// CHECK-NEXT: br i1 [[TMP275]], label [[RESOLVER_RETURN133:%.*]], label [[RESOLVER_ELSE134:%.*]]
+// CHECK: resolver_return133:
+// CHECK-NEXT: ret ptr @fmv._Mfp16fml
+// CHECK: resolver_else134:
+// CHECK-NEXT: [[TMP276:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP277:%.*]] = and i64 [[TMP276]], 65792
+// CHECK-NEXT: [[TMP278:%.*]] = icmp eq i64 [[TMP277]], 65792
+// CHECK-NEXT: [[TMP279:%.*]] = and i1 true, [[TMP278]]
+// CHECK-NEXT: br i1 [[TMP279]], label [[RESOLVER_RETURN135:%.*]], label [[RESOLVER_ELSE136:%.*]]
+// CHECK: resolver_return135:
+// CHECK-NEXT: ret ptr @fmv._Mfp16
+// CHECK: resolver_else136:
+// CHECK-NEXT: [[TMP280:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP281:%.*]] = and i64 [[TMP280]], 33536
+// CHECK-NEXT: [[TMP282:%.*]] = icmp eq i64 [[TMP281]], 33536
+// CHECK-NEXT: [[TMP283:%.*]] = and i1 true, [[TMP282]]
+// CHECK-NEXT: br i1 [[TMP283]], label [[RESOLVER_RETURN137:%.*]], label [[RESOLVER_ELSE138:%.*]]
+// CHECK: resolver_return137:
+// CHECK-NEXT: ret ptr @fmv._Maes
+// CHECK: resolver_else138:
+// CHECK-NEXT: [[TMP284:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP285:%.*]] = and i64 [[TMP284]], 13056
+// CHECK-NEXT: [[TMP286:%.*]] = icmp eq i64 [[TMP285]], 13056
+// CHECK-NEXT: [[TMP287:%.*]] = and i1 true, [[TMP286]]
+// CHECK-NEXT: br i1 [[TMP287]], label [[RESOLVER_RETURN139:%.*]], label [[RESOLVER_ELSE140:%.*]]
+// CHECK: resolver_return139:
+// CHECK-NEXT: ret ptr @fmv._Msha3
+// CHECK: resolver_else140:
+// CHECK-NEXT: [[TMP288:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP289:%.*]] = and i64 [[TMP288]], 4864
+// CHECK-NEXT: [[TMP290:%.*]] = icmp eq i64 [[TMP289]], 4864
+// CHECK-NEXT: [[TMP291:%.*]] = and i1 true, [[TMP290]]
+// CHECK-NEXT: br i1 [[TMP291]], label [[RESOLVER_RETURN141:%.*]], label [[RESOLVER_ELSE142:%.*]]
+// CHECK: resolver_return141:
+// CHECK-NEXT: ret ptr @fmv._Msha2
+// CHECK: resolver_else142:
+// CHECK-NEXT: [[TMP292:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP293:%.*]] = and i64 [[TMP292]], 1024
+// CHECK-NEXT: [[TMP294:%.*]] = icmp eq i64 [[TMP293]], 1024
+// CHECK-NEXT: [[TMP295:%.*]] = and i1 true, [[TMP294]]
+// CHECK-NEXT: br i1 [[TMP295]], label [[RESOLVER_RETURN143:%.*]], label [[RESOLVER_ELSE144:%.*]]
+// CHECK: resolver_return143:
+// CHECK-NEXT: ret ptr @fmv._Mcrc
+// CHECK: resolver_else144:
+// CHECK-NEXT: [[TMP296:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP297:%.*]] = and i64 [[TMP296]], 832
+// CHECK-NEXT: [[TMP298:%.*]] = icmp eq i64 [[TMP297]], 832
+// CHECK-NEXT: [[TMP299:%.*]] = and i1 true, [[TMP298]]
+// CHECK-NEXT: br i1 [[TMP299]], label [[RESOLVER_RETURN145:%.*]], label [[RESOLVER_ELSE146:%.*]]
+// CHECK: resolver_return145:
+// CHECK-NEXT: ret ptr @fmv._Mrdm
+// CHECK: resolver_else146:
+// CHECK-NEXT: [[TMP300:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP301:%.*]] = and i64 [[TMP300]], 800
+// CHECK-NEXT: [[TMP302:%.*]] = icmp eq i64 [[TMP301]], 800
+// CHECK-NEXT: [[TMP303:%.*]] = and i1 true, [[TMP302]]
+// CHECK-NEXT: br i1 [[TMP303]], label [[RESOLVER_RETURN147:%.*]], label [[RESOLVER_ELSE148:%.*]]
+// CHECK: resolver_return147:
+// CHECK-NEXT: ret ptr @fmv._Msm4
+// CHECK: resolver_else148:
+// CHECK-NEXT: [[TMP304:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP305:%.*]] = and i64 [[TMP304]], 784
+// CHECK-NEXT: [[TMP306:%.*]] = icmp eq i64 [[TMP305]], 784
+// CHECK-NEXT: [[TMP307:%.*]] = and i1 true, [[TMP306]]
+// CHECK-NEXT: br i1 [[TMP307]], label [[RESOLVER_RETURN149:%.*]], label [[RESOLVER_ELSE150:%.*]]
+// CHECK: resolver_return149:
+// CHECK-NEXT: ret ptr @fmv._Mdotprod
+// CHECK: resolver_else150:
+// CHECK-NEXT: [[TMP308:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP309:%.*]] = and i64 [[TMP308]], 768
+// CHECK-NEXT: [[TMP310:%.*]] = icmp eq i64 [[TMP309]], 768
+// CHECK-NEXT: [[TMP311:%.*]] = and i1 true, [[TMP310]]
+// CHECK-NEXT: br i1 [[TMP311]], label [[RESOLVER_RETURN151:%.*]], label [[RESOLVER_ELSE152:%.*]]
+// CHECK: resolver_return151:
+// CHECK-NEXT: ret ptr @fmv._Msimd
+// CHECK: resolver_else152:
+// CHECK-NEXT: [[TMP312:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP313:%.*]] = and i64 [[TMP312]], 256
+// CHECK-NEXT: [[TMP314:%.*]] = icmp eq i64 [[TMP313]], 256
+// CHECK-NEXT: [[TMP315:%.*]] = and i1 true, [[TMP314]]
+// CHECK-NEXT: br i1 [[TMP315]], label [[RESOLVER_RETURN153:%.*]], label [[RESOLVER_ELSE154:%.*]]
+// CHECK: resolver_return153:
+// CHECK-NEXT: ret ptr @fmv._Mfp
+// CHECK: resolver_else154:
+// CHECK-NEXT: [[TMP316:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP317:%.*]] = and i64 [[TMP316]], 128
+// CHECK-NEXT: [[TMP318:%.*]] = icmp eq i64 [[TMP317]], 128
+// CHECK-NEXT: [[TMP319:%.*]] = and i1 true, [[TMP318]]
+// CHECK-NEXT: br i1 [[TMP319]], label [[RESOLVER_RETURN155:%.*]], label [[RESOLVER_ELSE156:%.*]]
+// CHECK: resolver_return155:
+// CHECK-NEXT: ret ptr @fmv._Mlse
+// CHECK: resolver_else156:
+// CHECK-NEXT: [[TMP320:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP321:%.*]] = and i64 [[TMP320]], 6
+// CHECK-NEXT: [[TMP322:%.*]] = icmp eq i64 [[TMP321]], 6
+// CHECK-NEXT: [[TMP323:%.*]] = and i1 true, [[TMP322]]
+// CHECK-NEXT: br i1 [[TMP323]], label [[RESOLVER_RETURN157:%.*]], label [[RESOLVER_ELSE158:%.*]]
+// CHECK: resolver_return157:
+// CHECK-NEXT: ret ptr @fmv._Mflagm2
+// CHECK: resolver_else158:
+// CHECK-NEXT: [[TMP324:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP325:%.*]] = and i64 [[TMP324]], 2
+// CHECK-NEXT: [[TMP326:%.*]] = icmp eq i64 [[TMP325]], 2
+// CHECK-NEXT: [[TMP327:%.*]] = and i1 true, [[TMP326]]
+// CHECK-NEXT: br i1 [[TMP327]], label [[RESOLVER_RETURN159:%.*]], label [[RESOLVER_ELSE160:%.*]]
+// CHECK: resolver_return159:
+// CHECK-NEXT: ret ptr @fmv._Mflagm
+// CHECK: resolver_else160:
+// CHECK-NEXT: [[TMP328:%.*]] = load i64, ptr @__aarch64_cpu_features, align 8
+// CHECK-NEXT: [[TMP329:%.*]] = and i64 [[TMP328]], 1
+// CHECK-NEXT: [[TMP330:%.*]] = icmp eq i64 [[TMP329]], 1
+// CHECK-NEXT: [[TMP331:%.*]] = and i1 true, [[TMP330]]
+// CHECK-NEXT: br i1 [[TMP331]], label [[RESOLVER_RETURN161:%.*]], label [[RESOLVER_ELSE162:%.*]]
+// CHECK: resolver_return161:
+// CHECK-NEXT: ret ptr @fmv._Mrng
+// CHECK: resolver_else162:
// CHECK-NEXT: ret ptr @fmv.default
//
//.
diff --git a/compiler-rt/lib/builtins/cpu_model/aarch64.c b/compiler-rt/lib/builtins/cpu_model/aarch64.c
index e2178edff476e..24f65d54065be 100644
--- a/compiler-rt/lib/builtins/cpu_model/aarch64.c
+++ b/compiler-rt/lib/builtins/cpu_model/aarch64.c
@@ -65,6 +65,7 @@ _Bool __aarch64_have_lse_atomics
struct {
unsigned long long features;
unsigned long long features2;
+ // As features grows new fields could be added
} __aarch64_cpu_features __attribute__((visibility("hidden"), nocommon));
// The formatter wants to re-order these includes, but doing so is incorrect:
diff --git a/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/apple.inc b/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/apple.inc
index 74a317ba63555..3f30f894c503c 100644
--- a/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/apple.inc
+++ b/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/apple.inc
@@ -7,6 +7,9 @@
#define HAS_CPU_CAPABILITIES_PUBLIC_H 1
// FB13964283 - A few of these didn't make it into the public SDK yet.
+#ifndef CAP_BIT_FEAT_WFxT
+#define CAP_BIT_FEAT_WFxT 26
+#endif
#ifndef CAP_BIT_FEAT_SME
#define CAP_BIT_FEAT_SME 40
#endif
@@ -19,6 +22,30 @@
#ifndef CAP_BIT_FEAT_SME_I16I64
#define CAP_BIT_FEAT_SME_I16I64 43
#endif
+#ifndef CAP_BIT_FEAT_SME2p1
+#define CAP_BIT_FEAT_SME2p1 44
+#endif
+#ifndef CAP_BIT_FEAT_SME_F16F16
+#define CAP_BIT_FEAT_SME_F16F16 45
+#endif
+#ifndef CAP_BIT_FEAT_SME_F8F16
+#define CAP_BIT_FEAT_SME_F8F16 47
+#endif
+#ifndef CAP_BIT_FEAT_SME_F8F32
+#define CAP_BIT_FEAT_SME_F8F32 48
+#endif
+#ifndef CAP_BIT_FEAT_MTE2
+#define CAP_BIT_FEAT_MTE2 61
+#endif
+#ifndef CAP_BIT_FEAT_CSSC
+#define CAP_BIT_FEAT_CSSC 67
+#endif
+
+// Adjust the capability buffer size to fit in newer feature bits when using an older SDK.
+#if CAP_BIT_NB < 68
+#undef CAP_BIT_NB
+#define CAP_BIT_NB 68
+#endif
#endif
@@ -49,7 +76,7 @@ void __init_cpu_features_resolver(void) {
if (__atomic_load_n(&__aarch64_cpu_features.features, __ATOMIC_RELAXED))
return;
- uint64_t features = 0;
+ uint64_t features[2] = {0, 0};
#ifdef HAS_CPU_CAPABILITIES_PUBLIC_H
uint8_t feats_bitvec[(CAP_BIT_NB + 7) / 8] = {0};
@@ -62,7 +89,7 @@ void __init_cpu_features_resolver(void) {
#define CHECK_BIT(FROM, TO) \
do { \
if (feats_bitvec[FROM / 8] & (1u << ((FROM) & 7))) { \
- features |= (1ULL << TO); \
+ features[(TO) / 64] |= (1ULL << ((TO) % 64)); \
} \
} while (0)
@@ -85,6 +112,7 @@ void __init_cpu_features_resolver(void) {
CHECK_BIT(CAP_BIT_FEAT_DPB2, FEAT_DPB2);
CHECK_BIT(CAP_BIT_FEAT_BF16, FEAT_BF16);
CHECK_BIT(CAP_BIT_FEAT_I8MM, FEAT_I8MM);
+ CHECK_BIT(CAP_BIT_FEAT_WFxT, FEAT_WFXT);
CHECK_BIT(CAP_BIT_FEAT_DIT, FEAT_DIT);
CHECK_BIT(CAP_BIT_FEAT_FP16, FEAT_FP16);
CHECK_BIT(CAP_BIT_FEAT_SSBS, FEAT_SSBS2);
@@ -95,11 +123,21 @@ void __init_cpu_features_resolver(void) {
CHECK_BIT(CAP_BIT_FEAT_SME2, FEAT_SME2);
CHECK_BIT(CAP_BIT_FEAT_SME_F64F64, FEAT_SME_F64);
CHECK_BIT(CAP_BIT_FEAT_SME_I16I64, FEAT_SME_I64);
-
- features = deriveImplicitFeatures(features);
-
- __atomic_store_n(&__aarch64_cpu_features.features2, 0, __ATOMIC_RELAXED);
- __atomic_store_n(&__aarch64_cpu_features.features, features,
+ CHECK_BIT(CAP_BIT_FEAT_SME2p1, FEAT_SME2P1);
+ CHECK_BIT(CAP_BIT_FEAT_SME_F16F16, FEAT_SME_F16F16);
+ CHECK_BIT(CAP_BIT_FEAT_SME_F8F16, FEAT_SME_F8F16);
+ CHECK_BIT(CAP_BIT_FEAT_SME_F8F32, FEAT_SME_F8F32);
+ // Both SME FP8 extensions require the base FP8 feature.
+ CHECK_BIT(CAP_BIT_FEAT_SME_F8F16, FEAT_FP8);
+ CHECK_BIT(CAP_BIT_FEAT_SME_F8F32, FEAT_FP8);
+ CHECK_BIT(CAP_BIT_FEAT_MTE2, FEAT_MEMTAG2);
+ CHECK_BIT(CAP_BIT_FEAT_CSSC, FEAT_CSSC);
+
+ features[0] = deriveImplicitFeatures(features[0]);
+
+ __atomic_store_n(&__aarch64_cpu_features.features2, features[1],
+ __ATOMIC_RELAXED);
+ __atomic_store_n(&__aarch64_cpu_features.features, features[0],
__ATOMIC_RELEASE);
return;
}
@@ -140,12 +178,12 @@ void __init_cpu_features_resolver(void) {
for (size_t I = 0, E = sizeof(feature_checks) / sizeof(feature_checks[0]);
I != E; ++I)
if (isKnownAndSupported(feature_checks[I].sysctl_name))
- features |= (1ULL << feature_checks[I].feature);
+ features[0] |= (1ULL << feature_checks[I].feature);
- features = deriveImplicitFeatures(features);
+ features[0] = deriveImplicitFeatures(features[0]);
__atomic_store_n(&__aarch64_cpu_features.features2, 0, __ATOMIC_RELAXED);
- __atomic_store_n(&__aarch64_cpu_features.features, features,
+ __atomic_store_n(&__aarch64_cpu_features.features, features[0],
__ATOMIC_RELEASE);
}
diff --git a/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/hwcap.inc b/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/hwcap.inc
index 29b0a570a1412..e7fcf9b8ff808 100644
--- a/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/hwcap.inc
+++ b/compiler-rt/lib/builtins/cpu_model/aarch64/fmv/hwcap.inc
@@ -161,6 +161,5 @@ static void __init_cpu_features_constructor(unsigned long hwcap,
__atomic_store_n(&__aarch64_cpu_features.features2, feat[1],
__ATOMIC_RELAXED);
- __atomic_store_n(&__aarch64_cpu_features.features, feat[0],
- __ATOMIC_RELEASE);
+ __atomic_store_n(&__aarch64_cpu_features.features, feat[0], __ATOMIC_RELEASE);
}
diff --git a/compiler-rt/lib/builtins/cpu_model/aarch64/hwcap.inc b/compiler-rt/lib/builtins/cpu_model/aarch64/hwcap.inc
index d4e19cf912862..49cd9e754edd6 100644
--- a/compiler-rt/lib/builtins/cpu_model/aarch64/hwcap.inc
+++ b/compiler-rt/lib/builtins/cpu_model/aarch64/hwcap.inc
@@ -1,3 +1,4 @@
+//https://github.com/torvalds/linux/blob/master/arch/arm64/include/uapi/asm/hwcap.h
#ifndef _IFUNC_ARG_HWCAP
#define _IFUNC_ARG_HWCAP (1ULL << 62)
#endif
More information about the cfe-commits
mailing list