[libc-commits] [libc] [llvm] [libc] Use __builtin_fma(f) by default if LIBC_TARGET_CPU_HAS_FMA is defined. (PR #91535)
via libc-commits
libc-commits at lists.llvm.org
Wed May 8 18:38:46 PDT 2024
https://github.com/lntue updated https://github.com/llvm/llvm-project/pull/91535
>From 602f5ccd450fcc4d5fa29d7aceb1875fb326427a Mon Sep 17 00:00:00 2001
From: Tue Ly <lntue.h at gmail.com>
Date: Wed, 8 May 2024 20:55:05 +0000
Subject: [PATCH 1/3] [libc] Update FMA inclusion for ARM targets.
---
libc/src/__support/FPUtil/FMA.h | 2 +-
libc/src/__support/FPUtil/aarch64/FMA.h | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/libc/src/__support/FPUtil/FMA.h b/libc/src/__support/FPUtil/FMA.h
index 0e1ede02d5cc0..5d677503d2be9 100644
--- a/libc/src/__support/FPUtil/FMA.h
+++ b/libc/src/__support/FPUtil/FMA.h
@@ -16,7 +16,7 @@
#if defined(LIBC_TARGET_ARCH_IS_X86_64)
#include "x86_64/FMA.h"
-#elif defined(LIBC_TARGET_ARCH_IS_AARCH64)
+#elif defined(LIBC_TARGET_ARCH_IS_ANY_ARM)
#include "aarch64/FMA.h"
#elif defined(LIBC_TARGET_ARCH_IS_ANY_RISCV)
#include "riscv/FMA.h"
diff --git a/libc/src/__support/FPUtil/aarch64/FMA.h b/libc/src/__support/FPUtil/aarch64/FMA.h
index 6254a0673ff42..bfcc19f82a340 100644
--- a/libc/src/__support/FPUtil/aarch64/FMA.h
+++ b/libc/src/__support/FPUtil/aarch64/FMA.h
@@ -13,7 +13,7 @@
#include "src/__support/macros/properties/architectures.h"
#include "src/__support/macros/properties/cpu_features.h" // LIBC_TARGET_CPU_HAS_FMA
-#if !defined(LIBC_TARGET_ARCH_IS_AARCH64)
+#if !defined(LIBC_TARGET_ARCH_IS_ANY_ARM)
#error "Invalid include"
#endif
>From 2f784429bd88f51c69aa7b266ab503e397001179 Mon Sep 17 00:00:00 2001
From: Tue Ly <lntue.h at gmail.com>
Date: Thu, 9 May 2024 01:33:54 +0000
Subject: [PATCH 2/3] Use __builtin_fma(f) by default if
LIBC_TARGET_CPU_HAS_FMA is defined.
---
libc/src/__support/FPUtil/FMA.h | 26 +++++----
libc/src/__support/FPUtil/aarch64/FMA.h | 50 -----------------
libc/src/__support/FPUtil/gpu/FMA.h | 36 ------------
libc/src/__support/FPUtil/riscv/FMA.h | 54 ------------------
libc/src/__support/FPUtil/x86_64/FMA.h | 55 -------------------
.../llvm-project-overlay/libc/BUILD.bazel | 8 ---
6 files changed, 16 insertions(+), 213 deletions(-)
delete mode 100644 libc/src/__support/FPUtil/aarch64/FMA.h
delete mode 100644 libc/src/__support/FPUtil/gpu/FMA.h
delete mode 100644 libc/src/__support/FPUtil/riscv/FMA.h
delete mode 100644 libc/src/__support/FPUtil/x86_64/FMA.h
diff --git a/libc/src/__support/FPUtil/FMA.h b/libc/src/__support/FPUtil/FMA.h
index 5d677503d2be9..ed71b384a4d95 100644
--- a/libc/src/__support/FPUtil/FMA.h
+++ b/libc/src/__support/FPUtil/FMA.h
@@ -11,23 +11,29 @@
#include "src/__support/macros/properties/architectures.h"
#include "src/__support/macros/properties/cpu_features.h" // LIBC_TARGET_CPU_HAS_FMA
+#include "src/__support/CPP/type_traits.h"
#if defined(LIBC_TARGET_CPU_HAS_FMA)
-#if defined(LIBC_TARGET_ARCH_IS_X86_64)
-#include "x86_64/FMA.h"
-#elif defined(LIBC_TARGET_ARCH_IS_ANY_ARM)
-#include "aarch64/FMA.h"
-#elif defined(LIBC_TARGET_ARCH_IS_ANY_RISCV)
-#include "riscv/FMA.h"
-#elif defined(LIBC_TARGET_ARCH_IS_GPU)
-#include "gpu/FMA.h"
-#endif
+namespace LIBC_NAMESPACE {
+namespace fputil {
+
+template <typename T>
+LIBC_INLINE cpp::enable_if_t<cpp::is_same_v<T, float>, T> fma(T x, T y, T z) {
+ return __builtin_fmaf(x, y, z);
+}
+
+template <typename T>
+LIBC_INLINE cpp::enable_if_t<cpp::is_same_v<T, double>, T> fma(T x, T y, T z) {
+ return __builtin_fma(x, y, z);
+}
+
+} // namespace fputil
+} // namespace LIBC_NAMESPACE
#else
// FMA instructions are not available
#include "generic/FMA.h"
-#include "src/__support/CPP/type_traits.h"
namespace LIBC_NAMESPACE {
namespace fputil {
diff --git a/libc/src/__support/FPUtil/aarch64/FMA.h b/libc/src/__support/FPUtil/aarch64/FMA.h
deleted file mode 100644
index bfcc19f82a340..0000000000000
--- a/libc/src/__support/FPUtil/aarch64/FMA.h
+++ /dev/null
@@ -1,50 +0,0 @@
-//===-- Aarch64 implementations of the fma function -------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_SRC___SUPPORT_FPUTIL_AARCH64_FMA_H
-#define LLVM_LIBC_SRC___SUPPORT_FPUTIL_AARCH64_FMA_H
-
-#include "src/__support/macros/attributes.h" // LIBC_INLINE
-#include "src/__support/macros/properties/architectures.h"
-#include "src/__support/macros/properties/cpu_features.h" // LIBC_TARGET_CPU_HAS_FMA
-
-#if !defined(LIBC_TARGET_ARCH_IS_ANY_ARM)
-#error "Invalid include"
-#endif
-
-#if !defined(LIBC_TARGET_CPU_HAS_FMA)
-#error "FMA instructions are not supported"
-#endif
-
-#include "src/__support/CPP/type_traits.h"
-
-namespace LIBC_NAMESPACE {
-namespace fputil {
-
-template <typename T>
-LIBC_INLINE cpp::enable_if_t<cpp::is_same_v<T, float>, T> fma(T x, T y, T z) {
- float result;
- LIBC_INLINE_ASM("fmadd %s0, %s1, %s2, %s3\n\t"
- : "=w"(result)
- : "w"(x), "w"(y), "w"(z));
- return result;
-}
-
-template <typename T>
-LIBC_INLINE cpp::enable_if_t<cpp::is_same_v<T, double>, T> fma(T x, T y, T z) {
- double result;
- LIBC_INLINE_ASM("fmadd %d0, %d1, %d2, %d3\n\t"
- : "=w"(result)
- : "w"(x), "w"(y), "w"(z));
- return result;
-}
-
-} // namespace fputil
-} // namespace LIBC_NAMESPACE
-
-#endif // LLVM_LIBC_SRC___SUPPORT_FPUTIL_AARCH64_FMA_H
diff --git a/libc/src/__support/FPUtil/gpu/FMA.h b/libc/src/__support/FPUtil/gpu/FMA.h
deleted file mode 100644
index ef1cd26a72dd7..0000000000000
--- a/libc/src/__support/FPUtil/gpu/FMA.h
+++ /dev/null
@@ -1,36 +0,0 @@
-//===-- GPU implementations of the fma function -----------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_SRC___SUPPORT_FPUTIL_GPU_FMA_H
-#define LLVM_LIBC_SRC___SUPPORT_FPUTIL_GPU_FMA_H
-
-#include "src/__support/CPP/type_traits.h"
-
-// These intrinsics map to the FMA instructions in the target ISA for the GPU.
-// The default rounding mode generated from these will be to the nearest even.
-#if !__has_builtin(__builtin_fma) || !__has_builtin(__builtin_fmaf)
-#error "FMA builtins must be defined");
-#endif
-
-namespace LIBC_NAMESPACE {
-namespace fputil {
-
-template <typename T>
-LIBC_INLINE cpp::enable_if_t<cpp::is_same_v<T, float>, T> fma(T x, T y, T z) {
- return __builtin_fmaf(x, y, z);
-}
-
-template <typename T>
-LIBC_INLINE cpp::enable_if_t<cpp::is_same_v<T, double>, T> fma(T x, T y, T z) {
- return __builtin_fma(x, y, z);
-}
-
-} // namespace fputil
-} // namespace LIBC_NAMESPACE
-
-#endif // LLVM_LIBC_SRC___SUPPORT_FPUTIL_GPU_FMA_H
diff --git a/libc/src/__support/FPUtil/riscv/FMA.h b/libc/src/__support/FPUtil/riscv/FMA.h
deleted file mode 100644
index f01962174f16f..0000000000000
--- a/libc/src/__support/FPUtil/riscv/FMA.h
+++ /dev/null
@@ -1,54 +0,0 @@
-//===-- RISCV implementations of the fma function ---------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_SRC___SUPPORT_FPUTIL_RISCV_FMA_H
-#define LLVM_LIBC_SRC___SUPPORT_FPUTIL_RISCV_FMA_H
-
-#include "src/__support/macros/attributes.h" // LIBC_INLINE
-#include "src/__support/macros/properties/architectures.h"
-#include "src/__support/macros/properties/cpu_features.h" // LIBC_TARGET_CPU_HAS_FMA
-
-#if !defined(LIBC_TARGET_ARCH_IS_ANY_RISCV)
-#error "Invalid include"
-#endif
-
-#if !defined(LIBC_TARGET_CPU_HAS_FMA)
-#error "FMA instructions are not supported"
-#endif
-
-#include "src/__support/CPP/type_traits.h"
-
-namespace LIBC_NAMESPACE {
-namespace fputil {
-
-#ifdef __riscv_flen
-template <typename T>
-LIBC_INLINE cpp::enable_if_t<cpp::is_same_v<T, float>, T> fma(T x, T y, T z) {
- float result;
- LIBC_INLINE_ASM("fmadd.s %0, %1, %2, %3\n\t"
- : "=f"(result)
- : "f"(x), "f"(y), "f"(z));
- return result;
-}
-
-#if __riscv_flen >= 64
-template <typename T>
-LIBC_INLINE cpp::enable_if_t<cpp::is_same_v<T, double>, T> fma(T x, T y, T z) {
- double result;
- LIBC_INLINE_ASM("fmadd.d %0, %1, %2, %3\n\t"
- : "=f"(result)
- : "f"(x), "f"(y), "f"(z));
- return result;
-}
-#endif // __riscv_flen >= 64
-#endif // __riscv_flen
-
-} // namespace fputil
-} // namespace LIBC_NAMESPACE
-
-#endif // LLVM_LIBC_SRC___SUPPORT_FPUTIL_RISCV_FMA_H
diff --git a/libc/src/__support/FPUtil/x86_64/FMA.h b/libc/src/__support/FPUtil/x86_64/FMA.h
deleted file mode 100644
index 91ef7f96ff4d3..0000000000000
--- a/libc/src/__support/FPUtil/x86_64/FMA.h
+++ /dev/null
@@ -1,55 +0,0 @@
-//===-- x86_64 implementations of the fma function --------------*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_LIBC_SRC___SUPPORT_FPUTIL_X86_64_FMA_H
-#define LLVM_LIBC_SRC___SUPPORT_FPUTIL_X86_64_FMA_H
-
-#include "src/__support/macros/attributes.h" // LIBC_INLINE
-#include "src/__support/macros/properties/architectures.h"
-#include "src/__support/macros/properties/cpu_features.h" // LIBC_TARGET_CPU_HAS_FMA
-
-#if !defined(LIBC_TARGET_ARCH_IS_X86_64)
-#error "Invalid include"
-#endif
-
-#if !defined(LIBC_TARGET_CPU_HAS_FMA)
-#error "FMA instructions are not supported"
-#endif
-
-#include "src/__support/CPP/type_traits.h"
-#include <immintrin.h>
-
-namespace LIBC_NAMESPACE {
-namespace fputil {
-
-template <typename T>
-LIBC_INLINE cpp::enable_if_t<cpp::is_same_v<T, float>, T> fma(T x, T y, T z) {
- float result;
- __m128 xmm = _mm_load_ss(&x); // NOLINT
- __m128 ymm = _mm_load_ss(&y); // NOLINT
- __m128 zmm = _mm_load_ss(&z); // NOLINT
- __m128 r = _mm_fmadd_ss(xmm, ymm, zmm); // NOLINT
- _mm_store_ss(&result, r); // NOLINT
- return result;
-}
-
-template <typename T>
-LIBC_INLINE cpp::enable_if_t<cpp::is_same_v<T, double>, T> fma(T x, T y, T z) {
- double result;
- __m128d xmm = _mm_load_sd(&x); // NOLINT
- __m128d ymm = _mm_load_sd(&y); // NOLINT
- __m128d zmm = _mm_load_sd(&z); // NOLINT
- __m128d r = _mm_fmadd_sd(xmm, ymm, zmm); // NOLINT
- _mm_store_sd(&result, r); // NOLINT
- return result;
-}
-
-} // namespace fputil
-} // namespace LIBC_NAMESPACE
-
-#endif // LLVM_LIBC_SRC___SUPPORT_FPUTIL_X86_64_FMA_H
diff --git a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
index 055630cb6a005..6255ac998db10 100644
--- a/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
+++ b/utils/bazel/llvm-project-overlay/libc/BUILD.bazel
@@ -913,17 +913,9 @@ fma_common_hdrs = [
"src/__support/FPUtil/generic/FMA.h",
]
-fma_platform_hdrs = [
- "src/__support/FPUtil/x86_64/FMA.h",
- "src/__support/FPUtil/aarch64/FMA.h",
-]
-
libc_support_library(
name = "__support_fputil_fma",
hdrs = fma_common_hdrs,
- # These are conditionally included and will #error out if the platform
- # doesn't support FMA, so they can't be compiled on their own.
- textual_hdrs = fma_platform_hdrs,
deps = [
":__support_cpp_bit",
":__support_cpp_type_traits",
>From 4854b87278a5a0c36154e117281d63b992def2e3 Mon Sep 17 00:00:00 2001
From: Tue Ly <lntue.h at gmail.com>
Date: Thu, 9 May 2024 01:38:25 +0000
Subject: [PATCH 3/3] clang-format.
---
libc/src/__support/FPUtil/FMA.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/libc/src/__support/FPUtil/FMA.h b/libc/src/__support/FPUtil/FMA.h
index ed71b384a4d95..c277da49538bf 100644
--- a/libc/src/__support/FPUtil/FMA.h
+++ b/libc/src/__support/FPUtil/FMA.h
@@ -9,9 +9,9 @@
#ifndef LLVM_LIBC_SRC___SUPPORT_FPUTIL_FMA_H
#define LLVM_LIBC_SRC___SUPPORT_FPUTIL_FMA_H
+#include "src/__support/CPP/type_traits.h"
#include "src/__support/macros/properties/architectures.h"
#include "src/__support/macros/properties/cpu_features.h" // LIBC_TARGET_CPU_HAS_FMA
-#include "src/__support/CPP/type_traits.h"
#if defined(LIBC_TARGET_CPU_HAS_FMA)
More information about the libc-commits
mailing list