[llvm] RuntimeLibcalls: Add entries for vector sincospi functions (PR #166981)
Matt Arsenault via llvm-commits
llvm-commits at lists.llvm.org
Mon Nov 10 10:18:24 PST 2025
https://github.com/arsenm updated https://github.com/llvm/llvm-project/pull/166981
>From de4d7b44aabada57e061ff28b1adbc5d1ea2e223 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Thu, 6 Nov 2025 13:17:49 -0800
Subject: [PATCH 1/3] RuntimeLibcalls: Add entries for vector sincospi
functions
Add libcall entries for sleef and armpl sincospi implementations.
This is the start of adding the vector library functions; eventually
they should all be tracked here.
I'm starting with this case because this is a prerequisite to fix
reporting sincospi calls which do not exist on any common targets without
regressing vector codegen when these libraries are available.
---
llvm/include/llvm/IR/RuntimeLibcalls.h | 15 +--
llvm/include/llvm/IR/RuntimeLibcalls.td | 26 +++++
llvm/lib/CodeGen/TargetLoweringBase.cpp | 18 ++++
llvm/lib/IR/RuntimeLibcalls.cpp | 97 +++++++++++++++++++
.../Util/DeclareRuntimeLibcalls/armpl.ll | 12 +++
.../Util/DeclareRuntimeLibcalls/sleef.ll | 12 +++
6 files changed, 170 insertions(+), 10 deletions(-)
create mode 100644 llvm/test/Transforms/Util/DeclareRuntimeLibcalls/armpl.ll
create mode 100644 llvm/test/Transforms/Util/DeclareRuntimeLibcalls/sleef.ll
diff --git a/llvm/include/llvm/IR/RuntimeLibcalls.h b/llvm/include/llvm/IR/RuntimeLibcalls.h
index c822b6530a441..0afe32a4ecc3c 100644
--- a/llvm/include/llvm/IR/RuntimeLibcalls.h
+++ b/llvm/include/llvm/IR/RuntimeLibcalls.h
@@ -83,16 +83,7 @@ struct RuntimeLibcallsInfo {
const Triple &TT,
ExceptionHandling ExceptionModel = ExceptionHandling::None,
FloatABI::ABIType FloatABI = FloatABI::Default,
- EABI EABIVersion = EABI::Default, StringRef ABIName = "") {
- // FIXME: The ExceptionModel parameter is to handle the field in
- // TargetOptions. This interface fails to distinguish the forced disable
- // case for targets which support exceptions by default. This should
- // probably be a module flag and removed from TargetOptions.
- if (ExceptionModel == ExceptionHandling::None)
- ExceptionModel = TT.getDefaultExceptionHandling();
-
- initLibcalls(TT, ExceptionModel, FloatABI, EABIVersion, ABIName);
- }
+ EABI EABIVersion = EABI::Default, StringRef ABIName = "");
explicit RuntimeLibcallsInfo(const Module &M);
@@ -170,6 +161,10 @@ struct RuntimeLibcallsInfo {
getFunctionTy(LLVMContext &Ctx, const Triple &TT, const DataLayout &DL,
RTLIB::LibcallImpl LibcallImpl) const;
+ /// Returns true if the function has a vector mask argument, which is assumed
+ /// to be the last argument.
+ static bool hasVectorMaskArgument(RTLIB::LibcallImpl Impl);
+
private:
LLVM_ABI static iota_range<RTLIB::LibcallImpl>
lookupLibcallImplNameImpl(StringRef Name);
diff --git a/llvm/include/llvm/IR/RuntimeLibcalls.td b/llvm/include/llvm/IR/RuntimeLibcalls.td
index 24c1b035d0dda..1a752aed31992 100644
--- a/llvm/include/llvm/IR/RuntimeLibcalls.td
+++ b/llvm/include/llvm/IR/RuntimeLibcalls.td
@@ -182,6 +182,10 @@ foreach FPTy = ["F32", "F64", "F80", "F128", "PPCF128"] in {
def MODF_#FPTy : RuntimeLibcall;
}
+foreach VecTy = ["V4F32", "V2F64", "NXV4F32", "NXV2F64"] in {
+ def SINCOSPI_#VecTy : RuntimeLibcall;
+}
+
def FEGETENV : RuntimeLibcall;
def FESETENV : RuntimeLibcall;
@@ -1078,6 +1082,28 @@ def __security_check_cookie : RuntimeLibcallImpl<SECURITY_CHECK_COOKIE>;
def __security_check_cookie_arm64ec : RuntimeLibcallImpl<SECURITY_CHECK_COOKIE,
"#__security_check_cookie_arm64ec">;
+//===----------------------------------------------------------------------===//
+// sleef calls
+//===----------------------------------------------------------------------===//
+
+defset list<RuntimeLibcallImpl> SleefLibcalls = {
+ def _ZGVnN4vl4l4_sincospif : RuntimeLibcallImpl<SINCOSPI_V4F32>;
+ def _ZGVnN2vl8l8_sincospi : RuntimeLibcallImpl<SINCOSPI_V2F64>;
+ def _ZGVsNxvl4l4_sincospif : RuntimeLibcallImpl<SINCOSPI_NXV4F32>;
+ def _ZGVsNxvl8l8_sincospi : RuntimeLibcallImpl<SINCOSPI_NXV2F64>;
+}
+
+//===----------------------------------------------------------------------===//
+// ARMPL calls
+//===----------------------------------------------------------------------===//
+
+defset list<RuntimeLibcallImpl> ARMPLLibcalls = {
+ def armpl_vsincospiq_f32 : RuntimeLibcallImpl<SINCOSPI_V4F32>;
+ def armpl_vsincospiq_f64 : RuntimeLibcallImpl<SINCOSPI_V2F64>;
+ def armpl_svsincospi_f32_x : RuntimeLibcallImpl<SINCOSPI_NXV4F32>;
+ def armpl_svsincospi_f64_x : RuntimeLibcallImpl<SINCOSPI_NXV2F64>;
+}
+
//===----------------------------------------------------------------------===//
// F128 libm Runtime Libcalls
//===----------------------------------------------------------------------===//
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index 1cc591c17f9c3..814b4b57a0b9b 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -430,6 +430,24 @@ RTLIB::Libcall RTLIB::getSINCOS(EVT RetVT) {
}
RTLIB::Libcall RTLIB::getSINCOSPI(EVT RetVT) {
+ // TODO: Tablegen should generate this function
+ if (RetVT.isVector()) {
+ if (!RetVT.isSimple())
+ return RTLIB::UNKNOWN_LIBCALL;
+ switch (RetVT.getSimpleVT().SimpleTy) {
+ case MVT::v4f32:
+ return RTLIB::SINCOSPI_V4F32;
+ case MVT::v2f64:
+ return RTLIB::SINCOSPI_V2F64;
+ case MVT::nxv4f32:
+ return RTLIB::SINCOSPI_NXV4F32;
+ case MVT::nxv2f64:
+ return RTLIB::SINCOSPI_NXV2F64;
+ default:
+ return RTLIB::UNKNOWN_LIBCALL;
+ }
+ }
+
return getFPLibCall(RetVT, SINCOSPI_F32, SINCOSPI_F64, SINCOSPI_F80,
SINCOSPI_F128, SINCOSPI_PPCF128);
}
diff --git a/llvm/lib/IR/RuntimeLibcalls.cpp b/llvm/lib/IR/RuntimeLibcalls.cpp
index f4c5c6ff35af6..3d4462ed7fcb2 100644
--- a/llvm/lib/IR/RuntimeLibcalls.cpp
+++ b/llvm/lib/IR/RuntimeLibcalls.cpp
@@ -10,6 +10,7 @@
#include "llvm/ADT/FloatingPointMode.h"
#include "llvm/ADT/StringTable.h"
#include "llvm/IR/Module.h"
+#include "llvm/IR/SystemLibraries.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/xxhash.h"
#include "llvm/TargetParser/ARMTargetParser.h"
@@ -25,6 +26,40 @@ using namespace RTLIB;
#define DEFINE_GET_LOOKUP_LIBCALL_IMPL_NAME
#include "llvm/IR/RuntimeLibcalls.inc"
+RuntimeLibcallsInfo::RuntimeLibcallsInfo(const Triple &TT,
+ ExceptionHandling ExceptionModel,
+ FloatABI::ABIType FloatABI,
+ EABI EABIVersion, StringRef ABIName) {
+ // FIXME: The ExceptionModel parameter is to handle the field in
+ // TargetOptions. This interface fails to distinguish the forced disable
+ // case for targets which support exceptions by default. This should
+ // probably be a module flag and removed from TargetOptions.
+ if (ExceptionModel == ExceptionHandling::None)
+ ExceptionModel = TT.getDefaultExceptionHandling();
+
+ initLibcalls(TT, ExceptionModel, FloatABI, EABIVersion, ABIName);
+
+ // TODO: Tablegen should generate these sets
+ switch (ClVectorLibrary) {
+ case VectorLibrary::SLEEFGNUABI:
+ for (RTLIB::LibcallImpl Impl :
+ {RTLIB::impl__ZGVnN4vl4l4_sincospif, RTLIB::impl__ZGVnN2vl8l8_sincospi,
+ RTLIB::impl__ZGVsNxvl4l4_sincospif,
+ RTLIB::impl__ZGVsNxvl8l8_sincospi})
+ setAvailable(Impl);
+ break;
+ case VectorLibrary::ArmPL:
+ for (RTLIB::LibcallImpl Impl :
+ {RTLIB::impl_armpl_vsincospiq_f32, RTLIB::impl_armpl_vsincospiq_f64,
+ RTLIB::impl_armpl_svsincospi_f32_x,
+ RTLIB::impl_armpl_svsincospi_f64_x})
+ setAvailable(Impl);
+ break;
+ default:
+ break;
+ }
+}
+
RuntimeLibcallsInfo::RuntimeLibcallsInfo(const Module &M)
: RuntimeLibcallsInfo(M.getTargetTriple()) {
// TODO: Consider module flags
@@ -88,6 +123,8 @@ RuntimeLibcallsInfo::getFunctionTy(LLVMContext &Ctx, const Triple &TT,
static constexpr Attribute::AttrKind CommonFnAttrs[] = {
Attribute::NoCallback, Attribute::NoFree, Attribute::NoSync,
Attribute::NoUnwind, Attribute::WillReturn};
+ static constexpr Attribute::AttrKind CommonPtrArgAttrs[] = {
+ Attribute::NoAlias, Attribute::WriteOnly, Attribute::NonNull};
switch (LibcallImpl) {
case RTLIB::impl___sincos_stret:
@@ -151,9 +188,69 @@ RuntimeLibcallsInfo::getFunctionTy(LLVMContext &Ctx, const Triple &TT,
fcNegNormal));
return {FuncTy, Attrs};
}
+ case RTLIB::impl__ZGVnN4vl4l4_sincospif:
+ case RTLIB::impl__ZGVnN2vl8l8_sincospi:
+ case RTLIB::impl__ZGVsNxvl4l4_sincospif:
+ case RTLIB::impl__ZGVsNxvl8l8_sincospi:
+ case RTLIB::impl_armpl_vsincospiq_f32:
+ case RTLIB::impl_armpl_vsincospiq_f64:
+ case RTLIB::impl_armpl_svsincospi_f32_x:
+ case RTLIB::impl_armpl_svsincospi_f64_x: {
+ AttrBuilder FuncAttrBuilder(Ctx);
+
+ bool IsF32 = LibcallImpl == RTLIB::impl__ZGVnN4vl4l4_sincospif ||
+ LibcallImpl == RTLIB::impl__ZGVsNxvl4l4_sincospif ||
+ LibcallImpl == RTLIB::impl_armpl_vsincospiq_f32 ||
+ LibcallImpl == RTLIB::impl_armpl_svsincospi_f32_x;
+ Type *ScalarTy = IsF32 ? Type::getFloatTy(Ctx) : Type::getDoubleTy(Ctx);
+ unsigned EC = IsF32 ? 4 : 2;
+
+ bool IsScalable = LibcallImpl == RTLIB::impl__ZGVsNxvl4l4_sincospif ||
+ LibcallImpl == RTLIB::impl__ZGVsNxvl8l8_sincospi ||
+ LibcallImpl == RTLIB::impl_armpl_svsincospi_f32_x ||
+ LibcallImpl == RTLIB::impl_armpl_svsincospi_f64_x;
+ Type *VecTy =
+ IsScalable ? static_cast<Type *>(ScalableVectorType::get(ScalarTy, EC))
+ : static_cast<Type *>(FixedVectorType::get(ScalarTy, EC));
+
+ for (Attribute::AttrKind Attr : CommonFnAttrs)
+ FuncAttrBuilder.addAttribute(Attr);
+ FuncAttrBuilder.addMemoryAttr(MemoryEffects::argMemOnly(ModRefInfo::Mod));
+
+ AttributeList Attrs;
+ Attrs = Attrs.addFnAttributes(Ctx, FuncAttrBuilder);
+
+ {
+ AttrBuilder ArgAttrBuilder(Ctx);
+ for (Attribute::AttrKind AK : CommonPtrArgAttrs)
+ ArgAttrBuilder.addAttribute(AK);
+ ArgAttrBuilder.addAlignmentAttr(DL.getABITypeAlign(VecTy));
+ Attrs = Attrs.addParamAttributes(Ctx, 1, ArgAttrBuilder);
+ Attrs = Attrs.addParamAttributes(Ctx, 2, ArgAttrBuilder);
+ }
+
+ PointerType *PtrTy = PointerType::get(Ctx, 0);
+ SmallVector<Type *, 4> ArgTys = {VecTy, PtrTy, PtrTy};
+ if (IsScalable && hasVectorMaskArgument(LibcallImpl))
+ ArgTys.push_back(ScalableVectorType::get(Type::getInt1Ty(Ctx), EC));
+
+ return {FunctionType::get(Type::getVoidTy(Ctx), ArgTys, false), Attrs};
+ }
default:
return {};
}
return {};
}
+
+bool RuntimeLibcallsInfo::hasVectorMaskArgument(RTLIB::LibcallImpl Impl) {
+ /// FIXME: This should be generated by tablegen and support the argument at an
+ /// arbitrary position
+ switch (Impl) {
+ case RTLIB::impl_armpl_svsincospi_f32_x:
+ case RTLIB::impl_armpl_svsincospi_f64_x:
+ return true;
+ default:
+ return false;
+ }
+}
diff --git a/llvm/test/Transforms/Util/DeclareRuntimeLibcalls/armpl.ll b/llvm/test/Transforms/Util/DeclareRuntimeLibcalls/armpl.ll
new file mode 100644
index 0000000000000..c45f319f80122
--- /dev/null
+++ b/llvm/test/Transforms/Util/DeclareRuntimeLibcalls/armpl.ll
@@ -0,0 +1,12 @@
+; REQUIRES: aarch64-registered-target
+; RUN: opt -S -passes=declare-runtime-libcalls -mtriple=aarch64-unknown-linux -mattr=+neon,+sve -vector-library=ArmPL < %s | FileCheck %s
+
+; CHECK: declare void @armpl_svsincospi_f32_x(<vscale x 4 x float>, ptr noalias nonnull writeonly align 16, ptr noalias nonnull writeonly align 16, <vscale x 4 x i1>) [[ATTRS:#[0-9]+]]
+
+; CHECK: declare void @armpl_svsincospi_f64_x(<vscale x 2 x double>, ptr noalias nonnull writeonly align 16, ptr noalias nonnull writeonly align 16, <vscale x 2 x i1>) [[ATTRS:#[0-9]+]]
+
+; CHECK: declare void @armpl_vsincospiq_f32(<4 x float>, ptr noalias nonnull writeonly align 16, ptr noalias nonnull writeonly align 16) [[ATTRS]]
+
+; CHECK: declare void @armpl_vsincospiq_f64(<2 x double>, ptr noalias nonnull writeonly align 16, ptr noalias nonnull writeonly align 16) [[ATTRS]]
+
+; CHECK: attributes [[ATTRS]] = { nocallback nofree nosync nounwind willreturn memory(argmem: write) }
diff --git a/llvm/test/Transforms/Util/DeclareRuntimeLibcalls/sleef.ll b/llvm/test/Transforms/Util/DeclareRuntimeLibcalls/sleef.ll
new file mode 100644
index 0000000000000..7972e0ca1c487
--- /dev/null
+++ b/llvm/test/Transforms/Util/DeclareRuntimeLibcalls/sleef.ll
@@ -0,0 +1,12 @@
+; REQUIRES: aarch64-registered-target
+; RUN: opt -S -passes=declare-runtime-libcalls -mtriple=aarch64-unknown-linux -mattr=+neon,+sve -vector-library=sleefgnuabi < %s | FileCheck %s
+
+; CHECK: declare void @_ZGVnN2vl8l8_sincospi(<2 x double>, ptr noalias nonnull writeonly align 16, ptr noalias nonnull writeonly align 16) [[ATTRS:#[0-9]+]]
+
+; CHECK: declare void @_ZGVnN4vl4l4_sincospif(<4 x float>, ptr noalias nonnull writeonly align 16, ptr noalias nonnull writeonly align 16) [[ATTRS]]
+
+; CHECK: declare void @_ZGVsNxvl4l4_sincospif(<vscale x 4 x float>, ptr noalias nonnull writeonly align 16, ptr noalias nonnull writeonly align 16) [[ATTRS:#[0-9]+]]
+
+; CHECK: declare void @_ZGVsNxvl8l8_sincospi(<vscale x 2 x double>, ptr noalias nonnull writeonly align 16, ptr noalias nonnull writeonly align 16) [[ATTRS:#[0-9]+]]
+
+; CHECK: attributes [[ATTRS]] = { nocallback nofree nosync nounwind willreturn memory(argmem: write) }
>From 0d1d36b7f25dd0d4b0cdf13fa24a51304ef85ca0 Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Mon, 10 Nov 2025 09:12:37 -0800
Subject: [PATCH 2/3] Always use hasVectorMaskArgument
---
llvm/lib/IR/RuntimeLibcalls.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/IR/RuntimeLibcalls.cpp b/llvm/lib/IR/RuntimeLibcalls.cpp
index 3d4462ed7fcb2..ad486fd871d8a 100644
--- a/llvm/lib/IR/RuntimeLibcalls.cpp
+++ b/llvm/lib/IR/RuntimeLibcalls.cpp
@@ -231,8 +231,8 @@ RuntimeLibcallsInfo::getFunctionTy(LLVMContext &Ctx, const Triple &TT,
PointerType *PtrTy = PointerType::get(Ctx, 0);
SmallVector<Type *, 4> ArgTys = {VecTy, PtrTy, PtrTy};
- if (IsScalable && hasVectorMaskArgument(LibcallImpl))
- ArgTys.push_back(ScalableVectorType::get(Type::getInt1Ty(Ctx), EC));
+ if (hasVectorMaskArgument(LibcallImpl))
+ ArgTys.push_back(VectorType::get(Type::getInt1Ty(Ctx), EC, IsScalable));
return {FunctionType::get(Type::getVoidTy(Ctx), ArgTys, false), Attrs};
}
>From 500dbaeb46971fe86ea2fbb1556944760fa772ee Mon Sep 17 00:00:00 2001
From: Matt Arsenault <Matthew.Arsenault at amd.com>
Date: Mon, 10 Nov 2025 09:40:35 -0800
Subject: [PATCH 3/3] VectorType::get
---
llvm/lib/IR/RuntimeLibcalls.cpp | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/llvm/lib/IR/RuntimeLibcalls.cpp b/llvm/lib/IR/RuntimeLibcalls.cpp
index ad486fd871d8a..795621701d910 100644
--- a/llvm/lib/IR/RuntimeLibcalls.cpp
+++ b/llvm/lib/IR/RuntimeLibcalls.cpp
@@ -209,9 +209,7 @@ RuntimeLibcallsInfo::getFunctionTy(LLVMContext &Ctx, const Triple &TT,
LibcallImpl == RTLIB::impl__ZGVsNxvl8l8_sincospi ||
LibcallImpl == RTLIB::impl_armpl_svsincospi_f32_x ||
LibcallImpl == RTLIB::impl_armpl_svsincospi_f64_x;
- Type *VecTy =
- IsScalable ? static_cast<Type *>(ScalableVectorType::get(ScalarTy, EC))
- : static_cast<Type *>(FixedVectorType::get(ScalarTy, EC));
+ VectorType *VecTy = VectorType::get(ScalarTy, EC, IsScalable);
for (Attribute::AttrKind Attr : CommonFnAttrs)
FuncAttrBuilder.addAttribute(Attr);
More information about the llvm-commits
mailing list