[llvm] f6a359f - [CodeGen] Teach ReplaceWithVeclib split vector llvm.sincos when only sin/cos veclib mappings exist (#194639)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 14 18:54:14 PDT 2026
Author: Kito Cheng
Date: 2026-07-15T09:54:09+08:00
New Revision: f6a359fe20904afe5cc899d3c2fda258322f1f71
URL: https://github.com/llvm/llvm-project/commit/f6a359fe20904afe5cc899d3c2fda258322f1f71
DIFF: https://github.com/llvm/llvm-project/commit/f6a359fe20904afe5cc899d3c2fda258322f1f71.diff
LOG: [CodeGen] Teach ReplaceWithVeclib split vector llvm.sincos when only sin/cos veclib mappings exist (#194639)
Some vector math libraries provide vector sin and vector cos but no
vector sincos or no sincos with an ABI that LLVM can emit.
The one of the common case is glibc libmvec on x86: it exposes
`_ZGV{b,c,d,e}N{2,4,8,16}vvv_sincos{,f}` symbols, but those use a
vectors-of-pointers output ABI that expandMultipleResultFPLibCall does
not currently support. As a result, sincos will falls back to scalar
sincos calls even when the target has a fully working vector sin and
vector cos.
So we trying to split it into separate sin and cos calls, which will
then be replaced with vector calls if the target supports it, it
generally better than scalarized sincos calls.
Added:
llvm/test/CodeGen/X86/sincos-fpmath.ll
Modified:
llvm/lib/CodeGen/ReplaceWithVeclib.cpp
llvm/test/CodeGen/X86/veclib-llvm.sincos.ll
Removed:
################################################################################
diff --git a/llvm/lib/CodeGen/ReplaceWithVeclib.cpp b/llvm/lib/CodeGen/ReplaceWithVeclib.cpp
index 600b8d84e3926..4a240c914a252 100644
--- a/llvm/lib/CodeGen/ReplaceWithVeclib.cpp
+++ b/llvm/lib/CodeGen/ReplaceWithVeclib.cpp
@@ -94,6 +94,9 @@ static void replaceWithTLIFunction(IntrinsicInst *II, VFInfo &Info,
// safe for non-FP intrinsics, whose flags are simply empty).
auto *Replacement = IRBuilder.CreateCall(
TLIVecFunc, Args, OpBundles, /*FMFSource=*/II->getFastMathFlagsOrNone());
+ // Preserve fpmath for FP math
+ if (isa<FPMathOperator>(Replacement))
+ Replacement->copyMetadata(*II, {LLVMContext::MD_fpmath});
II->replaceAllUsesWith(Replacement);
Replacement->setCallingConv(TLIVecFunc->getCallingConv());
}
@@ -207,19 +210,113 @@ static bool replaceWithCallToVeclib(const TargetLibraryInfo &TLI,
return true;
}
+/// Returns true when \p TLI has a vector mapping for the scalar function name
+/// \p Name at \p EC (matching either masked or unmasked variants).
+static bool hasVectorMapping(const TargetLibraryInfo &TLI, StringRef Name,
+ ElementCount EC) {
+ return TLI.getVectorMappingInfo(Name, EC, /*Masked=*/false) ||
+ TLI.getVectorMappingInfo(Name, EC, /*Masked=*/true);
+}
+
+/// Returns true when \p TLI has a vector mapping for \p IID at the given
+/// element type and \p EC.
+static bool hasIntrinsicVectorMapping(const TargetLibraryInfo &TLI,
+ Intrinsic::ID IID, Type *ScalarTy,
+ ElementCount EC, Module *M) {
+ std::string Name = Intrinsic::getName(IID, {ScalarTy}, M);
+ return hasVectorMapping(TLI, Name, EC);
+}
+
+/// If \p II is a vector llvm.sincos with no direct vector library mapping but
+/// the target does have vector mappings for both llvm.sin and llvm.cos at the
+/// same element count, replace it with separate llvm.sin and llvm.cos calls
+/// and run the standard veclib replacement on each.
+static bool trySplitVectorSinCos(const TargetLibraryInfo &TLI,
+ IntrinsicInst *II,
+ SmallVectorImpl<Instruction *> &Replaced) {
+ if (II->getIntrinsicID() != Intrinsic::sincos)
+ return false;
+ Value *Arg = II->getArgOperand(0);
+ auto *VTy = dyn_cast<VectorType>(Arg->getType());
+ if (!VTy)
+ return false;
+
+ ElementCount EC = VTy->getElementCount();
+ Type *ScalarTy = VTy->getElementType();
+ Module *M = II->getModule();
+
+ // If a vector sincos mapping exists for the intrinsic name (e.g.
+ // "llvm.sincos.f32") or for the scalar libcall name ("sincos"/"sincosf"),
+ // leave the call alone -- SelectionDAG legalization will handle it via
+ // expandMultipleResultFPLibCall when the runtime libcall impl is enabled.
+ if (hasIntrinsicVectorMapping(TLI, Intrinsic::sincos, ScalarTy, EC, M))
+ return false;
+ LibFunc LF = NotLibFunc;
+ if (ScalarTy->isFloatTy())
+ LF = LibFunc_sincosf;
+ else if (ScalarTy->isDoubleTy())
+ LF = LibFunc_sincos;
+ if (LF != NotLibFunc && hasVectorMapping(TLI, TLI.getName(LF), EC))
+ return false;
+
+ // Splitting is only worthwhile when both sin and cos have vector mappings.
+ if (!hasIntrinsicVectorMapping(TLI, Intrinsic::sin, ScalarTy, EC, M) ||
+ !hasIntrinsicVectorMapping(TLI, Intrinsic::cos, ScalarTy, EC, M))
+ return false;
+
+ // All users must be extractvalue.
+ for (User *U : II->users()) {
+ if (!isa<ExtractValueInst>(U))
+ return false;
+ }
+
+ IRBuilder<> B(II);
+ Function *SinFn =
+ Intrinsic::getOrInsertDeclaration(M, Intrinsic::sin, Arg->getType());
+ Function *CosFn =
+ Intrinsic::getOrInsertDeclaration(M, Intrinsic::cos, Arg->getType());
+ CallInst *SinCall = B.CreateCall(SinFn, {Arg}, /*FMFSource=*/II, "sin");
+ CallInst *CosCall = B.CreateCall(CosFn, {Arg}, /*FMFSource=*/II, "cos");
+ SinCall->copyMetadata(*II, {LLVMContext::MD_fpmath});
+ CosCall->copyMetadata(*II, {LLVMContext::MD_fpmath});
+
+ // Forward extractvalue uses to the new calls.
+ for (User *U : make_early_inc_range(II->users())) {
+ auto *EV = cast<ExtractValueInst>(U);
+ EV->replaceAllUsesWith(EV->getIndices()[0] == 0 ? SinCall : CosCall);
+ EV->eraseFromParent();
+ }
+
+ // Replace each new call with the vector library function.
+ if (replaceWithCallToVeclib(TLI, cast<IntrinsicInst>(SinCall)))
+ Replaced.push_back(SinCall);
+ if (replaceWithCallToVeclib(TLI, cast<IntrinsicInst>(CosCall)))
+ Replaced.push_back(CosCall);
+
+ return true;
+}
+
static bool runImpl(const TargetLibraryInfo &TLI, Function &F) {
SmallVector<Instruction *> ReplacedCalls;
for (auto &I : instructions(F)) {
- // Process only intrinsic calls that return void or a vector.
- if (auto *II = dyn_cast<IntrinsicInst>(&I)) {
- if (II->getIntrinsicID() == Intrinsic::not_intrinsic)
- continue;
- if (!II->getType()->isVectorTy() && !II->getType()->isVoidTy())
- continue;
-
- if (replaceWithCallToVeclib(TLI, II))
- ReplacedCalls.push_back(&I);
+ auto *II = dyn_cast<IntrinsicInst>(&I);
+ if (!II)
+ continue;
+
+ // Vector llvm.sincos returns a struct so it does not fit the generic
+ // path below; try to split it into separate sin and cos calls when the
+ // target has vector mappings for them.
+ if (trySplitVectorSinCos(TLI, II, ReplacedCalls)) {
+ ReplacedCalls.push_back(&I);
+ continue;
}
+
+ // Process only intrinsic calls that return void or a vector.
+ if (!II->getType()->isVectorTy() && !II->getType()->isVoidTy())
+ continue;
+
+ if (replaceWithCallToVeclib(TLI, II))
+ ReplacedCalls.push_back(&I);
}
// Erase any intrinsic calls that were replaced with vector library calls.
for (auto *I : ReplacedCalls)
diff --git a/llvm/test/CodeGen/X86/sincos-fpmath.ll b/llvm/test/CodeGen/X86/sincos-fpmath.ll
new file mode 100644
index 0000000000000..8308e0fb7834c
--- /dev/null
+++ b/llvm/test/CodeGen/X86/sincos-fpmath.ll
@@ -0,0 +1,87 @@
+; RUN: opt -mtriple=x86_64-unknown-linux-gnu -vector-library=LIBMVEC -passes=replace-with-veclib -S < %s | FileCheck %s
+
+declare { <4 x float>, <4 x float> } @llvm.sincos.v4f32(<4 x float>)
+declare { <2 x double>, <2 x double> } @llvm.sincos.v2f64(<2 x double>)
+declare { <3 x float>, <3 x float> } @llvm.sincos.v3f32(<3 x float>)
+declare void @use_sincos_struct({ <4 x float>, <4 x float> })
+
+; v4f32 sincos -> _ZGVbN4v_sinf / _ZGVbN4v_cosf, both carrying !fpmath !0.
+define void @sincos_fpmath_v4f32(<4 x float> %x, ptr noalias %sin_out, ptr noalias %cos_out) {
+; CHECK-LABEL: @sincos_fpmath_v4f32(
+; CHECK: call <4 x float> @_ZGVbN4v_sinf(<4 x float> %x), !fpmath !0
+; CHECK: call <4 x float> @_ZGVbN4v_cosf(<4 x float> %x), !fpmath !0
+;
+ %r = call { <4 x float>, <4 x float> } @llvm.sincos.v4f32(<4 x float> %x), !fpmath !0
+ %s = extractvalue { <4 x float>, <4 x float> } %r, 0
+ %c = extractvalue { <4 x float>, <4 x float> } %r, 1
+ store <4 x float> %s, ptr %sin_out, align 16
+ store <4 x float> %c, ptr %cos_out, align 16
+ ret void
+}
+
+; v2f64 sincos -> _ZGVbN2v_sin / _ZGVbN2v_cos, both carrying !fpmath !1.
+define void @sincos_fpmath_v2f64(<2 x double> %x, ptr noalias %sin_out, ptr noalias %cos_out) {
+; CHECK-LABEL: @sincos_fpmath_v2f64(
+; CHECK: call <2 x double> @_ZGVbN2v_sin(<2 x double> %x), !fpmath !1
+; CHECK: call <2 x double> @_ZGVbN2v_cos(<2 x double> %x), !fpmath !1
+;
+ %r = call { <2 x double>, <2 x double> } @llvm.sincos.v2f64(<2 x double> %x), !fpmath !1
+ %s = extractvalue { <2 x double>, <2 x double> } %r, 0
+ %c = extractvalue { <2 x double>, <2 x double> } %r, 1
+ store <2 x double> %s, ptr %sin_out, align 16
+ store <2 x double> %c, ptr %cos_out, align 16
+ ret void
+}
+
+; When the original sincos has no fpmath metadata, the resulting vector sin
+; and cos calls should also have none.
+define void @sincos_no_fpmath_v4f32(<4 x float> %x, ptr noalias %sin_out, ptr noalias %cos_out) {
+; CHECK-LABEL: @sincos_no_fpmath_v4f32(
+; CHECK: call <4 x float> @_ZGVbN4v_sinf(<4 x float> %x){{$}}
+; CHECK-NOT: !fpmath
+; CHECK: call <4 x float> @_ZGVbN4v_cosf(<4 x float> %x){{$}}
+; CHECK-NOT: !fpmath
+;
+ %r = call { <4 x float>, <4 x float> } @llvm.sincos.v4f32(<4 x float> %x)
+ %s = extractvalue { <4 x float>, <4 x float> } %r, 0
+ %c = extractvalue { <4 x float>, <4 x float> } %r, 1
+ store <4 x float> %s, ptr %sin_out, align 16
+ store <4 x float> %c, ptr %cos_out, align 16
+ ret void
+}
+
+; A non-extractvalue user blocks the split, so llvm.sincos is left intact.
+define void @sincos_non_extractvalue_user_v4f32(<4 x float> %x, ptr noalias %sin_out) {
+; CHECK-LABEL: @sincos_non_extractvalue_user_v4f32(
+; CHECK: call { <4 x float>, <4 x float> } @llvm.sincos.v4f32(<4 x float> %x)
+; CHECK-NOT: _ZGV
+; CHECK: ret void
+;
+ %r = call { <4 x float>, <4 x float> } @llvm.sincos.v4f32(<4 x float> %x)
+ call void @use_sincos_struct({ <4 x float>, <4 x float> } %r)
+ %s = extractvalue { <4 x float>, <4 x float> } %r, 0
+ store <4 x float> %s, ptr %sin_out, align 16
+ ret void
+}
+
+; An odd vector width has no LIBMVEC sin/cos mapping, so llvm.sincos is left intact.
+define void @sincos_no_veclib_mapping_v3f32(<3 x float> %x, ptr noalias %sin_out, ptr noalias %cos_out) {
+; CHECK-LABEL: @sincos_no_veclib_mapping_v3f32(
+; CHECK: call { <3 x float>, <3 x float> } @llvm.sincos.v3f32(<3 x float> %x)
+; CHECK-NOT: _ZGV
+; CHECK: ret void
+;
+ %r = call { <3 x float>, <3 x float> } @llvm.sincos.v3f32(<3 x float> %x)
+ %s = extractvalue { <3 x float>, <3 x float> } %r, 0
+ %c = extractvalue { <3 x float>, <3 x float> } %r, 1
+ store <3 x float> %s, ptr %sin_out, align 16
+ store <3 x float> %c, ptr %cos_out, align 16
+ ret void
+}
+
+; Verify the exact !fpmath metadata values are preserved.
+; CHECK: !0 = !{float 2.500000e+00}
+; CHECK: !1 = !{float 4.000000e+00}
+
+!0 = !{float 2.5}
+!1 = !{float 4.0}
diff --git a/llvm/test/CodeGen/X86/veclib-llvm.sincos.ll b/llvm/test/CodeGen/X86/veclib-llvm.sincos.ll
index 0075e85865667..b7ce01cfe5622 100644
--- a/llvm/test/CodeGen/X86/veclib-llvm.sincos.ll
+++ b/llvm/test/CodeGen/X86/veclib-llvm.sincos.ll
@@ -13,10 +13,8 @@ define void @test_sincos_v4f32(<4 x float> %x, ptr noalias %out_sin, ptr noalias
; AMD: callq amd_vrs4_sincosf at PLT
;
; GLIBC-LABEL: test_sincos_v4f32:
-; GLIBC: callq sincosf at PLT
-; GLIBC: callq sincosf at PLT
-; GLIBC: callq sincosf at PLT
-; GLIBC: callq sincosf at PLT
+; GLIBC: callq _ZGVbN4v_sinf at PLT
+; GLIBC: callq _ZGVbN4v_cosf at PLT
%result = call { <4 x float>, <4 x float> } @llvm.sincos.v4f32(<4 x float> %x)
%result.0 = extractvalue { <4 x float>, <4 x float> } %result, 0
%result.1 = extractvalue { <4 x float>, <4 x float> } %result, 1
@@ -41,14 +39,8 @@ define void @test_sincos_v8f32(<8 x float> %x, ptr noalias %out_sin, ptr noalias
; AMD-AVX512: callq amd_vrs8_sincosf at PLT
;
; GLIBC-LABEL: test_sincos_v8f32:
-; GLIBC: callq sincosf at PLT
-; GLIBC: callq sincosf at PLT
-; GLIBC: callq sincosf at PLT
-; GLIBC: callq sincosf at PLT
-; GLIBC: callq sincosf at PLT
-; GLIBC: callq sincosf at PLT
-; GLIBC: callq sincosf at PLT
-; GLIBC: callq sincosf at PLT
+; GLIBC: callq _ZGVdN8v_sinf at PLT
+; GLIBC: callq _ZGVdN8v_cosf at PLT
%result = call { <8 x float>, <8 x float> } @llvm.sincos.v8f32(<8 x float> %x)
%result.0 = extractvalue { <8 x float>, <8 x float> } %result, 0
%result.1 = extractvalue { <8 x float>, <8 x float> } %result, 1
@@ -107,8 +99,8 @@ define void @test_sincos_v2f64(<2 x double> %x, ptr noalias %out_sin, ptr noalia
; AMD: callq amd_vrd2_sincos at PLT
;
; GLIBC-LABEL: test_sincos_v2f64:
-; GLIBC: callq sincos at PLT
-; GLIBC: callq sincos at PLT
+; GLIBC: callq _ZGVbN2v_sin at PLT
+; GLIBC: callq _ZGVbN2v_cos at PLT
%result = call { <2 x double>, <2 x double> } @llvm.sincos.v2f64(<2 x double> %x)
%result.0 = extractvalue { <2 x double>, <2 x double> } %result, 0
%result.1 = extractvalue { <2 x double>, <2 x double> } %result, 1
@@ -133,10 +125,8 @@ define void @test_sincos_v4f64(<4 x double> %x, ptr noalias %out_sin, ptr noalia
; AMD-AVX512: callq amd_vrd4_sincos at PLT
;
; GLIBC-LABEL: test_sincos_v4f64:
-; GLIBC: callq sincos at PLT
-; GLIBC: callq sincos at PLT
-; GLIBC: callq sincos at PLT
-; GLIBC: callq sincos at PLT
+; GLIBC: callq _ZGVdN4v_sin at PLT
+; GLIBC: callq _ZGVdN4v_cos at PLT
%result = call { <4 x double>, <4 x double> } @llvm.sincos.v4f64(<4 x double> %x)
%result.0 = extractvalue { <4 x double>, <4 x double> } %result, 0
%result.1 = extractvalue { <4 x double>, <4 x double> } %result, 1
More information about the llvm-commits
mailing list