[llvm] [CodeGen] Teach ReplaceWithVeclib split vector llvm.sincos when only sin/cos veclib mappings exist (PR #194639)
Kito Cheng via llvm-commits
llvm-commits at lists.llvm.org
Wed May 6 19:06:02 PDT 2026
================
@@ -208,19 +208,116 @@ static bool replaceWithCallToVeclib(const TargetLibraryInfo &TLI,
return true;
}
+/// Returns true when \p TLI has a vector mapping for the scalar function name
+/// \p Name at \p EC (matching either masked or unmasked variants).
+static bool hasVectorMapping(const TargetLibraryInfo &TLI, StringRef Name,
+ ElementCount EC) {
+ return TLI.getVectorMappingInfo(Name, EC, /*Masked=*/false) ||
+ TLI.getVectorMappingInfo(Name, EC, /*Masked=*/true);
+}
+
+/// Returns true when \p TLI has a vector mapping for \p IID at the given
+/// element type and \p EC.
+static bool hasIntrinsicVectorMapping(const TargetLibraryInfo &TLI,
+ Intrinsic::ID IID, Type *ScalarTy,
+ ElementCount EC, Module *M) {
+ std::string Name = Intrinsic::getName(IID, {ScalarTy}, M);
+ return hasVectorMapping(TLI, Name, EC);
+}
+
+/// If \p II is a vector llvm.sincos with no direct vector library mapping but
+/// the target does have vector mappings for both llvm.sin and llvm.cos at the
+/// same element count, replace it with separate llvm.sin and llvm.cos calls
+/// and run the standard veclib replacement on each.
+static bool trySplitVectorSinCos(const TargetLibraryInfo &TLI,
+ IntrinsicInst *II,
+ SmallVectorImpl<Instruction *> &Replaced) {
+ if (II->getIntrinsicID() != Intrinsic::sincos)
+ return false;
+ Value *Arg = II->getArgOperand(0);
+ auto *VTy = dyn_cast<VectorType>(Arg->getType());
+ if (!VTy)
+ return false;
+
+ ElementCount EC = VTy->getElementCount();
+ Type *ScalarTy = VTy->getElementType();
+ Module *M = II->getModule();
+
+ // If a vector sincos mapping exists for the intrinsic name (e.g.
+ // "llvm.sincos.f32") or for the scalar libcall name ("sincos"/"sincosf"),
+ // leave the call alone -- SelectionDAG legalization will handle it via
+ // expandMultipleResultFPLibCall when the runtime libcall impl is enabled.
+ if (hasIntrinsicVectorMapping(TLI, Intrinsic::sincos, ScalarTy, EC, M))
+ return false;
+ StringRef LibcallName;
+ if (ScalarTy->isFloatTy())
+ LibcallName = "sincosf";
+ else if (ScalarTy->isDoubleTy())
+ LibcallName = "sincos";
+ if (!LibcallName.empty() && hasVectorMapping(TLI, LibcallName, EC))
+ return false;
+
+ // Splitting is only worthwhile when both sin and cos have vector mappings.
+ if (!hasIntrinsicVectorMapping(TLI, Intrinsic::sin, ScalarTy, EC, M) ||
+ !hasIntrinsicVectorMapping(TLI, Intrinsic::cos, ScalarTy, EC, M))
+ return false;
+
+ // All users must be extractvalue with index 0 or 1; otherwise we cannot
+ // safely rewire results.
+ for (User *U : II->users()) {
+ auto *EV = dyn_cast<ExtractValueInst>(U);
+ if (!EV || EV->getNumIndices() != 1 ||
+ (EV->getIndices()[0] != 0 && EV->getIndices()[0] != 1))
+ return false;
+ }
+
+ IRBuilder<> B(II);
+ Function *SinFn =
+ Intrinsic::getOrInsertDeclaration(M, Intrinsic::sin, Arg->getType());
+ Function *CosFn =
+ Intrinsic::getOrInsertDeclaration(M, Intrinsic::cos, Arg->getType());
+ CallInst *SinCall = B.CreateCall(SinFn, {Arg}, "sin");
+ CallInst *CosCall = B.CreateCall(CosFn, {Arg}, "cos");
+ SinCall->copyFastMathFlags(II);
+ CosCall->copyFastMathFlags(II);
+
+ // Forward extractvalue uses to the new calls.
+ for (User *U : llvm::make_early_inc_range(II->users())) {
+ auto *EV = cast<ExtractValueInst>(U);
+ EV->replaceAllUsesWith(EV->getIndices()[0] == 0 ? SinCall : CosCall);
+ EV->eraseFromParent();
+ }
+
+ // Replace each new call with the vector library function.
+ if (replaceWithCallToVeclib(TLI, cast<IntrinsicInst>(SinCall)))
+ Replaced.push_back(SinCall);
+ if (replaceWithCallToVeclib(TLI, cast<IntrinsicInst>(CosCall)))
+ Replaced.push_back(CosCall);
+
+ return true;
+}
+
static bool runImpl(const TargetLibraryInfo &TLI, Function &F) {
SmallVector<Instruction *> ReplacedCalls;
for (auto &I : instructions(F)) {
- // Process only intrinsic calls that return void or a vector.
- if (auto *II = dyn_cast<IntrinsicInst>(&I)) {
- if (II->getIntrinsicID() == Intrinsic::not_intrinsic)
- continue;
- if (!II->getType()->isVectorTy() && !II->getType()->isVoidTy())
- continue;
-
- if (replaceWithCallToVeclib(TLI, II))
- ReplacedCalls.push_back(&I);
+ auto *II = dyn_cast<IntrinsicInst>(&I);
+ if (!II || II->getIntrinsicID() == Intrinsic::not_intrinsic)
----------------
kito-cheng wrote:
I plan to clean those fail will trigger by this case, and the verifier part might need a RFC, so I may not going forward on that way.
Full fail list when I remove that check:
```
Failed Tests:
LLVM :: CodeGen/AMDGPU/sched.barrier.inverted.mask.ll
LLVM :: CodeGen/AMDGPU/si-split-load-store-alias-info.ll
LLVM :: CodeGen/ARM/2012-08-27-CopyPhysRegCrash.ll
LLVM :: CodeGen/X86/blend-of-shift.ll
LLVM :: CodeGen/X86/shuffle-of-shift.ll
LLVM :: DebugInfo/COFF/fortran-basic.ll
```
And fail list if I add check on verifier:
```
LLVM :: Analysis/LoopAccessAnalysis/unsafe-and-rt-checks-convergent.ll
LLVM :: Assembler/immarg-param-attribute.ll
LLVM :: Assembler/invalid-immarg.ll
LLVM :: Assembler/invalid-immarg4.ll
LLVM :: Assembler/metadata-function-local.ll
LLVM :: Assembler/metadata.ll
LLVM :: Assembler/struct-ret-without-upgrade.ll
LLVM :: Assembler/token.ll
LLVM :: Bitcode/attributes.ll
LLVM :: Bitcode/autoupgrade-convert-fp16-intrinsics-malformed.ll
LLVM :: Bitcode/bcanalyzer-types.ll
LLVM :: Bitcode/compatibility.ll
LLVM :: Bitcode/ssse3_palignr.ll
LLVM :: Bitcode/upgrade-aarch64-sve-intrinsics.ll
LLVM :: CodeGen/AArch64/arm64-vecFold.ll
LLVM :: CodeGen/AArch64/arm64-vshift.ll
LLVM :: CodeGen/AArch64/fp-fcanonicalize.ll
LLVM :: CodeGen/AArch64/i128-math.ll
LLVM :: CodeGen/AArch64/sme2-intrinsics-mlall.ll
LLVM :: CodeGen/AMDGPU/global-saddr-atomics.gfx1030.ll
LLVM :: CodeGen/AMDGPU/image-attributes.ll
LLVM :: CodeGen/AMDGPU/image-resource-id.ll
LLVM :: CodeGen/AMDGPU/llvm.amdgcn.cvt.scalef32.pk.gfx950.ll
LLVM :: CodeGen/AMDGPU/llvm.amdgcn.ds.read.tr.gfx950.ll
LLVM :: CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.atomic.buffer.load.ll
LLVM :: CodeGen/AMDGPU/llvm.amdgcn.struct.atomic.buffer.load.ll
LLVM :: CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.atomic.buffer.load.ll
LLVM :: CodeGen/AMDGPU/lower-noalias-kernargs.ll
LLVM :: CodeGen/AMDGPU/memory-legalizer-multiple-mem-operands-nontemporal-1.mir
LLVM :: CodeGen/AMDGPU/sampler-resource-id.ll
LLVM :: CodeGen/AMDGPU/sched.barrier.inverted.mask.ll
LLVM :: CodeGen/AMDGPU/shl_add_ptr_csub.ll
LLVM :: CodeGen/AMDGPU/si-split-load-store-alias-info.ll
LLVM :: CodeGen/ARM/2012-08-27-CopyPhysRegCrash.ll
LLVM :: CodeGen/ARM/crc32.ll
LLVM :: CodeGen/ARM/reg_sequence.ll
LLVM :: CodeGen/ARM/vfp.ll
LLVM :: CodeGen/Generic/2005-01-18-SetUO-InfLoop.ll
LLVM :: CodeGen/Generic/isunord.ll
LLVM :: CodeGen/Hexagon/autohvx/bitwise-pred-128b.ll
LLVM :: CodeGen/Hexagon/autohvx/ripple_scalarize_scatter.ll
LLVM :: CodeGen/Hexagon/v60-align.ll
LLVM :: CodeGen/MIR/AMDGPU/syncscopes.mir
LLVM :: CodeGen/Mips/fp-fcanonicalize.ll
LLVM :: CodeGen/NVPTX/cp-async-bulk-tensor-reduce.ll
LLVM :: CodeGen/NVPTX/unknown-intrinsic.ll
LLVM :: CodeGen/PowerPC/fp-branch.ll
LLVM :: CodeGen/PowerPC/vec_mul_even_odd.ll
LLVM :: CodeGen/PowerPC/vec_rotate_shift.ll
LLVM :: CodeGen/RISCV/GlobalISel/fp-fcanonicalize.ll
LLVM :: CodeGen/RISCV/rvv/sifive-xsfmm-vset-insert.mir
LLVM :: CodeGen/SPIRV/allow_unknown_intrinsics.ll
LLVM :: CodeGen/SPIRV/extensions/SPV_INTEL_variable_length_array/builtin_alloca.ll
LLVM :: CodeGen/SPIRV/token/token-type-requires-extension.ll
LLVM :: CodeGen/Thumb2/LowOverheadLoops/dont-remove-loop-update.mir
LLVM :: CodeGen/Thumb2/LowOverheadLoops/wlstp.mir
LLVM :: CodeGen/VE/Vector/vec_select.ll
LLVM :: CodeGen/WebAssembly/f16-intrinsics.ll
LLVM :: CodeGen/WebAssembly/simd-relaxed-fmin.ll
LLVM :: CodeGen/WinEH/wineh-cloning.ll
LLVM :: CodeGen/WinEH/wineh-demotion.ll
LLVM :: CodeGen/X86/avx10_2_512convert-intrinsics.ll
LLVM :: CodeGen/X86/avx512cfmulsh-instrinsics.ll
LLVM :: CodeGen/X86/blend-of-shift.ll
LLVM :: CodeGen/X86/compare_folding.ll
LLVM :: CodeGen/X86/isnan.ll
LLVM :: CodeGen/X86/setuge.ll
LLVM :: CodeGen/X86/shuffle-of-shift.ll
LLVM :: DebugInfo/COFF/fortran-basic.ll
LLVM :: Feature/intrinsics.ll
LLVM :: Feature/metadata.ll
LLVM :: Instrumentation/MemorySanitizer/AArch64/arm64-vshift.ll
LLVM :: Linker/2009-09-03-mdnode.ll
LLVM :: Linker/linkmdnode.ll
LLVM :: Transforms/Attributor/convergent.ll
LLVM :: Transforms/GVN/convergent.ll
LLVM :: Transforms/GVN/intrinsics_in_cg.ll
LLVM :: Transforms/GVN/pre-skip-convergent.ll
LLVM :: Transforms/GlobalOpt/metadata.ll
LLVM :: Transforms/IROutliner/alloca-addrspace-1.ll
LLVM :: Transforms/InferAddressSpaces/NVPTX/isspacep.ll
LLVM :: Transforms/InstCombine/AArch64/sme-intrinsic-opts-counting-elems.ll
LLVM :: Transforms/InstCombine/NVPTX/nvvm-intrins.ll
LLVM :: Transforms/InstCombine/X86/x86-avx2-inseltpoison.ll
LLVM :: Transforms/InstCombine/X86/x86-avx2.ll
LLVM :: Transforms/InstSimplify/ConstProp/vscale-inseltpoison.ll
LLVM :: Transforms/InstSimplify/ConstProp/vscale.ll
LLVM :: Transforms/LoopDistribute/basic-with-memchecks.ll
LLVM :: Transforms/LoopDistribute/basic.ll
LLVM :: Transforms/LoopDistribute/convergent-no-cross-partition-checks.ll
LLVM :: Transforms/LoopDistribute/diagnostics.ll
LLVM :: Transforms/LoopDistribute/scev-inserted-runtime-check.ll
LLVM :: Transforms/LoopLoadElim/convergent.ll
LLVM :: Transforms/LoopVersioning/convergent.ll
LLVM :: Transforms/LoopVersioningLICM/convergent.ll
LLVM :: Transforms/NewGVN/convergent.ll
LLVM :: Transforms/ObjCARC/ensure-that-exception-unwind-path-is-visited.ll
LLVM :: Transforms/ObjCARC/invoke-2.ll
LLVM :: Transforms/ObjCARC/move-and-form-retain-autorelease.ll
LLVM :: Transforms/ObjCARC/nested.ll
LLVM :: Transforms/Reassociate/factorize-again.ll
LLVM :: Transforms/SLPVectorizer/AArch64/mismatched-intrinsics.ll
LLVM :: Transforms/SLPVectorizer/RISCV/math-function.ll
LLVM :: Transforms/SafeStack/X86/debug-loc2.ll
LLVM :: Transforms/SeparateConstOffsetFromGEP/AMDGPU/split-gep-and-gvn-addrspace-addressing-modes.ll
LLVM :: Transforms/SimplifyCFG/2003-08-17-FoldSwitch-dbg.ll
LLVM :: Transforms/SimplifyCFG/dbginfo.ll
LLVM :: Transforms/SimplifyCFG/empty-catchpad.ll
LLVM :: Transforms/StructurizeCFG/rebuild-ssa-infinite-loop-inseltpoison.ll
LLVM :: Transforms/StructurizeCFG/rebuild-ssa-infinite-loop.ll
LLVM :: tools/llvm-reduce/operands-skip-intrinsics.ll
LLVM :: tools/llvm-reduce/reduce-instructions-token.ll
LLVM :: tools/llvm-reduce/reduce-opcodes-call.ll
LLVM :: tools/llvm-reduce/reduce-operands-skip-token.ll
LLVM :: tools/llvm-reduce/reduce-operands-to-args-token.ll
LLVM :: tools/llvm-reduce/reduce-operands.ll
LLVM :: tools/llvm-reduce/remove-attributes-from-intrinsic-like-functions.ll
LLVM :: tools/llvm-reduce/remove-unused-declarations.ll
```
https://github.com/llvm/llvm-project/pull/194639
More information about the llvm-commits
mailing list