[llvm] [CodeGen] Don't scalarize scalable math intrinsics that have a veclib mapping (PR #216309)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 14 05:20:07 PDT 2026
https://github.com/matteo-rm created https://github.com/llvm/llvm-project/pull/216309
`PreISelIntrinsicLowering` scalarizes scalable vector math intrinsics whenever the operation is marked as Expand. This occurs even when the vector library implements a vectorized version of the operation. This PR adds a check to avoid scalarizing if the intrinsic is mapped by `ReplaceWithVeclib` later.
Example: compiling the following with `clang -O3 -march=armv8-a+sve -fveclib=SLEEF -S example.ll`
```llvm
define <vscale x 4 x float> @entry(<vscale x 4 x float> %0) {
%2 = call <vscale x 4 x float> @llvm.exp.nxv4f32(<vscale x 4 x float> %0)
ret <vscale x 4 x float> %2
}
declare <vscale x 4 x float> @llvm.exp.nxv4f32(<vscale x 4 x float>)
```
Produces scalarized code where a call to `_ZGVsMxv_expf` could instead be inserted.
>From 72d9d47373bd3e95d0959157dab80cbcc35c4416 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matt=C3=A9o=20Rizza=20Murgier?=
<matteo.rizza-murgier at sipearl.com>
Date: Thu, 13 Aug 2026 16:54:59 +0200
Subject: [PATCH 1/2] [CodeGen] Don't scalarize scalable math intr that have a
veclib mapping
---
llvm/include/llvm/CodeGen/ReplaceWithVeclib.h | 15 ++++++++++++
llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp | 7 ++++++
llvm/lib/CodeGen/ReplaceWithVeclib.cpp | 8 +++----
llvm/test/CodeGen/AArch64/veclib-llvm.exp.ll | 23 +++++++++++++++++++
4 files changed, 48 insertions(+), 5 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/veclib-llvm.exp.ll
diff --git a/llvm/include/llvm/CodeGen/ReplaceWithVeclib.h b/llvm/include/llvm/CodeGen/ReplaceWithVeclib.h
index d65a6d7a33df6..3baf5283fe2fd 100644
--- a/llvm/include/llvm/CodeGen/ReplaceWithVeclib.h
+++ b/llvm/include/llvm/CodeGen/ReplaceWithVeclib.h
@@ -18,9 +18,24 @@
#include "llvm/Pass.h"
#include "llvm/PassRegistry.h"
#include "llvm/Support/Compiler.h"
+#include "llvm/Support/TypeSize.h"
namespace llvm {
class Function;
+class Module;
+class TargetLibraryInfo;
+class Type;
+
+namespace Intrinsic {
+typedef unsigned ID;
+}
+
+/// Returns true when \p TLI has a vector mapping for \p IID at the given
+/// element type and \p EC.
+LLVM_ABI bool hasIntrinsicVectorMapping(const TargetLibraryInfo &TLI,
+ Intrinsic::ID IID, Type *ScalarTy,
+ ElementCount EC, Module *M);
+
struct ReplaceWithVeclib : public RequiredPassInfoMixin<ReplaceWithVeclib> {
LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
};
diff --git a/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp b/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp
index 4b5c97a23b091..aa9eedc8a7336 100644
--- a/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp
+++ b/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp
@@ -20,6 +20,7 @@
#include "llvm/CodeGen/ExpandVectorPredication.h"
#include "llvm/CodeGen/LibcallLoweringInfo.h"
#include "llvm/CodeGen/Passes.h"
+#include "llvm/CodeGen/ReplaceWithVeclib.h"
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/CodeGen/TargetPassConfig.h"
#include "llvm/IR/Function.h"
@@ -800,6 +801,12 @@ bool PreISelIntrinsicLowering::lowerIntrinsics(Module &M) const {
Type *Ty = CI->getArgOperand(0)->getType();
if (!TM || !isa<ScalableVectorType>(Ty))
return false;
+ // Avoid scalarizing if the vector library provides an implementation.
+ auto *VTy = cast<VectorType>(Ty);
+ if (hasIntrinsicVectorMapping(LookupTLI(*CI->getFunction()),
+ F.getIntrinsicID(), VTy->getElementType(),
+ VTy->getElementCount(), &M))
+ return false;
const TargetLowering *TL = TM->getSubtargetImpl(F)->getTargetLowering();
unsigned Op = TL->IntrinsicIDToISD(F.getIntrinsicID());
assert(Op != ISD::DELETED_NODE && "unsupported intrinsic");
diff --git a/llvm/lib/CodeGen/ReplaceWithVeclib.cpp b/llvm/lib/CodeGen/ReplaceWithVeclib.cpp
index 4a240c914a252..889472639ec12 100644
--- a/llvm/lib/CodeGen/ReplaceWithVeclib.cpp
+++ b/llvm/lib/CodeGen/ReplaceWithVeclib.cpp
@@ -218,11 +218,9 @@ static bool hasVectorMapping(const TargetLibraryInfo &TLI, StringRef Name,
TLI.getVectorMappingInfo(Name, EC, /*Masked=*/true);
}
-/// Returns true when \p TLI has a vector mapping for \p IID at the given
-/// element type and \p EC.
-static bool hasIntrinsicVectorMapping(const TargetLibraryInfo &TLI,
- Intrinsic::ID IID, Type *ScalarTy,
- ElementCount EC, Module *M) {
+bool llvm::hasIntrinsicVectorMapping(const TargetLibraryInfo &TLI,
+ Intrinsic::ID IID, Type *ScalarTy,
+ ElementCount EC, Module *M) {
std::string Name = Intrinsic::getName(IID, {ScalarTy}, M);
return hasVectorMapping(TLI, Name, EC);
}
diff --git a/llvm/test/CodeGen/AArch64/veclib-llvm.exp.ll b/llvm/test/CodeGen/AArch64/veclib-llvm.exp.ll
new file mode 100644
index 0000000000000..01f9dd1bf181f
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/veclib-llvm.exp.ll
@@ -0,0 +1,23 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter "bl" --version 6
+; RUN: llc -mtriple=aarch64-gnu-linux -mattr=+neon,+sve -vector-library=sleefgnuabi < %s | FileCheck %s -check-prefix=SLEEF
+; RUN: llc -mtriple=aarch64-gnu-linux -mattr=+neon,+sve -vector-library=ArmPL < %s | FileCheck %s -check-prefix=ARMPL
+
+define <vscale x 4 x float> @test_exp_nxv4f32(<vscale x 4 x float> %x) {
+; SLEEF-LABEL: test_exp_nxv4f32:
+; SLEEF: bl _ZGVsMxv_expf
+;
+; ARMPL-LABEL: test_exp_nxv4f32:
+; ARMPL: bl armpl_svexp_f32_x
+ %r = call <vscale x 4 x float> @llvm.exp.nxv4f32(<vscale x 4 x float> %x)
+ ret <vscale x 4 x float> %r
+}
+
+define <4 x float> @test_exp_v4f32(<4 x float> %x) {
+; SLEEF-LABEL: test_exp_v4f32:
+; SLEEF: bl _ZGVnN4v_expf
+;
+; ARMPL-LABEL: test_exp_v4f32:
+; ARMPL: bl armpl_vexpq_f32
+ %r = call <4 x float> @llvm.exp.v4f32(<4 x float> %x)
+ ret <4 x float> %r
+}
>From c565fec48515089f553728dcae1472c0b8db7411 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matt=C3=A9o=20Rizza=20Murgier?=
<matteo.rizza-murgier at sipearl.com>
Date: Fri, 14 Aug 2026 14:13:04 +0200
Subject: [PATCH 2/2] [CodeGen] Only skip scalarization if veclib pass is run
---
llvm/include/llvm/CodeGen/Passes.h | 5 +++-
llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp | 30 ++++++++++++-------
llvm/lib/CodeGen/TargetPassConfig.cpp | 3 +-
llvm/test/CodeGen/AArch64/veclib-llvm.exp.ll | 10 +++++++
4 files changed, 36 insertions(+), 12 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/Passes.h b/llvm/include/llvm/CodeGen/Passes.h
index 86bf5aea7b23a..1a570b6069841 100644
--- a/llvm/include/llvm/CodeGen/Passes.h
+++ b/llvm/include/llvm/CodeGen/Passes.h
@@ -508,7 +508,10 @@ LLVM_ABI ModulePass *createLibcallLoweringInfoWrapper();
/// instructions. This is unsafe to do earlier because a pass may combine the
/// constant initializer into the load, which may result in an overflowing
/// evaluation.
-LLVM_ABI ModulePass *createPreISelIntrinsicLoweringPass();
+/// If \p UseVeclibCalls is true, vector math intrinsics that have a vector
+/// library mapping are left for ReplaceWithVeclib instead of being expanded.
+LLVM_ABI ModulePass *
+createPreISelIntrinsicLoweringPass(bool UseVeclibCalls = false);
/// GlobalMerge - This pass merges internal (by default) globals into structs
/// to enable reuse of a base pointer by indexed addressing modes.
diff --git a/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp b/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp
index aa9eedc8a7336..e26e95a0b08e1 100644
--- a/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp
+++ b/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp
@@ -64,20 +64,24 @@ struct PreISelIntrinsicLowering {
const function_ref<TargetTransformInfo &(Function &)> LookupTTI;
const function_ref<TargetLibraryInfo &(Function &)> LookupTLI;
- /// If this is true, assume it's preferably to leave memory intrinsic calls
+ /// If this is true, assume it's preferable to leave memory intrinsic calls
/// for replacement with a library call later. Otherwise this depends on
/// TargetLoweringInfo availability of the corresponding function.
const bool UseMemIntrinsicLibFunc;
+ /// If this is true, assume ReplaceWithVeclib will run later and leave vector
+ /// math intrinsics that have a vector library mapping for it to lower.
+ const bool UseVeclibCalls;
+
explicit PreISelIntrinsicLowering(
const TargetMachine *TM_,
const ModuleLibcallLoweringInfo &ModuleLibcalls_,
function_ref<TargetTransformInfo &(Function &)> LookupTTI_,
function_ref<TargetLibraryInfo &(Function &)> LookupTLI_,
- bool UseMemIntrinsicLibFunc_ = true)
+ bool UseMemIntrinsicLibFunc_ = true, bool UseVeclibCalls_ = false)
: TM(TM_), ModuleLibcalls(ModuleLibcalls_), LookupTTI(LookupTTI_),
- LookupTLI(LookupTLI_), UseMemIntrinsicLibFunc(UseMemIntrinsicLibFunc_) {
- }
+ LookupTLI(LookupTLI_), UseMemIntrinsicLibFunc(UseMemIntrinsicLibFunc_),
+ UseVeclibCalls(UseVeclibCalls_) {}
static bool shouldExpandMemIntrinsicWithSize(Value *Size,
const TargetTransformInfo &TTI);
@@ -801,9 +805,11 @@ bool PreISelIntrinsicLowering::lowerIntrinsics(Module &M) const {
Type *Ty = CI->getArgOperand(0)->getType();
if (!TM || !isa<ScalableVectorType>(Ty))
return false;
- // Avoid scalarizing if the vector library provides an implementation.
+ // Avoid scalarizing if the vector library provides an implementation
+ // and ReplaceWithVeclib is run.
auto *VTy = cast<VectorType>(Ty);
- if (hasIntrinsicVectorMapping(LookupTLI(*CI->getFunction()),
+ if (UseVeclibCalls &&
+ hasIntrinsicVectorMapping(LookupTLI(*CI->getFunction()),
F.getIntrinsicID(), VTy->getElementType(),
VTy->getElementCount(), &M))
return false;
@@ -843,7 +849,9 @@ class PreISelIntrinsicLoweringLegacyPass : public ModulePass {
public:
static char ID;
- PreISelIntrinsicLoweringLegacyPass() : ModulePass(ID) {}
+ const bool UseVeclibCalls;
+ explicit PreISelIntrinsicLoweringLegacyPass(bool UseVeclibCalls_ = false)
+ : ModulePass(ID), UseVeclibCalls(UseVeclibCalls_) {}
void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.addRequired<TargetTransformInfoWrapperPass>();
@@ -864,7 +872,9 @@ class PreISelIntrinsicLoweringLegacyPass : public ModulePass {
};
const auto *TM = &getAnalysis<TargetPassConfig>().getTM<TargetMachine>();
- PreISelIntrinsicLowering Lowering(TM, ModuleLibcalls, LookupTTI, LookupTLI);
+ PreISelIntrinsicLowering Lowering(TM, ModuleLibcalls, LookupTTI, LookupTLI,
+ /*UseMemIntrinsicLibFunc=*/true,
+ UseVeclibCalls);
return Lowering.lowerIntrinsics(M);
}
};
@@ -885,8 +895,8 @@ INITIALIZE_PASS_END(PreISelIntrinsicLoweringLegacyPass,
"pre-isel-intrinsic-lowering",
"Pre-ISel Intrinsic Lowering", false, false)
-ModulePass *llvm::createPreISelIntrinsicLoweringPass() {
- return new PreISelIntrinsicLoweringLegacyPass();
+ModulePass *llvm::createPreISelIntrinsicLoweringPass(bool UseVeclibCalls) {
+ return new PreISelIntrinsicLoweringLegacyPass(UseVeclibCalls);
}
PreservedAnalyses
diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp
index 4996f5517fc0a..538c9dc18f10e 100644
--- a/llvm/lib/CodeGen/TargetPassConfig.cpp
+++ b/llvm/lib/CodeGen/TargetPassConfig.cpp
@@ -1089,7 +1089,8 @@ bool TargetPassConfig::addISelPasses() {
// PreISelIntrinsicLowering.
if (getOptLevel() != CodeGenOptLevel::None)
addPass(createObjCARCContractPass());
- addPass(createPreISelIntrinsicLoweringPass());
+ addPass(createPreISelIntrinsicLoweringPass(
+ getOptLevel() != CodeGenOptLevel::None && !DisableReplaceWithVecLib));
addPass(createExpandIRInstsPass(getOptLevel()));
addIRPasses();
diff --git a/llvm/test/CodeGen/AArch64/veclib-llvm.exp.ll b/llvm/test/CodeGen/AArch64/veclib-llvm.exp.ll
index 01f9dd1bf181f..2cc4cd94d19fb 100644
--- a/llvm/test/CodeGen/AArch64/veclib-llvm.exp.ll
+++ b/llvm/test/CodeGen/AArch64/veclib-llvm.exp.ll
@@ -1,6 +1,7 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter "bl" --version 6
; RUN: llc -mtriple=aarch64-gnu-linux -mattr=+neon,+sve -vector-library=sleefgnuabi < %s | FileCheck %s -check-prefix=SLEEF
; RUN: llc -mtriple=aarch64-gnu-linux -mattr=+neon,+sve -vector-library=ArmPL < %s | FileCheck %s -check-prefix=ARMPL
+; RUN: llc -O0 -mtriple=aarch64-gnu-linux -mattr=+neon,+sve -vector-library=sleefgnuabi < %s | FileCheck %s -check-prefix=O0
define <vscale x 4 x float> @test_exp_nxv4f32(<vscale x 4 x float> %x) {
; SLEEF-LABEL: test_exp_nxv4f32:
@@ -8,6 +9,9 @@ define <vscale x 4 x float> @test_exp_nxv4f32(<vscale x 4 x float> %x) {
;
; ARMPL-LABEL: test_exp_nxv4f32:
; ARMPL: bl armpl_svexp_f32_x
+;
+; O0-LABEL: test_exp_nxv4f32:
+; O0: bl expf
%r = call <vscale x 4 x float> @llvm.exp.nxv4f32(<vscale x 4 x float> %x)
ret <vscale x 4 x float> %r
}
@@ -18,6 +22,12 @@ define <4 x float> @test_exp_v4f32(<4 x float> %x) {
;
; ARMPL-LABEL: test_exp_v4f32:
; ARMPL: bl armpl_vexpq_f32
+;
+; O0-LABEL: test_exp_v4f32:
+; O0: bl expf
+; O0: bl expf
+; O0: bl expf
+; O0: bl expf
%r = call <4 x float> @llvm.exp.v4f32(<4 x float> %x)
ret <4 x float> %r
}
More information about the llvm-commits
mailing list