[llvm] [CodeGen] Don't scalarize scalable math intrinsics that have a veclib mapping (PR #216309)

via llvm-commits llvm-commits at lists.llvm.org
Fri Aug 14 05:20:07 PDT 2026


https://github.com/matteo-rm created https://github.com/llvm/llvm-project/pull/216309

`PreISelIntrinsicLowering` scalarizes scalable vector math intrinsics whenever the operation is marked as Expand. This occurs even when the vector library implements a vectorized version of the operation. This PR adds a check to avoid scalarizing if the intrinsic is mapped by `ReplaceWithVeclib` later.

Example: compiling the following with `clang -O3 -march=armv8-a+sve -fveclib=SLEEF -S example.ll`

```llvm
define <vscale x 4 x float> @entry(<vscale x 4 x float> %0) {
  %2 = call <vscale x 4 x float> @llvm.exp.nxv4f32(<vscale x 4 x float> %0)
  ret <vscale x 4 x float> %2
}

declare <vscale x 4 x float> @llvm.exp.nxv4f32(<vscale x 4 x float>)
```

Produces scalarized code where a call to `_ZGVsMxv_expf` could instead be inserted.

>From 72d9d47373bd3e95d0959157dab80cbcc35c4416 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matt=C3=A9o=20Rizza=20Murgier?=
 <matteo.rizza-murgier at sipearl.com>
Date: Thu, 13 Aug 2026 16:54:59 +0200
Subject: [PATCH 1/2] [CodeGen] Don't scalarize scalable math intr that have a
 veclib mapping

---
 llvm/include/llvm/CodeGen/ReplaceWithVeclib.h | 15 ++++++++++++
 llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp |  7 ++++++
 llvm/lib/CodeGen/ReplaceWithVeclib.cpp        |  8 +++----
 llvm/test/CodeGen/AArch64/veclib-llvm.exp.ll  | 23 +++++++++++++++++++
 4 files changed, 48 insertions(+), 5 deletions(-)
 create mode 100644 llvm/test/CodeGen/AArch64/veclib-llvm.exp.ll

diff --git a/llvm/include/llvm/CodeGen/ReplaceWithVeclib.h b/llvm/include/llvm/CodeGen/ReplaceWithVeclib.h
index d65a6d7a33df6..3baf5283fe2fd 100644
--- a/llvm/include/llvm/CodeGen/ReplaceWithVeclib.h
+++ b/llvm/include/llvm/CodeGen/ReplaceWithVeclib.h
@@ -18,9 +18,24 @@
 #include "llvm/Pass.h"
 #include "llvm/PassRegistry.h"
 #include "llvm/Support/Compiler.h"
+#include "llvm/Support/TypeSize.h"
 
 namespace llvm {
 class Function;
+class Module;
+class TargetLibraryInfo;
+class Type;
+
+namespace Intrinsic {
+typedef unsigned ID;
+}
+
+/// Returns true when \p TLI has a vector mapping for \p IID at the given
+/// element type and \p EC.
+LLVM_ABI bool hasIntrinsicVectorMapping(const TargetLibraryInfo &TLI,
+                                        Intrinsic::ID IID, Type *ScalarTy,
+                                        ElementCount EC, Module *M);
+
 struct ReplaceWithVeclib : public RequiredPassInfoMixin<ReplaceWithVeclib> {
   LLVM_ABI PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
 };
diff --git a/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp b/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp
index 4b5c97a23b091..aa9eedc8a7336 100644
--- a/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp
+++ b/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp
@@ -20,6 +20,7 @@
 #include "llvm/CodeGen/ExpandVectorPredication.h"
 #include "llvm/CodeGen/LibcallLoweringInfo.h"
 #include "llvm/CodeGen/Passes.h"
+#include "llvm/CodeGen/ReplaceWithVeclib.h"
 #include "llvm/CodeGen/TargetLowering.h"
 #include "llvm/CodeGen/TargetPassConfig.h"
 #include "llvm/IR/Function.h"
@@ -800,6 +801,12 @@ bool PreISelIntrinsicLowering::lowerIntrinsics(Module &M) const {
         Type *Ty = CI->getArgOperand(0)->getType();
         if (!TM || !isa<ScalableVectorType>(Ty))
           return false;
+        // Avoid scalarizing if the vector library provides an implementation.
+        auto *VTy = cast<VectorType>(Ty);
+        if (hasIntrinsicVectorMapping(LookupTLI(*CI->getFunction()),
+                                      F.getIntrinsicID(), VTy->getElementType(),
+                                      VTy->getElementCount(), &M))
+          return false;
         const TargetLowering *TL = TM->getSubtargetImpl(F)->getTargetLowering();
         unsigned Op = TL->IntrinsicIDToISD(F.getIntrinsicID());
         assert(Op != ISD::DELETED_NODE && "unsupported intrinsic");
diff --git a/llvm/lib/CodeGen/ReplaceWithVeclib.cpp b/llvm/lib/CodeGen/ReplaceWithVeclib.cpp
index 4a240c914a252..889472639ec12 100644
--- a/llvm/lib/CodeGen/ReplaceWithVeclib.cpp
+++ b/llvm/lib/CodeGen/ReplaceWithVeclib.cpp
@@ -218,11 +218,9 @@ static bool hasVectorMapping(const TargetLibraryInfo &TLI, StringRef Name,
          TLI.getVectorMappingInfo(Name, EC, /*Masked=*/true);
 }
 
-/// Returns true when \p TLI has a vector mapping for \p IID at the given
-/// element type and \p EC.
-static bool hasIntrinsicVectorMapping(const TargetLibraryInfo &TLI,
-                                      Intrinsic::ID IID, Type *ScalarTy,
-                                      ElementCount EC, Module *M) {
+bool llvm::hasIntrinsicVectorMapping(const TargetLibraryInfo &TLI,
+                                     Intrinsic::ID IID, Type *ScalarTy,
+                                     ElementCount EC, Module *M) {
   std::string Name = Intrinsic::getName(IID, {ScalarTy}, M);
   return hasVectorMapping(TLI, Name, EC);
 }
diff --git a/llvm/test/CodeGen/AArch64/veclib-llvm.exp.ll b/llvm/test/CodeGen/AArch64/veclib-llvm.exp.ll
new file mode 100644
index 0000000000000..01f9dd1bf181f
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/veclib-llvm.exp.ll
@@ -0,0 +1,23 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter "bl" --version 6
+; RUN: llc -mtriple=aarch64-gnu-linux -mattr=+neon,+sve -vector-library=sleefgnuabi < %s | FileCheck %s -check-prefix=SLEEF
+; RUN: llc -mtriple=aarch64-gnu-linux -mattr=+neon,+sve -vector-library=ArmPL < %s | FileCheck %s -check-prefix=ARMPL
+
+define <vscale x 4 x float> @test_exp_nxv4f32(<vscale x 4 x float> %x) {
+; SLEEF-LABEL: test_exp_nxv4f32:
+; SLEEF:    bl _ZGVsMxv_expf
+;
+; ARMPL-LABEL: test_exp_nxv4f32:
+; ARMPL:    bl armpl_svexp_f32_x
+  %r = call <vscale x 4 x float> @llvm.exp.nxv4f32(<vscale x 4 x float> %x)
+  ret <vscale x 4 x float> %r
+}
+
+define <4 x float> @test_exp_v4f32(<4 x float> %x) {
+; SLEEF-LABEL: test_exp_v4f32:
+; SLEEF:    bl _ZGVnN4v_expf
+;
+; ARMPL-LABEL: test_exp_v4f32:
+; ARMPL:    bl armpl_vexpq_f32
+  %r = call <4 x float> @llvm.exp.v4f32(<4 x float> %x)
+  ret <4 x float> %r
+}

>From c565fec48515089f553728dcae1472c0b8db7411 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Matt=C3=A9o=20Rizza=20Murgier?=
 <matteo.rizza-murgier at sipearl.com>
Date: Fri, 14 Aug 2026 14:13:04 +0200
Subject: [PATCH 2/2] [CodeGen] Only skip scalarization if veclib pass is run

---
 llvm/include/llvm/CodeGen/Passes.h            |  5 +++-
 llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp | 30 ++++++++++++-------
 llvm/lib/CodeGen/TargetPassConfig.cpp         |  3 +-
 llvm/test/CodeGen/AArch64/veclib-llvm.exp.ll  | 10 +++++++
 4 files changed, 36 insertions(+), 12 deletions(-)

diff --git a/llvm/include/llvm/CodeGen/Passes.h b/llvm/include/llvm/CodeGen/Passes.h
index 86bf5aea7b23a..1a570b6069841 100644
--- a/llvm/include/llvm/CodeGen/Passes.h
+++ b/llvm/include/llvm/CodeGen/Passes.h
@@ -508,7 +508,10 @@ LLVM_ABI ModulePass *createLibcallLoweringInfoWrapper();
 /// instructions.  This is unsafe to do earlier because a pass may combine the
 /// constant initializer into the load, which may result in an overflowing
 /// evaluation.
-LLVM_ABI ModulePass *createPreISelIntrinsicLoweringPass();
+/// If \p UseVeclibCalls is true, vector math intrinsics that have a vector
+/// library mapping are left for ReplaceWithVeclib instead of being expanded.
+LLVM_ABI ModulePass *
+createPreISelIntrinsicLoweringPass(bool UseVeclibCalls = false);
 
 /// GlobalMerge - This pass merges internal (by default) globals into structs
 /// to enable reuse of a base pointer by indexed addressing modes.
diff --git a/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp b/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp
index aa9eedc8a7336..e26e95a0b08e1 100644
--- a/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp
+++ b/llvm/lib/CodeGen/PreISelIntrinsicLowering.cpp
@@ -64,20 +64,24 @@ struct PreISelIntrinsicLowering {
   const function_ref<TargetTransformInfo &(Function &)> LookupTTI;
   const function_ref<TargetLibraryInfo &(Function &)> LookupTLI;
 
-  /// If this is true, assume it's preferably to leave memory intrinsic calls
+  /// If this is true, assume it's preferable to leave memory intrinsic calls
   /// for replacement with a library call later. Otherwise this depends on
   /// TargetLoweringInfo availability of the corresponding function.
   const bool UseMemIntrinsicLibFunc;
 
+  /// If this is true, assume ReplaceWithVeclib will run later and leave vector
+  /// math intrinsics that have a vector library mapping for it to lower.
+  const bool UseVeclibCalls;
+
   explicit PreISelIntrinsicLowering(
       const TargetMachine *TM_,
       const ModuleLibcallLoweringInfo &ModuleLibcalls_,
       function_ref<TargetTransformInfo &(Function &)> LookupTTI_,
       function_ref<TargetLibraryInfo &(Function &)> LookupTLI_,
-      bool UseMemIntrinsicLibFunc_ = true)
+      bool UseMemIntrinsicLibFunc_ = true, bool UseVeclibCalls_ = false)
       : TM(TM_), ModuleLibcalls(ModuleLibcalls_), LookupTTI(LookupTTI_),
-        LookupTLI(LookupTLI_), UseMemIntrinsicLibFunc(UseMemIntrinsicLibFunc_) {
-  }
+        LookupTLI(LookupTLI_), UseMemIntrinsicLibFunc(UseMemIntrinsicLibFunc_),
+        UseVeclibCalls(UseVeclibCalls_) {}
 
   static bool shouldExpandMemIntrinsicWithSize(Value *Size,
                                                const TargetTransformInfo &TTI);
@@ -801,9 +805,11 @@ bool PreISelIntrinsicLowering::lowerIntrinsics(Module &M) const {
         Type *Ty = CI->getArgOperand(0)->getType();
         if (!TM || !isa<ScalableVectorType>(Ty))
           return false;
-        // Avoid scalarizing if the vector library provides an implementation.
+        // Avoid scalarizing if the vector library provides an implementation
+        // and ReplaceWithVeclib is run.
         auto *VTy = cast<VectorType>(Ty);
-        if (hasIntrinsicVectorMapping(LookupTLI(*CI->getFunction()),
+        if (UseVeclibCalls &&
+            hasIntrinsicVectorMapping(LookupTLI(*CI->getFunction()),
                                       F.getIntrinsicID(), VTy->getElementType(),
                                       VTy->getElementCount(), &M))
           return false;
@@ -843,7 +849,9 @@ class PreISelIntrinsicLoweringLegacyPass : public ModulePass {
 public:
   static char ID;
 
-  PreISelIntrinsicLoweringLegacyPass() : ModulePass(ID) {}
+  const bool UseVeclibCalls;
+  explicit PreISelIntrinsicLoweringLegacyPass(bool UseVeclibCalls_ = false)
+      : ModulePass(ID), UseVeclibCalls(UseVeclibCalls_) {}
 
   void getAnalysisUsage(AnalysisUsage &AU) const override {
     AU.addRequired<TargetTransformInfoWrapperPass>();
@@ -864,7 +872,9 @@ class PreISelIntrinsicLoweringLegacyPass : public ModulePass {
     };
 
     const auto *TM = &getAnalysis<TargetPassConfig>().getTM<TargetMachine>();
-    PreISelIntrinsicLowering Lowering(TM, ModuleLibcalls, LookupTTI, LookupTLI);
+    PreISelIntrinsicLowering Lowering(TM, ModuleLibcalls, LookupTTI, LookupTLI,
+                                      /*UseMemIntrinsicLibFunc=*/true,
+                                      UseVeclibCalls);
     return Lowering.lowerIntrinsics(M);
   }
 };
@@ -885,8 +895,8 @@ INITIALIZE_PASS_END(PreISelIntrinsicLoweringLegacyPass,
                     "pre-isel-intrinsic-lowering",
                     "Pre-ISel Intrinsic Lowering", false, false)
 
-ModulePass *llvm::createPreISelIntrinsicLoweringPass() {
-  return new PreISelIntrinsicLoweringLegacyPass();
+ModulePass *llvm::createPreISelIntrinsicLoweringPass(bool UseVeclibCalls) {
+  return new PreISelIntrinsicLoweringLegacyPass(UseVeclibCalls);
 }
 
 PreservedAnalyses
diff --git a/llvm/lib/CodeGen/TargetPassConfig.cpp b/llvm/lib/CodeGen/TargetPassConfig.cpp
index 4996f5517fc0a..538c9dc18f10e 100644
--- a/llvm/lib/CodeGen/TargetPassConfig.cpp
+++ b/llvm/lib/CodeGen/TargetPassConfig.cpp
@@ -1089,7 +1089,8 @@ bool TargetPassConfig::addISelPasses() {
   // PreISelIntrinsicLowering.
   if (getOptLevel() != CodeGenOptLevel::None)
     addPass(createObjCARCContractPass());
-  addPass(createPreISelIntrinsicLoweringPass());
+  addPass(createPreISelIntrinsicLoweringPass(
+      getOptLevel() != CodeGenOptLevel::None && !DisableReplaceWithVecLib));
   addPass(createExpandIRInstsPass(getOptLevel()));
   addIRPasses();
 
diff --git a/llvm/test/CodeGen/AArch64/veclib-llvm.exp.ll b/llvm/test/CodeGen/AArch64/veclib-llvm.exp.ll
index 01f9dd1bf181f..2cc4cd94d19fb 100644
--- a/llvm/test/CodeGen/AArch64/veclib-llvm.exp.ll
+++ b/llvm/test/CodeGen/AArch64/veclib-llvm.exp.ll
@@ -1,6 +1,7 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter "bl" --version 6
 ; RUN: llc -mtriple=aarch64-gnu-linux -mattr=+neon,+sve -vector-library=sleefgnuabi < %s | FileCheck %s -check-prefix=SLEEF
 ; RUN: llc -mtriple=aarch64-gnu-linux -mattr=+neon,+sve -vector-library=ArmPL < %s | FileCheck %s -check-prefix=ARMPL
+; RUN: llc -O0 -mtriple=aarch64-gnu-linux -mattr=+neon,+sve -vector-library=sleefgnuabi < %s | FileCheck %s -check-prefix=O0
 
 define <vscale x 4 x float> @test_exp_nxv4f32(<vscale x 4 x float> %x) {
 ; SLEEF-LABEL: test_exp_nxv4f32:
@@ -8,6 +9,9 @@ define <vscale x 4 x float> @test_exp_nxv4f32(<vscale x 4 x float> %x) {
 ;
 ; ARMPL-LABEL: test_exp_nxv4f32:
 ; ARMPL:    bl armpl_svexp_f32_x
+;
+; O0-LABEL: test_exp_nxv4f32:
+; O0:    bl expf
   %r = call <vscale x 4 x float> @llvm.exp.nxv4f32(<vscale x 4 x float> %x)
   ret <vscale x 4 x float> %r
 }
@@ -18,6 +22,12 @@ define <4 x float> @test_exp_v4f32(<4 x float> %x) {
 ;
 ; ARMPL-LABEL: test_exp_v4f32:
 ; ARMPL:    bl armpl_vexpq_f32
+;
+; O0-LABEL: test_exp_v4f32:
+; O0:    bl expf
+; O0:    bl expf
+; O0:    bl expf
+; O0:    bl expf
   %r = call <4 x float> @llvm.exp.v4f32(<4 x float> %x)
   ret <4 x float> %r
 }



More information about the llvm-commits mailing list