[llvm] [BPF] Handle certain mem intrinsic functions with addr-space arguments (PR #160025)

via llvm-commits llvm-commits at lists.llvm.org
Sun Sep 21 20:40:29 PDT 2025


https://github.com/yonghong-song updated https://github.com/llvm/llvm-project/pull/160025

>From 3afb1d3142e2f276a997047eb82f885303903f69 Mon Sep 17 00:00:00 2001
From: Yonghong Song <yonghong.song at linux.dev>
Date: Sun, 21 Sep 2025 12:23:12 -0700
Subject: [PATCH] [BPF] Handle certain mem intrinsic functions with addr-space
 arguments

In linux kernel commit [1], we have a bpf selftest failure caused by llvm.
In this particular case, the BPFCheckAndAdjustIR pass has a function
insertASpaceCasts() which inserts proper addrspacecast insn at proper IR
places. It does not handle __builtin_memset() and hance caused selftest
failure.

Add support in insertASpaceCasts() to handle __builtin_(memset,memcpy,memmove}()
properly and this can fix the issue in [1] as well.

  [1] https://lore.kernel.org/all/20250920045805.3288551-1-yonghong.song@linux.dev/
---
 llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp   | 104 ++++++++++++++++--
 .../BPF/addr-space-memintrinsic-gep.ll        |  38 +++++++
 .../BPF/addr-space-memintrinsic-no-gep.ll     |  27 +++++
 3 files changed, 162 insertions(+), 7 deletions(-)
 create mode 100644 llvm/test/CodeGen/BPF/addr-space-memintrinsic-gep.ll
 create mode 100644 llvm/test/CodeGen/BPF/addr-space-memintrinsic-no-gep.ll

diff --git a/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp b/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp
index b202b20291aff..464123185c0ae 100644
--- a/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp
+++ b/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp
@@ -26,6 +26,7 @@
 #include "llvm/IR/IRBuilder.h"
 #include "llvm/IR/Instruction.h"
 #include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
 #include "llvm/IR/IntrinsicsBPF.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/Type.h"
@@ -478,9 +479,57 @@ static void aspaceWrapOperand(DenseMap<Value *, Value *> &Cache, Instruction *I,
   }
 }
 
+static Instruction *aspaceMemIntrinsic(DenseMap<Value *, Value *> &Cache,
+                                       Instruction *I, bool IsSet, bool IsCpy) {
+  auto *CI = cast<CallInst>(I);
+
+  auto wrapPtr = [&](Value *P) -> Value * {
+    if (auto *PTy = dyn_cast<PointerType>(P->getType())) {
+      if (PTy->getAddressSpace() == 0)
+        return P;
+    }
+    return aspaceWrapValue(Cache, CI->getFunction(), P);
+  };
+
+  auto *MI = cast<MemIntrinsic>(I);
+  IRBuilder<> B(I);
+
+  if (IsSet) {
+    // memset(dst, val, len, align, isvolatile, md)
+    Value *Dst = wrapPtr(CI->getArgOperand(0));
+    Value *Val = CI->getArgOperand(1);
+    Value *Len = CI->getArgOperand(2);
+
+    auto *MS = cast<MemSetInst>(I);
+    MaybeAlign Align = MS->getDestAlign();
+    bool IsVolatile = MS->isVolatile();
+
+    return B.CreateMemSet(Dst, Val, Len, Align, IsVolatile,
+                          MI->getAAMetadata());
+  }
+
+  // memcpy/memmove(dst, dst_align, src, src_align, len, isvolatile, md)
+  Value *Dst = wrapPtr(CI->getArgOperand(0));
+  Value *Src = wrapPtr(CI->getArgOperand(1));
+  Value *Len = CI->getArgOperand(2);
+
+  auto *MT = cast<MemTransferInst>(I);
+  MaybeAlign DstAlign = MT->getDestAlign();
+  MaybeAlign SrcAlign = MT->getSourceAlign();
+  bool IsVolatile = MT->isVolatile();
+
+  Instruction *New = IsCpy ? B.CreateMemCpy(Dst, DstAlign, Src, SrcAlign, Len,
+                                            IsVolatile, MI->getAAMetadata())
+                           : B.CreateMemMove(Dst, DstAlign, Src, SrcAlign, Len,
+                                             IsVolatile, MI->getAAMetadata());
+
+  return New;
+}
+
 // Support for BPF address spaces:
 // - for each function in the module M, update pointer operand of
 //   each memory access instruction (load/store/cmpxchg/atomicrmw)
+//   or intrinsic call insns (memset/memcpy/memmove)
 //   by casting it from non-zero address space to zero address space, e.g:
 //
 //   (load (ptr addrspace (N) %p) ...)
@@ -493,21 +542,62 @@ bool BPFCheckAndAdjustIR::insertASpaceCasts(Module &M) {
   for (Function &F : M) {
     DenseMap<Value *, Value *> CastsCache;
     for (BasicBlock &BB : F) {
-      for (Instruction &I : BB) {
+      for (Instruction &I : llvm::make_early_inc_range(BB)) {
         unsigned PtrOpNum;
 
-        if (auto *LD = dyn_cast<LoadInst>(&I))
+        if (auto *LD = dyn_cast<LoadInst>(&I)) {
           PtrOpNum = LD->getPointerOperandIndex();
-        else if (auto *ST = dyn_cast<StoreInst>(&I))
+          aspaceWrapOperand(CastsCache, &I, PtrOpNum);
+          continue;
+        }
+        if (auto *ST = dyn_cast<StoreInst>(&I)) {
           PtrOpNum = ST->getPointerOperandIndex();
-        else if (auto *CmpXchg = dyn_cast<AtomicCmpXchgInst>(&I))
+          aspaceWrapOperand(CastsCache, &I, PtrOpNum);
+          continue;
+        }
+        if (auto *CmpXchg = dyn_cast<AtomicCmpXchgInst>(&I)) {
           PtrOpNum = CmpXchg->getPointerOperandIndex();
-        else if (auto *RMW = dyn_cast<AtomicRMWInst>(&I))
+          aspaceWrapOperand(CastsCache, &I, PtrOpNum);
+          continue;
+        }
+        if (auto *RMW = dyn_cast<AtomicRMWInst>(&I)) {
           PtrOpNum = RMW->getPointerOperandIndex();
-        else
+          aspaceWrapOperand(CastsCache, &I, PtrOpNum);
+          continue;
+        }
+
+        auto *CI = dyn_cast<CallInst>(&I);
+        if (!CI)
+          continue;
+
+        Function *Callee = CI->getCalledFunction();
+        if (!Callee || !Callee->isIntrinsic())
+          continue;
+
+        // Check memset/memcpy/memmove
+        Intrinsic::ID ID = Callee->getIntrinsicID();
+        bool IsSet = ID == Intrinsic::memset;
+        bool IsCpy = ID == Intrinsic::memcpy;
+        bool IsMove = ID == Intrinsic::memmove;
+        if (!IsSet && !IsCpy && !IsMove)
+          continue;
+
+        auto isAS = [&](unsigned ArgIdx) {
+          Value *V = CI->getArgOperand(ArgIdx);
+          if (auto *PTy = dyn_cast<PointerType>(V->getType()))
+            return PTy->getAddressSpace() != 0;
+          return false;
+        };
+
+        // For memset: only dest is a pointer; for memcpy/memmove: dest & src.
+        bool HasAS = IsSet ? isAS(0) : (isAS(0) || isAS(1));
+        if (!HasAS)
           continue;
 
-        aspaceWrapOperand(CastsCache, &I, PtrOpNum);
+        Instruction *New = aspaceMemIntrinsic(CastsCache, &I, IsSet, IsCpy);
+        I.replaceAllUsesWith(New);
+        New->takeName(&I);
+        I.eraseFromParent();
       }
     }
     Changed |= !CastsCache.empty();
diff --git a/llvm/test/CodeGen/BPF/addr-space-memintrinsic-gep.ll b/llvm/test/CodeGen/BPF/addr-space-memintrinsic-gep.ll
new file mode 100644
index 0000000000000..9d0a2dbb4683b
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/addr-space-memintrinsic-gep.ll
@@ -0,0 +1,38 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt --bpf-check-and-opt-ir -S -mtriple=bpf-pc-linux < %s | FileCheck %s
+
+ at page1 = dso_local local_unnamed_addr addrspace(1) global [10 x ptr] zeroinitializer, align 8
+ at page2 = dso_local local_unnamed_addr addrspace(1) global [10 x ptr] zeroinitializer, align 8
+
+define dso_local void @memset_as() local_unnamed_addr {
+; CHECK-LABEL: define dso_local void @memset_as() local_unnamed_addr {
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 addrspacecast (ptr addrspace(1) getelementptr inbounds nuw (i8, ptr addrspace(1) @page1, i64 16) to ptr), i8 0, i64 16, i1 false)
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.memset.p1.i64(ptr addrspace(1) noundef nonnull align 8 dereferenceable(16) getelementptr inbounds nuw (i8, ptr addrspace(1) @page1, i64 16), i8 0, i64 16, i1 false)
+  ret void
+}
+
+declare void @llvm.memset.p1.i64(ptr addrspace(1) writeonly captures(none), i8, i64, i1 immarg)
+
+define dso_local void @memcpy_as() local_unnamed_addr {
+; CHECK-LABEL: define dso_local void @memcpy_as() local_unnamed_addr {
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 addrspacecast (ptr addrspace(1) getelementptr inbounds nuw (i8, ptr addrspace(1) @page2, i64 8) to ptr), ptr align 8 addrspacecast (ptr addrspace(1) getelementptr inbounds nuw (i8, ptr addrspace(1) @page1, i64 8) to ptr), i64 16, i1 false)
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) noundef nonnull align 8 dereferenceable(16) getelementptr inbounds nuw (i8, ptr addrspace(1) @page2, i64 8), ptr addrspace(1) noundef nonnull align 8 dereferenceable(16) getelementptr inbounds nuw (i8, ptr addrspace(1) @page1, i64 8), i64 16, i1 false)
+  ret void
+}
+
+declare void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) noalias writeonly captures(none), ptr addrspace(1) noalias readonly captures(none), i64, i1 immarg)
+
+define dso_local void @memmove_as() local_unnamed_addr {
+; CHECK-LABEL: define dso_local void @memmove_as() local_unnamed_addr {
+; CHECK-NEXT:    call void @llvm.memmove.p0.p0.i64(ptr align 8 addrspacecast (ptr addrspace(1) getelementptr inbounds nuw (i8, ptr addrspace(1) @page2, i64 16) to ptr), ptr align 8 addrspacecast (ptr addrspace(1) getelementptr inbounds nuw (i8, ptr addrspace(1) @page2, i64 8) to ptr), i64 16, i1 false)
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.memmove.p1.p1.i64(ptr addrspace(1) noundef nonnull align 8 dereferenceable(16) getelementptr inbounds nuw (i8, ptr addrspace(1) @page2, i64 16), ptr addrspace(1) noundef nonnull align 8 dereferenceable(16) getelementptr inbounds nuw (i8, ptr addrspace(1) @page2, i64 8), i64 16, i1 false)
+  ret void
+}
+
+declare void @llvm.memmove.p1.p1.i64(ptr addrspace(1) writeonly captures(none), ptr addrspace(1) readonly captures(none), i64, i1 immarg)
diff --git a/llvm/test/CodeGen/BPF/addr-space-memintrinsic-no-gep.ll b/llvm/test/CodeGen/BPF/addr-space-memintrinsic-no-gep.ll
new file mode 100644
index 0000000000000..92b1316373bbf
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/addr-space-memintrinsic-no-gep.ll
@@ -0,0 +1,27 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt --bpf-check-and-opt-ir -S -mtriple=bpf-pc-linux < %s | FileCheck %s
+
+ at page1 = dso_local local_unnamed_addr addrspace(1) global [10 x ptr] zeroinitializer, align 8
+ at page2 = dso_local local_unnamed_addr addrspace(1) global [10 x ptr] zeroinitializer, align 8
+
+define dso_local void @memset_as() local_unnamed_addr {
+; CHECK-LABEL: define dso_local void @memset_as() local_unnamed_addr {
+; CHECK-NEXT:    call void @llvm.memset.p0.i64(ptr align 8 addrspacecast (ptr addrspace(1) @page1 to ptr), i8 0, i64 16, i1 false)
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.memset.p1.i64(ptr addrspace(1) noundef align 8 dereferenceable(16) @page1, i8 0, i64 16, i1 false)
+  ret void
+}
+
+declare void @llvm.memset.p1.i64(ptr addrspace(1) writeonly captures(none), i8, i64, i1 immarg)
+
+define dso_local void @memcpy_as() local_unnamed_addr {
+; CHECK-LABEL: define dso_local void @memcpy_as() local_unnamed_addr {
+; CHECK-NEXT:    call void @llvm.memcpy.p0.p0.i64(ptr align 8 addrspacecast (ptr addrspace(1) @page2 to ptr), ptr align 8 addrspacecast (ptr addrspace(1) @page1 to ptr), i64 16, i1 false)
+; CHECK-NEXT:    ret void
+;
+  tail call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) noundef align 8 dereferenceable(16) @page2, ptr addrspace(1) noundef align 8 dereferenceable(16) @page1, i64 16, i1 false)
+  ret void
+}
+
+declare void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) noalias writeonly captures(none), ptr addrspace(1) noalias readonly captures(none), i64, i1 immarg)



More information about the llvm-commits mailing list