[llvm] [BPF] Handle certain mem intrinsic functions with addr-space arguments (PR #160025)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Oct 7 14:41:26 PDT 2025
https://github.com/yonghong-song updated https://github.com/llvm/llvm-project/pull/160025
>From 359b6450a35759c745e20187042aed6887a3aa3a Mon Sep 17 00:00:00 2001
From: Yonghong Song <yonghong.song at linux.dev>
Date: Sun, 21 Sep 2025 12:23:12 -0700
Subject: [PATCH] [BPF] Handle certain mem intrinsic functions with addr-space
arguments
In linux kernel commit [1], we have a bpf selftest failure caused by llvm.
In this particular case, the BPFCheckAndAdjustIR pass has a function
insertASpaceCasts() which inserts proper addrspacecast insn at proper IR
places. It does not handle __builtin_memset() and hance caused selftest
failure.
Add support in insertASpaceCasts() to handle
__builtin_(memset,memcpy,memmove,memset_inline,memcpy_inline}()
properly and this can fix the issue in [1] as well.
[1] https://lore.kernel.org/all/20250920045805.3288551-1-yonghong.song@linux.dev/
---
llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp | 138 +++++++++++++++++-
.../BPF/addr-space-memintrinsic-gep.ll | 60 ++++++++
.../BPF/addr-space-memintrinsic-no-gep.ll | 49 +++++++
3 files changed, 241 insertions(+), 6 deletions(-)
create mode 100644 llvm/test/CodeGen/BPF/addr-space-memintrinsic-gep.ll
create mode 100644 llvm/test/CodeGen/BPF/addr-space-memintrinsic-no-gep.ll
diff --git a/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp b/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp
index b202b20291aff..e3c39a1b8dda5 100644
--- a/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp
+++ b/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp
@@ -26,6 +26,7 @@
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/IntrinsicsBPF.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Type.h"
@@ -478,9 +479,95 @@ static void aspaceWrapOperand(DenseMap<Value *, Value *> &Cache, Instruction *I,
}
}
+static Value *wrapPtrIfASNotZero(DenseMap<Value *, Value *> &Cache,
+ CallInst *CI, Value *P) {
+ if (auto *PTy = dyn_cast<PointerType>(P->getType())) {
+ if (PTy->getAddressSpace() == 0)
+ return P;
+ }
+ return aspaceWrapValue(Cache, CI->getFunction(), P);
+}
+
+static Instruction *aspaceMemSet(Intrinsic::ID ID,
+ DenseMap<Value *, Value *> &Cache,
+ CallInst *CI) {
+ auto *MI = cast<MemIntrinsic>(CI);
+ IRBuilder<> B(CI);
+
+ Value *OldDst = CI->getArgOperand(0);
+ Value *NewDst = wrapPtrIfASNotZero(Cache, CI, OldDst);
+ if (OldDst == NewDst)
+ return nullptr;
+
+ // memset(new_dst, val, len, align, isvolatile, md)
+ Value *Val = CI->getArgOperand(1);
+ Value *Len = CI->getArgOperand(2);
+
+ auto *MS = cast<MemSetInst>(CI);
+ MaybeAlign Align = MS->getDestAlign();
+ bool IsVolatile = MS->isVolatile();
+
+ if (ID == Intrinsic::memset)
+ return B.CreateMemSet(NewDst, Val, Len, Align, IsVolatile,
+ MI->getAAMetadata());
+ else
+ return B.CreateMemSetInline(NewDst, Align, Val, Len, IsVolatile,
+ MI->getAAMetadata());
+}
+
+static Instruction *aspaceMemCpy(Intrinsic::ID ID,
+ DenseMap<Value *, Value *> &Cache,
+ CallInst *CI) {
+ auto *MI = cast<MemIntrinsic>(CI);
+ IRBuilder<> B(CI);
+
+ Value *OldDst = CI->getArgOperand(0);
+ Value *OldSrc = CI->getArgOperand(1);
+ Value *NewDst = wrapPtrIfASNotZero(Cache, CI, OldDst);
+ Value *NewSrc = wrapPtrIfASNotZero(Cache, CI, OldSrc);
+ if (OldDst == NewDst && OldSrc == NewSrc)
+ return nullptr;
+
+ // memcpy(new_dst, dst_align, new_src, src_align, len, isvolatile, md)
+ Value *Len = CI->getArgOperand(2);
+
+ auto *MT = cast<MemTransferInst>(CI);
+ MaybeAlign DstAlign = MT->getDestAlign();
+ MaybeAlign SrcAlign = MT->getSourceAlign();
+ bool IsVolatile = MT->isVolatile();
+
+ return B.CreateMemTransferInst(ID, NewDst, DstAlign, NewSrc, SrcAlign, Len,
+ IsVolatile, MI->getAAMetadata());
+}
+
+static Instruction *aspaceMemMove(DenseMap<Value *, Value *> &Cache,
+ CallInst *CI) {
+ auto *MI = cast<MemIntrinsic>(CI);
+ IRBuilder<> B(CI);
+
+ Value *OldDst = CI->getArgOperand(0);
+ Value *OldSrc = CI->getArgOperand(1);
+ Value *NewDst = wrapPtrIfASNotZero(Cache, CI, OldDst);
+ Value *NewSrc = wrapPtrIfASNotZero(Cache, CI, OldSrc);
+ if (OldDst == NewDst && OldSrc == NewSrc)
+ return nullptr;
+
+ // memmove(new_dst, dst_align, new_src, src_align, len, isvolatile, md)
+ Value *Len = CI->getArgOperand(2);
+
+ auto *MT = cast<MemTransferInst>(CI);
+ MaybeAlign DstAlign = MT->getDestAlign();
+ MaybeAlign SrcAlign = MT->getSourceAlign();
+ bool IsVolatile = MT->isVolatile();
+
+ return B.CreateMemMove(NewDst, DstAlign, NewSrc, SrcAlign, Len, IsVolatile,
+ MI->getAAMetadata());
+}
+
// Support for BPF address spaces:
// - for each function in the module M, update pointer operand of
// each memory access instruction (load/store/cmpxchg/atomicrmw)
+// or intrinsic call insns (memset/memcpy/memmove)
// by casting it from non-zero address space to zero address space, e.g:
//
// (load (ptr addrspace (N) %p) ...)
@@ -493,21 +580,60 @@ bool BPFCheckAndAdjustIR::insertASpaceCasts(Module &M) {
for (Function &F : M) {
DenseMap<Value *, Value *> CastsCache;
for (BasicBlock &BB : F) {
- for (Instruction &I : BB) {
+ for (Instruction &I : llvm::make_early_inc_range(BB)) {
unsigned PtrOpNum;
- if (auto *LD = dyn_cast<LoadInst>(&I))
+ if (auto *LD = dyn_cast<LoadInst>(&I)) {
PtrOpNum = LD->getPointerOperandIndex();
- else if (auto *ST = dyn_cast<StoreInst>(&I))
+ aspaceWrapOperand(CastsCache, &I, PtrOpNum);
+ continue;
+ }
+ if (auto *ST = dyn_cast<StoreInst>(&I)) {
PtrOpNum = ST->getPointerOperandIndex();
- else if (auto *CmpXchg = dyn_cast<AtomicCmpXchgInst>(&I))
+ aspaceWrapOperand(CastsCache, &I, PtrOpNum);
+ continue;
+ }
+ if (auto *CmpXchg = dyn_cast<AtomicCmpXchgInst>(&I)) {
PtrOpNum = CmpXchg->getPointerOperandIndex();
- else if (auto *RMW = dyn_cast<AtomicRMWInst>(&I))
+ aspaceWrapOperand(CastsCache, &I, PtrOpNum);
+ continue;
+ }
+ if (auto *RMW = dyn_cast<AtomicRMWInst>(&I)) {
PtrOpNum = RMW->getPointerOperandIndex();
+ aspaceWrapOperand(CastsCache, &I, PtrOpNum);
+ continue;
+ }
+
+ auto *CI = dyn_cast<CallInst>(&I);
+ if (!CI)
+ continue;
+
+ Function *Callee = CI->getCalledFunction();
+ if (!Callee || !Callee->isIntrinsic())
+ continue;
+
+ // Check memset/memcpy/memmove
+ Intrinsic::ID ID = Callee->getIntrinsicID();
+ bool IsSet = ID == Intrinsic::memset || ID == Intrinsic::memset_inline;
+ bool IsCpy = ID == Intrinsic::memcpy || ID == Intrinsic::memcpy_inline;
+ bool IsMove = ID == Intrinsic::memmove;
+ if (!IsSet && !IsCpy && !IsMove)
+ continue;
+
+ Instruction *New;
+ if (IsSet)
+ New = aspaceMemSet(ID, CastsCache, CI);
+ else if (IsCpy)
+ New = aspaceMemCpy(ID, CastsCache, CI);
else
+ New = aspaceMemMove(CastsCache, CI);
+
+ if (!New)
continue;
- aspaceWrapOperand(CastsCache, &I, PtrOpNum);
+ I.replaceAllUsesWith(New);
+ New->takeName(&I);
+ I.eraseFromParent();
}
}
Changed |= !CastsCache.empty();
diff --git a/llvm/test/CodeGen/BPF/addr-space-memintrinsic-gep.ll b/llvm/test/CodeGen/BPF/addr-space-memintrinsic-gep.ll
new file mode 100644
index 0000000000000..1db83915b00a3
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/addr-space-memintrinsic-gep.ll
@@ -0,0 +1,60 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt --bpf-check-and-opt-ir -S -mtriple=bpf-pc-linux < %s | FileCheck %s
+
+ at page1 = dso_local local_unnamed_addr addrspace(1) global [10 x ptr] zeroinitializer, align 8
+ at page2 = dso_local local_unnamed_addr addrspace(1) global [10 x ptr] zeroinitializer, align 8
+
+define dso_local void @test_memset() local_unnamed_addr {
+; CHECK-LABEL: define dso_local void @test_memset() local_unnamed_addr {
+; CHECK-NEXT: call void @llvm.memset.p0.i64(ptr align 8 addrspacecast (ptr addrspace(1) getelementptr inbounds nuw (i8, ptr addrspace(1) @page1, i64 16) to ptr), i8 0, i64 16, i1 false)
+; CHECK-NEXT: ret void
+;
+ tail call void @llvm.memset.p1.i64(ptr addrspace(1) noundef nonnull align 8 dereferenceable(16) getelementptr inbounds nuw (i8, ptr addrspace(1) @page1, i64 16), i8 0, i64 16, i1 false)
+ ret void
+}
+
+declare void @llvm.memset.p1.i64(ptr addrspace(1) writeonly captures(none), i8, i64, i1 immarg)
+
+define dso_local void @test_memcpy() local_unnamed_addr {
+; CHECK-LABEL: define dso_local void @test_memcpy() local_unnamed_addr {
+; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 addrspacecast (ptr addrspace(1) getelementptr inbounds nuw (i8, ptr addrspace(1) @page2, i64 8) to ptr), ptr align 8 addrspacecast (ptr addrspace(1) getelementptr inbounds nuw (i8, ptr addrspace(1) @page1, i64 8) to ptr), i64 16, i1 false)
+; CHECK-NEXT: ret void
+;
+ tail call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) noundef nonnull align 8 dereferenceable(16) getelementptr inbounds nuw (i8, ptr addrspace(1) @page2, i64 8), ptr addrspace(1) noundef nonnull align 8 dereferenceable(16) getelementptr inbounds nuw (i8, ptr addrspace(1) @page1, i64 8), i64 16, i1 false)
+ ret void
+}
+
+declare void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) noalias writeonly captures(none), ptr addrspace(1) noalias readonly captures(none), i64, i1 immarg)
+
+define dso_local void @test_memmove() local_unnamed_addr {
+; CHECK-LABEL: define dso_local void @test_memmove() local_unnamed_addr {
+; CHECK-NEXT: call void @llvm.memmove.p0.p0.i64(ptr align 8 addrspacecast (ptr addrspace(1) getelementptr inbounds nuw (i8, ptr addrspace(1) @page2, i64 16) to ptr), ptr align 8 addrspacecast (ptr addrspace(1) getelementptr inbounds nuw (i8, ptr addrspace(1) @page2, i64 8) to ptr), i64 16, i1 false)
+; CHECK-NEXT: ret void
+;
+ tail call void @llvm.memmove.p1.p1.i64(ptr addrspace(1) noundef nonnull align 8 dereferenceable(16) getelementptr inbounds nuw (i8, ptr addrspace(1) @page2, i64 16), ptr addrspace(1) noundef nonnull align 8 dereferenceable(16) getelementptr inbounds nuw (i8, ptr addrspace(1) @page2, i64 8), i64 16, i1 false)
+ ret void
+}
+
+declare void @llvm.memmove.p1.p1.i64(ptr addrspace(1) writeonly captures(none), ptr addrspace(1) readonly captures(none), i64, i1 immarg)
+
+define dso_local void @test_memset_inline() local_unnamed_addr {
+; CHECK-LABEL: define dso_local void @test_memset_inline() local_unnamed_addr {
+; CHECK-NEXT: call void @llvm.memset.inline.p0.i64(ptr align 8 addrspacecast (ptr addrspace(1) getelementptr inbounds nuw (i8, ptr addrspace(1) @page1, i64 16) to ptr), i8 0, i64 16, i1 false)
+; CHECK-NEXT: ret void
+;
+ tail call void @llvm.memset.inline.p1.i64(ptr addrspace(1) nonnull align 8 getelementptr inbounds nuw (i8, ptr addrspace(1) @page1, i64 16), i8 0, i64 16, i1 false)
+ ret void
+}
+
+declare void @llvm.memset.inline.p1.i64(ptr addrspace(1) writeonly captures(none), i8, i64, i1 immarg)
+
+define dso_local void @test_memcpy_inline() local_unnamed_addr {
+; CHECK-LABEL: define dso_local void @test_memcpy_inline() local_unnamed_addr {
+; CHECK-NEXT: call void @llvm.memcpy.inline.p0.p0.i64(ptr align 8 addrspacecast (ptr addrspace(1) getelementptr inbounds nuw (i8, ptr addrspace(1) @page2, i64 8) to ptr), ptr align 8 addrspacecast (ptr addrspace(1) getelementptr inbounds nuw (i8, ptr addrspace(1) @page1, i64 8) to ptr), i64 16, i1 false)
+; CHECK-NEXT: ret void
+;
+ tail call void @llvm.memcpy.inline.p1.p1.i64(ptr addrspace(1) nonnull align 8 getelementptr inbounds nuw (i8, ptr addrspace(1) @page2, i64 8), ptr addrspace(1) nonnull align 8 getelementptr inbounds nuw (i8, ptr addrspace(1) @page1, i64 8), i64 16, i1 false)
+ ret void
+}
+
+declare void @llvm.memcpy.inline.p1.p1.i64(ptr addrspace(1) noalias writeonly captures(none), ptr addrspace(1) noalias readonly captures(none), i64, i1 immarg)
diff --git a/llvm/test/CodeGen/BPF/addr-space-memintrinsic-no-gep.ll b/llvm/test/CodeGen/BPF/addr-space-memintrinsic-no-gep.ll
new file mode 100644
index 0000000000000..62fa2e4d4491f
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/addr-space-memintrinsic-no-gep.ll
@@ -0,0 +1,49 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt --bpf-check-and-opt-ir -S -mtriple=bpf-pc-linux < %s | FileCheck %s
+
+ at page1 = dso_local local_unnamed_addr addrspace(1) global [10 x ptr] zeroinitializer, align 8
+ at page2 = dso_local local_unnamed_addr addrspace(1) global [10 x ptr] zeroinitializer, align 8
+
+define dso_local void @test_memset() local_unnamed_addr {
+; CHECK-LABEL: define dso_local void @test_memset() local_unnamed_addr {
+; CHECK-NEXT: call void @llvm.memset.p0.i64(ptr align 8 addrspacecast (ptr addrspace(1) @page1 to ptr), i8 0, i64 16, i1 false)
+; CHECK-NEXT: ret void
+;
+ tail call void @llvm.memset.p1.i64(ptr addrspace(1) noundef align 8 dereferenceable(16) @page1, i8 0, i64 16, i1 false)
+ ret void
+}
+
+declare void @llvm.memset.p1.i64(ptr addrspace(1) writeonly captures(none), i8, i64, i1 immarg)
+
+define dso_local void @test_memcpy() local_unnamed_addr {
+; CHECK-LABEL: define dso_local void @test_memcpy() local_unnamed_addr {
+; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 addrspacecast (ptr addrspace(1) @page2 to ptr), ptr align 8 addrspacecast (ptr addrspace(1) @page1 to ptr), i64 16, i1 false)
+; CHECK-NEXT: ret void
+;
+ tail call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) noundef align 8 dereferenceable(16) @page2, ptr addrspace(1) noundef align 8 dereferenceable(16) @page1, i64 16, i1 false)
+ ret void
+}
+
+declare void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) noalias writeonly captures(none), ptr addrspace(1) noalias readonly captures(none), i64, i1 immarg)
+
+define dso_local void @test_memset_inline() local_unnamed_addr {
+; CHECK-LABEL: define dso_local void @test_memset_inline() local_unnamed_addr {
+; CHECK-NEXT: call void @llvm.memset.inline.p0.i64(ptr align 8 addrspacecast (ptr addrspace(1) @page1 to ptr), i8 0, i64 16, i1 false)
+; CHECK-NEXT: ret void
+;
+ tail call void @llvm.memset.inline.p1.i64(ptr addrspace(1) align 8 @page1, i8 0, i64 16, i1 false)
+ ret void
+}
+
+declare void @llvm.memset.inline.p1.i64(ptr addrspace(1) writeonly captures(none), i8, i64, i1 immarg)
+
+define dso_local void @test_memcpy_inline() local_unnamed_addr {
+; CHECK-LABEL: define dso_local void @test_memcpy_inline() local_unnamed_addr {
+; CHECK-NEXT: call void @llvm.memcpy.inline.p0.p0.i64(ptr align 8 addrspacecast (ptr addrspace(1) @page2 to ptr), ptr align 8 addrspacecast (ptr addrspace(1) @page1 to ptr), i64 16, i1 false)
+; CHECK-NEXT: ret void
+;
+ tail call void @llvm.memcpy.inline.p1.p1.i64(ptr addrspace(1) align 8 @page2, ptr addrspace(1) align 8 @page1, i64 16, i1 false)
+ ret void
+}
+
+declare void @llvm.memcpy.inline.p1.p1.i64(ptr addrspace(1) noalias writeonly captures(none), ptr addrspace(1) noalias readonly captures(none), i64, i1 immarg)
More information about the llvm-commits
mailing list