[llvm] [BPF] Handle certain mem intrinsic functions with addr-space arguments (PR #160025)
via llvm-commits
llvm-commits at lists.llvm.org
Sun Sep 21 20:40:29 PDT 2025
https://github.com/yonghong-song updated https://github.com/llvm/llvm-project/pull/160025
>From 3afb1d3142e2f276a997047eb82f885303903f69 Mon Sep 17 00:00:00 2001
From: Yonghong Song <yonghong.song at linux.dev>
Date: Sun, 21 Sep 2025 12:23:12 -0700
Subject: [PATCH] [BPF] Handle certain mem intrinsic functions with addr-space
arguments
In linux kernel commit [1], we have a bpf selftest failure caused by llvm.
In this particular case, the BPFCheckAndAdjustIR pass has a function
insertASpaceCasts() which inserts proper addrspacecast insn at proper IR
places. It does not handle __builtin_memset() and hance caused selftest
failure.
Add support in insertASpaceCasts() to handle __builtin_(memset,memcpy,memmove}()
properly and this can fix the issue in [1] as well.
[1] https://lore.kernel.org/all/20250920045805.3288551-1-yonghong.song@linux.dev/
---
llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp | 104 ++++++++++++++++--
.../BPF/addr-space-memintrinsic-gep.ll | 38 +++++++
.../BPF/addr-space-memintrinsic-no-gep.ll | 27 +++++
3 files changed, 162 insertions(+), 7 deletions(-)
create mode 100644 llvm/test/CodeGen/BPF/addr-space-memintrinsic-gep.ll
create mode 100644 llvm/test/CodeGen/BPF/addr-space-memintrinsic-no-gep.ll
diff --git a/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp b/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp
index b202b20291aff..464123185c0ae 100644
--- a/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp
+++ b/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp
@@ -26,6 +26,7 @@
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/IntrinsicsBPF.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Type.h"
@@ -478,9 +479,57 @@ static void aspaceWrapOperand(DenseMap<Value *, Value *> &Cache, Instruction *I,
}
}
+static Instruction *aspaceMemIntrinsic(DenseMap<Value *, Value *> &Cache,
+ Instruction *I, bool IsSet, bool IsCpy) {
+ auto *CI = cast<CallInst>(I);
+
+ auto wrapPtr = [&](Value *P) -> Value * {
+ if (auto *PTy = dyn_cast<PointerType>(P->getType())) {
+ if (PTy->getAddressSpace() == 0)
+ return P;
+ }
+ return aspaceWrapValue(Cache, CI->getFunction(), P);
+ };
+
+ auto *MI = cast<MemIntrinsic>(I);
+ IRBuilder<> B(I);
+
+ if (IsSet) {
+ // memset(dst, val, len, align, isvolatile, md)
+ Value *Dst = wrapPtr(CI->getArgOperand(0));
+ Value *Val = CI->getArgOperand(1);
+ Value *Len = CI->getArgOperand(2);
+
+ auto *MS = cast<MemSetInst>(I);
+ MaybeAlign Align = MS->getDestAlign();
+ bool IsVolatile = MS->isVolatile();
+
+ return B.CreateMemSet(Dst, Val, Len, Align, IsVolatile,
+ MI->getAAMetadata());
+ }
+
+ // memcpy/memmove(dst, dst_align, src, src_align, len, isvolatile, md)
+ Value *Dst = wrapPtr(CI->getArgOperand(0));
+ Value *Src = wrapPtr(CI->getArgOperand(1));
+ Value *Len = CI->getArgOperand(2);
+
+ auto *MT = cast<MemTransferInst>(I);
+ MaybeAlign DstAlign = MT->getDestAlign();
+ MaybeAlign SrcAlign = MT->getSourceAlign();
+ bool IsVolatile = MT->isVolatile();
+
+ Instruction *New = IsCpy ? B.CreateMemCpy(Dst, DstAlign, Src, SrcAlign, Len,
+ IsVolatile, MI->getAAMetadata())
+ : B.CreateMemMove(Dst, DstAlign, Src, SrcAlign, Len,
+ IsVolatile, MI->getAAMetadata());
+
+ return New;
+}
+
// Support for BPF address spaces:
// - for each function in the module M, update pointer operand of
// each memory access instruction (load/store/cmpxchg/atomicrmw)
+// or intrinsic call insns (memset/memcpy/memmove)
// by casting it from non-zero address space to zero address space, e.g:
//
// (load (ptr addrspace (N) %p) ...)
@@ -493,21 +542,62 @@ bool BPFCheckAndAdjustIR::insertASpaceCasts(Module &M) {
for (Function &F : M) {
DenseMap<Value *, Value *> CastsCache;
for (BasicBlock &BB : F) {
- for (Instruction &I : BB) {
+ for (Instruction &I : llvm::make_early_inc_range(BB)) {
unsigned PtrOpNum;
- if (auto *LD = dyn_cast<LoadInst>(&I))
+ if (auto *LD = dyn_cast<LoadInst>(&I)) {
PtrOpNum = LD->getPointerOperandIndex();
- else if (auto *ST = dyn_cast<StoreInst>(&I))
+ aspaceWrapOperand(CastsCache, &I, PtrOpNum);
+ continue;
+ }
+ if (auto *ST = dyn_cast<StoreInst>(&I)) {
PtrOpNum = ST->getPointerOperandIndex();
- else if (auto *CmpXchg = dyn_cast<AtomicCmpXchgInst>(&I))
+ aspaceWrapOperand(CastsCache, &I, PtrOpNum);
+ continue;
+ }
+ if (auto *CmpXchg = dyn_cast<AtomicCmpXchgInst>(&I)) {
PtrOpNum = CmpXchg->getPointerOperandIndex();
- else if (auto *RMW = dyn_cast<AtomicRMWInst>(&I))
+ aspaceWrapOperand(CastsCache, &I, PtrOpNum);
+ continue;
+ }
+ if (auto *RMW = dyn_cast<AtomicRMWInst>(&I)) {
PtrOpNum = RMW->getPointerOperandIndex();
- else
+ aspaceWrapOperand(CastsCache, &I, PtrOpNum);
+ continue;
+ }
+
+ auto *CI = dyn_cast<CallInst>(&I);
+ if (!CI)
+ continue;
+
+ Function *Callee = CI->getCalledFunction();
+ if (!Callee || !Callee->isIntrinsic())
+ continue;
+
+ // Check memset/memcpy/memmove
+ Intrinsic::ID ID = Callee->getIntrinsicID();
+ bool IsSet = ID == Intrinsic::memset;
+ bool IsCpy = ID == Intrinsic::memcpy;
+ bool IsMove = ID == Intrinsic::memmove;
+ if (!IsSet && !IsCpy && !IsMove)
+ continue;
+
+ auto isAS = [&](unsigned ArgIdx) {
+ Value *V = CI->getArgOperand(ArgIdx);
+ if (auto *PTy = dyn_cast<PointerType>(V->getType()))
+ return PTy->getAddressSpace() != 0;
+ return false;
+ };
+
+ // For memset: only dest is a pointer; for memcpy/memmove: dest & src.
+ bool HasAS = IsSet ? isAS(0) : (isAS(0) || isAS(1));
+ if (!HasAS)
continue;
- aspaceWrapOperand(CastsCache, &I, PtrOpNum);
+ Instruction *New = aspaceMemIntrinsic(CastsCache, &I, IsSet, IsCpy);
+ I.replaceAllUsesWith(New);
+ New->takeName(&I);
+ I.eraseFromParent();
}
}
Changed |= !CastsCache.empty();
diff --git a/llvm/test/CodeGen/BPF/addr-space-memintrinsic-gep.ll b/llvm/test/CodeGen/BPF/addr-space-memintrinsic-gep.ll
new file mode 100644
index 0000000000000..9d0a2dbb4683b
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/addr-space-memintrinsic-gep.ll
@@ -0,0 +1,38 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt --bpf-check-and-opt-ir -S -mtriple=bpf-pc-linux < %s | FileCheck %s
+
+ at page1 = dso_local local_unnamed_addr addrspace(1) global [10 x ptr] zeroinitializer, align 8
+ at page2 = dso_local local_unnamed_addr addrspace(1) global [10 x ptr] zeroinitializer, align 8
+
+define dso_local void @memset_as() local_unnamed_addr {
+; CHECK-LABEL: define dso_local void @memset_as() local_unnamed_addr {
+; CHECK-NEXT: call void @llvm.memset.p0.i64(ptr align 8 addrspacecast (ptr addrspace(1) getelementptr inbounds nuw (i8, ptr addrspace(1) @page1, i64 16) to ptr), i8 0, i64 16, i1 false)
+; CHECK-NEXT: ret void
+;
+ tail call void @llvm.memset.p1.i64(ptr addrspace(1) noundef nonnull align 8 dereferenceable(16) getelementptr inbounds nuw (i8, ptr addrspace(1) @page1, i64 16), i8 0, i64 16, i1 false)
+ ret void
+}
+
+declare void @llvm.memset.p1.i64(ptr addrspace(1) writeonly captures(none), i8, i64, i1 immarg)
+
+define dso_local void @memcpy_as() local_unnamed_addr {
+; CHECK-LABEL: define dso_local void @memcpy_as() local_unnamed_addr {
+; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 addrspacecast (ptr addrspace(1) getelementptr inbounds nuw (i8, ptr addrspace(1) @page2, i64 8) to ptr), ptr align 8 addrspacecast (ptr addrspace(1) getelementptr inbounds nuw (i8, ptr addrspace(1) @page1, i64 8) to ptr), i64 16, i1 false)
+; CHECK-NEXT: ret void
+;
+ tail call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) noundef nonnull align 8 dereferenceable(16) getelementptr inbounds nuw (i8, ptr addrspace(1) @page2, i64 8), ptr addrspace(1) noundef nonnull align 8 dereferenceable(16) getelementptr inbounds nuw (i8, ptr addrspace(1) @page1, i64 8), i64 16, i1 false)
+ ret void
+}
+
+declare void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) noalias writeonly captures(none), ptr addrspace(1) noalias readonly captures(none), i64, i1 immarg)
+
+define dso_local void @memmove_as() local_unnamed_addr {
+; CHECK-LABEL: define dso_local void @memmove_as() local_unnamed_addr {
+; CHECK-NEXT: call void @llvm.memmove.p0.p0.i64(ptr align 8 addrspacecast (ptr addrspace(1) getelementptr inbounds nuw (i8, ptr addrspace(1) @page2, i64 16) to ptr), ptr align 8 addrspacecast (ptr addrspace(1) getelementptr inbounds nuw (i8, ptr addrspace(1) @page2, i64 8) to ptr), i64 16, i1 false)
+; CHECK-NEXT: ret void
+;
+ tail call void @llvm.memmove.p1.p1.i64(ptr addrspace(1) noundef nonnull align 8 dereferenceable(16) getelementptr inbounds nuw (i8, ptr addrspace(1) @page2, i64 16), ptr addrspace(1) noundef nonnull align 8 dereferenceable(16) getelementptr inbounds nuw (i8, ptr addrspace(1) @page2, i64 8), i64 16, i1 false)
+ ret void
+}
+
+declare void @llvm.memmove.p1.p1.i64(ptr addrspace(1) writeonly captures(none), ptr addrspace(1) readonly captures(none), i64, i1 immarg)
diff --git a/llvm/test/CodeGen/BPF/addr-space-memintrinsic-no-gep.ll b/llvm/test/CodeGen/BPF/addr-space-memintrinsic-no-gep.ll
new file mode 100644
index 0000000000000..92b1316373bbf
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/addr-space-memintrinsic-no-gep.ll
@@ -0,0 +1,27 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt --bpf-check-and-opt-ir -S -mtriple=bpf-pc-linux < %s | FileCheck %s
+
+ at page1 = dso_local local_unnamed_addr addrspace(1) global [10 x ptr] zeroinitializer, align 8
+ at page2 = dso_local local_unnamed_addr addrspace(1) global [10 x ptr] zeroinitializer, align 8
+
+define dso_local void @memset_as() local_unnamed_addr {
+; CHECK-LABEL: define dso_local void @memset_as() local_unnamed_addr {
+; CHECK-NEXT: call void @llvm.memset.p0.i64(ptr align 8 addrspacecast (ptr addrspace(1) @page1 to ptr), i8 0, i64 16, i1 false)
+; CHECK-NEXT: ret void
+;
+ tail call void @llvm.memset.p1.i64(ptr addrspace(1) noundef align 8 dereferenceable(16) @page1, i8 0, i64 16, i1 false)
+ ret void
+}
+
+declare void @llvm.memset.p1.i64(ptr addrspace(1) writeonly captures(none), i8, i64, i1 immarg)
+
+define dso_local void @memcpy_as() local_unnamed_addr {
+; CHECK-LABEL: define dso_local void @memcpy_as() local_unnamed_addr {
+; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 addrspacecast (ptr addrspace(1) @page2 to ptr), ptr align 8 addrspacecast (ptr addrspace(1) @page1 to ptr), i64 16, i1 false)
+; CHECK-NEXT: ret void
+;
+ tail call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) noundef align 8 dereferenceable(16) @page2, ptr addrspace(1) noundef align 8 dereferenceable(16) @page1, i64 16, i1 false)
+ ret void
+}
+
+declare void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) noalias writeonly captures(none), ptr addrspace(1) noalias readonly captures(none), i64, i1 immarg)
More information about the llvm-commits
mailing list