[llvm] [BPF] Handle certain mem intrinsic functions with addr-space arguments (PR #160025)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 24 10:44:01 PDT 2025
https://github.com/yonghong-song updated https://github.com/llvm/llvm-project/pull/160025
>From 669cd1d4866266ba6c74e3ac35225cbb6b3f2bab Mon Sep 17 00:00:00 2001
From: Yonghong Song <yonghong.song at linux.dev>
Date: Sun, 21 Sep 2025 12:23:12 -0700
Subject: [PATCH] [BPF] Handle certain mem intrinsic functions with addr-space
arguments
In linux kernel commit [1], we have a bpf selftest failure caused by llvm.
In this particular case, the BPFCheckAndAdjustIR pass has a function
insertASpaceCasts() which inserts proper addrspacecast insn at proper IR
places. It does not handle __builtin_memset() and hance caused selftest
failure.
Add support in insertASpaceCasts() to handle __builtin_(memset,memcpy,memmove}()
properly and this can fix the issue in [1] as well.
[1] https://lore.kernel.org/all/20250920045805.3288551-1-yonghong.song@linux.dev/
---
llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp | 128 +++++++++++++++++-
.../BPF/addr-space-memintrinsic-gep.ll | 38 ++++++
.../BPF/addr-space-memintrinsic-no-gep.ll | 27 ++++
3 files changed, 186 insertions(+), 7 deletions(-)
create mode 100644 llvm/test/CodeGen/BPF/addr-space-memintrinsic-gep.ll
create mode 100644 llvm/test/CodeGen/BPF/addr-space-memintrinsic-no-gep.ll
diff --git a/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp b/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp
index b202b20291aff..316790e430ffd 100644
--- a/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp
+++ b/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp
@@ -26,6 +26,7 @@
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/IntrinsicsBPF.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Type.h"
@@ -478,9 +479,74 @@ static void aspaceWrapOperand(DenseMap<Value *, Value *> &Cache, Instruction *I,
}
}
+static Value *wrapPtrIfASNotZero(DenseMap<Value *, Value *> &Cache,
+ CallInst *CI, Value *P) {
+ if (auto *PTy = dyn_cast<PointerType>(P->getType())) {
+ if (PTy->getAddressSpace() == 0)
+ return P;
+ }
+ return aspaceWrapValue(Cache, CI->getFunction(), P);
+}
+
+static Instruction *aspaceMemSet(DenseMap<Value *, Value *> &Cache,
+ CallInst *CI) {
+ auto *MI = cast<MemIntrinsic>(CI);
+ IRBuilder<> B(CI);
+
+ // memset(dst, val, len, align, isvolatile, md)
+ Value *Dst = wrapPtrIfASNotZero(Cache, CI, CI->getArgOperand(0));
+ Value *Val = CI->getArgOperand(1);
+ Value *Len = CI->getArgOperand(2);
+
+ auto *MS = cast<MemSetInst>(CI);
+ MaybeAlign Align = MS->getDestAlign();
+ bool IsVolatile = MS->isVolatile();
+
+ return B.CreateMemSet(Dst, Val, Len, Align, IsVolatile, MI->getAAMetadata());
+}
+
+static Instruction *aspaceMemCpy(DenseMap<Value *, Value *> &Cache,
+ CallInst *CI) {
+ auto *MI = cast<MemIntrinsic>(CI);
+ IRBuilder<> B(CI);
+
+ // memcpy(dst, dst_align, src, src_align, len, isvolatile, md)
+ Value *Dst = wrapPtrIfASNotZero(Cache, CI, CI->getArgOperand(0));
+ Value *Src = wrapPtrIfASNotZero(Cache, CI, CI->getArgOperand(1));
+ Value *Len = CI->getArgOperand(2);
+
+ auto *MT = cast<MemTransferInst>(CI);
+ MaybeAlign DstAlign = MT->getDestAlign();
+ MaybeAlign SrcAlign = MT->getSourceAlign();
+ bool IsVolatile = MT->isVolatile();
+
+ return B.CreateMemCpy(Dst, DstAlign, Src, SrcAlign, Len, IsVolatile,
+ MI->getAAMetadata());
+}
+
+static Instruction *aspaceMemMove(DenseMap<Value *, Value *> &Cache,
+ CallInst *CI) {
+ auto *MI = cast<MemIntrinsic>(CI);
+ IRBuilder<> B(CI);
+
+ // memmove(dst, dst_align, src, src_align, len, isvolatile, md)
+ Value *Dst = wrapPtrIfASNotZero(Cache, CI, CI->getArgOperand(0));
+ Value *Src = wrapPtrIfASNotZero(Cache, CI, CI->getArgOperand(1));
+ Value *Len = CI->getArgOperand(2);
+
+ auto *MT = cast<MemTransferInst>(CI);
+ MaybeAlign DstAlign = MT->getDestAlign();
+ MaybeAlign SrcAlign = MT->getSourceAlign();
+ bool IsVolatile = MT->isVolatile();
+
+ return B.CreateMemMove(Dst, DstAlign, Src, SrcAlign, Len, IsVolatile,
+ MI->getAAMetadata());
+}
+
// Support for BPF address spaces:
// - for each function in the module M, update pointer operand of
// each memory access instruction (load/store/cmpxchg/atomicrmw)
+// or intrinsic call insns (memset/memcpy/memmove)
// by casting it from non-zero address space to zero address space, e.g:
//
// (load (ptr addrspace (N) %p) ...)
@@ -493,21 +559,69 @@ bool BPFCheckAndAdjustIR::insertASpaceCasts(Module &M) {
for (Function &F : M) {
DenseMap<Value *, Value *> CastsCache;
for (BasicBlock &BB : F) {
- for (Instruction &I : BB) {
+ for (Instruction &I : llvm::make_early_inc_range(BB)) {
unsigned PtrOpNum;
- if (auto *LD = dyn_cast<LoadInst>(&I))
+ if (auto *LD = dyn_cast<LoadInst>(&I)) {
PtrOpNum = LD->getPointerOperandIndex();
- else if (auto *ST = dyn_cast<StoreInst>(&I))
+ aspaceWrapOperand(CastsCache, &I, PtrOpNum);
+ continue;
+ }
+ if (auto *ST = dyn_cast<StoreInst>(&I)) {
PtrOpNum = ST->getPointerOperandIndex();
- else if (auto *CmpXchg = dyn_cast<AtomicCmpXchgInst>(&I))
+ aspaceWrapOperand(CastsCache, &I, PtrOpNum);
+ continue;
+ }
+ if (auto *CmpXchg = dyn_cast<AtomicCmpXchgInst>(&I)) {
PtrOpNum = CmpXchg->getPointerOperandIndex();
- else if (auto *RMW = dyn_cast<AtomicRMWInst>(&I))
+ aspaceWrapOperand(CastsCache, &I, PtrOpNum);
+ continue;
+ }
+ if (auto *RMW = dyn_cast<AtomicRMWInst>(&I)) {
PtrOpNum = RMW->getPointerOperandIndex();
- else
+ aspaceWrapOperand(CastsCache, &I, PtrOpNum);
continue;
+ }
+
+ auto *CI = dyn_cast<CallInst>(&I);
+ if (!CI)
+ continue;
+
+ Function *Callee = CI->getCalledFunction();
+ if (!Callee || !Callee->isIntrinsic())
+ continue;
+
+ // Check memset/memcpy/memmove
+ Intrinsic::ID ID = Callee->getIntrinsicID();
+ bool IsSet = ID == Intrinsic::memset;
+ bool IsCpy = ID == Intrinsic::memcpy;
+ bool IsMove = ID == Intrinsic::memmove;
+ if (!IsSet && !IsCpy && !IsMove)
+ continue;
+
+ auto isAS = [&](unsigned ArgIdx) {
+ Value *V = CI->getArgOperand(ArgIdx);
+ if (auto *PTy = dyn_cast<PointerType>(V->getType()))
+ return PTy->getAddressSpace() != 0;
+ return false;
+ };
+
+ // For memset: only dest is a pointer; for memcpy/memmove: dest & src.
+ bool HasAS = IsSet ? isAS(0) : (isAS(0) || isAS(1));
+ if (!HasAS)
+ continue;
+
+ Instruction *New;
+ if (IsSet)
+ New = aspaceMemSet(CastsCache, CI);
+ else if (IsCpy)
+ New = aspaceMemCpy(CastsCache, CI);
+ else
+ New = aspaceMemMove(CastsCache, CI);
- aspaceWrapOperand(CastsCache, &I, PtrOpNum);
+ I.replaceAllUsesWith(New);
+ New->takeName(&I);
+ I.eraseFromParent();
}
}
Changed |= !CastsCache.empty();
diff --git a/llvm/test/CodeGen/BPF/addr-space-memintrinsic-gep.ll b/llvm/test/CodeGen/BPF/addr-space-memintrinsic-gep.ll
new file mode 100644
index 0000000000000..9d0a2dbb4683b
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/addr-space-memintrinsic-gep.ll
@@ -0,0 +1,38 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt --bpf-check-and-opt-ir -S -mtriple=bpf-pc-linux < %s | FileCheck %s
+
+ at page1 = dso_local local_unnamed_addr addrspace(1) global [10 x ptr] zeroinitializer, align 8
+ at page2 = dso_local local_unnamed_addr addrspace(1) global [10 x ptr] zeroinitializer, align 8
+
+define dso_local void @memset_as() local_unnamed_addr {
+; CHECK-LABEL: define dso_local void @memset_as() local_unnamed_addr {
+; CHECK-NEXT: call void @llvm.memset.p0.i64(ptr align 8 addrspacecast (ptr addrspace(1) getelementptr inbounds nuw (i8, ptr addrspace(1) @page1, i64 16) to ptr), i8 0, i64 16, i1 false)
+; CHECK-NEXT: ret void
+;
+ tail call void @llvm.memset.p1.i64(ptr addrspace(1) noundef nonnull align 8 dereferenceable(16) getelementptr inbounds nuw (i8, ptr addrspace(1) @page1, i64 16), i8 0, i64 16, i1 false)
+ ret void
+}
+
+declare void @llvm.memset.p1.i64(ptr addrspace(1) writeonly captures(none), i8, i64, i1 immarg)
+
+define dso_local void @memcpy_as() local_unnamed_addr {
+; CHECK-LABEL: define dso_local void @memcpy_as() local_unnamed_addr {
+; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 addrspacecast (ptr addrspace(1) getelementptr inbounds nuw (i8, ptr addrspace(1) @page2, i64 8) to ptr), ptr align 8 addrspacecast (ptr addrspace(1) getelementptr inbounds nuw (i8, ptr addrspace(1) @page1, i64 8) to ptr), i64 16, i1 false)
+; CHECK-NEXT: ret void
+;
+ tail call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) noundef nonnull align 8 dereferenceable(16) getelementptr inbounds nuw (i8, ptr addrspace(1) @page2, i64 8), ptr addrspace(1) noundef nonnull align 8 dereferenceable(16) getelementptr inbounds nuw (i8, ptr addrspace(1) @page1, i64 8), i64 16, i1 false)
+ ret void
+}
+
+declare void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) noalias writeonly captures(none), ptr addrspace(1) noalias readonly captures(none), i64, i1 immarg)
+
+define dso_local void @memmove_as() local_unnamed_addr {
+; CHECK-LABEL: define dso_local void @memmove_as() local_unnamed_addr {
+; CHECK-NEXT: call void @llvm.memmove.p0.p0.i64(ptr align 8 addrspacecast (ptr addrspace(1) getelementptr inbounds nuw (i8, ptr addrspace(1) @page2, i64 16) to ptr), ptr align 8 addrspacecast (ptr addrspace(1) getelementptr inbounds nuw (i8, ptr addrspace(1) @page2, i64 8) to ptr), i64 16, i1 false)
+; CHECK-NEXT: ret void
+;
+ tail call void @llvm.memmove.p1.p1.i64(ptr addrspace(1) noundef nonnull align 8 dereferenceable(16) getelementptr inbounds nuw (i8, ptr addrspace(1) @page2, i64 16), ptr addrspace(1) noundef nonnull align 8 dereferenceable(16) getelementptr inbounds nuw (i8, ptr addrspace(1) @page2, i64 8), i64 16, i1 false)
+ ret void
+}
+
+declare void @llvm.memmove.p1.p1.i64(ptr addrspace(1) writeonly captures(none), ptr addrspace(1) readonly captures(none), i64, i1 immarg)
diff --git a/llvm/test/CodeGen/BPF/addr-space-memintrinsic-no-gep.ll b/llvm/test/CodeGen/BPF/addr-space-memintrinsic-no-gep.ll
new file mode 100644
index 0000000000000..92b1316373bbf
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/addr-space-memintrinsic-no-gep.ll
@@ -0,0 +1,27 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt --bpf-check-and-opt-ir -S -mtriple=bpf-pc-linux < %s | FileCheck %s
+
+ at page1 = dso_local local_unnamed_addr addrspace(1) global [10 x ptr] zeroinitializer, align 8
+ at page2 = dso_local local_unnamed_addr addrspace(1) global [10 x ptr] zeroinitializer, align 8
+
+define dso_local void @memset_as() local_unnamed_addr {
+; CHECK-LABEL: define dso_local void @memset_as() local_unnamed_addr {
+; CHECK-NEXT: call void @llvm.memset.p0.i64(ptr align 8 addrspacecast (ptr addrspace(1) @page1 to ptr), i8 0, i64 16, i1 false)
+; CHECK-NEXT: ret void
+;
+ tail call void @llvm.memset.p1.i64(ptr addrspace(1) noundef align 8 dereferenceable(16) @page1, i8 0, i64 16, i1 false)
+ ret void
+}
+
+declare void @llvm.memset.p1.i64(ptr addrspace(1) writeonly captures(none), i8, i64, i1 immarg)
+
+define dso_local void @memcpy_as() local_unnamed_addr {
+; CHECK-LABEL: define dso_local void @memcpy_as() local_unnamed_addr {
+; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 addrspacecast (ptr addrspace(1) @page2 to ptr), ptr align 8 addrspacecast (ptr addrspace(1) @page1 to ptr), i64 16, i1 false)
+; CHECK-NEXT: ret void
+;
+ tail call void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) noundef align 8 dereferenceable(16) @page2, ptr addrspace(1) noundef align 8 dereferenceable(16) @page1, i64 16, i1 false)
+ ret void
+}
+
+declare void @llvm.memcpy.p1.p1.i64(ptr addrspace(1) noalias writeonly captures(none), ptr addrspace(1) noalias readonly captures(none), i64, i1 immarg)
More information about the llvm-commits
mailing list