[clang] 2aacb56 - BPF address space insn (#84410)
via cfe-commits
cfe-commits at lists.llvm.org
Tue Mar 12 17:27:29 PDT 2024
Author: 4ast
Date: 2024-03-13T02:27:25+02:00
New Revision: 2aacb56e8361213c1bd69c2ceafdea3aa0ca9125
URL: https://github.com/llvm/llvm-project/commit/2aacb56e8361213c1bd69c2ceafdea3aa0ca9125
DIFF: https://github.com/llvm/llvm-project/commit/2aacb56e8361213c1bd69c2ceafdea3aa0ca9125.diff
LOG: BPF address space insn (#84410)
This commit aims to support BPF arena kernel side
[feature](https://lore.kernel.org/bpf/20240209040608.98927-1-alexei.starovoitov@gmail.com/):
- arena is a memory region accessible from both BPF program and
userspace;
- base pointers for this memory region differ between kernel and user
spaces;
- `dst_reg = addr_space_cast(src_reg, dst_addr_space, src_addr_space)`
translates src_reg, a pointer in src_addr_space to dst_reg, equivalent
pointer in dst_addr_space, {src,dst}_addr_space are immediate constants;
- number 0 is assigned to kernel address space;
- number 1 is assigned to user address space.
On the LLVM side, the goal is to make load and store operations on arena
pointers "transparent" for BPF programs:
- assume that pointers with non-zero address space are pointers to
arena memory;
- assume that arena is identified by address space number;
- assume that address space zero corresponds to kernel address space;
- assume that every BPF-side load or store from arena is done via
pointer in user address space, thus convert base pointers using
`addr_space_cast(src_reg, 0, 1)`;
Only load, store, cmpxchg and atomicrmw IR instructions are handled by
this transformation.
For example, the following C code:
```c
#define __as __attribute__((address_space(1)))
void copy(int __as *from, int __as *to) { *to = *from; }
```
Compiled to the following IR:
```llvm
define void @copy(ptr addrspace(1) %from, ptr addrspace(1) %to) {
entry:
%0 = load i32, ptr addrspace(1) %from, align 4
store i32 %0, ptr addrspace(1) %to, align 4
ret void
}
```
Is transformed to:
```llvm
%to2 = addrspacecast ptr addrspace(1) %to to ptr ;; !
%from1 = addrspacecast ptr addrspace(1) %from to ptr ;; !
%0 = load i32, ptr %from1, align 4, !tbaa !3
store i32 %0, ptr %to2, align 4, !tbaa !3
ret void
```
And compiled as:
```asm
r2 = addr_space_cast(r2, 0, 1)
r1 = addr_space_cast(r1, 0, 1)
r1 = *(u32 *)(r1 + 0)
*(u32 *)(r2 + 0) = r1
exit
```
Co-authored-by: Eduard Zingerman <eddyz87 at gmail.com>
Added:
llvm/lib/Target/BPF/BPFASpaceCastSimplifyPass.cpp
llvm/test/CodeGen/BPF/addr-space-auto-casts.ll
llvm/test/CodeGen/BPF/addr-space-cast.ll
llvm/test/CodeGen/BPF/addr-space-gep-chain.ll
llvm/test/CodeGen/BPF/addr-space-globals.ll
llvm/test/CodeGen/BPF/addr-space-globals2.ll
llvm/test/CodeGen/BPF/addr-space-phi.ll
llvm/test/CodeGen/BPF/addr-space-simplify-1.ll
llvm/test/CodeGen/BPF/addr-space-simplify-2.ll
llvm/test/CodeGen/BPF/addr-space-simplify-3.ll
llvm/test/CodeGen/BPF/addr-space-simplify-4.ll
llvm/test/CodeGen/BPF/addr-space-simplify-5.ll
Modified:
clang/lib/Basic/Targets/BPF.cpp
clang/test/Preprocessor/bpf-predefined-macros.c
llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp
llvm/lib/Target/BPF/BPF.h
llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp
llvm/lib/Target/BPF/BPFInstrInfo.td
llvm/lib/Target/BPF/BPFTargetMachine.cpp
llvm/lib/Target/BPF/CMakeLists.txt
llvm/test/CodeGen/BPF/assembler-disassembler.s
Removed:
################################################################################
diff --git a/clang/lib/Basic/Targets/BPF.cpp b/clang/lib/Basic/Targets/BPF.cpp
index e3fbbb720d0694..26a54f631fcfc4 100644
--- a/clang/lib/Basic/Targets/BPF.cpp
+++ b/clang/lib/Basic/Targets/BPF.cpp
@@ -35,6 +35,9 @@ void BPFTargetInfo::getTargetDefines(const LangOptions &Opts,
Builder.defineMacro("__BPF_CPU_VERSION__", "0");
return;
}
+
+ Builder.defineMacro("__BPF_FEATURE_ARENA_CAST");
+
if (CPU.empty() || CPU == "generic" || CPU == "v1") {
Builder.defineMacro("__BPF_CPU_VERSION__", "1");
return;
diff --git a/clang/test/Preprocessor/bpf-predefined-macros.c b/clang/test/Preprocessor/bpf-predefined-macros.c
index ff4d00ac3bcfcc..fea24d1ea0ff7b 100644
--- a/clang/test/Preprocessor/bpf-predefined-macros.c
+++ b/clang/test/Preprocessor/bpf-predefined-macros.c
@@ -61,6 +61,9 @@ int r;
#ifdef __BPF_FEATURE_ST
int s;
#endif
+#ifdef __BPF_FEATURE_ARENA_CAST
+int t;
+#endif
// CHECK: int b;
// CHECK: int c;
@@ -90,6 +93,11 @@ int s;
// CPU_V4: int r;
// CPU_V4: int s;
+// CPU_V1: int t;
+// CPU_V2: int t;
+// CPU_V3: int t;
+// CPU_V4: int t;
+
// CPU_GENERIC: int g;
// CPU_PROBE: int f;
diff --git a/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp b/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp
index 0d1eef60c3b550..3145bc3d19f5dc 100644
--- a/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp
+++ b/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp
@@ -271,6 +271,7 @@ struct BPFOperand : public MCParsedAsmOperand {
.Case("xchg32_32", true)
.Case("cmpxchg_64", true)
.Case("cmpxchg32_32", true)
+ .Case("addr_space_cast", true)
.Default(false);
}
};
diff --git a/llvm/lib/Target/BPF/BPF.h b/llvm/lib/Target/BPF/BPF.h
index 5c77d183e1ef3d..bbdbdbbde53228 100644
--- a/llvm/lib/Target/BPF/BPF.h
+++ b/llvm/lib/Target/BPF/BPF.h
@@ -66,6 +66,14 @@ class BPFIRPeepholePass : public PassInfoMixin<BPFIRPeepholePass> {
static bool isRequired() { return true; }
};
+class BPFASpaceCastSimplifyPass
+ : public PassInfoMixin<BPFASpaceCastSimplifyPass> {
+public:
+ PreservedAnalyses run(Function &F, FunctionAnalysisManager &AM);
+
+ static bool isRequired() { return true; }
+};
+
class BPFAdjustOptPass : public PassInfoMixin<BPFAdjustOptPass> {
public:
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
diff --git a/llvm/lib/Target/BPF/BPFASpaceCastSimplifyPass.cpp b/llvm/lib/Target/BPF/BPFASpaceCastSimplifyPass.cpp
new file mode 100644
index 00000000000000..f87b299bbba658
--- /dev/null
+++ b/llvm/lib/Target/BPF/BPFASpaceCastSimplifyPass.cpp
@@ -0,0 +1,92 @@
+//===-- BPFASpaceCastSimplifyPass.cpp - BPF addrspacecast simplications --===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "BPF.h"
+#include <optional>
+
+#define DEBUG_TYPE "bpf-aspace-simplify"
+
+using namespace llvm;
+
+namespace {
+
+struct CastGEPCast {
+ AddrSpaceCastInst *OuterCast;
+
+ // Match chain of instructions:
+ // %inner = addrspacecast N->M
+ // %gep = getelementptr %inner, ...
+ // %outer = addrspacecast M->N %gep
+ // Where I is %outer.
+ static std::optional<CastGEPCast> match(Value *I) {
+ auto *OuterCast = dyn_cast<AddrSpaceCastInst>(I);
+ if (!OuterCast)
+ return std::nullopt;
+ auto *GEP = dyn_cast<GetElementPtrInst>(OuterCast->getPointerOperand());
+ if (!GEP)
+ return std::nullopt;
+ auto *InnerCast = dyn_cast<AddrSpaceCastInst>(GEP->getPointerOperand());
+ if (!InnerCast)
+ return std::nullopt;
+ if (InnerCast->getSrcAddressSpace() != OuterCast->getDestAddressSpace())
+ return std::nullopt;
+ if (InnerCast->getDestAddressSpace() != OuterCast->getSrcAddressSpace())
+ return std::nullopt;
+ return CastGEPCast{OuterCast};
+ }
+
+ static PointerType *changeAddressSpace(PointerType *Ty, unsigned AS) {
+ return Ty->get(Ty->getContext(), AS);
+ }
+
+ // Assuming match(this->OuterCast) is true, convert:
+ // (addrspacecast M->N (getelementptr (addrspacecast N->M ptr) ...))
+ // To:
+ // (getelementptr ptr ...)
+ GetElementPtrInst *rewrite() {
+ auto *GEP = cast<GetElementPtrInst>(OuterCast->getPointerOperand());
+ auto *InnerCast = cast<AddrSpaceCastInst>(GEP->getPointerOperand());
+ unsigned AS = OuterCast->getDestAddressSpace();
+ auto *NewGEP = cast<GetElementPtrInst>(GEP->clone());
+ NewGEP->setName(GEP->getName());
+ NewGEP->insertAfter(OuterCast);
+ NewGEP->setOperand(0, InnerCast->getPointerOperand());
+ auto *GEPTy = cast<PointerType>(GEP->getType());
+ NewGEP->mutateType(changeAddressSpace(GEPTy, AS));
+ OuterCast->replaceAllUsesWith(NewGEP);
+ OuterCast->eraseFromParent();
+ if (GEP->use_empty())
+ GEP->eraseFromParent();
+ if (InnerCast->use_empty())
+ InnerCast->eraseFromParent();
+ return NewGEP;
+ }
+};
+
+} // anonymous namespace
+
+PreservedAnalyses BPFASpaceCastSimplifyPass::run(Function &F,
+ FunctionAnalysisManager &AM) {
+ SmallVector<CastGEPCast, 16> WorkList;
+ bool Changed = false;
+ for (BasicBlock &BB : F) {
+ for (Instruction &I : BB)
+ if (auto It = CastGEPCast::match(&I))
+ WorkList.push_back(It.value());
+ Changed |= !WorkList.empty();
+
+ while (!WorkList.empty()) {
+ CastGEPCast InsnChain = WorkList.pop_back_val();
+ GetElementPtrInst *NewGEP = InsnChain.rewrite();
+ for (User *U : NewGEP->users())
+ if (auto It = CastGEPCast::match(U))
+ WorkList.push_back(It.value());
+ }
+ }
+ return Changed ? PreservedAnalyses::none() : PreservedAnalyses::all();
+}
diff --git a/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp b/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp
index 81effc9b1db46c..edd59aaa6d01d2 100644
--- a/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp
+++ b/llvm/lib/Target/BPF/BPFCheckAndAdjustIR.cpp
@@ -14,6 +14,8 @@
// optimizations are done and those builtins can be removed.
// - remove llvm.bpf.getelementptr.and.load builtins.
// - remove llvm.bpf.getelementptr.and.store builtins.
+// - for loads and stores with base addresses from non-zero address space
+// cast base address to zero address space (support for BPF arenas).
//
//===----------------------------------------------------------------------===//
@@ -55,6 +57,7 @@ class BPFCheckAndAdjustIR final : public ModulePass {
bool removeCompareBuiltin(Module &M);
bool sinkMinMax(Module &M);
bool removeGEPBuiltins(Module &M);
+ bool insertASpaceCasts(Module &M);
};
} // End anonymous namespace
@@ -416,11 +419,124 @@ bool BPFCheckAndAdjustIR::removeGEPBuiltins(Module &M) {
return Changed;
}
+// Wrap ToWrap with cast to address space zero:
+// - if ToWrap is a getelementptr,
+// wrap it's base pointer instead and return a copy;
+// - if ToWrap is Instruction, insert address space cast
+// immediately after ToWrap;
+// - if ToWrap is not an Instruction (function parameter
+// or a global value), insert address space cast at the
+// beginning of the Function F;
+// - use Cache to avoid inserting too many casts;
+static Value *aspaceWrapValue(DenseMap<Value *, Value *> &Cache, Function *F,
+ Value *ToWrap) {
+ auto It = Cache.find(ToWrap);
+ if (It != Cache.end())
+ return It->getSecond();
+
+ if (auto *GEP = dyn_cast<GetElementPtrInst>(ToWrap)) {
+ Value *Ptr = GEP->getPointerOperand();
+ Value *WrappedPtr = aspaceWrapValue(Cache, F, Ptr);
+ auto *GEPTy = cast<PointerType>(GEP->getType());
+ auto *NewGEP = GEP->clone();
+ NewGEP->insertAfter(GEP);
+ NewGEP->mutateType(GEPTy->getPointerTo(0));
+ NewGEP->setOperand(GEP->getPointerOperandIndex(), WrappedPtr);
+ NewGEP->setName(GEP->getName());
+ Cache[ToWrap] = NewGEP;
+ return NewGEP;
+ }
+
+ IRBuilder IB(F->getContext());
+ if (Instruction *InsnPtr = dyn_cast<Instruction>(ToWrap))
+ IB.SetInsertPoint(*InsnPtr->getInsertionPointAfterDef());
+ else
+ IB.SetInsertPoint(F->getEntryBlock().getFirstInsertionPt());
+ auto *PtrTy = cast<PointerType>(ToWrap->getType());
+ auto *ASZeroPtrTy = PtrTy->getPointerTo(0);
+ auto *ACast = IB.CreateAddrSpaceCast(ToWrap, ASZeroPtrTy, ToWrap->getName());
+ Cache[ToWrap] = ACast;
+ return ACast;
+}
+
+// Wrap a pointer operand OpNum of instruction I
+// with cast to address space zero
+static void aspaceWrapOperand(DenseMap<Value *, Value *> &Cache, Instruction *I,
+ unsigned OpNum) {
+ Value *OldOp = I->getOperand(OpNum);
+ if (OldOp->getType()->getPointerAddressSpace() == 0)
+ return;
+
+ Value *NewOp = aspaceWrapValue(Cache, I->getFunction(), OldOp);
+ I->setOperand(OpNum, NewOp);
+ // Check if there are any remaining users of old GEP,
+ // delete those w/o users
+ for (;;) {
+ auto *OldGEP = dyn_cast<GetElementPtrInst>(OldOp);
+ if (!OldGEP)
+ break;
+ if (!OldGEP->use_empty())
+ break;
+ OldOp = OldGEP->getPointerOperand();
+ OldGEP->eraseFromParent();
+ }
+}
+
+// Support for BPF arenas:
+// - for each function in the module M, update pointer operand of
+// each memory access instruction (load/store/cmpxchg/atomicrmw)
+// by casting it from non-zero address space to zero address space, e.g:
+//
+// (load (ptr addrspace (N) %p) ...)
+// -> (load (addrspacecast ptr addrspace (N) %p to ptr))
+//
+// - assign section with name .arena.N for globals defined in
+// non-zero address space N
+bool BPFCheckAndAdjustIR::insertASpaceCasts(Module &M) {
+ bool Changed = false;
+ for (Function &F : M) {
+ DenseMap<Value *, Value *> CastsCache;
+ for (BasicBlock &BB : F) {
+ for (Instruction &I : BB) {
+ unsigned PtrOpNum;
+
+ if (auto *LD = dyn_cast<LoadInst>(&I))
+ PtrOpNum = LD->getPointerOperandIndex();
+ else if (auto *ST = dyn_cast<StoreInst>(&I))
+ PtrOpNum = ST->getPointerOperandIndex();
+ else if (auto *CmpXchg = dyn_cast<AtomicCmpXchgInst>(&I))
+ PtrOpNum = CmpXchg->getPointerOperandIndex();
+ else if (auto *RMW = dyn_cast<AtomicRMWInst>(&I))
+ PtrOpNum = RMW->getPointerOperandIndex();
+ else
+ continue;
+
+ aspaceWrapOperand(CastsCache, &I, PtrOpNum);
+ }
+ }
+ Changed |= !CastsCache.empty();
+ }
+ // Merge all globals within same address space into single
+ // .arena.<addr space no> section
+ for (GlobalVariable &G : M.globals()) {
+ if (G.getAddressSpace() == 0 || G.hasSection())
+ continue;
+ SmallString<16> SecName;
+ raw_svector_ostream OS(SecName);
+ OS << ".arena." << G.getAddressSpace();
+ G.setSection(SecName);
+ // Prevent having separate section for constants
+ G.setConstant(false);
+ }
+ return Changed;
+}
+
bool BPFCheckAndAdjustIR::adjustIR(Module &M) {
bool Changed = removePassThroughBuiltin(M);
Changed = removeCompareBuiltin(M) || Changed;
Changed = sinkMinMax(M) || Changed;
Changed = removeGEPBuiltins(M) || Changed;
+ Changed = insertASpaceCasts(M) || Changed;
return Changed;
}
diff --git a/llvm/lib/Target/BPF/BPFInstrInfo.td b/llvm/lib/Target/BPF/BPFInstrInfo.td
index 82d34702310668..7198e9499bc32a 100644
--- a/llvm/lib/Target/BPF/BPFInstrInfo.td
+++ b/llvm/lib/Target/BPF/BPFInstrInfo.td
@@ -420,6 +420,35 @@ let Predicates = [BPFHasMovsx] in {
}
}
+def ADDR_SPACE_CAST
+ : ALU_RR<BPF_ALU64, BPF_MOV, 64,
+ (outs GPR:$dst),
+ (ins GPR:$src, i64imm:$dst_as, i64imm:$src_as),
+ "$dst = addr_space_cast($src, $dst_as, $src_as)",
+ []> {
+ bits<64> dst_as;
+ bits<64> src_as;
+
+ let Inst{47-32} = 1;
+ let Inst{31-16} = dst_as{15-0};
+ let Inst{15-0} = src_as{15-0};
+}
+
+def SrcAddrSpace : SDNodeXForm<addrspacecast, [{
+ return CurDAG->getTargetConstant(
+ cast<AddrSpaceCastSDNode>(N)->getSrcAddressSpace(),
+ SDLoc(N), MVT::i64);
+}]>;
+
+def DstAddrSpace : SDNodeXForm<addrspacecast, [{
+ return CurDAG->getTargetConstant(
+ cast<AddrSpaceCastSDNode>(N)->getDestAddressSpace(),
+ SDLoc(N), MVT::i64);
+}]>;
+
+def : Pat<(addrspacecast:$this GPR:$src),
+ (ADDR_SPACE_CAST $src, (DstAddrSpace $this), (SrcAddrSpace $this))>;
+
def FI_ri
: TYPE_LD_ST<BPF_IMM.Value, BPF_DW.Value,
(outs GPR:$dst),
diff --git a/llvm/lib/Target/BPF/BPFTargetMachine.cpp b/llvm/lib/Target/BPF/BPFTargetMachine.cpp
index 08ac4b25540f70..5f26bec2e390c8 100644
--- a/llvm/lib/Target/BPF/BPFTargetMachine.cpp
+++ b/llvm/lib/Target/BPF/BPFTargetMachine.cpp
@@ -121,6 +121,10 @@ void BPFTargetMachine::registerPassBuilderCallbacks(
FPM.addPass(BPFPreserveStaticOffsetPass(false));
return true;
}
+ if (PassName == "bpf-aspace-simplify") {
+ FPM.addPass(BPFASpaceCastSimplifyPass());
+ return true;
+ }
return false;
});
PB.registerPipelineStartEPCallback(
@@ -135,6 +139,7 @@ void BPFTargetMachine::registerPassBuilderCallbacks(
PB.registerPeepholeEPCallback([=](FunctionPassManager &FPM,
OptimizationLevel Level) {
FPM.addPass(SimplifyCFGPass(SimplifyCFGOptions().hoistCommonInsts(true)));
+ FPM.addPass(BPFASpaceCastSimplifyPass());
});
PB.registerScalarOptimizerLateEPCallback(
[=](FunctionPassManager &FPM, OptimizationLevel Level) {
diff --git a/llvm/lib/Target/BPF/CMakeLists.txt b/llvm/lib/Target/BPF/CMakeLists.txt
index d88e7ade40b9a0..cb21ed03a86c1e 100644
--- a/llvm/lib/Target/BPF/CMakeLists.txt
+++ b/llvm/lib/Target/BPF/CMakeLists.txt
@@ -24,6 +24,7 @@ add_llvm_target(BPFCodeGen
BPFAbstractMemberAccess.cpp
BPFAdjustOpt.cpp
BPFAsmPrinter.cpp
+ BPFASpaceCastSimplifyPass.cpp
BPFCheckAndAdjustIR.cpp
BPFFrameLowering.cpp
BPFInstrInfo.cpp
diff --git a/llvm/test/CodeGen/BPF/addr-space-auto-casts.ll b/llvm/test/CodeGen/BPF/addr-space-auto-casts.ll
new file mode 100644
index 00000000000000..08e11e861c71cb
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/addr-space-auto-casts.ll
@@ -0,0 +1,78 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt --bpf-check-and-opt-ir -S -mtriple=bpf-pc-linux < %s | FileCheck %s
+
+define void @simple_store(ptr addrspace(272) %foo) {
+; CHECK-LABEL: define void @simple_store(
+; CHECK-SAME: ptr addrspace(272) [[FOO:%.*]]) {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[FOO1:%.*]] = addrspacecast ptr addrspace(272) [[FOO]] to ptr
+; CHECK-NEXT: [[ADD_PTR2:%.*]] = getelementptr inbounds i8, ptr [[FOO1]], i64 16
+; CHECK-NEXT: store volatile i32 57005, ptr [[ADD_PTR2]], align 4
+; CHECK-NEXT: [[ADD_PTR13:%.*]] = getelementptr inbounds i8, ptr [[FOO1]], i64 12
+; CHECK-NEXT: store volatile i32 48879, ptr [[ADD_PTR13]], align 4
+; CHECK-NEXT: ret void
+;
+entry:
+ %add.ptr = getelementptr inbounds i8, ptr addrspace(272) %foo, i64 16
+ store volatile i32 57005, ptr addrspace(272) %add.ptr, align 4
+ %add.ptr1 = getelementptr inbounds i8, ptr addrspace(272) %foo, i64 12
+ store volatile i32 48879, ptr addrspace(272) %add.ptr1, align 4
+ ret void
+}
+
+define void @separate_addr_store(ptr addrspace(272) %foo, ptr addrspace(272) %bar) {
+; CHECK-LABEL: define void @separate_addr_store(
+; CHECK-SAME: ptr addrspace(272) [[FOO:%.*]], ptr addrspace(272) [[BAR:%.*]]) {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[BAR3:%.*]] = addrspacecast ptr addrspace(272) [[BAR]] to ptr
+; CHECK-NEXT: [[FOO1:%.*]] = addrspacecast ptr addrspace(272) [[FOO]] to ptr
+; CHECK-NEXT: [[ADD_PTR2:%.*]] = getelementptr inbounds i8, ptr [[FOO1]], i64 16
+; CHECK-NEXT: store volatile i32 57005, ptr [[ADD_PTR2]], align 4
+; CHECK-NEXT: [[ADD_PTR14:%.*]] = getelementptr inbounds i8, ptr [[BAR3]], i64 12
+; CHECK-NEXT: store volatile i32 48879, ptr [[ADD_PTR14]], align 4
+; CHECK-NEXT: ret void
+;
+entry:
+ %add.ptr = getelementptr inbounds i8, ptr addrspace(272) %foo, i64 16
+ store volatile i32 57005, ptr addrspace(272) %add.ptr, align 4
+ %add.ptr1 = getelementptr inbounds i8, ptr addrspace(272) %bar, i64 12
+ store volatile i32 48879, ptr addrspace(272) %add.ptr1, align 4
+ ret void
+}
+
+define i32 @simple_load(ptr addrspace(272) %foo) {
+; CHECK-LABEL: define i32 @simple_load(
+; CHECK-SAME: ptr addrspace(272) [[FOO:%.*]]) {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[FOO1:%.*]] = addrspacecast ptr addrspace(272) [[FOO]] to ptr
+; CHECK-NEXT: [[TMP0:%.*]] = load i32, ptr [[FOO1]], align 4
+; CHECK-NEXT: ret i32 [[TMP0]]
+;
+entry:
+ %0 = load i32, ptr addrspace(272) %foo, align 4
+ ret i32 %0
+}
+
+define { i32, i1 } @simple_cmpxchg(ptr addrspace(1) %i) {
+; CHECK-LABEL: define { i32, i1 } @simple_cmpxchg(
+; CHECK-SAME: ptr addrspace(1) [[I:%.*]]) {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[I1:%.*]] = addrspacecast ptr addrspace(1) [[I]] to ptr
+; CHECK-NEXT: [[A:%.*]] = cmpxchg ptr [[I1]], i32 7, i32 42 monotonic monotonic, align 4
+; CHECK-NEXT: ret { i32, i1 } [[A]]
+;
+entry:
+ %a = cmpxchg ptr addrspace(1) %i, i32 7, i32 42 monotonic monotonic, align 4
+ ret { i32, i1 } %a
+}
+
+define void @simple_atomicrmw(ptr addrspace(1) %p) {
+; CHECK-LABEL: define void @simple_atomicrmw(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) {
+; CHECK-NEXT: [[P1:%.*]] = addrspacecast ptr addrspace(1) [[P]] to ptr
+; CHECK-NEXT: [[A:%.*]] = atomicrmw add ptr [[P1]], i64 42 monotonic, align 8
+; CHECK-NEXT: ret void
+;
+ %a = atomicrmw add ptr addrspace(1) %p, i64 42 monotonic, align 8
+ ret void
+}
diff --git a/llvm/test/CodeGen/BPF/addr-space-cast.ll b/llvm/test/CodeGen/BPF/addr-space-cast.ll
new file mode 100644
index 00000000000000..ad2860d8038ea7
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/addr-space-cast.ll
@@ -0,0 +1,22 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4
+; RUN: llc -march=bpfel -mcpu=v4 -filetype=asm -show-mc-encoding < %s | FileCheck %s
+
+define ptr addrspace(1) @foo(ptr %p) {
+; CHECK-LABEL: foo:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: r0 = addr_space_cast(r1, 1, 0) # encoding: [0xbf,0x10,0x01,0x00,0x00,0x00,0x01,0x00]
+; CHECK-NEXT: exit # encoding: [0x95,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
+entry:
+ %0 = addrspacecast ptr %p to ptr addrspace(1)
+ ret ptr addrspace(1) %0
+}
+
+define ptr @bar(ptr addrspace(1) %p) {
+; CHECK-LABEL: bar:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: r0 = addr_space_cast(r1, 0, 1) # encoding: [0xbf,0x10,0x01,0x00,0x01,0x00,0x00,0x00]
+; CHECK-NEXT: exit # encoding: [0x95,0x00,0x00,0x00,0x00,0x00,0x00,0x00]
+entry:
+ %0 = addrspacecast ptr addrspace(1) %p to ptr
+ ret ptr %0
+}
diff --git a/llvm/test/CodeGen/BPF/addr-space-gep-chain.ll b/llvm/test/CodeGen/BPF/addr-space-gep-chain.ll
new file mode 100644
index 00000000000000..3ac85fb9b12662
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/addr-space-gep-chain.ll
@@ -0,0 +1,25 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt --bpf-check-and-opt-ir -S -mtriple=bpf-pc-linux < %s | FileCheck %s
+
+define void @test(ptr addrspace(1) %p) {
+; CHECK-LABEL: define void @test(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[P1:%.*]] = addrspacecast ptr addrspace(1) [[P]] to ptr
+; CHECK-NEXT: [[A2:%.*]] = getelementptr inbounds i8, ptr [[P1]], i64 8
+; CHECK-NEXT: [[B3:%.*]] = getelementptr inbounds i8, ptr [[A2]], i64 16
+; CHECK-NEXT: [[C4:%.*]] = getelementptr inbounds i8, ptr [[B3]], i64 24
+; CHECK-NEXT: [[D5:%.*]] = getelementptr inbounds i8, ptr [[C4]], i64 32
+; CHECK-NEXT: store i64 11, ptr [[C4]], align 8
+; CHECK-NEXT: store i64 22, ptr [[D5]], align 8
+; CHECK-NEXT: ret void
+;
+entry:
+ %a = getelementptr inbounds i8, ptr addrspace(1) %p, i64 8
+ %b = getelementptr inbounds i8, ptr addrspace(1) %a, i64 16
+ %c = getelementptr inbounds i8, ptr addrspace(1) %b, i64 24
+ %d = getelementptr inbounds i8, ptr addrspace(1) %c, i64 32
+ store i64 11, ptr addrspace(1) %c, align 8
+ store i64 22, ptr addrspace(1) %d, align 8
+ ret void
+}
diff --git a/llvm/test/CodeGen/BPF/addr-space-globals.ll b/llvm/test/CodeGen/BPF/addr-space-globals.ll
new file mode 100644
index 00000000000000..878ba0dfce6cd1
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/addr-space-globals.ll
@@ -0,0 +1,30 @@
+; RUN: llc -march=bpfel -mcpu=v4 < %s | FileCheck %s
+
+; Generated from the following C code:
+;
+; #define __as __attribute__((address_space(272)))
+; __as const char a[2] = {1,2};
+; __as char b[2] = {3,4};
+; __as char c[2];
+;
+; Using the following command:
+;
+; clang --target=bpf -O2 -S -emit-llvm -o t.ll t.c
+
+
+ at a = dso_local local_unnamed_addr addrspace(272) constant [2 x i8] [i8 1, i8 2], align 1
+ at b = dso_local local_unnamed_addr addrspace(272) global [2 x i8] [i8 3, i8 4], align 1
+ at c = dso_local local_unnamed_addr addrspace(272) global [2 x i8] zeroinitializer, align 1
+
+; Verify that a,b,c reside in the same section
+
+; CHECK: .section .arena.272,"aw", at progbits
+; CHECK-NOT: .section
+; CHECK: .globl a
+; CHECK: .ascii "\001\002"
+; CHECK-NOT: .section
+; CHECK: .globl b
+; CHECK: .ascii "\003\004"
+; CHECK-NOT: .section
+; CHECK: .globl c
+; CHECK: .zero 2
diff --git a/llvm/test/CodeGen/BPF/addr-space-globals2.ll b/llvm/test/CodeGen/BPF/addr-space-globals2.ll
new file mode 100644
index 00000000000000..d1e2318948751e
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/addr-space-globals2.ll
@@ -0,0 +1,25 @@
+; RUN: llc -march=bpfel -mcpu=v4 < %s | FileCheck %s
+
+; Generated from the following C code:
+;
+; __attribute__((address_space(1))) char a[2] = {1,2};
+; __attribute__((address_space(2))) char b[2] = {3,4};
+;
+; Using the following command:
+;
+; clang --target=bpf -O2 -S -emit-llvm -o t.ll t.c
+
+ at a = dso_local local_unnamed_addr addrspace(1) global [2 x i8] [i8 1, i8 2], align 1
+ at b = dso_local local_unnamed_addr addrspace(2) global [2 x i8] [i8 3, i8 4], align 1
+
+; Verify that a,b reside in separate sections
+
+; CHECK: .section .arena.1,"aw", at progbits
+; CHECK-NOT: .section
+; CHECK: .globl a
+; CHECK: .ascii "\001\002"
+
+; CHECK: .section .arena.2,"aw", at progbits
+; CHECK-NOT: .section
+; CHECK: .globl b
+; CHECK: .ascii "\003\004"
diff --git a/llvm/test/CodeGen/BPF/addr-space-phi.ll b/llvm/test/CodeGen/BPF/addr-space-phi.ll
new file mode 100644
index 00000000000000..6d28b071f28086
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/addr-space-phi.ll
@@ -0,0 +1,53 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt --bpf-check-and-opt-ir -S -mtriple=bpf-pc-linux < %s | FileCheck %s
+
+; Generated from the following C code:
+;
+; #define __uptr __attribute__((address_space(1)))
+;
+; extern int __uptr *magic1();
+; extern int __uptr *magic2();
+;
+; void test(long i) {
+; int __uptr *a;
+;
+; if (i > 42)
+; a = magic1();
+; else
+; a = magic2();
+; a[5] = 7;
+; }
+;
+; Using the following command:
+;
+; clang --target=bpf -O2 -S -emit-llvm -o t.ll t.c
+
+define void @test(i64 noundef %i) {
+; CHECK: if.end:
+; CHECK-NEXT: [[A_0:%.*]] = phi ptr addrspace(1)
+; CHECK-NEXT: [[A_01:%.*]] = addrspacecast ptr addrspace(1) [[A_0]] to ptr
+; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[A_01]], i64 5
+; CHECK-NEXT: store i32 7, ptr [[ARRAYIDX2]], align 4
+; CHECK-NEXT: ret void
+;
+entry:
+ %cmp = icmp sgt i64 %i, 42
+ br i1 %cmp, label %if.then, label %if.else
+
+if.then: ; preds = %entry
+ %call = tail call ptr addrspace(1) @magic1()
+ br label %if.end
+
+if.else: ; preds = %entry
+ %call1 = tail call ptr addrspace(1) @magic2()
+ br label %if.end
+
+if.end: ; preds = %if.else, %if.then
+ %a.0 = phi ptr addrspace(1) [ %call, %if.then ], [ %call1, %if.else ]
+ %arrayidx = getelementptr inbounds i32, ptr addrspace(1) %a.0, i64 5
+ store i32 7, ptr addrspace(1) %arrayidx, align 4
+ ret void
+}
+
+declare ptr addrspace(1) @magic1(...)
+declare ptr addrspace(1) @magic2(...)
diff --git a/llvm/test/CodeGen/BPF/addr-space-simplify-1.ll b/llvm/test/CodeGen/BPF/addr-space-simplify-1.ll
new file mode 100644
index 00000000000000..32d67284d1c1b7
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/addr-space-simplify-1.ll
@@ -0,0 +1,19 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt -passes=bpf-aspace-simplify -mtriple=bpf-pc-linux -S < %s | FileCheck %s
+
+; Check that bpf-aspace-simplify pass removes unnecessary (for BPF)
+; address space casts for cast M->N -> GEP -> cast N->M chain.
+
+define dso_local ptr addrspace(1) @test (ptr addrspace(1) %p) {
+; CHECK-LABEL: define dso_local ptr addrspace(1) @test(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[B1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[P]], i64 8
+; CHECK-NEXT: ret ptr addrspace(1) [[B1]]
+;
+ entry:
+ %a = addrspacecast ptr addrspace(1) %p to ptr
+ %b = getelementptr inbounds i8, ptr %a, i64 8
+ %c = addrspacecast ptr %b to ptr addrspace(1)
+ ret ptr addrspace(1) %c
+}
diff --git a/llvm/test/CodeGen/BPF/addr-space-simplify-2.ll b/llvm/test/CodeGen/BPF/addr-space-simplify-2.ll
new file mode 100644
index 00000000000000..a2965554a97330
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/addr-space-simplify-2.ll
@@ -0,0 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt -passes=bpf-aspace-simplify -mtriple=bpf-pc-linux -S < %s | FileCheck %s
+
+; Check that bpf-aspace-simplify pass does not change
+; chain 'cast M->N -> GEP -> cast N->K'.
+
+define dso_local ptr addrspace(2) @test (ptr addrspace(1) %p) {
+; CHECK-LABEL: define dso_local ptr addrspace(2) @test(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[A:%.*]] = addrspacecast ptr addrspace(1) [[P]] to ptr
+; CHECK-NEXT: [[B:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 8
+; CHECK-NEXT: [[C:%.*]] = addrspacecast ptr [[B]] to ptr addrspace(2)
+; CHECK-NEXT: ret ptr addrspace(2) [[C]]
+;
+ entry:
+ %a = addrspacecast ptr addrspace(1) %p to ptr
+ %b = getelementptr inbounds i8, ptr %a, i64 8
+ %c = addrspacecast ptr %b to ptr addrspace(2)
+ ret ptr addrspace(2) %c
+}
diff --git a/llvm/test/CodeGen/BPF/addr-space-simplify-3.ll b/llvm/test/CodeGen/BPF/addr-space-simplify-3.ll
new file mode 100644
index 00000000000000..a7736c462b44b3
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/addr-space-simplify-3.ll
@@ -0,0 +1,26 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt -passes=bpf-aspace-simplify -mtriple=bpf-pc-linux -S < %s | FileCheck %s
+
+; Check that when bpf-aspace-simplify pass modifies chain
+; 'cast M->N -> GEP -> cast N->M' it does not remove GEP,
+; when that GEP is used by some other instruction.
+
+define dso_local ptr addrspace(1) @test (ptr addrspace(1) %p) {
+; CHECK-LABEL: define dso_local ptr addrspace(1) @test(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[A:%.*]] = addrspacecast ptr addrspace(1) [[P]] to ptr
+; CHECK-NEXT: [[B:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 8
+; CHECK-NEXT: [[B1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[P]], i64 8
+; CHECK-NEXT: call void @sink(ptr [[B]])
+; CHECK-NEXT: ret ptr addrspace(1) [[B1]]
+;
+ entry:
+ %a = addrspacecast ptr addrspace(1) %p to ptr
+ %b = getelementptr inbounds i8, ptr %a, i64 8
+ %c = addrspacecast ptr %b to ptr addrspace(1)
+ call void @sink(ptr %b)
+ ret ptr addrspace(1) %c
+}
+
+declare dso_local void @sink(ptr)
diff --git a/llvm/test/CodeGen/BPF/addr-space-simplify-4.ll b/llvm/test/CodeGen/BPF/addr-space-simplify-4.ll
new file mode 100644
index 00000000000000..b2c384bbb6abd3
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/addr-space-simplify-4.ll
@@ -0,0 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt -passes=bpf-aspace-simplify -mtriple=bpf-pc-linux -S < %s | FileCheck %s
+
+; Check that bpf-aspace-simplify pass simplifies chain
+; 'cast K->M -> cast M->N -> GEP -> cast N->M -> cast M->K'.
+
+define dso_local ptr addrspace(2) @test (ptr addrspace(2) %p) {
+; CHECK-LABEL: define dso_local ptr addrspace(2) @test(
+; CHECK-SAME: ptr addrspace(2) [[P:%.*]]) {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: [[C12:%.*]] = getelementptr inbounds i8, ptr addrspace(2) [[P]], i64 8
+; CHECK-NEXT: ret ptr addrspace(2) [[C12]]
+;
+ entry:
+ %a = addrspacecast ptr addrspace(2) %p to ptr addrspace(1)
+ %b = addrspacecast ptr addrspace(1) %a to ptr
+ %c = getelementptr inbounds i8, ptr %b, i64 8
+ %d = addrspacecast ptr %c to ptr addrspace(1)
+ %e = addrspacecast ptr addrspace (1) %d to ptr addrspace(2)
+ ret ptr addrspace(2) %e
+}
diff --git a/llvm/test/CodeGen/BPF/addr-space-simplify-5.ll b/llvm/test/CodeGen/BPF/addr-space-simplify-5.ll
new file mode 100644
index 00000000000000..b62d25384d9583
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/addr-space-simplify-5.ll
@@ -0,0 +1,25 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4
+; RUN: opt -passes=bpf-aspace-simplify -mtriple=bpf-pc-linux -S < %s | FileCheck %s
+
+; Check that bpf-aspace-simplify pass removes unnecessary (for BPF)
+; address space casts for cast M->N -> GEP -> cast N->M chain,
+; where chain is split between several BBs.
+
+define dso_local ptr addrspace(1) @test (ptr addrspace(1) %p) {
+; CHECK-LABEL: define dso_local ptr addrspace(1) @test(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]]) {
+; CHECK-NEXT: entry:
+; CHECK-NEXT: br label [[EXIT:%.*]]
+; CHECK: exit:
+; CHECK-NEXT: [[B1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[P]], i64 8
+; CHECK-NEXT: ret ptr addrspace(1) [[B1]]
+;
+entry:
+ %a = addrspacecast ptr addrspace(1) %p to ptr
+ %b = getelementptr inbounds i8, ptr %a, i64 8
+ br label %exit
+
+exit:
+ %c = addrspacecast ptr %b to ptr addrspace(1)
+ ret ptr addrspace(1) %c
+}
diff --git a/llvm/test/CodeGen/BPF/assembler-disassembler.s b/llvm/test/CodeGen/BPF/assembler-disassembler.s
index 2bc7421c2471c2..991d6edc683a30 100644
--- a/llvm/test/CodeGen/BPF/assembler-disassembler.s
+++ b/llvm/test/CodeGen/BPF/assembler-disassembler.s
@@ -289,3 +289,10 @@ r0 = *(u32*)skb[42]
r0 = *(u8*)skb[r1]
r0 = *(u16*)skb[r1]
r0 = *(u32*)skb[r1]
+
+// CHECK: bf 10 01 00 01 00 00 00 r0 = addr_space_cast(r1, 0x0, 0x1)
+// CHECK: bf 21 01 00 00 00 01 00 r1 = addr_space_cast(r2, 0x1, 0x0)
+// CHECK: bf 43 01 00 2a 00 07 00 r3 = addr_space_cast(r4, 0x7, 0x2a)
+r0 = addr_space_cast(r1, 0, 1)
+r1 = addr_space_cast(r2, 1, 0)
+r3 = addr_space_cast(r4, 7, 42)
More information about the cfe-commits
mailing list