[llvm] [BPF] Support Jump Table (PR #149715)
via llvm-commits
llvm-commits at lists.llvm.org
Sun Jul 20 08:57:12 PDT 2025
https://github.com/yonghong-song updated https://github.com/llvm/llvm-project/pull/149715
>From 3d7c4784baab30e8196398c58acfa3cc3d28121c Mon Sep 17 00:00:00 2001
From: Yonghong Song <yonghong.song at linux.dev>
Date: Mon, 31 Mar 2025 21:25:26 -0700
Subject: [PATCH 1/4] [BPF] Add jump table support with switch statements and
computed goto
NOTE 1: We probably need cpu v5 or other flags to enable this feature.
We can add it later when necessary.
NOTE 2: An option -bpf-min-jump-table-entries is implemented to control the minimum
number of entries to use a jump table on BPF. The default value 5 and this is
to make it easy to test. Eventually we will increase min jump table entries to be 13.
This patch adds jump table support. A new insn 'gotox <reg>' is
added to allow goto through a register. The register represents
the address in the current section.
Example 1 (switch statement):
=============================
Code:
struct simple_ctx {
int x;
int y;
int z;
};
int ret_user, ret_user2;
void bar(void);
int foo(struct simple_ctx *ctx, struct simple_ctx *ctx2)
{
switch (ctx->x) {
case 1: ret_user = 18; break;
case 20: ret_user = 6; break;
case 16: ret_user = 9; break;
case 6: ret_user = 16; break;
case 8: ret_user = 14; break;
case 30: ret_user = 2; break;
default: ret_user = 1; break;
}
bar();
switch (ctx2->x) {
case 0: ret_user2 = 8; break;
case 31: ret_user2 = 5; break;
case 13: ret_user2 = 8; break;
case 1: ret_user2 = 3; break;
case 11: ret_user2 = 4; break;
default: ret_user2 = 29; break;
}
return 0;
}
Run: clang --target=bpf -O2 -S test.c
The assembly code:
...
# %bb.1: # %entry
r1 <<= 3
r2 = .LJTI0_0 ll
r2 += r1
r1 = *(u64 *)(r2 + 0)
gotox r1
goto LBB0_8
LBB0_2:
w1 = 18
goto LBB0_9
...
# %bb.10: # %sw.epilog
r1 <<= 3
r2 = .LJTI0_1 ll
r2 += r1
r1 = *(u64 *)(r2 + 0)
gotox r1
goto LBB0_15
...
.section .rodata,"a", at progbits
.p2align 3, 0x0
.LJTI0_0:
.quad LBB0_2
.quad LBB0_8
.quad LBB0_8
.quad LBB0_8
.quad LBB0_7
...
.LJTI0_1:
.quad LBB0_11
.quad LBB0_13
Although we do have labels .LJTI0_0 and .LJTI0_1, but since they have
prefix '.L' so they won't appear in the .o file like
Run: llvm-objdump -Sr test.o
...
4: 67 01 00 00 03 00 00 00 r1 <<= 0x3
5: 18 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 r2 = 0x0 ll
0000000000000028: R_BPF_64_64 .rodata
7: 0f 12 00 00 00 00 00 00 r2 += r1
...
30: 67 01 00 00 03 00 00 00 r1 <<= 0x3
31: 18 02 00 00 f0 00 00 00 00 00 00 00 00 00 00 00 r2 = 0xf0 ll
00000000000000f8: R_BPF_64_64 .rodata
The size of jump table is not obvious. The libbpf needs to check all relocations
against .rodata section in order to get precise size in order to construct bpf
maps.
Example 2 (Simple computed goto):
=================================
Code:
int bar(int a) {
__label__ l1, l2;
void * volatile tgt;
int ret = 0;
if (a)
tgt = &&l1; // synthetic jump table generated here
else
tgt = &&l2; // another synthetic jump table
goto *tgt;
l1: ret += 1;
l2: ret += 2;
return ret;
}
Compile: clang --target=bpf -O2 -c test1.c
Objdump: llvm-objdump -Sr test1.o
0: 18 02 00 00 50 00 00 00 00 00 00 00 00 00 00 00 r2 = 0x50 ll
0000000000000000: R_BPF_64_64 .text
2: 16 01 02 00 00 00 00 00 if w1 == 0x0 goto +0x2 <bar+0x28>
3: 18 02 00 00 40 00 00 00 00 00 00 00 00 00 00 00 r2 = 0x40 ll
0000000000000018: R_BPF_64_64 .text
5: 7b 2a f8 ff 00 00 00 00 *(u64 *)(r10 - 0x8) = r2
6: 79 a1 f8 ff 00 00 00 00 r1 = *(u64 *)(r10 - 0x8)
7: 0d 01 00 00 00 00 00 00 gotox r1
8: b4 00 00 00 03 00 00 00 w0 = 0x3
9: 05 00 01 00 00 00 00 00 goto +0x1 <bar+0x58>
10: b4 00 00 00 02 00 00 00 w0 = 0x2
11: 95 00 00 00 00 00 00 00 exit
For this case, no need for jump table. Verifier is able to detect precise
gotox addresses for different branches.
Example 3 (More complicated computed goto):
===========================================
Code:
int foo(int a, int b) {
__label__ l1, l2, l3, l4;
void *jt1[] = {[0]=&&l1, [1]=&&l2};
void *jt2[] = {[0]=&&l3, [1]=&&l4};
int ret = 0;
goto *jt1[a % 2];
l1: ret += 1;
l2: ret += 3;
goto *jt2[b % 2];
l3: ret += 5;
l4: ret += 7;
return ret;
}
Compile: clang --target=bpf -O2 -S test2.c
Asm code:
# %bb.1: # %entry
r1 <<= 3
r2 = .LJTI0_0 ll
r2 += r1
r1 = *(u64 *)(r2 + 0)
gotox r1
goto LBB0_8
LBB0_2:
w1 = 18
goto LBB0_9
...
# %bb.10: # %sw.epilog
r1 <<= 3
r2 = .LJTI0_1 ll
r2 += r1
r1 = *(u64 *)(r2 + 0)
gotox r1
goto LBB0_15
LBB0_11:
w1 = 8
goto LBB0_16
...
.section .rodata,"a", at progbits
.p2align 3, 0x0
.LJTI0_0:
.quad LBB0_2
.quad LBB0_8
.quad LBB0_8
...
.quad LBB0_7
.LJTI0_1:
.quad LBB0_11
.quad LBB0_13
.quad LBB0_15
...
Similar to switch statement case, for the binary, the symbols .LJTI0_*
will not show up in the symbol table and jump table will be in .rodata
section.
With more libbpf work (dealing with .rodata sections etc.),
everything should work fine. But we could do better by
- Replacing symbols like .L<...> with symbols appearing in
symbol table.
- Add jump tables to .jumptables section instead of .rodata section.
This should make things easier for libbpf and for users.
Next two patches try to implement the above.
---
.../lib/Target/BPF/AsmParser/BPFAsmParser.cpp | 1 +
llvm/lib/Target/BPF/BPFISelLowering.cpp | 36 +++++++++++++++-
llvm/lib/Target/BPF/BPFISelLowering.h | 2 +
llvm/lib/Target/BPF/BPFInstrInfo.cpp | 41 +++++++++++++++++++
llvm/lib/Target/BPF/BPFInstrInfo.h | 3 ++
llvm/lib/Target/BPF/BPFInstrInfo.td | 18 ++++++++
llvm/lib/Target/BPF/BPFMCInstLower.cpp | 7 ++++
7 files changed, 106 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp b/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp
index a347794a9a30c..d96f403d2f814 100644
--- a/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp
+++ b/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp
@@ -234,6 +234,7 @@ struct BPFOperand : public MCParsedAsmOperand {
.Case("callx", true)
.Case("goto", true)
.Case("gotol", true)
+ .Case("gotox", true)
.Case("may_goto", true)
.Case("*", true)
.Case("exit", true)
diff --git a/llvm/lib/Target/BPF/BPFISelLowering.cpp b/llvm/lib/Target/BPF/BPFISelLowering.cpp
index f4f414d192df0..060fd0413e00f 100644
--- a/llvm/lib/Target/BPF/BPFISelLowering.cpp
+++ b/llvm/lib/Target/BPF/BPFISelLowering.cpp
@@ -38,6 +38,10 @@ static cl::opt<bool> BPFExpandMemcpyInOrder("bpf-expand-memcpy-in-order",
cl::Hidden, cl::init(false),
cl::desc("Expand memcpy into load/store pairs in order"));
+static cl::opt<unsigned> BPFMinimumJumpTableEntries(
+ "bpf-min-jump-table-entries", cl::init(5), cl::Hidden,
+ cl::desc("Set minimum number of entries to use a jump table on BPF"));
+
static void fail(const SDLoc &DL, SelectionDAG &DAG, const Twine &Msg,
SDValue Val = {}) {
std::string Str;
@@ -67,12 +71,13 @@ BPFTargetLowering::BPFTargetLowering(const TargetMachine &TM,
setOperationAction(ISD::BR_CC, MVT::i64, Custom);
setOperationAction(ISD::BR_JT, MVT::Other, Expand);
- setOperationAction(ISD::BRIND, MVT::Other, Expand);
setOperationAction(ISD::BRCOND, MVT::Other, Expand);
setOperationAction(ISD::TRAP, MVT::Other, Custom);
- setOperationAction({ISD::GlobalAddress, ISD::ConstantPool}, MVT::i64, Custom);
+ setOperationAction({ISD::GlobalAddress, ISD::ConstantPool, ISD::JumpTable,
+ ISD::BlockAddress},
+ MVT::i64, Custom);
setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64, Custom);
setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
@@ -159,6 +164,7 @@ BPFTargetLowering::BPFTargetLowering(const TargetMachine &TM,
setBooleanContents(ZeroOrOneBooleanContent);
setMaxAtomicSizeInBitsSupported(64);
+ setMinimumJumpTableEntries(BPFMinimumJumpTableEntries);
// Function alignments
setMinFunctionAlignment(Align(8));
@@ -316,10 +322,14 @@ SDValue BPFTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
report_fatal_error("unimplemented opcode: " + Twine(Op.getOpcode()));
case ISD::BR_CC:
return LowerBR_CC(Op, DAG);
+ case ISD::JumpTable:
+ return LowerJumpTable(Op, DAG);
case ISD::GlobalAddress:
return LowerGlobalAddress(Op, DAG);
case ISD::ConstantPool:
return LowerConstantPool(Op, DAG);
+ case ISD::BlockAddress:
+ return LowerBlockAddress(Op, DAG);
case ISD::SELECT_CC:
return LowerSELECT_CC(Op, DAG);
case ISD::SDIV:
@@ -780,6 +790,11 @@ SDValue BPFTargetLowering::LowerTRAP(SDValue Op, SelectionDAG &DAG) const {
return LowerCall(CLI, InVals);
}
+SDValue BPFTargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
+ JumpTableSDNode *N = cast<JumpTableSDNode>(Op);
+ return getAddr(N, DAG);
+}
+
const char *BPFTargetLowering::getTargetNodeName(unsigned Opcode) const {
switch ((BPFISD::NodeType)Opcode) {
case BPFISD::FIRST_NUMBER:
@@ -811,6 +826,17 @@ static SDValue getTargetNode(ConstantPoolSDNode *N, const SDLoc &DL, EVT Ty,
N->getOffset(), Flags);
}
+static SDValue getTargetNode(BlockAddressSDNode *N, const SDLoc &DL, EVT Ty,
+ SelectionDAG &DAG, unsigned Flags) {
+ return DAG.getTargetBlockAddress(N->getBlockAddress(), Ty, N->getOffset(),
+ Flags);
+}
+
+static SDValue getTargetNode(JumpTableSDNode *N, const SDLoc &DL, EVT Ty,
+ SelectionDAG &DAG, unsigned Flags) {
+ return DAG.getTargetJumpTable(N->getIndex(), Ty, Flags);
+}
+
template <class NodeTy>
SDValue BPFTargetLowering::getAddr(NodeTy *N, SelectionDAG &DAG,
unsigned Flags) const {
@@ -837,6 +863,12 @@ SDValue BPFTargetLowering::LowerConstantPool(SDValue Op,
return getAddr(N, DAG);
}
+SDValue BPFTargetLowering::LowerBlockAddress(SDValue Op,
+ SelectionDAG &DAG) const {
+ BlockAddressSDNode *N = cast<BlockAddressSDNode>(Op);
+ return getAddr(N, DAG);
+}
+
unsigned
BPFTargetLowering::EmitSubregExt(MachineInstr &MI, MachineBasicBlock *BB,
unsigned Reg, bool isSigned) const {
diff --git a/llvm/lib/Target/BPF/BPFISelLowering.h b/llvm/lib/Target/BPF/BPFISelLowering.h
index 8f60261c10e9e..13b1b746ad0c8 100644
--- a/llvm/lib/Target/BPF/BPFISelLowering.h
+++ b/llvm/lib/Target/BPF/BPFISelLowering.h
@@ -81,6 +81,8 @@ class BPFTargetLowering : public TargetLowering {
SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerTRAP(SDValue Op, SelectionDAG &DAG) const;
+ SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
+ SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const;
template <class NodeTy>
SDValue getAddr(NodeTy *N, SelectionDAG &DAG, unsigned Flags = 0) const;
diff --git a/llvm/lib/Target/BPF/BPFInstrInfo.cpp b/llvm/lib/Target/BPF/BPFInstrInfo.cpp
index 70bc163615f61..78626c39e80f7 100644
--- a/llvm/lib/Target/BPF/BPFInstrInfo.cpp
+++ b/llvm/lib/Target/BPF/BPFInstrInfo.cpp
@@ -181,6 +181,10 @@ bool BPFInstrInfo::analyzeBranch(MachineBasicBlock &MBB,
if (!isUnpredicatedTerminator(*I))
break;
+ // If a JX insn, we're done.
+ if (I->getOpcode() == BPF::JX)
+ break;
+
// A terminator that isn't a branch can't easily be handled
// by this analysis.
if (!I->isBranch())
@@ -259,3 +263,40 @@ unsigned BPFInstrInfo::removeBranch(MachineBasicBlock &MBB,
return Count;
}
+
+int BPFInstrInfo::getJumpTableIndex(const MachineInstr &MI) const {
+ // The pattern looks like:
+ // %0 = LD_imm64 %jump-table.0 ; load jump-table address
+ // %1 = ADD_rr %0, $another_reg ; address + offset
+ // %2 = LDD %1, 0 ; load the actual label
+ // JX %2
+ const MachineFunction &MF = *MI.getParent()->getParent();
+ const MachineRegisterInfo &MRI = MF.getRegInfo();
+
+ Register Reg = MI.getOperand(0).getReg();
+ if (!Reg.isVirtual())
+ return -1;
+ MachineInstr *Ldd = MRI.getUniqueVRegDef(Reg);
+ if (Ldd == nullptr || Ldd->getOpcode() != BPF::LDD)
+ return -1;
+
+ Reg = Ldd->getOperand(1).getReg();
+ if (!Reg.isVirtual())
+ return -1;
+ MachineInstr *Add = MRI.getUniqueVRegDef(Reg);
+ if (Add == nullptr || Add->getOpcode() != BPF::ADD_rr)
+ return -1;
+
+ Reg = Add->getOperand(1).getReg();
+ if (!Reg.isVirtual())
+ return -1;
+ MachineInstr *LDimm64 = MRI.getUniqueVRegDef(Reg);
+ if (LDimm64 == nullptr || LDimm64->getOpcode() != BPF::LD_imm64)
+ return -1;
+
+ const MachineOperand &MO = LDimm64->getOperand(1);
+ if (!MO.isJTI())
+ return -1;
+
+ return MO.getIndex();
+}
diff --git a/llvm/lib/Target/BPF/BPFInstrInfo.h b/llvm/lib/Target/BPF/BPFInstrInfo.h
index d8bbad44e314e..d88e37975980a 100644
--- a/llvm/lib/Target/BPF/BPFInstrInfo.h
+++ b/llvm/lib/Target/BPF/BPFInstrInfo.h
@@ -58,6 +58,9 @@ class BPFInstrInfo : public BPFGenInstrInfo {
MachineBasicBlock *FBB, ArrayRef<MachineOperand> Cond,
const DebugLoc &DL,
int *BytesAdded = nullptr) const override;
+
+ int getJumpTableIndex(const MachineInstr &MI) const override;
+
private:
void expandMEMCPY(MachineBasicBlock::iterator) const;
diff --git a/llvm/lib/Target/BPF/BPFInstrInfo.td b/llvm/lib/Target/BPF/BPFInstrInfo.td
index b21f1a0eee3b0..c61236c278dbb 100644
--- a/llvm/lib/Target/BPF/BPFInstrInfo.td
+++ b/llvm/lib/Target/BPF/BPFInstrInfo.td
@@ -216,6 +216,18 @@ class JMP_RI<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond>
let BPFClass = BPF_JMP;
}
+class JMP_IND<BPFJumpOp Opc, string OpcodeStr, list<dag> Pattern>
+ : TYPE_ALU_JMP<Opc.Value, BPF_X.Value,
+ (outs),
+ (ins GPR:$dst),
+ !strconcat(OpcodeStr, " $dst"),
+ Pattern> {
+ bits<4> dst;
+
+ let Inst{51-48} = dst;
+ let BPFClass = BPF_JMP;
+}
+
class JMP_JCOND<BPFJumpOp Opc, string OpcodeStr, list<dag> Pattern>
: TYPE_ALU_JMP<Opc.Value, BPF_K.Value,
(outs),
@@ -281,6 +293,10 @@ defm JSLT : J<BPF_JSLT, "s<", BPF_CC_LT, BPF_CC_LT_32>;
defm JSLE : J<BPF_JSLE, "s<=", BPF_CC_LE, BPF_CC_LE_32>;
defm JSET : J<BPF_JSET, "&", NoCond, NoCond>;
def JCOND : JMP_JCOND<BPF_JCOND, "may_goto", []>;
+
+let isIndirectBranch = 1 in {
+ def JX : JMP_IND<BPF_JA, "gotox", [(brind i64:$dst)]>;
+}
}
// ALU instructions
@@ -851,6 +867,8 @@ let usesCustomInserter = 1, isCodeGenOnly = 1 in {
// load 64-bit global addr into register
def : Pat<(BPFWrapper tglobaladdr:$in), (LD_imm64 tglobaladdr:$in)>;
def : Pat<(BPFWrapper tconstpool:$in), (LD_imm64 tconstpool:$in)>;
+def : Pat<(BPFWrapper tblockaddress:$in), (LD_imm64 tblockaddress:$in)>;
+def : Pat<(BPFWrapper tjumptable:$in), (LD_imm64 tjumptable:$in)>;
// 0xffffFFFF doesn't fit into simm32, optimize common case
def : Pat<(i64 (and (i64 GPR:$src), 0xffffFFFF)),
diff --git a/llvm/lib/Target/BPF/BPFMCInstLower.cpp b/llvm/lib/Target/BPF/BPFMCInstLower.cpp
index 040a1fb750702..e67b99a813945 100644
--- a/llvm/lib/Target/BPF/BPFMCInstLower.cpp
+++ b/llvm/lib/Target/BPF/BPFMCInstLower.cpp
@@ -77,6 +77,13 @@ void BPFMCInstLower::Lower(const MachineInstr *MI, MCInst &OutMI) const {
case MachineOperand::MO_ConstantPoolIndex:
MCOp = LowerSymbolOperand(MO, Printer.GetCPISymbol(MO.getIndex()));
break;
+ case MachineOperand::MO_JumpTableIndex:
+ MCOp = LowerSymbolOperand(MO, Printer.GetJTISymbol(MO.getIndex()));
+ break;
+ case MachineOperand::MO_BlockAddress:
+ MCOp = LowerSymbolOperand(
+ MO, Printer.GetBlockAddressSymbol(MO.getBlockAddress()));
+ break;
}
OutMI.addOperand(MCOp);
>From 62495bf7ef1e637e0460385088ffbbfc4fce7f6d Mon Sep 17 00:00:00 2001
From: Yonghong Song <yonghong.song at linux.dev>
Date: Fri, 18 Jul 2025 19:52:13 -0700
Subject: [PATCH 2/4] [BPF] Use visible symbols and dedicated jump tables for
switch statements
For jumptables from switch statements, generate 'llvm-readelf -s' visible
symbols and put jumptables into a dedicated section. Most work from
Eduard.
For the previous example 1,
Compile: clang --target=bpf -O2 -S test.c
Asm code:
# %bb.1: # %entry
r1 <<= 3
r2 = BPF.JT.0.0 ll
r2 += r1
r1 = *(u64 *)(r2 + 0)
gotox r1
goto LBB0_8
LBB0_2:
w1 = 18
goto LBB0_9
...
.section .jumptables,"", at progbits
BPF.JT.0.0:
.quad LBB0_2
.quad LBB0_8
...
.quad LBB0_7
.size BPF.JT.0.0, 120
BPF.JT.0.1:
...
And symbols BPF.JT.0.{0,1} will be in symbol table.
The final binary:
4: 67 01 00 00 03 00 00 00 r1 <<= 0x3
5: 18 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 r2 = 0x0 ll
0000000000000028: R_BPF_64_64 BPF.JT.0.0
7: 0f 12 00 00 00 00 00 00 r2 += r1
...
Symbol table:
4: 0000000000000000 120 NOTYPE GLOBAL DEFAULT 4 BPF.JT.0.0
5: 0000000000000000 4 OBJECT GLOBAL DEFAULT 6 ret_user
6: 0000000000000000 0 NOTYPE GLOBAL DEFAULT UND bar
7: 00000000000000f0 128 NOTYPE GLOBAL DEFAULT 4 BPF.JT.0.1
and
[ 4] .jumptables PROGBITS 0000000000000000 0001d8 0001f0 00 0 0 1
This will make things easier when inspecting the binary about jump table.
---
llvm/lib/Target/BPF/BPFAsmPrinter.cpp | 72 ++++++++++++-------
llvm/lib/Target/BPF/BPFAsmPrinter.h | 43 +++++++++++
llvm/lib/Target/BPF/BPFMCInstLower.cpp | 5 +-
llvm/lib/Target/BPF/BPFMCInstLower.h | 6 +-
.../BPF/BPFTargetLoweringObjectFile.cpp | 19 +++++
.../Target/BPF/BPFTargetLoweringObjectFile.h | 25 +++++++
llvm/lib/Target/BPF/BPFTargetMachine.cpp | 3 +-
llvm/lib/Target/BPF/CMakeLists.txt | 1 +
8 files changed, 145 insertions(+), 29 deletions(-)
create mode 100644 llvm/lib/Target/BPF/BPFAsmPrinter.h
create mode 100644 llvm/lib/Target/BPF/BPFTargetLoweringObjectFile.cpp
create mode 100644 llvm/lib/Target/BPF/BPFTargetLoweringObjectFile.h
diff --git a/llvm/lib/Target/BPF/BPFAsmPrinter.cpp b/llvm/lib/Target/BPF/BPFAsmPrinter.cpp
index e3843e0e112e2..2609671f111d4 100644
--- a/llvm/lib/Target/BPF/BPFAsmPrinter.cpp
+++ b/llvm/lib/Target/BPF/BPFAsmPrinter.cpp
@@ -11,52 +11,35 @@
//
//===----------------------------------------------------------------------===//
+#include "BPFAsmPrinter.h"
#include "BPF.h"
#include "BPFInstrInfo.h"
#include "BPFMCInstLower.h"
#include "BTFDebug.h"
#include "MCTargetDesc/BPFInstPrinter.h"
#include "TargetInfo/BPFTargetInfo.h"
+#include "llvm/BinaryFormat/ELF.h"
#include "llvm/CodeGen/AsmPrinter.h"
#include "llvm/CodeGen/MachineConstantPool.h"
#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/MachineJumpTableInfo.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/IR/Module.h"
#include "llvm/MC/MCAsmInfo.h"
+#include "llvm/MC/MCExpr.h"
#include "llvm/MC/MCInst.h"
#include "llvm/MC/MCStreamer.h"
#include "llvm/MC/MCSymbol.h"
+#include "llvm/MC/MCSymbolELF.h"
#include "llvm/MC/TargetRegistry.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/raw_ostream.h"
+#include "llvm/Target/TargetLoweringObjectFile.h"
using namespace llvm;
#define DEBUG_TYPE "asm-printer"
-namespace {
-class BPFAsmPrinter : public AsmPrinter {
-public:
- explicit BPFAsmPrinter(TargetMachine &TM,
- std::unique_ptr<MCStreamer> Streamer)
- : AsmPrinter(TM, std::move(Streamer), ID), BTF(nullptr) {}
-
- StringRef getPassName() const override { return "BPF Assembly Printer"; }
- bool doInitialization(Module &M) override;
- void printOperand(const MachineInstr *MI, int OpNum, raw_ostream &O);
- bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
- const char *ExtraCode, raw_ostream &O) override;
- bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum,
- const char *ExtraCode, raw_ostream &O) override;
-
- void emitInstruction(const MachineInstr *MI) override;
-
- static char ID;
-
-private:
- BTFDebug *BTF;
-};
-} // namespace
-
bool BPFAsmPrinter::doInitialization(Module &M) {
AsmPrinter::doInitialization(M);
@@ -150,6 +133,47 @@ void BPFAsmPrinter::emitInstruction(const MachineInstr *MI) {
EmitToStreamer(*OutStreamer, TmpInst);
}
+MCSymbol *BPFAsmPrinter::getJTPublicSymbol(unsigned JTI) {
+ SmallString<60> Name;
+ raw_svector_ostream(Name)
+ << "BPF.JT." << MF->getFunctionNumber() << '.' << JTI;
+ MCSymbol *S = OutContext.getOrCreateSymbol(Name);
+ if (auto *ES = dyn_cast<MCSymbolELF>(S))
+ ES->setBinding(ELF::STB_GLOBAL);
+ return S;
+}
+
+void BPFAsmPrinter::emitJumpTableInfo() {
+ const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
+ if (!MJTI)
+ return;
+
+ const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
+ if (JT.empty())
+ return;
+
+ const TargetLoweringObjectFile &TLOF = getObjFileLowering();
+ const Function &F = MF->getFunction();
+ MCSection *JTS = TLOF.getSectionForJumpTable(F, TM);
+ assert(MJTI->getEntryKind() == MachineJumpTableInfo::EK_BlockAddress);
+ unsigned EntrySize = MJTI->getEntrySize(getDataLayout());
+ OutStreamer->switchSection(JTS);
+ for (unsigned JTI = 0; JTI < JT.size(); JTI++) {
+ ArrayRef<MachineBasicBlock *> JTBBs = JT[JTI].MBBs;
+ if (JTBBs.empty())
+ continue;
+
+ MCSymbol *JTStart = getJTPublicSymbol(JTI);
+ OutStreamer->emitLabel(JTStart);
+ for (const MachineBasicBlock *MBB : JTBBs) {
+ const MCExpr *LHS = MCSymbolRefExpr::create(MBB->getSymbol(), OutContext);
+ OutStreamer->emitValue(LHS, EntrySize);
+ }
+ const MCExpr *JTSize = MCConstantExpr::create(JTBBs.size() * 4, OutContext);
+ OutStreamer->emitELFSize(JTStart, JTSize);
+ }
+}
+
char BPFAsmPrinter::ID = 0;
INITIALIZE_PASS(BPFAsmPrinter, "bpf-asm-printer", "BPF Assembly Printer", false,
diff --git a/llvm/lib/Target/BPF/BPFAsmPrinter.h b/llvm/lib/Target/BPF/BPFAsmPrinter.h
new file mode 100644
index 0000000000000..3ae2dbc094540
--- /dev/null
+++ b/llvm/lib/Target/BPF/BPFAsmPrinter.h
@@ -0,0 +1,43 @@
+//===-- BPFFrameLowering.h - Define frame lowering for BPF -----*- C++ -*--===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_BPF_BPFASMPRINTER_H
+#define LLVM_LIB_TARGET_BPF_BPFASMPRINTER_H
+
+#include "BTFDebug.h"
+#include "llvm/CodeGen/AsmPrinter.h"
+
+namespace llvm {
+
+class BPFAsmPrinter : public AsmPrinter {
+public:
+ explicit BPFAsmPrinter(TargetMachine &TM,
+ std::unique_ptr<MCStreamer> Streamer)
+ : AsmPrinter(TM, std::move(Streamer), ID), BTF(nullptr) {}
+
+ StringRef getPassName() const override { return "BPF Assembly Printer"; }
+ bool doInitialization(Module &M) override;
+ void printOperand(const MachineInstr *MI, int OpNum, raw_ostream &O);
+ bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
+ const char *ExtraCode, raw_ostream &O) override;
+ bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum,
+ const char *ExtraCode, raw_ostream &O) override;
+
+ void emitInstruction(const MachineInstr *MI) override;
+ MCSymbol *getJTPublicSymbol(unsigned JTI);
+ virtual void emitJumpTableInfo() override;
+
+ static char ID;
+
+private:
+ BTFDebug *BTF;
+};
+
+} // namespace llvm
+
+#endif /* LLVM_LIB_TARGET_BPF_BPFASMPRINTER_H */
diff --git a/llvm/lib/Target/BPF/BPFMCInstLower.cpp b/llvm/lib/Target/BPF/BPFMCInstLower.cpp
index e67b99a813945..295c27bd16c22 100644
--- a/llvm/lib/Target/BPF/BPFMCInstLower.cpp
+++ b/llvm/lib/Target/BPF/BPFMCInstLower.cpp
@@ -12,6 +12,8 @@
//===----------------------------------------------------------------------===//
#include "BPFMCInstLower.h"
+#include "BPFAsmPrinter.h"
+#include "BPFISelLowering.h"
#include "llvm/CodeGen/AsmPrinter.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineInstr.h"
@@ -19,6 +21,7 @@
#include "llvm/MC/MCContext.h"
#include "llvm/MC/MCExpr.h"
#include "llvm/MC/MCInst.h"
+#include "llvm/MC/MCStreamer.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/raw_ostream.h"
using namespace llvm;
@@ -78,7 +81,7 @@ void BPFMCInstLower::Lower(const MachineInstr *MI, MCInst &OutMI) const {
MCOp = LowerSymbolOperand(MO, Printer.GetCPISymbol(MO.getIndex()));
break;
case MachineOperand::MO_JumpTableIndex:
- MCOp = LowerSymbolOperand(MO, Printer.GetJTISymbol(MO.getIndex()));
+ MCOp = LowerSymbolOperand(MO, Printer.getJTPublicSymbol(MO.getIndex()));
break;
case MachineOperand::MO_BlockAddress:
MCOp = LowerSymbolOperand(
diff --git a/llvm/lib/Target/BPF/BPFMCInstLower.h b/llvm/lib/Target/BPF/BPFMCInstLower.h
index 4bd0f1f0bf1cf..483edd9a02831 100644
--- a/llvm/lib/Target/BPF/BPFMCInstLower.h
+++ b/llvm/lib/Target/BPF/BPFMCInstLower.h
@@ -12,7 +12,7 @@
#include "llvm/Support/Compiler.h"
namespace llvm {
-class AsmPrinter;
+class BPFAsmPrinter;
class MCContext;
class MCInst;
class MCOperand;
@@ -24,10 +24,10 @@ class MachineOperand;
class LLVM_LIBRARY_VISIBILITY BPFMCInstLower {
MCContext &Ctx;
- AsmPrinter &Printer;
+ BPFAsmPrinter &Printer;
public:
- BPFMCInstLower(MCContext &ctx, AsmPrinter &printer)
+ BPFMCInstLower(MCContext &ctx, BPFAsmPrinter &printer)
: Ctx(ctx), Printer(printer) {}
void Lower(const MachineInstr *MI, MCInst &OutMI) const;
diff --git a/llvm/lib/Target/BPF/BPFTargetLoweringObjectFile.cpp b/llvm/lib/Target/BPF/BPFTargetLoweringObjectFile.cpp
new file mode 100644
index 0000000000000..997f09870bad6
--- /dev/null
+++ b/llvm/lib/Target/BPF/BPFTargetLoweringObjectFile.cpp
@@ -0,0 +1,19 @@
+//===------------------ BPFTargetLoweringObjectFile.cpp -------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "BPFTargetLoweringObjectFile.h"
+#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCSectionELF.h"
+
+using namespace llvm;
+
+MCSection *BPFTargetLoweringObjectFileELF::getSectionForJumpTable(
+ const Function &F, const TargetMachine &TM,
+ const MachineJumpTableEntry *JTE) const {
+ return getContext().getELFSection(".jumptables", ELF::SHT_PROGBITS, 0);
+}
diff --git a/llvm/lib/Target/BPF/BPFTargetLoweringObjectFile.h b/llvm/lib/Target/BPF/BPFTargetLoweringObjectFile.h
new file mode 100644
index 0000000000000..f3064c0c8cb8a
--- /dev/null
+++ b/llvm/lib/Target/BPF/BPFTargetLoweringObjectFile.h
@@ -0,0 +1,25 @@
+//===============- BPFTargetLoweringObjectFile.h -*- C++ -*-================//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_BPF_BPFTARGETLOWERINGOBJECTFILE
+#define LLVM_LIB_TARGET_BPF_BPFTARGETLOWERINGOBJECTFILE
+
+#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
+#include "llvm/Target/TargetLoweringObjectFile.h"
+
+namespace llvm {
+class BPFTargetLoweringObjectFileELF : public TargetLoweringObjectFileELF {
+
+public:
+ virtual MCSection *
+ getSectionForJumpTable(const Function &F, const TargetMachine &TM,
+ const MachineJumpTableEntry *JTE) const override;
+};
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_BPF_BPFTARGETLOWERINGOBJECTFILE
diff --git a/llvm/lib/Target/BPF/BPFTargetMachine.cpp b/llvm/lib/Target/BPF/BPFTargetMachine.cpp
index 527a480354571..d538b6fe11675 100644
--- a/llvm/lib/Target/BPF/BPFTargetMachine.cpp
+++ b/llvm/lib/Target/BPF/BPFTargetMachine.cpp
@@ -12,6 +12,7 @@
#include "BPFTargetMachine.h"
#include "BPF.h"
+#include "BPFTargetLoweringObjectFile.h"
#include "BPFTargetTransformInfo.h"
#include "MCTargetDesc/BPFMCAsmInfo.h"
#include "TargetInfo/BPFTargetInfo.h"
@@ -80,7 +81,7 @@ BPFTargetMachine::BPFTargetMachine(const Target &T, const Triple &TT,
: CodeGenTargetMachineImpl(T, computeDataLayout(TT), TT, CPU, FS, Options,
getEffectiveRelocModel(RM),
getEffectiveCodeModel(CM, CodeModel::Small), OL),
- TLOF(std::make_unique<TargetLoweringObjectFileELF>()),
+ TLOF(std::make_unique<BPFTargetLoweringObjectFileELF>()),
Subtarget(TT, std::string(CPU), std::string(FS), *this) {
if (!DisableCheckUnreachable) {
this->Options.TrapUnreachable = true;
diff --git a/llvm/lib/Target/BPF/CMakeLists.txt b/llvm/lib/Target/BPF/CMakeLists.txt
index eade4cacb7100..3678f1335ca36 100644
--- a/llvm/lib/Target/BPF/CMakeLists.txt
+++ b/llvm/lib/Target/BPF/CMakeLists.txt
@@ -37,6 +37,7 @@ add_llvm_target(BPFCodeGen
BPFRegisterInfo.cpp
BPFSelectionDAGInfo.cpp
BPFSubtarget.cpp
+ BPFTargetLoweringObjectFile.cpp
BPFTargetMachine.cpp
BPFMIPeephole.cpp
BPFMIChecking.cpp
>From 0bac16d4de2737380891742b44542be899212b34 Mon Sep 17 00:00:00 2001
From: Yonghong Song <yonghong.song at linux.dev>
Date: Sat, 19 Jul 2025 17:39:02 -0700
Subject: [PATCH 3/4] [BPF] Use visible symbols and dedicated jump tables for
computed goto
For computed goto, the llvm won't generate JumpTableInfo. Note that
JumpTableInfo is generated for switch statements.
See previous example 3. Actually compiler has generated symbols
and jump tables. But unfortunately for previous patch, the symbols
are not visible in symbol table and the section is '.rodata' and
this makes it is hard to inspect binary for jump tables.
For computed goto, to make jump table symbol visible in symbol table,
a pass is added in BPFMIPeephole.cpp such that it will
- Change linkage from PrivateLinkage to InternalLinkage
to avoid adding '.L' prefix.
- Change section from '.rodata' to '.jumptables'.
With the above, the following are some asm and objdump codes.
Asm code:
...
r2 <<= 3
r3 = __const.foo.jt2 ll
r3 += r2
r1 <<= 32
r1 s>>= 32
r1 <<= 3
r2 = __const.foo.jt1 ll
r2 += r1
w0 = 0
r1 = *(u64 *)(r2 + 0)
gotox r1
goto LBB0_4
...
.type __const.foo.jt1, at object # @__const.foo.jt1
.section .jumptables,"a", at progbits
.p2align 3, 0x0
__const.foo.jt1:
.quad .Ltmp0
.quad .Ltmp1
.size __const.foo.jt1, 16
.type __const.foo.jt2, at object # @__const.foo.jt2
.p2align 3, 0x0
__const.foo.jt2:
.quad .Ltmp2
.quad .Ltmp3
.size __const.foo.jt2, 16
objdump code:
14: 67 02 00 00 03 00 00 00 r2 <<= 0x3
15: 18 03 00 00 10 00 00 00 00 00 00 00 00 00 00 00 r3 = 0x10 ll
0000000000000078: R_BPF_64_64 .jumptables
17: 0f 23 00 00 00 00 00 00 r3 += r2
18: 67 01 00 00 20 00 00 00 r1 <<= 0x20
19: c7 01 00 00 20 00 00 00 r1 s>>= 0x20
20: 67 01 00 00 03 00 00 00 r1 <<= 0x3
21: 18 02 00 00 00 00 00 00 00 00 00 00 00 00 00 00 r2 = 0x0 ll
00000000000000a8: R_BPF_64_64 .jumptables
23: 0f 12 00 00 00 00 00 00 r2 += r1
[ 4] .jumptables PROGBITS 0000000000000000 000178 000020 00 A 0 0 8
3: 0000000000000010 16 OBJECT LOCAL DEFAULT 4 __const.foo.jt2
4: 0000000000000000 16 OBJECT LOCAL DEFAULT 4 __const.foo.jt1
5: 0000000000000000 0 SECTION LOCAL DEFAULT 4 .jumptables
---
llvm/lib/Target/BPF/BPFMIPeephole.cpp | 47 +++++++++++++++++++++++++++
1 file changed, 47 insertions(+)
diff --git a/llvm/lib/Target/BPF/BPFMIPeephole.cpp b/llvm/lib/Target/BPF/BPFMIPeephole.cpp
index 6275d5b7721c6..9d3d78831e821 100644
--- a/llvm/lib/Target/BPF/BPFMIPeephole.cpp
+++ b/llvm/lib/Target/BPF/BPFMIPeephole.cpp
@@ -30,6 +30,7 @@
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/IR/Module.h"
#include "llvm/Support/Debug.h"
#include <set>
@@ -308,6 +309,7 @@ struct BPFMIPreEmitPeephole : public MachineFunctionPass {
const TargetRegisterInfo *TRI;
const BPFInstrInfo *TII;
bool SupportGotol;
+ bool DoneConvertPrivGlobal;
BPFMIPreEmitPeephole() : MachineFunctionPass(ID) {}
@@ -321,6 +323,7 @@ struct BPFMIPreEmitPeephole : public MachineFunctionPass {
bool insertMissingCallerSavedSpills();
bool removeMayGotoZero();
bool addExitAfterUnreachable();
+ bool convertPrivateGlobal();
public:
@@ -338,6 +341,7 @@ struct BPFMIPreEmitPeephole : public MachineFunctionPass {
Changed |= insertMissingCallerSavedSpills();
Changed |= removeMayGotoZero();
Changed |= addExitAfterUnreachable();
+ Changed |= convertPrivateGlobal();
return Changed;
}
};
@@ -348,6 +352,7 @@ void BPFMIPreEmitPeephole::initialize(MachineFunction &MFParm) {
TII = MF->getSubtarget<BPFSubtarget>().getInstrInfo();
TRI = MF->getSubtarget<BPFSubtarget>().getRegisterInfo();
SupportGotol = MF->getSubtarget<BPFSubtarget>().hasGotol();
+ DoneConvertPrivGlobal = false;
LLVM_DEBUG(dbgs() << "*** BPF PreEmit peephole pass ***\n\n");
}
@@ -750,6 +755,48 @@ bool BPFMIPreEmitPeephole::addExitAfterUnreachable() {
return true;
}
+bool BPFMIPreEmitPeephole::convertPrivateGlobal() {
+ if (DoneConvertPrivGlobal)
+ return false;
+ DoneConvertPrivGlobal = true;
+
+ // A jump table associated with indirectbr looks like:
+ // @__const.foo.jt1 = private unnamed_addr constant [2 x ptr]
+ // [ptr blockaddress(@foo, %l1), ptr blockaddress(@foo, %l2)], align 8
+ Module *M = MF->getFunction().getParent();
+ for (GlobalVariable &Global : M->globals()) {
+ if (Global.getLinkage() != GlobalValue::PrivateLinkage)
+ continue;
+ if (!Global.isConstant())
+ continue;
+ if (!Global.hasInitializer())
+ continue;
+
+ Constant *CV = dyn_cast<Constant>(Global.getInitializer());
+ if (!CV)
+ continue;
+ ConstantArray *CA = dyn_cast<ConstantArray>(CV);
+ if (!CA)
+ continue;
+
+ for (unsigned i = 1, e = CA->getNumOperands(); i != e; ++i) {
+ if (!dyn_cast<BlockAddress>(CA->getOperand(i)))
+ continue;
+ }
+
+ // All jump tables in the same .jumptables section.
+ Global.setSection(".jumptables");
+
+ // The global symbol with PrivateLinkage (e.g. __const.foo.jt1) will add
+ // a prefix like '.L' (e.g. .L__const.foo.jt1) which will prevent symbol
+ // from appearing in the symbol table. Changing linkage type from
+ // PrivateLinkage to InternalLinkage can preserve the symbol name and
+ // keep the symbol in the symbol table.
+ Global.setLinkage(GlobalValue::InternalLinkage);
+ }
+ return true;
+}
+
} // end default namespace
INITIALIZE_PASS(BPFMIPreEmitPeephole, "bpf-mi-pemit-peephole",
>From 9bcd328dade17d15fa07bf3d29b6c45d9caa4c8c Mon Sep 17 00:00:00 2001
From: Yonghong Song <yonghong.song at linux.dev>
Date: Sun, 20 Jul 2025 08:31:47 -0700
Subject: [PATCH 4/4] [BPF] Fix a test due to jumptable support
The fixed test is:
test/CodeGen/BPF/rodata_4.ll
---
llvm/test/CodeGen/BPF/rodata_4.ll | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/test/CodeGen/BPF/rodata_4.ll b/llvm/test/CodeGen/BPF/rodata_4.ll
index a446475a69c19..299e0713f3023 100644
--- a/llvm/test/CodeGen/BPF/rodata_4.ll
+++ b/llvm/test/CodeGen/BPF/rodata_4.ll
@@ -34,7 +34,7 @@ entry:
; CHECK: *(u16 *)(r1 + 0) = r2
ret i32 0
}
-; CHECK: .section .rodata,"a", at progbits
+; CHECK: .section .jumptables,"a", at progbits
; Function Attrs: argmemonly nounwind
declare void @llvm.memcpy.p0.p0.i64(ptr nocapture writeonly, ptr nocapture readonly, i64, i1) #1
More information about the llvm-commits
mailing list