[llvm] [RFC][BPF] Support Jump Table (PR #133856)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 11 08:38:19 PDT 2025
https://github.com/yonghong-song updated https://github.com/llvm/llvm-project/pull/133856
>From a8180a8f4d2da467b44da52dd18f1638c8932232 Mon Sep 17 00:00:00 2001
From: Yonghong Song <yonghong.song at linux.dev>
Date: Mon, 31 Mar 2025 21:25:26 -0700
Subject: [PATCH 1/5] [RFC][BPF] Support Jump Table
NOTE: We probably need cpu v5 or other flags to enable this feature.
We can add it later when necessary.
- Generate all jump tables in a single section named .jumptables.
- Represent each jump table as a symbol:
- value points to an offset within .jumptables;
- size encodes jump table size in bytes.
- Indirect jump is a gotox instruction:
- dst register is an index within the table;
- accompanied by a R_BPF_64_64 relocation pointing to a jump table
symbol.
clang -S:
.LJTI0_0:
.reloc 0, FK_SecRel_8, .BPF.JT.0.0
gotox r1
goto LBB0_2
LBB0_4:
...
.section .jumptables,"", at progbits
.L0_0_set_4 = ((LBB0_4-.LBPF.JX.0.0)>>3)-1
.L0_0_set_2 = ((LBB0_2-.LBPF.JX.0.0)>>3)-1
...
.BPF.JT.0.0:
.long .L0_0_set_4
.long .L0_0_set_2
...
llvm-readelf -r --sections --symbols:
Section Headers:
[Nr] Name Type Address Off Size ES Flg Lk Inf Al
...
[ 4] .jumptables PROGBITS 0000000000000000 000118 000100 00 0 0 1
...
Relocation section '.rel.text' at offset 0x2a8 contains 2 entries:
Offset Info Type Symbol's Value Symbol's Name
0000000000000010 0000000300000001 R_BPF_64_64 0000000000000000 .BPF.JT.0.0
...
Symbol table '.symtab' contains 6 entries:
Num: Value Size Type Bind Vis Ndx Name
...
2: 0000000000000000 112 FUNC GLOBAL DEFAULT 2 foo
3: 0000000000000000 128 NOTYPE GLOBAL DEFAULT 4 .BPF.JT.0.0
...
llvm-objdump -Sdr:
0000000000000000 <foo>:
...
2: gotox r1
0000000000000010: R_BPF_64_64 .BPF.JT.0.0
An option -bpf-min-jump-table-entries is implemented to control the minimum
number of entries to use a jump table on BPF. The default value 4, but it
can be changed with the following clang option
clang ... -mllvm -bpf-min-jump-table-entries=6
where the number of jump table cases needs to be >= 6 in order to
use jump table.
---
llvm/include/llvm/CodeGen/AsmPrinter.h | 3 +-
.../lib/Target/BPF/AsmParser/BPFAsmParser.cpp | 1 +
llvm/lib/Target/BPF/BPFAsmPrinter.cpp | 77 +++++++++++++++++++
llvm/lib/Target/BPF/BPFISelLowering.cpp | 42 +++++++++-
llvm/lib/Target/BPF/BPFISelLowering.h | 29 ++++++-
llvm/lib/Target/BPF/BPFInstrInfo.cpp | 12 +++
llvm/lib/Target/BPF/BPFInstrInfo.h | 3 +
llvm/lib/Target/BPF/BPFInstrInfo.td | 30 ++++++++
llvm/lib/Target/BPF/BPFMCInstLower.cpp | 28 +++++++
llvm/lib/Target/BPF/BPFMCInstLower.h | 2 +
.../BPF/BPFTargetLoweringObjectFile.cpp | 19 +++++
.../Target/BPF/BPFTargetLoweringObjectFile.h | 25 ++++++
llvm/lib/Target/BPF/BPFTargetMachine.cpp | 3 +-
llvm/lib/Target/BPF/CMakeLists.txt | 1 +
.../Target/BPF/MCTargetDesc/BPFAsmBackend.cpp | 13 ++++
15 files changed, 283 insertions(+), 5 deletions(-)
create mode 100644 llvm/lib/Target/BPF/BPFTargetLoweringObjectFile.cpp
create mode 100644 llvm/lib/Target/BPF/BPFTargetLoweringObjectFile.h
diff --git a/llvm/include/llvm/CodeGen/AsmPrinter.h b/llvm/include/llvm/CodeGen/AsmPrinter.h
index faab2503ced50..db328f4b2003e 100644
--- a/llvm/include/llvm/CodeGen/AsmPrinter.h
+++ b/llvm/include/llvm/CodeGen/AsmPrinter.h
@@ -663,7 +663,8 @@ class LLVM_ABI AsmPrinter : public MachineFunctionPass {
MCSymbol *GetExternalSymbolSymbol(const Twine &Sym) const;
/// Return the symbol for the specified jump table entry.
- MCSymbol *GetJTISymbol(unsigned JTID, bool isLinkerPrivate = false) const;
+ virtual MCSymbol *GetJTISymbol(unsigned JTID,
+ bool isLinkerPrivate = false) const;
/// Return the symbol for the specified jump table .set
/// FIXME: privatize to AsmPrinter.
diff --git a/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp b/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp
index a347794a9a30c..d96f403d2f814 100644
--- a/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp
+++ b/llvm/lib/Target/BPF/AsmParser/BPFAsmParser.cpp
@@ -234,6 +234,7 @@ struct BPFOperand : public MCParsedAsmOperand {
.Case("callx", true)
.Case("goto", true)
.Case("gotol", true)
+ .Case("gotox", true)
.Case("may_goto", true)
.Case("*", true)
.Case("exit", true)
diff --git a/llvm/lib/Target/BPF/BPFAsmPrinter.cpp b/llvm/lib/Target/BPF/BPFAsmPrinter.cpp
index e3843e0e112e2..d99b7bec4600a 100644
--- a/llvm/lib/Target/BPF/BPFAsmPrinter.cpp
+++ b/llvm/lib/Target/BPF/BPFAsmPrinter.cpp
@@ -17,18 +17,24 @@
#include "BTFDebug.h"
#include "MCTargetDesc/BPFInstPrinter.h"
#include "TargetInfo/BPFTargetInfo.h"
+#include "llvm/BinaryFormat/ELF.h"
#include "llvm/CodeGen/AsmPrinter.h"
#include "llvm/CodeGen/MachineConstantPool.h"
#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/MachineJumpTableInfo.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/IR/Module.h"
#include "llvm/MC/MCAsmInfo.h"
+#include "llvm/MC/MCExpr.h"
#include "llvm/MC/MCInst.h"
#include "llvm/MC/MCStreamer.h"
#include "llvm/MC/MCSymbol.h"
+#include "llvm/MC/MCSymbolELF.h"
#include "llvm/MC/TargetRegistry.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/raw_ostream.h"
+#include "llvm/Target/TargetLoweringObjectFile.h"
using namespace llvm;
#define DEBUG_TYPE "asm-printer"
@@ -49,6 +55,9 @@ class BPFAsmPrinter : public AsmPrinter {
const char *ExtraCode, raw_ostream &O) override;
void emitInstruction(const MachineInstr *MI) override;
+ virtual MCSymbol *GetJTISymbol(unsigned JTID,
+ bool isLinkerPrivate = false) const override;
+ virtual void emitJumpTableInfo() override;
static char ID;
@@ -150,6 +159,74 @@ void BPFAsmPrinter::emitInstruction(const MachineInstr *MI) {
EmitToStreamer(*OutStreamer, TmpInst);
}
+MCSymbol *BPFAsmPrinter::GetJTISymbol(unsigned JTID,
+ bool isLinkerPrivate) const {
+ SmallString<60> Name;
+ raw_svector_ostream(Name)
+ << "BPF.JT." << MF->getFunctionNumber() << '.' << JTID;
+ MCSymbol *S = OutContext.getOrCreateSymbol(Name);
+ if (auto *ES = dyn_cast<MCSymbolELF>(S))
+ ES->setBinding(ELF::STB_GLOBAL);
+ return S;
+}
+
+void BPFAsmPrinter::emitJumpTableInfo() {
+ const MachineJumpTableInfo *MJTI = MF->getJumpTableInfo();
+ if (!MJTI)
+ return;
+
+ const std::vector<MachineJumpTableEntry> &JT = MJTI->getJumpTables();
+ if (JT.empty())
+ return;
+
+ const TargetLoweringObjectFile &TLOF = getObjFileLowering();
+ const Function &F = MF->getFunction();
+ MCSection *JTS = TLOF.getSectionForJumpTable(F, TM);
+ assert(MJTI->getEntryKind() == MachineJumpTableInfo::EK_LabelDifference32);
+ unsigned EntrySize = MJTI->getEntrySize(getDataLayout());
+ OutStreamer->switchSection(JTS);
+ for (unsigned JTI = 0; JTI < JT.size(); JTI++) {
+ ArrayRef<MachineBasicBlock *> JTBBs = JT[JTI].MBBs;
+ if (JTBBs.empty())
+ continue;
+
+ SmallPtrSet<const MachineBasicBlock *, 16> EmittedSets;
+ const TargetLowering *TLI = MF->getSubtarget().getTargetLowering();
+ const MCExpr *Base = TLI->getPICJumpTableRelocBaseExpr(MF, JTI, OutContext);
+ for (const MachineBasicBlock *MBB : JTBBs) {
+ if (!EmittedSets.insert(MBB).second)
+ continue;
+
+ // Offset from gotox to target basic block expressed in number
+ // of instructions, e.g.:
+ //
+ // .L0_0_set_4 = ((LBB0_4 - .LBPF.JX.0.0) >> 3) - 1
+ const MCExpr *LHS = MCSymbolRefExpr::create(MBB->getSymbol(), OutContext);
+ OutStreamer->emitAssignment(
+ GetJTSetSymbol(JTI, MBB->getNumber()),
+ MCBinaryExpr::createSub(
+ MCBinaryExpr::createAShr(
+ MCBinaryExpr::createSub(LHS, Base, OutContext),
+ MCConstantExpr::create(3, OutContext), OutContext),
+ MCConstantExpr::create(1, OutContext), OutContext));
+ }
+ // BPF.JT.0.0:
+ // .long .L0_0_set_4
+ // .long .L0_0_set_2
+ // ...
+ // .size BPF.JT.0.0, 128
+ MCSymbol *JTStart = GetJTISymbol(JTI);
+ OutStreamer->emitLabel(JTStart);
+ for (const MachineBasicBlock *MBB : JTBBs) {
+ MCSymbol *SetSymbol = GetJTSetSymbol(JTI, MBB->getNumber());
+ const MCExpr *V = MCSymbolRefExpr::create(SetSymbol, OutContext);
+ OutStreamer->emitValue(V, EntrySize);
+ }
+ const MCExpr *JTSize = MCConstantExpr::create(JTBBs.size() * 4, OutContext);
+ OutStreamer->emitELFSize(JTStart, JTSize);
+ }
+}
+
char BPFAsmPrinter::ID = 0;
INITIALIZE_PASS(BPFAsmPrinter, "bpf-asm-printer", "BPF Assembly Printer", false,
diff --git a/llvm/lib/Target/BPF/BPFISelLowering.cpp b/llvm/lib/Target/BPF/BPFISelLowering.cpp
index f4f414d192df0..5af456ea2e054 100644
--- a/llvm/lib/Target/BPF/BPFISelLowering.cpp
+++ b/llvm/lib/Target/BPF/BPFISelLowering.cpp
@@ -18,6 +18,7 @@
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineJumpTableInfo.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
#include "llvm/CodeGen/ValueTypes.h"
@@ -25,6 +26,7 @@
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/DiagnosticPrinter.h"
#include "llvm/IR/Module.h"
+#include "llvm/MC/MCAsmInfo.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/MathExtras.h"
@@ -38,6 +40,10 @@ static cl::opt<bool> BPFExpandMemcpyInOrder("bpf-expand-memcpy-in-order",
cl::Hidden, cl::init(false),
cl::desc("Expand memcpy into load/store pairs in order"));
+static cl::opt<unsigned> BPFMinimumJumpTableEntries(
+ "bpf-min-jump-table-entries", cl::init(4), cl::Hidden,
+ cl::desc("Set minimum number of entries to use a jump table on BPF"));
+
static void fail(const SDLoc &DL, SelectionDAG &DAG, const Twine &Msg,
SDValue Val = {}) {
std::string Str;
@@ -66,8 +72,7 @@ BPFTargetLowering::BPFTargetLowering(const TargetMachine &TM,
setStackPointerRegisterToSaveRestore(BPF::R11);
setOperationAction(ISD::BR_CC, MVT::i64, Custom);
- setOperationAction(ISD::BR_JT, MVT::Other, Expand);
- setOperationAction(ISD::BRIND, MVT::Other, Expand);
+ setOperationAction(ISD::BR_JT, MVT::Other, Custom);
setOperationAction(ISD::BRCOND, MVT::Other, Expand);
setOperationAction(ISD::TRAP, MVT::Other, Custom);
@@ -159,6 +164,7 @@ BPFTargetLowering::BPFTargetLowering(const TargetMachine &TM,
setBooleanContents(ZeroOrOneBooleanContent);
setMaxAtomicSizeInBitsSupported(64);
+ setMinimumJumpTableEntries(BPFMinimumJumpTableEntries);
// Function alignments
setMinFunctionAlignment(Align(8));
@@ -332,6 +338,8 @@ SDValue BPFTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
return LowerATOMIC_LOAD_STORE(Op, DAG);
case ISD::TRAP:
return LowerTRAP(Op, DAG);
+ case ISD::BR_JT:
+ return LowerBR_JT(Op, DAG);
}
}
@@ -780,6 +788,16 @@ SDValue BPFTargetLowering::LowerTRAP(SDValue Op, SelectionDAG &DAG) const {
return LowerCall(CLI, InVals);
}
+SDValue BPFTargetLowering::LowerBR_JT(SDValue Op, SelectionDAG &DAG) const {
+ SDValue Chain = Op->getOperand(0);
+ SDValue Table = Op->getOperand(1);
+ SDValue Index = Op->getOperand(2);
+ JumpTableSDNode *JT = cast<JumpTableSDNode>(Table);
+ SDLoc DL(Op);
+ SDValue TargetJT = DAG.getTargetJumpTable(JT->getIndex(), MVT::i32);
+ return DAG.getNode(BPFISD::BPF_BR_JT, DL, MVT::Other, Chain, TargetJT, Index);
+}
+
const char *BPFTargetLowering::getTargetNodeName(unsigned Opcode) const {
switch ((BPFISD::NodeType)Opcode) {
case BPFISD::FIRST_NUMBER:
@@ -796,6 +814,8 @@ const char *BPFTargetLowering::getTargetNodeName(unsigned Opcode) const {
return "BPFISD::Wrapper";
case BPFISD::MEMCPY:
return "BPFISD::MEMCPY";
+ case BPFISD::BPF_BR_JT:
+ return "BPFISD::BPF_BR_JT";
}
return nullptr;
}
@@ -1069,3 +1089,21 @@ bool BPFTargetLowering::isLegalAddressingMode(const DataLayout &DL,
return true;
}
+
+MCSymbol *BPFTargetLowering::getJXAnchorSymbol(const MachineFunction *MF,
+ unsigned JTI) {
+ const MCAsmInfo *MAI = MF->getContext().getAsmInfo();
+ SmallString<60> Name;
+ raw_svector_ostream(Name) << MAI->getPrivateGlobalPrefix() << "BPF.JX."
+ << MF->getFunctionNumber() << '.' << JTI;
+ return MF->getContext().getOrCreateSymbol(Name);
+}
+
+unsigned BPFTargetLowering::getJumpTableEncoding() const {
+ return MachineJumpTableInfo::EK_LabelDifference32;
+}
+
+const MCExpr *BPFTargetLowering::getPICJumpTableRelocBaseExpr(
+ const MachineFunction *MF, unsigned JTI, MCContext &Ctx) const {
+ return MCSymbolRefExpr::create(getJXAnchorSymbol(MF, JTI), Ctx);
+}
diff --git a/llvm/lib/Target/BPF/BPFISelLowering.h b/llvm/lib/Target/BPF/BPFISelLowering.h
index 8f60261c10e9e..f533cd03b0c61 100644
--- a/llvm/lib/Target/BPF/BPFISelLowering.h
+++ b/llvm/lib/Target/BPF/BPFISelLowering.h
@@ -28,7 +28,8 @@ enum NodeType : unsigned {
SELECT_CC,
BR_CC,
Wrapper,
- MEMCPY
+ MEMCPY,
+ BPF_BR_JT,
};
}
@@ -66,6 +67,31 @@ class BPFTargetLowering : public TargetLowering {
MVT getScalarShiftAmountTy(const DataLayout &, EVT) const override;
+ // Always emit EK_LabelDifference32, computed as difference between
+ // JX instruction location and target basic block label.
+ virtual unsigned getJumpTableEncoding() const override;
+
+ // This is a label for JX instructions, used for jump table offsets
+ // computation, e.g.:
+ //
+ // .LBPF.JX.0.0: <------- this is the anchor
+ // .reloc 0, FK_SecRel_8, BPF.JT.0.0
+ // gotox r1
+ // ...
+ // .section .jumptables,"", at progbits
+ // .L0_0_set_7 = ((LBB0_7-.LBPF.JX.0.0)>>3)-1
+ // ...
+ // BPF.JT.0.0: <------- JT definition
+ // .long .L0_0_set_7
+ // ...
+ static MCSymbol *getJXAnchorSymbol(const MachineFunction *MF, unsigned JTI);
+
+ // Refers to a symbol returned by getJXAnchorSymbol(), used by
+ // AsmPrinter::emitJumpTableInfo() to define the .L0_0_set_7 etc above.
+ virtual const MCExpr *
+ getPICJumpTableRelocBaseExpr(const MachineFunction *MF, unsigned JTI,
+ MCContext &Ctx) const override;
+
private:
// Control Instruction Selection Features
bool HasAlu32;
@@ -81,6 +107,7 @@ class BPFTargetLowering : public TargetLowering {
SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerTRAP(SDValue Op, SelectionDAG &DAG) const;
+ SDValue LowerBR_JT(SDValue Op, SelectionDAG &DAG) const;
template <class NodeTy>
SDValue getAddr(NodeTy *N, SelectionDAG &DAG, unsigned Flags = 0) const;
diff --git a/llvm/lib/Target/BPF/BPFInstrInfo.cpp b/llvm/lib/Target/BPF/BPFInstrInfo.cpp
index 70bc163615f61..e9f65b0517f95 100644
--- a/llvm/lib/Target/BPF/BPFInstrInfo.cpp
+++ b/llvm/lib/Target/BPF/BPFInstrInfo.cpp
@@ -181,6 +181,10 @@ bool BPFInstrInfo::analyzeBranch(MachineBasicBlock &MBB,
if (!isUnpredicatedTerminator(*I))
break;
+ // If a JX insn, we're done.
+ if (I->getOpcode() == BPF::JX)
+ break;
+
// A terminator that isn't a branch can't easily be handled
// by this analysis.
if (!I->isBranch())
@@ -259,3 +263,11 @@ unsigned BPFInstrInfo::removeBranch(MachineBasicBlock &MBB,
return Count;
}
+
+int BPFInstrInfo::getJumpTableIndex(const MachineInstr &MI) const {
+ if (MI.getOpcode() != BPF::JX)
+ return -1;
+ const MachineOperand &MO = MI.getOperand(1);
+ assert(MO.isJTI() && "JX operand #0 should be isJTI");
+ return MO.getIndex();
+}
diff --git a/llvm/lib/Target/BPF/BPFInstrInfo.h b/llvm/lib/Target/BPF/BPFInstrInfo.h
index d8bbad44e314e..d88e37975980a 100644
--- a/llvm/lib/Target/BPF/BPFInstrInfo.h
+++ b/llvm/lib/Target/BPF/BPFInstrInfo.h
@@ -58,6 +58,9 @@ class BPFInstrInfo : public BPFGenInstrInfo {
MachineBasicBlock *FBB, ArrayRef<MachineOperand> Cond,
const DebugLoc &DL,
int *BytesAdded = nullptr) const override;
+
+ int getJumpTableIndex(const MachineInstr &MI) const override;
+
private:
void expandMEMCPY(MachineBasicBlock::iterator) const;
diff --git a/llvm/lib/Target/BPF/BPFInstrInfo.td b/llvm/lib/Target/BPF/BPFInstrInfo.td
index b21f1a0eee3b0..581e49419c37f 100644
--- a/llvm/lib/Target/BPF/BPFInstrInfo.td
+++ b/llvm/lib/Target/BPF/BPFInstrInfo.td
@@ -31,6 +31,8 @@ def SDT_BPFMEMCPY : SDTypeProfile<0, 4, [SDTCisVT<0, i64>,
SDTCisVT<1, i64>,
SDTCisVT<2, i64>,
SDTCisVT<3, i64>]>;
+def SDT_BPFBrJt : SDTypeProfile<0, 2, [SDTCisVT<0, i32>, // jump table
+ SDTCisVT<1, i64>]>; // index
def BPFcall : SDNode<"BPFISD::CALL", SDT_BPFCall,
[SDNPHasChain, SDNPOptInGlue, SDNPOutGlue,
@@ -49,6 +51,9 @@ def BPFWrapper : SDNode<"BPFISD::Wrapper", SDT_BPFWrapper>;
def BPFmemcpy : SDNode<"BPFISD::MEMCPY", SDT_BPFMEMCPY,
[SDNPHasChain, SDNPInGlue, SDNPOutGlue,
SDNPMayStore, SDNPMayLoad]>;
+def BPFBrJt : SDNode<"BPFISD::BPF_BR_JT", SDT_BPFBrJt,
+ [SDNPHasChain]>;
+
def BPFIsLittleEndian : Predicate<"Subtarget->isLittleEndian()">;
def BPFIsBigEndian : Predicate<"!Subtarget->isLittleEndian()">;
def BPFHasALU32 : Predicate<"Subtarget->getHasAlu32()">;
@@ -183,6 +188,15 @@ class TYPE_LD_ST<bits<3> mode, bits<2> size,
let Inst{60-59} = size;
}
+// For indirect jump
+class TYPE_IND_JMP<bits<4> op, bits<1> srctype,
+ dag outs, dag ins, string asmstr, list<dag> pattern>
+ : InstBPF<outs, ins, asmstr, pattern> {
+
+ let Inst{63-60} = op;
+ let Inst{59} = srctype;
+}
+
// jump instructions
class JMP_RR<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond>
: TYPE_ALU_JMP<Opc.Value, BPF_X.Value,
@@ -216,6 +230,18 @@ class JMP_RI<BPFJumpOp Opc, string OpcodeStr, PatLeaf Cond>
let BPFClass = BPF_JMP;
}
+class JMP_IND<BPFJumpOp Opc, string OpcodeStr, list<dag> Pattern>
+ : TYPE_ALU_JMP<Opc.Value, BPF_X.Value,
+ (outs),
+ (ins GPR:$dst, i32imm:$jt),
+ !strconcat(OpcodeStr, " $dst"),
+ Pattern> {
+ bits<4> dst;
+
+ let Inst{51-48} = dst;
+ let BPFClass = BPF_JMP;
+}
+
class JMP_JCOND<BPFJumpOp Opc, string OpcodeStr, list<dag> Pattern>
: TYPE_ALU_JMP<Opc.Value, BPF_K.Value,
(outs),
@@ -281,6 +307,10 @@ defm JSLT : J<BPF_JSLT, "s<", BPF_CC_LT, BPF_CC_LT_32>;
defm JSLE : J<BPF_JSLE, "s<=", BPF_CC_LE, BPF_CC_LE_32>;
defm JSET : J<BPF_JSET, "&", NoCond, NoCond>;
def JCOND : JMP_JCOND<BPF_JCOND, "may_goto", []>;
+
+let isIndirectBranch = 1 in {
+ def JX : JMP_IND<BPF_JA, "gotox", [(BPFBrJt tjumptable:$jt, i64:$dst)]>;
+}
}
// ALU instructions
diff --git a/llvm/lib/Target/BPF/BPFMCInstLower.cpp b/llvm/lib/Target/BPF/BPFMCInstLower.cpp
index 040a1fb750702..8d18ff58a92e0 100644
--- a/llvm/lib/Target/BPF/BPFMCInstLower.cpp
+++ b/llvm/lib/Target/BPF/BPFMCInstLower.cpp
@@ -12,6 +12,7 @@
//===----------------------------------------------------------------------===//
#include "BPFMCInstLower.h"
+#include "BPFISelLowering.h"
#include "llvm/CodeGen/AsmPrinter.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineInstr.h"
@@ -19,6 +20,7 @@
#include "llvm/MC/MCContext.h"
#include "llvm/MC/MCExpr.h"
#include "llvm/MC/MCInst.h"
+#include "llvm/MC/MCStreamer.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/raw_ostream.h"
using namespace llvm;
@@ -44,6 +46,29 @@ MCOperand BPFMCInstLower::LowerSymbolOperand(const MachineOperand &MO,
return MCOperand::createExpr(Expr);
}
+MCOperand BPFMCInstLower::LowerJTIOperand(const MachineInstr &MI,
+ const MachineOperand &MO,
+ int JTI) const {
+ // Emit relocation entry referencing jump table symbol plus a label
+ // for JX anchor, e.g.:
+ //
+ // .LBPF.JX.0.0:
+ // .reloc 0, FK_SecRel_8, BPF.JT.0.0
+ // gotox r1
+ assert((MI.getOpcode() == BPF::JX) &&
+ "Jump Table Index operands are expected only for JX instructions");
+ const MachineFunction *MF = MI.getMF();
+ Printer.OutStreamer->emitLabel(BPFTargetLowering::getJXAnchorSymbol(MF, JTI));
+ MCSymbol *JT = Printer.GetJTISymbol(JTI);
+ const MCExpr *Zero = MCConstantExpr::create(0, Ctx);
+ Printer.OutStreamer->emitRelocDirective(*Zero, "FK_SecRel_8",
+ MCSymbolRefExpr::create(JT, Ctx), {},
+ *Ctx.getSubtargetInfo());
+ // JTI parameter is used only to emit relocation and is not a part
+ // of JX instruction encoding, so this operand is not really used.
+ return MCOperand::createImm(JTI);
+}
+
void BPFMCInstLower::Lower(const MachineInstr *MI, MCInst &OutMI) const {
OutMI.setOpcode(MI->getOpcode());
@@ -77,6 +102,9 @@ void BPFMCInstLower::Lower(const MachineInstr *MI, MCInst &OutMI) const {
case MachineOperand::MO_ConstantPoolIndex:
MCOp = LowerSymbolOperand(MO, Printer.GetCPISymbol(MO.getIndex()));
break;
+ case MachineOperand::MO_JumpTableIndex:
+ MCOp = LowerJTIOperand(*MI, MO, MO.getIndex());
+ break;
}
OutMI.addOperand(MCOp);
diff --git a/llvm/lib/Target/BPF/BPFMCInstLower.h b/llvm/lib/Target/BPF/BPFMCInstLower.h
index 4bd0f1f0bf1cf..927e9dfaf8c68 100644
--- a/llvm/lib/Target/BPF/BPFMCInstLower.h
+++ b/llvm/lib/Target/BPF/BPFMCInstLower.h
@@ -32,6 +32,8 @@ class LLVM_LIBRARY_VISIBILITY BPFMCInstLower {
void Lower(const MachineInstr *MI, MCInst &OutMI) const;
MCOperand LowerSymbolOperand(const MachineOperand &MO, MCSymbol *Sym) const;
+ MCOperand LowerJTIOperand(const MachineInstr &MI, const MachineOperand &MO,
+ int JTI) const;
MCSymbol *GetGlobalAddressSymbol(const MachineOperand &MO) const;
MCSymbol *GetExternalSymbolSymbol(const MachineOperand &MO) const;
diff --git a/llvm/lib/Target/BPF/BPFTargetLoweringObjectFile.cpp b/llvm/lib/Target/BPF/BPFTargetLoweringObjectFile.cpp
new file mode 100644
index 0000000000000..997f09870bad6
--- /dev/null
+++ b/llvm/lib/Target/BPF/BPFTargetLoweringObjectFile.cpp
@@ -0,0 +1,19 @@
+//===------------------ BPFTargetLoweringObjectFile.cpp -------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "BPFTargetLoweringObjectFile.h"
+#include "llvm/MC/MCContext.h"
+#include "llvm/MC/MCSectionELF.h"
+
+using namespace llvm;
+
+MCSection *BPFTargetLoweringObjectFileELF::getSectionForJumpTable(
+ const Function &F, const TargetMachine &TM,
+ const MachineJumpTableEntry *JTE) const {
+ return getContext().getELFSection(".jumptables", ELF::SHT_PROGBITS, 0);
+}
diff --git a/llvm/lib/Target/BPF/BPFTargetLoweringObjectFile.h b/llvm/lib/Target/BPF/BPFTargetLoweringObjectFile.h
new file mode 100644
index 0000000000000..f3064c0c8cb8a
--- /dev/null
+++ b/llvm/lib/Target/BPF/BPFTargetLoweringObjectFile.h
@@ -0,0 +1,25 @@
+//===============- BPFTargetLoweringObjectFile.h -*- C++ -*-================//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_BPF_BPFTARGETLOWERINGOBJECTFILE
+#define LLVM_LIB_TARGET_BPF_BPFTARGETLOWERINGOBJECTFILE
+
+#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
+#include "llvm/Target/TargetLoweringObjectFile.h"
+
+namespace llvm {
+class BPFTargetLoweringObjectFileELF : public TargetLoweringObjectFileELF {
+
+public:
+ virtual MCSection *
+ getSectionForJumpTable(const Function &F, const TargetMachine &TM,
+ const MachineJumpTableEntry *JTE) const override;
+};
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_BPF_BPFTARGETLOWERINGOBJECTFILE
diff --git a/llvm/lib/Target/BPF/BPFTargetMachine.cpp b/llvm/lib/Target/BPF/BPFTargetMachine.cpp
index 527a480354571..d538b6fe11675 100644
--- a/llvm/lib/Target/BPF/BPFTargetMachine.cpp
+++ b/llvm/lib/Target/BPF/BPFTargetMachine.cpp
@@ -12,6 +12,7 @@
#include "BPFTargetMachine.h"
#include "BPF.h"
+#include "BPFTargetLoweringObjectFile.h"
#include "BPFTargetTransformInfo.h"
#include "MCTargetDesc/BPFMCAsmInfo.h"
#include "TargetInfo/BPFTargetInfo.h"
@@ -80,7 +81,7 @@ BPFTargetMachine::BPFTargetMachine(const Target &T, const Triple &TT,
: CodeGenTargetMachineImpl(T, computeDataLayout(TT), TT, CPU, FS, Options,
getEffectiveRelocModel(RM),
getEffectiveCodeModel(CM, CodeModel::Small), OL),
- TLOF(std::make_unique<TargetLoweringObjectFileELF>()),
+ TLOF(std::make_unique<BPFTargetLoweringObjectFileELF>()),
Subtarget(TT, std::string(CPU), std::string(FS), *this) {
if (!DisableCheckUnreachable) {
this->Options.TrapUnreachable = true;
diff --git a/llvm/lib/Target/BPF/CMakeLists.txt b/llvm/lib/Target/BPF/CMakeLists.txt
index eade4cacb7100..3678f1335ca36 100644
--- a/llvm/lib/Target/BPF/CMakeLists.txt
+++ b/llvm/lib/Target/BPF/CMakeLists.txt
@@ -37,6 +37,7 @@ add_llvm_target(BPFCodeGen
BPFRegisterInfo.cpp
BPFSelectionDAGInfo.cpp
BPFSubtarget.cpp
+ BPFTargetLoweringObjectFile.cpp
BPFTargetMachine.cpp
BPFMIPeephole.cpp
BPFMIChecking.cpp
diff --git a/llvm/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp b/llvm/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp
index 958790d49d087..7165a88fb17e7 100644
--- a/llvm/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp
+++ b/llvm/lib/Target/BPF/MCTargetDesc/BPFAsmBackend.cpp
@@ -34,6 +34,7 @@ class BPFAsmBackend : public MCAsmBackend {
createObjectTargetWriter() const override;
MCFixupKindInfo getFixupKindInfo(MCFixupKind Kind) const override;
+ std::optional<MCFixupKind> getFixupKind(StringRef Name) const override;
bool writeNopData(raw_ostream &OS, uint64_t Count,
const MCSubtargetInfo *STI) const override;
@@ -54,6 +55,18 @@ MCFixupKindInfo BPFAsmBackend::getFixupKindInfo(MCFixupKind Kind) const {
return Infos[Kind - FirstTargetFixupKind];
}
+std::optional<MCFixupKind> BPFAsmBackend::getFixupKind(StringRef Name) const {
+ if (Name == "FK_SecRel_8")
+ return FK_SecRel_8;
+ if (Name == "FK_BPF_PCRel_4")
+ return BPF::FK_BPF_PCRel_4;
+ if (Name == "FK_Data_8")
+ return FK_Data_8;
+ if (Name == "FK_Data_4")
+ return FK_Data_4;
+ return std::nullopt;
+}
+
bool BPFAsmBackend::writeNopData(raw_ostream &OS, uint64_t Count,
const MCSubtargetInfo *STI) const {
if ((Count % 8) != 0)
>From 024f351f9c5680e471e1cbbd1b47715890b6711b Mon Sep 17 00:00:00 2001
From: Eduard Zingerman <eddyz87 at gmail.com>
Date: Thu, 10 Jul 2025 14:14:29 -0700
Subject: [PATCH 2/5] [BPF] Don't insert unreachable 'goto' after JX
instructions
Update BPFInstrInfo::analyzeBranch() to comply with
TargetInstrInfo::analyzeBranch() requirements for JX instruction:
if branch instruction can't be categorized as a conditional with
true/false branches -- return true.
Because of this bug MachineBlockPlacement transformation inserted an
additional unreachabe jump after JX, e.g.:
bb.1.entry:
...
JX killed $r1, %jump-table.0
JMP %bb.2
Additionally, isNotDuplicable annotation is necessary to avoid machine
level transformations creating several JX instruction copies.
Such copies would refer to the same jump table and would make it not
possible to calculate jump offsets inside the table.
Files triggering such duplication are present in kernel selftests.
---
llvm/lib/Target/BPF/BPFInstrInfo.cpp | 7 ++++---
llvm/lib/Target/BPF/BPFInstrInfo.td | 2 +-
2 files changed, 5 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/Target/BPF/BPFInstrInfo.cpp b/llvm/lib/Target/BPF/BPFInstrInfo.cpp
index e9f65b0517f95..6275995ce9ba2 100644
--- a/llvm/lib/Target/BPF/BPFInstrInfo.cpp
+++ b/llvm/lib/Target/BPF/BPFInstrInfo.cpp
@@ -181,9 +181,10 @@ bool BPFInstrInfo::analyzeBranch(MachineBasicBlock &MBB,
if (!isUnpredicatedTerminator(*I))
break;
- // If a JX insn, we're done.
- if (I->getOpcode() == BPF::JX)
- break;
+ // From base method doc: ... returning true if it cannot be understood ...
+ // Indirect branch has multiple destinations and no true/false concepts.
+ if (I->isIndirectBranch())
+ return true;
// A terminator that isn't a branch can't easily be handled
// by this analysis.
diff --git a/llvm/lib/Target/BPF/BPFInstrInfo.td b/llvm/lib/Target/BPF/BPFInstrInfo.td
index 581e49419c37f..83dccb132c519 100644
--- a/llvm/lib/Target/BPF/BPFInstrInfo.td
+++ b/llvm/lib/Target/BPF/BPFInstrInfo.td
@@ -308,7 +308,7 @@ defm JSLE : J<BPF_JSLE, "s<=", BPF_CC_LE, BPF_CC_LE_32>;
defm JSET : J<BPF_JSET, "&", NoCond, NoCond>;
def JCOND : JMP_JCOND<BPF_JCOND, "may_goto", []>;
-let isIndirectBranch = 1 in {
+let isIndirectBranch = 1, isBarrier = 1, isNotDuplicable = 1 in {
def JX : JMP_IND<BPF_JA, "gotox", [(BPFBrJt tjumptable:$jt, i64:$dst)]>;
}
}
>From a6fecc9c1e8ed869ab0444c447d9523a2b3566d0 Mon Sep 17 00:00:00 2001
From: Eduard Zingerman <eddyz87 at gmail.com>
Date: Thu, 10 Jul 2025 17:41:30 -0700
Subject: [PATCH 3/5] [BPF] add test cases for jump tables
- one testing a general structure of the generated code;
- another testing that several jump tables within the same functions
are generated independently.
---
llvm/test/CodeGen/BPF/jump-table-multi.ll | 78 ++++++++++++++++++++
llvm/test/CodeGen/BPF/jump-table-simple.ll | 86 ++++++++++++++++++++++
2 files changed, 164 insertions(+)
create mode 100644 llvm/test/CodeGen/BPF/jump-table-multi.ll
create mode 100644 llvm/test/CodeGen/BPF/jump-table-simple.ll
diff --git a/llvm/test/CodeGen/BPF/jump-table-multi.ll b/llvm/test/CodeGen/BPF/jump-table-multi.ll
new file mode 100644
index 0000000000000..4f8b9ad428804
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/jump-table-multi.ll
@@ -0,0 +1,78 @@
+; RUN: llc -O2 -bpf-min-jump-table-entries=1 -mtriple=bpfel -mcpu=v4 < %s | FileCheck %s
+
+; Check that two jump tables of different size are generated
+
+define i64 @foo(i64 %v1, i64 %v2) {
+; CHECK: .LBPF.JX.0.0:
+; CHECK-NEXT: .reloc 0, FK_SecRel_8, BPF.JT.0.0
+; CHECK-NEXT: gotox r1
+
+; CHECK: .LBPF.JX.0.1:
+; CHECK-NEXT: .reloc 0, FK_SecRel_8, BPF.JT.0.1
+; CHECK-NEXT: gotox r2
+
+; CHECK: .section .jumptables,"", at progbits
+
+; CHECK-NEXT: [[m1:.*]] = (({{.*}}-.LBPF.JX.0.0)>>3)-1
+; CHECK-NEXT: [[m2:.*]] = (({{.*}}-.LBPF.JX.0.0)>>3)-1
+; CHECK-NEXT: BPF.JT.0.0:
+; CHECK-NEXT: .long [[m1]]
+; CHECK-NEXT: .long [[m2]]
+; CHECK-NEXT: .size BPF.JT.0.0, 8
+
+; CHECK-NEXT: [[m1:.*]] = (({{.*}}-.LBPF.JX.0.1)>>3)-1
+; CHECK-NEXT: [[m2:.*]] = (({{.*}}-.LBPF.JX.0.1)>>3)-1
+; CHECK-NEXT: [[m3:.*]] = (({{.*}}-.LBPF.JX.0.1)>>3)-1
+; CHECK-NEXT: BPF.JT.0.1:
+; CHECK-NEXT: .long [[m1]]
+; CHECK-NEXT: .long [[m2]]
+; CHECK-NEXT: .long [[m3]]
+; CHECK-NEXT: .size BPF.JT.0.1, 12
+
+entry:
+ switch i64 %v1, label %sw.default [
+ i64 0, label %sw.bb1
+ i64 1, label %sw.bb2
+ ]
+
+sw.bb1:
+ br label %sw.epilog
+
+sw.bb2:
+ br label %sw.epilog
+
+sw.default:
+ br label %sw.epilog
+
+sw.epilog:
+ %ret = phi i64 [ 42, %sw.default ], [ 3, %sw.bb1 ], [ 5, %sw.bb2 ]
+ switch i64 %v2, label %sw.default.1 [
+ i64 0, label %sw.bb1.1
+ i64 1, label %sw.bb2.1
+ i64 2, label %sw.bb3.1
+ ]
+
+sw.bb1.1:
+ br label %sw.epilog.1
+
+sw.bb2.1:
+ br label %sw.epilog.1
+
+sw.bb3.1:
+ br label %sw.epilog.1
+
+sw.default.1:
+ br label %sw.epilog.1
+
+sw.epilog.1:
+ %ret.1 = phi i64 [ 42, %sw.default.1 ], [ 3, %sw.bb1.1 ], [ 5, %sw.bb2.1 ], [ 7, %sw.bb3.1 ]
+ %ret.2 = add i64 %ret, %ret.1
+ ret i64 %ret.2
+}
+
+!llvm.module.flags = !{!0, !1}
+!llvm.ident = !{!2}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 7, !"frame-pointer", i32 2}
+!2 = !{!"clang some version"}
diff --git a/llvm/test/CodeGen/BPF/jump-table-simple.ll b/llvm/test/CodeGen/BPF/jump-table-simple.ll
new file mode 100644
index 0000000000000..8c048223dbc0b
--- /dev/null
+++ b/llvm/test/CodeGen/BPF/jump-table-simple.ll
@@ -0,0 +1,86 @@
+; Checks generated using command:
+;
+; llvm/utils/update_test_body.py llvm/test/CodeGen/BPF/jump-table-simple.ll
+;
+; RUN: rm -rf %t && split-file %s %t && cd %t
+; RUN: llc -O2 -bpf-min-jump-table-entries=1 -mtriple=bpfel -mcpu=v4 < test.ll | FileCheck %s
+;
+; Check general program structure generated for a jump table
+
+.ifdef GEN
+;--- test.ll
+define i64 @foo(i64 %v) {
+entry:
+ switch i64 %v, label %sw.default [
+ i64 0, label %sw.epilog
+ i64 1, label %sw.bb1
+ i64 2, label %sw.bb1
+ i64 3, label %sw.bb2
+ ]
+
+sw.bb1:
+ br label %sw.epilog
+
+sw.bb2:
+ br label %sw.epilog
+
+sw.default:
+ br label %sw.epilog
+
+sw.epilog:
+ %ret = phi i64 [ 42, %sw.default ], [ 3, %sw.bb1 ], [ 5, %sw.bb2 ], [ 7, %entry ]
+ ret i64 %ret
+}
+
+!llvm.module.flags = !{!0, !1}
+!llvm.ident = !{!2}
+
+!0 = !{i32 1, !"wchar_size", i32 4}
+!1 = !{i32 7, !"frame-pointer", i32 2}
+!2 = !{!"clang some version"}
+
+;--- gen
+echo ""
+echo "; Generated checks follow"
+echo ";"
+llc -O2 -bpf-min-jump-table-entries=1 -mtriple=bpfel -mcpu=v4 < test.ll \
+ | awk '/# -- End function/ {p=0} /@function/ {p=1} p {print "; CHECK" ": " $0}'
+.endif
+
+; Generated checks follow
+;
+; CHECK: .type foo, at function
+; CHECK: foo: # @foo
+; CHECK: .cfi_startproc
+; CHECK: # %bb.0: # %entry
+; CHECK: if r1 > 3 goto LBB0_5
+; CHECK: # %bb.1: # %entry
+; CHECK: .LBPF.JX.0.0:
+; CHECK: .reloc 0, FK_SecRel_8, BPF.JT.0.0
+; CHECK: gotox r1
+; CHECK: LBB0_3: # %sw.bb1
+; CHECK: r0 = 3
+; CHECK: goto LBB0_6
+; CHECK: LBB0_2:
+; CHECK: r0 = 7
+; CHECK: goto LBB0_6
+; CHECK: LBB0_4: # %sw.bb2
+; CHECK: r0 = 5
+; CHECK: goto LBB0_6
+; CHECK: LBB0_5: # %sw.default
+; CHECK: r0 = 42
+; CHECK: LBB0_6: # %sw.epilog
+; CHECK: exit
+; CHECK: .Lfunc_end0:
+; CHECK: .size foo, .Lfunc_end0-foo
+; CHECK: .cfi_endproc
+; CHECK: .section .jumptables,"", at progbits
+; CHECK: .L0_0_set_2 = ((LBB0_2-.LBPF.JX.0.0)>>3)-1
+; CHECK: .L0_0_set_3 = ((LBB0_3-.LBPF.JX.0.0)>>3)-1
+; CHECK: .L0_0_set_4 = ((LBB0_4-.LBPF.JX.0.0)>>3)-1
+; CHECK: BPF.JT.0.0:
+; CHECK: .long .L0_0_set_2
+; CHECK: .long .L0_0_set_3
+; CHECK: .long .L0_0_set_3
+; CHECK: .long .L0_0_set_4
+; CHECK: .size BPF.JT.0.0, 16
>From 9aec188a0a916a37cec809e02c0cdcb7369ffdaf Mon Sep 17 00:00:00 2001
From: Eduard Zingerman <eddyz87 at gmail.com>
Date: Thu, 10 Jul 2025 21:27:55 -0700
Subject: [PATCH 4/5] [BPF] generate gotox only for cpuv4
Coincidentally this fixes two test failures:
- LLVM :: CodeGen/BPF/CORE/offset-reloc-fieldinfo-2-bpfeb.ll
- LLVM :: CodeGen/BPF/CORE/offset-reloc-fieldinfo-2.ll
These tests invoke llc with -mcpuv1 and have a switch statement in the
IR. Both tests failed with assertion in
SelectionDAGLegalize::LegalizeOp():
for (const SDValue &Op : Node->op_values())
assert((TLI.getTypeAction(*DAG.getContext(), Op.getValueType()) ==
TargetLowering::TypeLegal ||
Op.getOpcode() == ISD::TargetConstant ||
Op.getOpcode() == ISD::Register) &&
"Unexpected illegal type!");
At the moment of the failure:
Op.getOpcode() == BPFISD::BPF_BR_JT
The error happened because one of the BPFBrJt parameters has i32 type:
def SDT_BPFBrJt : SDTypeProfile<0, 2, [SDTCisVT<0, i32>, // jump table
SDTCisVT<1, i64>]>; // index
def BPFBrJt : SDNode<"BPFISD::BPF_BR_JT", SDT_BPFBrJt,
[SDNPHasChain]>;
---
llvm/lib/Target/BPF/BPFISelLowering.cpp | 4 +++-
llvm/lib/Target/BPF/BPFSubtarget.cpp | 4 ++++
llvm/lib/Target/BPF/BPFSubtarget.h | 3 ++-
3 files changed, 9 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/BPF/BPFISelLowering.cpp b/llvm/lib/Target/BPF/BPFISelLowering.cpp
index 5af456ea2e054..742690238aa8e 100644
--- a/llvm/lib/Target/BPF/BPFISelLowering.cpp
+++ b/llvm/lib/Target/BPF/BPFISelLowering.cpp
@@ -72,8 +72,10 @@ BPFTargetLowering::BPFTargetLowering(const TargetMachine &TM,
setStackPointerRegisterToSaveRestore(BPF::R11);
setOperationAction(ISD::BR_CC, MVT::i64, Custom);
- setOperationAction(ISD::BR_JT, MVT::Other, Custom);
setOperationAction(ISD::BRCOND, MVT::Other, Expand);
+ LegalizeAction IndirectBrAction = STI.hasGotox() ? Custom : Expand;
+ setOperationAction(ISD::BR_JT, MVT::Other, IndirectBrAction);
+ setOperationAction(ISD::BRIND, MVT::Other, IndirectBrAction);
setOperationAction(ISD::TRAP, MVT::Other, Custom);
diff --git a/llvm/lib/Target/BPF/BPFSubtarget.cpp b/llvm/lib/Target/BPF/BPFSubtarget.cpp
index 4167547680b12..a11aa6933147c 100644
--- a/llvm/lib/Target/BPF/BPFSubtarget.cpp
+++ b/llvm/lib/Target/BPF/BPFSubtarget.cpp
@@ -43,6 +43,8 @@ static cl::opt<bool>
static cl::opt<bool> Disable_load_acq_store_rel(
"disable-load-acq-store-rel", cl::Hidden, cl::init(false),
cl::desc("Disable load-acquire and store-release insns"));
+static cl::opt<bool> Disable_gotox("disable-gotox", cl::Hidden, cl::init(false),
+ cl::desc("Disable gotox insn"));
void BPFSubtarget::anchor() {}
@@ -66,6 +68,7 @@ void BPFSubtarget::initializeEnvironment() {
HasGotol = false;
HasStoreImm = false;
HasLoadAcqStoreRel = false;
+ HasGotox = false;
}
void BPFSubtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) {
@@ -96,6 +99,7 @@ void BPFSubtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) {
HasGotol = !Disable_gotol;
HasStoreImm = !Disable_StoreImm;
HasLoadAcqStoreRel = !Disable_load_acq_store_rel;
+ HasGotox = !Disable_gotox;
return;
}
}
diff --git a/llvm/lib/Target/BPF/BPFSubtarget.h b/llvm/lib/Target/BPF/BPFSubtarget.h
index aed2211265e23..e870dfdc85ec9 100644
--- a/llvm/lib/Target/BPF/BPFSubtarget.h
+++ b/llvm/lib/Target/BPF/BPFSubtarget.h
@@ -65,7 +65,7 @@ class BPFSubtarget : public BPFGenSubtargetInfo {
// whether cpu v4 insns are enabled.
bool HasLdsx, HasMovsx, HasBswap, HasSdivSmod, HasGotol, HasStoreImm,
- HasLoadAcqStoreRel;
+ HasLoadAcqStoreRel, HasGotox;
std::unique_ptr<CallLowering> CallLoweringInfo;
std::unique_ptr<InstructionSelector> InstSelector;
@@ -94,6 +94,7 @@ class BPFSubtarget : public BPFGenSubtargetInfo {
bool hasGotol() const { return HasGotol; }
bool hasStoreImm() const { return HasStoreImm; }
bool hasLoadAcqStoreRel() const { return HasLoadAcqStoreRel; }
+ bool hasGotox() const { return HasGotox; }
bool isLittleEndian() const { return IsLittleEndian; }
>From ddc6c1f26e128fd8810180be70b4d00b844a1b61 Mon Sep 17 00:00:00 2001
From: Eduard Zingerman <eddyz87 at gmail.com>
Date: Thu, 10 Jul 2025 23:56:32 -0700
Subject: [PATCH 5/5] [BPF] consolidate jump table emit code in BPFAsmPrinter
The requirement to emit jump table entries as offsets measured in
instructions, e.g. as follows:
.L0_0_set_7 = ((LBB0_7-.LBPF.JX.0.0)>>3)-1
Makes it impossible to use generic AsmPrinter::emitJumpTableInfo()
function. Merge request used this generic function before
(and incorrect offsets were generated).
This generic function required two overloads:
- AsmPrinter::GetJTISymbol()
- TargetLowering::getPICJumpTableRelocBaseExpr()
Now all jump table emission logic is located in the
BPFAsmPrinter::emitJumpTableInfo(), which does not require above
overloads. Hence, remove the overloads and move corresponding code to
BPFAsmPrinter to keep it in one place.
---
llvm/include/llvm/CodeGen/AsmPrinter.h | 3 +-
llvm/lib/Target/BPF/BPFAsmPrinter.cpp | 59 +++++++++++--------------
llvm/lib/Target/BPF/BPFAsmPrinter.h | 44 ++++++++++++++++++
llvm/lib/Target/BPF/BPFISelLowering.cpp | 14 ------
llvm/lib/Target/BPF/BPFISelLowering.h | 21 ---------
llvm/lib/Target/BPF/BPFMCInstLower.cpp | 6 +--
llvm/lib/Target/BPF/BPFMCInstLower.h | 6 +--
7 files changed, 77 insertions(+), 76 deletions(-)
create mode 100644 llvm/lib/Target/BPF/BPFAsmPrinter.h
diff --git a/llvm/include/llvm/CodeGen/AsmPrinter.h b/llvm/include/llvm/CodeGen/AsmPrinter.h
index db328f4b2003e..faab2503ced50 100644
--- a/llvm/include/llvm/CodeGen/AsmPrinter.h
+++ b/llvm/include/llvm/CodeGen/AsmPrinter.h
@@ -663,8 +663,7 @@ class LLVM_ABI AsmPrinter : public MachineFunctionPass {
MCSymbol *GetExternalSymbolSymbol(const Twine &Sym) const;
/// Return the symbol for the specified jump table entry.
- virtual MCSymbol *GetJTISymbol(unsigned JTID,
- bool isLinkerPrivate = false) const;
+ MCSymbol *GetJTISymbol(unsigned JTID, bool isLinkerPrivate = false) const;
/// Return the symbol for the specified jump table .set
/// FIXME: privatize to AsmPrinter.
diff --git a/llvm/lib/Target/BPF/BPFAsmPrinter.cpp b/llvm/lib/Target/BPF/BPFAsmPrinter.cpp
index d99b7bec4600a..a6d8976a734dc 100644
--- a/llvm/lib/Target/BPF/BPFAsmPrinter.cpp
+++ b/llvm/lib/Target/BPF/BPFAsmPrinter.cpp
@@ -11,6 +11,7 @@
//
//===----------------------------------------------------------------------===//
+#include "BPFAsmPrinter.h"
#include "BPF.h"
#include "BPFInstrInfo.h"
#include "BPFMCInstLower.h"
@@ -39,33 +40,6 @@ using namespace llvm;
#define DEBUG_TYPE "asm-printer"
-namespace {
-class BPFAsmPrinter : public AsmPrinter {
-public:
- explicit BPFAsmPrinter(TargetMachine &TM,
- std::unique_ptr<MCStreamer> Streamer)
- : AsmPrinter(TM, std::move(Streamer), ID), BTF(nullptr) {}
-
- StringRef getPassName() const override { return "BPF Assembly Printer"; }
- bool doInitialization(Module &M) override;
- void printOperand(const MachineInstr *MI, int OpNum, raw_ostream &O);
- bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
- const char *ExtraCode, raw_ostream &O) override;
- bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum,
- const char *ExtraCode, raw_ostream &O) override;
-
- void emitInstruction(const MachineInstr *MI) override;
- virtual MCSymbol *GetJTISymbol(unsigned JTID,
- bool isLinkerPrivate = false) const override;
- virtual void emitJumpTableInfo() override;
-
- static char ID;
-
-private:
- BTFDebug *BTF;
-};
-} // namespace
-
bool BPFAsmPrinter::doInitialization(Module &M) {
AsmPrinter::doInitialization(M);
@@ -159,11 +133,31 @@ void BPFAsmPrinter::emitInstruction(const MachineInstr *MI) {
EmitToStreamer(*OutStreamer, TmpInst);
}
-MCSymbol *BPFAsmPrinter::GetJTISymbol(unsigned JTID,
- bool isLinkerPrivate) const {
+// This is a label for JX instructions, used for jump table offsets
+// computation, e.g.:
+//
+// .LBPF.JX.0.0: <------- this is the anchor
+// .reloc 0, FK_SecRel_8, BPF.JT.0.0
+// gotox r1
+// ...
+// .section .jumptables,"", at progbits
+// .L0_0_set_7 = ((LBB0_7-.LBPF.JX.0.0)>>3)-1
+// ...
+// BPF.JT.0.0: <------- JT definition
+// .long .L0_0_set_7
+// ...
+MCSymbol *BPFAsmPrinter::getJXAnchorSymbol(unsigned JTI) {
+ const MCAsmInfo *MAI = MF->getContext().getAsmInfo();
+ SmallString<60> Name;
+ raw_svector_ostream(Name) << MAI->getPrivateGlobalPrefix() << "BPF.JX."
+ << MF->getFunctionNumber() << '.' << JTI;
+ return MF->getContext().getOrCreateSymbol(Name);
+}
+
+MCSymbol *BPFAsmPrinter::getJTPublicSymbol(unsigned JTI) {
SmallString<60> Name;
raw_svector_ostream(Name)
- << "BPF.JT." << MF->getFunctionNumber() << '.' << JTID;
+ << "BPF.JT." << MF->getFunctionNumber() << '.' << JTI;
MCSymbol *S = OutContext.getOrCreateSymbol(Name);
if (auto *ES = dyn_cast<MCSymbolELF>(S))
ES->setBinding(ELF::STB_GLOBAL);
@@ -191,8 +185,7 @@ void BPFAsmPrinter::emitJumpTableInfo() {
continue;
SmallPtrSet<const MachineBasicBlock *, 16> EmittedSets;
- const TargetLowering *TLI = MF->getSubtarget().getTargetLowering();
- const MCExpr *Base = TLI->getPICJumpTableRelocBaseExpr(MF, JTI, OutContext);
+ auto *Base = MCSymbolRefExpr::create(getJXAnchorSymbol(JTI), OutContext);
for (const MachineBasicBlock *MBB : JTBBs) {
if (!EmittedSets.insert(MBB).second)
continue;
@@ -215,7 +208,7 @@ void BPFAsmPrinter::emitJumpTableInfo() {
// .long .L0_0_set_2
// ...
// .size BPF.JT.0.0, 128
- MCSymbol *JTStart = GetJTISymbol(JTI);
+ MCSymbol *JTStart = getJTPublicSymbol(JTI);
OutStreamer->emitLabel(JTStart);
for (const MachineBasicBlock *MBB : JTBBs) {
MCSymbol *SetSymbol = GetJTSetSymbol(JTI, MBB->getNumber());
diff --git a/llvm/lib/Target/BPF/BPFAsmPrinter.h b/llvm/lib/Target/BPF/BPFAsmPrinter.h
new file mode 100644
index 0000000000000..d46d789274a9e
--- /dev/null
+++ b/llvm/lib/Target/BPF/BPFAsmPrinter.h
@@ -0,0 +1,44 @@
+//===-- BPFFrameLowering.h - Define frame lowering for BPF -----*- C++ -*--===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_BPF_BPFASMPRINTER_H
+#define LLVM_LIB_TARGET_BPF_BPFASMPRINTER_H
+
+#include "BTFDebug.h"
+#include "llvm/CodeGen/AsmPrinter.h"
+
+namespace llvm {
+
+class BPFAsmPrinter : public AsmPrinter {
+public:
+ explicit BPFAsmPrinter(TargetMachine &TM,
+ std::unique_ptr<MCStreamer> Streamer)
+ : AsmPrinter(TM, std::move(Streamer), ID), BTF(nullptr) {}
+
+ StringRef getPassName() const override { return "BPF Assembly Printer"; }
+ bool doInitialization(Module &M) override;
+ void printOperand(const MachineInstr *MI, int OpNum, raw_ostream &O);
+ bool PrintAsmOperand(const MachineInstr *MI, unsigned OpNo,
+ const char *ExtraCode, raw_ostream &O) override;
+ bool PrintAsmMemoryOperand(const MachineInstr *MI, unsigned OpNum,
+ const char *ExtraCode, raw_ostream &O) override;
+
+ void emitInstruction(const MachineInstr *MI) override;
+ MCSymbol *getJTPublicSymbol(unsigned JTI);
+ MCSymbol *getJXAnchorSymbol(unsigned JTI);
+ virtual void emitJumpTableInfo() override;
+
+ static char ID;
+
+private:
+ BTFDebug *BTF;
+};
+
+} // namespace llvm
+
+#endif /* LLVM_LIB_TARGET_BPF_BPFASMPRINTER_H */
diff --git a/llvm/lib/Target/BPF/BPFISelLowering.cpp b/llvm/lib/Target/BPF/BPFISelLowering.cpp
index 742690238aa8e..98c9b2ed56959 100644
--- a/llvm/lib/Target/BPF/BPFISelLowering.cpp
+++ b/llvm/lib/Target/BPF/BPFISelLowering.cpp
@@ -1092,20 +1092,6 @@ bool BPFTargetLowering::isLegalAddressingMode(const DataLayout &DL,
return true;
}
-MCSymbol *BPFTargetLowering::getJXAnchorSymbol(const MachineFunction *MF,
- unsigned JTI) {
- const MCAsmInfo *MAI = MF->getContext().getAsmInfo();
- SmallString<60> Name;
- raw_svector_ostream(Name) << MAI->getPrivateGlobalPrefix() << "BPF.JX."
- << MF->getFunctionNumber() << '.' << JTI;
- return MF->getContext().getOrCreateSymbol(Name);
-}
-
unsigned BPFTargetLowering::getJumpTableEncoding() const {
return MachineJumpTableInfo::EK_LabelDifference32;
}
-
-const MCExpr *BPFTargetLowering::getPICJumpTableRelocBaseExpr(
- const MachineFunction *MF, unsigned JTI, MCContext &Ctx) const {
- return MCSymbolRefExpr::create(getJXAnchorSymbol(MF, JTI), Ctx);
-}
diff --git a/llvm/lib/Target/BPF/BPFISelLowering.h b/llvm/lib/Target/BPF/BPFISelLowering.h
index f533cd03b0c61..6e5bb2c814032 100644
--- a/llvm/lib/Target/BPF/BPFISelLowering.h
+++ b/llvm/lib/Target/BPF/BPFISelLowering.h
@@ -71,27 +71,6 @@ class BPFTargetLowering : public TargetLowering {
// JX instruction location and target basic block label.
virtual unsigned getJumpTableEncoding() const override;
- // This is a label for JX instructions, used for jump table offsets
- // computation, e.g.:
- //
- // .LBPF.JX.0.0: <------- this is the anchor
- // .reloc 0, FK_SecRel_8, BPF.JT.0.0
- // gotox r1
- // ...
- // .section .jumptables,"", at progbits
- // .L0_0_set_7 = ((LBB0_7-.LBPF.JX.0.0)>>3)-1
- // ...
- // BPF.JT.0.0: <------- JT definition
- // .long .L0_0_set_7
- // ...
- static MCSymbol *getJXAnchorSymbol(const MachineFunction *MF, unsigned JTI);
-
- // Refers to a symbol returned by getJXAnchorSymbol(), used by
- // AsmPrinter::emitJumpTableInfo() to define the .L0_0_set_7 etc above.
- virtual const MCExpr *
- getPICJumpTableRelocBaseExpr(const MachineFunction *MF, unsigned JTI,
- MCContext &Ctx) const override;
-
private:
// Control Instruction Selection Features
bool HasAlu32;
diff --git a/llvm/lib/Target/BPF/BPFMCInstLower.cpp b/llvm/lib/Target/BPF/BPFMCInstLower.cpp
index 8d18ff58a92e0..6b18a1468d57f 100644
--- a/llvm/lib/Target/BPF/BPFMCInstLower.cpp
+++ b/llvm/lib/Target/BPF/BPFMCInstLower.cpp
@@ -12,6 +12,7 @@
//===----------------------------------------------------------------------===//
#include "BPFMCInstLower.h"
+#include "BPFAsmPrinter.h"
#include "BPFISelLowering.h"
#include "llvm/CodeGen/AsmPrinter.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
@@ -57,9 +58,8 @@ MCOperand BPFMCInstLower::LowerJTIOperand(const MachineInstr &MI,
// gotox r1
assert((MI.getOpcode() == BPF::JX) &&
"Jump Table Index operands are expected only for JX instructions");
- const MachineFunction *MF = MI.getMF();
- Printer.OutStreamer->emitLabel(BPFTargetLowering::getJXAnchorSymbol(MF, JTI));
- MCSymbol *JT = Printer.GetJTISymbol(JTI);
+ Printer.OutStreamer->emitLabel(Printer.getJXAnchorSymbol(JTI));
+ MCSymbol *JT = Printer.getJTPublicSymbol(JTI);
const MCExpr *Zero = MCConstantExpr::create(0, Ctx);
Printer.OutStreamer->emitRelocDirective(*Zero, "FK_SecRel_8",
MCSymbolRefExpr::create(JT, Ctx), {},
diff --git a/llvm/lib/Target/BPF/BPFMCInstLower.h b/llvm/lib/Target/BPF/BPFMCInstLower.h
index 927e9dfaf8c68..46256bcfc83e9 100644
--- a/llvm/lib/Target/BPF/BPFMCInstLower.h
+++ b/llvm/lib/Target/BPF/BPFMCInstLower.h
@@ -12,7 +12,7 @@
#include "llvm/Support/Compiler.h"
namespace llvm {
-class AsmPrinter;
+class BPFAsmPrinter;
class MCContext;
class MCInst;
class MCOperand;
@@ -24,10 +24,10 @@ class MachineOperand;
class LLVM_LIBRARY_VISIBILITY BPFMCInstLower {
MCContext &Ctx;
- AsmPrinter &Printer;
+ BPFAsmPrinter &Printer;
public:
- BPFMCInstLower(MCContext &ctx, AsmPrinter &printer)
+ BPFMCInstLower(MCContext &ctx, BPFAsmPrinter &printer)
: Ctx(ctx), Printer(printer) {}
void Lower(const MachineInstr *MI, MCInst &OutMI) const;
More information about the llvm-commits
mailing list