[Lldb-commits] [lldb] 13b3943 - [lldb][bytecode] Add Python to formatter bytecode compiler (#113734)
via lldb-commits
lldb-commits at lists.llvm.org
Mon Mar 9 13:48:28 PDT 2026
Author: Dave Lee
Date: 2026-03-09T13:48:24-07:00
New Revision: 13b394312b97b4c0ea72a8d2a40c68d3fa392b3e
URL: https://github.com/llvm/llvm-project/commit/13b394312b97b4c0ea72a8d2a40c68d3fa392b3e
DIFF: https://github.com/llvm/llvm-project/commit/13b394312b97b4c0ea72a8d2a40c68d3fa392b3e.diff
LOG: [lldb][bytecode] Add Python to formatter bytecode compiler (#113734)
A compiler from Python to the assembly syntax of the [lldb data
formatter
bytecode](https://discourse.llvm.org/t/a-bytecode-for-lldb-data-formatters/82696).
Assisted-by: claude
Added:
lldb/test/Shell/ScriptInterpreter/Python/Inputs/FormatterBytecode/RigidArrayLLDBFormatter.txt
lldb/test/Shell/ScriptInterpreter/Python/python-bytecode.test
Modified:
lldb/examples/python/formatter_bytecode.py
Removed:
################################################################################
diff --git a/lldb/examples/python/formatter_bytecode.py b/lldb/examples/python/formatter_bytecode.py
index 471cb2c5b38ff..dfa6fda0f6402 100644
--- a/lldb/examples/python/formatter_bytecode.py
+++ b/lldb/examples/python/formatter_bytecode.py
@@ -17,9 +17,12 @@
import re
import io
+import ast
+import enum
import textwrap
+from copy import copy
from dataclasses import dataclass
-from typing import BinaryIO, TextIO, Tuple, Union
+from typing import Any, BinaryIO, Optional, Sequence, TextIO, Tuple, Union, cast
BINARY_VERSION = 1
@@ -173,22 +176,17 @@ def _segment_by_signature(input: list[str]) -> list[Tuple[str, list[str]]]:
signature = None
tokens = []
- def conclude_segment():
- if not tokens:
- raise ValueError(f"empty signature: {signature}")
- segments.append((signature, tokens))
-
for token in input:
if _SIGNATURE_LABEL.match(token):
if signature:
- conclude_segment()
+ segments.append((signature, tokens))
signature = token[1:-1] # strip leading @, trailing :
tokens = []
else:
tokens.append(token)
if signature:
- conclude_segment()
+ segments.append((signature, tokens))
return segments
@@ -295,7 +293,8 @@ def assemble_file(type_name: str, input: TextIO) -> BytecodeSection:
input_tokens = _tokenize(input.read())
signatures = []
for sig, tokens in _segment_by_signature(input_tokens):
- signatures.append((sig, assemble_tokens(tokens)))
+ if tokens:
+ signatures.append((sig, assemble_tokens(tokens)))
return BytecodeSection(type_name, flags=0, signatures=signatures)
@@ -663,6 +662,375 @@ def next_byte():
return data[-1]
+################################################################################
+# Python -> Bytecode Compiler
+################################################################################
+
+_BUILTINS = {
+ "Cast": "@cast",
+ "GetChildAtIndex": "@get_child_at_index",
+ "GetChildMemberWithName": "@get_child_with_name",
+ "GetSummary": "@summary",
+ "GetSyntheticValue": "@get_synthetic_value",
+ "GetTemplateArgumentType": "@get_template_argument_type",
+ "GetType": "@get_type",
+ "GetValueAsUnsigned": "@get_value_as_unsigned",
+}
+
+_COMPS = {
+ ast.Eq: "=",
+ ast.NotEq: "!=",
+ ast.Lt: "<",
+ ast.LtE: "=<",
+ ast.Gt: ">",
+ ast.GtE: "=>",
+}
+
+# Maps Python method names in a formatter class to their bytecode signatures.
+_METHOD_SIGS = {
+ "__init__": "@init",
+ "update": "@update",
+ "num_children": "@get_num_children",
+ "get_child_index": "@get_child_index",
+ "get_child_at_index": "@get_child_at_index",
+ "get_value": "@get_value",
+}
+
+
+class CompilerError(Exception):
+ lineno: int
+
+ def __init__(self, message, node: Union[ast.expr, ast.stmt]) -> None:
+ super().__init__(message)
+ self.lineno = node.lineno
+
+
+class Compiler(ast.NodeVisitor):
+ """
+ Compile Python LLDB data formatters to LLDB formatter bytecode.
+
+ This compiler is supports a limited subset of Python.
+
+ # Supported Features
+
+ * Top level functions implementing LLDB summary formatters
+ * Top level classes implementing LLDB synthetic formatters
+ * Partial support for the following, see below for more details:
+ - Object attributes (properties)
+ - Local variables
+ - Function calls
+ * Python language support
+ [x] If statements (including else, elif and nested if)
+ [x] Return statements
+ [x] String, integer, float, boolean, and None literals
+ [x] Binary comparisons
+ [ ] Boolean operators
+ [ ] Math operations
+
+ # Unsupported Features
+
+ Note: that this is not exhaustive, refer to the list of supported
+ features above.
+
+ * For and while loops
+ * Exceptions
+ * User defined general purpose functions and classes
+ * Lists, dicts, sets, and other container data types
+ * Iterators, comprehensions, yield, etc
+ * With statements
+ * Imports of any modules
+
+ # Variables
+
+ The compiler supports two kinds of variables, local variables and attribute
+ variables (properties), but there are limitations on both.
+
+ In __init__ and update, local variables are currently *not* supported, but
+ attributes can be assigned to. This matches the common case for these
+ functions.
+
+ In all other function bodies, local variables _are_ supported, but
+ attributes can only be read from, *not* assigned to. This also matches the
+ common case for these functions.
+
+ Variables (local and attributes) are tracked, allowing the compiler to know
+ their position in the stack. Variable reads can then be lowered to `pick`
+ instructions. See the compiler's `locals` and `attrs` attributes.
+
+ # Functions
+
+ Known functions are supported, a design that customizes the scope of what
+ formatters can and can't do. The functions known to the compiler are called
+ "selectors". The selectors are primarily SBValue API, although there are
+ also general purpose selectors. Formatters can only call selectors, not user
+ defined functions, and not SB methods that have not been defined as a
+ selector.
+ """
+
+ # Names of locals in bottom-to-top stack order. locals[0] is the
+ # oldest/deepest; locals[-1] is the most recently pushed.
+ locals: list[str]
+
+ # Names of visible attrs in bottom-to-top stack order. Always holds the
+ # full combined frame for the method being compiled: grows incrementally
+ # during __init__/update, and is set to the combined list before getter
+ # methods are compiled.
+ attrs: list[str]
+
+ # Temporaries currently on the stack above the locals/attrs frame.
+ # Always 0 at statement boundaries.
+ num_temps: int
+
+ # Bytecode signature of the method being compiled, or None for top-level
+ # functions.
+ current_sig: Optional[str]
+
+ buffer: io.StringIO
+
+ def __init__(self) -> None:
+ self.locals = []
+ self.attrs = []
+ self.num_temps = 0
+ self.current_sig = None
+ self.buffer = io.StringIO()
+
+ def compile(self, source_file: str) -> str:
+ with open(source_file) as f:
+ root = ast.parse(f.read())
+ self.visit(root)
+ return self.buffer.getvalue()
+
+ def visit_ClassDef(self, node: ast.ClassDef) -> None:
+ # Compile methods in a fixed order so that attrs is fully populated
+ # before getter methods are compiled.
+ methods = {}
+ for item in node.body:
+ if isinstance(item, ast.FunctionDef):
+ if item.name not in _METHOD_SIGS:
+ raise CompilerError(f"unsupported method: {item.name}", item)
+ methods[item.name] = item
+
+ self.attrs = []
+ if method := methods.get("__init__"):
+ self._compile_method(method)
+ # self.attrs now holds init's attrs. update's attrs are appended above
+ # them, so after update self.attrs is the combined init+update list.
+ if method := methods.get("update"):
+ self._compile_method(method)
+
+ for method_name, method in methods.items():
+ if method_name not in ("__init__", "update"):
+ self._compile_method(method)
+
+ def _compile_method(self, node: ast.FunctionDef) -> None:
+ self.current_sig = _METHOD_SIGS[node.name]
+ self.num_temps = 0
+
+ # Strip 'self' (and 'internal_dict' for __init__) from the arg list;
+ # the remaining args become the initial locals.
+ args = copy(node.args.args)
+ args.pop(0) # drop 'self'
+ if node.name == "__init__":
+ args.pop() # drop trailing 'internal_dict'
+
+ self.locals = [arg.arg for arg in args]
+
+ # Compile into a temporary buffer so the signature line can be
+ # emitted first.
+ saved_buffer = self.buffer
+ self.buffer = io.StringIO()
+
+ self._visit_each(node.body)
+
+ method_output = self.buffer.getvalue()
+ self.buffer = saved_buffer
+ self._output(f"{self.current_sig}:")
+ self._output(method_output)
+
+ self.locals.clear()
+ self.current_sig = None
+
+ def visit_FunctionDef(self, node: ast.FunctionDef) -> None:
+ # Top-level function (not inside a class).
+ self.current_sig = None
+ self.attrs = []
+ self.locals = [arg.arg for arg in node.args.args]
+ self._visit_each(node.body)
+ self.locals.clear()
+
+ def visit_Compare(self, node: ast.Compare) -> None:
+ self.visit(node.left)
+ # XXX: Does not handle multiple comparisons, ex: `0 < x < 10`
+ self.visit(node.comparators[0])
+ self._output(_COMPS[type(node.ops[0])])
+ # The comparison consumes two values and produces one.
+ self.num_temps -= 1
+
+ def visit_If(self, node: ast.If) -> None:
+ self.visit(node.test)
+ # `if`/`ifelse` consumes the condition.
+ self.num_temps = 0
+
+ self._output("{")
+ self._visit_each(node.body)
+ if node.orelse:
+ self.num_temps = 0
+ self._output("} {")
+ self._visit_each(node.orelse)
+ self._output("} ifelse")
+ else:
+ self._output("} if")
+
+ def visit_Return(self, node: ast.Return) -> None:
+ self.num_temps = 0
+ if node.value:
+ self.visit(node.value)
+ self._output("return")
+
+ def visit_Constant(self, node: ast.Constant) -> None:
+ if isinstance(node.value, str):
+ self._output(f'"{node.value}"')
+ elif isinstance(node.value, bool):
+ self._output(int(node.value))
+ else:
+ self._output(node.value)
+ self.num_temps += 1
+
+ def visit_Call(self, node: ast.Call) -> None:
+ func = node.func
+ if isinstance(func, ast.Attribute):
+ receiver = func.value
+ method = func.attr
+ # self is not a valid call receiver.
+ if isinstance(receiver, ast.Name) and receiver.id == "self":
+ raise CompilerError(
+ "self is not a valid call receiver; use self.attr to read an attribute",
+ node,
+ )
+ if selector := _BUILTINS.get(method):
+ self.visit(receiver)
+ self._visit_each(node.args)
+ self._output(f"{selector} call")
+ # `call` pops the receiver and all args, and pushes one result.
+ self.num_temps -= len(node.args)
+ return
+ raise CompilerError(f"unsupported method: {method}", node)
+
+ if isinstance(func, ast.Name):
+ raise CompilerError(f"unsupported function: {func.id}", node)
+
+ raise CompilerError("unsupported function call expression", node)
+
+ def visit_Assign(self, node: ast.Assign) -> None:
+ self.num_temps = 0
+
+ target = node.targets[0]
+
+ # Handle self.attr = expr (attribute assignment).
+ if (
+ isinstance(target, ast.Attribute)
+ and isinstance(target.value, ast.Name)
+ and target.value.id == "self"
+ ):
+ if self.current_sig not in ("@init", "@update"):
+ raise CompilerError(
+ "attribute assignment is only allowed in __init__ and update",
+ node,
+ )
+
+ attr = target.attr
+ if attr in self.attrs:
+ raise CompilerError(f"attribute '{attr}' is already assigned", node)
+
+ # If the RHS is an argument (the only kind of local permitted in
+ # __init__) - then it is already on the stack in place, and no
+ # evaluation is needed.
+ is_arg = (
+ isinstance(node.value, ast.Name)
+ and self._local_index(node.value) is not None
+ )
+ if not is_arg:
+ # Evaluate the RHS, leaving its value on the stack.
+ self.visit(node.value)
+
+ # Record the attr.
+ self.attrs.append(attr)
+ return
+
+ # Handle local variable assignment.
+ if self.current_sig in ("@init", "@update"):
+ raise CompilerError(
+ "local variable assignment is not allowed in __init__ or update; "
+ "use attribute assignment (self.attr = ...) instead",
+ node,
+ )
+
+ if isinstance(target, ast.Name):
+ names = [target]
+ elif isinstance(target, ast.Tuple):
+ names = cast(list[ast.Name], target.elts)
+ else:
+ raise CompilerError("unsupported assignment target", node)
+
+ # Visit RHS, leaving its value on the stack.
+ self.visit(node.value)
+
+ # Forget any previous bindings of these names.
+ # Their values are orphaned on the stack.
+ for name in names:
+ idx = self._local_index(name)
+ if idx is not None:
+ self.locals[idx] = ""
+
+ self.locals.extend(x.id for x in names)
+
+ def visit_Attribute(self, node: ast.Attribute) -> None:
+ # Only self.attr reads are supported here.
+ if not (isinstance(node.value, ast.Name) and node.value.id == "self"):
+ raise CompilerError(
+ "unsupported attribute access (only self.attr is supported)", node
+ )
+ attr_idx = self._attr_index(node.attr, node)
+ pick_idx = self.num_temps + attr_idx
+ self._output(f"{pick_idx} pick") # "# self.{node.attr}"
+ self.num_temps += 1
+
+ def visit_Name(self, node: ast.Name) -> None:
+ idx = self._stack_index(node)
+ if idx is None:
+ raise CompilerError(f"unknown local variable: {node.id}", node)
+ self._output(f"{idx} pick") # "# {node.id}"
+ self.num_temps += 1
+
+ def _visit_each(self, nodes: Sequence[ast.AST]) -> None:
+ for child in nodes:
+ self.visit(child)
+
+ def _attr_index(self, name: str, node: ast.expr) -> int:
+ # self.attrs is always the full visible attr frame, so the index is
+ # the direct pick offset with no further adjustment.
+ try:
+ return self.attrs.index(name)
+ except ValueError:
+ raise CompilerError(f"unknown attribute: {name}", node)
+
+ def _stack_index(self, name: ast.Name) -> Optional[int]:
+ # Offset past all attrs and any in-flight temporaries.
+ idx = self._local_index(name)
+ if idx is None:
+ return None
+ return len(self.attrs) + idx + self.num_temps
+
+ def _local_index(self, name: ast.Name) -> Optional[int]:
+ try:
+ return self.locals.index(name.id)
+ except ValueError:
+ return None
+
+ def _output(self, x: Any) -> None:
+ print(x, file=self.buffer)
+
+
################################################################################
# Helper functions.
################################################################################
@@ -717,6 +1085,12 @@ def _main():
mode = parser.add_mutually_exclusive_group()
mode.add_argument(
"-c",
+ "--compile",
+ action="store_true",
+ help="compile a Python LLDB data formatter into LLDB formatter bytecode",
+ )
+ mode.add_argument(
+ "-a",
"--assemble",
action="store_true",
help="assemble assembly into bytecode",
@@ -727,6 +1101,7 @@ def _main():
action="store_true",
help="disassemble bytecode",
)
+ parser.add_argument("-n", "--type-name", help="source type of formatter")
parser.add_argument(
"-o",
"--output",
@@ -742,7 +1117,28 @@ def _main():
parser.add_argument("-t", "--test", action="store_true", help="run unit tests")
args = parser.parse_args()
- if args.assemble:
+ if args.compile:
+ if not args.type_name:
+ parser.error("--type-name is required with --compile")
+ if not args.output:
+ parser.error("--output is required with --compile")
+ compiler = Compiler()
+ try:
+ assembly = compiler.compile(args.input)
+ except CompilerError as e:
+ print(f"{args.input}:{e.lineno}: {e}", file=sys.stderr)
+ return
+
+ section = assemble_file(args.type_name, io.StringIO(assembly))
+ if args.format == "binary":
+ with open(args.output, "wb") as output:
+ section.write_binary(output)
+ else: # args.format == "c"
+ with open(args.output, "w") as output:
+ section.write_source(output)
+ elif args.assemble:
+ if not args.type_name:
+ parser.error("--type-name is required with --assemble")
if not args.output:
parser.error("--output is required with --assemble")
with open(args.input) as input:
diff --git a/lldb/test/Shell/ScriptInterpreter/Python/Inputs/FormatterBytecode/RigidArrayLLDBFormatter.txt b/lldb/test/Shell/ScriptInterpreter/Python/Inputs/FormatterBytecode/RigidArrayLLDBFormatter.txt
new file mode 100644
index 0000000000000..a8daf9e20b85c
--- /dev/null
+++ b/lldb/test/Shell/ScriptInterpreter/Python/Inputs/FormatterBytecode/RigidArrayLLDBFormatter.txt
@@ -0,0 +1,38 @@
+
+#ifdef __APPLE__
+#define FORMATTER_SECTION "__DATA_CONST,__lldbformatters"
+#else
+#define FORMATTER_SECTION ".lldbformatters"
+#endif
+
+__attribute__((used, section(FORMATTER_SECTION)))
+unsigned char _RigidArray_synthetic[] =
+ // version
+ "\x01"
+ // remaining record size
+ "\x47"
+ // type name size
+ "\x0a"
+ // type name
+ "RigidArray"
+ // flags
+ "\x00"
+ // sig_update
+ "\x05"
+ // program size
+ "\x27"
+ // program
+ "\x21\x00\x03\x22\x08\x5f\x73\x74\x6f\x72\x61\x67\x65\x23\x12\x60\x23\x18\x60\x21\x00\x03\x22\x06\x5f\x63\x6f\x75\x6e\x74\x23\x12\x60\x23\x18\x60\x23\x21\x60"
+ // sig_get_num_children
+ "\x02"
+ // program size
+ "\x04"
+ // program
+ "\x21\x02\x03\x13"
+ // sig_get_child_at_index
+ "\x04"
+ // program size
+ "\x0a"
+ // program
+ "\x21\x01\x03\x21\x04\x03\x23\x11\x60\x13"
+;
diff --git a/lldb/test/Shell/ScriptInterpreter/Python/python-bytecode.test b/lldb/test/Shell/ScriptInterpreter/Python/python-bytecode.test
new file mode 100644
index 0000000000000..b4daa89067897
--- /dev/null
+++ b/lldb/test/Shell/ScriptInterpreter/Python/python-bytecode.test
@@ -0,0 +1,29 @@
+# RUN: mkdir -p %t
+# RUN: %python %S/../../../../examples/python/formatter_bytecode.py --compile %s --format c --type-name RigidArray --output %t/output.txt
+# RUN:
diff -u %S/Inputs/FormatterBytecode/RigidArrayLLDBFormatter.txt %t/output.txt
+import lldb
+
+
+class RigidArraySynthetic:
+ valobj: lldb.SBValue
+ storage: lldb.SBValue
+ count: int
+
+ def __init__(self, valobj: lldb.SBValue, _) -> None:
+ self.valobj = valobj
+
+ def num_children(self) -> int:
+ return self.count
+
+ def get_child_at_index(self, idx: int) -> lldb.SBValue:
+ return self.storage.GetChildAtIndex(idx)
+
+ def update(self) -> None:
+ self.storage = self.valobj.GetChildMemberWithName(
+ "_storage"
+ ).GetSyntheticValue()
+ self.count = (
+ self.valobj.GetChildMemberWithName("_count")
+ .GetSyntheticValue()
+ .GetValueAsUnsigned()
+ )
More information about the lldb-commits
mailing list