[llvm] [Dexter] Add basic structured script parsing (PR #193710)
Stephen Tozer via llvm-commits
llvm-commits at lists.llvm.org
Thu Apr 23 04:16:23 PDT 2026
https://github.com/SLTozer updated https://github.com/llvm/llvm-project/pull/193710
>From 5a89e2261b88fcb681a530eb54fbee6e68430947 Mon Sep 17 00:00:00 2001
From: Stephen Tozer <stephen.tozer at sony.com>
Date: Wed, 22 Apr 2026 14:28:05 +0100
Subject: [PATCH 1/2] [Dexter] Add basic structured script parsing
See PSA: https://discourse.llvm.org/t/psa-planned-changes-to-dexter/90402
This patch begins adding support for "structured scripts" to Dexter,
starting with some of the core classes and the ability to parse script
files. This patch does not add the ability to actually run scripts, or any
of the underlying behaviours required to do so.
NB: This patch adds a dependency on PyYAML, which is specified in a new
requirements.txt file.
---
.../debuginfo-tests/dexter/README.md | 7 +-
.../dexter/dex/dextIR/DextIR.py | 2 +
.../dexter/dex/test_script/Nodes.py | 202 +++++++++++++++
.../dexter/dex/test_script/Script.py | 238 ++++++++++++++++++
.../dexter/dex/test_script/__init__.py | 0
.../dexter/dex/tools/TestToolBase.py | 6 +
.../dexter/dex/tools/test/Tool.py | 23 ++
.../scripts/parser/bad-where-attr.test | 13 +
.../scripts/parser/error-locations.test | 23 ++
.../scripts/parser/invalid-script-nodes.test | 24 ++
.../scripts/parser/valid-parse.test | 20 ++
.../debuginfo-tests/dexter/requirements.txt | 1 +
12 files changed, 556 insertions(+), 3 deletions(-)
create mode 100644 cross-project-tests/debuginfo-tests/dexter/dex/test_script/Nodes.py
create mode 100644 cross-project-tests/debuginfo-tests/dexter/dex/test_script/Script.py
create mode 100644 cross-project-tests/debuginfo-tests/dexter/dex/test_script/__init__.py
create mode 100644 cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/parser/bad-where-attr.test
create mode 100644 cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/parser/error-locations.test
create mode 100644 cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/parser/invalid-script-nodes.test
create mode 100644 cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/parser/valid-parse.test
create mode 100644 cross-project-tests/debuginfo-tests/dexter/requirements.txt
diff --git a/cross-project-tests/debuginfo-tests/dexter/README.md b/cross-project-tests/debuginfo-tests/dexter/README.md
index 44c43435b20d5..7d9ec0ab41a1c 100644
--- a/cross-project-tests/debuginfo-tests/dexter/README.md
+++ b/cross-project-tests/debuginfo-tests/dexter/README.md
@@ -13,11 +13,12 @@ The following command evaluates your environment, listing the available and comp
dexter.py list-debuggers
## Dependencies
-[TODO] Add a requirements.txt or an install.py and document it here.
-### Python 3.6
+See: requirements.txt
-DExTer requires python version 3.6 or greater.
+### Python 3.10
+
+DExTer requires python version 3.10 or greater.
### pywin32 python package
diff --git a/cross-project-tests/debuginfo-tests/dexter/dex/dextIR/DextIR.py b/cross-project-tests/debuginfo-tests/dexter/dex/dextIR/DextIR.py
index 42500c4b9681d..9b00a45f2eb10 100644
--- a/cross-project-tests/debuginfo-tests/dexter/dex/dextIR/DextIR.py
+++ b/cross-project-tests/debuginfo-tests/dexter/dex/dextIR/DextIR.py
@@ -10,6 +10,7 @@
from dex.dextIR.DebuggerIR import DebuggerIR
from dex.dextIR.StepIR import StepIR, StepKind
+from dex.test_script.Script import DexterScript
def _step_kind_func(context, step):
@@ -55,6 +56,7 @@ def __init__(
self.debugger = debugger
self.commands = commands
self.steps: List[StepIR] = []
+ self.script: DexterScript | None = None
def __str__(self):
colors = "rgby"
diff --git a/cross-project-tests/debuginfo-tests/dexter/dex/test_script/Nodes.py b/cross-project-tests/debuginfo-tests/dexter/dex/test_script/Nodes.py
new file mode 100644
index 0000000000000..37e0a1e4e868d
--- /dev/null
+++ b/cross-project-tests/debuginfo-tests/dexter/dex/test_script/Nodes.py
@@ -0,0 +1,202 @@
+# DExTer : Debugging Experience Tester
+# ~~~~~~ ~ ~~ ~ ~~
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+"""This file defines all of the Nodes used to create Dexter scripts. All Nodes must be registered with the yaml
+constructor/representer in `setup_yaml_parser` before loading or printing any script.
+"""
+
+import abc
+from dataclasses import dataclass
+from typing import Any
+import yaml
+from dex.dextIR.ValueIR import ValueIR
+from dex.utils.Exceptions import Error
+
+
+def setup_yaml_parser(loader):
+ reg_classes = [
+ Where,
+ Value,
+ DexRange,
+ ]
+ for c in reg_classes:
+ c.register_yaml(loader)
+
+
+class DexterNodeError(Error):
+ """Class representing errors with Dexter node parsing."""
+
+ def __init__(self, node, msg):
+ super(DexterNodeError, self).__init__(msg)
+ self.msg = msg
+ self.node = node
+
+ def __str__(self):
+ return f"Error with node: {self.node}: {self.msg}"
+
+
+###################
+## Structural Nodes: These are used as keys in the Script, and collectively define Dexter's actions when running a test:
+## how it steps and navigates through the debuggee program, and what information it collects from the
+## debugger.
+
+
+class Where:
+ """ "One or more instances of this class define a range of steps in a debugging session. Any expects in the script
+ within scope of a "Where" will only be evaluated for the steps where the Where applies.
+ """
+
+ def __init__(self, attributes: dict):
+ self.file: str | None = attributes.pop("file", None)
+ self.function: list[str] | str | None = attributes.pop("function", None)
+ self.lines: int | DexRange | None = attributes.pop("lines", None)
+ self.after_hit_count: int | None = attributes.pop("after_hit_count", None)
+ self.for_hit_count: int | None = attributes.pop("for_hit_count", None)
+ self.conditions: dict = attributes.pop("conditions", None)
+ if attributes:
+ raise DexterNodeError(
+ self, f"unexpected attributes {', '.join(attributes)}"
+ )
+ if (
+ not self.function
+ and not self.lines
+ and (self.for_hit_count or self.after_hit_count)
+ ):
+ raise DexterNodeError(
+ self, "can't check hit counts without an explicit lines or function arg"
+ )
+
+ def __repr__(self):
+ elts = [
+ f"{name}={value}"
+ for name, value in self.get_attrs().items()
+ if value is not None
+ ]
+ return f"Where(" + ", ".join(elts) + ")"
+
+ def get_attrs(self) -> dict[str, Any]:
+ return {
+ "file": self.file,
+ "function": self.function,
+ "lines": self.lines,
+ "for_hit_count": self.for_hit_count,
+ "after_hit_count": self.after_hit_count,
+ "conditions": self.conditions,
+ }
+
+ @staticmethod
+ def constructor(loader: yaml.Loader, node):
+ return Where(loader.construct_mapping(node))
+
+ @staticmethod
+ def representer(dumper: yaml.Dumper, data: "Where"):
+ mapping = {
+ name: value for name, value in data.get_attrs().items() if value is not None
+ }
+ return dumper.represent_mapping("!where", mapping, flow_style=True)
+
+ @staticmethod
+ def register_yaml(loader):
+ yaml.add_constructor("!where", Where.constructor, loader)
+ yaml.add_representer(Where, Where.representer)
+
+ def get_lines(self) -> range:
+ """Returns the range of line numbers that this Where references, returning an empty range if this Where does not
+ refer to any lines."""
+ if not self.lines:
+ return range(-1)
+ if isinstance(self.lines, int):
+ return range(self.lines, self.lines + 1)
+ assert isinstance(self.lines, DexRange)
+ return self.lines.to_range()
+
+
+###################
+## Expect Nodes: These nodes define the expected outputs from the debugger - they are the only nodes that produce
+## metrics, and map to an expected value in the script.
+
+
+class Expect:
+ """An expectation of some debugger state that will be compared to actual observed debugger state and generate one
+ or more metrics as a measurement of the difference.
+ Expects are largely evaluated independently, but may influence each other through the evaluation context.
+ """
+
+ @staticmethod
+ def get_variable_result(value: ValueIR) -> str | None:
+ """For Expects that extract actual results from ValueIR, this method returns that result from the given value,
+ excluding any subvalues (i.e. struct members), or None if there is no valid result for this ValueIR.
+ """
+
+ @abc.abstractmethod
+ def get_watched_expr(self) -> str:
+ """Returns the list of expressions that this Expect wants to evaluate."""
+
+
+class Value(Expect):
+ def __init__(self, variable_name: str):
+ self.variable_name = variable_name
+ self.actual_values = None
+
+ @staticmethod
+ def get_variable_result(value: ValueIR) -> str | None:
+ if value.could_evaluate and not (
+ value.is_irretrievable or value.is_optimized_away
+ ):
+ return value.value
+ return None
+
+ def get_watched_expr(self) -> str:
+ return self.variable_name
+
+ def __repr__(self):
+ return f"Value({self.variable_name})"
+
+ @staticmethod
+ def constructor(loader: yaml.Loader, node):
+ return Value(loader.construct_scalar(node))
+
+ @staticmethod
+ def representer(dumper, data):
+ return dumper.represent_scalar("!value", data.variable_name)
+
+ @staticmethod
+ def register_yaml(loader):
+ yaml.add_constructor("!value", Value.constructor, loader)
+ yaml.add_representer(Value, Value.representer)
+
+
+##############
+## Utility Nodes: Can be used anywhere in a script as a form of syntactic sugar.
+
+
+ at dataclass(frozen=True)
+class DexRange:
+ start: int
+ stop: int
+
+ def __repr__(self) -> str:
+ return f"[{self.start} - {self.stop}]"
+
+ # We use an inclusive range in Dexter scripts, while python ranges are exclusive.
+ def to_range(self) -> range:
+ return range(self.start, self.stop + 1)
+
+ @staticmethod
+ def constructor(loader: yaml.Loader, node):
+ range_seq = loader.construct_sequence(node)
+ if len(range_seq) != 2 or not all(isinstance(elt, int) for elt in range_seq):
+ raise DexterNodeError(node, "range must have exactly 2 int elements")
+ return DexRange(range_seq[0], range_seq[1])
+
+ @staticmethod
+ def representer(dumper, data: "DexRange"):
+ return dumper.represent_sequence("!range", [data.start, data.stop])
+
+ @staticmethod
+ def register_yaml(loader):
+ yaml.add_constructor("!range", DexRange.constructor, loader)
+ yaml.add_representer(DexRange, DexRange.representer)
diff --git a/cross-project-tests/debuginfo-tests/dexter/dex/test_script/Script.py b/cross-project-tests/debuginfo-tests/dexter/dex/test_script/Script.py
new file mode 100644
index 0000000000000..88f24361d3b59
--- /dev/null
+++ b/cross-project-tests/debuginfo-tests/dexter/dex/test_script/Script.py
@@ -0,0 +1,238 @@
+# DExTer : Debugging Experience Tester
+# ~~~~~~ ~ ~~ ~ ~~
+#
+# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+"""This file defines the DexterScript class. Using Nodes as building blocks, the DexterScript defines a complete Dexter
+test, a structured definition of locations, values, and actions used to drive a debugging session and evaluate the
+results.
+"""
+
+from pathlib import PurePath
+import os
+from typing import Any, Callable
+import yaml
+
+from dex.test_script.Nodes import (
+ Expect,
+ Where,
+ setup_yaml_parser,
+)
+
+from dex.utils.Exceptions import Error
+from dex.utils.Timer import Timer
+
+
+class DexterScriptError(Error):
+ pass
+
+
+class Scope:
+ """Helper class used to simplify queries about the context of a Node in the Dexter Script. The context for a given
+ Node consists of some base context information in the root of the script, and then all Where nodes in the parent
+ chain of the current Node. Therefore each Script has a root Scope object, and each Node's context is given by a
+ Scope chain built from the root Scope and every Where between the root and the given Node.
+ """
+
+ def __init__(
+ self,
+ file: str | None = None,
+ where: Where | None = None,
+ parent_scope: "Scope | None" = None,
+ ):
+ """Can be initialized with either a file for the default Scope, or with the properties of a Where
+ for any script-nested Scope.
+ """
+ if where is not None:
+ assert (
+ parent_scope is not None
+ ), "Scope for a Where node must have a parent scope!"
+ assert (
+ file is None
+ ), "Scope for a Where node cannot have a separately-defined file!"
+ self.file = None
+ self.where = where
+ self.parent_scope = parent_scope
+ else:
+ assert (
+ parent_scope is None
+ ), "Scope for a Root node cannot have a parent scope!"
+ self.file = file
+ self.where = None
+ self.parent_scope = None
+
+ def add_where(self, where: Where):
+ """Adds `where` to this Scope's chain."""
+ return Scope(where=where, parent_scope=self)
+
+
+class DexterScript:
+ def __init__(
+ self,
+ context,
+ script_obj,
+ scope: Scope,
+ ):
+ self.context = context
+ self.script_obj = script_obj
+ self.root_scope = scope
+ # `visit_script` will validate the structure of the script, as it traverses the full script and raises an
+ # exception if it sees anything unexpected.
+ self.visit_script()
+
+ # If a truthy value is returned, abort further visiting and return that value..
+ def _visit_script(
+ self, script, scope: Scope, visit_where=None, visit_expect=None, visit_then=None
+ ) -> Any:
+ def do(visitor, *args):
+ if visitor:
+ return visitor(*args)
+ return None
+
+ if not isinstance(script, dict):
+ raise DexterScriptError(f"Found unexpected node: {script}")
+ for key, value in script.items():
+ if isinstance(key, Where):
+ if result := do(visit_where, key, scope):
+ return result
+ new_scope = scope.add_where(key)
+ if result := self._visit_script(
+ value, new_scope, visit_where, visit_expect, visit_then
+ ):
+ return result
+ elif isinstance(key, Expect):
+ if result := do(visit_expect, key, value, scope):
+ return result
+ else:
+ raise DexterScriptError(f"Found unexpected node: {key}")
+
+ # Any visitor function provided may return a truthy value to abort the visit and return that value.
+ def visit_script(
+ self,
+ visit_where: Callable[[Where, Scope], Any] | None = None,
+ visit_expect: Callable[[Expect, Any, Scope], Any] | None = None,
+ ) -> Any:
+ """Visits all nodes in the script in pre-order traversal, calling any non-none provided visitor functions for
+ each respective node type. Note that we do not visit expected values independently of their associated expect;
+ instead, visit_expect accepts the Expect node and it's expected value as an argument.
+
+ If any visit function returns a truth-y value, traversal will early-exit and this function returns that value;
+ otherwise, this function returns None."""
+ return self._visit_script(
+ self.script_obj, self.root_scope, visit_where, visit_expect
+ )
+
+ @property
+ def root_wheres(self) -> set[Where]:
+ return set(node for node in self.script_obj if isinstance(node, Where))
+
+ def dump(self) -> str:
+ return yaml.dump(self.script_obj)
+
+
+# Helper function to apply a line offset to the errors reported by YAML while loading, to account for the YAML documents
+# being embedded in part of a file.
+def try_load_yaml(yaml_doc, loader, line_offset=0):
+ """Helper function that loads a YAML document from within a file, where the document may start in the middle of the
+ file. In this case, the value of line_offset should be set to the start line of the YAML document, and this function
+ will fix-up any returned syntax errors to point to the correct line in the file."""
+ try:
+ return yaml.load(yaml_doc, loader)
+ except yaml.MarkedYAMLError as e:
+ # MarkedYAMLError is an error with a 'Mark' pointing to the location of the error; this helper function applies
+ # our line offset to the provided mark if it is present.
+ def adjust_mark_loc(mark: yaml.Mark | None) -> yaml.Mark | None:
+ if mark is None:
+ return None
+ return yaml.Mark(
+ mark.name,
+ mark.index,
+ mark.line + line_offset,
+ mark.column,
+ mark.buffer,
+ mark.pointer,
+ )
+
+ # Adjust the error marks and then propagate the adjusted error.
+ e.context_mark = adjust_mark_loc(e.context_mark)
+ e.problem_mark = adjust_mark_loc(e.problem_mark)
+ raise e
+
+
+def get_script(context, file, loader) -> DexterScript:
+ """Searches the given file for a valid Dexter script, and returns the first valid script that it finds or raises an
+ Error if none is found."""
+ if not os.path.exists(file):
+ raise Error(f"Provided script file '{file}' does not exist.")
+ with open(file, "r") as r:
+ lines = r.readlines()
+ if not lines:
+ raise Error(f"Provided script file '{file}' is empty.")
+
+ numbered_lines = [(idx + 1, line) for idx, line in enumerate(lines)]
+ root_scope = Scope(file=str(file))
+ start_line = None
+ attempted_scripts = []
+ start_line = next((idx for idx, line in numbered_lines if line == "---\n"), None)
+ if start_line is None:
+ # If we saw no '---', then assume the whole file is a document and try to parse it.
+ try:
+ return DexterScript(
+ context,
+ try_load_yaml("\n".join(lines), loader),
+ root_scope,
+ )
+ except (Error, yaml.YAMLError) as e:
+ raise Error(f"File '{file}' was not a valid Dexter script:\n{e}")
+ # If we have at least one valid document start, then check every document until we see one that is a valid Dexter
+ # test.
+ while start_line is not None:
+ stop_line = next(
+ (
+ idx
+ for idx, line in numbered_lines[start_line + 1 :]
+ if line.startswith("...")
+ ),
+ len(lines),
+ )
+ try:
+ return DexterScript(
+ context,
+ try_load_yaml(
+ "\n".join(lines[start_line:stop_line]), loader, start_line
+ ),
+ root_scope,
+ )
+ except (Error, yaml.YAMLError) as e:
+ attempted_scripts.append((start_line, e))
+ start_line = next(
+ (idx for idx, line in numbered_lines[stop_line + 1 :] if line == "---\n"),
+ None,
+ )
+ script_error_messages = "\n".join(
+ f"Script starting line {line}:\n{e}" for line, e in attempted_scripts
+ )
+ raise Error(
+ f"No valid Dexter script found in file '{file}'; candidates:\n{script_error_messages}"
+ )
+
+
+def get_dexter_script(context, test_file, source_root_dir):
+ setup_yaml_parser(yaml.CLoader)
+ with Timer("parsing script"):
+ script = get_script(context, test_file, yaml.CLoader)
+ assert script.root_scope.file == test_file
+ source_files = set()
+ source_dir = source_root_dir if source_root_dir else str(test_file)
+
+ def check_explicit_files(where: Where, _: Scope):
+ if not where.file:
+ return
+ declared_path = where.file
+ if not os.path.isabs(declared_path):
+ declared_path = os.path.join(source_dir, declared_path)
+ source_files.add(str(PurePath(declared_path)))
+
+ script.visit_script(visit_where=check_explicit_files)
+ return script, source_files
diff --git a/cross-project-tests/debuginfo-tests/dexter/dex/test_script/__init__.py b/cross-project-tests/debuginfo-tests/dexter/dex/test_script/__init__.py
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/cross-project-tests/debuginfo-tests/dexter/dex/tools/TestToolBase.py b/cross-project-tests/debuginfo-tests/dexter/dex/tools/TestToolBase.py
index ecfc8ebcb1507..c153193a0baca 100644
--- a/cross-project-tests/debuginfo-tests/dexter/dex/tools/TestToolBase.py
+++ b/cross-project-tests/debuginfo-tests/dexter/dex/tools/TestToolBase.py
@@ -65,6 +65,12 @@ def add_tool_arguments(self, parser, defaults):
default=None,
help="if passed, result names will include relative path from this directory",
)
+ parser.add_argument(
+ "--use-script",
+ action="store_true",
+ default=False,
+ help="if passed, Dexter will look for a structured YAML script instead of dexter commands",
+ )
def handle_options(self, defaults):
options = self.context.options
diff --git a/cross-project-tests/debuginfo-tests/dexter/dex/tools/test/Tool.py b/cross-project-tests/debuginfo-tests/dexter/dex/tools/test/Tool.py
index 693c05b97af7c..934cb37d92a74 100644
--- a/cross-project-tests/debuginfo-tests/dexter/dex/tools/test/Tool.py
+++ b/cross-project-tests/debuginfo-tests/dexter/dex/tools/test/Tool.py
@@ -19,6 +19,7 @@
from dex.debugger.DebuggerControllers.ConditionalController import ConditionalController
from dex.dextIR.DextIR import DextIR
from dex.heuristic import Heuristic
+from dex.test_script.Script import get_dexter_script
from dex.tools import TestToolBase
from dex.utils.Exceptions import DebuggerException
from dex.utils.Exceptions import BuildScriptException, HeuristicException
@@ -106,6 +107,11 @@ def add_tool_arguments(self, parser, defaults):
action="store_true",
help="calculate the average score of every test run",
)
+ parser.add_argument(
+ "--skip-run",
+ action="store_true",
+ help="if true, skip running the debugger and produce no output; used for testing purposes",
+ )
super(Tool, self).add_tool_arguments(parser, defaults)
def _init_debugger_controller(self):
@@ -115,6 +121,15 @@ def _init_debugger_controller(self):
dexter_version=self.context.version,
)
+ if self.context.options.use_script:
+ step_collection.script, new_source_files = get_dexter_script(
+ self.context,
+ self.context.options.test_files[0],
+ self.context.options.source_root_dir,
+ )
+ # Functionality not yet implemented.
+ return step_collection
+
step_collection.commands, new_source_files = get_command_infos(
self.context.options.test_files, self.context.options.source_root_dir
)
@@ -132,6 +147,10 @@ def _init_debugger_controller(self):
def _get_steps(self):
"""Generate a list of debugger steps from a test case."""
debugger_controller = self._init_debugger_controller()
+
+ if self.context.options.skip_run:
+ self.context.logger.warning("Skipping run...")
+ return debugger_controller
debugger_controller = run_debugger_subprocess(
debugger_controller, self.context.working_directory.path
)
@@ -227,6 +246,10 @@ def _run_test(self, test_name):
self.context.options.binary, self.context.options.executable
)
steps = self._get_steps()
+ if self.context.options.skip_run:
+ if steps.script is not None:
+ print(steps.script.dump())
+ return
self._record_steps(test_name, steps)
heuristic_score = Heuristic(self.context, steps)
self._record_score(test_name, heuristic_score)
diff --git a/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/parser/bad-where-attr.test b/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/parser/bad-where-attr.test
new file mode 100644
index 0000000000000..9d1d01413acec
--- /dev/null
+++ b/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/parser/bad-where-attr.test
@@ -0,0 +1,13 @@
+RUN: not %dexter_regression_test_run --binary %s --use-script -- %s 2>&1 | FileCheck %s
+
+This is a test that when the script document starts part-way through a file, any syntax errors are reported with the
+correct line number.
+
+CHECK: No valid Dexter script found in file
+CHECK-NEXT: Script starting line 10
+CHECK-NEXT: Error with node: Where(file=main.cpp): unexpected attributes bees
+
+---
+!where {file: "main.cpp", bees: nose}:
+ !value x: 5
+...
diff --git a/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/parser/error-locations.test b/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/parser/error-locations.test
new file mode 100644
index 0000000000000..941d5963ccb61
--- /dev/null
+++ b/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/parser/error-locations.test
@@ -0,0 +1,23 @@
+RUN: not %dexter_regression_test_run --binary %s --use-script -- %s 2>&1 | FileCheck %s
+
+This is a test that when the script document starts part-way through a file, any syntax errors are reported with the
+correct line number.
+NB: There is some weirdness with PyYAML where every comment line inside of a document causes 2 lines of offset for
+ subsequently-reported errors; this is out of Dexter's hands, so don't test for it here.
+
+CHECK: No valid Dexter script found in file
+
+CHECK-NEXT: Script starting line [[# @LINE + 3]]
+CHECK-NEXT: could not determine a constructor for the tag '!not_a_real_node'
+CHECK-NEXT: line [[# @LINE + 2]], column 1
+---
+!not_a_real_node {function: foo}:
+ !value x: 5
+...
+CHECK-NEXT: Script starting line [[# @LINE + 3]]
+CHECK-NEXT: could not determine a constructor for the tag '!also_not_a_real_node'
+CHECK-NEXT: line [[# @LINE + 2]], column 1
+---
+!also_not_a_real_node {function: foo}:
+ !value x: 5
+...
diff --git a/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/parser/invalid-script-nodes.test b/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/parser/invalid-script-nodes.test
new file mode 100644
index 0000000000000..6989f475a11e5
--- /dev/null
+++ b/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/parser/invalid-script-nodes.test
@@ -0,0 +1,24 @@
+RUN: not %dexter_regression_test_run --binary %s --use-script -- %s 2>&1 | FileCheck %s
+
+This is a test that Dexter validates that nodes appear in the correct place in scripts.
+
+
+CHECK: No valid Dexter script found in file
+CHECK-NEXT: Script starting line [[# @LINE + 1 ]]
+---
+# CHECK-NEXT: Found unexpected node: not a dict
+!where {function: foo}: not a dict
+...
+
+CHECK-NEXT: Script starting line [[# @LINE + 1 ]]
+---
+# CHECK-NEXT: Found unexpected node: Where(function=foo)
+!where {function: foo}
+...
+
+CHECK-NEXT: Script starting line [[# @LINE + 1 ]]
+---
+# CHECK-NEXT: Found unexpected node: [{Value(x): 5}]
+!where {function: foo}:
+- !value x: 5
+...
diff --git a/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/parser/valid-parse.test b/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/parser/valid-parse.test
new file mode 100644
index 0000000000000..a639fadf576fa
--- /dev/null
+++ b/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/parser/valid-parse.test
@@ -0,0 +1,20 @@
+RUN: %dexter_regression_test_run --binary %s --use-script --skip-run -- %s 2>&1 | FileCheck %s
+
+Provides a valid test case and checks that we can parse it successfully and print the script back out correctly.
+
+CHECK: ? !where {function: foo}
+CHECK-NEXT: : !value 'x': 5
+CHECK-NEXT: ? !where {file: lib.cpp, lines: !range [10, 20]}
+CHECK-NEXT: : !value 'y': 10
+CHECK-NEXT: ? !where {lines: 5}
+CHECK-NEXT: : !value 'z': bees
+
+
+---
+!where {function: foo}:
+ !value x: 5
+ !where {file: "lib.cpp", lines: !range [10, 20]}:
+ !value y: 10
+!where {lines: 5}:
+ !value z: bees
+...
diff --git a/cross-project-tests/debuginfo-tests/dexter/requirements.txt b/cross-project-tests/debuginfo-tests/dexter/requirements.txt
new file mode 100644
index 0000000000000..17eef56b9ca9e
--- /dev/null
+++ b/cross-project-tests/debuginfo-tests/dexter/requirements.txt
@@ -0,0 +1 @@
+PyYAML >= 6.0.0
>From acbcc150c53d231332c382f4d89eeea059777d2d Mon Sep 17 00:00:00 2001
From: Stephen Tozer <stephen.tozer at sony.com>
Date: Thu, 23 Apr 2026 12:16:07 +0100
Subject: [PATCH 2/2] Remove type annotations that require Python>=3.10
---
.../debuginfo-tests/dexter/README.md | 4 ++--
.../dexter/dex/dextIR/DextIR.py | 4 ++--
.../dexter/dex/test_script/Nodes.py | 18 +++++++++---------
.../dexter/dex/test_script/Script.py | 16 ++++++++--------
4 files changed, 21 insertions(+), 21 deletions(-)
diff --git a/cross-project-tests/debuginfo-tests/dexter/README.md b/cross-project-tests/debuginfo-tests/dexter/README.md
index 7d9ec0ab41a1c..99aa6fb601da7 100644
--- a/cross-project-tests/debuginfo-tests/dexter/README.md
+++ b/cross-project-tests/debuginfo-tests/dexter/README.md
@@ -16,9 +16,9 @@ The following command evaluates your environment, listing the available and comp
See: requirements.txt
-### Python 3.10
+### Python 3.8
-DExTer requires python version 3.10 or greater.
+DExTer requires python version 3.8 or greater.
### pywin32 python package
diff --git a/cross-project-tests/debuginfo-tests/dexter/dex/dextIR/DextIR.py b/cross-project-tests/debuginfo-tests/dexter/dex/dextIR/DextIR.py
index 9b00a45f2eb10..eb5ad1b97afc5 100644
--- a/cross-project-tests/debuginfo-tests/dexter/dex/dextIR/DextIR.py
+++ b/cross-project-tests/debuginfo-tests/dexter/dex/dextIR/DextIR.py
@@ -6,7 +6,7 @@
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
from collections import OrderedDict
import os
-from typing import List
+from typing import List, Union
from dex.dextIR.DebuggerIR import DebuggerIR
from dex.dextIR.StepIR import StepIR, StepKind
@@ -56,7 +56,7 @@ def __init__(
self.debugger = debugger
self.commands = commands
self.steps: List[StepIR] = []
- self.script: DexterScript | None = None
+ self.script: Union[DexterScript, None] = None
def __str__(self):
colors = "rgby"
diff --git a/cross-project-tests/debuginfo-tests/dexter/dex/test_script/Nodes.py b/cross-project-tests/debuginfo-tests/dexter/dex/test_script/Nodes.py
index 37e0a1e4e868d..7eb754e2b08b6 100644
--- a/cross-project-tests/debuginfo-tests/dexter/dex/test_script/Nodes.py
+++ b/cross-project-tests/debuginfo-tests/dexter/dex/test_script/Nodes.py
@@ -10,7 +10,7 @@
import abc
from dataclasses import dataclass
-from typing import Any
+from typing import Any, Dict, Optional, Union
import yaml
from dex.dextIR.ValueIR import ValueIR
from dex.utils.Exceptions import Error
@@ -50,11 +50,11 @@ class Where:
"""
def __init__(self, attributes: dict):
- self.file: str | None = attributes.pop("file", None)
- self.function: list[str] | str | None = attributes.pop("function", None)
- self.lines: int | DexRange | None = attributes.pop("lines", None)
- self.after_hit_count: int | None = attributes.pop("after_hit_count", None)
- self.for_hit_count: int | None = attributes.pop("for_hit_count", None)
+ self.file: Optional[str] = attributes.pop("file", None)
+ self.function: Union[list[str], str, None] = attributes.pop("function", None)
+ self.lines: Union[int, DexRange, None] = attributes.pop("lines", None)
+ self.after_hit_count: Optional[int] = attributes.pop("after_hit_count", None)
+ self.for_hit_count: Optional[int] = attributes.pop("for_hit_count", None)
self.conditions: dict = attributes.pop("conditions", None)
if attributes:
raise DexterNodeError(
@@ -77,7 +77,7 @@ def __repr__(self):
]
return f"Where(" + ", ".join(elts) + ")"
- def get_attrs(self) -> dict[str, Any]:
+ def get_attrs(self) -> Dict[str, Any]:
return {
"file": self.file,
"function": self.function,
@@ -126,7 +126,7 @@ class Expect:
"""
@staticmethod
- def get_variable_result(value: ValueIR) -> str | None:
+ def get_variable_result(value: ValueIR) -> Optional[str]:
"""For Expects that extract actual results from ValueIR, this method returns that result from the given value,
excluding any subvalues (i.e. struct members), or None if there is no valid result for this ValueIR.
"""
@@ -142,7 +142,7 @@ def __init__(self, variable_name: str):
self.actual_values = None
@staticmethod
- def get_variable_result(value: ValueIR) -> str | None:
+ def get_variable_result(value: ValueIR) -> Optional[str]:
if value.could_evaluate and not (
value.is_irretrievable or value.is_optimized_away
):
diff --git a/cross-project-tests/debuginfo-tests/dexter/dex/test_script/Script.py b/cross-project-tests/debuginfo-tests/dexter/dex/test_script/Script.py
index 88f24361d3b59..4e191c42618d0 100644
--- a/cross-project-tests/debuginfo-tests/dexter/dex/test_script/Script.py
+++ b/cross-project-tests/debuginfo-tests/dexter/dex/test_script/Script.py
@@ -11,7 +11,7 @@
from pathlib import PurePath
import os
-from typing import Any, Callable
+from typing import Any, Callable, Optional, Set, Union
import yaml
from dex.test_script.Nodes import (
@@ -37,9 +37,9 @@ class Scope:
def __init__(
self,
- file: str | None = None,
- where: Where | None = None,
- parent_scope: "Scope | None" = None,
+ file: Union[str, None] = None,
+ where: Union[Where, None] = None,
+ parent_scope: "Union[Scope, None]" = None,
):
"""Can be initialized with either a file for the default Scope, or with the properties of a Where
for any script-nested Scope.
@@ -110,8 +110,8 @@ def do(visitor, *args):
# Any visitor function provided may return a truthy value to abort the visit and return that value.
def visit_script(
self,
- visit_where: Callable[[Where, Scope], Any] | None = None,
- visit_expect: Callable[[Expect, Any, Scope], Any] | None = None,
+ visit_where: Optional[Callable[[Where, Scope], Any]] = None,
+ visit_expect: Optional[Callable[[Expect, Any, Scope], Any]] = None,
) -> Any:
"""Visits all nodes in the script in pre-order traversal, calling any non-none provided visitor functions for
each respective node type. Note that we do not visit expected values independently of their associated expect;
@@ -124,7 +124,7 @@ def visit_script(
)
@property
- def root_wheres(self) -> set[Where]:
+ def root_wheres(self) -> Set[Where]:
return set(node for node in self.script_obj if isinstance(node, Where))
def dump(self) -> str:
@@ -142,7 +142,7 @@ def try_load_yaml(yaml_doc, loader, line_offset=0):
except yaml.MarkedYAMLError as e:
# MarkedYAMLError is an error with a 'Mark' pointing to the location of the error; this helper function applies
# our line offset to the provided mark if it is present.
- def adjust_mark_loc(mark: yaml.Mark | None) -> yaml.Mark | None:
+ def adjust_mark_loc(mark: Optional[yaml.Mark]) -> Optional[yaml.Mark]:
if mark is None:
return None
return yaml.Mark(
More information about the llvm-commits
mailing list