[llvm-branch-commits] [llvm] [Dexter] Write expects for variables in Debugger scopes (PR #203254)
Stephen Tozer via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Tue Jun 16 07:12:11 PDT 2026
https://github.com/SLTozer updated https://github.com/llvm/llvm-project/pull/203254
>From d57cdcd0dfced530e5a0a6d6bd1117018062dccd Mon Sep 17 00:00:00 2001
From: Stephen Tozer <stephen.tozer at sony.com>
Date: Thu, 11 Jun 2026 11:00:50 +0100
Subject: [PATCH] [Dexter] Write expects for variables in Debugger scopes
Following on from the previous patch, this patch adds support for writing
expects from !value/all nodes, generating separate expects for each
variable in the requested debugger scope, for each continuous range of lines
it is live for.
---
.../dexter/dex/evaluation/ExpectWriter.py | 203 ++++++++++++++++--
.../Inputs/rewrite_scopes_expected.cpp | 49 +++++
.../Inputs/rewrite_scopes_list_expected.cpp | 71 ++++++
.../scripts/rewriting/rewrite_scopes.cpp | 40 ++++
.../scripts/rewriting/rewrite_scopes_list.cpp | 54 +++++
5 files changed, 399 insertions(+), 18 deletions(-)
create mode 100644 cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/rewriting/Inputs/rewrite_scopes_expected.cpp
create mode 100644 cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/rewriting/Inputs/rewrite_scopes_list_expected.cpp
create mode 100644 cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/rewriting/rewrite_scopes.cpp
create mode 100644 cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/rewriting/rewrite_scopes_list.cpp
diff --git a/cross-project-tests/debuginfo-tests/dexter/dex/evaluation/ExpectWriter.py b/cross-project-tests/debuginfo-tests/dexter/dex/evaluation/ExpectWriter.py
index 80392c3ea68a3..d288f4caed70f 100644
--- a/cross-project-tests/debuginfo-tests/dexter/dex/evaluation/ExpectWriter.py
+++ b/cross-project-tests/debuginfo-tests/dexter/dex/evaluation/ExpectWriter.py
@@ -6,16 +6,15 @@
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
"""Utilities for using debugger output to generate expected values that match that output."""
-from collections import Counter, OrderedDict, defaultdict
-from copy import deepcopy
-from enum import Enum, IntEnum
+from collections import defaultdict
from typing import Any, Dict, List, Optional, Set, Tuple, Union
from dex.dextIR import DextIR, StepIR, ValueIR
from dex.evaluation.StateMatch import get_active_where_matches
-from dex.test_script.Nodes import Expect, Then, Value, Where
+from dex.test_script.Nodes import DexRange, Expect, Line, Then, Value, ValueAll, Where
from dex.test_script.Script import DexterScript, Scope
from dex.tools.Main import Context
+from dex.utils.Exceptions import Error
class ExpectedValueWriter:
@@ -69,6 +68,92 @@ def freeze(input):
return result
+class ExpectedScopeWriter:
+ """Given a list of ValueIRs for all variables in a scope, generates a set of expected values for each."""
+
+ def __init__(self, expect: Expect, step: StepIR, values: List[ValueIR]):
+ self.expect = expect
+ self.step = step
+ self.values = values
+ self.expected_values = [ExpectedValueWriter(expect, value) for value in values]
+
+
+# (StartLine, StopLine) -> [(Var, ExpectedValues)]
+ExpectedScopeRewrites = Dict[Optional[Tuple[int, int]], List[Tuple[str, Any]]]
+
+
+def collect_scope_values(
+ step_scope_values: List[ExpectedScopeWriter],
+) -> ExpectedScopeRewrites:
+ if not step_scope_values:
+ return {}
+ assert all(
+ step_scope_values[0].step.current_location.path == sv.step.current_location.path
+ for sv in step_scope_values[1:]
+ ), "Dexter currently does not handle scope watches that span multiple files."
+ all_vars: Set[str] = set()
+ for step_scope_writer in step_scope_values:
+ all_vars.update(
+ ev_writer.root_value.expression
+ for ev_writer in step_scope_writer.expected_values
+ )
+ line_sorted_steps = sorted(
+ step_scope_values,
+ key=lambda step_scope_writer: step_scope_writer.step.current_location.lineno,
+ )
+
+ # Now we have a list of expected values for each variable sorted by the lines at which they appear; we use this to
+ # form blocks of continuous liveness. In general, for unoptimized code we expect variables to have a single
+ # continuous live range.
+ per_range_var_unique_expected_values: ExpectedScopeRewrites = defaultdict(list)
+ for var in sorted(all_vars):
+ # Now build a list of all continuous live ranges.
+ continuous_live_ranges: List[Tuple[int, int, List[ExpectedValueWriter]]] = []
+ is_live = False
+ ever_dead = False
+ for step_scope_writer in line_sorted_steps:
+ line = step_scope_writer.step.current_location.lineno
+ var_ev_writer = next(
+ (
+ ev_writer
+ for ev_writer in step_scope_writer.expected_values
+ if ev_writer.root_value.expression == var
+ ),
+ None,
+ )
+ if var_ev_writer is None or var_ev_writer.expected_value is None:
+ is_live = False
+ ever_dead = True
+ continue
+
+ if not is_live:
+ continuous_live_ranges.append((line, line, [var_ev_writer]))
+ else:
+ start, stop, range_evs = continuous_live_ranges[-1]
+ assert line >= stop
+ range_evs.append(var_ev_writer)
+ continuous_live_ranges[-1] = (start, line, range_evs)
+ is_live = True
+
+ if not continuous_live_ranges:
+ continue
+
+ if not ever_dead:
+ assert len(continuous_live_ranges) == 1
+ per_range_var_unique_expected_values[None].append(
+ (var, unique_expected_values(continuous_live_ranges[0][2]))
+ )
+ continue
+
+ # Finally, collect the results into the per_var map.
+ for start, stop, expected_values in continuous_live_ranges:
+ per_range_var_unique_expected_values[(start, stop)].append(
+ (var, unique_expected_values(expected_values))
+ )
+
+ return per_range_var_unique_expected_values
+
+
class StepExpectWriter:
"""Processes all active, unknown expects at a given debugger step and produces ExpectedValueWriter results for
each."""
@@ -82,13 +167,24 @@ def __init__(self, step: StepIR, script: DexterScript):
for where_match in self.state_match.values()
for expect in where_match.active_expects
}
- self.expect_matches: Dict[Expect, ExpectedValueWriter] = {}
+ self.expect_value_matches: Dict[Expect, ExpectedValueWriter] = {}
+ self.expect_scope_matches: Dict[Expect, ExpectedScopeWriter] = {}
def add_expected_values(expect: Expect, expected_value: Any, scope: Scope):
- assert isinstance(expect, Value), "Non-Value expects currently unsupported"
- if expect in active_expects and expected_value is None:
- self.expect_matches[expect] = ExpectedValueWriter(
- expect, step.watches[expect.get_watched_expr()]
+ if expect not in active_expects or expected_value is not None:
+ return
+ if (expr := expect.get_watched_expr()) is not None:
+ self.expect_value_matches[expect] = ExpectedValueWriter(
+ expect, step.watches[expr]
+ )
+ elif (scope_name := expect.get_watched_scope()) is not None:
+ scope_vars = step.scope_watches.get(scope_name, [])
+ self.expect_scope_matches[expect] = ExpectedScopeWriter(
+ expect, step, [step.watches[var] for var in scope_vars]
+ )
+ else:
+ raise Error(
+ f"Unexpected expect without watched expression or scope: {expect}"
)
script.visit_script(visit_expect=add_expected_values)
@@ -104,11 +200,19 @@ def __init__(self, context: Context, dext_ir: DextIR):
self.unknown_expect_rewrites: Dict[
Expect, List[Tuple[int, ExpectedValueWriter]]
] = {}
+ self.scope_expect_rewrites: Dict[
+ Expect, List[Tuple[int, ExpectedScopeWriter]]
+ ] = {}
self.new_script: Optional[DexterScript] = None
self.new_expected_values: Dict[Expect, Any] = {}
+ self.new_expected_scopes: Dict[Expect, ExpectedScopeRewrites] = {}
self.missing_expect_rewrites: List[Expect] = []
- def collect_unknown_expects(expect: Expect, expected_value: Any, scope: Scope):
+ def collect_expects_to_write(expect: Expect, expected_value: Any, scope: Scope):
+ if isinstance(expect, ValueAll):
+ assert expected_value is None
+ self.scope_expect_rewrites[expect] = []
+ return
assert isinstance(expect, Value), "Non-Value expects currently unsupported"
if expected_value is None:
self.unknown_expect_rewrites[expect] = []
@@ -117,19 +221,29 @@ def collect_unknown_expects(expect: Expect, expected_value: Any, scope: Scope):
assert (
script is not None
), "Cannot use ScriptExpectWriter on a non-script Dexter test."
- script.visit_script(visit_expect=collect_unknown_expects)
+ script.visit_script(visit_expect=collect_expects_to_write)
# If there are no expects to update, then there is no rewriting to be done - exit early.
- if not self.unknown_expect_rewrites:
+ if not self.unknown_expect_rewrites and not self.scope_expect_rewrites:
return
self.step_writers = [StepExpectWriter(step, script) for step in dext_ir.steps]
for step_writer in self.step_writers:
step_idx = step_writer.step.step_index
- for expect, expected_value_writer in step_writer.expect_matches.items():
+ for (
+ expect,
+ expected_value_writer,
+ ) in step_writer.expect_value_matches.items():
self.unknown_expect_rewrites[expect].append(
(step_idx, expected_value_writer)
)
+ for (
+ expect,
+ expected_scope_writer,
+ ) in step_writer.expect_scope_matches.items():
+ self.scope_expect_rewrites[expect].append(
+ (step_idx, expected_scope_writer)
+ )
self.new_expected_values = {
expect: expected_values
@@ -141,7 +255,13 @@ def collect_unknown_expects(expect: Expect, expected_value: Any, scope: Scope):
)
is not None
}
- self.new_script = rewrite_script(script, self.new_expected_values)
+ self.new_expected_scopes = {
+ expect: collect_scope_values([writer for idx, writer in expect_writers])
+ for expect, expect_writers in self.scope_expect_rewrites.items()
+ }
+ self.new_script = rewrite_script(
+ script, self.new_expected_values, self.new_expected_scopes
+ )
self.missing_expect_rewrites = [
expect
for expect in self.unknown_expect_rewrites
@@ -150,7 +270,10 @@ def collect_unknown_expects(expect: Expect, expected_value: Any, scope: Scope):
@property
def num_successful_rewrites(self):
- return len(self.new_expected_values)
+ return len(self.new_expected_values) + sum(
+ sum(len(var_expects) for var_expects in new_expected_scope.values())
+ for new_expected_scope in self.new_expected_scopes.values()
+ )
@property
def num_unsuccessful_rewrites(self):
@@ -158,7 +281,9 @@ def num_unsuccessful_rewrites(self):
def rewrite_script(
- script: DexterScript, add_expected_values: Dict[Expect, Any]
+ script: DexterScript,
+ add_expected_values: Dict[Expect, Any],
+ expected_scope_rewrites: Dict[Expect, ExpectedScopeRewrites],
) -> DexterScript:
"""Given a set of updates to apply to a provided script, returns a copy of the script_obj with the updates
applied.
@@ -183,12 +308,54 @@ def replace_then(then: Then, scope: Scope):
new_node_child_map[scope.where] = then
def replace_expect(expect: Expect, expected_value, scope: Scope):
- new_expected_value = add_expected_values.get(expect) or expected_value
- new_node_child_map[expect] = new_expected_value
scope_where_children = new_node_child_map.setdefault(scope.where, [])
assert isinstance(
scope_where_children, list
), f"Unexpected child for state node {scope.where}: {scope_where_children}"
+ if isinstance(expect, ValueAll):
+ assert (
+ expect in expected_scope_rewrites
+ ), "Script-writer error: Dexter missed rewriting !expect/all node."
+ scope_rewrites = expected_scope_rewrites[expect]
+ for line_range in sorted(
+ scope_rewrites.keys(), key=lambda lines: lines or (0, 0)
+ ):
+ var_expected_values = scope_rewrites[line_range]
+ # First we determine which node will be the parent for the new expect nodes; then we can start appending
+ # new expects to that parent's child list.
+ if line_range is None:
+ new_expect_sibling_list = scope_where_children
+ else:
+ start, stop = line_range
+ lines = (
+ Line(start)
+ if start == stop
+ else DexRange(Line(start), Line(stop))
+ )
+ new_expect_parent = Where({"lines": lines}, is_and=True)
+ # Reuse an existing !and node if one exists...
+ try:
+ existing_parent = next(
+ node
+ for node in scope_where_children
+ if str(node) == str(new_expect_parent)
+ )
+ new_expect_sibling_list = new_node_child_map.setdefault(
+ existing_parent, []
+ )
+ except StopIteration:
+ scope_where_children.append(new_expect_parent)
+ new_expect_sibling_list = new_node_child_map.setdefault(
+ new_expect_parent, []
+ )
+ for var, expected_values in var_expected_values:
+ new_expect = Value(var)
+ new_expect_sibling_list.append(new_expect)
+ new_node_child_map[new_expect] = expected_values
+ return
+ assert isinstance(expect, Value)
+ new_expected_value = add_expected_values.get(expect) or expected_value
+ new_node_child_map[expect] = new_expected_value
scope_where_children.append(expect)
script.visit_script(
diff --git a/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/rewriting/Inputs/rewrite_scopes_expected.cpp b/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/rewriting/Inputs/rewrite_scopes_expected.cpp
new file mode 100644
index 0000000000000..0786712bcf65c
--- /dev/null
+++ b/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/rewriting/Inputs/rewrite_scopes_expected.cpp
@@ -0,0 +1,49 @@
+// RUN: rm -rf %t
+// RUN: mkdir %t
+// RUN: %dexter_regression_test_cxx_build %s -o %t/test
+// RUN: %dexter_regression_test_run --use-script --binary %t/test \
+// RUN: --results-directory %t/results -- %s 2>&1 | FileCheck %s
+// RUN: diff %t/results/%{s:basename} \
+// RUN: %S/Inputs/rewrite_scopes_expected.cpp
+
+/// Tests that we can collect the values of all available variables with
+/// !value/all and produce corresponding variable expects.
+
+/// NB: The exact contents of this file are compared against the expect file in
+/// the Inputs/ directory; any changes to this file, including comments,
+/// will require updating the corresponding expected file.
+
+char There[] = "Here";
+
+int main() {
+ int One = 2;
+ char Red[] = "Blue";
+ return 0; // !dex_label ret
+}
+
+// CHECK: total_watched_steps: 3
+// CHECK: correct_steps: 3
+// CHECK: incorrect_steps: 0
+// CHECK: missing_var_steps: 0
+// CHECK: unexpected_value_steps: 0
+// CHECK: seen_values: 11
+// CHECK: missing_values: 0
+
+/*
+---
+? !where {lines: !label 'ret'}
+: !value 'One': '2'
+ !value 'Red':
+ '[0]': '''B'''
+ '[1]': '''l'''
+ '[2]': '''u'''
+ '[3]': '''e'''
+ '[4]': '''\0'''
+ !value '::There':
+ '[0]': '''H'''
+ '[1]': '''e'''
+ '[2]': '''r'''
+ '[3]': '''e'''
+ '[4]': '''\0'''
+...
+*/
diff --git a/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/rewriting/Inputs/rewrite_scopes_list_expected.cpp b/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/rewriting/Inputs/rewrite_scopes_list_expected.cpp
new file mode 100644
index 0000000000000..8c536f08b710b
--- /dev/null
+++ b/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/rewriting/Inputs/rewrite_scopes_list_expected.cpp
@@ -0,0 +1,71 @@
+// RUN: rm -rf %t
+// RUN: mkdir %t
+// RUN: %dexter_regression_test_cxx_build %s -o %t/test
+// RUN: %dexter_regression_test_run --use-script --binary %t/test \
+// RUN: --results-directory %t/results -- %s 2>&1 | FileCheck %s
+// RUN: diff %t/results/%{s:basename} \
+// RUN: %S/Inputs/rewrite_scopes_list_expected.cpp
+
+/// Tests that !value/all creates appropriate state nodes for the different
+/// variables:
+/// - TopFloor is live for the entire scope of the !value/all, so appears
+/// directly under the root !where
+/// - Elevator is live at two disjoint positions, so appears under two different
+/// !and nodes.
+/// - Ground and Button become live at the same time after the !value/all
+/// becomes active and for the remainder of the program, so are both placed
+/// under a single shared !and.
+
+/// NB: The exact contents of this file are compared against the expect file in
+/// the Inputs/ directory; any changes to this file, including comments,
+/// will require updating the corresponding expected file.
+
+void ding() {}
+void swapPassengers() {}
+
+int main() {
+ int TopFloor = 10;
+ int Ground = 0, Button = 6; // !dex_label start
+ for (int Elevator = Ground; Elevator < Button; ++Elevator) {
+ ding();
+ }
+ swapPassengers();
+ for (int Elevator = Button; Elevator < TopFloor; ++Elevator) {
+ ding();
+ }
+ return 0; // !dex_label ret
+}
+
+// CHECK: Rewrote script to add 5 expected values.
+
+// CHECK: total_watched_steps: 83
+// CHECK: correct_steps: 83
+// CHECK: incorrect_steps: 0
+// CHECK: missing_var_steps: 0
+// CHECK: unexpected_value_steps: 0
+// CHECK: seen_values: 13
+// CHECK: missing_values: 0
+
+/*
+---
+? !where {lines: !range [!label 'start', !label 'ret']}
+: !value 'TopFloor': '10'
+ ? !and {lines: !range [29, 36]}
+ : !value 'Button': '6'
+ !value 'Ground': '0'
+ ? !and {lines: 30}
+ : !value 'Elevator':
+ - '0'
+ - '1'
+ - '2'
+ - '3'
+ - '4'
+ - '5'
+ ? !and {lines: 34}
+ : !value 'Elevator':
+ - '6'
+ - '7'
+ - '8'
+ - '9'
+...
+*/
diff --git a/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/rewriting/rewrite_scopes.cpp b/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/rewriting/rewrite_scopes.cpp
new file mode 100644
index 0000000000000..fc0fec1a91bb0
--- /dev/null
+++ b/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/rewriting/rewrite_scopes.cpp
@@ -0,0 +1,40 @@
+// RUN: rm -rf %t
+// RUN: mkdir %t
+// RUN: %dexter_regression_test_cxx_build %s -o %t/test
+// RUN: %dexter_regression_test_run --use-script --binary %t/test \
+// RUN: --results-directory %t/results -- %s 2>&1 | FileCheck %s
+// RUN: diff %t/results/%{s:basename} \
+// RUN: %S/Inputs/rewrite_scopes_expected.cpp
+
+/// Tests that we can collect the values of all available variables with
+/// !value/all and produce corresponding variable expects.
+
+/// NB: The exact contents of this file are compared against the expect file in
+/// the Inputs/ directory; any changes to this file, including comments,
+/// will require updating the corresponding expected file.
+
+char There[] = "Here";
+
+int main() {
+ int One = 2;
+ char Red[] = "Blue";
+ return 0; // !dex_label ret
+}
+
+// CHECK: total_watched_steps: 3
+// CHECK: correct_steps: 3
+// CHECK: incorrect_steps: 0
+// CHECK: missing_var_steps: 0
+// CHECK: unexpected_value_steps: 0
+// CHECK: seen_values: 11
+// CHECK: missing_values: 0
+
+/*
+---
+!where {lines: !label ret}:
+ ? !value/all Locals
+ ? !value/all Globals
+ # Invalid scopes won't appear in the output.
+ ? !value/all NotARealScope
+...
+*/
diff --git a/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/rewriting/rewrite_scopes_list.cpp b/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/rewriting/rewrite_scopes_list.cpp
new file mode 100644
index 0000000000000..1291a30fb2796
--- /dev/null
+++ b/cross-project-tests/debuginfo-tests/dexter/feature_tests/scripts/rewriting/rewrite_scopes_list.cpp
@@ -0,0 +1,54 @@
+// RUN: rm -rf %t
+// RUN: mkdir %t
+// RUN: %dexter_regression_test_cxx_build %s -o %t/test
+// RUN: %dexter_regression_test_run --use-script --binary %t/test \
+// RUN: --results-directory %t/results -- %s 2>&1 | FileCheck %s
+// RUN: diff %t/results/%{s:basename} \
+// RUN: %S/Inputs/rewrite_scopes_list_expected.cpp
+
+/// Tests that !value/all creates appropriate state nodes for the different
+/// variables:
+/// - TopFloor is live for the entire scope of the !value/all, so appears
+/// directly under the root !where
+/// - Elevator is live at two disjoint positions, so appears under two different
+/// !and nodes.
+/// - Ground and Button become live at the same time after the !value/all
+/// becomes active and for the remainder of the program, so are both placed
+/// under a single shared !and.
+
+/// NB: The exact contents of this file are compared against the expect file in
+/// the Inputs/ directory; any changes to this file, including comments,
+/// will require updating the corresponding expected file.
+
+void ding() {}
+void swapPassengers() {}
+
+int main() {
+ int TopFloor = 10;
+ int Ground = 0, Button = 6; // !dex_label start
+ for (int Elevator = Ground; Elevator < Button; ++Elevator) {
+ ding();
+ }
+ swapPassengers();
+ for (int Elevator = Button; Elevator < TopFloor; ++Elevator) {
+ ding();
+ }
+ return 0; // !dex_label ret
+}
+
+// CHECK: Rewrote script to add 5 expected values.
+
+// CHECK: total_watched_steps: 83
+// CHECK: correct_steps: 83
+// CHECK: incorrect_steps: 0
+// CHECK: missing_var_steps: 0
+// CHECK: unexpected_value_steps: 0
+// CHECK: seen_values: 13
+// CHECK: missing_values: 0
+
+/*
+---
+!where {lines: !range [!label start, !label ret]}:
+ ? !value/all Locals
+...
+*/
More information about the llvm-branch-commits
mailing list