[llvm] 88dd82f - [opt-viewer] Fix demangler usage with multiprocessing (#228388)

[via llvm-commits]

Author: Ömer Sinan Ağacan
Date: 2026-10-03T22:22:02+02:00
New Revision: 88dd82f800f46c3ec4c16932a705091434e17ae0

URL: https://github.com/llvm/llvm-project/commit/88dd82f800f46c3ec4c16932a705091434e17ae0
DIFF: https://github.com/llvm/llvm-project/commit/88dd82f800f46c3ec4c16932a705091434e17ae0.diff

LOG: [opt-viewer] Fix demangler usage with multiprocessing (#228388)

Because of how `multiprocessing.Pool` creates new processes on macOS,
parallel `_render_file`s create their own demangler processes.

However the CLI flag `--demangler` also does not set an inherited state
that's used by the subprocesses, so the new demanglers are created from
the default command, which is `c++filt -n`.

This also means that the semaphore in class `Remark` is not useful on
macOS. It's also not properly cleaned up, causing warnings on exit:

    .../python3.9/multiprocessing/resource_tracker.py:216: UserWarning:
    resource_tracker: There appear to be 36 leaked semaphore objects to
    clean up at shutdown

With this patch we now pass the `--demangler` value to the `Remark`
class so the subprocesses can make use of it. We also remove the
semaphore and make each subprocess have their own demangler process.
This simplifies the code (no need for locking) as the demangler
processes are cheap to start and keep running and don't need to be
shared.

To observe the bug on macOS, generate inlining remarks with

-fsave-optimization-record -foptimization-record-passes=inline
-gline-tables-only

then render as HTML:

    python3 <llvm checkout>/llvm/tools/opt-viewer/opt-viewer.py \
      --demangler 'sed -l s/^/[CUSTOM]/' -s <source dir> \
      -o html <remark file dir>

In the output, you'll see that the index.html has the demangled symbols
starting with '[CUSTOM]', but if you click on the pages the demangled
symbols don't have the same prefix, because the `--demangler` command is
not used in the subprocesses.

The new demangling implementation should work on all platforms as it no
longer relies on process state inheriting behavior.

Added: 
    

Modified: 
    llvm/tools/opt-viewer/opt-viewer.py
    llvm/tools/opt-viewer/optrecord.py

Removed: 
    


################################################################################
diff  --git a/llvm/tools/opt-viewer/opt-viewer.py b/llvm/tools/opt-viewer/opt-viewer.py
index 6fb8a8dc795f1..51995fb85296a 100755
--- a/llvm/tools/opt-viewer/opt-viewer.py
+++ b/llvm/tools/opt-viewer/opt-viewer.py
@@ -286,9 +286,10 @@ def render(self, all_remarks):
         )
 
 
-def _render_file(source_dir, output_dir, ctx, no_highlight, entry, filter_):
+def _render_file(source_dir, output_dir, ctx, no_highlight, demangler, entry, filter_):
     global context
     context = ctx
+    optrecord.Remark.demangler_cmd = demangler
     filename, remarks = entry
     SourceFileRenderer(source_dir, output_dir, filename, no_highlight).render(remarks)
 
@@ -322,6 +323,7 @@ def generate_report(
     max_hottest_remarks_on_index,
     num_jobs,
     should_print_progress,
+    demangler,
 ):
     try:
         os.makedirs(output_dir)
@@ -369,7 +371,7 @@ def generate_report(
     )
 
     _render_file_bound = functools.partial(
-        _render_file, source_dir, output_dir, context, no_highlight
+        _render_file, source_dir, output_dir, context, no_highlight, demangler
     )
     if should_print_progress:
         print("Rendering HTML files...")
@@ -424,8 +426,8 @@ def main():
     )
     parser.add_argument(
         "--demangler",
-        help="Set the demangler to be used (defaults to %s)"
-        % optrecord.Remark.default_demangler,
+        default=optrecord.Remark.default_demangler,
+        help="Set the demangler to be used (defaults to %(default)s)",
     )
 
     parser.add_argument(
@@ -440,8 +442,7 @@ def main():
     args = parser.parse_args()
 
     print_progress = not args.no_progress_indicator
-    if args.demangler:
-        optrecord.Remark.set_demangler(args.demangler)
+    optrecord.Remark.demangler_cmd = args.demangler
 
     files = optrecord.find_opt_files(*args.yaml_dirs_or_files)
     if not files:
@@ -464,6 +465,7 @@ def main():
         args.max_hottest_remarks_on_index,
         args.jobs,
         print_progress,
+        args.demangler,
     )
 
 

diff  --git a/llvm/tools/opt-viewer/optrecord.py b/llvm/tools/opt-viewer/optrecord.py
index 94197e12456fb..ef70a743f0b2b 100644
--- a/llvm/tools/opt-viewer/optrecord.py
+++ b/llvm/tools/opt-viewer/optrecord.py
@@ -16,7 +16,6 @@
 from collections import defaultdict
 import fnmatch
 import functools
-from multiprocessing import Lock
 import os, os.path
 import subprocess
 from sys import intern
@@ -46,23 +45,25 @@ class Remark(yaml.YAMLObject):
     yaml_loader = Loader
 
     default_demangler = "c++filt -n"
+    demangler_cmd = default_demangler
     demangler_proc = None
-    demangler_lock = Lock()
-
-    @classmethod
-    def set_demangler(cls, demangler):
-        cls.demangler_proc = subprocess.Popen(
-            demangler.split(), stdin=subprocess.PIPE, stdout=subprocess.PIPE
-        )
+    demangler_owner_pid = None
 
     @classmethod
     def demangle(cls, name):
-        with cls.demangler_lock:
-            if not cls.demangler_proc:
-                cls.set_demangler(cls.default_demangler)
-            cls.demangler_proc.stdin.write((name + "\n").encode("utf-8"))
-            cls.demangler_proc.stdin.flush()
-            return cls.demangler_proc.stdout.readline().rstrip().decode("utf-8")
+        # Start a demangler on first use in each process. Workers created by
+        # `fork` inherit the parent's class state, including its pipes, so
+        # compare against the current pid rather than just checking for None.
+        if cls.demangler_owner_pid != os.getpid():
+            cls.demangler_proc = subprocess.Popen(
+                cls.demangler_cmd.split(),
+                stdin=subprocess.PIPE,
+                stdout=subprocess.PIPE,
+            )
+            cls.demangler_owner_pid = os.getpid()
+        cls.demangler_proc.stdin.write((name + "\n").encode("utf-8"))
+        cls.demangler_proc.stdin.flush()
+        return cls.demangler_proc.stdout.readline().rstrip().decode("utf-8")
 
     # Intern all strings since we have lot of duplication across filenames,
     # remark text.