[llvm] r370095 - Revert Autogenerate the shebang lines for tools/opt-viewer

Reid Kleckner via llvm-commits llvm-commits at lists.llvm.org
Tue Aug 27 11:31:30 PDT 2019


Author: rnk
Date: Tue Aug 27 11:31:29 2019
New Revision: 370095

URL: http://llvm.org/viewvc/llvm-project?rev=370095&view=rev
Log:
Revert Autogenerate the shebang lines for tools/opt-viewer

This reverts r369486 (git commit 8d18384809957cc923752e10a86adab129e3df48)

The opt-viewer tests don't pass after this change, and fixing them isn't
trivial. opt-viewer.py imports optmap, which requires adjusting
pythonpath, which is more work than I'm willing to do to fix forward.

Added:
    llvm/trunk/tools/opt-viewer/opt-diff.py   (with props)
    llvm/trunk/tools/opt-viewer/opt-stats.py   (with props)
    llvm/trunk/tools/opt-viewer/opt-viewer.py   (with props)
    llvm/trunk/tools/opt-viewer/optrecord.py
Removed:
    llvm/trunk/tools/opt-viewer/opt-diff.py.in
    llvm/trunk/tools/opt-viewer/opt-stats.py.in
    llvm/trunk/tools/opt-viewer/opt-viewer.py.in
    llvm/trunk/tools/opt-viewer/optrecord.py.in
Modified:
    llvm/trunk/CMakeLists.txt
    llvm/trunk/tools/opt-viewer/CMakeLists.txt

Modified: llvm/trunk/CMakeLists.txt
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/CMakeLists.txt?rev=370095&r1=370094&r2=370095&view=diff
==============================================================================
--- llvm/trunk/CMakeLists.txt (original)
+++ llvm/trunk/CMakeLists.txt Tue Aug 27 11:31:29 2019
@@ -660,8 +660,6 @@ if( ${PYTHON_VERSION_STRING} VERSION_LES
   message(FATAL_ERROR "Python 2.7 or newer is required")
 endif()
 
-get_filename_component(PYTHON_BASENAME ${PYTHON_EXECUTABLE} NAME)
-
 ######
 # LLVMBuild Integration
 #

Modified: llvm/trunk/tools/opt-viewer/CMakeLists.txt
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/tools/opt-viewer/CMakeLists.txt?rev=370095&r1=370094&r2=370095&view=diff
==============================================================================
--- llvm/trunk/tools/opt-viewer/CMakeLists.txt (original)
+++ llvm/trunk/tools/opt-viewer/CMakeLists.txt Tue Aug 27 11:31:29 2019
@@ -1,28 +1,13 @@
 set (files
-  "optpmap.py"
-  "style.css")
-
-set (generated_files
   "opt-diff.py"
   "opt-stats.py"
   "opt-viewer.py"
-  "optrecord.py")
-
-foreach (file ${generated_files})
-  configure_file(
-    ${CMAKE_CURRENT_SOURCE_DIR}/${file}.in
-    ${CMAKE_CURRENT_BINARY_DIR}/${file})
-endforeach (file)
+  "optpmap.py"
+  "optrecord.py"
+  "style.css")
 
 foreach (file ${files})
   install(PROGRAMS ${file}
     DESTINATION share/opt-viewer
     COMPONENT opt-viewer)
 endforeach (file)
-
-
-foreach (file ${generated_files})
-  install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/${file}
-    DESTINATION share/opt-viewer
-    COMPONENT opt-viewer)
-endforeach (file)

Added: llvm/trunk/tools/opt-viewer/opt-diff.py
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/tools/opt-viewer/opt-diff.py?rev=370095&view=auto
==============================================================================
--- llvm/trunk/tools/opt-viewer/opt-diff.py (added)
+++ llvm/trunk/tools/opt-viewer/opt-diff.py Tue Aug 27 11:31:29 2019
@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+
+desc = '''Generate the difference of two YAML files into a new YAML file (works on
+pair of directories too).  A new attribute 'Added' is set to True or False
+depending whether the entry is added or removed from the first input to the
+next.
+
+The tools requires PyYAML.'''
+
+import yaml
+# Try to use the C parser.
+try:
+    from yaml import CLoader as Loader
+except ImportError:
+    from yaml import Loader
+
+import optrecord
+import argparse
+from collections import defaultdict
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description=desc)
+    parser.add_argument(
+        'yaml_dir_or_file_1',
+        help='An optimization record file or a directory searched for optimization '
+             'record files that are used as the old version for the comparison')
+    parser.add_argument(
+        'yaml_dir_or_file_2',
+        help='An optimization record file or a directory searched for optimization '
+             'record files that are used as the new version for the comparison')
+    parser.add_argument(
+        '--jobs',
+        '-j',
+        default=None,
+        type=int,
+        help='Max job count (defaults to %(default)s, the current CPU count)')
+    parser.add_argument(
+        '--max-size',
+        '-m',
+        default=100000,
+        type=int,
+        help='Maximum number of remarks stored in an output file')
+    parser.add_argument(
+        '--no-progress-indicator',
+        '-n',
+        action='store_true',
+        default=False,
+        help='Do not display any indicator of how many YAML files were read.')
+    parser.add_argument('--output', '-o', default='diff{}.opt.yaml')
+    args = parser.parse_args()
+
+    files1 = optrecord.find_opt_files(args.yaml_dir_or_file_1)
+    files2 = optrecord.find_opt_files(args.yaml_dir_or_file_2)
+
+    print_progress = not args.no_progress_indicator
+    all_remarks1, _, _ = optrecord.gather_results(files1, args.jobs, print_progress)
+    all_remarks2, _, _ = optrecord.gather_results(files2, args.jobs, print_progress)
+
+    added = set(all_remarks2.values()) - set(all_remarks1.values())
+    removed = set(all_remarks1.values()) - set(all_remarks2.values())
+
+    for r in added:
+        r.Added = True
+    for r in removed:
+        r.Added = False
+
+    result = list(added | removed)
+    for r in result:
+        r.recover_yaml_structure()
+
+    for i in range(0, len(result), args.max_size):
+        with open(args.output.format(i / args.max_size), 'w') as stream:
+            yaml.dump_all(result[i:i + args.max_size], stream)

Propchange: llvm/trunk/tools/opt-viewer/opt-diff.py
------------------------------------------------------------------------------
    svn:executable = *

Removed: llvm/trunk/tools/opt-viewer/opt-diff.py.in
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/tools/opt-viewer/opt-diff.py.in?rev=370094&view=auto
==============================================================================
--- llvm/trunk/tools/opt-viewer/opt-diff.py.in (original)
+++ llvm/trunk/tools/opt-viewer/opt-diff.py.in (removed)
@@ -1,75 +0,0 @@
-#!/usr/bin/env @PYTHON_BASENAME@
-
-from __future__ import print_function
-
-desc = '''Generate the difference of two YAML files into a new YAML file (works on
-pair of directories too).  A new attribute 'Added' is set to True or False
-depending whether the entry is added or removed from the first input to the
-next.
-
-The tools requires PyYAML.'''
-
-import yaml
-# Try to use the C parser.
-try:
-    from yaml import CLoader as Loader
-except ImportError:
-    from yaml import Loader
-
-import optrecord
-import argparse
-from collections import defaultdict
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description=desc)
-    parser.add_argument(
-        'yaml_dir_or_file_1',
-        help='An optimization record file or a directory searched for optimization '
-             'record files that are used as the old version for the comparison')
-    parser.add_argument(
-        'yaml_dir_or_file_2',
-        help='An optimization record file or a directory searched for optimization '
-             'record files that are used as the new version for the comparison')
-    parser.add_argument(
-        '--jobs',
-        '-j',
-        default=None,
-        type=int,
-        help='Max job count (defaults to %(default)s, the current CPU count)')
-    parser.add_argument(
-        '--max-size',
-        '-m',
-        default=100000,
-        type=int,
-        help='Maximum number of remarks stored in an output file')
-    parser.add_argument(
-        '--no-progress-indicator',
-        '-n',
-        action='store_true',
-        default=False,
-        help='Do not display any indicator of how many YAML files were read.')
-    parser.add_argument('--output', '-o', default='diff{}.opt.yaml')
-    args = parser.parse_args()
-
-    files1 = optrecord.find_opt_files(args.yaml_dir_or_file_1)
-    files2 = optrecord.find_opt_files(args.yaml_dir_or_file_2)
-
-    print_progress = not args.no_progress_indicator
-    all_remarks1, _, _ = optrecord.gather_results(files1, args.jobs, print_progress)
-    all_remarks2, _, _ = optrecord.gather_results(files2, args.jobs, print_progress)
-
-    added = set(all_remarks2.values()) - set(all_remarks1.values())
-    removed = set(all_remarks1.values()) - set(all_remarks2.values())
-
-    for r in added:
-        r.Added = True
-    for r in removed:
-        r.Added = False
-
-    result = list(added | removed)
-    for r in result:
-        r.recover_yaml_structure()
-
-    for i in range(0, len(result), args.max_size):
-        with open(args.output.format(i / args.max_size), 'w') as stream:
-            yaml.dump_all(result[i:i + args.max_size], stream)

Added: llvm/trunk/tools/opt-viewer/opt-stats.py
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/tools/opt-viewer/opt-stats.py?rev=370095&view=auto
==============================================================================
--- llvm/trunk/tools/opt-viewer/opt-stats.py (added)
+++ llvm/trunk/tools/opt-viewer/opt-stats.py Tue Aug 27 11:31:29 2019
@@ -0,0 +1,78 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+
+desc = '''Generate statistics about optimization records from the YAML files
+generated with -fsave-optimization-record and -fdiagnostics-show-hotness.
+
+The tools requires PyYAML and Pygments Python packages.'''
+
+import optrecord
+import argparse
+import operator
+from collections import defaultdict
+from multiprocessing import cpu_count, Pool
+
+try:
+    from guppy import hpy
+    hp = hpy()
+except ImportError:
+    print("Memory consumption not shown because guppy is not installed")
+    hp = None
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description=desc)
+    parser.add_argument(
+        'yaml_dirs_or_files',
+        nargs='+',
+        help='List of optimization record files or directories searched '
+             'for optimization record files.')
+    parser.add_argument(
+        '--jobs',
+        '-j',
+        default=None,
+        type=int,
+        help='Max job count (defaults to %(default)s, the current CPU count)')
+    parser.add_argument(
+        '--no-progress-indicator',
+        '-n',
+        action='store_true',
+        default=False,
+        help='Do not display any indicator of how many YAML files were read.')
+    args = parser.parse_args()
+
+    print_progress = not args.no_progress_indicator
+
+    files = optrecord.find_opt_files(*args.yaml_dirs_or_files)
+    if not files:
+        parser.error("No *.opt.yaml files found")
+        sys.exit(1)
+
+    all_remarks, file_remarks, _ = optrecord.gather_results(
+        files, args.jobs, print_progress)
+    if print_progress:
+        print('\n')
+
+    bypass = defaultdict(int)
+    byname = defaultdict(int)
+    for r in optrecord.itervalues(all_remarks):
+        bypass[r.Pass] += 1
+        byname[r.Pass + "/" + r.Name] += 1
+
+    total = len(all_remarks)
+    print("{:24s} {:10d}".format("Total number of remarks", total))
+    if hp:
+        h = hp.heap()
+        print("{:24s} {:10d}".format("Memory per remark",
+                                     h.size / len(all_remarks)))
+    print('\n')
+
+    print("Top 10 remarks by pass:")
+    for (passname, count) in sorted(bypass.items(), key=operator.itemgetter(1),
+                                    reverse=True)[:10]:
+        print("  {:30s} {:2.0f}%". format(passname, count * 100. / total))
+
+    print("\nTop 10 remarks:")
+    for (name, count) in sorted(byname.items(), key=operator.itemgetter(1),
+                                reverse=True)[:10]:
+        print("  {:30s} {:2.0f}%". format(name, count * 100. / total))

Propchange: llvm/trunk/tools/opt-viewer/opt-stats.py
------------------------------------------------------------------------------
    svn:executable = *

Removed: llvm/trunk/tools/opt-viewer/opt-stats.py.in
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/tools/opt-viewer/opt-stats.py.in?rev=370094&view=auto
==============================================================================
--- llvm/trunk/tools/opt-viewer/opt-stats.py.in (original)
+++ llvm/trunk/tools/opt-viewer/opt-stats.py.in (removed)
@@ -1,78 +0,0 @@
-#!/usr/bin/env @PYTHON_BASENAME@
-
-from __future__ import print_function
-
-desc = '''Generate statistics about optimization records from the YAML files
-generated with -fsave-optimization-record and -fdiagnostics-show-hotness.
-
-The tools requires PyYAML and Pygments Python packages.'''
-
-import optrecord
-import argparse
-import operator
-from collections import defaultdict
-from multiprocessing import cpu_count, Pool
-
-try:
-    from guppy import hpy
-    hp = hpy()
-except ImportError:
-    print("Memory consumption not shown because guppy is not installed")
-    hp = None
-
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description=desc)
-    parser.add_argument(
-        'yaml_dirs_or_files',
-        nargs='+',
-        help='List of optimization record files or directories searched '
-             'for optimization record files.')
-    parser.add_argument(
-        '--jobs',
-        '-j',
-        default=None,
-        type=int,
-        help='Max job count (defaults to %(default)s, the current CPU count)')
-    parser.add_argument(
-        '--no-progress-indicator',
-        '-n',
-        action='store_true',
-        default=False,
-        help='Do not display any indicator of how many YAML files were read.')
-    args = parser.parse_args()
-
-    print_progress = not args.no_progress_indicator
-
-    files = optrecord.find_opt_files(*args.yaml_dirs_or_files)
-    if not files:
-        parser.error("No *.opt.yaml files found")
-        sys.exit(1)
-
-    all_remarks, file_remarks, _ = optrecord.gather_results(
-        files, args.jobs, print_progress)
-    if print_progress:
-        print('\n')
-
-    bypass = defaultdict(int)
-    byname = defaultdict(int)
-    for r in optrecord.itervalues(all_remarks):
-        bypass[r.Pass] += 1
-        byname[r.Pass + "/" + r.Name] += 1
-
-    total = len(all_remarks)
-    print("{:24s} {:10d}".format("Total number of remarks", total))
-    if hp:
-        h = hp.heap()
-        print("{:24s} {:10d}".format("Memory per remark",
-                                     h.size / len(all_remarks)))
-    print('\n')
-
-    print("Top 10 remarks by pass:")
-    for (passname, count) in sorted(bypass.items(), key=operator.itemgetter(1),
-                                    reverse=True)[:10]:
-        print("  {:30s} {:2.0f}%". format(passname, count * 100. / total))
-
-    print("\nTop 10 remarks:")
-    for (name, count) in sorted(byname.items(), key=operator.itemgetter(1),
-                                reverse=True)[:10]:
-        print("  {:30s} {:2.0f}%". format(name, count * 100. / total))

Added: llvm/trunk/tools/opt-viewer/opt-viewer.py
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/tools/opt-viewer/opt-viewer.py?rev=370095&view=auto
==============================================================================
--- llvm/trunk/tools/opt-viewer/opt-viewer.py (added)
+++ llvm/trunk/tools/opt-viewer/opt-viewer.py Tue Aug 27 11:31:29 2019
@@ -0,0 +1,382 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+
+import argparse
+import cgi
+import codecs
+import errno
+import functools
+from multiprocessing import cpu_count
+import os.path
+import re
+import shutil
+import sys
+
+from pygments import highlight
+from pygments.lexers.c_cpp import CppLexer
+from pygments.formatters import HtmlFormatter
+
+import optpmap
+import optrecord
+
+
+desc = '''Generate HTML output to visualize optimization records from the YAML files
+generated with -fsave-optimization-record and -fdiagnostics-show-hotness.
+
+The tools requires PyYAML and Pygments Python packages.'''
+
+
+# This allows passing the global context to the child processes.
+class Context:
+    def __init__(self, caller_loc = dict()):
+       # Map function names to their source location for function where inlining happened
+       self.caller_loc = caller_loc
+
+context = Context()
+
+def suppress(remark):
+    if remark.Name == 'sil.Specialized':
+        return remark.getArgDict()['Function'][0].startswith('\"Swift.')
+    elif remark.Name == 'sil.Inlined':
+        return remark.getArgDict()['Callee'][0].startswith(('\"Swift.', '\"specialized Swift.'))
+    return False
+
+class SourceFileRenderer:
+    def __init__(self, source_dir, output_dir, filename, no_highlight):
+        self.filename = filename
+        existing_filename = None
+        if os.path.exists(filename):
+            existing_filename = filename
+        else:
+            fn = os.path.join(source_dir, filename)
+            if os.path.exists(fn):
+                existing_filename = fn
+
+        self.no_highlight = no_highlight
+        self.stream = codecs.open(os.path.join(output_dir, optrecord.html_file_name(filename)), 'w', encoding='utf-8')
+        if existing_filename:
+            self.source_stream = open(existing_filename)
+        else:
+            self.source_stream = None
+            print('''
+<html>
+<h1>Unable to locate file {}</h1>
+</html>
+            '''.format(filename), file=self.stream)
+
+        self.html_formatter = HtmlFormatter(encoding='utf-8')
+        self.cpp_lexer = CppLexer(stripnl=False)
+
+    def render_source_lines(self, stream, line_remarks):
+        file_text = stream.read()
+
+        if self.no_highlight:
+            if sys.version_info.major >= 3:
+                html_highlighted = file_text
+            else:
+                html_highlighted = file_text.decode('utf-8')
+        else:
+            html_highlighted = highlight(
+            file_text,
+                self.cpp_lexer,
+                self.html_formatter)
+
+            # Note that the API is different between Python 2 and 3.  On
+            # Python 3, pygments.highlight() returns a bytes object, so we
+            # have to decode.  On Python 2, the output is str but since we
+            # support unicode characters and the output streams is unicode we
+            # decode too.
+            html_highlighted = html_highlighted.decode('utf-8')
+
+            # Take off the header and footer, these must be
+            #   reapplied line-wise, within the page structure
+            html_highlighted = html_highlighted.replace('<div class="highlight"><pre>', '')
+            html_highlighted = html_highlighted.replace('</pre></div>', '')
+
+        for (linenum, html_line) in enumerate(html_highlighted.split('\n'), start=1):
+            print(u'''
+<tr>
+<td><a name=\"L{linenum}\">{linenum}</a></td>
+<td></td>
+<td></td>
+<td><div class="highlight"><pre>{html_line}</pre></div></td>
+</tr>'''.format(**locals()), file=self.stream)
+
+            for remark in line_remarks.get(linenum, []):
+                if not suppress(remark):
+                    self.render_inline_remarks(remark, html_line)
+
+    def render_inline_remarks(self, r, line):
+        inlining_context = r.DemangledFunctionName
+        dl = context.caller_loc.get(r.Function)
+        if dl:
+            dl_dict = dict(list(dl))
+            link = optrecord.make_link(dl_dict['File'], dl_dict['Line'] - 2)
+            inlining_context = "<a href={link}>{r.DemangledFunctionName}</a>".format(**locals())
+
+        # Column is the number of characters *including* tabs, keep those and
+        # replace everything else with spaces.
+        indent = line[:max(r.Column, 1) - 1]
+        indent = re.sub('\S', ' ', indent)
+
+        # Create expanded message and link if we have a multiline message.
+        lines = r.message.split('\n')
+        if len(lines) > 1:
+            expand_link = '<a style="text-decoration: none;" href="" onclick="toggleExpandedMessage(this); return false;">+</a>'
+            message = lines[0]
+            expand_message = u'''
+<div class="full-info" style="display:none;">
+  <div class="col-left"><pre style="display:inline">{}</pre></div>
+  <div class="expanded col-left"><pre>{}</pre></div>
+</div>'''.format(indent, '\n'.join(lines[1:]))
+        else:
+            expand_link = ''
+            expand_message = ''
+            message = r.message
+        print(u'''
+<tr>
+<td></td>
+<td>{r.RelativeHotness}</td>
+<td class=\"column-entry-{r.color}\">{r.PassWithDiffPrefix}</td>
+<td><pre style="display:inline">{indent}</pre><span class=\"column-entry-yellow\">{expand_link} {message} </span>{expand_message}</td>
+<td class=\"column-entry-yellow\">{inlining_context}</td>
+</tr>'''.format(**locals()), file=self.stream)
+
+    def render(self, line_remarks):
+        if not self.source_stream:
+            return
+
+        print('''
+<html>
+<title>{}</title>
+<meta charset="utf-8" />
+<head>
+<link rel='stylesheet' type='text/css' href='style.css'>
+<script type="text/javascript">
+/* Simple helper to show/hide the expanded message of a remark. */
+function toggleExpandedMessage(e) {{
+  var FullTextElems = e.parentElement.parentElement.getElementsByClassName("full-info");
+  if (!FullTextElems || FullTextElems.length < 1) {{
+      return false;
+  }}
+  var FullText = FullTextElems[0];
+  if (FullText.style.display == 'none') {{
+    e.innerHTML = '-';
+    FullText.style.display = 'block';
+  }} else {{
+    e.innerHTML = '+';
+    FullText.style.display = 'none';
+  }}
+}}
+</script>
+</head>
+<body>
+<div class="centered">
+<table class="source">
+<thead>
+<tr>
+<th style="width: 2%">Line</td>
+<th style="width: 3%">Hotness</td>
+<th style="width: 10%">Optimization</td>
+<th style="width: 70%">Source</td>
+<th style="width: 15%">Inline Context</td>
+</tr>
+</thead>
+<tbody>'''.format(os.path.basename(self.filename)), file=self.stream)
+        self.render_source_lines(self.source_stream, line_remarks)
+
+        print('''
+</tbody>
+</table>
+</body>
+</html>''', file=self.stream)
+
+
+class IndexRenderer:
+    def __init__(self, output_dir, should_display_hotness, max_hottest_remarks_on_index):
+        self.stream = codecs.open(os.path.join(output_dir, 'index.html'), 'w', encoding='utf-8')
+        self.should_display_hotness = should_display_hotness
+        self.max_hottest_remarks_on_index = max_hottest_remarks_on_index
+
+    def render_entry(self, r, odd):
+        escaped_name = cgi.escape(r.DemangledFunctionName)
+        print(u'''
+<tr>
+<td class=\"column-entry-{odd}\"><a href={r.Link}>{r.DebugLocString}</a></td>
+<td class=\"column-entry-{odd}\">{r.RelativeHotness}</td>
+<td class=\"column-entry-{odd}\">{escaped_name}</td>
+<td class=\"column-entry-{r.color}\">{r.PassWithDiffPrefix}</td>
+</tr>'''.format(**locals()), file=self.stream)
+
+    def render(self, all_remarks):
+        print('''
+<html>
+<meta charset="utf-8" />
+<head>
+<link rel='stylesheet' type='text/css' href='style.css'>
+</head>
+<body>
+<div class="centered">
+<table>
+<tr>
+<td>Source Location</td>
+<td>Hotness</td>
+<td>Function</td>
+<td>Pass</td>
+</tr>''', file=self.stream)
+
+        max_entries = None
+        if self.should_display_hotness:
+            max_entries = self.max_hottest_remarks_on_index
+
+        for i, remark in enumerate(all_remarks[:max_entries]):
+            if not suppress(remark):
+                self.render_entry(remark, i % 2)
+        print('''
+</table>
+</body>
+</html>''', file=self.stream)
+
+
+def _render_file(source_dir, output_dir, ctx, no_highlight, entry, filter_):
+    global context
+    context = ctx
+    filename, remarks = entry
+    SourceFileRenderer(source_dir, output_dir, filename, no_highlight).render(remarks)
+
+
+def map_remarks(all_remarks):
+    # Set up a map between function names and their source location for
+    # function where inlining happened
+    for remark in optrecord.itervalues(all_remarks):
+        if isinstance(remark, optrecord.Passed) and remark.Pass == "inline" and remark.Name == "Inlined":
+            for arg in remark.Args:
+                arg_dict = dict(list(arg))
+                caller = arg_dict.get('Caller')
+                if caller:
+                    try:
+                        context.caller_loc[caller] = arg_dict['DebugLoc']
+                    except KeyError:
+                        pass
+
+
+def generate_report(all_remarks,
+                    file_remarks,
+                    source_dir,
+                    output_dir,
+                    no_highlight,
+                    should_display_hotness,
+                    max_hottest_remarks_on_index,
+                    num_jobs,
+                    should_print_progress):
+    try:
+        os.makedirs(output_dir)
+    except OSError as e:
+        if e.errno == errno.EEXIST and os.path.isdir(output_dir):
+            pass
+        else:
+            raise
+
+    if should_print_progress:
+        print('Rendering index page...')
+    if should_display_hotness:
+        sorted_remarks = sorted(optrecord.itervalues(all_remarks), key=lambda r: (r.Hotness, r.File, r.Line, r.Column, r.PassWithDiffPrefix, r.yaml_tag, r.Function), reverse=True)
+    else:
+        sorted_remarks = sorted(optrecord.itervalues(all_remarks), key=lambda r: (r.File, r.Line, r.Column, r.PassWithDiffPrefix, r.yaml_tag, r.Function))
+    IndexRenderer(output_dir, should_display_hotness, max_hottest_remarks_on_index).render(sorted_remarks)
+
+    shutil.copy(os.path.join(os.path.dirname(os.path.realpath(__file__)),
+            "style.css"), output_dir)
+
+    _render_file_bound = functools.partial(_render_file, source_dir, output_dir, context, no_highlight)
+    if should_print_progress:
+        print('Rendering HTML files...')
+    optpmap.pmap(_render_file_bound,
+                 file_remarks.items(),
+                 num_jobs,
+                 should_print_progress)
+
+
+def main():
+    parser = argparse.ArgumentParser(description=desc)
+    parser.add_argument(
+        'yaml_dirs_or_files',
+        nargs='+',
+        help='List of optimization record files or directories searched '
+             'for optimization record files.')
+    parser.add_argument(
+        '--output-dir',
+        '-o',
+        default='html',
+        help='Path to a directory where generated HTML files will be output. '
+             'If the directory does not already exist, it will be created. '
+             '"%(default)s" by default.')
+    parser.add_argument(
+        '--jobs',
+        '-j',
+        default=None,
+        type=int,
+        help='Max job count (defaults to %(default)s, the current CPU count)')
+    parser.add_argument(
+        '--source-dir',
+        '-s',
+        default='',
+        help='set source directory')
+    parser.add_argument(
+        '--no-progress-indicator',
+        '-n',
+        action='store_true',
+        default=False,
+        help='Do not display any indicator of how many YAML files were read '
+             'or rendered into HTML.')
+    parser.add_argument(
+        '--max-hottest-remarks-on-index',
+        default=1000,
+        type=int,
+        help='Maximum number of the hottest remarks to appear on the index page')
+    parser.add_argument(
+        '--no-highlight',
+        action='store_true',
+        default=False,
+        help='Do not use a syntax highlighter when rendering the source code')
+    parser.add_argument(
+        '--demangler',
+        help='Set the demangler to be used (defaults to %s)' % optrecord.Remark.default_demangler)
+
+    parser.add_argument(
+        '--filter',
+        default='',
+        help='Only display remarks from passes matching filter expression')
+
+    # Do not make this a global variable.  Values needed to be propagated through
+    # to individual classes and functions to be portable with multiprocessing across
+    # Windows and non-Windows.
+    args = parser.parse_args()
+
+    print_progress = not args.no_progress_indicator
+    if args.demangler:
+        optrecord.Remark.set_demangler(args.demangler)
+
+    files = optrecord.find_opt_files(*args.yaml_dirs_or_files)
+    if not files:
+        parser.error("No *.opt.yaml files found")
+        sys.exit(1)
+
+    all_remarks, file_remarks, should_display_hotness = \
+        optrecord.gather_results(files, args.jobs, print_progress, args.filter)
+
+    map_remarks(all_remarks)
+
+    generate_report(all_remarks,
+                    file_remarks,
+                    args.source_dir,
+                    args.output_dir,
+                    args.no_highlight,
+                    should_display_hotness,
+                    args.max_hottest_remarks_on_index,
+                    args.jobs,
+                    print_progress)
+
+if __name__ == '__main__':
+    main()

Propchange: llvm/trunk/tools/opt-viewer/opt-viewer.py
------------------------------------------------------------------------------
    svn:executable = *

Removed: llvm/trunk/tools/opt-viewer/opt-viewer.py.in
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/tools/opt-viewer/opt-viewer.py.in?rev=370094&view=auto
==============================================================================
--- llvm/trunk/tools/opt-viewer/opt-viewer.py.in (original)
+++ llvm/trunk/tools/opt-viewer/opt-viewer.py.in (removed)
@@ -1,382 +0,0 @@
-#!/usr/bin/env @PYTHON_BASENAME@
-
-from __future__ import print_function
-
-import argparse
-import cgi
-import codecs
-import errno
-import functools
-from multiprocessing import cpu_count
-import os.path
-import re
-import shutil
-import sys
-
-from pygments import highlight
-from pygments.lexers.c_cpp import CppLexer
-from pygments.formatters import HtmlFormatter
-
-import optpmap
-import optrecord
-
-
-desc = '''Generate HTML output to visualize optimization records from the YAML files
-generated with -fsave-optimization-record and -fdiagnostics-show-hotness.
-
-The tools requires PyYAML and Pygments Python packages.'''
-
-
-# This allows passing the global context to the child processes.
-class Context:
-    def __init__(self, caller_loc = dict()):
-       # Map function names to their source location for function where inlining happened
-       self.caller_loc = caller_loc
-
-context = Context()
-
-def suppress(remark):
-    if remark.Name == 'sil.Specialized':
-        return remark.getArgDict()['Function'][0].startswith('\"Swift.')
-    elif remark.Name == 'sil.Inlined':
-        return remark.getArgDict()['Callee'][0].startswith(('\"Swift.', '\"specialized Swift.'))
-    return False
-
-class SourceFileRenderer:
-    def __init__(self, source_dir, output_dir, filename, no_highlight):
-        self.filename = filename
-        existing_filename = None
-        if os.path.exists(filename):
-            existing_filename = filename
-        else:
-            fn = os.path.join(source_dir, filename)
-            if os.path.exists(fn):
-                existing_filename = fn
-
-        self.no_highlight = no_highlight
-        self.stream = codecs.open(os.path.join(output_dir, optrecord.html_file_name(filename)), 'w', encoding='utf-8')
-        if existing_filename:
-            self.source_stream = open(existing_filename)
-        else:
-            self.source_stream = None
-            print('''
-<html>
-<h1>Unable to locate file {}</h1>
-</html>
-            '''.format(filename), file=self.stream)
-
-        self.html_formatter = HtmlFormatter(encoding='utf-8')
-        self.cpp_lexer = CppLexer(stripnl=False)
-
-    def render_source_lines(self, stream, line_remarks):
-        file_text = stream.read()
-
-        if self.no_highlight:
-            if sys.version_info.major >= 3:
-                html_highlighted = file_text
-            else:
-                html_highlighted = file_text.decode('utf-8')
-        else:
-            html_highlighted = highlight(
-            file_text,
-                self.cpp_lexer,
-                self.html_formatter)
-
-            # Note that the API is different between Python 2 and 3.  On
-            # Python 3, pygments.highlight() returns a bytes object, so we
-            # have to decode.  On Python 2, the output is str but since we
-            # support unicode characters and the output streams is unicode we
-            # decode too.
-            html_highlighted = html_highlighted.decode('utf-8')
-
-            # Take off the header and footer, these must be
-            #   reapplied line-wise, within the page structure
-            html_highlighted = html_highlighted.replace('<div class="highlight"><pre>', '')
-            html_highlighted = html_highlighted.replace('</pre></div>', '')
-
-        for (linenum, html_line) in enumerate(html_highlighted.split('\n'), start=1):
-            print(u'''
-<tr>
-<td><a name=\"L{linenum}\">{linenum}</a></td>
-<td></td>
-<td></td>
-<td><div class="highlight"><pre>{html_line}</pre></div></td>
-</tr>'''.format(**locals()), file=self.stream)
-
-            for remark in line_remarks.get(linenum, []):
-                if not suppress(remark):
-                    self.render_inline_remarks(remark, html_line)
-
-    def render_inline_remarks(self, r, line):
-        inlining_context = r.DemangledFunctionName
-        dl = context.caller_loc.get(r.Function)
-        if dl:
-            dl_dict = dict(list(dl))
-            link = optrecord.make_link(dl_dict['File'], dl_dict['Line'] - 2)
-            inlining_context = "<a href={link}>{r.DemangledFunctionName}</a>".format(**locals())
-
-        # Column is the number of characters *including* tabs, keep those and
-        # replace everything else with spaces.
-        indent = line[:max(r.Column, 1) - 1]
-        indent = re.sub('\S', ' ', indent)
-
-        # Create expanded message and link if we have a multiline message.
-        lines = r.message.split('\n')
-        if len(lines) > 1:
-            expand_link = '<a style="text-decoration: none;" href="" onclick="toggleExpandedMessage(this); return false;">+</a>'
-            message = lines[0]
-            expand_message = u'''
-<div class="full-info" style="display:none;">
-  <div class="col-left"><pre style="display:inline">{}</pre></div>
-  <div class="expanded col-left"><pre>{}</pre></div>
-</div>'''.format(indent, '\n'.join(lines[1:]))
-        else:
-            expand_link = ''
-            expand_message = ''
-            message = r.message
-        print(u'''
-<tr>
-<td></td>
-<td>{r.RelativeHotness}</td>
-<td class=\"column-entry-{r.color}\">{r.PassWithDiffPrefix}</td>
-<td><pre style="display:inline">{indent}</pre><span class=\"column-entry-yellow\">{expand_link} {message} </span>{expand_message}</td>
-<td class=\"column-entry-yellow\">{inlining_context}</td>
-</tr>'''.format(**locals()), file=self.stream)
-
-    def render(self, line_remarks):
-        if not self.source_stream:
-            return
-
-        print('''
-<html>
-<title>{}</title>
-<meta charset="utf-8" />
-<head>
-<link rel='stylesheet' type='text/css' href='style.css'>
-<script type="text/javascript">
-/* Simple helper to show/hide the expanded message of a remark. */
-function toggleExpandedMessage(e) {{
-  var FullTextElems = e.parentElement.parentElement.getElementsByClassName("full-info");
-  if (!FullTextElems || FullTextElems.length < 1) {{
-      return false;
-  }}
-  var FullText = FullTextElems[0];
-  if (FullText.style.display == 'none') {{
-    e.innerHTML = '-';
-    FullText.style.display = 'block';
-  }} else {{
-    e.innerHTML = '+';
-    FullText.style.display = 'none';
-  }}
-}}
-</script>
-</head>
-<body>
-<div class="centered">
-<table class="source">
-<thead>
-<tr>
-<th style="width: 2%">Line</td>
-<th style="width: 3%">Hotness</td>
-<th style="width: 10%">Optimization</td>
-<th style="width: 70%">Source</td>
-<th style="width: 15%">Inline Context</td>
-</tr>
-</thead>
-<tbody>'''.format(os.path.basename(self.filename)), file=self.stream)
-        self.render_source_lines(self.source_stream, line_remarks)
-
-        print('''
-</tbody>
-</table>
-</body>
-</html>''', file=self.stream)
-
-
-class IndexRenderer:
-    def __init__(self, output_dir, should_display_hotness, max_hottest_remarks_on_index):
-        self.stream = codecs.open(os.path.join(output_dir, 'index.html'), 'w', encoding='utf-8')
-        self.should_display_hotness = should_display_hotness
-        self.max_hottest_remarks_on_index = max_hottest_remarks_on_index
-
-    def render_entry(self, r, odd):
-        escaped_name = cgi.escape(r.DemangledFunctionName)
-        print(u'''
-<tr>
-<td class=\"column-entry-{odd}\"><a href={r.Link}>{r.DebugLocString}</a></td>
-<td class=\"column-entry-{odd}\">{r.RelativeHotness}</td>
-<td class=\"column-entry-{odd}\">{escaped_name}</td>
-<td class=\"column-entry-{r.color}\">{r.PassWithDiffPrefix}</td>
-</tr>'''.format(**locals()), file=self.stream)
-
-    def render(self, all_remarks):
-        print('''
-<html>
-<meta charset="utf-8" />
-<head>
-<link rel='stylesheet' type='text/css' href='style.css'>
-</head>
-<body>
-<div class="centered">
-<table>
-<tr>
-<td>Source Location</td>
-<td>Hotness</td>
-<td>Function</td>
-<td>Pass</td>
-</tr>''', file=self.stream)
-
-        max_entries = None
-        if self.should_display_hotness:
-            max_entries = self.max_hottest_remarks_on_index
-
-        for i, remark in enumerate(all_remarks[:max_entries]):
-            if not suppress(remark):
-                self.render_entry(remark, i % 2)
-        print('''
-</table>
-</body>
-</html>''', file=self.stream)
-
-
-def _render_file(source_dir, output_dir, ctx, no_highlight, entry, filter_):
-    global context
-    context = ctx
-    filename, remarks = entry
-    SourceFileRenderer(source_dir, output_dir, filename, no_highlight).render(remarks)
-
-
-def map_remarks(all_remarks):
-    # Set up a map between function names and their source location for
-    # function where inlining happened
-    for remark in optrecord.itervalues(all_remarks):
-        if isinstance(remark, optrecord.Passed) and remark.Pass == "inline" and remark.Name == "Inlined":
-            for arg in remark.Args:
-                arg_dict = dict(list(arg))
-                caller = arg_dict.get('Caller')
-                if caller:
-                    try:
-                        context.caller_loc[caller] = arg_dict['DebugLoc']
-                    except KeyError:
-                        pass
-
-
-def generate_report(all_remarks,
-                    file_remarks,
-                    source_dir,
-                    output_dir,
-                    no_highlight,
-                    should_display_hotness,
-                    max_hottest_remarks_on_index,
-                    num_jobs,
-                    should_print_progress):
-    try:
-        os.makedirs(output_dir)
-    except OSError as e:
-        if e.errno == errno.EEXIST and os.path.isdir(output_dir):
-            pass
-        else:
-            raise
-
-    if should_print_progress:
-        print('Rendering index page...')
-    if should_display_hotness:
-        sorted_remarks = sorted(optrecord.itervalues(all_remarks), key=lambda r: (r.Hotness, r.File, r.Line, r.Column, r.PassWithDiffPrefix, r.yaml_tag, r.Function), reverse=True)
-    else:
-        sorted_remarks = sorted(optrecord.itervalues(all_remarks), key=lambda r: (r.File, r.Line, r.Column, r.PassWithDiffPrefix, r.yaml_tag, r.Function))
-    IndexRenderer(output_dir, should_display_hotness, max_hottest_remarks_on_index).render(sorted_remarks)
-
-    shutil.copy(os.path.join(os.path.dirname(os.path.realpath(__file__)),
-            "style.css"), output_dir)
-
-    _render_file_bound = functools.partial(_render_file, source_dir, output_dir, context, no_highlight)
-    if should_print_progress:
-        print('Rendering HTML files...')
-    optpmap.pmap(_render_file_bound,
-                 file_remarks.items(),
-                 num_jobs,
-                 should_print_progress)
-
-
-def main():
-    parser = argparse.ArgumentParser(description=desc)
-    parser.add_argument(
-        'yaml_dirs_or_files',
-        nargs='+',
-        help='List of optimization record files or directories searched '
-             'for optimization record files.')
-    parser.add_argument(
-        '--output-dir',
-        '-o',
-        default='html',
-        help='Path to a directory where generated HTML files will be output. '
-             'If the directory does not already exist, it will be created. '
-             '"%(default)s" by default.')
-    parser.add_argument(
-        '--jobs',
-        '-j',
-        default=None,
-        type=int,
-        help='Max job count (defaults to %(default)s, the current CPU count)')
-    parser.add_argument(
-        '--source-dir',
-        '-s',
-        default='',
-        help='set source directory')
-    parser.add_argument(
-        '--no-progress-indicator',
-        '-n',
-        action='store_true',
-        default=False,
-        help='Do not display any indicator of how many YAML files were read '
-             'or rendered into HTML.')
-    parser.add_argument(
-        '--max-hottest-remarks-on-index',
-        default=1000,
-        type=int,
-        help='Maximum number of the hottest remarks to appear on the index page')
-    parser.add_argument(
-        '--no-highlight',
-        action='store_true',
-        default=False,
-        help='Do not use a syntax highlighter when rendering the source code')
-    parser.add_argument(
-        '--demangler',
-        help='Set the demangler to be used (defaults to %s)' % optrecord.Remark.default_demangler)
-
-    parser.add_argument(
-        '--filter',
-        default='',
-        help='Only display remarks from passes matching filter expression')
-
-    # Do not make this a global variable.  Values needed to be propagated through
-    # to individual classes and functions to be portable with multiprocessing across
-    # Windows and non-Windows.
-    args = parser.parse_args()
-
-    print_progress = not args.no_progress_indicator
-    if args.demangler:
-        optrecord.Remark.set_demangler(args.demangler)
-
-    files = optrecord.find_opt_files(*args.yaml_dirs_or_files)
-    if not files:
-        parser.error("No *.opt.yaml files found")
-        sys.exit(1)
-
-    all_remarks, file_remarks, should_display_hotness = \
-        optrecord.gather_results(files, args.jobs, print_progress, args.filter)
-
-    map_remarks(all_remarks)
-
-    generate_report(all_remarks,
-                    file_remarks,
-                    args.source_dir,
-                    args.output_dir,
-                    args.no_highlight,
-                    should_display_hotness,
-                    args.max_hottest_remarks_on_index,
-                    args.jobs,
-                    print_progress)
-
-if __name__ == '__main__':
-    main()

Added: llvm/trunk/tools/opt-viewer/optrecord.py
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/tools/opt-viewer/optrecord.py?rev=370095&view=auto
==============================================================================
--- llvm/trunk/tools/opt-viewer/optrecord.py (added)
+++ llvm/trunk/tools/opt-viewer/optrecord.py Tue Aug 27 11:31:29 2019
@@ -0,0 +1,345 @@
+#!/usr/bin/env python
+
+from __future__ import print_function
+
+import yaml
+# Try to use the C parser.
+try:
+    from yaml import CLoader as Loader
+except ImportError:
+    print("For faster parsing, you may want to install libYAML for PyYAML")
+    from yaml import Loader
+
+import cgi
+from collections import defaultdict
+import fnmatch
+import functools
+from multiprocessing import Lock
+import os, os.path
+import subprocess
+try:
+    # The previously builtin function `intern()` was moved
+    # to the `sys` module in Python 3.
+    from sys import intern
+except:
+    pass
+
+import re
+
+import optpmap
+
+try:
+    dict.iteritems
+except AttributeError:
+    # Python 3
+    def itervalues(d):
+        return iter(d.values())
+    def iteritems(d):
+        return iter(d.items())
+else:
+    # Python 2
+    def itervalues(d):
+        return d.itervalues()
+    def iteritems(d):
+        return d.iteritems()
+
+
+def html_file_name(filename):
+    return filename.replace('/', '_').replace('#', '_') + ".html"
+
+
+def make_link(File, Line):
+    return "\"{}#L{}\"".format(html_file_name(File), Line)
+
+
+class Remark(yaml.YAMLObject):
+    # Work-around for http://pyyaml.org/ticket/154.
+    yaml_loader = Loader
+
+    default_demangler = 'c++filt -n'
+    demangler_proc = None
+
+    @classmethod
+    def set_demangler(cls, demangler):
+        cls.demangler_proc = subprocess.Popen(demangler.split(), stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+        cls.demangler_lock = Lock()
+
+    @classmethod
+    def demangle(cls, name):
+        with cls.demangler_lock:
+            cls.demangler_proc.stdin.write((name + '\n').encode('utf-8'))
+            cls.demangler_proc.stdin.flush()
+            return cls.demangler_proc.stdout.readline().rstrip().decode('utf-8')
+
+    # Intern all strings since we have lot of duplication across filenames,
+    # remark text.
+    #
+    # Change Args from a list of dicts to a tuple of tuples.  This saves
+    # memory in two ways.  One, a small tuple is significantly smaller than a
+    # small dict.  Two, using tuple instead of list allows Args to be directly
+    # used as part of the key (in Python only immutable types are hashable).
+    def _reduce_memory(self):
+        self.Pass = intern(self.Pass)
+        self.Name = intern(self.Name)
+        try:
+            # Can't intern unicode strings.
+            self.Function = intern(self.Function)
+        except:
+            pass
+
+        def _reduce_memory_dict(old_dict):
+            new_dict = dict()
+            for (k, v) in iteritems(old_dict):
+                if type(k) is str:
+                    k = intern(k)
+
+                if type(v) is str:
+                    v = intern(v)
+                elif type(v) is dict:
+                    # This handles [{'Caller': ..., 'DebugLoc': { 'File': ... }}]
+                    v = _reduce_memory_dict(v)
+                new_dict[k] = v
+            return tuple(new_dict.items())
+
+        self.Args = tuple([_reduce_memory_dict(arg_dict) for arg_dict in self.Args])
+
+    # The inverse operation of the dictonary-related memory optimization in
+    # _reduce_memory_dict.  E.g.
+    #     (('DebugLoc', (('File', ...) ... ))) -> [{'DebugLoc': {'File': ...} ....}]
+    def recover_yaml_structure(self):
+        def tuple_to_dict(t):
+            d = dict()
+            for (k, v) in t:
+                if type(v) is tuple:
+                    v = tuple_to_dict(v)
+                d[k] = v
+            return d
+
+        self.Args = [tuple_to_dict(arg_tuple) for arg_tuple in self.Args]
+
+    def canonicalize(self):
+        if not hasattr(self, 'Hotness'):
+            self.Hotness = 0
+        if not hasattr(self, 'Args'):
+            self.Args = []
+        self._reduce_memory()
+
+    @property
+    def File(self):
+        return self.DebugLoc['File']
+
+    @property
+    def Line(self):
+        return int(self.DebugLoc['Line'])
+
+    @property
+    def Column(self):
+        return self.DebugLoc['Column']
+
+    @property
+    def DebugLocString(self):
+        return "{}:{}:{}".format(self.File, self.Line, self.Column)
+
+    @property
+    def DemangledFunctionName(self):
+        return self.demangle(self.Function)
+
+    @property
+    def Link(self):
+        return make_link(self.File, self.Line)
+
+    def getArgString(self, mapping):
+        mapping = dict(list(mapping))
+        dl = mapping.get('DebugLoc')
+        if dl:
+            del mapping['DebugLoc']
+
+        assert(len(mapping) == 1)
+        (key, value) = list(mapping.items())[0]
+
+        if key == 'Caller' or key == 'Callee' or key == 'DirectCallee':
+            value = cgi.escape(self.demangle(value))
+
+        if dl and key != 'Caller':
+            dl_dict = dict(list(dl))
+            return u"<a href={}>{}</a>".format(
+                make_link(dl_dict['File'], dl_dict['Line']), value)
+        else:
+            return value
+
+    # Return a cached dictionary for the arguments.  The key for each entry is
+    # the argument key (e.g. 'Callee' for inlining remarks.  The value is a
+    # list containing the value (e.g. for 'Callee' the function) and
+    # optionally a DebugLoc.
+    def getArgDict(self):
+        if hasattr(self, 'ArgDict'):
+            return self.ArgDict
+        self.ArgDict = {}
+        for arg in self.Args:
+            if len(arg) == 2:
+                if arg[0][0] == 'DebugLoc':
+                    dbgidx = 0
+                else:
+                    assert(arg[1][0] == 'DebugLoc')
+                    dbgidx = 1
+
+                key = arg[1 - dbgidx][0]
+                entry = (arg[1 - dbgidx][1], arg[dbgidx][1])
+            else:
+                arg = arg[0]
+                key = arg[0]
+                entry = (arg[1], )
+
+            self.ArgDict[key] = entry
+        return self.ArgDict
+
+    def getDiffPrefix(self):
+        if hasattr(self, 'Added'):
+            if self.Added:
+                return '+'
+            else:
+                return '-'
+        return ''
+
+    @property
+    def PassWithDiffPrefix(self):
+        return self.getDiffPrefix() + self.Pass
+
+    @property
+    def message(self):
+        # Args is a list of mappings (dictionaries)
+        values = [self.getArgString(mapping) for mapping in self.Args]
+        return "".join(values)
+
+    @property
+    def RelativeHotness(self):
+        if self.max_hotness:
+            return "{0:.2f}%".format(self.Hotness * 100. / self.max_hotness)
+        else:
+            return ''
+
+    @property
+    def key(self):
+        return (self.__class__, self.PassWithDiffPrefix, self.Name, self.File,
+                self.Line, self.Column, self.Function, self.Args)
+
+    def __hash__(self):
+        return hash(self.key)
+
+    def __eq__(self, other):
+        return self.key == other.key
+
+    def __repr__(self):
+        return str(self.key)
+
+
+class Analysis(Remark):
+    yaml_tag = '!Analysis'
+
+    @property
+    def color(self):
+        return "white"
+
+
+class AnalysisFPCommute(Analysis):
+    yaml_tag = '!AnalysisFPCommute'
+
+
+class AnalysisAliasing(Analysis):
+    yaml_tag = '!AnalysisAliasing'
+
+
+class Passed(Remark):
+    yaml_tag = '!Passed'
+
+    @property
+    def color(self):
+        return "green"
+
+
+class Missed(Remark):
+    yaml_tag = '!Missed'
+
+    @property
+    def color(self):
+        return "red"
+
+class Failure(Missed):
+    yaml_tag = '!Failure'
+
+def get_remarks(input_file, filter_=None):
+    max_hotness = 0
+    all_remarks = dict()
+    file_remarks = defaultdict(functools.partial(defaultdict, list))
+
+    with open(input_file) as f:
+        docs = yaml.load_all(f, Loader=Loader)
+
+        filter_e = None
+        if filter_:
+            filter_e = re.compile(filter_)
+        for remark in docs:
+            remark.canonicalize()
+            # Avoid remarks withoug debug location or if they are duplicated
+            if not hasattr(remark, 'DebugLoc') or remark.key in all_remarks:
+                continue
+
+            if filter_e and not filter_e.search(remark.Pass):
+                continue
+
+            all_remarks[remark.key] = remark
+
+            file_remarks[remark.File][remark.Line].append(remark)
+
+            # If we're reading a back a diff yaml file, max_hotness is already
+            # captured which may actually be less than the max hotness found
+            # in the file.
+            if hasattr(remark, 'max_hotness'):
+                max_hotness = remark.max_hotness
+            max_hotness = max(max_hotness, remark.Hotness)
+
+    return max_hotness, all_remarks, file_remarks
+
+
+def gather_results(filenames, num_jobs, should_print_progress, filter_=None):
+    if should_print_progress:
+        print('Reading YAML files...')
+    if not Remark.demangler_proc:
+        Remark.set_demangler(Remark.default_demangler)
+    remarks = optpmap.pmap(
+        get_remarks, filenames, num_jobs, should_print_progress, filter_)
+    max_hotness = max(entry[0] for entry in remarks)
+
+    def merge_file_remarks(file_remarks_job, all_remarks, merged):
+        for filename, d in iteritems(file_remarks_job):
+            for line, remarks in iteritems(d):
+                for remark in remarks:
+                    # Bring max_hotness into the remarks so that
+                    # RelativeHotness does not depend on an external global.
+                    remark.max_hotness = max_hotness
+                    if remark.key not in all_remarks:
+                        merged[filename][line].append(remark)
+
+    all_remarks = dict()
+    file_remarks = defaultdict(functools.partial(defaultdict, list))
+    for _, all_remarks_job, file_remarks_job in remarks:
+        merge_file_remarks(file_remarks_job, all_remarks, file_remarks)
+        all_remarks.update(all_remarks_job)
+
+    return all_remarks, file_remarks, max_hotness != 0
+
+
+def find_opt_files(*dirs_or_files):
+    all = []
+    for dir_or_file in dirs_or_files:
+        if os.path.isfile(dir_or_file):
+            all.append(dir_or_file)
+        else:
+            for dir, subdirs, files in os.walk(dir_or_file):
+                # Exclude mounted directories and symlinks (os.walk default).
+                subdirs[:] = [d for d in subdirs
+                              if not os.path.ismount(os.path.join(dir, d))]
+                for file in files:
+                    if fnmatch.fnmatch(file, "*.opt.yaml*"):
+                        all.append(os.path.join(dir, file))
+    return all

Removed: llvm/trunk/tools/opt-viewer/optrecord.py.in
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/tools/opt-viewer/optrecord.py.in?rev=370094&view=auto
==============================================================================
--- llvm/trunk/tools/opt-viewer/optrecord.py.in (original)
+++ llvm/trunk/tools/opt-viewer/optrecord.py.in (removed)
@@ -1,345 +0,0 @@
-#!/usr/bin/env @PYTHON_BASENAME@
-
-from __future__ import print_function
-
-import yaml
-# Try to use the C parser.
-try:
-    from yaml import CLoader as Loader
-except ImportError:
-    print("For faster parsing, you may want to install libYAML for PyYAML")
-    from yaml import Loader
-
-import cgi
-from collections import defaultdict
-import fnmatch
-import functools
-from multiprocessing import Lock
-import os, os.path
-import subprocess
-try:
-    # The previously builtin function `intern()` was moved
-    # to the `sys` module in Python 3.
-    from sys import intern
-except:
-    pass
-
-import re
-
-import optpmap
-
-try:
-    dict.iteritems
-except AttributeError:
-    # Python 3
-    def itervalues(d):
-        return iter(d.values())
-    def iteritems(d):
-        return iter(d.items())
-else:
-    # Python 2
-    def itervalues(d):
-        return d.itervalues()
-    def iteritems(d):
-        return d.iteritems()
-
-
-def html_file_name(filename):
-    return filename.replace('/', '_').replace('#', '_') + ".html"
-
-
-def make_link(File, Line):
-    return "\"{}#L{}\"".format(html_file_name(File), Line)
-
-
-class Remark(yaml.YAMLObject):
-    # Work-around for http://pyyaml.org/ticket/154.
-    yaml_loader = Loader
-
-    default_demangler = 'c++filt -n'
-    demangler_proc = None
-
-    @classmethod
-    def set_demangler(cls, demangler):
-        cls.demangler_proc = subprocess.Popen(demangler.split(), stdin=subprocess.PIPE, stdout=subprocess.PIPE)
-        cls.demangler_lock = Lock()
-
-    @classmethod
-    def demangle(cls, name):
-        with cls.demangler_lock:
-            cls.demangler_proc.stdin.write((name + '\n').encode('utf-8'))
-            cls.demangler_proc.stdin.flush()
-            return cls.demangler_proc.stdout.readline().rstrip().decode('utf-8')
-
-    # Intern all strings since we have lot of duplication across filenames,
-    # remark text.
-    #
-    # Change Args from a list of dicts to a tuple of tuples.  This saves
-    # memory in two ways.  One, a small tuple is significantly smaller than a
-    # small dict.  Two, using tuple instead of list allows Args to be directly
-    # used as part of the key (in Python only immutable types are hashable).
-    def _reduce_memory(self):
-        self.Pass = intern(self.Pass)
-        self.Name = intern(self.Name)
-        try:
-            # Can't intern unicode strings.
-            self.Function = intern(self.Function)
-        except:
-            pass
-
-        def _reduce_memory_dict(old_dict):
-            new_dict = dict()
-            for (k, v) in iteritems(old_dict):
-                if type(k) is str:
-                    k = intern(k)
-
-                if type(v) is str:
-                    v = intern(v)
-                elif type(v) is dict:
-                    # This handles [{'Caller': ..., 'DebugLoc': { 'File': ... }}]
-                    v = _reduce_memory_dict(v)
-                new_dict[k] = v
-            return tuple(new_dict.items())
-
-        self.Args = tuple([_reduce_memory_dict(arg_dict) for arg_dict in self.Args])
-
-    # The inverse operation of the dictonary-related memory optimization in
-    # _reduce_memory_dict.  E.g.
-    #     (('DebugLoc', (('File', ...) ... ))) -> [{'DebugLoc': {'File': ...} ....}]
-    def recover_yaml_structure(self):
-        def tuple_to_dict(t):
-            d = dict()
-            for (k, v) in t:
-                if type(v) is tuple:
-                    v = tuple_to_dict(v)
-                d[k] = v
-            return d
-
-        self.Args = [tuple_to_dict(arg_tuple) for arg_tuple in self.Args]
-
-    def canonicalize(self):
-        if not hasattr(self, 'Hotness'):
-            self.Hotness = 0
-        if not hasattr(self, 'Args'):
-            self.Args = []
-        self._reduce_memory()
-
-    @property
-    def File(self):
-        return self.DebugLoc['File']
-
-    @property
-    def Line(self):
-        return int(self.DebugLoc['Line'])
-
-    @property
-    def Column(self):
-        return self.DebugLoc['Column']
-
-    @property
-    def DebugLocString(self):
-        return "{}:{}:{}".format(self.File, self.Line, self.Column)
-
-    @property
-    def DemangledFunctionName(self):
-        return self.demangle(self.Function)
-
-    @property
-    def Link(self):
-        return make_link(self.File, self.Line)
-
-    def getArgString(self, mapping):
-        mapping = dict(list(mapping))
-        dl = mapping.get('DebugLoc')
-        if dl:
-            del mapping['DebugLoc']
-
-        assert(len(mapping) == 1)
-        (key, value) = list(mapping.items())[0]
-
-        if key == 'Caller' or key == 'Callee' or key == 'DirectCallee':
-            value = cgi.escape(self.demangle(value))
-
-        if dl and key != 'Caller':
-            dl_dict = dict(list(dl))
-            return u"<a href={}>{}</a>".format(
-                make_link(dl_dict['File'], dl_dict['Line']), value)
-        else:
-            return value
-
-    # Return a cached dictionary for the arguments.  The key for each entry is
-    # the argument key (e.g. 'Callee' for inlining remarks.  The value is a
-    # list containing the value (e.g. for 'Callee' the function) and
-    # optionally a DebugLoc.
-    def getArgDict(self):
-        if hasattr(self, 'ArgDict'):
-            return self.ArgDict
-        self.ArgDict = {}
-        for arg in self.Args:
-            if len(arg) == 2:
-                if arg[0][0] == 'DebugLoc':
-                    dbgidx = 0
-                else:
-                    assert(arg[1][0] == 'DebugLoc')
-                    dbgidx = 1
-
-                key = arg[1 - dbgidx][0]
-                entry = (arg[1 - dbgidx][1], arg[dbgidx][1])
-            else:
-                arg = arg[0]
-                key = arg[0]
-                entry = (arg[1], )
-
-            self.ArgDict[key] = entry
-        return self.ArgDict
-
-    def getDiffPrefix(self):
-        if hasattr(self, 'Added'):
-            if self.Added:
-                return '+'
-            else:
-                return '-'
-        return ''
-
-    @property
-    def PassWithDiffPrefix(self):
-        return self.getDiffPrefix() + self.Pass
-
-    @property
-    def message(self):
-        # Args is a list of mappings (dictionaries)
-        values = [self.getArgString(mapping) for mapping in self.Args]
-        return "".join(values)
-
-    @property
-    def RelativeHotness(self):
-        if self.max_hotness:
-            return "{0:.2f}%".format(self.Hotness * 100. / self.max_hotness)
-        else:
-            return ''
-
-    @property
-    def key(self):
-        return (self.__class__, self.PassWithDiffPrefix, self.Name, self.File,
-                self.Line, self.Column, self.Function, self.Args)
-
-    def __hash__(self):
-        return hash(self.key)
-
-    def __eq__(self, other):
-        return self.key == other.key
-
-    def __repr__(self):
-        return str(self.key)
-
-
-class Analysis(Remark):
-    yaml_tag = '!Analysis'
-
-    @property
-    def color(self):
-        return "white"
-
-
-class AnalysisFPCommute(Analysis):
-    yaml_tag = '!AnalysisFPCommute'
-
-
-class AnalysisAliasing(Analysis):
-    yaml_tag = '!AnalysisAliasing'
-
-
-class Passed(Remark):
-    yaml_tag = '!Passed'
-
-    @property
-    def color(self):
-        return "green"
-
-
-class Missed(Remark):
-    yaml_tag = '!Missed'
-
-    @property
-    def color(self):
-        return "red"
-
-class Failure(Missed):
-    yaml_tag = '!Failure'
-
-def get_remarks(input_file, filter_=None):
-    max_hotness = 0
-    all_remarks = dict()
-    file_remarks = defaultdict(functools.partial(defaultdict, list))
-
-    with open(input_file) as f:
-        docs = yaml.load_all(f, Loader=Loader)
-
-        filter_e = None
-        if filter_:
-            filter_e = re.compile(filter_)
-        for remark in docs:
-            remark.canonicalize()
-            # Avoid remarks withoug debug location or if they are duplicated
-            if not hasattr(remark, 'DebugLoc') or remark.key in all_remarks:
-                continue
-
-            if filter_e and not filter_e.search(remark.Pass):
-                continue
-
-            all_remarks[remark.key] = remark
-
-            file_remarks[remark.File][remark.Line].append(remark)
-
-            # If we're reading a back a diff yaml file, max_hotness is already
-            # captured which may actually be less than the max hotness found
-            # in the file.
-            if hasattr(remark, 'max_hotness'):
-                max_hotness = remark.max_hotness
-            max_hotness = max(max_hotness, remark.Hotness)
-
-    return max_hotness, all_remarks, file_remarks
-
-
-def gather_results(filenames, num_jobs, should_print_progress, filter_=None):
-    if should_print_progress:
-        print('Reading YAML files...')
-    if not Remark.demangler_proc:
-        Remark.set_demangler(Remark.default_demangler)
-    remarks = optpmap.pmap(
-        get_remarks, filenames, num_jobs, should_print_progress, filter_)
-    max_hotness = max(entry[0] for entry in remarks)
-
-    def merge_file_remarks(file_remarks_job, all_remarks, merged):
-        for filename, d in iteritems(file_remarks_job):
-            for line, remarks in iteritems(d):
-                for remark in remarks:
-                    # Bring max_hotness into the remarks so that
-                    # RelativeHotness does not depend on an external global.
-                    remark.max_hotness = max_hotness
-                    if remark.key not in all_remarks:
-                        merged[filename][line].append(remark)
-
-    all_remarks = dict()
-    file_remarks = defaultdict(functools.partial(defaultdict, list))
-    for _, all_remarks_job, file_remarks_job in remarks:
-        merge_file_remarks(file_remarks_job, all_remarks, file_remarks)
-        all_remarks.update(all_remarks_job)
-
-    return all_remarks, file_remarks, max_hotness != 0
-
-
-def find_opt_files(*dirs_or_files):
-    all = []
-    for dir_or_file in dirs_or_files:
-        if os.path.isfile(dir_or_file):
-            all.append(dir_or_file)
-        else:
-            for dir, subdirs, files in os.walk(dir_or_file):
-                # Exclude mounted directories and symlinks (os.walk default).
-                subdirs[:] = [d for d in subdirs
-                              if not os.path.ismount(os.path.join(dir, d))]
-                for file in files:
-                    if fnmatch.fnmatch(file, "*.opt.yaml*"):
-                        all.append(os.path.join(dir, file))
-    return all




More information about the llvm-commits mailing list