[llvm] r308538 - [opt-viewer] Reduce memory consumption by another 20-25%
Adam Nemet via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 19 15:04:59 PDT 2017
Author: anemet
Date: Wed Jul 19 15:04:59 2017
New Revision: 308538
URL: http://llvm.org/viewvc/llvm-project?rev=308538&view=rev
Log:
[opt-viewer] Reduce memory consumption by another 20-25%
The Args field of the remark which consists of a list of mappings in YAML is
translated into a list of (small) dicts on Python. An empty dict is 280 bytes
on my system so we can save memory by using a tuple of tuples instead.
Making a tuple of tuples rather than a list of tuples allows Args to be shared
with the key of the remark. This is actually an even greater saving. (Keys
are alive throughout the entire run in all_remarks.)
Here are a few opt-stats runs with different input sizes while measuring heap
usage with heapy. Avg remark size is simply estimated as
heap-size / # of remarks:
| # of files | 60 | 114 | 308 | 605 | 1370 |
| # of remarks | 20K | 37K | 146K | 180K | 640K |
| total file size (MB) | 22 | 51 | 219 | 202 | 1034 |
|------------------------+------+------+------+------+------|
| Avg remark size before | 4339 | 4792 | 4761 | 4096 | 4607 |
| Avg remark size after | 3446 | 3641 | 3567 | 3146 | 3347 |
| Rate | 0.79 | 0.76 | 0.75 | 0.77 | 0.73 |
Differential Revision: https://reviews.llvm.org/D35611
Modified:
llvm/trunk/tools/opt-viewer/opt-diff.py
llvm/trunk/tools/opt-viewer/opt-viewer.py
llvm/trunk/tools/opt-viewer/optrecord.py
Modified: llvm/trunk/tools/opt-viewer/opt-diff.py
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/tools/opt-viewer/opt-diff.py?rev=308538&r1=308537&r2=308538&view=diff
==============================================================================
--- llvm/trunk/tools/opt-viewer/opt-diff.py (original)
+++ llvm/trunk/tools/opt-viewer/opt-diff.py Wed Jul 19 15:04:59 2017
@@ -60,5 +60,10 @@ if __name__ == '__main__':
r.Added = True
for r in removed:
r.Added = False
+
+ result = added | removed
+ for r in result:
+ r.recover_yaml_structure()
+
with open(args.output, 'w') as stream:
- yaml.dump_all(added | removed, stream)
+ yaml.dump_all(result, stream)
Modified: llvm/trunk/tools/opt-viewer/opt-viewer.py
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/tools/opt-viewer/opt-viewer.py?rev=308538&r1=308537&r2=308538&view=diff
==============================================================================
--- llvm/trunk/tools/opt-viewer/opt-viewer.py (original)
+++ llvm/trunk/tools/opt-viewer/opt-viewer.py Wed Jul 19 15:04:59 2017
@@ -82,7 +82,8 @@ class SourceFileRenderer:
inlining_context = r.DemangledFunctionName
dl = context.caller_loc.get(r.Function)
if dl:
- link = optrecord.make_link(dl['File'], dl['Line'] - 2)
+ dl_dict = dict(list(dl))
+ link = optrecord.make_link(dl_dict['File'], dl_dict['Line'] - 2)
inlining_context = "<a href={link}>{r.DemangledFunctionName}</a>".format(**locals())
# Column is the number of characters *including* tabs, keep those and
@@ -176,10 +177,11 @@ def map_remarks(all_remarks):
for remark in optrecord.itervalues(all_remarks):
if isinstance(remark, optrecord.Passed) and remark.Pass == "inline" and remark.Name == "Inlined":
for arg in remark.Args:
- caller = arg.get('Caller')
+ arg_dict = dict(list(arg))
+ caller = arg_dict.get('Caller')
if caller:
try:
- context.caller_loc[caller] = arg['DebugLoc']
+ context.caller_loc[caller] = arg_dict['DebugLoc']
except KeyError:
pass
Modified: llvm/trunk/tools/opt-viewer/optrecord.py
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/tools/opt-viewer/optrecord.py?rev=308538&r1=308537&r2=308538&view=diff
==============================================================================
--- llvm/trunk/tools/opt-viewer/optrecord.py (original)
+++ llvm/trunk/tools/opt-viewer/optrecord.py Wed Jul 19 15:04:59 2017
@@ -60,14 +60,19 @@ class Remark(yaml.YAMLObject):
# Work-around for http://pyyaml.org/ticket/154.
yaml_loader = Loader
- def _intern_strings(self):
+ # Intern all strings since we have lot of duplication across filenames,
+ # remark text.
+ #
+ # Change Args from a list of dicts to a tuple of tuples. This saves
+ # memory in two ways. One, a small tuple is significantly smaller than a
+ # small dict. Two, using tuple instead of list allows Args to be directly
+ # used as part of the key (in Python only immutable types are hashable).
+ def _reduce_memory(self):
self.Pass = intern(self.Pass)
self.Name = intern(self.Name)
self.Function = intern(self.Function)
- # Intern key and value if string and recurse if value is a dictionary.
- # This handles [{'Caller': ..., 'DebugLoc': { 'File': ... }}]
- def _intern_dict(old_dict):
+ def _reduce_memory_dict(old_dict):
new_dict = dict()
for (k, v) in old_dict.iteritems():
if type(k) is str:
@@ -76,18 +81,33 @@ class Remark(yaml.YAMLObject):
if type(v) is str:
v = intern(v)
elif type(v) is dict:
- v = _intern_dict(v)
+ # This handles [{'Caller': ..., 'DebugLoc': { 'File': ... }}]
+ v = _reduce_memory_dict(v)
new_dict[k] = v
- return new_dict
+ return tuple(new_dict.items())
- self.Args = [_intern_dict(arg_dict) for arg_dict in self.Args]
+ self.Args = tuple([_reduce_memory_dict(arg_dict) for arg_dict in self.Args])
+
+ # The inverse operation of the dictonary-related memory optimization in
+ # _reduce_memory_dict. E.g.
+ # (('DebugLoc', (('File', ...) ... ))) -> [{'DebugLoc': {'File': ...} ....}]
+ def recover_yaml_structure(self):
+ def tuple_to_dict(t):
+ d = dict()
+ for (k, v) in t:
+ if type(v) is tuple:
+ v = tuple_to_dict(v)
+ d[k] = v
+ return d
+
+ self.Args = [tuple_to_dict(arg_tuple) for arg_tuple in self.Args]
def canonicalize(self):
if not hasattr(self, 'Hotness'):
self.Hotness = 0
if not hasattr(self, 'Args'):
self.Args = []
- self._intern_strings()
+ self._reduce_memory()
@property
def File(self):
@@ -114,7 +134,7 @@ class Remark(yaml.YAMLObject):
return make_link(self.File, self.Line)
def getArgString(self, mapping):
- mapping = mapping.copy()
+ mapping = dict(list(mapping))
dl = mapping.get('DebugLoc')
if dl:
del mapping['DebugLoc']
@@ -126,8 +146,9 @@ class Remark(yaml.YAMLObject):
value = cgi.escape(demangle(value))
if dl and key != 'Caller':
+ dl_dict = dict(list(dl))
return "<a href={}>{}</a>".format(
- make_link(dl['File'], dl['Line']), value)
+ make_link(dl_dict['File'], dl_dict['Line']), value)
else:
return value
@@ -158,13 +179,8 @@ class Remark(yaml.YAMLObject):
@property
def key(self):
- k = (self.__class__, self.PassWithDiffPrefix, self.Name, self.File, self.Line, self.Column, self.Function)
- for arg in self.Args:
- for (key, value) in iteritems(arg):
- if type(value) is dict:
- value = tuple(value.items())
- k += (key, value)
- return k
+ return (self.__class__, self.PassWithDiffPrefix, self.Name, self.File,
+ self.Line, self.Column, self.Function, self.Args)
def __hash__(self):
return hash(self.key)
More information about the llvm-commits
mailing list