[LNT] r311754 - Move some utilities around; NFC
Matthias Braun via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 24 19:32:51 PDT 2017
Author: matze
Date: Thu Aug 24 19:32:51 2017
New Revision: 311754
URL: http://llvm.org/viewvc/llvm-project?rev=311754&view=rev
Log:
Move some utilities around; NFC
Reduce dependence on lnt.server.ui.util for code outside of
lnt.server.ui.
Modified:
lnt/trunk/lnt/server/reporting/analysis.py
lnt/trunk/lnt/server/reporting/dailyreport.py
lnt/trunk/lnt/server/ui/util.py
lnt/trunk/lnt/server/ui/views.py
lnt/trunk/lnt/util/stats.py
Modified: lnt/trunk/lnt/server/reporting/analysis.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/reporting/analysis.py?rev=311754&r1=311753&r2=311754&view=diff
==============================================================================
--- lnt/trunk/lnt/server/reporting/analysis.py (original)
+++ lnt/trunk/lnt/server/reporting/analysis.py Thu Aug 24 19:32:51 2017
@@ -1,10 +1,11 @@
"""
Utilities for helping with the analysis of data, for reporting purposes.
"""
+from lnt.testing import FAIL
from lnt.util import logger
+from lnt.util import multidict
+from lnt.util import stats
from lnt.util import stats
-from lnt.server.ui import util
-from lnt.testing import FAIL
REGRESSED = 'REGRESSED'
IMPROVED = 'IMPROVED'
@@ -46,7 +47,7 @@ def calc_geomean(run_values):
if not values:
return None
- return util.geometric_mean(values) - MIN_VALUE_PRECISION
+ return stats.geometric_mean(values) - MIN_VALUE_PRECISION
class ComparisonResult:
@@ -252,7 +253,7 @@ class RunInfo(object):
self.aggregation_fn = aggregation_fn
self.confidence_lv = confidence_lv
- self.sample_map = util.multidict()
+ self.sample_map = multidict.multidict()
self.profile_map = dict()
self.loaded_run_ids = set()
Modified: lnt/trunk/lnt/server/reporting/dailyreport.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/reporting/dailyreport.py?rev=311754&r1=311753&r2=311754&view=diff
==============================================================================
--- lnt/trunk/lnt/server/reporting/dailyreport.py (original)
+++ lnt/trunk/lnt/server/reporting/dailyreport.py Thu Aug 24 19:32:51 2017
@@ -1,19 +1,64 @@
+from collections import namedtuple
+from lnt.server.reporting.analysis import REGRESSED, UNCHANGED_FAIL
+from lnt.util import multidict
+import colorsys
import datetime
+import lnt.server.reporting.analysis
+import lnt.server.ui.app
import re
-import urllib
-
import sqlalchemy.sql
+import urllib
-import lnt.server.reporting.analysis
-import lnt.server.ui.app
+OrderAndHistory = namedtuple('OrderAndHistory', ['max_order', 'recent_orders'])
-from lnt.server.reporting.analysis import REGRESSED, UNCHANGED_FAIL
-from lnt.server.ui import util
+def _pairs(list):
+ return zip(list[:-1], list[1:])
-from collections import namedtuple
-OrderAndHistory = namedtuple('OrderAndHistory', ['max_order', 'recent_orders'])
+# The hash color palette avoids green and red as these colours are already used
+# in quite a few places to indicate "good" or "bad".
+_hash_color_palette = (
+ colorsys.hsv_to_rgb(h=45. / 360, s=0.3, v=0.9999), # warm yellow
+ colorsys.hsv_to_rgb(h=210. / 360, s=0.3, v=0.9999), # blue cyan
+ colorsys.hsv_to_rgb(h=300. / 360, s=0.3, v=0.9999), # mid magenta
+ colorsys.hsv_to_rgb(h=150. / 360, s=0.3, v=0.9999), # green cyan
+ colorsys.hsv_to_rgb(h=225. / 360, s=0.3, v=0.9999), # cool blue
+ colorsys.hsv_to_rgb(h=180. / 360, s=0.3, v=0.9999), # mid cyan
+)
+
+
+def _clamp(v, minVal, maxVal):
+ return min(max(v, minVal), maxVal)
+
+
+def _toColorString(col):
+ r, g, b = [_clamp(int(v * 255), 0, 255)
+ for v in col]
+ return "#%02x%02x%02x" % (r, g, b)
+
+
+def _get_rgb_colors_for_hashes(hash_strings):
+ hash2color = {}
+ unique_hash_counter = 0
+ for hash_string in hash_strings:
+ if hash_string is not None:
+ if hash_string in hash2color:
+ continue
+ hash2color[hash_string] = _hash_color_palette[unique_hash_counter]
+ unique_hash_counter += 1
+ if unique_hash_counter >= len(_hash_color_palette):
+ break
+ result = []
+ for hash_string in hash_strings:
+ if hash_string is None:
+ result.append(None)
+ else:
+ # If not one of the first N hashes, return rgb value 0,0,0 which is
+ # white.
+ rgb = hash2color.get(hash_string, (0.999, 0.999, 0.999))
+ result.append(_toColorString(rgb))
+ return result
# Helper classes to make the sparkline chart construction easier in the jinja
@@ -70,7 +115,7 @@ class DayResults:
hashes.append(None)
else:
hashes.append(dr.hash)
- rgb_colors = util.get_rgb_colors_for_hashes(hashes)
+ rgb_colors = _get_rgb_colors_for_hashes(hashes)
for i, dr in enumerate(self.day_results):
if dr is not None:
dr.hash_rgb_color = rgb_colors[i]
@@ -167,7 +212,7 @@ class DailyReport(object):
prior_runs = [ts.query(ts.Run).
filter(ts.Run.start_time > prior_day).
filter(ts.Run.start_time <= day).all()
- for day, prior_day in util.pairs(self.prior_days)]
+ for day, prior_day in _pairs(self.prior_days)]
if self.filter_machine_re is not None:
prior_runs = [[run for run in runs
@@ -189,7 +234,7 @@ class DailyReport(object):
historic_runs = [None] * len(prior_runs)
for i, runs in enumerate(prior_runs):
# Aggregate the runs by machine.
- machine_to_all_orders = util.multidict()
+ machine_to_all_orders = multidict.multidict()
for r in runs:
machine_to_all_orders[r.machine] = r.order
@@ -243,13 +288,13 @@ class DailyReport(object):
# overview across machines.
# Aggregate runs by machine ID and day index.
- self.machine_runs = machine_runs = util.multidict()
+ self.machine_runs = machine_runs = multidict.multidict()
for day_index, day_runs in enumerate(prior_runs):
for run in day_runs:
machine_runs[(run.machine_id, day_index)] = run
# Also aggregate past runs by day.
- self.machine_past_runs = util.multidict()
+ self.machine_past_runs = multidict.multidict()
for day_index, day_runs in enumerate(historic_runs):
for run in day_runs:
self.machine_past_runs[(run.machine_id, day_index)] = run
Modified: lnt/trunk/lnt/server/ui/util.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/ui/util.py?rev=311754&r1=311753&r2=311754&view=diff
==============================================================================
--- lnt/trunk/lnt/server/ui/util.py (original)
+++ lnt/trunk/lnt/server/ui/util.py Thu Aug 24 19:32:51 2017
@@ -11,10 +11,6 @@ def toColorString(col):
return "#%02x%02x%02x" % (r, g, b)
-def pairs(list):
- return zip(list[:-1], list[1:])
-
-
def safediv(a, b, default=None):
try:
return a / b
@@ -53,75 +49,6 @@ def makeBetterColor(h):
return colorsys.hsv_to_rgb(h, s, v)
-# The hash color palette avoids green and red as these colours are already used
-# in quite a few places to indicate "good" or "bad".
-hash_color_palette = (
- colorsys.hsv_to_rgb(h=45. / 360, s=0.3, v=0.9999), # warm yellow
- colorsys.hsv_to_rgb(h=210. / 360, s=0.3, v=0.9999), # blue cyan
- colorsys.hsv_to_rgb(h=300. / 360, s=0.3, v=0.9999), # mid magenta
- colorsys.hsv_to_rgb(h=150. / 360, s=0.3, v=0.9999), # green cyan
- colorsys.hsv_to_rgb(h=225. / 360, s=0.3, v=0.9999), # cool blue
- colorsys.hsv_to_rgb(h=180. / 360, s=0.3, v=0.9999), # mid cyan
-)
-
-
-def get_rgb_colors_for_hashes(hash_strings):
- hash2color = {}
- unique_hash_counter = 0
- for hash_string in hash_strings:
- if hash_string is not None:
- if hash_string in hash2color:
- continue
- hash2color[hash_string] = hash_color_palette[unique_hash_counter]
- unique_hash_counter += 1
- if unique_hash_counter >= len(hash_color_palette):
- break
- result = []
- for hash_string in hash_strings:
- if hash_string is None:
- result.append(None)
- else:
- # If not one of the first N hashes, return rgb value 0,0,0 which is
- # white.
- rgb = hash2color.get(hash_string, (0.999, 0.999, 0.999))
- result.append(toColorString(rgb))
- return result
-
-
-class multidict:
- def __init__(self, elts=()):
- self.data = {}
- for key, value in elts:
- self[key] = value
-
- def __contains__(self, item):
- return item in self.data
-
- def __getitem__(self, item):
- return self.data[item]
-
- def __setitem__(self, key, value):
- if key in self.data:
- self.data[key].append(value)
- else:
- self.data[key] = [value]
-
- def items(self):
- return self.data.items()
-
- def values(self):
- return self.data.values()
-
- def keys(self):
- return self.data.keys()
-
- def __len__(self):
- return len(self.data)
-
- def get(self, key, default=None):
- return self.data.get(key, default)
-
-
def any_true(list, predicate):
for i in list:
if predicate(i):
@@ -141,11 +68,6 @@ def all_false(list, predicate):
return not any_true(list, predicate)
-def geometric_mean(l):
- iPow = 1. / len(l)
- return reduce(lambda a, b: a * b, [v ** iPow for v in l])
-
-
def mean(l):
return sum(l) / len(l)
Modified: lnt/trunk/lnt/server/ui/views.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/ui/views.py?rev=311754&r1=311753&r2=311754&view=diff
==============================================================================
--- lnt/trunk/lnt/server/ui/views.py (original)
+++ lnt/trunk/lnt/server/ui/views.py Thu Aug 24 19:32:51 2017
@@ -41,6 +41,7 @@ from lnt.server.ui.util import FLASH_DAN
from lnt.server.ui.util import mean
from lnt.util import async_ops
from lnt.util import logger
+from lnt.util import multidict
from lnt.server.ui.util import baseline_key, convert_revision
@@ -269,7 +270,7 @@ def v4_machine(id):
# Gather all the runs on this machine.
ts = request.get_testsuite()
- associated_runs = util.multidict(
+ associated_runs = multidict.multidict(
(run_order, r)
for r, run_order in (ts.query(ts.Run, ts.Order)
.join(ts.Order)
@@ -845,8 +846,8 @@ def v4_graph():
(field.status_field.column.is_(None)))
# Aggregate by revision.
- data = util.multidict((rev, (val, date, run_id))
- for val, rev, date, run_id in q).items()
+ data = multidict.multidict((rev, (val, date, run_id))
+ for val, rev, date, run_id in q).items()
data.sort(key=lambda sample: convert_revision(sample[0]))
graph_datum.append((test.name, data, col, field, url))
@@ -905,8 +906,8 @@ def v4_graph():
.group_by(ts.Order.llvm_project_revision, ts.Test)
# Calculate geomean of each revision.
- data = util.multidict(((rev, date), val) for val, rev, date in q) \
- .items()
+ data = multidict.multidict(((rev, date), val) for val, rev, date in q) \
+ .items()
data = [(rev,
[(lnt.server.reporting.analysis.calc_geomean(vals), date)])
for ((rev, date), vals) in data]
@@ -1186,7 +1187,7 @@ def v4_global_status():
recent_runs = ts.query(ts.Run).filter(ts.Run.start_time > yesterday).all()
# Aggregate the runs by machine.
- recent_runs_by_machine = util.multidict()
+ recent_runs_by_machine = multidict.multidict()
for run in recent_runs:
recent_runs_by_machine[run.machine] = run
Modified: lnt/trunk/lnt/util/stats.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/util/stats.py?rev=311754&r1=311753&r2=311754&view=diff
==============================================================================
--- lnt/trunk/lnt/util/stats.py (original)
+++ lnt/trunk/lnt/util/stats.py Thu Aug 24 19:32:51 2017
@@ -28,6 +28,11 @@ def mean(l):
return None
+def geometric_mean(l):
+ iPow = 1. / len(l)
+ return reduce(lambda a, b: a * b, [v ** iPow for v in l])
+
+
def agg_mean(pairs):
"""Aggregation function in views.py receives input via enumerate and
produces a tuple.
More information about the llvm-commits
mailing list