[LNT] r207898 - Use Mann-Whitney U test to identify changes
Yi Kong
Yi.Kong at arm.com
Sat May 3 06:10:41 PDT 2014
Because virtualenv isolates the program from system, so the package from Ubuntu repo won't work. You need to run the installation script again to get SciPy as dependency.
________________________________________
From: Renato Golin [renato.golin at linaro.org]
Sent: 03 May 2014 11:47
To: Yi Kong
Cc: LLVM Commits; Daniel Dunbar; Chris Matthews
Subject: Re: [LNT] r207898 - Use Mann-Whitney U test to identify changes
Hi Yi,
I'm getting this error on our test-suite bot:
No distributions at all found for SciPy (from LNT==0.4.1dev)
http://lab.llvm.org:8011/builders/clang-native-arm-lnt/builds/6265/steps/venv.lnt.install/logs/stdio
Any idea how to pull that? Or why it's not finding the right module?
I did install python-scipy module from Ubuntu repo, but I don't think
that's related...
cheers,
--renato
On 3 May 2014 09:01, Yi Kong <Yi.Kong at arm.com> wrote:
> Author: kongyi
> Date: Sat May 3 03:01:30 2014
> New Revision: 207898
>
> URL: http://llvm.org/viewvc/llvm-project?rev=207898&view=rev
> Log:
> Use Mann-Whitney U test to identify changes
>
> Modified:
> lnt/trunk/lnt/server/reporting/analysis.py
> lnt/trunk/lnt/server/reporting/runs.py
> lnt/trunk/lnt/server/ui/templates/reporting/runs.html
> lnt/trunk/lnt/server/ui/templates/v4_run.html
> lnt/trunk/lnt/server/ui/views.py
> lnt/trunk/setup.py
>
> Modified: lnt/trunk/lnt/server/reporting/analysis.py
> URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/reporting/analysis.py?rev=207898&r1=207897&r2=207898&view=diff
> ==============================================================================
> --- lnt/trunk/lnt/server/reporting/analysis.py (original)
> +++ lnt/trunk/lnt/server/reporting/analysis.py Sat May 3 03:01:30 2014
> @@ -5,6 +5,7 @@ Utilities for helping with the analysis
> from lnt.util import stats
> from lnt.server.ui import util
> from lnt.testing import PASS, FAIL, XFAIL
> +from scipy.stats import mannwhitneyu
>
> REGRESSED = 'REGRESSED'
> IMPROVED = 'IMPROVED'
> @@ -13,8 +14,8 @@ UNCHANGED_FAIL = 'UNCHANGED_FAIL'
>
> class ComparisonResult:
> def __init__(self, cur_value, prev_value, delta, pct_delta, stddev, MAD,
> - cur_failed, prev_failed, samples, stddev_mean = None,
> - stddev_is_estimated = False):
> + cur_failed, prev_failed, samples, prev_samples, stddev_mean = None,
> + stddev_is_estimated = False, confidence_lv = .9):
> self.current = cur_value
> self.previous = prev_value
> self.delta = delta
> @@ -24,8 +25,17 @@ class ComparisonResult:
> self.failed = cur_failed
> self.prev_failed = prev_failed
> self.samples = samples
> + self.prev_samples = prev_samples
> self.stddev_mean = stddev_mean
> self.stddev_is_estimated = stddev_is_estimated
> + self.confidence_lv = confidence_lv
> +
> + try:
> + u, prob = mannwhitneyu(self.samples, self.prev_samples)
> + self.p = 1 - prob
> + except ValueError:
> + # All numbers are identical
> + self.p = 1
>
> def get_samples(self):
> return self.samples
> @@ -89,6 +99,12 @@ class ComparisonResult:
> if ignore_small and abs(self.delta) < .01:
> return UNCHANGED_PASS
>
> + # Use Mann-Whitney U test to test null hypothesis that result is
> + # unchanged.
> + if len(self.samples) >= 4 and len(self.prev_samples) >= 4:
> + if self.p <= self.confidence_lv:
> + return UNCHANGED_PASS
> +
> # If we have a comparison window, then measure using a symmetic
> # confidence interval.
> if self.stddev is not None:
> @@ -121,11 +137,16 @@ class ComparisonResult:
> else:
> return UNCHANGED_PASS
>
> +
> + def get_p_value(self):
> + return self.p
> +
> class RunInfo(object):
> def __init__(self, testsuite, runs_to_load,
> - aggregation_fn = min):
> + aggregation_fn = min, confidence_lv = .9):
> self.testsuite = testsuite
> self.aggregation_fn = aggregation_fn
> + self.confidence_lv = confidence_lv
>
> self.sample_map = util.multidict()
> self.loaded_run_ids = set()
> @@ -207,7 +228,8 @@ class RunInfo(object):
> run_value, prev_value, delta=None,
> pct_delta = None, stddev = stddev, MAD = MAD,
> cur_failed = run_failed, prev_failed = prev_failed,
> - samples = run_values)
> + samples = run_values, prev_samples = prev_values,
> + confidence_lv = self.confidence_lv)
>
> # Compute the comparison status for the test value.
> delta = run_value - prev_value
> @@ -241,8 +263,9 @@ class RunInfo(object):
>
> return ComparisonResult(run_value, prev_value, delta,
> pct_delta, stddev, MAD,
> - run_failed, prev_failed, run_values,
> - stddev_mean, stddev_is_estimated)
> + run_failed, prev_failed, run_values, prev_values,
> + stddev_mean, stddev_is_estimated,
> + self.confidence_lv)
>
> def _load_samples_for_runs(self, run_ids):
> # Find the set of new runs to load.
>
> Modified: lnt/trunk/lnt/server/reporting/runs.py
> URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/reporting/runs.py?rev=207898&r1=207897&r2=207898&view=diff
> ==============================================================================
> --- lnt/trunk/lnt/server/reporting/runs.py (original)
> +++ lnt/trunk/lnt/server/reporting/runs.py Sat May 3 03:01:30 2014
> @@ -14,7 +14,8 @@ import lnt.util.stats
> def generate_run_report(run, baseurl, only_html_body = False,
> num_comparison_runs = 10, result = None,
> compare_to = None, baseline = None,
> - comparison_window = None, aggregation_fn = min):
> + comparison_window = None, aggregation_fn = min,
> + confidence_lv = .5):
> """
> generate_run_report(...) -> (str: subject, str: text_report,
> str: html_report)
> @@ -67,7 +68,7 @@ def generate_run_report(run, baseurl, on
> if baseline:
> runs_to_load.add(baseline.id)
> sri = lnt.server.reporting.analysis.RunInfo(
> - ts, runs_to_load, aggregation_fn)
> + ts, runs_to_load, aggregation_fn, confidence_lv)
>
> # Get the test names, primary fields and total test counts.
> test_names = ts.query(ts.Test.name, ts.Test.id).\
>
> Modified: lnt/trunk/lnt/server/ui/templates/reporting/runs.html
> URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/ui/templates/reporting/runs.html?rev=207898&r1=207897&r2=207898&view=diff
> ==============================================================================
> --- lnt/trunk/lnt/server/ui/templates/reporting/runs.html (original)
> +++ lnt/trunk/lnt/server/ui/templates/reporting/runs.html Sat May 3 03:01:30 2014
> @@ -25,9 +25,11 @@
> <th style="{{ styles['th'] }}">{{ primary_name }}</th>
> <th style="{{ styles['th'] }}">Current</th>
> <th style="{{ styles['th'] }}">σ {{ primary_field_suffix }}</th>
> + <th style="{{ styles['th'] }}">p {{ primary_field_suffix }}</th>
> {% if secondary_info %}
> <th style="{{ styles['th'] }}">Δ {{ secondary_field_suffix }}</th>
> <th style="{{ styles['th'] }}">σ {{ secondary_field_suffix }}</th>
> + <th style="{{ styles['th'] }}">p {{ secondary_field_suffix }}</th>
> {% endif %}
> </tr>
>
> @@ -40,10 +42,12 @@
> <td style="{{ styles['td'] }}">{{ "%.4f" | format(cr.previous) }}</td>
> <td style="{{ styles['td'] }}">{{ "%.4f" | format(cr.current) }}</td>
> <td style="{{ styles['td'] }}">{{ "%.4f" | format_or_default(cr.stddev, '-') }}</td>
> + <td style="{{ styles['td'] }}">{{ "%.4f" | format_or_default(cr.p, '-') }}</td>
> {% if secondary_info %}
> {% set a_cr = secondary_info[(name, field)] %}
> {{ a_cr.pct_delta|aspctcell(style=styles['td'])|safe }}
> <td style="{{ styles['td'] }}">{{ "%.4f" | format_or_default(a_cr.stddev, '-') }}</td>
> + <td style="{{ styles['td'] }}">{{ "%.4f" | format_or_default(a_cr.p, '-') }}</td>
> {% endif %}
> </tr>
> {% endfor %}
>
> Modified: lnt/trunk/lnt/server/ui/templates/v4_run.html
> URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/ui/templates/v4_run.html?rev=207898&r1=207897&r2=207898&view=diff
> ==============================================================================
> --- lnt/trunk/lnt/server/ui/templates/v4_run.html (original)
> +++ lnt/trunk/lnt/server/ui/templates/v4_run.html Sat May 3 03:01:30 2014
> @@ -76,6 +76,10 @@
> {% if options.show_sample_counts %}
> <td>{{cr.get_samples()|length}}</td>
> {% endif %}
> +
> + {% if options.show_p_value %}
> + <td>{{ "%.4f" % cr.get_p_value() }}</td>
> + {% endif %}
> {% endmacro %}
>
> {% block sidebar %}
> @@ -244,6 +248,14 @@
> </td>
> </tr>
> <tr>
> + <td>Mann-Whitney test confidence level:</td>
> + <td><input type="text" name="MW_confidence_lv" value="{{ options.MW_confidence_lv }}"></td>
> + </tr>
> + <tr>
> + <td>Show Mann-Whitney p value:</td>
> + <td><input type="checkbox" name="show_p_value" value="yes" {{ "checked" if options.show_p_value else ""}}></td>
> + </tr>
> + <tr>
> <td colspan="2">
> {% if compare_to %}
> <input type="hidden" name="compare_to" value="{{compare_to.id}}">
> @@ -285,6 +297,7 @@
> {% if options.show_mad %}<th>MAD</th>{% endif %}
> {% if options.show_all_samples %}<th>Samples</th>{% endif %}
> {% if options.show_sample_counts %}<th>N</th>{% endif %}
> + {% if options.show_p_value %}<th>p</th>{% endif %}
> </tr>
> </thead>
> <tbody>
>
> Modified: lnt/trunk/lnt/server/ui/views.py
> URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/ui/views.py?rev=207898&r1=207897&r2=207898&view=diff
> ==============================================================================
> --- lnt/trunk/lnt/server/ui/views.py (original)
> +++ lnt/trunk/lnt/server/ui/views.py Sat May 3 03:01:30 2014
> @@ -191,6 +191,10 @@ class V4RequestInfo(object):
> 'median' : lnt.util.stats.median }.get(
> aggregation_fn_name, min)
>
> + # Get the MW confidence level
> + confidence_lv = float(request.args.get('MW_confidence_lv', .9))
> + self.confidence_lv = confidence_lv
> +
> # Find the neighboring runs, by order.
> prev_runs = list(ts.get_previous_runs_on_machine(run, N = 3))
> next_runs = list(ts.get_next_runs_on_machine(run, N = 3))
> @@ -246,7 +250,7 @@ class V4RequestInfo(object):
> only_html_body=only_html_body, result=None,
> compare_to=self.compare_to, baseline=self.baseline,
> comparison_window=self.comparison_window,
> - aggregation_fn=self.aggregation_fn)
> + aggregation_fn=self.aggregation_fn, confidence_lv=confidence_lv)
> _, self.text_report, self.html_report, self.sri = reports
>
> @v4_route("/<int:id>/report")
> @@ -314,6 +318,8 @@ def v4_run(id):
> options['num_comparison_runs'] = info.num_comparison_runs
> options['test_filter'] = test_filter_str = request.args.get(
> 'test_filter', '')
> + options['MW_confidence_lv'] = float(request.args.get('MW_confidence_lv', .9))
> + options['show_p_value'] = bool(request.args.get('show_p_value'))
> if test_filter_str:
> test_filter_re = re.compile(test_filter_str)
> else:
>
> Modified: lnt/trunk/setup.py
> URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/setup.py?rev=207898&r1=207897&r2=207898&view=diff
> ==============================================================================
> --- lnt/trunk/setup.py (original)
> +++ lnt/trunk/setup.py Sat May 3 03:01:30 2014
> @@ -90,5 +90,5 @@ http://llvm.org/svn/llvm-project/lnt/tru
> 'lnt = lnt.lnttool:main',
> ],
> },
> - install_requires=['SQLAlchemy', 'Flask'],
> + install_requires=['SQLAlchemy', 'Flask', 'SciPy'],
> )
>
>
> _______________________________________________
> llvm-commits mailing list
> llvm-commits at cs.uiuc.edu
> http://lists.cs.uiuc.edu/mailman/listinfo/llvm-commits
-- IMPORTANT NOTICE: The contents of this email and any attachments are confidential and may also be privileged. If you are not the intended recipient, please notify the sender immediately and do not disclose the contents to any other person, use it for any purpose, or store or copy the information in any medium. Thank you.
ARM Limited, Registered office 110 Fulbourn Road, Cambridge CB1 9NJ, Registered in England & Wales, Company No: 2557590
ARM Holdings plc, Registered office 110 Fulbourn Road, Cambridge CB1 9NJ, Registered in England & Wales, Company No: 2548782
More information about the llvm-commits
mailing list