[LNT] r207898 - Use Mann-Whitney U test to identify changes

Renato Golin renato.golin at linaro.org
Sat May 3 03:47:51 PDT 2014


Hi Yi,

I'm getting this error on our test-suite bot:

No distributions at all found for SciPy (from LNT==0.4.1dev)

http://lab.llvm.org:8011/builders/clang-native-arm-lnt/builds/6265/steps/venv.lnt.install/logs/stdio

Any idea how to pull that? Or why it's not finding the right module?

I did install python-scipy module from Ubuntu repo, but I don't think
that's related...

cheers,
--renato

On 3 May 2014 09:01, Yi Kong <Yi.Kong at arm.com> wrote:
> Author: kongyi
> Date: Sat May  3 03:01:30 2014
> New Revision: 207898
>
> URL: http://llvm.org/viewvc/llvm-project?rev=207898&view=rev
> Log:
> Use Mann-Whitney U test to identify changes
>
> Modified:
>     lnt/trunk/lnt/server/reporting/analysis.py
>     lnt/trunk/lnt/server/reporting/runs.py
>     lnt/trunk/lnt/server/ui/templates/reporting/runs.html
>     lnt/trunk/lnt/server/ui/templates/v4_run.html
>     lnt/trunk/lnt/server/ui/views.py
>     lnt/trunk/setup.py
>
> Modified: lnt/trunk/lnt/server/reporting/analysis.py
> URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/reporting/analysis.py?rev=207898&r1=207897&r2=207898&view=diff
> ==============================================================================
> --- lnt/trunk/lnt/server/reporting/analysis.py (original)
> +++ lnt/trunk/lnt/server/reporting/analysis.py Sat May  3 03:01:30 2014
> @@ -5,6 +5,7 @@ Utilities for helping with the analysis
>  from lnt.util import stats
>  from lnt.server.ui import util
>  from lnt.testing import PASS, FAIL, XFAIL
> +from scipy.stats import mannwhitneyu
>
>  REGRESSED = 'REGRESSED'
>  IMPROVED = 'IMPROVED'
> @@ -13,8 +14,8 @@ UNCHANGED_FAIL = 'UNCHANGED_FAIL'
>
>  class ComparisonResult:
>      def __init__(self, cur_value, prev_value, delta, pct_delta, stddev, MAD,
> -                 cur_failed, prev_failed, samples, stddev_mean = None,
> -                 stddev_is_estimated = False):
> +                 cur_failed, prev_failed, samples, prev_samples, stddev_mean = None,
> +                 stddev_is_estimated = False, confidence_lv = .9):
>          self.current = cur_value
>          self.previous = prev_value
>          self.delta = delta
> @@ -24,8 +25,17 @@ class ComparisonResult:
>          self.failed = cur_failed
>          self.prev_failed = prev_failed
>          self.samples = samples
> +        self.prev_samples = prev_samples
>          self.stddev_mean = stddev_mean
>          self.stddev_is_estimated = stddev_is_estimated
> +        self.confidence_lv = confidence_lv
> +
> +        try:
> +            u, prob = mannwhitneyu(self.samples, self.prev_samples)
> +            self.p = 1 - prob
> +        except ValueError:
> +            # All numbers are identical
> +            self.p = 1
>
>      def get_samples(self):
>          return self.samples
> @@ -89,6 +99,12 @@ class ComparisonResult:
>          if ignore_small and abs(self.delta) < .01:
>              return UNCHANGED_PASS
>
> +        # Use Mann-Whitney U test to test null hypothesis that result is
> +        # unchanged.
> +        if len(self.samples) >= 4 and len(self.prev_samples) >= 4:
> +            if self.p <= self.confidence_lv:
> +                return UNCHANGED_PASS
> +
>          # If we have a comparison window, then measure using a symmetic
>          # confidence interval.
>          if self.stddev is not None:
> @@ -121,11 +137,16 @@ class ComparisonResult:
>          else:
>              return UNCHANGED_PASS
>
> +
> +    def get_p_value(self):
> +        return self.p
> +
>  class RunInfo(object):
>      def __init__(self, testsuite, runs_to_load,
> -                 aggregation_fn = min):
> +                 aggregation_fn = min, confidence_lv = .9):
>          self.testsuite = testsuite
>          self.aggregation_fn = aggregation_fn
> +        self.confidence_lv = confidence_lv
>
>          self.sample_map = util.multidict()
>          self.loaded_run_ids = set()
> @@ -207,7 +228,8 @@ class RunInfo(object):
>                  run_value, prev_value, delta=None,
>                  pct_delta = None, stddev = stddev, MAD = MAD,
>                  cur_failed = run_failed, prev_failed = prev_failed,
> -                samples = run_values)
> +                samples = run_values, prev_samples = prev_values,
> +                confidence_lv = self.confidence_lv)
>
>          # Compute the comparison status for the test value.
>          delta = run_value - prev_value
> @@ -241,8 +263,9 @@ class RunInfo(object):
>
>          return ComparisonResult(run_value, prev_value, delta,
>                                  pct_delta, stddev, MAD,
> -                                run_failed, prev_failed, run_values,
> -                                stddev_mean, stddev_is_estimated)
> +                                run_failed, prev_failed, run_values, prev_values,
> +                                stddev_mean, stddev_is_estimated,
> +                                self.confidence_lv)
>
>      def _load_samples_for_runs(self, run_ids):
>          # Find the set of new runs to load.
>
> Modified: lnt/trunk/lnt/server/reporting/runs.py
> URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/reporting/runs.py?rev=207898&r1=207897&r2=207898&view=diff
> ==============================================================================
> --- lnt/trunk/lnt/server/reporting/runs.py (original)
> +++ lnt/trunk/lnt/server/reporting/runs.py Sat May  3 03:01:30 2014
> @@ -14,7 +14,8 @@ import lnt.util.stats
>  def generate_run_report(run, baseurl, only_html_body = False,
>                          num_comparison_runs = 10, result = None,
>                          compare_to = None, baseline = None,
> -                        comparison_window = None, aggregation_fn = min):
> +                        comparison_window = None, aggregation_fn = min,
> +                        confidence_lv = .5):
>      """
>      generate_run_report(...) -> (str: subject, str: text_report,
>                                   str: html_report)
> @@ -67,7 +68,7 @@ def generate_run_report(run, baseurl, on
>      if baseline:
>          runs_to_load.add(baseline.id)
>      sri = lnt.server.reporting.analysis.RunInfo(
> -        ts, runs_to_load, aggregation_fn)
> +        ts, runs_to_load, aggregation_fn, confidence_lv)
>
>      # Get the test names, primary fields and total test counts.
>      test_names = ts.query(ts.Test.name, ts.Test.id).\
>
> Modified: lnt/trunk/lnt/server/ui/templates/reporting/runs.html
> URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/ui/templates/reporting/runs.html?rev=207898&r1=207897&r2=207898&view=diff
> ==============================================================================
> --- lnt/trunk/lnt/server/ui/templates/reporting/runs.html (original)
> +++ lnt/trunk/lnt/server/ui/templates/reporting/runs.html Sat May  3 03:01:30 2014
> @@ -25,9 +25,11 @@
>           <th style="{{ styles['th'] }}">{{ primary_name }}</th>
>           <th style="{{ styles['th'] }}">Current</th>
>           <th style="{{ styles['th'] }}">σ {{ primary_field_suffix }}</th>
> +         <th style="{{ styles['th'] }}">p {{ primary_field_suffix }}</th>
>           {% if secondary_info %}
>           <th style="{{ styles['th'] }}">Δ {{ secondary_field_suffix }}</th>
>           <th style="{{ styles['th'] }}">σ {{ secondary_field_suffix }}</th>
> +         <th style="{{ styles['th'] }}">p {{ secondary_field_suffix }}</th>
>           {% endif %}
>           </tr>
>
> @@ -40,10 +42,12 @@
>             <td style="{{ styles['td'] }}">{{ "%.4f" | format(cr.previous) }}</td>
>             <td style="{{ styles['td'] }}">{{ "%.4f" | format(cr.current) }}</td>
>             <td style="{{ styles['td'] }}">{{ "%.4f" | format_or_default(cr.stddev, '-') }}</td>
> +           <td style="{{ styles['td'] }}">{{ "%.4f" | format_or_default(cr.p, '-') }}</td>
>             {% if secondary_info %}
>               {% set a_cr = secondary_info[(name, field)] %}
>               {{ a_cr.pct_delta|aspctcell(style=styles['td'])|safe }}
>               <td style="{{ styles['td'] }}">{{ "%.4f" | format_or_default(a_cr.stddev, '-') }}</td>
> +             <td style="{{ styles['td'] }}">{{ "%.4f" | format_or_default(a_cr.p, '-') }}</td>
>             {% endif %}
>           </tr>
>           {% endfor %}
>
> Modified: lnt/trunk/lnt/server/ui/templates/v4_run.html
> URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/ui/templates/v4_run.html?rev=207898&r1=207897&r2=207898&view=diff
> ==============================================================================
> --- lnt/trunk/lnt/server/ui/templates/v4_run.html (original)
> +++ lnt/trunk/lnt/server/ui/templates/v4_run.html Sat May  3 03:01:30 2014
> @@ -76,6 +76,10 @@
>    {% if options.show_sample_counts %}
>      <td>{{cr.get_samples()|length}}</td>
>    {% endif %}
> +
> +  {% if options.show_p_value %}
> +    <td>{{ "%.4f" % cr.get_p_value() }}</td>
> +  {% endif %}
>  {% endmacro %}
>
>  {% block sidebar %}
> @@ -244,6 +248,14 @@
>        </td>
>      </tr>
>      <tr>
> +      <td>Mann-Whitney test confidence level:</td>
> +      <td><input type="text" name="MW_confidence_lv" value="{{ options.MW_confidence_lv }}"></td>
> +    </tr>
> +    <tr>
> +      <td>Show Mann-Whitney p value:</td>
> +      <td><input type="checkbox" name="show_p_value" value="yes" {{ "checked" if options.show_p_value else ""}}></td>
> +    </tr>
> +    <tr>
>        <td colspan="2">
>          {% if compare_to %}
>            <input type="hidden" name="compare_to" value="{{compare_to.id}}">
> @@ -285,6 +297,7 @@
>              {% if options.show_mad %}<th>MAD</th>{% endif %}
>              {% if options.show_all_samples %}<th>Samples</th>{% endif %}
>              {% if options.show_sample_counts %}<th>N</th>{% endif %}
> +            {% if options.show_p_value %}<th>p</th>{% endif %}
>            </tr>
>          </thead>
>          <tbody>
>
> Modified: lnt/trunk/lnt/server/ui/views.py
> URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/ui/views.py?rev=207898&r1=207897&r2=207898&view=diff
> ==============================================================================
> --- lnt/trunk/lnt/server/ui/views.py (original)
> +++ lnt/trunk/lnt/server/ui/views.py Sat May  3 03:01:30 2014
> @@ -191,6 +191,10 @@ class V4RequestInfo(object):
>                                  'median' : lnt.util.stats.median }.get(
>              aggregation_fn_name, min)
>
> +        # Get the MW confidence level
> +        confidence_lv = float(request.args.get('MW_confidence_lv', .9))
> +        self.confidence_lv = confidence_lv
> +
>          # Find the neighboring runs, by order.
>          prev_runs = list(ts.get_previous_runs_on_machine(run, N = 3))
>          next_runs = list(ts.get_next_runs_on_machine(run, N = 3))
> @@ -246,7 +250,7 @@ class V4RequestInfo(object):
>              only_html_body=only_html_body, result=None,
>              compare_to=self.compare_to, baseline=self.baseline,
>              comparison_window=self.comparison_window,
> -            aggregation_fn=self.aggregation_fn)
> +            aggregation_fn=self.aggregation_fn, confidence_lv=confidence_lv)
>          _, self.text_report, self.html_report, self.sri = reports
>
>  @v4_route("/<int:id>/report")
> @@ -314,6 +318,8 @@ def v4_run(id):
>      options['num_comparison_runs'] = info.num_comparison_runs
>      options['test_filter'] = test_filter_str = request.args.get(
>          'test_filter', '')
> +    options['MW_confidence_lv'] = float(request.args.get('MW_confidence_lv', .9))
> +    options['show_p_value'] = bool(request.args.get('show_p_value'))
>      if test_filter_str:
>          test_filter_re = re.compile(test_filter_str)
>      else:
>
> Modified: lnt/trunk/setup.py
> URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/setup.py?rev=207898&r1=207897&r2=207898&view=diff
> ==============================================================================
> --- lnt/trunk/setup.py (original)
> +++ lnt/trunk/setup.py Sat May  3 03:01:30 2014
> @@ -90,5 +90,5 @@ http://llvm.org/svn/llvm-project/lnt/tru
>              'lnt = lnt.lnttool:main',
>              ],
>          },
> -    install_requires=['SQLAlchemy', 'Flask'],
> +    install_requires=['SQLAlchemy', 'Flask', 'SciPy'],
>  )
>
>
> _______________________________________________
> llvm-commits mailing list
> llvm-commits at cs.uiuc.edu
> http://lists.cs.uiuc.edu/mailman/listinfo/llvm-commits



More information about the llvm-commits mailing list