[LNT] r238965 - Improved regression detection with min of diffs

[Chris Matthews]

Author: cmatthews
Date: Wed Jun  3 16:01:23 2015
New Revision: 238965

URL: http://llvm.org/viewvc/llvm-project?rev=238965&view=rev
Log:
Improved regression detection with min of diffs

This change reduces false positives in LNT's regression detection
algorithm by having it use more past data.  Instead of calculating the
delta as the difference of min(current)-min(prev), we use the smallest
difference between min(current) and all prevs.  When the data has noise
or is multimodal this reduces our chances of flagging a false positive
regression.

Modified:
    lnt/trunk/lnt/server/reporting/analysis.py
    lnt/trunk/tests/server/reporting/analysis.py

Modified: lnt/trunk/lnt/server/reporting/analysis.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/server/reporting/analysis.py?rev=238965&r1=238964&r2=238965&view=diff
==============================================================================
--- lnt/trunk/lnt/server/reporting/analysis.py (original)
+++ lnt/trunk/lnt/server/reporting/analysis.py Wed Jun  3 16:01:23 2015
@@ -15,6 +15,21 @@ UNCHANGED_FAIL = 'UNCHANGED_FAIL'
 MIN_VALUE_PRECISION = 0.0001
 
 
+def absmin_diff(current, prevs):
+    """Min of differences between current sample and all previous samples.
+    Given more than one min, use the last one detected which is probably a 
+    newer value. Returns (difference, prev used)
+    """
+    diffs = [abs(current-prev) for prev in prevs]
+    smallest_pos = 0
+    smallest = diffs[0]
+    for i, diff in enumerate(diffs):
+        if diff <= smallest:
+            smallest = diff
+            smallest_pos = i
+    return current-prevs[smallest_pos], prevs[smallest_pos]
+
+
 def calc_geomean(run_values):
     # NOTE Geometric mean applied only to positive values, so fix it by
     # adding MIN_VALUE to each value and substract it from the result.
@@ -48,8 +63,8 @@ class ComparisonResult:
 
         # Compute the comparison status for the test value.
         if self.current and self.previous and self.previous != 0:
-            self.delta = self.current - self.previous
-            self.pct_delta = self.delta / self.previous
+            self.delta, value = absmin_diff(self.current, prev_samples)
+            self.pct_delta = self.delta / value
         else:
             self.delta = 0
             self.pct_delta = 0.0

Modified: lnt/trunk/tests/server/reporting/analysis.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/tests/server/reporting/analysis.py?rev=238965&r1=238964&r2=238965&view=diff
==============================================================================
--- lnt/trunk/tests/server/reporting/analysis.py (original)
+++ lnt/trunk/tests/server/reporting/analysis.py Wed Jun  3 16:01:23 2015
@@ -2,8 +2,10 @@
 #
 # RUN: python %s
 import unittest
+
 from lnt.server.reporting.analysis import ComparisonResult, REGRESSED, IMPROVED
 from lnt.server.reporting.analysis import UNCHANGED_PASS, UNCHANGED_FAIL
+from lnt.server.reporting.analysis import absmin_diff
 
 FLAT_LINE = [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0,
              1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]
@@ -309,5 +311,18 @@ class ComparisonResultTest(unittest.Test
         # Fixme
         # self.assertEquals(slow.get_value_status(), IMPROVED)
 
+
+class AbsMinTester(unittest.TestCase):
+
+    def test_absmin(self):
+        """Test finding smallest difference."""
+        self.assertEqual(absmin_diff(1, [2, 2, 3]), (-1, 2))
+        self.assertEqual(absmin_diff(1, [1, 2, 3]), (0, 1))
+        self.assertEqual(absmin_diff(1, [2]), (-1, 2))
+        self.assertEqual(absmin_diff(1.5, [1, 4, 4]), (0.5, 1))
+        self.assertEqual(absmin_diff(5, [1, 2, 1]), (3, 2))
+        self.assertEqual(absmin_diff(1, [2, 0, 3]), (1, 0))
+
+
 if __name__ == '__main__':
     unittest.main()