[LNT] r268593 - Add --pgo: LNT make test-suite generate then use PGO profiles

[Chris Matthews via llvm-commits]

Author: cmatthews
Date: Wed May  4 21:00:41 2016
New Revision: 268593

URL: http://llvm.org/viewvc/llvm-project?rev=268593&view=rev
Log:
Add --pgo: LNT make test-suite generate then use PGO profiles

This option exposes something that the test-suite could already do if
you knew how. Calling LNT with --pgo will run the test-suite twice, once
with the train input set to collect PGO profiles, and then in the usual
input set using those collected PGO profiles to rebuild the benchmarks.
Only the second run's data is submitted.

Modified:
    lnt/trunk/lnt/tests/test_suite.py
    lnt/trunk/tests/runtest/test_suite.py

Modified: lnt/trunk/lnt/tests/test_suite.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/tests/test_suite.py?rev=268593&r1=268592&r2=268593&view=diff
==============================================================================
--- lnt/trunk/lnt/tests/test_suite.py (original)
+++ lnt/trunk/lnt/tests/test_suite.py Wed May  4 21:00:41 2016
@@ -56,6 +56,7 @@ class TestSuiteTest(BuiltinTest):
     def __init__(self):
         self.configured = False
         self.compiled = False
+        self.trained = False
 
     def describe(self):
         return "LLVM test-suite"
@@ -172,6 +173,11 @@ class TestSuiteTest(BuiltinTest):
                               "test, this will not run all the tests.  Must be"
                               " used in conjunction with --only-test.",
                          action="store_true", default=False,)
+        group.add_option("", "--pgo", dest="pgo",
+                         help="Run the test-suite in training mode first and"
+                         " collect PGO data, then rerun with that training "
+                         "data.",
+                         action="store_true", default=False,)
 
         parser.add_option_group(group)
 
@@ -373,6 +379,10 @@ class TestSuiteTest(BuiltinTest):
         if not os.path.exists(path):
             mkdir_p(path)
             
+        if self.opts.pgo:
+            self._collect_pgo(path)
+            self.trained = True
+            
         if not self.configured and self._need_to_configure(path):
             self._configure(path)
             self._clean(path)
@@ -427,7 +437,7 @@ class TestSuiteTest(BuiltinTest):
         self._check_call([make_cmd, 'clean'],
                          cwd=subdir)
         
-    def _configure(self, path, execute=True):
+    def _configure(self, path, extra_flags=[], execute=True):
         cmake_cmd = self.opts.cmake
 
         defs = {
@@ -451,10 +461,19 @@ class TestSuiteTest(BuiltinTest):
             defs['TEST_SUITE_USE_PERF'] = 'ON'
         if self.opts.test_suite_externals:
             defs['TEST_SUITE_EXTERNALS_DIR'] = self.opts.test_suite_externals
+        if self.opts.pgo and self.trained:
+            defs['TEST_SUITE_PROFILE_USE'] = "On"
+            defs['TEST_SUITE_PROFILE_GENERATE'] = "Off"
+            # This could be redefined by user defines.
+            defs['TEST_SUITE_RUN_TYPE'] = "test"
+        
         if self.opts.cmake_defines:
             for item in self.opts.cmake_defines:
                 k, v = item.split('=', 1)
                 defs[k] = v
+        for item in extra_flags:
+            k, v = item.split('=', 1)
+            defs[k] = v
             
         lines = ['Configuring with {']
         for k, v in sorted(defs.items()):
@@ -481,6 +500,12 @@ class TestSuiteTest(BuiltinTest):
             self._check_call(cmake_cmd, cwd=path)
 
         return cmake_cmd
+        
+    def _collect_pgo(self, path):
+        flags = ["TEST_SUITE_PROFILE_GENERATE=On", "TEST_SUITE_RUN_TYPE=train"]
+        self._configure(path, extra_flags=flags)
+        self._make(path)
+        self._lit(path, True)
 
     def _make(self, path):
         make_cmd = self.opts.make

Modified: lnt/trunk/tests/runtest/test_suite.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/tests/runtest/test_suite.py?rev=268593&r1=268592&r2=268593&view=diff
==============================================================================
--- lnt/trunk/tests/runtest/test_suite.py (original)
+++ lnt/trunk/tests/runtest/test_suite.py Wed May  4 21:00:41 2016
@@ -351,14 +351,26 @@
 # CHECK-USE-PERF-ALL: Importing 1 profiles with
 # CHECK-USE-PERF-ALL: Profile /tmp/I/Do/Not/Exist.perf_data does not exist
 
+
 # Check a missing --cc on the command line
-# RUN: lnt runtest test-suite \
-# RUN:     --sandbox %t.SANDBOX \
-# RUN:     --no-timestamp \
-# RUN:     --test-suite %S/Inputs/test-suite-cmake \
 # RUN:     --use-cmake %S/Inputs/test-suite-cmake/fake-cmake \
 # RUN:     --use-make %S/Inputs/test-suite-cmake/fake-make \
 # RUN:     --use-lit %S/Inputs/test-suite-cmake/fake-lit \
 # RUN:     > %t.log 2> %t.err || true
 # RUN: FileCheck  --check-prefix CHECK-MISSING-CC < %t.err %s
 # CHECK-MISSING-CC: error: --cc is required
+
+# Check running with PGO
+# RUN: lnt runtest test-suite \
+# RUN:     --sandbox %t.SANDBOX \
+# RUN:     --no-timestamp \
+# RUN:     --test-suite %S/Inputs/test-suite-cmake \
+# RUN:     --cc %{shared_inputs}/FakeCompilers/clang-r154331 \
+# RUN:     --use-cmake %S/Inputs/test-suite-cmake/fake-cmake \
+# RUN:     --use-make %S/Inputs/test-suite-cmake/fake-make \
+# RUN:     --use-lit %S/Inputs/test-suite-cmake/fake-lit-profile \
+# RUN:     --verbose \
+# RUN:     --pgo \
+# RUN:     > %t.pgo.log 2> %t.pgo.err
+# RUN: FileCheck --check-prefix CHECK-PGO < %t.pgo.err %s
+# CHECK-PGO: pass