[LNT] r263930 - [profile] Add profile harvesting to test_suite.py
James Molloy via llvm-commits
llvm-commits at lists.llvm.org
Mon Mar 21 03:47:01 PDT 2016
Author: jamesm
Date: Mon Mar 21 05:47:00 2016
New Revision: 263930
URL: http://llvm.org/viewvc/llvm-project?rev=263930&view=rev
Log:
[profile] Add profile harvesting to test_suite.py
This is done via multiprocessing because it's on the critical path and can take a minute or two for a full LNT run (~300 profiles)
Added:
lnt/trunk/tests/runtest/Inputs/test-suite-cmake/fake-lit-profile (with props)
lnt/trunk/tests/runtest/Inputs/test-suite-cmake/fake-results-profile.json
Modified:
lnt/trunk/lnt/tests/test_suite.py
lnt/trunk/tests/runtest/Inputs/test-suite-cmake/fake-cmake
lnt/trunk/tests/runtest/test_suite.py
Modified: lnt/trunk/lnt/tests/test_suite.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/lnt/tests/test_suite.py?rev=263930&r1=263929&r2=263930&view=diff
==============================================================================
--- lnt/trunk/lnt/tests/test_suite.py (original)
+++ lnt/trunk/lnt/tests/test_suite.py Mon Mar 21 05:47:00 2016
@@ -1,8 +1,10 @@
import subprocess, tempfile, json, os, shlex, platform, pipes, sys
+import multiprocessing
from optparse import OptionParser, OptionGroup
import lnt.testing
+import lnt.testing.profile
import lnt.testing.util.compilers
from lnt.testing.util.misc import timestamp
from lnt.testing.util.commands import note, warning, fatal
@@ -17,6 +19,25 @@ from lnt.tests.builtintest import Builti
TEST_SUITE_KNOWN_ARCHITECTURES = ['ARM', 'AArch64', 'Mips', 'X86']
KNOWN_SAMPLE_KEYS = ['compile', 'exec', 'hash', 'score']
+# _importProfile imports a single profile. It must be at the top level (and
+# not within TestSuiteTest) so that multiprocessing can import it correctly.
+def _importProfile(name_filename):
+ name, filename = name_filename
+
+ if not os.path.exists(filename):
+ warning('Profile %s does not exist' % filename)
+ return None
+
+ pf = lnt.testing.profile.profile.Profile.fromFile(filename)
+ if not pf:
+ return None
+
+ pf.upgrade()
+ profilefile = pf.render()
+ return lnt.testing.TestSamples(name + '.profile',
+ [profilefile],
+ {},
+ str)
class TestSuiteTest(BuiltinTest):
def __init__(self):
@@ -118,9 +139,11 @@ class TestSuiteTest(BuiltinTest):
help="Number of compilation threads, defaults to "
"--threads", type=int, default=0, metavar="N")
group.add_option("", "--use-perf", dest="use_perf",
- help=("Use perf to obtain high accuracy timing"
- "[%default]"),
- action='store_true', default=False)
+ help=("Use Linux perf for high accuracy timing, profile "
+ "information or both"),
+ type='choice',
+ choices=['none', 'time', 'profile', 'all'],
+ default='none')
group.add_option("", "--run-under", dest="run_under",
help="Wrapper to run tests under ['%default']",
type=str, default="")
@@ -396,7 +419,7 @@ class TestSuiteTest(BuiltinTest):
defs['TEST_SUITE_RUN_UNDER'] = self._unix_quote_args(self.opts.run_under)
if self.opts.benchmarking_only:
defs['TEST_SUITE_BENCHMARKING_ONLY'] = 'ON'
- if self.opts.use_perf:
+ if self.opts.use_perf in ('time', 'all'):
defs['TEST_SUITE_USE_PERF'] = 'ON'
if self.opts.cmake_defines:
for item in self.opts.cmake_defines:
@@ -456,13 +479,15 @@ class TestSuiteTest(BuiltinTest):
subdir = path
if self.opts.only_test:
- components = [path] + self.opts.only_test[0]
+ components = [path] + [self.opts.only_test[0]]
subdir = os.path.join(*components)
extra_args = []
if not test:
extra_args = ['--no-execute']
-
+ if self.opts.use_perf in ('profile', 'all'):
+ extra_args += ['--param', 'profile=perf']
+
note('Testing...')
try:
self._check_call([lit_cmd,
@@ -524,6 +549,8 @@ class TestSuiteTest(BuiltinTest):
if only_test:
ignore.append('compile')
+ profiles_to_import = []
+
for test_data in data['tests']:
raw_name = test_data['name'].split(' :: ', 1)[1]
name = 'nts.' + raw_name.rsplit('.test', 1)[0]
@@ -543,6 +570,10 @@ class TestSuiteTest(BuiltinTest):
if 'metrics' in test_data:
for k,v in test_data['metrics'].items():
+ if k == 'profile':
+ profiles_to_import.append( (name, v) )
+ continue
+
if k not in LIT_METRIC_TO_LNT or LIT_METRIC_TO_LNT[k] in ignore:
continue
test_samples.append(
@@ -563,6 +594,23 @@ class TestSuiteTest(BuiltinTest):
[self._get_lnt_code(test_data['code'])],
test_info))
+ # Now import the profiles in parallel.
+ if profiles_to_import:
+ note('Importing %d profiles with %d threads...' %
+ (len(profiles_to_import), multiprocessing.cpu_count()))
+ TIMEOUT = 800
+ try:
+ pool = multiprocessing.Pool()
+ waiter = pool.map_async(_importProfile, profiles_to_import)
+ samples = waiter.get(TIMEOUT)
+ test_samples.extend([sample
+ for sample in samples
+ if sample is not None])
+ except multiprocessing.TimeoutError:
+ warning('Profiles had not completed importing after %s seconds.'
+ % TIMEOUT)
+ note('Aborting profile import and continuing')
+
if self.opts.single_result:
# If we got this far, the result we were looking for didn't exist.
raise RuntimeError("Result %s did not exist!" %
Modified: lnt/trunk/tests/runtest/Inputs/test-suite-cmake/fake-cmake
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/tests/runtest/Inputs/test-suite-cmake/fake-cmake?rev=263930&r1=263929&r2=263930&view=diff
==============================================================================
--- lnt/trunk/tests/runtest/Inputs/test-suite-cmake/fake-cmake (original)
+++ lnt/trunk/tests/runtest/Inputs/test-suite-cmake/fake-cmake Mon Mar 21 05:47:00 2016
@@ -9,7 +9,7 @@ fi
if [[ ! -f $1/CMakeLists.txt ]]; then
exit 1
else
- cp $1/fake-test $1/fake-results.json .
+ cp $1/fake-test $1/fake-results.json $1/fake-results-profile.json .
echo "Dummy" > CMakeCache.txt
mkdir subtest
cp $1/fake-test $1/fake-results.json subtest
Added: lnt/trunk/tests/runtest/Inputs/test-suite-cmake/fake-lit-profile
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/tests/runtest/Inputs/test-suite-cmake/fake-lit-profile?rev=263930&view=auto
==============================================================================
--- lnt/trunk/tests/runtest/Inputs/test-suite-cmake/fake-lit-profile (added)
+++ lnt/trunk/tests/runtest/Inputs/test-suite-cmake/fake-lit-profile Mon Mar 21 05:47:00 2016
@@ -0,0 +1,11 @@
+#!/usr/bin/python
+
+import argparse, shutil
+parser = argparse.ArgumentParser(description='dummy lit')
+parser.add_argument('-o')
+parser.add_argument('-j', type=int)
+parser.add_argument('bar')
+args, _ = parser.parse_known_args()
+
+shutil.copyfile(args.bar + '/fake-results-profile.json', args.o
+)
\ No newline at end of file
Propchange: lnt/trunk/tests/runtest/Inputs/test-suite-cmake/fake-lit-profile
------------------------------------------------------------------------------
svn:executable = *
Added: lnt/trunk/tests/runtest/Inputs/test-suite-cmake/fake-results-profile.json
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/tests/runtest/Inputs/test-suite-cmake/fake-results-profile.json?rev=263930&view=auto
==============================================================================
--- lnt/trunk/tests/runtest/Inputs/test-suite-cmake/fake-results-profile.json (added)
+++ lnt/trunk/tests/runtest/Inputs/test-suite-cmake/fake-results-profile.json Mon Mar 21 05:47:00 2016
@@ -0,0 +1,16 @@
+{
+ "tests": [
+ {
+ "name": "test-suite :: foo",
+ "code": "PASS",
+ "metrics": {
+ "compile_time": 1.3,
+ "exec_time": 1.4,
+ "score": 1.5,
+ "hash": "xyz",
+ "profile": "/tmp/I/Do/Not/Exist.perf_data",
+ "unknown": "unknown"
+ }
+ }
+ ]
+}
Modified: lnt/trunk/tests/runtest/test_suite.py
URL: http://llvm.org/viewvc/llvm-project/lnt/trunk/tests/runtest/test_suite.py?rev=263930&r1=263929&r2=263930&view=diff
==============================================================================
--- lnt/trunk/tests/runtest/test_suite.py (original)
+++ lnt/trunk/tests/runtest/test_suite.py Mon Mar 21 05:47:00 2016
@@ -260,7 +260,7 @@
# RUN: --use-cmake %S/Inputs/test-suite-cmake/fake-cmake \
# RUN: --use-make %S/Inputs/test-suite-cmake/fake-make \
# RUN: --use-lit %S/Inputs/test-suite-cmake/fake-lit \
-# RUN: --use-perf \
+# RUN: --use-perf=time \
# RUN: --verbose \
# RUN: > %t.log 2> %t.err
# RUN: FileCheck --check-prefix CHECK-USE-PERF < %t.err %s
@@ -331,3 +331,22 @@
# RUN: &> %t.cmake-cache2.err || true
# RUN: FileCheck --check-prefix CHECK-CACHE2 < %t.cmake-cache2.err %s
# CHECK-CACHE2: Could not find CMake cache file
+
+# Check importing profiles
+# RUN: lnt runtest test-suite \
+# RUN: --sandbox %t.SANDBOX \
+# RUN: --no-timestamp \
+# RUN: --test-suite %S/Inputs/test-suite-cmake \
+# RUN: --cc %{shared_inputs}/FakeCompilers/clang-r154331 \
+# RUN: --use-cmake %S/Inputs/test-suite-cmake/fake-cmake \
+# RUN: --use-make %S/Inputs/test-suite-cmake/fake-make \
+# RUN: --use-lit %S/Inputs/test-suite-cmake/fake-lit-profile \
+# RUN: --use-perf=all \
+# RUN: --verbose \
+# RUN: > %t.log 2> %t.err
+# RUN: FileCheck --check-prefix CHECK-USE-PERF-ALL < %t.err %s
+# CHECK-USE-PERF-ALL: Configuring with {
+# CHECK-USE-PERF-ALL: TEST_SUITE_USE_PERF: 'ON'
+# CHECK-USE-PERF-ALL: --param profile=perf
+# CHECK-USE-PERF-ALL: Importing 1 profiles with
+# CHECK-USE-PERF-ALL: Profile /tmp/I/Do/Not/Exist.perf_data does not exist
More information about the llvm-commits
mailing list