[test-suite] r266780 - lit: Dynamically load test modules

[Matthias Braun via llvm-commits]

Author: matze
Date: Tue Apr 19 12:49:12 2016
New Revision: 266780

URL: http://llvm.org/viewvc/llvm-project?rev=266780&view=rev
Log:
lit: Dynamically load test modules

You can now specify a list of test modules in lit.site.cfg.
This allows to add external test modules or do things like skip running
the tests and just collect compiletime and executable hashes.

Modified:
    test-suite/trunk/CMakeLists.txt
    test-suite/trunk/lit.cfg
    test-suite/trunk/lit.site.cfg.in
    test-suite/trunk/litsupport/test.py

Modified: test-suite/trunk/CMakeLists.txt
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/CMakeLists.txt?rev=266780&r1=266779&r2=266780&view=diff
==============================================================================
--- test-suite/trunk/CMakeLists.txt (original)
+++ test-suite/trunk/CMakeLists.txt Tue Apr 19 12:49:12 2016
@@ -163,6 +163,24 @@ add_subdirectory(SingleSource)
 add_subdirectory(MultiSource)
 add_subdirectory(External)
 
+set(LIT_MODULES "")
+list(APPEND LIT_MODULES run)
+list(APPEND LIT_MODULES codesize)
+list(APPEND LIT_MODULES hash)
+if(TEST_SUITE_RUN_UNDER)
+  list(APPEND LIT_MODULES run_under)
+endif()
+list(APPEND LIT_MODULES timeit)
+if(TEST_SUITE_REMOTE_HOST)
+  list(APPEND LIT_MODULES remote)
+endif()
+if(TEST_SUITE_PROFILE_GENERATE)
+  list(APPEND LIT_MODULES profilegen)
+endif()
+if(TEST_SUITE_TAKE_COMPILE_TIME)
+  list(APPEND LIT_MODULES compiletime)
+endif()
+
 # Produce lit.site.cfg
 configure_file("${PROJECT_SOURCE_DIR}/lit.site.cfg.in" "${CMAKE_BINARY_DIR}/lit.site.cfg")
 

Modified: test-suite/trunk/lit.cfg
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/lit.cfg?rev=266780&r1=266779&r2=266780&view=diff
==============================================================================
--- test-suite/trunk/lit.cfg (original)
+++ test-suite/trunk/lit.cfg Tue Apr 19 12:49:12 2016
@@ -1,7 +1,7 @@
-import site
-import os
-import json
 import logging
+import os
+import site
+import sys
 site.addsitedir(os.path.dirname(__file__))
 from litsupport import test
 
@@ -15,12 +15,15 @@ config.single_source = False
 if 'SSH_AUTH_SOCK' in os.environ:
     config.environment['SSH_AUTH_SOCK'] = os.environ['SSH_AUTH_SOCK']
 
+# Load previous test results so we can skip tests that did not change.
 previous_results_file = lit_config.params.get('previous', None)
 if previous_results_file:
+    import json
     config.previous_results = json.load(open(previous_results_file))
 else:
     config.previous_results = None
 
+# Setup logging
 logger = logging.getLogger()
 logger.setLevel(logging.DEBUG)
 file_log = logging.FileHandler("%s/test.log" % config.test_source_root,
@@ -30,3 +33,8 @@ logger.addHandler(file_log)
 console_log = logging.StreamHandler()
 console_log.setLevel(logging.WARNING)
 logger.addHandler(console_log)
+
+# Find and initialize lit modules.
+if lit_config.params.get('profile') == 'perf':
+    config.test_modules += ['perf']
+test.load_modules(config.test_modules)

Modified: test-suite/trunk/lit.site.cfg.in
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/lit.site.cfg.in?rev=266780&r1=266779&r2=266780&view=diff
==============================================================================
--- test-suite/trunk/lit.site.cfg.in (original)
+++ test-suite/trunk/lit.site.cfg.in Tue Apr 19 12:49:12 2016
@@ -11,5 +11,6 @@ config.strip_tool = "@CMAKE_STRIP@"
 config.profile_generate = @TEST_SUITE_PROFILE_GENERATE@
 config.llvm_profdata = "@TEST_SUITE_LLVM_PROFDATA@"
 config.llvm_size = "@TEST_SUITE_LLVM_SIZE@"
+config.test_modules = "@LIT_MODULES@".split(';')
 
 lit_config.load_config(config, "@CMAKE_SOURCE_DIR@/lit.cfg")

Modified: test-suite/trunk/litsupport/test.py
URL: http://llvm.org/viewvc/llvm-project/test-suite/trunk/litsupport/test.py?rev=266780&r1=266779&r2=266780&view=diff
==============================================================================
--- test-suite/trunk/litsupport/test.py (original)
+++ test-suite/trunk/litsupport/test.py Tue Apr 19 12:49:12 2016
@@ -1,7 +1,8 @@
-import os
+import importlib
 import lit
 import lit.util
 import logging
+import os
 from lit.formats import FileBasedTest
 from lit.TestRunner import getTempPaths
 from lit import Test
@@ -21,6 +22,7 @@ from litsupport import timeit
 
 
 SKIPPED = lit.Test.ResultCode('SKIPPED', False)
+modules = []
 
 
 class TestContext:
@@ -36,6 +38,22 @@ class TestContext:
         self.tmpBase = tmpBase
 
 
+def load_modules(test_modules):
+    for name in test_modules:
+        modulename = 'litsupport.%s' % name
+        try:
+            module = importlib.import_module(modulename)
+        except ImportError as e:
+            logging.error("Could not import module '%s'" % modulename)
+            sys.exit(1)
+        if not hasattr(module, 'mutatePlan'):
+            logging.error("Invalid test module '%s': No mutatePlan() function."
+                          % modulename)
+            sys.exit(1)
+        logging.info("Loaded test module %s" % module.__file__)
+        modules.append(module)
+
+
 class TestSuiteTest(FileBasedTest):
     def __init__(self):
         super(TestSuiteTest, self).__init__()
@@ -64,19 +82,9 @@ class TestSuiteTest(FileBasedTest):
                 result.addMetric('hash', val)
                 return result
 
-        # Prepare test plan
-        run.mutatePlan(context, plan)
-        run_under.mutatePlan(context, plan)
-        timeit.mutatePlan(context, plan)
-        compiletime.mutatePlan(context, plan)
-        codesize.mutatePlan(context, plan)
-        hash.mutatePlan(context, plan)
-        if config.profile_generate:
-            profilegen.mutatePlan(context, plan)
-        if config.remote_host:
-            remote.mutatePlan(context, plan)
-        if litConfig.params.get('profile') == 'perf':
-            perf.mutatePlan(context, plan)
+        # Let test modules modify the test plan.
+        for module in modules:
+            module.mutatePlan(context, plan)
 
         # Execute Test plan
         result = testplan.executePlanTestResult(context, plan)