r339629 - [analyzer] [NFC] Introduce separate targets for testing the analyzer: check-clang-analyzer and check-clang-analyzer-z3

[George Karpenkov via cfe-commits]

Author: george.karpenkov
Date: Mon Aug 13 16:12:43 2018
New Revision: 339629

URL: http://llvm.org/viewvc/llvm-project?rev=339629&view=rev
Log:
[analyzer] [NFC] Introduce separate targets for testing the analyzer: check-clang-analyzer and check-clang-analyzer-z3

Current testing setup for analyzer tests with Z3 is rather inconvenient:

There's no way to run the analyzer tests separately (I use
LIT_FILTER=Analysis ninja check-clang, but a direct target is nicer).

When Clang is built with Z3 support, there's no way to *not* run tests
with Z3 solver, and this is often desired, as tests with Z3 solver take
a very long time.

This patch introduces two extra targets:

 - check-clang-analyzer
 - check-clang-analyzer-z3

which solve those problems.

Differential Revision: https://reviews.llvm.org/D50594

Modified:
    cfe/trunk/test/Analysis/analyzer_test.py
    cfe/trunk/test/Analysis/lit.local.cfg
    cfe/trunk/test/CMakeLists.txt
    cfe/trunk/test/lit.site.cfg.py.in

Modified: cfe/trunk/test/Analysis/analyzer_test.py
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/Analysis/analyzer_test.py?rev=339629&r1=339628&r2=339629&view=diff
==============================================================================
--- cfe/trunk/test/Analysis/analyzer_test.py (original)
+++ cfe/trunk/test/Analysis/analyzer_test.py Mon Aug 13 16:12:43 2018
@@ -4,6 +4,10 @@ import lit.TestRunner
 # Custom format class for static analyzer tests
 class AnalyzerTest(lit.formats.ShTest):
 
+    def __init__(self, execute_external, use_z3_solver=False):
+        super(AnalyzerTest, self).__init__(execute_external)
+        self.use_z3_solver = use_z3_solver
+
     def execute(self, test, litConfig):
         results = []
 
@@ -19,7 +23,8 @@ class AnalyzerTest(lit.formats.ShTest):
                 return results[-1]
 
         # If z3 backend available, add an additional run line for it
-        if test.config.clang_staticanalyzer_z3 == '1':
+        if self.use_z3_solver == '1':
+            assert(test.config.clang_staticanalyzer_z3 == '1')
             results.append(self.executeWithAnalyzeSubstitution(
                 saved_test, litConfig, '-analyzer-constraints=z3 -DANALYZER_CM_Z3'))
 

Modified: cfe/trunk/test/Analysis/lit.local.cfg
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/Analysis/lit.local.cfg?rev=339629&r1=339628&r2=339629&view=diff
==============================================================================
--- cfe/trunk/test/Analysis/lit.local.cfg (original)
+++ cfe/trunk/test/Analysis/lit.local.cfg Mon Aug 13 16:12:43 2018
@@ -7,7 +7,7 @@ import site
 site.addsitedir(os.path.dirname(__file__))
 import analyzer_test
 config.test_format = analyzer_test.AnalyzerTest(
-        config.test_format.execute_external)
+        config.test_format.execute_external, config.use_z3_solver)
 
 if not config.root.clang_staticanalyzer:
     config.unsupported = True

Modified: cfe/trunk/test/CMakeLists.txt
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/CMakeLists.txt?rev=339629&r1=339628&r2=339629&view=diff
==============================================================================
--- cfe/trunk/test/CMakeLists.txt (original)
+++ cfe/trunk/test/CMakeLists.txt Mon Aug 13 16:12:43 2018
@@ -88,8 +88,15 @@ endif ()
 
 set(CLANG_TEST_PARAMS
   clang_site_config=${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg
+  USE_Z3_SOLVER=0
   )
 
+set(ANALYZER_TEST_PARAMS
+  USE_Z3_SOLVER=0)
+
+set(ANALYZER_TEST_PARAMS_Z3
+  USE_Z3_SOLVER=1)
+
 if( NOT CLANG_BUILT_STANDALONE )
   list(APPEND CLANG_TEST_DEPS
     llvm-config
@@ -126,6 +133,24 @@ add_lit_testsuite(check-clang "Running t
   )
 set_target_properties(check-clang PROPERTIES FOLDER "Clang tests")
 
+if (CLANG_ENABLE_STATIC_ANALYZER)
+  add_lit_testsuite(check-clang-analyzer "Running the Clang analyzer tests"
+    ${CMAKE_CURRENT_BINARY_DIR}/Analysis
+    PARAMS ${ANALYZER_TEST_PARAMS}
+    DEPENDS ${CLANG_TEST_DEPS})
+  set_target_properties(check-clang-analyzer PROPERTIES FOLDER "Clang tests")
+
+
+  if (CLANG_ANALYZER_WITH_Z3)
+    add_lit_testsuite(check-clang-analyzer-z3 "Running the Clang analyzer tests, using Z3 as a solver"
+      ${CMAKE_CURRENT_BINARY_DIR}/Analysis
+      PARAMS ${ANALYZER_TEST_PARAMS_Z3}
+      DEPENDS ${CLANG_TEST_DEPS})
+    set_target_properties(check-clang-analyzer-z3 PROPERTIES FOLDER "Clang tests")
+  endif()
+
+endif()
+
 add_lit_testsuites(CLANG ${CMAKE_CURRENT_SOURCE_DIR}
   PARAMS ${CLANG_TEST_PARAMS}
   DEPENDS ${CLANG_TEST_DEPS}

Modified: cfe/trunk/test/lit.site.cfg.py.in
URL: http://llvm.org/viewvc/llvm-project/cfe/trunk/test/lit.site.cfg.py.in?rev=339629&r1=339628&r2=339629&view=diff
==============================================================================
--- cfe/trunk/test/lit.site.cfg.py.in (original)
+++ cfe/trunk/test/lit.site.cfg.py.in Mon Aug 13 16:12:43 2018
@@ -26,6 +26,7 @@ config.enable_shared = @ENABLE_SHARED@
 config.enable_backtrace = @ENABLE_BACKTRACES@
 config.host_arch = "@HOST_ARCH@"
 config.python_executable = "@PYTHON_EXECUTABLE@"
+config.use_z3_solver = "@USE_Z3_SOLVER@"
 
 # Support substitution of the tools and libs dirs with user parameters. This is
 # used when we can't determine the tool dir at configuration time.
@@ -34,6 +35,7 @@ try:
     config.llvm_tools_dir = config.llvm_tools_dir % lit_config.params
     config.llvm_shlib_dir = config.llvm_shlib_dir % lit_config.params
     config.llvm_libs_dir = config.llvm_libs_dir % lit_config.params
+    config.use_z3_solver = lit_config.params['USE_Z3_SOLVER']
 except KeyError:
     e = sys.exc_info()[1]
     key, = e.args