[libcxx] r346888 - Rename cxx-benchmark-unittests target and convert to LIT.
Eric Fiselier
eric at efcs.ca
Wed Nov 14 12:38:47 PST 2018
Author: ericwf
Date: Wed Nov 14 12:38:46 2018
New Revision: 346888
URL: http://llvm.org/viewvc/llvm-project?rev=346888&view=rev
Log:
Rename cxx-benchmark-unittests target and convert to LIT.
This patch renames the cxx-benchmark-unittests to check-cxx-benchmarks
and converts the target to use LIT in order to make the tests run faster
and provide better output.
In particular this runs each benchmark in a suite one by one, allowing
more parallelism while ensuring output isn't garbage with multiple threads.
Additionally, it adds the CMake flag '-DLIBCXX_BENCHMARK_TEST_ARGS=<list>'
to specify what options are passed when running the benchmarks.
Added:
libcxx/trunk/benchmarks/lit.cfg.py
libcxx/trunk/benchmarks/lit.site.cfg.py.in
libcxx/trunk/utils/libcxx/test/googlebenchmark.py
Modified:
libcxx/trunk/CMakeLists.txt
libcxx/trunk/benchmarks/CMakeLists.txt
libcxx/trunk/docs/BuildingLibcxx.rst
libcxx/trunk/test/CMakeLists.txt
Modified: libcxx/trunk/CMakeLists.txt
URL: http://llvm.org/viewvc/llvm-project/libcxx/trunk/CMakeLists.txt?rev=346888&r1=346887&r2=346888&view=diff
==============================================================================
--- libcxx/trunk/CMakeLists.txt (original)
+++ libcxx/trunk/CMakeLists.txt Wed Nov 14 12:38:46 2018
@@ -83,6 +83,11 @@ option(LIBCXX_INCLUDE_TESTS "Build the l
# Benchmark options -----------------------------------------------------------
option(LIBCXX_INCLUDE_BENCHMARKS "Build the libc++ benchmarks and their dependencies" ON)
+
+set(LIBCXX_BENCHMARK_TEST_ARGS_DEFAULT --benchmark_min_time=0.01)
+set(LIBCXX_BENCHMARK_TEST_ARGS "${LIBCXX_BENCHMARK_TEST_ARGS_DEFAULT}" CACHE STRING
+ "Arguments to pass when running the benchmarks using check-cxx-benchmarks")
+
set(LIBCXX_BENCHMARK_NATIVE_STDLIB "" CACHE STRING
"Build the benchmarks against the specified native STL.
The value must be one of libc++/libstdc++")
@@ -766,6 +771,18 @@ include_directories(include)
add_subdirectory(include)
add_subdirectory(lib)
+set(LIBCXX_TEST_DEPS "")
+
+if (LIBCXX_ENABLE_EXPERIMENTAL_LIBRARY)
+ list(APPEND LIBCXX_TEST_DEPS cxx_experimental)
+endif()
+if (LIBCXX_ENABLE_FILESYSTEM)
+ list(APPEND LIBCXX_TEST_DEPS cxx_filesystem)
+endif()
+
+if (LIBCXX_BUILD_EXTERNAL_THREAD_LIBRARY)
+ list(APPEND LIBCXX_TEST_DEPS cxx_external_threads)
+endif()
if (LIBCXX_INCLUDE_BENCHMARKS)
add_subdirectory(benchmarks)
Modified: libcxx/trunk/benchmarks/CMakeLists.txt
URL: http://llvm.org/viewvc/llvm-project/libcxx/trunk/benchmarks/CMakeLists.txt?rev=346888&r1=346887&r2=346888&view=diff
==============================================================================
--- libcxx/trunk/benchmarks/CMakeLists.txt (original)
+++ libcxx/trunk/benchmarks/CMakeLists.txt Wed Nov 14 12:38:46 2018
@@ -180,15 +180,22 @@ foreach(test_path ${BENCHMARK_TESTS})
add_benchmark_test(${test_name} ${test_file})
endforeach()
+if (LIBCXX_INCLUDE_TESTS)
+ include(AddLLVM)
-add_custom_target(cxx-benchmark-unittests)
-foreach(libcxx_tg ${libcxx_benchmark_targets})
- message("Adding test ${libcxx_tg}")
- # Add a target that runs the benchmark for the smallest possible time, simply so we get test
- # and sanitizer coverage on the targets.
- add_custom_target(${libcxx_tg}_test
- COMMAND ${libcxx_tg} --benchmark_min_time=0.01
- COMMENT "Running test ${libcxx_tg}"
- )
- add_dependencies(cxx-benchmark-unittests ${libcxx_tg}_test)
-endforeach()
+ if (NOT DEFINED LIBCXX_TEST_DEPS)
+ message(FATAL_ERROR "Expected LIBCXX_TEST_DEPS to be defined")
+ endif()
+
+ configure_lit_site_cfg(
+ ${CMAKE_CURRENT_SOURCE_DIR}/lit.site.cfg.py.in
+ ${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg.py)
+
+ set(BENCHMARK_LIT_ARGS "--show-all --show-xfail --show-unsupported ${LIT_ARGS_DEFAULT}")
+
+ add_lit_testsuite(check-cxx-benchmarks
+ "Running libcxx benchmarks tests"
+ ${CMAKE_CURRENT_BINARY_DIR}
+ DEPENDS cxx-benchmarks ${LIBCXX_TEST_DEPS}
+ ARGS ${BENCHMARK_LIT_ARGS})
+endif()
Added: libcxx/trunk/benchmarks/lit.cfg.py
URL: http://llvm.org/viewvc/llvm-project/libcxx/trunk/benchmarks/lit.cfg.py?rev=346888&view=auto
==============================================================================
--- libcxx/trunk/benchmarks/lit.cfg.py (added)
+++ libcxx/trunk/benchmarks/lit.cfg.py Wed Nov 14 12:38:46 2018
@@ -0,0 +1,23 @@
+# -*- Python -*- vim: set ft=python ts=4 sw=4 expandtab tw=79:
+# Configuration file for the 'lit' test runner.
+import os
+import site
+
+site.addsitedir(os.path.join(os.path.dirname(os.path.dirname(__file__)), 'utils'))
+from libcxx.test.googlebenchmark import GoogleBenchmark
+
+# Tell pylint that we know config and lit_config exist somewhere.
+if 'PYLINT_IMPORT' in os.environ:
+ config = object()
+ lit_config = object()
+
+# name: The name of this test suite.
+config.name = 'libc++ benchmarks'
+config.suffixes = []
+
+config.test_exec_root = os.path.join(config.libcxx_obj_root, 'benchmarks')
+config.test_source_root = config.test_exec_root
+
+config.test_format = GoogleBenchmark(test_sub_dirs='.',
+ test_suffix='.libcxx.out',
+ benchmark_args=config.benchmark_args)
\ No newline at end of file
Added: libcxx/trunk/benchmarks/lit.site.cfg.py.in
URL: http://llvm.org/viewvc/llvm-project/libcxx/trunk/benchmarks/lit.site.cfg.py.in?rev=346888&view=auto
==============================================================================
--- libcxx/trunk/benchmarks/lit.site.cfg.py.in (added)
+++ libcxx/trunk/benchmarks/lit.site.cfg.py.in Wed Nov 14 12:38:46 2018
@@ -0,0 +1,10 @@
+ at LIT_SITE_CFG_IN_HEADER@
+
+import sys
+
+config.libcxx_src_root = "@LIBCXX_SOURCE_DIR@"
+config.libcxx_obj_root = "@LIBCXX_BINARY_DIR@"
+config.benchmark_args = "@LIBCXX_BENCHMARK_TEST_ARGS@".split(';')
+
+# Let the main config do the real work.
+lit_config.load_config(config, "@LIBCXX_SOURCE_DIR@/benchmarks/lit.cfg.py")
\ No newline at end of file
Modified: libcxx/trunk/docs/BuildingLibcxx.rst
URL: http://llvm.org/viewvc/llvm-project/libcxx/trunk/docs/BuildingLibcxx.rst?rev=346888&r1=346887&r2=346888&view=diff
==============================================================================
--- libcxx/trunk/docs/BuildingLibcxx.rst (original)
+++ libcxx/trunk/docs/BuildingLibcxx.rst Wed Nov 14 12:38:46 2018
@@ -316,6 +316,15 @@ libc++ Feature Options
Build the libc++ benchmark tests and the Google Benchmark library needed
to support them.
+.. option:: LIBCXX_BENCHMARK_TEST_ARGS:STRING
+
+ **Default**: ``--benchmark_min_time=0.01``
+
+ A semicolon list of arguments to pass when running the libc++ benchmarks using the
+ ``check-cxx-benchmarks`` rule. By default we run the benchmarks for a very short amount of time,
+ since the primary use of ``check-cxx-benchmarks`` is to get test and sanitizer coverage, not to
+ get accurate measurements.
+
.. option:: LIBCXX_BENCHMARK_NATIVE_STDLIB:STRING
**Default**:: ``""``
Modified: libcxx/trunk/test/CMakeLists.txt
URL: http://llvm.org/viewvc/llvm-project/libcxx/trunk/test/CMakeLists.txt?rev=346888&r1=346887&r2=346888&view=diff
==============================================================================
--- libcxx/trunk/test/CMakeLists.txt (original)
+++ libcxx/trunk/test/CMakeLists.txt Wed Nov 14 12:38:46 2018
@@ -55,17 +55,8 @@ set(LIBCXX_EXECUTOR "None" CACHE STRING
set(AUTO_GEN_COMMENT "## Autogenerated by libcxx configuration.\n# Do not edit!")
-set(LIBCXX_TEST_DEPS "")
-
-if (LIBCXX_ENABLE_EXPERIMENTAL_LIBRARY)
- list(APPEND LIBCXX_TEST_DEPS cxx_experimental)
-endif()
-if (LIBCXX_ENABLE_FILESYSTEM)
- list(APPEND LIBCXX_TEST_DEPS cxx_filesystem)
-endif()
-
-if (LIBCXX_BUILD_EXTERNAL_THREAD_LIBRARY)
- list(APPEND LIBCXX_TEST_DEPS cxx_external_threads)
+if (NOT DEFINED LIBCXX_TEST_DEPS)
+ message(FATAL_ERROR "Expected LIBCXX_TEST_DEPS to be defined")
endif()
if (LIBCXX_INCLUDE_TESTS)
Added: libcxx/trunk/utils/libcxx/test/googlebenchmark.py
URL: http://llvm.org/viewvc/llvm-project/libcxx/trunk/utils/libcxx/test/googlebenchmark.py?rev=346888&view=auto
==============================================================================
--- libcxx/trunk/utils/libcxx/test/googlebenchmark.py (added)
+++ libcxx/trunk/utils/libcxx/test/googlebenchmark.py Wed Nov 14 12:38:46 2018
@@ -0,0 +1,122 @@
+from __future__ import absolute_import
+import os
+import subprocess
+import sys
+
+import lit.Test
+import lit.TestRunner
+import lit.util
+from lit.formats.base import TestFormat
+
+kIsWindows = sys.platform in ['win32', 'cygwin']
+
+class GoogleBenchmark(TestFormat):
+ def __init__(self, test_sub_dirs, test_suffix, benchmark_args=[]):
+ self.benchmark_args = list(benchmark_args)
+ self.test_sub_dirs = os.path.normcase(str(test_sub_dirs)).split(';')
+
+ # On Windows, assume tests will also end in '.exe'.
+ exe_suffix = str(test_suffix)
+ if kIsWindows:
+ exe_suffix += '.exe'
+
+ # Also check for .py files for testing purposes.
+ self.test_suffixes = {exe_suffix, test_suffix + '.py'}
+
+ def getBenchmarkTests(self, path, litConfig, localConfig):
+ """getBenchmarkTests(path) - [name]
+
+ Return the tests available in gtest executable.
+
+ Args:
+ path: String path to a gtest executable
+ litConfig: LitConfig instance
+ localConfig: TestingConfig instance"""
+
+ # TODO: allow splitting tests according to the "benchmark family" so
+ # the output for a single family of tests all belongs to the same test
+ # target.
+ list_test_cmd = [path, '--benchmark_list_tests']
+ try:
+ output = subprocess.check_output(list_test_cmd,
+ env=localConfig.environment)
+ except subprocess.CalledProcessError as exc:
+ litConfig.warning(
+ "unable to discover google-benchmarks in %r: %s. Process output: %s"
+ % (path, sys.exc_info()[1], exc.output))
+ raise StopIteration
+
+ nested_tests = []
+ for ln in output.splitlines(False): # Don't keep newlines.
+ ln = lit.util.to_string(ln)
+ if not ln.strip():
+ continue
+
+ index = 0
+ while ln[index*2:index*2+2] == ' ':
+ index += 1
+ while len(nested_tests) > index:
+ nested_tests.pop()
+
+ ln = ln[index*2:]
+ if ln.endswith('.'):
+ nested_tests.append(ln)
+ elif any([name.startswith('DISABLED_')
+ for name in nested_tests + [ln]]):
+ # Gtest will internally skip these tests. No need to launch a
+ # child process for it.
+ continue
+ else:
+ yield ''.join(nested_tests) + ln
+
+ def getTestsInDirectory(self, testSuite, path_in_suite,
+ litConfig, localConfig):
+ source_path = testSuite.getSourcePath(path_in_suite)
+ for subdir in self.test_sub_dirs:
+ dir_path = os.path.join(source_path, subdir)
+ if not os.path.isdir(dir_path):
+ continue
+ for fn in lit.util.listdir_files(dir_path,
+ suffixes=self.test_suffixes):
+ # Discover the tests in this executable.
+ execpath = os.path.join(source_path, subdir, fn)
+ testnames = self.getBenchmarkTests(execpath, litConfig, localConfig)
+ for testname in testnames:
+ testPath = path_in_suite + (subdir, fn, testname)
+ yield lit.Test.Test(testSuite, testPath, localConfig,
+ file_path=execpath)
+
+ def execute(self, test, litConfig):
+ testPath,testName = os.path.split(test.getSourcePath())
+ while not os.path.exists(testPath):
+ # Handle GTest parametrized and typed tests, whose name includes
+ # some '/'s.
+ testPath, namePrefix = os.path.split(testPath)
+ testName = namePrefix + '/' + testName
+
+ cmd = [testPath, '--benchmark_filter=%s$' % testName ] + self.benchmark_args
+
+ if litConfig.noExecute:
+ return lit.Test.PASS, ''
+
+ try:
+ out, err, exitCode = lit.util.executeCommand(
+ cmd, env=test.config.environment,
+ timeout=litConfig.maxIndividualTestTime)
+ except lit.util.ExecuteCommandTimeoutException:
+ return (lit.Test.TIMEOUT,
+ 'Reached timeout of {} seconds'.format(
+ litConfig.maxIndividualTestTime)
+ )
+
+ if exitCode:
+ return lit.Test.FAIL, out + err
+
+ passing_test_line = testName
+ if passing_test_line not in out:
+ msg = ('Unable to find %r in google benchmark output:\n\n%s%s' %
+ (passing_test_line, out, err))
+ return lit.Test.UNRESOLVED, msg
+
+ return lit.Test.PASS, err + out
+
More information about the libcxx-commits
mailing list