[llvm] 47f59df - Revert "Reland "[lit] Use sharding for GoogleTest format""
Alex Brachet via llvm-commits
llvm-commits at lists.llvm.org
Mon Apr 4 09:34:37 PDT 2022
Author: Alex Brachet
Date: 2022-04-04T16:34:28Z
New Revision: 47f59df892893659d8e74e6afaf7c057a100bb1b
URL: https://github.com/llvm/llvm-project/commit/47f59df892893659d8e74e6afaf7c057a100bb1b
DIFF: https://github.com/llvm/llvm-project/commit/47f59df892893659d8e74e6afaf7c057a100bb1b.diff
LOG: Revert "Reland "[lit] Use sharding for GoogleTest format""
This reverts commit 948f3deca91a66caf4a618f826ff6de8277bed9c.
Added:
llvm/utils/lit/tests/Inputs/googletest-upstream-format/DummySubDir/OneTest.py
llvm/utils/lit/tests/Inputs/googletest-upstream-format/lit.cfg
llvm/utils/lit/tests/googletest-upstream-format.py
Modified:
llvm/unittests/Support/CrashRecoveryTest.cpp
llvm/unittests/Support/ProgramTest.cpp
llvm/utils/lit/lit/Test.py
llvm/utils/lit/lit/TestingConfig.py
llvm/utils/lit/lit/formats/googletest.py
llvm/utils/lit/lit/main.py
llvm/utils/lit/tests/Inputs/googletest-format/DummySubDir/OneTest.py
llvm/utils/lit/tests/Inputs/googletest-timeout/DummySubDir/OneTest.py
llvm/utils/lit/tests/Inputs/googletest-timeout/lit.cfg
llvm/utils/lit/tests/googletest-format.py
llvm/utils/lit/tests/googletest-timeout.py
Removed:
llvm/utils/lit/tests/Inputs/googletest-crash/DummySubDir/OneTest.py
llvm/utils/lit/tests/Inputs/googletest-crash/lit.cfg
llvm/utils/lit/tests/googletest-crash.py
################################################################################
diff --git a/llvm/unittests/Support/CrashRecoveryTest.cpp b/llvm/unittests/Support/CrashRecoveryTest.cpp
index c7350452034a3..e95513eb28416 100644
--- a/llvm/unittests/Support/CrashRecoveryTest.cpp
+++ b/llvm/unittests/Support/CrashRecoveryTest.cpp
@@ -178,11 +178,6 @@ TEST(CrashRecoveryTest, UnixCRCReturnCode) {
int Res = setenv("LLVM_CRC_UNIXCRCRETURNCODE", "1", 0);
ASSERT_EQ(Res, 0);
- Res = unsetenv("GTEST_SHARD_INDEX");
- ASSERT_EQ(Res, 0);
- Res = unsetenv("GTEST_TOTAL_SHARDS");
- ASSERT_EQ(Res, 0);
-
std::string Error;
bool ExecutionFailed;
int RetCode = ExecuteAndWait(Executable, argv, {}, {}, 0, 0, &Error,
diff --git a/llvm/unittests/Support/ProgramTest.cpp b/llvm/unittests/Support/ProgramTest.cpp
index f93a77ab67057..fdd0478f70515 100644
--- a/llvm/unittests/Support/ProgramTest.cpp
+++ b/llvm/unittests/Support/ProgramTest.cpp
@@ -95,9 +95,7 @@ class ProgramEnvTest : public testing::Test {
};
while (*EnvP != nullptr) {
- auto S = prepareEnvVar(*EnvP);
- if (!StringRef(S).startswith("GTEST_"))
- EnvTable.emplace_back(S);
+ EnvTable.emplace_back(prepareEnvVar(*EnvP));
++EnvP;
}
}
diff --git a/llvm/utils/lit/lit/Test.py b/llvm/utils/lit/lit/Test.py
index dc1c66e896c54..77b9c235e40d4 100644
--- a/llvm/utils/lit/lit/Test.py
+++ b/llvm/utils/lit/lit/Test.py
@@ -219,12 +219,11 @@ def getExecPath(self, components):
class Test:
"""Test - Information on a single test instance."""
- def __init__(self, suite, path_in_suite, config, file_path = None, gtest_json_file = None):
+ def __init__(self, suite, path_in_suite, config, file_path = None):
self.suite = suite
self.path_in_suite = path_in_suite
self.config = config
self.file_path = file_path
- self.gtest_json_file = gtest_json_file
# A list of conditions under which this test is expected to fail.
# Each condition is a boolean expression of features and target
@@ -259,7 +258,7 @@ def __init__(self, suite, path_in_suite, config, file_path = None, gtest_json_fi
# The previous test elapsed time, if applicable.
self.previous_elapsed = 0.0
- if suite.test_times and '/'.join(path_in_suite) in suite.test_times:
+ if '/'.join(path_in_suite) in suite.test_times:
time = suite.test_times['/'.join(path_in_suite)]
self.previous_elapsed = abs(time)
self.previous_failure = time < 0
diff --git a/llvm/utils/lit/lit/TestingConfig.py b/llvm/utils/lit/lit/TestingConfig.py
index a9660b39c9cdb..e80369377857b 100644
--- a/llvm/utils/lit/lit/TestingConfig.py
+++ b/llvm/utils/lit/lit/TestingConfig.py
@@ -28,7 +28,7 @@ def fromdefaults(litConfig):
'TMPDIR', 'TMP', 'TEMP', 'TEMPDIR', 'AVRLIT_BOARD',
'AVRLIT_PORT', 'FILECHECK_OPTS', 'VCINSTALLDIR',
'VCToolsinstallDir', 'VSINSTALLDIR', 'WindowsSdkDir',
- 'WindowsSDKLibVersion', 'SOURCE_DATE_EPOCH','GTEST_FILTER']
+ 'WindowsSDKLibVersion', 'SOURCE_DATE_EPOCH']
if sys.platform == 'win32':
pass_vars.append('COMSPEC')
diff --git a/llvm/utils/lit/lit/formats/googletest.py b/llvm/utils/lit/lit/formats/googletest.py
index 4ce14b78fe10d..5329f5773e54c 100644
--- a/llvm/utils/lit/lit/formats/googletest.py
+++ b/llvm/utils/lit/lit/formats/googletest.py
@@ -1,8 +1,8 @@
from __future__ import absolute_import
-import json
-import math
import os
+import re
import shlex
+import subprocess
import sys
import lit.Test
@@ -25,19 +25,74 @@ def __init__(self, test_sub_dirs, test_suffix, run_under = []):
self.test_suffixes = {exe_suffix, test_suffix + '.py'}
self.run_under = run_under
- def get_num_tests(self, path, localConfig):
- cmd = [path, '--gtest_list_tests', '--gtest_filter=-*DISABLED_*']
- if cmd[0].endswith('.py'):
- cmd = [sys.executable] + cmd
- out, _, exitCode = lit.util.executeCommand(cmd, env=localConfig.environment)
- if exitCode == 0:
- return sum(map(lambda line: line.startswith(' '), out.splitlines()))
- return None
+ def getGTestTests(self, path, litConfig, localConfig):
+ """getGTestTests(path) - [name]
+
+ Return the tests available in gtest executable.
+
+ Args:
+ path: String path to a gtest executable
+ litConfig: LitConfig instance
+ localConfig: TestingConfig instance"""
+
+ list_test_cmd = self.prepareCmd([path, '--gtest_list_tests'])
+
+ try:
+ output = subprocess.check_output(list_test_cmd,
+ env=localConfig.environment)
+ except subprocess.CalledProcessError as exc:
+ litConfig.warning(
+ "unable to discover google-tests in %r: %s. Process output: %s"
+ % (path, sys.exc_info()[1], exc.output))
+ # This doesn't look like a valid gtest file. This can
+ # have a number of causes, none of them good. For
+ # instance, we could have created a broken executable.
+ # Alternatively, someone has cruft in their test
+ # directory. If we don't return a test here, then no
+ # failures will get reported, so return a dummy test name
+ # so that the failure is reported later.
+ yield 'failed_to_discover_tests_from_gtest'
+ return
+
+ upstream_prefix = re.compile('Running main\(\) from .*gtest_main\.cc')
+ nested_tests = []
+ for ln in output.splitlines(False): # Don't keep newlines.
+ ln = lit.util.to_string(ln)
+
+ if upstream_prefix.fullmatch(ln):
+ # Upstream googletest prints this to stdout prior to running
+ # tests. LLVM removed that print statement in r61540, but we
+ # handle it here in case upstream googletest is being used.
+ continue
+
+ # The test name list includes trailing comments beginning with
+ # a '#' on some lines, so skip those. We don't support test names
+ # that use escaping to embed '#' into their name as the names come
+ # from C++ class and method names where such things are hard and
+ # uninteresting to support.
+ ln = ln.split('#', 1)[0].rstrip()
+ if not ln.lstrip():
+ continue
+
+ index = 0
+ while ln[index*2:index*2+2] == ' ':
+ index += 1
+ while len(nested_tests) > index:
+ nested_tests.pop()
+
+ ln = ln[index*2:]
+ if ln.endswith('.'):
+ nested_tests.append(ln)
+ elif any([name.startswith('DISABLED_')
+ for name in nested_tests + [ln]]):
+ # Gtest will internally skip these tests. No need to launch a
+ # child process for it.
+ continue
+ else:
+ yield ''.join(nested_tests) + ln
def getTestsInDirectory(self, testSuite, path_in_suite,
litConfig, localConfig):
- init_shard_size = 512 # number of tests in a shard
- core_count = lit.util.usable_core_count()
source_path = testSuite.getSourcePath(path_in_suite)
for subdir in self.test_sub_dirs:
dir_path = os.path.join(source_path, subdir)
@@ -47,40 +102,13 @@ def getTestsInDirectory(self, testSuite, path_in_suite,
suffixes=self.test_suffixes):
# Discover the tests in this executable.
execpath = os.path.join(source_path, subdir, fn)
- num_tests = self.get_num_tests(execpath, localConfig)
- if num_tests is not None:
- # Compute the number of shards.
- shard_size = init_shard_size
- nshard = int(math.ceil(num_tests/shard_size))
- while nshard < core_count and shard_size > 1:
- shard_size = shard_size//2
- nshard = int(math.ceil(num_tests/shard_size))
-
- # Create one lit test for each shard.
- for idx in range(nshard):
- testPath = path_in_suite + (subdir, fn,
- str(idx), str(nshard))
- json_file = '-'.join([execpath, testSuite.config.name,
- str(os.getpid()), str(idx),
- str(nshard)]) + '.json'
- yield lit.Test.Test(testSuite, testPath, localConfig,
- file_path=execpath,
- gtest_json_file=json_file)
- else:
- # This doesn't look like a valid gtest file. This can
- # have a number of causes, none of them good. For
- # instance, we could have created a broken executable.
- # Alternatively, someone has cruft in their test
- # directory. If we don't return a test here, then no
- # failures will get reported, so return a dummy test name
- # so that the failure is reported later.
- testPath = path_in_suite + (subdir, fn, 'failed_to_discover_tests_from_gtest')
- yield lit.Test.Test(testSuite, testPath, localConfig, file_path=execpath)
+ testnames = self.getGTestTests(execpath, litConfig, localConfig)
+ for testname in testnames:
+ testPath = path_in_suite + (subdir, fn, testname)
+ yield lit.Test.Test(testSuite, testPath, localConfig,
+ file_path=execpath)
def execute(self, test, litConfig):
- if test.gtest_json_file is None:
- return lit.Test.FAIL, ''
-
testPath,testName = os.path.split(test.getSourcePath())
while not os.path.exists(testPath):
# Handle GTest parametrized and typed tests, whose name includes
@@ -88,12 +116,7 @@ def execute(self, test, litConfig):
testPath, namePrefix = os.path.split(testPath)
testName = namePrefix + '/' + testName
- testName,total_shards = os.path.split(testName)
- testName,shard_idx = os.path.split(testName)
- shard_env = {'GTEST_COLOR':'no','GTEST_TOTAL_SHARDS':total_shards, 'GTEST_SHARD_INDEX':shard_idx, 'GTEST_OUTPUT':'json:'+test.gtest_json_file}
- test.config.environment.update(shard_env)
-
- cmd = [testPath]
+ cmd = [testPath, '--gtest_filter=' + testName]
cmd = self.prepareCmd(cmd)
if litConfig.useValgrind:
cmd = litConfig.valgrindArgs + cmd
@@ -101,43 +124,30 @@ def execute(self, test, litConfig):
if litConfig.noExecute:
return lit.Test.PASS, ''
- shard_envs= '\n'.join([k + '=' + v for k, v in shard_env.items()])
- shard_header = f"Script(shard):\n--\n{shard_envs}\n{' '.join(cmd)}\n--\n"
+ header = f"Script:\n--\n{' '.join(cmd)}\n--\n"
try:
- _, _, exitCode = lit.util.executeCommand(
+ out, err, exitCode = lit.util.executeCommand(
cmd, env=test.config.environment,
timeout=litConfig.maxIndividualTestTime)
except lit.util.ExecuteCommandTimeoutException:
return (lit.Test.TIMEOUT,
- f'{shard_header}Reached timeout of '
+ f'{header}Reached timeout of '
f'{litConfig.maxIndividualTestTime} seconds')
- if not os.path.exists(test.gtest_json_file):
- errmsg = f"shard JSON output does not exist: %s" % (test.gtest_json_file)
- return lit.Test.FAIL, shard_header + errmsg
-
if exitCode:
- output = shard_header + '\n'
- with open(test.gtest_json_file, encoding='utf-8') as f:
- testsuites = json.load(f)['testsuites']
- for testcase in testsuites:
- for testinfo in testcase['testsuite']:
- if testinfo['result'] == 'SUPPRESSED' or testinfo['result'] == 'SKIPPED':
- continue
- testname = testcase['name'] + '.' + testinfo['name']
- header = f"Script:\n--\n{' '.join(cmd)} --gtest_filter={testname}\n--\n"
- if 'failures' in testinfo:
- output += header
- for fail in testinfo['failures']:
- output += fail['failure'] + '\n'
- output += '\n'
- elif testinfo['result'] != 'COMPLETED':
- output += header
- output += 'unresolved test result\n'
- return lit.Test.FAIL, output
- else:
- return lit.Test.PASS, ''
+ return lit.Test.FAIL, header + out + err
+
+ if '[ SKIPPED ] 1 test,' in out:
+ return lit.Test.SKIPPED, ''
+
+ passing_test_line = '[ PASSED ] 1 test.'
+ if passing_test_line not in out:
+ return (lit.Test.UNRESOLVED,
+ f'{header}Unable to find {passing_test_line} '
+ f'in gtest output:\n\n{out}{err}')
+
+ return lit.Test.PASS,''
def prepareCmd(self, cmd):
"""Insert interpreter if needed.
@@ -156,61 +166,3 @@ def prepareCmd(self, cmd):
else:
cmd = shlex.split(self.run_under) + cmd
return cmd
-
- @staticmethod
- def post_process_shard_results(selected_tests, discovered_tests):
- def remove_gtest(tests):
- idxs = []
- for idx, t in enumerate(tests):
- if t.gtest_json_file:
- idxs.append(idx)
- for i in range(len(idxs)):
- del tests[idxs[i]-i]
-
- remove_gtest(discovered_tests)
- gtests = [t for t in selected_tests if t.gtest_json_file]
- remove_gtest(selected_tests)
- for test in gtests:
- # In case gtest has bugs such that no JSON file was emitted.
- if not os.path.exists(test.gtest_json_file):
- selected_tests.append(test)
- discovered_tests.append(test)
- continue
-
- # Load json file to retrieve results.
- with open(test.gtest_json_file, encoding='utf-8') as f:
- testsuites = json.load(f)['testsuites']
- for testcase in testsuites:
- for testinfo in testcase['testsuite']:
- # Ignore disabled tests.
- if testinfo['result'] == 'SUPPRESSED':
- continue
-
- testPath = test.path_in_suite[:-2] + (testcase['name'], testinfo['name'])
- subtest = lit.Test.Test(test.suite, testPath,
- test.config, test.file_path)
-
- testname = testcase['name'] + '.' + testinfo['name']
- header = f"Script:\n--\n{test.file_path} --gtest_filter={testname}\n--\n"
-
- output = ''
- if testinfo['result'] == 'SKIPPED':
- returnCode = lit.Test.SKIPPED
- elif 'failures' in testinfo:
- returnCode = lit.Test.FAIL
- output = header
- for fail in testinfo['failures']:
- output += fail['failure'] + '\n'
- elif testinfo['result'] == 'COMPLETED':
- returnCode = lit.Test.PASS
- else:
- returnCode = lit.Test.UNRESOLVED
- output = header + 'unresolved test result\n'
-
- subtest.setResult(lit.Test.Result(returnCode, output, float(testinfo['time'][:-1])))
-
- selected_tests.append(subtest)
- discovered_tests.append(subtest)
- os.remove(test.gtest_json_file)
-
- return selected_tests, discovered_tests
diff --git a/llvm/utils/lit/lit/main.py b/llvm/utils/lit/lit/main.py
index aacb579c8b03c..41f124a27ad7f 100755
--- a/llvm/utils/lit/lit/main.py
+++ b/llvm/utils/lit/lit/main.py
@@ -18,7 +18,6 @@
import lit.run
import lit.Test
import lit.util
-from lit.formats.googletest import GoogleTest
from lit.TestTimes import record_test_times
@@ -109,9 +108,6 @@ def main(builtin_params={}):
record_test_times(selected_tests, lit_config)
- selected_tests, discovered_tests = GoogleTest.post_process_shard_results(
- selected_tests, discovered_tests)
-
if opts.time_tests:
print_histogram(discovered_tests)
diff --git a/llvm/utils/lit/tests/Inputs/googletest-crash/DummySubDir/OneTest.py b/llvm/utils/lit/tests/Inputs/googletest-crash/DummySubDir/OneTest.py
deleted file mode 100644
index bebedb6f0e12f..0000000000000
--- a/llvm/utils/lit/tests/Inputs/googletest-crash/DummySubDir/OneTest.py
+++ /dev/null
@@ -1,45 +0,0 @@
-#!/usr/bin/env python
-
-import os
-import sys
-
-if len(sys.argv) == 3 and sys.argv[1] == "--gtest_list_tests":
- if sys.argv[2] != '--gtest_filter=-*DISABLED_*':
- raise ValueError("unexpected argument: %s" % (sys.argv[2]))
- print("""\
-FirstTest.
- subTestA
- subTestB
- subTestC
- subTestD
-ParameterizedTest/0.
- subTest
-ParameterizedTest/1.
- subTest""")
- sys.exit(0)
-elif len(sys.argv) != 1:
- # sharding and json output are specified using environment variables
- raise ValueError("unexpected argument: %r" % (' '.join(sys.argv[1:])))
-
-for e in ['GTEST_TOTAL_SHARDS', 'GTEST_SHARD_INDEX', 'GTEST_OUTPUT']:
- if e not in os.environ:
- raise ValueError("missing environment variables: " + e)
-
-if not os.environ['GTEST_OUTPUT'].startswith('json:'):
- raise ValueError("must emit json output: " + os.environ['GTEST_OUTPUT'])
-
-dummy_output = """\
-{
-"testsuites": [
-]
-}"""
-
-if os.environ['GTEST_SHARD_INDEX'] == '0':
- exit_code = 1
-else:
- json_filename = os.environ['GTEST_OUTPUT'].split(':', 1)[1]
- with open(json_filename, 'w') as f:
- f.write(dummy_output)
- exit_code = 0
-
-sys.exit(exit_code)
diff --git a/llvm/utils/lit/tests/Inputs/googletest-format/DummySubDir/OneTest.py b/llvm/utils/lit/tests/Inputs/googletest-format/DummySubDir/OneTest.py
index c66696f77a519..7bff6b6252b25 100644
--- a/llvm/utils/lit/tests/Inputs/googletest-format/DummySubDir/OneTest.py
+++ b/llvm/utils/lit/tests/Inputs/googletest-format/DummySubDir/OneTest.py
@@ -1,11 +1,11 @@
#!/usr/bin/env python
-import os
import sys
-if len(sys.argv) == 3 and sys.argv[1] == "--gtest_list_tests":
- if sys.argv[2] != '--gtest_filter=-*DISABLED_*':
- raise ValueError("unexpected argument: %s" % (sys.argv[2]))
+if len(sys.argv) != 2:
+ raise ValueError("unexpected number of args")
+
+if sys.argv[1] == "--gtest_list_tests":
print("""\
FirstTest.
subTestA
@@ -17,87 +17,31 @@
ParameterizedTest/1.
subTest""")
sys.exit(0)
-elif len(sys.argv) != 1:
- # sharding and json output are specified using environment variables
- raise ValueError("unexpected argument: %r" % (' '.join(sys.argv[1:])))
-
-for e in ['GTEST_TOTAL_SHARDS', 'GTEST_SHARD_INDEX', 'GTEST_OUTPUT']:
- if e not in os.environ:
- raise ValueError("missing environment variables: " + e)
-
-if not os.environ['GTEST_OUTPUT'].startswith('json:'):
- raise ValueError("must emit json output: " + os.environ['GTEST_OUTPUT'])
-
-output = """\
-{
-"testsuites": [
- {
- "name": "FirstTest",
- "testsuite": [
- {
- "name": "subTestA",
- "result": "COMPLETED",
- "time": "0.001s"
- },
- {
- "name": "subTestB",
- "result": "COMPLETED",
- "time": "0.001s",
- "failures": [
- {
- "failure": "I am subTest B, I FAIL\\nAnd I have two lines of output",
- "type": ""
- }
- ]
- },
- {
- "name": "subTestC",
- "result": "SKIPPED",
- "time": "0.001s"
- },
- {
- "name": "subTestD",
- "result": "UNRESOLVED",
- "time": "0.001s"
- }
- ]
- },
- {
- "name": "ParameterizedTest/0",
- "testsuite": [
- {
- "name": "subTest",
- "result": "COMPLETED",
- "time": "0.001s"
- }
- ]
- },
- {
- "name": "ParameterizedTest/1",
- "testsuite": [
- {
- "name": "subTest",
- "result": "COMPLETED",
- "time": "0.001s"
- }
- ]
- }
-]
-}"""
-
-dummy_output = """\
-{
-"testsuites": [
-]
-}"""
+elif not sys.argv[1].startswith("--gtest_filter="):
+ raise ValueError("unexpected argument: %r" % (sys.argv[1]))
-json_filename = os.environ['GTEST_OUTPUT'].split(':', 1)[1]
-with open(json_filename, 'w') as f:
- if os.environ['GTEST_SHARD_INDEX'] == '0':
- f.write(output)
- exit_code = 1
- else:
- f.write(dummy_output)
- exit_code = 0
-
-sys.exit(exit_code)
+test_name = sys.argv[1].split('=',1)[1]
+if test_name == 'FirstTest.subTestA':
+ print('I am subTest A, I PASS')
+ print('[ PASSED ] 1 test.')
+ sys.exit(0)
+elif test_name == 'FirstTest.subTestB':
+ print('I am subTest B, I FAIL')
+ print('And I have two lines of output')
+ sys.exit(1)
+elif test_name == 'FirstTest.subTestC':
+ print('I am subTest C, I am SKIPPED')
+ print('[ PASSED ] 0 tests.')
+ print('[ SKIPPED ] 1 test, listed below:')
+ print('[ SKIPPED ] FirstTest.subTestC')
+ sys.exit(0)
+elif test_name == 'FirstTest.subTestD':
+ print('I am subTest D, I am UNRESOLVED')
+ sys.exit(0)
+elif test_name in ('ParameterizedTest/0.subTest',
+ 'ParameterizedTest/1.subTest'):
+ print('I am a parameterized test, I also PASS')
+ print('[ PASSED ] 1 test.')
+ sys.exit(0)
+else:
+ raise SystemExit("error: invalid test name: %r" % (test_name,))
diff --git a/llvm/utils/lit/tests/Inputs/googletest-timeout/DummySubDir/OneTest.py b/llvm/utils/lit/tests/Inputs/googletest-timeout/DummySubDir/OneTest.py
index 3747232bf9e35..acf13a466fa71 100644
--- a/llvm/utils/lit/tests/Inputs/googletest-timeout/DummySubDir/OneTest.py
+++ b/llvm/utils/lit/tests/Inputs/googletest-timeout/DummySubDir/OneTest.py
@@ -1,66 +1,29 @@
#!/usr/bin/env python
-import os
import sys
+import time
-if len(sys.argv) == 3 and sys.argv[1] == "--gtest_list_tests":
- if sys.argv[2] != '--gtest_filter=-*DISABLED_*':
- raise ValueError("unexpected argument: %s" % (sys.argv[2]))
+if len(sys.argv) != 2:
+ raise ValueError("unexpected number of args")
+
+if sys.argv[1] == "--gtest_list_tests":
print("""\
T.
QuickSubTest
InfiniteLoopSubTest
""")
sys.exit(0)
-elif len(sys.argv) != 1:
- # sharding and json output are specified using environment variables
- raise ValueError("unexpected argument: %r" % (' '.join(sys.argv[1:])))
-
-for e in ['GTEST_TOTAL_SHARDS', 'GTEST_SHARD_INDEX', 'GTEST_OUTPUT', 'GTEST_FILTER']:
- if e not in os.environ:
- raise ValueError("missing environment variables: " + e)
-
-if not os.environ['GTEST_OUTPUT'].startswith('json:'):
- raise ValueError("must emit json output: " + os.environ['GTEST_OUTPUT'])
-
-output = """\
-{
-"testsuites": [
- {
- "name": "T",
- "testsuite": [
- {
- "name": "QuickSubTest",
- "result": "COMPLETED",
- "time": "2s"
- }
- ]
- }
-]
-}"""
+elif not sys.argv[1].startswith("--gtest_filter="):
+ raise ValueError("unexpected argument: %r" % (sys.argv[1]))
-dummy_output = """\
-{
-"testsuites": [
-]
-}"""
-
-json_filename = os.environ['GTEST_OUTPUT'].split(':', 1)[1]
-
-if os.environ['GTEST_SHARD_INDEX'] == '0':
- test_name = os.environ['GTEST_FILTER']
- if test_name == 'QuickSubTest':
- with open(json_filename, 'w') as f:
- f.write(output)
- exit_code = 0
- elif test_name == 'InfiniteLoopSubTest':
- while True:
- pass
- else:
- raise SystemExit("error: invalid test name: %r" % (test_name,))
+test_name = sys.argv[1].split('=',1)[1]
+if test_name == 'T.QuickSubTest':
+ print('I am QuickSubTest, I PASS')
+ print('[ PASSED ] 1 test.')
+ sys.exit(0)
+elif test_name == 'T.InfiniteLoopSubTest':
+ print('I am InfiniteLoopSubTest, I will hang')
+ while True:
+ pass
else:
- with open(json_filename, 'w') as f:
- f.write(dummy_output)
- exit_code = 0
-
-sys.exit(exit_code)
+ raise SystemExit("error: invalid test name: %r" % (test_name,))
diff --git a/llvm/utils/lit/tests/Inputs/googletest-timeout/lit.cfg b/llvm/utils/lit/tests/Inputs/googletest-timeout/lit.cfg
index af6a4846af56a..bf8a4db2bf90e 100644
--- a/llvm/utils/lit/tests/Inputs/googletest-timeout/lit.cfg
+++ b/llvm/utils/lit/tests/Inputs/googletest-timeout/lit.cfg
@@ -3,7 +3,6 @@ config.name = 'googletest-timeout'
config.test_format = lit.formats.GoogleTest('DummySubDir', 'Test')
configSetTimeout = lit_config.params.get('set_timeout', '0')
-config.environment['GTEST_FILTER'] = lit_config.params.get('gtest_filter')
if configSetTimeout == '1':
# Try setting the max individual test time in the configuration
diff --git a/llvm/utils/lit/tests/Inputs/googletest-upstream-format/DummySubDir/OneTest.py b/llvm/utils/lit/tests/Inputs/googletest-upstream-format/DummySubDir/OneTest.py
new file mode 100644
index 0000000000000..aa79a22c0b106
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/googletest-upstream-format/DummySubDir/OneTest.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+
+import os
+import sys
+
+if len(sys.argv) != 2:
+ raise ValueError("unexpected number of args")
+
+if sys.argv[1] == "--gtest_list_tests":
+ print(f"""\
+Running main() from {os.getcwd()}/gtest_main.cc
+FirstTest.
+ subTestA
+ subTestB
+ subTestC
+ subTestD
+ParameterizedTest/0.
+ subTest
+ParameterizedTest/1.
+ subTest""")
+ sys.exit(0)
+elif not sys.argv[1].startswith("--gtest_filter="):
+ raise ValueError("unexpected argument: %r" % (sys.argv[1]))
+
+test_name = sys.argv[1].split('=',1)[1]
+print('Running main() from gtest_main.cc')
+if test_name == 'FirstTest.subTestA':
+ print('I am subTest A, I PASS')
+ print('[ PASSED ] 1 test.')
+ sys.exit(0)
+elif test_name == 'FirstTest.subTestB':
+ print('I am subTest B, I FAIL')
+ print('And I have two lines of output')
+ sys.exit(1)
+elif test_name == 'FirstTest.subTestC':
+ print('I am subTest C, I am SKIPPED')
+ print('[ PASSED ] 0 tests.')
+ print('[ SKIPPED ] 1 test, listed below:')
+ print('[ SKIPPED ] FirstTest.subTestC')
+ sys.exit(0)
+elif test_name == 'FirstTest.subTestD':
+ print('I am subTest D, I am UNRESOLVED')
+ sys.exit(0)
+elif test_name in ('ParameterizedTest/0.subTest',
+ 'ParameterizedTest/1.subTest'):
+ print('I am a parameterized test, I also PASS')
+ print('[ PASSED ] 1 test.')
+ sys.exit(0)
+else:
+ raise SystemExit("error: invalid test name: %r" % (test_name,))
diff --git a/llvm/utils/lit/tests/Inputs/googletest-crash/lit.cfg b/llvm/utils/lit/tests/Inputs/googletest-upstream-format/lit.cfg
similarity index 66%
rename from llvm/utils/lit/tests/Inputs/googletest-crash/lit.cfg
rename to llvm/utils/lit/tests/Inputs/googletest-upstream-format/lit.cfg
index 8048411a70882..9fb5d2b0247be 100644
--- a/llvm/utils/lit/tests/Inputs/googletest-crash/lit.cfg
+++ b/llvm/utils/lit/tests/Inputs/googletest-upstream-format/lit.cfg
@@ -1,3 +1,3 @@
import lit.formats
-config.name = 'googletest-crash'
+config.name = 'googletest-upstream-format'
config.test_format = lit.formats.GoogleTest('DummySubDir', 'Test')
diff --git a/llvm/utils/lit/tests/googletest-crash.py b/llvm/utils/lit/tests/googletest-crash.py
deleted file mode 100644
index 1cbe05b4150ea..0000000000000
--- a/llvm/utils/lit/tests/googletest-crash.py
+++ /dev/null
@@ -1,20 +0,0 @@
-# Check GoogleTest shard test crashes are handled.
-
-# RUN: not %{lit} -v %{inputs}/googletest-crash | FileCheck %s
-
-# CHECK: -- Testing:
-# CHECK: FAIL: googletest-crash :: [[PATH:[Dd]ummy[Ss]ub[Dd]ir/]][[FILE:OneTest\.py]]/0
-# CHECK: *** TEST 'googletest-crash :: [[PATH]][[FILE]]/0{{.*}} FAILED ***
-# CHECK-NEXT: Script(shard):
-# CHECK-NEXT: --
-# CHECK-NEXT: GTEST_COLOR=no
-# CHECK-NEXT: GTEST_TOTAL_SHARDS=6
-# CHECK-NEXT: GTEST_SHARD_INDEX=0
-# CHECK-NEXT: GTEST_OUTPUT=json:[[JSON:.*\.json]]
-# CHECK-NEXT: [[FILE]]
-# CHECK-NEXT: --
-# CHECK-NEXT: shard JSON output does not exist: [[JSON]]
-# CHECK-NEXT: ***
-# CHECK: Failed Tests (1):
-# CHECK-NEXT: googletest-crash :: [[PATH]][[FILE]]/0/6
-# CHECK: Failed{{ *}}: 1
\ No newline at end of file
diff --git a/llvm/utils/lit/tests/googletest-format.py b/llvm/utils/lit/tests/googletest-format.py
index 5d5f1f9e96f04..b960d0cdc64e8 100644
--- a/llvm/utils/lit/tests/googletest-format.py
+++ b/llvm/utils/lit/tests/googletest-format.py
@@ -9,35 +9,28 @@
# END.
# CHECK: -- Testing:
-# CHECK: FAIL: googletest-format :: [[PATH:[Dd]ummy[Ss]ub[Dd]ir/]][[FILE:OneTest\.py]]/0
-# CHECK: *** TEST 'googletest-format :: [[PATH]][[FILE]]/0{{.*}} FAILED ***
-# CHECK-NEXT: Script(shard):
-# CHECK-NEXT: --
-# CHECK-NEXT: GTEST_COLOR=no
-# CHECK-NEXT: GTEST_TOTAL_SHARDS=6
-# CHECK-NEXT: GTEST_SHARD_INDEX=0
-# CHECK-NEXT: GTEST_OUTPUT=json:{{.*\.json}}
-# CHECK-NEXT: [[FILE]]
-# CHECK-NEXT: --
-# CHECK-EMPTY:
+# CHECK: PASS: googletest-format :: [[PATH:[Dd]ummy[Ss]ub[Dd]ir/]][[FILE:OneTest\.py]]/FirstTest.subTestA
+# CHECK: FAIL: googletest-format :: [[PATH]][[FILE]]/[[TEST:FirstTest\.subTestB]]
+# CHECK-NEXT: *** TEST 'googletest-format :: [[PATH]][[FILE]]/[[TEST]]' FAILED ***
# CHECK-NEXT: Script:
# CHECK-NEXT: --
-# CHECK-NEXT: [[FILE]] --gtest_filter=FirstTest.subTestB
+# CHECK-NEXT: [[FILE]] --gtest_filter=[[TEST]]
# CHECK-NEXT: --
# CHECK-NEXT: I am subTest B, I FAIL
# CHECK-NEXT: And I have two lines of output
-# CHECK-EMPTY:
-# CHECK: Script:
+# CHECK: ***
+# CHECK: SKIPPED: googletest-format :: [[PATH]][[FILE]]/FirstTest.subTestC
+# CHECK: UNRESOLVED: googletest-format :: [[PATH]][[FILE]]/[[TEST:FirstTest\.subTestD]]
+# CHECK-NEXT: *** TEST 'googletest-format :: [[PATH]][[FILE]]/[[TEST]]' FAILED ***
+# CHECK-NEXT: Script:
# CHECK-NEXT: --
-# CHECK-NEXT: [[FILE]] --gtest_filter=FirstTest.subTestD
+# CHECK-NEXT: [[FILE]] --gtest_filter=[[TEST]]
# CHECK-NEXT: --
-# CHECK-NEXT: unresolved test result
-# CHECK: ***
-# CHECK: Unresolved Tests (1):
-# CHECK-NEXT: googletest-format :: [[PATH]][[FILE]]/FirstTest/subTestD
-# CHECK: ***
-# CHECK-NEXT: Failed Tests (1):
-# CHECK-NEXT: googletest-format :: [[PATH]][[FILE]]/FirstTest/subTestB
+# CHECK-NEXT: Unable to find [ PASSED ] 1 test. in gtest output
+# CHECK: I am subTest D, I am UNRESOLVED
+# CHECK: PASS: googletest-format :: [[PATH]][[FILE]]/ParameterizedTest/0.subTest
+# CHECK: PASS: googletest-format :: [[PATH]][[FILE]]/ParameterizedTest/1.subTest
+# CHECK: Failed Tests (1)
# CHECK: Skipped{{ *}}: 1
# CHECK: Passed{{ *}}: 3
# CHECK: Unresolved{{ *}}: 1
diff --git a/llvm/utils/lit/tests/googletest-timeout.py b/llvm/utils/lit/tests/googletest-timeout.py
index dc80636340e6b..b7b9b64d4ed56 100644
--- a/llvm/utils/lit/tests/googletest-timeout.py
+++ b/llvm/utils/lit/tests/googletest-timeout.py
@@ -7,29 +7,24 @@
# Check that the per test timeout is enforced when running GTest tests.
#
# RUN: not %{lit} -v %{inputs}/googletest-timeout \
-# RUN: --param gtest_filter=InfiniteLoopSubTest --timeout=1 > %t.cmd.out
+# RUN: --filter=InfiniteLoopSubTest --timeout=1 > %t.cmd.out
# RUN: FileCheck --check-prefix=CHECK-INF < %t.cmd.out %s
# Check that the per test timeout is enforced when running GTest tests via
# the configuration file
#
# RUN: not %{lit} -v %{inputs}/googletest-timeout \
-# RUN: --param gtest_filter=InfiniteLoopSubTest --param set_timeout=1 \
+# RUN: --filter=InfiniteLoopSubTest --param set_timeout=1 \
# RUN: > %t.cfgset.out
# RUN: FileCheck --check-prefix=CHECK-INF < %t.cfgset.out %s
# CHECK-INF: -- Testing:
-# CHECK-INF: TIMEOUT: googletest-timeout :: [[PATH:[Dd]ummy[Ss]ub[Dd]ir/]][[FILE:OneTest\.py]]/0/2
-# CHECK-INF-NEXT: ******************** TEST 'googletest-timeout :: [[PATH]][[FILE]]/0/2' FAILED ********************
-# CHECK-INF-NEXT: Script(shard):
+# CHECK-INF: TIMEOUT: googletest-timeout :: [[PATH:[Dd]ummy[Ss]ub[Dd]ir/]][[FILE:OneTest\.py]]/[[TEST:T\.InfiniteLoopSubTest]]
+# CHECK-INF-NEXT: ******************** TEST 'googletest-timeout :: [[PATH]][[FILE]]/[[TEST]]' FAILED ********************
+# CHECK-INF-NEXT: Script:
# CHECK-INF-NEXT: --
-# CHECK-INF-NEXT: GTEST_COLOR=no
-# CHECK-INF-NEXT: GTEST_TOTAL_SHARDS=2
-# CHECK-INF-NEXT: GTEST_SHARD_INDEX=0
-# CHECK-INF-NEXT: GTEST_OUTPUT=json:{{.*\.json}}
-# CHECK-INF-NEXT: [[FILE]]
+# CHECK-INF-NEXT: [[FILE]] --gtest_filter=[[TEST]]
# CHECK-INF-NEXT: --
-# CHECK-INF-NEXT: Reached timeout of 1 seconds
# CHECK-INF: Timed Out: 1
###############################################################################
@@ -40,15 +35,15 @@
###############################################################################
# RUN: %{lit} -v %{inputs}/googletest-timeout \
-# RUN: --param gtest_filter=QuickSubTest --timeout=3600 > %t.cmd.out
+# RUN: --filter=QuickSubTest --timeout=3600 > %t.cmd.out
# RUN: FileCheck --check-prefix=CHECK-QUICK < %t.cmd.out %s
-# CHECK-QUICK: PASS: googletest-timeout :: {{[Dd]ummy[Ss]ub[Dd]ir}}/OneTest.py/0/2 {{.*}}
-# CHECK-QUICK: Passed: 1
+# CHECK-QUICK: PASS: googletest-timeout :: {{[Dd]ummy[Ss]ub[Dd]ir}}/OneTest.py/T.QuickSubTest
+# CHECK-QUICK: Passed : 1
# Test per test timeout via a config file and on the command line.
# The value set on the command line should override the config file.
-# RUN: %{lit} -v %{inputs}/googletest-timeout --param gtest_filter=QuickSubTest \
+# RUN: %{lit} -v %{inputs}/googletest-timeout --filter=QuickSubTest \
# RUN: --param set_timeout=1 --timeout=3600 \
# RUN: > %t.cmdover.out 2> %t.cmdover.err
# RUN: FileCheck --check-prefix=CHECK-QUICK < %t.cmdover.out %s
diff --git a/llvm/utils/lit/tests/googletest-upstream-format.py b/llvm/utils/lit/tests/googletest-upstream-format.py
new file mode 100644
index 0000000000000..cf0f73e2d31ed
--- /dev/null
+++ b/llvm/utils/lit/tests/googletest-upstream-format.py
@@ -0,0 +1,35 @@
+# Check the various features of the GoogleTest format.
+
+# RUN: not %{lit} -v %{inputs}/googletest-upstream-format > %t.out
+# RUN: FileCheck < %t.out %s
+#
+# END.
+
+# CHECK: -- Testing:
+# CHECK: PASS: googletest-upstream-format :: [[PATH:[Dd]ummy[Ss]ub[Dd]ir/]][[FILE:OneTest\.py]]/FirstTest.subTestA
+# CHECK: FAIL: googletest-upstream-format :: [[PATH]][[FILE]]/[[TEST:FirstTest\.subTestB]]
+# CHECK-NEXT: *** TEST 'googletest-upstream-format :: [[PATH]][[FILE]]/[[TEST]]' FAILED ***
+# CHECK-NEXT: Script:
+# CHECK-NEXT: --
+# CHECK-NEXT: [[FILE]] --gtest_filter=[[TEST]]
+# CHECK-NEXT: --
+# CHECK-NEXT: Running main() from gtest_main.cc
+# CHECK-NEXT: I am subTest B, I FAIL
+# CHECK-NEXT: And I have two lines of output
+# CHECK: SKIPPED: googletest-upstream-format :: [[PATH]][[FILE]]/FirstTest.subTestC
+# CHECK: UNRESOLVED: googletest-upstream-format :: [[PATH]][[FILE]]/[[TEST:FirstTest\.subTestD]]
+# CHECK-NEXT: *** TEST 'googletest-upstream-format :: [[PATH]][[FILE]]/[[TEST]]' FAILED ***
+# CHECK-NEXT: Script:
+# CHECK-NEXT: --
+# CHECK-NEXT: [[FILE]] --gtest_filter=[[TEST]]
+# CHECK-NEXT: --
+# CHECK-NEXT: Unable to find [ PASSED ] 1 test. in gtest output
+# CHECK: I am subTest D, I am UNRESOLVED
+# CHECK: ***
+# CHECK: PASS: googletest-upstream-format :: [[PATH]][[FILE]]/ParameterizedTest/0.subTest
+# CHECK: PASS: googletest-upstream-format :: [[PATH]][[FILE]]/ParameterizedTest/1.subTest
+# CHECK: Failed Tests (1)
+# CHECK: Skipped{{ *}}: 1
+# CHECK: Passed{{ *}}: 3
+# CHECK: Unresolved{{ *}}: 1
+# CHECK: Failed{{ *}}: 1
More information about the llvm-commits
mailing list