[llvm] gtest sharding (PR #67063)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 21 14:00:57 PDT 2023
https://github.com/zeroomega created https://github.com/llvm/llvm-project/pull/67063
[unittest] Add option to allow disabling sharding in unittest
By default, googletest based unit tests uses sharding to speed up the
testing. However, when these unit tests are through wrapper program on a
remote platform with large round trip time, the sharding will increase
the time cost dramatically. This patch adds an "use_sharding" in the lit
googletest configuration and it can be configured using
"LLVM_GTEST_USE_SHARDING" cmake option.
>From aade5b5abf83adad83d3f0fe49741838ff7d709e Mon Sep 17 00:00:00 2001
From: Haowei Wu <haowei at google.com>
Date: Tue, 19 Sep 2023 14:03:27 -0700
Subject: [PATCH 1/2] [unittest] Allow LLVM unit test to run under a wrapper
program.
This patch add CMake option "LLVM_GTEST_RUN_UNDER" to LLVM unittest
configuration. When specified, LLVM unittest will be run under the
wrapper program specified by this option. This feature can simplify
the setup to run LLVM unittest on a target platform that is different
than host.
---
llvm/test/Unit/lit.cfg.py | 6 +++++-
llvm/test/Unit/lit.site.cfg.py.in | 1 +
2 files changed, 6 insertions(+), 1 deletion(-)
diff --git a/llvm/test/Unit/lit.cfg.py b/llvm/test/Unit/lit.cfg.py
index f15c30dbcdb0aa6..61296d7ea0032ef 100644
--- a/llvm/test/Unit/lit.cfg.py
+++ b/llvm/test/Unit/lit.cfg.py
@@ -19,7 +19,11 @@
config.test_source_root = config.test_exec_root
# testFormat: The test format to use to interpret tests.
-config.test_format = lit.formats.GoogleTest(config.llvm_build_mode, "Tests")
+config.test_format = lit.formats.GoogleTest(
+ config.llvm_build_mode,
+ "Tests",
+ run_under=config.gtest_run_under,
+)
# Propagate the temp directory. Windows requires this because it uses \Windows\
# if none of these are present.
diff --git a/llvm/test/Unit/lit.site.cfg.py.in b/llvm/test/Unit/lit.site.cfg.py.in
index 1d7d76580149495..3536a34f796a28d 100644
--- a/llvm/test/Unit/lit.site.cfg.py.in
+++ b/llvm/test/Unit/lit.site.cfg.py.in
@@ -7,6 +7,7 @@ config.llvm_obj_root = path(r"@LLVM_BINARY_DIR@")
config.llvm_tools_dir = lit_config.substitute(path(r"@LLVM_TOOLS_DIR@"))
config.llvm_build_mode = lit_config.substitute("@LLVM_BUILD_MODE@")
config.shlibdir = lit_config.substitute(path(r"@SHLIBDIR@"))
+config.gtest_run_under = lit_config.substitute(r"@LLVM_GTEST_RUN_UNDER@")
# Let the main config do the real work.
lit_config.load_config(
>From 27c0ca99cbbac52e97a9f6bbbfc071ad1c105be7 Mon Sep 17 00:00:00 2001
From: Haowei Wu <haowei at google.com>
Date: Thu, 21 Sep 2023 13:51:12 -0700
Subject: [PATCH 2/2] [unittest] Add option to allow disabling sharding in
unittest
By default, googletest based unit tests uses sharding to speed up the
testing. However, when these unit tests are through wrapper program on a
remote platform with large round trip time, the sharding will increase
the time cost dramatically. This patch adds an "use_sharding" in the lit
googletest configuration and it can be configured using
"LLVM_GTEST_USE_SHARDING" cmake option.
---
llvm/test/Unit/lit.cfg.py | 1 +
llvm/test/Unit/lit.site.cfg.py.in | 1 +
llvm/utils/lit/lit/formats/googletest.py | 99 ++++++++++++-----
.../DummySubDir/OneTest.py | 105 ++++++++++++++++++
.../Inputs/googletest-no-sharding/lit.cfg | 4 +
.../utils/lit/tests/googletest-no-sharding.py | 43 +++++++
6 files changed, 226 insertions(+), 27 deletions(-)
create mode 100644 llvm/utils/lit/tests/Inputs/googletest-no-sharding/DummySubDir/OneTest.py
create mode 100644 llvm/utils/lit/tests/Inputs/googletest-no-sharding/lit.cfg
create mode 100644 llvm/utils/lit/tests/googletest-no-sharding.py
diff --git a/llvm/test/Unit/lit.cfg.py b/llvm/test/Unit/lit.cfg.py
index 61296d7ea0032ef..4e815444d339dcd 100644
--- a/llvm/test/Unit/lit.cfg.py
+++ b/llvm/test/Unit/lit.cfg.py
@@ -23,6 +23,7 @@
config.llvm_build_mode,
"Tests",
run_under=config.gtest_run_under,
+ use_sharding=config.use_gtest_sharding,
)
# Propagate the temp directory. Windows requires this because it uses \Windows\
diff --git a/llvm/test/Unit/lit.site.cfg.py.in b/llvm/test/Unit/lit.site.cfg.py.in
index 3536a34f796a28d..523d2bb3817e122 100644
--- a/llvm/test/Unit/lit.site.cfg.py.in
+++ b/llvm/test/Unit/lit.site.cfg.py.in
@@ -8,6 +8,7 @@ config.llvm_tools_dir = lit_config.substitute(path(r"@LLVM_TOOLS_DIR@"))
config.llvm_build_mode = lit_config.substitute("@LLVM_BUILD_MODE@")
config.shlibdir = lit_config.substitute(path(r"@SHLIBDIR@"))
config.gtest_run_under = lit_config.substitute(r"@LLVM_GTEST_RUN_UNDER@")
+config.use_gtest_sharding = lit_config.substitute(r"@LLVM_GTEST_USE_SHARD@")
# Let the main config do the real work.
lit_config.load_config(
diff --git a/llvm/utils/lit/lit/formats/googletest.py b/llvm/utils/lit/lit/formats/googletest.py
index f8304cbd05453b5..f05e7e351cfefa6 100644
--- a/llvm/utils/lit/lit/formats/googletest.py
+++ b/llvm/utils/lit/lit/formats/googletest.py
@@ -15,7 +15,7 @@
class GoogleTest(TestFormat):
- def __init__(self, test_sub_dirs, test_suffix, run_under=[]):
+ def __init__(self, test_sub_dirs, test_suffix, run_under=[], use_sharding=True):
self.seen_executables = set()
self.test_sub_dirs = str(test_sub_dirs).split(";")
@@ -27,6 +27,17 @@ def __init__(self, test_sub_dirs, test_suffix, run_under=[]):
# Also check for .py files for testing purposes.
self.test_suffixes = {exe_suffix, test_suffix + ".py"}
self.run_under = run_under
+ if type(use_sharding) is str:
+ use_sharding = use_sharding.lower()
+ if use_sharding in ("0", "off", "false", "no", "n", "ignore", "notfound"):
+ use_sharding = False
+ else:
+ use_sharding = True
+ elif type(use_sharding) is bool:
+ pass
+ else:
+ use_sharding = True
+ self.use_sharding = use_sharding
def get_num_tests(self, path, litConfig, localConfig):
list_test_cmd = self.prepareCmd(
@@ -68,24 +79,49 @@ def getTestsInDirectory(self, testSuite, path_in_suite, litConfig, localConfig):
self.seen_executables.add(execpath)
num_tests = self.get_num_tests(execpath, litConfig, localConfig)
if num_tests is not None:
- # Compute the number of shards.
- shard_size = init_shard_size
- nshard = int(math.ceil(num_tests / shard_size))
- while nshard < core_count and shard_size > 1:
- shard_size = shard_size // 2
+ if self.use_sharding:
+ # Compute the number of shards.
+ shard_size = init_shard_size
nshard = int(math.ceil(num_tests / shard_size))
-
- # Create one lit test for each shard.
- for idx in range(nshard):
- testPath = path_in_suite + (subdir, fn, str(idx), str(nshard))
+ while nshard < core_count and shard_size > 1:
+ shard_size = shard_size // 2
+ nshard = int(math.ceil(num_tests / shard_size))
+
+ # Create one lit test for each shard.
+ for idx in range(nshard):
+ testPath = path_in_suite + (
+ subdir,
+ fn,
+ str(idx),
+ str(nshard),
+ )
+ json_file = (
+ "-".join(
+ [
+ execpath,
+ testSuite.config.name,
+ str(os.getpid()),
+ str(idx),
+ str(nshard),
+ ]
+ )
+ + ".json"
+ )
+ yield lit.Test.Test(
+ testSuite,
+ testPath,
+ localConfig,
+ file_path=execpath,
+ gtest_json_file=json_file,
+ )
+ else:
+ testPath = path_in_suite + (subdir, fn)
json_file = (
"-".join(
[
execpath,
testSuite.config.name,
str(os.getpid()),
- str(idx),
- str(nshard),
]
)
+ ".json"
@@ -118,24 +154,33 @@ def execute(self, test, litConfig):
if test.gtest_json_file is None:
return lit.Test.FAIL, ""
- testPath, testName = os.path.split(test.getSourcePath())
- while not os.path.exists(testPath):
- # Handle GTest parametrized and typed tests, whose name includes
- # some '/'s.
- testPath, namePrefix = os.path.split(testPath)
- testName = namePrefix + "/" + testName
-
- testName, total_shards = os.path.split(testName)
- testName, shard_idx = os.path.split(testName)
+ testPath = test.getSourcePath()
from lit.cl_arguments import TestOrder
use_shuffle = TestOrder(litConfig.order) == TestOrder.RANDOM
- shard_env = {
- "GTEST_OUTPUT": "json:" + test.gtest_json_file,
- "GTEST_SHUFFLE": "1" if use_shuffle else "0",
- "GTEST_TOTAL_SHARDS": os.environ.get("GTEST_TOTAL_SHARDS", total_shards),
- "GTEST_SHARD_INDEX": os.environ.get("GTEST_SHARD_INDEX", shard_idx),
- }
+ if self.use_sharding:
+ testPath, testName = os.path.split(test.getSourcePath())
+ while not os.path.exists(testPath):
+ # Handle GTest parametrized and typed tests, whose name includes
+ # some '/'s.
+ testPath, namePrefix = os.path.split(testPath)
+ testName = namePrefix + "/" + testName
+
+ testName, total_shards = os.path.split(testName)
+ testName, shard_idx = os.path.split(testName)
+ shard_env = {
+ "GTEST_OUTPUT": "json:" + test.gtest_json_file,
+ "GTEST_SHUFFLE": "1" if use_shuffle else "0",
+ "GTEST_TOTAL_SHARDS": os.environ.get(
+ "GTEST_TOTAL_SHARDS", total_shards
+ ),
+ "GTEST_SHARD_INDEX": os.environ.get("GTEST_SHARD_INDEX", shard_idx),
+ }
+ else:
+ shard_env = {
+ "GTEST_OUTPUT": "json:" + test.gtest_json_file,
+ "GTEST_SHUFFLE": "1" if use_shuffle else "0",
+ }
test.config.environment.update(shard_env)
cmd = [testPath]
diff --git a/llvm/utils/lit/tests/Inputs/googletest-no-sharding/DummySubDir/OneTest.py b/llvm/utils/lit/tests/Inputs/googletest-no-sharding/DummySubDir/OneTest.py
new file mode 100644
index 000000000000000..8fcab0081a21209
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/googletest-no-sharding/DummySubDir/OneTest.py
@@ -0,0 +1,105 @@
+#!/usr/bin/env python
+
+import os
+import sys
+
+if len(sys.argv) == 3 and sys.argv[1] == "--gtest_list_tests":
+ if sys.argv[2] != "--gtest_filter=-*DISABLED_*":
+ raise ValueError("unexpected argument: %s" % (sys.argv[2]))
+ print(
+ """\
+FirstTest.
+ subTestA
+ subTestB
+ subTestC
+ subTestD
+ParameterizedTest/0.
+ subTest
+ParameterizedTest/1.
+ subTest"""
+ )
+ sys.exit(0)
+elif len(sys.argv) != 1:
+ # sharding and json output are specified using environment variables
+ raise ValueError("unexpected argument: %r" % (" ".join(sys.argv[1:])))
+
+for e in ["GTEST_OUTPUT"]:
+ if e not in os.environ:
+ raise ValueError("missing environment variables: " + e)
+
+if not os.environ["GTEST_OUTPUT"].startswith("json:"):
+ raise ValueError("must emit json output: " + os.environ["GTEST_OUTPUT"])
+
+output = """\
+{
+"random_seed": 123,
+"testsuites": [
+ {
+ "name": "FirstTest",
+ "testsuite": [
+ {
+ "name": "subTestA",
+ "result": "COMPLETED",
+ "time": "0.001s"
+ },
+ {
+ "name": "subTestB",
+ "result": "COMPLETED",
+ "time": "0.001s",
+ "failures": [
+ {
+ "failure": "I am subTest B, I FAIL\\nAnd I have two lines of output",
+ "type": ""
+ }
+ ]
+ },
+ {
+ "name": "subTestC",
+ "result": "SKIPPED",
+ "time": "0.001s"
+ },
+ {
+ "name": "subTestD",
+ "result": "UNRESOLVED",
+ "time": "0.001s"
+ }
+ ]
+ },
+ {
+ "name": "ParameterizedTest/0",
+ "testsuite": [
+ {
+ "name": "subTest",
+ "result": "COMPLETED",
+ "time": "0.001s"
+ }
+ ]
+ },
+ {
+ "name": "ParameterizedTest/1",
+ "testsuite": [
+ {
+ "name": "subTest",
+ "result": "COMPLETED",
+ "time": "0.001s"
+ }
+ ]
+ }
+]
+}"""
+
+dummy_output = """\
+{
+"testsuites": [
+]
+}"""
+
+json_filename = os.environ["GTEST_OUTPUT"].split(":", 1)[1]
+with open(json_filename, "w", encoding="utf-8") as f:
+ print("[ RUN ] FirstTest.subTestB", flush=True)
+ print("I am subTest B output", file=sys.stderr, flush=True)
+ print("[ FAILED ] FirstTest.subTestB (8 ms)", flush=True)
+ f.write(output)
+ exit_code = 1
+
+sys.exit(exit_code)
diff --git a/llvm/utils/lit/tests/Inputs/googletest-no-sharding/lit.cfg b/llvm/utils/lit/tests/Inputs/googletest-no-sharding/lit.cfg
new file mode 100644
index 000000000000000..5507fe65782dcbc
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/googletest-no-sharding/lit.cfg
@@ -0,0 +1,4 @@
+import lit.formats
+
+config.name = "googletest-no-sharding"
+config.test_format = lit.formats.GoogleTest("DummySubDir", "Test", use_sharding="False")
diff --git a/llvm/utils/lit/tests/googletest-no-sharding.py b/llvm/utils/lit/tests/googletest-no-sharding.py
new file mode 100644
index 000000000000000..3fa7a37cdd7090c
--- /dev/null
+++ b/llvm/utils/lit/tests/googletest-no-sharding.py
@@ -0,0 +1,43 @@
+# Check the various features of the GoogleTest format.
+
+# RUN: not %{lit} -v --order=random %{inputs}/googletest-no-sharding > %t.out
+# FIXME: Temporarily dump test output so we can debug failing tests on
+# buildbots.
+# RUN: cat %t.out
+# RUN: FileCheck < %t.out %s
+#
+# END.
+
+# CHECK: -- Testing:
+# CHECK: FAIL: googletest-no-sharding :: [[PATH:[Dd]ummy[Ss]ub[Dd]ir/]][[FILE:OneTest\.py]]
+# CHECK: *** TEST 'googletest-no-sharding :: [[PATH]][[FILE]]' FAILED ***
+# CHECK-NEXT: Script(shard):
+# CHECK-NEXT: --
+# CHECK-NEXT: GTEST_OUTPUT=json:{{[^[:space:]]*}} GTEST_SHUFFLE=1 GTEST_RANDOM_SEED=123 {{.*}}[[FILE]]
+# CHECK-NEXT: --
+# CHECK-EMPTY:
+# CHECK-NEXT: Script:
+# CHECK-NEXT: --
+# CHECK-NEXT: [[FILE]] --gtest_filter=FirstTest.subTestB
+# CHECK-NEXT: --
+# CHECK-NEXT: I am subTest B output
+# CHECK-EMPTY:
+# CHECK-NEXT: I am subTest B, I FAIL
+# CHECK-NEXT: And I have two lines of output
+# CHECK-EMPTY:
+# CHECK: Script:
+# CHECK-NEXT: --
+# CHECK-NEXT: [[FILE]] --gtest_filter=FirstTest.subTestD
+# CHECK-NEXT: --
+# CHECK-NEXT: unresolved test result
+# CHECK: ***
+# CHECK: ***
+# CHECK: Unresolved Tests (1):
+# CHECK-NEXT: googletest-no-sharding :: FirstTest/subTestD
+# CHECK: ***
+# CHECK-NEXT: Failed Tests (1):
+# CHECK-NEXT: googletest-no-sharding :: FirstTest/subTestB
+# CHECK: Skipped{{ *}}: 1
+# CHECK: Passed{{ *}}: 3
+# CHECK: Unresolved{{ *}}: 1
+# CHECK: Failed{{ *}}: 1
More information about the llvm-commits
mailing list