[llvm] e97b2d4 - [lit][unit] add a test for sanitizer-only test failures

[Yuanfang Chen via llvm-commits]

Author: Yuanfang Chen
Date: 2022-07-15T18:36:38-07:00
New Revision: e97b2d413849d3dbc8b49740ce5a07ed0382309c

URL: https://github.com/llvm/llvm-project/commit/e97b2d413849d3dbc8b49740ce5a07ed0382309c
DIFF: https://github.com/llvm/llvm-project/commit/e97b2d413849d3dbc8b49740ce5a07ed0382309c.diff

LOG: [lit][unit] add a test for sanitizer-only test failures

Follow-up for 6b02c53936b9e77fee.

Added: 
    llvm/utils/lit/tests/Inputs/googletest-sanitizer-error/DummySubDir/OneTest.py
    llvm/utils/lit/tests/Inputs/googletest-sanitizer-error/lit.cfg
    llvm/utils/lit/tests/googletest-sanitizer-error.py

Modified: 
    

Removed: 
    


################################################################################
diff  --git a/llvm/utils/lit/tests/Inputs/googletest-sanitizer-error/DummySubDir/OneTest.py b/llvm/utils/lit/tests/Inputs/googletest-sanitizer-error/DummySubDir/OneTest.py
new file mode 100644
index 000000000000..558117c7c9c0
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/googletest-sanitizer-error/DummySubDir/OneTest.py
@@ -0,0 +1,59 @@
+#!/usr/bin/env python
+
+import os
+import sys
+
+if len(sys.argv) == 3 and sys.argv[1] == "--gtest_list_tests":
+    if sys.argv[2] != '--gtest_filter=-*DISABLED_*':
+        raise ValueError("unexpected argument: %s" % (sys.argv[2]))
+    print("""\
+FirstTest.
+  subTestA""")
+    sys.exit(0)
+elif len(sys.argv) != 1:
+    # sharding and json output are specified using environment variables
+    raise ValueError("unexpected argument: %r" % (' '.join(sys.argv[1:])))
+
+for e in ['GTEST_TOTAL_SHARDS', 'GTEST_SHARD_INDEX', 'GTEST_OUTPUT']:
+    if e not in os.environ:
+        raise ValueError("missing environment variables: " + e)
+
+if not os.environ['GTEST_OUTPUT'].startswith('json:'):
+    raise ValueError("must emit json output: " + os.environ['GTEST_OUTPUT'])
+
+output = """\
+{
+"random_seed": 123,
+"testsuites": [
+    {
+        "name": "FirstTest",
+        "testsuite": [
+            {
+                "name": "subTestA",
+                "result": "COMPLETED",
+                "time": "0.001s"
+            }
+        ]
+    }
+]
+}"""
+
+dummy_output = """\
+{
+"testsuites": [
+]
+}"""
+
+json_filename = os.environ['GTEST_OUTPUT'].split(':', 1)[1]
+with open(json_filename, 'w', encoding='utf-8') as f:
+    if os.environ['GTEST_SHARD_INDEX'] == '0':
+        print('[ RUN      ] FirstTest.subTestA', flush=True)
+        print('[       OK ] FirstTest.subTestA (8 ms)', flush=True)
+
+        f.write(output)
+        exit_code = 1
+    else:
+        f.write(dummy_output)
+        exit_code = 0
+
+sys.exit(exit_code)

diff  --git a/llvm/utils/lit/tests/Inputs/googletest-sanitizer-error/lit.cfg b/llvm/utils/lit/tests/Inputs/googletest-sanitizer-error/lit.cfg
new file mode 100644
index 000000000000..43a8aabd96b3
--- /dev/null
+++ b/llvm/utils/lit/tests/Inputs/googletest-sanitizer-error/lit.cfg
@@ -0,0 +1,3 @@
+import lit.formats
+config.name = 'googletest-sanitizer-error'
+config.test_format = lit.formats.GoogleTest('DummySubDir', 'Test')

diff  --git a/llvm/utils/lit/tests/googletest-sanitizer-error.py b/llvm/utils/lit/tests/googletest-sanitizer-error.py
new file mode 100644
index 000000000000..741ad088424d
--- /dev/null
+++ b/llvm/utils/lit/tests/googletest-sanitizer-error.py
@@ -0,0 +1,31 @@
+# Check the output is expected when tests pass but sanitizer fails.
+# Note that there is only one shard which has only one sub-test. However, the summary
+# has one pass for the sub-test and one fail for the shard failure due to sanitizer
+# reported errors.
+
+# RUN: not %{lit} -v --order=random %{inputs}/googletest-sanitizer-error > %t.out
+# FIXME: Temporarily dump test output so we can debug failing tests on
+# buildbots.
+# RUN: cat %t.out
+# RUN: FileCheck < %t.out %s
+#
+# END.
+
+# CHECK: -- Testing:
+# CHECK: FAIL: googletest-sanitizer-error :: [[PATH:[Dd]ummy[Ss]ub[Dd]ir/]][[FILE:OneTest\.py]]/0
+# CHECK: *** TEST 'googletest-sanitizer-error :: [[PATH]][[FILE]]/0{{.*}} FAILED ***
+# CHECK-NEXT: Script(shard):
+# CHECK-NEXT: --
+# CHECK-NEXT: GTEST_OUTPUT=json:{{[^[:space:]]*}} GTEST_SHUFFLE=1 GTEST_TOTAL_SHARDS={{[1-6]}} GTEST_SHARD_INDEX=0 GTEST_RANDOM_SEED=123 {{.*}}[[FILE]]
+# CHECK-NEXT: --
+# CHECK-EMPTY:
+# CHECK-EMPTY:
+# CHECK:      [ RUN      ] FirstTest.subTestA
+# CHECK-NEXT: [       OK ] FirstTest.subTestA (8 ms)
+# CHECK:      --
+# CHECK-NEXT: exit: 1
+# CHECK-NEXT: --
+# CHECK:      Failed Tests (1):
+# CHECK-NEXT:   googletest-sanitizer-error :: [[PATH]][[FILE]]/0/1
+# CHECK: Passed{{ *}}: 1
+# CHECK: Failed{{ *}}: 1