[lldb-dev] [PATCH] make dosep.ty test driver multi-threadable

[Steve Pucci]

Hi all,

This patch provides support for running the dosep.ty test driver with
multiple threads.  It speeds up running the full test suite on my HP z620
Ubuntu machine with 32 hyperthreaded CPUs from 11 minutes to about 1m13s
(about 9x).

The default behavior is to run single-threaded as before.  If the
environment variable LLDB_TEST_THREADS is set, a Python work queue is set
up with that many worker threads.

To avoid collisions within a test directory where multiple tests make use
of the same prebuilt executable, the unit of work for the worker threads is
a single directory (that is, all tests within a directory are processed in
the normal serial way by a single thread).

I've run this way a number of times; the only issue I found was that the
TestProcessAttach.py test failed once, when attempting to attach to the
process "a.out" by name.  I assume this is because some other thread was
running an executable of that name at the same time, and we were attempting
to attach to the wrong one, so I changed that test to use a different
executable name (that change is also included in the attached patch).

There is actually an opportunity to speed the overall test time further
(maybe as much as another 1.5x or so) by dividing up a few long-running
tests into multiple directories, but I'm not planning to do that now.

This doesn't yet work with ninja makes, but tfiala or I will look at that
as well.

 - Steve
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.llvm.org/pipermail/lldb-dev/attachments/20140305/f59f62a7/attachment.html>
-------------- next part --------------
Index: test/dosep.ty
===================================================================
--- test/dosep.ty	(revision 202902)
+++ test/dosep.ty	(working copy)
@@ -5,34 +5,79 @@
 """
 
 import os, sys, platform
+import Queue, threading
+
 from optparse import OptionParser
 
 # Command template of the invocation of the test driver.
 template = '%s/dotest.py %s -p %s %s'
 
-def walk_and_invoke(test_root, dotest_options):
-    """Look for matched file and invoke test driver on it."""
+def process_dir(root, files, test_root, dotest_options):
+    """Examine a directory for tests, and invoke any found within it."""
     failed = []
     passed = []
-    for root, dirs, files in os.walk(test_root, topdown=False):
-        for name in files:
-            path = os.path.join(root, name)
+    for name in files:
+        path = os.path.join(root, name)
 
-            # We're only interested in the test file with the "Test*.py" naming pattern.
-            if not name.startswith("Test") or not name.endswith(".py"):
-                continue
+        # We're only interested in the test file with the "Test*.py" naming pattern.
+        if not name.startswith("Test") or not name.endswith(".py"):
+            continue
 
-            # Neither a symbolically linked file.
-            if os.path.islink(path):
-                continue
+        # Neither a symbolically linked file.
+        if os.path.islink(path):
+            continue
 
-            command = template % (test_root, dotest_options if dotest_options else "", name, root)
-            if 0 != os.system(command):
-                failed.append(name) 
-            else:
-                passed.append(name)
+        command = template % (test_root, dotest_options if dotest_options else "", name, root)
+        if 0 != os.system(command):
+            failed.append(name)
+        else:
+            passed.append(name)
     return (failed, passed)
 
+in_q = None
+out_q = None
+
+def process_dir_worker():
+    """Worker thread main loop when in multithreaded mode.
+    Takes one directory specification at a time and works on it."""
+    while True:
+        (root, files, test_root, dotest_options) = in_q.get()
+        (dir_failed, dir_passed) = process_dir(root, files, test_root, dotest_options)
+        out_q.put((dir_failed, dir_passed))
+        in_q.task_done()
+
+def walk_and_invoke(test_root, dotest_options, num_threads):
+    """Look for matched files and invoke test driver on each one.
+    In single-threaded mode, each test driver is invoked directly.
+    In multi-threaded mode, submit each test driver to a worker
+    queue, and then wait for all to complete."""
+    failed = []
+    passed = []
+    if (num_threads > 1):
+        print "Running multithreaded with " + str(num_threads) + " threads."
+        global in_q
+        global out_q
+        in_q = Queue.Queue()
+        out_q = Queue.Queue()
+        for i in range(num_threads):
+            t = threading.Thread(target=process_dir_worker)
+            t.daemon = True
+            t.start()
+    for root, dirs, files in os.walk(test_root, topdown=False):
+        if (num_threads > 1):
+            in_q.put((root, files, test_root, dotest_options))
+        else:
+            (dir_failed, dir_passed) = process_dir(root, files, test_root, dotest_options)
+            failed += dir_failed
+            passed += dir_passed
+    if (num_threads > 1):
+        in_q.join()
+        while not out_q.empty():
+            (dir_failed, dir_passed) = out_q.get()
+            failed += dir_failed
+            passed += dir_passed
+    return (failed, passed)
+
 def main():
     test_root = sys.path[0]
 
@@ -46,9 +91,16 @@
 
     opts, args = parser.parse_args()
     dotest_options = opts.dotest_options
+    num_threads_str = os.environ.get("LLDB_TEST_THREADS")
+    if num_threads_str:
+        num_threads = int(num_threads_str)
+        if num_threads < 1:
+            num_threads = 1
+    else:
+        num_threads = 1
 
     system_info = " ".join(platform.uname())
-    (failed, passed) = walk_and_invoke(test_root, dotest_options)
+    (failed, passed) = walk_and_invoke(test_root, dotest_options, num_threads)
     num_tests = len(failed) + len(passed)
 
     print "Ran %d tests." % num_tests
Index: test/functionalities/process_attach/Makefile
===================================================================
--- test/functionalities/process_attach/Makefile	(revision 202902)
+++ test/functionalities/process_attach/Makefile	(working copy)
@@ -2,4 +2,6 @@
 
 C_SOURCES := main.c
 
+EXE := ProcessAttach
+
 include $(LEVEL)/Makefile.rules
Index: test/functionalities/process_attach/TestProcessAttach.py
===================================================================
--- test/functionalities/process_attach/TestProcessAttach.py	(revision 202902)
+++ test/functionalities/process_attach/TestProcessAttach.py	(working copy)
@@ -8,6 +8,8 @@
 from lldbtest import *
 import lldbutil
 
+exe_name = "ProcessAttach"  # Must match Makefile
+
 class ProcessAttachTestCase(TestBase):
 
     mydir = TestBase.compute_mydir(__file__)
@@ -45,7 +47,7 @@
     def process_attach_by_id(self):
         """Test attach by process id"""
 
-        exe = os.path.join(os.getcwd(), "a.out")
+        exe = os.path.join(os.getcwd(), exe_name)
 
         # Spawn a new process
         popen = self.spawnSubprocess(exe)
@@ -62,13 +64,13 @@
     def process_attach_by_name(self):
         """Test attach by process name"""
 
-        exe = os.path.join(os.getcwd(), "a.out")
+        exe = os.path.join(os.getcwd(), exe_name)
 
         # Spawn a new process
         popen = self.spawnSubprocess(exe)
         self.addTearDownHook(self.cleanupSubprocesses)
 
-        self.runCmd("process attach -n a.out")
+        self.runCmd("process attach -n " + exe_name)
 
         target = self.dbg.GetSelectedTarget()