Multiprocessing tests runner (#267)

There are a few hacks to preserve quick Ctrl-C behavior.
I also added timings to the output and progress numbers.

test/test_all.py

@@ -10,8 +10,11 @@ from __future__ import print_function
 
 import BoostBuild
 
+import concurrent.futures
 import os
 import os.path
+import time
+import signal
 import sys
 
 xml = "--xml" in sys.argv
@@ -30,6 +33,24 @@ for s in ("BOOST_ROOT", "BOOST_BUILD_PATH", "JAM_TOOLSET", "BCCROOT",
 BoostBuild.set_defer_annotations(1)
 
 
+def iterfutures(futures):
+    while futures:
+        done, futures = concurrent.futures.wait(
+            futures,return_when=concurrent.futures.FIRST_COMPLETED)
+        for future in done:
+            yield future, futures
+
+
+def run_test(test):
+    ts = time.perf_counter()
+    exc = None
+    try:
+        __import__(test)
+    except BaseException as e:
+        exc = e
+    return test, time.perf_counter() - ts, exc, BoostBuild.annotations
+
+
 def run_tests(critical_tests, other_tests):
     """
       Runs first the critical_tests and then the other_tests.
@@ -49,21 +70,48 @@ def run_tests(critical_tests, other_tests):
         if len(x) > max_test_name_len:
             max_test_name_len = len(x)
 
+    cancelled = False
+    executor = concurrent.futures.ProcessPoolExecutor()
+
+    def handler(sig, frame):
+        cancelled = True
+        processes = executor._processes.values()
+        executor.shutdown(wait=False, cancel_futures=True)
+        for process in processes:
+            process.terminate()
+
+    signal.signal(signal.SIGINT, handler)
+
     pass_count = 0
     failures_count = 0
-
-    for test in all_tests:
+    start_ts = time.perf_counter()
+    isatty = sys.stdout.isatty() or "--interactive" in sys.argv
+    futures = {executor.submit(run_test, test): test for test in all_tests}
+    for future, pending in iterfutures(futures):
+        test = futures[future]
         if not xml:
             s = "%%-%ds :" % max_test_name_len % test
+            if isatty:
+                s = f"\r{s}"
             print(s, end='')
 
         passed = 0
+        ts = float('nan')
         try:
-            __import__(test)
+            test, ts, exc, annotations = future.result()
+            BoostBuild.annotations += annotations
+            if exc is not None:
+                raise exc from None
             passed = 1
+        except concurrent.futures.process.BrokenProcessPool:
+            # It could be us who broke the pool by terminating its threads
+            if not cancelled:
+                raise
         except KeyboardInterrupt:
             """This allows us to abort the testing manually using Ctrl-C."""
-            raise
+            print("\n\nTesting was cancelled by external signal.")
+            cancelled = True
+            break
         except SystemExit as e:
             """This is the regular way our test scripts are supposed to report
             test failures."""
@@ -98,10 +146,19 @@ def run_tests(critical_tests, other_tests):
 
         if not xml:
             if passed:
-                print("PASSED")
+                print(f"PASSED {ts * 1000:>5.0f}ms")
             else:
-                print("FAILED")
+                print(f"FAILED {ts * 1000:>5.0f}ms")
                 BoostBuild.flush_annotations()
+
+            if isatty:
+                msg = ", ".join(futures[future] for future in pending if future.running())
+                if msg:
+                    msg = f"[{len(futures) - len(pending)}/{len(futures)}] {msg}"
+                    max_len = max_test_name_len + len(" :PASSED 12345ms")
+                    if len(msg) > max_len:
+                        msg = msg[:max_len - 3] + "..."
+                    print(msg, end='')
         else:
             rs = "succeed"
             if not passed:
@@ -124,14 +181,15 @@ def run_tests(critical_tests, other_tests):
         open("test_results.txt", "w").close()
 
     if not xml:
-        print('''
+        print(f'''
         === Test summary ===
-        PASS: %d
-        FAIL: %d
-        ''' % (pass_count, failures_count))
+        PASS: {pass_count}
+        FAIL: {failures_count}
+        TIME: {time.perf_counter() - start_ts:.0f}s
+        ''')
 
     # exit with failure with failures
-    if failures_count > 0:
+    if cancelled or failures_count > 0:
         sys.exit(1)
 
 def last_failed_test():
@@ -367,7 +425,8 @@ if "--extras" in sys.argv:
     tests.append("example_customization")
     # Requires gettext tools.
     tests.append("example_gettext")
-elif not xml:
+elif not xml and __name__ == "__main__":
     print("Note: skipping extra tests")
 
-run_tests(critical_tests, tests)
+if __name__ == "__main__":
+    run_tests(critical_tests, tests)