commit a215673da0a89b0f5cfbc4f6a111bcfd3fa2fe17
author William A. Kennington III <wak@google.com> Sat Jun 30 18:38:09 2018 -0700
committer William A. Kennington III <wak@google.com> Mon Jul 16 11:30:54 2018 -0700
tree e1baab2d7db8229e83bf7cf8ed9f25b450ebc8ae
parent 82c8d73ae68030f5c987e9bcfb7bcb33c234c353

unit-test: Enable parallel building

Running one off unit test builds could saturate more of the provided CPU
if we enabled parallel make steps. Currently our make processes just use
single threaded mode. This allows us to use all of the available CPUs
when building and testing projects which vastly speeds up builds on
projects with many dependencies when run on machines with many cores. It
also ensures that our builds have specified their dependencies correctly
since the Makefile will be run out of order and based more on the DAG.

Change-Id: I8dfefa9bb4d314143e020c32d40b82661e0a3981
Signed-off-by: William A. Kennington III <wak@google.com>

scripts/unit-test.py

diff --git a/scripts/unit-test.py b/scripts/unit-test.py
index ae02d59..630ba63 100755
--- a/scripts/unit-test.py
+++ b/scripts/unit-test.py
@@ -13,6 +13,7 @@
 import os
 import sys
 import argparse
+import multiprocessing
 import re
 
 
@@ -274,6 +275,16 @@
     return deps
 
 
+make_parallel = [
+    'make',
+    # Run enough jobs to saturate all the cpus
+    '-j', str(multiprocessing.cpu_count()),
+    # Don't start more jobs if the load avg is too high
+    '-l', str(multiprocessing.cpu_count()),
+    # Synchronize the output so logs aren't intermixed in stdout / stderr
+    '-O',
+]
+
 def install_deps(dep_list):
     """
     Install each package in the ordered dep_list.
@@ -296,8 +307,8 @@
             conf_flags.extend(CONFIGURE_FLAGS.get(pkg))
         check_call_cmd(pkgdir, './bootstrap.sh')
         check_call_cmd(pkgdir, './configure', *conf_flags)
-        check_call_cmd(pkgdir, 'make')
-        check_call_cmd(pkgdir, 'make', 'install')
+        check_call_cmd(pkgdir, *make_parallel)
+        check_call_cmd(pkgdir, *(make_parallel + [ 'install' ]))
 
 
 def build_dep_tree(pkg, pkgdir, dep_added, head, dep_tree=None):
@@ -443,7 +454,7 @@
     check_call_cmd(os.path.join(WORKSPACE, UNIT_TEST_PKG), 'ldconfig')
     # Run package unit tests
     try:
-        cmd = [ 'make', 'check' ]
+        cmd = make_parallel + [ 'check' ]
         for i in range(0, args.repeat):
             check_call_cmd(os.path.join(WORKSPACE, UNIT_TEST_PKG),  *cmd)
     except CalledProcessError:
@@ -460,7 +471,7 @@
             cmd = [ 'make', '-n', 'check-valgrind' ]
             check_call(cmd, stdout=devnull, stderr=devnull)
             try:
-                cmd = [ 'make', 'check-valgrind' ]
+                cmd = make_parallel + [ 'check-valgrind' ]
                 check_call_cmd(top_dir,  *cmd)
             except CalledProcessError:
                 for root, _, files in os.walk(top_dir):
@@ -477,7 +488,7 @@
             cmd = [ 'make', '-n', 'check-code-coverage' ]
             check_call(cmd, stdout=devnull, stderr=devnull)
             try:
-                cmd = [ 'make', 'check-code-coverage' ]
+                cmd = make_parallel + [ 'check-code-coverage' ]
                 check_call_cmd(top_dir,  *cmd)
             except CalledProcessError:
                 raise Exception('Code coverage failed')