[infra] Fix build.py and gn.py to exit on SIGINT

It is very tricky to make scripts using the python multiprocessing
package terminate cleanly on SIGINT (ctrl-c).
Use subprocesses run in parallel instead for the simple uses
in these scripts.

This is necessary when running build.py from within another script,
so that it terminates cleanly when the process group receives
a broadcast SIGINT.

Change-Id: Ifc858225a49f369c4bd9cee62639ac6fc8ff737e
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/171120
Commit-Queue: William Hesse <whesse@google.com>
Reviewed-by: Jonas Termansen <sortie@google.com>

tools/build.py

@@ -7,7 +7,6 @@
 import argparse
 import io
 import json
-import multiprocessing
 import os
 import subprocess
 import sys
@@ -193,18 +192,6 @@ def RunOneBuildCommand(build_config, args, env):
     return 0
 
 
-def RunOneGomaBuildCommand(options):
-    (env, args) = options
-    try:
-        print(' '.join(args))
-        process = subprocess.Popen(args, env=env, stdin=None)
-        process.wait()
-        print(' '.join(args) + " done.")
-        return process.returncode
-    except KeyboardInterrupt:
-        return 1
-
-
 def SanitizerEnvironmentVariables():
     with io.open('tools/bots/test_matrix.json', encoding='utf-8') as fd:
         config = json.loads(fd.read())
@@ -266,11 +253,22 @@ def Main():
             return 1
 
     # Run goma builds in parallel.
-    pool = multiprocessing.Pool(multiprocessing.cpu_count())
-    results = pool.map(RunOneGomaBuildCommand, goma_builds, chunksize=1)
-    for r in results:
-        if r != 0:
-            return 1
+    active_goma_builds = []
+    for (env, args) in goma_builds:
+        print(' '.join(args))
+        process = subprocess.Popen(args, env=env)
+        active_goma_builds.append([args, process])
+    while active_goma_builds:
+        time.sleep(0.1)
+        for goma_build in active_goma_builds:
+            (args, process) = goma_build
+            if process.poll() is not None:
+                print(' '.join(args) + " done.")
+                active_goma_builds.remove(goma_build)
+                if process.returncode != 0:
+                    for (_, to_kill) in active_goma_builds:
+                        to_kill.terminate()
+                    return 1
 
     endtime = time.time()
     print("The build took %.3f seconds" % (endtime - starttime))

tools/gn.py

@@ -4,7 +4,6 @@
 # found in the LICENSE file.
 
 import argparse
-import multiprocessing
 import os
 import shutil
 import subprocess
@@ -471,12 +470,6 @@ def AddCommonConfigurationArgs(parser):
 
 def AddOtherArgs(parser):
     """Adds miscellaneous arguments to the parser."""
-    parser.add_argument('--workers',
-                        '-w',
-                        type=int,
-                        help='Number of simultaneous GN invocations',
-                        dest='workers',
-                        default=multiprocessing.cpu_count())
     parser.add_argument("-v",
                         "--verbose",
                         help='Verbose output.',
@@ -506,18 +499,6 @@ def parse_args(args):
     return options
 
 
-# Run the command, if it succeeds returns 0, if it fails, returns the commands
-# output as a string.
-def RunCommand(command):
-    try:
-        subprocess.check_output(
-            command, cwd=DART_ROOT, stderr=subprocess.STDOUT)
-        return 0
-    except subprocess.CalledProcessError as e:
-        return ("Command failed: " + ' '.join(command) + "\n" + "output: " +
-                e.output)
-
-
 def BuildGnCommand(args, mode, arch, target_os, sanitizer, out_dir):
     gn = os.path.join(DART_ROOT, 'buildtools',
                       'gn.exe' if utils.IsWindows() else 'gn')
@@ -551,24 +532,43 @@ def RunGnOnConfiguredConfigurations(args):
                     if args.verbose:
                         print("gn gen --check in %s" % out_dir)
 
-    pool = multiprocessing.Pool(args.workers)
-    results = pool.map(RunCommand, commands, chunksize=1)
-    for r in results:
-        if r != 0:
-            print(r.strip())
+    active_commands = []
+
+    def cleanup(command):
+        print("Command failed: " + ' '.join(command))
+        for (_, process) in active_commands:
+            process.terminate()
+
+    for command in commands:
+        try:
+            process = subprocess.Popen(command, cwd=DART_ROOT)
+            active_commands.append([command, process])
+        except Exception as e:
+            print('Error: %s' % e)
+            cleanup(command)
             return 1
+    while active_commands:
+        time.sleep(0.1)
+        for active_command in active_commands:
+            (command, process) = active_command
+            if process.poll() is not None:
+                active_commands.remove(active_command)
+                if process.returncode != 0:
+                    cleanup(command)
+                    return 1
+    return 0
 
 
 def Main(argv):
     starttime = time.time()
     args = parse_args(argv)
 
-    RunGnOnConfiguredConfigurations(args)
+    result = RunGnOnConfiguredConfigurations(args)
 
     endtime = time.time()
     if args.verbose:
         print("GN Time: %.3f seconds" % (endtime - starttime))
-    return 0
+    return result
 
 
 if __name__ == '__main__':