[infra] Simplify RBE configuration and allow concurrent builds.

Automatically detect the appropriate configuration file if RBE=1 is
set to request RBE on all projects, or if DART_RBE=1 is set to request
RBE for Dart only, or otherwise respect the explicit configuration file.

Automatically set the server_address location to the build directory
if it has not already been set. This is unfortunately not supported on
Windows due to the environment variable being set during build but it
needed to be set during gn.

Retain the older configuration files during the transitory period.

Bug: b/296994239
Fixes: b/320876546
Change-Id: I62d1fbfed35248477731cceda3f7267c605c4969
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/355400
Commit-Queue: Jonas Termansen <sortie@google.com>
Reviewed-by: William Hesse <whesse@google.com>

build/rbe/unix.cfg

@@ -0,0 +1,6 @@
+service=remotebuildexecution.googleapis.com:443
+instance=projects/flutter-rbe-prod/instances/default
+use_application_default_credentials=true
+enable_deps_cache=true
+xattr_digest=user.fuchsia.rbe.digest.sha256
+log_format=reducedtext

build/rbe/windows.cfg

@@ -0,0 +1,6 @@
+service=remotebuildexecution.googleapis.com:443
+instance=projects/flutter-rbe-prod/instances/default
+use_application_default_credentials=true
+enable_deps_cache=true
+server_address=pipe://reproxy.ipc
+log_format=reducedtext

build/toolchain/win/setup_toolchain.py

@@ -40,6 +40,7 @@ def _ExtractImportantEnvironment(output_of_set):
       'path',
       'pathext',
       'rbe_cfg', # Dart specific patch: RBE_cfg is needed by reclient.
+      'rbe_server_address', # Dart specific patch: RBE_server_address ditto.
       'systemroot',
       'temp',
       'tmp',

tools/build.py

@@ -140,7 +140,7 @@ rbe_started = None
 bootstrap_path = None
 
 
-def StartRBE(out_dir, use_goma):
+def StartRBE(out_dir, use_goma, env):
     global rbe_started, bootstrap_path
     rbe = 'goma' if use_goma else 'rbe'
     if rbe_started:
@@ -161,12 +161,27 @@ def StartRBE(out_dir, use_goma):
     if not os.path.exists(rbe_dir) or not os.path.isdir(rbe_dir):
         print(f'Could not find {rbe} at {rbe_dir}')
         return False
+    RBE_cfg = 'RBE_CFG' if HOST_OS == 'win32' else 'RBE_cfg'
+    RBE_server_address = ('RBE_SERVER_ADDRESS'
+                          if HOST_OS == 'win32' else 'RBE_server_address')
+    if not use_goma and not RBE_cfg in env:
+        env[RBE_cfg] = os.path.join(
+            os.getcwd(), 'build', 'rbe',
+            'windows.cfg' if HOST_OS == 'win32' else 'unix.cfg')
+    if not use_goma and not RBE_server_address in env:
+        with open(env[RBE_cfg], 'r') as f:
+            if not any([l.startswith('server_address') for l in f.readlines()]):
+                schema = 'pipe' if HOST_OS == 'win32' else 'unix'
+                socket = os.path.join(os.getcwd(), out_dir, 'reproxy.sock')
+                if HOST_OS == 'win32':
+                    socket = socket.replace('\\', '_').replace(':', '_')
+                env[RBE_server_address] = f'{schema}://{socket}'
     bootstrap = 'goma_ctl.py' if use_goma else 'bootstrap'
     bootstrap_path = os.path.join(rbe_dir, bootstrap)
     bootstrap_command = [bootstrap_path]
     if use_goma:
         bootstrap_command = ['python3', bootstrap_path, 'ensure_start']
-    process = subprocess.Popen(bootstrap_command)
+    process = subprocess.Popen(bootstrap_command, env=env)
     process.wait()
     if process.returncode != 0:
         print(f"Starting {rbe} failed. Try running it manually: " + "\n\t" +
@@ -186,7 +201,7 @@ def StopRBE():
 
 
 # Returns a tuple (build_config, command to run, whether rbe is used)
-def BuildOneConfig(options, targets, target_os, mode, arch, sanitizer):
+def BuildOneConfig(options, targets, target_os, mode, arch, sanitizer, env):
     build_config = utils.GetBuildConf(mode, arch, target_os, sanitizer)
     out_dir = utils.GetBuildRoot(HOST_OS, mode, arch, target_os, sanitizer)
     using_rbe = False
@@ -196,7 +211,7 @@ def BuildOneConfig(options, targets, target_os, mode, arch, sanitizer):
     use_rbe = UseRBE(out_dir)
     use_goma = UseGoma(out_dir)
     if use_rbe or use_goma:
-        if options.no_start_rbe or StartRBE(out_dir, use_goma):
+        if options.no_start_rbe or StartRBE(out_dir, use_goma, env):
             using_rbe = True
             command += [('-j%s' % str(options.j))]
             command += [('-l%s' % str(options.l))]
@@ -337,7 +352,7 @@ def Main():
                 for sanitizer in options.sanitizer:
                     configs.append(
                         BuildOneConfig(options, targets, target_os, mode, arch,
-                                       sanitizer))
+                                       sanitizer, env))
 
     exit_code = Build(configs, env, options)
 

tools/gn.py

@@ -411,13 +411,21 @@ def ProcessOptions(args):
     if HOST_OS != 'win' and args.use_crashpad:
         print("Crashpad is only supported on Windows")
         return False
-    if os.environ.get('RBE_cfg') == None and \
+    if os.environ.get('RBE') != '1' and \
+       os.environ.get('DART_RBE') != '1' and \
+       os.environ.get('RBE_cfg') == None and \
        (socket.getfqdn().endswith('.corp.google.com') or
-        socket.getfqdn().endswith('.c.googlers.com')) and \
-       sys.platform in ['linux']:
+        socket.getfqdn().endswith('.c.googlers.com')):
         print('You can speed up your build by following: go/dart-rbe')
         if not args.rbe and not args.goma:
             print('Goma is no longer enabled by default since RBE is ready.')
+    old_rbe_cfg = 'win-intel.cfg' if HOST_OS == 'win32' else 'linux-intel.cfg'
+    new_rbe_cfg = 'windows.cfg' if HOST_OS == 'win32' else 'unix.cfg'
+    if os.environ.get('RBE_cfg') == os.path.join(os.getcwd(), 'build', 'rbe',
+                                                 old_rbe_cfg):
+        print(f'warning: {old_rbe_cfg} is deprecated, please update your '
+              f'RBE_cfg variable to {new_rbe_cfg} use RBE=1 instead per '
+              'go/dart-rbe')
     return True
 
 
@@ -437,7 +445,9 @@ def ide_switch(host_os):
 def AddCommonGnOptionArgs(parser):
     """Adds arguments that will change the default GN arguments."""
 
-    use_rbe = os.environ.get('RBE_cfg') != None
+    use_rbe = os.environ.get('RBE') == '1' or \
+              os.environ.get('DART_RBE') == '1' or \
+              os.environ.get('RBE_cfg') != None
 
     parser.add_argument('--goma', help='Use goma', action='store_true')
     parser.add_argument('--no-goma',