Allows adding a storage 'realm' to the storage base URL (#131951)

Context: https://github.com/flutter/flutter/issues/131862

This PR injects a "realm" component to the storage base URL when the contents of the file `bin/internal/engine.realm` is non-empty.

As documented in the PR, when the realm is `flutter_archives_v2`, and `bin/internal/engine.version` contains the commit hash for a commit in a `flutter/engine` PR, then the artifacts pulled by the tool will be the artifacts built by the presubmit checks for the PR.

This works for everything but the following two cases:
1. Fuchsia artifacts are not uploaded to CIPD by the Fuchsia presubmit builds.
2. Web artifacts are not uploaded to gstatic by the web engine presubmit builds.

For (1), the flutter/flutter presubmit `fuchsia_precache` is driven by a shell script outside of the repo. It will fail when the `engine.version` and `engine.realm` don't point to a post-submit engine commit.

For (2), the flutter/flutter web presubmit tests that refer to artifacts in gstatic hang when the artifacts aren't found, so this PR skips them.

bin/internal/update_dart_sdk.ps1

@@ -20,6 +20,7 @@ $cachePath = "$flutterRoot\bin\cache"
 $dartSdkPath = "$cachePath\dart-sdk"
 $engineStamp = "$cachePath\engine-dart-sdk.stamp"
 $engineVersion = (Get-Content "$flutterRoot\bin\internal\engine.version")
+$engineRealm = (Get-Content "$flutterRoot\bin\internal\engine.realm")
 
 $oldDartSdkPrefix = "dart-sdk.old"
 
@@ -42,6 +43,9 @@ $dartSdkBaseUrl = $Env:FLUTTER_STORAGE_BASE_URL
 if (-not $dartSdkBaseUrl) {
     $dartSdkBaseUrl = "https://storage.googleapis.com"
 }
+if ($engineRealm) {
+    $dartSdkBaseUrl = "$dartSdkBaseUrl/$engineRealm"
+}
 $dartZipName = "dart-sdk-windows-x64.zip"
 $dartSdkUrl = "$dartSdkBaseUrl/flutter_infra_release/flutter/$engineVersion/$dartZipName"
 

bin/internal/update_dart_sdk.sh

@@ -20,6 +20,7 @@ DART_SDK_PATH="$FLUTTER_ROOT/bin/cache/dart-sdk"
 DART_SDK_PATH_OLD="$DART_SDK_PATH.old"
 ENGINE_STAMP="$FLUTTER_ROOT/bin/cache/engine-dart-sdk.stamp"
 ENGINE_VERSION=`cat "$FLUTTER_ROOT/bin/internal/engine.version"`
+ENGINE_REALM=`cat "$FLUTTER_ROOT/bin/internal/engine.realm"`
 OS="$(uname -s)"
 
 if [ ! -f "$ENGINE_STAMP" ] || [ "$ENGINE_VERSION" != `cat "$ENGINE_STAMP"` ]; then
@@ -121,7 +122,7 @@ if [ ! -f "$ENGINE_STAMP" ] || [ "$ENGINE_VERSION" != `cat "$ENGINE_STAMP"` ]; t
     FIND=find
   fi
 
-  DART_SDK_BASE_URL="${FLUTTER_STORAGE_BASE_URL:-https://storage.googleapis.com}"
+  DART_SDK_BASE_URL="${FLUTTER_STORAGE_BASE_URL:-https://storage.googleapis.com}${ENGINE_REALM:+/$ENGINE_REALM}"
   DART_SDK_URL="$DART_SDK_BASE_URL/flutter_infra_release/flutter/$ENGINE_VERSION/$DART_ZIP_NAME"
 
   # if the sdk path exists, copy it to a temporary location

dev/bots/test.dart

@@ -83,6 +83,7 @@ final String flutter = path.join(flutterRoot, 'bin', 'flutter$bat');
 final String dart = path.join(flutterRoot, 'bin', 'cache', 'dart-sdk', 'bin', 'dart$exe');
 final String pubCache = path.join(flutterRoot, '.pub-cache');
 final String engineVersionFile = path.join(flutterRoot, 'bin', 'internal', 'engine.version');
+final String engineRealmFile = path.join(flutterRoot, 'bin', 'internal', 'engine.realm');
 final String flutterPackagesVersionFile = path.join(flutterRoot, 'bin', 'internal', 'flutter_packages.version');
 
 String get platformFolderName {
@@ -1138,6 +1139,10 @@ Future<void> _runWebUnitTests(String webRenderer) async {
 /// Coarse-grained integration tests running on the Web.
 Future<void> _runWebLongRunningTests() async {
   final String engineVersion = File(engineVersionFile).readAsStringSync().trim();
+  final String engineRealm = File(engineRealmFile).readAsStringSync().trim();
+  if (engineRealm.isNotEmpty) {
+    return;
+  }
   final List<ShardRunner> tests = <ShardRunner>[
     for (final String buildMode in _kAllBuildModes) ...<ShardRunner>[
       () => _runFlutterDriverWebTest(

dev/conductor/core/lib/src/codesign.dart

@@ -125,7 +125,15 @@ class CodesignCommand extends Command<void> {
     await framework.checkout(revision);
 
     // Ensure artifacts present
-    await framework.runFlutter(<String>['precache', '--android', '--ios', '--macos']);
+    final io.ProcessResult result = await framework.runFlutter(
+      <String>['precache', '--android', '--ios', '--macos'],
+    );
+    if (result.exitCode != 0) {
+      stdio.printError(
+        'flutter precache: exitCode: ${result.exitCode}\n'
+        'stdout:\n${result.stdout}\nstderr:\n${result.stderr}',
+      );
+    }
 
     await verifyExist();
     if (argResults![kSignatures] as bool) {

dev/devicelab/bin/tasks/engine_dependency_proxy_test.dart

@@ -30,7 +30,8 @@ Future<void> main() async {
       await inDirectory(path.join(flutterProject.rootPath, 'android'), () async {
         section('Insert gradle testing script');
         final File build = File(path.join(
-	    flutterProject.rootPath, 'android', 'app', 'build.gradle'));
+          flutterProject.rootPath, 'android', 'app', 'build.gradle',
+        ));
         build.writeAsStringSync(
           '''
 task printEngineMavenUrl() {
@@ -44,6 +45,7 @@ task printEngineMavenUrl() {
         );
 
         section('Checking default maven URL');
+
         String gradleOutput = await eval(
           gradlewExecutable,
           <String>['printEngineMavenUrl', '-q'],
@@ -53,29 +55,39 @@ task printEngineMavenUrl() {
         String mavenUrl = outputLines.last;
         print('Returned maven url: $mavenUrl');
 
-        if (mavenUrl != 'https://storage.googleapis.com/download.flutter.io') {
+        String realm = File(
-          throw TaskResult.failure('Expected Android engine maven dependency URL to '
+          path.join(flutterDirectory.path, 'bin', 'internal', 'engine.realm'),
-              'resolve to https://storage.googleapis.com/download.flutter.io. Got '
+        ).readAsStringSync().trim();
-              '$mavenUrl instead');
+        if (realm.isNotEmpty) {
+          realm = '$realm/';
+        }
+
+        if (mavenUrl != 'https://storage.googleapis.com/${realm}download.flutter.io') {
+          throw TaskResult.failure(
+            'Expected Android engine maven dependency URL to '
+            'resolve to https://storage.googleapis.com/${realm}download.flutter.io. Got '
+            '$mavenUrl instead',
+          );
         }
 
         section('Checking overridden maven URL');
         gradleOutput = await eval(
-	    gradlewExecutable,
+          gradlewExecutable,
-	    <String>['printEngineMavenUrl','-q'],
+          <String>['printEngineMavenUrl','-q'],
-	    environment: <String, String>{
+          environment: <String, String>{
-              'FLUTTER_STORAGE_BASE_URL': 'https://my.special.proxy',
+            'FLUTTER_STORAGE_BASE_URL': 'https://my.special.proxy',
-            }
+          },
-	);
+        );
         outputLines = splitter.convert(gradleOutput);
         mavenUrl = outputLines.last;
 
-        if (mavenUrl != 'https://my.special.proxy/download.flutter.io') {
+        if (mavenUrl != 'https://my.special.proxy/${realm}download.flutter.io') {
           throw TaskResult.failure(
-	      'Expected overridden Android engine maven '
+            'Expected overridden Android engine maven '
-              'dependency URL to resolve to proxy location '
+            'dependency URL to resolve to proxy location '
-              'https://my.special.proxy/download.flutter.io. Got '
+            'https://my.special.proxy/${realm}download.flutter.io. Got '
-              '$mavenUrl instead');
+            '$mavenUrl instead',
+          );
         }
       });
     });

dev/tools/java_and_objc_doc.dart

@@ -15,11 +15,15 @@ const String kDocRoot = 'dev/docs/doc';
 /// the artifact store and extracts them to the location used for Dartdoc.
 Future<void> main(List<String> args) async {
   final String engineVersion = File('bin/internal/engine.version').readAsStringSync().trim();
+  String engineRealm = File('bin/internal/engine.realm').readAsStringSync().trim();
+  if (engineRealm.isNotEmpty) {
+    engineRealm = '$engineRealm/';
+  }
 
-  final String javadocUrl = 'https://storage.googleapis.com/flutter_infra_release/flutter/$engineVersion/android-javadoc.zip';
+  final String javadocUrl = 'https://storage.googleapis.com/${engineRealm}flutter_infra_release/flutter/$engineVersion/android-javadoc.zip';
   generateDocs(javadocUrl, 'javadoc', 'io/flutter/view/FlutterView.html');
 
-  final String objcdocUrl = 'https://storage.googleapis.com/flutter_infra_release/flutter/$engineVersion/ios-objcdoc.zip';
+  final String objcdocUrl = 'https://storage.googleapis.com/${engineRealm}flutter_infra_release/flutter/$engineVersion/ios-objcdoc.zip';
   generateDocs(objcdocUrl, 'objcdoc', 'Classes/FlutterViewController.html');
 }
 

packages/flutter_tools/gradle/aar_init_script.gradle

@@ -45,11 +45,18 @@ void configureProject(Project project, String outputDir) {
     }
 
     String storageUrl = System.getenv('FLUTTER_STORAGE_BASE_URL') ?: "https://storage.googleapis.com"
+
+    String engineRealm = Paths.get(getFlutterRoot(project), "bin", "internal", "engine.realm")
+        .toFile().text.trim()
+    if (engineRealm) {
+        engineRealm = engineRealm + "/"
+    }
+
     // This is a Flutter plugin project. Plugin projects don't apply the Flutter Gradle plugin,
     // as a result, add the dependency on the embedding.
     project.repositories {
         maven {
-            url "$storageUrl/download.flutter.io"
+            url "$storageUrl/${engineRealm}download.flutter.io"
         }
     }
     String engineVersion = Paths.get(getFlutterRoot(project), "bin", "internal", "engine.version")

packages/flutter_tools/gradle/resolve_dependencies.gradle

@@ -16,11 +16,17 @@ import java.nio.file.Paths
 
 String storageUrl = System.getenv('FLUTTER_STORAGE_BASE_URL') ?: "https://storage.googleapis.com"
 
+String engineRealm = Paths.get(flutterRoot.absolutePath, "bin", "internal", "engine.realm")
+        .toFile().text.trim()
+if (engineRealm) {
+    engineRealm = engineRealm + "/"
+}
+
 repositories {
     google()
     mavenCentral()
     maven {
-        url "$storageUrl/download.flutter.io"
+        url "$storageUrl/${engineRealm}download.flutter.io"
     }
 }
 

packages/flutter_tools/gradle/src/main/groovy/flutter.groovy

@@ -167,6 +167,7 @@ class FlutterPlugin implements Plugin<Project> {
     private String localEngineSrcPath
     private Properties localProperties
     private String engineVersion
+    private String engineRealm
 
     /**
      * Flutter Docs Website URLs for help messages.
@@ -192,11 +193,29 @@ class FlutterPlugin implements Plugin<Project> {
             }
         }
 
+        String flutterRootPath = resolveProperty("flutter.sdk", System.env.FLUTTER_ROOT)
+        if (flutterRootPath == null) {
+            throw new GradleException("Flutter SDK not found. Define location with flutter.sdk in the local.properties file or with a FLUTTER_ROOT environment variable.")
+        }
+        flutterRoot = project.file(flutterRootPath)
+        if (!flutterRoot.isDirectory()) {
+            throw new GradleException("flutter.sdk must point to the Flutter SDK directory")
+        }
+
+        engineVersion = useLocalEngine()
+            ? "+" // Match any version since there's only one.
+            : "1.0.0-" + Paths.get(flutterRoot.absolutePath, "bin", "internal", "engine.version").toFile().text.trim()
+
+        engineRealm = Paths.get(flutterRoot.absolutePath, "bin", "internal", "engine.realm").toFile().text.trim()
+        if (engineRealm) {
+            engineRealm = engineRealm + "/"
+        }
+
         // Configure the Maven repository.
         String hostedRepository = System.env.FLUTTER_STORAGE_BASE_URL ?: DEFAULT_MAVEN_HOST
         String repository = useLocalEngine()
             ? project.property('local-engine-repo')
-            : "$hostedRepository/download.flutter.io"
+            : "$hostedRepository/${engineRealm}download.flutter.io"
         rootProject.allprojects {
             repositories {
                 maven {
@@ -246,19 +265,6 @@ class FlutterPlugin implements Plugin<Project> {
             }
         }
 
-        String flutterRootPath = resolveProperty("flutter.sdk", System.env.FLUTTER_ROOT)
-        if (flutterRootPath == null) {
-            throw new GradleException("Flutter SDK not found. Define location with flutter.sdk in the local.properties file or with a FLUTTER_ROOT environment variable.")
-        }
-        flutterRoot = project.file(flutterRootPath)
-        if (!flutterRoot.isDirectory()) {
-            throw new GradleException("flutter.sdk must point to the Flutter SDK directory")
-        }
-
-        engineVersion = useLocalEngine()
-            ? "+" // Match any version since there's only one.
-            : "1.0.0-" + Paths.get(flutterRoot.absolutePath, "bin", "internal", "engine.version").toFile().text.trim()
-
         String flutterExecutableName = Os.isFamily(Os.FAMILY_WINDOWS) ? "flutter.bat" : "flutter"
         flutterExecutable = Paths.get(flutterRoot.absolutePath, "bin", flutterExecutableName).toFile();
 

packages/flutter_tools/lib/src/cache.dart

@@ -190,6 +190,7 @@ class Cache {
       httpClient: HttpClient(),
       allowedBaseUrls: <String>[
         storageBaseUrl,
+        realmlessStorageBaseUrl,
         cipdBaseUrl,
       ],
     );
@@ -447,6 +448,22 @@ class Cache {
   }
   String? _engineRevision;
 
+  /// The "realm" for the storage URL.
+  ///
+  /// For production artifacts from Engine post-submit and release builds,
+  /// this string will be empty, and the `storageBaseUrl` will be unmodified.
+  /// When non-empty, this string will be appended to the `storageBaseUrl` after
+  /// a '/'. For artifacts generated by Engine presubmits, the realm should be
+  /// "flutter_archives_v2".
+  String get storageRealm {
+    _storageRealm ??= getRealmFor('engine');
+    if (_storageRealm == null) {
+      throwToolExit('Could not determine engine realm.');
+    }
+    return _storageRealm!;
+  }
+  String? _storageRealm;
+
   /// The base for URLs that store Flutter engine artifacts that are fetched
   /// during the installation of the Flutter SDK.
   ///
@@ -459,11 +476,14 @@ class Cache {
   ///  * [cipdBaseUrl], which determines how CIPD artifacts are fetched.
   ///  * [Cache] class-level dartdocs that explain how artifact mirrors work.
   String get storageBaseUrl {
-    final String? overrideUrl = _platform.environment[kFlutterStorageBaseUrl];
+    String? overrideUrl = _platform.environment[kFlutterStorageBaseUrl];
     if (overrideUrl == null) {
-      return 'https://storage.googleapis.com';
+      return storageRealm.isEmpty
+        ? 'https://storage.googleapis.com'
+        : 'https://storage.googleapis.com/$storageRealm';
     }
     // verify that this is a valid URI.
+    overrideUrl = storageRealm.isEmpty ? overrideUrl : '$overrideUrl/$storageRealm';
     try {
       Uri.parse(overrideUrl);
     } on FormatException catch (err) {
@@ -473,6 +493,12 @@ class Cache {
     return overrideUrl;
   }
 
+  String get realmlessStorageBaseUrl {
+    return storageRealm.isEmpty
+      ? storageBaseUrl
+      : storageBaseUrl.replaceAll('/$storageRealm', '');
+  }
+
   /// The base for URLs that store Flutter engine artifacts in CIPD.
   ///
   /// For some platforms, such as Web and Fuchsia, CIPD artifacts are fetched
@@ -607,6 +633,16 @@ class Cache {
     return versionFile.existsSync() ? versionFile.readAsStringSync().trim() : null;
   }
 
+  String? getRealmFor(String artifactName) {
+    final File realmFile = _fileSystem.file(_fileSystem.path.join(
+      _rootOverride?.path ?? flutterRoot!,
+      'bin',
+      'internal',
+      '$artifactName.realm',
+    ));
+    return realmFile.existsSync() ? realmFile.readAsStringSync().trim() : '';
+  }
+
   /// Delete all stamp files maintained by the cache.
   void clearStampFiles() {
     try {

packages/flutter_tools/lib/src/flutter_cache.dart

@@ -836,7 +836,11 @@ class IosUsbArtifacts extends CachedArtifact {
   }
 
   @visibleForTesting
-  Uri get archiveUri => Uri.parse('${cache.storageBaseUrl}/flutter_infra_release/ios-usb-dependencies${cache.useUnsignedMacBinaries ? '/unsigned' : ''}/$name/$version/$name.zip');
+  Uri get archiveUri => Uri.parse(
+    '${cache.realmlessStorageBaseUrl}/flutter_infra_release/'
+    'ios-usb-dependencies${cache.useUnsignedMacBinaries ? '/unsigned' : ''}'
+    '/$name/$version/$name.zip',
+  );
 }
 
 // TODO(zanderso): upload debug desktop artifacts to host-debug and

packages/flutter_tools/test/general.shard/cache_test.dart

@@ -334,6 +334,37 @@ void main() {
       expect(logger.warningText, contains('Flutter assets will be downloaded from $baseUrl'));
       expect(logger.statusText, isEmpty);
     });
+
+    testWithoutContext('a non-empty realm is included in the storage url', () async {
+      final MemoryFileSystem fileSystem = MemoryFileSystem.test();
+      final Directory internalDir = fileSystem.currentDirectory
+        .childDirectory('cache')
+        .childDirectory('bin')
+        .childDirectory('internal');
+      final File engineVersionFile = internalDir.childFile('engine.version');
+      engineVersionFile.createSync(recursive: true);
+      engineVersionFile.writeAsStringSync('abcdef');
+
+      final File engineRealmFile = internalDir.childFile('engine.realm');
+      engineRealmFile.createSync(recursive: true);
+      engineRealmFile.writeAsStringSync('flutter_archives_v2');
+
+      final Cache cache = Cache.test(
+        processManager: FakeProcessManager.any(),
+        fileSystem: fileSystem,
+      );
+
+      expect(cache.storageBaseUrl, contains('flutter_archives_v2'));
+    });
+
+    test('bin/internal/engine.realm is empty', () async {
+      final FileSystem fileSystem = globals.fs;
+      final String realmFilePath = fileSystem.path.join(
+        getFlutterRoot(), 'bin', 'internal', 'engine.realm');
+      final String realm = fileSystem.file(realmFilePath).readAsStringSync().trim();
+      expect(realm, isEmpty,
+        reason: 'The checked-in engine.realm file must be empty.');
+    });
   });
 
   testWithoutContext('flattenNameSubdirs', () {