From 118c2df7764dbac21268ad7d6239de9a0ccefd7d Mon Sep 17 00:00:00 2001 From: Zachary Anderson Date: Wed, 9 Aug 2023 16:26:05 -0700 Subject: [PATCH] Allows adding a storage 'realm' to the storage base URL (#131951) Context: https://github.com/flutter/flutter/issues/131862 This PR injects a "realm" component to the storage base URL when the contents of the file `bin/internal/engine.realm` is non-empty. As documented in the PR, when the realm is `flutter_archives_v2`, and `bin/internal/engine.version` contains the commit hash for a commit in a `flutter/engine` PR, then the artifacts pulled by the tool will be the artifacts built by the presubmit checks for the PR. This works for everything but the following two cases: 1. Fuchsia artifacts are not uploaded to CIPD by the Fuchsia presubmit builds. 2. Web artifacts are not uploaded to gstatic by the web engine presubmit builds. For (1), the flutter/flutter presubmit `fuchsia_precache` is driven by a shell script outside of the repo. It will fail when the `engine.version` and `engine.realm` don't point to a post-submit engine commit. For (2), the flutter/flutter web presubmit tests that refer to artifacts in gstatic hang when the artifacts aren't found, so this PR skips them. --- bin/internal/engine.realm | 0 bin/internal/update_dart_sdk.ps1 | 4 ++ bin/internal/update_dart_sdk.sh | 3 +- dev/bots/test.dart | 5 +++ dev/conductor/core/lib/src/codesign.dart | 10 ++++- .../tasks/engine_dependency_proxy_test.dart | 44 ++++++++++++------- dev/tools/java_and_objc_doc.dart | 8 +++- .../gradle/aar_init_script.gradle | 9 +++- .../gradle/resolve_dependencies.gradle | 8 +++- .../gradle/src/main/groovy/flutter.groovy | 34 ++++++++------ packages/flutter_tools/lib/src/cache.dart | 40 ++++++++++++++++- .../flutter_tools/lib/src/flutter_cache.dart | 6 ++- .../test/general.shard/cache_test.dart | 31 +++++++++++++ 13 files changed, 163 insertions(+), 39 deletions(-) create mode 100644 bin/internal/engine.realm diff --git a/bin/internal/engine.realm b/bin/internal/engine.realm new file mode 100644 index 00000000000..e69de29bb2d diff --git a/bin/internal/update_dart_sdk.ps1 b/bin/internal/update_dart_sdk.ps1 index f4e2e77a558..6af62a54f73 100644 --- a/bin/internal/update_dart_sdk.ps1 +++ b/bin/internal/update_dart_sdk.ps1 @@ -20,6 +20,7 @@ $cachePath = "$flutterRoot\bin\cache" $dartSdkPath = "$cachePath\dart-sdk" $engineStamp = "$cachePath\engine-dart-sdk.stamp" $engineVersion = (Get-Content "$flutterRoot\bin\internal\engine.version") +$engineRealm = (Get-Content "$flutterRoot\bin\internal\engine.realm") $oldDartSdkPrefix = "dart-sdk.old" @@ -42,6 +43,9 @@ $dartSdkBaseUrl = $Env:FLUTTER_STORAGE_BASE_URL if (-not $dartSdkBaseUrl) { $dartSdkBaseUrl = "https://storage.googleapis.com" } +if ($engineRealm) { + $dartSdkBaseUrl = "$dartSdkBaseUrl/$engineRealm" +} $dartZipName = "dart-sdk-windows-x64.zip" $dartSdkUrl = "$dartSdkBaseUrl/flutter_infra_release/flutter/$engineVersion/$dartZipName" diff --git a/bin/internal/update_dart_sdk.sh b/bin/internal/update_dart_sdk.sh index 0f1a2380488..8aed2f47861 100755 --- a/bin/internal/update_dart_sdk.sh +++ b/bin/internal/update_dart_sdk.sh @@ -20,6 +20,7 @@ DART_SDK_PATH="$FLUTTER_ROOT/bin/cache/dart-sdk" DART_SDK_PATH_OLD="$DART_SDK_PATH.old" ENGINE_STAMP="$FLUTTER_ROOT/bin/cache/engine-dart-sdk.stamp" ENGINE_VERSION=`cat "$FLUTTER_ROOT/bin/internal/engine.version"` +ENGINE_REALM=`cat "$FLUTTER_ROOT/bin/internal/engine.realm"` OS="$(uname -s)" if [ ! -f "$ENGINE_STAMP" ] || [ "$ENGINE_VERSION" != `cat "$ENGINE_STAMP"` ]; then @@ -121,7 +122,7 @@ if [ ! -f "$ENGINE_STAMP" ] || [ "$ENGINE_VERSION" != `cat "$ENGINE_STAMP"` ]; t FIND=find fi - DART_SDK_BASE_URL="${FLUTTER_STORAGE_BASE_URL:-https://storage.googleapis.com}" + DART_SDK_BASE_URL="${FLUTTER_STORAGE_BASE_URL:-https://storage.googleapis.com}${ENGINE_REALM:+/$ENGINE_REALM}" DART_SDK_URL="$DART_SDK_BASE_URL/flutter_infra_release/flutter/$ENGINE_VERSION/$DART_ZIP_NAME" # if the sdk path exists, copy it to a temporary location diff --git a/dev/bots/test.dart b/dev/bots/test.dart index 26fa9c6cdbc..2b48df03287 100644 --- a/dev/bots/test.dart +++ b/dev/bots/test.dart @@ -83,6 +83,7 @@ final String flutter = path.join(flutterRoot, 'bin', 'flutter$bat'); final String dart = path.join(flutterRoot, 'bin', 'cache', 'dart-sdk', 'bin', 'dart$exe'); final String pubCache = path.join(flutterRoot, '.pub-cache'); final String engineVersionFile = path.join(flutterRoot, 'bin', 'internal', 'engine.version'); +final String engineRealmFile = path.join(flutterRoot, 'bin', 'internal', 'engine.realm'); final String flutterPackagesVersionFile = path.join(flutterRoot, 'bin', 'internal', 'flutter_packages.version'); String get platformFolderName { @@ -1138,6 +1139,10 @@ Future _runWebUnitTests(String webRenderer) async { /// Coarse-grained integration tests running on the Web. Future _runWebLongRunningTests() async { final String engineVersion = File(engineVersionFile).readAsStringSync().trim(); + final String engineRealm = File(engineRealmFile).readAsStringSync().trim(); + if (engineRealm.isNotEmpty) { + return; + } final List tests = [ for (final String buildMode in _kAllBuildModes) ...[ () => _runFlutterDriverWebTest( diff --git a/dev/conductor/core/lib/src/codesign.dart b/dev/conductor/core/lib/src/codesign.dart index c5cf2038a6c..5691aefe07e 100644 --- a/dev/conductor/core/lib/src/codesign.dart +++ b/dev/conductor/core/lib/src/codesign.dart @@ -125,7 +125,15 @@ class CodesignCommand extends Command { await framework.checkout(revision); // Ensure artifacts present - await framework.runFlutter(['precache', '--android', '--ios', '--macos']); + final io.ProcessResult result = await framework.runFlutter( + ['precache', '--android', '--ios', '--macos'], + ); + if (result.exitCode != 0) { + stdio.printError( + 'flutter precache: exitCode: ${result.exitCode}\n' + 'stdout:\n${result.stdout}\nstderr:\n${result.stderr}', + ); + } await verifyExist(); if (argResults![kSignatures] as bool) { diff --git a/dev/devicelab/bin/tasks/engine_dependency_proxy_test.dart b/dev/devicelab/bin/tasks/engine_dependency_proxy_test.dart index 43cad174406..9443a29bb25 100644 --- a/dev/devicelab/bin/tasks/engine_dependency_proxy_test.dart +++ b/dev/devicelab/bin/tasks/engine_dependency_proxy_test.dart @@ -30,7 +30,8 @@ Future main() async { await inDirectory(path.join(flutterProject.rootPath, 'android'), () async { section('Insert gradle testing script'); final File build = File(path.join( - flutterProject.rootPath, 'android', 'app', 'build.gradle')); + flutterProject.rootPath, 'android', 'app', 'build.gradle', + )); build.writeAsStringSync( ''' task printEngineMavenUrl() { @@ -44,6 +45,7 @@ task printEngineMavenUrl() { ); section('Checking default maven URL'); + String gradleOutput = await eval( gradlewExecutable, ['printEngineMavenUrl', '-q'], @@ -53,29 +55,39 @@ task printEngineMavenUrl() { String mavenUrl = outputLines.last; print('Returned maven url: $mavenUrl'); - if (mavenUrl != 'https://storage.googleapis.com/download.flutter.io') { - throw TaskResult.failure('Expected Android engine maven dependency URL to ' - 'resolve to https://storage.googleapis.com/download.flutter.io. Got ' - '$mavenUrl instead'); + String realm = File( + path.join(flutterDirectory.path, 'bin', 'internal', 'engine.realm'), + ).readAsStringSync().trim(); + if (realm.isNotEmpty) { + realm = '$realm/'; + } + + if (mavenUrl != 'https://storage.googleapis.com/${realm}download.flutter.io') { + throw TaskResult.failure( + 'Expected Android engine maven dependency URL to ' + 'resolve to https://storage.googleapis.com/${realm}download.flutter.io. Got ' + '$mavenUrl instead', + ); } section('Checking overridden maven URL'); gradleOutput = await eval( - gradlewExecutable, - ['printEngineMavenUrl','-q'], - environment: { - 'FLUTTER_STORAGE_BASE_URL': 'https://my.special.proxy', - } - ); + gradlewExecutable, + ['printEngineMavenUrl','-q'], + environment: { + 'FLUTTER_STORAGE_BASE_URL': 'https://my.special.proxy', + }, + ); outputLines = splitter.convert(gradleOutput); mavenUrl = outputLines.last; - if (mavenUrl != 'https://my.special.proxy/download.flutter.io') { + if (mavenUrl != 'https://my.special.proxy/${realm}download.flutter.io') { throw TaskResult.failure( - 'Expected overridden Android engine maven ' - 'dependency URL to resolve to proxy location ' - 'https://my.special.proxy/download.flutter.io. Got ' - '$mavenUrl instead'); + 'Expected overridden Android engine maven ' + 'dependency URL to resolve to proxy location ' + 'https://my.special.proxy/${realm}download.flutter.io. Got ' + '$mavenUrl instead', + ); } }); }); diff --git a/dev/tools/java_and_objc_doc.dart b/dev/tools/java_and_objc_doc.dart index af8b745baad..260cdc7b609 100644 --- a/dev/tools/java_and_objc_doc.dart +++ b/dev/tools/java_and_objc_doc.dart @@ -15,11 +15,15 @@ const String kDocRoot = 'dev/docs/doc'; /// the artifact store and extracts them to the location used for Dartdoc. Future main(List args) async { final String engineVersion = File('bin/internal/engine.version').readAsStringSync().trim(); + String engineRealm = File('bin/internal/engine.realm').readAsStringSync().trim(); + if (engineRealm.isNotEmpty) { + engineRealm = '$engineRealm/'; + } - final String javadocUrl = 'https://storage.googleapis.com/flutter_infra_release/flutter/$engineVersion/android-javadoc.zip'; + final String javadocUrl = 'https://storage.googleapis.com/${engineRealm}flutter_infra_release/flutter/$engineVersion/android-javadoc.zip'; generateDocs(javadocUrl, 'javadoc', 'io/flutter/view/FlutterView.html'); - final String objcdocUrl = 'https://storage.googleapis.com/flutter_infra_release/flutter/$engineVersion/ios-objcdoc.zip'; + final String objcdocUrl = 'https://storage.googleapis.com/${engineRealm}flutter_infra_release/flutter/$engineVersion/ios-objcdoc.zip'; generateDocs(objcdocUrl, 'objcdoc', 'Classes/FlutterViewController.html'); } diff --git a/packages/flutter_tools/gradle/aar_init_script.gradle b/packages/flutter_tools/gradle/aar_init_script.gradle index 07f8f552993..e6fa84cc17f 100644 --- a/packages/flutter_tools/gradle/aar_init_script.gradle +++ b/packages/flutter_tools/gradle/aar_init_script.gradle @@ -45,11 +45,18 @@ void configureProject(Project project, String outputDir) { } String storageUrl = System.getenv('FLUTTER_STORAGE_BASE_URL') ?: "https://storage.googleapis.com" + + String engineRealm = Paths.get(getFlutterRoot(project), "bin", "internal", "engine.realm") + .toFile().text.trim() + if (engineRealm) { + engineRealm = engineRealm + "/" + } + // This is a Flutter plugin project. Plugin projects don't apply the Flutter Gradle plugin, // as a result, add the dependency on the embedding. project.repositories { maven { - url "$storageUrl/download.flutter.io" + url "$storageUrl/${engineRealm}download.flutter.io" } } String engineVersion = Paths.get(getFlutterRoot(project), "bin", "internal", "engine.version") diff --git a/packages/flutter_tools/gradle/resolve_dependencies.gradle b/packages/flutter_tools/gradle/resolve_dependencies.gradle index 817b7306131..6355654f2ed 100644 --- a/packages/flutter_tools/gradle/resolve_dependencies.gradle +++ b/packages/flutter_tools/gradle/resolve_dependencies.gradle @@ -16,11 +16,17 @@ import java.nio.file.Paths String storageUrl = System.getenv('FLUTTER_STORAGE_BASE_URL') ?: "https://storage.googleapis.com" +String engineRealm = Paths.get(flutterRoot.absolutePath, "bin", "internal", "engine.realm") + .toFile().text.trim() +if (engineRealm) { + engineRealm = engineRealm + "/" +} + repositories { google() mavenCentral() maven { - url "$storageUrl/download.flutter.io" + url "$storageUrl/${engineRealm}download.flutter.io" } } diff --git a/packages/flutter_tools/gradle/src/main/groovy/flutter.groovy b/packages/flutter_tools/gradle/src/main/groovy/flutter.groovy index c0102b4ff21..95a750e6ef9 100644 --- a/packages/flutter_tools/gradle/src/main/groovy/flutter.groovy +++ b/packages/flutter_tools/gradle/src/main/groovy/flutter.groovy @@ -167,6 +167,7 @@ class FlutterPlugin implements Plugin { private String localEngineSrcPath private Properties localProperties private String engineVersion + private String engineRealm /** * Flutter Docs Website URLs for help messages. @@ -192,11 +193,29 @@ class FlutterPlugin implements Plugin { } } + String flutterRootPath = resolveProperty("flutter.sdk", System.env.FLUTTER_ROOT) + if (flutterRootPath == null) { + throw new GradleException("Flutter SDK not found. Define location with flutter.sdk in the local.properties file or with a FLUTTER_ROOT environment variable.") + } + flutterRoot = project.file(flutterRootPath) + if (!flutterRoot.isDirectory()) { + throw new GradleException("flutter.sdk must point to the Flutter SDK directory") + } + + engineVersion = useLocalEngine() + ? "+" // Match any version since there's only one. + : "1.0.0-" + Paths.get(flutterRoot.absolutePath, "bin", "internal", "engine.version").toFile().text.trim() + + engineRealm = Paths.get(flutterRoot.absolutePath, "bin", "internal", "engine.realm").toFile().text.trim() + if (engineRealm) { + engineRealm = engineRealm + "/" + } + // Configure the Maven repository. String hostedRepository = System.env.FLUTTER_STORAGE_BASE_URL ?: DEFAULT_MAVEN_HOST String repository = useLocalEngine() ? project.property('local-engine-repo') - : "$hostedRepository/download.flutter.io" + : "$hostedRepository/${engineRealm}download.flutter.io" rootProject.allprojects { repositories { maven { @@ -246,19 +265,6 @@ class FlutterPlugin implements Plugin { } } - String flutterRootPath = resolveProperty("flutter.sdk", System.env.FLUTTER_ROOT) - if (flutterRootPath == null) { - throw new GradleException("Flutter SDK not found. Define location with flutter.sdk in the local.properties file or with a FLUTTER_ROOT environment variable.") - } - flutterRoot = project.file(flutterRootPath) - if (!flutterRoot.isDirectory()) { - throw new GradleException("flutter.sdk must point to the Flutter SDK directory") - } - - engineVersion = useLocalEngine() - ? "+" // Match any version since there's only one. - : "1.0.0-" + Paths.get(flutterRoot.absolutePath, "bin", "internal", "engine.version").toFile().text.trim() - String flutterExecutableName = Os.isFamily(Os.FAMILY_WINDOWS) ? "flutter.bat" : "flutter" flutterExecutable = Paths.get(flutterRoot.absolutePath, "bin", flutterExecutableName).toFile(); diff --git a/packages/flutter_tools/lib/src/cache.dart b/packages/flutter_tools/lib/src/cache.dart index 13efbde879d..7cfa8ce22fe 100644 --- a/packages/flutter_tools/lib/src/cache.dart +++ b/packages/flutter_tools/lib/src/cache.dart @@ -190,6 +190,7 @@ class Cache { httpClient: HttpClient(), allowedBaseUrls: [ storageBaseUrl, + realmlessStorageBaseUrl, cipdBaseUrl, ], ); @@ -447,6 +448,22 @@ class Cache { } String? _engineRevision; + /// The "realm" for the storage URL. + /// + /// For production artifacts from Engine post-submit and release builds, + /// this string will be empty, and the `storageBaseUrl` will be unmodified. + /// When non-empty, this string will be appended to the `storageBaseUrl` after + /// a '/'. For artifacts generated by Engine presubmits, the realm should be + /// "flutter_archives_v2". + String get storageRealm { + _storageRealm ??= getRealmFor('engine'); + if (_storageRealm == null) { + throwToolExit('Could not determine engine realm.'); + } + return _storageRealm!; + } + String? _storageRealm; + /// The base for URLs that store Flutter engine artifacts that are fetched /// during the installation of the Flutter SDK. /// @@ -459,11 +476,14 @@ class Cache { /// * [cipdBaseUrl], which determines how CIPD artifacts are fetched. /// * [Cache] class-level dartdocs that explain how artifact mirrors work. String get storageBaseUrl { - final String? overrideUrl = _platform.environment[kFlutterStorageBaseUrl]; + String? overrideUrl = _platform.environment[kFlutterStorageBaseUrl]; if (overrideUrl == null) { - return 'https://storage.googleapis.com'; + return storageRealm.isEmpty + ? 'https://storage.googleapis.com' + : 'https://storage.googleapis.com/$storageRealm'; } // verify that this is a valid URI. + overrideUrl = storageRealm.isEmpty ? overrideUrl : '$overrideUrl/$storageRealm'; try { Uri.parse(overrideUrl); } on FormatException catch (err) { @@ -473,6 +493,12 @@ class Cache { return overrideUrl; } + String get realmlessStorageBaseUrl { + return storageRealm.isEmpty + ? storageBaseUrl + : storageBaseUrl.replaceAll('/$storageRealm', ''); + } + /// The base for URLs that store Flutter engine artifacts in CIPD. /// /// For some platforms, such as Web and Fuchsia, CIPD artifacts are fetched @@ -607,6 +633,16 @@ class Cache { return versionFile.existsSync() ? versionFile.readAsStringSync().trim() : null; } + String? getRealmFor(String artifactName) { + final File realmFile = _fileSystem.file(_fileSystem.path.join( + _rootOverride?.path ?? flutterRoot!, + 'bin', + 'internal', + '$artifactName.realm', + )); + return realmFile.existsSync() ? realmFile.readAsStringSync().trim() : ''; + } + /// Delete all stamp files maintained by the cache. void clearStampFiles() { try { diff --git a/packages/flutter_tools/lib/src/flutter_cache.dart b/packages/flutter_tools/lib/src/flutter_cache.dart index 252021cf78e..dcba2c6da54 100644 --- a/packages/flutter_tools/lib/src/flutter_cache.dart +++ b/packages/flutter_tools/lib/src/flutter_cache.dart @@ -836,7 +836,11 @@ class IosUsbArtifacts extends CachedArtifact { } @visibleForTesting - Uri get archiveUri => Uri.parse('${cache.storageBaseUrl}/flutter_infra_release/ios-usb-dependencies${cache.useUnsignedMacBinaries ? '/unsigned' : ''}/$name/$version/$name.zip'); + Uri get archiveUri => Uri.parse( + '${cache.realmlessStorageBaseUrl}/flutter_infra_release/' + 'ios-usb-dependencies${cache.useUnsignedMacBinaries ? '/unsigned' : ''}' + '/$name/$version/$name.zip', + ); } // TODO(zanderso): upload debug desktop artifacts to host-debug and diff --git a/packages/flutter_tools/test/general.shard/cache_test.dart b/packages/flutter_tools/test/general.shard/cache_test.dart index da9f6e10c8a..f5b7b6a6454 100644 --- a/packages/flutter_tools/test/general.shard/cache_test.dart +++ b/packages/flutter_tools/test/general.shard/cache_test.dart @@ -334,6 +334,37 @@ void main() { expect(logger.warningText, contains('Flutter assets will be downloaded from $baseUrl')); expect(logger.statusText, isEmpty); }); + + testWithoutContext('a non-empty realm is included in the storage url', () async { + final MemoryFileSystem fileSystem = MemoryFileSystem.test(); + final Directory internalDir = fileSystem.currentDirectory + .childDirectory('cache') + .childDirectory('bin') + .childDirectory('internal'); + final File engineVersionFile = internalDir.childFile('engine.version'); + engineVersionFile.createSync(recursive: true); + engineVersionFile.writeAsStringSync('abcdef'); + + final File engineRealmFile = internalDir.childFile('engine.realm'); + engineRealmFile.createSync(recursive: true); + engineRealmFile.writeAsStringSync('flutter_archives_v2'); + + final Cache cache = Cache.test( + processManager: FakeProcessManager.any(), + fileSystem: fileSystem, + ); + + expect(cache.storageBaseUrl, contains('flutter_archives_v2')); + }); + + test('bin/internal/engine.realm is empty', () async { + final FileSystem fileSystem = globals.fs; + final String realmFilePath = fileSystem.path.join( + getFlutterRoot(), 'bin', 'internal', 'engine.realm'); + final String realm = fileSystem.file(realmFilePath).readAsStringSync().trim(); + expect(realm, isEmpty, + reason: 'The checked-in engine.realm file must be empty.'); + }); }); testWithoutContext('flattenNameSubdirs', () {