mirror of
https://github.com/flutter/flutter
synced 2024-10-02 14:34:22 +00:00
Allows adding a storage 'realm' to the storage base URL (#131951)
Context: https://github.com/flutter/flutter/issues/131862 This PR injects a "realm" component to the storage base URL when the contents of the file `bin/internal/engine.realm` is non-empty. As documented in the PR, when the realm is `flutter_archives_v2`, and `bin/internal/engine.version` contains the commit hash for a commit in a `flutter/engine` PR, then the artifacts pulled by the tool will be the artifacts built by the presubmit checks for the PR. This works for everything but the following two cases: 1. Fuchsia artifacts are not uploaded to CIPD by the Fuchsia presubmit builds. 2. Web artifacts are not uploaded to gstatic by the web engine presubmit builds. For (1), the flutter/flutter presubmit `fuchsia_precache` is driven by a shell script outside of the repo. It will fail when the `engine.version` and `engine.realm` don't point to a post-submit engine commit. For (2), the flutter/flutter web presubmit tests that refer to artifacts in gstatic hang when the artifacts aren't found, so this PR skips them.
This commit is contained in:
parent
9c8f3950e3
commit
118c2df776
0
bin/internal/engine.realm
Normal file
0
bin/internal/engine.realm
Normal file
|
@ -20,6 +20,7 @@ $cachePath = "$flutterRoot\bin\cache"
|
|||
$dartSdkPath = "$cachePath\dart-sdk"
|
||||
$engineStamp = "$cachePath\engine-dart-sdk.stamp"
|
||||
$engineVersion = (Get-Content "$flutterRoot\bin\internal\engine.version")
|
||||
$engineRealm = (Get-Content "$flutterRoot\bin\internal\engine.realm")
|
||||
|
||||
$oldDartSdkPrefix = "dart-sdk.old"
|
||||
|
||||
|
@ -42,6 +43,9 @@ $dartSdkBaseUrl = $Env:FLUTTER_STORAGE_BASE_URL
|
|||
if (-not $dartSdkBaseUrl) {
|
||||
$dartSdkBaseUrl = "https://storage.googleapis.com"
|
||||
}
|
||||
if ($engineRealm) {
|
||||
$dartSdkBaseUrl = "$dartSdkBaseUrl/$engineRealm"
|
||||
}
|
||||
$dartZipName = "dart-sdk-windows-x64.zip"
|
||||
$dartSdkUrl = "$dartSdkBaseUrl/flutter_infra_release/flutter/$engineVersion/$dartZipName"
|
||||
|
||||
|
|
|
@ -20,6 +20,7 @@ DART_SDK_PATH="$FLUTTER_ROOT/bin/cache/dart-sdk"
|
|||
DART_SDK_PATH_OLD="$DART_SDK_PATH.old"
|
||||
ENGINE_STAMP="$FLUTTER_ROOT/bin/cache/engine-dart-sdk.stamp"
|
||||
ENGINE_VERSION=`cat "$FLUTTER_ROOT/bin/internal/engine.version"`
|
||||
ENGINE_REALM=`cat "$FLUTTER_ROOT/bin/internal/engine.realm"`
|
||||
OS="$(uname -s)"
|
||||
|
||||
if [ ! -f "$ENGINE_STAMP" ] || [ "$ENGINE_VERSION" != `cat "$ENGINE_STAMP"` ]; then
|
||||
|
@ -121,7 +122,7 @@ if [ ! -f "$ENGINE_STAMP" ] || [ "$ENGINE_VERSION" != `cat "$ENGINE_STAMP"` ]; t
|
|||
FIND=find
|
||||
fi
|
||||
|
||||
DART_SDK_BASE_URL="${FLUTTER_STORAGE_BASE_URL:-https://storage.googleapis.com}"
|
||||
DART_SDK_BASE_URL="${FLUTTER_STORAGE_BASE_URL:-https://storage.googleapis.com}${ENGINE_REALM:+/$ENGINE_REALM}"
|
||||
DART_SDK_URL="$DART_SDK_BASE_URL/flutter_infra_release/flutter/$ENGINE_VERSION/$DART_ZIP_NAME"
|
||||
|
||||
# if the sdk path exists, copy it to a temporary location
|
||||
|
|
|
@ -83,6 +83,7 @@ final String flutter = path.join(flutterRoot, 'bin', 'flutter$bat');
|
|||
final String dart = path.join(flutterRoot, 'bin', 'cache', 'dart-sdk', 'bin', 'dart$exe');
|
||||
final String pubCache = path.join(flutterRoot, '.pub-cache');
|
||||
final String engineVersionFile = path.join(flutterRoot, 'bin', 'internal', 'engine.version');
|
||||
final String engineRealmFile = path.join(flutterRoot, 'bin', 'internal', 'engine.realm');
|
||||
final String flutterPackagesVersionFile = path.join(flutterRoot, 'bin', 'internal', 'flutter_packages.version');
|
||||
|
||||
String get platformFolderName {
|
||||
|
@ -1138,6 +1139,10 @@ Future<void> _runWebUnitTests(String webRenderer) async {
|
|||
/// Coarse-grained integration tests running on the Web.
|
||||
Future<void> _runWebLongRunningTests() async {
|
||||
final String engineVersion = File(engineVersionFile).readAsStringSync().trim();
|
||||
final String engineRealm = File(engineRealmFile).readAsStringSync().trim();
|
||||
if (engineRealm.isNotEmpty) {
|
||||
return;
|
||||
}
|
||||
final List<ShardRunner> tests = <ShardRunner>[
|
||||
for (final String buildMode in _kAllBuildModes) ...<ShardRunner>[
|
||||
() => _runFlutterDriverWebTest(
|
||||
|
|
|
@ -125,7 +125,15 @@ class CodesignCommand extends Command<void> {
|
|||
await framework.checkout(revision);
|
||||
|
||||
// Ensure artifacts present
|
||||
await framework.runFlutter(<String>['precache', '--android', '--ios', '--macos']);
|
||||
final io.ProcessResult result = await framework.runFlutter(
|
||||
<String>['precache', '--android', '--ios', '--macos'],
|
||||
);
|
||||
if (result.exitCode != 0) {
|
||||
stdio.printError(
|
||||
'flutter precache: exitCode: ${result.exitCode}\n'
|
||||
'stdout:\n${result.stdout}\nstderr:\n${result.stderr}',
|
||||
);
|
||||
}
|
||||
|
||||
await verifyExist();
|
||||
if (argResults![kSignatures] as bool) {
|
||||
|
|
|
@ -30,7 +30,8 @@ Future<void> main() async {
|
|||
await inDirectory(path.join(flutterProject.rootPath, 'android'), () async {
|
||||
section('Insert gradle testing script');
|
||||
final File build = File(path.join(
|
||||
flutterProject.rootPath, 'android', 'app', 'build.gradle'));
|
||||
flutterProject.rootPath, 'android', 'app', 'build.gradle',
|
||||
));
|
||||
build.writeAsStringSync(
|
||||
'''
|
||||
task printEngineMavenUrl() {
|
||||
|
@ -44,6 +45,7 @@ task printEngineMavenUrl() {
|
|||
);
|
||||
|
||||
section('Checking default maven URL');
|
||||
|
||||
String gradleOutput = await eval(
|
||||
gradlewExecutable,
|
||||
<String>['printEngineMavenUrl', '-q'],
|
||||
|
@ -53,29 +55,39 @@ task printEngineMavenUrl() {
|
|||
String mavenUrl = outputLines.last;
|
||||
print('Returned maven url: $mavenUrl');
|
||||
|
||||
if (mavenUrl != 'https://storage.googleapis.com/download.flutter.io') {
|
||||
throw TaskResult.failure('Expected Android engine maven dependency URL to '
|
||||
'resolve to https://storage.googleapis.com/download.flutter.io. Got '
|
||||
'$mavenUrl instead');
|
||||
String realm = File(
|
||||
path.join(flutterDirectory.path, 'bin', 'internal', 'engine.realm'),
|
||||
).readAsStringSync().trim();
|
||||
if (realm.isNotEmpty) {
|
||||
realm = '$realm/';
|
||||
}
|
||||
|
||||
if (mavenUrl != 'https://storage.googleapis.com/${realm}download.flutter.io') {
|
||||
throw TaskResult.failure(
|
||||
'Expected Android engine maven dependency URL to '
|
||||
'resolve to https://storage.googleapis.com/${realm}download.flutter.io. Got '
|
||||
'$mavenUrl instead',
|
||||
);
|
||||
}
|
||||
|
||||
section('Checking overridden maven URL');
|
||||
gradleOutput = await eval(
|
||||
gradlewExecutable,
|
||||
<String>['printEngineMavenUrl','-q'],
|
||||
environment: <String, String>{
|
||||
'FLUTTER_STORAGE_BASE_URL': 'https://my.special.proxy',
|
||||
}
|
||||
);
|
||||
gradlewExecutable,
|
||||
<String>['printEngineMavenUrl','-q'],
|
||||
environment: <String, String>{
|
||||
'FLUTTER_STORAGE_BASE_URL': 'https://my.special.proxy',
|
||||
},
|
||||
);
|
||||
outputLines = splitter.convert(gradleOutput);
|
||||
mavenUrl = outputLines.last;
|
||||
|
||||
if (mavenUrl != 'https://my.special.proxy/download.flutter.io') {
|
||||
if (mavenUrl != 'https://my.special.proxy/${realm}download.flutter.io') {
|
||||
throw TaskResult.failure(
|
||||
'Expected overridden Android engine maven '
|
||||
'dependency URL to resolve to proxy location '
|
||||
'https://my.special.proxy/download.flutter.io. Got '
|
||||
'$mavenUrl instead');
|
||||
'Expected overridden Android engine maven '
|
||||
'dependency URL to resolve to proxy location '
|
||||
'https://my.special.proxy/${realm}download.flutter.io. Got '
|
||||
'$mavenUrl instead',
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
|
|
@ -15,11 +15,15 @@ const String kDocRoot = 'dev/docs/doc';
|
|||
/// the artifact store and extracts them to the location used for Dartdoc.
|
||||
Future<void> main(List<String> args) async {
|
||||
final String engineVersion = File('bin/internal/engine.version').readAsStringSync().trim();
|
||||
String engineRealm = File('bin/internal/engine.realm').readAsStringSync().trim();
|
||||
if (engineRealm.isNotEmpty) {
|
||||
engineRealm = '$engineRealm/';
|
||||
}
|
||||
|
||||
final String javadocUrl = 'https://storage.googleapis.com/flutter_infra_release/flutter/$engineVersion/android-javadoc.zip';
|
||||
final String javadocUrl = 'https://storage.googleapis.com/${engineRealm}flutter_infra_release/flutter/$engineVersion/android-javadoc.zip';
|
||||
generateDocs(javadocUrl, 'javadoc', 'io/flutter/view/FlutterView.html');
|
||||
|
||||
final String objcdocUrl = 'https://storage.googleapis.com/flutter_infra_release/flutter/$engineVersion/ios-objcdoc.zip';
|
||||
final String objcdocUrl = 'https://storage.googleapis.com/${engineRealm}flutter_infra_release/flutter/$engineVersion/ios-objcdoc.zip';
|
||||
generateDocs(objcdocUrl, 'objcdoc', 'Classes/FlutterViewController.html');
|
||||
}
|
||||
|
||||
|
|
|
@ -45,11 +45,18 @@ void configureProject(Project project, String outputDir) {
|
|||
}
|
||||
|
||||
String storageUrl = System.getenv('FLUTTER_STORAGE_BASE_URL') ?: "https://storage.googleapis.com"
|
||||
|
||||
String engineRealm = Paths.get(getFlutterRoot(project), "bin", "internal", "engine.realm")
|
||||
.toFile().text.trim()
|
||||
if (engineRealm) {
|
||||
engineRealm = engineRealm + "/"
|
||||
}
|
||||
|
||||
// This is a Flutter plugin project. Plugin projects don't apply the Flutter Gradle plugin,
|
||||
// as a result, add the dependency on the embedding.
|
||||
project.repositories {
|
||||
maven {
|
||||
url "$storageUrl/download.flutter.io"
|
||||
url "$storageUrl/${engineRealm}download.flutter.io"
|
||||
}
|
||||
}
|
||||
String engineVersion = Paths.get(getFlutterRoot(project), "bin", "internal", "engine.version")
|
||||
|
|
|
@ -16,11 +16,17 @@ import java.nio.file.Paths
|
|||
|
||||
String storageUrl = System.getenv('FLUTTER_STORAGE_BASE_URL') ?: "https://storage.googleapis.com"
|
||||
|
||||
String engineRealm = Paths.get(flutterRoot.absolutePath, "bin", "internal", "engine.realm")
|
||||
.toFile().text.trim()
|
||||
if (engineRealm) {
|
||||
engineRealm = engineRealm + "/"
|
||||
}
|
||||
|
||||
repositories {
|
||||
google()
|
||||
mavenCentral()
|
||||
maven {
|
||||
url "$storageUrl/download.flutter.io"
|
||||
url "$storageUrl/${engineRealm}download.flutter.io"
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -167,6 +167,7 @@ class FlutterPlugin implements Plugin<Project> {
|
|||
private String localEngineSrcPath
|
||||
private Properties localProperties
|
||||
private String engineVersion
|
||||
private String engineRealm
|
||||
|
||||
/**
|
||||
* Flutter Docs Website URLs for help messages.
|
||||
|
@ -192,11 +193,29 @@ class FlutterPlugin implements Plugin<Project> {
|
|||
}
|
||||
}
|
||||
|
||||
String flutterRootPath = resolveProperty("flutter.sdk", System.env.FLUTTER_ROOT)
|
||||
if (flutterRootPath == null) {
|
||||
throw new GradleException("Flutter SDK not found. Define location with flutter.sdk in the local.properties file or with a FLUTTER_ROOT environment variable.")
|
||||
}
|
||||
flutterRoot = project.file(flutterRootPath)
|
||||
if (!flutterRoot.isDirectory()) {
|
||||
throw new GradleException("flutter.sdk must point to the Flutter SDK directory")
|
||||
}
|
||||
|
||||
engineVersion = useLocalEngine()
|
||||
? "+" // Match any version since there's only one.
|
||||
: "1.0.0-" + Paths.get(flutterRoot.absolutePath, "bin", "internal", "engine.version").toFile().text.trim()
|
||||
|
||||
engineRealm = Paths.get(flutterRoot.absolutePath, "bin", "internal", "engine.realm").toFile().text.trim()
|
||||
if (engineRealm) {
|
||||
engineRealm = engineRealm + "/"
|
||||
}
|
||||
|
||||
// Configure the Maven repository.
|
||||
String hostedRepository = System.env.FLUTTER_STORAGE_BASE_URL ?: DEFAULT_MAVEN_HOST
|
||||
String repository = useLocalEngine()
|
||||
? project.property('local-engine-repo')
|
||||
: "$hostedRepository/download.flutter.io"
|
||||
: "$hostedRepository/${engineRealm}download.flutter.io"
|
||||
rootProject.allprojects {
|
||||
repositories {
|
||||
maven {
|
||||
|
@ -246,19 +265,6 @@ class FlutterPlugin implements Plugin<Project> {
|
|||
}
|
||||
}
|
||||
|
||||
String flutterRootPath = resolveProperty("flutter.sdk", System.env.FLUTTER_ROOT)
|
||||
if (flutterRootPath == null) {
|
||||
throw new GradleException("Flutter SDK not found. Define location with flutter.sdk in the local.properties file or with a FLUTTER_ROOT environment variable.")
|
||||
}
|
||||
flutterRoot = project.file(flutterRootPath)
|
||||
if (!flutterRoot.isDirectory()) {
|
||||
throw new GradleException("flutter.sdk must point to the Flutter SDK directory")
|
||||
}
|
||||
|
||||
engineVersion = useLocalEngine()
|
||||
? "+" // Match any version since there's only one.
|
||||
: "1.0.0-" + Paths.get(flutterRoot.absolutePath, "bin", "internal", "engine.version").toFile().text.trim()
|
||||
|
||||
String flutterExecutableName = Os.isFamily(Os.FAMILY_WINDOWS) ? "flutter.bat" : "flutter"
|
||||
flutterExecutable = Paths.get(flutterRoot.absolutePath, "bin", flutterExecutableName).toFile();
|
||||
|
||||
|
|
|
@ -190,6 +190,7 @@ class Cache {
|
|||
httpClient: HttpClient(),
|
||||
allowedBaseUrls: <String>[
|
||||
storageBaseUrl,
|
||||
realmlessStorageBaseUrl,
|
||||
cipdBaseUrl,
|
||||
],
|
||||
);
|
||||
|
@ -447,6 +448,22 @@ class Cache {
|
|||
}
|
||||
String? _engineRevision;
|
||||
|
||||
/// The "realm" for the storage URL.
|
||||
///
|
||||
/// For production artifacts from Engine post-submit and release builds,
|
||||
/// this string will be empty, and the `storageBaseUrl` will be unmodified.
|
||||
/// When non-empty, this string will be appended to the `storageBaseUrl` after
|
||||
/// a '/'. For artifacts generated by Engine presubmits, the realm should be
|
||||
/// "flutter_archives_v2".
|
||||
String get storageRealm {
|
||||
_storageRealm ??= getRealmFor('engine');
|
||||
if (_storageRealm == null) {
|
||||
throwToolExit('Could not determine engine realm.');
|
||||
}
|
||||
return _storageRealm!;
|
||||
}
|
||||
String? _storageRealm;
|
||||
|
||||
/// The base for URLs that store Flutter engine artifacts that are fetched
|
||||
/// during the installation of the Flutter SDK.
|
||||
///
|
||||
|
@ -459,11 +476,14 @@ class Cache {
|
|||
/// * [cipdBaseUrl], which determines how CIPD artifacts are fetched.
|
||||
/// * [Cache] class-level dartdocs that explain how artifact mirrors work.
|
||||
String get storageBaseUrl {
|
||||
final String? overrideUrl = _platform.environment[kFlutterStorageBaseUrl];
|
||||
String? overrideUrl = _platform.environment[kFlutterStorageBaseUrl];
|
||||
if (overrideUrl == null) {
|
||||
return 'https://storage.googleapis.com';
|
||||
return storageRealm.isEmpty
|
||||
? 'https://storage.googleapis.com'
|
||||
: 'https://storage.googleapis.com/$storageRealm';
|
||||
}
|
||||
// verify that this is a valid URI.
|
||||
overrideUrl = storageRealm.isEmpty ? overrideUrl : '$overrideUrl/$storageRealm';
|
||||
try {
|
||||
Uri.parse(overrideUrl);
|
||||
} on FormatException catch (err) {
|
||||
|
@ -473,6 +493,12 @@ class Cache {
|
|||
return overrideUrl;
|
||||
}
|
||||
|
||||
String get realmlessStorageBaseUrl {
|
||||
return storageRealm.isEmpty
|
||||
? storageBaseUrl
|
||||
: storageBaseUrl.replaceAll('/$storageRealm', '');
|
||||
}
|
||||
|
||||
/// The base for URLs that store Flutter engine artifacts in CIPD.
|
||||
///
|
||||
/// For some platforms, such as Web and Fuchsia, CIPD artifacts are fetched
|
||||
|
@ -607,6 +633,16 @@ class Cache {
|
|||
return versionFile.existsSync() ? versionFile.readAsStringSync().trim() : null;
|
||||
}
|
||||
|
||||
String? getRealmFor(String artifactName) {
|
||||
final File realmFile = _fileSystem.file(_fileSystem.path.join(
|
||||
_rootOverride?.path ?? flutterRoot!,
|
||||
'bin',
|
||||
'internal',
|
||||
'$artifactName.realm',
|
||||
));
|
||||
return realmFile.existsSync() ? realmFile.readAsStringSync().trim() : '';
|
||||
}
|
||||
|
||||
/// Delete all stamp files maintained by the cache.
|
||||
void clearStampFiles() {
|
||||
try {
|
||||
|
|
|
@ -836,7 +836,11 @@ class IosUsbArtifacts extends CachedArtifact {
|
|||
}
|
||||
|
||||
@visibleForTesting
|
||||
Uri get archiveUri => Uri.parse('${cache.storageBaseUrl}/flutter_infra_release/ios-usb-dependencies${cache.useUnsignedMacBinaries ? '/unsigned' : ''}/$name/$version/$name.zip');
|
||||
Uri get archiveUri => Uri.parse(
|
||||
'${cache.realmlessStorageBaseUrl}/flutter_infra_release/'
|
||||
'ios-usb-dependencies${cache.useUnsignedMacBinaries ? '/unsigned' : ''}'
|
||||
'/$name/$version/$name.zip',
|
||||
);
|
||||
}
|
||||
|
||||
// TODO(zanderso): upload debug desktop artifacts to host-debug and
|
||||
|
|
|
@ -334,6 +334,37 @@ void main() {
|
|||
expect(logger.warningText, contains('Flutter assets will be downloaded from $baseUrl'));
|
||||
expect(logger.statusText, isEmpty);
|
||||
});
|
||||
|
||||
testWithoutContext('a non-empty realm is included in the storage url', () async {
|
||||
final MemoryFileSystem fileSystem = MemoryFileSystem.test();
|
||||
final Directory internalDir = fileSystem.currentDirectory
|
||||
.childDirectory('cache')
|
||||
.childDirectory('bin')
|
||||
.childDirectory('internal');
|
||||
final File engineVersionFile = internalDir.childFile('engine.version');
|
||||
engineVersionFile.createSync(recursive: true);
|
||||
engineVersionFile.writeAsStringSync('abcdef');
|
||||
|
||||
final File engineRealmFile = internalDir.childFile('engine.realm');
|
||||
engineRealmFile.createSync(recursive: true);
|
||||
engineRealmFile.writeAsStringSync('flutter_archives_v2');
|
||||
|
||||
final Cache cache = Cache.test(
|
||||
processManager: FakeProcessManager.any(),
|
||||
fileSystem: fileSystem,
|
||||
);
|
||||
|
||||
expect(cache.storageBaseUrl, contains('flutter_archives_v2'));
|
||||
});
|
||||
|
||||
test('bin/internal/engine.realm is empty', () async {
|
||||
final FileSystem fileSystem = globals.fs;
|
||||
final String realmFilePath = fileSystem.path.join(
|
||||
getFlutterRoot(), 'bin', 'internal', 'engine.realm');
|
||||
final String realm = fileSystem.file(realmFilePath).readAsStringSync().trim();
|
||||
expect(realm, isEmpty,
|
||||
reason: 'The checked-in engine.realm file must be empty.');
|
||||
});
|
||||
});
|
||||
|
||||
testWithoutContext('flattenNameSubdirs', () {
|
||||
|
|
Loading…
Reference in a new issue