mirror of
https://github.com/flutter/flutter
synced 2024-08-27 20:10:11 +00:00
526 lines
19 KiB
Dart
526 lines
19 KiB
Dart
// Copyright 2014 The Flutter Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
// found in the LICENSE file.
|
|
|
|
|
|
/// This script removes published archives from the cloud storage and the
|
|
/// corresponding JSON metadata file that the website uses to determine what
|
|
/// releases are available.
|
|
///
|
|
/// If asked to remove a release that is currently the release on that channel,
|
|
/// it will replace that release with the next most recent release on that
|
|
/// channel.
|
|
|
|
import 'dart:async';
|
|
import 'dart:convert';
|
|
import 'dart:io' hide Platform;
|
|
|
|
import 'package:args/args.dart';
|
|
import 'package:path/path.dart' as path;
|
|
import 'package:platform/platform.dart' show LocalPlatform, Platform;
|
|
import 'package:process/process.dart';
|
|
|
|
const String gsBase = 'gs://flutter_infra_release';
|
|
const String releaseFolder = '/releases';
|
|
const String gsReleaseFolder = '$gsBase$releaseFolder';
|
|
const String baseUrl = 'https://storage.googleapis.com/flutter_infra_release';
|
|
|
|
/// Exception class for when a process fails to run, so we can catch
|
|
/// it and provide something more readable than a stack trace.
|
|
class UnpublishException implements Exception {
|
|
UnpublishException(this.message, [this.result]);
|
|
|
|
final String message;
|
|
final ProcessResult? result;
|
|
int get exitCode => result?.exitCode ?? -1;
|
|
|
|
@override
|
|
String toString() {
|
|
String output = runtimeType.toString();
|
|
if (message != null) {
|
|
output += ': $message';
|
|
}
|
|
final String stderr = result?.stderr as String? ?? '';
|
|
if (stderr.isNotEmpty) {
|
|
output += ':\n$stderr';
|
|
}
|
|
return output;
|
|
}
|
|
}
|
|
|
|
enum Channel { dev, beta, stable }
|
|
|
|
String getChannelName(Channel channel) {
|
|
switch (channel) {
|
|
case Channel.beta:
|
|
return 'beta';
|
|
case Channel.dev:
|
|
return 'dev';
|
|
case Channel.stable:
|
|
return 'stable';
|
|
}
|
|
}
|
|
|
|
Channel fromChannelName(String? name) {
|
|
switch (name) {
|
|
case 'beta':
|
|
return Channel.beta;
|
|
case 'dev':
|
|
return Channel.dev;
|
|
case 'stable':
|
|
return Channel.stable;
|
|
default:
|
|
throw ArgumentError('Invalid channel name.');
|
|
}
|
|
}
|
|
|
|
enum PublishedPlatform { linux, macos, windows }
|
|
|
|
String getPublishedPlatform(PublishedPlatform platform) {
|
|
switch (platform) {
|
|
case PublishedPlatform.linux:
|
|
return 'linux';
|
|
case PublishedPlatform.macos:
|
|
return 'macos';
|
|
case PublishedPlatform.windows:
|
|
return 'windows';
|
|
}
|
|
}
|
|
|
|
PublishedPlatform fromPublishedPlatform(String name) {
|
|
switch (name) {
|
|
case 'linux':
|
|
return PublishedPlatform.linux;
|
|
case 'macos':
|
|
return PublishedPlatform.macos;
|
|
case 'windows':
|
|
return PublishedPlatform.windows;
|
|
default:
|
|
throw ArgumentError('Invalid published platform name.');
|
|
}
|
|
}
|
|
|
|
/// A helper class for classes that want to run a process, optionally have the
|
|
/// stderr and stdout reported as the process runs, and capture the stdout
|
|
/// properly without dropping any.
|
|
class ProcessRunner {
|
|
/// Creates a [ProcessRunner].
|
|
///
|
|
/// The [processManager], [subprocessOutput], and [platform] arguments must
|
|
/// not be null.
|
|
ProcessRunner({
|
|
this.processManager = const LocalProcessManager(),
|
|
this.subprocessOutput = true,
|
|
this.defaultWorkingDirectory,
|
|
this.platform = const LocalPlatform(),
|
|
}) : assert(subprocessOutput != null),
|
|
assert(processManager != null),
|
|
assert(platform != null) {
|
|
environment = Map<String, String>.from(platform.environment);
|
|
}
|
|
|
|
/// The platform to use for a starting environment.
|
|
final Platform platform;
|
|
|
|
/// Set [subprocessOutput] to show output as processes run. Stdout from the
|
|
/// process will be printed to stdout, and stderr printed to stderr.
|
|
final bool subprocessOutput;
|
|
|
|
/// Set the [processManager] in order to inject a test instance to perform
|
|
/// testing.
|
|
final ProcessManager processManager;
|
|
|
|
/// Sets the default directory used when `workingDirectory` is not specified
|
|
/// to [runProcess].
|
|
final Directory? defaultWorkingDirectory;
|
|
|
|
/// The environment to run processes with.
|
|
late Map<String, String> environment;
|
|
|
|
/// Run the command and arguments in `commandLine` as a sub-process from
|
|
/// `workingDirectory` if set, or the [defaultWorkingDirectory] if not. Uses
|
|
/// [Directory.current] if [defaultWorkingDirectory] is not set.
|
|
///
|
|
/// Set `failOk` if [runProcess] should not throw an exception when the
|
|
/// command completes with a non-zero exit code.
|
|
Future<String> runProcess(
|
|
List<String> commandLine, {
|
|
Directory? workingDirectory,
|
|
bool failOk = false,
|
|
}) async {
|
|
workingDirectory ??= defaultWorkingDirectory ?? Directory.current;
|
|
if (subprocessOutput) {
|
|
stderr.write('Running "${commandLine.join(' ')}" in ${workingDirectory.path}.\n');
|
|
}
|
|
final List<int> output = <int>[];
|
|
final Completer<void> stdoutComplete = Completer<void>();
|
|
final Completer<void> stderrComplete = Completer<void>();
|
|
late Process process;
|
|
Future<int> allComplete() async {
|
|
await stderrComplete.future;
|
|
await stdoutComplete.future;
|
|
return process.exitCode;
|
|
}
|
|
|
|
try {
|
|
process = await processManager.start(
|
|
commandLine,
|
|
workingDirectory: workingDirectory.absolute.path,
|
|
environment: environment,
|
|
);
|
|
process.stdout.listen(
|
|
(List<int> event) {
|
|
output.addAll(event);
|
|
if (subprocessOutput) {
|
|
stdout.add(event);
|
|
}
|
|
},
|
|
onDone: () async => stdoutComplete.complete(),
|
|
);
|
|
if (subprocessOutput) {
|
|
process.stderr.listen(
|
|
(List<int> event) {
|
|
stderr.add(event);
|
|
},
|
|
onDone: () async => stderrComplete.complete(),
|
|
);
|
|
} else {
|
|
stderrComplete.complete();
|
|
}
|
|
} on ProcessException catch (e) {
|
|
final String message = 'Running "${commandLine.join(' ')}" in ${workingDirectory.path} '
|
|
'failed with:\n$e';
|
|
throw UnpublishException(message);
|
|
} on ArgumentError catch (e) {
|
|
final String message = 'Running "${commandLine.join(' ')}" in ${workingDirectory.path} '
|
|
'failed with:\n$e';
|
|
throw UnpublishException(message);
|
|
}
|
|
|
|
final int exitCode = await allComplete();
|
|
if (exitCode != 0 && !failOk) {
|
|
final String message = 'Running "${commandLine.join(' ')}" in ${workingDirectory.path} failed';
|
|
throw UnpublishException(
|
|
message,
|
|
ProcessResult(0, exitCode, null, 'returned $exitCode'),
|
|
);
|
|
}
|
|
return utf8.decoder.convert(output).trim();
|
|
}
|
|
}
|
|
|
|
class ArchiveUnpublisher {
|
|
ArchiveUnpublisher(
|
|
this.tempDir,
|
|
this.revisionsBeingRemoved,
|
|
this.channels,
|
|
this.platform, {
|
|
this.confirmed = false,
|
|
ProcessManager? processManager,
|
|
bool subprocessOutput = true,
|
|
}) : assert(revisionsBeingRemoved.length == 40),
|
|
metadataGsPath = '$gsReleaseFolder/${getMetadataFilename(platform)}',
|
|
_processRunner = ProcessRunner(
|
|
processManager: processManager ?? const LocalProcessManager(),
|
|
subprocessOutput: subprocessOutput,
|
|
);
|
|
|
|
final PublishedPlatform platform;
|
|
final String metadataGsPath;
|
|
final Set<Channel> channels;
|
|
final Set<String> revisionsBeingRemoved;
|
|
final bool confirmed;
|
|
final Directory tempDir;
|
|
final ProcessRunner _processRunner;
|
|
static String getMetadataFilename(PublishedPlatform platform) => 'releases_${getPublishedPlatform(platform)}.json';
|
|
|
|
/// Remove the archive from Google Storage.
|
|
Future<void> unpublishArchive() async {
|
|
final Map<String, dynamic> jsonData = await _loadMetadata();
|
|
final List<Map<String, String>> releases = (jsonData['releases'] as List<dynamic>).map<Map<String, String>>((dynamic entry) {
|
|
final Map<String, dynamic> mapEntry = entry as Map<String, dynamic>;
|
|
return mapEntry.cast<String, String>();
|
|
}).toList();
|
|
final Map<Channel, Map<String, String>> paths = await _getArchivePaths(releases);
|
|
releases.removeWhere((Map<String, String> value) => revisionsBeingRemoved.contains(value['hash']) && channels.contains(fromChannelName(value['channel'])));
|
|
releases.sort((Map<String, String> a, Map<String, String> b) {
|
|
final DateTime aDate = DateTime.parse(a['release_date']!);
|
|
final DateTime bDate = DateTime.parse(b['release_date']!);
|
|
return bDate.compareTo(aDate);
|
|
});
|
|
jsonData['releases'] = releases;
|
|
for (final Channel channel in channels) {
|
|
if (!revisionsBeingRemoved.contains((jsonData['current_release'] as Map<String, dynamic>)[getChannelName(channel)])) {
|
|
// Don't replace the current release if it's not one of the revisions we're removing.
|
|
continue;
|
|
}
|
|
final Map<String, String> replacementRelease = releases.firstWhere((Map<String, String> value) => value['channel'] == getChannelName(channel));
|
|
if (replacementRelease == null) {
|
|
throw UnpublishException('Unable to find previous release for channel ${getChannelName(channel)}.');
|
|
}
|
|
(jsonData['current_release'] as Map<String, dynamic>)[getChannelName(channel)] = replacementRelease['hash'];
|
|
print(
|
|
'${confirmed ? 'Reverting' : 'Would revert'} current ${getChannelName(channel)} '
|
|
'${getPublishedPlatform(platform)} release to ${replacementRelease['hash']} (version ${replacementRelease['version']}).'
|
|
);
|
|
}
|
|
await _cloudRemoveArchive(paths);
|
|
await _updateMetadata(jsonData);
|
|
}
|
|
|
|
Future<Map<Channel, Map<String, String>>> _getArchivePaths(List<Map<String, String>> releases) async {
|
|
final Set<String> hashes = <String>{};
|
|
final Map<Channel, Map<String, String>> paths = <Channel, Map<String, String>>{};
|
|
for (final Map<String, String> revision in releases) {
|
|
final String hash = revision['hash']!;
|
|
final Channel channel = fromChannelName(revision['channel']);
|
|
hashes.add(hash);
|
|
if (revisionsBeingRemoved.contains(hash) && channels.contains(channel)) {
|
|
paths[channel] ??= <String, String>{};
|
|
paths[channel]![hash] = revision['archive']!;
|
|
}
|
|
}
|
|
final Set<String> missingRevisions = revisionsBeingRemoved.difference(hashes.intersection(revisionsBeingRemoved));
|
|
if (missingRevisions.isNotEmpty) {
|
|
final bool plural = missingRevisions.length > 1;
|
|
throw UnpublishException('Revision${plural ? 's' : ''} $missingRevisions ${plural ? 'are' : 'is'} not present in the server metadata.');
|
|
}
|
|
return paths;
|
|
}
|
|
|
|
Future<Map<String, dynamic>> _loadMetadata() async {
|
|
final File metadataFile = File(
|
|
path.join(tempDir.absolute.path, getMetadataFilename(platform)),
|
|
);
|
|
// Always run this, even in dry runs.
|
|
await _runGsUtil(<String>['cp', metadataGsPath, metadataFile.absolute.path], confirm: true);
|
|
final String currentMetadata = metadataFile.readAsStringSync();
|
|
if (currentMetadata.isEmpty) {
|
|
throw UnpublishException('Empty metadata received from server');
|
|
}
|
|
|
|
Map<String, dynamic> jsonData;
|
|
try {
|
|
jsonData = json.decode(currentMetadata) as Map<String, dynamic>;
|
|
} on FormatException catch (e) {
|
|
throw UnpublishException('Unable to parse JSON metadata received from cloud: $e');
|
|
}
|
|
|
|
return jsonData;
|
|
}
|
|
|
|
Future<void> _updateMetadata(Map<String, dynamic> jsonData) async {
|
|
// We can't just cat the metadata from the server with 'gsutil cat', because
|
|
// Windows wants to echo the commands that execute in gsutil.bat to the
|
|
// stdout when we do that. So, we copy the file locally and then read it
|
|
// back in.
|
|
final File metadataFile = File(
|
|
path.join(tempDir.absolute.path, getMetadataFilename(platform)),
|
|
);
|
|
const JsonEncoder encoder = JsonEncoder.withIndent(' ');
|
|
metadataFile.writeAsStringSync(encoder.convert(jsonData));
|
|
print('${confirmed ? 'Overwriting' : 'Would overwrite'} $metadataGsPath with contents of ${metadataFile.absolute.path}');
|
|
await _cloudReplaceDest(metadataFile.absolute.path, metadataGsPath);
|
|
}
|
|
|
|
Future<String> _runGsUtil(
|
|
List<String> args, {
|
|
Directory? workingDirectory,
|
|
bool failOk = false,
|
|
bool confirm = false,
|
|
}) async {
|
|
final List<String> command = <String>['gsutil', '--', ...args];
|
|
if (confirm) {
|
|
return _processRunner.runProcess(
|
|
command,
|
|
workingDirectory: workingDirectory,
|
|
failOk: failOk,
|
|
);
|
|
} else {
|
|
print('Would run: ${command.join(' ')}');
|
|
return '';
|
|
}
|
|
}
|
|
|
|
Future<void> _cloudRemoveArchive(Map<Channel, Map<String, String>> paths) async {
|
|
final List<String> files = <String>[];
|
|
print('${confirmed ? 'Removing' : 'Would remove'} the following release archives:');
|
|
for (final Channel channel in paths.keys) {
|
|
final Map<String, String> hashes = paths[channel]!;
|
|
for (final String hash in hashes.keys) {
|
|
final String file = '$gsReleaseFolder/${hashes[hash]}';
|
|
files.add(file);
|
|
print(' $file');
|
|
}
|
|
}
|
|
await _runGsUtil(<String>['rm', ...files], failOk: true, confirm: confirmed);
|
|
}
|
|
|
|
Future<String> _cloudReplaceDest(String src, String dest) async {
|
|
assert(dest.startsWith('gs:'), '_cloudReplaceDest must have a destination in cloud storage.');
|
|
assert(!src.startsWith('gs:'), '_cloudReplaceDest must have a local source file.');
|
|
// We often don't have permission to overwrite, but
|
|
// we have permission to remove, so that's what we do first.
|
|
await _runGsUtil(<String>['rm', dest], failOk: true, confirm: confirmed);
|
|
String? mimeType;
|
|
if (dest.endsWith('.tar.xz')) {
|
|
mimeType = 'application/x-gtar';
|
|
}
|
|
if (dest.endsWith('.zip')) {
|
|
mimeType = 'application/zip';
|
|
}
|
|
if (dest.endsWith('.json')) {
|
|
mimeType = 'application/json';
|
|
}
|
|
final List<String> args = <String>[
|
|
// Use our preferred MIME type for the files we care about
|
|
// and let gsutil figure it out for anything else.
|
|
if (mimeType != null) ...<String>['-h', 'Content-Type:$mimeType'],
|
|
...<String>['cp', src, dest],
|
|
];
|
|
return _runGsUtil(args, confirm: confirmed);
|
|
}
|
|
}
|
|
|
|
void _printBanner(String message) {
|
|
final String banner = '*** $message ***';
|
|
print('\n');
|
|
print('*' * banner.length);
|
|
print(banner);
|
|
print('*' * banner.length);
|
|
print('\n');
|
|
}
|
|
|
|
/// Prepares a flutter git repo to be removed from the published cloud storage.
|
|
Future<void> main(List<String> rawArguments) async {
|
|
final List<String> allowedChannelValues = Channel.values.map<String>((Channel channel) => getChannelName(channel)).toList();
|
|
final List<String> allowedPlatformNames = PublishedPlatform.values.map<String>((PublishedPlatform platform) => getPublishedPlatform(platform)).toList();
|
|
final ArgParser argParser = ArgParser();
|
|
argParser.addOption(
|
|
'temp_dir',
|
|
help: 'A location where temporary files may be written. Defaults to a '
|
|
'directory in the system temp folder. If a temp_dir is not '
|
|
'specified, then by default a generated temporary directory will be '
|
|
'created, used, and removed automatically when the script exits.',
|
|
);
|
|
argParser.addMultiOption('revision',
|
|
help: 'The Flutter git repo revisions to remove from the published site. '
|
|
'Must be full 40-character hashes. More than one may be specified, '
|
|
'either by giving the option more than once, or by giving a comma '
|
|
'separated list. Required.');
|
|
argParser.addMultiOption(
|
|
'channel',
|
|
allowed: allowedChannelValues,
|
|
help: 'The Flutter channels to remove the archives corresponding to the '
|
|
'revisions given with --revision. More than one may be specified, '
|
|
'either by giving the option more than once, or by giving a '
|
|
'comma separated list. If not specified, then the archives from all '
|
|
'channels that a revision appears in will be removed.',
|
|
);
|
|
argParser.addMultiOption(
|
|
'platform',
|
|
allowed: allowedPlatformNames,
|
|
help: 'The Flutter platforms to remove the archive from. May specify more '
|
|
'than one, either by giving the option more than once, or by giving a '
|
|
'comma separated list. If not specified, then the archives from all '
|
|
'platforms that a revision appears in will be removed.',
|
|
);
|
|
argParser.addFlag(
|
|
'confirm',
|
|
help: 'If set, will actually remove the archive from Google Cloud Storage '
|
|
'upon successful execution of this script. Published archives will be '
|
|
'removed from this directory: $baseUrl$releaseFolder. This option '
|
|
'must be set to perform any action on the server, otherwise only a dry '
|
|
'run is performed.',
|
|
);
|
|
argParser.addFlag(
|
|
'help',
|
|
negatable: false,
|
|
help: 'Print help for this command.',
|
|
);
|
|
|
|
final ArgResults parsedArguments = argParser.parse(rawArguments);
|
|
|
|
if (parsedArguments['help'] as bool) {
|
|
print(argParser.usage);
|
|
exit(0);
|
|
}
|
|
|
|
void errorExit(String message, {int exitCode = -1}) {
|
|
stderr.write('Error: $message\n\n');
|
|
stderr.write('${argParser.usage}\n');
|
|
exit(exitCode);
|
|
}
|
|
|
|
final List<String> revisions = parsedArguments['revision'] as List<String>;
|
|
if (revisions.isEmpty) {
|
|
errorExit('Invalid argument: at least one --revision must be specified.');
|
|
}
|
|
for (final String revision in revisions) {
|
|
if (revision.length != 40) {
|
|
errorExit('Invalid argument: --revision "$revision" must be the entire hash, not just a prefix.');
|
|
}
|
|
if (revision.contains(RegExp(r'[^a-fA-F0-9]'))) {
|
|
errorExit('Invalid argument: --revision "$revision" contains non-hex characters.');
|
|
}
|
|
}
|
|
|
|
final String tempDirArg = parsedArguments['temp_dir'] as String;
|
|
Directory tempDir;
|
|
bool removeTempDir = false;
|
|
if (tempDirArg == null || tempDirArg.isEmpty) {
|
|
tempDir = Directory.systemTemp.createTempSync('flutter_package.');
|
|
removeTempDir = true;
|
|
} else {
|
|
tempDir = Directory(tempDirArg);
|
|
if (!tempDir.existsSync()) {
|
|
errorExit("Temporary directory $tempDirArg doesn't exist.");
|
|
}
|
|
}
|
|
|
|
if (!(parsedArguments['confirm'] as bool)) {
|
|
_printBanner('This will be just a dry run. To actually perform the changes below, re-run with --confirm argument.');
|
|
}
|
|
|
|
final List<String> channelArg = parsedArguments['channel'] as List<String>;
|
|
final List<String> channelOptions = channelArg.isNotEmpty ? channelArg : allowedChannelValues;
|
|
final Set<Channel> channels = channelOptions.map<Channel>((String value) => fromChannelName(value)).toSet();
|
|
final List<String> platformArg = parsedArguments['platform'] as List<String>;
|
|
final List<String> platformOptions = platformArg.isNotEmpty ? platformArg : allowedPlatformNames;
|
|
final List<PublishedPlatform> platforms = platformOptions.map<PublishedPlatform>((String value) => fromPublishedPlatform(value)).toList();
|
|
int exitCode = 0;
|
|
late String message;
|
|
late String stack;
|
|
try {
|
|
for (final PublishedPlatform platform in platforms) {
|
|
final ArchiveUnpublisher publisher = ArchiveUnpublisher(
|
|
tempDir,
|
|
revisions.toSet(),
|
|
channels,
|
|
platform,
|
|
confirmed: parsedArguments['confirm'] as bool,
|
|
);
|
|
await publisher.unpublishArchive();
|
|
}
|
|
} on UnpublishException catch (e, s) {
|
|
exitCode = e.exitCode;
|
|
message = e.message;
|
|
stack = s.toString();
|
|
} catch (e, s) {
|
|
exitCode = -1;
|
|
message = e.toString();
|
|
stack = s.toString();
|
|
} finally {
|
|
if (removeTempDir) {
|
|
tempDir.deleteSync(recursive: true);
|
|
}
|
|
if (exitCode != 0) {
|
|
errorExit('$message\n$stack', exitCode: exitCode);
|
|
}
|
|
if (!(parsedArguments['confirm'] as bool)) {
|
|
_printBanner('This was just a dry run. To actually perform the above changes, re-run with --confirm argument.');
|
|
}
|
|
exit(0);
|
|
}
|
|
}
|