mirror of
https://github.com/flutter/flutter
synced 2024-11-05 18:37:51 +00:00
536 lines
20 KiB
Dart
536 lines
20 KiB
Dart
// Copyright 2014 The Flutter Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
// found in the LICENSE file.
|
|
|
|
import 'dart:convert';
|
|
import 'dart:io' show stderr;
|
|
import 'dart:typed_data';
|
|
|
|
import 'package:convert/convert.dart';
|
|
import 'package:crypto/crypto.dart';
|
|
import 'package:file/file.dart';
|
|
import 'package:http/http.dart' as http;
|
|
import 'package:path/path.dart' as path;
|
|
import 'package:platform/platform.dart' show LocalPlatform, Platform;
|
|
import 'package:pool/pool.dart';
|
|
import 'package:process/process.dart';
|
|
|
|
import 'common.dart';
|
|
import 'process_runner.dart';
|
|
|
|
typedef HttpReader = Future<Uint8List> Function(Uri url, {Map<String, String> headers});
|
|
|
|
/// Creates a pre-populated Flutter archive from a git repo.
|
|
class ArchiveCreator {
|
|
/// [tempDir] is the directory to use for creating the archive. The script
|
|
/// will place several GiB of data there, so it should have available space.
|
|
///
|
|
/// The processManager argument is used to inject a mock of [ProcessManager] for
|
|
/// testing purposes.
|
|
///
|
|
/// If subprocessOutput is true, then output from processes invoked during
|
|
/// archive creation is echoed to stderr and stdout.
|
|
factory ArchiveCreator(
|
|
Directory tempDir,
|
|
Directory outputDir,
|
|
String revision,
|
|
Branch branch, {
|
|
required FileSystem fs,
|
|
HttpReader? httpReader,
|
|
Platform platform = const LocalPlatform(),
|
|
ProcessManager? processManager,
|
|
bool strict = true,
|
|
bool subprocessOutput = true,
|
|
}) {
|
|
final Directory flutterRoot = fs.directory(path.join(tempDir.path, 'flutter'));
|
|
final ProcessRunner processRunner = ProcessRunner(
|
|
processManager: processManager,
|
|
subprocessOutput: subprocessOutput,
|
|
platform: platform,
|
|
)..environment['PUB_CACHE'] = path.join(
|
|
tempDir.path, '.pub-cache',
|
|
);
|
|
final String flutterExecutable = path.join(
|
|
flutterRoot.absolute.path,
|
|
'bin',
|
|
'flutter',
|
|
);
|
|
final String dartExecutable = path.join(
|
|
flutterRoot.absolute.path,
|
|
'bin',
|
|
'cache',
|
|
'dart-sdk',
|
|
'bin',
|
|
'dart',
|
|
);
|
|
|
|
return ArchiveCreator._(
|
|
tempDir: tempDir,
|
|
platform: platform,
|
|
flutterRoot: flutterRoot,
|
|
fs: fs,
|
|
outputDir: outputDir,
|
|
revision: revision,
|
|
branch: branch,
|
|
strict: strict,
|
|
processRunner: processRunner,
|
|
httpReader: httpReader ?? http.readBytes,
|
|
flutterExecutable: flutterExecutable,
|
|
dartExecutable: dartExecutable,
|
|
);
|
|
}
|
|
|
|
ArchiveCreator._({
|
|
required this.branch,
|
|
required String dartExecutable,
|
|
required this.fs,
|
|
required String flutterExecutable,
|
|
required this.flutterRoot,
|
|
required this.httpReader,
|
|
required this.outputDir,
|
|
required this.platform,
|
|
required ProcessRunner processRunner,
|
|
required this.revision,
|
|
required this.strict,
|
|
required this.tempDir,
|
|
}) :
|
|
assert(revision.length == 40),
|
|
_processRunner = processRunner,
|
|
_flutter = flutterExecutable,
|
|
_dart = dartExecutable;
|
|
|
|
/// The platform to use for the environment and determining which
|
|
/// platform we're running on.
|
|
final Platform platform;
|
|
|
|
/// The branch to build the archive for. The branch must contain [revision].
|
|
final Branch branch;
|
|
|
|
/// The git revision hash to build the archive for. This revision has
|
|
/// to be available in the [branch], although it doesn't have to be
|
|
/// at HEAD, since we clone the branch and then reset to this revision
|
|
/// to create the archive.
|
|
final String revision;
|
|
|
|
/// The flutter root directory in the [tempDir].
|
|
final Directory flutterRoot;
|
|
|
|
/// The temporary directory used to build the archive in.
|
|
final Directory tempDir;
|
|
|
|
/// The directory to write the output file to.
|
|
final Directory outputDir;
|
|
|
|
final FileSystem fs;
|
|
|
|
/// True if the creator should be strict about checking requirements or not.
|
|
///
|
|
/// In strict mode, will insist that the [revision] be a tagged revision.
|
|
final bool strict;
|
|
|
|
final Uri _minGitUri = Uri.parse(mingitForWindowsUrl);
|
|
final ProcessRunner _processRunner;
|
|
|
|
/// Used to tell the [ArchiveCreator] which function to use for reading
|
|
/// bytes from a URL. Used in tests to inject a fake reader. Defaults to
|
|
/// [http.readBytes].
|
|
final HttpReader httpReader;
|
|
|
|
final Map<String, String> _version = <String, String>{};
|
|
late String _flutter;
|
|
late String _dart;
|
|
|
|
late final Future<String> _dartArch = (() async {
|
|
// Parse 'arch' out of a string like '... "os_arch"\n'.
|
|
return (await _runDart(<String>['--version']))
|
|
.trim().split(' ').last.replaceAll('"', '').split('_')[1];
|
|
})();
|
|
|
|
/// Returns a default archive name when given a Git revision.
|
|
/// Used when an output filename is not given.
|
|
Future<String> get _archiveName async {
|
|
final String os = platform.operatingSystem.toLowerCase();
|
|
// Include the intended host architecture in the file name for non-x64.
|
|
final String arch = await _dartArch == 'x64' ? '' : '${await _dartArch}_';
|
|
// We don't use .tar.xz on Mac because although it can unpack them
|
|
// on the command line (with tar), the "Archive Utility" that runs
|
|
// when you double-click on them just does some crazy behavior (it
|
|
// converts it to a compressed cpio archive, and when you double
|
|
// click on that, it converts it back to .tar.xz, without ever
|
|
// unpacking it!) So, we use .zip for Mac, and the files are about
|
|
// 220MB larger than they need to be. :-(
|
|
final String suffix = platform.isLinux ? 'tar.xz' : 'zip';
|
|
final String package = '${os}_$arch${_version[frameworkVersionTag]}';
|
|
return 'flutter_$package-${branch.name}.$suffix';
|
|
}
|
|
|
|
/// Checks out the flutter repo and prepares it for other operations.
|
|
///
|
|
/// Returns the version for this release as obtained from the git tags, and
|
|
/// the dart version as obtained from `flutter --version`.
|
|
Future<Map<String, String>> initializeRepo() async {
|
|
await _checkoutFlutter();
|
|
if (_version.isEmpty) {
|
|
_version.addAll(await _getVersion());
|
|
}
|
|
return _version;
|
|
}
|
|
|
|
/// Performs all of the steps needed to create an archive.
|
|
Future<File> createArchive() async {
|
|
assert(_version.isNotEmpty, 'Must run initializeRepo before createArchive');
|
|
final File outputFile = fs.file(path.join(
|
|
outputDir.absolute.path,
|
|
await _archiveName,
|
|
));
|
|
await _installMinGitIfNeeded();
|
|
await _populateCaches();
|
|
await _validate();
|
|
await _archiveFiles(outputFile);
|
|
return outputFile;
|
|
}
|
|
|
|
/// Validates the integrity of the release package.
|
|
///
|
|
/// Currently only checks that macOS binaries are codesigned. Will throw a
|
|
/// [PreparePackageException] if the test fails.
|
|
Future<void> _validate() async {
|
|
// Only validate in strict mode, which means `--publish`
|
|
if (!strict || !platform.isMacOS) {
|
|
return;
|
|
}
|
|
// Validate that the dart binary is codesigned
|
|
try {
|
|
// TODO(fujino): Use the conductor https://github.com/flutter/flutter/issues/81701
|
|
await _processRunner.runProcess(
|
|
<String>[
|
|
'codesign',
|
|
'-vvvv',
|
|
'--check-notarization',
|
|
_dart,
|
|
],
|
|
workingDirectory: flutterRoot,
|
|
);
|
|
} on PreparePackageException catch (e) {
|
|
throw PreparePackageException(
|
|
'The binary $_dart was not codesigned!\n${e.message}',
|
|
);
|
|
}
|
|
}
|
|
|
|
/// Returns the version map of this release, according the to tags in the
|
|
/// repo and the output of `flutter --version --machine`.
|
|
///
|
|
/// This looks for the tag attached to [revision] and, if it doesn't find one,
|
|
/// git will give an error.
|
|
///
|
|
/// If [strict] is true, the exact [revision] must be tagged to return the
|
|
/// version. If [strict] is not true, will look backwards in time starting at
|
|
/// [revision] to find the most recent version tag.
|
|
///
|
|
/// The version found as a git tag is added to the information given by
|
|
/// `flutter --version --machine` with the `frameworkVersionFromGit` tag, and
|
|
/// returned.
|
|
Future<Map<String, String>> _getVersion() async {
|
|
String gitVersion;
|
|
if (strict) {
|
|
try {
|
|
gitVersion = await _runGit(<String>['describe', '--tags', '--exact-match', revision]);
|
|
} on PreparePackageException catch (exception) {
|
|
throw PreparePackageException(
|
|
'Git error when checking for a version tag attached to revision $revision.\n'
|
|
'Perhaps there is no tag at that revision?:\n'
|
|
'$exception'
|
|
);
|
|
}
|
|
} else {
|
|
gitVersion = await _runGit(<String>['describe', '--tags', '--abbrev=0', revision]);
|
|
}
|
|
// Run flutter command twice, once to make sure the flutter command is built
|
|
// and ready (and thus won't output any junk on stdout the second time), and
|
|
// once to capture theJSON output. The second run should be fast.
|
|
await _runFlutter(<String>['--version', '--machine']);
|
|
final String versionJson = await _runFlutter(<String>['--version', '--machine']);
|
|
final Map<String, String> versionMap = <String, String>{};
|
|
final Map<String, dynamic> result = json.decode(versionJson) as Map<String, dynamic>;
|
|
result.forEach((String key, dynamic value) => versionMap[key] = value.toString());
|
|
versionMap[frameworkVersionTag] = gitVersion;
|
|
versionMap[dartTargetArchTag] = await _dartArch;
|
|
return versionMap;
|
|
}
|
|
|
|
/// Clone the Flutter repo and make sure that the git environment is sane
|
|
/// for when the user will unpack it.
|
|
Future<void> _checkoutFlutter() async {
|
|
// We want the user to start out the in the specified branch instead of a
|
|
// detached head. To do that, we need to make sure the branch points at the
|
|
// desired revision.
|
|
await _runGit(<String>['clone', '-b', branch.name, gobMirror], workingDirectory: tempDir);
|
|
await _runGit(<String>['reset', '--hard', revision]);
|
|
|
|
// Make the origin point to github instead of the chromium mirror.
|
|
await _runGit(<String>['remote', 'set-url', 'origin', githubRepo]);
|
|
|
|
// Minify `.git` footprint (saving about ~100 MB as of Oct 2022)
|
|
await _runGit(<String>['gc', '--prune=now', '--aggressive']);
|
|
}
|
|
|
|
/// Retrieve the MinGit executable from storage and unpack it.
|
|
Future<void> _installMinGitIfNeeded() async {
|
|
if (!platform.isWindows) {
|
|
return;
|
|
}
|
|
final Uint8List data = await httpReader(_minGitUri);
|
|
final File gitFile = fs.file(path.join(tempDir.absolute.path, 'mingit.zip'));
|
|
await gitFile.writeAsBytes(data, flush: true);
|
|
|
|
final Directory minGitPath = fs.directory(path.join(flutterRoot.absolute.path, 'bin', 'mingit'));
|
|
await minGitPath.create(recursive: true);
|
|
await _unzipArchive(gitFile, workingDirectory: minGitPath);
|
|
}
|
|
|
|
/// Downloads an archive of every package that is present in the temporary
|
|
/// pub-cache from pub.dev. Stores the archives in
|
|
/// $flutterRoot/.pub-preload-cache.
|
|
///
|
|
/// These archives will be installed in the user-level cache on first
|
|
/// following flutter command that accesses the cache.
|
|
///
|
|
/// Precondition: all packages currently in the PUB_CACHE of [_processRunner]
|
|
/// are installed from pub.dev.
|
|
Future<void> _downloadPubPackageArchives() async {
|
|
final Pool pool = Pool(10); // Number of simultaneous downloads.
|
|
final http.Client client = http.Client();
|
|
final Directory preloadCache = fs.directory(path.join(flutterRoot.path, '.pub-preload-cache'));
|
|
preloadCache.createSync(recursive: true);
|
|
/// Fetch a single package.
|
|
Future<void> fetchPackageArchive(String name, String version) async {
|
|
await pool.withResource(() async {
|
|
stderr.write('Fetching package archive for $name-$version.\n');
|
|
int retries = 7;
|
|
while (true) {
|
|
retries-=1;
|
|
try {
|
|
final Uri packageListingUrl = Uri.parse('https://pub.dev/api/packages/$name');
|
|
// Fetch the package listing to obtain the package download url.
|
|
final http.Response packageListingResponse = await client.get(packageListingUrl);
|
|
if (packageListingResponse.statusCode != 200) {
|
|
throw Exception('Downloading $packageListingUrl failed. Status code ${packageListingResponse.statusCode}.');
|
|
}
|
|
final dynamic decodedPackageListing = json.decode(packageListingResponse.body);
|
|
if (decodedPackageListing is! Map) {
|
|
throw const FormatException('Package listing should be a map');
|
|
}
|
|
final dynamic versions = decodedPackageListing['versions'];
|
|
if (versions is! List) {
|
|
throw const FormatException('.versions should be a list');
|
|
}
|
|
final Map<String, dynamic> versionDescription = versions.firstWhere(
|
|
(dynamic description) {
|
|
if (description is! Map) {
|
|
throw const FormatException('.versions elements should be maps');
|
|
}
|
|
return description['version'] == version;
|
|
},
|
|
orElse: () => throw FormatException('Could not find $name-$version in package listing')
|
|
) as Map<String, dynamic>;
|
|
final dynamic downloadUrl = versionDescription['archive_url'];
|
|
if (downloadUrl is! String) {
|
|
throw const FormatException('archive_url should be a string');
|
|
}
|
|
final dynamic archiveSha256 = versionDescription['archive_sha256'];
|
|
if (archiveSha256 is! String) {
|
|
throw const FormatException('archive_sha256 should be a string');
|
|
}
|
|
final http.Request request = http.Request('get', Uri.parse(downloadUrl));
|
|
final http.StreamedResponse response = await client.send(request);
|
|
if (response.statusCode != 200) {
|
|
throw Exception('Downloading ${request.url} failed. Status code ${response.statusCode}.');
|
|
}
|
|
final File archiveFile = fs.file(
|
|
path.join(preloadCache.path, '$name-$version.tar.gz'),
|
|
);
|
|
await response.stream.pipe(archiveFile.openWrite());
|
|
final Stream<List<int>> archiveStream = archiveFile.openRead();
|
|
final Digest r = await sha256.bind(archiveStream).first;
|
|
if (hex.encode(r.bytes) != archiveSha256) {
|
|
throw Exception('Hash mismatch of downloaded archive');
|
|
}
|
|
} on Exception catch (e) {
|
|
stderr.write('Failed downloading $name-$version. $e\n');
|
|
if (retries > 0) {
|
|
stderr.write('Retrying download of $name-$version...');
|
|
// Retry.
|
|
continue;
|
|
} else {
|
|
rethrow;
|
|
}
|
|
}
|
|
break;
|
|
}
|
|
});
|
|
}
|
|
final Map<String, dynamic> cacheDescription = json.decode(await _runFlutter(<String>['pub', 'cache', 'list'])) as Map<String, dynamic>;
|
|
final Map<String, dynamic> packages = cacheDescription['packages'] as Map<String, dynamic>;
|
|
final List<Future<void>> downloads = <Future<void>>[];
|
|
for (final MapEntry<String, dynamic> package in packages.entries) {
|
|
final String name = package.key;
|
|
final Map<String, dynamic> versions = package.value as Map<String, dynamic>;
|
|
for (final String version in versions.keys) {
|
|
downloads.add(fetchPackageArchive(name, version));
|
|
}
|
|
}
|
|
await Future.wait(downloads);
|
|
client.close();
|
|
}
|
|
|
|
/// Prepare the archive repo so that it has all of the caches warmed up and
|
|
/// is configured for the user to begin working.
|
|
Future<void> _populateCaches() async {
|
|
await _runFlutter(<String>['doctor']);
|
|
await _runFlutter(<String>['update-packages']);
|
|
await _runFlutter(<String>['precache']);
|
|
await _runFlutter(<String>['ide-config']);
|
|
|
|
// Create each of the templates, since they will call 'pub get' on
|
|
// themselves when created, and this will warm the cache with their
|
|
// dependencies too.
|
|
for (final String template in <String>['app', 'package', 'plugin']) {
|
|
final String createName = path.join(tempDir.path, 'create_$template');
|
|
await _runFlutter(
|
|
<String>['create', '--template=$template', createName],
|
|
// Run it outside the cloned Flutter repo to not nest git repos, since
|
|
// they'll be git repos themselves too.
|
|
workingDirectory: tempDir,
|
|
);
|
|
}
|
|
await _downloadPubPackageArchives();
|
|
// Yes, we could just skip all .packages files when constructing
|
|
// the archive, but some are checked in, and we don't want to skip
|
|
// those.
|
|
await _runGit(<String>[
|
|
'clean',
|
|
'-f',
|
|
// Do not -X as it could lead to entire bin/cache getting cleaned
|
|
'-x',
|
|
'--',
|
|
'**/.packages',
|
|
]);
|
|
/// Remove package_config files and any contents in .dart_tool
|
|
await _runGit(<String>[
|
|
'clean',
|
|
'-f',
|
|
'-x',
|
|
'--',
|
|
'**/.dart_tool/',
|
|
]);
|
|
|
|
// Ensure the above commands do not clean out the cache
|
|
final Directory flutterCache = fs.directory(path.join(flutterRoot.absolute.path, 'bin', 'cache'));
|
|
if (!flutterCache.existsSync()) {
|
|
throw Exception('The flutter cache was not found at ${flutterCache.path}!');
|
|
}
|
|
|
|
/// Remove git subfolder from .pub-cache, this contains the flutter goldens
|
|
/// and new flutter_gallery.
|
|
final Directory gitCache = fs.directory(path.join(flutterRoot.absolute.path, '.pub-cache', 'git'));
|
|
if (gitCache.existsSync()) {
|
|
gitCache.deleteSync(recursive: true);
|
|
}
|
|
}
|
|
|
|
/// Write the archive to the given output file.
|
|
Future<void> _archiveFiles(File outputFile) async {
|
|
if (outputFile.path.toLowerCase().endsWith('.zip')) {
|
|
await _createZipArchive(outputFile, flutterRoot);
|
|
} else if (outputFile.path.toLowerCase().endsWith('.tar.xz')) {
|
|
await _createTarArchive(outputFile, flutterRoot);
|
|
}
|
|
}
|
|
|
|
Future<String> _runDart(List<String> args, {Directory? workingDirectory}) {
|
|
return _processRunner.runProcess(
|
|
<String>[_dart, ...args],
|
|
workingDirectory: workingDirectory ?? flutterRoot,
|
|
);
|
|
}
|
|
|
|
Future<String> _runFlutter(List<String> args, {Directory? workingDirectory}) {
|
|
return _processRunner.runProcess(
|
|
<String>[_flutter, ...args],
|
|
workingDirectory: workingDirectory ?? flutterRoot,
|
|
);
|
|
}
|
|
|
|
Future<String> _runGit(List<String> args, {Directory? workingDirectory}) {
|
|
return _processRunner.runProcess(
|
|
<String>['git', ...args],
|
|
workingDirectory: workingDirectory ?? flutterRoot,
|
|
);
|
|
}
|
|
|
|
/// Unpacks the given zip file into the currentDirectory (if set), or the
|
|
/// same directory as the archive.
|
|
Future<String> _unzipArchive(File archive, {Directory? workingDirectory}) {
|
|
workingDirectory ??= fs.directory(path.dirname(archive.absolute.path));
|
|
List<String> commandLine;
|
|
if (platform.isWindows) {
|
|
commandLine = <String>[
|
|
'7za',
|
|
'x',
|
|
archive.absolute.path,
|
|
];
|
|
} else {
|
|
commandLine = <String>[
|
|
'unzip',
|
|
archive.absolute.path,
|
|
];
|
|
}
|
|
return _processRunner.runProcess(commandLine, workingDirectory: workingDirectory);
|
|
}
|
|
|
|
/// Create a zip archive from the directory source.
|
|
Future<String> _createZipArchive(File output, Directory source) async {
|
|
List<String> commandLine;
|
|
if (platform.isWindows) {
|
|
// Unhide the .git folder, https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/attrib.
|
|
await _processRunner.runProcess(
|
|
<String>['attrib', '-h', '.git'],
|
|
workingDirectory: fs.directory(source.absolute.path),
|
|
);
|
|
commandLine = <String>[
|
|
'7za',
|
|
'a',
|
|
'-tzip',
|
|
'-mx=9',
|
|
output.absolute.path,
|
|
path.basename(source.path),
|
|
];
|
|
} else {
|
|
commandLine = <String>[
|
|
'zip',
|
|
'-r',
|
|
'-9',
|
|
'--symlinks',
|
|
output.absolute.path,
|
|
path.basename(source.path),
|
|
];
|
|
}
|
|
return _processRunner.runProcess(
|
|
commandLine,
|
|
workingDirectory: fs.directory(path.dirname(source.absolute.path)),
|
|
);
|
|
}
|
|
|
|
/// Create a tar archive from the directory source.
|
|
Future<String> _createTarArchive(File output, Directory source) {
|
|
return _processRunner.runProcess(<String>[
|
|
'tar',
|
|
'cJf',
|
|
output.absolute.path,
|
|
// Print out input files as they get added, to debug hangs
|
|
'--verbose',
|
|
path.basename(source.absolute.path),
|
|
], workingDirectory: fs.directory(path.dirname(source.absolute.path)));
|
|
}
|
|
}
|