From 245f7a003cb206d53de11d312d65916da0dd790a Mon Sep 17 00:00:00 2001 From: Sigmund Cherem Date: Sat, 2 Jul 2022 04:25:21 +0000 Subject: [PATCH] [dart2js] Add a couple scripts to aid our migration. These are simple scripts to count progress and find migration candidates. Change-Id: I872d85891001349dadbcf1d67e64ab5aa993d2a5 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/250146 Reviewed-by: Stephen Adams Commit-Queue: Sigmund Cherem Reviewed-by: Mayank Patke --- pkg/compiler/pubspec.yaml | 1 + pkg/compiler/tool/null_safety/candidates.dart | 152 ++++++++++++++++++ pkg/compiler/tool/null_safety/tally.dart | 91 +++++++++++ 3 files changed, 244 insertions(+) create mode 100644 pkg/compiler/tool/null_safety/candidates.dart create mode 100644 pkg/compiler/tool/null_safety/tally.dart diff --git a/pkg/compiler/pubspec.yaml b/pkg/compiler/pubspec.yaml index ff1ae628d87..d25829f7504 100644 --- a/pkg/compiler/pubspec.yaml +++ b/pkg/compiler/pubspec.yaml @@ -34,3 +34,4 @@ dev_dependencies: modular_test: any sourcemap_testing: any testing: any + vm: any diff --git a/pkg/compiler/tool/null_safety/candidates.dart b/pkg/compiler/tool/null_safety/candidates.dart new file mode 100644 index 00000000000..6dc4dde5889 --- /dev/null +++ b/pkg/compiler/tool/null_safety/candidates.dart @@ -0,0 +1,152 @@ +// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +/// Script to identify good opportunities for null safety migration. +/// +/// This script sorts libraries based on a "migratable" order. We compute +/// this order by counting how many of a library's dependencies have been +/// migrated. + +import 'dart:io'; + +import 'package:_fe_analyzer_shared/src/messages/severity.dart' show Severity; +import 'package:_fe_analyzer_shared/src/parser/parser.dart'; +import 'package:_fe_analyzer_shared/src/scanner/io.dart' + show readBytesFromFileSync; +import 'package:_fe_analyzer_shared/src/scanner/scanner.dart'; +import 'package:front_end/src/api_prototype/front_end.dart'; +import 'package:front_end/src/api_prototype/language_version.dart'; +import 'package:front_end/src/api_prototype/terminal_color_support.dart' + show printDiagnosticMessage; +import 'package:front_end/src/base/processed_options.dart'; +import 'package:front_end/src/fasta/compiler_context.dart'; +import 'package:front_end/src/fasta/source/diet_parser.dart'; +import 'package:front_end/src/fasta/source/directive_listener.dart'; +import 'package:front_end/src/fasta/uri_translator.dart' show UriTranslator; +import 'package:kernel/target/targets.dart' show TargetFlags; +import 'package:vm/target/vm.dart' show VmTarget; + +void main(List args) async { + var prefix = args.isEmpty ? 'pkg/compiler/' : args.first; + var files = >{}; + var isLegacy = {}; + var isNullSafe = {}; + + var entryUri = Uri.parse('package:compiler/src/dart2js.dart'); + var options = CompilerOptions() + ..sdkRoot = Uri.base.resolve("sdk/") + ..onDiagnostic = _onDiagnosticMessageHandler + ..compileSdk = true + ..packagesFileUri = Uri.base.resolve('.dart_tool/package_config.json') + ..target = VmTarget(TargetFlags()); + var pOptions = ProcessedOptions(options: options); + var uriResolver = await pOptions.getUriTranslator(); + var context = CompilerContext(pOptions); + await context.runInContext((_) async { + collectSources(uriResolver, entryUri, files); + }); + + for (var file in files.keys) { + if (await uriUsesLegacyLanguageVersion(file, options)) { + isLegacy.add(file); + } else { + isNullSafe.add(file); + } + } + + var fileSummary = {}; + for (var file in files.keys) { + if (!file.path.contains(prefix)) continue; + var directives = extractDirectiveUris(files[file]!) + .map(file.resolve) + .where((uri) => uri.path.contains('pkg/compiler/')); + var migrated = directives.where(isNullSafe.contains).length; + var total = directives.length; + fileSummary[file] = FileData(isNullSafe.contains(file), total, migrated); + } + + var keys = fileSummary.keys.toList(); + keys.sort((a, b) { + var fa = fileSummary[a]!; + var fb = fileSummary[b]!; + if (fa.isNullSafe && !fb.isNullSafe) return -1; + if (fb.isNullSafe && !fa.isNullSafe) return 1; + if (fa.totalDependencies == 0 && fb.totalDependencies != 0) return -1; + if (fb.totalDependencies == 0 && fa.totalDependencies != 0) return 1; + if (fa.ratio != fb.ratio) return fb.ratio.compareTo(fa.ratio); + return fb.migratedDependencies.compareTo(fb.migratedDependencies); + }); + + for (var file in keys) { + var data = fileSummary[file]!; + String status; + String text = shorten(file); + if (data.isNullSafe) { + status = '\x1b[33mmigrated ---\x1b[0m | $text'; + } else if (data.totalDependencies == 0) { + status = '\x1b[32mready ---\x1b[0m | $text'; + } else if (data.ratio == 1.0) { + status = '\x1b[32mready 100%\x1b[0m | $text'; + } else { + var perc = (data.ratio * 100).toStringAsFixed(0).padLeft(3); + status = '\x1b[31mwait $perc%\x1b[0m' + ' | $text [${data.migratedDependencies} / ${data.totalDependencies}]'; + } + print(status); + } +} + +class FileData { + final bool isNullSafe; + final int totalDependencies; + final int migratedDependencies; + + double get ratio => migratedDependencies / totalDependencies; + FileData(this.isNullSafe, this.totalDependencies, this.migratedDependencies); +} + +void _onDiagnosticMessageHandler(DiagnosticMessage m) { + if (m.severity == Severity.internalProblem || m.severity == Severity.error) { + printDiagnosticMessage(m, stderr.writeln); + exitCode = 1; + } +} + +/// Add to [files] all sources reachable from [start]. +void collectSources( + UriTranslator uriResolver, Uri start, Map> files) { + void helper(Uri uri) { + if (uri.scheme == 'dart') return; + uri = uriResolver.translate(uri) ?? uri; + if (!uri.path.contains('pkg/compiler/')) return; + if (files.containsKey(uri)) return; + var contents = readBytesFromFileSync(uri); + files[uri] = contents; + for (var directiveUri in extractDirectiveUris(contents)) { + helper(uri.resolve(directiveUri)); + } + } + + helper(start); +} + +/// Parse [contents] as a Dart program and return the URIs that appear in its +/// import, export, and part directives. +Set extractDirectiveUris(List contents) { + var listener = new DirectiveListener(); + new TopLevelParser(listener, + useImplicitCreationExpression: useImplicitCreationExpressionInCfe) + .parseUnit(scan(contents).tokens); + // Note: this purposely ignores part files (listener.parts). + return new Set() + ..addAll(listener.imports.map((directive) => directive.uri!)) + ..addAll(listener.exports.map((directive) => directive.uri!)); +} + +String shorten(Uri uri) { + if (uri.scheme != 'file') return uri.toString(); + final prefix = Uri.base.path; + if (uri.path.startsWith(prefix)) return uri.path.substring(prefix.length); + return uri.toString(); +} diff --git a/pkg/compiler/tool/null_safety/tally.dart b/pkg/compiler/tool/null_safety/tally.dart new file mode 100644 index 00000000000..33a2d1a8223 --- /dev/null +++ b/pkg/compiler/tool/null_safety/tally.dart @@ -0,0 +1,91 @@ +// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +/// Script to count progress of the pkg/compiler/lib/ migration + +import 'dart:io'; + +void main(List args) { + var path = args.isEmpty ? 'pkg/compiler/lib/' : args.first; + var dart2jsDir = Directory.fromUri(Uri.base.resolve(path)); + var entries = []; + for (var file in dart2jsDir.listSync(recursive: true)) { + if (file is File && file.uri.path.endsWith('.dart')) { + entries.add(FileData(file)); + } + } + + var tally = Tally(); + for (var e in entries) { + tally.totalFiles++; + tally.totalBytes += e.sizeBytes; + tally.totalLOC += e.sizeLOC; + if (e.nullSafe) { + tally.migratedFiles++; + tally.migratedBytes += e.sizeBytes; + tally.migratedLOC += e.sizeLOC; + } + } + + print(tally.formatString()); + //print(tally.csvRow()); +} + +/// Details about each file in the package to properly count migration progress. +class FileData { + final Uri path; + final int sizeBytes; + final int sizeLOC; + final bool nullSafe; + + FileData._(this.path, this.sizeBytes, this.sizeLOC, this.nullSafe); + + factory FileData(File file) { + var contents = file.readAsStringSync(); + var length = contents.length; + var sizeLOC = '\n'.allMatches(contents).length; + var nullSafe = !contents.contains("// @dart = 2.10"); + return FileData._(file.uri, length, sizeLOC, nullSafe); + } +} + +/// Cumulative information about the status of the null safety migration. +class Tally { + int totalFiles = 0; + int migratedFiles = 0; + int totalBytes = 0; + int migratedBytes = 0; + int totalLOC = 0; + int migratedLOC = 0; + + /// Emit a readable table representation of the null safety progress. + String formatString() { + String _pad(String text, int width) { + return (' ' * (10 - text.length)) + text; + } + + String _row(String name, int a, int b) { + var padA = _pad('$a', 10); + var padB = _pad('$b', 10); + var padC = _pad((a * 100 / b).toStringAsFixed(2), 10); + return '${_pad(name, 8)} $padA $padB $padC%'; + } + + return '${_pad("", 10)} ${_pad("migrated", 10)} ${_pad("total", 10)} ${_pad("%", 10)}\n' + '${_row('Files', migratedFiles, totalFiles)}\n' + '${_row('Lines', migratedLOC, totalLOC)}\n' + '${_row('Bytes', migratedBytes, totalBytes)}'; + } + + /// Emit a csv representation of the null safety progress, useful to track + /// data over time. + String csvRow() => [ + totalFiles, + migratedFiles, + totalBytes, + migratedBytes, + totalLOC, + migratedLOC, + ].join(','); +}