development/flutter
development/flutter
1
0
Fork 0
mirror of https://github.com/flutter/flutter synced 2024-08-27 20:10:11 +00:00
Code Issues Packages Projects Releases Wiki Activity

Cleaner test.dart output. (#109206)

Browse source
This commit is contained in:
Ian Hickson 2022-08-14 01:53:54 -07:00 committed by GitHub
parent d50c5b1b61
commit 97901da149
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 582 additions and 751 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

118
dev/bots/analyze.dart
View file

@ -44,13 +44,13 @@ Future<void> main(List<String> arguments) async {
);
dart = path.join(dartSdk, 'bin', Platform.isWindows ? 'dart.exe' : 'dart');
pub = path.join(dartSdk, 'bin', Platform.isWindows ? 'pub.bat' : 'pub');
print('$clock STARTING ANALYSIS');
printProgress('STARTING ANALYSIS');
await run(arguments);
if (hasError) {
print('$clock ${bold}Test failed.$reset');
printProgress('${bold}Test failed.$reset');
reportErrorsAndExit();
}
print('$clock ${bold}Analysis successful.$reset');
printProgress('${bold}Analysis successful.$reset');
}
/// Scans [arguments] for an argument of the form `--dart-sdk` or
@ -88,85 +88,85 @@ Future<void> run(List<String> arguments) async {
foundError(<String>['The analyze.dart script must be run with --enable-asserts.']);
}
print('$clock No Double.clamp');
printProgress('No Double.clamp');
await verifyNoDoubleClamp(flutterRoot);
print('$clock All tool test files end in _test.dart...');
printProgress('All tool test files end in _test.dart...');
await verifyToolTestsEndInTestDart(flutterRoot);
print('$clock No sync*/async*');
printProgress('No sync*/async*');
await verifyNoSyncAsyncStar(flutterPackages);
await verifyNoSyncAsyncStar(flutterExamples, minimumMatches: 200);
print('$clock No runtimeType in toString...');
printProgress('No runtimeType in toString...');
await verifyNoRuntimeTypeInToString(flutterRoot);
print('$clock Debug mode instead of checked mode...');
printProgress('Debug mode instead of checked mode...');
await verifyNoCheckedMode(flutterRoot);
print('$clock Links for creating GitHub issues');
printProgress('Links for creating GitHub issues');
await verifyIssueLinks(flutterRoot);
print('$clock Unexpected binaries...');
printProgress('Unexpected binaries...');
await verifyNoBinaries(flutterRoot);
print('$clock Trailing spaces...');
printProgress('Trailing spaces...');
await verifyNoTrailingSpaces(flutterRoot); // assumes no unexpected binaries, so should be after verifyNoBinaries
print('$clock Deprecations...');
printProgress('Deprecations...');
await verifyDeprecations(flutterRoot);
print('$clock Goldens...');
printProgress('Goldens...');
await verifyGoldenTags(flutterPackages);
print('$clock Skip test comments...');
printProgress('Skip test comments...');
await verifySkipTestComments(flutterRoot);
print('$clock Licenses...');
printProgress('Licenses...');
await verifyNoMissingLicense(flutterRoot);
print('$clock Test imports...');
printProgress('Test imports...');
await verifyNoTestImports(flutterRoot);
print('$clock Bad imports (framework)...');
printProgress('Bad imports (framework)...');
await verifyNoBadImportsInFlutter(flutterRoot);
print('$clock Bad imports (tools)...');
printProgress('Bad imports (tools)...');
await verifyNoBadImportsInFlutterTools(flutterRoot);
print('$clock Internationalization...');
printProgress('Internationalization...');
await verifyInternationalizations(flutterRoot, dart);
print('$clock Integration test timeouts...');
printProgress('Integration test timeouts...');
await verifyIntegrationTestTimeouts(flutterRoot);
print('$clock null initialized debug fields...');
printProgress('null initialized debug fields...');
await verifyNullInitializedDebugExpensiveFields(flutterRoot);
// Ensure that all package dependencies are in sync.
print('$clock Package dependencies...');
printProgress('Package dependencies...');
await runCommand(flutter, <String>['update-packages', '--verify-only'],
workingDirectory: flutterRoot,
);
/// Ensure that no new dependencies have been accidentally
/// added to core packages.
print('$clock Package Allowlist...');
printProgress('Package Allowlist...');
await _checkConsumerDependencies();
// Analyze all the Dart code in the repo.
print('$clock Dart analysis...');
printProgress('Dart analysis...');
await _runFlutterAnalyze(flutterRoot, options: <String>[
'--flutter-repo',
...arguments,
]);
print('$clock Executable allowlist...');
printProgress('Executable allowlist...');
await _checkForNewExecutables();
// Try with the --watch analyzer, to make sure it returns success also.
// The --benchmark argument exits after one run.
print('$clock Dart analysis (with --watch)...');
printProgress('Dart analysis (with --watch)...');
await _runFlutterAnalyze(flutterRoot, options: <String>[
'--flutter-repo',
'--watch',
@ -175,14 +175,14 @@ Future<void> run(List<String> arguments) async {
]);
// Analyze the code in `{@tool snippet}` sections in the repo.
print('$clock Snippet code...');
printProgress('Snippet code...');
await runCommand(dart,
<String>['--enable-asserts', path.join(flutterRoot, 'dev', 'bots', 'analyze_snippet_code.dart'), '--verbose'],
workingDirectory: flutterRoot,
);
// Try analysis against a big version of the gallery; generate into a temporary directory.
print('$clock Dart analysis (mega gallery)...');
printProgress('Dart analysis (mega gallery)...');
final Directory outDir = Directory.systemTemp.createTempSync('flutter_mega_gallery.');
try {
await runCommand(dart,
@ -548,27 +548,23 @@ String _generateLicense(String prefix) {
Future<void> verifyNoMissingLicense(String workingDirectory, { bool checkMinimums = true }) async {
final int? overrideMinimumMatches = checkMinimums ? null : 0;
int failed = 0;
failed += await _verifyNoMissingLicenseForExtension(workingDirectory, 'dart', overrideMinimumMatches ?? 2000, _generateLicense('// '));
failed += await _verifyNoMissingLicenseForExtension(workingDirectory, 'java', overrideMinimumMatches ?? 39, _generateLicense('// '));
failed += await _verifyNoMissingLicenseForExtension(workingDirectory, 'h', overrideMinimumMatches ?? 30, _generateLicense('// '));
failed += await _verifyNoMissingLicenseForExtension(workingDirectory, 'm', overrideMinimumMatches ?? 30, _generateLicense('// '));
failed += await _verifyNoMissingLicenseForExtension(workingDirectory, 'cpp', overrideMinimumMatches ?? 0, _generateLicense('// '));
failed += await _verifyNoMissingLicenseForExtension(workingDirectory, 'swift', overrideMinimumMatches ?? 10, _generateLicense('// '));
failed += await _verifyNoMissingLicenseForExtension(workingDirectory, 'gradle', overrideMinimumMatches ?? 80, _generateLicense('// '));
failed += await _verifyNoMissingLicenseForExtension(workingDirectory, 'gn', overrideMinimumMatches ?? 0, _generateLicense('# '));
failed += await _verifyNoMissingLicenseForExtension(workingDirectory, 'sh', overrideMinimumMatches ?? 1, _generateLicense('# '), header: r'#!/usr/bin/env bash\n',);
failed += await _verifyNoMissingLicenseForExtension(workingDirectory, 'bat', overrideMinimumMatches ?? 1, _generateLicense('REM '), header: r'@ECHO off\n');
failed += await _verifyNoMissingLicenseForExtension(workingDirectory, 'ps1', overrideMinimumMatches ?? 1, _generateLicense('# '));
failed += await _verifyNoMissingLicenseForExtension(workingDirectory, 'html', overrideMinimumMatches ?? 1, '<!-- ${_generateLicense('')} -->', trailingBlank: false, header: r'<!DOCTYPE HTML>\n');
failed += await _verifyNoMissingLicenseForExtension(workingDirectory, 'xml', overrideMinimumMatches ?? 1, '<!-- ${_generateLicense('')} -->', header: r'(<\?xml version="1.0" encoding="utf-8"\?>\n)?');
failed += await _verifyNoMissingLicenseForExtension(workingDirectory, 'frag', overrideMinimumMatches ?? 1, _generateLicense('// '), header: r'#version 320 es(\n)+');
if (failed > 0) {
foundError(<String>['License check failed.']);
}
await _verifyNoMissingLicenseForExtension(workingDirectory, 'dart', overrideMinimumMatches ?? 2000, _generateLicense('// '));
await _verifyNoMissingLicenseForExtension(workingDirectory, 'java', overrideMinimumMatches ?? 39, _generateLicense('// '));
await _verifyNoMissingLicenseForExtension(workingDirectory, 'h', overrideMinimumMatches ?? 30, _generateLicense('// '));
await _verifyNoMissingLicenseForExtension(workingDirectory, 'm', overrideMinimumMatches ?? 30, _generateLicense('// '));
await _verifyNoMissingLicenseForExtension(workingDirectory, 'cpp', overrideMinimumMatches ?? 0, _generateLicense('// '));
await _verifyNoMissingLicenseForExtension(workingDirectory, 'swift', overrideMinimumMatches ?? 10, _generateLicense('// '));
await _verifyNoMissingLicenseForExtension(workingDirectory, 'gradle', overrideMinimumMatches ?? 80, _generateLicense('// '));
await _verifyNoMissingLicenseForExtension(workingDirectory, 'gn', overrideMinimumMatches ?? 0, _generateLicense('# '));
await _verifyNoMissingLicenseForExtension(workingDirectory, 'sh', overrideMinimumMatches ?? 1, _generateLicense('# '), header: r'#!/usr/bin/env bash\n',);
await _verifyNoMissingLicenseForExtension(workingDirectory, 'bat', overrideMinimumMatches ?? 1, _generateLicense('REM '), header: r'@ECHO off\n');
await _verifyNoMissingLicenseForExtension(workingDirectory, 'ps1', overrideMinimumMatches ?? 1, _generateLicense('# '));
await _verifyNoMissingLicenseForExtension(workingDirectory, 'html', overrideMinimumMatches ?? 1, '<!-- ${_generateLicense('')} -->', trailingBlank: false, header: r'<!DOCTYPE HTML>\n');
await _verifyNoMissingLicenseForExtension(workingDirectory, 'xml', overrideMinimumMatches ?? 1, '<!-- ${_generateLicense('')} -->', header: r'(<\?xml version="1.0" encoding="utf-8"\?>\n)?');
await _verifyNoMissingLicenseForExtension(workingDirectory, 'frag', overrideMinimumMatches ?? 1, _generateLicense('// '), header: r'#version 320 es(\n)+');
}
Future<int> _verifyNoMissingLicenseForExtension(
Future<void> _verifyNoMissingLicenseForExtension(
String workingDirectory,
String extension,
int minimumMatches,
@ -592,10 +588,8 @@ Future<int> _verifyNoMissingLicenseForExtension(
}
// Fail if any errors
if (errors.isNotEmpty) {
final String redLine = '$red━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━$reset';
final String fileDoes = errors.length == 1 ? 'file does' : '${errors.length} files do';
print(<String>[
redLine,
foundError(<String>[
'${bold}The following $fileDoes not have the right license header for $extension files:$reset',
...errors.map<String>((String error) => ' $error'),
'The expected license header is:',
@ -603,11 +597,8 @@ Future<int> _verifyNoMissingLicenseForExtension(
if (header.isNotEmpty) 'followed by the following license text:',
license,
if (trailingBlank) '...followed by a blank line.',
redLine,
].join('\n'));
return 1;
]);
}
return 0;
}
class _Line {
@ -1650,7 +1641,7 @@ Future<EvalResult> _evalCommand(String executable, List<String> arguments, {
final String relativeWorkingDir = path.relative(workingDirectory);
if (!runSilently) {
printProgress('RUNNING', relativeWorkingDir, commandDescription);
print('RUNNING: cd $cyan$relativeWorkingDir$reset; $green$commandDescription$reset');
}
final Stopwatch time = Stopwatch()..start();
@ -1669,12 +1660,12 @@ Future<EvalResult> _evalCommand(String executable, List<String> arguments, {
);
if (!runSilently) {
print('$clock ELAPSED TIME: $bold${prettyPrintDuration(time.elapsed)}$reset for $commandDescription in $relativeWorkingDir');
print('ELAPSED TIME: $bold${prettyPrintDuration(time.elapsed)}$reset for $commandDescription in $relativeWorkingDir');
}
if (exitCode != 0 && !allowNonZeroExit) {
stderr.write(result.stderr);
foundError(<String>[
result.stderr,
'${bold}ERROR:$red Last command exited with $exitCode.$reset',
'${bold}Command:$red $commandDescription$reset',
'${bold}Relative working directory:$red $relativeWorkingDir$reset',
@ -1838,9 +1829,8 @@ const Set<String> kExecutableAllowlist = <String>{
Future<void> _checkForNewExecutables() async {
// 0b001001001
const int executableBitMask = 0x49;
final List<File> files = await _gitFiles(flutterRoot);
int unexpectedExecutableCount = 0;
final List<String> errors = <String>[];
for (final File file in files) {
final String relativePath = path.relative(
file.path,
@ -1849,14 +1839,14 @@ Future<void> _checkForNewExecutables() async {
final FileStat stat = file.statSync();
final bool isExecutable = stat.mode & executableBitMask != 0x0;
if (isExecutable && !kExecutableAllowlist.contains(relativePath)) {
unexpectedExecutableCount += 1;
print('$relativePath is executable: ${(stat.mode & 0x1FF).toRadixString(2)}');
errors.add('$relativePath is executable: ${(stat.mode & 0x1FF).toRadixString(2)}');
}
}
if (unexpectedExecutableCount > 0) {
if (errors.isNotEmpty) {
throw Exception(
'found $unexpectedExecutableCount unexpected executable file'
'${unexpectedExecutableCount == 1 ? '' : 's'}! If this was intended, you '
'${errors.join('\n')}\n'
'found ${errors.length} unexpected executable file'
'${errors.length == 1 ? '' : 's'}! If this was intended, you '
'must add this file to kExecutableAllowlist in dev/bots/analyze.dart',
);
}

270
dev/bots/flutter_compact_formatter.dart
View file

@ -1,270 +0,0 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:convert';
import 'dart:io';
final Stopwatch _stopwatch = Stopwatch();
/// A wrapper around package:test's JSON reporter.
///
/// This class behaves similarly to the compact reporter, but suppresses all
/// output except for progress until the end of testing. In other words, errors,
/// [print] calls, and skipped test messages will not be printed during the run
/// of the suite.
///
/// It also processes the JSON data into a collection of [TestResult]s for any
/// other post processing needs, e.g. sending data to analytics.
class FlutterCompactFormatter {
FlutterCompactFormatter() {
_stopwatch.start();
}
/// Whether to use color escape codes in writing to stdout.
final bool useColor = stdout.supportsAnsiEscapes;
/// The terminal escape for green text, or the empty string if this is Windows
/// or not outputting to a terminal.
String get _green => useColor ? '\u001b[32m' : '';
/// The terminal escape for red text, or the empty string if this is Windows
/// or not outputting to a terminal.
String get _red => useColor ? '\u001b[31m' : '';
/// The terminal escape for yellow text, or the empty string if this is
/// Windows or not outputting to a terminal.
String get _yellow => useColor ? '\u001b[33m' : '';
/// The terminal escape for gray text, or the empty string if this is
/// Windows or not outputting to a terminal.
String get _gray => useColor ? '\u001b[1;30m' : '';
/// The terminal escape for bold text, or the empty string if this is
/// Windows or not outputting to a terminal.
String get _bold => useColor ? '\u001b[1m' : '';
/// The terminal escape for removing test coloring, or the empty string if
/// this is Windows or not outputting to a terminal.
String get _noColor => useColor ? '\u001b[0m' : '';
/// The terminal escape for clearing the line, or a carriage return if
/// this is Windows or not outputting to a terminal.
String get _clearLine => useColor ? '\x1b[2K\r' : '\r';
final Map<int, TestResult> _tests = <int, TestResult>{};
/// The test results from this run.
Iterable<TestResult> get tests => _tests.values;
/// The number of tests that were started.
int started = 0;
/// The number of test failures.
int failures = 0;
/// The number of skipped tests.
int skips = 0;
/// The number of successful tests.
int successes = 0;
/// Process a single line of JSON output from the JSON test reporter.
///
/// Callers are responsible for splitting multiple lines before calling this
/// method.
TestResult? processRawOutput(String raw) {
assert(raw != null);
// We might be getting messages from Flutter Tool about updating/building.
if (!raw.startsWith('{')) {
print(raw);
return null;
}
final Map<String, dynamic> decoded = json.decode(raw) as Map<String, dynamic>;
final TestResult? originalResult = _tests[decoded['testID']];
switch (decoded['type'] as String) {
case 'done':
stdout.write(_clearLine);
stdout.write('$_bold${_stopwatch.elapsed}$_noColor ');
stdout.writeln(
'$_green+$successes $_yellow~$skips $_red-$failures:$_bold$_gray Done.$_noColor');
break;
case 'testStart':
final Map<String, dynamic> testData = decoded['test'] as Map<String, dynamic>;
if (testData['url'] == null) {
started += 1;
stdout.write(_clearLine);
stdout.write('$_bold${_stopwatch.elapsed}$_noColor ');
stdout.write(
'$_green+$successes $_yellow~$skips $_red-$failures: $_gray${testData['name']}$_noColor');
break;
}
_tests[testData['id'] as int] = TestResult(
id: testData['id'] as int,
name: testData['name'] as String,
line: testData['root_line'] as int? ?? testData['line'] as int,
column: testData['root_column'] as int? ?? testData['column'] as int,
path: testData['root_url'] as String? ?? testData['url'] as String,
startTime: decoded['time'] as int,
);
break;
case 'testDone':
if (originalResult == null) {
break;
}
originalResult.endTime = decoded['time'] as int;
if (decoded['skipped'] == true) {
skips += 1;
originalResult.status = TestStatus.skipped;
} else {
if (decoded['result'] == 'success') {
originalResult.status =TestStatus.succeeded;
successes += 1;
} else {
originalResult.status = TestStatus.failed;
failures += 1;
}
}
break;
case 'error':
final String error = decoded['error'] as String;
final String stackTrace = decoded['stackTrace'] as String;
if (originalResult != null) {
originalResult.errorMessage = error;
originalResult.stackTrace = stackTrace;
} else {
if (error != null) {
stderr.writeln(error);
}
if (stackTrace != null) {
stderr.writeln(stackTrace);
}
}
break;
case 'print':
if (originalResult != null) {
originalResult.messages.add(decoded['message'] as String);
}
break;
case 'group':
case 'allSuites':
case 'start':
case 'suite':
default:
break;
}
return originalResult;
}
/// Print summary of test results.
void finish() {
final List<String> skipped = <String>[];
final List<String> failed = <String>[];
for (final TestResult result in _tests.values) {
switch (result.status) {
case TestStatus.started:
failed.add('${_red}Unexpectedly failed to complete a test!');
failed.add(result.toString() + _noColor);
break;
case TestStatus.skipped:
skipped.add(
'${_yellow}Skipped ${result.name} (${result.pathLineColumn}).$_noColor');
break;
case TestStatus.failed:
failed.addAll(<String>[
'$_bold${_red}Failed ${result.name} (${result.pathLineColumn}):',
result.errorMessage!,
_noColor + _red,
result.stackTrace!,
]);
failed.addAll(result.messages);
failed.add(_noColor);
break;
case TestStatus.succeeded:
break;
}
}
skipped.forEach(print);
failed.forEach(print);
if (failed.isEmpty) {
print('${_green}Completed, $successes test(s) passing ($skips skipped).$_noColor');
} else {
print('$_gray$failures test(s) failed.$_noColor');
}
}
}
/// The state of a test received from the JSON reporter.
enum TestStatus {
/// Test execution has started.
started,
/// Test completed successfully.
succeeded,
/// Test failed.
failed,
/// Test was skipped.
skipped,
}
/// The detailed status of a test run.
class TestResult {
TestResult({
required this.id,
required this.name,
required this.line,
required this.column,
required this.path,
required this.startTime,
this.status = TestStatus.started,
}) : assert(id != null),
assert(name != null),
assert(line != null),
assert(column != null),
assert(path != null),
assert(startTime != null),
assert(status != null),
messages = <String>[];
/// The state of the test.
TestStatus status;
/// The internal ID of the test used by the JSON reporter.
final int id;
/// The name of the test, specified via the `test` method.
final String name;
/// The line number from the original file.
final int line;
/// The column from the original file.
final int column;
/// The path of the original test file.
final String path;
/// A friendly print out of the [path], [line], and [column] of the test.
String get pathLineColumn => '$path:$line:$column';
/// The start time of the test, in milliseconds relative to suite startup.
final int startTime;
/// The stdout of the test.
final List<String> messages;
/// The error message from the test, from an `expect`, an [Exception] or
/// [Error].
String? errorMessage;
/// The stacktrace from a test failure.
String? stackTrace;
/// The time, in milliseconds relative to suite startup, that the test ended.
int? endTime;
/// The total time, in milliseconds, that the test took.
int get totalTime => (endTime ?? _stopwatch.elapsedMilliseconds) - startTime;
@override
String toString() => '{$runtimeType: {$id, $name, ${totalTime}ms, $pathLineColumn}}';
}

109
dev/bots/run_command.dart
View file

@ -50,23 +50,9 @@ class Command {
/// The raw process that was launched for this command.
final io.Process process;
final Stopwatch _time;
final Future<List<List<int>>>? _savedStdout;
final Future<List<List<int>>>? _savedStderr;
/// Evaluates when the [process] exits.
///
/// Returns the result of running the command.
Future<CommandResult> get onExit async {
final int exitCode = await process.exitCode;
_time.stop();
// Saved output is null when OutputMode.print is used.
final String? flattenedStdout = _savedStdout != null ? _flattenToString((await _savedStdout)!) : null;
final String? flattenedStderr = _savedStderr != null ? _flattenToString((await _savedStderr)!) : null;
return CommandResult._(exitCode, _time.elapsed, flattenedStdout, flattenedStderr);
}
final Future<String> _savedStdout;
final Future<String> _savedStderr;
}
/// The result of running a command using [startCommand] and [runCommand];
@ -105,46 +91,50 @@ Future<Command> startCommand(String executable, List<String> arguments, {
}) async {
final String commandDescription = '${path.relative(executable, from: workingDirectory)} ${arguments.join(' ')}';
final String relativeWorkingDir = path.relative(workingDirectory ?? io.Directory.current.path);
printProgress('RUNNING', relativeWorkingDir, commandDescription);
print('RUNNING: cd $cyan$relativeWorkingDir$reset; $green$commandDescription$reset');
final Stopwatch time = Stopwatch()..start();
final io.Process process = await io.Process.start(executable, arguments,
workingDirectory: workingDirectory,
environment: environment,
);
Future<List<List<int>>> savedStdout = Future<List<List<int>>>.value(<List<int>>[]);
Future<List<List<int>>> savedStderr = Future<List<List<int>>>.value(<List<int>>[]);
final Stream<List<int>> stdoutSource = process.stdout
.transform<String>(const Utf8Decoder())
.transform(const LineSplitter())
.where((String line) => removeLine == null || !removeLine(line))
.map((String line) {
final String formattedLine = '$line\n';
if (outputListener != null) {
outputListener(formattedLine, process);
}
return formattedLine;
})
.transform(const Utf8Encoder());
switch (outputMode) {
case OutputMode.print:
stdoutSource.listen((List<int> output) {
io.stdout.add(output);
savedStdout.then((List<List<int>> list) => list.add(output));
});
process.stderr.listen((List<int> output) {
io.stdout.add(output);
savedStdout.then((List<List<int>> list) => list.add(output));
});
break;
case OutputMode.capture:
savedStdout = stdoutSource.toList();
savedStderr = process.stderr.toList();
break;
}
return Command._(process, time, savedStdout, savedStderr);
return Command._(
process,
time,
process.stdout
.transform<String>(const Utf8Decoder())
.transform(const LineSplitter())
.where((String line) => removeLine == null || !removeLine(line))
.map<String>((String line) {
final String formattedLine = '$line\n';
if (outputListener != null) {
outputListener(formattedLine, process);
}
switch (outputMode) {
case OutputMode.print:
print(line);
break;
case OutputMode.capture:
break;
}
return line;
})
.join('\n'),
process.stderr
.transform<String>(const Utf8Decoder())
.transform(const LineSplitter())
.map<String>((String line) {
switch (outputMode) {
case OutputMode.print:
print(line);
break;
case OutputMode.capture:
break;
}
return line;
})
.join('\n'),
);
}
/// Runs the `executable` and waits until the process exits.
@ -182,7 +172,12 @@ Future<CommandResult> runCommand(String executable, List<String> arguments, {
outputListener: outputListener,
);
final CommandResult result = await command.onExit;
final CommandResult result = CommandResult._(
await command.process.exitCode,
command._time.elapsed,
await command._savedStdout,
await command._savedStderr,
);
if ((result.exitCode == 0) == expectNonZeroExit || (expectedExitCode != null && result.exitCode != expectedExitCode)) {
// Print the output when we get unexpected results (unless output was
@ -191,28 +186,24 @@ Future<CommandResult> runCommand(String executable, List<String> arguments, {
case OutputMode.print:
break;
case OutputMode.capture:
io.stdout.writeln(result.flattenedStdout);
io.stdout.writeln(result.flattenedStderr);
print(result.flattenedStdout);
print(result.flattenedStderr);
break;
}
foundError(<String>[
if (failureMessage != null)
failureMessage
else
'${bold}ERROR: ${red}Last command exited with ${result.exitCode} (expected: ${expectNonZeroExit ? (expectedExitCode ?? 'non-zero') : 'zero'}).$reset',
'$bold${red}Command exited with exit code ${result.exitCode} but expected: ${expectNonZeroExit ? (expectedExitCode ?? 'non-zero') : 'zero'} exit code.$reset',
'${bold}Command: $green$commandDescription$reset',
'${bold}Relative working directory: $cyan$relativeWorkingDir$reset',
]);
} else {
print('$clock ELAPSED TIME: ${prettyPrintDuration(result.elapsedTime)} for $green$commandDescription$reset in $cyan$relativeWorkingDir$reset');
print('ELAPSED TIME: ${prettyPrintDuration(result.elapsedTime)} for $green$commandDescription$reset in $cyan$relativeWorkingDir$reset');
}
return result;
}
/// Flattens a nested list of UTF-8 code units into a single string.
String _flattenToString(List<List<int>> chunks) =>
utf8.decode(chunks.expand<int>((List<int> ints) => ints).toList());
/// Specifies what to do with the command output from [runCommand] and [startCommand].
enum OutputMode {
/// Forwards standard output and standard error streams to the test process'

24
dev/bots/service_worker_test.dart
View file

@ -163,8 +163,8 @@ Future<void> runWebServiceWorkerTest({
Future<void> startAppServer({
required String cacheControl,
}) async {
final int serverPort = await findAvailablePort();
final int browserDebugPort = await findAvailablePort();
final int serverPort = await findAvailablePortAndPossiblyCauseFlakyTests();
final int browserDebugPort = await findAvailablePortAndPossiblyCauseFlakyTests();
server = await AppServer.start(
headless: headless,
cacheControl: cacheControl,
@ -201,7 +201,7 @@ Future<void> runWebServiceWorkerTest({
final bool shouldExpectFlutterJs = testType != ServiceWorkerTestType.withoutFlutterJs;
print('BEGIN runWebServiceWorkerTest(headless: $headless, testType: $testType)\n');
print('BEGIN runWebServiceWorkerTest(headless: $headless, testType: $testType)');
try {
/////
@ -417,7 +417,7 @@ Future<void> runWebServiceWorkerTest({
await server?.stop();
}
print('END runWebServiceWorkerTest(headless: $headless, testType: $testType)\n');
print('END runWebServiceWorkerTest(headless: $headless, testType: $testType)');
}
Future<void> runWebServiceWorkerTestWithCachingResources({
@ -435,8 +435,8 @@ Future<void> runWebServiceWorkerTestWithCachingResources({
Future<void> startAppServer({
required String cacheControl,
}) async {
final int serverPort = await findAvailablePort();
final int browserDebugPort = await findAvailablePort();
final int serverPort = await findAvailablePortAndPossiblyCauseFlakyTests();
final int browserDebugPort = await findAvailablePortAndPossiblyCauseFlakyTests();
server = await AppServer.start(
headless: headless,
cacheControl: cacheControl,
@ -472,7 +472,7 @@ Future<void> runWebServiceWorkerTestWithCachingResources({
final bool shouldExpectFlutterJs = testType != ServiceWorkerTestType.withoutFlutterJs;
print('BEGIN runWebServiceWorkerTestWithCachingResources(headless: $headless, testType: $testType)\n');
print('BEGIN runWebServiceWorkerTestWithCachingResources(headless: $headless, testType: $testType)');
try {
//////////////////////////////////////////////////////
@ -576,7 +576,7 @@ Future<void> runWebServiceWorkerTestWithCachingResources({
await server?.stop();
}
print('END runWebServiceWorkerTestWithCachingResources(headless: $headless, testType: $testType)\n');
print('END runWebServiceWorkerTestWithCachingResources(headless: $headless, testType: $testType)');
}
Future<void> runWebServiceWorkerTestWithBlockedServiceWorkers({
@ -593,8 +593,8 @@ Future<void> runWebServiceWorkerTestWithBlockedServiceWorkers({
Future<void> startAppServer({
required String cacheControl,
}) async {
final int serverPort = await findAvailablePort();
final int browserDebugPort = await findAvailablePort();
final int serverPort = await findAvailablePortAndPossiblyCauseFlakyTests();
final int browserDebugPort = await findAvailablePortAndPossiblyCauseFlakyTests();
server = await AppServer.start(
headless: headless,
cacheControl: cacheControl,
@ -628,7 +628,7 @@ Future<void> runWebServiceWorkerTestWithBlockedServiceWorkers({
workingDirectory: _testAppWebDirectory,
);
print('BEGIN runWebServiceWorkerTestWithBlockedServiceWorkers(headless: $headless)\n');
print('BEGIN runWebServiceWorkerTestWithBlockedServiceWorkers(headless: $headless)');
try {
await _rebuildApp(version: 1, testType: ServiceWorkerTestType.blockedServiceWorkers, target: _targetWithBlockedServiceWorkers);
@ -662,5 +662,5 @@ Future<void> runWebServiceWorkerTestWithBlockedServiceWorkers({
);
await server?.stop();
}
print('END runWebServiceWorkerTestWithBlockedServiceWorkers(headless: $headless)\n');
print('END runWebServiceWorkerTestWithBlockedServiceWorkers(headless: $headless)');
}

433
dev/bots/test.dart
View file

@ -2,9 +2,55 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
// Runs the tests for the flutter/flutter repository.
//
//
// By default, test output is filtered and only errors are shown. (If a
// particular test takes longer than _quietTimeout in utils.dart, the output is
// shown then also, in case something has hung.)
//
// --verbose stops the output cleanup and just outputs everything verbatim.
//
//
// By default, errors are non-fatal; all tests are executed and the output
// ends with a summary of the errors that were detected.
//
// Exit code is 1 if there was an error.
//
// --abort-on-error causes the script to exit immediately when hitting an error.
//
//
// By default, all tests are run. However, the tests support being split by
// shard and subshard. (Inspect the code to see what shards and subshards are
// supported.)
//
// If the CIRRUS_TASK_NAME environment variable exists, it is used to determine
// the shard and sub-shard, by parsing it in the form shard-subshard-platform,
// ignoring the platform.
//
// For local testing you can just set the SHARD and SUBSHARD environment
// variables. For example, to run all the framework tests you can just set
// SHARD=framework_tests. Some shards support named subshards, like
// SHARD=framework_tests SUBSHARD=widgets. Others support arbitrary numbered
// subsharding, like SHARD=build_tests SUBSHARD=1_2 (where 1_2 means "one of
// two" as in run the first half of the tests).
//
// So for example to run specifically the third subshard of the Web tests you
// would set SHARD=web_tests SUBSHARD=2 (it's zero-based).
//
// By default, where supported, tests within a shard are executed in a random
// order to (eventually) catch inter-test dependencies.
//
// --test-randomize-ordering-seed=<n> sets the shuffle seed for reproducing runs.
//
//
// All other arguments are treated as arguments to pass to the flutter tool when
// running tests.
import 'dart:convert';
import 'dart:core' as system show print;
import 'dart:core' hide print;
import 'dart:io' as system show exit;
import 'dart:io' hide exit;
import 'dart:math' as math;
import 'dart:typed_data';
@ -15,7 +61,6 @@ import 'package:file/local.dart';
import 'package:path/path.dart' as path;
import 'browser.dart';
import 'flutter_compact_formatter.dart';
import 'run_command.dart';
import 'service_worker_test.dart';
import 'utils.dart';
@ -61,8 +106,6 @@ final List<String> flutterTestArgs = <String>[];
/// if such flags are provided to `test.dart`.
final Map<String,String> localEngineEnv = <String, String>{};
final bool useFlutterTestFormatter = Platform.environment['FLUTTER_TEST_FORMATTER'] == 'true';
const String kShardKey = 'SHARD';
const String kSubshardKey = 'SUBSHARD';
@ -170,55 +213,68 @@ String get shuffleSeed {
/// SHARD=tool_tests bin/cache/dart-sdk/bin/dart dev/bots/test.dart
/// bin/cache/dart-sdk/bin/dart dev/bots/test.dart --local-engine=host_debug_unopt
Future<void> main(List<String> args) async {
print('$clock STARTING ANALYSIS');
flutterTestArgs.addAll(args);
final Set<String> removeArgs = <String>{};
for (final String arg in args) {
if (arg.startsWith('--local-engine=')) {
localEngineEnv['FLUTTER_LOCAL_ENGINE'] = arg.substring('--local-engine='.length);
try {
printProgress('STARTING ANALYSIS');
for (final String arg in args) {
if (arg.startsWith('--local-engine=')) {
localEngineEnv['FLUTTER_LOCAL_ENGINE'] = arg.substring('--local-engine='.length);
flutterTestArgs.add(arg);
} else if (arg.startsWith('--local-engine-src-path=')) {
localEngineEnv['FLUTTER_LOCAL_ENGINE_SRC_PATH'] = arg.substring('--local-engine-src-path='.length);
flutterTestArgs.add(arg);
} else if (arg.startsWith('--test-randomize-ordering-seed=')) {
_shuffleSeed = arg.substring('--test-randomize-ordering-seed='.length);
} else if (arg.startsWith('--verbose')) {
print = (Object? message) {
system.print(message);
};
} else if (arg.startsWith('--abort-on-error')) {
onError = () {
system.exit(1);
};
} else {
flutterTestArgs.add(arg);
}
}
if (arg.startsWith('--local-engine-src-path=')) {
localEngineEnv['FLUTTER_LOCAL_ENGINE_SRC_PATH'] = arg.substring('--local-engine-src-path='.length);
}
if (arg.startsWith('--test-randomize-ordering-seed=')) {
_shuffleSeed = arg.substring('--test-randomize-ordering-seed='.length);
removeArgs.add(arg);
}
if (arg == '--no-smoke-tests') {
// This flag is deprecated, ignore it.
removeArgs.add(arg);
if (Platform.environment.containsKey(CIRRUS_TASK_NAME)) {
printProgress('Running task: ${Platform.environment[CIRRUS_TASK_NAME]}');
}
await selectShard(<String, ShardRunner>{
'add_to_app_life_cycle_tests': _runAddToAppLifeCycleTests,
'build_tests': _runBuildTests,
'framework_coverage': _runFrameworkCoverage,
'framework_tests': _runFrameworkTests,
'tool_tests': _runToolTests,
// web_tool_tests is also used by HHH: https://dart.googlesource.com/recipes/+/refs/heads/master/recipes/dart/flutter_engine.py
'web_tool_tests': _runWebToolTests,
'tool_integration_tests': _runIntegrationToolTests,
'tool_host_cross_arch_tests': _runToolHostCrossArchTests,
// All the unit/widget tests run using `flutter test --platform=chrome --web-renderer=html`
'web_tests': _runWebHtmlUnitTests,
// All the unit/widget tests run using `flutter test --platform=chrome --web-renderer=canvaskit`
'web_canvaskit_tests': _runWebCanvasKitUnitTests,
// All web integration tests
'web_long_running_tests': _runWebLongRunningTests,
'flutter_plugins': _runFlutterPluginsTests,
'skp_generator': _runSkpGeneratorTests,
kTestHarnessShardName: _runTestHarnessTests, // Used for testing this script; also run as part of SHARD=framework_tests, SUBSHARD=misc.
});
} catch (error, stackTrace) {
foundError(<String>[
'UNEXPECTED ERROR!',
error.toString(),
...stackTrace.toString().split('\n'),
'The test.dart script should be corrected to catch this error and call foundError().',
'${yellow}Some tests are likely to have been skipped.$reset',
]);
system.exit(255);
}
flutterTestArgs.removeWhere((String arg) => removeArgs.contains(arg));
if (Platform.environment.containsKey(CIRRUS_TASK_NAME)) {
print('Running task: ${Platform.environment[CIRRUS_TASK_NAME]}');
}
print('═' * 80);
await selectShard(<String, ShardRunner>{
'add_to_app_life_cycle_tests': _runAddToAppLifeCycleTests,
'build_tests': _runBuildTests,
'framework_coverage': _runFrameworkCoverage,
'framework_tests': _runFrameworkTests,
'tool_tests': _runToolTests,
// web_tool_tests is also used by HHH: https://dart.googlesource.com/recipes/+/refs/heads/master/recipes/dart/flutter_engine.py
'web_tool_tests': _runWebToolTests,
'tool_integration_tests': _runIntegrationToolTests,
'tool_host_cross_arch_tests': _runToolHostCrossArchTests,
// All the unit/widget tests run using `flutter test --platform=chrome --web-renderer=html`
'web_tests': _runWebHtmlUnitTests,
// All the unit/widget tests run using `flutter test --platform=chrome --web-renderer=canvaskit`
'web_canvaskit_tests': _runWebCanvasKitUnitTests,
// All web integration tests
'web_long_running_tests': _runWebLongRunningTests,
'flutter_plugins': _runFlutterPluginsTests,
'skp_generator': _runSkpGeneratorTests,
kTestHarnessShardName: _runTestHarnessTests, // Used for testing this script.
});
if (hasError) {
print('$clock ${bold}Test failed.$reset');
printProgress('${bold}Test failed.$reset');
reportErrorsAndExit();
}
print('$clock ${bold}Test successful.$reset');
printProgress('${bold}Test successful.$reset');
system.exit(0);
}
final String _luciBotId = Platform.environment['SWARMING_BOT_ID'] ?? '';
@ -232,22 +288,36 @@ Future<void> _validateEngineHash() async {
// and then use this script to run Flutter's test suites.
// Because the artifacts have been changed, this particular test will return
// a false positive and should be skipped.
print('${yellow}Skipping Flutter Engine Version Validation for swarming '
'bot $_luciBotId.');
print('${yellow}Skipping Flutter Engine Version Validation for swarming bot $_luciBotId.');
return;
}
final String expectedVersion = File(engineVersionFile).readAsStringSync().trim();
final CommandResult result = await runCommand(flutterTester, <String>['--help'], outputMode: OutputMode.capture);
final String actualVersion = result.flattenedStderr!.split('\n').firstWhere((final String line) {
return line.startsWith('Flutter Engine Version:');
});
if (result.flattenedStdout!.isNotEmpty) {
foundError(<String>[
'${red}The stdout of `$flutterTester --help` was not empty:$reset',
...result.flattenedStdout!.split('\n').map((String line) => ' $gray┆$reset $line'),
]);
}
final String actualVersion;
try {
actualVersion = result.flattenedStderr!.split('\n').firstWhere((final String line) {
return line.startsWith('Flutter Engine Version:');
});
} on StateError {
foundError(<String>[
'${red}Could not find "Flutter Engine Version:" line in `${path.basename(flutterTester)} --help` stderr output:$reset',
...result.flattenedStderr!.split('\n').map((String line) => ' $gray┆$reset $line'),
]);
return;
}
if (!actualVersion.contains(expectedVersion)) {
foundError(<String>['${red}Expected "Flutter Engine Version: $expectedVersion", but found "$actualVersion".']);
foundError(<String>['${red}Expected "Flutter Engine Version: $expectedVersion", but found "$actualVersion".$reset']);
}
}
Future<void> _runTestHarnessTests() async {
print('${green}Running test harness tests...$reset');
printProgress('${green}Running test harness tests...$reset');
await _validateEngineHash();
@ -338,7 +408,7 @@ Future<void> _runTestHarnessTests() async {
final String _toolsPath = path.join(flutterRoot, 'packages', 'flutter_tools');
Future<void> _runGeneralToolTests() async {
await _dartRunTest(
await _runDartTest(
_toolsPath,
testPaths: <String>[path.join('test', 'general.shard')],
enableFlutterToolAsserts: false,
@ -351,7 +421,7 @@ Future<void> _runGeneralToolTests() async {
}
Future<void> _runCommandsToolTests() async {
await _dartRunTest(
await _runDartTest(
_toolsPath,
forceSingleCore: true,
testPaths: <String>[path.join('test', 'commands.shard')],
@ -359,7 +429,7 @@ Future<void> _runCommandsToolTests() async {
}
Future<void> _runWebToolTests() async {
await _dartRunTest(
await _runDartTest(
_toolsPath,
forceSingleCore: true,
testPaths: <String>[path.join('test', 'web.shard')],
@ -368,7 +438,7 @@ Future<void> _runWebToolTests() async {
}
Future<void> _runToolHostCrossArchTests() {
return _dartRunTest(
return _runDartTest(
_toolsPath,
// These are integration tests
forceSingleCore: true,
@ -382,7 +452,7 @@ Future<void> _runIntegrationToolTests() async {
.map<String>((FileSystemEntity entry) => path.relative(entry.path, from: _toolsPath))
.where((String testPath) => path.basename(testPath).endsWith('_test.dart')).toList();
await _dartRunTest(
await _runDartTest(
_toolsPath,
forceSingleCore: true,
testPaths: _selectIndexOfTotalSubshard<String>(allTests),
@ -545,7 +615,7 @@ Future<void> _flutterBuildApk(String relativePathToApplication, {
bool verifyCaching = false,
List<String> additionalArgs = const <String>[],
}) async {
print('${green}Testing APK build$reset for $cyan$relativePathToApplication$reset...');
printProgress('${green}Testing APK ${release ? 'release' : 'debug'} build$reset for $cyan$relativePathToApplication$reset...');
await _flutterBuild(relativePathToApplication, 'APK', 'apk',
release: release,
verifyCaching: verifyCaching,
@ -559,7 +629,7 @@ Future<void> _flutterBuildIpa(String relativePathToApplication, {
bool verifyCaching = false,
}) async {
assert(Platform.isMacOS);
print('${green}Testing IPA build$reset for $cyan$relativePathToApplication$reset...');
printProgress('${green}Testing IPA ${release ? 'release' : 'debug'} build$reset for $cyan$relativePathToApplication$reset...');
await _flutterBuild(relativePathToApplication, 'IPA', 'ios',
release: release,
verifyCaching: verifyCaching,
@ -574,7 +644,7 @@ Future<void> _flutterBuildLinux(String relativePathToApplication, {
}) async {
assert(Platform.isLinux);
await runCommand(flutter, <String>['config', '--enable-linux-desktop']);
print('${green}Testing Linux build$reset for $cyan$relativePathToApplication$reset...');
printProgress('${green}Testing Linux ${release ? 'release' : 'debug'} build$reset for $cyan$relativePathToApplication$reset...');
await _flutterBuild(relativePathToApplication, 'Linux', 'linux',
release: release,
verifyCaching: verifyCaching,
@ -589,7 +659,7 @@ Future<void> _flutterBuildMacOS(String relativePathToApplication, {
}) async {
assert(Platform.isMacOS);
await runCommand(flutter, <String>['config', '--enable-macos-desktop']);
print('${green}Testing macOS build$reset for $cyan$relativePathToApplication$reset...');
printProgress('${green}Testing macOS ${release ? 'release' : 'debug'} build$reset for $cyan$relativePathToApplication$reset...');
await _flutterBuild(relativePathToApplication, 'macOS', 'macos',
release: release,
verifyCaching: verifyCaching,
@ -603,7 +673,7 @@ Future<void> _flutterBuildWin32(String relativePathToApplication, {
List<String> additionalArgs = const <String>[],
}) async {
assert(Platform.isWindows);
print('${green}Testing Windows build$reset for $cyan$relativePathToApplication$reset...');
printProgress('${green}Testing ${release ? 'release' : 'debug'} Windows build$reset for $cyan$relativePathToApplication$reset...');
await _flutterBuild(relativePathToApplication, 'Windows', 'windows',
release: release,
verifyCaching: verifyCaching,
@ -634,7 +704,7 @@ Future<void> _flutterBuild(
);
if (verifyCaching) {
print('${green}Testing $platformLabel cache$reset for $cyan$relativePathToApplication$reset...');
printProgress('${green}Testing $platformLabel cache$reset for $cyan$relativePathToApplication$reset...');
await runCommand(flutter,
<String>[
'build',
@ -668,7 +738,7 @@ bool _allTargetsCached(File performanceFile) {
}
Future<void> _flutterBuildDart2js(String relativePathToApplication, String target, { bool expectNonZeroExit = false }) async {
print('${green}Testing Dart2JS build$reset for $cyan$relativePathToApplication$reset...');
printProgress('${green}Testing Dart2JS build$reset for $cyan$relativePathToApplication$reset...');
await runCommand(flutter,
<String>['build', 'web', '-v', '--target=$target'],
workingDirectory: path.join(flutterRoot, relativePathToApplication),
@ -681,12 +751,14 @@ Future<void> _flutterBuildDart2js(String relativePathToApplication, String targe
Future<void> _runAddToAppLifeCycleTests() async {
if (Platform.isMacOS) {
print('${green}Running add-to-app life cycle iOS integration tests$reset...');
printProgress('${green}Running add-to-app life cycle iOS integration tests$reset...');
final String addToAppDir = path.join(flutterRoot, 'dev', 'integration_tests', 'ios_add2app_life_cycle');
await runCommand('./build_and_test.sh',
<String>[],
workingDirectory: addToAppDir,
);
} else {
printProgress('${yellow}Skipped on this platform (only iOS has add-to-add lifecycle tests at this time).$reset');
}
}
@ -696,7 +768,7 @@ Future<void> _runFrameworkTests() async {
final List<String> trackWidgetCreationAlternatives = <String>['--track-widget-creation', '--no-track-widget-creation'];
Future<void> runWidgets() async {
print('${green}Running packages/flutter tests for$reset: ${cyan}test/widgets/$reset');
printProgress('${green}Running packages/flutter tests $reset for ${cyan}test/widgets/$reset');
for (final String trackWidgetCreationOption in trackWidgetCreationAlternatives) {
await _runFlutterTest(
path.join(flutterRoot, 'packages', 'flutter'),
@ -733,7 +805,7 @@ Future<void> _runFrameworkTests() async {
.where((Directory dir) => dir.path.endsWith('widgets') == false)
.map<String>((Directory dir) => path.join('test', path.basename(dir.path)) + path.separator)
.toList();
print('${green}Running packages/flutter tests$reset for: $cyan${tests.join(", ")}$reset');
printProgress('${green}Running packages/flutter tests$reset for $cyan${tests.join(", ")}$reset');
for (final String trackWidgetCreationOption in trackWidgetCreationAlternatives) {
await _runFlutterTest(
path.join(flutterRoot, 'packages', 'flutter'),
@ -774,30 +846,37 @@ Future<void> _runFrameworkTests() async {
required Set<String> allowed,
required Set<String> disallowed,
}) async {
await runCommand(
flutter,
<String>[
'build', 'appbundle', '--$modeArgument', path.join('lib', sourceFile),
],
workingDirectory: tracingDirectory,
);
final Archive archive = ZipDecoder().decodeBytes(File(path.join(tracingDirectory, 'build', 'app', 'outputs', 'bundle', modeArgument, 'app-$modeArgument.aab')).readAsBytesSync());
final ArchiveFile libapp = archive.findFile('base/lib/arm64-v8a/libapp.so')!;
final Uint8List libappBytes = libapp.content as Uint8List; // bytes decompressed here
final String libappStrings = utf8.decode(libappBytes, allowMalformed: true);
await runCommand(flutter, <String>['clean'], workingDirectory: tracingDirectory);
final List<String> results = <String>[];
for (final String pattern in allowed) {
if (!libappStrings.contains(pattern)) {
results.add('When building with --$modeArgument, expected to find "$pattern" in libapp.so but could not find it.');
try {
await runCommand(
flutter,
<String>[
'build', 'appbundle', '--$modeArgument', path.join('lib', sourceFile),
],
workingDirectory: tracingDirectory,
);
final Archive archive = ZipDecoder().decodeBytes(File(path.join(tracingDirectory, 'build', 'app', 'outputs', 'bundle', modeArgument, 'app-$modeArgument.aab')).readAsBytesSync());
final ArchiveFile libapp = archive.findFile('base/lib/arm64-v8a/libapp.so')!;
final Uint8List libappBytes = libapp.content as Uint8List; // bytes decompressed here
final String libappStrings = utf8.decode(libappBytes, allowMalformed: true);
await runCommand(flutter, <String>['clean'], workingDirectory: tracingDirectory);
final List<String> results = <String>[];
for (final String pattern in allowed) {
if (!libappStrings.contains(pattern)) {
results.add('When building with --$modeArgument, expected to find "$pattern" in libapp.so but could not find it.');
}
}
}
for (final String pattern in disallowed) {
if (libappStrings.contains(pattern)) {
results.add('When building with --$modeArgument, expected to not find "$pattern" in libapp.so but did find it.');
for (final String pattern in disallowed) {
if (libappStrings.contains(pattern)) {
results.add('When building with --$modeArgument, expected to not find "$pattern" in libapp.so but did find it.');
}
}
return results;
} catch (error, stackTrace) {
return <String>[
error.toString(),
...stackTrace.toString().trimRight().split('\n'),
];
}
return results;
}
final List<String> results = <String>[];
@ -877,12 +956,12 @@ Future<void> _runFrameworkTests() async {
}
Future<void> runMisc() async {
print('${green}Running package tests$reset for directories other than packages/flutter');
printProgress('${green}Running package tests$reset for directories other than packages/flutter');
await _runTestHarnessTests();
await runExampleTests();
await _dartRunTest(path.join(flutterRoot, 'dev', 'bots'));
await _dartRunTest(path.join(flutterRoot, 'dev', 'devicelab'), ensurePrecompiledTool: false); // See https://github.com/flutter/flutter/issues/86209
await _dartRunTest(path.join(flutterRoot, 'dev', 'conductor', 'core'), forceSingleCore: true);
await _runDartTest(path.join(flutterRoot, 'dev', 'bots'));
await _runDartTest(path.join(flutterRoot, 'dev', 'devicelab'), ensurePrecompiledTool: false); // See https://github.com/flutter/flutter/issues/86209
await _runDartTest(path.join(flutterRoot, 'dev', 'conductor', 'core'), forceSingleCore: true);
// TODO(gspencergoog): Remove the exception for fatalWarnings once https://github.com/flutter/flutter/pull/91127 has landed.
await _runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'android_semantics_testing'), fatalWarnings: false);
await _runFlutterTest(path.join(flutterRoot, 'dev', 'manual_tests'));
@ -942,7 +1021,7 @@ Future<void> _runFrameworkCoverage() async {
if (!coverageFile.existsSync()) {
foundError(<String>[
'${red}Coverage file not found.$reset',
'Expected to find: $cyan${coverageFile.absolute}$reset',
'Expected to find: $cyan${coverageFile.absolute.path}$reset',
'This file is normally obtained by running `${green}flutter update-packages$reset`.',
]);
return;
@ -954,7 +1033,7 @@ Future<void> _runFrameworkCoverage() async {
if (!coverageFile.existsSync()) {
foundError(<String>[
'${red}Coverage file not found.$reset',
'Expected to find: $cyan${coverageFile.absolute}$reset',
'Expected to find: $cyan${coverageFile.absolute.path}$reset',
'This file should have been generated by the `${green}flutter test --coverage$reset` script, but was not.',
]);
return;
@ -1172,7 +1251,7 @@ Future<void> _runFlutterDriverWebTest({
bool expectFailure = false,
bool silenceBrowserOutput = false,
}) async {
print('${green}Running integration tests $target in $buildMode mode.$reset');
printProgress('${green}Running integration tests $target in $buildMode mode.$reset');
await runCommand(
flutter,
<String>[ 'clean' ],
@ -1206,7 +1285,6 @@ Future<void> _runFlutterDriverWebTest({
return false;
},
);
print('${green}Integration test passed.$reset');
}
// Compiles a sample web app and checks that its JS doesn't contain certain
@ -1287,7 +1365,7 @@ Future<String> getFlutterPluginsVersion({
/// Executes the test suite for the flutter/plugins repo.
Future<void> _runFlutterPluginsTests() async {
Future<void> runAnalyze() async {
print('${green}Running analysis for flutter/plugins$reset');
printProgress('${green}Running analysis for flutter/plugins$reset');
final Directory checkout = Directory.systemTemp.createTempSync('flutter_plugins.');
await runCommand(
'git',
@ -1348,7 +1426,7 @@ Future<void> _runFlutterPluginsTests() async {
///
/// Generated SKPs are ditched, this just verifies that it can run without failure.
Future<void> _runSkpGeneratorTests() async {
print('${green}Running skp_generator from flutter/tests$reset');
printProgress('${green}Running skp_generator from flutter/tests$reset');
final Directory checkout = Directory.systemTemp.createTempSync('flutter_skp_generator.');
await runCommand(
'git',
@ -1388,9 +1466,12 @@ Future<bool> _isChromeDriverRunning() async {
Future<void> _ensureChromeDriverIsRunning() async {
// If we cannot connect to ChromeDriver, assume it is not running. Launch it.
if (!await _isChromeDriverRunning()) {
print('Starting chromedriver');
printProgress('Starting chromedriver');
// Assume chromedriver is in the PATH.
_chromeDriver = await startCommand(
// TODO(ianh): this is the only remaining consumer of startCommand other than runCommand
// and it doesn't use most of startCommand's features; we could simplify this a lot by
// inlining the relevant parts of startCommand here.
'chromedriver',
<String>['--port=4444'],
);
@ -1430,7 +1511,7 @@ Future<void> _stopChromeDriver() async {
/// The test is written using `package:integration_test` (despite the "e2e" in
/// the name, which is there for historic reasons).
Future<void> _runGalleryE2eWebTest(String buildMode, { bool canvasKit = false }) async {
print('${green}Running flutter_gallery integration test in --$buildMode using ${canvasKit ? 'CanvasKit' : 'HTML'} renderer.$reset');
printProgress('${green}Running flutter_gallery integration test in --$buildMode using ${canvasKit ? 'CanvasKit' : 'HTML'} renderer.$reset');
final String testAppDirectory = path.join(flutterRoot, 'dev', 'integration_tests', 'flutter_gallery');
await runCommand(
flutter,
@ -1461,7 +1542,6 @@ Future<void> _runGalleryE2eWebTest(String buildMode, { bool canvasKit = false })
'FLUTTER_WEB': 'true',
},
);
print('${green}Integration test passed.$reset');
}
Future<void> _runWebStackTraceTest(String buildMode, String entrypoint) async {
@ -1490,8 +1570,8 @@ Future<void> _runWebStackTraceTest(String buildMode, String entrypoint) async {
);
// Run the app.
final int serverPort = await findAvailablePort();
final int browserDebugPort = await findAvailablePort();
final int serverPort = await findAvailablePortAndPossiblyCauseFlakyTests();
final int browserDebugPort = await findAvailablePortAndPossiblyCauseFlakyTests();
final String result = await evalTestAppInChrome(
appUrl: 'http://localhost:$serverPort/index.html',
appDirectory: appBuildDirectory,
@ -1538,8 +1618,8 @@ Future<void> _runWebReleaseTest(String target, {
);
// Run the app.
final int serverPort = await findAvailablePort();
final int browserDebugPort = await findAvailablePort();
final int serverPort = await findAvailablePortAndPossiblyCauseFlakyTests();
final int browserDebugPort = await findAvailablePortAndPossiblyCauseFlakyTests();
final String result = await evalTestAppInChrome(
appUrl: 'http://localhost:$serverPort/index.html',
appDirectory: appBuildDirectory,
@ -1636,7 +1716,7 @@ Future<void> _runFlutterWebTest(String webRenderer, String workingDirectory, Lis
// properly when overriding the local engine (for example, because some platform
// dependent targets are only built on some engines).
// See https://github.com/flutter/flutter/issues/72368
Future<void> _dartRunTest(String workingDirectory, {
Future<void> _runDartTest(String workingDirectory, {
List<String>? testPaths,
bool enableFlutterToolAsserts = true,
bool useBuildRunner = false,
@ -1671,10 +1751,6 @@ Future<void> _dartRunTest(String workingDirectory, {
'run',
'test',
if (shuffleTests) '--test-randomize-ordering-seed=$shuffleSeed',
if (useFlutterTestFormatter)
'-rjson'
else
'-rcompact',
'-j$cpus',
if (!hasColor)
'--no-color',
@ -1702,29 +1778,13 @@ Future<void> _dartRunTest(String workingDirectory, {
// the tool themselves.
await runCommand(flutter, <String>['--version'], environment: environment);
}
if (useFlutterTestFormatter) {
final FlutterCompactFormatter formatter = FlutterCompactFormatter();
Stream<String> testOutput;
try {
testOutput = runAndGetStdout(
dart,
args,
workingDirectory: workingDirectory,
environment: environment,
);
} finally {
formatter.finish();
}
await _processTestOutput(formatter, testOutput);
} else {
await runCommand(
dart,
args,
workingDirectory: workingDirectory,
environment: environment,
removeLine: useBuildRunner ? (String line) => line.startsWith('[INFO]') : null,
);
}
await runCommand(
dart,
args,
workingDirectory: workingDirectory,
environment: environment,
removeLine: useBuildRunner ? (String line) => line.startsWith('[INFO]') : null,
);
}
Future<void> _runFlutterTest(String workingDirectory, {
@ -1741,9 +1801,9 @@ Future<void> _runFlutterTest(String workingDirectory, {
assert(!printOutput || outputChecker == null, 'Output either can be printed or checked but not both');
final List<String> tags = <String>[];
// Recipe configured reduced test shards will only execute tests with the
// Recipe-configured reduced test shards will only execute tests with the
// appropriate tag.
if ((Platform.environment['REDUCED_TEST_SET'] ?? 'False') == 'True') {
if (Platform.environment['REDUCED_TEST_SET'] == 'True') {
tags.addAll(<String>['-t', 'reduced-test-set']);
}
@ -1756,11 +1816,6 @@ Future<void> _runFlutterTest(String workingDirectory, {
...flutterTestArgs,
];
final bool shouldProcessOutput = useFlutterTestFormatter && !expectFailure && !options.contains('--coverage');
if (shouldProcessOutput) {
args.add('--machine');
}
if (script != null) {
final String fullScriptPath = path.join(workingDirectory, script);
if (!FileSystemEntity.isFileSync(fullScriptPath)) {
@ -1778,51 +1833,24 @@ Future<void> _runFlutterTest(String workingDirectory, {
args.addAll(tests);
if (!shouldProcessOutput) {
final OutputMode outputMode = outputChecker == null && printOutput
? OutputMode.print
: OutputMode.capture;
final OutputMode outputMode = outputChecker == null && printOutput
? OutputMode.print
: OutputMode.capture;
final CommandResult result = await runCommand(
flutter,
args,
workingDirectory: workingDirectory,
expectNonZeroExit: expectFailure,
outputMode: outputMode,
environment: environment,
);
final CommandResult result = await runCommand(
flutter,
args,
workingDirectory: workingDirectory,
expectNonZeroExit: expectFailure,
outputMode: outputMode,
environment: environment,
);
if (outputChecker != null) {
final String? message = outputChecker(result);
if (message != null) {
foundError(<String>[message]);
}
if (outputChecker != null) {
final String? message = outputChecker(result);
if (message != null) {
foundError(<String>[message]);
}
return;
}
if (useFlutterTestFormatter) {
final FlutterCompactFormatter formatter = FlutterCompactFormatter();
Stream<String> testOutput;
try {
testOutput = runAndGetStdout(
flutter,
args,
workingDirectory: workingDirectory,
expectNonZeroExit: expectFailure,
environment: environment,
);
} finally {
formatter.finish();
}
await _processTestOutput(formatter, testOutput);
} else {
await runCommand(
flutter,
args,
workingDirectory: workingDirectory,
expectNonZeroExit: expectFailure,
);
}
}
@ -1843,19 +1871,6 @@ enum CiProviders {
luci,
}
Future<void> _processTestOutput(
FlutterCompactFormatter formatter,
Stream<String> testOutput,
) async {
final Timer heartbeat = Timer.periodic(const Duration(seconds: 30), (Timer timer) {
print('Processing...');
});
await testOutput.forEach(formatter.processRawOutput);
heartbeat.cancel();
formatter.finish();
}
CiProviders? get ciProvider {
if (Platform.environment['CIRRUS_CI'] == 'true') {
return CiProviders.cirrus;
@ -1874,10 +1889,10 @@ CiProviders? get ciProvider {
Future<String?> verifyVersion(File file) async {
final RegExp pattern = RegExp(
r'^(\d+)\.(\d+)\.(\d+)((-\d+\.\d+)?\.pre(\.\d+)?)?$');
final String version = await file.readAsString();
if (!file.existsSync()) {
return 'The version logic failed to create the Flutter version file.';
}
final String version = await file.readAsString();
if (version == '0.0.0-unknown') {
return 'The version logic failed to determine the Flutter version.';
}
@ -1904,7 +1919,7 @@ List<T> _selectIndexOfTotalSubshard<T>(List<T> tests, {String subshardKey = kSub
print('$kSubshardKey environment variable is missing, skipping sharding');
return tests;
}
print('$bold$subshardKey=$subshardName$reset');
printProgress('$bold$subshardKey=$subshardName$reset');
final RegExp pattern = RegExp(r'^(\d+)_(\d+)$');
final Match? match = pattern.firstMatch(subshardName);
@ -1928,7 +1943,7 @@ List<T> _selectIndexOfTotalSubshard<T>(List<T> tests, {String subshardKey = kSub
final int start = (index - 1) * testsPerShard;
final int end = math.min(index * testsPerShard, tests.length);
print('Selecting subshard $index of $total (range ${start + 1}-$end of ${tests.length})');
print('Selecting subshard $index of $total (tests ${start + 1}-$end of ${tests.length})');
return tests.sublist(start, end);
}
@ -1939,19 +1954,6 @@ Future<void> _runShardRunnerIndexOfTotalSubshard(List<ShardRunner> tests) async
}
}
/// If the CIRRUS_TASK_NAME environment variable exists, we use that to determine
/// the shard and sub-shard (parsing it in the form shard-subshard-platform, ignoring
/// the platform).
///
/// For local testing you can just set the SHARD and SUBSHARD
/// environment variables. For example, to run all the framework tests you can
/// just set SHARD=framework_tests. Some shards support named subshards, like
/// SHARD=framework_tests SUBSHARD=widgets. Others support arbitrary numbered
/// subsharding, like SHARD=build_tests SUBSHARD=1_2 (where 1_2 means "one of two"
/// as in run the first half of the tests).
///
/// To run specifically the third subshard of
/// the Web tests you can set SHARD=web_tests SUBSHARD=2 (it's zero-based).
Future<void> selectShard(Map<String, ShardRunner> shards) => _runFromList(shards, kShardKey, 'shard', 0);
Future<void> selectSubshard(Map<String, ShardRunner> subshards) => _runFromList(subshards, kSubshardKey, 'subshard', 1);
@ -1966,11 +1968,11 @@ Future<void> _runFromList(Map<String, ShardRunner> items, String key, String nam
}
if (item == null) {
for (final String currentItem in items.keys) {
print('$bold$key=$currentItem$reset');
printProgress('$bold$key=$currentItem$reset');
await items[currentItem]!();
print('');
}
} else {
printProgress('$bold$key=$item$reset');
if (!items.containsKey(item)) {
foundError(<String>[
'${red}Invalid $name: $item$reset',
@ -1978,7 +1980,6 @@ Future<void> _runFromList(Map<String, ShardRunner> items, String key, String nam
]);
return;
}
print('$bold$key=$item$reset');
await items[item]!();
}
}

132
dev/bots/test/analyze_test.dart
View file

@ -16,7 +16,7 @@ Future<String> capture(AsyncVoidCallback callback, { bool shouldHaveErrors = fal
final StringBuffer buffer = StringBuffer();
final PrintCallback oldPrint = print;
try {
print = (Object line) {
print = (Object? line) {
buffer.writeln(line);
};
await callback();
@ -45,18 +45,18 @@ void main() {
test('analyze.dart - verifyDeprecations', () async {
final String result = await capture(() => verifyDeprecations(testRootPath, minimumMatches: 2), shouldHaveErrors: true);
final String lines = <String>[
'test/analyze-test-input/root/packages/foo/deprecation.dart:12: Deprecation notice does not match required pattern.',
'test/analyze-test-input/root/packages/foo/deprecation.dart:18: Deprecation notice should be a grammatically correct sentence and start with a capital letter; see style guide: STYLE_GUIDE_URL',
'test/analyze-test-input/root/packages/foo/deprecation.dart:25: Deprecation notice should be a grammatically correct sentence and end with a period.',
'test/analyze-test-input/root/packages/foo/deprecation.dart:29: Deprecation notice does not match required pattern.',
'test/analyze-test-input/root/packages/foo/deprecation.dart:32: Deprecation notice does not match required pattern.',
'test/analyze-test-input/root/packages/foo/deprecation.dart:37: Deprecation notice does not match required pattern.',
'test/analyze-test-input/root/packages/foo/deprecation.dart:41: Deprecation notice does not match required pattern.',
'test/analyze-test-input/root/packages/foo/deprecation.dart:48: End of deprecation notice does not match required pattern.',
'test/analyze-test-input/root/packages/foo/deprecation.dart:51: Unexpected deprecation notice indent.',
'test/analyze-test-input/root/packages/foo/deprecation.dart:70: Deprecation notice does not accurately indicate a beta branch version number; please see RELEASES_URL to find the latest beta build version number.',
'test/analyze-test-input/root/packages/foo/deprecation.dart:76: Deprecation notice does not accurately indicate a beta branch version number; please see RELEASES_URL to find the latest beta build version number.',
'test/analyze-test-input/root/packages/foo/deprecation.dart:99: Deprecation notice does not match required pattern. You might have used double quotes (") for the string instead of single quotes (\').',
'║ test/analyze-test-input/root/packages/foo/deprecation.dart:12: Deprecation notice does not match required pattern.',
'║ test/analyze-test-input/root/packages/foo/deprecation.dart:18: Deprecation notice should be a grammatically correct sentence and start with a capital letter; see style guide: STYLE_GUIDE_URL',
'║ test/analyze-test-input/root/packages/foo/deprecation.dart:25: Deprecation notice should be a grammatically correct sentence and end with a period.',
'║ test/analyze-test-input/root/packages/foo/deprecation.dart:29: Deprecation notice does not match required pattern.',
'║ test/analyze-test-input/root/packages/foo/deprecation.dart:32: Deprecation notice does not match required pattern.',
'║ test/analyze-test-input/root/packages/foo/deprecation.dart:37: Deprecation notice does not match required pattern.',
'║ test/analyze-test-input/root/packages/foo/deprecation.dart:41: Deprecation notice does not match required pattern.',
'║ test/analyze-test-input/root/packages/foo/deprecation.dart:48: End of deprecation notice does not match required pattern.',
'║ test/analyze-test-input/root/packages/foo/deprecation.dart:51: Unexpected deprecation notice indent.',
'║ test/analyze-test-input/root/packages/foo/deprecation.dart:70: Deprecation notice does not accurately indicate a beta branch version number; please see RELEASES_URL to find the latest beta build version number.',
'║ test/analyze-test-input/root/packages/foo/deprecation.dart:76: Deprecation notice does not accurately indicate a beta branch version number; please see RELEASES_URL to find the latest beta build version number.',
'║ test/analyze-test-input/root/packages/foo/deprecation.dart:99: Deprecation notice does not match required pattern. You might have used double quotes (") for the string instead of single quotes (\').',
]
.map((String line) {
return line
@ -66,55 +66,30 @@ void main() {
})
.join('\n');
expect(result,
'━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n'
'╔═╡ERROR╞═══════════════════════════════════════════════════════════════════════\n'
'$lines\n'
'See: https://github.com/flutter/flutter/wiki/Tree-hygiene#handling-breaking-changes\n'
'━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n'
'║ See: https://github.com/flutter/flutter/wiki/Tree-hygiene#handling-breaking-changes\n'
'╚═══════════════════════════════════════════════════════════════════════════════\n'
);
});
test('analyze.dart - verifyGoldenTags', () async {
final String result = await capture(() => verifyGoldenTags(testRootPath, minimumMatches: 6), shouldHaveErrors: true);
final List<String> result = (await capture(() => verifyGoldenTags(testRootPath, minimumMatches: 6), shouldHaveErrors: true)).split('\n');
const String noTag = "Files containing golden tests must be tagged using @Tags(<String>['reduced-test-set']) "
'at the top of the file before import statements.';
'at the top of the file before import statements.';
const String missingTag = "Files containing golden tests must be tagged with 'reduced-test-set'.";
String lines = <String>[
'test/analyze-test-input/root/packages/foo/golden_missing_tag.dart: $missingTag',
'test/analyze-test-input/root/packages/foo/golden_no_tag.dart: $noTag',
final List<String> lines = <String>[
'║ test/analyze-test-input/root/packages/foo/golden_missing_tag.dart: $missingTag',
'║ test/analyze-test-input/root/packages/foo/golden_no_tag.dart: $noTag',
]
.map((String line) {
return line
.replaceAll('/', Platform.isWindows ? r'\' : '/');
})
.join('\n');
try {
expect(
result,
'━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n'
'$lines\n'
'See: https://github.com/flutter/flutter/wiki/Writing-a-golden-file-test-for-package:flutter\n'
'━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n'
);
} catch (_) {
// This list of files may come up in one order or the other.
lines = <String>[
'test/analyze-test-input/root/packages/foo/golden_no_tag.dart: $noTag',
'test/analyze-test-input/root/packages/foo/golden_missing_tag.dart: $missingTag',
]
.map((String line) {
return line
.replaceAll('/', Platform.isWindows ? r'\' : '/');
})
.join('\n');
expect(
result,
'━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n'
'$lines\n'
'See: https://github.com/flutter/flutter/wiki/Writing-a-golden-file-test-for-package:flutter\n'
'━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n'
);
}
.map((String line) => line.replaceAll('/', Platform.isWindows ? r'\' : '/'))
.toList();
expect(result.length, 4 + lines.length, reason: 'output had unexpected number of lines:\n${result.join('\n')}');
expect(result[0], '╔═╡ERROR╞═══════════════════════════════════════════════════════════════════════');
expect(result.getRange(1, result.length - 3).toSet(), lines.toSet());
expect(result[result.length - 3], '║ See: https://github.com/flutter/flutter/wiki/Writing-a-golden-file-test-for-package:flutter');
expect(result[result.length - 2], '╚═══════════════════════════════════════════════════════════════════════════════');
expect(result[result.length - 1], ''); // trailing newline
});
test('analyze.dart - verifyNoMissingLicense', () async {
@ -122,33 +97,30 @@ void main() {
final String file = 'test/analyze-test-input/root/packages/foo/foo.dart'
.replaceAll('/', Platform.isWindows ? r'\' : '/');
expect(result,
'━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n'
'The following file does not have the right license header for dart files:\n'
' $file\n'
'The expected license header is:\n'
'// Copyright 2014 The Flutter Authors. All rights reserved.\n'
'// Use of this source code is governed by a BSD-style license that can be\n'
'// found in the LICENSE file.\n'
'...followed by a blank line.\n'
'━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n'
'━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n'
'License check failed.\n'
'━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n'
'╔═╡ERROR╞═══════════════════════════════════════════════════════════════════════\n'
'║ The following file does not have the right license header for dart files:\n'
'║ $file\n'
'║ The expected license header is:\n'
'║ // Copyright 2014 The Flutter Authors. All rights reserved.\n'
'║ // Use of this source code is governed by a BSD-style license that can be\n'
'║ // found in the LICENSE file.\n'
'║ ...followed by a blank line.\n'
'╚═══════════════════════════════════════════════════════════════════════════════\n'
);
});
test('analyze.dart - verifyNoTrailingSpaces', () async {
final String result = await capture(() => verifyNoTrailingSpaces(testRootPath, minimumMatches: 2), shouldHaveErrors: true);
final String lines = <String>[
'test/analyze-test-input/root/packages/foo/spaces.txt:5: trailing U+0020 space character',
'test/analyze-test-input/root/packages/foo/spaces.txt:9: trailing blank line',
'║ test/analyze-test-input/root/packages/foo/spaces.txt:5: trailing U+0020 space character',
'║ test/analyze-test-input/root/packages/foo/spaces.txt:9: trailing blank line',
]
.map((String line) => line.replaceAll('/', Platform.isWindows ? r'\' : '/'))
.join('\n');
expect(result,
'━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n'
'╔═╡ERROR╞═══════════════════════════════════════════════════════════════════════\n'
'$lines\n'
'━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n'
'╚═══════════════════════════════════════════════════════════════════════════════\n'
);
});
@ -159,16 +131,16 @@ void main() {
), shouldHaveErrors: !Platform.isWindows);
if (!Platform.isWindows) {
expect(result,
'━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n'
'test/analyze-test-input/root/packages/foo/serviceaccount.enc:0: file is not valid UTF-8\n'
'All files in this repository must be UTF-8. In particular, images and other binaries\n'
'must not be checked into this repository. This is because we are very sensitive to the\n'
'size of the repository as it is distributed to all our developers. If you have a binary\n'
'to which you need access, you should consider how to fetch it from another repository;\n'
'for example, the "assets-for-api-docs" repository is used for images in API docs.\n'
'To add assets to flutter_tools templates, see the instructions in the wiki:\n'
'https://github.com/flutter/flutter/wiki/Managing-template-image-assets\n'
'━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\n'
'╔═╡ERROR╞═══════════════════════════════════════════════════════════════════════\n'
'║ test/analyze-test-input/root/packages/foo/serviceaccount.enc:0: file is not valid UTF-8\n'
'║ All files in this repository must be UTF-8. In particular, images and other binaries\n'
'║ must not be checked into this repository. This is because we are very sensitive to the\n'
'║ size of the repository as it is distributed to all our developers. If you have a binary\n'
'║ to which you need access, you should consider how to fetch it from another repository;\n'
'║ for example, the "assets-for-api-docs" repository is used for images in API docs.\n'
'║ To add assets to flutter_tools templates, see the instructions in the wiki:\n'
'║ https://github.com/flutter/flutter/wiki/Managing-template-image-assets\n'
'╚═══════════════════════════════════════════════════════════════════════════════\n'
);
}
});

31
dev/bots/test/test_test.dart
View file

@ -17,7 +17,17 @@ import 'common.dart';
/// will include the process result's stdio in the failure message.
void expectExitCode(ProcessResult result, int expectedExitCode) {
if (result.exitCode != expectedExitCode) {
fail('Failure due to exit code ${result.exitCode}\nSTDOUT:\n${result.stdout}\nSTDERR:\n${result.stderr}');
fail(
'Process ${result.pid} exitted with the wrong exit code.\n'
'\n'
'EXPECTED: exit code $expectedExitCode\n'
'ACTUAL: exit code ${result.exitCode}\n'
'\n'
'STDOUT:\n'
'${result.stdout}\n'
'STDERR:\n'
'${result.stderr}'
);
}
}
@ -96,10 +106,13 @@ void main() {
group('test.dart script', () {
const ProcessManager processManager = LocalProcessManager();
Future<ProcessResult> runScript(
[Map<String, String>? environment, List<String> otherArgs = const <String>[]]) async {
Future<ProcessResult> runScript([
Map<String, String>? environment,
List<String> otherArgs = const <String>[],
]) async {
final String dart = path.absolute(
path.join('..', '..', 'bin', 'cache', 'dart-sdk', 'bin', 'dart'));
path.join('..', '..', 'bin', 'cache', 'dart-sdk', 'bin', 'dart'),
);
final ProcessResult scriptProcess = processManager.runSync(<String>[
dart,
'test.dart',
@ -112,21 +125,21 @@ void main() {
// When updating this test, try to pick shard numbers that ensure we're checking
// that unequal test distributions don't miss tests.
ProcessResult result = await runScript(
<String, String>{'SHARD': 'test_harness_tests', 'SUBSHARD': '1_3'},
<String, String>{'SHARD': kTestHarnessShardName, 'SUBSHARD': '1_3'},
);
expectExitCode(result, 0);
expect(result.stdout, contains('Selecting subshard 1 of 3 (range 1-3 of 8)'));
expect(result.stdout, contains('Selecting subshard 1 of 3 (tests 1-3 of 8)'));
result = await runScript(
<String, String>{'SHARD': 'test_harness_tests', 'SUBSHARD': '3_3'},
<String, String>{'SHARD': kTestHarnessShardName, 'SUBSHARD': '3_3'},
);
expectExitCode(result, 0);
expect(result.stdout, contains('Selecting subshard 3 of 3 (range 7-8 of 8)'));
expect(result.stdout, contains('Selecting subshard 3 of 3 (tests 7-8 of 8)'));
});
test('exits with code 1 when SUBSHARD index greater than total', () async {
final ProcessResult result = await runScript(
<String, String>{'SHARD': 'test_harness_tests', 'SUBSHARD': '100_99'},
<String, String>{'SHARD': kTestHarnessShardName, 'SUBSHARD': '100_99'},
);
expectExitCode(result, 1);
expect(result.stdout, contains('Invalid subshard name'));

216
dev/bots/utils.dart
View file

@ -2,56 +2,43 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:core' as core_internals show print;
import 'dart:async';
import 'dart:core' hide print;
import 'dart:io' as system show exit;
import 'dart:io' hide exit;
import 'dart:math' as math;
import 'package:meta/meta.dart';
const Duration _quietTimeout = Duration(minutes: 10); // how long the output should be hidden between calls to printProgress before just being verbose
final bool hasColor = stdout.supportsAnsiEscapes;
final String bold = hasColor ? '\x1B[1m' : ''; // used for shard titles
final String red = hasColor ? '\x1B[31m' : ''; // used for errors
final String green = hasColor ? '\x1B[32m' : ''; // used for section titles, commands
final String yellow = hasColor ? '\x1B[33m' : ''; // used for skips
final String cyan = hasColor ? '\x1B[36m' : ''; // used for paths
final String reverse = hasColor ? '\x1B[7m' : ''; // used for clocks
final String bold = hasColor ? '\x1B[1m' : ''; // shard titles
final String red = hasColor ? '\x1B[31m' : ''; // errors
final String green = hasColor ? '\x1B[32m' : ''; // section titles, commands
final String yellow = hasColor ? '\x1B[33m' : ''; // indications that a test was skipped (usually renders orange or brown)
final String cyan = hasColor ? '\x1B[36m' : ''; // paths
final String reverse = hasColor ? '\x1B[7m' : ''; // clocks
final String gray = hasColor ? '\x1B[30m' : ''; // subtle decorative items (usually renders as dark gray)
final String white = hasColor ? '\x1B[37m' : ''; // last log line (usually renders as light gray)
final String reset = hasColor ? '\x1B[0m' : '';
final String redLine = '$red━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━$reset';
typedef PrintCallback = void Function(Object line);
const int kESC = 0x1B;
const int kOpenSquareBracket = 0x5B;
const int kCSIParameterRangeStart = 0x30;
const int kCSIParameterRangeEnd = 0x3F;
const int kCSIIntermediateRangeStart = 0x20;
const int kCSIIntermediateRangeEnd = 0x2F;
const int kCSIFinalRangeStart = 0x40;
const int kCSIFinalRangeEnd = 0x7E;
// Allow print() to be overridden, for tests.
PrintCallback print = core_internals.print;
bool get hasError => _hasError;
bool _hasError = false;
Iterable<String> get errorMessages => _errorMessages;
List<String> _errorMessages = <String>[];
void foundError(List<String> messages) {
assert(messages.isNotEmpty);
print(redLine);
messages.forEach(print);
print(redLine);
_errorMessages.addAll(messages);
_hasError = true;
}
@visibleForTesting
void resetErrorStatus() {
_hasError = false;
_errorMessages.clear();
}
Never reportErrorsAndExit() {
print(redLine);
print('For your convenience, the error messages reported above are repeated here:');
_errorMessages.forEach(print);
print(redLine);
system.exit(1);
String get redLine {
if (hasColor) {
return '$red${'━' * stdout.terminalColumns}$reset';
}
return '━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━';
}
String get clock {
@ -75,14 +62,161 @@ String prettyPrintDuration(Duration duration) {
return result;
}
void printProgress(String action, String workingDir, String command) {
print('$clock $action: cd $cyan$workingDir$reset; $green$command$reset');
typedef PrintCallback = void Function(Object? line);
typedef VoidCallback = void Function();
// Allow print() to be overridden, for tests.
//
// Files that import this library should not import `print` from dart:core
// and should not use dart:io's `stdout` or `stderr`.
//
// By default this hides log lines between `printProgress` calls unless a
// timeout expires or anything calls `foundError`.
//
// Also used to implement `--verbose` in test.dart.
PrintCallback print = _printQuietly;
// Called by foundError and used to implement `--abort-on-error` in test.dart.
VoidCallback? onError;
bool get hasError => _hasError;
bool _hasError = false;
Iterable<String> get errorMessages => _errorMessages;
List<String> _errorMessages = <String>[];
final List<String> _pendingLogs = <String>[];
Timer? _hideTimer; // When this is null, the output is verbose.
void foundError(List<String> messages) {
assert(messages.isNotEmpty);
// Make the error message easy to notice in the logs by
// wrapping it in a red box.
final int width = math.max(15, (hasColor ? stdout.terminalColumns : 80) - 1);
print('$red╔═╡${bold}ERROR$reset$red╞═${"═" * (width - 9)}');
for (final String message in messages.expand((String line) => line.split('\n'))) {
print('$red║$reset $message');
}
print('$red╚${"═" * width}');
// Normally, "print" actually prints to the log. To make the errors visible,
// and to include useful context, print the entire log up to this point, and
// clear it. Subsequent messages will continue to not be logged until there is
// another error.
_pendingLogs.forEach(_printLoudly);
_pendingLogs.clear();
_errorMessages.addAll(messages);
_hasError = true;
if (onError != null) {
onError!();
}
}
@visibleForTesting
void resetErrorStatus() {
_hasError = false;
_errorMessages.clear();
_pendingLogs.clear();
_hideTimer?.cancel();
_hideTimer = null;
}
Never reportErrorsAndExit() {
_hideTimer?.cancel();
_hideTimer = null;
print(redLine);
print('For your convenience, the error messages reported above are repeated here:');
_errorMessages.forEach(print);
print(redLine);
system.exit(1);
}
void printProgress(String message) {
_pendingLogs.clear();
_hideTimer?.cancel();
_hideTimer = null;
print('$clock $message $reset');
if (hasColor) {
// This sets up a timer to switch to verbose mode when the tests take too long,
// so that if a test hangs we can see the logs.
// (This is only supported with a color terminal. When the terminal doesn't
// support colors, the scripts just print everything verbosely, that way in
// CI there's nothing hidden.)
_hideTimer = Timer(_quietTimeout, () {
_hideTimer = null;
_pendingLogs.forEach(_printLoudly);
_pendingLogs.clear();
});
}
}
final Pattern _lineBreak = RegExp(r'[\r\n]');
void _printQuietly(Object? message) {
// The point of this function is to avoid printing its output unless the timer
// has gone off in which case the function assumes verbose mode is active and
// prints everything. To show that progress is still happening though, rather
// than showing nothing at all, it instead shows the last line of output and
// keeps overwriting it. To do this in color mode, carefully measures the line
// of text ignoring color codes, which is what the parser below does.
if (_hideTimer != null) {
_pendingLogs.add(message.toString());
String line = '$message'.trimRight();
final int start = line.lastIndexOf(_lineBreak) + 1;
int index = start;
int length = 0;
while (index < line.length && length < stdout.terminalColumns) {
if (line.codeUnitAt(index) == kESC) { // 0x1B
index += 1;
if (index < line.length && line.codeUnitAt(index) == kOpenSquareBracket) { // 0x5B, [
// That was the start of a CSI sequence.
index += 1;
while (index < line.length && line.codeUnitAt(index) >= kCSIParameterRangeStart
&& line.codeUnitAt(index) <= kCSIParameterRangeEnd) { // 0x30..0x3F
index += 1; // ...parameter bytes...
}
while (index < line.length && line.codeUnitAt(index) >= kCSIIntermediateRangeStart
&& line.codeUnitAt(index) <= kCSIIntermediateRangeEnd) { // 0x20..0x2F
index += 1; // ...intermediate bytes...
}
if (index < line.length && line.codeUnitAt(index) >= kCSIFinalRangeStart
&& line.codeUnitAt(index) <= kCSIFinalRangeEnd) { // 0x40..0x7E
index += 1; // ...final byte.
}
}
} else {
index += 1;
length += 1;
}
}
line = line.substring(start, index);
if (line.isNotEmpty) {
stdout.write('\r\x1B[2K$white$line$reset');
}
} else {
_printLoudly('$message');
}
}
void _printLoudly(String message) {
if (hasColor) {
// Overwrite the last line written by _printQuietly.
stdout.writeln('\r\x1B[2K$reset${message.trimRight()}');
} else {
stdout.writeln(message);
}
}
// THE FOLLOWING CODE IS A VIOLATION OF OUR STYLE GUIDE
// BECAUSE IT INTRODUCES A VERY FLAKY RACE CONDITION
// https://github.com/flutter/flutter/wiki/Style-guide-for-Flutter-repo#never-check-if-a-port-is-available-before-using-it-never-add-timeouts-and-other-race-conditions
// DO NOT USE THE FOLLOWING FUNCTIONS
// DO NOT WRITE CODE LIKE THE FOLLOWING FUNCTIONS
// https://github.com/flutter/flutter/issues/109474
int _portCounter = 8080;
/// Finds the next available local port.
Future<int> findAvailablePort() async {
Future<int> findAvailablePortAndPossiblyCauseFlakyTests() async {
while (!await _isPortAvailable(_portCounter)) {
_portCounter += 1;
}