2019-11-27 23:04:02 +00:00
|
|
|
// Copyright 2014 The Flutter Authors. All rights reserved.
|
2016-09-14 20:22:53 +00:00
|
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
|
|
// found in the LICENSE file.
|
|
|
|
|
|
|
|
import 'dart:convert';
|
|
|
|
import 'dart:io';
|
|
|
|
|
|
|
|
import 'package:args/args.dart';
|
2020-04-10 23:53:52 +00:00
|
|
|
import 'package:flutter_devicelab/framework/ab.dart';
|
2016-09-14 20:22:53 +00:00
|
|
|
import 'package:flutter_devicelab/framework/runner.dart';
|
2020-10-07 23:37:09 +00:00
|
|
|
import 'package:flutter_devicelab/framework/task_result.dart';
|
2016-09-14 20:22:53 +00:00
|
|
|
import 'package:flutter_devicelab/framework/utils.dart';
|
|
|
|
|
2022-01-10 21:40:17 +00:00
|
|
|
/// Runs tasks.
|
|
|
|
///
|
|
|
|
/// The tasks are chosen depending on the command-line options.
|
|
|
|
Future<void> main(List<String> rawArgs) async {
|
|
|
|
// This is populated by a callback in the ArgParser.
|
|
|
|
final List<String> taskNames = <String>[];
|
|
|
|
final ArgParser argParser = createArgParser(taskNames);
|
2020-04-10 23:53:52 +00:00
|
|
|
|
2022-01-10 21:40:17 +00:00
|
|
|
ArgResults args;
|
|
|
|
try {
|
|
|
|
args = argParser.parse(rawArgs); // populates taskNames as a side-effect
|
|
|
|
} on FormatException catch (error) {
|
|
|
|
stderr.writeln('${error.message}\n');
|
|
|
|
stderr.writeln('Usage:\n');
|
|
|
|
stderr.writeln(argParser.usage);
|
|
|
|
exit(1);
|
|
|
|
}
|
2020-12-08 23:53:05 +00:00
|
|
|
|
2022-01-10 21:40:17 +00:00
|
|
|
/// Suppresses standard output, prints only standard error output.
|
|
|
|
final bool silent = (args['silent'] as bool?) ?? false;
|
2020-04-10 23:53:52 +00:00
|
|
|
|
2022-01-10 21:40:17 +00:00
|
|
|
/// The build of the local engine to use.
|
|
|
|
///
|
|
|
|
/// Required for A/B test mode.
|
|
|
|
final String? localEngine = args['local-engine'] as String?;
|
2020-04-10 23:53:52 +00:00
|
|
|
|
2023-08-11 00:25:30 +00:00
|
|
|
/// The build of the local engine to use as the host platform.
|
|
|
|
///
|
|
|
|
/// Required if [localEngine] is set.
|
|
|
|
final String? localEngineHost = args['local-engine-host'] as String?;
|
|
|
|
|
2023-03-31 21:25:00 +00:00
|
|
|
/// The build of the local Web SDK to use.
|
|
|
|
///
|
|
|
|
/// Required for A/B test mode.
|
|
|
|
final String? localWebSdk = args['local-web-sdk'] as String?;
|
|
|
|
|
2022-01-10 21:40:17 +00:00
|
|
|
/// The path to the engine "src/" directory.
|
|
|
|
final String? localEngineSrcPath = args['local-engine-src-path'] as String?;
|
2020-11-17 19:28:03 +00:00
|
|
|
|
2022-01-10 21:40:17 +00:00
|
|
|
/// The device-id to run test on.
|
|
|
|
final String? deviceId = args['device-id'] as String?;
|
2020-04-10 23:53:52 +00:00
|
|
|
|
2022-01-10 21:40:17 +00:00
|
|
|
/// Whether to exit on first test failure.
|
|
|
|
final bool exitOnFirstTestFailure = (args['exit'] as bool?) ?? false;
|
2020-12-23 00:45:38 +00:00
|
|
|
|
2022-01-10 21:40:17 +00:00
|
|
|
/// Whether to tell tasks to clean up after themselves.
|
|
|
|
final bool terminateStrayDartProcesses = (args['terminate-stray-dart-processes'] as bool?) ?? false;
|
2020-10-19 20:12:04 +00:00
|
|
|
|
2022-01-10 21:40:17 +00:00
|
|
|
/// The git branch being tested on.
|
|
|
|
final String? gitBranch = args['git-branch'] as String?;
|
2020-06-26 13:29:46 +00:00
|
|
|
|
2022-01-10 21:40:17 +00:00
|
|
|
/// Name of the LUCI builder this test is currently running on.
|
|
|
|
///
|
|
|
|
/// This is only passed on CI runs for Cocoon to be able to uniquely identify
|
|
|
|
/// this test run.
|
|
|
|
final String? luciBuilder = args['luci-builder'] as String?;
|
2016-09-14 20:22:53 +00:00
|
|
|
|
2022-01-10 21:40:17 +00:00
|
|
|
/// Path to write test results to.
|
|
|
|
final String? resultsPath = args['results-file'] as String?;
|
2020-12-23 00:45:38 +00:00
|
|
|
|
2023-01-17 22:01:11 +00:00
|
|
|
/// Use an emulator for this test if it is an android test.
|
|
|
|
final bool useEmulator = (args['use-emulator'] as bool?) ?? false;
|
|
|
|
|
2019-08-20 21:37:42 +00:00
|
|
|
if (args.wasParsed('list')) {
|
2022-01-10 21:40:17 +00:00
|
|
|
for (int i = 0; i < taskNames.length; i++) {
|
|
|
|
print('${(i + 1).toString().padLeft(3)} - ${taskNames[i]}');
|
2019-08-20 21:37:42 +00:00
|
|
|
}
|
2022-01-10 21:40:17 +00:00
|
|
|
exit(0);
|
2019-08-20 21:37:42 +00:00
|
|
|
}
|
|
|
|
|
2022-01-10 21:40:17 +00:00
|
|
|
if (taskNames.isEmpty) {
|
2016-09-14 20:22:53 +00:00
|
|
|
stderr.writeln('Failed to find tasks to run based on supplied options.');
|
2022-01-10 21:40:17 +00:00
|
|
|
exit(1);
|
2016-09-14 20:22:53 +00:00
|
|
|
}
|
|
|
|
|
2020-04-10 23:53:52 +00:00
|
|
|
if (args.wasParsed('ab')) {
|
2022-01-10 21:40:17 +00:00
|
|
|
final int runsPerTest = int.parse(args['ab'] as String);
|
|
|
|
final String resultsFile = args['ab-result-file'] as String? ?? 'ABresults#.json';
|
|
|
|
if (taskNames.length > 1) {
|
|
|
|
stderr.writeln('When running in A/B test mode exactly one task must be passed but got ${taskNames.join(', ')}.\n');
|
|
|
|
stderr.writeln(argParser.usage);
|
|
|
|
exit(1);
|
|
|
|
}
|
2023-03-31 21:25:00 +00:00
|
|
|
if (localEngine == null && localWebSdk == null) {
|
|
|
|
stderr.writeln('When running in A/B test mode --local-engine or --local-web-sdk is required.\n');
|
2022-01-10 21:40:17 +00:00
|
|
|
stderr.writeln(argParser.usage);
|
|
|
|
exit(1);
|
|
|
|
}
|
2023-08-11 00:25:30 +00:00
|
|
|
if (localEngineHost == null) {
|
|
|
|
stderr.writeln('When running in A/B test mode --local-engine-host is required.\n');
|
|
|
|
stderr.writeln(argParser.usage);
|
|
|
|
exit(1);
|
|
|
|
}
|
2022-01-10 21:40:17 +00:00
|
|
|
await _runABTest(
|
|
|
|
runsPerTest: runsPerTest,
|
|
|
|
silent: silent,
|
|
|
|
localEngine: localEngine,
|
2023-08-11 00:25:30 +00:00
|
|
|
localEngineHost: localEngineHost,
|
2023-03-31 21:25:00 +00:00
|
|
|
localWebSdk: localWebSdk,
|
2022-01-10 21:40:17 +00:00
|
|
|
localEngineSrcPath: localEngineSrcPath,
|
|
|
|
deviceId: deviceId,
|
|
|
|
resultsFile: resultsFile,
|
|
|
|
taskName: taskNames.single,
|
|
|
|
);
|
2020-04-10 23:53:52 +00:00
|
|
|
} else {
|
2022-01-10 21:40:17 +00:00
|
|
|
await runTasks(taskNames,
|
2019-06-07 19:49:09 +00:00
|
|
|
silent: silent,
|
2021-04-01 20:54:02 +00:00
|
|
|
localEngine: localEngine,
|
2023-08-29 00:31:14 +00:00
|
|
|
localEngineHost: localEngineHost,
|
2021-04-01 20:54:02 +00:00
|
|
|
localEngineSrcPath: localEngineSrcPath,
|
2020-06-26 13:29:46 +00:00
|
|
|
deviceId: deviceId,
|
2021-03-04 19:59:02 +00:00
|
|
|
exitOnFirstTestFailure: exitOnFirstTestFailure,
|
2022-01-10 21:40:17 +00:00
|
|
|
terminateStrayDartProcesses: terminateStrayDartProcesses,
|
2021-03-04 19:59:02 +00:00
|
|
|
gitBranch: gitBranch,
|
|
|
|
luciBuilder: luciBuilder,
|
|
|
|
resultsPath: resultsPath,
|
2023-01-17 22:01:11 +00:00
|
|
|
useEmulator: useEmulator,
|
2019-06-07 19:49:09 +00:00
|
|
|
);
|
2019-08-20 21:37:42 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-01-10 21:40:17 +00:00
|
|
|
Future<void> _runABTest({
|
|
|
|
required int runsPerTest,
|
|
|
|
required bool silent,
|
2023-03-31 21:25:00 +00:00
|
|
|
required String? localEngine,
|
2023-08-11 00:25:30 +00:00
|
|
|
required String localEngineHost,
|
2023-03-31 21:25:00 +00:00
|
|
|
required String? localWebSdk,
|
2022-01-10 21:40:17 +00:00
|
|
|
required String? localEngineSrcPath,
|
|
|
|
required String? deviceId,
|
|
|
|
required String resultsFile,
|
|
|
|
required String taskName,
|
|
|
|
}) async {
|
2020-04-10 23:53:52 +00:00
|
|
|
print('$taskName A/B test. Will run $runsPerTest times.');
|
|
|
|
|
2023-03-31 21:25:00 +00:00
|
|
|
assert(localEngine != null || localWebSdk != null);
|
|
|
|
|
2023-08-11 00:25:30 +00:00
|
|
|
final ABTest abTest = ABTest(
|
|
|
|
localEngine: (localEngine ?? localWebSdk)!,
|
|
|
|
localEngineHost: localEngineHost,
|
|
|
|
taskName: taskName,
|
|
|
|
);
|
2020-04-10 23:53:52 +00:00
|
|
|
for (int i = 1; i <= runsPerTest; i++) {
|
|
|
|
section('Run #$i');
|
|
|
|
|
|
|
|
print('Running with the default engine (A)');
|
2020-10-07 23:37:09 +00:00
|
|
|
final TaskResult defaultEngineResult = await runTask(
|
2020-04-10 23:53:52 +00:00
|
|
|
taskName,
|
|
|
|
silent: silent,
|
2020-06-26 13:29:46 +00:00
|
|
|
deviceId: deviceId,
|
2020-04-10 23:53:52 +00:00
|
|
|
);
|
|
|
|
|
|
|
|
print('Default engine result:');
|
|
|
|
print(const JsonEncoder.withIndent(' ').convert(defaultEngineResult));
|
|
|
|
|
2020-10-07 23:37:09 +00:00
|
|
|
if (!defaultEngineResult.succeeded) {
|
2020-04-10 23:53:52 +00:00
|
|
|
stderr.writeln('Task failed on the default engine.');
|
|
|
|
exit(1);
|
|
|
|
}
|
|
|
|
|
|
|
|
abTest.addAResult(defaultEngineResult);
|
|
|
|
|
|
|
|
print('Running with the local engine (B)');
|
2020-10-07 23:37:09 +00:00
|
|
|
final TaskResult localEngineResult = await runTask(
|
2020-04-10 23:53:52 +00:00
|
|
|
taskName,
|
|
|
|
silent: silent,
|
|
|
|
localEngine: localEngine,
|
2023-08-29 00:31:14 +00:00
|
|
|
localEngineHost: localEngineHost,
|
2023-03-31 21:25:00 +00:00
|
|
|
localWebSdk: localWebSdk,
|
2020-04-10 23:53:52 +00:00
|
|
|
localEngineSrcPath: localEngineSrcPath,
|
2020-06-26 13:29:46 +00:00
|
|
|
deviceId: deviceId,
|
2020-04-10 23:53:52 +00:00
|
|
|
);
|
|
|
|
|
|
|
|
print('Task localEngineResult:');
|
|
|
|
print(const JsonEncoder.withIndent(' ').convert(localEngineResult));
|
|
|
|
|
2020-10-07 23:37:09 +00:00
|
|
|
if (!localEngineResult.succeeded) {
|
2020-04-10 23:53:52 +00:00
|
|
|
stderr.writeln('Task failed on the local engine.');
|
|
|
|
exit(1);
|
|
|
|
}
|
|
|
|
|
|
|
|
abTest.addBResult(localEngineResult);
|
2020-04-16 21:36:42 +00:00
|
|
|
|
2023-05-16 16:14:23 +00:00
|
|
|
if (!silent && i < runsPerTest) {
|
2020-04-16 21:36:42 +00:00
|
|
|
section('A/B results so far');
|
|
|
|
print(abTest.printSummary());
|
|
|
|
}
|
2020-04-10 23:53:52 +00:00
|
|
|
}
|
2020-05-08 07:34:02 +00:00
|
|
|
abTest.finalize();
|
|
|
|
|
2022-01-10 21:40:17 +00:00
|
|
|
final File jsonFile = _uniqueFile(resultsFile);
|
2021-06-24 23:22:43 +00:00
|
|
|
jsonFile.writeAsStringSync(const JsonEncoder.withIndent(' ').convert(abTest.jsonMap));
|
2020-04-16 21:36:42 +00:00
|
|
|
|
2023-05-16 16:14:23 +00:00
|
|
|
if (!silent) {
|
2020-04-16 21:36:42 +00:00
|
|
|
section('Raw results');
|
|
|
|
print(abTest.rawResults());
|
|
|
|
}
|
|
|
|
|
|
|
|
section('Final A/B results');
|
2020-04-10 23:53:52 +00:00
|
|
|
print(abTest.printSummary());
|
2020-05-08 07:34:02 +00:00
|
|
|
|
|
|
|
print('');
|
|
|
|
print('Results saved to ${jsonFile.path}');
|
|
|
|
}
|
|
|
|
|
|
|
|
File _uniqueFile(String filenameTemplate) {
|
|
|
|
final List<String> parts = filenameTemplate.split('#');
|
|
|
|
if (parts.length != 2) {
|
|
|
|
return File(filenameTemplate);
|
|
|
|
}
|
|
|
|
File file = File(parts[0] + parts[1]);
|
|
|
|
int i = 1;
|
|
|
|
while (file.existsSync()) {
|
2020-12-23 00:45:38 +00:00
|
|
|
file = File(parts[0] + i.toString() + parts[1]);
|
2020-05-08 07:34:02 +00:00
|
|
|
i++;
|
|
|
|
}
|
|
|
|
return file;
|
2020-04-10 23:53:52 +00:00
|
|
|
}
|
|
|
|
|
2022-01-10 21:40:17 +00:00
|
|
|
ArgParser createArgParser(List<String> taskNames) {
|
|
|
|
return ArgParser()
|
|
|
|
..addMultiOption(
|
|
|
|
'task',
|
|
|
|
abbr: 't',
|
2023-11-29 08:41:27 +00:00
|
|
|
help: 'Name of a Dart file in bin/tasks.\n'
|
|
|
|
' Example: complex_layout__start_up\n'
|
2022-01-10 21:40:17 +00:00
|
|
|
'\n'
|
|
|
|
'This option may be repeated to specify multiple tasks.',
|
2023-11-29 08:41:27 +00:00
|
|
|
callback: (List<String> tasks) {
|
|
|
|
taskNames.addAll(tasks);
|
2022-01-10 21:40:17 +00:00
|
|
|
},
|
|
|
|
)
|
|
|
|
..addOption(
|
|
|
|
'device-id',
|
|
|
|
abbr: 'd',
|
|
|
|
help: 'Target device id (prefixes are allowed, names are not supported).\n'
|
|
|
|
'The option will be ignored if the test target does not run on a\n'
|
|
|
|
'mobile device. This still respects the device operating system\n'
|
|
|
|
'settings in the test case, and will results in error if no device\n'
|
|
|
|
'with given ID/ID prefix is found.',
|
|
|
|
)
|
|
|
|
..addOption(
|
|
|
|
'ab',
|
|
|
|
help: 'Runs an A/B test comparing the default engine with the local\n'
|
|
|
|
'engine build for one task. This option does not support running\n'
|
|
|
|
'multiple tasks. The value is the number of times to run the task.\n'
|
|
|
|
'The task is expected to be a benchmark that reports score keys.\n'
|
|
|
|
'The A/B test collects the metrics collected by the test and\n'
|
|
|
|
'produces a report containing averages, noise, and the speed-up\n'
|
|
|
|
'between the two engines. --local-engine is required when running\n'
|
|
|
|
'an A/B test.',
|
|
|
|
callback: (String? value) {
|
|
|
|
if (value != null && int.tryParse(value) == null) {
|
|
|
|
throw ArgParserException('Option --ab must be a number, but was "$value".');
|
|
|
|
}
|
|
|
|
},
|
|
|
|
)
|
|
|
|
..addOption(
|
|
|
|
'ab-result-file',
|
|
|
|
help: 'The filename in which to place the json encoded results of an A/B test.\n'
|
|
|
|
'The filename may contain a single # character to be replaced by a sequence\n'
|
|
|
|
'number if the name already exists.',
|
|
|
|
)
|
|
|
|
..addFlag(
|
|
|
|
'exit',
|
|
|
|
help: 'Exit on the first test failure. Currently flakes are intentionally (though '
|
|
|
|
'incorrectly) not considered to be failures.',
|
|
|
|
)
|
|
|
|
..addOption(
|
|
|
|
'git-branch',
|
|
|
|
help: '[Flutter infrastructure] Git branch of the current commit. LUCI\n'
|
|
|
|
'checkouts run in detached HEAD state, so the branch must be passed.',
|
|
|
|
)
|
|
|
|
..addOption(
|
|
|
|
'local-engine',
|
|
|
|
help: 'Name of a build output within the engine out directory, if you\n'
|
|
|
|
'are building Flutter locally. Use this to select a specific\n'
|
|
|
|
'version of the engine if you have built multiple engine targets.\n'
|
|
|
|
'This path is relative to --local-engine-src-path/out. This option\n'
|
|
|
|
'is required when running an A/B test (see the --ab option).',
|
|
|
|
)
|
2023-08-11 00:25:30 +00:00
|
|
|
..addOption(
|
|
|
|
'local-engine-host',
|
|
|
|
help: 'Name of a build output within the engine out directory, if you\n'
|
|
|
|
'are building Flutter locally. Use this to select a specific\n'
|
|
|
|
'version of the engine to use as the host platform if you have built '
|
|
|
|
'multiple engine targets.\n'
|
|
|
|
'This path is relative to --local-engine-src-path/out. This option\n'
|
|
|
|
'is required when running an A/B test (see the --ab option).',
|
|
|
|
)
|
2023-03-31 21:25:00 +00:00
|
|
|
..addOption(
|
|
|
|
'local-web-sdk',
|
|
|
|
help: 'Name of a build output within the engine out directory, if you\n'
|
|
|
|
'are building Flutter locally. Use this to select a specific\n'
|
|
|
|
'version of the engine if you have built multiple engine targets.\n'
|
|
|
|
'This path is relative to --local-engine-src-path/out. This option\n'
|
|
|
|
'is required when running an A/B test (see the --ab option).',
|
|
|
|
)
|
2022-01-10 21:40:17 +00:00
|
|
|
..addFlag(
|
|
|
|
'list',
|
|
|
|
abbr: 'l',
|
|
|
|
help: "Don't actually run the tasks, but list out the tasks that would\n"
|
|
|
|
'have been run, in the order they would have run.',
|
|
|
|
)
|
|
|
|
..addOption(
|
|
|
|
'local-engine-src-path',
|
|
|
|
help: 'Path to your engine src directory, if you are building Flutter\n'
|
|
|
|
'locally. Defaults to \$FLUTTER_ENGINE if set, or tries to guess at\n'
|
|
|
|
'the location based on the value of the --flutter-root option.',
|
|
|
|
)
|
|
|
|
..addOption('luci-builder', help: '[Flutter infrastructure] Name of the LUCI builder being run on.')
|
|
|
|
..addOption(
|
|
|
|
'results-file',
|
|
|
|
help: '[Flutter infrastructure] File path for test results. If passed with\n'
|
|
|
|
'task, will write test results to the file.'
|
|
|
|
)
|
|
|
|
..addOption(
|
|
|
|
'service-account-token-file',
|
|
|
|
help: '[Flutter infrastructure] Authentication for uploading results.',
|
|
|
|
)
|
|
|
|
..addFlag(
|
|
|
|
'silent',
|
|
|
|
help: 'Reduce verbosity slightly.',
|
|
|
|
)
|
|
|
|
..addFlag(
|
|
|
|
'terminate-stray-dart-processes',
|
|
|
|
defaultsTo: true,
|
|
|
|
help: 'Whether to send a SIGKILL signal to any Dart processes that are still '
|
|
|
|
'running when a task is completed. If any Dart processes are terminated '
|
|
|
|
'in this way, the test is considered to have failed.',
|
|
|
|
)
|
2023-01-17 22:01:11 +00:00
|
|
|
..addFlag(
|
|
|
|
'use-emulator',
|
|
|
|
help: 'If this is an android test, use an emulator to run the test instead of '
|
|
|
|
'a physical device.'
|
|
|
|
)
|
2022-01-10 21:40:17 +00:00
|
|
|
..addMultiOption(
|
|
|
|
'test',
|
|
|
|
hide: true,
|
|
|
|
callback: (List<String> value) {
|
|
|
|
if (value.isNotEmpty) {
|
|
|
|
throw const FormatException(
|
|
|
|
'Invalid option --test. Did you mean --task (-t)?',
|
|
|
|
);
|
2016-11-16 21:54:06 +00:00
|
|
|
}
|
2022-01-10 21:40:17 +00:00
|
|
|
},
|
|
|
|
);
|
|
|
|
}
|