development/flutter
development/flutter
1
0
Fork 0
mirror of https://github.com/flutter/flutter synced 2024-08-25 02:46:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

move CI tests into the main repo (#5758)

Browse source
This commit is contained in:
Yegor 2016-09-14 13:22:53 -07:00 committed by GitHub
parent 4ec5144427
commit 1ba1562293
42 changed files with 2624 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
dev/bots/test.sh
View file

@ -35,6 +35,7 @@ fi
(cd packages/flutter_test; flutter test)
(cd packages/flutter_tools; dart -c test/all.dart)
(cd dev/devicelab; dart -c test/all.dart)
(cd dev/manual_tests; flutter test)
(cd examples/hello_world; flutter test)
(cd examples/layers; flutter test)
@ -55,4 +56,4 @@ if [ -n "$COVERAGE_FLAG" ]; then
fi
# generate the API docs, upload them
dev/bots/docs.sh
dev/bots/docs.sh

9
dev/devicelab/.gitignore vendored Normal file
View file

@ -0,0 +1,9 @@
.DS_Store
.buildlog
.idea
.packages
.pub/
build/
packages
pubspec.lock
.atom/

96
dev/devicelab/README.md Normal file
View file

@ -0,0 +1,96 @@
# Flutter devicelab
This package contains the test framework and tests that run on physical devices.
More generally the tests are referred to as "tasks" in the API, but since we
primarily use it for testing, this document refers to them as "tests".
# Writing tests
A test is a simple Dart program that lives under `bin/tests` and uses
`package:flutter_devicelab/framework/framework.dart` to define and run a _task_.
Example:
```dart
import 'dart:async';
import 'package:flutter_devicelab/framework/framework.dart';
Future<Null> main() async {
await task(() async {
... do something interesting ...
// Aggregate results into a JSONable Map structure.
Map<String, dynamic> testResults = ...;
// Report success.
return new TaskResult.success(testResults);
// Or you can also report a failure.
return new TaskResult.failure('Something went wrong!');
});
}
```
Only one `task` is permitted per program. However, that task can run any number
of tests internally. A task has a name. It succeeds and fails independently of
other tasks, and is reported to the dashboard independently of other tasks.
A task runs in its own standalone Dart VM and reports results via Dart VM
service protocol. This ensures that tasks do not interfere with each other and
lets the CI system time out and clean up tasks that get stuck.
# Adding tests to the CI environment
The `manifest.yaml` file describes a subset of tests we run in the CI. To add
your test edit `manifest.yaml` and add the following in the "tasks" dictionary:
```
{NAME_OF_TEST}:
description: {DESCRIPTION}
stage: {STAGE}
required_agent_capabilities: {CAPABILITIES}
```
Where:
- `{NAME_OF_TEST}` is the name of your test that also matches the name of the
file in `bin/tests` without the `.dart` extension.
- `{DESCRIPTION}` is the plain English description of your test that helps
others understand what this test is testing.
- `{STAGE}` is `devicelab` if you want to run on Android, or `devicelab_ios` if
you want to run on iOS.
- `{CAPABILITIES}` is an array that lists the capabilities required of
the test agent (the computer that runs the test) to run your test. Available
capabilities are: `has-android-device`, `has-ios-device`.
# Running tests locally
Do make sure your tests pass locally before deploying to the CI environment.
Below is a handful of commands that run tests in a fashion very close to how the
CI environment runs them. These commands are also useful when you need to
reproduce a CI test failure locally.
To run a test use option `-t` (`--task`):
```sh
dart bin/run.dart -t {NAME_OF_TEST}
```
To run multiple tests repeat option `-t` (`--task`) multiple times:
```sh
dart bin/run.dart -t test1 -t test2 -t test3
```
To run all tests defined in `manifest.yaml` use option `-a` (`--all`):
```sh
dart bin/run.dart -a
```
To run tests from a specific stage use option `-s` (`--stage`):
```sh
dart bin/run.dart -s {NAME_OF_STAGE}
```

101
dev/devicelab/bin/run.dart Normal file
View file

@ -0,0 +1,101 @@
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'dart:convert';
import 'dart:io';
import 'package:args/args.dart';
import 'package:flutter_devicelab/framework/manifest.dart';
import 'package:flutter_devicelab/framework/runner.dart';
import 'package:flutter_devicelab/framework/utils.dart';
/// Runs tasks.
///
/// The tasks are chosen depending on the command-line options
/// (see [_argParser]).
Future<Null> main(List<String> rawArgs) async {
ArgResults args;
try {
args = _argParser.parse(rawArgs);
} on FormatException catch(error) {
stderr.writeln('${error.message}\n');
stderr.writeln('Usage:\n');
stderr.writeln(_argParser.usage);
exitCode = 1;
return null;
}
List<String> taskNames = <String>[];
if (args.wasParsed('task')) {
taskNames.addAll(args['task']);
} else if (args.wasParsed('stage')) {
String stageName = args['stage'];
List<ManifestTask> tasks = loadTaskManifest().tasks;
for (ManifestTask task in tasks) {
if (task.stage == stageName)
taskNames.add(task.name);
}
} else if (args.wasParsed('all')) {
List<ManifestTask> tasks = loadTaskManifest().tasks;
for (ManifestTask task in tasks) {
taskNames.add(task.name);
}
}
if (taskNames.isEmpty) {
stderr.writeln('Failed to find tasks to run based on supplied options.');
exitCode = 1;
return null;
}
for (String taskName in taskNames) {
section('Running task "$taskName"');
Map<String, dynamic> result = await runTask(taskName);
if (!result['success'])
exitCode = 1;
print('Task result:');
print(new JsonEncoder.withIndent(' ').convert(result));
section('Finished task "$taskName"');
}
}
/// Command-line options for the `run.dart` command.
final ArgParser _argParser = new ArgParser()
..addOption(
'task',
abbr: 't',
allowMultiple: true,
splitCommas: true,
help: 'Name of the task to run. This option may be repeated to '
'specify multiple tasks. A task selected by name does not have to be '
'defined in manifest.yaml. It only needs a Dart executable in bin/tasks.',
)
..addOption(
'stage',
abbr: 's',
help: 'Name of the stage. Runs all tasks for that stage. '
'The tasks and their stages are read from manifest.yaml.',
)
..addOption(
'all',
abbr: 'a',
help: 'Runs all tasks defined in manifest.yaml.',
)
..addOption(
'test',
hide: true,
allowMultiple: true,
splitCommas: true,
callback: (List<String> value) {
if (value.isNotEmpty) {
throw new FormatException(
'Invalid option --test. Did you mean --task (-t)?',
);
}
},
);

20
dev/devicelab/bin/tasks/analyzer_cli__analysis_time.dart Normal file
View file

@ -0,0 +1,20 @@
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'package:flutter_devicelab/tasks/analysis.dart';
import 'package:flutter_devicelab/framework/framework.dart';
import 'package:flutter_devicelab/framework/utils.dart';
Future<Null> main() async {
String revision = await getCurrentFlutterRepoCommit();
DateTime revisionTimestamp = await getFlutterRepoCommitTimestamp(revision);
String dartSdkVersion = await getDartVersion();
await task(createAnalyzerCliTest(
sdk: dartSdkVersion,
commit: revision,
timestamp: revisionTimestamp,
));
}

20
dev/devicelab/bin/tasks/analyzer_server__analysis_time.dart Normal file
View file

@ -0,0 +1,20 @@
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'package:flutter_devicelab/tasks/analysis.dart';
import 'package:flutter_devicelab/framework/framework.dart';
import 'package:flutter_devicelab/framework/utils.dart';
Future<Null> main() async {
String revision = await getCurrentFlutterRepoCommit();
DateTime revisionTimestamp = await getFlutterRepoCommitTimestamp(revision);
String dartSdkVersion = await getDartVersion();
await task(createAnalyzerServerTest(
sdk: dartSdkVersion,
commit: revision,
timestamp: revisionTimestamp,
));
}

12
dev/devicelab/bin/tasks/basic_material_app__size.dart Normal file
View file

@ -0,0 +1,12 @@
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'package:flutter_devicelab/tasks/size_tests.dart';
import 'package:flutter_devicelab/framework/framework.dart';
Future<Null> main() async {
await task(createBasicMaterialAppSizeTest());
}

12
dev/devicelab/bin/tasks/complex_layout__build.dart Normal file
View file

@ -0,0 +1,12 @@
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'package:flutter_devicelab/tasks/perf_tests.dart';
import 'package:flutter_devicelab/framework/framework.dart';
Future<Null> main() async {
await task(createComplexLayoutBuildTest());
}

12
dev/devicelab/bin/tasks/complex_layout__start_up.dart Normal file
View file

@ -0,0 +1,12 @@
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'package:flutter_devicelab/tasks/perf_tests.dart';
import 'package:flutter_devicelab/framework/framework.dart';
Future<Null> main() async {
await task(createComplexLayoutStartupTest(ios: false));
}

12
dev/devicelab/bin/tasks/complex_layout_ios__start_up.dart Normal file
View file

@ -0,0 +1,12 @@
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'package:flutter_devicelab/tasks/perf_tests.dart';
import 'package:flutter_devicelab/framework/framework.dart';
Future<Null> main() async {
await task(createComplexLayoutStartupTest(ios: true));
}

12
dev/devicelab/bin/tasks/complex_layout_scroll_perf__timeline_summary.dart Normal file
View file

@ -0,0 +1,12 @@
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'package:flutter_devicelab/tasks/perf_tests.dart';
import 'package:flutter_devicelab/framework/framework.dart';
Future<Null> main() async {
await task(createComplexLayoutScrollPerfTest(ios: false));
}

12
dev/devicelab/bin/tasks/complex_layout_scroll_perf_ios__timeline_summary.dart Normal file
View file

@ -0,0 +1,12 @@
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'package:flutter_devicelab/tasks/perf_tests.dart';
import 'package:flutter_devicelab/framework/framework.dart';
Future<Null> main() async {
await task(createComplexLayoutScrollPerfTest(ios: true));
}

12
dev/devicelab/bin/tasks/flutter_gallery__build.dart Normal file
View file

@ -0,0 +1,12 @@
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'package:flutter_devicelab/tasks/perf_tests.dart';
import 'package:flutter_devicelab/framework/framework.dart';
Future<Null> main() async {
await task(createFlutterGalleryBuildTest());
}

12
dev/devicelab/bin/tasks/flutter_gallery__start_up.dart Normal file
View file

@ -0,0 +1,12 @@
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'package:flutter_devicelab/tasks/perf_tests.dart';
import 'package:flutter_devicelab/framework/framework.dart';
Future<Null> main() async {
await task(createFlutterGalleryStartupTest(ios: false));
}

12
dev/devicelab/bin/tasks/flutter_gallery__transition_perf.dart Normal file
View file

@ -0,0 +1,12 @@
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'package:flutter_devicelab/tasks/gallery.dart';
import 'package:flutter_devicelab/framework/framework.dart';
Future<Null> main() async {
await task(createGalleryTransitionTest(ios: false));
}

12
dev/devicelab/bin/tasks/flutter_gallery_ios__start_up.dart Normal file
View file

@ -0,0 +1,12 @@
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'package:flutter_devicelab/tasks/perf_tests.dart';
import 'package:flutter_devicelab/framework/framework.dart';
Future<Null> main() async {
await task(createFlutterGalleryStartupTest(ios: true));
}

12
dev/devicelab/bin/tasks/flutter_gallery_ios__transition_perf.dart Normal file
View file

@ -0,0 +1,12 @@
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'package:flutter_devicelab/tasks/gallery.dart';
import 'package:flutter_devicelab/framework/framework.dart';
Future<Null> main() async {
await task(createGalleryTransitionTest(ios: true));
}

18
dev/devicelab/bin/tasks/mega_gallery__refresh_time.dart Normal file
View file

@ -0,0 +1,18 @@
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'package:flutter_devicelab/tasks/refresh.dart';
import 'package:flutter_devicelab/framework/framework.dart';
import 'package:flutter_devicelab/framework/utils.dart';
Future<Null> main() async {
String revision = await getCurrentFlutterRepoCommit();
DateTime revisionTimestamp = await getFlutterRepoCommitTimestamp(revision);
await task(createRefreshTest(
commit: revision,
timestamp: revisionTimestamp,
));
}

14
dev/devicelab/bin/tasks/smoke_test_failure.dart Normal file
View file

@ -0,0 +1,14 @@
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'package:flutter_devicelab/framework/framework.dart';
/// Smoke test of a task that fails by returning an unsuccessful response.
Future<Null> main() async {
await task(() async {
return new TaskResult.failure('Failed');
});
}

13
dev/devicelab/bin/tasks/smoke_test_setup_failure.dart Normal file
View file

@ -0,0 +1,13 @@
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
/// Creates a situation when the test framework was not properly initialized.
///
/// By not calling `task()` the VM service extension is not registered and
/// therefore will not accept requests to run tasks. When the runner attempts to
/// connect and run the test it will receive a "method not found" error from the
/// VM service, will likely retry and finally time out.
Future<Null> main() async {}

14
dev/devicelab/bin/tasks/smoke_test_success.dart Normal file
View file

@ -0,0 +1,14 @@
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'package:flutter_devicelab/framework/framework.dart';
/// Smoke test of a successful task.
Future<Null> main() async {
await task(() async {
return new TaskResult.success(<String, dynamic>{});
});
}

14
dev/devicelab/bin/tasks/smoke_test_throws.dart Normal file
View file

@ -0,0 +1,14 @@
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'package:flutter_devicelab/framework/framework.dart';
/// Smoke test of a task that fails with an exception.
Future<Null> main() async {
await task(() async {
throw 'failed';
});
}

202
dev/devicelab/lib/framework/adb.dart Normal file
View file

@ -0,0 +1,202 @@
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'dart:io';
import 'dart:math' as math;
import 'package:path/path.dart' as path;
import 'utils.dart';
typedef Future<Adb> AdbGetter();
/// Get an instance of [Adb].
///
/// See [realAdbGetter] for signature. This can be overwritten for testing.
AdbGetter adb = realAdbGetter;
Adb _currentDevice;
/// Picks a random Android device out of connected devices and sets it as
/// [_currentDevice].
Future<Null> pickNextDevice() async {
List<Adb> allDevices =
(await Adb.deviceIds).map((String id) => new Adb(deviceId: id)).toList();
if (allDevices.length == 0) throw 'No Android devices detected';
// TODO(yjbanov): filter out and warn about those with low battery level
_currentDevice = allDevices[new math.Random().nextInt(allDevices.length)];
}
Future<Adb> realAdbGetter() async {
if (_currentDevice == null) await pickNextDevice();
return _currentDevice;
}
/// Gets the ID of an unlocked device, unlocking it if necessary.
// TODO(yjbanov): abstract away iOS from Android.
Future<String> getUnlockedDeviceId({ bool ios: false }) async {
if (ios) {
// We currently do not have a way to lock/unlock iOS devices, or even to
// pick one out of many. So we pick the first random iPhone and assume it's
// already unlocked. For now we'll just keep them at minimum screen
// brightness so they don't drain battery too fast.
List<String> iosDeviceIds =
grep('UniqueDeviceID', from: await eval('ideviceinfo', <String>[]))
.map((String line) => line.split(' ').last)
.toList();
if (iosDeviceIds.isEmpty) throw 'No connected iOS devices found.';
return iosDeviceIds.first;
}
Adb device = await adb();
await device.unlock();
return device.deviceId;
}
/// Android Debug Bridge (`adb`) client that exposes a subset of functions
/// relevant to on-device testing.
class Adb {
Adb({ this.deviceId });
final String deviceId;
// Parses information about a device. Example:
//
// 015d172c98400a03 device usb:340787200X product:nakasi model:Nexus_7 device:grouper
static final RegExp _kDeviceRegex = new RegExp(r'^(\S+)\s+(\S+)(.*)');
/// Reports connection health for every device.
static Future<Map<String, HealthCheckResult>> checkDevices() async {
Map<String, HealthCheckResult> results = <String, HealthCheckResult>{};
for (String deviceId in await deviceIds) {
try {
Adb device = new Adb(deviceId: deviceId);
// Just a smoke test that we can read wakefulness state
// TODO(yjbanov): also check battery level
await device._getWakefulness();
results['android-device-$deviceId'] = new HealthCheckResult.success();
} catch (e, s) {
results['android-device-$deviceId'] = new HealthCheckResult.error(e, s);
}
}
return results;
}
/// Kills the `adb` server causing it to start a new instance upon next
/// command.
///
/// Restarting `adb` helps with keeping device connections alive. When `adb`
/// runs non-stop for too long it loses connections to devices.
static Future<Null> restart() async {
await exec(adbPath, <String>['kill-server'], canFail: false);
}
/// List of device IDs visible to `adb`.
static Future<List<String>> get deviceIds async {
List<String> output =
(await eval(adbPath, <String>['devices', '-l'], canFail: false))
.trim()
.split('\n');
List<String> results = <String>[];
for (String line in output) {
// Skip lines like: * daemon started successfully *
if (line.startsWith('* daemon ')) continue;
if (line.startsWith('List of devices')) continue;
if (_kDeviceRegex.hasMatch(line)) {
Match match = _kDeviceRegex.firstMatch(line);
String deviceID = match[1];
String deviceState = match[2];
if (!const <String>['unauthorized', 'offline'].contains(deviceState)) {
results.add(deviceID);
}
} else {
throw 'Failed to parse device from adb output: $line';
}
}
return results;
}
/// Whether the device is awake.
Future<bool> isAwake() async {
return await _getWakefulness() == 'Awake';
}
/// Whether the device is asleep.
Future<bool> isAsleep() async {
return await _getWakefulness() == 'Asleep';
}
/// Wake up the device if it is not awake using [togglePower].
Future<Null> wakeUp() async {
if (!(await isAwake())) await togglePower();
}
/// Send the device to sleep mode if it is not asleep using [togglePower].
Future<Null> sendToSleep() async {
if (!(await isAsleep())) await togglePower();
}
/// Sends `KEYCODE_POWER` (26), which causes the device to toggle its mode
/// between awake and asleep.
Future<Null> togglePower() async {
await shellExec('input', const <String>['keyevent', '26']);
}
/// Unlocks the device by sending `KEYCODE_MENU` (82).
///
/// This only works when the device doesn't have a secure unlock pattern.
Future<Null> unlock() async {
await wakeUp();
await shellExec('input', const <String>['keyevent', '82']);
}
/// Retrieves device's wakefulness state.
///
/// See: https://android.googlesource.com/platform/frameworks/base/+/master/core/java/android/os/PowerManagerInternal.java
Future<String> _getWakefulness() async {
String powerInfo = await shellEval('dumpsys', <String>['power']);
String wakefulness =
grep('mWakefulness=', from: powerInfo).single.split('=')[1].trim();
return wakefulness;
}
/// Executes [command] on `adb shell` and returns its exit code.
Future<Null> shellExec(String command, List<String> arguments,
{ Map<String, String> env }) async {
await exec(adbPath, <String>['shell', command]..addAll(arguments),
env: env, canFail: false);
}
/// Executes [command] on `adb shell` and returns its standard output as a [String].
Future<String> shellEval(String command, List<String> arguments,
{ Map<String, String> env }) {
return eval(adbPath, <String>['shell', command]..addAll(arguments),
env: env, canFail: false);
}
}
/// Path to the `adb` executable.
String get adbPath {
String androidHome = Platform.environment['ANDROID_HOME'];
if (androidHome == null)
throw 'ANDROID_HOME environment variable missing. This variable must '
'point to the Android SDK directory containing platform-tools.';
File adbPath = file(path.join(androidHome, 'platform-tools/adb'));
if (!adbPath.existsSync()) throw 'adb not found at: $adbPath';
return adbPath.absolute.path;
}

62
dev/devicelab/lib/framework/benchmarks.dart Normal file
View file

@ -0,0 +1,62 @@
// Copyright (c) 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'framework.dart';
/// A benchmark harness used to run a benchmark multiple times and report the
/// best result.
abstract class Benchmark {
Benchmark(this.name);
final String name;
TaskResult bestResult;
Future<Null> init() => new Future<Null>.value();
Future<num> run();
TaskResult get lastResult;
@override
String toString() => name;
}
/// Runs a [benchmark] [iterations] times and reports the best result.
///
/// Use [warmUpBenchmark] to discard cold performance results.
Future<num> runBenchmark(Benchmark benchmark, {
int iterations: 1,
bool warmUpBenchmark: false
}) async {
await benchmark.init();
List<num> allRuns = <num>[];
num minValue;
if (warmUpBenchmark)
await benchmark.run();
while (iterations > 0) {
iterations--;
print('');
try {
num result = await benchmark.run();
allRuns.add(result);
if (minValue == null || result < minValue) {
benchmark.bestResult = benchmark.lastResult;
minValue = result;
}
} catch (error) {
print('benchmark failed with error: $error');
}
}
return minValue;
}

217
dev/devicelab/lib/framework/framework.dart Normal file
View file

@ -0,0 +1,217 @@
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'dart:convert';
import 'dart:developer';
import 'dart:io';
import 'dart:isolate';
import 'package:logging/logging.dart';
import 'utils.dart';
/// Maximum amount of time a single task is allowed to take to run.
///
/// If exceeded the task is considered to have failed.
const Duration taskTimeout = const Duration(minutes: 10);
/// Represents a unit of work performed in the CI environment that can
/// succeed, fail and be retried independently of others.
typedef Future<TaskResult> TaskFunction();
bool _isTaskRegistered = false;
/// Registers a [task] to run, returns the result when it is complete.
///
/// Note, the task does not run immediately but waits for the request via the
/// VM service protocol to run it.
///
/// It is ok for a [task] to perform many things. However, only one task can be
/// registered per Dart VM.
Future<TaskResult> task(TaskFunction task) {
if (_isTaskRegistered)
throw new StateError('A task is already registered');
_isTaskRegistered = true;
// TODO: allow overriding logging.
Logger.root.level = Level.ALL;
Logger.root.onRecord.listen((LogRecord rec) {
print('${rec.level.name}: ${rec.time}: ${rec.message}');
});
_TaskRunner runner = new _TaskRunner(task);
runner.keepVmAliveUntilTaskRunRequested();
return runner.whenDone;
}
class _TaskRunner {
static final Logger logger = new Logger('TaskRunner');
final TaskFunction task;
// TODO: workaround for https://github.com/dart-lang/sdk/issues/23797
RawReceivePort _keepAlivePort;
Timer _startTaskTimeout;
bool _taskStarted = false;
final Completer<TaskResult> _completer = new Completer<TaskResult>();
_TaskRunner(this.task) {
registerExtension('ext.cocoonRunTask',
(String method, Map<String, String> parameters) async {
TaskResult result = await run();
return new ServiceExtensionResponse.result(JSON.encode(result.toJson()));
});
registerExtension('ext.cocoonRunnerReady',
(String method, Map<String, String> parameters) async {
return new ServiceExtensionResponse.result('"ready"');
});
}
/// Signals that this task runner finished running the task.
Future<TaskResult> get whenDone => _completer.future;
Future<TaskResult> run() async {
try {
_taskStarted = true;
TaskResult result = await _performTask().timeout(taskTimeout);
_completer.complete(result);
return result;
} on TimeoutException catch (_) {
return new TaskResult.failure('Task timed out after $taskTimeout');
} finally {
forceQuitRunningProcesses();
_closeKeepAlivePort();
}
}
/// Causes the Dart VM to stay alive until a request to run the task is
/// received via the VM service protocol.
void keepVmAliveUntilTaskRunRequested() {
if (_taskStarted)
throw new StateError('Task already started.');
// Merely creating this port object will cause the VM to stay alive and keep
// the VM service server running until the port is disposed of.
_keepAlivePort = new RawReceivePort();
// Timeout if nothing bothers to connect and ask us to run the task.
const Duration taskStartTimeout = const Duration(seconds: 10);
_startTaskTimeout = new Timer(taskStartTimeout, () {
if (!_taskStarted) {
logger.severe('Task did not start in $taskStartTimeout.');
_closeKeepAlivePort();
exitCode = 1;
}
});
}
/// Disables the keep-alive port, allowing the VM to exit.
void _closeKeepAlivePort() {
_startTaskTimeout?.cancel();
_keepAlivePort?.close();
}
Future<TaskResult> _performTask() async {
try {
return await task();
} catch (taskError, taskErrorStack) {
String message = 'Task failed: $taskError';
if (taskErrorStack != null) {
message += '\n\n$taskErrorStack';
}
return new TaskResult.failure(message);
}
}
}
/// A result of running a single task.
class TaskResult {
/// Constructs a successful result.
TaskResult.success(this.data, {this.benchmarkScoreKeys: const <String>[]})
: this.succeeded = true,
this.message = 'success' {
const JsonEncoder prettyJson = const JsonEncoder.withIndent(' ');
if (benchmarkScoreKeys != null) {
for (String key in benchmarkScoreKeys) {
if (!data.containsKey(key)) {
throw 'Invalid Golem score key "$key". It does not exist in task '
'result data ${prettyJson.convert(data)}';
} else if (data[key] is! num) {
throw 'Invalid Golem score for key "$key". It is expected to be a num '
'but was ${data[key].runtimeType}: ${prettyJson.convert(data[key])}';
}
}
}
}
/// Constructs a successful result using JSON data stored in a file.
factory TaskResult.successFromFile(File file,
{List<String> benchmarkScoreKeys}) {
return new TaskResult.success(JSON.decode(file.readAsStringSync()),
benchmarkScoreKeys: benchmarkScoreKeys);
}
/// Constructs an unsuccessful result.
TaskResult.failure(this.message)
: this.succeeded = false,
this.data = null,
this.benchmarkScoreKeys = const <String>[];
/// Whether the task succeeded.
final bool succeeded;
/// Task-specific JSON data
final Map<String, dynamic> data;
/// Keys in [data] that store scores that will be submitted to Golem.
///
/// Each key is also part of a benchmark's name tracked by Golem.
/// A benchmark name is computed by combining [Task.name] with a key
/// separated by a dot. For example, if a task's name is
/// `"complex_layout__start_up"` and score key is
/// `"engineEnterTimestampMicros"`, the score will be submitted to Golem under
/// `"complex_layout__start_up.engineEnterTimestampMicros"`.
///
/// This convention reduces the amount of configuration that needs to be done
/// to submit benchmark scores to Golem.
final List<String> benchmarkScoreKeys;
/// Whether the task failed.
bool get failed => !succeeded;
/// Explains the result in a human-readable format.
final String message;
/// Serializes this task result to JSON format.
///
/// The JSON format is as follows:
///
/// {
/// "success": true|false,
/// "data": arbitrary JSON data valid only for successful results,
/// "benchmarkScoreKeys": [
/// contains keys into "data" that represent benchmarks scores, which
/// can be uploaded, for example. to golem, valid only for successful
/// results
/// ],
/// "reason": failure reason string valid only for unsuccessful results
/// }
Map<String, dynamic> toJson() {
Map<String, dynamic> json = <String, dynamic>{
'success': succeeded,
};
if (succeeded) {
json['data'] = data;
json['benchmarkScoreKeys'] = benchmarkScoreKeys;
} else {
json['reason'] = message;
}
return json;
}
}

127
dev/devicelab/lib/framework/manifest.dart Normal file
View file

@ -0,0 +1,127 @@
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'package:meta/meta.dart';
import 'package:yaml/yaml.dart';
import 'utils.dart';
/// Loads manifest data from `manifest.yaml` file or from [yaml], if present.
Manifest loadTaskManifest([ String yaml ]) {
dynamic manifestYaml = yaml == null
? loadYaml(file('manifest.yaml').readAsStringSync())
: loadYamlNode(yaml);
_checkType(manifestYaml is Map, manifestYaml, 'Manifest', 'dictionary');
return _validateAndParseManifest(manifestYaml);
}
/// Contains CI task information.
class Manifest {
Manifest._(this.tasks);
/// CI tasks.
final List<ManifestTask> tasks;
}
/// A CI task.
class ManifestTask {
ManifestTask._({
@required this.name,
@required this.description,
@required this.stage,
@required this.requiredAgentCapabilities,
}) {
String taskName = 'task "$name"';
_checkIsNotBlank(name, 'Task name', taskName);
_checkIsNotBlank(description, 'Task description', taskName);
_checkIsNotBlank(stage, 'Task stage', taskName);
_checkIsNotBlank(requiredAgentCapabilities, 'requiredAgentCapabilities', taskName);
}
/// Task name as it appears on the dashboard.
final String name;
/// A human-readable description of the task.
final String description;
/// The stage this task should run in.
final String stage;
/// Capabilities required of the build agent to be able to perform this task.
final List<String> requiredAgentCapabilities;
}
/// Thrown when the manifest YAML is not valid.
class ManifestError extends Error {
ManifestError(this.message);
final String message;
@override
String toString() => '$ManifestError: $message';
}
// There's no good YAML validator, at least not for Dart, so we validate
// manually. It's not too much code and produces good error messages.
Manifest _validateAndParseManifest(Map<String, dynamic> manifestYaml) {
_checkKeys(manifestYaml, 'manifest', const <String>['tasks']);
return new Manifest._(_validateAndParseTasks(manifestYaml['tasks']));
}
List<ManifestTask> _validateAndParseTasks(dynamic tasksYaml) {
_checkType(tasksYaml is Map, tasksYaml, 'Value of "tasks"', 'dictionary');
return tasksYaml.keys.map((dynamic taskName) => _validateAndParseTask(taskName, tasksYaml[taskName])).toList();
}
ManifestTask _validateAndParseTask(dynamic taskName, dynamic taskYaml) {
_checkType(taskName is String, taskName, 'Task name', 'string');
_checkType(taskYaml is Map, taskYaml, 'Value of task "$taskName"', 'dictionary');
_checkKeys(taskYaml, 'Value of task "$taskName"', const <String>[
'description',
'stage',
'required_agent_capabilities',
]);
List<String> capabilities = _validateAndParseCapabilities(taskName, taskYaml['required_agent_capabilities']);
return new ManifestTask._(
name: taskName,
description: taskYaml['description'],
stage: taskYaml['stage'],
requiredAgentCapabilities: capabilities,
);
}
List<String> _validateAndParseCapabilities(String taskName, dynamic capabilitiesYaml) {
_checkType(capabilitiesYaml is List, capabilitiesYaml, 'required_agent_capabilities', 'list');
for (int i = 0; i < capabilitiesYaml.length; i++) {
dynamic capability = capabilitiesYaml[i];
_checkType(capability is String, capability, 'required_agent_capabilities[$i]', 'string');
}
return capabilitiesYaml;
}
void _checkType(bool isValid, dynamic value, String variableName, String typeName) {
if (!isValid) {
throw new ManifestError(
'$variableName must be a $typeName but was ${value.runtimeType}: $value',
);
}
}
void _checkIsNotBlank(dynamic value, String variableName, String ownerName) {
if (value == null || value.isEmpty) {
throw new ManifestError('$variableName must not be empty in $ownerName.');
}
}
void _checkKeys(Map<String, dynamic> map, String variableName, List<String> allowedKeys) {
for (String key in map.keys) {
if (!allowedKeys.contains(key)) {
throw new ManifestError(
'Unrecognized property "$key" in $variableName. '
'Allowed properties: ${allowedKeys.join(', ')}');
}
}
}

129
dev/devicelab/lib/framework/runner.dart Normal file
View file

@ -0,0 +1,129 @@
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'dart:convert';
import 'dart:io';
import 'package:vm_service_client/vm_service_client.dart';
import 'package:flutter_devicelab/framework/utils.dart';
/// Slightly longer than task timeout that gives the task runner a chance to
/// clean-up before forcefully quitting it.
const Duration taskTimeoutWithGracePeriod = const Duration(minutes: 11);
/// Runs a task in a separate Dart VM and collects the result using the VM
/// service protocol.
///
/// [taskName] is the name of the task. The corresponding task executable is
/// expected to be found under `bin/tasks`.
Future<Map<String, dynamic>> runTask(String taskName) async {
String taskExecutable = 'bin/tasks/$taskName.dart';
if (!file(taskExecutable).existsSync())
throw 'Executable Dart file not found: $taskExecutable';
int vmServicePort = await _findAvailablePort();
Process runner = await startProcess(dartBin, <String>[
'--enable-vm-service=$vmServicePort',
'--no-pause-isolates-on-exit',
taskExecutable,
]);
bool runnerFinished = false;
runner.exitCode.then((_) {
runnerFinished = true;
});
StreamSubscription<String> stdoutSub = runner.stdout
.transform(new Utf8Decoder())
.transform(new LineSplitter())
.listen((String line) {
stdout.writeln('[$taskName] [STDOUT] $line');
});
StreamSubscription<String> stderrSub = runner.stderr
.transform(new Utf8Decoder())
.transform(new LineSplitter())
.listen((String line) {
stderr.writeln('[$taskName] [STDERR] $line');
});
String waitingFor = 'connection';
try {
VMIsolate isolate = await _connectToRunnerIsolate(vmServicePort);
waitingFor = 'task completion';
Map<String, dynamic> taskResult =
await isolate.invokeExtension('ext.cocoonRunTask').timeout(taskTimeoutWithGracePeriod);
waitingFor = 'task process to exit';
await runner.exitCode.timeout(const Duration(seconds: 1));
return taskResult;
} on TimeoutException catch (timeout) {
runner.kill(ProcessSignal.SIGINT);
return <String, dynamic>{
'success': false,
'reason': 'Timeout waiting for $waitingFor: ${timeout.message}',
};
} finally {
if (!runnerFinished)
runner.kill(ProcessSignal.SIGKILL);
await stdoutSub.cancel();
await stderrSub.cancel();
}
}
Future<VMIsolate> _connectToRunnerIsolate(int vmServicePort) async {
String url = 'ws://localhost:$vmServicePort/ws';
DateTime started = new DateTime.now();
// TODO(yjbanov): due to lack of imagination at the moment the handshake with
// the task process is very rudimentary and requires this small
// delay to let the task process open up the VM service port.
// Otherwise we almost always hit the non-ready case first and
// wait a whole 1 second, which is annoying.
await new Future<Null>.delayed(const Duration(milliseconds: 100));
while (true) {
try {
// Make sure VM server is up by successfully opening and closing a socket.
await (await WebSocket.connect(url)).close();
// Look up the isolate.
VMServiceClient client = new VMServiceClient.connect(url);
VM vm = await client.getVM();
VMIsolate isolate = vm.isolates.single;
String response = await isolate.invokeExtension('ext.cocoonRunnerReady');
if (response != 'ready') throw 'not ready yet';
return isolate;
} catch (error) {
const Duration connectionTimeout = const Duration(seconds: 2);
if (new DateTime.now().difference(started) > connectionTimeout) {
throw new TimeoutException(
'Failed to connect to the task runner process',
connectionTimeout,
);
}
print('VM service not ready yet: $error');
const Duration pauseBetweenRetries = const Duration(milliseconds: 200);
print('Will retry in $pauseBetweenRetries.');
await new Future<Null>.delayed(pauseBetweenRetries);
}
}
}
Future<int> _findAvailablePort() async {
int port = 20000;
while (true) {
try {
ServerSocket socket =
await ServerSocket.bind(InternetAddress.LOOPBACK_IP_V4, port);
await socket.close();
return port;
} catch (_) {
port++;
}
}
}

412
dev/devicelab/lib/framework/utils.dart Normal file
View file

@ -0,0 +1,412 @@
// Copyright (c) 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'dart:convert';
import 'dart:io';
import 'package:meta/meta.dart';
import 'package:path/path.dart' as path;
import 'package:stack_trace/stack_trace.dart';
/// Virtual current working directory, which affect functions, such as [exec].
String cwd = Directory.current.path;
List<ProcessInfo> _runningProcesses = <ProcessInfo>[];
class ProcessInfo {
ProcessInfo(this.command, this.process);
final DateTime startTime = new DateTime.now();
final String command;
final Process process;
@override
String toString() {
return '''
command : $command
started : $startTime
pid : ${process.pid}
'''
.trim();
}
}
/// Result of a health check for a specific parameter.
class HealthCheckResult {
HealthCheckResult.success([this.details]) : succeeded = true;
HealthCheckResult.failure(this.details) : succeeded = false;
HealthCheckResult.error(dynamic error, dynamic stackTrace)
: succeeded = false,
details = 'ERROR: $error${'\n$stackTrace' ?? ''}';
final bool succeeded;
final String details;
@override
String toString() {
StringBuffer buf = new StringBuffer(succeeded ? 'succeeded' : 'failed');
if (details != null && details.trim().isNotEmpty) {
buf.writeln();
// Indent details by 4 spaces
for (String line in details.trim().split('\n')) {
buf.writeln(' $line');
}
}
return '$buf';
}
}
class BuildFailedError extends Error {
BuildFailedError(this.message);
final String message;
@override
String toString() => message;
}
void fail(String message) {
throw new BuildFailedError(message);
}
void rm(FileSystemEntity entity) {
if (entity.existsSync())
entity.deleteSync();
}
/// Remove recursively.
void rmTree(FileSystemEntity entity) {
if (entity.existsSync())
entity.deleteSync(recursive: true);
}
List<FileSystemEntity> ls(Directory directory) => directory.listSync();
Directory dir(String path) => new Directory(path);
File file(String path) => new File(path);
void copy(File sourceFile, Directory targetDirectory, {String name}) {
File target = file(
path.join(targetDirectory.path, name ?? path.basename(sourceFile.path)));
target.writeAsBytesSync(sourceFile.readAsBytesSync());
}
FileSystemEntity move(FileSystemEntity whatToMove,
{Directory to, String name}) {
return whatToMove
.renameSync(path.join(to.path, name ?? path.basename(whatToMove.path)));
}
/// Equivalent of `mkdir directory`.
void mkdir(Directory directory) {
directory.createSync();
}
/// Equivalent of `mkdir -p directory`.
void mkdirs(Directory directory) {
directory.createSync(recursive: true);
}
bool exists(FileSystemEntity entity) => entity.existsSync();
void section(String title) {
print('\n••• $title •••');
}
Future<String> getDartVersion() async {
// The Dart VM returns the version text to stderr.
ProcessResult result = Process.runSync(dartBin, <String>['--version']);
String version = result.stderr.trim();
// Convert:
// Dart VM version: 1.17.0-dev.2.0 (Tue May 3 12:14:52 2016) on "macos_x64"
// to:
// 1.17.0-dev.2.0
if (version.indexOf('(') != -1)
version = version.substring(0, version.indexOf('(')).trim();
if (version.indexOf(':') != -1)
version = version.substring(version.indexOf(':') + 1).trim();
return version.replaceAll('"', "'");
}
Future<String> getCurrentFlutterRepoCommit() {
if (!dir('${flutterDirectory.path}/.git').existsSync()) {
return null;
}
return inDirectory(flutterDirectory, () {
return eval('git', <String>['rev-parse', 'HEAD']);
});
}
Future<DateTime> getFlutterRepoCommitTimestamp(String commit) {
// git show -s --format=%at 4b546df7f0b3858aaaa56c4079e5be1ba91fbb65
return inDirectory(flutterDirectory, () async {
String unixTimestamp = await eval('git', <String>[
'show',
'-s',
'--format=%at',
commit,
]);
int secondsSinceEpoch = int.parse(unixTimestamp);
return new DateTime.fromMillisecondsSinceEpoch(secondsSinceEpoch * 1000);
});
}
Future<Process> startProcess(String executable, List<String> arguments,
{Map<String, String> env}) async {
String command = '$executable ${arguments?.join(" ") ?? ""}';
print('Executing: $command');
Process proc = await Process.start(executable, arguments,
environment: env, workingDirectory: cwd);
ProcessInfo procInfo = new ProcessInfo(command, proc);
_runningProcesses.add(procInfo);
proc.exitCode.then((_) {
_runningProcesses.remove(procInfo);
});
return proc;
}
Future<Null> forceQuitRunningProcesses() async {
if (_runningProcesses.isEmpty)
return;
// Give normally quitting processes a chance to report their exit code.
await new Future<Null>.delayed(const Duration(seconds: 1));
// Whatever's left, kill it.
for (ProcessInfo p in _runningProcesses) {
print('Force quitting process:\n$p');
if (!p.process.kill()) {
print('Failed to force quit process');
}
}
_runningProcesses.clear();
}
/// Executes a command and returns its exit code.
Future<int> exec(String executable, List<String> arguments,
{Map<String, String> env, bool canFail: false}) async {
Process proc = await startProcess(executable, arguments, env: env);
proc.stdout
.transform(UTF8.decoder)
.transform(const LineSplitter())
.listen(print);
proc.stderr
.transform(UTF8.decoder)
.transform(const LineSplitter())
.listen(stderr.writeln);
int exitCode = await proc.exitCode;
if (exitCode != 0 && !canFail)
fail('Executable failed with exit code $exitCode.');
return exitCode;
}
/// Executes a command and returns its standard output as a String.
///
/// Standard error is redirected to the current process' standard error stream.
Future<String> eval(String executable, List<String> arguments,
{Map<String, String> env, bool canFail: false}) async {
Process proc = await startProcess(executable, arguments, env: env);
proc.stderr.listen((List<int> data) {
stderr.add(data);
});
String output = await UTF8.decodeStream(proc.stdout);
int exitCode = await proc.exitCode;
if (exitCode != 0 && !canFail)
fail('Executable failed with exit code $exitCode.');
return output.trimRight();
}
Future<int> flutter(String command,
{List<String> options: const <String>[], bool canFail: false}) {
List<String> args = <String>[command]..addAll(options);
return exec(path.join(flutterDirectory.path, 'bin', 'flutter'), args,
canFail: canFail);
}
String get dartBin =>
path.join(flutterDirectory.path, 'bin', 'cache', 'dart-sdk', 'bin', 'dart');
Future<int> dart(List<String> args) => exec(dartBin, args);
Future<int> pub(String command) {
return exec(
path.join(flutterDirectory.path, 'bin', 'cache', 'dart-sdk', 'bin', 'pub'),
<String>[command]);
}
Future<dynamic> inDirectory(dynamic directory, Future<dynamic> action()) async {
String previousCwd = cwd;
try {
cd(directory);
return await action();
} finally {
cd(previousCwd);
}
}
void cd(dynamic directory) {
Directory d;
if (directory is String) {
cwd = directory;
d = dir(directory);
} else if (directory is Directory) {
cwd = directory.path;
d = directory;
} else {
throw 'Unsupported type ${directory.runtimeType} of $directory';
}
if (!d.existsSync())
throw 'Cannot cd into directory that does not exist: $directory';
}
Directory get flutterDirectory => dir('../..').absolute;
String requireEnvVar(String name) {
String value = Platform.environment[name];
if (value == null) fail('$name environment variable is missing. Quitting.');
return value;
}
dynamic/*=T*/ requireConfigProperty/*<T>*/(
Map<String, dynamic/*<T>*/ > map, String propertyName) {
if (!map.containsKey(propertyName))
fail('Configuration property not found: $propertyName');
return map[propertyName];
}
String jsonEncode(dynamic data) {
return new JsonEncoder.withIndent(' ').convert(data) + '\n';
}
Future<Null> getFlutter(String revision) async {
section('Get Flutter!');
if (exists(flutterDirectory)) {
rmTree(flutterDirectory);
}
await inDirectory(flutterDirectory.parent, () async {
await exec('git', <String>['clone', 'https://github.com/flutter/flutter.git']);
});
await inDirectory(flutterDirectory, () async {
await exec('git', <String>['checkout', revision]);
});
await flutter('config', options: <String>['--no-analytics']);
section('flutter doctor');
await flutter('doctor');
section('flutter update-packages');
await flutter('update-packages');
}
void checkNotNull(Object o1,
[Object o2 = 1,
Object o3 = 1,
Object o4 = 1,
Object o5 = 1,
Object o6 = 1,
Object o7 = 1,
Object o8 = 1,
Object o9 = 1,
Object o10 = 1]) {
if (o1 == null)
throw 'o1 is null';
if (o2 == null)
throw 'o2 is null';
if (o3 == null)
throw 'o3 is null';
if (o4 == null)
throw 'o4 is null';
if (o5 == null)
throw 'o5 is null';
if (o6 == null)
throw 'o6 is null';
if (o7 == null)
throw 'o7 is null';
if (o8 == null)
throw 'o8 is null';
if (o9 == null)
throw 'o9 is null';
if (o10 == null)
throw 'o10 is null';
}
/// Add benchmark values to a JSON results file.
///
/// If the file contains information about how long the benchmark took to run
/// (a `time` field), then return that info.
// TODO(yjbanov): move this data to __metadata__
num addBuildInfo(File jsonFile,
{num expected, String sdk, String commit, DateTime timestamp}) {
Map<String, dynamic> json;
if (jsonFile.existsSync())
json = JSON.decode(jsonFile.readAsStringSync());
else
json = <String, dynamic>{};
if (expected != null)
json['expected'] = expected;
if (sdk != null)
json['sdk'] = sdk;
if (commit != null)
json['commit'] = commit;
if (timestamp != null)
json['timestamp'] = timestamp.millisecondsSinceEpoch;
jsonFile.writeAsStringSync(jsonEncode(json));
// Return the elapsed time of the benchmark (if any).
return json['time'];
}
/// Splits [from] into lines and selects those that contain [pattern].
Iterable<String> grep(Pattern pattern, {@required String from}) {
return from.split('\n').where((String line) {
return line.contains(pattern);
});
}
/// Captures asynchronous stack traces thrown by [callback].
///
/// This is a convenience wrapper around [Chain] optimized for use with
/// `async`/`await`.
///
/// Example:
///
/// try {
/// await captureAsyncStacks(() { /* async things */ });
/// } catch (error, chain) {
///
/// }
Future<Null> runAndCaptureAsyncStacks(Future<Null> callback()) {
Completer<Null> completer = new Completer<Null>();
Chain.capture(() async {
await callback();
completer.complete();
}, onError: (dynamic error, Chain chain) async {
completer.completeError(error, chain);
});
return completer.future;
}

116
dev/devicelab/lib/tasks/analysis.dart Normal file
View file

@ -0,0 +1,116 @@
// Copyright (c) 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'dart:io';
import 'package:meta/meta.dart';
import 'package:path/path.dart' as path;
import '../framework/benchmarks.dart';
import '../framework/framework.dart';
import '../framework/utils.dart';
TaskFunction createAnalyzerCliTest({
@required String sdk,
@required String commit,
@required DateTime timestamp,
}) {
return new AnalyzerCliTask(sdk, commit, timestamp);
}
TaskFunction createAnalyzerServerTest({
@required String sdk,
@required String commit,
@required DateTime timestamp,
}) {
return new AnalyzerServerTask(sdk, commit, timestamp);
}
abstract class AnalyzerTask {
Benchmark benchmark;
Future<TaskResult> call() async {
section(benchmark.name);
await runBenchmark(benchmark, iterations: 3, warmUpBenchmark: true);
return benchmark.bestResult;
}
}
class AnalyzerCliTask extends AnalyzerTask {
AnalyzerCliTask(String sdk, String commit, DateTime timestamp) {
this.benchmark = new FlutterAnalyzeBenchmark(sdk, commit, timestamp);
}
}
class AnalyzerServerTask extends AnalyzerTask {
AnalyzerServerTask(String sdk, String commit, DateTime timestamp) {
this.benchmark = new FlutterAnalyzeAppBenchmark(sdk, commit, timestamp);
}
}
class FlutterAnalyzeBenchmark extends Benchmark {
FlutterAnalyzeBenchmark(this.sdk, this.commit, this.timestamp)
: super('flutter analyze --flutter-repo');
final String sdk;
final String commit;
final DateTime timestamp;
File get benchmarkFile =>
file(path.join(flutterDirectory.path, 'analysis_benchmark.json'));
@override
TaskResult get lastResult => new TaskResult.successFromFile(benchmarkFile);
@override
Future<num> run() async {
rm(benchmarkFile);
await inDirectory(flutterDirectory, () async {
await flutter('analyze', options: <String>[
'--flutter-repo',
'--benchmark',
]);
});
return addBuildInfo(benchmarkFile,
timestamp: timestamp, expected: 25.0, sdk: sdk, commit: commit);
}
}
class FlutterAnalyzeAppBenchmark extends Benchmark {
FlutterAnalyzeAppBenchmark(this.sdk, this.commit, this.timestamp)
: super('analysis server mega_gallery');
final String sdk;
final String commit;
final DateTime timestamp;
@override
TaskResult get lastResult => new TaskResult.successFromFile(benchmarkFile);
Directory get megaDir => dir(
path.join(flutterDirectory.path, 'dev/benchmarks/mega_gallery'));
File get benchmarkFile =>
file(path.join(megaDir.path, 'analysis_benchmark.json'));
@override
Future<Null> init() {
return inDirectory(flutterDirectory, () async {
await dart(<String>['dev/tools/mega_gallery.dart']);
});
}
@override
Future<num> run() async {
rm(benchmarkFile);
await inDirectory(megaDir, () async {
await flutter('analyze', options: <String>[
'--watch',
'--benchmark',
]);
});
return addBuildInfo(benchmarkFile,
timestamp: timestamp, expected: 10.0, sdk: sdk, commit: commit);
}
}

58
dev/devicelab/lib/tasks/gallery.dart Normal file
View file

@ -0,0 +1,58 @@
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'dart:convert';
import 'dart:io';
import 'package:meta/meta.dart';
import '../framework/adb.dart';
import '../framework/framework.dart';
import '../framework/utils.dart';
TaskFunction createGalleryTransitionTest({ @required bool ios: false }) {
return new GalleryTransitionTest(ios: ios);
}
class GalleryTransitionTest {
GalleryTransitionTest({ this.ios });
final bool ios;
Future<TaskResult> call() async {
String deviceId = await getUnlockedDeviceId(ios: ios);
Directory galleryDirectory =
dir('${flutterDirectory.path}/examples/flutter_gallery');
await inDirectory(galleryDirectory, () async {
await pub('get');
if (ios) {
// This causes an Xcode project to be created.
await flutter('build', options: <String>['ios', '--profile']);
}
await flutter('drive', options: <String>[
'--profile',
'--trace-startup',
'-t',
'test_driver/transitions_perf.dart',
'-d',
deviceId,
]);
});
// Route paths contains slashes, which Firebase doesn't accept in keys, so we
// remove them.
Map<String, dynamic> original = JSON.decode(file(
'${galleryDirectory.path}/build/transition_durations.timeline.json')
.readAsStringSync());
Map<String, dynamic> clean = new Map<String, dynamic>.fromIterable(
original.keys,
key: (String key) => key.replaceAll('/', ''),
value: (String key) => original[key]);
return new TaskResult.success(clean);
}
}

164
dev/devicelab/lib/tasks/perf_tests.dart Normal file
View file

@ -0,0 +1,164 @@
// Copyright (c) 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'dart:convert' show JSON;
import 'package:meta/meta.dart';
import '../framework/adb.dart';
import '../framework/framework.dart';
import '../framework/utils.dart';
TaskFunction createComplexLayoutScrollPerfTest({ @required bool ios: false }) {
return new PerfTest(
'${flutterDirectory.path}/dev/benchmarks/complex_layout',
'test_driver/scroll_perf.dart',
'complex_layout_scroll_perf',
ios: ios
);
}
TaskFunction createFlutterGalleryStartupTest({ bool ios: false }) {
return new StartupTest(
'${flutterDirectory.path}/examples/flutter_gallery',
ios: ios
);
}
TaskFunction createComplexLayoutStartupTest({ bool ios: false }) {
return new StartupTest(
'${flutterDirectory.path}/dev/benchmarks/complex_layout',
ios: ios
);
}
TaskFunction createFlutterGalleryBuildTest() {
return new BuildTest('${flutterDirectory.path}/examples/flutter_gallery');
}
TaskFunction createComplexLayoutBuildTest() {
return new BuildTest('${flutterDirectory.path}/dev/benchmarks/complex_layout');
}
/// Measure application startup performance.
class StartupTest {
static const Duration _startupTimeout = const Duration(minutes: 2);
StartupTest(this.testDirectory, { this.ios });
final String testDirectory;
final bool ios;
Future<TaskResult> call() async {
return await inDirectory(testDirectory, () async {
String deviceId = await getUnlockedDeviceId(ios: ios);
await pub('get');
if (ios) {
// This causes an Xcode project to be created.
await flutter('build', options: <String>['ios', '--profile']);
}
await flutter('run', options: <String>[
'--profile',
'--trace-startup',
'-d',
deviceId,
]).timeout(_startupTimeout);
Map<String, dynamic> data = JSON.decode(file('$testDirectory/build/start_up_info.json').readAsStringSync());
return new TaskResult.success(data, benchmarkScoreKeys: <String>[
'engineEnterTimestampMicros',
'timeToFirstFrameMicros',
]);
});
}
}
/// Measures application runtime performance, specifically per-frame
/// performance.
class PerfTest {
PerfTest(this.testDirectory, this.testTarget, this.timelineFileName, { this.ios });
final String testDirectory;
final String testTarget;
final String timelineFileName;
final bool ios;
Future<TaskResult> call() {
return inDirectory(testDirectory, () async {
String deviceId = await getUnlockedDeviceId(ios: ios);
await pub('get');
if (ios) {
// This causes an Xcode project to be created.
await flutter('build', options: <String>['ios', '--profile']);
}
await flutter('drive', options: <String>[
'-v',
'--profile',
'--trace-startup', // Enables "endless" timeline event buffering.
'-t',
testTarget,
'-d',
deviceId,
]);
Map<String, dynamic> data = JSON.decode(file('$testDirectory/build/$timelineFileName.timeline_summary.json').readAsStringSync());
return new TaskResult.success(data, benchmarkScoreKeys: <String>[
'average_frame_build_time_millis',
'worst_frame_build_time_millis',
'missed_frame_build_budget_count',
]);
});
}
}
class BuildTest {
BuildTest(this.testDirectory);
final String testDirectory;
Future<TaskResult> call() async {
return await inDirectory(testDirectory, () async {
Adb device = await adb();
await device.unlock();
await pub('get');
Stopwatch watch = new Stopwatch()..start();
await flutter('build', options: <String>[
'aot',
'--profile',
'--no-pub',
'--target-platform', 'android-arm' // Generate blobs instead of assembly.
]);
watch.stop();
int vmisolateSize = file("$testDirectory/build/aot/snapshot_aot_vmisolate").lengthSync();
int isolateSize = file("$testDirectory/build/aot/snapshot_aot_isolate").lengthSync();
int instructionsSize = file("$testDirectory/build/aot/snapshot_aot_instr").lengthSync();
int rodataSize = file("$testDirectory/build/aot/snapshot_aot_rodata").lengthSync();
int totalSize = vmisolateSize + isolateSize + instructionsSize + rodataSize;
Map<String, dynamic> data = <String, dynamic>{
'aot_snapshot_build_millis': watch.elapsedMilliseconds,
'aot_snapshot_size_vmisolate': vmisolateSize,
'aot_snapshot_size_isolate': isolateSize,
'aot_snapshot_size_instructions': instructionsSize,
'aot_snapshot_size_rodata': rodataSize,
'aot_snapshot_size_total': totalSize,
};
return new TaskResult.success(data, benchmarkScoreKeys: <String>[
'aot_snapshot_build_millis',
'aot_snapshot_size_vmisolate',
'aot_snapshot_size_isolate',
'aot_snapshot_size_instructions',
'aot_snapshot_size_rodata',
'aot_snapshot_size_total',
]);
});
}
}

75
dev/devicelab/lib/tasks/refresh.dart Normal file
View file

@ -0,0 +1,75 @@
// Copyright (c) 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'dart:io';
import 'package:path/path.dart' as path;
import '../framework/adb.dart';
import '../framework/benchmarks.dart';
import '../framework/framework.dart';
import '../framework/utils.dart';
TaskFunction createRefreshTest({ String commit, DateTime timestamp }) =>
new EditRefreshTask(commit, timestamp);
class EditRefreshTask {
EditRefreshTask(this.commit, this.timestamp) {
assert(commit != null);
assert(timestamp != null);
}
final String commit;
final DateTime timestamp;
Future<TaskResult> call() async {
Adb device = await adb();
await device.unlock();
Benchmark benchmark = new EditRefreshBenchmark(commit, timestamp);
section(benchmark.name);
await runBenchmark(benchmark, iterations: 3, warmUpBenchmark: true);
return benchmark.bestResult;
}
}
class EditRefreshBenchmark extends Benchmark {
EditRefreshBenchmark(this.commit, this.timestamp) : super('edit refresh');
final String commit;
final DateTime timestamp;
Directory get megaDir => dir(
path.join(flutterDirectory.path, 'dev/benchmarks/mega_gallery'));
File get benchmarkFile =>
file(path.join(megaDir.path, 'refresh_benchmark.json'));
@override
TaskResult get lastResult => new TaskResult.successFromFile(benchmarkFile);
@override
Future<Null> init() {
return inDirectory(flutterDirectory, () async {
await dart(<String>['dev/tools/mega_gallery.dart']);
});
}
@override
Future<num> run() async {
Adb device = await adb();
rm(benchmarkFile);
int exitCode = await inDirectory(megaDir, () async {
return await flutter('run',
options: <String>['-d', device.deviceId, '--benchmark'],
canFail: true);
});
if (exitCode != 0) return new Future<num>.error(exitCode);
return addBuildInfo(
benchmarkFile,
timestamp: timestamp,
expected: 200,
commit: commit,
);
}
}

38
dev/devicelab/lib/tasks/size_tests.dart Normal file
View file

@ -0,0 +1,38 @@
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:io';
import '../framework/framework.dart';
import '../framework/utils.dart';
TaskFunction createBasicMaterialAppSizeTest() {
return () async {
const String sampleAppName = 'sample_flutter_app';
Directory sampleDir = dir('${Directory.systemTemp.path}/$sampleAppName');
if (await sampleDir.exists())
rmTree(sampleDir);
int apkSizeInBytes;
await inDirectory(Directory.systemTemp, () async {
await flutter('create', options: <String>[sampleAppName]);
if (!(await sampleDir.exists()))
throw 'Failed to create sample Flutter app in ${sampleDir.path}';
await inDirectory(sampleDir, () async {
await pub('get');
await flutter('build', options: <String>['clean']);
await flutter('build', options: <String>['apk', '--release']);
apkSizeInBytes = await file('${sampleDir.path}/build/app.apk').length();
});
});
return new TaskResult.success(
<String, dynamic>{'release_size_in_bytes': apkSizeInBytes},
benchmarkScoreKeys: <String>['release_size_in_bytes']);
};
}

133
dev/devicelab/manifest.yaml Normal file
View file

@ -0,0 +1,133 @@
# Describes the tasks we run in the continuous integration (CI) environment.
#
# Cocoon[1] uses this file to generate a checklist of tasks to be performed for
# every master commit.
#
# [1] github.com/flutter/cocoon
# CI tasks.
#
# Each key in this dictionary is the unique name of a task, which also
# corresponds to a file in the "bin/" directory that the task runner will run.
#
# Due to historic reasons that may go away at some point, the suffix of the task
# name is significant. It is used by the dashboard to pick the right HTML
# template to display the results in a card. If you use a known name suffix also
# make sure that your task outputs data in the expected format for that card.
#
# Known suffixes:
#
# __analysis_time:
# Analyzer performance benchmarks.
# __refresh_time:
# Edit refresh cycle benchmarks.
# __start_up:
# Application startup speed benchmarks.
# __timeline_summary:
# Per-frame timings and missed/average/total counts.
# __transition_perf:
# Flutter Gallery app transitions benchmark.
# __size:
# Application size benchmarks.
tasks:
# Deviceless tests
# TODO: make these not require "has-android-device"; it is only there to
# ensure we have the Android SDK.
flutter_gallery__build:
description: >
Collects various performance metrics from AOT builds of the Flutter
Gallery.
stage: devicelab
required_agent_capabilities: ["has-android-device"]
complex_layout__build:
description: >
Collects various performance metrics from AOT builds of the Complex
Layout sample app.
stage: devicelab
required_agent_capabilities: ["has-android-device"]
basic_material_app__size:
description: >
Measures the APK/IPA sizes of a basic material app.
stage: devicelab
required_agent_capabilities: ["has-android-device"]
analyzer_cli__analysis_time:
description: >
Measures the speed of analyzing Flutter itself in batch mode.
stage: devicelab
required_agent_capabilities: ["has-android-device"]
analyzer_server__analysis_time:
description: >
Measures the speed of analyzing Flutter itself in server mode.
stage: devicelab
required_agent_capabilities: ["has-android-device"]
# Android on-device tests
complex_layout_scroll_perf__timeline_summary:
description: >
Measures the runtime performance of the Complex Layout sample app on
Android.
stage: devicelab
required_agent_capabilities: ["has-android-device"]
flutter_gallery__start_up:
description: >
Measures the startup time of the Flutter Gallery app on Android.
stage: devicelab
required_agent_capabilities: ["has-android-device"]
complex_layout__start_up:
description: >
Measures the startup time of the Complex Layout sample app on Android.
stage: devicelab
required_agent_capabilities: ["has-android-device"]
flutter_gallery__transition_perf:
description: >
Measures the performance of screen transitions in Flutter Gallery on
Android.
stage: devicelab
required_agent_capabilities: ["has-android-device"]
mega_gallery__refresh_time:
description: >
Measures AOT snapshot rebuild performance on a generated large app.
stage: devicelab
required_agent_capabilities: ["has-android-device"]
# iOS on-device tests
complex_layout_scroll_perf_ios__timeline_summary:
description: >
Measures the runtime performance of the Complex Layout sample app on
iOS.
stage: devicelab_ios
required_agent_capabilities: ["has-ios-device"]
flutter_gallery_ios__start_up:
stage: devicelab_ios
required_agent_capabilities: ["has-ios-device"]
description: >
Measures the startup time of the Flutter Gallery app on iOS.
complex_layout_ios__start_up:
description: >
Measures the startup time of the Complex Layout sample app on iOS.
stage: devicelab_ios
required_agent_capabilities: ["has-ios-device"]
flutter_gallery_ios__transition_perf:
stage: devicelab_ios
required_agent_capabilities: ["has-ios-device"]
description: >
Measures the performance of screen transitions in Flutter Gallery on
iOS.

22
dev/devicelab/pubspec.yaml Normal file
View file

@ -0,0 +1,22 @@
name: flutter_devicelab
version: 0.0.1
author: Flutter Authors <flutter-dev@googlegroups.com>
description: Flutter continuous integration performance and correctness tests.
homepage: https://github.com/flutter/flutter
environment:
sdk: '>=1.12.0 <2.0.0'
dependencies:
args: ^0.13.4
meta: ^1.0.3
path: ^1.3.0
stack_trace: ^1.4.0
vm_service_client: '^0.2.0'
# See packages/flutter_test/pubspec.yaml for why we're pinning this version
analyzer: 0.28.2-alpha.0
dev_dependencies:
# See packages/flutter_test/pubspec.yaml for why we're pinning this version
test: 0.12.15+4

190
dev/devicelab/test/adb_test.dart Normal file
View file

@ -0,0 +1,190 @@
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'package:test/test.dart';
import 'package:collection/collection.dart';
import 'package:flutter_devicelab/framework/adb.dart';
void main() {
group('adb', () {
Adb device;
setUp(() {
FakeAdb.resetLog();
adb = null;
device = new FakeAdb();
});
tearDown(() {
adb = realAdbGetter;
});
group('isAwake/isAsleep', () {
test('reads Awake', () async {
FakeAdb.pretendAwake();
expect(await device.isAwake(), isTrue);
expect(await device.isAsleep(), isFalse);
});
test('reads Asleep', () async {
FakeAdb.pretendAsleep();
expect(await device.isAwake(), isFalse);
expect(await device.isAsleep(), isTrue);
});
});
group('togglePower', () {
test('sends power event', () async {
await device.togglePower();
expectLog(<CommandArgs>[
cmd(command: 'input', arguments: <String>['keyevent', '26']),
]);
});
});
group('wakeUp', () {
test('when awake', () async {
FakeAdb.pretendAwake();
await device.wakeUp();
expectLog(<CommandArgs>[
cmd(command: 'dumpsys', arguments: <String>['power']),
]);
});
test('when asleep', () async {
FakeAdb.pretendAsleep();
await device.wakeUp();
expectLog(<CommandArgs>[
cmd(command: 'dumpsys', arguments: <String>['power']),
cmd(command: 'input', arguments: <String>['keyevent', '26']),
]);
});
});
group('sendToSleep', () {
test('when asleep', () async {
FakeAdb.pretendAsleep();
await device.sendToSleep();
expectLog(<CommandArgs>[
cmd(command: 'dumpsys', arguments: <String>['power']),
]);
});
test('when awake', () async {
FakeAdb.pretendAwake();
await device.sendToSleep();
expectLog(<CommandArgs>[
cmd(command: 'dumpsys', arguments: <String>['power']),
cmd(command: 'input', arguments: <String>['keyevent', '26']),
]);
});
});
group('unlock', () {
test('sends unlock event', () async {
FakeAdb.pretendAwake();
await device.unlock();
expectLog(<CommandArgs>[
cmd(command: 'dumpsys', arguments: <String>['power']),
cmd(command: 'input', arguments: <String>['keyevent', '82']),
]);
});
});
});
}
void expectLog(List<CommandArgs> log) {
expect(FakeAdb.commandLog, log);
}
CommandArgs cmd({ String command, List<String> arguments, Map<String, String> env }) => new CommandArgs(
command: command,
arguments: arguments,
env: env
);
typedef dynamic ExitErrorFactory();
class CommandArgs {
CommandArgs({ this.command, this.arguments, this.env });
final String command;
final List<String> arguments;
final Map<String, String> env;
@override
String toString() => 'CommandArgs(command: $command, arguments: $arguments, env: $env)';
@override
bool operator==(Object other) {
if (other.runtimeType != CommandArgs)
return false;
CommandArgs otherCmd = other;
return otherCmd.command == this.command &&
const ListEquality<String>().equals(otherCmd.arguments, this.arguments) &&
const MapEquality<String, String>().equals(otherCmd.env, this.env);
}
@override
int get hashCode => 17 * (17 * command.hashCode + _hashArguments) + _hashEnv;
int get _hashArguments => arguments != null
? const ListEquality<String>().hash(arguments)
: null.hashCode;
int get _hashEnv => env != null
? const MapEquality<String, String>().hash(env)
: null.hashCode;
}
class FakeAdb extends Adb {
FakeAdb({ String deviceId: null }) : super(deviceId: deviceId);
static String output = '';
static ExitErrorFactory exitErrorFactory = () => null;
static List<CommandArgs> commandLog = <CommandArgs>[];
static void resetLog() {
commandLog.clear();
}
static void pretendAwake() {
output = '''
mWakefulness=Awake
''';
}
static void pretendAsleep() {
output = '''
mWakefulness=Asleep
''';
}
@override
Future<String> shellEval(String command, List<String> arguments, {Map<String, String> env}) async {
commandLog.add(new CommandArgs(
command: command,
arguments: arguments,
env: env
));
return output;
}
@override
Future<Null> shellExec(String command, List<String> arguments, {Map<String, String> env}) async {
commandLog.add(new CommandArgs(
command: command,
arguments: arguments,
env: env
));
dynamic exitError = exitErrorFactory();
if (exitError != null)
throw exitError;
}
}

13
dev/devicelab/test/all.dart Normal file
View file

@ -0,0 +1,13 @@
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'adb_test.dart' as adb_test;
import 'manifest_test.dart' as manifest_test;
import 'run_test.dart' as run_test;
void main() {
adb_test.main();
manifest_test.main();
run_test.main();
}

143
dev/devicelab/test/manifest_test.dart Normal file
View file

@ -0,0 +1,143 @@
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'package:test/test.dart';
import 'package:flutter_devicelab/framework/manifest.dart';
void main() {
group('production manifest', () {
test('must be valid', () {
Manifest manifest = loadTaskManifest();
expect(manifest.tasks, isNotEmpty);
ManifestTask task = manifest.tasks.firstWhere((ManifestTask task) => task.name == 'flutter_gallery__start_up');
expect(task.description, 'Measures the startup time of the Flutter Gallery app on Android.\n');
expect(task.stage, 'devicelab');
expect(task.requiredAgentCapabilities, <String>['has-android-device']);
});
});
group('manifest parser', () {
void testManifestError(
String testDescription,
String errorMessage,
String yaml,
) {
test(testDescription, () {
try {
loadTaskManifest(yaml);
} on ManifestError catch(error) {
expect(error.message, errorMessage);
}
});
}
testManifestError(
'invalid top-level type',
'Manifest must be a dictionary but was YamlScalar: null',
'',
);
testManifestError(
'invalid top-level key',
'Unrecognized property "bad" in manifest. Allowed properties: tasks',
'''
bad:
key: yes
''',
);
testManifestError(
'invalid tasks list type',
'Value of "tasks" must be a dictionary but was YamlList: [a, b]',
'''
tasks:
- a
- b
'''
);
testManifestError(
'invalid task name type',
'Task name must be a string but was int: 1',
'''
tasks:
1: 2
'''
);
testManifestError(
'invalid task type',
'Value of task "foo" must be a dictionary but was int: 2',
'''
tasks:
foo: 2
'''
);
testManifestError(
'invalid task property',
'Unrecognized property "bar" in Value of task "foo". Allowed properties: description, stage, required_agent_capabilities',
'''
tasks:
foo:
bar: 2
'''
);
testManifestError(
'invalid required_agent_capabilities type',
'required_agent_capabilities must be a list but was int: 1',
'''
tasks:
foo:
required_agent_capabilities: 1
'''
);
testManifestError(
'invalid required_agent_capabilities element type',
'required_agent_capabilities[0] must be a string but was int: 1',
'''
tasks:
foo:
required_agent_capabilities: [1]
'''
);
testManifestError(
'missing description',
'Task description must not be empty in task "foo".',
'''
tasks:
foo:
required_agent_capabilities: ["a"]
'''
);
testManifestError(
'missing stage',
'Task stage must not be empty in task "foo".',
'''
tasks:
foo:
description: b
required_agent_capabilities: ["a"]
'''
);
testManifestError(
'missing stage',
'requiredAgentCapabilities must not be empty in task "foo".',
'''
tasks:
foo:
description: b
stage: c
required_agent_capabilities: []
'''
);
});
}

50
dev/devicelab/test/run_test.dart Normal file
View file

@ -0,0 +1,50 @@
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'dart:io';
import 'package:test/test.dart';
void main() {
group('run.dart script', () {
Future<int> runScript(List<String> testNames) async {
List<String> options = <String>['bin/run.dart'];
for (String testName in testNames) {
options..addAll(<String>['-t', testName]);
}
Process scriptProcess = await Process.start(
'../../bin/cache/dart-sdk/bin/dart',
options,
);
return scriptProcess.exitCode;
}
test('Exits with code 0 when succeeds', () async {
expect(await runScript(<String>['smoke_test_success']), 0);
});
test('Exits with code 1 when task throws', () async {
expect(await runScript(<String>['smoke_test_throws']), 1);
});
test('Exits with code 1 when fails', () async {
expect(await runScript(<String>['smoke_test_failure']), 1);
});
test('Exits with code 1 when fails to connect', () async {
expect(await runScript(<String>['smoke_test_setup_failure']), 1);
});
test('Exits with code 1 when results are mixed', () async {
expect(
await runScript(<String>[
'smoke_test_failure',
'smoke_test_success',
]),
1,
);
});
});
}

18
dev/devicelab/test/utils_test.dart Normal file
View file

@ -0,0 +1,18 @@
// Copyright (c) 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'package:test/test.dart';
import 'package:flutter_devicelab/framework/utils.dart';
void main() {
group('grep', () {
test('greps lines', () {
expect(grep('b', from: 'ab\ncd\nba'), <String>['ab', 'ba']);
});
test('understands RegExp', () {
expect(grep(new RegExp('^b'), from: 'ab\nba'), <String>['ba']);
});
});
}

2
packages/flutter/pubspec.yaml
View file

@ -7,7 +7,7 @@ homepage: http://flutter.io
dependencies:
collection: '>=1.9.1 <2.0.0'
intl: '>=0.14.0 <0.15.0'
meta: ^1.0.2
meta: ^1.0.3
vector_math: '>=2.0.3 <3.0.0'
sky_engine:

2
packages/flutter_tools/pubspec.yaml
View file

@ -22,7 +22,7 @@ dependencies:
json_schema: 1.0.3
linter: ^0.1.21
meta: ^1.0.0
meta: ^1.0.3
mustache4dart: ^1.0.0
package_config: '>=0.1.5 <2.0.0'
path: ^1.3.0