2016-09-14 20:22:53 +00:00
|
|
|
# Describes the tasks we run in the continuous integration (CI) environment.
|
|
|
|
|
#
|
|
|
|
|
# Cocoon[1] uses this file to generate a checklist of tasks to be performed for
|
|
|
|
|
# every master commit.
|
|
|
|
|
#
|
|
|
|
|
# [1] github.com/flutter/cocoon
|
|
|
|
|
|
|
|
|
|
# CI tasks.
|
|
|
|
|
#
|
|
|
|
|
# Each key in this dictionary is the unique name of a task, which also
|
|
|
|
|
# corresponds to a file in the "bin/" directory that the task runner will run.
|
|
|
|
|
#
|
2017-04-06 23:27:29 +00:00
|
|
|
# Required properties:
|
2016-09-14 20:22:53 +00:00
|
|
|
#
|
2017-04-06 23:27:29 +00:00
|
|
|
# * description: free form string
|
|
|
|
|
# describes what the task is about
|
|
|
|
|
# * stage: one of "devicelab", "devicelab_ios", "devicelab_win"
|
|
|
|
|
# tasks are grouped by stage so they appear next to each on the dashboard and have a distinct
|
|
|
|
|
# icon attached to them. Stages can be used to create dependencies, e.g. we may decide to not
|
|
|
|
|
# run device tests on red Travis.
|
|
|
|
|
# * required_agent_capabilities: a list of strings
|
|
|
|
|
# list of capabilities a devicelab agent must have to be able to run this task.
|
2016-09-14 20:22:53 +00:00
|
|
|
#
|
2017-04-06 23:27:29 +00:00
|
|
|
# Optional properties:
|
|
|
|
|
#
|
|
|
|
|
# * flaky: boolean true or false
|
|
|
|
|
# whether the task is considered flaky; the result of running a flaky task does not affect
|
2017-05-11 15:59:10 +00:00
|
|
|
# the overall build status. Tests should be marked flaky when newly added, until they have
|
|
|
|
|
# been proved for a few cycles. Tests that are actually flaky but not being actively worked
|
|
|
|
|
# on should be hidden because they just cause confusion otherwise.
|
2017-04-06 23:27:29 +00:00
|
|
|
# * timeout_in_minutes: integer
|
|
|
|
|
# a custom task timeout, specified in minutes.
|
2016-09-14 20:22:53 +00:00
|
|
|
|
|
|
|
|
tasks:
|
|
|
|
|
# Deviceless tests
|
|
|
|
|
|
|
|
|
|
# TODO: make these not require "has-android-device"; it is only there to
|
|
|
|
|
# ensure we have the Android SDK.
|
|
|
|
|
|
|
|
|
|
complex_layout__build:
|
|
|
|
|
description: >
|
|
|
|
|
Collects various performance metrics from AOT builds of the Complex
|
|
|
|
|
Layout sample app.
|
|
|
|
|
stage: devicelab
|
|
|
|
|
required_agent_capabilities: ["has-android-device"]
|
|
|
|
|
|
|
|
|
|
basic_material_app__size:
|
|
|
|
|
description: >
|
2016-12-05 20:46:59 +00:00
|
|
|
Measures the APK size of a basic material app.
|
2016-09-14 20:22:53 +00:00
|
|
|
stage: devicelab
|
|
|
|
|
required_agent_capabilities: ["has-android-device"]
|
|
|
|
|
|
|
|
|
|
# Android on-device tests
|
|
|
|
|
|
|
|
|
|
complex_layout_scroll_perf__timeline_summary:
|
|
|
|
|
description: >
|
|
|
|
|
Measures the runtime performance of the Complex Layout sample app on
|
|
|
|
|
Android.
|
|
|
|
|
stage: devicelab
|
|
|
|
|
required_agent_capabilities: ["has-android-device"]
|
|
|
|
|
|
2017-04-27 06:44:28 +00:00
|
|
|
channels_integration_test:
|
|
|
|
|
description: >
|
|
|
|
|
Checks that platform channels work on Android.
|
|
|
|
|
stage: devicelab
|
|
|
|
|
required_agent_capabilities: ["has-android-device"]
|
|
|
|
|
|
|
|
|
|
platform_channel_sample_test:
|
2017-03-20 21:19:03 +00:00
|
|
|
description: >
|
2017-03-27 09:17:31 +00:00
|
|
|
Runs a driver test on the Platform Channel sample app on Android.
|
2017-03-20 21:19:03 +00:00
|
|
|
stage: devicelab
|
|
|
|
|
required_agent_capabilities: ["has-android-device"]
|
|
|
|
|
|
2016-09-14 20:22:53 +00:00
|
|
|
complex_layout__start_up:
|
|
|
|
|
description: >
|
|
|
|
|
Measures the startup time of the Complex Layout sample app on Android.
|
|
|
|
|
stage: devicelab
|
|
|
|
|
required_agent_capabilities: ["has-android-device"]
|
|
|
|
|
|
2016-09-19 17:03:26 +00:00
|
|
|
hot_mode_dev_cycle__benchmark:
|
|
|
|
|
description: >
|
|
|
|
|
Measures the performance of Dart VM hot patching feature.
|
|
|
|
|
stage: devicelab
|
|
|
|
|
required_agent_capabilities: ["has-android-device"]
|
|
|
|
|
|
2016-12-01 19:37:16 +00:00
|
|
|
complex_layout_scroll_perf__memory:
|
|
|
|
|
description: >
|
|
|
|
|
Measures memory usage of the scroll performance test.
|
|
|
|
|
stage: devicelab
|
|
|
|
|
required_agent_capabilities: ["has-android-device"]
|
|
|
|
|
|
2016-12-03 00:50:27 +00:00
|
|
|
hello_world__memory:
|
|
|
|
|
description: >
|
|
|
|
|
Measures starting memory usage of the simplest Flutter app.
|
|
|
|
|
stage: devicelab
|
|
|
|
|
required_agent_capabilities: ["has-android-device"]
|
|
|
|
|
|
2017-01-09 22:57:14 +00:00
|
|
|
microbenchmarks:
|
|
|
|
|
description: >
|
|
|
|
|
Runs benchmarks from dev/benchmarks/microbenchmarks.
|
|
|
|
|
stage: devicelab
|
|
|
|
|
required_agent_capabilities: ["has-android-device"]
|
2017-04-06 17:48:45 +00:00
|
|
|
timeout_in_minutes: 30
|
2017-01-09 22:57:14 +00:00
|
|
|
|
2017-03-23 12:20:00 +00:00
|
|
|
flutter_view__start_up:
|
|
|
|
|
description: >
|
|
|
|
|
Verifies that Flutter View can be used from an Android project.
|
|
|
|
|
stage: devicelab
|
|
|
|
|
required_agent_capabilities: ["has-android-device"]
|
|
|
|
|
|
2017-05-11 21:29:20 +00:00
|
|
|
integration_ui:
|
|
|
|
|
description: >
|
|
|
|
|
Runs end-to-end Flutter tests on Android.
|
|
|
|
|
stage: devicelab
|
|
|
|
|
required_agent_capabilities: ["has-android-device"]
|
|
|
|
|
|
2017-05-25 18:47:36 +00:00
|
|
|
android_sample_catalog_generator:
|
|
|
|
|
description: >
|
|
|
|
|
Builds sample catalog markdown pages and Android screenshots
|
|
|
|
|
stage: devicelab
|
2017-06-14 16:05:53 +00:00
|
|
|
required_agent_capabilities: ["has-android-device"]
|
2017-05-25 18:47:36 +00:00
|
|
|
|
2017-06-13 19:49:07 +00:00
|
|
|
complex_layout_semantics_perf:
|
|
|
|
|
description: >
|
|
|
|
|
Measures duration of building the initial semantics tree.
|
|
|
|
|
stage: devicelab
|
|
|
|
|
required_agent_capabilities: ["linux/android"]
|
Deep linking: automatically push the route hiearchy on load. (#10894)
The main purpose of this PR is to make it so that when you set the
initial route and it's a hierarchical route (e.g. `/a/b/c`), it
implies multiple pushes, one for each step of the route (so in that
case, `/`, `/a`, `/a/b`, and `/a/b/c`, in that order). If any of those
routes don't exist, it falls back to '/'.
As part of doing that, I:
* Changed the default for MaterialApp.initialRoute to honor the
actual initial route.
* Added a MaterialApp.onUnknownRoute for handling bad routes.
* Added a feature to flutter_driver that allows the host test script
and the device test app to communicate.
* Added a test to make sure `flutter drive --route` works.
(Hopefully that will also prove `flutter run --route` works, though
this isn't testing the `flutter` tool's side of that. My main
concern is over whether the engine side works.)
* Fixed `flutter drive` to output the right target file name.
* Changed how the stocks app represents its data, so that we can
show a page for a stock before we know if it exists.
* Made it possible to show a stock page that doesn't exist. It shows
a progress indicator if we're loading the data, or else shows a
message saying it doesn't exist.
* Changed the pathing structure of routes in stocks to work more
sanely.
* Made search in the stocks app actually work (before it only worked
if we happened to accidentally trigger a rebuild). Added a test.
* Replaced some custom code in the stocks app with a BackButton.
* Added a "color" feature to BackButton to support the stocks use case.
* Spaced out the ErrorWidget text a bit more.
* Added `RouteSettings.copyWith`, which I ended up not using.
* Improved the error messages around routing.
While I was in some files I made a few formatting fixes, fixed some
code health issues, and also removed `flaky: true` from some devicelab
tests that have been stable for a while. Also added some documentation
here and there.
2017-06-23 21:58:29 +00:00
|
|
|
|
2017-06-24 01:13:40 +00:00
|
|
|
routing_test:
|
Deep linking: automatically push the route hiearchy on load. (#10894)
The main purpose of this PR is to make it so that when you set the
initial route and it's a hierarchical route (e.g. `/a/b/c`), it
implies multiple pushes, one for each step of the route (so in that
case, `/`, `/a`, `/a/b`, and `/a/b/c`, in that order). If any of those
routes don't exist, it falls back to '/'.
As part of doing that, I:
* Changed the default for MaterialApp.initialRoute to honor the
actual initial route.
* Added a MaterialApp.onUnknownRoute for handling bad routes.
* Added a feature to flutter_driver that allows the host test script
and the device test app to communicate.
* Added a test to make sure `flutter drive --route` works.
(Hopefully that will also prove `flutter run --route` works, though
this isn't testing the `flutter` tool's side of that. My main
concern is over whether the engine side works.)
* Fixed `flutter drive` to output the right target file name.
* Changed how the stocks app represents its data, so that we can
show a page for a stock before we know if it exists.
* Made it possible to show a stock page that doesn't exist. It shows
a progress indicator if we're loading the data, or else shows a
message saying it doesn't exist.
* Changed the pathing structure of routes in stocks to work more
sanely.
* Made search in the stocks app actually work (before it only worked
if we happened to accidentally trigger a rebuild). Added a test.
* Replaced some custom code in the stocks app with a BackButton.
* Added a "color" feature to BackButton to support the stocks use case.
* Spaced out the ErrorWidget text a bit more.
* Added `RouteSettings.copyWith`, which I ended up not using.
* Improved the error messages around routing.
While I was in some files I made a few formatting fixes, fixed some
code health issues, and also removed `flaky: true` from some devicelab
tests that have been stable for a while. Also added some documentation
here and there.
2017-06-23 21:58:29 +00:00
|
|
|
description: >
|
|
|
|
|
Verifies that `flutter drive --route` still works. No performance numbers.
|
|
|
|
|
stage: devicelab
|
|
|
|
|
required_agent_capabilities: ["linux/android"]
|
2017-07-11 00:20:49 +00:00
|
|
|
flaky: true
|
|
|
|
|
|
|
|
|
|
flutter_gallery_instrumentation_test:
|
|
|
|
|
description: >
|
|
|
|
|
Same as flutter_gallery__transition_perf but uses Android instrumentation
|
|
|
|
|
framework, and therefore does not require a host computer to run. This
|
|
|
|
|
test can run on off-the-shelf infrastructures, such as Firebase Test Lab.
|
|
|
|
|
stage: devicelab
|
|
|
|
|
required_agent_capabilities: ["linux/android"]
|
2017-06-13 19:49:07 +00:00
|
|
|
flaky: true
|
|
|
|
|
|
2016-09-14 20:22:53 +00:00
|
|
|
# iOS on-device tests
|
|
|
|
|
|
2017-04-27 06:44:28 +00:00
|
|
|
channels_integration_test_ios:
|
|
|
|
|
description: >
|
|
|
|
|
Checks that platform channels work on iOS.
|
|
|
|
|
stage: devicelab_ios
|
|
|
|
|
required_agent_capabilities: ["has-ios-device"]
|
2017-06-10 00:30:04 +00:00
|
|
|
flaky: true
|
2017-04-27 06:44:28 +00:00
|
|
|
|
|
|
|
|
platform_channel_sample_test_ios:
|
2017-03-20 21:19:03 +00:00
|
|
|
description: >
|
2017-03-27 09:17:31 +00:00
|
|
|
Runs a driver test on the Platform Channel sample app on iOS.
|
2017-03-20 21:19:03 +00:00
|
|
|
stage: devicelab_ios
|
|
|
|
|
required_agent_capabilities: ["has-ios-device"]
|
2017-06-10 00:30:04 +00:00
|
|
|
flaky: true
|
2017-03-20 21:19:03 +00:00
|
|
|
|
2016-09-14 20:22:53 +00:00
|
|
|
complex_layout_scroll_perf_ios__timeline_summary:
|
|
|
|
|
description: >
|
|
|
|
|
Measures the runtime performance of the Complex Layout sample app on
|
|
|
|
|
iOS.
|
|
|
|
|
stage: devicelab_ios
|
|
|
|
|
required_agent_capabilities: ["has-ios-device"]
|
2017-06-10 00:30:04 +00:00
|
|
|
flaky: true
|
2016-09-14 20:22:53 +00:00
|
|
|
|
2017-05-11 15:59:10 +00:00
|
|
|
# flutter_gallery_ios__start_up:
|
|
|
|
|
# description: >
|
|
|
|
|
# Measures the startup time of the Flutter Gallery app on iOS.
|
|
|
|
|
# stage: devicelab_ios
|
|
|
|
|
# required_agent_capabilities: ["has-ios-device"]
|
2017-05-20 04:48:51 +00:00
|
|
|
# flaky: true
|
2016-09-14 20:22:53 +00:00
|
|
|
|
2017-05-11 15:59:10 +00:00
|
|
|
# complex_layout_ios__start_up:
|
|
|
|
|
# description: >
|
|
|
|
|
# Measures the startup time of the Complex Layout sample app on iOS.
|
|
|
|
|
# stage: devicelab_ios
|
|
|
|
|
# required_agent_capabilities: ["has-ios-device"]
|
2017-05-20 04:48:51 +00:00
|
|
|
# flaky: true
|
2016-09-14 20:22:53 +00:00
|
|
|
|
2017-03-30 02:19:57 +00:00
|
|
|
flutter_gallery_ios__transition_perf:
|
|
|
|
|
description: >
|
|
|
|
|
Measures the performance of screen transitions in Flutter Gallery on
|
|
|
|
|
iOS.
|
|
|
|
|
stage: devicelab_ios
|
|
|
|
|
required_agent_capabilities: ["has-ios-device"]
|
2017-05-31 22:40:14 +00:00
|
|
|
flaky: true
|
2016-12-05 20:46:59 +00:00
|
|
|
|
|
|
|
|
basic_material_app_ios__size:
|
|
|
|
|
description: >
|
|
|
|
|
Measures the IPA size of a basic material app.
|
|
|
|
|
stage: devicelab_ios
|
2017-01-10 17:30:00 +00:00
|
|
|
required_agent_capabilities: ["has-ios-device"]
|
2017-06-10 00:30:04 +00:00
|
|
|
flaky: true
|
2017-01-10 17:30:00 +00:00
|
|
|
|
2017-05-11 15:59:10 +00:00
|
|
|
# microbenchmarks_ios:
|
|
|
|
|
# description: >
|
|
|
|
|
# Runs benchmarks from dev/benchmarks/microbenchmarks on iOS.
|
|
|
|
|
# stage: devicelab_ios
|
|
|
|
|
# required_agent_capabilities: ["has-ios-device"]
|
|
|
|
|
# timeout_in_minutes: 30
|
|
|
|
|
# flaky: true
|
|
|
|
|
|
|
|
|
|
# flutter_view_ios__start_up:
|
|
|
|
|
# description: >
|
|
|
|
|
# Verifies that Flutter View can be used from an iOS project.
|
|
|
|
|
# stage: devicelab_ios
|
|
|
|
|
# required_agent_capabilities: ["has-ios-device"]
|
2017-05-20 04:48:51 +00:00
|
|
|
# flaky: true
|
2017-03-24 10:45:57 +00:00
|
|
|
|
2017-05-11 21:29:20 +00:00
|
|
|
integration_ui_ios:
|
|
|
|
|
description: >
|
|
|
|
|
Runs end-to-end Flutter tests on iOS.
|
|
|
|
|
stage: devicelab_ios
|
|
|
|
|
required_agent_capabilities: ["has-ios-device"]
|
2017-06-10 00:30:04 +00:00
|
|
|
flaky: true
|
2017-05-11 21:29:20 +00:00
|
|
|
|
2017-05-25 18:47:36 +00:00
|
|
|
ios_sample_catalog_generator:
|
|
|
|
|
description: >
|
|
|
|
|
Builds sample catalog markdown pages and iOS screenshots
|
|
|
|
|
stage: devicelab_ios
|
|
|
|
|
required_agent_capabilities: ["has-ios-device"]
|
|
|
|
|
flaky: true
|
|
|
|
|
|
2017-03-16 20:47:14 +00:00
|
|
|
# Tests running on Windows host
|
|
|
|
|
|
2017-04-27 06:44:28 +00:00
|
|
|
channels_integration_test_win:
|
|
|
|
|
description: >
|
|
|
|
|
Checks that platform channels work when app is launched from Windows.
|
|
|
|
|
stage: devicelab_win
|
|
|
|
|
required_agent_capabilities: ["windows"]
|
|
|
|
|
|
2017-03-16 20:47:14 +00:00
|
|
|
flutter_gallery_win__build:
|
|
|
|
|
description: >
|
|
|
|
|
Collects various performance metrics from AOT builds of the Flutter
|
|
|
|
|
Gallery on Windows.
|
|
|
|
|
stage: devicelab_win
|
|
|
|
|
required_agent_capabilities: ["windows"]
|
|
|
|
|
|
2017-03-28 21:22:03 +00:00
|
|
|
hot_mode_dev_cycle_win__benchmark:
|
|
|
|
|
description: >
|
|
|
|
|
Measures the performance of Dart VM hot patching feature on Windows.
|
|
|
|
|
stage: devicelab_win
|
|
|
|
|
required_agent_capabilities: ["windows"]
|
2017-05-31 19:43:18 +00:00
|
|
|
|
|
|
|
|
# Tests running on Linux hosts
|
|
|
|
|
|
|
|
|
|
hot_mode_dev_cycle_linux__benchmark:
|
|
|
|
|
description: >
|
|
|
|
|
Measures the performance of Dart VM hot patching feature on a Linux host.
|
|
|
|
|
stage: devicelab
|
|
|
|
|
required_agent_capabilities: ["linux/android"]
|
|
|
|
|
|
|
|
|
|
dartdocs:
|
|
|
|
|
description: >
|
|
|
|
|
Tracks how many members are still lacking documentation.
|
|
|
|
|
stage: devicelab
|
|
|
|
|
required_agent_capabilities: ["linux/android"]
|
|
|
|
|
|
|
|
|
|
technical_debt__cost:
|
|
|
|
|
description: >
|
|
|
|
|
Estimates our technical debt (TODOs, analyzer ignores, etc).
|
|
|
|
|
stage: devicelab
|
|
|
|
|
required_agent_capabilities: ["linux/android"]
|
2017-05-31 22:13:08 +00:00
|
|
|
|
|
|
|
|
flutter_gallery__build:
|
|
|
|
|
description: >
|
|
|
|
|
Collects various performance metrics from AOT builds of the Flutter
|
|
|
|
|
Gallery.
|
|
|
|
|
stage: devicelab
|
|
|
|
|
required_agent_capabilities: ["linux/android"]
|
|
|
|
|
|
|
|
|
|
flutter_gallery__start_up:
|
|
|
|
|
description: >
|
|
|
|
|
Measures the startup time of the Flutter Gallery app on Android.
|
|
|
|
|
stage: devicelab
|
|
|
|
|
required_agent_capabilities: ["linux/android"]
|
|
|
|
|
|
|
|
|
|
flutter_gallery__transition_perf:
|
|
|
|
|
description: >
|
|
|
|
|
Measures the performance of screen transitions in Flutter Gallery on
|
|
|
|
|
Android.
|
|
|
|
|
stage: devicelab
|
|
|
|
|
required_agent_capabilities: ["linux/android"]
|
|
|
|
|
|
2017-06-14 15:09:05 +00:00
|
|
|
flutter_gallery__transition_perf_with_semantics:
|
|
|
|
|
description: >
|
|
|
|
|
Measures the delta in performance of screen transitions without and
|
|
|
|
|
with semantics enabled.
|
|
|
|
|
stage: devicelab
|
|
|
|
|
required_agent_capabilities: ["linux/android"]
|
2017-07-18 21:57:01 +00:00
|
|
|
flaky: true
|
2017-06-14 15:09:05 +00:00
|
|
|
|
2017-05-31 22:13:08 +00:00
|
|
|
flutter_gallery__memory_nav:
|
|
|
|
|
description: >
|
|
|
|
|
Measures memory usage after repeated navigation in Gallery.
|
|
|
|
|
stage: devicelab
|
|
|
|
|
required_agent_capabilities: ["linux/android"]
|
|
|
|
|
|
|
|
|
|
flutter_gallery__back_button_memory:
|
|
|
|
|
description: >
|
|
|
|
|
Measures memory usage after Android app suspend and resume.
|
|
|
|
|
stage: devicelab
|
|
|
|
|
required_agent_capabilities: ["linux/android"]
|
2017-06-13 20:14:16 +00:00
|
|
|
|
|
|
|
|
analyzer_benchmark:
|
|
|
|
|
description: >
|
|
|
|
|
Measures the speed of Dart analyzer.
|
|
|
|
|
stage: devicelab
|
|
|
|
|
required_agent_capabilities: ["linux/android"]
|