development/flutter
development/flutter
1
0
Fork 0
mirror of https://github.com/flutter/flutter synced 2024-10-13 03:32:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

[devicelab] reduce iterations, uninstall at end, and use --application-binary in all startup tests (#68516)

Browse source 
These devicelab tests are incredibly flaky. Currently they will build & install the same application 15 times. This causes the temp storage to fill up on android, and has a good chance of flaking on iOS due to install issues.

The change from 3 to 15 increased total test time for 2 -> 12 minutes on iOS, or roughly 5 more test equivalents for every single iOS test. Reduce the iteration count back to 5

Uninstall the app after each run so temp storage does not fill up.
This commit is contained in:
Jonah Williams 2020-10-19 14:16:04 -07:00 committed by GitHub
parent 98c519dbe2
commit a4c2075a45
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 38 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

44
dev/devicelab/lib/tasks/perf_tests.dart
View file

@ -454,24 +454,56 @@ class StartupTest {
Future<TaskResult> run() async {
return await inDirectory<TaskResult>(testDirectory, () async {
final String deviceId = (await devices.workingDevice).deviceId;
await flutter('packages', options: <String>['get']);
const int iterations = 15;
final Device device = await devices.workingDevice;
const int iterations = 5;
final List<Map<String, dynamic>> results = <Map<String, dynamic>>[];
for (int i = 0; i < iterations; ++i) {
section('Building application');
String applicationBinaryPath;
switch (deviceOperatingSystem) {
case DeviceOperatingSystem.android:
await flutter('build', options: <String>[
'apk',
'-v',
'--profile',
'--target-platform=android-arm,android-arm64',
]);
applicationBinaryPath = '$testDirectory/build/app/outputs/flutter-apk/app-profile.apk';
break;
case DeviceOperatingSystem.ios:
await flutter('build', options: <String>[
'ios',
'-v',
'--profile',
]);
applicationBinaryPath = '$testDirectory/build/ios/iphoneos/Runner.app';
break;
case DeviceOperatingSystem.fuchsia:
case DeviceOperatingSystem.fake:
break;
}
for (int i = 0; i < iterations; i += 1) {
await flutter('run', options: <String>[
'--no-android-gradle-daemon',
'--verbose',
'--profile',
'--trace-startup',
'-d',
deviceId,
device.deviceId,
if (applicationBinaryPath != null)
'--use-application-binary=$applicationBinaryPath',
]);
final Map<String, dynamic> data = json.decode(
file('$testDirectory/build/start_up_info.json').readAsStringSync(),
) as Map<String, dynamic>;
results.add(data);
await flutter('install', options: <String>[
'--uninstall-only',
'-d',
device.deviceId,
]);
}
final Map<String, dynamic> averageResults = _average(results, iterations);