mirror of
https://github.com/flutter/flutter
synced 2024-08-25 02:46:31 +00:00
Rerun devicelab task from test runner (#86394)
This commit is contained in:
parent
d056500bfe
commit
4d96a3fd50
|
@ -27,7 +27,9 @@ When a device in the lab is free, it will pickup tasks that need to be completed
|
|||
|
||||
1. If the task succeeds, the test runner reports the success and uploads its performance metrics to Flutter's infrastructure. Not
|
||||
all tasks record performance metrics.
|
||||
2. If the task fails, the test runner reports the failure to Flutter's infrastructure and no performance metrics are collected
|
||||
2. If task fails, an auto rerun happens. Whenever the last run succeeds, the task will be reported as a success. For this case,
|
||||
a flake will be flagged and populated to the test result.
|
||||
3. If the task fails in all reruns, the test runner reports the failure to Flutter's infrastructure and no performance metrics are collected
|
||||
|
||||
## Running tests locally
|
||||
|
||||
|
|
|
@ -47,6 +47,9 @@ class Cocoon {
|
|||
/// Url used to send results to.
|
||||
static const String baseCocoonApiUrl = 'https://flutter-dashboard.appspot.com/api';
|
||||
|
||||
/// Threshold to auto retry a failed test.
|
||||
static const int retryNumber = 2;
|
||||
|
||||
/// Underlying [FileSystem] to use.
|
||||
final FileSystem fs;
|
||||
|
||||
|
|
|
@ -4,6 +4,7 @@
|
|||
|
||||
import 'dart:async';
|
||||
import 'dart:convert';
|
||||
// import 'dart:core' as core;
|
||||
import 'dart:io';
|
||||
|
||||
import 'package:flutter_devicelab/common.dart';
|
||||
|
@ -15,6 +16,16 @@ import 'devices.dart';
|
|||
import 'task_result.dart';
|
||||
import 'utils.dart';
|
||||
|
||||
/// Run a list of tasks.
|
||||
///
|
||||
/// For each task, an auto rerun will be triggered when task fails.
|
||||
///
|
||||
/// If the task succeeds the first time, it will be recorded as successful.
|
||||
///
|
||||
/// If the task fails first, but gets passed in the end, the
|
||||
/// test will be recorded as successful but with a flake flag.
|
||||
///
|
||||
/// If the task fails all reruns, it will be recorded as failed.
|
||||
Future<void> runTasks(
|
||||
List<String> taskNames, {
|
||||
bool exitOnFirstTestFailure = false,
|
||||
|
@ -26,33 +37,45 @@ Future<void> runTasks(
|
|||
String? luciBuilder,
|
||||
String? resultsPath,
|
||||
List<String>? taskArgs,
|
||||
@visibleForTesting Map<String, String>? isolateParams,
|
||||
@visibleForTesting Function(String) print = print,
|
||||
@visibleForTesting List<String>? logs,
|
||||
}) async {
|
||||
for (final String taskName in taskNames) {
|
||||
section('Running task "$taskName"');
|
||||
final TaskResult result = await runTask(
|
||||
taskName,
|
||||
deviceId: deviceId,
|
||||
localEngine: localEngine,
|
||||
localEngineSrcPath: localEngineSrcPath,
|
||||
silent: silent,
|
||||
taskArgs: taskArgs,
|
||||
);
|
||||
|
||||
print('Task result:');
|
||||
print(const JsonEncoder.withIndent(' ').convert(result));
|
||||
section('Finished task "$taskName"');
|
||||
|
||||
if (resultsPath != null) {
|
||||
final Cocoon cocoon = Cocoon();
|
||||
await cocoon.writeTaskResultToFile(
|
||||
builderName: luciBuilder,
|
||||
gitBranch: gitBranch,
|
||||
result: result,
|
||||
TaskResult result = TaskResult.success(null);
|
||||
int retry = 0;
|
||||
while (retry <= Cocoon.retryNumber) {
|
||||
result = await rerunTask(
|
||||
taskName,
|
||||
deviceId: deviceId,
|
||||
localEngine: localEngine,
|
||||
localEngineSrcPath: localEngineSrcPath,
|
||||
silent: silent,
|
||||
taskArgs: taskArgs,
|
||||
resultsPath: resultsPath,
|
||||
gitBranch: gitBranch,
|
||||
luciBuilder: luciBuilder,
|
||||
isolateParams: isolateParams,
|
||||
);
|
||||
|
||||
section('Flaky status for "$taskName"');
|
||||
if (!result.succeeded) {
|
||||
retry++;
|
||||
} else {
|
||||
if (retry > 0) {
|
||||
print('Total ${retry+1} executions: $retry failures and 1 success');
|
||||
print('flaky: true');
|
||||
} else {
|
||||
print('Total ${retry+1} executions: 1 success');
|
||||
print('flaky: false');
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!result.succeeded) {
|
||||
print('Total $retry executions: 0 success');
|
||||
print('flaky: false');
|
||||
exitCode = 1;
|
||||
if (exitOnFirstTestFailure) {
|
||||
return;
|
||||
|
@ -61,6 +84,48 @@ Future<void> runTasks(
|
|||
}
|
||||
}
|
||||
|
||||
/// A rerun wrapper for `runTask`.
|
||||
///
|
||||
/// This separates reruns in separate sections.
|
||||
Future<TaskResult> rerunTask(
|
||||
String taskName, {
|
||||
String? deviceId,
|
||||
String? localEngine,
|
||||
String? localEngineSrcPath,
|
||||
bool silent = false,
|
||||
List<String>? taskArgs,
|
||||
String? resultsPath,
|
||||
String? gitBranch,
|
||||
String? luciBuilder,
|
||||
@visibleForTesting Map<String, String>? isolateParams,
|
||||
}) async {
|
||||
section('Running task "$taskName"');
|
||||
final TaskResult result = await runTask(
|
||||
taskName,
|
||||
deviceId: deviceId,
|
||||
localEngine: localEngine,
|
||||
localEngineSrcPath: localEngineSrcPath,
|
||||
silent: silent,
|
||||
taskArgs: taskArgs,
|
||||
isolateParams: isolateParams,
|
||||
);
|
||||
|
||||
print('Task result:');
|
||||
print(const JsonEncoder.withIndent(' ').convert(result));
|
||||
section('Finished task "$taskName"');
|
||||
|
||||
if (resultsPath != null) {
|
||||
final Cocoon cocoon = Cocoon();
|
||||
await cocoon.writeTaskResultToFile(
|
||||
builderName: luciBuilder,
|
||||
gitBranch: gitBranch,
|
||||
result: result,
|
||||
resultsPath: resultsPath,
|
||||
);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
/// Runs a task in a separate Dart VM and collects the result using the VM
|
||||
/// service protocol.
|
||||
///
|
||||
|
|
47
dev/devicelab/test/runner_test.dart
Normal file
47
dev/devicelab/test/runner_test.dart
Normal file
|
@ -0,0 +1,47 @@
|
|||
// Copyright 2014 The Flutter Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
// @dart = 2.8
|
||||
|
||||
import 'package:flutter_devicelab/framework/runner.dart';
|
||||
|
||||
import 'common.dart';
|
||||
|
||||
void main() {
|
||||
final Map<String, String> isolateParams = <String, String>{
|
||||
'runFlutterConfig': 'false',
|
||||
'runProcessCleanup': 'false',
|
||||
'timeoutInMinutes': '1',
|
||||
};
|
||||
List<String> printLog;
|
||||
void print(String s) => printLog.add(s);
|
||||
|
||||
group('run.dart script', () {
|
||||
test('Reruns - Test passes the first time.', () async {
|
||||
printLog = <String>[];
|
||||
await runTasks(
|
||||
<String>['smoke_test_success'],
|
||||
isolateParams: isolateParams,
|
||||
print: print,
|
||||
logs: printLog,
|
||||
);
|
||||
expect(printLog.length, 2);
|
||||
expect(printLog[0], 'Total 1 executions: 1 success');
|
||||
expect(printLog[1], 'flaky: false');
|
||||
});
|
||||
|
||||
test('Reruns - Test fails all reruns.', () async {
|
||||
printLog = <String>[];
|
||||
await runTasks(
|
||||
<String>['smoke_test_failure'],
|
||||
isolateParams: isolateParams,
|
||||
print: print,
|
||||
logs: printLog,
|
||||
);
|
||||
expect(printLog.length, 2);
|
||||
expect(printLog[0], 'Total 3 executions: 0 success');
|
||||
expect(printLog[1], 'flaky: false');
|
||||
});
|
||||
});
|
||||
}
|
Loading…
Reference in a new issue