// Copyright 2014 The Flutter Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. import 'dart:io'; import 'package:flutter_devicelab/framework/utils.dart' show rm; import 'package:path/path.dart' as path; import 'package:process/process.dart'; import 'common.dart'; void main() { const ProcessManager processManager = LocalProcessManager(); group('run.dart script', () { Future runScript(List testNames, [List otherArgs = const []]) async { final String dart = path.absolute( path.join('..', '..', 'bin', 'cache', 'dart-sdk', 'bin', 'dart')); final ProcessResult scriptProcess = processManager.runSync([ dart, 'bin/run.dart', '--no-terminate-stray-dart-processes', ...otherArgs, for (final String testName in testNames) ...['-t', testName], ]); return scriptProcess; } Future expectScriptResult( List testNames, int expectedExitCode, {String? deviceId} ) async { final ProcessResult result = await runScript(testNames, [ if (deviceId != null) ...['-d', deviceId], ]); expect(result.exitCode, expectedExitCode, reason: '[ stderr from test process ]\n\n${result.stderr}\n\n[ end of stderr ]' '\n\n[ stdout from test process ]\n\n${result.stdout}\n\n[ end of stdout ]'); } test('exits with code 0 when succeeds', () async { await expectScriptResult(['smoke_test_success'], 0); }); test('accepts file paths', () async { await expectScriptResult( ['bin/tasks/smoke_test_success.dart'], 0); }); test('rejects invalid file paths', () async { await expectScriptResult(['lib/framework/adb.dart'], 1); }); test('exits with code 1 when task throws', () async { await expectScriptResult(['smoke_test_throws'], 1); }); test('exits with code 1 when fails', () async { await expectScriptResult(['smoke_test_failure'], 1); }); test('exits with code 1 when fails to connect', () async { await expectScriptResult(['smoke_test_setup_failure'], 1); }, skip: true); // https://github.com/flutter/flutter/issues/53707 test('exits with code 1 when results are mixed', () async { await expectScriptResult( [ 'smoke_test_failure', 'smoke_test_success', ], 1, ); }); test('exits with code 0 when provided a valid device ID', () async { await expectScriptResult(['smoke_test_device'], 0, deviceId: 'FAKE'); }); test('exits with code 1 when provided a bad device ID', () async { await expectScriptResult(['smoke_test_device'], 1, deviceId: 'THIS_IS_NOT_VALID'); }); test('runs A/B test', () async { final Directory tempDirectory = Directory.systemTemp.createTempSync('flutter_devicelab_ab_test.'); final File abResultsFile = File(path.join(tempDirectory.path, 'test_results.json')); expect(abResultsFile.existsSync(), isFalse); final ProcessResult result = await runScript( ['smoke_test_success'], ['--ab=2', '--local-engine=host_debug_unopt', '--ab-result-file', abResultsFile.path], ); expect(result.exitCode, 0); String sectionHeader = !Platform.isWindows ? '═════════════════════════╡ ••• A/B results so far ••• ╞═════════════════════════' : 'A/B results so far'; expect( result.stdout, contains( '$sectionHeader\n' '\n' 'Score\tAverage A (noise)\tAverage B (noise)\tSpeed-up\n' 'metric1\t42.00 (0.00%)\t42.00 (0.00%)\t1.00x\t\n' 'metric2\t123.00 (0.00%)\t123.00 (0.00%)\t1.00x\t\n', ), ); sectionHeader = !Platform.isWindows ? '════════════════════════════╡ ••• Raw results ••• ╞═════════════════════════════' : 'Raw results'; expect( result.stdout, contains( '$sectionHeader\n' '\n' 'metric1:\n' ' A:\t42.00\t42.00\t\n' ' B:\t42.00\t42.00\t\n' 'metric2:\n' ' A:\t123.00\t123.00\t\n' ' B:\t123.00\t123.00\t\n', ), ); sectionHeader = !Platform.isWindows ? '═════════════════════════╡ ••• Final A/B results ••• ╞══════════════════════════' : 'Final A/B results'; expect( result.stdout, contains( '$sectionHeader\n' '\n' 'Score\tAverage A (noise)\tAverage B (noise)\tSpeed-up\n' 'metric1\t42.00 (0.00%)\t42.00 (0.00%)\t1.00x\t\n' 'metric2\t123.00 (0.00%)\t123.00 (0.00%)\t1.00x\t\n', ), ); expect(abResultsFile.existsSync(), isTrue); rm(tempDirectory, recursive: true); }); test('fails to upload results to Cocoon if flags given', () async { // CocoonClient will fail to find test-file, and will not send any http requests. final ProcessResult result = await runScript( ['smoke_test_success'], ['--service-account-file=test-file', '--task-key=task123'], ); expect(result.exitCode, 1); }); }); }