// Copyright 2014 The Flutter Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. import 'dart:convert'; import 'dart:io'; import 'package:args/args.dart'; import 'package:glob/glob.dart'; import 'package:meta/meta.dart'; import 'package:path/path.dart' as path; Future main(List arguments) async { exit(await run(arguments) ? 0 : 1); } Future run(List arguments) async { final ArgParser argParser = ArgParser( allowTrailingOptions: false, usageLineLength: 72, ) ..addOption( 'repeat', defaultsTo: '1', help: 'How many times to run each test. Set to a high value to look for flakes.', valueHelp: 'count', ) ..addFlag( 'skip-on-fetch-failure', defaultsTo: false, help: 'Whether to skip tests that we fail to download.', ) ..addFlag( 'skip-template', defaultsTo: false, help: 'Whether to skip tests named "template.test".', ) ..addFlag( 'verbose', defaultsTo: false, help: 'Describe what is happening in detail.', ) ..addFlag( 'help', defaultsTo: false, negatable: false, help: 'Print this help message.', ); void printHelp() { print('run_tests.dart [options...] path/to/file1.test path/to/file2.test...'); print('For details on the test registry format, see:'); print(' https://github.com/flutter/tests/blob/master/registry/template.test'); print(''); print(argParser.usage); print(''); } ArgResults parsedArguments; try { parsedArguments = argParser.parse(arguments); } on ArgParserException catch (error) { printHelp(); print('Error: ${error.message} Use --help for usage information.'); exit(1); } final int repeat = int.tryParse(parsedArguments['repeat'] as String); final bool skipOnFetchFailure = parsedArguments['skip-on-fetch-failure'] as bool; final bool skipTemplate = parsedArguments['skip-template'] as bool; final bool verbose = parsedArguments['verbose'] as bool; final bool help = parsedArguments['help'] as bool; final List files = parsedArguments .rest .expand((String path) => Glob(path).listSync()) .whereType() .where((File file) => !skipTemplate || path.basename(file.path) != 'template.test') .toList(); if (help || repeat == null || files.isEmpty) { printHelp(); if (verbose) { if (repeat == null) print('Error: Could not parse repeat count ("${parsedArguments['repeat']}")'); if (parsedArguments.rest.isEmpty) { print('Error: No file arguments specified.'); } else if (files.isEmpty) { print('Error: File arguments ("${parsedArguments.rest.join("\", \"")}") did not identify any real files.'); } } return help; } if (verbose) print('Starting run_tests.dart...'); int failures = 0; if (verbose) { final String s = files.length == 1 ? '' : 's'; print('${files.length} file$s specified.'); print(''); } for (final File file in files) { if (verbose) print('Processing ${file.path}...'); TestFile instructions; try { instructions = TestFile(file); } on FormatException catch (error) { print('ERROR: ${error.message}'); print(''); failures += 1; continue; } on FileSystemException catch (error) { print('ERROR: ${error.message}'); print(' ${file.path}'); print(''); failures += 1; continue; } final Directory checkout = Directory.systemTemp.createTempSync('flutter_customer_testing.${path.basenameWithoutExtension(file.path)}.'); if (verbose) print('Created temporary directory: ${checkout.path}'); try { bool success; bool showContacts = false; for (final String fetchCommand in instructions.fetch) { success = await shell(fetchCommand, checkout, verbose: verbose, silentFailure: skipOnFetchFailure); if (!success) { if (skipOnFetchFailure) { if (verbose) { print('Skipping (fetch failed).'); } else { print('Skipping ${file.path} (fetch failed).'); } } else { print('ERROR: Failed to fetch repository.'); failures += 1; showContacts = true; } break; } } assert(success != null); if (success) { if (verbose) print('Running tests...'); final Directory tests = Directory(path.join(checkout.path, 'tests')); // TODO(ianh): Once we have a way to update source code, run that command in each directory of instructions.update for (int iteration = 0; iteration < repeat; iteration += 1) { if (verbose && repeat > 1) print('Round ${iteration + 1} of $repeat.'); for (final String testCommand in instructions.tests) { success = await shell(testCommand, tests, verbose: verbose); if (!success) { print('ERROR: One or more tests from ${path.basenameWithoutExtension(file.path)} failed.'); failures += 1; showContacts = true; break; } } } if (verbose && success) print('Tests finished.'); } if (showContacts) { final String s = instructions.contacts.length == 1 ? '' : 's'; print('Contact$s: ${instructions.contacts.join(", ")}'); } } finally { if (verbose) print('Deleting temporary directory...'); try { checkout.deleteSync(recursive: true); } on FileSystemException { print('Failed to delete "${checkout.path}".'); } } if (verbose) print(''); } if (failures > 0) { final String s = failures == 1 ? '' : 's'; print('$failures failure$s.'); return false; } if (verbose) { print('All tests passed!'); } return true; } @immutable class TestFile { factory TestFile(File file) { final String errorPrefix = 'Could not parse: ${file.path}\n'; final List contacts = []; final List fetch = []; final List update = []; final List test = []; for (final String line in file.readAsLinesSync().map((String line) => line.trim())) { if (line.isEmpty) { // blank line } else if (line.startsWith('#')) { // comment } else if (line.startsWith('contact=')) { contacts.add(line.substring(8)); } else if (line.startsWith('fetch=')) { fetch.add(line.substring(6)); } else if (line.startsWith('update=')) { update.add(Directory(line.substring(7))); } else if (line.startsWith('test=')) { test.add(line.substring(5)); } else if (line.startsWith('test.windows=')) { if (Platform.isWindows) test.add(line.substring(13)); } else if (line.startsWith('test.macos=')) { if (Platform.isMacOS) test.add(line.substring(11)); } else if (line.startsWith('test.linux=')) { if (Platform.isLinux) test.add(line.substring(11)); } else if (line.startsWith('test.posix=')) { if (Platform.isLinux || Platform.isMacOS) test.add(line.substring(11)); } else { throw FormatException('${errorPrefix}Unexpected directive:\n$line'); } } if (contacts.isEmpty) throw FormatException('${errorPrefix}No contacts specified. At least one contact e-mail address must be specified.'); for (final String email in contacts) { if (!email.contains(_email) || email.endsWith('@example.com')) throw FormatException('${errorPrefix}The following e-mail address appears to be an invalid e-mail address: $email'); } if (fetch.isEmpty) throw FormatException('${errorPrefix}No "fetch" directives specified. Two lines are expected: "git clone https://github.com/USERNAME/REPOSITORY.git tests" and "git -C tests checkout HASH".'); if (fetch.length < 2) throw FormatException('${errorPrefix}Only one "fetch" directive specified. Two lines are expected: "git clone https://github.com/USERNAME/REPOSITORY.git tests" and "git -C tests checkout HASH".'); if (!fetch[0].contains(_fetch1)) throw FormatException('${errorPrefix}First "fetch" directive does not match expected pattern (expected "git clone https://github.com/USERNAME/REPOSITORY.git tests").'); if (!fetch[1].contains(_fetch2)) throw FormatException('${errorPrefix}Second "fetch" directive does not match expected pattern (expected "git -C tests checkout HASH").'); if (update.isEmpty) throw FormatException('${errorPrefix}No "update" directives specified. At least one directory must be specified. (It can be "." to just upgrade the root of the repository.)'); if (test.isEmpty) throw FormatException('${errorPrefix}No "test" directives specified for this platform. At least one command must be specified to run tests on each of Windows, MacOS, and Linux.'); return TestFile._( List.unmodifiable(contacts), List.unmodifiable(fetch), List.unmodifiable(update), List.unmodifiable(test), ); } const TestFile._(this.contacts, this.fetch, this.update, this.tests); // (e-mail regexp from HTML standard) static final RegExp _email = RegExp(r'''^[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$'''); static final RegExp _fetch1 = RegExp(r'^git(?: -c core.longPaths=true)? clone https://github.com/[-a-zA-Z0-9]+/[-_a-zA-Z0-9]+.git tests$'); static final RegExp _fetch2 = RegExp(r'^git(?: -c core.longPaths=true)? -C tests checkout [0-9a-f]+$'); final List contacts; final List fetch; final List update; final List tests; } final RegExp _spaces = RegExp(r' +'); Future shell(String command, Directory directory, { bool verbose = false, bool silentFailure = false }) async { if (verbose) print('>> $command'); Process process; if (Platform.isWindows) { process = await Process.start('CMD.EXE', ['/S', '/C', command], workingDirectory: directory.path); } else { final List segments = command.trim().split(_spaces); process = await Process.start(segments.first, segments.skip(1).toList(), workingDirectory: directory.path); } final List output = []; utf8.decoder.bind(process.stdout).transform(const LineSplitter()).listen(verbose ? printLog : output.add); utf8.decoder.bind(process.stderr).transform(const LineSplitter()).listen(verbose ? printLog : output.add); final bool success = await process.exitCode == 0; if (success || silentFailure) return success; if (!verbose) { print('>> $command'); output.forEach(printLog); } return success; } void printLog(String line) { print('| $line'.trimRight()); }