2019-11-27 23:04:02 +00:00
// Copyright 2014 The Flutter Authors. All rights reserved.
2019-07-10 19:10:28 +00:00
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import ' dart:convert ' ;
import ' dart:io ' ;
import ' package:args/args.dart ' ;
2021-02-16 23:29:12 +00:00
import ' package:file/local.dart ' ;
2019-07-10 19:10:28 +00:00
import ' package:glob/glob.dart ' ;
import ' package:meta/meta.dart ' ;
import ' package:path/path.dart ' as path ;
Future < void > main ( List < String > arguments ) async {
exit ( await run ( arguments ) ? 0 : 1 ) ;
}
Future < bool > run ( List < String > arguments ) async {
final ArgParser argParser = ArgParser (
allowTrailingOptions: false ,
usageLineLength: 72 ,
)
. . addOption (
' repeat ' ,
defaultsTo: ' 1 ' ,
help: ' How many times to run each test. Set to a high value to look for flakes. ' ,
valueHelp: ' count ' ,
)
2020-03-31 21:01:01 +00:00
. . addOption (
' shards ' ,
defaultsTo: ' 1 ' ,
help: ' How many shards to split the tests into. Used in continuous integration. ' ,
valueHelp: ' count ' ,
)
. . addOption (
' shard-index ' ,
defaultsTo: ' 0 ' ,
help: ' The current shard to run the tests with the range [0 .. shards - 1]. Used in continuous integration. ' ,
valueHelp: ' count ' ,
)
2019-07-10 19:10:28 +00:00
. . addFlag (
' skip-on-fetch-failure ' ,
defaultsTo: false ,
help: ' Whether to skip tests that we fail to download. ' ,
)
. . addFlag (
' skip-template ' ,
defaultsTo: false ,
help: ' Whether to skip tests named "template.test". ' ,
)
. . addFlag (
' verbose ' ,
defaultsTo: false ,
help: ' Describe what is happening in detail. ' ,
)
. . addFlag (
' help ' ,
defaultsTo: false ,
negatable: false ,
help: ' Print this help message. ' ,
) ;
2019-07-13 00:10:13 +00:00
void printHelp ( ) {
print ( ' run_tests.dart [options...] path/to/file1.test path/to/file2.test... ' ) ;
print ( ' For details on the test registry format, see: ' ) ;
print ( ' https://github.com/flutter/tests/blob/master/registry/template.test ' ) ;
print ( ' ' ) ;
print ( argParser . usage ) ;
print ( ' ' ) ;
}
ArgResults parsedArguments ;
try {
parsedArguments = argParser . parse ( arguments ) ;
} on ArgParserException catch ( error ) {
printHelp ( ) ;
print ( ' Error: ${ error . message } Use --help for usage information. ' ) ;
exit ( 1 ) ;
}
2019-07-10 19:10:28 +00:00
2019-12-05 21:34:06 +00:00
final int repeat = int . tryParse ( parsedArguments [ ' repeat ' ] as String ) ;
final bool skipOnFetchFailure = parsedArguments [ ' skip-on-fetch-failure ' ] as bool ;
final bool skipTemplate = parsedArguments [ ' skip-template ' ] as bool ;
final bool verbose = parsedArguments [ ' verbose ' ] as bool ;
final bool help = parsedArguments [ ' help ' ] as bool ;
2020-03-31 21:01:01 +00:00
final int numberShards = int . tryParse ( parsedArguments [ ' shards ' ] as String ) ;
final int shardIndex = int . tryParse ( parsedArguments [ ' shard-index ' ] as String ) ;
2019-07-10 19:10:28 +00:00
final List < File > files = parsedArguments
. rest
2021-02-16 23:29:12 +00:00
. expand ( ( String path ) = > Glob ( path ) . listFileSystemSync ( const LocalFileSystem ( ) ) )
2019-07-10 19:10:28 +00:00
. whereType < File > ( )
. where ( ( File file ) = > ! skipTemplate | | path . basename ( file . path ) ! = ' template.test ' )
. toList ( ) ;
if ( help | | repeat = = null | | files . isEmpty ) {
2019-07-13 00:10:13 +00:00
printHelp ( ) ;
if ( verbose ) {
if ( repeat = = null )
print ( ' Error: Could not parse repeat count (" ${ parsedArguments [ ' repeat ' ] } ") ' ) ;
if ( parsedArguments . rest . isEmpty ) {
print ( ' Error: No file arguments specified. ' ) ;
} else if ( files . isEmpty ) {
2020-02-11 19:58:27 +00:00
print ( ' Error: File arguments (" ${ parsedArguments . rest . join ( ' ", " ' ) } ") did not identify any real files. ' ) ;
2019-07-13 00:10:13 +00:00
}
}
2019-07-10 19:10:28 +00:00
return help ;
}
if ( verbose )
print ( ' Starting run_tests.dart... ' ) ;
2020-03-31 21:01:01 +00:00
if ( files . length < shardIndex )
print ( ' Warning: There are more shards than tests. Some shards will not run any tests. ' ) ;
if ( numberShards < = shardIndex ) {
print ( ' Error: There are more shard indexes than shards. ' ) ;
return help ;
}
// Best attempt at evenly splitting tests among the shards
final List < File > shardedFiles = < File > [ ] ;
for ( int i = shardIndex ; i < files . length ; i + = numberShards ) {
shardedFiles . add ( files [ i ] ) ;
}
2020-03-20 02:03:36 +00:00
int testCount = 0 ;
2019-07-10 19:10:28 +00:00
int failures = 0 ;
if ( verbose ) {
final String s = files . length = = 1 ? ' ' : ' s ' ;
2020-03-31 21:01:01 +00:00
final String ss = shardedFiles . length = = 1 ? ' ' : ' s ' ;
print ( ' ${ files . length } file $ s specified. ${ shardedFiles . length } test $ ss in shard # $ shardIndex . ' ) ;
2019-07-10 19:10:28 +00:00
print ( ' ' ) ;
}
2020-03-31 21:01:01 +00:00
if ( verbose ) {
print ( ' Tests in this shard: ' ) ;
for ( final File file in shardedFiles )
print ( file . path ) ;
}
print ( ' ' ) ;
for ( final File file in shardedFiles ) {
2019-07-10 19:10:28 +00:00
if ( verbose )
print ( ' Processing ${ file . path } ... ' ) ;
TestFile instructions ;
try {
instructions = TestFile ( file ) ;
} on FormatException catch ( error ) {
print ( ' ERROR: ${ error . message } ' ) ;
print ( ' ' ) ;
failures + = 1 ;
continue ;
} on FileSystemException catch ( error ) {
print ( ' ERROR: ${ error . message } ' ) ;
print ( ' ${ file . path } ' ) ;
print ( ' ' ) ;
failures + = 1 ;
continue ;
}
final Directory checkout = Directory . systemTemp . createTempSync ( ' flutter_customer_testing. ${ path . basenameWithoutExtension ( file . path ) } . ' ) ;
if ( verbose )
print ( ' Created temporary directory: ${ checkout . path } ' ) ;
try {
bool success ;
bool showContacts = false ;
2020-01-07 15:32:04 +00:00
for ( final String fetchCommand in instructions . fetch ) {
2019-07-10 19:10:28 +00:00
success = await shell ( fetchCommand , checkout , verbose: verbose , silentFailure: skipOnFetchFailure ) ;
if ( ! success ) {
if ( skipOnFetchFailure ) {
if ( verbose ) {
print ( ' Skipping (fetch failed). ' ) ;
} else {
print ( ' Skipping ${ file . path } (fetch failed). ' ) ;
}
} else {
print ( ' ERROR: Failed to fetch repository. ' ) ;
failures + = 1 ;
showContacts = true ;
}
break ;
}
}
assert ( success ! = null ) ;
if ( success ) {
if ( verbose )
print ( ' Running tests... ' ) ;
final Directory tests = Directory ( path . join ( checkout . path , ' tests ' ) ) ;
// TODO(ianh): Once we have a way to update source code, run that command in each directory of instructions.update
for ( int iteration = 0 ; iteration < repeat ; iteration + = 1 ) {
if ( verbose & & repeat > 1 )
print ( ' Round ${ iteration + 1 } of $ repeat . ' ) ;
2020-01-07 15:32:04 +00:00
for ( final String testCommand in instructions . tests ) {
2020-03-20 02:03:36 +00:00
testCount + = 1 ;
2019-07-10 19:10:28 +00:00
success = await shell ( testCommand , tests , verbose: verbose ) ;
if ( ! success ) {
print ( ' ERROR: One or more tests from ${ path . basenameWithoutExtension ( file . path ) } failed. ' ) ;
failures + = 1 ;
showContacts = true ;
break ;
}
}
}
if ( verbose & & success )
print ( ' Tests finished. ' ) ;
}
if ( showContacts ) {
final String s = instructions . contacts . length = = 1 ? ' ' : ' s ' ;
print ( ' Contact $ s : ${ instructions . contacts . join ( " , " ) } ' ) ;
}
} finally {
if ( verbose )
print ( ' Deleting temporary directory... ' ) ;
2020-02-04 06:44:45 +00:00
try {
checkout . deleteSync ( recursive: true ) ;
} on FileSystemException {
print ( ' Failed to delete " ${ checkout . path } ". ' ) ;
}
2019-07-10 19:10:28 +00:00
}
if ( verbose )
print ( ' ' ) ;
}
if ( failures > 0 ) {
final String s = failures = = 1 ? ' ' : ' s ' ;
print ( ' $ failures failure $ s . ' ) ;
return false ;
}
2020-03-20 02:03:36 +00:00
print ( ' $ testCount tests all passed! ' ) ;
2019-07-10 19:10:28 +00:00
return true ;
}
@ immutable
class TestFile {
factory TestFile ( File file ) {
final String errorPrefix = ' Could not parse: ${ file . path } \n ' ;
final List < String > contacts = < String > [ ] ;
final List < String > fetch = < String > [ ] ;
final List < Directory > update = < Directory > [ ] ;
final List < String > test = < String > [ ] ;
2020-03-20 02:03:36 +00:00
bool hasTests = false ;
2020-01-07 15:32:04 +00:00
for ( final String line in file . readAsLinesSync ( ) . map ( ( String line ) = > line . trim ( ) ) ) {
2019-07-10 19:10:28 +00:00
if ( line . isEmpty ) {
// blank line
} else if ( line . startsWith ( ' # ' ) ) {
// comment
} else if ( line . startsWith ( ' contact= ' ) ) {
contacts . add ( line . substring ( 8 ) ) ;
} else if ( line . startsWith ( ' fetch= ' ) ) {
fetch . add ( line . substring ( 6 ) ) ;
} else if ( line . startsWith ( ' update= ' ) ) {
update . add ( Directory ( line . substring ( 7 ) ) ) ;
} else if ( line . startsWith ( ' test= ' ) ) {
2020-03-20 02:03:36 +00:00
hasTests = true ;
2019-07-10 19:10:28 +00:00
test . add ( line . substring ( 5 ) ) ;
2019-10-28 22:51:22 +00:00
} else if ( line . startsWith ( ' test.windows= ' ) ) {
2020-03-20 02:03:36 +00:00
hasTests = true ;
2019-10-28 22:51:22 +00:00
if ( Platform . isWindows )
2019-12-09 20:58:03 +00:00
test . add ( line . substring ( 13 ) ) ;
2019-10-28 22:51:22 +00:00
} else if ( line . startsWith ( ' test.macos= ' ) ) {
2020-03-20 02:03:36 +00:00
hasTests = true ;
2019-10-28 22:51:22 +00:00
if ( Platform . isMacOS )
2019-12-09 20:58:03 +00:00
test . add ( line . substring ( 11 ) ) ;
2019-10-28 22:51:22 +00:00
} else if ( line . startsWith ( ' test.linux= ' ) ) {
2020-03-20 02:03:36 +00:00
hasTests = true ;
2019-10-28 22:51:22 +00:00
if ( Platform . isLinux )
2019-12-09 20:58:03 +00:00
test . add ( line . substring ( 11 ) ) ;
2019-10-28 22:51:22 +00:00
} else if ( line . startsWith ( ' test.posix= ' ) ) {
2020-03-20 02:03:36 +00:00
hasTests = true ;
2019-10-28 22:51:22 +00:00
if ( Platform . isLinux | | Platform . isMacOS )
2019-12-09 20:58:03 +00:00
test . add ( line . substring ( 11 ) ) ;
2019-07-10 19:10:28 +00:00
} else {
throw FormatException ( ' ${ errorPrefix } Unexpected directive: \n $ line ' ) ;
}
}
if ( contacts . isEmpty )
throw FormatException ( ' ${ errorPrefix } No contacts specified. At least one contact e-mail address must be specified. ' ) ;
2020-01-07 15:32:04 +00:00
for ( final String email in contacts ) {
2019-07-10 19:10:28 +00:00
if ( ! email . contains ( _email ) | | email . endsWith ( ' @example.com ' ) )
throw FormatException ( ' ${ errorPrefix } The following e-mail address appears to be an invalid e-mail address: $ email ' ) ;
}
if ( fetch . isEmpty )
throw FormatException ( ' ${ errorPrefix } No "fetch" directives specified. Two lines are expected: "git clone https://github.com/USERNAME/REPOSITORY.git tests" and "git -C tests checkout HASH". ' ) ;
if ( fetch . length < 2 )
throw FormatException ( ' ${ errorPrefix } Only one "fetch" directive specified. Two lines are expected: "git clone https://github.com/USERNAME/REPOSITORY.git tests" and "git -C tests checkout HASH". ' ) ;
if ( ! fetch [ 0 ] . contains ( _fetch1 ) )
throw FormatException ( ' ${ errorPrefix } First "fetch" directive does not match expected pattern (expected "git clone https://github.com/USERNAME/REPOSITORY.git tests"). ' ) ;
if ( ! fetch [ 1 ] . contains ( _fetch2 ) )
throw FormatException ( ' ${ errorPrefix } Second "fetch" directive does not match expected pattern (expected "git -C tests checkout HASH"). ' ) ;
if ( update . isEmpty )
throw FormatException ( ' ${ errorPrefix } No "update" directives specified. At least one directory must be specified. (It can be "." to just upgrade the root of the repository.) ' ) ;
2020-03-20 02:03:36 +00:00
if ( ! hasTests )
throw FormatException ( ' ${ errorPrefix } No "test" directives specified. At least one command must be specified to run tests. ' ) ;
2019-07-10 19:10:28 +00:00
return TestFile . _ (
List < String > . unmodifiable ( contacts ) ,
List < String > . unmodifiable ( fetch ) ,
List < Directory > . unmodifiable ( update ) ,
List < String > . unmodifiable ( test ) ,
) ;
}
const TestFile . _ ( this . contacts , this . fetch , this . update , this . tests ) ;
// (e-mail regexp from HTML standard)
2020-02-26 02:12:17 +00:00
static final RegExp _email = RegExp ( r"^[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$" ) ;
2020-02-04 06:44:45 +00:00
static final RegExp _fetch1 = RegExp ( r'^git(?: -c core.longPaths=true)? clone https://github.com/[-a-zA-Z0-9]+/[-_a-zA-Z0-9]+.git tests$' ) ;
static final RegExp _fetch2 = RegExp ( r'^git(?: -c core.longPaths=true)? -C tests checkout [0-9a-f]+$' ) ;
2019-07-10 19:10:28 +00:00
final List < String > contacts ;
final List < String > fetch ;
final List < Directory > update ;
final List < String > tests ;
}
final RegExp _spaces = RegExp ( r' +' ) ;
Future < bool > shell ( String command , Directory directory , { bool verbose = false , bool silentFailure = false } ) async {
if ( verbose )
print ( ' >> $ command ' ) ;
Process process ;
if ( Platform . isWindows ) {
2020-01-31 17:30:21 +00:00
process = await Process . start ( ' CMD.EXE ' , < String > [ ' /S ' , ' /C ' , command ] , workingDirectory: directory . path ) ;
2019-07-10 19:10:28 +00:00
} else {
final List < String > segments = command . trim ( ) . split ( _spaces ) ;
process = await Process . start ( segments . first , segments . skip ( 1 ) . toList ( ) , workingDirectory: directory . path ) ;
}
final List < String > output = < String > [ ] ;
utf8 . decoder . bind ( process . stdout ) . transform ( const LineSplitter ( ) ) . listen ( verbose ? printLog : output . add ) ;
utf8 . decoder . bind ( process . stderr ) . transform ( const LineSplitter ( ) ) . listen ( verbose ? printLog : output . add ) ;
final bool success = await process . exitCode = = 0 ;
if ( success | | silentFailure )
return success ;
if ( ! verbose ) {
print ( ' >> $ command ' ) ;
output . forEach ( printLog ) ;
}
return success ;
}
void printLog ( String line ) {
print ( ' | $ line ' . trimRight ( ) ) ;
}