mirror of
https://github.com/flutter/flutter
synced 2024-08-27 20:10:11 +00:00
Collect metrics - load balancing tests (#108752)
This commit is contained in:
parent
1e87fdd883
commit
8fd46ceeac
|
@ -63,6 +63,7 @@ import 'package:path/path.dart' as path;
|
|||
import 'browser.dart';
|
||||
import 'run_command.dart';
|
||||
import 'service_worker_test.dart';
|
||||
import 'tool_subsharding.dart';
|
||||
import 'utils.dart';
|
||||
|
||||
typedef ShardRunner = Future<void> Function();
|
||||
|
@ -464,6 +465,7 @@ Future<void> _runIntegrationToolTests() async {
|
|||
_toolsPath,
|
||||
forceSingleCore: true,
|
||||
testPaths: _selectIndexOfTotalSubshard<String>(allTests),
|
||||
collectMetrics: true,
|
||||
);
|
||||
}
|
||||
|
||||
|
@ -1736,6 +1738,7 @@ Future<void> _runDartTest(String workingDirectory, {
|
|||
bool includeLocalEngineEnv = false,
|
||||
bool ensurePrecompiledTool = true,
|
||||
bool shuffleTests = true,
|
||||
bool collectMetrics = false,
|
||||
}) async {
|
||||
int? cpus;
|
||||
final String? cpuVariable = Platform.environment['CPU']; // CPU is set in cirrus.yml
|
||||
|
@ -1757,6 +1760,8 @@ Future<void> _runDartTest(String workingDirectory, {
|
|||
cpus = 1;
|
||||
}
|
||||
|
||||
const LocalFileSystem fileSystem = LocalFileSystem();
|
||||
final File metricFile = fileSystem.file(path.join(flutterRoot, 'metrics.json'));
|
||||
final List<String> args = <String>[
|
||||
'run',
|
||||
'test',
|
||||
|
@ -1771,6 +1776,8 @@ Future<void> _runDartTest(String workingDirectory, {
|
|||
if (testPaths != null)
|
||||
for (final String testPath in testPaths)
|
||||
testPath,
|
||||
if (collectMetrics)
|
||||
'--file-reporter=json:${metricFile.path}',
|
||||
];
|
||||
final Map<String, String> environment = <String, String>{
|
||||
'FLUTTER_ROOT': flutterRoot,
|
||||
|
@ -1795,6 +1802,23 @@ Future<void> _runDartTest(String workingDirectory, {
|
|||
environment: environment,
|
||||
removeLine: useBuildRunner ? (String line) => line.startsWith('[INFO]') : null,
|
||||
);
|
||||
|
||||
if (collectMetrics) {
|
||||
try {
|
||||
final List<String> testList = <String>[];
|
||||
final Map<int, TestSpecs> allTestSpecs = generateMetrics(metricFile);
|
||||
for (final TestSpecs testSpecs in allTestSpecs.values) {
|
||||
testList.add(testSpecs.toJson());
|
||||
}
|
||||
if (testList.isNotEmpty) {
|
||||
final String testJson = json.encode(testList);
|
||||
final File testResults = fileSystem.file(path.join(flutterRoot, 'test_results.json'));
|
||||
testResults.writeAsStringSync(testJson);
|
||||
}
|
||||
} on fs.FileSystemException catch (e){
|
||||
print('Failed to generate metrics: $e');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Future<void> _runFlutterTest(String workingDirectory, {
|
||||
|
|
69
dev/bots/test/tool_subsharding_test.dart
Normal file
69
dev/bots/test/tool_subsharding_test.dart
Normal file
|
@ -0,0 +1,69 @@
|
|||
// Copyright 2014 The Flutter Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
import 'dart:io';
|
||||
|
||||
import 'package:file/memory.dart';
|
||||
|
||||
import '../tool_subsharding.dart';
|
||||
import 'common.dart';
|
||||
|
||||
void main() {
|
||||
group('generateMetrics', () {
|
||||
late MemoryFileSystem fileSystem;
|
||||
|
||||
setUp(() {
|
||||
fileSystem = MemoryFileSystem.test();
|
||||
});
|
||||
|
||||
test('empty metrics', () async {
|
||||
final File file = fileSystem.file('success_file');
|
||||
const String output = '''
|
||||
{"missing": "entry"}
|
||||
{"other": true}''';
|
||||
file.writeAsStringSync(output);
|
||||
final Map<int, TestSpecs> result = generateMetrics(file);
|
||||
expect(result, isEmpty);
|
||||
});
|
||||
|
||||
test('have metrics', () async {
|
||||
final File file = fileSystem.file('success_file');
|
||||
const String output = '''
|
||||
{"protocolVersion":"0.1.1","runnerVersion":"1.21.6","pid":93376,"type":"start","time":0}
|
||||
{"suite":{"id":0,"platform":"vm","path":"test/general.shard/project_validator_result_test.dart"},"type":"suite","time":0}
|
||||
{"count":1,"time":12,"type":"allSuites"}
|
||||
{"testID":1,"result":"success","skipped":false,"hidden":true,"type":"testDone","time":4798}
|
||||
{"test":{"id":4,"name":"ProjectValidatorResult success status","suiteID":0,"groupIDs":[2,3],"metadata":{"skip":false,"skipReason":null},"line":159,"column":16,"url":"file:///file","root_line":50,"root_column":5,"root_url":"file:///file"},"type":"testStart","time":4803}
|
||||
{"testID":4,"result":"success","skipped":false,"hidden":false,"type":"testDone","time":4837}
|
||||
{"suite":{"id":1,"platform":"vm","path":"other_path"},"type":"suite","time":1000}
|
||||
{"test":{"id":5,"name":"ProjectValidatorResult success status with warning","suiteID":0,"groupIDs":[2,3],"metadata":{"skip":false,"skipReason":null},"line":159,"column":16,"url":"file:///file","root_line":60,"root_column":5,"root_url":"file:///file"},"type":"testStart","time":4837}
|
||||
{"testID":5,"result":"success","skipped":false,"hidden":false,"type":"testDone","time":4839}
|
||||
{"test":{"id":6,"name":"ProjectValidatorResult error status","suiteID":0,"groupIDs":[2,3],"metadata":{"skip":false,"skipReason":null},"line":159,"column":16,"url":"file:///file","root_line":71,"root_column":5,"root_url":"file:///file"},"type":"testStart","time":4839}
|
||||
{"testID":6,"result":"success","skipped":false,"hidden":false,"type":"testDone","time":4841}
|
||||
{"group":{"id":7,"suiteID":0,"parentID":2,"name":"ProjectValidatorTask","metadata":{"skip":false,"skipReason":null},"testCount":1,"line":82,"column":3,"url":"file:///file"},"type":"group","time":4841}
|
||||
{"test":{"id":8,"name":"ProjectValidatorTask error status","suiteID":0,"groupIDs":[2,7],"metadata":{"skip":false,"skipReason":null},"line":159,"column":16,"url":"file:///file","root_line":89,"root_column":5,"root_url":"file:///file"},"type":"testStart","time":4842}
|
||||
{"testID":8,"result":"success","skipped":false,"hidden":false,"type":"testDone","time":4860}
|
||||
{"group":{"id":7,"suiteID":1,"parentID":2,"name":"ProjectValidatorTask","metadata":{"skip":false,"skipReason":null},"testCount":1,"line":82,"column":3,"url":"file:///file"},"type":"group","time":5000}
|
||||
{"success":true,"type":"done","time":4870}''';
|
||||
file.writeAsStringSync(output);
|
||||
final Map<int, TestSpecs> result = generateMetrics(file);
|
||||
expect(result, contains(0));
|
||||
expect(result, contains(1));
|
||||
expect(result[0]!.path, 'test/general.shard/project_validator_result_test.dart');
|
||||
expect(result[0]!.milliseconds, 4841);
|
||||
expect(result[1]!.path, 'other_path');
|
||||
expect(result[1]!.milliseconds, 4000);
|
||||
});
|
||||
|
||||
test('missing success entry', () async {
|
||||
final File file = fileSystem.file('success_file');
|
||||
const String output = '''
|
||||
{"suite":{"id":1,"platform":"vm","path":"other_path"},"type":"suite","time":1000}
|
||||
{"group":{"id":7,"suiteID":1,"parentID":2,"name":"name","metadata":{"skip":false,"skipReason":null},"testCount":1,"line":82,"column":3,"url":"file:///file"},"type":"group","time":5000}''';
|
||||
file.writeAsStringSync(output);
|
||||
final Map<int, TestSpecs> result = generateMetrics(file);
|
||||
expect(result, isEmpty);
|
||||
});
|
||||
});
|
||||
}
|
79
dev/bots/tool_subsharding.dart
Normal file
79
dev/bots/tool_subsharding.dart
Normal file
|
@ -0,0 +1,79 @@
|
|||
// Copyright 2014 The Flutter Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
import 'dart:convert';
|
||||
import 'dart:io';
|
||||
|
||||
class TestSpecs {
|
||||
|
||||
TestSpecs({
|
||||
required this.path,
|
||||
required this.startTime,
|
||||
});
|
||||
|
||||
final String path;
|
||||
int startTime;
|
||||
int? _endTime;
|
||||
|
||||
int get milliseconds {
|
||||
return endTime - startTime;
|
||||
}
|
||||
|
||||
set endTime(int value) {
|
||||
_endTime = value;
|
||||
}
|
||||
|
||||
int get endTime {
|
||||
if (_endTime == null) {
|
||||
return 0;
|
||||
}
|
||||
return _endTime!;
|
||||
}
|
||||
|
||||
String toJson() {
|
||||
return json.encode(
|
||||
<String, String>{'path': path, 'runtime': milliseconds.toString()}
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Intended to parse the output file of `dart test --file-reporter json:file_name
|
||||
Map<int, TestSpecs> generateMetrics(File metrics) {
|
||||
final Map<int, TestSpecs> allTestSpecs = <int, TestSpecs>{};
|
||||
if (!metrics.existsSync()) {
|
||||
return allTestSpecs;
|
||||
}
|
||||
|
||||
bool success = false;
|
||||
for(final String metric in metrics.readAsLinesSync()) {
|
||||
final Map<String, dynamic> entry = json.decode(metric) as Map<String, dynamic>;
|
||||
if (entry.containsKey('suite')) {
|
||||
final Map<dynamic, dynamic> suite = entry['suite'] as Map<dynamic, dynamic>;
|
||||
allTestSpecs[suite['id'] as int] = TestSpecs(
|
||||
path: suite['path'] as String,
|
||||
startTime: entry['time'] as int,
|
||||
);
|
||||
} else if (_isMetricDone(entry, allTestSpecs)) {
|
||||
final Map<dynamic, dynamic> group = entry['group'] as Map<dynamic, dynamic>;
|
||||
final int suiteID = group['suiteID'] as int;
|
||||
final TestSpecs testSpec = allTestSpecs[suiteID]!;
|
||||
testSpec.endTime = entry['time'] as int;
|
||||
} else if (entry.containsKey('success') && entry['success'] == true) {
|
||||
success = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (!success) { // means that not all tests succeeded therefore no metrics are stored
|
||||
return <int, TestSpecs>{};
|
||||
}
|
||||
return allTestSpecs;
|
||||
}
|
||||
|
||||
bool _isMetricDone(Map<String, dynamic> entry, Map<int, TestSpecs> allTestSpecs) {
|
||||
if (entry.containsKey('group') && entry['type'] as String == 'group') {
|
||||
final Map<dynamic, dynamic> group = entry['group'] as Map<dynamic, dynamic>;
|
||||
return allTestSpecs.containsKey(group['suiteID'] as int);
|
||||
}
|
||||
return false;
|
||||
}
|
Loading…
Reference in a new issue