languages/dart-sdk
languages/dart-sdk
1
0
Fork 0
mirror of https://github.com/dart-lang/sdk synced 2024-09-04 16:03:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

[frontend_server] This adds the ability for the frontend server to be a resident process, allowing it to live through invocations of the Dart CLI. This allows the CLI to utilize the frontend's incremental compilation mode and keep cached kernel files for better performance. This currently only supports VM targets.

Browse source 
When launched as a snapshot, the initial compile is slower than the existing pub approach because of a process start time of around 500 ms. When launched as a compiled executable, initial compile times are the same as the existing pub approach. Once launched, experimental results show times to produce a kernel file of at worst 1.5-2x faster than pub's solution and at best 10x faster than pub's solution. The typical workflow of making changes and recompiling results in an average of a 5x speedup with respect to pub's implementation.

Because compiler instances use a lot of memory, there is a limit on the number of active compilers that the resident server will keep alive, and will actively bring instances down when this limit is exceeded. If the user was previously compiling a given project during the lifespan of a ResidentFrontendServer and its compiler is taken down between requests, a new compiler instance will be allocated for the request. Performance is still between 1.5x-5x faster for this case when compared to pub's compile times.

Change-Id: If9ee1ecc71d660d34faf23381c764dc11d6a5902
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/252001
Reviewed-by: Siva Annamalai <asiva@google.com>
Reviewed-by: Jake Macdonald <jakemac@google.com>
Commit-Queue: Michael Richards <msrichards@google.com>
Reviewed-by: Alexander Aprelev <aam@google.com>
This commit is contained in:
Michael Richards 2022-08-02 18:08:21 +00:00 committed by Commit Bot
parent 571d9448ea
commit 1cd8b28cfe
7 changed files with 1039 additions and 93 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
pkg/frontend_server/bin/frontend_server_starter.dart
View file

@ -4,7 +4,7 @@ library frontend_server;
import 'dart:async';
import 'dart:io';
import 'package:frontend_server/frontend_server.dart';
import 'package:frontend_server/starter.dart';
Future<Null> main(List<String> args) async {
exitCode = await starter(args);

97
pkg/frontend_server/lib/frontend_server.dart
View file

@ -7,7 +7,7 @@ library frontend_server;
import 'dart:async';
import 'dart:convert';
import 'dart:io' show Directory, File, IOSink, stdin, stdout;
import 'dart:io' show File, IOSink, stdout;
import 'dart:typed_data' show BytesBuilder;
import 'package:args/args.dart';
@ -31,13 +31,11 @@ import 'package:kernel/kernel.dart'
show Component, loadComponentSourceFromBytes;
import 'package:kernel/target/targets.dart' show targets, TargetFlags;
import 'package:package_config/package_config.dart';
import 'package:path/path.dart' as path;
import 'package:usage/uuid/uuid.dart';
import 'package:vm/incremental_compiler.dart' show IncrementalCompiler;
import 'package:vm/kernel_front_end.dart';
import 'src/binary_protocol.dart';
import 'src/javascript_bundle.dart';
import 'src/strong_components.dart';
@ -205,6 +203,11 @@ ArgParser argParser = ArgParser(allowTrailingOptions: true)
..addFlag('print-incremental-dependencies',
help: 'Print list of sources added and removed from compilation',
defaultsTo: true)
..addOption('resident-info-file-name',
help:
'Allowing for incremental compilation of changes when using the Dart CLI.'
' Stores server information in this file for accessing later',
hide: true)
..addOption('verbosity',
help: 'Sets the verbosity level of the compilation',
defaultsTo: Verbosity.defaultValue,
@ -1370,91 +1373,3 @@ StreamSubscription<String> listenAndCompile(CompilerInterface compiler,
}
});
}
/// Entry point for this module, that creates `_FrontendCompiler` instance and
/// processes user input.
/// `compiler` is an optional parameter so it can be replaced with mocked
/// version for testing.
Future<int> starter(
List<String> args, {
CompilerInterface compiler,
Stream<List<int>> input,
StringSink output,
IncrementalCompiler generator,
BinaryPrinterFactory binaryPrinterFactory,
}) async {
ArgResults options;
try {
options = argParser.parse(args);
} catch (error) {
print('ERROR: $error\n');
print(usage);
return 1;
}
if (options['train']) {
if (options.rest.isEmpty) {
throw Exception('Must specify input.dart');
}
final String input = options.rest[0];
final String sdkRoot = options['sdk-root'];
final String platform = options['platform'];
final Directory temp =
Directory.systemTemp.createTempSync('train_frontend_server');
try {
final String outputTrainingDill = path.join(temp.path, 'app.dill');
final List<String> args = <String>[
'--incremental',
'--sdk-root=$sdkRoot',
'--output-dill=$outputTrainingDill',
];
if (platform != null) {
args.add('--platform=${Uri.file(platform)}');
}
options = argParser.parse(args);
compiler ??=
FrontendCompiler(output, printerFactory: binaryPrinterFactory);
await compiler.compile(input, options, generator: generator);
compiler.acceptLastDelta();
await compiler.recompileDelta();
compiler.acceptLastDelta();
compiler.resetIncrementalCompiler();
await compiler.recompileDelta();
compiler.acceptLastDelta();
await compiler.recompileDelta();
compiler.acceptLastDelta();
return 0;
} finally {
temp.deleteSync(recursive: true);
}
}
final binaryProtocolAddressStr = options['binary-protocol-address'];
if (binaryProtocolAddressStr is String) {
runBinaryProtocol(binaryProtocolAddressStr);
return 0;
}
compiler ??= FrontendCompiler(output,
printerFactory: binaryPrinterFactory,
unsafePackageSerialization: options["unsafe-package-serialization"],
incrementalSerialization: options["incremental-serialization"],
useDebuggerModuleNames: options['debugger-module-names'],
emitDebugMetadata: options['experimental-emit-debug-metadata'],
emitDebugSymbols: options['emit-debug-symbols']);
if (options.rest.isNotEmpty) {
return await compiler.compile(options.rest[0], options,
generator: generator)
? 0
: 254;
}
Completer<int> completer = Completer<int>();
var subscription = listenAndCompile(
compiler, input ?? stdin, options, completer,
generator: generator);
return completer.future..then((value) => subscription.cancel());
}

385
pkg/frontend_server/lib/src/resident_frontend_server.dart Normal file
View file

@ -0,0 +1,385 @@
// Copyright 2022 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// @dart = 2.9
import 'dart:async';
import 'dart:convert';
import 'dart:io' show File, InternetAddress, ServerSocket, Socket;
import 'dart:typed_data' show Uint8List;
import 'package:args/args.dart';
// front_end/src imports below that require lint `ignore_for_file`
// are a temporary state of things until frontend team builds better api
// that would replace api used below. This api was made private in
// an effort to discourage further use.
// ignore_for_file: implementation_imports
import 'package:front_end/src/api_unstable/vm.dart';
import '../frontend_server.dart';
extension on DateTime {
/// Truncates by [amount]
DateTime floorTime(Duration amount) {
return DateTime.fromMillisecondsSinceEpoch(this.millisecondsSinceEpoch -
this.millisecondsSinceEpoch % amount.inMilliseconds);
}
}
enum _ResidentState {
WAITING_FOR_FIRST_COMPILE,
COMPILING,
WAITING_FOR_RECOMPILE,
}
/// A wrapper around the FrontendCompiler, along with all the state needed
/// to perform incremental compilations
///
/// TODO: Fix the race condition that occurs when the ResidentCompiler returns
/// a kernel file to the CLI and another compilation request is given before
/// the VM is able to launch from the kernel that was returned in the first
/// compile request. The ResidentCompiler will be in the state of waiting for
/// a recompile request and will subsequently process the request and modify
/// the kernel file. However, it should be waiting for the VM to finish
/// launching itself from this kernel until it modifies the kernel.
/// As far as I can tell this race also exists in the current CLI run
/// command when using pub's precompile pipeline.
///
/// TODO Fix the race condition that occurs when the same entry point is
/// compiled concurrently.
class ResidentCompiler {
File _entryPoint;
File _outputDill;
File _currentPackage;
ArgResults _compileOptions;
FrontendCompiler _compiler;
DateTime _lastCompileStartTime =
DateTime.now().floorTime(Duration(seconds: 1));
_ResidentState _state = _ResidentState.WAITING_FOR_FIRST_COMPILE;
final StringBuffer _compilerOutput = StringBuffer();
final Set<Uri> trackedSources = <Uri>{};
final List<String> _formattedOutput = <String>[];
ResidentCompiler(this._entryPoint, this._outputDill, this._compileOptions) {
_compiler = FrontendCompiler(_compilerOutput);
updateState(_compileOptions);
}
/// The [ResidentCompiler] will use the [newOptions] for future compilation
/// requests.
void updateState(ArgResults newOptions) {
final packages = newOptions['packages'];
_compileOptions = newOptions;
_currentPackage = packages == null ? null : File(packages);
// Refresh the compiler's output for the next compile
_compilerOutput.clear();
_formattedOutput.clear();
_state = _ResidentState.WAITING_FOR_FIRST_COMPILE;
}
/// Determines whether the current compile options are outdated with respect
/// to the [newOptions]
///
/// TODO: account for all compiler options. See vm/bin/kernel_service.dart:88
bool areOptionsOutdated(ArgResults newOptions) {
final packagesPath = newOptions['packages'];
return (packagesPath != null && _currentPackage == null) ||
(packagesPath == null && _currentPackage != null) ||
(_currentPackage != null && _currentPackage.path != packagesPath) ||
(_currentPackage != null &&
!_lastCompileStartTime.isAfter(_currentPackage
.statSync()
.modified
.floorTime(Duration(seconds: 1))));
}
/// Compiles the entry point that this ResidentCompiler is hooked to.
/// Will perform incremental compilations when possible.
/// If the options are outdated, must use updateState to get a correct
/// compile.
Future<String> compile() async {
var incremental = false;
// If this entrypoint was previously compiled on this compiler instance,
// check which source files need to be recompiled in the incremental
// compilation request. If no files have been modified, we can return
// the cached kernel. Otherwise, perform an incremental compilation.
if (_state == _ResidentState.WAITING_FOR_RECOMPILE) {
var invalidatedUris =
await _getSourceFilesToRecompile(_lastCompileStartTime);
// No changes to source files detected and cached kernel file exists
// If a kernel file is removed in between compilation requests,
// fall through to procude the kernel in recompileDelta.
if (invalidatedUris.isEmpty && _outputDill.existsSync()) {
return _encodeCompilerOutput(
_outputDill.path, _formattedOutput, _compiler.errors.length,
usingCachedKernel: true);
}
_state = _ResidentState.COMPILING;
incremental = true;
invalidatedUris
.forEach((invalidatedUri) => _compiler.invalidate(invalidatedUri));
_compiler.errors.clear();
_lastCompileStartTime = DateTime.now().floorTime(Duration(seconds: 1));
await _compiler.recompileDelta(entryPoint: _entryPoint.path);
} else {
_state = _ResidentState.COMPILING;
_lastCompileStartTime = DateTime.now().floorTime(Duration(seconds: 1));
_compiler.errors.clear();
await _compiler.compile(_entryPoint.path, _compileOptions);
}
_interpretCompilerOutput(LineSplitter()
.convert(_compilerOutput.toString())
.where((line) => line.isNotEmpty)
.toList());
_compilerOutput.clear();
// forces the compiler to produce complete kernel files on each
// request, even when incrementally compiled.
_compiler
..acceptLastDelta()
..resetIncrementalCompiler();
_state = _ResidentState.WAITING_FOR_RECOMPILE;
return _encodeCompilerOutput(
_outputDill.path, _formattedOutput, _compiler.errors.length,
incrementalCompile: incremental);
}
/// Reads the compiler's [outputLines] to keep track of which files
/// need to be tracked. Adds correctly ANSI formatted output to
/// the [_formattedOutput] list.
void _interpretCompilerOutput(List<String> outputLines) {
_formattedOutput.clear();
var outputLineIndex = 0;
var acceptingErrorsOrVerboseOutput = true;
final boundaryKey = outputLines[outputLineIndex]
.substring(outputLines[outputLineIndex++].indexOf(' ') + 1);
var line = outputLines[outputLineIndex++];
while (acceptingErrorsOrVerboseOutput || !line.startsWith(boundaryKey)) {
if (acceptingErrorsOrVerboseOutput) {
if (line == boundaryKey) {
acceptingErrorsOrVerboseOutput = false;
} else {
_formattedOutput.add(line);
}
} else {
final diffUri = line.substring(1);
if (line.startsWith('+')) {
trackedSources.add(Uri.parse(diffUri));
} else if (line.startsWith('-')) {
trackedSources.remove(Uri.parse(diffUri));
}
}
line = outputLines[outputLineIndex++];
}
}
/// Returns a list of uris that need to be recompiled, based on the
/// [lastkernelCompileTime] timestamp.
/// Due to Windows timestamp granularity, all timestamps are truncated by
/// the second. This has no effect on correctness but may result in more
/// files being marked as invalid than are strictly required.
Future<List<Uri>> _getSourceFilesToRecompile(
DateTime lastKernelCompileTime) async {
final sourcesToRecompile = <Uri>[];
for (Uri uri in trackedSources) {
final sourceModifiedTime = File(uri.toFilePath())
.statSync()
.modified
.floorTime(Duration(seconds: 1));
if (!lastKernelCompileTime.isAfter(sourceModifiedTime)) {
sourcesToRecompile.add(uri);
}
}
return sourcesToRecompile;
}
/// Encodes [outputDillPath] and any [formattedErrors] in JSON to
/// be sent over the socket.
static String _encodeCompilerOutput(
String outputDillPath,
List<String> formattedErrors,
int errorCount, {
bool usingCachedKernel = false,
bool incrementalCompile = false,
}) {
return jsonEncode(<String, Object>{
"success": errorCount == 0,
"errorCount": errorCount,
"compilerOutputLines": formattedErrors,
"output-dill": outputDillPath,
if (usingCachedKernel) "returnedStoredKernel": true, // used for testing
if (incrementalCompile) "incremental": true, // used for testing
});
}
}
/// Maintains [FrontendCompiler] instances for kernel compilations, meant to be
/// used by the Dart CLI via sockets.
///
/// The [ResidentFrontendServer] manages compilation requests for VM targets
/// between any number of dart entrypoints, and utilizes incremental
/// compilation and existing kernel files for faster compile times.
///
/// Communication is handled on the socket set up by the
/// residentListenAndCompile method.
class ResidentFrontendServer {
static const _commandString = 'command';
static const _executableString = 'executable';
static const _packageString = 'packages';
static const _outputString = 'output-dill';
static const _shutdownString = 'shutdown';
static const _compilerLimit = 3;
static final shutdownCommand =
jsonEncode(<String, Object>{_commandString: _shutdownString});
static final _shutdownJsonResponse =
jsonEncode(<String, Object>{_shutdownString: true});
static final _sdkBinariesUri = computePlatformBinariesLocation();
static final _sdkUri = _sdkBinariesUri.resolve('../../');
static final _platformKernelUri =
_sdkBinariesUri.resolve('vm_platform_strong.dill');
static final Map<String, ResidentCompiler> compilers = {};
/// Takes in JSON [input] from the socket and compiles the request,
/// using incremental compilation if possible. Returns a JSON string to be
/// sent back to the client socket containing either an error message or the
/// kernel file to be used.
///
/// If the command is compile, paths the source file, package_config.json,
/// and the output-dill file must be provided via "executable", "packages",
/// and "output-dill".
static Future<String> handleRequest(String input) async {
Map<String, dynamic> request;
try {
request = jsonDecode(input);
} on FormatException {
return _encodeErrorMessage('$input is not valid JSON.');
}
switch (request[_commandString]) {
case 'compile':
if (request[_executableString] == null ||
request[_outputString] == null) {
return _encodeErrorMessage(
'compilation requests must include an $_executableString and an $_outputString path.');
}
final executablePath = request[_executableString];
final cachedDillPath = request[_outputString];
final options = argParser.parse(<String>[
'--sdk-root=${_sdkUri.toFilePath()}',
'--incremental',
if (request.containsKey(_packageString))
'--packages=${request[_packageString]}',
'--platform=${_platformKernelUri.path}',
'--output-dill=$cachedDillPath',
'--target=vm',
'--filesystem-scheme',
'org-dartlang-root',
if (request['verbose'] == true) '--verbose',
]);
var residentCompiler = compilers[executablePath];
if (residentCompiler == null) {
// Avoids using too much memory
if (compilers.length >= ResidentFrontendServer._compilerLimit) {
compilers.remove(compilers.keys.first);
}
residentCompiler = ResidentCompiler(
File(executablePath), File(cachedDillPath), options);
compilers[executablePath] = residentCompiler;
} else if (residentCompiler.areOptionsOutdated(options)) {
residentCompiler.updateState(options);
}
return await residentCompiler.compile();
case 'shutdown':
return _shutdownJsonResponse;
default:
return _encodeErrorMessage(
'Unsupported command: ${request[_commandString]}.');
}
}
/// Encodes the [message] in JSON to be sent over the socket.
static String _encodeErrorMessage(String message) =>
jsonEncode(<String, Object>{"success": false, "errorMessage": message});
/// Used to create compile requests for the ResidentFrontendServer.
/// Returns a JSON string that the resident compiler will be able to
/// interpret.
static String createCompileJSON(
{String executable,
String packages,
String outputDill,
bool verbose = false}) {
return jsonEncode(<String, Object>{
"command": "compile",
"executable": executable,
if (packages != null) "packages": packages,
"output-dill": outputDill,
"verbose": verbose,
});
}
}
/// Sends the JSON string [request] to the resident frontend server
/// and returns server's response in JSON
///
/// Clients must use this function when wanting to interact with a
/// ResidentFrontendServer instance.
Future<Map<String, dynamic>> sendAndReceiveResponse(
InternetAddress address, int port, String request) async {
try {
final client = await Socket.connect(address, port);
client.write(request);
final data = await client.first;
client.destroy();
return jsonDecode(String.fromCharCodes(data));
} catch (e) {
return <String, Object>{"success": false, "errorMessage": e.toString()};
}
}
/// Listens for compilation commands from socket connections on the
/// provided [address] and [port].
Future<StreamSubscription<Socket>> residentListenAndCompile(
InternetAddress address, int port, File serverInfoFile) async {
ServerSocket server;
try {
// TODO: have server shut itself off after period of inactivity
server = await ServerSocket.bind(address, port);
serverInfoFile
..writeAsStringSync(
'address:${server.address.address} port:${server.port}');
} catch (e) {
print('Error: $e\n');
return null;
}
print(
'Resident Frontend Compiler is listening at ${server.address.address}:${server.port}');
return server.listen((client) {
client.listen((Uint8List data) async {
String result = await ResidentFrontendServer.handleRequest(
String.fromCharCodes(data));
client.write(result);
if (result == ResidentFrontendServer._shutdownJsonResponse) {
if (serverInfoFile.existsSync()) {
serverInfoFile.deleteSync();
}
await server.close();
}
}, onError: (error) {
client.close();
}, onDone: () {
client.close();
});
}, onError: (_) {
if (serverInfoFile.existsSync()) {
serverInfoFile.deleteSync();
}
});
}

111
pkg/frontend_server/lib/starter.dart Normal file
View file

@ -0,0 +1,111 @@
// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE.md file.
// @dart = 2.9
import 'dart:async';
import 'dart:io' show Directory, File, InternetAddress, stdin;
import 'package:args/args.dart';
import 'package:path/path.dart' as path;
import 'package:vm/incremental_compiler.dart' show IncrementalCompiler;
import 'frontend_server.dart';
import 'src/binary_protocol.dart';
import 'src/resident_frontend_server.dart';
/// Entry point for this module, that creates either a `_FrontendCompiler`
/// instance or a `ResidentFrontendServer` instance and
/// processes user input.
/// `compiler` is an optional parameter so it can be replaced with mocked
/// version for testing.
Future<int> starter(
List<String> args, {
CompilerInterface compiler,
Stream<List<int>> input,
StringSink output,
IncrementalCompiler generator,
BinaryPrinterFactory binaryPrinterFactory,
}) async {
ArgResults options;
try {
options = argParser.parse(args);
} catch (error) {
print('ERROR: $error\n');
print(usage);
return 1;
}
if (options['resident-info-file-name'] != null) {
var serverSubscription = await residentListenAndCompile(
InternetAddress.loopbackIPv4,
0,
File(options['resident-info-file-name']));
return serverSubscription == null ? 1 : 0;
}
if (options['train']) {
if (options.rest.isEmpty) {
throw Exception('Must specify input.dart');
}
final String input = options.rest[0];
final String sdkRoot = options['sdk-root'];
final String platform = options['platform'];
final Directory temp =
Directory.systemTemp.createTempSync('train_frontend_server');
try {
final String outputTrainingDill = path.join(temp.path, 'app.dill');
final List<String> args = <String>[
'--incremental',
'--sdk-root=$sdkRoot',
'--output-dill=$outputTrainingDill',
];
if (platform != null) {
args.add('--platform=${Uri.file(platform)}');
}
options = argParser.parse(args);
compiler ??=
FrontendCompiler(output, printerFactory: binaryPrinterFactory);
await compiler.compile(input, options, generator: generator);
compiler.acceptLastDelta();
await compiler.recompileDelta();
compiler.acceptLastDelta();
compiler.resetIncrementalCompiler();
await compiler.recompileDelta();
compiler.acceptLastDelta();
await compiler.recompileDelta();
compiler.acceptLastDelta();
return 0;
} finally {
temp.deleteSync(recursive: true);
}
}
final binaryProtocolAddressStr = options['binary-protocol-address'];
if (binaryProtocolAddressStr is String) {
runBinaryProtocol(binaryProtocolAddressStr);
return 0;
}
compiler ??= FrontendCompiler(output,
printerFactory: binaryPrinterFactory,
unsafePackageSerialization: options["unsafe-package-serialization"],
incrementalSerialization: options["incremental-serialization"],
useDebuggerModuleNames: options['debugger-module-names'],
emitDebugMetadata: options['experimental-emit-debug-metadata'],
emitDebugSymbols: options['emit-debug-symbols']);
if (options.rest.isNotEmpty) {
return await compiler.compile(options.rest[0], options,
generator: generator)
? 0
: 254;
}
Completer<int> completer = Completer<int>();
var subscription = listenAndCompile(
compiler, input ?? stdin, options, completer,
generator: generator);
return completer.future..then((value) => subscription.cancel());
}

6
pkg/frontend_server/test/frontend_server_flutter.dart
View file

@ -1,3 +1,7 @@
// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE.md file.
// @dart = 2.9
import 'dart:async' show StreamController;
import 'dart:convert' show utf8, LineSplitter;
@ -13,7 +17,7 @@ import 'package:front_end/src/api_prototype/language_version.dart'
import 'package:front_end/src/api_unstable/vm.dart'
show CompilerOptions, NnbdMode, StandardFileSystem;
import 'package:frontend_server/frontend_server.dart';
import 'package:frontend_server/starter.dart';
main(List<String> args) async {
String flutterDir;

5
pkg/frontend_server/test/frontend_server_test.dart
View file

@ -1,3 +1,7 @@
// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE.md file.
// @dart = 2.9
// ignore_for_file: empty_catches
@ -10,6 +14,7 @@ import 'dart:typed_data';
import 'package:_fe_analyzer_shared/src/macros/compiler/request_channel.dart';
import 'package:front_end/src/api_unstable/vm.dart';
import 'package:frontend_server/frontend_server.dart';
import 'package:frontend_server/starter.dart';
import 'package:kernel/ast.dart' show Component;
import 'package:kernel/binary/ast_to_binary.dart';
import 'package:kernel/kernel.dart' show loadComponentFromBinary;

526
pkg/frontend_server/test/src/resident_frontend_server_test.dart Normal file
View file

@ -0,0 +1,526 @@
// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE.md file.
// @dart = 2.9
import 'dart:convert';
import 'dart:io';
import 'package:frontend_server/starter.dart';
import 'package:frontend_server/src/resident_frontend_server.dart';
import 'package:path/path.dart' as path;
import 'package:test/test.dart';
void main() async {
// Files are considered to be modified if the modification timestamp is
// during the same second of the last compile time due to the
// granularity of file stat on windows.
// Waiting for this number of milliseconds guarantees that the files in
// the unit tests will not be counted as modified.
const STAT_GRANULARITY = 1100;
group('Resident Frontend Server: invalid input: ', () {
test('no command given', () async {
final jsonResponse = await ResidentFrontendServer.handleRequest(
jsonEncode(<String, Object>{"no": "command"}));
expect(
jsonResponse,
equals(jsonEncode(<String, Object>{
"success": false,
"errorMessage": "Unsupported command: null."
})));
});
test('invalid command', () async {
final jsonResponse = await ResidentFrontendServer.handleRequest(
jsonEncode(<String, Object>{"command": "not a command"}));
expect(
jsonResponse,
equals(jsonEncode(<String, Object>{
"success": false,
"errorMessage": "Unsupported command: not a command."
})));
});
test('not a JSON request', () async {
final jsonResponse = await ResidentFrontendServer.handleRequest("hello");
expect(
jsonResponse,
equals(jsonEncode(<String, Object>{
"success": false,
"errorMessage": "hello is not valid JSON."
})));
});
test('missing files for compile command', () async {
final jsonResponse = await ResidentFrontendServer.handleRequest(
jsonEncode(<String, Object>{"command": "compile"}));
expect(
jsonResponse,
equals(jsonEncode(<String, Object>{
"success": false,
"errorMessage":
"compilation requests must include an executable and an output-dill path."
})));
});
});
group('Resident Frontend Server: compile tests: ', () {
Directory d;
File executable, package, cachedDill;
setUp(() async {
d = Directory.systemTemp.createTempSync();
executable = File(path.join(d.path, 'src1.dart'))
..createSync()
..writeAsStringSync('void main() {print("hello " "there");}');
package = File(path.join(d.path, '.dart_tool', 'package_config.json'))
..createSync(recursive: true)
..writeAsStringSync('''
{
"configVersion": 2,
"packages": [
{
"name": "hello",
"rootUri": "../",
"packageUri": "./"
}
]
}
''');
cachedDill = File(path.join(d.path, 'src1.dart.dill'));
});
tearDown(() async {
d.delete(recursive: true);
ResidentFrontendServer.compilers.clear();
});
test('initial compile, basic', () async {
final compileResult = jsonDecode(
await ResidentFrontendServer.handleRequest(
ResidentFrontendServer.createCompileJSON(
executable: executable.path,
packages: package.path,
outputDill: cachedDill.path)));
expect(compileResult['success'], true);
expect(compileResult['errorCount'], 0);
expect(compileResult['output-dill'], equals(cachedDill.path));
});
test('no package_config.json provided', () async {
final compileResult = jsonDecode(
await ResidentFrontendServer.handleRequest(
ResidentFrontendServer.createCompileJSON(
executable: executable.path, outputDill: cachedDill.path)));
expect(compileResult['success'], true);
expect(compileResult['errorCount'], 0);
expect(compileResult['output-dill'], equals(cachedDill.path));
});
test('incremental compilation', () async {
await Future.delayed(Duration(milliseconds: STAT_GRANULARITY));
final compileResults1 = jsonDecode(
await ResidentFrontendServer.handleRequest(
ResidentFrontendServer.createCompileJSON(
executable: executable.path,
packages: package.path,
outputDill: cachedDill.path,
)));
executable.writeAsStringSync(
executable.readAsStringSync().replaceFirst('there', 'world'));
final compileResults2 = jsonDecode(
await ResidentFrontendServer.handleRequest(
ResidentFrontendServer.createCompileJSON(
executable: executable.path,
packages: package.path,
outputDill: cachedDill.path,
)));
expect(compileResults1['success'], true);
expect(compileResults1['errorCount'],
allOf(0, equals(compileResults2['errorCount'])));
expect(compileResults2['output-dill'],
equals(compileResults1['output-dill']));
expect(compileResults2['incremental'], true);
expect(
ResidentFrontendServer
.compilers[executable.path].trackedSources.first,
equals(Uri.file(executable.path)));
expect(
ResidentFrontendServer
.compilers[executable.path].trackedSources.length,
1);
});
test(
'compiling twice with no modifications returns cached kernel without invoking compiler',
() async {
await Future.delayed(Duration(milliseconds: STAT_GRANULARITY));
final compileResults1 = jsonDecode(
await ResidentFrontendServer.handleRequest(
ResidentFrontendServer.createCompileJSON(
executable: executable.path,
packages: package.path,
outputDill: cachedDill.path)));
final compileResults2 = jsonDecode(
await ResidentFrontendServer.handleRequest(
ResidentFrontendServer.createCompileJSON(
executable: executable.path,
packages: package.path,
outputDill: cachedDill.path)));
expect(compileResults1['errorCount'],
allOf(0, equals(compileResults2['errorCount'])));
expect(compileResults1['output-dill'],
equals(compileResults2['output-dill']));
expect(compileResults2['returnedStoredKernel'], true);
expect(ResidentFrontendServer.compilers.length, 1);
});
test('switch entrypoints gracefully', () async {
final executable2 = File(path.join(d.path, 'src2.dart'))
..writeAsStringSync('void main() {}');
final entryPointDill = File(path.join(d.path, 'src2.dart.dill'));
final compileResults1 = jsonDecode(
await ResidentFrontendServer.handleRequest(
ResidentFrontendServer.createCompileJSON(
executable: executable.path,
packages: package.path,
outputDill: cachedDill.path)));
final compileResults2 = jsonDecode(
await ResidentFrontendServer.handleRequest(
ResidentFrontendServer.createCompileJSON(
executable: executable2.path,
packages: package.path,
outputDill: entryPointDill.path)));
expect(compileResults1['success'],
allOf(true, equals(compileResults2['success'])));
expect(
ResidentFrontendServer
.compilers[executable.path].trackedSources.length,
1);
expect(
ResidentFrontendServer
.compilers[executable.path].trackedSources.first,
equals(Uri.file(executable.path)));
expect(
ResidentFrontendServer
.compilers[executable2.path].trackedSources.length,
1);
expect(
ResidentFrontendServer
.compilers[executable2.path].trackedSources.first,
equals(Uri.file(executable2.path)));
expect(ResidentFrontendServer.compilers.length, 2);
});
test('Cached kernel is removed between compilation requests', () async {
await Future.delayed(Duration(milliseconds: STAT_GRANULARITY));
final compileResults1 = jsonDecode(
await ResidentFrontendServer.handleRequest(
ResidentFrontendServer.createCompileJSON(
executable: executable.path,
packages: package.path,
outputDill: cachedDill.path)));
executable.writeAsStringSync(
executable.readAsStringSync().replaceFirst('there', 'world'));
cachedDill.deleteSync();
expect(cachedDill.existsSync(), false);
final compileResults2 = jsonDecode(
await ResidentFrontendServer.handleRequest(
ResidentFrontendServer.createCompileJSON(
executable: executable.path,
packages: package.path,
outputDill: cachedDill.path)));
expect(compileResults1['success'], true);
expect(compileResults1['errorCount'],
allOf(equals(compileResults2['errorCount']), 0));
expect(compileResults2['returnedStoredKernel'], null);
expect(compileResults2['incremental'], true);
expect(cachedDill.existsSync(), true);
expect(ResidentFrontendServer.compilers.length, 1);
});
test('maintains tracked sources', () async {
await Future.delayed(Duration(milliseconds: STAT_GRANULARITY));
final executable2 = File(path.join(d.path, 'src2.dart'))
..createSync()
..writeAsStringSync('''
import 'src3.dart';
void main() {}''');
final executable3 = File(path.join(d.path, 'src3.dart'))
..createSync()
..writeAsStringSync('''
void fn() {}''');
// adding or removing package_config.json while maintaining the same entrypoint
// should not alter tracked sources
await ResidentFrontendServer.handleRequest(
ResidentFrontendServer.createCompileJSON(
executable: executable.path, outputDill: cachedDill.path));
await ResidentFrontendServer.handleRequest(
ResidentFrontendServer.createCompileJSON(
executable: executable.path,
packages: package.path,
outputDill: cachedDill.path));
final compileResult1 = jsonDecode(
await ResidentFrontendServer.handleRequest(
ResidentFrontendServer.createCompileJSON(
executable: executable.path, outputDill: cachedDill.path)));
expect(compileResult1['success'], true);
expect(compileResult1['returnedStoredKernel'], null);
expect(compileResult1['incremental'], null);
expect(
ResidentFrontendServer
.compilers[executable.path].trackedSources.length,
1);
expect(
ResidentFrontendServer
.compilers[executable.path].trackedSources.first,
equals(Uri.file(executable.path)));
// switching entrypoints, packages, and modifying packages
await ResidentFrontendServer.handleRequest(
ResidentFrontendServer.createCompileJSON(
executable: executable2.path, outputDill: cachedDill.path));
await ResidentFrontendServer.handleRequest(
ResidentFrontendServer.createCompileJSON(
executable: executable2.path,
packages: package.path,
outputDill: cachedDill.path));
package.writeAsStringSync(package.readAsStringSync());
// Forces package to be behind the next computed kernel by 1 second
// so that the final compilation will be incremental
await Future.delayed(Duration(milliseconds: STAT_GRANULARITY));
final compileResult2 = jsonDecode(
await ResidentFrontendServer.handleRequest(
ResidentFrontendServer.createCompileJSON(
executable: executable2.path,
packages: package.path,
outputDill: cachedDill.path)));
expect(compileResult2['success'], true);
expect(compileResult2['incremental'], null);
expect(compileResult2['returnedStoredKernel'], null);
expect(
ResidentFrontendServer
.compilers[executable2.path].trackedSources.length,
greaterThanOrEqualTo(2));
expect(
ResidentFrontendServer.compilers[executable2.path].trackedSources,
containsAll(
<Uri>{Uri.file(executable2.path), Uri.file(executable3.path)}));
// remove a source
executable2.writeAsStringSync('void main() {}');
final compileResult3 = jsonDecode(
await ResidentFrontendServer.handleRequest(
ResidentFrontendServer.createCompileJSON(
executable: executable2.path,
packages: package.path,
outputDill: cachedDill.path)));
expect(compileResult3['success'], true);
expect(compileResult3['incremental'], true);
expect(
ResidentFrontendServer
.compilers[executable2.path].trackedSources.length,
greaterThanOrEqualTo(1));
expect(ResidentFrontendServer.compilers[executable2.path].trackedSources,
containsAll(<Uri>{Uri.file(executable2.path)}));
});
test('continues to work after compiler error is produced', () async {
final originalContent = executable.readAsStringSync();
final newContent = originalContent.replaceAll(';', '@');
await Future.delayed(Duration(milliseconds: STAT_GRANULARITY));
executable.writeAsStringSync(newContent);
final compileResults1 = jsonDecode(
await ResidentFrontendServer.handleRequest(
ResidentFrontendServer.createCompileJSON(
executable: executable.path,
packages: package.path,
outputDill: cachedDill.path)));
executable.writeAsStringSync(originalContent);
final compileResults2 = jsonDecode(
await ResidentFrontendServer.handleRequest(
ResidentFrontendServer.createCompileJSON(
executable: executable.path,
packages: package.path,
outputDill: cachedDill.path)));
expect(compileResults1['success'], false);
expect(compileResults1['errorCount'], greaterThan(1));
expect(compileResults2['success'], true);
expect(compileResults2['errorCount'], 0);
expect(compileResults2['incremental'], true);
expect(
ResidentFrontendServer
.compilers[executable.path].trackedSources.length,
1);
expect(
ResidentFrontendServer
.compilers[executable.path].trackedSources.first,
equals(Uri.file(executable.path)));
});
test('using cached kernel maintains error messages', () async {
final originalContent = executable.readAsStringSync();
executable.writeAsStringSync(originalContent.replaceFirst(';', ''));
await Future.delayed(Duration(milliseconds: STAT_GRANULARITY));
final compileResults1 = jsonDecode(
await ResidentFrontendServer.handleRequest(
ResidentFrontendServer.createCompileJSON(
executable: executable.path,
packages: package.path,
outputDill: cachedDill.path,
)));
final compileResults2 = jsonDecode(
await ResidentFrontendServer.handleRequest(
ResidentFrontendServer.createCompileJSON(
executable: executable.path,
packages: package.path,
outputDill: cachedDill.path,
)));
executable.writeAsStringSync(originalContent);
final compileResults3 = jsonDecode(
await ResidentFrontendServer.handleRequest(
ResidentFrontendServer.createCompileJSON(
executable: executable.path,
packages: package.path,
outputDill: cachedDill.path,
)));
expect(compileResults2['returnedStoredKernel'], true);
expect(compileResults1['errorCount'],
allOf(1, equals(compileResults2['errorCount'])));
expect(
compileResults2['compilerOutputLines'] as List<dynamic>,
containsAllInOrder(
compileResults1['compilerOutputLines'] as List<dynamic>));
expect(compileResults3['errorCount'], 0);
expect(compileResults3['incremental'], true);
});
test('enforces compiler limit', () async {
final executable2 = File(path.join(d.path, 'src2.dart'))
..createSync()
..writeAsStringSync('''
import 'src3.dart';
void main() {}''');
final executable3 = File(path.join(d.path, 'src3.dart'))
..createSync()
..writeAsStringSync('''
void main() {}''');
final executable4 = File(path.join(d.path, 'src4.dart'))
..createSync()
..writeAsStringSync('''
void main() {}''');
final compileResults1 = jsonDecode(
await ResidentFrontendServer.handleRequest(
ResidentFrontendServer.createCompileJSON(
executable: executable.path,
packages: package.path,
outputDill: cachedDill.path)));
final compileResults2 = jsonDecode(
await ResidentFrontendServer.handleRequest(
ResidentFrontendServer.createCompileJSON(
executable: executable2.path,
packages: package.path,
outputDill: cachedDill.path)));
final compileResults3 = jsonDecode(
await ResidentFrontendServer.handleRequest(
ResidentFrontendServer.createCompileJSON(
executable: executable3.path,
packages: package.path,
outputDill: cachedDill.path)));
final compileResults4 = jsonDecode(
await ResidentFrontendServer.handleRequest(
ResidentFrontendServer.createCompileJSON(
executable: executable4.path,
packages: package.path,
outputDill: cachedDill.path)));
expect(
compileResults1['success'],
allOf(
true,
equals(compileResults2['success']),
equals(compileResults3['success']),
equals(compileResults4['success'])));
expect(ResidentFrontendServer.compilers.length, 3);
expect(
ResidentFrontendServer.compilers.containsKey(executable4.path), true);
});
});
group('Resident Frontend Server: socket tests: ', () {
Directory d;
File serverInfo;
setUp(() {
d = Directory.systemTemp.createTempSync();
serverInfo = File(path.join(d.path, 'info.txt'));
});
tearDown(() {
d.deleteSync(recursive: true);
});
test('ServerSocket fails to bind', () async {
final result = await residentListenAndCompile(
InternetAddress.loopbackIPv4, -1, serverInfo);
expect(serverInfo.existsSync(), false);
expect(result, null);
});
test('socket passes messages properly and shutsdown properly', () async {
await residentListenAndCompile(
InternetAddress.loopbackIPv4, 0, serverInfo);
expect(serverInfo.existsSync(), true);
final info = serverInfo.readAsStringSync();
final address = InternetAddress(
info.substring(info.indexOf(':') + 1, info.indexOf(' ')));
final port = int.parse(info.substring(info.lastIndexOf(':') + 1));
final shutdownResult = await sendAndReceiveResponse(
address, port, ResidentFrontendServer.shutdownCommand);
expect(shutdownResult, equals(<String, dynamic>{"shutdown": true}));
expect(serverInfo.existsSync(), false);
});
test('resident server starter', () async {
final returnValue =
starter(['--resident-info-file-name=${serverInfo.path}']);
expect(await returnValue, 0);
expect(serverInfo.existsSync(), true);
final info = serverInfo.readAsStringSync();
final address = InternetAddress(
info.substring(info.indexOf(':') + 1, info.indexOf(' ')));
final port = int.parse(info.substring(info.lastIndexOf(':') + 1));
var result = await sendAndReceiveResponse(
address, port, ResidentFrontendServer.shutdownCommand);
expect(result, equals(<String, dynamic>{"shutdown": true}));
expect(serverInfo.existsSync(), false);
result = await sendAndReceiveResponse(
address, port, ResidentFrontendServer.shutdownCommand);
expect(result['errorMessage'], contains('SocketException'));
});
});
}