Remove dartk and DFE workers support from testing scripts and kernel-service.

We only support running -c dark and -c dartkp configuration through Kernel isolate and fasta in non-batch mode.

R=kustermann@google.com
BUG=

Review-Url: https://codereview.chromium.org/2721543003 .
This commit is contained in:
Vyacheslav Egorov 2017-02-27 13:31:03 +01:00
parent c0404e4f23
commit b026aed3cf
5 changed files with 40 additions and 347 deletions

View file

@ -69,8 +69,6 @@ abstract class CompilerConfiguration {
bool useFastStartup = configuration['fast_startup'];
bool useKernelInDart2js = configuration['dart2js_with_kernel'];
bool verifyKernel = configuration['verify-ir'];
bool useDFE = !configuration['noDFE'];
bool useFasta = !configuration['noUseFasta'];
bool treeShake = !configuration['no-tree-shake'];
switch (compiler) {
@ -104,16 +102,6 @@ abstract class CompilerConfiguration {
useBlobs: useBlobs,
isAndroid: configuration['system'] == 'android');
case 'dartk':
if (!useDFE) {
return ComposedCompilerConfiguration.createDartKConfiguration(
isChecked: isChecked,
isHostChecked: isHostChecked,
useSdk: useSdk,
verify: verifyKernel,
strong: isStrong,
treeShake: treeShake);
}
return new NoneCompilerConfiguration(
isDebug: isDebug,
isChecked: isChecked,
@ -121,29 +109,15 @@ abstract class CompilerConfiguration {
useSdk: useSdk,
hotReload: hotReload,
hotReloadRollback: hotReloadRollback,
dfeMode: useFasta ? DFEMode.Fasta : DFEMode.DartK);
useDFE: true);
case 'dartkp':
if (!useDFE) {
return ComposedCompilerConfiguration.createDartKPConfiguration(
isChecked: isChecked,
isHostChecked: isHostChecked,
arch: configuration['arch'],
useBlobs: useBlobs,
isAndroid: configuration['system'] == 'android',
useSdk: useSdk,
verify: verifyKernel,
strong: isStrong,
treeShake: treeShake);
}
return new PrecompilerCompilerConfiguration(
isDebug: isDebug,
isChecked: isChecked,
arch: configuration['arch'],
useBlobs: useBlobs,
isAndroid: configuration['system'] == 'android',
dfeMode: useFasta ? DFEMode.Fasta : DFEMode.DartK);
useDFE: true);
case 'none':
return new NoneCompilerConfiguration(
isDebug: isDebug,
@ -210,23 +184,17 @@ abstract class CompilerConfiguration {
}
}
enum DFEMode {
None,
DartK,
Fasta
}
/// The "none" compiler.
class NoneCompilerConfiguration extends CompilerConfiguration {
final bool hotReload;
final bool hotReloadRollback;
final DFEMode dfeMode;
final bool useDFE;
NoneCompilerConfiguration(
{bool isDebug, bool isChecked, bool isHostChecked, bool useSdk,
bool this.hotReload,
bool this.hotReloadRollback,
DFEMode this.dfeMode: DFEMode.None})
this.useDFE: false})
: super._subclass(
isDebug: isDebug,
isChecked: isChecked,
@ -244,12 +212,9 @@ class NoneCompilerConfiguration extends CompilerConfiguration {
List<String> originalArguments,
CommandArtifact artifact) {
List<String> args = [];
if (dfeMode != DFEMode.None) {
if (useDFE) {
args.add('--dfe=utils/kernel-service/kernel-service.dart');
}
if (dfeMode == DFEMode.Fasta) {
args.add('-DDFE_USE_FASTA=true');
}
if (isChecked) {
args.add('--enable_asserts');
args.add('--enable_type_checks');
@ -613,10 +578,10 @@ class PrecompilerCompilerConfiguration extends CompilerConfiguration {
final String arch;
final bool useBlobs;
final bool isAndroid;
final DFEMode dfeMode;
final bool useDFE;
PrecompilerCompilerConfiguration({bool isDebug, bool isChecked,
this.arch, this.useBlobs, this.isAndroid, this.dfeMode: DFEMode.None})
this.arch, this.useBlobs, this.isAndroid, this.useDFE: false})
: super._subclass(isDebug: isDebug, isChecked: isChecked);
int computeTimeoutMultiplier() {
@ -657,12 +622,9 @@ class PrecompilerCompilerConfiguration extends CompilerConfiguration {
exec = "$buildDir/dart_bootstrap";
}
var args = new List();
if (dfeMode != DFEMode.None) {
if (useDFE) {
args.add('--dfe=utils/kernel-service/kernel-service.dart');
}
if (dfeMode == DFEMode.Fasta) {
args.add('-DDFE_USE_FASTA=true');
}
args.add("--snapshot-kind=app-aot");
if (useBlobs) {
args.add("--snapshot=$tempDir/out.aotsnapshot");

View file

@ -227,9 +227,6 @@ class StandaloneDartRuntimeConfiguration extends DartVmRuntimeConfiguration {
CommandArtifact artifact,
List<String> arguments,
Map<String, String> environmentOverrides) {
final bool needsDFERunner = suite.configuration['compiler'] == 'dartk' &&
suite.configuration['noUseFasta'] &&
!suite.configuration['noBatch'];
String script = artifact.filename;
String type = artifact.mimeType;
if (script != null &&
@ -239,7 +236,7 @@ class StandaloneDartRuntimeConfiguration extends DartVmRuntimeConfiguration {
}
String executable = suite.dartVmBinaryFileName;
return <Command>[
commandBuilder.getVmCommand(executable, arguments, environmentOverrides, needsDFERunner: needsDFERunner)
commandBuilder.getVmCommand(executable, arguments, environmentOverrides)
];
}
}

View file

@ -300,12 +300,6 @@ class TestOptionsParser {
new _TestOptionSpecification(
'no-tree-shake', 'Disable kernel IR tree shaking', ['--no-tree-shake'], [], false,
type: 'bool'),
new _TestOptionSpecification(
'noDFE', 'Do not use Kernel Isolate', ['--no-dfe'], [], false,
type: 'bool'),
new _TestOptionSpecification(
'noUseFasta', 'Use Fasta in Kernel Isolate', ['--no-use-fasta'], [], false,
type: 'bool'),
new _TestOptionSpecification(
'list', 'List tests only, do not run them', ['--list'], [], false,
type: 'bool'),

View file

@ -364,25 +364,14 @@ class AnalysisCommand extends ProcessCommand {
}
class VmCommand extends ProcessCommand {
final bool needsDFERunner;
VmCommand._(String executable, List<String> arguments,
Map<String, String> environmentOverrides,
bool this.needsDFERunner)
Map<String, String> environmentOverrides)
: super._("vm", executable, arguments, environmentOverrides);
void _buildHashCode(HashCodeBuilder builder) {
super._buildHashCode(builder);
builder.add(needsDFERunner);
}
bool _equal(VmCommand other) =>
super._equal(other) && needsDFERunner == other.needsDFERunner;
}
class VmBatchCommand extends ProcessCommand implements VmCommand {
final String dartFile;
final bool checked;
final needsDFERunner = false;
VmBatchCommand._(String executable, String dartFile, List<String> arguments,
Map<String, String> environmentOverrides, {this.checked: true})
@ -651,8 +640,8 @@ class CommandBuilder {
}
VmCommand getVmCommand(String executable, List<String> arguments,
Map<String, String> environmentOverrides, {bool needsDFERunner: false}) {
var command = new VmCommand._(executable, arguments, environmentOverrides, needsDFERunner);
Map<String, String> environmentOverrides) {
var command = new VmCommand._(executable, arguments, environmentOverrides);
return _getUniqueCommand(command);
}
@ -1901,12 +1890,10 @@ class RunningProcess {
bool compilationSkipped = false;
Completer<CommandOutput> completer;
Map configuration;
List<String> preArguments;
RunningProcess(this.command,
this.timeout,
{this.configuration,
this.preArguments});
{this.configuration});
Future<CommandOutput> run() {
completer = new Completer<CommandOutput>();
@ -1923,9 +1910,6 @@ class RunningProcess {
} else {
var processEnvironment = _createProcessEnvironment();
var args = command.arguments;
if (preArguments != null) {
args = []..addAll(preArguments)..addAll(args);
}
Future processFuture = io.Process.start(
command.executable, args,
environment: processEnvironment,
@ -2333,102 +2317,6 @@ class BatchRunnerProcess {
}
}
class BatchDFEProcess {
io.Process _process;
int _port = -1;
Function _processExitHandler;
Completer terminating = null;
bool locked = false;
Future<int> acquire() async {
try {
assert(!locked);
locked = true;
if (_process == null) {
await _startProcess();
}
return _port;
} catch(e) {
locked = false;
rethrow;
}
}
void release() {
locked = false;
}
Future terminate() {
locked = true;
if (_process == null) {
return new Future.value(true);
}
if (terminating == null) {
terminating = new Completer();
_process.kill();
}
return terminating.future;
}
_onExit(exitCode) {
if (terminating != null) {
terminating.complete();
return;
}
_process = null;
locked = false;
_port = -1;
}
static Future<String> _firstLine(stream) {
var completer = new Completer<String>();
stream.transform(UTF8.decoder)
.transform(new LineSplitter())
.listen((line) {
if (!completer.isCompleted) {
completer.complete(line);
}
// We need to drain a pipe continuously.
}, onDone: () {
if (!completer.isCompleted) {
completer.completeError(
"DFE kernel compiler server did not sucessfully start up");
}
});
return completer.future;
}
Future _startProcess() async {
final executable = io.Platform.executable;
final arguments = ['utils/kernel-service/kernel-service.dart', '--batch'];
try {
_port = -1;
_process = await io.Process.start(executable, arguments);
_process.exitCode.then(_onExit);
_process.stderr.transform(UTF8.decoder).listen(DebugLogger.error);
final readyMsg = await _firstLine(_process.stdout);
final data = readyMsg.split(' ');
assert(data[0] == 'READY');
_port = int.parse(data[1]);
} catch (e) {
print("Process error:");
print(" Command: $executable ${arguments.join(' ')}");
print(" Error: $e");
// If there is an error starting a batch process, chances are that
// it will always fail. So rather than re-trying a 1000+ times, we
// exit.
io.exit(1);
return true;
}
}
}
/**
* [TestCaseEnqueuer] takes a list of TestSuites, generates TestCases and
* builds a dependency graph of all commands in every TestSuite.
@ -2737,8 +2625,6 @@ class CommandExecutorImpl implements CommandExecutor {
// We keep a BrowserTestRunner for every configuration.
final _browserTestRunners = new Map<Map, BrowserTestRunner>();
List<BatchDFEProcess> _dfeProcesses = null;
bool _finishing = false;
CommandExecutorImpl(
@ -2763,13 +2649,9 @@ class CommandExecutorImpl implements CommandExecutor {
return Future.wait(futures);
}
Future _terminateDFEWorkers() =>
Future.wait((_dfeProcesses ?? <BatchDFEProcess>[]).map((p) => p.terminate()));
return Future.wait([
_terminateBatchRunners(),
_terminateBrowserRunners(),
_terminateDFEWorkers()
]);
}
@ -2818,13 +2700,6 @@ class CommandExecutorImpl implements CommandExecutor {
adbDevicePool.releaseDevice(device);
});
});
} else if (command is VmCommand && command.needsDFERunner) {
final runner = _getDFEProcess();
return runner.acquire().then((port) {
return new RunningProcess(command, timeout,
configuration: globalConfiguration,
preArguments: ['-DDFE_WORKER_PORT=${port}']).run();
}).whenComplete(() => runner.release());
} else if (command is VmBatchCommand) {
var name = command.displayName;
return _getBatchRunner(command.displayName + command.dartFile)
@ -2928,12 +2803,6 @@ class CommandExecutorImpl implements CommandExecutor {
throw new Exception('Unable to find inactive batch runner.');
}
BatchDFEProcess _getDFEProcess() {
_dfeProcesses ??= new List<BatchDFEProcess>.generate(maxProcesses,
(_) => new BatchDFEProcess());
return _dfeProcesses.firstWhere((runner) => !runner.locked);
}
Future<CommandOutput> _startBrowserControllerTest(
BrowserTestCommand browserCommand, int timeout) {
var completer = new Completer<CommandOutput>();

View file

@ -10,30 +10,13 @@
/// This is either invoked as the root script of the Kernel isolate when used
/// as a part of
///
/// dart --dfe=utils/kernel-service/kernel-service.dart ...
/// dart --dfe=utils/kernel-service/kernel-service.dart ...
///
/// invocation or it is invoked as a standalone script to perform batch mode
/// compilation requested via an HTTP interface
/// invocation or it is invoked as a standalone script to perform training for
/// the app-jit snapshot
///
/// dart utils/kernel-service/kernel-service.dart --batch
/// dart utils/kernel-service/kernel-service.dart --train <source-file>
///
/// The port for the batch mode worker is controlled by DFE_WORKER_PORT
/// environment declarations (set by -DDFE_WORKER_PORT=... command line flag).
/// When not set (or set to 0) an ephemeral port returned by the OS is used
/// instead.
///
/// When this script is used as a Kernel isolate root script and DFE_WORKER_PORT
/// is set to non-zero value then Kernel isolate will forward all compilation
/// requests it receives to the batch worker on the given port.
///
/// Set DFE_USE_FASTA environment declaration to true to use fasta front-end
/// instead of dartk. Note: we expect patched_sdk folder to contain
/// platform.dill file that contains patched SDK in the Kernel binary form.
/// This file can be created using the following command line:
///
/// export DART_AOT_SDK=<path-to-patched_sdk>
/// dart pkg/front_end/lib/src/fasta/bin/compile_platform.dart \
/// ${DART_AOT_SDK}/platform.dill
///
library runtime.tools.kernel_service;
@ -43,7 +26,6 @@ import 'dart:io';
import 'dart:isolate';
import 'dart:typed_data';
import 'package:kernel/analyzer/loader.dart';
import 'package:kernel/binary/ast_to_binary.dart';
import 'package:kernel/kernel.dart';
import 'package:kernel/target/targets.dart';
@ -57,8 +39,6 @@ import 'package:front_end/src/fasta/ast_kind.dart' show AstKind;
import 'package:front_end/src/fasta/errors.dart' show InputError;
const bool verbose = const bool.fromEnvironment('DFE_VERBOSE');
const int workerPort = const int.fromEnvironment('DFE_WORKER_PORT') ?? 0;
const bool useFasta = const bool.fromEnvironment('DFE_USE_FASTA');
class DataSink implements Sink<List<int>> {
final BytesBuilder builder = new BytesBuilder();
@ -93,19 +73,6 @@ class CompilationOk extends CompilationResult {
abstract class CompilationFail extends CompilationResult {
String get errorString;
Map<String, dynamic> toJson();
static CompilationFail fromJson(Map m) {
switch (m['status']) {
case STATUS_ERROR:
return new CompilationError(m['errors']);
case STATUS_CRASH:
return new CompilationCrash(m['exception'], m['stack']);
default:
throw "Can't deserialize CompilationFail from ${m}.";
}
}
}
class CompilationError extends CompilationFail {
@ -115,11 +82,6 @@ class CompilationError extends CompilationFail {
List toResponse() => [STATUS_ERROR, errorString];
Map<String, dynamic> toJson() => {
"status": STATUS_ERROR,
"errors": errors,
};
String get errorString => errors.take(10).join('\n');
String toString() => "CompilationError(${errorString})";
@ -133,18 +95,12 @@ class CompilationCrash extends CompilationFail {
List toResponse() => [STATUS_CRASH, errorString];
Map<String, dynamic> toJson() => {
"status": STATUS_CRASH,
"exception": exception,
"stack": stack,
};
String get errorString => "${exception}\n${stack}";
String toString() => "CompilationCrash(${errorString})";
}
Future<CompilationResult> parseScriptImpl(DartLoaderBatch batch_loader,
Future<CompilationResult> parseScriptImpl(
Uri fileName, String packageConfig, String sdkPath) async {
if (!FileSystemEntity.isFileSync(fileName.path)) {
throw "Input file '${fileName.path}' does not exist.";
@ -157,42 +113,24 @@ Future<CompilationResult> parseScriptImpl(DartLoaderBatch batch_loader,
Target target = getTarget("vm", new TargetFlags(strongMode: false));
Program program;
if (useFasta) {
final uriTranslator = await TranslateUri.parse(new Uri.file(packageConfig));
final Ticker ticker = new Ticker(isVerbose: verbose);
final DillTarget dillTarget = new DillTarget(ticker, uriTranslator);
dillTarget.read(new Uri.directory(sdkPath).resolve('platform.dill'));
final KernelTarget kernelTarget =
new KernelTarget(dillTarget, uriTranslator);
try {
kernelTarget.read(fileName);
await dillTarget.writeOutline(null);
program = await kernelTarget.writeOutline(null);
program = await kernelTarget.writeProgram(null, AstKind.Kernel);
if (kernelTarget.errors.isNotEmpty) {
return new CompilationError(kernelTarget.errors
.map((err) => err.toString())
.toList(growable: false));
}
} on InputError catch (e) {
return new CompilationError(<String>[e.format()]);
}
} else {
DartOptions dartOptions = new DartOptions(
strongMode: false,
strongModeSdk: false,
sdk: sdkPath,
packagePath: packageConfig,
customUriMappings: const {},
declaredVariables: const {});
program = new Program();
DartLoader loader =
await batch_loader.getLoader(program, dartOptions);
loader.loadProgram(fileName, target: target);
if (loader.errors.isNotEmpty) {
return new CompilationError(loader.errors.toList(growable: false));
final uriTranslator = await TranslateUri.parse(new Uri.file(packageConfig));
final Ticker ticker = new Ticker(isVerbose: verbose);
final DillTarget dillTarget = new DillTarget(ticker, uriTranslator);
dillTarget.read(new Uri.directory(sdkPath).resolve('platform.dill'));
final KernelTarget kernelTarget =
new KernelTarget(dillTarget, uriTranslator);
try {
kernelTarget.read(fileName);
await dillTarget.writeOutline(null);
program = await kernelTarget.writeOutline(null);
program = await kernelTarget.writeProgram(null, AstKind.Kernel);
if (kernelTarget.errors.isNotEmpty) {
return new CompilationError(kernelTarget.errors
.map((err) => err.toString())
.toList(growable: false));
}
} on InputError catch (e) {
return new CompilationError(<String>[e.format()]);
}
// Perform target-specific transformations.
@ -205,10 +143,10 @@ Future<CompilationResult> parseScriptImpl(DartLoaderBatch batch_loader,
return new CompilationOk(sink.builder.takeBytes());
}
Future<CompilationResult> parseScript(DartLoaderBatch loader, Uri fileName,
String packageConfig, String sdkPath) async {
Future<CompilationResult> parseScript(
Uri fileName, String packageConfig, String sdkPath) async {
try {
return await parseScriptImpl(loader, fileName, packageConfig, sdkPath);
return await parseScriptImpl(fileName, packageConfig, sdkPath);
} catch (err, stack) {
return new CompilationCrash(err.toString(), stack.toString());
}
@ -249,12 +187,7 @@ Future _processLoadRequestImpl(String inputFileUrl) async {
}""");
}
if (workerPort != 0) {
return await requestParse(scriptUri, packagesUri, patchedSdk);
} else {
return await parseScript(
new DartLoaderBatch(), scriptUri, packagesUri.path, patchedSdk.path);
}
return await parseScript(scriptUri, packagesUri.path, patchedSdk.path);
}
// Process a request from the runtime. See KernelIsolate::CompileToKernel in
@ -296,66 +229,6 @@ Future _processLoadRequest(request) async {
}
}
Future<CompilationResult> requestParse(
Uri scriptUri, Uri packagesUri, Uri patchedSdk) async {
if (verbose) {
print(
"DFE: forwarding request to worker at http://localhost:${workerPort}/");
}
HttpClient client = new HttpClient();
final rq = await client
.postUrl(new Uri(host: 'localhost', port: workerPort, scheme: 'http'));
rq.headers.contentType = ContentType.JSON;
rq.write(JSON.encode({
"inputFileUrl": scriptUri.toString(),
"packagesUri": packagesUri.toString(),
"patchedSdk": patchedSdk.toString(),
}));
final rs = await rq.close();
try {
if (rs.statusCode == HttpStatus.OK) {
final BytesBuilder bb = new BytesBuilder();
await rs.forEach(bb.add);
return new CompilationOk(bb.takeBytes());
} else {
return CompilationFail.fromJson(JSON.decode(await UTF8.decodeStream(rs)));
}
} finally {
await client.close();
}
}
void startBatchServer() {
final loader = new DartLoaderBatch();
HttpServer.bind('localhost', workerPort).then((server) {
print('READY ${server.port}');
server.listen((HttpRequest request) async {
final rq = JSON.decode(await UTF8.decodeStream(request));
final Uri scriptUri = Uri.parse(rq['inputFileUrl']);
final Uri packagesUri = Uri.parse(rq['packagesUri']);
final Uri patchedSdk = Uri.parse(rq['patchedSdk']);
final CompilationResult result = await parseScript(
loader, scriptUri, packagesUri.path, patchedSdk.path);
if (result is CompilationOk) {
request.response.statusCode = HttpStatus.OK;
request.response.headers.contentType = ContentType.BINARY;
request.response.add(result.binary);
request.response.close();
} else {
request.response.statusCode = HttpStatus.INTERNAL_SERVER_ERROR;
request.response.headers.contentType = ContentType.TEXT;
request.response.write(JSON.encode(result));
request.response.close();
}
});
ProcessSignal.SIGTERM.watch().first.then((_) => server.close());
});
}
train(String scriptUri) {
// TODO(28532): Enable on Windows.
if (Platform.isWindows) return;
@ -378,9 +251,7 @@ train(String scriptUri) {
}
main([args]) {
if (args?.length == 1 && args[0] == '--batch') {
startBatchServer();
} else if (args?.length == 2 && args[0] == '--train') {
if (args?.length == 2 && args[0] == '--train') {
// This entry point is used when creating an app snapshot. The argument
// provides a script to compile to warm-up generated code.
train(args[1]);