From cfccd80ef718f58860b9160965b09295bfe5e6fa Mon Sep 17 00:00:00 2001 From: Joshua Litt Date: Mon, 9 Nov 2020 21:35:14 +0000 Subject: [PATCH] [dart2js] Break out closed world computation from global inference step. Change-Id: Id743ad11344e94ebc56f732d5580c33ae8e45e46 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/169123 Commit-Queue: Joshua Litt Reviewed-by: Sigmund Cherem --- pkg/compiler/lib/src/commandline_options.dart | 2 + pkg/compiler/lib/src/compiler.dart | 87 +++++++---- pkg/compiler/lib/src/dart2js.dart | 76 +++++++++- pkg/compiler/lib/src/options.dart | 15 ++ .../lib/src/serialization/strategies.dart | 115 +++++++++++++-- pkg/compiler/lib/src/serialization/task.dart | 96 +++++++++--- .../serialization_test_helper.dart | 21 +-- pkg/compiler/tool/modular_test_suite.dart | 139 ++++++++++++++++-- 8 files changed, 460 insertions(+), 91 deletions(-) diff --git a/pkg/compiler/lib/src/commandline_options.dart b/pkg/compiler/lib/src/commandline_options.dart index b042613acd7..1c7d0ed5040 100644 --- a/pkg/compiler/lib/src/commandline_options.dart +++ b/pkg/compiler/lib/src/commandline_options.dart @@ -104,6 +104,8 @@ class Flags { static const String dillDependencies = '--dill-dependencies'; static const String readData = '--read-data'; static const String writeData = '--write-data'; + static const String writeClosedWorld = '--write-closed-world'; + static const String readClosedWorld = '--read-closed-world'; static const String readCodegen = '--read-codegen'; static const String writeCodegen = '--write-codegen'; static const String codegenShard = '--codegen-shard'; diff --git a/pkg/compiler/lib/src/compiler.dart b/pkg/compiler/lib/src/compiler.dart index 62d62d55687..a1dd1f526c5 100644 --- a/pkg/compiler/lib/src/compiler.dart +++ b/pkg/compiler/lib/src/compiler.dart @@ -8,6 +8,7 @@ import 'dart:async' show Future; import 'package:front_end/src/api_unstable/dart2js.dart' show clearStringTokenCanonicalizer; +import 'package:kernel/ast.dart' as ir; import '../compiler_new.dart' as api; import 'backend_strategy.dart'; @@ -37,6 +38,7 @@ import 'io/source_information.dart' show SourceInformation; import 'js_backend/backend.dart' show CodegenInputs, JavaScriptImpactStrategy; import 'js_backend/inferred_data.dart'; import 'js_model/js_strategy.dart'; +import 'js_model/js_world.dart'; import 'kernel/kernel_strategy.dart'; import 'kernel/loader.dart' show KernelLoaderTask, KernelResult; import 'null_compiler_output.dart' show NullCompilerOutput; @@ -231,10 +233,23 @@ abstract class Compiler { Future runInternal(Uri uri) async { clearState(); assert(uri != null); - // As far as I can tell, this branch is only used by test code. reporter.log('Compiling $uri (${options.buildId})'); - if (options.readDataUri != null) { + if (options.readClosedWorldUri != null) { + ir.Component component = + await serializationTask.deserializeComponentAndUpdateOptions(); + JsClosedWorld closedWorld = + await serializationTask.deserializeClosedWorld( + environment, abstractValueStrategy, component); + GlobalTypeInferenceResults globalTypeInferenceResults = + performGlobalTypeInference(closedWorld); + if (options.writeDataUri != null) { + serializationTask + .serializeGlobalTypeInference(globalTypeInferenceResults); + return; + } + await generateJavaScriptCode(globalTypeInferenceResults); + } else if (options.readDataUri != null) { GlobalTypeInferenceResults globalTypeInferenceResults = await serializationTask.deserializeGlobalTypeInference( environment, abstractValueStrategy); @@ -362,9 +377,6 @@ abstract class Compiler { JClosedWorld closedWorld = closeResolution(mainFunction, resolutionEnqueuer.worldBuilder); - if (retainDataForTesting) { - backendClosedWorldForTesting = closedWorld; - } return closedWorld; } @@ -409,37 +421,50 @@ abstract class Compiler { checkQueue(codegenEnqueuer); } + GlobalTypeInferenceResults globalTypeInferenceResultsTestMode( + GlobalTypeInferenceResults results) { + SerializationStrategy strategy = const BytesInMemorySerializationStrategy(); + List irData = strategy.unpackAndSerializeComponent(results); + List worldData = strategy.serializeGlobalTypeInferenceResults(results); + return strategy.deserializeGlobalTypeInferenceResults( + options, + reporter, + environment, + abstractValueStrategy, + strategy.deserializeComponent(irData), + worldData); + } + void compileFromKernel(Uri rootLibraryUri, Iterable libraries) { _userCodeLocations.add(new CodeLocation(rootLibraryUri)); selfTask.measureSubtask("compileFromKernel", () { - JClosedWorld closedWorld = selfTask.measureSubtask("computeClosedWorld", + JsClosedWorld closedWorld = selfTask.measureSubtask("computeClosedWorld", () => computeClosedWorld(rootLibraryUri, libraries)); - if (stopAfterClosedWorld) return; - if (closedWorld != null) { - GlobalTypeInferenceResults globalInferenceResults = - performGlobalTypeInference(closedWorld); - if (options.writeDataUri != null) { - serializationTask - .serializeGlobalTypeInference(globalInferenceResults); - return; - } - if (options.testMode) { - SerializationStrategy strategy = - const BytesInMemorySerializationStrategy(); - List irData = - strategy.serializeComponent(globalInferenceResults); - List worldData = strategy.serializeData(globalInferenceResults); - globalInferenceResults = strategy.deserializeData( - options, - reporter, - environment, - abstractValueStrategy, - strategy.deserializeComponent(irData), - worldData); - } - if (stopAfterTypeInference) return; - generateJavaScriptCode(globalInferenceResults); + if (closedWorld == null) return; + + if (retainDataForTesting) { + backendClosedWorldForTesting = closedWorld; } + + if (options.writeClosedWorldUri != null) { + serializationTask.serializeComponent( + closedWorld.elementMap.programEnv.mainComponent); + serializationTask.serializeClosedWorld(closedWorld); + return; + } + if (stopAfterClosedWorld) return; + GlobalTypeInferenceResults globalInferenceResults = + performGlobalTypeInference(closedWorld); + if (options.writeDataUri != null) { + serializationTask.serializeGlobalTypeInference(globalInferenceResults); + return; + } + if (options.testMode) { + globalInferenceResults = + globalTypeInferenceResultsTestMode(globalInferenceResults); + } + if (stopAfterTypeInference) return; + generateJavaScriptCode(globalInferenceResults); }); } diff --git a/pkg/compiler/lib/src/dart2js.dart b/pkg/compiler/lib/src/dart2js.dart index 7142b49e8a3..c1bb9b35b83 100644 --- a/pkg/compiler/lib/src/dart2js.dart +++ b/pkg/compiler/lib/src/dart2js.dart @@ -107,6 +107,8 @@ Future compile(List argv, Uri sourceMapOut; Uri readDataUri; Uri writeDataUri; + Uri readClosedWorldUri; + Uri writeClosedWorldUri; Uri readCodegenUri; Uri writeCodegenUri; int codegenShard; @@ -267,6 +269,14 @@ Future compile(List argv, } } + void setReadClosedWorld(String argument) { + if (argument != Flags.readClosedWorld) { + readClosedWorldUri = + fe.nativeToUri(extractPath(argument, isDirectory: false)); + } + readStrategy = ReadStrategy.fromClosedWorld; + } + void setDillDependencies(String argument) { String dependencies = extractParameter(argument); String uriDependencies = dependencies.splitMapJoin(',', @@ -299,6 +309,9 @@ Future compile(List argv, fail("Cannot use ${Flags.cfeOnly} " "and write serialized data simultaneously."); } + if (writeStrategy == WriteStrategy.toClosedWorld) { + fail("Cannot write closed world and data simultaneously."); + } if (writeStrategy == WriteStrategy.toCodegen) { fail("Cannot write serialized data and codegen simultaneously."); } @@ -308,11 +321,32 @@ Future compile(List argv, writeStrategy = WriteStrategy.toData; } + void setWriteClosedWorld(String argument) { + if (writeStrategy == WriteStrategy.toKernel) { + fail("Cannot use ${Flags.cfeOnly} " + "and write serialized data simultaneously."); + } + if (writeStrategy == WriteStrategy.toData) { + fail("Cannot write both closed world and data"); + } + if (writeStrategy == WriteStrategy.toCodegen) { + fail("Cannot write serialized data and codegen simultaneously."); + } + if (argument != Flags.writeClosedWorld) { + writeClosedWorldUri = + fe.nativeToUri(extractPath(argument, isDirectory: false)); + } + writeStrategy = WriteStrategy.toClosedWorld; + } + void setWriteCodegen(String argument) { if (writeStrategy == WriteStrategy.toKernel) { fail("Cannot use ${Flags.cfeOnly} " "and write serialized codegen simultaneously."); } + if (writeStrategy == WriteStrategy.toClosedWorld) { + fail("Cannot write closed world and codegen simultaneously."); + } if (writeStrategy == WriteStrategy.toData) { fail("Cannot write serialized data and codegen data simultaneously."); } @@ -415,6 +449,10 @@ Future compile(List argv, new OptionHandler('${Flags.dillDependencies}=.+', setDillDependencies), new OptionHandler('${Flags.readData}|${Flags.readData}=.+', setReadData), new OptionHandler('${Flags.writeData}|${Flags.writeData}=.+', setWriteData), + new OptionHandler('${Flags.readClosedWorld}|${Flags.readClosedWorld}=.+', + setReadClosedWorld), + new OptionHandler('${Flags.writeClosedWorld}|${Flags.writeClosedWorld}=.+', + setWriteClosedWorld), new OptionHandler( '${Flags.readCodegen}|${Flags.readCodegen}=.+', setReadCodegen), new OptionHandler( @@ -614,6 +652,19 @@ Future compile(List argv, "and read serialized codegen simultaneously."); } break; + case WriteStrategy.toClosedWorld: + out ??= Uri.base.resolve('out.dill'); + writeClosedWorldUri ??= Uri.base.resolve('$out.world'); + options.add('${Flags.writeClosedWorld}=${writeClosedWorldUri}'); + if (readStrategy == ReadStrategy.fromClosedWorld) { + fail("Cannot read and write serialized data simultaneously."); + } else if (readStrategy == ReadStrategy.fromData) { + fail("Cannot read from both closed world and data"); + } else if (readStrategy == ReadStrategy.fromCodegen) { + fail("Cannot read serialized codegen and " + "write serialized data simultaneously."); + } + break; case WriteStrategy.toData: out ??= Uri.base.resolve('out.dill'); writeDataUri ??= Uri.base.resolve('$out.data'); @@ -657,6 +708,10 @@ Future compile(List argv, switch (readStrategy) { case ReadStrategy.fromDart: break; + case ReadStrategy.fromClosedWorld: + readClosedWorldUri ??= Uri.base.resolve('$scriptName.world'); + options.add('${Flags.readClosedWorld}=${readClosedWorldUri}'); + break; case ReadStrategy.fromData: readDataUri ??= Uri.base.resolve('$scriptName.data'); options.add('${Flags.readData}=${readDataUri}'); @@ -673,6 +728,7 @@ Future compile(List argv, fail("${Flags.codegenShards} must be a positive integer."); } options.add('${Flags.codegenShards}=$codegenShards'); + break; } options.add('--out=$out'); if (writeStrategy == WriteStrategy.toJs) { @@ -707,6 +763,13 @@ Future compile(List argv, inputSize = inputProvider.dartCharactersRead; summary = 'Dart file $input '; break; + case ReadStrategy.fromClosedWorld: + inputName = 'bytes data'; + inputSize = inputProvider.dartCharactersRead; + String dataInput = + fe.relativizeUri(Uri.base, readClosedWorldUri, Platform.isWindows); + summary = 'Data files $input and $dataInput '; + break; case ReadStrategy.fromData: inputName = 'bytes data'; inputSize = inputProvider.dartCharactersRead; @@ -742,6 +805,15 @@ Future compile(List argv, String output = fe.relativizeUri(Uri.base, out, Platform.isWindows); summary += 'compiled to dill: ${output}.'; break; + case WriteStrategy.toClosedWorld: + processName = 'Serialized'; + outputName = 'bytes data'; + outputSize = outputProvider.totalDataWritten; + String output = fe.relativizeUri(Uri.base, out, Platform.isWindows); + String dataOutput = + fe.relativizeUri(Uri.base, writeClosedWorldUri, Platform.isWindows); + summary += 'serialized to dill and data: ${output} and ${dataOutput}.'; + break; case WriteStrategy.toData: processName = 'Serialized'; outputName = 'bytes data'; @@ -1235,5 +1307,5 @@ void batchMain(List batchArguments) { }); } -enum ReadStrategy { fromDart, fromData, fromCodegen } -enum WriteStrategy { toKernel, toData, toCodegen, toJs } +enum ReadStrategy { fromDart, fromClosedWorld, fromData, fromCodegen } +enum WriteStrategy { toKernel, toClosedWorld, toData, toCodegen, toJs } diff --git a/pkg/compiler/lib/src/options.dart b/pkg/compiler/lib/src/options.dart index 9157cbfbd56..378c5b4fa88 100644 --- a/pkg/compiler/lib/src/options.dart +++ b/pkg/compiler/lib/src/options.dart @@ -77,6 +77,17 @@ class CompilerOptions implements DiagnosticOptions { /// If this is set, the compilation stops after type inference. Uri writeDataUri; + /// Location from which the serialized closed world is read. + /// + /// If this is set, the [entryPoint] is expected to be a .dill file and the + /// frontend work is skipped. + Uri readClosedWorldUri; + + /// Location to which inference data is serialized. + /// + /// If this is set, the compilation stops after computing the closed world. + Uri writeClosedWorldUri; + /// Location from which codegen data is read. /// /// If this is set, the compilation starts at codegen enqueueing. @@ -497,6 +508,10 @@ class CompilerOptions implements DiagnosticOptions { _extractUriListOption(options, '${Flags.dillDependencies}') ..readDataUri = _extractUriOption(options, '${Flags.readData}=') ..writeDataUri = _extractUriOption(options, '${Flags.writeData}=') + ..readClosedWorldUri = + _extractUriOption(options, '${Flags.readClosedWorld}=') + ..writeClosedWorldUri = + _extractUriOption(options, '${Flags.writeClosedWorld}=') ..readCodegenUri = _extractUriOption(options, '${Flags.readCodegen}=') ..writeCodegenUri = _extractUriOption(options, '${Flags.writeCodegen}=') ..codegenShard = _extractIntOption(options, '${Flags.codegenShard}=') diff --git a/pkg/compiler/lib/src/serialization/strategies.dart b/pkg/compiler/lib/src/serialization/strategies.dart index 0c159f93368..37fbe893a38 100644 --- a/pkg/compiler/lib/src/serialization/strategies.dart +++ b/pkg/compiler/lib/src/serialization/strategies.dart @@ -24,13 +24,18 @@ import 'task.dart'; abstract class SerializationStrategy { const SerializationStrategy(); - List serializeComponent(GlobalTypeInferenceResults results) { + List unpackAndSerializeComponent(GlobalTypeInferenceResults results) { JsClosedWorld closedWorld = results.closedWorld; ir.Component component = closedWorld.elementMap.programEnv.mainComponent; - return ir.serializeComponent(component); + return serializeComponent(component); } - List serializeData(GlobalTypeInferenceResults results); + List serializeGlobalTypeInferenceResults( + GlobalTypeInferenceResults results); + + List serializeComponent(ir.Component component) { + return ir.serializeComponent(component); + } ir.Component deserializeComponent(List data) { ir.Component component = new ir.Component(); @@ -38,7 +43,17 @@ abstract class SerializationStrategy { return component; } - GlobalTypeInferenceResults deserializeData( + GlobalTypeInferenceResults deserializeGlobalTypeInferenceResults( + CompilerOptions options, + DiagnosticReporter reporter, + Environment environment, + AbstractValueStrategy abstractValueStrategy, + ir.Component component, + List data); + + List serializeClosedWorld(JsClosedWorld closedWorld); + + JsClosedWorld deserializeClosedWorld( CompilerOptions options, DiagnosticReporter reporter, Environment environment, @@ -53,15 +68,16 @@ class BytesInMemorySerializationStrategy extends SerializationStrategy { const BytesInMemorySerializationStrategy({this.useDataKinds: false}); @override - List serializeData(GlobalTypeInferenceResults results) { + List serializeGlobalTypeInferenceResults( + GlobalTypeInferenceResults results) { ByteSink byteSink = new ByteSink(); DataSink sink = new BinarySink(byteSink, useDataKinds: useDataKinds); - serializeGlobalTypeInferenceResults(results, sink); + serializeGlobalTypeInferenceResultsToSink(results, sink); return byteSink.builder.takeBytes(); } @override - GlobalTypeInferenceResults deserializeData( + GlobalTypeInferenceResults deserializeGlobalTypeInferenceResults( CompilerOptions options, DiagnosticReporter reporter, Environment environment, @@ -69,7 +85,28 @@ class BytesInMemorySerializationStrategy extends SerializationStrategy { ir.Component component, List data) { DataSource source = new BinarySourceImpl(data, useDataKinds: useDataKinds); - return deserializeGlobalTypeInferenceResults(options, reporter, environment, + return deserializeGlobalTypeInferenceResultsFromSource(options, reporter, + environment, abstractValueStrategy, component, source); + } + + @override + List serializeClosedWorld(JsClosedWorld closedWorld) { + ByteSink byteSink = new ByteSink(); + DataSink sink = new BinarySink(byteSink, useDataKinds: useDataKinds); + serializeClosedWorldToSink(closedWorld, sink); + return byteSink.builder.takeBytes(); + } + + @override + JsClosedWorld deserializeClosedWorld( + CompilerOptions options, + DiagnosticReporter reporter, + Environment environment, + AbstractValueStrategy abstractValueStrategy, + ir.Component component, + List data) { + DataSource source = new BinarySourceImpl(data, useDataKinds: useDataKinds); + return deserializeClosedWorldFromSource(options, reporter, environment, abstractValueStrategy, component, source); } } @@ -80,17 +117,18 @@ class BytesOnDiskSerializationStrategy extends SerializationStrategy { const BytesOnDiskSerializationStrategy({this.useDataKinds: false}); @override - List serializeData(GlobalTypeInferenceResults results) { + List serializeGlobalTypeInferenceResults( + GlobalTypeInferenceResults results) { Uri uri = Uri.base.resolve('world.data'); DataSink sink = new BinarySink( new BinaryOutputSinkAdapter(new RandomAccessBinaryOutputSink(uri)), useDataKinds: useDataKinds); - serializeGlobalTypeInferenceResults(results, sink); + serializeGlobalTypeInferenceResultsToSink(results, sink); return new File.fromUri(uri).readAsBytesSync(); } @override - GlobalTypeInferenceResults deserializeData( + GlobalTypeInferenceResults deserializeGlobalTypeInferenceResults( CompilerOptions options, DiagnosticReporter reporter, Environment environment, @@ -98,7 +136,30 @@ class BytesOnDiskSerializationStrategy extends SerializationStrategy { ir.Component component, List data) { DataSource source = new BinarySourceImpl(data, useDataKinds: useDataKinds); - return deserializeGlobalTypeInferenceResults(options, reporter, environment, + return deserializeGlobalTypeInferenceResultsFromSource(options, reporter, + environment, abstractValueStrategy, component, source); + } + + @override + List serializeClosedWorld(JsClosedWorld closedWorld) { + Uri uri = Uri.base.resolve('closed_world.data'); + DataSink sink = new BinarySink( + new BinaryOutputSinkAdapter(new RandomAccessBinaryOutputSink(uri)), + useDataKinds: useDataKinds); + serializeClosedWorldToSink(closedWorld, sink); + return new File.fromUri(uri).readAsBytesSync(); + } + + @override + JsClosedWorld deserializeClosedWorld( + CompilerOptions options, + DiagnosticReporter reporter, + Environment environment, + AbstractValueStrategy abstractValueStrategy, + ir.Component component, + List data) { + DataSource source = new BinarySourceImpl(data, useDataKinds: useDataKinds); + return deserializeClosedWorldFromSource(options, reporter, environment, abstractValueStrategy, component, source); } } @@ -110,15 +171,16 @@ class ObjectsInMemorySerializationStrategy const ObjectsInMemorySerializationStrategy({this.useDataKinds: true}); @override - List serializeData(GlobalTypeInferenceResults results) { + List serializeGlobalTypeInferenceResults( + GlobalTypeInferenceResults results) { List data = []; DataSink sink = new ObjectSink(data, useDataKinds: useDataKinds); - serializeGlobalTypeInferenceResults(results, sink); + serializeGlobalTypeInferenceResultsToSink(results, sink); return data; } @override - GlobalTypeInferenceResults deserializeData( + GlobalTypeInferenceResults deserializeGlobalTypeInferenceResults( CompilerOptions options, DiagnosticReporter reporter, Environment environment, @@ -126,7 +188,28 @@ class ObjectsInMemorySerializationStrategy ir.Component component, List data) { DataSource source = new ObjectSource(data, useDataKinds: useDataKinds); - return deserializeGlobalTypeInferenceResults(options, reporter, environment, + return deserializeGlobalTypeInferenceResultsFromSource(options, reporter, + environment, abstractValueStrategy, component, source); + } + + @override + List serializeClosedWorld(JsClosedWorld closedWorld) { + List data = []; + DataSink sink = new ObjectSink(data, useDataKinds: useDataKinds); + serializeClosedWorldToSink(closedWorld, sink); + return data; + } + + @override + JsClosedWorld deserializeClosedWorld( + CompilerOptions options, + DiagnosticReporter reporter, + Environment environment, + AbstractValueStrategy abstractValueStrategy, + ir.Component component, + List data) { + DataSource source = new ObjectSource(data, useDataKinds: useDataKinds); + return deserializeClosedWorldFromSource(options, reporter, environment, abstractValueStrategy, component, source); } } diff --git a/pkg/compiler/lib/src/serialization/task.dart b/pkg/compiler/lib/src/serialization/task.dart index 8a081d79c53..32d936c3c27 100644 --- a/pkg/compiler/lib/src/serialization/task.dart +++ b/pkg/compiler/lib/src/serialization/task.dart @@ -24,7 +24,7 @@ import '../util/sink_adapter.dart'; import '../world.dart'; import 'serialization.dart'; -void serializeGlobalTypeInferenceResults( +void serializeGlobalTypeInferenceResultsToSink( GlobalTypeInferenceResults results, DataSink sink) { JsClosedWorld closedWorld = results.closedWorld; InferredData inferredData = results.inferredData; @@ -34,7 +34,7 @@ void serializeGlobalTypeInferenceResults( sink.close(); } -GlobalTypeInferenceResults deserializeGlobalTypeInferenceResults( +GlobalTypeInferenceResults deserializeGlobalTypeInferenceResultsFromSource( CompilerOptions options, DiagnosticReporter reporter, Environment environment, @@ -49,6 +49,22 @@ GlobalTypeInferenceResults deserializeGlobalTypeInferenceResults( source, newClosedWorld.elementMap, newClosedWorld, newInferredData); } +void serializeClosedWorldToSink(JsClosedWorld closedWorld, DataSink sink) { + closedWorld.writeToDataSink(sink); + sink.close(); +} + +JsClosedWorld deserializeClosedWorldFromSource( + CompilerOptions options, + DiagnosticReporter reporter, + Environment environment, + AbstractValueStrategy abstractValueStrategy, + ir.Component component, + DataSource source) { + return new JsClosedWorld.readFromDataSource( + options, reporter, environment, abstractValueStrategy, component, source); +} + class SerializationTask extends CompilerTask { final CompilerOptions _options; final DiagnosticReporter _reporter; @@ -62,7 +78,7 @@ class SerializationTask extends CompilerTask { @override String get name => 'Serialization'; - void serializeGlobalTypeInference(GlobalTypeInferenceResults results) { + void serializeComponent(ir.Component component) { measureSubtask('serialize dill', () { // TODO(sigmund): remove entirely: we will do this immediately as soon as // we get the component in the kernel/loader.dart task once we refactor @@ -70,28 +86,15 @@ class SerializationTask extends CompilerTask { _reporter.log('Writing dill to ${_options.outputUri}'); api.BinaryOutputSink dillOutput = _outputProvider.createBinarySink(_options.outputUri); - JsClosedWorld closedWorld = results.closedWorld; - ir.Component component = closedWorld.elementMap.programEnv.mainComponent; BinaryOutputSinkAdapter irSink = new BinaryOutputSinkAdapter(dillOutput); ir.BinaryPrinter printer = new ir.BinaryPrinter(irSink); printer.writeComponentFile(component); irSink.close(); }); - - measureSubtask('serialize data', () { - _reporter.log('Writing data to ${_options.writeDataUri}'); - api.BinaryOutputSink dataOutput = - _outputProvider.createBinarySink(_options.writeDataUri); - DataSink sink = new BinarySink(new BinaryOutputSinkAdapter(dataOutput)); - serializeGlobalTypeInferenceResults(results, sink); - }); } - Future deserializeGlobalTypeInference( - Environment environment, - AbstractValueStrategy abstractValueStrategy) async { - ir.Component component = - await measureIoSubtask('deserialize dill', () async { + Future deserializeComponent() async { + return measureIoSubtask('deserialize dill', () async { _reporter.log('Reading dill from ${_options.entryPoint}'); api.Input> dillInput = await _provider .readFromUri(_options.entryPoint, inputKind: api.InputKind.binary); @@ -99,7 +102,9 @@ class SerializationTask extends CompilerTask { new ir.BinaryBuilder(dillInput.data).readComponent(component); return component; }); + } + void updateOptionsFromComponent(ir.Component component) { var isStrongDill = component.mode == ir.NonNullableByDefaultCompiledMode.Strong; var incompatibleNullSafetyMode = @@ -118,14 +123,65 @@ class SerializationTask extends CompilerTask { } else { _options.nullSafetyMode = NullSafetyMode.unsound; } + } + + Future deserializeComponentAndUpdateOptions() async { + ir.Component component = await deserializeComponent(); + updateOptionsFromComponent(component); + return component; + } + + void serializeClosedWorld(JsClosedWorld closedWorld) { + measureSubtask('serialize closed world', () { + _reporter.log('Writing closed world to ${_options.writeClosedWorldUri}'); + api.BinaryOutputSink dataOutput = + _outputProvider.createBinarySink(_options.writeClosedWorldUri); + DataSink sink = new BinarySink(new BinaryOutputSinkAdapter(dataOutput)); + serializeClosedWorldToSink(closedWorld, sink); + }); + } + + Future deserializeClosedWorld( + Environment environment, + AbstractValueStrategy abstractValueStrategy, + ir.Component component) async { + return await measureIoSubtask('deserialize closed world', () async { + _reporter.log('Reading data from ${_options.readClosedWorldUri}'); + api.Input> dataInput = await _provider.readFromUri( + _options.readClosedWorldUri, + inputKind: api.InputKind.binary); + DataSource source = new BinarySourceImpl(dataInput.data); + return deserializeClosedWorldFromSource(_options, _reporter, environment, + abstractValueStrategy, component, source); + }); + } + + void serializeGlobalTypeInference(GlobalTypeInferenceResults results) { + JsClosedWorld closedWorld = results.closedWorld; + ir.Component component = closedWorld.elementMap.programEnv.mainComponent; + serializeComponent(component); + + measureSubtask('serialize data', () { + _reporter.log('Writing data to ${_options.writeDataUri}'); + api.BinaryOutputSink dataOutput = + _outputProvider.createBinarySink(_options.writeDataUri); + DataSink sink = new BinarySink(new BinaryOutputSinkAdapter(dataOutput)); + serializeGlobalTypeInferenceResultsToSink(results, sink); + }); + } + + Future deserializeGlobalTypeInference( + Environment environment, + AbstractValueStrategy abstractValueStrategy) async { + ir.Component component = await deserializeComponentAndUpdateOptions(); return await measureIoSubtask('deserialize data', () async { _reporter.log('Reading data from ${_options.readDataUri}'); api.Input> dataInput = await _provider .readFromUri(_options.readDataUri, inputKind: api.InputKind.binary); DataSource source = new BinarySourceImpl(dataInput.data); - return deserializeGlobalTypeInferenceResults(_options, _reporter, - environment, abstractValueStrategy, component, source); + return deserializeGlobalTypeInferenceResultsFromSource(_options, + _reporter, environment, abstractValueStrategy, component, source); }); } diff --git a/pkg/compiler/test/serialization/serialization_test_helper.dart b/pkg/compiler/test/serialization/serialization_test_helper.dart index 389d37042d6..a3d44eaad3f 100644 --- a/pkg/compiler/test/serialization/serialization_test_helper.dart +++ b/pkg/compiler/test/serialization/serialization_test_helper.dart @@ -101,20 +101,21 @@ runTest( GlobalTypeInferenceResults cloneInferenceResults(Compiler compiler, GlobalTypeInferenceResults results, SerializationStrategy strategy) { - List irData = strategy.serializeComponent(results); + List irData = strategy.unpackAndSerializeComponent(results); - List worldData = strategy.serializeData(results); + List worldData = strategy.serializeGlobalTypeInferenceResults(results); print('data size: ${worldData.length}'); ir.Component newComponent = strategy.deserializeComponent(irData); - GlobalTypeInferenceResults newResults = strategy.deserializeData( - compiler.options, - compiler.reporter, - compiler.environment, - compiler.abstractValueStrategy, - newComponent, - worldData); - List newWorldData = strategy.serializeData(newResults); + GlobalTypeInferenceResults newResults = + strategy.deserializeGlobalTypeInferenceResults( + compiler.options, + compiler.reporter, + compiler.environment, + compiler.abstractValueStrategy, + newComponent, + worldData); + List newWorldData = strategy.serializeGlobalTypeInferenceResults(newResults); Expect.equals(worldData.length, newWorldData.length, "Reserialization data length mismatch."); for (int i = 0; i < worldData.length; i++) { diff --git a/pkg/compiler/tool/modular_test_suite.dart b/pkg/compiler/tool/modular_test_suite.dart index eb8953bc221..0fe73be6783 100644 --- a/pkg/compiler/tool/modular_test_suite.dart +++ b/pkg/compiler/tool/modular_test_suite.dart @@ -6,6 +6,7 @@ /// /// This is a shell that runs multiple tests, one per folder under `data/`. import 'dart:io'; +import 'dart:async'; import 'package:compiler/src/commandline_options.dart'; import 'package:front_end/src/compute_platform_binaries_location.dart' @@ -30,22 +31,40 @@ main(List args) async { _options = Options.parse(args); _packageConfig = await loadPackageConfigUri(packageConfigUri); await _resolveScripts(); - await runSuite( - sdkRoot.resolve('tests/modular/'), - 'tests/modular', - _options, - new IOPipeline([ - SourceToDillStep(), - GlobalAnalysisStep(), - Dart2jsCodegenStep(codeId0), - Dart2jsCodegenStep(codeId1), - Dart2jsEmissionStep(), - RunD8(), - ], cacheSharedModules: true)); + await Future.wait([ + runSuite( + sdkRoot.resolve('tests/modular/'), + 'tests/modular', + _options, + new IOPipeline([ + SourceToDillStep(), + ComputeClosedWorldStep(), + GlobalAnalysisStep(), + Dart2jsCodegenStep(codeId0), + Dart2jsCodegenStep(codeId1), + Dart2jsEmissionStep(), + RunD8(), + ], cacheSharedModules: true)), + // TODO(joshualitt) Delete this when we stop supporting this way of running + // the compiler. + runSuite( + sdkRoot.resolve('tests/modular/'), + 'tests/modular', + _options, + new IOPipeline([ + SourceToDillStep(), + LegacyGlobalAnalysisStep(), + Dart2jsCodegenStep(codeId0), + Dart2jsCodegenStep(codeId1), + Dart2jsEmissionStep(), + RunD8(), + ], cacheSharedModules: true)) + ]); } const dillId = const DataId("dill"); const updatedDillId = const DataId("udill"); +const closedWorldId = const DataId("world"); const globalDataId = const DataId("gdata"); const codeId = const ShardsDataId("code", 2); const codeId0 = const ShardDataId(codeId, 0); @@ -213,9 +232,105 @@ class SourceToDillStep implements IOModularStep { } } +// Step that invokes the dart2js closed world computation. +class ComputeClosedWorldStep implements IOModularStep { + @override + List get resultData => const [closedWorldId, updatedDillId]; + + @override + bool get needsSources => false; + + @override + List get dependencyDataNeeded => const [dillId]; + + @override + List get moduleDataNeeded => const [dillId]; + + @override + bool get onlyOnMain => true; + + @override + Future execute(Module module, Uri root, ModuleDataToRelativeUri toUri, + List flags) async { + if (_options.verbose) + print("\nstep: dart2js compute closed world on $module"); + Set transitiveDependencies = computeTransitiveDependencies(module); + Iterable dillDependencies = + transitiveDependencies.map((m) => '${toUri(m, dillId)}'); + List args = [ + '--packages=${sdkRoot.toFilePath()}/.packages', + _dart2jsScript, + // TODO(sigmund): remove this dependency on libraries.json + if (_options.useSdk) '--libraries-spec=$_librarySpecForSnapshot', + '${toUri(module, dillId)}', + for (String flag in flags) '--enable-experiment=$flag', + '${Flags.dillDependencies}=${dillDependencies.join(',')}', + '${Flags.writeClosedWorld}=${toUri(module, closedWorldId)}', + '--out=${toUri(module, updatedDillId)}', + ]; + var result = + await _runProcess(Platform.resolvedExecutable, args, root.toFilePath()); + + _checkExitCode(result, this, module); + } + + @override + void notifyCached(Module module) { + if (_options.verbose) + print("\ncached step: dart2js compute closed world on $module"); + } +} + // Step that invokes the dart2js global analysis on the main module by providing // the .dill files of all transitive modules as inputs. class GlobalAnalysisStep implements IOModularStep { + @override + List get resultData => const [globalDataId]; + + @override + bool get needsSources => false; + + @override + List get dependencyDataNeeded => const [updatedDillId]; + + @override + List get moduleDataNeeded => const [closedWorldId, updatedDillId]; + + @override + bool get onlyOnMain => true; + + @override + Future execute(Module module, Uri root, ModuleDataToRelativeUri toUri, + List flags) async { + if (_options.verbose) print("\nstep: dart2js global analysis on $module"); + List args = [ + '--packages=${sdkRoot.toFilePath()}/.packages', + _dart2jsScript, + // TODO(sigmund): remove this dependency on libraries.json + if (_options.useSdk) '--libraries-spec=$_librarySpecForSnapshot', + '${toUri(module, updatedDillId)}', + for (String flag in flags) '--enable-experiment=$flag', + '${Flags.readClosedWorld}=${toUri(module, closedWorldId)}', + '${Flags.writeData}=${toUri(module, globalDataId)}', + ]; + var result = + await _runProcess(Platform.resolvedExecutable, args, root.toFilePath()); + + _checkExitCode(result, this, module); + } + + @override + void notifyCached(Module module) { + if (_options.verbose) + print("\ncached step: dart2js global analysis on $module"); + } +} + +// Step that invokes the dart2js global analysis on the main module by providing +// the .dill files of all transitive modules as inputs. +// NOTE: This is the legacy combined closed world computation alongside global +// inference. +class LegacyGlobalAnalysisStep implements IOModularStep { @override List get resultData => const [globalDataId, updatedDillId];