languages/dart-sdk
languages/dart-sdk
1
0
Fork 0
mirror of https://github.com/dart-lang/sdk synced 2024-11-02 12:24:24 +00:00
Code Issues Projects Releases Packages Wiki Activity

[dart2js] Use impacts from codegen results for dump info.

Browse source 
Currently, serialized dump info metadata (output of emitter phase) contains all the impacts created during codegen in a Map<MemberEntity, WorldImpact>. These impacts are already serialized in the output of the codegen phase. This is bad for two reasons: 1) The serialized bytes are duplicated. 2) The impacts cannot be GCed right after the emitter uses them, the dump info registry maintains a reference to them.

To mitigate this I've updated the registry to only store impacts for entities where the impacts are generated during the emitter phase (today only parameter stubs). Rather than try to re-generate those impacts during the dump info phase, we serialize them with the dump info metadata. However, most impacts are generated during the codegen phase and for these we deserialize them from the codegen results at the beginning of the dump info phase.

Change-Id: I1302b4ab759bca3d492bc6b40194d7a7720ed6f8
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/354580
Commit-Queue: Nate Biggs <natebiggs@google.com>
Reviewed-by: Sigmund Cherem <sigmund@google.com>
This commit is contained in:
Nate Biggs 2024-03-07 19:25:15 +00:00 committed by Commit Queue
parent 8de00e2137
commit 52f996b5bc
7 changed files with 152 additions and 85 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
pkg/compiler/lib/src/common/codegen.dart
View file

@ -405,8 +405,9 @@ class OnDemandCodegenResults implements CodegenResults {
OnDemandCodegenResults(this.codegenInputs, this._functionCompiler);
@override
CodegenResult getCodegenResults(MemberEntity member) {
return _functionCompiler.compile(member);
({CodegenResult result, bool isGenerated}) getCodegenResults(
MemberEntity member) {
return (result: _functionCompiler.compile(member), isGenerated: true);
}
}
@ -2229,7 +2230,8 @@ class ModularName extends js.Name implements js.AstContainer {
/// Interface for reading the code generation results for all [MemberEntity]s.
abstract class CodegenResults {
CodegenInputs get codegenInputs;
CodegenResult getCodegenResults(MemberEntity member);
({CodegenResult result, bool isGenerated}) getCodegenResults(
MemberEntity member);
}
/// Deserialized code generation results.
@ -2246,12 +2248,15 @@ class DeserializedCodegenResults implements CodegenResults {
this.codegenInputs, this._map, this._functionCompiler);
@override
CodegenResult getCodegenResults(MemberEntity member) {
({CodegenResult result, bool isGenerated}) getCodegenResults(
MemberEntity member) {
// We only access these results once as it is picked up by the work queue
// so it is safe to remove and free up space in the map. With deferred
// deserialization this will also free the Deferrable holder.
// Some entities such as parameter stubs are generated lazily and so we have
// to compile them on the fly.
return _map.remove(member) ?? _functionCompiler.compile(member);
final deserialized = _map.remove(member);
if (deserialized != null) return (result: deserialized, isGenerated: false);
return (result: _functionCompiler.compile(member), isGenerated: true);
}
}

13
pkg/compiler/lib/src/compiler.dart
View file

@ -687,7 +687,10 @@ class Compiler {
codegenResults, inferredData, sourceLookup, closedWorld);
if (options.dumpInfo || options.dumpInfoWriteUri != null) {
final dumpInfoData = DumpInfoProgramData.fromEmitterResults(
backendStrategy, dumpInfoRegistry, programSize);
backendStrategy.emitterTask,
dumpInfoRegistry,
codegenResults,
programSize);
dumpInfoRegistry.clear();
if (options.dumpInfoWriteUri != null) {
serializationTask.serializeDumpInfoProgramData(
@ -716,10 +719,12 @@ class Compiler {
dumpInfoState = await dumpInfoTask.dumpInfoNew(
untrimmedComponentForDumpInfo!,
closedWorld,
globalTypeInferenceResults);
globalTypeInferenceResults,
codegenResults,
backendStrategy);
} else {
dumpInfoState =
await dumpInfoTask.dumpInfo(closedWorld, globalTypeInferenceResults);
dumpInfoState = await dumpInfoTask.dumpInfo(closedWorld,
globalTypeInferenceResults, codegenResults, backendStrategy);
}
if (retainDataForTesting) {
dumpInfoStateForTesting = dumpInfoState;

156
pkg/compiler/lib/src/dump_info.dart
View file

@ -6,9 +6,7 @@ library dump_info;
import 'dart:convert' show JsonEncoder, JsonDecoder;
import 'package:compiler/src/js_model/elements.dart';
import 'package:compiler/src/serialization/serialization.dart';
import 'package:compiler/src/universe/use.dart';
import 'package:compiler/src/js_model/js_strategy.dart';
import 'package:dart2js_info/info.dart';
import 'package:dart2js_info/json_info_codec.dart';
import 'package:dart2js_info/binary_serialization.dart' as dump_info;
@ -17,14 +15,13 @@ import 'package:kernel/core_types.dart' as ir;
import '../compiler_api.dart' as api;
import 'common.dart';
import 'common/codegen.dart';
import 'common/elements.dart' show JElementEnvironment;
import 'common/names.dart';
import 'common/tasks.dart' show CompilerTask, Measurer;
import 'common/ram_usage.dart';
import 'constants/values.dart'
show ConstantValue, ConstantValueKind, DeferredGlobalConstantValue;
import 'deferred_load/output_unit.dart'
show OutputUnit, OutputUnitData, deferredPartFileName;
import 'constants/values.dart' show ConstantValue;
import 'deferred_load/output_unit.dart' show OutputUnit, deferredPartFileName;
import 'elements/entities.dart';
import 'elements/entity_utils.dart' as entity_utils;
import 'elements/names.dart';
@ -32,11 +29,13 @@ import 'inferrer/abstract_value_domain.dart';
import 'inferrer/types.dart'
show GlobalTypeInferenceMemberResult, GlobalTypeInferenceResults;
import 'js/js.dart' as jsAst;
import 'js_model/js_strategy.dart';
import 'js_emitter/code_emitter_task.dart';
import 'js_model/elements.dart';
import 'js_model/js_world.dart' show JClosedWorld;
import 'js_backend/field_analysis.dart';
import 'options.dart';
import 'universe/world_impact.dart' show WorldImpact, WorldImpactBuilderImpl;
import 'serialization/serialization.dart';
import 'universe/world_impact.dart' show WorldImpact;
/// Collects data used for the dump info task.
///
@ -52,7 +51,16 @@ class DumpInfoJsAstRegistry {
final Map<Entity, List<CodeSpan>> _entityCode = {};
final Map<ConstantValue, CodeSpan> _constantCode = {};
final Map<MemberEntity, WorldImpact> _impactRegistry = {};
// Contains impacts that will be used immediately (if codegen is being run
// with this compiler execution) or serialized with partial dump info data.
// Impacts are not yet transformed by CodegenImpactTransformer.
final Map<MemberEntity, CodegenImpact> _impactRegistry = {};
// Contains members whose impacts should be deserialized from codegen results
// on subsequent dump info execution. Empty when dump info is being executed
// immediately without serialization.
final Set<MemberEntity> _serializedImpactMembers = {};
// Temporary structures used to collect data during the visit process with a
// low memory footprint.
@ -79,9 +87,14 @@ class DumpInfoJsAstRegistry {
_constantRegistry[code] = constant;
}
void registerImpact(MemberEntity member, WorldImpact impact) {
void registerImpact(MemberEntity member, CodegenImpact impact,
{required bool isGenerated}) {
if (_disabled) return;
_impactRegistry[member] = impact;
if (isGenerated || options.dumpInfoWriteUri == null) {
_impactRegistry[member] = impact;
} else {
_serializedImpactMembers.add(member);
}
}
bool get shouldEmitText => !useBinaryFormat;
@ -125,9 +138,10 @@ class DumpInfoJsAstRegistry {
assert(_stack.isEmpty);
assert(_entityRegistry.isEmpty);
assert(_constantRegistry.isEmpty);
_impactRegistry.clear();
_entityCode.clear();
_constantCode.clear();
_serializedImpactMembers.clear();
_impactRegistry.clear();
}
}
@ -144,7 +158,17 @@ class DumpInfoProgramData {
final Map<Entity, List<CodeSpan>> entityCode;
final Map<Entity, int> entityCodeSize;
final Map<ConstantValue, CodeSpan> constantCode;
final Map<MemberEntity, WorldImpact> impacts;
/// Contains members that are live and whose impacts are serialized in the
/// codegen results. This will be empty if dump info is being run without
/// serialization.
final Set<MemberEntity> serializedImpactMembers;
/// If dump info is being without serialziation, this will contain impacts for
/// all live members. Otherwise only contains impacts for members that were
/// created during the emitter phase and whose impacts are therefore not
/// included in codegen results.
final Map<MemberEntity, CodegenImpact> registeredImpacts;
DumpInfoProgramData._(
this.programSize,
@ -153,15 +177,16 @@ class DumpInfoProgramData {
this.entityCode,
this.entityCodeSize,
this.constantCode,
this.impacts,
this.serializedImpactMembers,
this.registeredImpacts,
{required this.neededClasses,
required this.neededClassTypes});
factory DumpInfoProgramData.fromEmitterResults(
JsBackendStrategy backendStrategy,
CodeEmitterTask emitterTask,
DumpInfoJsAstRegistry dumpInfoRegistry,
CodegenResults codegenResults,
int programSize) {
final emitterTask = backendStrategy.emitterTask;
final outputUnitSizes = emitterTask.emitter.generatedSizes;
var fragmentsToLoad = emitterTask.emitter.finalizedFragmentsToLoad;
@ -184,6 +209,7 @@ class DumpInfoProgramData {
entityCode,
entityCodeSize,
constantCode,
Set.from(dumpInfoRegistry._serializedImpactMembers),
Map.from(dumpInfoRegistry._impactRegistry),
neededClasses: neededClasses,
neededClassTypes: neededClassTypes);
@ -230,8 +256,7 @@ class DumpInfoProgramData {
sink.writeStringOrNull(codeSpan.text);
}
factory DumpInfoProgramData.readFromDataSource(
DataSourceReader source, OutputUnitData outputUnitData,
factory DumpInfoProgramData.readFromDataSource(DataSourceReader source,
{required bool includeCodeText}) {
final programSize = source.readInt();
final outputUnitSizesLength = source.readInt();
@ -263,26 +288,20 @@ class DumpInfoProgramData {
final codeSpan = _readCodeSpan(source, includeCodeText);
constantCode[constant] = codeSpan;
}
final impacts = source.readMemberMap((_) {
final impactBuilder = WorldImpactBuilderImpl();
source
.readList(() => StaticUse.readFromDataSource(source))
.forEach(impactBuilder.registerStaticUse);
source
.readList(() => DynamicUse.readFromDataSource(source))
.forEach(impactBuilder.registerDynamicUse);
source
.readList(() => ConstantUse.readFromDataSource(source))
.forEach((use) {
assert(use.value.kind == ConstantValueKind.DEFERRED_GLOBAL);
outputUnitData.registerConstantDeferredUse(
use.value as DeferredGlobalConstantValue);
});
return impactBuilder;
});
return DumpInfoProgramData._(programSize, outputUnitSizes,
fragmentDeferredMap, entityCode, entityCodeSize, constantCode, impacts,
neededClasses: neededClasses, neededClassTypes: neededClassTypes);
final serializedImpactMembers = source.readMembers().toSet();
final registeredImpacts = source.readMemberMap<MemberEntity, CodegenImpact>(
(_) => CodegenImpact.readFromDataSource(source));
return DumpInfoProgramData._(
programSize,
outputUnitSizes,
fragmentDeferredMap,
entityCode,
entityCodeSize,
constantCode,
serializedImpactMembers,
registeredImpacts,
neededClasses: neededClasses,
neededClassTypes: neededClassTypes);
}
void writeToDataSink(DataSinkWriter sink) {
@ -312,19 +331,9 @@ class DumpInfoProgramData {
sink.writeConstant(constant);
_writeCodeSpan(sink, codeSpan);
});
sink.writeMemberMap(impacts, (_, WorldImpact impact) {
// Only store relevant uses from the impact. Static and dynamic uses are
// queried explicitly and DEFERRED_GLOBAL constants must be registered
// into the closed world.
sink.writeList(
impact.staticUses, (StaticUse use) => use.writeToDataSink(sink));
sink.writeList(
impact.dynamicUses, (DynamicUse use) => use.writeToDataSink(sink));
sink.writeList(
impact.constantUses.where(
(use) => use.value.kind == ConstantValueKind.DEFERRED_GLOBAL),
(ConstantUse use) => use.writeToDataSink(sink));
});
sink.writeMembers(serializedImpactMembers);
sink.writeMemberMap(registeredImpacts,
(_, CodegenImpact impact) => impact.writeToDataSink(sink));
}
}
@ -360,7 +369,7 @@ class ElementInfoCollector {
/// output size. Either because it is a function being emitted or inlined,
/// or because it is an entity that holds dependencies to other entities.
bool shouldKeep(Entity entity) {
return dumpInfoTask._dumpInfoData.impacts.containsKey(entity) ||
return dumpInfoTask.impacts.containsKey(entity) ||
dumpInfoTask.inlineCount.containsKey(entity);
}
@ -1063,7 +1072,7 @@ class DumpInfoAnnotator {
/// output size. Either because it is a function being emitted or inlined,
/// or because it is an entity that holds dependencies to other entities.
bool shouldKeep(Entity entity) {
return dumpInfoTask._dumpInfoData.impacts.containsKey(entity) ||
return dumpInfoTask.impacts.containsKey(entity) ||
dumpInfoTask.inlineCount.containsKey(entity);
}
@ -1439,6 +1448,10 @@ class DumpInfoTask extends CompilerTask implements InfoReporter {
@override
String get name => "Dump Info";
/// Whether or not this dump info task is running using serialized dump info
/// data from an earlier dart2js invocation.
bool get useSerializedData => options.dumpInfoReadUri != null;
/// The size of the generated output.
late DumpInfoProgramData _dumpInfoData;
@ -1448,6 +1461,8 @@ class DumpInfoTask extends CompilerTask implements InfoReporter {
// inlined inside of it.
final Map<Entity, List<Entity>> inlineMap = <Entity, List<Entity>>{};
final Map<MemberEntity, WorldImpact> impacts = {};
/// Register the size of the generated output.
void registerDumpInfoProgramData(DumpInfoProgramData dumpInfoData) {
_dumpInfoData = dumpInfoData;
@ -1461,7 +1476,7 @@ class DumpInfoTask extends CompilerTask implements InfoReporter {
}
void unregisterImpact(MemberEntity impactSource) {
_dumpInfoData.impacts.remove(impactSource);
impacts.remove(impactSource);
}
/// Returns an iterable of [Selection]s that are used by [entity]. Each
@ -1469,7 +1484,7 @@ class DumpInfoTask extends CompilerTask implements InfoReporter {
/// selected the entity.
Iterable<Selection> getRetaining(
MemberEntity entity, JClosedWorld closedWorld) {
final impact = _dumpInfoData.impacts[entity];
final impact = impacts[entity];
if (impact == null) return const <Selection>[];
var selections = <Selection>[];
@ -1498,10 +1513,29 @@ class DumpInfoTask extends CompilerTask implements InfoReporter {
return _dumpInfoData.entityCode[entity] ?? const [];
}
Future<DumpInfoStateData> dumpInfo(JClosedWorld closedWorld,
GlobalTypeInferenceResults globalInferenceResults) async {
void _populateImpacts(JClosedWorld closedWorld, CodegenResults codegenResults,
JsBackendStrategy backendStrategy) {
backendStrategy.initialize(closedWorld, codegenResults.codegenInputs);
_dumpInfoData.registeredImpacts.forEach((member, impact) {
impacts[member] = backendStrategy.transformCodegenImpact(impact);
});
for (final member in _dumpInfoData.serializedImpactMembers) {
final (:result, :isGenerated) = codegenResults.getCodegenResults(member);
assert(!isGenerated, 'Should not be generating impact: $member');
impacts[member] = backendStrategy.transformCodegenImpact(result.impact);
}
}
Future<DumpInfoStateData> dumpInfo(
JClosedWorld closedWorld,
GlobalTypeInferenceResults globalInferenceResults,
CodegenResults codegenResults,
JsBackendStrategy backendStrategy) async {
late DumpInfoStateData dumpInfoState;
await measure(() async {
_populateImpacts(closedWorld, codegenResults, backendStrategy);
ElementInfoCollector elementInfoCollector = ElementInfoCollector(
options, this, closedWorld, globalInferenceResults)
..run();
@ -1521,9 +1555,13 @@ class DumpInfoTask extends CompilerTask implements InfoReporter {
Future<DumpInfoStateData> dumpInfoNew(
ir.Component component,
JClosedWorld closedWorld,
GlobalTypeInferenceResults globalInferenceResults) async {
GlobalTypeInferenceResults globalInferenceResults,
CodegenResults codegenResults,
JsBackendStrategy backendStrategy) async {
late DumpInfoStateData dumpInfoState;
await measure(() async {
_populateImpacts(closedWorld, codegenResults, backendStrategy);
KernelInfoCollector kernelInfoCollector =
KernelInfoCollector(component, options, this, closedWorld)..run();

8
pkg/compiler/lib/src/js_backend/impact_transformer.dart
View file

@ -36,7 +36,7 @@ class CodegenImpactTransformer {
final RuntimeTypesNeed _rtiNeed;
final NativeCodegenEnqueuer _nativeCodegenEnqueuer;
final Namer _namer;
final OneShotInterceptorData _oneShotInterceptorData;
final OneShotInterceptorData oneShotInterceptorData;
final RuntimeTypesChecksBuilder _rtiChecksBuilder;
final NativeEmitter _nativeEmitter;
@ -49,7 +49,7 @@ class CodegenImpactTransformer {
this._rtiNeed,
this._nativeCodegenEnqueuer,
this._namer,
this._oneShotInterceptorData,
this.oneShotInterceptorData,
this._rtiChecksBuilder,
this._nativeEmitter);
@ -162,7 +162,7 @@ class CodegenImpactTransformer {
}
for (Set<ClassEntity> classes in impact.specializedGetInterceptors) {
_oneShotInterceptorData.registerSpecializedGetInterceptor(classes);
oneShotInterceptorData.registerSpecializedGetInterceptor(classes);
}
if (impact.usesInterceptor) {
@ -208,7 +208,7 @@ class CodegenImpactTransformer {
}
for (Selector selector in impact.oneShotInterceptors) {
_oneShotInterceptorData.registerOneShotInterceptor(
oneShotInterceptorData.registerOneShotInterceptor(
selector, _namer, _closedWorld);
}

35
pkg/compiler/lib/src/js_model/js_strategy.dart
View file

@ -79,6 +79,8 @@ class JsBackendStrategy {
final Compiler _compiler;
late JsKernelToElementMap _elementMap;
bool _isInitialized = false;
/// Codegen support for generating table of interceptors and
/// constructors for custom elements.
late final CustomElementsCodegenAnalysis _customElementsCodegenAnalysis;
@ -240,11 +242,7 @@ class JsBackendStrategy {
CodegenInputs codegen,
CodegenResults codegenResults,
SourceLookup sourceLookup) {
OneShotInterceptorData oneShotInterceptorData = OneShotInterceptorData(
closedWorld.interceptorData,
closedWorld.commonElements,
closedWorld.nativeData);
_onCodegenEnqueuerStart(closedWorld, codegen, oneShotInterceptorData);
initialize(closedWorld, codegen);
ElementEnvironment elementEnvironment = closedWorld.elementEnvironment;
CommonElements commonElements = closedWorld.commonElements;
BackendImpacts impacts = BackendImpacts(commonElements, _compiler.options);
@ -255,7 +253,7 @@ class JsBackendStrategy {
closedWorld,
inferredData,
_compiler.abstractValueStrategy.createSelectorStrategy(),
oneShotInterceptorData);
_codegenImpactTransformer.oneShotInterceptorData);
return CodegenEnqueuer(
task,
worldBuilder,
@ -285,8 +283,17 @@ class JsBackendStrategy {
}
/// Called before the compiler starts running the codegen enqueuer.
void _onCodegenEnqueuerStart(JClosedWorld closedWorld, CodegenInputs codegen,
OneShotInterceptorData oneShotInterceptorData) {
void initialize(JClosedWorld closedWorld, CodegenInputs codegen) {
// This can be initialized during the emitter phase and when running dump
// info. Make sure if both are running together that this is only
// initialized once.
if (_isInitialized) return;
_isInitialized = true;
OneShotInterceptorData oneShotInterceptorData = OneShotInterceptorData(
closedWorld.interceptorData,
closedWorld.commonElements,
closedWorld.nativeData);
FixedNames fixedNames = codegen.fixedNames;
_namer = _compiler.options.enableMinification
? _compiler.options.useFrequencyNamer
@ -321,6 +328,10 @@ class JsBackendStrategy {
emitterTask.nativeEmitter);
}
WorldImpact transformCodegenImpact(CodegenImpact impact) {
return _codegenImpactTransformer.transformCodegenImpact(impact);
}
WorldImpact generateCode(
WorkItem work,
AbstractValueDomain abstractValueDomain,
@ -328,7 +339,7 @@ class JsBackendStrategy {
ComponentLookup componentLookup,
SourceLookup sourceLookup) {
MemberEntity member = work.element;
CodegenResult result = codegenResults.getCodegenResults(member);
var (:result, :isGenerated) = codegenResults.getCodegenResults(member);
if (_compiler.options.testMode) {
final indices = SerializationIndices(testMode: true);
bool useDataKinds = true;
@ -354,9 +365,13 @@ class JsBackendStrategy {
codegenImpactsForTesting ??= {};
codegenImpactsForTesting![member] = result.impact;
}
// Register the untransformed impact here as dump info will transform it
// again later if needed.
_compiler.dumpInfoRegistry
.registerImpact(member, result.impact, isGenerated: isGenerated);
WorldImpact worldImpact =
_codegenImpactTransformer.transformCodegenImpact(result.impact);
_compiler.dumpInfoRegistry.registerImpact(member, worldImpact);
result.applyModularState(_namer, emitterTask.emitter);
return worldImpact;
}

5
pkg/compiler/lib/src/serialization/task.dart
View file

@ -198,7 +198,8 @@ class SerializationTask extends CompilerTask {
final lazyMemberBodies =
backendStrategy.forEachCodegenMember((MemberEntity member) {
if (index % shards == shard) {
CodegenResult codegenResult = codegenResults.getCodegenResults(member);
final (result: codegenResult, isGenerated: _) =
codegenResults.getCodegenResults(member);
results[member] = codegenResult;
}
index++;
@ -304,7 +305,7 @@ class SerializationTask extends CompilerTask {
indices);
backendStrategy.prepareCodegenReader(source);
source.registerAbstractValueDomain(abstractValueDomain);
return DumpInfoProgramData.readFromDataSource(source, outputUnitData,
return DumpInfoProgramData.readFromDataSource(source,
includeCodeText: !_options.useDumpInfoBinaryFormat);
}
}

5
pkg/compiler/test/serialization/serialization_test_helper.dart
View file

@ -42,7 +42,10 @@ Future<void> generateJavaScriptCode(Compiler compiler,
codegenResults,
globalTypeInferenceResults,
DumpInfoProgramData.fromEmitterResults(
compiler.backendStrategy, compiler.dumpInfoRegistry, programSize));
compiler.backendStrategy.emitterTask,
compiler.dumpInfoRegistry,
codegenResults,
programSize));
}
}