languages/dart-sdk
languages/dart-sdk
1
0
Fork 0
mirror of https://github.com/dart-lang/sdk synced 2024-09-05 00:13:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

[CFE] Refactor macros setup.

Browse source 
Add `dispose` method to `ProcessedOptions` that does macro executor cleanup; add `dispose` method to MultiMacroExecutor for it to call.

R=asiva@google.com, jakemac@google.com

Change-Id: I90d9af133e84655ec14870400988fdee9adb219e
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/338440
Reviewed-by: Siva Annamalai <asiva@google.com>
Reviewed-by: Johnni Winther <johnniwinther@google.com>
Commit-Queue: Morgan :) <davidmorgan@google.com>
Reviewed-by: Jake Macdonald <jakemac@google.com>
This commit is contained in:
David Morgan 2023-11-30 08:08:11 +00:00 committed by Commit Queue
parent 32e46f7189
commit 26107a319a
11 changed files with 148 additions and 159 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
pkg/_fe_analyzer_shared/lib/src/macros/executor/multi_executor.dart
View file

@ -5,8 +5,8 @@
import 'dart:async';
import '../api.dart';
import '../executor/augmentation_library.dart';
import '../executor.dart';
import '../executor/augmentation_library.dart';
/// A [MacroExecutor] implementation which delegates most work to other
/// executors which are spawned through a provided callback.
@ -83,6 +83,13 @@ class MultiMacroExecutor extends MacroExecutor with AugmentationLibraryBuilder {
}
}
/// Shuts down all executors and clears all configuration.
Future<void> closeAndReset() async {
await Future.wait(_executorFactoryTokens
.toList()
.map((token) => unregisterExecutorFactory(token)));
}
/// Shuts down all executors, but does not clear [_libraryExecutorFactories]
/// or [_executorFactoryTokens].
@override

21
pkg/front_end/lib/src/api_prototype/compiler_options.dart
View file

@ -5,19 +5,18 @@
library front_end.compiler_options;
import 'package:_fe_analyzer_shared/src/macros/executor/multi_executor.dart';
import 'package:_fe_analyzer_shared/src/macros/executor/serialization.dart'
as macros show SerializationMode;
import 'package:_fe_analyzer_shared/src/messages/diagnostic_message.dart'
show DiagnosticMessage, DiagnosticMessageHandler;
import 'package:_fe_analyzer_shared/src/messages/severity.dart' show Severity;
import 'package:kernel/ast.dart' show Component, Version;
import 'package:kernel/default_language_version.dart' as kernel
show defaultLanguageVersion;
import 'package:kernel/target/targets.dart' show Target;
import '../api_unstable/util.dart';
import '../base/nnbd_mode.dart';
import '../macro_serializer.dart';
import 'experimental_flags.dart'
show
@ -25,18 +24,13 @@ import 'experimental_flags.dart'
ExperimentalFlag,
GlobalFeatures,
parseExperimentalFlag;
import 'experimental_flags.dart' as flags
show
getExperimentEnabledVersionInLibrary,
isExperimentEnabledInLibraryByVersion;
import 'file_system.dart' show FileSystem;
import 'standard_file_system.dart' show StandardFileSystem;
import '../api_unstable/util.dart';
export 'package:_fe_analyzer_shared/src/messages/diagnostic_message.dart'
show DiagnosticMessage;
@ -127,6 +121,15 @@ class CompilerOptions {
/// This is part of the experimental macro feature.
MacroSerializer? macroSerializer;
/// Raw precompiled macro options, each of the format
/// `<program-uri>;<macro-library-uri>`.
///
/// Multiple library URIs may be provided separated by additional semicolons.
List<String>? precompiledMacros;
/// The serialization mode to use for macro communication.
macros.SerializationMode? macroSerializationMode;
/// Whether to generate code for the SDK.
///
/// By default the front end resolves components using a prebuilt SDK summary.

62
pkg/front_end/lib/src/api_unstable/bazel_worker.dart
View file

@ -5,52 +5,37 @@
/// API needed by `utils/front_end/summary_worker.dart`, a tool used to compute
/// summaries in build systems like bazel, pub-build, and package-build.
import 'package:_fe_analyzer_shared/src/macros/executor/serialization.dart'
show SerializationMode;
import 'package:_fe_analyzer_shared/src/messages/diagnostic_message.dart'
show DiagnosticMessageHandler;
import 'package:front_end/src/api_prototype/compiler_options.dart';
import 'package:kernel/kernel.dart' show Component, Library, dummyComponent;
import 'package:kernel/target/targets.dart' show Target;
import '../api_prototype/experimental_flags.dart' show ExperimentalFlag;
import '../api_prototype/file_system.dart' show FileSystem;
import '../api_prototype/front_end.dart' show CompilerResult;
import '../base/nnbd_mode.dart' show NnbdMode;
import '../base/processed_options.dart' show ProcessedOptions;
import '../kernel_generator_impl.dart' show generateKernel;
import 'compiler_state.dart' show InitializedCompilerState;
import 'modular_incremental_compilation.dart' as modular
show initializeIncrementalCompiler;
export 'package:_fe_analyzer_shared/src/messages/diagnostic_message.dart'
show DiagnosticMessage;
export 'package:_fe_analyzer_shared/src/messages/severity.dart' show Severity;
export '../api_prototype/compiler_options.dart'
show parseExperimentalFlags, parseExperimentalArguments, Verbosity;
export '../api_prototype/experimental_flags.dart'
show ExperimentalFlag, parseExperimentalFlag;
export '../api_prototype/standard_file_system.dart' show StandardFileSystem;
export '../api_prototype/terminal_color_support.dart'
show printDiagnosticMessage;
export '../base/nnbd_mode.dart' show NnbdMode;
export '../fasta/kernel/utils.dart' show serializeComponent;
export 'compiler_state.dart' show InitializedCompilerState;
/// Initializes the compiler for a modular build.
@ -58,21 +43,24 @@ export 'compiler_state.dart' show InitializedCompilerState;
/// Re-uses cached components from [oldState.workerInputCache], and reloads them
/// as necessary based on [workerInputDigests].
Future<InitializedCompilerState> initializeIncrementalCompiler(
InitializedCompilerState? oldState,
Set<String> tags,
Uri? sdkSummary,
Uri? packagesFile,
Uri? librariesSpecificationUri,
List<Uri> additionalDills,
Map<Uri, List<int>> workerInputDigests,
Target target,
FileSystem fileSystem,
Iterable<String>? experiments,
bool outlineOnly,
Map<String, String> environmentDefines,
{bool trackNeededDillLibraries = false,
bool verbose = false,
NnbdMode nnbdMode = NnbdMode.Weak}) {
InitializedCompilerState? oldState,
Set<String> tags,
Uri? sdkSummary,
Uri? packagesFile,
Uri? librariesSpecificationUri,
List<Uri> additionalDills,
Map<Uri, List<int>> workerInputDigests,
Target target,
FileSystem fileSystem,
Iterable<String>? experiments,
bool outlineOnly,
Map<String, String> environmentDefines, {
bool trackNeededDillLibraries = false,
bool verbose = false,
NnbdMode nnbdMode = NnbdMode.Weak,
List<String> precompiledMacros = const [],
SerializationMode macroSerializationMode = SerializationMode.byteData,
}) {
List<Component> outputLoadedAdditionalDills =
new List<Component>.filled(additionalDills.length, dummyComponent);
Map<ExperimentalFlag, bool> experimentalFlags = parseExperimentalFlags(
@ -95,7 +83,9 @@ Future<InitializedCompilerState> initializeIncrementalCompiler(
trackNeededDillLibraries: trackNeededDillLibraries,
environmentDefines: environmentDefines,
verbose: verbose,
nnbdMode: nnbdMode);
nnbdMode: nnbdMode,
precompiledMacros: precompiledMacros,
macroSerializationMode: macroSerializationMode);
}
InitializedCompilerState initializeCompiler(
@ -110,6 +100,8 @@ InitializedCompilerState initializeCompiler(
Map<String, String>? environmentDefines, {
bool verbose = false,
NnbdMode nnbdMode = NnbdMode.Weak,
List<String> precompiledMacros = const [],
SerializationMode macroSerializationMode = SerializationMode.byteData,
}) {
// TODO(sigmund): use incremental compiler when it supports our use case.
// Note: it is common for the summary worker to invoke the compiler with the
@ -128,7 +120,9 @@ InitializedCompilerState initializeCompiler(
parseExperimentalArguments(experiments),
onError: (e) => throw e)
..verbose = verbose
..nnbdMode = nnbdMode;
..nnbdMode = nnbdMode
..precompiledMacros = precompiledMacros
..macroSerializationMode = macroSerializationMode;
ProcessedOptions processedOpts = new ProcessedOptions(options: options);

28
pkg/front_end/lib/src/api_unstable/modular_incremental_compilation.dart
View file

@ -2,28 +2,21 @@
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
import 'package:_fe_analyzer_shared/src/macros/executor/serialization.dart'
show SerializationMode;
import 'package:kernel/kernel.dart' show Component, CanonicalName, Library;
import 'package:kernel/target/targets.dart' show Target;
import '../api_prototype/compiler_options.dart' show CompilerOptions;
import '../api_prototype/experimental_flags.dart' show ExperimentalFlag;
import '../api_prototype/file_system.dart' show FileSystem;
import '../base/nnbd_mode.dart' show NnbdMode;
import '../base/processed_options.dart' show ProcessedOptions;
import '../fasta/compiler_context.dart' show CompilerContext;
import '../fasta/incremental_compiler.dart' show IncrementalCompiler;
import 'compiler_state.dart'
show InitializedCompilerState, WorkerInputComponent, digestsEqual;
import 'util.dart' show equalMaps, equalSets;
import 'util.dart' show equalLists, equalMaps, equalSets;
/// Initializes the compiler for a modular build.
///
@ -55,7 +48,10 @@ Future<InitializedCompilerState> initializeIncrementalCompiler(
bool omitPlatform = false,
bool trackNeededDillLibraries = false,
bool verbose = false,
NnbdMode nnbdMode = NnbdMode.Weak}) async {
NnbdMode nnbdMode = NnbdMode.Weak,
List<String> precompiledMacros = const [],
SerializationMode macroSerializationMode =
SerializationMode.byteData}) async {
bool isRetry = false;
while (true) {
try {
@ -88,13 +84,17 @@ Future<InitializedCompilerState> initializeIncrementalCompiler(
!equalSets(oldState.tags, tags) ||
(sdkSummary != null &&
(cachedSdkInput == null ||
!digestsEqual(cachedSdkInput.digest, sdkDigest)))) {
!digestsEqual(cachedSdkInput.digest, sdkDigest))) ||
// TODO(davidmorgan): add correct change detection for macros.
!equalLists(oldState.options.precompiledMacros, precompiledMacros) ||
oldState.options.macroSerializationMode != macroSerializationMode) {
// No - or immediately not correct - previous state.
// We'll load a new sdk, anything loaded already will have a wrong root.
workerInputCache.clear();
workerInputCacheLibs.clear();
// The sdk was either not cached or it has changed.
await oldState?.processedOpts.dispose();
options = new CompilerOptions()
..compileSdk = compileSdk
..sdkRoot = sdkRoot
@ -107,7 +107,9 @@ Future<InitializedCompilerState> initializeIncrementalCompiler(
..environmentDefines = environmentDefines
..explicitExperimentalFlags = explicitExperimentalFlags
..verbose = verbose
..nnbdMode = nnbdMode;
..nnbdMode = nnbdMode
..precompiledMacros = precompiledMacros
..macroSerializationMode = macroSerializationMode;
processedOpts = new ProcessedOptions(options: options);
if (sdkSummary != null && sdkDigest != null) {

88
pkg/front_end/lib/src/base/processed_options.dart
View file

@ -3,21 +3,23 @@
// BSD-style license that can be found in the LICENSE file.
import 'dart:io' show exitCode;
import 'dart:typed_data' show Uint8List;
import 'package:_fe_analyzer_shared/src/messages/severity.dart' show Severity;
import 'package:_fe_analyzer_shared/src/macros/executor/isolated_executor.dart'
as isolated_executor;
import 'package:_fe_analyzer_shared/src/macros/executor/multi_executor.dart';
import 'package:_fe_analyzer_shared/src/macros/executor/process_executor.dart'
as process_executor;
import 'package:_fe_analyzer_shared/src/macros/executor/serialization.dart'
show SerializationMode;
import 'package:_fe_analyzer_shared/src/messages/severity.dart' show Severity;
import 'package:_fe_analyzer_shared/src/util/libraries_specification.dart'
show
LibrariesSpecification,
LibrariesSpecificationException,
TargetLibrariesSpecification;
import 'package:front_end/src/fasta/kernel/macro/macro.dart';
import 'package:kernel/binary/ast_from_binary.dart' show BinaryBuilder;
import 'package:kernel/kernel.dart'
show
CanonicalName,
@ -25,27 +27,19 @@ import 'package:kernel/kernel.dart'
Location,
NonNullableByDefaultCompiledMode,
Version;
import 'package:kernel/target/targets.dart'
show NoneTarget, Target, TargetFlags;
import 'package:package_config/package_config.dart';
import '../api_prototype/compiler_options.dart'
show CompilerOptions, InvocationMode, Verbosity, DiagnosticMessage;
import '../api_prototype/experimental_flags.dart' as flags;
import '../api_prototype/file_system.dart'
show FileSystem, FileSystemEntity, FileSystemException;
import '../api_prototype/terminal_color_support.dart'
show printDiagnosticMessage;
import '../fasta/command_line_reporting.dart' as command_line_reporting;
import '../fasta/compiler_context.dart' show CompilerContext;
import '../fasta/fasta_codes.dart'
show
FormattedMessage,
@ -68,15 +62,10 @@ import '../fasta/fasta_codes.dart'
templateSdkRootNotFound,
templateSdkSpecificationNotFound,
templateSdkSummaryNotFound;
import '../fasta/messages.dart' show getLocation;
import '../fasta/problems.dart' show DebugAbort, unimplemented;
import '../fasta/ticker.dart' show Ticker;
import '../fasta/uri_translator.dart' show UriTranslator;
import 'nnbd_mode.dart';
/// All options needed for the front end implementation.
@ -213,7 +202,15 @@ class ProcessedOptions {
this.environmentDefines = options?.environmentDefines,
// TODO(sigmund, ahe): create ticker even earlier or pass in a stopwatch
// collecting time since the start of the VM.
this.ticker = new Ticker(isVerbose: options?.verbose ?? false);
this.ticker = new Ticker(isVerbose: options?.verbose ?? false) {
if (globalFeatures.macros.isEnabled) {
enableMacros = true;
forceEnableMacros = true;
} else {
enableMacros = false;
forceEnableMacros = false;
}
}
FormattedMessage format(
LocatedMessage message, Severity severity, List<LocatedMessage>? context,
@ -803,10 +800,57 @@ class ProcessedOptions {
}
}
MultiMacroExecutor get macroExecutor =>
_raw.macroExecutor ??= new MultiMacroExecutor();
MultiMacroExecutor get macroExecutor {
if (_raw.macroExecutor != null) return _raw.macroExecutor!;
MultiMacroExecutor executor = _raw.macroExecutor = new MultiMacroExecutor();
// Either set up or reset the state for macros based on experiment status.
List<ExecutorFactoryToken> registeredMacroExecutors =
<ExecutorFactoryToken>[];
if (globalFeatures.macros.isEnabled && _raw.precompiledMacros != null) {
// TODO: Handle multiple macro libraries compiled to a single precompiled
// kernel file.
for (List<String> parts
in _raw.precompiledMacros!.map((arg) => arg.split(';'))) {
Set<Uri> libraries = parts
.skip(1)
.map(Uri.parse)
.where((library) => !executor.libraryIsRegistered(library))
.toSet();
if (libraries.isEmpty) {
continue;
}
Uri programUri = Uri.base.resolve(parts[0]);
if (parts[0].endsWith('.dill')) {
// It's kernel, run as an isolate.
registeredMacroExecutors.add(executor.registerExecutorFactory(
() => isolated_executor.start(macroSerializationMode, programUri),
libraries));
} else {
// Otherwise, run as an executable.
registeredMacroExecutors.add(executor.registerExecutorFactory(
() => process_executor.start(
macroSerializationMode,
process_executor.CommunicationChannel.socket,
programUri.toFilePath()),
libraries));
break;
}
}
}
return executor;
}
SerializationMode get macroSerializationMode =>
_raw.macroSerializationMode ??= SerializationMode.byteData;
CompilerOptions get rawOptionsForTesting => _raw;
/// Disposes macro executor if one is configured.
Future<void> dispose() async {
await _raw.macroExecutor?.closeAndReset();
}
}
/// A [FileSystem] that only allows access to files that have been explicitly

1
pkg/front_end/test/spell_checking_list_code.txt
View file

@ -373,6 +373,7 @@ dartfix
dartino
dartlang
dashes
davidmorgan
dc
ddc
ddfea

1
pkg/front_end/test/spell_checking_list_common.txt
View file

@ -2736,6 +2736,7 @@ semantic
semantics
semi
semicolon
semicolons
send
sending
sends

1
pkg/front_end/testcases/strong.status
View file

@ -54,6 +54,7 @@ inference_update_2/external_field: semiFuzzFailureOnForceRebuildBodies # privacy
inference_update_2/field_promotion_and_no_such_method: semiFuzzFailureOnForceRebuildBodies # Private fields.
inference_update_2/field_promotion_name_conflicts: semiFuzzFailureOnForceRebuildBodies # Private fields.
inference_update_2/issue52452: semiFuzzFailureOnForceRebuildBodies # privacy.
macros/macro_class: SemiFuzzFailure # Declares macros.
patterns/exhaustiveness/issue51957: semiFuzzFailureOnForceRebuildBodies # can't split because of sealed class.
patterns/exhaustiveness/issue52041: semiFuzzFailureOnForceRebuildBodies # can't match on private field in split out library.
patterns/exhaustiveness/issue52048: semiFuzzFailureOnForceRebuildBodies # can't split because of sealed class.

1
pkg/front_end/testcases/weak.status
View file

@ -41,6 +41,7 @@ general/redirection/main: semiFuzzFailureOnForceRebuildBodies # privacy.
inference_update_2/abstract_field: semiFuzzFailureOnForceRebuildBodies # privacy.
inference_update_2/external_field: semiFuzzFailureOnForceRebuildBodies # privacy.
inference_update_2/issue52452: semiFuzzFailureOnForceRebuildBodies # privacy.
macros/macro_class: SemiFuzzFailure # Declares macros.
patterns/exhaustiveness/issue51957: semiFuzzFailureOnForceRebuildBodies # can't split because of sealed class.
patterns/exhaustiveness/issue52041: semiFuzzFailureOnForceRebuildBodies # can't match on private field in split out library.
nnbd_mixed/exhaustiveness: semiFuzzFailureOnForceRebuildBodies # can't split because of sealed class.

93
pkg/frontend_server/lib/compute_kernel.dart
View file

@ -12,12 +12,6 @@ library;
import 'dart:async';
import 'dart:io';
import 'package:_fe_analyzer_shared/src/macros/executor/isolated_executor.dart'
as isolated_executor;
import 'package:_fe_analyzer_shared/src/macros/executor/multi_executor.dart'
as multi_executor;
import 'package:_fe_analyzer_shared/src/macros/executor/process_executor.dart'
as process_executor;
import 'package:_fe_analyzer_shared/src/macros/executor/serialization.dart'
show SerializationMode;
import 'package:args/args.dart';
@ -28,7 +22,6 @@ import 'package:dev_compiler/src/kernel/target.dart';
import 'package:front_end/src/api_prototype/file_system.dart';
import 'package:front_end/src/api_prototype/incremental_kernel_generator.dart';
import 'package:front_end/src/api_unstable/bazel_worker.dart' as fe;
import 'package:front_end/src/fasta/kernel/macro/macro.dart';
import 'package:kernel/ast.dart'
show Component, Library, NonNullableByDefaultCompiledMode;
import 'package:kernel/target/targets.dart';
@ -321,6 +314,13 @@ Future<ComputeKernelResult> computeKernel(List<String> args,
}
}
SerializationMode macroSerializationMode =
switch (parsedArgs['macro-serialization-mode']) {
'json' => SerializationMode.json,
'bytedata' => SerializationMode.byteData,
_ => throw ArgumentError('Unrecognized macro serialization mode '
'${parsedArgs['macro-serialization-mode']}'),
};
if (usingIncrementalCompiler) {
// If digests weren't given and if not in worker mode, create fake data and
// ensure we don't have a previous state (as that wouldn't be safe with
@ -358,7 +358,9 @@ Future<ComputeKernelResult> computeKernel(List<String> args,
nullableEnvironmentDefines!,
trackNeededDillLibraries: recordUsedInputs,
verbose: verbose,
nnbdMode: nnbdMode);
nnbdMode: nnbdMode,
precompiledMacros: parsedArgs['precompiled-macro'],
macroSerializationMode: macroSerializationMode);
} else {
state = fe.initializeCompiler(
// TODO(sigmund): pass an old state once we can make use of it.
@ -372,69 +374,9 @@ Future<ComputeKernelResult> computeKernel(List<String> args,
parsedArgs['enable-experiment'] as List<String>,
nullableEnvironmentDefines,
verbose: verbose,
nnbdMode: nnbdMode);
}
// Either set up or reset the state for macros based on experiment status.
// TODO(jakemac,johnniwinther): Make this a part of `initializeCompiler`,
// if/when we want to make it more widely supported.
var registeredMacroExecutors = <multi_executor.ExecutorFactoryToken>[];
if (state.processedOpts.globalFeatures.macros.isEnabled) {
enableMacros = true;
forceEnableMacros = true;
SerializationMode serializationMode;
switch (parsedArgs['macro-serialization-mode']) {
case 'json':
serializationMode = SerializationMode.json;
break;
case 'bytedata':
serializationMode = SerializationMode.byteData;
break;
default:
throw ArgumentError('Unrecognized macro serialization mode '
'${parsedArgs['macro-serialization-mode']}');
}
// TODO: Handle invalidation of precompiled macros.
// TODO: Handle multiple macro libraries compiled to a single precompiled
// kernel file.
var macroExecutor = state.processedOpts.macroExecutor;
var format = parsedArgs['precompiled-macro-format'];
for (var parts in (parsedArgs['precompiled-macro'] as List<String>)
.map((arg) => arg.split(';'))) {
var libraries = parts
.skip(1)
.map(Uri.parse)
.where((library) => !macroExecutor.libraryIsRegistered(library))
.toSet();
if (libraries.isEmpty) {
continue;
}
var programUri = toUri(parts[0]);
switch (format) {
case 'kernel':
registeredMacroExecutors.add(macroExecutor.registerExecutorFactory(
() => isolated_executor.start(serializationMode, programUri),
libraries));
break;
case 'aot':
registeredMacroExecutors.add(macroExecutor.registerExecutorFactory(
() => process_executor.start(
serializationMode,
process_executor.CommunicationChannel.socket,
programUri.toFilePath()),
libraries));
break;
default:
throw ArgumentError('Unrecognized precompiled macro format $format');
}
}
} else {
enableMacros = false;
forceEnableMacros = false;
await state.options.macroExecutor?.close();
state.options.macroExecutor = null;
nnbdMode: nnbdMode,
precompiledMacros: parsedArgs['precompiled-macro'],
macroSerializationMode: macroSerializationMode);
}
void onDiagnostic(fe.DiagnosticMessage message) {
@ -532,14 +474,7 @@ Future<ComputeKernelResult> computeKernel(List<String> args,
}
state.options.onDiagnostic = null; // See http://dartbug.com/36983.
// Unregister any macros executors so the processes can be shut down.
// TODO(jakemac,johnniwinther): A better cleanup mechanism? Should these be
// longer lived?
if (state.options.macroExecutor != null &&
registeredMacroExecutors.isNotEmpty) {
await Future.wait(registeredMacroExecutors.map((token) =>
state.options.macroExecutor!.unregisterExecutorFactory(token)));
}
await state.processedOpts.dispose();
if (!wroteUsedDills && recordUsedInputs) {
// The path taken didn't record inputs used: Say we used everything.

2
pkg/frontend_server/pubspec.yaml
View file

@ -4,7 +4,7 @@ description: A resident kernel compiler
publish_to: none
environment:
sdk: '>=2.19.0 <3.0.0'
sdk: '^3.0.0'
# Use 'any' constraints here; we get our versions from the DEPS file.
dependencies: