Parallel build-mode and DDC changes for summary2.

It does not enable summary2 yet, but makes it possible to flip
one flag AnalysisDriver.useSummary2 to switch between summary1
and summary2.

I run presubmit scripts, and it seems that it works for both.
https://test.corp.google.com/ui#id=OCL:255717824:BASE:259228828:1563746330268:5481d33a
https://test.corp.google.com/ui#id=OCL:255717824:BASE:258892260:1563515248201:2d5ce329

One thing that is not implemented yet is tracking used dependencies
for summary2. So, //dart/bazel_codegen_tests:dependencies_test_run fails.


Change-Id: Iec9f3852e9b91236ce98e87041031daa0cc71a88
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/107689
Reviewed-by: Paul Berry <paulberry@google.com>
Reviewed-by: Brian Wilkerson <brianwilkerson@google.com>
Reviewed-by: Vijay Menon <vsm@google.com>
Commit-Queue: Konstantin Shcheglov <scheglov@google.com>
This commit is contained in:
Konstantin Shcheglov 2019-07-23 15:07:11 +00:00 committed by commit-bot@chromium.org
parent ae5d4973ab
commit ccdddd2eee
8 changed files with 242 additions and 75 deletions

View file

@ -20,6 +20,8 @@ import 'package:analyzer/src/summary/package_bundle_reader.dart';
import 'package:analyzer/src/summary/resynthesize.dart';
import 'package:analyzer/src/summary/summarize_ast.dart';
import 'package:analyzer/src/summary/summarize_elements.dart';
import 'package:analyzer/src/summary/summary_sdk.dart';
import 'package:analyzer/src/summary2/informative_data.dart';
import 'package:analyzer/src/summary2/link.dart' as summary2;
import 'package:analyzer/src/summary2/linked_bundle_context.dart' as summary2;
import 'package:analyzer/src/summary2/linked_element_factory.dart' as summary2;
@ -36,6 +38,8 @@ class DevCompilerResynthesizerBuilder {
_SourceCrawler _fileCrawler;
final List<_UnitInformativeData> _informativeData = [];
final PackageBundleAssembler _assembler;
List<int> summaryBytes;
@ -83,17 +87,21 @@ class DevCompilerResynthesizerBuilder {
);
context = RestrictedAnalysisContext(synchronousSession, _sourceFactory);
resynthesizer = StoreBasedSummaryResynthesizer(
context,
null,
context.sourceFactory,
/*strongMode*/ true,
SummaryDataStore([])
..addStore(_summaryData)
..addBundle(null, bundle),
);
resynthesizer.finishCoreAsyncLibraries();
context.typeProvider = resynthesizer.typeProvider;
if (AnalysisDriver.useSummary2) {
_createElementFactory(bundle);
} else {
resynthesizer = StoreBasedSummaryResynthesizer(
context,
null,
context.sourceFactory,
/*strongMode*/ true,
SummaryDataStore([])
..addStore(_summaryData)
..addBundle(null, bundle),
);
resynthesizer.finishCoreAsyncLibraries();
context.typeProvider = resynthesizer.typeProvider;
}
}
void _buildPackageBundleBytes() {
@ -126,31 +134,62 @@ class DevCompilerResynthesizerBuilder {
var inputLibraries = <summary2.LinkInputLibrary>[];
var sourceToUnit = _fileCrawler.sourceToUnit;
for (var librarySource in sourceToUnit.keys) {
for (var librarySource in _fileCrawler.librarySources) {
var libraryUriStr = '${librarySource.uri}';
var unit = sourceToUnit[librarySource];
if (_explicitSources.contains(librarySource.uri)) {
var isPart = unit.directives.any((d) => d is PartOfDirective);
if (isPart) {
continue;
}
}
var inputUnits = <summary2.LinkInputUnit>[];
inputUnits.add(
summary2.LinkInputUnit(null, librarySource, false, unit),
);
_informativeData.add(
_UnitInformativeData(
libraryUriStr,
libraryUriStr,
createInformativeData(unit),
),
);
for (var directive in unit.directives) {
if (directive is PartDirective) {
var partUri = directive.uri.stringValue;
var partSource = _sourceFactory.resolveUri(librarySource, partUri);
if (partSource != null) {
var partUnit = sourceToUnit[partSource];
var partRelativeUriStr = directive.uri.stringValue;
var partSource = _sourceFactory.resolveUri(
librarySource,
partRelativeUriStr,
);
// Add empty synthetic units for unresolved `part` URIs.
if (partSource == null) {
inputUnits.add(
summary2.LinkInputUnit(partUri, partSource, false, partUnit),
summary2.LinkInputUnit(
partRelativeUriStr,
null,
true,
_fsState.unresolvedFile.parse(),
),
);
continue;
}
var partUnit = sourceToUnit[partSource];
inputUnits.add(
summary2.LinkInputUnit(
partRelativeUriStr,
partSource,
partSource == null,
partUnit,
),
);
var unitUriStr = '${partSource.uri}';
_informativeData.add(
_UnitInformativeData(
libraryUriStr,
unitUriStr,
createInformativeData(partUnit),
),
);
}
}
@ -179,6 +218,40 @@ class DevCompilerResynthesizerBuilder {
var linkResult = summary2.link(elementFactory, inputLibraries);
_assembler.setBundle2(linkResult.bundle);
}
void _createElementFactory(PackageBundle newBundle) {
elementFactory = summary2.LinkedElementFactory(
context,
null,
summary2.Reference.root(),
);
for (var bundle in _summaryData.bundles) {
elementFactory.addBundle(
summary2.LinkedBundleContext(elementFactory, bundle.bundle2),
);
}
elementFactory.addBundle(
summary2.LinkedBundleContext(elementFactory, newBundle.bundle2),
);
for (var unitData in _informativeData) {
elementFactory.setInformativeData(
unitData.libraryUriStr,
unitData.unitUriStr,
unitData.data,
);
}
var dartCore = elementFactory.libraryOfUri('dart:core');
var dartAsync = elementFactory.libraryOfUri('dart:async');
var typeProvider = SummaryTypeProvider()
..initializeCore(dartCore)
..initializeAsync(dartAsync);
context.typeProvider = typeProvider;
dartCore.createLoadLibraryFunction(typeProvider);
dartAsync.createLoadLibraryFunction(typeProvider);
}
}
class _SourceCrawler {
@ -198,6 +271,7 @@ class _SourceCrawler {
final Map<String, UnlinkedUnitBuilder> uriToUnlinkedUnit = {};
final Map<Source, CompilationUnit> sourceToUnit = {};
final List<String> libraryUris = [];
final List<Source> librarySources = [];
_SourceCrawler(
this._fsState,
@ -230,7 +304,7 @@ class _SourceCrawler {
var uriStr = uri.toString();
// Maybe an input package contains the source.
if (_summaryData.unlinkedMap[uriStr] != null) {
if (_summaryData.hasUnlinkedUnit(uriStr)) {
return;
}
@ -273,6 +347,15 @@ class _SourceCrawler {
// Remember library URIs, for linking and compiling.
if (!isPart) {
libraryUris.add(uriStr);
librarySources.add(source);
}
}
}
class _UnitInformativeData {
final String libraryUriStr;
final String unitUriStr;
final List<UnlinkedInformativeData> data;
_UnitInformativeData(this.libraryUriStr, this.unitUriStr, this.data);
}

View file

@ -244,7 +244,7 @@ class FileState {
*/
bool get isExternalLibrary {
return _fsState.externalSummaries != null &&
_fsState.externalSummaries.linkedMap.containsKey(uriStr);
_fsState.externalSummaries.hasLinkedLibrary(uriStr);
}
/**
@ -253,8 +253,8 @@ class FileState {
*/
bool get isPart {
if (_fsState.externalSummaries != null &&
_fsState.externalSummaries.unlinkedMap.containsKey(uriStr)) {
return !_fsState.externalSummaries.linkedMap.containsKey(uriStr);
_fsState.externalSummaries.hasUnlinkedUnit(uriStr)) {
return _fsState.externalSummaries.isPartUnit(uriStr);
}
if (_unlinked2 != null) {
return !_unlinked2.hasLibraryDirective && _unlinked2.hasPartOfDirective;

View file

@ -7,6 +7,7 @@ import 'dart:math' show min;
import 'package:analyzer/dart/analysis/session.dart';
import 'package:analyzer/file_system/file_system.dart';
import 'package:analyzer/src/dart/analysis/driver.dart';
import 'package:analyzer/src/generated/engine.dart';
import 'package:analyzer/src/generated/source.dart';
import 'package:analyzer/src/generated/source_io.dart';
@ -130,10 +131,17 @@ class InSummaryUriResolver extends UriResolver {
Source resolveAbsolute(Uri uri, [Uri actualUri]) {
actualUri ??= uri;
String uriString = uri.toString();
UnlinkedUnit unit = _dataStore.unlinkedMap[uriString];
if (unit != null) {
if (AnalysisDriver.useSummary2) {
String summaryPath = _dataStore.uriToSummaryPath[uriString];
return new InSummarySource(actualUri, summaryPath);
if (summaryPath != null) {
return new InSummarySource(actualUri, summaryPath);
}
} else {
UnlinkedUnit unit = _dataStore.unlinkedMap[uriString];
if (unit != null) {
String summaryPath = _dataStore.uriToSummaryPath[uriString];
return new InSummarySource(actualUri, summaryPath);
}
}
return null;
}
@ -193,10 +201,13 @@ class SummaryDataStore {
final Map<String, LinkedLibrary> linkedMap = <String, LinkedLibrary>{};
/**
* Map from the URI of a library to the summary path that contained it.
* Map from the URI of a unit to the summary path that contained it.
*/
final Map<String, String> uriToSummaryPath = <String, String>{};
final Set<String> _libraryUris = Set<String>();
final Set<String> _partUris = Set<String>();
/**
* List of summary paths.
*/
@ -224,6 +235,7 @@ class SummaryDataStore {
*/
void addBundle(String path, PackageBundle bundle) {
bundles.add(bundle);
for (int i = 0; i < bundle.unlinkedUnitUris.length; i++) {
String uri = bundle.unlinkedUnitUris[i];
if (_disallowOverlappingSummaries &&
@ -239,6 +251,20 @@ class SummaryDataStore {
String uri = bundle.linkedLibraryUris[i];
addLinkedLibrary(uri, bundle.linkedLibraries[i]);
}
if (bundle.bundle2 != null) {
for (var library in bundle.bundle2.libraries) {
var libraryUri = library.uriStr;
_libraryUris.add(libraryUri);
for (var unit in library.units) {
var unitUri = unit.uriStr;
uriToSummaryPath[unitUri] = path;
if (unitUri != libraryUri) {
_partUris.add(unitUri);
}
}
}
}
}
/**
@ -293,7 +319,11 @@ class SummaryDataStore {
* with the given absolute [uri].
*/
bool hasLinkedLibrary(String uri) {
return linkedMap.containsKey(uri);
if (AnalysisDriver.useSummary2) {
return _libraryUris.contains(uri);
} else {
return linkedMap.containsKey(uri);
}
}
/**
@ -301,7 +331,22 @@ class SummaryDataStore {
* with the given absolute [uri].
*/
bool hasUnlinkedUnit(String uri) {
return unlinkedMap.containsKey(uri);
if (AnalysisDriver.useSummary2) {
return uriToSummaryPath.containsKey(uri);
} else {
return unlinkedMap.containsKey(uri);
}
}
/**
* Return `true` if the unit with the [uri] is a part unit in the store.
*/
bool isPartUnit(String uri) {
if (AnalysisDriver.useSummary2) {
return _partUris.contains(uri);
} else {
return !linkedMap.containsKey(uri);
}
}
void _fillMaps(String path, ResourceProvider resourceProvider) {

View file

@ -186,6 +186,9 @@ class _PackageBundleMock implements PackageBundle {
@override
List<String> unlinkedUnitUris;
@override
LinkedNodeBundle bundle2;
@override
noSuchMethod(Invocation invocation) {
throw new StateError('Unexpected invocation of ${invocation.memberName}');

View file

@ -195,8 +195,6 @@ class BuildMode with HasContextMixin {
PackageBundleAssembler assembler;
final Map<String, UnlinkedUnit> uriToUnit = <String, UnlinkedUnit>{};
final bool buildSummary2 = false;
final bool consumeSummary2 = false;
final Map<String, ParsedUnitResult> inputParsedUnitResults = {};
summary2.LinkedElementFactory elementFactory;
@ -277,21 +275,21 @@ class BuildMode with HasContextMixin {
});
// Build and assemble linked libraries.
if (!options.buildSummaryOnlyUnlinked) {
// Prepare URIs of unlinked units that should be linked.
var unlinkedUris = new Set<String>();
for (var bundle in unlinkedBundles) {
unlinkedUris.addAll(bundle.unlinkedUnitUris);
}
for (var src in explicitSources) {
unlinkedUris.add('${src.uri}');
}
// Perform linking.
_computeLinkedLibraries(unlinkedUris);
}
if (buildSummary2) {
if (AnalysisDriver.useSummary2) {
_computeLinkedLibraries2();
} else {
if (!options.buildSummaryOnlyUnlinked) {
// Prepare URIs of unlinked units that should be linked.
var unlinkedUris = new Set<String>();
for (var bundle in unlinkedBundles) {
unlinkedUris.addAll(bundle.unlinkedUnitUris);
}
for (var src in explicitSources) {
unlinkedUris.add('${src.uri}');
}
// Perform linking.
_computeLinkedLibraries(unlinkedUris);
}
}
// Write the whole package bundle.
@ -339,9 +337,6 @@ class BuildMode with HasContextMixin {
* add them to the [assembler].
*/
void _computeLinkedLibraries(Set<String> libraryUris) {
// Ensure that summary1 linking is done with summary1 rules.
AnalysisDriver.useSummary2 = false;
logger.run('Link output summary', () {
void trackDependency(String absoluteUri) {
if (dependencyTracker != null) {
@ -378,7 +373,6 @@ class BuildMode with HasContextMixin {
* [inputParsedUnitResults] to produce linked libraries in [assembler].
*/
void _computeLinkedLibraries2() {
AnalysisDriver.useSummary2 = consumeSummary2;
logger.run('Link output summary2', () {
var inputLibraries = <summary2.LinkInputLibrary>[];
@ -520,9 +514,6 @@ class BuildMode with HasContextMixin {
analysisOptions =
createAnalysisOptionsForCommandLineOptions(options, rootPath);
// Ensure that FileState prepare summary2 information if necessary.
AnalysisDriver.useSummary2 = consumeSummary2;
AnalysisDriverScheduler scheduler = new AnalysisDriverScheduler(logger);
analysisDriver = new AnalysisDriver(
scheduler,
@ -538,7 +529,7 @@ class BuildMode with HasContextMixin {
declaredVariables = new DeclaredVariables.fromMap(options.definedVariables);
analysisDriver.declaredVariables = declaredVariables;
if (buildSummary2) {
if (AnalysisDriver.useSummary2) {
_createLinkedElementFactory();
}
@ -609,10 +600,13 @@ class BuildMode with HasContextMixin {
return;
}
var result = await analysisDriver.parseFile(source.fullName);
inputParsedUnitResults[result.path] = result;
UnlinkedUnitBuilder unlinkedUnit = serializeAstUnlinked(result.unit);
uriToUnit[absoluteUri] = unlinkedUnit;
assembler.addUnlinkedUnit(source, unlinkedUnit);
if (AnalysisDriver.useSummary2) {
inputParsedUnitResults[result.path] = result;
} else {
UnlinkedUnitBuilder unlinkedUnit = serializeAstUnlinked(result.unit);
uriToUnit[absoluteUri] = unlinkedUnit;
assembler.addUnlinkedUnit(source, unlinkedUnit);
}
}
/**
@ -620,8 +614,6 @@ class BuildMode with HasContextMixin {
* is sent to a new file at that path.
*/
Future<void> _printErrors({String outputPath}) async {
AnalysisDriver.useSummary2 = consumeSummary2;
await logger.runAsync('Compute and print analysis errors', () async {
StringBuffer buffer = new StringBuffer();
var severityProcessor = (AnalysisError error) =>

View file

@ -7,6 +7,7 @@ import 'dart:io';
import 'package:analyzer/file_system/physical_file_system.dart';
import 'package:analyzer/src/command_line/arguments.dart';
import 'package:analyzer/src/context/builder.dart';
import 'package:analyzer/src/dart/analysis/driver.dart';
import 'package:analyzer/src/dart/analysis/experiments.dart';
import 'package:analyzer/src/util/sdk.dart';
import 'package:analyzer_cli/src/ansi.dart' as ansi;
@ -269,6 +270,11 @@ class CommandLineOptions {
}
if (options.buildSummaryOnlyUnlinked) {
if (AnalysisDriver.useSummary2) {
printAndFail('The option --build-summary-only-unlinked can not be used '
'together with summary2.');
return null; // Only reachable in testing.
}
if (!options.buildSummaryOnly) {
printAndFail(
'The option --build-summary-only-unlinked can be used only '

View file

@ -20,6 +20,7 @@ import 'package:analyzer/src/generated/sdk.dart';
import 'package:analyzer/src/generated/source.dart';
import 'package:analyzer/src/summary/package_bundle_reader.dart';
import 'package:analyzer/src/summary/resynthesize.dart';
import 'package:analyzer/src/summary2/linked_element_factory.dart';
import 'package:meta/meta.dart';
import '../compiler/shared_command.dart' show sdkLibraryVariables;
@ -130,11 +131,13 @@ class CompilerAnalysisDriver {
_extensionTypes ??= ExtensionTypeSet(
resynthesizerBuilder.context.typeProvider,
resynthesizerBuilder.resynthesizer,
resynthesizerBuilder.elementFactory,
);
return LinkedAnalysisDriver(
analysisOptions,
resynthesizerBuilder.resynthesizer,
resynthesizerBuilder.elementFactory,
sourceFactory,
resynthesizerBuilder.libraryUris,
declaredVariables,
@ -172,6 +175,7 @@ class CompilerAnalysisDriver {
class LinkedAnalysisDriver {
final AnalysisOptions analysisOptions;
final SummaryResynthesizer resynthesizer;
final LinkedElementFactory elementFactory;
final SourceFactory sourceFactory;
final List<String> libraryUris;
final DeclaredVariables declaredVariables;
@ -186,6 +190,7 @@ class LinkedAnalysisDriver {
LinkedAnalysisDriver(
this.analysisOptions,
this.resynthesizer,
this.elementFactory,
this.sourceFactory,
this.libraryUris,
this.declaredVariables,
@ -193,12 +198,22 @@ class LinkedAnalysisDriver {
this._fsState,
this._resourceProvider);
TypeProvider get typeProvider => resynthesizer.typeProvider;
TypeProvider get typeProvider {
if (resynthesizer != null) {
return resynthesizer.typeProvider;
} else {
return elementFactory.analysisContext.typeProvider;
}
}
/// True if [uri] refers to a Dart library (i.e. a Dart source file exists
/// with this uri, and it is not a part file).
bool _isLibraryUri(String uri) {
return resynthesizer.hasLibrarySummary(uri);
if (resynthesizer != null) {
return resynthesizer.hasLibrarySummary(uri);
} else {
return elementFactory.isLibraryUri(uri);
}
}
/// Analyzes the library at [uri] and returns the results of analysis for all
@ -208,16 +223,23 @@ class LinkedAnalysisDriver {
throw ArgumentError('"$libraryUri" is not a library');
}
AnalysisContext analysisContext;
if (resynthesizer != null) {
analysisContext = resynthesizer.context;
} else {
analysisContext = elementFactory.analysisContext;
}
var libraryFile = _fsState.getFileForUri(Uri.parse(libraryUri));
var analyzer = LibraryAnalyzer(
analysisOptions as AnalysisOptionsImpl,
declaredVariables,
resynthesizer.sourceFactory,
sourceFactory,
(uri) => _isLibraryUri('$uri'),
resynthesizer.context,
analysisContext,
resynthesizer,
null,
InheritanceManager2(resynthesizer.typeSystem),
elementFactory,
InheritanceManager2(analysisContext.typeSystem),
libraryFile,
_resourceProvider);
// TODO(jmesserly): ideally we'd use the existing public `analyze()` method,
@ -233,6 +255,10 @@ class LinkedAnalysisDriver {
}
LibraryElement getLibrary(String uri) {
return resynthesizer.getLibraryElement(uri);
if (resynthesizer != null) {
return resynthesizer.getLibraryElement(uri);
} else {
return elementFactory.libraryOfUri(uri);
}
}
}

View file

@ -3,11 +3,13 @@
// BSD-style license that can be found in the LICENSE file.
import 'dart:collection';
import 'package:analyzer/dart/element/element.dart'
show ClassElement, CompilationUnitElement, Element;
show ClassElement, CompilationUnitElement, Element, LibraryElement;
import 'package:analyzer/dart/element/type.dart' show DartType, InterfaceType;
import 'package:analyzer/src/generated/resolver.dart' show TypeProvider;
import 'package:analyzer/src/summary/resynthesize.dart';
import 'package:analyzer/src/summary2/linked_element_factory.dart';
import 'element_helpers.dart' show getAnnotationName, isBuiltinAnnotation;
/// Contains information about native JS types (those types provided by the
@ -29,6 +31,7 @@ import 'element_helpers.dart' show getAnnotationName, isBuiltinAnnotation;
/// `first` to the `Array.prototype`.
class ExtensionTypeSet {
final SummaryResynthesizer _resynthesizer;
final LinkedElementFactory _elementFactory;
// Abstract types that may be implemented by both native and non-native
// classes.
@ -38,7 +41,8 @@ class ExtensionTypeSet {
final _nativeTypes = HashSet<ClassElement>();
final _pendingLibraries = HashSet<String>();
ExtensionTypeSet(TypeProvider types, this._resynthesizer) {
ExtensionTypeSet(
TypeProvider types, this._resynthesizer, this._elementFactory) {
// TODO(vsm): Eventually, we want to make this extensible - i.e., find
// annotations in user code as well. It would need to be summarized in
// the element model - not searched this way on every compile. To make this
@ -108,14 +112,14 @@ class ExtensionTypeSet {
}
void _addExtensionTypesForLibrary(String libraryUri, List<String> typeNames) {
var library = _resynthesizer.getLibraryElement(libraryUri);
var library = _getLibraryByUri(libraryUri);
for (var typeName in typeNames) {
_addExtensionType(library.getType(typeName).type);
}
}
void _addExtensionTypes(String libraryUri) {
var library = _resynthesizer.getLibraryElement(libraryUri);
var library = _getLibraryByUri(libraryUri);
_visitCompilationUnit(library.definingCompilationUnit);
library.parts.forEach(_visitCompilationUnit);
}
@ -124,6 +128,14 @@ class ExtensionTypeSet {
_pendingLibraries.add(libraryUri);
}
LibraryElement _getLibraryByUri(String uriStr) {
if (_resynthesizer != null) {
return _resynthesizer.getLibraryElement(uriStr);
} else {
return _elementFactory.libraryOfUri(uriStr);
}
}
bool _processPending(Element element) {
if (_pendingLibraries.isEmpty) return false;
if (element is ClassElement) {