fix #27607, add dev_compiler summary to the SDK and move JS files

R=vsm@google.com

Review URL: https://codereview.chromium.org/2474523003 .

pkg/dev_compiler/lib/src/analyzer/context.dart

@@ -55,13 +55,25 @@ class AnalyzerOptions {
       this.packagePaths: const []})
       : dartSdkPath = dartSdkPath ?? getSdkDir().path;
 
-  AnalyzerOptions.fromArguments(ArgResults args)
+  factory AnalyzerOptions.fromArguments(ArgResults args) {
-      : summaryPaths = args['summary'] as List<String>,
+    var sdkPath = args['dart-sdk'] ?? getSdkDir().path;
-        dartSdkPath = args['dart-sdk'] ?? getSdkDir().path,
+    var sdkSummaryPath = args['dart-sdk-summary'];
-        dartSdkSummaryPath = args['dart-sdk-summary'],
+
-        customUrlMappings = _parseUrlMappings(args['url-mapping']),
+    if (sdkSummaryPath == null) {
-        packageRoot = args['package-root'],
+      sdkSummaryPath = path.join(sdkPath, 'lib', '_internal', 'ddc_sdk.sum');
-        packagePaths = (args['package-paths'] as String)?.split(',') ?? [];
+    } else if (sdkSummaryPath == 'build') {
+      // For building the SDK, we explicitly set the path to none.
+      sdkSummaryPath = null;
+    }
+
+    return new AnalyzerOptions(
+        summaryPaths: args['summary'] as List<String>,
+        dartSdkPath: sdkPath,
+        dartSdkSummaryPath: sdkSummaryPath,
+        customUrlMappings: _parseUrlMappings(args['url-mapping']),
+        packageRoot: args['package-root'],
+        packagePaths: (args['package-paths'] as String)?.split(',') ?? []);
+  }
 
   /// Whether to resolve 'package:' uris using the multi-package resolver.
   bool get useMultiPackage => packagePaths.isNotEmpty;
@@ -70,9 +82,10 @@ class AnalyzerOptions {
     parser
       ..addOption('summary',
           abbr: 's', help: 'summary file(s) to include', allowMultiple: true)
-      ..addOption('dart-sdk', help: 'Dart SDK Path', defaultsTo: null)
+      ..addOption('dart-sdk',
+          help: 'Dart SDK Path', defaultsTo: null, hide: true)
       ..addOption('dart-sdk-summary',
-          help: 'Dart SDK Summary Path', defaultsTo: null)
+          help: 'Dart SDK Summary Path', defaultsTo: null, hide: true)
       ..addOption('package-root',
           abbr: 'p', help: 'Package root to resolve "package:" imports')
       ..addOption('url-mapping',

pkg/dev_compiler/lib/src/compiler/command.dart

@@ -154,11 +154,17 @@ void _compile(ArgResults argResults, void printFn(Object obj)) {
 
   // Write JS file, as well as source map and summary (if requested).
   for (var i = 0; i < outPaths.length; i++) {
-    var outPath = outPaths[i];
+    module.writeCodeSync(moduleFormats[i], outPaths[i],
-    module.writeCodeSync(moduleFormats[i], singleOutFile, outPath);
+        singleOutFile: singleOutFile);
-    if (module.summaryBytes != null) {
+  }
-      var summaryPath =
+  if (module.summaryBytes != null) {
-          path.withoutExtension(outPath) + '.${compilerOpts.summaryExtension}';
+    var summaryPaths = compilerOpts.summaryOutPath != null
+        ? [compilerOpts.summaryOutPath]
+        : outPaths.map((p) =>
+            '${path.withoutExtension(p)}.${compilerOpts.summaryExtension}');
+
+    // place next to every compiled module
+    for (var summaryPath in summaryPaths) {
       // Only overwrite if summary changed.  This plays better with timestamp
       // based build systems.
       var file = new File(summaryPath);

pkg/dev_compiler/lib/src/compiler/compiler.dart

@@ -261,6 +261,10 @@ class CompilerOptions {
   /// source maps.
   final Map<String, String> bazelMapping;
 
+  /// If specified, the path to write the summary file.
+  /// Used when building the SDK.
+  final String summaryOutPath;
+
   const CompilerOptions(
       {this.sourceMap: true,
       this.sourceMapComment: true,
@@ -277,7 +281,8 @@ class CompilerOptions {
       this.nameTypeTests: true,
       this.hoistTypeTests: true,
       this.useAngular2Whitelist: false,
-      this.bazelMapping: const {}});
+      this.bazelMapping: const {},
+      this.summaryOutPath});
 
   CompilerOptions.fromArguments(ArgResults args)
       : sourceMap = args['source-map'],
@@ -295,7 +300,8 @@ class CompilerOptions {
         nameTypeTests = args['name-type-tests'],
         hoistTypeTests = args['hoist-type-tests'],
         useAngular2Whitelist = args['unsafe-angular2-whitelist'],
-        bazelMapping = _parseBazelMappings(args['bazel-mapping']);
+        bazelMapping = _parseBazelMappings(args['bazel-mapping']),
+        summaryOutPath = args['summary-out'];
 
   static void addArguments(ArgParser parser) {
     parser
@@ -349,7 +355,9 @@ class CompilerOptions {
               'to/library.dart as the path for library.dart in source maps.',
           allowMultiple: true,
           splitCommas: false,
-          hide: true);
+          hide: true)
+      ..addOption('summary-out',
+          help: 'location to write the summary file', hide: true);
   }
 
   static Map<String, String> _parseBazelMappings(Iterable argument) {
@@ -427,8 +435,8 @@ class JSModuleFile {
   //
   // TODO(jmesserly): this should match our old logic, but I'm not sure we are
   // correctly handling the pointer from the .js file to the .map file.
-  JSModuleCode getCode(
+  JSModuleCode getCode(ModuleFormat format, String jsUrl, String mapUrl,
-      ModuleFormat format, bool singleOutFile, String jsUrl, String mapUrl) {
+      {bool singleOutFile: false}) {
     var opts = new JS.JavaScriptPrintingOptions(
         emitTypes: options.closure,
         allowKeywordsInProperties: true,
@@ -443,7 +451,8 @@ class JSModuleFile {
       printer = new JS.SimpleJavaScriptPrintingContext();
     }
 
-    var tree = transformModuleFormat(format, singleOutFile, moduleTree);
+    var tree =
+        transformModuleFormat(format, moduleTree, singleOutFile: singleOutFile);
     tree.accept(
         new JS.Printer(opts, printer, localNamer: new JS.TemporaryNamer(tree)));
 
@@ -477,9 +486,10 @@ class JSModuleFile {
   ///
   /// If [mapPath] is not supplied but [options.sourceMap] is set, mapPath
   /// will default to [jsPath].map.
-  void writeCodeSync(ModuleFormat format, bool singleOutFile, String jsPath) {
+  void writeCodeSync(ModuleFormat format, String jsPath,
+      {bool singleOutFile: false}) {
     String mapPath = jsPath + '.map';
-    var code = getCode(format, singleOutFile, jsPath, mapPath);
+    var code = getCode(format, jsPath, mapPath, singleOutFile: singleOutFile);
     var c = code.code;
     if (singleOutFile) {
       // In singleOutFile mode we wrap each module in an eval statement to

pkg/dev_compiler/lib/src/compiler/module_builder.dart

@@ -65,9 +65,10 @@ void addModuleFormatOptions(ArgParser argParser, {bool allowMultiple: false}) {
         allowMultiple: allowMultiple,
         defaultsTo: 'amd')
     ..addFlag('single-out-file',
-        help: 'emit output so that libraries can be concatenated together into '
+        help: 'emit modules that can be concatenated into one file.\n'
-            'a single file. Only compatible with legacy and amd module formats.',
+            'Only compatible with legacy and amd module formats.',
-        defaultsTo: false);
+        defaultsTo: false,
+        hide: true);
 }
 
 /// Transforms an ES6 [module] into a given module [format].
@@ -79,16 +80,20 @@ void addModuleFormatOptions(ArgParser argParser, {bool allowMultiple: false}) {
 /// that affects the top-level module items, especially [ImportDeclaration]s and
 /// [ExportDeclaration]s.
 Program transformModuleFormat(
-    ModuleFormat format, bool singleOutFile, Program module) {
+    ModuleFormat format, Program module, {bool singleOutFile: false}) {
   switch (format) {
     case ModuleFormat.legacy:
-      return new LegacyModuleBuilder(singleOutFile).build(module);
+      // Legacy format always generates output compatible with single file mode.
+      return new LegacyModuleBuilder().build(module);
     case ModuleFormat.common:
-      return new CommonJSModuleBuilder(singleOutFile).build(module);
+      assert(!singleOutFile);
+      return new CommonJSModuleBuilder().build(module);
     case ModuleFormat.amd:
-      return new AmdModuleBuilder(singleOutFile).build(module);
+      // TODO(jmesserly): encode singleOutFile as a module format?
+      // Since it's irrelevant except for AMD.
+      return new AmdModuleBuilder(singleOutFile: singleOutFile).build(module);
     case ModuleFormat.es6:
-      assert(singleOutFile == false);
+      assert(!singleOutFile);
       return module;
   }
   return null; // unreachable. suppresses a bogus analyzer message
@@ -137,9 +142,6 @@ abstract class _ModuleBuilder {
 /// Generates modules for with our legacy `dart_library.js` loading mechanism.
 // TODO(jmesserly): remove this and replace with something that interoperates.
 class LegacyModuleBuilder extends _ModuleBuilder {
-  /// The legacy module format always generates output compatible with a single
-  /// file mode.
-  LegacyModuleBuilder(bool singleOutFile);
 
   Program build(Program module) {
     // Collect imports/exports/statements.
@@ -198,14 +200,6 @@ class LegacyModuleBuilder extends _ModuleBuilder {
 
 /// Generates CommonJS modules (used by Node.js).
 class CommonJSModuleBuilder extends _ModuleBuilder {
-  final bool singleOutFile;
-
-  CommonJSModuleBuilder(this.singleOutFile) {
-    // singleOutFile mode is not currently supported by the CommonJS module
-    // builder.
-    assert(singleOutFile == false);
-  }
-
   Program build(Program module) {
     var importStatements = <Statement>[];
 
@@ -257,7 +251,7 @@ class CommonJSModuleBuilder extends _ModuleBuilder {
 class AmdModuleBuilder extends _ModuleBuilder {
   final bool singleOutFile;
 
-  AmdModuleBuilder(this.singleOutFile);
+  AmdModuleBuilder({this.singleOutFile: false});
 
   Program build(Program module) {
     var importStatements = <Statement>[];

pkg/dev_compiler/lib/src/js_ast/js_ast.dart

@@ -11,7 +11,6 @@ import '../closure/closure_annotation.dart';
 part 'nodes.dart';
 part 'builder.dart';
 part 'js_types.dart';
-part 'module_transform.dart';
 part 'printer.dart';
 part 'template.dart';
 part 'type_printer.dart';

pkg/dev_compiler/lib/src/js_ast/module_transform.dart

@@ -1,25 +0,0 @@
-// Copyright (c) 2015, the Dart project authors.  Please see the AUTHORS file
-// for details. All rights reserved. Use of this source code is governed by a
-// BSD-style license that can be found in the LICENSE file.
-
-part of js_ast;
-
-/**
- * Transforms ECMAScript 6 modules to an ES 5 file using a module pattern.
- *
- * There are various module patterns in JavaScript, see
- * <http://babeljs.io/docs/usage/modules/> for some examples.
- *
- * At the moment, we only support our "custom Dart" conversion, roughly similar
- * to Asynchronous Module Definition (AMD), see also
- * <http://requirejs.org/docs/whyamd.html>. Like AMD, module files can
- * be loaded directly in the browser with no further transformation (e.g.
- * browserify, webpack).
- */
-// TODO(jmesserly): deprecate the "custom dart" form in favor of AMD.
-class CustomDartModuleTransform extends BaseVisitor {
-  // TODO(jmesserly): implement these. Module should transform to Program.
-  visitImportDeclaration(ImportDeclaration node) {}
-  visitExportDeclaration(ExportDeclaration node) {}
-  visitModule(Module node) {}
-}

pkg/dev_compiler/test/codegen_test.dart

@@ -68,7 +68,7 @@ main(List<String> arguments) {
 
   var sdkDir = path.join(repoDirectory, 'gen', 'patched_sdk');
   var sdkSummaryFile =
-      path.join(testDirectory, '..', 'lib', 'js', 'amd', 'dart_sdk.sum');
+      path.join(testDirectory, '..', 'lib', 'sdk', 'ddc_sdk.sum');
 
   var summaryPaths = new Directory(path.join(codegenOutputDir, 'pkg'))
       .listSync()
@@ -174,7 +174,7 @@ void _writeModule(String outPath, String expectPath, ModuleFormat format,
   if (errors.isNotEmpty && !errors.endsWith('\n')) errors += '\n';
   new File(outPath + '.txt').writeAsStringSync(errors);
 
-  result.writeCodeSync(format, false, outPath + '.js');
+  result.writeCodeSync(format, outPath + '.js');
 
   if (result.summaryBytes != null) {
     new File(outPath + '.sum').writeAsBytesSync(result.summaryBytes);
@@ -188,7 +188,7 @@ void _writeModule(String outPath, String expectPath, ModuleFormat format,
 
     var expectFile = new File(expectPath + '.js');
     if (result.isValid) {
-      result.writeCodeSync(format, false, expectFile.path);
+      result.writeCodeSync(format, expectFile.path);
     } else {
       expectFile.writeAsStringSync("//FAILED TO COMPILE");
     }

pkg/dev_compiler/tool/build_pkgs.sh

@@ -5,7 +5,7 @@ cd $( dirname "${BASH_SOURCE[0]}" )/..
 
 mkdir -p gen/codegen_output/pkg/
 
-SDK=--dart-sdk-summary=lib/js/amd/dart_sdk.sum
+SDK=--dart-sdk-summary=lib/sdk/ddc_sdk.sum
 
 # Build leaf packages.  These have no other package dependencies.
 

pkg/dev_compiler/tool/build_sdk.sh

@@ -11,6 +11,8 @@ echo "*** Compiling SDK to JavaScript"
 # TODO(jmesserly): break out dart:html & friends.
 dart -c tool/build_sdk.dart \
     --dart-sdk gen/patched_sdk \
+    --dart-sdk-summary=build \
+    --summary-out lib/sdk/ddc_sdk.sum \
     --modules=amd \
     -o lib/js/amd/dart_sdk.js \
     --modules=es6 \

pkg/dev_compiler/tool/build_test_pkgs.sh

@@ -5,7 +5,7 @@ cd $( dirname "${BASH_SOURCE[0]}" )/..
 
 mkdir -p gen/codegen_output/pkg/
 
-SDK=--dart-sdk-summary=lib/js/amd/dart_sdk.sum
+SDK=--dart-sdk-summary=lib/sdk/ddc_sdk.sum
 
 ./bin/dartdevc.dart $SDK -o gen/codegen_output/pkg/expect.js \
     package:expect/expect.dart \

pkg/dev_compiler/web/web_command.dart

@@ -197,11 +197,13 @@ class WebCompileCommand extends Command {
 
       JSModuleFile module = compiler.compile(unit, compilerOptions);
 
-      var moduleCode = module.isValid
+      var moduleCode = '';
-          ? module
+      if (module.isValid) {
-              .getCode(ModuleFormat.legacy, true, unit.name, unit.name + '.map')
+        moduleCode = module
-              .code
+            .getCode(ModuleFormat.legacy, unit.name, unit.name + '.map',
-          : '';
+                singleOutFile: true)
+            .code;
+      }
 
       return new CompileResult(
           code: moduleCode, isValid: module.isValid, errors: module.errors);

tools/create_sdk.py

@@ -171,8 +171,10 @@ def CopyAnalysisSummaries(snapshots, lib):
            join(lib, '_internal', 'strong.sum'))
 
 def CopyDevCompilerSdk(home, lib):
+  copyfile(join(home, 'pkg', 'dev_compiler', 'lib', 'sdk', 'ddc_sdk.sum'),
+           join(lib, '_internal', 'ddc_sdk.sum'))
   copytree(join(home, 'pkg', 'dev_compiler', 'lib', 'js'),
-           join(lib, '_internal', 'dev_compiler'))
+           join(lib, 'dev_compiler'))
 
 def Main():
   # Pull in all of the gypi files which will be munged into the sdk.