Skeleton code for running the forthcoming async/await compiler on pub.

R=nweiz@google.com

Review URL: https://codereview.chromium.org//472173004

git-svn-id: https://dart.googlecode.com/svn/branches/bleeding_edge/dart@39530 260f80e4-7a28-3924-810f-c04153c831b5

sdk/bin/pub

@@ -3,8 +3,9 @@
 # for details. All rights reserved. Use of this source code is governed by a
 # BSD-style license that can be found in the LICENSE file.
 
-# Run pub.dart on the Dart VM. This script assumes the Dart SDK's directory
-# structure.
+# Run pub.dart on the Dart VM. This script is only used when running pub from
+# within the Dart source repo. The shipped SDK instead uses "pub_sdk", which is
+# renamed to "pub" when the SDK is built.
 
 function follow_links() {
   file="$1"
@@ -38,30 +39,28 @@ if [[ $DART_VM_OPTIONS ]]; then
   VM_OPTIONS+=("${OPTIONS[@]}")
 fi
 
-if test -f "$SNAPSHOT"; then
-  # We are running the snapshot in the built SDK.
-  DART="$BIN_DIR/dart"
-  exec "$DART" "${VM_OPTIONS[@]}" "$SNAPSHOT" "$@"
-else
-  # We are running pub from source in the development repo.
-  if [ -z "$DART_CONFIGURATION" ];
-  then
-    DART_CONFIGURATION="ReleaseIA32"
-  fi
-
-  if [[ `uname` == 'Darwin' ]];
-  then
-    BUILD_DIR="$SDK_DIR/../xcodebuild/$DART_CONFIGURATION"
-  else
-    BUILD_DIR="$SDK_DIR/../out/$DART_CONFIGURATION"
-  fi
-
-  # Use the Dart binary in the built SDK so pub can find the version file next
-  # to it.
-  DART="$BUILD_DIR/dart-sdk/bin/dart"
-  PACKAGES_DIR="$BUILD_DIR/pub_packages/"
-
-  PUB="$SDK_DIR/lib/_internal/pub/bin/pub.dart"
-
-  exec "$DART" "${VM_OPTIONS[@]}" "--package-root=$PACKAGES_DIR" "$PUB" "$@"
+if [ -z "$DART_CONFIGURATION" ];
+then
+  DART_CONFIGURATION="ReleaseIA32"
 fi
+
+if [[ `uname` == 'Darwin' ]];
+then
+  BUILD_DIR="$SDK_DIR/../xcodebuild/$DART_CONFIGURATION"
+else
+  BUILD_DIR="$SDK_DIR/../out/$DART_CONFIGURATION"
+fi
+
+# Use the Dart binary in the built SDK so pub can find the version file next
+# to it.
+DART="$BUILD_DIR/dart-sdk/bin/dart"
+PACKAGES_DIR="$BUILD_DIR/pub_packages/"
+
+# Compile async/await down to vanilla Dart.
+# TODO(rnystrom): Remove this when #104 is fixed.
+ASYNC_COMPILER="$SDK_DIR/lib/_internal/pub/bin/async_compile.dart"
+"$DART" "--package-root=$PACKAGES_DIR" "$ASYNC_COMPILER" "$BUILD_DIR" --silent
+
+# Run the async/await compiled pub.
+PUB="$BUILD_DIR/pub_async/bin/pub.dart"
+exec "$DART" "${VM_OPTIONS[@]}" "--package-root=$PACKAGES_DIR" "$PUB" "$@"

sdk/bin/pub_sdk

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
+# Copyright (c) 2014, the Dart project authors.  Please see the AUTHORS file
 # for details. All rights reserved. Use of this source code is governed by a
 # BSD-style license that can be found in the LICENSE file.
 
@@ -21,8 +21,6 @@ PROG_NAME="$(follow_links "$BASH_SOURCE")"
 # Handle the case where dart-sdk/bin has been symlinked to.
 BIN_DIR="$(cd "${PROG_NAME%/*}" ; pwd -P)"
 
-SDK_DIR="$(cd "${BIN_DIR}/.." ; pwd -P)"
-
 SNAPSHOT="$BIN_DIR/snapshots/pub.dart.snapshot"
 
 unset VM_OPTIONS
@@ -38,32 +36,6 @@ if [[ $DART_VM_OPTIONS ]]; then
   VM_OPTIONS+=("${OPTIONS[@]}")
 fi
 
-VM_OPTIONS+=("--checked")
-
-if test -f "$SNAPSHOT"; then
-  # We are running the snapshot in the built SDK.
-  DART="$BIN_DIR/dart"
-  exec "$DART" "${VM_OPTIONS[@]}" "$SNAPSHOT" "$@"
-else
-  # We are running pub from source in the development repo.
-  if [ -z "$DART_CONFIGURATION" ];
-  then
-    DART_CONFIGURATION="ReleaseIA32"
-  fi
-
-  if [[ `uname` == 'Darwin' ]];
-  then
-    BUILD_DIR="$SDK_DIR/../xcodebuild/$DART_CONFIGURATION"
-  else
-    BUILD_DIR="$SDK_DIR/../out/$DART_CONFIGURATION"
-  fi
-
-  # Use the Dart binary in the built SDK so pub can find the version file next
-  # to it.
-  DART="$BUILD_DIR/dart-sdk/bin/dart"
-  PACKAGES_DIR="$BUILD_DIR/packages/"
-
-  PUB="$SDK_DIR/lib/_internal/pub/bin/pub.dart"
-
-  exec "$DART" "${VM_OPTIONS[@]}" "--package-root=$PACKAGES_DIR" "$PUB" "$@"
-fi
+# Run the pub snapshot.
+DART="$BIN_DIR/dart"
+exec "$DART" "${VM_OPTIONS[@]}" "$SNAPSHOT" "$@"

sdk/lib/_internal/pub/bin/async_compile.dart

@@ -0,0 +1,102 @@
+// Copyright (c) 2014, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:io';
+
+import 'package:path/path.dart' as p;
+
+/// The path to pub's root directory (sdk/lib/_internal/pub) in the Dart repo.
+///
+/// This assumes this script is itself being run from within the repo.
+final sourceDir = p.dirname(p.dirname(p.fromUri(Platform.script)));
+
+/// The [sourceDir] as a URL, for use in import strings.
+final sourceUrl = p.toUri(sourceDir).toString();
+
+/// The directory that compiler output should be written to.
+String buildDir;
+
+/// This runs the async/await compiler on all of the pub source code.
+///
+/// It reads from the repo and writes the compiled output into the given build
+/// directory (using the same file names and relative layout). Does not
+/// compile files that haven't changed since the last time they were compiled.
+// TODO(rnystrom): Remove this when #104 is fixed.
+void main(List<String> arguments) {
+  _validate(arguments.isNotEmpty, "Missing build directory.");
+  _validate(arguments.length <= 2, "Unexpected arguments.");
+  if (arguments.length == 2) {
+    _validate(arguments[1] == "--silent",
+        "Invalid argument '${arguments[1]}");
+  }
+
+  // Create the build output directory if it's not already there.
+  buildDir = p.join(p.normalize(arguments[0]), "pub_async");
+  new Directory(buildDir).createSync(recursive: true);
+
+  var silent = arguments.length == 2 && arguments[1] == "--silent";
+  var numFiles = 0;
+  var numCompiled = 0;
+
+  // Compile any modified or missing files.
+  for (var entry in new Directory(sourceDir).listSync(recursive: true)) {
+    if (p.extension(entry.path) != ".dart") continue;
+
+    // Skip tests.
+    // TODO(rnystrom): Do we want to use this for tests too?
+    if (p.isWithin(p.join(sourceDir, "test"), entry.path)) continue;
+
+    numFiles++;
+    var relative = p.relative(entry.path, from: sourceDir);
+
+    var sourceFile = entry as File;
+    var destPath = p.join(buildDir, relative);
+    var destFile = new File(destPath);
+    if (!destFile.existsSync() ||
+        entry.lastModifiedSync().isAfter(destFile.lastModifiedSync())) {
+      compile(sourceFile.path, sourceFile.readAsStringSync(), destPath);
+      numCompiled++;
+      if (!silent) print("Compiled ${sourceFile.path}.");
+    }
+  }
+
+  if (!silent) print("Compiled $numCompiled out of $numFiles files.");
+}
+
+final _compilerPattern = new RegExp(r"import '(\.\./)+compiler");
+
+void compile(String sourcePath, String source, String destPath) {
+  var destDir = new Directory(p.dirname(destPath));
+  destDir.createSync(recursive: true);
+
+  // TODO(rnystrom): Do real async/await transformation here!
+  source = source.replaceAll("ASYNC!", "");
+
+  // Pub imports dart2js using relative imports that reach outside of pub's
+  // source tree. Since the build directory is in a different location, we need
+  // to fix those to be valid relative imports from the build directory.
+  var compilerDir = p.url.join(sourceUrl, "../compiler");
+  var relative = p.url.relative(compilerDir, from: destDir.path);
+  source = source.replaceAll(_compilerPattern, "import '$relative");
+
+  try {
+    new File(destPath).writeAsStringSync(source);
+  } on IOException catch (ex) {
+    // Do nothing. This may happen if two instances of the compiler are running
+    // concurrently and compile the same file. The second one to try to write
+    // the output may fail if the file is still open. Since they are producing
+    // the same output anyway, just ignore it when the second one fails.
+  }
+}
+
+/// Validates command-line argument usage and exits with [message] if [valid]
+/// is `false`.
+void _validate(bool valid, String message) {
+  if (valid) return;
+
+  stderr.writeln(message);
+  stderr.writeln();
+  stderr.writeln("Usage: dart async_compile.dart <build dir> [--silent]");
+  exit(64);
+}

sdk/lib/_internal/pub/lib/src/io.dart

@@ -486,8 +486,24 @@ String get repoRoot {
   if (runningFromSdk) {
     throw new StateError("Can't get the repo root from the SDK.");
   }
-  return path.normalize(path.join(
-      path.dirname(libraryPath('pub.io')), '..', '..', '..', '..', '..', '..'));
+
+  // Get the path to the directory containing this very file.
+  var libDir = path.dirname(libraryPath('pub.io'));
+
+  // TODO(rnystrom): Remove this when #104 is fixed.
+  // If we are running from the async/await compiled build directory, walk out
+  // out of that. It will be something like:
+  //
+  //     <repo>/<build>/<config>/pub_async/lib/src
+  if (libDir.contains('pub_async')) {
+    return path.normalize(path.join(libDir, '..', '..', '..', '..', '..'));
+  }
+
+  // Otherwise, assume we're running directly from the source location in the
+  // repo:
+  //
+  //      <repo>/sdk/lib/_internal/pub/lib/src
+  return path.normalize(path.join(libDir, '..', '..', '..', '..', '..', '..'));
 }
 
 /// A line-by-line stream of standard input.

sdk/lib/_internal/pub/test/test_pub.dart

@@ -476,6 +476,39 @@ void confirmPublish(ScheduledProcess pub) {
   pub.writeLine("y");
 }
 
+/// Whether the async/await compiler has already been run.
+///
+/// If a test suite runs pub more than once, we only need to run the compiler
+/// the first time.
+// TODO(rnystrom): Remove this when #104 is fixed.
+bool _compiledAsync = false;
+
+/// Gets the path to the pub entrypoint Dart script to run.
+// TODO(rnystrom): This exists to run the async/await compiler on pub and then
+// get the path to the output of that. Once #104 is fixed, remove this.
+String _getPubPath(String dartBin) {
+  var buildDir = path.join(path.dirname(dartBin), '../../');
+
+  // Ensure the async/await compiler has been run once for this test suite. The
+  // compiler itself will only re-compile source files that have actually
+  // changed, so this is a no-op if everything is already compiled.
+  if (!_compiledAsync) {
+    var result = Process.runSync(dartBin, [
+      '--package-root=$_packageRoot/',
+      path.join(testDirectory, '..', 'bin', 'async_compile.dart'),
+      buildDir,
+      '--silent'
+    ]);
+    stdout.write(result.stdout);
+    stderr.write(result.stderr);
+    if (result.exitCode != 0) fail("Async/await compiler failed.");
+
+    _compiledAsync = true;
+  }
+
+  return path.join(buildDir, 'pub_async/bin/pub.dart');
+}
+
 /// Starts a Pub process and returns a [ScheduledProcess] that supports
 /// interaction with that process.
 ///
@@ -498,7 +531,10 @@ ScheduledProcess startPub({List args, Future<Uri> tokenEndpoint}) {
   }
 
   // Find the main pub entrypoint.
-  var pubPath = path.join(testDirectory, '..', 'bin', 'pub.dart');
+  var pubPath = _getPubPath(dartBin);
+  // TODO(rnystrom): Replace the above line with the following when #104 is
+  // fixed.
+  //var pubPath = path.join(testDirectory, '..', 'bin', 'pub.dart');
 
   var dartArgs = ['--package-root=$_packageRoot/', '--checked', pubPath,
       '--verbose'];

tools/create_pub_snapshot.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2014 The Dart Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""
+This script runs the async/await compiler on pub and then compiles the output
+of that to a snapshot.
+
+Usage: create_pub_snapshot.py <dart> <compiler> <package root> <output dir>
+
+When #104 is fixed, this script is no longer needed. Instead, replace the
+generate_pub_snapshot action in utils/pub.gyp with:
+
+    'action': [
+      '<(PRODUCT_DIR)/<(EXECUTABLE_PREFIX)dart<(EXECUTABLE_SUFFIX)',
+      '--package-root=<(PRODUCT_DIR)/pub_packages/',
+      '--snapshot=<(SHARED_INTERMEDIATE_DIR)/pub.dart.snapshot',
+      '../../sdk/lib/_internal/pub/bin/pub.dart',
+    ]
+"""
+
+import os
+import subprocess
+import sys
+
+
+def Main():
+  if len(sys.argv) < 5:
+    raise Exception("""Not enough arguments.
+
+Usage: create_pub_snapshot.py <dart> <compiler> <package root> <output file>""")
+
+  dart_path = sys.argv[1]
+  compiler = sys.argv[2]
+  package_root = sys.argv[3]
+  output_dir = sys.argv[4]
+
+  # Run the async compiler.
+  status = subprocess.call([
+    dart_path,
+    '--package-root=' + package_root,
+    compiler,
+    output_dir,
+    '--silent'
+  ])
+  if status != 0: return status
+
+  # Generate the snapshot from the output of that.
+  status = subprocess.call([
+    dart_path,
+    '--package-root=' + package_root,
+    '--snapshot=' + os.path.join(output_dir, 'pub.dart.snapshot'),
+    os.path.join(output_dir, 'pub_async/bin/pub.dart')
+  ])
+  if status != 0: return status
+
+
+if __name__ == '__main__':
+  sys.exit(Main())

tools/create_sdk.py

@@ -101,19 +101,24 @@ def CopyShellScript(src_file, dest_dir):
   if HOST_OS == 'win32':
     file_extension = '.bat'
 
+  # If we're copying an SDK-specific shell script, strip off the suffix.
+  dest_file = basename(src_file)
+  if dest_file.endswith('_sdk'):
+    dest_file = dest_file.replace('_sdk', '')
+
   src = src_file + file_extension
-  dest = join(dest_dir, basename(src_file) + file_extension)
+  dest = join(dest_dir, dest_file + file_extension)
   Copy(src, dest)
 
 
 def CopyDartScripts(home, sdk_root):
-  for executable in ['dart2js', 'dartanalyzer', 'dartfmt', 'docgen', 'pub']:
+  for executable in ['dart2js', 'dartanalyzer', 'dartfmt', 'docgen', 'pub_sdk']:
     CopyShellScript(os.path.join(home, 'sdk', 'bin', executable),
                     os.path.join(sdk_root, 'bin'))
 
 
 def CopySnapshots(snapshots, sdk_root):
-  for snapshot in ['analysis_server', 'dart2js', 'dartanalyzer', 'dartfmt', 
+  for snapshot in ['analysis_server', 'dart2js', 'dartanalyzer', 'dartfmt',
                    'utils_wrapper', 'pub']:
     snapshot += '.dart.snapshot'
     copyfile(join(snapshots, snapshot),

utils/pub/pub.gyp

@@ -29,10 +29,12 @@
             '<(SHARED_INTERMEDIATE_DIR)/pub.dart.snapshot',
           ],
           'action': [
+            'python',
+            '../../tools/create_pub_snapshot.py',
             '<(PRODUCT_DIR)/<(EXECUTABLE_PREFIX)dart<(EXECUTABLE_SUFFIX)',
-            '--package-root=<(PRODUCT_DIR)/pub_packages/',
-            '--snapshot=<(SHARED_INTERMEDIATE_DIR)/pub.dart.snapshot',
-            '../../sdk/lib/_internal/pub/bin/pub.dart',
+            '../../sdk/lib/_internal/pub/bin/async_compile.dart',
+            '<(PRODUCT_DIR)/pub_packages/',
+            '<(SHARED_INTERMEDIATE_DIR)',
           ],
         },
       ],