Generate docgen output along with api_docs as part of the build

R=danrubel@google.com, efortuna@google.com, kustermann@google.com, rnystrom@google.com

Review URL: https://codereview.chromium.org//73113002

git-svn-id: https://dart.googlecode.com/svn/branches/bleeding_edge/dart@30485 260f80e4-7a28-3924-810f-c04153c831b5

dart.gyp

@@ -86,6 +86,9 @@
       'dependencies': [
         'create_sdk',
       ],
+      'inputs': [
+        '<(PRODUCT_DIR)/dart-sdk/README',
+      ],
       'actions': [
         {
           'action_name': 'upload_sdk_py',
@@ -155,7 +158,10 @@
       'target_name': 'api_docs',
       'type': 'none',
       'dependencies': [
+        # TODO(alanknight) : Once we're fully switched over to the new
+        # viewer remove the old api_docs gyp file.
         'utils/apidoc/apidoc.gyp:api_docs',
+        'utils/apidoc/docgen.gyp:docgen',
       ],
     },
     {

pkg/docgen/bin/docgen.dart

@@ -10,6 +10,8 @@ import 'package:logging/logging.dart';
 import '../lib/docgen.dart';
 import 'package:path/path.dart' as path;
 
+List<String> excludedLibraries = [];
+
 /**
  * Analyzes Dart files and generates a representation of included libraries,
  * classes, and members.
@@ -18,15 +20,21 @@ void main(List<String> arguments) {
   logger.onRecord.listen((record) => print(record.message));
   var results = _initArgParser().parse(arguments);
 
+  var includeSdk = results['parse-sdk'] || results['include-sdk'];
+  var scriptDir = path.dirname(Platform.script.toFilePath());
+  var introFile =
+      path.join(path.dirname(scriptDir), 'doc', 'sdk-introduction.md');
+  var introduction = includeSdk ? introFile : results['introduction'];
   docgen(results.rest.map(path.normalize).toList(),
       packageRoot: results['package-root'],
       outputToYaml: !results['json'],
       includePrivate: results['include-private'],
-      includeSdk: results['parse-sdk'] || results['include-sdk'],
+      includeSdk: includeSdk,
       parseSdk: results['parse-sdk'],
-      append: results['append'] && new Directory('docs').existsSync(),
-      introduction: (results['parse-sdk'] || results['include-sdk']) ?
-          'sdk-introduction.md' : results['introduction']);
+      append: results['append'] && new Directory(results['out']).existsSync(),
+      introduction: introduction,
+      out: results['out'],
+      excludeLibraries: excludedLibraries);
 }
 
 /**
@@ -69,6 +77,12 @@ ArgParser _initArgParser() {
   parser.addOption('introduction',
       help: 'Adds the provided markdown text file as the introduction'
         ' for the outputted documentation.', defaultsTo: '');
-
+  parser.addOption('out',
+      help: 'The name of the output directory.',
+      defaultsTo: 'docs');
+  parser.addOption('exclude-lib',
+      help: 'Exclude the library by this name from the documentation',
+      allowMultiple: true,
+      callback: (libs) => excludedLibraries.addAll(libs));
   return parser;
 }

pkg/docgen/lib/docgen.dart

@@ -36,6 +36,10 @@ import '../../../sdk/lib/_internal/libraries.dart';
 
 var logger = new Logger('Docgen');
 
+const DEFAULT_OUTPUT_DIRECTORY = 'docs';
+
+var _outputDirectory;
+
 const String USAGE = 'Usage: dart docgen.dart [OPTIONS] fooDir/barFile';
 
 
@@ -62,6 +66,12 @@ Map<String, Indexable> entityMap = new Map<String, Indexable>();
 /// This is set from the command line arguments flag --include-private
 bool _includePrivate = false;
 
+/// Library names to explicitly exclude.
+///
+///   Set from the command line option
+/// --exclude-lib.
+List<String> _excluded;
+
 // TODO(janicejl): Make MDN content generic or pluggable. Maybe move
 // MDN-specific code to its own library that is imported into the default impl?
 /// Map of all the comments for dom elements from MDN.
@@ -79,10 +89,13 @@ Map _mdn;
 /// Returned Future completes with true if document generation is successful.
 Future<bool> docgen(List<String> files, {String packageRoot,
     bool outputToYaml: true, bool includePrivate: false, bool includeSdk: false,
-    bool parseSdk: false, bool append: false, String introduction: ''}) {
+    bool parseSdk: false, bool append: false, String introduction: '',
+    out: DEFAULT_OUTPUT_DIRECTORY, List<String> excludeLibraries}) {
+  _excluded = excludeLibraries;
   _includePrivate = includePrivate;
+  _outputDirectory = out;
   if (!append) {
-    var dir = new Directory('docs');
+    var dir = new Directory(_outputDirectory);
     if (dir.existsSync()) dir.deleteSync(recursive: true);
   }
 
@@ -96,26 +109,32 @@ Future<bool> docgen(List<String> files, {String packageRoot,
     }
   }
   logger.info('Package Root: ${packageRoot}');
+  var requestedLibraries = _listLibraries(files);
+  var allLibraries = []..addAll(requestedLibraries);
+  if (includeSdk) {
+    allLibraries.addAll(_listSdk());
+  }
 
-  return getMirrorSystem(files, packageRoot: packageRoot, parseSdk: parseSdk)
+  return getMirrorSystem(allLibraries, packageRoot: packageRoot,
+      parseSdk: parseSdk)
     .then((MirrorSystem mirrorSystem) {
       if (mirrorSystem.libraries.isEmpty) {
         throw new StateError('No library mirrors were created.');
       }
-      var librariesWeAskedFor = _listLibraries(files);
-      var librariesWeGot = mirrorSystem.libraries.values.where(
+      var availableLibraries = mirrorSystem.libraries.values.where(
           (each) => each.uri.scheme == 'file');
       _sdkLibraries = mirrorSystem.libraries.values.where(
           (each) => each.uri.scheme == 'dart');
       _coreLibrary = _sdkLibraries.singleWhere((lib) =>
           lib.uri.toString().startsWith('dart:core'));
-      var librariesWeGotByPath = new Map.fromIterables(
-          librariesWeGot.map((each) => each.uri.toFilePath()),
-          librariesWeGot);
-      var librariesToDocument = librariesWeAskedFor.map(
-          (each) => librariesWeGotByPath.putIfAbsent(each,
+      var availableLibrariesByPath = new Map.fromIterables(
+          availableLibraries.map((each) => each.uri.toFilePath()),
+          availableLibraries);
+      var librariesToDocument = requestedLibraries.map(
+          (each) => availableLibrariesByPath.putIfAbsent(each,
               () => throw "Missing library $each")).toList();
       librariesToDocument.addAll((includeSdk || parseSdk) ? _sdkLibraries : []);
+      librariesToDocument.removeWhere((x) => _excluded.contains(x.simpleName));
       _documentLibraries(librariesToDocument, includeSdk: includeSdk,
           outputToYaml: outputToYaml, append: append, parseSdk: parseSdk,
           introduction: introduction);
@@ -209,7 +228,7 @@ List<String> _listDartFromDir(String args) {
       }
     }
   });
-  return libraries;
+  return libraries.map(path.absolute).map(path.normalize).toList();
 }
 
 String _findPackageRoot(String directory) {
@@ -245,9 +264,8 @@ List<String> _listSdk() {
 
 /// Analyzes set of libraries by getting a mirror system and triggers the
 /// documentation of the libraries.
-Future<MirrorSystem> getMirrorSystem(List<String> args, {String packageRoot,
-    bool parseSdk: false}) {
-  var libraries = !parseSdk ? _listLibraries(args) : _listSdk();
+Future<MirrorSystem> getMirrorSystem(List<String> libraries,
+    {String packageRoot, bool parseSdk: false}) {
   if (libraries.isEmpty) throw new StateError('No Libraries.');
   // Finds the root of SDK library based off the location of docgen.
 
@@ -323,12 +341,12 @@ void _documentLibraries(List<LibraryMirror> libs, {bool includeSdk: false,
   var libraryMap;
   var linkResolver = (name) => fixReference(name, null, null, null);
   if (append) {
-    var docsDir = listDir('docs');
-    if (!docsDir.contains('docs/library_list.json')) {
+    var docsDir = listDir(_outputDirectory);
+    if (!docsDir.contains('$_outputDirectory/library_list.json')) {
       throw new StateError('No library_list.json');
     }
     libraryMap =
-        JSON.decode(new File('docs/library_list.json').readAsStringSync());
+        JSON.decode(new File('$_outputDirectory/library_list.json').readAsStringSync());
     libraryMap['libraries'].addAll(filteredEntities
         .where((e) => e is Library)
         .map((e) => e.previewMap));
@@ -369,7 +387,7 @@ void _documentLibraries(List<LibraryMirror> libs, {bool includeSdk: false,
       filteredEntities.map((e) => e.typeName));
   if (append) {
     var previousIndex =
-        JSON.decode(new File('docs/index.json').readAsStringSync());
+        JSON.decode(new File('$_outputDirectory/index.json').readAsStringSync());
     index.addAll(previousIndex);
   }
   _writeToFile(JSON.encode(index), 'index.json');
@@ -771,24 +789,21 @@ List<Type> _typeGenerics(TypeMirror mirror) {
   return [];
 }
 
-/// Writes text to a file in the 'docs' directory.
+/// Writes text to a file in the output directory.
 void _writeToFile(String text, String filename, {bool append: false}) {
-  Directory dir = new Directory('docs');
+  if (text == null) return;
+  Directory dir = new Directory(_outputDirectory);
   if (!dir.existsSync()) {
     dir.createSync();
   }
   // We assume there's a single extra level of directory structure for packages.
   if (path.split(filename).length > 1) {
-    var subdir = new Directory(path.join('docs', path.dirname(filename)));
+    var subdir = new Directory(path.join(_outputDirectory, path.dirname(filename)));
     if (!subdir.existsSync()) {
       subdir.createSync();
     }
   }
-
-  File file = new File('docs/$filename');
-  if (!file.existsSync()) {
-    file.createSync();
-  }
+  File file = new File(path.join(_outputDirectory, filename));
   file.writeAsStringSync(text, mode: append ? FileMode.APPEND : FileMode.WRITE);
 }
 

tools/bots/pub.py

@@ -35,12 +35,6 @@ def PubConfig(name, is_buildbot):
 
 
 def PubSteps(build_info):
-  with bot.BuildStep('Build API Docs'):
-    args = [sys.executable, './tools/build.py', '--mode=' + build_info.mode,
-            'api_docs']
-    print 'Generating API Docs: %s' % (' '.join(args))
-    bot.RunProcess(args)
-
   with bot.BuildStep('Build package-root'):
     args = [sys.executable, './tools/build.py', '--mode=' + build_info.mode,
             'packages']

utils/apidoc/docgen.gyp

@@ -0,0 +1,97 @@
+# Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+{
+  'variables' : {
+    'script_suffix%': '',
+  },
+  'conditions' : [
+    ['OS=="win"', {
+      'variables' : {
+        'script_suffix': '.bat',
+      },
+    }],
+  ],
+  'targets': [
+    {
+      'target_name': 'docgen',
+      'type': 'none',
+      'dependencies': [
+        '../../utils/compiler/compiler.gyp:dart2js',
+        '../../runtime/dart-runtime.gyp:dart',
+        '../../pkg/pkg.gyp:pkg_packages',
+        'apidoc.gyp:api_docs',
+      ],
+      'includes': [
+        '../../sdk/lib/core/corelib_sources.gypi',
+      ],
+      'actions': [
+        {
+          'action_name': 'run_docgen',
+          # The 'inputs' list records the files whose timestamps are
+          # compared to the files listed in 'outputs'.  If a file
+          # 'outputs' doesn't exist or if a file in 'inputs' is newer
+          # than a file in 'outputs', this action is executed.  Notice
+          # that the dependencies listed above has nothing to do with
+          # when this action is executed.  You must list a file in
+          # 'inputs' to make sure that it exists before the action is
+          # executed, or to make sure this action is re-run.
+          #
+          # We want to build the platform documentation whenever
+          # dartdoc, apidoc, or its dependency changes.  This prevents
+          # people from accidentally breaking apidoc when making
+          # changes to the platform libraries and or when modifying
+          # dart2js or the VM.
+          #
+          # In addition, we want to make sure that the platform
+          # documentation is regenerated when the platform sources
+          # changes.
+          #
+          # So we want this action to be re-run when a dart file
+          # changes in this directory, or in the SDK library (we may
+          # no longer need to list the files in ../../runtime/lib and
+          # ../../runtime/bin, as most of them has moved to
+          # ../../sdk/lib).
+          #
+          'inputs': [
+            '<(PRODUCT_DIR)/<(EXECUTABLE_PREFIX)dart<(EXECUTABLE_SUFFIX)',
+            '<(SHARED_INTERMEDIATE_DIR)/utils_wrapper.dart.snapshot',
+            '<!@(["python", "../../tools/list_files.py", "\\.(css|ico|js|json|png|sh|txt|yaml|py)$", ".", "../../sdk/lib/_internal/dartdoc"])',
+            '<!@(["python", "../../tools/list_files.py", "\\.dart$", ".", "../../sdk/lib", "../../runtime/lib", "../../runtime/bin"])',
+            '../../sdk/bin/dart',
+            '../../sdk/bin/dart.bat',
+            '../../sdk/bin/dart2js',
+            '../../sdk/bin/dart2js.bat',
+            '../../tools/only_in_release_mode.py',
+            # We sit inside the api_docs directory, so make sure it has run
+            # before we do. Otherwise it might run later and delete us.
+            '<(PRODUCT_DIR)/api_docs/index.html',
+          ],
+          'outputs': [
+            '<(PRODUCT_DIR)/api_docs/docgen/index.json',
+          ],
+          'action': [
+            'python',
+            '../../tools/only_in_release_mode.py',
+            '<@(_outputs)',
+            '--',
+            '<(PRODUCT_DIR)/<(EXECUTABLE_PREFIX)dart<(EXECUTABLE_SUFFIX)',
+            '--old_gen_heap_size=1024',
+            '--package-root=<(PRODUCT_DIR)/packages/',
+            '../../pkg/docgen/bin/docgen.dart',
+            '--out=<(PRODUCT_DIR)/api_docs/docgen',
+            '--json',
+            '--include-sdk',
+            '--package-root=<(PRODUCT_DIR)/packages',
+            '--exclude-lib=async_helper',
+            '--exclude-lib=expect',
+            '--exclude-lib=docgen',
+            '../../pkg',          
+          ],
+          'message': 'Running docgen: <(_action)',
+        },
+      ],
+    }
+  ],
+}