Add preview-dart-2 flag to analyzer cli and plumb it through test.py.

This allows us to collect stats about language-style tests as follows:

 ./tools/test.py -m release -c dart2analyzer -r none --preview-dart-2 --report \
      language language_2 corelib corelib_2

As of this morning, the command above has 87% of the tests passing.

Change-Id: I0f84211d5f33659d5923601267367be90c578107
Reviewed-on: https://dart-review.googlesource.com/3381
Reviewed-by: William Hesse <whesse@google.com>
Reviewed-by: Brian Wilkerson <brianwilkerson@google.com>
Reviewed-by: Bob Nystrom <rnystrom@google.com>
Reviewed-by: Dan Rubel <danrubel@google.com>

pkg/analyzer/lib/src/context/context.dart

@@ -279,6 +279,7 @@ class AnalysisContextImpl implements InternalAnalysisContext {
         _notEqual(this._options.lintRules, options.lintRules) ||
         this._options.preserveComments != options.preserveComments ||
         this._options.strongMode != options.strongMode ||
+        this._options.useFastaParser != options.useFastaParser ||
         this._options.enableAssertInitializer !=
             options.enableAssertInitializer ||
         this._options.enableLazyAssignmentOperators !=
@@ -324,6 +325,7 @@ class AnalysisContextImpl implements InternalAnalysisContext {
       _typeSystem = null;
     }
     this._options.strongMode = options.strongMode;
+    this._options.useFastaParser = options.useFastaParser;
     this._options.trackCacheDependencies = options.trackCacheDependencies;
     this._options.disableCacheFlushing = options.disableCacheFlushing;
     this._options.patchPaths = options.patchPaths;

pkg/analyzer/lib/src/dart/analysis/file_state.dart

@@ -14,8 +14,6 @@ import 'package:analyzer/src/dart/analysis/referenced_names.dart';
 import 'package:analyzer/src/dart/analysis/top_level_declaration.dart';
 import 'package:analyzer/src/dart/scanner/reader.dart';
 import 'package:analyzer/src/dart/scanner/scanner.dart';
-import 'package:analyzer/src/fasta/ast_builder.dart' as fasta;
-import 'package:analyzer/src/fasta/mock_element.dart' as fasta;
 import 'package:analyzer/src/generated/engine.dart';
 import 'package:analyzer/src/generated/parser.dart';
 import 'package:analyzer/src/generated/source.dart';
@@ -31,9 +29,6 @@ import 'package:crypto/crypto.dart';
 import 'package:front_end/byte_store.dart';
 import 'package:front_end/src/base/api_signature.dart';
 import 'package:front_end/src/base/performace_logger.dart';
-import 'package:front_end/src/fasta/builder/builder.dart' as fasta;
-import 'package:front_end/src/fasta/parser/parser.dart' as fasta;
-import 'package:front_end/src/fasta/scanner.dart' as fasta;
 import 'package:front_end/src/fasta/scanner/token.dart';
 import 'package:meta/meta.dart';
 
@@ -80,8 +75,6 @@ class FileContentOverlay {
  * should be called.
  */
 class FileState {
-  static const bool USE_FASTA_PARSER = false;
-
   final FileSystemState _fsState;
 
   /**
@@ -108,7 +101,6 @@ class FileState {
   final bool isInExternalSummaries;
 
   bool _exists;
-  List<int> _contentBytes;
   String _content;
   String _contentHash;
   LineInfo _lineInfo;
@@ -413,13 +405,6 @@ class FileState {
       _exists = false;
     }
 
-    if (USE_FASTA_PARSER) {
-      var bytes = UTF8.encode(_content);
-      _contentBytes = new Uint8List(bytes.length + 1);
-      _contentBytes.setRange(0, bytes.length, bytes);
-      _contentBytes[_contentBytes.length - 1] = 0;
-    }
-
     // Compute the content hash.
     List<int> contentBytes = UTF8.encode(_content);
     {
@@ -571,61 +556,26 @@ class FileState {
 
   CompilationUnit _parse(AnalysisErrorListener errorListener) {
     AnalysisOptions analysisOptions = _fsState._analysisOptions;
+    CharSequenceReader reader = new CharSequenceReader(content);
+    Scanner scanner = new Scanner(source, reader, errorListener);
+    scanner.scanGenericMethodComments = analysisOptions.strongMode;
+    Token token = PerformanceStatistics.scan.makeCurrentWhile(() {
+      return scanner.tokenize();
+    });
+    LineInfo lineInfo = new LineInfo(scanner.lineStarts);
 
-    if (USE_FASTA_PARSER) {
-      try {
-        fasta.ScannerResult scanResult =
-            PerformanceStatistics.scan.makeCurrentWhile(() {
-          return fasta.scan(
-            _contentBytes,
-            includeComments: true,
-            scanGenericMethodComments: analysisOptions.strongMode,
-          );
-        });
+    bool useFasta = analysisOptions.useFastaParser;
+    Parser parser = new Parser(source, errorListener, useFasta: useFasta);
+    parser.enableAssertInitializer = analysisOptions.enableAssertInitializer;
+    parser.parseGenericMethodComments = analysisOptions.strongMode;
+    CompilationUnit unit = parser.parseCompilationUnit(token);
+    unit.lineInfo = lineInfo;
 
-        var astBuilder = new fasta.AstBuilder(
-            new ErrorReporter(errorListener, source),
-            null,
-            null,
-            new fasta.Scope.top(isModifiable: true),
-            true,
-            uri);
-        astBuilder.parseGenericMethodComments = analysisOptions.strongMode;
+    // StringToken uses a static instance of StringCanonicalizer, so we need
+    // to clear it explicitly once we are done using it for this file.
+    StringToken.canonicalizer.clear();
 
-        var parser = new fasta.Parser(astBuilder);
-        astBuilder.parser = parser;
-        parser.parseUnit(scanResult.tokens);
-        var unit = astBuilder.pop() as CompilationUnit;
-
-        LineInfo lineInfo = new LineInfo(scanResult.lineStarts);
-        unit.lineInfo = lineInfo;
-        return unit;
-      } catch (e, st) {
-        print(e);
-        print(st);
-        rethrow;
-      }
-    } else {
-      CharSequenceReader reader = new CharSequenceReader(content);
-      Scanner scanner = new Scanner(source, reader, errorListener);
-      scanner.scanGenericMethodComments = analysisOptions.strongMode;
-      Token token = PerformanceStatistics.scan.makeCurrentWhile(() {
-        return scanner.tokenize();
-      });
-      LineInfo lineInfo = new LineInfo(scanner.lineStarts);
-
-      Parser parser = new Parser(source, errorListener);
-      parser.enableAssertInitializer = analysisOptions.enableAssertInitializer;
-      parser.parseGenericMethodComments = analysisOptions.strongMode;
-      CompilationUnit unit = parser.parseCompilationUnit(token);
-      unit.lineInfo = lineInfo;
-
-      // StringToken uses a static instance of StringCanonicalizer, so we need
-      // to clear it explicitly once we are done using it for this file.
-      StringToken.canonicalizer.clear();
-
-      return unit;
-    }
+    return unit;
   }
 
   /**

pkg/analyzer/lib/src/generated/engine.dart

@@ -1310,6 +1310,11 @@ abstract class AnalysisOptions {
    */
   bool get trackCacheDependencies;
 
+  /**
+   * Return `true` if analyzer should use the Dart 2.0 Front End parser.
+   */
+  bool get useFastaParser;
+
   /**
    * Reset the state of this set of analysis options to its original state.
    */
@@ -1454,6 +1459,9 @@ class AnalysisOptionsImpl implements AnalysisOptions {
   @override
   bool trackCacheDependencies = true;
 
+  @override
+  bool useFastaParser = false;
+
   @override
   bool disableCacheFlushing = false;
 
@@ -1511,6 +1519,7 @@ class AnalysisOptionsImpl implements AnalysisOptions {
     lintRules = options.lintRules;
     preserveComments = options.preserveComments;
     strongMode = options.strongMode;
+    useFastaParser = options.useFastaParser;
     if (options is AnalysisOptionsImpl) {
       declarationCasts = options.declarationCasts;
       strongModeHints = options.strongModeHints;
@@ -1692,6 +1701,7 @@ class AnalysisOptionsImpl implements AnalysisOptions {
     strongMode = false;
     strongModeHints = false;
     trackCacheDependencies = true;
+    useFastaParser = false;
   }
 
   @override

pkg/analyzer/lib/src/generated/parser_fasta.dart

@@ -66,6 +66,15 @@ class _Parser2 implements Parser {
     _astBuilder.parseGenericMethodComments = value;
   }
 
+  @override
+  bool enableAssertInitializer = true;
+
+  @override
+  bool enableUriInPartOf = true;
+
+  @override
+  bool enableNnbd = false;
+
   @override
   CompilationUnit parseCompilationUnit(Token token) {
     currentToken = token;

pkg/analyzer/lib/src/summary/summary_file_builder.dart

@@ -138,7 +138,8 @@ class _Builder {
     scanner.scanGenericMethodComments = strong;
     Token token = scanner.tokenize();
     LineInfo lineInfo = new LineInfo(scanner.lineStarts);
-    Parser parser = new Parser(source, errorListener);
+    Parser parser = new Parser(source, errorListener,
+        useFasta: context.analysisOptions.useFastaParser);
     parser.parseGenericMethodComments = strong;
     CompilationUnit unit = parser.parseCompilationUnit(token);
     unit.lineInfo = lineInfo;

pkg/analyzer/lib/src/task/dart.dart

@@ -3769,8 +3769,9 @@ class ParseDartTask extends SourceBasedAnalysisTask {
     RecordingErrorListener errorListener = new RecordingErrorListener();
     _errorReporter = new ErrorReporter(errorListener, _source);
 
-    Parser parser = new Parser(_source, errorListener);
     AnalysisOptions options = context.analysisOptions;
+    Parser parser =
+        new Parser(_source, errorListener, useFasta: options.useFastaParser);
     parser.enableAssertInitializer = options.enableAssertInitializer;
     parser.parseFunctionBodies =
         options.analyzeFunctionBodiesPredicate(_source);

pkg/analyzer_cli/lib/src/driver.dart

@@ -866,6 +866,9 @@ class Driver implements CommandLineStarter {
     if (options.enableAssertInitializer != null) {
       contextOptions.enableAssertInitializer = options.enableAssertInitializer;
     }
+    if (options.previewDart2) {
+      contextOptions.useFastaParser = true;
+    }
 
     _directoryToAnalysisOptions[contextRootDirectory] = contextOptions;
     return contextOptions;

pkg/analyzer_cli/lib/src/options.dart

@@ -114,6 +114,9 @@ class CommandLineOptions {
   /// (Or null if not enabled.)
   final String perfReport;
 
+  /// Whether to enable the Dart 2.0 Front End.
+  final bool previewDart2;
+
   /// Batch mode (for unit testing)
   final bool batchMode;
 
@@ -186,6 +189,7 @@ class CommandLineOptions {
         log = args['log'],
         machineFormat = args['format'] == 'machine',
         perfReport = args['x-perf-report'],
+        previewDart2 = args['preview-dart-2'],
         batchMode = args['batch'],
         showPackageWarnings = args['show-package-warnings'] ||
             args['package-warnings'] ||
@@ -506,6 +510,11 @@ class CommandLineOptions {
               'of "libraryUri".',
           allowMultiple: true,
           splitCommas: false,
+          hide: hide)
+      ..addFlag('preview-dart-2',
+          help: 'Enable the Dart 2.0 Front End implementation.',
+          defaultsTo: false,
+          negatable: false,
           hide: hide);
 
     try {

pkg/analyzer_cli/test/driver_test.dart

@@ -751,6 +751,12 @@ class OptionsTest extends BaseTest {
     expect(outSink.toString(), contains('Avoid empty else statements'));
   }
 
+  test_previewDart2() async {
+    await drive('data/options_tests_project/test_file.dart',
+        args: ['--preview-dart-2']);
+    expect(driver.context.analysisOptions.useFastaParser, isTrue);
+  }
+
   test_strongSdk() async {
     String testDir = path.join(testDirectory, 'data', 'strong_sdk');
     await drive(path.join(testDir, 'main.dart'), args: ['--strong']);

pkg/analyzer_cli/test/options_test.dart

@@ -76,6 +76,7 @@ main() {
         expect(options.warningsAreFatal, isFalse);
         expect(options.strongMode, isFalse);
         expect(options.lintsAreFatal, isFalse);
+        expect(options.previewDart2, isFalse);
       });
 
       test('batch', () {
@@ -256,6 +257,12 @@ main() {
               outStringBuffer.toString(), contains('Analytics are currently'));
         });
       }
+
+      test('preview FE', () {
+        CommandLineOptions options =
+            CommandLineOptions.parse(['--preview-dart-2', 'foo.dart']);
+        expect(options.previewDart2, isTrue);
+      });
     });
   });
   defineReflectiveTests(CommandLineOptionsTest);

tools/testing/dart/configuration.dart

@@ -39,6 +39,7 @@ class Configuration {
       this.isMinified,
       this.isVerbose,
       this.listTests,
+      this.previewDart2,
       this.printTiming,
       this.printReport,
       this.reportInJson,
@@ -104,6 +105,7 @@ class Configuration {
   final bool isMinified;
   final bool isVerbose;
   final bool listTests;
+  final bool previewDart2;
   final bool printTiming;
   final bool printReport;
   final bool reportInJson;
@@ -447,7 +449,8 @@ class Configuration {
         'use_sdk': useSdk,
         'builder_tag': builderTag,
         'fast_startup': useFastStartup,
-        'timeout': timeout
+        'timeout': timeout,
+        'preview_dart_2': previewDart2
       };
     }
     return _summaryMap;

tools/testing/dart/options.dart

@@ -177,6 +177,9 @@ simdbc, simdbc64''',
         'fast_startup', 'Pass the --fast-startup flag to dart2js.'),
     new _Option.bool('enable_asserts',
         'Pass the --enable-asserts flag to dart2js or to the vm.'),
+    new _Option.bool(
+        'preview_dart_2', 'Pass the --preview-dart-2 flag to analyzer.'),
+    // TODO(sigmund): replace dart2js_with_kernel with preview-dart-2.
     new _Option.bool(
         'dart2js_with_kernel', 'Pass the --use-kernel flag to dart2js.'),
     new _Option.bool('dart2js_with_kernel_in_ssa',
@@ -594,6 +597,7 @@ compiler.''')
                 isMinified: data["minified"] as bool,
                 isVerbose: data["verbose"] as bool,
                 listTests: data["list"] as bool,
+                previewDart2: data["preview_dart_2"] as bool,
                 printTiming: data["time"] as bool,
                 printReport: data["report"] as bool,
                 reportInJson: data["report_in_json"] as bool,

tools/testing/dart/test_suite.dart

@@ -1276,6 +1276,7 @@ class StandardTestSuite extends TestSuite {
     if (configuration.compiler == Compiler.dart2analyzer) {
       args.add('--format=machine');
       args.add('--no-hints');
+      if (configuration.previewDart2) args.add("--preview-dart-2");
 
       if (filePath.filename.contains("dart2js") ||
           filePath.directoryPath.segments().last.contains('html_common')) {