add previous token field

This adds fasta Token.previous and sets that field to point
to the previous token in the fasta token stream.

R=paulberry@google.com

Review-Url: https://codereview.chromium.org/2739583002 .

pkg/front_end/lib/src/fasta/analyzer/token_utils.dart

@@ -282,6 +282,7 @@ Token fromAnalyzerTokenStream(analyzer.Token analyzerToken) {
     token = token.next;
     while (token != null) {
       tail.next = fromAnalyzerToken(token);
+      tail.next.previousToken = tail;
       tail = tail.next;
       token = token.next;
     }
@@ -293,6 +294,7 @@ Token fromAnalyzerTokenStream(analyzer.Token analyzerToken) {
     token.precedingComments =
         translateComments(analyzerToken.precedingComments);
     tokenTail.next = token;
+    tokenTail.next.previousToken = tokenTail;
     tokenTail = token;
     matchGroups(analyzerToken, token);
     return analyzerToken.next;
@@ -302,6 +304,7 @@ Token fromAnalyzerTokenStream(analyzer.Token analyzerToken) {
     // TODO(paulberry): join up begingroup/endgroup.
     if (analyzerToken.type == TokenType.EOF) {
       tokenTail.next = new SymbolToken(EOF_INFO, analyzerToken.offset);
+      tokenTail.next.previousToken = tokenTail;
       tokenTail.next.precedingComments =
           translateComments(analyzerToken.precedingComments);
       return tokenHead.next;

pkg/front_end/lib/src/fasta/scanner/array_based_scanner.dart

@@ -40,6 +40,7 @@ abstract class ArrayBasedScanner extends AbstractScanner {
    */
   void appendToken(Token token) {
     tail.next = token;
+    tail.next.previousToken = tail;
     tail = tail.next;
     if (comments != null) {
       tail.precedingComments = comments;
@@ -225,6 +226,7 @@ abstract class ArrayBasedScanner extends AbstractScanner {
       commentsTail = comments;
     } else {
       commentsTail.next = newComment;
+      commentsTail.next.previousToken = commentsTail;
       commentsTail = commentsTail.next;
     }
   }

pkg/front_end/lib/src/fasta/scanner/token.dart

@@ -28,6 +28,15 @@ abstract class Token {
    */
   Token next;
 
+  /**
+   * The previous token in the token stream.
+   *
+   * Deprecated :: This exists for compatibility with the Analyzer token stream
+   * and will be removed at some future date.
+   */
+  @deprecated
+  Token previousToken;
+
   /**
    * Return the first comment in the list of comments that precede this token,
    * or `null` if there are no comments preceding this token. Additional

pkg/front_end/test/scanner_fasta_test.dart

@@ -194,6 +194,23 @@ class ScannerTest_Fasta extends ScannerTestBase {
     super.test_scriptTag_withArgs();
   }
 
+  void test_next_previous() {
+    const source = 'int a; /*1*/ /*2*/ /*3*/ B f(){if (a < 2) {}}';
+    fasta.Token token =
+        new fasta.StringScanner(source, includeComments: true).tokenize();
+    while (!token.isEof) {
+      expect(token.next.previousToken, token);
+      fasta.Token commentToken = token.precedingComments;
+      while (commentToken != null) {
+        if (commentToken.next != null) {
+          expect(commentToken.next.previousToken, commentToken);
+        }
+        commentToken = commentToken.next;
+      }
+      token = token.next;
+    }
+  }
+
   @override
   @failingTest
   void test_scriptTag_withoutSpace() {

pkg/front_end/tool/fasta/compile.dart

@@ -2,8 +2,6 @@
 // for details. All rights reserved. Use of this source code is governed by a
 // BSD-style license that can be found in the LICENSE file.
 
-import 'dart:math';
-
 import 'package:front_end/src/fasta/outline.dart' as outline;
 
 import 'standard_deviation.dart';