Tokenize one line at a time.

R=kasperl@google.com

Review URL: https://codereview.chromium.org//225903003

git-svn-id: https://dart.googlecode.com/svn/branches/bleeding_edge/dart@35688 260f80e4-7a28-3924-810f-c04153c831b5

site/try/build_try.gyp

@@ -25,6 +25,7 @@
         'try_dart_static_files': [
           'index.html',
           'dartlang-style.css',
+          'line_numbers.css',
           'iframe.html',
           'iframe.js',
           'dart-icon.png', # iOS icon.

site/try/index.html

@@ -16,6 +16,7 @@ See: http://www.google.com/fonts#UsePlace:use/Collection:Open+Sans:400,600,700,8
 -->
 <link href='http://fonts.googleapis.com/css?family=Open+Sans:400,600,700,800,300' rel='stylesheet' type='text/css'>
 <link rel="stylesheet" type="text/css" href="dartlang-style.css">
+<link rel="alternate stylesheet" type="text/css" href="line_numbers.css" title="line_numbers">
 <style>
 a.diagnostic {
   /* position: relative; */
@@ -33,7 +34,7 @@ a:hover.diagnostic span {
   position: absolute;
   /* left: 1em; */
   /* top: 2em; */
-  right: 10px;
+  left: 10px;
 }
 
 .offline {
@@ -148,7 +149,16 @@ a:hover.diagnostic span {
   height: auto;
 }
 
+.mainEditorPane {
+  overflow-y: auto;
+  overflow-x: hidden;
+  white-space: pre;
+  max-height: 80vh;
+}
 
+.lineNumber {
+  position: relative;
+}
 </style>
 
 <meta itemprop="name" content="Try Dart!">

site/try/line_numbers.css

@@ -0,0 +1,26 @@
+/* Copyright (c) 2014, the Dart project authors.  Please see the AUTHORS file
+ * for details. All rights reserved. Use of this source code is governed by a
+ * BSD-style license that can be found in the LICENSE file.
+ */
+
+.mainEditorPane {
+  padding: 0px;
+}
+
+div {
+  counter-reset: dart-line-number;
+}
+
+.lineNumber {
+  padding-left: 3.5em;
+}
+
+.lineNumber:before {
+  counter-increment: dart-line-number;
+  content: counter(dart-line-number) " ";
+  position: absolute;
+  left: 0px;
+  width: 3em;
+  text-align: right;
+  background-color: lightgoldenrodyellow;
+}

site/try/nossl.appcache

@@ -8,6 +8,7 @@ CACHE MANIFEST
 CACHE:
 index.html
 dartlang-style.css
+line_numbers.css
 leap.dart.js
 
 iframe.html

site/try/project_server.dart

@@ -7,7 +7,8 @@ library trydart.projectServer;
 import 'dart:io';
 
 import 'dart:async' show
-    Future;
+    Future,
+    Stream;
 
 import 'dart:convert' show
     HtmlEscape,
@@ -145,12 +146,6 @@ It is safe to delete tag '$GIT_TAG' if you don't need the backup.""";
     response.close();
   }
 
-  redirect(String location) {
-    response.statusCode = HttpStatus.FOUND;
-    response.headers.add(HttpHeaders.LOCATION, location);
-    response.close();
-  }
-
   badRequest(String problem) {
     response.statusCode = HttpStatus.BAD_REQUEST;
     response.write(htmlInfo("Bad request",

site/try/src/interaction_manager.dart

@@ -21,7 +21,9 @@ import 'package:compiler/implementation/scanner/scannerlib.dart' show
     EOF_TOKEN,
     ErrorToken,
     StringScanner,
-    Token;
+    Token,
+    UnmatchedToken,
+    UnterminatedToken;
 
 import 'package:compiler/implementation/source_file.dart' show
     StringSourceFile;
@@ -274,7 +276,16 @@ class InitialState extends InteractionState {
     int offset = 0;
     List<Node> nodes = <Node>[];
 
-    tokenizeAndHighlight(currentText, offset, trySelection, nodes);
+    String state = '';
+    for (String line in currentText.split(new RegExp('^', multiLine: true))) {
+      List<Node> lineNodes = <Node>[];
+      state =
+          tokenizeAndHighlight(line, state, offset, trySelection, lineNodes);
+      offset += line.length;
+      nodes.add(new SpanElement()
+          ..nodes.addAll(lineNodes)
+          ..classes.add('lineNumber'));
+    }
 
     mainEditorPane
         ..nodes.clear()
@@ -700,50 +711,77 @@ bool computeHasModifier(KeyboardEvent event) {
       event.getModifierState("OS");
 }
 
-void tokenizeAndHighlight(String currentText,
-                          int offset,
-                          TrySelection trySelection,
-                          List<Node> nodes) {
+String tokenizeAndHighlight(String line,
+                            String state,
+                            int start,
+                            TrySelection trySelection,
+                            List<Node> nodes) {
+  String newState = '';
+  int offset = state.length;
+  int adjustedStart = start - state.length;
+
   //   + offset  + charOffset  + globalOffset   + (charOffset + charCount)
   //   v         v             v                v
   // do          identifier_abcdefghijklmnopqrst
-  for (Token token = tokenize(currentText);
+  for (Token token = tokenize('$state$line');
        token.kind != EOF_TOKEN;
        token = token.next) {
     int charOffset = token.charOffset;
     int charCount = token.charCount;
 
-    if (charOffset < offset) continue; // Happens for scanner errors.
-
-    Decoration decoration = editor.getDecoration(token);
-
-    Token follow = token.next;
-    if (token is BeginGroupToken) {
-      follow = token.endGroup.next;
+    Token tokenToDecorate = token;
+    if (token is UnterminatedToken && isUnterminatedMultiLineToken(token)) {
+      newState += '${token.start}';
+      continue; // This might not be an error.
+    } else {
+      Token follow = token.next;
+      if (token is BeginGroupToken && token.endGroup != null) {
+        follow = token.endGroup.next;
+      }
+      if (follow is ErrorToken && follow.charOffset == token.charOffset) {
+        if (follow is UnmatchedToken) {
+          newState += '${follow.begin.value}';
+        } else {
+          tokenToDecorate = follow;
+        }
+      }
     }
-    if (follow is ErrorToken) {
-      decoration = editor.getDecoration(follow);
+
+    if (charOffset < offset) {
+      // Happens for scanner errors, or for the [state] prefix.
+      continue;
     }
 
+    Decoration decoration = editor.getDecoration(tokenToDecorate);
+
     if (decoration == null) continue;
 
     // Add a node for text before current token.
-    trySelection.addNodeFromSubstring(offset, charOffset, nodes);
+    trySelection.addNodeFromSubstring(
+        adjustedStart + offset, adjustedStart + charOffset, nodes);
 
     // Add a node for current token.
     trySelection.addNodeFromSubstring(
-        charOffset, charOffset + charCount, nodes, decoration);
+        adjustedStart + charOffset,
+        adjustedStart + charOffset + charCount, nodes, decoration);
 
     offset = charOffset + charCount;
   }
 
   // Add a node for anything after the last (decorated) token.
-  trySelection.addNodeFromSubstring(offset, currentText.length, nodes);
+  trySelection.addNodeFromSubstring(
+      adjustedStart + offset, start + line.length, nodes);
 
-  // Ensure text always ends with a newline.
-  if (!currentText.endsWith('\n')) {
-    nodes.add(new Text('\n'));
-  }
+  return newState;
+}
+
+bool isUnterminatedMultiLineToken(UnterminatedToken token) {
+  return
+      token.start == '/*' ||
+      token.start == "'''" ||
+      token.start == '"""' ||
+      token.start == "r'''" ||
+      token.start == 'r"""';
 }
 
 void normalizeMutationRecord(MutationRecord record, TrySelection selection) {

site/try/src/ui.dart

@@ -104,11 +104,9 @@ buildUI() {
   buildCode(interaction);
 
   (mainEditorPane = new DivElement())
-      ..classes.add('well')
+      ..classes.addAll(['well', 'mainEditorPane'])
       ..style.backgroundColor = currentTheme.background.color
       ..style.color = currentTheme.foreground.color
-      ..style.overflow = 'visible'
-      ..style.whiteSpace = 'pre'
       ..style.font = codeFont
       ..spellcheck = false;
 

tests/try/paste_content_rewriting_test.dart

@@ -23,13 +23,16 @@ import '../../pkg/expect/lib/expect.dart';
 import '../../pkg/async_helper/lib/async_helper.dart';
 
 const Map<String, String> tests = const <String, String> {
-  '<span><p>//...</p>}</span>': '//...\n}\n',
-  'someText': 'someText\n',
-  '"\$"': '"<DIAGNOSTIC>\$</DIAGNOSTIC>"\n',
-  '"\$\$"': '"<DIAGNOSTIC>\$</DIAGNOSTIC><DIAGNOSTIC>\$</DIAGNOSTIC>"\n',
-  '"\$\$4"': '"<DIAGNOSTIC>\$</DIAGNOSTIC><DIAGNOSTIC>\$</DIAGNOSTIC>4"\n',
-  '"\$\$4 "': '"<DIAGNOSTIC>\$</DIAGNOSTIC><DIAGNOSTIC>\$</DIAGNOSTIC>4 "\n',
-  '1e': '<DIAGNOSTIC>1e</DIAGNOSTIC>\n',
+  '<span><p>//...</p>}</span>': '//...\n}',
+  'someText': 'someText',
+  '"\$"': '"<DIAGNOSTIC>\$</DIAGNOSTIC>"',
+  '"\$\$"': '"<DIAGNOSTIC>\$</DIAGNOSTIC><DIAGNOSTIC>\$</DIAGNOSTIC>"',
+  '"\$\$4"': '"<DIAGNOSTIC>\$</DIAGNOSTIC><DIAGNOSTIC>\$</DIAGNOSTIC>4"',
+  '"\$\$4 "': '"<DIAGNOSTIC>\$</DIAGNOSTIC><DIAGNOSTIC>\$</DIAGNOSTIC>4 "',
+  '1e': '<DIAGNOSTIC>1e</DIAGNOSTIC>',
+  'r"""\n\n\'"""': 'r"""\n\n\'"""',
+  '"': '<DIAGNOSTIC>"</DIAGNOSTIC>',
+  '/**\n*/': '/**\n*/',
 };
 
 List<Node> queryDiagnosticNodes() {
@@ -76,7 +79,5 @@ void main() {
       ..observe(
           mainEditorPane, childList: true, characterData: true, subtree: true);
 
-  mainEditorPane.innerHtml = "<span><p>//...</p>}</span>";
-
   asyncTest(runTests);
 }