Compute accurate token count when loading from snapshot

With --compiler-stats, traverse heap at isolate shutdown to
find all token streams and compute the number of tokens in
the stream. This fixes incorrect token counts when loading
from snapshot.

R=asiva@google.com

Review URL: https://codereview.chromium.org//1131493007

git-svn-id: https://dart.googlecode.com/svn/branches/bleeding_edge/dart@45806 260f80e4-7a28-3924-810f-c04153c831b5

runtime/vm/compiler_stats.cc

@@ -5,6 +5,7 @@
 #include "vm/compiler_stats.h"
 
 #include "vm/flags.h"
+#include "vm/object_graph.h"
 #include "vm/timer.h"
 
 
@@ -13,6 +14,43 @@ namespace dart {
 DEFINE_FLAG(bool, compiler_stats, false, "Compiler stat counters.");
 
 
+class TokenStreamVisitor : public ObjectGraph::Visitor {
+ public:
+  explicit TokenStreamVisitor(CompilerStats* compiler_stats)
+      : obj_(Object::Handle()), stats_(compiler_stats) {
+  }
+
+  virtual Direction VisitObject(ObjectGraph::StackIterator* it) {
+    RawObject* raw_obj = it->Get();
+    if (raw_obj->IsFreeListElement()) {
+      return kProceed;
+    }
+    obj_ = raw_obj;
+    if (obj_.GetClassId() == TokenStream::kClassId) {
+      TokenStream::Iterator tkit(TokenStream::Cast(obj_),
+                                 0,
+                                 TokenStream::Iterator::kNoNewlines);
+      Token::Kind kind = tkit.CurrentTokenKind();
+      while (kind != Token::kEOS) {
+        ++stats_->num_tokens_total;
+        if (kind == Token::kIDENT) {
+          ++stats_->num_ident_tokens_total;
+        } else if (Token::NeedsLiteralToken(kind)) {
+          ++stats_->num_literal_tokens_total;
+        }
+        tkit.Advance();
+        kind = tkit.CurrentTokenKind();
+      }
+    }
+    return kProceed;
+  }
+
+ private:
+  Object& obj_;
+  CompilerStats* stats_;
+};
+
+
 CompilerStats::CompilerStats(Isolate* isolate)
     : isolate_(isolate),
       parser_timer(true, "parser timer"),
@@ -34,7 +72,6 @@ CompilerStats::CompilerStats(Isolate* isolate)
       num_ident_tokens_total(0),
       num_tokens_consumed(0),
       num_token_checks(0),
-      num_tokens_rewind(0),
       num_tokens_lookahead(0),
       num_classes_compiled(0),
       num_functions_compiled(0),
@@ -51,6 +88,16 @@ void CompilerStats::Print() {
   if (!FLAG_compiler_stats) {
     return;
   }
+
+  // Traverse the heap and compute number of tokens in all
+  // TokenStream objects.
+  num_tokens_total = 0;
+  num_literal_tokens_total = 0;
+  num_ident_tokens_total = 0;
+  TokenStreamVisitor visitor(this);
+  ObjectGraph graph(isolate_);
+  graph.IterateObjects(&visitor);
+
   OS::Print("==== Compiler Stats for isolate '%s' ====\n",
             isolate_->debugger_name());
   OS::Print("Number of tokens:   %" Pd64 "\n", num_tokens_total);
@@ -61,8 +108,6 @@ void CompilerStats::Print() {
             (1.0 * num_tokens_consumed) / num_tokens_total);
   OS::Print("Tokens checked:     %" Pd64 "  (%.2f times tokens consumed)\n",
             num_token_checks, (1.0 * num_token_checks) / num_tokens_consumed);
-  OS::Print("Token rewind:       %" Pd64 " (%" Pd64 "%% of tokens checked)\n",
-            num_tokens_rewind, (100 * num_tokens_rewind) / num_token_checks);
   OS::Print("Token lookahead:    %" Pd64 " (%" Pd64 "%% of tokens checked)\n",
             num_tokens_lookahead,
             (100 * num_tokens_lookahead) / num_token_checks);

runtime/vm/compiler_stats.h

@@ -49,7 +49,6 @@ class CompilerStats {
   int64_t num_ident_tokens_total;
   int64_t num_tokens_consumed;
   int64_t num_token_checks;
-  int64_t num_tokens_rewind;
   int64_t num_tokens_lookahead;
 
   int64_t num_classes_compiled;

runtime/vm/isolate.cc

@@ -1418,6 +1418,10 @@ void Isolate::Shutdown() {
     StackZone stack_zone(this);
     HandleScope handle_scope(this);
 
+    if (compiler_stats_ != NULL) {
+      compiler_stats()->Print();
+    }
+
     // Notify exit listeners that this isolate is shutting down.
     if (object_store() != NULL) {
       NotifyExitListeners();
@@ -1444,9 +1448,6 @@ void Isolate::Shutdown() {
     api_state()->weak_persistent_handles().VisitHandles(&visitor);
     api_state()->prologue_weak_persistent_handles().VisitHandles(&visitor);
 
-    if (compiler_stats_ != NULL) {
-      compiler_stats()->Print();
-    }
     if (FLAG_trace_isolates) {
       heap()->PrintSizes();
       megamorphic_cache_table()->PrintSizes();

runtime/vm/object.cc

@@ -7964,21 +7964,14 @@ RawTokenStream* TokenStream::New(const Scanner::GrowableTokenStream& tokens,
   for (intptr_t i = 0; i < len; i++) {
     Scanner::TokenDescriptor token = tokens[i];
     if (token.kind == Token::kIDENT) {  // Identifier token.
-      if (FLAG_compiler_stats) {
-        INC_STAT(isolate, num_ident_tokens_total, 1);
-      }
       data.AddIdentToken(token.literal);
     } else if (Token::NeedsLiteralToken(token.kind)) {  // Literal token.
-      if (FLAG_compiler_stats) {
-        INC_STAT(isolate, num_literal_tokens_total, 1);
-      }
       data.AddLiteralToken(token);
     } else {  // Keyword, pseudo keyword etc.
       ASSERT(token.kind < Token::kNumTokens);
       data.AddSimpleToken(token.kind);
     }
   }
-  INC_STAT(isolate, num_tokens_total, len);
   data.AddSimpleToken(Token::kEOS);  // End of stream.
 
   // Create and setup the token stream object.

runtime/vm/parser.cc

@@ -443,9 +443,6 @@ void Parser::set_current_class(const Class& value) {
 
 
 void Parser::SetPosition(intptr_t position) {
-  if (position < TokenPos() && position != 0) {
-    INC_STAT(I, num_tokens_rewind, (TokenPos() - position));
-  }
   tokens_iterator_.SetCurrentPosition(position);
   token_kind_ = Token::kILLEGAL;
 }