Added isolate + thread high watermark tracking to Observatory

Added tracking of memory usage inside of threads. In addition, the max memory usage is kept track of using a high watermark for both the threads and the isolates. Isolate high watermark information is updated when a thread exits the isolate. The isolate high watermark consists of the sum of all thread high watermarks (including the high watermark of the exiting thread). High watermark information for both threads and isolates is now visible in the isolate view in the Observatory.

BUG=
R=asiva@google.com

Review-Url: https://codereview.chromium.org/2608463002 .

runtime/observatory/lib/src/elements/isolate_view.dart

@@ -277,6 +277,17 @@ class IsolateViewElement extends HtmlElement implements Renderable {
                         ..text = 'object store'
                     ]
                 ],
+              new DivElement()
+                ..classes = ['memberItem']
+                ..children = [
+                  new DivElement()
+                    ..classes = ['memberName']
+                    ..text = 'high watermark',
+                  new DivElement()
+                    ..classes = ['memberValue']
+                    ..text = Utils.formatSize(_isolate.memoryHighWatermark)
+                    ..title = '${_isolate.memoryHighWatermark}B'
+                ],
               new BRElement(),
               new DivElement()
                 ..classes = ['memberItem']
@@ -342,6 +353,11 @@ class IsolateViewElement extends HtmlElement implements Renderable {
             new DivElement()
               ..classes = ['indent']
               ..text = 'kind ${t.kindString}',
+            new DivElement()
+              ..classes = ['indent']
+              ..title = '${t.memoryHighWatermark}B'
+              ..text =
+              'high watermark ${Utils.formatSize(t.memoryHighWatermark)}',
             new DivElement()
               ..children = t.zones
                 .map((z) => new DivElement()

runtime/observatory/lib/src/models/objects/isolate.dart

@@ -54,6 +54,11 @@ abstract class Isolate extends IsolateRef {
   /// The list of threads associated with this isolate.
   Iterable<Thread> get threads;
 
+  /// The maximum amount of memory in bytes allocated by the isolate in all
+  /// threads at a given time. Calculated using the high watermarks of each
+  /// thread alive when a thread is unscheduled.
+  int get memoryHighWatermark;
+
   /// The current pause on exception mode for this isolate.
   //ExceptionPauseMode get exceptionPauseMode;
 

runtime/observatory/lib/src/models/objects/thread.dart

@@ -20,6 +20,10 @@ abstract class Thread {
   /// The task type associated with the thread.
   ThreadKind get kind;
 
+  /// The maximum amount of memory in bytes allocated by a thread at a given
+  /// time throughout the entire life of the thread.
+  int get memoryHighWatermark;
+
   /// A list of all the zones held by the thread.
   Iterable<Zone> get zones;
 }

runtime/observatory/lib/src/service/object.dart

@@ -1507,6 +1507,9 @@ class Isolate extends ServiceObjectOwner implements M.Isolate {
   List<Thread> get threads => _threads;
   final List<Thread> _threads = new List<Thread>();
 
+  int get memoryHighWatermark => _memoryHighWatermark;
+  int _memoryHighWatermark;
+
   void _loadHeapSnapshot(ServiceEvent event) {
     if (_snapshotFetch == null || _snapshotFetch.isClosed) {
       // No outstanding snapshot request. Presumably another client asked for a
@@ -1635,6 +1638,8 @@ class Isolate extends ServiceObjectOwner implements M.Isolate {
     if(map['threads'] != null) {
       threads.addAll(map['threads']);
     }
+
+    _memoryHighWatermark = map['isolateMemoryHighWatermark'];
   }
 
   Future<TagProfile> updateTagProfile() {
@@ -3072,6 +3077,8 @@ class Thread extends ServiceObject implements M.Thread {
   M.ThreadKind _kind;
   String get kindString => _kindString;
   String _kindString;
+  int get memoryHighWatermark => _memoryHighWatermark;
+  int _memoryHighWatermark;
   List<Zone> get zones => _zones;
   final List<Zone> _zones = new List<Zone>();
 
@@ -3109,6 +3116,9 @@ class Thread extends ServiceObject implements M.Thread {
       default:
         assert(false);
     }
+
+    _memoryHighWatermark = map['threadMemoryHighWatermark'];
+
     zones.clear();
     zoneList.forEach((zone) {
       int capacity = zone['capacity'];

runtime/observatory/tests/service/get_zone_memory_info_rpc_test.dart

@@ -15,7 +15,7 @@ var tests = [
     // the correct fields needed to examine zone memory usage.
     for (Isolate isolate in vm.isolates) {
       await isolate.reload();
-
+      expect(isolate.memoryHighWatermark, isInt);
       expect(isolate.threads, isNotNull);
       List<Thread> threads = isolate.threads;
 
@@ -23,6 +23,7 @@ var tests = [
         expect(thread.type, equals('_Thread'));
         expect(thread.id, isNotNull);
         expect(thread.kind, isNotNull);
+        expect(thread.memoryHighWatermark, isInt);
         expect(thread.zones, isNotNull);
         List<Zone> zones = thread.zones;
 

runtime/vm/isolate.cc

@@ -2103,6 +2103,8 @@ void Isolate::PrintJSON(JSONStream* stream, bool ref) {
     }
   }
 
+  jsobj.AddProperty("isolateMemoryHighWatermark",
+                    isolate_memory_high_watermark_);
   jsobj.AddProperty("threads", thread_registry_);
 }
 #endif
@@ -2699,6 +2701,7 @@ void Isolate::UnscheduleThread(Thread* thread,
     // Ensure that the thread reports itself as being at a safepoint.
     thread->EnterSafepoint();
   }
+  UpdateIsolateHighWatermark();
   OSThread* os_thread = thread->os_thread();
   ASSERT(os_thread != NULL);
   os_thread->DisableThreadInterrupts();
@@ -2713,12 +2716,22 @@ void Isolate::UnscheduleThread(Thread* thread,
   thread->set_execution_state(Thread::kThreadInNative);
   thread->set_safepoint_state(Thread::SetAtSafepoint(true, 0));
   thread->clear_pending_functions();
+  thread->ClearThreadMemoryUsageStats();
   ASSERT(thread->no_safepoint_scope_depth() == 0);
   // Return thread structure.
   thread_registry()->ReturnThreadLocked(is_mutator, thread);
 }
 
 
+void Isolate::UpdateIsolateHighWatermark() {
+  intptr_t thread_watermarks_total =
+      thread_registry()->ThreadHighWatermarksTotalLocked();
+  if (thread_watermarks_total > isolate_memory_high_watermark_) {
+    isolate_memory_high_watermark_ = thread_watermarks_total;
+  }
+}
+
+
 static RawInstance* DeserializeObject(Thread* thread,
                                       uint8_t* obj_data,
                                       intptr_t obj_len) {

runtime/vm/isolate.h

@@ -178,7 +178,9 @@ class Isolate : public BaseIsolate {
 
   ThreadRegistry* thread_registry() const { return thread_registry_; }
   SafepointHandler* safepoint_handler() const { return safepoint_handler_; }
-
+  intptr_t GetIsolateHighWatermark() const {
+    return isolate_memory_high_watermark_;
+  }
   ClassTable* class_table() { return &class_table_; }
   static intptr_t class_table_offset() {
     return OFFSET_OF(Isolate, class_table_);
@@ -694,6 +696,10 @@ class Isolate : public BaseIsolate {
                         bool is_mutator,
                         bool bypass_safepoint = false);
 
+  // Updates the maximum memory usage in bytes of all zones in all threads of
+  // the current isolate.
+  void UpdateIsolateHighWatermark();
+
   // DEPRECATED: Use Thread's methods instead. During migration, these default
   // to using the mutator thread (which must also be the current thread).
   Zone* current_zone() const {
@@ -714,6 +720,7 @@ class Isolate : public BaseIsolate {
 
   ThreadRegistry* thread_registry_;
   SafepointHandler* safepoint_handler_;
+  intptr_t isolate_memory_high_watermark_;
   Dart_MessageNotifyCallback message_notify_callback_;
   char* name_;
   char* debugger_name_;

runtime/vm/thread.cc

@@ -74,6 +74,8 @@ Thread::Thread(Isolate* isolate)
       os_thread_(NULL),
       thread_lock_(new Monitor()),
       zone_(NULL),
+      current_thread_memory_(0),
+      thread_memory_high_watermark_(0),
       api_reusable_scope_(NULL),
       api_top_scope_(NULL),
       top_resource_(NULL),
@@ -213,6 +215,7 @@ void Thread::PrintJSON(JSONStream* stream) const {
   jsobj.AddPropertyF("id", "threads/%" Pd "",
                      OSThread::ThreadIdToIntPtr(os_thread()->trace_id()));
   jsobj.AddProperty("kind", TaskKindToCString(task_kind()));
+  jsobj.AddProperty("threadMemoryHighWatermark", thread_memory_high_watermark_);
   Zone* zone = zone_;
   {
     JSONArray zone_info_array(&jsobj, "zones");

runtime/vm/thread.h

@@ -264,6 +264,26 @@ class Thread : public BaseThread {
 
   bool ZoneIsOwnedByThread(Zone* zone) const;
 
+  void IncrementThreadMemoryUsage(intptr_t value) {
+    current_thread_memory_ += value;
+    if (current_thread_memory_ > thread_memory_high_watermark_) {
+      thread_memory_high_watermark_ = current_thread_memory_;
+    }
+  }
+
+  void DecrementThreadMemoryUsage(intptr_t value) {
+    current_thread_memory_ -= value;
+  }
+
+  intptr_t GetThreadHighWatermark() const {
+    return thread_memory_high_watermark_;
+  }
+
+  void ClearThreadMemoryUsageStats() {
+    thread_memory_high_watermark_ = 0;
+    current_thread_memory_ = 0;
+  }
+
   // The reusable api local scope for this thread.
   ApiLocalScope* api_reusable_scope() const { return api_reusable_scope_; }
   void set_api_reusable_scope(ApiLocalScope* value) {
@@ -679,6 +699,8 @@ class Thread : public BaseThread {
   OSThread* os_thread_;
   Monitor* thread_lock_;
   Zone* zone_;
+  intptr_t current_thread_memory_;
+  intptr_t thread_memory_high_watermark_;
   ApiLocalScope* api_reusable_scope_;
   ApiLocalScope* api_top_scope_;
   StackResource* top_resource_;

runtime/vm/thread_registry.cc

@@ -146,6 +146,7 @@ Thread* ThreadRegistry::GetFromFreelistLocked(Isolate* isolate) {
   return thread;
 }
 
+
 void ThreadRegistry::ReturnToFreelistLocked(Thread* thread) {
   ASSERT(thread != NULL);
   ASSERT(thread->os_thread_ == NULL);
@@ -157,4 +158,16 @@ void ThreadRegistry::ReturnToFreelistLocked(Thread* thread) {
   free_list_ = thread;
 }
 
+
+intptr_t ThreadRegistry::ThreadHighWatermarksTotalLocked() const {
+  ASSERT(threads_lock()->IsOwnedByCurrentThread());
+  intptr_t max_memory_usage_total = 0;
+  Thread* current = active_list_;
+  while (current != NULL) {
+    max_memory_usage_total += current->GetThreadHighWatermark();
+    current = current->next_;
+  }
+  return max_memory_usage_total;
+}
+
 }  // namespace dart

runtime/vm/thread_registry.h

@@ -37,6 +37,9 @@ class ThreadRegistry {
   void PrintJSON(JSONStream* stream) const;
 #endif
 
+  // Calculates the sum of the max memory usage in bytes of each thread.
+  intptr_t ThreadHighWatermarksTotalLocked() const;
+
  private:
   Thread* active_list() const { return active_list_; }
   Monitor* threads_lock() const { return threads_lock_; }

runtime/vm/thread_test.cc

@@ -320,12 +320,21 @@ TEST_CASE(ManySimpleTasksWithZones) {
   isolate->PrintJSON(&stream, false);
   const char* json = stream.ToCString();
 
+  Thread* current_thread = Thread::Current();
+  {
+    StackZone stack_zone(current_thread);
+    char* isolate_info_buf = OS::SCreate(current_thread->zone(),
+                                         "\"isolateMemoryHighWatermark\":"
+                                         "%" Pd "",
+                                         isolate->GetIsolateHighWatermark());
+    EXPECT_SUBSTRING(isolate_info_buf, json);
+  }
+
   // Confirm all expected entries are in the JSON output.
   for (intptr_t i = 0; i < kTaskCount + 1; i++) {
     Thread* thread = threads[i];
     Zone* top_zone = thread->zone();
 
-    Thread* current_thread = Thread::Current();
     StackZone stack_zone(current_thread);
     Zone* current_zone = current_thread->zone();
 
@@ -348,9 +357,11 @@ TEST_CASE(ManySimpleTasksWithZones) {
                     "\"type\":\"_Thread\","
                     "\"id\":\"threads\\/%" Pd
                     "\","
-                    "\"kind\":\"%s\"",
+                    "\"kind\":\"%s\","
+                    "\"threadMemoryHighWatermark\":%" Pd "",
                     OSThread::ThreadIdToIntPtr(thread->os_thread()->trace_id()),
-                    Thread::TaskKindToCString(thread->task_kind()));
+                    Thread::TaskKindToCString(thread->task_kind()),
+                    thread->GetThreadHighWatermark());
 
     EXPECT_SUBSTRING(thread_info_buf, json);
   }

runtime/vm/zone.cc

@@ -43,7 +43,13 @@ class Zone::Segment {
 
 void Zone::Segment::DeleteSegmentList(Segment* head) {
   Segment* current = head;
+  Thread* current_thread = Thread::Current();
   while (current != NULL) {
+    if (current_thread != NULL) {
+      // TODO(bkonyi) Handle special case of segment deletion within native
+      // isolate.
+      Thread::Current()->DecrementThreadMemoryUsage(current->size());
+    }
     Segment* next = current->next();
 #ifdef DEBUG
     // Zap the entire current segment (including the header).
@@ -68,6 +74,11 @@ Zone::Segment* Zone::Segment::New(intptr_t size, Zone::Segment* next) {
 #endif
   result->next_ = next;
   result->size_ = size;
+  if (Thread::Current() != NULL) {
+    // TODO(bkonyi) Handle special case of segment creation within native
+    // isolate.
+    Thread::Current()->IncrementThreadMemoryUsage(size);
+  }
   return result;
 }