[vm] Produce clearer error messages when operator new fails.

TEST=bots
Bug: https://github.com/dart-lang/sdk/issues/43642
Bug: https://github.com/dart-lang/sdk/issues/43862
Change-Id: I598a26a56762c141a1aef972f7d32f2c9594b244
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/171802
Commit-Queue: Ryan Macnak <rmacnak@google.com>
Reviewed-by: Alexander Aprelev <aam@google.com>
Reviewed-by: Martin Kustermann <kustermann@google.com>

runtime/platform/allocation.h

@@ -5,6 +5,7 @@
 #ifndef RUNTIME_PLATFORM_ALLOCATION_H_
 #define RUNTIME_PLATFORM_ALLOCATION_H_
 
+#include "platform/address_sanitizer.h"
 #include "platform/assert.h"
 
 namespace dart {
@@ -31,6 +32,36 @@ class AllStatic {
   DISALLOW_IMPLICIT_CONSTRUCTORS(AllStatic);
 };
 
+class MallocAllocated {
+ public:
+  MallocAllocated() {}
+
+  // Intercept operator new to produce clearer error messages when we run out
+  // of memory. Don't do this when running under ASAN so it can continue to
+  // check malloc/new/new[] are paired with free/delete/delete[] respectively.
+#if !defined(USING_ADDRESS_SANITIZER)
+  void* operator new(size_t size) {
+    void* result = ::malloc(size);
+    if (result == nullptr) {
+      OUT_OF_MEMORY();
+    }
+    return result;
+  }
+
+  void* operator new[](size_t size) {
+    void* result = ::malloc(size);
+    if (result == nullptr) {
+      OUT_OF_MEMORY();
+    }
+    return result;
+  }
+
+  void operator delete(void* pointer) { ::free(pointer); }
+
+  void operator delete[](void* pointer) { ::free(pointer); }
+#endif
+};
+
 }  // namespace dart
 
 #endif  // RUNTIME_PLATFORM_ALLOCATION_H_

runtime/platform/growable_array.h

@@ -254,14 +254,13 @@ class Malloc : public AllStatic {
   }
 };
 
-class EmptyBase {};
-
 template <typename T>
-class MallocGrowableArray : public BaseGrowableArray<T, EmptyBase, Malloc> {
+class MallocGrowableArray
+    : public BaseGrowableArray<T, MallocAllocated, Malloc> {
  public:
   explicit MallocGrowableArray(intptr_t initial_capacity)
-      : BaseGrowableArray<T, EmptyBase, Malloc>(initial_capacity, NULL) {}
-  MallocGrowableArray() : BaseGrowableArray<T, EmptyBase, Malloc>(NULL) {}
+      : BaseGrowableArray<T, MallocAllocated, Malloc>(initial_capacity, NULL) {}
+  MallocGrowableArray() : BaseGrowableArray<T, MallocAllocated, Malloc>(NULL) {}
 };
 
 }  // namespace dart

runtime/platform/hashmap.cc

@@ -157,9 +157,6 @@ SimpleHashMap::Entry* SimpleHashMap::Probe(void* key, uint32_t hash) {
 void SimpleHashMap::Initialize(uint32_t capacity) {
   ASSERT(dart::Utils::IsPowerOfTwo(capacity));
   map_ = new Entry[capacity];
-  if (map_ == NULL) {
-    OUT_OF_MEMORY();
-  }
   capacity_ = capacity;
   occupancy_ = 0;
 }

runtime/vm/allocation.h

@@ -54,10 +54,10 @@ class ZoneAllocated {
   ZoneAllocated() {}
 
   // Implicitly allocate the object in the current zone.
-  void* operator new(uword size);
+  void* operator new(size_t size);
 
   // Allocate the object in the given zone, which must be the current zone.
-  void* operator new(uword size, Zone* zone);
+  void* operator new(size_t size, Zone* zone);
 
   // Ideally, the delete operator should be protected instead of
   // public, but unfortunately the compiler sometimes synthesizes

runtime/vm/deopt_instructions.h

@@ -27,7 +27,7 @@ class TimelineEvent;
 
 // Holds all data relevant for execution of deoptimization instructions.
 // Structure is allocated in C-heap.
-class DeoptContext {
+class DeoptContext : public MallocAllocated {
  public:
   enum DestFrameOptions {
     kDestIsOriginalFrame,  // Replace the original frame with deopt frame.

runtime/vm/handles.h

@@ -116,7 +116,7 @@ class Handles {
   // is allocated from the chunk until we run out space in the chunk,
   // at this point another chunk is allocated. These chunks are chained
   // together.
-  class HandlesBlock {
+  class HandlesBlock : public MallocAllocated {
    public:
     explicit HandlesBlock(HandlesBlock* next)
         : next_handle_slot_(0), next_block_(next) {}

runtime/vm/handles_impl.h

@@ -157,9 +157,6 @@ void Handles<kHandleSizeInWords, kHandlesPerChunk, kOffsetOfRawPtr>::
   }
   if (scoped_blocks_->next_block() == NULL) {
     HandlesBlock* block = new HandlesBlock(NULL);
-    if (block == NULL) {
-      OUT_OF_MEMORY();
-    }
     scoped_blocks_->set_next_block(block);
   }
   scoped_blocks_ = scoped_blocks_->next_block();
@@ -207,9 +204,6 @@ void Handles<kHandleSizeInWords, kHandlesPerChunk, kOffsetOfRawPtr>::
                  CountScopedHandles());
   }
   zone_blocks_ = new HandlesBlock(zone_blocks_);
-  if (zone_blocks_ == NULL) {
-    OUT_OF_MEMORY();
-  }
 }
 
 #if defined(DEBUG)

runtime/vm/hash_map.h

@@ -423,15 +423,17 @@ class DirectChainedHashMap
 
 template <typename KeyValueTrait>
 class MallocDirectChainedHashMap
-    : public BaseDirectChainedHashMap<KeyValueTrait, EmptyBase, Malloc> {
+    : public BaseDirectChainedHashMap<KeyValueTrait, MallocAllocated, Malloc> {
  public:
   MallocDirectChainedHashMap()
-      : BaseDirectChainedHashMap<KeyValueTrait, EmptyBase, Malloc>(NULL) {}
+      : BaseDirectChainedHashMap<KeyValueTrait, MallocAllocated, Malloc>(NULL) {
+  }
 
   // The only use of the copy constructor seems to be in hash_map_test.cc.
   // Not disallowing it for now just in case there are other users.
   MallocDirectChainedHashMap(const MallocDirectChainedHashMap& other)
-      : BaseDirectChainedHashMap<KeyValueTrait, EmptyBase, Malloc>(other) {}
+      : BaseDirectChainedHashMap<KeyValueTrait, MallocAllocated, Malloc>(
+            other) {}
 
  private:
   void operator=(const MallocDirectChainedHashMap& other) = delete;

runtime/vm/heap/pointer_block.h

@@ -18,7 +18,7 @@ class ObjectPointerVisitor;
 
 // A set of ObjectPtr. Must be emptied before destruction (using Pop/Reset).
 template <int Size>
-class PointerBlock {
+class PointerBlock : public MallocAllocated {
  public:
   enum { kSize = Size };
 

runtime/vm/port_set.h

@@ -7,6 +7,7 @@
 
 #include "include/dart_api.h"
 
+#include "platform/allocation.h"
 #include "platform/globals.h"
 #include "platform/utils.h"
 
@@ -18,7 +19,7 @@ class PortSet {
   static constexpr Dart_Port kFreePort = static_cast<Dart_Port>(0);
   static constexpr Dart_Port kDeletedPort = static_cast<Dart_Port>(3);
 
-  struct Entry {
+  struct Entry : public MallocAllocated {
     Entry() : port(kFreePort) {}
 
     // Free entries have set this to 0.

runtime/vm/timeline.h

@@ -574,7 +574,7 @@ class TimelineBeginEndScope : public TimelineEventScope {
 };
 
 // A block of |TimelineEvent|s. Not thread safe.
-class TimelineEventBlock {
+class TimelineEventBlock : public MallocAllocated {
  public:
   static const intptr_t kBlockSize = 64;
 
@@ -707,7 +707,7 @@ class IsolateTimelineEventFilter : public TimelineEventFilter {
 };
 
 // Recorder of |TimelineEvent|s.
-class TimelineEventRecorder {
+class TimelineEventRecorder : public MallocAllocated {
  public:
   TimelineEventRecorder();
   virtual ~TimelineEventRecorder() {}