[vm] Ask the OS to release the memory for the clustered portion of the snapshot.

This region is mostly unused after loading, but it may be accessed again if
 - Isolate.spawn is invoke with isolate groups disabled
 - A secondary snapshot produced by splitting is loaded
 - An external typed data in the snapshot is accessed (usually a kernel file)
 - Likely other cases

Even if these cases did not exist, the region is often part of a shared library and so unable to be released independently.

madvise(DONT_NEED) on this region will cause the OS to release the memory in this region but keep the address space reservation and mapping. If it is touched again, it will be brought back in from the file. If it is not backed by a file, such as malloc memory, it will be brought back in as zeros and the program will likely fail.

TEST=ci
Bug: https://github.com/dart-lang/sdk/issues/44019
Bug: https://github.com/flutter/flutter/issues/92120
Change-Id: I315a049b0f7d440e181d0a5e87fa6770a2fd4f79
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/216580
Commit-Queue: Ryan Macnak <rmacnak@google.com>
Reviewed-by: Siva Annamalai <asiva@google.com>

runtime/include/dart_api.h

@@ -608,6 +608,7 @@ typedef struct {
   bool copy_parent_code;
   bool null_safety;
   bool is_system_isolate;
+  bool snapshot_is_dontneed_safe;
 } Dart_IsolateFlags;
 
 /**

runtime/vm/app_snapshot.cc

@@ -7833,6 +7833,8 @@ class HeapLocker : public StackResource {
 };
 
 void Deserializer::Deserialize(DeserializationRoots* roots) {
+  const void* clustered_start = CurrentBufferAddress();
+
   Array& refs = Array::Handle(zone_);
   num_base_objects_ = ReadUnsigned();
   num_objects_ = ReadUnsigned();
@@ -7926,8 +7928,8 @@ void Deserializer::Deserialize(DeserializationRoots* roots) {
 
   roots->PostLoad(this, refs);
 
-#if defined(DEBUG)
   auto isolate_group = thread()->isolate_group();
+#if defined(DEBUG)
   isolate_group->ValidateClassTable();
   if (isolate_group != Dart::vm_isolate()->group()) {
     isolate_group->heap()->Verify();
@@ -7941,6 +7943,13 @@ void Deserializer::Deserialize(DeserializationRoots* roots) {
       clusters_[i]->PostLoad(this, refs, primary);
     }
   }
+
+  if (isolate_group->snapshot_is_dontneed_safe()) {
+    size_t clustered_length = reinterpret_cast<uword>(CurrentBufferAddress()) -
+                              reinterpret_cast<uword>(clustered_start);
+    VirtualMemory::DontNeed(const_cast<void*>(clustered_start),
+                            clustered_length);
+  }
 }
 
 #if !defined(DART_PRECOMPILED_RUNTIME)

runtime/vm/isolate.h

@@ -161,7 +161,9 @@ typedef FixedCache<intptr_t, CatchEntryMovesRefPtr, 16> CatchEntryMovesCache;
     FLAG_use_field_guards)                                                     \
   V(PRODUCT, should_load_vmservice_library, ShouldLoadVmService,               \
     load_vmservice_library, false)                                             \
-  V(NONPRODUCT, use_osr, UseOsr, use_osr, FLAG_use_osr)
+  V(NONPRODUCT, use_osr, UseOsr, use_osr, FLAG_use_osr)                        \
+  V(NONPRODUCT, snapshot_is_dontneed_safe, SnapshotIsDontNeedSafe,             \
+    snapshot_is_dontneed_safe, false)
 
 #define BOOL_ISOLATE_FLAG_LIST_DEFAULT_GETTER(V)                               \
   V(PRODUCT, copy_parent_code, CopyParentCode, copy_parent_code, false)        \
@@ -786,7 +788,8 @@ class IsolateGroup : public IntrusiveDListEntry<IsolateGroup> {
   V(NullSafetySet)                                                             \
   V(Obfuscate)                                                                 \
   V(UseFieldGuards)                                                            \
-  V(UseOsr)
+  V(UseOsr)                                                                    \
+  V(SnapshotIsDontNeedSafe)
 
   // Isolate group specific flags.
   enum FlagBits {

runtime/vm/virtual_memory.h

@@ -46,6 +46,8 @@ class VirtualMemory {
   static void Protect(void* address, intptr_t size, Protection mode);
   void Protect(Protection mode) { return Protect(address(), size(), mode); }
 
+  static void DontNeed(void* address, intptr_t size);
+
   // Reserves and commits a virtual memory segment with size. If a segment of
   // the requested size cannot be allocated, NULL is returned.
   static VirtualMemory* Allocate(intptr_t size,

runtime/vm/virtual_memory_fuchsia.cc

@@ -299,6 +299,21 @@ void VirtualMemory::Protect(void* address, intptr_t size, Protection mode) {
   }
 }
 
+void VirtualMemory::DontNeed(void* address, intptr_t size) {
+  uword start_address = reinterpret_cast<uword>(address);
+  uword end_address = start_address + size;
+  uword page_address = Utils::RoundDown(start_address, PageSize());
+  zx_status_t status = zx_vmar_op_range(
+      getVmarForAddress(reinterpret_cast<uword>(address)), ZX_VMAR_OP_DONT_NEED,
+      page_address, end_address - page_address, nullptr, 0);
+  LOG_INFO("zx_vmar_op_range(DONTNEED, 0x%lx, 0x%lx)\n", page_address,
+           end_address - page_address);
+  if (status != ZX_OK) {
+    FATAL("zx_vmar_op_range(DONTNEED, 0x%lx, 0x%lx) failed: %s\n", page_address,
+          end_address - page_address, zx_status_get_string(status));
+  }
+}
+
 }  // namespace dart
 
 #endif  // defined(DART_HOST_OS_FUCHSIA)

runtime/vm/virtual_memory_posix.cc

@@ -560,6 +560,20 @@ void VirtualMemory::Protect(void* address, intptr_t size, Protection mode) {
            end_address - page_address, prot);
 }
 
+void VirtualMemory::DontNeed(void* address, intptr_t size) {
+  uword start_address = reinterpret_cast<uword>(address);
+  uword end_address = start_address + size;
+  uword page_address = Utils::RoundDown(start_address, PageSize());
+  if (madvise(reinterpret_cast<void*>(page_address), end_address - page_address,
+              MADV_DONTNEED) != 0) {
+    int error = errno;
+    const int kBufferSize = 1024;
+    char error_buf[kBufferSize];
+    FATAL("madvise error: %d (%s)", error,
+          Utils::StrError(error, error_buf, kBufferSize));
+  }
+}
+
 }  // namespace dart
 
 #endif  // defined(DART_HOST_OS_ANDROID) || defined(DART_HOST_OS_LINUX) ||     \

runtime/vm/virtual_memory_win.cc

@@ -241,6 +241,8 @@ void VirtualMemory::Protect(void* address, intptr_t size, Protection mode) {
   }
 }
 
+void VirtualMemory::DontNeed(void* address, intptr_t size) {}
+
 }  // namespace dart
 
 #endif  // defined(DART_HOST_OS_WINDOWS)