Kernel: Memory purging improvements

This adds the ability for a Region to define volatile/nonvolatile
areas within mapped memory using madvise(). This also means that
memory purging takes into account all views of the PurgeableVMObject
and only purges memory that is not needed by all of them. When calling
madvise() to change an area to nonvolatile memory, return whether
memory from that area was purged. At that time also try to remap
all memory that is requested to be nonvolatile, and if insufficient
pages are available notify the caller of that fact.

Kernel/FileSystem/ProcFS.cpp

@@ -331,7 +331,7 @@ static OwnPtr<KBuffer> procfs$pid_vm(InodeIdentifier identifier)
             region_object.add("user_accessible", region.is_user_accessible());
             region_object.add("purgeable", region.vmobject().is_purgeable());
             if (region.vmobject().is_purgeable()) {
-                region_object.add("volatile", static_cast<const PurgeableVMObject&>(region.vmobject()).is_volatile());
+                region_object.add("volatile", static_cast<const PurgeableVMObject&>(region.vmobject()).is_any_volatile());
             }
             region_object.add("cacheable", region.is_cacheable());
             region_object.add("kernel", region.is_kernel());

Kernel/Heap/SlabAllocator.cpp

@@ -130,7 +130,7 @@ static SlabAllocator<32> s_slab_allocator_32;
 static SlabAllocator<64> s_slab_allocator_64;
 static SlabAllocator<128> s_slab_allocator_128;
 
-static_assert(sizeof(Region) <= s_slab_allocator_64.slab_size());
+static_assert(sizeof(Region) <= s_slab_allocator_128.slab_size());
 
 template<typename Callback>
 void for_each_allocator(Callback callback)

Kernel/Process.cpp

@@ -130,7 +130,7 @@ Range Process::allocate_range(VirtualAddress vaddr, size_t size, size_t alignmen
 
 Region& Process::allocate_split_region(const Region& source_region, const Range& range, size_t offset_in_vmobject)
 {
-    auto& region = add_region(Region::create_user_accessible(range, source_region.vmobject(), offset_in_vmobject, source_region.name(), source_region.access()));
+    auto& region = add_region(Region::create_user_accessible(this, range, source_region.vmobject(), offset_in_vmobject, source_region.name(), source_region.access()));
     region.set_mmap(source_region.is_mmap());
     region.set_stack(source_region.is_stack());
     size_t page_offset_in_source_region = (offset_in_vmobject - source_region.offset_in_vmobject()) / PAGE_SIZE;
@@ -145,7 +145,7 @@ Region* Process::allocate_region(const Range& range, const String& name, int pro
 {
     ASSERT(range.is_valid());
     auto vmobject = AnonymousVMObject::create_with_size(range.size());
-    auto region = Region::create_user_accessible(range, vmobject, 0, name, prot_to_region_access_flags(prot));
+    auto region = Region::create_user_accessible(this, range, vmobject, 0, name, prot_to_region_access_flags(prot));
     region->map(page_directory());
     if (should_commit && !region->commit())
         return nullptr;
@@ -177,7 +177,7 @@ Region* Process::allocate_region_with_vmobject(const Range& range, NonnullRefPtr
         return nullptr;
     }
     offset_in_vmobject &= PAGE_MASK;
-    auto& region = add_region(Region::create_user_accessible(range, move(vmobject), offset_in_vmobject, name, prot_to_region_access_flags(prot)));
+    auto& region = add_region(Region::create_user_accessible(this, range, move(vmobject), offset_in_vmobject, name, prot_to_region_access_flags(prot)));
     region.map(page_directory());
     return &region;
 }
@@ -762,7 +762,7 @@ size_t Process::amount_purgeable_volatile() const
     size_t amount = 0;
     ScopedSpinLock lock(m_lock);
     for (auto& region : m_regions) {
-        if (region.vmobject().is_purgeable() && static_cast<const PurgeableVMObject&>(region.vmobject()).is_volatile())
+        if (region.vmobject().is_purgeable() && static_cast<const PurgeableVMObject&>(region.vmobject()).is_any_volatile())
             amount += region.amount_resident();
     }
     return amount;
@@ -773,7 +773,7 @@ size_t Process::amount_purgeable_nonvolatile() const
     size_t amount = 0;
     ScopedSpinLock lock(m_lock);
     for (auto& region : m_regions) {
-        if (region.vmobject().is_purgeable() && !static_cast<const PurgeableVMObject&>(region.vmobject()).is_volatile())
+        if (region.vmobject().is_purgeable() && !static_cast<const PurgeableVMObject&>(region.vmobject()).is_any_volatile())
             amount += region.amount_resident();
     }
     return amount;

Kernel/SharedBuffer.cpp

@@ -219,4 +219,29 @@ void SharedBuffer::seal()
     }
 }
 
+auto SharedBuffer::set_volatile_all(bool is_volatile, bool& was_purged) -> SetVolatileError
+{
+    was_purged = false;
+    auto pid = Process::current()->pid();
+    LOCKER(shared_buffers().lock());
+    for (size_t i = 0; i < m_refs.size(); ++i) {
+        auto& ref = m_refs[i];
+        if (ref.pid == pid) {
+            if (Region* region = ref.region.unsafe_ptr()) {
+                switch (region->set_volatile(region->vaddr(), region->size(), is_volatile, was_purged)) {
+                    case Region::SetVolatileError::Success:
+                        if (!was_purged && was_purged)
+                            klog() << "Region @ " << region->vaddr() << " - " << region->vaddr().offset(region->size()) << " was purged!";
+                        return SetVolatileError::Success;
+                    case Region::SetVolatileError::NotPurgeable:
+                        return SetVolatileError::NotPurgeable;
+                    case Region::SetVolatileError::OutOfMemory:
+                        return SetVolatileError::OutOfMemory;
+                }
+            }
+        }
+    }
+    return SetVolatileError::NotMapped;
+}
+
 }

Kernel/SharedBuffer.h

@@ -75,6 +75,13 @@ public:
     size_t size() const { return m_vmobject->size(); }
     void destroy_if_unused();
     void seal();
+    enum class SetVolatileError {
+        Success = 0,
+        NotPurgeable,
+        OutOfMemory,
+        NotMapped
+    };
+    SetVolatileError set_volatile_all(bool is_volatile, bool& was_purged);
     PurgeableVMObject& vmobject() { return m_vmobject; }
     const PurgeableVMObject& vmobject() const { return m_vmobject; }
     int id() const { return m_shbuf_id; }

Kernel/Syscalls/mmap.cpp

@@ -282,31 +282,30 @@ int Process::sys$madvise(void* address, size_t size, int advice)
         return -EINVAL;
     if (!region->is_mmap())
         return -EPERM;
-    if ((advice & MADV_SET_VOLATILE) && (advice & MADV_SET_NONVOLATILE))
+    bool set_volatile = advice & MADV_SET_VOLATILE;
+    bool set_nonvolatile = advice & MADV_SET_NONVOLATILE;
+    if (set_volatile && set_nonvolatile)
         return -EINVAL;
-    if (advice & MADV_SET_VOLATILE) {
+    if (set_volatile || set_nonvolatile) {
         if (!region->vmobject().is_purgeable())
             return -EPERM;
-        auto& vmobject = static_cast<PurgeableVMObject&>(region->vmobject());
-        vmobject.set_volatile(true);
+        bool was_purged = false;
+        switch (region->set_volatile(VirtualAddress(address), size, set_volatile, was_purged)) {
+        case Region::SetVolatileError::Success:
+            break;
+        case Region::SetVolatileError::NotPurgeable:
+            return -EPERM;
+        case Region::SetVolatileError::OutOfMemory:
+            return -ENOMEM;
+        }
+        if (set_nonvolatile)
+            return was_purged ? 1 : 0;
         return 0;
     }
-    if (advice & MADV_SET_NONVOLATILE) {
-        if (!region->vmobject().is_purgeable())
-            return -EPERM;
-        auto& vmobject = static_cast<PurgeableVMObject&>(region->vmobject());
-        if (!vmobject.is_volatile())
-            return 0;
-        vmobject.set_volatile(false);
-        bool was_purged = vmobject.was_purged();
-        vmobject.set_was_purged(false);
-        return was_purged ? 1 : 0;
-    }
     if (advice & MADV_GET_VOLATILE) {
         if (!region->vmobject().is_purgeable())
             return -EPERM;
-        auto& vmobject = static_cast<PurgeableVMObject&>(region->vmobject());
-        return vmobject.is_volatile() ? 0 : 1;
+        return region->is_volatile(VirtualAddress(address), size) ? 0 : 1;
     }
     return -EINVAL;
 }

Kernel/Syscalls/shbuf.cpp

@@ -174,14 +174,28 @@ int Process::sys$shbuf_set_volatile(int shbuf_id, bool state)
 #ifdef SHARED_BUFFER_DEBUG
     klog() << "Set shared buffer " << shbuf_id << " volatile: " << state;
 #endif
+
+    bool was_purged = false;
+    auto set_volatile = [&]() -> int {
+        switch (shared_buffer.set_volatile_all(state, was_purged)) {
+        case SharedBuffer::SetVolatileError::Success:
+            break;
+        case SharedBuffer::SetVolatileError::NotPurgeable:
+            return -EPERM;
+        case SharedBuffer::SetVolatileError::OutOfMemory:
+            return -ENOMEM;
+        case SharedBuffer::SetVolatileError::NotMapped:
+            return -EINVAL;
+        }
+        return 0;
+    };
+
     if (!state) {
-        bool was_purged = shared_buffer.vmobject().was_purged();
-        shared_buffer.vmobject().set_volatile(state);
-        shared_buffer.vmobject().set_was_purged(false);
+        if (int err = set_volatile())
+            return err;
         return was_purged ? 1 : 0;
     }
-    shared_buffer.vmobject().set_volatile(true);
-    return 0;
+    return set_volatile();
 }
 
 }

Kernel/VM/MemoryManager.cpp

@@ -451,7 +451,7 @@ OwnPtr<Region> MemoryManager::allocate_kernel_region_with_vmobject(const Range&
     ScopedSpinLock lock(s_mm_lock);
     OwnPtr<Region> region;
     if (user_accessible)
-        region = Region::create_user_accessible(range, vmobject, 0, name, access, cacheable);
+        region = Region::create_user_accessible(nullptr, range, vmobject, 0, name, access, cacheable);
     else
         region = Region::create_kernel_only(range, vmobject, 0, name, access, cacheable);
     if (region)

Kernel/VM/MemoryManager.h

@@ -38,7 +38,8 @@
 
 namespace Kernel {
 
-#define PAGE_ROUND_UP(x) ((((u32)(x)) + PAGE_SIZE - 1) & (~(PAGE_SIZE - 1)))
+#define PAGE_ROUND_UP(x) ((((FlatPtr)(x)) + PAGE_SIZE - 1) & (~(PAGE_SIZE - 1)))
+#define PAGE_ROUND_DOWN(x) (((FlatPtr)(x)) & ~(PAGE_SIZE - 1))
 
 template<typename T>
 inline T* low_physical_to_virtual(T* physical)

Kernel/VM/PurgeableVMObject.cpp

@@ -24,12 +24,201 @@
  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 
+#include <AK/BinarySearch.h>
+#include <AK/ScopeGuard.h>
+#include <Kernel/Process.h>
 #include <Kernel/VM/MemoryManager.h>
 #include <Kernel/VM/PhysicalPage.h>
 #include <Kernel/VM/PurgeableVMObject.h>
 
+//#define VOLATILE_PAGE_RANGES_DEBUG
+
 namespace Kernel {
 
+#ifdef VOLATILE_PAGE_RANGES_DEBUG
+inline LogStream& operator<<(const LogStream& stream, const VolatilePageRange& range)
+{
+    stream << "{" << range.base << " (" << range.count << ") purged: " << range.was_purged << "}";
+    return const_cast<LogStream&>(stream);
+}
+
+static void dump_volatile_page_ranges(const Vector<VolatilePageRange>& ranges)
+{
+   for (size_t i = 0; i < ranges.size(); i++) {
+       const auto& range = ranges[i];
+       klog() << "  [" << i << "] " << range;
+   }
+}
+#endif
+
+bool VolatilePageRanges::add(const VolatilePageRange& range)
+{
+    auto add_range = m_total_range.intersected(range);
+    if (add_range.is_empty())
+        return false;
+    add_range.was_purged = range.was_purged;
+
+#ifdef VOLATILE_PAGE_RANGES_DEBUG
+    klog() << "ADD " << range << " (total range: " << m_total_range << ") -->";
+    dump_volatile_page_ranges(m_ranges);
+    ScopeGuard debug_guard([&]() {
+        klog() << "After adding " << range << " (total range: " << m_total_range << ")";
+        dump_volatile_page_ranges(m_ranges);
+        klog() << "<-- ADD " << range << " (total range: " << m_total_range << ")";
+    });
+#endif
+
+    size_t nearby_index = 0;
+    auto* existing_range = binary_search(
+        m_ranges.span(), add_range, &nearby_index, [](auto& a, auto& b) {
+            if (a.intersects_or_adjacent(b))
+                return 0;
+            return (signed)(a.base - (b.base + b.count - 1));
+        });
+
+    size_t inserted_index = 0;
+    if (existing_range) {
+        if (*existing_range == add_range)
+            return false;
+
+        if (existing_range->was_purged != add_range.was_purged) {
+            // Found an intersecting or adjacent range, but the purge flag
+            // doesn't match. Subtract what we're adding from it, and
+            existing_range->subtract_intersecting(add_range);
+            if (existing_range->is_empty()) {
+                *existing_range = add_range;
+            } else {
+                m_ranges.insert(++nearby_index, add_range);
+                existing_range = &m_ranges[nearby_index];
+            }
+        } else {
+            // Found an intersecting or adjacent range that can be merged
+            existing_range->combine_intersecting_or_adjacent(add_range);
+        }
+        inserted_index = nearby_index;
+    } else {
+        // Insert into the sorted list
+        m_ranges.insert_before_matching(
+            VolatilePageRange(add_range), [&](auto& entry) {
+                return entry.base >= add_range.base + add_range.count;
+            },
+            nearby_index, &inserted_index);
+        existing_range = &m_ranges[inserted_index];
+    }
+
+    // See if we can merge any of the following ranges
+    inserted_index++;
+    while (inserted_index < m_ranges.size()) {
+        auto& next_range = m_ranges[inserted_index];
+        if (!next_range.intersects_or_adjacent(*existing_range))
+            break;
+        if (next_range.was_purged != existing_range->was_purged) {
+            // The purged flag of following range is not the same.
+            // Subtract the added/combined range from it
+            next_range.subtract_intersecting(*existing_range);
+            if (next_range.is_empty())
+                m_ranges.remove(inserted_index);
+        } else {
+            existing_range->combine_intersecting_or_adjacent(next_range);
+            m_ranges.remove(inserted_index);
+        }
+    }
+    return true;
+}
+
+bool VolatilePageRanges::remove(const VolatilePageRange& range, bool& was_purged)
+{
+    auto remove_range = m_total_range.intersected(range);
+    if (remove_range.is_empty())
+        return false;
+
+#ifdef VOLATILE_PAGE_RANGES_DEBUG
+    klog() << "REMOVE " << range << " (total range: " << m_total_range << ") -->";
+    dump_volatile_page_ranges(m_ranges);
+    ScopeGuard debug_guard([&]() {
+        klog() << "After removing " << range << " (total range: " << m_total_range << ")";
+        dump_volatile_page_ranges(m_ranges);
+        klog() << "<-- REMOVE " << range << " (total range: " << m_total_range << ") was_purged: " << was_purged;
+    });
+#endif
+
+    size_t nearby_index = 0;
+    auto* existing_range = binary_search(
+        m_ranges.span(), remove_range, &nearby_index, [](auto& a, auto& b) {
+            if (a.intersects(b))
+                return 0;
+            return (signed)(a.base - (b.base + b.count - 1));
+        });
+    if (!existing_range)
+        return false;
+
+    was_purged = existing_range->was_purged;
+    if (existing_range->range_equals(remove_range)) {
+        m_ranges.remove(nearby_index);
+    } else {
+        // See if we need to remove any of the following ranges
+        ASSERT(existing_range == &m_ranges[nearby_index]); // sanity check
+        while (nearby_index < m_ranges.size()) {
+            existing_range = &m_ranges[nearby_index];
+            if (!existing_range->intersects(range))
+                break;
+            was_purged |= existing_range->was_purged;
+            existing_range->subtract_intersecting(remove_range);
+            if (existing_range->is_empty()) {
+                m_ranges.remove(nearby_index);
+                break;
+            }
+        }
+    }
+    return true;
+}
+
+bool VolatilePageRanges::intersects(const VolatilePageRange& range) const
+{
+    auto* existing_range = binary_search(
+        m_ranges.span(), range, nullptr, [](auto& a, auto& b) {
+            if (a.intersects(b))
+                return 0;
+            return (signed)(a.base - (b.base + b.count - 1));
+        });
+    return existing_range != nullptr;
+}
+
+PurgeablePageRanges::PurgeablePageRanges(const VMObject& vmobject)
+    : m_volatile_ranges({0, vmobject.is_purgeable() ? static_cast<const PurgeableVMObject&>(vmobject).page_count() : 0})
+{
+}
+
+bool PurgeablePageRanges::add_volatile_range(const VolatilePageRange& range)
+{
+    if (range.is_empty())
+        return false;
+    ScopedSpinLock lock(m_volatile_ranges_lock);
+    return m_volatile_ranges.add(range);
+}
+
+bool PurgeablePageRanges::remove_volatile_range(const VolatilePageRange& range, bool& was_purged)
+{
+    if (range.is_empty())
+        return false;
+    ScopedSpinLock lock(m_volatile_ranges_lock);
+    return m_volatile_ranges.remove(range, was_purged);
+}
+
+bool PurgeablePageRanges::is_volatile_range(const VolatilePageRange& range) const
+{
+    if (range.is_empty())
+        return false;
+    ScopedSpinLock lock(m_volatile_ranges_lock);
+    return m_volatile_ranges.intersects(range);
+}
+
+void PurgeablePageRanges::set_was_purged(const VolatilePageRange& range)
+{
+    ScopedSpinLock lock(m_volatile_ranges_lock);
+    m_volatile_ranges.add({range.base, range.count, true});
+}
+
 NonnullRefPtr<PurgeableVMObject> PurgeableVMObject::create_with_size(size_t size)
 {
     return adopt(*new PurgeableVMObject(size));
@@ -42,9 +231,9 @@ PurgeableVMObject::PurgeableVMObject(size_t size)
 
 PurgeableVMObject::PurgeableVMObject(const PurgeableVMObject& other)
     : AnonymousVMObject(other)
-    , m_was_purged(other.m_was_purged)
-    , m_volatile(other.m_volatile)
+    , m_purgeable_ranges() // do *not* clone this
 {
+    // TODO: what about m_lock?
 }
 
 PurgeableVMObject::~PurgeableVMObject()
@@ -70,26 +259,75 @@ int PurgeableVMObject::purge_with_interrupts_disabled(Badge<MemoryManager>)
     return purge_impl();
 }
 
+void PurgeableVMObject::set_was_purged(const VolatilePageRange& range)
+{
+    ASSERT(m_lock.is_locked());
+    for (auto* purgeable_ranges : m_purgeable_ranges)
+        purgeable_ranges->set_was_purged(range);
+}
+
 int PurgeableVMObject::purge_impl()
 {
-    if (!m_volatile)
-        return 0;
     int purged_page_count = 0;
-    for (size_t i = 0; i < m_physical_pages.size(); ++i) {
-        if (m_physical_pages[i] && !m_physical_pages[i]->is_shared_zero_page())
-            ++purged_page_count;
-        m_physical_pages[i] = MM.shared_zero_page();
-    }
-    m_was_purged = true;
-
-    if (purged_page_count > 0) {
-        for_each_region([&](auto& region) {
-            if (&region.vmobject() == this)
-                region.remap();
-        });
-    }
+    ScopedSpinLock lock(m_lock);
+    for_each_volatile_range([&](const auto& range) {
+        int purged_in_range = 0;
+        auto range_end = range.base + range.count;
+        for (size_t i = range.base; i < range_end; i++) {
+            auto& phys_page = m_physical_pages[i];
+            if (phys_page && !phys_page->is_shared_zero_page())
+                ++purged_in_range;
+            phys_page = MM.shared_zero_page();
+        }
 
+        if (purged_in_range > 0) {
+            purged_page_count += purged_in_range;
+            set_was_purged(range);
+            for_each_region([&](auto& region) {
+                if (&region.vmobject() == this) {
+                    if (auto owner = region.get_owner()) {
+                        // we need to hold a reference the process here (if there is one) as we may not own this region
+                        klog() << "Purged " << purged_in_range << " pages from region " << region.name() << " owned by " << *owner << " at " << region.vaddr_from_page_index(range.base) << " - " << region.vaddr_from_page_index(range.base + range.count);
+                    } else {
+                        klog() << "Purged " << purged_in_range << " pages from region " << region.name() << " (no ownership) at " << region.vaddr_from_page_index(range.base) << " - " << region.vaddr_from_page_index(range.base + range.count);
+                    }
+                    region.remap_page_range(range.base, range.count);
+                }
+            });
+        }
+        return IterationDecision::Continue;
+    });
     return purged_page_count;
 }
 
+void PurgeableVMObject::register_purgeable_page_ranges(PurgeablePageRanges& purgeable_page_ranges)
+{
+    ScopedSpinLock lock(m_lock);
+    ASSERT(!m_purgeable_ranges.contains_slow(&purgeable_page_ranges));
+    m_purgeable_ranges.append(&purgeable_page_ranges);
+}
+
+void PurgeableVMObject::unregister_purgeable_page_ranges(PurgeablePageRanges& purgeable_page_ranges)
+{
+    ScopedSpinLock lock(m_lock);
+    for (size_t i = 0; i < m_purgeable_ranges.size(); i++) {
+        if (m_purgeable_ranges[i] != &purgeable_page_ranges)
+            continue;
+        m_purgeable_ranges.remove(i);
+        return;
+    }
+    ASSERT_NOT_REACHED();
+}
+
+bool PurgeableVMObject::is_any_volatile() const
+{
+    ScopedSpinLock lock(m_lock);
+    for (auto& volatile_ranges : m_purgeable_ranges) {
+        ScopedSpinLock lock(volatile_ranges->m_volatile_ranges_lock);
+        if (!volatile_ranges->is_empty())
+            return true;
+    }
+    return false;
+}
+
 }

Kernel/VM/PurgeableVMObject.h

@@ -26,10 +26,159 @@
 
 #pragma once
 
+#include <Kernel/SpinLock.h>
 #include <Kernel/VM/AnonymousVMObject.h>
 
 namespace Kernel {
 
+struct VolatilePageRange {
+    size_t base { 0 };
+    size_t count { 0 };
+    bool was_purged { false };
+
+    bool is_empty() const { return count == 0; }
+
+    bool intersects(const VolatilePageRange& other) const
+    {
+        return other.base < base + count || other.base + other.count > base;
+    }
+
+    bool intersects_or_adjacent(const VolatilePageRange& other) const
+    {
+        return other.base <= base + count || other.base + other.count >= base;
+    }
+
+    bool contains(const VolatilePageRange& other) const
+    {
+        return base <= other.base && base + count >= other.base + other.count;
+    }
+
+    VolatilePageRange intersected(const VolatilePageRange& other) const
+    {
+        auto b = max(base, other.base);
+        auto e = min(base + count, other.base + other.count);
+        if (b >= e)
+            return {};
+        return { b, e - b, was_purged };
+    }
+
+    void combine_intersecting_or_adjacent(const VolatilePageRange& other)
+    {
+        ASSERT(intersects_or_adjacent(other));
+        if (base <= other.base) {
+            count = (other.base - base) + other.count;
+        } else {
+            count = (base - other.base) + count;
+            base = other.base;
+        }
+        was_purged |= other.was_purged;
+    }
+
+    void subtract_intersecting(const VolatilePageRange& other)
+    {
+        if (!intersects(other))
+            return;
+        if (other.contains(*this)) {
+            count = 0;
+            return;
+        }
+        if (base <= other.base) {
+            count = (other.base - base);
+        } else {
+            auto new_base = other.base + other.count;
+            count = (base + count) - new_base;
+            base = new_base;
+        }
+    }
+
+    bool range_equals(const VolatilePageRange& other) const
+    {
+        return base == other.base && count == other.count;
+    }
+    bool operator==(const VolatilePageRange& other) const
+    {
+        return base == other.base && count == other.count && was_purged == other.was_purged;
+    }
+    bool operator!=(const VolatilePageRange& other) const
+    {
+        return base != other.base || count != other.count || was_purged != other.was_purged;
+    }
+};
+
+class VolatilePageRanges {
+public:
+    VolatilePageRanges(const VolatilePageRange& total_range)
+        : m_total_range(total_range)
+    {
+    }
+    VolatilePageRanges(const VolatilePageRanges& other)
+        : m_ranges(other.m_ranges)
+        , m_total_range(other.m_total_range)
+    {
+    }
+
+    bool is_empty() const { return m_ranges.is_empty(); }
+    void clear() { m_ranges.clear(); }
+
+    bool is_all() const
+    {
+        if (m_ranges.size() != 1)
+            return false;
+        return m_ranges[0] == m_total_range;
+    }
+
+    void set_all()
+    {
+        if (m_ranges.size() != 1)
+            m_ranges = { m_total_range };
+        else
+            m_ranges[0] = m_total_range;
+    }
+
+    bool intersects(const VolatilePageRange&) const;
+
+    bool add(const VolatilePageRange&);
+    bool remove(const VolatilePageRange&, bool&);
+
+    Vector<VolatilePageRange>& ranges() { return m_ranges; }
+    const Vector<VolatilePageRange>& ranges() const { return m_ranges; }
+
+private:
+    Vector<VolatilePageRange> m_ranges;
+    VolatilePageRange m_total_range;
+};
+
+class PurgeableVMObject;
+
+class PurgeablePageRanges {
+    friend class PurgeableVMObject;
+public:
+    PurgeablePageRanges(const VMObject&);
+
+    void set_purgeable_page_ranges(const PurgeablePageRanges& other)
+    {
+        if (this == &other)
+            return;
+        ScopedSpinLock lock(m_volatile_ranges_lock);
+        ScopedSpinLock other_lock(other.m_volatile_ranges_lock);
+        m_volatile_ranges = other.m_volatile_ranges;
+        return;
+    }
+
+    bool add_volatile_range(const VolatilePageRange& range);
+    bool remove_volatile_range(const VolatilePageRange& range, bool& was_purged);
+    bool is_volatile_range(const VolatilePageRange& range) const;
+
+    bool is_empty() const { return m_volatile_ranges.is_empty(); }
+
+    void set_was_purged(const VolatilePageRange&);
+
+    const VolatilePageRanges& volatile_ranges() const { return m_volatile_ranges; }
+protected:
+    VolatilePageRanges m_volatile_ranges;
+    mutable SpinLock<u8> m_volatile_ranges_lock;
+};
+
 class PurgeableVMObject final : public AnonymousVMObject {
 public:
     virtual ~PurgeableVMObject() override;
@@ -37,14 +186,49 @@ public:
     static NonnullRefPtr<PurgeableVMObject> create_with_size(size_t);
     virtual NonnullRefPtr<VMObject> clone() override;
 
+    void register_purgeable_page_ranges(PurgeablePageRanges&);
+    void unregister_purgeable_page_ranges(PurgeablePageRanges&);
+
     int purge();
     int purge_with_interrupts_disabled(Badge<MemoryManager>);
 
-    bool was_purged() const { return m_was_purged; }
-    void set_was_purged(bool b) { m_was_purged = b; }
+    bool is_any_volatile() const;
 
-    bool is_volatile() const { return m_volatile; }
-    void set_volatile(bool b) { m_volatile = b; }
+    template<typename F>
+    IterationDecision for_each_volatile_range(F f)
+    {
+        ASSERT(m_lock.is_locked());
+        // This is a little ugly. Basically, we're trying to find the
+        // volatile ranges that all share, because those are the only
+        // pages we can actually purge
+        for (auto* purgeable_range : m_purgeable_ranges) {
+            for (auto& r1 : purgeable_range->volatile_ranges().ranges()) {
+                VolatilePageRange range(r1);
+                for (auto* purgeable_range2 : m_purgeable_ranges) {
+                    if (purgeable_range2 == purgeable_range)
+                        continue;
+                    if (purgeable_range2->is_empty()) {
+                        // If just one doesn't allow any purging, we can
+                        // immediately bail
+                        return IterationDecision::Continue;
+                    }
+                    for (const auto& r2 : purgeable_range2->volatile_ranges().ranges()) {
+                        range = range.intersected(r2);
+                        if (range.is_empty())
+                            break;
+                    }
+                    if (range.is_empty())
+                        break;
+                }
+                if (range.is_empty())
+                    continue;
+                IterationDecision decision = f(range);
+                if (decision != IterationDecision::Continue)
+                    return decision;
+            }
+        }
+        return IterationDecision::Continue;
+    }
 
 private:
     explicit PurgeableVMObject(size_t);
@@ -53,6 +237,7 @@ private:
     virtual const char* class_name() const override { return "PurgeableVMObject"; }
 
     int purge_impl();
+    void set_was_purged(const VolatilePageRange&);
 
     PurgeableVMObject& operator=(const PurgeableVMObject&) = delete;
     PurgeableVMObject& operator=(PurgeableVMObject&&) = delete;
@@ -60,8 +245,8 @@ private:
 
     virtual bool is_purgeable() const override { return true; }
 
-    bool m_was_purged { false };
-    bool m_volatile { false };
+    Vector<PurgeablePageRanges*> m_purgeable_ranges;
+    mutable SpinLock<u8> m_lock;
 };
 
 }

Kernel/VM/Region.cpp

@@ -32,6 +32,7 @@
 #include <Kernel/VM/AnonymousVMObject.h>
 #include <Kernel/VM/MemoryManager.h>
 #include <Kernel/VM/PageDirectory.h>
+#include <Kernel/VM/PurgeableVMObject.h>
 #include <Kernel/VM/Region.h>
 #include <Kernel/VM/SharedInodeVMObject.h>
 
@@ -41,7 +42,8 @@
 namespace Kernel {
 
 Region::Region(const Range& range, NonnullRefPtr<VMObject> vmobject, size_t offset_in_vmobject, const String& name, u8 access, bool cacheable, bool kernel)
-    : m_range(range)
+    : PurgeablePageRanges(vmobject)
+    , m_range(range)
     , m_offset_in_vmobject(offset_in_vmobject)
     , m_vmobject(move(vmobject))
     , m_name(name)
@@ -49,11 +51,14 @@ Region::Region(const Range& range, NonnullRefPtr<VMObject> vmobject, size_t offs
     , m_cacheable(cacheable)
     , m_kernel(kernel)
 {
+    register_purgeable_page_ranges();
     MM.register_region(*this);
 }
 
 Region::~Region()
 {
+    unregister_purgeable_page_ranges();
+
     // Make sure we disable interrupts so we don't get interrupted between unmapping and unregistering.
     // Unmapping the region will give the VM back to the RangeAllocator, so an interrupt handler would
     // find the address<->region mappings in an invalid state there.
@@ -62,9 +67,26 @@ Region::~Region()
         unmap(ShouldDeallocateVirtualMemoryRange::Yes);
         ASSERT(!m_page_directory);
     }
+
     MM.unregister_region(*this);
 }
 
+void Region::register_purgeable_page_ranges()
+{
+    if (m_vmobject->is_purgeable()) {
+        auto& vmobject = static_cast<PurgeableVMObject&>(*m_vmobject);
+        vmobject.register_purgeable_page_ranges(*this);
+    }
+}
+
+void Region::unregister_purgeable_page_ranges()
+{
+    if (m_vmobject->is_purgeable()) {
+        auto& vmobject = static_cast<PurgeableVMObject&>(*m_vmobject);
+        vmobject.unregister_purgeable_page_ranges(*this);
+    }
+}
+
 NonnullOwnPtr<Region> Region::clone()
 {
     ASSERT(Process::current());
@@ -74,7 +96,8 @@ NonnullOwnPtr<Region> Region::clone()
         ASSERT(m_mmap);
         ASSERT(!m_shared);
         ASSERT(vmobject().is_anonymous());
-        auto zeroed_region = Region::create_user_accessible(m_range, AnonymousVMObject::create_with_size(size()), 0, m_name, m_access);
+        auto zeroed_region = Region::create_user_accessible(get_owner().ptr(), m_range, AnonymousVMObject::create_with_size(size()), 0, m_name, m_access);
+        zeroed_region->set_purgeable_page_ranges(*this);
         zeroed_region->set_mmap(m_mmap);
         zeroed_region->set_inherit_mode(m_inherit_mode);
         return zeroed_region;
@@ -89,7 +112,8 @@ NonnullOwnPtr<Region> Region::clone()
             ASSERT(vmobject().is_shared_inode());
 
         // Create a new region backed by the same VMObject.
-        auto region = Region::create_user_accessible(m_range, m_vmobject, m_offset_in_vmobject, m_name, m_access);
+        auto region = Region::create_user_accessible(get_owner().ptr(), m_range, m_vmobject, m_offset_in_vmobject, m_name, m_access);
+        region->set_purgeable_page_ranges(*this);
         region->set_mmap(m_mmap);
         region->set_shared(m_shared);
         return region;
@@ -104,7 +128,8 @@ NonnullOwnPtr<Region> Region::clone()
     // Set up a COW region. The parent (this) region becomes COW as well!
     ensure_cow_map().fill(true);
     remap();
-    auto clone_region = Region::create_user_accessible(m_range, m_vmobject->clone(), m_offset_in_vmobject, m_name, m_access);
+    auto clone_region = Region::create_user_accessible(get_owner().ptr(), m_range, m_vmobject->clone(), m_offset_in_vmobject, m_name, m_access);
+    clone_region->set_purgeable_page_ranges(*this);
     clone_region->ensure_cow_map();
     if (m_stack) {
         ASSERT(is_readable());
@@ -116,6 +141,59 @@ NonnullOwnPtr<Region> Region::clone()
     return clone_region;
 }
 
+void Region::set_vmobject(NonnullRefPtr<VMObject>&& obj)
+{
+    if (m_vmobject.ptr() == obj.ptr())
+        return;
+    unregister_purgeable_page_ranges();
+    m_vmobject = move(obj);
+    register_purgeable_page_ranges();
+}
+
+bool Region::is_volatile(VirtualAddress vaddr, size_t size) const
+{
+    if (!m_vmobject->is_purgeable())
+        return false;
+
+    auto offset_in_vmobject = vaddr.get() - (this->vaddr().get() - m_offset_in_vmobject);
+    size_t first_page_index = PAGE_ROUND_DOWN(offset_in_vmobject) / PAGE_SIZE;
+    size_t last_page_index = PAGE_ROUND_UP(offset_in_vmobject + size) / PAGE_SIZE;
+    return is_volatile_range({ first_page_index, last_page_index - first_page_index });
+}
+
+auto Region::set_volatile(VirtualAddress vaddr, size_t size, bool is_volatile, bool& was_purged) -> SetVolatileError
+{
+    was_purged = false;
+    if (!m_vmobject->is_purgeable())
+        return SetVolatileError::NotPurgeable;
+
+    auto offset_in_vmobject = vaddr.get() - (this->vaddr().get() - m_offset_in_vmobject);
+    if (is_volatile) {
+        // If marking pages as volatile, be prudent by not marking
+        // partial pages volatile to prevent potentially non-volatile
+        // data to be discarded. So rund up the first page and round
+        // down the last page.
+        size_t first_page_index = PAGE_ROUND_UP(offset_in_vmobject) / PAGE_SIZE;
+        size_t last_page_index = PAGE_ROUND_DOWN(offset_in_vmobject + size) / PAGE_SIZE;
+        if (first_page_index != last_page_index)
+            add_volatile_range({ first_page_index, last_page_index - first_page_index });
+    } else {
+        // If marking pages as non-volatile, round down the first page
+        // and round up the last page to make sure the beginning and
+        // end of the range doesn't inadvertedly get discarded.
+        size_t first_page_index = PAGE_ROUND_DOWN(offset_in_vmobject) / PAGE_SIZE;
+        size_t last_page_index = PAGE_ROUND_UP(offset_in_vmobject + size) / PAGE_SIZE;
+        if (remove_volatile_range({ first_page_index, last_page_index - first_page_index }, was_purged)) {
+            // Attempt to remap the page range. We want to make sure we have
+            // enough memory, if not we need to inform the caller of that
+            // fact
+            if (!remap_page_range(first_page_index, last_page_index - first_page_index))
+                return SetVolatileError::OutOfMemory;
+        }
+    }
+    return SetVolatileError::Success;
+}
+
 bool Region::commit()
 {
     ScopedSpinLock lock(s_mm_lock);
@@ -190,9 +268,11 @@ size_t Region::amount_shared() const
     return bytes;
 }
 
-NonnullOwnPtr<Region> Region::create_user_accessible(const Range& range, NonnullRefPtr<VMObject> vmobject, size_t offset_in_vmobject, const StringView& name, u8 access, bool cacheable)
+NonnullOwnPtr<Region> Region::create_user_accessible(Process* owner, const Range& range, NonnullRefPtr<VMObject> vmobject, size_t offset_in_vmobject, const StringView& name, u8 access, bool cacheable)
 {
     auto region = make<Region>(range, move(vmobject), offset_in_vmobject, name, access, cacheable, false);
+    if (owner)
+        region->m_owner = owner->make_weak_ptr();
     region->m_user_accessible = true;
     return region;
 }
@@ -259,6 +339,25 @@ bool Region::map_individual_page_impl(size_t page_index)
     return true;
 }
 
+bool Region::remap_page_range(size_t page_index, size_t page_count)
+{
+    bool success = true;
+    ScopedSpinLock lock(s_mm_lock);
+    ASSERT(m_page_directory);
+    ScopedSpinLock page_lock(m_page_directory->get_lock());
+    size_t index = page_index;
+    while (index < page_index + page_count) {
+        if (!map_individual_page_impl(index)) {
+            success = false;
+            break;
+        }
+        index++;
+    }
+    if (index > page_index)
+        MM.flush_tlb(vaddr_from_page_index(page_index), index - page_index);
+    return success;
+}
+
 bool Region::remap_page(size_t page_index, bool with_flush)
 {
     ScopedSpinLock lock(s_mm_lock);
@@ -534,4 +633,9 @@ PageFaultResponse Region::handle_inode_fault(size_t page_index_in_region)
     return PageFaultResponse::Continue;
 }
 
+RefPtr<Process> Region::get_owner()
+{
+    return m_owner.strong_ref();
+}
+
 }

Kernel/VM/Region.h

@@ -28,9 +28,11 @@
 
 #include <AK/InlineLinkedList.h>
 #include <AK/String.h>
+#include <AK/WeakPtr.h>
 #include <AK/Weakable.h>
 #include <Kernel/Arch/i386/CPU.h>
 #include <Kernel/Heap/SlabAllocator.h>
+#include <Kernel/VM/PurgeableVMObject.h>
 #include <Kernel/VM/RangeAllocator.h>
 #include <Kernel/VM/VMObject.h>
 
@@ -47,7 +49,8 @@ enum class PageFaultResponse {
 
 class Region final
     : public InlineLinkedListNode<Region>
-    , public Weakable<Region> {
+    , public Weakable<Region>
+    , public PurgeablePageRanges {
     friend class MemoryManager;
 
     MAKE_SLAB_ALLOCATED(Region)
@@ -63,7 +66,7 @@ public:
         ZeroedOnFork,
     };
 
-    static NonnullOwnPtr<Region> create_user_accessible(const Range&, NonnullRefPtr<VMObject>, size_t offset_in_vmobject, const StringView& name, u8 access, bool cacheable = true);
+    static NonnullOwnPtr<Region> create_user_accessible(Process*, const Range&, NonnullRefPtr<VMObject>, size_t offset_in_vmobject, const StringView& name, u8 access, bool cacheable = true);
     static NonnullOwnPtr<Region> create_kernel_only(const Range&, NonnullRefPtr<VMObject>, size_t offset_in_vmobject, const StringView& name, u8 access, bool cacheable = true);
 
     ~Region();
@@ -83,7 +86,7 @@ public:
 
     const VMObject& vmobject() const { return *m_vmobject; }
     VMObject& vmobject() { return *m_vmobject; }
-    void set_vmobject(NonnullRefPtr<VMObject>&& obj) { m_vmobject = obj; }
+    void set_vmobject(NonnullRefPtr<VMObject>&&);
 
     bool is_shared() const { return m_shared; }
     void set_shared(bool shared) { m_shared = shared; }
@@ -190,6 +193,18 @@ public:
 
     void set_inherit_mode(InheritMode inherit_mode) { m_inherit_mode = inherit_mode; }
 
+    bool remap_page_range(size_t page_index, size_t page_count);
+
+    bool is_volatile(VirtualAddress vaddr, size_t size) const;
+    enum class SetVolatileError {
+        Success = 0,
+        NotPurgeable,
+        OutOfMemory
+    };
+    SetVolatileError set_volatile(VirtualAddress vaddr, size_t size, bool is_volatile, bool& was_purged);
+
+    RefPtr<Process> get_owner();
+
 private:
     Bitmap& ensure_cow_map() const;
 
@@ -210,6 +225,9 @@ private:
 
     bool map_individual_page_impl(size_t page_index);
 
+    void register_purgeable_page_ranges();
+    void unregister_purgeable_page_ranges();
+
     RefPtr<PageDirectory> m_page_directory;
     Range m_range;
     size_t m_offset_in_vmobject { 0 };
@@ -224,6 +242,7 @@ private:
     bool m_mmap : 1 { false };
     bool m_kernel : 1 { false };
     mutable OwnPtr<Bitmap> m_cow_map;
+    WeakPtr<Process> m_owner;
 };
 
 inline unsigned prot_to_region_access_flags(int prot)

Meta/CMake/all_the_debug_macros.cmake

@@ -156,6 +156,7 @@ add_compile_definitions("UPDATE_COALESCING_DEBUG")
 add_compile_definitions("VERY_DEBUG")
 add_compile_definitions("VFS_DEBUG")
 add_compile_definitions("VMWAREBACKDOOR_DEBUG")
+add_compile_definitions("VOLATILE_PAGE_RANGES_DEBUG")
 add_compile_definitions("VRA_DEBUG")
 add_compile_definitions("WAITBLOCK_DEBUG")
 add_compile_definitions("WAITQUEUE_DEBUG")