[vm/aot] Optimize selector offsets for smaller instruction sequences.

The instruction sequences for dispatch table calls have three versions,
depending on the selector offset relative to where the dispatch table
register is pointing:
- Optimal: Zero offset
- Small:   A small range of positive and negative offsets
- Large:   Larger positive offsets
The exact limits for small offsets depend on the target architecture.

This commit changes the dispatch table layout algorithm to favor
offsets corresponding to smaller instruction sequences for selectors
with many callsites.

As a result, == (which is by far the most abundantly called selector)
gets the zero offset, and most commonly used selectors (such as [],
iterator, current and moveNext) usually get small offsets on ARM and
ARM64 targets (where the range of small offsets is reasonably large).

Change-Id: I4be9ef8e709c71681b5743084f280ce1673bc0ad
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/132663
Reviewed-by: Martin Kustermann <kustermann@google.com>

runtime/vm/compiler/aot/dispatch_table_generator.cc

@@ -93,16 +93,10 @@ class CidInterval {
 
 class SelectorRow {
  public:
-  SelectorRow(Zone* zone, int32_t selector_id)
-      : selector_id_(selector_id), class_ranges_(zone, 0), ranges_(zone, 0) {}
+  SelectorRow(Zone* zone, TableSelector* selector)
+      : selector_(selector), class_ranges_(zone, 0), ranges_(zone, 0) {}
 
-  int32_t selector_id() const { return selector_id_; }
-
-  void DefineSelectorImplementationForInterval(classid_t cid,
-                                               int16_t depth,
-                                               const Interval& range,
-                                               const Function& function);
-  bool Finalize();
+  TableSelector* selector() const { return selector_; }
 
   int32_t total_size() const { return total_size_; }
 
@@ -112,14 +106,27 @@ class SelectorRow {
     return class_ranges_;
   }
 
-  int32_t offset() const { return offset_; }
-  void set_offset(int32_t value) { offset_ = value; }
+  void DefineSelectorImplementationForInterval(classid_t cid,
+                                               int16_t depth,
+                                               const Interval& range,
+                                               const Function& function);
+  bool Finalize();
+
+  int32_t CallCount() const { return selector_->call_count; }
+
+  bool IsAllocated() const {
+    return selector_->offset != SelectorMap::kInvalidSelectorOffset;
+  }
+
+  void AllocateAt(int32_t offset) {
+    ASSERT(!IsAllocated());
+    selector_->offset = offset;
+  }
 
   void FillTable(ClassTable* class_table, DispatchTable* table);
 
  private:
-  int32_t selector_id_;
-  int32_t offset_ = SelectorMap::kInvalidSelectorOffset;
+  TableSelector* selector_;
   int32_t total_size_ = 0;
 
   GrowableArray<CidInterval> class_ranges_;
@@ -128,20 +135,23 @@ class SelectorRow {
 
 class RowFitter {
  public:
-  RowFitter() { free_slots_.Add(Interval(0, INT_MAX)); }
+  RowFitter() : first_slot_index_(0) { free_slots_.Add(Interval(0, INT_MAX)); }
 
-  int32_t Fit(SelectorRow* row);
+  // Try to fit a row at the specified offset and return whether it was
+  // successful. If successful, the entries taken up by the row are marked
+  // internally as occupied. If unsuccessful, next_offset is set to the next
+  // potential offset where the row might fit.
+  bool TryFit(SelectorRow* row, int32_t offset, int32_t* next_offset);
+
+  // If the row is not already allocated, try to fit it within the given range
+  // of offsets and allocate it if successful.
+  void FitAndAllocate(SelectorRow* row,
+                      int32_t min_offset,
+                      int32_t max_offset = INT32_MAX);
 
   int32_t TableSize() const { return free_slots_.Last().begin(); }
 
  private:
-  int32_t FindOffset(const GrowableArray<Interval>& ranges,
-                     intptr_t* result_slot_index);
-
-  int32_t MatchRemaining(int32_t offset,
-                         const GrowableArray<Interval>& ranges,
-                         intptr_t slot_index);
-
   intptr_t MoveForwardToCover(const Interval range, intptr_t slot_index);
 
   void UpdateFreeSlots(int32_t offset,
@@ -151,6 +161,7 @@ class RowFitter {
   intptr_t FitInFreeSlot(const Interval range, intptr_t slot_index);
 
   GrowableArray<Interval> free_slots_;
+  intptr_t first_slot_index_;
 };
 
 void SelectorRow::DefineSelectorImplementationForInterval(
@@ -243,94 +254,56 @@ void SelectorRow::FillTable(ClassTable* class_table, DispatchTable* table) {
     if (function->HasCode()) {
       code = function->CurrentCode();
       for (classid_t cid = range.begin(); cid < range.end(); cid++) {
-        table->SetCodeAt(offset_ + cid, code);
+        table->SetCodeAt(selector()->offset + cid, code);
       }
     }
   }
 }
 
-int32_t RowFitter::Fit(SelectorRow* row) {
-  ASSERT(row->ranges().length() > 0);
+void RowFitter::FitAndAllocate(SelectorRow* row,
+                               int32_t min_offset,
+                               int32_t max_offset) {
+  if (row->IsAllocated()) {
+    return;
+  }
+
+  int32_t next_offset;
+
+  int32_t offset = min_offset;
+  while (offset <= max_offset && !TryFit(row, offset, &next_offset)) {
+    offset = next_offset;
+  }
+  if (offset <= max_offset) {
+    row->AllocateAt(offset);
+  }
+}
+
+bool RowFitter::TryFit(SelectorRow* row, int32_t offset, int32_t* next_offset) {
   const GrowableArray<Interval>& ranges = row->ranges();
 
-  intptr_t slot_index;
-  const int32_t offset = FindOffset(ranges, &slot_index);
-  UpdateFreeSlots(offset, ranges, slot_index);
-
-  return offset;
-}
-
-int32_t RowFitter::FindOffset(const GrowableArray<Interval>& ranges,
-                              intptr_t* result_slot_index) {
-  const Interval first_range = ranges[0];
-
-  intptr_t index = 0;
-  int32_t min_start = 0;
-
-  while (index < free_slots_.length() - 1) {
-    const Interval slot = free_slots_[index];
-
-    int32_t start = Utils::Maximum(
-        min_start, Utils::Maximum(slot.begin(), first_range.begin()));
-    int32_t end = slot.end() - first_range.length();
-
-    while (start <= end) {
-      int32_t offset = start - first_range.begin();
-      ASSERT(offset >= 0);
-      ASSERT(slot.Contains(first_range.WithOffset(offset)));
-
-      // If the first block was the only block, we are done.
-      if (ranges.length() == 1) {
-        *result_slot_index = index;
-        return offset;
-      }
-
-      // Found an offset where the first range fits. Now match the
-      // remaining ones.
-      int32_t displacement = MatchRemaining(offset, ranges, index);
-
-      // Displacement is either 0 for a match, or a minimum distance to where
-      // a potential match can happen.
-      if (displacement == 0) {
-        *result_slot_index = index;
-        return offset;
-      }
-
-      start += displacement;
-    }
-
-    min_start = start;
-
-    index++;
+  Interval first_range = ranges[0].WithOffset(offset);
+  if (first_slot_index_ > 0 &&
+      free_slots_[first_slot_index_ - 1].end() >= first_range.end()) {
+    // Trying lower offset than last time. Start over in free slots.
+    first_slot_index_ = 0;
   }
+  first_slot_index_ = MoveForwardToCover(first_range, first_slot_index_);
+  intptr_t slot_index = first_slot_index_;
 
-  ASSERT(index == (free_slots_.length() - 1));
-  const Interval slot = free_slots_[index];
-  ASSERT(slot.end() == INT_MAX);
-
-  // If we are at end, we know it fits.
-  int32_t offset = Utils::Maximum(0, slot.begin() - first_range.begin());
-
-  *result_slot_index = index;
-  return offset;
-}
-
-int32_t RowFitter::MatchRemaining(int32_t offset,
-                                  const GrowableArray<Interval>& ranges,
-                                  intptr_t slot_index) {
-  intptr_t index = 1;
-  intptr_t length = ranges.length();
-
-  for (; index < length; index++) {
-    const Interval range = ranges[index].WithOffset(offset);
-
+  for (intptr_t index = 0; index < ranges.length(); index++) {
+    Interval range = ranges[index].WithOffset(offset);
     slot_index = MoveForwardToCover(range, slot_index);
+    ASSERT(slot_index < free_slots_.length());
     const Interval slot = free_slots_[slot_index];
-
-    if (range.begin() < slot.begin()) return slot.begin() - range.begin();
+    ASSERT(slot.end() >= range.end());
+    if (slot.begin() > range.begin()) {
+      *next_offset = offset + slot.begin() - range.begin();
+      return false;
+    }
   }
 
-  return 0;
+  UpdateFreeSlots(offset, ranges, first_slot_index_);
+  return true;
 }
 
 intptr_t RowFitter::MoveForwardToCover(const Interval range,
@@ -353,7 +326,7 @@ void RowFitter::UpdateFreeSlots(int32_t offset,
 
     // Assert that we have a valid slot.
     ASSERT(slot_index < free_slots_.length());
-    ASSERT(free_slots_[slot_index].begin() < range.end());
+    ASSERT(free_slots_[slot_index].Contains(range));
 
     slot_index = FitInFreeSlot(range, slot_index);
   }
@@ -418,10 +391,6 @@ void SelectorMap::SetSelectorProperties(int32_t sid,
   selectors_[sid].requires_args_descriptor |= requires_args_descriptor;
 }
 
-void SelectorMap::SetSelectorOffset(int32_t sid, int32_t offset) {
-  selectors_[sid].offset = offset;
-}
-
 DispatchTableGenerator::DispatchTableGenerator(Zone* zone)
     : zone_(zone),
       classes_(nullptr),
@@ -564,7 +533,7 @@ void DispatchTableGenerator::SetupSelectorRows() {
   // Initialize selector rows.
   SelectorRow* selector_rows = Z->Alloc<SelectorRow>(num_selectors_);
   for (intptr_t i = 0; i < num_selectors_; i++) {
-    new (&selector_rows[i]) SelectorRow(Z, i);
+    new (&selector_rows[i]) SelectorRow(Z, &selector_map_.selectors_[i]);
   }
 
   // Add implementation intervals to the selector rows for all classes that
@@ -611,20 +580,49 @@ void DispatchTableGenerator::SetupSelectorRows() {
 void DispatchTableGenerator::ComputeSelectorOffsets() {
   ASSERT(table_rows_.length() > 0);
 
-  // Sort the table rows according to size.
-  struct SelectorRowSorter {
+  RowFitter fitter;
+
+  // Sort the table rows according to popularity, descending.
+  struct PopularitySorter {
+    static int Compare(SelectorRow* const* a, SelectorRow* const* b) {
+      return (*b)->CallCount() - (*a)->CallCount();
+    }
+  };
+  table_rows_.Sort(PopularitySorter::Compare);
+
+  // Try to allocate at optimal offset.
+  const int32_t optimal_offset = DispatchTable::OriginElement();
+  for (intptr_t i = 0; i < table_rows_.length(); i++) {
+    fitter.FitAndAllocate(table_rows_[i], optimal_offset, optimal_offset);
+  }
+
+  // Sort the table rows according to popularity / size, descending.
+  struct PopularitySizeRatioSorter {
+    static int Compare(SelectorRow* const* a, SelectorRow* const* b) {
+      return (*b)->CallCount() * (*a)->total_size() -
+             (*a)->CallCount() * (*b)->total_size();
+    }
+  };
+  table_rows_.Sort(PopularitySizeRatioSorter::Compare);
+
+  // Try to allocate at small offsets.
+  const int32_t max_offset = DispatchTable::LargestSmallOffset();
+  for (intptr_t i = 0; i < table_rows_.length(); i++) {
+    fitter.FitAndAllocate(table_rows_[i], 0, max_offset);
+  }
+
+  // Sort the table rows according to size, descending.
+  struct SizeSorter {
     static int Compare(SelectorRow* const* a, SelectorRow* const* b) {
       return (*b)->total_size() - (*a)->total_size();
     }
   };
-  table_rows_.Sort(SelectorRowSorter::Compare);
+  table_rows_.Sort(SizeSorter::Compare);
 
-  RowFitter fitter;
+  // Allocate remaining rows at large offsets.
+  const int32_t min_large_offset = DispatchTable::LargestSmallOffset() + 1;
   for (intptr_t i = 0; i < table_rows_.length(); i++) {
-    SelectorRow* row = table_rows_[i];
-    const int32_t offset = fitter.Fit(row);
-    row->set_offset(offset);
-    selector_map_.SetSelectorOffset(row->selector_id(), offset);
+    fitter.FitAndAllocate(table_rows_[i], min_large_offset);
   }
 
   table_size_ = fitter.TableSize();

runtime/vm/compiler/aot/dispatch_table_generator.h

@@ -50,7 +50,6 @@ class SelectorMap {
   void SetSelectorProperties(int32_t sid,
                              bool on_null_interface,
                              bool requires_args_descriptor);
-  void SetSelectorOffset(int32_t sid, int32_t offset);
 
   int32_t NumIds() const { return selectors_.length(); }
 

runtime/vm/dispatch_table.h

@@ -49,6 +49,24 @@ class DispatchTable {
 #endif
   }
 
+  // The largest offset that can use a more compact instruction sequence.
+  static intptr_t LargestSmallOffset() {
+#if defined(TARGET_ARCH_X64)
+    // Origin + Max positive byte offset / 8
+    return 31;
+#elif defined(TARGET_ARCH_ARM)
+    // Origin + Max positive load offset / 4
+    return 2046;
+#elif defined(TARGET_ARCH_ARM64)
+    // Origin + Max consecutive add immediate value
+    return 8192;
+#else
+    // No AOT on IA32
+    UNREACHABLE();
+    return 0;
+#endif
+  }
+
   // Dispatch table array pointer to put into the dispatch table register.
   uword* ArrayOrigin() const { return &array()[OriginElement()]; }