Keep the line starts delta encoded in the VM heap as well - take 2

This reverts commit adf04d0ac6.

The original change used macros from stdint.h and it failed build on
the linux sdk bot:
https://uberchromegw.corp.google.com/i/client.dart/builders/dart-sdk-linux-be/builds/17348

In the newer version, constants from runtime/platform/globals.h are used
instead of the macros from stdint.h. Constants for int8 have been added as
they were not present already.

Change-Id: Ia4f27613fa5dca2cd6dbfeb37715ea70a85cec95
Reviewed-on: https://dart-review.googlesource.com/25083
Reviewed-by: Siva Annamalai <asiva@google.com>
Commit-Queue: Siva Chandra <sivachandra@google.com>

runtime/platform/globals.h

@@ -424,6 +424,9 @@ typedef simd128_value_t fpu_register_t;
   (((static_cast<uint64_t>(a) << 32) + 0x##b##u))
 
 // Integer constants.
+const int8_t kMinInt8 = 0x80;
+const int8_t kMaxInt8 = 0x7F;
+const uint8_t kMaxUint8 = 0xFF;
 const int16_t kMinInt16 = 0x8000;
 const int16_t kMaxInt16 = 0x7FFF;
 const uint16_t kMaxUint16 = 0xFFFF;

runtime/vm/compiler/frontend/kernel_binary_flowgraph.cc

@@ -9191,19 +9191,51 @@ String& StreamingFlowGraphBuilder::GetSourceFor(intptr_t index) {
 }
 
 RawTypedData* StreamingFlowGraphBuilder::GetLineStartsFor(intptr_t index) {
+  // Line starts are delta encoded. So get the max delta first so that we
+  // can store them as tighly as possible.
   AlternativeReadingScope alt(reader_);
   SetOffset(GetOffsetForSourceInfo(index));
-  SkipBytes(ReadUInt());       // skip uri.
+  SkipBytes(ReadUInt());                         // skip uri.
-  SkipBytes(ReadUInt());       // skip source.
+  SkipBytes(ReadUInt());                         // skip source.
-  intptr_t size = ReadUInt();  // read line starts length.
+  const intptr_t line_start_count = ReadUInt();  // read number of line start
+                                                 // entries.
+  MallocGrowableArray<int32_t> line_starts_array;
 
-  TypedData& line_starts_data = TypedData::Handle(
+  intptr_t max_delta = 0;
-      Z, TypedData::New(kTypedDataInt32ArrayCid, size, Heap::kOld));
+  for (intptr_t i = 0; i < line_start_count; ++i) {
-  intptr_t previous_line_start = 0;
+    int32_t delta = ReadUInt();
-  for (intptr_t j = 0; j < size; ++j) {
+    line_starts_array.Add(delta);
-    intptr_t line_start = ReadUInt() + previous_line_start;
+    if (delta > max_delta) {
-    line_starts_data.SetInt32(j * 4, line_start);
+      max_delta = delta;
-    previous_line_start = line_start;
+    }
+  }
+
+  intptr_t cid;
+  if (max_delta <= kMaxInt8) {
+    cid = kTypedDataInt8ArrayCid;
+  } else if (max_delta <= kMaxInt16) {
+    cid = kTypedDataInt16ArrayCid;
+  } else {
+    cid = kTypedDataInt32ArrayCid;
+  }
+
+  TypedData& line_starts_data =
+      TypedData::Handle(Z, TypedData::New(cid, line_start_count, Heap::kOld));
+  for (intptr_t j = 0; j < line_start_count; ++j) {
+    int32_t line_start = line_starts_array[j];
+    switch (cid) {
+      case kTypedDataInt8ArrayCid:
+        line_starts_data.SetInt8(j, static_cast<int8_t>(line_start));
+        break;
+      case kTypedDataInt16ArrayCid:
+        line_starts_data.SetInt16(j << 1, static_cast<int16_t>(line_start));
+        break;
+      case kTypedDataInt32ArrayCid:
+        line_starts_data.SetInt32(j << 2, line_start);
+        break;
+      default:
+        UNREACHABLE();
+    }
   }
   return line_starts_data.raw();
 }

runtime/vm/kernel.cc

@@ -28,6 +28,112 @@ bool FieldHasFunctionLiteralInitializer(const Field& field,
   return field_helper.FieldHasFunctionLiteralInitializer(start, end);
 }
 
+KernelLineStartsReader::KernelLineStartsReader(
+    const dart::TypedData& line_starts_data,
+    dart::Zone* zone)
+    : line_starts_data_(line_starts_data) {
+  TypedDataElementType type = line_starts_data_.ElementType();
+  if (type == kInt8ArrayElement) {
+    helper_ = new KernelInt8LineStartsHelper();
+  } else if (type == kInt16ArrayElement) {
+    helper_ = new KernelInt16LineStartsHelper();
+  } else if (type == kInt32ArrayElement) {
+    helper_ = new KernelInt32LineStartsHelper();
+  } else {
+    UNREACHABLE();
+  }
+}
+
+intptr_t KernelLineStartsReader::LineNumberForPosition(
+    intptr_t position) const {
+  intptr_t line_count = line_starts_data_.Length();
+  intptr_t current_start = 0;
+  for (intptr_t i = 0; i < line_count; ++i) {
+    current_start += helper_->At(line_starts_data_, i);
+    if (current_start > position) {
+      // If current_start is greater than the desired position, it means that
+      // it is for the line after |position|. However, since line numbers
+      // start at 1, we just return |i|.
+      return i;
+    }
+
+    if (current_start == position) {
+      return i + 1;
+    }
+  }
+  return line_count;
+}
+
+void KernelLineStartsReader::LocationForPosition(intptr_t position,
+                                                 intptr_t* line,
+                                                 intptr_t* col) const {
+  intptr_t line_count = line_starts_data_.Length();
+  intptr_t current_start = 0;
+  intptr_t previous_start = 0;
+  for (intptr_t i = 0; i < line_count; ++i) {
+    current_start += helper_->At(line_starts_data_, i);
+    if (current_start > position) {
+      *line = i;
+      if (col != NULL) {
+        *col = position - previous_start + 1;
+      }
+      return;
+    }
+    if (current_start == position) {
+      *line = i + 1;
+      if (col != NULL) {
+        *col = 1;
+      }
+      return;
+    }
+    previous_start = current_start;
+  }
+
+  // If the start of any of the lines did not cross |position|,
+  // then it means the position falls on the last line.
+  *line = line_count;
+  if (col != NULL) {
+    *col = position - current_start + 1;
+  }
+}
+
+void KernelLineStartsReader::TokenRangeAtLine(
+    intptr_t source_length,
+    intptr_t line_number,
+    TokenPosition* first_token_index,
+    TokenPosition* last_token_index) const {
+  ASSERT(line_number <= line_starts_data_.Length());
+  intptr_t cumulative = 0;
+  for (intptr_t i = 0; i < line_number; ++i) {
+    cumulative += helper_->At(line_starts_data_, i);
+  }
+  *first_token_index = dart::TokenPosition(cumulative);
+  if (line_number == line_starts_data_.Length()) {
+    *last_token_index = dart::TokenPosition(source_length);
+  } else {
+    *last_token_index = dart::TokenPosition(
+        cumulative + helper_->At(line_starts_data_, line_number) - 1);
+  }
+}
+
+int32_t KernelLineStartsReader::KernelInt8LineStartsHelper::At(
+    const dart::TypedData& data,
+    intptr_t index) const {
+  return data.GetInt8(index);
+}
+
+int32_t KernelLineStartsReader::KernelInt16LineStartsHelper::At(
+    const dart::TypedData& data,
+    intptr_t index) const {
+  return data.GetInt16(index << 1);
+}
+
+int32_t KernelLineStartsReader::KernelInt32LineStartsHelper::At(
+    const dart::TypedData& data,
+    intptr_t index) const {
+  return data.GetInt32(index << 2);
+}
+
 }  // namespace kernel
 
 }  // namespace dart

runtime/vm/kernel.h

@@ -114,6 +114,72 @@ class Program {
   DISALLOW_COPY_AND_ASSIGN(Program);
 };
 
+class KernelLineStartsReader {
+ public:
+  KernelLineStartsReader(const dart::TypedData& line_starts_data,
+                         dart::Zone* zone);
+
+  ~KernelLineStartsReader() { delete helper_; }
+
+  int32_t DeltaAt(intptr_t index) const {
+    return helper_->At(line_starts_data_, index);
+  }
+
+  intptr_t LineNumberForPosition(intptr_t position) const;
+
+  void LocationForPosition(intptr_t position,
+                           intptr_t* line,
+                           intptr_t* col) const;
+
+  void TokenRangeAtLine(intptr_t source_length,
+                        intptr_t line_number,
+                        dart::TokenPosition* first_token_index,
+                        dart::TokenPosition* last_token_index) const;
+
+ private:
+  class KernelLineStartsHelper {
+   public:
+    KernelLineStartsHelper() {}
+    virtual ~KernelLineStartsHelper() {}
+    virtual int32_t At(const dart::TypedData& data, intptr_t index) const = 0;
+
+   private:
+    DISALLOW_COPY_AND_ASSIGN(KernelLineStartsHelper);
+  };
+
+  class KernelInt8LineStartsHelper : public KernelLineStartsHelper {
+   public:
+    KernelInt8LineStartsHelper() {}
+    virtual int32_t At(const dart::TypedData& data, intptr_t index) const;
+
+   private:
+    DISALLOW_COPY_AND_ASSIGN(KernelInt8LineStartsHelper);
+  };
+
+  class KernelInt16LineStartsHelper : public KernelLineStartsHelper {
+   public:
+    KernelInt16LineStartsHelper() {}
+    virtual int32_t At(const dart::TypedData& data, intptr_t index) const;
+
+   private:
+    DISALLOW_COPY_AND_ASSIGN(KernelInt16LineStartsHelper);
+  };
+
+  class KernelInt32LineStartsHelper : public KernelLineStartsHelper {
+   public:
+    KernelInt32LineStartsHelper() {}
+    virtual int32_t At(const dart::TypedData& data, intptr_t index) const;
+
+   private:
+    DISALLOW_COPY_AND_ASSIGN(KernelInt32LineStartsHelper);
+  };
+
+  const dart::TypedData& line_starts_data_;
+  KernelLineStartsHelper* helper_;
+
+  DISALLOW_COPY_AND_ASSIGN(KernelLineStartsReader);
+};
+
 ParsedFunction* ParseStaticFieldInitializer(Zone* zone, const Field& field);
 
 bool FieldHasFunctionLiteralInitializer(const Field& field,

runtime/vm/object.cc

@@ -31,6 +31,7 @@
 #include "vm/hash_table.h"
 #include "vm/heap.h"
 #include "vm/isolate_reload.h"
+#include "vm/kernel.h"
 #include "vm/native_symbol.h"
 #include "vm/object_store.h"
 #include "vm/parser.h"
@@ -9224,18 +9225,21 @@ RawGrowableObjectArray* Script::GenerateLineNumberArray() const {
       info.Add(line_separator);  // New line.
       return info.raw();
     }
+#if !defined(DART_PRECOMPILED_RUNTIME)
     intptr_t line_count = line_starts_data.Length();
     ASSERT(line_count > 0);
     const Array& debug_positions_array = Array::Handle(debug_positions());
     intptr_t token_count = debug_positions_array.Length();
     int token_index = 0;
 
+    kernel::KernelLineStartsReader line_starts_reader(line_starts_data, zone);
+    intptr_t previous_start = 0;
     for (int line_index = 0; line_index < line_count; ++line_index) {
-      intptr_t start = line_starts_data.GetInt32(line_index * 4);
+      intptr_t start = previous_start + line_starts_reader.DeltaAt(line_index);
       // Output the rest of the tokens if we have no next line.
       intptr_t end = TokenPosition::kMaxSourcePos;
       if (line_index + 1 < line_count) {
-        end = line_starts_data.GetInt32((line_index + 1) * 4);
+        end = start + line_starts_reader.DeltaAt(line_index + 1);
       }
       bool first = true;
       while (token_index < token_count) {
@@ -9256,7 +9260,9 @@ RawGrowableObjectArray* Script::GenerateLineNumberArray() const {
         info.Add(value);
         ++token_index;
       }
+      previous_start = start;
     }
+#endif  // !defined(DART_PRECOMPILED_RUNTIME)
     return info.raw();
   }
 
@@ -9490,22 +9496,31 @@ intptr_t Script::GetTokenLineUsingLineStarts(
     set_line_starts(line_starts_data);
   }
 
-  ASSERT(line_starts_data.Length() > 0);
+  if (kind() == RawScript::kKernelTag) {
-  intptr_t offset = target_token_pos.Pos();
+#if !defined(DART_PRECOMPILED_RUNTIME)
-  intptr_t min = 0;
+    kernel::KernelLineStartsReader line_starts_reader(line_starts_data, zone);
-  intptr_t max = line_starts_data.Length() - 1;
+    return line_starts_reader.LineNumberForPosition(target_token_pos.value());
+#else
+    return 0;
+#endif  // !defined(DART_PRECOMPILED_RUNTIME)
+  } else {
+    ASSERT(line_starts_data.Length() > 0);
+    intptr_t offset = target_token_pos.Pos();
+    intptr_t min = 0;
+    intptr_t max = line_starts_data.Length() - 1;
 
-  // Binary search to find the line containing this offset.
+    // Binary search to find the line containing this offset.
-  while (min < max) {
+    while (min < max) {
-    int midpoint = (max - min + 1) / 2 + min;
+      int midpoint = (max - min + 1) / 2 + min;
-    int32_t token_pos = line_starts_data.GetInt32(midpoint * 4);
+      int32_t token_pos = line_starts_data.GetInt32(midpoint * 4);
-    if (token_pos > offset) {
+      if (token_pos > offset) {
-      max = midpoint - 1;
+        max = midpoint - 1;
-    } else {
+      } else {
-      min = midpoint;
+        min = midpoint;
+      }
     }
+    return min + 1;  // Line numbers start at 1.
   }
-  return min + 1;  // Line numbers start at 1.
 }
 
 void Script::GetTokenLocation(TokenPosition token_pos,
@@ -9528,31 +9543,15 @@ void Script::GetTokenLocation(TokenPosition token_pos,
       }
       return;
     }
+#if !defined(DART_PRECOMPILED_RUNTIME)
     ASSERT(line_starts_data.Length() > 0);
-    intptr_t offset = token_pos.value();
+    kernel::KernelLineStartsReader line_starts_reader(line_starts_data, zone);
-    intptr_t min = 0;
+    line_starts_reader.LocationForPosition(token_pos.value(), line, column);
-    intptr_t max = line_starts_data.Length() - 1;
-
-    // Binary search to find the line containing this offset.
-    while (min < max) {
-      intptr_t midpoint = (max - min + 1) / 2 + min;
-
-      int32_t value = line_starts_data.GetInt32(midpoint * 4);
-      if (value > offset) {
-        max = midpoint - 1;
-      } else {
-        min = midpoint;
-      }
-    }
-    *line = min + 1;  // Line numbers start at 1.
-    int32_t min_value = line_starts_data.GetInt32(min * 4);
-    if (column != NULL) {
-      *column = offset - min_value + 1;
-    }
     if (token_len != NULL) {
       // We don't explicitly save this data: Load the source
       // and find it from there.
       const String& source = String::Handle(zone, Source());
+      intptr_t offset = token_pos.value();
       *token_len = 1;
       if (offset < source.Length() &&
           Scanner::IsIdentStartChar(source.CharAt(offset))) {
@@ -9563,6 +9562,7 @@ void Script::GetTokenLocation(TokenPosition token_pos,
         }
       }
     }
+#endif  // !defined(DART_PRECOMPILED_RUNTIME)
     return;
   }
 
@@ -9626,16 +9626,13 @@ void Script::TokenRangeAtLine(intptr_t line_number,
       *last_token_index = TokenPosition::kNoSource;
       return;
     }
-    ASSERT(line_starts_data.Length() >= line_number);
+#if !defined(DART_PRECOMPILED_RUNTIME)
-    int32_t value = line_starts_data.GetInt32((line_number - 1) * 4);
+    kernel::KernelLineStartsReader line_starts_reader(
-    *first_token_index = TokenPosition(value);
+        line_starts_data, Thread::Current()->zone());
-    if (line_starts_data.Length() > line_number) {
+    line_starts_reader.TokenRangeAtLine(String::Handle(Source()).Length(),
-      value = line_starts_data.GetInt32(line_number * 4);
+                                        line_number, first_token_index,
-      *last_token_index = TokenPosition(value - 1);
+                                        last_token_index);
-    } else {
+#endif  // !defined(DART_PRECOMPILED_RUNTIME)
-      // Length of source is last possible token in this script.
-      *last_token_index = TokenPosition(String::Handle(Source()).Length());
-    }
     return;
   }