[kernel] Include urls even when we have no source

Previously, if we didn't include the source code, we wrote the url as
null. This for instance made it impossible to step through mixed in code
(at least when mixed in from the sdk).

This CL includes all used urls. If there's no source, the source is empty,
but the VM then tries to find the proper source to be able to display it
(e.g. the VM already has the sdk source).

This CL further more adds a service test that tests that we can actually
step into mixin in code from the sdk.

Change-Id: Ied9569723e23928769ebc980410aed60be6eaa22
Reviewed-on: https://dart-review.googlesource.com/51621
Commit-Queue: Jens Johansen <jensj@google.com>
Reviewed-by: Vyacheslav Egorov <vegorov@google.com>

pkg/front_end/test/summary_generator_test.dart

@@ -14,9 +14,12 @@ main() {
     var summary = await summarize(['a.dart'], allSources);
     var component = loadComponentFromBytes(summary);
 
-    // Note: the kernel representation always has a null key in the map,
-    // but otherwise no other data is included here.
-    expect(component.uriToSource.keys.single, null);
+    // Note: the kernel representation always includes the Uri entries, but
+    // doesn't include the actual source here.
+    for (Source source in component.uriToSource.values) {
+      expect(source.source.length, 0);
+      expect(source.lineStarts.length, 0);
+    }
   });
 
   test('summary includes declarations, but no method bodies', () async {

pkg/kernel/lib/binary/ast_to_binary.dart

@@ -220,16 +220,9 @@ class BinaryPrinter implements Visitor<void>, BinarySink {
 
   // Returns the new active file uri.
   Uri writeUriReference(Uri uri) {
-    if (_knownSourceUri.contains(uri)) {
-      final int index = _sourceUriIndexer.put(uri);
-      writeUInt30(index);
-      return uri;
-    } else {
-      // This is equivalent to `index = _sourceUriIndexer[null];`.
-      final int index = 0;
-      writeUInt30(index);
-      return null;
-    }
+    final int index = _sourceUriIndexer.put(uri);
+    writeUInt30(index);
+    return uri;
   }
 
   void writeList<T>(List<T> items, void writeItem(T x)) {
@@ -496,8 +489,9 @@ class BinaryPrinter implements Visitor<void>, BinarySink {
     Utf8Encoder utf8Encoder = const Utf8Encoder();
     for (Uri uri in _sourceUriIndexer.index.keys) {
       index[i] = getBufferOffset();
-
-      Source source = uriToSource[uri] ?? new Source(<int>[], const <int>[]);
+      Source source =
+          (_knownSourceUri.contains(uri) ? uriToSource[uri] : null) ??
+              new Source(<int>[], const <int>[]);
 
       writeByteList(utf8Encoder.convert(uri == null ? "" : "$uri"));
       writeByteList(source.source);

pkg/kernel/lib/clone.dart

@@ -406,7 +406,8 @@ class CloneVisitor implements TreeVisitor {
       ..annotations = cloneAnnotations && !node.annotations.isEmpty
           ? node.annotations.map(clone).toList()
           : const <Expression>[]
-      ..flags = node.flags;
+      ..flags = node.flags
+      ..fileEqualsOffset = _cloneFileOffset(node.fileEqualsOffset);
   }
 
   visitFunctionDeclaration(FunctionDeclaration node) {

runtime/observatory/tests/service/step_through_mixin_from_sdk_test.dart

@@ -0,0 +1,65 @@
+// Copyright (c) 2018, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'test_helper.dart';
+import 'service_test_common.dart';
+import 'dart:collection';
+
+const int LINE = 13;
+const String file = "step_through_mixin_from_sdk_test.dart";
+
+code() {
+  Foo foo = new Foo();
+  if (foo.contains(43)) {
+    print("Contains 43!");
+  } else {
+    print("Doesn't contain 43!");
+  }
+}
+
+class Foo extends Object with ListMixin<int> {
+  @override
+  int length = 1;
+
+  @override
+  int operator [](int index) {
+    return 42;
+  }
+
+  @override
+  void operator []=(int index, int value) {}
+}
+
+List<String> stops = [];
+List<String> expected = [
+  "$file:${LINE+0}:17", // on "Foo" (in "new Foo()")
+  "$file:${LINE+1}:11", // on "="
+  "list.dart:105:24", // on parameter to "contains"
+  "list.dart:106:23", // on "length" in "this.length"
+  "list.dart:107:16", // on "=" in "i = 0"
+  "list.dart:107:23", // on "<" in "i < length"
+  "list.dart:108:15", // on "[" in "this[i]"
+  "$file:${LINE+13}:23", // on parameter in "operator []"
+  "$file:${LINE+14}:5", // on "return"
+  "list.dart:108:19", // on "=="
+  "list.dart:109:26", // on "length" in "this.length"
+  "list.dart:109:18", // on "!="
+  "list.dart:107:34", // on "++" in "i++"
+  "list.dart:107:23", // on "<" in "i < length"
+  "list.dart:113:5", // on "return"
+  "$file:${LINE+4}:5", // on "print"
+  "$file:${LINE+6}:1" // on ending '}'
+];
+
+var tests = <IsolateTest>[
+  hasPausedAtStart,
+  setBreakpointAtLine(LINE),
+  runStepIntoThroughProgramRecordingStops(stops),
+  checkRecordedStops(stops, expected, removeDuplicates: true)
+];
+
+main(args) {
+  runIsolateTestsSynchronous(args, tests,
+      testeeConcurrent: code, pause_on_start: true, pause_on_exit: true);
+}

runtime/vm/kernel_loader.cc

@@ -1532,13 +1532,34 @@ const Object& KernelLoader::ClassForScriptAt(const Class& klass,
 
 RawScript* KernelLoader::LoadScriptAt(intptr_t index) {
   const String& uri_string = builder_.SourceTableUriFor(index);
-  const Script& script =
-      Script::Handle(Z, Script::New(uri_string, builder_.GetSourceFor(index),
-                                    RawScript::kKernelTag));
+  String& sources = builder_.GetSourceFor(index);
+  TypedData& line_starts =
+      TypedData::Handle(Z, builder_.GetLineStartsFor(index));
+  if (sources.Length() == 0 && line_starts.Length() == 0 &&
+      uri_string.Length() > 0) {
+    // Entry included only to provide URI - actual source should already exist
+    // in the VM, so try to find it.
+    Library& lib = Library::Handle(Z);
+    Script& script = Script::Handle(Z);
+    const GrowableObjectArray& libs =
+        GrowableObjectArray::Handle(isolate_->object_store()->libraries());
+    for (intptr_t i = 0; i < libs.Length(); i++) {
+      lib ^= libs.At(i);
+      script = lib.LookupScript(uri_string, /* useResolvedUri = */ true);
+      if (!script.IsNull() && script.kind() == RawScript::kKernelTag) {
+        sources ^= script.Source();
+        line_starts ^= script.line_starts();
+        break;
+      }
+    }
+  }
+
+  const Script& script = Script::Handle(
+      Z, Script::New(uri_string, sources, RawScript::kKernelTag));
+  String& script_url = String::Handle();
+  script_url = script.url();
   script.set_kernel_script_index(index);
   script.set_kernel_program_info(kernel_program_info_);
-  const TypedData& line_starts =
-      TypedData::Handle(Z, builder_.GetLineStartsFor(index));
   script.set_line_starts(line_starts);
   script.set_debug_positions(Array::Handle(Array::null()));
   script.set_yield_positions(Array::Handle(Array::null()));

runtime/vm/object.cc

@@ -11097,7 +11097,8 @@ RawArray* Library::LoadedScripts() const {
 
 // TODO(hausner): we might want to add a script dictionary to the
 // library class to make this lookup faster.
-RawScript* Library::LookupScript(const String& url) const {
+RawScript* Library::LookupScript(const String& url,
+                                 bool useResolvedUri /* = false */) const {
   const intptr_t url_length = url.Length();
   if (url_length == 0) {
     return Script::null();
@@ -11108,7 +11109,11 @@ RawScript* Library::LookupScript(const String& url) const {
   const intptr_t num_scripts = scripts.Length();
   for (int i = 0; i < num_scripts; i++) {
     script ^= scripts.At(i);
-    script_url = script.url();
+    if (!useResolvedUri) {
+      script_url = script.url();
+    } else {
+      script_url = script.resolved_url();
+    }
     const intptr_t start_idx = script_url.Length() - url_length;
     if ((start_idx == 0) && url.Equals(script_url)) {
       return script.raw();

runtime/vm/object.h

@@ -3600,6 +3600,8 @@ class Script : public Object {
     return raw_ptr()->tokens_;
   }
 
+  RawTypedData* line_starts() const;
+
   void set_line_starts(const TypedData& value) const;
 
   void set_debug_positions(const Array& value) const;
@@ -3657,7 +3659,6 @@ class Script : public Object {
   void set_kind(RawScript::Kind value) const;
   void set_load_timestamp(int64_t value) const;
   void set_tokens(const TokenStream& value) const;
-  RawTypedData* line_starts() const;
   RawArray* debug_positions() const;
 
   static RawScript* New();
@@ -3807,7 +3808,7 @@ class Library : public Object {
   RawFunction* LookupFunctionAllowPrivate(const String& name) const;
   RawFunction* LookupLocalFunction(const String& name) const;
   RawLibraryPrefix* LookupLocalLibraryPrefix(const String& name) const;
-  RawScript* LookupScript(const String& url) const;
+  RawScript* LookupScript(const String& url, bool useResolvedUri = false) const;
   RawArray* LoadedScripts() const;
 
   // Resolve name in the scope of this library. First check the cache