[ package:dds ] Initial commit of the Dart Development Service, package:dds

- Defined initial interface
- Currently can spawn an HTTP server and forward websocket and HTTP
requests to the VM service
- Simple smoke tests
- Initial documentation in dds_protocol.md and other book keeping
- Enabled analysis on bots

Change-Id: Ia11e9e33fd10b0b4700b704a29e2977341441cec
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/139542
Commit-Queue: Ben Konyi <bkonyi@google.com>
Reviewed-by: Ryan Macnak <rmacnak@google.com>

.dart_tool/package_config.json

@@ -200,6 +200,12 @@
       "packageUri": "lib/",
       "languageVersion": "2.3"
     },
+    {
+      "name": "dds",
+      "rootUri": "../pkg/dds",
+      "packageUri": "lib/",
+      "languageVersion": "2.3"
+    },
     {
       "name": "dev_compiler",
       "rootUri": "../pkg/dev_compiler",
@@ -567,6 +573,12 @@
       "packageUri": "lib/",
       "languageVersion": "1.22"
     },
+    {
+      "name": "shelf_proxy",
+      "rootUri": "../third_party/pkg/shelf_proxy",
+      "packageUri": "lib/",
+      "languageVersion": "2.3"
+    },
     {
       "name": "shelf_static",
       "rootUri": "../third_party/pkg/shelf_static",
@@ -784,4 +796,4 @@
       "languageVersion": "2.0"
     }
   ]
-}
+}

.packages

@@ -36,6 +36,7 @@ dart_style:third_party/pkg_tested/dart_style/lib
 dartdev:pkg/dartdev/lib
 dartdoc:third_party/pkg/dartdoc/lib
 dartfix:pkg/dartfix/lib
+dds:pkg/dds/lib
 dev_compiler:pkg/dev_compiler/lib
 diagnostic:pkg/diagnostic/lib
 expect:pkg/expect/lib
@@ -85,6 +86,7 @@ resource:third_party/pkg/resource/lib
 sdk_library_metadata:sdk/lib/_internal/sdk_library_metadata/lib
 shelf:third_party/pkg/shelf/lib
 shelf_packages_handler:third_party/pkg/shelf_packages_handler/lib
+shelf_proxy:third_party/pkg/shelf_proxy/lib
 shelf_static:third_party/pkg/shelf_static/lib
 shelf_web_socket:third_party/pkg/shelf_web_socket/lib
 smith:pkg/smith/lib

DEPS

@@ -127,6 +127,7 @@ vars = {
   "rust_revision": "60960a260f7b5c695fd0717311d72ce62dd4eb43",
   "shelf_static_rev": "v0.2.8",
   "shelf_packages_handler_tag": "2.0.0",
+  "shelf_proxy_tag": "0.1.0+7",
   "shelf_tag": "0.7.3+3",
   "shelf_web_socket_tag": "0.2.2+3",
   "source_map_stack_trace_tag": "2.0.0",
@@ -148,7 +149,7 @@ vars = {
   "usage_tag": "3.4.0",
   "watcher_rev": "0.9.7+14",
   "web_components_rev": "8f57dac273412a7172c8ade6f361b407e2e4ed02",
-  "web_socket_channel_tag": "1.0.9",
+  "web_socket_channel_tag": "1.0.15",
   "WebCore_rev": "fb11e887f77919450e497344da570d780e078bc8",
   "yaml_tag": "2.2.0",
   "zlib_rev": "c44fb7248079cc3d5563b14b3f758aee60d6b415",
@@ -365,6 +366,8 @@ deps = {
   Var("dart_root") + "/third_party/pkg/shelf_packages_handler":
       Var("dart_git") + "shelf_packages_handler.git"
       + "@" + Var("shelf_packages_handler_tag"),
+  Var("dart_root") + "/third_party/pkg/shelf_proxy":
+      Var("dart_git") + "shelf_proxy.git" + "@" + Var("shelf_proxy_tag"),
   Var("dart_root") + "/third_party/pkg/shelf_static":
       Var("dart_git") + "shelf_static.git" + "@" + Var("shelf_static_rev"),
   Var("dart_root") + "/third_party/pkg/shelf_web_socket":

pkg/dds/.gitignore

@@ -0,0 +1,5 @@
+# See https://dart.dev/guides/libraries/private-files
+.dart_tool/
+.packages
+build/
+pubspec.lock

pkg/dds/CHANGELOG.md

@@ -0,0 +1,3 @@
+# 0.1.0-dev
+
+- Initial release.

pkg/dds/LICENSE

@@ -0,0 +1,26 @@
+Copyright 2020, the Dart project authors. All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above
+      copyright notice, this list of conditions and the following
+      disclaimer in the documentation and/or other materials provided
+      with the distribution.
+    * Neither the name of Google Inc. nor the names of its
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

pkg/dds/README.md

@@ -0,0 +1,10 @@
+_This package is a work in-progress and the API may not be stable. Stay tuned for updates._
+
+A package used to spawn the Dart Developer Service (DDS), which is used to communicate with a Dart VM Service instance and provide extended functionality to the core VM Service Protocol.
+
+# Functionality
+
+Existing VM Service clients can issue both HTTP and websocket requests to a running DDS instance as if it were an instance of the VM Service itself. If a request corresponds to an RPC defined in the [VM Service Protocol][service-protocol], DDS will forward the request and return the response from the VM Service. Requests corresponding to an RPC defined in the [DDS Protocol][dds-protocol] will be handled directly by the DDS instance.
+
+[dds-protocol]: dds_protocol.md
+[service-protocol]: https://github.com/dart-lang/sdk/blob/master/runtime/vm/service/service.md

pkg/dds/analysis_options.yaml

@@ -0,0 +1,6 @@
+include: package:pedantic/analysis_options.1.7.0.yaml
+
+linter:
+  rules:
+    - directives_ordering
+    - prefer_generic_function_type_aliases

pkg/dds/dds_protocol.md

@@ -0,0 +1,56 @@
+# Dart Development Service Protocol 0.x
+
+This document describes _version 0.x_ of the Dart Development Service Protocol.
+This protocol is an extension of the Dart VM Service Protocol and implements it
+in it's entirety. For details on the VM Service Protocol, see the [Dart VM Service Protocol Specification][service-protocol].
+
+The Service Protocol uses [JSON-RPC 2.0][].
+
+[JSON-RPC 2.0]: http://www.jsonrpc.org/specification
+
+
+**Table of Contents**
+
+- [RPCs, Requests, and Responses](#rpcs-requests-and-responses)
+- [Events](#events)
+- [Types](#types)
+- [IDs and Names](#ids-and-names)
+- [Revision History](#revision-history)
+
+## RPCs, Requests, and Responses
+
+See the corresponding section in the VM Service protocol [here][service-protocol-rpcs-requests-and-responses].
+
+## Events
+
+See the corresponding section in the VM Service protocol [here][service-protocol-events].
+
+## Binary Events
+
+See the corresponding section in the VM Service protocol [here][service-protocol-binary-events].
+
+## Types
+
+See the corresponding section in the VM Service protocol [here][service-protocol-types].
+
+## IDs and Names
+
+See the corresponding section in the VM Service protocol [here][service-protocol-ids-and-names].
+
+## Public RPCs
+
+The DDS Protocol supports all [public RPCs defined in the VM Service protocol][service-protocol-public-rpcs].
+
+## Revision History
+
+version | comments
+------- | --------
+0.x | Initial revision
+
+[service-protocol]: https://github.com/dart-lang/sdk/blob/master/runtime/vm/service/service.md
+[service-protocol-rpcs-requests-and-responses]: https://github.com/dart-lang/sdk/blob/master/runtime/vm/service/service.md#rpcs-requests-and-responses
+[service-protocol-events]: https://github.com/dart-lang/sdk/blob/master/runtime/vm/service/service.md#events
+[service-protocol-binary-events]: https://github.com/dart-lang/sdk/blob/master/runtime/vm/service/service.md#binary-events
+[service-protocol-types]: https://github.com/dart-lang/sdk/blob/master/runtime/vm/service/service.md#types
+[service-protocol-ids-and-names]: https://github.com/dart-lang/sdk/blob/master/runtime/vm/service/service.md#ids-and-names
+[service-protocol-public-rpcs]: https://github.com/dart-lang/sdk/blob/master/runtime/vm/service/service.md#public-rpcs

pkg/dds/lib/dds.dart

@@ -0,0 +1,87 @@
+// Copyright (c) 2020, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+/// A library used to spawn the Dart Developer Service, used to communicate
+/// with a Dart VM Service instance.
+library dds;
+
+import 'dart:async';
+import 'dart:io';
+
+import 'package:shelf/shelf.dart';
+import 'package:shelf/shelf_io.dart' as io;
+import 'package:shelf_proxy/shelf_proxy.dart';
+import 'package:shelf_web_socket/shelf_web_socket.dart';
+import 'package:web_socket_channel/web_socket_channel.dart';
+
+part 'src/dds_impl.dart';
+
+/// An intermediary between a Dart VM service and its clients that offers
+/// additional functionality on top of the standard VM service protocol.
+///
+/// See the [Dart Development Service Protocol](https://github.com/dart-lang/sdk/blob/master/pkg/dds/dds_protocol.md)
+/// for details.
+abstract class DartDevelopmentService {
+  /// Creates a [DartDevelopmentService] instance which will communicate with a
+  /// VM service.
+  ///
+  /// [remoteVmServiceUri] is the address of the VM service that this
+  /// development service will communicate with.
+  ///
+  /// If provided, [serviceUri] will determine the address and port of the
+  /// spawned Dart Development Service.
+  static Future<DartDevelopmentService> startDartDevelopmentService(
+    Uri remoteVmServiceUri, {
+    Uri serviceUri,
+  }) async {
+    if (remoteVmServiceUri == null) {
+      throw ArgumentError.notNull('remoteVmServiceUri');
+    }
+    if (remoteVmServiceUri.scheme != 'http') {
+      throw ArgumentError(
+        'remoteVmServiceUri must have an HTTP scheme. Actual: ${remoteVmServiceUri.scheme}',
+      );
+    }
+    if (serviceUri != null && serviceUri.scheme != 'http') {
+      throw ArgumentError(
+        'serviceUri must have an HTTP scheme. Actual: ${serviceUri.scheme}',
+      );
+    }
+
+    final service = _DartDevelopmentService(remoteVmServiceUri, serviceUri);
+    await service.startService();
+    return service;
+  }
+
+  DartDevelopmentService._();
+
+  /// Stop accepting requests after gracefully handling existing requests.
+  Future<void> shutdown();
+
+  /// The HTTP [Uri] of the remote VM service instance that this service will
+  /// forward requests to.
+  Uri get remoteVmServiceUri;
+
+  /// The web socket [Uri] of the remote VM service instance that this service
+  /// will forward requests to.
+  ///
+  /// Can be used with [WebSocket] to communicate directly with the VM service.
+  Uri get remoteVmServiceWsUri;
+
+  /// The [Uri] VM service clients can use to communicate with this
+  /// [DartDevelopmentService] via HTTP.
+  ///
+  /// Returns `null` if the service is not running.
+  Uri get uri;
+
+  /// The [Uri] VM service clients can use to communicate with this
+  /// [DartDevelopmentService] via a [WebSocket].
+  ///
+  /// Returns `null` if the service is not running.
+  Uri get wsUri;
+
+  /// Set to `true` if this instance of [DartDevelopmentService] is accepting
+  /// requests.
+  bool get isRunning;
+}

pkg/dds/lib/src/dds_impl.dart

@@ -0,0 +1,85 @@
+// Copyright (c) 2020, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+part of dds;
+
+class _DartDevelopmentService implements DartDevelopmentService {
+  _DartDevelopmentService(this._remoteVmServiceUri, this._uri);
+
+  Future<void> startService() async {
+    // Establish the connection to the VM service.
+    _vmServiceSocket = await WebSocket.connect(remoteVmServiceWsUri.toString());
+    _vmServiceStream = _vmServiceSocket.asBroadcastStream();
+    // Once we have a connection to the VM service, we're ready to spawn the intermediary.
+    await _startDDSServer();
+  }
+
+  Future<void> _startDDSServer() async {
+    // No provided address, bind to an available port on localhost.
+    // TODO(bkonyi): handle case where there's no IPv4 loopback.
+    final host = uri?.host ?? InternetAddress.loopbackIPv4.host;
+    final port = uri?.port ?? 0;
+
+    // Start the DDS server.
+    _server = await io.serve(_handlers().handler, host, port);
+    _uri = Uri(scheme: 'http', host: host, port: _server.port);
+  }
+
+  /// Stop accepting requests after gracefully handling existing requests.
+  Future<void> shutdown() async {
+    await _server.close();
+    await _vmServiceSocket.close();
+  }
+
+  // Attempt to upgrade HTTP requests to a websocket before processing them as
+  // standard HTTP requests. The websocket handler will fail quickly if the
+  // request doesn't appear to be a websocket upgrade request.
+  Cascade _handlers() => Cascade().add(_webSocketHandler()).add(_httpHandler());
+
+  Handler _webSocketHandler() => webSocketHandler((WebSocketChannel ws) {
+        // TODO(bkonyi): actually process requests instead of blindly forwarding them.
+        _vmServiceStream.listen(
+          (event) => ws.sink.add(event),
+          onDone: () => ws.sink.close(),
+        );
+        ws.stream.listen((event) => _vmServiceSocket.add(event));
+      });
+
+  Handler _httpHandler() {
+    // TODO(bkonyi): actually process requests instead of blindly forwarding them.
+    final cascade = Cascade().add(proxyHandler(remoteVmServiceUri));
+    return cascade.handler;
+  }
+
+  Uri _toWebSocket(Uri uri) {
+    if (uri == null) {
+      return null;
+    }
+    final pathSegments = <String>[];
+    if (uri.pathSegments.isNotEmpty) {
+      pathSegments.addAll(uri.pathSegments.where(
+        // Strip out the empty string that appears at the end of path segments.
+        // Empty string elements will result in an extra '/' being added to the
+        // URI.
+        (s) => s.isNotEmpty,
+      ));
+    }
+    pathSegments.add('ws');
+    return uri.replace(scheme: 'ws', pathSegments: pathSegments);
+  }
+
+  Uri get remoteVmServiceUri => _remoteVmServiceUri;
+  Uri get remoteVmServiceWsUri => _toWebSocket(_remoteVmServiceUri);
+  Uri _remoteVmServiceUri;
+
+  Uri get uri => _uri;
+  Uri get wsUri => _toWebSocket(_uri);
+  Uri _uri;
+
+  bool get isRunning => _uri != null;
+
+  WebSocket _vmServiceSocket;
+  Stream _vmServiceStream;
+  HttpServer _server;
+}

pkg/dds/pubspec.yaml

@@ -0,0 +1,23 @@
+name: dds
+description: >-
+  A library used to spawn the Dart Developer Service, used to communicate with
+  a Dart VM Service instance.
+
+version: 0.1.0-dev
+
+homepage: https://github.com/dart-lang/sdk/tree/master/pkg/dds
+
+environment:
+  sdk: '>=2.6.0 <3.0.0'
+
+dependencies:
+  json_rpc_2: ^2.1.0
+  shelf: ^0.7.5
+  shelf_proxy: ^0.1.0+7
+  shelf_web_socket: ^0.2.3
+  web_socket_channel: ^1.1.0
+
+dev_dependencies:
+  pedantic: ^1.7.0
+  test: ^1.0.0
+  vm_service: ^4.0.0

pkg/dds/test/smoke.dart

@@ -0,0 +1,3 @@
+void main() {
+  print('Hello world!');
+}

pkg/dds/test/smoke_test.dart

@@ -0,0 +1,90 @@
+// Copyright (c) 2020, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:dds/dds.dart';
+import 'package:test/test.dart';
+import 'package:vm_service/vm_service_io.dart';
+
+Uri remoteVmServiceUri;
+
+Future<Process> spawnDartProcess(String script) async {
+  final executable = Platform.executable;
+  final tmpDir = await Directory.systemTemp.createTemp('dart_service');
+  final serviceInfoUri = tmpDir.uri.resolve('service_info.json');
+  final serviceInfoFile = await File.fromUri(serviceInfoUri).create();
+
+  final arguments = [
+    '--observe=0',
+    '--pause-isolates-on-start',
+    '--write-service-info=$serviceInfoUri',
+    script,
+  ];
+  final process = await Process.start(executable, arguments);
+  while ((await serviceInfoFile.length()) <= 5) {
+    await Future.delayed(const Duration(milliseconds: 50));
+  }
+  final content = await serviceInfoFile.readAsString();
+  final infoJson = json.decode(content);
+  remoteVmServiceUri = Uri.parse(infoJson['uri']);
+  return process;
+}
+
+void main() {
+  test('DDS Smoke Test', () async {
+    final process = await spawnDartProcess('smoke.dart');
+    final dds = await DartDevelopmentService.startDartDevelopmentService(
+      remoteVmServiceUri,
+    );
+    expect(dds.isRunning, true);
+
+    // Ensure basic websocket requests are forwarded correctly to the VM service.
+    final service = await vmServiceConnectUri(dds.wsUri.toString());
+    final version = await service.getVersion();
+    expect(version.major > 0, true);
+    expect(version.minor > 0, true);
+
+    // Ensure we can still make requests of the VM service via HTTP.
+    HttpClient client = HttpClient();
+    final request = await client.getUrl(remoteVmServiceUri.replace(
+      pathSegments: [
+        remoteVmServiceUri.pathSegments.first,
+        'getVersion',
+      ],
+    ));
+    final response = await request.close();
+    final Map<String, dynamic> jsonResponse =
+        (await response.transform(utf8.decoder).transform(json.decoder).single);
+    expect(jsonResponse['result']['type'], 'Version');
+    expect(jsonResponse['result']['major'] > 0, true);
+    expect(jsonResponse['result']['minor'] > 0, true);
+    await dds.shutdown();
+    process.kill();
+  });
+
+  test('Invalid args test', () async {
+    // null VM Service URI
+    expect(
+        () async =>
+            await DartDevelopmentService.startDartDevelopmentService(null),
+        throwsA(TypeMatcher<ArgumentError>()));
+
+    // Non-HTTP VM Service URI scheme
+    expect(
+        () async => await DartDevelopmentService.startDartDevelopmentService(
+              Uri.parse('dart-lang://localhost:1234'),
+            ),
+        throwsA(TypeMatcher<ArgumentError>()));
+
+    // Non-HTTP VM Service URI scheme
+    expect(
+        () async => await DartDevelopmentService.startDartDevelopmentService(
+              Uri.parse('http://localhost:1234'),
+              serviceUri: Uri.parse('dart-lang://localhost:2345'),
+            ),
+        throwsA(TypeMatcher<ArgumentError>()));
+  });
+}

pkg/pkg.status

@@ -120,6 +120,7 @@ analyzer_plugin/test/src/utilities/completion/optype_test: Slow, Pass
 mutation_observer: Skip # Skip tests on the VM if the package depends on dart:html
 
 [ $runtime != vm ]
+dds/test/*: SkipByDesign # Only meant to run on vm
 dev_compiler/test/options/*: SkipByDesign
 front_end/test/hot_reload_e2e_test: Skip
 frontend_server/test/*: SkipByDesign # Only meant to run on vm

tools/bots/test_matrix.json

@@ -3210,6 +3210,14 @@
             "pkg/vm_service"
           ]
         },
+        {
+          "name": "analyze pkg/dds",
+          "script": "out/ReleaseX64/dart-sdk/bin/dartanalyzer",
+          "arguments": [
+            "--fatal-warnings",
+            "pkg/dds"
+          ]
+        },
         {
           "name": "analyze runtime/observatory",
           "script": "out/ReleaseX64/dart-sdk/bin/dartanalyzer",