[dds] Add support for running DAP integration tests with an out-of-process server

Change-Id: Iffa9154951a6e5e4dc1fad783f8710dad5f1a5a8
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/201563
Reviewed-by: Ben Konyi <bkonyi@google.com>
Reviewed-by: Brian Wilkerson <brianwilkerson@google.com>
Commit-Queue: Ben Konyi <bkonyi@google.com>

pkg/dds/lib/src/dap/base_debug_adapter.dart

@@ -9,6 +9,9 @@ import 'protocol_common.dart';
 import 'protocol_generated.dart';
 import 'protocol_stream.dart';
 
+typedef _FromJsonHandler<T> = T Function(Map<String, Object?>);
+typedef _NullableFromJsonHandler<T> = T? Function(Map<String, Object?>?);
+
 /// A base class for debug adapters.
 ///
 /// Communicates over a [ByteStreamServerChannel] and turns messages into
@@ -138,8 +141,8 @@ abstract class BaseDebugAdapter<TLaunchArgs extends LaunchRequestArguments> {
   );
 
   /// Wraps a fromJson handler for requests that allow null arguments.
-  T? Function(Map<String, Object?>?) _allowNullArg<T extends RequestArguments>(
-    T Function(Map<String, Object?>) fromJson,
+  _NullableFromJsonHandler<T> _allowNullArg<T extends RequestArguments>(
+    _FromJsonHandler<T> fromJson,
   ) {
     return (data) => data == null ? null : fromJson(data);
   }

pkg/dds/lib/src/dap/server.dart

@@ -15,6 +15,8 @@ import 'protocol_stream_transformers.dart';
 
 /// A DAP server that binds to a port and runs in multi-session mode.
 class DapServer {
+  static const defaultPort = 9200;
+
   final ServerSocket _socket;
   final Logger? _logger;
   final _channels = <ByteStreamServerChannel>{};

pkg/dds/test/dap/integration/no_debug_test.dart

@@ -7,12 +7,10 @@ import 'package:test/test.dart';
 import 'test_support.dart';
 
 main() {
-  setUpAll(startServerAndClient);
-  tearDownAll(stopServerAndClient);
-
-  group('noDebug', () {
-    test('runs a simple script', () async {
-      final testFile = createTestFile(r'''
+  testDap((dap) async {
+    group('noDebug', () {
+      test('runs a simple script', () async {
+        final testFile = dap.createTestFile(r'''
 void main(List<String> args) async {
   print('Hello!');
   print('World!');
@@ -20,22 +18,23 @@ void main(List<String> args) async {
 }
     ''');
 
-      final outputEvents = await dapClient.collectOutput(
-        launch: () => dapClient.launch(
-          testFile.path,
-          noDebug: true,
-          args: ['one', 'two'],
-        ),
-      );
+        final outputEvents = await dap.client.collectOutput(
+          launch: () => dap.client.launch(
+            testFile.path,
+            noDebug: true,
+            args: ['one', 'two'],
+          ),
+        );
 
-      final output = outputEvents.map((e) => e.output).join();
-      expectLines(output, [
-        'Hello!',
-        'World!',
-        'args: [one, two]',
-        '',
-        'Exited.',
-      ]);
+        final output = outputEvents.map((e) => e.output).join();
+        expectLines(output, [
+          'Hello!',
+          'World!',
+          'args: [one, two]',
+          '',
+          'Exited.',
+        ]);
+      });
     });
   });
 }

pkg/dds/test/dap/integration/test_client.dart

@@ -19,10 +19,10 @@ class DapTestClient {
   final Socket _socket;
   final ByteStreamServerChannel _channel;
   late final StreamSubscription<String> _subscription;
+
   final Logger? _logger;
   final bool captureVmServiceTraffic;
   final _requestWarningDuration = const Duration(seconds: 2);
-
   final Map<int, _OutgoingRequest> _pendingRequests = {};
   final _eventController = StreamController<Event>.broadcast();
   int _seq = 1;

pkg/dds/test/dap/integration/test_server.dart

@@ -3,15 +3,17 @@
 // BSD-style license that can be found in the LICENSE file.
 
 import 'dart:async';
+import 'dart:convert';
+import 'dart:io';
+import 'dart:isolate';
 
 import 'package:dds/src/dap/server.dart';
+import 'package:path/path.dart' as path;
+import 'package:pedantic/pedantic.dart';
 
 abstract class DapTestServer {
-  String get host => _server.host;
-
-  int get port => _server.port;
-  DapServer get _server;
-
+  String get host;
+  int get port;
   FutureOr<void> stop();
 }
 
@@ -25,6 +27,9 @@ class InProcessDapTestServer extends DapTestServer {
 
   InProcessDapTestServer._(this._server);
 
+  String get host => _server.host;
+  int get port => _server.port;
+
   @override
   FutureOr<void> stop() async {
     await _server.stop();
@@ -35,3 +40,61 @@ class InProcessDapTestServer extends DapTestServer {
     return InProcessDapTestServer._(server);
   }
 }
+
+/// An instance of a DAP server running out-of-process.
+///
+/// This is how an editor will usually consume DAP so is a more accurate test
+/// but will be a little more difficult to debug tests as the debugger will not
+/// be attached to the process.
+class OutOfProcessDapTestServer extends DapTestServer {
+  /// Since each test library will spawn its own server (setup/teardown are
+  /// library-scoped) we'll use a different port for each one to avoid any issues
+  /// with overlapping tests.
+  static var _nextPort = DapServer.defaultPort;
+
+  var _isShuttingDown = false;
+  final Process _process;
+  final int port;
+  final String host;
+
+  OutOfProcessDapTestServer._(this._process, this.host, this.port) {
+    // The DAP server should generally not write to stdout/stderr (unless -v is
+    // passed), but it may do if it fails to start or crashes. If this happens,
+    // ensure these are included in the test output.
+    _process.stdout.transform(utf8.decoder).listen(print);
+    _process.stderr.transform(utf8.decoder).listen((s) => throw s);
+    unawaited(_process.exitCode.then((code) {
+      if (!_isShuttingDown && code != 0) {
+        throw 'Out-of-process DAP server terminated with code $code';
+      }
+    }));
+  }
+
+  @override
+  FutureOr<void> stop() async {
+    _isShuttingDown = true;
+    await _process.kill();
+    await _process.exitCode;
+  }
+
+  static Future<OutOfProcessDapTestServer> create() async {
+    final ddsEntryScript =
+        await Isolate.resolvePackageUri(Uri.parse('package:dds/dds.dart'));
+    final ddsLibFolder = path.dirname(ddsEntryScript!.toFilePath());
+    final dapServerScript =
+        path.join(ddsLibFolder, '../tool/dap/run_server.dart');
+
+    final port = OutOfProcessDapTestServer._nextPort++;
+    final host = 'localhost';
+    final _process = await Process.start(
+      Platform.resolvedExecutable,
+      [
+        dapServerScript,
+        '--host=$host',
+        '--port=$port',
+      ],
+    );
+
+    return OutOfProcessDapTestServer._(_process, host, port);
+  }
+}

pkg/dds/test/dap/integration/test_support.dart

@@ -11,19 +11,14 @@ import 'package:test/test.dart';
 import 'test_client.dart';
 import 'test_server.dart';
 
-late DapTestClient dapClient;
-late DapTestServer dapServer;
-
-final _testFolders = <Directory>[];
-
-/// Creates a file in a temporary folder to be used as an application for testing.
-File createTestFile(String content) {
-  final testAppDir = Directory.systemTemp.createTempSync('dart-sdk-dap-test');
-  _testFolders.add(testAppDir);
-  final testFile = File(path.join(testAppDir.path, 'test_file.dart'));
-  testFile.writeAsStringSync(content);
-  return testFile;
-}
+/// Whether to run the DAP server in-process with the tests, or externally in
+/// another process.
+///
+/// By default tests will run the DAP server out-of-process to match the real
+/// use from editors, but this complicates debugging the adapter. Set this env
+/// variables to run the server in-process for easier debugging (this can be
+/// simplified in VS Code by using a launch config with custom CodeLens links).
+final useInProcessDap = Platform.environment['DAP_TEST_INTERNAL'] == 'true';
 
 /// Expects [actual] to equal the lines [expected], ignoring differences in line
 /// endings.
@@ -31,18 +26,76 @@ void expectLines(String actual, List<String> expected) {
   expect(actual.replaceAll('\r\n', '\n'), equals(expected.join('\n')));
 }
 
-/// Starts a DAP server and a DAP client that connects to it for use in tests.
-FutureOr<void> startServerAndClient() async {
-  // TODO(dantup): An Out-of-process option.
-  dapServer = await InProcessDapTestServer.create();
-  dapClient = await DapTestClient.connect(dapServer.port);
+/// A helper function to wrap all tests in a library with setup/teardown functions
+/// to start a shared server for all tests in the library and an individual
+/// client for each test.
+testDap(FutureOr<void> Function(DapTestSession session) tests) {
+  final session = DapTestSession();
+
+  setUpAll(session.setUpAll);
+  tearDownAll(session.tearDownAll);
+  setUp(session.setUp);
+  tearDown(session.tearDown);
+
+  return tests(session);
 }
 
-/// Shuts down the DAP server and client created by [startServerAndClient].
-FutureOr<void> stopServerAndClient() async {
-  await dapClient.stop();
-  await dapServer.stop();
+/// A helper class provided to DAP integration tests run with [testDap] to
+/// easily share setup/teardown without sharing state across tests from different
+/// files.
+class DapTestSession {
+  late DapTestServer server;
+  late DapTestClient client;
+  final _testFolders = <Directory>[];
 
-  // Clean up any temp folders created during the test run.
-  _testFolders.forEach((dir) => dir.deleteSync(recursive: true));
+  /// Creates a file in a temporary folder to be used as an application for testing.
+  ///
+  /// The file will be deleted at the end of the test run.
+  File createTestFile(String content) {
+    final testAppDir = Directory.systemTemp.createTempSync('dart-sdk-dap-test');
+    _testFolders.add(testAppDir);
+    final testFile = File(path.join(testAppDir.path, 'test_file.dart'));
+    testFile.writeAsStringSync(content);
+    return testFile;
+  }
+
+  FutureOr<void> setUp() async {
+    client = await _startClient(server);
+  }
+
+  FutureOr<void> setUpAll() async {
+    server = await _startServer();
+  }
+
+  FutureOr<void> tearDown() => client.stop();
+
+  FutureOr<void> tearDownAll() async {
+    await server.stop();
+
+    // Clean up any temp folders created during the test runs.
+    _testFolders.forEach((dir) => dir.deleteSync(recursive: true));
+  }
+
+  /// Creates and connects a new [DapTestClient] to [server].
+  FutureOr<DapTestClient> _startClient(DapTestServer server) async {
+    // Since we don't get a signal from the DAP server when it's ready and we
+    // just started it, add a short retry to connections.
+    var attempt = 1;
+    while (attempt++ <= 5) {
+      try {
+        return await DapTestClient.connect(server.port);
+      } catch (e) {
+        await Future.delayed(const Duration(milliseconds: 200));
+      }
+    }
+
+    throw 'Failed to connect to DAP server after $attempt attempts';
+  }
+
+  /// Starts a DAP server that can be shared across tests.
+  FutureOr<DapTestServer> _startServer() async {
+    return useInProcessDap
+        ? await InProcessDapTestServer.create()
+        : await OutOfProcessDapTestServer.create();
+  }
 }

pkg/dds/tool/dap/run_server.dart

@@ -0,0 +1,46 @@
+// Copyright (c) 2021, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'package:args/args.dart';
+import 'package:dds/src/dap/server.dart';
+
+Future<void> main(List<String> arguments) async {
+  final args = argParser.parse(arguments);
+  if (args[argHelp]) {
+    print(argParser.usage);
+    return;
+  }
+
+  final port = int.parse(args[argPort]);
+  final host = args[argHost];
+
+  await DapServer.create(
+    host: host,
+    port: port,
+    logger: args[argVerbose] ? print : null,
+  );
+}
+
+const argHelp = 'help';
+const argHost = 'host';
+const argPort = 'port';
+const argVerbose = 'verbose';
+final argParser = ArgParser()
+  ..addFlag(argHelp, hide: true)
+  ..addOption(
+    argHost,
+    defaultsTo: 'localhost',
+    help: 'The hostname/IP to bind the server to',
+  )
+  ..addOption(
+    argPort,
+    abbr: 'p',
+    defaultsTo: DapServer.defaultPort.toString(),
+    help: 'The port to bind the server to',
+  )
+  ..addFlag(
+    argVerbose,
+    abbr: 'v',
+    help: 'Whether to print diagnostic output to stdout',
+  );