Add skeleton for parsing LSP spec from Markdown/TypeScript to generate Dart data classes

- Fixes to generation from spec
- Add basic code-gen with (very incomplete) tests
- Add some basic parsing of TypeScript interfaces in the LSP spec
- Add a group to the test
- Add code for extracting TypeScript codeblocks from Markdown

Change-Id: I733756d43744d89307b77527bd083cfacf670f56
Reviewed-on: https://dart-review.googlesource.com/c/79046
Commit-Queue: Danny Tuppeny <dantup@google.com>
Reviewed-by: Brian Wilkerson <brianwilkerson@google.com>

pkg/analysis_server/test/tool/lsp_spec/markdown_test.dart

@@ -0,0 +1,52 @@
+// Copyright (c) 2018, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'package:test/test.dart';
+
+import '../../../tool/lsp_spec/markdown.dart';
+
+main() {
+  group('markdown parser', () {
+    test('extracts a typescript fenced block from Markdown', () {
+      final String input = '''
+```typescript
+CONTENT
+```
+    ''';
+      final List<String> output = extractTypeScriptBlocks(input);
+      expect(output, hasLength(1));
+      expect(output, contains('CONTENT'));
+    });
+
+    test('does not extract unknown code blocks', () {
+      final String input = '''
+```
+CONTENT
+```
+
+```dart
+CONTENT
+```
+    ''';
+      final List<String> output = extractTypeScriptBlocks(input);
+      expect(output, hasLength(0));
+    });
+
+    test('extracts multiple code blocks', () {
+      final String input = '''
+```typescript
+CONTENT1
+```
+
+```typescript
+CONTENT2
+```
+    ''';
+      final List<String> output = extractTypeScriptBlocks(input);
+      expect(output, hasLength(2));
+      expect(output, contains('CONTENT1'));
+      expect(output, contains('CONTENT2'));
+    });
+  });
+}

pkg/analysis_server/test/tool/lsp_spec/typescript_test.dart

@@ -0,0 +1,173 @@
+// Copyright (c) 2018, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'package:test/test.dart';
+
+import '../../../tool/lsp_spec/typescript.dart';
+
+main() {
+  group('typescript parser', () {
+    test('parses an interface', () {
+      final String input = '''
+/**
+ * Some options.
+ */
+export interface SomeOptions {
+	/**
+	 * Options used by something.
+	 */
+	options?: OptionKind[];
+}
+    ''';
+      final List<ApiItem> output = extractTypes(input);
+      expect(output, hasLength(1));
+      expect(output[0], const TypeMatcher<Interface>());
+      final Interface interface = output[0];
+      expect(interface.name, equals('SomeOptions'));
+      expect(interface.comment, equals('Some options.'));
+      expect(interface.baseTypes, hasLength(0));
+      expect(interface.members, hasLength(1));
+      expect(interface.members[0], const TypeMatcher<Field>());
+      final Field field = interface.members[0];
+      expect(field.name, equals('options'));
+      expect(field.comment, equals('''Options used by something.'''));
+      expect(field.allowsNull, isFalse);
+      expect(field.allowsUndefined, isTrue);
+      expect(field.types, hasLength(1));
+      expect(field.types[0], equals('OptionKind[]'));
+    });
+
+    test('parses an interface with multiple fields', () {
+      final String input = '''
+export interface SomeOptions {
+	/**
+	 * Options0 used by something.
+	 */
+	options0: any;
+	/**
+	 * Options1 used by something.
+	 */
+	options1: any;
+}
+    ''';
+      final List<ApiItem> output = extractTypes(input);
+      expect(output, hasLength(1));
+      expect(output[0], const TypeMatcher<Interface>());
+      final Interface interface = output[0];
+      expect(interface.members, hasLength(2));
+      [0, 1].forEach((i) {
+        expect(interface.members[i], const TypeMatcher<Field>());
+        final Field field = interface.members[i];
+        expect(field.name, equals('options$i'));
+        expect(field.comment, equals('''Options$i used by something.'''));
+      });
+    });
+
+    test('flags nullable undefined values', () {
+      final String input = '''
+export interface A {
+  canBeNeither: string;
+	canBeNull: string | null;
+  canBeUndefined?: string;
+  canBeBoth?: string | null;
+}
+    ''';
+      final List<ApiItem> output = extractTypes(input);
+      final Interface interface = output[0];
+      expect(interface.members, hasLength(4));
+      interface.members.forEach((m) => expect(m, const TypeMatcher<Field>()));
+      final Field canBeNeither = interface.members[0],
+          canBeNull = interface.members[1],
+          canBeUndefined = interface.members[2],
+          canBeBoth = interface.members[3];
+      expect(canBeNeither.allowsNull, isFalse);
+      expect(canBeNeither.allowsUndefined, isFalse);
+      expect(canBeNull.allowsNull, isTrue);
+      expect(canBeNull.allowsUndefined, isFalse);
+      expect(canBeUndefined.allowsNull, isFalse);
+      expect(canBeUndefined.allowsUndefined, isTrue);
+      expect(canBeBoth.allowsNull, isTrue);
+      expect(canBeBoth.allowsUndefined, isTrue);
+    });
+
+    test('formats comments correctly', () {
+      final String input = '''
+/**
+ * Describes the what this class in lots of words that wrap onto
+ * multiple lines that will need re-wrapping to format nicely when
+ * converted into Dart.
+ *
+ * Blank lines should remain in-tact, as should:
+ *   - Indented
+ *   - Things
+ */
+export interface A {
+  a: a;
+}
+    ''';
+      final List<ApiItem> output = extractTypes(input);
+      final Interface interface = output[0];
+      expect(interface.comment, equals('''
+Describes the what this class in lots of words that wrap onto multiple lines that will need re-wrapping to format nicely when converted into Dart.
+
+Blank lines should remain in-tact, as should:
+  - Indented
+  - Things'''));
+    });
+
+    test('parses a type alias', () {
+      final String input = '''
+export type DocumentSelector = DocumentFilter[];
+    ''';
+      final List<ApiItem> output = extractTypes(input);
+      expect(output, hasLength(1));
+      expect(output[0], const TypeMatcher<TypeAlias>());
+      final TypeAlias typeAlias = output[0];
+      expect(typeAlias.name, equals('DocumentSelector'));
+      expect(typeAlias.baseType, equals('DocumentFilter[]'));
+    });
+
+    test('parses a namespace of constants', () {
+      final String input = '''
+export namespace ResourceOperationKind {
+	/**
+	 * Supports creating new files and folders.
+	 */
+	export const Create: ResourceOperationKind = 'create';
+
+	/**
+	 * Supports renaming existing files and folders.
+	 */
+	export const Rename: ResourceOperationKind = 'rename';
+
+	/**
+	 * Supports deleting existing files and folders.
+	 */
+	export const Delete: ResourceOperationKind = 'delete';
+}
+    ''';
+      final List<ApiItem> output = extractTypes(input);
+      expect(output, hasLength(1));
+      expect(output[0], const TypeMatcher<Namespace>());
+      final Namespace namespace = output[0];
+      expect(namespace.members, hasLength(3));
+      namespace.members.forEach((m) => expect(m, const TypeMatcher<Const>()));
+      final Const create = namespace.members[0],
+          rename = namespace.members[1],
+          delete = namespace.members[2];
+      expect(create.name, equals('Create'));
+      expect(create.type, equals('ResourceOperationKind'));
+      expect(
+          create.comment, equals('Supports creating new files and folders.'));
+      expect(rename.name, equals('Rename'));
+      expect(rename.type, equals('ResourceOperationKind'));
+      expect(rename.comment,
+          equals('Supports renaming existing files and folders.'));
+      expect(delete.name, equals('Delete'));
+      expect(delete.type, equals('ResourceOperationKind'));
+      expect(delete.comment,
+          equals('Supports deleting existing files and folders.'));
+    });
+  });
+}

pkg/analysis_server/test/tool/lsp_spec/typescript_to_dart_test.dart

@@ -0,0 +1,41 @@
+// Copyright (c) 2018, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'package:test/test.dart';
+
+import '../../../tool/lsp_spec/codegen_dart.dart';
+import '../../../tool/lsp_spec/typescript.dart';
+
+main() {
+  group('typescript converts to dart', () {
+    void convertAndCompare(String input, String expectedOutput) {
+      final String output = generateDartForTypes(extractTypes(input));
+      expect(output.trim(), equals(expectedOutput.trim()));
+    }
+
+    // TODO(dantup): These types are missing constructors, toJson, fromJson, etc.
+
+    test('for an interface', () {
+      final String input = '''
+/**
+ * Some options.
+ */
+export interface SomeOptions {
+	/**
+	 * Options used by something.
+	 */
+	options?: OptionKind[];
+}
+    ''';
+      final String expectedOutput = '''
+/// Some options.
+class SomeOptions {
+  /// Options used by something.
+  List<OptionKind> options;
+}
+    ''';
+      convertAndCompare(input, expectedOutput);
+    });
+  });
+}

pkg/analysis_server/tool/lsp_spec/codegen_dart.dart

@@ -0,0 +1,146 @@
+// Copyright (c) 2018, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'typescript.dart';
+
+String generateDartForTypes(List<ApiItem> types) {
+  final buffer = new IndentableStringBuffer();
+  types.forEach((t) => _writeType(buffer, t));
+  return buffer.toString();
+}
+
+String _mapType(String type) {
+  if (type.endsWith('[]')) {
+    return 'List<${_mapType(type.substring(0, type.length - 2))}>';
+  }
+  const types = <String, String>{
+    'boolean': 'bool',
+    'string': 'String',
+    'number': 'num',
+    'any': 'Object',
+  };
+  return types[type] ?? type;
+}
+
+Iterable<String> _wrapLines(List<String> lines, int maxLength) sync* {
+  lines = lines.map((l) => l.trimRight()).toList();
+  for (var line in lines) {
+    while (true) {
+      if (line.length <= maxLength) {
+        yield line;
+        break;
+      } else {
+        int lastSpace = line.lastIndexOf(' ', maxLength);
+        // If there was no valid place to wrap, yield the whole string.
+        if (lastSpace == -1) {
+          yield line;
+          break;
+        } else {
+          yield line.substring(0, lastSpace);
+          line = line.substring(lastSpace + 1);
+        }
+      }
+    }
+  }
+}
+
+void _writeConst(IndentableStringBuffer buffer, Const cons) {
+  _writeDocComment(buffer, cons.comment);
+  buffer.writeIndentedLn('static const ${cons.name} = ${cons.value};');
+}
+
+void _writeDocComment(IndentableStringBuffer buffer, String comment) {
+  comment = comment?.trim();
+  if (comment == null || comment.length == 0) {
+    return;
+  }
+  Iterable<String> lines = comment.split('\n');
+  // Wrap at 80 - 4 ('/// ') - indent characters.
+  lines = _wrapLines(lines, 80 - 4 - buffer.totalIndent);
+  lines.forEach((l) => buffer.writeIndentedLn('/// ${l.trim()}'));
+}
+
+void _writeField(IndentableStringBuffer buffer, Field field) {
+  _writeDocComment(buffer, field.comment);
+  if (field.types.length == 1) {
+    buffer.writeIndented(_mapType(field.types.first));
+  } else {
+    buffer.writeIndented('Either<${field.types.map(_mapType).join(', ')}>');
+  }
+  buffer.writeln(' ${field.name};');
+}
+
+void _writeInterface(IndentableStringBuffer buffer, Interface interface) {
+  _writeDocComment(buffer, interface.comment);
+  buffer
+    ..writeln('class ${interface.name} {')
+    ..indent();
+  // TODO(dantup): Generate constructors (inc. type checks for unions)
+  interface.members.forEach((m) => _writeMember(buffer, m));
+  // TODO(dantup): Generate toJson()
+  // TODO(dantup): Generate fromJson()
+  buffer
+    ..outdent()
+    ..writeln('}')
+    ..writeln();
+}
+
+void _writeMember(IndentableStringBuffer buffer, Member member) {
+  if (member is Field) {
+    _writeField(buffer, member);
+  } else if (member is Const) {
+    _writeConst(buffer, member);
+  } else {
+    throw 'Unknown type';
+  }
+}
+
+void _writeNamespace(IndentableStringBuffer buffer, Namespace namespace) {
+  _writeDocComment(buffer, namespace.comment);
+  buffer
+    ..writeln('abstract class ${namespace.name} {')
+    ..indent();
+  namespace.members.forEach((m) => _writeMember(buffer, m));
+  buffer
+    ..outdent()
+    ..writeln('}')
+    ..writeln();
+}
+
+void _writeType(IndentableStringBuffer buffer, ApiItem type) {
+  if (type is Interface) {
+    _writeInterface(buffer, type);
+  } else if (type is TypeAlias) {
+    _writeTypeAlias(buffer, type);
+  } else if (type is Namespace) {
+    _writeNamespace(buffer, type);
+  } else {
+    throw 'Unknown type';
+  }
+}
+
+void _writeTypeAlias(IndentableStringBuffer buffer, TypeAlias typeAlias) {
+  print('Skipping type alias ${typeAlias.name}');
+}
+
+class IndentableStringBuffer extends StringBuffer {
+  int _indentLevel = 0;
+  int _indentSpaces = 2;
+
+  int get totalIndent => _indentLevel * _indentSpaces;
+  String get _indentString => ' ' * totalIndent;
+
+  void indent() => _indentLevel++;
+  void outdent() => _indentLevel--;
+
+  void writeIndented(Object obj) {
+    write(_indentString);
+    write(obj);
+  }
+
+  void writeIndentedLn(Object obj) {
+    write(_indentString);
+    writeln(obj);
+  }
+}

pkg/analysis_server/tool/lsp_spec/generate_all.dart

@@ -0,0 +1,37 @@
+// Copyright (c) 2018, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:io';
+
+import 'package:http/http.dart' as http;
+import 'package:path/path.dart' as path;
+
+import 'codegen_dart.dart';
+import 'markdown.dart';
+import 'typescript.dart';
+
+main() async {
+  final String script = Platform.script.toFilePath();
+  // 3x parent = file -> lsp_spec -> tool -> analysis_server.
+  final String packageFolder = new File(script).parent.parent.parent.path;
+  final String outFolder = path.join(packageFolder, 'lib', 'lsp_protocol');
+  new Directory(outFolder).createSync();
+
+  final String spec = await fetchSpec();
+  final List<ApiItem> types =
+      extractTypeScriptBlocks(spec).map(extractTypes).expand((l) => l).toList();
+  final String output = generateDartForTypes(types);
+  // TODO(dantup): Add file header to output file before we start committing it.
+  new File(path.join(outFolder, 'protocol_generated.dart'))
+      .writeAsStringSync(output);
+}
+
+final Uri specUri = Uri.parse(
+    'https://raw.githubusercontent.com/Microsoft/language-server-protocol/gh-pages/specification.md');
+
+Future<String> fetchSpec() async {
+  final resp = await http.get(specUri);
+  return resp.body;
+}

pkg/analysis_server/tool/lsp_spec/markdown.dart

@@ -0,0 +1,15 @@
+// Copyright (c) 2018, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+final _typeScriptBlockPattern =
+    new RegExp(r'\B```typescript([\S\s]*?)\n```', multiLine: true);
+
+/// Extracts fenced code blocks that are explicitly marked as TypeScript from a
+/// markdown document.
+List<String> extractTypeScriptBlocks(String text) {
+  return _typeScriptBlockPattern
+      .allMatches(text)
+      .map((m) => m.group(1).trim())
+      .toList();
+}

pkg/analysis_server/tool/lsp_spec/typescript.dart

@@ -0,0 +1,182 @@
+// Copyright (c) 2018, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+// TODO(dantup): Regex seemed like a good choice when parsing the first few...
+// maybe it's not so great now. We should parse this properly if it turns out
+// there are issues with what we have here.
+const String _blockBody = r'\s*([\s\S]*?)\s*\n\s*';
+const String _comment = r'(?:\/\*\*((?:[\S\s](?!\*\/))+?)\s\*\/)?\s*';
+
+List<ApiItem> extractTypes(String code) {
+  return ApiItem.extractFrom(code);
+}
+
+String _cleanComment(String comment) {
+  if (comment == null) {
+    return null;
+  }
+  final _commentLinePrefixes = new RegExp(r'\n\s*\* ?', multiLine: true);
+  final _nonConcurrentNewlines = new RegExp(r'\n(?![\n\s])', multiLine: true);
+  final _newLinesThatRequireReinserting =
+      new RegExp(r'\n (\w)', multiLine: true);
+  // Remove any Windows newlines from the source.
+  comment = comment.replaceAll('\r', '');
+  // Remove the * prefixes.
+  comment = comment.replaceAll(_commentLinePrefixes, '\n');
+  // Remove and newlines that look like wrapped text.
+  comment = comment.replaceAll(_nonConcurrentNewlines, ' ');
+  // The above will remove one of the newlines when there are two, so we need
+  // to re-insert newlines for any block that starts immediately after a newline.
+  comment = comment.replaceAllMapped(
+      _newLinesThatRequireReinserting, (m) => '\n\n${m.group(1)}');
+  return comment.trim();
+}
+
+List<String> _parseTypes(String baseTypes, String sep) {
+  return baseTypes?.split(sep)?.map((t) => t.trim())?.toList() ?? [];
+}
+
+/// Base class for Interface, Field, Constant, etc. parsed from the LSP spec.
+abstract class ApiItem {
+  String name, comment;
+  ApiItem(this.name, String comment) : comment = _cleanComment(comment);
+
+  static List<ApiItem> extractFrom(String code) {
+    List<ApiItem> types = [];
+    types.addAll(Interface.extractFrom(code));
+    types.addAll(Namespace.extractFrom(code));
+    types.addAll(TypeAlias.extractFrom(code));
+    return types;
+  }
+}
+
+/// A Constant parsed from the LSP spec.
+class Const extends Member {
+  final String type, value;
+  Const(String name, String comment, this.type, this.value)
+      : super(name, comment);
+
+  static List<Const> extractFrom(String code) {
+    final RegExp _constPattern = new RegExp(
+        _comment +
+            r'''(?:export\s+)?const\s+(\w+)(?::\s+(\w+?))?\s*=\s*([\w\[\]'".]+)\s*;''',
+        multiLine: true);
+
+    return _constPattern.allMatches(code).map((m) {
+      final String comment = m.group(1);
+      final String name = m.group(2);
+      final String type = m.group(3);
+      final String value = m.group(4);
+      return new Const(name, comment, type, value);
+    }).toList();
+  }
+}
+
+/// A Field for an Interface parsed from the LSP spec.
+class Field extends Member {
+  final List<String> types;
+  final bool allowsNull, allowsUndefined;
+  Field(String name, String comment, this.types, this.allowsNull,
+      this.allowsUndefined)
+      : super(name, comment);
+
+  static List<Field> extractFrom(String code) {
+    final RegExp _fieldPattern = new RegExp(
+        _comment + r'(\w+\??)\s*:\s*([\w\[\]\s|]+)\s*;',
+        multiLine: true);
+
+    return _fieldPattern.allMatches(code).map((m) {
+      final String comment = m.group(1);
+      String name = m.group(2);
+      final List<String> types = _parseTypes(m.group(3), '|');
+      final bool allowsNull = types.contains('null');
+      if (allowsNull) {
+        types.remove('null');
+      }
+      final bool allowsUndefined = name.endsWith('?');
+      if (allowsUndefined) {
+        name = name.substring(0, name.length - 1);
+      }
+      return new Field(name, comment, types, allowsNull, allowsUndefined);
+    }).toList();
+  }
+}
+
+/// An Interface parsed from the LSP spec.
+class Interface extends ApiItem {
+  final List<String> baseTypes;
+  final List<Member> members;
+  Interface(String name, String comment, this.baseTypes, this.members)
+      : super(name, comment);
+
+  static List<Interface> extractFrom(String code) {
+    final RegExp _interfacePattern = new RegExp(
+        _comment +
+            r'(?:export\s+)?interface\s+(\w+)(?:\s+extends\s+([\w, ]+?))?\s*\{' +
+            _blockBody +
+            '\}',
+        multiLine: true);
+
+    return _interfacePattern.allMatches(code).map((match) {
+      final String comment = match.group(1);
+      final String name = match.group(2);
+      final List<String> baseTypes = _parseTypes(match.group(3), ',');
+      final String body = match.group(4);
+      final List<Member> members = Member.extractFrom(body);
+      return new Interface(name, comment, baseTypes, members);
+    }).toList();
+  }
+}
+
+/// A Field or Constant parsed from the LSP type.
+abstract class Member extends ApiItem {
+  Member(String name, String comment) : super(name, comment);
+
+  static List<Member> extractFrom(String code) {
+    List<Member> members = [];
+    members.addAll(Field.extractFrom(code));
+    members.addAll(Const.extractFrom(code));
+    return members;
+  }
+}
+
+/// A Namespace parsed from the LSP spec. Usually used to hold enum-like
+/// Constants.
+class Namespace extends ApiItem {
+  final List<Member> members;
+  Namespace(String name, String comment, this.members) : super(name, comment);
+
+  static List<Namespace> extractFrom(String code) {
+    final RegExp _namespacePattern = new RegExp(
+        _comment + r'(?:export\s+)?namespace\s+(\w+)\s*\{' + _blockBody + '\}',
+        multiLine: true);
+
+    return _namespacePattern.allMatches(code).map((match) {
+      final String comment = match.group(1);
+      final String name = match.group(2);
+      final String body = match.group(3);
+      final List<Member> members = Member.extractFrom(body);
+      return new Namespace(name, comment, members);
+    }).toList();
+  }
+}
+
+/// A type alias parsed from the LSP spec.
+class TypeAlias extends ApiItem {
+  final String baseType;
+  TypeAlias(name, comment, this.baseType) : super(name, comment);
+
+  static List<TypeAlias> extractFrom(String code) {
+    final RegExp _typeAliasPattern = new RegExp(
+        _comment + r'type\s+([\w]+)\s+=\s+([\w\[\]]+)\s*;',
+        multiLine: true);
+
+    return _typeAliasPattern.allMatches(code).map((match) {
+      final String comment = match.group(1);
+      final String name = match.group(2);
+      final String baseType = match.group(3);
+      return new TypeAlias(name, comment, baseType);
+    }).toList();
+  }
+}