[analysis_server] Add a new mechanism for marking up test code with regions/positions

Change-Id: I45ca24d2360f7201a563145a19c5291f83b9465c
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/259424
Reviewed-by: Konstantin Shcheglov <scheglov@google.com>

pkg/analysis_server/pubspec.yaml

@@ -32,4 +32,5 @@ dev_dependencies:
   lints: any
   logging: any
   matcher: any
+  string_scanner: any
   test_reflective_loader: any

pkg/analysis_server/test/src/computer/folding_computer_test.dart

@@ -9,6 +9,7 @@ import 'package:test/test.dart';
 import 'package:test_reflective_loader/test_reflective_loader.dart';
 
 import '../../abstract_context.dart';
+import '../../utils/test_code_format.dart';
 
 void main() {
   defineReflectiveSuite(() {
@@ -25,6 +26,19 @@ class FoldingComputerTest extends AbstractContextTest {
   };
 
   late String sourcePath;
+  late TestCode code;
+  List<FoldingRegion> regions = [];
+
+  /// Expects to find a [FoldingRegion] for the code marked [index] with a
+  /// [FoldingKind] of [kind].
+  void expectRegions(Map<int, FoldingKind> expected) {
+    final expectedRegions = expected.entries.map((entry) {
+      final range = code.ranges[entry.key].sourceRange;
+      return FoldingRegion(entry.value, range.offset, range.length);
+    }).toSet();
+
+    expect(regions, expectedRegions);
+  }
 
   @override
   void setUp() {
@@ -34,53 +48,63 @@ class FoldingComputerTest extends AbstractContextTest {
 
   Future<void> test_annotations() async {
     var content = '''
-@myMultilineAnnotation/*1:INC*/(
+@myMultilineAnnotation/*[0*/(
   "this",
   "is a test"
-)/*1:EXC:ANNOTATIONS*/
+)/*0]*/
 void f() {}
 
 @noFoldNecessary
 main2() {}
 
-@multipleAnnotations1/*2:INC*/(
+@multipleAnnotations1/*[1*/(
   "this",
   "is a test"
 )
 @multipleAnnotations2()
-@multipleAnnotations3/*2:EXC:ANNOTATIONS*/
+@multipleAnnotations3/*1]*/
 main3() {}
 
 @noFoldsForSingleClassAnnotation
 class MyClass {}
 
-@folded.classAnnotation1/*3:INC*/()
-@foldedClassAnnotation2/*3:EXC:ANNOTATIONS*/
-class MyClass2 {/*4:INC*/
-  @fieldAnnotation1/*5:INC*/
-  @fieldAnnotation2/*5:EXC:ANNOTATIONS*/
+@folded.classAnnotation1/*[2*/()
+@foldedClassAnnotation2/*2]*/
+class MyClass2 {/*[3*/
+  @fieldAnnotation1/*[4*/
+  @fieldAnnotation2/*4]*/
   int myField;
 
-  @getterAnnotation1/*6:INC*/
-  @getterAnnotation2/*6:EXC:ANNOTATIONS*/
+  @getterAnnotation1/*[5*/
+  @getterAnnotation2/*5]*/
   int get myThing => 1;
 
-  @setterAnnotation1/*7:INC*/
-  @setterAnnotation2/*7:EXC:ANNOTATIONS*/
+  @setterAnnotation1/*[6*/
+  @setterAnnotation2/*6]*/
   void set myThing(int value) {}
 
-  @methodAnnotation1/*8:INC*/
-  @methodAnnotation2/*8:EXC:ANNOTATIONS*/
+  @methodAnnotation1/*[7*/
+  @methodAnnotation2/*7]*/
   void myMethod() {}
 
-  @constructorAnnotation1/*9:INC*/
-  @constructorAnnotation1/*9:EXC:ANNOTATIONS*/
+  @constructorAnnotation1/*[8*/
+  @constructorAnnotation1/*8]*/
   MyClass2() {}
-/*4:INC:CLASS_BODY*/}
+/*3]*/}
 ''';
 
-    final regions = await _computeRegions(content);
-    _compareRegions(regions, content);
+    await _computeRegions(content);
+    expectRegions({
+      0: FoldingKind.ANNOTATIONS,
+      1: FoldingKind.ANNOTATIONS,
+      2: FoldingKind.ANNOTATIONS,
+      3: FoldingKind.CLASS_BODY,
+      4: FoldingKind.ANNOTATIONS,
+      5: FoldingKind.ANNOTATIONS,
+      6: FoldingKind.ANNOTATIONS,
+      7: FoldingKind.ANNOTATIONS,
+      8: FoldingKind.ANNOTATIONS,
+    });
   }
 
   Future<void> test_assertInitializer() async {
@@ -603,10 +627,12 @@ void f() {}
   }
 
   Future<List<FoldingRegion>> _computeRegions(String sourceContent) async {
-    newFile(sourcePath, sourceContent);
+    code = TestCode.parse(sourceContent);
+    newFile(sourcePath, code.code);
     var result =
         await (await session).getResolvedUnit(sourcePath) as ResolvedUnitResult;
     var computer = DartUnitFoldingComputer(result.lineInfo, result.unit);
-    return computer.compute();
+    regions = computer.compute();
+    return regions;
   }
 }

pkg/analysis_server/test/test_all.dart

@@ -25,6 +25,7 @@ import 'search/test_all.dart' as search;
 import 'services/test_all.dart' as services;
 import 'socket_server_test.dart' as socket_server;
 import 'src/test_all.dart' as src;
+import 'test_code_format_test.dart' as test_code_format;
 import 'tool/test_all.dart' as tool;
 import 'verify_error_fix_status_test.dart' as verify_error_fix_status;
 import 'verify_no_solo_test.dart' as verify_no_solo;
@@ -53,6 +54,7 @@ void main() {
     services.main();
     socket_server.main();
     src.main();
+    test_code_format.main();
     tool.main();
     verify_error_fix_status.main();
     verify_no_solo.main();

pkg/analysis_server/test/test_code_format_test.dart

@@ -0,0 +1,176 @@
+// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'package:analysis_server/lsp_protocol/protocol.dart' as lsp;
+import 'package:analyzer/source/source_range.dart';
+import 'package:test/test.dart';
+import 'package:test_reflective_loader/test_reflective_loader.dart';
+
+import 'utils/test_code_format.dart';
+
+void main() {
+  defineReflectiveSuite(() {
+    defineReflectiveTests(TestCodeFormatTest);
+  });
+}
+
+@reflectiveTest
+class TestCodeFormatTest {
+  void test_caret() {
+    final rawCode = '''
+int ^a = 1
+    ''';
+    final expectedCode = '''
+int a = 1
+    ''';
+    final code = TestCode.parse(rawCode);
+    expect(code.rawCode, rawCode);
+    expect(code.code, expectedCode);
+
+    expect(code.positions, hasLength(1));
+    expect(code.position.offset, 4);
+    expect(code.position.offset, code.positions[0].offset);
+    expect(code.position.lsp, lsp.Position(line: 0, character: 4));
+    expect(code.position.lsp, code.positions[0].lsp);
+
+    expect(code.ranges, isEmpty);
+  }
+
+  void test_noMarkers() {
+    final rawCode = '''
+int a = 1;
+''';
+    final code = TestCode.parse(rawCode);
+    expect(code.rawCode, rawCode);
+    expect(code.code, rawCode); // no difference
+    expect(code.positions, isEmpty);
+    expect(code.ranges, isEmpty);
+  }
+
+  void test_nonPositionCaret() {
+    final rawCode = '''
+String /*0*/a = '^^^';
+    ''';
+    final expectedCode = '''
+String a = '^^^';
+    ''';
+    final code = TestCode.parse(rawCode, treatCaretAsPosition: false);
+    expect(code.rawCode, rawCode);
+    expect(code.code, expectedCode);
+
+    expect(code.positions, hasLength(1));
+    expect(code.position.offset, 7);
+    expect(code.position.offset, code.positions[0].offset);
+    expect(code.position.lsp, lsp.Position(line: 0, character: 7));
+    expect(code.position.lsp, code.positions[0].lsp);
+
+    expect(code.ranges, isEmpty);
+  }
+
+  void test_positions() {
+    final rawCode = '''
+int /*0*/a = 1;/*1*/
+int b/*2*/ = 2;
+''';
+    final expectedCode = '''
+int a = 1;
+int b = 2;
+''';
+    final code = TestCode.parse(rawCode);
+    expect(code.rawCode, rawCode);
+    expect(code.code, expectedCode);
+    expect(code.ranges, isEmpty);
+
+    expect(code.positions[0].offset, 4);
+    expect(code.positions[1].offset, 10);
+    expect(code.positions[2].offset, 16);
+
+    expect(code.positions[0].lsp, lsp.Position(line: 0, character: 4));
+    expect(code.positions[1].lsp, lsp.Position(line: 0, character: 10));
+    expect(code.positions[2].lsp, lsp.Position(line: 1, character: 5));
+  }
+
+  void test_positions_reused() {
+    final rawCode = '''
+/*0*/ /*1*/ /*0*/
+''';
+    expect(() => TestCode.parse(rawCode), throwsArgumentError);
+  }
+
+  void test_positions_reusedCaret() {
+    final rawCode = '''
+^ ^
+''';
+    expect(() => TestCode.parse(rawCode), throwsArgumentError);
+  }
+
+  void test_positions_reusedCaretNumber() {
+    final rawCode = '''
+/*1*/ ^
+''';
+    expect(() => TestCode.parse(rawCode), throwsArgumentError);
+  }
+
+  void test_ranges() {
+    final rawCode = '''
+int /*[0*/a = 1;/*0]*/
+/*[1*/int b = 2;/*1]*/
+''';
+    final expectedCode = '''
+int a = 1;
+int b = 2;
+''';
+    final code = TestCode.parse(rawCode);
+    expect(code.rawCode, rawCode);
+    expect(code.code, expectedCode);
+    expect(code.positions, isEmpty);
+
+    expect(code.ranges, hasLength(2));
+    expect(code.ranges[0].sourceRange, SourceRange(4, 6));
+    expect(code.ranges[1].sourceRange, SourceRange(11, 10));
+    expect(
+        code.ranges[0].lsp,
+        lsp.Range(
+            start: lsp.Position(line: 0, character: 4),
+            end: lsp.Position(line: 0, character: 10)));
+    expect(
+        code.ranges[1].lsp,
+        lsp.Range(
+            start: lsp.Position(line: 1, character: 0),
+            end: lsp.Position(line: 1, character: 10)));
+
+    expect(code.ranges[0].text, 'a = 1;');
+    expect(code.ranges[1].text, 'int b = 2;');
+  }
+
+  void test_ranges_endReused() {
+    final rawCode = '''
+/*[0*/ /*0]*/
+/*[1*/ /*0]*/
+''';
+    expect(() => TestCode.parse(rawCode), throwsArgumentError);
+  }
+
+  void test_ranges_endWithoutStart() {
+    final rawCode = '''
+/*0]*/
+''';
+    expect(() => TestCode.parse(rawCode), throwsArgumentError);
+  }
+
+  void test_ranges_startReused() {
+    final rawCode = '''
+/*[0*/ /*0]*/
+/*[0*/ /*1]*/
+''';
+    expect(() => TestCode.parse(rawCode), throwsArgumentError);
+  }
+
+  void test_ranges_startWithoutEnd() {
+    final rawCode = '''
+/*[0*/
+''';
+    expect(() => TestCode.parse(rawCode), throwsArgumentError);
+  }
+}

pkg/analysis_server/test/utils/test_code_format.dart

@@ -0,0 +1,182 @@
+// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'package:analysis_server/lsp_protocol/protocol.dart'
+    show Position, Range;
+import 'package:analysis_server/src/lsp/mapping.dart';
+import 'package:analyzer/source/line_info.dart';
+import 'package:analyzer/source/source_range.dart';
+import 'package:collection/collection.dart';
+import 'package:string_scanner/string_scanner.dart';
+
+/// A class for parsing and representing test code that contains special markup
+/// to simplify marking up positions and ranges in test code.
+///
+/// Positions and ranges are marked with brackets inside block comments:
+///
+/// ```
+/// position ::= '/*' integer '*/
+/// rangeStart ::= '/*[' integer '*/
+/// rangeEnd ::= '/*' integer ']*/
+/// ```
+///
+/// Numbers start at 0 and positions and range starts must be consecutive.
+/// The same numbers can be used to represent both positions and ranges.
+///
+/// For convenience, a single position can also be marked with a `^` (which
+/// behaves the same as `/*0*/).
+class TestCode {
+  static final _positionPattern = RegExp(r'\/\*(\d+)\*\/');
+  static final _rangeStartPattern = RegExp(r'\/\*\[(\d+)\*\/');
+  static final _rangeEndPattern = RegExp(r'\/\*(\d+)\]\*\/');
+  final String code;
+  final String rawCode;
+
+  /// A map of positions marked in code, indexed by their number.
+  final List<TestCodePosition> positions;
+
+  /// A map of ranges marked in code, indexed by their number.
+  final List<TestCodeRange> ranges;
+
+  TestCode._({
+    required this.rawCode,
+    required this.code,
+    required this.positions,
+    required this.ranges,
+  });
+
+  TestCodePosition get position => positions.single;
+  TestCodeRange get range => ranges.single;
+
+  static TestCode parse(String rawCode, {bool treatCaretAsPosition = true}) {
+    final scanner = StringScanner(rawCode);
+    final codeBuffer = StringBuffer();
+    final positionOffsets = <int, int>{};
+    final rangeStartOffsets = <int, int>{};
+    final rangeEndOffsets = <int, int>{};
+    late int start;
+
+    int scannedNumber() => int.parse(scanner.lastMatch!.group(1)!);
+
+    void recordPosition(int number) {
+      if (positionOffsets.containsKey(number)) {
+        throw ArgumentError(
+            'Code contains multiple positions numbered $number');
+      } else if (number > positionOffsets.length) {
+        throw ArgumentError(
+            'Code contains position numbered $number but expected ${positionOffsets.length}');
+      }
+      positionOffsets[number] = start;
+    }
+
+    void recordRangeStart(int number) {
+      if (rangeStartOffsets.containsKey(number)) {
+        throw ArgumentError(
+            'Code contains multiple range starts numbered $number');
+      } else if (number > rangeStartOffsets.length) {
+        throw ArgumentError(
+            'Code contains range start numbered $number but expected ${rangeStartOffsets.length}');
+      }
+      rangeStartOffsets[number] = start;
+    }
+
+    void recordRangeEnd(int number) {
+      if (rangeEndOffsets.containsKey(number)) {
+        throw ArgumentError(
+            'Code contains multiple range ends numbered $number');
+      }
+      if (!rangeStartOffsets.containsKey(number)) {
+        throw ArgumentError(
+            'Code contains range end numbered $number without a preceeding start');
+      }
+      rangeEndOffsets[number] = start;
+    }
+
+    while (!scanner.isDone) {
+      start = codeBuffer.length;
+      if (treatCaretAsPosition && scanner.scan('^')) {
+        recordPosition(0);
+      } else if (scanner.scan(_positionPattern)) {
+        recordPosition(scannedNumber());
+      } else if (scanner.scan(_rangeStartPattern)) {
+        recordRangeStart(scannedNumber());
+      } else if (scanner.scan(_rangeEndPattern)) {
+        recordRangeEnd(scannedNumber());
+      } else {
+        codeBuffer.writeCharCode(scanner.readChar());
+      }
+    }
+
+    final unendedRanges =
+        rangeStartOffsets.keys.whereNot(rangeEndOffsets.keys.contains).toList();
+    if (unendedRanges.isNotEmpty) {
+      throw ArgumentError(
+          'Code contains range starts numbered $unendedRanges without ends');
+    }
+
+    final code = codeBuffer.toString();
+    final lineInfo = LineInfo.fromContent(code);
+
+    final positions = positionOffsets.map(
+      (number, offset) => MapEntry(
+        number,
+        TestCodePosition(
+          offset,
+          toPosition(lineInfo.getLocation(offset)),
+        ),
+      ),
+    );
+
+    final ranges = rangeStartOffsets.map(
+      (number, offset) => MapEntry(
+        number,
+        TestCodeRange(
+          code.substring(offset, rangeEndOffsets[number]!),
+          SourceRange(offset, rangeEndOffsets[number]! - offset),
+          toRange(lineInfo, offset, rangeEndOffsets[number]! - offset),
+        ),
+      ),
+    );
+
+    return TestCode._(
+      code: code,
+      rawCode: rawCode,
+      positions: positions.values.toList(),
+      ranges: ranges.values.toList(),
+    );
+  }
+}
+
+/// A marked position in the test code.
+class TestCodePosition {
+  /// The 0-based offset of the marker.
+  ///
+  /// Offsets are based on [TestCode.code], with all parsed markers removed.
+  final int offset;
+
+  /// The LSP [Position] of the marker.
+  ///
+  /// Positions are based on [TestCode.code], with all parsed markers removed.
+  final Position lsp;
+
+  TestCodePosition(this.offset, this.lsp);
+}
+
+class TestCodeRange {
+  /// The text from [TestCode.code] covered by this range.
+  final String text;
+
+  /// The [SourceRange] indicated by the markers.
+  ///
+  /// Offsets/lengths are based on [TestCode.code], with all parsed markers
+  /// removed.
+  final SourceRange sourceRange;
+
+  /// The LSP [Range] indicated by the markers.
+  ///
+  /// Ranges are based on [TestCode.code], with all parsed markers removed.
+  final Range lsp;
+
+  TestCodeRange(this.text, this.sourceRange, this.lsp);
+}