mirror of
https://github.com/dart-lang/sdk
synced 2024-09-16 03:36:59 +00:00
5909efe401
This fixes * lineStart tracking when synthetic strings added * token charOffset when scanning UTF-8 stream These changes prevent the crashes mentioned in #29976 and #29982. R=ahe@google.com Review-Url: https://codereview.chromium.org/2984713002 .
756 lines
25 KiB
Dart
756 lines
25 KiB
Dart
// Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file
|
|
// for details. All rights reserved. Use of this source code is governed by a
|
|
// BSD-style license that can be found in the LICENSE file.
|
|
import 'dart:convert';
|
|
|
|
import 'package:analyzer/src/fasta/token_utils.dart';
|
|
import 'package:front_end/src/fasta/fasta_codes.dart';
|
|
import 'package:front_end/src/fasta/scanner.dart' as usedForFuzzTesting;
|
|
import 'package:front_end/src/fasta/scanner/error_token.dart' as fasta;
|
|
import 'package:front_end/src/fasta/scanner/string_scanner.dart' as fasta;
|
|
import 'package:front_end/src/fasta/scanner/token.dart' as fasta;
|
|
import 'package:front_end/src/fasta/scanner/token_constants.dart' as fasta;
|
|
import 'package:front_end/src/fasta/scanner/utf8_bytes_scanner.dart' as fasta;
|
|
import 'package:front_end/src/scanner/errors.dart';
|
|
import 'package:front_end/src/scanner/token.dart';
|
|
import 'package:test/test.dart';
|
|
import 'package:test_reflective_loader/test_reflective_loader.dart';
|
|
|
|
import 'scanner_test.dart';
|
|
|
|
main() {
|
|
defineReflectiveSuite(() {
|
|
defineReflectiveTests(ScannerTest_Fasta);
|
|
defineReflectiveTests(ScannerTest_Fasta_FuzzTestAPI);
|
|
defineReflectiveTests(ScannerTest_Fasta_UTF8);
|
|
defineReflectiveTests(ScannerTest_Fasta_Direct);
|
|
defineReflectiveTests(ScannerTest_Fasta_Direct_UTF8);
|
|
});
|
|
}
|
|
|
|
@reflectiveTest
|
|
class ScannerTest_Fasta_FuzzTestAPI {
|
|
test_API() {
|
|
// These two API are used when fuzz testing the scanner.
|
|
String source = 'class A { }';
|
|
|
|
usedForFuzzTesting.ScannerResult result =
|
|
usedForFuzzTesting.scanString(source);
|
|
expect(result?.hasErrors, isFalse);
|
|
expect(result.tokens?.type, same(Keyword.CLASS));
|
|
|
|
// UTF8 encode source with trailing zero
|
|
List<int> bytes = UTF8.encode(source).toList();
|
|
bytes.add(0);
|
|
|
|
result = usedForFuzzTesting.scan(bytes);
|
|
expect(result?.hasErrors, isFalse);
|
|
expect(result.tokens?.type, same(Keyword.CLASS));
|
|
}
|
|
}
|
|
|
|
@reflectiveTest
|
|
class ScannerTest_Fasta_UTF8 extends ScannerTest_Fasta {
|
|
@override
|
|
createScanner(String source, {bool genericMethodComments: false}) {
|
|
List<int> encoded = UTF8.encode(source).toList(growable: true);
|
|
encoded.add(0); // Ensure 0 terminted bytes for UTF8 scanner
|
|
return new fasta.Utf8BytesScanner(encoded,
|
|
includeComments: true,
|
|
scanGenericMethodComments: genericMethodComments);
|
|
}
|
|
}
|
|
|
|
@reflectiveTest
|
|
class ScannerTest_Fasta extends ScannerTestBase {
|
|
ScannerTest_Fasta() {
|
|
usingFasta = true;
|
|
}
|
|
|
|
createScanner(String source, {bool genericMethodComments: false}) =>
|
|
new fasta.StringScanner(source,
|
|
includeComments: true,
|
|
scanGenericMethodComments: genericMethodComments);
|
|
|
|
@override
|
|
Token scanWithListener(String source, ErrorListener listener,
|
|
{bool genericMethodComments: false,
|
|
bool lazyAssignmentOperators: false}) {
|
|
var scanner =
|
|
createScanner(source, genericMethodComments: genericMethodComments);
|
|
var token = scanner.tokenize();
|
|
return new ToAnalyzerTokenStreamConverter_WithListener(listener)
|
|
.convertTokens(token);
|
|
}
|
|
|
|
void test_comments() {
|
|
const source = '''
|
|
/// Doc comment before class
|
|
/// second line
|
|
/// third
|
|
class Foo {
|
|
// Random comment
|
|
Object someField; // trailing comment
|
|
dynamic secondField;
|
|
/// Method doc
|
|
void someMethod(/* comment before closing paren */) {
|
|
// body comment
|
|
}
|
|
/** Doc comment 2 */
|
|
Foo2 bar() => new Baz();
|
|
} // EOF comment
|
|
''';
|
|
|
|
Token scanSource({bool includeComments}) {
|
|
return new fasta.StringScanner(source, includeComments: includeComments)
|
|
.tokenize();
|
|
}
|
|
|
|
int tokenCount = 0;
|
|
Token token = scanSource(includeComments: false);
|
|
while (!token.isEof) {
|
|
++tokenCount;
|
|
// Assert no comments
|
|
expect(token.precedingComments, isNull);
|
|
expect(token.type.kind, isNot(fasta.COMMENT_TOKEN));
|
|
token = token.next;
|
|
}
|
|
expect(token.precedingComments, isNull);
|
|
expect(tokenCount, 26);
|
|
|
|
tokenCount = 0;
|
|
int previousEnd = 0;
|
|
int spotCheckCount = 0;
|
|
int commentTokenCount = 0;
|
|
token = scanSource(includeComments: true);
|
|
while (!token.isEof) {
|
|
++tokenCount;
|
|
// Assert valid comments
|
|
fasta.CommentToken comment = token.precedingComments;
|
|
while (comment != null) {
|
|
++commentTokenCount;
|
|
expect(comment.type.kind, fasta.COMMENT_TOKEN);
|
|
expect(comment.charOffset, greaterThanOrEqualTo(previousEnd));
|
|
previousEnd = comment.charOffset + comment.charCount;
|
|
comment = comment.next;
|
|
}
|
|
expect(token.type.kind, isNot(fasta.COMMENT_TOKEN));
|
|
expect(token.charOffset, greaterThanOrEqualTo(previousEnd));
|
|
previousEnd = token.charOffset + token.charCount;
|
|
|
|
// Spot check for specific token/comment combinations
|
|
if (token.lexeme == 'class') {
|
|
++spotCheckCount;
|
|
expect(token.precedingComments?.lexeme, '/// Doc comment before class');
|
|
expect(token.precedingComments?.next?.lexeme, '/// second line');
|
|
expect(token.precedingComments?.next?.next?.lexeme, '/// third');
|
|
expect(token.precedingComments?.next?.next?.next, isNull);
|
|
} else if (token.lexeme == 'Foo2') {
|
|
++spotCheckCount;
|
|
expect(token.precedingComments?.lexeme, '/** Doc comment 2 */');
|
|
} else if (token.lexeme == ')') {
|
|
if (token.precedingComments != null) {
|
|
++spotCheckCount;
|
|
expect(token.precedingComments?.lexeme,
|
|
'/* comment before closing paren */');
|
|
expect(token.precedingComments?.next, isNull);
|
|
}
|
|
}
|
|
|
|
token = token.next;
|
|
}
|
|
expect(tokenCount, 26);
|
|
expect(spotCheckCount, 3);
|
|
expect(commentTokenCount, 9);
|
|
expect(token.precedingComments?.lexeme, '// EOF comment');
|
|
}
|
|
|
|
void test_CommentToken_remove() {
|
|
const code = '''
|
|
/// aaa
|
|
/// bbbb
|
|
/// ccccc
|
|
main() {}
|
|
''';
|
|
|
|
Token token;
|
|
fasta.CommentToken c1;
|
|
fasta.CommentToken c2;
|
|
fasta.CommentToken c3;
|
|
|
|
void prepareTokens() {
|
|
token = new fasta.StringScanner(code, includeComments: true).tokenize();
|
|
|
|
expect(token.type.kind, fasta.IDENTIFIER_TOKEN);
|
|
|
|
c1 = token.precedingComments;
|
|
c2 = c1.next;
|
|
c3 = c2.next;
|
|
expect(c3.next, isNull);
|
|
|
|
expect(c1.parent, token);
|
|
expect(c2.parent, token);
|
|
expect(c3.parent, token);
|
|
|
|
expect(c1.lexeme, '/// aaa');
|
|
expect(c2.lexeme, '/// bbbb');
|
|
expect(c3.lexeme, '/// ccccc');
|
|
}
|
|
|
|
// Remove the first token.
|
|
{
|
|
prepareTokens();
|
|
c1.remove();
|
|
expect(token.precedingComments, c2);
|
|
expect(c2.next, c3);
|
|
expect(c3.next, isNull);
|
|
}
|
|
|
|
// Remove the second token.
|
|
{
|
|
prepareTokens();
|
|
c2.remove();
|
|
expect(token.precedingComments, c1);
|
|
expect(c1.next, c3);
|
|
expect(c3.next, isNull);
|
|
}
|
|
|
|
// Remove the last token.
|
|
{
|
|
prepareTokens();
|
|
c3.remove();
|
|
expect(token.precedingComments, c1);
|
|
expect(c1.next, c2);
|
|
expect(c2.next, isNull);
|
|
}
|
|
}
|
|
|
|
@override
|
|
void test_mismatched_opener_in_interpolation() {
|
|
// When openers and closers are mismatched,
|
|
// fasta favors considering the opener to be mismatched
|
|
// and inserts synthetic closers as needed.
|
|
// r'"${({(}}"' is parsed as r'"${({()})}"'
|
|
// where both ')' are synthetic
|
|
ErrorListener listener = new ErrorListener();
|
|
BeginToken interpolationStart =
|
|
scanWithListener(r'"${({(}}"', listener).next;
|
|
BeginToken openParen1 = interpolationStart.next;
|
|
BeginToken openBrace = openParen1.next;
|
|
BeginToken openParen2 = openBrace.next;
|
|
var closeParen2 = openParen2.next;
|
|
var closeBrace = closeParen2.next;
|
|
var closeParen1 = closeBrace.next;
|
|
var interpolationEnd = closeParen1.next;
|
|
var stringEnd = interpolationEnd.next;
|
|
expect(stringEnd.next.type, TokenType.EOF);
|
|
expect(interpolationStart.endToken, same(interpolationEnd));
|
|
expect(openParen1.endToken, same(closeParen1));
|
|
expect(openBrace.endToken, same(closeBrace));
|
|
expect(openParen2.endToken, same(closeParen2));
|
|
listener.assertErrors([
|
|
new TestError(3, ScannerErrorCode.EXPECTED_TOKEN, [')']),
|
|
new TestError(5, ScannerErrorCode.EXPECTED_TOKEN, [')']),
|
|
]);
|
|
}
|
|
|
|
void test_next_previous() {
|
|
const source = 'int a; /*1*/ /*2*/ /*3*/ B f(){if (a < 2) {}}';
|
|
Token token =
|
|
new fasta.StringScanner(source, includeComments: true).tokenize();
|
|
while (!token.isEof) {
|
|
expect(token.next.previous, token);
|
|
fasta.CommentToken commentToken = token.precedingComments;
|
|
while (commentToken != null) {
|
|
if (commentToken.next != null) {
|
|
expect(commentToken.next.previous, commentToken);
|
|
}
|
|
commentToken = commentToken.next;
|
|
}
|
|
token = token.next;
|
|
}
|
|
}
|
|
|
|
@override
|
|
void test_unmatched_openers() {
|
|
ErrorListener listener = new ErrorListener();
|
|
BeginToken openBrace = scanWithListener('{[(', listener);
|
|
BeginToken openBracket = openBrace.next;
|
|
BeginToken openParen = openBracket.next;
|
|
var closeParen = openParen.next;
|
|
var closeBracket = closeParen.next;
|
|
var closeBrace = closeBracket.next;
|
|
expect(closeBrace.next.type, TokenType.EOF);
|
|
expect(openBrace.endToken, same(closeBrace));
|
|
expect(openBracket.endToken, same(closeBracket));
|
|
expect(openParen.endToken, same(closeParen));
|
|
listener.assertErrors([
|
|
new TestError(0, ScannerErrorCode.EXPECTED_TOKEN, ['}']),
|
|
new TestError(1, ScannerErrorCode.EXPECTED_TOKEN, [']']),
|
|
new TestError(2, ScannerErrorCode.EXPECTED_TOKEN, [')']),
|
|
]);
|
|
}
|
|
}
|
|
|
|
/// Base class for scanner tests that examine the token stream in Fasta format.
|
|
abstract class ScannerTest_Fasta_Base {
|
|
Token scan(String source);
|
|
|
|
expectToken(Token token, TokenType type, int offset, int length,
|
|
{bool isSynthetic: false, String lexeme}) {
|
|
String description = '${token.type} $token';
|
|
expect(token.type, type, reason: description);
|
|
expect(token.offset, offset, reason: description);
|
|
expect(token.length, length, reason: description);
|
|
expect(token.isSynthetic, isSynthetic, reason: description);
|
|
if (lexeme != null) {
|
|
expect(token.lexeme, lexeme, reason: description);
|
|
}
|
|
}
|
|
|
|
void test_string_simple_interpolation_missingIdentifier() {
|
|
Token token = scan("'\$x\$'");
|
|
expectToken(token, TokenType.STRING, 0, 1, lexeme: "'");
|
|
|
|
token = token.next;
|
|
expectToken(token, TokenType.STRING_INTERPOLATION_IDENTIFIER, 1, 1);
|
|
|
|
token = token.next;
|
|
expectToken(token, TokenType.IDENTIFIER, 2, 1, lexeme: 'x');
|
|
|
|
token = token.next;
|
|
expectToken(token, TokenType.STRING, 3, 0, lexeme: '', isSynthetic: true);
|
|
|
|
token = token.next;
|
|
expectToken(token, TokenType.STRING_INTERPOLATION_IDENTIFIER, 3, 1);
|
|
|
|
token = token.next;
|
|
expectToken(token, TokenType.IDENTIFIER, 4, 0,
|
|
lexeme: '', isSynthetic: true);
|
|
|
|
token = token.next;
|
|
expect((token as fasta.ErrorToken).errorCode,
|
|
same(codeUnexpectedDollarInString));
|
|
|
|
token = token.next;
|
|
expectToken(token, TokenType.STRING, 4, 1, lexeme: "'");
|
|
}
|
|
|
|
void test_string_simple_unterminated_interpolation_block() {
|
|
Token token = scan(r'"foo ${bar');
|
|
expectToken(token, TokenType.STRING, 0, 5, lexeme: '"foo ');
|
|
|
|
token = token.next;
|
|
expectToken(token, TokenType.STRING_INTERPOLATION_EXPRESSION, 5, 2);
|
|
BeginToken interpolationStart = token;
|
|
|
|
token = token.next;
|
|
expectToken(token, TokenType.IDENTIFIER, 7, 3, lexeme: 'bar');
|
|
|
|
// Expect interpolation to be terminated before string is closed
|
|
token = token.next;
|
|
expectToken(token, TokenType.CLOSE_CURLY_BRACKET, 10, 0,
|
|
isSynthetic: true, lexeme: '}');
|
|
expect(interpolationStart.endToken, same(token));
|
|
|
|
token = token.next;
|
|
expect((token as fasta.ErrorToken).errorCode, same(codeUnmatchedToken));
|
|
expect((token as fasta.UnmatchedToken).begin, same(interpolationStart));
|
|
|
|
token = token.next;
|
|
expectToken(token, TokenType.STRING, 10, 0, isSynthetic: true, lexeme: '"');
|
|
|
|
token = token.next;
|
|
expect((token as fasta.ErrorToken).errorCode, same(codeUnterminatedString));
|
|
}
|
|
|
|
void test_string_simple_unterminated_interpolation_block2() {
|
|
Token token = scan(r'"foo ${bar(baz[');
|
|
expectToken(token, TokenType.STRING, 0, 5, lexeme: '"foo ');
|
|
|
|
token = token.next;
|
|
expectToken(token, TokenType.STRING_INTERPOLATION_EXPRESSION, 5, 2);
|
|
BeginToken interpolationStart = token;
|
|
|
|
token = token.next;
|
|
expectToken(token, TokenType.IDENTIFIER, 7, 3, lexeme: 'bar');
|
|
|
|
token = token.next;
|
|
expectToken(token, TokenType.OPEN_PAREN, 10, 1);
|
|
BeginToken openParen = token;
|
|
|
|
token = token.next;
|
|
expectToken(token, TokenType.IDENTIFIER, 11, 3, lexeme: 'baz');
|
|
|
|
token = token.next;
|
|
expectToken(token, TokenType.OPEN_SQUARE_BRACKET, 14, 1);
|
|
BeginToken openSquareBracket = token;
|
|
|
|
token = token.next;
|
|
expectToken(token, TokenType.CLOSE_SQUARE_BRACKET, 15, 0,
|
|
isSynthetic: true, lexeme: ']');
|
|
expect(openSquareBracket.endToken, same(token));
|
|
|
|
token = token.next;
|
|
expect((token as fasta.ErrorToken).errorCode, same(codeUnmatchedToken));
|
|
expect((token as fasta.UnmatchedToken).begin, same(openSquareBracket));
|
|
|
|
token = token.next;
|
|
expectToken(token, TokenType.CLOSE_PAREN, 15, 0,
|
|
isSynthetic: true, lexeme: ')');
|
|
expect(openParen.endToken, same(token));
|
|
|
|
token = token.next;
|
|
expect((token as fasta.ErrorToken).errorCode, same(codeUnmatchedToken));
|
|
expect((token as fasta.UnmatchedToken).begin, same(openParen));
|
|
|
|
token = token.next;
|
|
expectToken(token, TokenType.CLOSE_CURLY_BRACKET, 15, 0,
|
|
isSynthetic: true, lexeme: '}');
|
|
expect(interpolationStart.endToken, same(token));
|
|
|
|
token = token.next;
|
|
expect((token as fasta.ErrorToken).errorCode, same(codeUnmatchedToken));
|
|
expect((token as fasta.UnmatchedToken).begin, same(interpolationStart));
|
|
|
|
token = token.next;
|
|
expectToken(token, TokenType.STRING, 15, 0, isSynthetic: true, lexeme: '"');
|
|
|
|
token = token.next;
|
|
expect((token as fasta.ErrorToken).errorCode, same(codeUnterminatedString));
|
|
}
|
|
|
|
void test_string_simple_missing_interpolation_identifier() {
|
|
Token token = scan(r'"foo $');
|
|
expectToken(token, TokenType.STRING, 0, 5, lexeme: '"foo ');
|
|
|
|
token = token.next;
|
|
expectToken(token, TokenType.STRING_INTERPOLATION_IDENTIFIER, 5, 1);
|
|
|
|
token = token.next;
|
|
expectToken(token, TokenType.IDENTIFIER, 6, 0,
|
|
isSynthetic: true, lexeme: '');
|
|
|
|
token = token.next;
|
|
expect((token as fasta.ErrorToken).errorCode,
|
|
same(codeUnexpectedDollarInString));
|
|
expect((token as fasta.UnterminatedToken).start, r'$');
|
|
|
|
token = token.next;
|
|
expectToken(token, TokenType.STRING, 6, 0, isSynthetic: true, lexeme: '"');
|
|
|
|
token = token.next;
|
|
expect((token as fasta.ErrorToken).errorCode, same(codeUnterminatedString));
|
|
}
|
|
|
|
void test_string_multi_unterminated() {
|
|
Token token = scan("'''string");
|
|
expectToken(token, TokenType.STRING, 0, 9,
|
|
lexeme: "'''string'''", isSynthetic: true);
|
|
|
|
token = token.next;
|
|
expect((token as fasta.ErrorToken).errorCode, same(codeUnterminatedString));
|
|
expect((token as fasta.ErrorToken).start, "'''");
|
|
}
|
|
|
|
void test_string_raw_multi_unterminated() {
|
|
Token token = scan("r'''string");
|
|
expectToken(token, TokenType.STRING, 0, 10,
|
|
lexeme: "r'''string'''", isSynthetic: true);
|
|
|
|
token = token.next;
|
|
expect((token as fasta.ErrorToken).errorCode, same(codeUnterminatedString));
|
|
expect((token as fasta.ErrorToken).start, "r'''");
|
|
}
|
|
|
|
void test_string_raw_simple_unterminated_eof() {
|
|
Token token = scan("r'string");
|
|
expectToken(token, TokenType.STRING, 0, 8,
|
|
lexeme: "r'string'", isSynthetic: true);
|
|
|
|
token = token.next;
|
|
expect((token as fasta.ErrorToken).errorCode, same(codeUnterminatedString));
|
|
expect((token as fasta.ErrorToken).start, "r'");
|
|
}
|
|
|
|
void test_string_raw_simple_unterminated_eol() {
|
|
Token token = scan("r'string\n");
|
|
expectToken(token, TokenType.STRING, 0, 8,
|
|
lexeme: "r'string'", isSynthetic: true);
|
|
|
|
token = token.next;
|
|
expect((token as fasta.ErrorToken).errorCode, same(codeUnterminatedString));
|
|
expect((token as fasta.ErrorToken).start, "r'");
|
|
}
|
|
|
|
void test_string_simple_unterminated_eof() {
|
|
Token token = scan("'string");
|
|
expectToken(token, TokenType.STRING, 0, 7,
|
|
lexeme: "'string'", isSynthetic: true);
|
|
|
|
token = token.next;
|
|
expect((token as fasta.ErrorToken).errorCode, same(codeUnterminatedString));
|
|
expect((token as fasta.ErrorToken).start, "'");
|
|
}
|
|
|
|
void test_string_simple_unterminated_eol() {
|
|
Token token = scan("'string\n");
|
|
expectToken(token, TokenType.STRING, 0, 7,
|
|
lexeme: "'string'", isSynthetic: true);
|
|
|
|
token = token.next;
|
|
expect((token as fasta.ErrorToken).errorCode, same(codeUnterminatedString));
|
|
expect((token as fasta.ErrorToken).start, "'");
|
|
}
|
|
|
|
void test_match_angle_brackets() {
|
|
var x = scan('x<y>');
|
|
BeginToken lessThan = x.next;
|
|
var y = lessThan.next;
|
|
var greaterThan = y.next;
|
|
expect(greaterThan.next.isEof, isTrue);
|
|
expect(lessThan.endGroup, same(greaterThan));
|
|
}
|
|
|
|
void test_match_angle_brackets_gt_gt() {
|
|
// When a ">>" appears in the token stream, Fasta's scanner matches it to
|
|
// the outer "<". The inner "<" is left unmatched.
|
|
var x = scan('x<y<z>>');
|
|
BeginToken lessThan1 = x.next;
|
|
var y = lessThan1.next;
|
|
BeginToken lessThan2 = y.next;
|
|
var z = lessThan2.next;
|
|
var greaterThans = z.next;
|
|
expect(greaterThans.next.isEof, isTrue);
|
|
expect(lessThan1.endGroup, same(greaterThans));
|
|
expect(lessThan2.endGroup, isNull);
|
|
}
|
|
|
|
void test_match_angle_brackets_interrupted_by_close_brace() {
|
|
// A "}" appearing in the token stream interrupts matching of "<" and ">".
|
|
BeginToken openBrace = scan('{x<y}>z');
|
|
var x = openBrace.next;
|
|
BeginToken lessThan = x.next;
|
|
var y = lessThan.next;
|
|
var closeBrace = y.next;
|
|
var greaterThan = closeBrace.next;
|
|
var z = greaterThan.next;
|
|
expect(z.next.isEof, isTrue);
|
|
expect(openBrace.endGroup, same(closeBrace));
|
|
expect(lessThan.endGroup, isNull);
|
|
}
|
|
|
|
void test_match_angle_brackets_interrupted_by_close_bracket() {
|
|
// A "]" appearing in the token stream interrupts matching of "<" and ">".
|
|
BeginToken openBracket = scan('[x<y]>z');
|
|
var x = openBracket.next;
|
|
BeginToken lessThan = x.next;
|
|
var y = lessThan.next;
|
|
var closeBracket = y.next;
|
|
var greaterThan = closeBracket.next;
|
|
var z = greaterThan.next;
|
|
expect(z.next.isEof, isTrue);
|
|
expect(openBracket.endGroup, same(closeBracket));
|
|
expect(lessThan.endGroup, isNull);
|
|
}
|
|
|
|
void test_match_angle_brackets_interrupted_by_close_paren() {
|
|
// A ")" appearing in the token stream interrupts matching of "<" and ">".
|
|
BeginToken openParen = scan('(x<y)>z');
|
|
var x = openParen.next;
|
|
BeginToken lessThan = x.next;
|
|
var y = lessThan.next;
|
|
var closeParen = y.next;
|
|
var greaterThan = closeParen.next;
|
|
var z = greaterThan.next;
|
|
expect(z.next.isEof, isTrue);
|
|
expect(openParen.endGroup, same(closeParen));
|
|
expect(lessThan.endGroup, isNull);
|
|
}
|
|
|
|
void test_match_angle_brackets_interrupted_by_interpolation_expr() {
|
|
// A "${" appearing in the token stream interrupts matching of "<" and ">".
|
|
var x = scan(r'x<"${y>z}"');
|
|
BeginToken lessThan = x.next;
|
|
var beginString = lessThan.next;
|
|
BeginToken beginInterpolation = beginString.next;
|
|
var y = beginInterpolation.next;
|
|
var greaterThan = y.next;
|
|
var z = greaterThan.next;
|
|
var endInterpolation = z.next;
|
|
var endString = endInterpolation.next;
|
|
expect(endString.next.isEof, isTrue);
|
|
expect(lessThan.endGroup, isNull);
|
|
expect(beginInterpolation.endGroup, same(endInterpolation));
|
|
}
|
|
|
|
void test_match_angle_brackets_interrupted_by_open_brace() {
|
|
// A "{" appearing in the token stream interrupts matching of "<" and ">".
|
|
var x = scan('x<{y>z}');
|
|
BeginToken lessThan = x.next;
|
|
BeginToken openBrace = lessThan.next;
|
|
var y = openBrace.next;
|
|
var greaterThan = y.next;
|
|
var z = greaterThan.next;
|
|
var closeBrace = z.next;
|
|
expect(closeBrace.next.isEof, isTrue);
|
|
expect(lessThan.endGroup, isNull);
|
|
expect(openBrace.endGroup, same(closeBrace));
|
|
}
|
|
|
|
void test_match_angle_brackets_interrupted_by_open_bracket() {
|
|
// A "[" appearing in the token stream interrupts matching of "<" and ">".
|
|
var x = scan('x<y[z>a]');
|
|
BeginToken lessThan = x.next;
|
|
var y = lessThan.next;
|
|
BeginToken openBracket = y.next;
|
|
var z = openBracket.next;
|
|
var greaterThan = z.next;
|
|
var a = greaterThan.next;
|
|
var closeBracket = a.next;
|
|
expect(closeBracket.next.isEof, isTrue);
|
|
expect(lessThan.endGroup, isNull);
|
|
expect(openBracket.endGroup, same(closeBracket));
|
|
}
|
|
|
|
void test_match_angle_brackets_interrupted_by_open_paren() {
|
|
// A "(" appearing in the token stream interrupts matching of "<" and ">".
|
|
var x = scan('x<y(z>a)');
|
|
BeginToken lessThan = x.next;
|
|
var y = lessThan.next;
|
|
BeginToken openParen = y.next;
|
|
var z = openParen.next;
|
|
var greaterThan = z.next;
|
|
var a = greaterThan.next;
|
|
var closeParen = a.next;
|
|
expect(closeParen.next.isEof, isTrue);
|
|
expect(lessThan.endGroup, isNull);
|
|
expect(openParen.endGroup, same(closeParen));
|
|
}
|
|
|
|
void test_match_angle_brackets_nested() {
|
|
var x = scan('x<y<z>,a>');
|
|
BeginToken lessThan1 = x.next;
|
|
var y = lessThan1.next;
|
|
BeginToken lessThan2 = y.next;
|
|
var z = lessThan2.next;
|
|
var greaterThan1 = z.next;
|
|
var comma = greaterThan1.next;
|
|
var a = comma.next;
|
|
var greaterThan2 = a.next;
|
|
expect(greaterThan2.next.isEof, isTrue);
|
|
expect(lessThan1.endGroup, same(greaterThan2));
|
|
expect(lessThan2.endGroup, same(greaterThan1));
|
|
}
|
|
|
|
void test_match_angle_brackets_unmatched_gt_gt() {
|
|
// When a ">>" appears in the token stream and there is no outer "<",
|
|
// Fasta's scanner leaves the inner "<" unmatched.
|
|
var x = scan('x<y>>z');
|
|
BeginToken lessThan = x.next;
|
|
var y = lessThan.next;
|
|
var greaterThans = y.next;
|
|
var z = greaterThans.next;
|
|
expect(z.next.isEof, isTrue);
|
|
expect(lessThan.endGroup, isNull);
|
|
}
|
|
}
|
|
|
|
/// Scanner tests that exercise the Fasta scanner directly.
|
|
@reflectiveTest
|
|
class ScannerTest_Fasta_Direct_UTF8 extends ScannerTest_Fasta_Direct {
|
|
createScanner(String source, {bool includeComments}) {
|
|
List<int> encoded = UTF8.encode(source).toList(growable: true);
|
|
encoded.add(0); // Ensure 0 terminted bytes for UTF8 scanner
|
|
return new fasta.Utf8BytesScanner(encoded,
|
|
includeComments: includeComments);
|
|
}
|
|
}
|
|
|
|
/// Scanner tests that exercise the Fasta scanner directly.
|
|
@reflectiveTest
|
|
class ScannerTest_Fasta_Direct extends ScannerTest_Fasta_Base {
|
|
createScanner(String source, {bool includeComments}) =>
|
|
new fasta.StringScanner(source, includeComments: includeComments);
|
|
|
|
@override
|
|
Token scan(String source) {
|
|
return createScanner(source, includeComments: true).tokenize();
|
|
}
|
|
|
|
void test_linestarts() {
|
|
var scanner = createScanner("var\r\ni\n=\n1;\n");
|
|
var token = scanner.tokenize();
|
|
expect(token.lexeme, 'var');
|
|
var lineStarts = scanner.lineStarts;
|
|
expect(lineStarts, orderedEquals([0, 5, 7, 9, 12, 13]));
|
|
}
|
|
|
|
void test_linestarts_synthetic_string() {
|
|
var scanner = createScanner("var\r\ns\n=\n'eh'\n'eh\n;\n");
|
|
Token firstToken = scanner.tokenize();
|
|
expect(firstToken.lexeme, 'var');
|
|
var lineStarts = scanner.lineStarts;
|
|
expect(lineStarts, orderedEquals([0, 5, 7, 9, 14, 18, 20, 21]));
|
|
var token = firstToken;
|
|
int index = 0;
|
|
while (!token.isEof) {
|
|
if (token is fasta.ErrorToken) {
|
|
expect(token.charOffset, 14,
|
|
reason: 'error token : $token, ${token.type}');
|
|
expect(token.charCount, 3,
|
|
reason: 'error token : $token, ${token.type}');
|
|
} else {
|
|
expect(token.charOffset, lineStarts[index],
|
|
reason: 'token # $index : $token, ${token.type}');
|
|
++index;
|
|
}
|
|
token = token.next;
|
|
}
|
|
}
|
|
|
|
void test_linestarts_synthetic_string_utf8() {
|
|
var scanner = createScanner("var\r\ns\n=\n'éh'\n'éh\n;\n");
|
|
Token firstToken = scanner.tokenize();
|
|
expect(firstToken.lexeme, 'var');
|
|
var lineStarts = scanner.lineStarts;
|
|
expect(lineStarts, orderedEquals([0, 5, 7, 9, 14, 18, 20, 21]));
|
|
var token = firstToken;
|
|
int index = 0;
|
|
while (!token.isEof) {
|
|
if (token is! fasta.ErrorToken) {
|
|
expect(token.charOffset, lineStarts[index],
|
|
reason: 'token # $index : $token, ${token.type}');
|
|
++index;
|
|
}
|
|
token = token.next;
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Override of [ToAnalyzerTokenStreamConverter] that verifies that there are no
|
|
/// errors.
|
|
class ToAnalyzerTokenStreamConverter_NoErrors
|
|
extends ToAnalyzerTokenStreamConverter {
|
|
@override
|
|
void reportError(
|
|
ScannerErrorCode errorCode, int offset, List<Object> arguments) {
|
|
fail('Unexpected error: $errorCode, $offset, $arguments');
|
|
}
|
|
}
|
|
|
|
/// Override of [ToAnalyzerTokenStreamConverter] that records errors in an
|
|
/// [ErrorListener].
|
|
class ToAnalyzerTokenStreamConverter_WithListener
|
|
extends ToAnalyzerTokenStreamConverter {
|
|
final ErrorListener _listener;
|
|
|
|
ToAnalyzerTokenStreamConverter_WithListener(this._listener);
|
|
|
|
@override
|
|
void reportError(
|
|
ScannerErrorCode errorCode, int offset, List<Object> arguments) {
|
|
_listener.errors.add(new TestError(offset, errorCode, arguments));
|
|
}
|
|
}
|