languages/dart-sdk
languages/dart-sdk
1
0
Fork 0
mirror of https://github.com/dart-lang/sdk synced 2024-11-02 08:20:31 +00:00
Code Issues Projects Releases Packages Wiki Activity

Refactor comment reference parsing, away from stack.

Browse source 
Now that comment reference parsing is done entirely in ast_builder, we
can simplify the implementation.

Work towards https://github.com/dart-lang/sdk/issues/50702

Change-Id: I0650706dfe31542454c7bc9832ec6104c141bd5d
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/316643
Commit-Queue: Samuel Rawlins <srawlins@google.com>
Reviewed-by: Paul Berry <paulberry@google.com>
This commit is contained in:
Sam Rawlins 2023-07-31 20:23:19 +00:00 committed by Commit Queue
parent c9d69866eb
commit 1934efbffb
6 changed files with 118 additions and 302 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
pkg/_fe_analyzer_shared/lib/src/parser/forwarding_listener.dart
View file

@ -1407,23 +1407,6 @@ class ForwardingListener implements Listener {
listener?.handleMixinWithClause(withKeyword);
}
@override
void handleCommentReference(
Token? newKeyword,
Token? firstToken,
Token? firstPeriod,
Token? secondToken,
Token? secondPeriod,
Token thirdToken) {
listener?.handleCommentReference(newKeyword, firstToken, firstPeriod,
secondToken, secondPeriod, thirdToken);
}
@override
void handleCommentReferenceText(String referenceSource, int referenceOffset) {
listener?.handleCommentReferenceText(referenceSource, referenceOffset);
}
@override
void handleConditionalExpressionColon() {
listener?.handleConditionalExpressionColon();
@ -1752,11 +1735,6 @@ class ForwardingListener implements Listener {
listener?.handleNoArguments(token);
}
@override
void handleNoCommentReference() {
listener?.handleNoCommentReference();
}
@override
void handleNoConstructorReferenceContinuationAfterTypeArguments(Token token) {
listener?.handleNoConstructorReferenceContinuationAfterTypeArguments(token);

33
pkg/_fe_analyzer_shared/lib/src/parser/listener.dart
View file

@ -2287,39 +2287,6 @@ class Listener implements UnescapeErrorListener {
logEvent("Script");
}
/// A single comment reference has been found
/// where [referenceSource] is the text between the `[` and `]`
/// and [referenceOffset] is the character offset in the token stream.
///
/// This event is generated by the parser when the parser's
/// `parseCommentReferences` method is called. For further processing,
/// a listener may scan the [referenceSource] and then pass the resulting
/// token stream to the parser's `parseOneCommentReference` method.
void handleCommentReferenceText(String referenceSource, int referenceOffset) {
logEvent("CommentReferenceText");
}
/// A single comment reference has been parsed.
/// * [newKeyword] may be null.
/// * [firstToken] and [firstPeriod] are either both tokens or both
/// `null`.
/// * [secondToken] and [secondPeriod] are either both tokens or both `null`.
/// * [thirdToken] can be an identifier or an operator.
///
/// This event is generated by the parser when the parser's
/// `parseOneCommentReference` method is called.
void handleCommentReference(
Token? newKeyword,
Token? firstToken,
Token? firstPeriod,
Token? secondToken,
Token? secondPeriod,
Token thirdToken) {}
/// This event is generated by the parser when the parser's
/// `parseOneCommentReference` method is called.
void handleNoCommentReference() {}
/// An expression was encountered consisting of type arguments applied to a
/// subexpression. This could validly represent any of the following:
/// - A type literal (`var x = List<int>;`)

221
pkg/analyzer/lib/src/fasta/ast_builder.dart
View file

@ -3830,61 +3830,6 @@ class AstBuilder extends StackListener {
);
}
@override
void handleCommentReference(
Token? newKeyword,
Token? firstToken,
Token? firstPeriod,
Token? secondToken,
Token? secondPeriod,
Token thirdToken,
) {
var identifier = SimpleIdentifierImpl(thirdToken);
if (firstToken != null) {
var target = PrefixedIdentifierImpl(
prefix: SimpleIdentifierImpl(firstToken),
period: firstPeriod!,
identifier: SimpleIdentifierImpl(secondToken!),
);
var expression = PropertyAccessImpl(
target: target,
operator: secondPeriod!,
propertyName: identifier,
);
push(
CommentReferenceImpl(
newKeyword: newKeyword,
expression: expression,
),
);
} else if (secondToken != null) {
var expression = PrefixedIdentifierImpl(
prefix: SimpleIdentifierImpl(secondToken),
period: secondPeriod!,
identifier: identifier,
);
push(
CommentReferenceImpl(
newKeyword: newKeyword,
expression: expression,
),
);
} else {
push(
CommentReferenceImpl(
newKeyword: newKeyword,
expression: identifier,
),
);
}
}
@override
void handleCommentReferenceText(String referenceSource, int referenceOffset) {
push(referenceSource);
push(referenceOffset);
}
@override
void handleConstFactory(Token constKeyword) {
debugEvent("ConstFactory");
@ -5585,28 +5530,24 @@ class AstBuilder extends StackListener {
/// [dartdoc] is the first token in the sequence.
List<CommentReferenceImpl> parseCommentReferences(Token dartdoc) {
// Parse dartdoc into potential comment reference source/offset pairs.
var count = dartdoc.lexeme.startsWith('///')
var sourcesAndOffsets = dartdoc.lexeme.startsWith('///')
? _parseReferencesInSingleLineComments(dartdoc)
: _parseReferencesInMultiLineComment(dartdoc);
var sourcesAndOffsets = List<Object?>.filled(count * 2, null);
popList(count * 2, sourcesAndOffsets);
var references = <CommentReferenceImpl>[];
// Parse each of the source/offset pairs into actual comment references.
count = 0;
var index = 0;
while (index < sourcesAndOffsets.length) {
var referenceSource = sourcesAndOffsets[index++] as String;
var referenceOffset = sourcesAndOffsets[index++] as int;
var result = scanString(referenceSource);
for (var (:source, :offset) in sourcesAndOffsets) {
var result = scanString(source);
if (!result.hasErrors) {
var token = result.tokens;
if (_parseOneCommentReference(token, referenceOffset)) {
++count;
var reference = _parseOneCommentReference(token, offset);
if (reference != null) {
references.add(reference);
}
}
}
return popTypedList<CommentReferenceImpl>(count) ?? const [];
return references;
}
List<CollectionElementImpl> popCollectionElements(int count) {
@ -5815,7 +5756,7 @@ class AstBuilder extends StackListener {
CommentImpl? _findComment(
List<AnnotationImpl>? metadata, Token tokenAfterMetadata) {
// Find the dartdoc tokens
// Find the dartdoc tokens.
var dartdoc = parser.findDartDoc(tokenAfterMetadata);
if (dartdoc == null) {
if (metadata == null) {
@ -5902,13 +5843,22 @@ class AstBuilder extends StackListener {
);
}
/// Parse the comment references in the text between [start] inclusive
/// Parses the comment references in the text between [start] inclusive
/// and [end] exclusive.
///
/// Return the number of comment references that were parsed.
int _parseCommentReferencesInText(Token commentToken, int start, int end) {
/// Returns information about the comment references as a list of records,
/// each with a `source` field and an `offset` field. The `source` is the text
/// between the delimiting `[` and `]` characters, not including them. The
/// `offset` is the offset of the comment reference in the containing
/// compilation unit.
///
/// For example, for the text `/// [a] and [b.c].`, two records are returned:
/// `(source: 'a', offset: 5)` and `(source: 'b.c', offset: 13)` (assuming the
/// comment is the beginning of the compilation unit).
List<({String source, int offset})> _parseCommentReferencesInText(
Token commentToken, int start, int end) {
var comment = commentToken.lexeme;
var count = 0;
var references = <({String source, int offset})>[];
var index = start;
while (index < end) {
var ch = comment.codeUnitAt(index);
@ -5932,10 +5882,10 @@ class AstBuilder extends StackListener {
// TODO(brianwilkerson) Handle the case where there's a library
// URI in the link text.
} else {
/*listener.*/ handleCommentReferenceText(
comment.substring(referenceStart, index),
commentToken.charOffset + referenceStart);
++count;
references.add((
source: comment.substring(referenceStart, index),
offset: commentToken.charOffset + referenceStart,
));
}
}
}
@ -5948,13 +5898,14 @@ class AstBuilder extends StackListener {
}
++index;
}
return count;
return references;
}
/// Parse the tokens in a single comment reference and generate either a
/// [_handleCommentReference] or [_handleNoCommentReference] event.
/// Return `true` if a comment reference was successfully parsed.
bool _parseOneCommentReference(Token token, int referenceOffset) {
/// Parses the text in a single comment reference.
///
/// Returns `null` if the text could not be parsed as a comment reference.
CommentReferenceImpl? _parseOneCommentReference(
Token token, int referenceOffset) {
var begin = token;
Token? newKeyword;
if (optional('new', token)) {
@ -5999,17 +5950,31 @@ class AstBuilder extends StackListener {
}
if (token.isUserDefinableOperator) {
if (token.next!.isEof) {
_parseOneCommentReferenceRest(begin, referenceOffset, newKeyword,
firstToken, firstPeriod, secondToken, secondPeriod, token);
return true;
return _parseOneCommentReferenceRest(
begin,
referenceOffset,
newKeyword,
firstToken,
firstPeriod,
secondToken,
secondPeriod,
token,
);
}
} else {
token = operatorKeyword ?? token;
if (token.next!.isEof) {
if (token.isIdentifier) {
_parseOneCommentReferenceRest(begin, referenceOffset, newKeyword,
firstToken, firstPeriod, secondToken, secondPeriod, token);
return true;
return _parseOneCommentReferenceRest(
begin,
referenceOffset,
newKeyword,
firstToken,
firstPeriod,
secondToken,
secondPeriod,
token,
);
}
var keyword = token.keyword;
if (newKeyword == null &&
@ -6026,11 +5991,29 @@ class AstBuilder extends StackListener {
}
}
}
handleNoCommentReference();
return false;
return null;
}
void _parseOneCommentReferenceRest(
/// Parses the parameters into a [CommentReferenceImpl].
///
/// If the reference begins with `new `, then pass the Token associated with
/// that text as [newToken].
///
/// If the reference contains a single identifier or operator (aside from the
/// optional [newToken]), then pass the associated Token as
/// [identifierOrOperator].
///
/// If the reference contains two identifiers separated by a period, then pass
/// the associated Tokens as [secondToken], [secondPeriod], and
/// [identifierOrOperator], in lexical order.
// TODO(srawlins): Rename the parameters or refactor this code to avoid the
// confusion of `null` values for the "first*" parameters and non-`null` values
// for the "second*" parameters.
///
/// If the reference contains three identifiers, each separated by a period,
/// then pass the associated Tokens as [firstToken], [firstPeriod],
/// [secondToken], [secondPeriod], and [identifierOrOperator].
CommentReferenceImpl _parseOneCommentReferenceRest(
Token begin,
int referenceOffset,
Token? newKeyword,
@ -6046,16 +6029,46 @@ class AstBuilder extends StackListener {
token = token.next!;
} while (!token.isEof);
handleCommentReference(newKeyword, firstToken, firstPeriod, secondToken,
secondPeriod, identifierOrOperator);
var identifier = SimpleIdentifierImpl(identifierOrOperator);
if (firstToken != null) {
var target = PrefixedIdentifierImpl(
prefix: SimpleIdentifierImpl(firstToken),
period: firstPeriod!,
identifier: SimpleIdentifierImpl(secondToken!),
);
var expression = PropertyAccessImpl(
target: target,
operator: secondPeriod!,
propertyName: identifier,
);
return CommentReferenceImpl(
newKeyword: newKeyword,
expression: expression,
);
} else if (secondToken != null) {
var expression = PrefixedIdentifierImpl(
prefix: SimpleIdentifierImpl(secondToken),
period: secondPeriod!,
identifier: identifier,
);
return CommentReferenceImpl(
newKeyword: newKeyword,
expression: expression,
);
} else {
return CommentReferenceImpl(
newKeyword: newKeyword,
expression: identifier,
);
}
}
/// Parse the comment references in a multi-line comment token.
/// Return the number of comment references parsed.
int _parseReferencesInMultiLineComment(Token multiLineDoc) {
/// Parses the comment references in a multi-line comment token.
List<({String source, int offset})> _parseReferencesInMultiLineComment(
Token multiLineDoc) {
var comment = multiLineDoc.lexeme;
assert(comment.startsWith('/**'));
var count = 0;
var references = <({String source, int offset})>[];
var length = comment.length;
var start = 3;
var inCodeBlock = false;
@ -6080,18 +6093,20 @@ class AstBuilder extends StackListener {
}
}
if (!inCodeBlock && !comment.startsWith('* ', start)) {
count += _parseCommentReferencesInText(multiLineDoc, start, end);
references
.addAll(_parseCommentReferencesInText(multiLineDoc, start, end));
}
start = end + 1;
}
return count;
return references;
}
/// Parse the comment references in a sequence of single line comment tokens
/// where [token] is the first comment token in the sequence.
/// Return the number of comment references parsed.
int _parseReferencesInSingleLineComments(Token? token) {
var count = 0;
List<({String source, int offset})> _parseReferencesInSingleLineComments(
Token? token) {
var references = <({String source, int offset})>[];
var inCodeBlock = false;
while (token != null && !token.isEof) {
var comment = token.lexeme;
@ -6110,14 +6125,14 @@ class AstBuilder extends StackListener {
parseReferences = true;
}
if (parseReferences) {
count += _parseCommentReferencesInText(
token, /* start = */ 3, comment.length);
references.addAll(_parseCommentReferencesInText(
token, /* start = */ 3, comment.length));
}
}
}
token = token.next;
}
return count;
return references;
}
List<NamedTypeImpl> _popNamedTypeList({

24
pkg/front_end/lib/src/fasta/kernel/macro/annotation_parser.dart
View file

@ -443,9 +443,6 @@ class _MacroListener implements Listener {
unrecognized = true;
}
/// Called for listener events that are ignored.
void _ignored() {}
@override
void beginAsOperatorType(Token operator) {
_unsupported();
@ -1669,22 +1666,6 @@ class _MacroListener implements Listener {
_unexpected();
}
@override
void handleCommentReference(
Token? newKeyword,
Token? firstToken,
Token? firstPeriod,
Token? secondToken,
Token? secondPeriod,
Token thirdToken) {
_ignored();
}
@override
void handleCommentReferenceText(String referenceSource, int referenceOffset) {
_ignored();
}
@override
void handleConditionalExpressionColon() {
_unhandled();
@ -1977,11 +1958,6 @@ class _MacroListener implements Listener {
_unhandled();
}
@override
void handleNoCommentReference() {
_ignored();
}
@override
void handleNoConstructorReferenceContinuationAfterTypeArguments(Token token) {
_unknown();

86
pkg/front_end/lib/src/fasta/util/parser_ast_helper.dart
View file

@ -3066,41 +3066,6 @@ abstract class AbstractParserAstListener implements Listener {
seen(data);
}
@override
void handleCommentReferenceText(String referenceSource, int referenceOffset) {
CommentReferenceTextHandle data = new CommentReferenceTextHandle(
ParserAstType.HANDLE,
referenceSource: referenceSource,
referenceOffset: referenceOffset);
seen(data);
}
@override
void handleCommentReference(
Token? newKeyword,
Token? firstToken,
Token? firstPeriod,
Token? secondToken,
Token? secondPeriod,
Token thirdToken) {
CommentReferenceHandle data = new CommentReferenceHandle(
ParserAstType.HANDLE,
newKeyword: newKeyword,
firstToken: firstToken,
firstPeriod: firstPeriod,
secondToken: secondToken,
secondPeriod: secondPeriod,
thirdToken: thirdToken);
seen(data);
}
@override
void handleNoCommentReference() {
NoCommentReferenceHandle data =
new NoCommentReferenceHandle(ParserAstType.HANDLE);
seen(data);
}
@override
void handleTypeArgumentApplication(Token openAngleBracket) {
TypeArgumentApplicationHandle data = new TypeArgumentApplicationHandle(
@ -8550,57 +8515,6 @@ class ScriptHandle extends ParserAstNode {
};
}
class CommentReferenceTextHandle extends ParserAstNode {
final String referenceSource;
final int referenceOffset;
CommentReferenceTextHandle(ParserAstType type,
{required this.referenceSource, required this.referenceOffset})
: super("CommentReferenceText", type);
@override
Map<String, Object?> get deprecatedArguments => {
"referenceSource": referenceSource,
"referenceOffset": referenceOffset,
};
}
class CommentReferenceHandle extends ParserAstNode {
final Token? newKeyword;
final Token? firstToken;
final Token? firstPeriod;
final Token? secondToken;
final Token? secondPeriod;
final Token thirdToken;
CommentReferenceHandle(ParserAstType type,
{this.newKeyword,
this.firstToken,
this.firstPeriod,
this.secondToken,
this.secondPeriod,
required this.thirdToken})
: super("CommentReference", type);
@override
Map<String, Object?> get deprecatedArguments => {
"newKeyword": newKeyword,
"firstToken": firstToken,
"firstPeriod": firstPeriod,
"secondToken": secondToken,
"secondPeriod": secondPeriod,
"thirdToken": thirdToken,
};
}
class NoCommentReferenceHandle extends ParserAstNode {
NoCommentReferenceHandle(ParserAstType type)
: super("NoCommentReference", type);
@override
Map<String, Object?> get deprecatedArguments => {};
}
class TypeArgumentApplicationHandle extends ParserAstNode {
final Token openAngleBracket;

34
pkg/front_end/test/parser_test_listener.dart
View file

@ -3217,40 +3217,6 @@ class ParserTestListener implements Listener {
doPrint('handleScript(' '$token)');
}
@override
void handleCommentReferenceText(String referenceSource, int referenceOffset) {
doPrint(
'handleCommentReferenceText(' '$referenceSource, ' '$referenceOffset)');
}
@override
void handleCommentReference(
Token? newKeyword,
Token? firstToken,
Token? firstPeriod,
Token? secondToken,
Token? secondPeriod,
Token thirdToken) {
seen(newKeyword);
seen(firstToken);
seen(firstPeriod);
seen(secondToken);
seen(secondPeriod);
seen(thirdToken);
doPrint('handleCommentReference('
'$newKeyword, '
'$firstToken, '
'$firstPeriod, '
'$secondToken, '
'$secondPeriod, '
'$thirdToken)');
}
@override
void handleNoCommentReference() {
doPrint('handleNoCommentReference()');
}
@override
void handleTypeArgumentApplication(Token openAngleBracket) {
seen(openAngleBracket);