mirror of
https://github.com/dart-lang/sdk
synced 2024-09-19 14:32:49 +00:00
8e6f238f86
Replace === null with == null. BUG=http://dartbug.com/6380 Review URL: https://codereview.chromium.org//11361190 git-svn-id: https://dart.googlecode.com/svn/branches/bleeding_edge/dart@14794 260f80e4-7a28-3924-810f-c04153c831b5
455 lines
11 KiB
Dart
455 lines
11 KiB
Dart
// Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file
|
|
// for details. All rights reserved. Use of this source code is governed by a
|
|
// BSD-style license that can be found in the LICENSE file.
|
|
// Generated by scripts/tokenizer_gen.py.
|
|
|
|
|
|
abstract class TokenSource {
|
|
Token next();
|
|
}
|
|
|
|
class InterpStack {
|
|
InterpStack next, previous;
|
|
final int quote;
|
|
final bool isMultiline;
|
|
int depth;
|
|
|
|
InterpStack(this.previous, this.quote, this.isMultiline): depth = -1;
|
|
|
|
InterpStack pop() {
|
|
return this.previous;
|
|
}
|
|
|
|
static InterpStack push(InterpStack stack, int quote, bool isMultiline) {
|
|
var newStack = new InterpStack(stack, quote, isMultiline);
|
|
if (stack != null) newStack.previous = stack;
|
|
return newStack;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* The base class for our tokenizer. The hand coded parts are in this file, with
|
|
* the generated parts in the subclass Tokenizer.
|
|
*/
|
|
class CSSTokenizerBase implements TokenSource {
|
|
final SourceFile _source;
|
|
final bool _skipWhitespace;
|
|
String _text;
|
|
|
|
int _index;
|
|
int _startIndex;
|
|
|
|
/** Keeps track of string interpolation state. */
|
|
InterpStack _interpStack;
|
|
|
|
CSSTokenizerBase(this._source, this._skipWhitespace, [index = 0])
|
|
: this._index = index {
|
|
_text = _source.text;
|
|
}
|
|
|
|
abstract Token next();
|
|
abstract int getIdentifierKind();
|
|
|
|
int _nextChar() {
|
|
if (_index < _text.length) {
|
|
return _text.charCodeAt(_index++);
|
|
} else {
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
int _peekChar() {
|
|
if (_index < _text.length) {
|
|
return _text.charCodeAt(_index);
|
|
} else {
|
|
return 0;
|
|
}
|
|
}
|
|
|
|
bool _maybeEatChar(int ch) {
|
|
if (_index < _text.length) {
|
|
if (_text.charCodeAt(_index) == ch) {
|
|
_index++;
|
|
return true;
|
|
} else {
|
|
return false;
|
|
}
|
|
} else {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
String _tokenText() {
|
|
if (_index < _text.length) {
|
|
return _text.substring(_startIndex, _index);
|
|
} else {
|
|
return _text.substring(_startIndex, _text.length);
|
|
}
|
|
}
|
|
|
|
Token _finishToken(int kind) {
|
|
return new Token(kind, _source, _startIndex, _index);
|
|
}
|
|
|
|
Token _errorToken([String message = null]) {
|
|
return new ErrorToken(
|
|
TokenKind.ERROR, _source, _startIndex, _index, message);
|
|
}
|
|
|
|
Token finishWhitespace() {
|
|
_index--;
|
|
while (_index < _text.length) {
|
|
final ch = _text.charCodeAt(_index++);
|
|
if (ch == 32/*' '*/ || ch == 9/*'\t'*/ || ch == 13/*'\r'*/) {
|
|
// do nothing
|
|
} else if (ch == 10/*'\n'*/) {
|
|
if (!_skipWhitespace) {
|
|
return _finishToken(TokenKind.WHITESPACE); // note the newline?
|
|
}
|
|
} else {
|
|
_index--;
|
|
if (_skipWhitespace) {
|
|
return next();
|
|
} else {
|
|
return _finishToken(TokenKind.WHITESPACE);
|
|
}
|
|
}
|
|
|
|
}
|
|
return _finishToken(TokenKind.END_OF_FILE);
|
|
}
|
|
|
|
Token finishSingleLineComment() {
|
|
while (true) {
|
|
int ch = _nextChar();
|
|
if (ch == 0 || ch == 10/*'\n'*/ || ch == 13/*'\r'*/) {
|
|
if (_skipWhitespace) {
|
|
return next();
|
|
} else {
|
|
return _finishToken(TokenKind.COMMENT);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
Token finishMultiLineComment() {
|
|
int nesting = 1;
|
|
do {
|
|
int ch = _nextChar();
|
|
if (ch == 0) {
|
|
return _errorToken();
|
|
} else if (ch == 42/*'*'*/) {
|
|
if (_maybeEatChar(47/*'/'*/)) {
|
|
nesting--;
|
|
}
|
|
} else if (ch == 47/*'/'*/) {
|
|
if (_maybeEatChar(42/*'*'*/)) {
|
|
nesting++;
|
|
}
|
|
}
|
|
} while (nesting > 0);
|
|
|
|
if (_skipWhitespace) {
|
|
return next();
|
|
} else {
|
|
return _finishToken(TokenKind.COMMENT);
|
|
}
|
|
}
|
|
|
|
void eatDigits() {
|
|
while (_index < _text.length) {
|
|
if (TokenizerHelpers.isDigit(_text.charCodeAt(_index))) {
|
|
_index++;
|
|
} else {
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
|
|
static int _hexDigit(int c) {
|
|
if(c >= 48/*0*/ && c <= 57/*9*/) {
|
|
return c - 48;
|
|
} else if (c >= 97/*a*/ && c <= 102/*f*/) {
|
|
return c - 87;
|
|
} else if (c >= 65/*A*/ && c <= 70/*F*/) {
|
|
return c - 55;
|
|
} else {
|
|
return -1;
|
|
}
|
|
}
|
|
|
|
int readHex([int hexLength]) {
|
|
int maxIndex;
|
|
if (hexLength == null) {
|
|
maxIndex = _text.length - 1;
|
|
} else {
|
|
// TODO(jimhug): What if this is too long?
|
|
maxIndex = _index + hexLength;
|
|
if (maxIndex >= _text.length) return -1;
|
|
}
|
|
var result = 0;
|
|
while (_index < maxIndex) {
|
|
final digit = _hexDigit(_text.charCodeAt(_index));
|
|
if (digit == -1) {
|
|
if (hexLength == null) {
|
|
return result;
|
|
} else {
|
|
return -1;
|
|
}
|
|
}
|
|
_hexDigit(_text.charCodeAt(_index));
|
|
// Multiply by 16 rather than shift by 4 since that will result in a
|
|
// correct value for numbers that exceed the 32 bit precision of JS
|
|
// 'integers'.
|
|
// TODO: Figure out a better solution to integer truncation. Issue 638.
|
|
result = (result * 16) + digit;
|
|
_index++;
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
Token finishNumber() {
|
|
eatDigits();
|
|
|
|
if (_peekChar() == 46/*.*/) {
|
|
// Handle the case of 1.toString().
|
|
_nextChar();
|
|
if (TokenizerHelpers.isDigit(_peekChar())) {
|
|
eatDigits();
|
|
return finishNumberExtra(TokenKind.DOUBLE);
|
|
} else {
|
|
_index--;
|
|
}
|
|
}
|
|
|
|
return finishNumberExtra(TokenKind.INTEGER);
|
|
}
|
|
|
|
Token finishNumberExtra(int kind) {
|
|
if (_maybeEatChar(101/*e*/) || _maybeEatChar(69/*E*/)) {
|
|
kind = TokenKind.DOUBLE;
|
|
_maybeEatChar(45/*-*/);
|
|
_maybeEatChar(43/*+*/);
|
|
eatDigits();
|
|
}
|
|
if (_peekChar() != 0 && TokenizerHelpers.isIdentifierStart(_peekChar())) {
|
|
_nextChar();
|
|
return _errorToken("illegal character in number");
|
|
}
|
|
|
|
return _finishToken(kind);
|
|
}
|
|
|
|
Token _makeStringToken(List<int> buf, bool isPart) {
|
|
final s = new String.fromCharCodes(buf);
|
|
final kind = isPart ? TokenKind.STRING_PART : TokenKind.STRING;
|
|
return new LiteralToken(kind, _source, _startIndex, _index, s);
|
|
}
|
|
|
|
Token _makeRawStringToken(bool isMultiline) {
|
|
String s;
|
|
if (isMultiline) {
|
|
// Skip initial newline in multiline strings
|
|
int start = _startIndex + 4;
|
|
if (_source.text[start] == '\n') start++;
|
|
s = _source.text.substring(start, _index - 3);
|
|
} else {
|
|
s = _source.text.substring(_startIndex + 2, _index - 1);
|
|
}
|
|
return new LiteralToken(TokenKind.STRING, _source, _startIndex, _index, s);
|
|
}
|
|
|
|
Token finishMultilineString(int quote) {
|
|
var buf = <int>[];
|
|
while (true) {
|
|
int ch = _nextChar();
|
|
if (ch == 0) {
|
|
return _errorToken();
|
|
} else if (ch == quote) {
|
|
if (_maybeEatChar(quote)) {
|
|
if (_maybeEatChar(quote)) {
|
|
return _makeStringToken(buf, false);
|
|
}
|
|
buf.add(quote);
|
|
}
|
|
buf.add(quote);
|
|
} else if (ch == 36/*$*/) {
|
|
// start of string interp
|
|
_interpStack = InterpStack.push(_interpStack, quote, true);
|
|
return _makeStringToken(buf, true);
|
|
} else if (ch == 92/*\*/) {
|
|
var escapeVal = readEscapeSequence();
|
|
if (escapeVal == -1) {
|
|
return _errorToken("invalid hex escape sequence");
|
|
} else {
|
|
buf.add(escapeVal);
|
|
}
|
|
} else {
|
|
buf.add(ch);
|
|
}
|
|
}
|
|
}
|
|
|
|
Token _finishOpenBrace() {
|
|
if (_interpStack != null) {
|
|
if (_interpStack.depth == -1) {
|
|
_interpStack.depth = 1;
|
|
} else {
|
|
assert(_interpStack.depth >= 0);
|
|
_interpStack.depth += 1;
|
|
}
|
|
}
|
|
return _finishToken(TokenKind.LBRACE);
|
|
}
|
|
|
|
Token _finishCloseBrace() {
|
|
if (_interpStack != null) {
|
|
_interpStack.depth -= 1;
|
|
assert(_interpStack.depth >= 0);
|
|
}
|
|
return _finishToken(TokenKind.RBRACE);
|
|
}
|
|
|
|
Token finishString(int quote) {
|
|
if (_maybeEatChar(quote)) {
|
|
if (_maybeEatChar(quote)) {
|
|
// skip an initial newline
|
|
_maybeEatChar(10/*'\n'*/);
|
|
return finishMultilineString(quote);
|
|
} else {
|
|
return _makeStringToken(new List<int>(), false);
|
|
}
|
|
}
|
|
return finishStringBody(quote);
|
|
}
|
|
|
|
Token finishRawString(int quote) {
|
|
if (_maybeEatChar(quote)) {
|
|
if (_maybeEatChar(quote)) {
|
|
return finishMultilineRawString(quote);
|
|
} else {
|
|
return _makeStringToken(<int>[], false);
|
|
}
|
|
}
|
|
while (true) {
|
|
int ch = _nextChar();
|
|
if (ch == quote) {
|
|
return _makeRawStringToken(false);
|
|
} else if (ch == 0) {
|
|
return _errorToken();
|
|
}
|
|
}
|
|
}
|
|
|
|
Token finishMultilineRawString(int quote) {
|
|
while (true) {
|
|
int ch = _nextChar();
|
|
if (ch == 0) {
|
|
return _errorToken();
|
|
} else if (ch == quote && _maybeEatChar(quote) && _maybeEatChar(quote)) {
|
|
return _makeRawStringToken(true);
|
|
}
|
|
}
|
|
}
|
|
|
|
Token finishStringBody(int quote) {
|
|
var buf = new List<int>();
|
|
while (true) {
|
|
int ch = _nextChar();
|
|
if (ch == quote) {
|
|
return _makeStringToken(buf, false);
|
|
} else if (ch == 36/*$*/) {
|
|
// start of string interp
|
|
_interpStack = InterpStack.push(_interpStack, quote, false);
|
|
return _makeStringToken(buf, true);
|
|
} else if (ch == 0) {
|
|
return _errorToken();
|
|
} else if (ch == 92/*\*/) {
|
|
var escapeVal = readEscapeSequence();
|
|
if (escapeVal == -1) {
|
|
return _errorToken("invalid hex escape sequence");
|
|
} else {
|
|
buf.add(escapeVal);
|
|
}
|
|
} else {
|
|
buf.add(ch);
|
|
}
|
|
}
|
|
}
|
|
|
|
int readEscapeSequence() {
|
|
final ch = _nextChar();
|
|
int hexValue;
|
|
switch (ch) {
|
|
case 110/*n*/:
|
|
return 0x0a/*'\n'*/;
|
|
case 114/*r*/:
|
|
return 0x0d/*'\r'*/;
|
|
case 102/*f*/:
|
|
return 0x0c/*'\f'*/;
|
|
case 98/*b*/:
|
|
return 0x08/*'\b'*/;
|
|
case 116/*t*/:
|
|
return 0x09/*'\t'*/;
|
|
case 118/*v*/:
|
|
return 0x0b/*'\v'*/;
|
|
case 120/*x*/:
|
|
hexValue = readHex(2);
|
|
break;
|
|
case 117/*u*/:
|
|
if (_maybeEatChar(123/*{*/)) {
|
|
hexValue = readHex();
|
|
if (!_maybeEatChar(125/*}*/)) {
|
|
return -1;
|
|
} else {
|
|
break;
|
|
}
|
|
} else {
|
|
hexValue = readHex(4);
|
|
break;
|
|
}
|
|
default: return ch;
|
|
}
|
|
|
|
if (hexValue == -1) return -1;
|
|
|
|
// According to the Unicode standard the high and low surrogate halves
|
|
// used by UTF-16 (U+D800 through U+DFFF) and values above U+10FFFF
|
|
// are not legal Unicode values.
|
|
if (hexValue < 0xD800 || hexValue > 0xDFFF && hexValue <= 0xFFFF) {
|
|
return hexValue;
|
|
} else if (hexValue <= 0x10FFFF){
|
|
world.fatal('unicode values greater than 2 bytes not implemented yet');
|
|
return -1;
|
|
} else {
|
|
return -1;
|
|
}
|
|
}
|
|
|
|
Token finishDot() {
|
|
if (TokenizerHelpers.isDigit(_peekChar())) {
|
|
eatDigits();
|
|
return finishNumberExtra(TokenKind.DOUBLE);
|
|
} else {
|
|
return _finishToken(TokenKind.DOT);
|
|
}
|
|
}
|
|
|
|
Token finishIdentifier(int ch) {
|
|
while (_index < _text.length) {
|
|
if (!TokenizerHelpers.isIdentifierPart(_text.charCodeAt(_index++))) {
|
|
_index--;
|
|
break;
|
|
}
|
|
}
|
|
int kind = getIdentifierKind();
|
|
if (kind == TokenKind.IDENTIFIER) {
|
|
return _finishToken(TokenKind.IDENTIFIER);
|
|
} else {
|
|
return _finishToken(kind);
|
|
}
|
|
}
|
|
}
|
|
|