mirror of
https://github.com/Microsoft/vscode
synced 2024-08-27 13:00:58 +00:00
Simplify tokenIterator
This commit is contained in:
parent
3ad82b90a3
commit
fa2fdbb037
|
@ -132,16 +132,25 @@ export class LineTokens {
|
|||
}
|
||||
|
||||
public findTokenAtOffset(offset:number): LineToken {
|
||||
if (this._textLength === 0) {
|
||||
return null;
|
||||
}
|
||||
let tokenIndex = this.findTokenIndexAtOffset(offset);
|
||||
let modeIndex = ModeTransition.findIndexInSegmentsArray(this.modeTransitions, offset);
|
||||
return new LineToken(this, tokenIndex, modeIndex);
|
||||
}
|
||||
|
||||
public first(): LineToken {
|
||||
public firstToken(): LineToken {
|
||||
if (this._textLength === 0) {
|
||||
return null;
|
||||
}
|
||||
return new LineToken(this, 0, 0);
|
||||
}
|
||||
|
||||
public last(): LineToken {
|
||||
public lastToken(): LineToken {
|
||||
if (this._textLength === 0) {
|
||||
return null;
|
||||
}
|
||||
return new LineToken(this, this._tokens.length - 1, this.modeTransitions.length - 1);
|
||||
}
|
||||
|
||||
|
|
|
@ -18,7 +18,6 @@ import {Position} from 'vs/editor/common/core/position';
|
|||
import {Range} from 'vs/editor/common/core/range';
|
||||
import {Selection} from 'vs/editor/common/core/selection';
|
||||
import {ModeTransition} from 'vs/editor/common/core/modeTransition';
|
||||
import {Token} from 'vs/editor/common/core/token';
|
||||
import {IndentRange} from 'vs/editor/common/model/indentRanges';
|
||||
import {ICommandHandlerDescription} from 'vs/platform/commands/common/commands';
|
||||
import {ContextKeyExpr, RawContextKey} from 'vs/platform/contextkey/common/contextkey';
|
||||
|
@ -1246,7 +1245,7 @@ export interface IWordRange {
|
|||
* @internal
|
||||
*/
|
||||
export interface ITokenInfo {
|
||||
token: Token;
|
||||
type: string;
|
||||
lineNumber: number;
|
||||
startColumn: number;
|
||||
endColumn: number;
|
||||
|
|
|
@ -527,6 +527,9 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
|
|||
const lineText = this._lines[lineNumber - 1].text;
|
||||
|
||||
const currentToken = lineTokens.findTokenAtOffset(position.column - 1);
|
||||
if (!currentToken) {
|
||||
return null;
|
||||
}
|
||||
const currentModeBrackets = LanguageConfigurationRegistry.getBracketsSupport(currentToken.modeId);
|
||||
|
||||
// If position is in between two tokens, try first looking in the previous token
|
||||
|
@ -624,11 +627,13 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
|
|||
currentToken = lineTokens.findTokenAtOffset(position.column - 1);
|
||||
searchStopOffset = position.column - 1;
|
||||
} else {
|
||||
currentToken = lineTokens.last();
|
||||
searchStopOffset = currentToken.endOffset;
|
||||
currentToken = lineTokens.lastToken();
|
||||
if (currentToken) {
|
||||
searchStopOffset = currentToken.endOffset;
|
||||
}
|
||||
}
|
||||
|
||||
do {
|
||||
while(currentToken) {
|
||||
if (currentToken.modeId === modeId && !ignoreBracketsInToken(currentToken.type)) {
|
||||
|
||||
while (true) {
|
||||
|
@ -658,8 +663,7 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
|
|||
if (currentToken) {
|
||||
searchStopOffset = currentToken.endOffset;
|
||||
}
|
||||
|
||||
} while(currentToken);
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
|
@ -682,11 +686,13 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
|
|||
currentToken = lineTokens.findTokenAtOffset(position.column - 1);
|
||||
searchStartOffset = position.column - 1;
|
||||
} else {
|
||||
currentToken = lineTokens.first();
|
||||
searchStartOffset = currentToken.startOffset;
|
||||
currentToken = lineTokens.firstToken();
|
||||
if (currentToken) {
|
||||
searchStartOffset = currentToken.startOffset;
|
||||
}
|
||||
}
|
||||
|
||||
do {
|
||||
while (currentToken) {
|
||||
if (currentToken.modeId === modeId && !ignoreBracketsInToken(currentToken.type)) {
|
||||
while (true) {
|
||||
let r = BracketsUtils.findNextBracketInToken(bracketRegex, lineNumber, lineText, searchStartOffset, currentToken.endOffset);
|
||||
|
@ -715,7 +721,7 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
|
|||
if (currentToken) {
|
||||
searchStartOffset = currentToken.startOffset;
|
||||
}
|
||||
} while (currentToken);
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
|
|
|
@ -5,117 +5,138 @@
|
|||
'use strict';
|
||||
|
||||
import * as editorCommon from 'vs/editor/common/editorCommon';
|
||||
import {Token} from 'vs/editor/common/core/token';
|
||||
import {LineTokens} from 'vs/editor/common/core/lineTokens';
|
||||
import {LineToken} from 'vs/editor/common/core/lineTokens';
|
||||
import {Position} from 'vs/editor/common/core/position';
|
||||
|
||||
class TokenInfo implements editorCommon.ITokenInfo {
|
||||
_tokenInfoBrand: void;
|
||||
|
||||
_actual: LineToken;
|
||||
public lineNumber: number;
|
||||
public startColumn: number;
|
||||
public endColumn: number;
|
||||
public type: string;
|
||||
|
||||
constructor(actual:LineToken, lineNumber:number) {
|
||||
this._actual = actual;
|
||||
this.lineNumber = lineNumber;
|
||||
this.startColumn = this._actual.startOffset + 1;
|
||||
this.endColumn = this._actual.endOffset + 1;
|
||||
this.type = this._actual.type;
|
||||
}
|
||||
}
|
||||
|
||||
function findClosestNonEmptyLine(model:editorCommon.ITokenizedModel, position:Position): Position {
|
||||
const lineNumber = position.lineNumber;
|
||||
if (model.getLineMaxColumn(lineNumber) !== 1) {
|
||||
return position;
|
||||
}
|
||||
|
||||
const lineCount = model.getLineCount();
|
||||
|
||||
// we need to go up or down
|
||||
let distance = 1;
|
||||
while (true) {
|
||||
let aboveLineNumber = lineNumber - distance;
|
||||
let belowLineNumber = lineNumber + distance;
|
||||
|
||||
if (aboveLineNumber < 1 && belowLineNumber > lineCount) {
|
||||
// No more lines above or below
|
||||
break;
|
||||
}
|
||||
|
||||
if (aboveLineNumber >= 1) {
|
||||
let aboveMaxColumn = model.getLineMaxColumn(aboveLineNumber);
|
||||
if (aboveMaxColumn !== 1) {
|
||||
// bingo!
|
||||
return new Position(aboveLineNumber, aboveMaxColumn);
|
||||
}
|
||||
}
|
||||
|
||||
if (belowLineNumber <= lineCount) {
|
||||
let belowMaxColumn = model.getLineMaxColumn(belowLineNumber);
|
||||
if (belowMaxColumn !== 1) {
|
||||
// bingo!
|
||||
return new Position(belowLineNumber, 1);
|
||||
}
|
||||
}
|
||||
|
||||
distance++;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export class TokenIterator implements editorCommon.ITokenIterator {
|
||||
|
||||
private _model:editorCommon.ITokenizedModel;
|
||||
private _currentLineNumber:number;
|
||||
private _currentTokenIndex:number;
|
||||
private _currentLineTokens:LineTokens;
|
||||
private _next:editorCommon.ITokenInfo;
|
||||
private _prev:editorCommon.ITokenInfo;
|
||||
private _lineCount:number;
|
||||
private _prev:TokenInfo;
|
||||
private _next:TokenInfo;
|
||||
|
||||
constructor(model:editorCommon.ITokenizedModel, position:editorCommon.IPosition) {
|
||||
constructor(model:editorCommon.ITokenizedModel, position:Position) {
|
||||
this._model = model;
|
||||
this._currentLineNumber = position.lineNumber;
|
||||
this._currentTokenIndex = 0;
|
||||
this._readLineTokens(this._currentLineNumber);
|
||||
this._next = null;
|
||||
this._lineCount = this._model.getLineCount();
|
||||
this._prev = null;
|
||||
this._next = null;
|
||||
|
||||
// start with a position to next/prev run
|
||||
var columnIndex = position.column - 1, tokenEndIndex = Number.MAX_VALUE;
|
||||
|
||||
for (var i = this._currentLineTokens.getTokenCount() - 1; i >= 0; i--) {
|
||||
let tokenStartIndex = this._currentLineTokens.getTokenStartOffset(i);
|
||||
|
||||
if (tokenStartIndex <= columnIndex && columnIndex <= tokenEndIndex) {
|
||||
|
||||
this._currentTokenIndex = i;
|
||||
this._next = this._current();
|
||||
this._prev = this._current();
|
||||
break;
|
||||
position = findClosestNonEmptyLine(model, position);
|
||||
if (position) {
|
||||
let lineTokens = this._model.getLineTokens(position.lineNumber);
|
||||
let currentToken = lineTokens.findTokenAtOffset(position.column - 1);
|
||||
if (currentToken) {
|
||||
this._prev = this._next = new TokenInfo(currentToken, position.lineNumber);
|
||||
}
|
||||
tokenEndIndex = tokenStartIndex;
|
||||
}
|
||||
}
|
||||
|
||||
private _readLineTokens(lineNumber:number): void {
|
||||
this._currentLineTokens = this._model.getLineTokens(lineNumber, false);
|
||||
}
|
||||
private _advanceNext(): void {
|
||||
if (!this._next) {
|
||||
return;
|
||||
}
|
||||
|
||||
let lineNumber = this._next.lineNumber;
|
||||
let next = this._next._actual.next();
|
||||
while (!next && lineNumber < this._lineCount) {
|
||||
lineNumber++;
|
||||
let currentLineTokens = this._model.getLineTokens(lineNumber);
|
||||
next = currentLineTokens.firstToken();
|
||||
}
|
||||
|
||||
private _advanceNext() {
|
||||
this._prev = this._next;
|
||||
this._next = null;
|
||||
if (this._currentTokenIndex + 1 < this._currentLineTokens.getTokenCount()) {
|
||||
// There are still tokens on current line
|
||||
this._currentTokenIndex++;
|
||||
this._next = this._current();
|
||||
|
||||
if (next) {
|
||||
this._next = new TokenInfo(next, lineNumber);
|
||||
} else {
|
||||
// find the next line with tokens
|
||||
while (this._currentLineNumber + 1 <= this._model.getLineCount()) {
|
||||
this._currentLineNumber++;
|
||||
this._readLineTokens(this._currentLineNumber);
|
||||
if (this._currentLineTokens.getTokenCount() > 0) {
|
||||
this._currentTokenIndex = 0;
|
||||
this._next = this._current();
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (this._next === null) {
|
||||
// prepare of a previous run
|
||||
this._readLineTokens(this._currentLineNumber);
|
||||
this._currentTokenIndex = this._currentLineTokens.getTokenCount();
|
||||
this._advancePrev();
|
||||
this._next = null;
|
||||
}
|
||||
this._next = null;
|
||||
}
|
||||
}
|
||||
|
||||
private _advancePrev() {
|
||||
private _advancePrev(): void {
|
||||
if (!this._prev) {
|
||||
return;
|
||||
}
|
||||
|
||||
let lineNumber = this._prev.lineNumber;
|
||||
let prev = this._prev._actual.prev();
|
||||
while (!prev && lineNumber > 1) {
|
||||
lineNumber--;
|
||||
let currentLineTokens = this._model.getLineTokens(lineNumber);
|
||||
prev = currentLineTokens.lastToken();
|
||||
}
|
||||
|
||||
this._next = this._prev;
|
||||
this._prev = null;
|
||||
if (this._currentTokenIndex > 0) {
|
||||
// There are still tokens on current line
|
||||
this._currentTokenIndex--;
|
||||
this._prev = this._current();
|
||||
|
||||
if (prev) {
|
||||
this._prev = new TokenInfo(prev, lineNumber);
|
||||
} else {
|
||||
// find previous line with tokens
|
||||
while (this._currentLineNumber > 1) {
|
||||
this._currentLineNumber--;
|
||||
this._readLineTokens(this._currentLineNumber);
|
||||
if (this._currentLineTokens.getTokenCount() > 0) {
|
||||
this._currentTokenIndex = this._currentLineTokens.getTokenCount() - 1;
|
||||
this._prev = this._current();
|
||||
break;
|
||||
}
|
||||
}
|
||||
this._prev = null;
|
||||
}
|
||||
}
|
||||
|
||||
private _current(): editorCommon.ITokenInfo {
|
||||
let startIndex = this._currentLineTokens.getTokenStartOffset(this._currentTokenIndex);
|
||||
let type = this._currentLineTokens.getTokenType(this._currentTokenIndex);
|
||||
let endIndex = this._currentLineTokens.getTokenEndOffset(this._currentTokenIndex);
|
||||
|
||||
return {
|
||||
token: new Token(startIndex, type),
|
||||
lineNumber: this._currentLineNumber,
|
||||
startColumn: startIndex + 1,
|
||||
endColumn: endIndex + 1
|
||||
};
|
||||
}
|
||||
|
||||
public hasNext(): boolean {
|
||||
return this._next !== null;
|
||||
}
|
||||
|
||||
public next(): editorCommon.ITokenInfo {
|
||||
var result = this._next;
|
||||
const result = this._next;
|
||||
this._advanceNext();
|
||||
return result;
|
||||
}
|
||||
|
@ -125,7 +146,7 @@ export class TokenIterator implements editorCommon.ITokenIterator {
|
|||
}
|
||||
|
||||
public prev(): editorCommon.ITokenInfo {
|
||||
var result = this._prev;
|
||||
const result = this._prev;
|
||||
this._advancePrev();
|
||||
return result;
|
||||
}
|
||||
|
|
|
@ -125,7 +125,7 @@ class Snapper {
|
|||
let content = model.getValueInRange({ startLineNumber: lineNumber, endLineNumber: lineNumber, startColumn: tokenInfo.startColumn, endColumn: tokenInfo.endColumn});
|
||||
result.push({
|
||||
c: content,
|
||||
t: this.normalizeType(tokenInfo.token.type),
|
||||
t: this.normalizeType(tokenInfo.type),
|
||||
r: {}
|
||||
});
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue