Simplify tokenIterator

src/vs/editor/common/core/lineTokens.ts

@@ -132,16 +132,25 @@ export class LineTokens {
 	}
 
 	public findTokenAtOffset(offset:number): LineToken {
+		if (this._textLength === 0) {
+			return null;
+		}
 		let tokenIndex = this.findTokenIndexAtOffset(offset);
 		let modeIndex = ModeTransition.findIndexInSegmentsArray(this.modeTransitions, offset);
 		return new LineToken(this, tokenIndex, modeIndex);
 	}
 
-	public first(): LineToken {
+	public firstToken(): LineToken {
+		if (this._textLength === 0) {
+			return null;
+		}
 		return new LineToken(this, 0, 0);
 	}
 
-	public last(): LineToken {
+	public lastToken(): LineToken {
+		if (this._textLength === 0) {
+			return null;
+		}
 		return new LineToken(this, this._tokens.length - 1, this.modeTransitions.length - 1);
 	}
 

src/vs/editor/common/editorCommon.ts

@@ -18,7 +18,6 @@ import {Position} from 'vs/editor/common/core/position';
 import {Range} from 'vs/editor/common/core/range';
 import {Selection} from 'vs/editor/common/core/selection';
 import {ModeTransition} from 'vs/editor/common/core/modeTransition';
-import {Token} from 'vs/editor/common/core/token';
 import {IndentRange} from 'vs/editor/common/model/indentRanges';
 import {ICommandHandlerDescription} from 'vs/platform/commands/common/commands';
 import {ContextKeyExpr, RawContextKey} from 'vs/platform/contextkey/common/contextkey';
@@ -1246,7 +1245,7 @@ export interface IWordRange {
  * @internal
  */
 export interface ITokenInfo {
-	token: Token;
+	type: string;
 	lineNumber: number;
 	startColumn: number;
 	endColumn: number;

src/vs/editor/common/model/textModelWithTokens.ts

@@ -527,6 +527,9 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
 		const lineText = this._lines[lineNumber - 1].text;
 
 		const currentToken = lineTokens.findTokenAtOffset(position.column - 1);
+		if (!currentToken) {
+			return null;
+		}
 		const currentModeBrackets = LanguageConfigurationRegistry.getBracketsSupport(currentToken.modeId);
 
 		// If position is in between two tokens, try first looking in the previous token
@@ -624,11 +627,13 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
 				currentToken = lineTokens.findTokenAtOffset(position.column - 1);
 				searchStopOffset = position.column - 1;
 			} else {
-				currentToken = lineTokens.last();
-				searchStopOffset = currentToken.endOffset;
+				currentToken = lineTokens.lastToken();
+				if (currentToken) {
+					searchStopOffset = currentToken.endOffset;
+				}
 			}
 
-			do {
+			while(currentToken) {
 				if (currentToken.modeId === modeId && !ignoreBracketsInToken(currentToken.type)) {
 
 					while (true) {
@@ -658,8 +663,7 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
 				if (currentToken) {
 					searchStopOffset = currentToken.endOffset;
 				}
-
-			} while(currentToken);
+			}
 		}
 
 		return null;
@@ -682,11 +686,13 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
 				currentToken = lineTokens.findTokenAtOffset(position.column - 1);
 				searchStartOffset = position.column - 1;
 			} else {
-				currentToken = lineTokens.first();
-				searchStartOffset = currentToken.startOffset;
+				currentToken = lineTokens.firstToken();
+				if (currentToken) {
+					searchStartOffset = currentToken.startOffset;
+				}
 			}
 
-			do {
+			while (currentToken) {
 				if (currentToken.modeId === modeId && !ignoreBracketsInToken(currentToken.type)) {
 					while (true) {
 						let r = BracketsUtils.findNextBracketInToken(bracketRegex, lineNumber, lineText, searchStartOffset, currentToken.endOffset);
@@ -715,7 +721,7 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke
 				if (currentToken) {
 					searchStartOffset = currentToken.startOffset;
 				}
-			} while (currentToken);
+			}
 		}
 
 		return null;

src/vs/editor/common/model/tokenIterator.ts

@@ -5,117 +5,138 @@
 'use strict';
 
 import * as editorCommon from 'vs/editor/common/editorCommon';
-import {Token} from 'vs/editor/common/core/token';
-import {LineTokens} from 'vs/editor/common/core/lineTokens';
+import {LineToken} from 'vs/editor/common/core/lineTokens';
+import {Position} from 'vs/editor/common/core/position';
+
+class TokenInfo implements editorCommon.ITokenInfo {
+	_tokenInfoBrand: void;
+
+	_actual: LineToken;
+	public lineNumber: number;
+	public startColumn: number;
+	public endColumn: number;
+	public type: string;
+
+	constructor(actual:LineToken, lineNumber:number) {
+		this._actual = actual;
+		this.lineNumber = lineNumber;
+		this.startColumn = this._actual.startOffset + 1;
+		this.endColumn = this._actual.endOffset + 1;
+		this.type = this._actual.type;
+	}
+}
+
+function findClosestNonEmptyLine(model:editorCommon.ITokenizedModel, position:Position): Position {
+	const lineNumber = position.lineNumber;
+	if (model.getLineMaxColumn(lineNumber) !== 1) {
+		return position;
+	}
+
+	const lineCount = model.getLineCount();
+
+	// we need to go up or down
+	let distance = 1;
+	while (true) {
+		let aboveLineNumber = lineNumber - distance;
+		let belowLineNumber = lineNumber + distance;
+
+		if (aboveLineNumber < 1 && belowLineNumber > lineCount) {
+			// No more lines above or below
+			break;
+		}
+
+		if (aboveLineNumber >= 1) {
+			let aboveMaxColumn = model.getLineMaxColumn(aboveLineNumber);
+			if (aboveMaxColumn !== 1) {
+				// bingo!
+				return new Position(aboveLineNumber, aboveMaxColumn);
+			}
+		}
+
+		if (belowLineNumber <= lineCount) {
+			let belowMaxColumn = model.getLineMaxColumn(belowLineNumber);
+			if (belowMaxColumn !== 1) {
+				// bingo!
+				return new Position(belowLineNumber, 1);
+			}
+		}
+
+		distance++;
+	}
+	return null;
+}
 
 export class TokenIterator implements editorCommon.ITokenIterator {
 
 	private _model:editorCommon.ITokenizedModel;
-	private _currentLineNumber:number;
-	private _currentTokenIndex:number;
-	private _currentLineTokens:LineTokens;
-	private _next:editorCommon.ITokenInfo;
-	private _prev:editorCommon.ITokenInfo;
+	private _lineCount:number;
+	private _prev:TokenInfo;
+	private _next:TokenInfo;
 
-	constructor(model:editorCommon.ITokenizedModel, position:editorCommon.IPosition) {
+	constructor(model:editorCommon.ITokenizedModel, position:Position) {
 		this._model = model;
-		this._currentLineNumber = position.lineNumber;
-		this._currentTokenIndex = 0;
-		this._readLineTokens(this._currentLineNumber);
-		this._next = null;
+		this._lineCount = this._model.getLineCount();
 		this._prev = null;
+		this._next = null;
 
-		// start with a position to next/prev run
-		var columnIndex = position.column - 1, tokenEndIndex = Number.MAX_VALUE;
-
-		for (var i = this._currentLineTokens.getTokenCount() - 1; i >= 0; i--) {
-			let tokenStartIndex = this._currentLineTokens.getTokenStartOffset(i);
-
-			if (tokenStartIndex <= columnIndex && columnIndex <= tokenEndIndex) {
-
-				this._currentTokenIndex = i;
-				this._next = this._current();
-				this._prev = this._current();
-				break;
+		position = findClosestNonEmptyLine(model, position);
+		if (position) {
+			let lineTokens = this._model.getLineTokens(position.lineNumber);
+			let currentToken = lineTokens.findTokenAtOffset(position.column - 1);
+			if (currentToken) {
+				this._prev = this._next = new TokenInfo(currentToken, position.lineNumber);
 			}
-			tokenEndIndex = tokenStartIndex;
 		}
 	}
 
-	private _readLineTokens(lineNumber:number): void {
-		this._currentLineTokens = this._model.getLineTokens(lineNumber, false);
-	}
+	private _advanceNext(): void {
+		if (!this._next) {
+			return;
+		}
+
+		let lineNumber = this._next.lineNumber;
+		let next = this._next._actual.next();
+		while (!next && lineNumber < this._lineCount) {
+			lineNumber++;
+			let currentLineTokens = this._model.getLineTokens(lineNumber);
+			next = currentLineTokens.firstToken();
+		}
 
-	private _advanceNext() {
 		this._prev = this._next;
-		this._next = null;
-		if (this._currentTokenIndex + 1 < this._currentLineTokens.getTokenCount()) {
-			// There are still tokens on current line
-			this._currentTokenIndex++;
-			this._next = this._current();
-
+		if (next) {
+			this._next = new TokenInfo(next, lineNumber);
 		} else {
-			// find the next line with tokens
-			while (this._currentLineNumber + 1 <= this._model.getLineCount()) {
-				this._currentLineNumber++;
-				this._readLineTokens(this._currentLineNumber);
-				if (this._currentLineTokens.getTokenCount() > 0) {
-					this._currentTokenIndex = 0;
-					this._next = this._current();
-					break;
-				}
-			}
-			if (this._next === null) {
-				// prepare of a previous run
-				this._readLineTokens(this._currentLineNumber);
-				this._currentTokenIndex = this._currentLineTokens.getTokenCount();
-				this._advancePrev();
-				this._next = null;
-			}
+			this._next = null;
 		}
 	}
 
-	private _advancePrev() {
+	private _advancePrev(): void {
+		if (!this._prev) {
+			return;
+		}
+
+		let lineNumber = this._prev.lineNumber;
+		let prev = this._prev._actual.prev();
+		while (!prev && lineNumber > 1) {
+			lineNumber--;
+			let currentLineTokens = this._model.getLineTokens(lineNumber);
+			prev = currentLineTokens.lastToken();
+		}
+
 		this._next = this._prev;
-		this._prev = null;
-		if (this._currentTokenIndex > 0) {
-			// There are still tokens on current line
-			this._currentTokenIndex--;
-			this._prev = this._current();
-
+		if (prev) {
+			this._prev = new TokenInfo(prev, lineNumber);
 		} else {
-			// find previous line with tokens
-			while (this._currentLineNumber > 1) {
-				this._currentLineNumber--;
-				this._readLineTokens(this._currentLineNumber);
-				if (this._currentLineTokens.getTokenCount() > 0) {
-					this._currentTokenIndex = this._currentLineTokens.getTokenCount() - 1;
-					this._prev = this._current();
-					break;
-				}
-			}
+			this._prev = null;
 		}
 	}
 
-	private _current(): editorCommon.ITokenInfo {
-		let startIndex = this._currentLineTokens.getTokenStartOffset(this._currentTokenIndex);
-		let type = this._currentLineTokens.getTokenType(this._currentTokenIndex);
-		let endIndex = this._currentLineTokens.getTokenEndOffset(this._currentTokenIndex);
-
-		return {
-			token: new Token(startIndex, type),
-			lineNumber: this._currentLineNumber,
-			startColumn: startIndex + 1,
-			endColumn: endIndex + 1
-		};
-	}
-
 	public hasNext(): boolean {
 		return this._next !== null;
 	}
 
 	public next(): editorCommon.ITokenInfo {
-		var result = this._next;
+		const result = this._next;
 		this._advanceNext();
 		return result;
 	}
@@ -125,7 +146,7 @@ export class TokenIterator implements editorCommon.ITokenIterator {
 	}
 
 	public prev(): editorCommon.ITokenInfo {
-		var result = this._prev;
+		const result = this._prev;
 		this._advancePrev();
 		return result;
 	}

src/vs/workbench/parts/themes/test/electron-browser/themes.test.contribution.ts

@@ -125,7 +125,7 @@ class Snapper {
 					let content = model.getValueInRange({ startLineNumber: lineNumber, endLineNumber: lineNumber, startColumn: tokenInfo.startColumn, endColumn: tokenInfo.endColumn});
 					result.push({
 						c: content,
-						t: this.normalizeType(tokenInfo.token.type),
+						t: this.normalizeType(tokenInfo.type),
 						r: {}
 					});
 				}