Remove ModeTransition

build/monaco/monaco.d.ts.recipe

@@ -48,11 +48,12 @@ declare module monaco {
 #include(vs/editor/common/core/position): Position
 #include(vs/editor/common/core/range): Range
 #include(vs/editor/common/core/selection): Selection, SelectionDirection
+#include(vs/editor/common/core/token): Token
 }
 
 declare module monaco.editor {
 
-#includeAll(vs/editor/browser/standalone/standaloneEditor;modes.=>languages.):
+#includeAll(vs/editor/browser/standalone/standaloneEditor;modes.=>languages.;editorCommon.=>):
 #include(vs/editor/common/services/webWorker): MonacoWebWorker, IWebWorkerOptions
 #include(vs/editor/browser/standalone/standaloneCodeEditor): IEditorConstructionOptions, IDiffEditorConstructionOptions, IStandaloneCodeEditor, IStandaloneDiffEditor
 export interface ICommandHandler {

src/vs/editor/browser/standalone/standaloneEditor.ts

@@ -6,9 +6,6 @@
 
 import 'vs/css!./media/standalone-tokens';
 import * as editorCommon from 'vs/editor/common/editorCommon';
-/* tslint:disable:duplicate-imports */
-import { IModel } from 'vs/editor/common/editorCommon';
-/* tslint:disable:duplicate-imports */
 import { ContentWidgetPositionPreference, OverlayWidgetPositionPreference } from 'vs/editor/browser/editorBrowser';
 import { StandaloneEditor, IStandaloneCodeEditor, StandaloneDiffEditor, IStandaloneDiffEditor, IEditorConstructionOptions, IDiffEditorConstructionOptions } from 'vs/editor/browser/standalone/standaloneCodeEditor';
 import { ScrollbarVisibility } from 'vs/base/common/scrollable';
@@ -33,9 +30,9 @@ import { IContextKeyService } from 'vs/platform/contextkey/common/contextkey';
 import { ICodeEditorService } from 'vs/editor/common/services/codeEditorService';
 import { IEditorWorkerService } from 'vs/editor/common/services/editorWorkerService';
 import { ITextModelResolverService } from 'vs/editor/common/services/resolverService';
-import { IState, ITokenizationSupport, TokenizationRegistry } from 'vs/editor/common/modes';
 import { NULL_STATE, nullTokenize } from 'vs/editor/common/modes/nullMode';
 import { IStandaloneColorService } from 'vs/editor/common/services/standaloneColorService';
+import { Token } from 'vs/editor/common/core/token';
 
 /**
  * @internal
@@ -136,7 +133,7 @@ export function createDiffNavigator(diffEditor: IStandaloneDiffEditor, opts?: ID
 	return new DiffNavigator(diffEditor, opts);
 }
 
-function doCreateModel(value: string, mode: TPromise<modes.IMode>, uri?: URI): IModel {
+function doCreateModel(value: string, mode: TPromise<modes.IMode>, uri?: URI): editorCommon.IModel {
 	return StaticServices.modelService.get().createModel(value, mode, uri);
 }
 
@@ -144,7 +141,7 @@ function doCreateModel(value: string, mode: TPromise<modes.IMode>, uri?: URI): I
  * Create a new editor model.
  * You can specify the language that should be set for this model or let the language be inferred from the `uri`.
  */
-export function createModel(value: string, language?: string, uri?: URI): IModel {
+export function createModel(value: string, language?: string, uri?: URI): editorCommon.IModel {
 	value = value || '';
 
 	if (!language) {
@@ -164,14 +161,14 @@ export function createModel(value: string, language?: string, uri?: URI): IModel
 /**
  * Change the language for a model.
  */
-export function setModelLanguage(model: IModel, language: string): void {
+export function setModelLanguage(model: editorCommon.IModel, language: string): void {
 	StaticServices.modelService.get().setMode(model, StaticServices.modeService.get().getOrCreateMode(language));
 }
 
 /**
  * Set the markers for a model.
  */
-export function setModelMarkers(model: IModel, owner: string, markers: IMarkerData[]): void {
+export function setModelMarkers(model: editorCommon.IModel, owner: string, markers: IMarkerData[]): void {
 	if (model) {
 		StaticServices.markerService.get().changeOne(owner, model.uri, markers);
 	}
@@ -180,14 +177,14 @@ export function setModelMarkers(model: IModel, owner: string, markers: IMarkerDa
 /**
  * Get the model that has `uri` if it exists.
  */
-export function getModel(uri: URI): IModel {
+export function getModel(uri: URI): editorCommon.IModel {
 	return StaticServices.modelService.get().getModel(uri);
 }
 
 /**
  * Get all the created models.
  */
-export function getModels(): IModel[] {
+export function getModels(): editorCommon.IModel[] {
 	return StaticServices.modelService.get().getModels();
 }
 
@@ -195,7 +192,7 @@ export function getModels(): IModel[] {
  * Emitted when a model is created.
  * @event
  */
-export function onDidCreateModel(listener: (model: IModel) => void): IDisposable {
+export function onDidCreateModel(listener: (model: editorCommon.IModel) => void): IDisposable {
 	return StaticServices.modelService.get().onModelAdded(listener);
 }
 
@@ -203,7 +200,7 @@ export function onDidCreateModel(listener: (model: IModel) => void): IDisposable
  * Emitted right before a model is disposed.
  * @event
  */
-export function onWillDisposeModel(listener: (model: IModel) => void): IDisposable {
+export function onWillDisposeModel(listener: (model: editorCommon.IModel) => void): IDisposable {
 	return StaticServices.modelService.get().onModelRemoved(listener);
 }
 
@@ -211,7 +208,7 @@ export function onWillDisposeModel(listener: (model: IModel) => void): IDisposab
  * Emitted when a different language is set to a model.
  * @event
  */
-export function onDidChangeModelLanguage(listener: (e: { readonly model: IModel; readonly oldLanguage: string; }) => void): IDisposable {
+export function onDidChangeModelLanguage(listener: (e: { readonly model: editorCommon.IModel; readonly oldLanguage: string; }) => void): IDisposable {
 	return StaticServices.modelService.get().onModelModeChanged((e) => {
 		listener({
 			model: e.model,
@@ -255,31 +252,21 @@ export function colorize(text: string, languageId: string, options: IColorizerOp
 /**
  * Colorize a line in a model.
  */
-export function colorizeModelLine(model: IModel, lineNumber: number, tabSize: number = 4): string {
+export function colorizeModelLine(model: editorCommon.IModel, lineNumber: number, tabSize: number = 4): string {
 	return Colorizer.colorizeModelLine(model, lineNumber, tabSize);
 }
 
-export class Token {
-	public readonly offset: number;
-	public readonly type: string;
-
-	constructor(offset: number, type: string) {
-		this.offset = offset;
-		this.type = type;
-	}
-}
-
 /**
  * @internal
  */
-function getSafeTokenizationSupport(languageId: string): ITokenizationSupport {
-	let tokenizationSupport = TokenizationRegistry.get(languageId);
+function getSafeTokenizationSupport(languageId: string): modes.ITokenizationSupport {
+	let tokenizationSupport = modes.TokenizationRegistry.get(languageId);
 	if (tokenizationSupport) {
 		return tokenizationSupport;
 	}
 	return {
 		getInitialState: () => NULL_STATE,
-		tokenize: (line: string, state: IState, deltaOffset: number) => nullTokenize(languageId, line, state, deltaOffset),
+		tokenize: (line: string, state: modes.IState, deltaOffset: number) => nullTokenize(languageId, line, state, deltaOffset),
 		tokenize3: undefined,
 	};
 }
@@ -300,7 +287,7 @@ export function tokenize(text: string, languageId: string): Token[][] {
 		let line = lines[i];
 		let tokenizationResult = tokenizationSupport.tokenize(line, state, 0);
 
-		result[i] = tokenizationResult.tokens.map((t) => new Token(t.startIndex, t.type));
+		result[i] = tokenizationResult.tokens;
 		state = tokenizationResult.endState;
 	}
 	return result;
@@ -358,7 +345,6 @@ export function createMonacoEditorAPI(): typeof monaco.editor {
 		BareFontInfo: <any>editorCommon.BareFontInfo,
 		FontInfo: <any>editorCommon.FontInfo,
 		TextModelResolvedOptions: <any>editorCommon.TextModelResolvedOptions,
-		Token: Token,
 
 		// vars
 		EditorType: editorCommon.EditorType,

src/vs/editor/browser/standalone/standaloneLanguages.ts

@@ -24,7 +24,6 @@ import { createTokenizationSupport } from 'vs/editor/common/modes/monarch/monarc
 import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry';
 import { IMarkerData } from 'vs/platform/markers/common/markers';
 import { Token } from 'vs/editor/common/core/token';
-import { ModeTransition } from 'vs/editor/common/core/modeTransition';
 import { IStandaloneColorService } from 'vs/editor/common/services/standaloneColorService';
 
 /**
@@ -90,18 +89,18 @@ export class TokenizationSupport2Adapter implements modes.ITokenizationSupport {
 		return this._actual.getInitialState();
 	}
 
-	private _toClassicTokens(tokens: modes.IToken2[], offsetDelta: number): Token[] {
+	private _toClassicTokens(tokens: modes.IToken2[], language: string, offsetDelta: number): Token[] {
 		let result: Token[] = [];
 		for (let i = 0, len = tokens.length; i < len; i++) {
 			let t = tokens[i];
-			result[i] = new Token(t.startIndex + offsetDelta, t.scopes);
+			result[i] = new Token(t.startIndex + offsetDelta, t.scopes, language);
 		}
 		return result;
 	}
 
 	public tokenize(line: string, state: modes.IState, offsetDelta: number): modes.ILineTokens {
 		let actualResult = this._actual.tokenize(line, state);
-		let tokens = this._toClassicTokens(actualResult.tokens, offsetDelta);
+		let tokens = this._toClassicTokens(actualResult.tokens, this._languageIdentifier.sid, offsetDelta);
 
 		let endState: modes.IState;
 		// try to save an object if possible
@@ -113,8 +112,7 @@ export class TokenizationSupport2Adapter implements modes.ITokenizationSupport {
 
 		return {
 			tokens: tokens,
-			endState: endState,
-			modeTransitions: [new ModeTransition(offsetDelta, this._languageIdentifier.sid)],
+			endState: endState
 		};
 	}
 

src/vs/editor/common/core/modeTransition.ts

@@ -1,23 +0,0 @@
-/*---------------------------------------------------------------------------------------------
- *  Copyright (c) Microsoft Corporation. All rights reserved.
- *  Licensed under the MIT License. See License.txt in the project root for license information.
- *--------------------------------------------------------------------------------------------*/
-'use strict';
-
-import { Arrays } from 'vs/editor/common/core/arrays';
-
-export class ModeTransition {
-	_modeTransitionBrand: void;
-
-	public readonly startIndex: number;
-	public readonly modeId: string;
-
-	constructor(startIndex: number, modeId: string) {
-		this.startIndex = startIndex | 0;
-		this.modeId = modeId;
-	}
-
-	public static findIndexInSegmentsArray(arr: ModeTransition[], desiredIndex: number): number {
-		return Arrays.findIndexInSegmentsArray(arr, desiredIndex);
-	}
-}

src/vs/editor/common/core/token.ts

@@ -4,45 +4,20 @@
  *--------------------------------------------------------------------------------------------*/
 'use strict';
 
-import { Arrays } from 'vs/editor/common/core/arrays';
-
 export class Token {
 	_tokenBrand: void;
 
-	public readonly startIndex: number;
+	public readonly offset: number;
 	public readonly type: string;
+	public readonly language: string;
 
-	constructor(startIndex: number, type: string) {
-		this.startIndex = startIndex | 0;// @perf
+	constructor(offset: number, type: string, language: string) {
+		this.offset = offset | 0;// @perf
 		this.type = type;
+		this.language = language;
 	}
 
 	public toString(): string {
-		return '(' + this.startIndex + ', ' + this.type + ')';
-	}
-
-	public equals(other: Token): boolean {
-		return (
-			this.startIndex === other.startIndex
-			&& this.type === other.type
-		);
-	}
-
-	public static findIndexInSegmentsArray(arr: Token[], desiredIndex: number): number {
-		return Arrays.findIndexInSegmentsArray(arr, desiredIndex);
-	}
-
-	public static equalsArray(a: Token[], b: Token[]): boolean {
-		let aLen = a.length;
-		let bLen = b.length;
-		if (aLen !== bLen) {
-			return false;
-		}
-		for (let i = 0; i < aLen; i++) {
-			if (!a[i].equals(b[i])) {
-				return false;
-			}
-		}
-		return true;
+		return '(' + this.offset + ', ' + this.type + ')';
 	}
 }

src/vs/editor/common/modes.ts

@@ -9,7 +9,6 @@ import { IDisposable } from 'vs/base/common/lifecycle';
 import URI from 'vs/base/common/uri';
 import { IFilter } from 'vs/base/common/filters';
 import * as editorCommon from 'vs/editor/common/editorCommon';
-import { ModeTransition } from 'vs/editor/common/core/modeTransition';
 import { Token } from 'vs/editor/common/core/token';
 import LanguageFeatureRegistry from 'vs/editor/common/modes/languageFeatureRegistry';
 import { CancellationToken } from 'vs/base/common/cancellation';
@@ -55,7 +54,6 @@ export interface IMode {
 export interface ILineTokens {
 	tokens: Token[];
 	endState: IState;
-	modeTransitions: ModeTransition[];
 }
 
 /**

src/vs/editor/common/modes/monarch/monarchLexer.ts

@@ -15,7 +15,6 @@ import * as monarchCommon from 'vs/editor/common/modes/monarch/monarchCommon';
 import { IModeService } from 'vs/editor/common/services/modeService';
 import { Token } from 'vs/editor/common/core/token';
 import { NULL_STATE, NULL_MODE_ID } from 'vs/editor/common/modes/nullMode';
-import { ModeTransition } from 'vs/editor/common/core/modeTransition';
 import { IStandaloneColorService } from 'vs/editor/common/services/standaloneColorService';
 import { Theme } from 'vs/editor/common/modes/supports/tokenization';
 
@@ -240,35 +239,29 @@ interface IMonarchTokensCollector {
 
 class MonarchClassicTokensCollector implements IMonarchTokensCollector {
 
-	private _modeTransitions: ModeTransition[];
 	private _tokens: Token[];
-
-	private _lastModeId: string;
+	private _language: string;
 	private _lastTokenType: string;
+	private _lastTokenLanguage: string;
 
 	constructor() {
-		this._modeTransitions = [];
 		this._tokens = [];
-
-		this._lastModeId = null;
+		this._language = null;
 		this._lastTokenType = null;
+		this._lastTokenLanguage = null;
 	}
 
 	public enterMode(startOffset: number, modeId: string): void {
-		if (this._lastModeId === modeId) {
-			// Avoid transitioning to the same mode (this can happen in case of empty embedded modes)
-			return;
-		}
-		this._lastModeId = modeId;
-		this._modeTransitions.push(new ModeTransition(startOffset, modeId));
+		this._language = modeId;
 	}
 
 	public emit(startOffset: number, type: string): void {
-		if (this._lastTokenType === type) {
+		if (this._lastTokenType === type && this._lastTokenLanguage === this._language) {
 			return;
 		}
 		this._lastTokenType = type;
-		this._tokens.push(new Token(startOffset, type));
+		this._lastTokenLanguage = this._language;
+		this._tokens.push(new Token(startOffset, type, this._language));
 	}
 
 	public nestedModeTokenize(embeddedModeLine: string, embeddedModeData: EmbeddedModeData, offsetDelta: number): modes.IState {
@@ -285,15 +278,14 @@ class MonarchClassicTokensCollector implements IMonarchTokensCollector {
 		let nestedResult = nestedModeTokenizationSupport.tokenize(embeddedModeLine, embeddedModeState, offsetDelta);
 		this._tokens = this._tokens.concat(nestedResult.tokens);
 		this._lastTokenType = null;
-		this._modeTransitions = this._modeTransitions.concat(nestedResult.modeTransitions);
-		this._lastModeId = null;
+		this._lastTokenLanguage = null;
+		this._language = null;
 		return nestedResult.endState;
 	}
 
 	public finalize(endState: MonarchLineState): modes.ILineTokens {
 		return {
 			tokens: this._tokens,
-			modeTransitions: this._modeTransitions,
 			endState: endState
 		};
 	}

src/vs/editor/common/modes/nullMode.ts

@@ -5,7 +5,6 @@
 'use strict';
 
 import { IState, ILineTokens, ILineTokens3, ColorId, MetadataConsts, LanguageIdentifier, FontStyle, StandardTokenType, LanguageId } from 'vs/editor/common/modes';
-import { ModeTransition } from 'vs/editor/common/core/modeTransition';
 import { Token } from 'vs/editor/common/core/token';
 
 class NullStateImpl implements IState {
@@ -26,14 +25,9 @@ export const NULL_MODE_ID = 'vs.editor.nullMode';
 export const NULL_LANGUAGE_IDENTIFIER = new LanguageIdentifier(NULL_MODE_ID, LanguageId.Null);
 
 export function nullTokenize(modeId: string, buffer: string, state: IState, deltaOffset: number): ILineTokens {
-	let tokens: Token[] = [new Token(deltaOffset, '')];
-
-	let modeTransitions: ModeTransition[] = [new ModeTransition(deltaOffset, modeId)];
-
 	return {
-		tokens: tokens,
-		endState: state,
-		modeTransitions: modeTransitions
+		tokens: [new Token(deltaOffset, '', modeId)],
+		endState: state
 	};
 }
 

src/vs/editor/common/modes/supports.ts

@@ -5,7 +5,6 @@
 'use strict';
 
 import * as modes from 'vs/editor/common/modes';
-import { ModeTransition } from 'vs/editor/common/core/modeTransition';
 import { Token } from 'vs/editor/common/core/token';
 import { LineTokens } from 'vs/editor/common/core/lineTokens';
 
@@ -13,12 +12,10 @@ export class RawLineTokens implements modes.ILineTokens {
 	_lineTokensBrand: void;
 
 	tokens: Token[];
-	modeTransitions: ModeTransition[];
 	endState: modes.IState;
 
-	constructor(tokens: Token[], modeTransitions: ModeTransition[], endState: modes.IState) {
+	constructor(tokens: Token[], endState: modes.IState) {
 		this.tokens = tokens;
-		this.modeTransitions = modeTransitions;
 		this.endState = endState;
 	}
 }

src/vs/editor/common/standalone/standaloneBase.ts

@@ -11,6 +11,7 @@ import { Range } from 'vs/editor/common/core/range';
 import { Selection, SelectionDirection } from 'vs/editor/common/core/selection';
 import { TPromise } from 'vs/base/common/winjs.base';
 import { CancellationTokenSource } from 'vs/base/common/cancellation';
+import { Token } from 'vs/editor/common/core/token';
 import URI from 'vs/base/common/uri';
 
 // --------------------------------------------
@@ -230,6 +231,7 @@ export function createMonacoBaseAPI(): typeof monaco {
 		SelectionDirection: SelectionDirection,
 		Severity: Severity,
 		Promise: TPromise,
-		Uri: URI
+		Uri: URI,
+		Token: Token
 	};
 }

src/vs/editor/node/textMate/TMSyntax.ts

@@ -16,7 +16,6 @@ import { TMState } from 'vs/editor/node/textMate/TMState';
 import { RawLineTokens } from 'vs/editor/common/modes/supports';
 import { IModeService } from 'vs/editor/common/services/modeService';
 import { IGrammar, Registry, StackElement, IToken, IEmbeddedLanguagesMap as IEmbeddedLanguagesMap2 } from 'vscode-textmate';
-import { ModeTransition } from 'vs/editor/common/core/modeTransition';
 import { Token } from 'vs/editor/common/core/token';
 import { languagesExtPoint } from 'vs/editor/common/services/modeServiceImpl';
 import { IThemeService } from 'vs/workbench/services/themes/common/themeService';
@@ -530,8 +529,7 @@ class Tokenizer {
 		if (line.length >= 20000) {
 			console.log(`Line (${line.substr(0, 15)}...): longer than 20k characters, tokenization skipped.`);
 			return new RawLineTokens(
-				[new Token(offsetDelta, '')],
-				[new ModeTransition(offsetDelta, this._modeId)],
+				[new Token(offsetDelta, '', this._modeId)],
 				state
 			);
 		}
@@ -544,8 +542,7 @@ class Tokenizer {
 				this._stackOverflowReported = true;
 			}
 			return new RawLineTokens(
-				[new Token(offsetDelta, '')],
-				[new ModeTransition(offsetDelta, this._modeId)],
+				[new Token(offsetDelta, '', this._modeId)],
 				state
 			);
 		}
@@ -581,10 +578,9 @@ export function decodeTextMateTokens(topLevelModeId: string, decodeMap: DecodeMa
 
 	// Create the result early and fill in the tokens later
 	let tokens: Token[] = [];
-	let modeTransitions: ModeTransition[] = [];
 
 	let lastTokenType: string = null;
-	let lastModeId: string = null;
+	let lastTokenLanguage: string = null;
 
 	for (let tokenIndex = 0, len = resultTokens.length; tokenIndex < len; tokenIndex++) {
 		let token = resultTokens[tokenIndex];
@@ -601,20 +597,15 @@ export function decodeTextMateTokens(topLevelModeId: string, decodeMap: DecodeMa
 		}
 
 		// do not push a new token if the type is exactly the same (also helps with ligatures)
-		if (tokenType !== lastTokenType) {
-			tokens.push(new Token(tokenStartIndex + offsetDelta, tokenType));
+		if (tokenType !== lastTokenType || tokenModeId !== lastTokenLanguage) {
+			tokens.push(new Token(tokenStartIndex + offsetDelta, tokenType, tokenModeId));
 			lastTokenType = tokenType;
-		}
-
-		if (tokenModeId !== lastModeId) {
-			modeTransitions.push(new ModeTransition(tokenStartIndex + offsetDelta, tokenModeId));
-			lastModeId = tokenModeId;
+			lastTokenLanguage = tokenModeId;
 		}
 	}
 
 	return new RawLineTokens(
 		tokens,
-		modeTransitions,
 		resultState
 	);
 }

src/vs/editor/test/node/textMate/TMSyntax.test.ts

@@ -104,14 +104,15 @@ suite('TextMate.decodeTextMateTokens', () => {
 			null
 		);
 
-		let actualModeTransitions = actual.modeTransitions.map((t) => { return { startIndex: t.startIndex, modeId: t.modeId }; });
-
-		assert.deepEqual(actualModeTransitions, [
-			{ startIndex: 0, modeId: 'html' },
-			{ startIndex: 11, modeId: 'css' },
-			{ startIndex: 17, modeId: 'html' },
-			{ startIndex: 33, modeId: 'javascript' },
-			{ startIndex: 41, modeId: 'html' },
+		assert.deepEqual(actual.tokens, [
+			{ offset: 0, language: 'html', type: '' },
+			{ offset: 4, language: 'html', type: 'style.tag.open' },
+			{ offset: 11, language: 'css', type: 'source.css' },
+			{ offset: 17, language: 'html', type: 'style.tag.close' },
+			{ offset: 25, language: 'html', type: 'tag.open.script' },
+			{ offset: 33, language: 'javascript', type: 'source.js' },
+			{ offset: 41, language: 'html', type: 'tag.close.script' },
+			{ offset: 50, language: 'html', type: '' },
 		]);
 	});
 
@@ -129,16 +130,13 @@ suite('TextMate.decodeTextMateTokens', () => {
 					{ startIndex: 10, endIndex: 11, scopes: ['text.html.php', 'meta.tag.any.html', 'punctuation.definition.tag.html'] }
 				],
 				tokens: [
-					{ startIndex: 0, type: 'meta.tag.any.html.punctuation.definition' },
-					{ startIndex: 1, type: 'meta.tag.any.html.entity.name' },
-					{ startIndex: 4, type: 'meta.tag.any.html.punctuation.definition' },
-					{ startIndex: 5, type: 'meta.tag.any.html.punctuation.definition.scope.between-tag-pair' },
-					{ startIndex: 6, type: 'meta.tag.any.html.punctuation.definition' },
-					{ startIndex: 7, type: 'meta.tag.any.html.entity.name' },
-					{ startIndex: 10, type: 'meta.tag.any.html.punctuation.definition' }
-				],
-				modeTransitions: [
-					{ startIndex: 0, modeId: 'html' }
+					{ offset: 0, language: 'html', type: 'meta.tag.any.html.punctuation.definition' },
+					{ offset: 1, language: 'html', type: 'meta.tag.any.html.entity.name' },
+					{ offset: 4, language: 'html', type: 'meta.tag.any.html.punctuation.definition' },
+					{ offset: 5, language: 'html', type: 'meta.tag.any.html.punctuation.definition.scope.between-tag-pair' },
+					{ offset: 6, language: 'html', type: 'meta.tag.any.html.punctuation.definition' },
+					{ offset: 7, language: 'html', type: 'meta.tag.any.html.entity.name' },
+					{ offset: 10, language: 'html', type: 'meta.tag.any.html.punctuation.definition' }
 				]
 			}, {
 				line: '<script>var x = 3;</script>',
@@ -159,25 +157,20 @@ suite('TextMate.decodeTextMateTokens', () => {
 					{ startIndex: 26, endIndex: 27, scopes: ['text.html.php', 'punctuation.definition.tag.html'] }
 				],
 				tokens: [
-					{ startIndex: 0, type: 'tag.html.punctuation.definition' },
-					{ startIndex: 1, type: 'tag.html.entity.name.script' },
-					{ startIndex: 7, type: 'tag.html.punctuation.definition.source.js.embedded' },
-					{ startIndex: 8, type: 'meta.html.source.js.embedded.var.expr.storage.type' },
-					{ startIndex: 11, type: 'meta.html.source.js.embedded.var.expr' },
-					{ startIndex: 12, type: 'meta.html.source.js.embedded.var.expr.var-single-variable.variable.other.readwrite' },
-					{ startIndex: 13, type: 'meta.html.source.js.embedded.var.expr.var-single-variable' },
-					{ startIndex: 14, type: 'meta.html.source.js.embedded.var.expr.keyword.operator.assignment' },
-					{ startIndex: 15, type: 'meta.html.source.js.embedded.var.expr' },
-					{ startIndex: 16, type: 'meta.html.source.js.embedded.var.expr.constant.numeric.decimal' },
-					{ startIndex: 17, type: 'html.punctuation.source.js.embedded.terminator.statement' },
-					{ startIndex: 18, type: 'tag.html.punctuation.definition.source.js.embedded' },
-					{ startIndex: 20, type: 'tag.html.entity.name.script.source.js.embedded' },
-					{ startIndex: 26, type: 'tag.html.punctuation.definition' }
-				],
-				modeTransitions: [
-					{ startIndex: 0, modeId: 'html' },
-					{ startIndex: 7, modeId: 'javascript' },
-					{ startIndex: 26, modeId: 'html' }
+					{ offset: 0, language: 'html', type: 'tag.html.punctuation.definition' },
+					{ offset: 1, language: 'html', type: 'tag.html.entity.name.script' },
+					{ offset: 7, language: 'javascript', type: 'tag.html.punctuation.definition.source.js.embedded' },
+					{ offset: 8, language: 'javascript', type: 'meta.html.source.js.embedded.var.expr.storage.type' },
+					{ offset: 11, language: 'javascript', type: 'meta.html.source.js.embedded.var.expr' },
+					{ offset: 12, language: 'javascript', type: 'meta.html.source.js.embedded.var.expr.var-single-variable.variable.other.readwrite' },
+					{ offset: 13, language: 'javascript', type: 'meta.html.source.js.embedded.var.expr.var-single-variable' },
+					{ offset: 14, language: 'javascript', type: 'meta.html.source.js.embedded.var.expr.keyword.operator.assignment' },
+					{ offset: 15, language: 'javascript', type: 'meta.html.source.js.embedded.var.expr' },
+					{ offset: 16, language: 'javascript', type: 'meta.html.source.js.embedded.var.expr.constant.numeric.decimal' },
+					{ offset: 17, language: 'javascript', type: 'html.punctuation.source.js.embedded.terminator.statement' },
+					{ offset: 18, language: 'javascript', type: 'tag.html.punctuation.definition.source.js.embedded' },
+					{ offset: 20, language: 'javascript', type: 'tag.html.entity.name.script.source.js.embedded' },
+					{ offset: 26, language: 'html', type: 'tag.html.punctuation.definition' }
 				]
 			}, {
 				line: '<style>body{background-color:red;}</style>',
@@ -197,24 +190,19 @@ suite('TextMate.decodeTextMateTokens', () => {
 					{ startIndex: 41, endIndex: 42, scopes: ['text.html.php', 'punctuation.definition.tag.html'] }
 				],
 				tokens: [
-					{ startIndex: 0, type: 'tag.html.punctuation.definition' },
-					{ startIndex: 1, type: 'tag.html.entity.name.style' },
-					{ startIndex: 6, type: 'tag.html.punctuation.definition.source.embedded.css' },
-					{ startIndex: 7, type: 'meta.tag.html.entity.name.source.embedded.css.selector' },
-					{ startIndex: 11, type: 'meta.html.punctuation.source.embedded.css.property-list.section.begin' },
-					{ startIndex: 12, type: 'meta.html.source.embedded.type.css.property-list.property-name.support' },
-					{ startIndex: 28, type: 'meta.html.punctuation.source.embedded.css.property-list.property-value.separator.key-value' },
-					{ startIndex: 29, type: 'meta.html.source.embedded.constant.css.property-list.support.property-value.color.w3c-standard-color-name' },
-					{ startIndex: 32, type: 'meta.html.punctuation.source.embedded.terminator.css.property-list.property-value.rule' },
-					{ startIndex: 33, type: 'meta.html.punctuation.source.embedded.css.property-list.section.end' },
-					{ startIndex: 34, type: 'tag.html.punctuation.definition' },
-					{ startIndex: 36, type: 'tag.html.entity.name.style' },
-					{ startIndex: 41, type: 'tag.html.punctuation.definition' }
-				],
-				modeTransitions: [
-					{ startIndex: 0, modeId: 'html' },
-					{ startIndex: 6, modeId: 'css' },
-					{ startIndex: 34, modeId: 'html' }
+					{ offset: 0, language: 'html', type: 'tag.html.punctuation.definition' },
+					{ offset: 1, language: 'html', type: 'tag.html.entity.name.style' },
+					{ offset: 6, language: 'css', type: 'tag.html.punctuation.definition.source.embedded.css' },
+					{ offset: 7, language: 'css', type: 'meta.tag.html.entity.name.source.embedded.css.selector' },
+					{ offset: 11, language: 'css', type: 'meta.html.punctuation.source.embedded.css.property-list.section.begin' },
+					{ offset: 12, language: 'css', type: 'meta.html.source.embedded.type.css.property-list.property-name.support' },
+					{ offset: 28, language: 'css', type: 'meta.html.punctuation.source.embedded.css.property-list.property-value.separator.key-value' },
+					{ offset: 29, language: 'css', type: 'meta.html.source.embedded.constant.css.property-list.support.property-value.color.w3c-standard-color-name' },
+					{ offset: 32, language: 'css', type: 'meta.html.punctuation.source.embedded.terminator.css.property-list.property-value.rule' },
+					{ offset: 33, language: 'css', type: 'meta.html.punctuation.source.embedded.css.property-list.section.end' },
+					{ offset: 34, language: 'html', type: 'tag.html.punctuation.definition' },
+					{ offset: 36, language: 'html', type: 'tag.html.entity.name.style' },
+					{ offset: 41, language: 'html', type: 'tag.html.punctuation.definition' }
 				]
 			}, {
 				line: '<?php',
@@ -222,10 +210,7 @@ suite('TextMate.decodeTextMateTokens', () => {
 					{ startIndex: 0, endIndex: 5, scopes: ['text.html.php', 'meta.embedded.block.php', 'punctuation.section.embedded.metatag.begin.php'] }
 				],
 				tokens: [
-					{ startIndex: 0, type: 'meta.punctuation.embedded.section.begin.block.php.metatag' }
-				],
-				modeTransitions: [
-					{ startIndex: 0, modeId: 'html' }
+					{ offset: 0, language: 'html', type: 'meta.punctuation.embedded.section.begin.block.php.metatag' }
 				]
 			}, {
 				line: '$query = \"SELECT col1, col2 FROM db; -- selects from sql\"; ',
@@ -246,25 +231,20 @@ suite('TextMate.decodeTextMateTokens', () => {
 					{ startIndex: 58, endIndex: 60, scopes: ['text.html.php', 'meta.embedded.block.php', 'source.php'] }
 				],
 				tokens: [
-					{ startIndex: 0, type: 'meta.punctuation.definition.source.embedded.variable.other.block.php' },
-					{ startIndex: 1, type: 'meta.source.embedded.variable.other.block.php' },
-					{ startIndex: 6, type: 'meta.source.embedded.block.php' },
-					{ startIndex: 7, type: 'meta.source.embedded.keyword.operator.assignment.block.php' },
-					{ startIndex: 8, type: 'meta.source.embedded.block.php' },
-					{ startIndex: 9, type: 'meta.punctuation.definition.source.embedded.begin.block.php.string.quoted.double.sql' },
-					{ startIndex: 10, type: 'meta.source.embedded.other.keyword.block.php.string.quoted.double.sql.DML' },
-					{ startIndex: 16, type: 'meta.source.embedded.block.php.string.quoted.double.sql' },
-					{ startIndex: 28, type: 'meta.source.embedded.other.keyword.block.php.string.quoted.double.sql.DML' },
-					{ startIndex: 32, type: 'meta.source.embedded.block.php.string.quoted.double.sql' },
-					{ startIndex: 37, type: 'meta.source.embedded.block.php.string.quoted.double.sql.comment.line.double-dash' },
-					{ startIndex: 56, type: 'meta.punctuation.definition.source.embedded.end.block.php.string.quoted.double.sql' },
-					{ startIndex: 57, type: 'meta.punctuation.source.embedded.terminator.block.php.expression' },
-					{ startIndex: 58, type: 'meta.source.embedded.block.php' }
-				],
-				modeTransitions: [
-					{ startIndex: 0, modeId: 'php' },
-					{ startIndex: 10, modeId: 'sql' },
-					{ startIndex: 56, modeId: 'php' }
+					{ offset: 0, language: 'php', type: 'meta.punctuation.definition.source.embedded.variable.other.block.php' },
+					{ offset: 1, language: 'php', type: 'meta.source.embedded.variable.other.block.php' },
+					{ offset: 6, language: 'php', type: 'meta.source.embedded.block.php' },
+					{ offset: 7, language: 'php', type: 'meta.source.embedded.keyword.operator.assignment.block.php' },
+					{ offset: 8, language: 'php', type: 'meta.source.embedded.block.php' },
+					{ offset: 9, language: 'php', type: 'meta.punctuation.definition.source.embedded.begin.block.php.string.quoted.double.sql' },
+					{ offset: 10, language: 'sql', type: 'meta.source.embedded.other.keyword.block.php.string.quoted.double.sql.DML' },
+					{ offset: 16, language: 'sql', type: 'meta.source.embedded.block.php.string.quoted.double.sql' },
+					{ offset: 28, language: 'sql', type: 'meta.source.embedded.other.keyword.block.php.string.quoted.double.sql.DML' },
+					{ offset: 32, language: 'sql', type: 'meta.source.embedded.block.php.string.quoted.double.sql' },
+					{ offset: 37, language: 'sql', type: 'meta.source.embedded.block.php.string.quoted.double.sql.comment.line.double-dash' },
+					{ offset: 56, language: 'php', type: 'meta.punctuation.definition.source.embedded.end.block.php.string.quoted.double.sql' },
+					{ offset: 57, language: 'php', type: 'meta.punctuation.source.embedded.terminator.block.php.expression' },
+					{ offset: 58, language: 'php', type: 'meta.source.embedded.block.php' }
 				]
 			}, {
 				line: '$a = <<<JSON { \"3\": \"4\" } JSON;',
@@ -295,31 +275,28 @@ suite('TextMate.decodeTextMateTokens', () => {
 					{ startIndex: 30, endIndex: 31, scopes: ['text.html.php', 'meta.embedded.block.php', 'source.php', 'punctuation.terminator.expression.php'] }
 				],
 				tokens: [
-					{ startIndex: 0, type: 'meta.punctuation.definition.source.embedded.variable.other.block.php' },
-					{ startIndex: 1, type: 'meta.source.embedded.variable.other.block.php' },
-					{ startIndex: 2, type: 'meta.source.embedded.block.php' },
-					{ startIndex: 3, type: 'meta.source.embedded.keyword.operator.assignment.block.php' },
-					{ startIndex: 4, type: 'meta.source.embedded.block.php' },
-					{ startIndex: 5, type: 'meta.source.embedded.keyword.operator.block.php.comparison' },
-					{ startIndex: 8, type: 'meta.source.embedded.other.constant.block.php' },
-					{ startIndex: 12, type: 'meta.source.embedded.block.php' },
-					{ startIndex: 13, type: 'meta.punctuation.scope.source.embedded.section.begin.block.php' },
-					{ startIndex: 14, type: 'meta.source.embedded.block.php' },
-					{ startIndex: 15, type: 'meta.punctuation.definition.source.embedded.begin.block.php.string.quoted.double' },
-					{ startIndex: 16, type: 'meta.source.embedded.block.php.string.quoted.double.string-contents' },
-					{ startIndex: 17, type: 'meta.punctuation.definition.source.embedded.end.block.php.string.quoted.double' },
-					{ startIndex: 18, type: 'meta.source.embedded.block.php' },
-					{ startIndex: 20, type: 'meta.punctuation.definition.source.embedded.begin.block.php.string.quoted.double' },
-					{ startIndex: 21, type: 'meta.source.embedded.block.php.string.quoted.double.string-contents' },
-					{ startIndex: 22, type: 'meta.punctuation.definition.source.embedded.end.block.php.string.quoted.double' },
-					{ startIndex: 23, type: 'meta.source.embedded.block.php' },
-					{ startIndex: 24, type: 'meta.punctuation.scope.source.embedded.section.end.block.php' },
-					{ startIndex: 25, type: 'meta.source.embedded.block.php' },
-					{ startIndex: 26, type: 'meta.source.embedded.other.constant.block.php' },
-					{ startIndex: 30, type: 'meta.punctuation.source.embedded.terminator.block.php.expression' }
-				],
-				modeTransitions: [
-					{ startIndex: 0, modeId: 'php' }
+					{ offset: 0, language: 'php', type: 'meta.punctuation.definition.source.embedded.variable.other.block.php' },
+					{ offset: 1, language: 'php', type: 'meta.source.embedded.variable.other.block.php' },
+					{ offset: 2, language: 'php', type: 'meta.source.embedded.block.php' },
+					{ offset: 3, language: 'php', type: 'meta.source.embedded.keyword.operator.assignment.block.php' },
+					{ offset: 4, language: 'php', type: 'meta.source.embedded.block.php' },
+					{ offset: 5, language: 'php', type: 'meta.source.embedded.keyword.operator.block.php.comparison' },
+					{ offset: 8, language: 'php', type: 'meta.source.embedded.other.constant.block.php' },
+					{ offset: 12, language: 'php', type: 'meta.source.embedded.block.php' },
+					{ offset: 13, language: 'php', type: 'meta.punctuation.scope.source.embedded.section.begin.block.php' },
+					{ offset: 14, language: 'php', type: 'meta.source.embedded.block.php' },
+					{ offset: 15, language: 'php', type: 'meta.punctuation.definition.source.embedded.begin.block.php.string.quoted.double' },
+					{ offset: 16, language: 'php', type: 'meta.source.embedded.block.php.string.quoted.double.string-contents' },
+					{ offset: 17, language: 'php', type: 'meta.punctuation.definition.source.embedded.end.block.php.string.quoted.double' },
+					{ offset: 18, language: 'php', type: 'meta.source.embedded.block.php' },
+					{ offset: 20, language: 'php', type: 'meta.punctuation.definition.source.embedded.begin.block.php.string.quoted.double' },
+					{ offset: 21, language: 'php', type: 'meta.source.embedded.block.php.string.quoted.double.string-contents' },
+					{ offset: 22, language: 'php', type: 'meta.punctuation.definition.source.embedded.end.block.php.string.quoted.double' },
+					{ offset: 23, language: 'php', type: 'meta.source.embedded.block.php' },
+					{ offset: 24, language: 'php', type: 'meta.punctuation.scope.source.embedded.section.end.block.php' },
+					{ offset: 25, language: 'php', type: 'meta.source.embedded.block.php' },
+					{ offset: 26, language: 'php', type: 'meta.source.embedded.other.constant.block.php' },
+					{ offset: 30, language: 'php', type: 'meta.punctuation.source.embedded.terminator.block.php.expression' }
 				]
 			}, {
 				line: '?>',
@@ -328,12 +305,8 @@ suite('TextMate.decodeTextMateTokens', () => {
 					{ startIndex: 1, endIndex: 2, scopes: ['text.html.php', 'meta.embedded.block.php', 'punctuation.section.embedded.metatag.end.php'] }
 				],
 				tokens: [
-					{ startIndex: 0, type: 'meta.punctuation.source.embedded.section.end.block.php.metatag' },
-					{ startIndex: 1, type: 'meta.punctuation.embedded.section.end.block.php.metatag' }
-				],
-				modeTransitions: [
-					{ startIndex: 0, modeId: 'php' },
-					{ startIndex: 1, modeId: 'html' }
+					{ offset: 0, language: 'php', type: 'meta.punctuation.source.embedded.section.end.block.php.metatag' },
+					{ offset: 1, language: 'html', type: 'meta.punctuation.embedded.section.end.block.php.metatag' }
 				]
 			}, {
 				line: '<div><?=\"something\"?></div>',
@@ -352,23 +325,18 @@ suite('TextMate.decodeTextMateTokens', () => {
 					{ startIndex: 26, endIndex: 27, scopes: ['text.html.php', 'meta.tag.block.any.html', 'punctuation.definition.tag.end.html'] }
 				],
 				tokens: [
-					{ startIndex: 0, type: 'meta.tag.any.html.punctuation.definition.begin.block' },
-					{ startIndex: 1, type: 'meta.tag.any.html.entity.name.block' },
-					{ startIndex: 4, type: 'meta.tag.any.html.punctuation.definition.end.block' },
-					{ startIndex: 5, type: 'meta.punctuation.embedded.section.begin.php.line' },
-					{ startIndex: 8, type: 'meta.punctuation.definition.source.embedded.begin.php.string.quoted.double.line' },
-					{ startIndex: 9, type: 'meta.source.embedded.php.string.quoted.double.line.string-contents' },
-					{ startIndex: 18, type: 'meta.punctuation.definition.source.embedded.end.php.string.quoted.double.line' },
-					{ startIndex: 19, type: 'meta.punctuation.source.embedded.section.end.php.line' },
-					{ startIndex: 20, type: 'meta.punctuation.embedded.section.end.php.line' },
-					{ startIndex: 21, type: 'meta.tag.any.html.punctuation.definition.begin.block' },
-					{ startIndex: 23, type: 'meta.tag.any.html.entity.name.block' },
-					{ startIndex: 26, type: 'meta.tag.any.html.punctuation.definition.end.block' }
-				],
-				modeTransitions: [
-					{ startIndex: 0, modeId: 'html' },
-					{ startIndex: 8, modeId: 'php' },
-					{ startIndex: 20, modeId: 'html' }
+					{ offset: 0, language: 'html', type: 'meta.tag.any.html.punctuation.definition.begin.block' },
+					{ offset: 1, language: 'html', type: 'meta.tag.any.html.entity.name.block' },
+					{ offset: 4, language: 'html', type: 'meta.tag.any.html.punctuation.definition.end.block' },
+					{ offset: 5, language: 'html', type: 'meta.punctuation.embedded.section.begin.php.line' },
+					{ offset: 8, language: 'php', type: 'meta.punctuation.definition.source.embedded.begin.php.string.quoted.double.line' },
+					{ offset: 9, language: 'php', type: 'meta.source.embedded.php.string.quoted.double.line.string-contents' },
+					{ offset: 18, language: 'php', type: 'meta.punctuation.definition.source.embedded.end.php.string.quoted.double.line' },
+					{ offset: 19, language: 'php', type: 'meta.punctuation.source.embedded.section.end.php.line' },
+					{ offset: 20, language: 'html', type: 'meta.punctuation.embedded.section.end.php.line' },
+					{ offset: 21, language: 'html', type: 'meta.tag.any.html.punctuation.definition.begin.block' },
+					{ offset: 23, language: 'html', type: 'meta.tag.any.html.entity.name.block' },
+					{ offset: 26, language: 'html', type: 'meta.tag.any.html.punctuation.definition.end.block' }
 				]
 			}
 		];
@@ -390,12 +358,7 @@ suite('TextMate.decodeTextMateTokens', () => {
 		for (let i = 0, len = tests.length; i < len; i++) {
 			let test = tests[i];
 			let actual = decodeTextMateTokens('html', decodeMap, test.line, 0, test.tmTokens, null);
-
-			let actualTokens = actual.tokens.map((t) => { return { startIndex: t.startIndex, type: t.type }; });
-			let actualModeTransitions = actual.modeTransitions.map((t) => { return { startIndex: t.startIndex, modeId: t.modeId }; });
-
-			assert.deepEqual(actualTokens, test.tokens, 'test ' + test.line);
-			assert.deepEqual(actualModeTransitions, test.modeTransitions, 'test ' + test.line);
+			assert.deepEqual(actual.tokens, test.tokens, 'test ' + test.line);
 		}
 	});
 
@@ -411,11 +374,10 @@ suite('TextMate.decodeTextMateTokens', () => {
 					{ startIndex: 14, endIndex: 15, scopes: ['text.html.basic', 'meta.tag.sgml.html', 'punctuation.definition.tag.html'] }
 				],
 				tokens: [
-					{ startIndex: 0, type: 'meta.tag.sgml.html.punctuation.definition' },
-					{ startIndex: 2, type: 'meta.tag.sgml.html.doctype' },
-					{ startIndex: 14, type: 'meta.tag.sgml.html.punctuation.definition' },
+					{ offset: 0, language: 'html', type: 'meta.tag.sgml.html.punctuation.definition' },
+					{ offset: 2, language: 'html', type: 'meta.tag.sgml.html.doctype' },
+					{ offset: 14, language: 'html', type: 'meta.tag.sgml.html.punctuation.definition' },
 				],
-				modeTransitions: [{ startIndex: 0, modeId: 'html' }]
 			}, {
 				line: '<!-- ',
 				tmTokens: [
@@ -423,28 +385,25 @@ suite('TextMate.decodeTextMateTokens', () => {
 					{ startIndex: 4, endIndex: 6, scopes: ['text.html.basic', 'comment.block.html'] }
 				],
 				tokens: [
-					{ startIndex: 0, type: 'html.punctuation.definition.comment.block' },
-					{ startIndex: 4, type: 'html.comment.block' },
+					{ offset: 0, language: 'html', type: 'html.punctuation.definition.comment.block' },
+					{ offset: 4, language: 'html', type: 'html.comment.block' },
 				],
-				modeTransitions: [{ startIndex: 0, modeId: 'html' }]
 			}, {
 				line: '\tComments are overrated',
 				tmTokens: [
 					{ startIndex: 0, endIndex: 24, scopes: ['text.html.basic', 'comment.block.html'] }
 				],
 				tokens: [
-					{ startIndex: 0, type: 'html.comment.block' },
+					{ offset: 0, language: 'html', type: 'html.comment.block' },
 				],
-				modeTransitions: [{ startIndex: 0, modeId: 'html' }]
 			}, {
 				line: '-->',
 				tmTokens: [
 					{ startIndex: 0, endIndex: 3, scopes: ['text.html.basic', 'comment.block.html', 'punctuation.definition.comment.html'] }
 				],
 				tokens: [
-					{ startIndex: 0, type: 'html.punctuation.definition.comment.block' },
+					{ offset: 0, language: 'html', type: 'html.punctuation.definition.comment.block' },
 				],
-				modeTransitions: [{ startIndex: 0, modeId: 'html' }]
 			}, {
 				line: '<html>',
 				tmTokens: [
@@ -453,11 +412,10 @@ suite('TextMate.decodeTextMateTokens', () => {
 					{ startIndex: 5, endIndex: 6, scopes: ['text.html.basic', 'meta.tag.structure.any.html', 'punctuation.definition.tag.html'] }
 				],
 				tokens: [
-					{ startIndex: 0, type: 'meta.tag.html.punctuation.definition.structure.any' },
-					{ startIndex: 1, type: 'meta.tag.html.structure.any.entity.name' },
-					{ startIndex: 5, type: 'meta.tag.html.punctuation.definition.structure.any' },
+					{ offset: 0, language: 'html', type: 'meta.tag.html.punctuation.definition.structure.any' },
+					{ offset: 1, language: 'html', type: 'meta.tag.html.structure.any.entity.name' },
+					{ offset: 5, language: 'html', type: 'meta.tag.html.punctuation.definition.structure.any' },
 				],
-				modeTransitions: [{ startIndex: 0, modeId: 'html' }]
 			}, {
 				line: '<head>',
 				tmTokens: [
@@ -466,11 +424,10 @@ suite('TextMate.decodeTextMateTokens', () => {
 					{ startIndex: 5, endIndex: 6, scopes: ['text.html.basic', 'meta.tag.structure.any.html', 'punctuation.definition.tag.html'] }
 				],
 				tokens: [
-					{ startIndex: 0, type: 'meta.tag.html.punctuation.definition.structure.any' },
-					{ startIndex: 1, type: 'meta.tag.html.structure.any.entity.name' },
-					{ startIndex: 5, type: 'meta.tag.html.punctuation.definition.structure.any' },
+					{ offset: 0, language: 'html', type: 'meta.tag.html.punctuation.definition.structure.any' },
+					{ offset: 1, language: 'html', type: 'meta.tag.html.structure.any.entity.name' },
+					{ offset: 5, language: 'html', type: 'meta.tag.html.punctuation.definition.structure.any' },
 				],
-				modeTransitions: [{ startIndex: 0, modeId: 'html' }]
 			}, {
 				line: '\t<title>HTML Sample</title>',
 				tmTokens: [
@@ -484,16 +441,15 @@ suite('TextMate.decodeTextMateTokens', () => {
 					{ startIndex: 26, endIndex: 27, scopes: ['text.html.basic', 'meta.tag.inline.any.html', 'punctuation.definition.tag.end.html'] }
 				],
 				tokens: [
-					{ startIndex: 0, type: '' },
-					{ startIndex: 1, type: 'meta.tag.html.punctuation.definition.any.inline.begin' },
-					{ startIndex: 2, type: 'meta.tag.html.any.entity.name.inline' },
-					{ startIndex: 7, type: 'meta.tag.html.punctuation.definition.any.inline.end' },
-					{ startIndex: 8, type: '' },
-					{ startIndex: 19, type: 'meta.tag.html.punctuation.definition.any.inline.begin' },
-					{ startIndex: 21, type: 'meta.tag.html.any.entity.name.inline' },
-					{ startIndex: 26, type: 'meta.tag.html.punctuation.definition.any.inline.end' },
+					{ offset: 0, language: 'html', type: '' },
+					{ offset: 1, language: 'html', type: 'meta.tag.html.punctuation.definition.any.inline.begin' },
+					{ offset: 2, language: 'html', type: 'meta.tag.html.any.entity.name.inline' },
+					{ offset: 7, language: 'html', type: 'meta.tag.html.punctuation.definition.any.inline.end' },
+					{ offset: 8, language: 'html', type: '' },
+					{ offset: 19, language: 'html', type: 'meta.tag.html.punctuation.definition.any.inline.begin' },
+					{ offset: 21, language: 'html', type: 'meta.tag.html.any.entity.name.inline' },
+					{ offset: 26, language: 'html', type: 'meta.tag.html.punctuation.definition.any.inline.end' },
 				],
-				modeTransitions: [{ startIndex: 0, modeId: 'html' }]
 			}, {
 				line: '\t<meta http-equiv=\"X-UA-Compatible\" content=\"IE=edge\">',
 				tmTokens: [
@@ -515,24 +471,23 @@ suite('TextMate.decodeTextMateTokens', () => {
 					{ startIndex: 53, endIndex: 54, scopes: ['text.html.basic', 'meta.tag.inline.any.html', 'punctuation.definition.tag.end.html'] }
 				],
 				tokens: [
-					{ startIndex: 0, type: '' },
-					{ startIndex: 1, type: 'meta.tag.html.punctuation.definition.any.inline.begin' },
-					{ startIndex: 2, type: 'meta.tag.html.any.entity.name.inline' },
-					{ startIndex: 6, type: 'meta.tag.html.any.inline' },
-					{ startIndex: 7, type: 'meta.tag.html.any.entity.inline.other.attribute-name' },
-					{ startIndex: 17, type: 'meta.tag.html.any.inline' },
-					{ startIndex: 18, type: 'meta.tag.html.punctuation.definition.any.inline.begin.string.quoted.double' },
-					{ startIndex: 19, type: 'meta.tag.html.any.inline.string.quoted.double' },
-					{ startIndex: 34, type: 'meta.tag.html.punctuation.definition.any.inline.end.string.quoted.double' },
-					{ startIndex: 35, type: 'meta.tag.html.any.inline' },
-					{ startIndex: 36, type: 'meta.tag.html.any.entity.inline.other.attribute-name' },
-					{ startIndex: 43, type: 'meta.tag.html.any.inline' },
-					{ startIndex: 44, type: 'meta.tag.html.punctuation.definition.any.inline.begin.string.quoted.double' },
-					{ startIndex: 45, type: 'meta.tag.html.any.inline.string.quoted.double' },
-					{ startIndex: 52, type: 'meta.tag.html.punctuation.definition.any.inline.end.string.quoted.double' },
-					{ startIndex: 53, type: 'meta.tag.html.punctuation.definition.any.inline.end' },
+					{ offset: 0, language: 'html', type: '' },
+					{ offset: 1, language: 'html', type: 'meta.tag.html.punctuation.definition.any.inline.begin' },
+					{ offset: 2, language: 'html', type: 'meta.tag.html.any.entity.name.inline' },
+					{ offset: 6, language: 'html', type: 'meta.tag.html.any.inline' },
+					{ offset: 7, language: 'html', type: 'meta.tag.html.any.entity.inline.other.attribute-name' },
+					{ offset: 17, language: 'html', type: 'meta.tag.html.any.inline' },
+					{ offset: 18, language: 'html', type: 'meta.tag.html.punctuation.definition.any.inline.begin.string.quoted.double' },
+					{ offset: 19, language: 'html', type: 'meta.tag.html.any.inline.string.quoted.double' },
+					{ offset: 34, language: 'html', type: 'meta.tag.html.punctuation.definition.any.inline.end.string.quoted.double' },
+					{ offset: 35, language: 'html', type: 'meta.tag.html.any.inline' },
+					{ offset: 36, language: 'html', type: 'meta.tag.html.any.entity.inline.other.attribute-name' },
+					{ offset: 43, language: 'html', type: 'meta.tag.html.any.inline' },
+					{ offset: 44, language: 'html', type: 'meta.tag.html.punctuation.definition.any.inline.begin.string.quoted.double' },
+					{ offset: 45, language: 'html', type: 'meta.tag.html.any.inline.string.quoted.double' },
+					{ offset: 52, language: 'html', type: 'meta.tag.html.punctuation.definition.any.inline.end.string.quoted.double' },
+					{ offset: 53, language: 'html', type: 'meta.tag.html.punctuation.definition.any.inline.end' },
 				],
-				modeTransitions: [{ startIndex: 0, modeId: 'html' }]
 			}, {
 				line: '\t<style type=\"text/css\">',
 				tmTokens: [
@@ -548,18 +503,17 @@ suite('TextMate.decodeTextMateTokens', () => {
 					{ startIndex: 23, endIndex: 24, scopes: ['text.html.basic', 'source.css.embedded.html', 'punctuation.definition.tag.html'] }
 				],
 				tokens: [
-					{ startIndex: 0, type: 'html.source.css.embedded' },
-					{ startIndex: 1, type: 'tag.html.punctuation.definition.source.css.embedded' },
-					{ startIndex: 2, type: 'tag.html.entity.name.source.css.embedded.style' },
-					{ startIndex: 7, type: 'html.source.css.embedded' },
-					{ startIndex: 8, type: 'html.entity.other.attribute-name.source.css.embedded' },
-					{ startIndex: 12, type: 'html.source.css.embedded' },
-					{ startIndex: 13, type: 'html.punctuation.definition.begin.string.quoted.double.source.css.embedded' },
-					{ startIndex: 14, type: 'html.string.quoted.double.source.css.embedded' },
-					{ startIndex: 22, type: 'html.punctuation.definition.end.string.quoted.double.source.css.embedded' },
-					{ startIndex: 23, type: 'tag.html.punctuation.definition.source.css.embedded' },
+					{ offset: 0, language: 'css', type: 'html.source.css.embedded' },
+					{ offset: 1, language: 'css', type: 'tag.html.punctuation.definition.source.css.embedded' },
+					{ offset: 2, language: 'css', type: 'tag.html.entity.name.source.css.embedded.style' },
+					{ offset: 7, language: 'css', type: 'html.source.css.embedded' },
+					{ offset: 8, language: 'css', type: 'html.entity.other.attribute-name.source.css.embedded' },
+					{ offset: 12, language: 'css', type: 'html.source.css.embedded' },
+					{ offset: 13, language: 'css', type: 'html.punctuation.definition.begin.string.quoted.double.source.css.embedded' },
+					{ offset: 14, language: 'css', type: 'html.string.quoted.double.source.css.embedded' },
+					{ offset: 22, language: 'css', type: 'html.punctuation.definition.end.string.quoted.double.source.css.embedded' },
+					{ offset: 23, language: 'css', type: 'tag.html.punctuation.definition.source.css.embedded' },
 				],
-				modeTransitions: [{ startIndex: 0, modeId: 'css' }]
 			}, {
 				line: '\t\th1 {',
 				tmTokens: [
@@ -569,12 +523,11 @@ suite('TextMate.decodeTextMateTokens', () => {
 					{ startIndex: 5, endIndex: 6, scopes: ['text.html.basic', 'source.css.embedded.html', 'meta.property-list.css', 'punctuation.section.property-list.begin.css'] }
 				],
 				tokens: [
-					{ startIndex: 0, type: 'meta.html.source.css.embedded.selector' },
-					{ startIndex: 2, type: 'meta.tag.html.entity.name.source.css.embedded.selector' },
-					{ startIndex: 4, type: 'meta.html.source.css.embedded.selector' },
-					{ startIndex: 5, type: 'meta.html.punctuation.begin.source.css.embedded.property-list.section' },
+					{ offset: 0, language: 'css', type: 'meta.html.source.css.embedded.selector' },
+					{ offset: 2, language: 'css', type: 'meta.tag.html.entity.name.source.css.embedded.selector' },
+					{ offset: 4, language: 'css', type: 'meta.html.source.css.embedded.selector' },
+					{ offset: 5, language: 'css', type: 'meta.html.punctuation.begin.source.css.embedded.property-list.section' },
 				],
-				modeTransitions: [{ startIndex: 0, modeId: 'css' }]
 			}, {
 				line: '\t\t\tcolor: #CCA3A3;',
 				tmTokens: [
@@ -587,15 +540,14 @@ suite('TextMate.decodeTextMateTokens', () => {
 					{ startIndex: 17, endIndex: 18, scopes: ['text.html.basic', 'source.css.embedded.html', 'meta.property-list.css', 'meta.property-value.css', 'punctuation.terminator.rule.css'] }
 				],
 				tokens: [
-					{ startIndex: 0, type: 'meta.html.source.css.embedded.property-list' },
-					{ startIndex: 3, type: 'meta.html.source.css.embedded.property-list.property-name.support.type' },
-					{ startIndex: 8, type: 'meta.html.punctuation.source.css.embedded.property-list.property-value.separator.key-value' },
-					{ startIndex: 9, type: 'meta.html.source.css.embedded.property-list.property-value' },
-					{ startIndex: 10, type: 'meta.html.punctuation.definition.other.source.css.embedded.property-list.property-value.constant.color.rgb-value' },
-					{ startIndex: 11, type: 'meta.html.other.source.css.embedded.property-list.property-value.constant.color.rgb-value' },
-					{ startIndex: 17, type: 'meta.html.punctuation.source.css.embedded.property-list.property-value.terminator.rule' },
+					{ offset: 0, language: 'css', type: 'meta.html.source.css.embedded.property-list' },
+					{ offset: 3, language: 'css', type: 'meta.html.source.css.embedded.property-list.property-name.support.type' },
+					{ offset: 8, language: 'css', type: 'meta.html.punctuation.source.css.embedded.property-list.property-value.separator.key-value' },
+					{ offset: 9, language: 'css', type: 'meta.html.source.css.embedded.property-list.property-value' },
+					{ offset: 10, language: 'css', type: 'meta.html.punctuation.definition.other.source.css.embedded.property-list.property-value.constant.color.rgb-value' },
+					{ offset: 11, language: 'css', type: 'meta.html.other.source.css.embedded.property-list.property-value.constant.color.rgb-value' },
+					{ offset: 17, language: 'css', type: 'meta.html.punctuation.source.css.embedded.property-list.property-value.terminator.rule' },
 				],
-				modeTransitions: [{ startIndex: 0, modeId: 'css' }]
 			}, {
 				line: '\t\t}',
 				tmTokens: [
@@ -603,10 +555,9 @@ suite('TextMate.decodeTextMateTokens', () => {
 					{ startIndex: 2, endIndex: 3, scopes: ['text.html.basic', 'source.css.embedded.html', 'meta.property-list.css', 'punctuation.section.property-list.end.css'] }
 				],
 				tokens: [
-					{ startIndex: 0, type: 'meta.html.source.css.embedded.property-list' },
-					{ startIndex: 2, type: 'meta.html.punctuation.end.source.css.embedded.property-list.section' },
+					{ offset: 0, language: 'css', type: 'meta.html.source.css.embedded.property-list' },
+					{ offset: 2, language: 'css', type: 'meta.html.punctuation.end.source.css.embedded.property-list.section' },
 				],
-				modeTransitions: [{ startIndex: 0, modeId: 'css' }]
 			}, {
 				line: '\t</style>',
 				tmTokens: [
@@ -616,12 +567,11 @@ suite('TextMate.decodeTextMateTokens', () => {
 					{ startIndex: 8, endIndex: 9, scopes: ['text.html.basic', 'source.css.embedded.html', 'punctuation.definition.tag.html'] }
 				],
 				tokens: [
-					{ startIndex: 0, type: 'html.source.css.embedded' },
-					{ startIndex: 1, type: 'tag.html.punctuation.definition.source.css.embedded' },
-					{ startIndex: 3, type: 'tag.html.entity.name.source.css.embedded.style' },
-					{ startIndex: 8, type: 'tag.html.punctuation.definition.source.css.embedded' },
+					{ offset: 0, language: 'css', type: 'html.source.css.embedded' },
+					{ offset: 1, language: 'css', type: 'tag.html.punctuation.definition.source.css.embedded' },
+					{ offset: 3, language: 'css', type: 'tag.html.entity.name.source.css.embedded.style' },
+					{ offset: 8, language: 'css', type: 'tag.html.punctuation.definition.source.css.embedded' },
 				],
-				modeTransitions: [{ startIndex: 0, modeId: 'css' }]
 			}, {
 				line: '\t<script type=\"text/javascript\">',
 				tmTokens: [
@@ -637,18 +587,17 @@ suite('TextMate.decodeTextMateTokens', () => {
 					{ startIndex: 31, endIndex: 32, scopes: ['text.html.basic', 'source.js.embedded.html', 'punctuation.definition.tag.html'] }
 				],
 				tokens: [
-					{ startIndex: 0, type: 'html.source.embedded.js' },
-					{ startIndex: 1, type: 'tag.html.punctuation.definition.source.embedded.js' },
-					{ startIndex: 2, type: 'tag.html.entity.name.source.embedded.js.script' },
-					{ startIndex: 8, type: 'html.source.embedded.js' },
-					{ startIndex: 9, type: 'html.entity.other.attribute-name.source.embedded.js' },
-					{ startIndex: 13, type: 'html.source.embedded.js' },
-					{ startIndex: 14, type: 'html.punctuation.definition.begin.string.quoted.double.source.embedded.js' },
-					{ startIndex: 15, type: 'html.string.quoted.double.source.embedded.js' },
-					{ startIndex: 30, type: 'html.punctuation.definition.end.string.quoted.double.source.embedded.js' },
-					{ startIndex: 31, type: 'tag.html.punctuation.definition.source.embedded.js' },
+					{ offset: 0, language: 'javascript', type: 'html.source.embedded.js' },
+					{ offset: 1, language: 'javascript', type: 'tag.html.punctuation.definition.source.embedded.js' },
+					{ offset: 2, language: 'javascript', type: 'tag.html.entity.name.source.embedded.js.script' },
+					{ offset: 8, language: 'javascript', type: 'html.source.embedded.js' },
+					{ offset: 9, language: 'javascript', type: 'html.entity.other.attribute-name.source.embedded.js' },
+					{ offset: 13, language: 'javascript', type: 'html.source.embedded.js' },
+					{ offset: 14, language: 'javascript', type: 'html.punctuation.definition.begin.string.quoted.double.source.embedded.js' },
+					{ offset: 15, language: 'javascript', type: 'html.string.quoted.double.source.embedded.js' },
+					{ offset: 30, language: 'javascript', type: 'html.punctuation.definition.end.string.quoted.double.source.embedded.js' },
+					{ offset: 31, language: 'javascript', type: 'tag.html.punctuation.definition.source.embedded.js' },
 				],
-				modeTransitions: [{ startIndex: 0, modeId: 'javascript' }]
 			}, {
 				line: '\t\twindow.alert(\"I am a sample...\");',
 				tmTokens: [
@@ -664,18 +613,17 @@ suite('TextMate.decodeTextMateTokens', () => {
 					{ startIndex: 34, endIndex: 35, scopes: ['text.html.basic', 'source.js.embedded.html', 'punctuation.terminator.statement.js'] }
 				],
 				tokens: [
-					{ startIndex: 0, type: 'html.source.embedded.js' },
-					{ startIndex: 2, type: 'html.source.embedded.support.js.variable.dom' },
-					{ startIndex: 8, type: 'html.punctuation.source.embedded.js.accessor' },
-					{ startIndex: 9, type: 'html.source.embedded.support.js.function' },
-					{ startIndex: 14, type: 'meta.html.source.embedded.js.brace.round' },
-					{ startIndex: 15, type: 'html.punctuation.definition.begin.string.quoted.double.source.embedded.js' },
-					{ startIndex: 16, type: 'html.string.quoted.double.source.embedded.js' },
-					{ startIndex: 32, type: 'html.punctuation.definition.end.string.quoted.double.source.embedded.js' },
-					{ startIndex: 33, type: 'meta.html.source.embedded.js.brace.round' },
-					{ startIndex: 34, type: 'html.punctuation.source.embedded.terminator.js.statement' },
+					{ offset: 0, language: 'javascript', type: 'html.source.embedded.js' },
+					{ offset: 2, language: 'javascript', type: 'html.source.embedded.support.js.variable.dom' },
+					{ offset: 8, language: 'javascript', type: 'html.punctuation.source.embedded.js.accessor' },
+					{ offset: 9, language: 'javascript', type: 'html.source.embedded.support.js.function' },
+					{ offset: 14, language: 'javascript', type: 'meta.html.source.embedded.js.brace.round' },
+					{ offset: 15, language: 'javascript', type: 'html.punctuation.definition.begin.string.quoted.double.source.embedded.js' },
+					{ offset: 16, language: 'javascript', type: 'html.string.quoted.double.source.embedded.js' },
+					{ offset: 32, language: 'javascript', type: 'html.punctuation.definition.end.string.quoted.double.source.embedded.js' },
+					{ offset: 33, language: 'javascript', type: 'meta.html.source.embedded.js.brace.round' },
+					{ offset: 34, language: 'javascript', type: 'html.punctuation.source.embedded.terminator.js.statement' },
 				],
-				modeTransitions: [{ startIndex: 0, modeId: 'javascript' }]
 			}, {
 				line: '\t</script>After',
 				tmTokens: [
@@ -686,13 +634,12 @@ suite('TextMate.decodeTextMateTokens', () => {
 					{ startIndex: 10, endIndex: 15, scopes: ['text.html.basic'] }
 				],
 				tokens: [
-					{ startIndex: 0, type: 'html.source.embedded.js' },
-					{ startIndex: 1, type: 'tag.html.punctuation.definition.source.embedded.js' },
-					{ startIndex: 3, type: 'tag.html.entity.name.source.embedded.js.script' },
-					{ startIndex: 9, type: 'tag.html.punctuation.definition.source.embedded.js' },
-					{ startIndex: 10, type: '' },
+					{ offset: 0, language: 'javascript', type: 'html.source.embedded.js' },
+					{ offset: 1, language: 'javascript', type: 'tag.html.punctuation.definition.source.embedded.js' },
+					{ offset: 3, language: 'javascript', type: 'tag.html.entity.name.source.embedded.js.script' },
+					{ offset: 9, language: 'javascript', type: 'tag.html.punctuation.definition.source.embedded.js' },
+					{ offset: 10, language: 'html', type: '' },
 				],
-				modeTransitions: [{ startIndex: 0, modeId: 'javascript' }, { startIndex: 10, modeId: 'html' }]
 			}, {
 				line: '</head>',
 				tmTokens: [
@@ -701,11 +648,10 @@ suite('TextMate.decodeTextMateTokens', () => {
 					{ startIndex: 6, endIndex: 7, scopes: ['text.html.basic', 'meta.tag.structure.any.html', 'punctuation.definition.tag.html'] }
 				],
 				tokens: [
-					{ startIndex: 0, type: 'meta.tag.html.punctuation.definition.structure.any' },
-					{ startIndex: 2, type: 'meta.tag.html.structure.any.entity.name' },
-					{ startIndex: 6, type: 'meta.tag.html.punctuation.definition.structure.any' },
+					{ offset: 0, language: 'html', type: 'meta.tag.html.punctuation.definition.structure.any' },
+					{ offset: 2, language: 'html', type: 'meta.tag.html.structure.any.entity.name' },
+					{ offset: 6, language: 'html', type: 'meta.tag.html.punctuation.definition.structure.any' },
 				],
-				modeTransitions: [{ startIndex: 0, modeId: 'html' }]
 			}, {
 				line: '<body>',
 				tmTokens: [
@@ -714,11 +660,10 @@ suite('TextMate.decodeTextMateTokens', () => {
 					{ startIndex: 5, endIndex: 6, scopes: ['text.html.basic', 'meta.tag.structure.any.html', 'punctuation.definition.tag.html'] }
 				],
 				tokens: [
-					{ startIndex: 0, type: 'meta.tag.html.punctuation.definition.structure.any' },
-					{ startIndex: 1, type: 'meta.tag.html.structure.any.entity.name' },
-					{ startIndex: 5, type: 'meta.tag.html.punctuation.definition.structure.any' },
+					{ offset: 0, language: 'html', type: 'meta.tag.html.punctuation.definition.structure.any' },
+					{ offset: 1, language: 'html', type: 'meta.tag.html.structure.any.entity.name' },
+					{ offset: 5, language: 'html', type: 'meta.tag.html.punctuation.definition.structure.any' },
 				],
-				modeTransitions: [{ startIndex: 0, modeId: 'html' }]
 			}, {
 				line: '\t<h1>Heading No.1</h1>',
 				tmTokens: [
@@ -732,16 +677,15 @@ suite('TextMate.decodeTextMateTokens', () => {
 					{ startIndex: 21, endIndex: 22, scopes: ['text.html.basic', 'meta.tag.block.any.html', 'punctuation.definition.tag.end.html'] }
 				],
 				tokens: [
-					{ startIndex: 0, type: '' },
-					{ startIndex: 1, type: 'meta.tag.html.punctuation.definition.block.any.begin' },
-					{ startIndex: 2, type: 'meta.tag.html.block.any.entity.name' },
-					{ startIndex: 4, type: 'meta.tag.html.punctuation.definition.block.any.end' },
-					{ startIndex: 5, type: '' },
-					{ startIndex: 17, type: 'meta.tag.html.punctuation.definition.block.any.begin' },
-					{ startIndex: 19, type: 'meta.tag.html.block.any.entity.name' },
-					{ startIndex: 21, type: 'meta.tag.html.punctuation.definition.block.any.end' },
+					{ offset: 0, language: 'html', type: '' },
+					{ offset: 1, language: 'html', type: 'meta.tag.html.punctuation.definition.block.any.begin' },
+					{ offset: 2, language: 'html', type: 'meta.tag.html.block.any.entity.name' },
+					{ offset: 4, language: 'html', type: 'meta.tag.html.punctuation.definition.block.any.end' },
+					{ offset: 5, language: 'html', type: '' },
+					{ offset: 17, language: 'html', type: 'meta.tag.html.punctuation.definition.block.any.begin' },
+					{ offset: 19, language: 'html', type: 'meta.tag.html.block.any.entity.name' },
+					{ offset: 21, language: 'html', type: 'meta.tag.html.punctuation.definition.block.any.end' },
 				],
-				modeTransitions: [{ startIndex: 0, modeId: 'html' }]
 			}, {
 				line: '\t<input disabled type=\"button\" value=\"Click me\" />',
 				tmTokens: [
@@ -765,26 +709,25 @@ suite('TextMate.decodeTextMateTokens', () => {
 					{ startIndex: 47, endIndex: 50, scopes: ['text.html.basic', 'meta.tag.inline.any.html', 'punctuation.definition.tag.end.html'] }
 				],
 				tokens: [
-					{ startIndex: 0, type: '' },
-					{ startIndex: 1, type: 'meta.tag.html.punctuation.definition.any.inline.begin' },
-					{ startIndex: 2, type: 'meta.tag.html.any.entity.name.inline' },
-					{ startIndex: 7, type: 'meta.tag.html.any.inline' },
-					{ startIndex: 8, type: 'meta.tag.html.any.entity.inline.other.attribute-name' },
-					{ startIndex: 16, type: 'meta.tag.html.any.inline' },
-					{ startIndex: 17, type: 'meta.tag.html.any.entity.inline.other.attribute-name' },
-					{ startIndex: 21, type: 'meta.tag.html.any.inline' },
-					{ startIndex: 22, type: 'meta.tag.html.punctuation.definition.any.inline.begin.string.quoted.double' },
-					{ startIndex: 23, type: 'meta.tag.html.any.inline.string.quoted.double' },
-					{ startIndex: 29, type: 'meta.tag.html.punctuation.definition.any.inline.end.string.quoted.double' },
-					{ startIndex: 30, type: 'meta.tag.html.any.inline' },
-					{ startIndex: 31, type: 'meta.tag.html.any.entity.inline.other.attribute-name' },
-					{ startIndex: 36, type: 'meta.tag.html.any.inline' },
-					{ startIndex: 37, type: 'meta.tag.html.punctuation.definition.any.inline.begin.string.quoted.double' },
-					{ startIndex: 38, type: 'meta.tag.html.any.inline.string.quoted.double' },
-					{ startIndex: 46, type: 'meta.tag.html.punctuation.definition.any.inline.end.string.quoted.double' },
-					{ startIndex: 47, type: 'meta.tag.html.punctuation.definition.any.inline.end' },
+					{ offset: 0, language: 'html', type: '' },
+					{ offset: 1, language: 'html', type: 'meta.tag.html.punctuation.definition.any.inline.begin' },
+					{ offset: 2, language: 'html', type: 'meta.tag.html.any.entity.name.inline' },
+					{ offset: 7, language: 'html', type: 'meta.tag.html.any.inline' },
+					{ offset: 8, language: 'html', type: 'meta.tag.html.any.entity.inline.other.attribute-name' },
+					{ offset: 16, language: 'html', type: 'meta.tag.html.any.inline' },
+					{ offset: 17, language: 'html', type: 'meta.tag.html.any.entity.inline.other.attribute-name' },
+					{ offset: 21, language: 'html', type: 'meta.tag.html.any.inline' },
+					{ offset: 22, language: 'html', type: 'meta.tag.html.punctuation.definition.any.inline.begin.string.quoted.double' },
+					{ offset: 23, language: 'html', type: 'meta.tag.html.any.inline.string.quoted.double' },
+					{ offset: 29, language: 'html', type: 'meta.tag.html.punctuation.definition.any.inline.end.string.quoted.double' },
+					{ offset: 30, language: 'html', type: 'meta.tag.html.any.inline' },
+					{ offset: 31, language: 'html', type: 'meta.tag.html.any.entity.inline.other.attribute-name' },
+					{ offset: 36, language: 'html', type: 'meta.tag.html.any.inline' },
+					{ offset: 37, language: 'html', type: 'meta.tag.html.punctuation.definition.any.inline.begin.string.quoted.double' },
+					{ offset: 38, language: 'html', type: 'meta.tag.html.any.inline.string.quoted.double' },
+					{ offset: 46, language: 'html', type: 'meta.tag.html.punctuation.definition.any.inline.end.string.quoted.double' },
+					{ offset: 47, language: 'html', type: 'meta.tag.html.punctuation.definition.any.inline.end' },
 				],
-				modeTransitions: [{ startIndex: 0, modeId: 'html' }]
 			}, {
 				line: '</body>',
 				tmTokens: [
@@ -793,11 +736,10 @@ suite('TextMate.decodeTextMateTokens', () => {
 					{ startIndex: 6, endIndex: 7, scopes: ['text.html.basic', 'meta.tag.structure.any.html', 'punctuation.definition.tag.html'] }
 				],
 				tokens: [
-					{ startIndex: 0, type: 'meta.tag.html.punctuation.definition.structure.any' },
-					{ startIndex: 2, type: 'meta.tag.html.structure.any.entity.name' },
-					{ startIndex: 6, type: 'meta.tag.html.punctuation.definition.structure.any' },
+					{ offset: 0, language: 'html', type: 'meta.tag.html.punctuation.definition.structure.any' },
+					{ offset: 2, language: 'html', type: 'meta.tag.html.structure.any.entity.name' },
+					{ offset: 6, language: 'html', type: 'meta.tag.html.punctuation.definition.structure.any' },
 				],
-				modeTransitions: [{ startIndex: 0, modeId: 'html' }]
 			}, {
 				line: '</html>',
 				tmTokens: [
@@ -806,11 +748,10 @@ suite('TextMate.decodeTextMateTokens', () => {
 					{ startIndex: 6, endIndex: 7, scopes: ['text.html.basic', 'meta.tag.structure.any.html', 'punctuation.definition.tag.html'] }
 				],
 				tokens: [
-					{ startIndex: 0, type: 'meta.tag.html.punctuation.definition.structure.any' },
-					{ startIndex: 2, type: 'meta.tag.html.structure.any.entity.name' },
-					{ startIndex: 6, type: 'meta.tag.html.punctuation.definition.structure.any' },
+					{ offset: 0, language: 'html', type: 'meta.tag.html.punctuation.definition.structure.any' },
+					{ offset: 2, language: 'html', type: 'meta.tag.html.structure.any.entity.name' },
+					{ offset: 6, language: 'html', type: 'meta.tag.html.punctuation.definition.structure.any' },
 				],
-				modeTransitions: [{ startIndex: 0, modeId: 'html' }]
 			}
 		];
 
@@ -830,11 +771,7 @@ suite('TextMate.decodeTextMateTokens', () => {
 			let test = tests[i];
 			let actual = decodeTextMateTokens('html', decodeMap, test.line, 0, test.tmTokens, null);
 
-			let actualTokens = actual.tokens.map((t) => { return { startIndex: t.startIndex, type: t.type }; });
-			let actualModeTransitions = actual.modeTransitions.map((t) => { return { startIndex: t.startIndex, modeId: t.modeId }; });
-
-			assert.deepEqual(actualTokens, test.tokens, 'test ' + test.line);
-			assert.deepEqual(actualModeTransitions, test.modeTransitions, 'test ' + test.line);
+			assert.deepEqual(actual.tokens, test.tokens, 'test ' + test.line);
 		}
 	});
 
@@ -847,12 +784,9 @@ suite('TextMate.decodeTextMateTokens', () => {
 					{ startIndex: 6, endIndex: 7, scopes: ['source.css.scss', 'meta.property-list.scss', 'punctuation.section.property-list.begin.bracket.curly.scss'] }
 				],
 				tokens: [
-					{ startIndex: 0, type: '' },
-					{ startIndex: 6, type: 'meta.property-list.scss.punctuation.section.begin.bracket.curly' }
+					{ offset: 0, language: 'scss', type: '' },
+					{ offset: 6, language: 'scss', type: 'meta.property-list.scss.punctuation.section.begin.bracket.curly' }
 				],
-				modeTransitions: [
-					{ startIndex: 0, modeId: 'scss' }
-				]
 			}, {
 				line: '    background: red;',
 				tmTokens: [
@@ -864,27 +798,21 @@ suite('TextMate.decodeTextMateTokens', () => {
 					{ startIndex: 19, endIndex: 20, scopes: ['source.css.scss', 'meta.property-list.scss', 'punctuation.terminator.rule.scss'] }
 				],
 				tokens: [
-					{ startIndex: 0, type: 'meta.property-list.scss' },
-					{ startIndex: 4, type: 'meta.property-list.scss.property-name.support.type' },
-					{ startIndex: 14, type: 'meta.property-list.scss.punctuation.separator.key-value' },
-					{ startIndex: 15, type: 'meta.property-list.scss' },
-					{ startIndex: 16, type: 'meta.property-list.scss.support.property-value.constant.color.w3c-standard-color-name' },
-					{ startIndex: 19, type: 'meta.property-list.scss.punctuation.terminator.rule' }
+					{ offset: 0, language: 'scss', type: 'meta.property-list.scss' },
+					{ offset: 4, language: 'scss', type: 'meta.property-list.scss.property-name.support.type' },
+					{ offset: 14, language: 'scss', type: 'meta.property-list.scss.punctuation.separator.key-value' },
+					{ offset: 15, language: 'scss', type: 'meta.property-list.scss' },
+					{ offset: 16, language: 'scss', type: 'meta.property-list.scss.support.property-value.constant.color.w3c-standard-color-name' },
+					{ offset: 19, language: 'scss', type: 'meta.property-list.scss.punctuation.terminator.rule' }
 				],
-				modeTransitions: [
-					{ startIndex: 0, modeId: 'scss' }
-				]
 			}, {
 				line: '}',
 				tmTokens: [
 					{ startIndex: 0, endIndex: 1, scopes: ['source.css.scss', 'meta.property-list.scss', 'punctuation.section.property-list.end.bracket.curly.scss'] }
 				],
 				tokens: [
-					{ startIndex: 0, type: 'meta.property-list.scss.punctuation.section.bracket.curly.end' }
+					{ offset: 0, language: 'scss', type: 'meta.property-list.scss.punctuation.section.bracket.curly.end' }
 				],
-				modeTransitions: [
-					{ startIndex: 0, modeId: 'scss' }
-				]
 			}
 		];
 
@@ -898,11 +826,7 @@ suite('TextMate.decodeTextMateTokens', () => {
 			let test = tests[i];
 			let actual = decodeTextMateTokens('scss', decodeMap, test.line, 0, test.tmTokens, null);
 
-			let actualTokens = actual.tokens.map((t) => { return { startIndex: t.startIndex, type: t.type }; });
-			let actualModeTransitions = actual.modeTransitions.map((t) => { return { startIndex: t.startIndex, modeId: t.modeId }; });
-
-			assert.deepEqual(actualTokens, test.tokens, 'test ' + test.line);
-			assert.deepEqual(actualModeTransitions, test.modeTransitions, 'test ' + test.line);
+			assert.deepEqual(actual.tokens, test.tokens, 'test ' + test.line);
 		}
 
 	});

src/vs/monaco.d.ts

@@ -751,6 +751,14 @@ declare module monaco {
          */
         RTL = 1,
     }
+    export class Token {
+        _tokenBrand: void;
+        readonly offset: number;
+        readonly type: string;
+        readonly language: string;
+        constructor(offset: number, type: string, language: string);
+        toString(): string;
+    }
 }
 
 declare module monaco.editor {
@@ -853,12 +861,6 @@ declare module monaco.editor {
      */
     export function colorizeModelLine(model: IModel, lineNumber: number, tabSize?: number): string;
 
-    export class Token {
-        readonly offset: number;
-        readonly type: string;
-        constructor(offset: number, type: string);
-    }
-
     /**
      * Tokenize `text` using language `languageId`
      */