diff --git a/build/monaco/monaco.d.ts.recipe b/build/monaco/monaco.d.ts.recipe index eea7f6c323a..6cc637db74a 100644 --- a/build/monaco/monaco.d.ts.recipe +++ b/build/monaco/monaco.d.ts.recipe @@ -48,11 +48,12 @@ declare module monaco { #include(vs/editor/common/core/position): Position #include(vs/editor/common/core/range): Range #include(vs/editor/common/core/selection): Selection, SelectionDirection +#include(vs/editor/common/core/token): Token } declare module monaco.editor { -#includeAll(vs/editor/browser/standalone/standaloneEditor;modes.=>languages.): +#includeAll(vs/editor/browser/standalone/standaloneEditor;modes.=>languages.;editorCommon.=>): #include(vs/editor/common/services/webWorker): MonacoWebWorker, IWebWorkerOptions #include(vs/editor/browser/standalone/standaloneCodeEditor): IEditorConstructionOptions, IDiffEditorConstructionOptions, IStandaloneCodeEditor, IStandaloneDiffEditor export interface ICommandHandler { diff --git a/src/vs/editor/browser/standalone/standaloneEditor.ts b/src/vs/editor/browser/standalone/standaloneEditor.ts index 4172b76b859..776ec46a520 100644 --- a/src/vs/editor/browser/standalone/standaloneEditor.ts +++ b/src/vs/editor/browser/standalone/standaloneEditor.ts @@ -6,9 +6,6 @@ import 'vs/css!./media/standalone-tokens'; import * as editorCommon from 'vs/editor/common/editorCommon'; -/* tslint:disable:duplicate-imports */ -import { IModel } from 'vs/editor/common/editorCommon'; -/* tslint:disable:duplicate-imports */ import { ContentWidgetPositionPreference, OverlayWidgetPositionPreference } from 'vs/editor/browser/editorBrowser'; import { StandaloneEditor, IStandaloneCodeEditor, StandaloneDiffEditor, IStandaloneDiffEditor, IEditorConstructionOptions, IDiffEditorConstructionOptions } from 'vs/editor/browser/standalone/standaloneCodeEditor'; import { ScrollbarVisibility } from 'vs/base/common/scrollable'; @@ -33,9 +30,9 @@ import { IContextKeyService } from 'vs/platform/contextkey/common/contextkey'; import { ICodeEditorService } from 'vs/editor/common/services/codeEditorService'; import { IEditorWorkerService } from 'vs/editor/common/services/editorWorkerService'; import { ITextModelResolverService } from 'vs/editor/common/services/resolverService'; -import { IState, ITokenizationSupport, TokenizationRegistry } from 'vs/editor/common/modes'; import { NULL_STATE, nullTokenize } from 'vs/editor/common/modes/nullMode'; import { IStandaloneColorService } from 'vs/editor/common/services/standaloneColorService'; +import { Token } from 'vs/editor/common/core/token'; /** * @internal @@ -136,7 +133,7 @@ export function createDiffNavigator(diffEditor: IStandaloneDiffEditor, opts?: ID return new DiffNavigator(diffEditor, opts); } -function doCreateModel(value: string, mode: TPromise, uri?: URI): IModel { +function doCreateModel(value: string, mode: TPromise, uri?: URI): editorCommon.IModel { return StaticServices.modelService.get().createModel(value, mode, uri); } @@ -144,7 +141,7 @@ function doCreateModel(value: string, mode: TPromise, uri?: URI): I * Create a new editor model. * You can specify the language that should be set for this model or let the language be inferred from the `uri`. */ -export function createModel(value: string, language?: string, uri?: URI): IModel { +export function createModel(value: string, language?: string, uri?: URI): editorCommon.IModel { value = value || ''; if (!language) { @@ -164,14 +161,14 @@ export function createModel(value: string, language?: string, uri?: URI): IModel /** * Change the language for a model. */ -export function setModelLanguage(model: IModel, language: string): void { +export function setModelLanguage(model: editorCommon.IModel, language: string): void { StaticServices.modelService.get().setMode(model, StaticServices.modeService.get().getOrCreateMode(language)); } /** * Set the markers for a model. */ -export function setModelMarkers(model: IModel, owner: string, markers: IMarkerData[]): void { +export function setModelMarkers(model: editorCommon.IModel, owner: string, markers: IMarkerData[]): void { if (model) { StaticServices.markerService.get().changeOne(owner, model.uri, markers); } @@ -180,14 +177,14 @@ export function setModelMarkers(model: IModel, owner: string, markers: IMarkerDa /** * Get the model that has `uri` if it exists. */ -export function getModel(uri: URI): IModel { +export function getModel(uri: URI): editorCommon.IModel { return StaticServices.modelService.get().getModel(uri); } /** * Get all the created models. */ -export function getModels(): IModel[] { +export function getModels(): editorCommon.IModel[] { return StaticServices.modelService.get().getModels(); } @@ -195,7 +192,7 @@ export function getModels(): IModel[] { * Emitted when a model is created. * @event */ -export function onDidCreateModel(listener: (model: IModel) => void): IDisposable { +export function onDidCreateModel(listener: (model: editorCommon.IModel) => void): IDisposable { return StaticServices.modelService.get().onModelAdded(listener); } @@ -203,7 +200,7 @@ export function onDidCreateModel(listener: (model: IModel) => void): IDisposable * Emitted right before a model is disposed. * @event */ -export function onWillDisposeModel(listener: (model: IModel) => void): IDisposable { +export function onWillDisposeModel(listener: (model: editorCommon.IModel) => void): IDisposable { return StaticServices.modelService.get().onModelRemoved(listener); } @@ -211,7 +208,7 @@ export function onWillDisposeModel(listener: (model: IModel) => void): IDisposab * Emitted when a different language is set to a model. * @event */ -export function onDidChangeModelLanguage(listener: (e: { readonly model: IModel; readonly oldLanguage: string; }) => void): IDisposable { +export function onDidChangeModelLanguage(listener: (e: { readonly model: editorCommon.IModel; readonly oldLanguage: string; }) => void): IDisposable { return StaticServices.modelService.get().onModelModeChanged((e) => { listener({ model: e.model, @@ -255,31 +252,21 @@ export function colorize(text: string, languageId: string, options: IColorizerOp /** * Colorize a line in a model. */ -export function colorizeModelLine(model: IModel, lineNumber: number, tabSize: number = 4): string { +export function colorizeModelLine(model: editorCommon.IModel, lineNumber: number, tabSize: number = 4): string { return Colorizer.colorizeModelLine(model, lineNumber, tabSize); } -export class Token { - public readonly offset: number; - public readonly type: string; - - constructor(offset: number, type: string) { - this.offset = offset; - this.type = type; - } -} - /** * @internal */ -function getSafeTokenizationSupport(languageId: string): ITokenizationSupport { - let tokenizationSupport = TokenizationRegistry.get(languageId); +function getSafeTokenizationSupport(languageId: string): modes.ITokenizationSupport { + let tokenizationSupport = modes.TokenizationRegistry.get(languageId); if (tokenizationSupport) { return tokenizationSupport; } return { getInitialState: () => NULL_STATE, - tokenize: (line: string, state: IState, deltaOffset: number) => nullTokenize(languageId, line, state, deltaOffset), + tokenize: (line: string, state: modes.IState, deltaOffset: number) => nullTokenize(languageId, line, state, deltaOffset), tokenize3: undefined, }; } @@ -300,7 +287,7 @@ export function tokenize(text: string, languageId: string): Token[][] { let line = lines[i]; let tokenizationResult = tokenizationSupport.tokenize(line, state, 0); - result[i] = tokenizationResult.tokens.map((t) => new Token(t.startIndex, t.type)); + result[i] = tokenizationResult.tokens; state = tokenizationResult.endState; } return result; @@ -358,7 +345,6 @@ export function createMonacoEditorAPI(): typeof monaco.editor { BareFontInfo: editorCommon.BareFontInfo, FontInfo: editorCommon.FontInfo, TextModelResolvedOptions: editorCommon.TextModelResolvedOptions, - Token: Token, // vars EditorType: editorCommon.EditorType, diff --git a/src/vs/editor/browser/standalone/standaloneLanguages.ts b/src/vs/editor/browser/standalone/standaloneLanguages.ts index 6abf05a07e4..b42e579b95b 100644 --- a/src/vs/editor/browser/standalone/standaloneLanguages.ts +++ b/src/vs/editor/browser/standalone/standaloneLanguages.ts @@ -24,7 +24,6 @@ import { createTokenizationSupport } from 'vs/editor/common/modes/monarch/monarc import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry'; import { IMarkerData } from 'vs/platform/markers/common/markers'; import { Token } from 'vs/editor/common/core/token'; -import { ModeTransition } from 'vs/editor/common/core/modeTransition'; import { IStandaloneColorService } from 'vs/editor/common/services/standaloneColorService'; /** @@ -90,18 +89,18 @@ export class TokenizationSupport2Adapter implements modes.ITokenizationSupport { return this._actual.getInitialState(); } - private _toClassicTokens(tokens: modes.IToken2[], offsetDelta: number): Token[] { + private _toClassicTokens(tokens: modes.IToken2[], language: string, offsetDelta: number): Token[] { let result: Token[] = []; for (let i = 0, len = tokens.length; i < len; i++) { let t = tokens[i]; - result[i] = new Token(t.startIndex + offsetDelta, t.scopes); + result[i] = new Token(t.startIndex + offsetDelta, t.scopes, language); } return result; } public tokenize(line: string, state: modes.IState, offsetDelta: number): modes.ILineTokens { let actualResult = this._actual.tokenize(line, state); - let tokens = this._toClassicTokens(actualResult.tokens, offsetDelta); + let tokens = this._toClassicTokens(actualResult.tokens, this._languageIdentifier.sid, offsetDelta); let endState: modes.IState; // try to save an object if possible @@ -113,8 +112,7 @@ export class TokenizationSupport2Adapter implements modes.ITokenizationSupport { return { tokens: tokens, - endState: endState, - modeTransitions: [new ModeTransition(offsetDelta, this._languageIdentifier.sid)], + endState: endState }; } diff --git a/src/vs/editor/common/core/modeTransition.ts b/src/vs/editor/common/core/modeTransition.ts deleted file mode 100644 index fc3f4213f93..00000000000 --- a/src/vs/editor/common/core/modeTransition.ts +++ /dev/null @@ -1,23 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ -'use strict'; - -import { Arrays } from 'vs/editor/common/core/arrays'; - -export class ModeTransition { - _modeTransitionBrand: void; - - public readonly startIndex: number; - public readonly modeId: string; - - constructor(startIndex: number, modeId: string) { - this.startIndex = startIndex | 0; - this.modeId = modeId; - } - - public static findIndexInSegmentsArray(arr: ModeTransition[], desiredIndex: number): number { - return Arrays.findIndexInSegmentsArray(arr, desiredIndex); - } -} diff --git a/src/vs/editor/common/core/token.ts b/src/vs/editor/common/core/token.ts index e5685c10bb8..e23278e8d2b 100644 --- a/src/vs/editor/common/core/token.ts +++ b/src/vs/editor/common/core/token.ts @@ -4,45 +4,20 @@ *--------------------------------------------------------------------------------------------*/ 'use strict'; -import { Arrays } from 'vs/editor/common/core/arrays'; - export class Token { _tokenBrand: void; - public readonly startIndex: number; + public readonly offset: number; public readonly type: string; + public readonly language: string; - constructor(startIndex: number, type: string) { - this.startIndex = startIndex | 0;// @perf + constructor(offset: number, type: string, language: string) { + this.offset = offset | 0;// @perf this.type = type; + this.language = language; } public toString(): string { - return '(' + this.startIndex + ', ' + this.type + ')'; - } - - public equals(other: Token): boolean { - return ( - this.startIndex === other.startIndex - && this.type === other.type - ); - } - - public static findIndexInSegmentsArray(arr: Token[], desiredIndex: number): number { - return Arrays.findIndexInSegmentsArray(arr, desiredIndex); - } - - public static equalsArray(a: Token[], b: Token[]): boolean { - let aLen = a.length; - let bLen = b.length; - if (aLen !== bLen) { - return false; - } - for (let i = 0; i < aLen; i++) { - if (!a[i].equals(b[i])) { - return false; - } - } - return true; + return '(' + this.offset + ', ' + this.type + ')'; } } diff --git a/src/vs/editor/common/modes.ts b/src/vs/editor/common/modes.ts index 93a3bc9e3f7..9e4e876f764 100644 --- a/src/vs/editor/common/modes.ts +++ b/src/vs/editor/common/modes.ts @@ -9,7 +9,6 @@ import { IDisposable } from 'vs/base/common/lifecycle'; import URI from 'vs/base/common/uri'; import { IFilter } from 'vs/base/common/filters'; import * as editorCommon from 'vs/editor/common/editorCommon'; -import { ModeTransition } from 'vs/editor/common/core/modeTransition'; import { Token } from 'vs/editor/common/core/token'; import LanguageFeatureRegistry from 'vs/editor/common/modes/languageFeatureRegistry'; import { CancellationToken } from 'vs/base/common/cancellation'; @@ -55,7 +54,6 @@ export interface IMode { export interface ILineTokens { tokens: Token[]; endState: IState; - modeTransitions: ModeTransition[]; } /** diff --git a/src/vs/editor/common/modes/monarch/monarchLexer.ts b/src/vs/editor/common/modes/monarch/monarchLexer.ts index 84f09f1c8e8..0ea192b5d8b 100644 --- a/src/vs/editor/common/modes/monarch/monarchLexer.ts +++ b/src/vs/editor/common/modes/monarch/monarchLexer.ts @@ -15,7 +15,6 @@ import * as monarchCommon from 'vs/editor/common/modes/monarch/monarchCommon'; import { IModeService } from 'vs/editor/common/services/modeService'; import { Token } from 'vs/editor/common/core/token'; import { NULL_STATE, NULL_MODE_ID } from 'vs/editor/common/modes/nullMode'; -import { ModeTransition } from 'vs/editor/common/core/modeTransition'; import { IStandaloneColorService } from 'vs/editor/common/services/standaloneColorService'; import { Theme } from 'vs/editor/common/modes/supports/tokenization'; @@ -240,35 +239,29 @@ interface IMonarchTokensCollector { class MonarchClassicTokensCollector implements IMonarchTokensCollector { - private _modeTransitions: ModeTransition[]; private _tokens: Token[]; - - private _lastModeId: string; + private _language: string; private _lastTokenType: string; + private _lastTokenLanguage: string; constructor() { - this._modeTransitions = []; this._tokens = []; - - this._lastModeId = null; + this._language = null; this._lastTokenType = null; + this._lastTokenLanguage = null; } public enterMode(startOffset: number, modeId: string): void { - if (this._lastModeId === modeId) { - // Avoid transitioning to the same mode (this can happen in case of empty embedded modes) - return; - } - this._lastModeId = modeId; - this._modeTransitions.push(new ModeTransition(startOffset, modeId)); + this._language = modeId; } public emit(startOffset: number, type: string): void { - if (this._lastTokenType === type) { + if (this._lastTokenType === type && this._lastTokenLanguage === this._language) { return; } this._lastTokenType = type; - this._tokens.push(new Token(startOffset, type)); + this._lastTokenLanguage = this._language; + this._tokens.push(new Token(startOffset, type, this._language)); } public nestedModeTokenize(embeddedModeLine: string, embeddedModeData: EmbeddedModeData, offsetDelta: number): modes.IState { @@ -285,15 +278,14 @@ class MonarchClassicTokensCollector implements IMonarchTokensCollector { let nestedResult = nestedModeTokenizationSupport.tokenize(embeddedModeLine, embeddedModeState, offsetDelta); this._tokens = this._tokens.concat(nestedResult.tokens); this._lastTokenType = null; - this._modeTransitions = this._modeTransitions.concat(nestedResult.modeTransitions); - this._lastModeId = null; + this._lastTokenLanguage = null; + this._language = null; return nestedResult.endState; } public finalize(endState: MonarchLineState): modes.ILineTokens { return { tokens: this._tokens, - modeTransitions: this._modeTransitions, endState: endState }; } diff --git a/src/vs/editor/common/modes/nullMode.ts b/src/vs/editor/common/modes/nullMode.ts index 65972a9fb4e..edf0171171a 100644 --- a/src/vs/editor/common/modes/nullMode.ts +++ b/src/vs/editor/common/modes/nullMode.ts @@ -5,7 +5,6 @@ 'use strict'; import { IState, ILineTokens, ILineTokens3, ColorId, MetadataConsts, LanguageIdentifier, FontStyle, StandardTokenType, LanguageId } from 'vs/editor/common/modes'; -import { ModeTransition } from 'vs/editor/common/core/modeTransition'; import { Token } from 'vs/editor/common/core/token'; class NullStateImpl implements IState { @@ -26,14 +25,9 @@ export const NULL_MODE_ID = 'vs.editor.nullMode'; export const NULL_LANGUAGE_IDENTIFIER = new LanguageIdentifier(NULL_MODE_ID, LanguageId.Null); export function nullTokenize(modeId: string, buffer: string, state: IState, deltaOffset: number): ILineTokens { - let tokens: Token[] = [new Token(deltaOffset, '')]; - - let modeTransitions: ModeTransition[] = [new ModeTransition(deltaOffset, modeId)]; - return { - tokens: tokens, - endState: state, - modeTransitions: modeTransitions + tokens: [new Token(deltaOffset, '', modeId)], + endState: state }; } diff --git a/src/vs/editor/common/modes/supports.ts b/src/vs/editor/common/modes/supports.ts index ccd31dd70e7..5ff3361c09d 100644 --- a/src/vs/editor/common/modes/supports.ts +++ b/src/vs/editor/common/modes/supports.ts @@ -5,7 +5,6 @@ 'use strict'; import * as modes from 'vs/editor/common/modes'; -import { ModeTransition } from 'vs/editor/common/core/modeTransition'; import { Token } from 'vs/editor/common/core/token'; import { LineTokens } from 'vs/editor/common/core/lineTokens'; @@ -13,12 +12,10 @@ export class RawLineTokens implements modes.ILineTokens { _lineTokensBrand: void; tokens: Token[]; - modeTransitions: ModeTransition[]; endState: modes.IState; - constructor(tokens: Token[], modeTransitions: ModeTransition[], endState: modes.IState) { + constructor(tokens: Token[], endState: modes.IState) { this.tokens = tokens; - this.modeTransitions = modeTransitions; this.endState = endState; } } diff --git a/src/vs/editor/common/standalone/standaloneBase.ts b/src/vs/editor/common/standalone/standaloneBase.ts index 3e169265ae5..4a737410439 100644 --- a/src/vs/editor/common/standalone/standaloneBase.ts +++ b/src/vs/editor/common/standalone/standaloneBase.ts @@ -11,6 +11,7 @@ import { Range } from 'vs/editor/common/core/range'; import { Selection, SelectionDirection } from 'vs/editor/common/core/selection'; import { TPromise } from 'vs/base/common/winjs.base'; import { CancellationTokenSource } from 'vs/base/common/cancellation'; +import { Token } from 'vs/editor/common/core/token'; import URI from 'vs/base/common/uri'; // -------------------------------------------- @@ -230,6 +231,7 @@ export function createMonacoBaseAPI(): typeof monaco { SelectionDirection: SelectionDirection, Severity: Severity, Promise: TPromise, - Uri: URI + Uri: URI, + Token: Token }; } diff --git a/src/vs/editor/node/textMate/TMSyntax.ts b/src/vs/editor/node/textMate/TMSyntax.ts index d717fcb5272..a54593c422f 100644 --- a/src/vs/editor/node/textMate/TMSyntax.ts +++ b/src/vs/editor/node/textMate/TMSyntax.ts @@ -16,7 +16,6 @@ import { TMState } from 'vs/editor/node/textMate/TMState'; import { RawLineTokens } from 'vs/editor/common/modes/supports'; import { IModeService } from 'vs/editor/common/services/modeService'; import { IGrammar, Registry, StackElement, IToken, IEmbeddedLanguagesMap as IEmbeddedLanguagesMap2 } from 'vscode-textmate'; -import { ModeTransition } from 'vs/editor/common/core/modeTransition'; import { Token } from 'vs/editor/common/core/token'; import { languagesExtPoint } from 'vs/editor/common/services/modeServiceImpl'; import { IThemeService } from 'vs/workbench/services/themes/common/themeService'; @@ -530,8 +529,7 @@ class Tokenizer { if (line.length >= 20000) { console.log(`Line (${line.substr(0, 15)}...): longer than 20k characters, tokenization skipped.`); return new RawLineTokens( - [new Token(offsetDelta, '')], - [new ModeTransition(offsetDelta, this._modeId)], + [new Token(offsetDelta, '', this._modeId)], state ); } @@ -544,8 +542,7 @@ class Tokenizer { this._stackOverflowReported = true; } return new RawLineTokens( - [new Token(offsetDelta, '')], - [new ModeTransition(offsetDelta, this._modeId)], + [new Token(offsetDelta, '', this._modeId)], state ); } @@ -581,10 +578,9 @@ export function decodeTextMateTokens(topLevelModeId: string, decodeMap: DecodeMa // Create the result early and fill in the tokens later let tokens: Token[] = []; - let modeTransitions: ModeTransition[] = []; let lastTokenType: string = null; - let lastModeId: string = null; + let lastTokenLanguage: string = null; for (let tokenIndex = 0, len = resultTokens.length; tokenIndex < len; tokenIndex++) { let token = resultTokens[tokenIndex]; @@ -601,20 +597,15 @@ export function decodeTextMateTokens(topLevelModeId: string, decodeMap: DecodeMa } // do not push a new token if the type is exactly the same (also helps with ligatures) - if (tokenType !== lastTokenType) { - tokens.push(new Token(tokenStartIndex + offsetDelta, tokenType)); + if (tokenType !== lastTokenType || tokenModeId !== lastTokenLanguage) { + tokens.push(new Token(tokenStartIndex + offsetDelta, tokenType, tokenModeId)); lastTokenType = tokenType; - } - - if (tokenModeId !== lastModeId) { - modeTransitions.push(new ModeTransition(tokenStartIndex + offsetDelta, tokenModeId)); - lastModeId = tokenModeId; + lastTokenLanguage = tokenModeId; } } return new RawLineTokens( tokens, - modeTransitions, resultState ); } diff --git a/src/vs/editor/test/node/textMate/TMSyntax.test.ts b/src/vs/editor/test/node/textMate/TMSyntax.test.ts index 30a699725d5..83e40f03e1c 100644 --- a/src/vs/editor/test/node/textMate/TMSyntax.test.ts +++ b/src/vs/editor/test/node/textMate/TMSyntax.test.ts @@ -104,14 +104,15 @@ suite('TextMate.decodeTextMateTokens', () => { null ); - let actualModeTransitions = actual.modeTransitions.map((t) => { return { startIndex: t.startIndex, modeId: t.modeId }; }); - - assert.deepEqual(actualModeTransitions, [ - { startIndex: 0, modeId: 'html' }, - { startIndex: 11, modeId: 'css' }, - { startIndex: 17, modeId: 'html' }, - { startIndex: 33, modeId: 'javascript' }, - { startIndex: 41, modeId: 'html' }, + assert.deepEqual(actual.tokens, [ + { offset: 0, language: 'html', type: '' }, + { offset: 4, language: 'html', type: 'style.tag.open' }, + { offset: 11, language: 'css', type: 'source.css' }, + { offset: 17, language: 'html', type: 'style.tag.close' }, + { offset: 25, language: 'html', type: 'tag.open.script' }, + { offset: 33, language: 'javascript', type: 'source.js' }, + { offset: 41, language: 'html', type: 'tag.close.script' }, + { offset: 50, language: 'html', type: '' }, ]); }); @@ -129,16 +130,13 @@ suite('TextMate.decodeTextMateTokens', () => { { startIndex: 10, endIndex: 11, scopes: ['text.html.php', 'meta.tag.any.html', 'punctuation.definition.tag.html'] } ], tokens: [ - { startIndex: 0, type: 'meta.tag.any.html.punctuation.definition' }, - { startIndex: 1, type: 'meta.tag.any.html.entity.name' }, - { startIndex: 4, type: 'meta.tag.any.html.punctuation.definition' }, - { startIndex: 5, type: 'meta.tag.any.html.punctuation.definition.scope.between-tag-pair' }, - { startIndex: 6, type: 'meta.tag.any.html.punctuation.definition' }, - { startIndex: 7, type: 'meta.tag.any.html.entity.name' }, - { startIndex: 10, type: 'meta.tag.any.html.punctuation.definition' } - ], - modeTransitions: [ - { startIndex: 0, modeId: 'html' } + { offset: 0, language: 'html', type: 'meta.tag.any.html.punctuation.definition' }, + { offset: 1, language: 'html', type: 'meta.tag.any.html.entity.name' }, + { offset: 4, language: 'html', type: 'meta.tag.any.html.punctuation.definition' }, + { offset: 5, language: 'html', type: 'meta.tag.any.html.punctuation.definition.scope.between-tag-pair' }, + { offset: 6, language: 'html', type: 'meta.tag.any.html.punctuation.definition' }, + { offset: 7, language: 'html', type: 'meta.tag.any.html.entity.name' }, + { offset: 10, language: 'html', type: 'meta.tag.any.html.punctuation.definition' } ] }, { line: '', @@ -159,25 +157,20 @@ suite('TextMate.decodeTextMateTokens', () => { { startIndex: 26, endIndex: 27, scopes: ['text.html.php', 'punctuation.definition.tag.html'] } ], tokens: [ - { startIndex: 0, type: 'tag.html.punctuation.definition' }, - { startIndex: 1, type: 'tag.html.entity.name.script' }, - { startIndex: 7, type: 'tag.html.punctuation.definition.source.js.embedded' }, - { startIndex: 8, type: 'meta.html.source.js.embedded.var.expr.storage.type' }, - { startIndex: 11, type: 'meta.html.source.js.embedded.var.expr' }, - { startIndex: 12, type: 'meta.html.source.js.embedded.var.expr.var-single-variable.variable.other.readwrite' }, - { startIndex: 13, type: 'meta.html.source.js.embedded.var.expr.var-single-variable' }, - { startIndex: 14, type: 'meta.html.source.js.embedded.var.expr.keyword.operator.assignment' }, - { startIndex: 15, type: 'meta.html.source.js.embedded.var.expr' }, - { startIndex: 16, type: 'meta.html.source.js.embedded.var.expr.constant.numeric.decimal' }, - { startIndex: 17, type: 'html.punctuation.source.js.embedded.terminator.statement' }, - { startIndex: 18, type: 'tag.html.punctuation.definition.source.js.embedded' }, - { startIndex: 20, type: 'tag.html.entity.name.script.source.js.embedded' }, - { startIndex: 26, type: 'tag.html.punctuation.definition' } - ], - modeTransitions: [ - { startIndex: 0, modeId: 'html' }, - { startIndex: 7, modeId: 'javascript' }, - { startIndex: 26, modeId: 'html' } + { offset: 0, language: 'html', type: 'tag.html.punctuation.definition' }, + { offset: 1, language: 'html', type: 'tag.html.entity.name.script' }, + { offset: 7, language: 'javascript', type: 'tag.html.punctuation.definition.source.js.embedded' }, + { offset: 8, language: 'javascript', type: 'meta.html.source.js.embedded.var.expr.storage.type' }, + { offset: 11, language: 'javascript', type: 'meta.html.source.js.embedded.var.expr' }, + { offset: 12, language: 'javascript', type: 'meta.html.source.js.embedded.var.expr.var-single-variable.variable.other.readwrite' }, + { offset: 13, language: 'javascript', type: 'meta.html.source.js.embedded.var.expr.var-single-variable' }, + { offset: 14, language: 'javascript', type: 'meta.html.source.js.embedded.var.expr.keyword.operator.assignment' }, + { offset: 15, language: 'javascript', type: 'meta.html.source.js.embedded.var.expr' }, + { offset: 16, language: 'javascript', type: 'meta.html.source.js.embedded.var.expr.constant.numeric.decimal' }, + { offset: 17, language: 'javascript', type: 'html.punctuation.source.js.embedded.terminator.statement' }, + { offset: 18, language: 'javascript', type: 'tag.html.punctuation.definition.source.js.embedded' }, + { offset: 20, language: 'javascript', type: 'tag.html.entity.name.script.source.js.embedded' }, + { offset: 26, language: 'html', type: 'tag.html.punctuation.definition' } ] }, { line: '', @@ -197,24 +190,19 @@ suite('TextMate.decodeTextMateTokens', () => { { startIndex: 41, endIndex: 42, scopes: ['text.html.php', 'punctuation.definition.tag.html'] } ], tokens: [ - { startIndex: 0, type: 'tag.html.punctuation.definition' }, - { startIndex: 1, type: 'tag.html.entity.name.style' }, - { startIndex: 6, type: 'tag.html.punctuation.definition.source.embedded.css' }, - { startIndex: 7, type: 'meta.tag.html.entity.name.source.embedded.css.selector' }, - { startIndex: 11, type: 'meta.html.punctuation.source.embedded.css.property-list.section.begin' }, - { startIndex: 12, type: 'meta.html.source.embedded.type.css.property-list.property-name.support' }, - { startIndex: 28, type: 'meta.html.punctuation.source.embedded.css.property-list.property-value.separator.key-value' }, - { startIndex: 29, type: 'meta.html.source.embedded.constant.css.property-list.support.property-value.color.w3c-standard-color-name' }, - { startIndex: 32, type: 'meta.html.punctuation.source.embedded.terminator.css.property-list.property-value.rule' }, - { startIndex: 33, type: 'meta.html.punctuation.source.embedded.css.property-list.section.end' }, - { startIndex: 34, type: 'tag.html.punctuation.definition' }, - { startIndex: 36, type: 'tag.html.entity.name.style' }, - { startIndex: 41, type: 'tag.html.punctuation.definition' } - ], - modeTransitions: [ - { startIndex: 0, modeId: 'html' }, - { startIndex: 6, modeId: 'css' }, - { startIndex: 34, modeId: 'html' } + { offset: 0, language: 'html', type: 'tag.html.punctuation.definition' }, + { offset: 1, language: 'html', type: 'tag.html.entity.name.style' }, + { offset: 6, language: 'css', type: 'tag.html.punctuation.definition.source.embedded.css' }, + { offset: 7, language: 'css', type: 'meta.tag.html.entity.name.source.embedded.css.selector' }, + { offset: 11, language: 'css', type: 'meta.html.punctuation.source.embedded.css.property-list.section.begin' }, + { offset: 12, language: 'css', type: 'meta.html.source.embedded.type.css.property-list.property-name.support' }, + { offset: 28, language: 'css', type: 'meta.html.punctuation.source.embedded.css.property-list.property-value.separator.key-value' }, + { offset: 29, language: 'css', type: 'meta.html.source.embedded.constant.css.property-list.support.property-value.color.w3c-standard-color-name' }, + { offset: 32, language: 'css', type: 'meta.html.punctuation.source.embedded.terminator.css.property-list.property-value.rule' }, + { offset: 33, language: 'css', type: 'meta.html.punctuation.source.embedded.css.property-list.section.end' }, + { offset: 34, language: 'html', type: 'tag.html.punctuation.definition' }, + { offset: 36, language: 'html', type: 'tag.html.entity.name.style' }, + { offset: 41, language: 'html', type: 'tag.html.punctuation.definition' } ] }, { line: ' { { startIndex: 0, endIndex: 5, scopes: ['text.html.php', 'meta.embedded.block.php', 'punctuation.section.embedded.metatag.begin.php'] } ], tokens: [ - { startIndex: 0, type: 'meta.punctuation.embedded.section.begin.block.php.metatag' } - ], - modeTransitions: [ - { startIndex: 0, modeId: 'html' } + { offset: 0, language: 'html', type: 'meta.punctuation.embedded.section.begin.block.php.metatag' } ] }, { line: '$query = \"SELECT col1, col2 FROM db; -- selects from sql\"; ', @@ -246,25 +231,20 @@ suite('TextMate.decodeTextMateTokens', () => { { startIndex: 58, endIndex: 60, scopes: ['text.html.php', 'meta.embedded.block.php', 'source.php'] } ], tokens: [ - { startIndex: 0, type: 'meta.punctuation.definition.source.embedded.variable.other.block.php' }, - { startIndex: 1, type: 'meta.source.embedded.variable.other.block.php' }, - { startIndex: 6, type: 'meta.source.embedded.block.php' }, - { startIndex: 7, type: 'meta.source.embedded.keyword.operator.assignment.block.php' }, - { startIndex: 8, type: 'meta.source.embedded.block.php' }, - { startIndex: 9, type: 'meta.punctuation.definition.source.embedded.begin.block.php.string.quoted.double.sql' }, - { startIndex: 10, type: 'meta.source.embedded.other.keyword.block.php.string.quoted.double.sql.DML' }, - { startIndex: 16, type: 'meta.source.embedded.block.php.string.quoted.double.sql' }, - { startIndex: 28, type: 'meta.source.embedded.other.keyword.block.php.string.quoted.double.sql.DML' }, - { startIndex: 32, type: 'meta.source.embedded.block.php.string.quoted.double.sql' }, - { startIndex: 37, type: 'meta.source.embedded.block.php.string.quoted.double.sql.comment.line.double-dash' }, - { startIndex: 56, type: 'meta.punctuation.definition.source.embedded.end.block.php.string.quoted.double.sql' }, - { startIndex: 57, type: 'meta.punctuation.source.embedded.terminator.block.php.expression' }, - { startIndex: 58, type: 'meta.source.embedded.block.php' } - ], - modeTransitions: [ - { startIndex: 0, modeId: 'php' }, - { startIndex: 10, modeId: 'sql' }, - { startIndex: 56, modeId: 'php' } + { offset: 0, language: 'php', type: 'meta.punctuation.definition.source.embedded.variable.other.block.php' }, + { offset: 1, language: 'php', type: 'meta.source.embedded.variable.other.block.php' }, + { offset: 6, language: 'php', type: 'meta.source.embedded.block.php' }, + { offset: 7, language: 'php', type: 'meta.source.embedded.keyword.operator.assignment.block.php' }, + { offset: 8, language: 'php', type: 'meta.source.embedded.block.php' }, + { offset: 9, language: 'php', type: 'meta.punctuation.definition.source.embedded.begin.block.php.string.quoted.double.sql' }, + { offset: 10, language: 'sql', type: 'meta.source.embedded.other.keyword.block.php.string.quoted.double.sql.DML' }, + { offset: 16, language: 'sql', type: 'meta.source.embedded.block.php.string.quoted.double.sql' }, + { offset: 28, language: 'sql', type: 'meta.source.embedded.other.keyword.block.php.string.quoted.double.sql.DML' }, + { offset: 32, language: 'sql', type: 'meta.source.embedded.block.php.string.quoted.double.sql' }, + { offset: 37, language: 'sql', type: 'meta.source.embedded.block.php.string.quoted.double.sql.comment.line.double-dash' }, + { offset: 56, language: 'php', type: 'meta.punctuation.definition.source.embedded.end.block.php.string.quoted.double.sql' }, + { offset: 57, language: 'php', type: 'meta.punctuation.source.embedded.terminator.block.php.expression' }, + { offset: 58, language: 'php', type: 'meta.source.embedded.block.php' } ] }, { line: '$a = << { { startIndex: 30, endIndex: 31, scopes: ['text.html.php', 'meta.embedded.block.php', 'source.php', 'punctuation.terminator.expression.php'] } ], tokens: [ - { startIndex: 0, type: 'meta.punctuation.definition.source.embedded.variable.other.block.php' }, - { startIndex: 1, type: 'meta.source.embedded.variable.other.block.php' }, - { startIndex: 2, type: 'meta.source.embedded.block.php' }, - { startIndex: 3, type: 'meta.source.embedded.keyword.operator.assignment.block.php' }, - { startIndex: 4, type: 'meta.source.embedded.block.php' }, - { startIndex: 5, type: 'meta.source.embedded.keyword.operator.block.php.comparison' }, - { startIndex: 8, type: 'meta.source.embedded.other.constant.block.php' }, - { startIndex: 12, type: 'meta.source.embedded.block.php' }, - { startIndex: 13, type: 'meta.punctuation.scope.source.embedded.section.begin.block.php' }, - { startIndex: 14, type: 'meta.source.embedded.block.php' }, - { startIndex: 15, type: 'meta.punctuation.definition.source.embedded.begin.block.php.string.quoted.double' }, - { startIndex: 16, type: 'meta.source.embedded.block.php.string.quoted.double.string-contents' }, - { startIndex: 17, type: 'meta.punctuation.definition.source.embedded.end.block.php.string.quoted.double' }, - { startIndex: 18, type: 'meta.source.embedded.block.php' }, - { startIndex: 20, type: 'meta.punctuation.definition.source.embedded.begin.block.php.string.quoted.double' }, - { startIndex: 21, type: 'meta.source.embedded.block.php.string.quoted.double.string-contents' }, - { startIndex: 22, type: 'meta.punctuation.definition.source.embedded.end.block.php.string.quoted.double' }, - { startIndex: 23, type: 'meta.source.embedded.block.php' }, - { startIndex: 24, type: 'meta.punctuation.scope.source.embedded.section.end.block.php' }, - { startIndex: 25, type: 'meta.source.embedded.block.php' }, - { startIndex: 26, type: 'meta.source.embedded.other.constant.block.php' }, - { startIndex: 30, type: 'meta.punctuation.source.embedded.terminator.block.php.expression' } - ], - modeTransitions: [ - { startIndex: 0, modeId: 'php' } + { offset: 0, language: 'php', type: 'meta.punctuation.definition.source.embedded.variable.other.block.php' }, + { offset: 1, language: 'php', type: 'meta.source.embedded.variable.other.block.php' }, + { offset: 2, language: 'php', type: 'meta.source.embedded.block.php' }, + { offset: 3, language: 'php', type: 'meta.source.embedded.keyword.operator.assignment.block.php' }, + { offset: 4, language: 'php', type: 'meta.source.embedded.block.php' }, + { offset: 5, language: 'php', type: 'meta.source.embedded.keyword.operator.block.php.comparison' }, + { offset: 8, language: 'php', type: 'meta.source.embedded.other.constant.block.php' }, + { offset: 12, language: 'php', type: 'meta.source.embedded.block.php' }, + { offset: 13, language: 'php', type: 'meta.punctuation.scope.source.embedded.section.begin.block.php' }, + { offset: 14, language: 'php', type: 'meta.source.embedded.block.php' }, + { offset: 15, language: 'php', type: 'meta.punctuation.definition.source.embedded.begin.block.php.string.quoted.double' }, + { offset: 16, language: 'php', type: 'meta.source.embedded.block.php.string.quoted.double.string-contents' }, + { offset: 17, language: 'php', type: 'meta.punctuation.definition.source.embedded.end.block.php.string.quoted.double' }, + { offset: 18, language: 'php', type: 'meta.source.embedded.block.php' }, + { offset: 20, language: 'php', type: 'meta.punctuation.definition.source.embedded.begin.block.php.string.quoted.double' }, + { offset: 21, language: 'php', type: 'meta.source.embedded.block.php.string.quoted.double.string-contents' }, + { offset: 22, language: 'php', type: 'meta.punctuation.definition.source.embedded.end.block.php.string.quoted.double' }, + { offset: 23, language: 'php', type: 'meta.source.embedded.block.php' }, + { offset: 24, language: 'php', type: 'meta.punctuation.scope.source.embedded.section.end.block.php' }, + { offset: 25, language: 'php', type: 'meta.source.embedded.block.php' }, + { offset: 26, language: 'php', type: 'meta.source.embedded.other.constant.block.php' }, + { offset: 30, language: 'php', type: 'meta.punctuation.source.embedded.terminator.block.php.expression' } ] }, { line: '?>', @@ -328,12 +305,8 @@ suite('TextMate.decodeTextMateTokens', () => { { startIndex: 1, endIndex: 2, scopes: ['text.html.php', 'meta.embedded.block.php', 'punctuation.section.embedded.metatag.end.php'] } ], tokens: [ - { startIndex: 0, type: 'meta.punctuation.source.embedded.section.end.block.php.metatag' }, - { startIndex: 1, type: 'meta.punctuation.embedded.section.end.block.php.metatag' } - ], - modeTransitions: [ - { startIndex: 0, modeId: 'php' }, - { startIndex: 1, modeId: 'html' } + { offset: 0, language: 'php', type: 'meta.punctuation.source.embedded.section.end.block.php.metatag' }, + { offset: 1, language: 'html', type: 'meta.punctuation.embedded.section.end.block.php.metatag' } ] }, { line: '
', @@ -352,23 +325,18 @@ suite('TextMate.decodeTextMateTokens', () => { { startIndex: 26, endIndex: 27, scopes: ['text.html.php', 'meta.tag.block.any.html', 'punctuation.definition.tag.end.html'] } ], tokens: [ - { startIndex: 0, type: 'meta.tag.any.html.punctuation.definition.begin.block' }, - { startIndex: 1, type: 'meta.tag.any.html.entity.name.block' }, - { startIndex: 4, type: 'meta.tag.any.html.punctuation.definition.end.block' }, - { startIndex: 5, type: 'meta.punctuation.embedded.section.begin.php.line' }, - { startIndex: 8, type: 'meta.punctuation.definition.source.embedded.begin.php.string.quoted.double.line' }, - { startIndex: 9, type: 'meta.source.embedded.php.string.quoted.double.line.string-contents' }, - { startIndex: 18, type: 'meta.punctuation.definition.source.embedded.end.php.string.quoted.double.line' }, - { startIndex: 19, type: 'meta.punctuation.source.embedded.section.end.php.line' }, - { startIndex: 20, type: 'meta.punctuation.embedded.section.end.php.line' }, - { startIndex: 21, type: 'meta.tag.any.html.punctuation.definition.begin.block' }, - { startIndex: 23, type: 'meta.tag.any.html.entity.name.block' }, - { startIndex: 26, type: 'meta.tag.any.html.punctuation.definition.end.block' } - ], - modeTransitions: [ - { startIndex: 0, modeId: 'html' }, - { startIndex: 8, modeId: 'php' }, - { startIndex: 20, modeId: 'html' } + { offset: 0, language: 'html', type: 'meta.tag.any.html.punctuation.definition.begin.block' }, + { offset: 1, language: 'html', type: 'meta.tag.any.html.entity.name.block' }, + { offset: 4, language: 'html', type: 'meta.tag.any.html.punctuation.definition.end.block' }, + { offset: 5, language: 'html', type: 'meta.punctuation.embedded.section.begin.php.line' }, + { offset: 8, language: 'php', type: 'meta.punctuation.definition.source.embedded.begin.php.string.quoted.double.line' }, + { offset: 9, language: 'php', type: 'meta.source.embedded.php.string.quoted.double.line.string-contents' }, + { offset: 18, language: 'php', type: 'meta.punctuation.definition.source.embedded.end.php.string.quoted.double.line' }, + { offset: 19, language: 'php', type: 'meta.punctuation.source.embedded.section.end.php.line' }, + { offset: 20, language: 'html', type: 'meta.punctuation.embedded.section.end.php.line' }, + { offset: 21, language: 'html', type: 'meta.tag.any.html.punctuation.definition.begin.block' }, + { offset: 23, language: 'html', type: 'meta.tag.any.html.entity.name.block' }, + { offset: 26, language: 'html', type: 'meta.tag.any.html.punctuation.definition.end.block' } ] } ]; @@ -390,12 +358,7 @@ suite('TextMate.decodeTextMateTokens', () => { for (let i = 0, len = tests.length; i < len; i++) { let test = tests[i]; let actual = decodeTextMateTokens('html', decodeMap, test.line, 0, test.tmTokens, null); - - let actualTokens = actual.tokens.map((t) => { return { startIndex: t.startIndex, type: t.type }; }); - let actualModeTransitions = actual.modeTransitions.map((t) => { return { startIndex: t.startIndex, modeId: t.modeId }; }); - - assert.deepEqual(actualTokens, test.tokens, 'test ' + test.line); - assert.deepEqual(actualModeTransitions, test.modeTransitions, 'test ' + test.line); + assert.deepEqual(actual.tokens, test.tokens, 'test ' + test.line); } }); @@ -411,11 +374,10 @@ suite('TextMate.decodeTextMateTokens', () => { { startIndex: 14, endIndex: 15, scopes: ['text.html.basic', 'meta.tag.sgml.html', 'punctuation.definition.tag.html'] } ], tokens: [ - { startIndex: 0, type: 'meta.tag.sgml.html.punctuation.definition' }, - { startIndex: 2, type: 'meta.tag.sgml.html.doctype' }, - { startIndex: 14, type: 'meta.tag.sgml.html.punctuation.definition' }, + { offset: 0, language: 'html', type: 'meta.tag.sgml.html.punctuation.definition' }, + { offset: 2, language: 'html', type: 'meta.tag.sgml.html.doctype' }, + { offset: 14, language: 'html', type: 'meta.tag.sgml.html.punctuation.definition' }, ], - modeTransitions: [{ startIndex: 0, modeId: 'html' }] }, { line: '', tmTokens: [ { startIndex: 0, endIndex: 3, scopes: ['text.html.basic', 'comment.block.html', 'punctuation.definition.comment.html'] } ], tokens: [ - { startIndex: 0, type: 'html.punctuation.definition.comment.block' }, + { offset: 0, language: 'html', type: 'html.punctuation.definition.comment.block' }, ], - modeTransitions: [{ startIndex: 0, modeId: 'html' }] }, { line: '', tmTokens: [ @@ -453,11 +412,10 @@ suite('TextMate.decodeTextMateTokens', () => { { startIndex: 5, endIndex: 6, scopes: ['text.html.basic', 'meta.tag.structure.any.html', 'punctuation.definition.tag.html'] } ], tokens: [ - { startIndex: 0, type: 'meta.tag.html.punctuation.definition.structure.any' }, - { startIndex: 1, type: 'meta.tag.html.structure.any.entity.name' }, - { startIndex: 5, type: 'meta.tag.html.punctuation.definition.structure.any' }, + { offset: 0, language: 'html', type: 'meta.tag.html.punctuation.definition.structure.any' }, + { offset: 1, language: 'html', type: 'meta.tag.html.structure.any.entity.name' }, + { offset: 5, language: 'html', type: 'meta.tag.html.punctuation.definition.structure.any' }, ], - modeTransitions: [{ startIndex: 0, modeId: 'html' }] }, { line: '', tmTokens: [ @@ -466,11 +424,10 @@ suite('TextMate.decodeTextMateTokens', () => { { startIndex: 5, endIndex: 6, scopes: ['text.html.basic', 'meta.tag.structure.any.html', 'punctuation.definition.tag.html'] } ], tokens: [ - { startIndex: 0, type: 'meta.tag.html.punctuation.definition.structure.any' }, - { startIndex: 1, type: 'meta.tag.html.structure.any.entity.name' }, - { startIndex: 5, type: 'meta.tag.html.punctuation.definition.structure.any' }, + { offset: 0, language: 'html', type: 'meta.tag.html.punctuation.definition.structure.any' }, + { offset: 1, language: 'html', type: 'meta.tag.html.structure.any.entity.name' }, + { offset: 5, language: 'html', type: 'meta.tag.html.punctuation.definition.structure.any' }, ], - modeTransitions: [{ startIndex: 0, modeId: 'html' }] }, { line: '\tHTML Sample', tmTokens: [ @@ -484,16 +441,15 @@ suite('TextMate.decodeTextMateTokens', () => { { startIndex: 26, endIndex: 27, scopes: ['text.html.basic', 'meta.tag.inline.any.html', 'punctuation.definition.tag.end.html'] } ], tokens: [ - { startIndex: 0, type: '' }, - { startIndex: 1, type: 'meta.tag.html.punctuation.definition.any.inline.begin' }, - { startIndex: 2, type: 'meta.tag.html.any.entity.name.inline' }, - { startIndex: 7, type: 'meta.tag.html.punctuation.definition.any.inline.end' }, - { startIndex: 8, type: '' }, - { startIndex: 19, type: 'meta.tag.html.punctuation.definition.any.inline.begin' }, - { startIndex: 21, type: 'meta.tag.html.any.entity.name.inline' }, - { startIndex: 26, type: 'meta.tag.html.punctuation.definition.any.inline.end' }, + { offset: 0, language: 'html', type: '' }, + { offset: 1, language: 'html', type: 'meta.tag.html.punctuation.definition.any.inline.begin' }, + { offset: 2, language: 'html', type: 'meta.tag.html.any.entity.name.inline' }, + { offset: 7, language: 'html', type: 'meta.tag.html.punctuation.definition.any.inline.end' }, + { offset: 8, language: 'html', type: '' }, + { offset: 19, language: 'html', type: 'meta.tag.html.punctuation.definition.any.inline.begin' }, + { offset: 21, language: 'html', type: 'meta.tag.html.any.entity.name.inline' }, + { offset: 26, language: 'html', type: 'meta.tag.html.punctuation.definition.any.inline.end' }, ], - modeTransitions: [{ startIndex: 0, modeId: 'html' }] }, { line: '\t', tmTokens: [ @@ -515,24 +471,23 @@ suite('TextMate.decodeTextMateTokens', () => { { startIndex: 53, endIndex: 54, scopes: ['text.html.basic', 'meta.tag.inline.any.html', 'punctuation.definition.tag.end.html'] } ], tokens: [ - { startIndex: 0, type: '' }, - { startIndex: 1, type: 'meta.tag.html.punctuation.definition.any.inline.begin' }, - { startIndex: 2, type: 'meta.tag.html.any.entity.name.inline' }, - { startIndex: 6, type: 'meta.tag.html.any.inline' }, - { startIndex: 7, type: 'meta.tag.html.any.entity.inline.other.attribute-name' }, - { startIndex: 17, type: 'meta.tag.html.any.inline' }, - { startIndex: 18, type: 'meta.tag.html.punctuation.definition.any.inline.begin.string.quoted.double' }, - { startIndex: 19, type: 'meta.tag.html.any.inline.string.quoted.double' }, - { startIndex: 34, type: 'meta.tag.html.punctuation.definition.any.inline.end.string.quoted.double' }, - { startIndex: 35, type: 'meta.tag.html.any.inline' }, - { startIndex: 36, type: 'meta.tag.html.any.entity.inline.other.attribute-name' }, - { startIndex: 43, type: 'meta.tag.html.any.inline' }, - { startIndex: 44, type: 'meta.tag.html.punctuation.definition.any.inline.begin.string.quoted.double' }, - { startIndex: 45, type: 'meta.tag.html.any.inline.string.quoted.double' }, - { startIndex: 52, type: 'meta.tag.html.punctuation.definition.any.inline.end.string.quoted.double' }, - { startIndex: 53, type: 'meta.tag.html.punctuation.definition.any.inline.end' }, + { offset: 0, language: 'html', type: '' }, + { offset: 1, language: 'html', type: 'meta.tag.html.punctuation.definition.any.inline.begin' }, + { offset: 2, language: 'html', type: 'meta.tag.html.any.entity.name.inline' }, + { offset: 6, language: 'html', type: 'meta.tag.html.any.inline' }, + { offset: 7, language: 'html', type: 'meta.tag.html.any.entity.inline.other.attribute-name' }, + { offset: 17, language: 'html', type: 'meta.tag.html.any.inline' }, + { offset: 18, language: 'html', type: 'meta.tag.html.punctuation.definition.any.inline.begin.string.quoted.double' }, + { offset: 19, language: 'html', type: 'meta.tag.html.any.inline.string.quoted.double' }, + { offset: 34, language: 'html', type: 'meta.tag.html.punctuation.definition.any.inline.end.string.quoted.double' }, + { offset: 35, language: 'html', type: 'meta.tag.html.any.inline' }, + { offset: 36, language: 'html', type: 'meta.tag.html.any.entity.inline.other.attribute-name' }, + { offset: 43, language: 'html', type: 'meta.tag.html.any.inline' }, + { offset: 44, language: 'html', type: 'meta.tag.html.punctuation.definition.any.inline.begin.string.quoted.double' }, + { offset: 45, language: 'html', type: 'meta.tag.html.any.inline.string.quoted.double' }, + { offset: 52, language: 'html', type: 'meta.tag.html.punctuation.definition.any.inline.end.string.quoted.double' }, + { offset: 53, language: 'html', type: 'meta.tag.html.punctuation.definition.any.inline.end' }, ], - modeTransitions: [{ startIndex: 0, modeId: 'html' }] }, { line: '\t', tmTokens: [ @@ -616,12 +567,11 @@ suite('TextMate.decodeTextMateTokens', () => { { startIndex: 8, endIndex: 9, scopes: ['text.html.basic', 'source.css.embedded.html', 'punctuation.definition.tag.html'] } ], tokens: [ - { startIndex: 0, type: 'html.source.css.embedded' }, - { startIndex: 1, type: 'tag.html.punctuation.definition.source.css.embedded' }, - { startIndex: 3, type: 'tag.html.entity.name.source.css.embedded.style' }, - { startIndex: 8, type: 'tag.html.punctuation.definition.source.css.embedded' }, + { offset: 0, language: 'css', type: 'html.source.css.embedded' }, + { offset: 1, language: 'css', type: 'tag.html.punctuation.definition.source.css.embedded' }, + { offset: 3, language: 'css', type: 'tag.html.entity.name.source.css.embedded.style' }, + { offset: 8, language: 'css', type: 'tag.html.punctuation.definition.source.css.embedded' }, ], - modeTransitions: [{ startIndex: 0, modeId: 'css' }] }, { line: '\tAfter', tmTokens: [ @@ -686,13 +634,12 @@ suite('TextMate.decodeTextMateTokens', () => { { startIndex: 10, endIndex: 15, scopes: ['text.html.basic'] } ], tokens: [ - { startIndex: 0, type: 'html.source.embedded.js' }, - { startIndex: 1, type: 'tag.html.punctuation.definition.source.embedded.js' }, - { startIndex: 3, type: 'tag.html.entity.name.source.embedded.js.script' }, - { startIndex: 9, type: 'tag.html.punctuation.definition.source.embedded.js' }, - { startIndex: 10, type: '' }, + { offset: 0, language: 'javascript', type: 'html.source.embedded.js' }, + { offset: 1, language: 'javascript', type: 'tag.html.punctuation.definition.source.embedded.js' }, + { offset: 3, language: 'javascript', type: 'tag.html.entity.name.source.embedded.js.script' }, + { offset: 9, language: 'javascript', type: 'tag.html.punctuation.definition.source.embedded.js' }, + { offset: 10, language: 'html', type: '' }, ], - modeTransitions: [{ startIndex: 0, modeId: 'javascript' }, { startIndex: 10, modeId: 'html' }] }, { line: '', tmTokens: [ @@ -701,11 +648,10 @@ suite('TextMate.decodeTextMateTokens', () => { { startIndex: 6, endIndex: 7, scopes: ['text.html.basic', 'meta.tag.structure.any.html', 'punctuation.definition.tag.html'] } ], tokens: [ - { startIndex: 0, type: 'meta.tag.html.punctuation.definition.structure.any' }, - { startIndex: 2, type: 'meta.tag.html.structure.any.entity.name' }, - { startIndex: 6, type: 'meta.tag.html.punctuation.definition.structure.any' }, + { offset: 0, language: 'html', type: 'meta.tag.html.punctuation.definition.structure.any' }, + { offset: 2, language: 'html', type: 'meta.tag.html.structure.any.entity.name' }, + { offset: 6, language: 'html', type: 'meta.tag.html.punctuation.definition.structure.any' }, ], - modeTransitions: [{ startIndex: 0, modeId: 'html' }] }, { line: '', tmTokens: [ @@ -714,11 +660,10 @@ suite('TextMate.decodeTextMateTokens', () => { { startIndex: 5, endIndex: 6, scopes: ['text.html.basic', 'meta.tag.structure.any.html', 'punctuation.definition.tag.html'] } ], tokens: [ - { startIndex: 0, type: 'meta.tag.html.punctuation.definition.structure.any' }, - { startIndex: 1, type: 'meta.tag.html.structure.any.entity.name' }, - { startIndex: 5, type: 'meta.tag.html.punctuation.definition.structure.any' }, + { offset: 0, language: 'html', type: 'meta.tag.html.punctuation.definition.structure.any' }, + { offset: 1, language: 'html', type: 'meta.tag.html.structure.any.entity.name' }, + { offset: 5, language: 'html', type: 'meta.tag.html.punctuation.definition.structure.any' }, ], - modeTransitions: [{ startIndex: 0, modeId: 'html' }] }, { line: '\t

Heading No.1

', tmTokens: [ @@ -732,16 +677,15 @@ suite('TextMate.decodeTextMateTokens', () => { { startIndex: 21, endIndex: 22, scopes: ['text.html.basic', 'meta.tag.block.any.html', 'punctuation.definition.tag.end.html'] } ], tokens: [ - { startIndex: 0, type: '' }, - { startIndex: 1, type: 'meta.tag.html.punctuation.definition.block.any.begin' }, - { startIndex: 2, type: 'meta.tag.html.block.any.entity.name' }, - { startIndex: 4, type: 'meta.tag.html.punctuation.definition.block.any.end' }, - { startIndex: 5, type: '' }, - { startIndex: 17, type: 'meta.tag.html.punctuation.definition.block.any.begin' }, - { startIndex: 19, type: 'meta.tag.html.block.any.entity.name' }, - { startIndex: 21, type: 'meta.tag.html.punctuation.definition.block.any.end' }, + { offset: 0, language: 'html', type: '' }, + { offset: 1, language: 'html', type: 'meta.tag.html.punctuation.definition.block.any.begin' }, + { offset: 2, language: 'html', type: 'meta.tag.html.block.any.entity.name' }, + { offset: 4, language: 'html', type: 'meta.tag.html.punctuation.definition.block.any.end' }, + { offset: 5, language: 'html', type: '' }, + { offset: 17, language: 'html', type: 'meta.tag.html.punctuation.definition.block.any.begin' }, + { offset: 19, language: 'html', type: 'meta.tag.html.block.any.entity.name' }, + { offset: 21, language: 'html', type: 'meta.tag.html.punctuation.definition.block.any.end' }, ], - modeTransitions: [{ startIndex: 0, modeId: 'html' }] }, { line: '\t', tmTokens: [ @@ -765,26 +709,25 @@ suite('TextMate.decodeTextMateTokens', () => { { startIndex: 47, endIndex: 50, scopes: ['text.html.basic', 'meta.tag.inline.any.html', 'punctuation.definition.tag.end.html'] } ], tokens: [ - { startIndex: 0, type: '' }, - { startIndex: 1, type: 'meta.tag.html.punctuation.definition.any.inline.begin' }, - { startIndex: 2, type: 'meta.tag.html.any.entity.name.inline' }, - { startIndex: 7, type: 'meta.tag.html.any.inline' }, - { startIndex: 8, type: 'meta.tag.html.any.entity.inline.other.attribute-name' }, - { startIndex: 16, type: 'meta.tag.html.any.inline' }, - { startIndex: 17, type: 'meta.tag.html.any.entity.inline.other.attribute-name' }, - { startIndex: 21, type: 'meta.tag.html.any.inline' }, - { startIndex: 22, type: 'meta.tag.html.punctuation.definition.any.inline.begin.string.quoted.double' }, - { startIndex: 23, type: 'meta.tag.html.any.inline.string.quoted.double' }, - { startIndex: 29, type: 'meta.tag.html.punctuation.definition.any.inline.end.string.quoted.double' }, - { startIndex: 30, type: 'meta.tag.html.any.inline' }, - { startIndex: 31, type: 'meta.tag.html.any.entity.inline.other.attribute-name' }, - { startIndex: 36, type: 'meta.tag.html.any.inline' }, - { startIndex: 37, type: 'meta.tag.html.punctuation.definition.any.inline.begin.string.quoted.double' }, - { startIndex: 38, type: 'meta.tag.html.any.inline.string.quoted.double' }, - { startIndex: 46, type: 'meta.tag.html.punctuation.definition.any.inline.end.string.quoted.double' }, - { startIndex: 47, type: 'meta.tag.html.punctuation.definition.any.inline.end' }, + { offset: 0, language: 'html', type: '' }, + { offset: 1, language: 'html', type: 'meta.tag.html.punctuation.definition.any.inline.begin' }, + { offset: 2, language: 'html', type: 'meta.tag.html.any.entity.name.inline' }, + { offset: 7, language: 'html', type: 'meta.tag.html.any.inline' }, + { offset: 8, language: 'html', type: 'meta.tag.html.any.entity.inline.other.attribute-name' }, + { offset: 16, language: 'html', type: 'meta.tag.html.any.inline' }, + { offset: 17, language: 'html', type: 'meta.tag.html.any.entity.inline.other.attribute-name' }, + { offset: 21, language: 'html', type: 'meta.tag.html.any.inline' }, + { offset: 22, language: 'html', type: 'meta.tag.html.punctuation.definition.any.inline.begin.string.quoted.double' }, + { offset: 23, language: 'html', type: 'meta.tag.html.any.inline.string.quoted.double' }, + { offset: 29, language: 'html', type: 'meta.tag.html.punctuation.definition.any.inline.end.string.quoted.double' }, + { offset: 30, language: 'html', type: 'meta.tag.html.any.inline' }, + { offset: 31, language: 'html', type: 'meta.tag.html.any.entity.inline.other.attribute-name' }, + { offset: 36, language: 'html', type: 'meta.tag.html.any.inline' }, + { offset: 37, language: 'html', type: 'meta.tag.html.punctuation.definition.any.inline.begin.string.quoted.double' }, + { offset: 38, language: 'html', type: 'meta.tag.html.any.inline.string.quoted.double' }, + { offset: 46, language: 'html', type: 'meta.tag.html.punctuation.definition.any.inline.end.string.quoted.double' }, + { offset: 47, language: 'html', type: 'meta.tag.html.punctuation.definition.any.inline.end' }, ], - modeTransitions: [{ startIndex: 0, modeId: 'html' }] }, { line: '', tmTokens: [ @@ -793,11 +736,10 @@ suite('TextMate.decodeTextMateTokens', () => { { startIndex: 6, endIndex: 7, scopes: ['text.html.basic', 'meta.tag.structure.any.html', 'punctuation.definition.tag.html'] } ], tokens: [ - { startIndex: 0, type: 'meta.tag.html.punctuation.definition.structure.any' }, - { startIndex: 2, type: 'meta.tag.html.structure.any.entity.name' }, - { startIndex: 6, type: 'meta.tag.html.punctuation.definition.structure.any' }, + { offset: 0, language: 'html', type: 'meta.tag.html.punctuation.definition.structure.any' }, + { offset: 2, language: 'html', type: 'meta.tag.html.structure.any.entity.name' }, + { offset: 6, language: 'html', type: 'meta.tag.html.punctuation.definition.structure.any' }, ], - modeTransitions: [{ startIndex: 0, modeId: 'html' }] }, { line: '', tmTokens: [ @@ -806,11 +748,10 @@ suite('TextMate.decodeTextMateTokens', () => { { startIndex: 6, endIndex: 7, scopes: ['text.html.basic', 'meta.tag.structure.any.html', 'punctuation.definition.tag.html'] } ], tokens: [ - { startIndex: 0, type: 'meta.tag.html.punctuation.definition.structure.any' }, - { startIndex: 2, type: 'meta.tag.html.structure.any.entity.name' }, - { startIndex: 6, type: 'meta.tag.html.punctuation.definition.structure.any' }, + { offset: 0, language: 'html', type: 'meta.tag.html.punctuation.definition.structure.any' }, + { offset: 2, language: 'html', type: 'meta.tag.html.structure.any.entity.name' }, + { offset: 6, language: 'html', type: 'meta.tag.html.punctuation.definition.structure.any' }, ], - modeTransitions: [{ startIndex: 0, modeId: 'html' }] } ]; @@ -830,11 +771,7 @@ suite('TextMate.decodeTextMateTokens', () => { let test = tests[i]; let actual = decodeTextMateTokens('html', decodeMap, test.line, 0, test.tmTokens, null); - let actualTokens = actual.tokens.map((t) => { return { startIndex: t.startIndex, type: t.type }; }); - let actualModeTransitions = actual.modeTransitions.map((t) => { return { startIndex: t.startIndex, modeId: t.modeId }; }); - - assert.deepEqual(actualTokens, test.tokens, 'test ' + test.line); - assert.deepEqual(actualModeTransitions, test.modeTransitions, 'test ' + test.line); + assert.deepEqual(actual.tokens, test.tokens, 'test ' + test.line); } }); @@ -847,12 +784,9 @@ suite('TextMate.decodeTextMateTokens', () => { { startIndex: 6, endIndex: 7, scopes: ['source.css.scss', 'meta.property-list.scss', 'punctuation.section.property-list.begin.bracket.curly.scss'] } ], tokens: [ - { startIndex: 0, type: '' }, - { startIndex: 6, type: 'meta.property-list.scss.punctuation.section.begin.bracket.curly' } + { offset: 0, language: 'scss', type: '' }, + { offset: 6, language: 'scss', type: 'meta.property-list.scss.punctuation.section.begin.bracket.curly' } ], - modeTransitions: [ - { startIndex: 0, modeId: 'scss' } - ] }, { line: ' background: red;', tmTokens: [ @@ -864,27 +798,21 @@ suite('TextMate.decodeTextMateTokens', () => { { startIndex: 19, endIndex: 20, scopes: ['source.css.scss', 'meta.property-list.scss', 'punctuation.terminator.rule.scss'] } ], tokens: [ - { startIndex: 0, type: 'meta.property-list.scss' }, - { startIndex: 4, type: 'meta.property-list.scss.property-name.support.type' }, - { startIndex: 14, type: 'meta.property-list.scss.punctuation.separator.key-value' }, - { startIndex: 15, type: 'meta.property-list.scss' }, - { startIndex: 16, type: 'meta.property-list.scss.support.property-value.constant.color.w3c-standard-color-name' }, - { startIndex: 19, type: 'meta.property-list.scss.punctuation.terminator.rule' } + { offset: 0, language: 'scss', type: 'meta.property-list.scss' }, + { offset: 4, language: 'scss', type: 'meta.property-list.scss.property-name.support.type' }, + { offset: 14, language: 'scss', type: 'meta.property-list.scss.punctuation.separator.key-value' }, + { offset: 15, language: 'scss', type: 'meta.property-list.scss' }, + { offset: 16, language: 'scss', type: 'meta.property-list.scss.support.property-value.constant.color.w3c-standard-color-name' }, + { offset: 19, language: 'scss', type: 'meta.property-list.scss.punctuation.terminator.rule' } ], - modeTransitions: [ - { startIndex: 0, modeId: 'scss' } - ] }, { line: '}', tmTokens: [ { startIndex: 0, endIndex: 1, scopes: ['source.css.scss', 'meta.property-list.scss', 'punctuation.section.property-list.end.bracket.curly.scss'] } ], tokens: [ - { startIndex: 0, type: 'meta.property-list.scss.punctuation.section.bracket.curly.end' } + { offset: 0, language: 'scss', type: 'meta.property-list.scss.punctuation.section.bracket.curly.end' } ], - modeTransitions: [ - { startIndex: 0, modeId: 'scss' } - ] } ]; @@ -898,11 +826,7 @@ suite('TextMate.decodeTextMateTokens', () => { let test = tests[i]; let actual = decodeTextMateTokens('scss', decodeMap, test.line, 0, test.tmTokens, null); - let actualTokens = actual.tokens.map((t) => { return { startIndex: t.startIndex, type: t.type }; }); - let actualModeTransitions = actual.modeTransitions.map((t) => { return { startIndex: t.startIndex, modeId: t.modeId }; }); - - assert.deepEqual(actualTokens, test.tokens, 'test ' + test.line); - assert.deepEqual(actualModeTransitions, test.modeTransitions, 'test ' + test.line); + assert.deepEqual(actual.tokens, test.tokens, 'test ' + test.line); } }); diff --git a/src/vs/monaco.d.ts b/src/vs/monaco.d.ts index e64352e7652..ab197a33284 100644 --- a/src/vs/monaco.d.ts +++ b/src/vs/monaco.d.ts @@ -751,6 +751,14 @@ declare module monaco { */ RTL = 1, } + export class Token { + _tokenBrand: void; + readonly offset: number; + readonly type: string; + readonly language: string; + constructor(offset: number, type: string, language: string); + toString(): string; + } } declare module monaco.editor { @@ -853,12 +861,6 @@ declare module monaco.editor { */ export function colorizeModelLine(model: IModel, lineNumber: number, tabSize?: number): string; - export class Token { - readonly offset: number; - readonly type: string; - constructor(offset: number, type: string); - } - /** * Tokenize `text` using language `languageId` */