development/vscode
development/vscode
1
0
Fork 0
mirror of https://github.com/Microsoft/vscode synced 2024-08-27 04:49:35 +00:00
Code Issues Packages Projects Releases Wiki Activity

Remove LanguageIdentifier and ensure tests dispose instantiated LanguagesRegistry objects

Browse source
This commit is contained in:
Alex Dima 2021-10-18 10:29:16 +02:00
parent 74541dceec
commit 11862795ea
No known key found for this signature in database
GPG key ID: 39563C1504FDD0C9
228 changed files with 2402 additions and 1667 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
build/monaco/monaco.d.ts.recipe
View file

@ -69,7 +69,7 @@ export interface ICommandHandler {
#include(vs/editor/standalone/browser/colorizer): IColorizerOptions, IColorizerElementOptions
#include(vs/base/common/scrollable): ScrollbarVisibility
#include(vs/platform/theme/common/themeService): ThemeColor
#includeAll(vs/editor/common/model;LanguageIdentifier=>languages.LanguageIdentifier): IScrollEvent
#includeAll(vs/editor/common/model): IScrollEvent
#includeAll(vs/editor/common/editorCommon;editorOptions.=>): IScrollEvent
#includeAll(vs/editor/common/model/textModelEvents):
#includeAll(vs/editor/common/controller/cursorEvents):

10
src/vs/editor/browser/core/markdownRenderer.ts
View file

@ -12,7 +12,7 @@ import { tokenizeToString } from 'vs/editor/common/modes/textToHtmlTokenizer';
import { ICodeEditor } from 'vs/editor/browser/editorBrowser';
import { Emitter } from 'vs/base/common/event';
import { IDisposable, DisposableStore } from 'vs/base/common/lifecycle';
import { ITokenizationSupport, TokenizationRegistry } from 'vs/editor/common/modes';
import { ILanguageIdCodec, ITokenizationSupport, TokenizationRegistry } from 'vs/editor/common/modes';
import { EditorOption } from 'vs/editor/common/config/editorOptions';
import { URI } from 'vs/base/common/uri';
@ -33,8 +33,8 @@ export interface IMarkdownRendererOptions {
export class MarkdownRenderer {
private static _ttpTokenizer = window.trustedTypes?.createPolicy('tokenizeToString', {
createHTML(value: string, tokenizer: ITokenizationSupport | undefined) {
return tokenizeToString(value, tokenizer);
createHTML(value: string, languageIdCodec: ILanguageIdCodec, tokenizer: ITokenizationSupport | undefined) {
return tokenizeToString(value, languageIdCodec, tokenizer);
}
});
@ -76,7 +76,7 @@ export class MarkdownRenderer {
if (languageAlias) {
modeId = this._modeService.getModeIdForLanguageName(languageAlias);
} else if (this._options.editor) {
modeId = this._options.editor.getModel()?.getLanguageIdentifier().language;
modeId = this._options.editor.getModel()?.getLanguageId();
}
if (!modeId) {
modeId = 'plaintext';
@ -86,7 +86,7 @@ export class MarkdownRenderer {
const element = document.createElement('span');
element.innerHTML = (MarkdownRenderer._ttpTokenizer?.createHTML(value, tokenization) ?? tokenizeToString(value, tokenization)) as string;
element.innerHTML = (MarkdownRenderer._ttpTokenizer?.createHTML(value, this._modeService.languageIdCodec, tokenization) ?? tokenizeToString(value, this._modeService.languageIdCodec, tokenization)) as string;
// use "good" font
let fontFamily = this._options.codeBlockFontFamily;

6
src/vs/editor/browser/widget/codeEditorWidget.ts
View file

@ -1498,7 +1498,7 @@ export class CodeEditorWidget extends Disposable implements editorBrowser.ICodeE
const listenersToRemove: IDisposable[] = [];
this._domElement.setAttribute('data-mode-id', model.getLanguageIdentifier().language);
this._domElement.setAttribute('data-mode-id', model.getLanguageId());
this._configuration.setIsDominatedByLongLines(model.isDominatedByLongLines());
this._configuration.setMaxLineNumber(model.getLineCount());
@ -1515,7 +1515,7 @@ export class CodeEditorWidget extends Disposable implements editorBrowser.ICodeE
listenersToRemove.push(model.onDidChangeDecorations((e) => this._onDidChangeModelDecorations.fire(e)));
listenersToRemove.push(model.onDidChangeLanguage((e) => {
this._domElement.setAttribute('data-mode-id', model.getLanguageIdentifier().language);
this._domElement.setAttribute('data-mode-id', model.getLanguageId());
this._onDidChangeModelLanguage.fire(e);
}));
listenersToRemove.push(model.onDidChangeLanguageConfiguration((e) => this._onDidChangeModelLanguageConfiguration.fire(e)));
@ -1961,7 +1961,7 @@ export class EditorModeContext extends Disposable {
return;
}
this._contextKeyService.bufferChangeEvents(() => {
this._langId.set(model.getLanguageIdentifier().language);
this._langId.set(model.getLanguageId());
this._hasCompletionItemProvider.set(modes.CompletionProviderRegistry.has(model));
this._hasCodeActionsProvider.set(modes.CodeActionProviderRegistry.has(model));
this._hasCodeLensProvider.set(modes.CodeLensProviderRegistry.has(model));

2
src/vs/editor/browser/widget/diffEditorWidget.ts
View file

@ -334,7 +334,7 @@ export class DiffEditorWidget extends Disposable implements editorBrowser.IDiffE
this._originalOverviewRuler = null;
this._modifiedOverviewRuler = null;
this._reviewPane = new DiffReview(this);
this._reviewPane = instantiationService.createInstance(DiffReview, this);
this._containerDomElement.appendChild(this._reviewPane.domNode.domNode);
this._containerDomElement.appendChild(this._reviewPane.shadow.domNode);
this._containerDomElement.appendChild(this._reviewPane.actionBarContainer.domNode);

20
src/vs/editor/browser/widget/diffReview.ts
View file

@ -32,6 +32,8 @@ import { registerThemingParticipant, ThemeIcon } from 'vs/platform/theme/common/
import { Constants } from 'vs/base/common/uint';
import { Codicon } from 'vs/base/common/codicons';
import { registerIcon } from 'vs/platform/theme/common/iconRegistry';
import { ILanguageIdCodec } from 'vs/editor/common/modes';
import { IModeService } from 'vs/editor/common/services/modeService';
const DIFF_LINES_PADDING = 3;
@ -92,7 +94,10 @@ export class DiffReview extends Disposable {
private _diffs: Diff[];
private _currentDiff: Diff | null;
constructor(diffEditor: DiffEditorWidget) {
constructor(
diffEditor: DiffEditorWidget,
@IModeService private readonly _modeService: IModeService
) {
super();
this._diffEditor = diffEditor;
this._isVisible = false;
@ -624,7 +629,7 @@ export class DiffReview extends Disposable {
let modLine = minModifiedLine;
for (let i = 0, len = diffs.length; i < len; i++) {
const diffEntry = diffs[i];
DiffReview._renderSection(container, diffEntry, modLine, lineHeight, this._width, originalOptions, originalModel, originalModelOpts, modifiedOptions, modifiedModel, modifiedModelOpts);
DiffReview._renderSection(container, diffEntry, modLine, lineHeight, this._width, originalOptions, originalModel, originalModelOpts, modifiedOptions, modifiedModel, modifiedModelOpts, this._modeService.languageIdCodec);
if (diffEntry.modifiedLineStart !== 0) {
modLine = diffEntry.modifiedLineEnd;
}
@ -638,7 +643,8 @@ export class DiffReview extends Disposable {
private static _renderSection(
dest: HTMLElement, diffEntry: DiffEntry, modLine: number, lineHeight: number, width: number,
originalOptions: IComputedEditorOptions, originalModel: ITextModel, originalModelOpts: TextModelResolvedOptions,
modifiedOptions: IComputedEditorOptions, modifiedModel: ITextModel, modifiedModelOpts: TextModelResolvedOptions
modifiedOptions: IComputedEditorOptions, modifiedModel: ITextModel, modifiedModelOpts: TextModelResolvedOptions,
languageIdCodec: ILanguageIdCodec
): void {
const type = diffEntry.getType();
@ -732,14 +738,14 @@ export class DiffReview extends Disposable {
let lineContent: string;
if (modifiedLine !== 0) {
let html: string | TrustedHTML = this._renderLine(modifiedModel, modifiedOptions, modifiedModelOpts.tabSize, modifiedLine);
let html: string | TrustedHTML = this._renderLine(modifiedModel, modifiedOptions, modifiedModelOpts.tabSize, modifiedLine, languageIdCodec);
if (DiffReview._ttPolicy) {
html = DiffReview._ttPolicy.createHTML(html as string);
}
cell.insertAdjacentHTML('beforeend', html as string);
lineContent = modifiedModel.getLineContent(modifiedLine);
} else {
let html: string | TrustedHTML = this._renderLine(originalModel, originalOptions, originalModelOpts.tabSize, originalLine);
let html: string | TrustedHTML = this._renderLine(originalModel, originalOptions, originalModelOpts.tabSize, originalLine, languageIdCodec);
if (DiffReview._ttPolicy) {
html = DiffReview._ttPolicy.createHTML(html as string);
}
@ -773,10 +779,10 @@ export class DiffReview extends Disposable {
}
}
private static _renderLine(model: ITextModel, options: IComputedEditorOptions, tabSize: number, lineNumber: number): string {
private static _renderLine(model: ITextModel, options: IComputedEditorOptions, tabSize: number, lineNumber: number, languageIdCodec: ILanguageIdCodec): string {
const lineContent = model.getLineContent(lineNumber);
const fontInfo = options.get(EditorOption.fontInfo);
const lineTokens = LineTokens.createEmpty(lineContent);
const lineTokens = LineTokens.createEmpty(lineContent, languageIdCodec);
const isBasicASCII = ViewLineRenderingData.isBasicASCII(lineContent, model.mightContainNonBasicASCII());
const containsRTL = ViewLineRenderingData.containsRTL(lineContent, isBasicASCII, model.mightContainRTL());
const r = renderViewLine(new RenderLineInput(

33
src/vs/editor/common/controller/cursorCommon.ts
View file

@ -11,7 +11,6 @@ import { ISelection, Selection } from 'vs/editor/common/core/selection';
import { ICommand, IConfiguration } from 'vs/editor/common/editorCommon';
import { ITextModel, PositionAffinity, TextModelResolvedOptions } from 'vs/editor/common/model';
import { TextModel } from 'vs/editor/common/model/textModel';
import { LanguageIdentifier } from 'vs/editor/common/modes';
import { AutoClosingPairs, IAutoClosingPair } from 'vs/editor/common/modes/languageConfiguration';
import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry';
import { ICoordinatesConverter } from 'vs/editor/common/viewModel/viewModel';
@ -81,7 +80,7 @@ export class CursorConfiguration {
public readonly surroundingPairs: CharacterMap;
public readonly shouldAutoCloseBefore: { quote: (ch: string) => boolean, bracket: (ch: string) => boolean };
private readonly _languageIdentifier: LanguageIdentifier;
private readonly _languageId: string;
private _electricChars: { [key: string]: boolean; } | null;
public static shouldRecreate(e: ConfigurationChangedEvent): boolean {
@ -103,11 +102,11 @@ export class CursorConfiguration {
}
constructor(
languageIdentifier: LanguageIdentifier,
languageId: string,
modelOptions: TextModelResolvedOptions,
configuration: IConfiguration
) {
this._languageIdentifier = languageIdentifier;
this._languageId = languageId;
const options = configuration.options;
const layoutInfo = options.get(EditorOption.layoutInfo);
@ -136,13 +135,13 @@ export class CursorConfiguration {
this._electricChars = null;
this.shouldAutoCloseBefore = {
quote: CursorConfiguration._getShouldAutoClose(languageIdentifier, this.autoClosingQuotes),
bracket: CursorConfiguration._getShouldAutoClose(languageIdentifier, this.autoClosingBrackets)
quote: CursorConfiguration._getShouldAutoClose(languageId, this.autoClosingQuotes),
bracket: CursorConfiguration._getShouldAutoClose(languageId, this.autoClosingBrackets)
};
this.autoClosingPairs = LanguageConfigurationRegistry.getAutoClosingPairs(languageIdentifier.id);
this.autoClosingPairs = LanguageConfigurationRegistry.getAutoClosingPairs(languageId);
let surroundingPairs = CursorConfiguration._getSurroundingPairs(languageIdentifier);
let surroundingPairs = CursorConfiguration._getSurroundingPairs(languageId);
if (surroundingPairs) {
for (const pair of surroundingPairs) {
this.surroundingPairs[pair.open] = pair.close;
@ -153,7 +152,7 @@ export class CursorConfiguration {
public get electricChars() {
if (!this._electricChars) {
this._electricChars = {};
let electricChars = CursorConfiguration._getElectricCharacters(this._languageIdentifier);
let electricChars = CursorConfiguration._getElectricCharacters(this._languageId);
if (electricChars) {
for (const char of electricChars) {
this._electricChars[char] = true;
@ -167,21 +166,21 @@ export class CursorConfiguration {
return TextModel.normalizeIndentation(str, this.indentSize, this.insertSpaces);
}
private static _getElectricCharacters(languageIdentifier: LanguageIdentifier): string[] | null {
private static _getElectricCharacters(languageId: string): string[] | null {
try {
return LanguageConfigurationRegistry.getElectricCharacters(languageIdentifier.id);
return LanguageConfigurationRegistry.getElectricCharacters(languageId);
} catch (e) {
onUnexpectedError(e);
return null;
}
}
private static _getShouldAutoClose(languageIdentifier: LanguageIdentifier, autoCloseConfig: EditorAutoClosingStrategy): (ch: string) => boolean {
private static _getShouldAutoClose(languageId: string, autoCloseConfig: EditorAutoClosingStrategy): (ch: string) => boolean {
switch (autoCloseConfig) {
case 'beforeWhitespace':
return autoCloseBeforeWhitespace;
case 'languageDefined':
return CursorConfiguration._getLanguageDefinedShouldAutoClose(languageIdentifier);
return CursorConfiguration._getLanguageDefinedShouldAutoClose(languageId);
case 'always':
return autoCloseAlways;
case 'never':
@ -189,9 +188,9 @@ export class CursorConfiguration {
}
}
private static _getLanguageDefinedShouldAutoClose(languageIdentifier: LanguageIdentifier): (ch: string) => boolean {
private static _getLanguageDefinedShouldAutoClose(languageId: string): (ch: string) => boolean {
try {
const autoCloseBeforeSet = LanguageConfigurationRegistry.getAutoCloseBeforeSet(languageIdentifier.id);
const autoCloseBeforeSet = LanguageConfigurationRegistry.getAutoCloseBeforeSet(languageId);
return c => autoCloseBeforeSet.indexOf(c) !== -1;
} catch (e) {
onUnexpectedError(e);
@ -199,9 +198,9 @@ export class CursorConfiguration {
}
}
private static _getSurroundingPairs(languageIdentifier: LanguageIdentifier): IAutoClosingPair[] | null {
private static _getSurroundingPairs(languageId: string): IAutoClosingPair[] | null {
try {
return LanguageConfigurationRegistry.getSurroundingPairs(languageIdentifier.id);
return LanguageConfigurationRegistry.getSurroundingPairs(languageId);
} catch (e) {
onUnexpectedError(e);
return null;

17
src/vs/editor/common/core/lineTokens.ts
View file

@ -3,7 +3,7 @@
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import { ColorId, FontStyle, LanguageId, MetadataConsts, StandardTokenType, TokenMetadata } from 'vs/editor/common/modes';
import { ColorId, FontStyle, ILanguageIdCodec, MetadataConsts, StandardTokenType, TokenMetadata } from 'vs/editor/common/modes';
export interface IViewLineTokens {
equals(other: IViewLineTokens): boolean;
@ -21,6 +21,7 @@ export class LineTokens implements IViewLineTokens {
private readonly _tokens: Uint32Array;
private readonly _tokensCount: number;
private readonly _text: string;
private readonly _languageIdCodec: ILanguageIdCodec;
public static defaultTokenMetadata = (
(FontStyle.None << MetadataConsts.FONT_STYLE_OFFSET)
@ -28,20 +29,21 @@ export class LineTokens implements IViewLineTokens {
| (ColorId.DefaultBackground << MetadataConsts.BACKGROUND_OFFSET)
) >>> 0;
public static createEmpty(lineContent: string): LineTokens {
public static createEmpty(lineContent: string, decoder: ILanguageIdCodec): LineTokens {
const defaultMetadata = LineTokens.defaultTokenMetadata;
const tokens = new Uint32Array(2);
tokens[0] = lineContent.length;
tokens[1] = defaultMetadata;
return new LineTokens(tokens, lineContent);
return new LineTokens(tokens, lineContent, decoder);
}
constructor(tokens: Uint32Array, text: string) {
constructor(tokens: Uint32Array, text: string, decoder: ILanguageIdCodec) {
this._tokens = tokens;
this._tokensCount = (this._tokens.length >>> 1);
this._text = text;
this._languageIdCodec = decoder;
}
public equals(other: IViewLineTokens): boolean {
@ -88,9 +90,10 @@ export class LineTokens implements IViewLineTokens {
return metadata;
}
public getLanguageId(tokenIndex: number): LanguageId {
public getLanguageId(tokenIndex: number): string {
const metadata = this._tokens[(tokenIndex << 1) + 1];
return TokenMetadata.getLanguageId(metadata);
const languageId = TokenMetadata.getLanguageId(metadata);
return this._languageIdCodec.decodeLanguageId(languageId);
}
public getStandardTokenType(tokenIndex: number): StandardTokenType {
@ -212,7 +215,7 @@ export class LineTokens implements IViewLineTokens {
}
}
return new LineTokens(new Uint32Array(newTokens), text);
return new LineTokens(new Uint32Array(newTokens), text, this._languageIdCodec);
}
}

14
src/vs/editor/common/model.ts
View file

@ -13,7 +13,7 @@ import { IRange, Range } from 'vs/editor/common/core/range';
import { Selection } from 'vs/editor/common/core/selection';
import { IModelContentChange, IModelContentChangedEvent, IModelDecorationsChangedEvent, IModelLanguageChangedEvent, IModelLanguageConfigurationChangedEvent, IModelOptionsChangedEvent, IModelTokensChangedEvent, ModelInjectedTextChangedEvent, ModelRawContentChangedEvent } from 'vs/editor/common/model/textModelEvents';
import { SearchData } from 'vs/editor/common/model/textModelSearch';
import { LanguageId, LanguageIdentifier, FormattingOptions } from 'vs/editor/common/modes';
import { FormattingOptions } from 'vs/editor/common/modes';
import { ThemeColor } from 'vs/platform/theme/common/themeService';
import { MultilineTokens, MultilineTokens2 } from 'vs/editor/common/model/tokensStore';
import { TextChange } from 'vs/editor/common/model/textChange';
@ -937,27 +937,21 @@ export interface ITextModel {
/**
* Get the language associated with this model.
* @internal
*/
getLanguageIdentifier(): LanguageIdentifier;
/**
* Get the language associated with this model.
*/
getModeId(): string;
getLanguageId(): string;
/**
* Set the current language mode associated with the model.
* @internal
*/
setMode(languageIdentifier: LanguageIdentifier): void;
setMode(languageId: string): void;
/**
* Returns the real (inner-most) language mode at a given position.
* The result might be inaccurate. Use `forceTokenization` to ensure accurate tokens.
* @internal
*/
getLanguageIdAtPosition(lineNumber: number, column: number): LanguageId;
getLanguageIdAtPosition(lineNumber: number, column: number): string;
/**
* Get the word under or besides `position`.

9
src/vs/editor/common/model/bracketPairColorizer/bracketPairColorizer.ts
View file

@ -12,7 +12,6 @@ import { DenseKeyProvider } from 'vs/editor/common/model/bracketPairColorizer/sm
import { DecorationProvider } from 'vs/editor/common/model/decorationProvider';
import { BackgroundTokenizationState, TextModel } from 'vs/editor/common/model/textModel';
import { IModelContentChangedEvent } from 'vs/editor/common/model/textModelEvents';
import { LanguageId } from 'vs/editor/common/modes';
import { ILanguageConfigurationService, ResolvedLanguageConfiguration } from 'vs/editor/common/modes/languageConfigurationRegistry';
import {
editorBracketHighlightingForeground1, editorBracketHighlightingForeground2, editorBracketHighlightingForeground3, editorBracketHighlightingForeground4, editorBracketHighlightingForeground5, editorBracketHighlightingForeground6, editorBracketHighlightingUnexpectedBracketForeground
@ -77,7 +76,7 @@ export class BracketPairColorizer extends Disposable implements DecorationProvid
this._register(
this.languageConfigurationService.onDidChange(e => {
if (!e.languageIdentifier || this.cache.value?.object.didLanguageChange(e.languageIdentifier.id)) {
if (!e.languageId || this.cache.value?.object.didLanguageChange(e.languageId)) {
this.cache.clear();
this.updateCache();
}
@ -178,7 +177,7 @@ class BracketPairColorizerImpl extends Disposable implements DecorationProvider
private readonly denseKeyProvider = new DenseKeyProvider<string>();
private readonly brackets = new LanguageAgnosticBracketTokens(this.denseKeyProvider, this.getLanguageConfiguration);
public didLanguageChange(languageId: LanguageId): boolean {
public didLanguageChange(languageId: string): boolean {
return this.brackets.didLanguageChange(languageId);
}
@ -186,7 +185,7 @@ class BracketPairColorizerImpl extends Disposable implements DecorationProvider
constructor(
private readonly textModel: TextModel,
private readonly getLanguageConfiguration: (languageId: LanguageId) => ResolvedLanguageConfiguration
private readonly getLanguageConfiguration: (languageId: string) => ResolvedLanguageConfiguration
) {
super();
@ -217,7 +216,7 @@ class BracketPairColorizerImpl extends Disposable implements DecorationProvider
if (textModel.backgroundTokenizationState === BackgroundTokenizationState.Uninitialized) {
// There are no token information yet
const brackets = this.brackets.getSingleLanguageBracketTokens(this.textModel.getLanguageIdentifier().id);
const brackets = this.brackets.getSingleLanguageBracketTokens(this.textModel.getLanguageId());
const tokenizer = new FastTokenizer(this.textModel.getValue(), brackets);
this.initialAstWithoutTokens = parseDocument(tokenizer, [], undefined, true);
this.astWithTokens = this.initialAstWithoutTokens;

17
src/vs/editor/common/model/bracketPairColorizer/brackets.ts
View file

@ -5,14 +5,13 @@
import { escapeRegExpCharacters } from 'vs/base/common/strings';
import { toLength } from 'vs/editor/common/model/bracketPairColorizer/length';
import { SmallImmutableSet, DenseKeyProvider, identityKeyProvider } from 'vs/editor/common/model/bracketPairColorizer/smallImmutableSet';
import { LanguageId } from 'vs/editor/common/modes';
import { ResolvedLanguageConfiguration } from 'vs/editor/common/modes/languageConfigurationRegistry';
import { BracketAstNode } from './ast';
import { OpeningBracketId, Token, TokenKind } from './tokenizer';
export class BracketTokens {
static createFromLanguage(configuration: ResolvedLanguageConfiguration, denseKeyProvider: DenseKeyProvider<string>): BracketTokens {
function getId(languageId: LanguageId, openingText: string): OpeningBracketId {
function getId(languageId: string, openingText: string): OpeningBracketId {
return denseKeyProvider.getKey(`${languageId}:::${openingText}`);
}
@ -25,7 +24,7 @@ export class BracketTokens {
openingBrackets.add(openingText);
let info = closingBrackets.get(closingText);
const openingTextId = getId(configuration.languageIdentifier.id, openingText);
const openingTextId = getId(configuration.languageId, openingText);
if (!info) {
info = { openingBrackets: SmallImmutableSet.getEmpty(), first: openingTextId };
closingBrackets.set(closingText, info);
@ -48,7 +47,7 @@ export class BracketTokens {
for (const openingText of openingBrackets) {
const length = toLength(0, openingText.length);
const openingTextId = getId(configuration.languageIdentifier.id, openingText);
const openingTextId = getId(configuration.languageId, openingText);
map.set(openingText, new Token(
length,
TokenKind.OpeningBracket,
@ -101,15 +100,15 @@ export class BracketTokens {
}
export class LanguageAgnosticBracketTokens {
private readonly languageIdToBracketTokens: Map<LanguageId, BracketTokens> = new Map();
private readonly languageIdToBracketTokens = new Map<string, BracketTokens>();
constructor(
private readonly denseKeyProvider: DenseKeyProvider<string>,
private readonly getLanguageConfiguration: (languageId: LanguageId) => ResolvedLanguageConfiguration,
private readonly getLanguageConfiguration: (languageId: string) => ResolvedLanguageConfiguration,
) {
}
public didLanguageChange(languageId: LanguageId): boolean {
public didLanguageChange(languageId: string): boolean {
const existing = this.languageIdToBracketTokens.get(languageId);
if (!existing) {
return false;
@ -118,7 +117,7 @@ export class LanguageAgnosticBracketTokens {
return existing.getRegExpStr() !== newRegExpStr;
}
getSingleLanguageBracketTokens(languageId: LanguageId): BracketTokens {
getSingleLanguageBracketTokens(languageId: string): BracketTokens {
let singleLanguageBracketTokens = this.languageIdToBracketTokens.get(languageId);
if (!singleLanguageBracketTokens) {
singleLanguageBracketTokens = BracketTokens.createFromLanguage(this.getLanguageConfiguration(languageId), this.denseKeyProvider);
@ -127,7 +126,7 @@ export class LanguageAgnosticBracketTokens {
return singleLanguageBracketTokens;
}
getToken(value: string, languageId: LanguageId): Token | undefined {
getToken(value: string, languageId: string): Token | undefined {
const singleLanguageBracketTokens = this.getSingleLanguageBracketTokens(languageId);
return singleLanguageBracketTokens.getToken(value);
}

80
src/vs/editor/common/model/textModel.ts
View file

@ -24,9 +24,9 @@ import { IModelContentChangedEvent, IModelDecorationsChangedEvent, IModelLanguag
import { SearchData, SearchParams, TextModelSearch } from 'vs/editor/common/model/textModelSearch';
import { TextModelTokenization } from 'vs/editor/common/model/textModelTokens';
import { getWordAtText } from 'vs/editor/common/model/wordHelper';
import { LanguageId, LanguageIdentifier, FormattingOptions } from 'vs/editor/common/modes';
import { FormattingOptions } from 'vs/editor/common/modes';
import { ILanguageConfigurationService, ResolvedLanguageConfiguration } from 'vs/editor/common/modes/languageConfigurationRegistry';
import { NULL_LANGUAGE_IDENTIFIER } from 'vs/editor/common/modes/nullMode';
import { NULL_MODE_ID } from 'vs/editor/common/modes/nullMode';
import { ignoreBracketsInToken } from 'vs/editor/common/modes/supports';
import { BracketsUtils, RichEditBracket, RichEditBrackets } from 'vs/editor/common/modes/supports/richEditBrackets';
import { ThemeColor } from 'vs/platform/theme/common/themeService';
@ -43,6 +43,7 @@ import { ArrayQueue, findLast } from 'vs/base/common/arrays';
import { BracketPairColorizer, IBracketPairs } from 'vs/editor/common/model/bracketPairColorizer/bracketPairColorizer';
import { DecorationProvider } from 'vs/editor/common/model/decorationProvider';
import { CursorColumns } from 'vs/editor/common/controller/cursorColumns';
import { IModeService } from 'vs/editor/common/services/modeService';
function createTextBufferBuilder() {
return new PieceTreeTextBufferBuilder();
@ -268,7 +269,6 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati
public readonly id: string;
public readonly isForSimpleWidget: boolean;
private readonly _associatedResource: URI;
private readonly _undoRedoService: IUndoRedoService;
private _attachedEditorCount: number;
private _buffer: model.ITextBuffer;
private _bufferDisposable: IDisposable;
@ -305,7 +305,7 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati
//#endregion
//#region Tokenization
private _languageIdentifier: LanguageIdentifier;
private _languageId: string;
private readonly _languageRegistryListener: IDisposable;
private readonly _tokens: TokensStore;
private readonly _tokens2: TokensStore2;
@ -334,20 +334,17 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati
private readonly _onBackgroundTokenizationStateChanged = this._register(new Emitter<void>());
public readonly onBackgroundTokenizationStateChanged: Event<void> = this._onBackgroundTokenizationStateChanged.event;
private readonly _languageConfigurationService: ILanguageConfigurationService;
constructor(
source: string | model.ITextBufferFactory,
creationOptions: model.ITextModelCreationOptions,
languageIdentifier: LanguageIdentifier | null,
languageId: string | null,
associatedResource: URI | null = null,
undoRedoService: IUndoRedoService,
languageConfigurationService: ILanguageConfigurationService
@IUndoRedoService private readonly _undoRedoService: IUndoRedoService,
@IModeService private readonly _modeService: IModeService,
@ILanguageConfigurationService private readonly _languageConfigurationService: ILanguageConfigurationService,
) {
super();
this._languageConfigurationService = languageConfigurationService;
this._register(this._eventEmitter.fastEvent((e: InternalModelContentChangeEvent) => {
this._onDidChangeContentOrInjectedText.fire(e.rawContentChangedEvent);
}));
@ -361,7 +358,6 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati
} else {
this._associatedResource = associatedResource;
}
this._undoRedoService = undoRedoService;
this._attachedEditorCount = 0;
const { textBuffer, disposable } = createTextBuffer(source, creationOptions.defaultEOL);
@ -394,11 +390,11 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati
this._isDisposed = false;
this._isDisposing = false;
this._languageIdentifier = languageIdentifier || NULL_LANGUAGE_IDENTIFIER;
this._languageId = languageId || NULL_MODE_ID;
this._languageRegistryListener = this._languageConfigurationService.onDidChange(
e => {
if (e.affects(this._languageIdentifier)) {
if (e.affects(this._languageId)) {
this._onDidChangeLanguageConfiguration.fire({});
}
}
@ -409,14 +405,14 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati
this._decorations = Object.create(null);
this._decorationsTree = new DecorationsTrees();
this._commandManager = new EditStack(this, undoRedoService);
this._commandManager = new EditStack(this, this._undoRedoService);
this._isUndoing = false;
this._isRedoing = false;
this._trimAutoWhitespaceLines = null;
this._tokens = new TokensStore();
this._tokens2 = new TokensStore2();
this._tokenization = new TextModelTokenization(this);
this._tokens = new TokensStore(this._modeService.languageIdCodec);
this._tokens2 = new TokensStore2(this._modeService.languageIdCodec);
this._tokenization = new TextModelTokenization(this, this._modeService.languageIdCodec);
this._bracketPairColorizer = this._register(new BracketPairColorizer(this, this._languageConfigurationService));
this._decorationProvider = this._bracketPairColorizer;
@ -1975,7 +1971,7 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati
throw new Error('Illegal value for lineNumber');
}
this._tokens.setTokens(this._languageIdentifier.id, lineNumber - 1, this._buffer.getLineLength(lineNumber), tokens, false);
this._tokens.setTokens(this._languageId, lineNumber - 1, this._buffer.getLineLength(lineNumber), tokens, false);
}
public setTokens(tokens: MultilineTokens[], backgroundTokenizationCompleted: boolean = false): void {
@ -1990,10 +1986,10 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati
for (let j = 0, lenJ = element.tokens.length; j < lenJ; j++) {
const lineNumber = element.startLineNumber + j;
if (hasChange) {
this._tokens.setTokens(this._languageIdentifier.id, lineNumber - 1, this._buffer.getLineLength(lineNumber), element.tokens[j], false);
this._tokens.setTokens(this._languageId, lineNumber - 1, this._buffer.getLineLength(lineNumber), element.tokens[j], false);
maxChangedLineNumber = lineNumber;
} else {
const lineHasChange = this._tokens.setTokens(this._languageIdentifier.id, lineNumber - 1, this._buffer.getLineLength(lineNumber), element.tokens[j], true);
const lineHasChange = this._tokens.setTokens(this._languageId, lineNumber - 1, this._buffer.getLineLength(lineNumber), element.tokens[j], true);
if (lineHasChange) {
hasChange = true;
minChangedLineNumber = lineNumber;
@ -2114,42 +2110,38 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati
private _getLineTokens(lineNumber: number): LineTokens {
const lineText = this.getLineContent(lineNumber);
const syntacticTokens = this._tokens.getTokens(this._languageIdentifier.id, lineNumber - 1, lineText);
const syntacticTokens = this._tokens.getTokens(this._languageId, lineNumber - 1, lineText);
return this._tokens2.addSemanticTokens(lineNumber, syntacticTokens);
}
public getLanguageIdentifier(): LanguageIdentifier {
return this._languageIdentifier;
public getLanguageId(): string {
return this._languageId;
}
public getModeId(): string {
return this._languageIdentifier.language;
}
public setMode(languageIdentifier: LanguageIdentifier): void {
if (this._languageIdentifier.id === languageIdentifier.id) {
public setMode(languageId: string): void {
if (this._languageId === languageId) {
// There's nothing to do
return;
}
let e: IModelLanguageChangedEvent = {
oldLanguage: this._languageIdentifier.language,
newLanguage: languageIdentifier.language
oldLanguage: this._languageId,
newLanguage: languageId
};
this._languageIdentifier = languageIdentifier;
this._languageId = languageId;
this._onDidChangeLanguage.fire(e);
this._onDidChangeLanguageConfiguration.fire({});
}
public getLanguageIdAtPosition(lineNumber: number, column: number): LanguageId {
public getLanguageIdAtPosition(lineNumber: number, column: number): string {
const position = this.validatePosition(new Position(lineNumber, column));
const lineTokens = this.getLineTokens(position.lineNumber);
return lineTokens.getLanguageId(lineTokens.findTokenIndexAtOffset(position.column - 1));
}
private getLanguageConfiguration(languageId: LanguageId): ResolvedLanguageConfiguration {
private getLanguageConfiguration(languageId: string): ResolvedLanguageConfiguration {
return this._languageConfigurationService.getLanguageConfiguration(languageId);
}
@ -2386,7 +2378,7 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati
private _findMatchingBracketUp(bracket: RichEditBracket, position: Position, continueSearchPredicate: ContinueBracketSearchPredicate): Range | null | BracketSearchCanceled {
// console.log('_findMatchingBracketUp: ', 'bracket: ', JSON.stringify(bracket), 'startPosition: ', String(position));
const languageId = bracket.languageIdentifier.id;
const languageId = bracket.languageId;
const reversedBracketRegex = bracket.reversedRegex;
let count = -1;
@ -2473,7 +2465,7 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati
private _findMatchingBracketDown(bracket: RichEditBracket, position: Position, continueSearchPredicate: ContinueBracketSearchPredicate): Range | null | BracketSearchCanceled {
// console.log('_findMatchingBracketDown: ', 'bracket: ', JSON.stringify(bracket), 'startPosition: ', String(position));
const languageId = bracket.languageIdentifier.id;
const languageId = bracket.languageId;
const bracketRegex = bracket.forwardRegex;
let count = 1;
@ -2561,7 +2553,7 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati
public findPrevBracket(_position: IPosition): model.IFoundBracket | null {
const position = this.validatePosition(_position);
let languageId: LanguageId = -1;
let languageId: string | null = null;
let modeBrackets: RichEditBrackets | null = null;
for (let lineNumber = position.lineNumber; lineNumber >= 1; lineNumber--) {
const lineTokens = this._getLineTokens(lineNumber);
@ -2639,7 +2631,7 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati
const position = this.validatePosition(_position);
const lineCount = this.getLineCount();
let languageId: LanguageId = -1;
let languageId: string | null = null;
let modeBrackets: RichEditBrackets | null = null;
for (let lineNumber = position.lineNumber; lineNumber <= lineCount; lineNumber++) {
const lineTokens = this._getLineTokens(lineNumber);
@ -2724,10 +2716,10 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati
}
const position = this.validatePosition(_position);
const lineCount = this.getLineCount();
const savedCounts = new Map<number, number[]>();
const savedCounts = new Map<string, number[]>();
let counts: number[] = [];
const resetCounts = (languageId: number, modeBrackets: RichEditBrackets | null) => {
const resetCounts = (languageId: string, modeBrackets: RichEditBrackets | null) => {
if (!savedCounts.has(languageId)) {
let tmp = [];
for (let i = 0, len = modeBrackets ? modeBrackets.brackets.length : 0; i < len; i++) {
@ -2768,7 +2760,7 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati
return null;
};
let languageId: LanguageId = -1;
let languageId: string | null = null;
let modeBrackets: RichEditBrackets | null = null;
for (let lineNumber = position.lineNumber; lineNumber <= lineCount; lineNumber++) {
const lineTokens = this._getLineTokens(lineNumber);
@ -2905,7 +2897,7 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati
throw new Error('Illegal value for lineNumber');
}
const foldingRules = this.getLanguageConfiguration(this._languageIdentifier.id).foldingRules;
const foldingRules = this.getLanguageConfiguration(this._languageId).foldingRules;
const offSide = Boolean(foldingRules && foldingRules.offSide);
let up_aboveContentLineIndex = -2; /* -2 is a marker for not having computed it */
@ -3267,7 +3259,7 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati
throw new Error('Illegal value for endLineNumber');
}
const foldingRules = this.getLanguageConfiguration(this._languageIdentifier.id).foldingRules;
const foldingRules = this.getLanguageConfiguration(this._languageId).foldingRules;
const offSide = Boolean(foldingRules && foldingRules.offSide);
let result: number[] = new Array<number>(endLineNumber - startLineNumber + 1);

41
src/vs/editor/common/model/textModelTokens.ts
View file

@ -9,7 +9,7 @@ import { LineTokens } from 'vs/editor/common/core/lineTokens';
import { Position } from 'vs/editor/common/core/position';
import { IRange } from 'vs/editor/common/core/range';
import { TokenizationResult2 } from 'vs/editor/common/core/token';
import { IState, ITokenizationSupport, LanguageIdentifier, TokenizationRegistry } from 'vs/editor/common/modes';
import { ILanguageIdCodec, IState, ITokenizationSupport, TokenizationRegistry } from 'vs/editor/common/modes';
import { nullTokenize2 } from 'vs/editor/common/modes/nullMode';
import { TextModel } from 'vs/editor/common/model/textModel';
import { Disposable } from 'vs/base/common/lifecycle';
@ -101,8 +101,8 @@ export class TokenizationStateStore {
if (insertCount === 0) {
return;
}
let beginState: (IState | null)[] = [];
let valid: boolean[] = [];
const beginState: (IState | null)[] = [];
const valid: boolean[] = [];
for (let i = 0; i < insertCount; i++) {
beginState[i] = null;
valid[i] = false;
@ -194,21 +194,22 @@ export class TokenizationStateStore {
export class TextModelTokenization extends Disposable {
private readonly _textModel: TextModel;
private readonly _tokenizationStateStore: TokenizationStateStore;
private _isDisposed: boolean;
private _tokenizationSupport: ITokenizationSupport | null;
constructor(textModel: TextModel) {
constructor(
private readonly _textModel: TextModel,
private readonly _languageIdCodec: ILanguageIdCodec
) {
super();
this._isDisposed = false;
this._textModel = textModel;
this._tokenizationStateStore = new TokenizationStateStore();
this._tokenizationSupport = null;
this._register(TokenizationRegistry.onDidChange((e) => {
const languageIdentifier = this._textModel.getLanguageIdentifier();
if (e.changedLanguages.indexOf(languageIdentifier.language) === -1) {
const languageId = this._textModel.getLanguageId();
if (e.changedLanguages.indexOf(languageId) === -1) {
return;
}
@ -349,7 +350,7 @@ export class TextModelTokenization extends Disposable {
if (!this._tokenizationSupport) {
return;
}
const languageIdentifier = this._textModel.getLanguageIdentifier();
const languageId = this._textModel.getLanguageId();
const linesLength = this._textModel.getLineCount();
const endLineIndex = lineNumber - 1;
@ -358,7 +359,7 @@ export class TextModelTokenization extends Disposable {
const text = this._textModel.getLineContent(lineIndex + 1);
const lineStartState = this._tokenizationStateStore.getBeginState(lineIndex);
const r = safeTokenize(languageIdentifier, this._tokenizationSupport, text, true, lineStartState!);
const r = safeTokenize(this._languageIdCodec, languageId, this._tokenizationSupport, text, true, lineStartState!);
builder.add(lineIndex + 1, r.tokens);
this._tokenizationStateStore.setEndState(linesLength, lineIndex, r.endState);
lineIndex = this._tokenizationStateStore.invalidLineStartIndex - 1; // -1 because the outer loop increments it
@ -383,10 +384,10 @@ export class TextModelTokenization extends Disposable {
}
let nonWhitespaceColumn = this._textModel.getLineFirstNonWhitespaceColumn(startLineNumber);
let fakeLines: string[] = [];
const fakeLines: string[] = [];
let initialState: IState | null = null;
for (let i = startLineNumber - 1; nonWhitespaceColumn > 0 && i >= 1; i--) {
let newNonWhitespaceIndex = this._textModel.getLineFirstNonWhitespaceColumn(i);
const newNonWhitespaceIndex = this._textModel.getLineFirstNonWhitespaceColumn(i);
if (newNonWhitespaceIndex === 0) {
continue;
@ -406,16 +407,16 @@ export class TextModelTokenization extends Disposable {
initialState = this._tokenizationSupport.getInitialState();
}
const languageIdentifier = this._textModel.getLanguageIdentifier();
const languageId = this._textModel.getLanguageId();
let state = initialState;
for (let i = fakeLines.length - 1; i >= 0; i--) {
let r = safeTokenize(languageIdentifier, this._tokenizationSupport, fakeLines[i], false, state);
const r = safeTokenize(this._languageIdCodec, languageId, this._tokenizationSupport, fakeLines[i], false, state);
state = r.endState;
}
for (let lineNumber = startLineNumber; lineNumber <= endLineNumber; lineNumber++) {
let text = this._textModel.getLineContent(lineNumber);
let r = safeTokenize(languageIdentifier, this._tokenizationSupport, text, true, state);
const text = this._textModel.getLineContent(lineNumber);
const r = safeTokenize(this._languageIdCodec, languageId, this._tokenizationSupport, text, true, state);
builder.add(lineNumber, r.tokens);
this._tokenizationStateStore.setFakeTokens(lineNumber - 1);
state = r.endState;
@ -424,11 +425,11 @@ export class TextModelTokenization extends Disposable {
}
function initializeTokenization(textModel: TextModel): [ITokenizationSupport | null, IState | null] {
const languageIdentifier = textModel.getLanguageIdentifier();
const languageId = textModel.getLanguageId();
let tokenizationSupport = (
textModel.isTooLargeForTokenization()
? null
: TokenizationRegistry.get(languageIdentifier.language)
: TokenizationRegistry.get(languageId)
);
let initialState: IState | null = null;
if (tokenizationSupport) {
@ -442,7 +443,7 @@ function initializeTokenization(textModel: TextModel): [ITokenizationSupport | n
return [tokenizationSupport, initialState];
}
function safeTokenize(languageIdentifier: LanguageIdentifier, tokenizationSupport: ITokenizationSupport | null, text: string, hasEOL: boolean, state: IState): TokenizationResult2 {
function safeTokenize(languageIdCodec: ILanguageIdCodec, languageId: string, tokenizationSupport: ITokenizationSupport | null, text: string, hasEOL: boolean, state: IState): TokenizationResult2 {
let r: TokenizationResult2 | null = null;
if (tokenizationSupport) {
@ -454,7 +455,7 @@ function safeTokenize(languageIdentifier: LanguageIdentifier, tokenizationSuppor
}
if (!r) {
r = nullTokenize2(languageIdentifier.id, text, state, 0);
r = nullTokenize2(languageIdCodec.encodeLanguageId(languageId), text, state, 0);
}
LineTokens.convertToEndOffset(r.tokens, text.length);

26
src/vs/editor/common/model/tokensStore.ts
View file

@ -7,7 +7,7 @@ import * as arrays from 'vs/base/common/arrays';
import { LineTokens } from 'vs/editor/common/core/lineTokens';
import { Position } from 'vs/editor/common/core/position';
import { IRange, Range } from 'vs/editor/common/core/range';
import { ColorId, FontStyle, LanguageId, MetadataConsts, StandardTokenType, TokenMetadata } from 'vs/editor/common/modes';
import { ColorId, FontStyle, ILanguageIdCodec, LanguageId, MetadataConsts, StandardTokenType, TokenMetadata } from 'vs/editor/common/modes';
import { writeUInt32BE, readUInt32BE } from 'vs/base/common/buffer';
import { CharCode } from 'vs/base/common/charCode';
@ -873,10 +873,12 @@ export class TokensStore2 {
private _pieces: MultilineTokens2[];
private _isComplete: boolean;
private readonly _languageIdCodec: ILanguageIdCodec;
constructor() {
constructor(languageIdCodec: ILanguageIdCodec) {
this._pieces = [];
this._isComplete = false;
this._languageIdCodec = languageIdCodec;
}
public flush(): void {
@ -1058,7 +1060,7 @@ export class TokensStore2 {
aIndex++;
}
return new LineTokens(new Uint32Array(result), aTokens.getLineContent());
return new LineTokens(new Uint32Array(result), aTokens.getLineContent(), this._languageIdCodec);
}
private static _findFirstPieceWithLine(pieces: MultilineTokens2[], lineNumber: number): number {
@ -1097,10 +1099,12 @@ export class TokensStore2 {
export class TokensStore {
private _lineTokens: (Uint32Array | ArrayBuffer | null)[];
private _len: number;
private readonly _languageIdCodec: ILanguageIdCodec;
constructor() {
constructor(languageIdCodec: ILanguageIdCodec) {
this._lineTokens = [];
this._len = 0;
this._languageIdCodec = languageIdCodec;
}
public flush(): void {
@ -1108,20 +1112,20 @@ export class TokensStore {
this._len = 0;
}
public getTokens(topLevelLanguageId: LanguageId, lineIndex: number, lineText: string): LineTokens {
public getTokens(topLevelLanguageId: string, lineIndex: number, lineText: string): LineTokens {
let rawLineTokens: Uint32Array | ArrayBuffer | null = null;
if (lineIndex < this._len) {
rawLineTokens = this._lineTokens[lineIndex];
}
if (rawLineTokens !== null && rawLineTokens !== EMPTY_LINE_TOKENS) {
return new LineTokens(toUint32Array(rawLineTokens), lineText);
return new LineTokens(toUint32Array(rawLineTokens), lineText, this._languageIdCodec);
}
let lineTokens = new Uint32Array(2);
const lineTokens = new Uint32Array(2);
lineTokens[0] = lineText.length;
lineTokens[1] = getDefaultMetadata(topLevelLanguageId);
return new LineTokens(lineTokens, lineText);
lineTokens[1] = getDefaultMetadata(this._languageIdCodec.encodeLanguageId(topLevelLanguageId));
return new LineTokens(lineTokens, lineText, this._languageIdCodec);
}
private static _massageTokens(topLevelLanguageId: LanguageId, lineTextLength: number, _tokens: Uint32Array | ArrayBuffer | null): Uint32Array | ArrayBuffer {
@ -1186,8 +1190,8 @@ export class TokensStore {
this._len += insertCount;
}
public setTokens(topLevelLanguageId: LanguageId, lineIndex: number, lineTextLength: number, _tokens: Uint32Array | ArrayBuffer | null, checkEquality: boolean): boolean {
const tokens = TokensStore._massageTokens(topLevelLanguageId, lineTextLength, _tokens);
public setTokens(topLevelLanguageId: string, lineIndex: number, lineTextLength: number, _tokens: Uint32Array | ArrayBuffer | null, checkEquality: boolean): boolean {
const tokens = TokensStore._massageTokens(this._languageIdCodec.encodeLanguageId(topLevelLanguageId), lineTextLength, _tokens);
this._ensureLine(lineIndex);
const oldTokens = this._lineTokens[lineIndex];
this._lineTokens[lineIndex] = tokens;

30
src/vs/editor/common/modes.ts
View file

@ -29,28 +29,6 @@ export const enum LanguageId {
PlainText = 1
}
/**
* @internal
*/
export class LanguageIdentifier {
/**
* A string identifier. Unique across languages. e.g. 'javascript'.
*/
public readonly language: string;
/**
* A numeric identifier. Unique across languages. e.g. 5
* Will vary at runtime based on registration order, etc.
*/
public readonly id: LanguageId;
constructor(language: string, id: LanguageId) {
this.language = language;
this.id = id;
}
}
/**
* A font style. Values are 2^x such that a bit mask can be used.
* @internal
@ -191,6 +169,14 @@ export class TokenMetadata {
}
}
/**
* @internal
*/
export interface ILanguageIdCodec {
encodeLanguageId(languageId: string): LanguageId;
decodeLanguageId(languageId: LanguageId): string;
}
/**
* @internal
*/

115
src/vs/editor/common/modes/languageConfigurationRegistry.ts
View file

@ -10,7 +10,6 @@ import { LineTokens } from 'vs/editor/common/core/lineTokens';
import { Range } from 'vs/editor/common/core/range';
import { ITextModel } from 'vs/editor/common/model';
import { DEFAULT_WORD_REGEXP, ensureValidWordDefinition } from 'vs/editor/common/model/wordHelper';
import { LanguageId, LanguageIdentifier } from 'vs/editor/common/modes';
import { EnterAction, FoldingRules, IAutoClosingPair, IndentAction, IndentationRule, LanguageConfiguration, StandardAutoClosingPairConditional, CompleteEnterAction, AutoClosingPairs, CharacterPair, ExplicitLanguageConfiguration } from 'vs/editor/common/modes/languageConfiguration';
import { createScopedLineTokens, ScopedLineTokens } from 'vs/editor/common/modes/supports';
import { CharacterPairSupport } from 'vs/editor/common/modes/supports/characterPair';
@ -35,8 +34,8 @@ export interface ICommentsConfiguration {
export interface IVirtualModel {
getLineTokens(lineNumber: number): LineTokens;
getLanguageIdentifier(): LanguageIdentifier;
getLanguageIdAtPosition(lineNumber: number, column: number): LanguageId;
getLanguageId(): string;
getLanguageIdAtPosition(lineNumber: number, column: number): string;
getLineContent(lineNumber: number): string;
}
@ -50,14 +49,14 @@ export interface ILanguageConfigurationService {
readonly _serviceBrand: undefined;
onDidChange: Event<LanguageConfigurationServiceChangeEvent>;
getLanguageConfiguration(languageId: LanguageId): ResolvedLanguageConfiguration;
getLanguageConfiguration(languageId: string): ResolvedLanguageConfiguration;
}
export class LanguageConfigurationServiceChangeEvent {
constructor(public readonly languageIdentifier: LanguageIdentifier | undefined) { }
constructor(public readonly languageId: string | undefined) { }
public affects(languageIdentifier: LanguageIdentifier): boolean {
return !this.languageIdentifier ? true : this.languageIdentifier.id === languageIdentifier.id;
public affects(languageId: string): boolean {
return !this.languageId ? true : this.languageId === languageId;
}
}
@ -69,7 +68,7 @@ export class LanguageConfigurationService extends Disposable implements ILanguag
private readonly onDidChangeEmitter = this._register(new Emitter<LanguageConfigurationServiceChangeEvent>());
public readonly onDidChange = this.onDidChangeEmitter.event;
private readonly configurations = new Map<LanguageId, ResolvedLanguageConfiguration>();
private readonly configurations = new Map<string, ResolvedLanguageConfiguration>();
constructor(
@IConfigurationService private readonly configurationService: IConfigurationService,
@ -87,28 +86,28 @@ export class LanguageConfigurationService extends Disposable implements ILanguag
.filter(([overrideLangName, keys]) =>
keys.some((k) => languageConfigKeys.has(k))
)
.map(([overrideLangName]) => this.modeService.getLanguageIdentifier(overrideLangName));
.map(([overrideLangName]) => this.modeService.validateLanguageId(overrideLangName));
if (globalConfigChanged) {
this.configurations.clear();
this.onDidChangeEmitter.fire(new LanguageConfigurationServiceChangeEvent(undefined));
} else {
for (const languageIdentifier of localConfigChanged) {
if (languageIdentifier) {
this.configurations.delete(languageIdentifier.id);
this.onDidChangeEmitter.fire(new LanguageConfigurationServiceChangeEvent(languageIdentifier));
for (const languageId of localConfigChanged) {
if (languageId) {
this.configurations.delete(languageId);
this.onDidChangeEmitter.fire(new LanguageConfigurationServiceChangeEvent(languageId));
}
}
}
}));
this._register(LanguageConfigurationRegistry.onDidChange((e) => {
this.configurations.delete(e.languageIdentifier.id);
this.onDidChangeEmitter.fire(new LanguageConfigurationServiceChangeEvent(e.languageIdentifier));
this.configurations.delete(e.languageId);
this.onDidChangeEmitter.fire(new LanguageConfigurationServiceChangeEvent(e.languageId));
}));
}
public getLanguageConfiguration(languageId: LanguageId): ResolvedLanguageConfiguration {
public getLanguageConfiguration(languageId: string): ResolvedLanguageConfiguration {
let result = this.configurations.get(languageId);
if (!result) {
result = computeConfig(languageId, this.configurationService, this.modeService);
@ -119,23 +118,23 @@ export class LanguageConfigurationService extends Disposable implements ILanguag
}
function computeConfig(
languageId: LanguageId,
languageId: string,
configurationService: IConfigurationService,
modeService: IModeService,
): ResolvedLanguageConfiguration {
let languageConfig = LanguageConfigurationRegistry.getLanguageConfiguration(languageId);
if (!languageConfig) {
const languageIdentifier = modeService.getLanguageIdentifier(languageId);
if (!languageIdentifier) {
const validLanguageId = modeService.validateLanguageId(languageId);
if (!validLanguageId) {
throw new Error('Unexpected languageId');
}
languageConfig = new ResolvedLanguageConfiguration(languageIdentifier, {});
languageConfig = new ResolvedLanguageConfiguration(validLanguageId, {});
}
const customizedConfig = getCustomizedLanguageConfig(languageConfig.languageIdentifier, configurationService);
const customizedConfig = getCustomizedLanguageConfig(languageConfig.languageId, configurationService);
const data = combineLanguageConfigurations([languageConfig.underlyingConfig, customizedConfig]);
const config = new ResolvedLanguageConfiguration(languageConfig.languageIdentifier, data);
const config = new ResolvedLanguageConfiguration(languageConfig.languageId, data);
return config;
}
@ -144,13 +143,13 @@ const customizedLanguageConfigKeys = {
colorizedBracketPairs: 'editor.language.colorizedBracketPairs'
};
function getCustomizedLanguageConfig(languageIdentifier: LanguageIdentifier, configurationService: IConfigurationService): LanguageConfiguration {
function getCustomizedLanguageConfig(languageId: string, configurationService: IConfigurationService): LanguageConfiguration {
const brackets = configurationService.getValue(customizedLanguageConfigKeys.brackets, {
overrideIdentifier: languageIdentifier.language,
overrideIdentifier: languageId,
});
const colorizedBracketPairs = configurationService.getValue(customizedLanguageConfigKeys.colorizedBracketPairs, {
overrideIdentifier: languageIdentifier.language,
overrideIdentifier: languageId,
});
return {
@ -172,11 +171,11 @@ function validateBracketPairs(data: unknown): CharacterPair[] | undefined {
}
export class LanguageConfigurationChangeEvent {
constructor(public readonly languageIdentifier: LanguageIdentifier) { }
constructor(public readonly languageId: string) { }
}
export class LanguageConfigurationRegistryImpl {
private readonly _entries = new Map<LanguageId, ComposedLanguageConfiguration>();
private readonly _entries = new Map<string, ComposedLanguageConfiguration>();
private readonly _onDidChange = new Emitter<LanguageConfigurationChangeEvent>();
public readonly onDidChange: Event<LanguageConfigurationChangeEvent> = this._onDidChange.event;
@ -184,35 +183,35 @@ export class LanguageConfigurationRegistryImpl {
/**
* @param priority Use a higher number for higher priority
*/
public register(languageIdentifier: LanguageIdentifier, configuration: LanguageConfiguration, priority: number = 0): IDisposable {
let entries = this._entries.get(languageIdentifier.id);
public register(languageId: string, configuration: LanguageConfiguration, priority: number = 0): IDisposable {
let entries = this._entries.get(languageId);
if (!entries) {
entries = new ComposedLanguageConfiguration(languageIdentifier);
this._entries.set(languageIdentifier.id, entries);
entries = new ComposedLanguageConfiguration(languageId);
this._entries.set(languageId, entries);
}
const disposable = entries.register(configuration, priority);
this._onDidChange.fire(new LanguageConfigurationChangeEvent(languageIdentifier));
this._onDidChange.fire(new LanguageConfigurationChangeEvent(languageId));
return toDisposable(() => {
disposable.dispose();
this._onDidChange.fire(new LanguageConfigurationChangeEvent(languageIdentifier));
this._onDidChange.fire(new LanguageConfigurationChangeEvent(languageId));
});
}
public getLanguageConfiguration(languageId: LanguageId): ResolvedLanguageConfiguration | null {
public getLanguageConfiguration(languageId: string): ResolvedLanguageConfiguration | null {
let entries = this._entries.get(languageId);
return entries?.getResolvedConfiguration() || null;
}
public getIndentationRules(languageId: LanguageId): IndentationRule | null {
public getIndentationRules(languageId: string): IndentationRule | null {
const value = this.getLanguageConfiguration(languageId);
return value ? value.indentationRules || null : null;
}
// begin electricCharacter
private _getElectricCharacterSupport(languageId: LanguageId): BracketElectricCharacterSupport | null {
private _getElectricCharacterSupport(languageId: string): BracketElectricCharacterSupport | null {
let value = this.getLanguageConfiguration(languageId);
if (!value) {
return null;
@ -220,7 +219,7 @@ export class LanguageConfigurationRegistryImpl {
return value.electricCharacter || null;
}
public getElectricCharacters(languageId: LanguageId): string[] {
public getElectricCharacters(languageId: string): string[] {
let electricCharacterSupport = this._getElectricCharacterSupport(languageId);
if (!electricCharacterSupport) {
return [];
@ -242,7 +241,7 @@ export class LanguageConfigurationRegistryImpl {
// end electricCharacter
public getComments(languageId: LanguageId): ICommentsConfiguration | null {
public getComments(languageId: string): ICommentsConfiguration | null {
let value = this.getLanguageConfiguration(languageId);
if (!value) {
return null;
@ -252,7 +251,7 @@ export class LanguageConfigurationRegistryImpl {
// begin characterPair
private _getCharacterPairSupport(languageId: LanguageId): CharacterPairSupport | null {
private _getCharacterPairSupport(languageId: string): CharacterPairSupport | null {
let value = this.getLanguageConfiguration(languageId);
if (!value) {
return null;
@ -260,12 +259,12 @@ export class LanguageConfigurationRegistryImpl {
return value.characterPair || null;
}
public getAutoClosingPairs(languageId: LanguageId): AutoClosingPairs {
public getAutoClosingPairs(languageId: string): AutoClosingPairs {
const characterPairSupport = this._getCharacterPairSupport(languageId);
return new AutoClosingPairs(characterPairSupport ? characterPairSupport.getAutoClosingPairs() : []);
}
public getAutoCloseBeforeSet(languageId: LanguageId): string {
public getAutoCloseBeforeSet(languageId: string): string {
let characterPairSupport = this._getCharacterPairSupport(languageId);
if (!characterPairSupport) {
return CharacterPairSupport.DEFAULT_AUTOCLOSE_BEFORE_LANGUAGE_DEFINED;
@ -273,7 +272,7 @@ export class LanguageConfigurationRegistryImpl {
return characterPairSupport.getAutoCloseBeforeSet();
}
public getSurroundingPairs(languageId: LanguageId): IAutoClosingPair[] {
public getSurroundingPairs(languageId: string): IAutoClosingPair[] {
let characterPairSupport = this._getCharacterPairSupport(languageId);
if (!characterPairSupport) {
return [];
@ -288,7 +287,7 @@ export class LanguageConfigurationRegistryImpl {
// end characterPair
public getWordDefinition(languageId: LanguageId): RegExp {
public getWordDefinition(languageId: string): RegExp {
let value = this.getLanguageConfiguration(languageId);
if (!value) {
return ensureValidWordDefinition(null);
@ -296,8 +295,8 @@ export class LanguageConfigurationRegistryImpl {
return ensureValidWordDefinition(value.wordDefinition || null);
}
public getWordDefinitions(): [LanguageId, RegExp][] {
let result: [LanguageId, RegExp][] = [];
public getWordDefinitions(): [string, RegExp][] {
let result: [string, RegExp][] = [];
for (const [language, entries] of this._entries) {
const value = entries.getResolvedConfiguration();
if (value) {
@ -307,7 +306,7 @@ export class LanguageConfigurationRegistryImpl {
return result;
}
public getFoldingRules(languageId: LanguageId): FoldingRules {
public getFoldingRules(languageId: string): FoldingRules {
let value = this.getLanguageConfiguration(languageId);
if (!value) {
return {};
@ -317,7 +316,7 @@ export class LanguageConfigurationRegistryImpl {
// begin Indent Rules
public getIndentRulesSupport(languageId: LanguageId): IndentRulesSupport | null {
public getIndentRulesSupport(languageId: string): IndentRulesSupport | null {
let value = this.getLanguageConfiguration(languageId);
if (!value) {
return null;
@ -372,7 +371,7 @@ export class LanguageConfigurationRegistryImpl {
return null;
}
const indentRulesSupport = this.getIndentRulesSupport(model.getLanguageIdentifier().id);
const indentRulesSupport = this.getIndentRulesSupport(model.getLanguageId());
if (!indentRulesSupport) {
return null;
}
@ -491,7 +490,7 @@ export class LanguageConfigurationRegistryImpl {
}
}
public getGoodIndentForLine(autoIndent: EditorAutoIndentStrategy, virtualModel: IVirtualModel, languageId: LanguageId, lineNumber: number, indentConverter: IIndentConverter): string | null {
public getGoodIndentForLine(autoIndent: EditorAutoIndentStrategy, virtualModel: IVirtualModel, languageId: string, lineNumber: number, indentConverter: IIndentConverter): string | null {
if (autoIndent < EditorAutoIndentStrategy.Full) {
return null;
}
@ -598,8 +597,8 @@ export class LanguageConfigurationRegistryImpl {
getLineTokens: (lineNumber: number) => {
return model.getLineTokens(lineNumber);
},
getLanguageIdentifier: () => {
return model.getLanguageIdentifier();
getLanguageId: () => {
return model.getLanguageId();
},
getLanguageIdAtPosition: (lineNumber: number, column: number) => {
return model.getLanguageIdAtPosition(lineNumber, column);
@ -693,7 +692,7 @@ export class LanguageConfigurationRegistryImpl {
}
public getIndentMetadata(model: ITextModel, lineNumber: number): number | null {
const indentRulesSupport = this.getIndentRulesSupport(model.getLanguageIdentifier().id);
const indentRulesSupport = this.getIndentRulesSupport(model.getLanguageId());
if (!indentRulesSupport) {
return null;
}
@ -790,7 +789,7 @@ export class LanguageConfigurationRegistryImpl {
// end onEnter
public getBracketsSupport(languageId: LanguageId): RichEditBrackets | null {
public getBracketsSupport(languageId: string): RichEditBrackets | null {
const value = this.getLanguageConfiguration(languageId);
if (!value) {
return null;
@ -798,7 +797,7 @@ export class LanguageConfigurationRegistryImpl {
return value.brackets || null;
}
public getColorizedBracketPairs(languageId: LanguageId): readonly CharacterPair[] {
public getColorizedBracketPairs(languageId: string): readonly CharacterPair[] {
return this.getLanguageConfiguration(languageId)?.characterPair.getColorizedBrackets() || [];
}
}
@ -810,7 +809,7 @@ class ComposedLanguageConfiguration {
private _order: number;
private _resolved: ResolvedLanguageConfiguration | null = null;
constructor(public readonly languageIdentifier: LanguageIdentifier) {
constructor(public readonly languageId: string) {
this._entries = [];
this._order = 0;
this._resolved = null;
@ -843,7 +842,7 @@ class ComposedLanguageConfiguration {
const config = this._resolve();
if (config) {
this._resolved = new ResolvedLanguageConfiguration(
this.languageIdentifier,
this.languageId,
config
);
}
@ -926,7 +925,7 @@ export class ResolvedLanguageConfiguration {
public readonly foldingRules: FoldingRules;
constructor(
public readonly languageIdentifier: LanguageIdentifier,
public readonly languageId: string,
public readonly underlyingConfig: LanguageConfiguration
) {
this._brackets = null;
@ -959,7 +958,7 @@ export class ResolvedLanguageConfiguration {
public get brackets(): RichEditBrackets | null {
if (!this._brackets && this.underlyingConfig.brackets) {
this._brackets = new RichEditBrackets(
this.languageIdentifier,
this.languageId,
this.underlyingConfig.brackets
);
}

4
src/vs/editor/common/modes/languageFeatureRegistry.ts
View file

@ -132,7 +132,7 @@ export class LanguageFeatureRegistry<T> {
let candidate = {
uri: model.uri.toString(),
language: model.getLanguageIdentifier().language
language: model.getLanguageId()
};
if (this._lastCandidate
@ -146,7 +146,7 @@ export class LanguageFeatureRegistry<T> {
this._lastCandidate = candidate;
for (let entry of this._entries) {
entry._score = score(entry.selector, model.uri, model.getLanguageIdentifier().language, shouldSynchronizeModel(model));
entry._score = score(entry.selector, model.uri, model.getLanguageId(), shouldSynchronizeModel(model));
if (isExclusive(entry.selector) && entry._score > 0) {
// support for one exclusive selector that overwrites

4
src/vs/editor/common/modes/modesRegistry.ts
View file

@ -5,7 +5,6 @@
import * as nls from 'vs/nls';
import { Emitter, Event } from 'vs/base/common/event';
import { LanguageId, LanguageIdentifier } from 'vs/editor/common/modes';
import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry';
import { ILanguageExtensionPoint } from 'vs/editor/common/services/modeService';
import { Registry } from 'vs/platform/registry/common/platform';
@ -60,7 +59,6 @@ Registry.add(Extensions.ModesRegistry, ModesRegistry);
export const PLAINTEXT_MODE_ID = 'plaintext';
export const PLAINTEXT_EXTENSION = '.txt';
export const PLAINTEXT_LANGUAGE_IDENTIFIER = new LanguageIdentifier(PLAINTEXT_MODE_ID, LanguageId.PlainText);
ModesRegistry.registerLanguage({
id: PLAINTEXT_MODE_ID,
@ -68,7 +66,7 @@ ModesRegistry.registerLanguage({
aliases: [nls.localize('plainText.alias', "Plain Text"), 'text'],
mimetypes: [Mimes.text]
});
LanguageConfigurationRegistry.register(PLAINTEXT_LANGUAGE_IDENTIFIER, {
LanguageConfigurationRegistry.register(PLAINTEXT_MODE_ID, {
brackets: [
['(', ')'],
['[', ']'],

4
src/vs/editor/common/modes/nullMode.ts
View file

@ -4,7 +4,7 @@
*--------------------------------------------------------------------------------------------*/
import { Token, TokenizationResult, TokenizationResult2 } from 'vs/editor/common/core/token';
import { ColorId, FontStyle, IState, LanguageId, LanguageIdentifier, MetadataConsts, StandardTokenType } from 'vs/editor/common/modes';
import { ColorId, FontStyle, IState, LanguageId, MetadataConsts, StandardTokenType } from 'vs/editor/common/modes';
class NullStateImpl implements IState {
@ -21,8 +21,6 @@ export const NULL_STATE: IState = new NullStateImpl();
export const NULL_MODE_ID = 'vs.editor.nullMode';
export const NULL_LANGUAGE_IDENTIFIER = new LanguageIdentifier(NULL_MODE_ID, LanguageId.Null);
export function nullTokenize(modeId: string, buffer: string, state: IState, deltaOffset: number): TokenizationResult {
return new TokenizationResult([new Token(deltaOffset, '', modeId)], state);
}

12
src/vs/editor/common/modes/supports.ts
View file

@ -4,7 +4,7 @@
*--------------------------------------------------------------------------------------------*/
import { LineTokens } from 'vs/editor/common/core/lineTokens';
import * as modes from 'vs/editor/common/modes';
import { StandardTokenType } from 'vs/editor/common/modes';
export function createScopedLineTokens(context: LineTokens, offset: number): ScopedLineTokens {
let tokenCount = context.getCount();
@ -34,7 +34,7 @@ export function createScopedLineTokens(context: LineTokens, offset: number): Sco
export class ScopedLineTokens {
_scopedLineTokensBrand: void = undefined;
public readonly languageId: modes.LanguageId;
public readonly languageId: string;
private readonly _actual: LineTokens;
private readonly _firstTokenIndex: number;
private readonly _lastTokenIndex: number;
@ -43,7 +43,7 @@ export class ScopedLineTokens {
constructor(
actual: LineTokens,
languageId: modes.LanguageId,
languageId: string,
firstTokenIndex: number,
lastTokenIndex: number,
firstCharOffset: number,
@ -75,15 +75,15 @@ export class ScopedLineTokens {
return this._actual.findTokenIndexAtOffset(offset + this.firstCharOffset) - this._firstTokenIndex;
}
public getStandardTokenType(tokenIndex: number): modes.StandardTokenType {
public getStandardTokenType(tokenIndex: number): StandardTokenType {
return this._actual.getStandardTokenType(tokenIndex + this._firstTokenIndex);
}
}
const enum IgnoreBracketsInTokens {
value = modes.StandardTokenType.Comment | modes.StandardTokenType.String | modes.StandardTokenType.RegEx
value = StandardTokenType.Comment | StandardTokenType.String | StandardTokenType.RegEx
}
export function ignoreBracketsInToken(standardTokenType: modes.StandardTokenType): boolean {
export function ignoreBracketsInToken(standardTokenType: StandardTokenType): boolean {
return (standardTokenType & IgnoreBracketsInTokens.value) !== 0;
}

11
src/vs/editor/common/modes/supports/richEditBrackets.ts
View file

@ -6,7 +6,6 @@
import * as strings from 'vs/base/common/strings';
import * as stringBuilder from 'vs/editor/common/core/stringBuilder';
import { Range } from 'vs/editor/common/core/range';
import { LanguageIdentifier } from 'vs/editor/common/modes';
import { CharacterPair } from 'vs/editor/common/modes/languageConfiguration';
interface InternalBracket {
@ -32,7 +31,7 @@ interface InternalBracket {
export class RichEditBracket {
_richEditBracketBrand: void = undefined;
readonly languageIdentifier: LanguageIdentifier;
readonly languageId: string;
/**
* A 0-based consecutive unique identifier for this bracket pair.
* If a language has 5 bracket pairs, out of which 2 are grouped together,
@ -78,8 +77,8 @@ export class RichEditBracket {
private readonly _openSet: Set<string>;
private readonly _closeSet: Set<string>;
constructor(languageIdentifier: LanguageIdentifier, index: number, open: string[], close: string[], forwardRegex: RegExp, reversedRegex: RegExp) {
this.languageIdentifier = languageIdentifier;
constructor(languageId: string, index: number, open: string[], close: string[], forwardRegex: RegExp, reversedRegex: RegExp) {
this.languageId = languageId;
this.index = index;
this.open = open;
this.close = close;
@ -215,12 +214,12 @@ export class RichEditBrackets {
*/
public readonly textIsOpenBracket: { [text: string]: boolean; };
constructor(languageIdentifier: LanguageIdentifier, _brackets: readonly CharacterPair[]) {
constructor(languageId: string, _brackets: readonly CharacterPair[]) {
const brackets = groupFuzzyBrackets(_brackets);
this.brackets = brackets.map((b, index) => {
return new RichEditBracket(
languageIdentifier,
languageId,
index,
b.open,
b.close,

18
src/vs/editor/common/modes/textToHtmlTokenizer.ts
View file

@ -7,7 +7,7 @@ import { CharCode } from 'vs/base/common/charCode';
import * as strings from 'vs/base/common/strings';
import { IViewLineTokens, LineTokens } from 'vs/editor/common/core/lineTokens';
import { TokenizationResult2 } from 'vs/editor/common/core/token';
import { IState, LanguageId } from 'vs/editor/common/modes';
import { ILanguageIdCodec, IState, LanguageId } from 'vs/editor/common/modes';
import { NULL_STATE, nullTokenize2 } from 'vs/editor/common/modes/nullMode';
export interface IReducedTokenizationSupport {
@ -20,8 +20,8 @@ const fallback: IReducedTokenizationSupport = {
tokenize2: (buffer: string, hasEOL: boolean, state: IState, deltaOffset: number) => nullTokenize2(LanguageId.Null, buffer, state, deltaOffset)
};
export function tokenizeToString(text: string, tokenizationSupport: IReducedTokenizationSupport = fallback): string {
return _tokenizeToString(text, tokenizationSupport || fallback);
export function tokenizeToString(text: string, languageIdCodec: ILanguageIdCodec, tokenizationSupport: IReducedTokenizationSupport = fallback): string {
return _tokenizeToString(text, languageIdCodec, tokenizationSupport || fallback);
}
export function tokenizeLineToHTML(text: string, viewLineTokens: IViewLineTokens, colorMap: string[], startOffset: number, endOffset: number, tabSize: number, useNbsp: boolean): string {
@ -120,21 +120,21 @@ export function tokenizeLineToHTML(text: string, viewLineTokens: IViewLineTokens
return result;
}
function _tokenizeToString(text: string, tokenizationSupport: IReducedTokenizationSupport): string {
function _tokenizeToString(text: string, languageIdCodec: ILanguageIdCodec, tokenizationSupport: IReducedTokenizationSupport): string {
let result = `<div class="monaco-tokenized-source">`;
let lines = strings.splitLines(text);
const lines = strings.splitLines(text);
let currentState = tokenizationSupport.getInitialState();
for (let i = 0, len = lines.length; i < len; i++) {
let line = lines[i];
const line = lines[i];
if (i > 0) {
result += `<br/>`;
}
let tokenizationResult = tokenizationSupport.tokenize2(line, true, currentState, 0);
const tokenizationResult = tokenizationSupport.tokenize2(line, true, currentState, 0);
LineTokens.convertToEndOffset(tokenizationResult.tokens, line.length);
let lineTokens = new LineTokens(tokenizationResult.tokens, line);
let viewLineTokens = lineTokens.inflate();
const lineTokens = new LineTokens(tokenizationResult.tokens, line, languageIdCodec);
const viewLineTokens = lineTokens.inflate();
let startOffset = 0;
for (let j = 0, lenJ = viewLineTokens.getCount(); j < lenJ; j++) {

8
src/vs/editor/common/services/editorWorkerServiceImpl.ts
View file

@ -168,7 +168,7 @@ class WordBasedCompletionItemProvider implements modes.CompletionItemProvider {
if (candidate === model) {
models.unshift(candidate.uri);
} else if (config.wordBasedSuggestionsMode === 'allDocuments' || candidate.getLanguageIdentifier().id === model.getLanguageIdentifier().id) {
} else if (config.wordBasedSuggestionsMode === 'allDocuments' || candidate.getLanguageId() === model.getLanguageId()) {
models.push(candidate.uri);
}
}
@ -178,7 +178,7 @@ class WordBasedCompletionItemProvider implements modes.CompletionItemProvider {
return undefined; // File too large, no other files
}
const wordDefRegExp = LanguageConfigurationRegistry.getWordDefinition(model.getLanguageIdentifier().id);
const wordDefRegExp = LanguageConfigurationRegistry.getWordDefinition(model.getLanguageId());
const word = model.getWordAtPosition(position);
const replace = !word ? Range.fromPositions(position) : new Range(position.lineNumber, word.startColumn, position.lineNumber, word.endColumn);
const insert = replace.setEndPosition(position.lineNumber, position.column);
@ -503,7 +503,7 @@ export class EditorWorkerClient extends Disposable implements IEditorWorkerClien
if (!model) {
return Promise.resolve(null);
}
let wordDefRegExp = LanguageConfigurationRegistry.getWordDefinition(model.getLanguageIdentifier().id);
let wordDefRegExp = LanguageConfigurationRegistry.getWordDefinition(model.getLanguageId());
let wordDef = wordDefRegExp.source;
let wordDefFlags = regExpFlags(wordDefRegExp);
return proxy.computeWordRanges(resource.toString(), range, wordDef, wordDefFlags);
@ -516,7 +516,7 @@ export class EditorWorkerClient extends Disposable implements IEditorWorkerClien
if (!model) {
return null;
}
let wordDefRegExp = LanguageConfigurationRegistry.getWordDefinition(model.getLanguageIdentifier().id);
let wordDefRegExp = LanguageConfigurationRegistry.getWordDefinition(model.getLanguageId());
let wordDef = wordDefRegExp.source;
let wordDefFlags = regExpFlags(wordDefRegExp);
return proxy.navigateValueSet(resource.toString(), range, up, wordDef, wordDefFlags);

2
src/vs/editor/common/services/getIconClasses.ts
View file

@ -85,7 +85,7 @@ function detectModeId(modelService: IModelService, modeService: IModeService, re
else {
const model = modelService.getModel(resource);
if (model) {
modeId = model.getModeId();
modeId = model.getLanguageId();
}
}

123
src/vs/editor/common/services/languagesRegistry.ts
View file

@ -9,9 +9,9 @@ import { Disposable } from 'vs/base/common/lifecycle';
import * as mime from 'vs/base/common/mime';
import * as strings from 'vs/base/common/strings';
import { URI } from 'vs/base/common/uri';
import { LanguageId, LanguageIdentifier } from 'vs/editor/common/modes';
import { ModesRegistry } from 'vs/editor/common/modes/modesRegistry';
import { NULL_LANGUAGE_IDENTIFIER, NULL_MODE_ID } from 'vs/editor/common/modes/nullMode';
import { ILanguageIdCodec, LanguageId } from 'vs/editor/common/modes';
import { ModesRegistry, PLAINTEXT_MODE_ID } from 'vs/editor/common/modes/modesRegistry';
import { NULL_MODE_ID } from 'vs/editor/common/modes/nullMode';
import { ILanguageExtensionPoint } from 'vs/editor/common/services/modeService';
import { Extensions, IConfigurationRegistry } from 'vs/platform/configuration/common/configurationRegistry';
import { Registry } from 'vs/platform/registry/common/platform';
@ -19,7 +19,7 @@ import { Registry } from 'vs/platform/registry/common/platform';
const hasOwnProperty = Object.prototype.hasOwnProperty;
export interface IResolvedLanguage {
identifier: LanguageIdentifier;
identifier: string;
name: string | null;
mimetypes: string[];
aliases: string[];
@ -28,31 +28,62 @@ export interface IResolvedLanguage {
configurationFiles: URI[];
}
export class LanguageIdCodec implements ILanguageIdCodec {
private _nextLanguageId: number;
private readonly _languageIdToLanguage: string[] = [];
private readonly _languageToLanguageId = new Map<string, number>();
constructor() {
this._register(NULL_MODE_ID, LanguageId.Null);
this._register(PLAINTEXT_MODE_ID, LanguageId.PlainText);
this._nextLanguageId = 2;
}
private _register(language: string, languageId: LanguageId): void {
this._languageIdToLanguage[languageId] = language;
this._languageToLanguageId.set(language, languageId);
}
public register(language: string): void {
if (this._languageToLanguageId.has(language)) {
return;
}
const languageId = this._nextLanguageId++;
this._register(language, languageId);
}
public encodeLanguageId(languageId: string): LanguageId {
return this._languageToLanguageId.get(languageId) || LanguageId.Null;
}
public decodeLanguageId(languageId: LanguageId): string {
return this._languageIdToLanguage[languageId] || NULL_MODE_ID;
}
}
export class LanguagesRegistry extends Disposable {
static instanceCount = 0;
private readonly _onDidChange: Emitter<void> = this._register(new Emitter<void>());
public readonly onDidChange: Event<void> = this._onDidChange.event;
private readonly _warnOnOverwrite: boolean;
private _nextLanguageId2: number;
private readonly _languageIdToLanguage: string[];
private readonly _languageToLanguageId: { [id: string]: number; };
public readonly languageIdCodec: LanguageIdCodec;
private _languages: { [id: string]: IResolvedLanguage; };
private _mimeTypesMap: { [mimeType: string]: LanguageIdentifier; };
private _nameMap: { [name: string]: LanguageIdentifier; };
private _lowercaseNameMap: { [name: string]: LanguageIdentifier; };
private _mimeTypesMap: { [mimeType: string]: string; };
private _nameMap: { [name: string]: string; };
private _lowercaseNameMap: { [name: string]: string; };
constructor(useModesRegistry = true, warnOnOverwrite = false) {
super();
LanguagesRegistry.instanceCount++;
this._warnOnOverwrite = warnOnOverwrite;
this._nextLanguageId2 = 1;
this._languageIdToLanguage = [];
this._languageToLanguageId = Object.create(null);
this.languageIdCodec = new LanguageIdCodec();
this._languages = {};
this._mimeTypesMap = {};
this._nameMap = {};
@ -60,16 +91,25 @@ export class LanguagesRegistry extends Disposable {
if (useModesRegistry) {
this._initializeFromRegistry();
this._register(ModesRegistry.onDidChangeLanguages((m) => this._initializeFromRegistry()));
this._register(ModesRegistry.onDidChangeLanguages((m) => {
// console.log(`onDidChangeLanguages - inst count: ${LanguagesRegistry.instanceCount}`);
this._initializeFromRegistry();
}));
}
}
override dispose() {
LanguagesRegistry.instanceCount--;
super.dispose();
}
private _initializeFromRegistry(): void {
this._languages = {};
this._mimeTypesMap = {};
this._nameMap = {};
this._lowercaseNameMap = {};
mime.clearTextMimes();
const desc = ModesRegistry.getLanguages();
this._registerLanguages(desc);
}
@ -102,18 +142,6 @@ export class LanguagesRegistry extends Disposable {
this._onDidChange.fire();
}
private _getLanguageId(language: string): number {
if (this._languageToLanguageId[language]) {
return this._languageToLanguageId[language];
}
const languageId = this._nextLanguageId2++;
this._languageIdToLanguage[languageId] = language;
this._languageToLanguageId[language] = languageId;
return languageId;
}
private _registerLanguage(lang: ILanguageExtensionPoint): void {
const langId = lang.id;
@ -121,9 +149,9 @@ export class LanguagesRegistry extends Disposable {
if (hasOwnProperty.call(this._languages, langId)) {
resolvedLanguage = this._languages[langId];
} else {
const languageId = this._getLanguageId(langId);
this.languageIdCodec.register(langId);
resolvedLanguage = {
identifier: new LanguageIdentifier(langId, languageId),
identifier: langId,
name: null,
mimetypes: [],
aliases: [],
@ -257,7 +285,7 @@ export class LanguagesRegistry extends Disposable {
if (!hasOwnProperty.call(this._lowercaseNameMap, languageNameLower)) {
return null;
}
return this._lowercaseNameMap[languageNameLower].language;
return this._lowercaseNameMap[languageNameLower];
}
public getConfigurationFiles(modeId: string): URI[] {
@ -286,7 +314,7 @@ export class LanguagesRegistry extends Disposable {
map((mimeTypeOrId) => mimeTypeOrId.trim()).
map((mimeTypeOrId) => {
if (hasOwnProperty.call(this._mimeTypesMap, mimeTypeOrId)) {
return this._mimeTypesMap[mimeTypeOrId].language;
return this._mimeTypesMap[mimeTypeOrId];
}
return mimeTypeOrId;
}).
@ -296,35 +324,26 @@ export class LanguagesRegistry extends Disposable {
);
}
public getLanguageIdentifier(_modeId: string | LanguageId): LanguageIdentifier | null {
if (_modeId === NULL_MODE_ID || _modeId === LanguageId.Null) {
return NULL_LANGUAGE_IDENTIFIER;
}
let modeId: string;
if (typeof _modeId === 'string') {
modeId = _modeId;
} else {
modeId = this._languageIdToLanguage[_modeId];
if (!modeId) {
return null;
}
public validateLanguageId(modeId: string | null): string | null {
if (!modeId || modeId === NULL_MODE_ID) {
return NULL_MODE_ID;
}
if (!hasOwnProperty.call(this._languages, modeId)) {
return null;
}
return this._languages[modeId].identifier;
return modeId;
}
public getModeIdsFromLanguageName(languageName: string): string[] {
public getModeIdFromLanguageName(languageName: string): string | null {
if (!languageName) {
return [];
return null;
}
if (hasOwnProperty.call(this._nameMap, languageName)) {
return [this._nameMap[languageName].language];
return this._nameMap[languageName];
}
return [];
return null;
}
public getModeIdsFromFilepathOrFirstLine(resource: URI | null, firstLine?: string): string[] {
@ -340,7 +359,7 @@ export class LanguagesRegistry extends Disposable {
return [];
}
const languageId = this._nameMap[languageName];
return this._languages[languageId.language].extensions;
return this._languages[languageId].extensions;
}
public getFilenames(languageName: string): string[] {
@ -348,6 +367,6 @@ export class LanguagesRegistry extends Disposable {
return [];
}
const languageId = this._nameMap[languageName];
return this._languages[languageId.language].filenames;
return this._languages[languageId].filenames;
}
}

12
src/vs/editor/common/services/modeService.ts
View file

@ -5,7 +5,7 @@
import { Event } from 'vs/base/common/event';
import { URI } from 'vs/base/common/uri';
import { LanguageId, LanguageIdentifier } from 'vs/editor/common/modes';
import { ILanguageIdCodec } from 'vs/editor/common/modes';
import { createDecorator } from 'vs/platform/instantiation/common/instantiation';
export const IModeService = createDecorator<IModeService>('modeService');
@ -22,14 +22,16 @@ export interface ILanguageExtensionPoint {
}
export interface ILanguageSelection {
readonly languageIdentifier: LanguageIdentifier;
readonly onDidChange: Event<LanguageIdentifier>;
readonly languageId: string;
readonly onDidChange: Event<string>;
}
export interface IModeService {
readonly _serviceBrand: undefined;
onDidEncounterLanguage: Event<LanguageIdentifier>;
readonly languageIdCodec: ILanguageIdCodec;
onDidEncounterLanguage: Event<string>;
onLanguagesMaybeChanged: Event<void>;
// --- reading
@ -43,7 +45,7 @@ export interface IModeService {
getModeIdForLanguageName(alias: string): string | null;
getModeIdByFilepathOrFirstLine(resource: URI, firstLine?: string): string | null;
getModeId(commaSeparatedMimetypesOrCommaSeparatedIds: string): string | null;
getLanguageIdentifier(modeId: string | LanguageId): LanguageIdentifier | null;
validateLanguageId(modeId: string): string | null;
getConfigurationFiles(modeId: string): URI[];
// --- instantiation

64
src/vs/editor/common/services/modeServiceImpl.ts
View file

@ -6,26 +6,26 @@
import { Emitter, Event } from 'vs/base/common/event';
import { Disposable, IDisposable } from 'vs/base/common/lifecycle';
import { URI } from 'vs/base/common/uri';
import { LanguageId, LanguageIdentifier } from 'vs/editor/common/modes';
import { NULL_LANGUAGE_IDENTIFIER } from 'vs/editor/common/modes/nullMode';
import { NULL_MODE_ID } from 'vs/editor/common/modes/nullMode';
import { LanguagesRegistry } from 'vs/editor/common/services/languagesRegistry';
import { ILanguageSelection, IModeService } from 'vs/editor/common/services/modeService';
import { firstOrDefault } from 'vs/base/common/arrays';
import { ILanguageIdCodec } from 'vs/editor/common/modes';
class LanguageSelection implements ILanguageSelection {
public languageIdentifier: LanguageIdentifier;
public languageId: string;
private readonly _selector: () => LanguageIdentifier;
private readonly _onDidChange: Emitter<LanguageIdentifier>;
public readonly onDidChange: Event<LanguageIdentifier>;
private readonly _selector: () => string;
private readonly _onDidChange: Emitter<string>;
public readonly onDidChange: Event<string>;
constructor(onLanguagesMaybeChanged: Event<void>, selector: () => LanguageIdentifier) {
constructor(onLanguagesMaybeChanged: Event<void>, selector: () => string) {
this._selector = selector;
this.languageIdentifier = this._selector();
this.languageId = this._selector();
let listener: IDisposable;
this._onDidChange = new Emitter<LanguageIdentifier>({
this._onDidChange = new Emitter<string>({
onFirstListenerAdd: () => {
listener = onLanguagesMaybeChanged(() => this._evaluate());
},
@ -37,38 +37,43 @@ class LanguageSelection implements ILanguageSelection {
}
private _evaluate(): void {
let languageIdentifier = this._selector();
if (languageIdentifier.id === this.languageIdentifier.id) {
const languageId = this._selector();
if (languageId === this.languageId) {
// no change
return;
}
this.languageIdentifier = languageIdentifier;
this._onDidChange.fire(this.languageIdentifier);
this.languageId = languageId;
this._onDidChange.fire(this.languageId);
}
}
export class ModeServiceImpl extends Disposable implements IModeService {
public _serviceBrand: undefined;
static instanceCount = 0;
private readonly _encounteredLanguages: Set<string>;
private readonly _registry: LanguagesRegistry;
public readonly languageIdCodec: ILanguageIdCodec;
private readonly _onDidEncounterLanguage = this._register(new Emitter<LanguageIdentifier>());
public readonly onDidEncounterLanguage: Event<LanguageIdentifier> = this._onDidEncounterLanguage.event;
private readonly _onDidEncounterLanguage = this._register(new Emitter<string>());
public readonly onDidEncounterLanguage: Event<string> = this._onDidEncounterLanguage.event;
protected readonly _onLanguagesMaybeChanged = this._register(new Emitter<void>({ leakWarningThreshold: 200 /* https://github.com/microsoft/vscode/issues/119968 */ }));
public readonly onLanguagesMaybeChanged: Event<void> = this._onLanguagesMaybeChanged.event;
constructor(warnOnOverwrite = false) {
super();
ModeServiceImpl.instanceCount++;
this._encounteredLanguages = new Set<string>();
this._registry = this._register(new LanguagesRegistry(true, warnOnOverwrite));
this.languageIdCodec = this._registry.languageIdCodec;
this._register(this._registry.onDidChange(() => this._onLanguagesMaybeChanged.fire()));
}
protected _onReady(): Promise<boolean> {
return Promise.resolve(true);
public override dispose(): void {
ModeServiceImpl.instanceCount--;
super.dispose();
}
public isRegisteredMode(mimetypeOrModeId: string): boolean {
@ -113,8 +118,8 @@ export class ModeServiceImpl extends Disposable implements IModeService {
return firstOrDefault(modeIds, null);
}
public getLanguageIdentifier(modeId: string | LanguageId): LanguageIdentifier | null {
return this._registry.getLanguageIdentifier(modeId);
public validateLanguageId(modeId: string | null): string | null {
return this._registry.validateLanguageId(modeId);
}
public getConfigurationFiles(modeId: string): URI[] {
@ -144,11 +149,11 @@ export class ModeServiceImpl extends Disposable implements IModeService {
});
}
private _createModeAndGetLanguageIdentifier(modeId: string | null): LanguageIdentifier {
private _createModeAndGetLanguageIdentifier(modeId: string | null): string {
// Fall back to plain text if no mode was found
const languageIdentifier = this.getLanguageIdentifier(modeId || 'plaintext') || NULL_LANGUAGE_IDENTIFIER;
this._getOrCreateMode(languageIdentifier.language);
return languageIdentifier;
const languageId = this.validateLanguageId(modeId || 'plaintext') || NULL_MODE_ID;
this._getOrCreateMode(languageId);
return languageId;
}
public triggerMode(commaSeparatedMimetypesOrCommaSeparatedIds: string): void {
@ -157,20 +162,15 @@ export class ModeServiceImpl extends Disposable implements IModeService {
this._getOrCreateMode(modeId || 'plaintext');
}
public waitForLanguageRegistration(): Promise<void> {
return this._onReady().then(() => { });
}
private _getModeIdByLanguageName(languageName: string): string | null {
const modeIds = this._registry.getModeIdsFromLanguageName(languageName);
return firstOrDefault(modeIds, null);
return this._registry.getModeIdFromLanguageName(languageName);
}
private _getOrCreateMode(modeId: string): void {
if (!this._encounteredLanguages.has(modeId)) {
this._encounteredLanguages.add(modeId);
const languageIdentifier = this.getLanguageIdentifier(modeId) || NULL_LANGUAGE_IDENTIFIER;
this._onDidEncounterLanguage.fire(languageIdentifier);
const languageId = this.validateLanguageId(modeId) || NULL_MODE_ID;
this._onDidEncounterLanguage.fire(languageId);
}
}
}

41
src/vs/editor/common/services/modelServiceImpl.ts
View file

@ -14,9 +14,9 @@ import { Range } from 'vs/editor/common/core/range';
import { DefaultEndOfLine, EndOfLinePreference, EndOfLineSequence, IIdentifiedSingleEditOperation, ITextBuffer, ITextBufferFactory, ITextModel, ITextModelCreationOptions } from 'vs/editor/common/model';
import { TextModel, createTextBuffer } from 'vs/editor/common/model/textModel';
import { IModelLanguageChangedEvent, IModelContentChangedEvent } from 'vs/editor/common/model/textModelEvents';
import { LanguageIdentifier, DocumentSemanticTokensProviderRegistry, DocumentSemanticTokensProvider, SemanticTokens, SemanticTokensEdits } from 'vs/editor/common/modes';
import { PLAINTEXT_LANGUAGE_IDENTIFIER } from 'vs/editor/common/modes/modesRegistry';
import { ILanguageSelection } from 'vs/editor/common/services/modeService';
import { DocumentSemanticTokensProviderRegistry, DocumentSemanticTokensProvider, SemanticTokens, SemanticTokensEdits } from 'vs/editor/common/modes';
import { PLAINTEXT_MODE_ID } from 'vs/editor/common/modes/modesRegistry';
import { ILanguageSelection, IModeService } from 'vs/editor/common/services/modeService';
import { IModelService, DocumentTokensProvider } from 'vs/editor/common/services/modelService';
import { ITextResourcePropertiesService } from 'vs/editor/common/services/textResourceConfigurationService';
import { IConfigurationService } from 'vs/platform/configuration/common/configuration';
@ -90,8 +90,8 @@ class ModelData implements IDisposable {
public setLanguage(languageSelection: ILanguageSelection): void {
this._disposeLanguageSelection();
this._languageSelection = languageSelection;
this._languageSelectionListener = this._languageSelection.onDidChange(() => this.model.setMode(languageSelection.languageIdentifier));
this.model.setMode(languageSelection.languageIdentifier);
this._languageSelectionListener = this._languageSelection.onDidChange(() => this.model.setMode(languageSelection.languageId));
this.model.setMode(languageSelection.languageId);
}
}
@ -162,6 +162,7 @@ export class ModelServiceImpl extends Disposable implements IModelService {
@IThemeService private readonly _themeService: IThemeService,
@ILogService private readonly _logService: ILogService,
@IUndoRedoService private readonly _undoRedoService: IUndoRedoService,
@IModeService private readonly _modeService: IModeService,
@ILanguageConfigurationService private readonly _languageConfigurationService: ILanguageConfigurationService
) {
super();
@ -169,7 +170,7 @@ export class ModelServiceImpl extends Disposable implements IModelService {
this._models = {};
this._disposedModels = new Map<string, DisposedModelInfo>();
this._disposedModelsHeapSize = 0;
this._semanticStyling = this._register(new SemanticStyling(this._themeService, this._logService));
this._semanticStyling = this._register(new SemanticStyling(this._themeService, this._modeService, this._logService));
this._register(this._configurationService.onDidChangeConfiguration(() => this._updateModelOptions()));
this._updateModelOptions();
@ -286,7 +287,7 @@ export class ModelServiceImpl extends Disposable implements IModelService {
for (let i = 0, len = keys.length; i < len; i++) {
const modelId = keys[i];
const modelData = this._models[modelId];
const language = modelData.model.getLanguageIdentifier().language;
const language = modelData.model.getLanguageId();
const uri = modelData.model.uri;
const oldOptions = oldOptionsByLanguageAndResource[language + uri];
const newOptions = this.getCreationOptions(language, uri, modelData.model.isForSimpleWidget);
@ -364,16 +365,17 @@ export class ModelServiceImpl extends Disposable implements IModelService {
}
}
private _createModelData(value: string | ITextBufferFactory, languageIdentifier: LanguageIdentifier, resource: URI | undefined, isForSimpleWidget: boolean): ModelData {
private _createModelData(value: string | ITextBufferFactory, languageId: string, resource: URI | undefined, isForSimpleWidget: boolean): ModelData {
// create & save the model
const options = this.getCreationOptions(languageIdentifier.language, resource, isForSimpleWidget);
const options = this.getCreationOptions(languageId, resource, isForSimpleWidget);
const model: TextModel = new TextModel(
value,
options,
languageIdentifier,
languageId,
resource,
this._undoRedoService,
this._languageConfigurationService
this._modeService,
this._languageConfigurationService,
);
if (resource && this._disposedModels.has(MODEL_ID(resource))) {
const disposedModelData = this._removeDisposedModel(resource)!;
@ -420,7 +422,7 @@ export class ModelServiceImpl extends Disposable implements IModelService {
}
public updateModel(model: ITextModel, value: string | ITextBufferFactory): void {
const options = this.getCreationOptions(model.getLanguageIdentifier().language, model.uri, model.isForSimpleWidget);
const options = this.getCreationOptions(model.getLanguageId(), model.uri, model.isForSimpleWidget);
const { textBuffer, disposable } = createTextBuffer(value, options.defaultEOL);
// Return early if the text is already set in that form
@ -496,10 +498,10 @@ export class ModelServiceImpl extends Disposable implements IModelService {
let modelData: ModelData;
if (languageSelection) {
modelData = this._createModelData(value, languageSelection.languageIdentifier, resource, isForSimpleWidget);
modelData = this._createModelData(value, languageSelection.languageId, resource, isForSimpleWidget);
this.setMode(modelData.model, languageSelection);
} else {
modelData = this._createModelData(value, PLAINTEXT_LANGUAGE_IDENTIFIER, resource, isForSimpleWidget);
modelData = this._createModelData(value, PLAINTEXT_MODE_ID, resource, isForSimpleWidget);
}
this._onModelAdded.fire(modelData.model);
@ -616,14 +618,14 @@ export class ModelServiceImpl extends Disposable implements IModelService {
modelData.dispose();
// clean up cache
delete this._modelCreationOptionsByLanguageAndResource[model.getLanguageIdentifier().language + model.uri];
delete this._modelCreationOptionsByLanguageAndResource[model.getLanguageId() + model.uri];
this._onModelRemoved.fire(model);
}
private _onDidChangeLanguage(model: ITextModel, e: IModelLanguageChangedEvent): void {
const oldModeId = e.oldLanguage;
const newModeId = model.getLanguageIdentifier().language;
const newModeId = model.getLanguageId();
const oldOptions = this.getCreationOptions(oldModeId, model.uri, model.isForSimpleWidget);
const newOptions = this.getCreationOptions(newModeId, model.uri, model.isForSimpleWidget);
ModelServiceImpl._setModelOptionsForModel(model, newOptions, oldOptions);
@ -638,7 +640,7 @@ export interface ILineSequence {
export const SEMANTIC_HIGHLIGHTING_SETTING_ID = 'editor.semanticHighlighting';
export function isSemanticColoringEnabled(model: ITextModel, themeService: IThemeService, configurationService: IConfigurationService): boolean {
const setting = configurationService.getValue<IEditorSemanticHighlightingOptions>(SEMANTIC_HIGHLIGHTING_SETTING_ID, { overrideIdentifier: model.getLanguageIdentifier().language, resource: model.uri })?.enabled;
const setting = configurationService.getValue<IEditorSemanticHighlightingOptions>(SEMANTIC_HIGHLIGHTING_SETTING_ID, { overrideIdentifier: model.getLanguageId(), resource: model.uri })?.enabled;
if (typeof setting === 'boolean') {
return setting;
}
@ -702,6 +704,7 @@ class SemanticStyling extends Disposable {
constructor(
private readonly _themeService: IThemeService,
private readonly _modeService: IModeService,
private readonly _logService: ILogService
) {
super();
@ -713,7 +716,7 @@ class SemanticStyling extends Disposable {
public get(provider: DocumentTokensProvider): SemanticTokensProviderStyling {
if (!this._caches.has(provider)) {
this._caches.set(provider, new SemanticTokensProviderStyling(provider.getLegend(), this._themeService, this._logService));
this._caches.set(provider, new SemanticTokensProviderStyling(provider.getLegend(), this._themeService, this._modeService, this._logService));
}
return this._caches.get(provider)!;
}
@ -953,7 +956,7 @@ export class ModelSemanticColoring extends Disposable {
this._currentDocumentResponse = new SemanticTokensResponse(provider, tokens.resultId, tokens.data);
const result = toMultilineTokens2(tokens, styling, this._model.getLanguageIdentifier());
const result = toMultilineTokens2(tokens, styling, this._model.getLanguageId());
// Adjust incoming semantic tokens
if (pendingChanges.length > 0) {

15
src/vs/editor/common/services/semanticTokensProviderStyling.ts
View file

@ -3,10 +3,11 @@
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import { SemanticTokensLegend, TokenMetadata, FontStyle, MetadataConsts, SemanticTokens, LanguageIdentifier } from 'vs/editor/common/modes';
import { SemanticTokensLegend, TokenMetadata, FontStyle, MetadataConsts, SemanticTokens } from 'vs/editor/common/modes';
import { IThemeService } from 'vs/platform/theme/common/themeService';
import { ILogService, LogLevel } from 'vs/platform/log/common/log';
import { MultilineTokens2, SparseEncodedTokens } from 'vs/editor/common/model/tokensStore';
import { IModeService } from 'vs/editor/common/services/modeService';
export const enum SemanticTokensProviderStylingConstants {
NO_STYLING = 0b01111111111111111111111111111111
@ -20,14 +21,16 @@ export class SemanticTokensProviderStyling {
constructor(
private readonly _legend: SemanticTokensLegend,
private readonly _themeService: IThemeService,
private readonly _modeService: IModeService,
private readonly _logService: ILogService
) {
this._hashTable = new HashTable();
this._hasWarnedOverlappingTokens = false;
}
public getMetadata(tokenTypeIndex: number, tokenModifierSet: number, languageId: LanguageIdentifier): number {
const entry = this._hashTable.get(tokenTypeIndex, tokenModifierSet, languageId.id);
public getMetadata(tokenTypeIndex: number, tokenModifierSet: number, languageId: string): number {
const encodedLanguageId = this._modeService.languageIdCodec.encodeLanguageId(languageId);
const entry = this._hashTable.get(tokenTypeIndex, tokenModifierSet, encodedLanguageId);
let metadata: number;
if (entry) {
metadata = entry.metadata;
@ -50,7 +53,7 @@ export class SemanticTokensProviderStyling {
tokenModifiers.push('not-in-legend');
}
const tokenStyle = this._themeService.getColorTheme().getTokenStyleMetadata(tokenType, tokenModifiers, languageId.language);
const tokenStyle = this._themeService.getColorTheme().getTokenStyleMetadata(tokenType, tokenModifiers, languageId);
if (typeof tokenStyle === 'undefined') {
metadata = SemanticTokensProviderStylingConstants.NO_STYLING;
} else {
@ -83,7 +86,7 @@ export class SemanticTokensProviderStyling {
metadata = SemanticTokensProviderStylingConstants.NO_STYLING;
tokenType = 'not-in-legend';
}
this._hashTable.add(tokenTypeIndex, tokenModifierSet, languageId.id, metadata);
this._hashTable.add(tokenTypeIndex, tokenModifierSet, encodedLanguageId, metadata);
if (this._logService.getLevel() === LogLevel.Trace) {
this._logService.trace(`SemanticTokensProviderStyling ${tokenTypeIndex} (${tokenType}) / ${tokenModifierSet} (${tokenModifiers.join(' ')}): foreground ${TokenMetadata.getForeground(metadata)}, fontStyle ${TokenMetadata.getFontStyle(metadata).toString(2)}`);
@ -116,7 +119,7 @@ const enum SemanticColoringConstants {
DesiredMaxAreas = 1024,
}
export function toMultilineTokens2(tokens: SemanticTokens, styling: SemanticTokensProviderStyling, languageId: LanguageIdentifier): MultilineTokens2[] {
export function toMultilineTokens2(tokens: SemanticTokens, styling: SemanticTokensProviderStyling, languageId: string): MultilineTokens2[] {
const srcData = tokens.data;
const tokenCount = (tokens.data.length / 5) | 0;
const tokensPerArea = Math.max(Math.ceil(tokenCount / SemanticColoringConstants.DesiredMaxAreas), SemanticColoringConstants.DesiredTokensPerArea);

2
src/vs/editor/common/services/textResourceConfigurationServiceImpl.ts
View file

@ -109,7 +109,7 @@ export class TextResourceConfigurationService extends Disposable implements ITex
private getLanguage(resource: URI, position: IPosition | null): string | null {
const model = this.modelService.getModel(resource);
if (model) {
return position ? this.modeService.getLanguageIdentifier(model.getLanguageIdAtPosition(position.lineNumber, position.column))!.language : model.getLanguageIdentifier().language;
return position ? model.getLanguageIdAtPosition(position.lineNumber, position.column) : model.getLanguageId();
}
return this.modeService.getModeIdByFilepathOrFirstLine(resource);
}

19
src/vs/editor/common/viewModel/viewModelImpl.ts
View file

@ -15,7 +15,7 @@ import { IConfiguration, IViewState, ScrollType, ICursorState, ICommand, INewScr
import { EndOfLinePreference, IActiveIndentGuideInfo, ITextModel, TrackedRangeStickiness, TextModelResolvedOptions, IIdentifiedSingleEditOperation, ICursorStateComputer, PositionAffinity, IndentGuide, BracketGuideOptions } from 'vs/editor/common/model';
import { ModelDecorationOverviewRulerOptions, ModelDecorationMinimapOptions } from 'vs/editor/common/model/textModel';
import * as textModelEvents from 'vs/editor/common/model/textModelEvents';
import { ColorId, LanguageId, TokenizationRegistry } from 'vs/editor/common/modes';
import { ColorId, TokenizationRegistry } from 'vs/editor/common/modes';
import { tokenizeLineToHTML } from 'vs/editor/common/modes/textToHtmlTokenizer';
import { MinimapTokensColorTracker } from 'vs/editor/common/viewModel/minimapTokensColorTracker';
import * as viewEvents from 'vs/editor/common/view/viewEvents';
@ -32,6 +32,7 @@ import { CursorChangeReason } from 'vs/editor/common/controller/cursorEvents';
import { IWhitespaceChangeAccessor } from 'vs/editor/common/viewLayout/linesLayout';
import { ViewModelEventDispatcher, OutgoingViewModelEvent, FocusChangedEvent, ScrollChangedEvent, ViewZonesChangedEvent, ViewModelEventsCollector, ReadOnlyEditAttemptEvent } from 'vs/editor/common/viewModel/viewModelEventDispatcher';
import { ViewEventHandler } from 'vs/editor/common/viewModel/viewEventHandler';
import { PLAINTEXT_MODE_ID } from 'vs/editor/common/modes/modesRegistry';
const USE_IDENTITY_LINES_COLLECTION = true;
@ -70,7 +71,7 @@ export class ViewModel extends Disposable implements IViewModel {
this.model = model;
this._eventDispatcher = new ViewModelEventDispatcher();
this.onEvent = this._eventDispatcher.onEvent;
this.cursorConfig = new CursorConfiguration(this.model.getLanguageIdentifier(), this.model.getOptions(), this._configuration);
this.cursorConfig = new CursorConfiguration(this.model.getLanguageId(), this.model.getOptions(), this._configuration);
this._tokenizeViewportSoon = this._register(new RunOnceScheduler(() => this.tokenizeViewport(), 50));
this._updateConfigurationViewLineCount = this._register(new RunOnceScheduler(() => this._updateConfigurationViewLineCountNow(), 0));
this._hasFocus = false;
@ -244,7 +245,7 @@ export class ViewModel extends Disposable implements IViewModel {
}
if (CursorConfiguration.shouldRecreate(e)) {
this.cursorConfig = new CursorConfiguration(this.model.getLanguageIdentifier(), this.model.getOptions(), this._configuration);
this.cursorConfig = new CursorConfiguration(this.model.getLanguageId(), this.model.getOptions(), this._configuration);
this._cursor.updateConfiguration(this.cursorConfig);
}
}
@ -405,12 +406,12 @@ export class ViewModel extends Disposable implements IViewModel {
this._register(this.model.onDidChangeLanguageConfiguration((e) => {
this._eventDispatcher.emitSingleViewEvent(new viewEvents.ViewLanguageConfigurationEvent());
this.cursorConfig = new CursorConfiguration(this.model.getLanguageIdentifier(), this.model.getOptions(), this._configuration);
this.cursorConfig = new CursorConfiguration(this.model.getLanguageId(), this.model.getOptions(), this._configuration);
this._cursor.updateConfiguration(this.cursorConfig);
}));
this._register(this.model.onDidChangeLanguage((e) => {
this.cursorConfig = new CursorConfiguration(this.model.getLanguageIdentifier(), this.model.getOptions(), this._configuration);
this.cursorConfig = new CursorConfiguration(this.model.getLanguageId(), this.model.getOptions(), this._configuration);
this._cursor.updateConfiguration(this.cursorConfig);
}));
@ -431,7 +432,7 @@ export class ViewModel extends Disposable implements IViewModel {
this._updateConfigurationViewLineCount.schedule();
}
this.cursorConfig = new CursorConfiguration(this.model.getLanguageIdentifier(), this.model.getOptions(), this._configuration);
this.cursorConfig = new CursorConfiguration(this.model.getLanguageId(), this.model.getOptions(), this._configuration);
this._cursor.updateConfiguration(this.cursorConfig);
}));
@ -829,8 +830,8 @@ export class ViewModel extends Disposable implements IViewModel {
}
public getRichTextToCopy(modelRanges: Range[], emptySelectionClipboard: boolean): { html: string, mode: string } | null {
const languageId = this.model.getLanguageIdentifier();
if (languageId.id === LanguageId.PlainText) {
const languageId = this.model.getLanguageId();
if (languageId === PLAINTEXT_MODE_ID) {
return null;
}
@ -870,7 +871,7 @@ export class ViewModel extends Disposable implements IViewModel {
}
return {
mode: languageId.language,
mode: languageId,
html: (
`<div style="`
+ `color: ${colorMap[ColorId.DefaultForeground]};`

20
src/vs/editor/contrib/bracketMatching/test/bracketMatching.test.ts
View file

@ -5,7 +5,6 @@
import * as assert from 'assert';
import { Position } from 'vs/editor/common/core/position';
import { Selection } from 'vs/editor/common/core/selection';
import { LanguageIdentifier } from 'vs/editor/common/modes';
import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry';
import { BracketMatchingController } from 'vs/editor/contrib/bracketMatching/bracketMatching';
import { withTestCodeEditor } from 'vs/editor/test/browser/testCodeEditor';
@ -14,12 +13,9 @@ import { MockMode } from 'vs/editor/test/common/mocks/mockMode';
suite('bracket matching', () => {
class BracketMode extends MockMode {
private static readonly _id = new LanguageIdentifier('bracketMode', 3);
constructor() {
super(BracketMode._id);
this._register(LanguageConfigurationRegistry.register(this.getLanguageIdentifier(), {
super('bracketMode');
this._register(LanguageConfigurationRegistry.register(this.languageId, {
brackets: [
['{', '}'],
['[', ']'],
@ -31,7 +27,7 @@ suite('bracket matching', () => {
test('issue #183: jump to matching bracket position', () => {
let mode = new BracketMode();
let model = createTextModel('var x = (3 + (5-7)) + ((5+3)+5);', undefined, mode.getLanguageIdentifier());
let model = createTextModel('var x = (3 + (5-7)) + ((5+3)+5);', undefined, mode.languageId);
withTestCodeEditor(null, { model: model }, (editor) => {
let bracketMatchingController = editor.registerAndInstantiateContribution(BracketMatchingController.ID, BracketMatchingController);
@ -63,7 +59,7 @@ suite('bracket matching', () => {
test('Jump to next bracket', () => {
let mode = new BracketMode();
let model = createTextModel('var x = (3 + (5-7)); y();', undefined, mode.getLanguageIdentifier());
let model = createTextModel('var x = (3 + (5-7)); y();', undefined, mode.languageId);
withTestCodeEditor(null, { model: model }, (editor) => {
let bracketMatchingController = editor.registerAndInstantiateContribution(BracketMatchingController.ID, BracketMatchingController);
@ -100,7 +96,7 @@ suite('bracket matching', () => {
test('Select to next bracket', () => {
let mode = new BracketMode();
let model = createTextModel('var x = (3 + (5-7)); y();', undefined, mode.getLanguageIdentifier());
let model = createTextModel('var x = (3 + (5-7)); y();', undefined, mode.languageId);
withTestCodeEditor(null, { model: model }, (editor) => {
let bracketMatchingController = editor.registerAndInstantiateContribution(BracketMatchingController.ID, BracketMatchingController);
@ -152,7 +148,7 @@ suite('bracket matching', () => {
'};',
].join('\n');
const mode = new BracketMode();
const model = createTextModel(text, undefined, mode.getLanguageIdentifier());
const model = createTextModel(text, undefined, mode.languageId);
withTestCodeEditor(null, { model: model }, (editor) => {
const bracketMatchingController = editor.registerAndInstantiateContribution(BracketMatchingController.ID, BracketMatchingController);
@ -177,7 +173,7 @@ suite('bracket matching', () => {
'};',
].join('\n');
const mode = new BracketMode();
const model = createTextModel(text, undefined, mode.getLanguageIdentifier());
const model = createTextModel(text, undefined, mode.languageId);
withTestCodeEditor(null, { model: model }, (editor) => {
const bracketMatchingController = editor.registerAndInstantiateContribution(BracketMatchingController.ID, BracketMatchingController);
@ -195,7 +191,7 @@ suite('bracket matching', () => {
test('issue #45369: Select to Bracket with multicursor', () => {
let mode = new BracketMode();
let model = createTextModel('{ } { } { }', undefined, mode.getLanguageIdentifier());
let model = createTextModel('{ } { } { }', undefined, mode.languageId);
withTestCodeEditor(null, { model: model }, (editor) => {
let bracketMatchingController = editor.registerAndInstantiateContribution(BracketMatchingController.ID, BracketMatchingController);

2
src/vs/editor/contrib/codeAction/test/codeAction.test.ts
View file

@ -29,7 +29,7 @@ function staticCodeActionProvider(...actions: modes.CodeAction[]): modes.CodeAct
suite('CodeAction', () => {
let langId = new modes.LanguageIdentifier('fooLang', 17);
let langId = 'fooLang';
let uri = URI.parse('untitled:path');
let model: TextModel;
const disposables = new DisposableStore();

12
src/vs/editor/contrib/codeAction/test/codeActionModel.test.ts
View file

@ -29,7 +29,7 @@ const testProvider = {
};
suite('CodeActionModel', () => {
const languageIdentifier = new modes.LanguageIdentifier('foo-lang', 3);
const languageId = 'foo-lang';
let uri = URI.parse('untitled:path');
let model: TextModel;
let markerService: MarkerService;
@ -39,7 +39,7 @@ suite('CodeActionModel', () => {
setup(() => {
disposables.clear();
markerService = new MarkerService();
model = createTextModel('foobar foo bar\nfarboo far boo', undefined, languageIdentifier, uri);
model = createTextModel('foobar foo bar\nfarboo far boo', undefined, languageId, uri);
editor = createTestCodeEditor({ model: model });
editor.setPosition({ lineNumber: 1, column: 1 });
});
@ -52,7 +52,7 @@ suite('CodeActionModel', () => {
});
test('Orcale -> marker added', done => {
const reg = modes.CodeActionProviderRegistry.register(languageIdentifier.language, testProvider);
const reg = modes.CodeActionProviderRegistry.register(languageId, testProvider);
disposables.add(reg);
const contextKeys = new MockContextKeyService();
@ -82,7 +82,7 @@ suite('CodeActionModel', () => {
});
test('Orcale -> position changed', () => {
const reg = modes.CodeActionProviderRegistry.register(languageIdentifier.language, testProvider);
const reg = modes.CodeActionProviderRegistry.register(languageId, testProvider);
disposables.add(reg);
markerService.changeOne('fake', uri, [{
@ -115,7 +115,7 @@ suite('CodeActionModel', () => {
});
test('Lightbulb is in the wrong place, #29933', async function () {
const reg = modes.CodeActionProviderRegistry.register(languageIdentifier.language, {
const reg = modes.CodeActionProviderRegistry.register(languageId, {
provideCodeActions(_doc, _range): modes.CodeActionList {
return { actions: [], dispose() { /* noop*/ } };
}
@ -156,7 +156,7 @@ suite('CodeActionModel', () => {
});
test('Orcale -> should only auto trigger once for cursor and marker update right after each other', done => {
const reg = modes.CodeActionProviderRegistry.register(languageIdentifier.language, testProvider);
const reg = modes.CodeActionProviderRegistry.register(languageId, testProvider);
disposables.add(reg);
let triggerCount = 0;

4
src/vs/editor/contrib/colorPicker/colorDetector.ts
View file

@ -75,9 +75,9 @@ export class ColorDetector extends Disposable implements IEditorContribution {
if (!model) {
return false;
}
const languageId = model.getLanguageIdentifier();
const languageId = model.getLanguageId();
// handle deprecated settings. [languageId].colorDecorators.enable
const deprecatedConfig = this._configurationService.getValue(languageId.language);
const deprecatedConfig = this._configurationService.getValue(languageId);
if (deprecatedConfig && typeof deprecatedConfig === 'object') {
const colorDecorators = (deprecatedConfig as any)['colorDecorators']; // deprecatedConfig.valueOf('.colorDecorators.enable');
if (colorDecorators && colorDecorators['enable'] !== undefined && !colorDecorators['enable']) {

6
src/vs/editor/contrib/comment/test/blockCommentCommand.test.ts
View file

@ -9,7 +9,7 @@ import { CommentMode } from 'vs/editor/test/common/commentMode';
function testBlockCommentCommand(lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection): void {
let mode = new CommentMode({ lineComment: '!@#', blockComment: ['<0', '0>'] });
testCommand(lines, mode.getLanguageIdentifier(), selection, (sel) => new BlockCommentCommand(sel, true), expectedLines, expectedSelection);
testCommand(lines, mode.languageId, selection, (sel) => new BlockCommentCommand(sel, true), expectedLines, expectedSelection);
mode.dispose();
}
@ -475,7 +475,7 @@ suite('Editor Contrib - Block Comment Command', () => {
test('insertSpace false', () => {
function testLineCommentCommand(lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection): void {
let mode = new CommentMode({ lineComment: '!@#', blockComment: ['<0', '0>'] });
testCommand(lines, mode.getLanguageIdentifier(), selection, (sel) => new BlockCommentCommand(sel, false), expectedLines, expectedSelection);
testCommand(lines, mode.languageId, selection, (sel) => new BlockCommentCommand(sel, false), expectedLines, expectedSelection);
mode.dispose();
}
@ -494,7 +494,7 @@ suite('Editor Contrib - Block Comment Command', () => {
test('insertSpace false does not remove space', () => {
function testLineCommentCommand(lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection): void {
let mode = new CommentMode({ lineComment: '!@#', blockComment: ['<0', '0>'] });
testCommand(lines, mode.getLanguageIdentifier(), selection, (sel) => new BlockCommentCommand(sel, false), expectedLines, expectedSelection);
testCommand(lines, mode.languageId, selection, (sel) => new BlockCommentCommand(sel, false), expectedLines, expectedSelection);
mode.dispose();
}

133
src/vs/editor/contrib/comment/test/lineCommentCommand.test.ts
View file

@ -4,30 +4,41 @@
*--------------------------------------------------------------------------------------------*/
import * as assert from 'assert';
import { DisposableStore } from 'vs/base/common/lifecycle';
import { Selection } from 'vs/editor/common/core/selection';
import { TokenizationResult2 } from 'vs/editor/common/core/token';
import * as modes from 'vs/editor/common/modes';
import { ICommand } from 'vs/editor/common/editorCommon';
import { ColorId, IState, MetadataConsts, TokenizationRegistry } from 'vs/editor/common/modes';
import { CommentRule } from 'vs/editor/common/modes/languageConfiguration';
import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry';
import { NULL_STATE } from 'vs/editor/common/modes/nullMode';
import { IModeService } from 'vs/editor/common/services/modeService';
import { ILinePreflightData, IPreflightData, ISimpleModel, LineCommentCommand, Type } from 'vs/editor/contrib/comment/lineCommentCommand';
import { testCommand } from 'vs/editor/test/browser/testCommand';
import { IInstantiationService, ServicesAccessor } from 'vs/platform/instantiation/common/instantiation';
import { CommentMode } from 'vs/editor/test/common/commentMode';
import { MockMode } from 'vs/editor/test/common/mocks/mockMode';
function createTestCommandHelper(commentsConfig: CommentRule, commandFactory: (selection: Selection) => ICommand): (lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection) => void {
return (lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection) => {
const setup = (accessor: ServicesAccessor, disposables: DisposableStore) => {
disposables.add(new CommentMode(commentsConfig));
};
testCommand(lines, CommentMode.id, selection, commandFactory, expectedLines, expectedSelection, false, setup);
};
}
suite('Editor Contrib - Line Comment Command', () => {
function testLineCommentCommand(lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection): void {
let mode = new CommentMode({ lineComment: '!@#', blockComment: ['<!@#', '#@!>'] });
testCommand(lines, mode.getLanguageIdentifier(), selection, (sel) => new LineCommentCommand(sel, 4, Type.Toggle, true, true), expectedLines, expectedSelection);
mode.dispose();
}
const testLineCommentCommand = createTestCommandHelper(
{ lineComment: '!@#', blockComment: ['<!@#', '#@!>'] },
(sel) => new LineCommentCommand(sel, 4, Type.Toggle, true, true)
);
function testAddLineCommentCommand(lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection): void {
let mode = new CommentMode({ lineComment: '!@#', blockComment: ['<!@#', '#@!>'] });
testCommand(lines, mode.getLanguageIdentifier(), selection, (sel) => new LineCommentCommand(sel, 4, Type.ForceAdd, true, true), expectedLines, expectedSelection);
mode.dispose();
}
const testAddLineCommentCommand = createTestCommandHelper(
{ lineComment: '!@#', blockComment: ['<!@#', '#@!>'] },
(sel) => new LineCommentCommand(sel, 4, Type.ForceAdd, true, true)
);
test('comment single line', function () {
testLineCommentCommand(
@ -45,11 +56,10 @@ suite('Editor Contrib - Line Comment Command', () => {
});
test('case insensitive', function () {
function testLineCommentCommand(lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection): void {
let mode = new CommentMode({ lineComment: 'rem' });
testCommand(lines, mode.getLanguageIdentifier(), selection, (sel) => new LineCommentCommand(sel, 4, Type.Toggle, true, true), expectedLines, expectedSelection);
mode.dispose();
}
const testLineCommentCommand = createTestCommandHelper(
{ lineComment: 'rem' },
(sel) => new LineCommentCommand(sel, 4, Type.Toggle, true, true)
);
testLineCommentCommand(
[
@ -629,11 +639,10 @@ suite('Editor Contrib - Line Comment Command', () => {
});
test('insertSpace false', () => {
function testLineCommentCommand(lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection): void {
let mode = new CommentMode({ lineComment: '!@#' });
testCommand(lines, mode.getLanguageIdentifier(), selection, (sel) => new LineCommentCommand(sel, 4, Type.Toggle, false, true), expectedLines, expectedSelection);
mode.dispose();
}
const testLineCommentCommand = createTestCommandHelper(
{ lineComment: '!@#' },
(sel) => new LineCommentCommand(sel, 4, Type.Toggle, false, true)
);
testLineCommentCommand(
[
@ -648,11 +657,10 @@ suite('Editor Contrib - Line Comment Command', () => {
});
test('insertSpace false does not remove space', () => {
function testLineCommentCommand(lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection): void {
let mode = new CommentMode({ lineComment: '!@#' });
testCommand(lines, mode.getLanguageIdentifier(), selection, (sel) => new LineCommentCommand(sel, 4, Type.Toggle, false, true), expectedLines, expectedSelection);
mode.dispose();
}
const testLineCommentCommand = createTestCommandHelper(
{ lineComment: '!@#' },
(sel) => new LineCommentCommand(sel, 4, Type.Toggle, false, true)
);
testLineCommentCommand(
[
@ -667,11 +675,11 @@ suite('Editor Contrib - Line Comment Command', () => {
});
suite('ignoreEmptyLines false', () => {
function testLineCommentCommand(lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection): void {
let mode = new CommentMode({ lineComment: '!@#', blockComment: ['<!@#', '#@!>'] });
testCommand(lines, mode.getLanguageIdentifier(), selection, (sel) => new LineCommentCommand(sel, 4, Type.Toggle, true, false), expectedLines, expectedSelection);
mode.dispose();
}
const testLineCommentCommand = createTestCommandHelper(
{ lineComment: '!@#', blockComment: ['<!@#', '#@!>'] },
(sel) => new LineCommentCommand(sel, 4, Type.Toggle, true, false)
);
test('does not ignore whitespace lines', () => {
testLineCommentCommand(
@ -760,11 +768,10 @@ suite('Editor Contrib - Line Comment Command', () => {
suite('Editor Contrib - Line Comment As Block Comment', () => {
function testLineCommentCommand(lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection): void {
let mode = new CommentMode({ lineComment: '', blockComment: ['(', ')'] });
testCommand(lines, mode.getLanguageIdentifier(), selection, (sel) => new LineCommentCommand(sel, 4, Type.Toggle, true, true), expectedLines, expectedSelection);
mode.dispose();
}
const testLineCommentCommand = createTestCommandHelper(
{ lineComment: '', blockComment: ['(', ')'] },
(sel) => new LineCommentCommand(sel, 4, Type.Toggle, true, true)
);
test('fall back to block comment command', function () {
testLineCommentCommand(
@ -871,11 +878,11 @@ suite('Editor Contrib - Line Comment As Block Comment', () => {
});
suite('Editor Contrib - Line Comment As Block Comment 2', () => {
function testLineCommentCommand(lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection): void {
let mode = new CommentMode({ lineComment: null, blockComment: ['<!@#', '#@!>'] });
testCommand(lines, mode.getLanguageIdentifier(), selection, (sel) => new LineCommentCommand(sel, 4, Type.Toggle, true, true), expectedLines, expectedSelection);
mode.dispose();
}
const testLineCommentCommand = createTestCommandHelper(
{ lineComment: null, blockComment: ['<!@#', '#@!>'] },
(sel) => new LineCommentCommand(sel, 4, Type.Toggle, true, true)
);
test('no selection => uses indentation', function () {
testLineCommentCommand(
@ -1068,29 +1075,33 @@ suite('Editor Contrib - Line Comment As Block Comment 2', () => {
suite('Editor Contrib - Line Comment in mixed modes', () => {
const OUTER_LANGUAGE_ID = new modes.LanguageIdentifier('outerMode', 3);
const INNER_LANGUAGE_ID = new modes.LanguageIdentifier('innerMode', 4);
const OUTER_LANGUAGE_ID = 'outerMode';
const INNER_LANGUAGE_ID = 'innerMode';
class OuterMode extends MockMode {
constructor(commentsConfig: CommentRule) {
constructor(
commentsConfig: CommentRule,
@IModeService modeService: IModeService
) {
super(OUTER_LANGUAGE_ID);
this._register(LanguageConfigurationRegistry.register(this.getLanguageIdentifier(), {
this._register(LanguageConfigurationRegistry.register(this.languageId, {
comments: commentsConfig
}));
this._register(modes.TokenizationRegistry.register(this.getLanguageIdentifier().language, {
getInitialState: (): modes.IState => NULL_STATE,
this._register(TokenizationRegistry.register(this.languageId, {
getInitialState: (): IState => NULL_STATE,
tokenize: () => {
throw new Error('not implemented');
},
tokenize2: (line: string, hasEOL: boolean, state: modes.IState): TokenizationResult2 => {
let languageId = (/^ /.test(line) ? INNER_LANGUAGE_ID : OUTER_LANGUAGE_ID);
tokenize2: (line: string, hasEOL: boolean, state: IState): TokenizationResult2 => {
const languageId = (/^ /.test(line) ? INNER_LANGUAGE_ID : OUTER_LANGUAGE_ID);
const encodedLanguageId = modeService.languageIdCodec.encodeLanguageId(languageId);
let tokens = new Uint32Array(1 << 1);
const tokens = new Uint32Array(1 << 1);
tokens[(0 << 1)] = 0;
tokens[(0 << 1) + 1] = (
(modes.ColorId.DefaultForeground << modes.MetadataConsts.FOREGROUND_OFFSET)
| (languageId.id << modes.MetadataConsts.LANGUAGEID_OFFSET)
(ColorId.DefaultForeground << MetadataConsts.FOREGROUND_OFFSET)
| (encodedLanguageId << MetadataConsts.LANGUAGEID_OFFSET)
);
return new TokenizationResult2(tokens, state);
}
@ -1101,26 +1112,30 @@ suite('Editor Contrib - Line Comment in mixed modes', () => {
class InnerMode extends MockMode {
constructor(commentsConfig: CommentRule) {
super(INNER_LANGUAGE_ID);
this._register(LanguageConfigurationRegistry.register(this.getLanguageIdentifier(), {
this._register(LanguageConfigurationRegistry.register(this.languageId, {
comments: commentsConfig
}));
}
}
function testLineCommentCommand(lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection): void {
let outerMode = new OuterMode({ lineComment: '//', blockComment: ['/*', '*/'] });
let innerMode = new InnerMode({ lineComment: null, blockComment: ['{/*', '*/}'] });
const setup = (accessor: ServicesAccessor, disposables: DisposableStore) => {
const instantiationService = accessor.get(IInstantiationService);
disposables.add(instantiationService.createInstance(OuterMode, { lineComment: '//', blockComment: ['/*', '*/'] }));
disposables.add(instantiationService.createInstance(InnerMode, { lineComment: null, blockComment: ['{/*', '*/}'] }));
};
testCommand(
lines,
outerMode.getLanguageIdentifier(),
OUTER_LANGUAGE_ID,
selection,
(sel) => new LineCommentCommand(sel, 4, Type.Toggle, true, true),
expectedLines,
expectedSelection,
true
true,
setup
);
innerMode.dispose();
outerMode.dispose();
}
test('issue #24047 (part 1): Commenting code in JSX files', () => {

2
src/vs/editor/contrib/documentSymbols/test/outlineModel.test.ts
View file

@ -38,6 +38,7 @@ suite('OutlineModel', function () {
assert.strictEqual(count, 2);
reg.dispose();
model.dispose();
});
test('OutlineModel#create, cached/cancel', async function () {
@ -69,6 +70,7 @@ suite('OutlineModel', function () {
assert.strictEqual(isCancelled, true);
reg.dispose();
model.dispose();
});
function fakeSymbolInformation(range: Range, name: string = 'foo'): DocumentSymbol {

21
src/vs/editor/contrib/find/test/findModel.test.ts
View file

@ -10,14 +10,9 @@ import { Position } from 'vs/editor/common/core/position';
import { Range } from 'vs/editor/common/core/range';
import { Selection } from 'vs/editor/common/core/selection';
import { PieceTreeTextBufferBuilder } from 'vs/editor/common/model/pieceTreeTextBuffer/pieceTreeTextBufferBuilder';
import { TextModel } from 'vs/editor/common/model/textModel';
import { FindModelBoundToEditorModel } from 'vs/editor/contrib/find/findModel';
import { FindReplaceState } from 'vs/editor/contrib/find/findState';
import { withTestCodeEditor } from 'vs/editor/test/browser/testCodeEditor';
import { TestLanguageConfigurationService } from 'vs/editor/test/common/modes/testLanguageConfigurationService';
import { TestDialogService } from 'vs/platform/dialogs/test/common/testDialogService';
import { TestNotificationService } from 'vs/platform/notification/test/common/testNotificationService';
import { UndoRedoService } from 'vs/platform/undoRedo/common/undoRedoService';
suite('FindModel', () => {
@ -46,20 +41,8 @@ suite('FindModel', () => {
ptBuilder.acceptChunk(text.substr(195, 59));
const factory = ptBuilder.finish();
withTestCodeEditor(
[],
{
model: new TextModel(
factory,
TextModel.DEFAULT_CREATION_OPTIONS,
null,
null,
new UndoRedoService(
new TestDialogService(),
new TestNotificationService()
),
new TestLanguageConfigurationService()
),
},
factory,
{},
(editor) => callback(editor as IActiveCodeEditor)
);
});

6
src/vs/editor/contrib/folding/folding.ts
View file

@ -791,7 +791,7 @@ class FoldAllBlockCommentsAction extends FoldingAction<void> {
if (!editorModel) {
return;
}
let comments = LanguageConfigurationRegistry.getComments(editorModel.getLanguageIdentifier().id);
const comments = LanguageConfigurationRegistry.getComments(editorModel.getLanguageId());
if (comments && comments.blockCommentStartToken) {
let regExp = new RegExp('^\\s*' + escapeRegExpCharacters(comments.blockCommentStartToken));
setCollapseStateForMatchingLines(foldingModel, regExp, true);
@ -824,7 +824,7 @@ class FoldAllRegionsAction extends FoldingAction<void> {
if (!editorModel) {
return;
}
let foldingRules = LanguageConfigurationRegistry.getFoldingRules(editorModel.getLanguageIdentifier().id);
const foldingRules = LanguageConfigurationRegistry.getFoldingRules(editorModel.getLanguageId());
if (foldingRules && foldingRules.markers && foldingRules.markers.start) {
let regExp = new RegExp(foldingRules.markers.start);
setCollapseStateForMatchingLines(foldingModel, regExp, true);
@ -857,7 +857,7 @@ class UnfoldAllRegionsAction extends FoldingAction<void> {
if (!editorModel) {
return;
}
let foldingRules = LanguageConfigurationRegistry.getFoldingRules(editorModel.getLanguageIdentifier().id);
const foldingRules = LanguageConfigurationRegistry.getFoldingRules(editorModel.getLanguageId());
if (foldingRules && foldingRules.markers && foldingRules.markers.start) {
let regExp = new RegExp(foldingRules.markers.start);
setCollapseStateForMatchingLines(foldingModel, regExp, false);

2
src/vs/editor/contrib/folding/indentRangeProvider.ts
View file

@ -25,7 +25,7 @@ export class IndentRangeProvider implements RangeProvider {
}
compute(cancelationToken: CancellationToken): Promise<FoldingRegions> {
let foldingRules = LanguageConfigurationRegistry.getFoldingRules(this.editorModel.getLanguageIdentifier().id);
let foldingRules = LanguageConfigurationRegistry.getFoldingRules(this.editorModel.getLanguageId());
let offSide = foldingRules && !!foldingRules.offSide;
let markers = foldingRules && foldingRules.markers;
return Promise.resolve(computeRanges(this.editorModel, offSide, markers));

2
src/vs/editor/contrib/folding/test/foldingRanges.test.ts
View file

@ -34,6 +34,7 @@ suite('FoldingRanges', () => {
assert.strictEqual(actual.getEndLineNumber(i), nRegions * 2 - i, 'end' + i);
assert.strictEqual(actual.getParentIndex(i), i - 1, 'parent' + i);
}
model.dispose();
});
@ -100,5 +101,6 @@ suite('FoldingRanges', () => {
for (let i = 0; i < nRegions; i++) {
assert.strictEqual(actual.isCollapsed(i), i % 3 === 0, 'line' + i);
}
model.dispose();
});
});

2
src/vs/editor/contrib/folding/test/hiddenRangeModel.test.ts
View file

@ -91,6 +91,8 @@ suite('Hidden Range Model', () => {
assert.strictEqual(hiddenRangeModel.isHidden(9), false);
assert.strictEqual(hiddenRangeModel.isHidden(10), false);
textModel.dispose();
});

2
src/vs/editor/contrib/folding/test/indentFold.test.ts
View file

@ -70,6 +70,8 @@ suite('Indentation Folding', () => {
assertLimit(2, [r1, r9], '2');
assertLimit(1, [r1], '1');
assertLimit(0, [], '0');
model.dispose();
});
});

2
src/vs/editor/contrib/folding/test/syntaxFold.test.ts
View file

@ -95,6 +95,8 @@ suite('Syntax folding', () => {
await assertLimit(2, [r1, r9], '2');
await assertLimit(1, [r1], '1');
await assertLimit(0, [], '0');
model.dispose();
});
});

4
src/vs/editor/contrib/gotoSymbol/peek/referencesWidget.ts
View file

@ -24,6 +24,7 @@ import { IModelDeltaDecoration, TrackedRangeStickiness } from 'vs/editor/common/
import { ModelDecorationOptions, TextModel } from 'vs/editor/common/model/textModel';
import { Location } from 'vs/editor/common/modes';
import { ILanguageConfigurationService } from 'vs/editor/common/modes/languageConfigurationRegistry';
import { IModeService } from 'vs/editor/common/services/modeService';
import { ITextEditorModel, ITextModelService } from 'vs/editor/common/services/resolverService';
import { AccessibilityProvider, DataSource, Delegate, FileReferencesRenderer, IdentityProvider, OneReferenceRenderer, StringRepresentationProvider, TreeElement } from 'vs/editor/contrib/gotoSymbol/peek/referencesTree';
import * as peekView from 'vs/editor/contrib/peekView/peekView';
@ -224,6 +225,7 @@ export class ReferenceWidget extends peekView.PeekViewWidget {
@ILabelService private readonly _uriLabel: ILabelService,
@IUndoRedoService private readonly _undoRedoService: IUndoRedoService,
@IKeybindingService private readonly _keybindingService: IKeybindingService,
@IModeService private readonly _modeService: IModeService,
@ILanguageConfigurationService private readonly _languageConfigurationService: ILanguageConfigurationService,
) {
super(editor, { showFrame: false, showArrow: true, isResizeable: true, isAccessible: true, supportOnTitleClick: true }, _instantiationService);
@ -313,7 +315,7 @@ export class ReferenceWidget extends peekView.PeekViewWidget {
};
this._preview = this._instantiationService.createInstance(EmbeddedCodeEditorWidget, this._previewContainer, options, this.editor);
dom.hide(this._previewContainer);
this._previewNotAvailableMessage = new TextModel(nls.localize('missingPreviewMessage', "no preview available"), TextModel.DEFAULT_CREATION_OPTIONS, null, null, this._undoRedoService, this._languageConfigurationService);
this._previewNotAvailableMessage = new TextModel(nls.localize('missingPreviewMessage', "no preview available"), TextModel.DEFAULT_CREATION_OPTIONS, null, null, this._undoRedoService, this._modeService, this._languageConfigurationService);
// tree
this._treeContainer = dom.append(containerElement, dom.$('div.ref-tree.inline'));

14
src/vs/editor/contrib/indentation/indentation.ts
View file

@ -30,7 +30,7 @@ export function getReindentEditOperations(model: ITextModel, startLineNumber: nu
return [];
}
let indentationRules = LanguageConfigurationRegistry.getIndentationRules(model.getLanguageIdentifier().id);
const indentationRules = LanguageConfigurationRegistry.getIndentationRules(model.getLanguageId());
if (!indentationRules) {
return [];
}
@ -219,7 +219,7 @@ export class ChangeIndentationSizeAction extends EditorAction {
return;
}
let creationOpts = modelService.getCreationOptions(model.getLanguageIdentifier().language, model.uri, model.isForSimpleWidget);
const creationOpts = modelService.getCreationOptions(model.getLanguageId(), model.uri, model.isForSimpleWidget);
const picks = [1, 2, 3, 4, 5, 6, 7, 8].map(n => ({
id: n.toString(),
label: n.toString(),
@ -294,7 +294,7 @@ export class DetectIndentation extends EditorAction {
return;
}
let creationOpts = modelService.getCreationOptions(model.getLanguageIdentifier().language, model.uri, model.isForSimpleWidget);
const creationOpts = modelService.getCreationOptions(model.getLanguageId(), model.uri, model.isForSimpleWidget);
model.detectIndentation(creationOpts.insertSpaces, creationOpts.tabSize);
}
}
@ -499,7 +499,7 @@ export class AutoIndentOnPaste implements IEditorContribution {
let firstLineText = model.getLineContent(startLineNumber);
if (!/\S/.test(firstLineText.substring(0, range.startColumn - 1))) {
let indentOfFirstLine = LanguageConfigurationRegistry.getGoodIndentForLine(autoIndent, model, model.getLanguageIdentifier().id, startLineNumber, indentConverter);
const indentOfFirstLine = LanguageConfigurationRegistry.getGoodIndentForLine(autoIndent, model, model.getLanguageId(), startLineNumber, indentConverter);
if (indentOfFirstLine !== null) {
let oldIndentation = strings.getLeadingWhitespace(firstLineText);
@ -543,8 +543,8 @@ export class AutoIndentOnPaste implements IEditorContribution {
getLineTokens: (lineNumber: number) => {
return model.getLineTokens(lineNumber);
},
getLanguageIdentifier: () => {
return model.getLanguageIdentifier();
getLanguageId: () => {
return model.getLanguageId();
},
getLanguageIdAtPosition: (lineNumber: number, column: number) => {
return model.getLanguageIdAtPosition(lineNumber, column);
@ -557,7 +557,7 @@ export class AutoIndentOnPaste implements IEditorContribution {
}
}
};
let indentOfSecondLine = LanguageConfigurationRegistry.getGoodIndentForLine(autoIndent, virtualModel, model.getLanguageIdentifier().id, startLineNumber + 1, indentConverter);
let indentOfSecondLine = LanguageConfigurationRegistry.getGoodIndentForLine(autoIndent, virtualModel, model.getLanguageId(), startLineNumber + 1, indentConverter);
if (indentOfSecondLine !== null) {
let newSpaceCntOfSecondLine = indentUtils.getSpaceCnt(indentOfSecondLine, tabSize);
let oldSpaceCntOfSecondLine = indentUtils.getSpaceCnt(strings.getLeadingWhitespace(model.getLineContent(startLineNumber + 1)), tabSize);

16
src/vs/editor/contrib/inlineCompletions/ghostTextWidget.ts
View file

@ -19,6 +19,8 @@ import { Range } from 'vs/editor/common/core/range';
import { createStringBuilder } from 'vs/editor/common/core/stringBuilder';
import { IDecorationRenderOptions } from 'vs/editor/common/editorCommon';
import { IModelDeltaDecoration } from 'vs/editor/common/model';
import { ILanguageIdCodec } from 'vs/editor/common/modes';
import { IModeService } from 'vs/editor/common/services/modeService';
import { ghostTextBorder, ghostTextForeground } from 'vs/editor/common/view/editorColorRegistry';
import { LineDecoration } from 'vs/editor/common/viewLayout/lineDecorations';
import { RenderLineInput, renderViewLine } from 'vs/editor/common/viewLayout/viewLineRenderer';
@ -33,13 +35,14 @@ const ttPolicy = window.trustedTypes?.createPolicy('editorGhostText', { createHT
export class GhostTextWidget extends Disposable {
private disposed = false;
private readonly partsWidget = this._register(this.instantiationService.createInstance(DecorationsWidget, this.editor));
private readonly additionalLinesWidget = this._register(new AdditionalLinesWidget(this.editor));
private readonly additionalLinesWidget = this._register(new AdditionalLinesWidget(this.editor, this.modeService.languageIdCodec));
private viewMoreContentWidget: ViewMoreLinesContentWidget | undefined = undefined;
constructor(
private readonly editor: ICodeEditor,
private readonly model: GhostTextWidgetModel,
@IInstantiationService private readonly instantiationService: IInstantiationService,
@IModeService private readonly modeService: IModeService,
) {
super();
@ -336,7 +339,10 @@ class AdditionalLinesWidget implements IDisposable {
private _viewZoneId: string | undefined = undefined;
public get viewZoneId(): string | undefined { return this._viewZoneId; }
constructor(private readonly editor: ICodeEditor) { }
constructor(
private readonly editor: ICodeEditor,
private readonly languageIdCodec: ILanguageIdCodec
) { }
public dispose(): void {
this.clear();
@ -368,7 +374,7 @@ class AdditionalLinesWidget implements IDisposable {
const heightInLines = Math.max(additionalLines.length, minReservedLineCount);
if (heightInLines > 0) {
const domNode = document.createElement('div');
renderLines(domNode, tabSize, additionalLines, this.editor.getOptions());
renderLines(domNode, tabSize, additionalLines, this.editor.getOptions(), this.languageIdCodec);
this._viewZoneId = changeAccessor.addZone({
afterLineNumber: lineNumber,
@ -385,7 +391,7 @@ interface LineData {
decorations: LineDecoration[];
}
function renderLines(domNode: HTMLElement, tabSize: number, lines: LineData[], opts: IComputedEditorOptions): void {
function renderLines(domNode: HTMLElement, tabSize: number, lines: LineData[], opts: IComputedEditorOptions, languageIdCodec: ILanguageIdCodec): void {
const disableMonospaceOptimizations = opts.get(EditorOption.disableMonospaceOptimizations);
const stopRenderingLineAfter = opts.get(EditorOption.stopRenderingLineAfter);
// To avoid visual confusion, we don't want to render visible whitespace
@ -408,7 +414,7 @@ function renderLines(domNode: HTMLElement, tabSize: number, lines: LineData[], o
const isBasicASCII = strings.isBasicASCII(line);
const containsRTL = strings.containsRTL(line);
const lineTokens = LineTokens.createEmpty(line);
const lineTokens = LineTokens.createEmpty(line, languageIdCodec);
renderViewLine(new RenderLineInput(
(fontInfo.isMonospace && !disableMonospaceOptimizations),

2
src/vs/editor/contrib/inlineCompletions/test/suggestWidgetModel.test.ts
View file

@ -99,6 +99,8 @@ suite('Suggest Widget Model', () => {
range: '[1,4 -> 1,4]',
text: 'ction'
});
model.dispose();
});
});

4
src/vs/editor/contrib/linesOperations/moveLinesCommand.ts
View file

@ -60,8 +60,8 @@ export class MoveLinesCommand implements ICommand {
getLineTokens: (lineNumber: number) => {
return model.getLineTokens(lineNumber);
},
getLanguageIdentifier: () => {
return model.getLanguageIdentifier();
getLanguageId: () => {
return model.getLanguageId();
},
getLanguageIdAtPosition: (lineNumber: number, column: number) => {
return model.getLanguageIdAtPosition(lineNumber, column);

19
src/vs/editor/contrib/linesOperations/test/moveLinesCommand.test.ts
View file

@ -4,7 +4,6 @@
*--------------------------------------------------------------------------------------------*/
import { EditorAutoIndentStrategy } from 'vs/editor/common/config/editorOptions';
import { Selection } from 'vs/editor/common/core/selection';
import { LanguageIdentifier } from 'vs/editor/common/modes';
import { IndentationRule } from 'vs/editor/common/modes/languageConfiguration';
import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry';
import { MoveLinesCommand } from 'vs/editor/contrib/linesOperations/moveLinesCommand';
@ -19,11 +18,11 @@ function testMoveLinesUpCommand(lines: string[], selection: Selection, expectedL
testCommand(lines, null, selection, (sel) => new MoveLinesCommand(sel, false, EditorAutoIndentStrategy.Advanced), expectedLines, expectedSelection);
}
function testMoveLinesDownWithIndentCommand(languageId: LanguageIdentifier, lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection): void {
function testMoveLinesDownWithIndentCommand(languageId: string, lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection): void {
testCommand(lines, languageId, selection, (sel) => new MoveLinesCommand(sel, true, EditorAutoIndentStrategy.Full), expectedLines, expectedSelection);
}
function testMoveLinesUpWithIndentCommand(languageId: LanguageIdentifier, lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection): void {
function testMoveLinesUpWithIndentCommand(languageId: string, lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection): void {
testCommand(lines, languageId, selection, (sel) => new MoveLinesCommand(sel, false, EditorAutoIndentStrategy.Full), expectedLines, expectedSelection);
}
@ -260,10 +259,10 @@ suite('Editor Contrib - Move Lines Command', () => {
});
class IndentRulesMode extends MockMode {
private static readonly _id = new LanguageIdentifier('moveLinesIndentMode', 7);
private static readonly _id = 'moveLinesIndentMode';
constructor(indentationRules: IndentationRule) {
super(IndentRulesMode._id);
this._register(LanguageConfigurationRegistry.register(this.getLanguageIdentifier(), {
this._register(LanguageConfigurationRegistry.register(this.languageId, {
indentationRules: indentationRules
}));
}
@ -282,7 +281,7 @@ suite('Editor contrib - Move Lines Command honors Indentation Rules', () => {
let mode = new IndentRulesMode(indentRules);
testMoveLinesUpWithIndentCommand(
mode.getLanguageIdentifier(),
mode.languageId,
[
'class X {',
'\tz = 2',
@ -305,7 +304,7 @@ suite('Editor contrib - Move Lines Command honors Indentation Rules', () => {
let mode = new IndentRulesMode(indentRules);
testMoveLinesDownWithIndentCommand(
mode.getLanguageIdentifier(),
mode.languageId,
[
'const value = 2;',
'const standardLanguageDescriptions = [',
@ -354,10 +353,10 @@ suite('Editor contrib - Move Lines Command honors Indentation Rules', () => {
});
class EnterRulesMode extends MockMode {
private static readonly _id = new LanguageIdentifier('moveLinesEnterMode', 8);
private static readonly _id = 'moveLinesEnterMode';
constructor() {
super(EnterRulesMode._id);
this._register(LanguageConfigurationRegistry.register(this.getLanguageIdentifier(), {
this._register(LanguageConfigurationRegistry.register(this.languageId, {
indentationRules: {
decreaseIndentPattern: /^\s*\[$/,
increaseIndentPattern: /^\s*\]$/,
@ -375,7 +374,7 @@ suite('Editor - contrib - Move Lines Command honors onEnter Rules', () => {
let mode = new EnterRulesMode();
testMoveLinesDownWithIndentCommand(
mode.getLanguageIdentifier(),
mode.languageId,
[
'if (true) {',

4
src/vs/editor/contrib/linkedEditing/linkedEditing.ts
View file

@ -120,9 +120,9 @@ export class LinkedEditingContribution extends Disposable implements IEditorCont
return;
}
this._languageWordPattern = LanguageConfigurationRegistry.getWordDefinition(model.getLanguageIdentifier().id);
this._languageWordPattern = LanguageConfigurationRegistry.getWordDefinition(model.getLanguageId());
this._localToDispose.add(model.onDidChangeLanguageConfiguration(() => {
this._languageWordPattern = LanguageConfigurationRegistry.getWordDefinition(model.getLanguageIdentifier().id);
this._languageWordPattern = LanguageConfigurationRegistry.getWordDefinition(model.getLanguageId());
}));
const rangeUpdateScheduler = new Delayer(this._debounceDuration);

12
src/vs/editor/contrib/linkedEditing/test/linkedEditing.test..ts
View file

@ -30,16 +30,16 @@ interface TestEditor {
redo(): void;
}
const languageIdentifier = new modes.LanguageIdentifier('linkedEditingTestLangage', 74);
LanguageConfigurationRegistry.register(languageIdentifier, {
wordPattern: /[a-zA-Z]+/
});
const languageId = 'linkedEditingTestLangage';
suite('linked editing', () => {
const disposables = new DisposableStore();
setup(() => {
disposables.clear();
disposables.add(LanguageConfigurationRegistry.register(languageId, {
wordPattern: /[a-zA-Z]+/
}));
});
teardown(() => {
@ -48,8 +48,8 @@ suite('linked editing', () => {
function createMockEditor(text: string | string[]): ITestCodeEditor {
const model = typeof text === 'string'
? createTextModel(text, undefined, languageIdentifier, mockFile)
: createTextModel(text.join('\n'), undefined, languageIdentifier, mockFile);
? createTextModel(text, undefined, languageId, mockFile)
: createTextModel(text.join('\n'), undefined, languageId, mockFile);
const editor = createTestCodeEditor({ model });
disposables.add(model);

35
src/vs/editor/contrib/smartSelect/test/smartSelect.test.ts
View file

@ -4,34 +4,28 @@
*--------------------------------------------------------------------------------------------*/
import * as assert from 'assert';
import { CancellationToken } from 'vs/base/common/cancellation';
import { DisposableStore } from 'vs/base/common/lifecycle';
import { URI } from 'vs/base/common/uri';
import { Position } from 'vs/editor/common/core/position';
import { IRange, Range } from 'vs/editor/common/core/range';
import { LanguageIdentifier, SelectionRangeProvider, SelectionRangeRegistry } from 'vs/editor/common/modes';
import { SelectionRangeProvider, SelectionRangeRegistry } from 'vs/editor/common/modes';
import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry';
import { ModelServiceImpl } from 'vs/editor/common/services/modelServiceImpl';
import { IModelService } from 'vs/editor/common/services/modelService';
import { BracketSelectionRangeProvider } from 'vs/editor/contrib/smartSelect/bracketSelections';
import { provideSelectionRanges } from 'vs/editor/contrib/smartSelect/smartSelect';
import { WordSelectionRangeProvider } from 'vs/editor/contrib/smartSelect/wordSelections';
import { createModelServices } from 'vs/editor/test/common/editorTestUtils';
import { MockMode, StaticLanguageSelector } from 'vs/editor/test/common/mocks/mockMode';
import { javascriptOnEnterRules } from 'vs/editor/test/common/modes/supports/javascriptOnEnterRules';
import { TestLanguageConfigurationService } from 'vs/editor/test/common/modes/testLanguageConfigurationService';
import { TestTextResourcePropertiesService } from 'vs/editor/test/common/services/testTextResourcePropertiesService';
import { TestConfigurationService } from 'vs/platform/configuration/test/common/testConfigurationService';
import { TestDialogService } from 'vs/platform/dialogs/test/common/testDialogService';
import { NullLogService } from 'vs/platform/log/common/log';
import { TestNotificationService } from 'vs/platform/notification/test/common/testNotificationService';
import { TestThemeService } from 'vs/platform/theme/test/common/testThemeService';
import { UndoRedoService } from 'vs/platform/undoRedo/common/undoRedoService';
class MockJSMode extends MockMode {
private static readonly _id = new LanguageIdentifier('mockJSMode', 3);
private static readonly _id = 'mockJSMode';
constructor() {
super(MockJSMode._id);
this._register(LanguageConfigurationRegistry.register(this.getLanguageIdentifier(), {
this._register(LanguageConfigurationRegistry.register(this.languageId, {
brackets: [
['(', ')'],
['{', '}'],
@ -56,24 +50,25 @@ suite('SmartSelect', () => {
BracketSelectionRangeProvider._maxDuration = OriginalBracketSelectionRangeProviderMaxDuration;
});
let modelService: ModelServiceImpl;
let disposables: DisposableStore;
let modelService: IModelService;
let mode: MockJSMode;
setup(() => {
const configurationService = new TestConfigurationService();
const dialogService = new TestDialogService();
modelService = new ModelServiceImpl(configurationService, new TestTextResourcePropertiesService(configurationService), new TestThemeService(), new NullLogService(), new UndoRedoService(dialogService, new TestNotificationService()), new TestLanguageConfigurationService());
mode = new MockJSMode();
const [instantiationService, _disposables] = createModelServices();
modelService = instantiationService.invokeFunction((accessor) => accessor.get(IModelService));
disposables = _disposables;
mode = disposables.add(new MockJSMode());
});
teardown(() => {
modelService.dispose();
mode.dispose();
disposables.dispose();
});
async function assertGetRangesToPosition(text: string[], lineNumber: number, column: number, ranges: Range[], selectLeadingAndTrailingWhitespace = true): Promise<void> {
let uri = URI.file('test.js');
let model = modelService.createModel(text.join('\n'), new StaticLanguageSelector(mode.getLanguageIdentifier()), uri);
let model = modelService.createModel(text.join('\n'), new StaticLanguageSelector(mode.languageId), uri);
let [actual] = await provideSelectionRanges(model, [new Position(lineNumber, column)], { selectLeadingAndTrailingWhitespace }, CancellationToken.None);
let actualStr = actual!.map(r => new Range(r.startLineNumber, r.startColumn, r.endLineNumber, r.endColumn).toString());
let desiredStr = ranges.reverse().map(r => String(r));
@ -220,7 +215,7 @@ suite('SmartSelect', () => {
let index = value.indexOf('|');
value = value.replace('|', '');
let model = modelService.createModel(value, new StaticLanguageSelector(mode.getLanguageIdentifier()), URI.parse('fake:lang'));
let model = modelService.createModel(value, new StaticLanguageSelector(mode.languageId), URI.parse('fake:lang'));
let pos = model.getPositionAt(index);
let all = await provider.provideSelectionRanges(model, [pos], CancellationToken.None);
let ranges = all![0];

22
src/vs/editor/contrib/snippet/test/snippetVariables.test.ts
View file

@ -4,6 +4,7 @@
*--------------------------------------------------------------------------------------------*/
import * as assert from 'assert';
import * as sinon from 'sinon';
import { DisposableStore } from 'vs/base/common/lifecycle';
import { sep } from 'vs/base/common/path';
import { isWindows } from 'vs/base/common/platform';
import { extUriBiasedIgnorePathCase } from 'vs/base/common/resources';
@ -65,6 +66,8 @@ suite('Snippet Variables Resolver', function () {
test('editor variables, file/dir', function () {
const disposables = new DisposableStore();
assertVariableResolve(resolver, 'TM_FILENAME', 'text.txt');
if (!isWindows) {
assertVariableResolve(resolver, 'TM_DIRECTORY', '/foo/files');
@ -73,7 +76,7 @@ suite('Snippet Variables Resolver', function () {
resolver = new ModelBasedVariableResolver(
labelService,
createTextModel('', undefined, undefined, URI.parse('http://www.pb.o/abc/def/ghi'))
disposables.add(createTextModel('', undefined, undefined, URI.parse('http://www.pb.o/abc/def/ghi')))
);
assertVariableResolve(resolver, 'TM_FILENAME', 'ghi');
if (!isWindows) {
@ -83,11 +86,12 @@ suite('Snippet Variables Resolver', function () {
resolver = new ModelBasedVariableResolver(
labelService,
createTextModel('', undefined, undefined, URI.parse('mem:fff.ts'))
disposables.add(createTextModel('', undefined, undefined, URI.parse('mem:fff.ts')))
);
assertVariableResolve(resolver, 'TM_DIRECTORY', '');
assertVariableResolve(resolver, 'TM_FILEPATH', 'fff.ts');
disposables.dispose();
});
test('Path delimiters in code snippet variables aren\'t specific to remote OS #76840', function () {
@ -103,6 +107,8 @@ suite('Snippet Variables Resolver', function () {
const resolver = new CompositeSnippetVariableResolver([new ModelBasedVariableResolver(labelService, model)]);
assertVariableResolve(resolver, 'TM_FILEPATH', '|foo|files|text.txt');
model.dispose();
});
test('editor variables, selection', function () {
@ -146,25 +152,29 @@ suite('Snippet Variables Resolver', function () {
test('More useful environment variables for snippets, #32737', function () {
const disposables = new DisposableStore();
assertVariableResolve(resolver, 'TM_FILENAME_BASE', 'text');
resolver = new ModelBasedVariableResolver(
labelService,
createTextModel('', undefined, undefined, URI.parse('http://www.pb.o/abc/def/ghi'))
disposables.add(createTextModel('', undefined, undefined, URI.parse('http://www.pb.o/abc/def/ghi')))
);
assertVariableResolve(resolver, 'TM_FILENAME_BASE', 'ghi');
resolver = new ModelBasedVariableResolver(
labelService,
createTextModel('', undefined, undefined, URI.parse('mem:.git'))
disposables.add(createTextModel('', undefined, undefined, URI.parse('mem:.git')))
);
assertVariableResolve(resolver, 'TM_FILENAME_BASE', '.git');
resolver = new ModelBasedVariableResolver(
labelService,
createTextModel('', undefined, undefined, URI.parse('mem:foo.'))
disposables.add(createTextModel('', undefined, undefined, URI.parse('mem:foo.')))
);
assertVariableResolve(resolver, 'TM_FILENAME_BASE', 'foo');
disposables.dispose();
});
@ -417,5 +427,7 @@ suite('Snippet Variables Resolver', function () {
} else {
assertVariableResolve(resolver, 'RELATIVE_FILEPATH', 'files\\text.txt');
}
model.dispose();
});
});

12
src/vs/editor/contrib/suggest/suggestMemory.ts
View file

@ -10,7 +10,6 @@ import { LRUCache, TernarySearchTree } from 'vs/base/common/map';
import { IPosition } from 'vs/editor/common/core/position';
import { ITextModel } from 'vs/editor/common/model';
import { CompletionItemKind, completionKindFromString } from 'vs/editor/common/modes';
import { IModeService } from 'vs/editor/common/services/modeService';
import { CompletionItem } from 'vs/editor/contrib/suggest/suggest';
import { IConfigurationService } from 'vs/platform/configuration/common/configuration';
import { registerSingleton } from 'vs/platform/instantiation/common/extensions';
@ -82,7 +81,7 @@ export class LRUMemory extends Memory {
private _seq = 0;
memorize(model: ITextModel, pos: IPosition, item: CompletionItem): void {
const key = `${model.getLanguageIdentifier().language}/${item.textLabel}`;
const key = `${model.getLanguageId()}/${item.textLabel}`;
this._cache.set(key, {
touch: this._seq++,
type: item.completion.kind,
@ -110,7 +109,7 @@ export class LRUMemory extends Memory {
// consider only top items
break;
}
const key = `${model.getLanguageIdentifier().language}/${items[i].textLabel}`;
const key = `${model.getLanguageId()}/${items[i].textLabel}`;
const item = this._cache.peek(key);
if (item && item.touch > seq && item.type === items[i].completion.kind && item.insertText === items[i].completion.insertText) {
seq = item.touch;
@ -158,7 +157,7 @@ export class PrefixMemory extends Memory {
memorize(model: ITextModel, pos: IPosition, item: CompletionItem): void {
const { word } = model.getWordUntilPosition(pos);
const key = `${model.getLanguageIdentifier().language}/${word}`;
const key = `${model.getLanguageId()}/${word}`;
this._trie.set(key, {
type: item.completion.kind,
insertText: item.completion.insertText,
@ -171,7 +170,7 @@ export class PrefixMemory extends Memory {
if (!word) {
return super.select(model, pos, items);
}
let key = `${model.getLanguageIdentifier().language}/${word}`;
let key = `${model.getLanguageId()}/${word}`;
let item = this._trie.get(key);
if (!item) {
item = this._trie.findSubstr(key);
@ -236,7 +235,6 @@ export class SuggestMemoryService implements ISuggestMemoryService {
constructor(
@IStorageService private readonly _storageService: IStorageService,
@IModeService private readonly _modeService: IModeService,
@IConfigurationService private readonly _configService: IConfigurationService,
) {
this._persistSoon = new RunOnceScheduler(() => this._saveState(), 500);
@ -264,7 +262,7 @@ export class SuggestMemoryService implements ISuggestMemoryService {
private _withStrategy(model: ITextModel, pos: IPosition): Memory {
const mode = this._configService.getValue<MemMode>('editor.suggestSelection', {
overrideIdentifier: this._modeService.getLanguageIdentifier(model.getLanguageIdAtPosition(pos.lineNumber, pos.column))?.language,
overrideIdentifier: model.getLanguageIdAtPosition(pos.lineNumber, pos.column),
resource: model.uri
});

6
src/vs/editor/contrib/suggest/test/suggestController.test.ts
View file

@ -70,11 +70,11 @@ suite('SuggestController', function () {
[IWorkspaceContextService, new class extends mock<IWorkspaceContextService>() { }],
);
model = createTextModel('', undefined, undefined, URI.from({ scheme: 'test-ctrl', path: '/path.tst' }));
editor = createTestCodeEditor({
model = disposables.add(createTextModel('', undefined, undefined, URI.from({ scheme: 'test-ctrl', path: '/path.tst' })));
editor = disposables.add(createTestCodeEditor({
model,
serviceCollection,
});
}));
editor.registerAndInstantiateContribution(SnippetController2.ID, SnippetController2);
controller = editor.registerAndInstantiateContribution(SuggestController.ID, SuggestController);

4
src/vs/editor/contrib/suggest/test/suggestMemory.test.ts
View file

@ -26,6 +26,10 @@ suite('SuggestMemories', function () {
];
});
teardown(() => {
buffer.dispose();
});
test('AbstractMemory, select', function () {
const mem = new class extends Memory {

100
src/vs/editor/contrib/suggest/test/suggestModel.test.ts
View file

@ -4,7 +4,7 @@
*--------------------------------------------------------------------------------------------*/
import * as assert from 'assert';
import { Event } from 'vs/base/common/event';
import { Disposable, dispose, IDisposable } from 'vs/base/common/lifecycle';
import { DisposableStore } from 'vs/base/common/lifecycle';
import { URI } from 'vs/base/common/uri';
import { mock } from 'vs/base/test/common/mock';
import { CoreEditingCommands } from 'vs/editor/browser/controller/coreCommands';
@ -16,17 +16,18 @@ import { TokenizationResult2 } from 'vs/editor/common/core/token';
import { Handler } from 'vs/editor/common/editorCommon';
import { ITextModel } from 'vs/editor/common/model';
import { TextModel } from 'vs/editor/common/model/textModel';
import { CompletionItemKind, CompletionItemProvider, CompletionList, CompletionProviderRegistry, CompletionTriggerKind, IState, LanguageIdentifier, MetadataConsts, TokenizationRegistry } from 'vs/editor/common/modes';
import { CompletionItemKind, CompletionItemProvider, CompletionList, CompletionProviderRegistry, CompletionTriggerKind, IState, MetadataConsts, TokenizationRegistry } from 'vs/editor/common/modes';
import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry';
import { NULL_STATE } from 'vs/editor/common/modes/nullMode';
import { IEditorWorkerService } from 'vs/editor/common/services/editorWorkerService';
import { IModeService } from 'vs/editor/common/services/modeService';
import { SnippetController2 } from 'vs/editor/contrib/snippet/snippetController2';
import { SuggestController } from 'vs/editor/contrib/suggest/suggestController';
import { ISuggestMemoryService } from 'vs/editor/contrib/suggest/suggestMemory';
import { LineContext, SuggestModel } from 'vs/editor/contrib/suggest/suggestModel';
import { ISelectedSuggestion } from 'vs/editor/contrib/suggest/suggestWidget';
import { createTestCodeEditor, ITestCodeEditor } from 'vs/editor/test/browser/testCodeEditor';
import { createTextModel } from 'vs/editor/test/common/editorTestUtils';
import { createModelServices, createTextModel, createTextModel2 } from 'vs/editor/test/common/editorTestUtils';
import { MockMode } from 'vs/editor/test/common/mocks/mockMode';
import { IClipboardService } from 'vs/platform/clipboard/common/clipboardService';
import { TestConfigurationService } from 'vs/platform/configuration/test/common/testConfigurationService';
@ -65,25 +66,28 @@ function createMockEditor(model: TextModel): ITestCodeEditor {
}
suite('SuggestModel - Context', function () {
const OUTER_LANGUAGE_ID = new LanguageIdentifier('outerMode', 3);
const INNER_LANGUAGE_ID = new LanguageIdentifier('innerMode', 4);
const OUTER_LANGUAGE_ID = 'outerMode';
const INNER_LANGUAGE_ID = 'innerMode';
class OuterMode extends MockMode {
constructor() {
constructor(
@IModeService modeService: IModeService
) {
super(OUTER_LANGUAGE_ID);
this._register(LanguageConfigurationRegistry.register(this.getLanguageIdentifier(), {}));
this._register(LanguageConfigurationRegistry.register(this.languageId, {}));
this._register(TokenizationRegistry.register(this.getLanguageIdentifier().language, {
this._register(TokenizationRegistry.register(this.languageId, {
getInitialState: (): IState => NULL_STATE,
tokenize: undefined!,
tokenize2: (line: string, hasEOL: boolean, state: IState): TokenizationResult2 => {
const tokensArr: number[] = [];
let prevLanguageId: LanguageIdentifier | undefined = undefined;
let prevLanguageId: string | undefined = undefined;
for (let i = 0; i < line.length; i++) {
const languageId = (line.charAt(i) === 'x' ? INNER_LANGUAGE_ID : OUTER_LANGUAGE_ID);
const encodedLanguageId = modeService.languageIdCodec.encodeLanguageId(languageId);
if (prevLanguageId !== languageId) {
tokensArr.push(i);
tokensArr.push((languageId.id << MetadataConsts.LANGUAGEID_OFFSET));
tokensArr.push((encodedLanguageId << MetadataConsts.LANGUAGEID_OFFSET));
}
prevLanguageId = languageId;
}
@ -101,7 +105,7 @@ suite('SuggestModel - Context', function () {
class InnerMode extends MockMode {
constructor() {
super(INNER_LANGUAGE_ID);
this._register(LanguageConfigurationRegistry.register(this.getLanguageIdentifier(), {}));
this._register(LanguageConfigurationRegistry.register(this.languageId, {}));
}
}
@ -113,34 +117,33 @@ suite('SuggestModel - Context', function () {
editor.dispose();
};
let disposables: Disposable[] = [];
let disposables: DisposableStore;
setup(() => {
disposables = [];
disposables = new DisposableStore();
});
teardown(function () {
dispose(disposables);
disposables = [];
disposables.dispose();
});
test('Context - shouldAutoTrigger', function () {
const model = createTextModel('Das Pferd frisst keinen Gurkensalat - Philipp Reis 1861.\nWer hat\'s erfunden?');
disposables.push(model);
disposables.add(model);
assertAutoTrigger(model, 3, true, 'end of word, Das|');
assertAutoTrigger(model, 4, false, 'no word Das |');
assertAutoTrigger(model, 1, false, 'middle of word D|as');
assertAutoTrigger(model, 55, false, 'number, 1861|');
model.dispose();
});
test('shouldAutoTrigger at embedded language boundaries', () => {
const outerMode = new OuterMode();
const innerMode = new InnerMode();
disposables.push(outerMode, innerMode);
const [instantiationService, disposables] = createModelServices();
const outerMode = disposables.add(instantiationService.createInstance(OuterMode));
disposables.add(instantiationService.createInstance(InnerMode));
const model = createTextModel('a<xx>a<x>', undefined, outerMode.getLanguageIdentifier());
disposables.push(model);
const model = disposables.add(createTextModel2(instantiationService, 'a<xx>a<x>', undefined, outerMode.languageId));
assertAutoTrigger(model, 1, true, 'a|<x — should trigger at end of word');
assertAutoTrigger(model, 2, false, 'a<|x — should NOT trigger at start of word');
@ -149,6 +152,8 @@ suite('SuggestModel - Context', function () {
assertAutoTrigger(model, 5, false, 'a<xx>|a — should NOT trigger at start of word');
assertAutoTrigger(model, 6, true, 'a<xx>a|< — should trigger at end of word');
assertAutoTrigger(model, 8, true, 'a<xx>a<x|> — should trigger at end of word at boundary');
disposables.dispose();
});
});
@ -183,13 +188,17 @@ suite('SuggestModel - TriggerAndCancelOracle', function () {
}
};
let disposables: IDisposable[] = [];
let disposables: DisposableStore;
let model: TextModel;
setup(function () {
disposables = dispose(disposables);
disposables = new DisposableStore();
model = createTextModel('abc def', undefined, undefined, URI.parse('test:somefile.ttt'));
disposables.push(model);
disposables.add(model);
});
teardown(() => {
disposables.dispose();
});
function withOracle(callback: (model: SuggestModel, editor: ITestCodeEditor) => any): Promise<any> {
@ -213,7 +222,8 @@ suite('SuggestModel - TriggerAndCancelOracle', function () {
new MockContextKeyService(),
new TestConfigurationService()
);
disposables.push(oracle, editor);
disposables.add(oracle);
disposables.add(editor);
try {
resolve(callback(oracle, editor));
@ -277,7 +287,7 @@ suite('SuggestModel - TriggerAndCancelOracle', function () {
test('events - suggest/empty', function () {
disposables.push(CompletionProviderRegistry.register({ scheme: 'test' }, alwaysEmptySupport));
disposables.add(CompletionProviderRegistry.register({ scheme: 'test' }, alwaysEmptySupport));
return withOracle(model => {
return Promise.all([
@ -299,7 +309,7 @@ suite('SuggestModel - TriggerAndCancelOracle', function () {
test('trigger - on type', function () {
disposables.push(CompletionProviderRegistry.register({ scheme: 'test' }, alwaysSomethingSupport));
disposables.add(CompletionProviderRegistry.register({ scheme: 'test' }, alwaysSomethingSupport));
return withOracle((model, editor) => {
return assertEvent(model.onDidSuggest, () => {
@ -318,7 +328,7 @@ suite('SuggestModel - TriggerAndCancelOracle', function () {
test('#17400: Keep filtering suggestModel.ts after space', function () {
disposables.push(CompletionProviderRegistry.register({ scheme: 'test' }, {
disposables.add(CompletionProviderRegistry.register({ scheme: 'test' }, {
provideCompletionItems(doc, pos): CompletionList {
return {
incomplete: false,
@ -368,7 +378,7 @@ suite('SuggestModel - TriggerAndCancelOracle', function () {
test('#21484: Trigger character always force a new completion session', function () {
disposables.push(CompletionProviderRegistry.register({ scheme: 'test' }, {
disposables.add(CompletionProviderRegistry.register({ scheme: 'test' }, {
provideCompletionItems(doc, pos): CompletionList {
return {
incomplete: false,
@ -382,7 +392,7 @@ suite('SuggestModel - TriggerAndCancelOracle', function () {
}
}));
disposables.push(CompletionProviderRegistry.register({ scheme: 'test' }, {
disposables.add(CompletionProviderRegistry.register({ scheme: 'test' }, {
triggerCharacters: ['.'],
provideCompletionItems(doc, pos): CompletionList {
return {
@ -430,7 +440,7 @@ suite('SuggestModel - TriggerAndCancelOracle', function () {
test('Intellisense Completion doesn\'t respect space after equal sign (.html file), #29353 [1/2]', function () {
disposables.push(CompletionProviderRegistry.register({ scheme: 'test' }, alwaysSomethingSupport));
disposables.add(CompletionProviderRegistry.register({ scheme: 'test' }, alwaysSomethingSupport));
return withOracle((model, editor) => {
@ -455,7 +465,7 @@ suite('SuggestModel - TriggerAndCancelOracle', function () {
test('Intellisense Completion doesn\'t respect space after equal sign (.html file), #29353 [2/2]', function () {
disposables.push(CompletionProviderRegistry.register({ scheme: 'test' }, alwaysSomethingSupport));
disposables.add(CompletionProviderRegistry.register({ scheme: 'test' }, alwaysSomethingSupport));
return withOracle((model, editor) => {
@ -480,7 +490,7 @@ suite('SuggestModel - TriggerAndCancelOracle', function () {
test('Incomplete suggestion results cause re-triggering when typing w/o further context, #28400 (1/2)', function () {
disposables.push(CompletionProviderRegistry.register({ scheme: 'test' }, {
disposables.add(CompletionProviderRegistry.register({ scheme: 'test' }, {
provideCompletionItems(doc, pos): CompletionList {
return {
incomplete: true,
@ -517,7 +527,7 @@ suite('SuggestModel - TriggerAndCancelOracle', function () {
test('Incomplete suggestion results cause re-triggering when typing w/o further context, #28400 (2/2)', function () {
disposables.push(CompletionProviderRegistry.register({ scheme: 'test' }, {
disposables.add(CompletionProviderRegistry.register({ scheme: 'test' }, {
provideCompletionItems(doc, pos): CompletionList {
return {
incomplete: true,
@ -560,7 +570,7 @@ suite('SuggestModel - TriggerAndCancelOracle', function () {
test('Trigger character is provided in suggest context', function () {
let triggerCharacter = '';
disposables.push(CompletionProviderRegistry.register({ scheme: 'test' }, {
disposables.add(CompletionProviderRegistry.register({ scheme: 'test' }, {
triggerCharacters: ['.'],
provideCompletionItems(doc, pos, context): CompletionList {
assert.strictEqual(context.triggerKind, CompletionTriggerKind.TriggerCharacter);
@ -593,7 +603,7 @@ suite('SuggestModel - TriggerAndCancelOracle', function () {
});
test('Mac press and hold accent character insertion does not update suggestions, #35269', function () {
disposables.push(CompletionProviderRegistry.register({ scheme: 'test' }, {
disposables.add(CompletionProviderRegistry.register({ scheme: 'test' }, {
provideCompletionItems(doc, pos): CompletionList {
return {
incomplete: true,
@ -636,7 +646,7 @@ suite('SuggestModel - TriggerAndCancelOracle', function () {
});
test('Backspace should not always cancel code completion, #36491', function () {
disposables.push(CompletionProviderRegistry.register({ scheme: 'test' }, alwaysSomethingSupport));
disposables.add(CompletionProviderRegistry.register({ scheme: 'test' }, alwaysSomethingSupport));
return withOracle(async (model, editor) => {
await assertEvent(model.onDidSuggest, () => {
@ -665,7 +675,7 @@ suite('SuggestModel - TriggerAndCancelOracle', function () {
});
test('Text changes for completion CodeAction are affected by the completion #39893', function () {
disposables.push(CompletionProviderRegistry.register({ scheme: 'test' }, {
disposables.add(CompletionProviderRegistry.register({ scheme: 'test' }, {
provideCompletionItems(doc, pos): CompletionList {
return {
incomplete: true,
@ -715,7 +725,7 @@ suite('SuggestModel - TriggerAndCancelOracle', function () {
test('Completion unexpectedly triggers on second keypress of an edit group in a snippet #43523', function () {
disposables.push(CompletionProviderRegistry.register({ scheme: 'test' }, alwaysSomethingSupport));
disposables.add(CompletionProviderRegistry.register({ scheme: 'test' }, alwaysSomethingSupport));
return withOracle((model, editor) => {
return assertEvent(model.onDidSuggest, () => {
@ -739,7 +749,7 @@ suite('SuggestModel - TriggerAndCancelOracle', function () {
let disposeA = 0;
let disposeB = 0;
disposables.push(CompletionProviderRegistry.register({ scheme: 'test' }, {
disposables.add(CompletionProviderRegistry.register({ scheme: 'test' }, {
provideCompletionItems(doc, pos) {
return {
incomplete: true,
@ -754,7 +764,7 @@ suite('SuggestModel - TriggerAndCancelOracle', function () {
};
}
}));
disposables.push(CompletionProviderRegistry.register({ scheme: 'test' }, {
disposables.add(CompletionProviderRegistry.register({ scheme: 'test' }, {
provideCompletionItems(doc, pos) {
return {
incomplete: false,
@ -808,7 +818,7 @@ suite('SuggestModel - TriggerAndCancelOracle', function () {
let countA = 0;
let countB = 0;
disposables.push(CompletionProviderRegistry.register({ scheme: 'test' }, {
disposables.add(CompletionProviderRegistry.register({ scheme: 'test' }, {
provideCompletionItems(doc, pos) {
countA += 1;
return {
@ -822,7 +832,7 @@ suite('SuggestModel - TriggerAndCancelOracle', function () {
};
}
}));
disposables.push(CompletionProviderRegistry.register({ scheme: 'test' }, {
disposables.add(CompletionProviderRegistry.register({ scheme: 'test' }, {
provideCompletionItems(doc, pos) {
countB += 1;
if (!doc.getWordUntilPosition(pos).word.startsWith('a')) {
@ -871,7 +881,7 @@ suite('SuggestModel - TriggerAndCancelOracle', function () {
test('registerCompletionItemProvider with letters as trigger characters block other completion items to show up #127815', async function () {
disposables.push(CompletionProviderRegistry.register({ scheme: 'test' }, {
disposables.add(CompletionProviderRegistry.register({ scheme: 'test' }, {
provideCompletionItems(doc, pos) {
return {
suggestions: [{
@ -883,7 +893,7 @@ suite('SuggestModel - TriggerAndCancelOracle', function () {
};
}
}));
disposables.push(CompletionProviderRegistry.register({ scheme: 'test' }, {
disposables.add(CompletionProviderRegistry.register({ scheme: 'test' }, {
triggerCharacters: ['a', '.'],
provideCompletionItems(doc, pos) {
return {

6
src/vs/editor/contrib/suggest/test/wordDistance.test.ts
View file

@ -28,11 +28,11 @@ suite('suggest, word distance', function () {
class BracketMode extends MockMode {
private static readonly _id = new modes.LanguageIdentifier('bracketMode', 3);
private static readonly _id = 'bracketMode';
constructor() {
super(BracketMode._id);
this._register(LanguageConfigurationRegistry.register(this.getLanguageIdentifier(), {
this._register(LanguageConfigurationRegistry.register(this.languageId, {
brackets: [
['{', '}'],
['[', ']'],
@ -48,7 +48,7 @@ suite('suggest, word distance', function () {
disposables.clear();
let mode = new BracketMode();
let model = createTextModel('function abc(aa, ab){\na\n}', undefined, mode.getLanguageIdentifier(), URI.parse('test:///some.path'));
let model = createTextModel('function abc(aa, ab){\na\n}', undefined, mode.languageId, URI.parse('test:///some.path'));
let editor = createTestCodeEditor({ model: model });
editor.updateOptions({ suggest: { localityBonus: true } });
editor.setPosition({ lineNumber: 2, column: 2 });

2
src/vs/editor/contrib/viewportSemanticTokens/viewportSemanticTokens.ts
View file

@ -117,7 +117,7 @@ class ViewportSemanticTokensContribution extends Disposable implements IEditorCo
if (!r || model.isDisposed() || model.getVersionId() !== requestVersionId) {
return;
}
model.setPartialSemanticTokens(range, toMultilineTokens2(r, styling, model.getLanguageIdentifier()));
model.setPartialSemanticTokens(range, toMultilineTokens2(r, styling, model.getLanguageId()));
}).then(() => this._removeOutstandingRequest(request), () => this._removeOutstandingRequest(request));
return request;
}

5
src/vs/editor/contrib/wordOperations/test/wordOperations.test.ts
View file

@ -9,7 +9,6 @@ import { ICodeEditor } from 'vs/editor/browser/editorBrowser';
import { EditorCommand } from 'vs/editor/browser/editorExtensions';
import { Position } from 'vs/editor/common/core/position';
import { Selection } from 'vs/editor/common/core/selection';
import { LanguageIdentifier } from 'vs/editor/common/modes';
import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry';
import { ViewModel } from 'vs/editor/common/viewModel/viewModelImpl';
import { deserializePipePositions, serializePipePositions, testRepeatedActionAndExtractPositions } from 'vs/editor/contrib/wordOperations/test/wordTestUtils';
@ -731,11 +730,11 @@ suite('WordOperations', () => {
});
test('deleteWordLeft - issue #91855: Matching (quote, bracket, paren) doesn\'t get deleted when hitting Ctrl+Backspace', () => {
const languageId = new LanguageIdentifier('myTestMode', 5);
const languageId = 'myTestMode';
class TestMode extends MockMode {
constructor() {
super(languageId);
this._register(LanguageConfigurationRegistry.register(this.getLanguageIdentifier(), {
this._register(LanguageConfigurationRegistry.register(this.languageId, {
autoClosingPairs: [
{ open: '\"', close: '\"' }
]

2
src/vs/editor/contrib/wordOperations/wordOperations.ts
View file

@ -339,7 +339,7 @@ export abstract class DeleteWordCommand extends EditorCommand {
const selections = editor.getSelections();
const autoClosingBrackets = editor.getOption(EditorOption.autoClosingBrackets);
const autoClosingQuotes = editor.getOption(EditorOption.autoClosingQuotes);
const autoClosingPairs = LanguageConfigurationRegistry.getAutoClosingPairs(model.getLanguageIdentifier().id);
const autoClosingPairs = LanguageConfigurationRegistry.getAutoClosingPairs(model.getLanguageId());
const viewModel = editor._getViewModel();
const commands = selections.map((sel) => {

25
src/vs/editor/standalone/browser/colorizer.ts
View file

@ -8,7 +8,7 @@ import { IDisposable } from 'vs/base/common/lifecycle';
import * as strings from 'vs/base/common/strings';
import { IViewLineTokens, LineTokens } from 'vs/editor/common/core/lineTokens';
import { ITextModel } from 'vs/editor/common/model';
import { ColorId, FontStyle, ITokenizationSupport, MetadataConsts, TokenizationRegistry } from 'vs/editor/common/modes';
import { ColorId, FontStyle, ILanguageIdCodec, ITokenizationSupport, MetadataConsts, TokenizationRegistry } from 'vs/editor/common/modes';
import { IModeService } from 'vs/editor/common/services/modeService';
import { RenderLineInput, renderViewLine2 as renderViewLine } from 'vs/editor/common/viewLayout/viewLineRenderer';
import { ViewLineRenderingData } from 'vs/editor/common/viewModel/viewModel';
@ -49,6 +49,7 @@ export class Colorizer {
}
public static colorize(modeService: IModeService, text: string, mimeType: string, options: IColorizerOptions | null | undefined): Promise<string> {
const languageIdCodec = modeService.languageIdCodec;
let tabSize = 4;
if (options && typeof options.tabSize === 'number') {
tabSize = options.tabSize;
@ -60,7 +61,7 @@ export class Colorizer {
let lines = strings.splitLines(text);
let language = modeService.getModeId(mimeType);
if (!language) {
return Promise.resolve(_fakeColorize(lines, tabSize));
return Promise.resolve(_fakeColorize(lines, tabSize, languageIdCodec));
}
// Send out the event to create the mode
@ -68,7 +69,7 @@ export class Colorizer {
const tokenizationSupport = TokenizationRegistry.get(language);
if (tokenizationSupport) {
return _colorize(lines, tabSize, tokenizationSupport);
return _colorize(lines, tabSize, tokenizationSupport, languageIdCodec);
}
const tokenizationSupportPromise = TokenizationRegistry.getPromise(language);
@ -76,7 +77,7 @@ export class Colorizer {
// A tokenizer will be registered soon
return new Promise<string>((resolve, reject) => {
tokenizationSupportPromise.then(tokenizationSupport => {
_colorize(lines, tabSize, tokenizationSupport).then(resolve, reject);
_colorize(lines, tabSize, tokenizationSupport, languageIdCodec).then(resolve, reject);
}, reject);
});
}
@ -96,10 +97,10 @@ export class Colorizer {
}
const tokenizationSupport = TokenizationRegistry.get(language!);
if (tokenizationSupport) {
_colorize(lines, tabSize, tokenizationSupport).then(resolve, reject);
_colorize(lines, tabSize, tokenizationSupport, languageIdCodec).then(resolve, reject);
return;
}
resolve(_fakeColorize(lines, tabSize));
resolve(_fakeColorize(lines, tabSize, languageIdCodec));
};
// wait 500ms for mode to load, then give up
@ -149,10 +150,10 @@ export class Colorizer {
}
}
function _colorize(lines: string[], tabSize: number, tokenizationSupport: ITokenizationSupport): Promise<string> {
function _colorize(lines: string[], tabSize: number, tokenizationSupport: ITokenizationSupport, languageIdCodec: ILanguageIdCodec): Promise<string> {
return new Promise<string>((c, e) => {
const execute = () => {
const result = _actualColorize(lines, tabSize, tokenizationSupport);
const result = _actualColorize(lines, tabSize, tokenizationSupport, languageIdCodec);
if (tokenizationSupport instanceof MonarchTokenizer) {
const status = tokenizationSupport.getLoadStatus();
if (status.loaded === false) {
@ -166,7 +167,7 @@ function _colorize(lines: string[], tabSize: number, tokenizationSupport: IToken
});
}
function _fakeColorize(lines: string[], tabSize: number): string {
function _fakeColorize(lines: string[], tabSize: number, languageIdCodec: ILanguageIdCodec): string {
let html: string[] = [];
const defaultMetadata = (
@ -183,7 +184,7 @@ function _fakeColorize(lines: string[], tabSize: number): string {
let line = lines[i];
tokens[0] = line.length;
const lineTokens = new LineTokens(tokens, line);
const lineTokens = new LineTokens(tokens, line, languageIdCodec);
const isBasicASCII = ViewLineRenderingData.isBasicASCII(line, /* check for basic ASCII */true);
const containsRTL = ViewLineRenderingData.containsRTL(line, isBasicASCII, /* check for RTL */true);
@ -216,7 +217,7 @@ function _fakeColorize(lines: string[], tabSize: number): string {
return html.join('');
}
function _actualColorize(lines: string[], tabSize: number, tokenizationSupport: ITokenizationSupport): string {
function _actualColorize(lines: string[], tabSize: number, tokenizationSupport: ITokenizationSupport, languageIdCodec: ILanguageIdCodec): string {
let html: string[] = [];
let state = tokenizationSupport.getInitialState();
@ -224,7 +225,7 @@ function _actualColorize(lines: string[], tabSize: number, tokenizationSupport:
let line = lines[i];
let tokenizeResult = tokenizationSupport.tokenize2(line, true, state, 0);
LineTokens.convertToEndOffset(tokenizeResult.tokens, line.length);
let lineTokens = new LineTokens(tokenizeResult.tokens, line);
let lineTokens = new LineTokens(tokenizeResult.tokens, line, languageIdCodec);
const isBasicASCII = ViewLineRenderingData.isBasicASCII(line, /* check for basic ASCII */true);
const containsRTL = ViewLineRenderingData.containsRTL(line, isBasicASCII, /* check for RTL */true);
let renderResult = renderViewLine(new RenderLineInput(

19
src/vs/editor/standalone/browser/inspectTokens/inspectTokens.ts
View file

@ -15,7 +15,7 @@ import { Position } from 'vs/editor/common/core/position';
import { Token } from 'vs/editor/common/core/token';
import { IEditorContribution } from 'vs/editor/common/editorCommon';
import { ITextModel } from 'vs/editor/common/model';
import { FontStyle, IState, ITokenizationSupport, LanguageIdentifier, StandardTokenType, TokenMetadata, TokenizationRegistry } from 'vs/editor/common/modes';
import { FontStyle, IState, ITokenizationSupport, StandardTokenType, TokenMetadata, TokenizationRegistry, ILanguageIdCodec } from 'vs/editor/common/modes';
import { NULL_STATE, nullTokenize, nullTokenize2 } from 'vs/editor/common/modes/nullMode';
import { IModeService } from 'vs/editor/common/services/modeService';
import { IStandaloneThemeService } from 'vs/editor/standalone/common/standaloneThemeService';
@ -103,7 +103,7 @@ interface ICompleteLineTokenization {
}
interface IDecodedMetadata {
languageIdentifier: LanguageIdentifier;
languageId: string;
tokenType: StandardTokenType;
fontStyle: FontStyle;
foreground: Color;
@ -130,15 +130,16 @@ function renderTokenText(tokenText: string): string {
return result;
}
function getSafeTokenizationSupport(languageIdentifier: LanguageIdentifier): ITokenizationSupport {
let tokenizationSupport = TokenizationRegistry.get(languageIdentifier.language);
function getSafeTokenizationSupport(languageIdCodec: ILanguageIdCodec, languageId: string): ITokenizationSupport {
const tokenizationSupport = TokenizationRegistry.get(languageId);
if (tokenizationSupport) {
return tokenizationSupport;
}
const encodedLanguageId = languageIdCodec.encodeLanguageId(languageId);
return {
getInitialState: () => NULL_STATE,
tokenize: (line: string, hasEOL: boolean, state: IState, deltaOffset: number) => nullTokenize(languageIdentifier.language, line, state, deltaOffset),
tokenize2: (line: string, hasEOL: boolean, state: IState, deltaOffset: number) => nullTokenize2(languageIdentifier.id, line, state, deltaOffset)
tokenize: (line: string, hasEOL: boolean, state: IState, deltaOffset: number) => nullTokenize(languageId, line, state, deltaOffset),
tokenize2: (line: string, hasEOL: boolean, state: IState, deltaOffset: number) => nullTokenize2(encodedLanguageId, line, state, deltaOffset)
};
}
@ -165,7 +166,7 @@ class InspectTokensWidget extends Disposable implements IContentWidget {
this._model = this._editor.getModel();
this._domNode = document.createElement('div');
this._domNode.className = 'tokens-inspect-widget';
this._tokenizationSupport = getSafeTokenizationSupport(this._model.getLanguageIdentifier());
this._tokenizationSupport = getSafeTokenizationSupport(this._modeService.languageIdCodec, this._model.getLanguageId());
this._compute(this._editor.getPosition());
this._register(this._editor.onDidChangeCursorPosition((e) => this._compute(this._editor.getPosition())));
this._editor.addContentWidget(this);
@ -218,7 +219,7 @@ class InspectTokensWidget extends Disposable implements IContentWidget {
$('tbody', undefined,
$('tr', undefined,
$('td.tm-metadata-key', undefined, 'language'),
$('td.tm-metadata-value', undefined, `${metadata ? metadata.languageIdentifier.language : '-?-'}`)
$('td.tm-metadata-value', undefined, `${metadata ? metadata.languageId : '-?-'}`)
),
$('tr', undefined,
$('td.tm-metadata-key', undefined, 'token type' as string),
@ -255,7 +256,7 @@ class InspectTokensWidget extends Disposable implements IContentWidget {
let foreground = TokenMetadata.getForeground(metadata);
let background = TokenMetadata.getBackground(metadata);
return {
languageIdentifier: this._modeService.getLanguageIdentifier(languageId)!,
languageId: this._modeService.languageIdCodec.decodeLanguageId(languageId),
tokenType: tokenType,
fontStyle: fontStyle,
foreground: colorMap[foreground],

2
src/vs/editor/standalone/browser/simpleServices.ts
View file

@ -94,7 +94,7 @@ export class SimpleModel implements IResolvedTextEditorModel {
}
public getMode(): string | undefined {
return this.model.getModeId();
return this.model.getLanguageId();
}
}

59
src/vs/editor/standalone/browser/standaloneLanguages.ts
View file

@ -14,7 +14,7 @@ import * as modes from 'vs/editor/common/modes';
import { LanguageConfiguration } from 'vs/editor/common/modes/languageConfiguration';
import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry';
import { ModesRegistry } from 'vs/editor/common/modes/modesRegistry';
import { ILanguageExtensionPoint } from 'vs/editor/common/services/modeService';
import { ILanguageExtensionPoint, IModeService } from 'vs/editor/common/services/modeService';
import * as standaloneEnums from 'vs/editor/common/standalone/standaloneEnums';
import { StaticServices } from 'vs/editor/standalone/browser/standaloneServices';
import { compile } from 'vs/editor/standalone/common/monarch/monarchCompile';
@ -40,8 +40,8 @@ export function getLanguages(): ILanguageExtensionPoint[] {
}
export function getEncodedLanguageId(languageId: string): number {
let lid = StaticServices.modeService.get().getLanguageIdentifier(languageId);
return lid ? lid.id : 0;
const modeService = StaticServices.modeService.get();
return modeService.languageIdCodec.encodeLanguageId(languageId);
}
/**
@ -49,8 +49,8 @@ export function getEncodedLanguageId(languageId: string): number {
* @event
*/
export function onLanguage(languageId: string, callback: () => void): IDisposable {
let disposable = StaticServices.modeService.get().onDidEncounterLanguage((languageIdentifier) => {
if (languageIdentifier.language === languageId) {
let disposable = StaticServices.modeService.get().onDidEncounterLanguage((languageId) => {
if (languageId === languageId) {
// stop listening
disposable.dispose();
// invoke actual listener
@ -64,11 +64,11 @@ export function onLanguage(languageId: string, callback: () => void): IDisposabl
* Set the editing configuration for a language.
*/
export function setLanguageConfiguration(languageId: string, configuration: LanguageConfiguration): IDisposable {
let languageIdentifier = StaticServices.modeService.get().getLanguageIdentifier(languageId);
if (!languageIdentifier) {
const validLanguageId = StaticServices.modeService.get().validateLanguageId(languageId);
if (!validLanguageId) {
throw new Error(`Cannot set configuration for unknown language ${languageId}`);
}
return LanguageConfigurationRegistry.register(languageIdentifier, configuration, 100);
return LanguageConfigurationRegistry.register(validLanguageId, configuration, 100);
}
/**
@ -76,11 +76,11 @@ export function setLanguageConfiguration(languageId: string, configuration: Lang
*/
export class EncodedTokenizationSupport2Adapter implements modes.ITokenizationSupport {
private readonly _languageIdentifier: modes.LanguageIdentifier;
private readonly _languageId: string;
private readonly _actual: EncodedTokensProvider;
constructor(languageIdentifier: modes.LanguageIdentifier, actual: EncodedTokensProvider) {
this._languageIdentifier = languageIdentifier;
constructor(languageId: string, actual: EncodedTokensProvider) {
this._languageId = languageId;
this._actual = actual;
}
@ -90,7 +90,7 @@ export class EncodedTokenizationSupport2Adapter implements modes.ITokenizationSu
public tokenize(line: string, hasEOL: boolean, state: modes.IState, offsetDelta: number): TokenizationResult {
if (typeof this._actual.tokenize === 'function') {
return TokenizationSupport2Adapter.adaptTokenize(this._languageIdentifier.language, <{ tokenize(line: string, state: modes.IState): ILineTokens; }>this._actual, line, state, offsetDelta);
return TokenizationSupport2Adapter.adaptTokenize(this._languageId, <{ tokenize(line: string, state: modes.IState): ILineTokens; }>this._actual, line, state, offsetDelta);
}
throw new Error('Not supported!');
}
@ -106,14 +106,12 @@ export class EncodedTokenizationSupport2Adapter implements modes.ITokenizationSu
*/
export class TokenizationSupport2Adapter implements modes.ITokenizationSupport {
private readonly _standaloneThemeService: IStandaloneThemeService;
private readonly _languageIdentifier: modes.LanguageIdentifier;
private readonly _actual: TokensProvider;
constructor(standaloneThemeService: IStandaloneThemeService, languageIdentifier: modes.LanguageIdentifier, actual: TokensProvider) {
this._standaloneThemeService = standaloneThemeService;
this._languageIdentifier = languageIdentifier;
this._actual = actual;
constructor(
private readonly _languageId: string,
private readonly _actual: TokensProvider,
private readonly _modeService: IModeService,
private readonly _standaloneThemeService: IStandaloneThemeService,
) {
}
public getInitialState(): modes.IState {
@ -159,11 +157,11 @@ export class TokenizationSupport2Adapter implements modes.ITokenizationSupport {
}
public tokenize(line: string, hasEOL: boolean, state: modes.IState, offsetDelta: number): TokenizationResult {
return TokenizationSupport2Adapter.adaptTokenize(this._languageIdentifier.language, this._actual, line, state, offsetDelta);
return TokenizationSupport2Adapter.adaptTokenize(this._languageId, this._actual, line, state, offsetDelta);
}
private _toBinaryTokens(tokens: IToken[], offsetDelta: number): Uint32Array {
const languageId = this._languageIdentifier.id;
private _toBinaryTokens(languageIdCodec: modes.ILanguageIdCodec, tokens: IToken[], offsetDelta: number): Uint32Array {
const languageId = languageIdCodec.encodeLanguageId(this._languageId);
const tokenTheme = this._standaloneThemeService.getColorTheme().tokenTheme;
let result: number[] = [], resultLen = 0;
@ -202,7 +200,7 @@ export class TokenizationSupport2Adapter implements modes.ITokenizationSupport {
public tokenize2(line: string, hasEOL: boolean, state: modes.IState, offsetDelta: number): TokenizationResult2 {
let actualResult = this._actual.tokenize(line, state);
let tokens = this._toBinaryTokens(actualResult.tokens, offsetDelta);
let tokens = this._toBinaryTokens(this._modeService.languageIdCodec, actualResult.tokens, offsetDelta);
let endState: modes.IState;
// try to save an object if possible
@ -331,15 +329,20 @@ export function setColorMap(colorMap: string[] | null): void {
* Set the tokens provider for a language (manual implementation).
*/
export function setTokensProvider(languageId: string, provider: TokensProvider | EncodedTokensProvider | Thenable<TokensProvider | EncodedTokensProvider>): IDisposable {
let languageIdentifier = StaticServices.modeService.get().getLanguageIdentifier(languageId);
if (!languageIdentifier) {
const validLanguageId = StaticServices.modeService.get().validateLanguageId(languageId);
if (!validLanguageId) {
throw new Error(`Cannot set tokens provider for unknown language ${languageId}`);
}
const create = (provider: TokensProvider | EncodedTokensProvider) => {
if (isEncodedTokensProvider(provider)) {
return new EncodedTokenizationSupport2Adapter(languageIdentifier!, provider);
return new EncodedTokenizationSupport2Adapter(validLanguageId, provider);
} else {
return new TokenizationSupport2Adapter(StaticServices.standaloneThemeService.get(), languageIdentifier!, provider);
return new TokenizationSupport2Adapter(
validLanguageId,
provider,
StaticServices.modeService.get(),
StaticServices.standaloneThemeService.get(),
);
}
};
if (isThenable<TokensProvider | EncodedTokensProvider>(provider)) {

1
src/vs/editor/standalone/browser/standaloneServices.ts
View file

@ -168,6 +168,7 @@ export module StaticServices {
standaloneThemeService.get(o),
logService.get(o),
undoRedoService.get(o),
modeService.get(o),
languageConfigurationService.get(o)
)
);

2
src/vs/editor/standalone/common/monarch/monarchLexer.ts
View file

@ -304,7 +304,7 @@ class MonarchModernTokensCollector implements IMonarchTokensCollector {
}
public enterMode(startOffset: number, modeId: string): void {
this._currentLanguageId = this._modeService.getLanguageIdentifier(modeId)!.id;
this._currentLanguageId = this._modeService.languageIdCodec.encodeLanguageId(modeId);
}
public emit(startOffset: number, type: string): void {

70
src/vs/editor/standalone/test/browser/standaloneLanguages.test.ts
View file

@ -6,9 +6,12 @@
import * as assert from 'assert';
import { Color } from 'vs/base/common/color';
import { Emitter } from 'vs/base/common/event';
import { DisposableStore } from 'vs/base/common/lifecycle';
import { Token } from 'vs/editor/common/core/token';
import { IState, LanguageId, LanguageIdentifier, MetadataConsts } from 'vs/editor/common/modes';
import { IState, LanguageId, MetadataConsts } from 'vs/editor/common/modes';
import { ModesRegistry } from 'vs/editor/common/modes/modesRegistry';
import { TokenTheme } from 'vs/editor/common/modes/supports/tokenization';
import { ModeServiceImpl } from 'vs/editor/common/services/modeServiceImpl';
import { ILineTokens, IToken, TokenizationSupport2Adapter, TokensProvider } from 'vs/editor/standalone/browser/standaloneLanguages';
import { IStandaloneTheme, IStandaloneThemeData, IStandaloneThemeService } from 'vs/editor/standalone/common/standaloneThemeService';
import { ColorIdentifier } from 'vs/platform/theme/common/colorRegistry';
@ -17,8 +20,8 @@ import { IFileIconTheme, IColorTheme, ITokenStyle } from 'vs/platform/theme/comm
suite('TokenizationSupport2Adapter', () => {
const languageIdentifier = new LanguageIdentifier('tttt', LanguageId.PlainText);
const tokenMetadata = (languageIdentifier.id << MetadataConsts.LANGUAGEID_OFFSET);
const languageId = 'tttt';
// const tokenMetadata = (LanguageId.PlainText << MetadataConsts.LANGUAGEID_OFFSET);
class MockTokenTheme extends TokenTheme {
private counter = 0;
@ -109,7 +112,15 @@ suite('TokenizationSupport2Adapter', () => {
}
}
const adapter = new TokenizationSupport2Adapter(new MockThemeService(), languageIdentifier, new BadTokensProvider());
const disposables = new DisposableStore();
const modeService = disposables.add(new ModeServiceImpl());
disposables.add(ModesRegistry.registerLanguage({ id: languageId }));
const adapter = new TokenizationSupport2Adapter(
languageId,
new BadTokensProvider(),
modeService,
new MockThemeService()
);
const actualClassicTokens = adapter.tokenize('whatever', true, MockState.INSTANCE, offsetDelta);
assert.deepStrictEqual(actualClassicTokens.tokens, expectedClassicTokens);
@ -119,10 +130,19 @@ suite('TokenizationSupport2Adapter', () => {
for (let i = 0; i < actualModernTokens.tokens.length; i++) {
modernTokens[i] = actualModernTokens.tokens[i];
}
// Add the encoded language id to the expected tokens
const encodedLanguageId = modeService.languageIdCodec.encodeLanguageId(languageId);
const tokenLanguageMetadata = (encodedLanguageId << MetadataConsts.LANGUAGEID_OFFSET);
for (let i = 1; i < expectedModernTokens.length; i += 2) {
expectedModernTokens[i] |= tokenLanguageMetadata;
}
assert.deepStrictEqual(modernTokens, expectedModernTokens);
disposables.dispose();
}
test('tokens always start at index 0 (no offset delta)', () => {
test(' (no offset delta)', () => {
testBadTokensProvider(
[
{ startIndex: 7, scopes: 'foo' },
@ -130,12 +150,12 @@ suite('TokenizationSupport2Adapter', () => {
],
0,
[
new Token(0, 'foo', languageIdentifier.language),
new Token(0, 'bar', languageIdentifier.language),
new Token(0, 'foo', languageId),
new Token(0, 'bar', languageId),
],
[
0, tokenMetadata | (0 << MetadataConsts.FOREGROUND_OFFSET),
0, tokenMetadata | (1 << MetadataConsts.FOREGROUND_OFFSET)
0, (0 << MetadataConsts.FOREGROUND_OFFSET),
0, (1 << MetadataConsts.FOREGROUND_OFFSET)
]
);
});
@ -149,14 +169,14 @@ suite('TokenizationSupport2Adapter', () => {
],
0,
[
new Token(0, 'foo', languageIdentifier.language),
new Token(5, 'bar', languageIdentifier.language),
new Token(5, 'foo', languageIdentifier.language),
new Token(0, 'foo', languageId),
new Token(5, 'bar', languageId),
new Token(5, 'foo', languageId),
],
[
0, tokenMetadata | (0 << MetadataConsts.FOREGROUND_OFFSET),
5, tokenMetadata | (1 << MetadataConsts.FOREGROUND_OFFSET),
5, tokenMetadata | (2 << MetadataConsts.FOREGROUND_OFFSET)
0, (0 << MetadataConsts.FOREGROUND_OFFSET),
5, (1 << MetadataConsts.FOREGROUND_OFFSET),
5, (2 << MetadataConsts.FOREGROUND_OFFSET)
]
);
});
@ -169,12 +189,12 @@ suite('TokenizationSupport2Adapter', () => {
],
7,
[
new Token(7, 'foo', languageIdentifier.language),
new Token(7, 'bar', languageIdentifier.language),
new Token(7, 'foo', languageId),
new Token(7, 'bar', languageId),
],
[
7, tokenMetadata | (0 << MetadataConsts.FOREGROUND_OFFSET),
7, tokenMetadata | (1 << MetadataConsts.FOREGROUND_OFFSET)
7, (0 << MetadataConsts.FOREGROUND_OFFSET),
7, (1 << MetadataConsts.FOREGROUND_OFFSET)
]
);
});
@ -188,14 +208,14 @@ suite('TokenizationSupport2Adapter', () => {
],
7,
[
new Token(7, 'foo', languageIdentifier.language),
new Token(12, 'bar', languageIdentifier.language),
new Token(12, 'foo', languageIdentifier.language),
new Token(7, 'foo', languageId),
new Token(12, 'bar', languageId),
new Token(12, 'foo', languageId),
],
[
7, tokenMetadata | (0 << MetadataConsts.FOREGROUND_OFFSET),
12, tokenMetadata | (1 << MetadataConsts.FOREGROUND_OFFSET),
12, tokenMetadata | (2 << MetadataConsts.FOREGROUND_OFFSET)
7, (0 << MetadataConsts.FOREGROUND_OFFSET),
12, (1 << MetadataConsts.FOREGROUND_OFFSET),
12, (2 << MetadataConsts.FOREGROUND_OFFSET)
]
);
});

5
src/vs/editor/standalone/test/monarch/monarch.test.ts
View file

@ -106,6 +106,7 @@ suite('Monarch', () => {
]);
innerModeTokenizationRegistration.dispose();
innerModeRegistration.dispose();
modeService.dispose();
});
test('microsoft/monaco-editor#1235: Empty Line Handling', () => {
@ -161,6 +162,7 @@ suite('Monarch', () => {
[],
[new Token(0, 'source.test', 'test')]
]);
modeService.dispose();
});
@ -209,6 +211,7 @@ suite('Monarch', () => {
new Token(18, 'number.test', 'test'),
]
]);
modeService.dispose();
});
test('issue #115662: monarchCompile function need an extra option which can control replacement', () => {
@ -262,6 +265,7 @@ suite('Monarch', () => {
new Token(0, 'ham.test', 'test'),
]
]);
modeService.dispose();
});
test('microsoft/monaco-editor#2424: Allow to target @@', () => {
@ -289,6 +293,7 @@ suite('Monarch', () => {
new Token(0, 'ham.test', 'test'),
]
]);
modeService.dispose();
});
});

21
src/vs/editor/test/browser/commands/shiftCommand.test.ts
View file

@ -8,7 +8,6 @@ import { ShiftCommand } from 'vs/editor/common/commands/shiftCommand';
import { Range } from 'vs/editor/common/core/range';
import { Selection } from 'vs/editor/common/core/selection';
import { IIdentifiedSingleEditOperation } from 'vs/editor/common/model';
import { LanguageIdentifier } from 'vs/editor/common/modes';
import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry';
import { getEditOperation, testCommand } from 'vs/editor/test/browser/testCommand';
import { withEditorModel } from 'vs/editor/test/common/editorTestUtils';
@ -29,11 +28,11 @@ export function createSingleEditOp(text: string, positionLineNumber: number, pos
class DocBlockCommentMode extends MockMode {
private static readonly _id = new LanguageIdentifier('commentMode', 3);
private static readonly _id = 'commentMode';
constructor() {
super(DocBlockCommentMode._id);
this._register(LanguageConfigurationRegistry.register(this.getLanguageIdentifier(), {
this._register(LanguageConfigurationRegistry.register(this.languageId, {
brackets: [
['(', ')'],
['{', '}'],
@ -45,8 +44,8 @@ class DocBlockCommentMode extends MockMode {
}
}
function testShiftCommand(lines: string[], languageIdentifier: LanguageIdentifier | null, useTabStops: boolean, selection: Selection, expectedLines: string[], expectedSelection: Selection): void {
testCommand(lines, languageIdentifier, selection, (sel) => new ShiftCommand(sel, {
function testShiftCommand(lines: string[], languageId: string | null, useTabStops: boolean, selection: Selection, expectedLines: string[], expectedSelection: Selection): void {
testCommand(lines, languageId, selection, (sel) => new ShiftCommand(sel, {
isUnshift: false,
tabSize: 4,
indentSize: 4,
@ -56,8 +55,8 @@ function testShiftCommand(lines: string[], languageIdentifier: LanguageIdentifie
}), expectedLines, expectedSelection);
}
function testUnshiftCommand(lines: string[], languageIdentifier: LanguageIdentifier | null, useTabStops: boolean, selection: Selection, expectedLines: string[], expectedSelection: Selection): void {
testCommand(lines, languageIdentifier, selection, (sel) => new ShiftCommand(sel, {
function testUnshiftCommand(lines: string[], languageId: string | null, useTabStops: boolean, selection: Selection, expectedLines: string[], expectedSelection: Selection): void {
testCommand(lines, languageId, selection, (sel) => new ShiftCommand(sel, {
isUnshift: true,
tabSize: 4,
indentSize: 4,
@ -565,7 +564,7 @@ suite('Editor Commands - ShiftCommand', () => {
' */',
'function hello() {}'
],
mode.getLanguageIdentifier(),
mode.languageId,
true,
new Selection(1, 1, 5, 20),
[
@ -586,7 +585,7 @@ suite('Editor Commands - ShiftCommand', () => {
' */',
'function hello() {}'
],
mode.getLanguageIdentifier(),
mode.languageId,
true,
new Selection(1, 1, 5, 20),
[
@ -607,7 +606,7 @@ suite('Editor Commands - ShiftCommand', () => {
'\t */',
'\tfunction hello() {}'
],
mode.getLanguageIdentifier(),
mode.languageId,
true,
new Selection(1, 1, 5, 21),
[
@ -635,7 +634,7 @@ suite('Editor Commands - ShiftCommand', () => {
' */',
'var foo = 0;'
],
mode.getLanguageIdentifier(),
mode.languageId,
true,
new Selection(1, 1, 7, 13),
[

294
src/vs/editor/test/browser/controller/cursor.test.ts
View file

File diff suppressed because it is too large Load diff

168
src/vs/editor/test/browser/testCodeEditor.ts
View file

@ -3,7 +3,7 @@
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import { DisposableStore, IDisposable } from 'vs/base/common/lifecycle';
import { DisposableStore, IDisposable, toDisposable } from 'vs/base/common/lifecycle';
import { ICodeEditor, IActiveCodeEditor, IEditorConstructionOptions } from 'vs/editor/browser/editorBrowser';
import { IEditorContributionCtor } from 'vs/editor/browser/editorExtensions';
import { ICodeEditorService } from 'vs/editor/browser/services/codeEditorService';
@ -11,28 +11,44 @@ import { View } from 'vs/editor/browser/view/viewImpl';
import { CodeEditorWidget, ICodeEditorWidgetOptions } from 'vs/editor/browser/widget/codeEditorWidget';
import * as editorOptions from 'vs/editor/common/config/editorOptions';
import { IConfiguration, IEditorContribution } from 'vs/editor/common/editorCommon';
import { ITextModel } from 'vs/editor/common/model';
import { TextModel } from 'vs/editor/common/model/textModel';
import { ITextBufferFactory, ITextModel } from 'vs/editor/common/model';
import { ILanguageConfigurationService } from 'vs/editor/common/modes/languageConfigurationRegistry';
import { IModelService } from 'vs/editor/common/services/modelService';
import { ModelServiceImpl } from 'vs/editor/common/services/modelServiceImpl';
import { IModeService } from 'vs/editor/common/services/modeService';
import { ModeServiceImpl } from 'vs/editor/common/services/modeServiceImpl';
import { ITextResourcePropertiesService } from 'vs/editor/common/services/textResourceConfigurationService';
import { ViewModel } from 'vs/editor/common/viewModel/viewModelImpl';
import { TestCodeEditorService, TestCommandService } from 'vs/editor/test/browser/editorTestServices';
import { createTextModel } from 'vs/editor/test/common/editorTestUtils';
import { createTextModel2 } from 'vs/editor/test/common/editorTestUtils';
import { TestConfiguration } from 'vs/editor/test/common/mocks/testConfiguration';
import { TestLanguageConfigurationService } from 'vs/editor/test/common/modes/testLanguageConfigurationService';
import { TestTextResourcePropertiesService } from 'vs/editor/test/common/services/testTextResourcePropertiesService';
import { ICommandService } from 'vs/platform/commands/common/commands';
import { IConfigurationService } from 'vs/platform/configuration/common/configuration';
import { TestConfigurationService } from 'vs/platform/configuration/test/common/testConfigurationService';
import { IContextKeyService, IContextKeyServiceTarget } from 'vs/platform/contextkey/common/contextkey';
import { BrandedService, IInstantiationService } from 'vs/platform/instantiation/common/instantiation';
import { IDialogService } from 'vs/platform/dialogs/common/dialogs';
import { TestDialogService } from 'vs/platform/dialogs/test/common/testDialogService';
import { SyncDescriptor } from 'vs/platform/instantiation/common/descriptors';
import { BrandedService, IInstantiationService, ServiceIdentifier } from 'vs/platform/instantiation/common/instantiation';
import { InstantiationService } from 'vs/platform/instantiation/common/instantiationService';
import { ServiceCollection } from 'vs/platform/instantiation/common/serviceCollection';
import { MockContextKeyService } from 'vs/platform/keybinding/test/common/mockKeybindingService';
import { ILogService, NullLogService } from 'vs/platform/log/common/log';
import { INotificationService } from 'vs/platform/notification/common/notification';
import { TestNotificationService } from 'vs/platform/notification/test/common/testNotificationService';
import { ITelemetryService } from 'vs/platform/telemetry/common/telemetry';
import { NullTelemetryService } from 'vs/platform/telemetry/common/telemetryUtils';
import { NullTelemetryServiceShape } from 'vs/platform/telemetry/common/telemetryUtils';
import { IThemeService } from 'vs/platform/theme/common/themeService';
import { TestThemeService } from 'vs/platform/theme/test/common/testThemeService';
import { IUndoRedoService } from 'vs/platform/undoRedo/common/undoRedo';
import { UndoRedoService } from 'vs/platform/undoRedo/common/undoRedoService';
export interface ITestCodeEditor extends IActiveCodeEditor {
getViewModel(): ViewModel | undefined;
registerAndInstantiateContribution<T extends IEditorContribution, Services extends BrandedService[]>(id: string, ctor: new (editor: ICodeEditor, ...services: Services) => T): T;
registerDisposable(disposable: IDisposable): void;
}
export class TestCodeEditor extends CodeEditorWidget implements ICodeEditor {
@ -63,6 +79,9 @@ export class TestCodeEditor extends CodeEditorWidget implements ICodeEditor {
this._contributions[id] = r;
return r;
}
public registerDisposable(disposable: IDisposable): void {
this._register(disposable);
}
}
class TestCodeEditorWithAutoModelDisposal extends TestCodeEditor {
@ -97,79 +116,97 @@ export interface TestCodeEditorCreationOptions extends editorOptions.IEditorOpti
hasTextFocus?: boolean;
}
export function withTestCodeEditor(text: string | string[] | null, options: TestCodeEditorCreationOptions, callback: (editor: ITestCodeEditor, viewModel: ViewModel) => void): void {
// create a model if necessary and remember it in order to dispose it.
export function withTestCodeEditor(text: string | string[] | ITextBufferFactory | null, options: TestCodeEditorCreationOptions, callback: (editor: ITestCodeEditor, viewModel: ViewModel) => void): void {
const [instantiationService, disposables] = createTestCodeEditorServices(options.serviceCollection);
delete options.serviceCollection;
// create a model if necessary
if (!options.model) {
if (typeof text === 'string') {
options.model = createTextModel(text);
} else if (text) {
options.model = createTextModel(text.join('\n'));
if (Array.isArray(text)) {
options.model = disposables.add(createTextModel2(instantiationService, text.join('\n')));
} else if (text !== null) {
options.model = disposables.add(createTextModel2(instantiationService, text));
}
}
const editor = createTestCodeEditor(options);
const editor = disposables.add(doCreateTestCodeEditor(instantiationService, options));
const viewModel = editor.getViewModel()!;
viewModel.setHasFocus(true);
callback(<ITestCodeEditor>editor, editor.getViewModel()!);
editor.dispose();
disposables.dispose();
}
export async function withAsyncTestCodeEditor(text: string | string[] | null, options: TestCodeEditorCreationOptions, callback: (editor: ITestCodeEditor, viewModel: ViewModel, instantiationService: IInstantiationService) => Promise<void>): Promise<void> {
// create a model if necessary and remember it in order to dispose it.
let model: TextModel | undefined;
export async function withAsyncTestCodeEditor(text: string | string[] | ITextBufferFactory | null, options: TestCodeEditorCreationOptions, callback: (editor: ITestCodeEditor, viewModel: ViewModel, instantiationService: IInstantiationService) => Promise<void>): Promise<void> {
const [instantiationService, disposables] = createTestCodeEditorServices(options.serviceCollection);
delete options.serviceCollection;
// create a model if necessary
if (!options.model) {
if (typeof text === 'string') {
model = options.model = createTextModel(text);
} else if (text) {
model = options.model = createTextModel(text.join('\n'));
if (Array.isArray(text)) {
options.model = disposables.add(createTextModel2(instantiationService, text.join('\n')));
} else if (text !== null) {
options.model = disposables.add(createTextModel2(instantiationService, text));
}
}
const [instantiationService, editor, disposable] = doCreateTestCodeEditor(options);
const editor = disposables.add(doCreateTestCodeEditor(instantiationService, options));
const viewModel = editor.getViewModel()!;
viewModel.setHasFocus(true);
await callback(<ITestCodeEditor>editor, editor.getViewModel()!, instantiationService);
editor.dispose();
model?.dispose();
disposable.dispose();
disposables.dispose();
}
export function createTestCodeEditor(options: TestCodeEditorCreationOptions): ITestCodeEditor {
const [, editor] = doCreateTestCodeEditor(options);
const [instantiationService, disposables] = createTestCodeEditorServices(options.serviceCollection);
delete options.serviceCollection;
const editor = doCreateTestCodeEditor(instantiationService, options);
editor.registerDisposable(disposables);
return editor;
}
function doCreateTestCodeEditor(options: TestCodeEditorCreationOptions): [IInstantiationService, ITestCodeEditor, IDisposable] {
const store = new DisposableStore();
export function createTestCodeEditorServices(services: ServiceCollection = new ServiceCollection()): [IInstantiationService, DisposableStore] {
const serviceIdentifiers: ServiceIdentifier<any>[] = [];
const define = <T>(id: ServiceIdentifier<T>, ctor: new (...args: any[]) => T) => {
if (!services.has(id)) {
services.set(id, new SyncDescriptor(ctor));
}
serviceIdentifiers.push(id);
};
const model = options.model;
delete options.model;
const services: ServiceCollection = options.serviceCollection || new ServiceCollection();
delete options.serviceCollection;
define(INotificationService, TestNotificationService);
define(IDialogService, TestDialogService);
define(IUndoRedoService, UndoRedoService);
define(IModeService, ModeServiceImpl);
define(ILanguageConfigurationService, TestLanguageConfigurationService);
define(IConfigurationService, TestConfigurationService);
define(ITextResourcePropertiesService, TestTextResourcePropertiesService);
define(IThemeService, TestThemeService);
define(ILogService, NullLogService);
define(IModelService, ModelServiceImpl);
define(ICodeEditorService, TestCodeEditorService);
define(IContextKeyService, MockContextKeyService);
define(ICommandService, TestCommandService);
define(ITelemetryService, NullTelemetryServiceShape);
const instantiationService: IInstantiationService = new InstantiationService(services);
const disposables = new DisposableStore();
disposables.add(toDisposable(() => {
for (const id of serviceIdentifiers) {
const instanceOrDescriptor = services.get(id);
if (typeof instanceOrDescriptor.dispose === 'function') {
instanceOrDescriptor.dispose();
}
}
}));
return [instantiationService, disposables];
}
if (!services.has(ICodeEditorService)) {
services.set(ICodeEditorService, store.add(new TestCodeEditorService()));
}
if (!services.has(IContextKeyService)) {
services.set(IContextKeyService, store.add(new MockContextKeyService()));
}
if (!services.has(INotificationService)) {
services.set(INotificationService, new TestNotificationService());
}
if (!services.has(ICommandService)) {
services.set(ICommandService, new TestCommandService(instantiationService));
}
if (!services.has(IThemeService)) {
services.set(IThemeService, new TestThemeService());
}
if (!services.has(ITelemetryService)) {
services.set(ITelemetryService, NullTelemetryService);
}
function doCreateTestCodeEditor(instantiationService: IInstantiationService, options: TestCodeEditorCreationOptions): ITestCodeEditor {
const model = options.model;
delete options.model;
const codeEditorWidgetOptions: ICodeEditorWidgetOptions = {
contributions: []
@ -185,5 +222,32 @@ function doCreateTestCodeEditor(options: TestCodeEditorCreationOptions): [IInsta
}
editor.setHasTextFocus(options.hasTextFocus);
editor.setModel(model);
return [instantiationService, <ITestCodeEditor>editor, store];
return <ITestCodeEditor>editor;
}
export interface TestCodeEditorCreationOptions2 extends editorOptions.IEditorOptions {
/**
* If the editor has text focus.
* Defaults to true.
*/
hasTextFocus?: boolean;
}
export function createTestCodeEditor2(instantiationService: IInstantiationService, model: ITextModel, options: TestCodeEditorCreationOptions2): TestCodeEditor {
const codeEditorWidgetOptions: ICodeEditorWidgetOptions = {
contributions: []
};
const editor = instantiationService.createInstance(
TestCodeEditor,
<HTMLElement><any>new TestEditorDomElement(),
options,
codeEditorWidgetOptions
);
if (typeof options.hasTextFocus === 'undefined') {
options.hasTextFocus = true;
}
editor.setHasTextFocus(options.hasTextFocus);
editor.setModel(model);
editor.getViewModel()!.setHasFocus(true);
return editor;
}

43
src/vs/editor/test/browser/testCommand.ts
View file

@ -8,40 +8,43 @@ import { IRange } from 'vs/editor/common/core/range';
import { Selection, ISelection } from 'vs/editor/common/core/selection';
import { ICommand, IEditOperationBuilder } from 'vs/editor/common/editorCommon';
import { IIdentifiedSingleEditOperation, ITextModel } from 'vs/editor/common/model';
import { createTextModel } from 'vs/editor/test/common/editorTestUtils';
import { LanguageIdentifier } from 'vs/editor/common/modes';
import { withTestCodeEditor } from 'vs/editor/test/browser/testCodeEditor';
import { createTestCodeEditor2, createTestCodeEditorServices } from 'vs/editor/test/browser/testCodeEditor';
import { TextModel } from 'vs/editor/common/model/textModel';
import { ServicesAccessor } from 'vs/platform/instantiation/common/instantiation';
import { DisposableStore } from 'vs/base/common/lifecycle';
export function testCommand(
lines: string[],
languageIdentifier: LanguageIdentifier | null,
languageId: string | null,
selection: Selection,
commandFactory: (selection: Selection) => ICommand,
expectedLines: string[],
expectedSelection: Selection,
forceTokenization?: boolean
forceTokenization?: boolean,
prepare?: (accessor: ServicesAccessor, disposables: DisposableStore) => void
): void {
let model = createTextModel(lines.join('\n'), undefined, languageIdentifier);
withTestCodeEditor('', { model: model }, (_editor, cursor) => {
if (!cursor) {
return;
}
const [instantiationService, disposables] = createTestCodeEditorServices();
if (prepare) {
instantiationService.invokeFunction(prepare, disposables);
}
const model = disposables.add(instantiationService.createInstance(TextModel, lines.join('\n'), TextModel.DEFAULT_CREATION_OPTIONS, languageId, null));
const editor = disposables.add(createTestCodeEditor2(instantiationService, model, {}));
const viewModel = editor.getViewModel()!;
if (forceTokenization) {
model.forceTokenization(model.getLineCount());
}
if (forceTokenization) {
model.forceTokenization(model.getLineCount());
}
cursor.setSelections('tests', [selection]);
viewModel.setSelections('tests', [selection]);
cursor.executeCommand(commandFactory(cursor.getSelection()), 'tests');
viewModel.executeCommand(commandFactory(viewModel.getSelection()), 'tests');
assert.deepStrictEqual(model.getLinesContent(), expectedLines);
assert.deepStrictEqual(model.getLinesContent(), expectedLines);
let actualSelection = cursor.getSelection();
assert.deepStrictEqual(actualSelection.toString(), expectedSelection.toString());
const actualSelection = viewModel.getSelection();
assert.deepStrictEqual(actualSelection.toString(), expectedSelection.toString());
});
model.dispose();
disposables.dispose();
}
/**

7
src/vs/editor/test/common/commentMode.ts
View file

@ -3,17 +3,16 @@
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import { LanguageIdentifier } from 'vs/editor/common/modes';
import { CommentRule } from 'vs/editor/common/modes/languageConfiguration';
import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry';
import { MockMode } from 'vs/editor/test/common/mocks/mockMode';
export class CommentMode extends MockMode {
private static readonly _id = new LanguageIdentifier('commentMode', 3);
public static readonly id = 'commentMode';
constructor(commentsConfig: CommentRule) {
super(CommentMode._id);
this._register(LanguageConfigurationRegistry.register(this.getLanguageIdentifier(), {
super(CommentMode.id);
this._register(LanguageConfigurationRegistry.register(this.languageId, {
comments: commentsConfig
}));
}

3
src/vs/editor/test/common/core/lineTokens.test.ts
View file

@ -6,6 +6,7 @@
import * as assert from 'assert';
import { IViewLineTokens, LineTokens } from 'vs/editor/common/core/lineTokens';
import { MetadataConsts } from 'vs/editor/common/modes';
import { LanguageIdCodec } from 'vs/editor/common/services/languagesRegistry';
suite('LineTokens', () => {
@ -24,7 +25,7 @@ suite('LineTokens', () => {
) >>> 0;
}
return new LineTokens(binTokens, text);
return new LineTokens(binTokens, text, new LanguageIdCodec());
}
function createTestLineTokens(): LineTokens {

82
src/vs/editor/test/common/editorTestUtils.ts
View file

@ -3,14 +3,39 @@
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import { DisposableStore, IDisposable, toDisposable } from 'vs/base/common/lifecycle';
import { URI } from 'vs/base/common/uri';
import { BracketPairColorizationOptions, DefaultEndOfLine, ITextModelCreationOptions } from 'vs/editor/common/model';
import { BracketPairColorizationOptions, DefaultEndOfLine, ITextBufferFactory, ITextModelCreationOptions } from 'vs/editor/common/model';
import { TextModel } from 'vs/editor/common/model/textModel';
import { LanguageIdentifier } from 'vs/editor/common/modes';
import { ILanguageConfigurationService } from 'vs/editor/common/modes/languageConfigurationRegistry';
import { IModeService } from 'vs/editor/common/services/modeService';
import { ModeServiceImpl } from 'vs/editor/common/services/modeServiceImpl';
import { ITextResourcePropertiesService } from 'vs/editor/common/services/textResourceConfigurationService';
import { TestLanguageConfigurationService } from 'vs/editor/test/common/modes/testLanguageConfigurationService';
import { IConfigurationService } from 'vs/platform/configuration/common/configuration';
import { TestConfigurationService } from 'vs/platform/configuration/test/common/testConfigurationService';
import { IDialogService } from 'vs/platform/dialogs/common/dialogs';
import { TestDialogService } from 'vs/platform/dialogs/test/common/testDialogService';
import { SyncDescriptor } from 'vs/platform/instantiation/common/descriptors';
import { IInstantiationService, ServiceIdentifier } from 'vs/platform/instantiation/common/instantiation';
import { InstantiationService } from 'vs/platform/instantiation/common/instantiationService';
import { ServiceCollection } from 'vs/platform/instantiation/common/serviceCollection';
import { ILogService, NullLogService } from 'vs/platform/log/common/log';
import { INotificationService } from 'vs/platform/notification/common/notification';
import { TestNotificationService } from 'vs/platform/notification/test/common/testNotificationService';
import { IThemeService } from 'vs/platform/theme/common/themeService';
import { TestThemeService } from 'vs/platform/theme/test/common/testThemeService';
import { IUndoRedoService } from 'vs/platform/undoRedo/common/undoRedo';
import { UndoRedoService } from 'vs/platform/undoRedo/common/undoRedoService';
import { TestTextResourcePropertiesService } from 'vs/editor/test/common/services/testTextResourcePropertiesService';
import { IModelService } from 'vs/editor/common/services/modelService';
import { ModelServiceImpl } from 'vs/editor/common/services/modelServiceImpl';
class TestTextModel extends TextModel {
public registerDisposable(disposable: IDisposable): void {
this._register(disposable);
}
}
export function withEditorModel(text: string[], callback: (model: TextModel) => void): void {
let model = createTextModel(text.join('\n'));
@ -30,8 +55,8 @@ export interface IRelaxedTextModelCreationOptions {
bracketColorizationOptions?: BracketPairColorizationOptions;
}
export function createTextModel(text: string, _options: IRelaxedTextModelCreationOptions = TextModel.DEFAULT_CREATION_OPTIONS, languageIdentifier: LanguageIdentifier | null = null, uri: URI | null = null): TextModel {
const options: ITextModelCreationOptions = {
function resolveOptions(_options: IRelaxedTextModelCreationOptions): ITextModelCreationOptions {
return {
tabSize: (typeof _options.tabSize === 'undefined' ? TextModel.DEFAULT_CREATION_OPTIONS.tabSize : _options.tabSize),
indentSize: (typeof _options.indentSize === 'undefined' ? TextModel.DEFAULT_CREATION_OPTIONS.indentSize : _options.indentSize),
insertSpaces: (typeof _options.insertSpaces === 'undefined' ? TextModel.DEFAULT_CREATION_OPTIONS.insertSpaces : _options.insertSpaces),
@ -42,8 +67,49 @@ export function createTextModel(text: string, _options: IRelaxedTextModelCreatio
largeFileOptimizations: (typeof _options.largeFileOptimizations === 'undefined' ? TextModel.DEFAULT_CREATION_OPTIONS.largeFileOptimizations : _options.largeFileOptimizations),
bracketPairColorizationOptions: (typeof _options.bracketColorizationOptions === 'undefined' ? TextModel.DEFAULT_CREATION_OPTIONS.bracketPairColorizationOptions : _options.bracketColorizationOptions),
};
const dialogService = new TestDialogService();
const notificationService = new TestNotificationService();
const undoRedoService = new UndoRedoService(dialogService, notificationService);
return new TextModel(text, options, languageIdentifier, uri, undoRedoService, new TestLanguageConfigurationService());
}
export function createTextModel(text: string | ITextBufferFactory, _options: IRelaxedTextModelCreationOptions = TextModel.DEFAULT_CREATION_OPTIONS, languageId: string | null = null, uri: URI | null = null): TextModel {
const [instantiationService, disposables] = createModelServices();
const model = createTextModel2(instantiationService, text, _options, languageId, uri);
model.registerDisposable(disposables);
return model;
}
export function createTextModel2(instantiationService: IInstantiationService, text: string | ITextBufferFactory, _options: IRelaxedTextModelCreationOptions = TextModel.DEFAULT_CREATION_OPTIONS, languageId: string | null = null, uri: URI | null = null): TestTextModel {
const options = resolveOptions(_options);
return instantiationService.createInstance(TestTextModel, text, options, languageId, uri);
}
export function createModelServices(services: ServiceCollection = new ServiceCollection()): [IInstantiationService, DisposableStore] {
const serviceIdentifiers: ServiceIdentifier<any>[] = [];
const define = <T>(id: ServiceIdentifier<T>, ctor: new (...args: any[]) => T) => {
if (!services.has(id)) {
services.set(id, new SyncDescriptor(ctor));
}
serviceIdentifiers.push(id);
};
define(INotificationService, TestNotificationService);
define(IDialogService, TestDialogService);
define(IUndoRedoService, UndoRedoService);
define(IModeService, ModeServiceImpl);
define(ILanguageConfigurationService, TestLanguageConfigurationService);
define(IConfigurationService, TestConfigurationService);
define(ITextResourcePropertiesService, TestTextResourcePropertiesService);
define(IThemeService, TestThemeService);
define(ILogService, NullLogService);
define(IModelService, ModelServiceImpl);
const instantiationService: IInstantiationService = new InstantiationService(services);
const disposables = new DisposableStore();
disposables.add(toDisposable(() => {
for (const id of serviceIdentifiers) {
const instanceOrDescriptor = services.get(id);
if (typeof instanceOrDescriptor.dispose === 'function') {
instanceOrDescriptor.dispose();
}
}
}));
return [instantiationService, disposables];
}

22
src/vs/editor/test/common/mocks/mockMode.ts
View file

@ -5,27 +5,19 @@
import { Event } from 'vs/base/common/event';
import { Disposable } from 'vs/base/common/lifecycle';
import { LanguageIdentifier } from 'vs/editor/common/modes';
import { ModesRegistry } from 'vs/editor/common/modes/modesRegistry';
import { ILanguageSelection } from 'vs/editor/common/services/modeService';
export class MockMode extends Disposable {
private readonly _languageIdentifier: LanguageIdentifier;
constructor(languageIdentifier: LanguageIdentifier) {
constructor(
public readonly languageId: string
) {
super();
this._languageIdentifier = languageIdentifier;
}
public getId(): string {
return this._languageIdentifier.language;
}
public getLanguageIdentifier(): LanguageIdentifier {
return this._languageIdentifier;
this._register(ModesRegistry.registerLanguage({ id: languageId }));
}
}
export class StaticLanguageSelector implements ILanguageSelection {
readonly onDidChange: Event<LanguageIdentifier> = Event.None;
constructor(public readonly languageIdentifier: LanguageIdentifier) { }
readonly onDidChange: Event<string> = Event.None;
constructor(public readonly languageId: string) { }
}

8
src/vs/editor/test/common/model/bracketPairColorizer/brackets.test.ts
View file

@ -8,14 +8,12 @@ import { DisposableStore } from 'vs/base/common/lifecycle';
import { LanguageAgnosticBracketTokens } from 'vs/editor/common/model/bracketPairColorizer/brackets';
import { SmallImmutableSet, DenseKeyProvider } from 'vs/editor/common/model/bracketPairColorizer/smallImmutableSet';
import { Token, TokenKind } from 'vs/editor/common/model/bracketPairColorizer/tokenizer';
import { LanguageIdentifier } from 'vs/editor/common/modes';
import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry';
import { TestLanguageConfigurationService } from 'vs/editor/test/common/modes/testLanguageConfigurationService';
suite('Bracket Pair Colorizer - Brackets', () => {
test('Basic', () => {
const languageId = 3;
const mode1 = new LanguageIdentifier('testMode1', languageId);
const languageId = 'testMode1';
const denseKeyProvider = new DenseKeyProvider<string>();
const getImmutableSet = (elements: string[]) => {
let newSet = SmallImmutableSet.getEmpty();
@ -27,7 +25,7 @@ suite('Bracket Pair Colorizer - Brackets', () => {
};
const disposableStore = new DisposableStore();
disposableStore.add(LanguageConfigurationRegistry.register(mode1, {
disposableStore.add(LanguageConfigurationRegistry.register(languageId, {
brackets: [
['{', '}'], ['[', ']'], ['(', ')'],
['begin', 'end'], ['case', 'endcase'], ['casez', 'endcase'], // Verilog
@ -58,7 +56,7 @@ suite('Bracket Pair Colorizer - Brackets', () => {
{ text: '\\right.', length: 7, kind: 'ClosingBracket', bracketId: getKey('\\left('), bracketIds: getImmutableSet(['\\left(', '\\left[']) },
{ text: '\\right]', length: 7, kind: 'ClosingBracket', bracketId: getKey('\\left['), bracketIds: getImmutableSet(['\\left[', '\\left.']) }
];
const bracketsActual = bracketsExpected.map(x => tokenToObject(brackets.getToken(x.text, 3), x.text));
const bracketsActual = bracketsExpected.map(x => tokenToObject(brackets.getToken(x.text, languageId), x.text));
assert.deepStrictEqual(bracketsActual, bracketsExpected);

5
src/vs/editor/test/common/model/bracketPairColorizer/getBracketPairsInRange.test.ts
View file

@ -8,7 +8,6 @@ import { Disposable, disposeOnReturn } from 'vs/base/common/lifecycle';
import { Position } from 'vs/editor/common/core/position';
import { Range } from 'vs/editor/common/core/range';
import { BracketPair } from 'vs/editor/common/model';
import { LanguageIdentifier } from 'vs/editor/common/modes';
import { LanguageConfiguration } from 'vs/editor/common/modes/languageConfiguration';
import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry';
import { createTextModel } from 'vs/editor/test/common/editorTestUtils';
@ -238,13 +237,13 @@ class MockLanguage extends Disposable {
private static id = 0;
public static create(options: MockLanguageOptions) {
const id = new LanguageIdentifier(`lang${this.id++}`, this.id++);
const id = `lang${this.id++}`;
return new MockLanguage(id, options);
}
constructor(
public readonly id: LanguageIdentifier,
public readonly id: string,
options: MockLanguageOptions
) {
super();

43
src/vs/editor/test/common/model/bracketPairColorizer/tokenizer.test.ts
View file

@ -4,39 +4,42 @@
*--------------------------------------------------------------------------------------------*/
import assert = require('assert');
import { DisposableStore } from 'vs/base/common/lifecycle';
import { TokenizationResult2 } from 'vs/editor/common/core/token';
import { LanguageAgnosticBracketTokens } from 'vs/editor/common/model/bracketPairColorizer/brackets';
import { Length, lengthAdd, lengthsToRange, lengthZero } from 'vs/editor/common/model/bracketPairColorizer/length';
import { DenseKeyProvider } from 'vs/editor/common/model/bracketPairColorizer/smallImmutableSet';
import { TextBufferTokenizer, Token, Tokenizer, TokenKind } from 'vs/editor/common/model/bracketPairColorizer/tokenizer';
import { TextModel } from 'vs/editor/common/model/textModel';
import { IState, ITokenizationSupport, LanguageId, LanguageIdentifier, MetadataConsts, StandardTokenType, TokenizationRegistry } from 'vs/editor/common/modes';
import { IState, ITokenizationSupport, LanguageId, MetadataConsts, StandardTokenType, TokenizationRegistry } from 'vs/editor/common/modes';
import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry';
import { createTextModel } from 'vs/editor/test/common/editorTestUtils';
import { ModesRegistry } from 'vs/editor/common/modes/modesRegistry';
import { IModeService } from 'vs/editor/common/services/modeService';
import { createModelServices, createTextModel2 } from 'vs/editor/test/common/editorTestUtils';
import { TestLanguageConfigurationService } from 'vs/editor/test/common/modes/testLanguageConfigurationService';
suite('Bracket Pair Colorizer - Tokenizer', () => {
test('Basic', () => {
const languageId = 2;
const mode1 = new LanguageIdentifier('testMode1', languageId);
const mode1 = 'testMode1';
const [instantiationService, disposableStore] = createModelServices();
const modeService = instantiationService.invokeFunction((accessor) => accessor.get(IModeService));
disposableStore.add(ModesRegistry.registerLanguage({ id: mode1 }));
const encodedMode1 = modeService.languageIdCodec.encodeLanguageId(mode1);
const denseKeyProvider = new DenseKeyProvider<string>();
const tStandard = (text: string) => new TokenInfo(text, mode1.id, StandardTokenType.Other);
const tComment = (text: string) => new TokenInfo(text, mode1.id, StandardTokenType.Comment);
const tStandard = (text: string) => new TokenInfo(text, encodedMode1, StandardTokenType.Other);
const tComment = (text: string) => new TokenInfo(text, encodedMode1, StandardTokenType.Comment);
const document = new TokenizedDocument([
tStandard(' { } '), tStandard('be'), tStandard('gin end'), tStandard('\n'),
tStandard('hello'), tComment('{'), tStandard('}'),
]);
const disposableStore = new DisposableStore();
disposableStore.add(TokenizationRegistry.register(mode1.language, document.getTokenizationSupport()));
disposableStore.add(TokenizationRegistry.register(mode1, document.getTokenizationSupport()));
disposableStore.add(LanguageConfigurationRegistry.register(mode1, {
brackets: [['{', '}'], ['[', ']'], ['(', ')'], ['begin', 'end']],
}));
const model = createTextModel(document.getText(), {}, mode1);
const model = disposableStore.add(createTextModel2(instantiationService, document.getText(), {}, mode1));
model.forceTokenization(model.getLineCount());
const languageConfigService = new TestLanguageConfigurationService();
@ -48,36 +51,36 @@ suite('Bracket Pair Colorizer - Tokenizer', () => {
{ text: ' ', bracketId: null, bracketIds: [], kind: 'Text' },
{
text: '{',
bracketId: '2:::{',
bracketIds: ['2:::{'],
bracketId: 'testMode1:::{',
bracketIds: ['testMode1:::{'],
kind: 'OpeningBracket',
},
{ text: ' ', bracketId: null, bracketIds: [], kind: 'Text' },
{
text: '}',
bracketId: '2:::{',
bracketIds: ['2:::{'],
bracketId: 'testMode1:::{',
bracketIds: ['testMode1:::{'],
kind: 'ClosingBracket',
},
{ text: ' ', bracketId: null, bracketIds: [], kind: 'Text' },
{
text: 'begin',
bracketId: '2:::begin',
bracketIds: ['2:::begin'],
bracketId: 'testMode1:::begin',
bracketIds: ['testMode1:::begin'],
kind: 'OpeningBracket',
},
{ text: ' ', bracketId: null, bracketIds: [], kind: 'Text' },
{
text: 'end',
bracketId: '2:::begin',
bracketIds: ['2:::begin'],
bracketId: 'testMode1:::begin',
bracketIds: ['testMode1:::begin'],
kind: 'ClosingBracket',
},
{ text: '\nhello{', bracketId: null, bracketIds: [], kind: 'Text' },
{
text: '}',
bracketId: '2:::{',
bracketIds: ['2:::{'],
bracketId: 'testMode1:::{',
bracketIds: ['testMode1:::{'],
kind: 'ClosingBracket',
},
]);

10
src/vs/editor/test/common/model/model.line.test.ts
View file

@ -7,7 +7,7 @@ import * as assert from 'assert';
import { LineTokens } from 'vs/editor/common/core/lineTokens';
import { Range } from 'vs/editor/common/core/range';
import { TextModel } from 'vs/editor/common/model/textModel';
import { LanguageIdentifier, MetadataConsts } from 'vs/editor/common/modes';
import { MetadataConsts } from 'vs/editor/common/modes';
import { ViewLineToken, ViewLineTokenFactory } from 'vs/editor/test/common/core/viewLineToken';
import { createTextModel } from 'vs/editor/test/common/editorTestUtils';
@ -107,7 +107,7 @@ suite('ModelLinesTokens', () => {
function testApplyEdits(initial: IBufferLineState[], edits: IEdit[], expected: IBufferLineState[]): void {
const initialText = initial.map(el => el.text).join('\n');
const model = createTextModel(initialText, TextModel.DEFAULT_CREATION_OPTIONS, new LanguageIdentifier('test', 0));
const model = createTextModel(initialText, TextModel.DEFAULT_CREATION_OPTIONS, 'test');
for (let lineIndex = 0; lineIndex < initial.length; lineIndex++) {
const lineTokens = initial[lineIndex].tokens;
const lineTextLength = model.getLineMaxColumn(lineIndex + 1) - 1;
@ -129,6 +129,8 @@ suite('ModelLinesTokens', () => {
assert.strictEqual(actualLine, expected[lineIndex].text);
assertLineTokens(actualTokens, expected[lineIndex].tokens);
}
model.dispose();
}
test('single delete 1', () => {
@ -443,7 +445,7 @@ suite('ModelLinesTokens', () => {
}
test('insertion on empty line', () => {
const model = createTextModel('some text', TextModel.DEFAULT_CREATION_OPTIONS, new LanguageIdentifier('test', 0));
const model = createTextModel('some text', TextModel.DEFAULT_CREATION_OPTIONS, 'test');
const tokens = TestToken.toTokens([new TestToken(0, 1)]);
LineTokens.convertToEndOffset(tokens, model.getLineMaxColumn(1) - 1);
model.setLineTokens(1, tokens);
@ -462,6 +464,8 @@ suite('ModelLinesTokens', () => {
const actualTokens = model.getLineTokens(1);
assertLineTokens(actualTokens, [new TestToken(0, 1)]);
model.dispose();
});
test('updates tokens on insertion 1', () => {

4
src/vs/editor/test/common/model/model.modes.test.ts
View file

@ -47,7 +47,7 @@ suite('Editor Model - Model Modes 1', () => {
const LANGUAGE_ID = 'modelModeTest1';
calledFor = [];
languageRegistration = modes.TokenizationRegistry.register(LANGUAGE_ID, tokenizationSupport);
thisModel = createTextModel(TEXT, undefined, new modes.LanguageIdentifier(LANGUAGE_ID, 0));
thisModel = createTextModel(TEXT, undefined, LANGUAGE_ID);
});
teardown(() => {
@ -200,7 +200,7 @@ suite('Editor Model - Model Modes 2', () => {
'Line5';
const LANGUAGE_ID = 'modelModeTest2';
languageRegistration = modes.TokenizationRegistry.register(LANGUAGE_ID, tokenizationSupport);
thisModel = createTextModel(TEXT, undefined, new modes.LanguageIdentifier(LANGUAGE_ID, 0));
thisModel = createTextModel(TEXT, undefined, LANGUAGE_ID);
});
teardown(() => {

41
src/vs/editor/test/common/model/model.test.ts
View file

@ -11,11 +11,12 @@ import { Range } from 'vs/editor/common/core/range';
import { TokenizationResult2 } from 'vs/editor/common/core/token';
import { TextModel } from 'vs/editor/common/model/textModel';
import { ModelRawContentChangedEvent, ModelRawFlush, ModelRawLineChanged, ModelRawLinesDeleted, ModelRawLinesInserted } from 'vs/editor/common/model/textModelEvents';
import { IState, LanguageIdentifier, MetadataConsts, TokenizationRegistry } from 'vs/editor/common/modes';
import { IState, MetadataConsts, TokenizationRegistry } from 'vs/editor/common/modes';
import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry';
import { NULL_STATE } from 'vs/editor/common/modes/nullMode';
import { MockMode } from 'vs/editor/test/common/mocks/mockMode';
import { createTextModel } from 'vs/editor/test/common/editorTestUtils';
import { createModelServices, createTextModel, createTextModel2 } from 'vs/editor/test/common/editorTestUtils';
import { IModeService } from 'vs/editor/common/services/modeService';
// --------- utils
@ -378,25 +379,30 @@ suite('Editor Model - Model Line Separators', () => {
suite('Editor Model - Words', () => {
const OUTER_LANGUAGE_ID = new LanguageIdentifier('outerMode', 3);
const INNER_LANGUAGE_ID = new LanguageIdentifier('innerMode', 4);
const OUTER_LANGUAGE_ID = 'outerMode';
const INNER_LANGUAGE_ID = 'innerMode';
class OuterMode extends MockMode {
constructor() {
constructor(
@IModeService modeService: IModeService
) {
super(OUTER_LANGUAGE_ID);
this._register(LanguageConfigurationRegistry.register(this.getLanguageIdentifier(), {}));
const languageIdCodec = modeService.languageIdCodec;
this._register(TokenizationRegistry.register(this.getLanguageIdentifier().language, {
this._register(LanguageConfigurationRegistry.register(this.languageId, {}));
this._register(TokenizationRegistry.register(this.languageId, {
getInitialState: (): IState => NULL_STATE,
tokenize: undefined!,
tokenize2: (line: string, hasEOL: boolean, state: IState): TokenizationResult2 => {
const tokensArr: number[] = [];
let prevLanguageId: LanguageIdentifier | undefined = undefined;
let prevLanguageId: string | undefined = undefined;
for (let i = 0; i < line.length; i++) {
const languageId = (line.charAt(i) === 'x' ? INNER_LANGUAGE_ID : OUTER_LANGUAGE_ID);
const encodedLanguageId = languageIdCodec.encodeLanguageId(languageId);
if (prevLanguageId !== languageId) {
tokensArr.push(i);
tokensArr.push((languageId.id << MetadataConsts.LANGUAGEID_OFFSET));
tokensArr.push((encodedLanguageId << MetadataConsts.LANGUAGEID_OFFSET));
}
prevLanguageId = languageId;
}
@ -414,7 +420,7 @@ suite('Editor Model - Words', () => {
class InnerMode extends MockMode {
constructor() {
super(INNER_LANGUAGE_ID);
this._register(LanguageConfigurationRegistry.register(this.getLanguageIdentifier(), {}));
this._register(LanguageConfigurationRegistry.register(this.languageId, {}));
}
}
@ -448,12 +454,11 @@ suite('Editor Model - Words', () => {
});
test('getWordAtPosition at embedded language boundaries', () => {
const outerMode = new OuterMode();
const innerMode = new InnerMode();
disposables.push(outerMode, innerMode);
const [instantiationService, disposables] = createModelServices();
const outerMode = disposables.add(instantiationService.createInstance(OuterMode));
disposables.add(new InnerMode());
const model = createTextModel('ab<xx>ab<x>', undefined, outerMode.getLanguageIdentifier());
disposables.push(model);
const model = disposables.add(createTextModel2(instantiationService, 'ab<xx>ab<x>', undefined, outerMode.languageId));
assert.deepStrictEqual(model.getWordAtPosition(new Position(1, 1)), { word: 'ab', startColumn: 1, endColumn: 3 });
assert.deepStrictEqual(model.getWordAtPosition(new Position(1, 2)), { word: 'ab', startColumn: 1, endColumn: 3 });
@ -462,15 +467,17 @@ suite('Editor Model - Words', () => {
assert.deepStrictEqual(model.getWordAtPosition(new Position(1, 5)), { word: 'xx', startColumn: 4, endColumn: 6 });
assert.deepStrictEqual(model.getWordAtPosition(new Position(1, 6)), { word: 'xx', startColumn: 4, endColumn: 6 });
assert.deepStrictEqual(model.getWordAtPosition(new Position(1, 7)), { word: 'ab', startColumn: 7, endColumn: 9 });
disposables.dispose();
});
test('issue #61296: VS code freezes when editing CSS file with emoji', () => {
const MODE_ID = new LanguageIdentifier('testMode', 4);
const MODE_ID = 'testMode';
const mode = new class extends MockMode {
constructor() {
super(MODE_ID);
this._register(LanguageConfigurationRegistry.register(this.getLanguageIdentifier(), {
this._register(LanguageConfigurationRegistry.register(this.languageId, {
wordPattern: /(#?-?\d*\.\d\w*%?)|(::?[\w-]*(?=[^,{;]*[,{]))|(([@#.!])?[\w-?]+%?|[@#!.])/g
}));
}

8
src/vs/editor/test/common/model/pieceTreeTextBuffer/pieceTreeTextBuffer.test.ts
View file

@ -1787,6 +1787,8 @@ suite('snapshot', () => {
]);
assert.strictEqual(model.getLinesContent().join('\n'), getValueInSnapshot(snapshot1));
model.dispose();
});
test('immutable snapshot 1', () => {
@ -1807,6 +1809,8 @@ suite('snapshot', () => {
]);
assert.strictEqual(model.getLinesContent().join('\n'), getValueInSnapshot(snapshot));
model.dispose();
});
test('immutable snapshot 2', () => {
@ -1827,6 +1831,8 @@ suite('snapshot', () => {
]);
assert.strictEqual(model.getLinesContent().join('\n'), getValueInSnapshot(snapshot));
model.dispose();
});
test('immutable snapshot 3', () => {
@ -1846,6 +1852,8 @@ suite('snapshot', () => {
]);
assert.notStrictEqual(model.getLinesContent().join('\n'), getValueInSnapshot(snapshot));
model.dispose();
});
});

29
src/vs/editor/test/common/model/textModel.test.ts
View file

@ -175,6 +175,7 @@ suite('Editor Model - TextModel', () => {
assert.strictEqual(m.getValueLengthInRange(new Range(1, 2, 3, 1)), 'y First Line\r\nMy Second Line\r\n'.length);
assert.strictEqual(m.getValueLengthInRange(new Range(1, 2, 3, 1000)), 'y First Line\r\nMy Second Line\r\nMy Third Line'.length);
assert.strictEqual(m.getValueLengthInRange(new Range(1, 1, 1000, 1000)), 'My First Line\r\nMy Second Line\r\nMy Third Line'.length);
m.dispose();
m = createTextModel('My First Line\nMy Second Line\nMy Third Line');
assert.strictEqual(m.getValueLengthInRange(new Range(1, 1, 1, 1)), ''.length);
@ -188,6 +189,7 @@ suite('Editor Model - TextModel', () => {
assert.strictEqual(m.getValueLengthInRange(new Range(1, 2, 3, 1)), 'y First Line\nMy Second Line\n'.length);
assert.strictEqual(m.getValueLengthInRange(new Range(1, 2, 3, 1000)), 'y First Line\nMy Second Line\nMy Third Line'.length);
assert.strictEqual(m.getValueLengthInRange(new Range(1, 1, 1000, 1000)), 'My First Line\nMy Second Line\nMy Third Line'.length);
m.dispose();
});
test('guess indentation 1', () => {
@ -687,6 +689,8 @@ suite('Editor Model - TextModel', () => {
assert.deepStrictEqual(m.validatePosition(new Position(Number.MAX_VALUE, Number.MAX_VALUE)), new Position(2, 9));
assert.deepStrictEqual(m.validatePosition(new Position(123.23, 47.5)), new Position(2, 9));
m.dispose();
});
test('validatePosition around high-low surrogate pairs 1', () => {
@ -714,6 +718,8 @@ suite('Editor Model - TextModel', () => {
assert.deepStrictEqual(m.validatePosition(new Position(Number.MAX_VALUE, Number.MAX_VALUE)), new Position(1, 5));
assert.deepStrictEqual(m.validatePosition(new Position(123.23, 47.5)), new Position(1, 5));
m.dispose();
});
test('validatePosition around high-low surrogate pairs 2', () => {
@ -728,6 +734,8 @@ suite('Editor Model - TextModel', () => {
assert.deepStrictEqual(m.validatePosition(new Position(1, 6)), new Position(1, 6));
assert.deepStrictEqual(m.validatePosition(new Position(1, 7)), new Position(1, 7));
m.dispose();
});
test('validatePosition handle NaN.', () => {
@ -740,6 +748,8 @@ suite('Editor Model - TextModel', () => {
assert.deepStrictEqual(m.validatePosition(new Position(NaN, NaN)), new Position(1, 1));
assert.deepStrictEqual(m.validatePosition(new Position(2, NaN)), new Position(2, 1));
assert.deepStrictEqual(m.validatePosition(new Position(NaN, 3)), new Position(1, 3));
m.dispose();
});
test('issue #71480: validatePosition handle floats', () => {
@ -753,6 +763,8 @@ suite('Editor Model - TextModel', () => {
assert.deepStrictEqual(m.validatePosition(new Position(2, 0.8)), new Position(2, 1), 'f');
assert.deepStrictEqual(m.validatePosition(new Position(1, 1.2)), new Position(1, 1), 'g');
assert.deepStrictEqual(m.validatePosition(new Position(2, 1.5)), new Position(2, 1), 'h');
m.dispose();
});
test('issue #71480: validateRange handle floats', () => {
@ -760,6 +772,8 @@ suite('Editor Model - TextModel', () => {
assert.deepStrictEqual(m.validateRange(new Range(0.2, 1.5, 0.8, 2.5)), new Range(1, 1, 1, 1));
assert.deepStrictEqual(m.validateRange(new Range(1.2, 1.7, 1.8, 2.2)), new Range(1, 1, 1, 2));
m.dispose();
});
test('validateRange around high-low surrogate pairs 1', () => {
@ -788,6 +802,8 @@ suite('Editor Model - TextModel', () => {
assert.deepStrictEqual(m.validateRange(new Range(1, 4, 1, 5)), new Range(1, 4, 1, 5));
assert.deepStrictEqual(m.validateRange(new Range(1, 5, 1, 5)), new Range(1, 5, 1, 5));
m.dispose();
});
test('validateRange around high-low surrogate pairs 2', () => {
@ -831,6 +847,8 @@ suite('Editor Model - TextModel', () => {
assert.deepStrictEqual(m.validateRange(new Range(1, 6, 1, 7)), new Range(1, 6, 1, 7));
assert.deepStrictEqual(m.validateRange(new Range(1, 7, 1, 7)), new Range(1, 7, 1, 7));
m.dispose();
});
test('modifyPosition', () => {
@ -861,6 +879,8 @@ suite('Editor Model - TextModel', () => {
assert.deepStrictEqual(m.modifyPosition(new Position(1, 2), -100), new Position(1, 1));
assert.deepStrictEqual(m.modifyPosition(new Position(2, 2), -100), new Position(1, 1));
assert.deepStrictEqual(m.modifyPosition(new Position(2, 9), -18), new Position(1, 1));
m.dispose();
});
test('normalizeIndentation 1', () => {
@ -940,6 +960,8 @@ suite('Editor Model - TextModel', () => {
assert.strictEqual(model.getLineFirstNonWhitespaceColumn(10), 4, '10');
assert.strictEqual(model.getLineFirstNonWhitespaceColumn(11), 0, '11');
assert.strictEqual(model.getLineFirstNonWhitespaceColumn(12), 0, '12');
model.dispose();
});
test('getLineLastNonWhitespaceColumn', () => {
@ -970,12 +992,15 @@ suite('Editor Model - TextModel', () => {
assert.strictEqual(model.getLineLastNonWhitespaceColumn(10), 4, '10');
assert.strictEqual(model.getLineLastNonWhitespaceColumn(11), 0, '11');
assert.strictEqual(model.getLineLastNonWhitespaceColumn(12), 0, '12');
model.dispose();
});
test('#50471. getValueInRange with invalid range', () => {
let m = createTextModel('My First Line\r\nMy Second Line\r\nMy Third Line');
assert.strictEqual(m.getValueInRange(new Range(1, NaN, 1, 3)), 'My');
assert.strictEqual(m.getValueInRange(new Range(NaN, NaN, NaN, NaN)), '');
m.dispose();
});
});
@ -984,11 +1009,13 @@ suite('TextModel.mightContainRTL', () => {
test('nope', () => {
let model = createTextModel('hello world!');
assert.strictEqual(model.mightContainRTL(), false);
model.dispose();
});
test('yes', () => {
let model = createTextModel('Hello,\nזוהי עובדה מבוססת שדעתו');
assert.strictEqual(model.mightContainRTL(), true);
model.dispose();
});
test('setValue resets 1', () => {
@ -996,6 +1023,7 @@ suite('TextModel.mightContainRTL', () => {
assert.strictEqual(model.mightContainRTL(), false);
model.setValue('Hello,\nזוהי עובדה מבוססת שדעתו');
assert.strictEqual(model.mightContainRTL(), true);
model.dispose();
});
test('setValue resets 2', () => {
@ -1003,6 +1031,7 @@ suite('TextModel.mightContainRTL', () => {
assert.strictEqual(model.mightContainRTL(), true);
model.setValue('hello world!');
assert.strictEqual(model.mightContainRTL(), false);
model.dispose();
});
});

212
src/vs/editor/test/common/model/textModelWithTokens.test.ts
View file

@ -4,18 +4,20 @@
*--------------------------------------------------------------------------------------------*/
import * as assert from 'assert';
import { DisposableStore, IDisposable } from 'vs/base/common/lifecycle';
import { DisposableStore } from 'vs/base/common/lifecycle';
import { Position } from 'vs/editor/common/core/position';
import { Range } from 'vs/editor/common/core/range';
import { TokenizationResult2 } from 'vs/editor/common/core/token';
import { IFoundBracket } from 'vs/editor/common/model';
import { TextModel } from 'vs/editor/common/model/textModel';
import { ITokenizationSupport, LanguageId, LanguageIdentifier, MetadataConsts, TokenizationRegistry, StandardTokenType } from 'vs/editor/common/modes';
import { ITokenizationSupport, MetadataConsts, TokenizationRegistry, StandardTokenType } from 'vs/editor/common/modes';
import { CharacterPair } from 'vs/editor/common/modes/languageConfiguration';
import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry';
import { ModesRegistry } from 'vs/editor/common/modes/modesRegistry';
import { NULL_STATE } from 'vs/editor/common/modes/nullMode';
import { IModeService } from 'vs/editor/common/services/modeService';
import { ViewLineToken } from 'vs/editor/test/common/core/viewLineToken';
import { createTextModel } from 'vs/editor/test/common/editorTestUtils';
import { createModelServices, createTextModel, createTextModel2 } from 'vs/editor/test/common/editorTestUtils';
suite('TextModelWithTokens', () => {
@ -67,17 +69,19 @@ suite('TextModelWithTokens', () => {
}
}
const languageIdentifier = new LanguageIdentifier('testMode', LanguageId.PlainText);
const languageId = 'testMode';
const disposables = new DisposableStore();
let registration = LanguageConfigurationRegistry.register(languageIdentifier, {
disposables.add(ModesRegistry.registerLanguage({ id: languageId }));
disposables.add(LanguageConfigurationRegistry.register(languageId, {
brackets: brackets
});
}));
let model = createTextModel(
const model = disposables.add(createTextModel(
contents.join('\n'),
TextModel.DEFAULT_CREATION_OPTIONS,
languageIdentifier
);
languageId
));
// findPrevBracket
{
@ -131,11 +135,10 @@ suite('TextModelWithTokens', () => {
}
}
model.dispose();
registration.dispose();
disposables.dispose();
}
test('brackets', () => {
test('brackets1', () => {
testBrackets([
'if (a == 3) { return (7 * (a + 5)); }'
], [
@ -158,28 +161,30 @@ function assertIsBracket(model: TextModel, testPosition: Position, expected: [Ra
suite('TextModelWithTokens - bracket matching', () => {
const languageIdentifier = new LanguageIdentifier('bracketMode1', LanguageId.PlainText);
let registration: IDisposable;
const languageId = 'bracketMode1';
let disposables: DisposableStore;
setup(() => {
registration = LanguageConfigurationRegistry.register(languageIdentifier, {
disposables = new DisposableStore();
disposables.add(ModesRegistry.registerLanguage({ id: languageId }));
disposables.add(LanguageConfigurationRegistry.register(languageId, {
brackets: [
['{', '}'],
['[', ']'],
['(', ')'],
]
});
}));
});
teardown(() => {
registration.dispose();
disposables.dispose();
});
test('bracket matching 1', () => {
let text =
')]}{[(' + '\n' +
')]}{[(';
let model = createTextModel(text, undefined, languageIdentifier);
let model = createTextModel(text, undefined, languageId);
assertIsNotBracket(model, 1, 1);
assertIsNotBracket(model, 1, 2);
@ -207,7 +212,7 @@ suite('TextModelWithTokens - bracket matching', () => {
'}, bar: {hallo: [{' + '\n' +
'}, {' + '\n' +
'}]}}';
let model = createTextModel(text, undefined, languageIdentifier);
let model = createTextModel(text, undefined, languageId);
let brackets: [Position, Range, Range][] = [
[new Position(1, 11), new Range(1, 11, 1, 12), new Range(5, 4, 5, 5)],
@ -260,14 +265,16 @@ suite('TextModelWithTokens', () => {
test('bracket matching 3', () => {
const languageIdentifier = new LanguageIdentifier('bracketMode2', LanguageId.PlainText);
const registration = LanguageConfigurationRegistry.register(languageIdentifier, {
const languageId = 'bracketMode2';
const disposables = new DisposableStore();
disposables.add(ModesRegistry.registerLanguage({ id: languageId }));
disposables.add(LanguageConfigurationRegistry.register(languageId, {
brackets: [
['if', 'end if'],
['loop', 'end loop'],
['begin', 'end']
],
});
}));
const text = [
'begin',
@ -285,7 +292,7 @@ suite('TextModelWithTokens', () => {
'end;',
].join('\n');
const model = createTextModel(text, undefined, languageIdentifier);
const model = disposables.add(createTextModel(text, undefined, languageId));
// <if> ... <end ifa> is not matched
assertIsNotBracket(model, 10, 9);
@ -302,19 +309,20 @@ suite('TextModelWithTokens', () => {
assertIsBracket(model, new Position(1, 1), [new Range(1, 1, 1, 6), new Range(6, 1, 6, 4)]);
assertIsBracket(model, new Position(6, 1), [new Range(6, 1, 6, 4), new Range(1, 1, 1, 6)]);
model.dispose();
registration.dispose();
disposables.dispose();
});
test('bracket matching 4', () => {
const languageIdentifier = new LanguageIdentifier('bracketMode2', LanguageId.PlainText);
const registration = LanguageConfigurationRegistry.register(languageIdentifier, {
const languageId = 'bracketMode2';
const disposables = new DisposableStore();
disposables.add(ModesRegistry.registerLanguage({ id: languageId }));
disposables.add(LanguageConfigurationRegistry.register(languageId, {
brackets: [
['recordbegin', 'endrecord'],
['simplerecordbegin', 'endrecord'],
],
});
}));
const text = [
'recordbegin',
@ -323,7 +331,7 @@ suite('TextModelWithTokens', () => {
'endrecord',
].join('\n');
const model = createTextModel(text, undefined, languageIdentifier);
const model = disposables.add(createTextModel(text, undefined, languageId));
// <recordbegin> ... <endrecord> is matched
assertIsBracket(model, new Position(1, 1), [new Range(1, 1, 1, 12), new Range(4, 1, 4, 10)]);
@ -333,19 +341,27 @@ suite('TextModelWithTokens', () => {
assertIsBracket(model, new Position(2, 3), [new Range(2, 3, 2, 20), new Range(3, 3, 3, 12)]);
assertIsBracket(model, new Position(3, 3), [new Range(3, 3, 3, 12), new Range(2, 3, 2, 20)]);
model.dispose();
registration.dispose();
disposables.dispose();
});
test('issue #95843: Highlighting of closing braces is indicating wrong brace when cursor is behind opening brace', () => {
const mode1 = new LanguageIdentifier('testMode1', 3);
const mode2 = new LanguageIdentifier('testMode2', 4);
const [instantiationService, disposables] = createModelServices();
const mode1 = 'testMode1';
const mode2 = 'testMode2';
const languageIdCodec = instantiationService.invokeFunction((accessor) => accessor.get(IModeService).languageIdCodec);
disposables.add(ModesRegistry.registerLanguage({ id: mode1 }));
disposables.add(ModesRegistry.registerLanguage({ id: mode2 }));
const encodedMode1 = languageIdCodec!.encodeLanguageId(mode1);
const encodedMode2 = languageIdCodec!.encodeLanguageId(mode2);
const otherMetadata1 = (
(mode1.id << MetadataConsts.LANGUAGEID_OFFSET)
(encodedMode1 << MetadataConsts.LANGUAGEID_OFFSET)
| (StandardTokenType.Other << MetadataConsts.TOKEN_TYPE_OFFSET)
) >>> 0;
const otherMetadata2 = (
(mode2.id << MetadataConsts.LANGUAGEID_OFFSET)
(encodedMode2 << MetadataConsts.LANGUAGEID_OFFSET)
| (StandardTokenType.Other << MetadataConsts.TOKEN_TYPE_OFFSET)
) >>> 0;
@ -395,17 +411,15 @@ suite('TextModelWithTokens', () => {
}
};
const disposableStore = new DisposableStore();
disposableStore.add(TokenizationRegistry.register(mode1.language, tokenizationSupport));
disposableStore.add(LanguageConfigurationRegistry.register(mode1, {
disposables.add(TokenizationRegistry.register(mode1, tokenizationSupport));
disposables.add(LanguageConfigurationRegistry.register(mode1, {
brackets: [
['{', '}'],
['[', ']'],
['(', ')']
],
}));
disposableStore.add(LanguageConfigurationRegistry.register(mode2, {
disposables.add(LanguageConfigurationRegistry.register(mode2, {
brackets: [
['{', '}'],
['[', ']'],
@ -413,11 +427,16 @@ suite('TextModelWithTokens', () => {
],
}));
const model = disposableStore.add(createTextModel([
'function f() {',
' return <p>{true}</p>;',
'}',
].join('\n'), undefined, mode1));
const model = disposables.add(createTextModel2(
instantiationService,
[
'function f() {',
' return <p>{true}</p>;',
'}',
].join('\n'),
undefined,
mode1
));
model.forceTokenization(1);
model.forceTokenization(2);
@ -425,17 +444,23 @@ suite('TextModelWithTokens', () => {
assert.deepStrictEqual(model.matchBracket(new Position(2, 14)), [new Range(2, 13, 2, 14), new Range(2, 18, 2, 19)]);
disposableStore.dispose();
disposables.dispose();
});
test('issue #88075: TypeScript brace matching is incorrect in `${}` strings', () => {
const mode = new LanguageIdentifier('testMode', 3);
const [instantiationService, disposables] = createModelServices();
const mode = 'testMode';
const languageIdCodec = instantiationService.invokeFunction((accessor) => accessor.get(IModeService).languageIdCodec);
const encodedMode = languageIdCodec!.encodeLanguageId(mode);
const otherMetadata = (
(mode.id << MetadataConsts.LANGUAGEID_OFFSET)
(encodedMode << MetadataConsts.LANGUAGEID_OFFSET)
| (StandardTokenType.Other << MetadataConsts.TOKEN_TYPE_OFFSET)
) >>> 0;
const stringMetadata = (
(mode.id << MetadataConsts.LANGUAGEID_OFFSET)
(encodedMode << MetadataConsts.LANGUAGEID_OFFSET)
| (StandardTokenType.String << MetadataConsts.TOKEN_TYPE_OFFSET)
) >>> 0;
@ -471,20 +496,25 @@ suite('TextModelWithTokens', () => {
}
};
const registration1 = TokenizationRegistry.register(mode.language, tokenizationSupport);
const registration2 = LanguageConfigurationRegistry.register(mode, {
disposables.add(TokenizationRegistry.register(mode, tokenizationSupport));
disposables.add(LanguageConfigurationRegistry.register(mode, {
brackets: [
['{', '}'],
['[', ']'],
['(', ')']
],
});
}));
const model = createTextModel([
'function hello() {',
' console.log(`${100}`);',
'}'
].join('\n'), undefined, mode);
const model = disposables.add(createTextModel2(
instantiationService,
[
'function hello() {',
' console.log(`${100}`);',
'}'
].join('\n'),
undefined,
mode
));
model.forceTokenization(1);
model.forceTokenization(2);
@ -493,9 +523,7 @@ suite('TextModelWithTokens', () => {
assert.deepStrictEqual(model.matchBracket(new Position(2, 23)), null);
assert.deepStrictEqual(model.matchBracket(new Position(2, 20)), null);
model.dispose();
registration1.dispose();
registration2.dispose();
disposables.dispose();
});
});
@ -531,8 +559,6 @@ suite('TextModelWithTokens regression tests', () => {
let _tokenId = 10;
const LANG_ID1 = 'indicisiveMode1';
const LANG_ID2 = 'indicisiveMode2';
const languageIdentifier1 = new LanguageIdentifier(LANG_ID1, 3);
const languageIdentifier2 = new LanguageIdentifier(LANG_ID2, 4);
const tokenizationSupport: ITokenizationSupport = {
getInitialState: () => NULL_STATE,
@ -556,12 +582,12 @@ suite('TextModelWithTokens regression tests', () => {
assertViewLineTokens(model, 1, true, [createViewLineToken(12, 1)]);
assertViewLineTokens(model, 2, true, [createViewLineToken(9, 1)]);
model.setMode(languageIdentifier1);
model.setMode(LANG_ID1);
assertViewLineTokens(model, 1, true, [createViewLineToken(12, 11)]);
assertViewLineTokens(model, 2, true, [createViewLineToken(9, 12)]);
model.setMode(languageIdentifier2);
model.setMode(LANG_ID2);
assertViewLineTokens(model, 1, false, [createViewLineToken(12, 1)]);
assertViewLineTokens(model, 2, false, [createViewLineToken(9, 1)]);
@ -581,16 +607,18 @@ suite('TextModelWithTokens regression tests', () => {
test('microsoft/monaco-editor#133: Error: Cannot read property \'modeId\' of undefined', () => {
const languageIdentifier = new LanguageIdentifier('testMode', LanguageId.PlainText);
const languageId = 'testMode';
let registration = LanguageConfigurationRegistry.register(languageIdentifier, {
const disposables = new DisposableStore();
disposables.add(ModesRegistry.registerLanguage({ id: languageId }));
disposables.add(LanguageConfigurationRegistry.register(languageId, {
brackets: [
['module', 'end module'],
['sub', 'end sub']
]
});
}));
let model = createTextModel([
const model = disposables.add(createTextModel([
'Imports System',
'Imports System.Collections.Generic',
'',
@ -600,43 +628,50 @@ suite('TextModelWithTokens regression tests', () => {
'\tEnd Sub',
'',
'End Module',
].join('\n'), undefined, languageIdentifier);
].join('\n'), undefined, languageId));
let actual = model.matchBracket(new Position(4, 1));
const actual = model.matchBracket(new Position(4, 1));
assert.deepStrictEqual(actual, [new Range(4, 1, 4, 7), new Range(9, 1, 9, 11)]);
model.dispose();
registration.dispose();
disposables.dispose();
});
test('issue #11856: Bracket matching does not work as expected if the opening brace symbol is contained in the closing brace symbol', () => {
const languageIdentifier = new LanguageIdentifier('testMode', LanguageId.PlainText);
let registration = LanguageConfigurationRegistry.register(languageIdentifier, {
const languageId = 'testMode';
const disposables = new DisposableStore();
disposables.add(ModesRegistry.registerLanguage({ id: languageId }));
disposables.add(LanguageConfigurationRegistry.register(languageId, {
brackets: [
['sequence', 'endsequence'],
['feature', 'endfeature']
]
});
}));
let model = createTextModel([
const model = disposables.add(createTextModel([
'sequence "outer"',
' sequence "inner"',
' endsequence',
'endsequence',
].join('\n'), undefined, languageIdentifier);
].join('\n'), undefined, languageId));
let actual = model.matchBracket(new Position(3, 9));
const actual = model.matchBracket(new Position(3, 9));
assert.deepStrictEqual(actual, [new Range(3, 6, 3, 17), new Range(2, 6, 2, 14)]);
model.dispose();
registration.dispose();
disposables.dispose();
});
test('issue #63822: Wrong embedded language detected for empty lines', () => {
const outerMode = new LanguageIdentifier('outerMode', 3);
const innerMode = new LanguageIdentifier('innerMode', 4);
const [instantiationService, disposables] = createModelServices();
const outerMode = 'outerMode';
const innerMode = 'innerMode';
disposables.add(ModesRegistry.registerLanguage({ id: outerMode }));
disposables.add(ModesRegistry.registerLanguage({ id: innerMode }));
const languageIdCodec = instantiationService.invokeFunction((accessor) => accessor.get(IModeService).languageIdCodec);
const encodedInnerMode = languageIdCodec.encodeLanguageId(innerMode);
const tokenizationSupport: ITokenizationSupport = {
getInitialState: () => NULL_STATE,
@ -645,21 +680,20 @@ suite('TextModelWithTokens regression tests', () => {
let tokens = new Uint32Array(2);
tokens[0] = 0;
tokens[1] = (
innerMode.id << MetadataConsts.LANGUAGEID_OFFSET
encodedInnerMode << MetadataConsts.LANGUAGEID_OFFSET
) >>> 0;
return new TokenizationResult2(tokens, state);
}
};
let registration = TokenizationRegistry.register(outerMode.language, tokenizationSupport);
disposables.add(TokenizationRegistry.register(outerMode, tokenizationSupport));
let model = createTextModel('A model with one line', undefined, outerMode);
const model = disposables.add(createTextModel2(instantiationService, 'A model with one line', undefined, outerMode));
model.forceTokenization(1);
assert.strictEqual(model.getLanguageIdAtPosition(1, 1), innerMode.id);
assert.strictEqual(model.getLanguageIdAtPosition(1, 1), innerMode);
model.dispose();
registration.dispose();
disposables.dispose();
});
});

31
src/vs/editor/test/common/model/tokensStore.test.ts
View file

@ -11,6 +11,7 @@ import { IIdentifiedSingleEditOperation } from 'vs/editor/common/model';
import { MetadataConsts, TokenMetadata, FontStyle, ColorId } from 'vs/editor/common/modes';
import { createTextModel } from 'vs/editor/test/common/editorTestUtils';
import { LineTokens } from 'vs/editor/common/core/lineTokens';
import { LanguageIdCodec } from 'vs/editor/common/services/languagesRegistry';
suite('TokensStore', () => {
@ -214,7 +215,8 @@ suite('TokensStore', () => {
});
test('partial tokens 1', () => {
const store = new TokensStore2();
const codec = new LanguageIdCodec();
const store = new TokensStore2(codec);
// setPartial: [1,1 -> 31,2], [(5,5-10),(10,5-10),(15,5-10),(20,5-10),(25,5-10),(30,5-10)]
store.setPartial(new Range(1, 1, 31, 2), [
@ -251,12 +253,13 @@ suite('TokensStore', () => {
])))
]);
const lineTokens = store.addSemanticTokens(10, new LineTokens(new Uint32Array([12, 1]), `enum Enum1 {`));
const lineTokens = store.addSemanticTokens(10, new LineTokens(new Uint32Array([12, 1]), `enum Enum1 {`, codec));
assert.strictEqual(lineTokens.getCount(), 3);
});
test('partial tokens 2', () => {
const store = new TokensStore2();
const codec = new LanguageIdCodec();
const store = new TokensStore2(codec);
// setPartial: [1,1 -> 31,2], [(5,5-10),(10,5-10),(15,5-10),(20,5-10),(25,5-10),(30,5-10)]
store.setPartial(new Range(1, 1, 31, 2), [
@ -292,12 +295,13 @@ suite('TokensStore', () => {
])))
]);
const lineTokens = store.addSemanticTokens(20, new LineTokens(new Uint32Array([12, 1]), `enum Enum1 {`));
const lineTokens = store.addSemanticTokens(20, new LineTokens(new Uint32Array([12, 1]), `enum Enum1 {`, codec));
assert.strictEqual(lineTokens.getCount(), 3);
});
test('partial tokens 3', () => {
const store = new TokensStore2();
const codec = new LanguageIdCodec();
const store = new TokensStore2(codec);
// setPartial: [1,1 -> 31,2], [(5,5-10),(10,5-10),(15,5-10),(20,5-10),(25,5-10),(30,5-10)]
store.setPartial(new Range(1, 1, 31, 2), [
@ -319,12 +323,13 @@ suite('TokensStore', () => {
])))
]);
const lineTokens = store.addSemanticTokens(5, new LineTokens(new Uint32Array([12, 1]), `enum Enum1 {`));
const lineTokens = store.addSemanticTokens(5, new LineTokens(new Uint32Array([12, 1]), `enum Enum1 {`, codec));
assert.strictEqual(lineTokens.getCount(), 3);
});
test('issue #94133: Semantic colors stick around when using (only) range provider', () => {
const store = new TokensStore2();
const codec = new LanguageIdCodec();
const store = new TokensStore2(codec);
// setPartial: [1,1 -> 1,20] [(1,9-11)]
store.setPartial(new Range(1, 1, 1, 20), [
@ -336,7 +341,7 @@ suite('TokensStore', () => {
// setPartial: [1,1 -> 1,20], []
store.setPartial(new Range(1, 1, 1, 20), []);
const lineTokens = store.addSemanticTokens(1, new LineTokens(new Uint32Array([12, 1]), `enum Enum1 {`));
const lineTokens = store.addSemanticTokens(1, new LineTokens(new Uint32Array([12, 1]), `enum Enum1 {`, codec));
assert.strictEqual(lineTokens.getCount(), 1);
});
@ -362,7 +367,8 @@ suite('TokensStore', () => {
return new MultilineTokens2(firstLineNumber, new SparseEncodedTokens(new Uint32Array(result)));
}
const store = new TokensStore2();
const codec = new LanguageIdCodec();
const store = new TokensStore2(codec);
// setPartial [36446,1 -> 36475,115] [(36448,24-29),(36448,33-46),(36448,47-54),(36450,25-35),(36450,36-50),(36451,28-33),(36451,36-49),(36451,50-57),(36452,35-53),(36452,54-62),(36454,33-38),(36454,41-54),(36454,55-60),(36455,35-53),(36455,54-62),(36457,33-44),(36457,45-49),(36457,50-56),(36457,62-83),(36457,84-88),(36458,35-53),(36458,54-62),(36460,33-37),(36460,38-42),(36460,47-57),(36460,58-67),(36461,35-53),(36461,54-62),(36463,34-38),(36463,39-45),(36463,46-51),(36463,54-63),(36463,64-71),(36463,76-80),(36463,81-87),(36463,88-92),(36463,97-107),(36463,108-119),(36464,35-53),(36464,54-62),(36466,33-71),(36466,72-76),(36467,35-53),(36467,54-62),(36469,24-29),(36469,33-46),(36469,47-54),(36470,24-35),(36470,38-46),(36473,25-35),(36473,36-51),(36474,28-33),(36474,36-49),(36474,50-58),(36475,35-53),(36475,54-62)]
store.setPartial(
new Range(36446, 1, 36475, 115),
@ -384,7 +390,7 @@ suite('TokensStore', () => {
[createTokens('[(36442,25-35),(36442,36-50),(36443,30-39),(36443,42-46),(36443,47-53),(36443,54-58),(36443,63-73),(36443,74-84),(36443,87-91),(36443,92-98),(36443,101-105),(36443,106-112),(36443,113-119),(36444,28-37),(36444,38-42),(36444,47-57),(36444,58-75),(36444,80-95),(36444,96-105),(36445,35-53),(36445,54-62),(36448,24-29),(36448,33-46),(36448,47-54),(36450,25-35),(36450,36-50),(36451,28-33),(36451,36-49),(36451,50-57),(36452,35-53),(36452,54-62),(36454,33-38),(36454,41-54),(36454,55-60),(36455,35-53),(36455,54-62),(36457,33-44),(36457,45-49),(36457,50-56),(36457,62-83),(36457,84-88),(36458,35-53),(36458,54-62),(36460,33-37),(36460,38-42),(36460,47-57),(36460,58-67),(36461,35-53),(36461,54-62),(36463,34-38),(36463,39-45),(36463,46-51),(36463,54-63),(36463,64-71),(36463,76-80),(36463,81-87),(36463,88-92),(36463,97-107),(36463,108-119),(36464,35-53),(36464,54-62),(36466,33-71),(36466,72-76),(36467,35-53),(36467,54-62),(36469,24-29),(36469,33-46),(36469,47-54),(36470,24-35)]')]
);
const lineTokens = store.addSemanticTokens(36451, new LineTokens(new Uint32Array([60, 1]), ` if (flags & ModifierFlags.Ambient) {`));
const lineTokens = store.addSemanticTokens(36451, new LineTokens(new Uint32Array([60, 1]), ` if (flags & ModifierFlags.Ambient) {`, codec));
assert.strictEqual(lineTokens.getCount(), 7);
});
@ -408,7 +414,8 @@ suite('TokensStore', () => {
return r;
}
const store = new TokensStore2();
const codec = new LanguageIdCodec();
const store = new TokensStore2(codec);
store.set([
new MultilineTokens2(1, new SparseEncodedTokens(new Uint32Array([
@ -421,7 +428,7 @@ suite('TokensStore', () => {
14, createTMMetadata(1, FontStyle.None, 53),
17, createTMMetadata(6, FontStyle.None, 53),
18, createTMMetadata(1, FontStyle.None, 53),
]), `const hello = 123;`));
]), `const hello = 123;`, codec));
const actual = toArr(lineTokens);
assert.deepStrictEqual(actual, [

6
src/vs/editor/test/common/modes/languageConfiguration.test.ts
View file

@ -4,7 +4,7 @@
*--------------------------------------------------------------------------------------------*/
import * as assert from 'assert';
import { LanguageIdentifier, StandardTokenType } from 'vs/editor/common/modes';
import { StandardTokenType } from 'vs/editor/common/modes';
import { StandardAutoClosingPairConditional } from 'vs/editor/common/modes/languageConfiguration';
import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry';
@ -91,10 +91,10 @@ suite('StandardAutoClosingPairConditional', () => {
});
test('language configurations priorities', () => {
const id = new LanguageIdentifier('testLang1', 15);
const id = 'testLang1';
const d1 = LanguageConfigurationRegistry.register(id, { comments: { lineComment: '1' } }, 100);
const d2 = LanguageConfigurationRegistry.register(id, { comments: { lineComment: '2' } }, 10);
assert.strictEqual(LanguageConfigurationRegistry.getComments(id.id)?.lineCommentToken, '1');
assert.strictEqual(LanguageConfigurationRegistry.getComments(id)?.lineCommentToken, '1');
d1.dispose();
d2.dispose();
});

8
src/vs/editor/test/common/modes/supports/electricCharacter.test.ts
View file

@ -4,12 +4,12 @@
*--------------------------------------------------------------------------------------------*/
import * as assert from 'assert';
import { LanguageIdentifier, StandardTokenType } from 'vs/editor/common/modes';
import { StandardTokenType } from 'vs/editor/common/modes';
import { BracketElectricCharacterSupport, IElectricAction } from 'vs/editor/common/modes/supports/electricCharacter';
import { RichEditBrackets } from 'vs/editor/common/modes/supports/richEditBrackets';
import { TokenText, createFakeScopedLineTokens } from 'vs/editor/test/common/modesTestUtils';
const fakeLanguageIdentifier = new LanguageIdentifier('test', 3);
const fakeLanguageId = 'test';
suite('Editor Modes - Auto Indentation', () => {
function _testOnElectricCharacter(electricCharacterSupport: BracketElectricCharacterSupport, line: TokenText[], character: string, offset: number): IElectricAction | null {
@ -28,7 +28,7 @@ suite('Editor Modes - Auto Indentation', () => {
test('getElectricCharacters uses all sources and dedups', () => {
let sup = new BracketElectricCharacterSupport(
new RichEditBrackets(fakeLanguageIdentifier, [
new RichEditBrackets(fakeLanguageId, [
['{', '}'],
['(', ')']
])
@ -39,7 +39,7 @@ suite('Editor Modes - Auto Indentation', () => {
test('matchOpenBracket', () => {
let sup = new BracketElectricCharacterSupport(
new RichEditBrackets(fakeLanguageIdentifier, [
new RichEditBrackets(fakeLanguageId, [
['{', '}'],
['(', ')']
])

7
src/vs/editor/test/common/modes/testLanguageConfigurationService.ts
View file

@ -5,7 +5,6 @@
import { Emitter } from 'vs/base/common/event';
import { IDisposable } from 'vs/base/common/lifecycle';
import { URI } from 'vs/base/common/uri';
import { LanguageId, LanguageIdentifier } from 'vs/editor/common/modes';
import { ILanguageConfigurationService, LanguageConfigurationRegistry, LanguageConfigurationServiceChangeEvent, ResolvedLanguageConfiguration } from 'vs/editor/common/modes/languageConfigurationRegistry';
export class TestLanguageConfigurationService implements ILanguageConfigurationService {
@ -16,7 +15,7 @@ export class TestLanguageConfigurationService implements ILanguageConfigurationS
private readonly onDidChangeEmitter = new Emitter<LanguageConfigurationServiceChangeEvent>({
onFirstListenerAdd: () => {
this.registration = LanguageConfigurationRegistry.onDidChange((e) => {
this.onDidChangeEmitter.fire(new LanguageConfigurationServiceChangeEvent(e.languageIdentifier));
this.onDidChangeEmitter.fire(new LanguageConfigurationServiceChangeEvent(e.languageId));
});
},
onLastListenerRemove: () => {
@ -26,8 +25,8 @@ export class TestLanguageConfigurationService implements ILanguageConfigurationS
});
public readonly onDidChange = this.onDidChangeEmitter.event;
getLanguageConfiguration(languageId: LanguageId, resource?: URI): ResolvedLanguageConfiguration {
getLanguageConfiguration(languageId: string, resource?: URI): ResolvedLanguageConfiguration {
return LanguageConfigurationRegistry.getLanguageConfiguration(languageId) ??
new ResolvedLanguageConfiguration(new LanguageIdentifier('unknown', languageId), {});
new ResolvedLanguageConfiguration('unknown', {});
}
}

15
src/vs/editor/test/common/modes/textToHtmlTokenizer.test.ts
View file

@ -5,8 +5,9 @@
import * as assert from 'assert';
import { TokenizationResult2 } from 'vs/editor/common/core/token';
import { ColorId, FontStyle, IState, LanguageIdentifier, MetadataConsts, TokenizationRegistry } from 'vs/editor/common/modes';
import { ColorId, FontStyle, IState, MetadataConsts, TokenizationRegistry } from 'vs/editor/common/modes';
import { tokenizeLineToHTML, tokenizeToString } from 'vs/editor/common/modes/textToHtmlTokenizer';
import { LanguageIdCodec } from 'vs/editor/common/services/languagesRegistry';
import { ViewLineToken, ViewLineTokens } from 'vs/editor/test/common/core/viewLineToken';
import { MockMode } from 'vs/editor/test/common/mocks/mockMode';
@ -18,9 +19,9 @@ suite('Editor Modes - textToHtmlTokenizer', () => {
test('TextToHtmlTokenizer 1', () => {
let mode = new Mode();
let support = TokenizationRegistry.get(mode.getId())!;
let support = TokenizationRegistry.get(mode.languageId)!;
let actual = tokenizeToString('.abc..def...gh', support);
let actual = tokenizeToString('.abc..def...gh', new LanguageIdCodec(), support);
let expected = [
{ className: 'mtk7', text: '.' },
{ className: 'mtk9', text: 'abc' },
@ -38,9 +39,9 @@ suite('Editor Modes - textToHtmlTokenizer', () => {
test('TextToHtmlTokenizer 2', () => {
let mode = new Mode();
let support = TokenizationRegistry.get(mode.getId())!;
let support = TokenizationRegistry.get(mode.languageId)!;
let actual = tokenizeToString('.abc..def...gh\n.abc..def...gh', support);
let actual = tokenizeToString('.abc..def...gh\n.abc..def...gh', new LanguageIdCodec(), support);
let expected1 = [
{ className: 'mtk7', text: '.' },
{ className: 'mtk9', text: 'abc' },
@ -280,11 +281,11 @@ suite('Editor Modes - textToHtmlTokenizer', () => {
class Mode extends MockMode {
private static readonly _id = new LanguageIdentifier('textToHtmlTokenizerMode', 3);
private static readonly _id = 'textToHtmlTokenizerMode';
constructor() {
super(Mode._id);
this._register(TokenizationRegistry.register(this.getId(), {
this._register(TokenizationRegistry.register(this.languageId, {
getInitialState: (): IState => null!,
tokenize: undefined!,
tokenize2: (line: string, hasEOL: boolean, state: IState): TokenizationResult2 => {

3
src/vs/editor/test/common/modesTestUtils.ts
View file

@ -6,6 +6,7 @@
import { LineTokens } from 'vs/editor/common/core/lineTokens';
import { MetadataConsts, StandardTokenType } from 'vs/editor/common/modes';
import { ScopedLineTokens, createScopedLineTokens } from 'vs/editor/common/modes/supports';
import { LanguageIdCodec } from 'vs/editor/common/services/languagesRegistry';
export interface TokenText {
text: string;
@ -30,5 +31,5 @@ export function createFakeScopedLineTokens(rawTokens: TokenText[]): ScopedLineTo
}
LineTokens.convertToEndOffset(tokens, line.length);
return createScopedLineTokens(new LineTokens(tokens, line), 0);
return createScopedLineTokens(new LineTokens(tokens, line, new LanguageIdCodec()), 0);
}

52
src/vs/editor/test/common/services/languagesRegistry.test.ts
View file

@ -20,6 +20,8 @@ suite('LanguagesRegistry', () => {
}]);
assert.deepStrictEqual(registry.getRegisteredLanguageNames(), []);
registry.dispose();
});
test('mode with alias does have a name', () => {
@ -34,6 +36,8 @@ suite('LanguagesRegistry', () => {
assert.deepStrictEqual(registry.getRegisteredLanguageNames(), ['ModeName']);
assert.deepStrictEqual(registry.getLanguageName('modeId'), 'ModeName');
registry.dispose();
});
test('mode without alias gets a name', () => {
@ -47,6 +51,8 @@ suite('LanguagesRegistry', () => {
assert.deepStrictEqual(registry.getRegisteredLanguageNames(), ['modeId']);
assert.deepStrictEqual(registry.getLanguageName('modeId'), 'modeId');
registry.dispose();
});
test('bug #4360: f# not shown in status bar', () => {
@ -68,6 +74,8 @@ suite('LanguagesRegistry', () => {
assert.deepStrictEqual(registry.getRegisteredLanguageNames(), ['ModeName']);
assert.deepStrictEqual(registry.getLanguageName('modeId'), 'ModeName');
registry.dispose();
});
test('issue #5278: Extension cannot override language name anymore', () => {
@ -89,6 +97,8 @@ suite('LanguagesRegistry', () => {
assert.deepStrictEqual(registry.getRegisteredLanguageNames(), ['BetterModeName']);
assert.deepStrictEqual(registry.getLanguageName('modeId'), 'BetterModeName');
registry.dispose();
});
test('mimetypes are generated if necessary', () => {
@ -99,6 +109,8 @@ suite('LanguagesRegistry', () => {
}]);
assert.deepStrictEqual(registry.getMimeForMode('modeId'), 'text/x-modeId');
registry.dispose();
});
test('first mimetype wins', () => {
@ -110,6 +122,8 @@ suite('LanguagesRegistry', () => {
}]);
assert.deepStrictEqual(registry.getMimeForMode('modeId'), 'text/modeId');
registry.dispose();
});
test('first mimetype wins 2', () => {
@ -125,6 +139,8 @@ suite('LanguagesRegistry', () => {
}]);
assert.deepStrictEqual(registry.getMimeForMode('modeId'), 'text/x-modeId');
registry.dispose();
});
test('aliases', () => {
@ -135,7 +151,7 @@ suite('LanguagesRegistry', () => {
}]);
assert.deepStrictEqual(registry.getRegisteredLanguageNames(), ['a']);
assert.deepStrictEqual(registry.getModeIdsFromLanguageName('a'), ['a']);
assert.deepStrictEqual(registry.getModeIdFromLanguageName('a'), 'a');
assert.deepStrictEqual(registry.getModeIdForLanguageNameLowercase('a'), 'a');
assert.deepStrictEqual(registry.getLanguageName('a'), 'a');
@ -145,9 +161,9 @@ suite('LanguagesRegistry', () => {
}]);
assert.deepStrictEqual(registry.getRegisteredLanguageNames(), ['A1']);
assert.deepStrictEqual(registry.getModeIdsFromLanguageName('a'), []);
assert.deepStrictEqual(registry.getModeIdsFromLanguageName('A1'), ['a']);
assert.deepStrictEqual(registry.getModeIdsFromLanguageName('A2'), []);
assert.deepStrictEqual(registry.getModeIdFromLanguageName('a'), null);
assert.deepStrictEqual(registry.getModeIdFromLanguageName('A1'), 'a');
assert.deepStrictEqual(registry.getModeIdFromLanguageName('A2'), null);
assert.deepStrictEqual(registry.getModeIdForLanguageNameLowercase('a'), 'a');
assert.deepStrictEqual(registry.getModeIdForLanguageNameLowercase('a1'), 'a');
assert.deepStrictEqual(registry.getModeIdForLanguageNameLowercase('a2'), 'a');
@ -159,17 +175,19 @@ suite('LanguagesRegistry', () => {
}]);
assert.deepStrictEqual(registry.getRegisteredLanguageNames(), ['A3']);
assert.deepStrictEqual(registry.getModeIdsFromLanguageName('a'), []);
assert.deepStrictEqual(registry.getModeIdsFromLanguageName('A1'), []);
assert.deepStrictEqual(registry.getModeIdsFromLanguageName('A2'), []);
assert.deepStrictEqual(registry.getModeIdsFromLanguageName('A3'), ['a']);
assert.deepStrictEqual(registry.getModeIdsFromLanguageName('A4'), []);
assert.deepStrictEqual(registry.getModeIdFromLanguageName('a'), null);
assert.deepStrictEqual(registry.getModeIdFromLanguageName('A1'), null);
assert.deepStrictEqual(registry.getModeIdFromLanguageName('A2'), null);
assert.deepStrictEqual(registry.getModeIdFromLanguageName('A3'), 'a');
assert.deepStrictEqual(registry.getModeIdFromLanguageName('A4'), null);
assert.deepStrictEqual(registry.getModeIdForLanguageNameLowercase('a'), 'a');
assert.deepStrictEqual(registry.getModeIdForLanguageNameLowercase('a1'), 'a');
assert.deepStrictEqual(registry.getModeIdForLanguageNameLowercase('a2'), 'a');
assert.deepStrictEqual(registry.getModeIdForLanguageNameLowercase('a3'), 'a');
assert.deepStrictEqual(registry.getModeIdForLanguageNameLowercase('a4'), 'a');
assert.deepStrictEqual(registry.getLanguageName('a'), 'A3');
registry.dispose();
});
test('empty aliases array means no alias', () => {
@ -180,7 +198,7 @@ suite('LanguagesRegistry', () => {
}]);
assert.deepStrictEqual(registry.getRegisteredLanguageNames(), ['a']);
assert.deepStrictEqual(registry.getModeIdsFromLanguageName('a'), ['a']);
assert.deepStrictEqual(registry.getModeIdFromLanguageName('a'), 'a');
assert.deepStrictEqual(registry.getModeIdForLanguageNameLowercase('a'), 'a');
assert.deepStrictEqual(registry.getLanguageName('a'), 'a');
@ -190,12 +208,14 @@ suite('LanguagesRegistry', () => {
}]);
assert.deepStrictEqual(registry.getRegisteredLanguageNames(), ['a']);
assert.deepStrictEqual(registry.getModeIdsFromLanguageName('a'), ['a']);
assert.deepStrictEqual(registry.getModeIdsFromLanguageName('b'), []);
assert.deepStrictEqual(registry.getModeIdFromLanguageName('a'), 'a');
assert.deepStrictEqual(registry.getModeIdFromLanguageName('b'), null);
assert.deepStrictEqual(registry.getModeIdForLanguageNameLowercase('a'), 'a');
assert.deepStrictEqual(registry.getModeIdForLanguageNameLowercase('b'), 'b');
assert.deepStrictEqual(registry.getLanguageName('a'), 'a');
assert.deepStrictEqual(registry.getLanguageName('b'), null);
registry.dispose();
});
test('extensions', () => {
@ -219,6 +239,8 @@ suite('LanguagesRegistry', () => {
assert.deepStrictEqual(registry.getExtensions('a'), []);
assert.deepStrictEqual(registry.getExtensions('aname'), []);
assert.deepStrictEqual(registry.getExtensions('aName'), ['aExt', 'aExt2']);
registry.dispose();
});
test('extensions of primary language registration come first', () => {
@ -245,6 +267,8 @@ suite('LanguagesRegistry', () => {
}]);
assert.deepStrictEqual(registry.getExtensions('a')[0], 'aExt');
registry.dispose();
});
test('filenames', () => {
@ -268,6 +292,8 @@ suite('LanguagesRegistry', () => {
assert.deepStrictEqual(registry.getFilenames('a'), []);
assert.deepStrictEqual(registry.getFilenames('aname'), []);
assert.deepStrictEqual(registry.getFilenames('aName'), ['aFilename', 'aFilename2']);
registry.dispose();
});
test('configuration', () => {
@ -291,5 +317,7 @@ suite('LanguagesRegistry', () => {
assert.deepStrictEqual(registry.getConfigurationFiles('a'), [URI.file('/path/to/aFilename'), URI.file('/path/to/aFilename2')]);
assert.deepStrictEqual(registry.getConfigurationFiles('aname'), []);
assert.deepStrictEqual(registry.getConfigurationFiles('aName'), []);
registry.dispose();
});
});

44
src/vs/editor/test/common/services/modelService.test.ts
View file

@ -36,19 +36,29 @@ import { TestLanguageConfigurationService } from 'vs/editor/test/common/modes/te
const GENERATE_TESTS = false;
suite('ModelService', () => {
let disposables: DisposableStore;
let modelService: ModelServiceImpl;
setup(() => {
disposables = new DisposableStore();
const configService = new TestConfigurationService();
configService.setUserConfiguration('files', { 'eol': '\n' });
configService.setUserConfiguration('files', { 'eol': '\r\n' }, URI.file(platform.isWindows ? 'c:\\myroot' : '/myroot'));
const dialogService = new TestDialogService();
modelService = new ModelServiceImpl(configService, new TestTextResourcePropertiesService(configService), new TestThemeService(), new NullLogService(), new UndoRedoService(dialogService, new TestNotificationService()), new TestLanguageConfigurationService());
modelService = disposables.add(new ModelServiceImpl(
configService,
new TestTextResourcePropertiesService(configService),
new TestThemeService(),
new NullLogService(),
new UndoRedoService(dialogService, new TestNotificationService()),
disposables.add(new ModeServiceImpl()),
new TestLanguageConfigurationService()
));
});
teardown(() => {
modelService.dispose();
disposables.dispose();
});
test('EOL setting respected depending on root', () => {
@ -63,14 +73,14 @@ suite('ModelService', () => {
test('_computeEdits no change', function () {
const model = createTextModel(
const model = disposables.add(createTextModel(
[
'This is line one', //16
'and this is line number two', //27
'it is followed by #3', //20
'and finished with the fourth.', //29
].join('\n')
);
));
const textBuffer = createTextBuffer(
[
@ -89,14 +99,14 @@ suite('ModelService', () => {
test('_computeEdits first line changed', function () {
const model = createTextModel(
const model = disposables.add(createTextModel(
[
'This is line one', //16
'and this is line number two', //27
'it is followed by #3', //20
'and finished with the fourth.', //29
].join('\n')
);
));
const textBuffer = createTextBuffer(
[
@ -117,14 +127,14 @@ suite('ModelService', () => {
test('_computeEdits EOL changed', function () {
const model = createTextModel(
const model = disposables.add(createTextModel(
[
'This is line one', //16
'and this is line number two', //27
'it is followed by #3', //20
'and finished with the fourth.', //29
].join('\n')
);
));
const textBuffer = createTextBuffer(
[
@ -143,14 +153,14 @@ suite('ModelService', () => {
test('_computeEdits EOL and other change 1', function () {
const model = createTextModel(
const model = disposables.add(createTextModel(
[
'This is line one', //16
'and this is line number two', //27
'it is followed by #3', //20
'and finished with the fourth.', //29
].join('\n')
);
));
const textBuffer = createTextBuffer(
[
@ -179,13 +189,13 @@ suite('ModelService', () => {
test('_computeEdits EOL and other change 2', function () {
const model = createTextModel(
const model = disposables.add(createTextModel(
[
'package main', // 1
'func foo() {', // 2
'}' // 3
].join('\n')
);
));
const textBuffer = createTextBuffer(
[
@ -335,6 +345,8 @@ suite('ModelService', () => {
// undo
model2.undo();
assert.strictEqual(model2.getValue(), 'text');
// dispose it
modelService.destroyModel(resource);
});
test('maintains version id and alternative version id for same resource and same content', () => {
@ -354,6 +366,8 @@ suite('ModelService', () => {
const model2 = modelService.createModel('text1', null, resource);
assert.strictEqual(model2.getVersionId(), versionId);
assert.strictEqual(model2.getAlternativeVersionId(), alternativeVersionId);
// dispose it
modelService.destroyModel(resource);
});
test('does not maintain undo for same resource and different content', () => {
@ -372,6 +386,8 @@ suite('ModelService', () => {
// undo
model2.undo();
assert.strictEqual(model2.getValue(), 'text2');
// dispose it
modelService.destroyModel(resource);
});
test('setValue should clear undo stack', () => {
@ -384,6 +400,8 @@ suite('ModelService', () => {
model.setValue('text2');
model.undo();
assert.strictEqual(model.getValue(), 'text2');
// dispose it
modelService.destroyModel(resource);
});
});
@ -406,6 +424,7 @@ suite('ModelSemanticColoring', () => {
themeService,
new NullLogService(),
new UndoRedoService(new TestDialogService(), new TestNotificationService()),
disposables.add(new ModeServiceImpl()),
new TestLanguageConfigurationService()
));
modeService = disposables.add(new ModeServiceImpl(false));
@ -482,6 +501,7 @@ function assertComputeEdits(lines1: string[], lines2: string[]): void {
model.pushEditOperations([], edits, null);
assert.strictEqual(model.getValue(), lines2.join('\n'));
model.dispose();
}
function getRandomInt(min: number, max: number): number {

20
src/vs/editor/test/common/viewModel/splitLinesCollection.test.ts
View file

@ -326,8 +326,8 @@ suite('SplitLinesCollection', () => {
]
];
let model: TextModel | null = null;
let languageRegistration: IDisposable | null = null;
let model: TextModel;
let languageRegistration: IDisposable;
setup(() => {
let _lineIndex = 0;
@ -349,16 +349,14 @@ suite('SplitLinesCollection', () => {
};
const LANGUAGE_ID = 'modelModeTest1';
languageRegistration = modes.TokenizationRegistry.register(LANGUAGE_ID, tokenizationSupport);
model = createTextModel(_text.join('\n'), undefined, new modes.LanguageIdentifier(LANGUAGE_ID, 0));
model = createTextModel(_text.join('\n'), undefined, LANGUAGE_ID);
// force tokenization
model.forceTokenization(model.getLineCount());
});
teardown(() => {
model!.dispose();
model = null;
languageRegistration!.dispose();
languageRegistration = null;
model.dispose();
languageRegistration.dispose();
});
@ -433,7 +431,7 @@ suite('SplitLinesCollection', () => {
}
test('getViewLinesData - no wrapping', () => {
withSplitLinesCollection(model!, 'off', 0, (splitLinesCollection) => {
withSplitLinesCollection(model, 'off', 0, (splitLinesCollection) => {
assert.strictEqual(splitLinesCollection.getViewLineCount(), 8);
assert.strictEqual(splitLinesCollection.modelPositionIsVisible(1, 1), true);
assert.strictEqual(splitLinesCollection.modelPositionIsVisible(2, 1), true);
@ -567,7 +565,7 @@ suite('SplitLinesCollection', () => {
});
test('getViewLinesData - with wrapping', () => {
withSplitLinesCollection(model!, 'wordWrapColumn', 30, (splitLinesCollection) => {
withSplitLinesCollection(model, 'wordWrapColumn', 30, (splitLinesCollection) => {
assert.strictEqual(splitLinesCollection.getViewLineCount(), 12);
assert.strictEqual(splitLinesCollection.modelPositionIsVisible(1, 1), true);
assert.strictEqual(splitLinesCollection.modelPositionIsVisible(2, 1), true);
@ -740,7 +738,7 @@ suite('SplitLinesCollection', () => {
});
test('getViewLinesData - with wrapping and injected text', () => {
model!.deltaDecorations([], [{
model.deltaDecorations([], [{
range: new Range(1, 9, 1, 9),
options: {
description: 'example',
@ -751,7 +749,7 @@ suite('SplitLinesCollection', () => {
}
}]);
withSplitLinesCollection(model!, 'wordWrapColumn', 30, (splitLinesCollection) => {
withSplitLinesCollection(model, 'wordWrapColumn', 30, (splitLinesCollection) => {
assert.strictEqual(splitLinesCollection.getViewLineCount(), 14);
assert.strictEqual(splitLinesCollection.getViewLineMaxColumn(1), 24);

2
src/vs/monaco.d.ts vendored
View file

@ -1877,7 +1877,7 @@ declare namespace monaco.editor {
/**
* Get the language associated with this model.
*/
getModeId(): string;
getLanguageId(): string;
/**
* Get the word under or besides `position`.
* @param position The position to look for a word.

2
src/vs/workbench/api/browser/mainThreadDocuments.ts
View file

@ -198,7 +198,7 @@ export class MainThreadDocuments extends Disposable implements MainThreadDocumen
if (!this._modelIsSynced.has(model.uri)) {
return;
}
this._proxy.$acceptModelModeChanged(model.uri, model.getLanguageIdentifier().language);
this._proxy.$acceptModelModeChanged(model.uri, model.getLanguageId());
}
private _onModelRemoved(modelUrl: URI): void {

2
src/vs/workbench/api/browser/mainThreadDocumentsAndEditors.ts
View file

@ -397,7 +397,7 @@ export class MainThreadDocumentsAndEditors {
versionId: model.getVersionId(),
lines: model.getLinesContent(),
EOL: model.getEOL(),
modeId: model.getLanguageIdentifier().language,
modeId: model.getLanguageId(),
isDirty: this._textFileService.isDirty(model.uri)
};
}

2
src/vs/workbench/api/browser/mainThreadEditor.ts
View file

@ -349,7 +349,7 @@ export class MainThreadTextEditor {
}
private _setIndentConfiguration(newConfiguration: ITextEditorConfigurationUpdate): void {
const creationOpts = this._modelService.getCreationOptions(this._model.getLanguageIdentifier().language, this._model.uri, this._model.isForSimpleWidget);
const creationOpts = this._modelService.getCreationOptions(this._model.getLanguageId(), this._model.uri, this._model.isForSimpleWidget);
if (newConfiguration.tabSize === 'auto' || newConfiguration.insertSpaces === 'auto') {
// one of the options was set to 'auto' => detect indentation

Some files were not shown because too many files have changed in this diff Show more