From d309e11579e3bfe877014834384e9f502e4996ad Mon Sep 17 00:00:00 2001 From: Aiday Marlen Kyzy Date: Mon, 27 May 2024 11:18:00 +0200 Subject: [PATCH] Remove brackets from comments, strings and regexes before evaluating the indentation (#210641) * wip * polishing the code * adding code * adding the language * reshuffling the code to avoid cyclic dependency * polihsing code * uncommenting tests * also adopting the indentation rules within the reindentation operation * using instead the sliced line tokens instead of the scoped line tokens * polishing the code * using start indices instead * using value everywhere * using the token data to type the tokens * setting to number instead of standard token type * using token data from autoindenttest.ts * using same code in both test files * placing instantiation service into the registerLanguage method * copying object into the node js autoindent.ts --- .../javascript-language-configuration.json | 4 +- .../language-configuration.json | 4 +- src/vs/editor/common/languages/autoIndent.ts | 170 ++++++------- src/vs/editor/common/languages/enterAction.ts | 35 +-- .../languageConfigurationRegistry.ts | 8 - src/vs/editor/common/languages/supports.ts | 13 +- .../supports/indentationLineProcessor.ts | 236 ++++++++++++++++++ .../supports/languageBracketsConfiguration.ts | 7 + .../languages/supports/richEditBrackets.ts | 5 +- src/vs/editor/common/tokens/lineTokens.ts | 66 ++++- .../contrib/indentation/common/indentation.ts | 49 ++-- .../test/browser/indentation.test.ts | 40 +-- .../browser/indentationLineProcessor.test.ts | 236 ++++++++++++++++++ .../editor/test/common/core/testLineToken.ts | 23 +- 14 files changed, 711 insertions(+), 185 deletions(-) create mode 100644 src/vs/editor/common/languages/supports/indentationLineProcessor.ts create mode 100644 src/vs/editor/contrib/indentation/test/browser/indentationLineProcessor.test.ts diff --git a/extensions/javascript/javascript-language-configuration.json b/extensions/javascript/javascript-language-configuration.json index fb2fb0397d7..f7c332337cb 100644 --- a/extensions/javascript/javascript-language-configuration.json +++ b/extensions/javascript/javascript-language-configuration.json @@ -111,10 +111,10 @@ }, "indentationRules": { "decreaseIndentPattern": { - "pattern": "^((?!.*?/\\*).*\\*\/)?\\s*[\\}\\]\\)].*$" + "pattern": "^\\s*[\\}\\]\\)].*$" }, "increaseIndentPattern": { - "pattern": "^((?!//).)*(\\{([^}\"'`/]*|(\\t|[ ])*//.*)|\\([^)\"'`/]*|\\[[^\\]\"'`/]*)$" + "pattern": "^.*(\\{[^}]*|\\([^)]*|\\[[^\\]]*)$" }, // e.g. * ...| or */| or *-----*/| "unIndentedLinePattern": { diff --git a/extensions/typescript-basics/language-configuration.json b/extensions/typescript-basics/language-configuration.json index 070b8911a82..25a23685738 100644 --- a/extensions/typescript-basics/language-configuration.json +++ b/extensions/typescript-basics/language-configuration.json @@ -129,10 +129,10 @@ }, "indentationRules": { "decreaseIndentPattern": { - "pattern": "^((?!.*?/\\*).*\\*\/)?\\s*[\\}\\]\\)].*$" + "pattern": "^\\s*[\\}\\]\\)].*$" }, "increaseIndentPattern": { - "pattern": "^((?!//).)*(\\{([^}\"'`/]*|(\\t|[ ])*//.*)|\\([^)\"'`/]*|\\[[^\\]\"'`/]*)$" + "pattern": "^.*(\\{[^}]*|\\([^)]*|\\[[^\\]]*)$" }, // e.g. * ...| or */| or *-----*/| "unIndentedLinePattern": { diff --git a/src/vs/editor/common/languages/autoIndent.ts b/src/vs/editor/common/languages/autoIndent.ts index 9ae3df974aa..680d3a7d5f9 100644 --- a/src/vs/editor/common/languages/autoIndent.ts +++ b/src/vs/editor/common/languages/autoIndent.ts @@ -7,17 +7,18 @@ import * as strings from 'vs/base/common/strings'; import { Range } from 'vs/editor/common/core/range'; import { ITextModel } from 'vs/editor/common/model'; import { IndentAction } from 'vs/editor/common/languages/languageConfiguration'; -import { createScopedLineTokens } from 'vs/editor/common/languages/supports'; -import { IndentConsts, IndentRulesSupport } from 'vs/editor/common/languages/supports/indentRules'; +import { IndentConsts } from 'vs/editor/common/languages/supports/indentRules'; import { EditorAutoIndentStrategy } from 'vs/editor/common/config/editorOptions'; -import { getScopedLineTokens, ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry'; -import { LineTokens } from 'vs/editor/common/tokens/lineTokens'; +import { ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry'; +import { IViewLineTokens } from 'vs/editor/common/tokens/lineTokens'; +import { IndentationContextProcessor, isLanguageDifferentFromLineStart, ProcessedIndentRulesSupport } from 'vs/editor/common/languages/supports/indentationLineProcessor'; export interface IVirtualModel { tokenization: { - getLineTokens(lineNumber: number): LineTokens; + getLineTokens(lineNumber: number): IViewLineTokens; getLanguageId(): string; getLanguageIdAtPosition(lineNumber: number, column: number): string; + forceTokenization?(lineNumber: number): void; }; getLineContent(lineNumber: number): string; } @@ -35,7 +36,7 @@ export interface IIndentConverter { * 0: every line above are invalid * else: nearest preceding line of the same language */ -function getPrecedingValidLine(model: IVirtualModel, lineNumber: number, indentRulesSupport: IndentRulesSupport) { +function getPrecedingValidLine(model: IVirtualModel, lineNumber: number, processedIndentRulesSupport: ProcessedIndentRulesSupport) { const languageId = model.tokenization.getLanguageIdAtPosition(lineNumber, 0); if (lineNumber > 1) { let lastLineNumber: number; @@ -46,7 +47,7 @@ function getPrecedingValidLine(model: IVirtualModel, lineNumber: number, indentR return resultLineNumber; } const text = model.getLineContent(lastLineNumber); - if (indentRulesSupport.shouldIgnore(text) || /^\s+$/.test(text) || text === '') { + if (processedIndentRulesSupport.shouldIgnore(lastLineNumber) || /^\s+$/.test(text) || text === '') { resultLineNumber = lastLineNumber; continue; } @@ -85,6 +86,7 @@ export function getInheritIndentForLine( if (!indentRulesSupport) { return null; } + const processedIndentRulesSupport = new ProcessedIndentRulesSupport(model, indentRulesSupport, languageConfigurationService); if (lineNumber <= 1) { return { @@ -106,7 +108,7 @@ export function getInheritIndentForLine( } } - const precedingUnIgnoredLine = getPrecedingValidLine(model, lineNumber, indentRulesSupport); + const precedingUnIgnoredLine = getPrecedingValidLine(model, lineNumber, processedIndentRulesSupport); if (precedingUnIgnoredLine < 0) { return null; } else if (precedingUnIgnoredLine < 1) { @@ -116,14 +118,15 @@ export function getInheritIndentForLine( }; } - const precedingUnIgnoredLineContent = model.getLineContent(precedingUnIgnoredLine); - if (indentRulesSupport.shouldIncrease(precedingUnIgnoredLineContent) || indentRulesSupport.shouldIndentNextLine(precedingUnIgnoredLineContent)) { + if (processedIndentRulesSupport.shouldIncrease(precedingUnIgnoredLine) || processedIndentRulesSupport.shouldIndentNextLine(precedingUnIgnoredLine)) { + const precedingUnIgnoredLineContent = model.getLineContent(precedingUnIgnoredLine); return { indentation: strings.getLeadingWhitespace(precedingUnIgnoredLineContent), action: IndentAction.Indent, line: precedingUnIgnoredLine }; - } else if (indentRulesSupport.shouldDecrease(precedingUnIgnoredLineContent)) { + } else if (processedIndentRulesSupport.shouldDecrease(precedingUnIgnoredLine)) { + const precedingUnIgnoredLineContent = model.getLineContent(precedingUnIgnoredLine); return { indentation: strings.getLeadingWhitespace(precedingUnIgnoredLineContent), action: null, @@ -150,7 +153,7 @@ export function getInheritIndentForLine( (previousLineIndentMetadata & IndentConsts.INDENT_NEXTLINE_MASK)) { let stopLine = 0; for (let i = previousLine - 1; i > 0; i--) { - if (indentRulesSupport.shouldIndentNextLine(model.getLineContent(i))) { + if (processedIndentRulesSupport.shouldIndentNextLine(i)) { continue; } stopLine = i; @@ -173,17 +176,16 @@ export function getInheritIndentForLine( } else { // search from precedingUnIgnoredLine until we find one whose indent is not temporary for (let i = precedingUnIgnoredLine; i > 0; i--) { - const lineContent = model.getLineContent(i); - if (indentRulesSupport.shouldIncrease(lineContent)) { + if (processedIndentRulesSupport.shouldIncrease(i)) { return { - indentation: strings.getLeadingWhitespace(lineContent), + indentation: strings.getLeadingWhitespace(model.getLineContent(i)), action: IndentAction.Indent, line: i }; - } else if (indentRulesSupport.shouldIndentNextLine(lineContent)) { + } else if (processedIndentRulesSupport.shouldIndentNextLine(i)) { let stopLine = 0; for (let j = i - 1; j > 0; j--) { - if (indentRulesSupport.shouldIndentNextLine(model.getLineContent(i))) { + if (processedIndentRulesSupport.shouldIndentNextLine(i)) { continue; } stopLine = j; @@ -195,9 +197,9 @@ export function getInheritIndentForLine( action: null, line: stopLine + 1 }; - } else if (indentRulesSupport.shouldDecrease(lineContent)) { + } else if (processedIndentRulesSupport.shouldDecrease(i)) { return { - indentation: strings.getLeadingWhitespace(lineContent), + indentation: strings.getLeadingWhitespace(model.getLineContent(i)), action: null, line: i }; @@ -235,8 +237,8 @@ export function getGoodIndentForLine( return null; } + const processedIndentRulesSupport = new ProcessedIndentRulesSupport(virtualModel, indentRulesSupport, languageConfigurationService); const indent = getInheritIndentForLine(autoIndent, virtualModel, lineNumber, undefined, languageConfigurationService); - const lineContent = virtualModel.getLineContent(lineNumber); if (indent) { const inheritLine = indent.line; @@ -268,7 +270,7 @@ export function getGoodIndentForLine( indentation = indentConverter.unshiftIndent(indentation); } - if (indentRulesSupport.shouldDecrease(lineContent)) { + if (processedIndentRulesSupport.shouldDecrease(lineNumber)) { indentation = indentConverter.unshiftIndent(indentation); } @@ -281,7 +283,7 @@ export function getGoodIndentForLine( } } - if (indentRulesSupport.shouldDecrease(lineContent)) { + if (processedIndentRulesSupport.shouldDecrease(lineNumber)) { if (indent.action === IndentAction.Indent) { return indent.indentation; } else { @@ -308,80 +310,44 @@ export function getIndentForEnter( if (autoIndent < EditorAutoIndentStrategy.Full) { return null; } - model.tokenization.forceTokenization(range.startLineNumber); - const lineTokens = model.tokenization.getLineTokens(range.startLineNumber); - const scopedLineTokens = createScopedLineTokens(lineTokens, range.startColumn - 1); - const scopedLineText = scopedLineTokens.getLineContent(); - - let embeddedLanguage = false; - let beforeEnterText: string; - if (scopedLineTokens.firstCharOffset > 0 && lineTokens.getLanguageId(0) !== scopedLineTokens.languageId) { - // we are in the embeded language content - embeddedLanguage = true; // if embeddedLanguage is true, then we don't touch the indentation of current line - beforeEnterText = scopedLineText.substr(0, range.startColumn - 1 - scopedLineTokens.firstCharOffset); - } else { - beforeEnterText = lineTokens.getLineContent().substring(0, range.startColumn - 1); - } - - let afterEnterText: string; - if (range.isEmpty()) { - afterEnterText = scopedLineText.substr(range.startColumn - 1 - scopedLineTokens.firstCharOffset); - } else { - const endScopedLineTokens = getScopedLineTokens(model, range.endLineNumber, range.endColumn); - afterEnterText = endScopedLineTokens.getLineContent().substr(range.endColumn - 1 - scopedLineTokens.firstCharOffset); - } - - const indentRulesSupport = languageConfigurationService.getLanguageConfiguration(scopedLineTokens.languageId).indentRulesSupport; + const languageId = model.getLanguageIdAtPosition(range.startLineNumber, range.startColumn); + const indentRulesSupport = languageConfigurationService.getLanguageConfiguration(languageId).indentRulesSupport; if (!indentRulesSupport) { return null; } - const beforeEnterResult = beforeEnterText; - const beforeEnterIndent = strings.getLeadingWhitespace(beforeEnterText); + model.tokenization.forceTokenization(range.startLineNumber); + const indentationContextProcessor = new IndentationContextProcessor(model, languageConfigurationService); + const processedContextTokens = indentationContextProcessor.getProcessedTokenContextAroundRange(range); + const afterEnterProcessedTokens = processedContextTokens.afterRangeProcessedTokens; + const beforeEnterProcessedTokens = processedContextTokens.beforeRangeProcessedTokens; + const beforeEnterIndent = strings.getLeadingWhitespace(beforeEnterProcessedTokens.getLineContent()); - const virtualModel: IVirtualModel = { - tokenization: { - getLineTokens: (lineNumber: number) => { - return model.tokenization.getLineTokens(lineNumber); - }, - getLanguageId: () => { - return model.getLanguageId(); - }, - getLanguageIdAtPosition: (lineNumber: number, column: number) => { - return model.getLanguageIdAtPosition(lineNumber, column); - }, - }, - getLineContent: (lineNumber: number) => { - if (lineNumber === range.startLineNumber) { - return beforeEnterResult; - } else { - return model.getLineContent(lineNumber); - } - } - }; - - const currentLineIndent = strings.getLeadingWhitespace(lineTokens.getLineContent()); + const virtualModel = createVirtualModelWithModifiedTokensAtLine(model, range.startLineNumber, beforeEnterProcessedTokens); + const languageIsDifferentFromLineStart = isLanguageDifferentFromLineStart(model, range.getStartPosition()); + const currentLine = model.getLineContent(range.startLineNumber); + const currentLineIndent = strings.getLeadingWhitespace(currentLine); const afterEnterAction = getInheritIndentForLine(autoIndent, virtualModel, range.startLineNumber + 1, undefined, languageConfigurationService); if (!afterEnterAction) { - const beforeEnter = embeddedLanguage ? currentLineIndent : beforeEnterIndent; + const beforeEnter = languageIsDifferentFromLineStart ? currentLineIndent : beforeEnterIndent; return { beforeEnter: beforeEnter, afterEnter: beforeEnter }; } - let afterEnterIndent = embeddedLanguage ? currentLineIndent : afterEnterAction.indentation; + let afterEnterIndent = languageIsDifferentFromLineStart ? currentLineIndent : afterEnterAction.indentation; if (afterEnterAction.action === IndentAction.Indent) { afterEnterIndent = indentConverter.shiftIndent(afterEnterIndent); } - if (indentRulesSupport.shouldDecrease(afterEnterText)) { + if (indentRulesSupport.shouldDecrease(afterEnterProcessedTokens.getLineContent())) { afterEnterIndent = indentConverter.unshiftIndent(afterEnterIndent); } return { - beforeEnter: embeddedLanguage ? currentLineIndent : beforeEnterIndent, + beforeEnter: languageIsDifferentFromLineStart ? currentLineIndent : beforeEnterIndent, afterEnter: afterEnterIndent }; } @@ -401,33 +367,28 @@ export function getIndentActionForType( if (autoIndent < EditorAutoIndentStrategy.Full) { return null; } - const scopedLineTokens = getScopedLineTokens(model, range.startLineNumber, range.startColumn); - - if (scopedLineTokens.firstCharOffset) { + const languageIsDifferentFromLineStart = isLanguageDifferentFromLineStart(model, range.getStartPosition()); + if (languageIsDifferentFromLineStart) { // this line has mixed languages and indentation rules will not work return null; } - const indentRulesSupport = languageConfigurationService.getLanguageConfiguration(scopedLineTokens.languageId).indentRulesSupport; + const languageId = model.getLanguageIdAtPosition(range.startLineNumber, range.startColumn); + const indentRulesSupport = languageConfigurationService.getLanguageConfiguration(languageId).indentRulesSupport; if (!indentRulesSupport) { return null; } - const scopedLineText = scopedLineTokens.getLineContent(); - const beforeTypeText = scopedLineText.substr(0, range.startColumn - 1 - scopedLineTokens.firstCharOffset); - - // selection support - let afterTypeText: string; - if (range.isEmpty()) { - afterTypeText = scopedLineText.substr(range.startColumn - 1 - scopedLineTokens.firstCharOffset); - } else { - const endScopedLineTokens = getScopedLineTokens(model, range.endLineNumber, range.endColumn); - afterTypeText = endScopedLineTokens.getLineContent().substr(range.endColumn - 1 - scopedLineTokens.firstCharOffset); - } + const indentationContextProcessor = new IndentationContextProcessor(model, languageConfigurationService); + const processedContextTokens = indentationContextProcessor.getProcessedTokenContextAroundRange(range); + const beforeRangeText = processedContextTokens.beforeRangeProcessedTokens.getLineContent(); + const afterRangeText = processedContextTokens.afterRangeProcessedTokens.getLineContent(); + const textAroundRange = beforeRangeText + afterRangeText; + const textAroundRangeWithCharacter = beforeRangeText + ch + afterRangeText; // If previous content already matches decreaseIndentPattern, it means indentation of this line should already be adjusted // Users might change the indentation by purpose and we should honor that instead of readjusting. - if (!indentRulesSupport.shouldDecrease(beforeTypeText + afterTypeText) && indentRulesSupport.shouldDecrease(beforeTypeText + ch + afterTypeText)) { + if (!indentRulesSupport.shouldDecrease(textAroundRange) && indentRulesSupport.shouldDecrease(textAroundRangeWithCharacter)) { // after typing `ch`, the content matches decreaseIndentPattern, we should adjust the indent to a good manner. // 1. Get inherited indent action const r = getInheritIndentForLine(autoIndent, model, range.startLineNumber, false, languageConfigurationService); @@ -460,3 +421,32 @@ export function getIndentMetadata( } return indentRulesSupport.getIndentMetadata(model.getLineContent(lineNumber)); } + +function createVirtualModelWithModifiedTokensAtLine(model: ITextModel, modifiedLineNumber: number, modifiedTokens: IViewLineTokens): IVirtualModel { + const virtualModel: IVirtualModel = { + tokenization: { + getLineTokens: (lineNumber: number): IViewLineTokens => { + if (lineNumber === modifiedLineNumber) { + return modifiedTokens; + } else { + return model.tokenization.getLineTokens(lineNumber); + } + }, + getLanguageId: (): string => { + return model.getLanguageId(); + }, + getLanguageIdAtPosition: (lineNumber: number, column: number): string => { + return model.getLanguageIdAtPosition(lineNumber, column); + }, + }, + getLineContent: (lineNumber: number): string => { + if (lineNumber === modifiedLineNumber) { + return modifiedTokens.getLineContent(); + } else { + return model.getLineContent(lineNumber); + } + } + }; + return virtualModel; +} + diff --git a/src/vs/editor/common/languages/enterAction.ts b/src/vs/editor/common/languages/enterAction.ts index 447665fe816..27669db6ebe 100644 --- a/src/vs/editor/common/languages/enterAction.ts +++ b/src/vs/editor/common/languages/enterAction.ts @@ -7,7 +7,8 @@ import { Range } from 'vs/editor/common/core/range'; import { ITextModel } from 'vs/editor/common/model'; import { IndentAction, CompleteEnterAction } from 'vs/editor/common/languages/languageConfiguration'; import { EditorAutoIndentStrategy } from 'vs/editor/common/config/editorOptions'; -import { getIndentationAtPosition, getScopedLineTokens, ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry'; +import { getIndentationAtPosition, ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry'; +import { IndentationContextProcessor } from 'vs/editor/common/languages/supports/indentationLineProcessor'; export function getEnterAction( autoIndent: EditorAutoIndentStrategy, @@ -15,33 +16,17 @@ export function getEnterAction( range: Range, languageConfigurationService: ILanguageConfigurationService ): CompleteEnterAction | null { - const scopedLineTokens = getScopedLineTokens(model, range.startLineNumber, range.startColumn); - const richEditSupport = languageConfigurationService.getLanguageConfiguration(scopedLineTokens.languageId); + model.tokenization.forceTokenization(range.startLineNumber); + const languageId = model.getLanguageIdAtPosition(range.startLineNumber, range.startColumn); + const richEditSupport = languageConfigurationService.getLanguageConfiguration(languageId); if (!richEditSupport) { return null; } - - const scopedLineText = scopedLineTokens.getLineContent(); - const beforeEnterText = scopedLineText.substr(0, range.startColumn - 1 - scopedLineTokens.firstCharOffset); - - // selection support - let afterEnterText: string; - if (range.isEmpty()) { - afterEnterText = scopedLineText.substr(range.startColumn - 1 - scopedLineTokens.firstCharOffset); - } else { - const endScopedLineTokens = getScopedLineTokens(model, range.endLineNumber, range.endColumn); - afterEnterText = endScopedLineTokens.getLineContent().substr(range.endColumn - 1 - scopedLineTokens.firstCharOffset); - } - - let previousLineText = ''; - if (range.startLineNumber > 1 && scopedLineTokens.firstCharOffset === 0) { - // This is not the first line and the entire line belongs to this mode - const oneLineAboveScopedLineTokens = getScopedLineTokens(model, range.startLineNumber - 1); - if (oneLineAboveScopedLineTokens.languageId === scopedLineTokens.languageId) { - // The line above ends with text belonging to the same mode - previousLineText = oneLineAboveScopedLineTokens.getLineContent(); - } - } + const indentationContextProcessor = new IndentationContextProcessor(model, languageConfigurationService); + const processedContextTokens = indentationContextProcessor.getProcessedTokenContextAroundRange(range); + const previousLineText = processedContextTokens.previousLineProcessedTokens.getLineContent(); + const beforeEnterText = processedContextTokens.beforeRangeProcessedTokens.getLineContent(); + const afterEnterText = processedContextTokens.afterRangeProcessedTokens.getLineContent(); const enterResult = richEditSupport.onEnter(autoIndent, previousLineText, beforeEnterText, afterEnterText); if (!enterResult) { diff --git a/src/vs/editor/common/languages/languageConfigurationRegistry.ts b/src/vs/editor/common/languages/languageConfigurationRegistry.ts index d8598afe6fd..7ff5ddec8a6 100644 --- a/src/vs/editor/common/languages/languageConfigurationRegistry.ts +++ b/src/vs/editor/common/languages/languageConfigurationRegistry.ts @@ -9,7 +9,6 @@ import * as strings from 'vs/base/common/strings'; import { ITextModel } from 'vs/editor/common/model'; import { DEFAULT_WORD_REGEXP, ensureValidWordDefinition } from 'vs/editor/common/core/wordHelper'; import { EnterAction, FoldingRules, IAutoClosingPair, IndentationRule, LanguageConfiguration, AutoClosingPairs, CharacterPair, ExplicitLanguageConfiguration } from 'vs/editor/common/languages/languageConfiguration'; -import { createScopedLineTokens, ScopedLineTokens } from 'vs/editor/common/languages/supports'; import { CharacterPairSupport } from 'vs/editor/common/languages/supports/characterPair'; import { BracketElectricCharacterSupport } from 'vs/editor/common/languages/supports/electricCharacter'; import { IndentRulesSupport } from 'vs/editor/common/languages/supports/indentRules'; @@ -181,13 +180,6 @@ export function getIndentationAtPosition(model: ITextModel, lineNumber: number, return indentation; } -export function getScopedLineTokens(model: ITextModel, lineNumber: number, columnNumber?: number): ScopedLineTokens { - model.tokenization.forceTokenization(lineNumber); - const lineTokens = model.tokenization.getLineTokens(lineNumber); - const column = (typeof columnNumber === 'undefined' ? model.getLineMaxColumn(lineNumber) - 1 : columnNumber - 1); - return createScopedLineTokens(lineTokens, column); -} - class ComposedLanguageConfiguration { private readonly _entries: LanguageConfigurationContribution[]; private _order: number; diff --git a/src/vs/editor/common/languages/supports.ts b/src/vs/editor/common/languages/supports.ts index 8fdfa17bb51..730fa2a1b73 100644 --- a/src/vs/editor/common/languages/supports.ts +++ b/src/vs/editor/common/languages/supports.ts @@ -3,8 +3,9 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { LineTokens } from 'vs/editor/common/tokens/lineTokens'; +import { IViewLineTokens, LineTokens } from 'vs/editor/common/tokens/lineTokens'; import { StandardTokenType } from 'vs/editor/common/encodedTokenAttributes'; +import { ILanguageIdCodec } from 'vs/editor/common/languages'; export function createScopedLineTokens(context: LineTokens, offset: number): ScopedLineTokens { const tokenCount = context.getCount(); @@ -34,6 +35,7 @@ export function createScopedLineTokens(context: LineTokens, offset: number): Sco export class ScopedLineTokens { _scopedLineTokensBrand: void = undefined; + public readonly languageIdCodec: ILanguageIdCodec; public readonly languageId: string; private readonly _actual: LineTokens; private readonly _firstTokenIndex: number; @@ -55,6 +57,7 @@ export class ScopedLineTokens { this._lastTokenIndex = lastTokenIndex; this.firstCharOffset = firstCharOffset; this._lastCharOffset = lastCharOffset; + this.languageIdCodec = actual.languageIdCodec; } public getLineContent(): string { @@ -62,6 +65,10 @@ export class ScopedLineTokens { return actualLineContent.substring(this.firstCharOffset, this._lastCharOffset); } + public getLineLength(): number { + return this._lastCharOffset - this.firstCharOffset; + } + public getActualLineContentBefore(offset: number): string { const actualLineContent = this._actual.getLineContent(); return actualLineContent.substring(0, this.firstCharOffset + offset); @@ -78,6 +85,10 @@ export class ScopedLineTokens { public getStandardTokenType(tokenIndex: number): StandardTokenType { return this._actual.getStandardTokenType(tokenIndex + this._firstTokenIndex); } + + public toIViewLineTokens(): IViewLineTokens { + return this._actual.sliceAndInflate(this.firstCharOffset, this._lastCharOffset, 0); + } } const enum IgnoreBracketsInTokens { diff --git a/src/vs/editor/common/languages/supports/indentationLineProcessor.ts b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts new file mode 100644 index 00000000000..919cb3cd4c8 --- /dev/null +++ b/src/vs/editor/common/languages/supports/indentationLineProcessor.ts @@ -0,0 +1,236 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as strings from 'vs/base/common/strings'; +import { Range } from 'vs/editor/common/core/range'; +import { ITextModel } from 'vs/editor/common/model'; +import { ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry'; +import { createScopedLineTokens, ScopedLineTokens } from 'vs/editor/common/languages/supports'; +import { IVirtualModel } from 'vs/editor/common/languages/autoIndent'; +import { IViewLineTokens, LineTokens } from 'vs/editor/common/tokens/lineTokens'; +import { IndentRulesSupport } from 'vs/editor/common/languages/supports/indentRules'; +import { StandardTokenType } from 'vs/editor/common/encodedTokenAttributes'; +import { Position } from 'vs/editor/common/core/position'; + +/** + * This class is a wrapper class around {@link IndentRulesSupport}. + * It processes the lines by removing the language configuration brackets from the regex, string and comment tokens. + * It then calls into the {@link IndentRulesSupport} to validate the indentation conditions. + */ +export class ProcessedIndentRulesSupport { + + private readonly _indentRulesSupport: IndentRulesSupport; + private readonly _indentationLineProcessor: IndentationLineProcessor; + + constructor( + model: IVirtualModel, + indentRulesSupport: IndentRulesSupport, + languageConfigurationService: ILanguageConfigurationService + ) { + this._indentRulesSupport = indentRulesSupport; + this._indentationLineProcessor = new IndentationLineProcessor(model, languageConfigurationService); + } + + /** + * Apply the new indentation and return whether the indentation level should be increased after the given line number + */ + public shouldIncrease(lineNumber: number, newIndentation?: string): boolean { + const processedLine = this._indentationLineProcessor.getProcessedLine(lineNumber, newIndentation); + return this._indentRulesSupport.shouldIncrease(processedLine); + } + + /** + * Apply the new indentation and return whether the indentation level should be decreased after the given line number + */ + public shouldDecrease(lineNumber: number, newIndentation?: string): boolean { + const processedLine = this._indentationLineProcessor.getProcessedLine(lineNumber, newIndentation); + return this._indentRulesSupport.shouldDecrease(processedLine); + } + + /** + * Apply the new indentation and return whether the indentation level should remain unchanged at the given line number + */ + public shouldIgnore(lineNumber: number, newIndentation?: string): boolean { + const processedLine = this._indentationLineProcessor.getProcessedLine(lineNumber, newIndentation); + return this._indentRulesSupport.shouldIgnore(processedLine); + } + + /** + * Apply the new indentation and return whether the indentation level should increase on the line after the given line number + */ + public shouldIndentNextLine(lineNumber: number, newIndentation?: string): boolean { + const processedLine = this._indentationLineProcessor.getProcessedLine(lineNumber, newIndentation); + return this._indentRulesSupport.shouldIndentNextLine(processedLine); + } + +} + +/** + * This class fetches the processed text around a range which can be used for indentation evaluation. + * It returns: + * - The processed text before the given range and on the same start line + * - The processed text after the given range and on the same end line + * - The processed text on the previous line + */ +export class IndentationContextProcessor { + + private readonly model: ITextModel; + private readonly indentationLineProcessor: IndentationLineProcessor; + + constructor( + model: ITextModel, + languageConfigurationService: ILanguageConfigurationService + ) { + this.model = model; + this.indentationLineProcessor = new IndentationLineProcessor(model, languageConfigurationService); + } + + /** + * Returns the processed text, stripped from the language configuration brackets within the string, comment and regex tokens, around the given range + */ + getProcessedTokenContextAroundRange(range: Range): { + beforeRangeProcessedTokens: IViewLineTokens; + afterRangeProcessedTokens: IViewLineTokens; + previousLineProcessedTokens: IViewLineTokens; + } { + const beforeRangeProcessedTokens = this._getProcessedTokensBeforeRange(range); + const afterRangeProcessedTokens = this._getProcessedTokensAfterRange(range); + const previousLineProcessedTokens = this._getProcessedPreviousLineTokens(range); + return { beforeRangeProcessedTokens, afterRangeProcessedTokens, previousLineProcessedTokens }; + } + + private _getProcessedTokensBeforeRange(range: Range): IViewLineTokens { + this.model.tokenization.forceTokenization(range.startLineNumber); + const lineTokens = this.model.tokenization.getLineTokens(range.startLineNumber); + const scopedLineTokens = createScopedLineTokens(lineTokens, range.startColumn - 1); + let slicedTokens: IViewLineTokens; + if (isLanguageDifferentFromLineStart(this.model, range.getStartPosition())) { + const columnIndexWithinScope = (range.startColumn - 1) - scopedLineTokens.firstCharOffset; + const firstCharacterOffset = scopedLineTokens.firstCharOffset; + const lastCharacterOffset = firstCharacterOffset + columnIndexWithinScope; + slicedTokens = lineTokens.sliceAndInflate(firstCharacterOffset, lastCharacterOffset, 0); + } else { + const columnWithinLine = range.startColumn - 1; + slicedTokens = lineTokens.sliceAndInflate(0, columnWithinLine, 0); + } + const processedTokens = this.indentationLineProcessor.getProcessedTokens(slicedTokens); + return processedTokens; + } + + private _getProcessedTokensAfterRange(range: Range): IViewLineTokens { + const position: Position = range.isEmpty() ? range.getStartPosition() : range.getEndPosition(); + this.model.tokenization.forceTokenization(position.lineNumber); + const lineTokens = this.model.tokenization.getLineTokens(position.lineNumber); + const scopedLineTokens = createScopedLineTokens(lineTokens, position.column - 1); + const columnIndexWithinScope = position.column - 1 - scopedLineTokens.firstCharOffset; + const firstCharacterOffset = scopedLineTokens.firstCharOffset + columnIndexWithinScope; + const lastCharacterOffset = scopedLineTokens.firstCharOffset + scopedLineTokens.getLineLength(); + const slicedTokens = lineTokens.sliceAndInflate(firstCharacterOffset, lastCharacterOffset, 0); + const processedTokens = this.indentationLineProcessor.getProcessedTokens(slicedTokens); + return processedTokens; + } + + private _getProcessedPreviousLineTokens(range: Range): IViewLineTokens { + const getScopedLineTokensAtEndColumnOfLine = (lineNumber: number): ScopedLineTokens => { + this.model.tokenization.forceTokenization(lineNumber); + const lineTokens = this.model.tokenization.getLineTokens(lineNumber); + const endColumnOfLine = this.model.getLineMaxColumn(lineNumber) - 1; + const scopedLineTokensAtEndColumn = createScopedLineTokens(lineTokens, endColumnOfLine); + return scopedLineTokensAtEndColumn; + }; + + this.model.tokenization.forceTokenization(range.startLineNumber); + const lineTokens = this.model.tokenization.getLineTokens(range.startLineNumber); + const scopedLineTokens = createScopedLineTokens(lineTokens, range.startColumn - 1); + const emptyTokens = LineTokens.createEmpty('', scopedLineTokens.languageIdCodec); + const previousLineNumber = range.startLineNumber - 1; + const isFirstLine = previousLineNumber === 0; + if (isFirstLine) { + return emptyTokens; + } + const canScopeExtendOnPreviousLine = scopedLineTokens.firstCharOffset === 0; + if (!canScopeExtendOnPreviousLine) { + return emptyTokens; + } + const scopedLineTokensAtEndColumnOfPreviousLine = getScopedLineTokensAtEndColumnOfLine(previousLineNumber); + const doesLanguageContinueOnPreviousLine = scopedLineTokens.languageId === scopedLineTokensAtEndColumnOfPreviousLine.languageId; + if (!doesLanguageContinueOnPreviousLine) { + return emptyTokens; + } + const previousSlicedLineTokens = scopedLineTokensAtEndColumnOfPreviousLine.toIViewLineTokens(); + const processedTokens = this.indentationLineProcessor.getProcessedTokens(previousSlicedLineTokens); + return processedTokens; + } +} + +/** + * This class performs the actual processing of the indentation lines. + * The brackets of the language configuration are removed from the regex, string and comment tokens. + */ +class IndentationLineProcessor { + + constructor( + private readonly model: IVirtualModel, + private readonly languageConfigurationService: ILanguageConfigurationService + ) { } + + /** + * Get the processed line for the given line number and potentially adjust the indentation level. + * Remove the language configuration brackets from the regex, string and comment tokens. + */ + getProcessedLine(lineNumber: number, newIndentation?: string): string { + const replaceIndentation = (line: string, newIndentation: string): string => { + const currentIndentation = strings.getLeadingWhitespace(line); + const adjustedLine = newIndentation + line.substring(currentIndentation.length); + return adjustedLine; + }; + + this.model.tokenization.forceTokenization?.(lineNumber); + const tokens = this.model.tokenization.getLineTokens(lineNumber); + let processedLine = this.getProcessedTokens(tokens).getLineContent(); + if (newIndentation !== undefined) { + processedLine = replaceIndentation(processedLine, newIndentation); + } + return processedLine; + } + + /** + * Process the line with the given tokens, remove the language configuration brackets from the regex, string and comment tokens. + */ + getProcessedTokens(tokens: IViewLineTokens): IViewLineTokens { + + const shouldRemoveBracketsFromTokenType = (tokenType: StandardTokenType): boolean => { + return tokenType === StandardTokenType.String + || tokenType === StandardTokenType.RegEx + || tokenType === StandardTokenType.Comment; + }; + + const languageId = tokens.getLanguageId(0); + const bracketsConfiguration = this.languageConfigurationService.getLanguageConfiguration(languageId).bracketsNew; + const bracketsRegExp = bracketsConfiguration.getBracketRegExp({ global: true }); + const textAndMetadata: { text: string; metadata: number }[] = []; + tokens.forEach((tokenIndex: number) => { + const tokenType = tokens.getStandardTokenType(tokenIndex); + let text = tokens.getTokenText(tokenIndex); + if (shouldRemoveBracketsFromTokenType(tokenType)) { + text = text.replace(bracketsRegExp, ''); + } + const metadata = tokens.getMetadata(tokenIndex); + textAndMetadata.push({ text, metadata }); + }); + const processedLineTokens = LineTokens.createFromTextAndMetadata(textAndMetadata, tokens.languageIdCodec); + return processedLineTokens; + } +} + +export function isLanguageDifferentFromLineStart(model: ITextModel, position: Position): boolean { + model.tokenization.forceTokenization(position.lineNumber); + const lineTokens = model.tokenization.getLineTokens(position.lineNumber); + const scopedLineTokens = createScopedLineTokens(lineTokens, position.column - 1); + const doesScopeStartAtOffsetZero = scopedLineTokens.firstCharOffset === 0; + const isScopedLanguageEqualToFirstLanguageOnLine = lineTokens.getLanguageId(0) === scopedLineTokens.languageId; + const languageIsDifferentFromLineStart = !doesScopeStartAtOffsetZero && !isScopedLanguageEqualToFirstLanguageOnLine; + return languageIsDifferentFromLineStart; +} diff --git a/src/vs/editor/common/languages/supports/languageBracketsConfiguration.ts b/src/vs/editor/common/languages/supports/languageBracketsConfiguration.ts index a989e2f35e4..4989395b264 100644 --- a/src/vs/editor/common/languages/supports/languageBracketsConfiguration.ts +++ b/src/vs/editor/common/languages/supports/languageBracketsConfiguration.ts @@ -4,7 +4,9 @@ *--------------------------------------------------------------------------------------------*/ import { CachedFunction } from 'vs/base/common/cache'; +import { RegExpOptions } from 'vs/base/common/strings'; import { LanguageConfiguration } from 'vs/editor/common/languages/languageConfiguration'; +import { createBracketOrRegExp } from 'vs/editor/common/languages/supports/richEditBrackets'; /** * Captures all bracket related configurations for a single language. @@ -91,6 +93,11 @@ export class LanguageBracketsConfiguration { public getBracketInfo(bracketText: string): BracketKind | undefined { return this.getOpeningBracketInfo(bracketText) || this.getClosingBracketInfo(bracketText); } + + public getBracketRegExp(options?: RegExpOptions): RegExp { + const brackets = Array.from([...this._openingBrackets.keys(), ...this._closingBrackets.keys()]); + return createBracketOrRegExp(brackets, options); + } } function filterValidBrackets(bracketPairs: [string, string][]): [string, string][] { diff --git a/src/vs/editor/common/languages/supports/richEditBrackets.ts b/src/vs/editor/common/languages/supports/richEditBrackets.ts index c6efd4ee7a7..7733719f049 100644 --- a/src/vs/editor/common/languages/supports/richEditBrackets.ts +++ b/src/vs/editor/common/languages/supports/richEditBrackets.ts @@ -7,6 +7,7 @@ import * as strings from 'vs/base/common/strings'; import * as stringBuilder from 'vs/editor/common/core/stringBuilder'; import { Range } from 'vs/editor/common/core/range'; import { CharacterPair } from 'vs/editor/common/languages/languageConfiguration'; +import { RegExpOptions } from 'vs/base/common/strings'; interface InternalBracket { open: string[]; @@ -408,9 +409,9 @@ function prepareBracketForRegExp(str: string): string { return (insertWordBoundaries ? `\\b${str}\\b` : str); } -function createBracketOrRegExp(pieces: string[]): RegExp { +export function createBracketOrRegExp(pieces: string[], options?: RegExpOptions): RegExp { const regexStr = `(${pieces.map(prepareBracketForRegExp).join(')|(')})`; - return strings.createRegExp(regexStr, true); + return strings.createRegExp(regexStr, true, options); } const toReversedString = (function () { diff --git a/src/vs/editor/common/tokens/lineTokens.ts b/src/vs/editor/common/tokens/lineTokens.ts index bdcda7cb180..43777995d51 100644 --- a/src/vs/editor/common/tokens/lineTokens.ts +++ b/src/vs/editor/common/tokens/lineTokens.ts @@ -7,8 +7,10 @@ import { ILanguageIdCodec } from 'vs/editor/common/languages'; import { FontStyle, ColorId, StandardTokenType, MetadataConsts, TokenMetadata, ITokenPresentation } from 'vs/editor/common/encodedTokenAttributes'; export interface IViewLineTokens { + languageIdCodec: ILanguageIdCodec; equals(other: IViewLineTokens): boolean; getCount(): number; + getStandardTokenType(tokenIndex: number): StandardTokenType; getForeground(tokenIndex: number): ColorId; getEndOffset(tokenIndex: number): number; getClassName(tokenIndex: number): string; @@ -18,6 +20,8 @@ export interface IViewLineTokens { getLineContent(): string; getMetadata(tokenIndex: number): number; getLanguageId(tokenIndex: number): string; + getTokenText(tokenIndex: number): string; + forEach(callback: (tokenIndex: number) => void): void; } export class LineTokens implements IViewLineTokens { @@ -26,7 +30,8 @@ export class LineTokens implements IViewLineTokens { private readonly _tokens: Uint32Array; private readonly _tokensCount: number; private readonly _text: string; - private readonly _languageIdCodec: ILanguageIdCodec; + + public readonly languageIdCodec: ILanguageIdCodec; public static defaultTokenMetadata = ( (FontStyle.None << MetadataConsts.FONT_STYLE_OFFSET) @@ -44,11 +49,23 @@ export class LineTokens implements IViewLineTokens { return new LineTokens(tokens, lineContent, decoder); } + public static createFromTextAndMetadata(data: { text: string; metadata: number }[], decoder: ILanguageIdCodec): LineTokens { + let offset: number = 0; + let fullText: string = ''; + const tokens = new Array(); + for (const { text, metadata } of data) { + tokens.push(offset + text.length, metadata); + offset += text.length; + fullText += text; + } + return new LineTokens(new Uint32Array(tokens), fullText, decoder); + } + constructor(tokens: Uint32Array, text: string, decoder: ILanguageIdCodec) { this._tokens = tokens; this._tokensCount = (this._tokens.length >>> 1); this._text = text; - this._languageIdCodec = decoder; + this.languageIdCodec = decoder; } public equals(other: IViewLineTokens): boolean { @@ -98,7 +115,7 @@ export class LineTokens implements IViewLineTokens { public getLanguageId(tokenIndex: number): string { const metadata = this._tokens[(tokenIndex << 1) + 1]; const languageId = TokenMetadata.getLanguageId(metadata); - return this._languageIdCodec.decodeLanguageId(languageId); + return this.languageIdCodec.decodeLanguageId(languageId); } public getStandardTokenType(tokenIndex: number): StandardTokenType { @@ -225,7 +242,21 @@ export class LineTokens implements IViewLineTokens { } } - return new LineTokens(new Uint32Array(newTokens), text, this._languageIdCodec); + return new LineTokens(new Uint32Array(newTokens), text, this.languageIdCodec); + } + + public getTokenText(tokenIndex: number): string { + const startOffset = this.getStartOffset(tokenIndex); + const endOffset = this.getEndOffset(tokenIndex); + const text = this._text.substring(startOffset, endOffset); + return text; + } + + public forEach(callback: (tokenIndex: number) => void): void { + const tokenCount = this.getCount(); + for (let tokenIndex = 0; tokenIndex < tokenCount; tokenIndex++) { + callback(tokenIndex); + } } } @@ -239,12 +270,15 @@ class SliceLineTokens implements IViewLineTokens { private readonly _firstTokenIndex: number; private readonly _tokensCount: number; + public readonly languageIdCodec: ILanguageIdCodec; + constructor(source: LineTokens, startOffset: number, endOffset: number, deltaOffset: number) { this._source = source; this._startOffset = startOffset; this._endOffset = endOffset; this._deltaOffset = deltaOffset; this._firstTokenIndex = source.findTokenIndexAtOffset(startOffset); + this.languageIdCodec = source.languageIdCodec; this._tokensCount = 0; for (let i = this._firstTokenIndex, len = source.getCount(); i < len; i++) { @@ -284,6 +318,10 @@ class SliceLineTokens implements IViewLineTokens { return this._tokensCount; } + public getStandardTokenType(tokenIndex: number): StandardTokenType { + return this._source.getStandardTokenType(this._firstTokenIndex + tokenIndex); + } + public getForeground(tokenIndex: number): ColorId { return this._source.getForeground(this._firstTokenIndex + tokenIndex); } @@ -308,4 +346,24 @@ class SliceLineTokens implements IViewLineTokens { public findTokenIndexAtOffset(offset: number): number { return this._source.findTokenIndexAtOffset(offset + this._startOffset - this._deltaOffset) - this._firstTokenIndex; } + + public getTokenText(tokenIndex: number): string { + const adjustedTokenIndex = this._firstTokenIndex + tokenIndex; + const tokenStartOffset = this._source.getStartOffset(adjustedTokenIndex); + const tokenEndOffset = this._source.getEndOffset(adjustedTokenIndex); + let text = this._source.getTokenText(adjustedTokenIndex); + if (tokenStartOffset < this._startOffset) { + text = text.substring(this._startOffset - tokenStartOffset); + } + if (tokenEndOffset > this._endOffset) { + text = text.substring(0, text.length - (tokenEndOffset - this._endOffset)); + } + return text; + } + + public forEach(callback: (tokenIndex: number) => void): void { + for (let tokenIndex = 0; tokenIndex < this.getCount(); tokenIndex++) { + callback(tokenIndex); + } + } } diff --git a/src/vs/editor/contrib/indentation/common/indentation.ts b/src/vs/editor/contrib/indentation/common/indentation.ts index 760a14919fb..41cc7cb8068 100644 --- a/src/vs/editor/contrib/indentation/common/indentation.ts +++ b/src/vs/editor/contrib/indentation/common/indentation.ts @@ -9,29 +9,26 @@ import { EditOperation, ISingleEditOperation } from 'vs/editor/common/core/editO import { normalizeIndentation } from 'vs/editor/common/core/indentation'; import { Selection } from 'vs/editor/common/core/selection'; import { ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry'; +import { ProcessedIndentRulesSupport } from 'vs/editor/common/languages/supports/indentationLineProcessor'; import { ITextModel } from 'vs/editor/common/model'; -export function getReindentEditOperations(model: ITextModel, languageConfigurationService: ILanguageConfigurationService, startLineNumber: number, endLineNumber: number, inheritedIndent?: string): ISingleEditOperation[] { +export function getReindentEditOperations(model: ITextModel, languageConfigurationService: ILanguageConfigurationService, startLineNumber: number, endLineNumber: number): ISingleEditOperation[] { if (model.getLineCount() === 1 && model.getLineMaxColumn(1) === 1) { // Model is empty return []; } - const indentationRules = languageConfigurationService.getLanguageConfiguration(model.getLanguageId()).indentationRules; - if (!indentationRules) { + const indentationRulesSupport = languageConfigurationService.getLanguageConfiguration(model.getLanguageId()).indentRulesSupport; + if (!indentationRulesSupport) { return []; } + const processedIndentRulesSupport = new ProcessedIndentRulesSupport(model, indentationRulesSupport, languageConfigurationService); endLineNumber = Math.min(endLineNumber, model.getLineCount()); // Skip `unIndentedLinePattern` lines while (startLineNumber <= endLineNumber) { - if (!indentationRules.unIndentedLinePattern) { - break; - } - - const text = model.getLineContent(startLineNumber); - if (!indentationRules.unIndentedLinePattern.test(text)) { + if (!processedIndentRulesSupport.shouldIgnore(startLineNumber)) { break; } @@ -54,37 +51,19 @@ export function getReindentEditOperations(model: ITextModel, languageConfigurati const indentEdits: ISingleEditOperation[] = []; // indentation being passed to lines below - let globalIndent: string; // Calculate indentation for the first line // If there is no passed-in indentation, we use the indentation of the first line as base. const currentLineText = model.getLineContent(startLineNumber); - let adjustedLineContent = currentLineText; - if (inheritedIndent !== undefined && inheritedIndent !== null) { - globalIndent = inheritedIndent; - const oldIndentation = strings.getLeadingWhitespace(currentLineText); - - adjustedLineContent = globalIndent + currentLineText.substring(oldIndentation.length); - if (indentationRules.decreaseIndentPattern && indentationRules.decreaseIndentPattern.test(adjustedLineContent)) { - globalIndent = unshiftIndent(globalIndent); - adjustedLineContent = globalIndent + currentLineText.substring(oldIndentation.length); - - } - if (currentLineText !== adjustedLineContent) { - indentEdits.push(EditOperation.replaceMove(new Selection(startLineNumber, 1, startLineNumber, oldIndentation.length + 1), normalizeIndentation(globalIndent, indentSize, insertSpaces))); - } - } else { - globalIndent = strings.getLeadingWhitespace(currentLineText); - } - + let globalIndent = strings.getLeadingWhitespace(currentLineText); // idealIndentForNextLine doesn't equal globalIndent when there is a line matching `indentNextLinePattern`. let idealIndentForNextLine: string = globalIndent; - if (indentationRules.increaseIndentPattern && indentationRules.increaseIndentPattern.test(adjustedLineContent)) { + if (processedIndentRulesSupport.shouldIncrease(startLineNumber)) { idealIndentForNextLine = shiftIndent(idealIndentForNextLine); globalIndent = shiftIndent(globalIndent); } - else if (indentationRules.indentNextLinePattern && indentationRules.indentNextLinePattern.test(adjustedLineContent)) { + else if (processedIndentRulesSupport.shouldIndentNextLine(startLineNumber)) { idealIndentForNextLine = shiftIndent(idealIndentForNextLine); } @@ -94,9 +73,9 @@ export function getReindentEditOperations(model: ITextModel, languageConfigurati for (let lineNumber = startLineNumber; lineNumber <= endLineNumber; lineNumber++) { const text = model.getLineContent(lineNumber); const oldIndentation = strings.getLeadingWhitespace(text); - const adjustedLineContent = idealIndentForNextLine + text.substring(oldIndentation.length); + const currentIdealIndent = idealIndentForNextLine; - if (indentationRules.decreaseIndentPattern && indentationRules.decreaseIndentPattern.test(adjustedLineContent)) { + if (processedIndentRulesSupport.shouldDecrease(lineNumber, currentIdealIndent)) { idealIndentForNextLine = unshiftIndent(idealIndentForNextLine); globalIndent = unshiftIndent(globalIndent); } @@ -106,14 +85,14 @@ export function getReindentEditOperations(model: ITextModel, languageConfigurati } // calculate idealIndentForNextLine - if (indentationRules.unIndentedLinePattern && indentationRules.unIndentedLinePattern.test(text)) { + if (processedIndentRulesSupport.shouldIgnore(lineNumber)) { // In reindent phase, if the line matches `unIndentedLinePattern` we inherit indentation from above lines // but don't change globalIndent and idealIndentForNextLine. continue; - } else if (indentationRules.increaseIndentPattern && indentationRules.increaseIndentPattern.test(adjustedLineContent)) { + } else if (processedIndentRulesSupport.shouldIncrease(lineNumber, currentIdealIndent)) { globalIndent = shiftIndent(globalIndent); idealIndentForNextLine = globalIndent; - } else if (indentationRules.indentNextLinePattern && indentationRules.indentNextLinePattern.test(adjustedLineContent)) { + } else if (processedIndentRulesSupport.shouldIndentNextLine(lineNumber, currentIdealIndent)) { idealIndentForNextLine = shiftIndent(idealIndentForNextLine); } else { idealIndentForNextLine = globalIndent; diff --git a/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts b/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts index 72350c3ed69..178e1db4061 100644 --- a/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts +++ b/src/vs/editor/contrib/indentation/test/browser/indentation.test.ts @@ -24,7 +24,7 @@ import { TypeOperations } from 'vs/editor/common/cursor/cursorTypeOperations'; import { cppBracketRules, goBracketRules, htmlBracketRules, latexBracketRules, luaBracketRules, phpBracketRules, rubyBracketRules, typescriptBracketRules, vbBracketRules } from 'vs/editor/test/common/modes/supports/bracketRules'; import { latexAutoClosingPairsRules } from 'vs/editor/test/common/modes/supports/autoClosingPairsRules'; -enum Language { +export enum Language { TypeScript = 'ts-test', Ruby = 'ruby-test', PHP = 'php-test', @@ -44,7 +44,7 @@ function testIndentationToTabsCommand(lines: string[], selection: Selection, tab testCommand(lines, null, selection, (accessor, sel) => new IndentationToTabsCommand(sel, tabSize), expectedLines, expectedSelection); } -function registerLanguage(instantiationService: TestInstantiationService, language: Language): IDisposable { +export function registerLanguage(instantiationService: TestInstantiationService, language: Language): IDisposable { const disposables = new DisposableStore(); const languageService = instantiationService.get(ILanguageService); disposables.add(registerLanguageConfiguration(instantiationService, language)); @@ -52,7 +52,7 @@ function registerLanguage(instantiationService: TestInstantiationService, langua return disposables; } -function registerLanguageConfiguration(instantiationService: TestInstantiationService, language: Language): IDisposable { +export function registerLanguageConfiguration(instantiationService: TestInstantiationService, language: Language): IDisposable { const languageConfigurationService = instantiationService.get(ILanguageConfigurationService); switch (language) { case Language.TypeScript: @@ -110,12 +110,12 @@ function registerLanguageConfiguration(instantiationService: TestInstantiationSe } } -interface StandardTokenTypeData { +export interface StandardTokenTypeData { startIndex: number; standardTokenType: StandardTokenType; } -function registerTokenizationSupport(instantiationService: TestInstantiationService, tokens: StandardTokenTypeData[][], languageId: string): IDisposable { +export function registerTokenizationSupport(instantiationService: TestInstantiationService, tokens: StandardTokenTypeData[][], languageId: string): IDisposable { let lineIndex = 0; const languageService = instantiationService.get(ILanguageService); const tokenizationSupport: ITokenizationSupport = { @@ -1007,9 +1007,7 @@ suite('Auto Indent On Type - TypeScript/JavaScript', () => { }); }); - // Failing tests... - - test.skip('issue #208232: incorrect indentation inside of comments', () => { + test('issue #208232: incorrect indentation inside of comments', () => { // https://github.com/microsoft/vscode/issues/208232 @@ -1023,6 +1021,12 @@ suite('Auto Indent On Type - TypeScript/JavaScript', () => { withTestCodeEditor(model, { autoIndent: "full" }, (editor, viewModel, instantiationService) => { disposables.add(registerLanguage(instantiationService, languageId)); + const tokens: StandardTokenTypeData[][] = [ + [{ startIndex: 0, standardTokenType: StandardTokenType.Comment }], + [{ startIndex: 0, standardTokenType: StandardTokenType.Comment }], + [{ startIndex: 0, standardTokenType: StandardTokenType.Comment }] + ]; + disposables.add(registerTokenizationSupport(instantiationService, tokens, languageId)); editor.setSelection(new Selection(2, 23, 2, 23)); viewModel.type("\n", 'keyboard'); assert.strictEqual(model.getValue(), [ @@ -1034,6 +1038,8 @@ suite('Auto Indent On Type - TypeScript/JavaScript', () => { }); }); + // Failing tests... + test.skip('issue #43244: indent after equal sign is detected', () => { // https://github.com/microsoft/vscode/issues/43244 @@ -1398,24 +1404,28 @@ suite('Auto Indent On Type - PHP', () => { ensureNoDisposablesAreLeakedInTestSuite(); - test('temp issue because there should be at least one passing test in a suite', () => { - assert.ok(true); - }); - - test.skip('issue #199050: should not indent after { detected in a string', () => { + test('issue #199050: should not indent after { detected in a string', () => { // https://github.com/microsoft/vscode/issues/199050 - const model = createTextModel("$phrase = preg_replace('#(\{1|%s).*#su', '', $phrase);", languageId, {}); + const model = createTextModel("preg_replace('{');", languageId, {}); disposables.add(model); withTestCodeEditor(model, { autoIndent: "full" }, (editor, viewModel, instantiationService) => { disposables.add(registerLanguage(instantiationService, languageId)); + const tokens: StandardTokenTypeData[][] = [ + [ + { startIndex: 0, standardTokenType: StandardTokenType.Other }, + { startIndex: 13, standardTokenType: StandardTokenType.String }, + { startIndex: 16, standardTokenType: StandardTokenType.Other }, + ] + ]; + disposables.add(registerTokenizationSupport(instantiationService, tokens, languageId)); editor.setSelection(new Selection(1, 54, 1, 54)); viewModel.type("\n", 'keyboard'); assert.strictEqual(model.getValue(), [ - "$phrase = preg_replace('#(\{1|%s).*#su', '', $phrase);", + "preg_replace('{');", "" ].join('\n')); }); diff --git a/src/vs/editor/contrib/indentation/test/browser/indentationLineProcessor.test.ts b/src/vs/editor/contrib/indentation/test/browser/indentationLineProcessor.test.ts new file mode 100644 index 00000000000..77afbe5c76d --- /dev/null +++ b/src/vs/editor/contrib/indentation/test/browser/indentationLineProcessor.test.ts @@ -0,0 +1,236 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +import * as assert from 'assert'; +import { DisposableStore } from 'vs/base/common/lifecycle'; +import { ensureNoDisposablesAreLeakedInTestSuite } from 'vs/base/test/common/utils'; +import { StandardTokenType } from 'vs/editor/common/encodedTokenAttributes'; +import { ILanguageConfigurationService } from 'vs/editor/common/languages/languageConfigurationRegistry'; +import { IndentationContextProcessor, ProcessedIndentRulesSupport } from 'vs/editor/common/languages/supports/indentationLineProcessor'; +import { Language, registerLanguage, registerTokenizationSupport, StandardTokenTypeData } from 'vs/editor/contrib/indentation/test/browser/indentation.test'; +import { withTestCodeEditor } from 'vs/editor/test/browser/testCodeEditor'; +import { createTextModel } from 'vs/editor/test/common/testTextModel'; +import { Range } from 'vs/editor/common/core/range'; + +suite('Indentation Context Processor - TypeScript/JavaScript', () => { + + const languageId = Language.TypeScript; + let disposables: DisposableStore; + + setup(() => { + disposables = new DisposableStore(); + }); + + teardown(() => { + disposables.dispose(); + }); + + ensureNoDisposablesAreLeakedInTestSuite(); + + test('brackets inside of string', () => { + + const model = createTextModel([ + 'const someVar = "{some text}"', + ].join('\n'), languageId, {}); + disposables.add(model); + + withTestCodeEditor(model, { autoIndent: "full" }, (editor, viewModel, instantiationService) => { + const tokens: StandardTokenTypeData[][] = [[ + { startIndex: 0, standardTokenType: StandardTokenType.Other }, + { startIndex: 16, standardTokenType: StandardTokenType.String }, + { startIndex: 28, standardTokenType: StandardTokenType.String } + ]]; + disposables.add(registerLanguage(instantiationService, languageId)); + disposables.add(registerTokenizationSupport(instantiationService, tokens, languageId)); + const languageConfigurationService = instantiationService.get(ILanguageConfigurationService); + const indentationContextProcessor = new IndentationContextProcessor(model, languageConfigurationService); + const processedContext = indentationContextProcessor.getProcessedTokenContextAroundRange(new Range(1, 23, 1, 23)); + assert.strictEqual(processedContext.beforeRangeProcessedTokens.getLineContent(), 'const someVar = "some'); + assert.strictEqual(processedContext.afterRangeProcessedTokens.getLineContent(), ' text"'); + assert.strictEqual(processedContext.previousLineProcessedTokens.getLineContent(), ''); + }); + }); + + test('brackets inside of comment', () => { + + const model = createTextModel([ + 'const someVar2 = /*(a])*/', + 'const someVar = /* [()] some other t{e}xt() */ "some text"', + ].join('\n'), languageId, {}); + disposables.add(model); + + withTestCodeEditor(model, { autoIndent: "full" }, (editor, viewModel, instantiationService) => { + const tokens: StandardTokenTypeData[][] = [ + [ + { startIndex: 0, standardTokenType: StandardTokenType.Other }, + { startIndex: 17, standardTokenType: StandardTokenType.Comment }, + ], + [ + { startIndex: 0, standardTokenType: StandardTokenType.Other }, + { startIndex: 16, standardTokenType: StandardTokenType.Comment }, + { startIndex: 46, standardTokenType: StandardTokenType.Other }, + { startIndex: 47, standardTokenType: StandardTokenType.String } + ]]; + disposables.add(registerLanguage(instantiationService, languageId)); + disposables.add(registerTokenizationSupport(instantiationService, tokens, languageId)); + const languageConfigurationService = instantiationService.get(ILanguageConfigurationService); + const indentationContextProcessor = new IndentationContextProcessor(model, languageConfigurationService); + const processedContext = indentationContextProcessor.getProcessedTokenContextAroundRange(new Range(2, 29, 2, 35)); + assert.strictEqual(processedContext.beforeRangeProcessedTokens.getLineContent(), 'const someVar = /* some'); + assert.strictEqual(processedContext.afterRangeProcessedTokens.getLineContent(), ' text */ "some text"'); + assert.strictEqual(processedContext.previousLineProcessedTokens.getLineContent(), 'const someVar2 = /*a*/'); + }); + }); + + test('brackets inside of regex', () => { + + const model = createTextModel([ + 'const someRegex2 = /(()))]/;', + 'const someRegex = /()a{h}{s}[(a}87(9a9()))]/;', + ].join('\n'), languageId, {}); + disposables.add(model); + + withTestCodeEditor(model, { autoIndent: "full" }, (editor, viewModel, instantiationService) => { + const tokens: StandardTokenTypeData[][] = [ + [ + { startIndex: 0, standardTokenType: StandardTokenType.Other }, + { startIndex: 19, standardTokenType: StandardTokenType.RegEx }, + { startIndex: 27, standardTokenType: StandardTokenType.Other }, + ], + [ + { startIndex: 0, standardTokenType: StandardTokenType.Other }, + { startIndex: 18, standardTokenType: StandardTokenType.RegEx }, + { startIndex: 44, standardTokenType: StandardTokenType.Other }, + ] + ]; + disposables.add(registerLanguage(instantiationService, languageId)); + disposables.add(registerTokenizationSupport(instantiationService, tokens, languageId)); + const languageConfigurationService = instantiationService.get(ILanguageConfigurationService); + const indentationContextProcessor = new IndentationContextProcessor(model, languageConfigurationService); + const processedContext = indentationContextProcessor.getProcessedTokenContextAroundRange(new Range(1, 25, 2, 33)); + assert.strictEqual(processedContext.beforeRangeProcessedTokens.getLineContent(), 'const someRegex2 = /'); + assert.strictEqual(processedContext.afterRangeProcessedTokens.getLineContent(), '879a9/;'); + assert.strictEqual(processedContext.previousLineProcessedTokens.getLineContent(), ''); + }); + }); +}); + +suite('Processed Indent Rules Support - TypeScript/JavaScript', () => { + + const languageId = Language.TypeScript; + let disposables: DisposableStore; + + setup(() => { + disposables = new DisposableStore(); + }); + + teardown(() => { + disposables.dispose(); + }); + + ensureNoDisposablesAreLeakedInTestSuite(); + + test('should increase', () => { + + const model = createTextModel([ + 'const someVar = {', + 'const someVar2 = "{"', + 'const someVar3 = /*{*/' + ].join('\n'), languageId, {}); + disposables.add(model); + + withTestCodeEditor(model, { autoIndent: "full" }, (editor, viewModel, instantiationService) => { + const tokens: StandardTokenTypeData[][] = [ + [ + { startIndex: 0, standardTokenType: StandardTokenType.Other } + ], + [ + { startIndex: 0, standardTokenType: StandardTokenType.Other }, + { startIndex: 17, standardTokenType: StandardTokenType.String }, + ], + [ + { startIndex: 0, standardTokenType: StandardTokenType.Other }, + { startIndex: 17, standardTokenType: StandardTokenType.Comment }, + ] + ]; + disposables.add(registerLanguage(instantiationService, languageId)); + disposables.add(registerTokenizationSupport(instantiationService, tokens, languageId)); + const languageConfigurationService = instantiationService.get(ILanguageConfigurationService); + const indentationRulesSupport = languageConfigurationService.getLanguageConfiguration(languageId).indentRulesSupport; + if (!indentationRulesSupport) { + assert.fail('indentationRulesSupport should be defined'); + } + const processedIndentRulesSupport = new ProcessedIndentRulesSupport(model, indentationRulesSupport, languageConfigurationService); + assert.strictEqual(processedIndentRulesSupport.shouldIncrease(1), true); + assert.strictEqual(processedIndentRulesSupport.shouldIncrease(2), false); + assert.strictEqual(processedIndentRulesSupport.shouldIncrease(3), false); + }); + }); + + test('should decrease', () => { + + const model = createTextModel([ + '}', + '"])some text}"', + '])*/' + ].join('\n'), languageId, {}); + disposables.add(model); + + withTestCodeEditor(model, { autoIndent: "full" }, (editor, viewModel, instantiationService) => { + const tokens: StandardTokenTypeData[][] = [ + [{ startIndex: 0, standardTokenType: StandardTokenType.Other }], + [{ startIndex: 0, standardTokenType: StandardTokenType.String }], + [{ startIndex: 0, standardTokenType: StandardTokenType.Comment }] + ]; + disposables.add(registerLanguage(instantiationService, languageId)); + disposables.add(registerTokenizationSupport(instantiationService, tokens, languageId)); + const languageConfigurationService = instantiationService.get(ILanguageConfigurationService); + const indentationRulesSupport = languageConfigurationService.getLanguageConfiguration(languageId).indentRulesSupport; + if (!indentationRulesSupport) { + assert.fail('indentationRulesSupport should be defined'); + } + const processedIndentRulesSupport = new ProcessedIndentRulesSupport(model, indentationRulesSupport, languageConfigurationService); + assert.strictEqual(processedIndentRulesSupport.shouldDecrease(1), true); + assert.strictEqual(processedIndentRulesSupport.shouldDecrease(2), false); + assert.strictEqual(processedIndentRulesSupport.shouldDecrease(3), false); + }); + }); + + test('should increase next line', () => { + + const model = createTextModel([ + 'if()', + 'const someString = "if()"', + 'const someRegex = /if()/' + ].join('\n'), languageId, {}); + disposables.add(model); + + withTestCodeEditor(model, { autoIndent: "full" }, (editor, viewModel, instantiationService) => { + const tokens: StandardTokenTypeData[][] = [ + [ + { startIndex: 0, standardTokenType: StandardTokenType.Other } + ], + [ + { startIndex: 0, standardTokenType: StandardTokenType.Other }, + { startIndex: 19, standardTokenType: StandardTokenType.String } + ], + [ + { startIndex: 0, standardTokenType: StandardTokenType.Other }, + { startIndex: 18, standardTokenType: StandardTokenType.RegEx } + ] + ]; + disposables.add(registerLanguage(instantiationService, languageId)); + disposables.add(registerTokenizationSupport(instantiationService, tokens, languageId)); + const languageConfigurationService = instantiationService.get(ILanguageConfigurationService); + const indentationRulesSupport = languageConfigurationService.getLanguageConfiguration(languageId).indentRulesSupport; + if (!indentationRulesSupport) { + assert.fail('indentationRulesSupport should be defined'); + } + const processedIndentRulesSupport = new ProcessedIndentRulesSupport(model, indentationRulesSupport, languageConfigurationService); + assert.strictEqual(processedIndentRulesSupport.shouldIndentNextLine(1), true); + assert.strictEqual(processedIndentRulesSupport.shouldIndentNextLine(2), false); + assert.strictEqual(processedIndentRulesSupport.shouldIndentNextLine(3), false); + }); + }); +}); diff --git a/src/vs/editor/test/common/core/testLineToken.ts b/src/vs/editor/test/common/core/testLineToken.ts index 8217c2042fb..f3c49807941 100644 --- a/src/vs/editor/test/common/core/testLineToken.ts +++ b/src/vs/editor/test/common/core/testLineToken.ts @@ -4,7 +4,8 @@ *--------------------------------------------------------------------------------------------*/ import { IViewLineTokens } from 'vs/editor/common/tokens/lineTokens'; -import { ColorId, TokenMetadata, ITokenPresentation } from 'vs/editor/common/encodedTokenAttributes'; +import { ColorId, TokenMetadata, ITokenPresentation, StandardTokenType } from 'vs/editor/common/encodedTokenAttributes'; +import { ILanguageIdCodec } from 'vs/editor/common/languages'; /** * A token on a line. @@ -22,6 +23,10 @@ export class TestLineToken { this._metadata = metadata; } + public getStandardTokenType(): StandardTokenType { + return TokenMetadata.getTokenType(this._metadata); + } + public getForeground(): ColorId { return TokenMetadata.getForeground(this._metadata); } @@ -79,6 +84,10 @@ export class TestLineTokens implements IViewLineTokens { return this._actual.length; } + public getStandardTokenType(tokenIndex: number): StandardTokenType { + return this._actual[tokenIndex].getStandardTokenType(); + } + public getForeground(tokenIndex: number): ColorId { return this._actual[tokenIndex].getForeground(); } @@ -114,6 +123,18 @@ export class TestLineTokens implements IViewLineTokens { public getLanguageId(tokenIndex: number): string { throw new Error('Method not implemented.'); } + + public getTokenText(tokenIndex: number): string { + throw new Error('Method not implemented.'); + } + + public forEach(callback: (tokenIndex: number) => void): void { + throw new Error('Not implemented'); + } + + public get languageIdCodec(): ILanguageIdCodec { + throw new Error('Not implemented'); + } } export class TestLineTokenFactory {