mirror of
https://github.com/Microsoft/vscode
synced 2024-08-27 13:00:58 +00:00
Refactors TextModelTokenization into GrammarTokenization.
This commit is contained in:
Henning Dieterichs
parent
9dde0d8202
commit
4ff4e12242
|
|
@ -1928,9 +1928,6 @@ export class TextModel extends Disposable implements model.ITextModel, IDecorati
|
|||
public getLanguageIdAtPosition(lineNumber: number, column: number): string {
|
||||
return this.tokenization.getLanguageIdAtPosition(lineNumber, column);
|
||||
}
|
||||
public setLineTokens(lineNumber: number, tokens: Uint32Array | ArrayBuffer | null): void {
|
||||
this._tokenizationTextModelPart.setLineTokens(lineNumber, tokens);
|
||||
}
|
||||
|
||||
public getWordAtPosition(position: IPosition): IWordAtPosition | null {
|
||||
return this._tokenizationTextModelPart.getWordAtPosition(position);
|
||||
|
|
|
|||
|
|
@ -4,8 +4,7 @@
|
|||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
import { IdleDeadline, runWhenIdle } from 'vs/base/common/async';
|
||||
import { BugIndicatingError, onUnexpectedError } from 'vs/base/common/errors';
|
||||
import { Disposable, MutableDisposable } from 'vs/base/common/lifecycle';
|
||||
import { onUnexpectedError } from 'vs/base/common/errors';
|
||||
import { setTimeout0 } from 'vs/base/common/platform';
|
||||
import { StopWatch } from 'vs/base/common/stopwatch';
|
||||
import { countEOL } from 'vs/editor/common/core/eolCounter';
|
||||
|
|
@ -13,13 +12,11 @@ import { LineRange } from 'vs/editor/common/core/lineRange';
|
|||
import { OffsetRange } from 'vs/editor/common/core/offsetRange';
|
||||
import { Position } from 'vs/editor/common/core/position';
|
||||
import { StandardTokenType } from 'vs/editor/common/encodedTokenAttributes';
|
||||
import { EncodedTokenizationResult, IBackgroundTokenizationStore, IBackgroundTokenizer, ILanguageIdCodec, IState, ITokenizationSupport, TokenizationRegistry } from 'vs/editor/common/languages';
|
||||
import { EncodedTokenizationResult, IBackgroundTokenizationStore, IBackgroundTokenizer, ILanguageIdCodec, IState, ITokenizationSupport } from 'vs/editor/common/languages';
|
||||
import { nullTokenizeEncoded } from 'vs/editor/common/languages/nullTokenize';
|
||||
import { ITextModel } from 'vs/editor/common/model';
|
||||
import { FixedArray } from 'vs/editor/common/model/fixedArray';
|
||||
import { TextModel } from 'vs/editor/common/model/textModel';
|
||||
import { TokenizationTextModelPart } from 'vs/editor/common/model/tokenizationTextModelPart';
|
||||
import { IModelContentChange, IModelContentChangedEvent, IModelLanguageChangedEvent } from 'vs/editor/common/textModelEvents';
|
||||
import { IModelContentChange } from 'vs/editor/common/textModelEvents';
|
||||
import { ContiguousMultilineTokensBuilder } from 'vs/editor/common/tokens/contiguousMultilineTokensBuilder';
|
||||
import { LineTokens } from 'vs/editor/common/tokens/lineTokens';
|
||||
|
||||
|
|
@ -27,266 +24,6 @@ const enum Constants {
|
|||
CHEAP_TOKENIZATION_LENGTH_LIMIT = 2048
|
||||
}
|
||||
|
||||
export class TextModelTokenization extends Disposable {
|
||||
private _tokenizationStateStore: TokenizerWithStateStore | null = null;
|
||||
private _defaultBackgroundTokenizer: DefaultBackgroundTokenizer | null = null;
|
||||
|
||||
private readonly backgroundTokenizer = this._register(new MutableDisposable<IBackgroundTokenizer>());
|
||||
|
||||
constructor(
|
||||
private readonly _textModel: TextModel,
|
||||
private readonly _tokenizationPart: TokenizationTextModelPart,
|
||||
private readonly _languageIdCodec: ILanguageIdCodec
|
||||
) {
|
||||
super();
|
||||
|
||||
this._register(TokenizationRegistry.onDidChange((e) => {
|
||||
const languageId = this._textModel.getLanguageId();
|
||||
if (e.changedLanguages.indexOf(languageId) === -1) {
|
||||
return;
|
||||
}
|
||||
|
||||
this._resetTokenizationState();
|
||||
this._tokenizationPart.clearTokens();
|
||||
}));
|
||||
|
||||
this._resetTokenizationState();
|
||||
}
|
||||
|
||||
public handleDidChangeContent(e: IModelContentChangedEvent): void {
|
||||
if (e.isFlush) {
|
||||
this._resetTokenizationState();
|
||||
} else if (!e.isEolChange) {
|
||||
if (this._tokenizationStateStore) {
|
||||
this._tokenizationStateStore.store.acceptChanges(e.changes);
|
||||
}
|
||||
this._defaultBackgroundTokenizer?.handleChanges();
|
||||
}
|
||||
}
|
||||
|
||||
public handleDidChangeAttached(): void {
|
||||
this._defaultBackgroundTokenizer?.handleChanges();
|
||||
}
|
||||
|
||||
public handleDidChangeLanguage(e: IModelLanguageChangedEvent): void {
|
||||
this._resetTokenizationState();
|
||||
this._tokenizationPart.clearTokens();
|
||||
}
|
||||
|
||||
private _resetTokenizationState(): void {
|
||||
const [tokenizationSupport, initialState] = initializeTokenization(this._textModel, this._tokenizationPart);
|
||||
if (tokenizationSupport && initialState) {
|
||||
this._tokenizationStateStore = new TokenizerWithStateStore(this._textModel.getLineCount(), tokenizationSupport);
|
||||
} else {
|
||||
this._tokenizationStateStore = null;
|
||||
}
|
||||
|
||||
this.backgroundTokenizer.clear();
|
||||
|
||||
this._defaultBackgroundTokenizer = null;
|
||||
if (this._tokenizationStateStore) {
|
||||
const b: IBackgroundTokenizationStore = {
|
||||
setTokens: (tokens) => {
|
||||
this._tokenizationPart.setTokens(tokens);
|
||||
},
|
||||
backgroundTokenizationFinished: () => {
|
||||
this._tokenizationPart.handleBackgroundTokenizationFinished();
|
||||
},
|
||||
setEndState: (lineNumber, state) => {
|
||||
if (!state) {
|
||||
throw new BugIndicatingError();
|
||||
}
|
||||
const firstInvalidEndStateLineNumber = this._tokenizationStateStore?.store.getFirstInvalidEndStateLineNumber() ?? undefined;
|
||||
if (firstInvalidEndStateLineNumber !== undefined && lineNumber >= firstInvalidEndStateLineNumber) {
|
||||
// Don't accept states for definitely valid states
|
||||
this._tokenizationStateStore?.store.setEndState(lineNumber, state);
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
if (tokenizationSupport && tokenizationSupport.createBackgroundTokenizer) {
|
||||
this.backgroundTokenizer.value = tokenizationSupport.createBackgroundTokenizer(this._textModel, b);
|
||||
}
|
||||
if (!this.backgroundTokenizer.value) {
|
||||
this.backgroundTokenizer.value = this._defaultBackgroundTokenizer =
|
||||
new DefaultBackgroundTokenizer(
|
||||
this._textModel,
|
||||
this._tokenizationStateStore,
|
||||
b,
|
||||
this._languageIdCodec
|
||||
);
|
||||
this._defaultBackgroundTokenizer.handleChanges();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public tokenizeViewport(startLineNumber: number, endLineNumber: number): void {
|
||||
const builder = new ContiguousMultilineTokensBuilder();
|
||||
this._heuristicallyTokenizeViewport(builder, startLineNumber, endLineNumber);
|
||||
this._tokenizationPart.setTokens(builder.finalize());
|
||||
this._defaultBackgroundTokenizer?.checkFinished();
|
||||
}
|
||||
|
||||
public reset(): void {
|
||||
this._resetTokenizationState();
|
||||
this._tokenizationPart.clearTokens();
|
||||
}
|
||||
|
||||
public forceTokenization(lineNumber: number): void {
|
||||
const builder = new ContiguousMultilineTokensBuilder();
|
||||
this._tokenizationStateStore?.updateTokensUntilLine(this._textModel, this._languageIdCodec, builder, lineNumber);
|
||||
this._tokenizationPart.setTokens(builder.finalize());
|
||||
this._defaultBackgroundTokenizer?.checkFinished();
|
||||
}
|
||||
|
||||
public getTokenTypeIfInsertingCharacter(position: Position, character: string): StandardTokenType {
|
||||
if (!this._tokenizationStateStore) {
|
||||
return StandardTokenType.Other;
|
||||
}
|
||||
|
||||
this.forceTokenization(position.lineNumber);
|
||||
const lineStartState = this._tokenizationStateStore.getStartState(position.lineNumber);
|
||||
if (!lineStartState) {
|
||||
return StandardTokenType.Other;
|
||||
}
|
||||
|
||||
const languageId = this._textModel.getLanguageId();
|
||||
const lineContent = this._textModel.getLineContent(position.lineNumber);
|
||||
|
||||
// Create the text as if `character` was inserted
|
||||
const text = (
|
||||
lineContent.substring(0, position.column - 1)
|
||||
+ character
|
||||
+ lineContent.substring(position.column - 1)
|
||||
);
|
||||
|
||||
const r = safeTokenize(this._languageIdCodec, languageId, this._tokenizationStateStore.tokenizationSupport, text, true, lineStartState);
|
||||
const lineTokens = new LineTokens(r.tokens, text, this._languageIdCodec);
|
||||
if (lineTokens.getCount() === 0) {
|
||||
return StandardTokenType.Other;
|
||||
}
|
||||
|
||||
const tokenIndex = lineTokens.findTokenIndexAtOffset(position.column - 1);
|
||||
return lineTokens.getStandardTokenType(tokenIndex);
|
||||
}
|
||||
|
||||
public tokenizeLineWithEdit(position: Position, length: number, newText: string): LineTokens | null {
|
||||
const lineNumber = position.lineNumber;
|
||||
const column = position.column;
|
||||
|
||||
if (!this._tokenizationStateStore) {
|
||||
return null;
|
||||
}
|
||||
|
||||
this.forceTokenization(lineNumber);
|
||||
const lineStartState = this._tokenizationStateStore.getStartState(lineNumber);
|
||||
if (!lineStartState) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const curLineContent = this._textModel.getLineContent(lineNumber);
|
||||
const newLineContent = curLineContent.substring(0, column - 1)
|
||||
+ newText + curLineContent.substring(column - 1 + length);
|
||||
|
||||
const languageId = this._textModel.getLanguageIdAtPosition(lineNumber, 0);
|
||||
const result = safeTokenize(
|
||||
this._languageIdCodec,
|
||||
languageId,
|
||||
this._tokenizationStateStore.tokenizationSupport,
|
||||
newLineContent,
|
||||
true,
|
||||
lineStartState
|
||||
);
|
||||
|
||||
const lineTokens = new LineTokens(result.tokens, newLineContent, this._languageIdCodec);
|
||||
return lineTokens;
|
||||
}
|
||||
|
||||
public isCheapToTokenize(lineNumber: number): boolean {
|
||||
if (!this._tokenizationStateStore) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const firstInvalidLineNumber = this._tokenizationStateStore.store.getFirstInvalidEndStateLineNumberOrMax();
|
||||
if (lineNumber < firstInvalidLineNumber) {
|
||||
return true;
|
||||
}
|
||||
if (lineNumber === firstInvalidLineNumber
|
||||
&& this._textModel.getLineLength(lineNumber) < Constants.CHEAP_TOKENIZATION_LENGTH_LIMIT) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* The result is not cached.
|
||||
*/
|
||||
private _heuristicallyTokenizeViewport(builder: ContiguousMultilineTokensBuilder, startLineNumber: number, endLineNumber: number): void {
|
||||
if (!this._tokenizationStateStore) {
|
||||
// nothing to do
|
||||
return;
|
||||
}
|
||||
if (endLineNumber <= this._tokenizationStateStore.store.getFirstInvalidEndStateLineNumberOrMax()) {
|
||||
// nothing to do
|
||||
return;
|
||||
}
|
||||
|
||||
if (startLineNumber <= this._tokenizationStateStore.store.getFirstInvalidEndStateLineNumberOrMax()) {
|
||||
// tokenization has reached the viewport start...
|
||||
this._tokenizationStateStore.updateTokensUntilLine(this._textModel, this._languageIdCodec, builder, endLineNumber);
|
||||
return;
|
||||
}
|
||||
|
||||
let state = this.guessStartState(startLineNumber);
|
||||
const languageId = this._textModel.getLanguageId();
|
||||
|
||||
for (let lineNumber = startLineNumber; lineNumber <= endLineNumber; lineNumber++) {
|
||||
const text = this._textModel.getLineContent(lineNumber);
|
||||
const r = safeTokenize(this._languageIdCodec, languageId, this._tokenizationStateStore.tokenizationSupport, text, true, state);
|
||||
builder.add(lineNumber, r.tokens);
|
||||
state = r.endState;
|
||||
}
|
||||
// We overrode the tokens. Because old states might get reused (thus stopping invalidation),
|
||||
// we have to explicitly request the tokens for this range again.
|
||||
this.backgroundTokenizer.value?.requestTokens(startLineNumber, endLineNumber + 1);
|
||||
}
|
||||
|
||||
private guessStartState(lineNumber: number): IState {
|
||||
let nonWhitespaceColumn = this._textModel.getLineFirstNonWhitespaceColumn(lineNumber);
|
||||
const likelyRelevantLines: string[] = [];
|
||||
let initialState: IState | null = null;
|
||||
for (let i = lineNumber - 1; nonWhitespaceColumn > 1 && i >= 1; i--) {
|
||||
const newNonWhitespaceIndex = this._textModel.getLineFirstNonWhitespaceColumn(i);
|
||||
// Ignore lines full of whitespace
|
||||
if (newNonWhitespaceIndex === 0) {
|
||||
continue;
|
||||
}
|
||||
if (newNonWhitespaceIndex < nonWhitespaceColumn) {
|
||||
likelyRelevantLines.push(this._textModel.getLineContent(i));
|
||||
nonWhitespaceColumn = newNonWhitespaceIndex;
|
||||
initialState = this._tokenizationStateStore!.getStartState(i);
|
||||
if (initialState) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!initialState) {
|
||||
initialState = this._tokenizationStateStore!.tokenizationSupport.getInitialState();
|
||||
}
|
||||
likelyRelevantLines.reverse();
|
||||
|
||||
const languageId = this._textModel.getLanguageId();
|
||||
let state = initialState;
|
||||
for (const line of likelyRelevantLines) {
|
||||
const r = safeTokenize(this._languageIdCodec, languageId, this._tokenizationStateStore!.tokenizationSupport, line, false, state);
|
||||
state = r.endState;
|
||||
}
|
||||
return state;
|
||||
}
|
||||
}
|
||||
|
||||
export class TokenizerWithStateStore<TState extends IState = IState> {
|
||||
private readonly initialState = this.tokenizationSupport.getInitialState();
|
||||
|
||||
|
|
@ -305,9 +42,20 @@ export class TokenizerWithStateStore<TState extends IState = IState> {
|
|||
}
|
||||
return this.store.getEndState(lineNumber - 1);
|
||||
}
|
||||
}
|
||||
|
||||
public updateTokensUntilLine(textModel: ITextModel, languageIdCodec: ILanguageIdCodec, builder: ContiguousMultilineTokensBuilder, lineNumber: number): void {
|
||||
const languageId = textModel.getLanguageId();
|
||||
export class TokenizerWithStateStoreAndTextModel<TState extends IState = IState> extends TokenizerWithStateStore<TState> {
|
||||
constructor(
|
||||
lineCount: number,
|
||||
tokenizationSupport: ITokenizationSupport,
|
||||
public readonly _textModel: ITextModel,
|
||||
public readonly _languageIdCodec: ILanguageIdCodec
|
||||
) {
|
||||
super(lineCount, tokenizationSupport);
|
||||
}
|
||||
|
||||
public updateTokensUntilLine(builder: ContiguousMultilineTokensBuilder, lineNumber: number): void {
|
||||
const languageId = this._textModel.getLanguageId();
|
||||
|
||||
while (true) {
|
||||
const nextLineNumber = this.store.getFirstInvalidEndStateLineNumber();
|
||||
|
|
@ -315,14 +63,145 @@ export class TokenizerWithStateStore<TState extends IState = IState> {
|
|||
break;
|
||||
}
|
||||
|
||||
const text = textModel.getLineContent(nextLineNumber);
|
||||
const text = this._textModel.getLineContent(nextLineNumber);
|
||||
const lineStartState = this.getStartState(nextLineNumber);
|
||||
|
||||
const r = safeTokenize(languageIdCodec, languageId, this.tokenizationSupport, text, true, lineStartState!);
|
||||
const r = safeTokenize(this._languageIdCodec, languageId, this.tokenizationSupport, text, true, lineStartState!);
|
||||
builder.add(nextLineNumber, r.tokens);
|
||||
this.store.setEndState(nextLineNumber, r.endState as TState);
|
||||
this!.store.setEndState(nextLineNumber, r.endState as TState);
|
||||
}
|
||||
}
|
||||
|
||||
/** assumes state is up to date */
|
||||
public getTokenTypeIfInsertingCharacter(position: Position, character: string): StandardTokenType {
|
||||
// TODO@hediet: use tokenizeLineWithEdit
|
||||
const lineStartState = this.getStartState(position.lineNumber);
|
||||
if (!lineStartState) {
|
||||
return StandardTokenType.Other;
|
||||
}
|
||||
|
||||
const languageId = this._textModel.getLanguageId();
|
||||
const lineContent = this._textModel.getLineContent(position.lineNumber);
|
||||
|
||||
// Create the text as if `character` was inserted
|
||||
const text = (
|
||||
lineContent.substring(0, position.column - 1)
|
||||
+ character
|
||||
+ lineContent.substring(position.column - 1)
|
||||
);
|
||||
|
||||
const r = safeTokenize(this._languageIdCodec, languageId, this.tokenizationSupport, text, true, lineStartState);
|
||||
const lineTokens = new LineTokens(r.tokens, text, this._languageIdCodec);
|
||||
if (lineTokens.getCount() === 0) {
|
||||
return StandardTokenType.Other;
|
||||
}
|
||||
|
||||
const tokenIndex = lineTokens.findTokenIndexAtOffset(position.column - 1);
|
||||
return lineTokens.getStandardTokenType(tokenIndex);
|
||||
}
|
||||
|
||||
/** assumes state is up to date */
|
||||
public tokenizeLineWithEdit(position: Position, length: number, newText: string): LineTokens | null {
|
||||
const lineNumber = position.lineNumber;
|
||||
const column = position.column;
|
||||
|
||||
const lineStartState = this.getStartState(lineNumber);
|
||||
if (!lineStartState) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const curLineContent = this._textModel.getLineContent(lineNumber);
|
||||
const newLineContent = curLineContent.substring(0, column - 1)
|
||||
+ newText + curLineContent.substring(column - 1 + length);
|
||||
|
||||
const languageId = this._textModel.getLanguageIdAtPosition(lineNumber, 0);
|
||||
const result = safeTokenize(
|
||||
this._languageIdCodec,
|
||||
languageId,
|
||||
this.tokenizationSupport,
|
||||
newLineContent,
|
||||
true,
|
||||
lineStartState
|
||||
);
|
||||
|
||||
const lineTokens = new LineTokens(result.tokens, newLineContent, this._languageIdCodec);
|
||||
return lineTokens;
|
||||
}
|
||||
|
||||
public isCheapToTokenize(lineNumber: number): boolean {
|
||||
const firstInvalidLineNumber = this.store.getFirstInvalidEndStateLineNumberOrMax();
|
||||
if (lineNumber < firstInvalidLineNumber) {
|
||||
return true;
|
||||
}
|
||||
if (lineNumber === firstInvalidLineNumber
|
||||
&& this._textModel.getLineLength(lineNumber) < Constants.CHEAP_TOKENIZATION_LENGTH_LIMIT) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* The result is not cached.
|
||||
*/
|
||||
public tokenizeHeuristically(builder: ContiguousMultilineTokensBuilder, startLineNumber: number, endLineNumber: number): { heuristicTokens: boolean } {
|
||||
if (endLineNumber <= this.store.getFirstInvalidEndStateLineNumberOrMax()) {
|
||||
// nothing to do
|
||||
return { heuristicTokens: false };
|
||||
}
|
||||
|
||||
if (startLineNumber <= this.store.getFirstInvalidEndStateLineNumberOrMax()) {
|
||||
// tokenization has reached the viewport start...
|
||||
this.updateTokensUntilLine(builder, endLineNumber);
|
||||
return { heuristicTokens: false };
|
||||
}
|
||||
|
||||
let state = this.guessStartState(startLineNumber);
|
||||
const languageId = this._textModel.getLanguageId();
|
||||
|
||||
for (let lineNumber = startLineNumber; lineNumber <= endLineNumber; lineNumber++) {
|
||||
const text = this._textModel.getLineContent(lineNumber);
|
||||
const r = safeTokenize(this._languageIdCodec, languageId, this.tokenizationSupport, text, true, state);
|
||||
builder.add(lineNumber, r.tokens);
|
||||
state = r.endState;
|
||||
}
|
||||
|
||||
return { heuristicTokens: true };
|
||||
}
|
||||
|
||||
private guessStartState(lineNumber: number): IState {
|
||||
let nonWhitespaceColumn = this._textModel.getLineFirstNonWhitespaceColumn(lineNumber);
|
||||
const likelyRelevantLines: string[] = [];
|
||||
let initialState: IState | null = null;
|
||||
for (let i = lineNumber - 1; nonWhitespaceColumn > 1 && i >= 1; i--) {
|
||||
const newNonWhitespaceIndex = this._textModel.getLineFirstNonWhitespaceColumn(i);
|
||||
// Ignore lines full of whitespace
|
||||
if (newNonWhitespaceIndex === 0) {
|
||||
continue;
|
||||
}
|
||||
if (newNonWhitespaceIndex < nonWhitespaceColumn) {
|
||||
likelyRelevantLines.push(this._textModel.getLineContent(i));
|
||||
nonWhitespaceColumn = newNonWhitespaceIndex;
|
||||
initialState = this.getStartState(i);
|
||||
if (initialState) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!initialState) {
|
||||
initialState = this.tokenizationSupport.getInitialState();
|
||||
}
|
||||
likelyRelevantLines.reverse();
|
||||
|
||||
const languageId = this._textModel.getLanguageId();
|
||||
let state = initialState;
|
||||
for (const line of likelyRelevantLines) {
|
||||
const r = safeTokenize(this._languageIdCodec, languageId, this.tokenizationSupport, line, false, state);
|
||||
state = r.endState;
|
||||
}
|
||||
return state;
|
||||
}
|
||||
}
|
||||
|
||||
export class TrackingTokenizationStateStore<TState extends IState> {
|
||||
|
|
@ -501,23 +380,6 @@ export class RangePriorityQueueImpl implements RangePriorityQueue {
|
|||
}
|
||||
}
|
||||
|
||||
function initializeTokenization(textModel: TextModel, tokenizationPart: TokenizationTextModelPart): [ITokenizationSupport, IState] | [null, null] {
|
||||
if (textModel.isTooLargeForTokenization()) {
|
||||
return [null, null];
|
||||
}
|
||||
const tokenizationSupport = TokenizationRegistry.get(tokenizationPart.getLanguageId());
|
||||
if (!tokenizationSupport) {
|
||||
return [null, null];
|
||||
}
|
||||
let initialState: IState;
|
||||
try {
|
||||
initialState = tokenizationSupport.getInitialState();
|
||||
} catch (e) {
|
||||
onUnexpectedError(e);
|
||||
return [null, null];
|
||||
}
|
||||
return [tokenizationSupport, initialState];
|
||||
}
|
||||
|
||||
function safeTokenize(languageIdCodec: ILanguageIdCodec, languageId: string, tokenizationSupport: ITokenizationSupport | null, text: string, hasEOL: boolean, state: IState): EncodedTokenizationResult {
|
||||
let r: EncodedTokenizationResult | null = null;
|
||||
|
|
@ -538,14 +400,12 @@ function safeTokenize(languageIdCodec: ILanguageIdCodec, languageId: string, tok
|
|||
return r;
|
||||
}
|
||||
|
||||
class DefaultBackgroundTokenizer implements IBackgroundTokenizer {
|
||||
export class DefaultBackgroundTokenizer implements IBackgroundTokenizer {
|
||||
private _isDisposed = false;
|
||||
|
||||
constructor(
|
||||
private readonly _textModel: ITextModel,
|
||||
private readonly _tokenizerWithStateStore: TokenizerWithStateStore,
|
||||
private readonly _tokenizerWithStateStore: TokenizerWithStateStoreAndTextModel,
|
||||
private readonly _backgroundTokenStore: IBackgroundTokenizationStore,
|
||||
private readonly _languageIdCodec: ILanguageIdCodec,
|
||||
) {
|
||||
}
|
||||
|
||||
|
|
@ -559,7 +419,7 @@ class DefaultBackgroundTokenizer implements IBackgroundTokenizer {
|
|||
|
||||
private _isScheduled = false;
|
||||
private _beginBackgroundTokenization(): void {
|
||||
if (this._isScheduled || !this._textModel.isAttachedToEditor() || !this._hasLinesToTokenize()) {
|
||||
if (this._isScheduled || !this._tokenizerWithStateStore._textModel.isAttachedToEditor() || !this._hasLinesToTokenize()) {
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -580,7 +440,7 @@ class DefaultBackgroundTokenizer implements IBackgroundTokenizer {
|
|||
const endTime = Date.now() + deadline.timeRemaining();
|
||||
|
||||
const execute = () => {
|
||||
if (this._isDisposed || !this._textModel.isAttachedToEditor() || !this._hasLinesToTokenize()) {
|
||||
if (this._isDisposed || !this._tokenizerWithStateStore._textModel.isAttachedToEditor() || !this._hasLinesToTokenize()) {
|
||||
// disposed in the meantime or detached or finished
|
||||
return;
|
||||
}
|
||||
|
|
@ -603,7 +463,7 @@ class DefaultBackgroundTokenizer implements IBackgroundTokenizer {
|
|||
* Tokenize for at least 1ms.
|
||||
*/
|
||||
private _backgroundTokenizeForAtLeast1ms(): void {
|
||||
const lineCount = this._textModel.getLineCount();
|
||||
const lineCount = this._tokenizerWithStateStore._textModel.getLineCount();
|
||||
const builder = new ContiguousMultilineTokensBuilder();
|
||||
const sw = StopWatch.create(false);
|
||||
|
||||
|
|
@ -635,10 +495,10 @@ class DefaultBackgroundTokenizer implements IBackgroundTokenizer {
|
|||
|
||||
private _tokenizeOneInvalidLine(builder: ContiguousMultilineTokensBuilder): number {
|
||||
if (!this._tokenizerWithStateStore || !this._hasLinesToTokenize()) {
|
||||
return this._textModel.getLineCount() + 1;
|
||||
return this._tokenizerWithStateStore._textModel.getLineCount() + 1;
|
||||
}
|
||||
const lineNumber = this._tokenizerWithStateStore.store.getFirstInvalidEndStateLineNumber()!;
|
||||
this._tokenizerWithStateStore.updateTokensUntilLine(this._textModel, this._languageIdCodec, builder, lineNumber);
|
||||
this._tokenizerWithStateStore.updateTokensUntilLine(builder, lineNumber);
|
||||
return lineNumber;
|
||||
}
|
||||
|
||||
|
|
@ -651,7 +511,7 @@ class DefaultBackgroundTokenizer implements IBackgroundTokenizer {
|
|||
}
|
||||
}
|
||||
|
||||
requestTokens(startLineNumber: number, endLineNumberExclusive: number): void {
|
||||
public requestTokens(startLineNumber: number, endLineNumberExclusive: number): void {
|
||||
this._tokenizerWithStateStore.store.invalidateEndStateRange(new LineRange(startLineNumber, endLineNumberExclusive));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,29 +3,34 @@
|
|||
* Licensed under the MIT License. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
import { Emitter, Event } from 'vs/base/common/event';
|
||||
import { CharCode } from 'vs/base/common/charCode';
|
||||
import { BugIndicatingError, onUnexpectedError } from 'vs/base/common/errors';
|
||||
import { Emitter, Event } from 'vs/base/common/event';
|
||||
import { Disposable, MutableDisposable } from 'vs/base/common/lifecycle';
|
||||
import { countEOL } from 'vs/editor/common/core/eolCounter';
|
||||
import { IPosition, Position } from 'vs/editor/common/core/position';
|
||||
import { Range } from 'vs/editor/common/core/range';
|
||||
import { getWordAtText, IWordAtPosition } from 'vs/editor/common/core/wordHelper';
|
||||
import { IWordAtPosition, getWordAtText } from 'vs/editor/common/core/wordHelper';
|
||||
import { StandardTokenType } from 'vs/editor/common/encodedTokenAttributes';
|
||||
import { IBackgroundTokenizationStore, IBackgroundTokenizer, ILanguageIdCodec, IState, ITokenizationSupport, TokenizationRegistry } from 'vs/editor/common/languages';
|
||||
import { ILanguageService } from 'vs/editor/common/languages/language';
|
||||
import { ILanguageConfigurationService, ResolvedLanguageConfiguration } from 'vs/editor/common/languages/languageConfigurationRegistry';
|
||||
import { BracketPairsTextModelPart } from 'vs/editor/common/model/bracketPairsTextModelPart/bracketPairsImpl';
|
||||
import { TextModel } from 'vs/editor/common/model/textModel';
|
||||
import { TextModelPart } from 'vs/editor/common/model/textModelPart';
|
||||
import { TextModelTokenization } from 'vs/editor/common/model/textModelTokens';
|
||||
import { DefaultBackgroundTokenizer, TokenizerWithStateStoreAndTextModel } from 'vs/editor/common/model/textModelTokens';
|
||||
import { IModelContentChangedEvent, IModelLanguageChangedEvent, IModelLanguageConfigurationChangedEvent, IModelTokensChangedEvent } from 'vs/editor/common/textModelEvents';
|
||||
import { BackgroundTokenizationState, ITokenizationTextModelPart } from 'vs/editor/common/tokenizationTextModelPart';
|
||||
import { ContiguousMultilineTokens } from 'vs/editor/common/tokens/contiguousMultilineTokens';
|
||||
import { ContiguousMultilineTokensBuilder } from 'vs/editor/common/tokens/contiguousMultilineTokensBuilder';
|
||||
import { ContiguousTokensStore } from 'vs/editor/common/tokens/contiguousTokensStore';
|
||||
import { LineTokens } from 'vs/editor/common/tokens/lineTokens';
|
||||
import { SparseMultilineTokens } from 'vs/editor/common/tokens/sparseMultilineTokens';
|
||||
import { SparseTokensStore } from 'vs/editor/common/tokens/sparseTokensStore';
|
||||
import { BracketPairsTextModelPart } from 'vs/editor/common/model/bracketPairsTextModelPart/bracketPairsImpl';
|
||||
import { BackgroundTokenizationState, ITokenizationTextModelPart } from 'vs/editor/common/tokenizationTextModelPart';
|
||||
import { countEOL } from 'vs/editor/common/core/eolCounter';
|
||||
import { BugIndicatingError } from 'vs/base/common/errors';
|
||||
|
||||
export class TokenizationTextModelPart extends TextModelPart implements ITokenizationTextModelPart {
|
||||
private readonly _semanticTokens: SparseTokensStore = new SparseTokensStore(this._languageService.languageIdCodec);
|
||||
|
||||
private readonly _onDidChangeLanguage: Emitter<IModelLanguageChangedEvent> = this._register(new Emitter<IModelLanguageChangedEvent>());
|
||||
public readonly onDidChangeLanguage: Event<IModelLanguageChangedEvent> = this._onDidChangeLanguage.event;
|
||||
|
||||
|
|
@ -35,57 +40,45 @@ export class TokenizationTextModelPart extends TextModelPart implements ITokeniz
|
|||
private readonly _onDidChangeTokens: Emitter<IModelTokensChangedEvent> = this._register(new Emitter<IModelTokensChangedEvent>());
|
||||
public readonly onDidChangeTokens: Event<IModelTokensChangedEvent> = this._onDidChangeTokens.event;
|
||||
|
||||
private readonly _tokens: ContiguousTokensStore;
|
||||
private readonly _semanticTokens: SparseTokensStore;
|
||||
private readonly _tokenization: TextModelTokenization;
|
||||
private readonly grammarTokens = this._register(new GrammarTokens(this._languageService.languageIdCodec, this._textModel, () => this._languageId));
|
||||
|
||||
constructor(
|
||||
private readonly _languageService: ILanguageService,
|
||||
private readonly _languageConfigurationService: ILanguageConfigurationService,
|
||||
private readonly _textModel: TextModel,
|
||||
private readonly bracketPairsTextModelPart: BracketPairsTextModelPart,
|
||||
private readonly _bracketPairsTextModelPart: BracketPairsTextModelPart,
|
||||
private _languageId: string,
|
||||
) {
|
||||
super();
|
||||
|
||||
this._tokens = new ContiguousTokensStore(
|
||||
this._languageService.languageIdCodec
|
||||
);
|
||||
this._semanticTokens = new SparseTokensStore(
|
||||
this._languageService.languageIdCodec
|
||||
);
|
||||
this._tokenization = this._register(new TextModelTokenization(
|
||||
_textModel,
|
||||
this,
|
||||
this._languageService.languageIdCodec
|
||||
));
|
||||
|
||||
this._register(this._languageConfigurationService.onDidChange(
|
||||
e => {
|
||||
if (e.affects(this._languageId)) {
|
||||
this._onDidChangeLanguageConfiguration.fire({});
|
||||
}
|
||||
this._register(this._languageConfigurationService.onDidChange(e => {
|
||||
if (e.affects(this._languageId)) {
|
||||
this._onDidChangeLanguageConfiguration.fire({});
|
||||
}
|
||||
));
|
||||
}));
|
||||
|
||||
this._register(this.grammarTokens.onDidChangeTokens(e => {
|
||||
this._emitModelTokensChangedEvent(e);
|
||||
}));
|
||||
|
||||
this._register(this.grammarTokens.onDidChangeBackgroundTokenizationState(e => {
|
||||
this._bracketPairsTextModelPart.handleDidChangeBackgroundTokenizationState();
|
||||
}));
|
||||
}
|
||||
|
||||
_hasListeners(): boolean {
|
||||
return (
|
||||
this._onDidChangeLanguage.hasListeners()
|
||||
return (this._onDidChangeLanguage.hasListeners()
|
||||
|| this._onDidChangeLanguageConfiguration.hasListeners()
|
||||
|| this._onDidChangeTokens.hasListeners()
|
||||
);
|
||||
|| this._onDidChangeTokens.hasListeners());
|
||||
}
|
||||
|
||||
public handleDidChangeContent(change: IModelContentChangedEvent): void {
|
||||
if (change.isFlush) {
|
||||
this._tokens.flush();
|
||||
public handleDidChangeContent(e: IModelContentChangedEvent): void {
|
||||
if (e.isFlush) {
|
||||
this._semanticTokens.flush();
|
||||
} else if (!change.isEolChange) { // We don't have to do anything on an EOL change
|
||||
for (const c of change.changes) {
|
||||
} else if (!e.isEolChange) { // We don't have to do anything on an EOL change
|
||||
for (const c of e.changes) {
|
||||
const [eolCount, firstLineLength, lastLineLength] = countEOL(c.text);
|
||||
|
||||
this._tokens.acceptEdit(c.range, eolCount, firstLineLength);
|
||||
this._semanticTokens.acceptEdit(
|
||||
c.range,
|
||||
eolCount,
|
||||
|
|
@ -96,115 +89,81 @@ export class TokenizationTextModelPart extends TextModelPart implements ITokeniz
|
|||
}
|
||||
}
|
||||
|
||||
this._tokenization.handleDidChangeContent(change);
|
||||
this.grammarTokens.handleDidChangeContent(e);
|
||||
}
|
||||
|
||||
public handleDidChangeAttached(): void {
|
||||
this._tokenization.handleDidChangeAttached();
|
||||
this.grammarTokens.handleDidChangeAttached();
|
||||
}
|
||||
|
||||
private _backgroundTokenizationState = BackgroundTokenizationState.InProgress;
|
||||
public get backgroundTokenizationState(): BackgroundTokenizationState {
|
||||
return this._backgroundTokenizationState;
|
||||
/**
|
||||
* Includes grammar and semantic tokens.
|
||||
*/
|
||||
public getLineTokens(lineNumber: number): LineTokens {
|
||||
this.validateLineNumber(lineNumber);
|
||||
const syntacticTokens = this.grammarTokens.getLineTokens(lineNumber);
|
||||
return this._semanticTokens.addSparseTokens(lineNumber, syntacticTokens);
|
||||
}
|
||||
|
||||
private _emitModelTokensChangedEvent(e: IModelTokensChangedEvent): void {
|
||||
if (!this._textModel._isDisposing()) {
|
||||
this._bracketPairsTextModelPart.handleDidChangeTokens(e);
|
||||
this._onDidChangeTokens.fire(e);
|
||||
}
|
||||
}
|
||||
|
||||
// #region Grammar Tokens
|
||||
|
||||
private validateLineNumber(lineNumber: number): void {
|
||||
if (lineNumber < 1 || lineNumber > this._textModel.getLineCount()) {
|
||||
throw new BugIndicatingError('Illegal value for lineNumber');
|
||||
}
|
||||
}
|
||||
|
||||
public setLineTokens(
|
||||
lineNumber: number,
|
||||
tokens: Uint32Array | ArrayBuffer | null
|
||||
): void {
|
||||
this.validateLineNumber(lineNumber);
|
||||
|
||||
this._tokens.setTokens(
|
||||
this._languageId,
|
||||
lineNumber - 1,
|
||||
this._textModel.getLineLength(lineNumber),
|
||||
tokens,
|
||||
false
|
||||
);
|
||||
}
|
||||
|
||||
public handleBackgroundTokenizationFinished(): void {
|
||||
if (this._backgroundTokenizationState === BackgroundTokenizationState.Completed) {
|
||||
// We already did a full tokenization and don't go back to progressing.
|
||||
return;
|
||||
}
|
||||
const newState = BackgroundTokenizationState.Completed;
|
||||
this._backgroundTokenizationState = newState;
|
||||
this.bracketPairsTextModelPart.handleDidChangeBackgroundTokenizationState();
|
||||
}
|
||||
|
||||
public get hasTokens(): boolean {
|
||||
return this._tokens.hasTokens;
|
||||
return this.grammarTokens.hasTokens;
|
||||
}
|
||||
|
||||
public setTokens(tokens: ContiguousMultilineTokens[]): void {
|
||||
if (tokens.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const ranges: { fromLineNumber: number; toLineNumber: number }[] = [];
|
||||
|
||||
for (let i = 0, len = tokens.length; i < len; i++) {
|
||||
const element = tokens[i];
|
||||
let minChangedLineNumber = 0;
|
||||
let maxChangedLineNumber = 0;
|
||||
let hasChange = false;
|
||||
for (
|
||||
let lineNumber = element.startLineNumber;
|
||||
lineNumber <= element.endLineNumber;
|
||||
lineNumber++
|
||||
) {
|
||||
if (hasChange) {
|
||||
this._tokens.setTokens(
|
||||
this._languageId,
|
||||
lineNumber - 1,
|
||||
this._textModel.getLineLength(lineNumber),
|
||||
element.getLineTokens(lineNumber),
|
||||
false
|
||||
);
|
||||
maxChangedLineNumber = lineNumber;
|
||||
} else {
|
||||
const lineHasChange = this._tokens.setTokens(
|
||||
this._languageId,
|
||||
lineNumber - 1,
|
||||
this._textModel.getLineLength(lineNumber),
|
||||
element.getLineTokens(lineNumber),
|
||||
true
|
||||
);
|
||||
if (lineHasChange) {
|
||||
hasChange = true;
|
||||
minChangedLineNumber = lineNumber;
|
||||
maxChangedLineNumber = lineNumber;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (hasChange) {
|
||||
ranges.push({
|
||||
fromLineNumber: minChangedLineNumber,
|
||||
toLineNumber: maxChangedLineNumber,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (ranges.length > 0) {
|
||||
this._emitModelTokensChangedEvent({
|
||||
tokenizationSupportChanged: false,
|
||||
semanticTokensApplied: false,
|
||||
ranges: ranges,
|
||||
});
|
||||
}
|
||||
public resetTokenization() {
|
||||
this.grammarTokens.resetTokenization();
|
||||
}
|
||||
|
||||
public setSemanticTokens(
|
||||
tokens: SparseMultilineTokens[] | null,
|
||||
isComplete: boolean
|
||||
): void {
|
||||
public get backgroundTokenizationState() {
|
||||
return this.grammarTokens.backgroundTokenizationState;
|
||||
}
|
||||
|
||||
public refreshTokens(startLineNumber: number, endLineNumber: number): void {
|
||||
this.grammarTokens.refreshTokens(startLineNumber, endLineNumber);
|
||||
}
|
||||
|
||||
public forceTokenization(lineNumber: number): void {
|
||||
this.validateLineNumber(lineNumber);
|
||||
this.grammarTokens.forceTokenization(lineNumber);
|
||||
}
|
||||
|
||||
public isCheapToTokenize(lineNumber: number): boolean {
|
||||
this.validateLineNumber(lineNumber);
|
||||
return this.grammarTokens.isCheapToTokenize(lineNumber);
|
||||
}
|
||||
|
||||
public tokenizeIfCheap(lineNumber: number): void {
|
||||
this.validateLineNumber(lineNumber);
|
||||
this.grammarTokens.tokenizeIfCheap(lineNumber);
|
||||
}
|
||||
|
||||
public getTokenTypeIfInsertingCharacter(lineNumber: number, column: number, character: string): StandardTokenType {
|
||||
return this.grammarTokens.getTokenTypeIfInsertingCharacter(lineNumber, column, character);
|
||||
}
|
||||
|
||||
public tokenizeLineWithEdit(position: IPosition, length: number, newText: string): LineTokens | null {
|
||||
return this.grammarTokens.tokenizeLineWithEdit(position, length, newText);
|
||||
}
|
||||
|
||||
// #endregion
|
||||
|
||||
// #region Semantic Tokens
|
||||
|
||||
public setSemanticTokens(tokens: SparseMultilineTokens[] | null, isComplete: boolean): void {
|
||||
this._semanticTokens.set(tokens, isComplete);
|
||||
|
||||
this._emitModelTokensChangedEvent({
|
||||
|
|
@ -222,10 +181,7 @@ export class TokenizationTextModelPart extends TextModelPart implements ITokeniz
|
|||
return !this._semanticTokens.isEmpty();
|
||||
}
|
||||
|
||||
public setPartialSemanticTokens(
|
||||
range: Range,
|
||||
tokens: SparseMultilineTokens[]
|
||||
): void {
|
||||
public setPartialSemanticTokens(range: Range, tokens: SparseMultilineTokens[]): void {
|
||||
if (this.hasCompleteSemanticTokens()) {
|
||||
return;
|
||||
}
|
||||
|
|
@ -245,130 +201,23 @@ export class TokenizationTextModelPart extends TextModelPart implements ITokeniz
|
|||
});
|
||||
}
|
||||
|
||||
public tokenizeViewport(
|
||||
startLineNumber: number,
|
||||
endLineNumber: number
|
||||
): void {
|
||||
startLineNumber = Math.max(1, startLineNumber);
|
||||
endLineNumber = Math.min(this._textModel.getLineCount(), endLineNumber);
|
||||
this._tokenization.tokenizeViewport(startLineNumber, endLineNumber);
|
||||
}
|
||||
// #endregion
|
||||
|
||||
public clearTokens(): void {
|
||||
this._tokens.flush();
|
||||
this._emitModelTokensChangedEvent({
|
||||
tokenizationSupportChanged: true,
|
||||
semanticTokensApplied: false,
|
||||
ranges: [
|
||||
{
|
||||
fromLineNumber: 1,
|
||||
toLineNumber: this._textModel.getLineCount(),
|
||||
},
|
||||
],
|
||||
});
|
||||
}
|
||||
|
||||
public clearSemanticTokens(): void {
|
||||
this._semanticTokens.flush();
|
||||
|
||||
this._emitModelTokensChangedEvent({
|
||||
tokenizationSupportChanged: false,
|
||||
semanticTokensApplied: false,
|
||||
ranges: [{ fromLineNumber: 1, toLineNumber: this._textModel.getLineCount() }],
|
||||
});
|
||||
}
|
||||
|
||||
private _emitModelTokensChangedEvent(e: IModelTokensChangedEvent): void {
|
||||
if (!this._textModel._isDisposing()) {
|
||||
this.bracketPairsTextModelPart.handleDidChangeTokens(e);
|
||||
this._onDidChangeTokens.fire(e);
|
||||
}
|
||||
}
|
||||
|
||||
public resetTokenization(): void {
|
||||
this._tokenization.reset();
|
||||
}
|
||||
|
||||
public forceTokenization(lineNumber: number): void {
|
||||
this.validateLineNumber(lineNumber);
|
||||
|
||||
this._tokenization.forceTokenization(lineNumber);
|
||||
}
|
||||
|
||||
public isCheapToTokenize(lineNumber: number): boolean {
|
||||
return this._tokenization.isCheapToTokenize(lineNumber);
|
||||
}
|
||||
|
||||
public tokenizeIfCheap(lineNumber: number): void {
|
||||
if (this.isCheapToTokenize(lineNumber)) {
|
||||
this.forceTokenization(lineNumber);
|
||||
}
|
||||
}
|
||||
|
||||
public getLineTokens(lineNumber: number): LineTokens {
|
||||
this.validateLineNumber(lineNumber);
|
||||
|
||||
const lineText = this._textModel.getLineContent(lineNumber);
|
||||
const syntacticTokens = this._tokens.getTokens(
|
||||
this._languageId,
|
||||
lineNumber - 1,
|
||||
lineText
|
||||
);
|
||||
return this._semanticTokens.addSparseTokens(lineNumber, syntacticTokens);
|
||||
}
|
||||
|
||||
public getTokenTypeIfInsertingCharacter(
|
||||
lineNumber: number,
|
||||
column: number,
|
||||
character: string
|
||||
): StandardTokenType {
|
||||
const position = this._textModel.validatePosition(new Position(lineNumber, column));
|
||||
return this._tokenization.getTokenTypeIfInsertingCharacter(
|
||||
position,
|
||||
character
|
||||
);
|
||||
}
|
||||
|
||||
public tokenizeLineWithEdit(
|
||||
position: IPosition,
|
||||
length: number,
|
||||
newText: string
|
||||
): LineTokens | null {
|
||||
const validatedPosition = this._textModel.validatePosition(position);
|
||||
return this._tokenization.tokenizeLineWithEdit(
|
||||
validatedPosition,
|
||||
length,
|
||||
newText
|
||||
);
|
||||
}
|
||||
|
||||
private getLanguageConfiguration(
|
||||
languageId: string
|
||||
): ResolvedLanguageConfiguration {
|
||||
return this._languageConfigurationService.getLanguageConfiguration(
|
||||
languageId
|
||||
);
|
||||
}
|
||||
|
||||
// Having tokens allows implementing additional helper methods
|
||||
// #region Utility Methods
|
||||
|
||||
public getWordAtPosition(_position: IPosition): IWordAtPosition | null {
|
||||
this.assertNotDisposed();
|
||||
|
||||
const position = this._textModel.validatePosition(_position);
|
||||
const lineContent = this._textModel.getLineContent(position.lineNumber);
|
||||
const lineTokens = this.getLineTokens(position.lineNumber);
|
||||
const tokenIndex = lineTokens.findTokenIndexAtOffset(position.column - 1);
|
||||
|
||||
// (1). First try checking right biased word
|
||||
const [rbStartOffset, rbEndOffset] = TokenizationTextModelPart._findLanguageBoundaries(
|
||||
lineTokens,
|
||||
tokenIndex
|
||||
);
|
||||
const [rbStartOffset, rbEndOffset] = TokenizationTextModelPart._findLanguageBoundaries(lineTokens, tokenIndex);
|
||||
const rightBiasedWord = getWordAtText(
|
||||
position.column,
|
||||
this.getLanguageConfiguration(
|
||||
lineTokens.getLanguageId(tokenIndex)
|
||||
).getWordDefinition(),
|
||||
this.getLanguageConfiguration(lineTokens.getLanguageId(tokenIndex)).getWordDefinition(),
|
||||
lineContent.substring(rbStartOffset, rbEndOffset),
|
||||
rbStartOffset
|
||||
);
|
||||
|
|
@ -390,9 +239,7 @@ export class TokenizationTextModelPart extends TextModelPart implements ITokeniz
|
|||
);
|
||||
const leftBiasedWord = getWordAtText(
|
||||
position.column,
|
||||
this.getLanguageConfiguration(
|
||||
lineTokens.getLanguageId(tokenIndex - 1)
|
||||
).getWordDefinition(),
|
||||
this.getLanguageConfiguration(lineTokens.getLanguageId(tokenIndex - 1)).getWordDefinition(),
|
||||
lineContent.substring(lbStartOffset, lbEndOffset),
|
||||
lbStartOffset
|
||||
);
|
||||
|
|
@ -409,19 +256,16 @@ export class TokenizationTextModelPart extends TextModelPart implements ITokeniz
|
|||
return null;
|
||||
}
|
||||
|
||||
private static _findLanguageBoundaries(
|
||||
lineTokens: LineTokens,
|
||||
tokenIndex: number
|
||||
): [number, number] {
|
||||
private getLanguageConfiguration(languageId: string): ResolvedLanguageConfiguration {
|
||||
return this._languageConfigurationService.getLanguageConfiguration(languageId);
|
||||
}
|
||||
|
||||
private static _findLanguageBoundaries(lineTokens: LineTokens, tokenIndex: number): [number, number] {
|
||||
const languageId = lineTokens.getLanguageId(tokenIndex);
|
||||
|
||||
// go left until a different language is hit
|
||||
let startOffset = 0;
|
||||
for (
|
||||
let i = tokenIndex;
|
||||
i >= 0 && lineTokens.getLanguageId(i) === languageId;
|
||||
i--
|
||||
) {
|
||||
for (let i = tokenIndex; i >= 0 && lineTokens.getLanguageId(i) === languageId; i--) {
|
||||
startOffset = lineTokens.getStartOffset(i);
|
||||
}
|
||||
|
||||
|
|
@ -441,22 +285,19 @@ export class TokenizationTextModelPart extends TextModelPart implements ITokeniz
|
|||
public getWordUntilPosition(position: IPosition): IWordAtPosition {
|
||||
const wordAtPosition = this.getWordAtPosition(position);
|
||||
if (!wordAtPosition) {
|
||||
return {
|
||||
word: '',
|
||||
startColumn: position.column,
|
||||
endColumn: position.column,
|
||||
};
|
||||
return { word: '', startColumn: position.column, endColumn: position.column, };
|
||||
}
|
||||
return {
|
||||
word: wordAtPosition.word.substr(
|
||||
0,
|
||||
position.column - wordAtPosition.startColumn
|
||||
),
|
||||
word: wordAtPosition.word.substr(0, position.column - wordAtPosition.startColumn),
|
||||
startColumn: wordAtPosition.startColumn,
|
||||
endColumn: position.column,
|
||||
};
|
||||
}
|
||||
|
||||
// #endregion
|
||||
|
||||
// #region Language Id handling
|
||||
|
||||
public getLanguageId(): string {
|
||||
return this._languageId;
|
||||
}
|
||||
|
|
@ -481,9 +322,270 @@ export class TokenizationTextModelPart extends TextModelPart implements ITokeniz
|
|||
|
||||
this._languageId = languageId;
|
||||
|
||||
this.bracketPairsTextModelPart.handleDidChangeLanguage(e);
|
||||
this._tokenization.handleDidChangeLanguage(e);
|
||||
this._bracketPairsTextModelPart.handleDidChangeLanguage(e);
|
||||
this.grammarTokens.resetTokenization();
|
||||
this._onDidChangeLanguage.fire(e);
|
||||
this._onDidChangeLanguageConfiguration.fire({});
|
||||
}
|
||||
|
||||
// #endregion
|
||||
}
|
||||
|
||||
class GrammarTokens extends Disposable {
|
||||
private _tokenizer: TokenizerWithStateStoreAndTextModel | null = null;
|
||||
private _defaultBackgroundTokenizer: DefaultBackgroundTokenizer | null = null;
|
||||
private readonly _backgroundTokenizer = this._register(new MutableDisposable<IBackgroundTokenizer>());
|
||||
|
||||
private readonly _tokens = new ContiguousTokensStore(this._languageIdCodec);
|
||||
|
||||
private _backgroundTokenizationState = BackgroundTokenizationState.InProgress;
|
||||
public get backgroundTokenizationState(): BackgroundTokenizationState {
|
||||
return this._backgroundTokenizationState;
|
||||
}
|
||||
|
||||
private readonly _onDidChangeBackgroundTokenizationState = this._register(new Emitter<void>());
|
||||
/** @internal, should not be exposed by the text model! */
|
||||
public readonly onDidChangeBackgroundTokenizationState: Event<void> = this._onDidChangeBackgroundTokenizationState.event;
|
||||
|
||||
private readonly _onDidChangeTokens = this._register(new Emitter<IModelTokensChangedEvent>());
|
||||
/** @internal, should not be exposed by the text model! */
|
||||
public readonly onDidChangeTokens: Event<IModelTokensChangedEvent> = this._onDidChangeTokens.event;
|
||||
|
||||
constructor(
|
||||
private readonly _languageIdCodec: ILanguageIdCodec,
|
||||
private readonly _textModel: TextModel,
|
||||
private getLanguageId: () => string,
|
||||
) {
|
||||
super();
|
||||
|
||||
this._register(TokenizationRegistry.onDidChange((e) => {
|
||||
const languageId = this._textModel.getLanguageId();
|
||||
if (e.changedLanguages.indexOf(languageId) === -1) {
|
||||
return;
|
||||
}
|
||||
this.resetTokenization();
|
||||
}));
|
||||
|
||||
this.resetTokenization();
|
||||
}
|
||||
|
||||
public resetTokenization(fireTokenChangeEvent: boolean = true): void {
|
||||
this._tokens.flush();
|
||||
if (fireTokenChangeEvent) {
|
||||
this._onDidChangeTokens.fire({
|
||||
tokenizationSupportChanged: true,
|
||||
semanticTokensApplied: false,
|
||||
ranges: [
|
||||
{
|
||||
fromLineNumber: 1,
|
||||
toLineNumber: this._textModel.getLineCount(),
|
||||
},
|
||||
],
|
||||
});
|
||||
}
|
||||
|
||||
const [tokenizationSupport, initialState] = initializeTokenization(this._textModel, this.getLanguageId());
|
||||
if (tokenizationSupport && initialState) {
|
||||
this._tokenizer = new TokenizerWithStateStoreAndTextModel(this._textModel.getLineCount(), tokenizationSupport, this._textModel, this._languageIdCodec);
|
||||
} else {
|
||||
this._tokenizer = null;
|
||||
}
|
||||
|
||||
this._backgroundTokenizer.clear();
|
||||
|
||||
this._defaultBackgroundTokenizer = null;
|
||||
if (this._tokenizer) {
|
||||
const b: IBackgroundTokenizationStore = {
|
||||
setTokens: (tokens) => {
|
||||
this.setTokens(tokens);
|
||||
},
|
||||
backgroundTokenizationFinished: () => {
|
||||
if (this._backgroundTokenizationState === BackgroundTokenizationState.Completed) {
|
||||
// We already did a full tokenization and don't go back to progressing.
|
||||
return;
|
||||
}
|
||||
const newState = BackgroundTokenizationState.Completed;
|
||||
this._backgroundTokenizationState = newState;
|
||||
this._onDidChangeBackgroundTokenizationState.fire();
|
||||
},
|
||||
setEndState: (lineNumber, state) => {
|
||||
if (!state) {
|
||||
throw new BugIndicatingError();
|
||||
}
|
||||
const firstInvalidEndStateLineNumber = this._tokenizer?.store.getFirstInvalidEndStateLineNumber() ?? undefined;
|
||||
if (firstInvalidEndStateLineNumber !== undefined && lineNumber >= firstInvalidEndStateLineNumber) {
|
||||
// Don't accept states for definitely valid states
|
||||
this._tokenizer?.store.setEndState(lineNumber, state);
|
||||
}
|
||||
},
|
||||
};
|
||||
|
||||
if (tokenizationSupport && tokenizationSupport.createBackgroundTokenizer) {
|
||||
this._backgroundTokenizer.value = tokenizationSupport.createBackgroundTokenizer(this._textModel, b);
|
||||
}
|
||||
if (!this._backgroundTokenizer.value) {
|
||||
this._backgroundTokenizer.value = this._defaultBackgroundTokenizer =
|
||||
new DefaultBackgroundTokenizer(this._tokenizer, b);
|
||||
this._defaultBackgroundTokenizer.handleChanges();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public handleDidChangeAttached() {
|
||||
this._defaultBackgroundTokenizer?.handleChanges();
|
||||
}
|
||||
|
||||
public handleDidChangeContent(e: IModelContentChangedEvent): void {
|
||||
if (e.isFlush) {
|
||||
// Don't fire the event, as the view might not have got the text change event yet
|
||||
this.resetTokenization(false);
|
||||
} else if (!e.isEolChange) { // We don't have to do anything on an EOL change
|
||||
for (const c of e.changes) {
|
||||
const [eolCount, firstLineLength] = countEOL(c.text);
|
||||
|
||||
this._tokens.acceptEdit(c.range, eolCount, firstLineLength);
|
||||
}
|
||||
|
||||
if (this._tokenizer) {
|
||||
this._tokenizer.store.acceptChanges(e.changes);
|
||||
}
|
||||
this._defaultBackgroundTokenizer?.handleChanges();
|
||||
}
|
||||
}
|
||||
|
||||
private setTokens(tokens: ContiguousMultilineTokens[]): { changes: { fromLineNumber: number; toLineNumber: number }[] } {
|
||||
if (tokens.length === 0) {
|
||||
return { changes: [] };
|
||||
}
|
||||
|
||||
const ranges: { fromLineNumber: number; toLineNumber: number }[] = [];
|
||||
|
||||
for (let i = 0, len = tokens.length; i < len; i++) {
|
||||
const element = tokens[i];
|
||||
let minChangedLineNumber = 0;
|
||||
let maxChangedLineNumber = 0;
|
||||
let hasChange = false;
|
||||
for (let lineNumber = element.startLineNumber; lineNumber <= element.endLineNumber; lineNumber++) {
|
||||
if (hasChange) {
|
||||
this._tokens.setTokens(this._textModel.getLanguageId(), lineNumber - 1, this._textModel.getLineLength(lineNumber), element.getLineTokens(lineNumber), false);
|
||||
maxChangedLineNumber = lineNumber;
|
||||
} else {
|
||||
const lineHasChange = this._tokens.setTokens(this._textModel.getLanguageId(), lineNumber - 1, this._textModel.getLineLength(lineNumber), element.getLineTokens(lineNumber), true);
|
||||
if (lineHasChange) {
|
||||
hasChange = true;
|
||||
minChangedLineNumber = lineNumber;
|
||||
maxChangedLineNumber = lineNumber;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (hasChange) {
|
||||
ranges.push({ fromLineNumber: minChangedLineNumber, toLineNumber: maxChangedLineNumber, });
|
||||
}
|
||||
}
|
||||
|
||||
if (ranges.length > 0) {
|
||||
this._onDidChangeTokens.fire({
|
||||
tokenizationSupportChanged: false,
|
||||
semanticTokensApplied: false,
|
||||
ranges: ranges,
|
||||
});
|
||||
}
|
||||
|
||||
return { changes: ranges };
|
||||
}
|
||||
|
||||
public refreshTokens(startLineNumber: number, endLineNumber: number): void {
|
||||
if (!this._tokenizer) {
|
||||
return;
|
||||
}
|
||||
|
||||
startLineNumber = Math.max(1, startLineNumber);
|
||||
endLineNumber = Math.min(this._textModel.getLineCount(), endLineNumber);
|
||||
|
||||
const builder = new ContiguousMultilineTokensBuilder();
|
||||
const { heuristicTokens } = this._tokenizer.tokenizeHeuristically(builder, startLineNumber, endLineNumber);
|
||||
const changedTokens = this.setTokens(builder.finalize());
|
||||
|
||||
if (heuristicTokens) {
|
||||
// We overrode tokens with heuristically computed ones.
|
||||
// Because old states might get reused (thus stopping invalidation),
|
||||
// we have to explicitly request the tokens for the changed ranges again.
|
||||
for (const c of changedTokens.changes) {
|
||||
this._backgroundTokenizer.value?.requestTokens(c.fromLineNumber, c.toLineNumber + 1);
|
||||
}
|
||||
}
|
||||
|
||||
this._defaultBackgroundTokenizer?.checkFinished();
|
||||
}
|
||||
|
||||
public forceTokenization(lineNumber: number): void {
|
||||
const builder = new ContiguousMultilineTokensBuilder();
|
||||
this._tokenizer?.updateTokensUntilLine(builder, lineNumber);
|
||||
this.setTokens(builder.finalize());
|
||||
this._defaultBackgroundTokenizer?.checkFinished();
|
||||
}
|
||||
|
||||
public isCheapToTokenize(lineNumber: number): boolean {
|
||||
if (!this._tokenizer) {
|
||||
return true;
|
||||
}
|
||||
return this._tokenizer.isCheapToTokenize(lineNumber);
|
||||
}
|
||||
|
||||
public tokenizeIfCheap(lineNumber: number): void {
|
||||
if (this.isCheapToTokenize(lineNumber)) {
|
||||
this.forceTokenization(lineNumber);
|
||||
}
|
||||
}
|
||||
|
||||
public getLineTokens(lineNumber: number): LineTokens {
|
||||
const lineText = this._textModel.getLineContent(lineNumber);
|
||||
return this._tokens.getTokens(
|
||||
this._textModel.getLanguageId(),
|
||||
lineNumber - 1,
|
||||
lineText
|
||||
);
|
||||
}
|
||||
|
||||
public getTokenTypeIfInsertingCharacter(lineNumber: number, column: number, character: string): StandardTokenType {
|
||||
if (!this._tokenizer) {
|
||||
return StandardTokenType.Other;
|
||||
}
|
||||
|
||||
const position = this._textModel.validatePosition(new Position(lineNumber, column));
|
||||
this.forceTokenization(position.lineNumber);
|
||||
return this._tokenizer.getTokenTypeIfInsertingCharacter(position, character);
|
||||
}
|
||||
|
||||
public tokenizeLineWithEdit(position: IPosition, length: number, newText: string): LineTokens | null {
|
||||
if (!this._tokenizer) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const validatedPosition = this._textModel.validatePosition(position);
|
||||
this.forceTokenization(validatedPosition.lineNumber);
|
||||
return this._tokenizer.tokenizeLineWithEdit(validatedPosition, length, newText);
|
||||
}
|
||||
|
||||
public get hasTokens(): boolean {
|
||||
return this._tokens.hasTokens;
|
||||
}
|
||||
}
|
||||
|
||||
function initializeTokenization(textModel: TextModel, languageId: string): [ITokenizationSupport, IState] | [null, null] {
|
||||
if (textModel.isTooLargeForTokenization()) {
|
||||
return [null, null];
|
||||
}
|
||||
const tokenizationSupport = TokenizationRegistry.get(languageId);
|
||||
if (!tokenizationSupport) {
|
||||
return [null, null];
|
||||
}
|
||||
let initialState: IState;
|
||||
try {
|
||||
initialState = tokenizationSupport.getInitialState();
|
||||
} catch (e) {
|
||||
onUnexpectedError(e);
|
||||
return [null, null];
|
||||
}
|
||||
return [tokenizationSupport, initialState];
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@
|
|||
import { IPosition } from 'vs/editor/common/core/position';
|
||||
import { Range } from 'vs/editor/common/core/range';
|
||||
import { StandardTokenType } from 'vs/editor/common/encodedTokenAttributes';
|
||||
import { ContiguousMultilineTokens } from 'vs/editor/common/tokens/contiguousMultilineTokens';
|
||||
import { LineTokens } from 'vs/editor/common/tokens/lineTokens';
|
||||
import { SparseMultilineTokens } from 'vs/editor/common/tokens/sparseMultilineTokens';
|
||||
|
||||
|
|
@ -14,11 +13,6 @@ import { SparseMultilineTokens } from 'vs/editor/common/tokens/sparseMultilineTo
|
|||
* Provides tokenization related functionality of the text model.
|
||||
*/
|
||||
export interface ITokenizationTextModelPart {
|
||||
/**
|
||||
* @internal
|
||||
*/
|
||||
setTokens(tokens: ContiguousMultilineTokens[]): void;
|
||||
|
||||
readonly hasTokens: boolean;
|
||||
|
||||
/**
|
||||
|
|
@ -91,7 +85,7 @@ export interface ITokenizationTextModelPart {
|
|||
/**
|
||||
* @internal
|
||||
*/
|
||||
tokenizeViewport(startLineNumber: number, endLineNumber: number): void;
|
||||
refreshTokens(startLineNumber: number, endLineNumber: number): void;
|
||||
|
||||
getLanguageId(): string;
|
||||
getLanguageIdAtPosition(lineNumber: number, column: number): string;
|
||||
|
|
|
|||
|
|
@ -195,7 +195,7 @@ export class ViewModel extends Disposable implements IViewModel {
|
|||
const modelVisibleRanges = this._toModelVisibleRanges(viewVisibleRange);
|
||||
|
||||
for (const modelVisibleRange of modelVisibleRanges) {
|
||||
this.model.tokenization.tokenizeViewport(modelVisibleRange.startLineNumber, modelVisibleRange.endLineNumber);
|
||||
this.model.tokenization.refreshTokens(modelVisibleRange.startLineNumber, modelVisibleRange.endLineNumber);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -10,6 +10,9 @@ import { computeIndentLevel } from 'vs/editor/common/model/utils';
|
|||
import { MetadataConsts } from 'vs/editor/common/encodedTokenAttributes';
|
||||
import { TestLineToken, TestLineTokenFactory } from 'vs/editor/test/common/core/testLineToken';
|
||||
import { createTextModel } from 'vs/editor/test/common/testTextModel';
|
||||
import { ITokenizationSupport, TokenizationRegistry, IState, IBackgroundTokenizationStore, EncodedTokenizationResult, TokenizationResult, IBackgroundTokenizer } from 'vs/editor/common/languages';
|
||||
import { ITextModel } from 'vs/editor/common/model';
|
||||
import { ContiguousMultilineTokensBuilder } from 'vs/editor/common/tokens/contiguousMultilineTokensBuilder';
|
||||
|
||||
interface ILineEdit {
|
||||
startColumn: number;
|
||||
|
|
@ -93,6 +96,56 @@ class TestToken {
|
|||
}
|
||||
}
|
||||
|
||||
class ManualTokenizationSupport implements ITokenizationSupport {
|
||||
private readonly tokens = new Map<number, Uint32Array>();
|
||||
private readonly stores = new Set<IBackgroundTokenizationStore>();
|
||||
|
||||
public setLineTokens(lineNumber: number, tokens: Uint32Array): void {
|
||||
const b = new ContiguousMultilineTokensBuilder();
|
||||
b.add(lineNumber, tokens);
|
||||
for (const s of this.stores) {
|
||||
s.setTokens(b.finalize());
|
||||
}
|
||||
}
|
||||
|
||||
getInitialState(): IState {
|
||||
return new LineState(1);
|
||||
}
|
||||
|
||||
tokenize(line: string, hasEOL: boolean, state: IState): TokenizationResult {
|
||||
throw new Error();
|
||||
}
|
||||
|
||||
tokenizeEncoded(line: string, hasEOL: boolean, state: IState): EncodedTokenizationResult {
|
||||
const s = state as LineState;
|
||||
return new EncodedTokenizationResult(this.tokens.get(s.lineNumber)!, new LineState(s.lineNumber + 1));
|
||||
}
|
||||
|
||||
/**
|
||||
* Can be/return undefined if default background tokenization should be used.
|
||||
*/
|
||||
createBackgroundTokenizer?(textModel: ITextModel, store: IBackgroundTokenizationStore): IBackgroundTokenizer | undefined {
|
||||
this.stores.add(store);
|
||||
return {
|
||||
dispose: () => {
|
||||
this.stores.delete(store);
|
||||
},
|
||||
requestTokens(startLineNumber, endLineNumberExclusive) {
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
class LineState implements IState {
|
||||
constructor(public readonly lineNumber: number) { }
|
||||
clone(): IState {
|
||||
return this;
|
||||
}
|
||||
equals(other: IState): boolean {
|
||||
return (other as LineState).lineNumber === this.lineNumber;
|
||||
}
|
||||
}
|
||||
|
||||
suite('ModelLinesTokens', () => {
|
||||
|
||||
interface IBufferLineState {
|
||||
|
|
@ -107,13 +160,18 @@ suite('ModelLinesTokens', () => {
|
|||
|
||||
function testApplyEdits(initial: IBufferLineState[], edits: IEdit[], expected: IBufferLineState[]): void {
|
||||
const initialText = initial.map(el => el.text).join('\n');
|
||||
|
||||
const s = new ManualTokenizationSupport();
|
||||
const d = TokenizationRegistry.register('test', s);
|
||||
|
||||
const model = createTextModel(initialText, 'test');
|
||||
model.onBeforeAttached();
|
||||
for (let lineIndex = 0; lineIndex < initial.length; lineIndex++) {
|
||||
const lineTokens = initial[lineIndex].tokens;
|
||||
const lineTextLength = model.getLineMaxColumn(lineIndex + 1) - 1;
|
||||
const tokens = TestToken.toTokens(lineTokens);
|
||||
LineTokens.convertToEndOffset(tokens, lineTextLength);
|
||||
model.setLineTokens(lineIndex + 1, tokens);
|
||||
s.setLineTokens(lineIndex + 1, tokens);
|
||||
}
|
||||
|
||||
model.applyEdits(edits.map((ed) => ({
|
||||
|
|
@ -131,6 +189,7 @@ suite('ModelLinesTokens', () => {
|
|||
}
|
||||
|
||||
model.dispose();
|
||||
d.dispose();
|
||||
}
|
||||
|
||||
test('single delete 1', () => {
|
||||
|
|
@ -445,17 +504,20 @@ suite('ModelLinesTokens', () => {
|
|||
}
|
||||
|
||||
test('insertion on empty line', () => {
|
||||
const s = new ManualTokenizationSupport();
|
||||
const d = TokenizationRegistry.register('test', s);
|
||||
|
||||
const model = createTextModel('some text', 'test');
|
||||
const tokens = TestToken.toTokens([new TestToken(0, 1)]);
|
||||
LineTokens.convertToEndOffset(tokens, model.getLineMaxColumn(1) - 1);
|
||||
model.setLineTokens(1, tokens);
|
||||
s.setLineTokens(1, tokens);
|
||||
|
||||
model.applyEdits([{
|
||||
range: new Range(1, 1, 1, 10),
|
||||
text: ''
|
||||
}]);
|
||||
|
||||
model.setLineTokens(1, new Uint32Array(0));
|
||||
s.setLineTokens(1, new Uint32Array(0));
|
||||
|
||||
model.applyEdits([{
|
||||
range: new Range(1, 1, 1, 1),
|
||||
|
|
@ -466,6 +528,7 @@ suite('ModelLinesTokens', () => {
|
|||
assertLineTokens(actualTokens, [new TestToken(0, 1)]);
|
||||
|
||||
model.dispose();
|
||||
d.dispose();
|
||||
});
|
||||
|
||||
test('updates tokens on insertion 1', () => {
|
||||
|
|
|
|||
Loading…
Reference in a new issue