Reduce number of times MD docs are re-tokenized (#152674)

This change reduces the number of times we retokenize a markdown file by doing the following:

- Use `MdTableOfContentsProvider` in more places
- Introduce a `IMarkdownParser` interface that lets us drop in a caching version of the tokenizer
This commit is contained in:
Matt Bierner 2022-06-20 23:43:01 -07:00 committed by GitHub
parent 10e6e87682
commit 2249b171f4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
23 changed files with 190 additions and 143 deletions

View file

@ -5,7 +5,7 @@
import * as vscode from 'vscode';
import { Command } from '../commandManager';
import { MarkdownEngine } from '../markdownEngine';
import { MdTableOfContentsProvider } from '../tableOfContents';
import { openDocumentLink } from '../util/openDocumentLink';
type UriComponents = {
@ -48,13 +48,13 @@ export class OpenDocumentLinkCommand implements Command {
}
public constructor(
private readonly engine: MarkdownEngine
private readonly tocProvider: MdTableOfContentsProvider,
) { }
public async execute(args: OpenDocumentLinkArgs) {
const fromResource = vscode.Uri.parse('').with(args.fromResource);
const targetResource = reviveUri(args.parts).with({ fragment: args.fragment });
return openDocumentLink(this.engine, targetResource, fromResource);
return openDocumentLink(this.tocProvider, targetResource, fromResource);
}
}

View file

@ -4,7 +4,7 @@
*--------------------------------------------------------------------------------------------*/
import { Command } from '../commandManager';
import { MarkdownEngine } from '../markdownEngine';
import { MarkdownItEngine } from '../markdownEngine';
import { MarkdownPreviewManager } from '../preview/previewManager';
export class RefreshPreviewCommand implements Command {
@ -12,7 +12,7 @@ export class RefreshPreviewCommand implements Command {
public constructor(
private readonly webviewManager: MarkdownPreviewManager,
private readonly engine: MarkdownEngine
private readonly engine: MarkdownItEngine
) { }
public execute() {

View file

@ -4,7 +4,7 @@
*--------------------------------------------------------------------------------------------*/
import { Command } from '../commandManager';
import { MarkdownEngine } from '../markdownEngine';
import { MarkdownItEngine } from '../markdownEngine';
import { MarkdownPreviewManager } from '../preview/previewManager';
export class ReloadPlugins implements Command {
@ -12,7 +12,7 @@ export class ReloadPlugins implements Command {
public constructor(
private readonly webviewManager: MarkdownPreviewManager,
private readonly engine: MarkdownEngine,
private readonly engine: MarkdownItEngine,
) { }
public execute(): void {

View file

@ -4,14 +4,14 @@
*--------------------------------------------------------------------------------------------*/
import { Command } from '../commandManager';
import { MarkdownEngine } from '../markdownEngine';
import { MarkdownItEngine } from '../markdownEngine';
import { SkinnyTextDocument } from '../workspaceContents';
export class RenderDocument implements Command {
public readonly id = 'markdown.api.render';
public constructor(
private readonly engine: MarkdownEngine
private readonly engine: MarkdownItEngine
) { }
public async execute(document: SkinnyTextDocument | string): Promise<string> {

View file

@ -20,7 +20,7 @@ import { registerRenameSupport } from './languageFeatures/rename';
import { registerSmartSelectSupport } from './languageFeatures/smartSelect';
import { registerWorkspaceSymbolSupport } from './languageFeatures/workspaceSymbolProvider';
import { Logger } from './logger';
import { MarkdownEngine } from './markdownEngine';
import { MarkdownItEngine, IMdParser, MdParsingProvider } from './markdownEngine';
import { getMarkdownExtensionContributions } from './markdownExtensions';
import { MarkdownContentProvider } from './preview/previewContentProvider';
import { MarkdownPreviewManager } from './preview/previewManager';
@ -28,7 +28,7 @@ import { ContentSecurityPolicyArbiter, ExtensionContentSecurityPolicyArbiter, Pr
import { githubSlugifier } from './slugify';
import { MdTableOfContentsProvider } from './tableOfContents';
import { loadDefaultTelemetryReporter, TelemetryReporter } from './telemetryReporter';
import { VsCodeMdWorkspaceContents } from './workspaceContents';
import { MdWorkspaceContents, VsCodeMdWorkspaceContents } from './workspaceContents';
export function activate(context: vscode.ExtensionContext) {
@ -39,16 +39,21 @@ export function activate(context: vscode.ExtensionContext) {
context.subscriptions.push(contributions);
const cspArbiter = new ExtensionContentSecurityPolicyArbiter(context.globalState, context.workspaceState);
const engine = new MarkdownEngine(contributions, githubSlugifier);
const logger = new Logger();
const commandManager = new CommandManager();
const engine = new MarkdownItEngine(contributions, githubSlugifier);
const workspaceContents = new VsCodeMdWorkspaceContents();
const parser = new MdParsingProvider(engine, workspaceContents);
const tocProvider = new MdTableOfContentsProvider(parser, workspaceContents);
context.subscriptions.push(workspaceContents, parser, tocProvider);
const contentProvider = new MarkdownContentProvider(engine, context, cspArbiter, contributions, logger);
const previewManager = new MarkdownPreviewManager(contentProvider, logger, contributions, engine);
const previewManager = new MarkdownPreviewManager(contentProvider, logger, contributions, tocProvider);
context.subscriptions.push(previewManager);
context.subscriptions.push(registerMarkdownLanguageFeatures(commandManager, engine));
context.subscriptions.push(registerMarkdownCommands(commandManager, previewManager, telemetryReporter, cspArbiter, engine));
context.subscriptions.push(registerMarkdownLanguageFeatures(parser, workspaceContents, commandManager, tocProvider));
context.subscriptions.push(registerMarkdownCommands(commandManager, previewManager, telemetryReporter, cspArbiter, engine, tocProvider));
context.subscriptions.push(vscode.workspace.onDidChangeConfiguration(() => {
logger.updateConfiguration();
@ -57,37 +62,34 @@ export function activate(context: vscode.ExtensionContext) {
}
function registerMarkdownLanguageFeatures(
parser: IMdParser,
workspaceContents: MdWorkspaceContents,
commandManager: CommandManager,
engine: MarkdownEngine
tocProvider: MdTableOfContentsProvider,
): vscode.Disposable {
const selector: vscode.DocumentSelector = { language: 'markdown', scheme: '*' };
const workspaceContents = new VsCodeMdWorkspaceContents();
const linkProvider = new MdLinkProvider(engine, workspaceContents);
const tocProvider = new MdTableOfContentsProvider(engine, workspaceContents);
const referencesProvider = new MdReferencesProvider(engine, workspaceContents, tocProvider);
const linkProvider = new MdLinkProvider(parser, workspaceContents);
const referencesProvider = new MdReferencesProvider(parser, workspaceContents, tocProvider);
const symbolProvider = new MdDocumentSymbolProvider(tocProvider);
return vscode.Disposable.from(
workspaceContents,
linkProvider,
referencesProvider,
tocProvider,
// Language features
registerDefinitionSupport(selector, referencesProvider),
registerDiagnosticSupport(selector, engine, workspaceContents, linkProvider, commandManager, referencesProvider, tocProvider),
registerDiagnosticSupport(selector, workspaceContents, linkProvider, commandManager, referencesProvider, tocProvider),
registerDocumentLinkSupport(selector, linkProvider),
registerDocumentSymbolSupport(selector, tocProvider),
registerDropIntoEditorSupport(selector),
registerFindFileReferenceSupport(commandManager, referencesProvider),
registerFoldingSupport(selector, engine, tocProvider),
registerFoldingSupport(selector, parser, tocProvider),
registerPasteSupport(selector),
registerPathCompletionSupport(selector, engine, linkProvider),
registerPathCompletionSupport(selector, parser, linkProvider),
registerReferencesSupport(selector, referencesProvider),
registerRenameSupport(selector, workspaceContents, referencesProvider, engine.slugifier),
registerSmartSelectSupport(selector, engine, tocProvider),
registerRenameSupport(selector, workspaceContents, referencesProvider, parser.slugifier),
registerSmartSelectSupport(selector, parser, tocProvider),
registerWorkspaceSymbolSupport(workspaceContents, symbolProvider),
);
}
@ -97,7 +99,8 @@ function registerMarkdownCommands(
previewManager: MarkdownPreviewManager,
telemetryReporter: TelemetryReporter,
cspArbiter: ContentSecurityPolicyArbiter,
engine: MarkdownEngine
engine: MarkdownItEngine,
tocProvider: MdTableOfContentsProvider,
): vscode.Disposable {
const previewSecuritySelector = new PreviewSecuritySelector(cspArbiter, previewManager);
@ -108,7 +111,7 @@ function registerMarkdownCommands(
commandManager.register(new commands.RefreshPreviewCommand(previewManager, engine));
commandManager.register(new commands.MoveCursorToPositionCommand());
commandManager.register(new commands.ShowPreviewSecuritySelectorCommand(previewSecuritySelector, previewManager));
commandManager.register(new commands.OpenDocumentLinkCommand(engine));
commandManager.register(new commands.OpenDocumentLinkCommand(tocProvider));
commandManager.register(new commands.ToggleLockCommand(previewManager));
commandManager.register(new commands.RenderDocument(engine));
commandManager.register(new commands.ReloadPlugins(previewManager, engine));

View file

@ -7,7 +7,6 @@ import * as picomatch from 'picomatch';
import * as vscode from 'vscode';
import * as nls from 'vscode-nls';
import { CommandManager } from '../commandManager';
import { MarkdownEngine } from '../markdownEngine';
import { MdTableOfContentsProvider } from '../tableOfContents';
import { MdTableOfContentsWatcher } from '../test/tableOfContentsWatcher';
import { Delayer } from '../util/async';
@ -305,12 +304,12 @@ export class DiagnosticManager extends Disposable {
public readonly ready: Promise<void>;
constructor(
engine: MarkdownEngine,
private readonly workspaceContents: MdWorkspaceContents,
private readonly computer: DiagnosticComputer,
private readonly configuration: DiagnosticConfiguration,
private readonly reporter: DiagnosticReporter,
private readonly referencesProvider: MdReferencesProvider,
tocProvider: MdTableOfContentsProvider,
delay = 300,
) {
super();
@ -346,7 +345,7 @@ export class DiagnosticManager extends Disposable {
}
}));
this.tableOfContentsWatcher = this._register(new MdTableOfContentsWatcher(engine, workspaceContents));
this.tableOfContentsWatcher = this._register(new MdTableOfContentsWatcher(workspaceContents, tocProvider));
this._register(this.tableOfContentsWatcher.onTocChanged(async e => {
// When the toc of a document changes, revalidate every file that linked to it too
const triggered = new ResourceMap<void>();
@ -638,7 +637,6 @@ class AddToIgnoreLinksQuickFixProvider implements vscode.CodeActionProvider {
export function registerDiagnosticSupport(
selector: vscode.DocumentSelector,
engine: MarkdownEngine,
workspaceContents: MdWorkspaceContents,
linkProvider: MdLinkProvider,
commandManager: CommandManager,
@ -647,12 +645,12 @@ export function registerDiagnosticSupport(
): vscode.Disposable {
const configuration = new VSCodeDiagnosticConfiguration();
const manager = new DiagnosticManager(
engine,
workspaceContents,
new DiagnosticComputer(workspaceContents, linkProvider, tocProvider),
configuration,
new DiagnosticCollectionReporter(),
referenceProvider);
referenceProvider,
tocProvider);
return vscode.Disposable.from(
configuration,
manager,

View file

@ -7,7 +7,7 @@ import * as vscode from 'vscode';
import * as nls from 'vscode-nls';
import * as uri from 'vscode-uri';
import { OpenDocumentLinkCommand } from '../commands/openDocumentLink';
import { MarkdownEngine } from '../markdownEngine';
import { IMdParser } from '../markdownEngine';
import { coalesce } from '../util/arrays';
import { noopToken } from '../util/cancellation';
import { Disposable } from '../util/dispose';
@ -233,8 +233,8 @@ const definitionPattern = /^([\t ]*\[(?!\^)((?:\\\]|[^\]])+)\]:\s*)([^<]\S*|<[^>
const inlineCodePattern = /(?:^|[^`])(`+)(?:.+?|.*?(?:(?:\r?\n).+?)*?)(?:\r?\n)?\1(?:$|[^`])/gm;
class NoLinkRanges {
public static async compute(document: SkinnyTextDocument, engine: MarkdownEngine): Promise<NoLinkRanges> {
const tokens = await engine.parse(document);
public static async compute(tokenizer: IMdParser, document: SkinnyTextDocument): Promise<NoLinkRanges> {
const tokens = await tokenizer.tokenize(document);
const multiline = tokens.filter(t => (t.type === 'code_block' || t.type === 'fence' || t.type === 'html_block') && !!t.map).map(t => t.map) as [number, number][];
const text = document.getText();
@ -270,11 +270,11 @@ class NoLinkRanges {
export class MdLinkComputer {
constructor(
private readonly engine: MarkdownEngine
private readonly tokenizer: IMdParser,
) { }
public async getAllLinks(document: SkinnyTextDocument, token: vscode.CancellationToken): Promise<MdLink[]> {
const noLinkRanges = await NoLinkRanges.compute(document, this.engine);
const noLinkRanges = await NoLinkRanges.compute(this.tokenizer, document);
if (token.isCancellationRequested) {
return [];
}
@ -436,11 +436,11 @@ export class MdLinkProvider extends Disposable {
private readonly linkComputer: MdLinkComputer;
constructor(
engine: MarkdownEngine,
tokenizer: IMdParser,
workspaceContents: MdWorkspaceContents,
) {
super();
this.linkComputer = new MdLinkComputer(engine);
this.linkComputer = new MdLinkComputer(tokenizer);
this._linkCache = this._register(new MdDocumentInfoCache(workspaceContents, doc => this.linkComputer.getAllLinks(doc, noopToken)));
}

View file

@ -3,9 +3,9 @@
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import Token = require('markdown-it/lib/token');
import type Token = require('markdown-it/lib/token');
import * as vscode from 'vscode';
import { MarkdownEngine } from '../markdownEngine';
import { IMdParser } from '../markdownEngine';
import { MdTableOfContentsProvider } from '../tableOfContents';
import { SkinnyTextDocument } from '../workspaceContents';
@ -18,7 +18,7 @@ interface MarkdownItTokenWithMap extends Token {
export class MdFoldingProvider implements vscode.FoldingRangeProvider {
constructor(
private readonly engine: MarkdownEngine,
private readonly parser: IMdParser,
private readonly tocProvide: MdTableOfContentsProvider,
) { }
@ -36,7 +36,7 @@ export class MdFoldingProvider implements vscode.FoldingRangeProvider {
}
private async getRegions(document: SkinnyTextDocument): Promise<vscode.FoldingRange[]> {
const tokens = await this.engine.parse(document);
const tokens = await this.parser.tokenize(document);
const regionMarkers = tokens.filter(isRegionMarker)
.map(token => ({ line: token.map[0], isStart: isStartRegion(token.content) }));
@ -67,7 +67,7 @@ export class MdFoldingProvider implements vscode.FoldingRangeProvider {
}
private async getBlockFoldingRanges(document: SkinnyTextDocument): Promise<vscode.FoldingRange[]> {
const tokens = await this.engine.parse(document);
const tokens = await this.parser.tokenize(document);
const multiLineListItems = tokens.filter(isFoldableToken);
return multiLineListItems.map(listItem => {
const start = listItem.map[0];
@ -115,8 +115,8 @@ const isFoldableToken = (token: Token): token is MarkdownItTokenWithMap => {
export function registerFoldingSupport(
selector: vscode.DocumentSelector,
engine: MarkdownEngine,
parser: IMdParser,
tocProvider: MdTableOfContentsProvider,
): vscode.Disposable {
return vscode.languages.registerFoldingRangeProvider(selector, new MdFoldingProvider(engine, tocProvider));
return vscode.languages.registerFoldingRangeProvider(selector, new MdFoldingProvider(parser, tocProvider));
}

View file

@ -5,7 +5,7 @@
import { dirname, resolve } from 'path';
import * as vscode from 'vscode';
import { MarkdownEngine } from '../markdownEngine';
import { IMdParser } from '../markdownEngine';
import { TableOfContents } from '../tableOfContents';
import { resolveUriToMarkdownFile } from '../util/openDocumentLink';
import { SkinnyTextDocument } from '../workspaceContents';
@ -82,7 +82,7 @@ function tryDecodeUriComponent(str: string): string {
export class MdVsCodePathCompletionProvider implements vscode.CompletionItemProvider {
constructor(
private readonly engine: MarkdownEngine,
private readonly parser: IMdParser,
private readonly linkProvider: MdLinkProvider,
) { }
@ -249,7 +249,7 @@ export class MdVsCodePathCompletionProvider implements vscode.CompletionItemProv
}
private async *provideHeaderSuggestions(document: SkinnyTextDocument, position: vscode.Position, context: CompletionContext, insertionRange: vscode.Range): AsyncIterable<vscode.CompletionItem> {
const toc = await TableOfContents.createForDocumentOrNotebook(this.engine, document);
const toc = await TableOfContents.createForDocumentOrNotebook(this.parser, document);
for (const entry of toc.entries) {
const replacementRange = new vscode.Range(insertionRange.start, position.translate({ characterDelta: context.linkSuffix.length }));
yield {
@ -349,8 +349,8 @@ export class MdVsCodePathCompletionProvider implements vscode.CompletionItemProv
export function registerPathCompletionSupport(
selector: vscode.DocumentSelector,
engine: MarkdownEngine,
parser: IMdParser,
linkProvider: MdLinkProvider,
): vscode.Disposable {
return vscode.languages.registerCompletionItemProvider(selector, new MdVsCodePathCompletionProvider(engine, linkProvider), '.', '/', '#');
return vscode.languages.registerCompletionItemProvider(selector, new MdVsCodePathCompletionProvider(parser, linkProvider), '.', '/', '#');
}

View file

@ -4,7 +4,7 @@
*--------------------------------------------------------------------------------------------*/
import * as vscode from 'vscode';
import * as uri from 'vscode-uri';
import { MarkdownEngine } from '../markdownEngine';
import { IMdParser } from '../markdownEngine';
import { MdTableOfContentsProvider, TocEntry } from '../tableOfContents';
import { noopToken } from '../util/cancellation';
import { Disposable } from '../util/dispose';
@ -68,13 +68,13 @@ export class MdReferencesProvider extends Disposable {
private readonly _linkComputer: MdLinkComputer;
public constructor(
private readonly engine: MarkdownEngine,
private readonly parser: IMdParser,
private readonly workspaceContents: MdWorkspaceContents,
private readonly tocProvider: MdTableOfContentsProvider,
) {
super();
this._linkComputer = new MdLinkComputer(engine);
this._linkComputer = new MdLinkComputer(parser);
this._linkCache = this._register(new MdWorkspaceInfoCache(workspaceContents, doc => this._linkComputer.getAllLinks(doc, noopToken)));
}
@ -114,7 +114,7 @@ export class MdReferencesProvider extends Disposable {
for (const link of links) {
if (link.href.kind === 'internal'
&& this.looksLikeLinkToDoc(link.href, document.uri)
&& this.engine.slugifier.fromHeading(link.href.fragment).value === header.slug.value
&& this.parser.slugifier.fromHeading(link.href.fragment).value === header.slug.value
) {
references.push({
kind: 'link',
@ -204,7 +204,7 @@ export class MdReferencesProvider extends Disposable {
continue;
}
if (this.engine.slugifier.fromHeading(link.href.fragment).equals(this.engine.slugifier.fromHeading(sourceLink.href.fragment))) {
if (this.parser.slugifier.fromHeading(link.href.fragment).equals(this.parser.slugifier.fromHeading(sourceLink.href.fragment))) {
const isTriggerLocation = sourceLink.source.resource.fsPath === link.source.resource.fsPath && sourceLink.source.hrefRange.isEqual(link.source.hrefRange);
references.push({
kind: 'link',

View file

@ -4,7 +4,7 @@
*--------------------------------------------------------------------------------------------*/
import Token = require('markdown-it/lib/token');
import * as vscode from 'vscode';
import { MarkdownEngine } from '../markdownEngine';
import { IMdParser } from '../markdownEngine';
import { MdTableOfContentsProvider, TocEntry } from '../tableOfContents';
import { SkinnyTextDocument } from '../workspaceContents';
@ -15,7 +15,7 @@ interface MarkdownItTokenWithMap extends Token {
export class MdSmartSelect implements vscode.SelectionRangeProvider {
constructor(
private readonly engine: MarkdownEngine,
private readonly parser: IMdParser,
private readonly tocProvider: MdTableOfContentsProvider,
) { }
@ -37,9 +37,7 @@ export class MdSmartSelect implements vscode.SelectionRangeProvider {
}
private async getBlockSelectionRange(document: SkinnyTextDocument, position: vscode.Position, headerRange?: vscode.SelectionRange): Promise<vscode.SelectionRange | undefined> {
const tokens = await this.engine.parse(document);
const tokens = await this.parser.tokenize(document);
const blockTokens = getBlockTokensForPosition(tokens, position, headerRange);
if (blockTokens.length === 0) {
@ -253,8 +251,8 @@ function getFirstChildHeader(document: SkinnyTextDocument, header?: TocEntry, to
export function registerSmartSelectSupport(
selector: vscode.DocumentSelector,
engine: MarkdownEngine,
parser: IMdParser,
tocProvider: MdTableOfContentsProvider,
): vscode.Disposable {
return vscode.languages.registerSelectionRangeProvider(selector, new MdSmartSelect(engine, tocProvider));
return vscode.languages.registerSelectionRangeProvider(selector, new MdSmartSelect(parser, tocProvider));
}

View file

@ -65,6 +65,15 @@ export class MdDocumentInfoCache<T> extends Disposable {
return doc && this.onDidChangeDocument(doc, true)?.value;
}
public async getForDocument(document: SkinnyTextDocument): Promise<T> {
const existing = this._cache.get(document.uri);
if (existing) {
return existing;
}
return this.onDidChangeDocument(document, true)!.value;
}
public async entries(): Promise<Array<[vscode.Uri, T]>> {
return this._cache.entries();
}

View file

@ -3,15 +3,17 @@
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import MarkdownIt = require('markdown-it');
import Token = require('markdown-it/lib/token');
import type MarkdownIt = require('markdown-it');
import type Token = require('markdown-it/lib/token');
import * as vscode from 'vscode';
import { MdDocumentInfoCache } from './languageFeatures/workspaceCache';
import { MarkdownContributionProvider } from './markdownExtensions';
import { Slugifier } from './slugify';
import { Disposable } from './util/dispose';
import { stringHash } from './util/hash';
import { WebviewResourceProvider } from './util/resources';
import { isOfScheme, Schemes } from './util/schemes';
import { SkinnyTextDocument } from './workspaceContents';
import { MdWorkspaceContents, SkinnyTextDocument } from './workspaceContents';
const UNICODE_NEWLINE_REGEX = /\u2028|\u2029/g;
@ -91,7 +93,12 @@ interface RenderEnv {
resourceProvider: WebviewResourceProvider | undefined;
}
export class MarkdownEngine {
export interface IMdParser {
readonly slugifier: Slugifier;
tokenize(document: SkinnyTextDocument): Promise<Token[]>;
}
export class MarkdownItEngine implements IMdParser {
private md?: Promise<MarkdownIt>;
@ -213,7 +220,7 @@ export class MarkdownEngine {
};
}
public async parse(document: SkinnyTextDocument): Promise<Token[]> {
public async tokenize(document: SkinnyTextDocument): Promise<Token[]> {
const config = this.getConfig(document.uri);
const engine = await this.getEngine(config);
return this.tokenizeDocument(document, config, engine);
@ -427,3 +434,27 @@ function normalizeHighlightLang(lang: string | undefined) {
return lang;
}
}
export class MdParsingProvider extends Disposable implements IMdParser {
private readonly _cache: MdDocumentInfoCache<Token[]>;
public readonly slugifier: Slugifier;
constructor(
engine: MarkdownItEngine,
workspaceContents: MdWorkspaceContents,
) {
super();
this.slugifier = engine.slugifier;
this._cache = this._register(new MdDocumentInfoCache<Token[]>(workspaceContents, doc => {
return engine.tokenize(doc);
}));
}
public tokenize(document: SkinnyTextDocument): Promise<Token[]> {
return this._cache.getForDocument(document);
}
}

View file

@ -7,8 +7,8 @@ import * as vscode from 'vscode';
import * as nls from 'vscode-nls';
import * as uri from 'vscode-uri';
import { Logger } from '../logger';
import { MarkdownEngine } from '../markdownEngine';
import { MarkdownContributionProvider } from '../markdownExtensions';
import { MdTableOfContentsProvider } from '../tableOfContents';
import { Disposable } from '../util/dispose';
import { isMarkdownFile } from '../util/file';
import { openDocumentLink, resolveDocumentLink, resolveUriToMarkdownFile } from '../util/openDocumentLink';
@ -116,11 +116,11 @@ class MarkdownPreview extends Disposable implements WebviewResourceProvider {
resource: vscode.Uri,
startingScroll: StartingScrollLocation | undefined,
private readonly delegate: MarkdownPreviewDelegate,
private readonly engine: MarkdownEngine,
private readonly _contentProvider: MarkdownContentProvider,
private readonly _previewConfigurations: MarkdownPreviewConfigurationManager,
private readonly _logger: Logger,
private readonly _contributionProvider: MarkdownContributionProvider,
private readonly _tocProvider: MdTableOfContentsProvider,
) {
super();
@ -456,7 +456,7 @@ class MarkdownPreview extends Disposable implements WebviewResourceProvider {
}
}
return openDocumentLink(this.engine, targetResource, this.resource);
return openDocumentLink(this._tocProvider, targetResource, this.resource);
}
//#region WebviewResourceProvider
@ -504,10 +504,10 @@ export class StaticMarkdownPreview extends Disposable implements ManagedMarkdown
topmostLineMonitor: TopmostLineMonitor,
logger: Logger,
contributionProvider: MarkdownContributionProvider,
engine: MarkdownEngine,
tocProvider: MdTableOfContentsProvider,
scrollLine?: number,
): StaticMarkdownPreview {
return new StaticMarkdownPreview(webview, resource, contentProvider, previewConfigurations, topmostLineMonitor, logger, contributionProvider, engine, scrollLine);
return new StaticMarkdownPreview(webview, resource, contentProvider, previewConfigurations, topmostLineMonitor, logger, contributionProvider, tocProvider, scrollLine);
}
private readonly preview: MarkdownPreview;
@ -520,7 +520,7 @@ export class StaticMarkdownPreview extends Disposable implements ManagedMarkdown
topmostLineMonitor: TopmostLineMonitor,
logger: Logger,
contributionProvider: MarkdownContributionProvider,
engine: MarkdownEngine,
tocProvider: MdTableOfContentsProvider,
scrollLine?: number,
) {
super();
@ -532,7 +532,7 @@ export class StaticMarkdownPreview extends Disposable implements ManagedMarkdown
fragment
}), StaticMarkdownPreview.customEditorViewType, this._webviewPanel.viewColumn);
}
}, engine, contentProvider, _previewConfigurations, logger, contributionProvider));
}, contentProvider, _previewConfigurations, logger, contributionProvider, tocProvider));
this._register(this._webviewPanel.onDidDispose(() => {
this.dispose();
@ -616,12 +616,12 @@ export class DynamicMarkdownPreview extends Disposable implements ManagedMarkdow
logger: Logger,
topmostLineMonitor: TopmostLineMonitor,
contributionProvider: MarkdownContributionProvider,
engine: MarkdownEngine,
tocProvider: MdTableOfContentsProvider,
): DynamicMarkdownPreview {
webview.iconPath = contentProvider.iconPath;
return new DynamicMarkdownPreview(webview, input,
contentProvider, previewConfigurations, logger, topmostLineMonitor, contributionProvider, engine);
contentProvider, previewConfigurations, logger, topmostLineMonitor, contributionProvider, tocProvider);
}
public static create(
@ -632,7 +632,7 @@ export class DynamicMarkdownPreview extends Disposable implements ManagedMarkdow
logger: Logger,
topmostLineMonitor: TopmostLineMonitor,
contributionProvider: MarkdownContributionProvider,
engine: MarkdownEngine,
tocProvider: MdTableOfContentsProvider,
): DynamicMarkdownPreview {
const webview = vscode.window.createWebviewPanel(
DynamicMarkdownPreview.viewType,
@ -642,7 +642,7 @@ export class DynamicMarkdownPreview extends Disposable implements ManagedMarkdow
webview.iconPath = contentProvider.iconPath;
return new DynamicMarkdownPreview(webview, input,
contentProvider, previewConfigurations, logger, topmostLineMonitor, contributionProvider, engine);
contentProvider, previewConfigurations, logger, topmostLineMonitor, contributionProvider, tocProvider);
}
private constructor(
@ -653,7 +653,7 @@ export class DynamicMarkdownPreview extends Disposable implements ManagedMarkdow
private readonly _logger: Logger,
private readonly _topmostLineMonitor: TopmostLineMonitor,
private readonly _contributionProvider: MarkdownContributionProvider,
private readonly _engine: MarkdownEngine,
private readonly _tocProvider: MdTableOfContentsProvider,
) {
super();
@ -805,10 +805,10 @@ export class DynamicMarkdownPreview extends Disposable implements ManagedMarkdow
this.update(link, fragment ? new StartingScrollFragment(fragment) : undefined);
}
},
this._engine,
this._contentProvider,
this._previewConfigurations,
this._logger,
this._contributionProvider);
this._contributionProvider,
this._tocProvider);
}
}

View file

@ -7,7 +7,7 @@ import * as vscode from 'vscode';
import * as nls from 'vscode-nls';
import * as uri from 'vscode-uri';
import { Logger } from '../logger';
import { MarkdownEngine } from '../markdownEngine';
import { MarkdownItEngine } from '../markdownEngine';
import { MarkdownContributionProvider } from '../markdownExtensions';
import { WebviewResourceProvider } from '../util/resources';
import { MarkdownPreviewConfiguration, MarkdownPreviewConfigurationManager } from './previewConfig';
@ -47,7 +47,7 @@ export interface MarkdownContentProviderOutput {
export class MarkdownContentProvider {
constructor(
private readonly engine: MarkdownEngine,
private readonly engine: MarkdownItEngine,
private readonly context: vscode.ExtensionContext,
private readonly cspArbiter: ContentSecurityPolicyArbiter,
private readonly contributionProvider: MarkdownContributionProvider,

View file

@ -5,8 +5,8 @@
import * as vscode from 'vscode';
import { Logger } from '../logger';
import { MarkdownEngine } from '../markdownEngine';
import { MarkdownContributionProvider } from '../markdownExtensions';
import { MdTableOfContentsProvider } from '../tableOfContents';
import { Disposable, disposeAll } from '../util/dispose';
import { isMarkdownFile } from '../util/file';
import { DynamicMarkdownPreview, ManagedMarkdownPreview, StaticMarkdownPreview } from './preview';
@ -71,7 +71,7 @@ export class MarkdownPreviewManager extends Disposable implements vscode.Webview
private readonly _contentProvider: MarkdownContentProvider,
private readonly _logger: Logger,
private readonly _contributions: MarkdownContributionProvider,
private readonly _engine: MarkdownEngine,
private readonly _tocProvider: MdTableOfContentsProvider,
) {
super();
@ -166,7 +166,7 @@ export class MarkdownPreviewManager extends Disposable implements vscode.Webview
this._logger,
this._topmostLineMonitor,
this._contributions,
this._engine);
this._tocProvider);
this.registerDynamicPreview(preview);
}
@ -184,7 +184,7 @@ export class MarkdownPreviewManager extends Disposable implements vscode.Webview
this._topmostLineMonitor,
this._logger,
this._contributions,
this._engine,
this._tocProvider,
lineNumber
);
this.registerStaticPreview(preview);
@ -209,7 +209,7 @@ export class MarkdownPreviewManager extends Disposable implements vscode.Webview
this._logger,
this._topmostLineMonitor,
this._contributions,
this._engine);
this._tocProvider);
this.setPreviewActiveContext(true);
this._activePreview = preview;

View file

@ -5,7 +5,7 @@
import * as vscode from 'vscode';
import { MdDocumentInfoCache } from './languageFeatures/workspaceCache';
import { MarkdownEngine } from './markdownEngine';
import { IMdParser } from './markdownEngine';
import { githubSlugifier, Slug, Slugifier } from './slugify';
import { Disposable } from './util/dispose';
import { isMarkdownFile } from './util/file';
@ -63,12 +63,12 @@ export interface TocEntry {
export class TableOfContents {
public static async create(engine: MarkdownEngine, document: SkinnyTextDocument,): Promise<TableOfContents> {
const entries = await this.buildToc(engine, document);
return new TableOfContents(entries, engine.slugifier);
public static async create(parser: IMdParser, document: SkinnyTextDocument,): Promise<TableOfContents> {
const entries = await this.buildToc(parser, document);
return new TableOfContents(entries, parser.slugifier);
}
public static async createForDocumentOrNotebook(engine: MarkdownEngine, document: SkinnyTextDocument): Promise<TableOfContents> {
public static async createForDocumentOrNotebook(parser: IMdParser, document: SkinnyTextDocument): Promise<TableOfContents> {
if (document.uri.scheme === 'vscode-notebook-cell') {
const notebook = vscode.workspace.notebookDocuments
.find(notebook => notebook.getCells().some(cell => cell.document === document));
@ -78,20 +78,20 @@ export class TableOfContents {
for (const cell of notebook.getCells()) {
if (cell.kind === vscode.NotebookCellKind.Markup && isMarkdownFile(cell.document)) {
entries.push(...(await this.buildToc(engine, cell.document)));
entries.push(...(await this.buildToc(parser, cell.document)));
}
}
return new TableOfContents(entries, engine.slugifier);
return new TableOfContents(entries, parser.slugifier);
}
}
return this.create(engine, document);
return this.create(parser, document);
}
private static async buildToc(engine: MarkdownEngine, document: SkinnyTextDocument): Promise<TocEntry[]> {
private static async buildToc(parser: IMdParser, document: SkinnyTextDocument): Promise<TocEntry[]> {
const toc: TocEntry[] = [];
const tokens = await engine.parse(document);
const tokens = await parser.tokenize(document);
const existingSlugEntries = new Map<string, { count: number }>();
@ -103,11 +103,11 @@ export class TableOfContents {
const lineNumber = heading.map[0];
const line = document.lineAt(lineNumber);
let slug = engine.slugifier.fromHeading(line.text);
let slug = parser.slugifier.fromHeading(line.text);
const existingSlugEntry = existingSlugEntries.get(slug.value);
if (existingSlugEntry) {
++existingSlugEntry.count;
slug = engine.slugifier.fromHeading(slug.value + '-' + existingSlugEntry.count);
slug = parser.slugifier.fromHeading(slug.value + '-' + existingSlugEntry.count);
} else {
existingSlugEntries.set(slug.value, { count: 0 });
}
@ -181,16 +181,20 @@ export class MdTableOfContentsProvider extends Disposable {
private readonly _cache: MdDocumentInfoCache<TableOfContents>;
constructor(
engine: MarkdownEngine,
parser: IMdParser,
workspaceContents: MdWorkspaceContents,
) {
super();
this._cache = this._register(new MdDocumentInfoCache<TableOfContents>(workspaceContents, doc => {
return TableOfContents.create(engine, doc);
return TableOfContents.create(parser, doc);
}));
}
public async get(resource: vscode.Uri): Promise<TableOfContents> {
return (await this._cache.get(resource)) ?? TableOfContents.empty;
return await this._cache.get(resource) ?? TableOfContents.empty;
}
public getForDocument(doc: SkinnyTextDocument): Promise<TableOfContents> {
return this._cache.getForDocument(doc);
}
}

View file

@ -440,12 +440,12 @@ suite('Markdown: Diagnostics manager', () => {
const tocProvider = new MdTableOfContentsProvider(engine, workspace);
const referencesProvider = new MdReferencesProvider(engine, workspace, tocProvider);
const manager = new DiagnosticManager(
engine,
workspace,
new DiagnosticComputer(workspace, linkProvider, tocProvider),
configuration,
reporter,
referencesProvider,
tocProvider,
0);
_disposables.push(manager, referencesProvider);
return manager;

View file

@ -4,7 +4,7 @@
*--------------------------------------------------------------------------------------------*/
import * as vscode from 'vscode';
import { MarkdownEngine } from '../markdownEngine';
import { MarkdownItEngine } from '../markdownEngine';
import { MarkdownContributionProvider, MarkdownContributions } from '../markdownExtensions';
import { githubSlugifier } from '../slugify';
import { Disposable } from '../util/dispose';
@ -15,6 +15,6 @@ const emptyContributions = new class extends Disposable implements MarkdownContr
readonly onContributionsChanged = this._register(new vscode.EventEmitter<this>()).event;
};
export function createNewMarkdownEngine(): MarkdownEngine {
return new MarkdownEngine(emptyContributions, githubSlugifier);
export function createNewMarkdownEngine(): MarkdownItEngine {
return new MarkdownItEngine(emptyContributions, githubSlugifier);
}

View file

@ -6,11 +6,11 @@
import * as assert from 'assert';
import * as vscode from 'vscode';
import { MdSmartSelect } from '../languageFeatures/smartSelect';
import { createNewMarkdownEngine } from './engine';
import { InMemoryDocument } from '../util/inMemoryDocument';
import { CURSOR, getCursorPositions, joinLines } from './util';
import { MdTableOfContentsProvider } from '../tableOfContents';
import { InMemoryDocument } from '../util/inMemoryDocument';
import { createNewMarkdownEngine } from './engine';
import { InMemoryWorkspaceMarkdownDocuments } from './inMemoryWorkspace';
import { CURSOR, getCursorPositions, joinLines } from './util';
const testFileName = vscode.Uri.file('test.md');

View file

@ -7,16 +7,22 @@ import * as assert from 'assert';
import 'mocha';
import * as vscode from 'vscode';
import { TableOfContents } from '../tableOfContents';
import { createNewMarkdownEngine } from './engine';
import { InMemoryDocument } from '../util/inMemoryDocument';
import { SkinnyTextDocument } from '../workspaceContents';
import { createNewMarkdownEngine } from './engine';
const testFileName = vscode.Uri.file('test.md');
function createToc(doc: SkinnyTextDocument): Promise<TableOfContents> {
const engine = createNewMarkdownEngine();
return TableOfContents.create(engine, doc);
}
suite('markdown.TableOfContentsProvider', () => {
test('Lookup should not return anything for empty document', async () => {
const doc = new InMemoryDocument(testFileName, '');
const provider = await TableOfContents.create(createNewMarkdownEngine(), doc);
const provider = await createToc(doc);
assert.strictEqual(provider.lookup(''), undefined);
assert.strictEqual(provider.lookup('foo'), undefined);
@ -24,7 +30,7 @@ suite('markdown.TableOfContentsProvider', () => {
test('Lookup should not return anything for document with no headers', async () => {
const doc = new InMemoryDocument(testFileName, 'a *b*\nc');
const provider = await TableOfContents.create(createNewMarkdownEngine(), doc);
const provider = await createToc(doc);
assert.strictEqual(provider.lookup(''), undefined);
assert.strictEqual(provider.lookup('foo'), undefined);
@ -34,7 +40,7 @@ suite('markdown.TableOfContentsProvider', () => {
test('Lookup should return basic #header', async () => {
const doc = new InMemoryDocument(testFileName, `# a\nx\n# c`);
const provider = await TableOfContents.create(createNewMarkdownEngine(), doc);
const provider = await createToc(doc);
{
const entry = provider.lookup('a');
@ -53,7 +59,7 @@ suite('markdown.TableOfContentsProvider', () => {
test('Lookups should be case in-sensitive', async () => {
const doc = new InMemoryDocument(testFileName, `# fOo\n`);
const provider = await TableOfContents.create(createNewMarkdownEngine(), doc);
const provider = await createToc(doc);
assert.strictEqual((provider.lookup('fOo'))!.line, 0);
assert.strictEqual((provider.lookup('foo'))!.line, 0);
@ -62,7 +68,7 @@ suite('markdown.TableOfContentsProvider', () => {
test('Lookups should ignore leading and trailing white-space, and collapse internal whitespace', async () => {
const doc = new InMemoryDocument(testFileName, `# f o o \n`);
const provider = await TableOfContents.create(createNewMarkdownEngine(), doc);
const provider = await createToc(doc);
assert.strictEqual((provider.lookup('f o o'))!.line, 0);
assert.strictEqual((provider.lookup(' f o o'))!.line, 0);
@ -77,14 +83,14 @@ suite('markdown.TableOfContentsProvider', () => {
test('should handle special characters #44779', async () => {
const doc = new InMemoryDocument(testFileName, `# Indentação\n`);
const provider = await TableOfContents.create(createNewMarkdownEngine(), doc);
const provider = await createToc(doc);
assert.strictEqual((provider.lookup('indentação'))!.line, 0);
});
test('should handle special characters 2, #48482', async () => {
const doc = new InMemoryDocument(testFileName, `# Инструкция - Делай Раз, Делай Два\n`);
const provider = await TableOfContents.create(createNewMarkdownEngine(), doc);
const provider = await createToc(doc);
assert.strictEqual((provider.lookup('инструкция---делай-раз-делай-два'))!.line, 0);
});
@ -97,7 +103,7 @@ suite('markdown.TableOfContentsProvider', () => {
### Заголовок Header 3
## Заголовок`);
const provider = await TableOfContents.create(createNewMarkdownEngine(), doc);
const provider = await createToc(doc);
assert.strictEqual((provider.lookup('header-2'))!.line, 0);
assert.strictEqual((provider.lookup('header-3'))!.line, 1);
@ -109,7 +115,7 @@ suite('markdown.TableOfContentsProvider', () => {
test('Lookup should support suffixes for repeated headers', async () => {
const doc = new InMemoryDocument(testFileName, `# a\n# a\n## a`);
const provider = await TableOfContents.create(createNewMarkdownEngine(), doc);
const provider = await createToc(doc);
{
const entry = provider.lookup('a');

View file

@ -4,12 +4,11 @@
*--------------------------------------------------------------------------------------------*/
import * as vscode from 'vscode';
import { MarkdownEngine } from '../markdownEngine';
import { TableOfContents } from '../tableOfContents';
import { MdWorkspaceContents, SkinnyTextDocument } from '../workspaceContents';
import { MdTableOfContentsProvider, TableOfContents } from '../tableOfContents';
import { equals } from '../util/arrays';
import { Disposable } from '../util/dispose';
import { ResourceMap } from '../util/resourceMap';
import { MdWorkspaceContents, SkinnyTextDocument } from '../workspaceContents';
/**
* Check if the items in a table of contents have changed.
@ -32,8 +31,8 @@ export class MdTableOfContentsWatcher extends Disposable {
public readonly onTocChanged = this._onTocChanged.event;
public constructor(
private readonly engine: MarkdownEngine,
private readonly workspaceContents: MdWorkspaceContents,
private readonly tocProvider: MdTableOfContentsProvider,
) {
super();
@ -43,13 +42,13 @@ export class MdTableOfContentsWatcher extends Disposable {
}
private async onDidCreateDocument(document: SkinnyTextDocument) {
const toc = await TableOfContents.create(this.engine, document);
const toc = await this.tocProvider.getForDocument(document);
this._files.set(document.uri, { toc });
}
private async onDidChangeDocument(document: SkinnyTextDocument) {
const existing = this._files.get(document.uri);
const newToc = await TableOfContents.create(this.engine, document);
const newToc = await this.tocProvider.getForDocument(document);
if (!existing || hasTableOfContentsChanged(existing.toc, newToc)) {
this._onTocChanged.fire({ uri: document.uri });

View file

@ -6,8 +6,7 @@
import * as path from 'path';
import * as vscode from 'vscode';
import * as uri from 'vscode-uri';
import { MarkdownEngine } from '../markdownEngine';
import { TableOfContents } from '../tableOfContents';
import { MdTableOfContentsProvider } from '../tableOfContents';
import { isMarkdownFile } from './file';
export interface OpenDocumentLinkArgs {
@ -37,10 +36,10 @@ export function resolveDocumentLink(href: string, markdownFile: vscode.Uri): vsc
return vscode.Uri.joinPath(dirnameUri, hrefPath).with({ fragment });
}
export async function openDocumentLink(engine: MarkdownEngine, targetResource: vscode.Uri, fromResource: vscode.Uri): Promise<void> {
export async function openDocumentLink(tocProvider: MdTableOfContentsProvider, targetResource: vscode.Uri, fromResource: vscode.Uri): Promise<void> {
const column = getViewColumn(fromResource);
if (await tryNavigateToFragmentInActiveEditor(engine, targetResource)) {
if (await tryNavigateToFragmentInActiveEditor(tocProvider, targetResource)) {
return;
}
@ -58,7 +57,7 @@ export async function openDocumentLink(engine: MarkdownEngine, targetResource: v
try {
const stat = await vscode.workspace.fs.stat(dotMdResource);
if (stat.type === vscode.FileType.File) {
await tryOpenMdFile(engine, dotMdResource, column);
await tryOpenMdFile(tocProvider, dotMdResource, column);
return;
}
} catch {
@ -69,19 +68,19 @@ export async function openDocumentLink(engine: MarkdownEngine, targetResource: v
return vscode.commands.executeCommand('revealInExplorer', targetResource);
}
await tryOpenMdFile(engine, targetResource, column);
await tryOpenMdFile(tocProvider, targetResource, column);
}
async function tryOpenMdFile(engine: MarkdownEngine, resource: vscode.Uri, column: vscode.ViewColumn): Promise<boolean> {
async function tryOpenMdFile(tocProvider: MdTableOfContentsProvider, resource: vscode.Uri, column: vscode.ViewColumn): Promise<boolean> {
await vscode.commands.executeCommand('vscode.open', resource.with({ fragment: '' }), column);
return tryNavigateToFragmentInActiveEditor(engine, resource);
return tryNavigateToFragmentInActiveEditor(tocProvider, resource);
}
async function tryNavigateToFragmentInActiveEditor(engine: MarkdownEngine, resource: vscode.Uri): Promise<boolean> {
async function tryNavigateToFragmentInActiveEditor(tocProvider: MdTableOfContentsProvider, resource: vscode.Uri): Promise<boolean> {
const activeEditor = vscode.window.activeTextEditor;
if (activeEditor?.document.uri.fsPath === resource.fsPath) {
if (isMarkdownFile(activeEditor.document)) {
if (await tryRevealLineUsingTocFragment(engine, activeEditor, resource.fragment)) {
if (await tryRevealLineUsingTocFragment(tocProvider, activeEditor, resource.fragment)) {
return true;
}
}
@ -103,8 +102,8 @@ function getViewColumn(resource: vscode.Uri): vscode.ViewColumn {
}
}
async function tryRevealLineUsingTocFragment(engine: MarkdownEngine, editor: vscode.TextEditor, fragment: string): Promise<boolean> {
const toc = await TableOfContents.create(engine, editor.document);
async function tryRevealLineUsingTocFragment(tocProvider: MdTableOfContentsProvider, editor: vscode.TextEditor, fragment: string): Promise<boolean> {
const toc = await tocProvider.getForDocument(editor.document);
const entry = toc.lookup(fragment);
if (entry) {
const lineStart = new vscode.Range(entry.line, 0, entry.line, 0);