Microsoft/vscode
1
0
Fork 0
mirror of https://github.com/microsoft/vscode.git synced 2026-04-23 18:19:12 +01:00
Code Issues Packages Projects Releases Wiki Activity

Reduce number of times MD docs are re-tokenized (#152674)

Browse Source 
This change reduces the number of times we retokenize a markdown file by doing the following:

- Use `MdTableOfContentsProvider` in more places
- Introduce a `IMarkdownParser` interface that lets us drop in a caching version of the tokenizer
This commit is contained in:
Matt Bierner
2022-06-20 23:43:01 -07:00
committed by GitHub
parent 10e6e87682
commit 2249b171f4
23 changed files with 190 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
extensions/markdown-language-features/src/test/diagnostic.test.ts
View File

@@ -440,12 +440,12 @@ suite('Markdown: Diagnostics manager', () => {
const tocProvider = new MdTableOfContentsProvider(engine, workspace);
const referencesProvider = new MdReferencesProvider(engine, workspace, tocProvider);
const manager = new DiagnosticManager(
engine,
workspace,
new DiagnosticComputer(workspace, linkProvider, tocProvider),
configuration,
reporter,
referencesProvider,
tocProvider,
0);
_disposables.push(manager, referencesProvider);
return manager;

6
extensions/markdown-language-features/src/test/engine.ts
View File

@@ -4,7 +4,7 @@
*--------------------------------------------------------------------------------------------*/
import * as vscode from 'vscode';
import { MarkdownEngine } from '../markdownEngine';
import { MarkdownItEngine } from '../markdownEngine';
import { MarkdownContributionProvider, MarkdownContributions } from '../markdownExtensions';
import { githubSlugifier } from '../slugify';
import { Disposable } from '../util/dispose';
@@ -15,6 +15,6 @@ const emptyContributions = new class extends Disposable implements MarkdownContr
readonly onContributionsChanged = this._register(new vscode.EventEmitter<this>()).event;
};
export function createNewMarkdownEngine(): MarkdownEngine {
return new MarkdownEngine(emptyContributions, githubSlugifier);
export function createNewMarkdownEngine(): MarkdownItEngine {
return new MarkdownItEngine(emptyContributions, githubSlugifier);
}

6
extensions/markdown-language-features/src/test/smartSelect.test.ts
View File

@@ -6,11 +6,11 @@
import * as assert from 'assert';
import * as vscode from 'vscode';
import { MdSmartSelect } from '../languageFeatures/smartSelect';
import { createNewMarkdownEngine } from './engine';
import { InMemoryDocument } from '../util/inMemoryDocument';
import { CURSOR, getCursorPositions, joinLines } from './util';
import { MdTableOfContentsProvider } from '../tableOfContents';
import { InMemoryDocument } from '../util/inMemoryDocument';
import { createNewMarkdownEngine } from './engine';
import { InMemoryWorkspaceMarkdownDocuments } from './inMemoryWorkspace';
import { CURSOR, getCursorPositions, joinLines } from './util';
const testFileName = vscode.Uri.file('test.md');

26
extensions/markdown-language-features/src/test/tableOfContentsProvider.test.ts
View File

@@ -7,16 +7,22 @@ import * as assert from 'assert';
import 'mocha';
import * as vscode from 'vscode';
import { TableOfContents } from '../tableOfContents';
import { createNewMarkdownEngine } from './engine';
import { InMemoryDocument } from '../util/inMemoryDocument';
import { SkinnyTextDocument } from '../workspaceContents';
import { createNewMarkdownEngine } from './engine';
const testFileName = vscode.Uri.file('test.md');
function createToc(doc: SkinnyTextDocument): Promise<TableOfContents> {
const engine = createNewMarkdownEngine();
return TableOfContents.create(engine, doc);
}
suite('markdown.TableOfContentsProvider', () => {
test('Lookup should not return anything for empty document', async () => {
const doc = new InMemoryDocument(testFileName, '');
const provider = await TableOfContents.create(createNewMarkdownEngine(), doc);
const provider = await createToc(doc);
assert.strictEqual(provider.lookup(''), undefined);
assert.strictEqual(provider.lookup('foo'), undefined);
@@ -24,7 +30,7 @@ suite('markdown.TableOfContentsProvider', () => {
test('Lookup should not return anything for document with no headers', async () => {
const doc = new InMemoryDocument(testFileName, 'a *b*\nc');
const provider = await TableOfContents.create(createNewMarkdownEngine(), doc);
const provider = await createToc(doc);
assert.strictEqual(provider.lookup(''), undefined);
assert.strictEqual(provider.lookup('foo'), undefined);
@@ -34,7 +40,7 @@ suite('markdown.TableOfContentsProvider', () => {
test('Lookup should return basic #header', async () => {
const doc = new InMemoryDocument(testFileName, `# a\nx\n# c`);
const provider = await TableOfContents.create(createNewMarkdownEngine(), doc);
const provider = await createToc(doc);
{
const entry = provider.lookup('a');
@@ -53,7 +59,7 @@ suite('markdown.TableOfContentsProvider', () => {
test('Lookups should be case in-sensitive', async () => {
const doc = new InMemoryDocument(testFileName, `# fOo\n`);
const provider = await TableOfContents.create(createNewMarkdownEngine(), doc);
const provider = await createToc(doc);
assert.strictEqual((provider.lookup('fOo'))!.line, 0);
assert.strictEqual((provider.lookup('foo'))!.line, 0);
@@ -62,7 +68,7 @@ suite('markdown.TableOfContentsProvider', () => {
test('Lookups should ignore leading and trailing white-space, and collapse internal whitespace', async () => {
const doc = new InMemoryDocument(testFileName, `# f o o \n`);
const provider = await TableOfContents.create(createNewMarkdownEngine(), doc);
const provider = await createToc(doc);
assert.strictEqual((provider.lookup('f o o'))!.line, 0);
assert.strictEqual((provider.lookup(' f o o'))!.line, 0);
@@ -77,14 +83,14 @@ suite('markdown.TableOfContentsProvider', () => {
test('should handle special characters #44779', async () => {
const doc = new InMemoryDocument(testFileName, `# Indentação\n`);
const provider = await TableOfContents.create(createNewMarkdownEngine(), doc);
const provider = await createToc(doc);
assert.strictEqual((provider.lookup('indentação'))!.line, 0);
});
test('should handle special characters 2, #48482', async () => {
const doc = new InMemoryDocument(testFileName, `# Инструкция - Делай Раз, Делай Два\n`);
const provider = await TableOfContents.create(createNewMarkdownEngine(), doc);
const provider = await createToc(doc);
assert.strictEqual((provider.lookup('инструкция---делай-раз-делай-два'))!.line, 0);
});
@@ -97,7 +103,7 @@ suite('markdown.TableOfContentsProvider', () => {
### Заголовок Header 3
## Заголовок`);
const provider = await TableOfContents.create(createNewMarkdownEngine(), doc);
const provider = await createToc(doc);
assert.strictEqual((provider.lookup('header-2'))!.line, 0);
assert.strictEqual((provider.lookup('header-3'))!.line, 1);
@@ -109,7 +115,7 @@ suite('markdown.TableOfContentsProvider', () => {
test('Lookup should support suffixes for repeated headers', async () => {
const doc = new InMemoryDocument(testFileName, `# a\n# a\n## a`);
const provider = await TableOfContents.create(createNewMarkdownEngine(), doc);
const provider = await createToc(doc);
{
const entry = provider.lookup('a');

11
extensions/markdown-language-features/src/test/tableOfContentsWatcher.ts
View File

@@ -4,12 +4,11 @@
*--------------------------------------------------------------------------------------------*/
import * as vscode from 'vscode';
import { MarkdownEngine } from '../markdownEngine';
import { TableOfContents } from '../tableOfContents';
import { MdWorkspaceContents, SkinnyTextDocument } from '../workspaceContents';
import { MdTableOfContentsProvider, TableOfContents } from '../tableOfContents';
import { equals } from '../util/arrays';
import { Disposable } from '../util/dispose';
import { ResourceMap } from '../util/resourceMap';
import { MdWorkspaceContents, SkinnyTextDocument } from '../workspaceContents';
/**
* Check if the items in a table of contents have changed.
@@ -32,8 +31,8 @@ export class MdTableOfContentsWatcher extends Disposable {
public readonly onTocChanged = this._onTocChanged.event;
public constructor(
private readonly engine: MarkdownEngine,
private readonly workspaceContents: MdWorkspaceContents,
private readonly tocProvider: MdTableOfContentsProvider,
) {
super();
@@ -43,13 +42,13 @@ export class MdTableOfContentsWatcher extends Disposable {
}
private async onDidCreateDocument(document: SkinnyTextDocument) {
const toc = await TableOfContents.create(this.engine, document);
const toc = await this.tocProvider.getForDocument(document);
this._files.set(document.uri, { toc });
}
private async onDidChangeDocument(document: SkinnyTextDocument) {
const existing = this._files.get(document.uri);
const newToc = await TableOfContents.create(this.engine, document);
const newToc = await this.tocProvider.getForDocument(document);
if (!existing || hasTableOfContentsChanged(existing.toc, newToc)) {
this._onTocChanged.fire({ uri: document.uri });