Microsoft/vscode
1
0
Fork 0
mirror of https://github.com/microsoft/vscode.git synced 2025-12-24 12:19:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix-65575 tokenize method added

Browse Source
This commit is contained in:
pkoushik
2019-01-04 20:07:21 +05:30
parent 1ef045875d
commit 661d191638
1 changed files with 32 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
extensions/markdown-language-features/src/markdownEngine.ts
View File

@@ -13,14 +13,15 @@ import { SkinnyTextDocument } from './tableOfContentsProvider';
import { getUriForLinkWithKnownExternalScheme } from './util/links';
const FrontMatterRegex = /^---\s*[^]*?(-{3}|\.{3})\s*/;
export class MarkdownEngine {
private md?: MarkdownIt;
private firstLine?: number;
private currentDocument?: vscode.Uri;
private _slugCount = new Map<string, number>();
private cachedToken = new Set<Token>();
private cachedTokens = new Map<vscode.Uri, Token[]>();
public constructor(
@@ -96,6 +97,24 @@ export class MarkdownEngine {
return { text, offset };
}
private tokenize(document: SkinnyTextDocument, engine?: MarkdownIt): Token[] {
const UNICODE_NEWLINE_REGEX = /\u2028|\u2029/g;
const { text, offset } = this.stripFrontmatter(document.getText());
let tokens: Token[];
if (this.cachedTokens.has(document.uri)) {
tokens = this.cachedTokens.get(document.uri)!;
}
tokens = engine!.parse(text.replace(UNICODE_NEWLINE_REGEX, ''), {}).map(token => {
if (token.map) {
token.map[0] += offset;
token.map[1] += offset;
}
return token;
});
this.cachedTokens.set(document.uri, tokens);
return tokens;
}
public async render(document: SkinnyTextDocument, stripFrontmatter: boolean): Promise<string> {
let offset = 0;
let text = document.getText();
@@ -109,30 +128,25 @@ export class MarkdownEngine {
this._slugCount = new Map<string, number>();
const engine = await this.getEngine(document.uri);
if (this.cachedToken) {
return engine.renderer.render([...this.cachedToken].map(token => {
return token;
}), this.md, {});
let tokens: Token[];
tokens = this.tokenize(document);
if (!tokens) {
tokens = this.tokenize(document, engine);
}
return engine.render(text);
return engine.renderer.render(tokens, this.md, {});
}
public async parse(document: SkinnyTextDocument): Promise<Token[]> {
const UNICODE_NEWLINE_REGEX = /\u2028|\u2029/g;
const { text, offset } = this.stripFrontmatter(document.getText());
this.currentDocument = document.uri;
this._slugCount = new Map<string, number>();
const engine = await this.getEngine(document.uri);
return engine.parse(text.replace(UNICODE_NEWLINE_REGEX, ''), {}).map(token => {
if (token.map) {
token.map[0] += offset;
token.map[1] += offset;
}
this.cachedToken.add(token);
return token;
});
let tokens: Token[];
tokens = this.tokenize(document);
if (!tokens) {
tokens = this.tokenize(document, engine);
}
return tokens;
}
private addLineNumberRenderer(md: any, ruleName: string): void {