Microsoft/vscode
1
0
Fork 0
mirror of https://github.com/microsoft/vscode.git synced 2026-05-08 17:19:48 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #58703 from Microsoft/aeschli/tokenizersupport-initialization

Browse Source 
ITokenizationRegistry to support TokenizationSupport promises
This commit is contained in:
Martin Aeschlimann
2018-09-17 17:06:25 +02:00
committed by GitHub
parent b2e70a254d 51ef4de3a7
commit f22940a29f
7 changed files with 66 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/vs/editor/common/modes.ts
+11
View File
@@ -1186,12 +1186,23 @@ export interface ITokenizationRegistry {
*/
register(language: string, support: ITokenizationSupport): IDisposable;
/**
* Register a promise for a tokenization support.
*/
registerPromise(language: string, promise: Thenable<ITokenizationSupport>): Thenable<IDisposable>;
/**
* Get the tokenization support for a language.
* Returns null if not found.
*/
get(language: string): ITokenizationSupport;
/**
* Get the promise of a tokenization support for a language.
* `null` is returned if no support is available and no promise for the support has been registered yet.
*/
getPromise(language: string): Thenable<ITokenizationSupport>;
/**
* Set the new color map that all tokens will use in their ColorId binary encoded bits for foreground and background.
*/
src/vs/editor/common/modes/textToHtmlTokenizer.ts
+10 -16
View File
@@ -5,13 +5,19 @@
'use strict';
import * as strings from 'vs/base/common/strings';
import { IState, ITokenizationSupport, TokenizationRegistry, LanguageId } from 'vs/editor/common/modes';
import { NULL_STATE, nullTokenize2 } from 'vs/editor/common/modes/nullMode';
import { ITokenizationSupport, IState, LanguageId } from 'vs/editor/common/modes';
import { LineTokens, IViewLineTokens } from 'vs/editor/common/core/lineTokens';
import { CharCode } from 'vs/base/common/charCode';
import { NULL_STATE, nullTokenize2 } from 'vs/editor/common/modes/nullMode';
export function tokenizeToString(text: string, languageId: string): string {
return _tokenizeToString(text, _getSafeTokenizationSupport(languageId));
const fallback = {
getInitialState: () => NULL_STATE,
tokenize: undefined,
tokenize2: (buffer: string, state: IState, deltaOffset: number) => nullTokenize2(LanguageId.Null, buffer, state, deltaOffset)
};
export function tokenizeToString(text: string, tokenizationSupport: ITokenizationSupport = fallback): string {
return _tokenizeToString(text, tokenizationSupport);
}
export function tokenizeLineToHTML(text: string, viewLineTokens: IViewLineTokens, colorMap: string[], startOffset: number, endOffset: number, tabSize: number): string {
@@ -83,18 +89,6 @@ export function tokenizeLineToHTML(text: string, viewLineTokens: IViewLineTokens
return result;
}
function _getSafeTokenizationSupport(languageId: string): ITokenizationSupport {
let tokenizationSupport = TokenizationRegistry.get(languageId);
if (tokenizationSupport) {
return tokenizationSupport;
}
return {
getInitialState: () => NULL_STATE,
tokenize: undefined,
tokenize2: (buffer: string, state: IState, deltaOffset: number) => nullTokenize2(LanguageId.Null, buffer, state, deltaOffset)
};
}
function _tokenizeToString(text: string, tokenizationSupport: ITokenizationSupport): string {
let result = `<div class="monaco-tokenized-source">`;
let lines = text.split(/\r\n|\r|\n/);
src/vs/editor/common/modes/tokenizationRegistry.ts
+24 -1
View File
@@ -8,10 +8,12 @@ import { IDisposable, toDisposable } from 'vs/base/common/lifecycle';
import { Event, Emitter } from 'vs/base/common/event';
import { ColorId, ITokenizationRegistry, ITokenizationSupport, ITokenizationSupportChangedEvent } from 'vs/editor/common/modes';
import { Color } from 'vs/base/common/color';
import { TPromise } from 'vs/base/common/winjs.base';
export class TokenizationRegistryImpl implements ITokenizationRegistry {
private _map: { [language: string]: ITokenizationSupport };
private _promises: { [language: string]: Thenable<IDisposable> };
private readonly _onDidChange: Emitter<ITokenizationSupportChangedEvent> = new Emitter<ITokenizationSupportChangedEvent>();
public readonly onDidChange: Event<ITokenizationSupportChangedEvent> = this._onDidChange.event;
@@ -20,6 +22,7 @@ export class TokenizationRegistryImpl implements ITokenizationRegistry {
constructor() {
this._map = Object.create(null);
this._promises = Object.create(null);
this._colorMap = null;
}
@@ -30,7 +33,7 @@ export class TokenizationRegistryImpl implements ITokenizationRegistry {
});
}
public register(language: string, support: ITokenizationSupport): IDisposable {
public register(language: string, support: ITokenizationSupport) {
this._map[language] = support;
this.fire([language]);
return toDisposable(() => {
@@ -42,6 +45,26 @@ export class TokenizationRegistryImpl implements ITokenizationRegistry {
});
}
public registerPromise(language: string, supportPromise: Thenable<ITokenizationSupport>): Thenable<IDisposable> {
const promise = this._promises[language] = supportPromise.then(support => {
delete this._promises[language];
return this.register(language, support);
});
return promise;
}
public getPromise(language: string): Thenable<ITokenizationSupport> {
const support = this.get(language);
if (support) {
return TPromise.as(support);
}
const promise = this._promises[language];
if (promise) {
return promise.then(_ => this.get(language));
}
return null;
}
public get(language: string): ITokenizationSupport {
return (this._map[language] || null);
}
src/vs/editor/contrib/markdown/markdownRenderer.ts
+6 -1
View File
@@ -17,6 +17,7 @@ import { ICodeEditor } from 'vs/editor/browser/editorBrowser';
import { optional } from 'vs/platform/instantiation/common/instantiation';
import { Event, Emitter } from 'vs/base/common/event';
import { IDisposable, dispose } from 'vs/base/common/lifecycle';
import { TokenizationRegistry } from 'vs/editor/common/modes';
export interface IMarkdownRenderResult extends IDisposable {
element: HTMLElement;
@@ -45,7 +46,11 @@ export class MarkdownRenderer {
: this._editor.getModel().getLanguageIdentifier().language;
return this._modeService.getOrCreateMode(modeId).then(_ => {
return tokenizeToString(value, modeId);
const promise = TokenizationRegistry.getPromise(modeId);
if (promise) {
return promise.then(support => tokenizeToString(value, support));
}
return tokenizeToString(value, null);
}).then(code => {
return `<span style="font-family: ${this._editor.getConfiguration().fontInfo.fontFamily}">${code}</span>`;
});
src/vs/editor/test/common/modes/textToHtmlTokenizer.test.ts
+4 -2
View File
@@ -19,8 +19,9 @@ suite('Editor Modes - textToHtmlTokenizer', () => {
test('TextToHtmlTokenizer 1', () => {
let mode = new Mode();
let support = TokenizationRegistry.get(mode.getId());
let actual = tokenizeToString('.abc..def...gh', mode.getId());
let actual = tokenizeToString('.abc..def...gh', support);
let expected = [
{ className: 'mtk7', text: '.' },
{ className: 'mtk9', text: 'abc' },
@@ -38,8 +39,9 @@ suite('Editor Modes - textToHtmlTokenizer', () => {
test('TextToHtmlTokenizer 2', () => {
let mode = new Mode();
let support = TokenizationRegistry.get(mode.getId());
let actual = tokenizeToString('.abc..def...gh\n.abc..def...gh', mode.getId());
let actual = tokenizeToString('.abc..def...gh\n.abc..def...gh', support);
let expected1 = [
{ className: 'mtk7', text: '.' },
{ className: 'mtk9', text: 'abc' },
src/vs/workbench/parts/update/electron-browser/releaseNotesEditor.ts
+4 -7
View File
@@ -11,7 +11,7 @@ import { OS } from 'vs/base/common/platform';
import { URI } from 'vs/base/common/uri';
import { TPromise } from 'vs/base/common/winjs.base';
import { asText } from 'vs/base/node/request';
import { IMode, TokenizationRegistry } from 'vs/editor/common/modes';
import { TokenizationRegistry, ITokenizationSupport } from 'vs/editor/common/modes';
import { generateTokensCSSForColorMap } from 'vs/editor/common/modes/supports/tokenization';
import { tokenizeToString } from 'vs/editor/common/modes/textToHtmlTokenizer';
import { IModeService } from 'vs/editor/common/services/modeService';
@@ -192,21 +192,18 @@ export class ReleaseNotesManager {
}
private async getRenderer(text: string) {
const result: TPromise<IMode>[] = [];
let result: TPromise<ITokenizationSupport>[] = [];
const renderer = new marked.Renderer();
renderer.code = (code, lang) => {
const modeId = this._modeService.getModeIdForLanguageName(lang);
result.push(this._modeService.getOrCreateMode(modeId));
result.push(this._modeService.getOrCreateMode(modeId).then(_ => TokenizationRegistry.getPromise(modeId)));
return '';
};
marked(text, { renderer });
await TPromise.join(result);
renderer.code = (code, lang) => {
const modeId = this._modeService.getModeIdForLanguageName(lang);
return `<code>${tokenizeToString(code, modeId)}</code>`;
};
renderer.code = (code, lang) => `<code>${tokenizeToString(code, TokenizationRegistry.get(lang))}</code>`;
return renderer;
}
}
src/vs/workbench/services/textMate/electron-browser/TMSyntax.ts
+7 -3
View File
@@ -397,9 +397,13 @@ export class TextMateService implements ITextMateService {
}
private registerDefinition(modeId: string): void {
this._createGrammar(modeId).then((r) => {
TokenizationRegistry.register(modeId, new TMTokenization(this._scopeRegistry, r.languageId, r.grammar, r.initialState, r.containsEmbeddedLanguages, this._notificationService));
}, onUnexpectedError);
const promise = this._createGrammar(modeId).then((r) => {
return new TMTokenization(this._scopeRegistry, r.languageId, r.grammar, r.initialState, r.containsEmbeddedLanguages, this._notificationService);
}, e => {
onUnexpectedError(e);
return null;
});
TokenizationRegistry.registerPromise(modeId, promise);
}
}