From e019fcbeb7513c24f94fc2fc8cf67a67b29dec1e Mon Sep 17 00:00:00 2001 From: Martin Aeschlimann Date: Fri, 14 Sep 2018 17:59:09 +0200 Subject: [PATCH 1/2] promises in ITokenizationRegistry --- src/vs/editor/common/modes.ts | 11 ++++++ .../common/modes/textToHtmlTokenizer.ts | 26 +++++-------- .../common/modes/tokenizationRegistry.ts | 25 +++++++++++- .../common/services/tokenizationService.ts | 39 +++++++++++++++++++ .../contrib/markdown/markdownRenderer.ts | 7 +++- .../common/modes/textToHtmlTokenizer.test.ts | 6 ++- .../electron-browser/releaseNotesEditor.ts | 11 ++---- .../textMate/electron-browser/TMSyntax.ts | 10 +++-- 8 files changed, 105 insertions(+), 30 deletions(-) create mode 100644 src/vs/editor/common/services/tokenizationService.ts diff --git a/src/vs/editor/common/modes.ts b/src/vs/editor/common/modes.ts index 23a2b611dc9..fd46c75c791 100644 --- a/src/vs/editor/common/modes.ts +++ b/src/vs/editor/common/modes.ts @@ -1191,12 +1191,23 @@ export interface ITokenizationRegistry { */ register(language: string, support: ITokenizationSupport): IDisposable; + /** + * Register a promise for a tokenization support. + */ + registerPromise(language: string, promise: Thenable): Thenable; + /** * Get the tokenization support for a language. * Returns null if not found. */ get(language: string): ITokenizationSupport; + /** + * Get the promise of a tokenization support for a language. + * `null` is returned if no support is available and no promise for the support has been registered yet. + */ + getPromise(language: string): Thenable; + /** * Set the new color map that all tokens will use in their ColorId binary encoded bits for foreground and background. */ diff --git a/src/vs/editor/common/modes/textToHtmlTokenizer.ts b/src/vs/editor/common/modes/textToHtmlTokenizer.ts index 3c2bd218343..168cdeb1643 100644 --- a/src/vs/editor/common/modes/textToHtmlTokenizer.ts +++ b/src/vs/editor/common/modes/textToHtmlTokenizer.ts @@ -5,13 +5,19 @@ 'use strict'; import * as strings from 'vs/base/common/strings'; -import { IState, ITokenizationSupport, TokenizationRegistry, LanguageId } from 'vs/editor/common/modes'; -import { NULL_STATE, nullTokenize2 } from 'vs/editor/common/modes/nullMode'; +import { ITokenizationSupport, IState, LanguageId } from 'vs/editor/common/modes'; import { LineTokens, IViewLineTokens } from 'vs/editor/common/core/lineTokens'; import { CharCode } from 'vs/base/common/charCode'; +import { NULL_STATE, nullTokenize2 } from 'vs/editor/common/modes/nullMode'; -export function tokenizeToString(text: string, languageId: string): string { - return _tokenizeToString(text, _getSafeTokenizationSupport(languageId)); +const fallback = { + getInitialState: () => NULL_STATE, + tokenize: undefined, + tokenize2: (buffer: string, state: IState, deltaOffset: number) => nullTokenize2(LanguageId.Null, buffer, state, deltaOffset) +}; + +export function tokenizeToString(text: string, tokenizationSupport: ITokenizationSupport = fallback): string { + return _tokenizeToString(text, tokenizationSupport); } export function tokenizeLineToHTML(text: string, viewLineTokens: IViewLineTokens, colorMap: string[], startOffset: number, endOffset: number, tabSize: number): string { @@ -83,18 +89,6 @@ export function tokenizeLineToHTML(text: string, viewLineTokens: IViewLineTokens return result; } -function _getSafeTokenizationSupport(languageId: string): ITokenizationSupport { - let tokenizationSupport = TokenizationRegistry.get(languageId); - if (tokenizationSupport) { - return tokenizationSupport; - } - return { - getInitialState: () => NULL_STATE, - tokenize: undefined, - tokenize2: (buffer: string, state: IState, deltaOffset: number) => nullTokenize2(LanguageId.Null, buffer, state, deltaOffset) - }; -} - function _tokenizeToString(text: string, tokenizationSupport: ITokenizationSupport): string { let result = `
`; let lines = text.split(/\r\n|\r|\n/); diff --git a/src/vs/editor/common/modes/tokenizationRegistry.ts b/src/vs/editor/common/modes/tokenizationRegistry.ts index dd13f213e76..ec66585a4d0 100644 --- a/src/vs/editor/common/modes/tokenizationRegistry.ts +++ b/src/vs/editor/common/modes/tokenizationRegistry.ts @@ -8,10 +8,12 @@ import { IDisposable, toDisposable } from 'vs/base/common/lifecycle'; import { Event, Emitter } from 'vs/base/common/event'; import { ColorId, ITokenizationRegistry, ITokenizationSupport, ITokenizationSupportChangedEvent } from 'vs/editor/common/modes'; import { Color } from 'vs/base/common/color'; +import { TPromise } from 'vs/base/common/winjs.base'; export class TokenizationRegistryImpl implements ITokenizationRegistry { private _map: { [language: string]: ITokenizationSupport }; + private _promises: { [language: string]: Thenable }; private readonly _onDidChange: Emitter = new Emitter(); public readonly onDidChange: Event = this._onDidChange.event; @@ -20,6 +22,7 @@ export class TokenizationRegistryImpl implements ITokenizationRegistry { constructor() { this._map = Object.create(null); + this._promises = Object.create(null); this._colorMap = null; } @@ -30,7 +33,7 @@ export class TokenizationRegistryImpl implements ITokenizationRegistry { }); } - public register(language: string, support: ITokenizationSupport): IDisposable { + public register(language: string, support: ITokenizationSupport) { this._map[language] = support; this.fire([language]); return toDisposable(() => { @@ -42,6 +45,26 @@ export class TokenizationRegistryImpl implements ITokenizationRegistry { }); } + public registerPromise(language: string, supportPromise: Thenable): Thenable { + const promise = this._promises[language] = supportPromise.then(support => { + delete this._promises[language]; + return this.register(language, support); + }); + return promise; + } + + public getPromise(language: string): Thenable { + const support = this.get(language); + if (support) { + return TPromise.as(support); + } + const promise = this._promises[language]; + if (promise) { + return promise.then(_ => this.get(language)); + } + return null; + } + public get(language: string): ITokenizationSupport { return (this._map[language] || null); } diff --git a/src/vs/editor/common/services/tokenizationService.ts b/src/vs/editor/common/services/tokenizationService.ts new file mode 100644 index 00000000000..7106f8144c2 --- /dev/null +++ b/src/vs/editor/common/services/tokenizationService.ts @@ -0,0 +1,39 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +'use strict'; + +import { Event } from 'vs/base/common/event'; +import { TPromise } from 'vs/base/common/winjs.base'; +import { createDecorator } from 'vs/platform/instantiation/common/instantiation'; +import { IMode, LanguageId, LanguageIdentifier } from 'vs/editor/common/modes'; +import { URI } from 'vs/base/common/uri'; + +export const ITokenizationService = createDecorator('tokenizationService'); + +export interface ITokenizationService { + _serviceBrand: any; + + onDidCreateMode: Event; + + // --- reading + isRegisteredMode(mimetypeOrModeId: string): boolean; + getRegisteredModes(): string[]; + getRegisteredLanguageNames(): string[]; + getExtensions(alias: string): string[]; + getFilenames(alias: string): string[]; + getMimeForMode(modeId: string): string; + getLanguageName(modeId: string): string; + getModeIdForLanguageName(alias: string): string; + getModeIdByFilenameOrFirstLine(filename: string, firstLine?: string): string; + getModeId(commaSeparatedMimetypesOrCommaSeparatedIds: string): string; + getLanguageIdentifier(modeId: string | LanguageId): LanguageIdentifier; + getConfigurationFiles(modeId: string): URI[]; + + // --- instantiation + getMode(commaSeparatedMimetypesOrCommaSeparatedIds: string): IMode; + getOrCreateMode(commaSeparatedMimetypesOrCommaSeparatedIds: string): TPromise; + getOrCreateModeByLanguageName(languageName: string): TPromise; + getOrCreateModeByFilenameOrFirstLine(filename: string, firstLine?: string): TPromise; +} diff --git a/src/vs/editor/contrib/markdown/markdownRenderer.ts b/src/vs/editor/contrib/markdown/markdownRenderer.ts index 6f7d78e7db4..61825299b89 100644 --- a/src/vs/editor/contrib/markdown/markdownRenderer.ts +++ b/src/vs/editor/contrib/markdown/markdownRenderer.ts @@ -17,6 +17,7 @@ import { ICodeEditor } from 'vs/editor/browser/editorBrowser'; import { optional } from 'vs/platform/instantiation/common/instantiation'; import { Event, Emitter } from 'vs/base/common/event'; import { IDisposable, dispose } from 'vs/base/common/lifecycle'; +import { TokenizationRegistry } from 'vs/editor/common/modes'; export interface IMarkdownRenderResult extends IDisposable { element: HTMLElement; @@ -45,7 +46,11 @@ export class MarkdownRenderer { : this._editor.getModel().getLanguageIdentifier().language; return this._modeService.getOrCreateMode(modeId).then(_ => { - return tokenizeToString(value, modeId); + const promise = TokenizationRegistry.getPromise(modeId); + if (promise) { + return promise.then(support => tokenizeToString(value, support)); + } + return tokenizeToString(value, null); }).then(code => { return `${code}`; }); diff --git a/src/vs/editor/test/common/modes/textToHtmlTokenizer.test.ts b/src/vs/editor/test/common/modes/textToHtmlTokenizer.test.ts index 7449d71b00d..0f2af297d93 100644 --- a/src/vs/editor/test/common/modes/textToHtmlTokenizer.test.ts +++ b/src/vs/editor/test/common/modes/textToHtmlTokenizer.test.ts @@ -19,8 +19,9 @@ suite('Editor Modes - textToHtmlTokenizer', () => { test('TextToHtmlTokenizer 1', () => { let mode = new Mode(); + let support = TokenizationRegistry.get(mode.getId()); - let actual = tokenizeToString('.abc..def...gh', mode.getId()); + let actual = tokenizeToString('.abc..def...gh', support); let expected = [ { className: 'mtk7', text: '.' }, { className: 'mtk9', text: 'abc' }, @@ -38,8 +39,9 @@ suite('Editor Modes - textToHtmlTokenizer', () => { test('TextToHtmlTokenizer 2', () => { let mode = new Mode(); + let support = TokenizationRegistry.get(mode.getId()); - let actual = tokenizeToString('.abc..def...gh\n.abc..def...gh', mode.getId()); + let actual = tokenizeToString('.abc..def...gh\n.abc..def...gh', support); let expected1 = [ { className: 'mtk7', text: '.' }, { className: 'mtk9', text: 'abc' }, diff --git a/src/vs/workbench/parts/update/electron-browser/releaseNotesEditor.ts b/src/vs/workbench/parts/update/electron-browser/releaseNotesEditor.ts index 88110f40e35..01447e9fe53 100644 --- a/src/vs/workbench/parts/update/electron-browser/releaseNotesEditor.ts +++ b/src/vs/workbench/parts/update/electron-browser/releaseNotesEditor.ts @@ -11,7 +11,7 @@ import { OS } from 'vs/base/common/platform'; import { URI } from 'vs/base/common/uri'; import { TPromise } from 'vs/base/common/winjs.base'; import { asText } from 'vs/base/node/request'; -import { IMode, TokenizationRegistry } from 'vs/editor/common/modes'; +import { TokenizationRegistry, ITokenizationSupport } from 'vs/editor/common/modes'; import { generateTokensCSSForColorMap } from 'vs/editor/common/modes/supports/tokenization'; import { tokenizeToString } from 'vs/editor/common/modes/textToHtmlTokenizer'; import { IModeService } from 'vs/editor/common/services/modeService'; @@ -192,21 +192,18 @@ export class ReleaseNotesManager { } private async getRenderer(text: string) { - const result: TPromise[] = []; + let result: TPromise[] = []; const renderer = new marked.Renderer(); renderer.code = (code, lang) => { const modeId = this._modeService.getModeIdForLanguageName(lang); - result.push(this._modeService.getOrCreateMode(modeId)); + result.push(this._modeService.getOrCreateMode(modeId).then(_ => TokenizationRegistry.getPromise(modeId))); return ''; }; marked(text, { renderer }); await TPromise.join(result); - renderer.code = (code, lang) => { - const modeId = this._modeService.getModeIdForLanguageName(lang); - return `${tokenizeToString(code, modeId)}`; - }; + renderer.code = (code, lang) => `${tokenizeToString(code, TokenizationRegistry.get(lang))}`; return renderer; } } diff --git a/src/vs/workbench/services/textMate/electron-browser/TMSyntax.ts b/src/vs/workbench/services/textMate/electron-browser/TMSyntax.ts index 0294ecefead..8dc3f78076c 100644 --- a/src/vs/workbench/services/textMate/electron-browser/TMSyntax.ts +++ b/src/vs/workbench/services/textMate/electron-browser/TMSyntax.ts @@ -397,9 +397,13 @@ export class TextMateService implements ITextMateService { } private registerDefinition(modeId: string): void { - this._createGrammar(modeId).then((r) => { - TokenizationRegistry.register(modeId, new TMTokenization(this._scopeRegistry, r.languageId, r.grammar, r.initialState, r.containsEmbeddedLanguages, this._notificationService)); - }, onUnexpectedError); + const promise = this._createGrammar(modeId).then((r) => { + return new TMTokenization(this._scopeRegistry, r.languageId, r.grammar, r.initialState, r.containsEmbeddedLanguages, this._notificationService); + }, e => { + onUnexpectedError(e); + return null; + }); + TokenizationRegistry.registerPromise(modeId, promise); } } From 51ef4de3a77f5c41cf47f1aea764a7bced76743f Mon Sep 17 00:00:00 2001 From: Martin Aeschlimann Date: Mon, 17 Sep 2018 16:58:25 +0200 Subject: [PATCH 2/2] ITokenizationService unnecessary --- .../common/services/tokenizationService.ts | 39 ------------------- 1 file changed, 39 deletions(-) delete mode 100644 src/vs/editor/common/services/tokenizationService.ts diff --git a/src/vs/editor/common/services/tokenizationService.ts b/src/vs/editor/common/services/tokenizationService.ts deleted file mode 100644 index 7106f8144c2..00000000000 --- a/src/vs/editor/common/services/tokenizationService.ts +++ /dev/null @@ -1,39 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ -'use strict'; - -import { Event } from 'vs/base/common/event'; -import { TPromise } from 'vs/base/common/winjs.base'; -import { createDecorator } from 'vs/platform/instantiation/common/instantiation'; -import { IMode, LanguageId, LanguageIdentifier } from 'vs/editor/common/modes'; -import { URI } from 'vs/base/common/uri'; - -export const ITokenizationService = createDecorator('tokenizationService'); - -export interface ITokenizationService { - _serviceBrand: any; - - onDidCreateMode: Event; - - // --- reading - isRegisteredMode(mimetypeOrModeId: string): boolean; - getRegisteredModes(): string[]; - getRegisteredLanguageNames(): string[]; - getExtensions(alias: string): string[]; - getFilenames(alias: string): string[]; - getMimeForMode(modeId: string): string; - getLanguageName(modeId: string): string; - getModeIdForLanguageName(alias: string): string; - getModeIdByFilenameOrFirstLine(filename: string, firstLine?: string): string; - getModeId(commaSeparatedMimetypesOrCommaSeparatedIds: string): string; - getLanguageIdentifier(modeId: string | LanguageId): LanguageIdentifier; - getConfigurationFiles(modeId: string): URI[]; - - // --- instantiation - getMode(commaSeparatedMimetypesOrCommaSeparatedIds: string): IMode; - getOrCreateMode(commaSeparatedMimetypesOrCommaSeparatedIds: string): TPromise; - getOrCreateModeByLanguageName(languageName: string): TPromise; - getOrCreateModeByFilenameOrFirstLine(filename: string, firstLine?: string): TPromise; -}