diff --git a/build/monaco/monaco.d.ts.recipe b/build/monaco/monaco.d.ts.recipe index bb7ad20348c..eea7f6c323a 100644 --- a/build/monaco/monaco.d.ts.recipe +++ b/build/monaco/monaco.d.ts.recipe @@ -63,7 +63,7 @@ export interface ICommandHandler { #include(vs/platform/markers/common/markers): IMarkerData #include(vs/editor/browser/standalone/colorizer): IColorizerOptions, IColorizerElementOptions #include(vs/base/common/scrollable): ScrollbarVisibility -#includeAll(vs/editor/common/editorCommon;IMode=>languages.IMode): IPosition, IRange, ISelection, SelectionDirection, IScrollEvent +#includeAll(vs/editor/common/editorCommon;IMode=>languages.IMode;LanguageIdentifier=>languages.LanguageIdentifier): IPosition, IRange, ISelection, SelectionDirection, IScrollEvent #includeAll(vs/editor/browser/editorBrowser;editorCommon.=>): } @@ -71,6 +71,15 @@ declare module monaco.languages { #includeAll(vs/editor/browser/standalone/standaloneLanguages;modes.=>;editorCommon.=>editor.;IMarkerData=>editor.IMarkerData): #includeAll(vs/editor/common/modes/languageConfiguration): +/** + * An identifier for a registered language. + */ +export class LanguageIdentifier { + public readonly sid: string; + public readonly iid: number; + + constructor(sid: string, iid: number); +} #includeAll(vs/editor/common/modes;editorCommon.IRange=>IRange;editorCommon.IPosition=>IPosition;editorCommon.=>editor.;IToken2=>IToken;ILineTokens2=>ILineTokens): #include(vs/editor/common/services/modeService): ILanguageExtensionPoint #includeAll(vs/editor/common/modes/monarch/monarchTypes): diff --git a/src/vs/editor/browser/services/standaloneColorServiceImpl.ts b/src/vs/editor/browser/services/standaloneColorServiceImpl.ts new file mode 100644 index 00000000000..f5e164bb841 --- /dev/null +++ b/src/vs/editor/browser/services/standaloneColorServiceImpl.ts @@ -0,0 +1,43 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +'use strict'; + +import Event, { Emitter } from 'vs/base/common/event'; +import { Theme } from 'vs/editor/common/modes/supports/tokenization'; +import { IStandaloneColorService } from 'vs/editor/common/services/standaloneColorService'; +import { vs } from 'vs/editor/common/standalone/themes'; +import * as dom from 'vs/base/browser/dom'; + +export class StandaloneColorServiceImpl implements IStandaloneColorService { + + _serviceBrand: any; + + private _onThemeChanged: Emitter = new Emitter(); + public onThemeChanged: Event = this._onThemeChanged.event; + + private _theme: Theme; + private _styleElement: HTMLStyleElement; + + constructor() { + this._theme = Theme.createFromRawTheme(vs); + this._styleElement = dom.createStyleSheet(); + + let colorMap = this._theme.getColorMap(); + let rules: string[] = []; + for (let i = 0, len = colorMap.length; i < len; i++) { + let color = colorMap[i]; + rules[i] = `.mtk${i} { color: #${color}; }`; + } + rules.push('.mtki { font-style: italic; }'); + rules.push('.mtkb { font-weight: bold; }'); + rules.push('.mtku { text-decoration: underline; }'); + + this._styleElement.innerHTML = rules.join('\n'); + } + + public getTheme(): Theme { + return this._theme; + } +} diff --git a/src/vs/editor/browser/standalone/colorizer.ts b/src/vs/editor/browser/standalone/colorizer.ts index 80d447da386..692513c2b8c 100644 --- a/src/vs/editor/browser/standalone/colorizer.ts +++ b/src/vs/editor/browser/standalone/colorizer.ts @@ -12,6 +12,7 @@ import { IModeService } from 'vs/editor/common/services/modeService'; import { renderLine, RenderLineInput } from 'vs/editor/common/viewLayout/viewLineRenderer'; import { ViewLineToken } from 'vs/editor/common/core/viewLineToken'; import { LineParts } from 'vs/editor/common/core/lineParts'; +import { LineTokens } from 'vs/editor/common/core/lineTokens'; import * as strings from 'vs/base/common/strings'; export interface IColorizerOptions { @@ -141,12 +142,12 @@ function _fakeColorize(lines: string[], tabSize: number): string { function _actualColorize(lines: string[], tabSize: number, tokenizationSupport: ITokenizationSupport): string { let html: string[] = []; let state = tokenizationSupport.getInitialState(); + let colorMap = TokenizationRegistry.getColorMap(); for (let i = 0, length = lines.length; i < length; i++) { let line = lines[i]; - - let tokenizeResult = tokenizationSupport.tokenize(line, state, 0); - + let tokenizeResult = tokenizationSupport.tokenize3(line, state, 0); + let lineTokens = new LineTokens(colorMap, tokenizeResult.tokens, line); let renderResult = renderLine(new RenderLineInput( line, tabSize, @@ -154,7 +155,7 @@ function _actualColorize(lines: string[], tabSize: number, tokenizationSupport: -1, 'none', false, - new LineParts(tokenizeResult.tokens.map(t => new ViewLineToken(t.startIndex, t.type)), line.length + 1) + new LineParts(lineTokens.inflate(), line.length + 1) )); html = html.concat(renderResult.output); diff --git a/src/vs/editor/browser/standalone/standaloneLanguages.ts b/src/vs/editor/browser/standalone/standaloneLanguages.ts index f78f63280ec..dd261b8d898 100644 --- a/src/vs/editor/browser/standalone/standaloneLanguages.ts +++ b/src/vs/editor/browser/standalone/standaloneLanguages.ts @@ -23,7 +23,10 @@ import { compile } from 'vs/editor/common/modes/monarch/monarchCompile'; import { createTokenizationSupport } from 'vs/editor/common/modes/monarch/monarchLexer'; import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry'; import { IMarkerData } from 'vs/platform/markers/common/markers'; -import { TokenizationSupport2Adapter } from 'vs/editor/common/services/modeServiceImpl'; +import { Token } from 'vs/editor/common/core/token'; +import { ModeTransition } from 'vs/editor/common/core/modeTransition'; +import { IStandaloneColorService } from 'vs/editor/common/services/standaloneColorService'; + /** * Register information about a new language. */ @@ -61,14 +64,101 @@ export function onLanguage(languageId: string, callback: () => void): IDisposabl * Set the editing configuration for a language. */ export function setLanguageConfiguration(languageId: string, configuration: LanguageConfiguration): IDisposable { - return LanguageConfigurationRegistry.register(languageId, configuration); + let languageIdentifier = StaticServices.modeService.get().getLanguageIdentifier(languageId); + if (!languageIdentifier) { + throw new Error(`Cannot set configuration for unknown language ${languageId}`); + } + return LanguageConfigurationRegistry.register(languageIdentifier, configuration); +} + +/** + * @internal + */ +export class TokenizationSupport2Adapter implements modes.ITokenizationSupport { + + private readonly _standaloneColorService: IStandaloneColorService; + private readonly _languageIdentifier: modes.LanguageIdentifier; + private readonly _actual: modes.TokensProvider; + + constructor(standaloneColorService: IStandaloneColorService, languageIdentifier: modes.LanguageIdentifier, actual: modes.TokensProvider) { + this._standaloneColorService = standaloneColorService; + this._languageIdentifier = languageIdentifier; + this._actual = actual; + } + + public getInitialState(): modes.IState { + return this._actual.getInitialState(); + } + + private _toClassicTokens(tokens: modes.IToken2[], offsetDelta: number): Token[] { + let result: Token[] = []; + for (let i = 0, len = tokens.length; i < len; i++) { + let t = tokens[i]; + result[i] = new Token(t.startIndex + offsetDelta, t.scopes); + } + return result; + } + + public tokenize(line: string, state: modes.IState, offsetDelta: number): modes.ILineTokens { + let actualResult = this._actual.tokenize(line, state); + let tokens = this._toClassicTokens(actualResult.tokens, offsetDelta); + + let endState: modes.IState; + // try to save an object if possible + if (actualResult.endState.equals(state)) { + endState = state; + } else { + endState = actualResult.endState; + } + + return { + tokens: tokens, + endState: endState, + modeTransitions: [new ModeTransition(offsetDelta, this._languageIdentifier.sid)], + }; + } + + private _toBinaryTokens(tokens: modes.IToken2[], offsetDelta: number): Uint32Array { + let languageId = this._languageIdentifier.iid; + let theme = this._standaloneColorService.getTheme(); + + let result = new Uint32Array(tokens.length << 1); + for (let i = 0, len = tokens.length; i < len; i++) { + let t = tokens[i]; + result[(i << 1)] = t.startIndex; + result[(i << 1) + 1] = theme.match(languageId, t.scopes); + } + return result; + } + + public tokenize3(line: string, state: modes.IState, offsetDelta: number): modes.ILineTokens3 { + let actualResult = this._actual.tokenize(line, state); + let tokens = this._toBinaryTokens(actualResult.tokens, offsetDelta); + + let endState: modes.IState; + // try to save an object if possible + if (actualResult.endState.equals(state)) { + endState = state; + } else { + endState = actualResult.endState; + } + + return { + tokens: tokens, + endState: endState + }; + } } /** * Set the tokens provider for a language (manual implementation). */ export function setTokensProvider(languageId: string, provider: modes.TokensProvider): IDisposable { - let adapter = new TokenizationSupport2Adapter(languageId, provider); + let languageIdentifier = StaticServices.modeService.get().getLanguageIdentifier(languageId); + if (!languageIdentifier) { + throw new Error(`Cannot set tokens provider for unknown language ${languageId}`); + } + let adapter = new TokenizationSupport2Adapter(StaticServices.standaloneColorService.get(), languageIdentifier, provider); return modes.TokenizationRegistry.register(languageId, adapter); } @@ -77,7 +167,7 @@ export function setTokensProvider(languageId: string, provider: modes.TokensProv */ export function setMonarchTokensProvider(languageId: string, languageDef: IMonarchLanguage): IDisposable { let lexer = compile(languageId, languageDef); - let adapter = createTokenizationSupport(StaticServices.modeService.get(), languageId, lexer); + let adapter = createTokenizationSupport(StaticServices.modeService.get(), StaticServices.standaloneColorService.get(), languageId, lexer); return modes.TokenizationRegistry.register(languageId, adapter); } @@ -515,6 +605,9 @@ export function createMonacoLanguagesAPI(): typeof monaco.languages { DocumentHighlightKind: modes.DocumentHighlightKind, CompletionItemKind: CompletionItemKind, SymbolKind: modes.SymbolKind, - IndentAction: IndentAction + IndentAction: IndentAction, + + // classes + LanguageIdentifier: modes.LanguageIdentifier }; } diff --git a/src/vs/editor/browser/standalone/standaloneServices.ts b/src/vs/editor/browser/standalone/standaloneServices.ts index 40581337c32..c2430823fce 100644 --- a/src/vs/editor/browser/standalone/standaloneServices.ts +++ b/src/vs/editor/browser/standalone/standaloneServices.ts @@ -39,6 +39,8 @@ import { import { ContextKeyService } from 'vs/platform/contextkey/browser/contextKeyService'; import { IMenuService } from 'vs/platform/actions/common/actions'; import { MenuService } from 'vs/platform/actions/common/menuService'; +import { IStandaloneColorService } from 'vs/editor/common/services/standaloneColorService'; +import { StandaloneColorServiceImpl } from 'vs/editor/browser/services/standaloneColorServiceImpl'; export interface IEditorContextViewService extends IContextViewService { dispose(): void; @@ -138,6 +140,8 @@ export module StaticServices { export const progressService = define(IProgressService, () => new SimpleProgressService()); export const storageService = define(IStorageService, () => NullStorageService); + + export const standaloneColorService = define(IStandaloneColorService, () => new StandaloneColorServiceImpl()); } export class DynamicStandaloneServices extends Disposable { diff --git a/src/vs/editor/browser/viewParts/lines/viewLines.css b/src/vs/editor/browser/viewParts/lines/viewLines.css index bc58cf8e45e..d26b34f8597 100644 --- a/src/vs/editor/browser/viewParts/lines/viewLines.css +++ b/src/vs/editor/browser/viewParts/lines/viewLines.css @@ -16,8 +16,6 @@ .monaco-editor .lines-content, .monaco-editor .view-line, -.monaco-editor .view-line > span, -.monaco-editor .view-line > span > span, .monaco-editor .view-lines { -webkit-user-select: text; -ms-user-select: text; @@ -29,8 +27,6 @@ .monaco-editor.ie .lines-content, .monaco-editor.ie .view-line, -.monaco-editor.ie .view-line > span, -.monaco-editor.ie .view-line > span > span, .monaco-editor.ie .view-lines { -ms-user-select: none; user-select: none; @@ -56,9 +52,9 @@ top: 0; } -/* bootstrap fix */ -.monaco-editor .view-line > span > span { +/* TODO@tokenization bootstrap fix */ +/*.monaco-editor .view-line > span > span { float: none; min-height: inherit; margin-left: inherit; -} \ No newline at end of file +}*/ \ No newline at end of file diff --git a/src/vs/editor/browser/widget/media/tokens.css b/src/vs/editor/browser/widget/media/tokens.css index eee8ea2d1aa..03d60885426 100644 --- a/src/vs/editor/browser/widget/media/tokens.css +++ b/src/vs/editor/browser/widget/media/tokens.css @@ -9,30 +9,31 @@ .monaco-editor.hc-black .token { mix-blend-mode: difference; } - -.monaco-editor.vs .token { color: #000000; } .monaco-editor.vs .token.vs-whitespace { color: rgba(51, 51, 51, 0.2) !important; } +.monaco-editor.vs-dark .token.vs-whitespace { color: rgba(227, 228, 226, 0.16) !important; } +.monaco-editor.hc-black .token.vs-whitespace{ color: rgba(227, 228, 226, 0.16) !important; } + +/* TODO@tokenization */ +/*.monaco-editor.vs .token { color: #000000; } .monaco-editor.vs .token.info-token { color: #316bcd; } .monaco-editor.vs .token.warn-token { color: #cd9731; } .monaco-editor.vs .token.error-token { color: #cd3131; } .monaco-editor.vs .token.debug-token { color: purple; } .monaco-editor.vs-dark .token { color: #D4D4D4; } -.monaco-editor.vs-dark .token.vs-whitespace { color: rgba(227, 228, 226, 0.16) !important; } .monaco-editor.vs-dark .token.info-token { color: #6796e6; } .monaco-editor.vs-dark .token.warn-token { color: #cd9731; } .monaco-editor.vs-dark .token.error-token { color: #f44747; } .monaco-editor.vs-dark .token.debug-token { color: #b267e6; } .monaco-editor.hc-black .token { color: #FFFFFF; } -.monaco-editor.hc-black .token.vs-whitespace{ color: rgba(227, 228, 226, 0.16) !important; } .monaco-editor.hc-black .token.info-token { color: #6796e6; } .monaco-editor.hc-black .token.warn-token { color: #008000; } .monaco-editor.hc-black .token.error-token { color: #FF0000; } -.monaco-editor.hc-black .token.debug-token { color: #b267e6; } +.monaco-editor.hc-black .token.debug-token { color: #b267e6; }*/ /* ----- mimic text bundle support for themes ----- */ -.monaco-editor .markup.bold { font-weight: bold; } +/*.monaco-editor .markup.bold { font-weight: bold; } .monaco-editor .markup.italic { font-style: italic; } -.monaco-editor .markup.underline { text-decoration: underline; } +.monaco-editor .markup.underline { text-decoration: underline; }*/ diff --git a/src/vs/editor/common/controller/cursorCollection.ts b/src/vs/editor/common/controller/cursorCollection.ts index 6671d58d51f..00974bb03ea 100644 --- a/src/vs/editor/common/controller/cursorCollection.ts +++ b/src/vs/editor/common/controller/cursorCollection.ts @@ -330,7 +330,7 @@ export class CursorCollection { let electricChars: string[] = null; try { - electricChars = LanguageConfigurationRegistry.getElectricCharacters(this.model.getMode().getId()); + electricChars = LanguageConfigurationRegistry.getElectricCharacters(this.model.getLanguageIdentifier().iid); } catch (e) { onUnexpectedError(e); electricChars = null; @@ -343,7 +343,7 @@ export class CursorCollection { let autoClosingPairs: IAutoClosingPair[]; try { - autoClosingPairs = LanguageConfigurationRegistry.getAutoClosingPairs(this.model.getMode().getId()); + autoClosingPairs = LanguageConfigurationRegistry.getAutoClosingPairs(this.model.getLanguageIdentifier().iid); } catch (e) { onUnexpectedError(e); autoClosingPairs = null; @@ -357,7 +357,7 @@ export class CursorCollection { let surroundingPairs: IAutoClosingPair[]; try { - surroundingPairs = LanguageConfigurationRegistry.getSurroundingPairs(this.model.getMode().getId()); + surroundingPairs = LanguageConfigurationRegistry.getSurroundingPairs(this.model.getLanguageIdentifier().iid); } catch (e) { onUnexpectedError(e); surroundingPairs = null; diff --git a/src/vs/editor/common/controller/textAreaState.ts b/src/vs/editor/common/controller/textAreaState.ts index d73b1e00bff..b12723009ea 100644 --- a/src/vs/editor/common/controller/textAreaState.ts +++ b/src/vs/editor/common/controller/textAreaState.ts @@ -9,6 +9,7 @@ import { commonPrefixLength, commonSuffixLength } from 'vs/base/common/strings'; import { Range } from 'vs/editor/common/core/range'; import { EndOfLinePreference } from 'vs/editor/common/editorCommon'; import { Position } from 'vs/editor/common/core/position'; +import { Constants } from 'vs/editor/common/core/uint'; export interface IClipboardEvent { canUseTextData(): boolean; @@ -380,7 +381,7 @@ export class NVDAPagedTextAreaState extends TextAreaState { let offset = page * NVDAPagedTextAreaState._LINES_PER_PAGE; let startLineNumber = offset + 1; let endLineNumber = offset + NVDAPagedTextAreaState._LINES_PER_PAGE; - return new Range(startLineNumber, 1, endLineNumber, Number.MAX_VALUE); + return new Range(startLineNumber, 1, endLineNumber, Constants.MAX_SAFE_SMALL_INTEGER); } public fromEditorSelection(model: ISimpleModel, selection: Range): TextAreaState { diff --git a/src/vs/editor/common/core/lineTokens.ts b/src/vs/editor/common/core/lineTokens.ts index 48b2227c30f..f0030ed7cb1 100644 --- a/src/vs/editor/common/core/lineTokens.ts +++ b/src/vs/editor/common/core/lineTokens.ts @@ -4,22 +4,12 @@ *--------------------------------------------------------------------------------------------*/ 'use strict'; -import { TokensBinaryEncoding, TokensInflatorMap } from 'vs/editor/common/model/tokensBinaryEncoding'; -import { ModeTransition } from 'vs/editor/common/core/modeTransition'; +import { TokenMetadata } from 'vs/editor/common/model/tokensBinaryEncoding'; import { ViewLineToken } from 'vs/editor/common/core/viewLineToken'; - -/** - * A standard token type. Values are 2^x such that a bit mask can be used. - */ -export const enum StandardTokenType { - Other = 0, - Comment = 1, - String = 2, - RegEx = 4 -} +import { ColorId, FontStyle, StandardTokenType, LanguageId } from 'vs/editor/common/modes'; const STANDARD_TOKEN_TYPE_REGEXP = /\b(comment|string|regex)\b/; -function toStandardTokenType(tokenType: string): StandardTokenType { +export function toStandardTokenType(tokenType: string): StandardTokenType { let m = tokenType.match(STANDARD_TOKEN_TYPE_REGEXP); if (!m) { return StandardTokenType.Other; @@ -38,87 +28,94 @@ function toStandardTokenType(tokenType: string): StandardTokenType { export class LineToken { _lineTokenBrand: void; - private _source: LineTokens; - private _tokenIndex: number; - private _modeIndex: number; + private readonly _colorMap: string[]; + private readonly _source: LineTokens; + private readonly _tokenIndex: number; + private readonly _metadata: number; public readonly startOffset: number; public readonly endOffset: number; - public readonly standardType: StandardTokenType; - public readonly modeId: string; + public readonly hasPrev: boolean; public readonly hasNext: boolean; - constructor(source: LineTokens, tokenIndex: number, modeIndex: number) { + public get languageId(): LanguageId { + return TokenMetadata.getLanguageId(this._metadata); + } + + public get tokenType(): StandardTokenType { + return TokenMetadata.getTokenType(this._metadata); + } + + public get fontStyle(): FontStyle { + return TokenMetadata.getFontStyle(this._metadata); + } + + public get foregroundId(): ColorId { + return TokenMetadata.getForeground(this._metadata); + } + + public get foreground(): string { + return this._colorMap[this.foregroundId]; + } + + public get backgroundId(): ColorId { + return TokenMetadata.getBackground(this._metadata); + } + + public get background(): string { + return this._colorMap[this.backgroundId]; + } + + constructor(colorMap: string[], source: LineTokens, tokenIndex: number, tokenCount: number, startOffset: number, endOffset: number, metadata: number) { + this._colorMap = colorMap; this._source = source; this._tokenIndex = tokenIndex; - this._modeIndex = modeIndex; + this._metadata = metadata; + + this.startOffset = startOffset; + this.endOffset = endOffset; - this.startOffset = this._source.getTokenStartOffset(this._tokenIndex); - this.endOffset = this._source.getTokenEndOffset(this._tokenIndex); - this.standardType = this._source.getStandardTokenType(this._tokenIndex); - this.modeId = this._source.modeTransitions[this._modeIndex].modeId; this.hasPrev = (this._tokenIndex > 0); - this.hasNext = (this._tokenIndex + 1 < this._source.getTokenCount()); + this.hasNext = (this._tokenIndex + 1 < tokenCount); } public prev(): LineToken { if (!this.hasPrev) { return null; } - if (this._modeIndex === 0) { - return new LineToken(this._source, this._tokenIndex - 1, this._modeIndex); - } - const modeTransitions = this._source.modeTransitions; - const currentModeTransition = modeTransitions[this._modeIndex]; - const prevStartOffset = this._source.getTokenStartOffset(this._tokenIndex - 1); - if (prevStartOffset < currentModeTransition.startIndex) { - // Going to previous mode transition - return new LineToken(this._source, this._tokenIndex - 1, this._modeIndex - 1); - } - return new LineToken(this._source, this._tokenIndex - 1, this._modeIndex); + return this._source.tokenAt(this._tokenIndex - 1); } public next(): LineToken { if (!this.hasNext) { return null; } - const modeTransitions = this._source.modeTransitions; - if (this._modeIndex === modeTransitions.length - 1) { - return new LineToken(this._source, this._tokenIndex + 1, this._modeIndex); - } - const nextModeTransition = modeTransitions[this._modeIndex + 1]; - const nextStartOffset = this._source.getTokenStartOffset(this._tokenIndex + 1); - if (nextStartOffset >= nextModeTransition.startIndex) { - // Going to next mode transition - return new LineToken(this._source, this._tokenIndex + 1, this._modeIndex + 1); - } - return new LineToken(this._source, this._tokenIndex + 1, this._modeIndex); + return this._source.tokenAt(this._tokenIndex + 1); } } export class LineTokens { _lineTokensBrand: void; - private readonly _map: TokensInflatorMap; - private readonly _tokens: number[]; + private readonly _colorMap: string[]; + private readonly _tokens: Uint32Array; + private readonly _tokensCount: number; private readonly _text: string; private readonly _textLength: number; - readonly modeTransitions: ModeTransition[]; - - constructor(map: TokensInflatorMap, tokens: number[], modeTransitions: ModeTransition[], text: string) { - this._map = map; + constructor(colorMap: string[], tokens: Uint32Array, text: string) { + this._colorMap = colorMap; this._tokens = tokens; - this.modeTransitions = modeTransitions; + this._tokensCount = (this._tokens.length >>> 1); this._text = text; this._textLength = this._text.length; } public getTokenCount(): number { - return this._tokens.length; + return this._tokensCount; } public getLineContent(): string { @@ -126,69 +123,77 @@ export class LineTokens { } public getTokenStartOffset(tokenIndex: number): number { - return TokensBinaryEncoding.getStartIndex(this._tokens[tokenIndex]); + return this._tokens[(tokenIndex << 1)]; } - /** - * Deprecated. Do not use. - * @deprecated - */ - public getTokenType(tokenIndex: number): string { - return TokensBinaryEncoding.getType(this._map, this._tokens[tokenIndex]); + public getLanguageId(tokenIndex: number): LanguageId { + let metadata = this._tokens[(tokenIndex << 1) + 1]; + return TokenMetadata.getLanguageId(metadata); } public getStandardTokenType(tokenIndex: number): StandardTokenType { - return toStandardTokenType(this.getTokenType(tokenIndex)); + let metadata = this._tokens[(tokenIndex << 1) + 1]; + return TokenMetadata.getTokenType(metadata); } public getTokenEndOffset(tokenIndex: number): number { - if (tokenIndex + 1 < this._tokens.length) { - return TokensBinaryEncoding.getStartIndex(this._tokens[tokenIndex + 1]); + if (tokenIndex + 1 < this._tokensCount) { + return this._tokens[(tokenIndex + 1) << 1]; } return this._textLength; } /** * Find the token containing offset `offset`. - * For example, with the following tokens [0, 5), [5, 9), [9, infinity) - * Searching for 0, 1, 2, 3 or 4 will return 0. - * Searching for 5, 6, 7 or 8 will return 1. - * Searching for 9, 10, 11, ... will return 2. + * ``` + * For example, with the following tokens [0, 5), [5, 9), [9, infinity) + * Searching for 0, 1, 2, 3 or 4 will return 0. + * Searching for 5, 6, 7 or 8 will return 1. + * Searching for 9, 10, 11, ... will return 2. + * ``` * @param offset The search offset * @return The index of the token containing the offset. */ public findTokenIndexAtOffset(offset: number): number { - return TokensBinaryEncoding.findIndexOfOffset(this._tokens, offset); + return TokenMetadata.findIndexInSegmentsArray(this._tokens, offset); } public findTokenAtOffset(offset: number): LineToken { - if (this._textLength === 0) { - return null; - } let tokenIndex = this.findTokenIndexAtOffset(offset); - let modeIndex = ModeTransition.findIndexInSegmentsArray(this.modeTransitions, offset); - return new LineToken(this, tokenIndex, modeIndex); + return this.tokenAt(tokenIndex); + } + + public tokenAt(tokenIndex: number): LineToken { + let startOffset = this._tokens[(tokenIndex << 1)]; + let endOffset: number; + if (tokenIndex + 1 < this._tokensCount) { + endOffset = this._tokens[(tokenIndex + 1) << 1]; + } else { + endOffset = this._textLength; + } + let metadata = this._tokens[(tokenIndex << 1) + 1]; + return new LineToken(this._colorMap, this, tokenIndex, this._tokensCount, startOffset, endOffset, metadata); } public firstToken(): LineToken { if (this._textLength === 0) { return null; } - return new LineToken(this, 0, 0); + return this.tokenAt(0); } public lastToken(): LineToken { if (this._textLength === 0) { return null; } - return new LineToken(this, this._tokens.length - 1, this.modeTransitions.length - 1); + return this.tokenAt(this._tokensCount - 1); } public inflate(): ViewLineToken[] { - return TokensBinaryEncoding.inflateArr(this._map, this._tokens); + return TokenMetadata.inflateArr(this._tokens); } public sliceAndInflate(startOffset: number, endOffset: number, deltaStartIndex: number): ViewLineToken[] { - return TokensBinaryEncoding.sliceAndInflate(this._map, this._tokens, startOffset, endOffset, deltaStartIndex); + return TokenMetadata.sliceAndInflate(this._tokens, startOffset, endOffset, deltaStartIndex); } } diff --git a/src/vs/editor/common/core/viewLineToken.ts b/src/vs/editor/common/core/viewLineToken.ts index ac6b4e922ec..e64d7ddb593 100644 --- a/src/vs/editor/common/core/viewLineToken.ts +++ b/src/vs/editor/common/core/viewLineToken.ts @@ -17,6 +17,7 @@ export class ViewLineToken { constructor(startIndex: number, type: string) { this.startIndex = startIndex | 0;// @perf + // TODO@tokenization: remove check? this.type = type.replace(/[^a-z0-9\-]/gi, ' '); } diff --git a/src/vs/editor/common/editorCommon.ts b/src/vs/editor/common/editorCommon.ts index 2c169e356fb..b8835ea782b 100644 --- a/src/vs/editor/common/editorCommon.ts +++ b/src/vs/editor/common/editorCommon.ts @@ -10,8 +10,8 @@ import * as types from 'vs/base/common/types'; import URI from 'vs/base/common/uri'; import { TPromise } from 'vs/base/common/winjs.base'; import { ServicesAccessor, IConstructorSignature1 } from 'vs/platform/instantiation/common/instantiation'; -import { IMode } from 'vs/editor/common/modes'; -import { LineTokens, StandardTokenType } from 'vs/editor/common/core/lineTokens'; +import { IMode, LanguageId, LanguageIdentifier, StandardTokenType } from 'vs/editor/common/modes'; +import { LineTokens } from 'vs/editor/common/core/lineTokens'; import { ScrollbarVisibility } from 'vs/base/common/scrollable'; import { IDisposable } from 'vs/base/common/lifecycle'; import { Position } from 'vs/editor/common/core/position'; @@ -1294,7 +1294,7 @@ export interface IWordRange { * @internal */ export interface ITokenInfo { - readonly standardType: StandardTokenType; + readonly type: StandardTokenType; readonly lineNumber: number; readonly startColumn: number; readonly endColumn: number; @@ -1818,9 +1818,16 @@ export interface IReadOnlyModel extends ITextModel { /** * Get the language associated with this model. + * TODO@tokenization + * @deprecated */ getModeId(): string; + /** + * Get the language associated with this model. + */ + getLanguageIdentifier(): LanguageIdentifier; + /** * Get the word under or besides `position`. * @param position The position to look for a word. @@ -1863,25 +1870,34 @@ export interface ITokenizedModel extends ITextModel { /** * Get the current language mode associated with the model. + * TODO@tokenization + * @deprecated */ getMode(): IMode; /** * Get the language associated with this model. + * TODO@tokenization + * @deprecated */ getModeId(): string; + /** + * Get the language associated with this model. + */ + getLanguageIdentifier(): LanguageIdentifier; + /** * Set the current language mode associated with the model. * @internal */ - setMode(languageId: string): void; + setMode(languageIdentifier: LanguageIdentifier): void; /** * Returns the true (inner-most) language mode at a given position. * @internal */ - getModeIdAtPosition(lineNumber: number, column: number): string; + getLanguageIdAtPosition(lineNumber: number, column: number): LanguageId; /** * Get the word under or besides `position`. diff --git a/src/vs/editor/common/model/editableTextModel.ts b/src/vs/editor/common/model/editableTextModel.ts index 8105b0262c1..65dcb6ef52d 100644 --- a/src/vs/editor/common/model/editableTextModel.ts +++ b/src/vs/editor/common/model/editableTextModel.ts @@ -13,6 +13,7 @@ import * as strings from 'vs/base/common/strings'; import { Selection } from 'vs/editor/common/core/selection'; import { Position } from 'vs/editor/common/core/position'; import { IDisposable } from 'vs/base/common/lifecycle'; +import { LanguageIdentifier } from 'vs/editor/common/modes'; export interface IValidatedEditOperation { sortIndex: number; @@ -49,9 +50,9 @@ export class EditableTextModel extends TextModelWithDecorations implements edito private _trimAutoWhitespaceLines: number[]; - constructor(allowedEventTypes: string[], rawText: editorCommon.IRawText, languageId: string) { + constructor(allowedEventTypes: string[], rawText: editorCommon.IRawText, languageIdentifier: LanguageIdentifier) { allowedEventTypes.push(editorCommon.EventType.ModelRawContentChanged); - super(allowedEventTypes, rawText, languageId); + super(allowedEventTypes, rawText, languageIdentifier); this._commandManager = new EditStack(this); diff --git a/src/vs/editor/common/model/model.ts b/src/vs/editor/common/model/model.ts index e69a16223fe..4af20f75a77 100644 --- a/src/vs/editor/common/model/model.ts +++ b/src/vs/editor/common/model/model.ts @@ -13,6 +13,7 @@ import { EditableTextModel } from 'vs/editor/common/model/editableTextModel'; import { TextModel } from 'vs/editor/common/model/textModel'; import { IDisposable } from 'vs/base/common/lifecycle'; import { BulkListenerCallback } from 'vs/base/common/eventEmitter'; +import { LanguageIdentifier } from 'vs/editor/common/modes'; // The hierarchy is: // Model -> EditableTextModel -> TextModelWithDecorations -> TextModelWithTrackedRanges -> TextModelWithMarkers -> TextModelWithTokens -> TextModel @@ -51,9 +52,9 @@ export class Model extends EditableTextModel implements IModel { return super.addBulkListener(listener); } - public static createFromString(text: string, options: ITextModelCreationOptions = TextModel.DEFAULT_CREATION_OPTIONS, languageId: string = null, uri: URI = null): Model { + public static createFromString(text: string, options: ITextModelCreationOptions = TextModel.DEFAULT_CREATION_OPTIONS, languageIdentifier: LanguageIdentifier = null, uri: URI = null): Model { let rawText = TextModel.toRawText(text, options); - return new Model(rawText, languageId, uri); + return new Model(rawText, languageIdentifier, uri); } public id: string; @@ -74,8 +75,8 @@ export class Model extends EditableTextModel implements IModel { * The resource associated with this model. If the value is not provided an * unique in memory URL is constructed as the associated resource. */ - constructor(rawText: IRawText, languageId: string, associatedResource: URI = null) { - super([EventType.ModelDispose], rawText, languageId); + constructor(rawText: IRawText, languageIdentifier: LanguageIdentifier, associatedResource: URI = null) { + super([EventType.ModelDispose], rawText, languageIdentifier); // Generate a new unique model id MODEL_ID++; diff --git a/src/vs/editor/common/model/modelLine.ts b/src/vs/editor/common/model/modelLine.ts index f3773cd4c9c..7c7340aef50 100644 --- a/src/vs/editor/common/model/modelLine.ts +++ b/src/vs/editor/common/model/modelLine.ts @@ -4,13 +4,11 @@ *--------------------------------------------------------------------------------------------*/ 'use strict'; -import { IState } from 'vs/editor/common/modes'; -import { TokensBinaryEncoding, DEFLATED_TOKENS_EMPTY_TEXT, DEFLATED_TOKENS_NON_EMPTY_TEXT, TokensBinaryEncodingValues, TokensInflatorMap } from 'vs/editor/common/model/tokensBinaryEncoding'; -import { ModeTransition } from 'vs/editor/common/core/modeTransition'; -import { Token } from 'vs/editor/common/core/token'; +import { IState, FontStyle, StandardTokenType, MetadataConsts, ColorId, LanguageId } from 'vs/editor/common/modes'; import { CharCode } from 'vs/base/common/charCode'; import { LineTokens } from 'vs/editor/common/core/lineTokens'; import { Position } from 'vs/editor/common/core/position'; +import { Constants } from 'vs/editor/common/core/uint'; export interface ILineEdit { startColumn: number; @@ -204,8 +202,7 @@ export class ModelLine { } private _state: IState; - private _modeTransitions: ModeTransition[]; - private _lineTokens: number[]; + private _lineTokens: ArrayBuffer; private _markers: LineMarker[]; constructor(lineNumber: number, text: string, tabSize: number) { @@ -213,7 +210,6 @@ export class ModelLine { this._metadata = 0; this._setText(text, tabSize); this._state = null; - this._modeTransitions = null; this._lineTokens = null; this._markers = null; } @@ -222,7 +218,6 @@ export class ModelLine { public resetTokenizationState(): void { this._state = null; - this._modeTransitions = null; this._lineTokens = null; } @@ -236,40 +231,43 @@ export class ModelLine { // --- END STATE - // --- BEGIN MODE TRANSITIONS - - public getModeTransitions(topLevelModeId: string): ModeTransition[] { - if (this._modeTransitions) { - return this._modeTransitions; - } else { - return [new ModeTransition(0, topLevelModeId)]; - } - } - - // --- END MODE TRANSITIONS - // --- BEGIN TOKENS - public setTokens(map: TokensInflatorMap, tokens: Token[], modeTransitions: ModeTransition[]): void { - this._lineTokens = toLineTokensFromInflated(map, tokens, this._text.length); - this._modeTransitions = toModeTransitions(map.topLevelModeId, modeTransitions); + private static _getDefaultMetadata(topLevelLanguageId: LanguageId): number { + return ( + (topLevelLanguageId << MetadataConsts.LANGUAGEID_OFFSET) + | (StandardTokenType.Other << MetadataConsts.TOKEN_TYPE_OFFSET) + | (FontStyle.None << MetadataConsts.FONT_STYLE_OFFSET) + | (ColorId.DefaultForeground << MetadataConsts.FOREGROUND_OFFSET) + | (ColorId.DefaultBackground << MetadataConsts.BACKGROUND_OFFSET) + ) >>> 0; } - private _setLineTokensFromDeflated(tokens: number[]): void { - this._lineTokens = toLineTokensFromDeflated(tokens, this._text.length); - } - - public getTokens(map: TokensInflatorMap): LineTokens { - let lineTokens = this._lineTokens; - if (!lineTokens) { - if (this._text.length === 0) { - lineTokens = DEFLATED_TOKENS_EMPTY_TEXT; - } else { - lineTokens = DEFLATED_TOKENS_NON_EMPTY_TEXT; + public setTokens(topLevelLanguageId: LanguageId, tokens: Uint32Array): void { + if (!tokens || tokens.length === 0) { + this._lineTokens = null; + return; + } + if (tokens.length === 2) { + // there is one token + if (tokens[0] === 0 && tokens[1] === ModelLine._getDefaultMetadata(topLevelLanguageId)) { + this._lineTokens = null; + return; } } + this._lineTokens = tokens.buffer; + } - return new LineTokens(map, lineTokens, this.getModeTransitions(map.topLevelModeId), this._text); + public getTokens(topLevelLanguageId: LanguageId, colorMap: string[]): LineTokens { + let rawLineTokens = this._lineTokens; + if (rawLineTokens) { + return new LineTokens(colorMap, new Uint32Array(rawLineTokens), this._text); + } + + let lineTokens = new Uint32Array(2); + lineTokens[0] = 0; + lineTokens[1] = ModelLine._getDefaultMetadata(topLevelLanguageId); + return new LineTokens(colorMap, lineTokens, this._text); } // --- END TOKENS @@ -280,53 +278,59 @@ export class ModelLine { return NO_OP_TOKENS_ADJUSTER; } - let tokens = this._lineTokens; - let tokensLength = tokens.length; - let tokensIndex = 0; - let currentTokenStartIndex = 0; + let lineTokens = new Uint32Array(this._lineTokens); + let tokensLength = (lineTokens.length >>> 1); + let tokenIndex = 0; + let tokenStartOffset = 0; + let removeTokensCount = 0; let adjust = (toColumn: number, delta: number, minimumAllowedColumn: number) => { - // console.log('before call: tokensIndex: ' + tokensIndex + ': ' + String(this.getTokens())); - // console.log('adjustTokens: ' + toColumn + ' with delta: ' + delta + ' and [' + minimumAllowedColumn + ']'); - // console.log('currentTokenStartIndex: ' + currentTokenStartIndex); + // console.log(`------------------------------------------------------------------`); + // console.log(`before call: tokenIndex: ${tokenIndex}: ${lineTokens}`); + // console.log(`adjustTokens: ${toColumn} with delta: ${delta} and [${minimumAllowedColumn}]`); + // console.log(`tokenStartOffset: ${tokenStartOffset}`); let minimumAllowedIndex = minimumAllowedColumn - 1; - while (currentTokenStartIndex < toColumn && tokensIndex < tokensLength) { + while (tokenStartOffset < toColumn && tokenIndex < tokensLength) { - if (currentTokenStartIndex > 0 && delta !== 0) { + if (tokenStartOffset > 0 && delta !== 0) { // adjust token's `startIndex` by `delta` - let deflatedType = (tokens[tokensIndex] / TokensBinaryEncodingValues.TYPE_OFFSET) & TokensBinaryEncodingValues.TYPE_MASK; - let newStartIndex = Math.max(minimumAllowedIndex, currentTokenStartIndex + delta); - let newToken = deflatedType * TokensBinaryEncodingValues.TYPE_OFFSET + newStartIndex * TokensBinaryEncodingValues.START_INDEX_OFFSET; + let newTokenStartOffset = Math.max(minimumAllowedIndex, tokenStartOffset + delta); + lineTokens[(tokenIndex << 1)] = newTokenStartOffset; + + // console.log(` * adjusted token start offset for token at ${tokenIndex}: ${newTokenStartOffset}`); if (delta < 0) { - // pop all previous tokens that have become `collapsed` - while (tokensIndex > 0) { - let prevTokenStartIndex = (tokens[tokensIndex - 1] / TokensBinaryEncodingValues.START_INDEX_OFFSET) & TokensBinaryEncodingValues.START_INDEX_MASK; - if (prevTokenStartIndex >= newStartIndex) { - // Token at `tokensIndex` - 1 is now `collapsed` => pop it - tokens.splice(tokensIndex - 1, 1); - tokensLength--; - tokensIndex--; + let tmpTokenIndex = tokenIndex; + while (tmpTokenIndex > 0) { + let prevTokenStartOffset = lineTokens[((tmpTokenIndex - 1) << 1)]; + if (prevTokenStartOffset >= newTokenStartOffset) { + if (prevTokenStartOffset !== Constants.MAX_UINT_32) { + // console.log(` * marking for deletion token at ${tmpTokenIndex - 1}`); + lineTokens[((tmpTokenIndex - 1) << 1)] = Constants.MAX_UINT_32; + removeTokensCount++; + } + tmpTokenIndex--; } else { break; } } } - tokens[tokensIndex] = newToken; } - tokensIndex++; - - if (tokensIndex < tokensLength) { - currentTokenStartIndex = (tokens[tokensIndex] / TokensBinaryEncodingValues.START_INDEX_OFFSET) & TokensBinaryEncodingValues.START_INDEX_MASK; + tokenIndex++; + if (tokenIndex < tokensLength) { + tokenStartOffset = lineTokens[(tokenIndex << 1)]; } } - // console.log('after call: tokensIndex: ' + tokensIndex + ': ' + String(this.getTokens())); + // console.log(`after call: tokenIndex: ${tokenIndex}: ${lineTokens}`); }; let finish = (delta: number, lineTextLength: number) => { - adjust(Number.MAX_VALUE, delta, 1); + adjust(Constants.MAX_SAFE_SMALL_INTEGER, delta, 1); + + // Mark overflowing tokens for deletion & delete marked tokens + this._deleteMarkedTokens(this._markOverflowingTokensForDeletion(removeTokensCount, lineTextLength)); }; return { @@ -335,6 +339,63 @@ export class ModelLine { }; } + private _markOverflowingTokensForDeletion(removeTokensCount: number, lineTextLength: number): number { + if (!this._lineTokens) { + return removeTokensCount; + } + + let lineTokens = new Uint32Array(this._lineTokens); + let tokensLength = (lineTokens.length >>> 1); + + if (removeTokensCount + 1 === tokensLength) { + // no more removing, cannot end up without any tokens for mode transition reasons + return removeTokensCount; + } + + for (let tokenIndex = tokensLength - 1; tokenIndex > 0; tokenIndex--) { + let tokenStartOffset = lineTokens[(tokenIndex << 1)]; + if (tokenStartOffset < lineTextLength) { + // valid token => stop iterating + return removeTokensCount; + } + + // this token now overflows the text => mark it for removal + if (tokenStartOffset !== Constants.MAX_UINT_32) { + // console.log(` * marking for deletion token at ${tokenIndex}`); + lineTokens[(tokenIndex << 1)] = Constants.MAX_UINT_32; + removeTokensCount++; + + if (removeTokensCount + 1 === tokensLength) { + // no more removing, cannot end up without any tokens for mode transition reasons + return removeTokensCount; + } + } + } + + return removeTokensCount; + } + + private _deleteMarkedTokens(removeTokensCount: number): void { + if (removeTokensCount === 0) { + return; + } + + let lineTokens = new Uint32Array(this._lineTokens); + let tokensLength = (lineTokens.length >>> 1); + let newTokens = new Uint32Array(((tokensLength - removeTokensCount) << 1)), newTokenIdx = 0; + for (let i = 0; i < tokensLength; i++) { + let startOffset = lineTokens[(i << 1)]; + if (startOffset === Constants.MAX_UINT_32) { + // marked for deletion + continue; + } + let metadata = lineTokens[(i << 1) + 1]; + newTokens[newTokenIdx++] = startOffset; + newTokens[newTokenIdx++] = metadata; + } + this._lineTokens = newTokens.buffer; + } + private _setText(text: string, tabSize: number): void { this._text = text; if (tabSize === 0) { @@ -343,24 +404,6 @@ export class ModelLine { } else { this._setPlusOneIndentLevel(computePlusOneIndentLevel(text, tabSize)); } - - let tokens = this._lineTokens; - if (tokens) { - let lineTextLength = this._text.length; - - // Remove overflowing tokens - while (tokens.length > 0) { - let lastTokenStartIndex = (tokens[tokens.length - 1] / TokensBinaryEncodingValues.START_INDEX_OFFSET) & TokensBinaryEncodingValues.START_INDEX_MASK; - if (lastTokenStartIndex < lineTextLength) { - // Valid token - break; - } - // This token now overflows the text => remove it - tokens.pop(); - } - - this._setLineTokensFromDeflated(tokens); - } } // private _printMarkers(): string { @@ -453,7 +496,7 @@ export class ModelLine { }; let finish = (delta: number, lineTextLength: number) => { - adjustDelta(Number.MAX_VALUE, delta, 1, MarkerMoveSemantics.MarkerDefined); + adjustDelta(Constants.MAX_SAFE_SMALL_INTEGER, delta, 1, MarkerMoveSemantics.MarkerDefined); // console.log('------------- FINAL MARKERS: ' + this._printMarkers()); }; @@ -561,6 +604,9 @@ export class ModelLine { this._setText(myText, tabSize); + // Mark overflowing tokens for deletion & delete marked tokens + this._deleteMarkedTokens(this._markOverflowingTokensForDeletion(0, this._text.length)); + var otherLine = new ModelLine(this._lineNumber + 1, otherText, tabSize); if (otherMarkers) { otherLine.addMarkers(otherMarkers); @@ -571,35 +617,34 @@ export class ModelLine { public append(markersTracker: MarkersTracker, other: ModelLine, tabSize: number): void { // console.log('--> append: THIS :: ' + this._printMarkers()); // console.log('--> append: OTHER :: ' + this._printMarkers()); - var thisTextLength = this._text.length; + let thisTextLength = this._text.length; this._setText(this._text + other._text, tabSize); - let otherTokens = other._lineTokens; - if (otherTokens) { + let otherRawTokens = other._lineTokens; + if (otherRawTokens) { // Other has real tokens + let otherTokens = new Uint32Array(otherRawTokens); + // Adjust other tokens if (thisTextLength > 0) { - for (let i = 0, len = otherTokens.length; i < len; i++) { - let token = otherTokens[i]; - - let deflatedStartIndex = (token / TokensBinaryEncodingValues.START_INDEX_OFFSET) & TokensBinaryEncodingValues.START_INDEX_MASK; - let deflatedType = (token / TokensBinaryEncodingValues.TYPE_OFFSET) & TokensBinaryEncodingValues.TYPE_MASK; - let newStartIndex = deflatedStartIndex + thisTextLength; - let newToken = deflatedType * TokensBinaryEncodingValues.TYPE_OFFSET + newStartIndex * TokensBinaryEncodingValues.START_INDEX_OFFSET; - - otherTokens[i] = newToken; + for (let i = 0, len = (otherTokens.length >>> 1); i < len; i++) { + otherTokens[(i << 1)] = otherTokens[(i << 1)] + thisTextLength; } } // Append other tokens - let myLineTokens = this._lineTokens; - if (myLineTokens) { + let myRawTokens = this._lineTokens; + if (myRawTokens) { // I have real tokens - this._setLineTokensFromDeflated(myLineTokens.concat(otherTokens)); + let myTokens = new Uint32Array(myRawTokens); + let result = new Uint32Array(myTokens.length + otherTokens.length); + result.set(myTokens, 0); + result.set(otherTokens, myTokens.length); + this._lineTokens = result.buffer; } else { // I don't have real tokens - this._setLineTokensFromDeflated(otherTokens); + this._lineTokens = otherTokens.buffer; } } @@ -712,45 +757,3 @@ export class ModelLine { } } } - -function toLineTokensFromInflated(map: TokensInflatorMap, tokens: Token[], textLength: number): number[] { - if (textLength === 0) { - return null; - } - if (!tokens || tokens.length === 0) { - return null; - } - if (tokens.length === 1) { - if (tokens[0].startIndex === 0 && tokens[0].type === '') { - return null; - } - } - - return TokensBinaryEncoding.deflateArr(map, tokens); -} - -function toLineTokensFromDeflated(tokens: number[], textLength: number): number[] { - if (textLength === 0) { - return null; - } - if (!tokens || tokens.length === 0) { - return null; - } - if (tokens.length === 1) { - if (tokens[0] === 0) { - return null; - } - } - return tokens; -} - -function toModeTransitions(topLevelModeId: string, modeTransitions: ModeTransition[]): ModeTransition[] { - - if (!modeTransitions || modeTransitions.length === 0) { - return null; - } else if (modeTransitions.length === 1 && modeTransitions[0].startIndex === 0 && modeTransitions[0].modeId === topLevelModeId) { - return null; - } - - return modeTransitions; -} diff --git a/src/vs/editor/common/model/textModelWithDecorations.ts b/src/vs/editor/common/model/textModelWithDecorations.ts index 0439168ae7d..654172d5344 100644 --- a/src/vs/editor/common/model/textModelWithDecorations.ts +++ b/src/vs/editor/common/model/textModelWithDecorations.ts @@ -13,6 +13,7 @@ import * as editorCommon from 'vs/editor/common/editorCommon'; import { MarkersTracker, LineMarker } from 'vs/editor/common/model/modelLine'; import { Position } from 'vs/editor/common/core/position'; import { INewMarker, TextModelWithMarkers } from 'vs/editor/common/model/textModelWithMarkers'; +import { LanguageIdentifier } from 'vs/editor/common/modes'; class DecorationsTracker { @@ -138,9 +139,9 @@ export class TextModelWithDecorations extends TextModelWithMarkers implements ed private _internalDecorations: { [internalDecorationId: number]: InternalDecoration; }; private _multiLineDecorationsMap: { [key: string]: InternalDecoration; }; - constructor(allowedEventTypes: string[], rawText: editorCommon.IRawText, languageId: string) { + constructor(allowedEventTypes: string[], rawText: editorCommon.IRawText, languageIdentifier: LanguageIdentifier) { allowedEventTypes.push(editorCommon.EventType.ModelDecorationsChanged); - super(allowedEventTypes, rawText, languageId); + super(allowedEventTypes, rawText, languageIdentifier); this._instanceId = nextInstanceId(); this._lastDecorationId = 0; diff --git a/src/vs/editor/common/model/textModelWithMarkers.ts b/src/vs/editor/common/model/textModelWithMarkers.ts index 85fbb0af312..7c69a44c7da 100644 --- a/src/vs/editor/common/model/textModelWithMarkers.ts +++ b/src/vs/editor/common/model/textModelWithMarkers.ts @@ -9,6 +9,7 @@ import { Position } from 'vs/editor/common/core/position'; import { IRawText, ITextModelWithMarkers } from 'vs/editor/common/editorCommon'; import { LineMarker } from 'vs/editor/common/model/modelLine'; import { TextModelWithTokens } from 'vs/editor/common/model/textModelWithTokens'; +import { LanguageIdentifier } from 'vs/editor/common/modes'; export interface IMarkerIdToMarkerMap { [key: string]: LineMarker; @@ -26,8 +27,9 @@ export class TextModelWithMarkers extends TextModelWithTokens implements ITextMo private _markerIdGenerator: IdGenerator; protected _markerIdToMarker: IMarkerIdToMarkerMap; - constructor(allowedEventTypes: string[], rawText: IRawText, languageId: string) { - super(allowedEventTypes, rawText, languageId); + + constructor(allowedEventTypes: string[], rawText: IRawText, languageIdentifier: LanguageIdentifier) { + super(allowedEventTypes, rawText, languageIdentifier); this._markerIdGenerator = new IdGenerator((++_INSTANCE_COUNT) + ';'); this._markerIdToMarker = Object.create(null); } diff --git a/src/vs/editor/common/model/textModelWithTokens.ts b/src/vs/editor/common/model/textModelWithTokens.ts index 593235cf73c..529b74e0c48 100644 --- a/src/vs/editor/common/model/textModelWithTokens.ts +++ b/src/vs/editor/common/model/textModelWithTokens.ts @@ -13,12 +13,10 @@ import { Range } from 'vs/editor/common/core/range'; import * as editorCommon from 'vs/editor/common/editorCommon'; import { TextModel } from 'vs/editor/common/model/textModel'; import { TokenIterator } from 'vs/editor/common/model/tokenIterator'; -import { ITokenizationSupport, ILineTokens, IMode, IState, TokenizationRegistry } from 'vs/editor/common/modes'; -import { NULL_MODE_ID, nullTokenize } from 'vs/editor/common/modes/nullMode'; +import { ITokenizationSupport, ILineTokens3, IMode, IState, TokenizationRegistry, LanguageId, LanguageIdentifier } from 'vs/editor/common/modes'; +import { NULL_LANGUAGE_IDENTIFIER, nullTokenize3 } from 'vs/editor/common/modes/nullMode'; import { ignoreBracketsInToken } from 'vs/editor/common/modes/supports'; import { BracketsUtils, RichEditBrackets, RichEditBracket } from 'vs/editor/common/modes/supports/richEditBrackets'; -import { ModeTransition } from 'vs/editor/common/core/modeTransition'; -import { TokensInflatorMap } from 'vs/editor/common/model/tokensBinaryEncoding'; import { Position } from 'vs/editor/common/core/position'; import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry'; import { LineTokens, LineToken } from 'vs/editor/common/core/lineTokens'; @@ -26,14 +24,18 @@ import { getWordAtText } from 'vs/editor/common/model/wordHelper'; class Mode implements IMode { - private _languageId: string; + private _languageIdentifier: LanguageIdentifier; - constructor(languageId: string) { - this._languageId = languageId; + constructor(languageId: LanguageIdentifier) { + this._languageIdentifier = languageId; } getId(): string { - return this._languageId; + return this._languageIdentifier.sid; + } + + getLanguageIdentifier(): LanguageIdentifier { + return this._languageIdentifier; } } @@ -76,24 +78,24 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke private static MODE_TOKENIZATION_FAILED_MSG = nls.localize('mode.tokenizationSupportFailed', "The mode has failed while tokenizing the input."); - private _languageId: string; + private _languageIdentifier: LanguageIdentifier; private _tokenizationListener: IDisposable; + private _colorMap: string[]; private _tokenizationSupport: ITokenizationSupport; - private _tokensInflatorMap: TokensInflatorMap; private _invalidLineStartIndex: number; private _lastState: IState; private _revalidateTokensTimeout: number; - constructor(allowedEventTypes: string[], rawText: editorCommon.IRawText, languageId: string) { + constructor(allowedEventTypes: string[], rawText: editorCommon.IRawText, languageIdentifier: LanguageIdentifier) { allowedEventTypes.push(editorCommon.EventType.ModelTokensChanged); allowedEventTypes.push(editorCommon.EventType.ModelModeChanged); super(allowedEventTypes, rawText); - this._languageId = languageId || NULL_MODE_ID; + this._languageIdentifier = languageIdentifier || NULL_LANGUAGE_IDENTIFIER; this._tokenizationListener = TokenizationRegistry.onDidChange((e) => { - if (e.languageId !== this._languageId) { + if (e.languageId !== this._languageIdentifier.sid) { return; } @@ -105,10 +107,6 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke }] }); }); - this._tokensInflatorMap = null; - - this._invalidLineStartIndex = 0; - this._lastState = null; this._revalidateTokensTimeout = -1; @@ -119,7 +117,6 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke this._tokenizationListener.dispose(); this._clearTimers(); this._lastState = null; - this._tokensInflatorMap = null; super.dispose(); } @@ -142,7 +139,7 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke this._tokenizationSupport = null; if (!this.isTooLargeForHavingAMode()) { - this._tokenizationSupport = TokenizationRegistry.get(this._languageId); + this._tokenizationSupport = TokenizationRegistry.get(this._languageIdentifier.sid); } if (this._tokenizationSupport) { @@ -160,8 +157,8 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke } } + this._colorMap = TokenizationRegistry.getColorMap(); this._lastState = null; - this._tokensInflatorMap = new TokensInflatorMap(this.getModeId()); this._invalidLineStartIndex = 0; this._beginBackgroundTokenization(); } @@ -198,29 +195,38 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke this._updateTokensUntilLine(eventBuilder, lineNumber, true); }); } - return this._lines[lineNumber - 1].getTokens(this._tokensInflatorMap); + + return this._getLineTokens(lineNumber); + } + + private _getLineTokens(lineNumber: number): LineTokens { + return this._lines[lineNumber - 1].getTokens(this._languageIdentifier.iid, this._colorMap); } public getMode(): IMode { - return new Mode(this._languageId); + return new Mode(this._languageIdentifier); } public getModeId(): string { return this.getMode().getId(); } - public setMode(languageId: string): void { - if (this._languageId === languageId) { + public getLanguageIdentifier(): LanguageIdentifier { + return this._languageIdentifier; + } + + public setMode(languageIdentifier: LanguageIdentifier): void { + if (this._languageIdentifier.iid === languageIdentifier.iid) { // There's nothing to do return; } let e: editorCommon.IModelModeChangedEvent = { - oldMode: new Mode(this._languageId), - newMode: new Mode(languageId) + oldMode: new Mode(this._languageIdentifier), + newMode: new Mode(languageIdentifier) }; - this._languageId = languageId; + this._languageIdentifier = languageIdentifier; // Cancel tokenization, clear all tokens and begin tokenizing this._resetTokenizationState(); @@ -234,9 +240,9 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke this._emitModelModeChangedEvent(e); } - public getModeIdAtPosition(_lineNumber: number, _column: number): string { + public getLanguageIdAtPosition(_lineNumber: number, _column: number): LanguageId { if (!this._tokenizationSupport) { - return this.getModeId(); + return this._languageIdentifier.iid; } let { lineNumber, column } = this.validatePosition({ lineNumber: _lineNumber, column: _column }); @@ -244,10 +250,9 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke this._updateTokensUntilLine(eventBuilder, lineNumber, true); }); - let modeTransitions = this._lines[lineNumber - 1].getModeTransitions(this.getModeId()); - let modeTransitionIndex = ModeTransition.findIndexInSegmentsArray(modeTransitions, column - 1); - - return modeTransitions[modeTransitionIndex].modeId; + let lineTokens = this._getLineTokens(lineNumber); + let token = lineTokens.findTokenAtOffset(column - 1); + return token.languageId; } protected _invalidateLine(lineIndex: number): void { @@ -352,42 +357,35 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke return; } - var linesLength = this._lines.length; - var endLineIndex = lineNumber - 1; + const linesLength = this._lines.length; + const endLineIndex = lineNumber - 1; // Validate all states up to and including endLineIndex - for (var lineIndex = this._invalidLineStartIndex; lineIndex <= endLineIndex; lineIndex++) { - var endStateIndex = lineIndex + 1; - var r: ILineTokens = null; - var text = this._lines[lineIndex].text; + for (let lineIndex = this._invalidLineStartIndex; lineIndex <= endLineIndex; lineIndex++) { + const endStateIndex = lineIndex + 1; + let r: ILineTokens3 = null; + const text = this._lines[lineIndex].text; try { // Tokenize only the first X characters let freshState = this._lines[lineIndex].getState().clone(); - r = this._tokenizationSupport.tokenize(this._lines[lineIndex].text, freshState, 0); + r = this._tokenizationSupport.tokenize3(this._lines[lineIndex].text, freshState, 0); } catch (e) { e.friendlyMessage = TextModelWithTokens.MODE_TOKENIZATION_FAILED_MSG; onUnexpectedError(e); } if (!r) { - r = nullTokenize(this.getModeId(), text, this._lines[lineIndex].getState(), 0); + r = nullTokenize3(this._languageIdentifier.iid, text, this._lines[lineIndex].getState(), 0); } - if (!r.modeTransitions) { - r.modeTransitions = []; - } - if (r.modeTransitions.length === 0) { - // Make sure there is at least the transition to the top-most mode - r.modeTransitions.push(new ModeTransition(0, this.getModeId())); - } - this._lines[lineIndex].setTokens(this._tokensInflatorMap, r.tokens, r.modeTransitions); + this._lines[lineIndex].setTokens(this._languageIdentifier.iid, r.tokens); eventBuilder.registerChangedTokens(lineIndex + 1); this._lines[lineIndex].isInvalid = false; if (endStateIndex < linesLength) { if (this._lines[endStateIndex].getState() !== null && r.endState.equals(this._lines[endStateIndex].getState())) { // The end state of this line remains the same - var nextInvalidLineIndex = lineIndex + 1; + let nextInvalidLineIndex = lineIndex + 1; while (nextInvalidLineIndex < linesLength) { if (this._lines[nextInvalidLineIndex].isInvalid) { break; @@ -438,38 +436,32 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke // this line is not tokenized return getWordAtText( position.column, - LanguageConfigurationRegistry.getWordDefinition(this.getModeId()), + LanguageConfigurationRegistry.getWordDefinition(this._languageIdentifier.iid), lineContent, 0 ); } - let modeTransitions = this._lines[position.lineNumber - 1].getModeTransitions(this.getModeId()); + let lineTokens = this._getLineTokens(position.lineNumber); let offset = position.column - 1; - let modeIndex = ModeTransition.findIndexInSegmentsArray(modeTransitions, offset); - - let modeStartOffset = modeTransitions[modeIndex].startIndex; - let modeEndOffset = (modeIndex + 1 < modeTransitions.length ? modeTransitions[modeIndex + 1].startIndex : lineContent.length); - let modeId = modeTransitions[modeIndex].modeId; + let token = lineTokens.findTokenAtOffset(offset); let result = getWordAtText( position.column, - LanguageConfigurationRegistry.getWordDefinition(modeId), - lineContent.substring(modeStartOffset, modeEndOffset), - modeStartOffset + LanguageConfigurationRegistry.getWordDefinition(token.languageId), + lineContent.substring(token.startOffset, token.endOffset), + token.startOffset ); - if (!result && modeIndex > 0 && modeTransitions[modeIndex].startIndex === offset) { + if (!result && token.hasPrev && token.startOffset === offset) { // The position is right at the beginning of `modeIndex`, so try looking at `modeIndex` - 1 too - let prevModeStartOffset = modeTransitions[modeIndex - 1].startIndex; - let prevModeId = modeTransitions[modeIndex - 1].modeId; - + let prevToken = token.prev(); result = getWordAtText( position.column, - LanguageConfigurationRegistry.getWordDefinition(prevModeId), - lineContent.substring(prevModeStartOffset, modeStartOffset), - prevModeStartOffset + LanguageConfigurationRegistry.getWordDefinition(prevToken.languageId), + lineContent.substring(prevToken.startOffset, prevToken.endOffset), + prevToken.startOffset ); } @@ -503,16 +495,15 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke let bracket = _bracket.toLowerCase(); let position = this.validatePosition(_position); - let modeTransitions = this._lines[position.lineNumber - 1].getModeTransitions(this.getModeId()); - let currentModeIndex = ModeTransition.findIndexInSegmentsArray(modeTransitions, position.column - 1); - let currentMode = modeTransitions[currentModeIndex]; - let currentModeBrackets = LanguageConfigurationRegistry.getBracketsSupport(currentMode.modeId); + let lineTokens = this._getLineTokens(position.lineNumber); + let token = lineTokens.findTokenAtOffset(position.column - 1); + let bracketsSupport = LanguageConfigurationRegistry.getBracketsSupport(token.languageId); - if (!currentModeBrackets) { + if (!bracketsSupport) { return null; } - let data = currentModeBrackets.textIsBracket[bracket]; + let data = bracketsSupport.textIsBracket[bracket]; if (!data) { return null; @@ -527,17 +518,17 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke private _matchBracket(position: Position): [Range, Range] { const lineNumber = position.lineNumber; - const lineTokens = this._lines[lineNumber - 1].getTokens(this._tokensInflatorMap); + let lineTokens = this._getLineTokens(lineNumber); const lineText = this._lines[lineNumber - 1].text; const currentToken = lineTokens.findTokenAtOffset(position.column - 1); if (!currentToken) { return null; } - const currentModeBrackets = LanguageConfigurationRegistry.getBracketsSupport(currentToken.modeId); + const currentModeBrackets = LanguageConfigurationRegistry.getBracketsSupport(currentToken.languageId); // check that the token is not to be ignored - if (currentModeBrackets && !ignoreBracketsInToken(currentToken.standardType)) { + if (currentModeBrackets && !ignoreBracketsInToken(currentToken.tokenType)) { // limit search to not go before `maxBracketLength` let searchStartOffset = Math.max(currentToken.startOffset, position.column - 1 - currentModeBrackets.maxBracketLength); // limit search to not go after `maxBracketLength` @@ -585,10 +576,10 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke // If position is in between two tokens, try also looking in the previous token if (currentToken.hasPrev && currentToken.startOffset === position.column - 1) { const prevToken = currentToken.prev(); - const prevModeBrackets = LanguageConfigurationRegistry.getBracketsSupport(prevToken.modeId); + const prevModeBrackets = LanguageConfigurationRegistry.getBracketsSupport(prevToken.languageId); // check that previous token is not to be ignored - if (prevModeBrackets && !ignoreBracketsInToken(prevToken.standardType)) { + if (prevModeBrackets && !ignoreBracketsInToken(prevToken.tokenType)) { // limit search in case previous token is very large, there's no need to go beyond `maxBracketLength` const searchStartOffset = Math.max(prevToken.startOffset, position.column - 1 - prevModeBrackets.maxBracketLength); const searchEndOffset = currentToken.startOffset; @@ -631,12 +622,12 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke private _findMatchingBracketUp(bracket: RichEditBracket, position: Position): Range { // console.log('_findMatchingBracketUp: ', 'bracket: ', JSON.stringify(bracket), 'startPosition: ', String(position)); - const modeId = bracket.modeId; + const languageId = bracket.languageIdentifier.iid; const reversedBracketRegex = bracket.reversedRegex; let count = -1; for (let lineNumber = position.lineNumber; lineNumber >= 1; lineNumber--) { - const lineTokens = this._lines[lineNumber - 1].getTokens(this._tokensInflatorMap); + const lineTokens = this._getLineTokens(lineNumber); const lineText = this._lines[lineNumber - 1].text; let currentToken: LineToken; @@ -652,7 +643,7 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke } while (currentToken) { - if (currentToken.modeId === modeId && !ignoreBracketsInToken(currentToken.standardType)) { + if (currentToken.languageId === languageId && !ignoreBracketsInToken(currentToken.tokenType)) { while (true) { let r = BracketsUtils.findPrevBracketInToken(reversedBracketRegex, lineNumber, lineText, currentToken.startOffset, searchStopOffset); @@ -690,12 +681,12 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke private _findMatchingBracketDown(bracket: RichEditBracket, position: Position): Range { // console.log('_findMatchingBracketDown: ', 'bracket: ', JSON.stringify(bracket), 'startPosition: ', String(position)); - const modeId = bracket.modeId; + const languageId = bracket.languageIdentifier.iid; const bracketRegex = bracket.forwardRegex; let count = 1; for (let lineNumber = position.lineNumber, lineCount = this.getLineCount(); lineNumber <= lineCount; lineNumber++) { - const lineTokens = this._lines[lineNumber - 1].getTokens(this._tokensInflatorMap); + const lineTokens = this._getLineTokens(lineNumber); const lineText = this._lines[lineNumber - 1].text; let currentToken: LineToken; @@ -711,7 +702,7 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke } while (currentToken) { - if (currentToken.modeId === modeId && !ignoreBracketsInToken(currentToken.standardType)) { + if (currentToken.languageId === languageId && !ignoreBracketsInToken(currentToken.tokenType)) { while (true) { let r = BracketsUtils.findNextBracketInToken(bracketRegex, lineNumber, lineText, searchStartOffset, currentToken.endOffset); if (!r) { @@ -748,10 +739,10 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke public findPrevBracket(_position: editorCommon.IPosition): editorCommon.IFoundBracket { const position = this.validatePosition(_position); - let modeId: string = null; - let modeBrackets: RichEditBrackets; + let languageId: LanguageId = -1; + let modeBrackets: RichEditBrackets = null; for (let lineNumber = position.lineNumber; lineNumber >= 1; lineNumber--) { - const lineTokens = this._lines[lineNumber - 1].getTokens(this._tokensInflatorMap); + const lineTokens = this._getLineTokens(lineNumber); const lineText = this._lines[lineNumber - 1].text; let currentToken: LineToken; @@ -767,11 +758,11 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke } while (currentToken) { - if (modeId !== currentToken.modeId) { - modeId = currentToken.modeId; - modeBrackets = LanguageConfigurationRegistry.getBracketsSupport(modeId); + if (languageId !== currentToken.languageId) { + languageId = currentToken.languageId; + modeBrackets = LanguageConfigurationRegistry.getBracketsSupport(languageId); } - if (modeBrackets && !ignoreBracketsInToken(currentToken.standardType)) { + if (modeBrackets && !ignoreBracketsInToken(currentToken.tokenType)) { let r = BracketsUtils.findPrevBracketInToken(modeBrackets.reversedRegex, lineNumber, lineText, currentToken.startOffset, searchStopOffset); if (r) { return this._toFoundBracket(modeBrackets, r); @@ -791,10 +782,10 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke public findNextBracket(_position: editorCommon.IPosition): editorCommon.IFoundBracket { const position = this.validatePosition(_position); - let modeId: string = null; - let modeBrackets: RichEditBrackets; + let languageId: LanguageId = -1; + let modeBrackets: RichEditBrackets = null; for (let lineNumber = position.lineNumber, lineCount = this.getLineCount(); lineNumber <= lineCount; lineNumber++) { - const lineTokens = this._lines[lineNumber - 1].getTokens(this._tokensInflatorMap); + const lineTokens = this._getLineTokens(lineNumber); const lineText = this._lines[lineNumber - 1].text; let currentToken: LineToken; @@ -810,11 +801,11 @@ export class TextModelWithTokens extends TextModel implements editorCommon.IToke } while (currentToken) { - if (modeId !== currentToken.modeId) { - modeId = currentToken.modeId; - modeBrackets = LanguageConfigurationRegistry.getBracketsSupport(modeId); + if (languageId !== currentToken.languageId) { + languageId = currentToken.languageId; + modeBrackets = LanguageConfigurationRegistry.getBracketsSupport(languageId); } - if (modeBrackets && !ignoreBracketsInToken(currentToken.standardType)) { + if (modeBrackets && !ignoreBracketsInToken(currentToken.tokenType)) { let r = BracketsUtils.findNextBracketInToken(modeBrackets.forwardRegex, lineNumber, lineText, searchStartOffset, currentToken.endOffset); if (r) { return this._toFoundBracket(modeBrackets, r); diff --git a/src/vs/editor/common/model/tokenIterator.ts b/src/vs/editor/common/model/tokenIterator.ts index 11026ec137c..338b72e8e3d 100644 --- a/src/vs/editor/common/model/tokenIterator.ts +++ b/src/vs/editor/common/model/tokenIterator.ts @@ -5,8 +5,9 @@ 'use strict'; import * as editorCommon from 'vs/editor/common/editorCommon'; -import { LineToken, StandardTokenType } from 'vs/editor/common/core/lineTokens'; +import { LineToken } from 'vs/editor/common/core/lineTokens'; import { Position } from 'vs/editor/common/core/position'; +import { StandardTokenType } from 'vs/editor/common/modes'; class TokenInfo implements editorCommon.ITokenInfo { _tokenInfoBrand: void; @@ -15,14 +16,14 @@ class TokenInfo implements editorCommon.ITokenInfo { public readonly lineNumber: number; public readonly startColumn: number; public readonly endColumn: number; - public readonly standardType: StandardTokenType; + public readonly type: StandardTokenType; constructor(actual: LineToken, lineNumber: number) { this._actual = actual; this.lineNumber = lineNumber; this.startColumn = this._actual.startOffset + 1; this.endColumn = this._actual.endOffset + 1; - this.standardType = this._actual.standardType; + this.type = this._actual.tokenType; } } diff --git a/src/vs/editor/common/model/tokensBinaryEncoding.ts b/src/vs/editor/common/model/tokensBinaryEncoding.ts index 65c5824487b..18884f1de39 100644 --- a/src/vs/editor/common/model/tokensBinaryEncoding.ts +++ b/src/vs/editor/common/model/tokensBinaryEncoding.ts @@ -4,187 +4,118 @@ *--------------------------------------------------------------------------------------------*/ 'use strict'; -import { onUnexpectedError } from 'vs/base/common/errors'; -import * as strings from 'vs/base/common/strings'; import { ViewLineToken } from 'vs/editor/common/core/viewLineToken'; -import { Token } from 'vs/editor/common/core/token'; +import { ColorId, FontStyle, StandardTokenType, MetadataConsts, LanguageId } from 'vs/editor/common/modes'; -export const enum TokensBinaryEncodingValues { - START_INDEX_MASK = 0xffffffff, - TYPE_MASK = 0xffff, - START_INDEX_OFFSET = 1, - TYPE_OFFSET = 4294967296 // Math.pow(2, 32) -} +export class TokenMetadata { -const DEFAULT_VIEW_TOKEN = new ViewLineToken(0, ''); -const INFLATED_TOKENS_EMPTY_TEXT: ViewLineToken[] = []; -export const DEFLATED_TOKENS_EMPTY_TEXT: number[] = []; -const INFLATED_TOKENS_NON_EMPTY_TEXT: ViewLineToken[] = [DEFAULT_VIEW_TOKEN]; -export const DEFLATED_TOKENS_NON_EMPTY_TEXT: number[] = [0]; - -export class TokensInflatorMap { - _tokensInflatorMapBrand: void; - - public topLevelModeId: string; - public _inflate: string[]; - - public _deflate: { - [token: string]: number; - }; - - constructor(topLevelModeId: string) { - this.topLevelModeId = topLevelModeId; - this._inflate = ['']; - this._deflate = { '': 0 }; + public static toBinaryStr(metadata: number): string { + let r = metadata.toString(2); + while (r.length < 32) { + r = '0' + r; + } + return r; } -} -export class TokensBinaryEncoding { + public static printMetadata(metadata: number): void { + let languageId = TokenMetadata.getLanguageId(metadata); + let tokenType = TokenMetadata.getTokenType(metadata); + let fontStyle = TokenMetadata.getFontStyle(metadata); + let foreground = TokenMetadata.getForeground(metadata); + let background = TokenMetadata.getBackground(metadata); - public static deflateArr(map: TokensInflatorMap, tokens: Token[]): number[] { - if (tokens.length === 0) { - return DEFLATED_TOKENS_EMPTY_TEXT; + console.log({ + languageId: languageId, + tokenType: tokenType, + fontStyle: fontStyle, + foreground: foreground, + background: background, + }); + } + + public static getLanguageId(metadata: number): LanguageId { + return (metadata & MetadataConsts.LANGUAGEID_MASK) >>> MetadataConsts.LANGUAGEID_OFFSET; + } + + public static getTokenType(metadata: number): StandardTokenType { + return (metadata & MetadataConsts.TOKEN_TYPE_MASK) >>> MetadataConsts.TOKEN_TYPE_OFFSET; + } + + public static getFontStyle(metadata: number): FontStyle { + return (metadata & MetadataConsts.FONT_STYLE_MASK) >>> MetadataConsts.FONT_STYLE_OFFSET; + } + + public static getForeground(metadata: number): ColorId { + return (metadata & MetadataConsts.FOREGROUND_MASK) >>> MetadataConsts.FOREGROUND_OFFSET; + } + + public static getBackground(metadata: number): ColorId { + return (metadata & MetadataConsts.BACKGROUND_MASK) >>> MetadataConsts.BACKGROUND_OFFSET; + } + + private static _getClassNameFromMetadata(metadata: number): string { + let foreground = this.getForeground(metadata); + let className = 'mtk' + foreground; + + let fontStyle = this.getFontStyle(metadata); + if (fontStyle & FontStyle.Italic) { + className += ' mtki'; } - if (tokens.length === 1 && tokens[0].startIndex === 0 && !tokens[0].type) { - return DEFLATED_TOKENS_NON_EMPTY_TEXT; + if (fontStyle & FontStyle.Bold) { + className += ' mtkb'; + } + if (fontStyle & FontStyle.Underline) { + className += ' mtku'; } - var i: number, - len: number, - deflatedToken: number, - deflated: number, - token: Token, - inflateMap = map._inflate, - deflateMap = map._deflate, - prevStartIndex: number = -1, - result: number[] = new Array(tokens.length); + return className; + } - for (i = 0, len = tokens.length; i < len; i++) { - token = tokens[i]; + public static inflateArr(tokens: Uint32Array): ViewLineToken[] { + let tokenCount = (tokens.length >>> 1); + let result: ViewLineToken[] = []; - if (token.startIndex <= prevStartIndex) { - token = new Token(prevStartIndex + 1, token.type); - onUnexpectedError({ - message: 'Invalid tokens detected', - tokens: tokens - }); - } + for (let i = 0; i < tokenCount; i++) { + let startOffset = tokens[(i << 1)]; + let metadata = tokens[(i << 1) + 1]; - if (deflateMap.hasOwnProperty(token.type)) { - deflatedToken = deflateMap[token.type]; - } else { - deflatedToken = inflateMap.length; - deflateMap[token.type] = deflatedToken; - inflateMap.push(token.type); - } - - // http://stackoverflow.com/a/2803010 - // All numbers in JavaScript are actually IEEE-754 compliant floating-point doubles. - // These have a 53-bit mantissa which should mean that any integer value with a magnitude - // of approximately 9 quadrillion or less -- more specifically, 9,007,199,254,740,991 -- - // will be represented accurately. - - // http://stackoverflow.com/a/6729252 - // Bitwise operations cast numbers to 32bit representation in JS - - // 32 bits for startIndex => up to 2^32 = 4,294,967,296 - // 16 bits for token => up to 2^16 = 65,536 - - // [token][startIndex] - deflated = deflatedToken * TokensBinaryEncodingValues.TYPE_OFFSET + token.startIndex * TokensBinaryEncodingValues.START_INDEX_OFFSET; - - result[i] = deflated; - - prevStartIndex = token.startIndex; + result[i] = new ViewLineToken(startOffset, this._getClassNameFromMetadata(metadata)); } - return result; } - public static getStartIndex(binaryEncodedToken: number): number { - return (binaryEncodedToken / TokensBinaryEncodingValues.START_INDEX_OFFSET) & TokensBinaryEncodingValues.START_INDEX_MASK; - } + public static sliceAndInflate(tokens: Uint32Array, startOffset: number, endOffset: number, deltaStartIndex: number): ViewLineToken[] { + let tokenIndex = this.findIndexInSegmentsArray(tokens, startOffset); + let result: ViewLineToken[] = [], resultLen = 0; - public static getType(map: TokensInflatorMap, binaryEncodedToken: number): string { - var deflatedType = (binaryEncodedToken / TokensBinaryEncodingValues.TYPE_OFFSET) & TokensBinaryEncodingValues.TYPE_MASK; - if (deflatedType === 0) { - return strings.empty; - } - return map._inflate[deflatedType]; - } + result[resultLen++] = new ViewLineToken(0, this._getClassNameFromMetadata(tokens[(tokenIndex << 1) + 1])); - public static inflateArr(map: TokensInflatorMap, binaryEncodedTokens: number[]): ViewLineToken[] { - if (binaryEncodedTokens.length === 0) { - return INFLATED_TOKENS_EMPTY_TEXT; - } - if (binaryEncodedTokens.length === 1 && binaryEncodedTokens[0] === 0) { - return INFLATED_TOKENS_NON_EMPTY_TEXT; - } + for (let i = tokenIndex + 1, len = (tokens.length >>> 1); i < len; i++) { + let originalStartOffset = tokens[(i << 1)]; - let result: ViewLineToken[] = []; - const inflateMap = map._inflate; - - for (let i = 0, len = binaryEncodedTokens.length; i < len; i++) { - let deflated = binaryEncodedTokens[i]; - - let startIndex = (deflated / TokensBinaryEncodingValues.START_INDEX_OFFSET) & TokensBinaryEncodingValues.START_INDEX_MASK; - let deflatedType = (deflated / TokensBinaryEncodingValues.TYPE_OFFSET) & TokensBinaryEncodingValues.TYPE_MASK; - - result.push(new ViewLineToken(startIndex, inflateMap[deflatedType])); - } - - return result; - } - - public static findIndexOfOffset(binaryEncodedTokens: number[], offset: number): number { - return this.findIndexInSegmentsArray(binaryEncodedTokens, offset); - } - - public static sliceAndInflate(map: TokensInflatorMap, binaryEncodedTokens: number[], startOffset: number, endOffset: number, deltaStartIndex: number): ViewLineToken[] { - if (binaryEncodedTokens.length === 0) { - return INFLATED_TOKENS_EMPTY_TEXT; - } - if (binaryEncodedTokens.length === 1 && binaryEncodedTokens[0] === 0) { - return INFLATED_TOKENS_NON_EMPTY_TEXT; - } - - let startIndex = this.findIndexInSegmentsArray(binaryEncodedTokens, startOffset); - let result: ViewLineToken[] = []; - const inflateMap = map._inflate; - - let originalToken = binaryEncodedTokens[startIndex]; - let deflatedType = (originalToken / TokensBinaryEncodingValues.TYPE_OFFSET) & TokensBinaryEncodingValues.TYPE_MASK; - let newStartIndex = 0; - result.push(new ViewLineToken(newStartIndex, inflateMap[deflatedType])); - - for (let i = startIndex + 1, len = binaryEncodedTokens.length; i < len; i++) { - originalToken = binaryEncodedTokens[i]; - let originalStartIndex = (originalToken / TokensBinaryEncodingValues.START_INDEX_OFFSET) & TokensBinaryEncodingValues.START_INDEX_MASK; - - if (originalStartIndex >= endOffset) { + if (originalStartOffset >= endOffset) { break; } - deflatedType = (originalToken / TokensBinaryEncodingValues.TYPE_OFFSET) & TokensBinaryEncodingValues.TYPE_MASK; - newStartIndex = originalStartIndex - startOffset + deltaStartIndex; - result.push(new ViewLineToken(newStartIndex, inflateMap[deflatedType])); + let newStartOffset = originalStartOffset - startOffset + deltaStartIndex; + let metadata = tokens[(i << 1) + 1]; + + result[resultLen++] = new ViewLineToken(newStartOffset, this._getClassNameFromMetadata(metadata)); } return result; } - private static findIndexInSegmentsArray(arr: number[], desiredIndex: number): number { + public static findIndexInSegmentsArray(tokens: Uint32Array, desiredIndex: number): number { - var low = 0, - high = arr.length - 1, - mid: number, - value: number; + let low = 0; + let high = (tokens.length >>> 1) - 1; while (low < high) { - mid = low + Math.ceil((high - low) / 2); + let mid = low + Math.ceil((high - low) / 2); - value = arr[mid] & 0xffffffff; + let value = tokens[(mid << 1)]; if (value > desiredIndex) { high = mid - 1; @@ -195,4 +126,4 @@ export class TokensBinaryEncoding { return low; } -} \ No newline at end of file +} diff --git a/src/vs/editor/common/modes.ts b/src/vs/editor/common/modes.ts index 9a714b4dc4f..cd8f4246f92 100644 --- a/src/vs/editor/common/modes.ts +++ b/src/vs/editor/common/modes.ts @@ -20,8 +20,22 @@ import Event, { Emitter } from 'vs/base/common/event'; /** * @internal */ -export interface IModeDescriptor { - id: string; +export const enum LanguageId { + Null = 0, + PlainText = 1 +} + +/** + * @internal + */ +export class LanguageIdentifier { + public readonly sid: string; + public readonly iid: LanguageId; + + constructor(sid: string, iid: LanguageId) { + this.sid = sid; + this.iid = iid; + } } /** @@ -31,6 +45,8 @@ export interface IMode { getId(): string; + getLanguageIdentifier(): LanguageIdentifier; + } /** @@ -42,6 +58,38 @@ export interface ILineTokens { modeTransitions: ModeTransition[]; } +/** + * A font style. Values are 2^x such that a bit mask can be used. + * @internal + */ +export const enum FontStyle { + NotSet = -1, + None = 0, + Italic = 1, + Bold = 2, + Underline = 4 +} + +/** + * @internal + */ +export const enum ColorId { + None = 0, + DefaultForeground = 1, + DefaultBackground = 2 +} + +/** + * A standard token type. Values are 2^x such that a bit mask can be used. + * @internal + */ +export const enum StandardTokenType { + Other = 0, + Comment = 1, + String = 2, + RegEx = 4 +} + /** * Helpers to manage the "collapsed" metadata of an entire StackElement stack. * The following assumptions have been made: @@ -49,18 +97,18 @@ export interface ILineTokens { * - unique color count < 512 => needs 9 bits * * The binary format is: - * -------------------------------------------- - * 3322 2222 2222 1111 1111 1100 0000 0000 - * 1098 7654 3210 9876 5432 1098 7654 3210 - * -------------------------------------------- - * xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx - * bbbb bbbb bfff ffff ffFF FTTT LLLL LLLL - * -------------------------------------------- - * L = LanguageId (8 bits) - * T = StandardTokenType (3 bits) - * F = FontStyle (3 bits) - * f = foreground color (9 bits) - * b = background color (9 bits) + * - ------------------------------------------- + * 3322 2222 2222 1111 1111 1100 0000 0000 + * 1098 7654 3210 9876 5432 1098 7654 3210 + * - ------------------------------------------- + * xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx + * bbbb bbbb bfff ffff ffFF FTTT LLLL LLLL + * - ------------------------------------------- + * - L = LanguageId (8 bits) + * - T = StandardTokenType (3 bits) + * - F = FontStyle (3 bits) + * - f = foreground color (9 bits) + * - b = background color (9 bits) * * @internal */ @@ -102,15 +150,15 @@ export interface ITokenizationSupport { // add offsetDelta to each of the returned indices tokenize(line: string, state: IState, offsetDelta: number): ILineTokens; - // tokenize3(line: string, state: IState, offsetDelta: number): ILineTokens3; + tokenize3(line: string, state: IState, offsetDelta: number): ILineTokens3; } /** - * A token. Only supports a single scope, but will soon support a scope array. + * A token. */ export interface IToken2 { startIndex: number; - scopes: string | string[]; + scopes: string; } /** * The result of a line tokenization. @@ -790,8 +838,11 @@ export class TokenizationRegistryImpl { private _onDidChange: Emitter = new Emitter(); public onDidChange: Event = this._onDidChange.event; + private _colorMap: string[]; + constructor() { this._map = Object.create(null); + this._colorMap = null; } /** @@ -819,6 +870,10 @@ export class TokenizationRegistryImpl { public get(languageId: string): ITokenizationSupport { return (this._map[languageId] || null); } + + public getColorMap(): string[] { + return this._colorMap; + } } /** diff --git a/src/vs/editor/common/modes/abstractMode.ts b/src/vs/editor/common/modes/abstractMode.ts index 54605c7caad..b25a8d20160 100644 --- a/src/vs/editor/common/modes/abstractMode.ts +++ b/src/vs/editor/common/modes/abstractMode.ts @@ -4,17 +4,21 @@ *--------------------------------------------------------------------------------------------*/ 'use strict'; -import * as modes from 'vs/editor/common/modes'; +import { IMode, LanguageIdentifier } from 'vs/editor/common/modes'; -export class FrankensteinMode implements modes.IMode { +export class FrankensteinMode implements IMode { - private _modeId: string; + private _languageIdentifier: LanguageIdentifier; - constructor(descriptor: modes.IModeDescriptor) { - this._modeId = descriptor.id; + constructor(languageIdentifier: LanguageIdentifier) { + this._languageIdentifier = languageIdentifier; } public getId(): string { - return this._modeId; + return this._languageIdentifier.sid; + } + + public getLanguageIdentifier(): LanguageIdentifier { + return this._languageIdentifier; } } diff --git a/src/vs/editor/common/modes/languageConfiguration.ts b/src/vs/editor/common/modes/languageConfiguration.ts index a0afe5f5a0f..178809e9e84 100644 --- a/src/vs/editor/common/modes/languageConfiguration.ts +++ b/src/vs/editor/common/modes/languageConfiguration.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ 'use strict'; -import { StandardTokenType } from 'vs/editor/common/core/lineTokens'; +import { StandardTokenType } from 'vs/editor/common/modes'; /** * Describes how comments for a language work. diff --git a/src/vs/editor/common/modes/languageConfigurationRegistry.ts b/src/vs/editor/common/modes/languageConfigurationRegistry.ts index 4d6d1ac0f04..0a3ef8005d8 100644 --- a/src/vs/editor/common/modes/languageConfigurationRegistry.ts +++ b/src/vs/editor/common/modes/languageConfigurationRegistry.ts @@ -17,6 +17,7 @@ import { DEFAULT_WORD_REGEXP, ensureValidWordDefinition } from 'vs/editor/common import { createScopedLineTokens } from 'vs/editor/common/modes/supports'; import { LineTokens } from 'vs/editor/common/core/lineTokens'; import { IndentAction, EnterAction, IAutoClosingPair, LanguageConfiguration } from 'vs/editor/common/modes/languageConfiguration'; +import { LanguageIdentifier, LanguageId } from 'vs/editor/common/modes'; /** * Interface used to support insertion of mode specific comments. @@ -29,16 +30,16 @@ export interface ICommentsConfiguration { export class RichEditSupport { - private _conf: LanguageConfiguration; + private readonly _conf: LanguageConfiguration; - public electricCharacter: BracketElectricCharacterSupport; - public comments: ICommentsConfiguration; - public characterPair: CharacterPairSupport; - public wordDefinition: RegExp; - public onEnter: OnEnterSupport; - public brackets: RichEditBrackets; + public readonly electricCharacter: BracketElectricCharacterSupport; + public readonly comments: ICommentsConfiguration; + public readonly characterPair: CharacterPairSupport; + public readonly wordDefinition: RegExp; + public readonly onEnter: OnEnterSupport; + public readonly brackets: RichEditBrackets; - constructor(modeId: string, previous: RichEditSupport, rawConf: LanguageConfiguration) { + constructor(languageIdentifier: LanguageIdentifier, previous: RichEditSupport, rawConf: LanguageConfiguration) { let prev: LanguageConfiguration = null; if (previous) { @@ -48,12 +49,12 @@ export class RichEditSupport { this._conf = RichEditSupport._mergeConf(prev, rawConf); if (this._conf.brackets) { - this.brackets = new RichEditBrackets(modeId, this._conf.brackets); + this.brackets = new RichEditBrackets(languageIdentifier, this._conf.brackets); } - this._handleOnEnter(modeId, this._conf); + this.onEnter = RichEditSupport._handleOnEnter(this._conf); - this._handleComments(modeId, this._conf); + this.comments = RichEditSupport._handleComments(this._conf); this.characterPair = new CharacterPairSupport(this._conf); this.electricCharacter = new BracketElectricCharacterSupport(this.brackets, this.characterPair.getAutoClosingPairs(), this._conf.__electricCharacterSupport); @@ -74,7 +75,7 @@ export class RichEditSupport { }; } - private _handleOnEnter(modeId: string, conf: LanguageConfiguration): void { + private static _handleOnEnter(conf: LanguageConfiguration): OnEnterSupport { // on enter let onEnter: IOnEnterSupportOptions = {}; let empty = true; @@ -93,66 +94,75 @@ export class RichEditSupport { } if (!empty) { - this.onEnter = new OnEnterSupport(onEnter); + return new OnEnterSupport(onEnter); } + return null; } - private _handleComments(modeId: string, conf: LanguageConfiguration): void { + private static _handleComments(conf: LanguageConfiguration): ICommentsConfiguration { let commentRule = conf.comments; + if (!commentRule) { + return null; + } // comment configuration - if (commentRule) { - this.comments = {}; + let comments: ICommentsConfiguration = {}; - if (commentRule.lineComment) { - this.comments.lineCommentToken = commentRule.lineComment; - } - if (commentRule.blockComment) { - let [blockStart, blockEnd] = commentRule.blockComment; - this.comments.blockCommentStartToken = blockStart; - this.comments.blockCommentEndToken = blockEnd; - } + if (commentRule.lineComment) { + comments.lineCommentToken = commentRule.lineComment; + } + if (commentRule.blockComment) { + let [blockStart, blockEnd] = commentRule.blockComment; + comments.blockCommentStartToken = blockStart; + comments.blockCommentEndToken = blockEnd; } - } + return comments; + } } export class LanguageConfigurationRegistryImpl { - private _entries: { [languageId: string]: RichEditSupport; }; + private _entries: RichEditSupport[]; private _onDidChange: Emitter = new Emitter(); public onDidChange: Event = this._onDidChange.event; constructor() { - this._entries = Object.create(null); + this._entries = []; } - public register(languageId: string, configuration: LanguageConfiguration): IDisposable { - let previous = this._entries[languageId] || null; - this._entries[languageId] = new RichEditSupport(languageId, previous, configuration); + public register(languageIdentifier: LanguageIdentifier, configuration: LanguageConfiguration): IDisposable { + let previous = this._getRichEditSupport(languageIdentifier.iid); + let current = new RichEditSupport(languageIdentifier, previous, configuration); + this._entries[languageIdentifier.iid] = current; this._onDidChange.fire(void 0); return { - dispose: () => { } + dispose: () => { + if (this._entries[languageIdentifier.iid] === current) { + this._entries[languageIdentifier.iid] = previous; + this._onDidChange.fire(void 0); + } + } }; } - private _getRichEditSupport(modeId: string): RichEditSupport { - return this._entries[modeId]; + private _getRichEditSupport(languageId: LanguageId): RichEditSupport { + return this._entries[languageId] || null; } // begin electricCharacter - private _getElectricCharacterSupport(modeId: string): BracketElectricCharacterSupport { - let value = this._getRichEditSupport(modeId); + private _getElectricCharacterSupport(languageId: LanguageId): BracketElectricCharacterSupport { + let value = this._getRichEditSupport(languageId); if (!value) { return null; } return value.electricCharacter || null; } - public getElectricCharacters(modeId: string): string[] { - let electricCharacterSupport = this._getElectricCharacterSupport(modeId); + public getElectricCharacters(languageId: LanguageId): string[] { + let electricCharacterSupport = this._getElectricCharacterSupport(languageId); if (!electricCharacterSupport) { return []; } @@ -164,7 +174,7 @@ export class LanguageConfigurationRegistryImpl { */ public onElectricCharacter(character: string, context: LineTokens, column: number): IElectricAction { let scopedLineTokens = createScopedLineTokens(context, column - 1); - let electricCharacterSupport = this._getElectricCharacterSupport(scopedLineTokens.modeId); + let electricCharacterSupport = this._getElectricCharacterSupport(scopedLineTokens.languageId); if (!electricCharacterSupport) { return null; } @@ -173,8 +183,8 @@ export class LanguageConfigurationRegistryImpl { // end electricCharacter - public getComments(modeId: string): ICommentsConfiguration { - let value = this._getRichEditSupport(modeId); + public getComments(languageId: LanguageId): ICommentsConfiguration { + let value = this._getRichEditSupport(languageId); if (!value) { return null; } @@ -183,24 +193,24 @@ export class LanguageConfigurationRegistryImpl { // begin characterPair - private _getCharacterPairSupport(modeId: string): CharacterPairSupport { - let value = this._getRichEditSupport(modeId); + private _getCharacterPairSupport(languageId: LanguageId): CharacterPairSupport { + let value = this._getRichEditSupport(languageId); if (!value) { return null; } return value.characterPair || null; } - public getAutoClosingPairs(modeId: string): IAutoClosingPair[] { - let characterPairSupport = this._getCharacterPairSupport(modeId); + public getAutoClosingPairs(languageId: LanguageId): IAutoClosingPair[] { + let characterPairSupport = this._getCharacterPairSupport(languageId); if (!characterPairSupport) { return []; } return characterPairSupport.getAutoClosingPairs(); } - public getSurroundingPairs(modeId: string): IAutoClosingPair[] { - let characterPairSupport = this._getCharacterPairSupport(modeId); + public getSurroundingPairs(languageId: LanguageId): IAutoClosingPair[] { + let characterPairSupport = this._getCharacterPairSupport(languageId); if (!characterPairSupport) { return []; } @@ -209,7 +219,7 @@ export class LanguageConfigurationRegistryImpl { public shouldAutoClosePair(character: string, context: LineTokens, column: number): boolean { let scopedLineTokens = createScopedLineTokens(context, column - 1); - let characterPairSupport = this._getCharacterPairSupport(scopedLineTokens.modeId); + let characterPairSupport = this._getCharacterPairSupport(scopedLineTokens.languageId); if (!characterPairSupport) { return false; } @@ -218,8 +228,8 @@ export class LanguageConfigurationRegistryImpl { // end characterPair - public getWordDefinition(modeId: string): RegExp { - let value = this._getRichEditSupport(modeId); + public getWordDefinition(languageId: LanguageId): RegExp { + let value = this._getRichEditSupport(languageId); if (!value) { return ensureValidWordDefinition(null); } @@ -228,8 +238,8 @@ export class LanguageConfigurationRegistryImpl { // begin onEnter - private _getOnEnterSupport(modeId: string): OnEnterSupport { - let value = this._getRichEditSupport(modeId); + private _getOnEnterSupport(languageId: LanguageId): OnEnterSupport { + let value = this._getRichEditSupport(languageId); if (!value) { return null; } @@ -239,7 +249,7 @@ export class LanguageConfigurationRegistryImpl { public getRawEnterActionAtPosition(model: ITokenizedModel, lineNumber: number, column: number): EnterAction { let lineTokens = model.getLineTokens(lineNumber, false); let scopedLineTokens = createScopedLineTokens(lineTokens, column - 1); - let onEnterSupport = this._getOnEnterSupport(scopedLineTokens.modeId); + let onEnterSupport = this._getOnEnterSupport(scopedLineTokens.languageId); if (!onEnterSupport) { return null; } @@ -254,7 +264,7 @@ export class LanguageConfigurationRegistryImpl { let oneLineAboveLineTokens = model.getLineTokens(lineNumber - 1, false); let oneLineAboveMaxColumn = model.getLineMaxColumn(lineNumber - 1); let oneLineAboveScopedLineTokens = createScopedLineTokens(oneLineAboveLineTokens, oneLineAboveMaxColumn - 1); - if (oneLineAboveScopedLineTokens.modeId === scopedLineTokens.modeId) { + if (oneLineAboveScopedLineTokens.languageId === scopedLineTokens.languageId) { // The line above ends with text belonging to the same mode oneLineAboveText = oneLineAboveScopedLineTokens.getLineContent(); } @@ -307,8 +317,8 @@ export class LanguageConfigurationRegistryImpl { // end onEnter - public getBracketsSupport(modeId: string): RichEditBrackets { - let value = this._getRichEditSupport(modeId); + public getBracketsSupport(languageId: LanguageId): RichEditBrackets { + let value = this._getRichEditSupport(languageId); if (!value) { return null; } diff --git a/src/vs/editor/common/modes/modesRegistry.ts b/src/vs/editor/common/modes/modesRegistry.ts index f6fe33c690b..259fdda2dcf 100644 --- a/src/vs/editor/common/modes/modesRegistry.ts +++ b/src/vs/editor/common/modes/modesRegistry.ts @@ -9,6 +9,8 @@ import Event, { Emitter } from 'vs/base/common/event'; import { Registry } from 'vs/platform/platform'; import { ILanguageExtensionPoint } from 'vs/editor/common/services/modeService'; import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry'; +import { LanguageIdentifier, LanguageId } from 'vs/editor/common/modes'; + // Define extension point ids export var Extensions = { ModesRegistry: 'editor.modesRegistry' @@ -44,6 +46,7 @@ export var ModesRegistry = new EditorModesRegistry(); Registry.add(Extensions.ModesRegistry, ModesRegistry); export const PLAINTEXT_MODE_ID = 'plaintext'; +export const PLAINTEXT_LANGUAGE_IDENTIFIER = new LanguageIdentifier(PLAINTEXT_MODE_ID, LanguageId.PlainText); ModesRegistry.registerLanguage({ id: PLAINTEXT_MODE_ID, @@ -51,7 +54,7 @@ ModesRegistry.registerLanguage({ aliases: [nls.localize('plainText.alias', "Plain Text"), 'text'], mimetypes: ['text/plain'] }); -LanguageConfigurationRegistry.register(PLAINTEXT_MODE_ID, { +LanguageConfigurationRegistry.register(PLAINTEXT_LANGUAGE_IDENTIFIER, { brackets: [ ['(', ')'], ['[', ']'], diff --git a/src/vs/editor/common/modes/monarch/monarchLexer.ts b/src/vs/editor/common/modes/monarch/monarchLexer.ts index 361ac989169..897b0526147 100644 --- a/src/vs/editor/common/modes/monarch/monarchLexer.ts +++ b/src/vs/editor/common/modes/monarch/monarchLexer.ts @@ -16,6 +16,8 @@ import { IModeService } from 'vs/editor/common/services/modeService'; import { Token } from 'vs/editor/common/core/token'; import { NULL_STATE, NULL_MODE_ID } from 'vs/editor/common/modes/nullMode'; import { ModeTransition } from 'vs/editor/common/core/modeTransition'; +import { IStandaloneColorService } from 'vs/editor/common/services/standaloneColorService'; +import { Theme } from 'vs/editor/common/modes/supports/tokenization'; const CACHE_STACK_DEPTH = 5; @@ -297,16 +299,105 @@ class MonarchClassicTokensCollector implements IMonarchTokensCollector { } } +class MonarchModernTokensCollector implements IMonarchTokensCollector { + + private _modeService: IModeService; + private _theme: Theme; + private _prependTokens: Uint32Array; + private _tokens: number[]; + private _currentLanguageId: modes.LanguageId; + private _lastTokenMetadata: number; + + constructor(modeService: IModeService, theme: Theme) { + this._modeService = modeService; + this._theme = theme; + this._prependTokens = null; + this._tokens = []; + this._currentLanguageId = modes.LanguageId.Null; + this._lastTokenMetadata = 0; + } + + public enterMode(startOffset: number, modeId: string): void { + this._currentLanguageId = this._modeService.getLanguageIdentifier(modeId).iid; + } + + public emit(startOffset: number, type: string): void { + let metadata = this._theme.match(this._currentLanguageId, type); + if (this._lastTokenMetadata === metadata) { + return; + } + this._lastTokenMetadata = metadata; + this._tokens.push(startOffset); + this._tokens.push(metadata); + } + + private static _merge(a: Uint32Array, b: number[], c: Uint32Array): Uint32Array { + let aLen = (a !== null ? a.length : 0); + let bLen = b.length; + let cLen = (c !== null ? c.length : 0); + + // Fast path + if (bLen === 0) { + if (aLen === 0) { + return c; + } + if (cLen === 0) { + return a; + } + } + + let result = new Uint32Array(aLen + bLen + cLen); + if (a !== null) { + result.set(a); + } + for (let i = 0; i < bLen; i++) { + result[aLen + i] = b[i]; + } + if (c !== null) { + result.set(c, aLen + bLen); + } + return result; + } + + public nestedModeTokenize(embeddedModeLine: string, embeddedModeData: EmbeddedModeData, offsetDelta: number): modes.IState { + const nestedModeId = embeddedModeData.modeId; + const embeddedModeState = embeddedModeData.state; + + const nestedModeTokenizationSupport = modes.TokenizationRegistry.get(nestedModeId); + if (!nestedModeTokenizationSupport) { + this.enterMode(offsetDelta, nestedModeId); + this.emit(offsetDelta, ''); + return embeddedModeState; + } + + let nestedResult = nestedModeTokenizationSupport.tokenize3(embeddedModeLine, embeddedModeState, offsetDelta); + this._prependTokens = MonarchModernTokensCollector._merge(this._prependTokens, this._tokens, nestedResult.tokens); + this._tokens = []; + this._currentLanguageId = 0; + this._lastTokenMetadata = 0; + return nestedResult.endState; + } + + public finalize(endState: MonarchLineState): modes.ILineTokens3 { + return { + tokens: MonarchModernTokensCollector._merge(this._prependTokens, this._tokens, null), + endState: endState + }; + } +} + export class MonarchTokenizer implements modes.ITokenizationSupport { private readonly _modeService: IModeService; + private readonly _standaloneColorService: IStandaloneColorService; private readonly _modeId: string; private readonly _lexer: monarchCommon.ILexer; private _embeddedModes: { [modeId: string]: boolean; }; private _tokenizationRegistryListener: IDisposable; - constructor(modeService: IModeService, modeId: string, lexer: monarchCommon.ILexer) { + constructor(modeService: IModeService, standaloneColorService: IStandaloneColorService, modeId: string, lexer: monarchCommon.ILexer) { this._modeService = modeService; + this._standaloneColorService = standaloneColorService; this._modeId = modeId; this._lexer = lexer; this._embeddedModes = Object.create(null); @@ -335,12 +426,15 @@ export class MonarchTokenizer implements modes.ITokenizationSupport { return MonarchLineStateFactory.create(rootState, null); } - public tokenize(line: string, _lineState: modes.IState, offsetDelta: number): modes.ILineTokens { - let lineState = (_lineState); - + public tokenize(line: string, lineState: modes.IState, offsetDelta: number): modes.ILineTokens { let tokensCollector = new MonarchClassicTokensCollector(); - let endLineState = this._tokenize(line, lineState, offsetDelta, tokensCollector); + let endLineState = this._tokenize(line, lineState, offsetDelta, tokensCollector); + return tokensCollector.finalize(endLineState); + } + public tokenize3(line: string, lineState: modes.IState, offsetDelta: number): modes.ILineTokens3 { + let tokensCollector = new MonarchModernTokensCollector(this._modeService, this._standaloneColorService.getTheme()); + let endLineState = this._tokenize(line, lineState, offsetDelta, tokensCollector); return tokensCollector.finalize(endLineState); } @@ -746,6 +840,6 @@ function findBracket(lexer: monarchCommon.ILexer, matched: string) { return null; } -export function createTokenizationSupport(_modeService: IModeService, modeId: string, lexer: monarchCommon.ILexer): modes.ITokenizationSupport { - return new MonarchTokenizer(_modeService, modeId, lexer); +export function createTokenizationSupport(modeService: IModeService, standaloneColorService: IStandaloneColorService, modeId: string, lexer: monarchCommon.ILexer): modes.ITokenizationSupport { + return new MonarchTokenizer(modeService, standaloneColorService, modeId, lexer); } diff --git a/src/vs/editor/common/modes/nullMode.ts b/src/vs/editor/common/modes/nullMode.ts index 7c47983486e..65972a9fb4e 100644 --- a/src/vs/editor/common/modes/nullMode.ts +++ b/src/vs/editor/common/modes/nullMode.ts @@ -4,7 +4,7 @@ *--------------------------------------------------------------------------------------------*/ 'use strict'; -import { IState, ILineTokens } from 'vs/editor/common/modes'; +import { IState, ILineTokens, ILineTokens3, ColorId, MetadataConsts, LanguageIdentifier, FontStyle, StandardTokenType, LanguageId } from 'vs/editor/common/modes'; import { ModeTransition } from 'vs/editor/common/core/modeTransition'; import { Token } from 'vs/editor/common/core/token'; @@ -23,6 +23,8 @@ export const NULL_STATE: IState = new NullStateImpl(); export const NULL_MODE_ID = 'vs.editor.nullMode'; +export const NULL_LANGUAGE_IDENTIFIER = new LanguageIdentifier(NULL_MODE_ID, LanguageId.Null); + export function nullTokenize(modeId: string, buffer: string, state: IState, deltaOffset: number): ILineTokens { let tokens: Token[] = [new Token(deltaOffset, '')]; @@ -34,3 +36,20 @@ export function nullTokenize(modeId: string, buffer: string, state: IState, delt modeTransitions: modeTransitions }; } + +export function nullTokenize3(languageId: LanguageId, buffer: string, state: IState, deltaOffset: number): ILineTokens3 { + let tokens = new Uint32Array(2); + tokens[0] = deltaOffset; + tokens[1] = ( + (languageId << MetadataConsts.LANGUAGEID_OFFSET) + | (StandardTokenType.Other << MetadataConsts.TOKEN_TYPE_OFFSET) + | (FontStyle.None << MetadataConsts.FONT_STYLE_OFFSET) + | (ColorId.DefaultForeground << MetadataConsts.FOREGROUND_OFFSET) + | (ColorId.DefaultBackground << MetadataConsts.BACKGROUND_OFFSET) + ) >>> 0; + + return { + tokens: tokens, + endState: state + }; +} diff --git a/src/vs/editor/common/modes/snippetsRegistry.ts b/src/vs/editor/common/modes/snippetsRegistry.ts index 9aaf64e18ff..55d19ec8a50 100644 --- a/src/vs/editor/common/modes/snippetsRegistry.ts +++ b/src/vs/editor/common/modes/snippetsRegistry.ts @@ -7,7 +7,7 @@ import { localize } from 'vs/nls'; import * as strings from 'vs/base/common/strings'; import { ITokenizedModel, IPosition } from 'vs/editor/common/editorCommon'; -import { ISuggestion } from 'vs/editor/common/modes'; +import { ISuggestion, LanguageIdentifier, LanguageId } from 'vs/editor/common/modes'; import { Registry } from 'vs/platform/platform'; export const Extensions = { @@ -19,12 +19,12 @@ export interface ISnippetsRegistry { /** * Register a snippet to the registry. */ - registerSnippets(modeId: string, snippets: ISnippet[], owner?: string): void; + registerSnippets(languageIdentifier: LanguageIdentifier, snippets: ISnippet[], owner?: string): void; /** * Visit all snippets */ - visitSnippets(modeId: string, accept: (snippet: ISnippet) => void): void; + visitSnippets(languageId: LanguageId, accept: (snippet: ISnippet) => void): void; /** * Get all snippet completions for the given position @@ -47,18 +47,18 @@ interface ISnippetSuggestion extends ISuggestion { class SnippetsRegistry implements ISnippetsRegistry { - private _snippets: { [modeId: string]: { [owner: string]: ISnippet[] } } = Object.create(null); + private _snippets: { [owner: string]: ISnippet[] }[] = []; - public registerSnippets(modeId: string, snippets: ISnippet[], owner = ''): void { - let snippetsByMode = this._snippets[modeId]; + public registerSnippets(languageIdentifier: LanguageIdentifier, snippets: ISnippet[], owner = ''): void { + let snippetsByMode = this._snippets[languageIdentifier.iid]; if (!snippetsByMode) { - this._snippets[modeId] = snippetsByMode = {}; + this._snippets[languageIdentifier.iid] = snippetsByMode = {}; } snippetsByMode[owner] = snippets; } - public visitSnippets(modeId: string, accept: (snippet: ISnippet) => boolean): void { - let snippetsByMode = this._snippets[modeId]; + public visitSnippets(languageId: LanguageId, accept: (snippet: ISnippet) => boolean): void { + let snippetsByMode = this._snippets[languageId]; if (snippetsByMode) { for (let s in snippetsByMode) { let result = snippetsByMode[s].every(accept); @@ -70,8 +70,8 @@ class SnippetsRegistry implements ISnippetsRegistry { } public getSnippetCompletions(model: ITokenizedModel, position: IPosition): ISuggestion[] { - const modeId = model.getModeIdAtPosition(position.lineNumber, position.column); - if (!this._snippets[modeId]) { + const languageId = model.getLanguageIdAtPosition(position.lineNumber, position.column); + if (!this._snippets[languageId]) { return; } @@ -81,7 +81,7 @@ class SnippetsRegistry implements ISnippetsRegistry { const currentWord = word ? word.word.substring(0, position.column - word.startColumn).toLowerCase() : ''; const currentFullWord = getNonWhitespacePrefix(model, position).toLowerCase(); - this.visitSnippets(modeId, s => { + this.visitSnippets(languageId, s => { let overwriteBefore: number; if (currentWord.length === 0 && currentFullWord.length === 0) { // if there's no prefix, only show snippets at the beginning of the line, or after a whitespace diff --git a/src/vs/editor/common/modes/supports.ts b/src/vs/editor/common/modes/supports.ts index 95ea02db568..ccd31dd70e7 100644 --- a/src/vs/editor/common/modes/supports.ts +++ b/src/vs/editor/common/modes/supports.ts @@ -7,7 +7,7 @@ import * as modes from 'vs/editor/common/modes'; import { ModeTransition } from 'vs/editor/common/core/modeTransition'; import { Token } from 'vs/editor/common/core/token'; -import { LineTokens, StandardTokenType } from 'vs/editor/common/core/lineTokens'; +import { LineTokens } from 'vs/editor/common/core/lineTokens'; export class RawLineTokens implements modes.ILineTokens { _lineTokensBrand: void; @@ -24,34 +24,34 @@ export class RawLineTokens implements modes.ILineTokens { } export function createScopedLineTokens(context: LineTokens, offset: number): ScopedLineTokens { - let modeTransitions = context.modeTransitions; - if (modeTransitions.length === 1) { - return new ScopedLineTokens(context, modeTransitions[0].modeId, 0, context.getTokenCount(), 0, context.getLineContent().length); + let tokenCount = context.getTokenCount(); + let tokenIndex = context.findTokenIndexAtOffset(offset); + let desiredLanguageId = context.getLanguageId(tokenIndex); + + let lastTokenIndex = tokenIndex; + while (lastTokenIndex + 1 < tokenCount && context.getLanguageId(lastTokenIndex + 1) === desiredLanguageId) { + lastTokenIndex++; } - let modeIndex = ModeTransition.findIndexInSegmentsArray(modeTransitions, offset); - let nestedModeId = modeTransitions[modeIndex].modeId; - let modeStartIndex = modeTransitions[modeIndex].startIndex; - - let firstTokenIndex = context.findTokenIndexAtOffset(modeStartIndex); - let lastCharOffset = -1; - let lastTokenIndex = -1; - if (modeIndex + 1 < modeTransitions.length) { - lastTokenIndex = context.findTokenIndexAtOffset(modeTransitions[modeIndex + 1].startIndex); - lastCharOffset = context.getTokenStartOffset(lastTokenIndex); - } else { - lastTokenIndex = context.getTokenCount(); - lastCharOffset = context.getLineContent().length; + let firstTokenIndex = tokenIndex; + while (firstTokenIndex > 0 && context.getLanguageId(firstTokenIndex - 1) === desiredLanguageId) { + firstTokenIndex--; } - let firstCharOffset = context.getTokenStartOffset(firstTokenIndex); - return new ScopedLineTokens(context, nestedModeId, firstTokenIndex, lastTokenIndex, firstCharOffset, lastCharOffset); + return new ScopedLineTokens( + context, + desiredLanguageId, + firstTokenIndex, + lastTokenIndex + 1, + context.getTokenStartOffset(firstTokenIndex), + context.getTokenEndOffset(lastTokenIndex) + ); } export class ScopedLineTokens { _scopedLineTokensBrand: void; - public readonly modeId: string; + public readonly languageId: modes.LanguageId; private readonly _actual: LineTokens; private readonly _firstTokenIndex: number; private readonly _lastTokenIndex: number; @@ -60,14 +60,14 @@ export class ScopedLineTokens { constructor( actual: LineTokens, - modeId: string, + languageId: modes.LanguageId, firstTokenIndex: number, lastTokenIndex: number, firstCharOffset: number, lastCharOffset: number ) { this._actual = actual; - this.modeId = modeId; + this.languageId = languageId; this._firstTokenIndex = firstTokenIndex; this._lastTokenIndex = lastTokenIndex; this.firstCharOffset = firstCharOffset; @@ -91,15 +91,15 @@ export class ScopedLineTokens { return this._actual.getTokenStartOffset(tokenIndex + this._firstTokenIndex) - this.firstCharOffset; } - public getStandardTokenType(tokenIndex: number): StandardTokenType { + public getStandardTokenType(tokenIndex: number): modes.StandardTokenType { return this._actual.getStandardTokenType(tokenIndex + this._firstTokenIndex); } } const enum IgnoreBracketsInTokens { - value = StandardTokenType.Comment | StandardTokenType.String | StandardTokenType.RegEx + value = modes.StandardTokenType.Comment | modes.StandardTokenType.String | modes.StandardTokenType.RegEx } -export function ignoreBracketsInToken(standardTokenType: StandardTokenType): boolean { +export function ignoreBracketsInToken(standardTokenType: modes.StandardTokenType): boolean { return (standardTokenType & IgnoreBracketsInTokens.value) !== 0; } diff --git a/src/vs/editor/common/modes/supports/richEditBrackets.ts b/src/vs/editor/common/modes/supports/richEditBrackets.ts index e88dceb376e..817e0c67b9f 100644 --- a/src/vs/editor/common/modes/supports/richEditBrackets.ts +++ b/src/vs/editor/common/modes/supports/richEditBrackets.ts @@ -7,6 +7,7 @@ import * as strings from 'vs/base/common/strings'; import { Range } from 'vs/editor/common/core/range'; import { CharacterPair } from 'vs/editor/common/modes/languageConfiguration'; +import { LanguageIdentifier } from 'vs/editor/common/modes'; interface ISimpleInternalBracket { open: string; @@ -16,14 +17,14 @@ interface ISimpleInternalBracket { export class RichEditBracket { _richEditBracketBrand: void; - readonly modeId: string; + readonly languageIdentifier: LanguageIdentifier; readonly open: string; readonly close: string; readonly forwardRegex: RegExp; readonly reversedRegex: RegExp; - constructor(modeId: string, open: string, close: string, forwardRegex: RegExp, reversedRegex: RegExp) { - this.modeId = modeId; + constructor(languageIdentifier: LanguageIdentifier, open: string, close: string, forwardRegex: RegExp, reversedRegex: RegExp) { + this.languageIdentifier = languageIdentifier; this.open = open; this.close = close; this.forwardRegex = forwardRegex; @@ -41,10 +42,10 @@ export class RichEditBrackets { public readonly textIsBracket: { [text: string]: RichEditBracket; }; public readonly textIsOpenBracket: { [text: string]: boolean; }; - constructor(modeId: string, brackets: CharacterPair[]) { + constructor(languageIdentifier: LanguageIdentifier, brackets: CharacterPair[]) { this.brackets = brackets.map((b) => { return new RichEditBracket( - modeId, + languageIdentifier, b[0], b[1], getRegexForBracketPair({ open: b[0], close: b[1] }), diff --git a/src/vs/editor/common/modes/supports/tokenization.ts b/src/vs/editor/common/modes/supports/tokenization.ts index 2e3e8d78005..cd9967c0895 100644 --- a/src/vs/editor/common/modes/supports/tokenization.ts +++ b/src/vs/editor/common/modes/supports/tokenization.ts @@ -5,6 +5,8 @@ 'use strict'; import { Map, createMap } from 'vs/editor/common/core/map'; +import { ColorId, FontStyle, MetadataConsts, LanguageId } from 'vs/editor/common/modes'; +import { toStandardTokenType } from 'vs/editor/common/core/lineTokens'; export interface IThemeRule { token: string; @@ -13,35 +15,27 @@ export interface IThemeRule { fontStyle?: string; } -export const enum FontStyle { - NotSet = -1, - None = 0, - Italic = 1, - Bold = 2, - Underline = 4 -} - export class ParsedThemeRule { _parsedThemeRuleBrand: void; - readonly scope: string; + readonly token: string; readonly index: number; /** * -1 if not set. An or mask of `FontStyle` otherwise. */ - readonly fontStyle: number; + readonly fontStyle: FontStyle; readonly foreground: string; readonly background: string; constructor( - scope: string, + token: string, index: number, fontStyle: number, foreground: string, background: string, ) { - this.scope = scope; + this.token = token; this.index = index; this.fontStyle = fontStyle; this.foreground = foreground; @@ -110,7 +104,7 @@ function resolveParsedThemeRules(parsedThemeRules: ParsedThemeRule[]): Theme { // Sort rules lexicographically, and then by index if necessary parsedThemeRules.sort((a, b) => { - let r = strcmp(a.scope, b.scope); + let r = strcmp(a.token, b.token); if (r !== 0) { return r; } @@ -119,9 +113,9 @@ function resolveParsedThemeRules(parsedThemeRules: ParsedThemeRule[]): Theme { // Determine defaults let defaultFontStyle = FontStyle.None; - let defaultForeground = '#000000'; - let defaultBackground = '#ffffff'; - while (parsedThemeRules.length >= 1 && parsedThemeRules[0].scope === '') { + let defaultForeground = '000000'; + let defaultBackground = 'ffffff'; + while (parsedThemeRules.length >= 1 && parsedThemeRules[0].token === '') { let incomingDefaults = parsedThemeRules.shift(); if (incomingDefaults.fontStyle !== FontStyle.NotSet) { defaultFontStyle = incomingDefaults.fontStyle; @@ -134,34 +128,38 @@ function resolveParsedThemeRules(parsedThemeRules: ParsedThemeRule[]): Theme { } } let colorMap = new ColorMap(); + // ensure default foreground gets id 1 and default background gets id 2 let defaults = new ThemeTrieElementRule(defaultFontStyle, colorMap.getId(defaultForeground), colorMap.getId(defaultBackground)); - let root = new ThemeTrieElement(new ThemeTrieElementRule(FontStyle.NotSet, 0, 0)); + let root = new ThemeTrieElement(defaults); for (let i = 0, len = parsedThemeRules.length; i < len; i++) { let rule = parsedThemeRules[i]; - root.insert(rule.scope, rule.fontStyle, colorMap.getId(rule.foreground), colorMap.getId(rule.background)); + root.insert(rule.token, rule.fontStyle, colorMap.getId(rule.foreground), colorMap.getId(rule.background)); } - return new Theme(colorMap, defaults, root); + return new Theme(colorMap, root); } export class ColorMap { private _lastColorId: number; private _id2color: string[]; - private _color2id: Map; + private _color2id: Map; constructor() { this._lastColorId = 0; this._id2color = []; - this._color2id = createMap(); + this._color2id = createMap(); } - public getId(color: string): number { + public getId(color: string): ColorId { if (color === null) { return 0; } color = color.toUpperCase(); + if (!/^[0-9A-F]{6}$/.test(color)) { + throw new Error('Illegal color name: ' + color); + } let value = this._color2id.get(color); if (value) { return value; @@ -190,13 +188,11 @@ export class Theme { private readonly _colorMap: ColorMap; private readonly _root: ThemeTrieElement; - private readonly _defaults: ThemeTrieElementRule; private readonly _cache: Map; - constructor(colorMap: ColorMap, defaults: ThemeTrieElementRule, root: ThemeTrieElement) { + constructor(colorMap: ColorMap, root: ThemeTrieElement) { this._colorMap = colorMap; this._root = root; - this._defaults = defaults; this._cache = createMap(); } @@ -204,10 +200,6 @@ export class Theme { return this._colorMap.getColorMap(); } - public getDefaults(): ThemeTrieElementRule { - return this._defaults; - } - /** * used for testing purposes */ @@ -215,14 +207,25 @@ export class Theme { return this._root.toExternalThemeTrieElement(); } - public match(scopeName: string): ThemeTrieElementRule { - let result = this._cache.get(scopeName); + public _match(token: string): ThemeTrieElementRule { + let result = this._cache.get(token); if (typeof result === 'undefined') { - result = this._root.match(scopeName); - this._cache.set(scopeName, result); + result = this._root.match(token); + this._cache.set(token, result); } return result; } + + public match(languageId: LanguageId, token: string): number { + let rule = this._match(token); + let standardToken = toStandardTokenType(token); + + return ( + rule.metadata + | (standardToken << MetadataConsts.TOKEN_TYPE_OFFSET) + | (languageId << MetadataConsts.LANGUAGEID_OFFSET) + ) >>> 0; + } } export function strcmp(a: string, b: string): number { @@ -238,18 +241,24 @@ export function strcmp(a: string, b: string): number { export class ThemeTrieElementRule { _themeTrieElementRuleBrand: void; - fontStyle: number; - foreground: number; - background: number; + private _fontStyle: FontStyle; + private _foreground: ColorId; + private _background: ColorId; + public metadata: number; - constructor(fontStyle: number, foreground: number, background: number) { - this.fontStyle = fontStyle; - this.foreground = foreground; - this.background = background; + constructor(fontStyle: FontStyle, foreground: ColorId, background: ColorId) { + this._fontStyle = fontStyle; + this._foreground = foreground; + this._background = background; + this.metadata = ( + (this._fontStyle << MetadataConsts.FONT_STYLE_OFFSET) + | (this._foreground << MetadataConsts.FOREGROUND_OFFSET) + | (this._background << MetadataConsts.BACKGROUND_OFFSET) + ) >>> 0; } public clone(): ThemeTrieElementRule { - return new ThemeTrieElementRule(this.fontStyle, this.foreground, this.background); + return new ThemeTrieElementRule(this._fontStyle, this._foreground, this._background); } public static cloneArr(arr: ThemeTrieElementRule[]): ThemeTrieElementRule[] { @@ -260,16 +269,21 @@ export class ThemeTrieElementRule { return r; } - public acceptOverwrite(fontStyle: number, foreground: number, background: number): void { + public acceptOverwrite(fontStyle: FontStyle, foreground: ColorId, background: ColorId): void { if (fontStyle !== FontStyle.NotSet) { - this.fontStyle = fontStyle; + this._fontStyle = fontStyle; } - if (foreground !== 0) { - this.foreground = foreground; + if (foreground !== ColorId.None) { + this._foreground = foreground; } - if (background !== 0) { - this.background = background; + if (background !== ColorId.None) { + this._background = background; } + this.metadata = ( + (this._fontStyle << MetadataConsts.FONT_STYLE_OFFSET) + | (this._foreground << MetadataConsts.FOREGROUND_OFFSET) + | (this._background << MetadataConsts.BACKGROUND_OFFSET) + ) >>> 0; } } @@ -306,20 +320,20 @@ export class ThemeTrieElement { return new ExternalThemeTrieElement(this._mainRule, children); } - public match(scope: string): ThemeTrieElementRule { - if (scope === '') { + public match(token: string): ThemeTrieElementRule { + if (token === '') { return this._mainRule; } - let dotIndex = scope.indexOf('.'); + let dotIndex = token.indexOf('.'); let head: string; let tail: string; if (dotIndex === -1) { - head = scope; + head = token; tail = ''; } else { - head = scope.substring(0, dotIndex); - tail = scope.substring(dotIndex + 1); + head = token.substring(0, dotIndex); + tail = token.substring(dotIndex + 1); } let child = this._children.get(head); @@ -330,22 +344,22 @@ export class ThemeTrieElement { return this._mainRule; } - public insert(scope: string, fontStyle: number, foreground: number, background: number): void { - if (scope === '') { + public insert(token: string, fontStyle: FontStyle, foreground: ColorId, background: ColorId): void { + if (token === '') { // Merge into the main rule this._mainRule.acceptOverwrite(fontStyle, foreground, background); return; } - let dotIndex = scope.indexOf('.'); + let dotIndex = token.indexOf('.'); let head: string; let tail: string; if (dotIndex === -1) { - head = scope; + head = token; tail = ''; } else { - head = scope.substring(0, dotIndex); - tail = scope.substring(dotIndex + 1); + head = token.substring(0, dotIndex); + tail = token.substring(dotIndex + 1); } let child = this._children.get(head); diff --git a/src/vs/editor/common/modes/textToHtmlTokenizer.ts b/src/vs/editor/common/modes/textToHtmlTokenizer.ts index 8a0f28c74d5..fa7bb112675 100644 --- a/src/vs/editor/common/modes/textToHtmlTokenizer.ts +++ b/src/vs/editor/common/modes/textToHtmlTokenizer.ts @@ -4,17 +4,13 @@ *--------------------------------------------------------------------------------------------*/ 'use strict'; -import { IHTMLContentElement } from 'vs/base/common/htmlContent'; import * as strings from 'vs/base/common/strings'; -import { IState, ITokenizationSupport, TokenizationRegistry } from 'vs/editor/common/modes'; -import { NULL_STATE, nullTokenize } from 'vs/editor/common/modes/nullMode'; +import { IState, ITokenizationSupport, TokenizationRegistry, LanguageId } from 'vs/editor/common/modes'; +import { NULL_STATE, nullTokenize3 } from 'vs/editor/common/modes/nullMode'; +import { LineTokens } from 'vs/editor/common/core/lineTokens'; -export function tokenizeToHtmlContent(text: string, languageId: string): IHTMLContentElement { - return _tokenizeToHtmlContent(text, _getSafeTokenizationSupport(languageId)); -} - -export function tokenizeToString(text: string, languageId: string, extraTokenClass?: string): string { - return _tokenizeToString(text, _getSafeTokenizationSupport(languageId), extraTokenClass); +export function tokenizeToString(text: string, languageId: string): string { + return _tokenizeToString(text, _getSafeTokenizationSupport(languageId)); } function _getSafeTokenizationSupport(languageId: string): ITokenizationSupport { @@ -24,106 +20,41 @@ function _getSafeTokenizationSupport(languageId: string): ITokenizationSupport { } return { getInitialState: () => NULL_STATE, - tokenize: (buffer: string, state: IState, deltaOffset: number) => nullTokenize(null, buffer, state, deltaOffset) + tokenize: undefined, + tokenize3: (buffer: string, state: IState, deltaOffset: number) => nullTokenize3(LanguageId.Null, buffer, state, deltaOffset) }; } -function _tokenizeToHtmlContent(text: string, tokenizationSupport: ITokenizationSupport): IHTMLContentElement { - var result: IHTMLContentElement = { - tagName: 'div', - style: 'white-space: pre-wrap', - children: [] - }; +function _tokenizeToString(text: string, tokenizationSupport: ITokenizationSupport): string { + let result = `
`; + let lines = text.split(/\r\n|\r|\n/); + let currentState = tokenizationSupport.getInitialState(); + for (let i = 0, len = lines.length; i < len; i++) { + let line = lines[i]; - var emitToken = (className: string, tokenText: string) => { - result.children.push({ - tagName: 'span', - className: className, - text: tokenText - }); - }; + if (i > 0) { + result += `
`; + } - var emitNewLine = () => { - result.children.push({ - tagName: 'br' - }); - }; + let tokenizationResult = tokenizationSupport.tokenize3(line, currentState, 0); + let lineTokens = new LineTokens(null, tokenizationResult.tokens, line); + let viewLineTokens = lineTokens.inflate(); - _tokenizeLines(text, tokenizationSupport, emitToken, emitNewLine); + let startOffset = 0; + let className = viewLineTokens[0].type; + for (let j = 1, lenJ = viewLineTokens.length; j < lenJ; j++) { + let viewLineToken = viewLineTokens[j]; + + result += `${strings.escape(line.substring(startOffset, viewLineToken.startIndex))}`; + startOffset = viewLineToken.startIndex; + className = viewLineToken.type; + } + result += `${strings.escape(line.substring(startOffset))}`; + + currentState = tokenizationResult.endState; + } + + result += `
`; return result; } - -function _tokenizeToString(text: string, tokenizationSupport: ITokenizationSupport, extraTokenClass: string = ''): string { - if (extraTokenClass && extraTokenClass.length > 0) { - extraTokenClass = ' ' + extraTokenClass; - } - - var result = ''; - - var emitToken = (className: string, tokenText: string) => { - result += '' + strings.escape(tokenText) + ''; - }; - - var emitNewLine = () => { - result += '
'; - }; - - result = `
`; - _tokenizeLines(text, tokenizationSupport, emitToken, emitNewLine); - result += '
'; - - return result; -} - -interface IEmitTokenFunc { - (className: string, innerText: string): void; -} -interface IEmitNewLineFunc { - (): void; -} - -function _tokenizeLines(text: string, tokenizationSupport: ITokenizationSupport, emitToken: IEmitTokenFunc, emitNewLine: IEmitNewLineFunc): void { - var lines = text.split(/\r\n|\r|\n/); - var currentState = tokenizationSupport.getInitialState(); - for (var i = 0; i < lines.length; i++) { - currentState = _tokenizeLine(lines[i], tokenizationSupport, emitToken, currentState); - - // Keep new lines - if (i < lines.length - 1) { - emitNewLine(); - } - } -} - -function _tokenizeLine(line: string, tokenizationSupport: ITokenizationSupport, emitToken: IEmitTokenFunc, startState: IState): IState { - var tokenized = tokenizationSupport.tokenize(line, startState, 0), - endState = tokenized.endState, - tokens = tokenized.tokens, - offset = 0, - tokenText: string; - - // For each token inject spans with proper class names based on token type - for (var j = 0; j < tokens.length; j++) { - var token = tokens[j]; - - // Tokens only provide a startIndex from where they are valid from. As such, we need to - // look ahead the value of the token by advancing until the next tokens start inex or the - // end of the line. - if (j < tokens.length - 1) { - tokenText = line.substring(offset, tokens[j + 1].startIndex); - offset = tokens[j + 1].startIndex; - } else { - tokenText = line.substr(offset); - } - - var className = 'token'; - var safeType = token.type.replace(/[^a-z0-9\-]/gi, ' '); - if (safeType.length > 0) { - className += ' ' + safeType; - } - emitToken(className, tokenText); - } - - return endState; -} diff --git a/src/vs/editor/common/services/editorWorkerServiceImpl.ts b/src/vs/editor/common/services/editorWorkerServiceImpl.ts index ae7ae9bbd7e..309d746e102 100644 --- a/src/vs/editor/common/services/editorWorkerServiceImpl.ts +++ b/src/vs/editor/common/services/editorWorkerServiceImpl.ts @@ -344,7 +344,7 @@ export class EditorWorkerClient extends Disposable { if (!model) { return null; } - let wordDefRegExp = LanguageConfigurationRegistry.getWordDefinition(model.getModeId()); + let wordDefRegExp = LanguageConfigurationRegistry.getWordDefinition(model.getLanguageIdentifier().iid); let wordDef = wordDefRegExp.source; let wordDefFlags = (wordDefRegExp.global ? 'g' : '') + (wordDefRegExp.ignoreCase ? 'i' : '') + (wordDefRegExp.multiline ? 'm' : ''); return proxy.textualSuggest(resource.toString(), position, wordDef, wordDefFlags); @@ -357,7 +357,7 @@ export class EditorWorkerClient extends Disposable { if (!model) { return null; } - let wordDefRegExp = LanguageConfigurationRegistry.getWordDefinition(model.getModeId()); + let wordDefRegExp = LanguageConfigurationRegistry.getWordDefinition(model.getLanguageIdentifier().iid); let wordDef = wordDefRegExp.source; let wordDefFlags = (wordDefRegExp.global ? 'g' : '') + (wordDefRegExp.ignoreCase ? 'i' : '') + (wordDefRegExp.multiline ? 'm' : ''); return proxy.navigateValueSet(resource.toString(), range, up, wordDef, wordDefFlags); diff --git a/src/vs/editor/common/services/languagesRegistry.ts b/src/vs/editor/common/services/languagesRegistry.ts index e24abfc987f..08ef758a467 100644 --- a/src/vs/editor/common/services/languagesRegistry.ts +++ b/src/vs/editor/common/services/languagesRegistry.ts @@ -10,10 +10,12 @@ import * as mime from 'vs/base/common/mime'; import * as strings from 'vs/base/common/strings'; import { ModesRegistry } from 'vs/editor/common/modes/modesRegistry'; import { ILanguageExtensionPoint } from 'vs/editor/common/services/modeService'; +import { LanguageId, LanguageIdentifier } from 'vs/editor/common/modes'; var hasOwnProperty = Object.prototype.hasOwnProperty; export interface IResolvedLanguage { + id: LanguageId; name: string; mimetypes: string[]; aliases: string[]; @@ -24,20 +26,24 @@ export interface IResolvedLanguage { export class LanguagesRegistry { + private _nextLanguageId: number; private _languages: { [id: string]: IResolvedLanguage; }; private mime2LanguageId: { [mimeType: string]: string; }; private name2LanguageId: { [name: string]: string; }; private lowerName2Id: { [name: string]: string; }; + private languageIds: string[]; private _onDidAddModes: Emitter = new Emitter(); public onDidAddModes: Event = this._onDidAddModes.event; constructor(useModesRegistry = true) { + this._nextLanguageId = 1; this._languages = {}; this.mime2LanguageId = {}; this.name2LanguageId = {}; this.lowerName2Id = {}; + this.languageIds = []; if (useModesRegistry) { this._registerLanguages(ModesRegistry.getLanguages()); @@ -83,7 +89,9 @@ export class LanguagesRegistry { if (hasOwnProperty.call(this._languages, langId)) { resolvedLanguage = this._languages[langId]; } else { + let languageId = this._nextLanguageId++; resolvedLanguage = { + id: languageId, name: null, mimetypes: [], aliases: [], @@ -91,6 +99,7 @@ export class LanguagesRegistry { filenames: [], configurationFiles: [] }; + this.languageIds[languageId] = langId; this._languages[langId] = resolvedLanguage; } @@ -116,8 +125,6 @@ export class LanguagesRegistry { resolvedLanguage.mimetypes.push(primaryMime); } - - if (Array.isArray(lang.extensions)) { for (let extension of lang.extensions) { mime.registerTextMime({ id: langId, mime: primaryMime, extension: extension }); @@ -246,6 +253,23 @@ export class LanguagesRegistry { ); } + public getLanguageIdentifier(_modeId: string | LanguageId): LanguageIdentifier { + let modeId: string; + if (typeof _modeId === 'string') { + modeId = _modeId; + } else { + modeId = this.languageIds[_modeId]; + if (!modeId) { + return null; + } + } + + if (!hasOwnProperty.call(this._languages, modeId)) { + return null; + } + return new LanguageIdentifier(modeId, this._languages[modeId].id); + } + public getModeIdsFromLanguageName(languageName: string): string[] { if (!languageName) { return []; diff --git a/src/vs/editor/common/services/modeService.ts b/src/vs/editor/common/services/modeService.ts index a4b0ca702cc..558d3030ed5 100644 --- a/src/vs/editor/common/services/modeService.ts +++ b/src/vs/editor/common/services/modeService.ts @@ -7,7 +7,7 @@ import Event from 'vs/base/common/event'; import { TPromise } from 'vs/base/common/winjs.base'; import { createDecorator } from 'vs/platform/instantiation/common/instantiation'; -import * as modes from 'vs/editor/common/modes'; +import { IMode, LanguageId, LanguageIdentifier } from 'vs/editor/common/modes'; export var IModeService = createDecorator('modeService'); @@ -42,7 +42,7 @@ export interface IModeService { _serviceBrand: any; onDidAddModes: Event; - onDidCreateMode: Event; + onDidCreateMode: Event; // --- reading isRegisteredMode(mimetypeOrModeId: string): boolean; @@ -55,12 +55,13 @@ export interface IModeService { getModeIdForLanguageName(alias: string): string; getModeIdByFilenameOrFirstLine(filename: string, firstLine?: string): string; getModeId(commaSeparatedMimetypesOrCommaSeparatedIds: string): string; + getLanguageIdentifier(modeId: string | LanguageId): LanguageIdentifier; getConfigurationFiles(modeId: string): string[]; // --- instantiation lookup(commaSeparatedMimetypesOrCommaSeparatedIds: string): IModeLookupResult[]; - getMode(commaSeparatedMimetypesOrCommaSeparatedIds: string): modes.IMode; - getOrCreateMode(commaSeparatedMimetypesOrCommaSeparatedIds: string): TPromise; - getOrCreateModeByLanguageName(languageName: string): TPromise; - getOrCreateModeByFilenameOrFirstLine(filename: string, firstLine?: string): TPromise; + getMode(commaSeparatedMimetypesOrCommaSeparatedIds: string): IMode; + getOrCreateMode(commaSeparatedMimetypesOrCommaSeparatedIds: string): TPromise; + getOrCreateModeByLanguageName(languageName: string): TPromise; + getOrCreateModeByFilenameOrFirstLine(filename: string, firstLine?: string): TPromise; } diff --git a/src/vs/editor/common/services/modeServiceImpl.ts b/src/vs/editor/common/services/modeServiceImpl.ts index bedcb16e851..924e16c758b 100644 --- a/src/vs/editor/common/services/modeServiceImpl.ts +++ b/src/vs/editor/common/services/modeServiceImpl.ts @@ -14,14 +14,12 @@ import { IFilesConfiguration } from 'vs/platform/files/common/files'; import { IExtensionService } from 'vs/platform/extensions/common/extensions'; import { IExtensionPoint, IExtensionPointUser, ExtensionMessageCollector, ExtensionsRegistry } from 'vs/platform/extensions/common/extensionsRegistry'; import { IInstantiationService } from 'vs/platform/instantiation/common/instantiation'; -import * as modes from 'vs/editor/common/modes'; +import { IMode, LanguageId, LanguageIdentifier } from 'vs/editor/common/modes'; import { FrankensteinMode } from 'vs/editor/common/modes/abstractMode'; import { ModesRegistry } from 'vs/editor/common/modes/modesRegistry'; import { LanguagesRegistry } from 'vs/editor/common/services/languagesRegistry'; import { ILanguageExtensionPoint, IValidLanguageExtensionPoint, IModeLookupResult, IModeService } from 'vs/editor/common/services/modeService'; import { IConfigurationService } from 'vs/platform/configuration/common/configuration'; -import { Token } from 'vs/editor/common/core/token'; -import { ModeTransition } from 'vs/editor/common/core/modeTransition'; export const languagesExtPoint: IExtensionPoint = ExtensionsRegistry.registerExtensionPoint('languages', [], { description: nls.localize('vscode.extension.contributes.languages', 'Contributes language declarations.'), @@ -135,15 +133,15 @@ export class ModeServiceImpl implements IModeService { private _instantiationService: IInstantiationService; protected _extensionService: IExtensionService; - private _instantiatedModes: { [modeId: string]: modes.IMode; }; + private _instantiatedModes: { [modeId: string]: IMode; }; private _registry: LanguagesRegistry; private _onDidAddModes: Emitter = new Emitter(); public onDidAddModes: Event = this._onDidAddModes.event; - private _onDidCreateMode: Emitter = new Emitter(); - public onDidCreateMode: Event = this._onDidCreateMode.event; + private _onDidCreateMode: Emitter = new Emitter(); + public onDidCreateMode: Event = this._onDidCreateMode.event; constructor( instantiationService: IInstantiationService, @@ -200,6 +198,10 @@ export class ModeServiceImpl implements IModeService { return null; } + public getLanguageIdentifier(modeId: string | LanguageId): LanguageIdentifier { + return this._registry.getLanguageIdentifier(modeId); + } + public getConfigurationFiles(modeId: string): string[] { return this._registry.getConfigurationFiles(modeId); } @@ -222,7 +224,7 @@ export class ModeServiceImpl implements IModeService { return r; } - public getMode(commaSeparatedMimetypesOrCommaSeparatedIds: string): modes.IMode { + public getMode(commaSeparatedMimetypesOrCommaSeparatedIds: string): IMode { var modeIds = this._registry.extractModeIds(commaSeparatedMimetypesOrCommaSeparatedIds); var isPlainText = false; @@ -235,7 +237,7 @@ export class ModeServiceImpl implements IModeService { if (isPlainText) { // Try to do it synchronously - var r: modes.IMode = null; + var r: IMode = null; this.getOrCreateMode(commaSeparatedMimetypesOrCommaSeparatedIds).then((mode) => { r = mode; }).done(null, onUnexpectedError); @@ -267,7 +269,7 @@ export class ModeServiceImpl implements IModeService { return this._extensionService.onReady(); } - public getOrCreateMode(commaSeparatedMimetypesOrCommaSeparatedIds: string): TPromise { + public getOrCreateMode(commaSeparatedMimetypesOrCommaSeparatedIds: string): TPromise { return this.onReady().then(() => { var modeId = this.getModeId(commaSeparatedMimetypesOrCommaSeparatedIds); // Fall back to plain text if no mode was found @@ -275,7 +277,7 @@ export class ModeServiceImpl implements IModeService { }); } - public getOrCreateModeByLanguageName(languageName: string): TPromise { + public getOrCreateModeByLanguageName(languageName: string): TPromise { return this.onReady().then(() => { var modeId = this.getModeIdByLanguageName(languageName); // Fall back to plain text if no mode was found @@ -283,7 +285,7 @@ export class ModeServiceImpl implements IModeService { }); } - public getOrCreateModeByFilenameOrFirstLine(filename: string, firstLine?: string): TPromise { + public getOrCreateModeByFilenameOrFirstLine(filename: string, firstLine?: string): TPromise { return this.onReady().then(() => { var modeId = this.getModeIdByFilenameOrFirstLine(filename, firstLine); // Fall back to plain text if no mode was found @@ -291,11 +293,10 @@ export class ModeServiceImpl implements IModeService { }); } - private _getOrCreateMode(modeId: string): modes.IMode { + private _getOrCreateMode(modeId: string): IMode { if (!this._instantiatedModes.hasOwnProperty(modeId)) { - this._instantiatedModes[modeId] = this._instantiationService.createInstance(FrankensteinMode, { - id: modeId - }); + let languageIdentifier = this.getLanguageIdentifier(modeId); + this._instantiatedModes[modeId] = new FrankensteinMode(languageIdentifier); this._onDidCreateMode.fire(this._instantiatedModes[modeId]); @@ -305,49 +306,6 @@ export class ModeServiceImpl implements IModeService { } } -export class TokenizationSupport2Adapter implements modes.ITokenizationSupport { - - private _modeId: string; - private _actual: modes.TokensProvider; - - constructor(modeId: string, actual: modes.TokensProvider) { - this._modeId = modeId; - this._actual = actual; - } - - public getInitialState(): modes.IState { - return this._actual.getInitialState(); - } - - public tokenize(line: string, state: modes.IState, offsetDelta: number): modes.ILineTokens { - let actualResult = this._actual.tokenize(line, state); - let tokens: Token[] = []; - actualResult.tokens.forEach((t) => { - if (typeof t.scopes === 'string') { - tokens.push(new Token(t.startIndex + offsetDelta, t.scopes)); - } else if (Array.isArray(t.scopes) && t.scopes.length === 1) { - tokens.push(new Token(t.startIndex + offsetDelta, t.scopes[0])); - } else { - throw new Error('Only token scopes as strings or of precisely 1 length are supported at this time!'); - } - }); - - let endState: modes.IState; - // try to save an object if possible - if (actualResult.endState.equals(state)) { - endState = state; - } else { - endState = actualResult.endState; - } - - return { - tokens: tokens, - endState: endState, - modeTransitions: [new ModeTransition(offsetDelta, this._modeId)], - }; - } -} - export class MainThreadModeServiceImpl extends ModeServiceImpl { private _configurationService: IConfigurationService; private _onReadyPromise: TPromise; diff --git a/src/vs/editor/common/services/modelServiceImpl.ts b/src/vs/editor/common/services/modelServiceImpl.ts index 3eb47ac4ac6..7452e25be92 100644 --- a/src/vs/editor/common/services/modelServiceImpl.ts +++ b/src/vs/editor/common/services/modelServiceImpl.ts @@ -18,13 +18,13 @@ import { anonymize } from 'vs/platform/telemetry/common/telemetry'; import { Range } from 'vs/editor/common/core/range'; import * as editorCommon from 'vs/editor/common/editorCommon'; import { Model } from 'vs/editor/common/model/model'; -import { IMode } from 'vs/editor/common/modes'; +import { IMode, LanguageIdentifier } from 'vs/editor/common/modes'; import { IModelService } from 'vs/editor/common/services/modelService'; import * as platform from 'vs/base/common/platform'; import { IConfigurationService } from 'vs/platform/configuration/common/configuration'; import { DEFAULT_INDENTATION, DEFAULT_TRIM_AUTO_WHITESPACE } from 'vs/editor/common/config/defaultConfig'; import { IMessageService } from 'vs/platform/message/common/message'; -import { PLAINTEXT_MODE_ID } from 'vs/editor/common/modes/modesRegistry'; +import { PLAINTEXT_LANGUAGE_IDENTIFIER } from 'vs/editor/common/modes/modesRegistry'; function MODEL_ID(resource: URI): string { return resource.toString(); @@ -350,13 +350,13 @@ export class ModelServiceImpl implements IModelService { // --- begin IModelService - private _createModelData(value: string | editorCommon.IRawText, languageId: string, resource: URI): ModelData { + private _createModelData(value: string | editorCommon.IRawText, languageIdentifier: LanguageIdentifier, resource: URI): ModelData { // create & save the model let model: Model; if (typeof value === 'string') { - model = Model.createFromString(value, this._modelCreationOptions, languageId, resource); + model = Model.createFromString(value, this._modelCreationOptions, languageIdentifier, resource); } else { - model = new Model(value, languageId, resource); + model = new Model(value, languageIdentifier, resource); } let modelId = MODEL_ID(model.uri); @@ -375,10 +375,10 @@ export class ModelServiceImpl implements IModelService { let modelData: ModelData; if (!modeOrPromise || TPromise.is(modeOrPromise)) { - modelData = this._createModelData(value, PLAINTEXT_MODE_ID, resource); + modelData = this._createModelData(value, PLAINTEXT_LANGUAGE_IDENTIFIER, resource); this.setMode(modelData.model, modeOrPromise); } else { - modelData = this._createModelData(value, modeOrPromise.getId(), resource); + modelData = this._createModelData(value, modeOrPromise.getLanguageIdentifier(), resource); } // handle markers (marker service => model) @@ -398,11 +398,11 @@ export class ModelServiceImpl implements IModelService { if (TPromise.is(modeOrPromise)) { modeOrPromise.then((mode) => { if (!model.isDisposed()) { - model.setMode(mode.getId()); + model.setMode(mode.getLanguageIdentifier()); } }); } else { - model.setMode(modeOrPromise.getId()); + model.setMode(modeOrPromise.getLanguageIdentifier()); } } diff --git a/src/vs/editor/common/services/standaloneColorService.ts b/src/vs/editor/common/services/standaloneColorService.ts new file mode 100644 index 00000000000..67ff7bfaa71 --- /dev/null +++ b/src/vs/editor/common/services/standaloneColorService.ts @@ -0,0 +1,19 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ +'use strict'; + +import Event from 'vs/base/common/event'; +import { createDecorator } from 'vs/platform/instantiation/common/instantiation'; +import { Theme } from 'vs/editor/common/modes/supports/tokenization'; + +export var IStandaloneColorService = createDecorator('standaloneColorService'); + +export interface IStandaloneColorService { + _serviceBrand: any; + + onThemeChanged: Event; + + getTheme(): Theme; +} diff --git a/src/vs/editor/common/viewLayout/viewLineParts.ts b/src/vs/editor/common/viewLayout/viewLineParts.ts index d219bb82c8e..cbf7394f819 100644 --- a/src/vs/editor/common/viewLayout/viewLineParts.ts +++ b/src/vs/editor/common/viewLayout/viewLineParts.ts @@ -59,7 +59,7 @@ export function getColumnOfLinePartOffset(stopRenderingLineAfter: number, linePa // invariant: offsetOf(min) <= offset <= offsetOf(max) while (min + 1 < max) { - let mid = Math.floor((min + max) / 2); + let mid = ((min + max) >>> 1); let midOffset = charOffsetInPart[mid]; if (midOffset === offset) { diff --git a/src/vs/editor/common/viewLayout/whitespaceComputer.ts b/src/vs/editor/common/viewLayout/whitespaceComputer.ts index 42d5b250bd9..218b7bb2b2d 100644 --- a/src/vs/editor/common/viewLayout/whitespaceComputer.ts +++ b/src/vs/editor/common/viewLayout/whitespaceComputer.ts @@ -75,7 +75,7 @@ export class WhitespaceComputer { mid: number; while (low < high) { - mid = Math.floor((low + high) / 2); + mid = ((low + high) >>> 1); if (value === sortedArray[mid]) { if (valueOrdinal < ordinals[mid]) { diff --git a/src/vs/editor/contrib/comment/common/blockCommentCommand.ts b/src/vs/editor/contrib/comment/common/blockCommentCommand.ts index b21f9c3fa67..10a71e077f3 100644 --- a/src/vs/editor/contrib/comment/common/blockCommentCommand.ts +++ b/src/vs/editor/contrib/comment/common/blockCommentCommand.ts @@ -121,8 +121,8 @@ export class BlockCommentCommand implements editorCommon.ICommand { var endLineNumber = this._selection.endLineNumber; var endColumn = this._selection.endColumn; - let modeId = model.getModeIdAtPosition(startLineNumber, startColumn); - let config = LanguageConfigurationRegistry.getComments(modeId); + let languageId = model.getLanguageIdAtPosition(startLineNumber, startColumn); + let config = LanguageConfigurationRegistry.getComments(languageId); if (!config || !config.blockCommentStartToken || !config.blockCommentEndToken) { // Mode does not support block comments return; diff --git a/src/vs/editor/contrib/comment/common/lineCommentCommand.ts b/src/vs/editor/contrib/comment/common/lineCommentCommand.ts index 7b0853ab141..89d408721bd 100644 --- a/src/vs/editor/contrib/comment/common/lineCommentCommand.ts +++ b/src/vs/editor/contrib/comment/common/lineCommentCommand.ts @@ -11,7 +11,7 @@ import { Range } from 'vs/editor/common/core/range'; import { Selection } from 'vs/editor/common/core/selection'; import * as editorCommon from 'vs/editor/common/editorCommon'; import { BlockCommentCommand } from './blockCommentCommand'; -import { ICommentsConfiguration, LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry'; +import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry'; import { CharCode } from 'vs/base/common/charCode'; export interface IInsertionPoint { @@ -63,31 +63,25 @@ export class LineCommentCommand implements editorCommon.ICommand { * Returns null if any of the lines doesn't support a line comment string. */ public static _gatherPreflightCommentStrings(model: editorCommon.ITokenizedModel, startLineNumber: number, endLineNumber: number): ILinePreflightData[] { - var lines: ILinePreflightData[] = [], - config: ICommentsConfiguration, - commentStr: string, - seenModes: { [modeId: string]: string; } = Object.create(null), - i: number, - lineCount: number, - lineNumber: number, - modeId: string; - - for (i = 0, lineCount = endLineNumber - startLineNumber + 1; i < lineCount; i++) { - lineNumber = startLineNumber + i; - modeId = model.getModeIdAtPosition(lineNumber, 1); + let commentStrForLanguage: string[] = []; + let lines: ILinePreflightData[] = []; + for (let i = 0, lineCount = endLineNumber - startLineNumber + 1; i < lineCount; i++) { + let lineNumber = startLineNumber + i; + let languageId = model.getLanguageIdAtPosition(lineNumber, 1); // Find the commentStr for this line, if none is found then bail out: we cannot do line comments - if (seenModes[modeId]) { - commentStr = seenModes[modeId]; + let commentStr: string; + if (commentStrForLanguage[languageId]) { + commentStr = commentStrForLanguage[languageId]; } else { - config = LanguageConfigurationRegistry.getComments(modeId); + let config = LanguageConfigurationRegistry.getComments(languageId); commentStr = (config ? config.lineCommentToken : null); if (!commentStr) { // Mode does not support line comments return null; } - seenModes[modeId] = commentStr; + commentStrForLanguage[languageId] = commentStr; } lines.push({ @@ -271,8 +265,8 @@ export class LineCommentCommand implements editorCommon.ICommand { * Given an unsuccessful analysis, delegate to the block comment command */ private _executeBlockComment(model: editorCommon.ITokenizedModel, builder: editorCommon.IEditOperationBuilder, s: Selection): void { - let modeId = model.getModeIdAtPosition(s.startLineNumber, s.startColumn); - let config = LanguageConfigurationRegistry.getComments(modeId); + let languageId = model.getLanguageIdAtPosition(s.startLineNumber, s.startColumn); + let config = LanguageConfigurationRegistry.getComments(languageId); if (!config || !config.blockCommentStartToken || !config.blockCommentEndToken) { // Mode does not support block comments return; diff --git a/src/vs/editor/contrib/comment/test/common/blockCommentCommand.test.ts b/src/vs/editor/contrib/comment/test/common/blockCommentCommand.test.ts index 01c77173dd0..8f8f2cae118 100644 --- a/src/vs/editor/contrib/comment/test/common/blockCommentCommand.test.ts +++ b/src/vs/editor/contrib/comment/test/common/blockCommentCommand.test.ts @@ -7,11 +7,12 @@ import { Selection } from 'vs/editor/common/core/selection'; import { BlockCommentCommand } from 'vs/editor/contrib/comment/common/blockCommentCommand'; import { testCommand } from 'vs/editor/test/common/commands/commandTestUtils'; -import { CommentMode } from 'vs/editor/test/common/testModes'; +import { CommentMode } from 'vs/editor/test/common/commentMode'; function testBlockCommentCommand(lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection): void { - var mode = new CommentMode({ lineComment: '!@#', blockComment: ['<0', '0>'] }); - testCommand(lines, mode.getId(), selection, (sel) => new BlockCommentCommand(sel), expectedLines, expectedSelection); + let mode = new CommentMode({ lineComment: '!@#', blockComment: ['<0', '0>'] }); + testCommand(lines, mode.getLanguageIdentifier(), selection, (sel) => new BlockCommentCommand(sel), expectedLines, expectedSelection); + mode.dispose(); } suite('Editor Contrib - Block Comment Command', () => { diff --git a/src/vs/editor/contrib/comment/test/common/lineCommentCommand.test.ts b/src/vs/editor/contrib/comment/test/common/lineCommentCommand.test.ts index d5d5cc97cb3..802efd52419 100644 --- a/src/vs/editor/contrib/comment/test/common/lineCommentCommand.test.ts +++ b/src/vs/editor/contrib/comment/test/common/lineCommentCommand.test.ts @@ -8,18 +8,20 @@ import * as assert from 'assert'; import { Selection } from 'vs/editor/common/core/selection'; import { ILinePreflightData, IPreflightData, ISimpleModel, LineCommentCommand, Type } from 'vs/editor/contrib/comment/common/lineCommentCommand'; import { testCommand } from 'vs/editor/test/common/commands/commandTestUtils'; -import { CommentMode } from 'vs/editor/test/common/testModes'; +import { CommentMode } from 'vs/editor/test/common/commentMode'; suite('Editor Contrib - Line Comment Command', () => { function testLineCommentCommand(lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection): void { - var mode = new CommentMode({ lineComment: '!@#', blockComment: [''] }); - testCommand(lines, mode.getId(), selection, (sel) => new LineCommentCommand(sel, 4, Type.Toggle), expectedLines, expectedSelection); + let mode = new CommentMode({ lineComment: '!@#', blockComment: [''] }); + testCommand(lines, mode.getLanguageIdentifier(), selection, (sel) => new LineCommentCommand(sel, 4, Type.Toggle), expectedLines, expectedSelection); + mode.dispose(); } function testAddLineCommentCommand(lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection): void { - var mode = new CommentMode({ lineComment: '!@#', blockComment: [''] }); - testCommand(lines, mode.getId(), selection, (sel) => new LineCommentCommand(sel, 4, Type.ForceAdd), expectedLines, expectedSelection); + let mode = new CommentMode({ lineComment: '!@#', blockComment: [''] }); + testCommand(lines, mode.getLanguageIdentifier(), selection, (sel) => new LineCommentCommand(sel, 4, Type.ForceAdd), expectedLines, expectedSelection); + mode.dispose(); } test('comment single line', function () { @@ -520,8 +522,9 @@ suite('Editor Contrib - Line Comment Command', () => { suite('Editor Contrib - Line Comment As Block Comment', () => { function testLineCommentCommand(lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection): void { - var mode = new CommentMode({ lineComment: '', blockComment: ['(', ')'] }); - testCommand(lines, mode.getId(), selection, (sel) => new LineCommentCommand(sel, 4, Type.Toggle), expectedLines, expectedSelection); + let mode = new CommentMode({ lineComment: '', blockComment: ['(', ')'] }); + testCommand(lines, mode.getLanguageIdentifier(), selection, (sel) => new LineCommentCommand(sel, 4, Type.Toggle), expectedLines, expectedSelection); + mode.dispose(); } test('fall back to block comment command', function () { @@ -630,8 +633,9 @@ suite('Editor Contrib - Line Comment As Block Comment', () => { suite('Editor Contrib - Line Comment As Block Comment 2', () => { function testLineCommentCommand(lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection): void { - var mode = new CommentMode({ lineComment: null, blockComment: [''] }); - testCommand(lines, mode.getId(), selection, (sel) => new LineCommentCommand(sel, 4, Type.Toggle), expectedLines, expectedSelection); + let mode = new CommentMode({ lineComment: null, blockComment: [''] }); + testCommand(lines, mode.getLanguageIdentifier(), selection, (sel) => new LineCommentCommand(sel, 4, Type.Toggle), expectedLines, expectedSelection); + mode.dispose(); } test('no selection => uses indentation', function () { diff --git a/src/vs/editor/contrib/smartSelect/common/tokenTree.ts b/src/vs/editor/contrib/smartSelect/common/tokenTree.ts index e1d8221e963..9292763e3da 100644 --- a/src/vs/editor/contrib/smartSelect/common/tokenTree.ts +++ b/src/vs/editor/contrib/smartSelect/common/tokenTree.ts @@ -7,10 +7,11 @@ import { Position } from 'vs/editor/common/core/position'; import { Range } from 'vs/editor/common/core/range'; import { IModel, IPosition } from 'vs/editor/common/editorCommon'; -import { LineToken, StandardTokenType } from 'vs/editor/common/core/lineTokens'; +import { LineToken } from 'vs/editor/common/core/lineTokens'; import { ignoreBracketsInToken } from 'vs/editor/common/modes/supports'; import { BracketsUtils, RichEditBrackets } from 'vs/editor/common/modes/supports/richEditBrackets'; import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry'; +import { LanguageId, StandardTokenType } from 'vs/editor/common/modes'; export const enum TokenTreeBracket { None = 0, @@ -128,16 +129,16 @@ class RawToken { public lineText: string; public startOffset: number; public endOffset: number; - public standardType: StandardTokenType; - public modeId: string; + public type: StandardTokenType; + public languageId: LanguageId; constructor(source: LineToken, lineNumber: number, lineText: string) { this.lineNumber = lineNumber; this.lineText = lineText; this.startOffset = source.startOffset; this.endOffset = source.endOffset; - this.standardType = source.standardType; - this.modeId = source.modeId; + this.type = source.tokenType; + this.languageId = source.languageId; } } @@ -188,14 +189,14 @@ class TokenScanner { private _rawTokenScanner: ModelRawTokenScanner; private _nextBuff: Token[]; - private _cachedModeBrackets: RichEditBrackets; - private _cachedModeId: string; + private _cachedLanguageBrackets: RichEditBrackets; + private _cachedLanguageId: LanguageId; constructor(model: IModel) { this._rawTokenScanner = new ModelRawTokenScanner(model); this._nextBuff = []; - this._cachedModeBrackets = null; - this._cachedModeId = null; + this._cachedLanguageBrackets = null; + this._cachedLanguageId = -1; } next(): Token { @@ -209,17 +210,17 @@ class TokenScanner { } const lineNumber = token.lineNumber; const lineText = token.lineText; - const standardTokenType = token.standardType; + const tokenType = token.type; let startOffset = token.startOffset; const endOffset = token.endOffset; - if (this._cachedModeId !== token.modeId) { - this._cachedModeId = token.modeId; - this._cachedModeBrackets = LanguageConfigurationRegistry.getBracketsSupport(this._cachedModeId); + if (this._cachedLanguageId !== token.languageId) { + this._cachedLanguageId = token.languageId; + this._cachedLanguageBrackets = LanguageConfigurationRegistry.getBracketsSupport(this._cachedLanguageId); } - const modeBrackets = this._cachedModeBrackets; + const modeBrackets = this._cachedLanguageBrackets; - if (!modeBrackets || ignoreBracketsInToken(standardTokenType)) { + if (!modeBrackets || ignoreBracketsInToken(tokenType)) { return new Token( new Range(lineNumber, startOffset + 1, lineNumber, endOffset + 1), TokenTreeBracket.None, @@ -252,7 +253,7 @@ class TokenScanner { this._nextBuff.push(new Token( new Range(lineNumber, foundBracketStartOffset + 1, lineNumber, foundBracketEndOffset + 1), bracketIsOpen ? TokenTreeBracket.Open : TokenTreeBracket.Close, - `${bracketData.modeId};${bracketData.open};${bracketData.close}` + `${bracketData.languageIdentifier.sid};${bracketData.open};${bracketData.close}` )); startOffset = foundBracketEndOffset; diff --git a/src/vs/editor/contrib/smartSelect/test/common/tokenSelectionSupport.test.ts b/src/vs/editor/contrib/smartSelect/test/common/tokenSelectionSupport.test.ts index 1a27b066ea6..7e7ad380507 100644 --- a/src/vs/editor/contrib/smartSelect/test/common/tokenSelectionSupport.test.ts +++ b/src/vs/editor/contrib/smartSelect/test/common/tokenSelectionSupport.test.ts @@ -7,7 +7,7 @@ import * as assert from 'assert'; import URI from 'vs/base/common/uri'; import { Range } from 'vs/editor/common/core/range'; -import { IMode } from 'vs/editor/common/modes'; +import { LanguageIdentifier } from 'vs/editor/common/modes'; import { IndentAction } from 'vs/editor/common/modes/languageConfiguration'; import { TokenSelectionSupport } from 'vs/editor/contrib/smartSelect/common/tokenSelectionSupport'; import { MockMode } from 'vs/editor/test/common/mocks/mockMode'; @@ -17,10 +17,12 @@ import { TestConfigurationService } from 'vs/platform/configuration/test/common/ class MockJSMode extends MockMode { - constructor() { - super('mock-js'); + private static _id = new LanguageIdentifier('mockJSMode', 3); - LanguageConfigurationRegistry.register(this.getId(), { + constructor() { + super(MockJSMode._id); + + this._register(LanguageConfigurationRegistry.register(this.getLanguageIdentifier(), { brackets: [ ['(', ')'], ['{', '}'], @@ -55,25 +57,30 @@ class MockJSMode extends MockMode { action: { indentAction: IndentAction.None, removeText: 1 } } ] - }); + })); } } suite('TokenSelectionSupport', () => { - let modelService: ModelServiceImpl; + let modelService: ModelServiceImpl = null; let tokenSelectionSupport: TokenSelectionSupport; - let _mode: IMode = new MockJSMode(); + let mode: MockJSMode = null; setup(() => { - modelService = new ModelServiceImpl(null, new TestConfigurationService(), null); tokenSelectionSupport = new TokenSelectionSupport(modelService); + mode = new MockJSMode(); + }); + + teardown(() => { + modelService.dispose(); + mode.dispose(); }); function assertGetRangesToPosition(text: string[], lineNumber: number, column: number, ranges: Range[]): void { let uri = URI.file('test.js'); - modelService.createModel(text.join('\n'), _mode, uri); + modelService.createModel(text.join('\n'), mode, uri); let actual = tokenSelectionSupport.getRangesToPositionSync(uri, { lineNumber: lineNumber, diff --git a/src/vs/editor/contrib/suggest/browser/tabCompletion.ts b/src/vs/editor/contrib/suggest/browser/tabCompletion.ts index ea1e7cb7948..7a2ab2a9229 100644 --- a/src/vs/editor/contrib/suggest/browser/tabCompletion.ts +++ b/src/vs/editor/contrib/suggest/browser/tabCompletion.ts @@ -57,7 +57,7 @@ export class TabCompletionController implements editorCommon.IEditorContribution } if (selectFn) { - snippetsRegistry.visitSnippets(editor.getModel().getModeId(), s => { + snippetsRegistry.visitSnippets(editor.getModel().getLanguageIdentifier().iid, s => { if (selectFn(s)) { this._currentSnippets.push(s); } diff --git a/src/vs/editor/contrib/suggest/common/snippetCompletion.ts b/src/vs/editor/contrib/suggest/common/snippetCompletion.ts index 2d2e67b7a4f..4b441d3e2e1 100644 --- a/src/vs/editor/contrib/suggest/common/snippetCompletion.ts +++ b/src/vs/editor/contrib/suggest/common/snippetCompletion.ts @@ -37,10 +37,10 @@ class ShowSnippetsActions extends EditorAction { } const {lineNumber, column} = editor.getPosition(); - const modeId = editor.getModel().getModeIdAtPosition(lineNumber, column); + const languageId = editor.getModel().getLanguageIdAtPosition(lineNumber, column); const picks: ISnippetPick[] = []; - Registry.as(Extensions.Snippets).visitSnippets(modeId, snippet => { + Registry.as(Extensions.Snippets).visitSnippets(languageId, snippet => { picks.push({ label: snippet.prefix, detail: snippet.description, diff --git a/src/vs/editor/node/languageConfigurationExtensionPoint.ts b/src/vs/editor/node/languageConfigurationExtensionPoint.ts index 61f8f204da1..91a93ba15be 100644 --- a/src/vs/editor/node/languageConfigurationExtensionPoint.ts +++ b/src/vs/editor/node/languageConfigurationExtensionPoint.ts @@ -14,6 +14,7 @@ import { Extensions, IJSONContributionRegistry } from 'vs/platform/jsonschemas/c import { Registry } from 'vs/platform/platform'; import { IJSONSchema } from 'vs/base/common/jsonSchema'; import { MainProcessTextMateSyntax } from 'vs/editor/node/textMate/TMSyntax'; +import { LanguageIdentifier } from 'vs/editor/common/modes'; interface ILanguageConfiguration { comments?: CommentRule; @@ -25,44 +26,48 @@ interface ILanguageConfiguration { export class LanguageConfigurationFileHandler { private _modeService: IModeService; - private _done: { [modeId: string]: boolean; }; + private _done: boolean[]; constructor( tmSyntax: MainProcessTextMateSyntax, @IModeService modeService: IModeService ) { this._modeService = modeService; - this._done = Object.create(null); + this._done = []; // Listen for hints that a language configuration is needed/usefull and then load it once - this._modeService.onDidCreateMode((mode) => this._loadConfigurationsForMode(mode.getId())); - tmSyntax.onDidEncounterLanguage((language) => this._loadConfigurationsForMode(language)); + this._modeService.onDidCreateMode((mode) => this._loadConfigurationsForMode(mode.getLanguageIdentifier())); + tmSyntax.onDidEncounterLanguage((language) => { + // TODO@tokenization + throw new Error('TODO@tokenization'); + // this._loadConfigurationsForMode(language); + }); } - private _loadConfigurationsForMode(modeId: string): void { - if (this._done[modeId]) { + private _loadConfigurationsForMode(languageIdentifier: LanguageIdentifier): void { + if (this._done[languageIdentifier.iid]) { return; } - this._done[modeId] = true; + this._done[languageIdentifier.iid] = true; - let configurationFiles = this._modeService.getConfigurationFiles(modeId); - configurationFiles.forEach((configFilePath) => this._handleConfigFile(modeId, configFilePath)); + let configurationFiles = this._modeService.getConfigurationFiles(languageIdentifier.sid); + configurationFiles.forEach((configFilePath) => this._handleConfigFile(languageIdentifier, configFilePath)); } - private _handleConfigFile(modeId: string, configFilePath: string): void { + private _handleConfigFile(languageIdentifier: LanguageIdentifier, configFilePath: string): void { readFile(configFilePath).then((fileContents) => { var errors = []; var configuration = parse(fileContents.toString(), errors); if (errors.length) { console.error(nls.localize('parseErrors', "Errors parsing {0}: {1}", configFilePath, errors.join('\n'))); } - this._handleConfig(modeId, configuration); + this._handleConfig(languageIdentifier, configuration); }, (err) => { console.error(err); }); } - private _handleConfig(modeId: string, configuration: ILanguageConfiguration): void { + private _handleConfig(languageIdentifier: LanguageIdentifier, configuration: ILanguageConfiguration): void { let richEditConfig: LanguageConfiguration = {}; @@ -82,7 +87,7 @@ export class LanguageConfigurationFileHandler { richEditConfig.surroundingPairs = this._mapCharacterPairs(configuration.surroundingPairs); } - LanguageConfigurationRegistry.register(modeId, richEditConfig); + LanguageConfigurationRegistry.register(languageIdentifier, richEditConfig); } private _mapCharacterPairs(pairs: (CharacterPair | IAutoClosingPairConditional)[]): IAutoClosingPairConditional[] { diff --git a/src/vs/editor/node/textMate/TMSnippets.ts b/src/vs/editor/node/textMate/TMSnippets.ts index 359af387762..3902495c507 100644 --- a/src/vs/editor/node/textMate/TMSnippets.ts +++ b/src/vs/editor/node/textMate/TMSnippets.ts @@ -14,6 +14,7 @@ import { ISnippetsRegistry, Extensions, ISnippet } from 'vs/editor/common/modes/ import { IModeService } from 'vs/editor/common/services/modeService'; import platform = require('vs/platform/platform'); import { languagesExtPoint } from 'vs/editor/common/services/modeServiceImpl'; +import { LanguageIdentifier } from 'vs/editor/common/modes'; export interface ISnippetsExtensionPoint { language: string; @@ -72,22 +73,25 @@ export class MainProcessTextMateSnippet { } let modeId = snippet.language; - let disposable = this._modeService.onDidCreateMode(mode => { - if (mode.getId() !== modeId) { - return; - } - readAndRegisterSnippets(modeId, normalizedAbsolutePath, extensionName); - disposable.dispose(); - }); + let languageIdentifier = this._modeService.getLanguageIdentifier(modeId); + if (languageIdentifier) { + let disposable = this._modeService.onDidCreateMode(mode => { + if (mode.getId() !== modeId) { + return; + } + readAndRegisterSnippets(languageIdentifier, normalizedAbsolutePath, extensionName); + disposable.dispose(); + }); + } } } let snippetsRegistry = platform.Registry.as(Extensions.Snippets); -export function readAndRegisterSnippets(modeId: string, filePath: string, ownerName: string): TPromise { +export function readAndRegisterSnippets(languageIdentifier: LanguageIdentifier, filePath: string, ownerName: string): TPromise { return readFile(filePath).then(fileContents => { let snippets = parseSnippetFile(fileContents.toString(), ownerName); - snippetsRegistry.registerSnippets(modeId, snippets, filePath); + snippetsRegistry.registerSnippets(languageIdentifier, snippets, filePath); }); } diff --git a/src/vs/editor/node/textMate/TMSyntax.ts b/src/vs/editor/node/textMate/TMSyntax.ts index bb71d484c9a..741ad5449de 100644 --- a/src/vs/editor/node/textMate/TMSyntax.ts +++ b/src/vs/editor/node/textMate/TMSyntax.ts @@ -282,7 +282,11 @@ function createTokenizationSupport(languageRegistration: TMLanguageRegistration, var tokenizer = new Tokenizer(languageRegistration, modeId, grammar); return { getInitialState: () => new TMState(null), - tokenize: (line, state, offsetDelta) => tokenizer.tokenize(line, state, offsetDelta) + tokenize: (line, state, offsetDelta) => tokenizer.tokenize(line, state, offsetDelta), + tokenize3: () => { + // TODO@tokenization + throw new Error('TODO@tokenization'); + } }; } diff --git a/src/vs/editor/test/common/commands/commandTestUtils.ts b/src/vs/editor/test/common/commands/commandTestUtils.ts index ff4f923d496..0d2af66fd57 100644 --- a/src/vs/editor/test/common/commands/commandTestUtils.ts +++ b/src/vs/editor/test/common/commands/commandTestUtils.ts @@ -10,19 +10,20 @@ import { Range } from 'vs/editor/common/core/range'; import { Selection } from 'vs/editor/common/core/selection'; import * as editorCommon from 'vs/editor/common/editorCommon'; import { Model } from 'vs/editor/common/model/model'; +import { LanguageIdentifier } from 'vs/editor/common/modes'; import { MockConfiguration } from 'vs/editor/test/common/mocks/mockConfiguration'; import { viewModelHelper } from 'vs/editor/test/common/editorTestUtils'; export function testCommand( lines: string[], - mode: string, + languageIdentifier: LanguageIdentifier, selection: Selection, commandFactory: (selection: Selection) => editorCommon.ICommand, expectedLines: string[], expectedSelection: Selection ): void { - let model = Model.createFromString(lines.join('\n'), undefined, mode); + let model = Model.createFromString(lines.join('\n'), undefined, languageIdentifier); let config = new MockConfiguration(null); let cursor = new Cursor(0, config, model, viewModelHelper(model), false); diff --git a/src/vs/editor/test/common/commands/shiftCommand.test.ts b/src/vs/editor/test/common/commands/shiftCommand.test.ts index a85e9a05742..7c1fb500b72 100644 --- a/src/vs/editor/test/common/commands/shiftCommand.test.ts +++ b/src/vs/editor/test/common/commands/shiftCommand.test.ts @@ -13,6 +13,7 @@ import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageCo import { createSingleEditOp, getEditOperation, testCommand } from 'vs/editor/test/common/commands/commandTestUtils'; import { withEditorModel } from 'vs/editor/test/common/editorTestUtils'; import { MockMode } from 'vs/editor/test/common/mocks/mockMode'; +import { LanguageIdentifier } from 'vs/editor/common/modes'; function testShiftCommand(lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection): void { testCommand(lines, null, selection, (sel) => new ShiftCommand(sel, { @@ -32,9 +33,11 @@ function testUnshiftCommand(lines: string[], selection: Selection, expectedLines class DocBlockCommentMode extends MockMode { + private static _id = new LanguageIdentifier('commentMode', 3); + constructor() { - super(); - LanguageConfigurationRegistry.register(this.getId(), { + super(DocBlockCommentMode._id); + this._register(LanguageConfigurationRegistry.register(this.getLanguageIdentifier(), { brackets: [ ['(', ')'], ['{', '}'], @@ -69,24 +72,28 @@ class DocBlockCommentMode extends MockMode { action: { indentAction: IndentAction.None, removeText: 1 } } ] - }); + })); } } function testShiftCommandInDocBlockCommentMode(lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection): void { - testCommand(lines, new DocBlockCommentMode().getId(), selection, (sel) => new ShiftCommand(sel, { + let mode = new DocBlockCommentMode(); + testCommand(lines, mode.getLanguageIdentifier(), selection, (sel) => new ShiftCommand(sel, { isUnshift: false, tabSize: 4, oneIndent: '\t' }), expectedLines, expectedSelection); + mode.dispose(); } function testUnshiftCommandInDocBlockCommentMode(lines: string[], selection: Selection, expectedLines: string[], expectedSelection: Selection): void { - testCommand(lines, new DocBlockCommentMode().getId(), selection, (sel) => new ShiftCommand(sel, { + let mode = new DocBlockCommentMode(); + testCommand(lines, mode.getLanguageIdentifier(), selection, (sel) => new ShiftCommand(sel, { isUnshift: true, tabSize: 4, oneIndent: '\t' }), expectedLines, expectedSelection); + mode.dispose(); } suite('Editor Commands - ShiftCommand', () => { diff --git a/src/vs/editor/test/common/testModes.ts b/src/vs/editor/test/common/commentMode.ts similarity index 74% rename from src/vs/editor/test/common/testModes.ts rename to src/vs/editor/test/common/commentMode.ts index eeda213c85e..232906169e1 100644 --- a/src/vs/editor/test/common/testModes.ts +++ b/src/vs/editor/test/common/commentMode.ts @@ -6,13 +6,16 @@ import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry'; import { CommentRule } from 'vs/editor/common/modes/languageConfiguration'; +import { LanguageIdentifier } from 'vs/editor/common/modes'; import { MockMode } from 'vs/editor/test/common/mocks/mockMode'; export class CommentMode extends MockMode { + private static _id = new LanguageIdentifier('commentMode', 3); + constructor(commentsConfig: CommentRule) { - super(); - LanguageConfigurationRegistry.register(this.getId(), { + super(CommentMode._id); + this._register(LanguageConfigurationRegistry.register(this.getLanguageIdentifier(), { comments: commentsConfig - }); + })); } } diff --git a/src/vs/editor/test/common/controller/cursor.test.ts b/src/vs/editor/test/common/controller/cursor.test.ts index 2dbba9e223c..9ca9b5d1e0b 100644 --- a/src/vs/editor/test/common/controller/cursor.test.ts +++ b/src/vs/editor/test/common/controller/cursor.test.ts @@ -21,6 +21,7 @@ import { IndentAction } from 'vs/editor/common/modes/languageConfiguration'; import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry'; import { MockConfiguration } from 'vs/editor/test/common/mocks/mockConfiguration'; import { MockMode } from 'vs/editor/test/common/mocks/mockMode'; +import { LanguageIdentifier } from 'vs/editor/common/modes'; import { viewModelHelper } from 'vs/editor/test/common/editorTestUtils'; let H = Handler; @@ -1121,25 +1122,31 @@ suite('Editor Controller - Cursor', () => { }); class SurroundingMode extends MockMode { + + private static _id = new LanguageIdentifier('surroundingMode', 3); + constructor() { - super(); - LanguageConfigurationRegistry.register(this.getId(), { + super(SurroundingMode._id); + this._register(LanguageConfigurationRegistry.register(this.getLanguageIdentifier(), { autoClosingPairs: [{ open: '(', close: ')' }] - }); + })); } } class OnEnterMode extends MockMode { + + private static _id = new LanguageIdentifier('onEnterMode', 3); + constructor(indentAction: IndentAction) { - super(); - LanguageConfigurationRegistry.register(this.getId(), { + super(OnEnterMode._id); + this._register(LanguageConfigurationRegistry.register(this.getLanguageIdentifier(), { onEnterRules: [{ beforeText: /.*/, action: { indentAction: indentAction } }] - }); + })); } } @@ -1206,23 +1213,27 @@ suite('Editor Controller - Regression tests', () => { test('issue #183: jump to matching bracket position', () => { class BracketMode extends MockMode { + + private static _id = new LanguageIdentifier('bracketMode', 3); + constructor() { - super(); - LanguageConfigurationRegistry.register(this.getId(), { + super(BracketMode._id); + this._register(LanguageConfigurationRegistry.register(this.getLanguageIdentifier(), { brackets: [ ['{', '}'], ['[', ']'], ['(', ')'], ] - }); + })); } } + let mode = new BracketMode(); usingCursor({ text: [ 'var x = (3 + (5-7));' ], - modeId: new BracketMode().getId() + languageIdentifier: mode.getLanguageIdentifier() }, (model, cursor) => { // ensure is tokenized model.getLineTokens(1, false); @@ -1238,9 +1249,11 @@ suite('Editor Controller - Regression tests', () => { cursorCommand(cursor, H.JumpToBracket, null, 'keyboard'); assertCursor(cursor, new Position(1, 10)); }); + mode.dispose(); }); test('bug #16543: Tab should indent to correct indentation spot immediately', () => { + let mode = new OnEnterMode(IndentAction.Indent); usingCursor({ text: [ 'function baz() {', @@ -1257,7 +1270,7 @@ suite('Editor Controller - Regression tests', () => { tabSize: 4, trimAutoWhitespace: true }, - modeId: new OnEnterMode(IndentAction.Indent).getId(), + languageIdentifier: mode.getLanguageIdentifier(), }, (model, cursor) => { moveTo(cursor, 4, 1, false); assertCursor(cursor, new Selection(4, 1, 4, 1)); @@ -1265,9 +1278,11 @@ suite('Editor Controller - Regression tests', () => { cursorCommand(cursor, H.Tab, null, 'keyboard'); assert.equal(model.getLineContent(4), '\t\t'); }); + mode.dispose(); }); test('bug #2938 (1): When pressing Tab on white-space only lines, indent straight to the right spot (similar to empty lines)', () => { + let mode = new OnEnterMode(IndentAction.Indent); usingCursor({ text: [ '\tfunction baz() {', @@ -1284,7 +1299,7 @@ suite('Editor Controller - Regression tests', () => { tabSize: 4, trimAutoWhitespace: true }, - modeId: new OnEnterMode(IndentAction.Indent).getId(), + languageIdentifier: mode.getLanguageIdentifier(), }, (model, cursor) => { moveTo(cursor, 4, 2, false); assertCursor(cursor, new Selection(4, 2, 4, 2)); @@ -1292,10 +1307,12 @@ suite('Editor Controller - Regression tests', () => { cursorCommand(cursor, H.Tab, null, 'keyboard'); assert.equal(model.getLineContent(4), '\t\t\t'); }); + mode.dispose(); }); test('bug #2938 (2): When pressing Tab on white-space only lines, indent straight to the right spot (similar to empty lines)', () => { + let mode = new OnEnterMode(IndentAction.Indent); usingCursor({ text: [ '\tfunction baz() {', @@ -1312,7 +1329,7 @@ suite('Editor Controller - Regression tests', () => { tabSize: 4, trimAutoWhitespace: true }, - modeId: new OnEnterMode(IndentAction.Indent).getId(), + languageIdentifier: mode.getLanguageIdentifier(), }, (model, cursor) => { moveTo(cursor, 4, 1, false); assertCursor(cursor, new Selection(4, 1, 4, 1)); @@ -1320,9 +1337,11 @@ suite('Editor Controller - Regression tests', () => { cursorCommand(cursor, H.Tab, null, 'keyboard'); assert.equal(model.getLineContent(4), '\t\t\t'); }); + mode.dispose(); }); test('bug #2938 (3): When pressing Tab on white-space only lines, indent straight to the right spot (similar to empty lines)', () => { + let mode = new OnEnterMode(IndentAction.Indent); usingCursor({ text: [ '\tfunction baz() {', @@ -1339,7 +1358,7 @@ suite('Editor Controller - Regression tests', () => { tabSize: 4, trimAutoWhitespace: true }, - modeId: new OnEnterMode(IndentAction.Indent).getId(), + languageIdentifier: mode.getLanguageIdentifier(), }, (model, cursor) => { moveTo(cursor, 4, 3, false); assertCursor(cursor, new Selection(4, 3, 4, 3)); @@ -1347,9 +1366,11 @@ suite('Editor Controller - Regression tests', () => { cursorCommand(cursor, H.Tab, null, 'keyboard'); assert.equal(model.getLineContent(4), '\t\t\t\t'); }); + mode.dispose(); }); test('bug #2938 (4): When pressing Tab on white-space only lines, indent straight to the right spot (similar to empty lines)', () => { + let mode = new OnEnterMode(IndentAction.Indent); usingCursor({ text: [ '\tfunction baz() {', @@ -1366,7 +1387,7 @@ suite('Editor Controller - Regression tests', () => { tabSize: 4, trimAutoWhitespace: true }, - modeId: new OnEnterMode(IndentAction.Indent).getId(), + languageIdentifier: mode.getLanguageIdentifier(), }, (model, cursor) => { moveTo(cursor, 4, 4, false); assertCursor(cursor, new Selection(4, 4, 4, 4)); @@ -1374,6 +1395,7 @@ suite('Editor Controller - Regression tests', () => { cursorCommand(cursor, H.Tab, null, 'keyboard'); assert.equal(model.getLineContent(4), '\t\t\t\t\t'); }); + mode.dispose(); }); test('Bug 18276:[editor] Indentation broken when selection is empty', () => { @@ -1402,11 +1424,12 @@ suite('Editor Controller - Regression tests', () => { }); test('bug #16815:Shift+Tab doesn\'t go back to tabstop', () => { + let mode = new OnEnterMode(IndentAction.IndentOutdent); usingCursor({ text: [ ' function baz() {' ], - modeId: new OnEnterMode(IndentAction.IndentOutdent).getId(), + languageIdentifier: mode.getLanguageIdentifier(), modelOpts: { insertSpaces: true, tabSize: 4, detectIndentation: false, defaultEOL: DefaultEndOfLine.LF, trimAutoWhitespace: true } }, (model, cursor) => { moveTo(cursor, 1, 6, false); @@ -1416,6 +1439,7 @@ suite('Editor Controller - Regression tests', () => { assert.equal(model.getLineContent(1), ' function baz() {'); assertCursor(cursor, new Selection(1, 5, 1, 5)); }); + mode.dispose(); }); test('Bug #18293:[regression][editor] Can\'t outdent whitespace line', () => { @@ -1505,11 +1529,12 @@ suite('Editor Controller - Regression tests', () => { }); test('Bug #11476: Double bracket surrounding + undo is broken', () => { + let mode = new SurroundingMode(); usingCursor({ text: [ 'hello' ], - modeId: new SurroundingMode().getId(), + languageIdentifier: mode.getLanguageIdentifier(), modelOpts: { tabSize: 4, insertSpaces: true, detectIndentation: false, defaultEOL: DefaultEndOfLine.LF, trimAutoWhitespace: true } }, (model, cursor) => { moveTo(cursor, 1, 3, false); @@ -1522,16 +1547,18 @@ suite('Editor Controller - Regression tests', () => { cursorCommand(cursor, H.Type, { text: '(' }, 'keyboard'); assertCursor(cursor, new Selection(1, 5, 1, 7)); }); + mode.dispose(); }); test('issue #1140: Backspace stops prematurely', () => { + let mode = new SurroundingMode(); usingCursor({ text: [ 'function baz() {', ' return 1;', '};' ], - modeId: new SurroundingMode().getId(), + languageIdentifier: mode.getLanguageIdentifier(), modelOpts: { tabSize: 4, insertSpaces: true, detectIndentation: false, defaultEOL: DefaultEndOfLine.LF, trimAutoWhitespace: true } }, (model, cursor) => { moveTo(cursor, 3, 2, false); @@ -1543,6 +1570,7 @@ suite('Editor Controller - Regression tests', () => { assert.equal(model.getLineCount(), 1); assert.equal(model.getLineContent(1), 'function baz(;'); }); + mode.dispose(); }); test('issue #1336: Insert cursor below on last line adds a cursor to the end of the current line', () => { @@ -1598,7 +1626,7 @@ suite('Editor Controller - Regression tests', () => { 'and more lines', 'just some text', ], - modeId: null, + languageIdentifier: null, modelOpts: { insertSpaces: true, tabSize: 4, detectIndentation: false, defaultEOL: DefaultEndOfLine.LF, trimAutoWhitespace: true } }, (model, cursor) => { moveTo(cursor, 1, 1, false); @@ -1640,7 +1668,7 @@ suite('Editor Controller - Regression tests', () => { 'and more lines', 'just some text', ], - modeId: null, + languageIdentifier: null, modelOpts: { insertSpaces: true, tabSize: 4, detectIndentation: false, defaultEOL: DefaultEndOfLine.LF, trimAutoWhitespace: true } }, (model, cursor) => { moveTo(cursor, 3, 1, false); @@ -2119,6 +2147,7 @@ suite('Editor Controller - Regression tests', () => { }); test('issue Microsoft/monaco-editor#108 part 1/2: Auto indentation on Enter with selection is half broken', () => { + let mode = new OnEnterMode(IndentAction.None); usingCursor({ text: [ 'function baz() {', @@ -2133,7 +2162,7 @@ suite('Editor Controller - Regression tests', () => { tabSize: 4, trimAutoWhitespace: true }, - modeId: new OnEnterMode(IndentAction.None).getId(), + languageIdentifier: mode.getLanguageIdentifier(), }, (model, cursor) => { moveTo(cursor, 3, 8, false); moveTo(cursor, 2, 12, true); @@ -2143,9 +2172,11 @@ suite('Editor Controller - Regression tests', () => { assert.equal(model.getLineContent(3), '\treturn x;'); assertCursor(cursor, new Position(3, 2)); }); + mode.dispose(); }); test('issue Microsoft/monaco-editor#108 part 2/2: Auto indentation on Enter with selection is half broken', () => { + let mode = new OnEnterMode(IndentAction.None); usingCursor({ text: [ 'function baz() {', @@ -2160,7 +2191,7 @@ suite('Editor Controller - Regression tests', () => { tabSize: 4, trimAutoWhitespace: true }, - modeId: new OnEnterMode(IndentAction.None).getId(), + languageIdentifier: mode.getLanguageIdentifier(), }, (model, cursor) => { moveTo(cursor, 2, 12, false); moveTo(cursor, 3, 8, true); @@ -2170,6 +2201,7 @@ suite('Editor Controller - Regression tests', () => { assert.equal(model.getLineContent(3), '\treturn x;'); assertCursor(cursor, new Position(3, 2)); }); + mode.dispose(); }); test('issue #9675: Undo/Redo adds a stop in between CHN Characters', () => { @@ -2290,11 +2322,12 @@ suite('Editor Controller - Cursor Configuration', () => { }); test('Enter auto-indents with insertSpaces setting 1', () => { + let mode = new OnEnterMode(IndentAction.Indent); usingCursor({ text: [ '\thello' ], - modeId: new OnEnterMode(IndentAction.Indent).getId(), + languageIdentifier: mode.getLanguageIdentifier(), modelOpts: { insertSpaces: true, tabSize: 4, detectIndentation: false, defaultEOL: DefaultEndOfLine.LF, trimAutoWhitespace: true } }, (model, cursor) => { moveTo(cursor, 1, 7, false); @@ -2303,14 +2336,16 @@ suite('Editor Controller - Cursor Configuration', () => { cursorCommand(cursor, H.Type, { text: '\n' }, 'keyboard'); assert.equal(model.getValue(EndOfLinePreference.CRLF), '\thello\r\n '); }); + mode.dispose(); }); test('Enter auto-indents with insertSpaces setting 2', () => { + let mode = new OnEnterMode(IndentAction.None); usingCursor({ text: [ '\thello' ], - modeId: new OnEnterMode(IndentAction.None).getId(), + languageIdentifier: mode.getLanguageIdentifier(), modelOpts: { insertSpaces: true, tabSize: 4, detectIndentation: false, defaultEOL: DefaultEndOfLine.LF, trimAutoWhitespace: true } }, (model, cursor) => { moveTo(cursor, 1, 7, false); @@ -2319,14 +2354,16 @@ suite('Editor Controller - Cursor Configuration', () => { cursorCommand(cursor, H.Type, { text: '\n' }, 'keyboard'); assert.equal(model.getValue(EndOfLinePreference.CRLF), '\thello\r\n '); }); + mode.dispose(); }); test('Enter auto-indents with insertSpaces setting 3', () => { + let mode = new OnEnterMode(IndentAction.IndentOutdent); usingCursor({ text: [ '\thell()' ], - modeId: new OnEnterMode(IndentAction.IndentOutdent).getId(), + languageIdentifier: mode.getLanguageIdentifier(), modelOpts: { insertSpaces: true, tabSize: 4, detectIndentation: false, defaultEOL: DefaultEndOfLine.LF, trimAutoWhitespace: true } }, (model, cursor) => { moveTo(cursor, 1, 7, false); @@ -2335,6 +2372,7 @@ suite('Editor Controller - Cursor Configuration', () => { cursorCommand(cursor, H.Type, { text: '\n' }, 'keyboard'); assert.equal(model.getValue(EndOfLinePreference.CRLF), '\thell(\r\n \r\n )'); }); + mode.dispose(); }); test('Insert line before', () => { @@ -2475,6 +2513,7 @@ suite('Editor Controller - Cursor Configuration', () => { }); test('issue #6862: Editor removes auto inserted indentation when formatting on type', () => { + let mode = new OnEnterMode(IndentAction.IndentOutdent); usingCursor({ text: [ 'function foo (params: string) {}' @@ -2486,7 +2525,7 @@ suite('Editor Controller - Cursor Configuration', () => { defaultEOL: DefaultEndOfLine.LF, trimAutoWhitespace: true }, - modeId: new OnEnterMode(IndentAction.IndentOutdent).getId(), + languageIdentifier: mode.getLanguageIdentifier(), }, (model, cursor) => { moveTo(cursor, 1, 32); @@ -2515,6 +2554,7 @@ suite('Editor Controller - Cursor Configuration', () => { assert.equal(model.getLineContent(2), ' '); assert.equal(model.getLineContent(3), '}'); }); + mode.dispose(); }); test('removeAutoWhitespace on: removes only whitespace the cursor added 2', () => { @@ -2787,13 +2827,13 @@ suite('Editor Controller - Cursor Configuration', () => { interface ICursorOpts { text: string[]; - modeId?: string; + languageIdentifier?: LanguageIdentifier; modelOpts?: ITextModelCreationOptions; editorOpts?: IEditorOptions; } function usingCursor(opts: ICursorOpts, callback: (model: Model, cursor: Cursor) => void): void { - let model = Model.createFromString(opts.text.join('\n'), opts.modelOpts, opts.modeId); + let model = Model.createFromString(opts.text.join('\n'), opts.modelOpts, opts.languageIdentifier); let config = new MockConfiguration(opts.editorOpts); let cursor = new Cursor(1, config, model, viewModelHelper(model), false); @@ -2805,9 +2845,12 @@ function usingCursor(opts: ICursorOpts, callback: (model: Model, cursor: Cursor) } class ElectricCharMode extends MockMode { + + private static _id = new LanguageIdentifier('electricCharMode', 3); + constructor() { - super(); - LanguageConfigurationRegistry.register(this.getId(), { + super(ElectricCharMode._id); + this._register(LanguageConfigurationRegistry.register(this.getLanguageIdentifier(), { __electricCharacterSupport: { docComment: { open: '/**', close: ' */' } }, @@ -2816,54 +2859,61 @@ class ElectricCharMode extends MockMode { ['[', ']'], ['(', ')'] ] - }); + })); } } suite('ElectricCharacter', () => { test('does nothing if no electric char', () => { + let mode = new ElectricCharMode(); usingCursor({ text: [ ' if (a) {', '' ], - modeId: new ElectricCharMode().getId() + languageIdentifier: mode.getLanguageIdentifier() }, (model, cursor) => { moveTo(cursor, 2, 1); cursorCommand(cursor, H.Type, { text: '*' }, 'keyboard'); assert.deepEqual(model.getLineContent(2), '*'); }); + mode.dispose(); }); test('indents in order to match bracket', () => { + let mode = new ElectricCharMode(); usingCursor({ text: [ ' if (a) {', '' ], - modeId: new ElectricCharMode().getId() + languageIdentifier: mode.getLanguageIdentifier() }, (model, cursor) => { moveTo(cursor, 2, 1); cursorCommand(cursor, H.Type, { text: '}' }, 'keyboard'); assert.deepEqual(model.getLineContent(2), ' }'); }); + mode.dispose(); }); test('unindents in order to match bracket', () => { + let mode = new ElectricCharMode(); usingCursor({ text: [ ' if (a) {', ' ' ], - modeId: new ElectricCharMode().getId() + languageIdentifier: mode.getLanguageIdentifier() }, (model, cursor) => { moveTo(cursor, 2, 5); cursorCommand(cursor, H.Type, { text: '}' }, 'keyboard'); assert.deepEqual(model.getLineContent(2), ' }'); }); + mode.dispose(); }); test('matches with correct bracket', () => { + let mode = new ElectricCharMode(); usingCursor({ text: [ ' if (a) {', @@ -2871,15 +2921,17 @@ suite('ElectricCharacter', () => { ' }', ' ' ], - modeId: new ElectricCharMode().getId() + languageIdentifier: mode.getLanguageIdentifier() }, (model, cursor) => { moveTo(cursor, 4, 1); cursorCommand(cursor, H.Type, { text: '}' }, 'keyboard'); assert.deepEqual(model.getLineContent(4), ' } '); }); + mode.dispose(); }); test('does nothing if bracket does not match', () => { + let mode = new ElectricCharMode(); usingCursor({ text: [ ' if (a) {', @@ -2887,81 +2939,92 @@ suite('ElectricCharacter', () => { ' }', ' } ' ], - modeId: new ElectricCharMode().getId() + languageIdentifier: mode.getLanguageIdentifier() }, (model, cursor) => { moveTo(cursor, 4, 6); cursorCommand(cursor, H.Type, { text: '}' }, 'keyboard'); assert.deepEqual(model.getLineContent(4), ' } }'); }); + mode.dispose(); }); test('matches bracket even in line with content', () => { + let mode = new ElectricCharMode(); usingCursor({ text: [ ' if (a) {', '// hello' ], - modeId: new ElectricCharMode().getId() + languageIdentifier: mode.getLanguageIdentifier() }, (model, cursor) => { moveTo(cursor, 2, 1); cursorCommand(cursor, H.Type, { text: '}' }, 'keyboard'); assert.deepEqual(model.getLineContent(2), ' }// hello'); }); + mode.dispose(); }); test('is no-op if bracket is lined up', () => { + let mode = new ElectricCharMode(); usingCursor({ text: [ ' if (a) {', ' ' ], - modeId: new ElectricCharMode().getId() + languageIdentifier: mode.getLanguageIdentifier() }, (model, cursor) => { moveTo(cursor, 2, 3); cursorCommand(cursor, H.Type, { text: '}' }, 'keyboard'); assert.deepEqual(model.getLineContent(2), ' }'); }); + mode.dispose(); }); test('is no-op if there is non-whitespace text before', () => { + let mode = new ElectricCharMode(); usingCursor({ text: [ ' if (a) {', 'a' ], - modeId: new ElectricCharMode().getId() + languageIdentifier: mode.getLanguageIdentifier() }, (model, cursor) => { moveTo(cursor, 2, 2); cursorCommand(cursor, H.Type, { text: '}' }, 'keyboard'); assert.deepEqual(model.getLineContent(2), ' a}'); }); + mode.dispose(); }); test('appends text', () => { + let mode = new ElectricCharMode(); usingCursor({ text: [ ' if (a) {', '/*' ], - modeId: new ElectricCharMode().getId() + languageIdentifier: mode.getLanguageIdentifier() }, (model, cursor) => { moveTo(cursor, 2, 3); cursorCommand(cursor, H.Type, { text: '*' }, 'keyboard'); assert.deepEqual(model.getLineContent(2), '/** */'); }); + mode.dispose(); }); test('appends text 2', () => { + let mode = new ElectricCharMode(); usingCursor({ text: [ ' if (a) {', ' /*' ], - modeId: new ElectricCharMode().getId() + languageIdentifier: mode.getLanguageIdentifier() }, (model, cursor) => { moveTo(cursor, 2, 5); cursorCommand(cursor, H.Type, { text: '*' }, 'keyboard'); assert.deepEqual(model.getLineContent(2), ' /** */'); }); + mode.dispose(); }); }); diff --git a/src/vs/editor/test/common/mocks/mockMode.ts b/src/vs/editor/test/common/mocks/mockMode.ts index cca5ca0478d..9c2d6d1c10d 100644 --- a/src/vs/editor/test/common/mocks/mockMode.ts +++ b/src/vs/editor/test/common/mocks/mockMode.ts @@ -4,24 +4,22 @@ *--------------------------------------------------------------------------------------------*/ 'use strict'; -import { IMode } from 'vs/editor/common/modes'; +import { Disposable } from 'vs/base/common/lifecycle'; +import { IMode, LanguageIdentifier } from 'vs/editor/common/modes'; -let instanceCount = 0; -function generateMockModeId(): string { - return 'mockMode' + (++instanceCount); -} +export class MockMode extends Disposable implements IMode { + private _languageIdentifier: LanguageIdentifier; -export class MockMode implements IMode { - private _id: string; - - constructor(id?: string) { - if (typeof id === 'undefined') { - id = generateMockModeId(); - } - this._id = id; + constructor(languageIdentifier: LanguageIdentifier) { + super(); + this._languageIdentifier = languageIdentifier; } public getId(): string { - return this._id; + return this._languageIdentifier.sid; + } + + public getLanguageIdentifier(): LanguageIdentifier { + return this._languageIdentifier; } } diff --git a/src/vs/editor/test/common/model/model.line.test.ts b/src/vs/editor/test/common/model/model.line.test.ts index dd0957e3889..0210b17139e 100644 --- a/src/vs/editor/test/common/model/model.line.test.ts +++ b/src/vs/editor/test/common/model/model.line.test.ts @@ -7,13 +7,17 @@ import * as assert from 'assert'; import { LineTokens } from 'vs/editor/common/core/lineTokens'; import { ModelLine, ILineEdit, LineMarker, MarkersTracker } from 'vs/editor/common/model/modelLine'; -import { TokensInflatorMap } from 'vs/editor/common/model/tokensBinaryEncoding'; -import { Token } from 'vs/editor/common/core/token'; +import { MetadataConsts } from 'vs/editor/common/modes'; import { Position } from 'vs/editor/common/core/position'; -function assertLineTokens(actual: LineTokens, expected: Token[]): void { - var inflatedActual = actual.inflate(); - assert.deepEqual(inflatedActual, expected, 'Line tokens are equal'); +function assertLineTokens(actual: LineTokens, expected: TestToken[]): void { + let inflatedActual = actual.inflate(); + assert.deepEqual(inflatedActual, expected.map((token) => { + return { + startIndex: token.startOffset, + type: 'mtk' + token.color + }; + }), 'Line tokens are equal'); } const NO_TAB_SIZE = 0; @@ -261,37 +265,63 @@ suite('Editor Model - modelLine.append text', () => { }); }); +class TestToken { + public readonly startOffset: number; + public readonly color: number; + + constructor(startOffset: number, color: number) { + this.startOffset = startOffset; + this.color = color; + } + + public static toTokens(tokens: TestToken[]): Uint32Array { + if (tokens === null) { + return null; + } + let tokensLen = tokens.length; + let result = new Uint32Array((tokensLen << 1)); + for (let i = 0; i < tokensLen; i++) { + let token = tokens[i]; + result[(i << 1)] = token.startOffset; + result[(i << 1) + 1] = ( + token.color << MetadataConsts.FOREGROUND_OFFSET + ) >>> 0; + } + return result; + } +} + suite('Editor Model - modelLine.applyEdits text & tokens', () => { - function testLineEditTokens(initialText: string, initialTokens: Token[], edits: ILineEdit[], expectedText: string, expectedTokens: Token[]): void { + + + function testLineEditTokens(initialText: string, initialTokens: TestToken[], edits: ILineEdit[], expectedText: string, expectedTokens: TestToken[]): void { let line = new ModelLine(1, initialText, NO_TAB_SIZE); - let map = new TokensInflatorMap(null); - line.setTokens(map, initialTokens, []); + line.setTokens(0, TestToken.toTokens(initialTokens)); line.applyEdits(new MarkersTracker(), edits, NO_TAB_SIZE); assert.equal(line.text, expectedText); - assertLineTokens(line.getTokens(map), expectedTokens); + assertLineTokens(line.getTokens(0, []), expectedTokens); } test('insertion on empty line', () => { let line = new ModelLine(1, 'some text', NO_TAB_SIZE); - let map = new TokensInflatorMap(null); - line.setTokens(map, [new Token(0, 'bar')], []); + line.setTokens(0, TestToken.toTokens([new TestToken(0, 1)])); line.applyEdits(new MarkersTracker(), [{ startColumn: 1, endColumn: 10, text: '', forceMoveMarkers: false }], NO_TAB_SIZE); - line.setTokens(map, [], []); + line.setTokens(0, new Uint32Array(0)); line.applyEdits(new MarkersTracker(), [{ startColumn: 1, endColumn: 1, text: 'a', forceMoveMarkers: false }], NO_TAB_SIZE); - assertLineTokens(line.getTokens(map), [new Token(0, '')]); + assertLineTokens(line.getTokens(0, []), [new TestToken(0, 1)]); }); test('updates tokens on insertion 1', () => { testLineEditTokens( 'abcd efgh', [ - new Token(0, '1'), - new Token(4, '2'), - new Token(5, '3') + new TestToken(0, 1), + new TestToken(4, 2), + new TestToken(5, 3) ], [{ startColumn: 1, @@ -301,9 +331,9 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => { }], 'aabcd efgh', [ - new Token(0, '1'), - new Token(5, '2'), - new Token(6, '3') + new TestToken(0, 1), + new TestToken(5, 2), + new TestToken(6, 3) ] ); }); @@ -312,9 +342,9 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => { testLineEditTokens( 'aabcd efgh', [ - new Token(0, '1'), - new Token(5, '2'), - new Token(6, '3') + new TestToken(0, 1), + new TestToken(5, 2), + new TestToken(6, 3) ], [{ startColumn: 2, @@ -324,9 +354,9 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => { }], 'axabcd efgh', [ - new Token(0, '1'), - new Token(6, '2'), - new Token(7, '3') + new TestToken(0, 1), + new TestToken(6, 2), + new TestToken(7, 3) ] ); }); @@ -335,9 +365,9 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => { testLineEditTokens( 'axabcd efgh', [ - new Token(0, '1'), - new Token(6, '2'), - new Token(7, '3') + new TestToken(0, 1), + new TestToken(6, 2), + new TestToken(7, 3) ], [{ startColumn: 3, @@ -347,9 +377,9 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => { }], 'axstuabcd efgh', [ - new Token(0, '1'), - new Token(9, '2'), - new Token(10, '3') + new TestToken(0, 1), + new TestToken(9, 2), + new TestToken(10, 3) ] ); }); @@ -358,9 +388,9 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => { testLineEditTokens( 'axstuabcd efgh', [ - new Token(0, '1'), - new Token(9, '2'), - new Token(10, '3') + new TestToken(0, 1), + new TestToken(9, 2), + new TestToken(10, 3) ], [{ startColumn: 10, @@ -370,9 +400,9 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => { }], 'axstuabcd\t efgh', [ - new Token(0, '1'), - new Token(10, '2'), - new Token(11, '3') + new TestToken(0, 1), + new TestToken(10, 2), + new TestToken(11, 3) ] ); }); @@ -381,9 +411,9 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => { testLineEditTokens( 'axstuabcd\t efgh', [ - new Token(0, '1'), - new Token(10, '2'), - new Token(11, '3') + new TestToken(0, 1), + new TestToken(10, 2), + new TestToken(11, 3) ], [{ startColumn: 12, @@ -393,9 +423,9 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => { }], 'axstuabcd\t ddefgh', [ - new Token(0, '1'), - new Token(10, '2'), - new Token(13, '3') + new TestToken(0, 1), + new TestToken(10, 2), + new TestToken(13, 3) ] ); }); @@ -404,9 +434,9 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => { testLineEditTokens( 'axstuabcd\t ddefgh', [ - new Token(0, '1'), - new Token(10, '2'), - new Token(13, '3') + new TestToken(0, 1), + new TestToken(10, 2), + new TestToken(13, 3) ], [{ startColumn: 18, @@ -416,9 +446,9 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => { }], 'axstuabcd\t ddefghxyz', [ - new Token(0, '1'), - new Token(10, '2'), - new Token(13, '3') + new TestToken(0, 1), + new TestToken(10, 2), + new TestToken(13, 3) ] ); }); @@ -427,9 +457,9 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => { testLineEditTokens( 'axstuabcd\t ddefghxyz', [ - new Token(0, '1'), - new Token(10, '2'), - new Token(13, '3') + new TestToken(0, 1), + new TestToken(10, 2), + new TestToken(13, 3) ], [{ startColumn: 1, @@ -439,9 +469,9 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => { }], 'xaxstuabcd\t ddefghxyz', [ - new Token(0, '1'), - new Token(11, '2'), - new Token(14, '3') + new TestToken(0, 1), + new TestToken(11, 2), + new TestToken(14, 3) ] ); }); @@ -450,9 +480,9 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => { testLineEditTokens( 'xaxstuabcd\t ddefghxyz', [ - new Token(0, '1'), - new Token(11, '2'), - new Token(14, '3') + new TestToken(0, 1), + new TestToken(11, 2), + new TestToken(14, 3) ], [{ startColumn: 22, @@ -462,9 +492,9 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => { }], 'xaxstuabcd\t ddefghxyzx', [ - new Token(0, '1'), - new Token(11, '2'), - new Token(14, '3') + new TestToken(0, 1), + new TestToken(11, 2), + new TestToken(14, 3) ] ); }); @@ -473,9 +503,9 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => { testLineEditTokens( 'xaxstuabcd\t ddefghxyzx', [ - new Token(0, '1'), - new Token(11, '2'), - new Token(14, '3') + new TestToken(0, 1), + new TestToken(11, 2), + new TestToken(14, 3) ], [{ startColumn: 2, @@ -485,9 +515,9 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => { }], 'xaxstuabcd\t ddefghxyzx', [ - new Token(0, '1'), - new Token(11, '2'), - new Token(14, '3') + new TestToken(0, 1), + new TestToken(11, 2), + new TestToken(14, 3) ] ); }); @@ -504,7 +534,7 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => { }], 'a', [ - new Token(0, '') + new TestToken(0, 1) ] ); }); @@ -513,9 +543,9 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => { testLineEditTokens( 'abcdefghij', [ - new Token(0, '1'), - new Token(3, '2'), - new Token(6, '3') + new TestToken(0, 1), + new TestToken(3, 2), + new TestToken(6, 3) ], [{ startColumn: 4, @@ -525,8 +555,8 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => { }], 'abcghij', [ - new Token(0, '1'), - new Token(3, '3') + new TestToken(0, 1), + new TestToken(3, 3) ] ); }); @@ -535,9 +565,9 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => { testLineEditTokens( 'abcdefghij', [ - new Token(0, '1'), - new Token(3, '2'), - new Token(6, '3') + new TestToken(0, 1), + new TestToken(3, 2), + new TestToken(6, 3) ], [{ startColumn: 4, @@ -547,9 +577,9 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => { }], 'abchellodefghij', [ - new Token(0, '1'), - new Token(8, '2'), - new Token(11, '3') + new TestToken(0, 1), + new TestToken(8, 2), + new TestToken(11, 3) ] ); }); @@ -558,9 +588,9 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => { testLineEditTokens( 'abcd efgh', [ - new Token(0, '1'), - new Token(4, '2'), - new Token(5, '3') + new TestToken(0, 1), + new TestToken(4, 2), + new TestToken(5, 3) ], [{ startColumn: 1, @@ -570,9 +600,9 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => { }], 'bcd efgh', [ - new Token(0, '1'), - new Token(3, '2'), - new Token(4, '3') + new TestToken(0, 1), + new TestToken(3, 2), + new TestToken(4, 3) ] ); }); @@ -581,9 +611,9 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => { testLineEditTokens( 'abcd efgh', [ - new Token(0, '1'), - new Token(4, '2'), - new Token(5, '3') + new TestToken(0, 1), + new TestToken(4, 2), + new TestToken(5, 3) ], [{ startColumn: 2, @@ -593,9 +623,9 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => { }], 'ad efgh', [ - new Token(0, '1'), - new Token(2, '2'), - new Token(3, '3') + new TestToken(0, 1), + new TestToken(2, 2), + new TestToken(3, 3) ] ); }); @@ -604,9 +634,9 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => { testLineEditTokens( 'abcd efgh', [ - new Token(0, '1'), - new Token(4, '2'), - new Token(5, '3') + new TestToken(0, 1), + new TestToken(4, 2), + new TestToken(5, 3) ], [{ startColumn: 1, @@ -616,8 +646,8 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => { }], ' efgh', [ - new Token(0, '2'), - new Token(1, '3') + new TestToken(0, 2), + new TestToken(1, 3) ] ); }); @@ -626,9 +656,9 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => { testLineEditTokens( 'abcd efgh', [ - new Token(0, '1'), - new Token(4, '2'), - new Token(5, '3') + new TestToken(0, 1), + new TestToken(4, 2), + new TestToken(5, 3) ], [{ startColumn: 5, @@ -638,8 +668,8 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => { }], 'abcdefgh', [ - new Token(0, '1'), - new Token(4, '3') + new TestToken(0, 1), + new TestToken(4, 3) ] ); }); @@ -648,9 +678,9 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => { testLineEditTokens( 'abcd efgh', [ - new Token(0, '1'), - new Token(4, '2'), - new Token(5, '3') + new TestToken(0, 1), + new TestToken(4, 2), + new TestToken(5, 3) ], [{ startColumn: 5, @@ -660,8 +690,8 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => { }], 'abcdfgh', [ - new Token(0, '1'), - new Token(4, '3') + new TestToken(0, 1), + new TestToken(4, 3) ] ); }); @@ -670,9 +700,9 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => { testLineEditTokens( 'abcd efgh', [ - new Token(0, '1'), - new Token(4, '2'), - new Token(5, '3') + new TestToken(0, 1), + new TestToken(4, 2), + new TestToken(5, 3) ], [{ startColumn: 5, @@ -682,7 +712,7 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => { }], 'abcd', [ - new Token(0, '1') + new TestToken(0, 1) ] ); }); @@ -691,9 +721,9 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => { testLineEditTokens( 'abcd efgh', [ - new Token(0, '1'), - new Token(4, '2'), - new Token(5, '3') + new TestToken(0, 1), + new TestToken(4, 2), + new TestToken(5, 3) ], [{ startColumn: 1, @@ -702,7 +732,9 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => { forceMoveMarkers: false }], '', - [] + [ + new TestToken(0, 3) + ] ); }); @@ -710,9 +742,9 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => { testLineEditTokens( 'abcd efgh', [ - new Token(0, '1'), - new Token(4, '2'), - new Token(5, '3') + new TestToken(0, 1), + new TestToken(4, 2), + new TestToken(5, 3) ], [{ startColumn: 1, @@ -722,9 +754,9 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => { }], 'abcd efgh', [ - new Token(0, '1'), - new Token(4, '2'), - new Token(5, '3') + new TestToken(0, 1), + new TestToken(4, 2), + new TestToken(5, 3) ] ); }); @@ -733,9 +765,9 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => { testLineEditTokens( 'abcd efgh', [ - new Token(0, '1'), - new Token(4, '2'), - new Token(5, '3') + new TestToken(0, 1), + new TestToken(4, 2), + new TestToken(5, 3) ], [{ startColumn: 1, @@ -745,9 +777,9 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => { }], 'cd efgh', [ - new Token(0, '1'), - new Token(2, '2'), - new Token(3, '3') + new TestToken(0, 1), + new TestToken(2, 2), + new TestToken(3, 3) ] ); }); @@ -756,9 +788,9 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => { testLineEditTokens( 'abcd efgh', [ - new Token(0, '1'), - new Token(4, '2'), - new Token(5, '3') + new TestToken(0, 1), + new TestToken(4, 2), + new TestToken(5, 3) ], [{ startColumn: 5, @@ -768,7 +800,7 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => { }], 'abcd', [ - new Token(0, '1') + new TestToken(0, 1) ] ); }); @@ -777,11 +809,11 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => { testLineEditTokens( 'Hello world, ciao', [ - new Token(0, 'hello'), - new Token(5, ''), - new Token(6, 'world'), - new Token(11, ''), - new Token(13, '') + new TestToken(0, 1), + new TestToken(5, 0), + new TestToken(6, 2), + new TestToken(11, 0), + new TestToken(13, 0) ], [{ startColumn: 1, @@ -791,11 +823,11 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => { }], 'Hi world, ciao', [ - new Token(0, 'hello'), - new Token(2, ''), - new Token(3, 'world'), - new Token(8, ''), - new Token(10, ''), + new TestToken(0, 1), + new TestToken(2, 0), + new TestToken(3, 2), + new TestToken(8, 0), + new TestToken(10, 0), ] ); }); @@ -804,11 +836,11 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => { testLineEditTokens( 'Hello world, ciao', [ - new Token(0, 'hello'), - new Token(5, ''), - new Token(6, 'world'), - new Token(11, ''), - new Token(13, ''), + new TestToken(0, 1), + new TestToken(5, 0), + new TestToken(6, 2), + new TestToken(11, 0), + new TestToken(13, 0), ], [{ startColumn: 1, @@ -823,41 +855,42 @@ suite('Editor Model - modelLine.applyEdits text & tokens', () => { }], 'Hi wmy friends, ciao', [ - new Token(0, 'hello'), - new Token(2, ''), - new Token(3, 'world'), - new Token(14, ''), - new Token(16, ''), + new TestToken(0, 1), + new TestToken(2, 0), + new TestToken(3, 2), + new TestToken(14, 0), + new TestToken(16, 0), ] ); }); }); suite('Editor Model - modelLine.split text & tokens', () => { - function testLineSplitTokens(initialText: string, initialTokens: Token[], splitColumn: number, expectedText1: string, expectedText2: string, expectedTokens: Token[]): void { + function testLineSplitTokens(initialText: string, initialTokens: TestToken[], splitColumn: number, expectedText1: string, expectedText2: string, expectedTokens: TestToken[]): void { let line = new ModelLine(1, initialText, NO_TAB_SIZE); - let map = new TokensInflatorMap(null); - line.setTokens(map, initialTokens, []); + line.setTokens(0, TestToken.toTokens(initialTokens)); let other = line.split(new MarkersTracker(), splitColumn, false, NO_TAB_SIZE); assert.equal(line.text, expectedText1); assert.equal(other.text, expectedText2); - assertLineTokens(line.getTokens(map), expectedTokens); + assertLineTokens(line.getTokens(0, []), expectedTokens); } test('split at the beginning', () => { testLineSplitTokens( 'abcd efgh', [ - new Token(0, '1'), - new Token(4, '2'), - new Token(5, '3') + new TestToken(0, 1), + new TestToken(4, 2), + new TestToken(5, 3) ], 1, '', 'abcd efgh', - [] + [ + new TestToken(0, 1), + ] ); }); @@ -865,17 +898,17 @@ suite('Editor Model - modelLine.split text & tokens', () => { testLineSplitTokens( 'abcd efgh', [ - new Token(0, '1'), - new Token(4, '2'), - new Token(5, '3') + new TestToken(0, 1), + new TestToken(4, 2), + new TestToken(5, 3) ], 10, 'abcd efgh', '', [ - new Token(0, '1'), - new Token(4, '2'), - new Token(5, '3') + new TestToken(0, 1), + new TestToken(4, 2), + new TestToken(5, 3) ] ); }); @@ -884,15 +917,15 @@ suite('Editor Model - modelLine.split text & tokens', () => { testLineSplitTokens( 'abcd efgh', [ - new Token(0, '1'), - new Token(4, '2'), - new Token(5, '3') + new TestToken(0, 1), + new TestToken(4, 2), + new TestToken(5, 3) ], 5, 'abcd', ' efgh', [ - new Token(0, '1') + new TestToken(0, 1) ] ); }); @@ -901,52 +934,50 @@ suite('Editor Model - modelLine.split text & tokens', () => { testLineSplitTokens( 'abcd efgh', [ - new Token(0, '1'), - new Token(4, '2'), - new Token(5, '3') + new TestToken(0, 1), + new TestToken(4, 2), + new TestToken(5, 3) ], 6, 'abcd ', 'efgh', [ - new Token(0, '1'), - new Token(4, '2') + new TestToken(0, 1), + new TestToken(4, 2) ] ); }); }); suite('Editor Model - modelLine.append text & tokens', () => { - function testLineAppendTokens(aText: string, aTokens: Token[], bText: string, bTokens: Token[], expectedText: string, expectedTokens: Token[]): void { - let inflator = new TokensInflatorMap(null); - + function testLineAppendTokens(aText: string, aTokens: TestToken[], bText: string, bTokens: TestToken[], expectedText: string, expectedTokens: TestToken[]): void { let a = new ModelLine(1, aText, NO_TAB_SIZE); - a.setTokens(inflator, aTokens, []); + a.setTokens(0, TestToken.toTokens(aTokens)); let b = new ModelLine(2, bText, NO_TAB_SIZE); - b.setTokens(inflator, bTokens, []); + b.setTokens(0, TestToken.toTokens(bTokens)); a.append(new MarkersTracker(), b, NO_TAB_SIZE); assert.equal(a.text, expectedText); - assertLineTokens(a.getTokens(inflator), expectedTokens); + assertLineTokens(a.getTokens(0, []), expectedTokens); } test('append empty 1', () => { testLineAppendTokens( 'abcd efgh', [ - new Token(0, '1'), - new Token(4, '2'), - new Token(5, '3') + new TestToken(0, 1), + new TestToken(4, 2), + new TestToken(5, 3) ], '', [], 'abcd efgh', [ - new Token(0, '1'), - new Token(4, '2'), - new Token(5, '3') + new TestToken(0, 1), + new TestToken(4, 2), + new TestToken(5, 3) ] ); }); @@ -957,15 +988,15 @@ suite('Editor Model - modelLine.append text & tokens', () => { [], 'abcd efgh', [ - new Token(0, '1'), - new Token(4, '2'), - new Token(5, '3') + new TestToken(0, 1), + new TestToken(4, 2), + new TestToken(5, 3) ], 'abcd efgh', [ - new Token(0, '1'), - new Token(4, '2'), - new Token(5, '3') + new TestToken(0, 1), + new TestToken(4, 2), + new TestToken(5, 3) ] ); }); @@ -974,24 +1005,24 @@ suite('Editor Model - modelLine.append text & tokens', () => { testLineAppendTokens( 'abcd efgh', [ - new Token(0, '1'), - new Token(4, '2'), - new Token(5, '3') + new TestToken(0, 1), + new TestToken(4, 2), + new TestToken(5, 3) ], 'abcd efgh', [ - new Token(0, '4'), - new Token(4, '5'), - new Token(5, '6') + new TestToken(0, 4), + new TestToken(4, 5), + new TestToken(5, 6) ], 'abcd efghabcd efgh', [ - new Token(0, '1'), - new Token(4, '2'), - new Token(5, '3'), - new Token(9, '4'), - new Token(13, '5'), - new Token(14, '6') + new TestToken(0, 1), + new TestToken(4, 2), + new TestToken(5, 3), + new TestToken(9, 4), + new TestToken(13, 5), + new TestToken(14, 6) ] ); }); @@ -1000,18 +1031,18 @@ suite('Editor Model - modelLine.append text & tokens', () => { testLineAppendTokens( 'abcd ', [ - new Token(0, '1'), - new Token(4, '2') + new TestToken(0, 1), + new TestToken(4, 2) ], 'efgh', [ - new Token(0, '3') + new TestToken(0, 3) ], 'abcd efgh', [ - new Token(0, '1'), - new Token(4, '2'), - new Token(5, '3') + new TestToken(0, 1), + new TestToken(4, 2), + new TestToken(5, 3) ] ); }); @@ -1020,18 +1051,18 @@ suite('Editor Model - modelLine.append text & tokens', () => { testLineAppendTokens( 'abcd', [ - new Token(0, '1'), + new TestToken(0, 1), ], ' efgh', [ - new Token(0, '2'), - new Token(1, '3') + new TestToken(0, 2), + new TestToken(1, 3) ], 'abcd efgh', [ - new Token(0, '1'), - new Token(4, '2'), - new Token(5, '3') + new TestToken(0, 1), + new TestToken(4, 2), + new TestToken(5, 3) ] ); }); diff --git a/src/vs/editor/test/common/model/model.modes.test.ts b/src/vs/editor/test/common/model/model.modes.test.ts index d00e81f627d..fe43a52a6e6 100644 --- a/src/vs/editor/test/common/model/model.modes.test.ts +++ b/src/vs/editor/test/common/model/model.modes.test.ts @@ -5,161 +5,160 @@ 'use strict'; import * as assert from 'assert'; +import { IDisposable } from 'vs/base/common/lifecycle'; import { EditOperation } from 'vs/editor/common/core/editOperation'; import { Position } from 'vs/editor/common/core/position'; import { Range } from 'vs/editor/common/core/range'; import { Model } from 'vs/editor/common/model/model'; import * as modes from 'vs/editor/common/modes'; -import { Token } from 'vs/editor/common/core/token'; +import { NULL_STATE } from 'vs/editor/common/modes/nullMode'; // --------- utils suite('Editor Model - Model Modes 1', () => { - const LANGUAGE_ID = 'modelModeTest1'; + let calledFor: string[] = []; - let calledState = { - calledFor: [] - }; - let thisModel: Model; - - function checkAndClear(calledState: { calledFor: string[] }, arr: string[]) { - assert.deepEqual(calledState.calledFor, arr); - calledState.calledFor = []; + function checkAndClear(arr: string[]) { + assert.deepEqual(calledFor, arr); + calledFor = []; } - class ModelState1 implements modes.IState { - clone(): modes.IState { return this; } - equals(other: modes.IState): boolean { return this === other; } - } - - modes.TokenizationRegistry.register(LANGUAGE_ID, { - getInitialState: () => new ModelState1(), - tokenize: (line: string, state: modes.IState): modes.ILineTokens => { - calledState.calledFor.push(line.charAt(0)); + const tokenizationSupport: modes.ITokenizationSupport = { + getInitialState: () => NULL_STATE, + tokenize: undefined, + tokenize3: (line: string, state: modes.IState): modes.ILineTokens3 => { + calledFor.push(line.charAt(0)); return { - tokens: [new Token(0, '')], - endState: state, - modeTransitions: null + tokens: null, + endState: state }; } - }); + }; + + let thisModel: Model = null; + let languageRegistration: IDisposable = null; setup(() => { - calledState.calledFor = []; - var text = + const TEXT = '1\r\n' + '2\n' + '3\n' + '4\r\n' + '5'; - thisModel = Model.createFromString(text, undefined, LANGUAGE_ID); + const LANGUAGE_ID = 'modelModeTest1'; + calledFor = []; + languageRegistration = modes.TokenizationRegistry.register(LANGUAGE_ID, tokenizationSupport); + thisModel = Model.createFromString(TEXT, undefined, new modes.LanguageIdentifier(LANGUAGE_ID, 0)); }); teardown(() => { thisModel.dispose(); + thisModel = null; + languageRegistration.dispose(); + languageRegistration = null; + calledFor = []; }); + test('model calls syntax highlighter 1', () => { thisModel.getLineTokens(1); - checkAndClear(calledState, ['1']); + checkAndClear(['1']); }); test('model calls syntax highlighter 2', () => { thisModel.getLineTokens(2); - checkAndClear(calledState, ['1', '2']); + checkAndClear(['1', '2']); thisModel.getLineTokens(2); - checkAndClear(calledState, []); + checkAndClear([]); }); test('model caches states', () => { thisModel.getLineTokens(1); - checkAndClear(calledState, ['1']); + checkAndClear(['1']); thisModel.getLineTokens(2); - checkAndClear(calledState, ['2']); + checkAndClear(['2']); thisModel.getLineTokens(3); - checkAndClear(calledState, ['3']); + checkAndClear(['3']); thisModel.getLineTokens(4); - checkAndClear(calledState, ['4']); + checkAndClear(['4']); thisModel.getLineTokens(5); - checkAndClear(calledState, ['5']); + checkAndClear(['5']); thisModel.getLineTokens(5); - checkAndClear(calledState, []); + checkAndClear([]); }); test('model invalidates states for one line insert', () => { thisModel.getLineTokens(5); - checkAndClear(calledState, ['1', '2', '3', '4', '5']); + checkAndClear(['1', '2', '3', '4', '5']); thisModel.applyEdits([EditOperation.insert(new Position(1, 1), '-')]); thisModel.getLineTokens(5); - checkAndClear(calledState, ['-']); + checkAndClear(['-']); thisModel.getLineTokens(5); - checkAndClear(calledState, []); + checkAndClear([]); }); test('model invalidates states for many lines insert', () => { thisModel.getLineTokens(5); - checkAndClear(calledState, ['1', '2', '3', '4', '5']); + checkAndClear(['1', '2', '3', '4', '5']); thisModel.applyEdits([EditOperation.insert(new Position(1, 1), '0\n-\n+')]); assert.equal(thisModel.getLineCount(), 7); thisModel.getLineTokens(7); - checkAndClear(calledState, ['0', '-', '+']); + checkAndClear(['0', '-', '+']); thisModel.getLineTokens(7); - checkAndClear(calledState, []); + checkAndClear([]); }); test('model invalidates states for one new line', () => { thisModel.getLineTokens(5); - checkAndClear(calledState, ['1', '2', '3', '4', '5']); + checkAndClear(['1', '2', '3', '4', '5']); thisModel.applyEdits([EditOperation.insert(new Position(1, 2), '\n')]); thisModel.applyEdits([EditOperation.insert(new Position(2, 1), 'a')]); thisModel.getLineTokens(6); - checkAndClear(calledState, ['1', 'a']); + checkAndClear(['1', 'a']); }); test('model invalidates states for one line delete', () => { thisModel.getLineTokens(5); - checkAndClear(calledState, ['1', '2', '3', '4', '5']); + checkAndClear(['1', '2', '3', '4', '5']); thisModel.applyEdits([EditOperation.insert(new Position(1, 2), '-')]); thisModel.getLineTokens(5); - checkAndClear(calledState, ['1']); + checkAndClear(['1']); thisModel.applyEdits([EditOperation.delete(new Range(1, 1, 1, 2))]); thisModel.getLineTokens(5); - checkAndClear(calledState, ['-']); + checkAndClear(['-']); thisModel.getLineTokens(5); - checkAndClear(calledState, []); + checkAndClear([]); }); test('model invalidates states for many lines delete', () => { thisModel.getLineTokens(5); - checkAndClear(calledState, ['1', '2', '3', '4', '5']); + checkAndClear(['1', '2', '3', '4', '5']); thisModel.applyEdits([EditOperation.delete(new Range(1, 1, 3, 1))]); thisModel.getLineTokens(3); - checkAndClear(calledState, ['3']); + checkAndClear(['3']); thisModel.getLineTokens(3); - checkAndClear(calledState, []); + checkAndClear([]); }); }); suite('Editor Model - Model Modes 2', () => { - const LANGUAGE_ID = 'modelModeTest2'; - class ModelState2 implements modes.IState { prevLineContent: string; @@ -176,33 +175,33 @@ suite('Editor Model - Model Modes 2', () => { } } - modes.TokenizationRegistry.register(LANGUAGE_ID, { + const tokenizationSupport: modes.ITokenizationSupport = { getInitialState: () => new ModelState2(''), - tokenize: (line: string, state: modes.IState): modes.ILineTokens => { + tokenize: undefined, + tokenize3: (line: string, state: modes.IState): modes.ILineTokens3 => { (state).prevLineContent = line; return { - tokens: [new Token(0, '')], - endState: state, - modeTransitions: null + tokens: null, + endState: state }; } - }); + }; - function invalidEqual(model, indexArray) { - var i, len, asHash = {}; - for (i = 0, len = indexArray.length; i < len; i++) { - asHash[indexArray[i]] = true; - } - for (i = 0, len = model.getLineCount(); i < len; i++) { - assert.equal(model._lines[i].isInvalid, asHash.hasOwnProperty(i)); + function invalidEqual(model: Model, expected: number[]): void { + let actual: number[] = []; + for (let i = 0, len = model.getLineCount(); i < len; i++) { + if (model._lines[i].isInvalid) { + actual.push(i); + } } + assert.deepEqual(actual, expected); } - function stateEqual(state, content) { - assert.equal(state.prevLineContent, content); + function stateEqual(state: modes.IState, content: string): void { + assert.equal((state).prevLineContent, content); } - function statesEqual(model: Model, states: string[]) { + function statesEqual(model: Model, states: string[]): void { var i, len = states.length - 1; for (i = 0; i < len; i++) { stateEqual(model._lines[i].getState(), states[i]); @@ -210,21 +209,28 @@ suite('Editor Model - Model Modes 2', () => { stateEqual((model)._lastState, states[len]); } - var thisModel: Model; + let thisModel: Model = null; + let languageRegistration: IDisposable = null; setup(() => { - var text = + const TEXT = 'Line1' + '\r\n' + 'Line2' + '\n' + 'Line3' + '\n' + 'Line4' + '\r\n' + 'Line5'; - thisModel = Model.createFromString(text, undefined, LANGUAGE_ID); + const LANGUAGE_ID = 'modelModeTest2'; + languageRegistration = modes.TokenizationRegistry.register(LANGUAGE_ID, tokenizationSupport); + thisModel = Model.createFromString(TEXT, undefined, new modes.LanguageIdentifier(LANGUAGE_ID, 0)); }); teardown(() => { thisModel.dispose(); + thisModel = null; + languageRegistration.dispose(); + languageRegistration = null; }); + test('getTokensForInvalidLines one text insert', () => { thisModel.getLineTokens(5); statesEqual(thisModel, ['', 'Line1', 'Line2', 'Line3', 'Line4', 'Line5']); @@ -291,42 +297,46 @@ suite('Editor Model - Model Modes 2', () => { suite('Editor Model - Token Iterator', () => { - const LANGUAGE_ID = 'modelModeTestTokenIterator'; - - class NState implements modes.IState { - clone(): modes.IState { return this; } - equals(other: modes.IState): boolean { return this === other; } - } - - modes.TokenizationRegistry.register(LANGUAGE_ID, { - getInitialState: (): modes.IState => new NState(), - tokenize: (line: string, state: modes.IState): modes.ILineTokens => { - let tokens: Token[] = []; - for (let i = 0; i < line.length / 3; i++) { - let from = 3 * i; - let to = from + 3; - tokens.push(new Token(from, 'n-3-' + line.substring(from, to))); + const tokenizationSupport: modes.ITokenizationSupport = { + getInitialState: (): modes.IState => NULL_STATE, + tokenize: undefined, + tokenize3: (line: string, state: modes.IState): modes.ILineTokens3 => { + if (line.length % 3 !== 0) { + throw new Error('Unexpected line length in ' + line); + } + let tokensCount = line.length / 3; + let tokens = new Uint32Array(tokensCount << 1); + for (let i = 0; i < tokensCount; i++) { + tokens[(i << 1)] = 3 * i; + tokens[(i << 1) + 1] = ( + i << modes.MetadataConsts.FOREGROUND_OFFSET + ) >>> 0; } return { tokens: tokens, - endState: state, - modeTransitions: null + endState: state }; } - }); + }; - var thisModel: Model; + let thisModel: Model = null; + let languageRegistration: IDisposable = null; setup(() => { - var text = + const TEXT = 'foobarfoobar' + '\r\n' + 'foobarfoobar' + '\r\n' + 'foobarfoobar' + '\r\n'; - thisModel = Model.createFromString(text, undefined, LANGUAGE_ID); + const LANGUAGE_ID = 'modelModeTestTokenIterator'; + languageRegistration = modes.TokenizationRegistry.register(LANGUAGE_ID, tokenizationSupport); + thisModel = Model.createFromString(TEXT, undefined, new modes.LanguageIdentifier(LANGUAGE_ID, 0)); }); teardown(() => { thisModel.dispose(); + thisModel = null; + languageRegistration.dispose(); + languageRegistration = null; }); test('all tokens with ranges', () => { @@ -518,5 +528,3 @@ suite('Editor Model - Token Iterator', () => { } }); }); - - diff --git a/src/vs/editor/test/common/model/textModelWithTokens.test.ts b/src/vs/editor/test/common/model/textModelWithTokens.test.ts index 9d87b3c44ec..96323722131 100644 --- a/src/vs/editor/test/common/model/textModelWithTokens.test.ts +++ b/src/vs/editor/test/common/model/textModelWithTokens.test.ts @@ -6,17 +6,17 @@ import * as assert from 'assert'; import { Model } from 'vs/editor/common/model/model'; +import { IDisposable } from 'vs/base/common/lifecycle'; import { ViewLineToken } from 'vs/editor/common/core/viewLineToken'; -import { TokenizationRegistry, IState } from 'vs/editor/common/modes'; +import { ITokenizationSupport, TokenizationRegistry, LanguageId, LanguageIdentifier, MetadataConsts } from 'vs/editor/common/modes'; import { CharacterPair } from 'vs/editor/common/modes/languageConfiguration'; -import { MockMode } from 'vs/editor/test/common/mocks/mockMode'; -import { Token } from 'vs/editor/common/core/token'; import { Range } from 'vs/editor/common/core/range'; import { Position } from 'vs/editor/common/core/position'; import { IFoundBracket } from 'vs/editor/common/editorCommon'; import { TextModel } from 'vs/editor/common/model/textModel'; import { TextModelWithTokens } from 'vs/editor/common/model/textModelWithTokens'; import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry'; +import { NULL_STATE } from 'vs/editor/common/modes/nullMode'; suite('TextModelWithTokens', () => { @@ -68,16 +68,17 @@ suite('TextModelWithTokens', () => { } } - class Mode extends MockMode { - constructor() { - super(); - LanguageConfigurationRegistry.register(this.getId(), { - brackets: brackets - }); - } - } + const languageIdentifier = new LanguageIdentifier('testMode', LanguageId.PlainText); - let model = new TextModelWithTokens([], TextModel.toRawText(contents.join('\n'), TextModel.DEFAULT_CREATION_OPTIONS), new Mode().getId()); + let registration = LanguageConfigurationRegistry.register(languageIdentifier, { + brackets: brackets + }); + + let model = new TextModelWithTokens( + [], + TextModel.toRawText(contents.join('\n'), TextModel.DEFAULT_CREATION_OPTIONS), + languageIdentifier + ); // findPrevBracket { @@ -132,6 +133,7 @@ suite('TextModelWithTokens', () => { } model.dispose(); + registration.dispose(); } test('brackets', () => { @@ -143,8 +145,6 @@ suite('TextModelWithTokens', () => { ['(', ')'] ]); }); - - }); suite('TextModelWithTokens - bracket matching', () => { @@ -159,21 +159,29 @@ suite('TextModelWithTokens - bracket matching', () => { assert.deepEqual(actual, expected, 'matches brackets at ' + testPosition); } - const LANGUAGE_ID = 'bracketMode1'; + const languageIdentifier = new LanguageIdentifier('bracketMode1', LanguageId.PlainText); + let registration: IDisposable = null; - LanguageConfigurationRegistry.register(LANGUAGE_ID, { - brackets: [ - ['{', '}'], - ['[', ']'], - ['(', ')'], - ] + setup(() => { + registration = LanguageConfigurationRegistry.register(languageIdentifier, { + brackets: [ + ['{', '}'], + ['[', ']'], + ['(', ')'], + ] + }); + }); + + teardown(() => { + registration.dispose(); + registration = null; }); test('bracket matching 1', () => { let text = ')]}{[(' + '\n' + ')]}{[('; - let model = Model.createFromString(text, undefined, LANGUAGE_ID); + let model = Model.createFromString(text, undefined, languageIdentifier); isNotABracket(model, 1, 1); isNotABracket(model, 1, 2); @@ -201,7 +209,7 @@ suite('TextModelWithTokens - bracket matching', () => { '}, bar: {hallo: [{' + '\n' + '}, {' + '\n' + '}]}}'; - let model = Model.createFromString(text, undefined, LANGUAGE_ID); + let model = Model.createFromString(text, undefined, languageIdentifier); let brackets: [Position, Range, Range][] = [ [new Position(1, 11), new Range(1, 11, 1, 12), new Range(5, 4, 5, 5)], @@ -259,52 +267,57 @@ suite('TextModelWithTokens regression tests', () => { assert.deepEqual(actual, expected); } - let _tokenId = 0; - class IndicisiveModeState implements IState { - clone(): IState { return this; } - equals(other: IState): boolean { return true; } - } - class IndicisiveMode extends MockMode { - constructor() { - super(); - TokenizationRegistry.register(this.getId(), { - getInitialState: () => { - return new IndicisiveModeState(); - }, - tokenize: (line, state, offsetDelta) => { - let myId = ++_tokenId; - return { - tokens: [new Token(0, 'custom.' + myId)], - endState: state, - modeTransitions: [] - }; - } - }); + let _tokenId = 10; + const LANG_ID1 = 'indicisiveMode1'; + const LANG_ID2 = 'indicisiveMode2'; + const languageIdentifier1 = new LanguageIdentifier(LANG_ID1, 3); + const languageIdentifier2 = new LanguageIdentifier(LANG_ID2, 4); + + const tokenizationSupport: ITokenizationSupport = { + getInitialState: () => NULL_STATE, + tokenize: undefined, + tokenize3: (line, state) => { + let myId = ++_tokenId; + let tokens = new Uint32Array(2); + tokens[0] = 0; + tokens[1] = ( + myId << MetadataConsts.FOREGROUND_OFFSET + ) >>> 0; + return { + tokens: tokens, + endState: state + }; } - } + }; + + let registration1 = TokenizationRegistry.register(LANG_ID1, tokenizationSupport); + let registration2 = TokenizationRegistry.register(LANG_ID2, tokenizationSupport); + let model = Model.createFromString('A model with\ntwo lines'); - assertViewLineTokens(model, 1, true, [new ViewLineToken(0, '')]); - assertViewLineTokens(model, 2, true, [new ViewLineToken(0, '')]); + assertViewLineTokens(model, 1, true, [new ViewLineToken(0, 'mtk1')]); + assertViewLineTokens(model, 2, true, [new ViewLineToken(0, 'mtk1')]); - model.setMode(new IndicisiveMode().getId()); + model.setMode(languageIdentifier1); - assertViewLineTokens(model, 1, true, [new ViewLineToken(0, 'custom.1')]); - assertViewLineTokens(model, 2, true, [new ViewLineToken(0, 'custom.2')]); + assertViewLineTokens(model, 1, true, [new ViewLineToken(0, 'mtk11')]); + assertViewLineTokens(model, 2, true, [new ViewLineToken(0, 'mtk12')]); - model.setMode(new IndicisiveMode().getId()); + model.setMode(languageIdentifier2); - assertViewLineTokens(model, 1, false, [new ViewLineToken(0, '')]); - assertViewLineTokens(model, 2, false, [new ViewLineToken(0, '')]); + assertViewLineTokens(model, 1, false, [new ViewLineToken(0, 'mtk1')]); + assertViewLineTokens(model, 2, false, [new ViewLineToken(0, 'mtk1')]); model.dispose(); + registration1.dispose(); + registration2.dispose(); }); test('Microsoft/monaco-editor#133: Error: Cannot read property \'modeId\' of undefined', () => { - const LANGUAGE_ID = 'bracketMode2'; + const languageIdentifier = new LanguageIdentifier('testMode', LanguageId.PlainText); - LanguageConfigurationRegistry.register(LANGUAGE_ID, { + let registration = LanguageConfigurationRegistry.register(languageIdentifier, { brackets: [ ['module', 'end module'], ['sub', 'end sub'] @@ -321,11 +334,12 @@ suite('TextModelWithTokens regression tests', () => { '\tEnd Sub', '', 'End Module', - ].join('\n'), undefined, LANGUAGE_ID); + ].join('\n'), undefined, languageIdentifier); let actual = model.matchBracket(new Position(4, 1)); assert.deepEqual(actual, [new Range(4, 1, 4, 7), new Range(9, 1, 9, 11)]); model.dispose(); + registration.dispose(); }); -}); \ No newline at end of file +}); diff --git a/src/vs/editor/test/common/modes/languageConfiguration.test.ts b/src/vs/editor/test/common/modes/languageConfiguration.test.ts index ec970be8dbe..bef57289a1f 100644 --- a/src/vs/editor/test/common/modes/languageConfiguration.test.ts +++ b/src/vs/editor/test/common/modes/languageConfiguration.test.ts @@ -7,7 +7,7 @@ import * as assert from 'assert'; import { StandardAutoClosingPairConditional } from 'vs/editor/common/modes/languageConfiguration'; -import { StandardTokenType } from 'vs/editor/common/core/lineTokens'; +import { StandardTokenType } from 'vs/editor/common/modes'; suite('StandardAutoClosingPairConditional', () => { diff --git a/src/vs/editor/test/common/modes/supports/characterPair.test.ts b/src/vs/editor/test/common/modes/supports/characterPair.test.ts index 2bcf2d5c70a..c21003f23ff 100644 --- a/src/vs/editor/test/common/modes/supports/characterPair.test.ts +++ b/src/vs/editor/test/common/modes/supports/characterPair.test.ts @@ -7,6 +7,7 @@ import * as assert from 'assert'; import { CharacterPairSupport } from 'vs/editor/common/modes/supports/characterPair'; import { TokenText, createFakeScopedLineTokens } from 'vs/editor/test/common/modesTestUtils'; +import { StandardTokenType } from 'vs/editor/common/modes'; suite('CharacterPairSupport', () => { @@ -53,7 +54,7 @@ suite('CharacterPairSupport', () => { }); function testShouldAutoClose(characterPairSupport: CharacterPairSupport, line: TokenText[], character: string, column: number): boolean { - return characterPairSupport.shouldAutoClosePair(character, createFakeScopedLineTokens('test', line), column); + return characterPairSupport.shouldAutoClosePair(character, createFakeScopedLineTokens(line), column); } test('shouldAutoClosePair in empty line', () => { @@ -64,58 +65,58 @@ suite('CharacterPairSupport', () => { test('shouldAutoClosePair in not interesting line 1', () => { let sup = new CharacterPairSupport({ autoClosingPairs: [{ open: '{', close: '}', notIn: ['string', 'comment'] }] }); - assert.equal(testShouldAutoClose(sup, [{ text: 'do', type: 'keyword' }], '{', 3), true); - assert.equal(testShouldAutoClose(sup, [{ text: 'do', type: 'keyword' }], 'a', 3), true); + assert.equal(testShouldAutoClose(sup, [{ text: 'do', type: StandardTokenType.Other }], '{', 3), true); + assert.equal(testShouldAutoClose(sup, [{ text: 'do', type: StandardTokenType.Other }], 'a', 3), true); }); test('shouldAutoClosePair in not interesting line 2', () => { let sup = new CharacterPairSupport({ autoClosingPairs: [{ open: '{', close: '}' }] }); - assert.equal(testShouldAutoClose(sup, [{ text: 'do', type: 'string' }], '{', 3), true); - assert.equal(testShouldAutoClose(sup, [{ text: 'do', type: 'string' }], 'a', 3), true); + assert.equal(testShouldAutoClose(sup, [{ text: 'do', type: StandardTokenType.String }], '{', 3), true); + assert.equal(testShouldAutoClose(sup, [{ text: 'do', type: StandardTokenType.String }], 'a', 3), true); }); test('shouldAutoClosePair in interesting line 1', () => { let sup = new CharacterPairSupport({ autoClosingPairs: [{ open: '{', close: '}', notIn: ['string', 'comment'] }] }); - assert.equal(testShouldAutoClose(sup, [{ text: '"a"', type: 'string' }], '{', 1), false); - assert.equal(testShouldAutoClose(sup, [{ text: '"a"', type: 'string' }], 'a', 1), true); - assert.equal(testShouldAutoClose(sup, [{ text: '"a"', type: 'string' }], '{', 2), false); - assert.equal(testShouldAutoClose(sup, [{ text: '"a"', type: 'string' }], 'a', 2), true); - assert.equal(testShouldAutoClose(sup, [{ text: '"a"', type: 'string' }], '{', 3), false); - assert.equal(testShouldAutoClose(sup, [{ text: '"a"', type: 'string' }], 'a', 3), true); - assert.equal(testShouldAutoClose(sup, [{ text: '"a"', type: 'string' }], '{', 4), false); - assert.equal(testShouldAutoClose(sup, [{ text: '"a"', type: 'string' }], 'a', 4), true); + assert.equal(testShouldAutoClose(sup, [{ text: '"a"', type: StandardTokenType.String }], '{', 1), false); + assert.equal(testShouldAutoClose(sup, [{ text: '"a"', type: StandardTokenType.String }], 'a', 1), true); + assert.equal(testShouldAutoClose(sup, [{ text: '"a"', type: StandardTokenType.String }], '{', 2), false); + assert.equal(testShouldAutoClose(sup, [{ text: '"a"', type: StandardTokenType.String }], 'a', 2), true); + assert.equal(testShouldAutoClose(sup, [{ text: '"a"', type: StandardTokenType.String }], '{', 3), false); + assert.equal(testShouldAutoClose(sup, [{ text: '"a"', type: StandardTokenType.String }], 'a', 3), true); + assert.equal(testShouldAutoClose(sup, [{ text: '"a"', type: StandardTokenType.String }], '{', 4), false); + assert.equal(testShouldAutoClose(sup, [{ text: '"a"', type: StandardTokenType.String }], 'a', 4), true); }); test('shouldAutoClosePair in interesting line 2', () => { let sup = new CharacterPairSupport({ autoClosingPairs: [{ open: '{', close: '}', notIn: ['string', 'comment'] }] }); - assert.equal(testShouldAutoClose(sup, [{ text: 'x=', type: 'op' }, { text: '"a"', type: 'string' }, { text: ';', type: 'punct' }], '{', 1), true); - assert.equal(testShouldAutoClose(sup, [{ text: 'x=', type: 'op' }, { text: '"a"', type: 'string' }, { text: ';', type: 'punct' }], 'a', 1), true); - assert.equal(testShouldAutoClose(sup, [{ text: 'x=', type: 'op' }, { text: '"a"', type: 'string' }, { text: ';', type: 'punct' }], '{', 2), true); - assert.equal(testShouldAutoClose(sup, [{ text: 'x=', type: 'op' }, { text: '"a"', type: 'string' }, { text: ';', type: 'punct' }], 'a', 2), true); - assert.equal(testShouldAutoClose(sup, [{ text: 'x=', type: 'op' }, { text: '"a"', type: 'string' }, { text: ';', type: 'punct' }], '{', 3), true); - assert.equal(testShouldAutoClose(sup, [{ text: 'x=', type: 'op' }, { text: '"a"', type: 'string' }, { text: ';', type: 'punct' }], 'a', 3), true); - assert.equal(testShouldAutoClose(sup, [{ text: 'x=', type: 'op' }, { text: '"a"', type: 'string' }, { text: ';', type: 'punct' }], '{', 4), false); - assert.equal(testShouldAutoClose(sup, [{ text: 'x=', type: 'op' }, { text: '"a"', type: 'string' }, { text: ';', type: 'punct' }], 'a', 4), true); - assert.equal(testShouldAutoClose(sup, [{ text: 'x=', type: 'op' }, { text: '"a"', type: 'string' }, { text: ';', type: 'punct' }], '{', 5), false); - assert.equal(testShouldAutoClose(sup, [{ text: 'x=', type: 'op' }, { text: '"a"', type: 'string' }, { text: ';', type: 'punct' }], 'a', 5), true); - assert.equal(testShouldAutoClose(sup, [{ text: 'x=', type: 'op' }, { text: '"a"', type: 'string' }, { text: ';', type: 'punct' }], '{', 6), false); - assert.equal(testShouldAutoClose(sup, [{ text: 'x=', type: 'op' }, { text: '"a"', type: 'string' }, { text: ';', type: 'punct' }], 'a', 6), true); - assert.equal(testShouldAutoClose(sup, [{ text: 'x=', type: 'op' }, { text: '"a"', type: 'string' }, { text: ';', type: 'punct' }], '{', 7), true); - assert.equal(testShouldAutoClose(sup, [{ text: 'x=', type: 'op' }, { text: '"a"', type: 'string' }, { text: ';', type: 'punct' }], 'a', 7), true); + assert.equal(testShouldAutoClose(sup, [{ text: 'x=', type: StandardTokenType.Other }, { text: '"a"', type: StandardTokenType.String }, { text: ';', type: StandardTokenType.Other }], '{', 1), true); + assert.equal(testShouldAutoClose(sup, [{ text: 'x=', type: StandardTokenType.Other }, { text: '"a"', type: StandardTokenType.String }, { text: ';', type: StandardTokenType.Other }], 'a', 1), true); + assert.equal(testShouldAutoClose(sup, [{ text: 'x=', type: StandardTokenType.Other }, { text: '"a"', type: StandardTokenType.String }, { text: ';', type: StandardTokenType.Other }], '{', 2), true); + assert.equal(testShouldAutoClose(sup, [{ text: 'x=', type: StandardTokenType.Other }, { text: '"a"', type: StandardTokenType.String }, { text: ';', type: StandardTokenType.Other }], 'a', 2), true); + assert.equal(testShouldAutoClose(sup, [{ text: 'x=', type: StandardTokenType.Other }, { text: '"a"', type: StandardTokenType.String }, { text: ';', type: StandardTokenType.Other }], '{', 3), true); + assert.equal(testShouldAutoClose(sup, [{ text: 'x=', type: StandardTokenType.Other }, { text: '"a"', type: StandardTokenType.String }, { text: ';', type: StandardTokenType.Other }], 'a', 3), true); + assert.equal(testShouldAutoClose(sup, [{ text: 'x=', type: StandardTokenType.Other }, { text: '"a"', type: StandardTokenType.String }, { text: ';', type: StandardTokenType.Other }], '{', 4), false); + assert.equal(testShouldAutoClose(sup, [{ text: 'x=', type: StandardTokenType.Other }, { text: '"a"', type: StandardTokenType.String }, { text: ';', type: StandardTokenType.Other }], 'a', 4), true); + assert.equal(testShouldAutoClose(sup, [{ text: 'x=', type: StandardTokenType.Other }, { text: '"a"', type: StandardTokenType.String }, { text: ';', type: StandardTokenType.Other }], '{', 5), false); + assert.equal(testShouldAutoClose(sup, [{ text: 'x=', type: StandardTokenType.Other }, { text: '"a"', type: StandardTokenType.String }, { text: ';', type: StandardTokenType.Other }], 'a', 5), true); + assert.equal(testShouldAutoClose(sup, [{ text: 'x=', type: StandardTokenType.Other }, { text: '"a"', type: StandardTokenType.String }, { text: ';', type: StandardTokenType.Other }], '{', 6), false); + assert.equal(testShouldAutoClose(sup, [{ text: 'x=', type: StandardTokenType.Other }, { text: '"a"', type: StandardTokenType.String }, { text: ';', type: StandardTokenType.Other }], 'a', 6), true); + assert.equal(testShouldAutoClose(sup, [{ text: 'x=', type: StandardTokenType.Other }, { text: '"a"', type: StandardTokenType.String }, { text: ';', type: StandardTokenType.Other }], '{', 7), true); + assert.equal(testShouldAutoClose(sup, [{ text: 'x=', type: StandardTokenType.Other }, { text: '"a"', type: StandardTokenType.String }, { text: ';', type: StandardTokenType.Other }], 'a', 7), true); }); test('shouldAutoClosePair in interesting line 3', () => { let sup = new CharacterPairSupport({ autoClosingPairs: [{ open: '{', close: '}', notIn: ['string', 'comment'] }] }); - assert.equal(testShouldAutoClose(sup, [{ text: ' ', type: '' }, { text: '//a', type: 'comment' }], '{', 1), true); - assert.equal(testShouldAutoClose(sup, [{ text: ' ', type: '' }, { text: '//a', type: 'comment' }], 'a', 1), true); - assert.equal(testShouldAutoClose(sup, [{ text: ' ', type: '' }, { text: '//a', type: 'comment' }], '{', 2), true); - assert.equal(testShouldAutoClose(sup, [{ text: ' ', type: '' }, { text: '//a', type: 'comment' }], 'a', 2), true); - assert.equal(testShouldAutoClose(sup, [{ text: ' ', type: '' }, { text: '//a', type: 'comment' }], '{', 3), false); - assert.equal(testShouldAutoClose(sup, [{ text: ' ', type: '' }, { text: '//a', type: 'comment' }], 'a', 3), true); - assert.equal(testShouldAutoClose(sup, [{ text: ' ', type: '' }, { text: '//a', type: 'comment' }], '{', 4), false); - assert.equal(testShouldAutoClose(sup, [{ text: ' ', type: '' }, { text: '//a', type: 'comment' }], 'a', 4), true); - assert.equal(testShouldAutoClose(sup, [{ text: ' ', type: '' }, { text: '//a', type: 'comment' }], '{', 5), false); - assert.equal(testShouldAutoClose(sup, [{ text: ' ', type: '' }, { text: '//a', type: 'comment' }], 'a', 5), true); + assert.equal(testShouldAutoClose(sup, [{ text: ' ', type: StandardTokenType.Other }, { text: '//a', type: StandardTokenType.Comment }], '{', 1), true); + assert.equal(testShouldAutoClose(sup, [{ text: ' ', type: StandardTokenType.Other }, { text: '//a', type: StandardTokenType.Comment }], 'a', 1), true); + assert.equal(testShouldAutoClose(sup, [{ text: ' ', type: StandardTokenType.Other }, { text: '//a', type: StandardTokenType.Comment }], '{', 2), true); + assert.equal(testShouldAutoClose(sup, [{ text: ' ', type: StandardTokenType.Other }, { text: '//a', type: StandardTokenType.Comment }], 'a', 2), true); + assert.equal(testShouldAutoClose(sup, [{ text: ' ', type: StandardTokenType.Other }, { text: '//a', type: StandardTokenType.Comment }], '{', 3), false); + assert.equal(testShouldAutoClose(sup, [{ text: ' ', type: StandardTokenType.Other }, { text: '//a', type: StandardTokenType.Comment }], 'a', 3), true); + assert.equal(testShouldAutoClose(sup, [{ text: ' ', type: StandardTokenType.Other }, { text: '//a', type: StandardTokenType.Comment }], '{', 4), false); + assert.equal(testShouldAutoClose(sup, [{ text: ' ', type: StandardTokenType.Other }, { text: '//a', type: StandardTokenType.Comment }], 'a', 4), true); + assert.equal(testShouldAutoClose(sup, [{ text: ' ', type: StandardTokenType.Other }, { text: '//a', type: StandardTokenType.Comment }], '{', 5), false); + assert.equal(testShouldAutoClose(sup, [{ text: ' ', type: StandardTokenType.Other }, { text: '//a', type: StandardTokenType.Comment }], 'a', 5), true); }); }); diff --git a/src/vs/editor/test/common/modes/supports/electricCharacter.test.ts b/src/vs/editor/test/common/modes/supports/electricCharacter.test.ts index ab4733df599..3c3b1b0b327 100644 --- a/src/vs/editor/test/common/modes/supports/electricCharacter.test.ts +++ b/src/vs/editor/test/common/modes/supports/electricCharacter.test.ts @@ -8,10 +8,13 @@ import * as assert from 'assert'; import { BracketElectricCharacterSupport, IElectricAction } from 'vs/editor/common/modes/supports/electricCharacter'; import { createFakeScopedLineTokens, TokenText } from 'vs/editor/test/common/modesTestUtils'; import { RichEditBrackets } from 'vs/editor/common/modes/supports/richEditBrackets'; +import { LanguageIdentifier, StandardTokenType } from 'vs/editor/common/modes'; + +const fakeLanguageIdentifier = new LanguageIdentifier('test', 3); suite('Editor Modes - Auto Indentation', () => { function _testOnElectricCharacter(electricCharacterSupport: BracketElectricCharacterSupport, line: TokenText[], character: string, offset: number): IElectricAction { - return electricCharacterSupport.onElectricCharacter(character, createFakeScopedLineTokens('test', line), offset); + return electricCharacterSupport.onElectricCharacter(character, createFakeScopedLineTokens(line), offset); } function testDoesNothing(electricCharacterSupport: BracketElectricCharacterSupport, line: TokenText[], character: string, offset: number): void { @@ -33,19 +36,19 @@ suite('Editor Modes - Auto Indentation', () => { var brackets = new BracketElectricCharacterSupport(null, [{ open: '/**', close: ' */' }], null); testAppends(brackets, [ - { text: '/*', type: 'doc' }, + { text: '/*', type: StandardTokenType.Other }, ], '*', 3, ' */'); testDoesNothing(brackets, [ - { text: '/*', type: 'doc' }, - { text: ' ', type: 'doc' }, - { text: '*/', type: 'doc' }, + { text: '/*', type: StandardTokenType.Other }, + { text: ' ', type: StandardTokenType.Other }, + { text: '*/', type: StandardTokenType.Other }, ], '*', 3); }); test('getElectricCharacters uses all sources and dedups', () => { var sup = new BracketElectricCharacterSupport( - new RichEditBrackets('test', [ + new RichEditBrackets(fakeLanguageIdentifier, [ ['{', '}'], ['(', ')'] ]), [ @@ -61,7 +64,7 @@ suite('Editor Modes - Auto Indentation', () => { test('auto-close', () => { var sup = new BracketElectricCharacterSupport( - new RichEditBrackets('test', [ + new RichEditBrackets(fakeLanguageIdentifier, [ ['{', '}'], ['(', ')'] ]), [ @@ -74,36 +77,36 @@ suite('Editor Modes - Auto Indentation', () => { testDoesNothing(sup, [], 'a', 0); - testDoesNothing(sup, [{ text: 'egi', type: '' }], 'b', 1); - testDoesNothing(sup, [{ text: 'bgi', type: '' }], 'e', 2); - testDoesNothing(sup, [{ text: 'bei', type: '' }], 'g', 3); - testDoesNothing(sup, [{ text: 'beg', type: '' }], 'i', 4); + testDoesNothing(sup, [{ text: 'egi', type: StandardTokenType.Other }], 'b', 1); + testDoesNothing(sup, [{ text: 'bgi', type: StandardTokenType.Other }], 'e', 2); + testDoesNothing(sup, [{ text: 'bei', type: StandardTokenType.Other }], 'g', 3); + testDoesNothing(sup, [{ text: 'beg', type: StandardTokenType.Other }], 'i', 4); - testDoesNothing(sup, [{ text: 'egin', type: '' }], 'b', 1); - testDoesNothing(sup, [{ text: 'bgin', type: '' }], 'e', 2); - testDoesNothing(sup, [{ text: 'bein', type: '' }], 'g', 3); - testDoesNothing(sup, [{ text: 'begn', type: '' }], 'i', 4); - testAppends(sup, [{ text: 'begi', type: '' }], 'n', 5, 'end'); + testDoesNothing(sup, [{ text: 'egin', type: StandardTokenType.Other }], 'b', 1); + testDoesNothing(sup, [{ text: 'bgin', type: StandardTokenType.Other }], 'e', 2); + testDoesNothing(sup, [{ text: 'bein', type: StandardTokenType.Other }], 'g', 3); + testDoesNothing(sup, [{ text: 'begn', type: StandardTokenType.Other }], 'i', 4); + testAppends(sup, [{ text: 'begi', type: StandardTokenType.Other }], 'n', 5, 'end'); - testDoesNothing(sup, [{ text: '3gin', type: '' }], 'b', 1); - testDoesNothing(sup, [{ text: 'bgin', type: '' }], '3', 2); - testDoesNothing(sup, [{ text: 'b3in', type: '' }], 'g', 3); - testDoesNothing(sup, [{ text: 'b3gn', type: '' }], 'i', 4); - testDoesNothing(sup, [{ text: 'b3gi', type: '' }], 'n', 5); + testDoesNothing(sup, [{ text: '3gin', type: StandardTokenType.Other }], 'b', 1); + testDoesNothing(sup, [{ text: 'bgin', type: StandardTokenType.Other }], '3', 2); + testDoesNothing(sup, [{ text: 'b3in', type: StandardTokenType.Other }], 'g', 3); + testDoesNothing(sup, [{ text: 'b3gn', type: StandardTokenType.Other }], 'i', 4); + testDoesNothing(sup, [{ text: 'b3gi', type: StandardTokenType.Other }], 'n', 5); - testDoesNothing(sup, [{ text: 'begi', type: 'string' }], 'n', 5); + testDoesNothing(sup, [{ text: 'begi', type: StandardTokenType.String }], 'n', 5); - testAppends(sup, [{ text: '"', type: 'string' }, { text: 'begi', type: '' }], 'n', 6, 'end'); - testDoesNothing(sup, [{ text: '"', type: 'string' }, { text: 'begi', type: 'string' }], 'n', 6); + testAppends(sup, [{ text: '"', type: StandardTokenType.String }, { text: 'begi', type: StandardTokenType.Other }], 'n', 6, 'end'); + testDoesNothing(sup, [{ text: '"', type: StandardTokenType.String }, { text: 'begi', type: StandardTokenType.String }], 'n', 6); - testAppends(sup, [{ text: '/*', type: 'string' }], '*', 3, ' */'); + testAppends(sup, [{ text: '/*', type: StandardTokenType.String }], '*', 3, ' */'); - testDoesNothing(sup, [{ text: 'begi', type: '' }, { text: 'end', type: '' }], 'n', 5); + testDoesNothing(sup, [{ text: 'begi', type: StandardTokenType.Other }, { text: 'end', type: StandardTokenType.Other }], 'n', 5); }); test('matchOpenBracket', () => { var sup = new BracketElectricCharacterSupport( - new RichEditBrackets('test', [ + new RichEditBrackets(fakeLanguageIdentifier, [ ['{', '}'], ['(', ')'] ]), [ @@ -114,12 +117,12 @@ suite('Editor Modes - Auto Indentation', () => { { docComment: { open: '/**', close: ' */' } } ); - testDoesNothing(sup, [{ text: '\t{', type: '' }], '\t', 1); - testDoesNothing(sup, [{ text: '\t{', type: '' }], '\t', 2); - testDoesNothing(sup, [{ text: '\t\t', type: '' }], '{', 3); + testDoesNothing(sup, [{ text: '\t{', type: StandardTokenType.Other }], '\t', 1); + testDoesNothing(sup, [{ text: '\t{', type: StandardTokenType.Other }], '\t', 2); + testDoesNothing(sup, [{ text: '\t\t', type: StandardTokenType.Other }], '{', 3); - testDoesNothing(sup, [{ text: '\t}', type: '' }], '\t', 1); - testDoesNothing(sup, [{ text: '\t}', type: '' }], '\t', 2); - testMatchBracket(sup, [{ text: '\t\t', type: '' }], '}', 3, '}'); + testDoesNothing(sup, [{ text: '\t}', type: StandardTokenType.Other }], '\t', 1); + testDoesNothing(sup, [{ text: '\t}', type: StandardTokenType.Other }], '\t', 2); + testMatchBracket(sup, [{ text: '\t\t', type: StandardTokenType.Other }], '}', 3, '}'); }); }); diff --git a/src/vs/editor/test/common/modes/supports/tokenization.test.ts b/src/vs/editor/test/common/modes/supports/tokenization.test.ts index 272d5f56a38..28187687942 100644 --- a/src/vs/editor/test/common/modes/supports/tokenization.test.ts +++ b/src/vs/editor/test/common/modes/supports/tokenization.test.ts @@ -5,58 +5,57 @@ 'use strict'; import * as assert from 'assert'; -import { strcmp, parseTheme, Theme, ParsedThemeRule, ColorMap, ExternalThemeTrieElement, ThemeTrieElementRule, FontStyle } from 'vs/editor/common/modes/supports/tokenization'; +import { strcmp, parseTheme, Theme, ParsedThemeRule, ColorMap, ExternalThemeTrieElement, ThemeTrieElementRule } from 'vs/editor/common/modes/supports/tokenization'; +import { FontStyle } from 'vs/editor/common/modes'; suite('Theme matching', () => { test('gives higher priority to deeper matches', () => { let theme = Theme.createFromRawTheme([ - { token: '', foreground: '#100000', background: '#200000' }, - { token: 'punctuation.definition.string.begin.html', foreground: '#300000' }, - { token: 'punctuation.definition.string', foreground: '#400000' }, + { token: '', foreground: '100000', background: '200000' }, + { token: 'punctuation.definition.string.begin.html', foreground: '300000' }, + { token: 'punctuation.definition.string', foreground: '400000' }, ]); let colorMap = new ColorMap(); - const _NOT_SET = 0; - colorMap.getId('#100000'); - colorMap.getId('#200000'); - colorMap.getId('#400000'); - const _D = colorMap.getId('#300000'); + colorMap.getId('100000'); + const _B = colorMap.getId('200000'); + colorMap.getId('400000'); + const _D = colorMap.getId('300000'); - let actual = theme.match('punctuation.definition.string.begin.html'); + let actual = theme._match('punctuation.definition.string.begin.html'); - assert.deepEqual(actual, new ThemeTrieElementRule(FontStyle.NotSet, _D, _NOT_SET)); + assert.deepEqual(actual, new ThemeTrieElementRule(FontStyle.None, _D, _B)); }); test('can match', () => { let theme = Theme.createFromRawTheme([ - { token: '', foreground: '#F8F8F2', background: '#272822' }, - { token: 'source', background: '#100000' }, - { token: 'something', background: '#100000' }, - { token: 'bar', background: '#200000' }, - { token: 'baz', background: '#200000' }, + { token: '', foreground: 'F8F8F2', background: '272822' }, + { token: 'source', background: '100000' }, + { token: 'something', background: '100000' }, + { token: 'bar', background: '200000' }, + { token: 'baz', background: '200000' }, { token: 'bar', fontStyle: 'bold' }, - { token: 'constant', fontStyle: 'italic', foreground: '#300000' }, - { token: 'constant.numeric', foreground: '#400000' }, + { token: 'constant', fontStyle: 'italic', foreground: '300000' }, + { token: 'constant.numeric', foreground: '400000' }, { token: 'constant.numeric.hex', fontStyle: 'bold' }, { token: 'constant.numeric.oct', fontStyle: 'bold italic underline' }, - { token: 'constant.numeric.dec', fontStyle: '', foreground: '#500000' }, - { token: 'storage.object.bar', fontStyle: '', foreground: '#600000' }, + { token: 'constant.numeric.dec', fontStyle: '', foreground: '500000' }, + { token: 'storage.object.bar', fontStyle: '', foreground: '600000' }, ]); let colorMap = new ColorMap(); - const _NOT_SET = 0; - colorMap.getId('#F8F8F2'); - colorMap.getId('#272822'); - const _C = colorMap.getId('#200000'); - const _D = colorMap.getId('#300000'); - const _E = colorMap.getId('#400000'); - const _F = colorMap.getId('#500000'); - const _G = colorMap.getId('#100000'); - const _H = colorMap.getId('#600000'); + const _A = colorMap.getId('F8F8F2'); + const _B = colorMap.getId('272822'); + const _C = colorMap.getId('200000'); + const _D = colorMap.getId('300000'); + const _E = colorMap.getId('400000'); + const _F = colorMap.getId('500000'); + const _G = colorMap.getId('100000'); + const _H = colorMap.getId('600000'); function assertMatch(scopeName: string, expected: ThemeTrieElementRule): void { - let actual = theme.match(scopeName); + let actual = theme._match(scopeName); assert.deepEqual(actual, expected, 'when matching <<' + scopeName + '>>'); } @@ -65,7 +64,7 @@ suite('Theme matching', () => { } function assertNoMatch(scopeName: string): void { - assertMatch(scopeName, new ThemeTrieElementRule(FontStyle.NotSet, _NOT_SET, _NOT_SET)); + assertMatch(scopeName, new ThemeTrieElementRule(FontStyle.None, _A, _B)); } // matches defaults @@ -74,51 +73,51 @@ suite('Theme matching', () => { assertNoMatch('asdfg'); // matches source - assertSimpleMatch('source', FontStyle.NotSet, _NOT_SET, _G); - assertSimpleMatch('source.ts', FontStyle.NotSet, _NOT_SET, _G); - assertSimpleMatch('source.tss', FontStyle.NotSet, _NOT_SET, _G); + assertSimpleMatch('source', FontStyle.None, _A, _G); + assertSimpleMatch('source.ts', FontStyle.None, _A, _G); + assertSimpleMatch('source.tss', FontStyle.None, _A, _G); // matches something - assertSimpleMatch('something', FontStyle.NotSet, _NOT_SET, _G); - assertSimpleMatch('something.ts', FontStyle.NotSet, _NOT_SET, _G); - assertSimpleMatch('something.tss', FontStyle.NotSet, _NOT_SET, _G); + assertSimpleMatch('something', FontStyle.None, _A, _G); + assertSimpleMatch('something.ts', FontStyle.None, _A, _G); + assertSimpleMatch('something.tss', FontStyle.None, _A, _G); // matches baz - assertSimpleMatch('baz', FontStyle.NotSet, _NOT_SET, _C); - assertSimpleMatch('baz.ts', FontStyle.NotSet, _NOT_SET, _C); - assertSimpleMatch('baz.tss', FontStyle.NotSet, _NOT_SET, _C); + assertSimpleMatch('baz', FontStyle.None, _A, _C); + assertSimpleMatch('baz.ts', FontStyle.None, _A, _C); + assertSimpleMatch('baz.tss', FontStyle.None, _A, _C); // matches constant - assertSimpleMatch('constant', FontStyle.Italic, _D, _NOT_SET); - assertSimpleMatch('constant.string', FontStyle.Italic, _D, _NOT_SET); - assertSimpleMatch('constant.hex', FontStyle.Italic, _D, _NOT_SET); + assertSimpleMatch('constant', FontStyle.Italic, _D, _B); + assertSimpleMatch('constant.string', FontStyle.Italic, _D, _B); + assertSimpleMatch('constant.hex', FontStyle.Italic, _D, _B); // matches constant.numeric - assertSimpleMatch('constant.numeric', FontStyle.Italic, _E, _NOT_SET); - assertSimpleMatch('constant.numeric.baz', FontStyle.Italic, _E, _NOT_SET); + assertSimpleMatch('constant.numeric', FontStyle.Italic, _E, _B); + assertSimpleMatch('constant.numeric.baz', FontStyle.Italic, _E, _B); // matches constant.numeric.hex - assertSimpleMatch('constant.numeric.hex', FontStyle.Bold, _E, _NOT_SET); - assertSimpleMatch('constant.numeric.hex.baz', FontStyle.Bold, _E, _NOT_SET); + assertSimpleMatch('constant.numeric.hex', FontStyle.Bold, _E, _B); + assertSimpleMatch('constant.numeric.hex.baz', FontStyle.Bold, _E, _B); // matches constant.numeric.oct - assertSimpleMatch('constant.numeric.oct', FontStyle.Bold | FontStyle.Italic | FontStyle.Underline, _E, _NOT_SET); - assertSimpleMatch('constant.numeric.oct.baz', FontStyle.Bold | FontStyle.Italic | FontStyle.Underline, _E, _NOT_SET); + assertSimpleMatch('constant.numeric.oct', FontStyle.Bold | FontStyle.Italic | FontStyle.Underline, _E, _B); + assertSimpleMatch('constant.numeric.oct.baz', FontStyle.Bold | FontStyle.Italic | FontStyle.Underline, _E, _B); // matches constant.numeric.dec - assertSimpleMatch('constant.numeric.dec', FontStyle.None, _F, _NOT_SET); - assertSimpleMatch('constant.numeric.dec.baz', FontStyle.None, _F, _NOT_SET); + assertSimpleMatch('constant.numeric.dec', FontStyle.None, _F, _B); + assertSimpleMatch('constant.numeric.dec.baz', FontStyle.None, _F, _B); // matches storage.object.bar - assertSimpleMatch('storage.object.bar', FontStyle.None, _H, _NOT_SET); - assertSimpleMatch('storage.object.bar.baz', FontStyle.None, _H, _NOT_SET); + assertSimpleMatch('storage.object.bar', FontStyle.None, _H, _B); + assertSimpleMatch('storage.object.bar.baz', FontStyle.None, _H, _B); // does not match storage.object.bar - assertSimpleMatch('storage.object.bart', FontStyle.NotSet, _NOT_SET, _NOT_SET); - assertSimpleMatch('storage.object', FontStyle.NotSet, _NOT_SET, _NOT_SET); - assertSimpleMatch('storage', FontStyle.NotSet, _NOT_SET, _NOT_SET); + assertSimpleMatch('storage.object.bart', FontStyle.None, _A, _B); + assertSimpleMatch('storage.object', FontStyle.None, _A, _B); + assertSimpleMatch('storage', FontStyle.None, _A, _B); - assertSimpleMatch('bar', FontStyle.Bold, _NOT_SET, _C); + assertSimpleMatch('bar', FontStyle.Bold, _A, _C); }); }); @@ -127,31 +126,31 @@ suite('Theme parsing', () => { test('can parse', () => { let actual = parseTheme([ - { token: '', foreground: '#F8F8F2', background: '#272822' }, - { token: 'source', background: '#100000' }, - { token: 'something', background: '#100000' }, - { token: 'bar', background: '#010000' }, - { token: 'baz', background: '#010000' }, + { token: '', foreground: 'F8F8F2', background: '272822' }, + { token: 'source', background: '100000' }, + { token: 'something', background: '100000' }, + { token: 'bar', background: '010000' }, + { token: 'baz', background: '010000' }, { token: 'bar', fontStyle: 'bold' }, - { token: 'constant', fontStyle: 'italic', foreground: '#ff0000' }, - { token: 'constant.numeric', foreground: '#00ff00' }, + { token: 'constant', fontStyle: 'italic', foreground: 'ff0000' }, + { token: 'constant.numeric', foreground: '00ff00' }, { token: 'constant.numeric.hex', fontStyle: 'bold' }, { token: 'constant.numeric.oct', fontStyle: 'bold italic underline' }, - { token: 'constant.numeric.dec', fontStyle: '', foreground: '#0000ff' }, + { token: 'constant.numeric.dec', fontStyle: '', foreground: '0000ff' }, ]); let expected = [ - new ParsedThemeRule('', 0, FontStyle.NotSet, '#F8F8F2', '#272822'), - new ParsedThemeRule('source', 1, FontStyle.NotSet, null, '#100000'), - new ParsedThemeRule('something', 2, FontStyle.NotSet, null, '#100000'), - new ParsedThemeRule('bar', 3, FontStyle.NotSet, null, '#010000'), - new ParsedThemeRule('baz', 4, FontStyle.NotSet, null, '#010000'), + new ParsedThemeRule('', 0, FontStyle.NotSet, 'F8F8F2', '272822'), + new ParsedThemeRule('source', 1, FontStyle.NotSet, null, '100000'), + new ParsedThemeRule('something', 2, FontStyle.NotSet, null, '100000'), + new ParsedThemeRule('bar', 3, FontStyle.NotSet, null, '010000'), + new ParsedThemeRule('baz', 4, FontStyle.NotSet, null, '010000'), new ParsedThemeRule('bar', 5, FontStyle.Bold, null, null), - new ParsedThemeRule('constant', 6, FontStyle.Italic, '#ff0000', null), - new ParsedThemeRule('constant.numeric', 7, FontStyle.NotSet, '#00ff00', null), + new ParsedThemeRule('constant', 6, FontStyle.Italic, 'ff0000', null), + new ParsedThemeRule('constant.numeric', 7, FontStyle.NotSet, '00ff00', null), new ParsedThemeRule('constant.numeric.hex', 8, FontStyle.Bold, null, null), new ParsedThemeRule('constant.numeric.oct', 9, FontStyle.Bold | FontStyle.Italic | FontStyle.Underline, null, null), - new ParsedThemeRule('constant.numeric.dec', 10, FontStyle.None, '#0000ff', null), + new ParsedThemeRule('constant.numeric.dec', 10, FontStyle.None, '0000ff', null), ]; assert.deepEqual(actual, expected); @@ -170,12 +169,10 @@ suite('Theme resolving', () => { test('always has defaults', () => { let actual = Theme.createFromParsedTheme([]); let colorMap = new ColorMap(); - const _NOT_SET = 0; - const _A = colorMap.getId('#000000'); - const _B = colorMap.getId('#ffffff'); + const _A = colorMap.getId('000000'); + const _B = colorMap.getId('ffffff'); assert.deepEqual(actual.getColorMap(), colorMap.getColorMap()); - assert.deepEqual(actual.getDefaults(), new ThemeTrieElementRule(FontStyle.None, _A, _B)); - assert.deepEqual(actual.getThemeTrieElement(), new ExternalThemeTrieElement(new ThemeTrieElementRule(FontStyle.NotSet, _NOT_SET, _NOT_SET))); + assert.deepEqual(actual.getThemeTrieElement(), new ExternalThemeTrieElement(new ThemeTrieElementRule(FontStyle.None, _A, _B))); }); test('respects incoming defaults 1', () => { @@ -183,12 +180,10 @@ suite('Theme resolving', () => { new ParsedThemeRule('', -1, FontStyle.NotSet, null, null) ]); let colorMap = new ColorMap(); - const _NOT_SET = 0; - const _A = colorMap.getId('#000000'); - const _B = colorMap.getId('#ffffff'); + const _A = colorMap.getId('000000'); + const _B = colorMap.getId('ffffff'); assert.deepEqual(actual.getColorMap(), colorMap.getColorMap()); - assert.deepEqual(actual.getDefaults(), new ThemeTrieElementRule(FontStyle.None, _A, _B)); - assert.deepEqual(actual.getThemeTrieElement(), new ExternalThemeTrieElement(new ThemeTrieElementRule(FontStyle.NotSet, _NOT_SET, _NOT_SET))); + assert.deepEqual(actual.getThemeTrieElement(), new ExternalThemeTrieElement(new ThemeTrieElementRule(FontStyle.None, _A, _B))); }); test('respects incoming defaults 2', () => { @@ -196,12 +191,10 @@ suite('Theme resolving', () => { new ParsedThemeRule('', -1, FontStyle.None, null, null) ]); let colorMap = new ColorMap(); - const _NOT_SET = 0; - const _A = colorMap.getId('#000000'); - const _B = colorMap.getId('#ffffff'); + const _A = colorMap.getId('000000'); + const _B = colorMap.getId('ffffff'); assert.deepEqual(actual.getColorMap(), colorMap.getColorMap()); - assert.deepEqual(actual.getDefaults(), new ThemeTrieElementRule(FontStyle.None, _A, _B)); - assert.deepEqual(actual.getThemeTrieElement(), new ExternalThemeTrieElement(new ThemeTrieElementRule(FontStyle.NotSet, _NOT_SET, _NOT_SET))); + assert.deepEqual(actual.getThemeTrieElement(), new ExternalThemeTrieElement(new ThemeTrieElementRule(FontStyle.None, _A, _B))); }); test('respects incoming defaults 3', () => { @@ -209,109 +202,95 @@ suite('Theme resolving', () => { new ParsedThemeRule('', -1, FontStyle.Bold, null, null) ]); let colorMap = new ColorMap(); - const _NOT_SET = 0; - const _A = colorMap.getId('#000000'); - const _B = colorMap.getId('#ffffff'); + const _A = colorMap.getId('000000'); + const _B = colorMap.getId('ffffff'); assert.deepEqual(actual.getColorMap(), colorMap.getColorMap()); - assert.deepEqual(actual.getDefaults(), new ThemeTrieElementRule(FontStyle.Bold, _A, _B)); - assert.deepEqual(actual.getThemeTrieElement(), new ExternalThemeTrieElement(new ThemeTrieElementRule(FontStyle.NotSet, _NOT_SET, _NOT_SET))); + assert.deepEqual(actual.getThemeTrieElement(), new ExternalThemeTrieElement(new ThemeTrieElementRule(FontStyle.Bold, _A, _B))); }); test('respects incoming defaults 4', () => { let actual = Theme.createFromParsedTheme([ - new ParsedThemeRule('', -1, FontStyle.NotSet, '#ff0000', null) + new ParsedThemeRule('', -1, FontStyle.NotSet, 'ff0000', null) ]); let colorMap = new ColorMap(); - const _NOT_SET = 0; - const _A = colorMap.getId('#ff0000'); - const _B = colorMap.getId('#ffffff'); + const _A = colorMap.getId('ff0000'); + const _B = colorMap.getId('ffffff'); assert.deepEqual(actual.getColorMap(), colorMap.getColorMap()); - assert.deepEqual(actual.getDefaults(), new ThemeTrieElementRule(FontStyle.None, _A, _B)); - assert.deepEqual(actual.getThemeTrieElement(), new ExternalThemeTrieElement(new ThemeTrieElementRule(FontStyle.NotSet, _NOT_SET, _NOT_SET))); + assert.deepEqual(actual.getThemeTrieElement(), new ExternalThemeTrieElement(new ThemeTrieElementRule(FontStyle.None, _A, _B))); }); test('respects incoming defaults 5', () => { let actual = Theme.createFromParsedTheme([ - new ParsedThemeRule('', -1, FontStyle.NotSet, null, '#ff0000') + new ParsedThemeRule('', -1, FontStyle.NotSet, null, 'ff0000') ]); let colorMap = new ColorMap(); - const _NOT_SET = 0; - const _A = colorMap.getId('#000000'); - const _B = colorMap.getId('#ff0000'); + const _A = colorMap.getId('000000'); + const _B = colorMap.getId('ff0000'); assert.deepEqual(actual.getColorMap(), colorMap.getColorMap()); - assert.deepEqual(actual.getDefaults(), new ThemeTrieElementRule(FontStyle.None, _A, _B)); - assert.deepEqual(actual.getThemeTrieElement(), new ExternalThemeTrieElement(new ThemeTrieElementRule(FontStyle.NotSet, _NOT_SET, _NOT_SET))); + assert.deepEqual(actual.getThemeTrieElement(), new ExternalThemeTrieElement(new ThemeTrieElementRule(FontStyle.None, _A, _B))); }); test('can merge incoming defaults', () => { let actual = Theme.createFromParsedTheme([ - new ParsedThemeRule('', -1, FontStyle.NotSet, null, '#ff0000'), - new ParsedThemeRule('', -1, FontStyle.NotSet, '#00ff00', null), + new ParsedThemeRule('', -1, FontStyle.NotSet, null, 'ff0000'), + new ParsedThemeRule('', -1, FontStyle.NotSet, '00ff00', null), new ParsedThemeRule('', -1, FontStyle.Bold, null, null), ]); let colorMap = new ColorMap(); - const _NOT_SET = 0; - const _A = colorMap.getId('#00ff00'); - const _B = colorMap.getId('#ff0000'); + const _A = colorMap.getId('00ff00'); + const _B = colorMap.getId('ff0000'); assert.deepEqual(actual.getColorMap(), colorMap.getColorMap()); - assert.deepEqual(actual.getDefaults(), new ThemeTrieElementRule(FontStyle.Bold, _A, _B)); - assert.deepEqual(actual.getThemeTrieElement(), new ExternalThemeTrieElement(new ThemeTrieElementRule(FontStyle.NotSet, _NOT_SET, _NOT_SET))); + assert.deepEqual(actual.getThemeTrieElement(), new ExternalThemeTrieElement(new ThemeTrieElementRule(FontStyle.Bold, _A, _B))); }); test('defaults are inherited', () => { let actual = Theme.createFromParsedTheme([ - new ParsedThemeRule('', -1, FontStyle.NotSet, '#F8F8F2', '#272822'), - new ParsedThemeRule('var', -1, FontStyle.NotSet, '#ff0000', null) + new ParsedThemeRule('', -1, FontStyle.NotSet, 'F8F8F2', '272822'), + new ParsedThemeRule('var', -1, FontStyle.NotSet, 'ff0000', null) ]); let colorMap = new ColorMap(); - const _NOT_SET = 0; - const _A = colorMap.getId('#F8F8F2'); - const _B = colorMap.getId('#272822'); - const _C = colorMap.getId('#ff0000'); + const _A = colorMap.getId('F8F8F2'); + const _B = colorMap.getId('272822'); + const _C = colorMap.getId('ff0000'); assert.deepEqual(actual.getColorMap(), colorMap.getColorMap()); - assert.deepEqual(actual.getDefaults(), new ThemeTrieElementRule(FontStyle.None, _A, _B)); - let root = new ExternalThemeTrieElement(new ThemeTrieElementRule(FontStyle.NotSet, _NOT_SET, _NOT_SET), { - 'var': new ExternalThemeTrieElement(new ThemeTrieElementRule(FontStyle.NotSet, _C, _NOT_SET)) + let root = new ExternalThemeTrieElement(new ThemeTrieElementRule(FontStyle.None, _A, _B), { + 'var': new ExternalThemeTrieElement(new ThemeTrieElementRule(FontStyle.None, _C, _B)) }); assert.deepEqual(actual.getThemeTrieElement(), root); }); test('same rules get merged', () => { let actual = Theme.createFromParsedTheme([ - new ParsedThemeRule('', -1, FontStyle.NotSet, '#F8F8F2', '#272822'), + new ParsedThemeRule('', -1, FontStyle.NotSet, 'F8F8F2', '272822'), new ParsedThemeRule('var', 1, FontStyle.Bold, null, null), - new ParsedThemeRule('var', 0, FontStyle.NotSet, '#ff0000', null), + new ParsedThemeRule('var', 0, FontStyle.NotSet, 'ff0000', null), ]); let colorMap = new ColorMap(); - const _NOT_SET = 0; - const _A = colorMap.getId('#F8F8F2'); - const _B = colorMap.getId('#272822'); - const _C = colorMap.getId('#ff0000'); + const _A = colorMap.getId('F8F8F2'); + const _B = colorMap.getId('272822'); + const _C = colorMap.getId('ff0000'); assert.deepEqual(actual.getColorMap(), colorMap.getColorMap()); - assert.deepEqual(actual.getDefaults(), new ThemeTrieElementRule(FontStyle.None, _A, _B)); - let root = new ExternalThemeTrieElement(new ThemeTrieElementRule(FontStyle.NotSet, _NOT_SET, _NOT_SET), { - 'var': new ExternalThemeTrieElement(new ThemeTrieElementRule(FontStyle.Bold, _C, _NOT_SET)) + let root = new ExternalThemeTrieElement(new ThemeTrieElementRule(FontStyle.None, _A, _B), { + 'var': new ExternalThemeTrieElement(new ThemeTrieElementRule(FontStyle.Bold, _C, _B)) }); assert.deepEqual(actual.getThemeTrieElement(), root); }); test('rules are inherited 1', () => { let actual = Theme.createFromParsedTheme([ - new ParsedThemeRule('', -1, FontStyle.NotSet, '#F8F8F2', '#272822'), - new ParsedThemeRule('var', -1, FontStyle.Bold, '#ff0000', null), - new ParsedThemeRule('var.identifier', -1, FontStyle.NotSet, '#00ff00', null), + new ParsedThemeRule('', -1, FontStyle.NotSet, 'F8F8F2', '272822'), + new ParsedThemeRule('var', -1, FontStyle.Bold, 'ff0000', null), + new ParsedThemeRule('var.identifier', -1, FontStyle.NotSet, '00ff00', null), ]); let colorMap = new ColorMap(); - const _NOT_SET = 0; - const _A = colorMap.getId('#F8F8F2'); - const _B = colorMap.getId('#272822'); - const _C = colorMap.getId('#ff0000'); - const _D = colorMap.getId('#00ff00'); + const _A = colorMap.getId('F8F8F2'); + const _B = colorMap.getId('272822'); + const _C = colorMap.getId('ff0000'); + const _D = colorMap.getId('00ff00'); assert.deepEqual(actual.getColorMap(), colorMap.getColorMap()); - assert.deepEqual(actual.getDefaults(), new ThemeTrieElementRule(FontStyle.None, _A, _B)); - let root = new ExternalThemeTrieElement(new ThemeTrieElementRule(FontStyle.NotSet, _NOT_SET, _NOT_SET), { - 'var': new ExternalThemeTrieElement(new ThemeTrieElementRule(FontStyle.Bold, _C, _NOT_SET), { - 'identifier': new ExternalThemeTrieElement(new ThemeTrieElementRule(FontStyle.Bold, _D, _NOT_SET)) + let root = new ExternalThemeTrieElement(new ThemeTrieElementRule(FontStyle.None, _A, _B), { + 'var': new ExternalThemeTrieElement(new ThemeTrieElementRule(FontStyle.Bold, _C, _B), { + 'identifier': new ExternalThemeTrieElement(new ThemeTrieElementRule(FontStyle.Bold, _D, _B)) }) }); assert.deepEqual(actual.getThemeTrieElement(), root); @@ -319,35 +298,33 @@ suite('Theme resolving', () => { test('rules are inherited 2', () => { let actual = Theme.createFromParsedTheme([ - new ParsedThemeRule('', -1, FontStyle.NotSet, '#F8F8F2', '#272822'), - new ParsedThemeRule('var', -1, FontStyle.Bold, '#ff0000', null), - new ParsedThemeRule('var.identifier', -1, FontStyle.NotSet, '#00ff00', null), - new ParsedThemeRule('constant', 4, FontStyle.Italic, '#100000', null), - new ParsedThemeRule('constant.numeric', 5, FontStyle.NotSet, '#200000', null), + new ParsedThemeRule('', -1, FontStyle.NotSet, 'F8F8F2', '272822'), + new ParsedThemeRule('var', -1, FontStyle.Bold, 'ff0000', null), + new ParsedThemeRule('var.identifier', -1, FontStyle.NotSet, '00ff00', null), + new ParsedThemeRule('constant', 4, FontStyle.Italic, '100000', null), + new ParsedThemeRule('constant.numeric', 5, FontStyle.NotSet, '200000', null), new ParsedThemeRule('constant.numeric.hex', 6, FontStyle.Bold, null, null), new ParsedThemeRule('constant.numeric.oct', 7, FontStyle.Bold | FontStyle.Italic | FontStyle.Underline, null, null), - new ParsedThemeRule('constant.numeric.dec', 8, FontStyle.None, '#300000', null), + new ParsedThemeRule('constant.numeric.dec', 8, FontStyle.None, '300000', null), ]); let colorMap = new ColorMap(); - const _NOT_SET = 0; - const _A = colorMap.getId('#F8F8F2'); - const _B = colorMap.getId('#272822'); - const _C = colorMap.getId('#100000'); - const _D = colorMap.getId('#200000'); - const _E = colorMap.getId('#300000'); - const _F = colorMap.getId('#ff0000'); - const _G = colorMap.getId('#00ff00'); + const _A = colorMap.getId('F8F8F2'); + const _B = colorMap.getId('272822'); + const _C = colorMap.getId('100000'); + const _D = colorMap.getId('200000'); + const _E = colorMap.getId('300000'); + const _F = colorMap.getId('ff0000'); + const _G = colorMap.getId('00ff00'); assert.deepEqual(actual.getColorMap(), colorMap.getColorMap()); - assert.deepEqual(actual.getDefaults(), new ThemeTrieElementRule(FontStyle.None, _A, _B)); - let root = new ExternalThemeTrieElement(new ThemeTrieElementRule(FontStyle.NotSet, _NOT_SET, _NOT_SET), { - 'var': new ExternalThemeTrieElement(new ThemeTrieElementRule(FontStyle.Bold, _F, _NOT_SET), { - 'identifier': new ExternalThemeTrieElement(new ThemeTrieElementRule(FontStyle.Bold, _G, _NOT_SET)) + let root = new ExternalThemeTrieElement(new ThemeTrieElementRule(FontStyle.None, _A, _B), { + 'var': new ExternalThemeTrieElement(new ThemeTrieElementRule(FontStyle.Bold, _F, _B), { + 'identifier': new ExternalThemeTrieElement(new ThemeTrieElementRule(FontStyle.Bold, _G, _B)) }), - 'constant': new ExternalThemeTrieElement(new ThemeTrieElementRule(FontStyle.Italic, _C, _NOT_SET), { - 'numeric': new ExternalThemeTrieElement(new ThemeTrieElementRule(FontStyle.Italic, _D, _NOT_SET), { - 'hex': new ExternalThemeTrieElement(new ThemeTrieElementRule(FontStyle.Bold, _D, _NOT_SET)), - 'oct': new ExternalThemeTrieElement(new ThemeTrieElementRule(FontStyle.Bold | FontStyle.Italic | FontStyle.Underline, _D, _NOT_SET)), - 'dec': new ExternalThemeTrieElement(new ThemeTrieElementRule(FontStyle.None, _E, _NOT_SET)), + 'constant': new ExternalThemeTrieElement(new ThemeTrieElementRule(FontStyle.Italic, _C, _B), { + 'numeric': new ExternalThemeTrieElement(new ThemeTrieElementRule(FontStyle.Italic, _D, _B), { + 'hex': new ExternalThemeTrieElement(new ThemeTrieElementRule(FontStyle.Bold, _D, _B)), + 'oct': new ExternalThemeTrieElement(new ThemeTrieElementRule(FontStyle.Bold | FontStyle.Italic | FontStyle.Underline, _D, _B)), + 'dec': new ExternalThemeTrieElement(new ThemeTrieElementRule(FontStyle.None, _E, _B)), }) }) }); diff --git a/src/vs/editor/test/common/modes/textToHtmlTokenizer.test.ts b/src/vs/editor/test/common/modes/textToHtmlTokenizer.test.ts index 8213e9febf0..85ca8b122ec 100644 --- a/src/vs/editor/test/common/modes/textToHtmlTokenizer.test.ts +++ b/src/vs/editor/test/common/modes/textToHtmlTokenizer.test.ts @@ -5,78 +5,98 @@ 'use strict'; import * as assert from 'assert'; -import { TokenizationRegistry, IState, ILineTokens } from 'vs/editor/common/modes'; -import { tokenizeToHtmlContent } from 'vs/editor/common/modes/textToHtmlTokenizer'; +import { TokenizationRegistry, IState, ILineTokens3, LanguageIdentifier, ColorId, MetadataConsts } from 'vs/editor/common/modes'; +import { tokenizeToString } from 'vs/editor/common/modes/textToHtmlTokenizer'; import { MockMode } from 'vs/editor/test/common/mocks/mockMode'; -import { Token } from 'vs/editor/common/core/token'; suite('Editor Modes - textToHtmlTokenizer', () => { - test('TextToHtmlTokenizer', () => { - var mode = new Mode(); - var result = tokenizeToHtmlContent('.abc..def...gh', mode.getId()); + function toStr(pieces: { className: string; text: string }[]): string { + let resultArr = pieces.map((t) => `${t.text}`); + return resultArr.join(''); + } - assert.ok(!!result); + test('TextToHtmlTokenizer 1', () => { + let mode = new Mode(); - var children = result.children; - assert.equal(children.length, 6); + let actual = tokenizeToString('.abc..def...gh', mode.getId()); + let expected = [ + { className: 'mtk7', text: '.' }, + { className: 'mtk9', text: 'abc' }, + { className: 'mtk7', text: '..' }, + { className: 'mtk9', text: 'def' }, + { className: 'mtk7', text: '...' }, + { className: 'mtk9', text: 'gh' }, + ]; + let expectedStr = `
${toStr(expected)}
`; - assert.equal(children[0].text, '.'); - assert.equal(children[0].className, 'token'); - assert.equal(children[0].tagName, 'span'); + assert.equal(actual, expectedStr); - assert.equal(children[1].text, 'abc'); - assert.equal(children[1].className, 'token text'); - assert.equal(children[1].tagName, 'span'); + mode.dispose(); + }); - assert.equal(children[2].text, '..'); - assert.equal(children[2].className, 'token'); - assert.equal(children[2].tagName, 'span'); + test('TextToHtmlTokenizer 2', () => { + let mode = new Mode(); - assert.equal(children[3].text, 'def'); - assert.equal(children[3].className, 'token text'); - assert.equal(children[3].tagName, 'span'); + let actual = tokenizeToString('.abc..def...gh\n.abc..def...gh', mode.getId()); + let expected1 = [ + { className: 'mtk7', text: '.' }, + { className: 'mtk9', text: 'abc' }, + { className: 'mtk7', text: '..' }, + { className: 'mtk9', text: 'def' }, + { className: 'mtk7', text: '...' }, + { className: 'mtk9', text: 'gh' }, + ]; + let expected2 = [ + { className: 'mtk7', text: '.' }, + { className: 'mtk9', text: 'abc' }, + { className: 'mtk7', text: '..' }, + { className: 'mtk9', text: 'def' }, + { className: 'mtk7', text: '...' }, + { className: 'mtk9', text: 'gh' }, + ]; + let expectedStr1 = toStr(expected1); + let expectedStr2 = toStr(expected2); + let expectedStr = `
${expectedStr1}
${expectedStr2}
`; - assert.equal(children[4].text, '...'); - assert.equal(children[4].className, 'token'); - assert.equal(children[4].tagName, 'span'); + assert.equal(actual, expectedStr); - assert.equal(children[5].text, 'gh'); - assert.equal(children[5].className, 'token text'); - assert.equal(children[5].tagName, 'span'); - - result = tokenizeToHtmlContent('.abc..def...gh\n.abc..def...gh', mode.getId()); - - assert.ok(!!result); - - children = result.children; - assert.equal(children.length, 12 + 1 /* +1 for the line break */); - - assert.equal(children[6].tagName, 'br'); + mode.dispose(); }); }); class Mode extends MockMode { + + private static _id = new LanguageIdentifier('textToHtmlTokenizerMode', 3); + constructor() { - super(); - TokenizationRegistry.register(this.getId(), { + super(Mode._id); + this._register(TokenizationRegistry.register(this.getId(), { getInitialState: (): IState => null, - tokenize: (line: string, state: IState): ILineTokens => { - let tokens: Token[] = []; + tokenize: undefined, + tokenize3: (line: string, state: IState): ILineTokens3 => { + let tokensArr: number[] = []; + let prevColor: ColorId = -1; for (let i = 0; i < line.length; i++) { - let chr = line.charAt(i); - let type = chr === '.' ? '' : 'text'; - if (tokens.length > 0 && tokens[tokens.length - 1].type === type) { - continue; + let colorId = line.charAt(i) === '.' ? 7 : 9; + if (prevColor !== colorId) { + tokensArr.push(i); + tokensArr.push(( + colorId << MetadataConsts.FOREGROUND_OFFSET + ) >>> 0); } - tokens.push(new Token(i, type)); + prevColor = colorId; + } + + let tokens = new Uint32Array(tokensArr.length); + for (let i = 0; i < tokens.length; i++) { + tokens[i] = tokensArr[i]; } return { tokens: tokens, - endState: null, - modeTransitions: null + endState: null }; } - }); + })); } } diff --git a/src/vs/editor/test/common/modesTestUtils.ts b/src/vs/editor/test/common/modesTestUtils.ts index 2b3701fb005..fd73790e008 100644 --- a/src/vs/editor/test/common/modesTestUtils.ts +++ b/src/vs/editor/test/common/modesTestUtils.ts @@ -4,37 +4,31 @@ *--------------------------------------------------------------------------------------------*/ 'use strict'; -import { ModeTransition } from 'vs/editor/common/core/modeTransition'; -import { Token } from 'vs/editor/common/core/token'; import { LineTokens } from 'vs/editor/common/core/lineTokens'; -import { TokensBinaryEncoding, TokensInflatorMap } from 'vs/editor/common/model/tokensBinaryEncoding'; import { createScopedLineTokens, ScopedLineTokens } from 'vs/editor/common/modes/supports'; - -export function createFakeLineTokens(line: string, modeId: string, tokens: Token[], modeTransitions: ModeTransition[]): LineTokens { - let map = new TokensInflatorMap(modeId); - let deflatedTokens = TokensBinaryEncoding.deflateArr(map, tokens); - return new LineTokens(map, deflatedTokens, modeTransitions, line); -} +import { StandardTokenType, MetadataConsts } from 'vs/editor/common/modes'; export interface TokenText { text: string; - type: string; + type: StandardTokenType; } -export function createFakeScopedLineTokens(modeId: string, tokens: TokenText[]): ScopedLineTokens { - return createScopedLineTokens(createLineContextFromTokenText(modeId, tokens), 0); -} - -function createLineContextFromTokenText(modeId: string, tokens: TokenText[]): LineTokens { +export function createFakeScopedLineTokens(rawTokens: TokenText[]): ScopedLineTokens { + let tokens = new Uint32Array(rawTokens.length << 1); let line = ''; - let processedTokens: Token[] = []; - let indexSoFar = 0; - for (let i = 0; i < tokens.length; ++i) { - processedTokens.push(new Token(indexSoFar, tokens[i].type)); - line += tokens[i].text; - indexSoFar += tokens[i].text.length; + for (let i = 0, len = rawTokens.length; i < len; i++) { + let rawToken = rawTokens[i]; + + let startOffset = line.length; + let metadata = ( + (rawToken.type << MetadataConsts.TOKEN_TYPE_OFFSET) + ) >>> 0; + + tokens[(i << 1)] = startOffset; + tokens[(i << 1) + 1] = metadata; + line += rawToken.text; } - return createFakeLineTokens(line, modeId, processedTokens, [new ModeTransition(0, modeId)]); + return createScopedLineTokens(new LineTokens([], tokens, line), 0); } diff --git a/src/vs/monaco.d.ts b/src/vs/monaco.d.ts index ba7d847ed8e..5fccd649a67 100644 --- a/src/vs/monaco.d.ts +++ b/src/vs/monaco.d.ts @@ -2062,8 +2062,14 @@ declare module monaco.editor { readonly uri: Uri; /** * Get the language associated with this model. + * TODO@tokenization + * @deprecated */ getModeId(): string; + /** + * Get the language associated with this model. + */ + getLanguageIdentifier(): languages.LanguageIdentifier; /** * Get the word under or besides `position`. * @param position The position to look for a word. @@ -2086,12 +2092,20 @@ declare module monaco.editor { export interface ITokenizedModel extends ITextModel { /** * Get the current language mode associated with the model. + * TODO@tokenization + * @deprecated */ getMode(): languages.IMode; /** * Get the language associated with this model. + * TODO@tokenization + * @deprecated */ getModeId(): string; + /** + * Get the language associated with this model. + */ + getLanguageIdentifier(): languages.LanguageIdentifier; /** * Get the word under or besides `position`. * @param position The position to look for a word. @@ -4230,20 +4244,30 @@ declare module monaco.languages { */ removeText?: number; } + /** + * An identifier for a registered language. + */ + export class LanguageIdentifier { + public readonly sid: string; + public readonly iid: number; + + constructor(sid: string, iid: number); + } /** * A mode. Will soon be obsolete. */ export interface IMode { getId(): string; + getLanguageIdentifier(): LanguageIdentifier; } /** - * A token. Only supports a single scope, but will soon support a scope array. + * A token. */ export interface IToken { startIndex: number; - scopes: string | string[]; + scopes: string; } /** diff --git a/src/vs/workbench/api/node/mainThreadLanguageFeatures.ts b/src/vs/workbench/api/node/mainThreadLanguageFeatures.ts index 275a9350c95..c49f0b41b32 100644 --- a/src/vs/workbench/api/node/mainThreadLanguageFeatures.ts +++ b/src/vs/workbench/api/node/mainThreadLanguageFeatures.ts @@ -19,20 +19,24 @@ import { ExtHostContext, MainThreadLanguageFeaturesShape, ExtHostLanguageFeature import { LanguageConfigurationRegistry } from 'vs/editor/common/modes/languageConfigurationRegistry'; import { LanguageConfiguration } from 'vs/editor/common/modes/languageConfiguration'; import { IHeapService } from './mainThreadHeapService'; +import { IModeService } from 'vs/editor/common/services/modeService'; export class MainThreadLanguageFeatures extends MainThreadLanguageFeaturesShape { private _proxy: ExtHostLanguageFeaturesShape; private _heapService: IHeapService; + private _modeService: IModeService; private _registrations: { [handle: number]: IDisposable; } = Object.create(null); constructor( @IThreadService threadService: IThreadService, - @IHeapService heapService: IHeapService + @IHeapService heapService: IHeapService, + @IModeService modeService: IModeService, ) { super(); this._proxy = threadService.get(ExtHostContext.ExtHostLanguageFeatures); this._heapService = heapService; + this._modeService = modeService; } $unregister(handle: number): TPromise { @@ -255,7 +259,11 @@ export class MainThreadLanguageFeatures extends MainThreadLanguageFeaturesShape }; } - this._registrations[handle] = LanguageConfigurationRegistry.register(languageId, configuration); + let languageIdentifier = this._modeService.getLanguageIdentifier(languageId); + if (languageIdentifier) { + this._registrations[handle] = LanguageConfigurationRegistry.register(languageIdentifier, configuration); + } + return undefined; } diff --git a/src/vs/workbench/parts/emmet/node/editorAccessor.ts b/src/vs/workbench/parts/emmet/node/editorAccessor.ts index 1b6703e5e7d..bbe2427c51d 100644 --- a/src/vs/workbench/parts/emmet/node/editorAccessor.ts +++ b/src/vs/workbench/parts/emmet/node/editorAccessor.ts @@ -10,6 +10,7 @@ import strings = require('vs/base/common/strings'); import snippets = require('vs/editor/contrib/snippet/common/snippet'); import { Range } from 'vs/editor/common/core/range'; import { SnippetController } from 'vs/editor/contrib/snippet/common/snippetController'; +import { LanguageId, LanguageIdentifier } from 'vs/editor/common/modes'; import emmet = require('emmet'); @@ -18,8 +19,13 @@ export interface IGrammarContributions { getGrammar(mode: string): string; } +export interface ILanguageIdentifierResolver { + getLanguageIdentifier(modeId: LanguageId): LanguageIdentifier; +} + export class EditorAccessor implements emmet.Editor { + private _languageIdentifierResolver: ILanguageIdentifierResolver; private _editor: ICommonCodeEditor; private _syntaxProfiles: any; private _excludedLanguages: any; @@ -29,7 +35,8 @@ export class EditorAccessor implements emmet.Editor { private emmetSupportedModes = ['html', 'xhtml', 'css', 'xml', 'xsl', 'haml', 'jade', 'jsx', 'slim', 'scss', 'sass', 'less', 'stylus', 'styl']; - constructor(editor: ICommonCodeEditor, syntaxProfiles: any, excludedLanguages: String[], grammars: IGrammarContributions) { + constructor(languageIdentifierResolver: ILanguageIdentifierResolver, editor: ICommonCodeEditor, syntaxProfiles: any, excludedLanguages: String[], grammars: IGrammarContributions) { + this._languageIdentifierResolver = languageIdentifierResolver; this._editor = editor; this._syntaxProfiles = syntaxProfiles; this._excludedLanguages = excludedLanguages; @@ -137,7 +144,8 @@ export class EditorAccessor implements emmet.Editor { public getSyntax(): string { let position = this._editor.getSelection().getStartPosition(); - let modeId = this._editor.getModel().getModeIdAtPosition(position.lineNumber, position.column); + let languageId = this._editor.getModel().getLanguageIdAtPosition(position.lineNumber, position.column); + let modeId = this._languageIdentifierResolver.getLanguageIdentifier(languageId).sid; let syntax = modeId.split('.').pop(); if (this._excludedLanguages.indexOf(syntax) !== -1) { diff --git a/src/vs/workbench/parts/emmet/node/emmetActions.ts b/src/vs/workbench/parts/emmet/node/emmetActions.ts index f3cde468d4d..4b84a0ba694 100644 --- a/src/vs/workbench/parts/emmet/node/emmetActions.ts +++ b/src/vs/workbench/parts/emmet/node/emmetActions.ts @@ -11,6 +11,7 @@ import { EditorAction, ServicesAccessor } from 'vs/editor/common/editorCommonExt import { ICommandKeybindingsOptions } from 'vs/editor/common/config/config'; import { IInstantiationService } from 'vs/platform/instantiation/common/instantiation'; import { grammarsExtPoint, ITMSyntaxExtensionPoint } from 'vs/editor/node/textMate/TMSyntax'; +import { IModeService } from 'vs/editor/common/services/modeService'; import { EditorAccessor, IGrammarContributions } from 'vs/workbench/parts/emmet/node/editorAccessor'; import { IConfigurationService } from 'vs/platform/configuration/common/configuration'; @@ -146,10 +147,12 @@ export abstract class EmmetEditorAction extends EditorAction { const configurationService = accessor.get(IConfigurationService); const instantiationService = accessor.get(IInstantiationService); const extensionService = accessor.get(IExtensionService); + const modeService = accessor.get(IModeService); return this._withGrammarContributions(extensionService).then((grammarContributions) => { let editorAccessor = new EditorAccessor( + modeService, editor, configurationService.getConfiguration().emmet.syntaxProfiles, configurationService.getConfiguration().emmet.excludeLanguages, diff --git a/src/vs/workbench/parts/emmet/test/node/editorAccessor.test.ts b/src/vs/workbench/parts/emmet/test/node/editorAccessor.test.ts index fda5cb55426..fc27d82bdc4 100644 --- a/src/vs/workbench/parts/emmet/test/node/editorAccessor.test.ts +++ b/src/vs/workbench/parts/emmet/test/node/editorAccessor.test.ts @@ -5,9 +5,10 @@ 'use strict'; -import { EditorAccessor, IGrammarContributions } from 'vs/workbench/parts/emmet/node/editorAccessor'; +import { EditorAccessor, ILanguageIdentifierResolver, IGrammarContributions } from 'vs/workbench/parts/emmet/node/editorAccessor'; import { withMockCodeEditor } from 'vs/editor/test/common/mocks/mockCodeEditor'; import assert = require('assert'); +import { LanguageId, LanguageIdentifier } from 'vs/editor/common/modes'; // // To run the emmet tests only change .vscode/launch.json @@ -49,8 +50,17 @@ suite('Emmet', () => { withMockCodeEditor([], {}, (editor) => { function testIsEnabled(mode: string, scopeName: string, isEnabled = true, profile = {}, excluded = []) { - editor.getModel().setMode(mode); - let editorAccessor = new EditorAccessor(editor, profile, excluded, new MockGrammarContributions(scopeName)); + const languageIdentifier = new LanguageIdentifier(mode, 73); + const languageIdentifierResolver: ILanguageIdentifierResolver = { + getLanguageIdentifier: (languageId: LanguageId) => { + if (languageId === 73) { + return languageIdentifier; + } + throw new Error('Unexpected'); + } + }; + editor.getModel().setMode(languageIdentifier); + let editorAccessor = new EditorAccessor(languageIdentifierResolver, editor, profile, excluded, new MockGrammarContributions(scopeName)); assert.equal(editorAccessor.isEmmetEnabledMode(), isEnabled); } @@ -90,9 +100,18 @@ suite('Emmet', () => { ], {}, (editor) => { function testIsEnabled(mode: string, scopeName: string, isEnabled = true, profile = {}, excluded = []) { - editor.getModel().setMode(mode); + const languageIdentifier = new LanguageIdentifier(mode, 73); + const languageIdentifierResolver: ILanguageIdentifierResolver = { + getLanguageIdentifier: (languageId: LanguageId) => { + if (languageId === 73) { + return languageIdentifier; + } + throw new Error('Unexpected'); + } + }; + editor.getModel().setMode(languageIdentifier); editor.setPosition({ lineNumber: 1, column: 3 }); - let editorAccessor = new EditorAccessor(editor, profile, excluded, new MockGrammarContributions(scopeName)); + let editorAccessor = new EditorAccessor(languageIdentifierResolver, editor, profile, excluded, new MockGrammarContributions(scopeName)); assert.equal(editorAccessor.isEmmetEnabledMode(), isEnabled); } @@ -105,8 +124,17 @@ suite('Emmet', () => { withMockCodeEditor([], {}, (editor) => { function testSyntax(mode: string, scopeName: string, expectedSyntax: string, profile = {}, excluded = []) { - editor.getModel().setMode(mode); - let editorAccessor = new EditorAccessor(editor, profile, excluded, new MockGrammarContributions(scopeName)); + const languageIdentifier = new LanguageIdentifier(mode, 73); + const languageIdentifierResolver: ILanguageIdentifierResolver = { + getLanguageIdentifier: (languageId: LanguageId) => { + if (languageId === 73) { + return languageIdentifier; + } + throw new Error('Unexpected'); + } + }; + editor.getModel().setMode(languageIdentifier); + let editorAccessor = new EditorAccessor(languageIdentifierResolver, editor, profile, excluded, new MockGrammarContributions(scopeName)); assert.equal(editorAccessor.getSyntax(), expectedSyntax); } diff --git a/src/vs/workbench/parts/snippets/electron-browser/snippetsTracker.ts b/src/vs/workbench/parts/snippets/electron-browser/snippetsTracker.ts index 1dce853f6a9..8f2d8e84874 100644 --- a/src/vs/workbench/parts/snippets/electron-browser/snippetsTracker.ts +++ b/src/vs/workbench/parts/snippets/electron-browser/snippetsTracker.ts @@ -18,6 +18,7 @@ import { IFileService } from 'vs/platform/files/common/files'; import { ILifecycleService } from 'vs/platform/lifecycle/common/lifecycle'; import { IEnvironmentService } from 'vs/platform/environment/common/environment'; import { watch, FSWatcher } from 'fs'; +import { IModeService } from 'vs/editor/common/services/modeService'; export class SnippetsTracker implements workbenchExt.IWorkbenchContribution { private static FILE_WATCH_DELAY = 200; @@ -30,6 +31,7 @@ export class SnippetsTracker implements workbenchExt.IWorkbenchContribution { constructor( @IFileService private fileService: IFileService, @ILifecycleService private lifecycleService: ILifecycleService, + @IModeService private modeService: IModeService, @IEnvironmentService environmentService: IEnvironmentService ) { this.snippetFolder = paths.join(environmentService.appSettingsHome, 'snippets'); @@ -70,7 +72,10 @@ export class SnippetsTracker implements workbenchExt.IWorkbenchContribution { return winjs.TPromise.join(snippetFiles.map(snippetFile => { var modeId = snippetFile.replace(/\.json$/, '').toLowerCase(); var snippetPath = paths.join(this.snippetFolder, snippetFile); - return readAndRegisterSnippets(modeId, snippetPath, localize('userSnippet', "User Snippet")); + let languageIdentifier = this.modeService.getLanguageIdentifier(modeId); + if (languageIdentifier) { + return readAndRegisterSnippets(languageIdentifier, snippetPath, localize('userSnippet', "User Snippet")); + } })); }); } diff --git a/src/vs/workbench/parts/themes/test/electron-browser/themes.test.contribution.ts b/src/vs/workbench/parts/themes/test/electron-browser/themes.test.contribution.ts index 95ecdfa2b74..260750d1379 100644 --- a/src/vs/workbench/parts/themes/test/electron-browser/themes.test.contribution.ts +++ b/src/vs/workbench/parts/themes/test/electron-browser/themes.test.contribution.ts @@ -8,8 +8,8 @@ import { TPromise } from 'vs/base/common/winjs.base'; import paths = require('vs/base/common/paths'); import URI from 'vs/base/common/uri'; -import { TextModelWithTokens } from 'vs/editor/common/model/textModelWithTokens'; -import { TextModel } from 'vs/editor/common/model/textModel'; +// import { TextModelWithTokens } from 'vs/editor/common/model/textModelWithTokens'; +// import { TextModel } from 'vs/editor/common/model/textModel'; import { IModeService } from 'vs/editor/common/services/modeService'; import pfs = require('vs/base/node/pfs'); import { CommandsRegistry } from 'vs/platform/commands/common/commands'; @@ -45,9 +45,9 @@ class Snapper { return element; } - private normalizeType(type: string): string { - return type.split('.').sort().join('.'); - } + // private normalizeType(type: string): string { + // return type.split('.').sort().join('.'); + // } private getStyle(testNode: Element, scope: string): string { @@ -116,25 +116,27 @@ class Snapper { public captureSyntaxTokens(fileName: string, content: string): TPromise { return this.modeService.getOrCreateModeByFilenameOrFirstLine(fileName).then(mode => { - let result: Data[] = []; - let model = new TextModelWithTokens([], TextModel.toRawText(content, TextModel.DEFAULT_CREATION_OPTIONS), mode.getId()); - for (let lineNumber = 1, lineCount = model.getLineCount(); lineNumber <= lineCount; lineNumber++) { - let lineTokens = model.getLineTokens(lineNumber, false); - let lineContent = model.getLineContent(lineNumber); + // TODO@tokenization + throw new Error('TODO@tokenization'); + // let result: Data[] = []; + // let model = new TextModelWithTokens([], TextModel.toRawText(content, TextModel.DEFAULT_CREATION_OPTIONS), mode.getId()); + // for (let lineNumber = 1, lineCount = model.getLineCount(); lineNumber <= lineCount; lineNumber++) { + // let lineTokens = model.getLineTokens(lineNumber, false); + // let lineContent = model.getLineContent(lineNumber); - for (let i = 0, len = lineTokens.getTokenCount(); i < len; i++) { - let tokenType = lineTokens.getTokenType(i); - let tokenStartOffset = lineTokens.getTokenStartOffset(i); - let tokenEndOffset = lineTokens.getTokenEndOffset(i); + // for (let i = 0, len = lineTokens.getTokenCount(); i < len; i++) { + // let tokenType = lineTokens.getTokenType(i); + // let tokenStartOffset = lineTokens.getTokenStartOffset(i); + // let tokenEndOffset = lineTokens.getTokenEndOffset(i); - result.push({ - c: lineContent.substring(tokenStartOffset, tokenEndOffset), - t: this.normalizeType(tokenType), - r: {} - }); - } - } - return this.appendThemeInformation(result); + // result.push({ + // c: lineContent.substring(tokenStartOffset, tokenEndOffset), + // t: this.normalizeType(tokenType), + // r: {} + // }); + // } + // } + // return this.appendThemeInformation(result); }); } }