diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 14bd4c852c7..2fbb67c9f37 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -410,9 +410,9 @@ "resolved": "https://registry.npmjs.org/vscode-debugprotocol/-/vscode-debugprotocol-1.15.0.tgz" }, "vscode-textmate": { - "version": "2.3.2", - "from": "vscode-textmate@2.3.2", - "resolved": "https://registry.npmjs.org/vscode-textmate/-/vscode-textmate-2.3.2.tgz" + "version": "3.0.0", + "from": "vscode-textmate@3.0.0", + "resolved": "https://registry.npmjs.org/vscode-textmate/-/vscode-textmate-3.0.0.tgz" }, "windows-foreground-love": { "version": "0.1.0", diff --git a/package.json b/package.json index d099f65135a..09ee3437ad7 100644 --- a/package.json +++ b/package.json @@ -33,7 +33,7 @@ "pty.js": "https://github.com/Tyriar/pty.js/tarball/c75c2dcb6dcad83b0cb3ef2ae42d0448fb912642", "semver": "4.3.6", "vscode-debugprotocol": "1.15.0", - "vscode-textmate": "2.3.2", + "vscode-textmate": "3.0.0", "winreg": "1.2.0", "xterm": "git+https://github.com/Tyriar/xterm.js.git#vscode-release/1.9", "yauzl": "2.3.1" diff --git a/src/typings/vscode-textmate.d.ts b/src/typings/vscode-textmate.d.ts new file mode 100644 index 00000000000..10fd8c39ea8 --- /dev/null +++ b/src/typings/vscode-textmate.d.ts @@ -0,0 +1,166 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +declare module "vscode-textmate" { + /** + * A single theme setting. + */ + export interface IRawThemeSetting { + readonly name?: string; + readonly scope?: string | string[]; + readonly settings: { + readonly fontStyle?: string; + readonly foreground?: string; + readonly background?: string; + }; + } + /** + * A TextMate theme. + */ + export interface IRawTheme { + readonly name?: string; + readonly settings: IRawThemeSetting[]; + } + /** + * A registry helper that can locate grammar file paths given scope names. + */ + export interface RegistryOptions { + theme?: IRawTheme; + getFilePath(scopeName: string): string; + getInjections?(scopeName: string): string[]; + } + /** + * A map from scope name to a language id. Please do not use language id 0. + */ + export interface IEmbeddedLanguagesMap { + [scopeName: string]: number; + } + export const enum StandardTokenType { + Other = 0, + Comment = 1, + String = 2, + RegEx = 4, + } + /** + * The registry that will hold all grammars. + */ + export class Registry { + private readonly _locator; + private readonly _syncRegistry; + constructor(locator?: RegistryOptions); + /** + * Change the theme. Once called, no previous `ruleStack` should be used anymore. + */ + setTheme(theme: IRawTheme): void; + /** + * Returns a lookup array for color ids. + */ + getColorMap(): string[]; + /** + * Load the grammar for `scopeName` and all referenced included grammars asynchronously. + * Please do not use language id 0. + */ + loadGrammarWithEmbeddedLanguages(initialScopeName: string, initialLanguage: number, embeddedLanguages: IEmbeddedLanguagesMap, callback: (err: any, grammar: IGrammar) => void): void; + /** + * Load the grammar for `scopeName` and all referenced included grammars asynchronously. + */ + loadGrammar(initialScopeName: string, callback: (err: any, grammar: IGrammar) => void): void; + private _loadGrammar(initialScopeName, callback); + /** + * Load the grammar at `path` synchronously. + */ + loadGrammarFromPathSync(path: string, initialLanguage?: number, embeddedLanguages?: IEmbeddedLanguagesMap): IGrammar; + /** + * Get the grammar for `scopeName`. The grammar must first be created via `loadGrammar` or `loadGrammarFromPathSync`. + */ + grammarForScopeName(scopeName: string, initialLanguage?: number, embeddedLanguages?: IEmbeddedLanguagesMap): IGrammar; + } + /** + * A grammar + */ + export interface IGrammar { + /** + * Tokenize `lineText` using previous line state `prevState`. + */ + tokenizeLine(lineText: string, prevState: StackElement): ITokenizeLineResult; + /** + * Tokenize `lineText` using previous line state `prevState`. + * The result contains the tokens in binary format, resolved with the following information: + * - language + * - token type (regex, string, comment, other) + * - font style + * - foreground color + * - background color + * e.g. for getting the languageId: `(metadata & MetadataConsts.LANGUAGEID_MASK) >>> MetadataConsts.LANGUAGEID_OFFSET` + */ + tokenizeLine2(lineText: string, prevState: StackElement): ITokenizeLineResult2; + } + export interface ITokenizeLineResult { + readonly tokens: IToken[]; + /** + * The `prevState` to be passed on to the next line tokenization. + */ + readonly ruleStack: StackElement; + } + /** + * Helpers to manage the "collapsed" metadata of an entire StackElement stack. + * The following assumptions have been made: + * - languageId < 256 => needs 8 bits + * - unique color count < 512 => needs 9 bits + * + * The binary format is: + * - ------------------------------------------- + * 3322 2222 2222 1111 1111 1100 0000 0000 + * 1098 7654 3210 9876 5432 1098 7654 3210 + * - ------------------------------------------- + * xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx + * bbbb bbbb bfff ffff ffFF FTTT LLLL LLLL + * - ------------------------------------------- + * - L = LanguageId (8 bits) + * - T = StandardTokenType (3 bits) + * - F = FontStyle (3 bits) + * - f = foreground color (9 bits) + * - b = background color (9 bits) + */ + export const enum MetadataConsts { + LANGUAGEID_MASK = 255, + TOKEN_TYPE_MASK = 1792, + FONT_STYLE_MASK = 14336, + FOREGROUND_MASK = 8372224, + BACKGROUND_MASK = 4286578688, + LANGUAGEID_OFFSET = 0, + TOKEN_TYPE_OFFSET = 8, + FONT_STYLE_OFFSET = 11, + FOREGROUND_OFFSET = 14, + BACKGROUND_OFFSET = 23, + } + export interface ITokenizeLineResult2 { + /** + * The tokens in binary format. Each token occupies two array indices. For token i: + * - at offset 2*i => startIndex + * - at offset 2*i + 1 => metadata + * + */ + readonly tokens: Uint32Array; + /** + * The `prevState` to be passed on to the next line tokenization. + */ + readonly ruleStack: StackElement; + } + export interface IToken { + startIndex: number; + readonly endIndex: number; + readonly scopes: string[]; + } + /** + * **IMPORTANT** - Immutable! + */ + export interface StackElement { + _stackElementBrand: void; + readonly depth: number; + equals(other: StackElement): boolean; + } + +} \ No newline at end of file diff --git a/src/vs/editor/node/textMate/TMSyntax.ts b/src/vs/editor/node/textMate/TMSyntax.ts index d60e5d696fe..d6957c02fda 100644 --- a/src/vs/editor/node/textMate/TMSyntax.ts +++ b/src/vs/editor/node/textMate/TMSyntax.ts @@ -438,12 +438,7 @@ export class DecodeMap { } function depth(stackElement: StackElement): number { - let result = 0; - while (stackElement) { - result++; - stackElement = stackElement._parent; - } - return result; + return stackElement.depth; } class Tokenizer { @@ -502,12 +497,14 @@ class Tokenizer { // TODO: replace with something like ruleStack.toDebugString function printRuleStack(ruleStack: StackElement): string[] { - let scopes = []; - while (ruleStack) { - scopes.push(ruleStack['_scopeName']); - ruleStack = ruleStack._parent; - } - return scopes; + // TODO@tokenization + throw new Error('Not implemented'); + // let scopes = []; + // while (ruleStack) { + // scopes.push(ruleStack['_scopeName']); + // ruleStack = ruleStack._parent; + // } + // return scopes; } export function decodeTextMateTokens(topLevelModeId: string, decodeMap: DecodeMap, line: string, offsetDelta: number, resultTokens: IToken[], resultState: TMState): RawLineTokens { diff --git a/src/vs/editor/node/textMate/vscode-textmate.d.ts b/src/vs/editor/node/textMate/vscode-textmate.d.ts deleted file mode 100644 index 481741bce33..00000000000 --- a/src/vs/editor/node/textMate/vscode-textmate.d.ts +++ /dev/null @@ -1,80 +0,0 @@ -/*--------------------------------------------------------------------------------------------- - * Copyright (c) Microsoft Corporation. All rights reserved. - * Licensed under the MIT License. See License.txt in the project root for license information. - *--------------------------------------------------------------------------------------------*/ - -declare module "vscode-textmate" { - - /** - * A registry helper that can locate grammar file paths given scope names. - */ - export interface IGrammarLocator { - getFilePath(scopeName: string): string; - getInjections?(scopeName: string): string[]; - } - - /** - * The registry that will hold all grammars. - */ - export class Registry { - - constructor(locator?: IGrammarLocator); - - /** - * Load the grammar for `scopeName` and all referenced included grammars asynchronously. - */ - public loadGrammar(scopeName: string, callback: (err: any, grammar: IGrammar) => void): void; - - /** - * Load the grammar at `path` synchronously. - */ - public loadGrammarFromPathSync(path: string): IGrammar; - - /** - * Get the grammar for `scopeName`. The grammar must first be created via `loadGrammar` or `loadGrammarFromPathSync`. - */ - public grammarForScopeName(scopeName: string): IGrammar; - } - - export interface IGrammarInfo { - fileTypes: string[]; - name: string; - scopeName: string; - firstLineMatch: string; - } - - /** - * A grammar - */ - export interface IGrammar { - /** - * Tokenize `lineText` using previous line state `prevState`. - */ - tokenizeLine(lineText: string, prevState: StackElement): ITokenizeLineResult; - } - - export interface ITokenizeLineResult { - tokens: IToken[]; - /** - * The `prevState` to be passed on to the next line tokenization. - */ - ruleStack: StackElement; - } - - export interface IToken { - startIndex: number; - endIndex: number; - scopes: string[]; - } - - /** - * Should not be used by consumers, as its shape might change at any time. - */ - export interface StackElement { - _stackElementBrand: void; - _parent: StackElement; - - equals(other: StackElement): boolean; - } - -} \ No newline at end of file