diff --git a/src/vs/editor/common/model/textModelTokens.ts b/src/vs/editor/common/model/textModelTokens.ts index 80cc3b4e829..f944e5e5f1d 100644 --- a/src/vs/editor/common/model/textModelTokens.ts +++ b/src/vs/editor/common/model/textModelTokens.ts @@ -511,14 +511,14 @@ function safeTokenize(languageIdCodec: ILanguageIdCodec, languageId: string, tok if (tokenizationSupport) { try { - r = tokenizationSupport.tokenizeEncoded(text, hasEOL, state.clone(), 0); + r = tokenizationSupport.tokenizeEncoded(text, hasEOL, state.clone()); } catch (e) { onUnexpectedError(e); } } if (!r) { - r = nullTokenizeEncoded(languageIdCodec.encodeLanguageId(languageId), text, state, 0); + r = nullTokenizeEncoded(languageIdCodec.encodeLanguageId(languageId), state); } LineTokens.convertToEndOffset(r.tokens, text.length); diff --git a/src/vs/editor/common/modes.ts b/src/vs/editor/common/modes.ts index b38301bcf4a..822ea6c6fad 100644 --- a/src/vs/editor/common/modes.ts +++ b/src/vs/editor/common/modes.ts @@ -201,10 +201,9 @@ export interface ITokenizationSupport { getInitialState(): IState; - // add offsetDelta to each of the returned indices - tokenize(line: string, hasEOL: boolean, state: IState, offsetDelta: number): TokenizationResult; + tokenize(line: string, hasEOL: boolean, state: IState): TokenizationResult; - tokenizeEncoded(line: string, hasEOL: boolean, state: IState, offsetDelta: number): EncodedTokenizationResult; + tokenizeEncoded(line: string, hasEOL: boolean, state: IState): EncodedTokenizationResult; } /** diff --git a/src/vs/editor/common/modes/nullMode.ts b/src/vs/editor/common/modes/nullMode.ts index dc654e480de..f9226149835 100644 --- a/src/vs/editor/common/modes/nullMode.ts +++ b/src/vs/editor/common/modes/nullMode.ts @@ -19,13 +19,13 @@ class NullStateImpl implements IState { export const NULL_STATE: IState = new NullStateImpl(); -export function nullTokenize(languageId: string, buffer: string, state: IState, deltaOffset: number): TokenizationResult { - return new TokenizationResult([new Token(deltaOffset, '', languageId)], state); +export function nullTokenize(languageId: string, state: IState): TokenizationResult { + return new TokenizationResult([new Token(0, '', languageId)], state); } -export function nullTokenizeEncoded(languageId: LanguageId, buffer: string, state: IState | null, deltaOffset: number): EncodedTokenizationResult { +export function nullTokenizeEncoded(languageId: LanguageId, state: IState | null): EncodedTokenizationResult { let tokens = new Uint32Array(2); - tokens[0] = deltaOffset; + tokens[0] = 0; tokens[1] = ( (languageId << MetadataConsts.LANGUAGEID_OFFSET) | (StandardTokenType.Other << MetadataConsts.TOKEN_TYPE_OFFSET) diff --git a/src/vs/editor/common/modes/textToHtmlTokenizer.ts b/src/vs/editor/common/modes/textToHtmlTokenizer.ts index f41c7cac62a..bfd5204b9f6 100644 --- a/src/vs/editor/common/modes/textToHtmlTokenizer.ts +++ b/src/vs/editor/common/modes/textToHtmlTokenizer.ts @@ -14,7 +14,7 @@ export type IReducedTokenizationSupport = Omit const fallback: IReducedTokenizationSupport = { getInitialState: () => NULL_STATE, - tokenizeEncoded: (buffer: string, hasEOL: boolean, state: IState, deltaOffset: number) => nullTokenizeEncoded(LanguageId.Null, buffer, state, deltaOffset) + tokenizeEncoded: (buffer: string, hasEOL: boolean, state: IState) => nullTokenizeEncoded(LanguageId.Null, state) }; export function tokenizeToStringSync(languageService: ILanguageService, text: string, languageId: string): string { @@ -136,7 +136,7 @@ export function _tokenizeToString(text: string, languageIdCodec: ILanguageIdCode result += `
`; } - const tokenizationResult = tokenizationSupport.tokenizeEncoded(line, true, currentState, 0); + const tokenizationResult = tokenizationSupport.tokenizeEncoded(line, true, currentState); LineTokens.convertToEndOffset(tokenizationResult.tokens, line.length); const lineTokens = new LineTokens(tokenizationResult.tokens, line, languageIdCodec); const viewLineTokens = lineTokens.inflate(); diff --git a/src/vs/editor/standalone/browser/colorizer.ts b/src/vs/editor/standalone/browser/colorizer.ts index a7dada0840d..34902014698 100644 --- a/src/vs/editor/standalone/browser/colorizer.ts +++ b/src/vs/editor/standalone/browser/colorizer.ts @@ -179,7 +179,7 @@ function _actualColorize(lines: string[], tabSize: number, tokenizationSupport: for (let i = 0, length = lines.length; i < length; i++) { let line = lines[i]; - let tokenizeResult = tokenizationSupport.tokenizeEncoded(line, true, state, 0); + let tokenizeResult = tokenizationSupport.tokenizeEncoded(line, true, state); LineTokens.convertToEndOffset(tokenizeResult.tokens, line.length); let lineTokens = new LineTokens(tokenizeResult.tokens, line, languageIdCodec); const isBasicASCII = ViewLineRenderingData.isBasicASCII(line, /* check for basic ASCII */true); diff --git a/src/vs/editor/standalone/browser/inspectTokens/inspectTokens.ts b/src/vs/editor/standalone/browser/inspectTokens/inspectTokens.ts index 284397862e8..b539711d549 100644 --- a/src/vs/editor/standalone/browser/inspectTokens/inspectTokens.ts +++ b/src/vs/editor/standalone/browser/inspectTokens/inspectTokens.ts @@ -138,8 +138,8 @@ function getSafeTokenizationSupport(languageIdCodec: ILanguageIdCodec, languageI const encodedLanguageId = languageIdCodec.encodeLanguageId(languageId); return { getInitialState: () => NULL_STATE, - tokenize: (line: string, hasEOL: boolean, state: IState, deltaOffset: number) => nullTokenize(languageId, line, state, deltaOffset), - tokenizeEncoded: (line: string, hasEOL: boolean, state: IState, deltaOffset: number) => nullTokenizeEncoded(encodedLanguageId, line, state, deltaOffset) + tokenize: (line: string, hasEOL: boolean, state: IState) => nullTokenize(languageId, state), + tokenizeEncoded: (line: string, hasEOL: boolean, state: IState) => nullTokenizeEncoded(encodedLanguageId, state) }; } @@ -297,8 +297,8 @@ class InspectTokensWidget extends Disposable implements IContentWidget { private _getTokensAtLine(lineNumber: number): ICompleteLineTokenization { let stateBeforeLine = this._getStateBeforeLine(lineNumber); - let tokenizationResult1 = this._tokenizationSupport.tokenize(this._model.getLineContent(lineNumber), true, stateBeforeLine, 0); - let tokenizationResult2 = this._tokenizationSupport.tokenizeEncoded(this._model.getLineContent(lineNumber), true, stateBeforeLine, 0); + let tokenizationResult1 = this._tokenizationSupport.tokenize(this._model.getLineContent(lineNumber), true, stateBeforeLine); + let tokenizationResult2 = this._tokenizationSupport.tokenizeEncoded(this._model.getLineContent(lineNumber), true, stateBeforeLine); return { startState: stateBeforeLine, @@ -312,7 +312,7 @@ class InspectTokensWidget extends Disposable implements IContentWidget { let state: IState = this._tokenizationSupport.getInitialState(); for (let i = 1; i < lineNumber; i++) { - let tokenizationResult = this._tokenizationSupport.tokenize(this._model.getLineContent(i), true, state, 0); + let tokenizationResult = this._tokenizationSupport.tokenize(this._model.getLineContent(i), true, state); state = tokenizationResult.endState; } diff --git a/src/vs/editor/standalone/browser/standaloneEditor.ts b/src/vs/editor/standalone/browser/standaloneEditor.ts index e2f918f37c4..2b5c0089449 100644 --- a/src/vs/editor/standalone/browser/standaloneEditor.ts +++ b/src/vs/editor/standalone/browser/standaloneEditor.ts @@ -274,7 +274,7 @@ function getSafeTokenizationSupport(language: string): Omit NULL_STATE, - tokenize: (line: string, hasEOL: boolean, state: modes.IState, deltaOffset: number) => nullTokenize(language, line, state, deltaOffset) + tokenize: (line: string, hasEOL: boolean, state: modes.IState) => nullTokenize(language, state) }; } @@ -291,7 +291,7 @@ export function tokenize(text: string, languageId: string): Token[][] { let state = tokenizationSupport.getInitialState(); for (let i = 0, len = lines.length; i < len; i++) { let line = lines[i]; - let tokenizationResult = tokenizationSupport.tokenize(line, true, state, 0); + let tokenizationResult = tokenizationSupport.tokenize(line, true, state); result[i] = tokenizationResult.tokens; state = tokenizationResult.endState; diff --git a/src/vs/editor/standalone/browser/standaloneLanguages.ts b/src/vs/editor/standalone/browser/standaloneLanguages.ts index d208dfdeb51..ea4904173ae 100644 --- a/src/vs/editor/standalone/browser/standaloneLanguages.ts +++ b/src/vs/editor/standalone/browser/standaloneLanguages.ts @@ -18,7 +18,7 @@ import { ILanguageExtensionPoint, ILanguageService } from 'vs/editor/common/serv import * as standaloneEnums from 'vs/editor/common/standalone/standaloneEnums'; import { StaticServices } from 'vs/editor/standalone/browser/standaloneServices'; import { compile } from 'vs/editor/standalone/common/monarch/monarchCompile'; -import { createTokenizationSupport } from 'vs/editor/standalone/common/monarch/monarchLexer'; +import { MonarchTokenizer } from 'vs/editor/standalone/common/monarch/monarchLexer'; import { IMonarchLanguage } from 'vs/editor/standalone/common/monarch/monarchTypes'; import { IStandaloneThemeService } from 'vs/editor/standalone/common/standaloneThemeService'; import { IMarkerData } from 'vs/platform/markers/common/markers'; @@ -88,9 +88,9 @@ export class EncodedTokenizationSupportAdapter implements modes.ITokenizationSup return this._actual.getInitialState(); } - public tokenize(line: string, hasEOL: boolean, state: modes.IState, offsetDelta: number): TokenizationResult { + public tokenize(line: string, hasEOL: boolean, state: modes.IState): TokenizationResult { if (typeof this._actual.tokenize === 'function') { - return TokenizationSupportAdapter.adaptTokenize(this._languageId, <{ tokenize(line: string, state: modes.IState): ILineTokens; }>this._actual, line, state, offsetDelta); + return TokenizationSupportAdapter.adaptTokenize(this._languageId, <{ tokenize(line: string, state: modes.IState): ILineTokens; }>this._actual, line, state); } throw new Error('Not supported!'); } @@ -118,7 +118,7 @@ export class TokenizationSupportAdapter implements modes.ITokenizationSupport { return this._actual.getInitialState(); } - private static _toClassicTokens(tokens: IToken[], language: string, offsetDelta: number): Token[] { + private static _toClassicTokens(tokens: IToken[], language: string): Token[] { let result: Token[] = []; let previousStartIndex: number = 0; for (let i = 0, len = tokens.length; i < len; i++) { @@ -134,16 +134,16 @@ export class TokenizationSupportAdapter implements modes.ITokenizationSupport { startIndex = previousStartIndex; } - result[i] = new Token(startIndex + offsetDelta, t.scopes, language); + result[i] = new Token(startIndex, t.scopes, language); previousStartIndex = startIndex; } return result; } - public static adaptTokenize(language: string, actual: { tokenize(line: string, state: modes.IState): ILineTokens; }, line: string, state: modes.IState, offsetDelta: number): TokenizationResult { + public static adaptTokenize(language: string, actual: { tokenize(line: string, state: modes.IState): ILineTokens; }, line: string, state: modes.IState): TokenizationResult { let actualResult = actual.tokenize(line, state); - let tokens = TokenizationSupportAdapter._toClassicTokens(actualResult.tokens, language, offsetDelta); + let tokens = TokenizationSupportAdapter._toClassicTokens(actualResult.tokens, language); let endState: modes.IState; // try to save an object if possible @@ -156,11 +156,11 @@ export class TokenizationSupportAdapter implements modes.ITokenizationSupport { return new TokenizationResult(tokens, endState); } - public tokenize(line: string, hasEOL: boolean, state: modes.IState, offsetDelta: number): TokenizationResult { - return TokenizationSupportAdapter.adaptTokenize(this._languageId, this._actual, line, state, offsetDelta); + public tokenize(line: string, hasEOL: boolean, state: modes.IState): TokenizationResult { + return TokenizationSupportAdapter.adaptTokenize(this._languageId, this._actual, line, state); } - private _toBinaryTokens(languageIdCodec: modes.ILanguageIdCodec, tokens: IToken[], offsetDelta: number): Uint32Array { + private _toBinaryTokens(languageIdCodec: modes.ILanguageIdCodec, tokens: IToken[]): Uint32Array { const languageId = languageIdCodec.encodeLanguageId(this._languageId); const tokenTheme = this._standaloneThemeService.getColorTheme().tokenTheme; @@ -185,7 +185,7 @@ export class TokenizationSupportAdapter implements modes.ITokenizationSupport { startIndex = previousStartIndex; } - result[resultLen++] = startIndex + offsetDelta; + result[resultLen++] = startIndex; result[resultLen++] = metadata; previousStartIndex = startIndex; @@ -198,9 +198,9 @@ export class TokenizationSupportAdapter implements modes.ITokenizationSupport { return actualResult; } - public tokenizeEncoded(line: string, hasEOL: boolean, state: modes.IState, offsetDelta: number): EncodedTokenizationResult { + public tokenizeEncoded(line: string, hasEOL: boolean, state: modes.IState): EncodedTokenizationResult { let actualResult = this._actual.tokenize(line, state); - let tokens = this._toBinaryTokens(this._languageService.languageIdCodec, actualResult.tokens, offsetDelta); + let tokens = this._toBinaryTokens(this._languageService.languageIdCodec, actualResult.tokens); let endState: modes.IState; // try to save an object if possible @@ -367,7 +367,7 @@ export function registerTokensProviderFactory(languageId: string, factory: Token if (isATokensProvider(result)) { return createTokenizationSupportAdapter(languageId, result); } - return createTokenizationSupport(StaticServices.languageService.get(), StaticServices.standaloneThemeService.get(), languageId, compile(languageId, result)); + return new MonarchTokenizer(StaticServices.languageService.get(), StaticServices.standaloneThemeService.get(), languageId, compile(languageId, result)); } }; return modes.TokenizationRegistry.registerFactory(languageId, adaptedFactory); @@ -398,7 +398,7 @@ export function setTokensProvider(languageId: string, provider: TokensProvider | */ export function setMonarchTokensProvider(languageId: string, languageDef: IMonarchLanguage | Thenable): IDisposable { const create = (languageDef: IMonarchLanguage) => { - return createTokenizationSupport(StaticServices.languageService.get(), StaticServices.standaloneThemeService.get(), languageId, compile(languageId, languageDef)); + return new MonarchTokenizer(StaticServices.languageService.get(), StaticServices.standaloneThemeService.get(), languageId, compile(languageId, languageDef)); }; if (isThenable(languageDef)) { return registerTokensProviderFactory(languageId, { create: () => languageDef }); diff --git a/src/vs/editor/standalone/common/monarch/monarchLexer.ts b/src/vs/editor/standalone/common/monarch/monarchLexer.ts index 43d83cbd030..35cdbda1348 100644 --- a/src/vs/editor/standalone/common/monarch/monarchLexer.ts +++ b/src/vs/editor/standalone/common/monarch/monarchLexer.ts @@ -229,7 +229,7 @@ class MonarchLineState implements modes.IState { } interface IMonarchTokensCollector { - enterLanguage(startOffset: number, languageId: string): void; + enterLanguage(languageId: string): void; emit(startOffset: number, type: string): void; nestedLanguageTokenize(embeddedLanguageLine: string, hasEOL: boolean, embeddedLanguageData: EmbeddedLanguageData, offsetDelta: number): modes.IState; } @@ -248,7 +248,7 @@ class MonarchClassicTokensCollector implements IMonarchTokensCollector { this._lastTokenLanguage = null; } - public enterLanguage(startOffset: number, languageId: string): void { + public enterLanguage(languageId: string): void { this._languageId = languageId; } @@ -267,13 +267,19 @@ class MonarchClassicTokensCollector implements IMonarchTokensCollector { const nestedLanguageTokenizationSupport = modes.TokenizationRegistry.get(nestedLanguageId); if (!nestedLanguageTokenizationSupport) { - this.enterLanguage(offsetDelta, nestedLanguageId); + this.enterLanguage(nestedLanguageId); this.emit(offsetDelta, ''); return embeddedModeState; } - let nestedResult = nestedLanguageTokenizationSupport.tokenize(embeddedLanguageLine, hasEOL, embeddedModeState, offsetDelta); - this._tokens = this._tokens.concat(nestedResult.tokens); + const nestedResult = nestedLanguageTokenizationSupport.tokenize(embeddedLanguageLine, hasEOL, embeddedModeState); + if (offsetDelta !== 0) { + for (const token of nestedResult.tokens) { + this._tokens.push(new Token(token.offset + offsetDelta, token.type, token.language)); + } + } else { + this._tokens = this._tokens.concat(nestedResult.tokens); + } this._lastTokenType = null; this._lastTokenLanguage = null; this._languageId = null; @@ -303,7 +309,7 @@ class MonarchModernTokensCollector implements IMonarchTokensCollector { this._lastTokenMetadata = 0; } - public enterLanguage(startOffset: number, languageId: string): void { + public enterLanguage(languageId: string): void { this._currentLanguageId = this._languageService.languageIdCodec.encodeLanguageId(languageId); } @@ -351,12 +357,18 @@ class MonarchModernTokensCollector implements IMonarchTokensCollector { const nestedLanguageTokenizationSupport = modes.TokenizationRegistry.get(nestedLanguageId); if (!nestedLanguageTokenizationSupport) { - this.enterLanguage(offsetDelta, nestedLanguageId); + this.enterLanguage(nestedLanguageId); this.emit(offsetDelta, ''); return embeddedModeState; } - let nestedResult = nestedLanguageTokenizationSupport.tokenizeEncoded(embeddedLanguageLine, hasEOL, embeddedModeState, offsetDelta); + const nestedResult = nestedLanguageTokenizationSupport.tokenizeEncoded(embeddedLanguageLine, hasEOL, embeddedModeState); + if (offsetDelta !== 0) { + for (let i = 0, len = nestedResult.tokens.length; i < len; i += 2) { + nestedResult.tokens[i] += offsetDelta; + } + } + this._prependTokens = MonarchModernTokensCollector._merge(this._prependTokens, this._tokens, nestedResult.tokens); this._tokens = []; this._currentLanguageId = 0; @@ -455,23 +467,23 @@ export class MonarchTokenizer implements modes.ITokenizationSupport { return MonarchLineStateFactory.create(rootState, null); } - public tokenize(line: string, hasEOL: boolean, lineState: modes.IState, offsetDelta: number): TokenizationResult { + public tokenize(line: string, hasEOL: boolean, lineState: modes.IState): TokenizationResult { let tokensCollector = new MonarchClassicTokensCollector(); - let endLineState = this._tokenize(line, hasEOL, lineState, offsetDelta, tokensCollector); + let endLineState = this._tokenize(line, hasEOL, lineState, tokensCollector); return tokensCollector.finalize(endLineState); } - public tokenizeEncoded(line: string, hasEOL: boolean, lineState: modes.IState, offsetDelta: number): EncodedTokenizationResult { + public tokenizeEncoded(line: string, hasEOL: boolean, lineState: modes.IState): EncodedTokenizationResult { let tokensCollector = new MonarchModernTokensCollector(this._languageService, this._standaloneThemeService.getColorTheme().tokenTheme); - let endLineState = this._tokenize(line, hasEOL, lineState, offsetDelta, tokensCollector); + let endLineState = this._tokenize(line, hasEOL, lineState, tokensCollector); return tokensCollector.finalize(endLineState); } - private _tokenize(line: string, hasEOL: boolean, lineState: MonarchLineState, offsetDelta: number, collector: IMonarchTokensCollector): MonarchLineState { + private _tokenize(line: string, hasEOL: boolean, lineState: MonarchLineState, collector: IMonarchTokensCollector): MonarchLineState { if (lineState.embeddedLanguageData) { - return this._nestedTokenize(line, hasEOL, lineState, offsetDelta, collector); + return this._nestedTokenize(line, hasEOL, lineState, 0, collector); } else { - return this._myTokenize(line, hasEOL, lineState, offsetDelta, collector); + return this._myTokenize(line, hasEOL, lineState, 0, collector); } } @@ -545,7 +557,7 @@ export class MonarchTokenizer implements modes.ITokenizationSupport { } private _myTokenize(lineWithoutLF: string, hasEOL: boolean, lineState: MonarchLineState, offsetDelta: number, tokensCollector: IMonarchTokensCollector): MonarchLineState { - tokensCollector.enterLanguage(offsetDelta, this._languageId); + tokensCollector.enterLanguage(this._languageId); const lineWithoutLFLength = lineWithoutLF.length; const line = (hasEOL && this._lexer.includeLF ? lineWithoutLF + '\n' : lineWithoutLF); @@ -886,7 +898,3 @@ function findBracket(lexer: monarchCommon.ILexer, matched: string) { } return null; } - -export function createTokenizationSupport(languageService: ILanguageService, standaloneThemeService: IStandaloneThemeService, languageId: string, lexer: monarchCommon.ILexer): modes.ITokenizationSupport { - return new MonarchTokenizer(languageService, standaloneThemeService, languageId, lexer); -} diff --git a/src/vs/editor/standalone/test/browser/standaloneLanguages.test.ts b/src/vs/editor/standalone/test/browser/standaloneLanguages.test.ts index ffa32c3ed78..bd587dec9e7 100644 --- a/src/vs/editor/standalone/test/browser/standaloneLanguages.test.ts +++ b/src/vs/editor/standalone/test/browser/standaloneLanguages.test.ts @@ -106,7 +106,7 @@ suite('TokenizationSupport2Adapter', () => { } } - function testBadTokensProvider(providerTokens: IToken[], offsetDelta: number, expectedClassicTokens: Token[], expectedModernTokens: number[]): void { + function testBadTokensProvider(providerTokens: IToken[], expectedClassicTokens: Token[], expectedModernTokens: number[]): void { class BadTokensProvider implements TokensProvider { public getInitialState(): IState { @@ -130,10 +130,10 @@ suite('TokenizationSupport2Adapter', () => { new MockThemeService() ); - const actualClassicTokens = adapter.tokenize('whatever', true, MockState.INSTANCE, offsetDelta); + const actualClassicTokens = adapter.tokenize('whatever', true, MockState.INSTANCE); assert.deepStrictEqual(actualClassicTokens.tokens, expectedClassicTokens); - const actualModernTokens = adapter.tokenizeEncoded('whatever', true, MockState.INSTANCE, offsetDelta); + const actualModernTokens = adapter.tokenizeEncoded('whatever', true, MockState.INSTANCE); const modernTokens: number[] = []; for (let i = 0; i < actualModernTokens.tokens.length; i++) { modernTokens[i] = actualModernTokens.tokens[i]; @@ -150,13 +150,12 @@ suite('TokenizationSupport2Adapter', () => { disposables.dispose(); } - test(' (no offset delta)', () => { + test('tokens always start at index 0', () => { testBadTokensProvider( [ { startIndex: 7, scopes: 'foo' }, { startIndex: 0, scopes: 'bar' } ], - 0, [ new Token(0, 'foo', languageId), new Token(0, 'bar', languageId), @@ -168,14 +167,13 @@ suite('TokenizationSupport2Adapter', () => { ); }); - test('tokens always start after each other (no offset delta)', () => { + test('tokens always start after each other', () => { testBadTokensProvider( [ { startIndex: 0, scopes: 'foo' }, { startIndex: 5, scopes: 'bar' }, { startIndex: 3, scopes: 'foo' }, ], - 0, [ new Token(0, 'foo', languageId), new Token(5, 'bar', languageId), @@ -188,44 +186,4 @@ suite('TokenizationSupport2Adapter', () => { ] ); }); - - test('tokens always start at index 0 (with offset delta)', () => { - testBadTokensProvider( - [ - { startIndex: 7, scopes: 'foo' }, - { startIndex: 0, scopes: 'bar' } - ], - 7, - [ - new Token(7, 'foo', languageId), - new Token(7, 'bar', languageId), - ], - [ - 7, (0 << MetadataConsts.FOREGROUND_OFFSET), - 7, (1 << MetadataConsts.FOREGROUND_OFFSET) - ] - ); - }); - - test('tokens always start after each other (with offset delta)', () => { - testBadTokensProvider( - [ - { startIndex: 0, scopes: 'foo' }, - { startIndex: 5, scopes: 'bar' }, - { startIndex: 3, scopes: 'foo' }, - ], - 7, - [ - new Token(7, 'foo', languageId), - new Token(12, 'bar', languageId), - new Token(12, 'foo', languageId), - ], - [ - 7, (0 << MetadataConsts.FOREGROUND_OFFSET), - 12, (1 << MetadataConsts.FOREGROUND_OFFSET), - 12, (2 << MetadataConsts.FOREGROUND_OFFSET) - ] - ); - }); - }); diff --git a/src/vs/editor/standalone/test/monarch/monarch.test.ts b/src/vs/editor/standalone/test/monarch/monarch.test.ts index 6ecf23a0d0c..00fce58a94e 100644 --- a/src/vs/editor/standalone/test/monarch/monarch.test.ts +++ b/src/vs/editor/standalone/test/monarch/monarch.test.ts @@ -23,7 +23,7 @@ suite('Monarch', () => { const actualTokens: Token[][] = []; let state = tokenizer.getInitialState(); for (const line of lines) { - const result = tokenizer.tokenize(line, true, state, 0); + const result = tokenizer.tokenize(line, true, state); actualTokens.push(result.tokens); state = result.endState; } diff --git a/src/vs/editor/test/browser/controller/cursor.test.ts b/src/vs/editor/test/browser/controller/cursor.test.ts index 0ca43701874..25e3eb76c59 100644 --- a/src/vs/editor/test/browser/controller/cursor.test.ts +++ b/src/vs/editor/test/browser/controller/cursor.test.ts @@ -4835,7 +4835,7 @@ suite('autoClosingPairs', () => { this._register(TokenizationRegistry.register(this.languageId, { getInitialState: () => new BaseState(), tokenize: undefined!, - tokenizeEncoded: function (line: string, hasEOL: boolean, _state: IState, offsetDelta: number): EncodedTokenizationResult { + tokenizeEncoded: function (line: string, hasEOL: boolean, _state: IState): EncodedTokenizationResult { let state = _state; const tokens: { length: number; type: StandardTokenType; }[] = []; const generateToken = (length: number, type: StandardTokenType, newState?: State) => { diff --git a/src/vs/editor/test/common/model/bracketPairColorizer/tokenizer.test.ts b/src/vs/editor/test/common/model/bracketPairColorizer/tokenizer.test.ts index 763f30c7ab7..5c2c710cc15 100644 --- a/src/vs/editor/test/common/model/bracketPairColorizer/tokenizer.test.ts +++ b/src/vs/editor/test/common/model/bracketPairColorizer/tokenizer.test.ts @@ -175,7 +175,7 @@ class TokenizedDocument { return { getInitialState: () => new State(0), tokenize: () => { throw new Error('Method not implemented.'); }, - tokenizeEncoded: (line: string, hasEOL: boolean, state: IState, offsetDelta: number): EncodedTokenizationResult => { + tokenizeEncoded: (line: string, hasEOL: boolean, state: IState): EncodedTokenizationResult => { const state2 = state as State; const tokens = this.tokensByLine[state2.lineNumber]; const arr = new Array(); diff --git a/src/vs/workbench/services/textMate/browser/abstractTextMateService.ts b/src/vs/workbench/services/textMate/browser/abstractTextMateService.ts index 34e4ae5c8d4..6763dfa772e 100644 --- a/src/vs/workbench/services/textMate/browser/abstractTextMateService.ts +++ b/src/vs/workbench/services/textMate/browser/abstractTextMateService.ts @@ -446,18 +446,14 @@ class TMTokenizationSupport implements ITokenizationSupport { return this._actual.getInitialState(); } - tokenize(line: string, hasEOL: boolean, state: IState, offsetDelta: number): TokenizationResult { + tokenize(line: string, hasEOL: boolean, state: IState): TokenizationResult { throw new Error('Not supported!'); } - tokenizeEncoded(line: string, hasEOL: boolean, state: StackElement, offsetDelta: number): EncodedTokenizationResult { - if (offsetDelta !== 0) { - throw new Error('Unexpected: offsetDelta should be 0.'); - } - + tokenizeEncoded(line: string, hasEOL: boolean, state: StackElement): EncodedTokenizationResult { // Do not attempt to tokenize if a line is too long if (line.length >= this._maxTokenizationLineLength) { - return nullTokenizeEncoded(this._encodedLanguageId, line, state, offsetDelta); + return nullTokenizeEncoded(this._encodedLanguageId, state); } return this._actual.tokenizeEncoded(line, state);