diff --git a/src/vs/workbench/api/browser/mainThreadLanguageFeatures.ts b/src/vs/workbench/api/browser/mainThreadLanguageFeatures.ts index d5d0675a301..8b3849988cc 100644 --- a/src/vs/workbench/api/browser/mainThreadLanguageFeatures.ts +++ b/src/vs/workbench/api/browser/mainThreadLanguageFeatures.ts @@ -617,7 +617,7 @@ class MainThreadSemanticColoringCacheEntry implements modes.SemanticColoring { } } -class MainThreadSemanticColoringProvider implements modes.SemanticColoringProvider { +export class MainThreadSemanticColoringProvider implements modes.SemanticColoringProvider { private readonly _cache = new Map(); diff --git a/src/vs/workbench/api/common/extHostLanguageFeatures.ts b/src/vs/workbench/api/common/extHostLanguageFeatures.ts index b9a231f870a..1bbb8bb5686 100644 --- a/src/vs/workbench/api/common/extHostLanguageFeatures.ts +++ b/src/vs/workbench/api/common/extHostLanguageFeatures.ts @@ -622,12 +622,6 @@ const enum SemanticColoringConstants { */ DesiredTokensPerArea = 400, - /** - * If the semantic coloring result contains a single area and the area contains - * more than 600 tokens, then split the area into mulitple areas. - */ - SplitSingleAreaTokenCountThreshold = 600, - /** * Try to keep the total number of areas under 1024 if possible, * simply compensate by having more tokens per area... @@ -640,15 +634,21 @@ interface ISemanticColoringAreaPair { dto: ISemanticTokensAreaDto; } -class SemanticColoringAdapter { +export class SemanticColoringAdapter { - private readonly _previousResults = new Map(); + private readonly _previousResults: Map; + private readonly _splitSingleAreaTokenCountThreshold: number; private _nextResultId = 1; constructor( private readonly _documents: ExtHostDocuments, private readonly _provider: vscode.SemanticColoringProvider, - ) { } + private readonly _desiredTokensPerArea = SemanticColoringConstants.DesiredTokensPerArea, + private readonly _desiredMaxAreas = SemanticColoringConstants.DesiredMaxAreas + ) { + this._previousResults = new Map(); + this._splitSingleAreaTokenCountThreshold = Math.round(1.5 * this._desiredTokensPerArea); + } provideSemanticColoring(resource: URI, previousSemanticColoringResultId: number, token: CancellationToken): Promise { const doc = this._documents.getDocument(resource); @@ -736,7 +736,6 @@ class SemanticColoringAdapter { } private _deltaEncodeArea(oldAreas: Uint32Array[], newArea: SemanticColoringArea): VSBuffer { - console.log(`_deltaEncodeArea`); const newAreaData = newArea.data; const prependAreas: ISemanticColoringAreaPair[] = []; const appendAreas: ISemanticColoringAreaPair[] = []; @@ -748,7 +747,7 @@ class SemanticColoringAdapter { const oldAreaData = oldAreas[i]; const oldAreaTokenCount = (oldAreaData.length / 5) | 0; if (newTokenEndIndex - newTokenStartIndex < oldAreaTokenCount) { - // thre are too many old tokens, this cannot work + // there are too many old tokens, this cannot work break; } @@ -786,7 +785,7 @@ class SemanticColoringAdapter { } newTokenEndIndex -= oldAreaTokenCount; - appendAreas.push({ + appendAreas.unshift({ data: oldAreaData, dto: { type: 'delta', @@ -798,18 +797,14 @@ class SemanticColoringAdapter { if (prependAreas.length === 0 && appendAreas.length === 0) { // There is no reuse possibility! - console.log(`no reuse possibility`); return this._fullEncodeAreas([newArea]); } if (newTokenStartIndex === newTokenEndIndex) { // 100% reuse! - console.log(`100% reuse`); return this._saveResultAndEncode(prependAreas.concat(appendAreas)); } - console.log(`some reuse`); - // Extract the mid area const newTokenStartDeltaLine = newAreaData[5 * newTokenStartIndex]; const newMidAreaData = new Uint32Array(5 * (newTokenEndIndex - newTokenStartIndex)); @@ -830,7 +825,7 @@ class SemanticColoringAdapter { } const newMidArea = new SemanticColoringArea(newArea.line + newTokenStartDeltaLine, newMidAreaData); - const newMidAreas = SemanticColoringAdapter._splitAreaIntoMultipleAreasIfNecessary(newMidArea); + const newMidAreas = this._splitAreaIntoMultipleAreasIfNecessary(newMidArea); const newMidAreasPairs: ISemanticColoringAreaPair[] = newMidAreas.map(a => { return { data: a.data, @@ -869,9 +864,8 @@ class SemanticColoringAdapter { // } private _fullEncodeAreas(areas: SemanticColoringArea[]): VSBuffer { - console.log(`_fullEncodeAreas`); if (areas.length === 1) { - areas = SemanticColoringAdapter._splitAreaIntoMultipleAreasIfNecessary(areas[0]); + areas = this._splitAreaIntoMultipleAreasIfNecessary(areas[0]); } return this._saveResultAndEncode(areas.map(a => { @@ -897,15 +891,15 @@ class SemanticColoringAdapter { return encodeSemanticTokensDto(dto); } - private static _splitAreaIntoMultipleAreasIfNecessary(area: vscode.SemanticColoringArea): SemanticColoringArea[] { + private _splitAreaIntoMultipleAreasIfNecessary(area: vscode.SemanticColoringArea): SemanticColoringArea[] { const srcAreaLine = area.line; const srcAreaData = area.data; const tokenCount = (srcAreaData.length / 5) | 0; - if (tokenCount <= SemanticColoringConstants.SplitSingleAreaTokenCountThreshold) { + if (tokenCount <= this._splitSingleAreaTokenCountThreshold) { return [area]; } - const tokensPerArea = Math.max(Math.ceil(tokenCount / SemanticColoringConstants.DesiredMaxAreas), SemanticColoringConstants.DesiredTokensPerArea); + const tokensPerArea = Math.max(Math.ceil(tokenCount / this._desiredMaxAreas), this._desiredTokensPerArea); let result: SemanticColoringArea[] = []; let tokenIndex = 0; diff --git a/src/vs/workbench/test/electron-browser/api/semanticTokens.test.ts b/src/vs/workbench/test/electron-browser/api/semanticTokens.test.ts new file mode 100644 index 00000000000..a7e12e704e6 --- /dev/null +++ b/src/vs/workbench/test/electron-browser/api/semanticTokens.test.ts @@ -0,0 +1,308 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as assert from 'assert'; +import { URI } from 'vs/base/common/uri'; +import * as types from 'vs/workbench/api/common/extHostTypes'; +import { TestRPCProtocol } from './testRPCProtocol'; +import { SemanticColoringAdapter } from 'vs/workbench/api/common/extHostLanguageFeatures'; +import { ExtHostDocuments } from 'vs/workbench/api/common/extHostDocuments'; +import { ExtHostDocumentsAndEditors } from 'vs/workbench/api/common/extHostDocumentsAndEditors'; +import { ExtHostContext } from 'vs/workbench/api/common/extHost.protocol'; +import * as vscode from 'vscode'; +import { CancellationToken } from 'vs/base/common/cancellation'; +import { decodeSemanticTokensDto, ISemanticTokensDto } from 'vs/workbench/api/common/shared/semanticTokens'; +import { ExtHostDocumentData } from 'vs/workbench/api/common/extHostDocumentData'; + +suite('SemanticColoringAdapter', () => { + + const resource = URI.parse('foo:bar'); + const rpcProtocol = new TestRPCProtocol(); + + const initialText = [ + 'const enum E01 {}', + 'const enum E02 {}', + 'const enum E03 {}', + 'const enum E04 {}', + 'const enum E05 {}', + 'const enum E06 {}', + 'const enum E07 {}', + 'const enum E08 {}', + 'const enum E09 {}', + 'const enum E10 {}', + 'const enum E11 {}', + 'const enum E12 {}', + 'const enum E13 {}', + 'const enum E14 {}', + 'const enum E15 {}', + 'const enum E16 {}', + 'const enum E17 {}', + 'const enum E18 {}', + 'const enum E19 {}', + 'const enum E20 {}', + 'const enum E21 {}', + 'const enum E22 {}', + 'const enum E23 {}', + ].join('\n'); + + const extHostDocumentsAndEditors = new ExtHostDocumentsAndEditors(rpcProtocol); + extHostDocumentsAndEditors.$acceptDocumentsAndEditorsDelta({ + addedDocuments: [{ + isDirty: false, + versionId: 1, + modeId: 'javascript', + uri: resource, + lines: initialText.split(/\n/), + EOL: '\n', + }] + }); + const extHostDocuments = new ExtHostDocuments(rpcProtocol, extHostDocumentsAndEditors); + rpcProtocol.set(ExtHostContext.ExtHostDocuments, extHostDocuments); + + const semanticTokensProvider = new class implements vscode.SemanticColoringProvider { + provideSemanticColoring(document: vscode.TextDocument, token: vscode.CancellationToken): types.SemanticColoring { + const lines = document.getText().split(/\r\n|\r|\n/g); + const tokens: number[] = []; + const pushToken = (line: number, startCharacter: number, endCharacter: number, type: number) => { + tokens.push(line); + tokens.push(startCharacter); + tokens.push(endCharacter); + tokens.push(type); + tokens.push(0); + }; + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + const m = line.match(/^(const enum )([\w\d]+) \{\}/); + if (m) { + pushToken(i, m[1].length, m[1].length + m[2].length, parseInt(m[2].substr(1))); + } + } + return new types.SemanticColoring([new types.SemanticColoringArea(0, new Uint32Array(tokens))]); + } + }; + + let adapter: SemanticColoringAdapter; + let doc: ExtHostDocumentData; + + setup(() => { + adapter = new SemanticColoringAdapter(extHostDocuments, semanticTokensProvider, 10); + doc = extHostDocumentsAndEditors.getDocument(resource)!; + const docLineCount = doc.document.lineCount; + const allRange = { startLineNumber: 1, startColumn: 1, endLineNumber: docLineCount, endColumn: doc.document.lineAt(docLineCount - 1).text.length + 1 }; + doc.onEvents({ + versionId: 1, + eol: '\n', + changes: [{ + range: allRange, + rangeOffset: 0, + rangeLength: 0, + text: initialText + }] + }); + }); + + type SimpleTokensDto = { type: 'full'; line: number; tokens: number[]; } | { type: 'delta'; line: number; oldIndex: number }; + + function assertDTO(actual: ISemanticTokensDto, expected: SimpleTokensDto[]): void { + const simpleActual: SimpleTokensDto[] = actual.areas.map((area) => { + if (area.type === 'full') { + const tokenCount = (area.data.length / 5) | 0; + let tokens: number[] = []; + for (let i = 0; i < tokenCount; i++) { + tokens.push(area.data[5 * i]); + } + return { + type: 'full', + line: area.line, + tokens: tokens + }; + } + return { + type: 'delta', + line: area.line, + oldIndex: area.oldIndex + }; + }); + assert.deepEqual(simpleActual, expected); + } + + test('single area - breaks it up', async () => { + const dto = (await adapter.provideSemanticColoring(resource, 0, CancellationToken.None))!; + const result = decodeSemanticTokensDto(dto); + assertDTO(result, [ + { type: 'full', line: 1, tokens: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] }, + { type: 'full', line: 11, tokens: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] }, + { type: 'full', line: 21, tokens: [0, 1, 2] }, + ]); + }); + + test('single area - after a not important change', async () => { + const result1 = decodeSemanticTokensDto((await adapter.provideSemanticColoring(resource, 0, CancellationToken.None))!); + assertDTO(result1, [ + { type: 'full', line: 1, tokens: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] }, + { type: 'full', line: 11, tokens: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] }, + { type: 'full', line: 21, tokens: [0, 1, 2] }, + ]); + + doc.onEvents({ + versionId: 2, + eol: '\n', + changes: [{ + range: { startLineNumber: 2, startColumn: 18, endLineNumber: 2, endColumn: 18 }, + rangeOffset: 0, + rangeLength: 0, + text: '//' + }] + }); + + const result2 = decodeSemanticTokensDto((await adapter.provideSemanticColoring(resource, result1.id, CancellationToken.None))!); + assertDTO(result2, [ + { type: 'delta', line: 1, oldIndex: 0 }, + { type: 'delta', line: 11, oldIndex: 1 }, + { type: 'delta', line: 21, oldIndex: 2 }, + ]); + adapter.releaseSemanticColoring(result1.id); + }); + + test('single area - after a single removal in the first block', async () => { + const result1 = decodeSemanticTokensDto((await adapter.provideSemanticColoring(resource, 0, CancellationToken.None))!); + assertDTO(result1, [ + { type: 'full', line: 1, tokens: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] }, + { type: 'full', line: 11, tokens: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] }, + { type: 'full', line: 21, tokens: [0, 1, 2] }, + ]); + + doc.onEvents({ + versionId: 2, + eol: '\n', + changes: [{ + range: { startLineNumber: 2, startColumn: 1, endLineNumber: 2, endColumn: 1 }, + rangeOffset: 0, + rangeLength: 0, + text: '//' + }] + }); + + const result2 = decodeSemanticTokensDto((await adapter.provideSemanticColoring(resource, result1.id, CancellationToken.None))!); + assertDTO(result2, [ + { type: 'full', line: 1, tokens: [0, 2, 3, 4, 5, 6, 7, 8, 9] }, + { type: 'delta', line: 11, oldIndex: 1 }, + { type: 'delta', line: 21, oldIndex: 2 }, + ]); + adapter.releaseSemanticColoring(result1.id); + }); + + test('single area - after a not important change', async () => { + const result1 = decodeSemanticTokensDto((await adapter.provideSemanticColoring(resource, 0, CancellationToken.None))!); + assertDTO(result1, [ + { type: 'full', line: 1, tokens: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] }, + { type: 'full', line: 11, tokens: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] }, + { type: 'full', line: 21, tokens: [0, 1, 2] }, + ]); + + doc.onEvents({ + versionId: 2, + eol: '\n', + changes: [{ + range: { startLineNumber: 2, startColumn: 18, endLineNumber: 2, endColumn: 18 }, + rangeOffset: 0, + rangeLength: 0, + text: '//' + }] + }); + + const result2 = decodeSemanticTokensDto((await adapter.provideSemanticColoring(resource, result1.id, CancellationToken.None))!); + assertDTO(result2, [ + { type: 'delta', line: 1, oldIndex: 0 }, + { type: 'delta', line: 11, oldIndex: 1 }, + { type: 'delta', line: 21, oldIndex: 2 }, + ]); + adapter.releaseSemanticColoring(result1.id); + }); + + test('single area - after a down shift of all the blocks', async () => { + const result1 = decodeSemanticTokensDto((await adapter.provideSemanticColoring(resource, 0, CancellationToken.None))!); + assertDTO(result1, [ + { type: 'full', line: 1, tokens: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] }, + { type: 'full', line: 11, tokens: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] }, + { type: 'full', line: 21, tokens: [0, 1, 2] }, + ]); + + doc.onEvents({ + versionId: 2, + eol: '\n', + changes: [{ + range: { startLineNumber: 1, startColumn: 1, endLineNumber: 1, endColumn: 1 }, + rangeOffset: 0, + rangeLength: 0, + text: '\n' + }] + }); + + const result2 = decodeSemanticTokensDto((await adapter.provideSemanticColoring(resource, result1.id, CancellationToken.None))!); + assertDTO(result2, [ + { type: 'delta', line: 2, oldIndex: 0 }, + { type: 'delta', line: 12, oldIndex: 1 }, + { type: 'delta', line: 22, oldIndex: 2 }, + ]); + adapter.releaseSemanticColoring(result1.id); + }); + + test('single area - after a single removal in the last block', async () => { + const result1 = decodeSemanticTokensDto((await adapter.provideSemanticColoring(resource, 0, CancellationToken.None))!); + assertDTO(result1, [ + { type: 'full', line: 1, tokens: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] }, + { type: 'full', line: 11, tokens: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] }, + { type: 'full', line: 21, tokens: [0, 1, 2] }, + ]); + + doc.onEvents({ + versionId: 2, + eol: '\n', + changes: [{ + range: { startLineNumber: 22, startColumn: 1, endLineNumber: 22, endColumn: 1 }, + rangeOffset: 0, + rangeLength: 0, + text: '//' + }] + }); + + const result2 = decodeSemanticTokensDto((await adapter.provideSemanticColoring(resource, result1.id, CancellationToken.None))!); + assertDTO(result2, [ + { type: 'delta', line: 1, oldIndex: 0 }, + { type: 'delta', line: 11, oldIndex: 1 }, + { type: 'full', line: 21, tokens: [0, 2] }, + ]); + adapter.releaseSemanticColoring(result1.id); + }); + + test('single area - after a single addition in the first block', async () => { + const result1 = decodeSemanticTokensDto((await adapter.provideSemanticColoring(resource, 0, CancellationToken.None))!); + assertDTO(result1, [ + { type: 'full', line: 1, tokens: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] }, + { type: 'full', line: 11, tokens: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] }, + { type: 'full', line: 21, tokens: [0, 1, 2] }, + ]); + + doc.onEvents({ + versionId: 2, + eol: '\n', + changes: [{ + range: { startLineNumber: 2, startColumn: 1, endLineNumber: 2, endColumn: 1 }, + rangeOffset: 0, + rangeLength: 0, + text: 'const enum E00 {}\n' + }] + }); + + const result2 = decodeSemanticTokensDto((await adapter.provideSemanticColoring(resource, result1.id, CancellationToken.None))!); + assertDTO(result2, [ + { type: 'full', line: 1, tokens: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10] }, + { type: 'delta', line: 12, oldIndex: 1 }, + { type: 'delta', line: 22, oldIndex: 2 }, + ]); + adapter.releaseSemanticColoring(result1.id); + }); +});