From 94248bf149cdcc6deafa59502f4aeeda8ffdd45c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dirk=20B=C3=A4umer?= Date: Thu, 23 Oct 2025 11:31:07 +0200 Subject: [PATCH] Make prompt tests work again. Port over more tests from client (#1542) --- .../contextProviderRegistryMultiLanguage.ts | 24 +- .../src/prompt/contextProviderRegistryTs.ts | 4 +- .../src/prompt/contextProviderStatistics.ts | 2 +- .../vscode-node/lib/src/prompt/repository.ts | 2 +- ...ntextProviderRegistryMultiLanguage.test.ts | 90 ++ .../test/contextProviderRegistryTs.test.ts | 54 + .../test/contextProviderStatistics.test.ts | 256 ++++ .../lib/src/prompt/test/parseBlock.test.ts | 38 + .../lib/src/prompt/test/prompt.test.ts | 296 ++++ .../lib/src/prompt/test/repository.test.ts | 97 ++ .../prompt/src/test/indentation.test.ts | 128 +- .../src/test/indentationLanguages.test.ts | 100 +- .../src/test/indentationParsing.test.ts | 316 ++-- .../prompt/src/test/parseBlock.test.ts | 1350 ++++++++--------- .../prompt/src/test/tokenizer.test.ts | 14 +- 15 files changed, 1811 insertions(+), 960 deletions(-) create mode 100644 extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/test/contextProviderRegistryMultiLanguage.test.ts create mode 100644 extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/test/contextProviderRegistryTs.test.ts create mode 100644 extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/test/contextProviderStatistics.test.ts create mode 100644 extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/test/parseBlock.test.ts create mode 100644 extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/test/prompt.test.ts create mode 100644 extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/test/repository.test.ts diff --git a/extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/contextProviderRegistryMultiLanguage.ts b/extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/contextProviderRegistryMultiLanguage.ts index 820ab697244..c507fad3489 100644 --- a/extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/contextProviderRegistryMultiLanguage.ts +++ b/extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/contextProviderRegistryMultiLanguage.ts @@ -6,8 +6,8 @@ import { Context } from '../context'; import { Features } from '../experiments/features'; import { logger } from '../logger'; -import { ActiveExperiments } from './contextProviderRegistry'; import { TelemetryWithExp } from '../telemetry'; +import { ActiveExperiments } from './contextProviderRegistry'; const MULTI_LANGUAGE_CONTEXT_PROVIDER_ID = 'fallbackContextProvider'; @@ -36,7 +36,7 @@ interface MultiLanguageContextProviderParams { mlcpEnableImports: boolean; } -const multiLanguageContextProviderParamsDefault: MultiLanguageContextProviderParams = { +export const multiLanguageContextProviderParamsDefault: MultiLanguageContextProviderParams = { mlcpMaxContextItems: 20, mlcpMaxSymbolMatches: 20, mlcpEnableImports: false, @@ -83,3 +83,23 @@ function getMultiLanguageContextProviderParamsFromExp( return params; } + +export function getMultiLanguageContextProviderParamsFromActiveExperiments( + activeExperiments: Map +): MultiLanguageContextProviderParams { + const params = { ...multiLanguageContextProviderParamsDefault }; + + if (activeExperiments.has('mlcpMaxContextItems')) { + params.mlcpMaxContextItems = Number(activeExperiments.get('mlcpMaxContextItems')); + } + + if (activeExperiments.has('mlcpMaxSymbolMatches')) { + params.mlcpMaxSymbolMatches = Number(activeExperiments.get('mlcpMaxSymbolMatches')); + } + + if (activeExperiments.has('mlcpEnableImports')) { + params.mlcpEnableImports = String(activeExperiments.get('mlcpEnableImports')) === 'true'; + } + + return params; +} diff --git a/extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/contextProviderRegistryTs.ts b/extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/contextProviderRegistryTs.ts index 27a5a16a1bf..4f1285c28b2 100644 --- a/extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/contextProviderRegistryTs.ts +++ b/extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/contextProviderRegistryTs.ts @@ -6,10 +6,10 @@ import { Context } from '../context'; import { Features } from '../experiments/features'; import { logger } from '../logger'; -import { ActiveExperiments } from './contextProviderRegistry'; import { TelemetryWithExp } from '../telemetry'; +import { ActiveExperiments } from './contextProviderRegistry'; -const TS_CONTEXT_PROVIDER_ID = 'typescript-ai-context-provider'; +export const TS_CONTEXT_PROVIDER_ID = 'typescript-ai-context-provider'; interface ContextProviderParams { [key: string]: string | number | boolean; diff --git a/extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/contextProviderStatistics.ts b/extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/contextProviderStatistics.ts index ab35c1623e1..8d8998c54b6 100644 --- a/extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/contextProviderStatistics.ts +++ b/extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/contextProviderStatistics.ts @@ -17,7 +17,7 @@ import { SupportedContextItemWithId } from './contextProviders/contextItemSchema export type PromptExpectation = 'included' | 'content_excluded'; -type PromptMatcher = { +export type PromptMatcher = { source: SupportedContextItemWithId; expectedTokens: number; actualTokens: number; diff --git a/extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/repository.ts b/extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/repository.ts index f582711157c..c71f7e08466 100644 --- a/extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/repository.ts +++ b/extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/repository.ts @@ -76,7 +76,7 @@ const backgroundRepoInfo = computeInBackgroundAndMemoize { +export async function extractRepoInfo(ctx: Context, uri: FileIdentifier): Promise { const fsUri = getFsUri(uri); if (!fsUri) { return undefined; } diff --git a/extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/test/contextProviderRegistryMultiLanguage.test.ts b/extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/test/contextProviderRegistryMultiLanguage.test.ts new file mode 100644 index 00000000000..5199e0dd884 --- /dev/null +++ b/extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/test/contextProviderRegistryMultiLanguage.test.ts @@ -0,0 +1,90 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as assert from 'assert'; +import { + getMultiLanguageContextProviderParamsFromActiveExperiments, + multiLanguageContextProviderParamsDefault, +} from '../contextProviderRegistryMultiLanguage'; + +suite('contextProviderRegistryMultiLanguage', function () { + let activeExperiments: Map; + + setup(function () { + activeExperiments = new Map(); + }); + + suite('getMultiLanguageContextProviderConfigFromActiveExperiments', function () { + test('returns default config when no experiments are set', function () { + const result = getMultiLanguageContextProviderParamsFromActiveExperiments(new Map()); + + assert.deepStrictEqual(result, multiLanguageContextProviderParamsDefault); + }); + + test('overrides defaults with experiment values', function () { + activeExperiments.set('mlcpMaxContextItems', '50'); + activeExperiments.set('mlcpMaxSymbolMatches', 30); + activeExperiments.set('mlcpEnableImports', true); + + const result = getMultiLanguageContextProviderParamsFromActiveExperiments(activeExperiments); + + assert.strictEqual(result.mlcpMaxContextItems, 50); + assert.strictEqual(result.mlcpMaxSymbolMatches, 30); + assert.strictEqual(result.mlcpEnableImports, true); + }); + + test('converts string values to appropriate types', function () { + activeExperiments.set('mlcpMaxContextItems', '25'); + activeExperiments.set('mlcpEnableImports', 'true'); + + const result = getMultiLanguageContextProviderParamsFromActiveExperiments(activeExperiments); + + assert.strictEqual(result.mlcpMaxContextItems, 25); + assert.strictEqual(result.mlcpEnableImports, true); + }); + + test('converts string values for false to appropriate types', function () { + activeExperiments.set('mlcpMaxContextItems', '25'); + activeExperiments.set('mlcpEnableImports', 'false'); + + const result = getMultiLanguageContextProviderParamsFromActiveExperiments(activeExperiments); + + assert.strictEqual(result.mlcpMaxContextItems, 25); + assert.strictEqual(result.mlcpEnableImports, false); + }); + + test('handles partial overrides', function () { + activeExperiments.set('mlcpEnableImports', true); + + const result = getMultiLanguageContextProviderParamsFromActiveExperiments(activeExperiments); + + assert.strictEqual( + result.mlcpMaxContextItems, + multiLanguageContextProviderParamsDefault.mlcpMaxContextItems + ); + assert.strictEqual( + result.mlcpMaxSymbolMatches, + multiLanguageContextProviderParamsDefault.mlcpMaxSymbolMatches + ); + assert.strictEqual(result.mlcpEnableImports, true); + }); + + test('converts falsy values correctly', function () { + activeExperiments.set('mlcpMaxContextItems', 0); + activeExperiments.set('mlcpEnableImports', false); + + const result = getMultiLanguageContextProviderParamsFromActiveExperiments(activeExperiments); + + assert.strictEqual(result.mlcpMaxContextItems, 0); + assert.strictEqual(result.mlcpEnableImports, false); + }); + + test('returns false for imports when not set', function () { + const result = getMultiLanguageContextProviderParamsFromActiveExperiments(activeExperiments); + + assert.strictEqual(result.mlcpEnableImports, false); + }); + }); +}); diff --git a/extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/test/contextProviderRegistryTs.test.ts b/extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/test/contextProviderRegistryTs.test.ts new file mode 100644 index 00000000000..632040cbb0b --- /dev/null +++ b/extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/test/contextProviderRegistryTs.test.ts @@ -0,0 +1,54 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; +import { Context } from '../../context'; +import { TelemetryWithExp } from '../../telemetry'; +import { createLibTestingContext } from '../../test/context'; +import { ActiveExperiments } from '../contextProviderRegistry'; +import { fillInTsActiveExperiments, TS_CONTEXT_PROVIDER_ID } from '../contextProviderRegistryTs'; + +suite('contextProviderRegistryTs', function () { + let ctx: Context; + let activeExperiments: ActiveExperiments; + let telemetryData: TelemetryWithExp; + + setup(function () { + ctx = createLibTestingContext(); + activeExperiments = new Map(); + telemetryData = TelemetryWithExp.createEmptyConfigForTesting(); + telemetryData.filtersAndExp.exp.variables['copilottscontextproviderparams'] = JSON.stringify({ + booleanProperty: true, + }); + }); + + test('does not add active experiments if no provider is active', function () { + fillInTsActiveExperiments(ctx, [], activeExperiments, telemetryData); + + assert.ok(activeExperiments.size === 0); + }); + + test('adds active experiments if TS provider is active', function () { + fillInTsActiveExperiments(ctx, [TS_CONTEXT_PROVIDER_ID], activeExperiments, telemetryData); + + assert.ok(activeExperiments.has('booleanProperty')); + assert.strictEqual(activeExperiments.get('booleanProperty'), true); + }); + + test('adds active experiments in debug mode', function () { + fillInTsActiveExperiments(ctx, ['*'], activeExperiments, telemetryData); + + assert.ok(activeExperiments.has('booleanProperty')); + assert.strictEqual(activeExperiments.get('booleanProperty'), true); + }); + + test('bad JSON is ignored', function () { + telemetryData.filtersAndExp.exp.variables['copilottscontextproviderparams'] = '{"badJSON": true'; + + fillInTsActiveExperiments(ctx, [TS_CONTEXT_PROVIDER_ID], activeExperiments, telemetryData); + + assert.ok(activeExperiments.size === 0); + }); +}); diff --git a/extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/test/contextProviderStatistics.test.ts b/extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/test/contextProviderStatistics.test.ts new file mode 100644 index 00000000000..1e69c6a2659 --- /dev/null +++ b/extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/test/contextProviderStatistics.test.ts @@ -0,0 +1,256 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; +import { ResolutionStatus } from '../../../../types/src/index'; +import { TraitWithId } from '../contextProviders/contextItemSchemas'; +import { PromptMatcher } from '../contextProviderStatistics'; +import { TestContextProviderStatistics } from './contextProviderStatistics'; + +suite('contextProviderStatistics', function () { + let statistics: TestContextProviderStatistics; + const resolutions: ResolutionStatus[] = ['partial', 'full']; + + setup(function () { + statistics = new TestContextProviderStatistics(); + }); + + const trait1: TraitWithId = { + name: 'trait1', + value: 'value1', + id: '1234', + type: 'Trait', + }; + const trait2: TraitWithId = { + name: 'trait2', + value: 'value2', + id: '5678', + type: 'Trait', + }; + + test('can set expectations', function () { + statistics.addExpectations('bar', [ + [trait1, 'included'], + [trait2, 'content_excluded'], + ]); + assert.deepStrictEqual(statistics.expectations.size, 1); + assert.deepStrictEqual(statistics.expectations.get('bar')?.length, 2); + }); + + test('can add expectations', function () { + statistics.addExpectations('bar', [ + [trait1, 'included'], + [trait2, 'content_excluded'], + ]); + + const trait3: TraitWithId = { + name: 'trait3', + value: 'value3', + id: '9012', + type: 'Trait', + }; + const trait4: TraitWithId = { + name: 'trait4', + value: 'value4', + id: '3456', + type: 'Trait', + }; + + statistics.addExpectations('bar', [ + [trait3, 'included'], + [trait4, 'content_excluded'], + ]); + assert.deepStrictEqual(statistics.expectations.size, 1); + assert.deepStrictEqual(statistics.expectations.get('bar')?.length, 4); + }); + + test('computing match unsets expectations and resolution', function () { + statistics.addExpectations('bar', [ + [trait1, 'included'], + [trait2, 'content_excluded'], + ]); + statistics.setLastResolution('bar', 'full'); + assert.deepStrictEqual(statistics.expectations.size, 1); + assert.deepStrictEqual(statistics.lastResolution.size, 1); + + statistics.computeMatch([]); + + assert.deepStrictEqual(statistics.expectations.size, 0); + assert.deepStrictEqual(statistics.lastResolution.size, 0); + }); + + test('does not compute match for empty expectations', function () { + statistics.addExpectations('bar', []); + + statistics.computeMatch([]); + + assert.deepStrictEqual(statistics.statistics.size, 0); + }); + + for (const resolution of resolutions) { + test(`can match full expectations, resolution: ${resolution}`, function () { + statistics.addExpectations('foo', [ + [trait1, 'included'], + [trait2, 'included'], + ]); + statistics.setLastResolution('foo', resolution); + + const promptMatcher: PromptMatcher[] = [ + { + expectedTokens: 7, + actualTokens: 7, + source: trait1, + }, + { + expectedTokens: 10, + actualTokens: 10, + source: trait2, + }, + ]; + + statistics.computeMatch(promptMatcher); + const stats = statistics.get('foo'); + assert.ok(stats); + assert.deepStrictEqual(stats.resolution, resolution); + assert.deepStrictEqual(stats.usage, 'full'); + assert.deepStrictEqual(stats.usageDetails, [ + { id: '1234', usage: 'full', expectedTokens: 7, actualTokens: 7, type: 'Trait' }, + { id: '5678', usage: 'full', expectedTokens: 10, actualTokens: 10, type: 'Trait' }, + ]); + }); + + test(`can match partial expectations, resolution: ${resolution}`, function () { + statistics.addExpectations('foo', [ + [trait1, 'included'], + [trait2, 'included'], + ]); + statistics.setLastResolution('foo', resolution); + + const promptMatchers: PromptMatcher[] = [ + { + expectedTokens: 7, + actualTokens: 7, + source: trait1, + }, + { + expectedTokens: 10, + actualTokens: 5, + source: trait2, + }, + ]; + + statistics.computeMatch(promptMatchers); + const stats = statistics.get('foo'); + assert.ok(stats); + assert.deepStrictEqual(stats.resolution, resolution); + assert.deepStrictEqual(stats.usage, 'partial'); + assert.deepStrictEqual(stats.usageDetails, [ + { id: '1234', usage: 'full', expectedTokens: 7, actualTokens: 7, type: 'Trait' }, + { id: '5678', usage: 'partial', expectedTokens: 10, actualTokens: 5, type: 'Trait' }, + ]); + }); + + test(`full elision is no usage, resolution: ${resolution}`, function () { + statistics.addExpectations('foo', [ + [trait1, 'included'], + [trait2, 'included'], + ]); + statistics.setLastResolution('foo', resolution); + + const promptMatchers: PromptMatcher[] = [ + { + expectedTokens: 7, + actualTokens: 0, + source: trait1, + }, + { + expectedTokens: 10, + actualTokens: 0, + source: trait2, + }, + ]; + + statistics.computeMatch(promptMatchers); + const stats = statistics.get('foo'); + assert.ok(stats); + assert.deepStrictEqual(stats.resolution, resolution); + assert.deepStrictEqual(stats.usage, 'none'); + assert.deepStrictEqual(stats.usageDetails, [ + { id: '1234', usage: 'none', expectedTokens: 7, actualTokens: 0, type: 'Trait' }, + { id: '5678', usage: 'none', expectedTokens: 10, actualTokens: 0, type: 'Trait' }, + ]); + }); + + test(`some content excluded items make it partial, resolution: ${resolution}`, function () { + statistics.addExpectations('foo', [ + [trait1, 'included'], + [trait2, 'content_excluded'], + ]); + statistics.setLastResolution('foo', resolution); + + const promptMatchers: PromptMatcher[] = [ + { + expectedTokens: 7, + actualTokens: 7, + source: trait1, + }, + ]; + + statistics.computeMatch(promptMatchers); + const stats = statistics.get('foo'); + assert.ok(stats); + assert.deepStrictEqual(stats.resolution, resolution); + assert.deepStrictEqual(stats.usage, 'partial'); + assert.deepStrictEqual(stats.usageDetails, [ + { id: '1234', usage: 'full', expectedTokens: 7, actualTokens: 7, type: 'Trait' }, + { id: '5678', usage: 'none_content_excluded', type: 'Trait' }, + ]); + }); + + test(`all content excluded items make it none, resolution: ${resolution}`, function () { + statistics.addExpectations('foo', [ + [trait1, 'content_excluded'], + [trait2, 'content_excluded'], + ]); + statistics.setLastResolution('foo', resolution); + + statistics.computeMatch([]); + const stats = statistics.get('foo'); + assert.ok(stats); + assert.deepStrictEqual(stats.resolution, resolution); + assert.deepStrictEqual(stats.usage, 'none'); + assert.deepStrictEqual(stats.usageDetails, [ + { id: '1234', usage: 'none_content_excluded', type: 'Trait' }, + { id: '5678', usage: 'none_content_excluded', type: 'Trait' }, + ]); + }); + } + + test('none resolution is always no match', function () { + statistics.addExpectations('foo', [ + [trait1, 'included'], + [trait1, 'included'], + ]); + statistics.setLastResolution('foo', 'none'); + + statistics.computeMatch([]); + + const stats = statistics.get('foo'); + assert.deepStrictEqual(stats!, { usage: 'none', resolution: 'none' }); + }); + + test('error resolution is always no match', function () { + statistics.addExpectations('foo', [ + [trait1, 'content_excluded'], + [trait2, 'content_excluded'], + ]); + statistics.setLastResolution('foo', 'error'); + + statistics.computeMatch([]); + + const stats = statistics.get('foo'); + assert.deepStrictEqual(stats!, { usage: 'none', resolution: 'error' }); + }); +}); diff --git a/extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/test/parseBlock.test.ts b/extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/test/parseBlock.test.ts new file mode 100644 index 00000000000..4a1fccea1c7 --- /dev/null +++ b/extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/test/parseBlock.test.ts @@ -0,0 +1,38 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as assert from 'assert'; +import { contextIndentationFromText } from '../parseBlock'; + +suite('Indentation', function () { + test('single line -> only current', function () { + assert.deepStrictEqual(contextIndentationFromText('x', 0, 'language'), { + prev: undefined, + current: 0, + next: undefined, + }); + }); + test('single line with line after -> only current & next', function () { + assert.deepStrictEqual(contextIndentationFromText('x\ny', 0, 'language'), { + prev: undefined, + current: 0, + next: 0, + }); + }); + test('after indent -> only current & prev', function () { + assert.deepStrictEqual(contextIndentationFromText('x\n y', 4, 'language'), { + prev: 0, + current: 1, + next: undefined, + }); + }); + test('after indent but before text -> only current from line above', function () { + assert.deepStrictEqual(contextIndentationFromText('x\n y', 3, 'language'), { + prev: undefined, + current: 0, + next: undefined, + }); + }); +}); diff --git a/extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/test/prompt.test.ts b/extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/test/prompt.test.ts new file mode 100644 index 00000000000..cd073594309 --- /dev/null +++ b/extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/test/prompt.test.ts @@ -0,0 +1,296 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as assert from 'assert'; +import * as sinon from 'sinon'; +import dedent from 'ts-dedent'; +import { + DEFAULT_MAX_COMPLETION_LENGTH, + DEFAULT_MAX_PROMPT_LENGTH, + DEFAULT_NUM_SNIPPETS, + DEFAULT_PROMPT_ALLOCATION_PERCENT, + DEFAULT_SUFFIX_MATCH_THRESHOLD, + PromptOptions, +} from '../../../../prompt/src/prompt'; +import { defaultSimilarFilesOptions } from '../../../../prompt/src/snippetInclusion/similarFiles'; +import { CopilotContentExclusionManager } from '../../contentExclusion/contentExclusionManager'; +import { Context } from '../../context'; +import { ExpTreatmentVariables } from '../../experiments/expConfig'; +import { TelemetryWithExp } from '../../telemetry'; +import { createLibTestingContext } from '../../test/context'; +import { createTextDocument, InMemoryNotebookDocument, TestTextDocumentManager } from '../../test/textDocument'; +import { INotebookCell, IPosition } from '../../textDocument'; +import { TextDocumentManager } from '../../textDocumentManager'; +import { CompletionsPromptRenderer } from '../components/completionsPromptRenderer'; +import { _copilotContentExclusion, _promptError, getPromptOptions } from '../prompt'; +import { extractPromptInternal } from './prompt'; + +suite('Prompt unit tests', function () { + let ctx: Context; + let sandbox: sinon.SinonSandbox; + + setup(function () { + sandbox = sinon.createSandbox(); + ctx = createLibTestingContext(); + }); + + teardown(function () { + sandbox.restore(); + }); + + test('defaults to 8K max prompt length', async function () { + const content = 'function add()\n'; + const sourceDoc = createTextDocument('file:///foo.js', 'javascript', 0, content); + const cursorPosition: IPosition = { + line: 0, + character: 13, + }; + + const rendererStub = sandbox.stub(CompletionsPromptRenderer.prototype, 'render').throws('unspecified error'); + + const prompt = await extractPromptInternal( + ctx, + 'COMPLETION_ID', + sourceDoc, + cursorPosition, + TelemetryWithExp.createEmptyConfigForTesting() + ); + + assert.deepStrictEqual(prompt, _promptError); + assert.ok(rendererStub.calledOnce, 'should call renderer'); + assert.strictEqual( + rendererStub.firstCall.args[1].promptTokenLimit, + 8192 - DEFAULT_MAX_COMPLETION_LENGTH, + 'should default to 8192 max total tokens, 7692 max prompt tokens' + ); + }); + + test('default EXP prompt options are the same as default PromptOptions object', function () { + const promptOptionsFromExp = getPromptOptions(ctx, TelemetryWithExp.createEmptyConfigForTesting(), ''); + const defaultPromptOptions: PromptOptions = { + maxPromptLength: DEFAULT_MAX_PROMPT_LENGTH, + numberOfSnippets: DEFAULT_NUM_SNIPPETS, + similarFilesOptions: defaultSimilarFilesOptions, + suffixMatchThreshold: DEFAULT_SUFFIX_MATCH_THRESHOLD, + suffixPercent: DEFAULT_PROMPT_ALLOCATION_PERCENT.suffix, + }; + + assert.deepStrictEqual(promptOptionsFromExp, defaultPromptOptions); + }); + + test('default C++ EXP prompt options use tuned values', function () { + const promptOptionsFromExp: PromptOptions = getPromptOptions( + ctx, + TelemetryWithExp.createEmptyConfigForTesting(), + 'cpp' + ); + + assert.deepStrictEqual(promptOptionsFromExp.similarFilesOptions, { + snippetLength: 60, + threshold: 0.0, + maxTopSnippets: 16, + maxCharPerFile: 100000, + maxNumberOfFiles: 200, + maxSnippetsPerFile: 4, + useSubsetMatching: false, + }); + assert.deepStrictEqual(promptOptionsFromExp.numberOfSnippets, 16); + }); + + test('default Java EXP prompt options are correct', function () { + const telemetryWithExp = TelemetryWithExp.createEmptyConfigForTesting(); + const expVars = telemetryWithExp.filtersAndExp.exp.variables; + + Object.assign(expVars, { + [ExpTreatmentVariables.UseSubsetMatching]: true, + }); + + const promptOptionsFromExp = getPromptOptions(ctx, telemetryWithExp, 'java'); + assert.deepStrictEqual(promptOptionsFromExp.similarFilesOptions, { + snippetLength: 60, + threshold: 0.0, + maxTopSnippets: 4, + maxCharPerFile: 10000, + maxNumberOfFiles: 20, + maxSnippetsPerFile: 1, + useSubsetMatching: true, + }); + assert.deepStrictEqual(promptOptionsFromExp.numberOfSnippets, 4); + }); + + test('should return without a prompt if the file blocked by repository control', async function () { + const evaluateStub = sandbox.stub(CopilotContentExclusionManager.prototype, 'evaluate'); + evaluateStub.callsFake(_ => { + return Promise.resolve({ isBlocked: true }); + }); + + const content = 'function add()\n'; + const sourceDoc = createTextDocument('file:///foo.js', 'javascript', 0, content); + const cursorPosition: IPosition = { + line: 0, + character: 13, + }; + const response = await extractPromptInternal( + ctx, + 'COMPLETION_ID', + sourceDoc, + cursorPosition, + TelemetryWithExp.createEmptyConfigForTesting() + ); + assert.ok(response); + assert.strictEqual(response, _copilotContentExclusion); + }); + + test('prompt for ipython notebooks, using only the current cell language as shebang', async function () { + await assertPromptForCell( + ctx, + cells[4], + dedent( + `import math + +def add(a, b): + return a + b + +def product(c, d):` + ), + ['#!/usr/bin/env python3'] + ); + }); + + test('prompt for ipython notebooks, using only the current cell language for known language', async function () { + await assertPromptForCell( + ctx, + cells[5], + dedent( + `def product(c, d):` + ), + ['Language: julia'] + ); + }); + + test('prompt for ipython notebooks, using only the current cell language for unknown language', async function () { + await assertPromptForCell(ctx, cells[6], dedent(`foo bar baz`), ['Language: unknown-great-language']); + }); + + test('exception telemetry', async function () { + this.skip(); + /* todo@dbaeumer need to understand how we handle exception in chat + class TestExceptionTextDocumentManager extends TestTextDocumentManager { + override textDocuments() { + return Promise.reject(new Error('test error')); + } + } + const tdm = new TestExceptionTextDocumentManager(ctx); + tdm.setTextDocument('file:///a/1.py', 'python', 'import torch'); + ctx.forceSet(TextDocumentManager, tdm); + NeighborSource.reset(); + + const { reporter, enhancedReporter } = await withInMemoryTelemetry(ctx, async ctx => { + const document = createTextDocument('file:///a/2.py', 'python', 0, 'import torch'); + await extractPromptInternal( + ctx, + 'COMPLETION_ID', + document, + { line: 0, character: 0 }, + TelemetryWithExp.createEmptyConfigForTesting() + ); + }); + + assert.ok(reporter.hasException); + assert.deepStrictEqual( + reporter.firstException?.properties?.origin, + 'PromptComponents.CompletionsPromptFactory' + ); + assert.strictEqual(reporter.exceptions.length, 1); + + assert.ok(enhancedReporter.hasException); + assert.deepStrictEqual( + enhancedReporter.firstException?.properties?.origin, + 'PromptComponents.CompletionsPromptFactory' + ); + assert.strictEqual(enhancedReporter.exceptions.length, 1); + */ + }); +}); + +async function assertPromptForCell(ctx: Context, sourceCell: INotebookCell, expectedPrefix: string, expectedContext?: string[]) { + const notebook = new InMemoryNotebookDocument(cells); + const sourceDoc = sourceCell.document; + + (ctx.get(TextDocumentManager) as TestTextDocumentManager).setNotebookDocument(sourceDoc, notebook); + + const cursorPosition: IPosition = { + line: 0, + character: sourceDoc.getText().length, + }; + const response = await extractPromptInternal( + ctx, + 'COMPLETION_ID', + sourceDoc, + cursorPosition, + TelemetryWithExp.createEmptyConfigForTesting() + ); + assert.ok(response); + assert.strictEqual(response.type, 'prompt'); + assert.strictEqual(response.prompt.prefix, expectedPrefix); + if (expectedContext !== undefined) { + assert.deepEqual(response.prompt.context, expectedContext); + } +} + +const cells: INotebookCell[] = [ + { + index: 1, + document: createTextDocument('file:///test/a.ipynb#1', 'python', 1, 'import math'), + metadata: {}, + kind: 2, + }, + { + index: 2, + document: createTextDocument( + 'file:///test/a.ipynb#2', + 'markdown', + 1, + 'This is an addition function\nIt is used to add two numbers' + ), + metadata: {}, + kind: 1, + }, + { + index: 3, + document: createTextDocument('file:///test/a.ipynb#3', 'python', 2, 'def add(a, b):\n return a + b'), + metadata: {}, + kind: 2, + }, + { + index: 4, + document: createTextDocument( + 'file:///test/a.ipynb#4', + 'markdown', + 2, + 'This is a product function\nYou guessed it: it multiplies two numbers' + ), + metadata: {}, + kind: 2, + }, + { + index: 5, + document: createTextDocument('file:///test/a.ipynb#5', 'python', 3, 'def product(c, d):'), + metadata: {}, + kind: 2, + }, + { + index: 6, + document: createTextDocument('file:///test/a.ipynb#6', 'julia', 3, 'def product(c, d):'), + metadata: {}, + kind: 2, + }, + { + index: 7, + document: createTextDocument('file:///test/a.ipynb#7', 'unknown-great-language', 3, 'foo bar baz'), + metadata: {}, + kind: 2, + }, +]; diff --git a/extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/test/repository.test.ts b/extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/test/repository.test.ts new file mode 100644 index 00000000000..7eaa5fb6e80 --- /dev/null +++ b/extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/test/repository.test.ts @@ -0,0 +1,97 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; +import path from 'path'; +import { Context } from '../../context'; +import { FileSystem } from '../../fileSystem'; +import { createLibTestingContext } from '../../test/context'; +import { FakeFileSystem } from '../../test/filesystem'; +import { makeFsUri } from '../../util/uri'; +import { ComputationStatus, extractRepoInfo, extractRepoInfoInBackground } from '../repository'; + +suite('Extract repo info tests', function () { + const baseFolder = { uri: makeFsUri(path.resolve(__dirname, '../../../../../../../../')) }; + + class Nested { + nested: Nested | undefined; + } + + test('avoid using context as cache key', function () { + const ctx = new Context(); + ctx.set(FileSystem, new FakeFileSystem({})); + const n = new Nested(); + ctx.set(Nested, n); + n.nested = n; + + const maybe = extractRepoInfoInBackground(ctx, makeFsUri(__filename)); + + assert.deepStrictEqual(maybe, ComputationStatus.PENDING); + }); + + test('Extract repo info', async function () { + const ctx = createLibTestingContext(); + const info = await extractRepoInfo(ctx, baseFolder.uri); + + assert.ok(info); + + // url and pathname get their own special treatment because they depend on how the repo was cloned. + const { url, pathname, repoId, ...repoInfo } = info; + + assert.deepStrictEqual(repoInfo, { + baseFolder, + hostname: 'github.com' + }); + assert.ok(repoId); + assert.deepStrictEqual( + { org: repoId.org, repo: repoId.repo, type: repoId.type }, + { org: 'microsoft', repo: 'vscode-copilot-chat', type: 'github' } + ); + assert.ok( + [ + 'git@github.com:microsoft/vscode-copilot-chat', + 'https://github.com/microsoft/vscode-copilot-chat', + 'https://github.com/microsoft/vscode-copilot-chat.git', + ].includes(url), + `url is ${url}` + ); + assert.ok(pathname.startsWith('/github/vscode-copilot-chat') || pathname.startsWith('/microsoft/vscode-copilot-chat')); + + assert.deepStrictEqual(await extractRepoInfo(ctx, 'file:///tmp/does/not/exist/.git/config'), undefined); + }); + + test('Extract repo info - Jupyter Notebook vscode-notebook-cell ', async function () { + const cellUri = baseFolder.uri.replace(/^file:/, 'vscode-notebook-cell:'); + assert.ok(cellUri.startsWith('vscode-notebook-cell:')); + const ctx = createLibTestingContext(); + const info = await extractRepoInfo(ctx, cellUri); + + assert.ok(info); + + // url and pathname get their own special treatment because they depend on how the repo was cloned. + const { url, pathname, repoId, ...repoInfo } = info; + + assert.deepStrictEqual(repoInfo, { + baseFolder, + hostname: 'github.com' + }); + assert.ok(repoId); + assert.deepStrictEqual( + { org: repoId.org, repo: repoId.repo, type: repoId.type }, + { org: 'microsoft', repo: 'vscode-copilot-chat', type: 'github' } + ); + assert.ok( + [ + 'git@github.com:microsoft/vscode-copilot-chat', + 'https://github.com/microsoft/vscode-copilot-chat', + 'https://github.com/microsoft/vscode-copilot-chat.git', + ].includes(url), + `url is ${url}` + ); + assert.ok(pathname.startsWith('/github/vscode-copilot-chat') || pathname.startsWith('/microsoft/vscode-copilot-chat')); + + assert.deepStrictEqual(await extractRepoInfo(ctx, 'file:///tmp/does/not/exist/.git/config'), undefined); + }); +}); diff --git a/extensions/copilot/src/extension/completions-core/vscode-node/prompt/src/test/indentation.test.ts b/extensions/copilot/src/extension/completions-core/vscode-node/prompt/src/test/indentation.test.ts index 3795835a54f..cec97d0fcf6 100644 --- a/extensions/copilot/src/extension/completions-core/vscode-node/prompt/src/test/indentation.test.ts +++ b/extensions/copilot/src/extension/completions-core/vscode-node/prompt/src/test/indentation.test.ts @@ -40,11 +40,11 @@ function doParseTest(source: string, expectedTree: IndentationTree) { const SOURCE = { source: dedent` - f1: - a1 - f2: - a2 - a3 +f1: + a1 +f2: + a2 + a3 `, name: '', }; @@ -52,79 +52,79 @@ const SOURCE = { suite('Test compareTreeWithSpec', function () { const SOURCE_MISSING_CHILD = { source: dedent` - f1: - a1 - f2: - a2 - `, +f1: + a1 +f2: + a2 +`, name: 'missing child', }; const SOURCE_EXTRA_CHILD = { source: dedent` - f1: - a1 - f2: - a2 - a3 - a4 - `, +f1: + a1 +f2: + a2 + a3 + a4 +`, name: 'extra_child', }; const SOURCE_MISSING_SIBLING = { source: dedent` - f1: - a1 - `, +f1: + a1 +`, name: 'missing sibling', }; const SOURCE_EXTRA_SIBLING = { source: dedent` - f1: - a1 - f2: - a2 - a3 - f3: - a4 - `, +f1: + a1 +f2: + a2 + a3 +f3: + a4 +`, name: 'extra_sibling', }; const SOURCE_EXTRA_MIDDLE_BLANK_LINE = { source: dedent` - f1: - a1 +f1: + a1 - f2: - a2 - a3 - `, +f2: + a2 + a3 +`, name: 'extra middle blank line', }; const SOURCE_EXTRA_TRAILING_BLANK_LINE = { source: dedent` - f1: - a1 - f2: - a2 - a3 +f1: + a1 +f2: + a2 + a3 - `, +`, name: 'extra trailing blank line', }; const SOURCE_EXTRA_INDENTATION = { source: dedent` - f1: - a1 - f2: - a2 - a3 - `, +f1: + a1 +f2: + a2 + a3 +`, name: 'extra indentation', }; @@ -334,15 +334,15 @@ suite('Test core functions: other', function () { }); test('deparseAndCutTree cuts at labels', function () { const source = dedent` - 1 - 2 - 3 - 4 - 5 - 6 - 7 - 8 - 9`; +1 + 2 + 3 +4 + 5 + 6 +7 + 8 + 9`; const tree = parseRaw(source) as IndentationTree; tree.subs[0].subs[1].label = 'cut'; tree.subs[1].subs[0].label = 'cut'; @@ -390,15 +390,15 @@ suite('Test core functions: other', function () { }); test('VisitTreeConditionally', function () { const tree = parseRaw(dedent` - 1 - 2 - 3 - 4 - 5 - 6 - 7 - 8 - 9`); +1 + 2 + 3 +4 + 5 + 6 +7 + 8 + 9`); const traceTopDownAll: string[] = []; visitTree( tree, diff --git a/extensions/copilot/src/extension/completions-core/vscode-node/prompt/src/test/indentationLanguages.test.ts b/extensions/copilot/src/extension/completions-core/vscode-node/prompt/src/test/indentationLanguages.test.ts index 7db294677ca..fe3e0a1bce6 100644 --- a/extensions/copilot/src/extension/completions-core/vscode-node/prompt/src/test/indentationLanguages.test.ts +++ b/extensions/copilot/src/extension/completions-core/vscode-node/prompt/src/test/indentationLanguages.test.ts @@ -2,10 +2,10 @@ * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { blankNode, isLine, lineNode, parseTree, topNode, virtualNode, visitTree } from '../indentation'; import * as assert from 'assert'; import { suite } from 'mocha'; import dedent from 'ts-dedent'; +import { blankNode, isLine, lineNode, parseTree, topNode, virtualNode, visitTree } from '../indentation'; import { compareTreeWithSpec } from './testHelpers'; /** Test some language specific parsing techniques */ @@ -57,23 +57,23 @@ suite('Java', function () { test('labelLines java', function () { const tree = parseTree( dedent` - package com.example; - import java.awt.*; - @annotation - final public class A { - /** A javadoc - * Second line - */ - public static void main(String[] args) { - // single-line comment - /* Multiline - * comment - */ - System.out.println("Hello, world!"); - } - } - public interface I { } - `, +package com.example; +import java.awt.*; +@annotation +final public class A { + /** A javadoc + * Second line + */ + public static void main(String[] args) { + // single-line comment + /* Multiline + * comment + */ + System.out.println("Hello, world!"); + } +} +public interface I { } +`, 'java' ); compareTreeWithSpec( @@ -113,14 +113,14 @@ suite('Java', function () { //TODO: Add a field with annotation on separate line const tree = parseTree( dedent` - class A { - int a; - /** Javadoc */ - int b; - // Comment - @Native int c; - } - `, +class A { + int a; + /** Javadoc */ + int b; + // Comment + @Native int c; +} +`, 'java' ); compareTreeWithSpec( @@ -147,18 +147,18 @@ suite('Java', function () { test('parse Java inner class', function () { const tree = parseTree( dedent` - class A { - int a; +class A { + int a; - class Inner { - int b; - } + class Inner { + int b; + } - interface InnerInterface { - int myMethod(); - } - } - `, + interface InnerInterface { + int myMethod(); + } +} +`, 'java' ); compareTreeWithSpec( @@ -198,25 +198,25 @@ suite('Java', function () { suite('Markdown', function () { test('header processing in markdown', function () { const source = dedent` - A +A - # B - C - D +# B +C +D - ## E - F - G +## E +F +G - # H - I +# H +I - ### J - K +### J +K - L - M - `; +L +M +`; const mdParsedTree = parseTree(source, 'markdown'); compareTreeWithSpec( diff --git a/extensions/copilot/src/extension/completions-core/vscode-node/prompt/src/test/indentationParsing.test.ts b/extensions/copilot/src/extension/completions-core/vscode-node/prompt/src/test/indentationParsing.test.ts index e1fe264bf92..9d465e324df 100644 --- a/extensions/copilot/src/extension/completions-core/vscode-node/prompt/src/test/indentationParsing.test.ts +++ b/extensions/copilot/src/extension/completions-core/vscode-node/prompt/src/test/indentationParsing.test.ts @@ -68,17 +68,17 @@ suite('Test core parsing elements', function () { test('groupBlocks basic cases', function () { const source = dedent` - A +A - B - C - D +B +C +D - E - F +E +F - G - H`; +G +H`; const tree = parseRaw(source); const blockTree = groupBlocks(tree); function assertChildrenAreTheFollowingLines( @@ -104,24 +104,24 @@ suite('Test core parsing elements', function () { // blank lines after last child, lone blank lines, // consecutive lone blank lines, offside blocks let tree = parseRaw(dedent` - A +A - B - C - D + B + C + D - E + E - F + F - G - H - I - J +G + H + I + J - K - `); + K +`); tree = groupBlocks(tree); compareTreeWithSpec( tree, @@ -156,13 +156,13 @@ suite('Test core parsing elements', function () { test('groupBlocks consecutive blanks as oldest children', function () { let tree = parseRaw(dedent` - A +A - B1 - B2 - C - `); + B1 + B2 +C +`); tree = groupBlocks(tree); compareTreeWithSpec( tree, @@ -200,12 +200,12 @@ suite('Test core parsing elements', function () { test('groupBlocks with different delimiter', function () { let tree = parseRaw(dedent` - A - B - C - D - E - `) as IndentationTree; +A +B +C +D +E +`) as IndentationTree; const isDelimiter = (node: IndentationTree) => isLine(node) && (node.sourceLine.trim() === 'B' || node.sourceLine.trim() === 'D'); tree = groupBlocks(tree, isDelimiter); @@ -224,19 +224,19 @@ suite('Raw parsing', function () { test('parseRaw', function () { compareTreeWithSpec( parseRaw(dedent` - A - a - B - b1 - b2 - C - c1 - c2 - c3 - D - d1 - d2 - `), +A + a +B + b1 + b2 +C + c1 + c2 + c3 +D + d1 + d2 +`), topNode([ lineNode(0, 0, 'A', [lineNode(2, 1, 'a', [])]), lineNode(0, 2, 'B', [lineNode(2, 3, 'b1', []), lineNode(2, 4, 'b2', [])]), @@ -249,19 +249,19 @@ suite('Raw parsing', function () { test('parseRaw blanks', function () { compareTreeWithSpec( parseRaw(dedent` - E - e1 +E + e1 - e2 - F + e2 +F - f1 - G - g1 + f1 +G + g1 - H +H - `), +`), topNode([ lineNode(0, 0, 'E', [lineNode(2, 1, 'e1', []), blankNode(2), lineNode(2, 3, 'e2', [])]), lineNode(0, 4, 'F', [blankNode(5), lineNode(2, 6, 'f1', [])]), @@ -275,24 +275,24 @@ suite('Raw parsing', function () { test('combineBraces', function () { const tree = parseTree(dedent` - A { - } - B - b1 { - bb1 - } - b2 { - bb2 +A { +} +B + b1 { + bb1 + } + b2 { + bb2 - } - } - C { - c1 - c2 - c3 - c4 - } - `); + } +} +C { + c1 + c2 + c3 + c4 +} +`); compareTreeWithSpec( tree, topNode([ @@ -329,10 +329,10 @@ suite('Raw parsing', function () { suite('Test bracket indentation spec', function () { test('Opener merged to older sibling', function () { const source = dedent` - A - ( - B - C`; +A +( + B + C`; const treeRaw = parseRaw(source); const treeCode = parseTree(source, ''); @@ -357,9 +357,9 @@ suite('Test bracket indentation spec', function () { test('Closer merged, simplest case', function () { const source = dedent` - A - B - )`; +A + B +)`; const treeRaw = parseRaw(source); const treeCode = parseTree(source, ''); @@ -378,13 +378,13 @@ suite('Test bracket indentation spec', function () { test('Closer merged, multi-body case', function () { const source = dedent` - A - B - C - ) + ( - D - E - )`; +A + B + C +) + ( + D + E +)`; const treeRaw = parseRaw(source); const treeCode = parseTree(source, ''); @@ -402,20 +402,20 @@ suite('Test bracket indentation spec', function () { test('closer starting their next subblock, ifelse', function () { const source = dedent` - if (new) { - print(“hello”) - print(“world”) - } else { - print(“goodbye”) - }`; + if (new) { + print(“hello”) + print(“world”) + } else { + print(“goodbye”) + }`; const sourceParsedAsIf = dedent` - if (new) { - -> virtual - print(“hello”) - print(“world”) - } else { - print(“goodbye”) - }`; + if (new) { + -> virtual + print(“hello”) + print(“world”) + } else { + print(“goodbye”) + }`; const treeRaw = parseRaw(source); const treeCode = parseTree(source, ''); @@ -425,10 +425,10 @@ suite('Test bracket indentation spec', function () { treeRaw, topNode([ lineNode(0, 0, 'if (new) {', [ - lineNode(4, 1, 'print(“hello”)', []), - lineNode(4, 2, 'print(“world”)', []), + lineNode(4, 1, 'print(“hello”)', []), + lineNode(4, 2, 'print(“world”)', []), ]), - lineNode(0, 3, '} else {', [lineNode(4, 4, 'print(“goodbye”)', [])]), + lineNode(0, 3, '} else {', [lineNode(4, 4, 'print(“goodbye”)', [])]), lineNode(0, 5, '}', []), ]) ); @@ -436,8 +436,8 @@ suite('Test bracket indentation spec', function () { treeCode, topNode([ lineNode(0, 0, 'if (new) {', [ - virtualNode(0, [lineNode(4, 1, 'print(“hello”)', []), lineNode(4, 2, 'print(“world”)', [])]), - lineNode(0, 3, '} else {', [lineNode(4, 4, 'print(“goodbye”)', [])]), + virtualNode(0, [lineNode(4, 1, 'print(“hello”)', []), lineNode(4, 2, 'print(“world”)', [])]), + lineNode(0, 3, '} else {', [lineNode(4, 4, 'print(“goodbye”)', [])]), lineNode(0, 5, '}', []), ]), ]) @@ -449,11 +449,11 @@ suite('Test bracket indentation spec', function () { suite('Special indentation styles', function () { test('Allman style example (function)', function () { const source = dedent` - function test() - { - print(“hello”) - print(“world”) - }`; + function test() + { + print(“hello”) + print(“world”) + }`; const treeRaw = parseRaw(source); const treeCode = parseTree(source, ''); @@ -464,8 +464,8 @@ suite('Special indentation styles', function () { topNode([ lineNode(0, 0, 'function test()', [ lineNode(0, 1, '{', [], 'opener'), - lineNode(4, 2, 'print(“hello”)', []), - lineNode(4, 3, 'print(“world”)', []), + lineNode(4, 2, 'print(“hello”)', []), + lineNode(4, 3, 'print(“world”)', []), lineNode(0, 4, '}', [], 'closer'), ]), ]) @@ -476,7 +476,7 @@ suite('Special indentation styles', function () { treeRaw, topNode([ lineNode(0, 0, 'function test()', []), - lineNode(0, 1, '{', [lineNode(4, 2, 'print(“hello”)', []), lineNode(4, 3, 'print(“world”)', [])]), + lineNode(0, 1, '{', [lineNode(4, 2, 'print(“hello”)', []), lineNode(4, 3, 'print(“world”)', [])]), lineNode(0, 4, '}', []), ]) ); @@ -485,17 +485,17 @@ suite('Special indentation styles', function () { /** This test is a case where our parsing isn't yet optimal */ test('Allman style example (if-then-else)', function () { const source = dedent` - if (condition) - { - print(“hello”) - print(“world”) - } - else - { - print(“goodbye”) - print(“phone”) - } - `; + if (condition) + { + print(“hello”) + print(“world”) + } + else + { + print(“goodbye”) + print(“phone”) + } + `; const treeCode = parseTree(source, ''); @@ -506,14 +506,14 @@ suite('Special indentation styles', function () { topNode([ lineNode(0, 0, 'if (condition)', [ lineNode(0, 1, '{', [], 'opener'), - lineNode(4, 2, 'print(“hello”)', []), - lineNode(4, 3, 'print(“world”)', []), + lineNode(4, 2, 'print(“hello”)', []), + lineNode(4, 3, 'print(“world”)', []), lineNode(0, 4, '}', [], 'closer'), ]), lineNode(0, 5, 'else ', [ lineNode(0, 6, '{', [], 'opener'), - lineNode(4, 7, 'print(“goodbye”)', []), - lineNode(4, 8, 'print(“phone”)', []), + lineNode(4, 7, 'print(“goodbye”)', []), + lineNode(4, 8, 'print(“phone”)', []), lineNode(0, 9, '}', [], 'closer'), ]), ]) @@ -522,14 +522,14 @@ suite('Special indentation styles', function () { test('K&R style example (if-then-else)', function () { const source = dedent` - if (condition) { - print(“hello”) - print(“world”) - } else { - print(“goodbye”) - print(“phone”) - } - `; + if (condition) { + print(“hello”) + print(“world”) + } else { + print(“goodbye”) + print(“phone”) + } + `; const treeCode = parseTree(source, ''); @@ -539,12 +539,12 @@ suite('Special indentation styles', function () { treeCode, topNode([ lineNode(0, 0, 'if (condition) {', [ - virtualNode(0, [lineNode(4, 2, 'print(“hello”)', []), lineNode(4, 3, 'print(“world”)', [])]), + virtualNode(0, [lineNode(4, 2, 'print(“hello”)', []), lineNode(4, 3, 'print(“world”)', [])]), lineNode( 0, 4, '} else {', - [lineNode(4, 5, 'print(“goodbye”)', []), lineNode(4, 6, 'print(“phone”)', [])], + [lineNode(4, 5, 'print(“goodbye”)', []), lineNode(4, 6, 'print(“phone”)', [])], 'closer' ), lineNode(0, 7, '}', [], 'closer'), @@ -555,11 +555,11 @@ suite('Special indentation styles', function () { test('combineBraces GNU style indentation 1', function () { let tree: IndentationTree = parseRaw(dedent` - A - { - stmt - } - `); +A + { + stmt + } +`); labelLines(tree, buildLabelRules({ opener: /^{$/, closer: /^}$/ })); tree = combineClosersAndOpeners(tree); compareTreeWithSpec( @@ -574,15 +574,15 @@ suite('Special indentation styles', function () { test('combineBraces GNU style indentation 2', function () { let tree: IndentationTree = parseRaw(dedent` - B - { - stmt +B +{ + stmt - } +} - end - `); +end +`); labelLines(tree, buildLabelRules({ opener: /^{$/, closer: /^}$/ })); tree = combineClosersAndOpeners(tree); tree = flattenVirtual(tree); @@ -604,11 +604,11 @@ suite('Special indentation styles', function () { test('combineBraces GNU style indentation 3', function () { let tree: IndentationTree = parseRaw(dedent` - C - { +C +{ - } - `); +} +`); labelLines(tree, buildLabelRules({ opener: /^{$/, closer: /^}$/ })); tree = combineClosersAndOpeners(tree); tree = flattenVirtual(tree); @@ -626,15 +626,15 @@ suite('Special indentation styles', function () { test('combineBraces GNU style indentation 4', function () { let tree: IndentationTree = parseRaw(dedent` - D - { - d - { - stmt +D +{ + d + { + stmt - } - } - `); + } +} +`); labelLines(tree, buildLabelRules({ opener: /^{$/, closer: /^}$/ })); tree = combineClosersAndOpeners(tree); tree = flattenVirtual(tree); diff --git a/extensions/copilot/src/extension/completions-core/vscode-node/prompt/src/test/parseBlock.test.ts b/extensions/copilot/src/extension/completions-core/vscode-node/prompt/src/test/parseBlock.test.ts index 80dd49ea0fb..dbc89bb5955 100644 --- a/extensions/copilot/src/extension/completions-core/vscode-node/prompt/src/test/parseBlock.test.ts +++ b/extensions/copilot/src/extension/completions-core/vscode-node/prompt/src/test/parseBlock.test.ts @@ -26,8 +26,8 @@ enum TrimMode { * A convenience class for testing BlockParser.isEmptyBlockStart. * * To use this, pass a string containing a snippet of source code, and use - * 🟢 for cursor positions at which isEmptyBlockStart should return true, - * and ❌ for cursor positions where it should return false. Then call + * 🟢 for cursor positions at which isEmptyBlockStart should return true, + * and ❌ for cursor positions where it should return false. Then call * .test() to run the tests. * * By default, for each cursor position it trims the line from the cursor @@ -51,10 +51,10 @@ class IsEmptyBlockStartTestCase { // Must use for...of loop to avoid surrogate pair/UTF-16 weirdness for (const char of testCase) { switch (char) { - case '🟢': + case '🟢': expectTrueOffsets.push(i); break; - case '❌': + case '❌': expectFalseOffsets.push(i); break; default: @@ -92,13 +92,13 @@ class IsEmptyBlockStartTestCase { const blockParser = getBlockParser(this.languageId); for (const offset of this.expectTrueOffsets) { const text = this.trimText(offset); - const msg = `${this.text.slice(0, offset)}â–ˆ${this.text.slice(offset)}`; + const msg = `${this.text.slice(0, offset)}█${this.text.slice(offset)}`; // common helper to all breaks assert.strictEqual(await blockParser.isEmptyBlockStart(text, offset), true, msg); } for (const offset of this.expectFalseOffsets) { const text = this.trimText(offset); - const msg = `${this.text.slice(0, offset)}â–ˆ${this.text.slice(offset)}`; + const msg = `${this.text.slice(0, offset)}█${this.text.slice(offset)}`; assert.strictEqual(await blockParser.isEmptyBlockStart(text, offset), false, msg); } } @@ -142,7 +142,7 @@ function runTestCase(languageId: string, testCase: TestCase) { // cursor position is after the before text const offset = testCase.before.length; // print the text with a cursor indicator on failure - const prettyPrint = ('\n' + testCase.before + 'â–ˆ' + bodyWithAfter).split('\n').join('\n\t| '); + const prettyPrint = ('\n' + testCase.before + '█' + bodyWithAfter).split('\n').join('\n\t| '); test(`empty block start:${expectedEmpty}`, async function () { const isEmpty = await blockParser.isEmptyBlockStart(text, offset); @@ -174,15 +174,15 @@ function getNodeStartTestCase(testCase: string): [string, number[], number[], nu for (const char of testCase) { switch (char) { //Test cases that should pass the test - case '🟢': + case '🟢': positiveTests.push(i); break; //Test cases that should fail the test - case '❌': + case '❌': rejectedTests.push(i); break; //Location used for the assertions (begining of the node we want to detect) - case '🔵': + case '🔵': expectedResult = i; break; default: @@ -199,9 +199,9 @@ function getNodeStartTestCase(testCase: string): [string, number[], number[], nu * Helper function for testing `getNodeStart` * * To use this, pass a language ID and a string containing a snippet of source code, and use - * 🔵 for a location that's used for assertion ( begining of the node we want to detect) - * 🟢 for cursor positions at which `getNodeStart` should return the position 🔵, - * and ❌ for cursor positions where it shouldn't. + * 🔵 for a location that's used for assertion ( begining of the node we want to detect) + * 🟢 for cursor positions at which `getNodeStart` should return the position 🔵, + * and ❌ for cursor positions where it shouldn't. */ async function testGetNodeStart(languageId: string, testCase: string) { const blockParser = getBlockParser(languageId); @@ -240,9 +240,9 @@ suite('parseBlock Tests', function () { suite('Python isEmptyBlockStart tests', function () { test('Invalid positions', async function () { const text = dedent` - def foo(): - pass - `; + def foo(): + pass + `; const blockParser = getBlockParser('python'); await assert.rejects(blockParser.isEmptyBlockStart(text, text.length + 1)); }); @@ -250,53 +250,53 @@ suite('parseBlock Tests', function () { test('simple examples', async function () { const testCases: IsEmptyBlockStartTestCase[] = [ IsEmptyBlockStartTestCase.python(dedent` - ❌d❌e❌f🟢 🟢f🟢o🟢o🟢(🟢)🟢:🟢 - 🟢p❌a❌s❌s❌ - `), + ❌d❌e❌f🟢 🟢f🟢o🟢o🟢(🟢)🟢:🟢 + 🟢p❌a❌s❌s❌ + `), IsEmptyBlockStartTestCase.python(dedent` - ❌i❌f🟢 🟢f🟢o🟢o🟢:🟢 - 🟢p❌a❌s❌s❌ - ❌e❌l❌i❌f🟢 🟢b🟢a🟢r🟢:🟢 - 🟢p❌a❌s❌s❌ - e❌l❌s❌e🟢:🟢 - 🟢p❌a❌ss❌ - `), + ❌i❌f🟢 🟢f🟢o🟢o🟢:🟢 + 🟢p❌a❌s❌s❌ + ❌e❌l❌i❌f🟢 🟢b🟢a🟢r🟢:🟢 + 🟢p❌a❌s❌s❌ + e❌l❌s❌e🟢:🟢 + 🟢p❌a❌ss❌ + `), IsEmptyBlockStartTestCase.python(dedent` - ❌i❌f🟢 🟢f🟢o🟢o🟢:🟢 - 🟢p❌a❌s❌s❌ - e❌l❌s❌e🟢:🟢 - 🟢p❌a❌s❌s❌ - `), + ❌i❌f🟢 🟢f🟢o🟢o🟢:🟢 + 🟢p❌a❌s❌s❌ + e❌l❌s❌e🟢:🟢 + 🟢p❌a❌s❌s❌ + `), IsEmptyBlockStartTestCase.python(dedent` - ❌t❌r❌y🟢:🟢 - 🟢p❌a❌s❌s❌ - ❌e❌x❌c❌e❌p❌t🟢 🟢E🟢:🟢 - 🟢p❌a❌s❌s❌ - ❌f❌i❌n❌a❌l❌l❌y🟢:🟢 - 🟢p❌a❌s❌s❌ - `), + ❌t❌r❌y🟢:🟢 + 🟢p❌a❌s❌s❌ + ❌e❌x❌c❌e❌p❌t🟢 🟢E🟢:🟢 + 🟢p❌a❌s❌s❌ + ❌f❌i❌n❌a❌l❌l❌y🟢:🟢 + 🟢p❌a❌s❌s❌ + `), IsEmptyBlockStartTestCase.python(dedent` - ❌t❌r❌y🟢:🟢 - 🟢p❌a❌s❌s❌ - ❌f❌i❌n❌a❌l❌l❌y🟢:🟢 - 🟢p❌a❌s❌s❌ - `), + ❌t❌r❌y🟢:🟢 + 🟢p❌a❌s❌s❌ + ❌f❌i❌n❌a❌l❌l❌y🟢:🟢 + 🟢p❌a❌s❌s❌ + `), IsEmptyBlockStartTestCase.python(dedent` - ❌f❌o❌r🟢 🟢f🟢o🟢o🟢 🟢i🟢n🟢 🟢b🟢a🟢r🟢:🟢 - 🟢p❌a❌s❌s❌ - `), + ❌f❌o❌r🟢 🟢f🟢o🟢o🟢 🟢i🟢n🟢 🟢b🟢a🟢r🟢:🟢 + 🟢p❌a❌s❌s❌ + `), IsEmptyBlockStartTestCase.python(dedent` - ❌w❌h❌i❌l❌e🟢 🟢f🟢o🟢o🟢:🟢 - 🟢p❌a❌s❌s❌ - `), + ❌w❌h❌i❌l❌e🟢 🟢f🟢o🟢o🟢:🟢 + 🟢p❌a❌s❌s❌ + `), IsEmptyBlockStartTestCase.python(dedent` - ❌w❌i❌t❌h🟢 🟢o🟢p🟢e🟢n🟢(🟢)🟢 🟢a🟢s🟢 🟢f🟢:🟢 - 🟢p❌a❌s❌s❌ - `), + ❌w❌i❌t❌h🟢 🟢o🟢p🟢e🟢n🟢(🟢)🟢 🟢a🟢s🟢 🟢f🟢:🟢 + 🟢p❌a❌s❌s❌ + `), IsEmptyBlockStartTestCase.python(dedent` - ❌c❌l❌a❌s❌s🟢 🟢F🟢o🟢o🟢:🟢 - 🟢p❌a❌s❌s❌ - `), + ❌c❌l❌a❌s❌s🟢 🟢F🟢o🟢o🟢:🟢 + 🟢p❌a❌s❌s❌ + `), ]; for (const testCase of testCases) { @@ -306,12 +306,12 @@ suite('parseBlock Tests', function () { test('func_decl', async function () { const testCases = [ - IsEmptyBlockStartTestCase.python('❌d❌e❌f🟢 🟢f🟢o🟢o🟢(🟢)🟢:🟢'), + IsEmptyBlockStartTestCase.python('❌d❌e❌f🟢 🟢f🟢o🟢o🟢(🟢)🟢:🟢'), IsEmptyBlockStartTestCase.python(dedent` - ❌d❌e❌f🟢 🟢f🟢o🟢o🟢(🟢)🟢:🟢 - 🟢 🟢 🟢 🟢 🟢 - 🟢 - `), + ❌d❌e❌f🟢 🟢f🟢o🟢o🟢(🟢)🟢:🟢 + 🟢 🟢 🟢 🟢 🟢 + 🟢 + `), ]; for (const testCase of testCases) { @@ -321,11 +321,11 @@ suite('parseBlock Tests', function () { test('multiline_func_decl', async function () { const testCase = IsEmptyBlockStartTestCase.python(dedent` - ❌d❌e❌f🟢 🟢f🟢o🟢o🟢(🟢a🟢,🟢 - 🟢b🟢,🟢 - 🟢c🟢)🟢:🟢 - 🟢 - `); + ❌d❌e❌f🟢 🟢f🟢o🟢o🟢(🟢a🟢,🟢 + 🟢b🟢,🟢 + 🟢c🟢)🟢:🟢 + 🟢 + `); await testCase.test(); }); @@ -334,13 +334,13 @@ suite('parseBlock Tests', function () { // Trailing whitespace is intentional, do not remove! const testCase = IsEmptyBlockStartTestCase.python( dedent` - """This is a module.""" - import foo + """This is a module.""" + import foo - ❌d❌e❌f🟢 🟢f🟢u🟢n🟢c🟢1🟢(🟢)🟢:🟢 🟢 🟢 + ❌d❌e❌f🟢 🟢f🟢u🟢n🟢c🟢1🟢(🟢)🟢:🟢 🟢 🟢 - print("Running at toplevel") - ` + print("Running at toplevel") + ` ).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE); // break 1 await testCase.test(); @@ -348,7 +348,7 @@ suite('parseBlock Tests', function () { test('func_decl_with_type_hints', async function () { const testCase = IsEmptyBlockStartTestCase.python( - '❌d❌e❌f🟢 🟢s🟢u🟢m🟢(🟢a🟢:🟢 🟢i🟢n🟢t🟢,🟢 🟢b🟢:🟢 🟢i🟢n🟢t🟢)🟢 🟢-🟢>🟢 🟢I🟢n🟢t🟢:🟢' + '❌d❌e❌f🟢 🟢s🟢u🟢m🟢(🟢a🟢:🟢 🟢i🟢n🟢t🟢,🟢 🟢b🟢:🟢 🟢i🟢n🟢t🟢)🟢 🟢-🟢>🟢 🟢I🟢n🟢t🟢:🟢' ); await testCase.test(); }); @@ -356,11 +356,11 @@ suite('parseBlock Tests', function () { test('block not empty', async function () { const testCase = IsEmptyBlockStartTestCase.python( dedent` - def func1(): - ❌ - pass❌ - ❌ - ` + def func1(): + ❌ + pass❌ + ❌ + ` ).setTrimMode(TrimMode.NO_TRIM); await testCase.test(); }); @@ -368,13 +368,13 @@ suite('parseBlock Tests', function () { test('docstring', async function () { const testCases = [ IsEmptyBlockStartTestCase.python(dedent` - def my_func(): - 🟢 🟢 🟢 🟢 🟢"🟢"🟢"🟢T🟢h🟢i🟢s🟢 🟢i🟢s🟢 🟢a🟢 🟢d🟢o🟢c🟢s🟢t🟢r🟢i🟢n🟢g🟢.🟢"🟢"🟢"🟢 - `), + def my_func(): + 🟢 🟢 🟢 🟢 🟢"🟢"🟢"🟢T🟢h🟢i🟢s🟢 🟢i🟢s🟢 🟢a🟢 🟢d🟢o🟢c🟢s🟢t🟢r🟢i🟢n🟢g🟢.🟢"🟢"🟢"🟢 + `), IsEmptyBlockStartTestCase.python(dedent` - def my_func(): - 🟢 🟢 🟢 🟢 🟢'🟢'🟢'🟢T🟢h🟢i🟢s🟢 🟢i🟢s🟢 🟢a🟢 🟢d🟢o🟢c🟢s🟢t🟢r🟢i🟢n🟢g🟢.🟢'🟢'🟢'🟢 - `), + def my_func(): + 🟢 🟢 🟢 🟢 🟢'🟢'🟢'🟢T🟢h🟢i🟢s🟢 🟢i🟢s🟢 🟢a🟢 🟢d🟢o🟢c🟢s🟢t🟢r🟢i🟢n🟢g🟢.🟢'🟢'🟢'🟢 + `), ]; for (const testCase of testCases) { // break 2 @@ -385,17 +385,17 @@ suite('parseBlock Tests', function () { test('multiline docstring', async function () { const testCases = [ IsEmptyBlockStartTestCase.python(dedent` - def my_func(): - """🟢T🟢h🟢i🟢s🟢 🟢i🟢s🟢 🟢a🟢 🟢m🟢u🟢l🟢t🟢i🟢l🟢i🟢n🟢e🟢 🟢d🟢o🟢c🟢s🟢t🟢r🟢i🟢n🟢g🟢.🟢 - 🟢 - 🟢H🟢e🟢r🟢e🟢'🟢s🟢 🟢a🟢n🟢o🟢t🟢h🟢e🟢r🟢 🟢l🟢i🟢n🟢e🟢.🟢"🟢"🟢"🟢 - `), + def my_func(): + """🟢T🟢h🟢i🟢s🟢 🟢i🟢s🟢 🟢a🟢 🟢m🟢u🟢l🟢t🟢i🟢l🟢i🟢n🟢e🟢 🟢d🟢o🟢c🟢s🟢t🟢r🟢i🟢n🟢g🟢.🟢 + 🟢 + 🟢H🟢e🟢r🟢e🟢'🟢s🟢 🟢a🟢n🟢o🟢t🟢h🟢e🟢r🟢 🟢l🟢i🟢n🟢e🟢.🟢"🟢"🟢"🟢 + `), IsEmptyBlockStartTestCase.python(dedent` - def my_func(): - '''🟢T🟢h🟢i🟢s🟢 🟢i🟢s🟢 🟢a🟢 🟢m🟢u🟢l🟢t🟢i🟢l🟢i🟢n🟢e🟢 🟢d🟢o🟢c🟢s🟢t🟢r🟢i🟢n🟢g🟢.🟢 - 🟢 - 🟢H🟢e🟢r🟢e🟢'🟢s🟢 🟢a🟢n🟢o🟢t🟢h🟢e🟢r🟢 🟢l🟢i🟢n🟢e🟢.🟢'🟢'🟢'🟢 - `), + def my_func(): + '''🟢T🟢h🟢i🟢s🟢 🟢i🟢s🟢 🟢a🟢 🟢m🟢u🟢l🟢t🟢i🟢l🟢i🟢n🟢e🟢 🟢d🟢o🟢c🟢s🟢t🟢r🟢i🟢n🟢g🟢.🟢 + 🟢 + 🟢H🟢e🟢r🟢e🟢'🟢s🟢 🟢a🟢n🟢o🟢t🟢h🟢e🟢r🟢 🟢l🟢i🟢n🟢e🟢.🟢'🟢'🟢'🟢 + `), ]; for (const testCase of testCases) { @@ -413,19 +413,19 @@ suite('parseBlock Tests', function () { const testCases = [ IsEmptyBlockStartTestCase.python( dedent` - def my_func():❌ - "❌"❌"❌T❌h❌i❌s❌ ❌i❌s❌ ❌a❌ ❌d❌o❌c❌s❌t❌r❌i❌n❌g❌.❌"❌"❌"❌ - pass - ` + def my_func():❌ + "❌"❌"❌T❌h❌i❌s❌ ❌i❌s❌ ❌a❌ ❌d❌o❌c❌s❌t❌r❌i❌n❌g❌.❌"❌"❌"❌ + pass + ` ).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE), IsEmptyBlockStartTestCase.python( dedent` - def my_func():❌ - "❌"❌"❌T❌h❌i❌s❌ ❌i❌s❌ ❌a❌ ❌d❌o❌c❌s❌t❌r❌i❌n❌g❌.❌ + def my_func():❌ + "❌"❌"❌T❌h❌i❌s❌ ❌i❌s❌ ❌a❌ ❌d❌o❌c❌s❌t❌r❌i❌n❌g❌.❌ - ❌H❌e❌r❌e❌'❌s❌ ❌a❌n❌o❌t❌h❌e❌r❌ ❌l❌i❌n❌e❌.❌"❌"❌"❌ - pass - ` + ❌H❌e❌r❌e❌'❌s❌ ❌a❌n❌o❌t❌h❌e❌r❌ ❌l❌i❌n❌e❌.❌"❌"❌"❌ + pass + ` ).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE), ]; @@ -435,20 +435,20 @@ suite('parseBlock Tests', function () { }); test('Not EOL', async function () { - const testCase = IsEmptyBlockStartTestCase.python('def my_❌func():').setTrimMode(TrimMode.NO_TRIM); + const testCase = IsEmptyBlockStartTestCase.python('def my_❌func():').setTrimMode(TrimMode.NO_TRIM); await testCase.test(); }); test('if-elif-else', async function () { const testCases = [ IsEmptyBlockStartTestCase.python(dedent` - ❌i❌f🟢 🟢f🟢o🟢o🟢:🟢 - 🟢pass❌ - ❌e❌l❌i❌f🟢 🟢b🟢a🟢r🟢:🟢 - 🟢pass❌ - ❌e❌l❌s❌e🟢: - 🟢pass❌ - `), + ❌i❌f🟢 🟢f🟢o🟢o🟢:🟢 + 🟢pass❌ + ❌e❌l❌i❌f🟢 🟢b🟢a🟢r🟢:🟢 + 🟢pass❌ + ❌e❌l❌s❌e🟢: + 🟢pass❌ + `), ]; for (const testCase of testCases) { @@ -460,27 +460,27 @@ suite('parseBlock Tests', function () { test('block in error state', async function () { const testCases = [ IsEmptyBlockStartTestCase.python(dedent` - def create_tables(conn):🟢 - """Create the tables students, courses and enrolled🟢"""🟢 - conn = sqlite3.connect(results_db_path)❌ - c = conn.cursor()❌ - c.execute('''CREATE TABLE students (❌ - ❌ - `), + def create_tables(conn):🟢 + """Create the tables students, courses and enrolled🟢"""🟢 + conn = sqlite3.connect(results_db_path)❌ + c = conn.cursor()❌ + c.execute('''CREATE TABLE students (❌ + ❌ + `), IsEmptyBlockStartTestCase.python(dedent` - if True:🟢 - conn = sqlite3.connect(results_db_path)❌ - c = conn.cursor()❌ - c.execute('''CREATE TABLE students (❌ - ❌ - `), + if True:🟢 + conn = sqlite3.connect(results_db_path)❌ + c = conn.cursor()❌ + c.execute('''CREATE TABLE students (❌ + ❌ + `), IsEmptyBlockStartTestCase.python(dedent` - try:🟢 - conn = sqlite3.connect(results_db_path)❌ - c = conn.cursor()❌ - c.execute('''CREATE TABLE students (❌ - ❌ - `), + try:🟢 + conn = sqlite3.connect(results_db_path)❌ + c = conn.cursor()❌ + c.execute('''CREATE TABLE students (❌ + ❌ + `), ]; for (const testCase of testCases) { await testCase.test(); @@ -492,15 +492,15 @@ suite('parseBlock Tests', function () { test('arrow_function', async function () { const testCases = [ IsEmptyBlockStartTestCase.javascript(dedent` - ❌(❌a❌)❌ ❌=❌>🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌(❌a❌)❌ ❌=❌>🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), IsEmptyBlockStartTestCase.javascript(dedent` - ❌a❌ ❌=❌>🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌a❌ ❌=❌>🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), // Note: We don't try to give a multline-suggestion immediately after "async". // "async" is a keyword but not a reserved one, so it may be used as an // identifier. Therefore when you have a partially written async function declaration, @@ -508,15 +508,15 @@ suite('parseBlock Tests', function () { // is parsed as a call of a function named "async" with arguments "a"). We'd have to do // very hacky things to support this. IsEmptyBlockStartTestCase.javascript(dedent` - ❌a❌s❌y❌n❌c❌ ❌(❌a❌)❌ ❌=❌>🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌a❌s❌y❌n❌c❌ ❌(❌a❌)❌ ❌=❌>🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), IsEmptyBlockStartTestCase.javascript(dedent` - ❌a❌s❌y❌n❌c❌ ❌a❌ ❌=❌>🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌a❌s❌y❌n❌c❌ ❌a❌ ❌=❌>🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), ]; for (const testCase of testCases) { @@ -527,28 +527,28 @@ suite('parseBlock Tests', function () { test('try_statement, catch_clause, finally_clause', async function () { const testCases: IsEmptyBlockStartTestCase[] = [ IsEmptyBlockStartTestCase.javascript(dedent` - ❌t❌r❌y🟢 🟢{🟢 - 🟢;❌ - ❌}❌ ❌c❌a❌t❌c❌h🟢 🟢(🟢e🟢)🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌t❌r❌y🟢 🟢{🟢 + 🟢;❌ + ❌}❌ ❌c❌a❌t❌c❌h🟢 🟢(🟢e🟢)🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), IsEmptyBlockStartTestCase.javascript(dedent` - ❌t❌r❌y🟢 🟢{🟢 - 🟢;❌ - ❌}❌ ❌f❌i❌n❌a❌l❌l❌y🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌t❌r❌y🟢 🟢{🟢 + 🟢;❌ + ❌}❌ ❌f❌i❌n❌a❌l❌l❌y🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), IsEmptyBlockStartTestCase.javascript(dedent` - ❌t❌r❌y🟢 🟢{🟢 - 🟢;❌ - ❌}❌ ❌c❌a❌t❌c❌h🟢 🟢(🟢e🟢)🟢 🟢{🟢 - 🟢;❌ - ❌}❌ ❌f❌i❌n❌a❌l❌l❌y🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌t❌r❌y🟢 🟢{🟢 + 🟢;❌ + ❌}❌ ❌c❌a❌t❌c❌h🟢 🟢(🟢e🟢)🟢 🟢{🟢 + 🟢;❌ + ❌}❌ ❌f❌i❌n❌a❌l❌l❌y🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), ]; for (const testCase of testCases) { @@ -558,10 +558,10 @@ suite('parseBlock Tests', function () { test('do_statement', async function () { const testCase = IsEmptyBlockStartTestCase.javascript(dedent` - ❌d❌o🟢 🟢{🟢 - 🟢;❌ - ❌}❌ ❌w❌h❌i❌l❌e❌ ❌(❌t❌r❌u❌e❌)❌;❌ - `); + ❌d❌o🟢 🟢{🟢 + 🟢;❌ + ❌}❌ ❌w❌h❌i❌l❌e❌ ❌(❌t❌r❌u❌e❌)❌;❌ + `); await testCase.test(); }); @@ -570,15 +570,15 @@ suite('parseBlock Tests', function () { test('for_in_statement', async function () { const testCases = [ IsEmptyBlockStartTestCase.javascript(dedent` - ❌f❌o❌r🟢 🟢(🟢l🟢e🟢t🟢 🟢v🟢a🟢r🟢 🟢i🟢n🟢 🟢o🟢b🟢j🟢e🟢c🟢t🟢)🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌f❌o❌r🟢 🟢(🟢l🟢e🟢t🟢 🟢v🟢a🟢r🟢 🟢i🟢n🟢 🟢o🟢b🟢j🟢e🟢c🟢t🟢)🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), IsEmptyBlockStartTestCase.javascript(dedent` - ❌f❌o❌r🟢 🟢(🟢l🟢e🟢t🟢 🟢v🟢a🟢r🟢 🟢o🟢f🟢 🟢o🟢b🟢j🟢e🟢c🟢t🟢)🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌f❌o❌r🟢 🟢(🟢l🟢e🟢t🟢 🟢v🟢a🟢r🟢 🟢o🟢f🟢 🟢o🟢b🟢j🟢e🟢c🟢t🟢)🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), ]; for (const testCase of testCases) { @@ -588,10 +588,10 @@ suite('parseBlock Tests', function () { test('for_statement', async function () { const testCase = IsEmptyBlockStartTestCase.javascript(dedent` - ❌f❌o❌r🟢 🟢(🟢l🟢e🟢t🟢 🟢i🟢 🟢=🟢 🟢0🟢;🟢 🟢i🟢 🟢<🟢 🟢5🟢;🟢 🟢i🟢+🟢+🟢)🟢 {🟢 - ;❌ - ❌}❌ - `); + ❌f❌o❌r🟢 🟢(🟢l🟢e🟢t🟢 🟢i🟢 🟢=🟢 🟢0🟢;🟢 🟢i🟢 🟢<🟢 🟢5🟢;🟢 🟢i🟢+🟢+🟢)🟢 {🟢 + ;❌ + ❌}❌ + `); await testCase.test(); }); @@ -599,24 +599,24 @@ suite('parseBlock Tests', function () { test('if_statement', async function () { const testCases = [ IsEmptyBlockStartTestCase.javascript(dedent` - ❌i❌f🟢 🟢(🟢f🟢o🟢o🟢)🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌i❌f🟢 🟢(🟢f🟢o🟢o🟢)🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), IsEmptyBlockStartTestCase.javascript(dedent` - ❌i❌f🟢 🟢(🟢f🟢o🟢o🟢)🟢 🟢{🟢 - 🟢;❌ - ❌}❌ ❌e❌l❌s❌e🟢 🟢i❌f🟢 🟢(🟢b🟢a🟢r🟢)🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌i❌f🟢 🟢(🟢f🟢o🟢o🟢)🟢 🟢{🟢 + 🟢;❌ + ❌}❌ ❌e❌l❌s❌e🟢 🟢i❌f🟢 🟢(🟢b🟢a🟢r🟢)🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), IsEmptyBlockStartTestCase.javascript(dedent` - ❌i❌f🟢 🟢(🟢f🟢o🟢o🟢)🟢 🟢{🟢 - 🟢;❌ - ❌}❌ ❌e❌l❌s❌e🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌i❌f🟢 🟢(🟢f🟢o🟢o🟢)🟢 🟢{🟢 + 🟢;❌ + ❌}❌ ❌e❌l❌s❌e🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), ]; for (const testCase of testCases) { @@ -626,12 +626,12 @@ suite('parseBlock Tests', function () { test('method_definition', async function () { const testCase = IsEmptyBlockStartTestCase.javascript(dedent` - class Foo { - 🟢b❌a❌r❌(❌)🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - } - `); + class Foo { + 🟢b❌a❌r❌(❌)🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + } + `); await testCase.test(); }); @@ -640,32 +640,32 @@ suite('parseBlock Tests', function () { // We don't give multline suggestions for switch_case and switch_default // because they are almost never blocks. const testCase = IsEmptyBlockStartTestCase.javascript(dedent` - switch (foo) { - ❌c❌a❌s❌e❌ ❌b❌a❌r❌:❌ - ❌b❌r❌e❌a❌k❌;❌ - ❌d❌e❌f❌a❌u❌l❌t❌:❌ - ❌b❌r❌e❌a❌k❌;❌ - } - `); + switch (foo) { + ❌c❌a❌s❌e❌ ❌b❌a❌r❌:❌ + ❌b❌r❌e❌a❌k❌;❌ + ❌d❌e❌f❌a❌u❌l❌t❌:❌ + ❌b❌r❌e❌a❌k❌;❌ + } + `); await testCase.test(); }); test('while_statement', async function () { const testCase = IsEmptyBlockStartTestCase.javascript(dedent` - ❌w❌h❌i❌l❌e🟢 🟢(🟢t🟢r🟢u🟢e🟢)🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `); + ❌w❌h❌i❌l❌e🟢 🟢(🟢t🟢r🟢u🟢e🟢)🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `); await testCase.test(); }); test('with_statement', async function () { const testCase = IsEmptyBlockStartTestCase.javascript(dedent` - ❌w❌i❌t❌h🟢 🟢(🟢o🟢)🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `); + ❌w❌i❌t❌h🟢 🟢(🟢o🟢)🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `); await testCase.test(); }); @@ -677,15 +677,15 @@ suite('parseBlock Tests', function () { test('function', async function () { const testCases = [ IsEmptyBlockStartTestCase.javascript(dedent` - ❌l❌e❌t❌ ❌f❌ ❌=❌ ❌f❌u❌n❌c❌t❌i❌o❌n🟢 🟢(🟢)🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌l❌e❌t❌ ❌f❌ ❌=❌ ❌f❌u❌n❌c❌t❌i❌o❌n🟢 🟢(🟢)🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), IsEmptyBlockStartTestCase.javascript(dedent` - ❌l❌e❌t❌ ❌f❌ ❌=❌ ❌a❌s❌y❌n❌c❌ ❌f❌u❌n❌c❌t❌i❌o❌n🟢 🟢(🟢)🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌l❌e❌t❌ ❌f❌ ❌=❌ ❌a❌s❌y❌n❌c❌ ❌f❌u❌n❌c❌t❌i❌o❌n🟢 🟢(🟢)🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), ]; for (const testCase of testCases) { @@ -696,20 +696,20 @@ suite('parseBlock Tests', function () { test('function_declaration', async function () { const testCases = [ IsEmptyBlockStartTestCase.javascript(dedent` - ❌f❌u❌n❌c❌t❌i❌o❌n🟢 🟢f🟢(🟢)🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌f❌u❌n❌c❌t❌i❌o❌n🟢 🟢f🟢(🟢)🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), IsEmptyBlockStartTestCase.javascript(dedent` - ❌a❌s❌y❌n❌c❌ ❌f❌u❌n❌c❌t❌i❌o❌n🟢 🟢f🟢(🟢)🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌a❌s❌y❌n❌c❌ ❌f❌u❌n❌c❌t❌i❌o❌n🟢 🟢f🟢(🟢)🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), IsEmptyBlockStartTestCase.javascript(dedent` - ❌f❌u❌n❌c❌t❌i❌o❌n🟢 🟢f🟢(🟢)🟢 🟢{🟢 - 🟢 🟢 🟢 🟢 🟢 - 🟢}❌ - `), + ❌f❌u❌n❌c❌t❌i❌o❌n🟢 🟢f🟢(🟢)🟢 🟢{🟢 + 🟢 🟢 🟢 🟢 🟢 + 🟢}❌ + `), ]; for (const testCase of testCases) { @@ -720,15 +720,15 @@ suite('parseBlock Tests', function () { test('generator_function', async function () { const testCases = [ IsEmptyBlockStartTestCase.javascript(dedent` - ❌l❌e❌t❌ ❌g❌ ❌=❌ ❌f❌u❌n❌c❌t❌i❌o❌n🟢*🟢 🟢g🟢e🟢n🟢e🟢r🟢a🟢t🟢o🟢r🟢(🟢)🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌l❌e❌t❌ ❌g❌ ❌=❌ ❌f❌u❌n❌c❌t❌i❌o❌n🟢*🟢 🟢g🟢e🟢n🟢e🟢r🟢a🟢t🟢o🟢r🟢(🟢)🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), IsEmptyBlockStartTestCase.javascript(dedent` - ❌l❌e❌t❌ ❌g❌ ❌=❌ ❌a❌s❌y❌n❌c❌ ❌f❌u❌n❌c❌t❌i❌o❌n🟢*🟢 🟢g🟢e🟢n🟢e🟢r🟢a🟢t🟢o🟢r🟢(🟢)🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌l❌e❌t❌ ❌g❌ ❌=❌ ❌a❌s❌y❌n❌c❌ ❌f❌u❌n❌c❌t❌i❌o❌n🟢*🟢 🟢g🟢e🟢n🟢e🟢r🟢a🟢t🟢o🟢r🟢(🟢)🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), ]; for (const testCase of testCases) { await testCase.test(); @@ -738,15 +738,15 @@ suite('parseBlock Tests', function () { test('generator_function_declaration', async function () { const testCases = [ IsEmptyBlockStartTestCase.javascript(dedent` - ❌f❌u❌n❌c❌t❌i❌o❌n🟢*🟢 🟢g🟢e🟢n🟢e🟢r🟢a🟢t🟢o🟢r🟢(🟢)🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌f❌u❌n❌c❌t❌i❌o❌n🟢*🟢 🟢g🟢e🟢n🟢e🟢r🟢a🟢t🟢o🟢r🟢(🟢)🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), IsEmptyBlockStartTestCase.javascript(dedent` - ❌a❌s❌y❌n❌c❌ ❌f❌u❌n❌c❌t❌i❌o❌n🟢*🟢 🟢g🟢e🟢n🟢e🟢r🟢a🟢t🟢o🟢r🟢(🟢)🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌a❌s❌y❌n❌c❌ ❌f❌u❌n❌c❌t❌i❌o❌n🟢*🟢 🟢g🟢e🟢n🟢e🟢r🟢a🟢t🟢o🟢r🟢(🟢)🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), ]; for (const testCase of testCases) { await testCase.test(); @@ -755,19 +755,19 @@ suite('parseBlock Tests', function () { test('class', async function () { const testCase = IsEmptyBlockStartTestCase.javascript(dedent` - ❌l❌e❌t❌ ❌c❌ ❌=❌ ❌c❌l❌a❌s❌s🟢 🟢C🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `); + ❌l❌e❌t❌ ❌c❌ ❌=❌ ❌c❌l❌a❌s❌s🟢 🟢C🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `); await testCase.test(); }); test('class_declaration', async function () { const testCase = IsEmptyBlockStartTestCase.javascript(dedent` - ❌c❌l❌a❌s❌s🟢 🟢C🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `); + ❌c❌l❌a❌s❌s🟢 🟢C🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `); await testCase.test(); }); @@ -780,31 +780,31 @@ suite('parseBlock Tests', function () { const testCases = [ IsEmptyBlockStartTestCase.javascript( dedent` - ❌f❌u❌n❌c❌t❌i❌o❌n🟢 🟢f🟢o🟢o🟢(🟢)🟢 🟢{🟢 - 🟢 - function bar() {} - ` + ❌f❌u❌n❌c❌t❌i❌o❌n🟢 🟢f🟢o🟢o🟢(🟢)🟢 🟢{🟢 + 🟢 + function bar() {} + ` ).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE), IsEmptyBlockStartTestCase.javascript( dedent` - ❌f❌u❌n❌c❌t❌i❌o❌n❌ ❌f❌o❌o❌(❌)❌ ❌{❌ - ❌ - function bar() {} - ` + ❌f❌u❌n❌c❌t❌i❌o❌n❌ ❌f❌o❌o❌(❌)❌ ❌{❌ + ❌ + function bar() {} + ` ).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE), IsEmptyBlockStartTestCase.javascript( dedent` - ❌f❌u❌n❌c❌t❌i❌o❌n🟢 🟢f🟢o🟢o🟢(🟢)🟢 🟢{🟢 - 🟢 - let a = 10; - ` + ❌f❌u❌n❌c❌t❌i❌o❌n🟢 🟢f🟢o🟢o🟢(🟢)🟢 🟢{🟢 + 🟢 + let a = 10; + ` ).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE), IsEmptyBlockStartTestCase.javascript( dedent` - ❌f❌u❌n❌c❌t❌i❌o❌n❌ ❌f❌o❌o❌(❌)❌ ❌{❌ - ❌ - let a = 10; - ` + ❌f❌u❌n❌c❌t❌i❌o❌n❌ ❌f❌o❌o❌(❌)❌ ❌{❌ + ❌ + let a = 10; + ` ).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE), ]; @@ -817,29 +817,29 @@ suite('parseBlock Tests', function () { const testCases = [ IsEmptyBlockStartTestCase.javascript( dedent` - () => doIt(❌ - ❌f❌o❌o❌.❌f❌o❌o❌,❌ - ❌b❌a❌r❌.❌b❌a❌z❌,❌ - ❌b❌a❌z❌.❌b❌a❌z❌ - ); - ` + () => doIt(❌ + ❌f❌o❌o❌.❌f❌o❌o❌,❌ + ❌b❌a❌r❌.❌b❌a❌z❌,❌ + ❌b❌a❌z❌.❌b❌a❌z❌ + ); + ` ).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE), IsEmptyBlockStartTestCase.javascript( dedent` - () => doIt(❌ - ❌'❌a❌'❌,❌ - ❌'❌a❌'❌,❌ - ❌'❌a❌'❌ - ); - ` + () => doIt(❌ + ❌'❌a❌'❌,❌ + ❌'❌a❌'❌,❌ + ❌'❌a❌'❌ + ); + ` ).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE), IsEmptyBlockStartTestCase.javascript(dedent` - () => doIt(❌ - ❌'❌a❌'❌,❌ - ❌'❌a❌'❌,❌ - ❌'❌a❌'❌ - ); - `), + () => doIt(❌ + ❌'❌a❌'❌,❌ + ❌'❌a❌'❌,❌ + ❌'❌a❌'❌ + ); + `), ]; for (const testCase of testCases) { @@ -853,10 +853,10 @@ suite('parseBlock Tests', function () { // suggestion until after "global," when it transitions from an identifer to a keyword. test('ambient_declaration', async function () { const testCase = IsEmptyBlockStartTestCase.typescript(dedent` - ❌d❌e❌c❌l❌a❌r❌e❌ ❌g❌l❌o❌b❌a❌l🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `); + ❌d❌e❌c❌l❌a❌r❌e❌ ❌g❌l❌o❌b❌a❌l🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `); await testCase.test(); }); @@ -865,10 +865,10 @@ suite('parseBlock Tests', function () { // suggestion until the open quote, when it transitions from an identifer to a keyword. test('internal_module', async function () { const testCase = IsEmptyBlockStartTestCase.typescript(dedent` - ❌n❌a❌m❌e❌s❌p❌a❌c❌e❌ ❌"🟢f🟢o🟢o🟢"🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `); + ❌n❌a❌m❌e❌s❌p❌a❌c❌e❌ ❌"🟢f🟢o🟢o🟢"🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `); await testCase.test(); }); @@ -877,10 +877,10 @@ suite('parseBlock Tests', function () { // suggestion until the open quote, when it transitions from an identifer to a keyword. test('module', async function () { const testCase = IsEmptyBlockStartTestCase.typescript(dedent` - ❌m❌o❌d❌u❌l❌e❌ ❌"🟢f🟢o🟢o🟢"🟢 🟢{🟢 - ;❌ - ❌}❌ - `); + ❌m❌o❌d❌u❌l❌e❌ ❌"🟢f🟢o🟢o🟢"🟢 🟢{🟢 + ;❌ + ❌}❌ + `); await testCase.test(); }); @@ -888,35 +888,35 @@ suite('parseBlock Tests', function () { test('arrow_function', async function () { const testCases = [ IsEmptyBlockStartTestCase.typescript(dedent` - ❌(❌a❌)❌ ❌=❌>🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌(❌a❌)❌ ❌=❌>🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), IsEmptyBlockStartTestCase.typescript(dedent` - ❌(❌a❌:❌ ❌s❌t❌r❌i❌n❌g❌)❌:❌ ❌v❌o❌i❌d❌ ❌=❌>🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌(❌a❌:❌ ❌s❌t❌r❌i❌n❌g❌)❌:❌ ❌v❌o❌i❌d❌ ❌=❌>🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), IsEmptyBlockStartTestCase.typescript(dedent` - ❌a❌ ❌=❌>🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌a❌ ❌=❌>🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), IsEmptyBlockStartTestCase.typescript(dedent` - ❌a❌s❌y❌n❌c❌ ❌(❌a❌)❌ ❌=❌>🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌a❌s❌y❌n❌c❌ ❌(❌a❌)❌ ❌=❌>🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), IsEmptyBlockStartTestCase.typescript(dedent` - ❌a❌s❌y❌n❌c❌ ❌(❌a❌:❌ ❌s❌t❌r❌i❌n❌g❌)❌:❌ ❌v❌o❌i❌d❌ ❌=❌>🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌a❌s❌y❌n❌c❌ ❌(❌a❌:❌ ❌s❌t❌r❌i❌n❌g❌)❌:❌ ❌v❌o❌i❌d❌ ❌=❌>🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), IsEmptyBlockStartTestCase.typescript(dedent` - ❌a❌s❌y❌n❌c❌ ❌a❌ ❌=❌>🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌a❌s❌y❌n❌c❌ ❌a❌ ❌=❌>🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), ]; for (const testCase of testCases) { @@ -930,28 +930,28 @@ suite('parseBlock Tests', function () { test('try_statement, catch_clause, finally_clause', async function () { const testCases: IsEmptyBlockStartTestCase[] = [ IsEmptyBlockStartTestCase.typescript(dedent` - ❌t❌r❌y🟢 🟢{🟢 - 🟢;❌ - ❌}❌ ❌c❌a❌t❌c❌h🟢 🟢(🟢e🟢)🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌t❌r❌y🟢 🟢{🟢 + 🟢;❌ + ❌}❌ ❌c❌a❌t❌c❌h🟢 🟢(🟢e🟢)🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), IsEmptyBlockStartTestCase.typescript(dedent` - ❌t❌r❌y🟢 🟢{🟢 - 🟢;❌ - ❌}❌ ❌f❌i❌n❌a❌l❌l❌y🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌t❌r❌y🟢 🟢{🟢 + 🟢;❌ + ❌}❌ ❌f❌i❌n❌a❌l❌l❌y🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), IsEmptyBlockStartTestCase.typescript(dedent` - ❌t❌r❌y🟢 🟢{🟢 - 🟢;❌ - ❌}❌ ❌c❌a❌t❌c❌h🟢 🟢(🟢e🟢)🟢 🟢{🟢 - 🟢;❌ - ❌}❌ ❌f❌i❌n❌a❌l❌l❌y🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌t❌r❌y🟢 🟢{🟢 + 🟢;❌ + ❌}❌ ❌c❌a❌t❌c❌h🟢 🟢(🟢e🟢)🟢 🟢{🟢 + 🟢;❌ + ❌}❌ ❌f❌i❌n❌a❌l❌l❌y🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), ]; for (const testCase of testCases) { @@ -961,10 +961,10 @@ suite('parseBlock Tests', function () { test('do_statement', async function () { const testCase = IsEmptyBlockStartTestCase.typescript(dedent` - ❌d❌o🟢 🟢{🟢 - 🟢;❌ - ❌}❌ ❌w❌h❌i❌l❌e❌ ❌(❌t❌r❌u❌e❌)❌;❌ - `); + ❌d❌o🟢 🟢{🟢 + 🟢;❌ + ❌}❌ ❌w❌h❌i❌l❌e❌ ❌(❌t❌r❌u❌e❌)❌;❌ + `); await testCase.test(); }); @@ -973,15 +973,15 @@ suite('parseBlock Tests', function () { test('for_in_statement', async function () { const testCases = [ IsEmptyBlockStartTestCase.typescript(dedent` - ❌f❌o❌r🟢 🟢(🟢l🟢e🟢t🟢 🟢v🟢a🟢r🟢 🟢i🟢n🟢 🟢o🟢b🟢j🟢e🟢c🟢t🟢)🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌f❌o❌r🟢 🟢(🟢l🟢e🟢t🟢 🟢v🟢a🟢r🟢 🟢i🟢n🟢 🟢o🟢b🟢j🟢e🟢c🟢t🟢)🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), IsEmptyBlockStartTestCase.typescript(dedent` - ❌f❌o❌r🟢 🟢(🟢l🟢e🟢t🟢 🟢v🟢a🟢r🟢 🟢o🟢f🟢 🟢o🟢b🟢j🟢e🟢c🟢t🟢)🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌f❌o❌r🟢 🟢(🟢l🟢e🟢t🟢 🟢v🟢a🟢r🟢 🟢o🟢f🟢 🟢o🟢b🟢j🟢e🟢c🟢t🟢)🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), ]; for (const testCase of testCases) { @@ -992,15 +992,15 @@ suite('parseBlock Tests', function () { test('for_statement', async function () { const testCases = [ IsEmptyBlockStartTestCase.typescript(dedent` - ❌f❌o❌r🟢 🟢(🟢l🟢e🟢t🟢 🟢i🟢 🟢=🟢 🟢0🟢;🟢 🟢i🟢 🟢<🟢 🟢5🟢;🟢 🟢i🟢+🟢+🟢)🟢 {🟢 - ;❌ - ❌}❌ - `), + ❌f❌o❌r🟢 🟢(🟢l🟢e🟢t🟢 🟢i🟢 🟢=🟢 🟢0🟢;🟢 🟢i🟢 🟢<🟢 🟢5🟢;🟢 🟢i🟢+🟢+🟢)🟢 {🟢 + ;❌ + ❌}❌ + `), IsEmptyBlockStartTestCase.typescript(dedent` - ❌f❌o❌r🟢 🟢(🟢l🟢e🟢t🟢 🟢i🟢:🟢 🟢i🟢n🟢t🟢 🟢=🟢 🟢0🟢;🟢 🟢i🟢 🟢<🟢 🟢5🟢;🟢 🟢i🟢+🟢+🟢)🟢 {🟢 - ;❌ - ❌}❌ - `), + ❌f❌o❌r🟢 🟢(🟢l🟢e🟢t🟢 🟢i🟢:🟢 🟢i🟢n🟢t🟢 🟢=🟢 🟢0🟢;🟢 🟢i🟢 🟢<🟢 🟢5🟢;🟢 🟢i🟢+🟢+🟢)🟢 {🟢 + ;❌ + ❌}❌ + `), ]; for (const testCase of testCases) { @@ -1011,24 +1011,24 @@ suite('parseBlock Tests', function () { test('if_statement', async function () { const testCases = [ IsEmptyBlockStartTestCase.typescript(dedent` - ❌i❌f🟢 🟢(🟢f🟢o🟢o🟢)🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌i❌f🟢 🟢(🟢f🟢o🟢o🟢)🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), IsEmptyBlockStartTestCase.typescript(dedent` - ❌i❌f🟢 🟢(🟢f🟢o🟢o🟢)🟢 🟢{🟢 - 🟢;❌ - ❌}❌ ❌e❌l❌s❌e🟢 🟢i❌f🟢 🟢(🟢b🟢a🟢r🟢)🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌i❌f🟢 🟢(🟢f🟢o🟢o🟢)🟢 🟢{🟢 + 🟢;❌ + ❌}❌ ❌e❌l❌s❌e🟢 🟢i❌f🟢 🟢(🟢b🟢a🟢r🟢)🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), IsEmptyBlockStartTestCase.typescript(dedent` - ❌i❌f🟢 🟢(🟢f🟢o🟢o🟢)🟢 🟢{🟢 - 🟢;❌ - ❌}❌ ❌e❌l❌s❌e🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌i❌f🟢 🟢(🟢f🟢o🟢o🟢)🟢 🟢{🟢 + 🟢;❌ + ❌}❌ ❌e❌l❌s❌e🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), ]; for (const testCase of testCases) { @@ -1039,27 +1039,27 @@ suite('parseBlock Tests', function () { test('method_definition', async function () { const testCases = [ IsEmptyBlockStartTestCase.typescript(dedent` - class Foo { - 🟢b❌a❌r❌(❌)🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - } - `), + class Foo { + 🟢b❌a❌r❌(❌)🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + } + `), IsEmptyBlockStartTestCase.typescript(dedent` - class Foo { - 🟢b❌a❌r❌(❌i❌:❌ ❌i❌n❌t❌)🟢:❌ ❌v🟢o🟢i🟢d🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - } - `), + class Foo { + 🟢b❌a❌r❌(❌i❌:❌ ❌i❌n❌t❌)🟢:❌ ❌v🟢o🟢i🟢d🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + } + `), // TODO(eaftan): fix sibling function issue and enable this test // IsEmptyBlockStartTestCase.typescript(dedent` // class Foo { - // f❌o❌o❌(❌)🟢 🟢{🟢 - // 🟢}❌ + // f❌o❌o❌(❌)🟢 🟢{🟢 + // 🟢}❌ - // ❌b❌a❌r❌(❌)🟢 🟢{🟢 - // 🟢}❌ + // ❌b❌a❌r❌(❌)🟢 🟢{🟢 + // 🟢}❌ // } // `).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE), ]; @@ -1071,15 +1071,15 @@ suite('parseBlock Tests', function () { test('method_signature', async function () { const testCases = [ IsEmptyBlockStartTestCase.typescript(dedent` - class Foo { - 🟢b❌a❌r❌(❌)🟢;❌ - } - `), + class Foo { + 🟢b❌a❌r❌(❌)🟢;❌ + } + `), IsEmptyBlockStartTestCase.typescript(dedent` - class Foo { - 🟢b❌a❌r❌(❌i❌:❌ ❌i❌n❌t❌)🟢:❌ ❌v🟢o🟢i🟢d🟢;❌ - } - `), + class Foo { + 🟢b❌a❌r❌(❌i❌:❌ ❌i❌n❌t❌)🟢:❌ ❌v🟢o🟢i🟢d🟢;❌ + } + `), ]; for (const testCase of testCases) { @@ -1091,23 +1091,23 @@ suite('parseBlock Tests', function () { // We don't give multline suggestions for switch_case and switch_default // because they are almost never blocks. const testCase = IsEmptyBlockStartTestCase.typescript(dedent` - switch (foo) { - ❌c❌a❌s❌e❌ ❌b❌a❌r❌:❌ - ❌b❌r❌e❌a❌k❌;❌ - ❌d❌e❌f❌a❌u❌l❌t❌:❌ - ❌b❌r❌e❌a❌k❌;❌ - } - `); + switch (foo) { + ❌c❌a❌s❌e❌ ❌b❌a❌r❌:❌ + ❌b❌r❌e❌a❌k❌;❌ + ❌d❌e❌f❌a❌u❌l❌t❌:❌ + ❌b❌r❌e❌a❌k❌;❌ + } + `); await testCase.test(); }); test('while_statement', async function () { const testCase = IsEmptyBlockStartTestCase.typescript(dedent` - ❌w❌h❌i❌l❌e🟢 🟢(🟢t🟢r🟢u🟢e🟢)🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `); + ❌w❌h❌i❌l❌e🟢 🟢(🟢t🟢r🟢u🟢e🟢)🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `); await testCase.test(); }); @@ -1119,25 +1119,25 @@ suite('parseBlock Tests', function () { test('function', async function () { const testCases = [ IsEmptyBlockStartTestCase.typescript(dedent` - ❌l❌e❌t❌ ❌f❌ ❌=❌ ❌f❌u❌n❌c❌t❌i❌o❌n🟢 🟢(🟢)🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌l❌e❌t❌ ❌f❌ ❌=❌ ❌f❌u❌n❌c❌t❌i❌o❌n🟢 🟢(🟢)🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), IsEmptyBlockStartTestCase.typescript(dedent` - ❌l❌e❌t❌ ❌f❌ ❌=❌ ❌f❌u❌n❌c❌t❌i❌o❌n🟢 🟢(🟢i🟢:🟢 🟢i🟢n🟢t🟢)🟢:🟢 🟢v🟢o🟢i🟢d🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌l❌e❌t❌ ❌f❌ ❌=❌ ❌f❌u❌n❌c❌t❌i❌o❌n🟢 🟢(🟢i🟢:🟢 🟢i🟢n🟢t🟢)🟢:🟢 🟢v🟢o🟢i🟢d🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), IsEmptyBlockStartTestCase.typescript(dedent` - ❌l❌e❌t❌ ❌f❌ ❌=❌ ❌a❌s❌y❌n❌c❌ ❌f❌u❌n❌c❌t❌i❌o❌n🟢 🟢(🟢)🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌l❌e❌t❌ ❌f❌ ❌=❌ ❌a❌s❌y❌n❌c❌ ❌f❌u❌n❌c❌t❌i❌o❌n🟢 🟢(🟢)🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), IsEmptyBlockStartTestCase.typescript(dedent` - ❌l❌e❌t❌ ❌f❌ ❌=❌ ❌a❌s❌y❌n❌c❌ ❌f❌u❌n❌c❌t❌i❌o❌n🟢 🟢(i🟢:🟢 🟢i🟢n🟢t🟢)🟢:🟢 🟢v🟢o🟢i🟢d🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌l❌e❌t❌ ❌f❌ ❌=❌ ❌a❌s❌y❌n❌c❌ ❌f❌u❌n❌c❌t❌i❌o❌n🟢 🟢(i🟢:🟢 🟢i🟢n🟢t🟢)🟢:🟢 🟢v🟢o🟢i🟢d🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), ]; for (const testCase of testCases) { @@ -1148,64 +1148,64 @@ suite('parseBlock Tests', function () { test('function_declaration', async function () { const testCases = [ IsEmptyBlockStartTestCase.typescript(dedent` - ❌f❌u❌n❌c❌t❌i❌o❌n🟢 🟢f🟢(🟢)🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌f❌u❌n❌c❌t❌i❌o❌n🟢 🟢f🟢(🟢)🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), IsEmptyBlockStartTestCase.typescript(dedent` - ❌f❌u❌n❌c❌t❌i❌o❌n🟢 🟢f🟢(i🟢:🟢 🟢i🟢n🟢t🟢)🟢:🟢 🟢v🟢o🟢i🟢d🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌f❌u❌n❌c❌t❌i❌o❌n🟢 🟢f🟢(i🟢:🟢 🟢i🟢n🟢t🟢)🟢:🟢 🟢v🟢o🟢i🟢d🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), IsEmptyBlockStartTestCase.typescript(dedent` - ❌a❌s❌y❌n❌c❌ ❌f❌u❌n❌c❌t❌i❌o❌n🟢 🟢f🟢(🟢)🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌a❌s❌y❌n❌c❌ ❌f❌u❌n❌c❌t❌i❌o❌n🟢 🟢f🟢(🟢)🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), IsEmptyBlockStartTestCase.typescript(dedent` - ❌a❌s❌y❌n❌c❌ ❌f❌u❌n❌c❌t❌i❌o❌n🟢 🟢f🟢(🟢i🟢:🟢 🟢i🟢n🟢t🟢)🟢:🟢 🟢v🟢o🟢i🟢d🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌a❌s❌y❌n❌c❌ ❌f❌u❌n❌c❌t❌i❌o❌n🟢 🟢f🟢(🟢i🟢:🟢 🟢i🟢n🟢t🟢)🟢:🟢 🟢v🟢o🟢i🟢d🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), IsEmptyBlockStartTestCase.typescript(dedent` - ❌f❌u❌n❌c❌t❌i❌o❌n🟢 🟢f🟢(🟢)🟢 🟢{🟢 - 🟢 🟢 🟢 🟢 🟢 - 🟢}❌ - `), + ❌f❌u❌n❌c❌t❌i❌o❌n🟢 🟢f🟢(🟢)🟢 🟢{🟢 + 🟢 🟢 🟢 🟢 🟢 + 🟢}❌ + `), IsEmptyBlockStartTestCase.typescript(dedent` - ❌f❌u❌n❌c❌t❌i❌o❌n🟢 🟢f🟢(🟢i🟢:🟢 🟢i🟢n🟢t🟢)🟢:🟢 🟢v🟢o🟢i🟢d🟢 🟢{🟢 - 🟢 🟢 🟢 🟢 🟢 - 🟢}❌ - `), + ❌f❌u❌n❌c❌t❌i❌o❌n🟢 🟢f🟢(🟢i🟢:🟢 🟢i🟢n🟢t🟢)🟢:🟢 🟢v🟢o🟢i🟢d🟢 🟢{🟢 + 🟢 🟢 🟢 🟢 🟢 + 🟢}❌ + `), IsEmptyBlockStartTestCase.typescript( dedent` - ❌f❌u❌n❌c❌t❌i❌o❌n🟢 🟢f🟢(❌x❌ ❌:❌ ❌n❌u❌m❌b❌e❌r❌,❌ - 🟢y🟢 🟢:🟢 🟢n🟢u🟢m🟢b🟢e🟢r🟢)🟢 🟢:🟢 🟢n🟢u🟢m🟢b🟢e🟢r🟢;❌ - ` + ❌f❌u❌n❌c❌t❌i❌o❌n🟢 🟢f🟢(❌x❌ ❌:❌ ❌n❌u❌m❌b❌e❌r❌,❌ + 🟢y🟢 🟢:🟢 🟢n🟢u🟢m🟢b🟢e🟢r🟢)🟢 🟢:🟢 🟢n🟢u🟢m🟢b🟢e🟢r🟢;❌ + ` ).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE), IsEmptyBlockStartTestCase.typescript( dedent` - ❌f❌u❌n❌c❌t❌i❌o❌n🟢 🟢f🟢(🟢 - 🟢 - let x = 0; - ` + ❌f❌u❌n❌c❌t❌i❌o❌n🟢 🟢f🟢(🟢 + 🟢 + let x = 0; + ` ).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE), IsEmptyBlockStartTestCase.typescript( dedent` - function f(❌ - /** first parameter */ - x: number, - /** second parameter */ - y: number); - ` + function f(❌ + /** first parameter */ + x: number, + /** second parameter */ + y: number); + ` ).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE), IsEmptyBlockStartTestCase.typescript( dedent` - function getPosition() : {❌ - start: number,❌ - end: number❌ - }; - ` + function getPosition() : {❌ + start: number,❌ + end: number❌ + }; + ` ).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE), ]; @@ -1217,25 +1217,25 @@ suite('parseBlock Tests', function () { test('generator_function', async function () { const testCases = [ IsEmptyBlockStartTestCase.typescript(dedent` - ❌l❌e❌t❌ ❌g❌ ❌=❌ ❌f❌u❌n❌c❌t❌i❌o❌n🟢*🟢 🟢g🟢e🟢n🟢e🟢r🟢a🟢t🟢o🟢r🟢(🟢)🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌l❌e❌t❌ ❌g❌ ❌=❌ ❌f❌u❌n❌c❌t❌i❌o❌n🟢*🟢 🟢g🟢e🟢n🟢e🟢r🟢a🟢t🟢o🟢r🟢(🟢)🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), IsEmptyBlockStartTestCase.typescript(dedent` - ❌l❌e❌t❌ ❌g❌ ❌=❌ ❌f❌u❌n❌c❌t❌i❌o❌n🟢*🟢 🟢g🟢e🟢n🟢e🟢r🟢a🟢t🟢o🟢r🟢(🟢i🟢:🟢 🟢i🟢n🟢t🟢)🟢:🟢 🟢v🟢o🟢i🟢d🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌l❌e❌t❌ ❌g❌ ❌=❌ ❌f❌u❌n❌c❌t❌i❌o❌n🟢*🟢 🟢g🟢e🟢n🟢e🟢r🟢a🟢t🟢o🟢r🟢(🟢i🟢:🟢 🟢i🟢n🟢t🟢)🟢:🟢 🟢v🟢o🟢i🟢d🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), IsEmptyBlockStartTestCase.typescript(dedent` - ❌l❌e❌t❌ ❌g❌ ❌=❌ ❌a❌s❌y❌n❌c❌ ❌f❌u❌n❌c❌t❌i❌o❌n🟢*🟢 🟢g🟢e🟢n🟢e🟢r🟢a🟢t🟢o🟢r🟢(🟢)🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌l❌e❌t❌ ❌g❌ ❌=❌ ❌a❌s❌y❌n❌c❌ ❌f❌u❌n❌c❌t❌i❌o❌n🟢*🟢 🟢g🟢e🟢n🟢e🟢r🟢a🟢t🟢o🟢r🟢(🟢)🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), IsEmptyBlockStartTestCase.typescript(dedent` - ❌l❌e❌t❌ ❌g❌ ❌=❌ ❌a❌s❌y❌n❌c❌ ❌f❌u❌n❌c❌t❌i❌o❌n🟢*🟢 🟢g🟢e🟢n🟢e🟢r🟢a🟢t🟢o🟢r🟢(🟢i🟢:🟢 🟢i🟢n🟢t🟢)🟢:🟢 🟢v🟢o🟢i🟢d🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌l❌e❌t❌ ❌g❌ ❌=❌ ❌a❌s❌y❌n❌c❌ ❌f❌u❌n❌c❌t❌i❌o❌n🟢*🟢 🟢g🟢e🟢n🟢e🟢r🟢a🟢t🟢o🟢r🟢(🟢i🟢:🟢 🟢i🟢n🟢t🟢)🟢:🟢 🟢v🟢o🟢i🟢d🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), ]; for (const testCase of testCases) { await testCase.test(); @@ -1245,25 +1245,25 @@ suite('parseBlock Tests', function () { test('generator_function_declaration', async function () { const testCases = [ IsEmptyBlockStartTestCase.typescript(dedent` - ❌f❌u❌n❌c❌t❌i❌o❌n🟢*🟢 🟢g🟢e🟢n🟢e🟢r🟢a🟢t🟢o🟢r🟢(🟢)🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌f❌u❌n❌c❌t❌i❌o❌n🟢*🟢 🟢g🟢e🟢n🟢e🟢r🟢a🟢t🟢o🟢r🟢(🟢)🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), IsEmptyBlockStartTestCase.typescript(dedent` - ❌f❌u❌n❌c❌t❌i❌o❌n🟢*🟢 🟢g🟢e🟢n🟢e🟢r🟢a🟢t🟢o🟢r🟢(🟢i🟢:🟢 🟢i🟢n🟢t🟢)🟢:🟢 🟢v🟢o🟢i🟢d🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌f❌u❌n❌c❌t❌i❌o❌n🟢*🟢 🟢g🟢e🟢n🟢e🟢r🟢a🟢t🟢o🟢r🟢(🟢i🟢:🟢 🟢i🟢n🟢t🟢)🟢:🟢 🟢v🟢o🟢i🟢d🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), IsEmptyBlockStartTestCase.typescript(dedent` - ❌a❌s❌y❌n❌c❌ ❌f❌u❌n❌c❌t❌i❌o❌n🟢*🟢 🟢g🟢e🟢n🟢e🟢r🟢a🟢t🟢o🟢r🟢(🟢)🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌a❌s❌y❌n❌c❌ ❌f❌u❌n❌c❌t❌i❌o❌n🟢*🟢 🟢g🟢e🟢n🟢e🟢r🟢a🟢t🟢o🟢r🟢(🟢)🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), IsEmptyBlockStartTestCase.typescript(dedent` - ❌a❌s❌y❌n❌c❌ ❌f❌u❌n❌c❌t❌i❌o❌n🟢*🟢 🟢g🟢e🟢n🟢e🟢r🟢a🟢t🟢o🟢r🟢(🟢i🟢:🟢 🟢i🟢n🟢t🟢)🟢:🟢 🟢v🟢o🟢i🟢d🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `), + ❌a❌s❌y❌n❌c❌ ❌f❌u❌n❌c❌t❌i❌o❌n🟢*🟢 🟢g🟢e🟢n🟢e🟢r🟢a🟢t🟢o🟢r🟢(🟢i🟢:🟢 🟢i🟢n🟢t🟢)🟢:🟢 🟢v🟢o🟢i🟢d🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `), ]; for (const testCase of testCases) { await testCase.test(); @@ -1272,28 +1272,28 @@ suite('parseBlock Tests', function () { test('class', async function () { const testCase = IsEmptyBlockStartTestCase.typescript(dedent` - ❌l❌e❌t❌ ❌c❌ ❌=❌ ❌c❌l❌a❌s❌s🟢 🟢C🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `); + ❌l❌e❌t❌ ❌c❌ ❌=❌ ❌c❌l❌a❌s❌s🟢 🟢C🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `); await testCase.test(); }); test('class_declaration', async function () { const testCase = IsEmptyBlockStartTestCase.typescript(dedent` - ❌c❌l❌a❌s❌s🟢 🟢C🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `); + ❌c❌l❌a❌s❌s🟢 🟢C🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `); await testCase.test(); }); test('abstract_class_declaration', async function () { const testCase = IsEmptyBlockStartTestCase.typescript(dedent` - ❌a❌b❌s❌t❌r❌a❌c❌t❌ ❌c❌l❌a❌s❌s🟢 🟢C🟢 🟢{🟢 - 🟢;❌ - ❌}❌ - `); + ❌a❌b❌s❌t❌r❌a❌c❌t❌ ❌c❌l❌a❌s❌s🟢 🟢C🟢 🟢{🟢 + 🟢;❌ + ❌}❌ + `); await testCase.test(); }); @@ -1306,31 +1306,31 @@ suite('parseBlock Tests', function () { const testCases = [ IsEmptyBlockStartTestCase.typescript( dedent` - ❌f❌u❌n❌c❌t❌i❌o❌n🟢 🟢f🟢o🟢o🟢(🟢)🟢 🟢{🟢 - 🟢 - function bar() {} - ` + ❌f❌u❌n❌c❌t❌i❌o❌n🟢 🟢f🟢o🟢o🟢(🟢)🟢 🟢{🟢 + 🟢 + function bar() {} + ` ).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE), IsEmptyBlockStartTestCase.typescript( dedent` - ❌f❌u❌n❌c❌t❌i❌o❌n❌ ❌f❌o❌o❌(❌)❌ ❌{❌ - ❌ - function bar() {} - ` + ❌f❌u❌n❌c❌t❌i❌o❌n❌ ❌f❌o❌o❌(❌)❌ ❌{❌ + ❌ + function bar() {} + ` ).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE), IsEmptyBlockStartTestCase.typescript( dedent` - ❌f❌u❌n❌c❌t❌i❌o❌n🟢 🟢f🟢o🟢o🟢(🟢)🟢 🟢{🟢 - 🟢 - let a = 10; - ` + ❌f❌u❌n❌c❌t❌i❌o❌n🟢 🟢f🟢o🟢o🟢(🟢)🟢 🟢{🟢 + 🟢 + let a = 10; + ` ).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE), IsEmptyBlockStartTestCase.typescript( dedent` - ❌f❌u❌n❌c❌t❌i❌o❌n❌ ❌f❌o❌o❌(❌)❌ ❌{❌ - ❌ - let a = 10; - ` + ❌f❌u❌n❌c❌t❌i❌o❌n❌ ❌f❌o❌o❌(❌)❌ ❌{❌ + ❌ + let a = 10; + ` ).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE), ]; @@ -1343,29 +1343,29 @@ suite('parseBlock Tests', function () { const testCases = [ IsEmptyBlockStartTestCase.typescript( dedent` - () => doIt(❌ - ❌f❌o❌o❌.❌f❌o❌o❌,❌ - ❌b❌a❌r❌.❌b❌a❌z❌,❌ - ❌b❌a❌z❌.❌b❌a❌z❌ - ); - ` + () => doIt(❌ + ❌f❌o❌o❌.❌f❌o❌o❌,❌ + ❌b❌a❌r❌.❌b❌a❌z❌,❌ + ❌b❌a❌z❌.❌b❌a❌z❌ + ); + ` ).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE), IsEmptyBlockStartTestCase.typescript( dedent` - () => doIt(❌ - ❌'❌a❌'❌,❌ - ❌'❌a❌'❌,❌ - ❌'❌a❌'❌ - ); - ` + () => doIt(❌ + ❌'❌a❌'❌,❌ + ❌'❌a❌'❌,❌ + ❌'❌a❌'❌ + ); + ` ).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE), IsEmptyBlockStartTestCase.typescript(dedent` - () => doIt(❌ - ❌'❌a❌'❌,❌ - ❌'❌a❌'❌,❌ - ❌'❌a❌'❌ - ); - `), + () => doIt(❌ + ❌'❌a❌'❌,❌ + ❌'❌a❌'❌,❌ + ❌'❌a❌'❌ + ); + `), ]; for (const testCase of testCases) { @@ -1375,10 +1375,10 @@ suite('parseBlock Tests', function () { test('function type', async function () { const testCase = IsEmptyBlockStartTestCase.typescript(dedent` - ❌f❌u❌n❌c❌t❌i❌o❌n🟢 🟢f🟢(🟢c🟢b🟢:🟢 🟢(🟢)🟢 🟢=🟢>🟢 🟢v🟢o🟢i🟢d🟢)🟢 🟢{🟢 - 🟢c❌b❌(❌)❌;❌ - ❌}❌ - `); + ❌f❌u❌n❌c❌t❌i❌o❌n🟢 🟢f🟢(🟢c🟢b🟢:🟢 🟢(🟢)🟢 🟢=🟢>🟢 🟢v🟢o🟢i🟢d🟢)🟢 🟢{🟢 + 🟢c❌b❌(❌)❌;❌ + ❌}❌ + `); await testCase.test(); }); @@ -1388,25 +1388,25 @@ suite('parseBlock Tests', function () { test('simple examples', async function () { const testCases = [ IsEmptyBlockStartTestCase.ruby(dedent` - def 🟢greet🟢 - 🟢puts "Hello"❌ - ❌puts "Bye"❌ - end - `), + def 🟢greet🟢 + 🟢puts "Hello"❌ + ❌puts "Bye"❌ + end + `), IsEmptyBlockStartTestCase.ruby( dedent` - def 🟢greet❌ - 🟢puts "Hello"❌ - end - ` + def 🟢greet❌ + 🟢puts "Hello"❌ + end + ` ).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE), IsEmptyBlockStartTestCase.ruby( dedent` - def 🟢greet❌ - ❌puts "Hello"❌ - ❌puts "Bye"❌ - end - ` + def 🟢greet❌ + ❌puts "Hello"❌ + ❌puts "Bye"❌ + end + ` ).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE), ]; for (const testCase of testCases) { @@ -1419,25 +1419,25 @@ suite('parseBlock Tests', function () { test('simple examples', async function () { const testCases = [ IsEmptyBlockStartTestCase.go(dedent` - func 🟢greet🟢()🟢 {🟢 - 🟢fmt.Println("Hello")❌ - ❌fmt.Println("Bye")❌ - } - `), + func 🟢greet🟢()🟢 {🟢 + 🟢fmt.Println("Hello")❌ + ❌fmt.Println("Bye")❌ + } + `), IsEmptyBlockStartTestCase.go( dedent` - func 🟢greet🟢()🟢 {❌ - 🟢fmt.Println("Hello")❌ - } - ` + func 🟢greet🟢()🟢 {❌ + 🟢fmt.Println("Hello")❌ + } + ` ).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE), IsEmptyBlockStartTestCase.go( dedent` - func 🟢greet🟢()🟢 {❌ - ❌fmt.Println("Hello")❌ - ❌fmt.Println("Bye")❌ - } - ` + func 🟢greet🟢()🟢 {❌ + ❌fmt.Println("Hello")❌ + ❌fmt.Println("Bye")❌ + } + ` ).setTrimMode(TrimMode.TRIM_TO_END_OF_LINE), ]; for (const testCase of testCases) { @@ -1491,165 +1491,165 @@ suite('parseBlock Tests', function () { suite('Python getBlockStart tests', function () { test('class_definition', async function () { const code = dedent` - 🔵class MyClass:🟢 - 🟢"""A simple🟢 example class"""🟢 - 🟢i = 12🟢345🟢 - 🟢 - ❌def❌ f(self):❌ - ❌return❌ 'hello❌ world'❌ + 🔵class MyClass:🟢 + 🟢"""A simple🟢 example class"""🟢 + 🟢i = 12🟢345🟢 + 🟢 + ❌def❌ f(self):❌ + ❌return❌ 'hello❌ world'❌ - `; + `; await testGetNodeStart('python', code); }); test('elif_clause', async function () { const code = dedent` - def ❌sample():❌ - ❌if 1❌: - ❌pass❌ - 🔵elif🟢 2🟢:🟢 - 🟢p🟢a🟢s🟢s - ❌else:❌ - ❌pass❌ - `; + def ❌sample():❌ + ❌if 1❌: + ❌pass❌ + 🔵elif🟢 2🟢:🟢 + 🟢p🟢a🟢s🟢s + ❌else:❌ + ❌pass❌ + `; await testGetNodeStart('python', code); }); test('else_clause', async function () { const code = dedent` - ❌def ❌sample():❌ - ❌if 1:❌ - ❌pass❌ - ❌elif 2:❌ - ❌pass❌ - 🔵else🟢:🟢 - 🟢p🟢a🟢s🟢s - `; + ❌def ❌sample():❌ + ❌if 1:❌ + ❌pass❌ + ❌elif 2:❌ + ❌pass❌ + 🔵else🟢:🟢 + 🟢p🟢a🟢s🟢s + `; await testGetNodeStart('python', code); }); test('except_clause', async function () { const code = dedent` - ❌def❌ ❌sample❌()❌:❌ - ❌try:❌ - ❌pass❌ - 🔵except🟢:🟢 - 🟢p🟢a🟢s🟢s - `; + ❌def❌ ❌sample❌()❌:❌ + ❌try:❌ + ❌pass❌ + 🔵except🟢:🟢 + 🟢p🟢a🟢s🟢s + `; await testGetNodeStart('python', code); }); test('finally_clause', async function () { const code = dedent` - ❌def❌ sa❌mple❌()❌:❌ - ❌try: - ❌pass❌ - 🔵finally🟢:🟢 - 🟢p🟢a🟢s🟢s - `; + ❌def❌ sa❌mple❌()❌:❌ + ❌try: + ❌pass❌ + 🔵finally🟢:🟢 + 🟢p🟢a🟢s🟢s + `; await testGetNodeStart('python', code); }); test('for_statement', async function () { const code = dedent` - ❌def❌ ❌sample(❌):❌ - ❌fruits❌ = ❌["apple", "banana", "cherry"]❌ - 🔵for🟢 x in🟢 fr🟢uits🟢:🟢 - 🟢p🟢a🟢s🟢s - `; + ❌def❌ ❌sample(❌):❌ + ❌fruits❌ = ❌["apple", "banana", "cherry"]❌ + 🔵for🟢 x in🟢 fr🟢uits🟢:🟢 + 🟢p🟢a🟢s🟢s + `; await testGetNodeStart('python', code); }); test('function_definition', async function () { const code = dedent` - 🔵def🟢 sam🟢ple🟢(🟢)🟢: - 🟢"""Sample 🟢comment"""🟢 - 🟢fruits🟢 = 🟢["apple", 🟢"banana",🟢 "cherry"]🟢 - ❌for❌ x❌ in❌ fruits❌:❌ - ❌p❌a❌s❌s❌ - `; + 🔵def🟢 sam🟢ple🟢(🟢)🟢: + 🟢"""Sample 🟢comment"""🟢 + 🟢fruits🟢 = 🟢["apple", 🟢"banana",🟢 "cherry"]🟢 + ❌for❌ x❌ in❌ fruits❌:❌ + ❌p❌a❌s❌s❌ + `; await testGetNodeStart('python', code); }); test('if_statement', async function () { const code = dedent` - ❌def ❌sample❌(❌)❌:❌ - 🔵if 🟢1🟢:🟢 - 🟢p🟢a🟢s🟢s - ❌elif❌ 2:❌ - ❌pass - ❌else:❌ - ❌pass - `; + ❌def ❌sample❌(❌)❌:❌ + 🔵if 🟢1🟢:🟢 + 🟢p🟢a🟢s🟢s + ❌elif❌ 2:❌ + ❌pass + ❌else:❌ + ❌pass + `; await testGetNodeStart('python', code); }); test('try_statement', async function () { const code = dedent` - ❌def❌ ❌sample❌(❌)❌:❌ - 🔵try🟢:🟢 - 🟢p🟢a🟢s🟢s - ❌fin❌all❌y:❌ - ❌pass❌ - `; + ❌def❌ ❌sample❌(❌)❌:❌ + 🔵try🟢:🟢 + 🟢p🟢a🟢s🟢s + ❌fin❌all❌y:❌ + ❌pass❌ + `; await testGetNodeStart('python', code); }); test('while_statement', async function () { const code = dedent` - ❌def❌ sa❌mple(❌)❌:❌ - 🔵while🟢 🟢Tr🟢ue🟢:🟢 - 🟢p🟢a🟢s🟢s - `; + ❌def❌ sa❌mple(❌)❌:❌ + 🔵while🟢 🟢Tr🟢ue🟢:🟢 + 🟢p🟢a🟢s🟢s + `; await testGetNodeStart('python', code); }); test('with_statement', async function () { const code = dedent` - ❌def❌ ❌sa❌mple❌(❌)❌:❌ - 🔵with🟢 🟢open🟢(🟢'fil🟢e_pa🟢th'🟢, 🟢'w')🟢 🟢as🟢 🟢f🟢i🟢l🟢e🟢:🟢 - 🟢p🟢a🟢s🟢s - `; + ❌def❌ ❌sa❌mple❌(❌)❌:❌ + 🔵with🟢 🟢open🟢(🟢'fil🟢e_pa🟢th'🟢, 🟢'w')🟢 🟢as🟢 🟢f🟢i🟢l🟢e🟢:🟢 + 🟢p🟢a🟢s🟢s + `; await testGetNodeStart('python', code); }); }); - // tests for JavaScript and TypeScript: `⦃...⦄` delineates the body, `〚...〛` the type annotations, + // tests for JavaScript and TypeScript: `⦃...⦄` delineates the body, `〚...〛` the type annotations, // which are stripped off for JavaScript const test1 = dedent` - function getTextOrNull(document〚: doc | null〛) { - if (document === null) - ⦃ return null; - return document.getText(); - }⦄ + function getTextOrNull(document〚: doc | null〛) { + if (document === null) + ⦃ return null; + return document.getText(); + }⦄ - // this is a comment`; + // this is a comment`; const test2 = dedent` - function getB(capital〚: boolean〛) { - if (capital) { - return "B"; - } else {⦃ - return "b"; - }⦄ - }`; + function getB(capital〚: boolean〛) { + if (capital) { + return "B"; + } else {⦃ + return "b"; + }⦄ + }`; function mkTestCase(src: string, stripTypes: boolean) { - if (stripTypes) { src = src.replace(/〚.*?〛/g, ''); } - const bodyStart = src.indexOf('⦃'); - const bodyEnd = src.indexOf('⦄'); + if (stripTypes) { src = src.replace(/〚.*?〛/g, ''); } + const bodyStart = src.indexOf('⦃'); + const bodyEnd = src.indexOf('⦄'); return { before: src.slice(0, bodyStart), body: src.slice(bodyStart + 1, bodyEnd), diff --git a/extensions/copilot/src/extension/completions-core/vscode-node/prompt/src/test/tokenizer.test.ts b/extensions/copilot/src/extension/completions-core/vscode-node/prompt/src/test/tokenizer.test.ts index 354778fc375..35355c4297a 100644 --- a/extensions/copilot/src/extension/completions-core/vscode-node/prompt/src/test/tokenizer.test.ts +++ b/extensions/copilot/src/extension/completions-core/vscode-node/prompt/src/test/tokenizer.test.ts @@ -2,11 +2,11 @@ * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { ApproximateTokenizer, getTokenizer, TokenizerName } from '../tokenization'; import * as assert from 'assert'; import * as fs from 'fs'; import { suite } from 'mocha'; import { resolve } from 'path'; +import { ApproximateTokenizer, getTokenizer, TokenizerName } from '../tokenization'; // Read the source files and normalize the line endings const source = fs.readFileSync(resolve(__dirname, 'testdata/example.py'), 'utf8').replace(/\r\n?/g, '\n'); @@ -96,7 +96,7 @@ suite('Tokenizer Test Suite - cl100k', function () { }); test('emojis', function () { - const str = 'hello 👋 world 🌍'; + const str = 'hello 👋 world 🌍'; assert.deepStrictEqual(tokenizer.tokenize(str), [15339, 62904, 233, 1917, 11410, 234, 235]); assert.strictEqual(tokenizer.detokenize(tokenizer.tokenize(str)), str); }); @@ -119,9 +119,9 @@ suite('Tokenizer Test Suite - cl100k', function () { } // Test special characters - assert.deepStrictEqual(tokenizer.tokenize('\n\n👋'), [271, 9468, 239, 233]); + assert.deepStrictEqual(tokenizer.tokenize('\n\n👋'), [271, 9468, 239, 233]); assert.deepStrictEqual(tokenizer.tokenize('\n\n '), [271, 220]); - assert.deepStrictEqual(tokenizer.tokenize('\n\n 👋'), [271, 62904, 233]); + assert.deepStrictEqual(tokenizer.tokenize('\n\n 👋'), [271, 62904, 233]); assert.deepStrictEqual(tokenizer.tokenize('\n\n\t'), [271, 197]); assert.deepStrictEqual(tokenizer.tokenize('\n\n\r'), [271, 201]); @@ -296,7 +296,7 @@ suite('Tokenizer Test Suite - o200k', function () { }); test('emojis', function () { - const str = 'hello 👋 world 🌍'; + const str = 'hello 👋 world 🌍'; assert.deepStrictEqual(tokenizer.tokenize(str), [24912, 61138, 233, 2375, 130321, 235]); assert.strictEqual(tokenizer.detokenize(tokenizer.tokenize(str)), str); }); @@ -319,9 +319,9 @@ suite('Tokenizer Test Suite - o200k', function () { } // Test special characters - assert.deepStrictEqual(tokenizer.tokenize('\n\n👋'), [279, 28823, 233]); + assert.deepStrictEqual(tokenizer.tokenize('\n\n👋'), [279, 28823, 233]); assert.deepStrictEqual(tokenizer.tokenize('\n\n '), [279, 220]); - assert.deepStrictEqual(tokenizer.tokenize('\n\n 👋'), [279, 61138, 233]); + assert.deepStrictEqual(tokenizer.tokenize('\n\n 👋'), [279, 61138, 233]); assert.deepStrictEqual(tokenizer.tokenize('\n\n\t'), [279, 197]); assert.deepStrictEqual(tokenizer.tokenize('\n\n\r'), [279, 201]);