Make prompt tests work again. Port over more tests from client (#1542)

extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/contextProviderRegistryMultiLanguage.ts

@@ -6,8 +6,8 @@
 import { Context } from '../context';
 import { Features } from '../experiments/features';
 import { logger } from '../logger';
-import { ActiveExperiments } from './contextProviderRegistry';
 import { TelemetryWithExp } from '../telemetry';
+import { ActiveExperiments } from './contextProviderRegistry';
 
 const MULTI_LANGUAGE_CONTEXT_PROVIDER_ID = 'fallbackContextProvider';
 
@@ -36,7 +36,7 @@ interface MultiLanguageContextProviderParams {
 	mlcpEnableImports: boolean;
 }
 
-const multiLanguageContextProviderParamsDefault: MultiLanguageContextProviderParams = {
+export const multiLanguageContextProviderParamsDefault: MultiLanguageContextProviderParams = {
 	mlcpMaxContextItems: 20,
 	mlcpMaxSymbolMatches: 20,
 	mlcpEnableImports: false,
@@ -83,3 +83,23 @@ function getMultiLanguageContextProviderParamsFromExp(
 
 	return params;
 }
+
+export function getMultiLanguageContextProviderParamsFromActiveExperiments(
+	activeExperiments: Map<string, string | number | boolean | string[]>
+): MultiLanguageContextProviderParams {
+	const params = { ...multiLanguageContextProviderParamsDefault };
+
+	if (activeExperiments.has('mlcpMaxContextItems')) {
+		params.mlcpMaxContextItems = Number(activeExperiments.get('mlcpMaxContextItems'));
+	}
+
+	if (activeExperiments.has('mlcpMaxSymbolMatches')) {
+		params.mlcpMaxSymbolMatches = Number(activeExperiments.get('mlcpMaxSymbolMatches'));
+	}
+
+	if (activeExperiments.has('mlcpEnableImports')) {
+		params.mlcpEnableImports = String(activeExperiments.get('mlcpEnableImports')) === 'true';
+	}
+
+	return params;
+}

extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/contextProviderRegistryTs.ts

@@ -6,10 +6,10 @@
 import { Context } from '../context';
 import { Features } from '../experiments/features';
 import { logger } from '../logger';
-import { ActiveExperiments } from './contextProviderRegistry';
 import { TelemetryWithExp } from '../telemetry';
+import { ActiveExperiments } from './contextProviderRegistry';
 
-const TS_CONTEXT_PROVIDER_ID = 'typescript-ai-context-provider';
+export const TS_CONTEXT_PROVIDER_ID = 'typescript-ai-context-provider';
 
 interface ContextProviderParams {
 	[key: string]: string | number | boolean;

extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/contextProviderStatistics.ts

@@ -17,7 +17,7 @@ import { SupportedContextItemWithId } from './contextProviders/contextItemSchema
 
 export type PromptExpectation = 'included' | 'content_excluded';
 
-type PromptMatcher = {
+export type PromptMatcher = {
 	source: SupportedContextItemWithId;
 	expectedTokens: number;
 	actualTokens: number;

extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/repository.ts

@@ -76,7 +76,7 @@ const backgroundRepoInfo = computeInBackgroundAndMemoize<RepoInfo | undefined, [
  * If it does appear to be part of a git repository, but its information is not parsable,
  * it returns a RepoInfo object with hostname, user and repo set to "".
  */
-async function extractRepoInfo(ctx: Context, uri: FileIdentifier): Promise<RepoInfo | undefined> {
+export async function extractRepoInfo(ctx: Context, uri: FileIdentifier): Promise<RepoInfo | undefined> {
 	const fsUri = getFsUri(uri);
 	if (!fsUri) { return undefined; }
 

extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/test/contextProviderRegistryMultiLanguage.test.ts

@@ -0,0 +1,90 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import * as assert from 'assert';
+import {
+	getMultiLanguageContextProviderParamsFromActiveExperiments,
+	multiLanguageContextProviderParamsDefault,
+} from '../contextProviderRegistryMultiLanguage';
+
+suite('contextProviderRegistryMultiLanguage', function () {
+	let activeExperiments: Map<string, string | number | boolean | string[]>;
+
+	setup(function () {
+		activeExperiments = new Map();
+	});
+
+	suite('getMultiLanguageContextProviderConfigFromActiveExperiments', function () {
+		test('returns default config when no experiments are set', function () {
+			const result = getMultiLanguageContextProviderParamsFromActiveExperiments(new Map());
+
+			assert.deepStrictEqual(result, multiLanguageContextProviderParamsDefault);
+		});
+
+		test('overrides defaults with experiment values', function () {
+			activeExperiments.set('mlcpMaxContextItems', '50');
+			activeExperiments.set('mlcpMaxSymbolMatches', 30);
+			activeExperiments.set('mlcpEnableImports', true);
+
+			const result = getMultiLanguageContextProviderParamsFromActiveExperiments(activeExperiments);
+
+			assert.strictEqual(result.mlcpMaxContextItems, 50);
+			assert.strictEqual(result.mlcpMaxSymbolMatches, 30);
+			assert.strictEqual(result.mlcpEnableImports, true);
+		});
+
+		test('converts string values to appropriate types', function () {
+			activeExperiments.set('mlcpMaxContextItems', '25');
+			activeExperiments.set('mlcpEnableImports', 'true');
+
+			const result = getMultiLanguageContextProviderParamsFromActiveExperiments(activeExperiments);
+
+			assert.strictEqual(result.mlcpMaxContextItems, 25);
+			assert.strictEqual(result.mlcpEnableImports, true);
+		});
+
+		test('converts string values for false to appropriate types', function () {
+			activeExperiments.set('mlcpMaxContextItems', '25');
+			activeExperiments.set('mlcpEnableImports', 'false');
+
+			const result = getMultiLanguageContextProviderParamsFromActiveExperiments(activeExperiments);
+
+			assert.strictEqual(result.mlcpMaxContextItems, 25);
+			assert.strictEqual(result.mlcpEnableImports, false);
+		});
+
+		test('handles partial overrides', function () {
+			activeExperiments.set('mlcpEnableImports', true);
+
+			const result = getMultiLanguageContextProviderParamsFromActiveExperiments(activeExperiments);
+
+			assert.strictEqual(
+				result.mlcpMaxContextItems,
+				multiLanguageContextProviderParamsDefault.mlcpMaxContextItems
+			);
+			assert.strictEqual(
+				result.mlcpMaxSymbolMatches,
+				multiLanguageContextProviderParamsDefault.mlcpMaxSymbolMatches
+			);
+			assert.strictEqual(result.mlcpEnableImports, true);
+		});
+
+		test('converts falsy values correctly', function () {
+			activeExperiments.set('mlcpMaxContextItems', 0);
+			activeExperiments.set('mlcpEnableImports', false);
+
+			const result = getMultiLanguageContextProviderParamsFromActiveExperiments(activeExperiments);
+
+			assert.strictEqual(result.mlcpMaxContextItems, 0);
+			assert.strictEqual(result.mlcpEnableImports, false);
+		});
+
+		test('returns false for imports when not set', function () {
+			const result = getMultiLanguageContextProviderParamsFromActiveExperiments(activeExperiments);
+
+			assert.strictEqual(result.mlcpEnableImports, false);
+		});
+	});
+});

extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/test/contextProviderRegistryTs.test.ts

@@ -0,0 +1,54 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import assert from 'assert';
+import { Context } from '../../context';
+import { TelemetryWithExp } from '../../telemetry';
+import { createLibTestingContext } from '../../test/context';
+import { ActiveExperiments } from '../contextProviderRegistry';
+import { fillInTsActiveExperiments, TS_CONTEXT_PROVIDER_ID } from '../contextProviderRegistryTs';
+
+suite('contextProviderRegistryTs', function () {
+	let ctx: Context;
+	let activeExperiments: ActiveExperiments;
+	let telemetryData: TelemetryWithExp;
+
+	setup(function () {
+		ctx = createLibTestingContext();
+		activeExperiments = new Map();
+		telemetryData = TelemetryWithExp.createEmptyConfigForTesting();
+		telemetryData.filtersAndExp.exp.variables['copilottscontextproviderparams'] = JSON.stringify({
+			booleanProperty: true,
+		});
+	});
+
+	test('does not add active experiments if no provider is active', function () {
+		fillInTsActiveExperiments(ctx, [], activeExperiments, telemetryData);
+
+		assert.ok(activeExperiments.size === 0);
+	});
+
+	test('adds active experiments if TS provider is active', function () {
+		fillInTsActiveExperiments(ctx, [TS_CONTEXT_PROVIDER_ID], activeExperiments, telemetryData);
+
+		assert.ok(activeExperiments.has('booleanProperty'));
+		assert.strictEqual(activeExperiments.get('booleanProperty'), true);
+	});
+
+	test('adds active experiments in debug mode', function () {
+		fillInTsActiveExperiments(ctx, ['*'], activeExperiments, telemetryData);
+
+		assert.ok(activeExperiments.has('booleanProperty'));
+		assert.strictEqual(activeExperiments.get('booleanProperty'), true);
+	});
+
+	test('bad JSON is ignored', function () {
+		telemetryData.filtersAndExp.exp.variables['copilottscontextproviderparams'] = '{"badJSON": true';
+
+		fillInTsActiveExperiments(ctx, [TS_CONTEXT_PROVIDER_ID], activeExperiments, telemetryData);
+
+		assert.ok(activeExperiments.size === 0);
+	});
+});

extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/test/contextProviderStatistics.test.ts

@@ -0,0 +1,256 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import assert from 'assert';
+import { ResolutionStatus } from '../../../../types/src/index';
+import { TraitWithId } from '../contextProviders/contextItemSchemas';
+import { PromptMatcher } from '../contextProviderStatistics';
+import { TestContextProviderStatistics } from './contextProviderStatistics';
+
+suite('contextProviderStatistics', function () {
+	let statistics: TestContextProviderStatistics;
+	const resolutions: ResolutionStatus[] = ['partial', 'full'];
+
+	setup(function () {
+		statistics = new TestContextProviderStatistics();
+	});
+
+	const trait1: TraitWithId = {
+		name: 'trait1',
+		value: 'value1',
+		id: '1234',
+		type: 'Trait',
+	};
+	const trait2: TraitWithId = {
+		name: 'trait2',
+		value: 'value2',
+		id: '5678',
+		type: 'Trait',
+	};
+
+	test('can set expectations', function () {
+		statistics.addExpectations('bar', [
+			[trait1, 'included'],
+			[trait2, 'content_excluded'],
+		]);
+		assert.deepStrictEqual(statistics.expectations.size, 1);
+		assert.deepStrictEqual(statistics.expectations.get('bar')?.length, 2);
+	});
+
+	test('can add expectations', function () {
+		statistics.addExpectations('bar', [
+			[trait1, 'included'],
+			[trait2, 'content_excluded'],
+		]);
+
+		const trait3: TraitWithId = {
+			name: 'trait3',
+			value: 'value3',
+			id: '9012',
+			type: 'Trait',
+		};
+		const trait4: TraitWithId = {
+			name: 'trait4',
+			value: 'value4',
+			id: '3456',
+			type: 'Trait',
+		};
+
+		statistics.addExpectations('bar', [
+			[trait3, 'included'],
+			[trait4, 'content_excluded'],
+		]);
+		assert.deepStrictEqual(statistics.expectations.size, 1);
+		assert.deepStrictEqual(statistics.expectations.get('bar')?.length, 4);
+	});
+
+	test('computing match unsets expectations and resolution', function () {
+		statistics.addExpectations('bar', [
+			[trait1, 'included'],
+			[trait2, 'content_excluded'],
+		]);
+		statistics.setLastResolution('bar', 'full');
+		assert.deepStrictEqual(statistics.expectations.size, 1);
+		assert.deepStrictEqual(statistics.lastResolution.size, 1);
+
+		statistics.computeMatch([]);
+
+		assert.deepStrictEqual(statistics.expectations.size, 0);
+		assert.deepStrictEqual(statistics.lastResolution.size, 0);
+	});
+
+	test('does not compute match for empty expectations', function () {
+		statistics.addExpectations('bar', []);
+
+		statistics.computeMatch([]);
+
+		assert.deepStrictEqual(statistics.statistics.size, 0);
+	});
+
+	for (const resolution of resolutions) {
+		test(`can match full expectations, resolution: ${resolution}`, function () {
+			statistics.addExpectations('foo', [
+				[trait1, 'included'],
+				[trait2, 'included'],
+			]);
+			statistics.setLastResolution('foo', resolution);
+
+			const promptMatcher: PromptMatcher[] = [
+				{
+					expectedTokens: 7,
+					actualTokens: 7,
+					source: trait1,
+				},
+				{
+					expectedTokens: 10,
+					actualTokens: 10,
+					source: trait2,
+				},
+			];
+
+			statistics.computeMatch(promptMatcher);
+			const stats = statistics.get('foo');
+			assert.ok(stats);
+			assert.deepStrictEqual(stats.resolution, resolution);
+			assert.deepStrictEqual(stats.usage, 'full');
+			assert.deepStrictEqual(stats.usageDetails, [
+				{ id: '1234', usage: 'full', expectedTokens: 7, actualTokens: 7, type: 'Trait' },
+				{ id: '5678', usage: 'full', expectedTokens: 10, actualTokens: 10, type: 'Trait' },
+			]);
+		});
+
+		test(`can match partial expectations, resolution: ${resolution}`, function () {
+			statistics.addExpectations('foo', [
+				[trait1, 'included'],
+				[trait2, 'included'],
+			]);
+			statistics.setLastResolution('foo', resolution);
+
+			const promptMatchers: PromptMatcher[] = [
+				{
+					expectedTokens: 7,
+					actualTokens: 7,
+					source: trait1,
+				},
+				{
+					expectedTokens: 10,
+					actualTokens: 5,
+					source: trait2,
+				},
+			];
+
+			statistics.computeMatch(promptMatchers);
+			const stats = statistics.get('foo');
+			assert.ok(stats);
+			assert.deepStrictEqual(stats.resolution, resolution);
+			assert.deepStrictEqual(stats.usage, 'partial');
+			assert.deepStrictEqual(stats.usageDetails, [
+				{ id: '1234', usage: 'full', expectedTokens: 7, actualTokens: 7, type: 'Trait' },
+				{ id: '5678', usage: 'partial', expectedTokens: 10, actualTokens: 5, type: 'Trait' },
+			]);
+		});
+
+		test(`full elision is no usage, resolution: ${resolution}`, function () {
+			statistics.addExpectations('foo', [
+				[trait1, 'included'],
+				[trait2, 'included'],
+			]);
+			statistics.setLastResolution('foo', resolution);
+
+			const promptMatchers: PromptMatcher[] = [
+				{
+					expectedTokens: 7,
+					actualTokens: 0,
+					source: trait1,
+				},
+				{
+					expectedTokens: 10,
+					actualTokens: 0,
+					source: trait2,
+				},
+			];
+
+			statistics.computeMatch(promptMatchers);
+			const stats = statistics.get('foo');
+			assert.ok(stats);
+			assert.deepStrictEqual(stats.resolution, resolution);
+			assert.deepStrictEqual(stats.usage, 'none');
+			assert.deepStrictEqual(stats.usageDetails, [
+				{ id: '1234', usage: 'none', expectedTokens: 7, actualTokens: 0, type: 'Trait' },
+				{ id: '5678', usage: 'none', expectedTokens: 10, actualTokens: 0, type: 'Trait' },
+			]);
+		});
+
+		test(`some content excluded items make it partial, resolution: ${resolution}`, function () {
+			statistics.addExpectations('foo', [
+				[trait1, 'included'],
+				[trait2, 'content_excluded'],
+			]);
+			statistics.setLastResolution('foo', resolution);
+
+			const promptMatchers: PromptMatcher[] = [
+				{
+					expectedTokens: 7,
+					actualTokens: 7,
+					source: trait1,
+				},
+			];
+
+			statistics.computeMatch(promptMatchers);
+			const stats = statistics.get('foo');
+			assert.ok(stats);
+			assert.deepStrictEqual(stats.resolution, resolution);
+			assert.deepStrictEqual(stats.usage, 'partial');
+			assert.deepStrictEqual(stats.usageDetails, [
+				{ id: '1234', usage: 'full', expectedTokens: 7, actualTokens: 7, type: 'Trait' },
+				{ id: '5678', usage: 'none_content_excluded', type: 'Trait' },
+			]);
+		});
+
+		test(`all content excluded items make it none, resolution: ${resolution}`, function () {
+			statistics.addExpectations('foo', [
+				[trait1, 'content_excluded'],
+				[trait2, 'content_excluded'],
+			]);
+			statistics.setLastResolution('foo', resolution);
+
+			statistics.computeMatch([]);
+			const stats = statistics.get('foo');
+			assert.ok(stats);
+			assert.deepStrictEqual(stats.resolution, resolution);
+			assert.deepStrictEqual(stats.usage, 'none');
+			assert.deepStrictEqual(stats.usageDetails, [
+				{ id: '1234', usage: 'none_content_excluded', type: 'Trait' },
+				{ id: '5678', usage: 'none_content_excluded', type: 'Trait' },
+			]);
+		});
+	}
+
+	test('none resolution is always no match', function () {
+		statistics.addExpectations('foo', [
+			[trait1, 'included'],
+			[trait1, 'included'],
+		]);
+		statistics.setLastResolution('foo', 'none');
+
+		statistics.computeMatch([]);
+
+		const stats = statistics.get('foo');
+		assert.deepStrictEqual(stats!, { usage: 'none', resolution: 'none' });
+	});
+
+	test('error resolution is always no match', function () {
+		statistics.addExpectations('foo', [
+			[trait1, 'content_excluded'],
+			[trait2, 'content_excluded'],
+		]);
+		statistics.setLastResolution('foo', 'error');
+
+		statistics.computeMatch([]);
+
+		const stats = statistics.get('foo');
+		assert.deepStrictEqual(stats!, { usage: 'none', resolution: 'error' });
+	});
+});

extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/test/parseBlock.test.ts

@@ -0,0 +1,38 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import * as assert from 'assert';
+import { contextIndentationFromText } from '../parseBlock';
+
+suite('Indentation', function () {
+	test('single line -> only current', function () {
+		assert.deepStrictEqual(contextIndentationFromText('x', 0, 'language'), {
+			prev: undefined,
+			current: 0,
+			next: undefined,
+		});
+	});
+	test('single line with line after -> only current & next', function () {
+		assert.deepStrictEqual(contextIndentationFromText('x\ny', 0, 'language'), {
+			prev: undefined,
+			current: 0,
+			next: 0,
+		});
+	});
+	test('after indent -> only current & prev', function () {
+		assert.deepStrictEqual(contextIndentationFromText('x\n y', 4, 'language'), {
+			prev: 0,
+			current: 1,
+			next: undefined,
+		});
+	});
+	test('after indent but before text -> only current from line above', function () {
+		assert.deepStrictEqual(contextIndentationFromText('x\n y', 3, 'language'), {
+			prev: undefined,
+			current: 0,
+			next: undefined,
+		});
+	});
+});

extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/test/prompt.test.ts

@@ -0,0 +1,296 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import * as assert from 'assert';
+import * as sinon from 'sinon';
+import dedent from 'ts-dedent';
+import {
+	DEFAULT_MAX_COMPLETION_LENGTH,
+	DEFAULT_MAX_PROMPT_LENGTH,
+	DEFAULT_NUM_SNIPPETS,
+	DEFAULT_PROMPT_ALLOCATION_PERCENT,
+	DEFAULT_SUFFIX_MATCH_THRESHOLD,
+	PromptOptions,
+} from '../../../../prompt/src/prompt';
+import { defaultSimilarFilesOptions } from '../../../../prompt/src/snippetInclusion/similarFiles';
+import { CopilotContentExclusionManager } from '../../contentExclusion/contentExclusionManager';
+import { Context } from '../../context';
+import { ExpTreatmentVariables } from '../../experiments/expConfig';
+import { TelemetryWithExp } from '../../telemetry';
+import { createLibTestingContext } from '../../test/context';
+import { createTextDocument, InMemoryNotebookDocument, TestTextDocumentManager } from '../../test/textDocument';
+import { INotebookCell, IPosition } from '../../textDocument';
+import { TextDocumentManager } from '../../textDocumentManager';
+import { CompletionsPromptRenderer } from '../components/completionsPromptRenderer';
+import { _copilotContentExclusion, _promptError, getPromptOptions } from '../prompt';
+import { extractPromptInternal } from './prompt';
+
+suite('Prompt unit tests', function () {
+	let ctx: Context;
+	let sandbox: sinon.SinonSandbox;
+
+	setup(function () {
+		sandbox = sinon.createSandbox();
+		ctx = createLibTestingContext();
+	});
+
+	teardown(function () {
+		sandbox.restore();
+	});
+
+	test('defaults to 8K max prompt length', async function () {
+		const content = 'function add()\n';
+		const sourceDoc = createTextDocument('file:///foo.js', 'javascript', 0, content);
+		const cursorPosition: IPosition = {
+			line: 0,
+			character: 13,
+		};
+
+		const rendererStub = sandbox.stub(CompletionsPromptRenderer.prototype, 'render').throws('unspecified error');
+
+		const prompt = await extractPromptInternal(
+			ctx,
+			'COMPLETION_ID',
+			sourceDoc,
+			cursorPosition,
+			TelemetryWithExp.createEmptyConfigForTesting()
+		);
+
+		assert.deepStrictEqual(prompt, _promptError);
+		assert.ok(rendererStub.calledOnce, 'should call renderer');
+		assert.strictEqual(
+			rendererStub.firstCall.args[1].promptTokenLimit,
+			8192 - DEFAULT_MAX_COMPLETION_LENGTH,
+			'should default to 8192 max total tokens, 7692 max prompt tokens'
+		);
+	});
+
+	test('default EXP prompt options are the same as default PromptOptions object', function () {
+		const promptOptionsFromExp = getPromptOptions(ctx, TelemetryWithExp.createEmptyConfigForTesting(), '');
+		const defaultPromptOptions: PromptOptions = {
+			maxPromptLength: DEFAULT_MAX_PROMPT_LENGTH,
+			numberOfSnippets: DEFAULT_NUM_SNIPPETS,
+			similarFilesOptions: defaultSimilarFilesOptions,
+			suffixMatchThreshold: DEFAULT_SUFFIX_MATCH_THRESHOLD,
+			suffixPercent: DEFAULT_PROMPT_ALLOCATION_PERCENT.suffix,
+		};
+
+		assert.deepStrictEqual(promptOptionsFromExp, defaultPromptOptions);
+	});
+
+	test('default C++ EXP prompt options use tuned values', function () {
+		const promptOptionsFromExp: PromptOptions = getPromptOptions(
+			ctx,
+			TelemetryWithExp.createEmptyConfigForTesting(),
+			'cpp'
+		);
+
+		assert.deepStrictEqual(promptOptionsFromExp.similarFilesOptions, {
+			snippetLength: 60,
+			threshold: 0.0,
+			maxTopSnippets: 16,
+			maxCharPerFile: 100000,
+			maxNumberOfFiles: 200,
+			maxSnippetsPerFile: 4,
+			useSubsetMatching: false,
+		});
+		assert.deepStrictEqual(promptOptionsFromExp.numberOfSnippets, 16);
+	});
+
+	test('default Java EXP prompt options are correct', function () {
+		const telemetryWithExp = TelemetryWithExp.createEmptyConfigForTesting();
+		const expVars = telemetryWithExp.filtersAndExp.exp.variables;
+
+		Object.assign(expVars, {
+			[ExpTreatmentVariables.UseSubsetMatching]: true,
+		});
+
+		const promptOptionsFromExp = getPromptOptions(ctx, telemetryWithExp, 'java');
+		assert.deepStrictEqual(promptOptionsFromExp.similarFilesOptions, {
+			snippetLength: 60,
+			threshold: 0.0,
+			maxTopSnippets: 4,
+			maxCharPerFile: 10000,
+			maxNumberOfFiles: 20,
+			maxSnippetsPerFile: 1,
+			useSubsetMatching: true,
+		});
+		assert.deepStrictEqual(promptOptionsFromExp.numberOfSnippets, 4);
+	});
+
+	test('should return without a prompt if the file blocked by repository control', async function () {
+		const evaluateStub = sandbox.stub(CopilotContentExclusionManager.prototype, 'evaluate');
+		evaluateStub.callsFake(_ => {
+			return Promise.resolve({ isBlocked: true });
+		});
+
+		const content = 'function add()\n';
+		const sourceDoc = createTextDocument('file:///foo.js', 'javascript', 0, content);
+		const cursorPosition: IPosition = {
+			line: 0,
+			character: 13,
+		};
+		const response = await extractPromptInternal(
+			ctx,
+			'COMPLETION_ID',
+			sourceDoc,
+			cursorPosition,
+			TelemetryWithExp.createEmptyConfigForTesting()
+		);
+		assert.ok(response);
+		assert.strictEqual(response, _copilotContentExclusion);
+	});
+
+	test('prompt for ipython notebooks, using only the current cell language as shebang', async function () {
+		await assertPromptForCell(
+			ctx,
+			cells[4],
+			dedent(
+				`import math
+
+def add(a, b):
+    return a + b
+
+def product(c, d):`
+			),
+			['#!/usr/bin/env python3']
+		);
+	});
+
+	test('prompt for ipython notebooks, using only the current cell language for known language', async function () {
+		await assertPromptForCell(
+			ctx,
+			cells[5],
+			dedent(
+				`def product(c, d):`
+			),
+			['Language: julia']
+		);
+	});
+
+	test('prompt for ipython notebooks, using only the current cell language for unknown language', async function () {
+		await assertPromptForCell(ctx, cells[6], dedent(`foo bar baz`), ['Language: unknown-great-language']);
+	});
+
+	test('exception telemetry', async function () {
+		this.skip();
+		/* todo@dbaeumer need to understand how we handle exception in chat
+		class TestExceptionTextDocumentManager extends TestTextDocumentManager {
+			override textDocuments() {
+				return Promise.reject(new Error('test error'));
+			}
+		}
+		const tdm = new TestExceptionTextDocumentManager(ctx);
+		tdm.setTextDocument('file:///a/1.py', 'python', 'import torch');
+		ctx.forceSet(TextDocumentManager, tdm);
+		NeighborSource.reset();
+
+		const { reporter, enhancedReporter } = await withInMemoryTelemetry(ctx, async ctx => {
+			const document = createTextDocument('file:///a/2.py', 'python', 0, 'import torch');
+			await extractPromptInternal(
+				ctx,
+				'COMPLETION_ID',
+				document,
+				{ line: 0, character: 0 },
+				TelemetryWithExp.createEmptyConfigForTesting()
+			);
+		});
+
+		assert.ok(reporter.hasException);
+		assert.deepStrictEqual(
+			reporter.firstException?.properties?.origin,
+			'PromptComponents.CompletionsPromptFactory'
+		);
+		assert.strictEqual(reporter.exceptions.length, 1);
+
+		assert.ok(enhancedReporter.hasException);
+		assert.deepStrictEqual(
+			enhancedReporter.firstException?.properties?.origin,
+			'PromptComponents.CompletionsPromptFactory'
+		);
+		assert.strictEqual(enhancedReporter.exceptions.length, 1);
+		*/
+	});
+});
+
+async function assertPromptForCell(ctx: Context, sourceCell: INotebookCell, expectedPrefix: string, expectedContext?: string[]) {
+	const notebook = new InMemoryNotebookDocument(cells);
+	const sourceDoc = sourceCell.document;
+
+	(ctx.get(TextDocumentManager) as TestTextDocumentManager).setNotebookDocument(sourceDoc, notebook);
+
+	const cursorPosition: IPosition = {
+		line: 0,
+		character: sourceDoc.getText().length,
+	};
+	const response = await extractPromptInternal(
+		ctx,
+		'COMPLETION_ID',
+		sourceDoc,
+		cursorPosition,
+		TelemetryWithExp.createEmptyConfigForTesting()
+	);
+	assert.ok(response);
+	assert.strictEqual(response.type, 'prompt');
+	assert.strictEqual(response.prompt.prefix, expectedPrefix);
+	if (expectedContext !== undefined) {
+		assert.deepEqual(response.prompt.context, expectedContext);
+	}
+}
+
+const cells: INotebookCell[] = [
+	{
+		index: 1,
+		document: createTextDocument('file:///test/a.ipynb#1', 'python', 1, 'import math'),
+		metadata: {},
+		kind: 2,
+	},
+	{
+		index: 2,
+		document: createTextDocument(
+			'file:///test/a.ipynb#2',
+			'markdown',
+			1,
+			'This is an addition function\nIt is used to add two numbers'
+		),
+		metadata: {},
+		kind: 1,
+	},
+	{
+		index: 3,
+		document: createTextDocument('file:///test/a.ipynb#3', 'python', 2, 'def add(a, b):\n    return a + b'),
+		metadata: {},
+		kind: 2,
+	},
+	{
+		index: 4,
+		document: createTextDocument(
+			'file:///test/a.ipynb#4',
+			'markdown',
+			2,
+			'This is a product function\nYou guessed it: it multiplies two numbers'
+		),
+		metadata: {},
+		kind: 2,
+	},
+	{
+		index: 5,
+		document: createTextDocument('file:///test/a.ipynb#5', 'python', 3, 'def product(c, d):'),
+		metadata: {},
+		kind: 2,
+	},
+	{
+		index: 6,
+		document: createTextDocument('file:///test/a.ipynb#6', 'julia', 3, 'def product(c, d):'),
+		metadata: {},
+		kind: 2,
+	},
+	{
+		index: 7,
+		document: createTextDocument('file:///test/a.ipynb#7', 'unknown-great-language', 3, 'foo bar baz'),
+		metadata: {},
+		kind: 2,
+	},
+];

extensions/copilot/src/extension/completions-core/vscode-node/lib/src/prompt/test/repository.test.ts

@@ -0,0 +1,97 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import assert from 'assert';
+import path from 'path';
+import { Context } from '../../context';
+import { FileSystem } from '../../fileSystem';
+import { createLibTestingContext } from '../../test/context';
+import { FakeFileSystem } from '../../test/filesystem';
+import { makeFsUri } from '../../util/uri';
+import { ComputationStatus, extractRepoInfo, extractRepoInfoInBackground } from '../repository';
+
+suite('Extract repo info tests', function () {
+	const baseFolder = { uri: makeFsUri(path.resolve(__dirname, '../../../../../../../../')) };
+
+	class Nested {
+		nested: Nested | undefined;
+	}
+
+	test('avoid using context as cache key', function () {
+		const ctx = new Context();
+		ctx.set(FileSystem, new FakeFileSystem({}));
+		const n = new Nested();
+		ctx.set(Nested, n);
+		n.nested = n;
+
+		const maybe = extractRepoInfoInBackground(ctx, makeFsUri(__filename));
+
+		assert.deepStrictEqual(maybe, ComputationStatus.PENDING);
+	});
+
+	test('Extract repo info', async function () {
+		const ctx = createLibTestingContext();
+		const info = await extractRepoInfo(ctx, baseFolder.uri);
+
+		assert.ok(info);
+
+		// url and pathname get their own special treatment because they depend on how the repo was cloned.
+		const { url, pathname, repoId, ...repoInfo } = info;
+
+		assert.deepStrictEqual(repoInfo, {
+			baseFolder,
+			hostname: 'github.com'
+		});
+		assert.ok(repoId);
+		assert.deepStrictEqual(
+			{ org: repoId.org, repo: repoId.repo, type: repoId.type },
+			{ org: 'microsoft', repo: 'vscode-copilot-chat', type: 'github' }
+		);
+		assert.ok(
+			[
+				'git@github.com:microsoft/vscode-copilot-chat',
+				'https://github.com/microsoft/vscode-copilot-chat',
+				'https://github.com/microsoft/vscode-copilot-chat.git',
+			].includes(url),
+			`url is ${url}`
+		);
+		assert.ok(pathname.startsWith('/github/vscode-copilot-chat') || pathname.startsWith('/microsoft/vscode-copilot-chat'));
+
+		assert.deepStrictEqual(await extractRepoInfo(ctx, 'file:///tmp/does/not/exist/.git/config'), undefined);
+	});
+
+	test('Extract repo info - Jupyter Notebook vscode-notebook-cell ', async function () {
+		const cellUri = baseFolder.uri.replace(/^file:/, 'vscode-notebook-cell:');
+		assert.ok(cellUri.startsWith('vscode-notebook-cell:'));
+		const ctx = createLibTestingContext();
+		const info = await extractRepoInfo(ctx, cellUri);
+
+		assert.ok(info);
+
+		// url and pathname get their own special treatment because they depend on how the repo was cloned.
+		const { url, pathname, repoId, ...repoInfo } = info;
+
+		assert.deepStrictEqual(repoInfo, {
+			baseFolder,
+			hostname: 'github.com'
+		});
+		assert.ok(repoId);
+		assert.deepStrictEqual(
+			{ org: repoId.org, repo: repoId.repo, type: repoId.type },
+			{ org: 'microsoft', repo: 'vscode-copilot-chat', type: 'github' }
+		);
+		assert.ok(
+			[
+				'git@github.com:microsoft/vscode-copilot-chat',
+				'https://github.com/microsoft/vscode-copilot-chat',
+				'https://github.com/microsoft/vscode-copilot-chat.git',
+			].includes(url),
+			`url is ${url}`
+		);
+		assert.ok(pathname.startsWith('/github/vscode-copilot-chat') || pathname.startsWith('/microsoft/vscode-copilot-chat'));
+
+		assert.deepStrictEqual(await extractRepoInfo(ctx, 'file:///tmp/does/not/exist/.git/config'), undefined);
+	});
+});

extensions/copilot/src/extension/completions-core/vscode-node/prompt/src/test/indentation.test.ts

@@ -40,11 +40,11 @@ function doParseTest<T>(source: string, expectedTree: IndentationTree<T>) {
 
 const SOURCE = {
 	source: dedent`
-	f1:
-		a1
-	f2:
-		a2
-		a3
+f1:
+    a1
+f2:
+    a2
+    a3
 `,
 	name: '',
 };
@@ -52,79 +52,79 @@ const SOURCE = {
 suite('Test compareTreeWithSpec', function () {
 	const SOURCE_MISSING_CHILD = {
 		source: dedent`
-		f1:
-			a1
-		f2:
-			a2
-	`,
+f1:
+    a1
+f2:
+    a2
+`,
 		name: 'missing child',
 	};
 
 	const SOURCE_EXTRA_CHILD = {
 		source: dedent`
-		f1:
-			a1
-		f2:
-			a2
-			a3
-			a4
-	`,
+f1:
+    a1
+f2:
+    a2
+    a3
+    a4
+`,
 		name: 'extra_child',
 	};
 
 	const SOURCE_MISSING_SIBLING = {
 		source: dedent`
-		f1:
-			a1
-	`,
+f1:
+    a1
+`,
 		name: 'missing sibling',
 	};
 
 	const SOURCE_EXTRA_SIBLING = {
 		source: dedent`
-		f1:
-			a1
-		f2:
-			a2
-			a3
-		f3:
-			a4
-	`,
+f1:
+    a1
+f2:
+    a2
+    a3
+f3:
+    a4
+`,
 		name: 'extra_sibling',
 	};
 
 	const SOURCE_EXTRA_MIDDLE_BLANK_LINE = {
 		source: dedent`
-		f1:
-			a1
+f1:
+    a1
 
-		f2:
-			a2
-			a3
-	`,
+f2:
+    a2
+    a3
+`,
 		name: 'extra middle blank line',
 	};
 
 	const SOURCE_EXTRA_TRAILING_BLANK_LINE = {
 		source: dedent`
-		f1:
-			a1
-		f2:
-			a2
-			a3
+f1:
+    a1
+f2:
+    a2
+    a3
 
-	`,
+`,
 		name: 'extra trailing blank line',
 	};
 
 	const SOURCE_EXTRA_INDENTATION = {
 		source: dedent`
-		f1:
-				a1
-		f2:
-			a2
-				a3
-	`,
+f1:
+        a1
+f2:
+    a2
+        a3
+`,
 		name: 'extra indentation',
 	};
 
@@ -334,15 +334,15 @@ suite('Test core functions: other', function () {
 	});
 	test('deparseAndCutTree cuts at labels', function () {
 		const source = dedent`
-			1
-				2
-				3
-			4
-				5
-				6
-			7
-				8
-				9`;
+1
+    2
+    3
+4
+    5
+    6
+7
+    8
+    9`;
 		const tree = parseRaw(source) as IndentationTree<string>;
 		tree.subs[0].subs[1].label = 'cut';
 		tree.subs[1].subs[0].label = 'cut';
@@ -390,15 +390,15 @@ suite('Test core functions: other', function () {
 	});
 	test('VisitTreeConditionally', function () {
 		const tree = parseRaw(dedent`
-			1
-				2
-				3
-			4
-				5
-				6
-			7
-				8
-				9`);
+1
+    2
+    3
+4
+    5
+    6
+7
+    8
+    9`);
 		const traceTopDownAll: string[] = [];
 		visitTree(
 			tree,

extensions/copilot/src/extension/completions-core/vscode-node/prompt/src/test/indentationLanguages.test.ts

@@ -2,10 +2,10 @@
  *  Copyright (c) Microsoft Corporation. All rights reserved.
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
-import { blankNode, isLine, lineNode, parseTree, topNode, virtualNode, visitTree } from '../indentation';
 import * as assert from 'assert';
 import { suite } from 'mocha';
 import dedent from 'ts-dedent';
+import { blankNode, isLine, lineNode, parseTree, topNode, virtualNode, visitTree } from '../indentation';
 import { compareTreeWithSpec } from './testHelpers';
 
 /** Test some language specific parsing techniques */
@@ -57,23 +57,23 @@ suite('Java', function () {
 	test('labelLines java', function () {
 		const tree = parseTree(
 			dedent`
-			package com.example;
-			import java.awt.*;
-			@annotation
-			final public class A {
-				/** A javadoc
-				 *  Second line
-				 */
-				public static void main(String[] args) {
-					// single-line comment
-					/* Multiline
-					 * comment
-					 */
-					System.out.println("Hello, world!");
-				}
-			}
-			public interface I { }
-			`,
+package com.example;
+import java.awt.*;
+@annotation
+final public class A {
+    /** A javadoc
+     *  Second line
+     */
+    public static void main(String[] args) {
+        // single-line comment
+        /* Multiline
+         * comment
+         */
+        System.out.println("Hello, world!");
+    }
+}
+public interface I { }
+`,
 			'java'
 		);
 		compareTreeWithSpec(
@@ -113,14 +113,14 @@ suite('Java', function () {
 		//TODO: Add a field with annotation on separate line
 		const tree = parseTree(
 			dedent`
-			class A {
-				int a;
-				/** Javadoc */
-				int b;
-				// Comment
-				@Native int c;
-			}
-			`,
+class A {
+    int a;
+    /** Javadoc */
+    int b;
+    // Comment
+    @Native int c;
+}
+`,
 			'java'
 		);
 		compareTreeWithSpec(
@@ -147,18 +147,18 @@ suite('Java', function () {
 	test('parse Java inner class', function () {
 		const tree = parseTree(
 			dedent`
-			class A {
-				int a;
+class A {
+    int a;
 
-				class Inner {
-					int b;
-				}
+    class Inner {
+        int b;
+    }
 
-				interface InnerInterface {
-					int myMethod();
-				}
-			}
-			`,
+    interface InnerInterface {
+        int myMethod();
+    }
+}
+`,
 			'java'
 		);
 		compareTreeWithSpec(
@@ -198,25 +198,25 @@ suite('Java', function () {
 suite('Markdown', function () {
 	test('header processing in markdown', function () {
 		const source = dedent`
-		A
+A
 
-		# B
-		C
-		D
+# B
+C
+D
 
-		## E
-		F
-		G
+## E
+F
+G
 
-		# H
-		I
+# H
+I
 
-		### J
-		K
+### J
+K
 
-		L
-		M
-		`;
+L
+M
+`;
 		const mdParsedTree = parseTree(source, 'markdown');
 
 		compareTreeWithSpec(

extensions/copilot/src/extension/completions-core/vscode-node/prompt/src/test/indentationParsing.test.ts

@@ -68,17 +68,17 @@ suite('Test core parsing elements', function () {
 
 	test('groupBlocks basic cases', function () {
 		const source = dedent`
-			A
+A
 
-			B
-			C
-			D
+B
+C
+D
 
-			E
-			F
+E
+F
 
-			G
-			H`;
+G
+H`;
 		const tree = parseRaw(source);
 		const blockTree = groupBlocks(tree);
 		function assertChildrenAreTheFollowingLines(
@@ -104,24 +104,24 @@ suite('Test core parsing elements', function () {
 		// blank lines after last child, lone blank lines,
 		// consecutive lone blank lines, offside blocks
 		let tree = parseRaw(dedent`
-			A
+A
 
-			  B
-			  C
-				D
+  B
+  C
+    D
 
-			  E
+  E
 
 
-			  F
+  F
 
-			G
-				H
-				I
-			  J
+G
+    H
+    I
+  J
 
-			  K
-			`);
+  K
+`);
 		tree = groupBlocks(tree);
 		compareTreeWithSpec(
 			tree,
@@ -156,13 +156,13 @@ suite('Test core parsing elements', function () {
 
 	test('groupBlocks consecutive blanks as oldest children', function () {
 		let tree = parseRaw(dedent`
-		A
+A
 
 
-			B1
-			B2
-		C
-		`);
+    B1
+    B2
+C
+`);
 		tree = groupBlocks(tree);
 		compareTreeWithSpec(
 			tree,
@@ -200,12 +200,12 @@ suite('Test core parsing elements', function () {
 
 	test('groupBlocks with different delimiter', function () {
 		let tree = parseRaw(dedent`
-			A
-			B
-			C
-			D
-			E
-			`) as IndentationTree<string>;
+A
+B
+C
+D
+E
+`) as IndentationTree<string>;
 		const isDelimiter = (node: IndentationTree<string>) =>
 			isLine(node) && (node.sourceLine.trim() === 'B' || node.sourceLine.trim() === 'D');
 		tree = groupBlocks(tree, isDelimiter);
@@ -224,19 +224,19 @@ suite('Raw parsing', function () {
 	test('parseRaw', function () {
 		compareTreeWithSpec(
 			parseRaw(dedent`
-			A
-			  a
-			B
-			  b1
-			  b2
-			C
-				c1
-				c2
-			  c3
-			D
-			  d1
-				d2
-			`),
+A
+  a
+B
+  b1
+  b2
+C
+    c1
+    c2
+  c3
+D
+  d1
+    d2
+`),
 			topNode([
 				lineNode(0, 0, 'A', [lineNode(2, 1, 'a', [])]),
 				lineNode(0, 2, 'B', [lineNode(2, 3, 'b1', []), lineNode(2, 4, 'b2', [])]),
@@ -249,19 +249,19 @@ suite('Raw parsing', function () {
 	test('parseRaw blanks', function () {
 		compareTreeWithSpec(
 			parseRaw(dedent`
-			E
-			  e1
+E
+  e1
 
-			  e2
-			F
+  e2
+F
 
-			  f1
-			G
-			  g1
+  f1
+G
+  g1
 
-			H
+H
 
-			`),
+`),
 			topNode([
 				lineNode(0, 0, 'E', [lineNode(2, 1, 'e1', []), blankNode(2), lineNode(2, 3, 'e2', [])]),
 				lineNode(0, 4, 'F', [blankNode(5), lineNode(2, 6, 'f1', [])]),
@@ -275,24 +275,24 @@ suite('Raw parsing', function () {
 
 	test('combineBraces', function () {
 		const tree = parseTree(dedent`
-			A {
-			}
-			B
-			  b1 {
-				bb1
-			  }
-			  b2 {
-				bb2
+A {
+}
+B
+  b1 {
+    bb1
+  }
+  b2 {
+    bb2
 
-			  }
-			}
-			C {
-				c1
-				c2
-			  c3
-			  c4
-			}
-			`);
+  }
+}
+C {
+    c1
+    c2
+  c3
+  c4
+}
+`);
 		compareTreeWithSpec(
 			tree,
 			topNode([
@@ -329,10 +329,10 @@ suite('Raw parsing', function () {
 suite('Test bracket indentation spec', function () {
 	test('Opener merged to older sibling', function () {
 		const source = dedent`
-		A
-		(
-			B
-			C`;
+A
+(
+    B
+    C`;
 		const treeRaw = parseRaw(source);
 		const treeCode = parseTree(source, '');
 
@@ -357,9 +357,9 @@ suite('Test bracket indentation spec', function () {
 
 	test('Closer merged, simplest case', function () {
 		const source = dedent`
-		A
-			B
-		)`;
+A
+    B
+)`;
 		const treeRaw = parseRaw(source);
 		const treeCode = parseTree(source, '');
 
@@ -378,13 +378,13 @@ suite('Test bracket indentation spec', function () {
 
 	test('Closer merged, multi-body case', function () {
 		const source = dedent`
-		A
-			B
-			C
-		) + (
-			D
-			E
-		)`;
+A
+    B
+    C
+) + (
+    D
+    E
+)`;
 		const treeRaw = parseRaw(source);
 		const treeCode = parseTree(source, '');
 
@@ -402,20 +402,20 @@ suite('Test bracket indentation spec', function () {
 
 	test('closer starting their next subblock, ifelse', function () {
 		const source = dedent`
-			if (new) {
-				print(“hello”)
-				print(“world”)
-			} else {
-				print(“goodbye”)
-			}`;
+            if (new) {
+                print(“hello”)
+                print(“world”)
+            } else {
+                print(“goodbye”)
+            }`;
 		const sourceParsedAsIf = dedent`
-			if (new) {
-				-> virtual
-					print(“hello”)
-					print(“world”)
-				} else {
-					print(“goodbye”)
-				}`;
+            if (new) {
+                -> virtual
+                    print(“hello”)
+                    print(“world”)
+                } else {
+                    print(“goodbye”)
+                }`;
 
 		const treeRaw = parseRaw(source);
 		const treeCode = parseTree(source, '');
@@ -425,10 +425,10 @@ suite('Test bracket indentation spec', function () {
 			treeRaw,
 			topNode([
 				lineNode(0, 0, 'if (new) {', [
-					lineNode(4, 1, 'print(“hello”)', []),
-					lineNode(4, 2, 'print(“world”)', []),
+					lineNode(4, 1, 'print(“hello”)', []),
+					lineNode(4, 2, 'print(“world”)', []),
 				]),
-				lineNode(0, 3, '} else {', [lineNode(4, 4, 'print(“goodbye”)', [])]),
+				lineNode(0, 3, '} else {', [lineNode(4, 4, 'print(“goodbye”)', [])]),
 				lineNode(0, 5, '}', []),
 			])
 		);
@@ -436,8 +436,8 @@ suite('Test bracket indentation spec', function () {
 			treeCode,
 			topNode([
 				lineNode(0, 0, 'if (new) {', [
-					virtualNode(0, [lineNode(4, 1, 'print(“hello”)', []), lineNode(4, 2, 'print(“world”)', [])]),
-					lineNode(0, 3, '} else {', [lineNode(4, 4, 'print(“goodbye”)', [])]),
+					virtualNode(0, [lineNode(4, 1, 'print(“hello”)', []), lineNode(4, 2, 'print(“world”)', [])]),
+					lineNode(0, 3, '} else {', [lineNode(4, 4, 'print(“goodbye”)', [])]),
 					lineNode(0, 5, '}', []),
 				]),
 			])
@@ -449,11 +449,11 @@ suite('Test bracket indentation spec', function () {
 suite('Special indentation styles', function () {
 	test('Allman style example (function)', function () {
 		const source = dedent`
-		function test()
-		{
-			print(“hello”)
-			print(“world”)
-		}`;
+        function test()
+        {
+            print(“hello”)
+            print(“world”)
+        }`;
 
 		const treeRaw = parseRaw(source);
 		const treeCode = parseTree(source, '');
@@ -464,8 +464,8 @@ suite('Special indentation styles', function () {
 			topNode([
 				lineNode(0, 0, 'function test()', [
 					lineNode(0, 1, '{', [], 'opener'),
-					lineNode(4, 2, 'print(“hello”)', []),
-					lineNode(4, 3, 'print(“world”)', []),
+					lineNode(4, 2, 'print(“hello”)', []),
+					lineNode(4, 3, 'print(“world”)', []),
 					lineNode(0, 4, '}', [], 'closer'),
 				]),
 			])
@@ -476,7 +476,7 @@ suite('Special indentation styles', function () {
 			treeRaw,
 			topNode([
 				lineNode(0, 0, 'function test()', []),
-				lineNode(0, 1, '{', [lineNode(4, 2, 'print(“hello”)', []), lineNode(4, 3, 'print(“world”)', [])]),
+				lineNode(0, 1, '{', [lineNode(4, 2, 'print(“hello”)', []), lineNode(4, 3, 'print(“world”)', [])]),
 				lineNode(0, 4, '}', []),
 			])
 		);
@@ -485,17 +485,17 @@ suite('Special indentation styles', function () {
 	/** This test is a case where our parsing isn't yet optimal */
 	test('Allman style example (if-then-else)', function () {
 		const source = dedent`
-		if (condition)
-		{
-			print(“hello”)
-			print(“world”)
-		}
-		else
-		{
-			print(“goodbye”)
-			print(“phone”)
-		}
-		`;
+        if (condition)
+        {
+            print(“hello”)
+            print(“world”)
+        }
+        else
+        {
+            print(“goodbye”)
+            print(“phone”)
+        }
+        `;
 
 		const treeCode = parseTree(source, '');
 
@@ -506,14 +506,14 @@ suite('Special indentation styles', function () {
 			topNode([
 				lineNode(0, 0, 'if (condition)', [
 					lineNode(0, 1, '{', [], 'opener'),
-					lineNode(4, 2, 'print(“hello”)', []),
-					lineNode(4, 3, 'print(“world”)', []),
+					lineNode(4, 2, 'print(“hello”)', []),
+					lineNode(4, 3, 'print(“world”)', []),
 					lineNode(0, 4, '}', [], 'closer'),
 				]),
 				lineNode(0, 5, 'else ', [
 					lineNode(0, 6, '{', [], 'opener'),
-					lineNode(4, 7, 'print(“goodbye”)', []),
-					lineNode(4, 8, 'print(“phone”)', []),
+					lineNode(4, 7, 'print(“goodbye”)', []),
+					lineNode(4, 8, 'print(“phone”)', []),
 					lineNode(0, 9, '}', [], 'closer'),
 				]),
 			])
@@ -522,14 +522,14 @@ suite('Special indentation styles', function () {
 
 	test('K&R style example (if-then-else)', function () {
 		const source = dedent`
-		if (condition) {
-			print(“hello”)
-			print(“world”)
-		} else {
-			print(“goodbye”)
-			print(“phone”)
-		}
-		`;
+        if (condition) {
+            print(“hello”)
+            print(“world”)
+        } else {
+            print(“goodbye”)
+            print(“phone”)
+        }
+        `;
 
 		const treeCode = parseTree(source, '');
 
@@ -539,12 +539,12 @@ suite('Special indentation styles', function () {
 			treeCode,
 			topNode([
 				lineNode(0, 0, 'if (condition) {', [
-					virtualNode(0, [lineNode(4, 2, 'print(“hello”)', []), lineNode(4, 3, 'print(“world”)', [])]),
+					virtualNode(0, [lineNode(4, 2, 'print(“hello”)', []), lineNode(4, 3, 'print(“world”)', [])]),
 					lineNode(
 						0,
 						4,
 						'} else {',
-						[lineNode(4, 5, 'print(“goodbye”)', []), lineNode(4, 6, 'print(“phone”)', [])],
+						[lineNode(4, 5, 'print(“goodbye”)', []), lineNode(4, 6, 'print(“phone”)', [])],
 						'closer'
 					),
 					lineNode(0, 7, '}', [], 'closer'),
@@ -555,11 +555,11 @@ suite('Special indentation styles', function () {
 
 	test('combineBraces GNU style indentation 1', function () {
 		let tree: IndentationTree<string> = parseRaw(dedent`
-			A
-			  {
-				stmt
-			  }
-			`);
+A
+  {
+    stmt
+  }
+`);
 		labelLines(tree, buildLabelRules({ opener: /^{$/, closer: /^}$/ }));
 		tree = combineClosersAndOpeners(tree);
 		compareTreeWithSpec(
@@ -574,15 +574,15 @@ suite('Special indentation styles', function () {
 
 	test('combineBraces GNU style indentation 2', function () {
 		let tree: IndentationTree<string> = parseRaw(dedent`
-				B
-				{
-					stmt
+B
+{
+    stmt
 
-				}
+}
 
 
-				end
-				`);
+end
+`);
 		labelLines(tree, buildLabelRules({ opener: /^{$/, closer: /^}$/ }));
 		tree = combineClosersAndOpeners(tree);
 		tree = flattenVirtual(tree);
@@ -604,11 +604,11 @@ suite('Special indentation styles', function () {
 
 	test('combineBraces GNU style indentation 3', function () {
 		let tree: IndentationTree<string> = parseRaw(dedent`
-				C
-				{
+C
+{
 
-				}
-				`);
+}
+`);
 		labelLines(tree, buildLabelRules({ opener: /^{$/, closer: /^}$/ }));
 		tree = combineClosersAndOpeners(tree);
 		tree = flattenVirtual(tree);
@@ -626,15 +626,15 @@ suite('Special indentation styles', function () {
 
 	test('combineBraces GNU style indentation 4', function () {
 		let tree: IndentationTree<string> = parseRaw(dedent`
-				D
-				{
-					d
-					{
-						stmt
+D
+{
+    d
+    {
+        stmt
 
-					}
-				}
-				`);
+    }
+}
+`);
 		labelLines(tree, buildLabelRules({ opener: /^{$/, closer: /^}$/ }));
 		tree = combineClosersAndOpeners(tree);
 		tree = flattenVirtual(tree);

extensions/copilot/src/extension/completions-core/vscode-node/prompt/src/test/tokenizer.test.ts

@@ -2,11 +2,11 @@
  *  Copyright (c) Microsoft Corporation. All rights reserved.
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
-import { ApproximateTokenizer, getTokenizer, TokenizerName } from '../tokenization';
 import * as assert from 'assert';
 import * as fs from 'fs';
 import { suite } from 'mocha';
 import { resolve } from 'path';
+import { ApproximateTokenizer, getTokenizer, TokenizerName } from '../tokenization';
 
 // Read the source files and normalize the line endings
 const source = fs.readFileSync(resolve(__dirname, 'testdata/example.py'), 'utf8').replace(/\r\n?/g, '\n');
@@ -96,7 +96,7 @@ suite('Tokenizer Test Suite - cl100k', function () {
 	});
 
 	test('emojis', function () {
-		const str = 'hello 👋 world 🌍';
+		const str = 'hello 👋 world 🌍';
 		assert.deepStrictEqual(tokenizer.tokenize(str), [15339, 62904, 233, 1917, 11410, 234, 235]);
 		assert.strictEqual(tokenizer.detokenize(tokenizer.tokenize(str)), str);
 	});
@@ -119,9 +119,9 @@ suite('Tokenizer Test Suite - cl100k', function () {
 		}
 
 		// Test special characters
-		assert.deepStrictEqual(tokenizer.tokenize('\n\n👋'), [271, 9468, 239, 233]);
+		assert.deepStrictEqual(tokenizer.tokenize('\n\n👋'), [271, 9468, 239, 233]);
 		assert.deepStrictEqual(tokenizer.tokenize('\n\n '), [271, 220]);
-		assert.deepStrictEqual(tokenizer.tokenize('\n\n 👋'), [271, 62904, 233]);
+		assert.deepStrictEqual(tokenizer.tokenize('\n\n 👋'), [271, 62904, 233]);
 		assert.deepStrictEqual(tokenizer.tokenize('\n\n\t'), [271, 197]);
 		assert.deepStrictEqual(tokenizer.tokenize('\n\n\r'), [271, 201]);
 
@@ -296,7 +296,7 @@ suite('Tokenizer Test Suite - o200k', function () {
 	});
 
 	test('emojis', function () {
-		const str = 'hello 👋 world 🌍';
+		const str = 'hello 👋 world 🌍';
 		assert.deepStrictEqual(tokenizer.tokenize(str), [24912, 61138, 233, 2375, 130321, 235]);
 		assert.strictEqual(tokenizer.detokenize(tokenizer.tokenize(str)), str);
 	});
@@ -319,9 +319,9 @@ suite('Tokenizer Test Suite - o200k', function () {
 		}
 
 		// Test special characters
-		assert.deepStrictEqual(tokenizer.tokenize('\n\n👋'), [279, 28823, 233]);
+		assert.deepStrictEqual(tokenizer.tokenize('\n\n👋'), [279, 28823, 233]);
 		assert.deepStrictEqual(tokenizer.tokenize('\n\n '), [279, 220]);
-		assert.deepStrictEqual(tokenizer.tokenize('\n\n 👋'), [279, 61138, 233]);
+		assert.deepStrictEqual(tokenizer.tokenize('\n\n 👋'), [279, 61138, 233]);
 		assert.deepStrictEqual(tokenizer.tokenize('\n\n\t'), [279, 197]);
 		assert.deepStrictEqual(tokenizer.tokenize('\n\n\r'), [279, 201]);