testing: add test configurations api

src/vs/workbench/api/browser/mainThreadTesting.ts

@@ -5,14 +5,15 @@
 
 import { VSBuffer } from 'vs/base/common/buffer';
 import { CancellationToken } from 'vs/base/common/cancellation';
-import { Disposable, IDisposable, MutableDisposable } from 'vs/base/common/lifecycle';
+import { Disposable, DisposableStore, IDisposable, MutableDisposable, toDisposable } from 'vs/base/common/lifecycle';
 import { isDefined } from 'vs/base/common/types';
 import { URI } from 'vs/base/common/uri';
 import { Range } from 'vs/editor/common/core/range';
 import { extHostNamedCustomer } from 'vs/workbench/api/common/extHostCustomers';
 import { TestResultState } from 'vs/workbench/api/common/extHostTypes';
 import { MutableObservableValue } from 'vs/workbench/contrib/testing/common/observableValue';
-import { ExtensionRunTestsRequest, ITestItem, ITestMessage, ITestRunTask, RunTestsRequest, TestDiffOpType, TestsDiff } from 'vs/workbench/contrib/testing/common/testCollection';
+import { ExtensionRunTestsRequest, ITestItem, ITestMessage, ITestRunConfiguration, ITestRunTask, ResolvedTestRunRequest, TestDiffOpType, TestsDiff } from 'vs/workbench/contrib/testing/common/testCollection';
+import { ITestConfigurationService } from 'vs/workbench/contrib/testing/common/testConfigurationService';
 import { TestCoverage } from 'vs/workbench/contrib/testing/common/testCoverage';
 import { LiveTestResult } from 'vs/workbench/contrib/testing/common/testResult';
 import { ITestResultService } from 'vs/workbench/contrib/testing/common/testResultService';
@@ -42,6 +43,7 @@ export class MainThreadTesting extends Disposable implements MainThreadTestingSh
 	constructor(
 		extHostContext: IExtHostContext,
 		@ITestService private readonly testService: ITestService,
+		@ITestConfigurationService private readonly testConfiguration: ITestConfigurationService,
 		@ITestResultService private readonly resultService: ITestResultService,
 	) {
 		super();
@@ -65,6 +67,27 @@ export class MainThreadTesting extends Disposable implements MainThreadTestingSh
 		}));
 	}
 
+	/**
+	 * @inheritdoc
+	 */
+	$publishTestRunConfig(config: ITestRunConfiguration): void {
+		this.testConfiguration.addConfiguration(config);
+	}
+
+	/**
+	 * @inheritdoc
+	 */
+	$updateTestRunConfig(controllerId: string, configId: number, update: Partial<ITestRunConfiguration>): void {
+		this.testConfiguration.updateConfiguration(controllerId, configId, update);
+	}
+
+	/**
+	 * @inheritdoc
+	 */
+	$removeTestRunConfig(controllerId: string, configId: number): void {
+		this.testConfiguration.removeConfiguration(controllerId, configId);
+	}
+
 	/**
 	 * @inheritdoc
 	 */
@@ -158,10 +181,13 @@ export class MainThreadTesting extends Disposable implements MainThreadTestingSh
 	 * @inheritdoc
 	 */
 	public $registerTestController(controllerId: string) {
-		const disposable = this.testService.registerTestController(controllerId, {
+		const disposable = new DisposableStore();
+
+		disposable.add(toDisposable(() => this.testConfiguration.removeConfiguration(controllerId)));
+		disposable.add(this.testService.registerTestController(controllerId, {
 			runTests: (req, token) => this.proxy.$runControllerTests(req, token),
 			expandTest: (src, levels) => this.proxy.$expandTest(src, isFinite(levels) ? levels : -1),
-		});
+		}));
 
 		this.testProviderRegistrations.set(controllerId, disposable);
 	}
@@ -169,9 +195,9 @@ export class MainThreadTesting extends Disposable implements MainThreadTestingSh
 	/**
 	 * @inheritdoc
 	 */
-	public $unregisterTestController(id: string) {
-		this.testProviderRegistrations.get(id)?.dispose();
-		this.testProviderRegistrations.delete(id);
+	public $unregisterTestController(controllerId: string) {
+		this.testProviderRegistrations.get(controllerId)?.dispose();
+		this.testProviderRegistrations.delete(controllerId);
 	}
 
 	/**
@@ -197,8 +223,8 @@ export class MainThreadTesting extends Disposable implements MainThreadTestingSh
 		this.testService.publishDiff(controllerId, diff);
 	}
 
-	public async $runTests(req: RunTestsRequest, token: CancellationToken): Promise<string> {
-		const result = await this.testService.runTests(req, token);
+	public async $runTests(req: ResolvedTestRunRequest, token: CancellationToken): Promise<string> {
+		const result = await this.testService.runResolvedTests(req, token);
 		return result.id;
 	}
 

src/vs/workbench/api/common/extHost.api.impl.ts

@@ -1261,6 +1261,7 @@ export function createApiFactoryAndRegisterActors(accessor: ServicesAccessor): I
 			TestResultState: extHostTypes.TestResultState,
 			TestRunRequest: extHostTypes.TestRunRequest,
 			TestMessage: extHostTypes.TestMessage,
+			TestRunConfigurationGroup: extHostTypes.TestRunConfigurationGroup,
 			TextSearchCompleteMessageType: TextSearchCompleteMessageType,
 			TestMessageSeverity: extHostTypes.TestMessageSeverity,
 			CoveredCount: extHostTypes.CoveredCount,

src/vs/workbench/api/common/extHost.protocol.ts

@@ -57,7 +57,7 @@ import { ICellRange } from 'vs/workbench/contrib/notebook/common/notebookRange';
 import { InputValidationType } from 'vs/workbench/contrib/scm/common/scm';
 import { ITextQueryBuilderOptions } from 'vs/workbench/contrib/search/common/queryBuilder';
 import { ISerializableEnvironmentVariableCollection } from 'vs/workbench/contrib/terminal/common/environmentVariable';
-import { ExtensionRunTestsRequest, ISerializedTestResults, ITestItem, ITestMessage, ITestRunTask, RunTestForControllerRequest, RunTestsRequest, ITestIdWithSrc, TestsDiff, IFileCoverage, CoverageDetails } from 'vs/workbench/contrib/testing/common/testCollection';
+import { ExtensionRunTestsRequest, ISerializedTestResults, ITestItem, ITestMessage, ITestRunTask, RunTestForControllerRequest, ResolvedTestRunRequest, ITestIdWithSrc, TestsDiff, IFileCoverage, CoverageDetails, ITestRunConfiguration } from 'vs/workbench/contrib/testing/common/testCollection';
 import { InternalTimelineOptions, Timeline, TimelineChangeEvent, TimelineOptions, TimelineProviderDescriptor } from 'vs/workbench/contrib/timeline/common/timeline';
 import { EditorGroupColumn } from 'vs/workbench/services/editor/common/editorGroupColumn';
 import { ActivationKind, ExtensionHostKind, MissingExtensionDependency } from 'vs/workbench/services/extensions/common/extensions';
@@ -2090,6 +2090,8 @@ export interface ExtHostTestingShape {
 }
 
 export interface MainThreadTestingShape {
+	// --- test lifecycle:
+
 	/** Registeres that there's a test controller with the given ID */
 	$registerTestController(controllerId: string): void;
 	/** Diposes of the test controller with the given ID */
@@ -2100,11 +2102,21 @@ export interface MainThreadTestingShape {
 	$unsubscribeFromDiffs(): void;
 	/** Publishes that new tests were available on the given source. */
 	$publishDiff(controllerId: string, diff: TestsDiff): void;
-	/** Request by an extension to run tests. */
-	$runTests(req: RunTestsRequest, token: CancellationToken): Promise<string>;
+
+	// --- test run configurations:
+
+	/** Called when a new test run configuration is available */
+	$publishTestRunConfig(config: ITestRunConfiguration): void;
+	/** Updates an existing test run configuration */
+	$updateTestRunConfig(controllerId: string, configId: number, update: Partial<ITestRunConfiguration>): void;
+	/** Removes a previously-published test run config */
+	$removeTestRunConfig(controllerId: string, configId: number): void;
+
 
 	// --- test run handling:
 
+	/** Request by an extension to run tests. */
+	$runTests(req: ResolvedTestRunRequest, token: CancellationToken): Promise<string>;
 	/**
 	 * Adds tests to the run. The tests are given in descending depth. The first
 	 * item will be a previously-known test, or a test root.

src/vs/workbench/api/common/extHostTesting.ts

@@ -9,6 +9,7 @@ import { VSBuffer } from 'vs/base/common/buffer';
 import { CancellationToken, CancellationTokenSource } from 'vs/base/common/cancellation';
 import { Emitter, Event } from 'vs/base/common/event';
 import { once } from 'vs/base/common/functional';
+import { hash } from 'vs/base/common/hash';
 import { Iterable } from 'vs/base/common/iterator';
 import { Disposable, DisposableStore, toDisposable } from 'vs/base/common/lifecycle';
 import { MarshalledId } from 'vs/base/common/marshalling';
@@ -19,15 +20,16 @@ import { ExtHostTestingShape, MainContext, MainThreadTestingShape } from 'vs/wor
 import { ExtHostCommands } from 'vs/workbench/api/common/extHostCommands';
 import { IExtHostRpcService } from 'vs/workbench/api/common/extHostRpcService';
 import * as Convert from 'vs/workbench/api/common/extHostTypeConverters';
-import { TestItemImpl } from 'vs/workbench/api/common/extHostTypes';
+import { TestItemImpl, TestRunConfigurationGroup, TestRunRequest } from 'vs/workbench/api/common/extHostTypes';
 import { SingleUseTestCollection, TestPosition } from 'vs/workbench/contrib/testing/common/ownedTestCollection';
-import { AbstractIncrementalTestCollection, CoverageDetails, IFileCoverage, IncrementalChangeCollector, IncrementalTestCollectionItem, InternalTestItem, ISerializedTestResults, ITestIdWithSrc, ITestItem, RunTestForControllerRequest, TestsDiff } from 'vs/workbench/contrib/testing/common/testCollection';
+import { AbstractIncrementalTestCollection, CoverageDetails, IFileCoverage, IncrementalChangeCollector, IncrementalTestCollectionItem, InternalTestItem, ISerializedTestResults, ITestIdWithSrc, ITestItem, RunTestForControllerRequest, TestRunConfigurationBitset, TestsDiff } from 'vs/workbench/contrib/testing/common/testCollection';
 import type * as vscode from 'vscode';
 
 export class ExtHostTesting implements ExtHostTestingShape {
 	private readonly resultsChangedEmitter = new Emitter<void>();
 	private readonly controllers = new Map</* controller ID */ string, {
 		controller: vscode.TestController,
+		configurations: Map<number, vscode.TestRunConfiguration>,
 		collection: SingleUseTestCollection,
 	}>();
 	private readonly proxy: MainThreadTestingShape;
@@ -55,12 +57,25 @@ export class ExtHostTesting implements ExtHostTestingShape {
 		const disposable = new DisposableStore();
 		const collection = disposable.add(new SingleUseTestCollection(controllerId));
 		const initialExpand = disposable.add(new RunOnceScheduler(() => collection.expand(collection.root.id, 0), 0));
+		const configurations = new Map<number, vscode.TestRunConfiguration>();
 
 		const controller: vscode.TestController = {
 			root: collection.root,
 			get id() {
 				return controllerId;
 			},
+			createRunConfiguration: (label, group, runHandler, isDefault) => {
+				// Derive the config ID from a hash so that the same config will tend
+				// to have the same hashes, allowing re-run requests to work across reloads.
+				let configId = hash(label);
+				while (configurations.has(configId)) {
+					configId++;
+				}
+
+				const config = new TestRunConfigurationImpl(this.proxy, controllerId, configId, label, group, runHandler, isDefault);
+				configurations.set(configId, config);
+				return config;
+			},
 			createTestRun: (request, name, persist = true) => {
 				return this.runTracker.createTestRun(controllerId, request, name, persist);
 			},
@@ -88,7 +103,7 @@ export class ExtHostTesting implements ExtHostTestingShape {
 		this.proxy.$registerTestController(controllerId);
 		disposable.add(toDisposable(() => this.proxy.$unregisterTestController(controllerId)));
 
-		this.controllers.set(controllerId, { controller, collection });
+		this.controllers.set(controllerId, { controller, collection, configurations });
 		disposable.add(toDisposable(() => this.controllers.delete(controllerId)));
 
 		disposable.add(collection.onDidGenerateDiff(diff => this.proxy.$publishDiff(controllerId, diff)));
@@ -108,16 +123,32 @@ export class ExtHostTesting implements ExtHostTestingShape {
 	 * Implements vscode.test.runTests
 	 */
 	public async runTests(req: vscode.TestRunRequest, token = CancellationToken.None) {
+		const config = tryGetConfigFromTestRunReq(req);
+		if (!config) {
+			throw new Error('The request passed to `vscode.test.runTests` must include a configuration');
+		}
+
+		if (!req.tests.length) {
+			return;
+		}
+
 		const testListToProviders = (tests: ReadonlyArray<vscode.TestItem>) =>
 			tests
 				.map(this.getInternalTestForReference, this)
 				.filter(isDefined)
-				.map(t => ({ controllerId: t.controllerId, testId: t.item.extId }));
+				.map(t => ({ controllerId: t.controllerId, testId: t.item.extId, configId: config }));
 
 		await this.proxy.$runTests({
-			exclude: req.exclude ? testListToProviders(req.exclude).map(t => t.testId) : undefined,
-			tests: testListToProviders(req.tests),
-			debug: req.debug
+			targets: [{
+				testIds: req.tests.map(t => t.id),
+				configGroup: configGroupToBitset[config.group],
+				configLabel: config.label,
+				configId: config.configId,
+				controllerId: config.controllerId,
+			}],
+			exclude: req.exclude
+				? testListToProviders(req.exclude).map(t => ({ testId: t.testId, controllerId: t.controllerId }))
+				: undefined,
 		}, token);
 	}
 
@@ -182,7 +213,12 @@ export class ExtHostTesting implements ExtHostTestingShape {
 			return;
 		}
 
-		const { controller, collection } = lookup;
+		const { collection, configurations } = lookup;
+		const configuration = configurations.get(req.configId);
+		if (!configuration) {
+			return;
+		}
+
 		const includeTests = req.testIds
 			.map((testId) => collection.tree.get(testId))
 			.filter(isDefined);
@@ -198,16 +234,16 @@ export class ExtHostTesting implements ExtHostTestingShape {
 			return;
 		}
 
-		const publicReq: vscode.TestRunRequest = {
-			tests: includeTests.map(t => t.actual),
-			exclude: excludeTests.map(t => t.actual),
-			debug: req.debug,
-		};
+		const publicReq = new TestRunRequest(
+			includeTests.map(t => t.actual),
+			excludeTests.map(t => t.actual),
+			configuration,
+		);
 
 		const tracker = this.runTracker.prepareForMainThreadTestRun(publicReq, TestRunDto.fromInternal(req), token);
 
 		try {
-			await controller.runHandler?.(publicReq, token);
+			configuration.runHandler(publicReq, token);
 		} finally {
 			if (tracker.isRunning && !token.isCancellationRequested) {
 				await Event.toPromise(tracker.onEnd);
@@ -357,8 +393,10 @@ export class TestRunCoordinator {
 		// If there is not an existing tracked extension for the request, start
 		// a new, detached session.
 		const dto = TestRunDto.fromPublic(controllerId, request);
+		const config = tryGetConfigFromTestRunReq(request);
 		this.proxy.$startedExtensionTestRun({
-			debug: request.debug,
+			controllerId,
+			config: config && { group: configGroupToBitset[config.group], id: config.configId },
 			exclude: request.exclude?.map(t => t.id) ?? [],
 			id: dto.id,
 			tests: request.tests.map(t => t.id),
@@ -378,6 +416,18 @@ export class TestRunCoordinator {
 	}
 }
 
+const tryGetConfigFromTestRunReq = (request: vscode.TestRunRequest) => {
+	if (!request.configuration) {
+		return undefined;
+	}
+
+	if (!(request.configuration instanceof TestRunConfigurationImpl)) {
+		throw new Error(`TestRunRequest.configuration is not an instance created from TestController.createRunConfiguration`);
+	}
+
+	return request.configuration;
+};
+
 export class TestRunDto {
 	public static fromPublic(controllerId: string, request: vscode.TestRunRequest) {
 		return new TestRunDto(
@@ -748,3 +798,77 @@ class TestObservers {
 		return { observers: 0, tests, };
 	}
 }
+
+export class TestRunConfigurationImpl implements vscode.TestRunConfiguration {
+	readonly #proxy: MainThreadTestingShape;
+	private _configureHandler?: (() => void);
+
+	public get label() {
+		return this._label;
+	}
+
+	public set label(label: string) {
+		if (label !== this._label) {
+			this._label = label;
+			this.#proxy.$updateTestRunConfig(this.controllerId, this.configId, { label });
+		}
+	}
+
+	public get isDefault() {
+		return this._isDefault;
+	}
+
+	public set isDefault(isDefault: boolean) {
+		if (isDefault !== this._isDefault) {
+			this._isDefault = isDefault;
+			this.#proxy.$updateTestRunConfig(this.controllerId, this.configId, { isDefault });
+		}
+	}
+
+	public get configureHandler() {
+		return this._configureHandler;
+	}
+
+	public set configureHandler(handler: undefined | (() => void)) {
+		if (handler !== this._configureHandler) {
+			this._configureHandler = handler;
+			this.#proxy.$updateTestRunConfig(this.controllerId, this.configId, { hasConfigurationHandler: !!handler });
+		}
+	}
+
+	constructor(
+		proxy: MainThreadTestingShape,
+		public readonly controllerId: string,
+		public readonly configId: number,
+		private _label: string,
+		public readonly group: vscode.TestRunConfigurationGroup,
+		public runHandler: vscode.TestRunHandler,
+		private _isDefault = false,
+	) {
+		this.#proxy = proxy;
+
+		const groupBitset = configGroupToBitset[group];
+		if (typeof groupBitset !== 'number') {
+			throw new Error(`Unknown TestRunConfiguration.group ${group}`);
+		}
+
+		this.#proxy.$publishTestRunConfig({
+			configId,
+			controllerId,
+			label: _label,
+			group: groupBitset,
+			isDefault: _isDefault,
+			hasConfigurationHandler: false,
+		});
+	}
+
+	dispose(): void {
+		this.#proxy.$removeTestRunConfig(this.controllerId, this.configId);
+	}
+}
+
+const configGroupToBitset: { [K in TestRunConfigurationGroup]: TestRunConfigurationBitset } = {
+	[TestRunConfigurationGroup.Coverage]: TestRunConfigurationBitset.Coverage,
+	[TestRunConfigurationGroup.Debug]: TestRunConfigurationBitset.Debug,
+	[TestRunConfigurationGroup.Run]: TestRunConfigurationBitset.Run,
+};

src/vs/workbench/api/common/extHostTypeConverters.ts

@@ -1643,9 +1643,7 @@ export namespace TestItem {
 			label: item.label,
 			uri: item.uri,
 			range: Range.from(item.range) || null,
-			debuggable: item.debuggable ?? false,
 			description: item.description || null,
-			runnable: item.runnable ?? true,
 			error: item.error ? (MarkdownString.fromStrict(item.error) || null) : null,
 		};
 	}
@@ -1656,10 +1654,8 @@ export namespace TestItem {
 			label: item.label,
 			uri: item.uri,
 			range: Range.from(item.range) || null,
-			debuggable: false,
 			description: item.description || null,
 			error: null,
-			runnable: true,
 		};
 	}
 
@@ -1673,18 +1669,14 @@ export namespace TestItem {
 			invalidateResults: () => undefined,
 			canResolveChildren: false,
 			busy: false,
-			debuggable: item.debuggable,
 			description: item.description || undefined,
-			runnable: item.runnable,
 		};
 	}
 
 	export function to(item: ITestItem, parent?: vscode.TestItem): types.TestItemImpl {
 		const testItem = new types.TestItemImpl(item.extId, item.label, URI.revive(item.uri), undefined, parent);
 		testItem.range = Range.to(item.range || undefined);
-		testItem.debuggable = item.debuggable;
 		testItem.description = item.description || undefined;
-		testItem.runnable = item.runnable;
 		return testItem;
 	}
 
@@ -1717,7 +1709,7 @@ export namespace TestResults {
 		const byInternalId = new Map<string, SerializedTestResultItem>();
 		for (const item of serialized.items) {
 			byInternalId.set(item.item.extId, item);
-			if (item.direct) {
+			if (serialized.request.targets.some(t => t.controllerId === item.controllerId && t.testIds.includes(item.item.extId))) {
 				roots.push(item);
 			}
 		}

src/vs/workbench/api/common/extHostTypes.ts

@@ -3331,11 +3331,17 @@ const rangeComparator = (a: vscode.Range | undefined, b: vscode.Range | undefine
 	return a.isEqual(b);
 };
 
+export enum TestRunConfigurationGroup {
+	Run = 1,
+	Debug = 2,
+	Coverage = 3,
+}
+
 export class TestRunRequest implements vscode.TestRunRequest {
 	constructor(
 		public readonly tests: vscode.TestItem[],
 		public readonly exclude?: vscode.TestItem[] | undefined,
-		public readonly debug = false,
+		public readonly configuration?: vscode.TestRunConfiguration,
 	) { }
 }
 
@@ -3347,8 +3353,6 @@ export class TestItemImpl implements vscode.TestItem {
 
 	public range!: vscode.Range | undefined;
 	public description!: string | undefined;
-	public runnable!: boolean;
-	public debuggable!: boolean;
 	public label!: string;
 	public error!: string | vscode.MarkdownString;
 	public busy!: boolean;
@@ -3384,8 +3388,6 @@ export class TestItemImpl implements vscode.TestItem {
 			range: testItemPropAccessor(api, 'range', undefined, rangeComparator),
 			label: testItemPropAccessor(api, 'label', label, strictEqualComparator),
 			description: testItemPropAccessor(api, 'description', undefined, strictEqualComparator),
-			runnable: testItemPropAccessor(api, 'runnable', true, strictEqualComparator),
-			debuggable: testItemPropAccessor(api, 'debuggable', false, strictEqualComparator),
 			canResolveChildren: testItemPropAccessor(api, 'canResolveChildren', false, strictEqualComparator),
 			busy: testItemPropAccessor(api, 'busy', false, strictEqualComparator),
 			error: testItemPropAccessor(api, 'error', undefined, strictEqualComparator),