From 6e69d8b462ff61e8428c5d30d479e7477f04372a Mon Sep 17 00:00:00 2001 From: Connor Peet Date: Tue, 7 May 2024 16:15:34 -0700 Subject: [PATCH 1/2] testing: initial attributable test coverage API This implements the data flow for attributable test coverage. Todo for tomorrow is to support this in our test runner (probably making module to generate per-test coverage for generic JS tests) and then building some UX for it. --- src/vs/base/common/assert.ts | 4 +- src/vs/base/common/prefixTree.ts | 5 + src/vs/workbench/api/common/extHostTesting.ts | 55 +++-- .../api/common/extHostTypeConverters.ts | 4 +- src/vs/workbench/api/common/extHostTypes.ts | 1 + .../api/test/browser/extHostTesting.test.ts | 33 +-- .../contrib/testing/common/testCoverage.ts | 72 +++++-- .../contrib/testing/common/testTypes.ts | 11 + .../testing/test/common/testCoverage.test.ts | 194 ++++++++++++++++++ .../common/extensionsApiProposals.ts | 1 + .../vscode.proposed.attributableCoverage.d.ts | 28 +++ 11 files changed, 359 insertions(+), 49 deletions(-) create mode 100644 src/vs/workbench/contrib/testing/test/common/testCoverage.test.ts create mode 100644 src/vscode-dts/vscode.proposed.attributableCoverage.d.ts diff --git a/src/vs/base/common/assert.ts b/src/vs/base/common/assert.ts index 4ded48fb1de..bbd344d55c7 100644 --- a/src/vs/base/common/assert.ts +++ b/src/vs/base/common/assert.ts @@ -29,9 +29,9 @@ export function assertNever(value: never, message = 'Unreachable'): never { throw new Error(message); } -export function assert(condition: boolean): void { +export function assert(condition: boolean, message = 'unexpected state'): asserts condition { if (!condition) { - throw new BugIndicatingError('Assertion Failed'); + throw new BugIndicatingError(`Assertion Failed: ${message}`); } } diff --git a/src/vs/base/common/prefixTree.ts b/src/vs/base/common/prefixTree.ts index 0c2bdca384b..8e839e2b4ff 100644 --- a/src/vs/base/common/prefixTree.ts +++ b/src/vs/base/common/prefixTree.ts @@ -46,6 +46,11 @@ export class WellDefinedPrefixTree { this.opNode(key, n => n._value = mutate(n._value === unset ? undefined : n._value)); } + /** Mutates nodes along the path in the prefix tree. */ + mutatePath(key: Iterable, mutate: (node: IPrefixTreeNode) => void): void { + this.opNode(key, () => { }, n => mutate(n)); + } + /** Deletes a node from the prefix tree, returning the value it contained. */ delete(key: Iterable): V | undefined { const path = this.getPathToKey(key); diff --git a/src/vs/workbench/api/common/extHostTesting.ts b/src/vs/workbench/api/common/extHostTesting.ts index 44a28328eab..7e50f1fa9fc 100644 --- a/src/vs/workbench/api/common/extHostTesting.ts +++ b/src/vs/workbench/api/common/extHostTesting.ts @@ -23,11 +23,12 @@ import { ExtHostDocumentsAndEditors } from 'vs/workbench/api/common/extHostDocum import { IExtHostRpcService } from 'vs/workbench/api/common/extHostRpcService'; import { ExtHostTestItemCollection, TestItemImpl, TestItemRootImpl, toItemFromContext } from 'vs/workbench/api/common/extHostTestItem'; import * as Convert from 'vs/workbench/api/common/extHostTypeConverters'; -import { TestRunProfileKind, TestRunRequest } from 'vs/workbench/api/common/extHostTypes'; +import { TestRunProfileKind, TestRunRequest, FileCoverage } from 'vs/workbench/api/common/extHostTypes'; import { TestCommandId } from 'vs/workbench/contrib/testing/common/constants'; import { TestId, TestIdPathParts, TestPosition } from 'vs/workbench/contrib/testing/common/testId'; import { InvalidTestItemError } from 'vs/workbench/contrib/testing/common/testItemCollection'; import { AbstractIncrementalTestCollection, CoverageDetails, ICallProfileRunHandler, ISerializedTestResults, IStartControllerTests, IStartControllerTestsResult, ITestErrorMessage, ITestItem, ITestItemContext, ITestMessageMenuArgs, ITestRunProfile, IncrementalChangeCollector, IncrementalTestCollectionItem, InternalTestItem, TestResultState, TestRunProfileBitset, TestsDiff, TestsDiffOp, isStartControllerTests } from 'vs/workbench/contrib/testing/common/testTypes'; +import { checkProposedApiEnabled } from 'vs/workbench/services/extensions/common/extensions'; import type * as vscode from 'vscode'; interface ControllerInfo { @@ -154,7 +155,7 @@ export class ExtHostTesting extends Disposable implements ExtHostTestingShape { return new TestItemImpl(controllerId, id, label, uri); }, createTestRun: (request, name, persist = true) => { - return this.runTracker.createTestRun(controllerId, collection, request, name, persist); + return this.runTracker.createTestRun(extension, controllerId, collection, request, name, persist); }, invalidateTestResults: items => { if (items === undefined) { @@ -353,7 +354,7 @@ export class ExtHostTesting extends Disposable implements ExtHostTestingShape { return {}; } - const { collection, profiles } = lookup; + const { collection, profiles, extension } = lookup; const profile = profiles.get(req.profileId); if (!profile) { return {}; @@ -382,6 +383,7 @@ export class ExtHostTesting extends Disposable implements ExtHostTestingShape { ); const tracker = isStartControllerTests(req) && this.runTracker.prepareForMainThreadTestRun( + extension, publicReq, TestRunDto.fromInternal(req, lookup.collection), profile, @@ -460,6 +462,7 @@ class TestRunTracker extends Disposable { private readonly proxy: MainThreadTestingShape, private readonly logService: ILogService, private readonly profile: vscode.TestRunProfile | undefined, + private readonly extension: IRelaxedExtensionDescription, parentToken?: CancellationToken, ) { super(); @@ -539,16 +542,43 @@ class TestRunTracker extends Disposable { }; let ended = false; + + // one-off map used to associate test items with incrementing IDs in `addCoverage`. + // There's no need to include their entire ID, we just want to make sure they're + // stable and unique. Normal map is okay since TestRun lifetimes are limited. + const testItemCoverageId = new Map(); const run: vscode.TestRun = { isPersisted: this.dto.isPersisted, token: this.cts.token, name, onDidDispose: this.onDidDispose, - addCoverage: coverage => { + addCoverage: (coverage) => { + if (ended) { + return; + } + + const testItem = coverage instanceof FileCoverage ? coverage.testItem : undefined; + let testItemIdPart: undefined | number; + if (testItem) { + checkProposedApiEnabled(this.extension, 'attributableCoverage'); + if (!this.dto.isIncluded(testItem)) { + throw new Error('Attempted to `addCoverage` for a test item not included in the run'); + } + + testItemIdPart = testItemCoverageId.get(testItem); + if (testItemIdPart === undefined) { + testItemIdPart = testItemCoverageId.size; + testItemCoverageId.set(testItem, testItemIdPart); + } + } + const uriStr = coverage.uri.toString(); - const id = new TestId([runId, taskId, uriStr]).toString(); + const id = new TestId(testItemIdPart + ? [runId, taskId, uriStr, String(testItemIdPart)] + : [runId, taskId, uriStr], + ).toString(); this.publishedCoverage.set(uriStr, coverage); - this.proxy.$appendCoverage(runId, taskId, Convert.TestCoverage.fromFile(id, coverage)); + this.proxy.$appendCoverage(runId, taskId, Convert.TestCoverage.fromFile(ctrlId, id, coverage)); }, //#region state mutation enqueued: guardTestMutation(test => { @@ -599,6 +629,7 @@ class TestRunTracker extends Disposable { } ended = true; + testItemCoverageId.clear(); this.proxy.$finishedTestRunTask(runId, taskId); if (!--this.running) { this.markEnded(); @@ -706,8 +737,8 @@ export class TestRunCoordinator { * `$startedExtensionTestRun` is not invoked. The run must eventually * be cancelled manually. */ - public prepareForMainThreadTestRun(req: vscode.TestRunRequest, dto: TestRunDto, profile: vscode.TestRunProfile, token: CancellationToken) { - return this.getTracker(req, dto, profile, token); + public prepareForMainThreadTestRun(extension: IRelaxedExtensionDescription, req: vscode.TestRunRequest, dto: TestRunDto, profile: vscode.TestRunProfile, token: CancellationToken) { + return this.getTracker(req, dto, profile, extension, token); } /** @@ -729,7 +760,7 @@ export class TestRunCoordinator { /** * Implements the public `createTestRun` API. */ - public createTestRun(controllerId: string, collection: ExtHostTestItemCollection, request: vscode.TestRunRequest, name: string | undefined, persist: boolean): vscode.TestRun { + public createTestRun(extension: IRelaxedExtensionDescription, controllerId: string, collection: ExtHostTestItemCollection, request: vscode.TestRunRequest, name: string | undefined, persist: boolean): vscode.TestRun { const existing = this.tracked.get(request); if (existing) { return existing.createRun(name); @@ -750,7 +781,7 @@ export class TestRunCoordinator { persist }); - const tracker = this.getTracker(request, dto, request.profile); + const tracker = this.getTracker(request, dto, request.profile, extension); Event.once(tracker.onEnd)(() => { this.proxy.$finishedExtensionTestRun(dto.id); }); @@ -758,8 +789,8 @@ export class TestRunCoordinator { return tracker.createRun(name); } - private getTracker(req: vscode.TestRunRequest, dto: TestRunDto, profile: vscode.TestRunProfile | undefined, token?: CancellationToken) { - const tracker = new TestRunTracker(dto, this.proxy, this.logService, profile, token); + private getTracker(req: vscode.TestRunRequest, dto: TestRunDto, profile: vscode.TestRunProfile | undefined, extension: IRelaxedExtensionDescription, token?: CancellationToken) { + const tracker = new TestRunTracker(dto, this.proxy, this.logService, profile, extension, token); this.tracked.set(req, tracker); this.trackedById.set(tracker.id, tracker); return tracker; diff --git a/src/vs/workbench/api/common/extHostTypeConverters.ts b/src/vs/workbench/api/common/extHostTypeConverters.ts index 6d3beb0e99a..defa6ec84bf 100644 --- a/src/vs/workbench/api/common/extHostTypeConverters.ts +++ b/src/vs/workbench/api/common/extHostTypeConverters.ts @@ -2052,7 +2052,7 @@ export namespace TestCoverage { } } - export function fromFile(id: string, coverage: vscode.FileCoverage): IFileCoverage.Serialized { + export function fromFile(controllerId: string, id: string, coverage: vscode.FileCoverage): IFileCoverage.Serialized { types.validateTestCoverageCount(coverage.statementCoverage); types.validateTestCoverageCount(coverage.branchCoverage); types.validateTestCoverageCount(coverage.declarationCoverage); @@ -2063,6 +2063,8 @@ export namespace TestCoverage { statement: fromCoverageCount(coverage.statementCoverage), branch: coverage.branchCoverage && fromCoverageCount(coverage.branchCoverage), declaration: coverage.declarationCoverage && fromCoverageCount(coverage.declarationCoverage), + testId: coverage instanceof types.FileCoverage && coverage.testItem ? + TestId.fromExtHostTestItem(coverage.testItem, controllerId).toString() : undefined, }; } } diff --git a/src/vs/workbench/api/common/extHostTypes.ts b/src/vs/workbench/api/common/extHostTypes.ts index 220035f98f0..9637b4086a2 100644 --- a/src/vs/workbench/api/common/extHostTypes.ts +++ b/src/vs/workbench/api/common/extHostTypes.ts @@ -4119,6 +4119,7 @@ export class FileCoverage implements vscode.FileCoverage { public statementCoverage: vscode.TestCoverageCount, public branchCoverage?: vscode.TestCoverageCount, public declarationCoverage?: vscode.TestCoverageCount, + public testItem?: vscode.TestItem, ) { } } diff --git a/src/vs/workbench/api/test/browser/extHostTesting.test.ts b/src/vs/workbench/api/test/browser/extHostTesting.test.ts index 0a4723049b6..b82376cd88d 100644 --- a/src/vs/workbench/api/test/browser/extHostTesting.test.ts +++ b/src/vs/workbench/api/test/browser/extHostTesting.test.ts @@ -14,7 +14,7 @@ import { URI } from 'vs/base/common/uri'; import { mock, mockObject, MockObject } from 'vs/base/test/common/mock'; import { ensureNoDisposablesAreLeakedInTestSuite } from 'vs/base/test/common/utils'; import * as editorRange from 'vs/editor/common/core/range'; -import { ExtensionIdentifier } from 'vs/platform/extensions/common/extensions'; +import { ExtensionIdentifier, IRelaxedExtensionDescription } from 'vs/platform/extensions/common/extensions'; import { NullLogService } from 'vs/platform/log/common/log'; import { MainThreadTestingShape } from 'vs/workbench/api/common/extHost.protocol'; import { ExtHostCommands } from 'vs/workbench/api/common/extHostCommands'; @@ -637,6 +637,7 @@ suite('ExtHost Testing', () => { let req: TestRunRequest; let dto: TestRunDto; + const ext: IRelaxedExtensionDescription = {} as any; teardown(() => { for (const { id } of c.trackers) { @@ -671,11 +672,11 @@ suite('ExtHost Testing', () => { }); test('tracks a run started from a main thread request', () => { - const tracker = ds.add(c.prepareForMainThreadTestRun(req, dto, configuration, cts.token)); + const tracker = ds.add(c.prepareForMainThreadTestRun(ext, req, dto, configuration, cts.token)); assert.strictEqual(tracker.hasRunningTasks, false); - const task1 = c.createTestRun('ctrl', single, req, 'run1', true); - const task2 = c.createTestRun('ctrl', single, req, 'run2', true); + const task1 = c.createTestRun(ext, 'ctrl', single, req, 'run1', true); + const task2 = c.createTestRun(ext, 'ctrl', single, req, 'run2', true); assert.strictEqual(proxy.$startedExtensionTestRun.called, false); assert.strictEqual(tracker.hasRunningTasks, true); @@ -696,8 +697,8 @@ suite('ExtHost Testing', () => { test('run cancel force ends after a timeout', () => { const clock = sinon.useFakeTimers(); try { - const tracker = ds.add(c.prepareForMainThreadTestRun(req, dto, configuration, cts.token)); - const task = c.createTestRun('ctrl', single, req, 'run1', true); + const tracker = ds.add(c.prepareForMainThreadTestRun(ext, req, dto, configuration, cts.token)); + const task = c.createTestRun(ext, 'ctrl', single, req, 'run1', true); const onEnded = sinon.stub(); ds.add(tracker.onEnd(onEnded)); @@ -721,8 +722,8 @@ suite('ExtHost Testing', () => { }); test('run cancel force ends on second cancellation request', () => { - const tracker = ds.add(c.prepareForMainThreadTestRun(req, dto, configuration, cts.token)); - const task = c.createTestRun('ctrl', single, req, 'run1', true); + const tracker = ds.add(c.prepareForMainThreadTestRun(ext, req, dto, configuration, cts.token)); + const task = c.createTestRun(ext, 'ctrl', single, req, 'run1', true); const onEnded = sinon.stub(); ds.add(tracker.onEnd(onEnded)); @@ -740,7 +741,7 @@ suite('ExtHost Testing', () => { }); test('tracks a run started from an extension request', () => { - const task1 = c.createTestRun('ctrl', single, req, 'hello world', false); + const task1 = c.createTestRun(ext, 'ctrl', single, req, 'hello world', false); const tracker = Iterable.first(c.trackers)!; assert.strictEqual(tracker.hasRunningTasks, true); @@ -757,8 +758,8 @@ suite('ExtHost Testing', () => { }] ]); - const task2 = c.createTestRun('ctrl', single, req, 'run2', true); - const task3Detached = c.createTestRun('ctrl', single, { ...req }, 'task3Detached', true); + const task2 = c.createTestRun(ext, 'ctrl', single, req, 'run2', true); + const task3Detached = c.createTestRun(ext, 'ctrl', single, { ...req }, 'task3Detached', true); task1.end(); assert.strictEqual(proxy.$finishedExtensionTestRun.called, false); @@ -772,7 +773,7 @@ suite('ExtHost Testing', () => { }); test('adds tests to run smartly', () => { - const task1 = c.createTestRun('ctrlId', single, req, 'hello world', false); + const task1 = c.createTestRun(ext, 'ctrlId', single, req, 'hello world', false); const tracker = Iterable.first(c.trackers)!; const expectedArgs: unknown[][] = []; assert.deepStrictEqual(proxy.$addTestsToRun.args, expectedArgs); @@ -811,7 +812,7 @@ suite('ExtHost Testing', () => { const test2 = new TestItemImpl('ctrlId', 'id-d', 'test d', URI.file('/testd.txt')); test1.range = test2.range = new Range(new Position(0, 0), new Position(1, 0)); single.root.children.replace([test1, test2]); - const task = c.createTestRun('ctrlId', single, req, 'hello world', false); + const task = c.createTestRun(ext, 'ctrlId', single, req, 'hello world', false); const message1 = new TestMessage('some message'); message1.location = new Location(URI.file('/a.txt'), new Position(0, 0)); @@ -852,7 +853,7 @@ suite('ExtHost Testing', () => { }); test('guards calls after runs are ended', () => { - const task = c.createTestRun('ctrl', single, req, 'hello world', false); + const task = c.createTestRun(ext, 'ctrl', single, req, 'hello world', false); task.end(); task.failed(single.root, new TestMessage('some message')); @@ -864,7 +865,7 @@ suite('ExtHost Testing', () => { }); test('excludes tests outside tree or explicitly excluded', () => { - const task = c.createTestRun('ctrlId', single, { + const task = c.createTestRun(ext, 'ctrlId', single, { profile: configuration, include: [single.root.children.get('id-a')!], exclude: [single.root.children.get('id-a')!.children.get('id-aa')!], @@ -894,7 +895,7 @@ suite('ExtHost Testing', () => { const childB = new TestItemImpl('ctrlId', 'id-child', 'child', undefined); testB!.children.replace([childB]); - const task1 = c.createTestRun('ctrl', single, new TestRunRequestImpl(), 'hello world', false); + const task1 = c.createTestRun(ext, 'ctrl', single, new TestRunRequestImpl(), 'hello world', false); const tracker = Iterable.first(c.trackers)!; task1.passed(childA); diff --git a/src/vs/workbench/contrib/testing/common/testCoverage.ts b/src/vs/workbench/contrib/testing/common/testCoverage.ts index 10d4f23cea3..ada6359fded 100644 --- a/src/vs/workbench/contrib/testing/common/testCoverage.ts +++ b/src/vs/workbench/contrib/testing/common/testCoverage.ts @@ -3,6 +3,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ +import { assert } from 'vs/base/common/assert'; import { CancellationToken } from 'vs/base/common/cancellation'; import { ResourceMap } from 'vs/base/common/map'; import { deepClone } from 'vs/base/common/objects'; @@ -34,8 +35,7 @@ export class TestCoverage { private readonly accessor: ICoverageAccessor, ) { } - public append(rawCoverage: IFileCoverage, tx: ITransaction | undefined) { - const coverage = new FileCoverage(rawCoverage, this.accessor); + public append(coverage: IFileCoverage, tx: ITransaction | undefined) { const previous = this.getComputedForUri(coverage.uri); const applyDelta = (kind: 'statement' | 'branch' | 'declaration', node: ComputedFileCoverage) => { if (!node[kind]) { @@ -53,27 +53,51 @@ export class TestCoverage { // version. const canonical = [...this.treePathForUri(coverage.uri, /* canonical = */ true)]; const chain: IPrefixTreeNode[] = []; - this.tree.insert(this.treePathForUri(coverage.uri, /* canonical = */ false), coverage, node => { + const isPerTestCoverage = !!coverage.testId; + this.tree.mutatePath(this.treePathForUri(coverage.uri, /* canonical = */ false), node => { chain.push(node); if (chain.length === canonical.length) { - node.value = coverage; - } else if (!node.value) { - // clone because later intersertions can modify the counts: - const intermediate = deepClone(rawCoverage); - intermediate.id = String(incId++); - intermediate.uri = this.treePathToUri(canonical.slice(0, chain.length)); - node.value = new ComputedFileCoverage(intermediate); - } else { - applyDelta('statement', node.value); - applyDelta('branch', node.value); - applyDelta('declaration', node.value); - node.value.didChange.trigger(tx); + // we reached our destination node, apply the coverage as necessary: + if (isPerTestCoverage) { + const v = node.value ??= new FileCoverage(IFileCoverage.empty(String(incId++), coverage.uri), this.accessor); + assert(v instanceof FileCoverage, 'coverage is unexpectedly computed'); + v.perTestData ??= new Map(); + v.perTestData.set(coverage.testId!.toString(), new FileCoverage(coverage, this.accessor)); + this.fileCoverage.set(coverage.uri, v); + } else if (node.value) { + const v = node.value; + // if ID was generated from a test-specific coverage, reassign it to get its real ID in the extension host. + v.id = coverage.id; + v.statement = coverage.statement; + v.branch = coverage.branch; + v.declaration = coverage.declaration; + v.existsInExtHost = true; + } else { + const v = node.value = new FileCoverage(coverage, this.accessor); + v.existsInExtHost = true; + this.fileCoverage.set(coverage.uri, v); + } + } else if (!isPerTestCoverage) { + // Otherwise, if this is not a partial per-test coverage, merge the + // coverage changes into the chain. Per-test coverages are not complete + // and we don't want to consider them for computation. + if (!node.value) { + // clone because later intersertions can modify the counts: + const intermediate = deepClone(coverage); + intermediate.id = String(incId++); + intermediate.uri = this.treePathToUri(canonical.slice(0, chain.length)); + node.value = new ComputedFileCoverage(intermediate); + } else { + applyDelta('statement', node.value); + applyDelta('branch', node.value); + applyDelta('declaration', node.value); + node.value.didChange.trigger(tx); + } } }); - this.fileCoverage.set(coverage.uri, coverage); - if (chain) { + if (chain && !isPerTestCoverage) { this.didAddCoverage.trigger(tx, chain); } } @@ -131,13 +155,20 @@ export const getTotalCoveragePercent = (statement: ICoverageCount, branch: ICove }; export abstract class AbstractFileCoverage { - public readonly id: string; + public id: string; public readonly uri: URI; public statement: ICoverageCount; public branch?: ICoverageCount; public declaration?: ICoverageCount; public readonly didChange = observableSignal(this); + /** + * Whether this coverage item exists in the extension host. This is false + * if we have only {@link perTestData} and not summary data for the file, or + * if the node is computed for a directory. + */ + public existsInExtHost = false; + /** * Gets the total coverage percent based on information provided. * This is based on the Clover total coverage formula @@ -170,6 +201,11 @@ export class FileCoverage extends AbstractFileCoverage { return this._details instanceof Array || this.resolved; } + /** + * Per-test coverage data for this file, if available. + */ + public perTestData?: Map; + constructor(coverage: IFileCoverage, private readonly accessor: ICoverageAccessor) { super(coverage); } diff --git a/src/vs/workbench/contrib/testing/common/testTypes.ts b/src/vs/workbench/contrib/testing/common/testTypes.ts index cd14796eb0a..d1f779a475d 100644 --- a/src/vs/workbench/contrib/testing/common/testTypes.ts +++ b/src/vs/workbench/contrib/testing/common/testTypes.ts @@ -559,6 +559,7 @@ export namespace ICoverageCount { export interface IFileCoverage { id: string; uri: URI; + testId?: TestId; statement: ICoverageCount; branch?: ICoverageCount; declaration?: ICoverageCount; @@ -568,6 +569,7 @@ export namespace IFileCoverage { export interface Serialized { id: string; uri: UriComponents; + testId: string | undefined; statement: ICoverageCount; branch?: ICoverageCount; declaration?: ICoverageCount; @@ -578,6 +580,7 @@ export namespace IFileCoverage { statement: original.statement, branch: original.branch, declaration: original.declaration, + testId: original.testId?.toString(), uri: original.uri.toJSON(), }); @@ -586,8 +589,16 @@ export namespace IFileCoverage { statement: serialized.statement, branch: serialized.branch, declaration: serialized.declaration, + testId: serialized.testId ? TestId.fromString(serialized.testId) : undefined, uri: uriIdentity.asCanonicalUri(URI.revive(serialized.uri)), }); + + export const empty = (id: string, uri: URI): IFileCoverage => ({ + id, + uri, + testId: undefined, + statement: ICoverageCount.empty(), + }); } function serializeThingWithLocation(serialized: T): T & { location?: IRange | IPosition } { diff --git a/src/vs/workbench/contrib/testing/test/common/testCoverage.test.ts b/src/vs/workbench/contrib/testing/test/common/testCoverage.test.ts new file mode 100644 index 00000000000..bad192a7c02 --- /dev/null +++ b/src/vs/workbench/contrib/testing/test/common/testCoverage.test.ts @@ -0,0 +1,194 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as assert from 'assert'; +import { SinonSandbox, createSandbox } from 'sinon'; +import { Iterable } from 'vs/base/common/iterator'; +import { URI } from 'vs/base/common/uri'; +import { ensureNoDisposablesAreLeakedInTestSuite } from 'vs/base/test/common/utils'; +import { onObservableChange } from 'vs/workbench/contrib/testing/common/observableUtils'; +import { ICoverageAccessor, TestCoverage } from 'vs/workbench/contrib/testing/common/testCoverage'; +import { TestId } from 'vs/workbench/contrib/testing/common/testId'; +import { IFileCoverage } from 'vs/workbench/contrib/testing/common/testTypes'; + +suite('TestCoverage', () => { + let sandbox: SinonSandbox; + let coverageAccessor: ICoverageAccessor; + let testCoverage: TestCoverage; + + const ds = ensureNoDisposablesAreLeakedInTestSuite(); + + setup(() => { + sandbox = createSandbox(); + coverageAccessor = { + getCoverageDetails: sandbox.stub().resolves([]), + }; + testCoverage = new TestCoverage('taskId', { extUri: { ignorePathCasing: () => true } } as any, coverageAccessor); + }); + + teardown(() => { + sandbox.restore(); + }); + + function addTests() { + const raw1: IFileCoverage = { + id: '1', + uri: URI.file('/path/to/file'), + statement: { covered: 10, total: 20 }, + branch: { covered: 5, total: 10 }, + declaration: { covered: 2, total: 5 }, + }; + + testCoverage.append(raw1, undefined); + + const raw2: IFileCoverage = { + id: '1', + uri: URI.file('/path/to/file2'), + statement: { covered: 5, total: 10 }, + branch: { covered: 1, total: 5 }, + }; + + testCoverage.append(raw2, undefined); + + return { raw1, raw2 }; + } + + test('should look up file coverage', async () => { + const { raw1 } = addTests(); + + const fileCoverage = testCoverage.getUri(raw1.uri); + assert.equal(fileCoverage?.id, raw1.id); + assert.deepEqual(fileCoverage?.statement, raw1.statement); + assert.deepEqual(fileCoverage?.branch, raw1.branch); + assert.deepEqual(fileCoverage?.declaration, raw1.declaration); + assert.strictEqual(fileCoverage?.existsInExtHost, true); + + assert.strictEqual(testCoverage.getComputedForUri(raw1.uri), testCoverage.getUri(raw1.uri)); + assert.strictEqual(testCoverage.getComputedForUri(URI.file('/path/to/x')), undefined); + assert.strictEqual(testCoverage.getUri(URI.file('/path/to/x')), undefined); + }); + + test('should compute coverage for directories', async () => { + const { raw1 } = addTests(); + const dirCoverage = testCoverage.getComputedForUri(URI.file('/path/to')); + assert.deepEqual(dirCoverage?.statement, { covered: 15, total: 30 }); + assert.deepEqual(dirCoverage?.branch, { covered: 6, total: 15 }); + assert.deepEqual(dirCoverage?.declaration, raw1.declaration); + assert.strictEqual(dirCoverage?.existsInExtHost, false); + }); + + test('should incrementally diff updates to existing files', async () => { + addTests(); + + const raw3: IFileCoverage = { + id: '1', + uri: URI.file('/path/to/file'), + statement: { covered: 12, total: 24 }, + branch: { covered: 7, total: 10 }, + declaration: { covered: 2, total: 5 }, + }; + + testCoverage.append(raw3, undefined); + + const fileCoverage = testCoverage.getUri(raw3.uri); + assert.deepEqual(fileCoverage?.statement, raw3.statement); + assert.deepEqual(fileCoverage?.branch, raw3.branch); + assert.deepEqual(fileCoverage?.declaration, raw3.declaration); + + const dirCoverage = testCoverage.getComputedForUri(URI.file('/path/to')); + assert.deepEqual(dirCoverage?.statement, { covered: 17, total: 34 }); + assert.deepEqual(dirCoverage?.branch, { covered: 8, total: 15 }); + assert.deepEqual(dirCoverage?.declaration, raw3.declaration); + }); + + test('should emit changes', async () => { + const changes: string[][] = []; + ds.add(onObservableChange(testCoverage.didAddCoverage, value => + changes.push(value.map(v => v.value!.uri.toString())))); + + addTests(); + + assert.deepStrictEqual(changes, [ + [ + "file:///", + "file:///", + "file:///", + "file:///path", + "file:///path/to", + "file:///path/to/file", + ], + [ + "file:///", + "file:///", + "file:///", + "file:///path", + "file:///path/to", + "file:///path/to/file2", + ], + ]); + }); + + test('adds per-test data to files', async () => { + const { raw1 } = addTests(); + + const raw3: IFileCoverage = { + id: '1', + testId: TestId.fromString('my-test'), + uri: URI.file('/path/to/file'), + statement: { covered: 12, total: 24 }, + branch: { covered: 7, total: 10 }, + declaration: { covered: 2, total: 5 }, + }; + testCoverage.append(raw3, undefined); + + const fileCoverage = testCoverage.getUri(raw1.uri); + assert.strictEqual(fileCoverage?.perTestData?.size, 1); + + const perTestCoverage = Iterable.first(fileCoverage!.perTestData!.values()); + assert.deepStrictEqual(perTestCoverage?.statement, raw3.statement); + assert.deepStrictEqual(perTestCoverage?.branch, raw3.branch); + assert.deepStrictEqual(perTestCoverage?.declaration, raw3.declaration); + + // should be unchanged: + assert.deepEqual(fileCoverage?.statement, { covered: 10, total: 20 }); + assert.deepEqual(fileCoverage?.existsInExtHost, true); + const dirCoverage = testCoverage.getComputedForUri(URI.file('/path/to')); + assert.deepEqual(dirCoverage?.statement, { covered: 15, total: 30 }); + }); + + test('works if per-test data is added first', async () => { + const raw3: IFileCoverage = { + id: '1', + testId: TestId.fromString('my-test'), + uri: URI.file('/path/to/file'), + statement: { covered: 12, total: 24 }, + branch: { covered: 7, total: 10 }, + declaration: { covered: 2, total: 5 }, + }; + testCoverage.append(raw3, undefined); + + const fileCoverage = testCoverage.getUri(raw3.uri); + assert.deepEqual(fileCoverage?.existsInExtHost, false); + + addTests(); + + assert.strictEqual(fileCoverage?.perTestData?.size, 1); + const perTestCoverage = Iterable.first(fileCoverage!.perTestData!.values()); + assert.deepStrictEqual(perTestCoverage?.statement, raw3.statement); + assert.deepStrictEqual(perTestCoverage?.branch, raw3.branch); + assert.deepStrictEqual(perTestCoverage?.declaration, raw3.declaration); + + // should be the expected values: + assert.deepEqual(fileCoverage?.statement, { covered: 10, total: 20 }); + assert.deepEqual(fileCoverage?.existsInExtHost, true); + const dirCoverage = testCoverage.getComputedForUri(URI.file('/path/to')); + assert.deepEqual(dirCoverage?.statement, { covered: 15, total: 30 }); + }); +}); diff --git a/src/vs/workbench/services/extensions/common/extensionsApiProposals.ts b/src/vs/workbench/services/extensions/common/extensionsApiProposals.ts index ccc37fcf750..19988575941 100644 --- a/src/vs/workbench/services/extensions/common/extensionsApiProposals.ts +++ b/src/vs/workbench/services/extensions/common/extensionsApiProposals.ts @@ -9,6 +9,7 @@ export const allApiProposals = Object.freeze({ activeComment: 'https://raw.githubusercontent.com/microsoft/vscode/main/src/vscode-dts/vscode.proposed.activeComment.d.ts', aiRelatedInformation: 'https://raw.githubusercontent.com/microsoft/vscode/main/src/vscode-dts/vscode.proposed.aiRelatedInformation.d.ts', aiTextSearchProvider: 'https://raw.githubusercontent.com/microsoft/vscode/main/src/vscode-dts/vscode.proposed.aiTextSearchProvider.d.ts', + attributableCoverage: 'https://raw.githubusercontent.com/microsoft/vscode/main/src/vscode-dts/vscode.proposed.attributableCoverage.d.ts', authGetSessions: 'https://raw.githubusercontent.com/microsoft/vscode/main/src/vscode-dts/vscode.proposed.authGetSessions.d.ts', authLearnMore: 'https://raw.githubusercontent.com/microsoft/vscode/main/src/vscode-dts/vscode.proposed.authLearnMore.d.ts', authSession: 'https://raw.githubusercontent.com/microsoft/vscode/main/src/vscode-dts/vscode.proposed.authSession.d.ts', diff --git a/src/vscode-dts/vscode.proposed.attributableCoverage.d.ts b/src/vscode-dts/vscode.proposed.attributableCoverage.d.ts new file mode 100644 index 00000000000..63000738c0f --- /dev/null +++ b/src/vscode-dts/vscode.proposed.attributableCoverage.d.ts @@ -0,0 +1,28 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +declare module 'vscode' { + export class FileCoverage2 extends FileCoverage { + /** + * Test {@link TestItem} this file coverage is generated from. If undefined, + * the editor will assume the coverage is the overall summary coverage for + * the entire file. + * + * If per-test coverage is available, an extension should append multiple + * `FileCoverage` instances with this property set for each test item. It + * must also append a `FileCoverage` instance without this property set to + * represent the overall coverage of the file. + */ + testItem?: TestItem; + + constructor( + uri: Uri, + statementCoverage: TestCoverageCount, + branchCoverage?: TestCoverageCount, + declarationCoverage?: TestCoverageCount, + testItem?: TestItem, + ); + } +} From 6e24e5f2bdf801a3b70f6c3068073e4af8dc6a40 Mon Sep 17 00:00:00 2001 From: Connor Peet Date: Wed, 8 May 2024 08:10:08 -0700 Subject: [PATCH 2/2] fix compile error --- src/vs/workbench/api/common/extHost.api.impl.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/vs/workbench/api/common/extHost.api.impl.ts b/src/vs/workbench/api/common/extHost.api.impl.ts index d0b5accb676..6642a348f93 100644 --- a/src/vs/workbench/api/common/extHost.api.impl.ts +++ b/src/vs/workbench/api/common/extHost.api.impl.ts @@ -1683,6 +1683,7 @@ export function createApiFactoryAndRegisterActors(accessor: ServicesAccessor): I DataTransferItem: extHostTypes.DataTransferItem, TestCoverageCount: extHostTypes.TestCoverageCount, FileCoverage: extHostTypes.FileCoverage, + FileCoverage2: extHostTypes.FileCoverage, StatementCoverage: extHostTypes.StatementCoverage, BranchCoverage: extHostTypes.BranchCoverage, DeclarationCoverage: extHostTypes.DeclarationCoverage,