diff --git a/.vscode/extensions/vscode-selfhost-test-provider/package.json b/.vscode/extensions/vscode-selfhost-test-provider/package.json index f27953f3cdb..cda2d07ddf3 100644 --- a/.vscode/extensions/vscode-selfhost-test-provider/package.json +++ b/.vscode/extensions/vscode-selfhost-test-provider/package.json @@ -65,10 +65,13 @@ "license": "MIT", "scripts": { "compile": "gulp compile-extension:vscode-selfhost-test-provider", - "watch": "gulp watch-extension:vscode-selfhost-test-provider" + "watch": "gulp watch-extension:vscode-selfhost-test-provider", + "test": "npx mocha --ui tdd 'out/*.test.js'" }, "devDependencies": { - "@types/node": "18.x" + "@types/mocha": "^10.0.6", + "@types/node": "18.x", + "v8-to-istanbul": "^9.2.0" }, "dependencies": { "@jridgewell/trace-mapping": "^0.3.25", diff --git a/.vscode/extensions/vscode-selfhost-test-provider/src/coverageProvider.ts b/.vscode/extensions/vscode-selfhost-test-provider/src/coverageProvider.ts index 16fa7843336..dd100be9a26 100644 --- a/.vscode/extensions/vscode-selfhost-test-provider/src/coverageProvider.ts +++ b/.vscode/extensions/vscode-selfhost-test-provider/src/coverageProvider.ts @@ -4,5 +4,120 @@ *--------------------------------------------------------------------------------------------*/ import { IstanbulCoverageContext } from 'istanbul-to-vscode'; +import { SourceMapStore } from './testOutputScanner'; +import * as vscode from 'vscode'; +import { IScriptCoverage, OffsetToPosition, RangeCoverageTracker } from './v8CoverageWrangling'; +import * as v8ToIstanbul from 'v8-to-istanbul'; -export const coverageContext = new IstanbulCoverageContext(); +export const istanbulCoverageContext = new IstanbulCoverageContext(); + +/** + * Tracks coverage in per-script coverage mode. There are two modes of coverage + * in this extension: generic istanbul reports, and reports from the runtime + * sent before and after each test case executes. This handles the latter. + */ +export class PerTestCoverageTracker { + private readonly scripts = new Map(); + + constructor( + private readonly initialCoverage: IScriptCoverage, + private readonly maps: SourceMapStore, + ) {} + + public add(coverage: IScriptCoverage, test?: vscode.TestItem) { + const script = this.scripts.get(coverage.scriptId); + if (script) { + return script.add(coverage, test); + } + if (!coverage.source) { + throw new Error('expected to have source the first time a script is seen'); + } + + const src = new Script(coverage.url, coverage.source, this.maps); + } +} + +class Script { + private converter: OffsetToPosition; + + /** Tracking the overall coverage for the file */ + private overall = new ScriptProjection(); + /** Range tracking per-test item */ + private readonly perItem = new Map(); + + constructor( + public readonly url: string, + source: string, + private readonly maps: SourceMapStore, + ) { + this.converter = new OffsetToPosition(source); + } + + public add(coverage: IScriptCoverage, test?: vscode.TestItem) { + this.overall.add(coverage); + if (test) { + const p = new ScriptProjection(); + p.add(coverage); + this.perItem.set(test, p); + } + } + + public report(run: vscode.TestRun) { + + } +} + +class ScriptProjection { + /** Range tracking for non-block coverage in the file */ + private file = new RangeCoverageTracker(); + /** Range tracking for block coverage in the file */ + private readonly blocks = new Map(); + + public add(coverage: IScriptCoverage) { + + for (const fn of coverage.functions) { + if (fn.isBlockCoverage) { + const key = `${fn.ranges[0].startOffset}/${fn.ranges[0].endOffset}`; + const block = this.blocks.get(key); + if (block) { + for (let i = 1; i < fn.ranges.length; i++) { + block.setCovered(fn.ranges[i].startOffset, fn.ranges[i].endOffset, fn.ranges[i].count > 0); + } + } else { + this.blocks.set(key, RangeCoverageTracker.initializeBlock(fn.ranges)); + } + } else { + for (const range of fn.ranges) { + this.file.setCovered(range.startOffset, range.endOffset, range.count > 0); + } + } + } + } + + public report(run: vscode.TestRun, convert: OffsetToPosition, item?: vscode.TestItem) { + const ranges = [...this.file]; + for (const block of this.blocks.values()) { + for (const range of block) { + ranges.push(range); + } + } + + let ri = 0; + ranges.sort((a, b) => a.end - b.end); + + let offset = 0; + for (let i = 0; i < convert.lines.length; i++) { + const lineEnd = offset + convert.lines[i] + 1; + + const coverage = new RangeCoverageTracker(); + for (let i = ri; i < ranges.length && ranges[i].start < lineEnd; i++) { + coverage.setCovered(ranges[i].start - offset, ranges[i].end - offset, ranges[i].covered); + } + + while (ri < ranges.length && ranges[ri].end < lineEnd) { + ri++; + } + } + + } +} diff --git a/.vscode/extensions/vscode-selfhost-test-provider/src/extension.ts b/.vscode/extensions/vscode-selfhost-test-provider/src/extension.ts index 153f5efb2d9..cda561e1325 100644 --- a/.vscode/extensions/vscode-selfhost-test-provider/src/extension.ts +++ b/.vscode/extensions/vscode-selfhost-test-provider/src/extension.ts @@ -7,7 +7,7 @@ import { randomBytes } from 'crypto'; import { tmpdir } from 'os'; import * as path from 'path'; import * as vscode from 'vscode'; -import { coverageContext } from './coverageProvider'; +import { istanbulCoverageContext } from './coverageProvider'; import { FailingDeepStrictEqualAssertFixer } from './failingDeepStrictEqualAssertFixer'; import { registerSnapshotUpdate } from './snapshot'; import { scanTestOutput } from './testOutputScanner'; @@ -86,17 +86,20 @@ export async function activate(context: vscode.ExtensionContext) { let coverageDir: string | undefined; let currentArgs = args; if (kind === vscode.TestRunProfileKind.Coverage) { - coverageDir = path.join(tmpdir(), `vscode-test-coverage-${randomBytes(8).toString('hex')}`); - currentArgs = [ - ...currentArgs, - '--coverage', - '--coveragePath', - coverageDir, - '--coverageFormats', - 'json', - '--coverageFormats', - 'html', - ]; + // todo: browser runs currently don't support per-test coverage + if (args.includes('--browser')) { + coverageDir = path.join(tmpdir(), `vscode-test-coverage-${randomBytes(8).toString('hex')}`); + currentArgs = [ + ...currentArgs, + '--coverage', + '--coveragePath', + coverageDir, + '--coverageFormats', + 'json', + ]; + } else { + currentArgs = [...currentArgs, '--per-test-coverage']; + } } return await scanTestOutput( @@ -180,7 +183,7 @@ export async function activate(context: vscode.ExtensionContext) { true ); - coverage.loadDetailedCoverage = coverageContext.loadDetailedCoverage; + coverage.loadDetailedCoverage = istanbulCoverageContext.loadDetailedCoverage; for (const [name, arg] of browserArgs) { const cfg = ctrl.createRunProfile( diff --git a/.vscode/extensions/vscode-selfhost-test-provider/src/testOutputScanner.ts b/.vscode/extensions/vscode-selfhost-test-provider/src/testOutputScanner.ts index 39da7213325..6ed9b7fd973 100644 --- a/.vscode/extensions/vscode-selfhost-test-provider/src/testOutputScanner.ts +++ b/.vscode/extensions/vscode-selfhost-test-provider/src/testOutputScanner.ts @@ -12,7 +12,7 @@ import { import * as styles from 'ansi-styles'; import { ChildProcessWithoutNullStreams } from 'child_process'; import * as vscode from 'vscode'; -import { coverageContext } from './coverageProvider'; +import { IScriptCoverage, istanbulCoverageContext } from './coverageProvider'; import { attachTestMessageMetadata } from './metadata'; import { snapshotComment } from './snapshot'; import { getContentFromFilesystem } from './testTree'; @@ -24,6 +24,10 @@ export const enum MochaEvent { Pass = 'pass', Fail = 'fail', End = 'end', + + // custom events: + CoverageInit = 'coverage init', + CoverageIncrement = 'coverage increment', } export interface IStartEvent { @@ -62,12 +66,20 @@ export interface IEndEvent { end: string /* ISO date */; } +export interface ITestCoverageCoverage { + file: string; + fullTitle: string; + coverage: IScriptCoverage; +} + export type MochaEventTuple = | [MochaEvent.Start, IStartEvent] | [MochaEvent.TestStart, ITestStartEvent] | [MochaEvent.Pass, IPassEvent] | [MochaEvent.Fail, IFailEvent] - | [MochaEvent.End, IEndEvent]; + | [MochaEvent.End, IEndEvent] + | [MochaEvent.CoverageInit, IScriptCoverage] + | [MochaEvent.CoverageIncrement, ITestCoverageCoverage]; const LF = '\n'.charCodeAt(0); @@ -315,7 +327,7 @@ export async function scanTestOutput( if (coverageDir) { try { - await coverageContext.apply(task, coverageDir, { + await istanbulCoverageContext.apply(task, coverageDir, { mapFileUri: uri => store.getSourceFile(uri.toString()), mapLocation: (uri, position) => store.getSourceLocation(uri.toString(), position.line, position.character), diff --git a/.vscode/extensions/vscode-selfhost-test-provider/src/v8CoverageWrangling.test.ts b/.vscode/extensions/vscode-selfhost-test-provider/src/v8CoverageWrangling.test.ts new file mode 100644 index 00000000000..ad22e317860 --- /dev/null +++ b/.vscode/extensions/vscode-selfhost-test-provider/src/v8CoverageWrangling.test.ts @@ -0,0 +1,106 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as assert from 'assert'; +import { RangeCoverageTracker } from './v8CoverageWrangling'; + +suite('v8CoverageWrangling', () => { + suite('RangeCoverageTracker', () => { + test('covers new range', () => { + const rt = new RangeCoverageTracker(); + rt.cover(5, 10); + assert.deepStrictEqual([...rt], [{ start: 5, end: 10, covered: true }]); + }); + + test('non overlapping ranges', () => { + const rt = new RangeCoverageTracker(); + rt.cover(5, 10); + rt.cover(15, 20); + assert.deepStrictEqual([...rt], [ + { start: 5, end: 10, covered: true }, + { start: 15, end: 20, covered: true }, + ]); + }); + + test('covers exact', () => { + const rt = new RangeCoverageTracker(); + rt.uncovered(5, 10); + rt.cover(5, 10); + assert.deepStrictEqual([...rt], [ + { start: 5, end: 10, covered: true }, + ]); + }); + + test('overlap at start', () => { + const rt = new RangeCoverageTracker(); + rt.uncovered(5, 10); + rt.cover(2, 7); + assert.deepStrictEqual([...rt], [ + { start: 2, end: 5, covered: true }, + { start: 5, end: 7, covered: true }, + { start: 7, end: 10, covered: false }, + ]); + }); + + test('overlap at end', () => { + const rt = new RangeCoverageTracker(); + rt.cover(5, 10); + rt.uncovered(2, 7); + assert.deepStrictEqual([...rt], [ + { start: 2, end: 5, covered: false }, + { start: 5, end: 7, covered: true }, + { start: 7, end: 10, covered: true }, + ]); + }); + + test('inner contained', () => { + const rt = new RangeCoverageTracker(); + rt.cover(5, 10); + rt.uncovered(2, 12); + assert.deepStrictEqual([...rt], [ + { start: 2, end: 5, covered: false }, + { start: 5, end: 10, covered: true }, + { start: 10, end: 12, covered: false }, + ]); + }); + + test('outer contained', () => { + const rt = new RangeCoverageTracker(); + rt.uncovered(5, 10); + rt.cover(7, 9); + assert.deepStrictEqual([...rt], [ + { start: 5, end: 7, covered: false }, + { start: 7, end: 9, covered: true }, + { start: 9, end: 10, covered: false }, + ]); + }); + + test('boundary touching', () => { + const rt = new RangeCoverageTracker(); + rt.uncovered(5, 10); + rt.cover(10, 15); + rt.uncovered(15, 20); + assert.deepStrictEqual([...rt], [ + { start: 5, end: 10, covered: false }, + { start: 10, end: 15, covered: true }, + { start: 15, end: 20, covered: false }, + ]); + }); + + test('initializeBlock', () => { + const rt = RangeCoverageTracker.initializeBlock([ + { count: 1, startOffset: 5, endOffset: 30 }, + { count: 1, startOffset: 8, endOffset: 10 }, + { count: 0, startOffset: 15, endOffset: 20 }, + ]); + + assert.deepStrictEqual([...rt], [ + { start: 5, end: 15, covered: true }, + { start: 15, end: 20, covered: false }, + { start: 20, end: 30, covered: true }, + ]); + }); + }); +}); diff --git a/.vscode/extensions/vscode-selfhost-test-provider/src/v8CoverageWrangling.ts b/.vscode/extensions/vscode-selfhost-test-provider/src/v8CoverageWrangling.ts new file mode 100644 index 00000000000..60ed7da4dd9 --- /dev/null +++ b/.vscode/extensions/vscode-selfhost-test-provider/src/v8CoverageWrangling.ts @@ -0,0 +1,156 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +export interface ICoverageRange { + start: number; + end: number; + covered: boolean; +} + +export interface IV8FunctionCoverage { + functionName: string; + isBlockCoverage: boolean; + ranges: IV8CoverageRange[]; +} + +export interface IV8CoverageRange { + startOffset: number; + endOffset: number; + count: number; +} + +/** V8 Script coverage data */ +export interface IScriptCoverage { + scriptId: string; + url: string; + // Script source added by the runner the first time the script is emitted. + source?: string; + functions: IV8FunctionCoverage[]; +} + + +export class RangeCoverageTracker implements Iterable { + /** + * A noncontiguous, non-overlapping, ordered set of ranges and whether + * that range has been covered. + */ + private ranges: readonly ICoverageRange[] = []; + + /** + * Adds a coverage tracker initialized for a function with {@link isBlockCoverage} set to true. + */ + public static initializeBlock(ranges: IV8CoverageRange[]) { + let start = ranges[0].startOffset; + const rt = new RangeCoverageTracker(); + if (!ranges[0].count) { + rt.uncovered(start, ranges[0].endOffset); + return rt; + } + + for (let i = 1; i < ranges.length; i++) { + const range = ranges[i]; + if (range.count) { + continue; + } + + rt.cover(start, range.startOffset); + rt.uncovered(range.startOffset, range.endOffset); + start = range.endOffset; + } + + rt.cover(start, ranges[0].endOffset); + return rt; + } + + /** Marks a range covered */ + public cover(start: number, end: number) { + this.setCovered(start, end, true); + } + + /** Marks a range as uncovered */ + public uncovered(start: number, end: number) { + this.setCovered(start, end, false); + } + + /** Iterates over coverage ranges */ + [Symbol.iterator]() { + return this.ranges[Symbol.iterator](); + } + + public setCovered(start: number, end: number, covered: boolean) { + const newRanges: ICoverageRange[] = []; + let i = 0; + for (; i < this.ranges.length && this.ranges[i].end <= start; i++) { + newRanges.push(this.ranges[i]); + } + + newRanges.push({ start, end, covered }); + for (; i < this.ranges.length; i++) { + const range = this.ranges[i]; + const last = newRanges[newRanges.length - 1]; + + if (range.start < last.start && range.end > last.end) { + // range contains last: + newRanges.pop(); + newRanges.push({ start: range.start, end: last.start, covered: range.covered }); + newRanges.push({ start: last.start, end: last.end, covered: range.covered || last.covered }); + newRanges.push({ start: last.end, end: range.end, covered: range.covered }); + } else if (range.start > last.start && range.end <= last.end) { + // last contains range: + newRanges.pop(); + newRanges.push({ start: last.start, end: range.start, covered: last.covered }); + newRanges.push({ start: range.start, end: range.end, covered: range.covered || last.covered }); + newRanges.push({ start: range.end, end: last.end, covered: last.covered }); + } else if (range.start < last.start && range.end <= last.end) { + // range overlaps start of last: + newRanges.pop(); + newRanges.push({ start: range.start, end: last.start, covered: range.covered }); + newRanges.push({ start: last.start, end: range.end, covered: range.covered || last.covered }); + newRanges.push({ start: range.end, end: last.end, covered: last.covered }); + } else if (range.start > last.start && range.end > last.end) { + // range overlaps end of last: + newRanges.pop(); + newRanges.push({ start: last.start, end: range.start, covered: last.covered }); + newRanges.push({ start: range.start, end: last.end, covered: range.covered || last.covered }); + newRanges.push({ start: last.end, end: range.end, covered: range.covered }); + } else { + // ranges are equal: + last.covered ||= range.covered; + } + } + + this.ranges = newRanges; + } +} + +export class OffsetToPosition { + /** Line numbers to byte offsets. */ + public readonly lines: number[] = []; + + constructor(public readonly source: string) { + this.lines.push(0); + for (let i = source.indexOf('\n'); i !== -1; i = source.indexOf('\n', i + 1)) { + this.lines.push(i + 1); + } + } + + /** + * Converts from a file offset to a base 0 line/column . + */ + public convert(offset: number): { line: number; column: number } { + let low = 0; + let high = this.lines.length; + while (low < high) { + const mid = Math.floor((low + high) / 2); + if (this.lines[mid] > offset) { + high = mid; + } else { + low = mid + 1; + } + } + + return { line: low - 1, column: offset - this.lines[low - 1] }; + } +} diff --git a/.vscode/extensions/vscode-selfhost-test-provider/tsconfig.json b/.vscode/extensions/vscode-selfhost-test-provider/tsconfig.json index 4bc025b62ba..b21867bb03d 100644 --- a/.vscode/extensions/vscode-selfhost-test-provider/tsconfig.json +++ b/.vscode/extensions/vscode-selfhost-test-provider/tsconfig.json @@ -3,7 +3,8 @@ "compilerOptions": { "outDir": "./out", "types": [ - "node" + "node", + "mocha", ] }, "include": [ diff --git a/.vscode/extensions/vscode-selfhost-test-provider/yarn.lock b/.vscode/extensions/vscode-selfhost-test-provider/yarn.lock index bf2295ed7b3..bff97849843 100644 --- a/.vscode/extensions/vscode-selfhost-test-provider/yarn.lock +++ b/.vscode/extensions/vscode-selfhost-test-provider/yarn.lock @@ -12,7 +12,7 @@ resolved "https://registry.yarnpkg.com/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz#d7c6e6755c78567a951e04ab52ef0fd26de59f32" integrity sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg== -"@jridgewell/trace-mapping@^0.3.25": +"@jridgewell/trace-mapping@^0.3.12", "@jridgewell/trace-mapping@^0.3.25": version "0.3.25" resolved "https://registry.yarnpkg.com/@jridgewell/trace-mapping/-/trace-mapping-0.3.25.tgz#15f190e98895f3fc23276ee14bc76b675c2e50f0" integrity sha512-vNk6aEwybGtawWmy/PzwnGDOjCkLWSD2wqvjGGAgOAwCGWySYXfYoxt00IJkTF+8Lb57DwOb3Aa0o9CApepiYQ== @@ -20,11 +20,16 @@ "@jridgewell/resolve-uri" "^3.1.0" "@jridgewell/sourcemap-codec" "^1.4.14" -"@types/istanbul-lib-coverage@^2.0.6": +"@types/istanbul-lib-coverage@^2.0.1", "@types/istanbul-lib-coverage@^2.0.6": version "2.0.6" resolved "https://registry.yarnpkg.com/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.6.tgz#7739c232a1fee9b4d3ce8985f314c0c6d33549d7" integrity sha512-2QF/t/auWm0lsy8XtKVPG19v3sSOQlJe/YHZgfjb/KBBHOGSV+J2q/S671rcq9uTBrLAXmZpqJiaQbMT+zNU1w== +"@types/mocha@^10.0.6": + version "10.0.6" + resolved "https://registry.yarnpkg.com/@types/mocha/-/mocha-10.0.6.tgz#818551d39113081048bdddbef96701b4e8bb9d1b" + integrity sha512-dJvrYWxP/UcXm36Qn36fxhUKu8A/xMRXVT2cliFF1Z7UA9liG5Psj3ezNSZw+5puH2czDXRLcXQxf8JbJt0ejg== + "@types/node@18.x": version "18.19.26" resolved "https://registry.yarnpkg.com/@types/node/-/node-18.19.26.tgz#18991279d0a0e53675285e8cf4a0823766349729" @@ -37,6 +42,11 @@ ansi-styles@^5.2.0: resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-5.2.0.tgz#07449690ad45777d1924ac2abb2fc8895dba836b" integrity sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA== +convert-source-map@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-2.0.0.tgz#4b560f649fc4e918dd0ab75cf4961e8bc882d82a" + integrity sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg== + istanbul-to-vscode@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/istanbul-to-vscode/-/istanbul-to-vscode-2.0.1.tgz#84994d06e604b68cac7301840f338b1e74eb888b" @@ -48,3 +58,12 @@ undici-types@~5.26.4: version "5.26.5" resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617" integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA== + +v8-to-istanbul@^9.2.0: + version "9.2.0" + resolved "https://registry.yarnpkg.com/v8-to-istanbul/-/v8-to-istanbul-9.2.0.tgz#2ed7644a245cddd83d4e087b9b33b3e62dfd10ad" + integrity sha512-/EH/sDgxU2eGxajKdwLCDmQ4FWq+kpi3uCmBGpw1xJtnAxEjlD8j8PEiGWpCIMIs3ciNAgH0d3TTJiUkYzyZjA== + dependencies: + "@jridgewell/trace-mapping" "^0.3.12" + "@types/istanbul-lib-coverage" "^2.0.1" + convert-source-map "^2.0.0"