testing: add temporary failure tracker to the selfhost test runner (#212134)

For /fixTestFailures, I want to get more 'real world' tests and test fixes. This makes a change in the selfhost test provider such that when a test fails and is then fixed, we record the code changes into a JSON file in the `.build` directory. In a few days I'll follow up with team members to collect their test failures and use them as evaluation tests for copilot. The FailureTracker will be removed when I've gotten enough data.
2026-05-02 06:21:50 +01:00 · 2024-05-06 15:54:33 -07:00
parent 3f91c9bcd7
commit 26120e5bf4
5 changed files with 151 additions and 6 deletions
--- a/.vscode/extensions/vscode-selfhost-test-provider/src/failureTracker.ts
+++ b/.vscode/extensions/vscode-selfhost-test-provider/src/failureTracker.ts
@@ -0,0 +1,126 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { spawn } from 'child_process';
+import { readFile, writeFile } from 'fs/promises';
+import { join } from 'path';
+import * as vscode from 'vscode';
+
+interface IGitState {
+	commitId: string;
+	tracked: string;
+	untracked: Record<string, string>;
+}
+
+interface ITrackedRemediation {
+	snapshot: vscode.TestResultSnapshot;
+	failing: IGitState;
+	passing: IGitState;
+}
+
+const MAX_FAILURES = 10;
+
+export class FailureTracker {
+	private readonly disposables: vscode.Disposable[] = [];
+	private readonly lastFailed = new Map<
+		string,
+		{ snapshot: vscode.TestResultSnapshot; failing: IGitState }
+	>();
+
+	private readonly logFile = join(this.rootDir, '.build/vscode-test-failures.json');
+	private logs?: ITrackedRemediation[];
+
+	constructor(private readonly rootDir: string) {
+		this.disposables.push(
+			vscode.tests.onDidChangeTestResults(() => {
+				const last = vscode.tests.testResults[0];
+				if (!last) {
+					return;
+				}
+
+				let gitState: Promise<IGitState> | undefined;
+				const getGitState = () => gitState ?? (gitState = this.captureGitState());
+
+				const queue = [last.results];
+				for (let i = 0; i < queue.length; i++) {
+					for (const snapshot of queue[i]) {
+						// only interested in states of leaf tests
+						if (snapshot.children.length) {
+							queue.push(snapshot.children);
+							continue;
+						}
+
+						const key = `${snapshot.uri}/${snapshot.id}`;
+						const prev = this.lastFailed.get(key);
+						if (snapshot.taskStates.some(s => s.state === vscode.TestResultState.Failed)) {
+							// unset the parent to avoid a circular JSON structure:
+							getGitState().then(s => this.lastFailed.set(key, { snapshot: { ...snapshot, parent: undefined }, failing: s }));
+						} else if (prev) {
+							this.lastFailed.delete(key);
+							getGitState().then(s => this.append({ ...prev, passing: s }));
+						}
+					}
+				}
+			})
+		);
+	}
+
+	private async append(log: ITrackedRemediation) {
+		if (!this.logs) {
+			try {
+				this.logs = JSON.parse(await readFile(this.logFile, 'utf-8'));
+			} catch {
+				this.logs = [];
+			}
+		}
+
+		const logs = this.logs!;
+		logs.push(log);
+		if (logs.length > MAX_FAILURES) {
+			logs.splice(0, logs.length - MAX_FAILURES);
+		}
+
+		await writeFile(this.logFile, JSON.stringify(logs, undefined, 2));
+	}
+
+	private async captureGitState() {
+		const [commitId, tracked, untracked] = await Promise.all([
+			this.exec('git', ['rev-parse', 'HEAD']),
+			this.exec('git', ['diff', 'HEAD']),
+			this.exec('git', ['ls-files', '--others', '--exclude-standard']).then(async output => {
+				const mapping: Record<string, string> = {};
+				await Promise.all(
+					output
+						.trim()
+						.split('\n')
+						.map(async f => {
+							mapping[f] = await readFile(join(this.rootDir, f), 'utf-8');
+						})
+				);
+				return mapping;
+			}),
+		]);
+		return { commitId, tracked, untracked };
+	}
+
+	public dispose() {
+		this.disposables.forEach(d => d.dispose());
+	}
+
+	private exec(command: string, args: string[]): Promise<string> {
+		return new Promise((resolve, reject) => {
+			const child = spawn(command, args, { stdio: 'pipe', cwd: this.rootDir });
+			let output = '';
+			child.stdout.setEncoding('utf-8').on('data', b => (output += b));
+			child.stderr.setEncoding('utf-8').on('data', b => (output += b));
+			child.on('error', reject);
+			child.on('exit', code =>
+				code === 0
+					? resolve(output)
+					: reject(new Error(`Failed with error code ${code}\n${output}`))
+			);
+		});
+	}
+}