vscode/test/integration/agentHost/runner.ts

/*---------------------------------------------------------------------------------------------
 *  Copyright (c) Microsoft Corporation. All rights reserved.
 *  Licensed under the MIT License. See License.txt in the project root for license information.
 *--------------------------------------------------------------------------------------------*/

const childProcess: typeof import('child_process') = require('child_process');
const fs: typeof import('fs') = require('fs');
const os: typeof import('os') = require('os');
const path: typeof import('path') = require('path');
const { spawn, spawnSync } = childProcess;
const { appendFileSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } = fs;
const { availableParallelism, cpus, tmpdir } = os;
const { basename, dirname, extname, join, resolve } = path;

const repoRoot = resolve(__dirname, '../../..');
const testScript = join(repoRoot, 'scripts', process.platform === 'win32' ? 'test-integration.bat' : 'test-integration.sh');
const windowsTestWrapper = join(__dirname, 'child.ps1');
const incompatibleFlags = [
	'AGENT_HOST_REPLAY_RECORD',
	'AGENT_HOST_UPDATE_AHP_SNAPSHOTS',
	'AGENT_HOST_UPDATE_SNAPSHOTS',
];
type StorageMode = 'disk' | 'tmpfs' | 'split';
const diskLifecyclePattern = [
	'session metadata history and provider context survive a host restart',
	'archiving a never-restored session survives a host restart',
	'a created automation survives an agent host restart',
	'automation lifecycle: cancelled request deduplication survives a restart',
	'automation lifecycle: an unavailable model produces a durable failed run',
	'regression coverage: cold resume retains ordered user and assistant context',
	'regression coverage: identical prompts remain separate completed turns after resume',
	'detached worktree safety: archive commits local edits and restores them on unarchive',
	'detached worktree safety: archived handles remain restorable after a host restart',
	'detached worktree safety: deletion recovers when the checkout was already removed externally',
].join('|');

interface ISuite {
	readonly id: string;
	readonly label: string;
	readonly file: string;
}

interface IRunResult {
	readonly suite: ISuite;
	readonly succeeded: boolean;
	readonly durationSeconds: number;
	readonly failure?: string;
	readonly failureDetails?: string;
}

interface IObservedSurface {
	readonly commands: readonly string[];
	readonly notifications: readonly string[];
	readonly actions: readonly string[];
}

const suites: readonly ISuite[] = [
	{
		id: 'conformance',
		label: 'Conformance',
		file: 'src/vs/platform/agentHost/test/node/e2e/conformance/agentHostConformance.integrationTest.ts',
	},
	{
		id: 'claude',
		label: 'Claude',
		file: 'src/vs/platform/agentHost/test/node/e2e/providers/claudeAgentHostE2E.integrationTest.ts',
	},
	{
		id: 'codex',
		label: 'Codex',
		file: 'src/vs/platform/agentHost/test/node/e2e/providers/codexAgentHostE2E.integrationTest.ts',
	},
	{
		id: 'copilot',
		label: 'Copilot',
		file: 'src/vs/platform/agentHost/test/node/e2e/providers/copilotAgentHostE2E.integrationTest.ts',
	},
	{
		id: 'copilot-otel',
		label: 'Copilot OTel',
		file: 'src/vs/platform/agentHost/test/node/e2e/providers/copilotOtelAgentHostE2E.integrationTest.ts',
	},
	{
		id: 'copilot-managed-settings',
		label: 'Copilot managed settings',
		file: 'src/vs/platform/agentHost/test/node/e2e/providers/copilotManagedSettingsAgentHostE2E.integrationTest.ts',
	},
];

async function main(): Promise<void> {
	validateEnvironment();
	const { jobs, storage, forwardedArgs } = parseArguments(process.argv.slice(2));
	await prepareTestRuntime();
	if (storage !== 'disk') {
		runStoragePasses(storage, jobs, forwardedArgs);
		return;
	}
	const storageBacking = process.env['AGENT_HOST_E2E_STORAGE_BACKING'] ?? 'disk';
	if (process.platform === 'linux') {
		const filesystem = fs.statfsSync(tmpdir());
		if (storageBacking === 'tmpfs' && filesystem.type !== 0x01021994) {
			throw new Error(`RAM-backed E2E tests require tmpfs temporary storage: ${tmpdir()}`);
		}
		if (storageBacking === 'disk' && (filesystem.type === 0x01021994 || filesystem.type === 0x858458f6)) {
			throw new Error(`Disk-backed E2E tests require non-tmpfs temporary storage: ${tmpdir()}`);
		}
		console.log(`Agent Host E2E storage: ${storageBacking}, temporary directory=${tmpdir()}, filesystemType=${filesystem.type}`);
	}

	const startedAt = process.hrtime.bigint();
	const surfaceOutputs = prepareSurfaceOutputs();
	const results: IRunResult[] = [];
	let nextSuite = 0;

	const stopResourceDiagnostics = startResourceDiagnostics(jobs);
	try {
		const workers = Array.from({ length: jobs }, async () => {
			while (nextSuite < suites.length) {
				const suiteIndex = nextSuite++;
				const suite = suites[suiteIndex];
				results[suiteIndex] = await runSuite(suite, forwardedArgs, surfaceOutputs.get(suite.id));
			}
		});
		await Promise.all(workers);
	} finally {
		stopResourceDiagnostics();
	}

	const failures = results.filter(result => !result.succeeded);
	if (surfaceOutputs.size > 0 && failures.length === 0) {
		mergeSurfaceOutputs(surfaceOutputs);
	}

	const durationSeconds = elapsedSeconds(startedAt);
	console.log(`\nAgent Host E2E suites completed in ${durationSeconds.toFixed(1)}s (${jobs} parallel ${jobs === 1 ? 'worker' : 'workers'}).`);
	for (const result of results) {
		console.log(`  ${result.succeeded ? 'PASS' : 'FAIL'} ${result.suite.label}: ${result.durationSeconds.toFixed(1)}s`);
	}
	if (failures.length > 0) {
		printFailureDetails(failures);
		process.exitCode = 1;
	}
}

function runStoragePasses(storage: 'tmpfs' | 'split', jobs: number, args: readonly string[]): void {
	const environment = { ...process.env, VSCODE_SKIP_PRELAUNCH: '1' };
	delete environment.ELECTRON_RUN_AS_NODE;
	const runnerArgs = [__filename, '--storage', 'disk', '--jobs', String(jobs)];
	const tmpfs = spawnSync('bash', [
		join(__dirname, 'tmpfs.sh'),
		process.execPath, ...runnerArgs, ...args,
	], { cwd: repoRoot, env: { ...environment, TMPDIR: '/tmp' }, stdio: 'inherit' });
	if (tmpfs.error) {
		throw tmpfs.error;
	}
	if (tmpfs.status !== 0) {
		process.exitCode = 1;
	}
	if (storage === 'split') {
		const diskArgs = args.filter((argument, index) =>
			!['--grep', '-g', '-f', '--invert'].includes(argument)
			&& !['--grep', '-g', '-f'].includes(args[index - 1])
			&& !argument.startsWith('--grep='));
		console.log('\nRunning required disk-backed Agent Host persistence and lifecycle coverage.');
		const disk = spawnSync(process.execPath, [...runnerArgs, ...diskArgs, '--grep', diskLifecyclePattern], {
			cwd: repoRoot,
			env: {
				...environment,
				AGENT_HOST_E2E_STORAGE_BACKING: 'disk',
				AGENT_HOST_RECORD_PROTOCOL_SURFACE: '0',
				AGENT_HOST_E2E_COVERAGE: '0',
			},
			stdio: 'inherit',
		});
		if (disk.error) {
			throw disk.error;
		}
		if (disk.status !== 0) {
			process.exitCode = 1;
		}
	}
}

function startResourceDiagnostics(jobs: number): () => void {
	if (process.platform !== 'linux') {
		return () => { };
	}
	const directory = join(repoRoot, '.build', 'logs', 'integration-tests');
	mkdirSync(directory, { recursive: true });
	const output = join(directory, `agent-host-resources-${process.pid}.jsonl`);
	const files = ['/proc/pressure/cpu', '/proc/pressure/io', '/proc/pressure/memory', '/proc/stat', '/proc/meminfo', '/proc/diskstats', '/proc/self/mountinfo'];
	const sample = () => {
		try {
			const resources = Object.fromEntries(files.map(file => [file, existsSync(file) ? readFileSync(file, 'utf8') : 'unavailable']));
			appendFileSync(output, JSON.stringify({
				timestamp: new Date().toISOString(),
				jobs,
				storage: process.env['AGENT_HOST_E2E_STORAGE_BACKING'] ?? 'disk',
				tmpdir: tmpdir(),
				resources,
			}) + '\n');
		} catch (error) {
			console.warn(`[agent-host-e2e] Failed to collect worker resource diagnostics: ${error instanceof Error ? error.message : String(error)}`);
		}
	};
	sample();
	const timer = setInterval(sample, 5_000);
	timer.unref();
	return () => {
		clearInterval(timer);
		sample();
	};
}

function validateEnvironment(): void {
	const enabledFlags = incompatibleFlags.filter(flag => process.env[flag] === '1');
	if (enabledFlags.length > 0) {
		throw new Error(`Parallel Agent Host E2E runs only support deterministic replay; unset ${enabledFlags.join(', ')}`);
	}
}

function parseArguments(args: readonly string[]): { jobs: number; storage: StorageMode; forwardedArgs: readonly string[] } {
	const forwardedArgs: string[] = [];
	let requestedJobs: string | undefined = process.env['AGENT_HOST_E2E_JOBS'];
	const linuxCI = process.platform === 'linux' && (process.env['CI'] === 'true' || process.env['TF_BUILD']?.toLowerCase() === 'true');
	let requestedStorage = process.env['AGENT_HOST_E2E_STORAGE'] ?? (linuxCI ? 'split' : 'disk');

	for (let index = 0; index < args.length; index++) {
		const argument = args[index];
		if (argument === '--storage') {
			requestedStorage = args[++index];
		} else if (argument.startsWith('--storage=')) {
			requestedStorage = argument.slice('--storage='.length);
		} else if (argument === '--jobs') {
			requestedJobs = args[++index];
			if (!requestedJobs) {
				throw new Error('--jobs requires a value');
			}
		} else if (argument.startsWith('--jobs=')) {
			requestedJobs = argument.slice('--jobs='.length);
		} else {
			forwardedArgs.push(argument);
		}
	}

	const ownedArguments = ['--run', '--runGlob', '--glob', '--runGrep', '--testSplit'];
	const conflictingArgument = forwardedArgs.find(argument => ownedArguments.some(owned => argument === owned || argument.startsWith(`${owned}=`)));
	if (conflictingArgument) {
		throw new Error(`${conflictingArgument} is managed by the Agent Host E2E runner`);
	}

	const defaultJobs = Math.min(suites.length, availableParallelism?.() ?? cpus().length);
	const jobs = requestedJobs === undefined ? defaultJobs : Number(requestedJobs);
	if (!Number.isInteger(jobs) || jobs < 1) {
		throw new Error(`Invalid Agent Host E2E worker count: ${requestedJobs}`);
	}
	if (requestedStorage !== 'disk' && requestedStorage !== 'tmpfs' && requestedStorage !== 'split') {
		throw new Error(`Invalid Agent Host E2E storage mode: ${requestedStorage}`);
	}
	if (requestedStorage !== 'disk' && process.platform !== 'linux') {
		throw new Error('Agent Host tmpfs storage requires Linux');
	}
	return { jobs: Math.min(jobs, suites.length), storage: requestedStorage, forwardedArgs };
}

async function prepareTestRuntime(): Promise<void> {
	const environment = { ...process.env };
	delete environment.ELECTRON_RUN_AS_NODE;

	mkdirSync(join(repoRoot, '.build', 'crashes'), { recursive: true });
	if (!existsSync(join(repoRoot, 'node_modules'))) {
		runSync(process.platform === 'win32' ? 'npm.cmd' : 'npm', ['install'], environment);
	}
	const { shouldDownloadElectron } = await import('../../../build/lib/electronVersion.ts');
	if (shouldDownloadElectron(repoRoot, environment)) {
		runSync(process.platform === 'win32' ? 'npm.cmd' : 'npm', ['run', 'electron'], environment);
	}
}

function runSync(command: string, args: readonly string[], environment: NodeJS.ProcessEnv): void {
	const result = spawnSync(command, args, {
		cwd: repoRoot,
		env: environment,
		stdio: 'inherit',
	});
	if (result.error) {
		throw result.error;
	}
	if (result.status !== 0) {
		const reason = result.signal ? `signal ${result.signal}` : `code ${result.status}`;
		throw new Error(`${command} exited with ${reason}`);
	}
}

async function runSuite(suite: ISuite, forwardedArgs: readonly string[], surfaceOutput: string | undefined): Promise<IRunResult> {
	console.log(`Starting Agent Host E2E — ${suite.label}`);
	const startedAt = process.hrtime.bigint();
	const environment = {
		...process.env,
		VSCODE_SKIP_PRELAUNCH: '1',
		...(surfaceOutput ? { AGENT_HOST_PROTOCOL_SURFACE_OUT: surfaceOutput } : {}),
	};
	delete environment.ELECTRON_RUN_AS_NODE;

	return new Promise(resolveResult => {
		const testArguments = ['--run', suite.file, ...suiteArguments(forwardedArgs, suite)];
		const child = process.platform === 'win32'
			? spawn(join(process.env['SYSTEMROOT'] ?? 'C:\\Windows', 'System32', 'WindowsPowerShell', 'v1.0', 'powershell.exe'), [
				'-NoLogo',
				'-NoProfile',
				'-NonInteractive',
				'-ExecutionPolicy', 'Bypass',
				'-File', windowsTestWrapper,
				testScript,
				...testArguments,
			], {
				cwd: repoRoot,
				env: environment,
				stdio: ['ignore', 'pipe', 'pipe'],
			})
			: spawn(testScript, testArguments, {
				cwd: repoRoot,
				env: environment,
				stdio: ['ignore', 'pipe', 'pipe'],
			});
		let output = '';
		child.stdout.setEncoding('utf8');
		child.stderr.setEncoding('utf8');
		child.stdout.on('data', chunk => output += chunk);
		child.stderr.on('data', chunk => output += chunk);
		child.on('error', error => {
			resolveResult({
				suite,
				succeeded: false,
				durationSeconds: elapsedSeconds(startedAt),
				failure: error.message,
				failureDetails: extractFailureDetails(output),
			});
		});
		child.on('close', (code, signal) => {
			const succeeded = code === 0;
			const failure = succeeded ? undefined : signal ? `signal ${signal}` : `code ${code}`;
			console.log(`\n===== Agent Host E2E — ${suite.label} =====`);
			process.stdout.write(output);
			if (!output.endsWith('\n')) {
				process.stdout.write('\n');
			}
			resolveResult({
				suite,
				succeeded,
				durationSeconds: elapsedSeconds(startedAt),
				failure,
				failureDetails: succeeded ? undefined : extractFailureDetails(output),
			});
		});
	});
}

function extractFailureDetails(output: string): string | undefined {
	const lines = output.split(/\r?\n/);
	for (let index = lines.length - 1; index >= 0; index--) {
		const line = lines[index].replace(/\x1b\[[0-9;]*m/g, '').trim();
		if (/^\d+ failing$/.test(line)) {
			return `${lines.slice(index).join('\n').trimEnd()}\n`;
		}
	}

	const trimmed = output.trim();
	return trimmed.length > 0 ? `${trimmed}\n` : undefined;
}

function printFailureDetails(failures: readonly IRunResult[]): void {
	console.log('\nAgent Host E2E failure details:');
	for (const result of failures) {
		console.log(`\n===== Agent Host E2E — ${result.suite.label} failure =====`);
		if (result.failureDetails) {
			process.stdout.write(result.failureDetails);
		}
		console.log(`Agent Host E2E — ${result.suite.label} failed with ${result.failure ?? 'an unknown error'}`);
	}
}

function suiteArguments(args: readonly string[], suite: ISuite): readonly string[] {
	const result = [...args];
	const tfsIndex = result.indexOf('--tfs');
	if (tfsIndex >= 0 && result[tfsIndex + 1]) {
		const storage = process.env['AGENT_HOST_E2E_STORAGE_BACKING'];
		result[tfsIndex + 1] = `${result[tfsIndex + 1]} ${suite.label}${storage ? ` ${storage}` : ''}`;
	}
	return result;
}

function prepareSurfaceOutputs(): ReadonlyMap<string, string> {
	if (process.env['AGENT_HOST_RECORD_PROTOCOL_SURFACE'] !== '1') {
		return new Map();
	}

	const combinedOutput = process.env['AGENT_HOST_PROTOCOL_SURFACE_OUT']
		?? join(repoRoot, '.build', 'agent-host-e2e-coverage', 'protocol-surface', 'observed.json');
	const extension = extname(combinedOutput);
	const stem = basename(combinedOutput, extension);
	const outputs = new Map<string, string>();
	for (const suite of suites) {
		const output = join(dirname(combinedOutput), `${stem}-${suite.id}${extension}`);
		rmSync(output, { force: true });
		outputs.set(suite.id, output);
	}
	return outputs;
}

function mergeSurfaceOutputs(outputs: ReadonlyMap<string, string>): void {
	const combinedOutput = process.env['AGENT_HOST_PROTOCOL_SURFACE_OUT']
		?? join(repoRoot, '.build', 'agent-host-e2e-coverage', 'protocol-surface', 'observed.json');
	const commands = new Set<string>();
	const notifications = new Set<string>();
	const actions = new Set<string>();

	for (const output of outputs.values()) {
		if (!existsSync(output)) {
			if (process.env['AGENT_HOST_E2E_COVERAGE'] === '1') {
				throw new Error(`Missing protocol surface observations from ${output}`);
			}
			continue;
		}
		const observed = readObservedSurface(output);
		observed.commands.forEach(command => commands.add(command));
		observed.notifications.forEach(notification => notifications.add(notification));
		observed.actions.forEach(action => actions.add(action));
		rmSync(output, { force: true });
	}

	mkdirSync(dirname(combinedOutput), { recursive: true });
	writeFileSync(combinedOutput, `${JSON.stringify({
		commands: [...commands].sort(),
		notifications: [...notifications].sort(),
		actions: [...actions].sort(),
	}, undefined, '\t')}\n`);
}

function readObservedSurface(file: string): IObservedSurface {
	const value: unknown = JSON.parse(readFileSync(file, 'utf8'));
	if (!isRecord(value)
		|| !isStringArray(value.commands)
		|| !isStringArray(value.notifications)
		|| !isStringArray(value.actions)
	) {
		throw new Error(`Invalid protocol surface observations in ${file}`);
	}
	return {
		commands: value.commands,
		notifications: value.notifications,
		actions: value.actions,
	};
}

function isRecord(value: unknown): value is Record<string, unknown> {
	return typeof value === 'object' && value !== null;
}

function isStringArray(value: unknown): value is readonly string[] {
	return Array.isArray(value) && value.every(entry => typeof entry === 'string');
}

function elapsedSeconds(startedAt: bigint): number {
	return Number(process.hrtime.bigint() - startedAt) / 1_000_000_000;
}

main().catch(error => {
	console.error(error);
	process.exitCode = 1;
});