mirror of
https://github.com/microsoft/vscode.git
synced 2026-05-15 12:51:00 +01:00
Paul
1936 lines
84 KiB
JavaScript
1936 lines
84 KiB
JavaScript
/*---------------------------------------------------------------------------------------------
|
|
* Copyright (c) Microsoft Corporation. All rights reserved.
|
|
* Licensed under the MIT License. See License.txt in the project root for license information.
|
|
*--------------------------------------------------------------------------------------------*/
|
|
|
|
// @ts-check
|
|
|
|
/**
|
|
* Chat performance benchmark.
|
|
*
|
|
* Uses the real copilot extension with IS_SCENARIO_AUTOMATION=1 and a local
|
|
* mock LLM server. Measures the full stack: prompt building, context
|
|
* gathering, tool resolution, rendering, GC, and layout overhead.
|
|
*
|
|
* Usage:
|
|
* npm run perf:chat # all scenarios vs 1.115.0
|
|
* npm run perf:chat -- --runs 10 # 10 runs per scenario
|
|
* npm run perf:chat -- --scenario text-only # single scenario
|
|
* npm run perf:chat -- --no-baseline # skip baseline comparison
|
|
* npm run perf:chat -- --build 1.110.0 --baseline-build 1.115.0
|
|
* npm run perf:chat -- --resume .chat-simulation-data/2026-04-14/results.json --runs 3
|
|
*/
|
|
|
|
const path = require('path');
|
|
const fs = require('fs');
|
|
const {
|
|
ROOT, DATA_DIR, METRIC_DEFS, loadConfig,
|
|
resolveBuild, isVersionString, buildEnv, buildArgs, prepareRunDir,
|
|
robustStats, welchTTest, summarize, markDuration, launchVSCode,
|
|
getNextExtHostInspectPort, connectToExtHostInspector, getRepoRoot,
|
|
} = require('./common/utils');
|
|
const { getUserTurns, getScenarioIds } = require('./common/mock-llm-server');
|
|
const { registerPerfScenarios, getScenarioDescription } = require('./common/perf-scenarios');
|
|
|
|
// -- Config (edit config.jsonc to change defaults) ---------------------------
|
|
|
|
const CONFIG = loadConfig('perfRegression');
|
|
|
|
// -- CLI args ----------------------------------------------------------------
|
|
|
|
function parseArgs() {
|
|
const args = process.argv.slice(2);
|
|
const opts = {
|
|
runs: CONFIG.runsPerScenario ?? 5,
|
|
verbose: false,
|
|
ci: false,
|
|
noCache: false,
|
|
force: false,
|
|
heapSnapshots: false,
|
|
/** @type {string[]} */
|
|
scenarios: [],
|
|
/** @type {string | undefined} */
|
|
build: undefined,
|
|
/** @type {string | undefined} */
|
|
baseline: undefined,
|
|
/** @type {string | undefined} */
|
|
baselineBuild: CONFIG.baselineBuild ?? '1.115.0',
|
|
saveBaseline: false,
|
|
threshold: CONFIG.regressionThreshold ?? 0.2,
|
|
/** @type {Record<string, number | string>} */
|
|
metricThresholds: CONFIG.metricThresholds ?? {},
|
|
/** @type {string | undefined} */
|
|
resume: undefined,
|
|
productionBuild: false,
|
|
/** @type {Record<string, any>} */
|
|
settingsOverrides: {},
|
|
/** @type {Record<string, any>} */
|
|
testSettingsOverrides: {},
|
|
/** @type {Record<string, any>} */
|
|
baselineSettingsOverrides: {},
|
|
cleanupDiagnostics: false,
|
|
};
|
|
for (let i = 0; i < args.length; i++) {
|
|
switch (args[i]) {
|
|
case '--runs': opts.runs = parseInt(args[++i], 10); break;
|
|
case '--verbose': opts.verbose = true; break;
|
|
case '--scenario': case '-s': opts.scenarios.push(args[++i]); break;
|
|
case '--build': case '-b': opts.build = args[++i]; break;
|
|
case '--baseline': opts.baseline = args[++i]; break;
|
|
case '--baseline-build': opts.baselineBuild = args[++i]; break;
|
|
case '--no-baseline': opts.baselineBuild = undefined; break;
|
|
case '--save-baseline': opts.saveBaseline = true; break;
|
|
case '--threshold': opts.threshold = parseFloat(args[++i]); break;
|
|
case '--resume': opts.resume = args[++i]; break;
|
|
case '--production-build': opts.productionBuild = true; break;
|
|
case '--setting': case '--test-setting': case '--baseline-setting': {
|
|
const kv = args[++i];
|
|
const eq = kv.indexOf('=');
|
|
if (eq === -1) { console.error(`${args[i - 1]} requires key=value, got: ${kv}`); process.exit(1); }
|
|
const key = kv.slice(0, eq);
|
|
const raw = kv.slice(eq + 1);
|
|
// Parse booleans and numbers, keep rest as strings
|
|
const val = raw === 'true' ? true : raw === 'false' ? false : /^-?\d+(\.\d+)?$/.test(raw) ? Number(raw) : raw;
|
|
const flag = args[i - 1];
|
|
if (flag === '--test-setting') { opts.testSettingsOverrides[key] = val; }
|
|
else if (flag === '--baseline-setting') { opts.baselineSettingsOverrides[key] = val; }
|
|
else { opts.settingsOverrides[key] = val; }
|
|
break;
|
|
}
|
|
case '--no-cache': opts.noCache = true; break;
|
|
case '--force': opts.force = true; break;
|
|
case '--heap-snapshots': opts.heapSnapshots = true; break;
|
|
case '--ci': opts.ci = true; opts.noCache = true; opts.heapSnapshots = true; opts.cleanupDiagnostics = true; break;
|
|
case '--cleanup-diagnostics': opts.cleanupDiagnostics = true; break;
|
|
case '--help': case '-h':
|
|
console.log([
|
|
'Chat performance benchmark',
|
|
'',
|
|
'Options:',
|
|
' --runs <n> Number of runs per scenario (default: 5)',
|
|
' --scenario <id> Scenario to run (repeatable; default: all)',
|
|
' --build <path|ver> Path to VS Code build, or a version to download',
|
|
' (e.g. "1.110.0", "insiders", commit hash, or local path)',
|
|
' --baseline <path> Compare against a baseline JSON file',
|
|
' --baseline-build <v> Version or path to benchmark as baseline',
|
|
' (e.g. "1.115.0", "insiders", commit hash, or local path)',
|
|
' --no-baseline Skip baseline comparison entirely',
|
|
' --save-baseline Save results as the new baseline (requires --baseline <path>)',
|
|
' --resume <path> Resume a previous run, adding more iterations to increase',
|
|
' confidence. Merges new runs with existing rawRuns data',
|
|
' --threshold <frac> Regression threshold fraction (default: 0.2 = 20%)',
|
|
' --production-build Build a local bundled package (via gulp vscode) for',
|
|
' apples-to-apples comparison against a release baseline',
|
|
' --setting <k=v> Set a VS Code setting override for all builds (repeatable)',
|
|
' --test-setting <k=v> Set a VS Code setting override for test build only',
|
|
' --baseline-setting <k=v> Set a VS Code setting override for baseline build only',
|
|
' e.g. --setting chat.experimental.incrementalRendering.enabled=true',
|
|
' --no-cache Ignore cached baseline data, always run fresh',
|
|
' --force Skip build mode mismatch confirmation',
|
|
' --heap-snapshots Take heap snapshots (slow; auto-enabled in --ci mode)',
|
|
' --ci CI mode: write Markdown summary to ci-summary.md (implies --no-cache, --heap-snapshots, --cleanup-diagnostics)',
|
|
' --cleanup-diagnostics Remove heap snapshots, CPU profiles, and traces after each run to save disk space',
|
|
' --verbose Print per-run details',
|
|
'',
|
|
'Scenarios: ' + getScenarioIds().join(', '),
|
|
].join('\n'));
|
|
process.exit(0);
|
|
}
|
|
}
|
|
if (opts.scenarios.length === 0) {
|
|
opts.scenarios = getScenarioIds();
|
|
} else {
|
|
const knownIds = new Set(getScenarioIds());
|
|
const unknown = opts.scenarios.filter(s => !knownIds.has(s));
|
|
if (unknown.length > 0) {
|
|
console.error(`Unknown scenario(s): ${unknown.join(', ')}\nAvailable: ${[...knownIds].join(', ')}`);
|
|
process.exit(1);
|
|
}
|
|
}
|
|
return opts;
|
|
}
|
|
|
|
// -- Build mode detection ----------------------------------------------------
|
|
|
|
/**
|
|
* Classify an electron path into a build mode.
|
|
* @param {string} electronPath
|
|
* @returns {'dev' | 'production' | 'release'}
|
|
*/
|
|
function detectBuildMode(electronPath) {
|
|
if (electronPath.includes('.vscode-test')) {
|
|
return 'release';
|
|
}
|
|
if (electronPath.includes('VSCode-')) {
|
|
return 'production';
|
|
}
|
|
return 'dev';
|
|
}
|
|
|
|
/**
|
|
* Return a human-readable label for a build mode.
|
|
* @param {'dev' | 'production' | 'release'} mode
|
|
* @returns {string}
|
|
*/
|
|
function buildModeLabel(mode) {
|
|
switch (mode) {
|
|
case 'dev': return 'development (unbundled)';
|
|
case 'production': return 'production (bundled, local)';
|
|
case 'release': return 'release (bundled, downloaded)';
|
|
}
|
|
}
|
|
|
|
// -- Production build --------------------------------------------------------
|
|
|
|
/**
|
|
* Build a local production (bundled) VS Code package using `gulp vscode`.
|
|
* Returns the path to the Electron executable in the packaged output.
|
|
*
|
|
* The gulp task compiles TypeScript, bundles JS, and packages with Electron
|
|
* into `../VSCode-<platform>-<arch>/`. This is the same process used for
|
|
* release builds, minus minification and mangling.
|
|
*/
|
|
function buildProductionBuild() {
|
|
const product = require(path.join(ROOT, 'product.json'));
|
|
const platform = process.platform;
|
|
const arch = process.arch;
|
|
const destDir = path.join(ROOT, '..', `VSCode-${platform}-${arch}`);
|
|
|
|
console.log('[chat-simulation] Building local production package (gulp vscode)...');
|
|
console.log('[chat-simulation] This may take a few minutes on the first run.');
|
|
|
|
const { execSync } = require('child_process');
|
|
try {
|
|
execSync('npm run gulp -- vscode', {
|
|
cwd: ROOT,
|
|
stdio: 'inherit',
|
|
timeout: 10 * 60 * 1000, // 10 minute timeout
|
|
});
|
|
} catch (e) {
|
|
// The copilot shim step may fail locally when the copilot SDK is not
|
|
// fully packaged (it is normally supplied via CI). As long as the
|
|
// Electron executable was produced we can still benchmark.
|
|
console.warn('[chat-simulation] gulp vscode exited with errors (see above). Checking if executable was still produced...');
|
|
}
|
|
|
|
/** @type {string} */
|
|
let electronPath;
|
|
if (platform === 'darwin') {
|
|
electronPath = path.join(destDir, `${product.nameLong}.app`, 'Contents', 'MacOS', product.nameShort);
|
|
} else if (platform === 'linux') {
|
|
electronPath = path.join(destDir, product.applicationName);
|
|
} else {
|
|
electronPath = path.join(destDir, `${product.nameShort}.exe`);
|
|
}
|
|
|
|
if (!fs.existsSync(electronPath)) {
|
|
console.error(`[chat-simulation] Production build failed — executable not found at: ${electronPath}`);
|
|
process.exit(1);
|
|
}
|
|
|
|
// Merge product.overrides.json into the packaged product.json.
|
|
// The overrides file contains extensionsGallery and other config that
|
|
// the OSS product.json lacks. In dev builds these are loaded at
|
|
// runtime when VSCODE_DEV is set, but the production build doesn't
|
|
// set that flag so we bake them in.
|
|
const overridesPath = path.join(ROOT, 'product.overrides.json');
|
|
if (fs.existsSync(overridesPath)) {
|
|
/** @type {string} */
|
|
let appDir;
|
|
if (platform === 'darwin') {
|
|
appDir = path.join(destDir, `${product.nameLong}.app`, 'Contents', 'Resources', 'app');
|
|
} else {
|
|
appDir = path.join(destDir, 'resources', 'app');
|
|
}
|
|
const packagedProductPath = path.join(appDir, 'product.json');
|
|
if (fs.existsSync(packagedProductPath)) {
|
|
const packagedProduct = JSON.parse(fs.readFileSync(packagedProductPath, 'utf-8'));
|
|
const overrides = JSON.parse(fs.readFileSync(overridesPath, 'utf-8'));
|
|
const merged = Object.assign(packagedProduct, overrides);
|
|
fs.writeFileSync(packagedProductPath, JSON.stringify(merged, null, '\t'));
|
|
console.log('[chat-simulation] Merged product.overrides.json into packaged product.json');
|
|
}
|
|
}
|
|
|
|
console.log(`[chat-simulation] Production build ready: ${electronPath}`);
|
|
return electronPath;
|
|
}
|
|
|
|
/**
|
|
* @typedef {{ type: 'fraction', value: number } | { type: 'absolute', value: number }} MetricThreshold
|
|
*/
|
|
|
|
/**
|
|
* Parse a metric threshold value from config.
|
|
* - A number is treated as a fraction (e.g. 0.2 = 20%).
|
|
* - A string like "100ms" or "5" is treated as an absolute delta.
|
|
* @param {number | string} raw
|
|
* @returns {MetricThreshold}
|
|
*/
|
|
function parseMetricThreshold(raw) {
|
|
if (typeof raw === 'number') {
|
|
return { type: 'fraction', value: raw };
|
|
}
|
|
// Strip unit suffix (ms, MB, etc.) and parse the number
|
|
const num = parseFloat(raw);
|
|
if (isNaN(num)) {
|
|
throw new Error(`Invalid metric threshold: ${raw}`);
|
|
}
|
|
return { type: 'absolute', value: num };
|
|
}
|
|
|
|
/**
|
|
* Get the regression threshold for a specific metric.
|
|
* Uses per-metric override from config if available, otherwise the global threshold.
|
|
* @param {{ threshold: number, metricThresholds?: Record<string, number | string> }} opts
|
|
* @param {string} metric
|
|
* @returns {MetricThreshold}
|
|
*/
|
|
function getMetricThreshold(opts, metric) {
|
|
const raw = opts.metricThresholds?.[metric];
|
|
if (raw !== undefined) {
|
|
return parseMetricThreshold(raw);
|
|
}
|
|
return { type: 'fraction', value: opts.threshold };
|
|
}
|
|
|
|
/**
|
|
* Check whether a change exceeds the threshold.
|
|
* @param {MetricThreshold} threshold
|
|
* @param {number} change - fractional change (e.g. 0.5 = 50% increase)
|
|
* @param {number} absoluteDelta - absolute difference (cur.median - bas.median)
|
|
* @returns {boolean}
|
|
*/
|
|
function exceedsThreshold(threshold, change, absoluteDelta) {
|
|
if (threshold.type === 'absolute') {
|
|
return absoluteDelta > threshold.value;
|
|
}
|
|
return change > threshold.value;
|
|
}
|
|
|
|
// -- Metrics -----------------------------------------------------------------
|
|
|
|
/**
|
|
* @typedef {{
|
|
* timeToUIUpdated: number,
|
|
* timeToFirstToken: number,
|
|
* timeToComplete: number,
|
|
* timeToRenderComplete: number,
|
|
* instructionCollectionTime: number,
|
|
* agentInvokeTime: number,
|
|
* heapUsedBefore: number,
|
|
* heapUsedAfter: number,
|
|
* heapDelta: number,
|
|
* heapDeltaPostGC: number,
|
|
* majorGCs: number,
|
|
* minorGCs: number,
|
|
* gcDurationMs: number,
|
|
* layoutCount: number,
|
|
* layoutDurationMs: number,
|
|
* recalcStyleCount: number,
|
|
* forcedReflowCount: number,
|
|
* longTaskCount: number,
|
|
* longAnimationFrameCount: number,
|
|
* longAnimationFrameTotalMs: number,
|
|
* frameCount: number,
|
|
* compositeLayers: number,
|
|
* paintCount: number,
|
|
* hasInternalMarks: boolean,
|
|
* responseHasContent: boolean,
|
|
* internalFirstToken: number,
|
|
* profilePath: string,
|
|
* tracePath: string,
|
|
* snapshotPath: string,
|
|
* extHostHeapUsedBefore: number,
|
|
* extHostHeapUsedAfter: number,
|
|
* extHostHeapDelta: number,
|
|
* extHostHeapDeltaPostGC: number,
|
|
* extHostProfilePath: string,
|
|
* extHostSnapshotPath: string,
|
|
* }} RunMetrics
|
|
*/
|
|
|
|
// -- Single run --------------------------------------------------------------
|
|
|
|
/**
|
|
* @param {string} electronPath
|
|
* @param {string} scenario
|
|
* @param {{ url: string, requestCount: () => number, waitForRequests: (n: number, ms: number) => Promise<void>, completionCount: () => number, waitForCompletion: (n: number, ms: number) => Promise<void> }} mockServer
|
|
* @param {boolean} verbose
|
|
* @param {string} runIndex
|
|
* @param {string} runDir - timestamped run directory for diagnostics
|
|
* @param {'baseline' | 'test'} role - whether this is a baseline or test run
|
|
* @param {Record<string, any>} [settingsOverrides] - custom VS Code settings
|
|
* @param {{ heapSnapshots?: boolean }} [runOpts] - additional run options
|
|
* @returns {Promise<RunMetrics>}
|
|
*/
|
|
async function runOnce(electronPath, scenario, mockServer, verbose, runIndex, runDir, role, settingsOverrides, runOpts) {
|
|
const takeHeapSnapshots = runOpts?.heapSnapshots ?? false;
|
|
const { userDataDir, extDir, logsDir } = prepareRunDir(runIndex, mockServer, settingsOverrides);
|
|
const isDevBuild = !electronPath.includes('.vscode-test') && !electronPath.includes('VSCode-');
|
|
// Extract a clean build label from the path.
|
|
// Dev: .build/electron/Code - OSS.app/.../Code - OSS → "dev"
|
|
// Stable: .vscode-test/vscode-darwin-arm64-1.115.0/Visual Studio Code.app/.../Electron → "1.115.0"
|
|
// Production: ../VSCode-darwin-arm64/Code - OSS.app/.../Code - OSS → "production"
|
|
let buildLabel = 'dev';
|
|
if (!isDevBuild) {
|
|
const vscodeTestMatch = electronPath.match(/vscode-test\/vscode-[^/]*?-(\d+\.\d+\.\d+)/);
|
|
if (vscodeTestMatch) {
|
|
buildLabel = vscodeTestMatch[1];
|
|
} else if (electronPath.includes('VSCode-')) {
|
|
buildLabel = 'production';
|
|
} else {
|
|
buildLabel = path.basename(electronPath);
|
|
}
|
|
}
|
|
|
|
// For dev builds from a different repo, derive the repo root from the
|
|
// electron path so that the build loads its own out/ source code.
|
|
const appRoot = isDevBuild ? (getRepoRoot(electronPath) || ROOT) : ROOT;
|
|
if (isDevBuild && appRoot !== ROOT) {
|
|
if (verbose) {
|
|
console.log(` [debug] Using appRoot from electron path: ${appRoot}`);
|
|
}
|
|
}
|
|
|
|
// Create a per-run diagnostics directory: <runDir>/<role>-<build>/<scenario>-<i>/
|
|
const runDiagDir = path.join(runDir, `${role}-${buildLabel}`, runIndex.replace(/^baseline-/, ''));
|
|
fs.mkdirSync(runDiagDir, { recursive: true });
|
|
|
|
const tracePath = path.join(runDiagDir, 'trace.json');
|
|
const extHostInspectPort = getNextExtHostInspectPort();
|
|
const vscode = await launchVSCode(
|
|
electronPath,
|
|
buildArgs(userDataDir, extDir, logsDir, { isDevBuild, extHostInspectPort, traceFile: tracePath, appRoot }),
|
|
buildEnv(mockServer, { isDevBuild }),
|
|
{ verbose },
|
|
);
|
|
activeVSCode = vscode;
|
|
const window = vscode.page;
|
|
|
|
// Declared outside try so the finally block can clean up
|
|
/** @type {{ send: (method: string, params?: any) => Promise<any>, on: (event: string, listener: (params: any) => void) => void, close: () => void } | null} */
|
|
let extHostInspector = null;
|
|
/** @type {{ usedSize: number, totalSize: number } | null} */
|
|
let extHostHeapBefore = null;
|
|
/** @type {Omit<RunMetrics, 'majorGCs' | 'minorGCs' | 'gcDurationMs' | 'longTaskCount' | 'longAnimationFrameCount' | 'longAnimationFrameTotalMs' | 'timeToUIUpdated' | 'timeToFirstToken' | 'timeToComplete' | 'timeToRenderComplete' | 'layoutDurationMs' | 'instructionCollectionTime' | 'agentInvokeTime' | 'hasInternalMarks' | 'internalFirstToken'> | null} */
|
|
let partialMetrics = null;
|
|
// Timing vars hoisted for access in post-close trace parsing
|
|
let submitTime = 0;
|
|
let firstResponseTime = 0;
|
|
let responseCompleteTime = 0;
|
|
let renderCompleteTime = 0;
|
|
|
|
try {
|
|
await window.waitForSelector('.monaco-workbench', { timeout: 60_000 });
|
|
|
|
const cdp = await window.context().newCDPSession(window);
|
|
await cdp.send('Performance.enable');
|
|
const heapBefore = /** @type {any} */ (await cdp.send('Runtime.getHeapUsage'));
|
|
|
|
const metricsBefore = await cdp.send('Performance.getMetrics');
|
|
|
|
// Open chat
|
|
const chatShortcut = process.platform === 'darwin' ? 'Control+Meta+KeyI' : 'Control+Alt+KeyI';
|
|
await window.keyboard.press(chatShortcut);
|
|
|
|
const CHAT_VIEW = 'div[id="workbench.panel.chat"]';
|
|
const chatEditorSel = `${CHAT_VIEW} .interactive-input-part .monaco-editor[role="code"]`;
|
|
|
|
await window.waitForSelector(CHAT_VIEW, { timeout: 15_000 });
|
|
await window.waitForFunction(
|
|
(selector) => Array.from(document.querySelectorAll(selector)).some(el => {
|
|
const rect = el.getBoundingClientRect();
|
|
return rect.width > 0 && rect.height > 0;
|
|
}),
|
|
chatEditorSel, { timeout: 15_000 },
|
|
);
|
|
|
|
// Dismiss dialogs
|
|
const dismissDialog = async () => {
|
|
for (const sel of ['.chat-setup-dialog', '.dialog-shadow', '.monaco-dialog-box']) {
|
|
const el = await window.$(sel);
|
|
if (el) { await window.keyboard.press('Escape'); await new Promise(r => setTimeout(r, 500)); break; }
|
|
}
|
|
};
|
|
await dismissDialog();
|
|
|
|
// Wait for extension activation
|
|
const reqsBefore = mockServer.requestCount();
|
|
try { await mockServer.waitForRequests(reqsBefore + 4, 30_000); } catch { }
|
|
if (verbose) {
|
|
console.log(` [debug] Extension active (${mockServer.requestCount() - reqsBefore} new requests)`);
|
|
}
|
|
|
|
// Connect to extension host inspector for profiling/heap data
|
|
try {
|
|
extHostInspector = await connectToExtHostInspector(extHostInspectPort, { verbose, timeoutMs: 15_000 });
|
|
await extHostInspector.send('HeapProfiler.enable');
|
|
await extHostInspector.send('Profiler.enable');
|
|
await extHostInspector.send('Profiler.start');
|
|
extHostHeapBefore = await extHostInspector.send('Runtime.getHeapUsage');
|
|
if (verbose && extHostHeapBefore) {
|
|
console.log(` [ext-host] Heap before: ${Math.round(extHostHeapBefore.usedSize / 1024 / 1024)}MB`);
|
|
}
|
|
} catch (err) {
|
|
if (verbose) {
|
|
console.log(` [ext-host] Could not connect to inspector: ${err}`);
|
|
}
|
|
}
|
|
|
|
// Wait for model resolution
|
|
await new Promise(r => setTimeout(r, 3000));
|
|
await dismissDialog();
|
|
|
|
// Focus input
|
|
await window.click(chatEditorSel);
|
|
const focusStart = Date.now();
|
|
while (Date.now() - focusStart < 5_000) {
|
|
const focused = await window.evaluate((sel) => {
|
|
const el = document.querySelector(sel);
|
|
return el && (el.classList.contains('focused') || el.contains(document.activeElement));
|
|
}, chatEditorSel).catch(() => false);
|
|
if (focused) { break; }
|
|
await new Promise(r => setTimeout(r, 50));
|
|
}
|
|
|
|
// Type message — use the smoke-test driver's typeInEditor when available
|
|
// (dev builds), fall back to pressSequentially for stable/insiders builds.
|
|
const chatMessage = `[scenario:${scenario}] Explain how this code works`;
|
|
const actualInputSelector = await window.evaluate((editorSel) => {
|
|
const editor = document.querySelector(editorSel);
|
|
if (!editor) { throw new Error('Chat editor not found'); }
|
|
return editor.querySelector('.native-edit-context') ? editorSel + ' .native-edit-context' : editorSel + ' textarea';
|
|
}, chatEditorSel);
|
|
|
|
const hasDriver = await window.evaluate(() =>
|
|
// @ts-ignore
|
|
!!globalThis.driver?.typeInEditor
|
|
).catch(() => false);
|
|
|
|
if (hasDriver) {
|
|
await window.evaluate(({ selector, text }) => {
|
|
// @ts-ignore
|
|
return globalThis.driver.typeInEditor(selector, text);
|
|
}, { selector: actualInputSelector, text: chatMessage });
|
|
} else {
|
|
// Fallback: click the input element and use pressSequentially
|
|
await window.click(actualInputSelector);
|
|
await new Promise(r => setTimeout(r, 200));
|
|
await window.locator(actualInputSelector).pressSequentially(chatMessage, { delay: 0 });
|
|
}
|
|
|
|
// Start CPU profiler to capture call stacks during the interaction
|
|
await cdp.send('Profiler.enable');
|
|
await cdp.send('Profiler.start');
|
|
|
|
// Submit
|
|
const completionsBefore = mockServer.completionCount();
|
|
submitTime = Date.now();
|
|
await window.keyboard.press('Enter');
|
|
|
|
// Wait for mock server to serve the response
|
|
try { await mockServer.waitForCompletion(completionsBefore + 1, 60_000); } catch { }
|
|
firstResponseTime = Date.now();
|
|
|
|
// Wait for DOM response to settle
|
|
await dismissDialog();
|
|
const responseSelector = `${CHAT_VIEW} .interactive-item-container.interactive-response`;
|
|
await window.waitForFunction(
|
|
(sel) => {
|
|
const responses = document.querySelectorAll(sel);
|
|
if (responses.length === 0) { return false; }
|
|
return !responses[responses.length - 1].classList.contains('chat-response-loading');
|
|
},
|
|
responseSelector, { timeout: 30_000 },
|
|
);
|
|
responseCompleteTime = Date.now();
|
|
|
|
// -- User turn injection loop -----------------------------------------
|
|
// For multi-turn scenarios with user follow-ups, type each follow-up
|
|
// message and wait for the model's response to settle.
|
|
const userTurns = getUserTurns(scenario);
|
|
for (let ut = 0; ut < userTurns.length; ut++) {
|
|
const userTurn = userTurns[ut];
|
|
if (verbose) {
|
|
console.log(` [debug] User follow-up ${ut + 1}/${userTurns.length}: "${userTurn.message}"`);
|
|
}
|
|
|
|
// Brief pause to let the UI settle between turns
|
|
await new Promise(r => setTimeout(r, 500));
|
|
|
|
// Focus the chat input
|
|
await window.click(chatEditorSel);
|
|
const utFocusStart = Date.now();
|
|
while (Date.now() - utFocusStart < 3_000) {
|
|
const focused = await window.evaluate((sel) => {
|
|
const el = document.querySelector(sel);
|
|
return el && (el.classList.contains('focused') || el.contains(document.activeElement));
|
|
}, chatEditorSel).catch(() => false);
|
|
if (focused) { break; }
|
|
await new Promise(r => setTimeout(r, 50));
|
|
}
|
|
|
|
// Type the follow-up message
|
|
if (hasDriver) {
|
|
await window.evaluate(({ selector, text }) => {
|
|
// @ts-ignore
|
|
return globalThis.driver.typeInEditor(selector, text);
|
|
}, { selector: actualInputSelector, text: userTurn.message });
|
|
} else {
|
|
await window.click(actualInputSelector);
|
|
await new Promise(r => setTimeout(r, 200));
|
|
await window.locator(actualInputSelector).pressSequentially(userTurn.message, { delay: 0 });
|
|
}
|
|
|
|
// Submit follow-up
|
|
const utCompBefore = mockServer.completionCount();
|
|
await window.keyboard.press('Enter');
|
|
|
|
// Wait for mock server to serve the response for this turn
|
|
try { await mockServer.waitForCompletion(utCompBefore + 1, 60_000); } catch { }
|
|
|
|
// Wait for the new response to finish rendering.
|
|
// The chat list is virtualized — old response elements are
|
|
// recycled out of the DOM as new ones appear, so we cannot
|
|
// rely on counting DOM elements. Instead, scroll to the
|
|
// bottom and wait for no response to be in loading state.
|
|
await dismissDialog();
|
|
await window.evaluate((chatViewSel) => {
|
|
const input = document.querySelector(chatViewSel + ' .interactive-input-part');
|
|
if (input) { input.scrollIntoView({ block: 'end' }); }
|
|
}, CHAT_VIEW);
|
|
await new Promise(r => setTimeout(r, 200));
|
|
|
|
await window.waitForFunction(
|
|
(sel) => {
|
|
const responses = document.querySelectorAll(sel);
|
|
if (responses.length === 0) { return false; }
|
|
return !responses[responses.length - 1].classList.contains('chat-response-loading');
|
|
},
|
|
responseSelector,
|
|
{ timeout: 30_000 },
|
|
);
|
|
responseCompleteTime = Date.now();
|
|
|
|
if (verbose) {
|
|
const utResponseInfo = await window.evaluate((sel) => {
|
|
const responses = document.querySelectorAll(sel);
|
|
const last = responses[responses.length - 1];
|
|
return last ? (last.textContent || '').substring(0, 150) : '(empty)';
|
|
}, responseSelector);
|
|
console.log(` [debug] Follow-up response (first 150 chars): ${utResponseInfo}`);
|
|
}
|
|
}
|
|
|
|
// Stop CPU profiler and save the profile
|
|
const { profile } = /** @type {any} */ (await cdp.send('Profiler.stop'));
|
|
const profilePath = path.join(runDiagDir, 'profile.cpuprofile');
|
|
fs.writeFileSync(profilePath, JSON.stringify(profile));
|
|
if (verbose) {
|
|
console.log(` [debug] CPU profile saved to ${profilePath}`);
|
|
}
|
|
|
|
const responseInfo = await window.evaluate((sel) => {
|
|
const responses = document.querySelectorAll(sel);
|
|
const last = responses[responses.length - 1];
|
|
if (!last) { return { hasContent: false, text: '' }; }
|
|
const text = last.textContent || '';
|
|
return { hasContent: text.trim().length > 0, text: text.substring(0, 200) };
|
|
}, responseSelector);
|
|
|
|
if (verbose) {
|
|
console.log(` [debug] Response content (first 200 chars): ${responseInfo.text}`);
|
|
console.log(` [debug] Client-side timing: firstResponse=${firstResponseTime - submitTime}ms, complete=${responseCompleteTime - submitTime}ms`);
|
|
}
|
|
|
|
// Wait for the typewriter animation to finish rendering.
|
|
// The chat UI animates streamed content word-by-word after the
|
|
// response stream completes. We need to wait until all content
|
|
// is rendered before capturing layout/style metrics, otherwise
|
|
// we miss the rendering phase where batching optimizations matter.
|
|
await window.waitForFunction(
|
|
(sel) => {
|
|
const responses = document.querySelectorAll(sel);
|
|
const last = responses[responses.length - 1];
|
|
if (!last) { return true; }
|
|
// The typewriter animation is done when there are no
|
|
// elements with the 'typewriter' or 'animating' class,
|
|
// and no pending cursor animations.
|
|
const hasAnimating = last.querySelector('.chat-animated-word, .chat-typewriter-cursor');
|
|
return !hasAnimating;
|
|
},
|
|
responseSelector,
|
|
{ timeout: 30_000 },
|
|
).catch(() => {
|
|
// Fallback: if the selector-based check doesn't work (e.g.
|
|
// the CSS classes differ across versions), wait for content
|
|
// to stabilize by polling textContent.
|
|
});
|
|
|
|
// Additional stabilization: poll until textContent stops changing.
|
|
// This catches any remaining animation regardless of CSS class names.
|
|
{
|
|
let prev = '';
|
|
let stableCount = 0;
|
|
const stabilizeStart = Date.now();
|
|
while (stableCount < 3 && Date.now() - stabilizeStart < 10_000) {
|
|
const current = await window.evaluate((sel) => {
|
|
const responses = document.querySelectorAll(sel);
|
|
const last = responses[responses.length - 1];
|
|
return last ? (last.textContent || '') : '';
|
|
}, responseSelector).catch(() => '');
|
|
if (current === prev) {
|
|
stableCount++;
|
|
} else {
|
|
stableCount = 0;
|
|
prev = current;
|
|
}
|
|
await new Promise(r => setTimeout(r, 100));
|
|
}
|
|
}
|
|
renderCompleteTime = Date.now();
|
|
if (verbose) {
|
|
console.log(` [debug] Render stabilized: ${renderCompleteTime - responseCompleteTime}ms after stream complete`);
|
|
}
|
|
|
|
const heapAfter = /** @type {any} */ (await cdp.send('Runtime.getHeapUsage'));
|
|
const metricsAfter = await cdp.send('Performance.getMetrics');
|
|
|
|
// -- Extension host metrics (non-snapshot) ---------------------------
|
|
let extHostHeapUsedBefore = -1;
|
|
let extHostHeapUsedAfter = -1;
|
|
let extHostHeapDelta = -1;
|
|
let extHostHeapDeltaPostGC = -1;
|
|
let extHostProfilePath = '';
|
|
let extHostSnapshotPath = '';
|
|
if (extHostInspector && extHostHeapBefore) {
|
|
try {
|
|
extHostHeapUsedBefore = Math.round(extHostHeapBefore.usedSize / 1024 / 1024);
|
|
|
|
// Stop CPU profiler and save
|
|
const extProfile = await extHostInspector.send('Profiler.stop');
|
|
extHostProfilePath = path.join(runDiagDir, 'exthost-profile.cpuprofile');
|
|
fs.writeFileSync(extHostProfilePath, JSON.stringify(extProfile.profile));
|
|
if (verbose) {
|
|
console.log(` [ext-host] CPU profile saved to ${extHostProfilePath}`);
|
|
}
|
|
|
|
// Heap usage after interaction
|
|
const extHostHeapAfter = await extHostInspector.send('Runtime.getHeapUsage');
|
|
extHostHeapUsedAfter = Math.round(extHostHeapAfter.usedSize / 1024 / 1024);
|
|
extHostHeapDelta = extHostHeapUsedAfter - extHostHeapUsedBefore;
|
|
|
|
// Force GC and measure retained heap
|
|
try {
|
|
await extHostInspector.send('Runtime.evaluate', { expression: 'gc()', awaitPromise: false, includeCommandLineAPI: true });
|
|
await new Promise(r => setTimeout(r, 200));
|
|
const extHostHeapPostGC = await extHostInspector.send('Runtime.getHeapUsage');
|
|
extHostHeapDeltaPostGC = Math.round(extHostHeapPostGC.usedSize / 1024 / 1024) - extHostHeapUsedBefore;
|
|
} catch {
|
|
extHostHeapDeltaPostGC = -1;
|
|
}
|
|
|
|
if (verbose) {
|
|
console.log(` [ext-host] Heap: before=${extHostHeapUsedBefore}MB, after=${extHostHeapUsedAfter}MB, delta=${extHostHeapDelta}MB, deltaPostGC=${extHostHeapDeltaPostGC}MB`);
|
|
}
|
|
} catch (err) {
|
|
if (verbose) {
|
|
console.log(` [ext-host] Error collecting metrics: ${err}`);
|
|
}
|
|
}
|
|
}
|
|
|
|
// -- Heap snapshots (opt-in, parallelized) ---------------------------
|
|
let snapshotPath = '';
|
|
if (takeHeapSnapshots) {
|
|
const snapshotPromises = [];
|
|
|
|
// Renderer snapshot
|
|
snapshotPromises.push((async () => {
|
|
const p = path.join(runDiagDir, 'heap.heapsnapshot');
|
|
await cdp.send('HeapProfiler.enable');
|
|
const chunks = /** @type {string[]} */ ([]);
|
|
cdp.on('HeapProfiler.addHeapSnapshotChunk', (/** @type {any} */ params) => {
|
|
chunks.push(params.chunk);
|
|
});
|
|
await cdp.send('HeapProfiler.takeHeapSnapshot', { reportProgress: false });
|
|
fs.writeFileSync(p, chunks.join(''));
|
|
return p;
|
|
})());
|
|
|
|
// Extension host snapshot (parallel with renderer)
|
|
if (extHostInspector && extHostHeapBefore) {
|
|
snapshotPromises.push((async () => {
|
|
const p = path.join(runDiagDir, 'exthost-heap.heapsnapshot');
|
|
const chunks = /** @type {string[]} */ ([]);
|
|
extHostInspector.on('HeapProfiler.addHeapSnapshotChunk', (/** @type {any} */ params) => {
|
|
chunks.push(params.chunk);
|
|
});
|
|
await extHostInspector.send('HeapProfiler.takeHeapSnapshot', { reportProgress: false });
|
|
fs.writeFileSync(p, chunks.join(''));
|
|
return p;
|
|
})());
|
|
}
|
|
|
|
const snapshotResults = await Promise.all(snapshotPromises);
|
|
snapshotPath = snapshotResults[0];
|
|
if (snapshotResults.length > 1) {
|
|
extHostSnapshotPath = snapshotResults[1];
|
|
}
|
|
|
|
if (verbose) {
|
|
console.log(` [debug] Renderer snapshot saved to ${snapshotPath}`);
|
|
if (extHostSnapshotPath) {
|
|
console.log(` [ext-host] Snapshot saved to ${extHostSnapshotPath}`);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Close ext host inspector now that snapshots (if any) are done
|
|
if (extHostInspector) {
|
|
extHostInspector.close();
|
|
}
|
|
|
|
// Store partial metrics here so we can combine with trace data after close.
|
|
|
|
/** @param {any} r @param {string} name */
|
|
function getMetric(r, name) {
|
|
const e = r.metrics?.find((/** @type {any} */ m) => m.name === name);
|
|
return e ? e.value : 0;
|
|
}
|
|
|
|
partialMetrics = {
|
|
heapUsedBefore: Math.round(heapBefore.usedSize / 1024 / 1024),
|
|
heapUsedAfter: Math.round(heapAfter.usedSize / 1024 / 1024),
|
|
heapDelta: Math.round((heapAfter.usedSize - heapBefore.usedSize) / 1024 / 1024),
|
|
heapDeltaPostGC: await (async () => {
|
|
// Force a full GC then measure heap to get deterministic retained-memory delta.
|
|
// --js-flags=--expose-gc is not required: CDP's Runtime.evaluate can call gc()
|
|
// when includeCommandLineAPI is true.
|
|
try {
|
|
await cdp.send('Runtime.evaluate', { expression: 'gc()', awaitPromise: false, includeCommandLineAPI: true });
|
|
await new Promise(r => setTimeout(r, 200));
|
|
const heapPostGC = /** @type {any} */ (await cdp.send('Runtime.getHeapUsage'));
|
|
return Math.round((heapPostGC.usedSize - heapBefore.usedSize) / 1024 / 1024);
|
|
} catch {
|
|
return -1; // gc() not available in this build
|
|
}
|
|
})(),
|
|
layoutCount: getMetric(metricsAfter, 'LayoutCount') - getMetric(metricsBefore, 'LayoutCount'),
|
|
recalcStyleCount: getMetric(metricsAfter, 'RecalcStyleCount') - getMetric(metricsBefore, 'RecalcStyleCount'),
|
|
forcedReflowCount: getMetric(metricsAfter, 'ForcedStyleRecalcs') - getMetric(metricsBefore, 'ForcedStyleRecalcs'),
|
|
frameCount: getMetric(metricsAfter, 'FrameCount') - getMetric(metricsBefore, 'FrameCount'),
|
|
compositeLayers: getMetric(metricsAfter, 'CompositeLayers') - getMetric(metricsBefore, 'CompositeLayers'),
|
|
paintCount: getMetric(metricsAfter, 'PaintCount') - getMetric(metricsBefore, 'PaintCount'),
|
|
responseHasContent: responseInfo.hasContent,
|
|
profilePath,
|
|
tracePath,
|
|
snapshotPath,
|
|
extHostHeapUsedBefore,
|
|
extHostHeapUsedAfter,
|
|
extHostHeapDelta,
|
|
extHostHeapDeltaPostGC,
|
|
extHostProfilePath,
|
|
extHostSnapshotPath,
|
|
};
|
|
} finally {
|
|
if (extHostInspector) {
|
|
try { extHostInspector.close(); } catch { }
|
|
}
|
|
activeVSCode = null;
|
|
await vscode.close();
|
|
}
|
|
|
|
// Read the trace file written by VS Code on exit via --trace-startup-file
|
|
/** @type {Array<any>} */
|
|
let traceEvents = [];
|
|
try {
|
|
const traceData = JSON.parse(fs.readFileSync(tracePath, 'utf-8'));
|
|
traceEvents = traceData.traceEvents || [];
|
|
} catch {
|
|
// Trace file may not exist if VS Code crashed before shutdown
|
|
}
|
|
|
|
// Extract code/chat/* perf marks from blink.user_timing trace events.
|
|
// These appear as instant ('R' or 'I') events with timestamps in microseconds.
|
|
const chatMarks = traceEvents
|
|
.filter(e => e.cat === 'blink.user_timing' && e.name && e.name.startsWith('code/chat/'))
|
|
.map(e => ({ name: e.name, startTime: e.ts / 1000 }));
|
|
|
|
if (verbose && chatMarks.length > 0) {
|
|
console.log(` [trace] chatMarks (${chatMarks.length}): ${chatMarks.map((/** @type {any} */ m) => m.name.split('/').slice(-1)[0]).join(', ')}`);
|
|
}
|
|
|
|
// Parse timing — prefer internal code/chat/* marks (precise, in-process)
|
|
// with client-side Date.now() as fallback for older builds without marks.
|
|
const timeToUIUpdated = markDuration(chatMarks, 'request/start', 'request/uiUpdated');
|
|
const internalFirstToken = markDuration(chatMarks, 'request/start', 'request/firstToken');
|
|
const timeToFirstToken = internalFirstToken >= 0 ? internalFirstToken : (firstResponseTime - submitTime);
|
|
const timeToComplete = responseCompleteTime - submitTime;
|
|
const timeToRenderComplete = renderCompleteTime - submitTime;
|
|
const instructionCollectionTime = markDuration(chatMarks, 'request/willCollectInstructions', 'request/didCollectInstructions');
|
|
const agentInvokeTime = markDuration(chatMarks, 'agent/willInvoke', 'agent/didInvoke');
|
|
|
|
// Parse GC events from trace.
|
|
// Use the trace-event category and phase fields which are stable
|
|
// across V8 versions, rather than matching event name substrings.
|
|
let majorGCs = 0, minorGCs = 0, gcDurationMs = 0;
|
|
for (const event of traceEvents) {
|
|
const isGC = event.cat === 'v8.gc'
|
|
|| event.cat === 'devtools.timeline,v8'
|
|
|| (typeof event.cat === 'string' && event.cat.split(',').some((/** @type {string} */ c) => {
|
|
const t = c.trim();
|
|
return t === 'v8.gc' || t === 'disabled-by-default-v8.gc' || t === 'disabled-by-default-v8.gc_stats';
|
|
}));
|
|
if (!isGC) { continue; }
|
|
// Only count complete ('X') or duration-begin ('B') events to
|
|
// avoid double-counting begin/end pairs.
|
|
if (event.ph && event.ph !== 'X' && event.ph !== 'B') { continue; }
|
|
const name = event.name || '';
|
|
if (/Major|MarkCompact|MSC|MC|IncrementalMarking|FinalizeMC/i.test(name)) { majorGCs++; }
|
|
else if (/Minor|Scaveng/i.test(name)) { minorGCs++; }
|
|
else { minorGCs++; } // default unknown GC events to minor
|
|
if (event.dur) { gcDurationMs += event.dur / 1000; }
|
|
}
|
|
// Parse Layout duration from devtools.timeline trace events.
|
|
let layoutDurationMs = 0;
|
|
for (const event of traceEvents) {
|
|
if (event.name === 'Layout' && event.ph === 'X' && event.dur) {
|
|
layoutDurationMs += event.dur / 1000;
|
|
}
|
|
}
|
|
|
|
let longTaskCount = 0;
|
|
for (const event of traceEvents) {
|
|
if (event.name === 'RunTask' && event.dur && event.dur > 50_000) { longTaskCount++; }
|
|
}
|
|
|
|
// Parse Long Animation Frame (LoAF) events from devtools.timeline trace.
|
|
// AnimationFrame events use async flow pairs (ph:'s' start, ph:'f' finish)
|
|
// with matching ids. Compute duration from each s→f pair.
|
|
let longAnimationFrameCount = 0;
|
|
let longAnimationFrameTotalMs = 0;
|
|
{
|
|
/** @type {Map<number, number>} */
|
|
const frameStarts = new Map();
|
|
for (const event of traceEvents) {
|
|
if (event.cat === 'devtools.timeline' && event.name === 'AnimationFrame') {
|
|
if (event.ph === 's') {
|
|
frameStarts.set(event.id, event.ts);
|
|
} else if (event.ph === 'f' && frameStarts.has(event.id)) {
|
|
const durationMs = (event.ts - /** @type {number} */(frameStarts.get(event.id))) / 1000;
|
|
frameStarts.delete(event.id);
|
|
if (durationMs > 50) {
|
|
longAnimationFrameCount++;
|
|
longAnimationFrameTotalMs += durationMs;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return {
|
|
...partialMetrics,
|
|
timeToUIUpdated, timeToFirstToken, timeToComplete, timeToRenderComplete, instructionCollectionTime, agentInvokeTime,
|
|
hasInternalMarks: chatMarks.length > 0,
|
|
internalFirstToken,
|
|
majorGCs, minorGCs,
|
|
gcDurationMs: Math.round(gcDurationMs * 100) / 100,
|
|
layoutDurationMs: Math.round(layoutDurationMs * 100) / 100,
|
|
longTaskCount,
|
|
longAnimationFrameCount,
|
|
longAnimationFrameTotalMs: Math.round(longAnimationFrameTotalMs * 100) / 100,
|
|
};
|
|
}
|
|
|
|
// -- CI summary generation ---------------------------------------------------
|
|
|
|
const GITHUB_REPO = 'https://github.com/microsoft/vscode';
|
|
|
|
/**
|
|
* Format a build identifier as a Markdown link when possible.
|
|
* - Commit SHAs link to the commit page.
|
|
* - Semver versions link to the release tag page.
|
|
* - Everything else (e.g. "baseline", "dev (local)") is returned as inline code.
|
|
* @param {string} label
|
|
* @returns {string}
|
|
*/
|
|
function formatBuildLink(label) {
|
|
if (/^[0-9a-f]{7,40}$/.test(label)) {
|
|
const short = label.substring(0, 7);
|
|
return `[\`${short}\`](${GITHUB_REPO}/commit/${label})`;
|
|
}
|
|
if (/^\d+\.\d+\.\d+/.test(label)) {
|
|
return `[\`${label}\`](${GITHUB_REPO}/releases/tag/${label})`;
|
|
}
|
|
return `\`${label}\``;
|
|
}
|
|
|
|
/**
|
|
* Build a GitHub compare link between two build identifiers, if both are
|
|
* commit-like or version-like references. Returns empty string otherwise.
|
|
* @param {string} base
|
|
* @param {string} test
|
|
* @returns {string}
|
|
*/
|
|
function formatCompareLink(base, test) {
|
|
const isRef = (/** @type {string} */ v) => /^[0-9a-f]{7,40}$/.test(v) || /^\d+\.\d+\.\d+/.test(v);
|
|
if (!isRef(base) || !isRef(test)) {
|
|
return '';
|
|
}
|
|
return `[compare](${GITHUB_REPO}/compare/${base}...${test})`;
|
|
}
|
|
|
|
/**
|
|
* Generate a detailed Markdown summary table for CI.
|
|
* Printed to stdout and written to ci-summary.md.
|
|
*
|
|
* @param {Record<string, any>} jsonReport
|
|
* @param {Record<string, any> | null} baseline
|
|
* @param {{ threshold: number, metricThresholds?: Record<string, number | string>, runs: number, baselineBuild?: string, build?: string }} opts
|
|
*/
|
|
function generateCISummary(jsonReport, baseline, opts) {
|
|
const baseLabel = opts.baselineBuild || 'baseline';
|
|
const testBuildMode = jsonReport.buildMode || 'dev';
|
|
const testLabel = testBuildMode === 'dev' ? 'dev (local)'
|
|
: testBuildMode === 'production' ? 'production (local)'
|
|
: opts.build || testBuildMode;
|
|
const baseLink = formatBuildLink(baseLabel);
|
|
const testLink = formatBuildLink(testLabel);
|
|
const compareLink = formatCompareLink(baseLabel, testLabel);
|
|
const allMetrics = [
|
|
['timeToFirstToken', 'timing', 'ms'],
|
|
['timeToComplete', 'timing', 'ms'],
|
|
['layoutCount', 'rendering', ''],
|
|
['recalcStyleCount', 'rendering', ''],
|
|
['forcedReflowCount', 'rendering', ''],
|
|
['longTaskCount', 'rendering', ''],
|
|
['longAnimationFrameCount', 'rendering', ''],
|
|
['longAnimationFrameTotalMs', 'rendering', 'ms'],
|
|
['frameCount', 'rendering', ''],
|
|
['compositeLayers', 'rendering', ''],
|
|
['paintCount', 'rendering', ''],
|
|
['heapDelta', 'memory', 'MB'],
|
|
['heapDeltaPostGC', 'memory', 'MB'],
|
|
['gcDurationMs', 'memory', 'ms'],
|
|
['extHostHeapDelta', 'extHost', 'MB'],
|
|
['extHostHeapDeltaPostGC', 'extHost', 'MB'],
|
|
];
|
|
const regressionMetricNames = new Set(['timeToFirstToken', 'timeToComplete', 'forcedReflowCount', 'longTaskCount', 'longAnimationFrameCount']);
|
|
|
|
const lines = [];
|
|
const scenarios = Object.keys(jsonReport.scenarios);
|
|
|
|
// -- Collect verdicts per scenario/metric --------------------------------
|
|
/** @type {Map<string, { metric: string, verdict: string, change: number, pValue: string, basStr: string, curStr: string }[]>} */
|
|
const scenarioVerdicts = new Map();
|
|
let totalRegressions = 0;
|
|
let totalImprovements = 0;
|
|
|
|
for (const scenario of scenarios) {
|
|
const current = jsonReport.scenarios[scenario];
|
|
const base = baseline?.scenarios?.[scenario];
|
|
/** @type {{ metric: string, verdict: string, change: number, pValue: string, basStr: string, curStr: string }[]} */
|
|
const verdicts = [];
|
|
|
|
if (base) {
|
|
for (const [metric, group, unit] of allMetrics) {
|
|
const cur = current[group]?.[metric];
|
|
const bas = base[group]?.[metric];
|
|
if (!cur || !bas || bas.median === null || bas.median === undefined) { continue; }
|
|
|
|
const change = bas.median !== 0 ? (cur.median - bas.median) / bas.median : 0;
|
|
const isRegressionMetric = regressionMetricNames.has(metric);
|
|
|
|
const curRaw = (current.rawRuns || []).map((/** @type {any} */ r) => r[metric]).filter((/** @type {any} */ v) => v >= 0);
|
|
const basRaw = (base.rawRuns || []).map((/** @type {any} */ r) => r[metric]).filter((/** @type {any} */ v) => v >= 0);
|
|
const ttest = welchTTest(basRaw, curRaw);
|
|
const pStr = ttest ? `${ttest.pValue}` : 'n/a';
|
|
|
|
const metricThreshold = getMetricThreshold(opts, metric);
|
|
const absoluteDelta = cur.median - bas.median;
|
|
let verdict = '';
|
|
if (isRegressionMetric) {
|
|
if (exceedsThreshold(metricThreshold, change, absoluteDelta)) {
|
|
if (!ttest || ttest.significant) {
|
|
verdict = 'REGRESSION';
|
|
totalRegressions++;
|
|
} else {
|
|
verdict = 'noise';
|
|
}
|
|
} else if (exceedsThreshold(metricThreshold, -change, -absoluteDelta) && ttest?.significant) {
|
|
verdict = 'improved';
|
|
totalImprovements++;
|
|
} else {
|
|
verdict = 'ok';
|
|
}
|
|
} else {
|
|
verdict = 'info';
|
|
}
|
|
|
|
const basStr = `${bas.median}${unit} \xb1${bas.stddev}${unit}`;
|
|
const curStr = `${cur.median}${unit} \xb1${cur.stddev}${unit}`;
|
|
verdicts.push({ metric, verdict, change, pValue: pStr, basStr, curStr });
|
|
}
|
|
}
|
|
scenarioVerdicts.set(scenario, verdicts);
|
|
}
|
|
|
|
// -- Header with verdict up front ----------------------------------------
|
|
const hasRegressions = totalRegressions > 0;
|
|
const verdictIcon = hasRegressions ? '\u274C' : '\u2705';
|
|
const verdictText = hasRegressions
|
|
? `${totalRegressions} regression(s) detected`
|
|
: totalImprovements > 0
|
|
? `No regressions \u2014 ${totalImprovements} improvement(s)`
|
|
: 'No significant changes';
|
|
|
|
lines.push(`# ${verdictIcon} Chat Performance: ${verdictText}`);
|
|
lines.push('');
|
|
lines.push(`| | |`);
|
|
lines.push(`|---|---|`);
|
|
lines.push(`| **Baseline** | ${baseLink} |`);
|
|
lines.push(`| **Test** | ${testLink} |`);
|
|
if (compareLink) {
|
|
lines.push(`| **Diff** | ${compareLink} |`);
|
|
}
|
|
lines.push(`| **Runs per scenario** | ${opts.runs} |`);
|
|
const overrides = Object.entries(opts.metricThresholds || {}).filter(([, v]) => {
|
|
const parsed = parseMetricThreshold(v);
|
|
return parsed.type !== 'fraction' || parsed.value !== opts.threshold;
|
|
});
|
|
if (overrides.length > 0) {
|
|
const overrideStr = overrides.map(([k, v]) => {
|
|
const parsed = parseMetricThreshold(v);
|
|
return `${k}: ${parsed.type === 'absolute' ? `${parsed.value}${k.includes('Ms') || k.includes('Time') || k.includes('time') ? 'ms' : ''}` : `${(parsed.value * 100).toFixed(0)}%`}`;
|
|
}).join(', ');
|
|
lines.push(`| **Regression threshold** | ${(opts.threshold * 100).toFixed(0)}% (${overrideStr}) |`);
|
|
} else {
|
|
lines.push(`| **Regression threshold** | ${(opts.threshold * 100).toFixed(0)}% |`);
|
|
}
|
|
lines.push(`| **Scenarios** | ${scenarios.length} |`);
|
|
lines.push(`| **Platform** | ${process.platform} / ${process.arch} |`);
|
|
if (jsonReport.buildMode) {
|
|
lines.push(`| **Build mode** | ${jsonReport.buildMode} |`);
|
|
}
|
|
lines.push('');
|
|
if (jsonReport.mismatchedBuildMode) {
|
|
lines.push('> **⚠ Build mode mismatch:** The test and baseline builds use different build modes.');
|
|
lines.push('> Results may not be directly comparable. For apples-to-apples comparisons,');
|
|
lines.push('> use the same build type for both (e.g. `--production-build` with a local');
|
|
lines.push('> baseline path, or two version strings).');
|
|
lines.push('');
|
|
}
|
|
|
|
// -- At-a-glance overview table: one row per scenario --------------------
|
|
lines.push(`## Overview`);
|
|
lines.push('');
|
|
lines.push('| Scenario | Description | TTFT | Complete | Layouts | Styles | LoAF | Verdict |');
|
|
lines.push('|----------|-------------|-----:|---------:|--------:|-------:|-----:|:-------:|');
|
|
|
|
for (const scenario of scenarios) {
|
|
const verdicts = scenarioVerdicts.get(scenario) || [];
|
|
const get = (/** @type {string} */ m) => verdicts.find(v => v.metric === m);
|
|
|
|
const ttft = get('timeToFirstToken');
|
|
const complete = get('timeToComplete');
|
|
const layouts = get('layoutCount');
|
|
const styles = get('recalcStyleCount');
|
|
const loaf = get('longAnimationFrameCount');
|
|
|
|
const fmtCell = (/** @type {{ change: number, verdict: string } | undefined} */ v) => {
|
|
if (!v) { return '\u2014'; }
|
|
const pct = `${v.change > 0 ? '+' : ''}${(v.change * 100).toFixed(0)}%`;
|
|
return pct;
|
|
};
|
|
|
|
const fmtVerdict = (/** @type {{ verdict: string, change: number }[]} */ vs) => {
|
|
const hasRegression = vs.some(v => v.verdict === 'REGRESSION');
|
|
const hasImproved = vs.some(v => v.verdict === 'improved');
|
|
if (hasRegression) { return '\u274C Regressed'; }
|
|
if (hasImproved) { return '\u2B06\uFE0F Improved'; }
|
|
return '\u2705 OK';
|
|
};
|
|
|
|
const keyVerdicts = [ttft, complete, layouts, styles, loaf].filter(Boolean);
|
|
const rowVerdict = fmtVerdict(/** @type {any[]} */(keyVerdicts));
|
|
|
|
lines.push(`| ${scenario} | ${getScenarioDescription(scenario)} | ${fmtCell(ttft)} | ${fmtCell(complete)} | ${fmtCell(layouts)} | ${fmtCell(styles)} | ${fmtCell(loaf)} | ${rowVerdict} |`);
|
|
}
|
|
lines.push('');
|
|
|
|
// -- Regressions & improvements detail section ---------------------------
|
|
const hasNotable = [...scenarioVerdicts.values()].some(vs => vs.some(v => v.verdict === 'REGRESSION' || v.verdict === 'improved'));
|
|
if (hasNotable) {
|
|
lines.push('## Regressions & Improvements');
|
|
lines.push('');
|
|
lines.push('Only metrics that regressed or improved significantly are shown below.');
|
|
lines.push('');
|
|
|
|
for (const scenario of scenarios) {
|
|
const verdicts = scenarioVerdicts.get(scenario) || [];
|
|
const notable = verdicts.filter(v => v.verdict === 'REGRESSION' || v.verdict === 'improved');
|
|
if (notable.length === 0) { continue; }
|
|
|
|
const icon = notable.some(v => v.verdict === 'REGRESSION') ? '\u274C' : '\u2B06\uFE0F';
|
|
lines.push(`### ${icon} ${scenario}`);
|
|
lines.push('');
|
|
lines.push('| Metric | Baseline | Test | Change | p-value | Verdict |');
|
|
lines.push('|--------|----------|------|--------|---------|---------|');
|
|
for (const v of notable) {
|
|
const pct = `${v.change > 0 ? '+' : ''}${(v.change * 100).toFixed(1)}%`;
|
|
const verdictIcon = v.verdict === 'REGRESSION' ? '\u274C' : '\u2B06\uFE0F';
|
|
lines.push(`| ${v.metric} | ${v.basStr} | ${v.curStr} | ${pct} | ${v.pValue} | ${verdictIcon} ${v.verdict} |`);
|
|
}
|
|
lines.push('');
|
|
}
|
|
}
|
|
|
|
// -- Full metric tables in collapsible section ---------------------------
|
|
lines.push('<details><summary>Full metric details per scenario</summary>');
|
|
lines.push('');
|
|
|
|
for (const scenario of scenarios) {
|
|
const verdicts = scenarioVerdicts.get(scenario) || [];
|
|
const base = baseline?.scenarios?.[scenario];
|
|
|
|
lines.push(`### ${scenario}`);
|
|
lines.push('');
|
|
|
|
if (!base) {
|
|
const current = jsonReport.scenarios[scenario];
|
|
lines.push('> No baseline data for this scenario.');
|
|
lines.push('');
|
|
lines.push('| Metric | Value | StdDev | CV | n |');
|
|
lines.push('|--------|------:|-------:|---:|--:|');
|
|
for (const [metric, group, unit] of allMetrics) {
|
|
const cur = current[group]?.[metric];
|
|
if (!cur) { continue; }
|
|
lines.push(`| ${metric} | ${cur.median}${unit} | \xb1${cur.stddev}${unit} | ${(cur.cv * 100).toFixed(0)}% | ${cur.n} |`);
|
|
}
|
|
lines.push('');
|
|
continue;
|
|
}
|
|
|
|
lines.push(`| Metric | Baseline | Test | Change | p-value | Verdict |`);
|
|
lines.push(`|--------|----------|------|--------|---------|---------|`);
|
|
|
|
for (const v of verdicts) {
|
|
const pct = `${v.change > 0 ? '+' : ''}${(v.change * 100).toFixed(1)}%`;
|
|
let verdictDisplay = v.verdict;
|
|
if (v.verdict === 'REGRESSION') { verdictDisplay = '\u274C REGRESSION'; }
|
|
else if (v.verdict === 'improved') { verdictDisplay = '\u2B06\uFE0F improved'; }
|
|
else if (v.verdict === 'ok') { verdictDisplay = '\u2705 ok'; }
|
|
else if (v.verdict === 'noise') { verdictDisplay = '\uD83C\uDF2B\uFE0F noise'; }
|
|
else if (v.verdict === 'info') { verdictDisplay = '\u2139\uFE0F'; }
|
|
lines.push(`| ${v.metric} | ${v.basStr} | ${v.curStr} | ${pct} | ${v.pValue} | ${verdictDisplay} |`);
|
|
}
|
|
lines.push('');
|
|
}
|
|
lines.push('</details>');
|
|
lines.push('');
|
|
|
|
// -- Raw run data in collapsible section ---------------------------------
|
|
lines.push('<details><summary>Raw run data</summary>');
|
|
lines.push('');
|
|
for (const scenario of scenarios) {
|
|
const current = jsonReport.scenarios[scenario];
|
|
lines.push(`### ${scenario}`);
|
|
lines.push('');
|
|
lines.push('| Run | TTFT (ms) | Complete (ms) | Layouts | Style Recalcs | LoAF Count | LoAF (ms) | Frames | Heap Delta (MB) | Internal Marks |');
|
|
lines.push('|----:|----------:|--------------:|--------:|--------------:|-----------:|----------:|-------:|----------------:|:--------------:|');
|
|
const runs = current.rawRuns || [];
|
|
for (let i = 0; i < runs.length; i++) {
|
|
const r = runs[i];
|
|
const round2 = (/** @type {number} */ v) => Math.round(v * 100) / 100;
|
|
lines.push(`| ${i + 1} | ${round2(r.timeToFirstToken)} | ${r.timeToComplete} | ${r.layoutCount} | ${r.recalcStyleCount} | ${r.longAnimationFrameCount ?? '-'} | ${r.longAnimationFrameTotalMs !== null && r.longAnimationFrameTotalMs !== undefined ? round2(r.longAnimationFrameTotalMs) : '-'} | ${r.frameCount ?? '-'} | ${r.heapDelta} | ${r.hasInternalMarks ? 'yes' : 'no'} |`);
|
|
}
|
|
lines.push('');
|
|
}
|
|
if (baseline) {
|
|
for (const scenario of scenarios) {
|
|
const base = baseline.scenarios?.[scenario];
|
|
if (!base) { continue; }
|
|
lines.push(`### ${scenario} (baseline)`);
|
|
lines.push('');
|
|
lines.push('| Run | TTFT (ms) | Complete (ms) | Layouts | Style Recalcs | LoAF Count | LoAF (ms) | Frames | Heap Delta (MB) | Internal Marks |');
|
|
lines.push('|----:|----------:|--------------:|--------:|--------------:|-----------:|----------:|-------:|----------------:|:--------------:|');
|
|
const runs = base.rawRuns || [];
|
|
for (let i = 0; i < runs.length; i++) {
|
|
const r = runs[i];
|
|
const round2 = (/** @type {number} */ v) => Math.round(v * 100) / 100;
|
|
lines.push(`| ${i + 1} | ${round2(r.timeToFirstToken)} | ${r.timeToComplete} | ${r.layoutCount} | ${r.recalcStyleCount} | ${r.longAnimationFrameCount ?? '-'} | ${r.longAnimationFrameTotalMs !== null && r.longAnimationFrameTotalMs !== undefined ? round2(r.longAnimationFrameTotalMs) : '-'} | ${r.frameCount ?? '-'} | ${r.heapDelta} | ${r.hasInternalMarks ? 'yes' : 'no'} |`);
|
|
}
|
|
lines.push('');
|
|
}
|
|
}
|
|
lines.push('</details>');
|
|
lines.push('');
|
|
|
|
return lines.join('\n');
|
|
}
|
|
|
|
// -- Cleanup on SIGINT/SIGTERM -----------------------------------------------
|
|
|
|
/** @type {{ close: () => Promise<void> } | null} */
|
|
let activeVSCode = null;
|
|
/** @type {{ close: () => Promise<void> } | null} */
|
|
let activeMockServer = null;
|
|
|
|
function installSignalHandlers() {
|
|
const cleanup = async () => {
|
|
console.log('\n[chat-simulation] Caught interrupt, cleaning up...');
|
|
try { await activeVSCode?.close(); } catch { }
|
|
try { await activeMockServer?.close(); } catch { }
|
|
process.exit(130);
|
|
};
|
|
process.on('SIGINT', cleanup);
|
|
process.on('SIGTERM', cleanup);
|
|
}
|
|
|
|
// -- Diagnostic cleanup ------------------------------------------------------
|
|
|
|
/**
|
|
* Remove large diagnostic files (heap snapshots, CPU profiles, traces) from
|
|
* a run's metrics to free disk space. Keeps the JSON results data intact.
|
|
* @param {RunMetrics} metrics
|
|
*/
|
|
function cleanupRunDiagnostics(metrics) {
|
|
const filesToDelete = [
|
|
metrics.profilePath,
|
|
metrics.tracePath,
|
|
metrics.snapshotPath,
|
|
metrics.extHostProfilePath,
|
|
metrics.extHostSnapshotPath,
|
|
];
|
|
for (const filePath of filesToDelete) {
|
|
if (filePath && fs.existsSync(filePath)) {
|
|
try {
|
|
fs.rmSync(filePath, { force: true });
|
|
} catch {
|
|
// Ignore cleanup errors
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Clean up diagnostics for all scenarios that did NOT regress.
|
|
* Keeps diagnostics for regressed scenarios so they can be investigated.
|
|
* @param {Record<string, RunMetrics[]>} allResults - test results by scenario
|
|
* @param {Set<string>} regressedScenarios - scenarios that regressed
|
|
*/
|
|
function cleanupNonRegressedDiagnostics(allResults, regressedScenarios) {
|
|
for (const [scenario, runs] of Object.entries(allResults)) {
|
|
if (regressedScenarios.has(scenario)) {
|
|
continue;
|
|
}
|
|
for (const metrics of runs) {
|
|
cleanupRunDiagnostics(metrics);
|
|
}
|
|
}
|
|
}
|
|
|
|
// -- Main --------------------------------------------------------------------
|
|
|
|
async function main() {
|
|
registerPerfScenarios();
|
|
const opts = parseArgs();
|
|
|
|
installSignalHandlers();
|
|
|
|
const { startServer } = require('./common/mock-llm-server');
|
|
const mockServer = await startServer(0);
|
|
activeMockServer = mockServer;
|
|
console.log(`[chat-simulation] Mock LLM server: ${mockServer.url}`);
|
|
|
|
// -- Resume mode --------------------------------------------------------
|
|
if (opts.resume) {
|
|
if (!fs.existsSync(opts.resume)) {
|
|
console.error(`[chat-simulation] Resume file not found: ${opts.resume}`);
|
|
process.exit(1);
|
|
}
|
|
const prevResults = JSON.parse(fs.readFileSync(opts.resume, 'utf-8'));
|
|
const prevDir = path.dirname(opts.resume);
|
|
|
|
// Find the associated baseline JSON in the same directory
|
|
const baselineFiles = fs.readdirSync(prevDir).filter((/** @type {string} */ f) => f.startsWith('baseline-') && f.endsWith('.json'));
|
|
const baselineFile = baselineFiles.length > 0 ? path.join(prevDir, baselineFiles[0]) : null;
|
|
const prevBaseline = baselineFile ? JSON.parse(fs.readFileSync(baselineFile, 'utf-8')) : null;
|
|
|
|
// Determine which scenarios to resume (default: all from previous run)
|
|
const resumeScenarios = opts.scenarios.length > 0
|
|
? opts.scenarios.filter(s => prevResults.scenarios?.[s])
|
|
: Object.keys(prevResults.scenarios || {});
|
|
|
|
if (resumeScenarios.length === 0) {
|
|
console.error('[chat-simulation] No matching scenarios found in previous results');
|
|
process.exit(1);
|
|
}
|
|
|
|
const testElectron = await resolveBuild(opts.build);
|
|
const baselineVersion = prevBaseline?.baselineBuildVersion;
|
|
const baselineElectron = baselineVersion ? await resolveBuild(baselineVersion) : null;
|
|
|
|
const runsToAdd = opts.runs;
|
|
console.log(`[chat-simulation] Resuming from: ${opts.resume}`);
|
|
console.log(`[chat-simulation] Adding ${runsToAdd} runs per scenario`);
|
|
console.log(`[chat-simulation] Scenarios: ${resumeScenarios.join(', ')}`);
|
|
if (prevBaseline) {
|
|
console.log(`[chat-simulation] Baseline: ${baselineVersion} (${prevBaseline.scenarios?.[resumeScenarios[0]]?.rawRuns?.length || 0} existing runs)`);
|
|
}
|
|
console.log('');
|
|
|
|
for (const scenario of resumeScenarios) {
|
|
console.log(`[chat-simulation] === Resuming: ${scenario} ===`);
|
|
const prevTestRuns = prevResults.scenarios[scenario]?.rawRuns || [];
|
|
const prevBaseRuns = prevBaseline?.scenarios?.[scenario]?.rawRuns || [];
|
|
|
|
// Run additional test iterations
|
|
console.log(`[chat-simulation] Test build (${prevTestRuns.length} existing + ${runsToAdd} new)`);
|
|
for (let i = 0; i < runsToAdd; i++) {
|
|
const runIdx = `${scenario}-resume-${prevTestRuns.length + i}`;
|
|
console.log(`[chat-simulation] Run ${i + 1}/${runsToAdd}...`);
|
|
try {
|
|
const m = await runOnce(testElectron, scenario, mockServer, opts.verbose, runIdx, prevDir, 'test', { ...opts.settingsOverrides, ...opts.testSettingsOverrides }, { heapSnapshots: opts.heapSnapshots });
|
|
// Clean up previous run's diagnostics to bound disk usage; keep the latest
|
|
if (opts.cleanupDiagnostics && prevTestRuns.length > 0) { cleanupRunDiagnostics(prevTestRuns[prevTestRuns.length - 1]); }
|
|
prevTestRuns.push(m);
|
|
if (opts.verbose) {
|
|
const src = m.hasInternalMarks ? 'internal' : 'client-side';
|
|
console.log(` [${src}] firstToken=${m.timeToFirstToken}ms, complete=${m.timeToComplete}ms`);
|
|
}
|
|
} catch (err) { console.error(` Run ${i + 1} failed: ${err}`); }
|
|
}
|
|
|
|
// Run additional baseline iterations
|
|
if (baselineElectron && prevBaseline?.scenarios?.[scenario]) {
|
|
console.log(`[chat-simulation] Baseline build (${prevBaseRuns.length} existing + ${runsToAdd} new)`);
|
|
for (let i = 0; i < runsToAdd; i++) {
|
|
const runIdx = `baseline-${scenario}-resume-${prevBaseRuns.length + i}`;
|
|
console.log(`[chat-simulation] Run ${i + 1}/${runsToAdd}...`);
|
|
try {
|
|
const m = await runOnce(baselineElectron, scenario, mockServer, opts.verbose, runIdx, prevDir, 'baseline', { ...opts.settingsOverrides, ...opts.baselineSettingsOverrides }, { heapSnapshots: opts.heapSnapshots });
|
|
// Clean up previous run's diagnostics to bound disk usage; keep the latest
|
|
if (opts.cleanupDiagnostics && prevBaseRuns.length > 0) { cleanupRunDiagnostics(prevBaseRuns[prevBaseRuns.length - 1]); }
|
|
prevBaseRuns.push(m);
|
|
} catch (err) { console.error(` Run ${i + 1} failed: ${err}`); }
|
|
}
|
|
}
|
|
|
|
// Recompute stats with merged data
|
|
const sd = /** @type {any} */ ({ runs: prevTestRuns.length, timing: {}, memory: {}, rendering: {}, extHost: {}, rawRuns: prevTestRuns });
|
|
for (const [metric, group] of METRIC_DEFS) { sd[group][metric] = robustStats(prevTestRuns.map((/** @type {any} */ r) => r[metric])); }
|
|
prevResults.scenarios[scenario] = sd;
|
|
|
|
if (prevBaseline?.scenarios?.[scenario]) {
|
|
const bsd = /** @type {any} */ ({ runs: prevBaseRuns.length, timing: {}, memory: {}, rendering: {}, extHost: {}, rawRuns: prevBaseRuns });
|
|
for (const [metric, group] of METRIC_DEFS) { bsd[group][metric] = robustStats(prevBaseRuns.map((/** @type {any} */ r) => r[metric])); }
|
|
prevBaseline.scenarios[scenario] = bsd;
|
|
}
|
|
console.log(`[chat-simulation] Merged: test n=${prevTestRuns.length}${prevBaseRuns.length > 0 ? `, baseline n=${prevBaseRuns.length}` : ''}`);
|
|
console.log('');
|
|
}
|
|
|
|
// Write updated files back
|
|
prevResults.runsPerScenario = Math.max(prevResults.runsPerScenario || 0, ...Object.values(prevResults.scenarios).map((/** @type {any} */ s) => s.runs));
|
|
prevResults.lastResumed = new Date().toISOString();
|
|
fs.writeFileSync(opts.resume, JSON.stringify(prevResults, null, 2));
|
|
console.log(`[chat-simulation] Updated results: ${opts.resume}`);
|
|
|
|
if (prevBaseline && baselineFile) {
|
|
prevBaseline.lastResumed = new Date().toISOString();
|
|
fs.writeFileSync(baselineFile, JSON.stringify(prevBaseline, null, 2));
|
|
// Also update cached baseline
|
|
const cachedPath = path.join(DATA_DIR, path.basename(baselineFile));
|
|
fs.writeFileSync(cachedPath, JSON.stringify(prevBaseline, null, 2));
|
|
console.log(`[chat-simulation] Updated baseline: ${baselineFile}`);
|
|
}
|
|
|
|
// -- Re-run comparison with merged data --------------------------------
|
|
opts.baseline = baselineFile || undefined;
|
|
const jsonReport = prevResults;
|
|
jsonReport._resultsPath = opts.resume;
|
|
|
|
// Fall through to comparison logic below
|
|
await printComparison(jsonReport, opts);
|
|
await mockServer.close();
|
|
return;
|
|
}
|
|
|
|
// -- Normal (non-resume) flow -------------------------------------------
|
|
// --production-build: build a local bundled (non-dev) package from the
|
|
// current source tree using `gulp vscode`. This produces the same
|
|
// packaging as a release build (bundled JS, no VSCODE_DEV) while still
|
|
// testing your local changes.
|
|
if (opts.productionBuild && !opts.build) {
|
|
const prodBuildPath = buildProductionBuild();
|
|
opts.build = prodBuildPath;
|
|
console.log(`[chat-simulation] --production-build: using local production build at ${prodBuildPath}`);
|
|
}
|
|
|
|
const electronPath = await resolveBuild(opts.build);
|
|
|
|
if (!fs.existsSync(electronPath)) {
|
|
console.error(`Electron not found at: ${electronPath}`);
|
|
console.error('Run "node build/lib/preLaunch.ts" first, or pass --build <path>');
|
|
process.exit(1);
|
|
}
|
|
|
|
// Detect build modes for both test and baseline builds
|
|
const testBuildMode = detectBuildMode(electronPath);
|
|
|
|
// Resolve the baseline build path early so we can detect its mode.
|
|
// For version strings this downloads; for local paths it resolves directly.
|
|
const isBaselineVersionString = opts.baselineBuild && isVersionString(opts.baselineBuild);
|
|
const isBaselineLocalPath = opts.baselineBuild && !isBaselineVersionString;
|
|
/** @type {string | undefined} */
|
|
let baselineElectronPath;
|
|
if (isBaselineLocalPath) {
|
|
baselineElectronPath = await resolveBuild(opts.baselineBuild);
|
|
if (!fs.existsSync(baselineElectronPath)) {
|
|
console.error(`Baseline build not found at: ${baselineElectronPath}`);
|
|
process.exit(1);
|
|
}
|
|
}
|
|
const baselineBuildMode = opts.baselineBuild
|
|
? (isBaselineVersionString ? 'release' : detectBuildMode(baselineElectronPath || ''))
|
|
: undefined;
|
|
|
|
const isMismatchedBuildMode = baselineBuildMode !== undefined && testBuildMode !== baselineBuildMode;
|
|
|
|
// Create a timestamped run directory for all output
|
|
const runTimestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
|
|
const runDir = path.join(DATA_DIR, runTimestamp);
|
|
fs.mkdirSync(runDir, { recursive: true });
|
|
console.log(`[chat-simulation] Output: ${runDir}`);
|
|
|
|
// Compute effective settings per role
|
|
const testSettings = { ...opts.settingsOverrides, ...opts.testSettingsOverrides };
|
|
const baselineSettings = { ...opts.settingsOverrides, ...opts.baselineSettingsOverrides };
|
|
|
|
// -- Baseline build --------------------------------------------------
|
|
if (opts.baselineBuild) {
|
|
// Use a sanitized label for file names — replace path separators for local paths
|
|
const baselineLabel = isBaselineLocalPath
|
|
? path.basename(path.resolve(opts.baselineBuild))
|
|
: opts.baselineBuild;
|
|
const baselineJsonPath = path.join(runDir, `baseline-${baselineLabel}.json`);
|
|
|
|
// Local paths: always run fresh (no caching — the build may have changed)
|
|
// Version strings: use caching as before
|
|
const cachedPath = isBaselineLocalPath ? null : path.join(DATA_DIR, `baseline-${baselineLabel}.json`);
|
|
const cachedBaseline = cachedPath && !opts.noCache && fs.existsSync(cachedPath)
|
|
? JSON.parse(fs.readFileSync(cachedPath, 'utf-8'))
|
|
: null;
|
|
|
|
if (cachedBaseline?.baselineBuildVersion === opts.baselineBuild) {
|
|
// Check if the cache covers all requested scenarios
|
|
const cachedScenarios = new Set(Object.keys(cachedBaseline.scenarios || {}));
|
|
const missingScenarios = opts.scenarios.filter((/** @type {string} */ s) => !cachedScenarios.has(s));
|
|
|
|
// Also check if cached scenarios have fewer runs than requested
|
|
const shortScenarios = opts.scenarios.filter((/** @type {string} */ s) => {
|
|
const cached = cachedBaseline.scenarios?.[s];
|
|
return cached && (cached.rawRuns?.length || 0) < opts.runs;
|
|
});
|
|
|
|
if (missingScenarios.length === 0 && shortScenarios.length === 0) {
|
|
console.log(`[chat-simulation] Using cached baseline for ${opts.baselineBuild}`);
|
|
fs.writeFileSync(baselineJsonPath, JSON.stringify(cachedBaseline, null, 2));
|
|
opts.baseline = baselineJsonPath;
|
|
} else {
|
|
const scenariosToRun = [...new Set([...missingScenarios, ...shortScenarios])];
|
|
if (missingScenarios.length > 0) {
|
|
console.log(`[chat-simulation] Cached baseline missing scenarios: ${missingScenarios.join(', ')}`);
|
|
}
|
|
if (shortScenarios.length > 0) {
|
|
console.log(`[chat-simulation] Cached baseline needs more runs for: ${shortScenarios.map((/** @type {string} */ s) => `${s} (${cachedBaseline.scenarios[s].rawRuns?.length || 0}/${opts.runs})`).join(', ')}`);
|
|
}
|
|
console.log(`[chat-simulation] Running baseline for ${scenariosToRun.length} scenario(s)...`);
|
|
const baselineExePath = baselineElectronPath || await resolveBuild(opts.baselineBuild);
|
|
for (const scenario of scenariosToRun) {
|
|
const existingRuns = cachedBaseline.scenarios?.[scenario]?.rawRuns || [];
|
|
const runsNeeded = opts.runs - existingRuns.length;
|
|
/** @type {RunMetrics[]} */
|
|
const newResults = [];
|
|
for (let i = 0; i < runsNeeded; i++) {
|
|
try {
|
|
const m = await runOnce(baselineExePath, scenario, mockServer, opts.verbose, `baseline-${scenario}-${existingRuns.length + i}`, runDir, 'baseline', baselineSettings, { heapSnapshots: opts.heapSnapshots });
|
|
// Clean up previous run's diagnostics to bound disk usage; keep the latest
|
|
if (opts.cleanupDiagnostics && newResults.length > 0) { cleanupRunDiagnostics(newResults[newResults.length - 1]); }
|
|
newResults.push(m);
|
|
}
|
|
catch (err) { console.error(`[chat-simulation] Baseline run ${i + 1} failed: ${err}`); }
|
|
}
|
|
const allRuns = [...existingRuns, ...newResults];
|
|
if (allRuns.length > 0) {
|
|
const sd = /** @type {any} */ ({ runs: allRuns.length, timing: {}, memory: {}, rendering: {}, extHost: {}, rawRuns: allRuns });
|
|
for (const [metric, group] of METRIC_DEFS) { sd[group][metric] = robustStats(allRuns.map((/** @type {any} */ r) => r[metric])); }
|
|
cachedBaseline.scenarios[scenario] = sd;
|
|
}
|
|
}
|
|
cachedBaseline.runsPerScenario = opts.runs;
|
|
fs.writeFileSync(baselineJsonPath, JSON.stringify(cachedBaseline, null, 2));
|
|
if (cachedPath) {
|
|
fs.writeFileSync(cachedPath, JSON.stringify(cachedBaseline, null, 2));
|
|
}
|
|
opts.baseline = baselineJsonPath;
|
|
}
|
|
} else {
|
|
const baselineExePath = baselineElectronPath || await resolveBuild(opts.baselineBuild);
|
|
console.log(`[chat-simulation] Benchmarking baseline build (${baselineLabel})...`);
|
|
/** @type {Record<string, RunMetrics[]>} */
|
|
const baselineResults = {};
|
|
for (const scenario of opts.scenarios) {
|
|
/** @type {RunMetrics[]} */
|
|
const results = [];
|
|
for (let i = 0; i < opts.runs; i++) {
|
|
try {
|
|
const m = await runOnce(baselineExePath, scenario, mockServer, opts.verbose, `baseline-${scenario}-${i}`, runDir, 'baseline', baselineSettings, { heapSnapshots: opts.heapSnapshots });
|
|
// Clean up previous run's diagnostics to bound disk usage; keep the latest
|
|
if (opts.cleanupDiagnostics && results.length > 0) { cleanupRunDiagnostics(results[results.length - 1]); }
|
|
results.push(m);
|
|
}
|
|
catch (err) { console.error(`[chat-simulation] Baseline run ${i + 1} failed: ${err}`); }
|
|
}
|
|
if (results.length > 0) { baselineResults[scenario] = results; }
|
|
}
|
|
const baselineReport = {
|
|
timestamp: new Date().toISOString(),
|
|
baselineBuildVersion: opts.baselineBuild,
|
|
platform: process.platform,
|
|
runsPerScenario: opts.runs,
|
|
scenarios: /** @type {Record<string, any>} */ ({}),
|
|
};
|
|
for (const [scenario, results] of Object.entries(baselineResults)) {
|
|
const sd = /** @type {any} */ ({ runs: results.length, timing: {}, memory: {}, rendering: {}, extHost: {}, rawRuns: results });
|
|
for (const [metric, group] of METRIC_DEFS) { sd[group][metric] = robustStats(results.map(r => /** @type {any} */(r)[metric])); }
|
|
baselineReport.scenarios[scenario] = sd;
|
|
}
|
|
fs.writeFileSync(baselineJsonPath, JSON.stringify(baselineReport, null, 2));
|
|
// Cache at the top level for reuse across runs (version strings only)
|
|
if (cachedPath) {
|
|
fs.writeFileSync(cachedPath, JSON.stringify(baselineReport, null, 2));
|
|
}
|
|
opts.baseline = baselineJsonPath;
|
|
}
|
|
console.log('');
|
|
}
|
|
|
|
// -- Run benchmarks --------------------------------------------------
|
|
console.log(`[chat-simulation] Electron: ${electronPath}`);
|
|
console.log(`[chat-simulation] Build mode: ${buildModeLabel(testBuildMode)}`);
|
|
if (baselineBuildMode) {
|
|
console.log(`[chat-simulation] Baseline mode: ${buildModeLabel(baselineBuildMode)}`);
|
|
}
|
|
console.log(`[chat-simulation] Runs per scenario: ${opts.runs}`);
|
|
console.log(`[chat-simulation] Scenarios: ${opts.scenarios.join(', ')}`);
|
|
if (Object.keys(opts.settingsOverrides).length > 0) {
|
|
console.log(`[chat-simulation] Settings overrides (all): ${JSON.stringify(opts.settingsOverrides)}`);
|
|
}
|
|
if (Object.keys(opts.testSettingsOverrides).length > 0) {
|
|
console.log(`[chat-simulation] Settings overrides (test): ${JSON.stringify(opts.testSettingsOverrides)}`);
|
|
}
|
|
if (Object.keys(opts.baselineSettingsOverrides).length > 0) {
|
|
console.log(`[chat-simulation] Settings overrides (baseline): ${JSON.stringify(opts.baselineSettingsOverrides)}`);
|
|
}
|
|
|
|
if (isMismatchedBuildMode) {
|
|
console.log('');
|
|
console.log(`[chat-simulation] ⚠ WARNING: Build mode mismatch — test is ${testBuildMode}, baseline is ${baselineBuildMode}.`);
|
|
console.log('[chat-simulation] Results may not be directly comparable. For apples-to-apples');
|
|
console.log('[chat-simulation] comparisons, use the same build type for both.');
|
|
if (testBuildMode === 'dev') {
|
|
console.log('[chat-simulation] To use a local production build instead:');
|
|
console.log('[chat-simulation] npm run perf:chat -- --production-build');
|
|
}
|
|
if (!opts.ci && !opts.force) {
|
|
const readline = require('readline');
|
|
const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
|
|
const answer = await new Promise(resolve => rl.question('[chat-simulation] Continue anyway? [y/N] ', resolve));
|
|
rl.close();
|
|
if (String(answer).toLowerCase() !== 'y') {
|
|
console.log('[chat-simulation] Aborted.');
|
|
await mockServer.close();
|
|
process.exit(0);
|
|
}
|
|
}
|
|
}
|
|
console.log('');
|
|
|
|
/** @type {Record<string, RunMetrics[]>} */
|
|
const allResults = {};
|
|
let anyFailed = false;
|
|
|
|
for (const scenario of opts.scenarios) {
|
|
console.log(`[chat-simulation] === Scenario: ${scenario} ===`);
|
|
/** @type {RunMetrics[]} */
|
|
const results = [];
|
|
for (let i = 0; i < opts.runs; i++) {
|
|
console.log(`[chat-simulation] Run ${i + 1}/${opts.runs}...`);
|
|
try {
|
|
const metrics = await runOnce(electronPath, scenario, mockServer, opts.verbose, `${scenario}-${i}`, runDir, 'test', testSettings, { heapSnapshots: opts.heapSnapshots });
|
|
// Clean up previous run's diagnostics to bound disk usage; keep the latest
|
|
if (opts.cleanupDiagnostics && results.length > 0) { cleanupRunDiagnostics(results[results.length - 1]); }
|
|
results.push(metrics);
|
|
if (opts.verbose) {
|
|
const src = metrics.hasInternalMarks ? 'internal' : 'client-side';
|
|
console.log(` [${src}] firstToken=${metrics.timeToFirstToken}ms, complete=${metrics.timeToComplete}ms, heap=delta${metrics.heapDelta}MB, longTasks=${metrics.longTaskCount}${metrics.hasInternalMarks ? `, internalTTFT=${metrics.internalFirstToken}ms` : ''}`);
|
|
}
|
|
} catch (err) { console.error(` Run ${i + 1} failed: ${err}`); }
|
|
}
|
|
if (results.length === 0) { console.error(`[chat-simulation] All runs failed for scenario: ${scenario}`); anyFailed = true; }
|
|
else { allResults[scenario] = results; }
|
|
console.log('');
|
|
}
|
|
|
|
// -- Summary ---------------------------------------------------------
|
|
console.log('[chat-simulation] ======================= Summary =======================');
|
|
for (const [scenario, results] of Object.entries(allResults)) {
|
|
console.log('');
|
|
console.log(` -- ${scenario} (${results.length} runs) --`);
|
|
console.log('');
|
|
console.log(' Timing:');
|
|
console.log(summarize(results.map(r => r.timeToFirstToken), ' Request → First token ', 'ms'));
|
|
console.log(summarize(results.map(r => r.timeToComplete), ' Request → Complete ', 'ms'));
|
|
console.log(summarize(results.map(r => r.timeToRenderComplete), ' Request → Rendered ', 'ms'));
|
|
console.log('');
|
|
console.log(' Rendering:');
|
|
console.log(summarize(results.map(r => r.layoutCount), ' Layouts ', ''));
|
|
console.log(summarize(results.map(r => r.layoutDurationMs), ' Layout duration ', 'ms'));
|
|
console.log(summarize(results.map(r => r.recalcStyleCount), ' Style recalcs ', ''));
|
|
console.log(summarize(results.map(r => r.forcedReflowCount), ' Forced reflows ', ''));
|
|
console.log(summarize(results.map(r => r.longTaskCount), ' Long tasks (>50ms) ', ''));
|
|
console.log(summarize(results.map(r => r.longAnimationFrameCount), ' Long anim. frames ', ''));
|
|
console.log(summarize(results.map(r => r.longAnimationFrameTotalMs), ' LoAF total duration ', 'ms'));
|
|
console.log(summarize(results.map(r => r.frameCount), ' Frames ', ''));
|
|
console.log(summarize(results.map(r => r.compositeLayers), ' Composite layers ', ''));
|
|
console.log(summarize(results.map(r => r.paintCount), ' Paints ', ''));
|
|
console.log('');
|
|
console.log(' Memory:');
|
|
console.log(summarize(results.map(r => r.heapDelta), ' Heap delta ', 'MB'));
|
|
console.log(summarize(results.map(r => r.heapDeltaPostGC), ' Heap delta (post-GC) ', 'MB'));
|
|
console.log(summarize(results.map(r => r.gcDurationMs), ' GC duration ', 'ms'));
|
|
if (results.some(r => r.extHostHeapDelta >= 0)) {
|
|
console.log('');
|
|
console.log(' Extension Host:');
|
|
console.log(summarize(results.map(r => r.extHostHeapUsedBefore), ' Heap before ', 'MB'));
|
|
console.log(summarize(results.map(r => r.extHostHeapUsedAfter), ' Heap after ', 'MB'));
|
|
console.log(summarize(results.map(r => r.extHostHeapDelta), ' Heap delta ', 'MB'));
|
|
console.log(summarize(results.map(r => r.extHostHeapDeltaPostGC), ' Heap delta (post-GC) ', 'MB'));
|
|
}
|
|
}
|
|
|
|
// -- JSON output -----------------------------------------------------
|
|
const jsonPath = path.join(runDir, 'results.json');
|
|
const jsonReport = /** @type {{ timestamp: string, platform: NodeJS.Platform, runsPerScenario: number, buildMode: string, mismatchedBuildMode: boolean, scenarios: Record<string, any>, _resultsPath?: string }} */ ({
|
|
timestamp: new Date().toISOString(),
|
|
platform: process.platform,
|
|
runsPerScenario: opts.runs,
|
|
buildMode: testBuildMode,
|
|
mismatchedBuildMode: !!isMismatchedBuildMode,
|
|
scenarios: /** @type {Record<string, any>} */ ({}),
|
|
});
|
|
for (const [scenario, results] of Object.entries(allResults)) {
|
|
const sd = /** @type {any} */ ({ runs: results.length, timing: {}, memory: {}, rendering: {}, extHost: {}, rawRuns: results });
|
|
for (const [metric, group] of METRIC_DEFS) { sd[group][metric] = robustStats(results.map(r => /** @type {any} */(r)[metric])); }
|
|
jsonReport.scenarios[scenario] = sd;
|
|
}
|
|
fs.writeFileSync(jsonPath, JSON.stringify(jsonReport, null, 2));
|
|
jsonReport._resultsPath = jsonPath;
|
|
console.log('');
|
|
console.log(`[chat-simulation] Results written to ${jsonPath}`);
|
|
|
|
// -- Save baseline ---------------------------------------------------
|
|
if (opts.saveBaseline) {
|
|
if (!opts.baseline) { console.error('[chat-simulation] --save-baseline requires --baseline <path>'); process.exit(1); }
|
|
fs.writeFileSync(opts.baseline, JSON.stringify(jsonReport, null, 2));
|
|
console.log(`[chat-simulation] Baseline saved to ${opts.baseline}`);
|
|
}
|
|
|
|
// -- Baseline comparison ---------------------------------------------
|
|
const regressedScenarios = await printComparison(jsonReport, opts);
|
|
|
|
// Clean up diagnostics for scenarios that did not regress
|
|
if (opts.cleanupDiagnostics) {
|
|
cleanupNonRegressedDiagnostics(allResults, regressedScenarios);
|
|
}
|
|
|
|
if (anyFailed) { process.exit(1); }
|
|
await mockServer.close();
|
|
}
|
|
|
|
/**
|
|
* Print baseline comparison and exit with code 1 if regressions found.
|
|
* Returns the set of scenario IDs that regressed.
|
|
* @param {Record<string, any>} jsonReport
|
|
* @param {{ threshold: number, metricThresholds?: Record<string, number | string>, baseline?: string, ci?: boolean, resume?: string, build?: string, baselineBuild?: string, runs: number, cleanupDiagnostics?: boolean }} opts
|
|
* @returns {Promise<Set<string>>}
|
|
*/
|
|
async function printComparison(jsonReport, opts) {
|
|
let regressionFound = false;
|
|
let inconclusiveFound = false;
|
|
/** @type {Set<string>} */
|
|
const regressedScenarios = new Set();
|
|
if (opts.baseline && fs.existsSync(opts.baseline)) {
|
|
const baseline = JSON.parse(fs.readFileSync(opts.baseline, 'utf-8'));
|
|
console.log('');
|
|
console.log(`[chat-simulation] =========== Baseline Comparison (threshold: ${(opts.threshold * 100).toFixed(0)}%) ===========`);
|
|
console.log(`[chat-simulation] Baseline: ${baseline.baselineBuildVersion || baseline.timestamp}`);
|
|
if (jsonReport.mismatchedBuildMode) {
|
|
console.log(`[chat-simulation] ⚠ Note: build mode mismatch — test is ${jsonReport.buildMode}, baseline differs.`);
|
|
console.log('[chat-simulation] Results may not be directly comparable.');
|
|
}
|
|
console.log('');
|
|
|
|
// Metrics that trigger regression failure when they exceed the threshold
|
|
const regressionMetrics = [
|
|
// [metric, group, unit]
|
|
['timeToFirstToken', 'timing', 'ms'],
|
|
['timeToComplete', 'timing', 'ms'],
|
|
['layoutCount', 'rendering', ''],
|
|
['recalcStyleCount', 'rendering', ''],
|
|
['forcedReflowCount', 'rendering', ''],
|
|
['longTaskCount', 'rendering', ''],
|
|
];
|
|
// Informational metrics — shown in comparison but don't trigger failure
|
|
const infoMetrics = [
|
|
['heapDelta', 'memory', 'MB'],
|
|
['gcDurationMs', 'memory', 'ms'],
|
|
['extHostHeapDelta', 'extHost', 'MB'],
|
|
['extHostHeapDeltaPostGC', 'extHost', 'MB'],
|
|
];
|
|
|
|
for (const scenario of Object.keys(jsonReport.scenarios)) {
|
|
const current = jsonReport.scenarios[scenario];
|
|
const base = baseline.scenarios?.[scenario];
|
|
if (!base) { console.log(` ${scenario}: (no baseline)`); continue; }
|
|
|
|
/** @type {string[]} */
|
|
const diffs = [];
|
|
let scenarioRegression = false;
|
|
|
|
for (const [metric, group, unit] of regressionMetrics) {
|
|
const cur = current[group]?.[metric];
|
|
const bas = base[group]?.[metric];
|
|
if (!cur || !bas || !bas.median) { continue; }
|
|
const change = (cur.median - bas.median) / bas.median;
|
|
const pct = `${change > 0 ? '+' : ''}${(change * 100).toFixed(1)}%`;
|
|
|
|
// Statistical significance via Welch's t-test on raw run values
|
|
const curRaw = (current.rawRuns || []).map((/** @type {any} */ r) => r[metric]).filter((/** @type {any} */ v) => v >= 0);
|
|
const basRaw = (base.rawRuns || []).map((/** @type {any} */ r) => r[metric]).filter((/** @type {any} */ v) => v >= 0);
|
|
const ttest = welchTTest(basRaw, curRaw);
|
|
|
|
const metricThreshold = getMetricThreshold(opts, metric);
|
|
const absoluteDelta = cur.median - bas.median;
|
|
let flag = '';
|
|
if (exceedsThreshold(metricThreshold, change, absoluteDelta)) {
|
|
if (!ttest) {
|
|
flag = ' ← possible regression (n too small for significance test)';
|
|
inconclusiveFound = true;
|
|
} else if (ttest.significant) {
|
|
flag = ` ← REGRESSION (p=${ttest.pValue}, ${ttest.confidence} confidence)`;
|
|
scenarioRegression = true;
|
|
regressionFound = true;
|
|
} else {
|
|
flag = ` (likely noise — p=${ttest.pValue}, not significant)`;
|
|
inconclusiveFound = true;
|
|
}
|
|
} else if (ttest && change > 0 && ttest.significant && ttest.confidence === 'high') {
|
|
flag = ` (significant increase, p=${ttest.pValue})`;
|
|
}
|
|
diffs.push(` ${metric}: ${bas.median}${unit} → ${cur.median}${unit} (${pct})${flag}`);
|
|
}
|
|
for (const [metric, group, unit] of infoMetrics) {
|
|
const cur = current[group]?.[metric];
|
|
const bas = base[group]?.[metric];
|
|
if (!cur || !bas || bas.median === null || bas.median === undefined) { continue; }
|
|
const change = bas.median !== 0 ? (cur.median - bas.median) / bas.median : 0;
|
|
const pct = `${change > 0 ? '+' : ''}${(change * 100).toFixed(1)}%`;
|
|
diffs.push(` ${metric}: ${bas.median}${unit} → ${cur.median}${unit} (${pct}) [info]`);
|
|
}
|
|
console.log(` ${scenario}: ${scenarioRegression ? 'FAIL' : 'OK'}`);
|
|
if (scenarioRegression) { regressedScenarios.add(scenario); }
|
|
diffs.forEach(d => console.log(d));
|
|
}
|
|
|
|
console.log('');
|
|
console.log(regressionFound
|
|
? `[chat-simulation] REGRESSION DETECTED — exceeded ${(opts.threshold * 100).toFixed(0)}% threshold with statistical significance`
|
|
: `[chat-simulation] All metrics within ${(opts.threshold * 100).toFixed(0)}% of baseline (or not statistically significant)`);
|
|
|
|
if (inconclusiveFound && !regressionFound) {
|
|
// Find the results.json path to suggest in the hint
|
|
const resultsPath = Object.keys(jsonReport.scenarios).length > 0
|
|
? (jsonReport._resultsPath || opts.resume || 'path/to/results.json')
|
|
: 'path/to/results.json';
|
|
// Estimate required runs from the observed effect size and variance
|
|
// using power analysis for Welch's t-test (alpha=0.05, 80% power).
|
|
// n_per_group = 2 * ((z_alpha/2 + z_beta) / d)^2 where d = Cohen's d
|
|
let maxNeeded = 0;
|
|
for (const scenario of Object.keys(jsonReport.scenarios)) {
|
|
const current = jsonReport.scenarios[scenario];
|
|
const base = baseline.scenarios?.[scenario];
|
|
if (!base) { continue; }
|
|
for (const [metric, group] of [['timeToFirstToken', 'timing'], ['timeToComplete', 'timing'], ['layoutCount', 'rendering'], ['recalcStyleCount', 'rendering']]) {
|
|
const curRaw = (current.rawRuns || []).map((/** @type {any} */ r) => r[metric]).filter((/** @type {any} */ v) => v >= 0);
|
|
const basRaw = (base.rawRuns || []).map((/** @type {any} */ r) => r[metric]).filter((/** @type {any} */ v) => v >= 0);
|
|
if (curRaw.length < 2 || basRaw.length < 2) { continue; }
|
|
const meanA = basRaw.reduce((/** @type {number} */ s, /** @type {number} */ v) => s + v, 0) / basRaw.length;
|
|
const meanB = curRaw.reduce((/** @type {number} */ s, /** @type {number} */ v) => s + v, 0) / curRaw.length;
|
|
const varA = basRaw.reduce((/** @type {number} */ s, /** @type {number} */ v) => s + (v - meanA) ** 2, 0) / (basRaw.length - 1);
|
|
const varB = curRaw.reduce((/** @type {number} */ s, /** @type {number} */ v) => s + (v - meanB) ** 2, 0) / (curRaw.length - 1);
|
|
const pooledSD = Math.sqrt((varA + varB) / 2);
|
|
if (pooledSD === 0) { continue; }
|
|
const d = Math.abs(meanB - meanA) / pooledSD;
|
|
if (d === 0) { continue; }
|
|
// z_0.025 = 1.96, z_0.2 = 0.842
|
|
const nPerGroup = Math.ceil(2 * ((1.96 + 0.842) / d) ** 2);
|
|
const currentN = Math.min(curRaw.length, basRaw.length);
|
|
maxNeeded = Math.max(maxNeeded, nPerGroup - currentN);
|
|
}
|
|
}
|
|
const suggestedRuns = Math.max(1, Math.min(maxNeeded, 20));
|
|
console.log('');
|
|
console.log('[chat-simulation] Some metrics exceeded the threshold but were not statistically significant.');
|
|
console.log('[chat-simulation] To increase confidence, add more runs with --resume:');
|
|
console.log(`[chat-simulation] npm run perf:chat -- --resume ${resultsPath} --runs ${suggestedRuns}`);
|
|
}
|
|
}
|
|
|
|
// -- CI summary ------------------------------------------------------
|
|
if (opts.ci) {
|
|
const ciBaseline = opts.baseline && fs.existsSync(opts.baseline)
|
|
? JSON.parse(fs.readFileSync(opts.baseline, 'utf-8'))
|
|
: null;
|
|
const summary = generateCISummary(jsonReport, ciBaseline, {
|
|
threshold: opts.threshold,
|
|
metricThresholds: opts.metricThresholds,
|
|
runs: jsonReport.runsPerScenario || opts.runs,
|
|
baselineBuild: ciBaseline?.baselineBuildVersion || opts.baselineBuild,
|
|
build: opts.build,
|
|
});
|
|
|
|
// Write to file for GitHub Actions $GITHUB_STEP_SUMMARY
|
|
const summaryPath = path.join(DATA_DIR, 'ci-summary.md');
|
|
fs.writeFileSync(summaryPath, summary);
|
|
console.log(`[chat-simulation] CI summary written to ${summaryPath}`);
|
|
|
|
// Also print the full summary table to stdout
|
|
console.log('');
|
|
console.log('==================================================================');
|
|
console.log(' CHAT PERF COMPARISON RESULTS ');
|
|
console.log('==================================================================');
|
|
console.log('');
|
|
console.log(summary);
|
|
}
|
|
|
|
if (regressionFound) { process.exit(1); }
|
|
return regressedScenarios;
|
|
}
|
|
|
|
main().catch(err => { console.error(err); process.exit(1); });
|