From e6967d8b45daa06db787c14d8d18bf4e3e528db1 Mon Sep 17 00:00:00 2001 From: Giuseppe Cianci Date: Wed, 17 Jun 2026 18:14:08 +0200 Subject: [PATCH] Codex: add Agents-Window smoke test (real app-server, mock LLM) Adds an end-to-end smoke test for the codex harness that uses the REAL codex app-server and fakes only the LLM. It reuses the existing Agents-Window smoke harness, which already drives: real Agents-Window UI -> agent host -> real spawned `codex` binary -> real CodexProxyService -> mock LLM server The mock LLM server (scripts/chat-simulation/common/mock-llm-server.ts) already speaks codex's OpenAI `/responses` API and ships a `gpt-5.3-codex` `/responses` chat-default model, so no LLM-side changes are needed. - test/smoke/.../agentsWindow.test.ts: new `Agents Window (Codex)` suite via the existing setupAgentHostSuite helper with `chat.agentHost.codexAgent.enabled` (codex is off by default). The test selects the `Codex` session type, submits a scenario prompt, and asserts the mocked reply plus a `chat/turnStarted` frame in the AHP transcript (proving the codex harness, not a renderer fallback, served it). Adds a warmUpCodexModel helper mirroring warmUpClaudeModel. - test/automation/src/agentsWindow.ts: new isSessionTypeAvailable(label) used to skip gracefully when codex isn't registered. Codex only registers when its SDK is resolvable (product.agentSdks.codex in packaged builds, or VSCODE_AGENT_HOST_CODEX_SDK_ROOT in dev); when absent the test skips rather than fails. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- test/automation/src/agentsWindow.ts | 31 +++++ .../areas/agentsWindow/agentsWindow.test.ts | 110 ++++++++++++++++++ 2 files changed, 141 insertions(+) diff --git a/test/automation/src/agentsWindow.ts b/test/automation/src/agentsWindow.ts index 6d97bf69823..847ae6adaaa 100644 --- a/test/automation/src/agentsWindow.ts +++ b/test/automation/src/agentsWindow.ts @@ -80,6 +80,37 @@ export class AgentsWindow { await this.code.waitForElement(SESSION_TYPE_PICKER_VISIBLE, undefined, retryCount); } + /** + * Returns whether the given session type appears in the new-session + * picker. Opens the picker, polls its async-populated rows for up to + * `timeoutMs`, then dismisses the dropdown (Escape) so it does not block + * subsequent interactions. Use this to gate tests on optionally-registered + * providers (e.g. Codex, which only registers when its SDK is available) + * instead of relying on {@link selectSessionType} throwing. + */ + async isSessionTypeAvailable(label: string, timeoutMs: number = 10_000): Promise { + await this.code.waitForElement(SESSION_TYPE_PICKER_VISIBLE); + await this.code.waitAndClick(SESSION_TYPE_PICKER_VISIBLE); + + const itemSel = `.action-widget .monaco-list-row`; + const needle = label.toLowerCase(); + const deadline = Date.now() + timeoutMs; + let found = false; + while (Date.now() < deadline) { + const items = await this.code.getElements(itemSel, /* recursive */ true); + const labels = (items ?? []).map(i => (i.textContent ?? '').trim()); + if (labels.some(t => t.toLowerCase().includes(needle))) { + found = true; + break; + } + await new Promise(r => setTimeout(r, 250)); + } + + // Dismiss the dropdown so it does not intercept later clicks. + await this.code.dispatchKeybinding('escape', async () => { await this.code.waitForElement(itemSel, el => !el, 20); }); + return found; + } + /** * Select the given session type from the new-session picker. * diff --git a/test/smoke/src/areas/agentsWindow/agentsWindow.test.ts b/test/smoke/src/areas/agentsWindow/agentsWindow.test.ts index 889021356c3..643f2835d8e 100644 --- a/test/smoke/src/areas/agentsWindow/agentsWindow.test.ts +++ b/test/smoke/src/areas/agentsWindow/agentsWindow.test.ts @@ -32,6 +32,15 @@ const LOCAL_REPLY = 'MOCKED_LOCAL_RESPONSE'; const CLAUDE_SCENARIO_ID = 'smoke-hello-claude'; const CLAUDE_REPLY = 'MOCKED_CLAUDE_RESPONSE'; +const CODEX_SCENARIO_ID = 'smoke-hello-codex'; +const CODEX_REPLY = 'MOCKED_CODEX_RESPONSE'; + +// Lightweight throwaway scenario used by {@link warmUpCodexModel} to pre-pay +// the Codex session cold-start cost (native codex app-server spawn + model +// list resolution) before the real assertion runs. +const CODEX_WARMUP_SCENARIO_ID = 'smoke-hello-codex-warmup'; +const CODEX_WARMUP_REPLY = 'MOCKED_CODEX_WARMUP_RESPONSE'; + // Lightweight throwaway scenario used by {@link warmUpClaudeModel} to // pre-pay the Claude session cold-start cost (bundled SDK import, language // model server startup, SDK subprocess spawn, plugin loading) before the @@ -538,6 +547,79 @@ export function setup(logger: Logger) { } }); }); + + describe('Agents Window (Codex)', () => { + + const codex = setupAgentHostSuite(logger, { + serverLabel: 'Codex', + registerScenarios: ({ ScenarioBuilder, registerScenario }) => { + registerScenario(CODEX_SCENARIO_ID, new ScenarioBuilder().emit(CODEX_REPLY).build()); + registerScenario(CODEX_WARMUP_SCENARIO_ID, new ScenarioBuilder().emit(CODEX_WARMUP_REPLY).build()); + }, + settings: { + // Register the Codex provider in the agent host process (it is + // off by default). The provider only actually appears if the + // codex SDK is resolvable (product.agentSdks.codex in packaged + // builds, or VSCODE_AGENT_HOST_CODEX_SDK_ROOT in dev) — the test + // skips gracefully when it is not. + 'chat.agentHost.codexAgent.enabled': true, + }, + }); + + it('Test Codex session', async function () { + this.timeout(5 * 60 * 1000); + + const app = this.app as Application; + + // Gate on Codex availability OUTSIDE the try/catch below so that the + // Pending thrown by `this.skip()` is not swallowed (and re-thrown as a + // failure) by the failure-diagnostics handler. Codex registers only + // when its native SDK is resolvable; until it ships in the build this + // keeps the suite green instead of red. + await app.workbench.agentsWindow.waitForNewSessionView(); + const codexAvailable = await app.workbench.agentsWindow.isSessionTypeAvailable('Codex'); + if (!codexAvailable) { + logger.log('[Agents Window/Codex] Codex session type not available (no product.agentSdks.codex / VSCODE_AGENT_HOST_CODEX_SDK_ROOT); skipping'); + this.skip(); + } + + try { + // Pre-pay the Codex session cold-start cost: the first Codex session + // in a fresh agent host has to spawn the native codex app-server and + // resolve its model list before the first /responses request can + // complete. A throwaway prompt absorbs that so the real assertion + // runs against a warm pipeline. + await warmUpCodexModel(app, logger, 'Agents Window/Codex'); + + const requestsBefore = codex.mockServer.requestCount(); + logger.log(`[Agents Window/Codex] submitting prompt; requestCount=${requestsBefore}`); + await app.workbench.agentsWindow.submitNewSessionPrompt(`hello world [scenario:${CODEX_SCENARIO_ID}]`); + + const text = await app.workbench.agentsWindow.waitForAssistantText(CODEX_REPLY); + logger.log(`[Agents Window/Codex] response (length=${text.length}): ${text}`); + + assert.ok( + codex.mockServer.requestCount() > requestsBefore, + `expected the mock LLM server to have received a new request from the Codex session (before=${requestsBefore}, after=${codex.mockServer.requestCount()})` + ); + + // Confirm the request flowed through the AgentHost process (the codex + // harness) and not a renderer-side fallback by checking for a + // `chat/turnStarted` frame in the AHP JSONL transcript. The transcript + // is written through an async queue, so poll briefly. + const ahpLogDir = path.join(codex.logsPath, 'ahp'); + const ahpFrames = await waitForLogContent(() => readAhpFrames(ahpLogDir), '"type":"chat/turnStarted"'); + assert.ok( + ahpFrames.includes('"type":"chat/turnStarted"'), + `expected the AgentHost process to have received a chat/turnStarted dispatchAction (checked ${ahpJsonlFiles(ahpLogDir).length} jsonl files under ${ahpLogDir}); if missing, the renderer-side extension likely served the reply instead` + ); + } catch (error) { + logger.log(`[Agents Window/Codex] FAILURE: ${error instanceof Error ? error.stack ?? error.message : String(error)}`); + await dumpFailureDiagnostics(app, logger, 'Agents Window/Codex', { sendButtonSelector: AGENTS_SEND_BUTTON_SELECTOR }); + throw error; + } + }); + }); } /** @@ -650,6 +732,34 @@ async function warmUpClaudeModel(app: Application, logger: Logger, label: string await app.workbench.agentsWindow.selectSessionType('Claude'); } +/** + * Pre-pays the Codex session cold-start cost: the first Codex session in a + * fresh agent host has to spawn the native `codex app-server` binary and + * resolve its model list before the first `/responses` request can complete. + * + * Assumes the Agents Window is showing a new-session view AND that the 'Codex' + * session type is available (callers gate on + * {@link AgentsWindow.isSessionTypeAvailable} first). Sends a throwaway prompt + * to a 'Codex' session, ignores its outcome (the warm-up itself may hit the + * cold start), then leaves a fresh new-session view with 'Codex' selected so + * the caller can submit the real prompt against a warm pipeline. + */ +async function warmUpCodexModel(app: Application, logger: Logger, label: string): Promise { + await app.workbench.agentsWindow.waitForNewSessionView(); + await app.workbench.agentsWindow.selectSessionType('Codex'); + await app.workbench.agentsWindow.submitNewSessionPrompt(`hello world [scenario:${CODEX_WARMUP_SCENARIO_ID}]`); + try { + await app.workbench.agentsWindow.waitForAssistantText(CODEX_WARMUP_REPLY, 60_000); + } catch (error) { + // Ignore — the warm-up itself may hit the cold-start race; the caller's + // real attempt runs against an already-warmed pipeline. + logger.log(`${label} warm-up attempt did not produce the expected reply (likely the cold-start race); proceeding with the real attempt. Reason: ${error instanceof Error ? error.message : String(error)}`); + } + await app.workbench.agentsWindow.startNewSession(); + await app.workbench.agentsWindow.waitForNewSessionView(); + await app.workbench.agentsWindow.selectSessionType('Codex'); +} + /** * Accessors for the per-suite state owned by {@link setupAgentHostSuite}. * Implemented with getters so tests read the values populated by the