diff --git a/scripts/chat-simulation/common/mock-llm-server.js b/scripts/chat-simulation/common/mock-llm-server.js index 6f19270a96e..7b3dbf888e9 100644 --- a/scripts/chat-simulation/common/mock-llm-server.js +++ b/scripts/chat-simulation/common/mock-llm-server.js @@ -995,6 +995,24 @@ async function handleMessagesApi(body, res) { break; } } + + // Anthropic's Messages API also accepts a top-level `system` parameter + // (string or array of `{ type: 'text', text }` blocks). Some session + // types (e.g. Claude Code) embed the user prompt there alongside the + // system instructions, so scan it as a fallback when no tag was found + // in the messages array. + if (!isScenarioRequest && parsed.system !== undefined) { + const systemContent = typeof parsed.system === 'string' + ? parsed.system + : Array.isArray(parsed.system) + ? parsed.system.map((/** @type {any} */ c) => c.text || '').join('') + : ''; + const match = systemContent.match(/\[scenario:([^\]]+)\]/); + if (match && SCENARIOS[match[1]]) { + scenarioId = match[1]; + isScenarioRequest = true; + } + } } catch { } const scenario = SCENARIOS[scenarioId] || SCENARIOS[DEFAULT_SCENARIO]; diff --git a/test/automation/src/chat.ts b/test/automation/src/chat.ts index 9274114851e..a3596e6ba37 100644 --- a/test/automation/src/chat.ts +++ b/test/automation/src/chat.ts @@ -9,11 +9,13 @@ const CHAT_VIEW = 'div[id="workbench.panel.chat"]'; const CHAT_EDITOR = '.editor-instance .interactive-session'; const CHAT_INPUT_EDITOR = `${CHAT_VIEW} .interactive-input-part .monaco-editor[role="code"]`; const CHAT_INPUT_EDITOR_FOCUSED = `${CHAT_VIEW} .interactive-input-part .monaco-editor.focused[role="code"]`; +const CHAT_SEND_BUTTON_ENABLED = `${CHAT_VIEW} .chat-input-toolbars > .chat-execute-toolbar .monaco-action-bar .action-item:not(.disabled) > .action-label.codicon-arrow-up`; const CHAT_RESPONSE = `${CHAT_VIEW} .interactive-item-container.interactive-response`; const CHAT_RESPONSE_COMPLETE = `${CHAT_RESPONSE}:not(.chat-response-loading)`; const CHAT_FOOTER_DETAILS = `${CHAT_VIEW} .chat-footer-details`; const CHAT_EDITOR_INPUT_EDITOR = `${CHAT_EDITOR} .interactive-input-part .monaco-editor[role="code"]`; const CHAT_EDITOR_INPUT_EDITOR_FOCUSED = `${CHAT_EDITOR} .interactive-input-part .monaco-editor.focused[role="code"]`; +const CHAT_EDITOR_SEND_BUTTON_ENABLED = `${CHAT_EDITOR} .chat-input-toolbars > .chat-execute-toolbar .monaco-action-bar .action-item:not(.disabled) > .action-label.codicon-arrow-up`; const CHAT_EDITOR_RESPONSE = `${CHAT_EDITOR} .interactive-item-container.interactive-response`; const CHAT_EDITOR_RESPONSE_COMPLETE = `${CHAT_EDITOR_RESPONSE}:not(.chat-response-loading)`; @@ -48,23 +50,38 @@ export class Chat { // Wait for the editor to be focused await this.waitForInputFocus(); - // Type the message using pressSequentially - this works with Monaco editors - // Note: Newlines are replaced with spaces since Enter key submits in chat input + // Insert via Monaco's executeEdits rather than character-by-character + // keypresses so suggestion widgets (e.g. the `[`-triggered chat reference + // picker) cannot intercept characters and corrupt the prompt. + // Newlines are replaced with spaces since Enter submits in chat input. const sanitizedMessage = message.replace(/\n/g, ' '); - await this.code.driver.currentPage.locator(this.chatInputSelector).pressSequentially(sanitizedMessage); + await this.code.waitForTypeInEditor(this.chatInputSelector, sanitizedMessage); - // Submit the message - await this.code.dispatchKeybinding('enter', () => Promise.resolve()); + // Wait for the send button to be enabled before clicking. The send + // button stays disabled until the chat participant is fully ready to + // receive a request — relying on Enter alone is fragile for providers + // that initialize asynchronously. + await this.code.waitForElement(CHAT_SEND_BUTTON_ENABLED, undefined, 600); + await this.code.waitAndClick(CHAT_SEND_BUTTON_ENABLED); } async sendEditorMessage(message: string): Promise { await this.code.waitAndClick(CHAT_EDITOR_INPUT_EDITOR); await this.code.waitForElement(CHAT_EDITOR_INPUT_EDITOR_FOCUSED); + // Insert via Monaco's executeEdits rather than character-by-character + // keypresses so suggestion widgets (e.g. the `[`-triggered chat reference + // picker) cannot intercept characters and corrupt the prompt. const sanitizedMessage = message.replace(/\n/g, ' '); - await this.code.driver.currentPage.locator(this.chatEditorInputSelector).pressSequentially(sanitizedMessage); + await this.code.waitForTypeInEditor(this.chatEditorInputSelector, sanitizedMessage); - await this.code.dispatchKeybinding('enter', () => Promise.resolve()); + // Wait for the send button to be enabled before clicking. The send + // button stays disabled until the chat session participant is fully + // ready to receive a request — relying on Enter alone is fragile for + // session types whose providers initialize asynchronously (e.g. Claude + // Agent). + await this.code.waitForElement(CHAT_EDITOR_SEND_BUTTON_ENABLED, undefined, 600); + await this.code.waitAndClick(CHAT_EDITOR_SEND_BUTTON_ENABLED); } async waitForResponse(retryCount?: number): Promise { @@ -86,6 +103,11 @@ export class Chat { return (await response.textContent()) ?? ''; } + async getLatestResponseText(): Promise { + const response = this.code.driver.currentPage.locator(CHAT_RESPONSE_COMPLETE).last(); + return (await response.textContent()) ?? ''; + } + async waitForModelInFooter(): Promise { await this.code.waitForElements(CHAT_FOOTER_DETAILS, false, el => { return el.some(el => { diff --git a/test/smoke/extensions/vscode-smoketest-ext-host/extension.js b/test/smoke/extensions/vscode-smoketest-ext-host/extension.js index 89f08a97cd9..67fa974bf55 100644 --- a/test/smoke/extensions/vscode-smoketest-ext-host/extension.js +++ b/test/smoke/extensions/vscode-smoketest-ext-host/extension.js @@ -102,6 +102,18 @@ function activate(context) { await vscode.commands.executeCommand(command); }) ); + + context.subscriptions.push( + vscode.commands.registerCommand('smoketest.openClaudeChat', async () => { + const command = 'workbench.action.chat.openNewSessionEditor.claude-code'; + await vscode.workspace.getConfiguration('chat').update('disableAIFeatures', false, vscode.ConfigurationTarget.Global); + await vscode.workspace.getConfiguration('github.copilot.chat').update('claudeAgent.enabled', true, vscode.ConfigurationTarget.Global); + await vscode.commands.executeCommand('github.copilot.debug.extensionState'); + await waitForCommand(command, 30_000); + await vscode.commands.executeCommand('workbench.action.closeAllEditors'); + await vscode.commands.executeCommand(command); + }) + ); } function deactivate() { diff --git a/test/smoke/extensions/vscode-smoketest-ext-host/package.json b/test/smoke/extensions/vscode-smoketest-ext-host/package.json index 6ccaddfee93..6cea05f86d7 100644 --- a/test/smoke/extensions/vscode-smoketest-ext-host/package.json +++ b/test/smoke/extensions/vscode-smoketest-ext-host/package.json @@ -27,6 +27,10 @@ { "command": "smoketest.openCopilotCliChat", "title": "Smoke Test: Open CLI Test Session" + }, + { + "command": "smoketest.openClaudeChat", + "title": "Smoke Test: Open Claude Test Session" } ] } diff --git a/test/smoke/src/areas/chat/chatSessions.test.ts b/test/smoke/src/areas/chat/chatSessions.test.ts new file mode 100644 index 00000000000..84584369baa --- /dev/null +++ b/test/smoke/src/areas/chat/chatSessions.test.ts @@ -0,0 +1,155 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as assert from 'assert'; +import { Application, Logger } from '../../../../automation'; +import { getCopilotSmokeTestEnv, getMockLlmServerPath, installAllHandlers, MockLlmServer } from '../../utils'; + +/** + * Per-test scenarios. Each test uses a unique scenario id so that the mock + * reply is distinct — this catches stale-content bugs where the previous + * test's response is mistakenly accepted as the current test's response. + */ +const COPILOT_CLI_SCENARIO_ID = 'smoke-chat-sessions-copilot-cli'; +const COPILOT_CLI_REPLY = 'MOCKED_CHAT_SESSIONS_COPILOT_CLI_RESPONSE'; + +const CLAUDE_SCENARIO_ID = 'smoke-chat-sessions-claude'; +const CLAUDE_REPLY = 'MOCKED_CHAT_SESSIONS_CLAUDE_RESPONSE'; + +const LOCAL_SCENARIO_ID = 'smoke-chat-sessions-local'; +const LOCAL_REPLY = 'MOCKED_CHAT_SESSIONS_LOCAL_RESPONSE'; + +export function setup(logger: Logger) { + + describe('Chat Sessions', function () { + this.timeout(3 * 60 * 1000); + this.retries(0); + + let mockServer: MockLlmServer; + + // Start the mock server BEFORE installAllHandlers' `before` runs so + // the mock URL is available when we configure the app's env vars via + // `optionsTransform`. + before(async function () { + const { startServer, ScenarioBuilder, registerScenario } = require(getMockLlmServerPath()); + + // Fallback for ancillary requests (title/branch) that don't carry a [scenario:...] tag. + registerScenario('text-only', new ScenarioBuilder().emit('OK').build()); + + // One scenario per session type, each emitting a distinct reply + // so the assertion is unambiguous. + registerScenario(COPILOT_CLI_SCENARIO_ID, new ScenarioBuilder().emit(COPILOT_CLI_REPLY).build()); + registerScenario(CLAUDE_SCENARIO_ID, new ScenarioBuilder().emit(CLAUDE_REPLY).build()); + registerScenario(LOCAL_SCENARIO_ID, new ScenarioBuilder().emit(LOCAL_REPLY).build()); + + mockServer = await startServer(0, { logger: (msg: string) => logger.log(msg) }); + logger.log(`Chat Sessions mock LLM server started at ${mockServer.url}`); + }); + + installAllHandlers(logger, opts => ({ + ...opts, + extraEnv: { + ...(opts.extraEnv ?? {}), + ...getCopilotSmokeTestEnv(mockServer), + }, + })); + + before(async function () { + const app = this.app as Application; + + // overrideProxyUrl/overrideCapiUrl redirect Copilot SDK + CAPI traffic + // to the mock server. allowAnonymousAccess skips the token-validation + // gate when there is no real GitHub session. The MCP/githubMcpServer + // settings prevent real-network MCP connections during the test. + await app.workbench.settingsEditor.addUserSettings([ + ['github.copilot.advanced.debug.overrideProxyUrl', JSON.stringify(mockServer.url)], + ['github.copilot.advanced.debug.overrideCapiUrl', JSON.stringify(mockServer.url)], + ['chat.allowAnonymousAccess', 'true'], + ['github.copilot.chat.githubMcpServer.enabled', 'false'], + ['chat.mcp.discovery.enabled', 'false'], + ['chat.mcp.enabled', 'false'], + // Force the bundled Claude Agent SDK (avoid the experiment that + // would route through the ms-vscode.vscode-claude-sdk extension, + // which would attempt a network install during the smoke run). + ['github.copilot.chat.claudeAgent.useSdkExtension', 'false'], + ]); + }); + + after(async function () { + await mockServer?.close(); + }); + + it('Test Copilot CLI session', async function () { + const app = this.app as Application; + const requestsBefore = mockServer.requestCount(); + + await app.workbench.quickaccess.runCommand('smoketest.openCopilotCliChat'); + await app.workbench.chat.waitForChatEditor(600); + await app.workbench.chat.sendEditorMessage(`hello world [scenario:${COPILOT_CLI_SCENARIO_ID}]`); + await app.workbench.chat.waitForEditorResponse(1500); + + const responseText = (await app.workbench.chat.getLatestEditorResponseText()).trim(); + logger.log(`Chat Sessions (Copilot CLI) response: ${responseText}`); + + assert.ok( + responseText.includes(COPILOT_CLI_REPLY), + `Expected Copilot CLI response to include mocked scenario response "${COPILOT_CLI_REPLY}".\n\nResponse:\n${responseText}` + ); + assert.ok( + mockServer.requestCount() > requestsBefore, + 'expected the mock LLM server to have received a new request from the Copilot CLI session' + ); + }); + + it('Test Claude session', async function () { + const app = this.app as Application; + const requestsBefore = mockServer.requestCount(); + logger.log(`Chat Sessions (Claude) mock requests before: ${requestsBefore}`); + + await app.workbench.quickaccess.runCommand('smoketest.openClaudeChat'); + await app.workbench.chat.waitForChatEditor(600); + await app.workbench.chat.sendEditorMessage(`hello world [scenario:${CLAUDE_SCENARIO_ID}]`); + logger.log(`Chat Sessions (Claude) mock requests after submit: ${mockServer.requestCount()}`); + await app.workbench.chat.waitForEditorResponse(1500); + + const responseText = (await app.workbench.chat.getLatestEditorResponseText()).trim(); + logger.log(`Chat Sessions (Claude) response: ${responseText}`); + logger.log(`Chat Sessions (Claude) mock requests after response: ${mockServer.requestCount()}`); + + assert.ok( + responseText.includes(CLAUDE_REPLY), + `Expected Claude response to include mocked scenario response "${CLAUDE_REPLY}".\n\nResponse:\n${responseText}` + ); + assert.ok( + mockServer.requestCount() > requestsBefore, + 'expected the mock LLM server to have received a new request from the Claude session' + ); + }); + + it('Test Local session', async function () { + const app = this.app as Application; + const requestsBefore = mockServer.requestCount(); + + // "Local" in the regular VS Code window is the default chat + // experience in the chat view (sidebar / aux bar). + await app.workbench.quickaccess.runCommand('workbench.action.chat.open'); + await app.workbench.chat.waitForChatView(); + await app.workbench.chat.sendMessage(`hello world [scenario:${LOCAL_SCENARIO_ID}]`); + await app.workbench.chat.waitForResponse(1500); + + const responseText = (await app.workbench.chat.getLatestResponseText()).trim(); + logger.log(`Chat Sessions (Local) response: ${responseText}`); + + assert.ok( + responseText.includes(LOCAL_REPLY), + `Expected Local response to include mocked scenario response "${LOCAL_REPLY}".\n\nResponse:\n${responseText}` + ); + assert.ok( + mockServer.requestCount() > requestsBefore, + 'expected the mock LLM server to have received a new request from the Local session' + ); + }); + }); +} diff --git a/test/smoke/src/main.ts b/test/smoke/src/main.ts index c528b71de8b..aaca9222443 100644 --- a/test/smoke/src/main.ts +++ b/test/smoke/src/main.ts @@ -29,6 +29,7 @@ import { setup as setupTerminalTests } from './areas/terminal/terminal.test'; import { setup as setupTaskTests } from './areas/task/task.test'; import { setup as setupChatTests } from './areas/chat/chatDisabled.test'; import { setup as setupCopilotCliTests } from './areas/chat/copilotCli.test'; +import { setup as setupChatSessionsTests } from './areas/chat/chatSessions.test'; import { setup as setupAccessibilityTests } from './areas/accessibility/accessibility.test'; import { setup as setupAgentsWindowTests } from './areas/agentsWindow/agentsWindow.test'; @@ -421,6 +422,7 @@ describe(`VSCode Smoke Tests (${opts.web ? 'Web' : 'Electron'})`, () => { if (!opts.web && !opts.remote) { setupLaunchTests(logger); } if (!opts.web) { setupChatTests(logger); } if (!opts.web && !opts.remote && quality !== Quality.Dev && quality !== Quality.OSS) { setupCopilotCliTests(logger); } + if (!opts.web && !opts.remote && quality !== Quality.Dev && quality !== Quality.OSS) { setupChatSessionsTests(logger); } if (!opts.web && !opts.remote && quality !== Quality.Dev && quality !== Quality.OSS) { setupAgentsWindowTests(logger); } setupAccessibilityTests(logger, opts, quality); });