mirror of
https://github.com/microsoft/vscode.git
synced 2026-09-10 20:44:25 +01:00
Add "hello" tests to VS Code smoke tests (#319719)
* Add "hello" tests to VS Code smoke tests * Address CCR feedback
This commit is contained in:
Alex Ross
@@ -995,6 +995,24 @@ async function handleMessagesApi(body, res) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Anthropic's Messages API also accepts a top-level `system` parameter
|
||||
// (string or array of `{ type: 'text', text }` blocks). Some session
|
||||
// types (e.g. Claude Code) embed the user prompt there alongside the
|
||||
// system instructions, so scan it as a fallback when no tag was found
|
||||
// in the messages array.
|
||||
if (!isScenarioRequest && parsed.system !== undefined) {
|
||||
const systemContent = typeof parsed.system === 'string'
|
||||
? parsed.system
|
||||
: Array.isArray(parsed.system)
|
||||
? parsed.system.map((/** @type {any} */ c) => c.text || '').join('')
|
||||
: '';
|
||||
const match = systemContent.match(/\[scenario:([^\]]+)\]/);
|
||||
if (match && SCENARIOS[match[1]]) {
|
||||
scenarioId = match[1];
|
||||
isScenarioRequest = true;
|
||||
}
|
||||
}
|
||||
} catch { }
|
||||
|
||||
const scenario = SCENARIOS[scenarioId] || SCENARIOS[DEFAULT_SCENARIO];
|
||||
|
||||
@@ -9,11 +9,13 @@ const CHAT_VIEW = 'div[id="workbench.panel.chat"]';
|
||||
const CHAT_EDITOR = '.editor-instance .interactive-session';
|
||||
const CHAT_INPUT_EDITOR = `${CHAT_VIEW} .interactive-input-part .monaco-editor[role="code"]`;
|
||||
const CHAT_INPUT_EDITOR_FOCUSED = `${CHAT_VIEW} .interactive-input-part .monaco-editor.focused[role="code"]`;
|
||||
const CHAT_SEND_BUTTON_ENABLED = `${CHAT_VIEW} .chat-input-toolbars > .chat-execute-toolbar .monaco-action-bar .action-item:not(.disabled) > .action-label.codicon-arrow-up`;
|
||||
const CHAT_RESPONSE = `${CHAT_VIEW} .interactive-item-container.interactive-response`;
|
||||
const CHAT_RESPONSE_COMPLETE = `${CHAT_RESPONSE}:not(.chat-response-loading)`;
|
||||
const CHAT_FOOTER_DETAILS = `${CHAT_VIEW} .chat-footer-details`;
|
||||
const CHAT_EDITOR_INPUT_EDITOR = `${CHAT_EDITOR} .interactive-input-part .monaco-editor[role="code"]`;
|
||||
const CHAT_EDITOR_INPUT_EDITOR_FOCUSED = `${CHAT_EDITOR} .interactive-input-part .monaco-editor.focused[role="code"]`;
|
||||
const CHAT_EDITOR_SEND_BUTTON_ENABLED = `${CHAT_EDITOR} .chat-input-toolbars > .chat-execute-toolbar .monaco-action-bar .action-item:not(.disabled) > .action-label.codicon-arrow-up`;
|
||||
const CHAT_EDITOR_RESPONSE = `${CHAT_EDITOR} .interactive-item-container.interactive-response`;
|
||||
const CHAT_EDITOR_RESPONSE_COMPLETE = `${CHAT_EDITOR_RESPONSE}:not(.chat-response-loading)`;
|
||||
|
||||
@@ -48,23 +50,38 @@ export class Chat {
|
||||
// Wait for the editor to be focused
|
||||
await this.waitForInputFocus();
|
||||
|
||||
// Type the message using pressSequentially - this works with Monaco editors
|
||||
// Note: Newlines are replaced with spaces since Enter key submits in chat input
|
||||
// Insert via Monaco's executeEdits rather than character-by-character
|
||||
// keypresses so suggestion widgets (e.g. the `[`-triggered chat reference
|
||||
// picker) cannot intercept characters and corrupt the prompt.
|
||||
// Newlines are replaced with spaces since Enter submits in chat input.
|
||||
const sanitizedMessage = message.replace(/\n/g, ' ');
|
||||
await this.code.driver.currentPage.locator(this.chatInputSelector).pressSequentially(sanitizedMessage);
|
||||
await this.code.waitForTypeInEditor(this.chatInputSelector, sanitizedMessage);
|
||||
|
||||
// Submit the message
|
||||
await this.code.dispatchKeybinding('enter', () => Promise.resolve());
|
||||
// Wait for the send button to be enabled before clicking. The send
|
||||
// button stays disabled until the chat participant is fully ready to
|
||||
// receive a request — relying on Enter alone is fragile for providers
|
||||
// that initialize asynchronously.
|
||||
await this.code.waitForElement(CHAT_SEND_BUTTON_ENABLED, undefined, 600);
|
||||
await this.code.waitAndClick(CHAT_SEND_BUTTON_ENABLED);
|
||||
}
|
||||
|
||||
async sendEditorMessage(message: string): Promise<void> {
|
||||
await this.code.waitAndClick(CHAT_EDITOR_INPUT_EDITOR);
|
||||
await this.code.waitForElement(CHAT_EDITOR_INPUT_EDITOR_FOCUSED);
|
||||
|
||||
// Insert via Monaco's executeEdits rather than character-by-character
|
||||
// keypresses so suggestion widgets (e.g. the `[`-triggered chat reference
|
||||
// picker) cannot intercept characters and corrupt the prompt.
|
||||
const sanitizedMessage = message.replace(/\n/g, ' ');
|
||||
await this.code.driver.currentPage.locator(this.chatEditorInputSelector).pressSequentially(sanitizedMessage);
|
||||
await this.code.waitForTypeInEditor(this.chatEditorInputSelector, sanitizedMessage);
|
||||
|
||||
await this.code.dispatchKeybinding('enter', () => Promise.resolve());
|
||||
// Wait for the send button to be enabled before clicking. The send
|
||||
// button stays disabled until the chat session participant is fully
|
||||
// ready to receive a request — relying on Enter alone is fragile for
|
||||
// session types whose providers initialize asynchronously (e.g. Claude
|
||||
// Agent).
|
||||
await this.code.waitForElement(CHAT_EDITOR_SEND_BUTTON_ENABLED, undefined, 600);
|
||||
await this.code.waitAndClick(CHAT_EDITOR_SEND_BUTTON_ENABLED);
|
||||
}
|
||||
|
||||
async waitForResponse(retryCount?: number): Promise<void> {
|
||||
@@ -86,6 +103,11 @@ export class Chat {
|
||||
return (await response.textContent()) ?? '';
|
||||
}
|
||||
|
||||
async getLatestResponseText(): Promise<string> {
|
||||
const response = this.code.driver.currentPage.locator(CHAT_RESPONSE_COMPLETE).last();
|
||||
return (await response.textContent()) ?? '';
|
||||
}
|
||||
|
||||
async waitForModelInFooter(): Promise<void> {
|
||||
await this.code.waitForElements(CHAT_FOOTER_DETAILS, false, el => {
|
||||
return el.some(el => {
|
||||
|
||||
@@ -102,6 +102,18 @@ function activate(context) {
|
||||
await vscode.commands.executeCommand(command);
|
||||
})
|
||||
);
|
||||
|
||||
context.subscriptions.push(
|
||||
vscode.commands.registerCommand('smoketest.openClaudeChat', async () => {
|
||||
const command = 'workbench.action.chat.openNewSessionEditor.claude-code';
|
||||
await vscode.workspace.getConfiguration('chat').update('disableAIFeatures', false, vscode.ConfigurationTarget.Global);
|
||||
await vscode.workspace.getConfiguration('github.copilot.chat').update('claudeAgent.enabled', true, vscode.ConfigurationTarget.Global);
|
||||
await vscode.commands.executeCommand('github.copilot.debug.extensionState');
|
||||
await waitForCommand(command, 30_000);
|
||||
await vscode.commands.executeCommand('workbench.action.closeAllEditors');
|
||||
await vscode.commands.executeCommand(command);
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
function deactivate() {
|
||||
|
||||
@@ -27,6 +27,10 @@
|
||||
{
|
||||
"command": "smoketest.openCopilotCliChat",
|
||||
"title": "Smoke Test: Open CLI Test Session"
|
||||
},
|
||||
{
|
||||
"command": "smoketest.openClaudeChat",
|
||||
"title": "Smoke Test: Open Claude Test Session"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
/*---------------------------------------------------------------------------------------------
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License. See License.txt in the project root for license information.
|
||||
*--------------------------------------------------------------------------------------------*/
|
||||
|
||||
import * as assert from 'assert';
|
||||
import { Application, Logger } from '../../../../automation';
|
||||
import { getCopilotSmokeTestEnv, getMockLlmServerPath, installAllHandlers, MockLlmServer } from '../../utils';
|
||||
|
||||
/**
|
||||
* Per-test scenarios. Each test uses a unique scenario id so that the mock
|
||||
* reply is distinct — this catches stale-content bugs where the previous
|
||||
* test's response is mistakenly accepted as the current test's response.
|
||||
*/
|
||||
const COPILOT_CLI_SCENARIO_ID = 'smoke-chat-sessions-copilot-cli';
|
||||
const COPILOT_CLI_REPLY = 'MOCKED_CHAT_SESSIONS_COPILOT_CLI_RESPONSE';
|
||||
|
||||
const CLAUDE_SCENARIO_ID = 'smoke-chat-sessions-claude';
|
||||
const CLAUDE_REPLY = 'MOCKED_CHAT_SESSIONS_CLAUDE_RESPONSE';
|
||||
|
||||
const LOCAL_SCENARIO_ID = 'smoke-chat-sessions-local';
|
||||
const LOCAL_REPLY = 'MOCKED_CHAT_SESSIONS_LOCAL_RESPONSE';
|
||||
|
||||
export function setup(logger: Logger) {
|
||||
|
||||
describe('Chat Sessions', function () {
|
||||
this.timeout(3 * 60 * 1000);
|
||||
this.retries(0);
|
||||
|
||||
let mockServer: MockLlmServer;
|
||||
|
||||
// Start the mock server BEFORE installAllHandlers' `before` runs so
|
||||
// the mock URL is available when we configure the app's env vars via
|
||||
// `optionsTransform`.
|
||||
before(async function () {
|
||||
const { startServer, ScenarioBuilder, registerScenario } = require(getMockLlmServerPath());
|
||||
|
||||
// Fallback for ancillary requests (title/branch) that don't carry a [scenario:...] tag.
|
||||
registerScenario('text-only', new ScenarioBuilder().emit('OK').build());
|
||||
|
||||
// One scenario per session type, each emitting a distinct reply
|
||||
// so the assertion is unambiguous.
|
||||
registerScenario(COPILOT_CLI_SCENARIO_ID, new ScenarioBuilder().emit(COPILOT_CLI_REPLY).build());
|
||||
registerScenario(CLAUDE_SCENARIO_ID, new ScenarioBuilder().emit(CLAUDE_REPLY).build());
|
||||
registerScenario(LOCAL_SCENARIO_ID, new ScenarioBuilder().emit(LOCAL_REPLY).build());
|
||||
|
||||
mockServer = await startServer(0, { logger: (msg: string) => logger.log(msg) });
|
||||
logger.log(`Chat Sessions mock LLM server started at ${mockServer.url}`);
|
||||
});
|
||||
|
||||
installAllHandlers(logger, opts => ({
|
||||
...opts,
|
||||
extraEnv: {
|
||||
...(opts.extraEnv ?? {}),
|
||||
...getCopilotSmokeTestEnv(mockServer),
|
||||
},
|
||||
}));
|
||||
|
||||
before(async function () {
|
||||
const app = this.app as Application;
|
||||
|
||||
// overrideProxyUrl/overrideCapiUrl redirect Copilot SDK + CAPI traffic
|
||||
// to the mock server. allowAnonymousAccess skips the token-validation
|
||||
// gate when there is no real GitHub session. The MCP/githubMcpServer
|
||||
// settings prevent real-network MCP connections during the test.
|
||||
await app.workbench.settingsEditor.addUserSettings([
|
||||
['github.copilot.advanced.debug.overrideProxyUrl', JSON.stringify(mockServer.url)],
|
||||
['github.copilot.advanced.debug.overrideCapiUrl', JSON.stringify(mockServer.url)],
|
||||
['chat.allowAnonymousAccess', 'true'],
|
||||
['github.copilot.chat.githubMcpServer.enabled', 'false'],
|
||||
['chat.mcp.discovery.enabled', 'false'],
|
||||
['chat.mcp.enabled', 'false'],
|
||||
// Force the bundled Claude Agent SDK (avoid the experiment that
|
||||
// would route through the ms-vscode.vscode-claude-sdk extension,
|
||||
// which would attempt a network install during the smoke run).
|
||||
['github.copilot.chat.claudeAgent.useSdkExtension', 'false'],
|
||||
]);
|
||||
});
|
||||
|
||||
after(async function () {
|
||||
await mockServer?.close();
|
||||
});
|
||||
|
||||
it('Test Copilot CLI session', async function () {
|
||||
const app = this.app as Application;
|
||||
const requestsBefore = mockServer.requestCount();
|
||||
|
||||
await app.workbench.quickaccess.runCommand('smoketest.openCopilotCliChat');
|
||||
await app.workbench.chat.waitForChatEditor(600);
|
||||
await app.workbench.chat.sendEditorMessage(`hello world [scenario:${COPILOT_CLI_SCENARIO_ID}]`);
|
||||
await app.workbench.chat.waitForEditorResponse(1500);
|
||||
|
||||
const responseText = (await app.workbench.chat.getLatestEditorResponseText()).trim();
|
||||
logger.log(`Chat Sessions (Copilot CLI) response: ${responseText}`);
|
||||
|
||||
assert.ok(
|
||||
responseText.includes(COPILOT_CLI_REPLY),
|
||||
`Expected Copilot CLI response to include mocked scenario response "${COPILOT_CLI_REPLY}".\n\nResponse:\n${responseText}`
|
||||
);
|
||||
assert.ok(
|
||||
mockServer.requestCount() > requestsBefore,
|
||||
'expected the mock LLM server to have received a new request from the Copilot CLI session'
|
||||
);
|
||||
});
|
||||
|
||||
it('Test Claude session', async function () {
|
||||
const app = this.app as Application;
|
||||
const requestsBefore = mockServer.requestCount();
|
||||
logger.log(`Chat Sessions (Claude) mock requests before: ${requestsBefore}`);
|
||||
|
||||
await app.workbench.quickaccess.runCommand('smoketest.openClaudeChat');
|
||||
await app.workbench.chat.waitForChatEditor(600);
|
||||
await app.workbench.chat.sendEditorMessage(`hello world [scenario:${CLAUDE_SCENARIO_ID}]`);
|
||||
logger.log(`Chat Sessions (Claude) mock requests after submit: ${mockServer.requestCount()}`);
|
||||
await app.workbench.chat.waitForEditorResponse(1500);
|
||||
|
||||
const responseText = (await app.workbench.chat.getLatestEditorResponseText()).trim();
|
||||
logger.log(`Chat Sessions (Claude) response: ${responseText}`);
|
||||
logger.log(`Chat Sessions (Claude) mock requests after response: ${mockServer.requestCount()}`);
|
||||
|
||||
assert.ok(
|
||||
responseText.includes(CLAUDE_REPLY),
|
||||
`Expected Claude response to include mocked scenario response "${CLAUDE_REPLY}".\n\nResponse:\n${responseText}`
|
||||
);
|
||||
assert.ok(
|
||||
mockServer.requestCount() > requestsBefore,
|
||||
'expected the mock LLM server to have received a new request from the Claude session'
|
||||
);
|
||||
});
|
||||
|
||||
it('Test Local session', async function () {
|
||||
const app = this.app as Application;
|
||||
const requestsBefore = mockServer.requestCount();
|
||||
|
||||
// "Local" in the regular VS Code window is the default chat
|
||||
// experience in the chat view (sidebar / aux bar).
|
||||
await app.workbench.quickaccess.runCommand('workbench.action.chat.open');
|
||||
await app.workbench.chat.waitForChatView();
|
||||
await app.workbench.chat.sendMessage(`hello world [scenario:${LOCAL_SCENARIO_ID}]`);
|
||||
await app.workbench.chat.waitForResponse(1500);
|
||||
|
||||
const responseText = (await app.workbench.chat.getLatestResponseText()).trim();
|
||||
logger.log(`Chat Sessions (Local) response: ${responseText}`);
|
||||
|
||||
assert.ok(
|
||||
responseText.includes(LOCAL_REPLY),
|
||||
`Expected Local response to include mocked scenario response "${LOCAL_REPLY}".\n\nResponse:\n${responseText}`
|
||||
);
|
||||
assert.ok(
|
||||
mockServer.requestCount() > requestsBefore,
|
||||
'expected the mock LLM server to have received a new request from the Local session'
|
||||
);
|
||||
});
|
||||
});
|
||||
}
|
||||
@@ -29,6 +29,7 @@ import { setup as setupTerminalTests } from './areas/terminal/terminal.test';
|
||||
import { setup as setupTaskTests } from './areas/task/task.test';
|
||||
import { setup as setupChatTests } from './areas/chat/chatDisabled.test';
|
||||
import { setup as setupCopilotCliTests } from './areas/chat/copilotCli.test';
|
||||
import { setup as setupChatSessionsTests } from './areas/chat/chatSessions.test';
|
||||
import { setup as setupAccessibilityTests } from './areas/accessibility/accessibility.test';
|
||||
import { setup as setupAgentsWindowTests } from './areas/agentsWindow/agentsWindow.test';
|
||||
|
||||
@@ -421,6 +422,7 @@ describe(`VSCode Smoke Tests (${opts.web ? 'Web' : 'Electron'})`, () => {
|
||||
if (!opts.web && !opts.remote) { setupLaunchTests(logger); }
|
||||
if (!opts.web) { setupChatTests(logger); }
|
||||
if (!opts.web && !opts.remote && quality !== Quality.Dev && quality !== Quality.OSS) { setupCopilotCliTests(logger); }
|
||||
if (!opts.web && !opts.remote && quality !== Quality.Dev && quality !== Quality.OSS) { setupChatSessionsTests(logger); }
|
||||
if (!opts.web && !opts.remote && quality !== Quality.Dev && quality !== Quality.OSS) { setupAgentsWindowTests(logger); }
|
||||
setupAccessibilityTests(logger, opts, quality);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user