From 92bb5ba22c8bd783ecc5aac83fca2c0d47472c54 Mon Sep 17 00:00:00 2001 From: Christof Marti Date: Tue, 8 Sep 2026 15:06:15 +0200 Subject: [PATCH] Agent Host: Smoke test Dev Container sessions (#334507) * Smoke test Dev Container Agent Host sessions Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Run Dev Container smoke tests across platforms Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Stabilize Dev Container smoke setup Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Support packaged Dev Container smoke runs Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Fix Dev Container smoke coverage Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Fix optional protected resource scopes Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Fix Windows environment assertion Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Probe Linux Docker for Dev Container smoke test Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Handle missing Docker in smoke probe Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../chat-simulation/common/mock-llm-server.ts | 4 +- .../node/devContainerAgentHostService.ts | 10 +- .../node/sshRemoteAgentHostHelpers.ts | 5 +- .../node/devContainerAgentHostService.test.ts | 15 ++ .../node/sshRemoteAgentHostHelpers.test.ts | 5 +- .../browser/remoteAgentHost.contribution.ts | 32 +++- .../agentSessions/agentHost/agentHostAuth.ts | 22 ++- .../agentHost/agentHostChatContribution.ts | 34 ++-- test/automation/src/agentsWindow.ts | 34 ++++ .../areas/agentsWindow/agentsWindow.test.ts | 150 +++++++++++++++++- test/smoke/src/utils.ts | 15 +- 11 files changed, 284 insertions(+), 42 deletions(-) diff --git a/scripts/chat-simulation/common/mock-llm-server.ts b/scripts/chat-simulation/common/mock-llm-server.ts index 9d942b7d39dd..6be4d596ddc7 100644 --- a/scripts/chat-simulation/common/mock-llm-server.ts +++ b/scripts/chat-simulation/common/mock-llm-server.ts @@ -1865,6 +1865,8 @@ interface CapturedRequest { interface StartServerOptions { logger?: (msg: string) => void; verbose?: boolean; + /** Address to listen on. Defaults to loopback. */ + host?: string; /** Reject requests that do not carry this exact header. */ requiredRequestHeader?: { name: string; @@ -1936,7 +1938,7 @@ function _startServer(port = 0, options?: StartServerOptions): Promise !fn()); handleRequest(req, res); }); - server.listen(port, '127.0.0.1', () => { + server.listen(port, options?.host ?? '127.0.0.1', () => { const addr = server.address(); const actualPort = typeof addr === 'object' && addr ? addr.port : port; const url = `http://127.0.0.1:${actualPort}`; diff --git a/src/vs/platform/agentHost/node/devContainerAgentHostService.ts b/src/vs/platform/agentHost/node/devContainerAgentHostService.ts index 3a832d303ae1..6d5b931213b1 100644 --- a/src/vs/platform/agentHost/node/devContainerAgentHostService.ts +++ b/src/vs/platform/agentHost/node/devContainerAgentHostService.ts @@ -387,10 +387,12 @@ export class DevContainerAgentHostMainService extends Disposable implements IDev } protected _resolveShellEnvironment(): Promise { - this._shellEnvironment ??= this._resolveUserShellEnvironment().catch(error => { - this._logService.error(`${LOG_PREFIX} Unable to resolve shell environment; using inherited environment`, error); - return process.env; - }); + this._shellEnvironment ??= this._resolveUserShellEnvironment() + .then(environment => ({ ...process.env, ...environment })) + .catch(error => { + this._logService.error(`${LOG_PREFIX} Unable to resolve shell environment; using inherited environment`, error); + return process.env; + }); return this._shellEnvironment; } diff --git a/src/vs/platform/agentHost/node/sshRemoteAgentHostHelpers.ts b/src/vs/platform/agentHost/node/sshRemoteAgentHostHelpers.ts index 4c51afdeac61..7bb586da2b80 100644 --- a/src/vs/platform/agentHost/node/sshRemoteAgentHostHelpers.ts +++ b/src/vs/platform/agentHost/node/sshRemoteAgentHostHelpers.ts @@ -239,14 +239,15 @@ export function buildCleanupOldCLIsCommand(serverDataFolderName: string, quality */ export function buildFindFallbackCLICommand(serverDataFolderName: string, quality: string): string { const root = getRemoteCLIInstallRoot(serverDataFolderName); + const relativeRoot = root.slice(2); const archive = getRemoteCLIArchiveName(quality); const commitGlob = '[0-9a-f]'.repeat(40); const q = validateShellToken(quality, 'quality'); const legacyDir = q === 'stable' ? '~/.vscode-cli' : `~/.vscode-cli-${q}`; const legacyBin = `${legacyDir}/${archive}`; return [ - `ls -1t -- ${root}/${archive}-${commitGlob} 2>/dev/null`, - `ls -1 -- ${legacyBin} 2>/dev/null`, + `(cd ~ && ls -1t -- ${relativeRoot}/${archive}-${commitGlob} 2>/dev/null | sed 's#^#~/#')`, + `test ! -f ${legacyBin} || printf '%s\\n' ${shellEscape(legacyBin)}`, 'true', ].join('; '); } diff --git a/src/vs/platform/agentHost/test/node/devContainerAgentHostService.test.ts b/src/vs/platform/agentHost/test/node/devContainerAgentHostService.test.ts index 9cfaa7bf68c7..4d0e76047d79 100644 --- a/src/vs/platform/agentHost/test/node/devContainerAgentHostService.test.ts +++ b/src/vs/platform/agentHost/test/node/devContainerAgentHostService.test.ts @@ -11,6 +11,7 @@ import { tmpdir } from 'os'; import { CancellationToken } from '../../../../base/common/cancellation.js'; import { DeferredPromise } from '../../../../base/common/async.js'; import { join } from '../../../../base/common/path.js'; +import { getCaseInsensitive } from '../../../../base/common/objects.js'; import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../base/test/common/utils.js'; import { mock } from '../../../../base/test/common/mock.js'; import { runWithFakedTimers } from '../../../../base/test/common/timeTravelScheduler.js'; @@ -308,6 +309,20 @@ suite('Dev Container Agent Host Main Service', () => { assert.strictEqual(await service.resolveShellEnvironment(), process.env); }); + test('merges the resolved shell environment with the inherited environment', async () => { + const service = store.add(new TestDevContainerAgentHostMainService('', false, undefined, { VSCODE_TEST_VALUE: 'resolved' })); + + const environment = await service.resolveShellEnvironment(); + + assert.deepStrictEqual({ + path: getCaseInsensitive(environment, 'PATH'), + testValue: environment.VSCODE_TEST_VALUE, + }, { + path: getCaseInsensitive(process.env, 'PATH'), + testValue: 'resolved', + }); + }); + test('reuses a standalone endpoint and exposes its relay', async () => { const service = store.add(new TestDevContainerAgentHostMainService()); const output: string[] = []; diff --git a/src/vs/platform/agentHost/test/node/sshRemoteAgentHostHelpers.test.ts b/src/vs/platform/agentHost/test/node/sshRemoteAgentHostHelpers.test.ts index bbc9e340b4e8..0ed36c0cc46f 100644 --- a/src/vs/platform/agentHost/test/node/sshRemoteAgentHostHelpers.test.ts +++ b/src/vs/platform/agentHost/test/node/sshRemoteAgentHostHelpers.test.ts @@ -385,10 +385,11 @@ suite('SSH Remote Agent Host Helpers', () => { test('lists commit-keyed candidates then legacy paths for insider', () => { const cmd = buildFindFallbackCLICommand('.vscode-server-insiders', 'insider'); // New commit-keyed candidates in shared install root, sorted newest-first. - assert.ok(cmd.includes('~/.vscode-server-insiders/code-insiders-'), `cmd missing new path: ${cmd}`); + assert.ok(cmd.includes('.vscode-server-insiders/code-insiders-'), `cmd missing new path: ${cmd}`); assert.ok(/ls -1t/.test(cmd), 'should sort commit-keyed candidates by mtime'); + assert.ok(cmd.includes(`sed 's#^#~/#'`), `cmd should normalize candidates to validated paths: ${cmd}`); // Legacy single-binary path (insider has the `-insider` dir suffix). - assert.ok(cmd.includes('~/.vscode-cli-insider/code-insiders'), `cmd missing legacy path: ${cmd}`); + assert.ok(cmd.includes(`printf '%s\\n' '~/.vscode-cli-insider/code-insiders'`), `cmd missing legacy path: ${cmd}`); }); test('uses no-suffix legacy dir for stable', () => { diff --git a/src/vs/sessions/contrib/providers/remoteAgentHost/browser/remoteAgentHost.contribution.ts b/src/vs/sessions/contrib/providers/remoteAgentHost/browser/remoteAgentHost.contribution.ts index c10dac504457..b769cece0511 100644 --- a/src/vs/sessions/contrib/providers/remoteAgentHost/browser/remoteAgentHost.contribution.ts +++ b/src/vs/sessions/contrib/providers/remoteAgentHost/browser/remoteAgentHost.contribution.ts @@ -19,6 +19,7 @@ import { AgentHostLocalFilePermissionsSettingId } from '../../../../../platform/ import { type ProtectedResourceMetadata } from '../../../../../platform/agentHost/common/state/protocol/state.js'; import { type AgentInfo, type RootState } from '../../../../../platform/agentHost/common/state/sessionState.js'; import { NotificationType, type INotification } from '../../../../../platform/agentHost/common/state/sessionActions.js'; +import { IConfigurationService } from '../../../../../platform/configuration/common/configuration.js'; import { ConfigurationScope, Extensions as ConfigurationExtensions, IConfigurationRegistry } from '../../../../../platform/configuration/common/configurationRegistry.js'; import { IDefaultAccountService } from '../../../../../platform/defaultAccount/common/defaultAccount.js'; import { IInstantiationService, ServicesAccessor } from '../../../../../platform/instantiation/common/instantiation.js'; @@ -27,7 +28,7 @@ import { Registry } from '../../../../../platform/registry/common/platform.js'; import { IWorkbenchContribution, registerWorkbenchContribution2, WorkbenchPhase } from '../../../../../workbench/common/contributions.js'; import { registerAction2 } from '../../../../../platform/actions/common/actions.js'; import { OpenAgentHostStateFileAction } from '../../agentHost/browser/openAgentHostStateFileAction.js'; -import { authenticateProtectedResources, AgentHostAuthenticationRecovery, AgentHostAuthTokenCache, resolveAuthenticationInteractively, revokeAuthenticationForRemovedSessions } from '../../../../../workbench/contrib/chat/browser/agentSessions/agentHost/agentHostAuth.js'; +import { authenticateAgentProtectedResourcesWithToken, authenticateProtectedResources, authenticateProtectedResourcesWithToken, AgentHostAuthenticationRecovery, AgentHostAuthTokenCache, resolveAuthenticationInteractively, revokeAuthenticationForRemovedSessions } from '../../../../../workbench/contrib/chat/browser/agentSessions/agentHost/agentHostAuth.js'; import { AgentHostLanguageModelProvider, agentHostProviderSupportsAutoModel } from '../../../../../workbench/contrib/chat/browser/agentSessions/agentHost/agentHostLanguageModelProvider.js'; import { AgentHostSessionHandler } from '../../../../../workbench/contrib/chat/browser/agentSessions/agentHost/agentHostSessionHandler.js'; import { IAgentHostActiveClientService } from '../../../../../workbench/contrib/chat/browser/agentSessions/agentHost/agentHostActiveClientService.js'; @@ -45,6 +46,7 @@ import { RemoteAgentHostSessionsProvider } from './remoteAgentHostSessionsProvid import { IRemoteAgentHostConnectionCustomizationService, RemoteAgentHostConnectionCustomizationService } from './remoteAgentHostConnectionCustomization.js'; import { InstantiationType, registerSingleton } from '../../../../../platform/instantiation/common/extensions.js'; import { IAgentHostTerminalService } from '../../../../../workbench/contrib/terminal/browser/agentHostTerminalService.js'; +import { IWorkbenchEnvironmentService } from '../../../../../workbench/services/environment/common/environmentService.js'; import { ITelemetryService } from '../../../../../platform/telemetry/common/telemetry.js'; import { logTerminalRecovery } from '../../../../common/sessionsTelemetry.js'; @@ -140,6 +142,7 @@ export class RemoteAgentHostContribution extends Disposable implements IWorkbenc /** Per-connection state: client state + per-agent registrations. */ private readonly _connections = this._register(new DisposableMap()); + private readonly _enableSmokeTestDriver: boolean; constructor( @IRemoteAgentHostService private readonly _remoteAgentHostService: IRemoteAgentHostService, @@ -156,8 +159,11 @@ export class RemoteAgentHostContribution extends Disposable implements IWorkbenc @ITelemetryService private readonly _telemetryService: ITelemetryService, @IAgentHostActiveClientService private readonly _activeClientService: IAgentHostActiveClientService, @IRemoteAgentHostConnectionCustomizationService private readonly _connectionCustomizations: IRemoteAgentHostConnectionCustomizationService, + @IConfigurationService private readonly _configurationService: IConfigurationService, + @IWorkbenchEnvironmentService environmentService: IWorkbenchEnvironmentService, ) { super(); + this._enableSmokeTestDriver = !!environmentService.enableSmokeTestDriver; this._register(this._remoteAgentHostService.onDidChangeConnections(() => this._reconcile())); this._register(this._defaultAccountService.onDidChangeDefaultAccount(() => this._authenticateAllConnections())); @@ -490,6 +496,14 @@ export class RemoteAgentHostContribution extends Disposable implements IWorkbenc const authTokenCache = this._connections.get(address)?.authTokenCache; provider?.setAuthenticationPending(true); try { + const testToken = this._getScenarioAutomationToken(); + if (testToken !== undefined) { + await authenticateAgentProtectedResourcesWithToken(agents, testToken, { + authTokenCache, + authenticate: this._authenticateCallback(address, connection), + }); + return; + } await this._instantiationService.invokeFunction(authenticateProtectedResources, agents, { authTokenCache, logPrefix: '[RemoteAgentHost]', @@ -502,6 +516,14 @@ export class RemoteAgentHostContribution extends Disposable implements IWorkbenc } } + private _getScenarioAutomationToken(): string | undefined { + if (!this._enableSmokeTestDriver) { + return undefined; + } + const token = this._configurationService.getValue('chat.agentHost.unsafeTestToken'); + return typeof token === 'string' && token.length > 0 ? token : undefined; + } + private _handleAuthenticationRequiredNotification(address: string, connection: IAgentConnection, notification: INotification): void { if (notification.type !== NotificationType.AuthRequired) { return; @@ -560,6 +582,14 @@ export class RemoteAgentHostContribution extends Disposable implements IWorkbenc */ private async _resolveAuthenticationInteractively(address: string, connection: IAgentConnection, protectedResources: readonly ProtectedResourceMetadata[]): Promise { const authTokenCache = this._connections.get(address)?.authTokenCache; + const testToken = this._getScenarioAutomationToken(); + if (testToken !== undefined) { + await authenticateProtectedResourcesWithToken(protectedResources, testToken, { + authTokenCache, + authenticate: this._authenticateCallback(address, connection), + }); + return protectedResources.length > 0; + } return this._instantiationService.invokeFunction(resolveAuthenticationInteractively, protectedResources, { authTokenCache, logPrefix: '[RemoteAgentHost]', diff --git a/src/vs/workbench/contrib/chat/browser/agentSessions/agentHost/agentHostAuth.ts b/src/vs/workbench/contrib/chat/browser/agentSessions/agentHost/agentHostAuth.ts index 73d6807a4375..81db903d83b9 100644 --- a/src/vs/workbench/contrib/chat/browser/agentSessions/agentHost/agentHostAuth.ts +++ b/src/vs/workbench/contrib/chat/browser/agentSessions/agentHost/agentHostAuth.ts @@ -405,7 +405,7 @@ export interface IAgentHostMcpAuthenticationOptionsBase { async function forwardAuthenticationToken( options: Pick, resource: string, - scopes: readonly string[], + scopes: readonly string[] | undefined, token: string, ): Promise { throwIfAuthenticationStale(options); @@ -441,6 +441,26 @@ export async function authenticateProtectedResources( } } +export async function authenticateAgentProtectedResourcesWithToken( + agents: readonly AgentInfo[], + token: string, + options: Pick, +): Promise { + for (const agent of agents) { + await authenticateProtectedResourcesWithToken(agent.protectedResources ?? [], token, options); + } +} + +export async function authenticateProtectedResourcesWithToken( + protectedResources: readonly ProtectedResourceMetadata[], + token: string, + options: Pick, +): Promise { + for (const resource of protectedResources) { + await forwardAuthenticationToken(options, resource.resource, resource.scopes_supported, token); + } +} + /** * Reconciles resources backed by an authentication session that was explicitly * removed. diff --git a/src/vs/workbench/contrib/chat/browser/agentSessions/agentHost/agentHostChatContribution.ts b/src/vs/workbench/contrib/chat/browser/agentSessions/agentHost/agentHostChatContribution.ts index a2a28a1520b8..76af318e0f85 100644 --- a/src/vs/workbench/contrib/chat/browser/agentSessions/agentHost/agentHostChatContribution.ts +++ b/src/vs/workbench/contrib/chat/browser/agentSessions/agentHost/agentHostChatContribution.ts @@ -36,7 +36,7 @@ import { Target } from '../../../common/promptSyntax/promptTypes.js'; import { AgentCustomizationItemProvider } from './agentCustomizationItemProvider.js'; import { agentHostProviderHasBuiltInGitHubMcpServer, COPILOT_CHAT_GITHUB_MCP_COLLECTION_ID } from './agentHostMcpServerSupport.js'; import { AgentHostDownloadProgress } from './agentHostDownloadProgress.js'; -import { authenticateProtectedResources, AgentHostAuthenticationRecovery, AgentHostAuthTokenCache, resolveAuthenticationInteractively, revokeAuthenticationForRemovedSessions } from './agentHostAuth.js'; +import { authenticateAgentProtectedResourcesWithToken, authenticateProtectedResources, authenticateProtectedResourcesWithToken, AgentHostAuthenticationRecovery, AgentHostAuthTokenCache, resolveAuthenticationInteractively, revokeAuthenticationForRemovedSessions } from './agentHostAuth.js'; import { AgentHostLanguageModelProvider, agentHostProviderSupportsAutoModel } from './agentHostLanguageModelProvider.js'; import { AgentHostSessionHandler } from './agentHostSessionHandler.js'; import { AgentHostPromptCacheNotification } from './agentHostPromptCacheNotification.js'; @@ -415,7 +415,11 @@ export class AgentHostContribution extends Disposable implements IWorkbenchContr try { const testToken = this._getScenarioAutomationToken(); if (testToken !== undefined) { - await this._seedTestToken(agents, testToken, generation); + await authenticateAgentProtectedResourcesWithToken(agents, testToken, { + authTokenCache: this._authTokenCache, + isCurrent: () => this._isAuthenticationCurrent(generation), + authenticate: request => this._authenticateIfCurrent(request, generation), + }); return; } await this._instantiationService.invokeFunction(authenticateProtectedResources, agents, { @@ -475,14 +479,11 @@ export class AgentHostContribution extends Disposable implements IWorkbenchContr } const testToken = this._getScenarioAutomationToken(); if (testToken !== undefined) { - for (const resource of protectedResources) { - await this._authTokenCache.authenticate( - resource.resource, - resource.scopes_supported, - testToken, - () => this._authenticateIfCurrent({ resource: resource.resource, token: testToken }, generation), - ); - } + await authenticateProtectedResourcesWithToken(protectedResources, testToken, { + authTokenCache: this._authTokenCache, + isCurrent: () => this._isAuthenticationCurrent(generation), + authenticate: request => this._authenticateIfCurrent(request, generation), + }); return protectedResources.length > 0; } return this._instantiationService.invokeFunction(resolveAuthenticationInteractively, protectedResources, { @@ -493,19 +494,6 @@ export class AgentHostContribution extends Disposable implements IWorkbenchContr }); } - private async _seedTestToken(agents: readonly AgentInfo[], token: string, generation: number): Promise { - for (const agent of agents) { - for (const resource of agent.protectedResources ?? []) { - await this._authTokenCache.authenticate( - resource.resource, - resource.scopes_supported, - token, - () => this._authenticateIfCurrent({ resource: resource.resource, token }, generation), - ); - } - } - } - private _getScenarioAutomationToken(): string | undefined { // Smoke-test escape hatch. if (!this._enableSmokeTestDriver) { diff --git a/test/automation/src/agentsWindow.ts b/test/automation/src/agentsWindow.ts index 92526c45be4b..9944c35a0675 100644 --- a/test/automation/src/agentsWindow.ts +++ b/test/automation/src/agentsWindow.ts @@ -14,6 +14,8 @@ const SESSION_TYPE_PICKER = '.sessions-chat-session-type-picker .action-label'; const SESSION_TYPE_PICKER_VISIBLE = `${SESSION_TYPE_PICKER}:not(.hidden)`; const NEW_CHAT_EDITOR = `${NEW_SESSION_VIEW} .sessions-chat-editor .monaco-editor[role="code"]`; const SEND_BUTTON_ENABLED = `${NEW_SESSION_VIEW} .sessions-chat-send-button .monaco-button:not(.disabled)`; +const DEV_CONTAINER_CHECKBOX = `${NEW_SESSION_VIEW} .sessions-chat-dev-container-checkbox .monaco-checkbox`; +const NEW_WORKTREE_CHECKBOX = `${NEW_SESSION_VIEW} .sessions-chat-isolation-checkbox .monaco-checkbox`; const ACTIVE_SESSION = `${AGENTS_WORKBENCH} .session-view.is-active`; const ACTIVE_SESSION_INPUT_EDITOR = `${ACTIVE_SESSION} .interactive-session .interactive-input-part .monaco-editor[role="code"]`; const ACTIVE_SESSION_SEND_BUTTON_ENABLED = `${ACTIVE_SESSION} .interactive-session .chat-input-toolbars > .chat-execute-toolbar .monaco-action-bar .action-item:not(.disabled) > .action-label.codicon-arrow-up-compact`; @@ -115,6 +117,38 @@ export class AgentsWindow { await this.code.waitForElement(ACTIVE_SESSION_INPUT_EDITOR, undefined, retryCount); } + async selectDevContainer(): Promise { + const page = this.code.driver.currentPage; + const devContainer = page.locator(DEV_CONTAINER_CHECKBOX).first(); + const newWorktree = page.locator(NEW_WORKTREE_CHECKBOX).first(); + const deadline = Date.now() + 120_000; + while (Date.now() < deadline) { + if (await newWorktree.count() > 0 && await newWorktree.getAttribute('aria-checked') === 'true') { + await newWorktree.click(); + continue; + } + if (await devContainer.count() > 0 && await devContainer.getAttribute('aria-disabled') !== 'true') { + break; + } + await new Promise(resolve => setTimeout(resolve, 100)); + } + if (await devContainer.count() === 0) { + throw new Error('Timed out waiting for Dev Container checkbox to appear'); + } + if (await devContainer.getAttribute('aria-disabled') === 'true') { + throw new Error('Timed out waiting for Dev Container checkbox to become enabled'); + } + if (await devContainer.getAttribute('aria-checked') !== 'true') { + await devContainer.click(); + } + while (await devContainer.getAttribute('aria-checked') !== 'true') { + if (Date.now() >= deadline) { + throw new Error('Timed out waiting for Dev Container checkbox to become checked'); + } + await new Promise(resolve => setTimeout(resolve, 100)); + } + } + private async isSessionTypeSelected(label: string): Promise { const picker = this.code.driver.currentPage.locator(SESSION_TYPE_PICKER_VISIBLE).first(); return ((await picker.textContent()) ?? '').trim().toLowerCase() === label.trim().toLowerCase(); diff --git a/test/smoke/src/areas/agentsWindow/agentsWindow.test.ts b/test/smoke/src/areas/agentsWindow/agentsWindow.test.ts index 5e8b7476de91..c11f3927c00a 100644 --- a/test/smoke/src/areas/agentsWindow/agentsWindow.test.ts +++ b/test/smoke/src/areas/agentsWindow/agentsWindow.test.ts @@ -42,6 +42,7 @@ const AGENT_HOST_REPLY = 'MOCKED_AGENT_HOST_RESPONSE'; const AGENT_HOST_MODEL = 'gpt-5.3-codex'; const AGENT_HOST_REPLACEMENT_SCENARIO_ID = 'smoke-agent-host-session-replacement'; const AGENT_HOST_REPLACEMENT_REPLY = 'MOCKED_AGENT_HOST_REPLACEMENT_RESPONSE'; +const DEV_CONTAINER_SCENARIO_ID = 'smoke-dev-container-agent-host'; const AGENT_HOST_SANDBOX_SCENARIO_ID = 'smoke-hello-agent-host-sandbox'; const AGENT_HOST_SANDBOX_REPLY = 'MOCKED_AGENT_HOST_SANDBOX_RESPONSE'; @@ -55,6 +56,27 @@ const AGENT_HOST_SDK_SANDBOX_REPLY = 'MOCKED_AGENT_HOST_SDK_SANDBOX_RESPONSE'; const AGENT_HOST_WARMUP_SCENARIO_ID = 'smoke-hello-agent-host-warmup'; const AGENT_HOST_WARMUP_REPLY = 'MOCKED_AGENT_HOST_WARMUP_RESPONSE'; +function probeLinuxDocker(): { readonly available: boolean; readonly reason?: string } { + const result = cp.spawnSync('docker', ['info', '--format', '{{.OSType}}'], { + encoding: 'utf8', + timeout: 15_000, + windowsHide: true, + }); + const operatingSystem = result.stdout?.trim().toLowerCase() ?? ''; + if (result.status === 0 && operatingSystem === 'linux') { + return { available: true }; + } + const stderr = result.stderr?.trim() ?? ''; + return { + available: false, + reason: result.error?.message + ?? (stderr || undefined) + ?? (operatingSystem + ? `Docker daemon reports '${operatingSystem}' containers` + : `docker info exited with code ${result.status ?? 'unknown'}`), + }; +} + export function setup(logger: Logger) { describe('Agents Window (local AgentHost)', () => { @@ -220,6 +242,108 @@ export function setup(logger: Logger) { }); }); + const linuxDocker = probeLinuxDocker(); + const runDevContainerSuite = linuxDocker.available || process.platform === 'linux'; + if (!linuxDocker.available) { + logger.log(process.platform === 'linux' + ? `Linux Docker probe failed: ${linuxDocker.reason}` + : `Skipping Agents Window (Dev Container AgentHost): ${linuxDocker.reason}`); + } + (runDevContainerSuite ? describe : describe.skip)('Agents Window (Dev Container AgentHost)', () => { + if (process.platform === 'linux') { + before(() => assert.ok(linuxDocker.available, `Expected a reachable Linux Docker daemon: ${linuxDocker.reason}`)); + } + + const devContainer = setupAgentHostSuite(logger, { + serverLabel: 'Dev Container AgentHost', + mockServerHost: '0.0.0.0', + registerScenarios: ({ ScenarioBuilder, registerScenario }) => { + registerScenario(DEV_CONTAINER_SCENARIO_ID, new ScenarioBuilder().emit('OK').build()); + }, + settings: { + 'chat.agentHost.devContainer.enabled': true, + 'chat.remoteAgentHostsEnabled': true, + }, + prepareWorkspace: workspacePath => { + const configDirectory = path.join(workspacePath, '.devcontainer'); + const mockServerUrl = `http://vscode-smoke.test:${devContainer.mockServer.port}`; + fs.mkdirSync(configDirectory, { recursive: true }); + fs.writeFileSync(path.join(configDirectory, 'devcontainer.json'), JSON.stringify({ + name: 'Agents Window Smoke', + image: 'mcr.microsoft.com/devcontainers/base:ubuntu-24.04', + remoteUser: 'vscode', + runArgs: ['--add-host=vscode-smoke.test:host-gateway'], + containerEnv: { + COPILOT_API_URL: mockServerUrl, + COPILOT_DEBUG_GITHUB_API_URL: mockServerUrl, + GITHUB_COPILOT_API_TOKEN: 'smoketest-fake-agent-host-token', + VSCODE_AGENT_HOST_CAPI_URL_OVERRIDE: mockServerUrl, + VSCODE_SMOKE_TEST_PROXY_HEADER: 'dev-container', + }, + postCreateCommand: [ + 'set -e', + 'case "$(uname -m)" in x86_64) cli_arch=x64 ;; aarch64|arm64) cli_arch=arm64 ;; *) exit 1 ;; esac', + 'mkdir -p ~/.vscode-cli-insider', + 'curl -fsSL "https://update.code.visualstudio.com/latest/cli-linux-${cli_arch}/insider" | tar xz -C ~/.vscode-cli-insider', + 'chmod +x ~/.vscode-cli-insider/code-insiders', + ].join(' && '), + }, null, 2)); + }, + cleanupWorkspace: workspacePath => { + fs.rmSync(path.join(workspacePath, '.devcontainer'), { recursive: true, force: true }); + const containerIds = cp.execFileSync('docker', [ + 'ps', + '-aq', + '--filter', + `label=devcontainer.local_folder=${workspacePath}`, + ], { encoding: 'utf8' }).trim().split(/\s+/).filter(Boolean); + if (containerIds.length > 0) { + cp.execFileSync('docker', ['rm', '--force', ...containerIds], { stdio: 'pipe' }); + } + }, + }); + + it('Starts a session in a Dev Container', async function () { + this.timeout(10 * 60 * 1000); + cp.execFileSync('docker', ['info'], { stdio: 'pipe' }); + const app = this.app as Application; + + try { + const requestsBefore = devContainer.mockServer.requestCount(); + await app.workbench.agentsWindow.waitForNewSessionView(); + await app.workbench.agentsWindow.selectSessionType('Copilot'); + await app.workbench.agentsWindow.selectDevContainer(); + await app.workbench.agentsWindow.submitNewSessionPrompt(`start Dev Container [scenario:${DEV_CONTAINER_SCENARIO_ID}]`, 1_800); + await app.workbench.agentsWindow.waitForActiveSessionView(5 * 60 * 1000); + const text = await app.workbench.agentsWindow.waitForAssistantText('OK', 2 * 60 * 1000); + logger.log(`Agents Window (Dev Container AgentHost) response: ${text}`); + assert.ok( + devContainer.mockServer.requestCount() > requestsBefore, + 'expected the mock LLM server to receive the Dev Container Agent Host request' + ); + + const ahpFrames = await waitForLogContent( + () => readAhpFrames(path.join(devContainer.logsPath, 'ahp')), + /"transport":"devcontainer"/, + 30_000, + ); + assert.match(ahpFrames, /"vscode\.clientConnectionKind":"dev_container"/); + assert.match(ahpFrames, /"transport":"devcontainer"/); + + const rendererLogs = await waitForLogContent( + () => readRendererLogs(devContainer.logsPath), + /\[AgentHost\] _invokeAgent called for resource: remote-devcontainer__/, + 30_000, + ); + assert.match(rendererLogs, /\[AgentHost\] _invokeAgent called for resource: remote-devcontainer__/); + } catch (error) { + logger.log(`Agents Window (Dev Container AgentHost) FAILURE: ${error instanceof Error ? error.stack ?? error.message : String(error)}`); + await dumpFailureDiagnostics(app, logger, 'Agents Window (Dev Container AgentHost)', { sendButtonSelector: AGENTS_SEND_BUTTON_SELECTOR }); + throw error; + } + }); + }); + describe('Agents Window (local AgentHost, SDK sandbox)', () => { // Variant of the AgentHost suite that leaves @@ -513,11 +637,15 @@ interface IAgentHostSuiteContext { */ function setupAgentHostSuite(logger: Logger, config: { readonly serverLabel: string; + readonly mockServerHost?: string; readonly registerScenarios: (api: { ScenarioBuilder: any; registerScenario: (id: string, scenario: unknown) => void }) => void; readonly settings: Record; + readonly prepareWorkspace?: (workspacePath: string) => Promise | void; + readonly cleanupWorkspace?: (workspacePath: string) => Promise | void; }): IAgentHostSuiteContext { let mockServer: MockLlmServer; let logsPath: string; + let workspacePath: string | undefined; before(async function () { const { startServer, ScenarioBuilder, registerScenario } = require(getMockLlmServerPath()); @@ -526,7 +654,10 @@ function setupAgentHostSuite(logger: Logger, config: { registerScenario(AGENT_HOST_WARMUP_SCENARIO_ID, new ScenarioBuilder().emit(AGENT_HOST_WARMUP_REPLY).build()); config.registerScenarios({ ScenarioBuilder, registerScenario }); - mockServer = await startServer(0, mockServerStartOptions((msg: string) => logger.log(msg))); + mockServer = await startServer(0, { + ...mockServerStartOptions((msg: string) => logger.log(msg)), + host: config.mockServerHost, + }); logger.log(`Mock LLM server (${config.serverLabel}) started at ${getMockLlmServerUrl(mockServer)}`); }); @@ -554,6 +685,8 @@ function setupAgentHostSuite(logger: Logger, config: { VSCODE_AGENT_HOST_CAPI_URL_OVERRIDE: getMockLlmServerUrl(mockServer), }, })); + workspacePath = (this.app as Application).workspacePathOrFolder; + await config.prepareWorkspace?.(workspacePath); // Pre-seed settings.json on disk into BOTH the default profile and the // Agents profile so Agent Host startup observes the test configuration. @@ -588,7 +721,11 @@ function setupAgentHostSuite(logger: Logger, config: { await (this.app as Application).start(); }); - installAppAfterHandler(); + installAppAfterHandler(undefined, async () => { + if (workspacePath) { + await config.cleanupWorkspace?.(workspacePath); + } + }); before(async function () { const app = this.app as Application; @@ -641,3 +778,12 @@ function ahpJsonlFiles(ahpLogDir: string): string[] { function readAhpFrames(ahpLogDir: string): string { return ahpJsonlFiles(ahpLogDir).map(f => fs.readFileSync(path.join(ahpLogDir, f), 'utf8')).join('\n'); } + +function readRendererLogs(logsPath: string): string { + return fs.existsSync(logsPath) + ? fs.readdirSync(logsPath, { withFileTypes: true }) + .filter(entry => entry.isDirectory() && /^window\d+$/.test(entry.name)) + .map(entry => readFileIfExists(path.join(logsPath, entry.name, 'renderer.log'))) + .join('\n') + : ''; +} diff --git a/test/smoke/src/utils.ts b/test/smoke/src/utils.ts index 0693f14fcfad..0789de8468f7 100644 --- a/test/smoke/src/utils.ts +++ b/test/smoke/src/utils.ts @@ -10,6 +10,7 @@ import { dirname, join } from 'path'; import { Application, ApplicationOptions, IModelConfigSection, Logger } from '../../automation'; export interface MockLlmServer { + readonly port: number; readonly url: string; requestCount(): number; getRequests(): readonly { readonly path: string; readonly method: string; readonly body: unknown }[]; @@ -142,12 +143,14 @@ function installAppBeforeHandler(optionsTransform?: (opts: ApplicationOptions) = export function installAppAfterHandler(appFn?: () => Application | undefined, joinFn?: () => Promise) { after(async function () { const app: Application = appFn?.() ?? this.app; - if (app) { - await app.stop(); - } - - if (joinFn) { - await joinFn(); + try { + if (app) { + await app.stop(); + } + } finally { + if (joinFn) { + await joinFn(); + } } }); }