From a095788f98e686b19fd2c0ef3499b22759495639 Mon Sep 17 00:00:00 2001 From: vritant24 Date: Mon, 3 Aug 2026 11:20:21 -0700 Subject: [PATCH 1/2] Agent Host changes for agents/gemini-provider-issue-investigation --- .../agentHost/common/agentHostByokLm.ts | 10 +- .../node/copilot/byokResponsesTranslation.ts | 23 +++- .../test/node/byokLmProxyService.test.ts | 112 ++++++++++++++++++ .../agentHost/agentHostByokLmHandler.ts | 23 +++- .../agentHostByokLmHandler.test.ts | 17 ++- 5 files changed, 178 insertions(+), 7 deletions(-) diff --git a/src/vs/platform/agentHost/common/agentHostByokLm.ts b/src/vs/platform/agentHost/common/agentHostByokLm.ts index 0323e351059..ab4ba49e08b 100644 --- a/src/vs/platform/agentHost/common/agentHostByokLm.ts +++ b/src/vs/platform/agentHost/common/agentHostByokLm.ts @@ -25,10 +25,18 @@ export interface IByokLmTextPart { readonly text: string; } +export interface IByokLmImagePart { + readonly type: 'image'; + readonly mimeType: string; + readonly data: string; +} + +export type IByokLmContentPart = IByokLmTextPart | IByokLmImagePart; + export interface IByokLmMessageItem { readonly type: 'message'; readonly role: 'system' | 'developer' | 'user' | 'assistant'; - readonly content: IByokLmTextPart[]; + readonly content: IByokLmContentPart[]; } export interface IByokLmReasoningItem { diff --git a/src/vs/platform/agentHost/node/copilot/byokResponsesTranslation.ts b/src/vs/platform/agentHost/node/copilot/byokResponsesTranslation.ts index acd2767a83a..d04f97b9727 100644 --- a/src/vs/platform/agentHost/node/copilot/byokResponsesTranslation.ts +++ b/src/vs/platform/agentHost/node/copilot/byokResponsesTranslation.ts @@ -3,9 +3,11 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ +import { decodeBase64 } from '../../../../base/common/buffer.js'; import { IByokLmChatRequest, IByokLmChatResult, + IByokLmContentPart, IByokLmInputItem, IByokLmOutputItem, IByokLmTool, @@ -14,6 +16,7 @@ import { interface IResponsesContentPart { readonly type?: string; readonly text?: string; + readonly image_url?: string; } interface IResponsesSummaryPart { @@ -71,7 +74,7 @@ function toBridgeRole(role: string | undefined): 'system' | 'developer' | 'user' } } -function toTextParts(content: string | IResponsesContentPart[] | undefined, itemIndex: number): Array<{ type: 'text'; text: string }> { +function toContentParts(content: string | IResponsesContentPart[] | undefined, itemIndex: number): IByokLmContentPart[] { if (typeof content === 'string') { return content ? [{ type: 'text', text: content }] : []; } @@ -82,6 +85,22 @@ function toTextParts(content: string | IResponsesContentPart[] | undefined, item if ((part.type === 'input_text' || part.type === 'output_text' || part.type === 'text') && typeof part.text === 'string') { return { type: 'text' as const, text: part.text }; } + if (part.type === 'input_image' && typeof part.image_url === 'string') { + const match = /^data:(?image\/[^;,]+)(?:;[^,]*)?;base64,(?.*)$/.exec(part.image_url); + if (match?.groups) { + try { + decodeBase64(match.groups.data); + } catch { + throw new ResponsesTranslationError(`Invalid input[${itemIndex}].content[${contentIndex}].image_url`); + } + return { + type: 'image' as const, + mimeType: match.groups.mimeType, + data: match.groups.data, + }; + } + throw new ResponsesTranslationError(`Unsupported input[${itemIndex}].content[${contentIndex}].image_url`); + } throw new ResponsesTranslationError(`Unsupported input[${itemIndex}].content[${contentIndex}] type '${part.type ?? ''}'`); }); } @@ -99,7 +118,7 @@ function toBridgeInputItem(item: IResponsesInputItem, index: number): IByokLmInp return { type: 'message', role: toBridgeRole(item.role), - content: toTextParts(item.content, index), + content: toContentParts(item.content, index), }; case 'reasoning': return { diff --git a/src/vs/platform/agentHost/test/node/byokLmProxyService.test.ts b/src/vs/platform/agentHost/test/node/byokLmProxyService.test.ts index 7f036f71355..b47dd63b3ec 100644 --- a/src/vs/platform/agentHost/test/node/byokLmProxyService.test.ts +++ b/src/vs/platform/agentHost/test/node/byokLmProxyService.test.ts @@ -138,6 +138,118 @@ suite('ByokLmProxyService', () => { assert.deepStrictEqual(captured?.input, [{ type: 'message', role: 'user', content: [{ type: 'text', text: 'hi' }] }]); }); + test('forwards image input on the initial and subsequent turns', async () => { + const captured: IByokLmChatRequest[] = []; + const statuses: number[] = []; + const imageMessage = { + type: 'message', + role: 'user', + content: [ + { type: 'input_text', text: 'What is in this image?' }, + { type: 'input_image', image_url: 'data:image/png;base64,iVBORw0KGgo=' }, + ], + }; + + await withProxy( + async request => { + captured.push(request); + return { output: [] }; + }, + async handle => { + for (const input of [ + [imageMessage], + [imageMessage, { type: 'message', role: 'user', content: [{ type: 'input_text', text: 'Try again without a new image.' }] }], + ]) { + const response = await fetch(responsesUrl(handle, 'gemini'), { + method: 'POST', + headers: authHeaders(handle), + body: JSON.stringify({ model: 'gemini-3.6-flash', input }), + }); + statuses.push(response.status); + await response.text(); + } + }, + ); + + assert.deepStrictEqual({ statuses, input: captured.map(request => request.input) }, { + statuses: [200, 200], + input: [ + [ + { + type: 'message', + role: 'user', + content: [ + { type: 'text', text: 'What is in this image?' }, + { type: 'image', mimeType: 'image/png', data: 'iVBORw0KGgo=' }, + ], + }, + ], + [ + { + type: 'message', + role: 'user', + content: [ + { type: 'text', text: 'What is in this image?' }, + { type: 'image', mimeType: 'image/png', data: 'iVBORw0KGgo=' }, + ], + }, + { + type: 'message', + role: 'user', + content: [ + { type: 'text', text: 'Try again without a new image.' }, + ], + }, + ], + ], + }); + }); + + test('rejects image URLs that cannot be forwarded as inline data', async () => { + await withProxy( + async () => ({ output: [] }), + async handle => { + const responses: Array<{ status: number; body: unknown }> = []; + for (const imageUrl of ['https://example.com/image.png', 'data:image/png;base64,not valid']) { + const response = await fetch(responsesUrl(handle, 'gemini'), { + method: 'POST', + headers: authHeaders(handle), + body: JSON.stringify({ + model: 'gemini-3.6-flash', + input: [{ + type: 'message', + role: 'user', + content: [{ type: 'input_image', image_url: imageUrl }], + }], + }), + }); + responses.push({ status: response.status, body: await response.json() }); + } + + assert.deepStrictEqual(responses, [ + { + status: 400, + body: { + error: { + message: 'Unsupported input[0].content[0].image_url', + type: 'invalid_request_error', + }, + }, + }, + { + status: 400, + body: { + error: { + message: 'Invalid input[0].content[0].image_url', + type: 'invalid_request_error', + }, + }, + }, + ]); + }, + ); + }); + test('forwards custom tool call history with freeform input', async () => { let captured: IByokLmChatRequest | undefined; await withProxy( diff --git a/src/vs/workbench/contrib/chat/browser/agentSessions/agentHost/agentHostByokLmHandler.ts b/src/vs/workbench/contrib/chat/browser/agentSessions/agentHost/agentHostByokLmHandler.ts index 27a22293686..f19515b4847 100644 --- a/src/vs/workbench/contrib/chat/browser/agentSessions/agentHost/agentHostByokLmHandler.ts +++ b/src/vs/workbench/contrib/chat/browser/agentSessions/agentHost/agentHostByokLmHandler.ts @@ -6,11 +6,12 @@ import { CancellationToken } from '../../../../../../base/common/cancellation.js'; import { Emitter, Event } from '../../../../../../base/common/event.js'; import { Disposable } from '../../../../../../base/common/lifecycle.js'; -import { VSBuffer } from '../../../../../../base/common/buffer.js'; +import { decodeBase64, VSBuffer } from '../../../../../../base/common/buffer.js'; import { IAgentHostByokLmHandler, IByokLmChatRequest, IByokLmChatResult, + IByokLmContentPart, IByokLmInputItem, IByokLmModelInfo, IByokLmOutputItem, @@ -18,6 +19,7 @@ import { } from '../../../../../../platform/agentHost/common/agentHostByokLm.js'; import { ILogService } from '../../../../../../platform/log/common/log.js'; import { + ChatImageMimeType, ChatMessageRole, IChatMessage, IChatMessagePart, @@ -207,7 +209,7 @@ export class AgentHostByokLmHandler extends Disposable implements IAgentHostByok case 'message': return { role: this._toChatRole(item.role), - content: [{ type: 'text', value: item.content.map(part => part.text).join('') }], + content: this._toChatMessageParts(item.content), }; case 'reasoning': { return { @@ -256,6 +258,23 @@ export class AgentHostByokLmHandler extends Disposable implements IAgentHostByok } } + private _toChatMessageParts(parts: IByokLmContentPart[]): IChatMessagePart[] { + const result: IChatMessagePart[] = []; + for (const part of parts) { + if (part.type === 'text') { + const previous = result.at(-1); + if (previous?.type === 'text') { + previous.value += part.text; + } else { + result.push({ type: 'text', value: part.text }); + } + } else { + result.push({ type: 'image_url', value: { mimeType: part.mimeType as ChatImageMimeType, data: decodeBase64(part.data) } }); + } + } + return result.length ? result : [{ type: 'text', value: '' }]; + } + private _appendTextOutput(output: IByokLmOutputItem[], value: string): void { const previous = output.at(-1); if (previous?.type === 'message') { diff --git a/src/vs/workbench/contrib/chat/test/browser/agentSessions/agentHostByokLmHandler.test.ts b/src/vs/workbench/contrib/chat/test/browser/agentSessions/agentHostByokLmHandler.test.ts index d9ed95f9857..8897ebc4bd8 100644 --- a/src/vs/workbench/contrib/chat/test/browser/agentSessions/agentHostByokLmHandler.test.ts +++ b/src/vs/workbench/contrib/chat/test/browser/agentSessions/agentHostByokLmHandler.test.ts @@ -251,7 +251,14 @@ suite('AgentHostByokLmHandler', () => { { type: 'custom_tool_call', callId: 't2', name: 'apply_patch', input: 'patch' }, { type: 'function_call_output', callId: 't1', output: 'sunny' }, { type: 'custom_tool_call_output', callId: 't2', output: 'Done!' }, - { type: 'message', role: 'user', content: [{ type: 'text', text: 'hi' }] }, + { + type: 'message', + role: 'user', + content: [ + { type: 'text', text: 'hi' }, + { type: 'image', mimeType: 'image/png', data: 'aW1hZ2U=' }, + ], + }, ], }, CancellationToken.None, @@ -280,7 +287,13 @@ suite('AgentHostByokLmHandler', () => { }, { role: ChatMessageRole.User, content: [{ type: 'tool_result', toolCallId: 't1', value: [{ type: 'text', value: 'sunny' }] }] }, { role: ChatMessageRole.User, content: [{ type: 'tool_result', toolCallId: 't2', value: [{ type: 'text', value: 'Done!' }] }] }, - { role: ChatMessageRole.User, content: [{ type: 'text', value: 'hi' }] }, + { + role: ChatMessageRole.User, + content: [ + { type: 'text', value: 'hi' }, + { type: 'image_url', value: { mimeType: 'image/png', data: VSBuffer.fromString('image') } }, + ], + }, ], options: { modelOptions: { temperature: 0.5 }, From 73a9c5aa010a73a85621970eb7ce2028f1819192 Mon Sep 17 00:00:00 2001 From: vritant24 Date: Mon, 3 Aug 2026 11:26:53 -0700 Subject: [PATCH 2/2] Validate BYOK image MIME types Reject image data URLs outside the renderer image contract before they cross the Agent Host bridge, and map accepted MIME types exhaustively. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../agentHost/common/agentHostByokLm.ts | 4 +++- .../node/copilot/byokResponsesTranslation.ts | 17 +++++++++++++++++ .../test/node/byokLmProxyService.test.ts | 11 ++++++++++- .../agentHost/agentHostByokLmHandler.ts | 18 +++++++++++++++++- 4 files changed, 47 insertions(+), 3 deletions(-) diff --git a/src/vs/platform/agentHost/common/agentHostByokLm.ts b/src/vs/platform/agentHost/common/agentHostByokLm.ts index ab4ba49e08b..66c2c89a37b 100644 --- a/src/vs/platform/agentHost/common/agentHostByokLm.ts +++ b/src/vs/platform/agentHost/common/agentHostByokLm.ts @@ -25,9 +25,11 @@ export interface IByokLmTextPart { readonly text: string; } +export type ByokLmImageMimeType = 'image/png' | 'image/jpeg' | 'image/gif' | 'image/webp' | 'image/bmp'; + export interface IByokLmImagePart { readonly type: 'image'; - readonly mimeType: string; + readonly mimeType: ByokLmImageMimeType; readonly data: string; } diff --git a/src/vs/platform/agentHost/node/copilot/byokResponsesTranslation.ts b/src/vs/platform/agentHost/node/copilot/byokResponsesTranslation.ts index d04f97b9727..4292ee91ca1 100644 --- a/src/vs/platform/agentHost/node/copilot/byokResponsesTranslation.ts +++ b/src/vs/platform/agentHost/node/copilot/byokResponsesTranslation.ts @@ -5,6 +5,7 @@ import { decodeBase64 } from '../../../../base/common/buffer.js'; import { + ByokLmImageMimeType, IByokLmChatRequest, IByokLmChatResult, IByokLmContentPart, @@ -24,6 +25,19 @@ interface IResponsesSummaryPart { readonly text?: string; } +function isSupportedImageMimeType(mimeType: string): mimeType is ByokLmImageMimeType { + switch (mimeType) { + case 'image/png': + case 'image/jpeg': + case 'image/gif': + case 'image/webp': + case 'image/bmp': + return true; + default: + return false; + } +} + interface IResponsesInputItem { readonly type?: string; readonly role?: string; @@ -88,6 +102,9 @@ function toContentParts(content: string | IResponsesContentPart[] | undefined, i if (part.type === 'input_image' && typeof part.image_url === 'string') { const match = /^data:(?image\/[^;,]+)(?:;[^,]*)?;base64,(?.*)$/.exec(part.image_url); if (match?.groups) { + if (!isSupportedImageMimeType(match.groups.mimeType)) { + throw new ResponsesTranslationError(`Unsupported input[${itemIndex}].content[${contentIndex}].image_url MIME type '${match.groups.mimeType}'`); + } try { decodeBase64(match.groups.data); } catch { diff --git a/src/vs/platform/agentHost/test/node/byokLmProxyService.test.ts b/src/vs/platform/agentHost/test/node/byokLmProxyService.test.ts index b47dd63b3ec..b2c9309de26 100644 --- a/src/vs/platform/agentHost/test/node/byokLmProxyService.test.ts +++ b/src/vs/platform/agentHost/test/node/byokLmProxyService.test.ts @@ -210,7 +210,7 @@ suite('ByokLmProxyService', () => { async () => ({ output: [] }), async handle => { const responses: Array<{ status: number; body: unknown }> = []; - for (const imageUrl of ['https://example.com/image.png', 'data:image/png;base64,not valid']) { + for (const imageUrl of ['https://example.com/image.png', 'data:image/svg+xml;base64,PHN2Zz4=', 'data:image/png;base64,not valid']) { const response = await fetch(responsesUrl(handle, 'gemini'), { method: 'POST', headers: authHeaders(handle), @@ -236,6 +236,15 @@ suite('ByokLmProxyService', () => { }, }, }, + { + status: 400, + body: { + error: { + message: 'Unsupported input[0].content[0].image_url MIME type \'image/svg+xml\'', + type: 'invalid_request_error', + }, + }, + }, { status: 400, body: { diff --git a/src/vs/workbench/contrib/chat/browser/agentSessions/agentHost/agentHostByokLmHandler.ts b/src/vs/workbench/contrib/chat/browser/agentSessions/agentHost/agentHostByokLmHandler.ts index f19515b4847..b9239ca3f54 100644 --- a/src/vs/workbench/contrib/chat/browser/agentSessions/agentHost/agentHostByokLmHandler.ts +++ b/src/vs/workbench/contrib/chat/browser/agentSessions/agentHost/agentHostByokLmHandler.ts @@ -8,6 +8,7 @@ import { Emitter, Event } from '../../../../../../base/common/event.js'; import { Disposable } from '../../../../../../base/common/lifecycle.js'; import { decodeBase64, VSBuffer } from '../../../../../../base/common/buffer.js'; import { + ByokLmImageMimeType, IAgentHostByokLmHandler, IByokLmChatRequest, IByokLmChatResult, @@ -269,12 +270,27 @@ export class AgentHostByokLmHandler extends Disposable implements IAgentHostByok result.push({ type: 'text', value: part.text }); } } else { - result.push({ type: 'image_url', value: { mimeType: part.mimeType as ChatImageMimeType, data: decodeBase64(part.data) } }); + result.push({ type: 'image_url', value: { mimeType: this._toChatImageMimeType(part.mimeType), data: decodeBase64(part.data) } }); } } return result.length ? result : [{ type: 'text', value: '' }]; } + private _toChatImageMimeType(mimeType: ByokLmImageMimeType): ChatImageMimeType { + switch (mimeType) { + case 'image/png': + return ChatImageMimeType.PNG; + case 'image/jpeg': + return ChatImageMimeType.JPEG; + case 'image/gif': + return ChatImageMimeType.GIF; + case 'image/webp': + return ChatImageMimeType.WEBP; + case 'image/bmp': + return ChatImageMimeType.BMP; + } + } + private _appendTextOutput(output: IByokLmOutputItem[], value: string): void { const previous = output.at(-1); if (previous?.type === 'message') {