From df984eb56f52bced08e6accea9aaca718a88a01b Mon Sep 17 00:00:00 2001 From: Justin Chen <54879025+justschen@users.noreply.github.com> Date: Mon, 28 Jul 2025 11:00:38 -0700 Subject: [PATCH] make sure to resize images coming from tool calls (#257785) * make sure to resize images coming from tool calls * try/catch, resize in the right areas * new image service * some cleanup * clean up service usage * no more electron main service * app.ts cleanup * address comments * wrap in try catch and revert * proper revert --- .../imageResize/browser/imageResizeService.ts | 167 ++++++++++++++++++ .../imageResize/common/imageResizeService.ts | 19 ++ .../browser/contrib/chatInputCompletions.ts | 5 +- .../mcpLanguageModelToolContribution.ts | 23 ++- .../imageResize/browser/imageResizeService.ts | 10 ++ .../electron-browser/imageResizeService.ts | 10 ++ src/vs/workbench/workbench.desktop.main.ts | 1 + .../workbench/workbench.web.main.internal.ts | 1 + 8 files changed, 227 insertions(+), 9 deletions(-) create mode 100644 src/vs/platform/imageResize/browser/imageResizeService.ts create mode 100644 src/vs/platform/imageResize/common/imageResizeService.ts create mode 100644 src/vs/workbench/services/imageResize/browser/imageResizeService.ts create mode 100644 src/vs/workbench/services/imageResize/electron-browser/imageResizeService.ts diff --git a/src/vs/platform/imageResize/browser/imageResizeService.ts b/src/vs/platform/imageResize/browser/imageResizeService.ts new file mode 100644 index 00000000000..6a960f7b399 --- /dev/null +++ b/src/vs/platform/imageResize/browser/imageResizeService.ts @@ -0,0 +1,167 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { decodeBase64, VSBuffer } from '../../../base/common/buffer.js'; +import { joinPath } from '../../../base/common/resources.js'; +import { URI } from '../../../base/common/uri.js'; +import { IFileService } from '../../files/common/files.js'; +import { InstantiationType, registerSingleton } from '../../instantiation/common/extensions.js'; +import { ILogService } from '../../log/common/log.js'; +import { IImageResizeService } from '../common/imageResizeService.js'; + + +export class ImageResizeService implements IImageResizeService { + + declare readonly _serviceBrand: undefined; + + /** + * Resizes an image provided as a UInt8Array string. Resizing is based on Open AI's algorithm for tokenzing images. + * https://platform.openai.com/docs/guides/vision#calculating-costs + * @param data - The UInt8Array string of the image to resize. + * @returns A promise that resolves to the UInt8Array string of the resized image. + */ + + async resizeImage(data: Uint8Array | string, mimeType?: string): Promise { + const isGif = mimeType === 'image/gif'; + + if (typeof data === 'string') { + data = this.convertStringToUInt8Array(data); + } + + return new Promise((resolve, reject) => { + const blob = new Blob([data as Uint8Array], { type: mimeType }); + const img = new Image(); + const url = URL.createObjectURL(blob); + img.src = url; + + img.onload = () => { + URL.revokeObjectURL(url); + let { width, height } = img; + + if ((width <= 768 || height <= 768) && !isGif) { + resolve(data); + return; + } + + // Calculate the new dimensions while maintaining the aspect ratio + if (width > 2048 || height > 2048) { + const scaleFactor = 2048 / Math.max(width, height); + width = Math.round(width * scaleFactor); + height = Math.round(height * scaleFactor); + } + + const scaleFactor = 768 / Math.min(width, height); + width = Math.round(width * scaleFactor); + height = Math.round(height * scaleFactor); + + const canvas = document.createElement('canvas'); + canvas.width = width; + canvas.height = height; + const ctx = canvas.getContext('2d'); + if (ctx) { + ctx.drawImage(img, 0, 0, width, height); + canvas.toBlob((blob) => { + if (blob) { + const reader = new FileReader(); + reader.onload = () => { + resolve(new Uint8Array(reader.result as ArrayBuffer)); + }; + reader.onerror = (error) => reject(error); + reader.readAsArrayBuffer(blob); + } else { + reject(new Error('Failed to create blob from canvas')); + } + }, mimeType || 'image/png'); + } else { + reject(new Error('Failed to get canvas context')); + } + }; + img.onerror = (error) => { + URL.revokeObjectURL(url); + reject(error); + }; + }); + } + + convertStringToUInt8Array(data: string): Uint8Array { + const base64Data = data.includes(',') ? data.split(',')[1] : data; + if (this.isValidBase64(base64Data)) { + return decodeBase64(base64Data).buffer; + } + return new TextEncoder().encode(data); + } + + // Only used for URLs + convertUint8ArrayToString(data: Uint8Array): string { + try { + const decoder = new TextDecoder(); + const decodedString = decoder.decode(data); + return decodedString; + } catch { + return ''; + } + } + + isValidBase64(str: string): boolean { + try { + decodeBase64(str); + return true; + } catch { + return false; + } + } + + async createFileForMedia(fileService: IFileService, imagesFolder: URI, dataTransfer: Uint8Array, mimeType: string): Promise { + const exists = await fileService.exists(imagesFolder); + if (!exists) { + await fileService.createFolder(imagesFolder); + } + + const ext = mimeType.split('/')[1] || 'png'; + const filename = `image-${Date.now()}.${ext}`; + const fileUri = joinPath(imagesFolder, filename); + + const buffer = VSBuffer.wrap(dataTransfer); + await fileService.writeFile(fileUri, buffer); + + return fileUri; + } + + async cleanupOldImages(fileService: IFileService, logService: ILogService, imagesFolder: URI): Promise { + const exists = await fileService.exists(imagesFolder); + if (!exists) { + return; + } + + const duration = 7 * 24 * 60 * 60 * 1000; // 7 days + const files = await fileService.resolve(imagesFolder); + if (!files.children) { + return; + } + + await Promise.all(files.children.map(async (file) => { + try { + const timestamp = this.getTimestampFromFilename(file.name); + if (timestamp && (Date.now() - timestamp > duration)) { + await fileService.del(file.resource); + } + } catch (err) { + logService.error('Failed to clean up old images', err); + } + })); + } + + getTimestampFromFilename(filename: string): number | undefined { + const match = filename.match(/image-(\d+)\./); + if (match) { + return parseInt(match[1], 10); + } + return undefined; + } + + +} + +registerSingleton(IImageResizeService, ImageResizeService, InstantiationType.Delayed); diff --git a/src/vs/platform/imageResize/common/imageResizeService.ts b/src/vs/platform/imageResize/common/imageResizeService.ts new file mode 100644 index 00000000000..fc67f21ee13 --- /dev/null +++ b/src/vs/platform/imageResize/common/imageResizeService.ts @@ -0,0 +1,19 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { createDecorator } from '../../instantiation/common/instantiation.js'; + +export const IImageResizeService = createDecorator('imageResizeMainService'); + + +export interface IImageResizeService { + + readonly _serviceBrand: undefined; + + /** + * Resizes an image to a maximum dimension of 768px while maintaining aspect ratio. + */ + resizeImage(data: Uint8Array | string, mimeType?: string): Promise; +} diff --git a/src/vs/workbench/contrib/chat/browser/contrib/chatInputCompletions.ts b/src/vs/workbench/contrib/chat/browser/contrib/chatInputCompletions.ts index 83c3ac88104..43bf922e0c9 100644 --- a/src/vs/workbench/contrib/chat/browser/contrib/chatInputCompletions.ts +++ b/src/vs/workbench/contrib/chat/browser/contrib/chatInputCompletions.ts @@ -56,6 +56,7 @@ import { ToolSet } from '../../common/languageModelToolsService.js'; import { IPromptsService } from '../../common/promptSyntax/service/promptsService.js'; import { ChatSubmitAction } from '../actions/chatExecuteActions.js'; import { IChatWidget, IChatWidgetService } from '../chat.js'; +import { resizeImage } from '../imageUtils.js'; import { ChatDynamicVariableModel } from './chatDynamicVariables.js'; class SlashCommandCompletions extends Disposable { @@ -647,11 +648,13 @@ class StartParameterizedPromptAction extends Action2 { }); } } else if (mimeType && getAttachableImageExtension(mimeType)) { + const resized = await resizeImage(contents) + .catch(() => decodeBase64(contents).buffer); chatWidget.attachmentModel.addContext({ id: generateUuid(), name: localize('mcp.prompt.image', 'Prompt Image'), fullName: localize('mcp.prompt.image', 'Prompt Image'), - value: decodeBase64(contents).buffer, + value: resized, kind: 'image', references: validURI && [{ reference: validURI, kind: 'reference' }], }); diff --git a/src/vs/workbench/contrib/mcp/common/mcpLanguageModelToolContribution.ts b/src/vs/workbench/contrib/mcp/common/mcpLanguageModelToolContribution.ts index 80f77cc8c7c..899d39079c3 100644 --- a/src/vs/workbench/contrib/mcp/common/mcpLanguageModelToolContribution.ts +++ b/src/vs/workbench/contrib/mcp/common/mcpLanguageModelToolContribution.ts @@ -15,6 +15,7 @@ import { basename } from '../../../../base/common/resources.js'; import { URI } from '../../../../base/common/uri.js'; import { localize } from '../../../../nls.js'; import { IFileService } from '../../../../platform/files/common/files.js'; +import { IImageResizeService } from '../../../../platform/imageResize/common/imageResizeService.js'; import { IInstantiationService } from '../../../../platform/instantiation/common/instantiation.js'; import { IProductService } from '../../../../platform/product/common/productService.js'; import { StorageScope } from '../../../../platform/storage/common/storage.js'; @@ -163,6 +164,7 @@ class McpToolImplementation implements IToolImpl { private readonly _server: IMcpServer, @IProductService private readonly _productService: IProductService, @IFileService private readonly _fileService: IFileService, + @IImageResizeService private readonly _imageResizeService: IImageResizeService, ) { } async prepareToolInvocation(context: IToolInvocationPreparationContext): Promise { @@ -222,14 +224,18 @@ class McpToolImplementation implements IToolImpl { } } - // Rewrite image rsources to images so they are inlined nicely - const addAsInlineData = (mimeType: string, value: string, uri?: URI) => { + // Rewrite image resources to images so they are inlined nicely + const addAsInlineData = async (mimeType: string, value: string, uri?: URI): Promise => { details.output.push({ type: 'embed', mimeType, value, uri }); if (isForModel) { - result.content.push({ - kind: 'data', - value: { mimeType, data: decodeBase64(value) } - }); + let finalData: VSBuffer; + try { + const resized = await this._imageResizeService.resizeImage(decodeBase64(value).buffer, mimeType); + finalData = VSBuffer.wrap(resized); + } catch { + finalData = decodeBase64(value); + } + result.content.push({ kind: 'data', value: { mimeType, data: finalData } }); } }; @@ -246,7 +252,7 @@ class McpToolImplementation implements IToolImpl { } } else if (item.type === 'image' || item.type === 'audio') { // default to some image type if not given to hint - addAsInlineData(item.mimeType || 'image/png', item.data); + await addAsInlineData(item.mimeType || 'image/png', item.data); } else if (item.type === 'resource_link') { const uri = McpResourceURI.fromServer(this._server.definition, item.uri); details.output.push({ @@ -274,7 +280,7 @@ class McpToolImplementation implements IToolImpl { } else if (item.type === 'resource') { const uri = McpResourceURI.fromServer(this._server.definition, item.resource.uri); if (item.resource.mimeType && getAttachableImageExtension(item.resource.mimeType) && 'blob' in item.resource) { - addAsInlineData(item.resource.mimeType, item.resource.blob, uri); + await addAsInlineData(item.resource.mimeType, item.resource.blob, uri); } else { details.output.push({ type: 'embed', @@ -305,4 +311,5 @@ class McpToolImplementation implements IToolImpl { result.toolResultDetails = details; return result; } + } diff --git a/src/vs/workbench/services/imageResize/browser/imageResizeService.ts b/src/vs/workbench/services/imageResize/browser/imageResizeService.ts new file mode 100644 index 00000000000..848d6657271 --- /dev/null +++ b/src/vs/workbench/services/imageResize/browser/imageResizeService.ts @@ -0,0 +1,10 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { IImageResizeService } from '../../../../platform/imageResize/common/imageResizeService.js'; +import { ImageResizeService } from '../../../../platform/imageResize/browser/imageResizeService.js'; +import { InstantiationType, registerSingleton } from '../../../../platform/instantiation/common/extensions.js'; + +registerSingleton(IImageResizeService, ImageResizeService, InstantiationType.Delayed); diff --git a/src/vs/workbench/services/imageResize/electron-browser/imageResizeService.ts b/src/vs/workbench/services/imageResize/electron-browser/imageResizeService.ts new file mode 100644 index 00000000000..848d6657271 --- /dev/null +++ b/src/vs/workbench/services/imageResize/electron-browser/imageResizeService.ts @@ -0,0 +1,10 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { IImageResizeService } from '../../../../platform/imageResize/common/imageResizeService.js'; +import { ImageResizeService } from '../../../../platform/imageResize/browser/imageResizeService.js'; +import { InstantiationType, registerSingleton } from '../../../../platform/instantiation/common/extensions.js'; + +registerSingleton(IImageResizeService, ImageResizeService, InstantiationType.Delayed); diff --git a/src/vs/workbench/workbench.desktop.main.ts b/src/vs/workbench/workbench.desktop.main.ts index af84852ebd2..f5d13c8f52b 100644 --- a/src/vs/workbench/workbench.desktop.main.ts +++ b/src/vs/workbench/workbench.desktop.main.ts @@ -55,6 +55,7 @@ import './services/themes/electron-browser/nativeHostColorSchemeService.js'; import './services/extensionManagement/electron-browser/extensionManagementService.js'; import './services/mcp/electron-browser/mcpWorkbenchManagementService.js'; import './services/encryption/electron-browser/encryptionService.js'; +import './services/imageResize/electron-browser/imageResizeService.js'; import './services/browserElements/electron-browser/browserElementsService.js'; import './services/secrets/electron-browser/secretStorageService.js'; import './services/localization/electron-browser/languagePackService.js'; diff --git a/src/vs/workbench/workbench.web.main.internal.ts b/src/vs/workbench/workbench.web.main.internal.ts index eae3a102dee..2b4feb52c6d 100644 --- a/src/vs/workbench/workbench.web.main.internal.ts +++ b/src/vs/workbench/workbench.web.main.internal.ts @@ -58,6 +58,7 @@ import './services/localization/browser/localeService.js'; import './services/path/browser/pathService.js'; import './services/themes/browser/browserHostColorSchemeService.js'; import './services/encryption/browser/encryptionService.js'; +import './services/imageResize/browser/imageResizeService.js'; import './services/secrets/browser/secretStorageService.js'; import './services/workingCopy/browser/workingCopyBackupService.js'; import './services/tunnel/browser/tunnelService.js';