From cc86f3c7a341af86a97b054e5efcedcca39ac40f Mon Sep 17 00:00:00 2001 From: Rob Lourens Date: Thu, 12 Mar 2026 23:01:55 -0700 Subject: [PATCH] Split image viewing out of read file (#4394) * Add dedicated view image tool (Written by Copilot) * Remove stale read file test import (Written by Copilot) * Add image extension coverage test (Written by Copilot) * Update test snapshots --- extensions/copilot/package.json | 20 +++ extensions/copilot/package.nls.json | 2 + .../all_non_edit_tools.spec.snap | 1 + .../all_tools.spec.snap | 1 + .../all_non_edit_tools.spec.snap | 1 + .../all_tools.spec.snap | 1 + .../all_non_edit_tools.spec.snap | 1 + .../all_tools.spec.snap | 1 + .../all_non_edit_tools.spec.snap | 1 + .../all_tools.spec.snap | 1 + .../src/extension/tools/common/toolNames.ts | 3 + .../src/extension/tools/node/allTools.ts | 2 +- .../extension/tools/node/imageToolUtils.ts | 29 ++++ .../src/extension/tools/node/readFileTool.tsx | 76 ++------- .../tools/node/test/imageToolUtils.spec.ts | 31 ++++ .../tools/node/test/readFile.spec.tsx | 156 +----------------- .../tools/node/test/viewImage.spec.tsx | 127 ++++++++++++++ .../extension/tools/node/viewImageTool.tsx | 106 ++++++++++++ 18 files changed, 345 insertions(+), 215 deletions(-) create mode 100644 extensions/copilot/src/extension/tools/node/imageToolUtils.ts create mode 100644 extensions/copilot/src/extension/tools/node/test/imageToolUtils.spec.ts create mode 100644 extensions/copilot/src/extension/tools/node/test/viewImage.spec.tsx create mode 100644 extensions/copilot/src/extension/tools/node/viewImageTool.tsx diff --git a/extensions/copilot/package.json b/extensions/copilot/package.json index 66220ead1ce..665b13a9cf6 100644 --- a/extensions/copilot/package.json +++ b/extensions/copilot/package.json @@ -375,6 +375,25 @@ ] } }, + { + "name": "copilot_viewImage", + "toolReferenceName": "viewImage", + "displayName": "%copilot.tools.viewImage.name%", + "userDescription": "%copilot.tools.viewImage.userDescription%", + "modelDescription": "View the contents of an image file. Use this instead of read_file for supported image files such as png, jpg, jpeg, gif, and webp. The tool returns the image directly to multimodal models and does not take line ranges or offsets.", + "inputSchema": { + "type": "object", + "properties": { + "filePath": { + "description": "The absolute path of the image file to view.", + "type": "string" + } + }, + "required": [ + "filePath" + ] + } + }, { "name": "copilot_listDirectory", "toolReferenceName": "listDirectory", @@ -1180,6 +1199,7 @@ "getNotebookSummary", "problems", "readFile", + "viewImage", "readNotebookCellOutput" ] }, diff --git a/extensions/copilot/package.nls.json b/extensions/copilot/package.nls.json index bea4963b50b..ed9ced01dea 100644 --- a/extensions/copilot/package.nls.json +++ b/extensions/copilot/package.nls.json @@ -294,6 +294,8 @@ "copilot.tools.applyPatch.name": "Apply Patch", "copilot.tools.readFile.name": "Read File", "copilot.tools.readFile.userDescription": "Read the contents of a file", + "copilot.tools.viewImage.name": "View Image", + "copilot.tools.viewImage.userDescription": "View the contents of an image file", "copilot.tools.listDirectory.name": "List Dir", "copilot.tools.listDirectory.userDescription": "List the contents of a directory", "copilot.tools.getTaskOutput.name": "Get Task Output", diff --git a/extensions/copilot/src/extension/prompts/node/agent/test/__snapshots__/agentPrompts-claude-opus-4.5/all_non_edit_tools.spec.snap b/extensions/copilot/src/extension/prompts/node/agent/test/__snapshots__/agentPrompts-claude-opus-4.5/all_non_edit_tools.spec.snap index 0d484ce9dc5..ee88d461245 100644 --- a/extensions/copilot/src/extension/prompts/node/agent/test/__snapshots__/agentPrompts-claude-opus-4.5/all_non_edit_tools.spec.snap +++ b/extensions/copilot/src/extension/prompts/node/agent/test/__snapshots__/agentPrompts-claude-opus-4.5/all_non_edit_tools.spec.snap @@ -91,6 +91,7 @@ search_workspace_symbols test_failure test_search tool_replay +view_image diff --git a/extensions/copilot/src/extension/prompts/node/agent/test/__snapshots__/agentPrompts-claude-opus-4.5/all_tools.spec.snap b/extensions/copilot/src/extension/prompts/node/agent/test/__snapshots__/agentPrompts-claude-opus-4.5/all_tools.spec.snap index f3248c2f38c..1f40bd0666e 100644 --- a/extensions/copilot/src/extension/prompts/node/agent/test/__snapshots__/agentPrompts-claude-opus-4.5/all_tools.spec.snap +++ b/extensions/copilot/src/extension/prompts/node/agent/test/__snapshots__/agentPrompts-claude-opus-4.5/all_tools.spec.snap @@ -90,6 +90,7 @@ search_workspace_symbols test_failure test_search tool_replay +view_image diff --git a/extensions/copilot/src/extension/prompts/node/agent/test/__snapshots__/agentPrompts-claude-opus-4.6-fast/all_non_edit_tools.spec.snap b/extensions/copilot/src/extension/prompts/node/agent/test/__snapshots__/agentPrompts-claude-opus-4.6-fast/all_non_edit_tools.spec.snap index 6b9e68afc63..c1039169354 100644 --- a/extensions/copilot/src/extension/prompts/node/agent/test/__snapshots__/agentPrompts-claude-opus-4.6-fast/all_non_edit_tools.spec.snap +++ b/extensions/copilot/src/extension/prompts/node/agent/test/__snapshots__/agentPrompts-claude-opus-4.6-fast/all_non_edit_tools.spec.snap @@ -123,6 +123,7 @@ search_workspace_symbols test_failure test_search tool_replay +view_image diff --git a/extensions/copilot/src/extension/prompts/node/agent/test/__snapshots__/agentPrompts-claude-opus-4.6-fast/all_tools.spec.snap b/extensions/copilot/src/extension/prompts/node/agent/test/__snapshots__/agentPrompts-claude-opus-4.6-fast/all_tools.spec.snap index 3a09c13596e..40feb1ed886 100644 --- a/extensions/copilot/src/extension/prompts/node/agent/test/__snapshots__/agentPrompts-claude-opus-4.6-fast/all_tools.spec.snap +++ b/extensions/copilot/src/extension/prompts/node/agent/test/__snapshots__/agentPrompts-claude-opus-4.6-fast/all_tools.spec.snap @@ -122,6 +122,7 @@ search_workspace_symbols test_failure test_search tool_replay +view_image diff --git a/extensions/copilot/src/extension/prompts/node/agent/test/__snapshots__/agentPrompts-claude-opus-4.6/all_non_edit_tools.spec.snap b/extensions/copilot/src/extension/prompts/node/agent/test/__snapshots__/agentPrompts-claude-opus-4.6/all_non_edit_tools.spec.snap index 6b9e68afc63..c1039169354 100644 --- a/extensions/copilot/src/extension/prompts/node/agent/test/__snapshots__/agentPrompts-claude-opus-4.6/all_non_edit_tools.spec.snap +++ b/extensions/copilot/src/extension/prompts/node/agent/test/__snapshots__/agentPrompts-claude-opus-4.6/all_non_edit_tools.spec.snap @@ -123,6 +123,7 @@ search_workspace_symbols test_failure test_search tool_replay +view_image diff --git a/extensions/copilot/src/extension/prompts/node/agent/test/__snapshots__/agentPrompts-claude-opus-4.6/all_tools.spec.snap b/extensions/copilot/src/extension/prompts/node/agent/test/__snapshots__/agentPrompts-claude-opus-4.6/all_tools.spec.snap index 3a09c13596e..40feb1ed886 100644 --- a/extensions/copilot/src/extension/prompts/node/agent/test/__snapshots__/agentPrompts-claude-opus-4.6/all_tools.spec.snap +++ b/extensions/copilot/src/extension/prompts/node/agent/test/__snapshots__/agentPrompts-claude-opus-4.6/all_tools.spec.snap @@ -122,6 +122,7 @@ search_workspace_symbols test_failure test_search tool_replay +view_image diff --git a/extensions/copilot/src/extension/prompts/node/agent/test/__snapshots__/agentPrompts-claude-sonnet-4.5/all_non_edit_tools.spec.snap b/extensions/copilot/src/extension/prompts/node/agent/test/__snapshots__/agentPrompts-claude-sonnet-4.5/all_non_edit_tools.spec.snap index 0d484ce9dc5..ee88d461245 100644 --- a/extensions/copilot/src/extension/prompts/node/agent/test/__snapshots__/agentPrompts-claude-sonnet-4.5/all_non_edit_tools.spec.snap +++ b/extensions/copilot/src/extension/prompts/node/agent/test/__snapshots__/agentPrompts-claude-sonnet-4.5/all_non_edit_tools.spec.snap @@ -91,6 +91,7 @@ search_workspace_symbols test_failure test_search tool_replay +view_image diff --git a/extensions/copilot/src/extension/prompts/node/agent/test/__snapshots__/agentPrompts-claude-sonnet-4.5/all_tools.spec.snap b/extensions/copilot/src/extension/prompts/node/agent/test/__snapshots__/agentPrompts-claude-sonnet-4.5/all_tools.spec.snap index f3248c2f38c..1f40bd0666e 100644 --- a/extensions/copilot/src/extension/prompts/node/agent/test/__snapshots__/agentPrompts-claude-sonnet-4.5/all_tools.spec.snap +++ b/extensions/copilot/src/extension/prompts/node/agent/test/__snapshots__/agentPrompts-claude-sonnet-4.5/all_tools.spec.snap @@ -90,6 +90,7 @@ search_workspace_symbols test_failure test_search tool_replay +view_image diff --git a/extensions/copilot/src/extension/tools/common/toolNames.ts b/extensions/copilot/src/extension/tools/common/toolNames.ts index cd4bbee0295..1720dd45e21 100644 --- a/extensions/copilot/src/extension/tools/common/toolNames.ts +++ b/extensions/copilot/src/extension/tools/common/toolNames.ts @@ -26,6 +26,7 @@ export enum ToolName { FindFiles = 'file_search', FindTextInFiles = 'grep_search', ReadFile = 'read_file', + ViewImage = 'view_image', ListDirectory = 'list_dir', GetErrors = 'get_errors', GetScmChanges = 'get_changed_files', @@ -82,6 +83,7 @@ export enum ContributedToolName { FindFiles = 'copilot_findFiles', FindTextInFiles = 'copilot_findTextInFiles', ReadFile = 'copilot_readFile', + ViewImage = 'copilot_viewImage', ListDirectory = 'copilot_listDirectory', GetErrors = 'copilot_getErrors', GetScmChanges = 'copilot_getChangedFiles', @@ -158,6 +160,7 @@ export const toolCategories: Record = { [ToolName.Codebase]: ToolCategory.Core, [ToolName.FindTextInFiles]: ToolCategory.Core, [ToolName.ReadFile]: ToolCategory.Core, + [ToolName.ViewImage]: ToolCategory.Core, [ToolName.CreateFile]: ToolCategory.Core, [ToolName.ApplyPatch]: ToolCategory.Core, [ToolName.ReplaceString]: ToolCategory.Core, diff --git a/extensions/copilot/src/extension/tools/node/allTools.ts b/extensions/copilot/src/extension/tools/node/allTools.ts index f1a9d21e5ce..ad2f2306ef2 100644 --- a/extensions/copilot/src/extension/tools/node/allTools.ts +++ b/extensions/copilot/src/extension/tools/node/allTools.ts @@ -35,6 +35,6 @@ import './searchWorkspaceSymbolsTool'; import './testFailureTool'; import './toolReplayTool'; import './toolSearchTool'; +import './viewImageTool'; import './vscodeAPITool'; import './vscodeCmdTool'; - diff --git a/extensions/copilot/src/extension/tools/node/imageToolUtils.ts b/extensions/copilot/src/extension/tools/node/imageToolUtils.ts new file mode 100644 index 00000000000..27ee5542a12 --- /dev/null +++ b/extensions/copilot/src/extension/tools/node/imageToolUtils.ts @@ -0,0 +1,29 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { URI } from '../../../util/vs/base/common/uri'; +import { ChatImageMimeType } from '../../conversation/common/languageModelChatMessageHelpers'; + +/** Maximum image file size in bytes (20 MB) */ +export const MAX_IMAGE_FILE_SIZE = 20 * 1024 * 1024; + +const imageExtensionToMimeType: Record = { + '.png': ChatImageMimeType.PNG, + '.jpg': ChatImageMimeType.JPEG, + '.jpeg': ChatImageMimeType.JPEG, + '.gif': ChatImageMimeType.GIF, + '.webp': ChatImageMimeType.WEBP, +}; + +export function getImageMimeType(uri: URI): ChatImageMimeType | undefined { + const path = uri.path.toLowerCase(); + for (const [ext, mime] of Object.entries(imageExtensionToMimeType)) { + if (path.endsWith(ext)) { + return mime; + } + } + + return undefined; +} diff --git a/extensions/copilot/src/extension/tools/node/readFileTool.tsx b/extensions/copilot/src/extension/tools/node/readFileTool.tsx index 2a585e60cef..788d794da1c 100644 --- a/extensions/copilot/src/extension/tools/node/readFileTool.tsx +++ b/extensions/copilot/src/extension/tools/node/readFileTool.tsx @@ -24,8 +24,7 @@ import { clamp } from '../../../util/vs/base/common/numbers'; import { dirname, extUriBiasedIgnorePathCase } from '../../../util/vs/base/common/resources'; import { URI } from '../../../util/vs/base/common/uri'; import { IInstantiationService } from '../../../util/vs/platform/instantiation/common/instantiation'; -import { LanguageModelDataPart, LanguageModelPromptTsxPart, LanguageModelTextPart, LanguageModelToolResult, Location, MarkdownString, Range } from '../../../vscodeTypes'; -import { ChatImageMimeType } from '../../conversation/common/languageModelChatMessageHelpers'; +import { LanguageModelPromptTsxPart, LanguageModelToolResult, Location, MarkdownString, Range } from '../../../vscodeTypes'; import { IBuildPromptContext } from '../../prompt/common/intents'; import { renderPromptElementJSON } from '../../prompts/node/base/promptRenderer'; import { BinaryFileHexdump, hexdumpIfBinary } from '../../prompts/node/panel/binaryFileHexdump'; @@ -33,11 +32,12 @@ import { CodeBlock } from '../../prompts/node/panel/safeElements'; import { ToolName } from '../common/toolNames'; import { ICopilotTool, ToolRegistry } from '../common/toolsRegistry'; import { formatUriForFileWidget } from '../common/toolUtils'; +import { getImageMimeType } from './imageToolUtils'; import { assertFileNotContentExcluded, assertFileOkForTool, isFileExternalAndNeedsConfirmation, resolveToolInputPath } from './toolUtils'; export const readFileV2Description: vscode.LanguageModelToolInformation = { name: ToolName.ReadFile, - description: 'Read the contents of a file. Line numbers are 1-indexed. This tool will truncate its output at 2000 lines and may be called repeatedly with offset and limit parameters to read larger files in chunks. For image files (png, jpg, jpeg, gif, webp), the full image content will be returned. Binary files use offset/limit as byte offsets.', + description: 'Read the contents of a file. Line numbers are 1-indexed. This tool will truncate its output at 2000 lines and may be called repeatedly with offset and limit parameters to read larger files in chunks. Binary files use offset/limit as byte offsets.', tags: ['vscode_codesearch'], source: undefined, inputSchema: { @@ -74,27 +74,6 @@ export interface IReadFileParamsV2 { const MAX_LINES_PER_READ = 2000; -/** Maximum image file size in bytes (20 MB) */ -const MAX_IMAGE_FILE_SIZE = 20 * 1024 * 1024; - -const imageExtensionToMimeType: Record = { - '.png': ChatImageMimeType.PNG, - '.jpg': ChatImageMimeType.JPEG, - '.jpeg': ChatImageMimeType.JPEG, - '.gif': ChatImageMimeType.GIF, - '.webp': ChatImageMimeType.WEBP, -}; - -function getImageMimeType(uri: URI): ChatImageMimeType | undefined { - const path = uri.path.toLowerCase(); - for (const [ext, mime] of Object.entries(imageExtensionToMimeType)) { - if (path.endsWith(ext)) { - return mime; - } - } - return undefined; -} - export type ReadFileParams = IReadFileParamsV1 | IReadFileParamsV2; const isParamsV2 = (params: ReadFileParams): params is IReadFileParamsV2 => @@ -157,10 +136,8 @@ export class ReadFileTool implements ICopilotTool { try { uri = resolveToolInputPath(options.input.filePath, this.promptPathRepresentationService); - // Handle image files - const imageMimeType = getImageMimeType(uri); - if (imageMimeType) { - return await this.invokeForImage(uri, imageMimeType, options); + if (getImageMimeType(uri)) { + throw new Error(`Cannot read image files with ${ToolName.ReadFile}. Use ${ToolName.ViewImage} instead.`); } // Handle binary files — read raw bytes and check for null bytes @@ -223,30 +200,6 @@ export class ReadFileTool implements ICopilotTool { } } - private async invokeForImage(uri: URI, imageMimeType: ChatImageMimeType, options: vscode.LanguageModelToolInvocationOptions): Promise { - const input = options.input; - const hasLineParams = isParamsV2(input) - ? (input.offset !== undefined || input.limit !== undefined) - : true; - if (hasLineParams) { - throw new Error(`Cannot specify line ranges when reading an image file. Use read_file with only the filePath parameter for image files.`); - } - - const stat = await this.fileSystemService.stat(uri); - if (stat.size > MAX_IMAGE_FILE_SIZE) { - void this.sendReadFileTelemetry('imageTooLarge', options, { start: 0, end: 0, truncated: false }, uri); - return new LanguageModelToolResult([ - new LanguageModelTextPart(`Cannot read image file ${this.promptPathRepresentationService.getFilePath(uri)}: file size (${Math.round(stat.size / (1024 * 1024))}MB) exceeds the maximum allowed size of ${Math.round(MAX_IMAGE_FILE_SIZE / (1024 * 1024))}MB.`) - ]); - } - - const imageData = await this.fileSystemService.readFile(uri, true); - void this.sendReadFileTelemetry('success', options, { start: 0, end: 0, truncated: false }, uri); - return new LanguageModelToolResult([ - LanguageModelDataPart.image(imageData, imageMimeType), - ]); - } - async prepareInvocation(options: vscode.LanguageModelToolInvocationPrepareOptions, token: vscode.CancellationToken): Promise { const { input } = options; if (!input.filePath.length) { @@ -257,7 +210,9 @@ export class ReadFileTool implements ICopilotTool { let documentSnapshot: NotebookDocumentSnapshot | TextDocumentSnapshot; try { uri = resolveToolInputPath(input.filePath, this.promptPathRepresentationService); - const isImage = !!getImageMimeType(uri); + if (getImageMimeType(uri)) { + throw new Error(`Cannot read image files with ${ToolName.ReadFile}. Use ${ToolName.ViewImage} instead.`); + } // Check if file is external (outside workspace, not open in editor, etc.) const isExternal = await this.instantiationService.invokeFunction( @@ -274,14 +229,11 @@ export class ReadFileTool implements ICopilotTool { const message = this.workspaceService.getWorkspaceFolders().length === 1 ? new MarkdownString(l10n.t`${formatUriForFileWidget(uri)} is outside of the current folder in ${formatUriForFileWidget(folderUri)}.`) : new MarkdownString(l10n.t`${formatUriForFileWidget(uri)} is outside of the current workspace in ${formatUriForFileWidget(folderUri)}.`); - const readingLabel = isImage ? l10n.t`Reading image ${formatUriForFileWidget(uri)}` : l10n.t`Reading ${formatUriForFileWidget(uri)}`; - const readLabel = isImage ? l10n.t`Read image ${formatUriForFileWidget(uri)}` : l10n.t`Read ${formatUriForFileWidget(uri)}`; - // Return confirmation request for external file // The folder-based "allow this session" option is provided by the core confirmation contribution return { - invocationMessage: new MarkdownString(readingLabel), - pastTenseMessage: new MarkdownString(readLabel), + invocationMessage: new MarkdownString(l10n.t`Reading ${formatUriForFileWidget(uri)}`), + pastTenseMessage: new MarkdownString(l10n.t`Read ${formatUriForFileWidget(uri)}`), confirmationMessages: { title: l10n.t`Allow reading external files?`, message, @@ -291,14 +243,6 @@ export class ReadFileTool implements ICopilotTool { await this.instantiationService.invokeFunction(accessor => assertFileOkForTool(accessor, uri!, this._promptContext, { readOnly: true })); - // For images, we're done - no snapshot/range logic needed - if (isImage) { - return { - invocationMessage: new MarkdownString(l10n.t`Reading image ${formatUriForFileWidget(uri)}`), - pastTenseMessage: new MarkdownString(l10n.t`Read image ${formatUriForFileWidget(uri)}`), - }; - } - try { documentSnapshot = await this.getSnapshot(uri); } catch (e) { diff --git a/extensions/copilot/src/extension/tools/node/test/imageToolUtils.spec.ts b/extensions/copilot/src/extension/tools/node/test/imageToolUtils.spec.ts new file mode 100644 index 00000000000..4c421da4086 --- /dev/null +++ b/extensions/copilot/src/extension/tools/node/test/imageToolUtils.spec.ts @@ -0,0 +1,31 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { expect, suite, test } from 'vitest'; +import { URI } from '../../../../util/vs/base/common/uri'; +import { ChatImageMimeType } from '../../../conversation/common/languageModelChatMessageHelpers'; +import { getImageMimeType } from '../imageToolUtils'; + +suite('imageToolUtils', () => { + test('recognizes all supported image extensions', () => { + expect([ + getImageMimeType(URI.file('/workspace/image.png')), + getImageMimeType(URI.file('/workspace/image.jpg')), + getImageMimeType(URI.file('/workspace/image.jpeg')), + getImageMimeType(URI.file('/workspace/image.gif')), + getImageMimeType(URI.file('/workspace/image.webp')), + ]).toEqual([ + ChatImageMimeType.PNG, + ChatImageMimeType.JPEG, + ChatImageMimeType.JPEG, + ChatImageMimeType.GIF, + ChatImageMimeType.WEBP, + ]); + }); + + test('does not recognize unsupported extensions as images', () => { + expect(getImageMimeType(URI.file('/workspace/image.bmp'))).toBeUndefined(); + }); +}); \ No newline at end of file diff --git a/extensions/copilot/src/extension/tools/node/test/readFile.spec.tsx b/extensions/copilot/src/extension/tools/node/test/readFile.spec.tsx index f7008e20b93..d647ca5f580 100644 --- a/extensions/copilot/src/extension/tools/node/test/readFile.spec.tsx +++ b/extensions/copilot/src/extension/tools/node/test/readFile.spec.tsx @@ -17,7 +17,7 @@ import { CancellationToken } from '../../../../util/vs/base/common/cancellation' import { URI } from '../../../../util/vs/base/common/uri'; import { SyncDescriptor } from '../../../../util/vs/platform/instantiation/common/descriptors'; import { IInstantiationService } from '../../../../util/vs/platform/instantiation/common/instantiation'; -import { LanguageModelDataPart, MarkdownString } from '../../../../vscodeTypes'; +import { MarkdownString } from '../../../../vscodeTypes'; import { createExtensionUnitTestingServices } from '../../../test/node/services'; import { ToolName } from '../../common/toolNames'; import { IToolsService } from '../../common/toolsService'; @@ -492,7 +492,7 @@ suite('ReadFile', () => { }); suite('image files', () => { - test('returns image data for image file', async () => { + test('throws for image files and points to view_image', async () => { const services = createExtensionUnitTestingServices(); const mockFs = new MockFileSystemService(); mockFs.mockFile(URI.file('/workspace/photo.jpg'), 'fake-image-bytes'); @@ -506,20 +506,15 @@ suite('ReadFile', () => { const readFileTool = testAccessor.get(IInstantiationService).createInstance(ReadFileTool); const input: IReadFileParamsV2 = { filePath: '/workspace/photo.jpg' }; - const result = await readFileTool.invoke( + await expect(readFileTool.invoke( { input, toolInvocationToken: null as never }, CancellationToken.None - ); - - // The result should contain a LanguageModelDataPart with image data - const imagePart = result.content.find(part => part instanceof LanguageModelDataPart); - expect(imagePart).toBeDefined(); - expect((imagePart as LanguageModelDataPart).mimeType).toBe('image/jpeg'); + )).rejects.toThrow('Use view_image instead'); testAccessor.dispose(); }); - test('throws when reading image with offset/limit params', async () => { + test('prepareInvocation throws for image files and points to view_image', async () => { const services = createExtensionUnitTestingServices(); const mockFs = new MockFileSystemService(); mockFs.mockFile(URI.file('/workspace/photo.png'), 'fake-image-bytes'); @@ -532,146 +527,11 @@ suite('ReadFile', () => { const testAccessor = services.createTestingAccessor(); const readFileTool = testAccessor.get(IInstantiationService).createInstance(ReadFileTool); - const input: IReadFileParamsV2 = { filePath: '/workspace/photo.png', offset: 1, limit: 10 }; - await expect(readFileTool.invoke( - { input, toolInvocationToken: null as never }, - CancellationToken.None - )).rejects.toThrow('Cannot specify line ranges when reading an image file'); - - testAccessor.dispose(); - }); - - test('throws when reading image with v1 params', async () => { - const services = createExtensionUnitTestingServices(); - const mockFs = new MockFileSystemService(); - mockFs.mockFile(URI.file('/workspace/photo.png'), 'fake-image-bytes'); - services.define(IFileSystemService, mockFs); - services.define(IWorkspaceService, new SyncDescriptor( - TestWorkspaceService, - [[URI.file('/workspace')], []] - )); - - const testAccessor = services.createTestingAccessor(); - const readFileTool = testAccessor.get(IInstantiationService).createInstance(ReadFileTool); - - const input: IReadFileParamsV1 = { filePath: '/workspace/photo.png', startLine: 1, endLine: 5 }; - await expect(readFileTool.invoke( - { input, toolInvocationToken: null as never }, - CancellationToken.None - )).rejects.toThrow('Cannot specify line ranges when reading an image file'); - - testAccessor.dispose(); - }); - - test('returns error for oversized image files', async () => { - const services = createExtensionUnitTestingServices(); - const mockFs = new class extends MockFileSystemService { - override async stat(resource: URI) { - const result = await super.stat(resource); - if (resource.toString() === URI.file('/workspace/huge.png').toString()) { - return { ...result, size: 21 * 1024 * 1024 }; - } - return result; - } - }(); - // Create a small mock file whose stat reports a size over the 20MB limit - mockFs.mockFile(URI.file('/workspace/huge.png'), 'fake-image-bytes'); - services.define(IFileSystemService, mockFs); - services.define(IWorkspaceService, new SyncDescriptor( - TestWorkspaceService, - [[URI.file('/workspace')], []] - )); - - const testAccessor = services.createTestingAccessor(); - const readFileTool = testAccessor.get(IInstantiationService).createInstance(ReadFileTool); - - const input: IReadFileParamsV2 = { filePath: '/workspace/huge.png' }; - const result = await readFileTool.invoke( - { input, toolInvocationToken: null as never }, - CancellationToken.None - ); - - const text = await toolResultToString(testAccessor, result); - expect(text).toContain('exceeds the maximum allowed size'); - - testAccessor.dispose(); - }); - - test('prepareInvocation returns image-specific messages', async () => { - const services = createExtensionUnitTestingServices(); - const mockFs = new MockFileSystemService(); - mockFs.mockFile(URI.file('/workspace/icon.png'), 'fake-image-data'); - services.define(IFileSystemService, mockFs); - services.define(IWorkspaceService, new SyncDescriptor( - TestWorkspaceService, - [[URI.file('/workspace')], []] - )); - - const testAccessor = services.createTestingAccessor(); - const readFileTool = testAccessor.get(IInstantiationService).createInstance(ReadFileTool); - - const input: IReadFileParamsV2 = { filePath: '/workspace/icon.png' }; - const result = await readFileTool.prepareInvocation( + const input: IReadFileParamsV2 = { filePath: '/workspace/photo.png' }; + await expect(readFileTool.prepareInvocation( { input }, CancellationToken.None - ); - - expect(result).toBeDefined(); - expect((result!.invocationMessage as MarkdownString).value).toContain('Reading image'); - expect((result!.pastTenseMessage as MarkdownString).value).toContain('Read image'); - - testAccessor.dispose(); - }); - - test('recognizes all supported image extensions', async () => { - const services = createExtensionUnitTestingServices(); - const mockFs = new MockFileSystemService(); - for (const ext of ['.png', '.jpg', '.jpeg', '.gif', '.webp']) { - mockFs.mockFile(URI.file(`/workspace/image${ext}`), 'data'); - } - services.define(IFileSystemService, mockFs); - services.define(IWorkspaceService, new SyncDescriptor( - TestWorkspaceService, - [[URI.file('/workspace')], []] - )); - - const testAccessor = services.createTestingAccessor(); - const readFileTool = testAccessor.get(IInstantiationService).createInstance(ReadFileTool); - - for (const ext of ['.png', '.jpg', '.jpeg', '.gif', '.webp']) { - const input: IReadFileParamsV2 = { filePath: `/workspace/image${ext}` }; - const result = await readFileTool.prepareInvocation( - { input }, - CancellationToken.None - ); - expect(result).toBeDefined(); - expect((result!.invocationMessage as MarkdownString).value).toContain('Reading image'); - } - - testAccessor.dispose(); - }); - - test('does not treat unsupported extensions as images', async () => { - const testDoc = createTextDocumentData(URI.file('/workspace/image.bmp'), 'not an image', 'plaintext').document; - - const services = createExtensionUnitTestingServices(); - services.define(IWorkspaceService, new SyncDescriptor( - TestWorkspaceService, - [[URI.file('/workspace')], [testDoc]] - )); - - const testAccessor = services.createTestingAccessor(); - const readFileTool = testAccessor.get(IInstantiationService).createInstance(ReadFileTool); - - const input: IReadFileParamsV2 = { filePath: '/workspace/image.bmp' }; - const result = await readFileTool.prepareInvocation( - { input }, - CancellationToken.None - ); - - expect(result).toBeDefined(); - // Should be a normal "Reading" message, not "Reading image" - expect((result!.invocationMessage as MarkdownString).value).not.toContain('Reading image'); + )).rejects.toThrow('Use view_image instead'); testAccessor.dispose(); }); diff --git a/extensions/copilot/src/extension/tools/node/test/viewImage.spec.tsx b/extensions/copilot/src/extension/tools/node/test/viewImage.spec.tsx new file mode 100644 index 00000000000..4fca660744d --- /dev/null +++ b/extensions/copilot/src/extension/tools/node/test/viewImage.spec.tsx @@ -0,0 +1,127 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { expect, suite, test } from 'vitest'; +import { IFileSystemService } from '../../../../platform/filesystem/common/fileSystemService'; +import { MockFileSystemService } from '../../../../platform/filesystem/node/test/mockFileSystemService'; +import { TestWorkspaceService } from '../../../../platform/test/node/testWorkspaceService'; +import { IWorkspaceService } from '../../../../platform/workspace/common/workspaceService'; +import { createTextDocumentData } from '../../../../util/common/test/shims/textDocument'; +import { CancellationToken } from '../../../../util/vs/base/common/cancellation'; +import { URI } from '../../../../util/vs/base/common/uri'; +import { SyncDescriptor } from '../../../../util/vs/platform/instantiation/common/descriptors'; +import { IInstantiationService } from '../../../../util/vs/platform/instantiation/common/instantiation'; +import { LanguageModelDataPart, MarkdownString } from '../../../../vscodeTypes'; +import { createExtensionUnitTestingServices } from '../../../test/node/services'; +import { IViewImageParams, ViewImageTool } from '../viewImageTool'; +import { toolResultToString } from './toolTestUtils'; + +suite('ViewImage', () => { + test('returns image data for image file', async () => { + const services = createExtensionUnitTestingServices(); + const mockFs = new MockFileSystemService(); + mockFs.mockFile(URI.file('/workspace/photo.jpg'), 'fake-image-bytes'); + services.define(IFileSystemService, mockFs); + services.define(IWorkspaceService, new SyncDescriptor( + TestWorkspaceService, + [[URI.file('/workspace')], []] + )); + + const accessor = services.createTestingAccessor(); + const viewImageTool = accessor.get(IInstantiationService).createInstance(ViewImageTool); + + const input: IViewImageParams = { filePath: '/workspace/photo.jpg' }; + const result = await viewImageTool.invoke( + { input, toolInvocationToken: null as never }, + CancellationToken.None + ); + + const imagePart = result.content.find(part => part instanceof LanguageModelDataPart); + expect(imagePart).toBeDefined(); + expect((imagePart as LanguageModelDataPart).mimeType).toBe('image/jpeg'); + + accessor.dispose(); + }); + + test('returns error for oversized image files', async () => { + const services = createExtensionUnitTestingServices(); + const mockFs = new class extends MockFileSystemService { + override async stat(resource: URI) { + const result = await super.stat(resource); + if (resource.toString() === URI.file('/workspace/huge.png').toString()) { + return { ...result, size: 21 * 1024 * 1024 }; + } + return result; + } + }(); + mockFs.mockFile(URI.file('/workspace/huge.png'), 'fake-image-bytes'); + services.define(IFileSystemService, mockFs); + services.define(IWorkspaceService, new SyncDescriptor( + TestWorkspaceService, + [[URI.file('/workspace')], []] + )); + + const accessor = services.createTestingAccessor(); + const viewImageTool = accessor.get(IInstantiationService).createInstance(ViewImageTool); + + const input: IViewImageParams = { filePath: '/workspace/huge.png' }; + const result = await viewImageTool.invoke( + { input, toolInvocationToken: null as never }, + CancellationToken.None + ); + + const text = await toolResultToString(accessor, result); + expect(text).toContain('exceeds the maximum allowed size'); + + accessor.dispose(); + }); + + test('prepareInvocation returns image-specific messages', async () => { + const services = createExtensionUnitTestingServices(); + const mockFs = new MockFileSystemService(); + mockFs.mockFile(URI.file('/workspace/icon.png'), 'fake-image-data'); + services.define(IFileSystemService, mockFs); + services.define(IWorkspaceService, new SyncDescriptor( + TestWorkspaceService, + [[URI.file('/workspace')], []] + )); + + const accessor = services.createTestingAccessor(); + const viewImageTool = accessor.get(IInstantiationService).createInstance(ViewImageTool); + + const input: IViewImageParams = { filePath: '/workspace/icon.png' }; + const result = await viewImageTool.prepareInvocation( + { input }, + CancellationToken.None + ); + + expect(result).toBeDefined(); + expect((result!.invocationMessage as MarkdownString).value).toContain('Viewing image'); + expect((result!.pastTenseMessage as MarkdownString).value).toContain('Viewed image'); + + accessor.dispose(); + }); + + test('throws for non-image files and points to read_file', async () => { + const document = createTextDocumentData(URI.file('/workspace/file.ts'), 'const x = 1;', 'typescript').document; + + const services = createExtensionUnitTestingServices(); + services.define(IWorkspaceService, new SyncDescriptor( + TestWorkspaceService, + [[URI.file('/workspace')], [document]] + )); + + const accessor = services.createTestingAccessor(); + const viewImageTool = accessor.get(IInstantiationService).createInstance(ViewImageTool); + + const input: IViewImageParams = { filePath: '/workspace/file.ts' }; + await expect(viewImageTool.prepareInvocation( + { input }, + CancellationToken.None + )).rejects.toThrow('Use read_file for non-image files'); + + accessor.dispose(); + }); +}); diff --git a/extensions/copilot/src/extension/tools/node/viewImageTool.tsx b/extensions/copilot/src/extension/tools/node/viewImageTool.tsx new file mode 100644 index 00000000000..3a840eb42f9 --- /dev/null +++ b/extensions/copilot/src/extension/tools/node/viewImageTool.tsx @@ -0,0 +1,106 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import * as l10n from '@vscode/l10n'; +import type * as vscode from 'vscode'; +import { IFileSystemService } from '../../../platform/filesystem/common/fileSystemService'; +import { IPromptPathRepresentationService } from '../../../platform/prompts/common/promptPathRepresentationService'; +import { IWorkspaceService } from '../../../platform/workspace/common/workspaceService'; +import { dirname } from '../../../util/vs/base/common/resources'; +import { URI } from '../../../util/vs/base/common/uri'; +import { IInstantiationService } from '../../../util/vs/platform/instantiation/common/instantiation'; +import { LanguageModelDataPart, LanguageModelTextPart, LanguageModelToolResult, MarkdownString } from '../../../vscodeTypes'; +import { IBuildPromptContext } from '../../prompt/common/intents'; +import { ToolName } from '../common/toolNames'; +import { ICopilotTool, ToolRegistry } from '../common/toolsRegistry'; +import { formatUriForFileWidget } from '../common/toolUtils'; +import { getImageMimeType, MAX_IMAGE_FILE_SIZE } from './imageToolUtils'; +import { assertFileNotContentExcluded, assertFileOkForTool, isFileExternalAndNeedsConfirmation, resolveToolInputPath } from './toolUtils'; + +export interface IViewImageParams { + filePath: string; +} + +export class ViewImageTool implements ICopilotTool { + public static readonly toolName = ToolName.ViewImage; + + private _promptContext: IBuildPromptContext | undefined; + + constructor( + @IWorkspaceService private readonly workspaceService: IWorkspaceService, + @IPromptPathRepresentationService private readonly promptPathRepresentationService: IPromptPathRepresentationService, + @IInstantiationService private readonly instantiationService: IInstantiationService, + @IFileSystemService private readonly fileSystemService: IFileSystemService, + ) { } + + async invoke(options: vscode.LanguageModelToolInvocationOptions, _token: vscode.CancellationToken): Promise { + const uri = resolveToolInputPath(options.input.filePath, this.promptPathRepresentationService); + const imageMimeType = getImageMimeType(uri); + if (!imageMimeType) { + throw new Error(`Cannot view ${this.promptPathRepresentationService.getFilePath(uri)} with ${ToolName.ViewImage}. Use ${ToolName.ReadFile} for non-image files.`); + } + + const stat = await this.fileSystemService.stat(uri); + if (stat.size > MAX_IMAGE_FILE_SIZE) { + return new LanguageModelToolResult([ + new LanguageModelTextPart(`Cannot view image file ${this.promptPathRepresentationService.getFilePath(uri)}: file size (${Math.round(stat.size / (1024 * 1024))}MB) exceeds the maximum allowed size of ${Math.round(MAX_IMAGE_FILE_SIZE / (1024 * 1024))}MB.`) + ]); + } + + const imageData = await this.fileSystemService.readFile(uri, true); + return new LanguageModelToolResult([ + LanguageModelDataPart.image(imageData, imageMimeType), + ]); + } + + async prepareInvocation(options: vscode.LanguageModelToolInvocationPrepareOptions, _token: vscode.CancellationToken): Promise { + const uri = resolveToolInputPath(options.input.filePath, this.promptPathRepresentationService); + this.assertImageFile(uri); + + const isExternal = await this.instantiationService.invokeFunction( + accessor => isFileExternalAndNeedsConfirmation(accessor, uri, this._promptContext, { readOnly: true }) + ); + + if (isExternal) { + await this.instantiationService.invokeFunction( + accessor => assertFileNotContentExcluded(accessor, uri) + ); + + const folderUri = dirname(uri); + const message = this.workspaceService.getWorkspaceFolders().length === 1 + ? new MarkdownString(l10n.t`${formatUriForFileWidget(uri)} is outside of the current folder in ${formatUriForFileWidget(folderUri)}.`) + : new MarkdownString(l10n.t`${formatUriForFileWidget(uri)} is outside of the current workspace in ${formatUriForFileWidget(folderUri)}.`); + + return { + invocationMessage: new MarkdownString(l10n.t`Viewing image ${formatUriForFileWidget(uri)}`), + pastTenseMessage: new MarkdownString(l10n.t`Viewed image ${formatUriForFileWidget(uri)}`), + confirmationMessages: { + title: l10n.t`Allow viewing external images?`, + message, + } + }; + } + + await this.instantiationService.invokeFunction(accessor => assertFileOkForTool(accessor, uri, this._promptContext, { readOnly: true })); + + return { + invocationMessage: new MarkdownString(l10n.t`Viewing image ${formatUriForFileWidget(uri)}`), + pastTenseMessage: new MarkdownString(l10n.t`Viewed image ${formatUriForFileWidget(uri)}`), + }; + } + + async resolveInput(input: IViewImageParams, promptContext: IBuildPromptContext): Promise { + this._promptContext = promptContext; + return input; + } + + private assertImageFile(uri: URI): void { + if (!getImageMimeType(uri)) { + throw new Error(`Cannot view ${this.promptPathRepresentationService.getFilePath(uri)} with ${ToolName.ViewImage}. Use ${ToolName.ReadFile} for non-image files.`); + } + } +} + +ToolRegistry.registerTool(ViewImageTool);