Split image viewing out of read file (#4394)

* Add dedicated view image tool (Written by Copilot)

* Remove stale read file test import (Written by Copilot)

* Add image extension coverage test (Written by Copilot)

* Update test snapshots
This commit is contained in:
Rob Lourens
2026-03-12 23:01:55 -07:00
committed by GitHub
parent 812a77a816
commit cc86f3c7a3
18 changed files with 345 additions and 215 deletions
+20
View File
@@ -375,6 +375,25 @@
]
}
},
{
"name": "copilot_viewImage",
"toolReferenceName": "viewImage",
"displayName": "%copilot.tools.viewImage.name%",
"userDescription": "%copilot.tools.viewImage.userDescription%",
"modelDescription": "View the contents of an image file. Use this instead of read_file for supported image files such as png, jpg, jpeg, gif, and webp. The tool returns the image directly to multimodal models and does not take line ranges or offsets.",
"inputSchema": {
"type": "object",
"properties": {
"filePath": {
"description": "The absolute path of the image file to view.",
"type": "string"
}
},
"required": [
"filePath"
]
}
},
{
"name": "copilot_listDirectory",
"toolReferenceName": "listDirectory",
@@ -1180,6 +1199,7 @@
"getNotebookSummary",
"problems",
"readFile",
"viewImage",
"readNotebookCellOutput"
]
},
+2
View File
@@ -294,6 +294,8 @@
"copilot.tools.applyPatch.name": "Apply Patch",
"copilot.tools.readFile.name": "Read File",
"copilot.tools.readFile.userDescription": "Read the contents of a file",
"copilot.tools.viewImage.name": "View Image",
"copilot.tools.viewImage.userDescription": "View the contents of an image file",
"copilot.tools.listDirectory.name": "List Dir",
"copilot.tools.listDirectory.userDescription": "List the contents of a directory",
"copilot.tools.getTaskOutput.name": "Get Task Output",
@@ -91,6 +91,7 @@ search_workspace_symbols
test_failure
test_search
tool_replay
view_image
</availableDeferredTools>
</toolSearchInstructions>
@@ -90,6 +90,7 @@ search_workspace_symbols
test_failure
test_search
tool_replay
view_image
</availableDeferredTools>
</toolSearchInstructions>
@@ -123,6 +123,7 @@ search_workspace_symbols
test_failure
test_search
tool_replay
view_image
</availableDeferredTools>
</toolSearchInstructions>
@@ -122,6 +122,7 @@ search_workspace_symbols
test_failure
test_search
tool_replay
view_image
</availableDeferredTools>
</toolSearchInstructions>
@@ -123,6 +123,7 @@ search_workspace_symbols
test_failure
test_search
tool_replay
view_image
</availableDeferredTools>
</toolSearchInstructions>
@@ -122,6 +122,7 @@ search_workspace_symbols
test_failure
test_search
tool_replay
view_image
</availableDeferredTools>
</toolSearchInstructions>
@@ -91,6 +91,7 @@ search_workspace_symbols
test_failure
test_search
tool_replay
view_image
</availableDeferredTools>
</toolSearchInstructions>
@@ -90,6 +90,7 @@ search_workspace_symbols
test_failure
test_search
tool_replay
view_image
</availableDeferredTools>
</toolSearchInstructions>
@@ -26,6 +26,7 @@ export enum ToolName {
FindFiles = 'file_search',
FindTextInFiles = 'grep_search',
ReadFile = 'read_file',
ViewImage = 'view_image',
ListDirectory = 'list_dir',
GetErrors = 'get_errors',
GetScmChanges = 'get_changed_files',
@@ -82,6 +83,7 @@ export enum ContributedToolName {
FindFiles = 'copilot_findFiles',
FindTextInFiles = 'copilot_findTextInFiles',
ReadFile = 'copilot_readFile',
ViewImage = 'copilot_viewImage',
ListDirectory = 'copilot_listDirectory',
GetErrors = 'copilot_getErrors',
GetScmChanges = 'copilot_getChangedFiles',
@@ -158,6 +160,7 @@ export const toolCategories: Record<ToolName, ToolCategory> = {
[ToolName.Codebase]: ToolCategory.Core,
[ToolName.FindTextInFiles]: ToolCategory.Core,
[ToolName.ReadFile]: ToolCategory.Core,
[ToolName.ViewImage]: ToolCategory.Core,
[ToolName.CreateFile]: ToolCategory.Core,
[ToolName.ApplyPatch]: ToolCategory.Core,
[ToolName.ReplaceString]: ToolCategory.Core,
@@ -35,6 +35,6 @@ import './searchWorkspaceSymbolsTool';
import './testFailureTool';
import './toolReplayTool';
import './toolSearchTool';
import './viewImageTool';
import './vscodeAPITool';
import './vscodeCmdTool';
@@ -0,0 +1,29 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import { URI } from '../../../util/vs/base/common/uri';
import { ChatImageMimeType } from '../../conversation/common/languageModelChatMessageHelpers';
/** Maximum image file size in bytes (20 MB) */
export const MAX_IMAGE_FILE_SIZE = 20 * 1024 * 1024;
const imageExtensionToMimeType: Record<string, ChatImageMimeType> = {
'.png': ChatImageMimeType.PNG,
'.jpg': ChatImageMimeType.JPEG,
'.jpeg': ChatImageMimeType.JPEG,
'.gif': ChatImageMimeType.GIF,
'.webp': ChatImageMimeType.WEBP,
};
export function getImageMimeType(uri: URI): ChatImageMimeType | undefined {
const path = uri.path.toLowerCase();
for (const [ext, mime] of Object.entries(imageExtensionToMimeType)) {
if (path.endsWith(ext)) {
return mime;
}
}
return undefined;
}
@@ -24,8 +24,7 @@ import { clamp } from '../../../util/vs/base/common/numbers';
import { dirname, extUriBiasedIgnorePathCase } from '../../../util/vs/base/common/resources';
import { URI } from '../../../util/vs/base/common/uri';
import { IInstantiationService } from '../../../util/vs/platform/instantiation/common/instantiation';
import { LanguageModelDataPart, LanguageModelPromptTsxPart, LanguageModelTextPart, LanguageModelToolResult, Location, MarkdownString, Range } from '../../../vscodeTypes';
import { ChatImageMimeType } from '../../conversation/common/languageModelChatMessageHelpers';
import { LanguageModelPromptTsxPart, LanguageModelToolResult, Location, MarkdownString, Range } from '../../../vscodeTypes';
import { IBuildPromptContext } from '../../prompt/common/intents';
import { renderPromptElementJSON } from '../../prompts/node/base/promptRenderer';
import { BinaryFileHexdump, hexdumpIfBinary } from '../../prompts/node/panel/binaryFileHexdump';
@@ -33,11 +32,12 @@ import { CodeBlock } from '../../prompts/node/panel/safeElements';
import { ToolName } from '../common/toolNames';
import { ICopilotTool, ToolRegistry } from '../common/toolsRegistry';
import { formatUriForFileWidget } from '../common/toolUtils';
import { getImageMimeType } from './imageToolUtils';
import { assertFileNotContentExcluded, assertFileOkForTool, isFileExternalAndNeedsConfirmation, resolveToolInputPath } from './toolUtils';
export const readFileV2Description: vscode.LanguageModelToolInformation = {
name: ToolName.ReadFile,
description: 'Read the contents of a file. Line numbers are 1-indexed. This tool will truncate its output at 2000 lines and may be called repeatedly with offset and limit parameters to read larger files in chunks. For image files (png, jpg, jpeg, gif, webp), the full image content will be returned. Binary files use offset/limit as byte offsets.',
description: 'Read the contents of a file. Line numbers are 1-indexed. This tool will truncate its output at 2000 lines and may be called repeatedly with offset and limit parameters to read larger files in chunks. Binary files use offset/limit as byte offsets.',
tags: ['vscode_codesearch'],
source: undefined,
inputSchema: {
@@ -74,27 +74,6 @@ export interface IReadFileParamsV2 {
const MAX_LINES_PER_READ = 2000;
/** Maximum image file size in bytes (20 MB) */
const MAX_IMAGE_FILE_SIZE = 20 * 1024 * 1024;
const imageExtensionToMimeType: Record<string, ChatImageMimeType> = {
'.png': ChatImageMimeType.PNG,
'.jpg': ChatImageMimeType.JPEG,
'.jpeg': ChatImageMimeType.JPEG,
'.gif': ChatImageMimeType.GIF,
'.webp': ChatImageMimeType.WEBP,
};
function getImageMimeType(uri: URI): ChatImageMimeType | undefined {
const path = uri.path.toLowerCase();
for (const [ext, mime] of Object.entries(imageExtensionToMimeType)) {
if (path.endsWith(ext)) {
return mime;
}
}
return undefined;
}
export type ReadFileParams = IReadFileParamsV1 | IReadFileParamsV2;
const isParamsV2 = (params: ReadFileParams): params is IReadFileParamsV2 =>
@@ -157,10 +136,8 @@ export class ReadFileTool implements ICopilotTool<ReadFileParams> {
try {
uri = resolveToolInputPath(options.input.filePath, this.promptPathRepresentationService);
// Handle image files
const imageMimeType = getImageMimeType(uri);
if (imageMimeType) {
return await this.invokeForImage(uri, imageMimeType, options);
if (getImageMimeType(uri)) {
throw new Error(`Cannot read image files with ${ToolName.ReadFile}. Use ${ToolName.ViewImage} instead.`);
}
// Handle binary files — read raw bytes and check for null bytes
@@ -223,30 +200,6 @@ export class ReadFileTool implements ICopilotTool<ReadFileParams> {
}
}
private async invokeForImage(uri: URI, imageMimeType: ChatImageMimeType, options: vscode.LanguageModelToolInvocationOptions<ReadFileParams>): Promise<LanguageModelToolResult> {
const input = options.input;
const hasLineParams = isParamsV2(input)
? (input.offset !== undefined || input.limit !== undefined)
: true;
if (hasLineParams) {
throw new Error(`Cannot specify line ranges when reading an image file. Use read_file with only the filePath parameter for image files.`);
}
const stat = await this.fileSystemService.stat(uri);
if (stat.size > MAX_IMAGE_FILE_SIZE) {
void this.sendReadFileTelemetry('imageTooLarge', options, { start: 0, end: 0, truncated: false }, uri);
return new LanguageModelToolResult([
new LanguageModelTextPart(`Cannot read image file ${this.promptPathRepresentationService.getFilePath(uri)}: file size (${Math.round(stat.size / (1024 * 1024))}MB) exceeds the maximum allowed size of ${Math.round(MAX_IMAGE_FILE_SIZE / (1024 * 1024))}MB.`)
]);
}
const imageData = await this.fileSystemService.readFile(uri, true);
void this.sendReadFileTelemetry('success', options, { start: 0, end: 0, truncated: false }, uri);
return new LanguageModelToolResult([
LanguageModelDataPart.image(imageData, imageMimeType),
]);
}
async prepareInvocation(options: vscode.LanguageModelToolInvocationPrepareOptions<ReadFileParams>, token: vscode.CancellationToken): Promise<vscode.PreparedToolInvocation | undefined> {
const { input } = options;
if (!input.filePath.length) {
@@ -257,7 +210,9 @@ export class ReadFileTool implements ICopilotTool<ReadFileParams> {
let documentSnapshot: NotebookDocumentSnapshot | TextDocumentSnapshot;
try {
uri = resolveToolInputPath(input.filePath, this.promptPathRepresentationService);
const isImage = !!getImageMimeType(uri);
if (getImageMimeType(uri)) {
throw new Error(`Cannot read image files with ${ToolName.ReadFile}. Use ${ToolName.ViewImage} instead.`);
}
// Check if file is external (outside workspace, not open in editor, etc.)
const isExternal = await this.instantiationService.invokeFunction(
@@ -274,14 +229,11 @@ export class ReadFileTool implements ICopilotTool<ReadFileParams> {
const message = this.workspaceService.getWorkspaceFolders().length === 1 ? new MarkdownString(l10n.t`${formatUriForFileWidget(uri)} is outside of the current folder in ${formatUriForFileWidget(folderUri)}.`) : new MarkdownString(l10n.t`${formatUriForFileWidget(uri)} is outside of the current workspace in ${formatUriForFileWidget(folderUri)}.`);
const readingLabel = isImage ? l10n.t`Reading image ${formatUriForFileWidget(uri)}` : l10n.t`Reading ${formatUriForFileWidget(uri)}`;
const readLabel = isImage ? l10n.t`Read image ${formatUriForFileWidget(uri)}` : l10n.t`Read ${formatUriForFileWidget(uri)}`;
// Return confirmation request for external file
// The folder-based "allow this session" option is provided by the core confirmation contribution
return {
invocationMessage: new MarkdownString(readingLabel),
pastTenseMessage: new MarkdownString(readLabel),
invocationMessage: new MarkdownString(l10n.t`Reading ${formatUriForFileWidget(uri)}`),
pastTenseMessage: new MarkdownString(l10n.t`Read ${formatUriForFileWidget(uri)}`),
confirmationMessages: {
title: l10n.t`Allow reading external files?`,
message,
@@ -291,14 +243,6 @@ export class ReadFileTool implements ICopilotTool<ReadFileParams> {
await this.instantiationService.invokeFunction(accessor => assertFileOkForTool(accessor, uri!, this._promptContext, { readOnly: true }));
// For images, we're done - no snapshot/range logic needed
if (isImage) {
return {
invocationMessage: new MarkdownString(l10n.t`Reading image ${formatUriForFileWidget(uri)}`),
pastTenseMessage: new MarkdownString(l10n.t`Read image ${formatUriForFileWidget(uri)}`),
};
}
try {
documentSnapshot = await this.getSnapshot(uri);
} catch (e) {
@@ -0,0 +1,31 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import { expect, suite, test } from 'vitest';
import { URI } from '../../../../util/vs/base/common/uri';
import { ChatImageMimeType } from '../../../conversation/common/languageModelChatMessageHelpers';
import { getImageMimeType } from '../imageToolUtils';
suite('imageToolUtils', () => {
test('recognizes all supported image extensions', () => {
expect([
getImageMimeType(URI.file('/workspace/image.png')),
getImageMimeType(URI.file('/workspace/image.jpg')),
getImageMimeType(URI.file('/workspace/image.jpeg')),
getImageMimeType(URI.file('/workspace/image.gif')),
getImageMimeType(URI.file('/workspace/image.webp')),
]).toEqual([
ChatImageMimeType.PNG,
ChatImageMimeType.JPEG,
ChatImageMimeType.JPEG,
ChatImageMimeType.GIF,
ChatImageMimeType.WEBP,
]);
});
test('does not recognize unsupported extensions as images', () => {
expect(getImageMimeType(URI.file('/workspace/image.bmp'))).toBeUndefined();
});
});
@@ -17,7 +17,7 @@ import { CancellationToken } from '../../../../util/vs/base/common/cancellation'
import { URI } from '../../../../util/vs/base/common/uri';
import { SyncDescriptor } from '../../../../util/vs/platform/instantiation/common/descriptors';
import { IInstantiationService } from '../../../../util/vs/platform/instantiation/common/instantiation';
import { LanguageModelDataPart, MarkdownString } from '../../../../vscodeTypes';
import { MarkdownString } from '../../../../vscodeTypes';
import { createExtensionUnitTestingServices } from '../../../test/node/services';
import { ToolName } from '../../common/toolNames';
import { IToolsService } from '../../common/toolsService';
@@ -492,7 +492,7 @@ suite('ReadFile', () => {
});
suite('image files', () => {
test('returns image data for image file', async () => {
test('throws for image files and points to view_image', async () => {
const services = createExtensionUnitTestingServices();
const mockFs = new MockFileSystemService();
mockFs.mockFile(URI.file('/workspace/photo.jpg'), 'fake-image-bytes');
@@ -506,20 +506,15 @@ suite('ReadFile', () => {
const readFileTool = testAccessor.get(IInstantiationService).createInstance(ReadFileTool);
const input: IReadFileParamsV2 = { filePath: '/workspace/photo.jpg' };
const result = await readFileTool.invoke(
await expect(readFileTool.invoke(
{ input, toolInvocationToken: null as never },
CancellationToken.None
);
// The result should contain a LanguageModelDataPart with image data
const imagePart = result.content.find(part => part instanceof LanguageModelDataPart);
expect(imagePart).toBeDefined();
expect((imagePart as LanguageModelDataPart).mimeType).toBe('image/jpeg');
)).rejects.toThrow('Use view_image instead');
testAccessor.dispose();
});
test('throws when reading image with offset/limit params', async () => {
test('prepareInvocation throws for image files and points to view_image', async () => {
const services = createExtensionUnitTestingServices();
const mockFs = new MockFileSystemService();
mockFs.mockFile(URI.file('/workspace/photo.png'), 'fake-image-bytes');
@@ -532,146 +527,11 @@ suite('ReadFile', () => {
const testAccessor = services.createTestingAccessor();
const readFileTool = testAccessor.get(IInstantiationService).createInstance(ReadFileTool);
const input: IReadFileParamsV2 = { filePath: '/workspace/photo.png', offset: 1, limit: 10 };
await expect(readFileTool.invoke(
{ input, toolInvocationToken: null as never },
CancellationToken.None
)).rejects.toThrow('Cannot specify line ranges when reading an image file');
testAccessor.dispose();
});
test('throws when reading image with v1 params', async () => {
const services = createExtensionUnitTestingServices();
const mockFs = new MockFileSystemService();
mockFs.mockFile(URI.file('/workspace/photo.png'), 'fake-image-bytes');
services.define(IFileSystemService, mockFs);
services.define(IWorkspaceService, new SyncDescriptor(
TestWorkspaceService,
[[URI.file('/workspace')], []]
));
const testAccessor = services.createTestingAccessor();
const readFileTool = testAccessor.get(IInstantiationService).createInstance(ReadFileTool);
const input: IReadFileParamsV1 = { filePath: '/workspace/photo.png', startLine: 1, endLine: 5 };
await expect(readFileTool.invoke(
{ input, toolInvocationToken: null as never },
CancellationToken.None
)).rejects.toThrow('Cannot specify line ranges when reading an image file');
testAccessor.dispose();
});
test('returns error for oversized image files', async () => {
const services = createExtensionUnitTestingServices();
const mockFs = new class extends MockFileSystemService {
override async stat(resource: URI) {
const result = await super.stat(resource);
if (resource.toString() === URI.file('/workspace/huge.png').toString()) {
return { ...result, size: 21 * 1024 * 1024 };
}
return result;
}
}();
// Create a small mock file whose stat reports a size over the 20MB limit
mockFs.mockFile(URI.file('/workspace/huge.png'), 'fake-image-bytes');
services.define(IFileSystemService, mockFs);
services.define(IWorkspaceService, new SyncDescriptor(
TestWorkspaceService,
[[URI.file('/workspace')], []]
));
const testAccessor = services.createTestingAccessor();
const readFileTool = testAccessor.get(IInstantiationService).createInstance(ReadFileTool);
const input: IReadFileParamsV2 = { filePath: '/workspace/huge.png' };
const result = await readFileTool.invoke(
{ input, toolInvocationToken: null as never },
CancellationToken.None
);
const text = await toolResultToString(testAccessor, result);
expect(text).toContain('exceeds the maximum allowed size');
testAccessor.dispose();
});
test('prepareInvocation returns image-specific messages', async () => {
const services = createExtensionUnitTestingServices();
const mockFs = new MockFileSystemService();
mockFs.mockFile(URI.file('/workspace/icon.png'), 'fake-image-data');
services.define(IFileSystemService, mockFs);
services.define(IWorkspaceService, new SyncDescriptor(
TestWorkspaceService,
[[URI.file('/workspace')], []]
));
const testAccessor = services.createTestingAccessor();
const readFileTool = testAccessor.get(IInstantiationService).createInstance(ReadFileTool);
const input: IReadFileParamsV2 = { filePath: '/workspace/icon.png' };
const result = await readFileTool.prepareInvocation(
const input: IReadFileParamsV2 = { filePath: '/workspace/photo.png' };
await expect(readFileTool.prepareInvocation(
{ input },
CancellationToken.None
);
expect(result).toBeDefined();
expect((result!.invocationMessage as MarkdownString).value).toContain('Reading image');
expect((result!.pastTenseMessage as MarkdownString).value).toContain('Read image');
testAccessor.dispose();
});
test('recognizes all supported image extensions', async () => {
const services = createExtensionUnitTestingServices();
const mockFs = new MockFileSystemService();
for (const ext of ['.png', '.jpg', '.jpeg', '.gif', '.webp']) {
mockFs.mockFile(URI.file(`/workspace/image${ext}`), 'data');
}
services.define(IFileSystemService, mockFs);
services.define(IWorkspaceService, new SyncDescriptor(
TestWorkspaceService,
[[URI.file('/workspace')], []]
));
const testAccessor = services.createTestingAccessor();
const readFileTool = testAccessor.get(IInstantiationService).createInstance(ReadFileTool);
for (const ext of ['.png', '.jpg', '.jpeg', '.gif', '.webp']) {
const input: IReadFileParamsV2 = { filePath: `/workspace/image${ext}` };
const result = await readFileTool.prepareInvocation(
{ input },
CancellationToken.None
);
expect(result).toBeDefined();
expect((result!.invocationMessage as MarkdownString).value).toContain('Reading image');
}
testAccessor.dispose();
});
test('does not treat unsupported extensions as images', async () => {
const testDoc = createTextDocumentData(URI.file('/workspace/image.bmp'), 'not an image', 'plaintext').document;
const services = createExtensionUnitTestingServices();
services.define(IWorkspaceService, new SyncDescriptor(
TestWorkspaceService,
[[URI.file('/workspace')], [testDoc]]
));
const testAccessor = services.createTestingAccessor();
const readFileTool = testAccessor.get(IInstantiationService).createInstance(ReadFileTool);
const input: IReadFileParamsV2 = { filePath: '/workspace/image.bmp' };
const result = await readFileTool.prepareInvocation(
{ input },
CancellationToken.None
);
expect(result).toBeDefined();
// Should be a normal "Reading" message, not "Reading image"
expect((result!.invocationMessage as MarkdownString).value).not.toContain('Reading image');
)).rejects.toThrow('Use view_image instead');
testAccessor.dispose();
});
@@ -0,0 +1,127 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import { expect, suite, test } from 'vitest';
import { IFileSystemService } from '../../../../platform/filesystem/common/fileSystemService';
import { MockFileSystemService } from '../../../../platform/filesystem/node/test/mockFileSystemService';
import { TestWorkspaceService } from '../../../../platform/test/node/testWorkspaceService';
import { IWorkspaceService } from '../../../../platform/workspace/common/workspaceService';
import { createTextDocumentData } from '../../../../util/common/test/shims/textDocument';
import { CancellationToken } from '../../../../util/vs/base/common/cancellation';
import { URI } from '../../../../util/vs/base/common/uri';
import { SyncDescriptor } from '../../../../util/vs/platform/instantiation/common/descriptors';
import { IInstantiationService } from '../../../../util/vs/platform/instantiation/common/instantiation';
import { LanguageModelDataPart, MarkdownString } from '../../../../vscodeTypes';
import { createExtensionUnitTestingServices } from '../../../test/node/services';
import { IViewImageParams, ViewImageTool } from '../viewImageTool';
import { toolResultToString } from './toolTestUtils';
suite('ViewImage', () => {
test('returns image data for image file', async () => {
const services = createExtensionUnitTestingServices();
const mockFs = new MockFileSystemService();
mockFs.mockFile(URI.file('/workspace/photo.jpg'), 'fake-image-bytes');
services.define(IFileSystemService, mockFs);
services.define(IWorkspaceService, new SyncDescriptor(
TestWorkspaceService,
[[URI.file('/workspace')], []]
));
const accessor = services.createTestingAccessor();
const viewImageTool = accessor.get(IInstantiationService).createInstance(ViewImageTool);
const input: IViewImageParams = { filePath: '/workspace/photo.jpg' };
const result = await viewImageTool.invoke(
{ input, toolInvocationToken: null as never },
CancellationToken.None
);
const imagePart = result.content.find(part => part instanceof LanguageModelDataPart);
expect(imagePart).toBeDefined();
expect((imagePart as LanguageModelDataPart).mimeType).toBe('image/jpeg');
accessor.dispose();
});
test('returns error for oversized image files', async () => {
const services = createExtensionUnitTestingServices();
const mockFs = new class extends MockFileSystemService {
override async stat(resource: URI) {
const result = await super.stat(resource);
if (resource.toString() === URI.file('/workspace/huge.png').toString()) {
return { ...result, size: 21 * 1024 * 1024 };
}
return result;
}
}();
mockFs.mockFile(URI.file('/workspace/huge.png'), 'fake-image-bytes');
services.define(IFileSystemService, mockFs);
services.define(IWorkspaceService, new SyncDescriptor(
TestWorkspaceService,
[[URI.file('/workspace')], []]
));
const accessor = services.createTestingAccessor();
const viewImageTool = accessor.get(IInstantiationService).createInstance(ViewImageTool);
const input: IViewImageParams = { filePath: '/workspace/huge.png' };
const result = await viewImageTool.invoke(
{ input, toolInvocationToken: null as never },
CancellationToken.None
);
const text = await toolResultToString(accessor, result);
expect(text).toContain('exceeds the maximum allowed size');
accessor.dispose();
});
test('prepareInvocation returns image-specific messages', async () => {
const services = createExtensionUnitTestingServices();
const mockFs = new MockFileSystemService();
mockFs.mockFile(URI.file('/workspace/icon.png'), 'fake-image-data');
services.define(IFileSystemService, mockFs);
services.define(IWorkspaceService, new SyncDescriptor(
TestWorkspaceService,
[[URI.file('/workspace')], []]
));
const accessor = services.createTestingAccessor();
const viewImageTool = accessor.get(IInstantiationService).createInstance(ViewImageTool);
const input: IViewImageParams = { filePath: '/workspace/icon.png' };
const result = await viewImageTool.prepareInvocation(
{ input },
CancellationToken.None
);
expect(result).toBeDefined();
expect((result!.invocationMessage as MarkdownString).value).toContain('Viewing image');
expect((result!.pastTenseMessage as MarkdownString).value).toContain('Viewed image');
accessor.dispose();
});
test('throws for non-image files and points to read_file', async () => {
const document = createTextDocumentData(URI.file('/workspace/file.ts'), 'const x = 1;', 'typescript').document;
const services = createExtensionUnitTestingServices();
services.define(IWorkspaceService, new SyncDescriptor(
TestWorkspaceService,
[[URI.file('/workspace')], [document]]
));
const accessor = services.createTestingAccessor();
const viewImageTool = accessor.get(IInstantiationService).createInstance(ViewImageTool);
const input: IViewImageParams = { filePath: '/workspace/file.ts' };
await expect(viewImageTool.prepareInvocation(
{ input },
CancellationToken.None
)).rejects.toThrow('Use read_file for non-image files');
accessor.dispose();
});
});
@@ -0,0 +1,106 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import * as l10n from '@vscode/l10n';
import type * as vscode from 'vscode';
import { IFileSystemService } from '../../../platform/filesystem/common/fileSystemService';
import { IPromptPathRepresentationService } from '../../../platform/prompts/common/promptPathRepresentationService';
import { IWorkspaceService } from '../../../platform/workspace/common/workspaceService';
import { dirname } from '../../../util/vs/base/common/resources';
import { URI } from '../../../util/vs/base/common/uri';
import { IInstantiationService } from '../../../util/vs/platform/instantiation/common/instantiation';
import { LanguageModelDataPart, LanguageModelTextPart, LanguageModelToolResult, MarkdownString } from '../../../vscodeTypes';
import { IBuildPromptContext } from '../../prompt/common/intents';
import { ToolName } from '../common/toolNames';
import { ICopilotTool, ToolRegistry } from '../common/toolsRegistry';
import { formatUriForFileWidget } from '../common/toolUtils';
import { getImageMimeType, MAX_IMAGE_FILE_SIZE } from './imageToolUtils';
import { assertFileNotContentExcluded, assertFileOkForTool, isFileExternalAndNeedsConfirmation, resolveToolInputPath } from './toolUtils';
export interface IViewImageParams {
filePath: string;
}
export class ViewImageTool implements ICopilotTool<IViewImageParams> {
public static readonly toolName = ToolName.ViewImage;
private _promptContext: IBuildPromptContext | undefined;
constructor(
@IWorkspaceService private readonly workspaceService: IWorkspaceService,
@IPromptPathRepresentationService private readonly promptPathRepresentationService: IPromptPathRepresentationService,
@IInstantiationService private readonly instantiationService: IInstantiationService,
@IFileSystemService private readonly fileSystemService: IFileSystemService,
) { }
async invoke(options: vscode.LanguageModelToolInvocationOptions<IViewImageParams>, _token: vscode.CancellationToken): Promise<LanguageModelToolResult> {
const uri = resolveToolInputPath(options.input.filePath, this.promptPathRepresentationService);
const imageMimeType = getImageMimeType(uri);
if (!imageMimeType) {
throw new Error(`Cannot view ${this.promptPathRepresentationService.getFilePath(uri)} with ${ToolName.ViewImage}. Use ${ToolName.ReadFile} for non-image files.`);
}
const stat = await this.fileSystemService.stat(uri);
if (stat.size > MAX_IMAGE_FILE_SIZE) {
return new LanguageModelToolResult([
new LanguageModelTextPart(`Cannot view image file ${this.promptPathRepresentationService.getFilePath(uri)}: file size (${Math.round(stat.size / (1024 * 1024))}MB) exceeds the maximum allowed size of ${Math.round(MAX_IMAGE_FILE_SIZE / (1024 * 1024))}MB.`)
]);
}
const imageData = await this.fileSystemService.readFile(uri, true);
return new LanguageModelToolResult([
LanguageModelDataPart.image(imageData, imageMimeType),
]);
}
async prepareInvocation(options: vscode.LanguageModelToolInvocationPrepareOptions<IViewImageParams>, _token: vscode.CancellationToken): Promise<vscode.PreparedToolInvocation | undefined> {
const uri = resolveToolInputPath(options.input.filePath, this.promptPathRepresentationService);
this.assertImageFile(uri);
const isExternal = await this.instantiationService.invokeFunction(
accessor => isFileExternalAndNeedsConfirmation(accessor, uri, this._promptContext, { readOnly: true })
);
if (isExternal) {
await this.instantiationService.invokeFunction(
accessor => assertFileNotContentExcluded(accessor, uri)
);
const folderUri = dirname(uri);
const message = this.workspaceService.getWorkspaceFolders().length === 1
? new MarkdownString(l10n.t`${formatUriForFileWidget(uri)} is outside of the current folder in ${formatUriForFileWidget(folderUri)}.`)
: new MarkdownString(l10n.t`${formatUriForFileWidget(uri)} is outside of the current workspace in ${formatUriForFileWidget(folderUri)}.`);
return {
invocationMessage: new MarkdownString(l10n.t`Viewing image ${formatUriForFileWidget(uri)}`),
pastTenseMessage: new MarkdownString(l10n.t`Viewed image ${formatUriForFileWidget(uri)}`),
confirmationMessages: {
title: l10n.t`Allow viewing external images?`,
message,
}
};
}
await this.instantiationService.invokeFunction(accessor => assertFileOkForTool(accessor, uri, this._promptContext, { readOnly: true }));
return {
invocationMessage: new MarkdownString(l10n.t`Viewing image ${formatUriForFileWidget(uri)}`),
pastTenseMessage: new MarkdownString(l10n.t`Viewed image ${formatUriForFileWidget(uri)}`),
};
}
async resolveInput(input: IViewImageParams, promptContext: IBuildPromptContext): Promise<IViewImageParams> {
this._promptContext = promptContext;
return input;
}
private assertImageFile(uri: URI): void {
if (!getImageMimeType(uri)) {
throw new Error(`Cannot view ${this.promptPathRepresentationService.getFilePath(uri)} with ${ToolName.ViewImage}. Use ${ToolName.ReadFile} for non-image files.`);
}
}
}
ToolRegistry.registerTool(ViewImageTool);