diff --git a/src/vs/base/common/marshallingIds.ts b/src/vs/base/common/marshallingIds.ts index 9ea80910eeb..272aa3594e5 100644 --- a/src/vs/base/common/marshallingIds.ts +++ b/src/vs/base/common/marshallingIds.ts @@ -27,5 +27,4 @@ export const enum MarshalledId { LanguageModelTextPart, LanguageModelPromptTsxPart, LanguageModelDataPart, - LanguageModelExtraDataPart, } diff --git a/src/vs/platform/extensions/common/extensionsApiProposals.ts b/src/vs/platform/extensions/common/extensionsApiProposals.ts index bef924d6d9b..c43841d4784 100644 --- a/src/vs/platform/extensions/common/extensionsApiProposals.ts +++ b/src/vs/platform/extensions/common/extensionsApiProposals.ts @@ -231,7 +231,7 @@ const _allApiProposals = { }, languageModelDataPart: { proposal: 'https://raw.githubusercontent.com/microsoft/vscode/main/src/vscode-dts/vscode.proposed.languageModelDataPart.d.ts', - version: 2 + version: 3 }, languageModelSystem: { proposal: 'https://raw.githubusercontent.com/microsoft/vscode/main/src/vscode-dts/vscode.proposed.languageModelSystem.d.ts', diff --git a/src/vs/workbench/api/common/extHost.api.impl.ts b/src/vs/workbench/api/common/extHost.api.impl.ts index 71f70cb8489..9b9274fdfd6 100644 --- a/src/vs/workbench/api/common/extHost.api.impl.ts +++ b/src/vs/workbench/api/common/extHost.api.impl.ts @@ -1828,7 +1828,6 @@ export function createApiFactoryAndRegisterActors(accessor: ServicesAccessor): I LanguageModelToolResult: extHostTypes.LanguageModelToolResult, LanguageModelToolResult2: extHostTypes.LanguageModelToolResult2, LanguageModelDataPart: extHostTypes.LanguageModelDataPart, - LanguageModelExtraDataPart: extHostTypes.LanguageModelExtraDataPart, ExtendedLanguageModelToolResult: extHostTypes.ExtendedLanguageModelToolResult, PreparedTerminalToolInvocation: extHostTypes.PreparedTerminalToolInvocation, LanguageModelChatToolMode: extHostTypes.LanguageModelChatToolMode, diff --git a/src/vs/workbench/api/common/extHostTypeConverters.ts b/src/vs/workbench/api/common/extHostTypeConverters.ts index 0a8dfb99651..6a964ad5d4b 100644 --- a/src/vs/workbench/api/common/extHostTypeConverters.ts +++ b/src/vs/workbench/api/common/extHostTypeConverters.ts @@ -45,7 +45,7 @@ import { IChatRequestVariableEntry, isImageVariableEntry } from '../../contrib/c import { IChatAgentMarkdownContentWithVulnerability, IChatCodeCitation, IChatCommandButton, IChatConfirmation, IChatContentInlineReference, IChatContentReference, IChatExtensionsContent, IChatFollowup, IChatMarkdownContent, IChatMoveMessage, IChatProgressMessage, IChatResponseCodeblockUriPart, IChatTaskDto, IChatTaskResult, IChatTextEdit, IChatTreeData, IChatUserActionEvent, IChatWarningMessage } from '../../contrib/chat/common/chatService.js'; import { IToolData, IToolResult } from '../../contrib/chat/common/languageModelToolsService.js'; import * as chatProvider from '../../contrib/chat/common/languageModels.js'; -import { IChatResponseDataPart, IChatResponsePromptTsxPart, IChatResponseTextPart } from '../../contrib/chat/common/languageModels.js'; +import { IChatMessageDataPart, IChatResponseDataPart, IChatResponsePromptTsxPart, IChatResponseTextPart } from '../../contrib/chat/common/languageModels.js'; import { DebugTreeItemCollapsibleState, IDebugVisualizationTreeItem } from '../../contrib/debug/common/debug.js'; import * as notebooks from '../../contrib/notebook/common/notebookCommon.js'; import { CellEditType } from '../../contrib/notebook/common/notebookCommon.js'; @@ -2329,12 +2329,14 @@ export namespace LanguageModelChatMessage { } }); return new types.LanguageModelToolResultPart(c.toolCallId, content, c.isError); - } else if (c.type === 'image_url' || c.type === 'extra_data') { + } else if (c.type === 'image_url') { // Non-stable types return undefined; - } else { + } else if (c.type === 'tool_use') { return new types.LanguageModelToolCallPart(c.toolCallId, c.name, c.parameters); } + + return undefined; }).filter(c => c !== undefined); const role = LanguageModelChatMessageRole.to(message.role); @@ -2426,8 +2428,8 @@ export namespace LanguageModelChatMessage2 { return new types.LanguageModelToolResultPart2(c.toolCallId, content, c.isError); } else if (c.type === 'image_url') { return new types.LanguageModelDataPart(c.value.data.buffer, c.value.mimeType); - } else if (c.type === 'extra_data') { - return new types.LanguageModelExtraDataPart(c.kind, c.data); + } else if (c.type === 'data') { + return new types.LanguageModelDataPart(c.data.buffer, c.mimeType); } else { return new types.LanguageModelToolCallPart(c.toolCallId, c.name, c.parameters); } @@ -2479,15 +2481,23 @@ export namespace LanguageModelChatMessage2 { isError: c.isError }; } else if (c instanceof types.LanguageModelDataPart) { - const value: chatProvider.IChatImageURLPart = { - mimeType: c.mimeType as chatProvider.ChatImageMimeType, - data: VSBuffer.wrap(c.data), - }; + if (isImageDataPart(c)) { + const value: chatProvider.IChatImageURLPart = { + mimeType: c.mimeType as chatProvider.ChatImageMimeType, + data: VSBuffer.wrap(c.data), + }; - return { - type: 'image_url', - value: value - }; + return { + type: 'image_url', + value: value + }; + } else { + return { + type: 'data', + mimeType: c.mimeType, + data: VSBuffer.wrap(c.data), + } satisfies IChatMessageDataPart; + } } else if (c instanceof types.LanguageModelToolCallPart) { return { type: 'tool_use', @@ -2500,12 +2510,6 @@ export namespace LanguageModelChatMessage2 { type: 'text', value: c.value }; - } else if (c instanceof types.LanguageModelExtraDataPart) { - return { - type: 'extra_data', - kind: c.kind, - data: c.data - } satisfies chatProvider.IChatMessagePart; } else { if (typeof c !== 'string') { throw new Error('Unexpected chat message content type llm 2'); @@ -2526,6 +2530,19 @@ export namespace LanguageModelChatMessage2 { } } +function isImageDataPart(part: types.LanguageModelDataPart): boolean { + switch (part.mimeType) { + case types.ChatImageMimeType.PNG: + case types.ChatImageMimeType.JPEG: + case types.ChatImageMimeType.GIF: + case types.ChatImageMimeType.WEBP: + case types.ChatImageMimeType.BMP: + return true; + default: + return false; + } +} + export namespace ChatResponseMarkdownPart { export function from(part: vscode.ChatResponseMarkdownPart): Dto { return { diff --git a/src/vs/workbench/api/common/extHostTypes.ts b/src/vs/workbench/api/common/extHostTypes.ts index 4590e039171..37e3fa81713 100644 --- a/src/vs/workbench/api/common/extHostTypes.ts +++ b/src/vs/workbench/api/common/extHostTypes.ts @@ -4872,19 +4872,19 @@ export class LanguageModelChatMessage implements vscode.LanguageModelChatMessage export class LanguageModelChatMessage2 implements vscode.LanguageModelChatMessage2 { - static User(content: string | (LanguageModelTextPart | LanguageModelToolResultPart2 | LanguageModelToolCallPart | LanguageModelDataPart | LanguageModelExtraDataPart)[], name?: string): LanguageModelChatMessage2 { + static User(content: string | (LanguageModelTextPart | LanguageModelToolResultPart2 | LanguageModelToolCallPart | LanguageModelDataPart)[], name?: string): LanguageModelChatMessage2 { return new LanguageModelChatMessage2(LanguageModelChatMessageRole.User, content, name); } - static Assistant(content: string | (LanguageModelTextPart | LanguageModelToolResultPart2 | LanguageModelToolCallPart | LanguageModelDataPart | LanguageModelExtraDataPart)[], name?: string): LanguageModelChatMessage2 { + static Assistant(content: string | (LanguageModelTextPart | LanguageModelToolResultPart2 | LanguageModelToolCallPart | LanguageModelDataPart)[], name?: string): LanguageModelChatMessage2 { return new LanguageModelChatMessage2(LanguageModelChatMessageRole.Assistant, content, name); } role: vscode.LanguageModelChatMessageRole; - private _content: (LanguageModelTextPart | LanguageModelToolResultPart2 | LanguageModelToolCallPart | LanguageModelDataPart | LanguageModelExtraDataPart)[] = []; + private _content: (LanguageModelTextPart | LanguageModelToolResultPart2 | LanguageModelToolCallPart | LanguageModelDataPart)[] = []; - set content(value: string | (LanguageModelTextPart | LanguageModelToolResultPart2 | LanguageModelToolCallPart | LanguageModelDataPart | LanguageModelExtraDataPart)[]) { + set content(value: string | (LanguageModelTextPart | LanguageModelToolResultPart2 | LanguageModelToolCallPart | LanguageModelDataPart)[]) { if (typeof value === 'string') { // we changed this and still support setting content with a string property. this keep the API runtime stable // despite the breaking change in the type definition. @@ -4894,7 +4894,7 @@ export class LanguageModelChatMessage2 implements vscode.LanguageModelChatMessag } } - get content(): (LanguageModelTextPart | LanguageModelToolResultPart2 | LanguageModelToolCallPart | LanguageModelDataPart | LanguageModelExtraDataPart)[] { + get content(): (LanguageModelTextPart | LanguageModelToolResultPart2 | LanguageModelToolCallPart | LanguageModelDataPart)[] { return this._content; } @@ -4910,7 +4910,7 @@ export class LanguageModelChatMessage2 implements vscode.LanguageModelChatMessag } } - get content2(): (string | LanguageModelToolResultPart2 | LanguageModelToolCallPart | LanguageModelDataPart | LanguageModelExtraDataPart)[] | undefined { + get content2(): (string | LanguageModelToolResultPart2 | LanguageModelToolCallPart | LanguageModelDataPart)[] | undefined { return this.content.map(part => { if (part instanceof LanguageModelTextPart) { return part.value; @@ -4921,7 +4921,7 @@ export class LanguageModelChatMessage2 implements vscode.LanguageModelChatMessag name: string | undefined; - constructor(role: vscode.LanguageModelChatMessageRole, content: string | (LanguageModelTextPart | LanguageModelToolResultPart2 | LanguageModelToolCallPart | LanguageModelDataPart | LanguageModelExtraDataPart)[], name?: string) { + constructor(role: vscode.LanguageModelChatMessageRole, content: string | (LanguageModelTextPart | LanguageModelToolResultPart2 | LanguageModelToolCallPart | LanguageModelDataPart)[], name?: string) { this.role = role; this.content = content; this.name = name; @@ -4970,13 +4970,13 @@ export class LanguageModelDataPart implements vscode.LanguageModelDataPart { return new LanguageModelDataPart(data, mimeType as string); } - static json(value: object): vscode.LanguageModelDataPart { + static json(value: object, mime: string = 'text/x-json'): vscode.LanguageModelDataPart { const rawStr = JSON.stringify(value, undefined, '\t'); - return new LanguageModelDataPart(VSBuffer.fromString(rawStr).buffer, 'json'); + return new LanguageModelDataPart(VSBuffer.fromString(rawStr).buffer, mime); } - static text(value: string): vscode.LanguageModelDataPart { - return new LanguageModelDataPart(VSBuffer.fromString(value).buffer, 'text/plain'); + static text(value: string, mime: string = Mimes.text): vscode.LanguageModelDataPart { + return new LanguageModelDataPart(VSBuffer.fromString(value).buffer, mime); } toJSON() { @@ -4996,24 +4996,6 @@ export enum ChatImageMimeType { BMP = 'image/bmp', } -export class LanguageModelExtraDataPart implements vscode.LanguageModelExtraDataPart { - kind: string; - data: any; - - constructor(kind: string, data: any) { - this.kind = kind; - this.data = data; - } - - toJSON() { - return { - $mid: MarshalledId.LanguageModelExtraDataPart, - kind: this.kind, - data: this.data, - }; - } -} - export class LanguageModelPromptTsxPart { value: unknown; diff --git a/src/vs/workbench/contrib/chat/common/languageModels.ts b/src/vs/workbench/contrib/chat/common/languageModels.ts index 62ed7dcdf82..5927d2b1c79 100644 --- a/src/vs/workbench/contrib/chat/common/languageModels.ts +++ b/src/vs/workbench/contrib/chat/common/languageModels.ts @@ -36,10 +36,10 @@ export interface IChatMessageImagePart { value: IChatImageURLPart; } -export interface IChatMessageExtraDataPart { - type: 'extra_data'; - kind: string; - data: any; +export interface IChatMessageDataPart { + type: 'data'; + mimeType: string; + data: VSBuffer; } export interface IChatImageURLPart { @@ -81,7 +81,7 @@ export interface IChatMessageToolResultPart { isError?: boolean; } -export type IChatMessagePart = IChatMessageTextPart | IChatMessageToolResultPart | IChatResponseToolUsePart | IChatMessageImagePart | IChatMessageExtraDataPart; +export type IChatMessagePart = IChatMessageTextPart | IChatMessageToolResultPart | IChatResponseToolUsePart | IChatMessageImagePart | IChatMessageDataPart; export interface IChatMessage { readonly name?: string | undefined; diff --git a/src/vscode-dts/vscode.proposed.languageModelDataPart.d.ts b/src/vscode-dts/vscode.proposed.languageModelDataPart.d.ts index f365ede02e6..902ed64ec97 100644 --- a/src/vscode-dts/vscode.proposed.languageModelDataPart.d.ts +++ b/src/vscode-dts/vscode.proposed.languageModelDataPart.d.ts @@ -3,7 +3,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -// version: 2 +// version: 3 declare module 'vscode' { @@ -23,7 +23,7 @@ declare module 'vscode' { * @param content The content of the message. * @param name The optional name of a user for the message. */ - static User(content: string | Array, name?: string): LanguageModelChatMessage2; + static User(content: string | Array, name?: string): LanguageModelChatMessage2; /** * Utility to create a new assistant message. @@ -31,7 +31,7 @@ declare module 'vscode' { * @param content The content of the message. * @param name The optional name of a user for the message. */ - static Assistant(content: string | Array, name?: string): LanguageModelChatMessage2; + static Assistant(content: string | Array, name?: string): LanguageModelChatMessage2; /** * The role of this message. @@ -42,7 +42,7 @@ declare module 'vscode' { * A string or heterogeneous array of things that a message can contain as content. Some parts may be message-type * specific for some models. */ - content: Array; + content: Array; /** * The optional name of a user for this message. @@ -56,7 +56,7 @@ declare module 'vscode' { * @param content The content of the message. * @param name The optional name of a user for the message. */ - constructor(role: LanguageModelChatMessageRole, content: string | Array, name?: string); + constructor(role: LanguageModelChatMessageRole, content: string | Array, name?: string); } /** @@ -70,9 +70,9 @@ declare module 'vscode' { */ static image(data: Uint8Array, mimeType: ChatImageMimeType): LanguageModelDataPart; - static json(value: object): LanguageModelDataPart; + static json(value: any, mime?: string): LanguageModelDataPart; - static text(value: string): LanguageModelDataPart; + static text(value: string, mime?: string): LanguageModelDataPart; /** * The mime type which determines how the data property is interpreted. @@ -102,31 +102,6 @@ declare module 'vscode' { BMP = 'image/bmp', } - /** - * Tagging onto this proposal, because otherwise managing two different extensions of LanguageModelChatMessage could be confusing. - * A language model response part containing arbitrary model-specific data, returned from a {@link LanguageModelChatResponse}. - * TODO@API naming, looking at LanguageModelChatRequestOptions.modelOptions, but LanguageModelModelData is not very good. - * LanguageModelOpaqueData from prompt-tsx? - */ - export class LanguageModelExtraDataPart { - /** - * The type of data. The allowed values and data types here are model-specific. - */ - kind: string; - - /** - * Extra model-specific data. - */ - data: any; - - /** - * Construct an extra data part with the given content. - * @param value The image content of the part. - */ - constructor(kind: string, data: any); - } - - /** * The result of a tool call. This is the counterpart of a {@link LanguageModelToolCallPart tool call} and * it can only be included in the content of a User message