diff --git a/src/vs/base/common/marshallingIds.ts b/src/vs/base/common/marshallingIds.ts index c83402b47c5..9ea80910eeb 100644 --- a/src/vs/base/common/marshallingIds.ts +++ b/src/vs/base/common/marshallingIds.ts @@ -26,5 +26,6 @@ export const enum MarshalledId { LanguageModelToolResult, LanguageModelTextPart, LanguageModelPromptTsxPart, - LanguageModelDataPart + LanguageModelDataPart, + LanguageModelExtraDataPart, } diff --git a/src/vs/platform/extensions/common/extensionsApiProposals.ts b/src/vs/platform/extensions/common/extensionsApiProposals.ts index e21a27208f3..1636bf4db20 100644 --- a/src/vs/platform/extensions/common/extensionsApiProposals.ts +++ b/src/vs/platform/extensions/common/extensionsApiProposals.ts @@ -33,7 +33,7 @@ const _allApiProposals = { }, chatParticipantPrivate: { proposal: 'https://raw.githubusercontent.com/microsoft/vscode/main/src/vscode-dts/vscode.proposed.chatParticipantPrivate.d.ts', - version: 7 + version: 8 }, chatProvider: { proposal: 'https://raw.githubusercontent.com/microsoft/vscode/main/src/vscode-dts/vscode.proposed.chatProvider.d.ts', @@ -231,6 +231,7 @@ const _allApiProposals = { }, languageModelDataPart: { proposal: 'https://raw.githubusercontent.com/microsoft/vscode/main/src/vscode-dts/vscode.proposed.languageModelDataPart.d.ts', + version: 1 }, languageModelSystem: { proposal: 'https://raw.githubusercontent.com/microsoft/vscode/main/src/vscode-dts/vscode.proposed.languageModelSystem.d.ts', diff --git a/src/vs/workbench/api/common/extHost.api.impl.ts b/src/vs/workbench/api/common/extHost.api.impl.ts index 3165c3c4a79..c23e6900be5 100644 --- a/src/vs/workbench/api/common/extHost.api.impl.ts +++ b/src/vs/workbench/api/common/extHost.api.impl.ts @@ -1814,6 +1814,7 @@ export function createApiFactoryAndRegisterActors(accessor: ServicesAccessor): I LanguageModelError: extHostTypes.LanguageModelError, LanguageModelToolResult: extHostTypes.LanguageModelToolResult, LanguageModelDataPart: extHostTypes.LanguageModelDataPart, + LanguageModelExtraDataPart: extHostTypes.LanguageModelExtraDataPart, ChatImageMimeType: extHostTypes.ChatImageMimeType, ExtendedLanguageModelToolResult: extHostTypes.ExtendedLanguageModelToolResult, PreparedTerminalToolInvocation: extHostTypes.PreparedTerminalToolInvocation, diff --git a/src/vs/workbench/api/common/extHostTypeConverters.ts b/src/vs/workbench/api/common/extHostTypeConverters.ts index a42e18e171b..a9a9ef3b486 100644 --- a/src/vs/workbench/api/common/extHostTypeConverters.ts +++ b/src/vs/workbench/api/common/extHostTypeConverters.ts @@ -2328,10 +2328,9 @@ export namespace LanguageModelChatMessage { } }); return new types.LanguageModelToolResultPart(c.toolCallId, content, c.isError); - } else if (c.type === 'image_url') { - // No image support for LanguageModelChatMessage + } else if (c.type === 'image_url' || c.type === 'extra_data') { + // Non-stable types return undefined; - } else { return new types.LanguageModelToolCallPart(c.toolCallId, c.name, c.parameters); } @@ -2429,6 +2428,8 @@ export namespace LanguageModelChatMessage2 { }; return new types.LanguageModelDataPart(value); + } else if (c.type === 'extra_data') { + return new types.LanguageModelExtraDataPart(c.kind, c.data); } else { return new types.LanguageModelToolCallPart(c.toolCallId, c.name, c.parameters); } @@ -2493,6 +2494,12 @@ export namespace LanguageModelChatMessage2 { type: 'text', value: c.value }; + } else if (c instanceof types.LanguageModelExtraDataPart) { + return { + type: 'extra_data', + kind: c.kind, + data: c.data + } satisfies chatProvider.IChatMessagePart; } else { if (typeof c !== 'string') { throw new Error('Unexpected chat message content type'); diff --git a/src/vs/workbench/api/common/extHostTypes.ts b/src/vs/workbench/api/common/extHostTypes.ts index b35ec4328a2..bcac0be39f8 100644 --- a/src/vs/workbench/api/common/extHostTypes.ts +++ b/src/vs/workbench/api/common/extHostTypes.ts @@ -4871,22 +4871,21 @@ export class LanguageModelChatMessage implements vscode.LanguageModelChatMessage } } - export class LanguageModelChatMessage2 implements vscode.LanguageModelChatMessage2 { - static User(content: string | (LanguageModelTextPart | LanguageModelToolResultPart | LanguageModelToolCallPart | LanguageModelDataPart)[], name?: string): LanguageModelChatMessage2 { + static User(content: string | (LanguageModelTextPart | LanguageModelToolResultPart | LanguageModelToolCallPart | LanguageModelDataPart | LanguageModelExtraDataPart)[], name?: string): LanguageModelChatMessage2 { return new LanguageModelChatMessage2(LanguageModelChatMessageRole.User, content, name); } - static Assistant(content: string | (LanguageModelTextPart | LanguageModelToolResultPart | LanguageModelToolCallPart | LanguageModelDataPart)[], name?: string): LanguageModelChatMessage2 { + static Assistant(content: string | (LanguageModelTextPart | LanguageModelToolResultPart | LanguageModelToolCallPart | LanguageModelDataPart | LanguageModelExtraDataPart)[], name?: string): LanguageModelChatMessage2 { return new LanguageModelChatMessage2(LanguageModelChatMessageRole.Assistant, content, name); } role: vscode.LanguageModelChatMessageRole; - private _content: (LanguageModelTextPart | LanguageModelToolResultPart | LanguageModelToolCallPart | LanguageModelDataPart)[] = []; + private _content: (LanguageModelTextPart | LanguageModelToolResultPart | LanguageModelToolCallPart | LanguageModelDataPart | LanguageModelExtraDataPart)[] = []; - set content(value: string | (LanguageModelTextPart | LanguageModelToolResultPart | LanguageModelToolCallPart | LanguageModelDataPart)[]) { + set content(value: string | (LanguageModelTextPart | LanguageModelToolResultPart | LanguageModelToolCallPart | LanguageModelDataPart | LanguageModelExtraDataPart)[]) { if (typeof value === 'string') { // we changed this and still support setting content with a string property. this keep the API runtime stable // despite the breaking change in the type definition. @@ -4896,7 +4895,7 @@ export class LanguageModelChatMessage2 implements vscode.LanguageModelChatMessag } } - get content(): (LanguageModelTextPart | LanguageModelToolResultPart | LanguageModelToolCallPart | LanguageModelDataPart)[] { + get content(): (LanguageModelTextPart | LanguageModelToolResultPart | LanguageModelToolCallPart | LanguageModelDataPart | LanguageModelExtraDataPart)[] { return this._content; } @@ -4912,7 +4911,7 @@ export class LanguageModelChatMessage2 implements vscode.LanguageModelChatMessag } } - get content2(): (string | LanguageModelToolResultPart | LanguageModelToolCallPart | LanguageModelDataPart)[] | undefined { + get content2(): (string | LanguageModelToolResultPart | LanguageModelToolCallPart | LanguageModelDataPart | LanguageModelExtraDataPart)[] | undefined { return this.content.map(part => { if (part instanceof LanguageModelTextPart) { return part.value; @@ -4923,7 +4922,7 @@ export class LanguageModelChatMessage2 implements vscode.LanguageModelChatMessag name: string | undefined; - constructor(role: vscode.LanguageModelChatMessageRole, content: string | (LanguageModelTextPart | LanguageModelToolResultPart | LanguageModelToolCallPart | LanguageModelDataPart)[], name?: string) { + constructor(role: vscode.LanguageModelChatMessageRole, content: string | (LanguageModelTextPart | LanguageModelToolResultPart | LanguageModelToolCallPart | LanguageModelDataPart | LanguageModelExtraDataPart)[], name?: string) { this.role = role; this.content = content; this.name = name; @@ -4974,6 +4973,24 @@ export class LanguageModelDataPart implements vscode.LanguageModelDataPart { } } +export class LanguageModelExtraDataPart implements vscode.LanguageModelExtraDataPart { + kind: string; + data: any; + + constructor(kind: string, data: any) { + this.kind = kind; + this.data = data; + } + + toJSON() { + return { + $mid: MarshalledId.LanguageModelExtraDataPart, + kind: this.kind, + data: this.data, + }; + } +} + /** * Enum for supported image MIME types. */ diff --git a/src/vs/workbench/contrib/chat/common/languageModels.ts b/src/vs/workbench/contrib/chat/common/languageModels.ts index ba03b133a1a..9d03fb8a208 100644 --- a/src/vs/workbench/contrib/chat/common/languageModels.ts +++ b/src/vs/workbench/contrib/chat/common/languageModels.ts @@ -35,6 +35,12 @@ export interface IChatMessageImagePart { value: IChatImageURLPart; } +export interface IChatMessageExtraDataPart { + type: 'extra_data'; + kind: string; + data: any; +} + export interface IChatImageURLPart { /** * The image's MIME type (e.g., "image/png", "image/jpeg"). @@ -74,7 +80,7 @@ export interface IChatMessageToolResultPart { isError?: boolean; } -export type IChatMessagePart = IChatMessageTextPart | IChatMessageToolResultPart | IChatResponseToolUsePart | IChatMessageImagePart; +export type IChatMessagePart = IChatMessageTextPart | IChatMessageToolResultPart | IChatResponseToolUsePart | IChatMessageImagePart | IChatMessageExtraDataPart; export interface IChatMessage { readonly name?: string | undefined; diff --git a/src/vscode-dts/vscode.proposed.chatParticipantPrivate.d.ts b/src/vscode-dts/vscode.proposed.chatParticipantPrivate.d.ts index 65f6729e38e..d056a6f0711 100644 --- a/src/vscode-dts/vscode.proposed.chatParticipantPrivate.d.ts +++ b/src/vscode-dts/vscode.proposed.chatParticipantPrivate.d.ts @@ -3,7 +3,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -// version: 7 +// version: 8 declare module 'vscode' { diff --git a/src/vscode-dts/vscode.proposed.languageModelDataPart.d.ts b/src/vscode-dts/vscode.proposed.languageModelDataPart.d.ts index e5160b12799..a9e888ea892 100644 --- a/src/vscode-dts/vscode.proposed.languageModelDataPart.d.ts +++ b/src/vscode-dts/vscode.proposed.languageModelDataPart.d.ts @@ -3,6 +3,8 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ +// version: 1 + declare module 'vscode' { export interface LanguageModelChat { @@ -21,7 +23,7 @@ declare module 'vscode' { * @param content The content of the message. * @param name The optional name of a user for the message. */ - static User(content: string | Array, name?: string): LanguageModelChatMessage2; + static User(content: string | Array, name?: string): LanguageModelChatMessage2; /** * Utility to create a new assistant message. @@ -29,7 +31,7 @@ declare module 'vscode' { * @param content The content of the message. * @param name The optional name of a user for the message. */ - static Assistant(content: string | Array, name?: string): LanguageModelChatMessage2; + static Assistant(content: string | Array, name?: string): LanguageModelChatMessage2; /** * The role of this message. @@ -40,7 +42,7 @@ declare module 'vscode' { * A string or heterogeneous array of things that a message can contain as content. Some parts may be message-type * specific for some models. */ - content: Array; + content: Array; /** * The optional name of a user for this message. @@ -54,7 +56,7 @@ declare module 'vscode' { * @param content The content of the message. * @param name The optional name of a user for the message. */ - constructor(role: LanguageModelChatMessageRole, content: string | Array, name?: string); + constructor(role: LanguageModelChatMessageRole, content: string | Array, name?: string); } /** @@ -95,4 +97,28 @@ declare module 'vscode' { */ data: Uint8Array; } + + /** + * Tagging onto this proposal, because otherwise managing two different extensions of LangaugeModelChatMessage could be confusing. + * A language model response part containing arbitrary model-specific data, returned from a {@link LanguageModelChatResponse}. + * TODO@API naming, looking at LanguageModelChatRequestOptions.modelOptions, but LanguageModelModelData is not very good. + * LanguageModelOpaqueData from prompt-tsx? + */ + export class LanguageModelExtraDataPart { + /** + * The type of data. The allowed values and data types here are model-specific. + */ + kind: string; + + /** + * Extra model-specific data. + */ + data: any; + + /** + * Construct an extra data part with the given content. + * @param value The image content of the part. + */ + constructor(kind: string, data: any); + } }