diff --git a/src/vs/platform/extensions/common/extensionsApiProposals.ts b/src/vs/platform/extensions/common/extensionsApiProposals.ts index 4eb66e7ea6e..d2d4c1fff25 100644 --- a/src/vs/platform/extensions/common/extensionsApiProposals.ts +++ b/src/vs/platform/extensions/common/extensionsApiProposals.ts @@ -246,7 +246,7 @@ const _allApiProposals = { }, lmTools: { proposal: 'https://raw.githubusercontent.com/microsoft/vscode/main/src/vscode-dts/vscode.proposed.lmTools.d.ts', - version: 5 + version: 6 }, mappedEditsProvider: { proposal: 'https://raw.githubusercontent.com/microsoft/vscode/main/src/vscode-dts/vscode.proposed.mappedEditsProvider.d.ts', diff --git a/src/vs/workbench/api/common/extHost.api.impl.ts b/src/vs/workbench/api/common/extHost.api.impl.ts index 4c4b70a2209..a9c8d66c70c 100644 --- a/src/vs/workbench/api/common/extHost.api.impl.ts +++ b/src/vs/workbench/api/common/extHost.api.impl.ts @@ -1766,9 +1766,9 @@ export function createApiFactoryAndRegisterActors(accessor: ServicesAccessor): I ChatRequestNotebookData: extHostTypes.ChatRequestNotebookData, LanguageModelChatMessageRole: extHostTypes.LanguageModelChatMessageRole, LanguageModelChatMessage: extHostTypes.LanguageModelChatMessage, - LanguageModelChatMessageFunctionResultPart: extHostTypes.LanguageModelFunctionResultPart, + LanguageModelChatMessageToolResultPart: extHostTypes.LanguageModelToolResultPart, LanguageModelChatResponseTextPart: extHostTypes.LanguageModelTextPart, - LanguageModelChatResponseFunctionUsePart: extHostTypes.LanguageModelFunctionUsePart, + LanguageModelChatResponseToolCallPart: extHostTypes.LanguageModelToolCallPart, LanguageModelError: extHostTypes.LanguageModelError, NewSymbolName: extHostTypes.NewSymbolName, NewSymbolNameTag: extHostTypes.NewSymbolNameTag, diff --git a/src/vs/workbench/api/common/extHostLanguageModels.ts b/src/vs/workbench/api/common/extHostLanguageModels.ts index b67999350af..dd0ccfc98b0 100644 --- a/src/vs/workbench/api/common/extHostLanguageModels.ts +++ b/src/vs/workbench/api/common/extHostLanguageModels.ts @@ -37,13 +37,13 @@ type LanguageModelData = { class LanguageModelResponseStream { - readonly stream = new AsyncIterableSource(); + readonly stream = new AsyncIterableSource(); constructor( readonly option: number, - stream?: AsyncIterableSource + stream?: AsyncIterableSource ) { - this.stream = stream ?? new AsyncIterableSource(); + this.stream = stream ?? new AsyncIterableSource(); } } @@ -52,7 +52,7 @@ class LanguageModelResponse { readonly apiObject: vscode.LanguageModelChatResponse; private readonly _responseStreams = new Map(); - private readonly _defaultStream = new AsyncIterableSource(); + private readonly _defaultStream = new AsyncIterableSource(); private _isDone: boolean = false; constructor() { @@ -100,11 +100,11 @@ class LanguageModelResponse { this._responseStreams.set(fragment.index, res); } - let out: vscode.LanguageModelChatResponseTextPart | vscode.LanguageModelChatResponseFunctionUsePart; + let out: vscode.LanguageModelChatResponseTextPart | vscode.LanguageModelChatResponseToolCallPart; if (fragment.part.type === 'text') { out = new extHostTypes.LanguageModelTextPart(fragment.part.value); } else { - out = new extHostTypes.LanguageModelFunctionUsePart(fragment.part.name, fragment.part.parameters); + out = new extHostTypes.LanguageModelToolCallPart(fragment.part.name, fragment.part.toolCallId, fragment.part.parameters); } res.stream.emitOne(out); } @@ -201,8 +201,8 @@ export class ExtHostLanguageModels implements ExtHostLanguageModelsShape { } let part: IChatResponsePart | undefined; - if (fragment.part instanceof extHostTypes.LanguageModelFunctionUsePart) { - part = { type: 'function_use', name: fragment.part.name, parameters: fragment.part.parameters }; + if (fragment.part instanceof extHostTypes.LanguageModelToolCallPart) { + part = { type: 'tool_use', name: fragment.part.name, parameters: fragment.part.parameters, toolCallId: fragment.part.toolCallId }; } else if (fragment.part instanceof extHostTypes.LanguageModelTextPart) { part = { type: 'text', value: fragment.part.value }; } @@ -396,7 +396,7 @@ export class ExtHostLanguageModels implements ExtHostLanguageModelsShape { if (message.role as number === extHostTypes.LanguageModelChatMessageRole.System) { checkProposedApiEnabled(extension, 'languageModelSystem'); } - if (message.content2 instanceof extHostTypes.LanguageModelFunctionResultPart) { + if (message.content2.some(part => part instanceof extHostTypes.LanguageModelToolResultPart)) { checkProposedApiEnabled(extension, 'lmTools'); } internalMessages.push(typeConvert.LanguageModelChatMessage.from(message)); diff --git a/src/vs/workbench/api/common/extHostTypeConverters.ts b/src/vs/workbench/api/common/extHostTypeConverters.ts index ede0cd5a525..c62d9d059a6 100644 --- a/src/vs/workbench/api/common/extHostTypeConverters.ts +++ b/src/vs/workbench/api/common/extHostTypeConverters.ts @@ -2313,18 +2313,17 @@ export namespace LanguageModelChatMessageRole { export namespace LanguageModelChatMessage { export function to(message: chatProvider.IChatMessage): vscode.LanguageModelChatMessage { - let content: string = ''; - let content2: vscode.LanguageModelChatMessageFunctionResultPart | undefined; - if (message.content.type === 'text') { - content = message.content.value; - } else { - content2 = new types.LanguageModelFunctionResultPart(message.content.name, message.content.value, message.content.isError); - } + const content2 = message.content.map(c => { + if (c.type === 'text') { + return c.value; + } else { + return new types.LanguageModelToolResultPart(c.name, c.name, c.value, c.isError); + } + }); + const content = content2.find(c => typeof c === 'string') ?? ''; const role = LanguageModelChatMessageRole.to(message.role); const result = new types.LanguageModelChatMessage(role, content, message.name); - if (content2 !== undefined) { - result.content2 = content2; - } + result.content2 = content2; return result; } @@ -2333,21 +2332,22 @@ export namespace LanguageModelChatMessage { const role = LanguageModelChatMessageRole.from(message.role); const name = message.name; - let content: chatProvider.IChatMessagePart; - - if (message.content2 instanceof types.LanguageModelFunctionResultPart) { - content = { - type: 'function_result', - name: message.content2.name, - value: message.content2.content, - isError: message.content2.isError - }; - } else { - content = { - type: 'text', - value: message.content - }; - } + const content = message.content2.map((c): chatProvider.IChatMessagePart => { + if (message.content2 instanceof types.LanguageModelToolResultPart) { + return { + type: 'tool_result', + name: message.content2.name, + toolCallId: message.content2.toolCallId, + value: message.content2.content, + isError: message.content2.isError + }; + } else { + return { + type: 'text', + value: message.content + }; + } + }); return { role, diff --git a/src/vs/workbench/api/common/extHostTypes.ts b/src/vs/workbench/api/common/extHostTypes.ts index 6e232711ebb..8227ee92056 100644 --- a/src/vs/workbench/api/common/extHostTypes.ts +++ b/src/vs/workbench/api/common/extHostTypes.ts @@ -4557,14 +4557,16 @@ export enum LanguageModelChatMessageRole { System = 3 } -export class LanguageModelFunctionResultPart implements vscode.LanguageModelChatMessageFunctionResultPart { +export class LanguageModelToolResultPart implements vscode.LanguageModelChatMessageToolResultPart { name: string; + toolCallId: string; content: string; isError: boolean; - constructor(name: string, content: string, isError?: boolean) { + constructor(name: string, toolCallId: string, content: string, isError?: boolean) { this.name = name; + this.toolCallId = toolCallId; this.content = content; this.isError = isError ?? false; } @@ -4572,9 +4574,9 @@ export class LanguageModelFunctionResultPart implements vscode.LanguageModelChat export class LanguageModelChatMessage implements vscode.LanguageModelChatMessage { - static User(content: string | LanguageModelFunctionResultPart, name?: string): LanguageModelChatMessage { + static User(content: string | LanguageModelToolResultPart, name?: string): LanguageModelChatMessage { const value = new LanguageModelChatMessage(LanguageModelChatMessageRole.User, typeof content === 'string' ? content : '', name); - value.content2 = content; + value.content2 = [content]; return value; } @@ -4584,23 +4586,25 @@ export class LanguageModelChatMessage implements vscode.LanguageModelChatMessage role: vscode.LanguageModelChatMessageRole; content: string; - content2: string | vscode.LanguageModelChatMessageFunctionResultPart; + content2: (string | vscode.LanguageModelChatMessageToolResultPart | vscode.LanguageModelChatResponseToolCallPart)[]; name: string | undefined; constructor(role: vscode.LanguageModelChatMessageRole, content: string, name?: string) { this.role = role; this.content = content; - this.content2 = content; + this.content2 = [content]; this.name = name; } } -export class LanguageModelFunctionUsePart implements vscode.LanguageModelChatResponseFunctionUsePart { +export class LanguageModelToolCallPart implements vscode.LanguageModelChatResponseToolCallPart { name: string; + toolCallId: string; parameters: any; - constructor(name: string, parameters: any) { + constructor(name: string, toolCallId: string, parameters: any) { this.name = name; + this.toolCallId = toolCallId; this.parameters = parameters; } } diff --git a/src/vs/workbench/contrib/chat/common/chatServiceImpl.ts b/src/vs/workbench/contrib/chat/common/chatServiceImpl.ts index a1b0b20833b..3ca4524e80e 100644 --- a/src/vs/workbench/contrib/chat/common/chatServiceImpl.ts +++ b/src/vs/workbench/contrib/chat/common/chatServiceImpl.ts @@ -679,8 +679,8 @@ export class ChatService extends Disposable implements IChatService { if (!request.response) { continue; } - history.push({ role: ChatMessageRole.User, content: { type: 'text', value: request.message.text } }); - history.push({ role: ChatMessageRole.Assistant, content: { type: 'text', value: request.response.response.toString() } }); + history.push({ role: ChatMessageRole.User, content: [{ type: 'text', value: request.message.text }] }); + history.push({ role: ChatMessageRole.Assistant, content: [{ type: 'text', value: request.response.response.toString() }] }); } const message = parsedRequest.text; const commandResult = await this.chatSlashCommandService.executeCommand(commandPart.slashCommand.command, message.substring(commandPart.slashCommand.command.length + 1).trimStart(), new Progress(p => { diff --git a/src/vs/workbench/contrib/chat/common/languageModels.ts b/src/vs/workbench/contrib/chat/common/languageModels.ts index b19a50e5e44..2d8915e9f1e 100644 --- a/src/vs/workbench/contrib/chat/common/languageModels.ts +++ b/src/vs/workbench/contrib/chat/common/languageModels.ts @@ -27,19 +27,20 @@ export interface IChatMessageTextPart { value: string; } -export interface IChatMessageFunctionResultPart { - type: 'function_result'; +export interface IChatMessageToolResultPart { + type: 'tool_result'; name: string; + toolCallId: string; value: any; isError?: boolean; } -export type IChatMessagePart = IChatMessageTextPart | IChatMessageFunctionResultPart; +export type IChatMessagePart = IChatMessageTextPart | IChatMessageToolResultPart; export interface IChatMessage { readonly name?: string | undefined; readonly role: ChatMessageRole; - readonly content: IChatMessagePart; + readonly content: IChatMessagePart[]; } export interface IChatResponseTextPart { @@ -47,13 +48,14 @@ export interface IChatResponseTextPart { value: string; } -export interface IChatResponceFunctionUsePart { - type: 'function_use'; +export interface IChatResponseToolUsePart { + type: 'tool_use'; name: string; + toolCallId: string; parameters: any; } -export type IChatResponsePart = IChatResponseTextPart | IChatResponceFunctionUsePart; +export type IChatResponsePart = IChatResponseTextPart | IChatResponseToolUsePart; export interface IChatResponseFragment { index: number; diff --git a/src/vs/workbench/contrib/chat/test/common/languageModels.test.ts b/src/vs/workbench/contrib/chat/test/common/languageModels.test.ts index ebdde0c0d22..4018b862996 100644 --- a/src/vs/workbench/contrib/chat/test/common/languageModels.test.ts +++ b/src/vs/workbench/contrib/chat/test/common/languageModels.test.ts @@ -149,7 +149,7 @@ suite('LanguageModels', function () { const cts = new CancellationTokenSource(); - const request = await languageModels.sendChatRequest(first, nullExtensionDescription.identifier, [{ role: ChatMessageRole.User, content: { type: 'text', value: 'hello' } }], {}, cts.token); + const request = await languageModels.sendChatRequest(first, nullExtensionDescription.identifier, [{ role: ChatMessageRole.User, content: [{ type: 'text', value: 'hello' }] }], {}, cts.token); assert.ok(request); diff --git a/src/vscode-dts/vscode.proposed.chatProvider.d.ts b/src/vscode-dts/vscode.proposed.chatProvider.d.ts index 9abbdc5d1aa..e645ffb2dc4 100644 --- a/src/vscode-dts/vscode.proposed.chatProvider.d.ts +++ b/src/vscode-dts/vscode.proposed.chatProvider.d.ts @@ -12,7 +12,7 @@ declare module 'vscode' { export interface ChatResponseFragment2 { index: number; - part: LanguageModelChatResponseTextPart | LanguageModelChatResponseFunctionUsePart; + part: LanguageModelChatResponseTextPart | LanguageModelChatResponseToolCallPart; } // @API extension ship a d.ts files for their options diff --git a/src/vscode-dts/vscode.proposed.lmTools.d.ts b/src/vscode-dts/vscode.proposed.lmTools.d.ts index 860a64f4c69..95c48887fd3 100644 --- a/src/vscode-dts/vscode.proposed.lmTools.d.ts +++ b/src/vscode-dts/vscode.proposed.lmTools.d.ts @@ -3,7 +3,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -// version: 5 +// version: 6 // https://github.com/microsoft/vscode/issues/213274 declare module 'vscode' { @@ -11,7 +11,8 @@ declare module 'vscode' { // TODO@API capabilities // API -> LM: an tool/function that is available to the language model - export interface LanguageModelChatFunction { + export interface LanguageModelChatTool { + // TODO@API should use "id" here to match vscode tools, or keep name to match OpenAI? name: string; description: string; parametersSchema?: JSONSchema; @@ -19,8 +20,8 @@ declare module 'vscode' { // API -> LM: add tools as request option export interface LanguageModelChatRequestOptions { - // TODO@API this will a heterogeneous array of different types of tools - tools?: LanguageModelChatFunction[]; + // TODO@API this will be a heterogeneous array of different types of tools + tools?: LanguageModelChatTool[]; /** * Force a specific tool to be used. @@ -29,11 +30,12 @@ declare module 'vscode' { } // LM -> USER: function that should be used - export class LanguageModelChatResponseFunctionUsePart { + export class LanguageModelChatResponseToolCallPart { name: string; + toolCallId: string; parameters: any; - constructor(name: string, parameters: any); + constructor(name: string, parameters: any, toolCallId: string); } // LM -> USER: text chunk @@ -44,22 +46,30 @@ declare module 'vscode' { } export interface LanguageModelChatResponse { - - stream: AsyncIterable; + stream: AsyncIterable; } // USER -> LM: the result of a function call - export class LanguageModelChatMessageFunctionResultPart { + export class LanguageModelChatMessageToolResultPart { name: string; + toolCallId: string; content: string; isError: boolean; - constructor(name: string, content: string, isError?: boolean); + constructor(name: string, toolCallId: string, content: string, isError?: boolean); } export interface LanguageModelChatMessage { - content2: string | LanguageModelChatMessageFunctionResultPart; + /** + * A heterogeneous array of other things that a message can contain as content. + * Some parts would be message-type specific for some models and wouldn't go together, + * but it's up to the chat provider to decide what to do about that. + * Can drop parts that are not valid for the message type. + * LanguageModelChatMessageToolResultPart: only on User messages + * LanguageModelChatResponseToolCallPart: only on Assistant messages + */ + content2: (string | LanguageModelChatMessageToolResultPart | LanguageModelChatResponseToolCallPart)[]; } export interface LanguageModelToolResult {