From da2eb39383ee70f7f9caa047352d5dbc19fe2483 Mon Sep 17 00:00:00 2001 From: Rob Lourens Date: Mon, 5 Aug 2024 16:04:21 -0700 Subject: [PATCH] Support returning different types from chat tools API (#224894) See #213274 --- .../common/extensionsApiProposals.ts | 2 +- .../browser/mainThreadLanguageModelTools.ts | 4 ++-- .../workbench/api/common/extHost.protocol.ts | 6 +++--- .../api/common/extHostLanguageModelTools.ts | 12 ++++++----- .../api/common/extHostTypeConverters.ts | 20 ++++++++++++++++++- .../chat/common/languageModelToolsService.ts | 11 +++++++--- .../vscode.proposed.chatVariableResolver.d.ts | 17 ---------------- src/vscode-dts/vscode.proposed.lmTools.d.ts | 20 ++++++++++++++++--- 8 files changed, 57 insertions(+), 35 deletions(-) diff --git a/src/vs/platform/extensions/common/extensionsApiProposals.ts b/src/vs/platform/extensions/common/extensionsApiProposals.ts index 673643f7173..b2b3f5cfb98 100644 --- a/src/vs/platform/extensions/common/extensionsApiProposals.ts +++ b/src/vs/platform/extensions/common/extensionsApiProposals.ts @@ -240,7 +240,7 @@ const _allApiProposals = { }, lmTools: { proposal: 'https://raw.githubusercontent.com/microsoft/vscode/main/src/vscode-dts/vscode.proposed.lmTools.d.ts', - version: 2 + version: 3 }, mappedEditsProvider: { proposal: 'https://raw.githubusercontent.com/microsoft/vscode/main/src/vscode-dts/vscode.proposed.mappedEditsProvider.d.ts', diff --git a/src/vs/workbench/api/browser/mainThreadLanguageModelTools.ts b/src/vs/workbench/api/browser/mainThreadLanguageModelTools.ts index fbda6ced5a3..1e1a25f966c 100644 --- a/src/vs/workbench/api/browser/mainThreadLanguageModelTools.ts +++ b/src/vs/workbench/api/browser/mainThreadLanguageModelTools.ts @@ -6,7 +6,7 @@ import { CancellationToken } from 'vs/base/common/cancellation'; import { Disposable, DisposableMap } from 'vs/base/common/lifecycle'; import { ExtHostLanguageModelToolsShape, ExtHostContext, MainContext, MainThreadLanguageModelToolsShape } from 'vs/workbench/api/common/extHost.protocol'; -import { IToolData, ILanguageModelToolsService } from 'vs/workbench/contrib/chat/common/languageModelToolsService'; +import { IToolData, ILanguageModelToolsService, IToolResult } from 'vs/workbench/contrib/chat/common/languageModelToolsService'; import { IExtHostContext, extHostNamedCustomer } from 'vs/workbench/services/extensions/common/extHostCustomers'; @extHostNamedCustomer(MainContext.MainThreadLanguageModelTools) @@ -29,7 +29,7 @@ export class MainThreadLanguageModelTools extends Disposable implements MainThre return Array.from(this._languageModelToolsService.getTools()); } - $invokeTool(name: string, parameters: any, token: CancellationToken): Promise { + $invokeTool(name: string, parameters: any, token: CancellationToken): Promise { return this._languageModelToolsService.invokeTool(name, parameters, token); } diff --git a/src/vs/workbench/api/common/extHost.protocol.ts b/src/vs/workbench/api/common/extHost.protocol.ts index 76f6a658ba6..ad8ed5c219e 100644 --- a/src/vs/workbench/api/common/extHost.protocol.ts +++ b/src/vs/workbench/api/common/extHost.protocol.ts @@ -56,7 +56,7 @@ import { IChatProgressResponseContent } from 'vs/workbench/contrib/chat/common/c import { ChatAgentVoteDirection, IChatFollowup, IChatProgress, IChatResponseErrorDetails, IChatTask, IChatTaskDto, IChatUserActionEvent } from 'vs/workbench/contrib/chat/common/chatService'; import { IChatRequestVariableValue, IChatVariableData, IChatVariableResolverProgress } from 'vs/workbench/contrib/chat/common/chatVariables'; import { IChatMessage, IChatResponseFragment, ILanguageModelChatMetadata, ILanguageModelChatSelector, ILanguageModelsChangeEvent } from 'vs/workbench/contrib/chat/common/languageModels'; -import { IToolData, IToolDelta } from 'vs/workbench/contrib/chat/common/languageModelToolsService'; +import { IToolData, IToolDelta, IToolResult } from 'vs/workbench/contrib/chat/common/languageModelToolsService'; import { DebugConfigurationProviderTriggerKind, IAdapterDescriptor, IConfig, IDebugSessionReplMode, IDebugTestRunReference, IDebugVisualization, IDebugVisualizationContext, IDebugVisualizationTreeItem, MainThreadDebugVisualization } from 'vs/workbench/contrib/debug/common/debug'; import * as notebookCommon from 'vs/workbench/contrib/notebook/common/notebookCommon'; import { CellExecutionUpdateType } from 'vs/workbench/contrib/notebook/common/notebookExecutionService'; @@ -1310,7 +1310,7 @@ export interface MainThreadChatVariablesShape extends IDisposable { export interface MainThreadLanguageModelToolsShape extends IDisposable { $getTools(): Promise[]>; - $invokeTool(name: string, parameters: any, token: CancellationToken): Promise; + $invokeTool(name: string, parameters: any, token: CancellationToken): Promise; $registerTool(id: string): void; $unregisterTool(name: string): void; } @@ -1323,7 +1323,7 @@ export interface ExtHostChatVariablesShape { export interface ExtHostLanguageModelToolsShape { $acceptToolDelta(delta: IToolDelta): Promise; - $invokeTool(id: string, parameters: any, token: CancellationToken): Promise; + $invokeTool(id: string, parameters: any, token: CancellationToken): Promise; } export interface MainThreadUrlsShape extends IDisposable { diff --git a/src/vs/workbench/api/common/extHostLanguageModelTools.ts b/src/vs/workbench/api/common/extHostLanguageModelTools.ts index bd77b53bd44..f383a4ad265 100644 --- a/src/vs/workbench/api/common/extHostLanguageModelTools.ts +++ b/src/vs/workbench/api/common/extHostLanguageModelTools.ts @@ -9,7 +9,7 @@ import { revive } from 'vs/base/common/marshalling'; import { IExtensionDescription } from 'vs/platform/extensions/common/extensions'; import { ExtHostLanguageModelToolsShape, IMainContext, MainContext, MainThreadLanguageModelToolsShape } from 'vs/workbench/api/common/extHost.protocol'; import * as typeConvert from 'vs/workbench/api/common/extHostTypeConverters'; -import { IToolData, IToolDelta } from 'vs/workbench/contrib/chat/common/languageModelToolsService'; +import { IToolData, IToolDelta, IToolResult } from 'vs/workbench/contrib/chat/common/languageModelToolsService'; import type * as vscode from 'vscode'; export class ExtHostLanguageModelTools implements ExtHostLanguageModelToolsShape { @@ -30,9 +30,10 @@ export class ExtHostLanguageModelTools implements ExtHostLanguageModelToolsShape }); } - async invokeTool(name: string, parameters: any, token: CancellationToken): Promise { + async invokeTool(name: string, parameters: any, token: CancellationToken): Promise { // Making the round trip here because not all tools were necessarily registered in this EH - return await this._proxy.$invokeTool(name, parameters, token); + const result = await this._proxy.$invokeTool(name, parameters, token); + return typeConvert.LanguageModelToolResult.to(result); } async $acceptToolDelta(delta: IToolDelta): Promise { @@ -50,13 +51,14 @@ export class ExtHostLanguageModelTools implements ExtHostLanguageModelToolsShape .map(tool => typeConvert.LanguageModelToolDescription.to(tool)); } - async $invokeTool(name: string, parameters: any, token: CancellationToken): Promise { + async $invokeTool(name: string, parameters: any, token: CancellationToken): Promise { const item = this._registeredTools.get(name); if (!item) { throw new Error(`Unknown tool ${name}`); } - return await item.tool.invoke(parameters, token); + const extensionResult = await item.tool.invoke(parameters, token); + return typeConvert.LanguageModelToolResult.from(extensionResult); } registerTool(extension: IExtensionDescription, name: string, tool: vscode.LanguageModelTool): IDisposable { diff --git a/src/vs/workbench/api/common/extHostTypeConverters.ts b/src/vs/workbench/api/common/extHostTypeConverters.ts index 144cd0fcdac..e062e5e0267 100644 --- a/src/vs/workbench/api/common/extHostTypeConverters.ts +++ b/src/vs/workbench/api/common/extHostTypeConverters.ts @@ -53,7 +53,7 @@ import { ACTIVE_GROUP, SIDE_GROUP } from 'vs/workbench/services/editor/common/ed import { Dto } from 'vs/workbench/services/extensions/common/proxyIdentifier'; import type * as vscode from 'vscode'; import * as types from './extHostTypes'; -import { IToolData } from 'vs/workbench/contrib/chat/common/languageModelToolsService'; +import { IToolData, IToolResult } from 'vs/workbench/contrib/chat/common/languageModelToolsService'; export namespace Command { @@ -2724,6 +2724,24 @@ export namespace ChatAgentUserActionEvent { } } +export namespace LanguageModelToolResult { + export function from(result: vscode.LanguageModelToolResult): IToolResult { + return { + ...result, + string: result.toString(), + }; + } + + export function to(result: IToolResult): vscode.LanguageModelToolResult { + const copy: vscode.LanguageModelToolResult = { + ...result, + toString: () => result.string, + }; + delete copy.string; + + return copy; + } +} export namespace TerminalQuickFix { export function from(quickFix: vscode.TerminalQuickFixTerminalCommand | vscode.TerminalQuickFixOpener | vscode.Command, converter: Command.ICommandsConverter, disposables: DisposableStore): extHostProtocol.ITerminalQuickFixTerminalCommandDto | extHostProtocol.ITerminalQuickFixOpenerDto | extHostProtocol.ICommandDto | undefined { diff --git a/src/vs/workbench/contrib/chat/common/languageModelToolsService.ts b/src/vs/workbench/contrib/chat/common/languageModelToolsService.ts index 09e8903a15f..ad5866a8fa4 100644 --- a/src/vs/workbench/contrib/chat/common/languageModelToolsService.ts +++ b/src/vs/workbench/contrib/chat/common/languageModelToolsService.ts @@ -27,8 +27,13 @@ interface IToolEntry { impl?: IToolImpl; } +export interface IToolResult { + [contentType: string]: any; + string: string; +} + export interface IToolImpl { - invoke(parameters: any, token: CancellationToken): Promise; + invoke(parameters: any, token: CancellationToken): Promise; } export const ILanguageModelToolsService = createDecorator('ILanguageModelToolsService'); @@ -44,7 +49,7 @@ export interface ILanguageModelToolsService { registerToolData(toolData: IToolData): IDisposable; registerToolImplementation(name: string, tool: IToolImpl): IDisposable; getTools(): Iterable>; - invokeTool(name: string, parameters: any, token: CancellationToken): Promise; + invokeTool(name: string, parameters: any, token: CancellationToken): Promise; } export class LanguageModelToolsService implements ILanguageModelToolsService { @@ -94,7 +99,7 @@ export class LanguageModelToolsService implements ILanguageModelToolsService { return Iterable.map(this._tools.values(), i => i.data); } - async invokeTool(name: string, parameters: any, token: CancellationToken): Promise { + async invokeTool(name: string, parameters: any, token: CancellationToken): Promise { let tool = this._tools.get(name); if (!tool) { throw new Error(`Tool ${name} was not contributed`); diff --git a/src/vscode-dts/vscode.proposed.chatVariableResolver.d.ts b/src/vscode-dts/vscode.proposed.chatVariableResolver.d.ts index 62f07234bb1..ec386ec928c 100644 --- a/src/vscode-dts/vscode.proposed.chatVariableResolver.d.ts +++ b/src/vscode-dts/vscode.proposed.chatVariableResolver.d.ts @@ -77,23 +77,6 @@ declare module 'vscode' { Full = 3 } - export interface ChatVariableValue { - /** - * The detail level of this chat variable value. If possible, variable resolvers should try to offer shorter values that will consume fewer tokens in an LLM prompt. - */ - level: ChatVariableLevel; - - /** - * The variable's value, which can be included in an LLM prompt as-is, or the chat participant may decide to read the value and do something else with it. - */ - value: string | Uri; - - /** - * A description of this value, which could be provided to the LLM as a hint. - */ - description?: string; - } - export interface ChatVariableResolverResponseStream { /** * Push a progress part to this stream. Short-hand for diff --git a/src/vscode-dts/vscode.proposed.lmTools.d.ts b/src/vscode-dts/vscode.proposed.lmTools.d.ts index 6ba1a489db8..42711e0b0f6 100644 --- a/src/vscode-dts/vscode.proposed.lmTools.d.ts +++ b/src/vscode-dts/vscode.proposed.lmTools.d.ts @@ -3,7 +3,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -// version: 2 +// version: 3 // https://github.com/microsoft/vscode/issues/213274 declare module 'vscode' { @@ -64,6 +64,18 @@ declare module 'vscode' { content2: string | LanguageModelChatMessageFunctionResultPart; } + export interface LanguageModelToolResult { + /** + * The result can contain arbitrary representations of the content. An example might be 'prompt-tsx' to indicate an element that can be rendered with the @vscode/prompt-tsx library. + */ + [contentType: string]: any; + + /** + * A string representation of the result which can be incorporated back into an LLM prompt without any special handling. + */ + toString(): string; + } + // Tool registration/invoking between extensions export namespace lm { @@ -79,8 +91,9 @@ declare module 'vscode' { /** * Invoke a tool with the given parameters. + * TODO@API Could request a set of contentTypes to be returned so they don't all need to be computed? */ - export function invokeTool(name: string, parameters: Object, token: CancellationToken): Thenable; + export function invokeTool(name: string, parameters: Object, token: CancellationToken): Thenable; } // Is the same as LanguageModelChatFunction now, but could have more details in the future @@ -91,6 +104,7 @@ declare module 'vscode' { } export interface LanguageModelTool { - invoke(parameters: any, token: CancellationToken): Thenable; + // TODO@API should it be LanguageModelToolResult | string? + invoke(parameters: any, token: CancellationToken): Thenable; } }