diff --git a/src/vs/platform/extensions/common/extensionsApiProposals.ts b/src/vs/platform/extensions/common/extensionsApiProposals.ts index 4267b5062af..6b89bc737ef 100644 --- a/src/vs/platform/extensions/common/extensionsApiProposals.ts +++ b/src/vs/platform/extensions/common/extensionsApiProposals.ts @@ -244,7 +244,7 @@ const _allApiProposals = { }, lmTools: { proposal: 'https://raw.githubusercontent.com/microsoft/vscode/main/src/vscode-dts/vscode.proposed.lmTools.d.ts', - version: 9 + version: 10 }, mappedEditsProvider: { proposal: 'https://raw.githubusercontent.com/microsoft/vscode/main/src/vscode-dts/vscode.proposed.mappedEditsProvider.d.ts', diff --git a/src/vs/workbench/api/common/extHost.api.impl.ts b/src/vs/workbench/api/common/extHost.api.impl.ts index bfce6627972..7e52f48078b 100644 --- a/src/vs/workbench/api/common/extHost.api.impl.ts +++ b/src/vs/workbench/api/common/extHost.api.impl.ts @@ -1495,9 +1495,9 @@ export function createApiFactoryAndRegisterActors(accessor: ServicesAccessor): I checkProposedApiEnabled(extension, 'lmTools'); return extHostLanguageModelTools.registerTool(extension, name, tool); }, - invokeTool(toolId: string, parameters: vscode.LanguageModelToolInvocationOptions, token: vscode.CancellationToken) { + invokeTool(name: string, parameters: vscode.LanguageModelToolInvocationOptions, token: vscode.CancellationToken) { checkProposedApiEnabled(extension, 'lmTools'); - return extHostLanguageModelTools.invokeTool(toolId, parameters, token); + return extHostLanguageModelTools.invokeTool(name, parameters, token); }, get tools() { checkProposedApiEnabled(extension, 'lmTools'); @@ -1780,6 +1780,9 @@ export function createApiFactoryAndRegisterActors(accessor: ServicesAccessor): I LanguageModelTextPart: extHostTypes.LanguageModelTextPart, LanguageModelToolCallPart: extHostTypes.LanguageModelToolCallPart, LanguageModelError: extHostTypes.LanguageModelError, + LanguageModelToolResult: extHostTypes.LanguageModelToolResult, + LanguageModelChatToolMode: extHostTypes.LanguageModelChatToolMode, + LanguageModelPromptTsxPart: extHostTypes.LanguageModelPromptTsxPart, NewSymbolName: extHostTypes.NewSymbolName, NewSymbolNameTag: extHostTypes.NewSymbolNameTag, NewSymbolNameTriggerKind: extHostTypes.NewSymbolNameTriggerKind, diff --git a/src/vs/workbench/api/common/extHostLanguageModelTools.ts b/src/vs/workbench/api/common/extHostLanguageModelTools.ts index 1fd511758bf..dd0ba4f95d8 100644 --- a/src/vs/workbench/api/common/extHostLanguageModelTools.ts +++ b/src/vs/workbench/api/common/extHostLanguageModelTools.ts @@ -44,13 +44,9 @@ export class ExtHostLanguageModelTools implements ExtHostLanguageModelToolsShape } async invokeTool(toolId: string, options: vscode.LanguageModelToolInvocationOptions, token: CancellationToken): Promise { - if (!options.requestedContentTypes?.length) { - throw new Error('LanguageModelToolInvocationOptions.requestedContentTypes is required to be set'); - } - const callId = generateUuid(); - if (options.tokenOptions) { - this._tokenCountFuncs.set(callId, options.tokenOptions.countTokens); + if (options.tokenizationOptions) { + this._tokenCountFuncs.set(callId, options.tokenizationOptions.countTokens); } try { // Making the round trip here because not all tools were necessarily registered in this EH @@ -58,11 +54,10 @@ export class ExtHostLanguageModelTools implements ExtHostLanguageModelToolsShape toolId, callId, parameters: options.parameters, - tokenBudget: options.tokenOptions?.tokenBudget, + tokenBudget: options.tokenizationOptions?.tokenBudget, context: options.toolInvocationToken as IToolInvocationContext | undefined, - requestedContentTypes: options.requestedContentTypes, }, token); - return result; + return typeConvert.LanguageModelToolResult.to(result); } finally { this._tokenCountFuncs.delete(callId); } @@ -75,7 +70,7 @@ export class ExtHostLanguageModelTools implements ExtHostLanguageModelToolsShape } } - get tools(): vscode.LanguageModelToolDescription[] { + get tools(): vscode.LanguageModelToolInformation[] { return Array.from(this._allTools.values()) .map(tool => typeConvert.LanguageModelToolDescription.to(tool)); } @@ -86,9 +81,9 @@ export class ExtHostLanguageModelTools implements ExtHostLanguageModelToolsShape throw new Error(`Unknown tool ${dto.toolId}`); } - const options: vscode.LanguageModelToolInvocationOptions = { parameters: dto.parameters, toolInvocationToken: dto.context, requestedContentTypes: dto.requestedContentTypes }; + const options: vscode.LanguageModelToolInvocationOptions = { parameters: dto.parameters, toolInvocationToken: dto.context }; if (dto.tokenBudget !== undefined) { - options.tokenOptions = { + options.tokenizationOptions = { tokenBudget: dto.tokenBudget, countTokens: this._tokenCountFuncs.get(dto.callId) || ((value, token = CancellationToken.None) => this._proxy.$countTokensForInvocation(dto.callId, value, token)) @@ -106,19 +101,7 @@ export class ExtHostLanguageModelTools implements ExtHostLanguageModelToolsShape throw new CancellationError(); } - for (const key of Object.keys(extensionResult)) { - const value = extensionResult[key]; - if (value instanceof Promise) { - throw new Error(`Tool result for '${key}' cannot be a Promise`); - } else if (!options.requestedContentTypes.includes(key) && key !== 'toString') { - // This could help the scenario where a tool updated the prompt-tsx library, but did not update the contentType in package.json. - // Or, where a tool author didn't declare supportedContentTypes and isn't checking the list of requestedContentTypes. - // toString check can be temp, just to help with tools that are already published. - throw new Error(`Tool result for '${key}' was not requested from ${dto.toolId}.`); - } - } - - return extensionResult; + return typeConvert.LanguageModelToolResult.from(extensionResult); } async $prepareToolInvocation(toolId: string, parameters: any, token: CancellationToken): Promise { @@ -127,11 +110,11 @@ export class ExtHostLanguageModelTools implements ExtHostLanguageModelToolsShape throw new Error(`Unknown tool ${toolId}`); } - if (!item.tool.prepareToolInvocation) { + if (!item.tool.prepareInvocation) { return undefined; } - const result = await item.tool.prepareToolInvocation({ parameters }, token); + const result = await item.tool.prepareInvocation({ parameters }, token); if (!result) { return undefined; } diff --git a/src/vs/workbench/api/common/extHostLanguageModels.ts b/src/vs/workbench/api/common/extHostLanguageModels.ts index 2e8bbc88575..5fa89a2e915 100644 --- a/src/vs/workbench/api/common/extHostLanguageModels.ts +++ b/src/vs/workbench/api/common/extHostLanguageModels.ts @@ -204,7 +204,7 @@ export class ExtHostLanguageModels implements ExtHostLanguageModelsShape { let part: IChatResponsePart | undefined; if (fragment.part instanceof extHostTypes.LanguageModelToolCallPart) { - part = { type: 'tool_use', name: fragment.part.name, parameters: fragment.part.parameters, toolCallId: fragment.part.toolCallId }; + part = { type: 'tool_use', name: fragment.part.name, parameters: fragment.part.parameters, toolCallId: fragment.part.callId }; } else if (fragment.part instanceof extHostTypes.LanguageModelTextPart) { part = { type: 'text', value: fragment.part.value }; } diff --git a/src/vs/workbench/api/common/extHostTypeConverters.ts b/src/vs/workbench/api/common/extHostTypeConverters.ts index 8b84d0ad41c..5fb3f8c0fa9 100644 --- a/src/vs/workbench/api/common/extHostTypeConverters.ts +++ b/src/vs/workbench/api/common/extHostTypeConverters.ts @@ -39,7 +39,7 @@ import { IViewBadge } from '../../common/views.js'; import { ChatAgentLocation, IChatAgentRequest, IChatAgentResult } from '../../contrib/chat/common/chatAgents.js'; import { IChatRequestVariableEntry } from '../../contrib/chat/common/chatModel.js'; import { IChatAgentDetection, IChatAgentMarkdownContentWithVulnerability, IChatCodeCitation, IChatCommandButton, IChatConfirmation, IChatContentInlineReference, IChatContentReference, IChatFollowup, IChatMarkdownContent, IChatMoveMessage, IChatProgressMessage, IChatResponseCodeblockUriPart, IChatTaskDto, IChatTaskResult, IChatTextEdit, IChatTreeData, IChatUserActionEvent, IChatWarningMessage } from '../../contrib/chat/common/chatService.js'; -import { IToolData } from '../../contrib/chat/common/languageModelToolsService.js'; +import { IToolData, IToolResult } from '../../contrib/chat/common/languageModelToolsService.js'; import * as chatProvider from '../../contrib/chat/common/languageModels.js'; import { DebugTreeItemCollapsibleState, IDebugVisualizationTreeItem } from '../../contrib/debug/common/debug.js'; import * as notebooks from '../../contrib/notebook/common/notebookCommon.js'; @@ -54,6 +54,8 @@ import * as extHostProtocol from './extHost.protocol.js'; import { CommandsConverter } from './extHostCommands.js'; import { getPrivateApiFor } from './extHostTestingPrivateApi.js'; import * as types from './extHostTypes.js'; +import { IChatResponseTextPart, IChatResponsePromptTsxPart } from '../../contrib/chat/common/languageModels.js'; +import { LanguageModelTextPart, LanguageModelPromptTsxPart } from './extHostTypes.js'; export namespace Command { @@ -2349,7 +2351,14 @@ export namespace LanguageModelChatMessage { if (c.type === 'text') { return c.value; } else if (c.type === 'tool_result') { - return new types.LanguageModelToolResultPart(c.toolCallId, c.value, c.isError); + const content: (LanguageModelTextPart | LanguageModelPromptTsxPart)[] = c.value.map(part => { + if (part.type === 'text') { + return new types.LanguageModelTextPart(part.value); + } else { + return new types.LanguageModelPromptTsxPart(part.value, part.mime); + } + }); + return new types.LanguageModelToolResultPart(c.toolCallId, content, c.isError); } else { return new types.LanguageModelToolCallPart(c.name, c.toolCallId, c.parameters); } @@ -2370,14 +2379,30 @@ export namespace LanguageModelChatMessage { if (c instanceof types.LanguageModelToolResultPart) { return { type: 'tool_result', - toolCallId: c.toolCallId, - value: c.content, + toolCallId: c.callId, + value: coalesce(c.content.map(part => { + if (part instanceof types.LanguageModelTextPart) { + return { + type: 'text', + value: part.value + } satisfies IChatResponseTextPart; + } else if (part instanceof types.LanguageModelPromptTsxPart) { + return { + type: 'prompt_tsx', + value: part.value, + mime: part.mime + } satisfies IChatResponsePromptTsxPart; + } else { + // Strip unknown parts + return undefined; + } + })), isError: c.isError }; } else if (c instanceof types.LanguageModelToolCallPart) { return { type: 'tool_use', - toolCallId: c.toolCallId, + toolCallId: c.callId, name: c.name, parameters: c.parameters }; @@ -2809,7 +2834,7 @@ export namespace ChatLanguageModelToolReference { } return { - id: variable.id, + name: variable.id, range: variable.range && [variable.range.start, variable.range.endExclusive], }; } @@ -2925,14 +2950,46 @@ export namespace DebugTreeItem { } export namespace LanguageModelToolDescription { - export function to(item: IToolData): vscode.LanguageModelToolDescription { + export function to(item: IToolData): vscode.LanguageModelToolInformation { return { // Note- the reason this is a unique 'name' is just to avoid confusion with the toolCallId name: item.id, description: item.modelDescription, parametersSchema: item.parametersSchema, - supportedContentTypes: item.supportedContentTypes, tags: item.tags ?? [], }; } } + +export namespace LanguageModelToolResult { + export function to(result: IToolResult): vscode.LanguageModelToolResult { + return new types.LanguageModelToolResult(result.content.map(item => { + if (item.kind === 'text') { + return new types.LanguageModelTextPart(item.value); + } else { + return new types.LanguageModelPromptTsxPart(item.value, item.mime); + } + })); + } + + export function from(result: vscode.LanguageModelToolResult): IToolResult { + return { + content: result.content.map(item => { + if (item instanceof types.LanguageModelTextPart) { + return { + kind: 'text', + value: item.value + }; + } else if (item instanceof types.LanguageModelPromptTsxPart) { + return { + kind: 'promptTsx', + value: item.value, + mime: item.mime + }; + } else { + throw new Error('Unknown LanguageModelToolResult part type'); + } + }) + }; + } +} diff --git a/src/vs/workbench/api/common/extHostTypes.ts b/src/vs/workbench/api/common/extHostTypes.ts index f50208a09ad..eeaf2db8d5c 100644 --- a/src/vs/workbench/api/common/extHostTypes.ts +++ b/src/vs/workbench/api/common/extHostTypes.ts @@ -4585,12 +4585,12 @@ export enum LanguageModelChatMessageRole { export class LanguageModelToolResultPart implements vscode.LanguageModelToolResultPart { - toolCallId: string; - content: string; + callId: string; + content: (LanguageModelTextPart | LanguageModelPromptTsxPart | unknown)[]; isError: boolean; - constructor(toolCallId: string, content: string, isError?: boolean) { - this.toolCallId = toolCallId; + constructor(callId: string, content: (LanguageModelTextPart | LanguageModelPromptTsxPart | unknown)[], isError?: boolean) { + this.callId = callId; this.content = content; this.isError = isError ?? false; } @@ -4623,12 +4623,12 @@ export class LanguageModelChatMessage implements vscode.LanguageModelChatMessage export class LanguageModelToolCallPart implements vscode.LanguageModelToolCallPart { name: string; - toolCallId: string; + callId: string; parameters: any; constructor(name: string, toolCallId: string, parameters: any) { this.name = name; - this.toolCallId = toolCallId; + this.callId = toolCallId; this.parameters = parameters; } } @@ -4638,7 +4638,16 @@ export class LanguageModelTextPart implements vscode.LanguageModelTextPart { constructor(value: string) { this.value = value; + } +} +export class LanguageModelPromptTsxPart { + value: unknown; + mime: string; + + constructor(value: unknown, mime: string) { + this.value = value; + this.mime = mime; } } @@ -4703,6 +4712,15 @@ export class LanguageModelError extends Error { } +export class LanguageModelToolResult { + constructor(public content: (LanguageModelTextPart | LanguageModelPromptTsxPart | unknown)[]) { } +} + +export enum LanguageModelChatToolMode { + Auto = 1, + Required = 2 +} + //#endregion //#region ai diff --git a/src/vs/workbench/contrib/chat/common/languageModelToolsService.ts b/src/vs/workbench/contrib/chat/common/languageModelToolsService.ts index ab5f6d1ae23..0000ce531d9 100644 --- a/src/vs/workbench/contrib/chat/common/languageModelToolsService.ts +++ b/src/vs/workbench/contrib/chat/common/languageModelToolsService.ts @@ -24,7 +24,6 @@ export interface IToolData { modelDescription: string; parametersSchema?: IJSONSchema; canBeReferencedInPrompt?: boolean; - supportedContentTypes: string[]; } export interface IToolInvocation { @@ -33,7 +32,6 @@ export interface IToolInvocation { parameters: Object; tokenBudget?: number; context: IToolInvocationContext | undefined; - requestedContentTypes: string[]; } export interface IToolInvocationContext { @@ -41,7 +39,18 @@ export interface IToolInvocationContext { } export interface IToolResult { - [contentType: string]: any; + content: (IToolResultPromptTsxPart | IToolResultTextPart)[]; +} + +export interface IToolResultPromptTsxPart { + kind: 'promptTsx'; + mime: string; + value: unknown; +} + +export interface IToolResultTextPart { + kind: 'text'; + value: string; } export interface IToolConfirmationMessages { diff --git a/src/vs/workbench/contrib/chat/common/languageModels.ts b/src/vs/workbench/contrib/chat/common/languageModels.ts index ee5b7d181c8..b8e49d63123 100644 --- a/src/vs/workbench/contrib/chat/common/languageModels.ts +++ b/src/vs/workbench/contrib/chat/common/languageModels.ts @@ -32,7 +32,7 @@ export interface IChatMessageTextPart { export interface IChatMessageToolResultPart { type: 'tool_result'; toolCallId: string; - value: any; + value: (IChatResponseTextPart | IChatResponsePromptTsxPart)[]; isError?: boolean; } @@ -49,6 +49,12 @@ export interface IChatResponseTextPart { value: string; } +export interface IChatResponsePromptTsxPart { + type: 'prompt_tsx'; + value: unknown; + mime: string; +} + export interface IChatResponseToolUsePart { type: 'tool_use'; name: string; diff --git a/src/vs/workbench/contrib/chat/common/tools/languageModelToolsContribution.ts b/src/vs/workbench/contrib/chat/common/tools/languageModelToolsContribution.ts index 36df5b60125..90a9293489b 100644 --- a/src/vs/workbench/contrib/chat/common/tools/languageModelToolsContribution.ts +++ b/src/vs/workbench/contrib/chat/common/tools/languageModelToolsContribution.ts @@ -27,7 +27,6 @@ interface IRawToolContribution { userDescription?: string; parametersSchema?: IJSONSchema; canBeReferencedInPrompt?: boolean; - supportedContentTypes?: string[]; } const languageModelToolsExtensionPoint = extensionsRegistry.ExtensionsRegistry.registerExtensionPoint({ @@ -43,7 +42,21 @@ const languageModelToolsExtensionPoint = extensionsRegistry.ExtensionsRegistry.r items: { additionalProperties: false, type: 'object', - defaultSnippets: [{ body: { name: '', description: '' } }], + defaultSnippets: [{ + body: { + name: '${1}', + modelDescription: '${2}', + parametersSchema: { + type: 'object', + properties: { + '${3:paramName}': { + type: 'string', + description: '${4:description}' + } + } + }, + } + }], required: ['name', 'displayName', 'modelDescription'], properties: { name: { @@ -101,13 +114,6 @@ const languageModelToolsExtensionPoint = extensionsRegistry.ExtensionsRegistry.r markdownDescription: localize('condition', "Condition which must be true for this tool to be enabled. Note that a tool may still be invoked by another extension even when its `when` condition is false."), type: 'string' }, - supportedContentTypes: { - markdownDescription: localize('contentTypes', "The list of content types that this tool can return. It's recommended that all tools support `text/plain`, which would indicate any text-based content. Another example is the contentType exported by the `@vscode/prompt-tsx` library, which would let a tool return a `PromptElementJSON` which can be easily rendered in a prompt by an extension using `@vscode/prompt-tsx`."), - type: 'array', - items: { - type: 'string' - } - }, tags: { description: localize('toolTags', "A set of tags that roughly describe the tool's capabilities. A tool user may use these to filter the set of tools to just ones that are relevant for the task at hand."), type: 'array', @@ -170,7 +176,6 @@ export class LanguageModelToolsExtensionPointHandler implements IWorkbenchContri id: rawTool.name, icon, when: rawTool.when ? ContextKeyExpr.deserialize(rawTool.when) : undefined, - supportedContentTypes: rawTool.supportedContentTypes ? rawTool.supportedContentTypes : [], }; const disposable = languageModelToolsService.registerToolData(tool); this._registrationDisposables.set(toToolKey(extension.description.identifier, rawTool.name), disposable); diff --git a/src/vs/workbench/contrib/chat/test/browser/languageModelToolsService.test.ts b/src/vs/workbench/contrib/chat/test/browser/languageModelToolsService.test.ts index 2bd4519971f..48e21693f02 100644 --- a/src/vs/workbench/contrib/chat/test/browser/languageModelToolsService.test.ts +++ b/src/vs/workbench/contrib/chat/test/browser/languageModelToolsService.test.ts @@ -31,7 +31,6 @@ suite('LanguageModelToolsService', () => { const toolData: IToolData = { id: 'testTool', modelDescription: 'Test Tool', - supportedContentTypes: [], displayName: 'Test Tool' }; @@ -45,14 +44,13 @@ suite('LanguageModelToolsService', () => { const toolData: IToolData = { id: 'testTool', modelDescription: 'Test Tool', - supportedContentTypes: [], displayName: 'Test Tool' }; store.add(service.registerToolData(toolData)); const toolImpl: IToolImpl = { - invoke: async () => ({ 'text/plain': 'result' }), + invoke: async () => ({ content: [{ kind: 'text', value: 'result' }] }), }; store.add(service.registerToolImplementation('testTool', toolImpl)); @@ -65,7 +63,6 @@ suite('LanguageModelToolsService', () => { id: 'testTool1', modelDescription: 'Test Tool 1', when: ContextKeyEqualsExpr.create('testKey', false), - supportedContentTypes: [], displayName: 'Test Tool' }; @@ -73,14 +70,12 @@ suite('LanguageModelToolsService', () => { id: 'testTool2', modelDescription: 'Test Tool 2', when: ContextKeyEqualsExpr.create('testKey', true), - supportedContentTypes: [], displayName: 'Test Tool' }; const toolData3: IToolData = { id: 'testTool3', modelDescription: 'Test Tool 3', - supportedContentTypes: [], displayName: 'Test Tool' }; @@ -98,7 +93,6 @@ suite('LanguageModelToolsService', () => { const toolData: IToolData = { id: 'testTool', modelDescription: 'Test Tool', - supportedContentTypes: [], displayName: 'Test Tool' }; @@ -109,7 +103,7 @@ suite('LanguageModelToolsService', () => { assert.strictEqual(invocation.callId, '1'); assert.strictEqual(invocation.toolId, 'testTool'); assert.deepStrictEqual(invocation.parameters, { a: 1 }); - return { 'text/plain': 'result' }; + return { content: [{ kind: 'text', value: 'result' }] }; } }; @@ -123,10 +117,9 @@ suite('LanguageModelToolsService', () => { a: 1 }, context: undefined, - requestedContentTypes: ['text/plain'] }; const result = await service.invokeTool(dto, async () => 0, CancellationToken.None); - assert.strictEqual(result['text/plain'], 'result'); + assert.strictEqual(result.content[0].value, 'result'); }); }); diff --git a/src/vs/workbench/contrib/chat/test/common/mockLanguageModelToolsService.ts b/src/vs/workbench/contrib/chat/test/common/mockLanguageModelToolsService.ts index 57e5eca8540..a192334cb9d 100644 --- a/src/vs/workbench/contrib/chat/test/common/mockLanguageModelToolsService.ts +++ b/src/vs/workbench/contrib/chat/test/common/mockLanguageModelToolsService.ts @@ -37,7 +37,7 @@ export class MockLanguageModelToolsService implements ILanguageModelToolsService async invokeTool(dto: IToolInvocation, countTokens: CountTokensCallback, token: CancellationToken): Promise { return { - string: '' + content: [{ kind: 'text', value: 'result' }] }; } } diff --git a/src/vscode-dts/vscode.d.ts b/src/vscode-dts/vscode.d.ts index ec203e749da..676ca798aa4 100644 --- a/src/vscode-dts/vscode.d.ts +++ b/src/vscode-dts/vscode.d.ts @@ -19475,8 +19475,8 @@ declare module 'vscode' { * Make a chat request using a language model. * * *Note* that language model use may be subject to access restrictions and user consent. Calling this function - * for the first time (for a extension) will show a consent dialog to the user and because of that this function - * must _only be called in response to a user action!_ Extension can use {@link LanguageModelAccessInformation.canSendRequest} + * for the first time (for an extension) will show a consent dialog to the user and because of that this function + * must _only be called in response to a user action!_ Extensions can use {@link LanguageModelAccessInformation.canSendRequest} * to check if they have the necessary permissions to make a request. * * This function will return a rejected promise if making a request to the language model is not @@ -19487,6 +19487,10 @@ declare module 'vscode' { * - quota limits exceeded, see {@link LanguageModelError.Blocked `Blocked`} * - other issues in which case extension must check {@link LanguageModelError.cause `LanguageModelError.cause`} * + * An extension can make use of language model tool calling by passing a set of tools to + * {@link LanguageModelChatRequestOptions.tools}. The language model will return a {@link LanguageModelToolCallPart} and + * the extension can invoke the tool and make another request with the result. + * * @param messages An array of message instances. * @param options Options that control the request. * @param token A cancellation token which controls the request. See {@link CancellationTokenSource} for how to create one. diff --git a/src/vscode-dts/vscode.proposed.lmTools.d.ts b/src/vscode-dts/vscode.proposed.lmTools.d.ts index aa4f1722e54..f96f821e447 100644 --- a/src/vscode-dts/vscode.proposed.lmTools.d.ts +++ b/src/vscode-dts/vscode.proposed.lmTools.d.ts @@ -3,13 +3,14 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -// version: 9 +// version: 10 // https://github.com/microsoft/vscode/issues/213274 declare module 'vscode' { /** - * A tool that is available to the language model via {@link LanguageModelChatRequestOptions}. + * A tool that is available to the language model via {@link LanguageModelChatRequestOptions}. A language model uses all the + * properties of this interface to decide which tool to call, and how to call it. */ export interface LanguageModelChatTool { /** @@ -28,24 +29,47 @@ declare module 'vscode' { parametersSchema?: object; } + /** + * A tool-calling mode for the language model to use. + */ + export enum LanguageModelChatToolMode { + /** + * The language model can choose to call a tool or generate a message. Is the default. + */ + Auto = 1, + + /** + * The language model must call one of the provided tools. Note- some models only support a single tool when using this + * mode. TODO@API - do we throw, or just pick the first tool? Or only offer an API that allows callers to pick a single + * tool? Go back to `toolChoice?: string`? + */ + Required = 2 + } + export interface LanguageModelChatRequestOptions { /** - * An optional list of tools that are available to the language model. + * An optional list of tools that are available to the language model. These could be registered tools available via + * {@link lm.tools}, or private tools that are just implemented within the calling extension. + * + * If the LLM requests to call one of these tools, it will return a {@link LanguageModelToolCallPart} in + * {@link LanguageModelChatResponse.stream}. It's the caller's responsibility to invoke the tool. If it's a tool + * registered in {@link lm.tools}, that means calling {@link lm.invokeTool}. + * + * Then, the tool result can be provided to the LLM by creating an Assistant-type {@link LanguageModelChatMessage} with a + * {@link LanguageModelToolCallPart}, followed by a User-type message with a {@link LanguageModelToolResultPart}. */ tools?: LanguageModelChatTool[]; /** - * Force a specific tool to be used. + * The tool-selecting mode to use. {@link LanguageModelChatToolMode.Auto} by default. */ - // TODO@API? - toolChoice?: string; + toolMode?: LanguageModelChatToolMode; } /** * A language model response part indicating a tool call, returned from a {@link LanguageModelChatResponse}, and also can be - * included as a content part on a {@link LanguageModelChatMessage}, to represent a previous tool call in a - * chat request. + * included as a content part on a {@link LanguageModelChatMessage}, to represent a previous tool call in a chat request. */ export class LanguageModelToolCallPart { /** @@ -56,15 +80,17 @@ declare module 'vscode' { /** * The ID of the tool call. This is a unique identifier for the tool call within the chat request. */ - // TODO@API name callId - toolCallId: string; + callId: string; /** * The parameters with which to call the tool. */ parameters: object; - constructor(name: string, toolCallId: string, parameters: object); + /** + * Create a new LanguageModelToolCallPart. + */ + constructor(name: string, callId: string, parameters: object); } /** @@ -79,6 +105,23 @@ declare module 'vscode' { constructor(value: string); } + /** + * A language model response part containing a PromptElementJSON from `@vscode/prompt-tsx`. + */ + export class LanguageModelPromptTsxPart { + /** + * The content of the part. + */ + value: unknown; + + /** + * The mimeType of this part, exported from the `@vscode/prompt-tsx` library. + */ + mime: string; + + constructor(value: unknown, mime: string); + } + export interface LanguageModelChatResponse { /** * A stream of parts that make up the response. Could be extended with more types in the future. @@ -93,15 +136,14 @@ declare module 'vscode' { /** * The ID of the tool call. */ - // TODO@API name callId - toolCallId: string; + callId: string; /** - * The content of the tool result. + * The value of the tool result. */ - content: string; + content: (LanguageModelTextPart | LanguageModelPromptTsxPart | unknown)[]; - constructor(toolCallId: string, content: string); + constructor(callId: string, content: (LanguageModelTextPart | LanguageModelPromptTsxPart | unknown)[]); } export interface LanguageModelChatMessage { @@ -112,21 +154,13 @@ declare module 'vscode' { content2: (string | LanguageModelToolResultPart | LanguageModelToolCallPart)[]; } - // Tool registration/invoking between extensions - /** * A result returned from a tool invocation. */ - // TODO@API should we align this with NotebookCellOutput and NotebookCellOutputItem - export interface LanguageModelToolResult { - /** - * The result can contain arbitrary representations of the content. A tool user can set - * {@link LanguageModelToolInvocationOptions.requested} to request particular types, and a tool implementation should only - * compute the types that were requested. `text/plain` is recommended to be supported by all tools, which would indicate - * any text-based content. Another example might be a `PromptElementJSON` from `@vscode/prompt-tsx`, using the - * `contentType` exported by that library. - */ - [contentType: string]: any; + export class LanguageModelToolResult { + content: (LanguageModelTextPart | LanguageModelPromptTsxPart | unknown)[]; + + constructor(content: (LanguageModelTextPart | LanguageModelPromptTsxPart | unknown)[]); } export namespace lm { @@ -140,14 +174,13 @@ declare module 'vscode' { /** * A list of all available tools. */ - // TODO@API nit `readonly LanguageModelToolDescription[]` - export const tools: ReadonlyArray; + export const tools: readonly LanguageModelToolInformation[]; /** * Invoke a tool with the given parameters. + * TODO@API describe tool calling flow here, LanguageModelToolInvocationOptions */ - // TODO@API name - export function invokeTool(id: string, options: LanguageModelToolInvocationOptions, token: CancellationToken): Thenable; + export function invokeTool(name: string, options: LanguageModelToolInvocationOptions, token: CancellationToken): Thenable; } /** @@ -171,39 +204,36 @@ declare module 'vscode' { /** * The parameters with which to invoke the tool. The parameters must match the schema defined in - * {@link LanguageModelToolDescription.parametersSchema} + * {@link LanguageModelToolInformation.parametersSchema} */ parameters: T; /** - * A tool user can request that particular content types be returned from the tool, depending on what the tool user - * supports. All tools are recommended to support `text/plain`. See {@link LanguageModelToolResult}. + * Options to hint at how many tokens the tool should return in its response, and enable the tool to count tokens + * accurately. */ - requestedContentTypes: string[]; + tokenizationOptions?: LanguageModelToolTokenizationOptions; + } + + export interface LanguageModelToolTokenizationOptions { + /** + * If known, the maximum number of tokens the tool should emit in its result. + */ + tokenBudget: number; /** - * Options to hint at how many tokens the tool should return in its response. + * Count the number of tokens in a message using the model specific tokenizer-logic. + * @param text A string. + * @param token Optional cancellation token. See {@link CancellationTokenSource} for how to create one. + * @returns A thenable that resolves to the number of tokens. */ - tokenOptions?: { - /** - * If known, the maximum number of tokens the tool should emit in its result. - */ - tokenBudget: number; - - /** - * Count the number of tokens in a message using the model specific tokenizer-logic. - * @param text A string. - * @param token Optional cancellation token. See {@link CancellationTokenSource} for how to create one. - * @returns A thenable that resolves to the number of tokens. - */ - countTokens(text: string, token?: CancellationToken): Thenable; - }; + countTokens(text: string, token?: CancellationToken): Thenable; } /** - * A description of an available tool. + * Information about a registered tool available in {@link lm.tools}. */ - export interface LanguageModelToolDescription { + export interface LanguageModelToolInformation { /** * A unique name for the tool. */ @@ -217,20 +247,13 @@ declare module 'vscode' { /** * A JSON schema for the parameters this tool accepts. */ - // TODO@API put simple sample in snippet - readonly parametersSchema?: object; - - /** - * The list of content types that the tool has declared support for. See {@link LanguageModelToolResult}. - */ - // TODO@API put text/plain as default in snippet - readonly supportedContentTypes: string[]; + readonly parametersSchema: object | undefined; /** * A set of tags, declared by the tool, that roughly describe the tool's capabilities. A tool user may use these to filter * the set of tools to just ones that are relevant for the task at hand. */ - readonly tags: string[]; + readonly tags: readonly string[]; } /** @@ -250,7 +273,7 @@ declare module 'vscode' { } /** - * Options for {@link LanguageModelTool.prepareToolInvocation}. + * Options for {@link LanguageModelTool.prepareInvocation}. */ export interface LanguageModelToolInvocationPrepareOptions { /** @@ -266,21 +289,21 @@ declare module 'vscode' { /** * Invoke the tool with the given parameters and return a result. * - * TODO@API options.parameters have been validated against the scheme at. + * The provided {@link LanguageModelToolInvocationOptions.parameters} have been validated against the schema declared for + * this tool. */ invoke(options: LanguageModelToolInvocationOptions, token: CancellationToken): ProviderResult; /** * Called once before a tool is invoked. May be implemented to signal that a tool needs user confirmation before running, - * and to customize the progress message that appears while the tool is running. + * and to customize the progress message that appears while the tool is running. Must be free of side-effects. A call to + * `prepareInvocation` is not necessarily followed by a call to `invoke`. */ - // TODO@API must be side-effect free, not every prepare does an invoke - // TODO@API name: prepare, prepareInvocation - prepareToolInvocation?(options: LanguageModelToolInvocationPrepareOptions, token: CancellationToken): ProviderResult; + prepareInvocation?(options: LanguageModelToolInvocationPrepareOptions, token: CancellationToken): ProviderResult; } /** - * The result of a call to {@link LanguageModelTool.prepareToolInvocation}. + * The result of a call to {@link LanguageModelTool.prepareInvocation}. */ export interface PreparedToolInvocation { /** @@ -299,10 +322,9 @@ declare module 'vscode' { */ export interface ChatLanguageModelToolReference { /** - * The tool's ID. Refers to a tool listed in {@link lm.tools}. + * The tool name. Refers to a tool listed in {@link lm.tools}. */ - // TODO@API name - readonly id: string; + readonly name: string; /** * The start and end index of the reference in the {@link ChatRequest.prompt prompt}. When undefined, the reference was