diff --git a/extensions/copilot/src/extension/chatSessions/claude/common/test/claudeMessageDispatch.spec.ts b/extensions/copilot/src/extension/chatSessions/claude/common/test/claudeMessageDispatch.spec.ts index 92b73e91f9a..47ff1eb9419 100644 --- a/extensions/copilot/src/extension/chatSessions/claude/common/test/claudeMessageDispatch.spec.ts +++ b/extensions/copilot/src/extension/chatSessions/claude/common/test/claudeMessageDispatch.spec.ts @@ -506,60 +506,60 @@ describe('handleUserMessage', () => { expect(mockSpan.end).toHaveBeenCalled(); }); - it('emits languageModelToolInvoked telemetry for completed tool results', () => { - const toolUse: Anthropic.Beta.Messages.BetaToolUseBlock = { - type: 'tool_use', id: 'tool-telemetry', name: ClaudeToolNames.Read, input: { file_path: '/test.ts' }, - }; - state.unprocessedToolCalls.set('tool-telemetry', toolUse); - state.toolStartTimes.set('tool-telemetry', Date.now()); + it('emits languageModelToolInvoked telemetry for completed tool results', () => { + const toolUse: Anthropic.Beta.Messages.BetaToolUseBlock = { + type: 'tool_use', id: 'tool-telemetry', name: ClaudeToolNames.Read, input: { file_path: '/test.ts' }, + }; + state.unprocessedToolCalls.set('tool-telemetry', toolUse); + state.toolStartTimes.set('tool-telemetry', Date.now()); - handleUserMessage( - makeUserMessage([{ type: 'tool_result', tool_use_id: 'tool-telemetry', content: 'file contents here' }]), - accessor, TEST_SESSION_ID, request, state, - ); + handleUserMessage( + makeUserMessage([{ type: 'tool_result', tool_use_id: 'tool-telemetry', content: 'file contents here' }]), + accessor, TEST_SESSION_ID, request, state, + ); - expect(services.telemetryService.sendMSFTTelemetryEvent).toHaveBeenCalledWith('languageModelToolInvoked', { - result: 'success', - chatSessionId: 'claude-code:/test-session-id', - toolId: ClaudeToolNames.Read, - toolExtensionId: undefined, - toolSourceKind: 'claudeCode', - }, { invocationTimeMs: expect.any(Number) }); - }); + expect(services.telemetryService.sendMSFTTelemetryEvent).toHaveBeenCalledWith('languageModelToolInvoked', { + result: 'success', + chatSessionId: 'claude-code:/test-session-id', + toolId: ClaudeToolNames.Read, + toolExtensionId: undefined, + toolSourceKind: 'claudeCode', + }, { invocationTimeMs: expect.any(Number) }); + }); - it('maps Claude Code MCP, error, and denied tool telemetry', () => { - const mcpToolUse: Anthropic.Beta.Messages.BetaToolUseBlock = { - type: 'tool_use', id: 'tool-mcp', name: 'mcp__server__tool', input: {}, - }; - const deniedToolUse: Anthropic.Beta.Messages.BetaToolUseBlock = { - type: 'tool_use', id: 'tool-denied-telemetry', name: ClaudeToolNames.Bash, input: { command: 'rm -rf /' }, - }; - state.unprocessedToolCalls.set('tool-mcp', mcpToolUse); - state.unprocessedToolCalls.set('tool-denied-telemetry', deniedToolUse); + it('maps Claude Code MCP, error, and denied tool telemetry', () => { + const mcpToolUse: Anthropic.Beta.Messages.BetaToolUseBlock = { + type: 'tool_use', id: 'tool-mcp', name: 'mcp__server__tool', input: {}, + }; + const deniedToolUse: Anthropic.Beta.Messages.BetaToolUseBlock = { + type: 'tool_use', id: 'tool-denied-telemetry', name: ClaudeToolNames.Bash, input: { command: 'rm -rf /' }, + }; + state.unprocessedToolCalls.set('tool-mcp', mcpToolUse); + state.unprocessedToolCalls.set('tool-denied-telemetry', deniedToolUse); - handleUserMessage( - makeUserMessage([ - { type: 'tool_result', tool_use_id: 'tool-mcp', content: 'failed', is_error: true }, - { type: 'tool_result', tool_use_id: 'tool-denied-telemetry', content: DENY_TOOL_MESSAGE }, - ]), - accessor, TEST_SESSION_ID, request, state, - ); + handleUserMessage( + makeUserMessage([ + { type: 'tool_result', tool_use_id: 'tool-mcp', content: 'failed', is_error: true }, + { type: 'tool_result', tool_use_id: 'tool-denied-telemetry', content: DENY_TOOL_MESSAGE }, + ]), + accessor, TEST_SESSION_ID, request, state, + ); - const events = services.telemetryService.sendMSFTTelemetryEvent.mock.calls.map(call => ({ - properties: call[1] as TelemetryEventProperties, - measurements: call[2] as TelemetryEventMeasurements | undefined, - })); - expect(events).toEqual([ - { - properties: { result: 'error', chatSessionId: 'claude-code:/test-session-id', toolId: 'mcp__server__tool', toolExtensionId: undefined, toolSourceKind: 'mcp' }, - measurements: undefined, - }, - { - properties: { result: 'userCancelled', chatSessionId: 'claude-code:/test-session-id', toolId: ClaudeToolNames.Bash, toolExtensionId: undefined, toolSourceKind: 'claudeCode' }, - measurements: undefined, - }, - ]); - }); + const events = services.telemetryService.sendMSFTTelemetryEvent.mock.calls.map(call => ({ + properties: call[1] as TelemetryEventProperties, + measurements: call[2] as TelemetryEventMeasurements | undefined, + })); + expect(events).toEqual([ + { + properties: { result: 'error', chatSessionId: 'claude-code:/test-session-id', toolId: 'mcp__server__tool', toolExtensionId: undefined, toolSourceKind: 'mcp' }, + measurements: undefined, + }, + { + properties: { result: 'userCancelled', chatSessionId: 'claude-code:/test-session-id', toolId: ClaudeToolNames.Bash, toolExtensionId: undefined, toolSourceKind: 'claudeCode' }, + measurements: undefined, + }, + ]); + }); it('skips tool_result blocks with no matching tool call', () => { handleUserMessage( diff --git a/src/vs/platform/agentHost/common/languageModelToolTelemetry.ts b/src/vs/platform/agentHost/common/languageModelToolTelemetry.ts new file mode 100644 index 00000000000..796c7d3ca14 --- /dev/null +++ b/src/vs/platform/agentHost/common/languageModelToolTelemetry.ts @@ -0,0 +1,32 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +export type LanguageModelToolTelemetryData = { + chatSessionId: string | undefined; + toolId: string; + toolExtensionId: string | undefined; + toolSourceKind: string; +}; + +export type LanguageModelToolTelemetryClassification = { + chatSessionId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the chat session that the tool was used within, if applicable.' }; + toolId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the tool used.' }; + toolExtensionId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The extension that contributed the tool.' }; + toolSourceKind: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The source (mcp/extension/internal) of the tool.' }; +}; + +export type LanguageModelToolInvokedEvent = LanguageModelToolTelemetryData & { + result: 'success' | 'error' | 'userCancelled'; + prepareTimeMs?: number; + invocationTimeMs?: number; +}; + +export type LanguageModelToolInvokedClassification = LanguageModelToolTelemetryClassification & { + result: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether invoking the LanguageModelTool resulted in an error.' }; + prepareTimeMs?: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'Time spent in prepareToolInvocation method in milliseconds.' }; + invocationTimeMs?: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'Time spent in tool invoke method in milliseconds.' }; + owner: 'roblourens'; + comment: 'Provides insight into the usage of language model tools.'; +}; diff --git a/src/vs/platform/agentHost/node/copilot/copilotAgentSession.ts b/src/vs/platform/agentHost/node/copilot/copilotAgentSession.ts index 0b3a9644555..2c840e7810e 100644 --- a/src/vs/platform/agentHost/node/copilot/copilotAgentSession.ts +++ b/src/vs/platform/agentHost/node/copilot/copilotAgentSession.ts @@ -25,6 +25,7 @@ import { ITelemetryService } from '../../../telemetry/common/telemetry.js'; import { platformSessionSchema } from '../../common/agentHostSchema.js'; import { AgentSignal } from '../../common/agentService.js'; import { stripRedundantCdPrefix } from '../../common/commandLineHelpers.js'; +import type { LanguageModelToolInvokedClassification, LanguageModelToolInvokedEvent } from '../../common/languageModelToolTelemetry.js'; import { SessionConfigKey } from '../../common/sessionConfigKeys.js'; import { ISessionDatabase, ISessionDataService, SESSION_ATTACHMENTS_DIRNAME } from '../../common/sessionDataService.js'; import { MessageAttachmentKind, type FileEdit, type MessageAttachment, type ToolDefinition } from '../../common/state/protocol/state.js'; @@ -246,28 +247,6 @@ export interface IActiveClientSnapshot { readonly plugins: readonly IParsedPlugin[]; } -type LanguageModelToolInvokedEvent = { - result: 'success' | 'error' | 'userCancelled'; - chatSessionId: string | undefined; - toolId: string; - toolExtensionId: string | undefined; - toolSourceKind: string; - prepareTimeMs?: number; - invocationTimeMs?: number; -}; - -type LanguageModelToolInvokedClassification = { - result: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether invoking the LanguageModelTool resulted in an error.' }; - chatSessionId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the chat session that the tool was used within, if applicable.' }; - toolId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the tool used.' }; - toolExtensionId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The extension that contributed the tool.' }; - toolSourceKind: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The source (mcp/extension/internal) of the tool.' }; - prepareTimeMs?: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'Time spent in prepareToolInvocation method in milliseconds.' }; - invocationTimeMs?: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'Time spent in tool invoke method in milliseconds.' }; - owner: 'roblourens'; - comment: 'Provides insight into the usage of language model tools.'; -}; - /** * Factory function that produces a {@link CopilotSessionWrapper}. * Called by {@link CopilotAgentSession.initializeSession} with the diff --git a/src/vs/workbench/contrib/chat/browser/tools/languageModelToolsService.ts b/src/vs/workbench/contrib/chat/browser/tools/languageModelToolsService.ts index af82cdf409e..6cd590d2800 100644 --- a/src/vs/workbench/contrib/chat/browser/tools/languageModelToolsService.ts +++ b/src/vs/workbench/contrib/chat/browser/tools/languageModelToolsService.ts @@ -29,6 +29,7 @@ import { ICommandService } from '../../../../../platform/commands/common/command import { IConfigurationService } from '../../../../../platform/configuration/common/configuration.js'; import { IContextKey, IContextKeyService } from '../../../../../platform/contextkey/common/contextkey.js'; import { IDialogService } from '../../../../../platform/dialogs/common/dialogs.js'; +import type { LanguageModelToolInvokedClassification, LanguageModelToolInvokedEvent, LanguageModelToolTelemetryClassification, LanguageModelToolTelemetryData } from '../../../../../platform/agentHost/common/languageModelToolTelemetry.js'; import { IInstantiationService } from '../../../../../platform/instantiation/common/instantiation.js'; import * as JSONContributionRegistry from '../../../../../platform/jsonschemas/common/jsonContributionRegistry.js'; import { ILogService } from '../../../../../platform/log/common/log.js'; @@ -1708,29 +1709,7 @@ function getToolSetFullReferenceName(toolSet: IToolSet) { } -type LanguageModelToolInvokedEvent = { - result: 'success' | 'error' | 'userCancelled'; - chatSessionId: string | undefined; - toolId: string; - toolExtensionId: string | undefined; - toolSourceKind: string; - prepareTimeMs?: number; - invocationTimeMs?: number; -}; - -type LanguageModelToolInvokedClassification = { - result: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether invoking the LanguageModelTool resulted in an error.' }; - chatSessionId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the chat session that the tool was used within, if applicable.' }; - toolId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the tool used.' }; - toolExtensionId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The extension that contributed the tool.' }; - toolSourceKind: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The source (mcp/extension/internal) of the tool.' }; - prepareTimeMs?: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'Time spent in prepareToolInvocation method in milliseconds.' }; - invocationTimeMs?: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'Time spent in tool invoke method in milliseconds.' }; - owner: 'roblourens'; - comment: 'Provides insight into the usage of language model tools.'; -}; - -type ToolApprovalEvent = { +type ToolApprovalEvent = LanguageModelToolTelemetryData & { confirmKind: string; settingId: string | undefined; lmServiceScope: string | undefined; @@ -1738,13 +1717,9 @@ type ToolApprovalEvent = { confirmationNotNeededReason: string | undefined; sandboxWrapped: boolean | undefined; requestUnsandboxedExecution: boolean | undefined; - chatSessionId: string | undefined; - toolId: string; - toolExtensionId: string | undefined; - toolSourceKind: string; }; -type ToolApprovalClassification = { +type ToolApprovalClassification = LanguageModelToolTelemetryClassification & { confirmKind: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'How the confirmation was resolved (userAction, setting, lmServicePerTool, confirmationNotNeeded, denied, skipped). Anything other than userAction implies auto-approval. "denied" and "skipped" mean the tool did not run; otherwise it ran (note: a custom Deny button click resolves as userAction since the tool still runs and the chosen label is passed to it; see customButtonKind to distinguish).' }; settingId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'When confirmKind is setting, the configuration id that auto-approved the tool.' }; lmServiceScope: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'When confirmKind is lmServicePerTool, the scope (session/workspace/profile).' }; @@ -1752,10 +1727,6 @@ type ToolApprovalClassification = { confirmationNotNeededReason: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'When confirmKind is confirmationNotNeeded, a stable identifier for why the tool did not require confirmation. Limited to a known allowlist (e.g. auto-approve-all, inlineChat); set to "other" for any other reason; undefined when no reason was supplied.' }; sandboxWrapped: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'For terminal tool calls, whether this specific invocation runs inside the agent terminal sandbox. Undefined for non-terminal tools.' }; requestUnsandboxedExecution: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'For terminal tool calls, whether the model requested to bypass the sandbox for this invocation. Undefined for non-terminal tools.' }; - chatSessionId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the chat session that the tool was used within, if applicable.' }; - toolId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the tool used.' }; - toolExtensionId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The extension that contributed the tool.' }; - toolSourceKind: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The source (mcp/extension/internal) of the tool.' }; owner: 'chrmarti'; comment: 'Provides insight into how tool confirmations are resolved (user action vs. auto-approval).'; };