Microsoft/vscode
1
0
Fork 0
mirror of https://github.com/microsoft/vscode.git synced 2026-05-19 06:39:55 +01:00
Code Issues Packages Projects Releases Wiki Activity

share definitions

Browse Source
This commit is contained in:
amunger
2026-05-13 15:03:05 -07:00
parent ced7594cb8
commit b5cae35bcf
4 changed files with 85 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files
extensions/copilot/src/extension/chatSessions/claude/common/test/claudeMessageDispatch.spec.ts
+49 -49
View File
@@ -506,60 +506,60 @@ describe('handleUserMessage', () => {
expect(mockSpan.end).toHaveBeenCalled();
});
it('emits languageModelToolInvoked telemetry for completed tool results', () => {
const toolUse: Anthropic.Beta.Messages.BetaToolUseBlock = {
type: 'tool_use', id: 'tool-telemetry', name: ClaudeToolNames.Read, input: { file_path: '/test.ts' },
};
state.unprocessedToolCalls.set('tool-telemetry', toolUse);
state.toolStartTimes.set('tool-telemetry', Date.now());
it('emits languageModelToolInvoked telemetry for completed tool results', () => {
const toolUse: Anthropic.Beta.Messages.BetaToolUseBlock = {
type: 'tool_use', id: 'tool-telemetry', name: ClaudeToolNames.Read, input: { file_path: '/test.ts' },
};
state.unprocessedToolCalls.set('tool-telemetry', toolUse);
state.toolStartTimes.set('tool-telemetry', Date.now());
handleUserMessage(
makeUserMessage([{ type: 'tool_result', tool_use_id: 'tool-telemetry', content: 'file contents here' }]),
accessor, TEST_SESSION_ID, request, state,
);
handleUserMessage(
makeUserMessage([{ type: 'tool_result', tool_use_id: 'tool-telemetry', content: 'file contents here' }]),
accessor, TEST_SESSION_ID, request, state,
);
expect(services.telemetryService.sendMSFTTelemetryEvent).toHaveBeenCalledWith('languageModelToolInvoked', {
result: 'success',
chatSessionId: 'claude-code:/test-session-id',
toolId: ClaudeToolNames.Read,
toolExtensionId: undefined,
toolSourceKind: 'claudeCode',
}, { invocationTimeMs: expect.any(Number) });
});
expect(services.telemetryService.sendMSFTTelemetryEvent).toHaveBeenCalledWith('languageModelToolInvoked', {
result: 'success',
chatSessionId: 'claude-code:/test-session-id',
toolId: ClaudeToolNames.Read,
toolExtensionId: undefined,
toolSourceKind: 'claudeCode',
}, { invocationTimeMs: expect.any(Number) });
});
it('maps Claude Code MCP, error, and denied tool telemetry', () => {
const mcpToolUse: Anthropic.Beta.Messages.BetaToolUseBlock = {
type: 'tool_use', id: 'tool-mcp', name: 'mcp__server__tool', input: {},
};
const deniedToolUse: Anthropic.Beta.Messages.BetaToolUseBlock = {
type: 'tool_use', id: 'tool-denied-telemetry', name: ClaudeToolNames.Bash, input: { command: 'rm -rf /' },
};
state.unprocessedToolCalls.set('tool-mcp', mcpToolUse);
state.unprocessedToolCalls.set('tool-denied-telemetry', deniedToolUse);
it('maps Claude Code MCP, error, and denied tool telemetry', () => {
const mcpToolUse: Anthropic.Beta.Messages.BetaToolUseBlock = {
type: 'tool_use', id: 'tool-mcp', name: 'mcp__server__tool', input: {},
};
const deniedToolUse: Anthropic.Beta.Messages.BetaToolUseBlock = {
type: 'tool_use', id: 'tool-denied-telemetry', name: ClaudeToolNames.Bash, input: { command: 'rm -rf /' },
};
state.unprocessedToolCalls.set('tool-mcp', mcpToolUse);
state.unprocessedToolCalls.set('tool-denied-telemetry', deniedToolUse);
handleUserMessage(
makeUserMessage([
{ type: 'tool_result', tool_use_id: 'tool-mcp', content: 'failed', is_error: true },
{ type: 'tool_result', tool_use_id: 'tool-denied-telemetry', content: DENY_TOOL_MESSAGE },
]),
accessor, TEST_SESSION_ID, request, state,
);
handleUserMessage(
makeUserMessage([
{ type: 'tool_result', tool_use_id: 'tool-mcp', content: 'failed', is_error: true },
{ type: 'tool_result', tool_use_id: 'tool-denied-telemetry', content: DENY_TOOL_MESSAGE },
]),
accessor, TEST_SESSION_ID, request, state,
);
const events = services.telemetryService.sendMSFTTelemetryEvent.mock.calls.map(call => ({
properties: call[1] as TelemetryEventProperties,
measurements: call[2] as TelemetryEventMeasurements | undefined,
}));
expect(events).toEqual([
{
properties: { result: 'error', chatSessionId: 'claude-code:/test-session-id', toolId: 'mcp__server__tool', toolExtensionId: undefined, toolSourceKind: 'mcp' },
measurements: undefined,
},
{
properties: { result: 'userCancelled', chatSessionId: 'claude-code:/test-session-id', toolId: ClaudeToolNames.Bash, toolExtensionId: undefined, toolSourceKind: 'claudeCode' },
measurements: undefined,
},
]);
});
const events = services.telemetryService.sendMSFTTelemetryEvent.mock.calls.map(call => ({
properties: call[1] as TelemetryEventProperties,
measurements: call[2] as TelemetryEventMeasurements | undefined,
}));
expect(events).toEqual([
{
properties: { result: 'error', chatSessionId: 'claude-code:/test-session-id', toolId: 'mcp__server__tool', toolExtensionId: undefined, toolSourceKind: 'mcp' },
measurements: undefined,
},
{
properties: { result: 'userCancelled', chatSessionId: 'claude-code:/test-session-id', toolId: ClaudeToolNames.Bash, toolExtensionId: undefined, toolSourceKind: 'claudeCode' },
measurements: undefined,
},
]);
});
it('skips tool_result blocks with no matching tool call', () => {
handleUserMessage(
src/vs/platform/agentHost/common/languageModelToolTelemetry.ts
+32
View File
@@ -0,0 +1,32 @@
/*---------------------------------------------------------------------------------------------
* Copyright (c) Microsoft Corporation. All rights reserved.
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
export type LanguageModelToolTelemetryData = {
chatSessionId: string | undefined;
toolId: string;
toolExtensionId: string | undefined;
toolSourceKind: string;
};
export type LanguageModelToolTelemetryClassification = {
chatSessionId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the chat session that the tool was used within, if applicable.' };
toolId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the tool used.' };
toolExtensionId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The extension that contributed the tool.' };
toolSourceKind: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The source (mcp/extension/internal) of the tool.' };
};
export type LanguageModelToolInvokedEvent = LanguageModelToolTelemetryData & {
result: 'success' | 'error' | 'userCancelled';
prepareTimeMs?: number;
invocationTimeMs?: number;
};
export type LanguageModelToolInvokedClassification = LanguageModelToolTelemetryClassification & {
result: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether invoking the LanguageModelTool resulted in an error.' };
prepareTimeMs?: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'Time spent in prepareToolInvocation method in milliseconds.' };
invocationTimeMs?: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'Time spent in tool invoke method in milliseconds.' };
owner: 'roblourens';
comment: 'Provides insight into the usage of language model tools.';
};
src/vs/platform/agentHost/node/copilot/copilotAgentSession.ts
+1 -22
View File
@@ -25,6 +25,7 @@ import { ITelemetryService } from '../../../telemetry/common/telemetry.js';
import { platformSessionSchema } from '../../common/agentHostSchema.js';
import { AgentSignal } from '../../common/agentService.js';
import { stripRedundantCdPrefix } from '../../common/commandLineHelpers.js';
import type { LanguageModelToolInvokedClassification, LanguageModelToolInvokedEvent } from '../../common/languageModelToolTelemetry.js';
import { SessionConfigKey } from '../../common/sessionConfigKeys.js';
import { ISessionDatabase, ISessionDataService, SESSION_ATTACHMENTS_DIRNAME } from '../../common/sessionDataService.js';
import { MessageAttachmentKind, type FileEdit, type MessageAttachment, type ToolDefinition } from '../../common/state/protocol/state.js';
@@ -246,28 +247,6 @@ export interface IActiveClientSnapshot {
readonly plugins: readonly IParsedPlugin[];
}
type LanguageModelToolInvokedEvent = {
result: 'success' | 'error' | 'userCancelled';
chatSessionId: string | undefined;
toolId: string;
toolExtensionId: string | undefined;
toolSourceKind: string;
prepareTimeMs?: number;
invocationTimeMs?: number;
};
type LanguageModelToolInvokedClassification = {
result: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether invoking the LanguageModelTool resulted in an error.' };
chatSessionId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the chat session that the tool was used within, if applicable.' };
toolId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the tool used.' };
toolExtensionId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The extension that contributed the tool.' };
toolSourceKind: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The source (mcp/extension/internal) of the tool.' };
prepareTimeMs?: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'Time spent in prepareToolInvocation method in milliseconds.' };
invocationTimeMs?: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'Time spent in tool invoke method in milliseconds.' };
owner: 'roblourens';
comment: 'Provides insight into the usage of language model tools.';
};
/**
* Factory function that produces a {@link CopilotSessionWrapper}.
* Called by {@link CopilotAgentSession.initializeSession} with the
src/vs/workbench/contrib/chat/browser/tools/languageModelToolsService.ts
+3 -32
View File
@@ -29,6 +29,7 @@ import { ICommandService } from '../../../../../platform/commands/common/command
import { IConfigurationService } from '../../../../../platform/configuration/common/configuration.js';
import { IContextKey, IContextKeyService } from '../../../../../platform/contextkey/common/contextkey.js';
import { IDialogService } from '../../../../../platform/dialogs/common/dialogs.js';
import type { LanguageModelToolInvokedClassification, LanguageModelToolInvokedEvent, LanguageModelToolTelemetryClassification, LanguageModelToolTelemetryData } from '../../../../../platform/agentHost/common/languageModelToolTelemetry.js';
import { IInstantiationService } from '../../../../../platform/instantiation/common/instantiation.js';
import * as JSONContributionRegistry from '../../../../../platform/jsonschemas/common/jsonContributionRegistry.js';
import { ILogService } from '../../../../../platform/log/common/log.js';
@@ -1708,29 +1709,7 @@ function getToolSetFullReferenceName(toolSet: IToolSet) {
}
type LanguageModelToolInvokedEvent = {
result: 'success' | 'error' | 'userCancelled';
chatSessionId: string | undefined;
toolId: string;
toolExtensionId: string | undefined;
toolSourceKind: string;
prepareTimeMs?: number;
invocationTimeMs?: number;
};
type LanguageModelToolInvokedClassification = {
result: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether invoking the LanguageModelTool resulted in an error.' };
chatSessionId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the chat session that the tool was used within, if applicable.' };
toolId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the tool used.' };
toolExtensionId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The extension that contributed the tool.' };
toolSourceKind: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The source (mcp/extension/internal) of the tool.' };
prepareTimeMs?: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'Time spent in prepareToolInvocation method in milliseconds.' };
invocationTimeMs?: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'Time spent in tool invoke method in milliseconds.' };
owner: 'roblourens';
comment: 'Provides insight into the usage of language model tools.';
};
type ToolApprovalEvent = {
type ToolApprovalEvent = LanguageModelToolTelemetryData & {
confirmKind: string;
settingId: string | undefined;
lmServiceScope: string | undefined;
@@ -1738,13 +1717,9 @@ type ToolApprovalEvent = {
confirmationNotNeededReason: string | undefined;
sandboxWrapped: boolean | undefined;
requestUnsandboxedExecution: boolean | undefined;
chatSessionId: string | undefined;
toolId: string;
toolExtensionId: string | undefined;
toolSourceKind: string;
};
type ToolApprovalClassification = {
type ToolApprovalClassification = LanguageModelToolTelemetryClassification & {
confirmKind: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'How the confirmation was resolved (userAction, setting, lmServicePerTool, confirmationNotNeeded, denied, skipped). Anything other than userAction implies auto-approval. "denied" and "skipped" mean the tool did not run; otherwise it ran (note: a custom Deny button click resolves as userAction since the tool still runs and the chosen label is passed to it; see customButtonKind to distinguish).' };
settingId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'When confirmKind is setting, the configuration id that auto-approved the tool.' };
lmServiceScope: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'When confirmKind is lmServicePerTool, the scope (session/workspace/profile).' };
@@ -1752,10 +1727,6 @@ type ToolApprovalClassification = {
confirmationNotNeededReason: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'When confirmKind is confirmationNotNeeded, a stable identifier for why the tool did not require confirmation. Limited to a known allowlist (e.g. auto-approve-all, inlineChat); set to "other" for any other reason; undefined when no reason was supplied.' };
sandboxWrapped: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'For terminal tool calls, whether this specific invocation runs inside the agent terminal sandbox. Undefined for non-terminal tools.' };
requestUnsandboxedExecution: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'For terminal tool calls, whether the model requested to bypass the sandbox for this invocation. Undefined for non-terminal tools.' };
chatSessionId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the chat session that the tool was used within, if applicable.' };
toolId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the tool used.' };
toolExtensionId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The extension that contributed the tool.' };
toolSourceKind: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The source (mcp/extension/internal) of the tool.' };
owner: 'chrmarti';
comment: 'Provides insight into how tool confirmations are resolved (user action vs. auto-approval).';
};