From 8f4fc8d3273653bfb73b8d4c3cfc7eeecd00722c Mon Sep 17 00:00:00 2001 From: Connor Peet Date: Fri, 21 Aug 2026 10:56:37 -0700 Subject: [PATCH] chat: identify Agent Host telemetry sessions (#332000) * chat: identify Agent Host telemetry sessions Adds a session-mode property to chat request, user-action, edit, and follow-up telemetry. This lets telemetry split Agent Host sessions from legacy sessions, including local fallback sessions. - Adds isAgentHostSession to workbench chat telemetry events.\n- Adds whole-file edit outcomes and tags hunk outcomes.\n- Tags shared accepted and rejected edit telemetry.\n- Adds focused tests for Agent Host and legacy telemetry values. (Commit message generated by Copilot) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * chat: tag code block telemetry sessions Tags sidebar code-block telemetry with the actual session mode. Corrects the remaining-edits value for user-modified file outcomes. - Shares Agent Host session resource detection.\n- Tags code-block suggestions and acceptance actions.\n- Reports pending edits when users modify a reviewed file.\n- Extends focused telemetry coverage. (Commit message generated by Copilot) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../browser/actions/chatCodeblockActions.ts | 4 + .../browser/actions/codeBlockOperations.ts | 2 + .../chatEditingModifiedFileEntry.ts | 6 +- .../chatMarkdownContentPart.ts | 3 +- .../common/chatService/chatServiceImpl.ts | 2 +- .../chatService/chatServiceTelemetry.ts | 91 +++++++++++++++---- .../chat/common/chatSessionsService.ts | 4 + .../common/chatService/chatService.test.ts | 63 ++++++++++++- .../aiEditTelemetry/aiEditTelemetryService.ts | 3 + .../aiEditTelemetryServiceImpl.ts | 9 ++ .../test/browser/editTelemetry.test.ts | 37 ++++++++ 11 files changed, 198 insertions(+), 26 deletions(-) diff --git a/src/vs/workbench/contrib/chat/browser/actions/chatCodeblockActions.ts b/src/vs/workbench/contrib/chat/browser/actions/chatCodeblockActions.ts index 640998f1220..af600ebf796 100644 --- a/src/vs/workbench/contrib/chat/browser/actions/chatCodeblockActions.ts +++ b/src/vs/workbench/contrib/chat/browser/actions/chatCodeblockActions.ts @@ -33,6 +33,7 @@ import { reviewEdits } from './reviewEdits.js'; import { ITerminalEditorService, ITerminalGroupService, ITerminalService } from '../../../terminal/browser/terminal.js'; import { ChatContextKeys } from '../../common/actions/chatContextKeys.js'; import { ChatCopyKind, IChatService } from '../../common/chatService/chatService.js'; +import { isAgentHostSessionResource } from '../../common/chatSessionsService.js'; import { IChatRequestViewModel, IChatResponseViewModel, isRequestVM, isResponseVM } from '../../common/model/chatViewModel.js'; import { ChatAgentLocation } from '../../common/constants.js'; import { IChatCodeBlockContextProviderService, IChatWidgetService } from '../chat.js'; @@ -201,6 +202,7 @@ export function registerChatCodeBlockActions() { applyCodeBlockSuggestionId: undefined, source: undefined, sourceRequestId: undefined, + isAgentHostSession: isAgentHostSessionResource(context.element.sessionResource), }); } } @@ -269,6 +271,7 @@ export function registerChatCodeBlockActions() { applyCodeBlockSuggestionId: undefined, source: undefined, sourceRequestId: undefined, + isAgentHostSession: isAgentHostSessionResource(element.sessionResource), }); } @@ -427,6 +430,7 @@ export function registerChatCodeBlockActions() { applyCodeBlockSuggestionId: undefined, source: undefined, sourceRequestId: undefined, + isAgentHostSession: isAgentHostSessionResource(context.element.sessionResource), }); } } diff --git a/src/vs/workbench/contrib/chat/browser/actions/codeBlockOperations.ts b/src/vs/workbench/contrib/chat/browser/actions/codeBlockOperations.ts index c3650b84faa..06507add6b7 100644 --- a/src/vs/workbench/contrib/chat/browser/actions/codeBlockOperations.ts +++ b/src/vs/workbench/contrib/chat/browser/actions/codeBlockOperations.ts @@ -36,6 +36,7 @@ import { CellKind, ICellEditOperation, NOTEBOOK_EDITOR_ID } from '../../../noteb import { INotebookService } from '../../../notebook/common/notebookService.js'; import { ICodeMapperCodeBlock, ICodeMapperRequest, ICodeMapperResponse, ICodeMapperService } from '../../common/editing/chatCodeMapperService.js'; import { ChatUserAction, IChatService } from '../../common/chatService/chatService.js'; +import { isAgentHostSessionResource } from '../../common/chatSessionsService.js'; import { IChatRequestViewModel, isRequestVM, isResponseVM } from '../../common/model/chatViewModel.js'; import { ICodeBlockActionContext } from '../widget/chatContentParts/codeBlockPart.js'; @@ -91,6 +92,7 @@ export class InsertCodeBlockOperation { applyCodeBlockSuggestionId: undefined, source: undefined, sourceRequestId: undefined, + isAgentHostSession: isAgentHostSessionResource(context.element.sessionResource), }); } } diff --git a/src/vs/workbench/contrib/chat/browser/chatEditing/chatEditingModifiedFileEntry.ts b/src/vs/workbench/contrib/chat/browser/chatEditing/chatEditingModifiedFileEntry.ts index 94f750d632f..dc6e1279c03 100644 --- a/src/vs/workbench/contrib/chat/browser/chatEditing/chatEditingModifiedFileEntry.ts +++ b/src/vs/workbench/contrib/chat/browser/chatEditing/chatEditingModifiedFileEntry.ts @@ -24,6 +24,7 @@ import { IFilesConfigurationService } from '../../../../services/filesConfigurat import { IAiEditTelemetryService } from '../../../editTelemetry/browser/telemetry/aiEditTelemetry/aiEditTelemetryService.js'; import { ICellEditOperation } from '../../../notebook/common/notebookCommon.js'; import { ChatUserAction, IChatService } from '../../common/chatService/chatService.js'; +import { isAgentHostSessionResource } from '../../common/chatSessionsService.js'; import { ChatEditKind, IModifiedEntryTelemetryInfo, IModifiedFileEntry, IModifiedFileEntryEditorIntegration, ISnapshotEntry, ModifiedFileEntryState } from '../../common/editing/chatEditingService.js'; import { IChatResponseModel } from '../../common/model/chatModel.js'; @@ -269,10 +270,11 @@ export abstract class AbstractChatEditingModifiedFileEntry extends Disposable im protected abstract _doReject(): Promise; protected _notifySessionAction(outcome: 'accepted' | 'rejected' | 'userModified') { - this._notifyAction({ kind: 'chatEditingSessionAction', uri: this.modifiedURI, hasRemainingEdits: false, outcome }); + this._notifyAction({ kind: 'chatEditingSessionAction', uri: this.modifiedURI, hasRemainingEdits: outcome === 'userModified', outcome }); } protected _notifyAction(action: ChatUserAction) { + const isAgentHostSession = isAgentHostSessionResource(this._telemetryInfo.sessionResource); if (action.kind === 'chatEditingHunkAction' && action.outcome === 'accepted') { this._aiEditTelemetryService.handleCodeAccepted({ suggestionId: undefined, // TODO@hediet try to figure this out @@ -291,6 +293,7 @@ export abstract class AbstractChatEditingModifiedFileEntry extends Disposable im languageId: action.languageId, source: undefined, sourceRequestId: this._telemetryInfo.requestId, + isAgentHostSession, }); } else if (action.kind === 'chatEditingHunkAction' && action.outcome === 'rejected') { this._aiEditTelemetryService.handleCodeRejected({ @@ -310,6 +313,7 @@ export abstract class AbstractChatEditingModifiedFileEntry extends Disposable im languageId: action.languageId, source: undefined, sourceRequestId: this._telemetryInfo.requestId, + isAgentHostSession, }); } diff --git a/src/vs/workbench/contrib/chat/browser/widget/chatContentParts/chatMarkdownContentPart.ts b/src/vs/workbench/contrib/chat/browser/widget/chatContentParts/chatMarkdownContentPart.ts index 5811f5f7b3d..d73c6fe24ee 100644 --- a/src/vs/workbench/contrib/chat/browser/widget/chatContentParts/chatMarkdownContentPart.ts +++ b/src/vs/workbench/contrib/chat/browser/widget/chatContentParts/chatMarkdownContentPart.ts @@ -47,7 +47,7 @@ import { extractCodeblockUrisFromText, extractVulnerabilitiesFromText } from '.. import { IEditSessionEntryDiff } from '../../../common/editing/chatEditingService.js'; import { IChatProgressRenderableResponseContent } from '../../../common/model/chatModel.js'; import { IChatContentInlineReference, IChatMarkdownContent, IChatService, IChatUndoStop } from '../../../common/chatService/chatService.js'; -import { IChatSessionsService } from '../../../common/chatSessionsService.js'; +import { IChatSessionsService, isAgentHostSessionResource } from '../../../common/chatSessionsService.js'; import { isRequestVM, isResponseVM } from '../../../common/model/chatViewModel.js'; import { ChatConfiguration } from '../../../common/constants.js'; import { IChatCodeBlockInfo } from '../../chat.js'; @@ -384,6 +384,7 @@ export class ChatMarkdownContentPart extends Disposable implements IChatContentP applyCodeBlockSuggestionId: undefined, source: undefined, sourceRequestId: undefined, + isAgentHostSession: isAgentHostSessionResource(element.sessionResource), }) }; })); diff --git a/src/vs/workbench/contrib/chat/common/chatService/chatServiceImpl.ts b/src/vs/workbench/contrib/chat/common/chatService/chatServiceImpl.ts index d297690e19d..493c470cfe6 100644 --- a/src/vs/workbench/contrib/chat/common/chatService/chatServiceImpl.ts +++ b/src/vs/workbench/contrib/chat/common/chatService/chatServiceImpl.ts @@ -1810,7 +1810,7 @@ export class ChatService extends Disposable implements IChatService { agentOrCommandFollowups.then(followups => { model.setFollowups(completedRequest, followups); const commandForTelemetry = agentSlashCommandPart ? agentSlashCommandPart.command.name : commandPart?.slashCommand.command; - this._chatServiceTelemetry.retrievedFollowups(agentPart?.agent.id ?? '', commandForTelemetry, followups?.length ?? 0); + this._chatServiceTelemetry.retrievedFollowups(model.sessionResource, agentPart?.agent.id ?? '', commandForTelemetry, followups?.length ?? 0); }); } } diff --git a/src/vs/workbench/contrib/chat/common/chatService/chatServiceTelemetry.ts b/src/vs/workbench/contrib/chat/common/chatService/chatServiceTelemetry.ts index 00aed85de18..d711dad0665 100644 --- a/src/vs/workbench/contrib/chat/common/chatService/chatServiceTelemetry.ts +++ b/src/vs/workbench/contrib/chat/common/chatService/chatServiceTelemetry.ts @@ -14,15 +14,24 @@ import { isImageVariableEntry } from '../attachments/chatVariableEntries.js'; import { ChatAgentLocation, ChatModeKind, ChatPermissionLevel } from '../constants.js'; import { ILanguageModelsService } from '../languageModels.js'; import { chatSessionResourceToId, getChatSessionType } from '../model/chatUri.js'; +import { isAgentHostSessionResource } from '../chatSessionsService.js'; import { isRemoteAgentHostSessionType, parseRemoteAgentHostHarness } from '../../../../../platform/agentHost/common/agentHostSessionType.js'; -type ChatVoteEvent = { +type ChatSessionModeEvent = { + isAgentHostSession: boolean; +}; + +type ChatSessionModeClassification = { + isAgentHostSession: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether the action was performed in an Agent Host-backed chat session.' }; +}; + +type ChatVoteEvent = ChatSessionModeEvent & { direction: 'up' | 'down'; agentId: string; command: string | undefined; }; -type ChatVoteClassification = { +type ChatVoteClassification = ChatSessionModeClassification & { direction: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether the user voted up or down.' }; agentId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the chat agent that this vote is for.' }; command: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The name of the slash command that this vote is for.' }; @@ -30,13 +39,13 @@ type ChatVoteClassification = { comment: 'Provides insight into the performance of Chat agents.'; }; -type ChatCopyEvent = { +type ChatCopyEvent = ChatSessionModeEvent & { copyKind: 'action' | 'toolbar'; agentId: string; command: string | undefined; }; -type ChatCopyClassification = { +type ChatCopyClassification = ChatSessionModeClassification & { copyKind: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'How the copy was initiated.' }; agentId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the chat agent that the copy acted on.' }; command: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The name of the slash command the copy acted on.' }; @@ -44,13 +53,13 @@ type ChatCopyClassification = { comment: 'Provides insight into the usage of Chat features.'; }; -type ChatInsertEvent = { +type ChatInsertEvent = ChatSessionModeEvent & { newFile: boolean; agentId: string; command: string | undefined; }; -type ChatInsertClassification = { +type ChatInsertClassification = ChatSessionModeClassification & { newFile: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether the code was inserted into a new untitled file.' }; agentId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the chat agent that this insertion is for.' }; command: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The name of the slash command that this insertion is for.' }; @@ -58,7 +67,7 @@ type ChatInsertClassification = { comment: 'Provides insight into the usage of Chat features.'; }; -type ChatApplyEvent = { +type ChatApplyEvent = ChatSessionModeEvent & { newFile: boolean; agentId: string; command: string | undefined; @@ -66,7 +75,7 @@ type ChatApplyEvent = { editsProposed: boolean; }; -type ChatApplyClassification = { +type ChatApplyClassification = ChatSessionModeClassification & { newFile: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether the code was inserted into a new untitled file.' }; agentId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the chat agent that this insertion is for.' }; command: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The name of the slash command that this insertion is for.' }; @@ -76,25 +85,25 @@ type ChatApplyClassification = { comment: 'Provides insight into the usage of Chat features.'; }; -type ChatFollowupEvent = { +type ChatFollowupEvent = ChatSessionModeEvent & { agentId: string; command: string | undefined; }; -type ChatFollowupClassification = { +type ChatFollowupClassification = ChatSessionModeClassification & { agentId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the related chat agent.' }; command: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The name of the related slash command.' }; owner: 'roblourens'; comment: 'Provides insight into the usage of Chat features.'; }; -type ChatTerminalEvent = { +type ChatTerminalEvent = ChatSessionModeEvent & { languageId: string; agentId: string; command: string | undefined; }; -type ChatTerminalClassification = { +type ChatTerminalClassification = ChatSessionModeClassification & { languageId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The language of the code that was run in the terminal.' }; agentId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the related chat agent.' }; command: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The name of the related slash command.' }; @@ -102,13 +111,13 @@ type ChatTerminalClassification = { comment: 'Provides insight into the usage of Chat features.'; }; -type ChatFollowupsRetrievedEvent = { +type ChatFollowupsRetrievedEvent = ChatSessionModeEvent & { agentId: string; command: string | undefined; numFollowups: number; }; -type ChatFollowupsRetrievedClassification = { +type ChatFollowupsRetrievedClassification = ChatSessionModeClassification & { agentId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the related chat agent.' }; command: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The name of the related slash command.' }; numFollowups: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The number of followup prompts returned by the agent.' }; @@ -116,7 +125,27 @@ type ChatFollowupsRetrievedClassification = { comment: 'Provides insight into the usage of Chat features.'; }; -type ChatEditHunkEvent = { +type ChatEditSessionEvent = ChatSessionModeEvent & { + agentId: string; + outcome: 'accepted' | 'rejected' | 'userModified'; + hasRemainingEdits: boolean; + requestId: string; + modelId: string; + modeId: string; +}; + +type ChatEditSessionClassification = ChatSessionModeClassification & { + agentId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the related chat agent.' }; + outcome: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The outcome of the edited file action.' }; + hasRemainingEdits: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether there are remaining edits in the file after this action.' }; + requestId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the chat request that produced the edit.' }; + modelId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The AI model used to generate the edit.' }; + modeId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The chat mode used for the request (e.g. ask, edit, agent).' }; + owner: 'roblourens'; + comment: 'Provides insight into the usage of Chat features.'; +}; + +type ChatEditHunkEvent = ChatSessionModeEvent & { agentId: string; outcome: 'accepted' | 'rejected'; lineCount: number; @@ -126,7 +155,7 @@ type ChatEditHunkEvent = { modeId: string; }; -type ChatEditHunkClassification = { +type ChatEditHunkClassification = ChatSessionModeClassification & { agentId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the related chat agent.' }; outcome: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The outcome of the edit hunk action.' }; lineCount: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The number of lines in the relevant change.' }; @@ -138,7 +167,7 @@ type ChatEditHunkClassification = { comment: 'Provides insight into the usage of Chat features.'; }; -export type ChatProviderInvokedEvent = { +export type ChatProviderInvokedEvent = ChatSessionModeEvent & { timeToFirstProgress: number | undefined; totalTime: number | undefined; result: 'success' | 'error' | 'errorWithOutput' | 'cancelled' | 'filtered'; @@ -161,7 +190,7 @@ export type ChatProviderInvokedEvent = { harness: string | undefined; }; -export type ChatProviderInvokedClassification = { +export type ChatProviderInvokedClassification = ChatSessionModeClassification & { timeToFirstProgress: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'The time in milliseconds from invoking the provider to getting the first data.' }; totalTime: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'The total time it took to run the provider\'s `provideResponseWithProgress`.' }; result: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether invoking the ChatProvider resulted in an error.' }; @@ -192,26 +221,31 @@ export class ChatServiceTelemetry { ) { } notifyUserAction(action: IChatUserActionEvent): void { + const isAgentHostSession = getIsAgentHostSessionForTelemetry(action.sessionResource); if (action.action.kind === 'vote') { this.telemetryService.publicLog2('interactiveSessionVote', { + isAgentHostSession, direction: action.action.direction === ChatAgentVoteDirection.Up ? 'up' : 'down', agentId: action.agentId ?? '', command: action.command, }); } else if (action.action.kind === 'copy') { this.telemetryService.publicLog2('interactiveSessionCopy', { + isAgentHostSession, copyKind: action.action.copyKind === ChatCopyKind.Action ? 'action' : 'toolbar', agentId: action.agentId ?? '', command: action.command, }); } else if (action.action.kind === 'insert') { this.telemetryService.publicLog2('interactiveSessionInsert', { + isAgentHostSession, newFile: !!action.action.newFile, agentId: action.agentId ?? '', command: action.command, }); } else if (action.action.kind === 'apply') { this.telemetryService.publicLog2('interactiveSessionApply', { + isAgentHostSession, newFile: !!action.action.newFile, codeMapper: action.action.codeMapper, agentId: action.agentId ?? '', @@ -220,17 +254,30 @@ export class ChatServiceTelemetry { }); } else if (action.action.kind === 'runInTerminal') { this.telemetryService.publicLog2('interactiveSessionRunInTerminal', { + isAgentHostSession, languageId: action.action.languageId ?? '', agentId: action.agentId ?? '', command: action.command, }); } else if (action.action.kind === 'followUp') { this.telemetryService.publicLog2('chatFollowupClicked', { + isAgentHostSession, agentId: action.agentId ?? '', command: action.command, }); + } else if (action.action.kind === 'chatEditingSessionAction') { + this.telemetryService.publicLog2('chatEditSession', { + isAgentHostSession, + agentId: action.agentId ?? '', + outcome: action.action.outcome, + hasRemainingEdits: action.action.hasRemainingEdits, + requestId: action.requestId, + modelId: escapeModelIdForTelemetry(action.modelId) ?? '', + modeId: action.modeId ?? '', + }); } else if (action.action.kind === 'chatEditingHunkAction') { this.telemetryService.publicLog2('chatEditHunk', { + isAgentHostSession, agentId: action.agentId ?? '', outcome: action.action.outcome, lineCount: action.action.lineCount, @@ -242,8 +289,9 @@ export class ChatServiceTelemetry { } } - retrievedFollowups(agentId: string, command: string | undefined, numFollowups: number): void { + retrievedFollowups(sessionResource: URI, agentId: string, command: string | undefined, numFollowups: number): void { this.telemetryService.publicLog2('chatFollowupsRetrieved', { + isAgentHostSession: getIsAgentHostSessionForTelemetry(sessionResource), agentId, command, numFollowups, @@ -325,6 +373,7 @@ export class ChatRequestTelemetry { chatMode: this.opts.options?.modeInfo?.telemetryModeName ?? this.opts.options?.modeInfo?.telemetryModeId, sessionType: getChatSessionTypeForTelemetry(this.opts.sessionResource), harness: getHarnessForTelemetry(this.opts.sessionResource), + isAgentHostSession: getIsAgentHostSessionForTelemetry(this.opts.sessionResource), }); } @@ -378,6 +427,10 @@ function getChatSessionTypeForTelemetry(sessionResource: URI): string { return isRemoteAgentHostSessionType(sessionType) ? 'remote-agent-host' : sessionType; } +function getIsAgentHostSessionForTelemetry(sessionResource: URI): boolean { + return isAgentHostSessionResource(sessionResource); +} + /** * For remote agent host sessions, the underlying harness/provider so remote * activity can be split by harness (the collapsed sessionType cannot). See diff --git a/src/vs/workbench/contrib/chat/common/chatSessionsService.ts b/src/vs/workbench/contrib/chat/common/chatSessionsService.ts index 2a00f4c78ff..cd031faf26f 100644 --- a/src/vs/workbench/contrib/chat/common/chatSessionsService.ts +++ b/src/vs/workbench/contrib/chat/common/chatSessionsService.ts @@ -400,6 +400,10 @@ export function isAgentHostTarget(target: string): boolean { return isLocalAgentHostTarget(target) || isRemoteAgentHostTarget(target); } +export function isAgentHostSessionResource(resource: URI): boolean { + return isAgentHostTarget(resource.scheme); +} + /** * The session type used for local agent chat sessions. */ diff --git a/src/vs/workbench/contrib/chat/test/common/chatService/chatService.test.ts b/src/vs/workbench/contrib/chat/test/common/chatService/chatService.test.ts index fa9cb0decff..27482ee542a 100644 --- a/src/vs/workbench/contrib/chat/test/common/chatService/chatService.test.ts +++ b/src/vs/workbench/contrib/chat/test/common/chatService/chatService.test.ts @@ -28,7 +28,8 @@ import { MockContextKeyService } from '../../../../../../platform/keybinding/tes import { ILogService, NullLogService } from '../../../../../../platform/log/common/log.js'; import { IStorageService, StorageScope, WillSaveStateReason } from '../../../../../../platform/storage/common/storage.js'; import { ITelemetryService } from '../../../../../../platform/telemetry/common/telemetry.js'; -import { NullTelemetryService } from '../../../../../../platform/telemetry/common/telemetryUtils.js'; +import { NullTelemetryService, NullTelemetryServiceShape } from '../../../../../../platform/telemetry/common/telemetryUtils.js'; +import { ClassifiedEvent, IGDPRProperty, OmitMetadata, StrictPropertyCheck } from '../../../../../../platform/telemetry/common/gdprTypings.js'; import { IUserDataProfilesService, toUserDataProfile } from '../../../../../../platform/userDataProfile/common/userDataProfile.js'; import { IWorkspaceContextService } from '../../../../../../platform/workspace/common/workspace.js'; import { IWorkbenchAssignmentService } from '../../../../../services/assignment/common/assignmentService.js'; @@ -45,8 +46,9 @@ import { IChatRequestVariableEntry } from '../../../common/attachments/chatVaria import { IChatVariablesService } from '../../../common/attachments/chatVariables.js'; import { IChatDebugService } from '../../../common/chatDebugService.js'; import { ChatDebugServiceImpl } from '../../../common/chatDebugServiceImpl.js'; -import { ChatRequestQueueKind, ChatSendResult, IChatFollowup, IChatModelReference, IChatProgress, IChatService, ResponseModelState } from '../../../common/chatService/chatService.js'; +import { ChatRequestQueueKind, ChatSendResult, IChatFollowup, IChatModelReference, IChatProgress, IChatService, IChatUserActionEvent, ResponseModelState } from '../../../common/chatService/chatService.js'; import { backfillTransferredModel, backfillRestoredPickerState, ChatService } from '../../../common/chatService/chatServiceImpl.js'; +import { ChatServiceTelemetry } from '../../../common/chatService/chatServiceTelemetry.js'; import { ChatRequestOriginKind } from '../../../common/chatRequestOrigin.js'; import { ChatAgentLocation, ChatModeKind } from '../../../common/constants.js'; import { ChatEditingSessionState, IChatEditingService, IChatEditingSession, IModifiedFileEntry, ModifiedFileEntryState } from '../../../common/editing/chatEditingService.js'; @@ -62,7 +64,7 @@ import { MockChatVariablesService } from '../mockChatVariables.js'; import { MockPromptsService } from '../promptSyntax/service/mockPromptsService.js'; import { MockLanguageModelToolsService } from '../tools/mockLanguageModelToolsService.js'; import { MockChatService } from './mockChatService.js'; -import { ChatSessionOptionsMap, IChatSession, IChatSessionContentProvider, IChatSessionHistoryItem, IChatSessionItem, IChatSessionServerRequest, IChatSessionsService } from '../../../common/chatSessionsService.js'; +import { ChatSessionOptionsMap, IChatSession, IChatSessionContentProvider, IChatSessionHistoryItem, IChatSessionItem, IChatSessionServerRequest, IChatSessionsService, SessionType } from '../../../common/chatSessionsService.js'; import { MockChatSessionsService } from '../mockChatSessionsService.js'; import { AGENT_DEBUG_LOG_FILE_LOGGING_ENABLED_SETTING, COPILOT_SKILL_URI_SCHEME, TROUBLESHOOT_SKILL_PATH } from '../../../common/promptSyntax/promptTypes.js'; import { ChatRequestSlashPromptPart } from '../../../common/requestParser/chatParserTypes.js'; @@ -2098,8 +2100,61 @@ suite('ChatService', () => { assert.deepStrictEqual(providerInvokedEvents.map(event => ({ sessionType: event.sessionType, + isAgentHostSession: event.isAgentHostSession, hasRequestId: typeof event.requestId === 'string', - })), [{ sessionType: 'remote-agent-host', hasRequestId: true }]); + })), [{ sessionType: 'remote-agent-host', isAgentHostSession: true, hasRequestId: true }]); + }); + + test('user action telemetry distinguishes agent host sessions from local sessions', () => { + const telemetryEvents: { readonly name: string; readonly isAgentHostSession: boolean }[] = []; + class TestTelemetryService extends NullTelemetryServiceShape { + override publicLog2> = never, T extends IGDPRProperty = never>(name?: string, data?: StrictPropertyCheck): void { + const isAgentHostSession = data && typeof data === 'object' ? Reflect.get(data, 'isAgentHostSession') : undefined; + if ((name === 'chatEditHunk' || name === 'chatEditSession') && typeof isAgentHostSession === 'boolean') { + telemetryEvents.push({ name, isAgentHostSession }); + } + } + } + const telemetry = new ChatServiceTelemetry(new TestTelemetryService()); + const sessionAction = { + action: { + kind: 'chatEditingSessionAction', + uri: URI.file('/test/file.ts'), + outcome: 'accepted', + hasRemainingEdits: false, + }, + agentId: 'agent', + command: undefined, + requestId: 'request', + result: undefined, + } satisfies Omit; + const action = { + action: { + kind: 'chatEditingHunkAction', + uri: URI.file('/test/file.ts'), + lineCount: 1, + linesAdded: 1, + linesRemoved: 0, + outcome: 'accepted', + hasRemainingEdits: false, + }, + agentId: 'agent', + command: undefined, + requestId: 'request', + result: undefined, + } satisfies Omit; + + telemetry.notifyUserAction({ ...sessionAction, sessionResource: URI.from({ scheme: SessionType.AgentHostCopilot, path: '/session' }) }); + telemetry.notifyUserAction({ ...sessionAction, sessionResource: URI.from({ scheme: SessionType.Local, path: '/session' }) }); + telemetry.notifyUserAction({ ...action, sessionResource: URI.from({ scheme: SessionType.AgentHostCopilot, path: '/session' }) }); + telemetry.notifyUserAction({ ...action, sessionResource: URI.from({ scheme: SessionType.Local, path: '/session' }) }); + + assert.deepStrictEqual(telemetryEvents, [ + { name: 'chatEditSession', isAgentHostSession: true }, + { name: 'chatEditSession', isAgentHostSession: false }, + { name: 'chatEditHunk', isAgentHostSession: true }, + { name: 'chatEditHunk', isAgentHostSession: false }, + ]); }); test('sendRequest with agentIdSilent passes agent host session capabilities to the request parser', async () => { diff --git a/src/vs/workbench/contrib/editTelemetry/browser/telemetry/aiEditTelemetry/aiEditTelemetryService.ts b/src/vs/workbench/contrib/editTelemetry/browser/telemetry/aiEditTelemetry/aiEditTelemetryService.ts index b349871d0e8..a6eac7374c8 100644 --- a/src/vs/workbench/contrib/editTelemetry/browser/telemetry/aiEditTelemetry/aiEditTelemetryService.ts +++ b/src/vs/workbench/contrib/editTelemetry/browser/telemetry/aiEditTelemetry/aiEditTelemetryService.ts @@ -65,6 +65,9 @@ export interface IEditTelemetryBaseData { /** Source controlled id. For agent edits (sideBarChat/highlightedEdit) this is the chat request id. */ sourceRequestId: string | undefined; + + /** Whether the edit was generated by an Agent Host-backed chat session. */ + isAgentHostSession?: boolean; } export interface IEditTelemetryCodeSuggestedData extends IEditTelemetryBaseData { diff --git a/src/vs/workbench/contrib/editTelemetry/browser/telemetry/aiEditTelemetry/aiEditTelemetryServiceImpl.ts b/src/vs/workbench/contrib/editTelemetry/browser/telemetry/aiEditTelemetry/aiEditTelemetryServiceImpl.ts index cc3bbf0e368..b557846e4da 100644 --- a/src/vs/workbench/contrib/editTelemetry/browser/telemetry/aiEditTelemetry/aiEditTelemetryServiceImpl.ts +++ b/src/vs/workbench/contrib/editTelemetry/browser/telemetry/aiEditTelemetry/aiEditTelemetryServiceImpl.ts @@ -45,6 +45,7 @@ export class AiEditTelemetryServiceImpl implements IAiEditTelemetryService { modelId: string | undefined; applyCodeBlockSuggestionId: string | undefined; sourceRequestId: string | undefined; + isAgentHostSession: boolean | undefined; }, { owner: 'hediet'; comment: 'Reports when code from AI is suggested to the user. @sentToGitHub'; @@ -69,6 +70,7 @@ export class AiEditTelemetryServiceImpl implements IAiEditTelemetryService { modelId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The AI model used to generate the suggestion.' }; applyCodeBlockSuggestionId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'If this suggestion is for applying a suggested code block, this is the id of the suggested code block.' }; sourceRequestId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The chat request ID that produced the suggestion, for correlating suggestions with specific requests.' }; + isAgentHostSession: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether the edit was generated by an Agent Host-backed chat session.' }; }>('editTelemetry.codeSuggested', { eventId: this._randomService.generatePrefixedUuid('evt'), suggestionId: suggestionId as unknown as string, @@ -89,6 +91,7 @@ export class AiEditTelemetryServiceImpl implements IAiEditTelemetryService { modelId: escapeModelIdForTelemetry(data.modelId), applyCodeBlockSuggestionId: data.applyCodeBlockSuggestionId as unknown as string, sourceRequestId: data.sourceRequestId, + isAgentHostSession: data.isAgentHostSession, ...forwardToChannelIf(isCopilotLikeExtension(data.source?.extensionId)), }); @@ -119,6 +122,7 @@ export class AiEditTelemetryServiceImpl implements IAiEditTelemetryService { modelId: string | undefined; applyCodeBlockSuggestionId: string | undefined; sourceRequestId: string | undefined; + isAgentHostSession: boolean | undefined; acceptanceMethod: | 'insertAtCursor' @@ -152,6 +156,7 @@ export class AiEditTelemetryServiceImpl implements IAiEditTelemetryService { applyCodeBlockSuggestionId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'If this suggestion is for applying a suggested code block, this is the id of the suggested code block.' }; sourceRequestId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The chat request ID that produced the edit, for correlating accepts/rejects with specific requests.' }; + isAgentHostSession: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether the edit was generated by an Agent Host-backed chat session.' }; acceptanceMethod: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'How the user accepted the code suggestion. See #IEditTelemetryCodeAcceptedData.acceptanceMethod for possible values.' }; }>('editTelemetry.codeAccepted', { eventId: this._randomService.generatePrefixedUuid('evt'), @@ -173,6 +178,7 @@ export class AiEditTelemetryServiceImpl implements IAiEditTelemetryService { modelId: escapeModelIdForTelemetry(data.modelId), applyCodeBlockSuggestionId: data.applyCodeBlockSuggestionId as unknown as string, sourceRequestId: data.sourceRequestId, + isAgentHostSession: data.isAgentHostSession, acceptanceMethod: data.acceptanceMethod, ...forwardToChannelIf(isCopilotLikeExtension(data.source?.extensionId)), @@ -201,6 +207,7 @@ export class AiEditTelemetryServiceImpl implements IAiEditTelemetryService { modelId: string | undefined; applyCodeBlockSuggestionId: string | undefined; sourceRequestId: string | undefined; + isAgentHostSession: boolean | undefined; rejectionMethod: 'reject'; }, { @@ -228,6 +235,7 @@ export class AiEditTelemetryServiceImpl implements IAiEditTelemetryService { applyCodeBlockSuggestionId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'If this suggestion is for applying a suggested code block, this is the id of the suggested code block.' }; sourceRequestId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The chat request ID that produced the edit, for correlating accepts/rejects with specific requests.' }; + isAgentHostSession: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether the edit was generated by an Agent Host-backed chat session.' }; rejectionMethod: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'How the user rejected the code suggestion. See #IEditTelemetryCodeRejectedData.rejectionMethod for possible values.' }; }>('editTelemetry.codeRejected', { eventId: this._randomService.generatePrefixedUuid('evt'), @@ -249,6 +257,7 @@ export class AiEditTelemetryServiceImpl implements IAiEditTelemetryService { modelId: escapeModelIdForTelemetry(data.modelId), applyCodeBlockSuggestionId: data.applyCodeBlockSuggestionId as unknown as string, sourceRequestId: data.sourceRequestId, + isAgentHostSession: data.isAgentHostSession, rejectionMethod: data.rejectionMethod, ...forwardToChannelIf(isCopilotLikeExtension(data.source?.extensionId)), diff --git a/src/vs/workbench/contrib/editTelemetry/test/browser/editTelemetry.test.ts b/src/vs/workbench/contrib/editTelemetry/test/browser/editTelemetry.test.ts index 9020dd44fcf..c87eba43df6 100644 --- a/src/vs/workbench/contrib/editTelemetry/test/browser/editTelemetry.test.ts +++ b/src/vs/workbench/contrib/editTelemetry/test/browser/editTelemetry.test.ts @@ -36,6 +36,43 @@ import { ITextFileService } from '../../../../services/textfile/common/textfiles suite('Edit Telemetry', () => { ensureNoDisposablesAreLeakedInTestSuite(); + test('reports Agent Host session mode for accepted and rejected edits', () => { + const instantiationService = new TestInstantiationService(); + const sentTelemetry: { readonly eventName: string; readonly data: Record | undefined }[] = []; + instantiationService.stub(ITelemetryService, { + publicLog2(eventName, data) { + sentTelemetry.push({ eventName, data }); + }, + }); + instantiationService.stub(IRandomService, new DeterministicRandomService()); + const aiEditTelemetryService = instantiationService.createInstance(AiEditTelemetryServiceImpl); + const baseData = { + suggestionId: undefined, + presentation: 'highlightedEdit' as const, + feature: 'inlineChat' as const, + source: undefined, + languageId: undefined, + editDeltaInfo: undefined, + modeId: undefined, + applyCodeBlockSuggestionId: undefined, + modelId: undefined, + sourceRequestId: undefined, + }; + + aiEditTelemetryService.createSuggestionId({ ...baseData, isAgentHostSession: true }); + aiEditTelemetryService.handleCodeAccepted({ ...baseData, acceptanceMethod: 'accept', isAgentHostSession: true }); + aiEditTelemetryService.handleCodeRejected({ ...baseData, rejectionMethod: 'reject', isAgentHostSession: false }); + + assert.deepStrictEqual(sentTelemetry.map(event => ({ + eventName: event.eventName, + isAgentHostSession: event.data?.isAgentHostSession, + })), [ + { eventName: 'editTelemetry.codeSuggested', isAgentHostSession: true }, + { eventName: 'editTelemetry.codeAccepted', isAgentHostSession: true }, + { eventName: 'editTelemetry.codeRejected', isAgentHostSession: false }, + ]); + }); + test('1', async () => runWithFakedTimers({}, async () => { const disposables = new DisposableStore(); const instantiationService = disposables.add(new TestInstantiationService(new ServiceCollection(