chat: identify Agent Host telemetry sessions (#332000)

* chat: identify Agent Host telemetry sessions

Adds a session-mode property to chat request, user-action, edit, and follow-up telemetry. This lets telemetry split Agent Host sessions from legacy sessions, including local fallback sessions.

- Adds isAgentHostSession to workbench chat telemetry events.\n- Adds whole-file edit outcomes and tags hunk outcomes.\n- Tags shared accepted and rejected edit telemetry.\n- Adds focused tests for Agent Host and legacy telemetry values.

(Commit message generated by Copilot)

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

* chat: tag code block telemetry sessions

Tags sidebar code-block telemetry with the actual session mode. Corrects the remaining-edits value for user-modified file outcomes.

- Shares Agent Host session resource detection.\n- Tags code-block suggestions and acceptance actions.\n- Reports pending edits when users modify a reviewed file.\n- Extends focused telemetry coverage.

(Commit message generated by Copilot)

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

---------

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
Connor Peet
2026-08-21 17:56:37 +00:00
committed by GitHub
co-authored by Copilot
parent eb6755c072
commit 8f4fc8d327
11 changed files with 198 additions and 26 deletions
@@ -33,6 +33,7 @@ import { reviewEdits } from './reviewEdits.js';
import { ITerminalEditorService, ITerminalGroupService, ITerminalService } from '../../../terminal/browser/terminal.js';
import { ChatContextKeys } from '../../common/actions/chatContextKeys.js';
import { ChatCopyKind, IChatService } from '../../common/chatService/chatService.js';
import { isAgentHostSessionResource } from '../../common/chatSessionsService.js';
import { IChatRequestViewModel, IChatResponseViewModel, isRequestVM, isResponseVM } from '../../common/model/chatViewModel.js';
import { ChatAgentLocation } from '../../common/constants.js';
import { IChatCodeBlockContextProviderService, IChatWidgetService } from '../chat.js';
@@ -201,6 +202,7 @@ export function registerChatCodeBlockActions() {
applyCodeBlockSuggestionId: undefined,
source: undefined,
sourceRequestId: undefined,
isAgentHostSession: isAgentHostSessionResource(context.element.sessionResource),
});
}
}
@@ -269,6 +271,7 @@ export function registerChatCodeBlockActions() {
applyCodeBlockSuggestionId: undefined,
source: undefined,
sourceRequestId: undefined,
isAgentHostSession: isAgentHostSessionResource(element.sessionResource),
});
}
@@ -427,6 +430,7 @@ export function registerChatCodeBlockActions() {
applyCodeBlockSuggestionId: undefined,
source: undefined,
sourceRequestId: undefined,
isAgentHostSession: isAgentHostSessionResource(context.element.sessionResource),
});
}
}
@@ -36,6 +36,7 @@ import { CellKind, ICellEditOperation, NOTEBOOK_EDITOR_ID } from '../../../noteb
import { INotebookService } from '../../../notebook/common/notebookService.js';
import { ICodeMapperCodeBlock, ICodeMapperRequest, ICodeMapperResponse, ICodeMapperService } from '../../common/editing/chatCodeMapperService.js';
import { ChatUserAction, IChatService } from '../../common/chatService/chatService.js';
import { isAgentHostSessionResource } from '../../common/chatSessionsService.js';
import { IChatRequestViewModel, isRequestVM, isResponseVM } from '../../common/model/chatViewModel.js';
import { ICodeBlockActionContext } from '../widget/chatContentParts/codeBlockPart.js';
@@ -91,6 +92,7 @@ export class InsertCodeBlockOperation {
applyCodeBlockSuggestionId: undefined,
source: undefined,
sourceRequestId: undefined,
isAgentHostSession: isAgentHostSessionResource(context.element.sessionResource),
});
}
}
@@ -24,6 +24,7 @@ import { IFilesConfigurationService } from '../../../../services/filesConfigurat
import { IAiEditTelemetryService } from '../../../editTelemetry/browser/telemetry/aiEditTelemetry/aiEditTelemetryService.js';
import { ICellEditOperation } from '../../../notebook/common/notebookCommon.js';
import { ChatUserAction, IChatService } from '../../common/chatService/chatService.js';
import { isAgentHostSessionResource } from '../../common/chatSessionsService.js';
import { ChatEditKind, IModifiedEntryTelemetryInfo, IModifiedFileEntry, IModifiedFileEntryEditorIntegration, ISnapshotEntry, ModifiedFileEntryState } from '../../common/editing/chatEditingService.js';
import { IChatResponseModel } from '../../common/model/chatModel.js';
@@ -269,10 +270,11 @@ export abstract class AbstractChatEditingModifiedFileEntry extends Disposable im
protected abstract _doReject(): Promise<void>;
protected _notifySessionAction(outcome: 'accepted' | 'rejected' | 'userModified') {
this._notifyAction({ kind: 'chatEditingSessionAction', uri: this.modifiedURI, hasRemainingEdits: false, outcome });
this._notifyAction({ kind: 'chatEditingSessionAction', uri: this.modifiedURI, hasRemainingEdits: outcome === 'userModified', outcome });
}
protected _notifyAction(action: ChatUserAction) {
const isAgentHostSession = isAgentHostSessionResource(this._telemetryInfo.sessionResource);
if (action.kind === 'chatEditingHunkAction' && action.outcome === 'accepted') {
this._aiEditTelemetryService.handleCodeAccepted({
suggestionId: undefined, // TODO@hediet try to figure this out
@@ -291,6 +293,7 @@ export abstract class AbstractChatEditingModifiedFileEntry extends Disposable im
languageId: action.languageId,
source: undefined,
sourceRequestId: this._telemetryInfo.requestId,
isAgentHostSession,
});
} else if (action.kind === 'chatEditingHunkAction' && action.outcome === 'rejected') {
this._aiEditTelemetryService.handleCodeRejected({
@@ -310,6 +313,7 @@ export abstract class AbstractChatEditingModifiedFileEntry extends Disposable im
languageId: action.languageId,
source: undefined,
sourceRequestId: this._telemetryInfo.requestId,
isAgentHostSession,
});
}
@@ -47,7 +47,7 @@ import { extractCodeblockUrisFromText, extractVulnerabilitiesFromText } from '..
import { IEditSessionEntryDiff } from '../../../common/editing/chatEditingService.js';
import { IChatProgressRenderableResponseContent } from '../../../common/model/chatModel.js';
import { IChatContentInlineReference, IChatMarkdownContent, IChatService, IChatUndoStop } from '../../../common/chatService/chatService.js';
import { IChatSessionsService } from '../../../common/chatSessionsService.js';
import { IChatSessionsService, isAgentHostSessionResource } from '../../../common/chatSessionsService.js';
import { isRequestVM, isResponseVM } from '../../../common/model/chatViewModel.js';
import { ChatConfiguration } from '../../../common/constants.js';
import { IChatCodeBlockInfo } from '../../chat.js';
@@ -384,6 +384,7 @@ export class ChatMarkdownContentPart extends Disposable implements IChatContentP
applyCodeBlockSuggestionId: undefined,
source: undefined,
sourceRequestId: undefined,
isAgentHostSession: isAgentHostSessionResource(element.sessionResource),
})
};
}));
@@ -1810,7 +1810,7 @@ export class ChatService extends Disposable implements IChatService {
agentOrCommandFollowups.then(followups => {
model.setFollowups(completedRequest, followups);
const commandForTelemetry = agentSlashCommandPart ? agentSlashCommandPart.command.name : commandPart?.slashCommand.command;
this._chatServiceTelemetry.retrievedFollowups(agentPart?.agent.id ?? '', commandForTelemetry, followups?.length ?? 0);
this._chatServiceTelemetry.retrievedFollowups(model.sessionResource, agentPart?.agent.id ?? '', commandForTelemetry, followups?.length ?? 0);
});
}
}
@@ -14,15 +14,24 @@ import { isImageVariableEntry } from '../attachments/chatVariableEntries.js';
import { ChatAgentLocation, ChatModeKind, ChatPermissionLevel } from '../constants.js';
import { ILanguageModelsService } from '../languageModels.js';
import { chatSessionResourceToId, getChatSessionType } from '../model/chatUri.js';
import { isAgentHostSessionResource } from '../chatSessionsService.js';
import { isRemoteAgentHostSessionType, parseRemoteAgentHostHarness } from '../../../../../platform/agentHost/common/agentHostSessionType.js';
type ChatVoteEvent = {
type ChatSessionModeEvent = {
isAgentHostSession: boolean;
};
type ChatSessionModeClassification = {
isAgentHostSession: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether the action was performed in an Agent Host-backed chat session.' };
};
type ChatVoteEvent = ChatSessionModeEvent & {
direction: 'up' | 'down';
agentId: string;
command: string | undefined;
};
type ChatVoteClassification = {
type ChatVoteClassification = ChatSessionModeClassification & {
direction: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether the user voted up or down.' };
agentId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the chat agent that this vote is for.' };
command: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The name of the slash command that this vote is for.' };
@@ -30,13 +39,13 @@ type ChatVoteClassification = {
comment: 'Provides insight into the performance of Chat agents.';
};
type ChatCopyEvent = {
type ChatCopyEvent = ChatSessionModeEvent & {
copyKind: 'action' | 'toolbar';
agentId: string;
command: string | undefined;
};
type ChatCopyClassification = {
type ChatCopyClassification = ChatSessionModeClassification & {
copyKind: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'How the copy was initiated.' };
agentId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the chat agent that the copy acted on.' };
command: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The name of the slash command the copy acted on.' };
@@ -44,13 +53,13 @@ type ChatCopyClassification = {
comment: 'Provides insight into the usage of Chat features.';
};
type ChatInsertEvent = {
type ChatInsertEvent = ChatSessionModeEvent & {
newFile: boolean;
agentId: string;
command: string | undefined;
};
type ChatInsertClassification = {
type ChatInsertClassification = ChatSessionModeClassification & {
newFile: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether the code was inserted into a new untitled file.' };
agentId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the chat agent that this insertion is for.' };
command: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The name of the slash command that this insertion is for.' };
@@ -58,7 +67,7 @@ type ChatInsertClassification = {
comment: 'Provides insight into the usage of Chat features.';
};
type ChatApplyEvent = {
type ChatApplyEvent = ChatSessionModeEvent & {
newFile: boolean;
agentId: string;
command: string | undefined;
@@ -66,7 +75,7 @@ type ChatApplyEvent = {
editsProposed: boolean;
};
type ChatApplyClassification = {
type ChatApplyClassification = ChatSessionModeClassification & {
newFile: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether the code was inserted into a new untitled file.' };
agentId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the chat agent that this insertion is for.' };
command: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The name of the slash command that this insertion is for.' };
@@ -76,25 +85,25 @@ type ChatApplyClassification = {
comment: 'Provides insight into the usage of Chat features.';
};
type ChatFollowupEvent = {
type ChatFollowupEvent = ChatSessionModeEvent & {
agentId: string;
command: string | undefined;
};
type ChatFollowupClassification = {
type ChatFollowupClassification = ChatSessionModeClassification & {
agentId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the related chat agent.' };
command: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The name of the related slash command.' };
owner: 'roblourens';
comment: 'Provides insight into the usage of Chat features.';
};
type ChatTerminalEvent = {
type ChatTerminalEvent = ChatSessionModeEvent & {
languageId: string;
agentId: string;
command: string | undefined;
};
type ChatTerminalClassification = {
type ChatTerminalClassification = ChatSessionModeClassification & {
languageId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The language of the code that was run in the terminal.' };
agentId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the related chat agent.' };
command: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The name of the related slash command.' };
@@ -102,13 +111,13 @@ type ChatTerminalClassification = {
comment: 'Provides insight into the usage of Chat features.';
};
type ChatFollowupsRetrievedEvent = {
type ChatFollowupsRetrievedEvent = ChatSessionModeEvent & {
agentId: string;
command: string | undefined;
numFollowups: number;
};
type ChatFollowupsRetrievedClassification = {
type ChatFollowupsRetrievedClassification = ChatSessionModeClassification & {
agentId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the related chat agent.' };
command: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The name of the related slash command.' };
numFollowups: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The number of followup prompts returned by the agent.' };
@@ -116,7 +125,27 @@ type ChatFollowupsRetrievedClassification = {
comment: 'Provides insight into the usage of Chat features.';
};
type ChatEditHunkEvent = {
type ChatEditSessionEvent = ChatSessionModeEvent & {
agentId: string;
outcome: 'accepted' | 'rejected' | 'userModified';
hasRemainingEdits: boolean;
requestId: string;
modelId: string;
modeId: string;
};
type ChatEditSessionClassification = ChatSessionModeClassification & {
agentId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the related chat agent.' };
outcome: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The outcome of the edited file action.' };
hasRemainingEdits: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether there are remaining edits in the file after this action.' };
requestId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the chat request that produced the edit.' };
modelId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The AI model used to generate the edit.' };
modeId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The chat mode used for the request (e.g. ask, edit, agent).' };
owner: 'roblourens';
comment: 'Provides insight into the usage of Chat features.';
};
type ChatEditHunkEvent = ChatSessionModeEvent & {
agentId: string;
outcome: 'accepted' | 'rejected';
lineCount: number;
@@ -126,7 +155,7 @@ type ChatEditHunkEvent = {
modeId: string;
};
type ChatEditHunkClassification = {
type ChatEditHunkClassification = ChatSessionModeClassification & {
agentId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the related chat agent.' };
outcome: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The outcome of the edit hunk action.' };
lineCount: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The number of lines in the relevant change.' };
@@ -138,7 +167,7 @@ type ChatEditHunkClassification = {
comment: 'Provides insight into the usage of Chat features.';
};
export type ChatProviderInvokedEvent = {
export type ChatProviderInvokedEvent = ChatSessionModeEvent & {
timeToFirstProgress: number | undefined;
totalTime: number | undefined;
result: 'success' | 'error' | 'errorWithOutput' | 'cancelled' | 'filtered';
@@ -161,7 +190,7 @@ export type ChatProviderInvokedEvent = {
harness: string | undefined;
};
export type ChatProviderInvokedClassification = {
export type ChatProviderInvokedClassification = ChatSessionModeClassification & {
timeToFirstProgress: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'The time in milliseconds from invoking the provider to getting the first data.' };
totalTime: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'The total time it took to run the provider\'s `provideResponseWithProgress`.' };
result: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether invoking the ChatProvider resulted in an error.' };
@@ -192,26 +221,31 @@ export class ChatServiceTelemetry {
) { }
notifyUserAction(action: IChatUserActionEvent): void {
const isAgentHostSession = getIsAgentHostSessionForTelemetry(action.sessionResource);
if (action.action.kind === 'vote') {
this.telemetryService.publicLog2<ChatVoteEvent, ChatVoteClassification>('interactiveSessionVote', {
isAgentHostSession,
direction: action.action.direction === ChatAgentVoteDirection.Up ? 'up' : 'down',
agentId: action.agentId ?? '',
command: action.command,
});
} else if (action.action.kind === 'copy') {
this.telemetryService.publicLog2<ChatCopyEvent, ChatCopyClassification>('interactiveSessionCopy', {
isAgentHostSession,
copyKind: action.action.copyKind === ChatCopyKind.Action ? 'action' : 'toolbar',
agentId: action.agentId ?? '',
command: action.command,
});
} else if (action.action.kind === 'insert') {
this.telemetryService.publicLog2<ChatInsertEvent, ChatInsertClassification>('interactiveSessionInsert', {
isAgentHostSession,
newFile: !!action.action.newFile,
agentId: action.agentId ?? '',
command: action.command,
});
} else if (action.action.kind === 'apply') {
this.telemetryService.publicLog2<ChatApplyEvent, ChatApplyClassification>('interactiveSessionApply', {
isAgentHostSession,
newFile: !!action.action.newFile,
codeMapper: action.action.codeMapper,
agentId: action.agentId ?? '',
@@ -220,17 +254,30 @@ export class ChatServiceTelemetry {
});
} else if (action.action.kind === 'runInTerminal') {
this.telemetryService.publicLog2<ChatTerminalEvent, ChatTerminalClassification>('interactiveSessionRunInTerminal', {
isAgentHostSession,
languageId: action.action.languageId ?? '',
agentId: action.agentId ?? '',
command: action.command,
});
} else if (action.action.kind === 'followUp') {
this.telemetryService.publicLog2<ChatFollowupEvent, ChatFollowupClassification>('chatFollowupClicked', {
isAgentHostSession,
agentId: action.agentId ?? '',
command: action.command,
});
} else if (action.action.kind === 'chatEditingSessionAction') {
this.telemetryService.publicLog2<ChatEditSessionEvent, ChatEditSessionClassification>('chatEditSession', {
isAgentHostSession,
agentId: action.agentId ?? '',
outcome: action.action.outcome,
hasRemainingEdits: action.action.hasRemainingEdits,
requestId: action.requestId,
modelId: escapeModelIdForTelemetry(action.modelId) ?? '',
modeId: action.modeId ?? '',
});
} else if (action.action.kind === 'chatEditingHunkAction') {
this.telemetryService.publicLog2<ChatEditHunkEvent, ChatEditHunkClassification>('chatEditHunk', {
isAgentHostSession,
agentId: action.agentId ?? '',
outcome: action.action.outcome,
lineCount: action.action.lineCount,
@@ -242,8 +289,9 @@ export class ChatServiceTelemetry {
}
}
retrievedFollowups(agentId: string, command: string | undefined, numFollowups: number): void {
retrievedFollowups(sessionResource: URI, agentId: string, command: string | undefined, numFollowups: number): void {
this.telemetryService.publicLog2<ChatFollowupsRetrievedEvent, ChatFollowupsRetrievedClassification>('chatFollowupsRetrieved', {
isAgentHostSession: getIsAgentHostSessionForTelemetry(sessionResource),
agentId,
command,
numFollowups,
@@ -325,6 +373,7 @@ export class ChatRequestTelemetry {
chatMode: this.opts.options?.modeInfo?.telemetryModeName ?? this.opts.options?.modeInfo?.telemetryModeId,
sessionType: getChatSessionTypeForTelemetry(this.opts.sessionResource),
harness: getHarnessForTelemetry(this.opts.sessionResource),
isAgentHostSession: getIsAgentHostSessionForTelemetry(this.opts.sessionResource),
});
}
@@ -378,6 +427,10 @@ function getChatSessionTypeForTelemetry(sessionResource: URI): string {
return isRemoteAgentHostSessionType(sessionType) ? 'remote-agent-host' : sessionType;
}
function getIsAgentHostSessionForTelemetry(sessionResource: URI): boolean {
return isAgentHostSessionResource(sessionResource);
}
/**
* For remote agent host sessions, the underlying harness/provider so remote
* activity can be split by harness (the collapsed sessionType cannot). See
@@ -400,6 +400,10 @@ export function isAgentHostTarget(target: string): boolean {
return isLocalAgentHostTarget(target) || isRemoteAgentHostTarget(target);
}
export function isAgentHostSessionResource(resource: URI): boolean {
return isAgentHostTarget(resource.scheme);
}
/**
* The session type used for local agent chat sessions.
*/
@@ -28,7 +28,8 @@ import { MockContextKeyService } from '../../../../../../platform/keybinding/tes
import { ILogService, NullLogService } from '../../../../../../platform/log/common/log.js';
import { IStorageService, StorageScope, WillSaveStateReason } from '../../../../../../platform/storage/common/storage.js';
import { ITelemetryService } from '../../../../../../platform/telemetry/common/telemetry.js';
import { NullTelemetryService } from '../../../../../../platform/telemetry/common/telemetryUtils.js';
import { NullTelemetryService, NullTelemetryServiceShape } from '../../../../../../platform/telemetry/common/telemetryUtils.js';
import { ClassifiedEvent, IGDPRProperty, OmitMetadata, StrictPropertyCheck } from '../../../../../../platform/telemetry/common/gdprTypings.js';
import { IUserDataProfilesService, toUserDataProfile } from '../../../../../../platform/userDataProfile/common/userDataProfile.js';
import { IWorkspaceContextService } from '../../../../../../platform/workspace/common/workspace.js';
import { IWorkbenchAssignmentService } from '../../../../../services/assignment/common/assignmentService.js';
@@ -45,8 +46,9 @@ import { IChatRequestVariableEntry } from '../../../common/attachments/chatVaria
import { IChatVariablesService } from '../../../common/attachments/chatVariables.js';
import { IChatDebugService } from '../../../common/chatDebugService.js';
import { ChatDebugServiceImpl } from '../../../common/chatDebugServiceImpl.js';
import { ChatRequestQueueKind, ChatSendResult, IChatFollowup, IChatModelReference, IChatProgress, IChatService, ResponseModelState } from '../../../common/chatService/chatService.js';
import { ChatRequestQueueKind, ChatSendResult, IChatFollowup, IChatModelReference, IChatProgress, IChatService, IChatUserActionEvent, ResponseModelState } from '../../../common/chatService/chatService.js';
import { backfillTransferredModel, backfillRestoredPickerState, ChatService } from '../../../common/chatService/chatServiceImpl.js';
import { ChatServiceTelemetry } from '../../../common/chatService/chatServiceTelemetry.js';
import { ChatRequestOriginKind } from '../../../common/chatRequestOrigin.js';
import { ChatAgentLocation, ChatModeKind } from '../../../common/constants.js';
import { ChatEditingSessionState, IChatEditingService, IChatEditingSession, IModifiedFileEntry, ModifiedFileEntryState } from '../../../common/editing/chatEditingService.js';
@@ -62,7 +64,7 @@ import { MockChatVariablesService } from '../mockChatVariables.js';
import { MockPromptsService } from '../promptSyntax/service/mockPromptsService.js';
import { MockLanguageModelToolsService } from '../tools/mockLanguageModelToolsService.js';
import { MockChatService } from './mockChatService.js';
import { ChatSessionOptionsMap, IChatSession, IChatSessionContentProvider, IChatSessionHistoryItem, IChatSessionItem, IChatSessionServerRequest, IChatSessionsService } from '../../../common/chatSessionsService.js';
import { ChatSessionOptionsMap, IChatSession, IChatSessionContentProvider, IChatSessionHistoryItem, IChatSessionItem, IChatSessionServerRequest, IChatSessionsService, SessionType } from '../../../common/chatSessionsService.js';
import { MockChatSessionsService } from '../mockChatSessionsService.js';
import { AGENT_DEBUG_LOG_FILE_LOGGING_ENABLED_SETTING, COPILOT_SKILL_URI_SCHEME, TROUBLESHOOT_SKILL_PATH } from '../../../common/promptSyntax/promptTypes.js';
import { ChatRequestSlashPromptPart } from '../../../common/requestParser/chatParserTypes.js';
@@ -2098,8 +2100,61 @@ suite('ChatService', () => {
assert.deepStrictEqual(providerInvokedEvents.map(event => ({
sessionType: event.sessionType,
isAgentHostSession: event.isAgentHostSession,
hasRequestId: typeof event.requestId === 'string',
})), [{ sessionType: 'remote-agent-host', hasRequestId: true }]);
})), [{ sessionType: 'remote-agent-host', isAgentHostSession: true, hasRequestId: true }]);
});
test('user action telemetry distinguishes agent host sessions from local sessions', () => {
const telemetryEvents: { readonly name: string; readonly isAgentHostSession: boolean }[] = [];
class TestTelemetryService extends NullTelemetryServiceShape {
override publicLog2<E extends ClassifiedEvent<OmitMetadata<T>> = never, T extends IGDPRProperty = never>(name?: string, data?: StrictPropertyCheck<T, E>): void {
const isAgentHostSession = data && typeof data === 'object' ? Reflect.get(data, 'isAgentHostSession') : undefined;
if ((name === 'chatEditHunk' || name === 'chatEditSession') && typeof isAgentHostSession === 'boolean') {
telemetryEvents.push({ name, isAgentHostSession });
}
}
}
const telemetry = new ChatServiceTelemetry(new TestTelemetryService());
const sessionAction = {
action: {
kind: 'chatEditingSessionAction',
uri: URI.file('/test/file.ts'),
outcome: 'accepted',
hasRemainingEdits: false,
},
agentId: 'agent',
command: undefined,
requestId: 'request',
result: undefined,
} satisfies Omit<IChatUserActionEvent, 'sessionResource'>;
const action = {
action: {
kind: 'chatEditingHunkAction',
uri: URI.file('/test/file.ts'),
lineCount: 1,
linesAdded: 1,
linesRemoved: 0,
outcome: 'accepted',
hasRemainingEdits: false,
},
agentId: 'agent',
command: undefined,
requestId: 'request',
result: undefined,
} satisfies Omit<IChatUserActionEvent, 'sessionResource'>;
telemetry.notifyUserAction({ ...sessionAction, sessionResource: URI.from({ scheme: SessionType.AgentHostCopilot, path: '/session' }) });
telemetry.notifyUserAction({ ...sessionAction, sessionResource: URI.from({ scheme: SessionType.Local, path: '/session' }) });
telemetry.notifyUserAction({ ...action, sessionResource: URI.from({ scheme: SessionType.AgentHostCopilot, path: '/session' }) });
telemetry.notifyUserAction({ ...action, sessionResource: URI.from({ scheme: SessionType.Local, path: '/session' }) });
assert.deepStrictEqual(telemetryEvents, [
{ name: 'chatEditSession', isAgentHostSession: true },
{ name: 'chatEditSession', isAgentHostSession: false },
{ name: 'chatEditHunk', isAgentHostSession: true },
{ name: 'chatEditHunk', isAgentHostSession: false },
]);
});
test('sendRequest with agentIdSilent passes agent host session capabilities to the request parser', async () => {
@@ -65,6 +65,9 @@ export interface IEditTelemetryBaseData {
/** Source controlled id. For agent edits (sideBarChat/highlightedEdit) this is the chat request id. */
sourceRequestId: string | undefined;
/** Whether the edit was generated by an Agent Host-backed chat session. */
isAgentHostSession?: boolean;
}
export interface IEditTelemetryCodeSuggestedData extends IEditTelemetryBaseData {
@@ -45,6 +45,7 @@ export class AiEditTelemetryServiceImpl implements IAiEditTelemetryService {
modelId: string | undefined;
applyCodeBlockSuggestionId: string | undefined;
sourceRequestId: string | undefined;
isAgentHostSession: boolean | undefined;
}, {
owner: 'hediet';
comment: 'Reports when code from AI is suggested to the user. @sentToGitHub';
@@ -69,6 +70,7 @@ export class AiEditTelemetryServiceImpl implements IAiEditTelemetryService {
modelId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The AI model used to generate the suggestion.' };
applyCodeBlockSuggestionId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'If this suggestion is for applying a suggested code block, this is the id of the suggested code block.' };
sourceRequestId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The chat request ID that produced the suggestion, for correlating suggestions with specific requests.' };
isAgentHostSession: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether the edit was generated by an Agent Host-backed chat session.' };
}>('editTelemetry.codeSuggested', {
eventId: this._randomService.generatePrefixedUuid('evt'),
suggestionId: suggestionId as unknown as string,
@@ -89,6 +91,7 @@ export class AiEditTelemetryServiceImpl implements IAiEditTelemetryService {
modelId: escapeModelIdForTelemetry(data.modelId),
applyCodeBlockSuggestionId: data.applyCodeBlockSuggestionId as unknown as string,
sourceRequestId: data.sourceRequestId,
isAgentHostSession: data.isAgentHostSession,
...forwardToChannelIf(isCopilotLikeExtension(data.source?.extensionId)),
});
@@ -119,6 +122,7 @@ export class AiEditTelemetryServiceImpl implements IAiEditTelemetryService {
modelId: string | undefined;
applyCodeBlockSuggestionId: string | undefined;
sourceRequestId: string | undefined;
isAgentHostSession: boolean | undefined;
acceptanceMethod:
| 'insertAtCursor'
@@ -152,6 +156,7 @@ export class AiEditTelemetryServiceImpl implements IAiEditTelemetryService {
applyCodeBlockSuggestionId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'If this suggestion is for applying a suggested code block, this is the id of the suggested code block.' };
sourceRequestId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The chat request ID that produced the edit, for correlating accepts/rejects with specific requests.' };
isAgentHostSession: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether the edit was generated by an Agent Host-backed chat session.' };
acceptanceMethod: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'How the user accepted the code suggestion. See #IEditTelemetryCodeAcceptedData.acceptanceMethod for possible values.' };
}>('editTelemetry.codeAccepted', {
eventId: this._randomService.generatePrefixedUuid('evt'),
@@ -173,6 +178,7 @@ export class AiEditTelemetryServiceImpl implements IAiEditTelemetryService {
modelId: escapeModelIdForTelemetry(data.modelId),
applyCodeBlockSuggestionId: data.applyCodeBlockSuggestionId as unknown as string,
sourceRequestId: data.sourceRequestId,
isAgentHostSession: data.isAgentHostSession,
acceptanceMethod: data.acceptanceMethod,
...forwardToChannelIf(isCopilotLikeExtension(data.source?.extensionId)),
@@ -201,6 +207,7 @@ export class AiEditTelemetryServiceImpl implements IAiEditTelemetryService {
modelId: string | undefined;
applyCodeBlockSuggestionId: string | undefined;
sourceRequestId: string | undefined;
isAgentHostSession: boolean | undefined;
rejectionMethod: 'reject';
}, {
@@ -228,6 +235,7 @@ export class AiEditTelemetryServiceImpl implements IAiEditTelemetryService {
applyCodeBlockSuggestionId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'If this suggestion is for applying a suggested code block, this is the id of the suggested code block.' };
sourceRequestId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The chat request ID that produced the edit, for correlating accepts/rejects with specific requests.' };
isAgentHostSession: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether the edit was generated by an Agent Host-backed chat session.' };
rejectionMethod: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'How the user rejected the code suggestion. See #IEditTelemetryCodeRejectedData.rejectionMethod for possible values.' };
}>('editTelemetry.codeRejected', {
eventId: this._randomService.generatePrefixedUuid('evt'),
@@ -249,6 +257,7 @@ export class AiEditTelemetryServiceImpl implements IAiEditTelemetryService {
modelId: escapeModelIdForTelemetry(data.modelId),
applyCodeBlockSuggestionId: data.applyCodeBlockSuggestionId as unknown as string,
sourceRequestId: data.sourceRequestId,
isAgentHostSession: data.isAgentHostSession,
rejectionMethod: data.rejectionMethod,
...forwardToChannelIf(isCopilotLikeExtension(data.source?.extensionId)),
@@ -36,6 +36,43 @@ import { ITextFileService } from '../../../../services/textfile/common/textfiles
suite('Edit Telemetry', () => {
ensureNoDisposablesAreLeakedInTestSuite();
test('reports Agent Host session mode for accepted and rejected edits', () => {
const instantiationService = new TestInstantiationService();
const sentTelemetry: { readonly eventName: string; readonly data: Record<string, unknown> | undefined }[] = [];
instantiationService.stub(ITelemetryService, {
publicLog2(eventName, data) {
sentTelemetry.push({ eventName, data });
},
});
instantiationService.stub(IRandomService, new DeterministicRandomService());
const aiEditTelemetryService = instantiationService.createInstance(AiEditTelemetryServiceImpl);
const baseData = {
suggestionId: undefined,
presentation: 'highlightedEdit' as const,
feature: 'inlineChat' as const,
source: undefined,
languageId: undefined,
editDeltaInfo: undefined,
modeId: undefined,
applyCodeBlockSuggestionId: undefined,
modelId: undefined,
sourceRequestId: undefined,
};
aiEditTelemetryService.createSuggestionId({ ...baseData, isAgentHostSession: true });
aiEditTelemetryService.handleCodeAccepted({ ...baseData, acceptanceMethod: 'accept', isAgentHostSession: true });
aiEditTelemetryService.handleCodeRejected({ ...baseData, rejectionMethod: 'reject', isAgentHostSession: false });
assert.deepStrictEqual(sentTelemetry.map(event => ({
eventName: event.eventName,
isAgentHostSession: event.data?.isAgentHostSession,
})), [
{ eventName: 'editTelemetry.codeSuggested', isAgentHostSession: true },
{ eventName: 'editTelemetry.codeAccepted', isAgentHostSession: true },
{ eventName: 'editTelemetry.codeRejected', isAgentHostSession: false },
]);
});
test('1', async () => runWithFakedTimers({}, async () => {
const disposables = new DisposableStore();
const instantiationService = disposables.add(new TestInstantiationService(new ServiceCollection(