mirror of
https://github.com/microsoft/vscode.git
synced 2026-08-24 08:45:18 +01:00
chat: identify Agent Host telemetry sessions (#332000)
* chat: identify Agent Host telemetry sessions Adds a session-mode property to chat request, user-action, edit, and follow-up telemetry. This lets telemetry split Agent Host sessions from legacy sessions, including local fallback sessions. - Adds isAgentHostSession to workbench chat telemetry events.\n- Adds whole-file edit outcomes and tags hunk outcomes.\n- Tags shared accepted and rejected edit telemetry.\n- Adds focused tests for Agent Host and legacy telemetry values. (Commit message generated by Copilot) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * chat: tag code block telemetry sessions Tags sidebar code-block telemetry with the actual session mode. Corrects the remaining-edits value for user-modified file outcomes. - Shares Agent Host session resource detection.\n- Tags code-block suggestions and acceptance actions.\n- Reports pending edits when users modify a reviewed file.\n- Extends focused telemetry coverage. (Commit message generated by Copilot) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
@@ -33,6 +33,7 @@ import { reviewEdits } from './reviewEdits.js';
|
||||
import { ITerminalEditorService, ITerminalGroupService, ITerminalService } from '../../../terminal/browser/terminal.js';
|
||||
import { ChatContextKeys } from '../../common/actions/chatContextKeys.js';
|
||||
import { ChatCopyKind, IChatService } from '../../common/chatService/chatService.js';
|
||||
import { isAgentHostSessionResource } from '../../common/chatSessionsService.js';
|
||||
import { IChatRequestViewModel, IChatResponseViewModel, isRequestVM, isResponseVM } from '../../common/model/chatViewModel.js';
|
||||
import { ChatAgentLocation } from '../../common/constants.js';
|
||||
import { IChatCodeBlockContextProviderService, IChatWidgetService } from '../chat.js';
|
||||
@@ -201,6 +202,7 @@ export function registerChatCodeBlockActions() {
|
||||
applyCodeBlockSuggestionId: undefined,
|
||||
source: undefined,
|
||||
sourceRequestId: undefined,
|
||||
isAgentHostSession: isAgentHostSessionResource(context.element.sessionResource),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -269,6 +271,7 @@ export function registerChatCodeBlockActions() {
|
||||
applyCodeBlockSuggestionId: undefined,
|
||||
source: undefined,
|
||||
sourceRequestId: undefined,
|
||||
isAgentHostSession: isAgentHostSessionResource(element.sessionResource),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -427,6 +430,7 @@ export function registerChatCodeBlockActions() {
|
||||
applyCodeBlockSuggestionId: undefined,
|
||||
source: undefined,
|
||||
sourceRequestId: undefined,
|
||||
isAgentHostSession: isAgentHostSessionResource(context.element.sessionResource),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -36,6 +36,7 @@ import { CellKind, ICellEditOperation, NOTEBOOK_EDITOR_ID } from '../../../noteb
|
||||
import { INotebookService } from '../../../notebook/common/notebookService.js';
|
||||
import { ICodeMapperCodeBlock, ICodeMapperRequest, ICodeMapperResponse, ICodeMapperService } from '../../common/editing/chatCodeMapperService.js';
|
||||
import { ChatUserAction, IChatService } from '../../common/chatService/chatService.js';
|
||||
import { isAgentHostSessionResource } from '../../common/chatSessionsService.js';
|
||||
import { IChatRequestViewModel, isRequestVM, isResponseVM } from '../../common/model/chatViewModel.js';
|
||||
import { ICodeBlockActionContext } from '../widget/chatContentParts/codeBlockPart.js';
|
||||
|
||||
@@ -91,6 +92,7 @@ export class InsertCodeBlockOperation {
|
||||
applyCodeBlockSuggestionId: undefined,
|
||||
source: undefined,
|
||||
sourceRequestId: undefined,
|
||||
isAgentHostSession: isAgentHostSessionResource(context.element.sessionResource),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,6 +24,7 @@ import { IFilesConfigurationService } from '../../../../services/filesConfigurat
|
||||
import { IAiEditTelemetryService } from '../../../editTelemetry/browser/telemetry/aiEditTelemetry/aiEditTelemetryService.js';
|
||||
import { ICellEditOperation } from '../../../notebook/common/notebookCommon.js';
|
||||
import { ChatUserAction, IChatService } from '../../common/chatService/chatService.js';
|
||||
import { isAgentHostSessionResource } from '../../common/chatSessionsService.js';
|
||||
import { ChatEditKind, IModifiedEntryTelemetryInfo, IModifiedFileEntry, IModifiedFileEntryEditorIntegration, ISnapshotEntry, ModifiedFileEntryState } from '../../common/editing/chatEditingService.js';
|
||||
import { IChatResponseModel } from '../../common/model/chatModel.js';
|
||||
|
||||
@@ -269,10 +270,11 @@ export abstract class AbstractChatEditingModifiedFileEntry extends Disposable im
|
||||
protected abstract _doReject(): Promise<void>;
|
||||
|
||||
protected _notifySessionAction(outcome: 'accepted' | 'rejected' | 'userModified') {
|
||||
this._notifyAction({ kind: 'chatEditingSessionAction', uri: this.modifiedURI, hasRemainingEdits: false, outcome });
|
||||
this._notifyAction({ kind: 'chatEditingSessionAction', uri: this.modifiedURI, hasRemainingEdits: outcome === 'userModified', outcome });
|
||||
}
|
||||
|
||||
protected _notifyAction(action: ChatUserAction) {
|
||||
const isAgentHostSession = isAgentHostSessionResource(this._telemetryInfo.sessionResource);
|
||||
if (action.kind === 'chatEditingHunkAction' && action.outcome === 'accepted') {
|
||||
this._aiEditTelemetryService.handleCodeAccepted({
|
||||
suggestionId: undefined, // TODO@hediet try to figure this out
|
||||
@@ -291,6 +293,7 @@ export abstract class AbstractChatEditingModifiedFileEntry extends Disposable im
|
||||
languageId: action.languageId,
|
||||
source: undefined,
|
||||
sourceRequestId: this._telemetryInfo.requestId,
|
||||
isAgentHostSession,
|
||||
});
|
||||
} else if (action.kind === 'chatEditingHunkAction' && action.outcome === 'rejected') {
|
||||
this._aiEditTelemetryService.handleCodeRejected({
|
||||
@@ -310,6 +313,7 @@ export abstract class AbstractChatEditingModifiedFileEntry extends Disposable im
|
||||
languageId: action.languageId,
|
||||
source: undefined,
|
||||
sourceRequestId: this._telemetryInfo.requestId,
|
||||
isAgentHostSession,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
+2
-1
@@ -47,7 +47,7 @@ import { extractCodeblockUrisFromText, extractVulnerabilitiesFromText } from '..
|
||||
import { IEditSessionEntryDiff } from '../../../common/editing/chatEditingService.js';
|
||||
import { IChatProgressRenderableResponseContent } from '../../../common/model/chatModel.js';
|
||||
import { IChatContentInlineReference, IChatMarkdownContent, IChatService, IChatUndoStop } from '../../../common/chatService/chatService.js';
|
||||
import { IChatSessionsService } from '../../../common/chatSessionsService.js';
|
||||
import { IChatSessionsService, isAgentHostSessionResource } from '../../../common/chatSessionsService.js';
|
||||
import { isRequestVM, isResponseVM } from '../../../common/model/chatViewModel.js';
|
||||
import { ChatConfiguration } from '../../../common/constants.js';
|
||||
import { IChatCodeBlockInfo } from '../../chat.js';
|
||||
@@ -384,6 +384,7 @@ export class ChatMarkdownContentPart extends Disposable implements IChatContentP
|
||||
applyCodeBlockSuggestionId: undefined,
|
||||
source: undefined,
|
||||
sourceRequestId: undefined,
|
||||
isAgentHostSession: isAgentHostSessionResource(element.sessionResource),
|
||||
})
|
||||
};
|
||||
}));
|
||||
|
||||
@@ -1810,7 +1810,7 @@ export class ChatService extends Disposable implements IChatService {
|
||||
agentOrCommandFollowups.then(followups => {
|
||||
model.setFollowups(completedRequest, followups);
|
||||
const commandForTelemetry = agentSlashCommandPart ? agentSlashCommandPart.command.name : commandPart?.slashCommand.command;
|
||||
this._chatServiceTelemetry.retrievedFollowups(agentPart?.agent.id ?? '', commandForTelemetry, followups?.length ?? 0);
|
||||
this._chatServiceTelemetry.retrievedFollowups(model.sessionResource, agentPart?.agent.id ?? '', commandForTelemetry, followups?.length ?? 0);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,15 +14,24 @@ import { isImageVariableEntry } from '../attachments/chatVariableEntries.js';
|
||||
import { ChatAgentLocation, ChatModeKind, ChatPermissionLevel } from '../constants.js';
|
||||
import { ILanguageModelsService } from '../languageModels.js';
|
||||
import { chatSessionResourceToId, getChatSessionType } from '../model/chatUri.js';
|
||||
import { isAgentHostSessionResource } from '../chatSessionsService.js';
|
||||
import { isRemoteAgentHostSessionType, parseRemoteAgentHostHarness } from '../../../../../platform/agentHost/common/agentHostSessionType.js';
|
||||
|
||||
type ChatVoteEvent = {
|
||||
type ChatSessionModeEvent = {
|
||||
isAgentHostSession: boolean;
|
||||
};
|
||||
|
||||
type ChatSessionModeClassification = {
|
||||
isAgentHostSession: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether the action was performed in an Agent Host-backed chat session.' };
|
||||
};
|
||||
|
||||
type ChatVoteEvent = ChatSessionModeEvent & {
|
||||
direction: 'up' | 'down';
|
||||
agentId: string;
|
||||
command: string | undefined;
|
||||
};
|
||||
|
||||
type ChatVoteClassification = {
|
||||
type ChatVoteClassification = ChatSessionModeClassification & {
|
||||
direction: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether the user voted up or down.' };
|
||||
agentId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the chat agent that this vote is for.' };
|
||||
command: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The name of the slash command that this vote is for.' };
|
||||
@@ -30,13 +39,13 @@ type ChatVoteClassification = {
|
||||
comment: 'Provides insight into the performance of Chat agents.';
|
||||
};
|
||||
|
||||
type ChatCopyEvent = {
|
||||
type ChatCopyEvent = ChatSessionModeEvent & {
|
||||
copyKind: 'action' | 'toolbar';
|
||||
agentId: string;
|
||||
command: string | undefined;
|
||||
};
|
||||
|
||||
type ChatCopyClassification = {
|
||||
type ChatCopyClassification = ChatSessionModeClassification & {
|
||||
copyKind: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'How the copy was initiated.' };
|
||||
agentId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the chat agent that the copy acted on.' };
|
||||
command: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The name of the slash command the copy acted on.' };
|
||||
@@ -44,13 +53,13 @@ type ChatCopyClassification = {
|
||||
comment: 'Provides insight into the usage of Chat features.';
|
||||
};
|
||||
|
||||
type ChatInsertEvent = {
|
||||
type ChatInsertEvent = ChatSessionModeEvent & {
|
||||
newFile: boolean;
|
||||
agentId: string;
|
||||
command: string | undefined;
|
||||
};
|
||||
|
||||
type ChatInsertClassification = {
|
||||
type ChatInsertClassification = ChatSessionModeClassification & {
|
||||
newFile: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether the code was inserted into a new untitled file.' };
|
||||
agentId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the chat agent that this insertion is for.' };
|
||||
command: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The name of the slash command that this insertion is for.' };
|
||||
@@ -58,7 +67,7 @@ type ChatInsertClassification = {
|
||||
comment: 'Provides insight into the usage of Chat features.';
|
||||
};
|
||||
|
||||
type ChatApplyEvent = {
|
||||
type ChatApplyEvent = ChatSessionModeEvent & {
|
||||
newFile: boolean;
|
||||
agentId: string;
|
||||
command: string | undefined;
|
||||
@@ -66,7 +75,7 @@ type ChatApplyEvent = {
|
||||
editsProposed: boolean;
|
||||
};
|
||||
|
||||
type ChatApplyClassification = {
|
||||
type ChatApplyClassification = ChatSessionModeClassification & {
|
||||
newFile: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether the code was inserted into a new untitled file.' };
|
||||
agentId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the chat agent that this insertion is for.' };
|
||||
command: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The name of the slash command that this insertion is for.' };
|
||||
@@ -76,25 +85,25 @@ type ChatApplyClassification = {
|
||||
comment: 'Provides insight into the usage of Chat features.';
|
||||
};
|
||||
|
||||
type ChatFollowupEvent = {
|
||||
type ChatFollowupEvent = ChatSessionModeEvent & {
|
||||
agentId: string;
|
||||
command: string | undefined;
|
||||
};
|
||||
|
||||
type ChatFollowupClassification = {
|
||||
type ChatFollowupClassification = ChatSessionModeClassification & {
|
||||
agentId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the related chat agent.' };
|
||||
command: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The name of the related slash command.' };
|
||||
owner: 'roblourens';
|
||||
comment: 'Provides insight into the usage of Chat features.';
|
||||
};
|
||||
|
||||
type ChatTerminalEvent = {
|
||||
type ChatTerminalEvent = ChatSessionModeEvent & {
|
||||
languageId: string;
|
||||
agentId: string;
|
||||
command: string | undefined;
|
||||
};
|
||||
|
||||
type ChatTerminalClassification = {
|
||||
type ChatTerminalClassification = ChatSessionModeClassification & {
|
||||
languageId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The language of the code that was run in the terminal.' };
|
||||
agentId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the related chat agent.' };
|
||||
command: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The name of the related slash command.' };
|
||||
@@ -102,13 +111,13 @@ type ChatTerminalClassification = {
|
||||
comment: 'Provides insight into the usage of Chat features.';
|
||||
};
|
||||
|
||||
type ChatFollowupsRetrievedEvent = {
|
||||
type ChatFollowupsRetrievedEvent = ChatSessionModeEvent & {
|
||||
agentId: string;
|
||||
command: string | undefined;
|
||||
numFollowups: number;
|
||||
};
|
||||
|
||||
type ChatFollowupsRetrievedClassification = {
|
||||
type ChatFollowupsRetrievedClassification = ChatSessionModeClassification & {
|
||||
agentId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the related chat agent.' };
|
||||
command: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The name of the related slash command.' };
|
||||
numFollowups: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The number of followup prompts returned by the agent.' };
|
||||
@@ -116,7 +125,27 @@ type ChatFollowupsRetrievedClassification = {
|
||||
comment: 'Provides insight into the usage of Chat features.';
|
||||
};
|
||||
|
||||
type ChatEditHunkEvent = {
|
||||
type ChatEditSessionEvent = ChatSessionModeEvent & {
|
||||
agentId: string;
|
||||
outcome: 'accepted' | 'rejected' | 'userModified';
|
||||
hasRemainingEdits: boolean;
|
||||
requestId: string;
|
||||
modelId: string;
|
||||
modeId: string;
|
||||
};
|
||||
|
||||
type ChatEditSessionClassification = ChatSessionModeClassification & {
|
||||
agentId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the related chat agent.' };
|
||||
outcome: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The outcome of the edited file action.' };
|
||||
hasRemainingEdits: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether there are remaining edits in the file after this action.' };
|
||||
requestId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the chat request that produced the edit.' };
|
||||
modelId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The AI model used to generate the edit.' };
|
||||
modeId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The chat mode used for the request (e.g. ask, edit, agent).' };
|
||||
owner: 'roblourens';
|
||||
comment: 'Provides insight into the usage of Chat features.';
|
||||
};
|
||||
|
||||
type ChatEditHunkEvent = ChatSessionModeEvent & {
|
||||
agentId: string;
|
||||
outcome: 'accepted' | 'rejected';
|
||||
lineCount: number;
|
||||
@@ -126,7 +155,7 @@ type ChatEditHunkEvent = {
|
||||
modeId: string;
|
||||
};
|
||||
|
||||
type ChatEditHunkClassification = {
|
||||
type ChatEditHunkClassification = ChatSessionModeClassification & {
|
||||
agentId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The ID of the related chat agent.' };
|
||||
outcome: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The outcome of the edit hunk action.' };
|
||||
lineCount: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The number of lines in the relevant change.' };
|
||||
@@ -138,7 +167,7 @@ type ChatEditHunkClassification = {
|
||||
comment: 'Provides insight into the usage of Chat features.';
|
||||
};
|
||||
|
||||
export type ChatProviderInvokedEvent = {
|
||||
export type ChatProviderInvokedEvent = ChatSessionModeEvent & {
|
||||
timeToFirstProgress: number | undefined;
|
||||
totalTime: number | undefined;
|
||||
result: 'success' | 'error' | 'errorWithOutput' | 'cancelled' | 'filtered';
|
||||
@@ -161,7 +190,7 @@ export type ChatProviderInvokedEvent = {
|
||||
harness: string | undefined;
|
||||
};
|
||||
|
||||
export type ChatProviderInvokedClassification = {
|
||||
export type ChatProviderInvokedClassification = ChatSessionModeClassification & {
|
||||
timeToFirstProgress: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'The time in milliseconds from invoking the provider to getting the first data.' };
|
||||
totalTime: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'The total time it took to run the provider\'s `provideResponseWithProgress`.' };
|
||||
result: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether invoking the ChatProvider resulted in an error.' };
|
||||
@@ -192,26 +221,31 @@ export class ChatServiceTelemetry {
|
||||
) { }
|
||||
|
||||
notifyUserAction(action: IChatUserActionEvent): void {
|
||||
const isAgentHostSession = getIsAgentHostSessionForTelemetry(action.sessionResource);
|
||||
if (action.action.kind === 'vote') {
|
||||
this.telemetryService.publicLog2<ChatVoteEvent, ChatVoteClassification>('interactiveSessionVote', {
|
||||
isAgentHostSession,
|
||||
direction: action.action.direction === ChatAgentVoteDirection.Up ? 'up' : 'down',
|
||||
agentId: action.agentId ?? '',
|
||||
command: action.command,
|
||||
});
|
||||
} else if (action.action.kind === 'copy') {
|
||||
this.telemetryService.publicLog2<ChatCopyEvent, ChatCopyClassification>('interactiveSessionCopy', {
|
||||
isAgentHostSession,
|
||||
copyKind: action.action.copyKind === ChatCopyKind.Action ? 'action' : 'toolbar',
|
||||
agentId: action.agentId ?? '',
|
||||
command: action.command,
|
||||
});
|
||||
} else if (action.action.kind === 'insert') {
|
||||
this.telemetryService.publicLog2<ChatInsertEvent, ChatInsertClassification>('interactiveSessionInsert', {
|
||||
isAgentHostSession,
|
||||
newFile: !!action.action.newFile,
|
||||
agentId: action.agentId ?? '',
|
||||
command: action.command,
|
||||
});
|
||||
} else if (action.action.kind === 'apply') {
|
||||
this.telemetryService.publicLog2<ChatApplyEvent, ChatApplyClassification>('interactiveSessionApply', {
|
||||
isAgentHostSession,
|
||||
newFile: !!action.action.newFile,
|
||||
codeMapper: action.action.codeMapper,
|
||||
agentId: action.agentId ?? '',
|
||||
@@ -220,17 +254,30 @@ export class ChatServiceTelemetry {
|
||||
});
|
||||
} else if (action.action.kind === 'runInTerminal') {
|
||||
this.telemetryService.publicLog2<ChatTerminalEvent, ChatTerminalClassification>('interactiveSessionRunInTerminal', {
|
||||
isAgentHostSession,
|
||||
languageId: action.action.languageId ?? '',
|
||||
agentId: action.agentId ?? '',
|
||||
command: action.command,
|
||||
});
|
||||
} else if (action.action.kind === 'followUp') {
|
||||
this.telemetryService.publicLog2<ChatFollowupEvent, ChatFollowupClassification>('chatFollowupClicked', {
|
||||
isAgentHostSession,
|
||||
agentId: action.agentId ?? '',
|
||||
command: action.command,
|
||||
});
|
||||
} else if (action.action.kind === 'chatEditingSessionAction') {
|
||||
this.telemetryService.publicLog2<ChatEditSessionEvent, ChatEditSessionClassification>('chatEditSession', {
|
||||
isAgentHostSession,
|
||||
agentId: action.agentId ?? '',
|
||||
outcome: action.action.outcome,
|
||||
hasRemainingEdits: action.action.hasRemainingEdits,
|
||||
requestId: action.requestId,
|
||||
modelId: escapeModelIdForTelemetry(action.modelId) ?? '',
|
||||
modeId: action.modeId ?? '',
|
||||
});
|
||||
} else if (action.action.kind === 'chatEditingHunkAction') {
|
||||
this.telemetryService.publicLog2<ChatEditHunkEvent, ChatEditHunkClassification>('chatEditHunk', {
|
||||
isAgentHostSession,
|
||||
agentId: action.agentId ?? '',
|
||||
outcome: action.action.outcome,
|
||||
lineCount: action.action.lineCount,
|
||||
@@ -242,8 +289,9 @@ export class ChatServiceTelemetry {
|
||||
}
|
||||
}
|
||||
|
||||
retrievedFollowups(agentId: string, command: string | undefined, numFollowups: number): void {
|
||||
retrievedFollowups(sessionResource: URI, agentId: string, command: string | undefined, numFollowups: number): void {
|
||||
this.telemetryService.publicLog2<ChatFollowupsRetrievedEvent, ChatFollowupsRetrievedClassification>('chatFollowupsRetrieved', {
|
||||
isAgentHostSession: getIsAgentHostSessionForTelemetry(sessionResource),
|
||||
agentId,
|
||||
command,
|
||||
numFollowups,
|
||||
@@ -325,6 +373,7 @@ export class ChatRequestTelemetry {
|
||||
chatMode: this.opts.options?.modeInfo?.telemetryModeName ?? this.opts.options?.modeInfo?.telemetryModeId,
|
||||
sessionType: getChatSessionTypeForTelemetry(this.opts.sessionResource),
|
||||
harness: getHarnessForTelemetry(this.opts.sessionResource),
|
||||
isAgentHostSession: getIsAgentHostSessionForTelemetry(this.opts.sessionResource),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -378,6 +427,10 @@ function getChatSessionTypeForTelemetry(sessionResource: URI): string {
|
||||
return isRemoteAgentHostSessionType(sessionType) ? 'remote-agent-host' : sessionType;
|
||||
}
|
||||
|
||||
function getIsAgentHostSessionForTelemetry(sessionResource: URI): boolean {
|
||||
return isAgentHostSessionResource(sessionResource);
|
||||
}
|
||||
|
||||
/**
|
||||
* For remote agent host sessions, the underlying harness/provider so remote
|
||||
* activity can be split by harness (the collapsed sessionType cannot). See
|
||||
|
||||
@@ -400,6 +400,10 @@ export function isAgentHostTarget(target: string): boolean {
|
||||
return isLocalAgentHostTarget(target) || isRemoteAgentHostTarget(target);
|
||||
}
|
||||
|
||||
export function isAgentHostSessionResource(resource: URI): boolean {
|
||||
return isAgentHostTarget(resource.scheme);
|
||||
}
|
||||
|
||||
/**
|
||||
* The session type used for local agent chat sessions.
|
||||
*/
|
||||
|
||||
@@ -28,7 +28,8 @@ import { MockContextKeyService } from '../../../../../../platform/keybinding/tes
|
||||
import { ILogService, NullLogService } from '../../../../../../platform/log/common/log.js';
|
||||
import { IStorageService, StorageScope, WillSaveStateReason } from '../../../../../../platform/storage/common/storage.js';
|
||||
import { ITelemetryService } from '../../../../../../platform/telemetry/common/telemetry.js';
|
||||
import { NullTelemetryService } from '../../../../../../platform/telemetry/common/telemetryUtils.js';
|
||||
import { NullTelemetryService, NullTelemetryServiceShape } from '../../../../../../platform/telemetry/common/telemetryUtils.js';
|
||||
import { ClassifiedEvent, IGDPRProperty, OmitMetadata, StrictPropertyCheck } from '../../../../../../platform/telemetry/common/gdprTypings.js';
|
||||
import { IUserDataProfilesService, toUserDataProfile } from '../../../../../../platform/userDataProfile/common/userDataProfile.js';
|
||||
import { IWorkspaceContextService } from '../../../../../../platform/workspace/common/workspace.js';
|
||||
import { IWorkbenchAssignmentService } from '../../../../../services/assignment/common/assignmentService.js';
|
||||
@@ -45,8 +46,9 @@ import { IChatRequestVariableEntry } from '../../../common/attachments/chatVaria
|
||||
import { IChatVariablesService } from '../../../common/attachments/chatVariables.js';
|
||||
import { IChatDebugService } from '../../../common/chatDebugService.js';
|
||||
import { ChatDebugServiceImpl } from '../../../common/chatDebugServiceImpl.js';
|
||||
import { ChatRequestQueueKind, ChatSendResult, IChatFollowup, IChatModelReference, IChatProgress, IChatService, ResponseModelState } from '../../../common/chatService/chatService.js';
|
||||
import { ChatRequestQueueKind, ChatSendResult, IChatFollowup, IChatModelReference, IChatProgress, IChatService, IChatUserActionEvent, ResponseModelState } from '../../../common/chatService/chatService.js';
|
||||
import { backfillTransferredModel, backfillRestoredPickerState, ChatService } from '../../../common/chatService/chatServiceImpl.js';
|
||||
import { ChatServiceTelemetry } from '../../../common/chatService/chatServiceTelemetry.js';
|
||||
import { ChatRequestOriginKind } from '../../../common/chatRequestOrigin.js';
|
||||
import { ChatAgentLocation, ChatModeKind } from '../../../common/constants.js';
|
||||
import { ChatEditingSessionState, IChatEditingService, IChatEditingSession, IModifiedFileEntry, ModifiedFileEntryState } from '../../../common/editing/chatEditingService.js';
|
||||
@@ -62,7 +64,7 @@ import { MockChatVariablesService } from '../mockChatVariables.js';
|
||||
import { MockPromptsService } from '../promptSyntax/service/mockPromptsService.js';
|
||||
import { MockLanguageModelToolsService } from '../tools/mockLanguageModelToolsService.js';
|
||||
import { MockChatService } from './mockChatService.js';
|
||||
import { ChatSessionOptionsMap, IChatSession, IChatSessionContentProvider, IChatSessionHistoryItem, IChatSessionItem, IChatSessionServerRequest, IChatSessionsService } from '../../../common/chatSessionsService.js';
|
||||
import { ChatSessionOptionsMap, IChatSession, IChatSessionContentProvider, IChatSessionHistoryItem, IChatSessionItem, IChatSessionServerRequest, IChatSessionsService, SessionType } from '../../../common/chatSessionsService.js';
|
||||
import { MockChatSessionsService } from '../mockChatSessionsService.js';
|
||||
import { AGENT_DEBUG_LOG_FILE_LOGGING_ENABLED_SETTING, COPILOT_SKILL_URI_SCHEME, TROUBLESHOOT_SKILL_PATH } from '../../../common/promptSyntax/promptTypes.js';
|
||||
import { ChatRequestSlashPromptPart } from '../../../common/requestParser/chatParserTypes.js';
|
||||
@@ -2098,8 +2100,61 @@ suite('ChatService', () => {
|
||||
|
||||
assert.deepStrictEqual(providerInvokedEvents.map(event => ({
|
||||
sessionType: event.sessionType,
|
||||
isAgentHostSession: event.isAgentHostSession,
|
||||
hasRequestId: typeof event.requestId === 'string',
|
||||
})), [{ sessionType: 'remote-agent-host', hasRequestId: true }]);
|
||||
})), [{ sessionType: 'remote-agent-host', isAgentHostSession: true, hasRequestId: true }]);
|
||||
});
|
||||
|
||||
test('user action telemetry distinguishes agent host sessions from local sessions', () => {
|
||||
const telemetryEvents: { readonly name: string; readonly isAgentHostSession: boolean }[] = [];
|
||||
class TestTelemetryService extends NullTelemetryServiceShape {
|
||||
override publicLog2<E extends ClassifiedEvent<OmitMetadata<T>> = never, T extends IGDPRProperty = never>(name?: string, data?: StrictPropertyCheck<T, E>): void {
|
||||
const isAgentHostSession = data && typeof data === 'object' ? Reflect.get(data, 'isAgentHostSession') : undefined;
|
||||
if ((name === 'chatEditHunk' || name === 'chatEditSession') && typeof isAgentHostSession === 'boolean') {
|
||||
telemetryEvents.push({ name, isAgentHostSession });
|
||||
}
|
||||
}
|
||||
}
|
||||
const telemetry = new ChatServiceTelemetry(new TestTelemetryService());
|
||||
const sessionAction = {
|
||||
action: {
|
||||
kind: 'chatEditingSessionAction',
|
||||
uri: URI.file('/test/file.ts'),
|
||||
outcome: 'accepted',
|
||||
hasRemainingEdits: false,
|
||||
},
|
||||
agentId: 'agent',
|
||||
command: undefined,
|
||||
requestId: 'request',
|
||||
result: undefined,
|
||||
} satisfies Omit<IChatUserActionEvent, 'sessionResource'>;
|
||||
const action = {
|
||||
action: {
|
||||
kind: 'chatEditingHunkAction',
|
||||
uri: URI.file('/test/file.ts'),
|
||||
lineCount: 1,
|
||||
linesAdded: 1,
|
||||
linesRemoved: 0,
|
||||
outcome: 'accepted',
|
||||
hasRemainingEdits: false,
|
||||
},
|
||||
agentId: 'agent',
|
||||
command: undefined,
|
||||
requestId: 'request',
|
||||
result: undefined,
|
||||
} satisfies Omit<IChatUserActionEvent, 'sessionResource'>;
|
||||
|
||||
telemetry.notifyUserAction({ ...sessionAction, sessionResource: URI.from({ scheme: SessionType.AgentHostCopilot, path: '/session' }) });
|
||||
telemetry.notifyUserAction({ ...sessionAction, sessionResource: URI.from({ scheme: SessionType.Local, path: '/session' }) });
|
||||
telemetry.notifyUserAction({ ...action, sessionResource: URI.from({ scheme: SessionType.AgentHostCopilot, path: '/session' }) });
|
||||
telemetry.notifyUserAction({ ...action, sessionResource: URI.from({ scheme: SessionType.Local, path: '/session' }) });
|
||||
|
||||
assert.deepStrictEqual(telemetryEvents, [
|
||||
{ name: 'chatEditSession', isAgentHostSession: true },
|
||||
{ name: 'chatEditSession', isAgentHostSession: false },
|
||||
{ name: 'chatEditHunk', isAgentHostSession: true },
|
||||
{ name: 'chatEditHunk', isAgentHostSession: false },
|
||||
]);
|
||||
});
|
||||
|
||||
test('sendRequest with agentIdSilent passes agent host session capabilities to the request parser', async () => {
|
||||
|
||||
+3
@@ -65,6 +65,9 @@ export interface IEditTelemetryBaseData {
|
||||
|
||||
/** Source controlled id. For agent edits (sideBarChat/highlightedEdit) this is the chat request id. */
|
||||
sourceRequestId: string | undefined;
|
||||
|
||||
/** Whether the edit was generated by an Agent Host-backed chat session. */
|
||||
isAgentHostSession?: boolean;
|
||||
}
|
||||
|
||||
export interface IEditTelemetryCodeSuggestedData extends IEditTelemetryBaseData {
|
||||
|
||||
+9
@@ -45,6 +45,7 @@ export class AiEditTelemetryServiceImpl implements IAiEditTelemetryService {
|
||||
modelId: string | undefined;
|
||||
applyCodeBlockSuggestionId: string | undefined;
|
||||
sourceRequestId: string | undefined;
|
||||
isAgentHostSession: boolean | undefined;
|
||||
}, {
|
||||
owner: 'hediet';
|
||||
comment: 'Reports when code from AI is suggested to the user. @sentToGitHub';
|
||||
@@ -69,6 +70,7 @@ export class AiEditTelemetryServiceImpl implements IAiEditTelemetryService {
|
||||
modelId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The AI model used to generate the suggestion.' };
|
||||
applyCodeBlockSuggestionId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'If this suggestion is for applying a suggested code block, this is the id of the suggested code block.' };
|
||||
sourceRequestId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The chat request ID that produced the suggestion, for correlating suggestions with specific requests.' };
|
||||
isAgentHostSession: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether the edit was generated by an Agent Host-backed chat session.' };
|
||||
}>('editTelemetry.codeSuggested', {
|
||||
eventId: this._randomService.generatePrefixedUuid('evt'),
|
||||
suggestionId: suggestionId as unknown as string,
|
||||
@@ -89,6 +91,7 @@ export class AiEditTelemetryServiceImpl implements IAiEditTelemetryService {
|
||||
modelId: escapeModelIdForTelemetry(data.modelId),
|
||||
applyCodeBlockSuggestionId: data.applyCodeBlockSuggestionId as unknown as string,
|
||||
sourceRequestId: data.sourceRequestId,
|
||||
isAgentHostSession: data.isAgentHostSession,
|
||||
|
||||
...forwardToChannelIf(isCopilotLikeExtension(data.source?.extensionId)),
|
||||
});
|
||||
@@ -119,6 +122,7 @@ export class AiEditTelemetryServiceImpl implements IAiEditTelemetryService {
|
||||
modelId: string | undefined;
|
||||
applyCodeBlockSuggestionId: string | undefined;
|
||||
sourceRequestId: string | undefined;
|
||||
isAgentHostSession: boolean | undefined;
|
||||
|
||||
acceptanceMethod:
|
||||
| 'insertAtCursor'
|
||||
@@ -152,6 +156,7 @@ export class AiEditTelemetryServiceImpl implements IAiEditTelemetryService {
|
||||
|
||||
applyCodeBlockSuggestionId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'If this suggestion is for applying a suggested code block, this is the id of the suggested code block.' };
|
||||
sourceRequestId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The chat request ID that produced the edit, for correlating accepts/rejects with specific requests.' };
|
||||
isAgentHostSession: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether the edit was generated by an Agent Host-backed chat session.' };
|
||||
acceptanceMethod: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'How the user accepted the code suggestion. See #IEditTelemetryCodeAcceptedData.acceptanceMethod for possible values.' };
|
||||
}>('editTelemetry.codeAccepted', {
|
||||
eventId: this._randomService.generatePrefixedUuid('evt'),
|
||||
@@ -173,6 +178,7 @@ export class AiEditTelemetryServiceImpl implements IAiEditTelemetryService {
|
||||
modelId: escapeModelIdForTelemetry(data.modelId),
|
||||
applyCodeBlockSuggestionId: data.applyCodeBlockSuggestionId as unknown as string,
|
||||
sourceRequestId: data.sourceRequestId,
|
||||
isAgentHostSession: data.isAgentHostSession,
|
||||
acceptanceMethod: data.acceptanceMethod,
|
||||
|
||||
...forwardToChannelIf(isCopilotLikeExtension(data.source?.extensionId)),
|
||||
@@ -201,6 +207,7 @@ export class AiEditTelemetryServiceImpl implements IAiEditTelemetryService {
|
||||
modelId: string | undefined;
|
||||
applyCodeBlockSuggestionId: string | undefined;
|
||||
sourceRequestId: string | undefined;
|
||||
isAgentHostSession: boolean | undefined;
|
||||
|
||||
rejectionMethod: 'reject';
|
||||
}, {
|
||||
@@ -228,6 +235,7 @@ export class AiEditTelemetryServiceImpl implements IAiEditTelemetryService {
|
||||
|
||||
applyCodeBlockSuggestionId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'If this suggestion is for applying a suggested code block, this is the id of the suggested code block.' };
|
||||
sourceRequestId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The chat request ID that produced the edit, for correlating accepts/rejects with specific requests.' };
|
||||
isAgentHostSession: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether the edit was generated by an Agent Host-backed chat session.' };
|
||||
rejectionMethod: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'How the user rejected the code suggestion. See #IEditTelemetryCodeRejectedData.rejectionMethod for possible values.' };
|
||||
}>('editTelemetry.codeRejected', {
|
||||
eventId: this._randomService.generatePrefixedUuid('evt'),
|
||||
@@ -249,6 +257,7 @@ export class AiEditTelemetryServiceImpl implements IAiEditTelemetryService {
|
||||
modelId: escapeModelIdForTelemetry(data.modelId),
|
||||
applyCodeBlockSuggestionId: data.applyCodeBlockSuggestionId as unknown as string,
|
||||
sourceRequestId: data.sourceRequestId,
|
||||
isAgentHostSession: data.isAgentHostSession,
|
||||
rejectionMethod: data.rejectionMethod,
|
||||
|
||||
...forwardToChannelIf(isCopilotLikeExtension(data.source?.extensionId)),
|
||||
|
||||
@@ -36,6 +36,43 @@ import { ITextFileService } from '../../../../services/textfile/common/textfiles
|
||||
suite('Edit Telemetry', () => {
|
||||
ensureNoDisposablesAreLeakedInTestSuite();
|
||||
|
||||
test('reports Agent Host session mode for accepted and rejected edits', () => {
|
||||
const instantiationService = new TestInstantiationService();
|
||||
const sentTelemetry: { readonly eventName: string; readonly data: Record<string, unknown> | undefined }[] = [];
|
||||
instantiationService.stub(ITelemetryService, {
|
||||
publicLog2(eventName, data) {
|
||||
sentTelemetry.push({ eventName, data });
|
||||
},
|
||||
});
|
||||
instantiationService.stub(IRandomService, new DeterministicRandomService());
|
||||
const aiEditTelemetryService = instantiationService.createInstance(AiEditTelemetryServiceImpl);
|
||||
const baseData = {
|
||||
suggestionId: undefined,
|
||||
presentation: 'highlightedEdit' as const,
|
||||
feature: 'inlineChat' as const,
|
||||
source: undefined,
|
||||
languageId: undefined,
|
||||
editDeltaInfo: undefined,
|
||||
modeId: undefined,
|
||||
applyCodeBlockSuggestionId: undefined,
|
||||
modelId: undefined,
|
||||
sourceRequestId: undefined,
|
||||
};
|
||||
|
||||
aiEditTelemetryService.createSuggestionId({ ...baseData, isAgentHostSession: true });
|
||||
aiEditTelemetryService.handleCodeAccepted({ ...baseData, acceptanceMethod: 'accept', isAgentHostSession: true });
|
||||
aiEditTelemetryService.handleCodeRejected({ ...baseData, rejectionMethod: 'reject', isAgentHostSession: false });
|
||||
|
||||
assert.deepStrictEqual(sentTelemetry.map(event => ({
|
||||
eventName: event.eventName,
|
||||
isAgentHostSession: event.data?.isAgentHostSession,
|
||||
})), [
|
||||
{ eventName: 'editTelemetry.codeSuggested', isAgentHostSession: true },
|
||||
{ eventName: 'editTelemetry.codeAccepted', isAgentHostSession: true },
|
||||
{ eventName: 'editTelemetry.codeRejected', isAgentHostSession: false },
|
||||
]);
|
||||
});
|
||||
|
||||
test('1', async () => runWithFakedTimers({}, async () => {
|
||||
const disposables = new DisposableStore();
|
||||
const instantiationService = disposables.add(new TestInstantiationService(new ServiceCollection(
|
||||
|
||||
Reference in New Issue
Block a user