From 197fc9911c625314f7162bbf0dd2a2da54a86e90 Mon Sep 17 00:00:00 2001 From: Josh Spicer <23246594+joshspicer@users.noreply.github.com> Date: Thu, 12 Mar 2026 11:10:58 -0700 Subject: [PATCH] Add telemetry to Chat Customizations editor (#301173) * Add telemetry to Chat Customizations editor Instrument 7 key user interactions in the AI Customization Management Editor with GDPR-compliant publicLog2 telemetry events: - chatCustomizationEditor.opened: tracks editor opens with initial section - chatCustomizationEditor.sectionChanged: tracks sidebar navigation - chatCustomizationEditor.itemSelected: tracks item selection with type/storage - chatCustomizationEditor.createItem: tracks AI-guided and manual creation - chatCustomizationEditor.saveItem: tracks save actions (builtin override + existing) - chatCustomizationEditor.deleteItem: tracks confirmed deletions - chatCustomizationEditor.search: tracks search usage with result counts Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * fix: improve search telemetry logging in Chat Customizations editor --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../aiCustomizationListWidget.ts | 31 ++++- .../aiCustomizationManagement.contribution.ts | 23 ++++ .../aiCustomizationManagementEditor.ts | 107 ++++++++++++++++++ 3 files changed, 159 insertions(+), 2 deletions(-) diff --git a/src/vs/workbench/contrib/chat/browser/aiCustomization/aiCustomizationListWidget.ts b/src/vs/workbench/contrib/chat/browser/aiCustomization/aiCustomizationListWidget.ts index f7b67fbc41f..42b5b2cf824 100644 --- a/src/vs/workbench/contrib/chat/browser/aiCustomization/aiCustomizationListWidget.ts +++ b/src/vs/workbench/contrib/chat/browser/aiCustomization/aiCustomizationListWidget.ts @@ -48,11 +48,28 @@ import { HookType, HOOK_METADATA } from '../../common/promptSyntax/hookTypes.js' import { parse as parseJSONC } from '../../../../../base/common/json.js'; import { Schemas } from '../../../../../base/common/network.js'; import { OS } from '../../../../../base/common/platform.js'; +import { ITelemetryService } from '../../../../../platform/telemetry/common/telemetry.js'; export { truncateToFirstSentence } from './aiCustomizationListWidgetUtils.js'; const $ = DOM.$; +//#region Telemetry + +type CustomizationEditorSearchEvent = { + section: string; + resultCount: number; +}; + +type CustomizationEditorSearchClassification = { + section: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The active section when the search was performed.' }; + resultCount: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'The number of items matching the search query.' }; + owner: 'joshspicer'; + comment: 'Tracks search usage in the Chat Customizations editor.'; +}; + +//#endregion + const ITEM_HEIGHT = 44; const GROUP_HEADER_HEIGHT = 36; const GROUP_HEADER_HEIGHT_WITH_SEPARATOR = 40; @@ -403,6 +420,7 @@ export class AICustomizationListWidget extends Disposable { @IHoverService private readonly hoverService: IHoverService, @IFileService private readonly fileService: IFileService, @IPathService private readonly pathService: IPathService, + @ITelemetryService private readonly telemetryService: ITelemetryService, ) { super(); this.element = $('.ai-customization-list-widget'); @@ -430,7 +448,15 @@ export class AICustomizationListWidget extends Disposable { this._register(this.searchInput.onDidChange(() => { this.searchQuery = this.searchInput.value; - this.delayedFilter.trigger(() => this.filterItems()); + this.delayedFilter.trigger(() => { + const matchCount = this.filterItems(); + if (this.searchQuery.trim()) { + this.telemetryService.publicLog2('chatCustomizationEditor.search', { + section: this.currentSection, + resultCount: matchCount, + }); + } + }); })); // Add button container next to search @@ -1011,7 +1037,7 @@ export class AICustomizationListWidget extends Disposable { /** * Filters items based on the current search query and builds grouped display entries. */ - private filterItems(): void { + private filterItems(): number { let matchedItems: IAICustomizationListItem[]; if (!this.searchQuery.trim()) { @@ -1095,6 +1121,7 @@ export class AICustomizationListWidget extends Disposable { this.list.splice(0, this.list.length, this.displayEntries); this.updateEmptyState(); + return matchedItems.length; } /** diff --git a/src/vs/workbench/contrib/chat/browser/aiCustomization/aiCustomizationManagement.contribution.ts b/src/vs/workbench/contrib/chat/browser/aiCustomization/aiCustomizationManagement.contribution.ts index 7011d2094f6..f5a73f856af 100644 --- a/src/vs/workbench/contrib/chat/browser/aiCustomization/aiCustomizationManagement.contribution.ts +++ b/src/vs/workbench/contrib/chat/browser/aiCustomization/aiCustomizationManagement.contribution.ts @@ -38,6 +38,23 @@ import { IDialogService } from '../../../../../platform/dialogs/common/dialogs.j import { basename, dirname } from '../../../../../base/common/resources.js'; import { Schemas } from '../../../../../base/common/network.js'; import { isWindows, isMacintosh } from '../../../../../base/common/platform.js'; +import { ITelemetryService } from '../../../../../platform/telemetry/common/telemetry.js'; + +//#region Telemetry + +type CustomizationEditorDeleteItemEvent = { + promptType: string; + storage: string; +}; + +type CustomizationEditorDeleteItemClassification = { + promptType: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The type of customization being deleted.' }; + storage: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The storage location of the deleted item.' }; + owner: 'joshspicer'; + comment: 'Tracks item deletion in the Chat Customizations editor.'; +}; + +//#endregion //#region Editor Registration @@ -230,6 +247,12 @@ registerAction2(class extends Action2 { }); if (confirmation.confirmed) { + const telemetryService = accessor.get(ITelemetryService); + telemetryService.publicLog2('chatCustomizationEditor.deleteItem', { + promptType: promptType ?? '', + storage: storage ?? '', + }); + // For skills, delete the parent folder (e.g. .github/skills/my-skill/) // since each skill is a folder containing SKILL.md. const deleteTarget = isSkill ? dirname(uri) : uri; diff --git a/src/vs/workbench/contrib/chat/browser/aiCustomization/aiCustomizationManagementEditor.ts b/src/vs/workbench/contrib/chat/browser/aiCustomization/aiCustomizationManagementEditor.ts index 6de3558b5c3..0c6212e0c5c 100644 --- a/src/vs/workbench/contrib/chat/browser/aiCustomization/aiCustomizationManagementEditor.ts +++ b/src/vs/workbench/contrib/chat/browser/aiCustomization/aiCustomizationManagementEditor.ts @@ -81,6 +81,74 @@ import { IAgentPluginItem } from '../agentPluginEditor/agentPluginItems.js'; const $ = DOM.$; +//#region Telemetry + +type CustomizationEditorOpenedEvent = { + section: string; +}; + +type CustomizationEditorOpenedClassification = { + section: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The initially selected section when the editor opens.' }; + owner: 'joshspicer'; + comment: 'Tracks when the Chat Customizations editor is opened.'; +}; + +type CustomizationEditorSectionChangedEvent = { + section: string; +}; + +type CustomizationEditorSectionChangedClassification = { + section: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The section the user navigated to.' }; + owner: 'joshspicer'; + comment: 'Tracks section navigation within the Chat Customizations editor.'; +}; + +type CustomizationEditorItemSelectedEvent = { + section: string; + promptType: string; + storage: string; +}; + +type CustomizationEditorItemSelectedClassification = { + section: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The active section when the item was selected.' }; + promptType: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The prompt type of the selected item.' }; + storage: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The storage location of the selected item (local, user, extension, plugin, builtin).' }; + owner: 'joshspicer'; + comment: 'Tracks item selection in the Chat Customizations editor.'; +}; + +type CustomizationEditorCreateItemEvent = { + section: string; + promptType: string; + creationMode: 'ai' | 'manual'; + target: string; +}; + +type CustomizationEditorCreateItemClassification = { + section: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The active section when the item was created.' }; + promptType: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The type of customization being created.' }; + creationMode: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether the item was created via AI-guided flow or manual creation.' }; + target: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The target storage for the new item (workspace, user).' }; + owner: 'joshspicer'; + comment: 'Tracks customization creation in the Chat Customizations editor.'; +}; + +type CustomizationEditorSaveItemEvent = { + promptType: string; + storage: string; + saveTarget: string; +}; + +type CustomizationEditorSaveItemClassification = { + promptType: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The type of customization being saved.' }; + storage: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The original storage location of the item.' }; + saveTarget: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The target storage for the save (workspace, user, existing).' }; + owner: 'joshspicer'; + comment: 'Tracks save actions in the Chat Customizations editor.'; +}; + +//#endregion + export const aiCustomizationManagementSashBorder = registerColor( 'aiCustomizationManagement.sashBorder', PANEL_BORDER, @@ -485,6 +553,11 @@ export class AICustomizationManagementEditor extends EditorPane { // Handle item selection this.editorDisposables.add(this.listWidget.onDidSelectItem(item => { + this.telemetryService.publicLog2('chatCustomizationEditor.itemSelected', { + section: this.selectedSection, + promptType: item.promptType, + storage: item.storage, + }); const isWorkspaceFile = item.storage === PromptsStorage.local; const isReadOnly = item.storage === PromptsStorage.extension || item.storage === PromptsStorage.plugin || item.storage === BUILTIN_STORAGE; this.showEmbeddedEditor(item.uri, item.name, item.promptType, item.storage, isWorkspaceFile, isReadOnly); @@ -576,6 +649,10 @@ export class AICustomizationManagementEditor extends EditorPane { return; } + this.telemetryService.publicLog2('chatCustomizationEditor.sectionChanged', { + section, + }); + if (this.viewMode === 'editor') { this.goBackToList(); } @@ -667,6 +744,12 @@ export class AICustomizationManagementEditor extends EditorPane { * Creates a new customization using the AI-guided flow. */ private async createNewItemWithAI(type: PromptsType): Promise { + this.telemetryService.publicLog2('chatCustomizationEditor.createItem', { + section: this.selectedSection, + promptType: type, + creationMode: 'ai', + target: 'workspace', + }); if (this.input) { this.group.closeEditor(this.input); } @@ -677,6 +760,12 @@ export class AICustomizationManagementEditor extends EditorPane { * Creates a new prompt file and opens it in the embedded editor. */ private async createNewItemManual(type: PromptsType, target: 'workspace' | 'user'): Promise { + this.telemetryService.publicLog2('chatCustomizationEditor.createItem', { + section: this.selectedSection, + promptType: type, + creationMode: 'manual', + target, + }); if (type === PromptsType.hook) { if (this.workspaceService.isSessionsWindow) { @@ -741,6 +830,10 @@ export class AICustomizationManagementEditor extends EditorPane { this.inEditorContextKey.set(true); this.sectionContextKey.set(this.selectedSection); + this.telemetryService.publicLog2('chatCustomizationEditor.opened', { + section: this.selectedSection, + }); + await super.setInput(input, options, context, token); if (this.dimension) { @@ -966,6 +1059,13 @@ export class AICustomizationManagementEditor extends EditorPane { private goBackToList(): void { const fileUri = this.currentEditingUri; const backgroundSaveRequest = this.createExistingCustomizationSaveRequest(); + if (backgroundSaveRequest) { + this.telemetryService.publicLog2('chatCustomizationEditor.saveItem', { + promptType: this.currentEditingPromptType ?? '', + storage: String(this.currentEditingStorage ?? ''), + saveTarget: 'existing', + }); + } if (fileUri && this.currentEditingStorage === BUILTIN_STORAGE) { this.disposeBuiltinEditingSession(fileUri); } @@ -1134,6 +1234,13 @@ export class AICustomizationManagementEditor extends EditorPane { } backgroundSaveRequest = this.createBuiltinPromptSaveRequest(selection); + if (backgroundSaveRequest) { + this.telemetryService.publicLog2('chatCustomizationEditor.saveItem', { + promptType: this.currentEditingPromptType ?? '', + storage: String(this.currentEditingStorage ?? ''), + saveTarget: selection.target, + }); + } } this.goBackToList();