From b365ecd0b8f3f3e37946c2bea2bc98bfafbb338e Mon Sep 17 00:00:00 2001 From: Vijay Upadya <41652029+vijayupadya@users.noreply.github.com> Date: Fri, 29 May 2026 14:16:38 -0700 Subject: [PATCH] Chronicle: per-subcommand telemetry attribution for sessionStoreSql tool (#319054) chronicle: per-subcommand telemetry attribution for sessionStoreSql tool Add a 'subcommand' enum input on copilot_sessionStoreSql so each /chronicle slash command (standup, tips, cost-tips, search, improve, reindex) tags its tool calls. Plumb it through _invokeQuery/_invokeStandup/_invokeReindex and emit it on the chronicle.sqlQuery telemetry event (with 'unknown' for ad-hoc model calls). Also add command/target/success dimensions while preserving the legacy 'source' value so existing dashboards keep working. Update each chronicle-*.prompt.md to instruct the model to set the subcommand on every call. --- .../prompts/chronicle-cost-tips.prompt.md | 2 + .../prompts/chronicle-improve.prompt.md | 2 + .../prompts/chronicle-reindex.prompt.md | 2 + .../assets/prompts/chronicle-search.prompt.md | 2 + .../prompts/chronicle-standup.prompt.md | 2 + .../assets/prompts/chronicle-tips.prompt.md | 2 + extensions/copilot/package.json | 12 +++ .../tools/node/sessionStoreSqlTool.ts | 98 +++++++++++++------ 8 files changed, 90 insertions(+), 32 deletions(-) diff --git a/extensions/copilot/assets/prompts/chronicle-cost-tips.prompt.md b/extensions/copilot/assets/prompts/chronicle-cost-tips.prompt.md index 816e0fb4029..fe66163d992 100644 --- a/extensions/copilot/assets/prompts/chronicle-cost-tips.prompt.md +++ b/extensions/copilot/assets/prompts/chronicle-cost-tips.prompt.md @@ -3,3 +3,5 @@ name: chronicle:cost-tips description: Get personalized tips to reduce token usage and Copilot cost --- Analyze my recent chat session history and give me personalized, data-grounded tips to reduce token usage and Copilot cost. Use the **chronicle** skill — it documents the `copilot_sessionStoreSql` tool, the session-store schema, and the Cost Tips workflow for finding expensive sessions, token-heavy patterns, and concrete habit changes. + +When you invoke `copilot_sessionStoreSql`, set `subcommand: "cost-tips"` on every call. diff --git a/extensions/copilot/assets/prompts/chronicle-improve.prompt.md b/extensions/copilot/assets/prompts/chronicle-improve.prompt.md index 6c4e8ee1016..a54c3f48898 100644 --- a/extensions/copilot/assets/prompts/chronicle-improve.prompt.md +++ b/extensions/copilot/assets/prompts/chronicle-improve.prompt.md @@ -3,3 +3,5 @@ name: chronicle:improve description: Improve agent instructions based on friction patterns in your session history --- Analyze my recent chat session history for friction patterns and suggest improvements to my agent instructions file. Use the **chronicle** skill — it documents the `copilot_sessionStoreSql` tool, the session-store schema, and the Improve workflow for detecting repeated failures, user corrections, and recurring friction across sessions, then proposing data-grounded additions to the project's agent instructions. + +When you invoke `copilot_sessionStoreSql`, set `subcommand: "improve"` on every call. diff --git a/extensions/copilot/assets/prompts/chronicle-reindex.prompt.md b/extensions/copilot/assets/prompts/chronicle-reindex.prompt.md index fcb1eac903b..712f6bda5a4 100644 --- a/extensions/copilot/assets/prompts/chronicle-reindex.prompt.md +++ b/extensions/copilot/assets/prompts/chronicle-reindex.prompt.md @@ -3,3 +3,5 @@ name: chronicle:reindex description: Rebuild the local session index and sync to cloud --- Reindex my session store to pick up any missing sessions. Add 'force' to re-process already indexed sessions. + +When you invoke `copilot_sessionStoreSql`, set `subcommand: "reindex"`. diff --git a/extensions/copilot/assets/prompts/chronicle-search.prompt.md b/extensions/copilot/assets/prompts/chronicle-search.prompt.md index 58759c7c7b8..44dde61e6c8 100644 --- a/extensions/copilot/assets/prompts/chronicle-search.prompt.md +++ b/extensions/copilot/assets/prompts/chronicle-search.prompt.md @@ -3,3 +3,5 @@ name: chronicle:search description: Search recent chat sessions by keyword, file path, or PR/issue ref --- Search my Copilot session history for the query I provide — a keyword, a file path, or a PR/issue/commit ref — and list the matching sessions. Use the **chronicle** skill — it documents the `copilot_sessionStoreSql` tool, the session-store schema (the `sessions` table primary key is `id`; conversation content lives in `turns`, not on `sessions`; on local SQLite use the FTS5 `search_index` table and select `session_id` directly — never join `search_index.rowid` to `turns.rowid`), and the Search workflow including the cloud perf rules (aggregate-once via `WITH hits ... JOIN sessions`, default 90-day window on `turns`). + +When you invoke `copilot_sessionStoreSql`, set `subcommand: "search"` on every call. diff --git a/extensions/copilot/assets/prompts/chronicle-standup.prompt.md b/extensions/copilot/assets/prompts/chronicle-standup.prompt.md index 7b7504b63f8..c910988d6f8 100644 --- a/extensions/copilot/assets/prompts/chronicle-standup.prompt.md +++ b/extensions/copilot/assets/prompts/chronicle-standup.prompt.md @@ -3,3 +3,5 @@ name: chronicle:standup description: Generate a standup report from recent chat sessions --- Generate a standup report from my recent coding sessions. Use the **chronicle** skill — it documents the `copilot_sessionStoreSql` tool and the Standup workflow (call with `action: "standup"` to pre-fetch the last 24h of sessions, turns, files, and refs). + +When you invoke `copilot_sessionStoreSql`, set `subcommand: "standup"`. diff --git a/extensions/copilot/assets/prompts/chronicle-tips.prompt.md b/extensions/copilot/assets/prompts/chronicle-tips.prompt.md index c6704187793..d865ea582dc 100644 --- a/extensions/copilot/assets/prompts/chronicle-tips.prompt.md +++ b/extensions/copilot/assets/prompts/chronicle-tips.prompt.md @@ -3,3 +3,5 @@ name: chronicle:tips description: Get personalized tips based on your chat session usage patterns --- Analyze my recent chat session history and give me personalized tips to improve my workflow. Use the **chronicle** skill — it documents the `copilot_sessionStoreSql` tool, the session-store schema, and the Tips workflow for investigating usage patterns from `sessions`, `turns`, `session_files`, and `session_refs`. + +When you invoke `copilot_sessionStoreSql`, set `subcommand: "tips"` on every call. diff --git a/extensions/copilot/package.json b/extensions/copilot/package.json index 1267e2329fe..5dda5a69a52 100644 --- a/extensions/copilot/package.json +++ b/extensions/copilot/package.json @@ -1262,6 +1262,18 @@ "description": { "type": "string", "description": "A 2-5 word summary of what this call does (e.g. 'Recent sessions overview', 'Generate standup', 'Reindex sessions')." + }, + "subcommand": { + "type": "string", + "enum": [ + "standup", + "tips", + "cost-tips", + "search", + "improve", + "reindex" + ], + "description": "The chronicle subcommand that triggered this call (e.g. 'tips' for /chronicle tips). Used for telemetry attribution only — pass this whenever the call originates from a /chronicle slash command." } }, "required": [ diff --git a/extensions/copilot/src/extension/tools/node/sessionStoreSqlTool.ts b/extensions/copilot/src/extension/tools/node/sessionStoreSqlTool.ts index 5ee961bfc90..66c83fa826e 100644 --- a/extensions/copilot/src/extension/tools/node/sessionStoreSqlTool.ts +++ b/extensions/copilot/src/extension/tools/node/sessionStoreSqlTool.ts @@ -59,6 +59,8 @@ export interface SessionStoreSqlParams { readonly query?: string; readonly force?: boolean; readonly description: string; + /** Originating /chronicle slash command (e.g. 'tips', 'cost-tips', 'search', 'improve'). Used for telemetry attribution only. */ + readonly subcommand?: 'standup' | 'tips' | 'cost-tips' | 'search' | 'improve' | 'reindex'; } /** Cloud SQL dialect sessions query. */ @@ -99,17 +101,18 @@ class SessionStoreSqlTool implements ICopilotTool { token: CancellationToken, ): Promise { const action = options.input.action ?? 'query'; + const subcommand = options.input.subcommand; switch (action) { case 'standup': - return this._invokeStandup(token); + return this._invokeStandup(subcommand ?? 'standup', token); case 'reindex': - return this._invokeReindex(options.input.force ?? false, token); + return this._invokeReindex(options.input.force ?? false, subcommand ?? 'reindex', token); default: - return this._invokeQuery(options.input.query ?? '', token); + return this._invokeQuery(options.input.query ?? '', subcommand, token); } } - private async _invokeQuery(rawQuery: string, token: CancellationToken): Promise { + private async _invokeQuery(rawQuery: string, subcommand: SessionStoreSqlParams['subcommand'], token: CancellationToken): Promise { // Strip trailing semicolons — models often append them const sql = rawQuery.trim().replace(/;+\s*$/, ''); @@ -120,7 +123,7 @@ class SessionStoreSqlTool implements ICopilotTool { // Security check: block mutating / side-effecting statements for (const pattern of BLOCKED_PATTERNS) { if (pattern.test(sql)) { - this._sendTelemetry('blocked', 0, 0, false, 'blocked_mutating_sql'); + this._sendTelemetry({ command: 'query', subcommand, target: 'local', blocked: true, rowCount: 0, durationMs: 0, success: false, error: 'blocked_mutating_sql' }); return new LanguageModelToolResult([ new LanguageModelTextPart('Error: Blocked SQL statement. Only SELECT or WITH queries are allowed.'), ]); @@ -131,7 +134,7 @@ class SessionStoreSqlTool implements ICopilotTool { // comments first so a comment prefix cannot smuggle a non-query past the check. const firstKeywordSrc = stripLeadingCommentsAndWhitespace(sql); if (!/^(SELECT|WITH)\b/i.test(firstKeywordSrc)) { - this._sendTelemetry('blocked', 0, 0, false, 'blocked_not_select_or_with'); + this._sendTelemetry({ command: 'query', subcommand, target: 'local', blocked: true, rowCount: 0, durationMs: 0, success: false, error: 'blocked_not_select_or_with' }); return new LanguageModelToolResult([ new LanguageModelTextPart('Error: Blocked SQL statement. Only SELECT or WITH queries are allowed.'), ]); @@ -139,7 +142,7 @@ class SessionStoreSqlTool implements ICopilotTool { // Block multiple statements — only one query per call if (sql.includes(';')) { - this._sendTelemetry('blocked', 0, 0, false, 'multiple_statements'); + this._sendTelemetry({ command: 'query', subcommand, target: 'local', blocked: true, rowCount: 0, durationMs: 0, success: false, error: 'multiple_statements' }); return new LanguageModelToolResult([ new LanguageModelTextPart('Error: Only one SQL statement per call. Remove semicolons and split into separate calls.'), ]); @@ -149,6 +152,8 @@ class SessionStoreSqlTool implements ICopilotTool { const hasCloud = this._indexingPreference.hasCloudConsent(); const startTime = Date.now(); let source = hasCloud ? 'cloud' : 'local'; + let target: 'local' | 'cloud' = hasCloud ? 'cloud' : 'local'; + let fallback = false; try { let rows: Record[]; @@ -161,13 +166,15 @@ class SessionStoreSqlTool implements ICopilotTool { if (cloudResult && 'error' in cloudResult) { // Cloud query failed — surface the error so model can fix its query - this._sendTelemetry('cloud', 0, Date.now() - startTime, false, cloudResult.error.substring(0, 100)); + this._sendTelemetry({ command: 'query', subcommand, target: 'cloud', rowCount: 0, durationMs: Date.now() - startTime, success: false, error: cloudResult.error.substring(0, 100) }); return new LanguageModelToolResult([new LanguageModelTextPart( `Error from cloud: ${cloudResult.error}\n\nReminder: Cloud uses DuckDB SQL syntax. Use \`now() - INTERVAL '1 day'\` for date math, \`ILIKE\` for text search (no FTS5/MATCH).` )]); } else if (!cloudResult) { // Auth/network failure — fall back to local source = 'local_fallback'; + target = 'local'; + fallback = true; rows = this._executeLocal(sql); } else { rows = cloudResult.rows; @@ -183,14 +190,14 @@ class SessionStoreSqlTool implements ICopilotTool { truncated = true; } - this._sendTelemetry(source, rows.length, Date.now() - startTime, true); + this._sendTelemetry({ command: 'query', subcommand, target, fallback, rowCount: rows.length, durationMs: Date.now() - startTime, success: true }); // Format as table const result = formatSqlResult(rows, truncated, source); return new LanguageModelToolResult([new LanguageModelTextPart(result)]); } catch (err) { const message = err instanceof Error ? err.message : String(err); - this._sendTelemetry(source, 0, Date.now() - startTime, false, message.substring(0, 100)); + this._sendTelemetry({ command: 'query', subcommand, target, fallback, rowCount: 0, durationMs: Date.now() - startTime, success: false, error: message.substring(0, 100) }); return new LanguageModelToolResult([new LanguageModelTextPart(`Error: ${message}`)]); } } @@ -208,8 +215,10 @@ class SessionStoreSqlTool implements ICopilotTool { * Standup action: pre-fetch last 24h sessions + turns + files + refs, * merge local/cloud, dedup, and return formatted data for the model to summarise. */ - private async _invokeStandup(_token: CancellationToken): Promise { + private async _invokeStandup(subcommand: NonNullable, _token: CancellationToken): Promise { const startTime = Date.now(); + const hadCloudConsent = this._indexingPreference.hasCloudConsent(); + const target: 'local' | 'cloud' = hadCloudConsent ? 'cloud' : 'local'; try { // Always query local SQLite (has current machine's sessions) @@ -217,7 +226,7 @@ class SessionStoreSqlTool implements ICopilotTool { // Query cloud if user has cloud consent let cloudSessions: { sessions: AnnotatedSession[]; refs: AnnotatedRef[] } = { sessions: [], refs: [] }; - if (this._indexingPreference.hasCloudConsent()) { + if (hadCloudConsent) { cloudSessions = await this._queryCloudStore(); } @@ -290,11 +299,11 @@ class SessionStoreSqlTool implements ICopilotTool { } const prompt = buildStandupPrompt(capped, cappedRefs, cappedTurns, cappedFiles); - this._sendTelemetry('standup', capped.length, Date.now() - startTime, true); + this._sendTelemetry({ command: 'standup', subcommand, target, rowCount: capped.length, durationMs: Date.now() - startTime, success: true }); return new LanguageModelToolResult([new LanguageModelTextPart(prompt)]); } catch (err) { const message = err instanceof Error ? err.message : String(err); - this._sendTelemetry('standup', 0, Date.now() - startTime, false, message.substring(0, 100)); + this._sendTelemetry({ command: 'standup', subcommand, target, rowCount: 0, durationMs: Date.now() - startTime, success: false, error: message.substring(0, 100) }); return new LanguageModelToolResult([new LanguageModelTextPart(`Error fetching standup data: ${message}`)]); } } @@ -303,8 +312,10 @@ class SessionStoreSqlTool implements ICopilotTool { * Reindex action: rebuild the local session store from debug logs, * then trigger cloud sync if enabled. */ - private async _invokeReindex(force: boolean, token: CancellationToken): Promise { + private async _invokeReindex(force: boolean, subcommand: NonNullable, token: CancellationToken): Promise { const startTime = Date.now(); + const hadCloudConsent = this._indexingPreference.hasCloudConsent(); + const target: 'local' | 'cloud' = hadCloudConsent ? 'cloud' : 'local'; try { const statsBefore = this._sessionStore.getStats(); @@ -352,11 +363,11 @@ class SessionStoreSqlTool implements ICopilotTool { } } - this._sendTelemetry('reindex', result.processed, Date.now() - startTime, true); + this._sendTelemetry({ command: 'reindex', subcommand, target, rowCount: result.processed, durationMs: Date.now() - startTime, success: true }); return new LanguageModelToolResult([new LanguageModelTextPart(lines.join('\n'))]); } catch (err) { const message = err instanceof Error ? err.message : String(err); - this._sendTelemetry('reindex', 0, Date.now() - startTime, false, message.substring(0, 100)); + this._sendTelemetry({ command: 'reindex', subcommand, target, rowCount: 0, durationMs: Date.now() - startTime, success: false, error: message.substring(0, 100) }); return new LanguageModelToolResult([new LanguageModelTextPart(`Error during reindex: ${message}`)]); } } @@ -464,33 +475,56 @@ class SessionStoreSqlTool implements ICopilotTool { } } - private _sendTelemetry(source: string, rowCount: number, durationMs: number, success: boolean, error?: string): void { + private _sendTelemetry(args: { + command: 'query' | 'standup' | 'reindex'; + subcommand?: SessionStoreSqlParams['subcommand']; + target: 'local' | 'cloud'; + blocked?: boolean; + fallback?: boolean; + rowCount: number; + durationMs: number; + success: boolean; + error?: string; + }): void { + const { command, subcommand, target, blocked, fallback, rowCount, durationMs, success, error } = args; + // Back-compat: derive the original `source` value so existing dashboards keep working. + const source = blocked + ? 'blocked' + : fallback + ? 'local_fallback' + : command === 'query' + ? target + : command; + const properties = { + command, + subcommand: subcommand ?? 'unknown', + target, + source, + success: success ? 'true' : 'false', + }; + const measurements = { rowCount, durationMs }; if (success) { /* __GDPR__ "chronicle.sqlQuery" : { "owner": "vijayu", -"comment": "Tracks session store SQL query execution and failures", -"source": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Query target: local, cloud, or blocked." }, +"comment": "Tracks chronicle session-store tool invocations (query/standup/reindex) and outcomes", +"command": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Tool action invoked: query, standup, or reindex." }, +"subcommand": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Originating /chronicle slash command (standup, tips, cost-tips, search, improve, reindex) or 'unknown' for ad-hoc model calls." }, +"target": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Whether the invocation primarily targeted the local SQLite store or the cloud session store." }, +"source": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Fine-grained source: local, cloud, local_fallback, blocked, standup, or reindex (kept for back-compat)." }, +"success": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Whether the invocation succeeded (true/false)." }, "error": { "classification": "CallstackOrException", "purpose": "PerformanceAndHealth", "comment": "Truncated error message." }, "rowCount": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "isMeasurement": true, "comment": "Number of rows returned." }, -"durationMs": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "isMeasurement": true, "comment": "Query duration in milliseconds." } +"durationMs": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "isMeasurement": true, "comment": "Invocation duration in milliseconds." } } */ - this._telemetryService.sendMSFTTelemetryEvent('chronicle.sqlQuery', { - source, - }, { - rowCount, - durationMs, - }); + this._telemetryService.sendMSFTTelemetryEvent('chronicle.sqlQuery', properties, measurements); } else { this._telemetryService.sendMSFTTelemetryErrorEvent('chronicle.sqlQuery', { - source, + ...properties, error: error ?? 'unknown', - }, { - rowCount, - durationMs, - }); + }, measurements); } }