From ddb6f98ce6b3bb7d4aa22a6bbc3011c4b4f4e75b Mon Sep 17 00:00:00 2001 From: Zhichao Li <57812115+zhichli@users.noreply.github.com> Date: Mon, 2 Mar 2026 12:46:30 -0800 Subject: [PATCH] feat(otel): Add OpenTelemetry GenAI instrumentation to Copilot Chat (#3917) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add OTel GenAI instrumentation foundation Phase 0 complete: - spec.md: Full spec with decisions, GenAI semconv, dual-write, eval signals, lessons from Gemini CLI + Claude Code - plan.md: E2E demo plan (chat ext + eval repo + Azure backend) - src/platform/otel/: IOTelService, config, attributes, metrics, events, message formatters, NodeOTelService, file exporters - package.json: Added @opentelemetry/* dependencies OTel opt-in behind OTEL_EXPORTER_OTLP_ENDPOINT env var. * refactor: reorder OTel type imports for consistency * refactor: reorder OTel type imports for consistency * feat(otel): wire OTel spans into chat extension — Phase 1 core - Register IOTelService in DI (NodeOTelService when enabled, NoopOTelService when disabled) - Add OTelContrib lifecycle contribution for OTel init/shutdown - Add `chat {model}` inference span in ChatMLFetcherImpl._doFetchAndStreamChat() - Add `execute_tool {name}` span in ToolsService.invokeTool() - Add `invoke_agent {participant}` parent span in ToolCallingLoop.run() - Record gen_ai.client.operation.duration, tool call count/duration, agent metrics - Thread IOTelService through all ToolCallingLoop subclasses - Update test files with NoopOTelService - Zero overhead when OTel is disabled (noop providers, no dynamic imports) * feat(otel): add embeddings span, config UI settings, and unit tests - Add `embeddings {model}` span in RemoteEmbeddingsComputer.computeEmbeddings() - Add VS Code settings under github.copilot.chat.otel.* in package.json (enabled, exporterType, otlpEndpoint, captureContent, outfile) - Wire VS Code settings into resolveOTelConfig in services.ts - Add unit tests for: - resolveOTelConfig: env precedence, kill switch, all config paths (16 tests) - NoopOTelService: zero-overhead noop behavior (8 tests) - GenAiMetrics: metric recording with correct attributes (7 tests) * test(otel): add unit tests for messageFormatters, genAiEvents, fileExporters - messageFormatters: 18 tests covering toInputMessages, toOutputMessages, toSystemInstructions, toToolDefinitions (edge cases, empty inputs, invalid JSON) - genAiEvents: 9 tests covering all 4 event emitters, content capture on/off - fileExporters: 5 tests covering write/read round-trip for span, log, metric exporters plus aggregation temporality Total OTel test suite: 63 tests across 6 files * feat(otel): record token usage and time-to-first-token metrics Add gen_ai.client.token.usage (input/output) and copilot_chat.time_to_first_token histogram metrics at the fetchMany success path where token counts and TTFT are available from the processSuccessfulResponse result. * docs: finalize sprint plan with completion status * style: apply formatter changes to OTel files * feat(otel): emit gen_ai.client.inference.operation.details event with token usage Wire emitInferenceDetailsEvent into fetchMany success path where full token usage (prompt_tokens, completion_tokens), resolved model, request ID, and finish reasons are available from processSuccessfulResponse. This follows the OTel GenAI spec pattern: - Spans: timing + hierarchy + error tracking - Events: full request/response details including token counts The data mirrors what RequestLogger captures for chat-export-logs.json. * feat(otel): add aggregated token usage to invoke_agent span Per the OTel GenAI agent spans spec, add gen_ai.usage.input_tokens and gen_ai.usage.output_tokens as Recommended attributes on the invoke_agent span. Tokens are accumulated across all LLM turns by listening to onDidReceiveResponse events during the agent loop, then set on the span before it ends. Ref: https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-agent-spans/ * feat(otel): add token usage attributes to chat inference span Defer the `chat {model}` span completion from _doFetchAndStreamChat to fetchMany where processSuccessfulResponse has extracted token counts. The chat span now carries: - gen_ai.usage.input_tokens (prompt_tokens) - gen_ai.usage.output_tokens (completion_tokens) - gen_ai.response.model (resolved model) The span handle is returned from _doFetchAndStreamChat via the result object so fetchMany can set attributes and end it after tokens are known. This matches the chat-export-logs.json pattern where each request entry carries full usage data alongside the response. * style: apply formatter changes * fix: correct import paths in otelContrib and add IOTelService to test * feat: add diagnostic span exporter to log first successful export and failures * feat: add content capture to OTel spans (messages, responses, tool args/results) - Chat spans: add copilot.debug_name attribute for identifying orphan spans - Chat spans: capture gen_ai.input.messages and gen_ai.output.messages when captureContent enabled - Tool spans: capture gen_ai.tool.call.arguments and gen_ai.tool.call.result when captureContent enabled - Extension chat endpoint: capture input/output messages when captureContent enabled - Add CopilotAttr.DEBUG_NAME constant * fix: register IOTelService in chatLib setupServices for NES test * fix: register OTel ConfigKey settings in Advanced namespace for configurations test * fix: register IOTelService in shared test services (createExtensionUnitTestingServices) * fix: register IOTelService in platform test services * feat(otel): enhance GenAI span attributes per OTel semantic conventions - Change gen_ai.provider.name from 'openai' to 'github' for CAPI models - Rename CopilotAttr to CopilotChatAttr, prefix values with copilot_chat.* - Add GITHUB to GenAiProviderName enum - Replace copilot.debug_name with gen_ai.agent.name on chat spans - Add gen_ai.request.temperature, gen_ai.request.top_p to chat spans - Add gen_ai.response.id, gen_ai.response.finish_reasons on success - Add gen_ai.usage.cache_read.input_tokens from cached_tokens - Add copilot_chat.request.max_prompt_tokens and copilot_chat.time_to_first_token - Add gen_ai.tool.description to execute_tool spans - Fix gen_ai.tool.call.id to read chatStreamToolCallId (was reading nonexistent prop) - Fix tool result capture to handle PromptTsxPart and DataPart (not just TextPart) - Add gen_ai.input.messages and gen_ai.output.messages to invoke_agent span (opt-in) - Move gen_ai.tool.definitions from chat spans to invoke_agent span (opt-in) - Add gen_ai.system_instructions to chat spans (opt-in) - Fix error.type raw strings to use StdAttr.ERROR_TYPE constant - Centralize hardcoded copilot.turn_count and copilot.endpoint_type into CopilotChatAttr - Add COPILOT_OTEL_CAPTURE_CONTENT=true to launch.json for testing - Document span hierarchy fixes needed in plan.md * feat(otel): connect subagent spans to parent trace via context propagation - Add TraceContext type and getActiveTraceContext() to IOTelService - Add storeTraceContext/getStoredTraceContext for cross-boundary propagation - Add parentTraceContext option to SpanOptions for explicit parent linking - Implement in NodeOTelService using OTel remote span context - Capture trace context when execute_tool runSubagent fires (keyed by toolCallId) - Restore parent context in subagent invoke_agent span (via subAgentInvocationId) - Auto-cleanup stored contexts after 5 minutes to prevent memory leaks - Update test mocks with new IOTelService methods - Update plan.md with investigation findings * fix(otel): fix subagent trace context key to use parentRequestId The previous implementation stored trace context keyed by chatStreamToolCallId (model-assigned tool call ID), but looked it up by subAgentInvocationId (VS Code internal invocation.callId UUID). These are different IDs that don't match across the IPC boundary. Fix: key by chatRequestId on store side (available on invocation options), and look up by parentRequestId on subagent side (same value, available on ChatRequest). Both reference the parent agent's request ID. Verified: 21-span trace with subagent correctly nested under parent agent. * fix(otel): add model attrs to invoke_agent and max_prompt_tokens to BYOK chat - Set gen_ai.request.model on invoke_agent span from endpoint - Track gen_ai.response.model from last LLM response resolvedModel - Add copilot_chat.request.max_prompt_tokens to BYOK chat spans - Document upstream gaps in plan.md (BYOK token usage, programmatic tool IDs) * test(otel): add trace context propagation tests for subagent linkage Tests verify: - storeTraceContext/getStoredTraceContext round-trip and single-use semantics - getActiveTraceContext returns context inside startActiveSpan - parentTraceContext makes child span inherit traceId from parent - Independent spans get different traceIds without parentTraceContext - Full subagent flow: store context in tool call, retrieve in subagent * fix(otel): add finish_reasons and ttft to BYOK chat spans, document orphan spans - Set gen_ai.response.finish_reasons on BYOK chat success - Set copilot_chat.time_to_first_token on BYOK chat success - Document Gap 4: duplicate orphan spans from CopilotLanguageModelWrapper - Identify all orphan span categories (title, progressMessages, promptCategorization, wrapper) * docs(otel): update Gap 4 analysis — wrapper spans have actual token usage data The copilotLanguageModelWrapper orphan spans are the actual CAPI HTTP handlers, not duplicates. They contain real token usage, cache read tokens, resolved model names, and temperature — all missing from the consumer-side extChatEndpoint spans due to VS Code LM API limitations. Updated plan.md with: - Side-by-side attribute comparison table - Three fix approaches (context propagation, span suppression, enrichment) - Recommendation: Option 1 (propagate trace context through IPC) * feat(otel): propagate trace context through BYOK IPC to link wrapper spans - Pass _otelTraceContext through modelOptions alongside _capturingTokenCorrelationId - Inject IOTelService into CopilotLanguageModelWrapper - Wrap makeRequest in startActiveSpan with parentTraceContext when available - This creates a byok-provider bridge span that makes chatMLFetcher's chat span a child of the original invoke_agent trace, bringing real token usage data into the agent trace hierarchy * debug(otel): add debug attribute to verify trace context capture in BYOK path * fix(otel): remove debug attribute, BYOK trace context propagation verified working Verified: 63-span trace with Azure BYOK (gpt-5) correctly shows: - byok-provider bridge spans linking wrapper chat spans into agent trace - Real token usage (in:21458 out:1730 cache:19072) visible on wrapper chat spans - hasCtx:true on all extChatEndpoint spans confirming context capture - Two subagent invoke_agent spans correctly nested under main agent - Zero orphan copilotLanguageModelWrapper spans * refactor(otel): replace byok-provider bridge span with invisible context propagation Add runWithTraceContext() to IOTelService — sets parent trace context without creating a visible span. The wrapper's chat spans now appear directly as children of invoke_agent, eliminating the noisy byok-provider intermediary span. Before: invoke_agent → byok-provider → chat (wrapper) After: invoke_agent → chat (wrapper) * refactor(otel): remove duplicate BYOK consumer-side chat span The extChatEndpoint no longer creates its own chat span. The wrapper's chatMLFetcher span (via CopilotLanguageModelWrapper) is the single source of truth with full token usage, cache data, and resolved model. Before: invoke_agent → chat (empty, extChatEndpoint) + chat (rich, wrapper) After: invoke_agent → chat (rich, wrapper only) * fix(otel): restore chat span for non-wrapper BYOK providers (Anthropic, Gemini) The previous commit removed the extChatEndpoint chat span, which was correct for Azure/OpenAI BYOK (served by CopilotLanguageModelWrapper via chatMLFetcher). But Anthropic and Gemini BYOK providers call their native SDKs directly, bypassing CopilotLanguageModelWrapper — so they need the consumer-side span. Now: always create a chat span in extChatEndpoint with basic metadata (model, provider, response.id, finish_reasons). For wrapper-based providers, the chatMLFetcher also creates a richer sibling span with token usage. * fix(otel): skip consumer chat span for wrapper-based BYOK providers Only create the extChatEndpoint chat span for non-wrapper providers (Anthropic, Gemini) that need it as their only span. Wrapper-based providers (Azure, OpenAI, OpenRouter, Ollama, xAI) get a single rich span from chatMLFetcher via CopilotLanguageModelWrapper. Result: 1 chat span per LLM call for all provider types. * fix: remove unnecessary 'google' from non-wrapper vendor set * feat(otel): add rich chat span with usage data for Anthropic BYOK provider Move chat span creation into AnthropicLMProvider where actual API response data (token usage, cache reads) is available. The span is linked to the agent trace via runWithTraceContext and enriched with: - gen_ai.usage.input_tokens / output_tokens - gen_ai.usage.cache_read.input_tokens - gen_ai.response.model / response.id / finish_reasons Remove consumer-side extChatEndpoint span for all vendors (nonWrapperVendors now empty) since both wrapper-based and Anthropic providers create their own spans with full data. Next: apply same pattern to Gemini provider. * feat(otel): add rich chat span for Gemini BYOK, clean up extChatEndpoint - Add OTel chat span with full usage data to GeminiNativeBYOKLMProvider - Remove all consumer-side span code from extChatEndpoint (dead code) - Each provider now owns its chat span with real API response data: * CAPI: chatMLFetcher * OpenAI-compat BYOK: CopilotLanguageModelWrapper → chatMLFetcher * Anthropic: AnthropicLMProvider * Gemini: GeminiNativeBYOKLMProvider - Fix Gemini test to pass IOTelService * feat(otel): enrich Anthropic/Gemini chat spans with full metadata Add to both providers: - copilot_chat.request.max_prompt_tokens (model.maxInputTokens) - server.address (api.anthropic.com / generativelanguage.googleapis.com) - gen_ai.conversation.id (requestId) - copilot_chat.time_to_first_token (result.ttft) Now matches CAPI chat span attribute parity. * feat(otel): add server.address to CAPI/Azure BYOK chat spans Extract hostname from urlOrRequestMetadata when it's a URL string and set as server.address on the chat span. Works for both CAPI and CopilotLanguageModelWrapper (Azure BYOK) paths. * feat(otel): add max_tokens and output_messages to Anthropic/Gemini chat spans - gen_ai.request.max_tokens from model.maxOutputTokens - gen_ai.output.messages (opt-in) from response text - Closes remaining attribute gaps vs CAPI/Azure BYOK spans * fix(otel): capture tool calls in output_messages for chat spans When model responds with tool calls instead of text, the output_messages attribute was empty. Now captures both text parts and tool call parts in the output_messages, matching the OTel GenAI output messages schema. Also: Azure BYOK invoke_agent zero tokens is a known upstream gap — extChatEndpoint returns hardcoded usage:0 since VS Code LM API doesn't expose actual usage from the provider side. * fix(otel): capture tool calls in output_messages for Anthropic/Gemini BYOK spans Same fix as CAPI — when model responds with tool calls, include them in gen_ai.output.messages alongside text parts. All three provider paths (CAPI, Anthropic, Gemini) now consistently capture both text and tool call parts in output messages. * fix(otel): add input_messages and agent_name to Anthropic/Gemini chat spans - gen_ai.input.messages (opt-in) captured from provider messages parameter - gen_ai.agent.name set to AnthropicBYOK / GeminiBYOK for identification Closes the last attribute gaps vs CAPI/Azure BYOK chat spans. * fix(otel): fix input_messages serialization for Anthropic/Gemini BYOK - Map enum role values to names (1→user, 2→assistant, 3→system) - Extract text from LanguageModelTextPart content arrays instead of showing '[complex]' for all messages - Use OTel GenAI input messages schema with role + parts format * docs(otel): add remaining metrics/events work to plan.md Coverage matrix showing: - Anthropic/Gemini BYOK missing: operation.duration, token.usage, time_to_first_token metrics, and inference.details event - CAPI and Azure BYOK (via wrapper) fully covered - Tool/agent/session metrics covered across all providers - 4 tasks (M1-M4) to close the gap * feat(otel): add metrics and inference events to Anthropic/Gemini BYOK providers Both providers now record: - gen_ai.client.operation.duration histogram - gen_ai.client.token.usage histograms (input + output) - copilot_chat.time_to_first_token histogram - gen_ai.client.inference.operation.details log event All metrics/events now have full parity across CAPI, Azure BYOK, Anthropic BYOK, and Gemini BYOK. * fix(otel): fix LoggerProvider constructor — use 'processors' key (SDK v2) The OTel SDK v2 changed the LoggerProvider constructor option from 'logRecordProcessors' to 'processors'. The old key was silently ignored, causing all log records to be dropped. This is why logs never appeared in Loki despite traces working fine. * docs: add agent monitoring guide with OTel usage and Claude/Gemini comparison * docs: remove Claude/Gemini comparison from monitoring guide * docs: add OTel comparison with Claude Code and Gemini CLI * docs: reorganize monitoring docs — user guide + dev architecture - agent_monitoring.md: polished user-facing guide (for VS Code website) - agent_monitoring_arch.md: developer-facing architecture & instrumentation guide - Removed internal plan/spec/comparison files from repo (moved to ~/Documents) * fix(otel): restore _doFetchViaHttp body and _fetchWithInstrumentation after rebase * fix(otel): propagate otelSpan through WebSocket/HTTP routing paths The otelSpan was created in _doFetchAndStreamChat but not included in returns from _doFetchViaWebSocket and _doFetchViaHttp, causing the caller (fetchMany) to always receive undefined for otelSpan. Fix: await both routing paths and spread otelSpan into the result. * docs(otel): improve monitoring docs, add collector setup, fix trace context - Expand agent_monitoring.md with detailed span/metric/event attribute tables - Add BYOK provider coverage, subagent trace propagation docs - Add Backend Considerations: Azure App Insights (via collector), Langfuse, Grafana - Add End-to-End Setup & Verification section with KQL examples - Add OTel Collector config + docker-compose for Azure App Insights - Fix: emit inference details event before span.end() in chatMLFetcher (fixes 'No trace ID' log records in App Insights) - Fix: pass active context in emitLogRecord for trace correlation - Update launch.json to point at OTel Collector (localhost:4328) * docs(otel): merge Backend Considerations and E2E sections to remove redundancy * docs(otel): remove internal dev debug reference from user-facing guide * docs(otel): remove Grafana section and Jaeger refs from App Insights section * docs(otel): trim Backend section to factual setup guides, remove claims * docs(otel): final accuracy audit — fix false claims against code - Mark copilot_chat.session.start event as 'not yet emitted' (defined but no call site) - Mark copilot_chat.agent.turn event as 'not yet emitted' (defined but no call site) - Mark copilot_chat.session.count metric as 'not yet wired up' - Fix OTEL_EXPORTER_OTLP_PROTOCOL desc: only 'grpc' changes behavior - Fix telemetry kill switch claim: vscodeTelemetryLevel not wired in services.ts - Remove false toolCalling.tsx instrumentation point from arch doc - Fix docker-compose comments: wrong port numbers (16686→16687, 4318→4328) - Add reference to full collector config file from inline snippet * docs(otel): remove telemetry.telemetryLevel references — OTel is independent * feat(otel): wire up session.start event, agent.turn event, and session.count metric - emitSessionStartEvent + incrementSessionCount at invoke_agent start (top-level only) - emitAgentTurnEvent per LLM response in onDidReceiveResponse listener - Remove 'not yet wired' markers from docs * chore: untrack .playwright-mcp/ and add to .gitignore * chore: remove otel spec reference files * chore(otel): remove OpenTelemetry environment variables from launch configurations * fix(otel): add 64KB truncation limit for content capture attributes Prevents OTLP batch export failures when large prompts/responses are captured. Aligned with gemini-cli's limitTotalLength pattern. Applied truncateForOTel() to all JSON.stringify calls feeding span attributes across chatMLFetcher, toolCallingLoop, toolsService, anthropicProvider, geminiNativeProvider, and genAiEvents. * refactor(otel): make GenAiMetrics methods static to avoid per-call allocations Aligned with gemini-cli pattern of module-level metric functions. Eliminates 17+ throwaway GenAiMetrics instances per agent run. * fix(otel): fix timer leak, cap buffered ops, rate-limit export logs - storeTraceContext: track timers for clearTimeout on retrieval/shutdown, add 100-entry max with LRU eviction - BufferedSpanHandle: cap _ops at 200 to prevent unbounded growth - DiagnosticSpanExporter: rate-limit failure logs to once per 60s * docs(otel): fix Jaeger UI port to match docker-compose (16687) * chore(otel): update sprint plan — mark P0/P1 tasks done * fix(otel): remove as any casts in BYOK provider content capture Use proper Array.isArray + instanceof checks instead of as any[] casts for LanguageModelChatMessage.content iteration. * refactor(otel): extract OTelModelOptions shared interface Replaces 3 duplicated inline type assertions for _otelTraceContext and _capturingTokenCorrelationId with a single shared interface. * refactor(otel): route OTel logs through ILogService output channel Replace console.info/error/warn in NodeOTelService with a log callback. OTelContrib logs essential status to the Copilot Chat output channel for user troubleshooting (enabled/disabled, exporter config, shutdown). * fix(otel): remove orphaned OTel ConfigKey definitions OTel config is read via workspace.getConfiguration in services.ts, not through IConfigurationService.get(ConfigKey). These constants were unused dead code. * test(otel): add comprehensive OTel instrumentation tests - Agent trace hierarchy (invoke_agent → chat → execute_tool, subagent propagation, error states, metrics, events) - BYOK provider span emission (CLIENT kind, token usage, error.type, content capture gating, parentTraceContext linking) - chatMLFetcher two-phase span lifecycle (create → enrich → end, error path, operation duration metric) - Service robustness (runWithTraceContext, startActiveSpan error lifecycle, storeTraceContext overwrite) - CapturingOTelService reusable test mock for all OTel assertions * chore: apply formatter import sorting * chore: remove outdated sprint plan document * feat(otel): add OTel configuration settings for tracing and logging * fix(otel): ensure metric reader is flushed and shutdown properly --- extensions/copilot/.gitignore | 3 + .../docs/monitoring/agent_monitoring.md | 506 ++++++++++++ .../docs/monitoring/agent_monitoring_arch.md | 297 +++++++ .../docs/monitoring/docker-compose.yaml | 36 + .../monitoring/otel-collector-config.yaml | 50 ++ extensions/copilot/package-lock.json | 770 ++++++++++++++++-- extensions/copilot/package.json | 54 ++ .../byok/vscode-node/anthropicProvider.ts | 121 ++- .../byok/vscode-node/geminiNativeProvider.ts | 121 ++- .../test/geminiNativeProvider.spec.ts | 7 +- .../vscode-node/languageModelAccess.ts | 23 +- .../extension/vscode-node/contributions.ts | 2 + .../extension/vscode-node/services.ts | 34 +- .../extension/intents/node/toolCallingLoop.ts | 113 +++ .../test/node/toolCallingLoopHooks.spec.ts | 5 + .../mcp/vscode-node/mcpToolCallingLoop.tsx | 4 +- .../extension/otel/vscode-node/otelContrib.ts | 38 + .../extension/prompt/node/chatMLFetcher.ts | 220 ++++- .../prompt/node/codebaseToolCalling.ts | 4 +- .../node/defaultIntentRequestHandler.ts | 4 +- .../node/searchSubagentToolCallingLoop.ts | 4 +- .../chatMLFetcherResponseApiTelemetry.spec.ts | 3 + .../node/test/chatMLFetcherRetry.spec.ts | 3 + .../tools/vscode-node/toolsService.ts | 79 +- .../copilot/src/lib/node/chatLibMain.ts | 4 + .../common/configurationService.ts | 7 + .../common/remoteEmbeddingsComputer.ts | 193 +++-- .../endpoint/vscode-node/extChatEndpoint.ts | 39 +- .../platform/otel/common/genAiAttributes.ts | 116 +++ .../src/platform/otel/common/genAiEvents.ts | 116 +++ .../src/platform/otel/common/genAiMetrics.ts | 99 +++ .../copilot/src/platform/otel/common/index.ts | 13 + .../platform/otel/common/messageFormatters.ts | 147 ++++ .../platform/otel/common/noopOtelService.ts | 60 ++ .../src/platform/otel/common/otelConfig.ts | 180 ++++ .../src/platform/otel/common/otelService.ts | 124 +++ .../common/test/agentTraceHierarchy.spec.ts | 226 +++++ .../common/test/byokProviderSpans.spec.ts | 133 +++ .../otel/common/test/capturingOTelService.ts | 160 ++++ .../test/chatMLFetcherSpanLifecycle.spec.ts | 112 +++ .../otel/common/test/genAiEvents.spec.ts | 160 ++++ .../otel/common/test/genAiMetrics.spec.ts | 124 +++ .../common/test/messageFormatters.spec.ts | 204 +++++ .../otel/common/test/noopOtelService.spec.ts | 57 ++ .../otel/common/test/otelConfig.spec.ts | 164 ++++ .../common/test/serviceRobustness.spec.ts | 104 +++ .../src/platform/otel/node/fileExporters.ts | 65 ++ .../src/platform/otel/node/otelServiceImpl.ts | 612 ++++++++++++++ .../otel/node/test/fileExporters.spec.ts | 116 +++ .../node/test/traceContextPropagation.spec.ts | 144 ++++ .../src/platform/test/node/services.ts | 4 + .../test/base/cachingEmbeddingsFetcher.ts | 3 + 52 files changed, 5761 insertions(+), 226 deletions(-) create mode 100644 extensions/copilot/docs/monitoring/agent_monitoring.md create mode 100644 extensions/copilot/docs/monitoring/agent_monitoring_arch.md create mode 100644 extensions/copilot/docs/monitoring/docker-compose.yaml create mode 100644 extensions/copilot/docs/monitoring/otel-collector-config.yaml create mode 100644 extensions/copilot/src/extension/otel/vscode-node/otelContrib.ts create mode 100644 extensions/copilot/src/platform/otel/common/genAiAttributes.ts create mode 100644 extensions/copilot/src/platform/otel/common/genAiEvents.ts create mode 100644 extensions/copilot/src/platform/otel/common/genAiMetrics.ts create mode 100644 extensions/copilot/src/platform/otel/common/index.ts create mode 100644 extensions/copilot/src/platform/otel/common/messageFormatters.ts create mode 100644 extensions/copilot/src/platform/otel/common/noopOtelService.ts create mode 100644 extensions/copilot/src/platform/otel/common/otelConfig.ts create mode 100644 extensions/copilot/src/platform/otel/common/otelService.ts create mode 100644 extensions/copilot/src/platform/otel/common/test/agentTraceHierarchy.spec.ts create mode 100644 extensions/copilot/src/platform/otel/common/test/byokProviderSpans.spec.ts create mode 100644 extensions/copilot/src/platform/otel/common/test/capturingOTelService.ts create mode 100644 extensions/copilot/src/platform/otel/common/test/chatMLFetcherSpanLifecycle.spec.ts create mode 100644 extensions/copilot/src/platform/otel/common/test/genAiEvents.spec.ts create mode 100644 extensions/copilot/src/platform/otel/common/test/genAiMetrics.spec.ts create mode 100644 extensions/copilot/src/platform/otel/common/test/messageFormatters.spec.ts create mode 100644 extensions/copilot/src/platform/otel/common/test/noopOtelService.spec.ts create mode 100644 extensions/copilot/src/platform/otel/common/test/otelConfig.spec.ts create mode 100644 extensions/copilot/src/platform/otel/common/test/serviceRobustness.spec.ts create mode 100644 extensions/copilot/src/platform/otel/node/fileExporters.ts create mode 100644 extensions/copilot/src/platform/otel/node/otelServiceImpl.ts create mode 100644 extensions/copilot/src/platform/otel/node/test/fileExporters.spec.ts create mode 100644 extensions/copilot/src/platform/otel/node/test/traceContextPropagation.spec.ts diff --git a/extensions/copilot/.gitignore b/extensions/copilot/.gitignore index d0a89e7b094..ad1103f2689 100644 --- a/extensions/copilot/.gitignore +++ b/extensions/copilot/.gitignore @@ -40,3 +40,6 @@ test/aml/out # claude .claude/settings.local.json + +# playwright +.playwright-mcp/ diff --git a/extensions/copilot/docs/monitoring/agent_monitoring.md b/extensions/copilot/docs/monitoring/agent_monitoring.md new file mode 100644 index 00000000000..0a98ddef065 --- /dev/null +++ b/extensions/copilot/docs/monitoring/agent_monitoring.md @@ -0,0 +1,506 @@ +# Monitoring Agent Usage with OpenTelemetry + +Copilot Chat can export **traces**, **metrics**, and **events** via [OpenTelemetry](https://opentelemetry.io/) (OTel) — giving you real-time visibility into agent interactions, LLM calls, tool executions, and token usage. + +All signal names and attributes follow the [OTel GenAI Semantic Conventions](https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai/), so the data works with any OTel-compatible backend: Jaeger, Grafana, Azure Monitor, Datadog, Honeycomb, and more. + +## Quick Start + +Set these environment variables before launching VS Code: + +```bash +# Enable OTel and point to a local collector +export COPILOT_OTEL_ENABLED=true +export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 + +# Launch VS Code +code . +``` + +That's it. Traces, metrics, and events start flowing to your collector. + +> **Tip:** To get started quickly with a local trace viewer, run [Jaeger](https://www.jaegertracing.io/) in Docker: +> ```bash +> docker run -d --name jaeger -p 16686:16686 -p 4318:4318 jaegertracing/jaeger:latest +> ``` +> Then open http://localhost:16687 and look for service `copilot-chat`. + +--- + +## Configuration + +### VS Code Settings + +Open **Settings** (`Ctrl+,`) and search for `copilot otel`: + +| Setting | Type | Default | Description | +|---|---|---|---| +| `github.copilot.chat.otel.enabled` | boolean | `false` | Enable OTel emission | +| `github.copilot.chat.otel.exporterType` | string | `"otlp-http"` | `otlp-http`, `otlp-grpc`, `console`, or `file` | +| `github.copilot.chat.otel.otlpEndpoint` | string | `"http://localhost:4318"` | OTLP collector endpoint | +| `github.copilot.chat.otel.captureContent` | boolean | `false` | Capture full prompt/response content | +| `github.copilot.chat.otel.outfile` | string | `""` | File path for JSON-lines output | + +### Environment Variables + +Environment variables **always take precedence** over VS Code settings. + +| Variable | Default | Description | +|---|---|---| +| `COPILOT_OTEL_ENABLED` | `false` | Enable OTel. Also enabled when `OTEL_EXPORTER_OTLP_ENDPOINT` is set. | +| `COPILOT_OTEL_ENDPOINT` | — | OTLP endpoint URL (takes precedence over `OTEL_EXPORTER_OTLP_ENDPOINT`) | +| `OTEL_EXPORTER_OTLP_ENDPOINT` | — | Standard OTel OTLP endpoint URL | +| `OTEL_EXPORTER_OTLP_PROTOCOL` | `http/protobuf` | OTLP protocol. Only `grpc` changes behavior; all other values use HTTP. | +| `COPILOT_OTEL_PROTOCOL` | — | Override OTLP protocol (`grpc` or `http`). Falls back to `OTEL_EXPORTER_OTLP_PROTOCOL`. | +| `OTEL_SERVICE_NAME` | `copilot-chat` | Service name in resource attributes | +| `OTEL_RESOURCE_ATTRIBUTES` | — | Extra resource attributes (`key1=val1,key2=val2`) | +| `COPILOT_OTEL_CAPTURE_CONTENT` | `false` | Capture full prompt/response content | +| `COPILOT_OTEL_LOG_LEVEL` | `info` | Min log level: `trace`, `debug`, `info`, `warn`, `error` | +| `COPILOT_OTEL_FILE_EXPORTER_PATH` | — | Write all signals to this file (JSON-lines) | +| `COPILOT_OTEL_HTTP_INSTRUMENTATION` | `false` | Enable HTTP-level OTel instrumentation | +| `OTEL_EXPORTER_OTLP_HEADERS` | — | Auth headers (e.g., `Authorization=Bearer token`) | + +### Activation + +OTel is **off by default** with zero overhead. It activates when: + +- `COPILOT_OTEL_ENABLED=true`, or +- `OTEL_EXPORTER_OTLP_ENDPOINT` is set, or +- `github.copilot.chat.otel.enabled` is `true` + + +--- + +## What Gets Exported + +### Traces + +Copilot Chat emits a hierarchical span tree for each agent interaction: + +``` +invoke_agent copilot [~15s] + ├── chat gpt-4o [~3s] (LLM requests tool calls) + ├── execute_tool readFile [~50ms] + ├── execute_tool runCommand [~2s] + ├── chat gpt-4o [~4s] (LLM generates final response) + └── (span ends) +``` + +**`invoke_agent`** — wraps the entire agent orchestration (all LLM calls + tool executions). + +| Attribute | Requirement | Example | +|---|---|---| +| `gen_ai.operation.name` | Required | `invoke_agent` | +| `gen_ai.provider.name` | Required | `github` | +| `gen_ai.agent.name` | Required | `copilot` | +| `gen_ai.conversation.id` | Required | `a1b2c3d4-...` | +| `gen_ai.request.model` | Recommended | `gpt-4o` | +| `gen_ai.response.model` | Recommended | `gpt-4o-2024-08-06` | +| `gen_ai.usage.input_tokens` | Recommended | `12500` | +| `gen_ai.usage.output_tokens` | Recommended | `3200` | +| `copilot_chat.turn_count` | Always | `4` | +| `error.type` | On error | `Error` | +| `gen_ai.input.messages` | Opt-in (captureContent) | `[{"role":"user",...}]` | +| `gen_ai.output.messages` | Opt-in (captureContent) | `[{"role":"assistant",...}]` | +| `gen_ai.tool.definitions` | Opt-in (captureContent) | `[{"type":"function",...}]` | + +**`chat`** — one span per LLM API call (span kind: `CLIENT`). + +| Attribute | Requirement | Example | +|---|---|---| +| `gen_ai.operation.name` | Required | `chat` | +| `gen_ai.provider.name` | Required | `github` | +| `gen_ai.request.model` | Required | `gpt-4o` | +| `gen_ai.conversation.id` | Required | `a1b2c3d4-...` | +| `gen_ai.request.max_tokens` | Always | `2048` | +| `gen_ai.request.temperature` | When set | `0.1` | +| `gen_ai.request.top_p` | When set | `0.95` | +| `copilot_chat.request.max_prompt_tokens` | Always | `128000` | +| `gen_ai.response.id` | On response | `chatcmpl-abc123` | +| `gen_ai.response.model` | On response | `gpt-4o-2024-08-06` | +| `gen_ai.response.finish_reasons` | On response | `["stop"]` | +| `gen_ai.usage.input_tokens` | On response | `1500` | +| `gen_ai.usage.output_tokens` | On response | `250` | +| `copilot_chat.time_to_first_token` | On response | `450` | +| `server.address` | When available | `api.github.com` | +| `copilot_chat.debug_name` | When available | `agentMode` | +| `error.type` | On error | `TimeoutError` | +| `gen_ai.input.messages` | Opt-in (captureContent) | `[{"role":"system",...}]` | +| `gen_ai.system_instructions` | Opt-in (captureContent) | `[{"type":"text",...}]` | + +**`execute_tool`** — one span per tool invocation (span kind: `INTERNAL`). + +| Attribute | Requirement | Example | +|---|---|---| +| `gen_ai.operation.name` | Required | `execute_tool` | +| `gen_ai.tool.name` | Required | `readFile` | +| `gen_ai.tool.type` | Required | `function` or `extension` (MCP tools) | +| `gen_ai.tool.call.id` | Recommended | `call_abc123` | +| `gen_ai.tool.description` | When available | `Read the contents of a file` | +| `error.type` | On error | `FileNotFoundError` | +| `gen_ai.tool.call.arguments` | Opt-in (captureContent) | `{"filePath":"/src/index.ts"}` | +| `gen_ai.tool.call.result` | Opt-in (captureContent) | `(file contents or summary)` | + +### Metrics + +#### GenAI Convention Metrics + +| Metric | Type | Unit | Description | +|---|---|---|---| +| `gen_ai.client.operation.duration` | Histogram | s | LLM API call duration | +| `gen_ai.client.token.usage` | Histogram | tokens | Token counts (input/output) | + +**`gen_ai.client.operation.duration` attributes:** + +| Attribute | Description | +|---|---| +| `gen_ai.operation.name` | Operation type (e.g., `chat`) | +| `gen_ai.provider.name` | Provider (e.g., `github`, `anthropic`) | +| `gen_ai.request.model` | Requested model | +| `gen_ai.response.model` | Resolved model (if different) | +| `server.address` | Server hostname | +| `server.port` | Server port | +| `error.type` | Error class (if failed) | + +**`gen_ai.client.token.usage` attributes:** + +| Attribute | Description | +|---|---| +| `gen_ai.operation.name` | Operation type | +| `gen_ai.provider.name` | Provider name | +| `gen_ai.token.type` | `input` or `output` | +| `gen_ai.request.model` | Requested model | +| `gen_ai.response.model` | Resolved model | +| `server.address` | Server hostname | + +#### Extension-Specific Metrics + +| Metric | Type | Unit | Description | +|---|---|---|---| +| `copilot_chat.tool.call.count` | Counter | calls | Tool invocations by name and success | +| `copilot_chat.tool.call.duration` | Histogram | ms | Tool execution latency | +| `copilot_chat.agent.invocation.duration` | Histogram | s | Agent mode end-to-end duration | +| `copilot_chat.agent.turn.count` | Histogram | turns | LLM round-trips per agent invocation | +| `copilot_chat.session.count` | Counter | sessions | Chat sessions started | +| `copilot_chat.time_to_first_token` | Histogram | s | Time to first SSE token | + +**`copilot_chat.tool.call.count` attributes:** `gen_ai.tool.name`, `success` (boolean) + +**`copilot_chat.tool.call.duration` attributes:** `gen_ai.tool.name` + +**`copilot_chat.agent.invocation.duration` attributes:** `gen_ai.agent.name` + +**`copilot_chat.agent.turn.count` attributes:** `gen_ai.agent.name` + +**`copilot_chat.time_to_first_token` attributes:** `gen_ai.request.model` + +### Events + +#### `gen_ai.client.inference.operation.details` + +Emitted after each LLM API call with full inference metadata. + +| Attribute | Description | +|---|---| +| `gen_ai.operation.name` | Always `chat` | +| `gen_ai.request.model` | Requested model | +| `gen_ai.response.model` | Resolved model | +| `gen_ai.response.id` | Response ID | +| `gen_ai.response.finish_reasons` | Stop reasons (e.g., `["stop"]`) | +| `gen_ai.usage.input_tokens` | Input token count | +| `gen_ai.usage.output_tokens` | Output token count | +| `gen_ai.request.temperature` | Temperature (if set) | +| `gen_ai.request.max_tokens` | Max tokens (if set) | +| `error.type` | Error class (if failed) | +| `gen_ai.input.messages` | Full prompt messages (captureContent only) | +| `gen_ai.system_instructions` | System prompt (captureContent only) | +| `gen_ai.tool.definitions` | Tool schemas (captureContent only) | + +#### `copilot_chat.session.start` + +Emitted when a new chat session begins (top-level agent invocations only, not subagents). + +| Attribute | Description | +|---|---| +| `session.id` | Session identifier | +| `gen_ai.request.model` | Initial model | +| `gen_ai.agent.name` | Chat participant name | + +#### `copilot_chat.tool.call` + +Emitted when a tool invocation completes. + +| Attribute | Description | +|---|---| +| `gen_ai.tool.name` | Tool name | +| `duration_ms` | Execution time in milliseconds | +| `success` | `true` or `false` | +| `error.type` | Error class (if failed) | + +#### `copilot_chat.agent.turn` + +Emitted for each LLM round-trip within an agent invocation. + +| Attribute | Description | +|---|---| +| `turn.index` | Turn number (0-indexed) | +| `gen_ai.usage.input_tokens` | Input tokens this turn | +| `gen_ai.usage.output_tokens` | Output tokens this turn | +| `tool_call_count` | Number of tool calls this turn | + +### Resource Attributes + +All signals carry: + +| Attribute | Value | +|---|---| +| `service.name` | `copilot-chat` (configurable via `OTEL_SERVICE_NAME`) | +| `service.version` | Extension version | +| `session.id` | Unique per VS Code window | + +Add custom resource attributes with `OTEL_RESOURCE_ATTRIBUTES`: + +```bash +export OTEL_RESOURCE_ATTRIBUTES="team.id=platform,department=engineering" +``` + +These custom attributes are included in all traces, metrics, and events, allowing you to: + +- Filter metrics by team or department +- Create team-specific dashboards and alerts +- Track usage across organizational boundaries + +> **Note:** `OTEL_RESOURCE_ATTRIBUTES` uses comma-separated `key=value` pairs. Values cannot contain spaces, commas, or semicolons. Use percent-encoding for special characters (e.g., `org.name=John%27s%20Org`). + +--- + +## Content Capture + +By default, **no prompt content, responses, or tool arguments are captured** — only metadata like model names, token counts, and durations. + +To capture full content: + +```bash +export COPILOT_OTEL_CAPTURE_CONTENT=true +``` + +This populates these span attributes: + +| Attribute | Content | +|---|---| +| `gen_ai.input.messages` | Full prompt messages (JSON) | +| `gen_ai.output.messages` | Full response messages (JSON) | +| `gen_ai.system_instructions` | System prompt | +| `gen_ai.tool.definitions` | Tool schemas | +| `gen_ai.tool.call.arguments` | Tool input arguments | +| `gen_ai.tool.call.result` | Tool output | + +Content is captured in full with no truncation. + +> **Warning:** Content capture may include sensitive information such as code, file contents, and user prompts. Only enable in trusted environments. + +--- + +## Example Configurations + +**OTLP/gRPC:** + +```bash +export COPILOT_OTEL_ENABLED=true +export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 +export OTEL_EXPORTER_OTLP_PROTOCOL=grpc +``` + +**Remote collector with authentication:** + +```bash +export COPILOT_OTEL_ENABLED=true +export OTEL_EXPORTER_OTLP_ENDPOINT=https://collector.example.com:4318 +export OTEL_EXPORTER_OTLP_HEADERS="Authorization=Bearer your-token" +``` + +**File-based output (offline / CI):** + +```bash +export COPILOT_OTEL_ENABLED=true +export COPILOT_OTEL_FILE_EXPORTER_PATH=/tmp/copilot-otel.jsonl +``` + +**Console output (quick debugging):** + +```json +{ + "github.copilot.chat.otel.enabled": true, + "github.copilot.chat.otel.exporterType": "console" +} +``` + +--- + +## Subagent Trace Propagation + +When an agent invokes a subagent (e.g., via the `runSubagent` tool), Copilot Chat automatically propagates the trace context so the subagent's `invoke_agent` span is parented to the calling agent's `execute_tool` span. This produces a connected trace tree: + +``` +invoke_agent copilot [~30s] + ├── chat gpt-4o [~3s] + ├── execute_tool runSubagent [~20s] + │ └── invoke_agent Explore [~18s] ← child via trace context + │ ├── chat gpt-4o [~2s] + │ ├── execute_tool searchFiles [~200ms] + │ ├── execute_tool readFile [~50ms] + │ └── chat gpt-4o [~3s] + ├── chat gpt-4o [~4s] + └── (span ends) +``` + +This propagation works across async boundaries — the parent's trace context is stored when `runSubagent` starts and retrieved when the subagent begins its `invoke_agent` span. + +--- + +## Interpreting the Data + +**Traces** — Visualize the full agent execution in Jaeger or Grafana Tempo. Each `invoke_agent` span contains child `chat` and `execute_tool` spans, making it easy to identify bottlenecks and debug failures. Subagent invocations appear as nested `invoke_agent` spans under `execute_tool runSubagent`. + +**Metrics** — Track token usage trends by model and provider, monitor tool success rates via `copilot_chat.tool.call.count`, and watch perceived latency with `copilot_chat.time_to_first_token`. All metrics carry the same resource attributes (`service.name`, `service.version`, `session.id`) for consistent filtering. + +**Events** — `copilot_chat.session.start` tracks session creation. `copilot_chat.tool.call` events provide per-invocation timing and error details. `gen_ai.client.inference.operation.details` gives the full LLM call record including token usage and, when content capture is enabled, the complete prompt/response messages. Use `gen_ai.conversation.id` to correlate all signals belonging to the same session. + +--- + +## Initialization & Buffering + +The OTel SDK is loaded asynchronously via dynamic imports to avoid blocking extension startup. Events emitted before initialization completes are buffered (up to 1,000 items) and replayed once the SDK is ready. If initialization fails, buffered events are discarded and all subsequent calls become no-ops — the extension continues to function normally. + +First successful span export is logged to the console (`[OTel] First span batch exported successfully via ...`) to confirm end-to-end connectivity. + +--- + +## Backend Setup & Verification + +Copilot Chat's OTel data works with any OTLP-compatible backend. This section covers setup and verification for recommended backends. + +### OTel Collector + Azure Application Insights + +[Azure Application Insights](https://learn.microsoft.com/en-us/azure/azure-monitor/app/app-insights-overview) ingests OTel traces, metrics, and logs through an [OTel Collector](https://opentelemetry.io/docs/collector/) with the `azuremonitor` exporter. This repo includes a ready-to-use collector setup in `docs/monitoring/`. + +**1. Start the collector:** + +```bash +# Set your App Insights connection string (from Azure Portal → App Insights → Overview) +export APPLICATIONINSIGHTS_CONNECTION_STRING="InstrumentationKey=...;IngestionEndpoint=..." + +# Start the OTel Collector +cd docs/monitoring +docker compose up -d +``` + +**2. Verify the collector is healthy:** + +```bash +# Should return 200 +curl -s -o /dev/null -w "%{http_code}" http://localhost:4328/v1/traces \ + -X POST -H "Content-Type: application/json" -d '{"resourceSpans":[]}' +``` + +**3. Launch VS Code:** + +```bash +COPILOT_OTEL_ENABLED=true \ +COPILOT_OTEL_CAPTURE_CONTENT=true \ +OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4328 \ +code . +``` + +**4. Generate telemetry** — Send a chat message in Copilot Chat (e.g., "explain this file" in agent mode). + +**5. Verify in App Insights:** + +- **Traces:** Application Insights → Transaction search → filter by "Trace" or "Request". + +- **Logs:** Application Insights → Logs: + ```kql + traces + | where timestamp > ago(1h) + | where message contains "GenAI" or message contains "copilot_chat" + | project timestamp, message, customDimensions + | order by timestamp desc + ``` + +- **Metrics:** Application Insights → Metrics → "Custom" namespace, or via Logs: + ```kql + customMetrics + | where timestamp > ago(1h) + | where name startswith "gen_ai" or name startswith "copilot_chat" + | summarize avg(value), count() by name + ``` + +> **Note:** Traces typically appear within 1-2 minutes. Metrics may take 5-10 minutes. + +**Collector config** (`docs/monitoring/otel-collector-config.yaml`): + +```yaml +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + grpc: + endpoint: 0.0.0.0:4317 + +exporters: + azuremonitor: + connection_string: "${APPLICATIONINSIGHTS_CONNECTION_STRING}" + debug: + verbosity: basic + +service: + pipelines: + traces: + receivers: [otlp] + exporters: [azuremonitor, debug] + metrics: + receivers: [otlp] + exporters: [azuremonitor, debug] +``` + +> **Note:** The docker-compose maps ports to `4328`/`4327` on the host to avoid conflicts. Adjust in `docker-compose.yaml` if needed. Add additional exporters (e.g., `otlphttp/jaeger`) to fan out to multiple backends. See `docs/monitoring/otel-collector-config.yaml` for the full config including `batch` processor and `logs` pipeline. + + +### Langfuse + +[Langfuse](https://langfuse.com/) is an open-source LLM observability platform with native OTLP ingestion and support for [OTel GenAI Semantic Conventions](https://opentelemetry.io/docs/specs/semconv/gen-ai/gen-ai-agent-spans/). See the [Langfuse docs](https://langfuse.com/docs/opentelemetry/introduction) for full details on capabilities and limitations. + +**Setup:** + +```bash +export COPILOT_OTEL_ENABLED=true +export OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:3000/api/public/otel +export OTEL_EXPORTER_OTLP_HEADERS="Authorization=Basic $(echo -n ':' | base64)" +export COPILOT_OTEL_CAPTURE_CONTENT=true +``` + +Replace `` and `` with your Langfuse API keys from **Settings → API Keys**. + +**Verify:** Open Langfuse → **Traces**. You should see `invoke_agent` traces with nested `chat` and `execute_tool` spans. + +### Other Backends + +Any OTLP-compatible backend works with Copilot Chat's OTel output. Some options: + +| Backend | Description | +|---|---| +| **[Jaeger](https://www.jaegertracing.io/)** | Open-source distributed tracing platform | +| **[Grafana Tempo](https://grafana.com/oss/tempo/) + [Prometheus](https://prometheus.io/)** | Open-source traces + metrics stack | + +Refer to each backend's documentation for OTLP ingestion setup. + +--- + +## Security & Privacy + +- **Off by default.** No OTel data is emitted unless explicitly enabled. When disabled, the OTel SDK is not loaded at all — zero runtime overhead. +- **No content by default.** Prompts, responses, and tool arguments require opt-in via `captureContent`. +- **No PII in default attributes.** Session IDs, model names, and token counts are not personally identifiable. +- **User-configured endpoints.** Data goes only where you point it — no phone-home behavior. +- **Dynamic imports only.** OTel SDK packages are loaded on-demand, ensuring zero bundle impact when disabled. diff --git a/extensions/copilot/docs/monitoring/agent_monitoring_arch.md b/extensions/copilot/docs/monitoring/agent_monitoring_arch.md new file mode 100644 index 00000000000..7e3fa956245 --- /dev/null +++ b/extensions/copilot/docs/monitoring/agent_monitoring_arch.md @@ -0,0 +1,297 @@ +# OTel Instrumentation — Developer Guide + +This document describes the architecture, code structure, and conventions for the OpenTelemetry instrumentation in the Copilot Chat extension. It is intended for developers contributing to or maintaining this codebase. + +For user-facing configuration and usage, see [agent_monitoring.md](agent_monitoring.md). + +--- + +## Architecture Overview + +``` +┌──────────────────────────────────────────────────────────────────┐ +│ VS Code Copilot Chat Extension │ +│ │ +│ ┌─────────────┐ ┌──────────────┐ ┌──────────┐ ┌──────────┐ │ +│ │ ChatML │ │ Tool Calling │ │ Tools │ │ Prompts │ │ +│ │ Fetcher │ │ Loop │ │ Service │ │ │ │ +│ └──────┬──────┘ └──────┬───────┘ └────┬─────┘ └────┬─────┘ │ +│ │ │ │ │ │ +│ ▼ ▼ ▼ ▼ │ +│ ┌──────────────────────────────────────────────────────────┐ │ +│ │ IOTelService (DI) │ │ +│ │ ┌─────────┐ ┌──────────┐ ┌─────────┐ ┌───────────┐ │ │ +│ │ │ Tracer │ │ Meter │ │ Logger │ │ Semantic │ │ │ +│ │ │ (spans) │ │ (metrics)│ │ (events)│ │ Helpers │ │ │ +│ │ └────┬────┘ └────┬─────┘ └────┬────┘ └───────────┘ │ │ +│ └───────┼─────────────┼────────────┼──────────────────────┘ │ +│ ▼ ▼ ▼ │ +│ ┌─────────────────────────────────────────────┐ │ +│ │ OTel SDK (BatchSpanProcessor, │ │ +│ │ BatchLogRecordProcessor, │ │ +│ │ PeriodicExportingMetricReader) │ │ +│ └──────────────────┬──────────────────────────┘ │ +│ ▼ │ +│ ┌─────────────────────────────────────────────┐ │ +│ │ Exporters: OTLP/HTTP | OTLP/gRPC | │ │ +│ │ Console | File (JSON-lines) │ │ +│ └─────────────────────────────────────────────┘ │ +└──────────────────────────────────────────────────────────────────┘ +``` + +--- + +## File Structure + +``` +src/platform/otel/ +├── common/ +│ ├── otelService.ts # IOTelService interface + ISpanHandle +│ ├── otelConfig.ts # Config resolution (env → settings → defaults) +│ ├── noopOtelService.ts # Zero-cost no-op implementation +│ ├── genAiAttributes.ts # GenAI semantic convention attribute keys +│ ├── genAiEvents.ts # Event emitter helpers +│ ├── genAiMetrics.ts # GenAiMetrics class (metric recording) +│ ├── messageFormatters.ts # Message → OTel JSON schema converters +│ ├── index.ts # Public API barrel export +│ └── test/ # Unit tests +└── node/ + ├── otelServiceImpl.ts # NodeOTelService (real SDK implementation) + ├── fileExporters.ts # File-based span/log/metric exporters + └── test/ # Unit tests + +src/extension/otel/ +└── vscode-node/ + └── otelContrib.ts # Lifecycle contribution (shutdown hook) +``` + +### Instrumentation Points + +| File | What Gets Instrumented | +|---|---| +| `src/extension/prompt/node/chatMLFetcher.ts` | `chat` spans — one per LLM API call. Used by standard CAPI endpoints **and** all OpenAI-compatible BYOK providers (Azure, OpenAI, Ollama, OpenRouter, xAI, CustomOAI) via `CopilotLanguageModelWrapper` → `endpoint.makeChatRequest` | +| `src/extension/byok/vscode-node/anthropicProvider.ts` | `chat` spans — BYOK Anthropic requests (native SDK, instrumented directly) | +| `src/extension/byok/vscode-node/geminiNativeProvider.ts` | `chat` spans — BYOK Gemini requests (native SDK, instrumented directly) | +| `src/extension/intents/node/toolCallingLoop.ts` | `invoke_agent` spans — wraps agent orchestration | +| `src/extension/tools/vscode-node/toolsService.ts` | `execute_tool` spans — one per tool invocation | +| `src/extension/extension/vscode-node/services.ts` | Service registration (config → NodeOTelService or NoopOTelService) | + +--- + +## Service Layer + +### `IOTelService` Interface + +The core abstraction. All consumers depend on this interface, never on the OTel SDK directly. It exposes methods for starting spans, recording metrics, emitting log records, managing trace context propagation, and lifecycle (`flush`/`shutdown`). + +### Implementations + +| Class | When Used | Characteristics | +|---|---|---| +| `NoopOTelService` | OTel disabled (default) | All methods are empty. Zero cost. | +| `NodeOTelService` | OTel enabled | Full SDK with dynamic imports, buffering, batched processors. | + +### Registration + +In `services.ts`, the config is resolved from env + settings, then the appropriate implementation is registered: + +```typescript +const otelConfig = resolveOTelConfig({ env: process.env, ... }); +if (otelConfig.enabled) { + const { NodeOTelService } = require('.../otelServiceImpl'); + builder.define(IOTelService, new NodeOTelService(otelConfig)); +} else { + builder.define(IOTelService, new NoopOTelService(otelConfig)); +} +``` + +The `require()` (not `import()`) is intentional here — it avoids loading the SDK at all when disabled, while the `NodeOTelService` constructor internally uses `import()` for all OTel packages. + +--- + +## Configuration Resolution + +`resolveOTelConfig()` in `otelConfig.ts` implements layered precedence: + +1. `COPILOT_OTEL_*` env vars (highest) +2. `OTEL_EXPORTER_OTLP_*` standard env vars +3. VS Code settings (`github.copilot.chat.otel.*`) +4. Defaults (lowest) + +Kill switch: If `telemetry.telemetryLevel === 'off'`, the config resolver returns a disabled config. Note: `vscodeTelemetryLevel` must be passed by the call site — currently not wired in `services.ts`. + +Endpoint parsing: gRPC → origin only (`scheme://host:port`). HTTP → full href. + +--- + +## Span Conventions + +### Naming + +Follow the OTel GenAI conventions: + +| Operation | Span Name | Kind | +|---|---|---| +| Agent orchestration | `invoke_agent {agent_name}` | `INTERNAL` | +| LLM API call | `chat {model}` | `CLIENT` | +| Tool execution | `execute_tool {tool_name}` | `INTERNAL` | + +### Attributes + +Use the constants from `genAiAttributes.ts`: + +```typescript +import { GenAiAttr, GenAiOperationName, CopilotChatAttr, StdAttr } from '../../platform/otel/common/index'; + +span.setAttributes({ + [GenAiAttr.OPERATION_NAME]: GenAiOperationName.CHAT, + [GenAiAttr.REQUEST_MODEL]: model, + [GenAiAttr.USAGE_INPUT_TOKENS]: inputTokens, + [StdAttr.ERROR_TYPE]: error.constructor.name, +}); +``` + +### Error Handling + +On error, set both status and `error.type`: + +```typescript +span.setStatus(SpanStatusCode.ERROR, error.message); +span.setAttribute(StdAttr.ERROR_TYPE, error.constructor.name); +``` + +### Content Capture + +Always gate content capture on `otel.config.captureContent`: + +```typescript +if (this._otelService.config.captureContent) { + span.setAttribute(GenAiAttr.INPUT_MESSAGES, JSON.stringify(messages)); +} +``` + +--- + +## Adding Instrumentation to New Code + +### Pattern: Wrapping an Operation with a Span + +```typescript +class MyService { + constructor(@IOTelService private readonly _otel: IOTelService) {} + + async doWork(): Promise { + return this._otel.startActiveSpan( + 'execute_tool myTool', + { kind: SpanKind.INTERNAL, attributes: { [GenAiAttr.TOOL_NAME]: 'myTool' } }, + async (span) => { + try { + const result = await this._actualWork(); + span.setStatus(SpanStatusCode.OK); + return result; + } catch (err) { + span.setStatus(SpanStatusCode.ERROR, err instanceof Error ? err.message : String(err)); + span.setAttribute(StdAttr.ERROR_TYPE, err instanceof Error ? err.constructor.name : 'Error'); + throw err; + } + }, + ); + } +} +``` + +### Pattern: Recording Metrics + +Use `GenAiMetrics` for standard metric recording: + +```typescript +const metrics = new GenAiMetrics(this._otelService); +metrics.recordTokenUsage(1500, 'input', { + operationName: GenAiOperationName.CHAT, + providerName: GenAiProviderName.GITHUB, + requestModel: 'gpt-4o', +}); +metrics.recordToolCallCount('readFile', true); +metrics.recordTimeToFirstToken('gpt-4o', 0.45); +``` + +### Pattern: Emitting Events + +```typescript +import { emitToolCallEvent, emitInferenceDetailsEvent } from '../../platform/otel/common/index'; + +emitToolCallEvent(this._otelService, 'readFile', 50, true); +emitInferenceDetailsEvent(this._otelService, { model: 'gpt-4o' }, { inputTokens: 1500 }); +``` + +### Pattern: Cross-Boundary Trace Propagation + +When spawning a subagent, store the current trace context and retrieve it in the child: + +```typescript +// Parent: store context before spawning subagent +const traceContext = this._otelService.getActiveTraceContext(); +if (traceContext) { + this._otelService.storeTraceContext(`subagent:${requestId}`, traceContext); +} + +// Child: retrieve and use as parent +const parentCtx = this._otelService.getStoredTraceContext(`subagent:${requestId}`); +return this._otelService.startActiveSpan('invoke_agent child', { parentTraceContext: parentCtx }, async (span) => { + // child spans are now part of the same trace +}); +``` + +--- + +## Buffering & Initialization + +`NodeOTelService` buffers operations during async SDK initialization. Once init completes, the buffer is drained in order; on failure, it is discarded and all future calls become no-ops. `BufferedSpanHandle` captures span mutations during this window and replays them onto the real span once available. + +--- + +## Exporters + +Four exporter types are supported: OTLP/HTTP (default), OTLP/gRPC, Console (stdout), and File (JSON-lines). All OTel SDK packages are dynamically imported — none are loaded when OTel is disabled. `DiagnosticSpanExporter` wraps the span exporter to log the first successful export (confirms connectivity). + +--- + +## GenAI Semantic Convention Reference + +All attribute names follow [OTel GenAI Semantic Conventions](https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai/). + +Constants are defined in `genAiAttributes.ts`: + +- `GenAiAttr.*` — Standard `gen_ai.*` attribute keys +- `CopilotChatAttr.*` — Extension-specific `copilot_chat.*` keys +- `StdAttr.*` — Standard OTel keys (`error.type`, `server.address`, `server.port`) +- `GenAiOperationName.*` — Operation name values (`chat`, `invoke_agent`, `execute_tool`) +- `GenAiProviderName.*` — Provider values (`github`, `openai`, `anthropic`) + +Message formatting helpers in `messageFormatters.ts` convert internal message types to the OTel JSON schema: + +- `toInputMessages()` — CAPI messages → OTel input format +- `toOutputMessages()` — Model response choices → OTel output format +- `toSystemInstructions()` — System message → OTel system instruction format +- `toToolDefinitions()` — Tool schemas → OTel tool definition format + +--- + +## Testing + +Unit tests live alongside the source: + +``` +src/platform/otel/common/test/ +├── genAiEvents.spec.ts +├── genAiMetrics.spec.ts +├── messageFormatters.spec.ts +├── noopOtelService.spec.ts +└── otelConfig.spec.ts + +src/platform/otel/node/test/ +├── fileExporters.spec.ts +└── traceContextPropagation.spec.ts +``` + +Run with: `npm test -- --grep "OTel"` diff --git a/extensions/copilot/docs/monitoring/docker-compose.yaml b/extensions/copilot/docs/monitoring/docker-compose.yaml new file mode 100644 index 00000000000..e35eccf1626 --- /dev/null +++ b/extensions/copilot/docs/monitoring/docker-compose.yaml @@ -0,0 +1,36 @@ +# Copilot Chat OTel monitoring stack +# +# Starts an OpenTelemetry Collector that accepts OTLP on :4318 (HTTP) and :4317 (gRPC), +# then forwards traces/metrics/logs to Azure Application Insights and a local Jaeger instance. +# +# Usage: +# # Set your App Insights connection string: +# export APPLICATIONINSIGHTS_CONNECTION_STRING="InstrumentationKey=...;IngestionEndpoint=..." +# +# # Start the stack: +# docker compose up -d +# +# # View traces in Jaeger: +# open http://localhost:16687 +# +# # Then launch VS Code with: +# COPILOT_OTEL_ENABLED=true OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4328 code . + +services: + otel-collector: + image: otel/opentelemetry-collector-contrib:latest + command: ["--config=/etc/otel-collector-config.yaml"] + volumes: + - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml:ro + ports: + - "4327:4317" # OTLP gRPC (host:4327 → container:4317) + - "4328:4318" # OTLP HTTP (host:4328 → container:4318) + environment: + - APPLICATIONINSIGHTS_CONNECTION_STRING=${APPLICATIONINSIGHTS_CONNECTION_STRING:-} + restart: unless-stopped + + jaeger: + image: jaegertracing/jaeger:latest + ports: + - "16687:16686" # Jaeger UI (host:16687 to avoid conflict) + restart: unless-stopped diff --git a/extensions/copilot/docs/monitoring/otel-collector-config.yaml b/extensions/copilot/docs/monitoring/otel-collector-config.yaml new file mode 100644 index 00000000000..9024cb91aea --- /dev/null +++ b/extensions/copilot/docs/monitoring/otel-collector-config.yaml @@ -0,0 +1,50 @@ +# OpenTelemetry Collector configuration for Copilot Chat +# Receives OTLP from Copilot Chat and exports to multiple backends. +# +# Usage: +# docker compose -f docs/monitoring/docker-compose.yaml up -d +# +# Then set in VS Code or launch.json: +# OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318 + +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + grpc: + endpoint: 0.0.0.0:4317 + +processors: + batch: + timeout: 5s + send_batch_size: 256 + +exporters: + # Azure Application Insights via connection string + # Replace with your App Insights connection string + azuremonitor: + connection_string: "${APPLICATIONINSIGHTS_CONNECTION_STRING}" + + # Debug exporter — prints to collector stdout (useful for troubleshooting) + debug: + verbosity: basic + + # Local Jaeger for trace visualization + otlphttp/jaeger: + endpoint: http://jaeger:4318 + +service: + pipelines: + traces: + receivers: [otlp] + processors: [batch] + exporters: [azuremonitor, otlphttp/jaeger, debug] + metrics: + receivers: [otlp] + processors: [batch] + exporters: [azuremonitor, debug] + logs: + receivers: [otlp] + processors: [batch] + exporters: [azuremonitor, debug] diff --git a/extensions/copilot/package-lock.json b/extensions/copilot/package-lock.json index 4439525887c..9ca0743599a 100644 --- a/extensions/copilot/package-lock.json +++ b/extensions/copilot/package-lock.json @@ -18,6 +18,19 @@ "@humanwhocodes/gitignore-to-minimatch": "1.0.2", "@microsoft/tiktokenizer": "^1.0.10", "@modelcontextprotocol/sdk": "^1.25.2", + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/api-logs": "^0.212.0", + "@opentelemetry/exporter-logs-otlp-grpc": "^0.212.0", + "@opentelemetry/exporter-logs-otlp-http": "^0.212.0", + "@opentelemetry/exporter-metrics-otlp-grpc": "^0.212.0", + "@opentelemetry/exporter-metrics-otlp-http": "^0.212.0", + "@opentelemetry/exporter-trace-otlp-grpc": "^0.212.0", + "@opentelemetry/exporter-trace-otlp-http": "^0.212.0", + "@opentelemetry/resources": "^2.5.1", + "@opentelemetry/sdk-logs": "^0.212.0", + "@opentelemetry/sdk-metrics": "^2.5.1", + "@opentelemetry/sdk-trace-node": "^2.5.1", + "@opentelemetry/semantic-conventions": "^1.39.0", "@sinclair/typebox": "^0.34.41", "@vscode/copilot-api": "^0.2.13", "@vscode/extension-telemetry": "^1.5.1", @@ -3309,6 +3322,37 @@ "csstype": "^3.1.3" } }, + "node_modules/@grpc/grpc-js": { + "version": "1.14.3", + "resolved": "https://registry.npmjs.org/@grpc/grpc-js/-/grpc-js-1.14.3.tgz", + "integrity": "sha512-Iq8QQQ/7X3Sac15oB6p0FmUg/klxQvXLeileoqrTRGJYLV+/9tubbr9ipz0GKHjmXVsgFPo/+W+2cA8eNcR+XA==", + "license": "Apache-2.0", + "dependencies": { + "@grpc/proto-loader": "^0.8.0", + "@js-sdsl/ordered-map": "^4.4.2" + }, + "engines": { + "node": ">=12.10.0" + } + }, + "node_modules/@grpc/proto-loader": { + "version": "0.8.0", + "resolved": "https://registry.npmjs.org/@grpc/proto-loader/-/proto-loader-0.8.0.tgz", + "integrity": "sha512-rc1hOQtjIWGxcxpb9aHAfLpIctjEnsDehj0DAiVfBlmT84uvR0uUtN2hEi/ecvWVjXUGf5qPF4qEgiLOx1YIMQ==", + "license": "Apache-2.0", + "dependencies": { + "lodash.camelcase": "^4.3.0", + "long": "^5.0.0", + "protobufjs": "^7.5.3", + "yargs": "^17.7.2" + }, + "bin": { + "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/@hediet/node-reload": { "version": "0.8.0", "resolved": "https://registry.npmjs.org/@hediet/node-reload/-/node-reload-0.8.0.tgz", @@ -3867,6 +3911,16 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@js-sdsl/ordered-map": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/@js-sdsl/ordered-map/-/ordered-map-4.4.2.tgz", + "integrity": "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw==", + "license": "MIT", + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/js-sdsl" + } + }, "node_modules/@keyv/serialize": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/@keyv/serialize/-/serialize-1.1.1.tgz", @@ -4248,13 +4302,38 @@ } }, "node_modules/@opentelemetry/api": { - "version": "1.6.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.6.0.tgz", - "integrity": "sha512-OWlrQAnWn9577PhVgqjUvMr1pg57Bc4jv0iL4w0PRuOSRvq67rvHW9Ie/dZVMvCzhSCB+UxhcY/PmCmFj33Q+g==", + "version": "1.9.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", + "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", + "license": "Apache-2.0", "engines": { "node": ">=8.0.0" } }, + "node_modules/@opentelemetry/api-logs": { + "version": "0.212.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.212.0.tgz", + "integrity": "sha512-TEEVrLbNROUkYY51sBJGk7lO/OLjuepch8+hmpM6ffMJQ2z/KVCjdHuCFX6fJj8OkJP2zckPjrJzQtXU3IAsFg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api": "^1.3.0" + }, + "engines": { + "node": ">=8.0.0" + } + }, + "node_modules/@opentelemetry/context-async-hooks": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/context-async-hooks/-/context-async-hooks-2.5.1.tgz", + "integrity": "sha512-MHbu8XxCHcBn6RwvCt2Vpn1WnLMNECfNKYB14LI5XypcgH4IE0/DiVifVR9tAkwPMyLXN8dOoPJfya3IryLQVw==", + "license": "Apache-2.0", + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, "node_modules/@opentelemetry/core": { "version": "1.30.1", "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-1.30.1.tgz", @@ -4279,6 +4358,250 @@ "node": ">=14" } }, + "node_modules/@opentelemetry/exporter-logs-otlp-grpc": { + "version": "0.212.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-logs-otlp-grpc/-/exporter-logs-otlp-grpc-0.212.0.tgz", + "integrity": "sha512-/0bk6fQG+eSFZ4L6NlckGTgUous/ib5+OVdg0x4OdwYeHzV3lTEo3it1HgnPY6UKpmX7ki+hJvxjsOql8rCeZA==", + "license": "Apache-2.0", + "dependencies": { + "@grpc/grpc-js": "^1.14.3", + "@opentelemetry/core": "2.5.1", + "@opentelemetry/otlp-exporter-base": "0.212.0", + "@opentelemetry/otlp-grpc-exporter-base": "0.212.0", + "@opentelemetry/otlp-transformer": "0.212.0", + "@opentelemetry/sdk-logs": "0.212.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-logs-otlp-grpc/node_modules/@opentelemetry/core": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.5.1.tgz", + "integrity": "sha512-Dwlc+3HAZqpgTYq0MUyZABjFkcrKTePwuiFVLjahGD8cx3enqihmpAmdgNFO1R4m/sIe5afjJrA25Prqy4NXlA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-logs-otlp-http": { + "version": "0.212.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-logs-otlp-http/-/exporter-logs-otlp-http-0.212.0.tgz", + "integrity": "sha512-JidJasLwG/7M9RTxV/64xotDKmFAUSBc9SNlxI32QYuUMK5rVKhHNWMPDzC7E0pCAL3cu+FyiKvsTwLi2KqPYw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.212.0", + "@opentelemetry/core": "2.5.1", + "@opentelemetry/otlp-exporter-base": "0.212.0", + "@opentelemetry/otlp-transformer": "0.212.0", + "@opentelemetry/sdk-logs": "0.212.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-logs-otlp-http/node_modules/@opentelemetry/core": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.5.1.tgz", + "integrity": "sha512-Dwlc+3HAZqpgTYq0MUyZABjFkcrKTePwuiFVLjahGD8cx3enqihmpAmdgNFO1R4m/sIe5afjJrA25Prqy4NXlA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-metrics-otlp-grpc": { + "version": "0.212.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-metrics-otlp-grpc/-/exporter-metrics-otlp-grpc-0.212.0.tgz", + "integrity": "sha512-/6Gqf9wpBq22XsomR1i0iPGnbQtCq2Vwnrq5oiDPjYSqveBdK1jtQbhGfmpK2mLLxk4cPDtD1ZEYdIou5K8EaA==", + "license": "Apache-2.0", + "dependencies": { + "@grpc/grpc-js": "^1.14.3", + "@opentelemetry/core": "2.5.1", + "@opentelemetry/exporter-metrics-otlp-http": "0.212.0", + "@opentelemetry/otlp-exporter-base": "0.212.0", + "@opentelemetry/otlp-grpc-exporter-base": "0.212.0", + "@opentelemetry/otlp-transformer": "0.212.0", + "@opentelemetry/resources": "2.5.1", + "@opentelemetry/sdk-metrics": "2.5.1" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-metrics-otlp-grpc/node_modules/@opentelemetry/core": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.5.1.tgz", + "integrity": "sha512-Dwlc+3HAZqpgTYq0MUyZABjFkcrKTePwuiFVLjahGD8cx3enqihmpAmdgNFO1R4m/sIe5afjJrA25Prqy4NXlA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-metrics-otlp-http": { + "version": "0.212.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-metrics-otlp-http/-/exporter-metrics-otlp-http-0.212.0.tgz", + "integrity": "sha512-8hgBw3aTTRpSTkU4b9MLf/2YVLnfWp+hfnLq/1Fa2cky+vx6HqTodo+Zv1GTIrAKMOOwgysOjufy0gTxngqeBg==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.5.1", + "@opentelemetry/otlp-exporter-base": "0.212.0", + "@opentelemetry/otlp-transformer": "0.212.0", + "@opentelemetry/resources": "2.5.1", + "@opentelemetry/sdk-metrics": "2.5.1" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-metrics-otlp-http/node_modules/@opentelemetry/core": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.5.1.tgz", + "integrity": "sha512-Dwlc+3HAZqpgTYq0MUyZABjFkcrKTePwuiFVLjahGD8cx3enqihmpAmdgNFO1R4m/sIe5afjJrA25Prqy4NXlA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-trace-otlp-grpc": { + "version": "0.212.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-trace-otlp-grpc/-/exporter-trace-otlp-grpc-0.212.0.tgz", + "integrity": "sha512-9xTuYWp8ClBhljDGAoa0NSsJcsxJsC9zCFKMSZJp1Osb9pjXCMRdA6fwXtlubyqe7w8FH16EWtQNKx/FWi+Ghw==", + "license": "Apache-2.0", + "dependencies": { + "@grpc/grpc-js": "^1.14.3", + "@opentelemetry/core": "2.5.1", + "@opentelemetry/otlp-exporter-base": "0.212.0", + "@opentelemetry/otlp-grpc-exporter-base": "0.212.0", + "@opentelemetry/otlp-transformer": "0.212.0", + "@opentelemetry/resources": "2.5.1", + "@opentelemetry/sdk-trace-base": "2.5.1" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-trace-otlp-grpc/node_modules/@opentelemetry/core": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.5.1.tgz", + "integrity": "sha512-Dwlc+3HAZqpgTYq0MUyZABjFkcrKTePwuiFVLjahGD8cx3enqihmpAmdgNFO1R4m/sIe5afjJrA25Prqy4NXlA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-trace-otlp-grpc/node_modules/@opentelemetry/sdk-trace-base": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.5.1.tgz", + "integrity": "sha512-iZH3Gw8cxQn0gjpOjJMmKLd9GIaNh/E3v3ST67vyzLSxHBs14HsG4dy7jMYyC5WXGdBVEcM7U/XTF5hCQxjDMw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.5.1", + "@opentelemetry/resources": "2.5.1", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-trace-otlp-http": { + "version": "0.212.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/exporter-trace-otlp-http/-/exporter-trace-otlp-http-0.212.0.tgz", + "integrity": "sha512-v/0wMozNoiEPRolzC4YoPo4rAT0q8r7aqdnRw3Nu7IDN0CGFzNQazkfAlBJ6N5y0FYJkban7Aw5WnN73//6YlA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.5.1", + "@opentelemetry/otlp-exporter-base": "0.212.0", + "@opentelemetry/otlp-transformer": "0.212.0", + "@opentelemetry/resources": "2.5.1", + "@opentelemetry/sdk-trace-base": "2.5.1" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/exporter-trace-otlp-http/node_modules/@opentelemetry/core": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.5.1.tgz", + "integrity": "sha512-Dwlc+3HAZqpgTYq0MUyZABjFkcrKTePwuiFVLjahGD8cx3enqihmpAmdgNFO1R4m/sIe5afjJrA25Prqy4NXlA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/exporter-trace-otlp-http/node_modules/@opentelemetry/sdk-trace-base": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.5.1.tgz", + "integrity": "sha512-iZH3Gw8cxQn0gjpOjJMmKLd9GIaNh/E3v3ST67vyzLSxHBs14HsG4dy7jMYyC5WXGdBVEcM7U/XTF5hCQxjDMw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.5.1", + "@opentelemetry/resources": "2.5.1", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, "node_modules/@opentelemetry/instrumentation": { "version": "0.41.2", "resolved": "https://registry.npmjs.org/@opentelemetry/instrumentation/-/instrumentation-0.41.2.tgz", @@ -4297,29 +4620,239 @@ "@opentelemetry/api": "^1.3.0" } }, - "node_modules/@opentelemetry/resources": { - "version": "1.30.1", - "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-1.30.1.tgz", - "integrity": "sha512-5UxZqiAgLYGFjS4s9qm5mBVo433u+dSPUFWVWXmLAD4wB65oMCoXaJP1KJa9DIYYMeHu3z4BZcStG3LC593cWA==", + "node_modules/@opentelemetry/otlp-exporter-base": { + "version": "0.212.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-exporter-base/-/otlp-exporter-base-0.212.0.tgz", + "integrity": "sha512-HoMv5pQlzbuxiMS0hN7oiUtg8RsJR5T7EhZccumIWxYfNo/f4wFc7LPDfFK6oHdG2JF/+qTocfqIHoom+7kLpw==", "license": "Apache-2.0", "dependencies": { - "@opentelemetry/core": "1.30.1", - "@opentelemetry/semantic-conventions": "1.28.0" + "@opentelemetry/core": "2.5.1", + "@opentelemetry/otlp-transformer": "0.212.0" }, "engines": { - "node": ">=14" + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/otlp-exporter-base/node_modules/@opentelemetry/core": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.5.1.tgz", + "integrity": "sha512-Dwlc+3HAZqpgTYq0MUyZABjFkcrKTePwuiFVLjahGD8cx3enqihmpAmdgNFO1R4m/sIe5afjJrA25Prqy4NXlA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, - "node_modules/@opentelemetry/resources/node_modules/@opentelemetry/semantic-conventions": { - "version": "1.28.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.28.0.tgz", - "integrity": "sha512-lp4qAiMTD4sNWW4DbKLBkfiMZ4jbAboJIGOQr5DvciMRI494OapieI9qiODpOt0XBr1LjIDy1xAGAnVs5supTA==", + "node_modules/@opentelemetry/otlp-grpc-exporter-base": { + "version": "0.212.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-grpc-exporter-base/-/otlp-grpc-exporter-base-0.212.0.tgz", + "integrity": "sha512-YidOSlzpsun9uw0iyIWrQp6HxpMtBlECE3tiHGAsnpEqJWbAUWcMnIffvIuvTtTQ1OyRtwwaE79dWSQ8+eiB7g==", "license": "Apache-2.0", + "dependencies": { + "@grpc/grpc-js": "^1.14.3", + "@opentelemetry/core": "2.5.1", + "@opentelemetry/otlp-exporter-base": "0.212.0", + "@opentelemetry/otlp-transformer": "0.212.0" + }, "engines": { - "node": ">=14" + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/otlp-grpc-exporter-base/node_modules/@opentelemetry/core": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.5.1.tgz", + "integrity": "sha512-Dwlc+3HAZqpgTYq0MUyZABjFkcrKTePwuiFVLjahGD8cx3enqihmpAmdgNFO1R4m/sIe5afjJrA25Prqy4NXlA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/otlp-transformer": { + "version": "0.212.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/otlp-transformer/-/otlp-transformer-0.212.0.tgz", + "integrity": "sha512-bj7zYFOg6Db7NUwsRZQ/WoVXpAf41WY2gsd3kShSfdpZQDRKHWJiRZIg7A8HvWsf97wb05rMFzPbmSHyjEl9tw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.212.0", + "@opentelemetry/core": "2.5.1", + "@opentelemetry/resources": "2.5.1", + "@opentelemetry/sdk-logs": "0.212.0", + "@opentelemetry/sdk-metrics": "2.5.1", + "@opentelemetry/sdk-trace-base": "2.5.1", + "protobufjs": "8.0.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": "^1.3.0" + } + }, + "node_modules/@opentelemetry/otlp-transformer/node_modules/@opentelemetry/core": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.5.1.tgz", + "integrity": "sha512-Dwlc+3HAZqpgTYq0MUyZABjFkcrKTePwuiFVLjahGD8cx3enqihmpAmdgNFO1R4m/sIe5afjJrA25Prqy4NXlA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/otlp-transformer/node_modules/@opentelemetry/sdk-trace-base": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.5.1.tgz", + "integrity": "sha512-iZH3Gw8cxQn0gjpOjJMmKLd9GIaNh/E3v3ST67vyzLSxHBs14HsG4dy7jMYyC5WXGdBVEcM7U/XTF5hCQxjDMw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.5.1", + "@opentelemetry/resources": "2.5.1", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/otlp-transformer/node_modules/protobufjs": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-8.0.0.tgz", + "integrity": "sha512-jx6+sE9h/UryaCZhsJWbJtTEy47yXoGNYI4z8ZaRncM0zBKeRqjO2JEcOUYwrYGb1WLhXM1FfMzW3annvFv0rw==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/node": ">=13.7.0", + "long": "^5.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/@opentelemetry/resources": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-2.5.1.tgz", + "integrity": "sha512-BViBCdE/GuXRlp9k7nS1w6wJvY5fnFX5XvuEtWsTAOQFIO89Eru7lGW3WbfbxtCuZ/GbrJfAziXG0w0dpxL7eQ==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.5.1", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/resources/node_modules/@opentelemetry/core": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.5.1.tgz", + "integrity": "sha512-Dwlc+3HAZqpgTYq0MUyZABjFkcrKTePwuiFVLjahGD8cx3enqihmpAmdgNFO1R4m/sIe5afjJrA25Prqy4NXlA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-logs": { + "version": "0.212.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-logs/-/sdk-logs-0.212.0.tgz", + "integrity": "sha512-qglb5cqTf0mOC1sDdZ7nfrPjgmAqs2OxkzOPIf2+Rqx8yKBK0pS7wRtB1xH30rqahBIut9QJDbDePyvtyqvH/Q==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/api-logs": "0.212.0", + "@opentelemetry/core": "2.5.1", + "@opentelemetry/resources": "2.5.1" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.4.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-logs/node_modules/@opentelemetry/core": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.5.1.tgz", + "integrity": "sha512-Dwlc+3HAZqpgTYq0MUyZABjFkcrKTePwuiFVLjahGD8cx3enqihmpAmdgNFO1R4m/sIe5afjJrA25Prqy4NXlA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-metrics": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-metrics/-/sdk-metrics-2.5.1.tgz", + "integrity": "sha512-RKMn3QKi8nE71ULUo0g/MBvq1N4icEBo7cQSKnL3URZT16/YH3nSVgWegOjwx7FRBTrjOIkMJkCUn/ZFIEfn4A==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.5.1", + "@opentelemetry/resources": "2.5.1" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.9.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-metrics/node_modules/@opentelemetry/core": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.5.1.tgz", + "integrity": "sha512-Dwlc+3HAZqpgTYq0MUyZABjFkcrKTePwuiFVLjahGD8cx3enqihmpAmdgNFO1R4m/sIe5afjJrA25Prqy4NXlA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "node_modules/@opentelemetry/sdk-trace-base": { @@ -4339,6 +4872,22 @@ "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, + "node_modules/@opentelemetry/sdk-trace-base/node_modules/@opentelemetry/resources": { + "version": "1.30.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/resources/-/resources-1.30.1.tgz", + "integrity": "sha512-5UxZqiAgLYGFjS4s9qm5mBVo433u+dSPUFWVWXmLAD4wB65oMCoXaJP1KJa9DIYYMeHu3z4BZcStG3LC593cWA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "1.30.1", + "@opentelemetry/semantic-conventions": "1.28.0" + }, + "engines": { + "node": ">=14" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, "node_modules/@opentelemetry/sdk-trace-base/node_modules/@opentelemetry/semantic-conventions": { "version": "1.28.0", "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.28.0.tgz", @@ -4348,10 +4897,59 @@ "node": ">=14" } }, + "node_modules/@opentelemetry/sdk-trace-node": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-node/-/sdk-trace-node-2.5.1.tgz", + "integrity": "sha512-9lopQ6ZoElETOEN0csgmtEV5/9C7BMfA7VtF4Jape3i954b6sTY2k3Xw3CxUTKreDck/vpAuJM+EDo4zheUw+A==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/context-async-hooks": "2.5.1", + "@opentelemetry/core": "2.5.1", + "@opentelemetry/sdk-trace-base": "2.5.1" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace-node/node_modules/@opentelemetry/core": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/core/-/core-2.5.1.tgz", + "integrity": "sha512-Dwlc+3HAZqpgTYq0MUyZABjFkcrKTePwuiFVLjahGD8cx3enqihmpAmdgNFO1R4m/sIe5afjJrA25Prqy4NXlA==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.0.0 <1.10.0" + } + }, + "node_modules/@opentelemetry/sdk-trace-node/node_modules/@opentelemetry/sdk-trace-base": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/@opentelemetry/sdk-trace-base/-/sdk-trace-base-2.5.1.tgz", + "integrity": "sha512-iZH3Gw8cxQn0gjpOjJMmKLd9GIaNh/E3v3ST67vyzLSxHBs14HsG4dy7jMYyC5WXGdBVEcM7U/XTF5hCQxjDMw==", + "license": "Apache-2.0", + "dependencies": { + "@opentelemetry/core": "2.5.1", + "@opentelemetry/resources": "2.5.1", + "@opentelemetry/semantic-conventions": "^1.29.0" + }, + "engines": { + "node": "^18.19.0 || >=20.6.0" + }, + "peerDependencies": { + "@opentelemetry/api": ">=1.3.0 <1.10.0" + } + }, "node_modules/@opentelemetry/semantic-conventions": { - "version": "1.30.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.30.0.tgz", - "integrity": "sha512-4VlGgo32k2EQ2wcCY3vEU28A0O13aOtHz3Xt2/2U5FAh9EfhD6t6DqL5Z6yAnRCntbTFDU4YfbpyzSlHNWycPw==", + "version": "1.39.0", + "resolved": "https://registry.npmjs.org/@opentelemetry/semantic-conventions/-/semantic-conventions-1.39.0.tgz", + "integrity": "sha512-R5R9tb2AXs2IRLNKLBJDynhkfmx7mX0vi8NkhZb3gUkPWHn6HXk5J8iQ/dql0U3ApfWym4kXXmBDRGO+oeOfjg==", "license": "Apache-2.0", "engines": { "node": ">=14" @@ -4712,6 +5310,70 @@ "node": ">=18" } }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", + "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", + "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", + "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1", + "@protobufjs/inquire": "^1.1.0" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/inquire": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", + "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", + "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==", + "license": "BSD-3-Clause" + }, "node_modules/@rollup/plugin-virtual": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/@rollup/plugin-virtual/-/plugin-virtual-3.0.2.tgz", @@ -5951,7 +6613,6 @@ "version": "22.16.3", "resolved": "https://registry.npmjs.org/@types/node/-/node-22.16.3.tgz", "integrity": "sha512-sr4Xz74KOUeYadexo1r8imhRtlVXcs+j3XK3TcoiYk7B1t3YRVJgtaD3cwX73NYb71pmVuMLNRhJ9XKdoDB74g==", - "dev": true, "license": "MIT", "dependencies": { "undici-types": "~6.21.0" @@ -7539,7 +8200,6 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, "engines": { "node": ">=8" } @@ -7548,7 +8208,6 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, "dependencies": { "color-convert": "^2.0.1" }, @@ -7631,15 +8290,6 @@ "node": ">=18.0.0" } }, - "node_modules/applicationinsights/node_modules/@opentelemetry/api": { - "version": "1.9.0", - "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.0.tgz", - "integrity": "sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==", - "license": "Apache-2.0", - "engines": { - "node": ">=8.0.0" - } - }, "node_modules/are-docs-informative": { "version": "0.0.2", "resolved": "https://registry.npmjs.org/are-docs-informative/-/are-docs-informative-0.0.2.tgz", @@ -8921,7 +9571,6 @@ "version": "8.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", - "dev": true, "dependencies": { "string-width": "^4.2.0", "strip-ansi": "^6.0.1", @@ -8934,14 +9583,12 @@ "node_modules/cliui/node_modules/emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" }, "node_modules/cliui/node_modules/is-fullwidth-code-point": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "dev": true, "engines": { "node": ">=8" } @@ -8950,7 +9597,6 @@ "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", @@ -8964,7 +9610,6 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dev": true, "dependencies": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", @@ -9024,7 +9669,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, "dependencies": { "color-name": "~1.1.4" }, @@ -9035,8 +9679,7 @@ "node_modules/color-name": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, "node_modules/colorette": { "version": "2.0.20", @@ -10255,7 +10898,6 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.1.1.tgz", "integrity": "sha512-k0er2gUkLf8O0zKJiAhmkTnJlTvINGv7ygDNPbeIsX/TJjGJZHuh9B2UxbsaEkmlEo9MfhrSzmhIlhRlI2GXnw==", - "dev": true, "engines": { "node": ">=6" } @@ -11435,7 +12077,6 @@ "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", - "dev": true, "engines": { "node": "6.* || 8.* || >= 10.*" } @@ -13720,6 +14361,12 @@ "dev": true, "license": "MIT" }, + "node_modules/lodash.camelcase": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.camelcase/-/lodash.camelcase-4.3.0.tgz", + "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==", + "license": "MIT" + }, "node_modules/lodash.includes": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", @@ -13936,6 +14583,12 @@ "url": "https://github.com/chalk/wrap-ansi?sponsor=1" } }, + "node_modules/long": { + "version": "5.3.2", + "resolved": "https://registry.npmjs.org/long/-/long-5.3.2.tgz", + "integrity": "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==", + "license": "Apache-2.0" + }, "node_modules/loose-envify": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", @@ -16422,6 +17075,30 @@ "dev": true, "license": "MIT" }, + "node_modules/protobufjs": { + "version": "7.5.4", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-7.5.4.tgz", + "integrity": "sha512-CvexbZtbov6jW2eXAvLukXjXUW1TzFaivC46BpWc/3BpcCysb5Vffu+B3XHMm8lVEuy2Mm4XGex8hBSg1yapPg==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/node": ">=13.7.0", + "long": "^5.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/proxy-addr": { "version": "2.0.7", "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", @@ -16816,7 +17493,6 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", - "dev": true, "engines": { "node": ">=0.10.0" } @@ -18265,7 +18941,6 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, "dependencies": { "ansi-regex": "^5.0.1" }, @@ -19262,7 +19937,6 @@ "version": "6.21.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", - "dev": true, "license": "MIT" }, "node_modules/unicorn-magic": { @@ -20099,7 +20773,6 @@ "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", - "dev": true, "engines": { "node": ">=10" } @@ -20127,7 +20800,6 @@ "version": "17.7.2", "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.7.2.tgz", "integrity": "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==", - "dev": true, "dependencies": { "cliui": "^8.0.1", "escalade": "^3.1.1", @@ -20145,7 +20817,6 @@ "version": "21.1.1", "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", - "dev": true, "license": "ISC", "engines": { "node": ">=12" @@ -20181,14 +20852,12 @@ "node_modules/yargs/node_modules/emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" }, "node_modules/yargs/node_modules/is-fullwidth-code-point": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "dev": true, "engines": { "node": ">=8" } @@ -20197,7 +20866,6 @@ "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", diff --git a/extensions/copilot/package.json b/extensions/copilot/package.json index 4a189074a40..1c83c4bec6b 100644 --- a/extensions/copilot/package.json +++ b/extensions/copilot/package.json @@ -4420,6 +4420,47 @@ "experimental", "onExp" ] + }, + "github.copilot.chat.otel.enabled": { + "type": "boolean", + "default": false, + "markdownDescription": "Enable OpenTelemetry trace/metric/log emission for Copilot Chat operations. Env var `COPILOT_OTEL_ENABLED` takes precedence.", + "tags": [ + "advanced" + ] + }, + "github.copilot.chat.otel.exporterType": { + "type": "string", + "enum": ["otlp-grpc", "otlp-http", "console", "file"], + "default": "otlp-http", + "markdownDescription": "OTel exporter type for Copilot Chat telemetry.", + "tags": [ + "advanced" + ] + }, + "github.copilot.chat.otel.otlpEndpoint": { + "type": "string", + "default": "http://localhost:4318", + "markdownDescription": "OTLP collector endpoint URL for Copilot Chat OTel data. Env var `OTEL_EXPORTER_OTLP_ENDPOINT` takes precedence.", + "tags": [ + "advanced" + ] + }, + "github.copilot.chat.otel.captureContent": { + "type": "boolean", + "default": false, + "markdownDescription": "Capture input/output messages, system instructions, and tool definitions in OTel telemetry. **Contains potentially sensitive data.** Env var `COPILOT_OTEL_CAPTURE_CONTENT` takes precedence.", + "tags": [ + "advanced" + ] + }, + "github.copilot.chat.otel.outfile": { + "type": "string", + "default": "", + "markdownDescription": "File path for file-based OTel exporter output (JSON-lines). When set, overrides exporter type to `file`.", + "tags": [ + "advanced" + ] } } } @@ -5924,6 +5965,19 @@ "@humanwhocodes/gitignore-to-minimatch": "1.0.2", "@microsoft/tiktokenizer": "^1.0.10", "@modelcontextprotocol/sdk": "^1.25.2", + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/api-logs": "^0.212.0", + "@opentelemetry/exporter-logs-otlp-grpc": "^0.212.0", + "@opentelemetry/exporter-logs-otlp-http": "^0.212.0", + "@opentelemetry/exporter-metrics-otlp-grpc": "^0.212.0", + "@opentelemetry/exporter-metrics-otlp-http": "^0.212.0", + "@opentelemetry/exporter-trace-otlp-grpc": "^0.212.0", + "@opentelemetry/exporter-trace-otlp-http": "^0.212.0", + "@opentelemetry/resources": "^2.5.1", + "@opentelemetry/sdk-logs": "^0.212.0", + "@opentelemetry/sdk-metrics": "^2.5.1", + "@opentelemetry/sdk-trace-node": "^2.5.1", + "@opentelemetry/semantic-conventions": "^1.39.0", "@sinclair/typebox": "^0.34.41", "@vscode/copilot-api": "^0.2.13", "@vscode/extension-telemetry": "^1.5.1", diff --git a/extensions/copilot/src/extension/byok/vscode-node/anthropicProvider.ts b/extensions/copilot/src/extension/byok/vscode-node/anthropicProvider.ts index f28f58c8f1d..6191de750bd 100644 --- a/extensions/copilot/src/extension/byok/vscode-node/anthropicProvider.ts +++ b/extensions/copilot/src/extension/byok/vscode-node/anthropicProvider.ts @@ -13,6 +13,8 @@ import { ILogService } from '../../../platform/log/common/logService'; import { ContextManagementResponse, getContextManagementFromConfig, isAnthropicContextEditingEnabled, isAnthropicMemoryToolEnabled, isAnthropicToolSearchEnabled, nonDeferredToolNames, TOOL_SEARCH_TOOL_NAME, TOOL_SEARCH_TOOL_TYPE, ToolSearchToolResult, ToolSearchToolSearchResult } from '../../../platform/networking/common/anthropic'; import { IResponseDelta, OpenAiFunctionTool } from '../../../platform/networking/common/fetch'; import { APIUsage } from '../../../platform/networking/common/openai'; +import { CopilotChatAttr, emitInferenceDetailsEvent, GenAiAttr, GenAiMetrics, GenAiOperationName, type OTelModelOptions, StdAttr, truncateForOTel } from '../../../platform/otel/common/index'; +import { IOTelService, SpanKind, SpanStatusCode } from '../../../platform/otel/common/otelService'; import { IRequestLogger, retrieveCapturingTokenByCorrelation, runWithCapturingToken } from '../../../platform/requestLogger/node/requestLogger'; import { IExperimentationService } from '../../../platform/telemetry/common/nullExperimentationService'; import { ITelemetryService } from '../../../platform/telemetry/common/telemetry'; @@ -35,7 +37,8 @@ export class AnthropicLMProvider extends AbstractLanguageModelChatProvider { @IRequestLogger private readonly _requestLogger: IRequestLogger, @IConfigurationService private readonly _configurationService: IConfigurationService, @IExperimentationService private readonly _experimentationService: IExperimentationService, - @ITelemetryService private readonly _telemetryService: ITelemetryService + @ITelemetryService private readonly _telemetryService: ITelemetryService, + @IOTelService private readonly _otelService: IOTelService, ) { super(AnthropicLMProvider.providerName.toLowerCase(), AnthropicLMProvider.providerName, knownModels, byokStorageService, logService); @@ -90,9 +93,15 @@ export class AnthropicLMProvider extends AbstractLanguageModelChatProvider { async provideLanguageModelChatResponse(model: ExtendedLanguageModelChatInformation, messages: Array, options: ProvideLanguageModelChatResponseOptions, progress: Progress, token: CancellationToken): Promise { // Restore CapturingToken context if correlation ID was passed through modelOptions. // This handles the case where AsyncLocalStorage context was lost crossing VS Code IPC. - const correlationId = (options as { modelOptions?: { _capturingTokenCorrelationId?: string } }).modelOptions?._capturingTokenCorrelationId; + const correlationId = (options as { modelOptions?: OTelModelOptions }).modelOptions?._capturingTokenCorrelationId; const capturingToken = correlationId ? retrieveCapturingTokenByCorrelation(correlationId) : undefined; + // Restore OTel trace context to link spans back to the agent trace + const parentTraceContext = (options as { modelOptions?: OTelModelOptions }).modelOptions?._otelTraceContext ?? undefined; + + // OTel span handle — created outside doRequest, enriched inside with usage data + let otelSpan: ReturnType | undefined; + const doRequest = async () => { const issuedTime = Date.now(); const apiKey = model.configuration?.apiKey; @@ -314,6 +323,61 @@ export class AnthropicLMProvider extends AbstractLanguageModelChatProvider { resolvedModel: model.id }, responseDeltas); + // Enrich OTel span with usage data from the Anthropic response + if (otelSpan && result.usage) { + otelSpan.setAttributes({ + [GenAiAttr.USAGE_INPUT_TOKENS]: result.usage.prompt_tokens ?? 0, + [GenAiAttr.USAGE_OUTPUT_TOKENS]: result.usage.completion_tokens ?? 0, + ...(result.usage.prompt_tokens_details?.cached_tokens + ? { [GenAiAttr.USAGE_CACHE_READ_INPUT_TOKENS]: result.usage.prompt_tokens_details.cached_tokens } + : {}), + [GenAiAttr.RESPONSE_MODEL]: model.id, + [GenAiAttr.RESPONSE_ID]: requestId, + [GenAiAttr.RESPONSE_FINISH_REASONS]: ['stop'], + [GenAiAttr.CONVERSATION_ID]: requestId, + ...(result.ttft ? { [CopilotChatAttr.TIME_TO_FIRST_TOKEN]: result.ttft } : {}), + [GenAiAttr.REQUEST_MAX_TOKENS]: model.maxOutputTokens ?? 0, + }); + // Opt-in content capture + if (this._otelService.config.captureContent) { + const responseText = wrappedProgress.items + .filter((p): p is LanguageModelTextPart => p instanceof LanguageModelTextPart) + .map(p => p.value).join(''); + const toolCalls = wrappedProgress.items + .filter((p): p is LanguageModelToolCallPart => p instanceof LanguageModelToolCallPart) + .map(tc => ({ type: 'tool_call' as const, id: tc.callId, name: tc.name, arguments: tc.input })); + const parts: Array<{ type: string; content?: string; id?: string; name?: string; arguments?: unknown }> = []; + if (responseText) { parts.push({ type: 'text', content: responseText }); } + parts.push(...toolCalls); + if (parts.length > 0) { + otelSpan.setAttribute(GenAiAttr.OUTPUT_MESSAGES, truncateForOTel(JSON.stringify([{ role: 'assistant', parts }]))); + } + } + } + + // Record OTel metrics for this Anthropic LLM call + if (result.usage) { + const durationSec = (Date.now() - issuedTime) / 1000; + const metricAttrs = { operationName: GenAiOperationName.CHAT, providerName: 'anthropic', requestModel: model.id, responseModel: model.id }; + GenAiMetrics.recordOperationDuration(this._otelService, durationSec, metricAttrs); + if (result.usage.prompt_tokens) { GenAiMetrics.recordTokenUsage(this._otelService, result.usage.prompt_tokens, 'input', metricAttrs); } + if (result.usage.completion_tokens) { GenAiMetrics.recordTokenUsage(this._otelService, result.usage.completion_tokens, 'output', metricAttrs); } + if (result.ttft) { GenAiMetrics.recordTimeToFirstToken(this._otelService, model.id, result.ttft / 1000); } + } + + // Emit OTel inference details event + emitInferenceDetailsEvent( + this._otelService, + { model: model.id, maxTokens: model.maxOutputTokens }, + result.usage ? { + id: requestId, + model: model.id, + finishReasons: ['stop'], + inputTokens: result.usage.prompt_tokens, + outputTokens: result.usage.completion_tokens, + } : undefined, + ); + // Send success telemetry matching response.success format /* __GDPR__ "response.success" : { @@ -410,11 +474,56 @@ export class AnthropicLMProvider extends AbstractLanguageModelChatProvider { } }; - // Execute with restored CapturingToken context if available - if (capturingToken) { - return runWithCapturingToken(capturingToken, doRequest); + // Create OTel span and execute with trace context + CapturingToken + const executeRequest = async () => { + otelSpan = this._otelService.startSpan(`chat ${model.id}`, { + kind: SpanKind.CLIENT, + attributes: { + [GenAiAttr.OPERATION_NAME]: GenAiOperationName.CHAT, + [GenAiAttr.PROVIDER_NAME]: 'anthropic', + [GenAiAttr.REQUEST_MODEL]: model.id, + [GenAiAttr.AGENT_NAME]: 'AnthropicBYOK', + [CopilotChatAttr.MAX_PROMPT_TOKENS]: model.maxInputTokens, + [StdAttr.SERVER_ADDRESS]: 'api.anthropic.com', + }, + }); + // Opt-in: capture input messages + if (this._otelService.config.captureContent) { + try { + const roleNames: Record = { 1: 'user', 2: 'assistant', 3: 'system' }; + const inputMsgs = messages.map(m => { + const msg = m as LanguageModelChatMessage; + const role = roleNames[msg.role] ?? String(msg.role); + const textParts: string[] = []; + if (Array.isArray(msg.content)) { + for (const p of msg.content) { + if (p instanceof LanguageModelTextPart) { textParts.push(p.value); } + } + } + const content = textParts.length > 0 ? textParts.join('') : '[non-text content]'; + return { role, parts: [{ type: 'text', content }] }; + }); + otelSpan.setAttribute(GenAiAttr.INPUT_MESSAGES, truncateForOTel(JSON.stringify(inputMsgs))); + } catch { /* swallow */ } + } + try { + const result = capturingToken + ? await runWithCapturingToken(capturingToken, doRequest) + : await doRequest(); + otelSpan.setStatus(SpanStatusCode.OK); + return result; + } catch (err) { + otelSpan.setStatus(SpanStatusCode.ERROR, err instanceof Error ? err.message : String(err)); + throw err; + } finally { + otelSpan.end(); + } + }; + + if (parentTraceContext) { + return this._otelService.runWithTraceContext(parentTraceContext, executeRequest); } - return doRequest(); + return executeRequest(); } async provideTokenCount(model: LanguageModelChatInformation, text: string | LanguageModelChatMessage | LanguageModelChatMessage2, token: CancellationToken): Promise { diff --git a/extensions/copilot/src/extension/byok/vscode-node/geminiNativeProvider.ts b/extensions/copilot/src/extension/byok/vscode-node/geminiNativeProvider.ts index db364d818a1..6afba73d80d 100644 --- a/extensions/copilot/src/extension/byok/vscode-node/geminiNativeProvider.ts +++ b/extensions/copilot/src/extension/byok/vscode-node/geminiNativeProvider.ts @@ -9,6 +9,8 @@ import { ChatFetchResponseType, ChatLocation } from '../../../platform/chat/comm import { ILogService } from '../../../platform/log/common/logService'; import { IResponseDelta, OpenAiFunctionTool } from '../../../platform/networking/common/fetch'; import { APIUsage } from '../../../platform/networking/common/openai'; +import { CopilotChatAttr, emitInferenceDetailsEvent, GenAiAttr, GenAiMetrics, GenAiOperationName, type OTelModelOptions, StdAttr, truncateForOTel } from '../../../platform/otel/common/index'; +import { IOTelService, SpanKind, SpanStatusCode } from '../../../platform/otel/common/otelService'; import { IRequestLogger, retrieveCapturingTokenByCorrelation, runWithCapturingToken } from '../../../platform/requestLogger/node/requestLogger'; import { ITelemetryService } from '../../../platform/telemetry/common/telemetry'; import { toErrorMessage } from '../../../util/common/errorMessage'; @@ -29,7 +31,8 @@ export class GeminiNativeBYOKLMProvider extends AbstractLanguageModelChatProvide byokStorageService: IBYOKStorageService, @ILogService logService: ILogService, @IRequestLogger private readonly _requestLogger: IRequestLogger, - @ITelemetryService private readonly _telemetryService: ITelemetryService + @ITelemetryService private readonly _telemetryService: ITelemetryService, + @IOTelService private readonly _otelService: IOTelService, ) { super(GeminiNativeBYOKLMProvider.providerName.toLowerCase(), GeminiNativeBYOKLMProvider.providerName, knownModels, byokStorageService, logService); } @@ -73,9 +76,15 @@ export class GeminiNativeBYOKLMProvider extends AbstractLanguageModelChatProvide async provideLanguageModelChatResponse(model: ExtendedLanguageModelChatInformation, messages: Array, options: ProvideLanguageModelChatResponseOptions, progress: Progress, token: CancellationToken): Promise { // Restore CapturingToken context if correlation ID was passed through modelOptions. // This handles the case where AsyncLocalStorage context was lost crossing VS Code IPC. - const correlationId = (options as { modelOptions?: { _capturingTokenCorrelationId?: string } }).modelOptions?._capturingTokenCorrelationId; + const correlationId = (options as { modelOptions?: OTelModelOptions }).modelOptions?._capturingTokenCorrelationId; const capturingToken = correlationId ? retrieveCapturingTokenByCorrelation(correlationId) : undefined; + // Restore OTel trace context to link spans back to the agent trace + const parentTraceContext = (options as { modelOptions?: OTelModelOptions }).modelOptions?._otelTraceContext ?? undefined; + + // OTel span handle — created outside doRequest, enriched inside with usage data + let otelSpan: ReturnType | undefined; + const doRequest = async () => { const issuedTime = Date.now(); const apiKey = model.configuration?.apiKey; @@ -180,6 +189,61 @@ export class GeminiNativeBYOKLMProvider extends AbstractLanguageModelChatProvide }; })); + // Enrich OTel span with usage data from the Gemini response + if (otelSpan && result.usage) { + otelSpan.setAttributes({ + [GenAiAttr.USAGE_INPUT_TOKENS]: result.usage.prompt_tokens ?? 0, + [GenAiAttr.USAGE_OUTPUT_TOKENS]: result.usage.completion_tokens ?? 0, + ...(result.usage.prompt_tokens_details?.cached_tokens + ? { [GenAiAttr.USAGE_CACHE_READ_INPUT_TOKENS]: result.usage.prompt_tokens_details.cached_tokens } + : {}), + [GenAiAttr.RESPONSE_MODEL]: model.id, + [GenAiAttr.RESPONSE_ID]: requestId, + [GenAiAttr.RESPONSE_FINISH_REASONS]: ['stop'], + [GenAiAttr.CONVERSATION_ID]: requestId, + ...(result.ttft ? { [CopilotChatAttr.TIME_TO_FIRST_TOKEN]: result.ttft } : {}), + [GenAiAttr.REQUEST_MAX_TOKENS]: model.maxOutputTokens ?? 0, + }); + // Opt-in content capture + if (this._otelService.config.captureContent) { + const responseText = wrappedProgress.items + .filter((p): p is LanguageModelTextPart => p instanceof LanguageModelTextPart) + .map(p => p.value).join(''); + const toolCalls = wrappedProgress.items + .filter((p): p is LanguageModelToolCallPart => p instanceof LanguageModelToolCallPart) + .map(tc => ({ type: 'tool_call' as const, id: tc.callId, name: tc.name, arguments: tc.input })); + const parts: Array<{ type: string; content?: string; id?: string; name?: string; arguments?: unknown }> = []; + if (responseText) { parts.push({ type: 'text', content: responseText }); } + parts.push(...toolCalls); + if (parts.length > 0) { + otelSpan.setAttribute(GenAiAttr.OUTPUT_MESSAGES, truncateForOTel(JSON.stringify([{ role: 'assistant', parts }]))); + } + } + } + + // Record OTel metrics for this Gemini LLM call + if (result.usage) { + const durationSec = (Date.now() - issuedTime) / 1000; + const metricAttrs = { operationName: GenAiOperationName.CHAT, providerName: 'gemini', requestModel: model.id, responseModel: model.id }; + GenAiMetrics.recordOperationDuration(this._otelService, durationSec, metricAttrs); + if (result.usage.prompt_tokens) { GenAiMetrics.recordTokenUsage(this._otelService, result.usage.prompt_tokens, 'input', metricAttrs); } + if (result.usage.completion_tokens) { GenAiMetrics.recordTokenUsage(this._otelService, result.usage.completion_tokens, 'output', metricAttrs); } + if (result.ttft) { GenAiMetrics.recordTimeToFirstToken(this._otelService, model.id, result.ttft / 1000); } + } + + // Emit OTel inference details event + emitInferenceDetailsEvent( + this._otelService, + { model: model.id, maxTokens: model.maxOutputTokens }, + result.usage ? { + id: requestId, + model: model.id, + finishReasons: ['stop'], + inputTokens: result.usage.prompt_tokens, + outputTokens: result.usage.completion_tokens, + } : undefined, + ); + // Send success telemetry matching response.success format /* __GDPR__ "response.success" : { @@ -266,11 +330,56 @@ export class GeminiNativeBYOKLMProvider extends AbstractLanguageModelChatProvide } }; - // Execute with restored CapturingToken context if available - if (capturingToken) { - return runWithCapturingToken(capturingToken, doRequest); + // Create OTel span and execute with trace context + CapturingToken + const executeRequest = async () => { + otelSpan = this._otelService.startSpan(`chat ${model.id}`, { + kind: SpanKind.CLIENT, + attributes: { + [GenAiAttr.OPERATION_NAME]: GenAiOperationName.CHAT, + [GenAiAttr.PROVIDER_NAME]: 'gemini', + [GenAiAttr.REQUEST_MODEL]: model.id, + [GenAiAttr.AGENT_NAME]: 'GeminiBYOK', + [CopilotChatAttr.MAX_PROMPT_TOKENS]: model.maxInputTokens, + [StdAttr.SERVER_ADDRESS]: 'generativelanguage.googleapis.com', + }, + }); + // Opt-in: capture input messages + if (this._otelService.config.captureContent) { + try { + const roleNames: Record = { 1: 'user', 2: 'assistant', 3: 'system' }; + const inputMsgs = messages.map(m => { + const msg = m as LanguageModelChatMessage; + const role = roleNames[msg.role] ?? String(msg.role); + const textParts: string[] = []; + if (Array.isArray(msg.content)) { + for (const p of msg.content) { + if (p instanceof LanguageModelTextPart) { textParts.push(p.value); } + } + } + const content = textParts.length > 0 ? textParts.join('') : '[non-text content]'; + return { role, parts: [{ type: 'text', content }] }; + }); + otelSpan.setAttribute(GenAiAttr.INPUT_MESSAGES, truncateForOTel(JSON.stringify(inputMsgs))); + } catch { /* swallow */ } + } + try { + const result = capturingToken + ? await runWithCapturingToken(capturingToken, doRequest) + : await doRequest(); + otelSpan.setStatus(SpanStatusCode.OK); + return result; + } catch (err) { + otelSpan.setStatus(SpanStatusCode.ERROR, err instanceof Error ? err.message : String(err)); + throw err; + } finally { + otelSpan.end(); + } + }; + + if (parentTraceContext) { + return this._otelService.runWithTraceContext(parentTraceContext, executeRequest); } - return doRequest(); + return executeRequest(); } async provideTokenCount(model: LanguageModelChatInformation, text: string | LanguageModelChatMessage | LanguageModelChatMessage2, token: CancellationToken): Promise { diff --git a/extensions/copilot/src/extension/byok/vscode-node/test/geminiNativeProvider.spec.ts b/extensions/copilot/src/extension/byok/vscode-node/test/geminiNativeProvider.spec.ts index 9f943a24c5f..71ee2bd1de2 100644 --- a/extensions/copilot/src/extension/byok/vscode-node/test/geminiNativeProvider.spec.ts +++ b/extensions/copilot/src/extension/byok/vscode-node/test/geminiNativeProvider.spec.ts @@ -5,6 +5,7 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; import * as vscode from 'vscode'; +import { NoopOTelService, resolveOTelConfig } from '../../../../platform/otel/common/index'; import type { CapturingToken } from '../../../../platform/requestLogger/common/capturingToken'; import type { IRequestLogger } from '../../../../platform/requestLogger/node/requestLogger'; import { NullTelemetryService } from '../../../../platform/telemetry/common/nullTelemetryService'; @@ -104,7 +105,7 @@ describe('GeminiNativeBYOKLMProvider', () => { it.skip('throws a clear error when no API key is configured (no silent return)', async () => { const { GeminiNativeBYOKLMProvider } = await import('../geminiNativeProvider'); const storage = createStorageService({ getAPIKey: vi.fn().mockResolvedValue(undefined) }); - const provider = new GeminiNativeBYOKLMProvider(undefined, storage, new TestLogService(), createRequestLogger(), new NullTelemetryService()); + const provider = new GeminiNativeBYOKLMProvider(undefined, storage, new TestLogService(), createRequestLogger(), new NullTelemetryService(), new NoopOTelService(resolveOTelConfig({ env: {}, extensionVersion: '1.0.0', sessionId: 'test' }))); const model: vscode.LanguageModelChatInformation = { id: 'gemini-2.0-flash', @@ -234,7 +235,7 @@ describe('GeminiNativeBYOKLMProvider', () => { mockHandleAPIKeyUpdate.mockResolvedValue({ apiKey: undefined, deleted: false, cancelled: true }); - const provider = new GeminiNativeBYOKLMProvider(undefined, storage, new TestLogService(), createRequestLogger(), new NullTelemetryService()); + const provider = new GeminiNativeBYOKLMProvider(undefined, storage, new TestLogService(), createRequestLogger(), new NullTelemetryService(), new NoopOTelService(resolveOTelConfig({ env: {}, extensionVersion: '1.0.0', sessionId: 'test' }))); const tokenSource = new vscode.CancellationTokenSource(); const models = await provider.provideLanguageModelChatInformation({ silent: false }, tokenSource.token); @@ -280,7 +281,7 @@ describe('GeminiNativeBYOKLMProvider', () => { } }; - const provider = new GeminiNativeBYOKLMProvider(knownModels, storage, new TestLogService(), createRequestLogger(), new NullTelemetryService()); + const provider = new GeminiNativeBYOKLMProvider(knownModels, storage, new TestLogService(), createRequestLogger(), new NullTelemetryService(), new NoopOTelService(resolveOTelConfig({ env: {}, extensionVersion: '1.0.0', sessionId: 'test' }))); const tokenSource = new vscode.CancellationTokenSource(); const models = await provider.provideLanguageModelChatInformation({ silent: false }, tokenSource.token); diff --git a/extensions/copilot/src/extension/conversation/vscode-node/languageModelAccess.ts b/extensions/copilot/src/extension/conversation/vscode-node/languageModelAccess.ts index 9579d933ff5..9578baa4585 100644 --- a/extensions/copilot/src/extension/conversation/vscode-node/languageModelAccess.ts +++ b/extensions/copilot/src/extension/conversation/vscode-node/languageModelAccess.ts @@ -25,6 +25,7 @@ import { ILogService } from '../../../platform/log/common/logService'; import { isAnthropicToolSearchEnabled } from '../../../platform/networking/common/anthropic'; import { FinishedCallback, OpenAiFunctionTool, OptionalChatRequestParams } from '../../../platform/networking/common/fetch'; import { IChatEndpoint, IEndpoint } from '../../../platform/networking/common/networking'; +import { IOTelService, type OTelModelOptions } from '../../../platform/otel/common/otelService'; import { retrieveCapturingTokenByCorrelation, runWithCapturingToken } from '../../../platform/requestLogger/node/requestLogger'; import { IExperimentationService } from '../../../platform/telemetry/common/nullExperimentationService'; import { ITelemetryService } from '../../../platform/telemetry/common/telemetry'; @@ -434,6 +435,7 @@ export class CopilotLanguageModelWrapper extends Disposable { @IAuthenticationService private readonly _authenticationService: IAuthenticationService, @IEnvService private readonly _envService: IEnvService, @IConfigurationService private readonly _configurationService: IConfigurationService, + @IOTelService private readonly _otelService: IOTelService, ) { super(); } @@ -544,14 +546,27 @@ export class CopilotLanguageModelWrapper extends Disposable { // Restore CapturingToken context if correlation ID was passed through modelOptions. // This handles BYOK providers where the original AsyncLocalStorage context was lost // when crossing the VS Code IPC boundary. - const correlationId = (_options as { modelOptions?: { _capturingTokenCorrelationId?: string } }).modelOptions?._capturingTokenCorrelationId; + const correlationId = (_options as { modelOptions?: OTelModelOptions }).modelOptions?._capturingTokenCorrelationId; const capturingToken = correlationId ? retrieveCapturingTokenByCorrelation(correlationId) : undefined; + // Restore OTel trace context if passed through modelOptions. + // This links the wrapper's chat span back to the original invoke_agent trace. + const parentTraceContext = (_options as { modelOptions?: OTelModelOptions }).modelOptions?._otelTraceContext ?? undefined; + const makeRequest = () => endpoint.makeChatRequest('copilotLanguageModelWrapper', messages, callback, token, ChatLocation.Other, { extensionId }, options, extensionId !== 'core', telemetryProperties); - const result = capturingToken - ? await runWithCapturingToken(capturingToken, makeRequest) - : await makeRequest(); + // Run request within the parent OTel context (no extra span) so chat spans in chatMLFetcher inherit the agent trace + const wrappedRequest = parentTraceContext + ? () => this._otelService.runWithTraceContext(parentTraceContext, async () => { + return capturingToken + ? await runWithCapturingToken(capturingToken, makeRequest) + : await makeRequest(); + }) + : () => capturingToken + ? runWithCapturingToken(capturingToken, makeRequest) + : makeRequest(); + + const result = await wrappedRequest(); if (result.type !== ChatFetchResponseType.Success) { if (result.type === ChatFetchResponseType.ExtensionBlocked) { diff --git a/extensions/copilot/src/extension/extension/vscode-node/contributions.ts b/extensions/copilot/src/extension/extension/vscode-node/contributions.ts index f70e10ef51f..688885d895e 100644 --- a/extensions/copilot/src/extension/extension/vscode-node/contributions.ts +++ b/extensions/copilot/src/extension/extension/vscode-node/contributions.ts @@ -37,6 +37,7 @@ import { McpSetupCommands } from '../../mcp/vscode-node/commands'; import { NotebookFollowCommands } from '../../notebook/vscode-node/followActions'; import { CopilotDebugCommandContribution } from '../../onboardDebug/vscode-node/copilotDebugCommandContribution'; import { OnboardTerminalTestsContribution } from '../../onboardDebug/vscode-node/onboardTerminalTestsContribution'; +import { OTelContrib } from '../../otel/vscode-node/otelContrib'; import { PowerStateLogger } from '../../power/vscode-node/powerStateLogger'; import { DebugCommandsContribution } from '../../prompt/vscode-node/debugCommands'; import { RenameSuggestionsContrib } from '../../prompt/vscode-node/renameSuggestions'; @@ -101,6 +102,7 @@ export const vscodeNodeContributions: IExtensionContributionFactory[] = [ workspaceIndexingContribution, asContributionFactory(ChatSessionsContrib), asContributionFactory(GitHubMcpContrib), + asContributionFactory(OTelContrib), ]; /** diff --git a/extensions/copilot/src/extension/extension/vscode-node/services.ts b/extensions/copilot/src/extension/extension/vscode-node/services.ts index 468eba9ea10..0767e19cf28 100644 --- a/extensions/copilot/src/extension/extension/vscode-node/services.ts +++ b/extensions/copilot/src/extension/extension/vscode-node/services.ts @@ -3,7 +3,7 @@ * Licensed under the MIT License. See License.txt in the project root for license information. *--------------------------------------------------------------------------------------------*/ -import { ExtensionContext, ExtensionMode, env } from 'vscode'; +import { ExtensionContext, ExtensionMode, env, workspace } from 'vscode'; import { IAuthenticationService } from '../../../platform/authentication/common/authentication'; import { ICopilotTokenManager } from '../../../platform/authentication/common/copilotTokenManager'; import { StaticGitHubAuthenticationService } from '../../../platform/authentication/common/staticGitHubAuthenticationService'; @@ -46,8 +46,11 @@ import { ILanguageContextService } from '../../../platform/languageServer/common import { ICompletionsFetchService } from '../../../platform/nesFetch/common/completionsFetchService'; import { CompletionsFetchService } from '../../../platform/nesFetch/node/completionsFetchServiceImpl'; import { IFetcherService } from '../../../platform/networking/common/fetcherService'; -import { IChatWebSocketManager, ChatWebSocketManager } from '../../../platform/networking/node/chatWebSocketManager'; +import { ChatWebSocketManager, IChatWebSocketManager } from '../../../platform/networking/node/chatWebSocketManager'; import { FetcherService } from '../../../platform/networking/vscode-node/fetcherServiceImpl'; +import { NoopOTelService } from '../../../platform/otel/common/noopOtelService'; +import { resolveOTelConfig } from '../../../platform/otel/common/otelConfig'; +import { IOTelService } from '../../../platform/otel/common/otelService'; import { IParserService } from '../../../platform/parser/node/parserService'; import { ParserServiceImpl } from '../../../platform/parser/node/parserServiceImpl'; import { IProxyModelsService } from '../../../platform/proxyModels/common/proxyModelsService'; @@ -253,6 +256,33 @@ export function registerServices(builder: IInstantiationServiceBuilder, extensio builder.define(ITrajectoryLogger, new SyncDescriptor(TrajectoryLogger)); builder.define(IAgentDebugEventService, new SyncDescriptor(AgentDebugEventServiceImpl)); builder.define(IToolResultContentRenderer, new SyncDescriptor(ToolResultContentRenderer)); + + // OTel service — resolve config from env + settings, create appropriate impl + const otelSettings = workspace.getConfiguration('github.copilot.chat.otel'); + const otelConfig = resolveOTelConfig({ + env: process.env, + settingEnabled: otelSettings.get('enabled'), + settingExporterType: otelSettings.get<'otlp-grpc' | 'otlp-http' | 'console' | 'file'>('exporterType'), + settingOtlpEndpoint: otelSettings.get('otlpEndpoint'), + settingCaptureContent: otelSettings.get('captureContent'), + settingOutfile: otelSettings.get('outfile') || undefined, + extensionVersion: extensionContext.extension.packageJSON.version ?? '0.0.0', + sessionId: env.sessionId, + }); + if (otelConfig.enabled) { + // Dynamic import to avoid loading OTel SDK when disabled + const { NodeOTelService } = require('../../../platform/otel/node/otelServiceImpl') as typeof import('../../../platform/otel/node/otelServiceImpl'); + // Log callback routes OTel messages to the extension output channel (via ILogService wired in OTelContrib) + // During early init before ILogService is available, messages go to console as fallback + const logFn: import('../../../platform/otel/node/otelServiceImpl').OTelLogFn = (level, msg) => { + if (level === 'error') { console.error(msg); } + else if (level === 'warn') { console.warn(msg); } + else { console.info(msg); } + }; + builder.define(IOTelService, new NodeOTelService(otelConfig, logFn)); + } else { + builder.define(IOTelService, new NoopOTelService(otelConfig)); + } } function setupMSFTExperimentationService(builder: IInstantiationServiceBuilder, extensionContext: ExtensionContext) { diff --git a/extensions/copilot/src/extension/intents/node/toolCallingLoop.ts b/extensions/copilot/src/extension/intents/node/toolCallingLoop.ts index 23962be7494..1f5170d01d8 100644 --- a/extensions/copilot/src/extension/intents/node/toolCallingLoop.ts +++ b/extensions/copilot/src/extension/intents/node/toolCallingLoop.ts @@ -19,6 +19,8 @@ import { ILogService } from '../../../platform/log/common/logService'; import { isOpenAIContextManagementResponse, OpenAiFunctionDef } from '../../../platform/networking/common/fetch'; import { IMakeChatRequestOptions } from '../../../platform/networking/common/networking'; import { OpenAIContextManagementResponse } from '../../../platform/networking/common/openai'; +import { CopilotChatAttr, emitAgentTurnEvent, emitSessionStartEvent, GenAiAttr, GenAiMetrics, GenAiOperationName, GenAiProviderName, StdAttr, truncateForOTel } from '../../../platform/otel/common/index'; +import { IOTelService, SpanKind, SpanStatusCode } from '../../../platform/otel/common/otelService'; import { IRequestLogger } from '../../../platform/requestLogger/node/requestLogger'; import { IExperimentationService } from '../../../platform/telemetry/common/nullExperimentationService'; import { ITelemetryService } from '../../../platform/telemetry/common/telemetry'; @@ -194,6 +196,7 @@ export abstract class ToolCallingLoop { + const agentName = (this.options.request as { participant?: string }).participant ?? 'GitHub Copilot Chat'; + + // If this is a subagent request, look up the parent trace context stored by the parent agent's execute_tool span + const parentRequestId = (this.options.request as { parentRequestId?: string }).parentRequestId; + const parentTraceContext = parentRequestId + ? this._otelService.getStoredTraceContext(`subagent:${parentRequestId}`) + : undefined; + + return this._otelService.startActiveSpan( + `invoke_agent ${agentName}`, + { + kind: SpanKind.INTERNAL, + attributes: { + [GenAiAttr.OPERATION_NAME]: GenAiOperationName.INVOKE_AGENT, + [GenAiAttr.PROVIDER_NAME]: GenAiProviderName.GITHUB, + [GenAiAttr.AGENT_NAME]: agentName, + [GenAiAttr.CONVERSATION_ID]: this.options.conversation.sessionId, + }, + parentTraceContext, + }, + async (span) => { + const otelStartTime = Date.now(); + + // Emit session start event and metric for top-level agent invocations (not subagents) + if (!parentTraceContext) { + GenAiMetrics.incrementSessionCount(this._otelService); + try { + const endpoint = await this._endpointProvider.getChatEndpoint(this.options.request); + emitSessionStartEvent(this._otelService, this.options.conversation.sessionId, endpoint.model, agentName); + } catch { + emitSessionStartEvent(this._otelService, this.options.conversation.sessionId, 'unknown', agentName); + } + } + + // Set request model from the endpoint + try { + const endpoint = await this._endpointProvider.getChatEndpoint(this.options.request); + span.setAttribute(GenAiAttr.REQUEST_MODEL, endpoint.model); + } catch { /* endpoint not available yet, will be set on response */ } + + // Capture user input message (opt-in) + if (this._otelService.config.captureContent) { + span.setAttribute(GenAiAttr.INPUT_MESSAGES, truncateForOTel(JSON.stringify([ + { role: 'user', parts: [{ type: 'text', content: this.turn.request.message }] } + ]))); + } + + // Accumulate token usage across all LLM turns per GenAI agent span spec + let totalInputTokens = 0; + let totalOutputTokens = 0; + let lastResolvedModel: string | undefined; + let turnIndex = 0; + const tokenListener = this.onDidReceiveResponse(({ response }) => { + const turnInputTokens = response.type === ChatFetchResponseType.Success ? (response.usage?.prompt_tokens || 0) : 0; + const turnOutputTokens = response.type === ChatFetchResponseType.Success ? (response.usage?.completion_tokens || 0) : 0; + if (response.type === ChatFetchResponseType.Success && response.usage) { + totalInputTokens += turnInputTokens; + totalOutputTokens += turnOutputTokens; + } + if (response.type === ChatFetchResponseType.Success && response.resolvedModel) { + lastResolvedModel = response.resolvedModel; + } + emitAgentTurnEvent(this._otelService, turnIndex, turnInputTokens, turnOutputTokens, 0); + turnIndex++; + }); + + try { + const result = await this._runLoop(outputStream, token); + span.setAttributes({ + [CopilotChatAttr.TURN_COUNT]: result.toolCallRounds.length, + [GenAiAttr.USAGE_INPUT_TOKENS]: totalInputTokens, + [GenAiAttr.USAGE_OUTPUT_TOKENS]: totalOutputTokens, + ...(lastResolvedModel ? { [GenAiAttr.RESPONSE_MODEL]: lastResolvedModel } : {}), + }); + // Capture agent output message and tool definitions (opt-in) + if (this._otelService.config.captureContent) { + const lastRound = result.toolCallRounds.at(-1); + if (lastRound?.response) { + const responseText = Array.isArray(lastRound.response) ? lastRound.response.join('') : lastRound.response; + span.setAttribute(GenAiAttr.OUTPUT_MESSAGES, truncateForOTel(JSON.stringify([ + { role: 'assistant', parts: [{ type: 'text', content: responseText }] } + ]))); + } + // Log tool definitions once on the agent span (same set across all turns) + if (result.availableTools.length > 0) { + span.setAttribute(GenAiAttr.TOOL_DEFINITIONS, truncateForOTel(JSON.stringify( + result.availableTools.map(t => ({ type: 'function', name: t.name, description: t.description })) + ))); + } + } + span.setStatus(SpanStatusCode.OK); + + // Record agent-level metrics + const durationSec = (Date.now() - otelStartTime) / 1000; + GenAiMetrics.recordAgentDuration(this._otelService, agentName, durationSec); + GenAiMetrics.recordAgentTurnCount(this._otelService, agentName, result.toolCallRounds.length); + + return result; + } catch (err) { + span.setStatus(SpanStatusCode.ERROR, err instanceof Error ? err.message : String(err)); + span.setAttribute(StdAttr.ERROR_TYPE, err instanceof Error ? err.constructor.name : 'Error'); + throw err; + } finally { + tokenListener.dispose(); + } + }, + ); + } + + private async _runLoop(outputStream: ChatResponseStream | undefined, token: CancellationToken): Promise { let i = 0; let lastResult: IToolCallSingleResult | undefined; let lastRequestMessagesStartingIndexForRun: number | undefined; diff --git a/extensions/copilot/src/extension/intents/test/node/toolCallingLoopHooks.spec.ts b/extensions/copilot/src/extension/intents/test/node/toolCallingLoopHooks.spec.ts index 0b255b8d92f..c3bb784c671 100644 --- a/extensions/copilot/src/extension/intents/test/node/toolCallingLoopHooks.spec.ts +++ b/extensions/copilot/src/extension/intents/test/node/toolCallingLoopHooks.spec.ts @@ -15,6 +15,9 @@ import { IBuildPromptContext } from '../../../prompt/common/intents'; import { IBuildPromptResult, nullRenderPromptResult } from '../../../prompt/node/intents'; import { createExtensionUnitTestingServices } from '../../../test/node/services'; import { IToolCallingLoopOptions, ToolCallingLoop } from '../../node/toolCallingLoop'; +import { NoopOTelService } from '../../../../platform/otel/common/noopOtelService'; +import { resolveOTelConfig } from '../../../../platform/otel/common/otelConfig'; +import { IOTelService } from '../../../../platform/otel/common/otelService'; /** * Configurable mock implementation of IChatHookService for testing. @@ -168,6 +171,7 @@ describe('ToolCallingLoop SessionStart hook', () => { const serviceCollection = disposables.add(createExtensionUnitTestingServices()); // Must define the mock service BEFORE creating the accessor serviceCollection.define(IChatHookService, mockChatHookService); + serviceCollection.define(IOTelService, new NoopOTelService(resolveOTelConfig({ env: {}, extensionVersion: '0.0.0', sessionId: 'test' }))); const accessor = serviceCollection.createTestingAccessor(); instantiationService = accessor.get(IInstantiationService); @@ -548,6 +552,7 @@ describe('ToolCallingLoop SubagentStart hook', () => { const serviceCollection = disposables.add(createExtensionUnitTestingServices()); serviceCollection.define(IChatHookService, mockChatHookService); + serviceCollection.define(IOTelService, new NoopOTelService(resolveOTelConfig({ env: {}, extensionVersion: '0.0.0', sessionId: 'test' }))); const accessor = serviceCollection.createTestingAccessor(); instantiationService = accessor.get(IInstantiationService); diff --git a/extensions/copilot/src/extension/mcp/vscode-node/mcpToolCallingLoop.tsx b/extensions/copilot/src/extension/mcp/vscode-node/mcpToolCallingLoop.tsx index 8adbe3c20b0..1fb492c2909 100644 --- a/extensions/copilot/src/extension/mcp/vscode-node/mcpToolCallingLoop.tsx +++ b/extensions/copilot/src/extension/mcp/vscode-node/mcpToolCallingLoop.tsx @@ -11,6 +11,7 @@ import { ISessionTranscriptService } from '../../../platform/chat/common/session import { IConfigurationService } from '../../../platform/configuration/common/configurationService'; import { IEndpointProvider } from '../../../platform/endpoint/common/endpointProvider'; import { ILogService } from '../../../platform/log/common/logService'; +import { IOTelService } from '../../../platform/otel/common/otelService'; import { IRequestLogger } from '../../../platform/requestLogger/node/requestLogger'; import { IExperimentationService } from '../../../platform/telemetry/common/nullExperimentationService'; import { ITelemetryService } from '../../../platform/telemetry/common/telemetry'; @@ -42,8 +43,9 @@ export class McpToolCallingLoop extends ToolCallingLoop { + this._logService.error('[OTel] Error during shutdown:', String(err)); + }); + super.dispose(); + } +} diff --git a/extensions/copilot/src/extension/prompt/node/chatMLFetcher.ts b/extensions/copilot/src/extension/prompt/node/chatMLFetcher.ts index 39e07d946c6..ff72236cf8b 100644 --- a/extensions/copilot/src/extension/prompt/node/chatMLFetcher.ts +++ b/extensions/copilot/src/extension/prompt/node/chatMLFetcher.ts @@ -28,6 +28,8 @@ import { sendEngineMessagesTelemetry } from '../../../platform/networking/node/c import { IChatWebSocketManager } from '../../../platform/networking/node/chatWebSocketManager'; import { sendCommunicationErrorTelemetry } from '../../../platform/networking/node/stream'; import { ChatFailKind, ChatRequestCanceled, ChatRequestFailed, ChatResults, FetchResponseKind } from '../../../platform/openai/node/fetch'; +import { CopilotChatAttr, emitInferenceDetailsEvent, GenAiAttr, GenAiMetrics, GenAiOperationName, GenAiProviderName, StdAttr, toInputMessages, toSystemInstructions, truncateForOTel } from '../../../platform/otel/common/index'; +import { IOTelService, ISpanHandle, SpanKind, SpanStatusCode } from '../../../platform/otel/common/otelService'; import { IRequestLogger } from '../../../platform/requestLogger/node/requestLogger'; import { IExperimentationService } from '../../../platform/telemetry/common/nullExperimentationService'; import { ITelemetryService, TelemetryProperties } from '../../../platform/telemetry/common/telemetry'; @@ -116,6 +118,7 @@ export class ChatMLFetcherImpl extends AbstractChatMLFetcher { @IPowerService private readonly _powerService: IPowerService, @IInstantiationService private readonly _instantiationService: IInstantiationService, @IChatWebSocketManager private readonly _webSocketManager: IChatWebSocketManager, + @IOTelService private readonly _otelService: IOTelService, ) { super(options); } @@ -182,6 +185,7 @@ export class ChatMLFetcherImpl extends AbstractChatMLFetcher { let actualStatusCode: number | undefined; let suspendEventSeen: boolean | undefined; let resumeEventSeen: boolean | undefined; + let otelInferenceSpan: ISpanHandle | undefined; try { let response: ChatResults | ChatRequestFailed | ChatRequestCanceled; const payloadValidationResult = isValidChatPayload(opts.messages, postOptions, chatEndpoint, this._configurationService, this._experimentationService); @@ -221,6 +225,24 @@ export class ChatMLFetcherImpl extends AbstractChatMLFetcher { actualStatusCode = fetchResult.statusCode; suspendEventSeen = fetchResult.suspendEventSeen; resumeEventSeen = fetchResult.resumeEventSeen; + otelInferenceSpan = fetchResult.otelSpan; + // Tag span with debug name so orphaned spans (title, progressMessages, etc.) are identifiable + otelInferenceSpan?.setAttribute(GenAiAttr.AGENT_NAME, debugName); + // Capture request content when enabled + if (this._otelService.config.captureContent && otelInferenceSpan) { + const capiMessages = requestBody.messages as ReadonlyArray<{ role?: string; content?: string; tool_calls?: ReadonlyArray<{ id: string; function: { name: string; arguments: string } }> }>; + if (capiMessages) { + otelInferenceSpan.setAttribute(GenAiAttr.INPUT_MESSAGES, truncateForOTel(JSON.stringify(toInputMessages(capiMessages)))); + } + // Capture system instructions (first system message) + const systemMsg = capiMessages?.find(m => m.role === 'system'); + if (systemMsg?.content) { + const sysInstructions = toSystemInstructions(systemMsg.content); + if (sysInstructions) { + otelInferenceSpan.setAttribute(GenAiAttr.SYSTEM_INSTRUCTIONS, truncateForOTel(JSON.stringify(sysInstructions))); + } + } + } tokenCount = await chatEndpoint.acquireTokenizer().countMessagesTokens(messages); const extensionId = source?.extensionId ?? EXTENSION_ID; this._onDidMakeChatMLRequest.fire({ @@ -289,6 +311,79 @@ export class ChatMLFetcherImpl extends AbstractChatMLFetcher { } pendingLoggedChatRequest?.resolve(result, streamRecorder.deltas); + + // Record OTel token usage metrics if available + if (result.type === ChatFetchResponseType.Success && result.usage) { + const metricAttrs = { + operationName: GenAiOperationName.CHAT, + providerName: GenAiProviderName.GITHUB, + requestModel: chatEndpoint.model, + responseModel: result.resolvedModel, + }; + if (result.usage.prompt_tokens) { + GenAiMetrics.recordTokenUsage(this._otelService, result.usage.prompt_tokens, 'input', metricAttrs); + } + if (result.usage.completion_tokens) { + GenAiMetrics.recordTokenUsage(this._otelService, result.usage.completion_tokens, 'output', metricAttrs); + } + + // Set token usage and response details on the chat span before ending it + otelInferenceSpan?.setAttributes({ + [GenAiAttr.USAGE_INPUT_TOKENS]: result.usage.prompt_tokens ?? 0, + [GenAiAttr.USAGE_OUTPUT_TOKENS]: result.usage.completion_tokens ?? 0, + [GenAiAttr.RESPONSE_MODEL]: result.resolvedModel ?? chatEndpoint.model, + [GenAiAttr.RESPONSE_ID]: result.requestId, + [GenAiAttr.RESPONSE_FINISH_REASONS]: ['stop'], + ...(result.usage.prompt_tokens_details?.cached_tokens + ? { [GenAiAttr.USAGE_CACHE_READ_INPUT_TOKENS]: result.usage.prompt_tokens_details.cached_tokens } + : {}), + [CopilotChatAttr.TIME_TO_FIRST_TOKEN]: timeToFirstToken, + }); + } + // Capture response content when enabled + if (this._otelService.config.captureContent && otelInferenceSpan && result.type === ChatFetchResponseType.Success) { + const responseText = streamRecorder.deltas.map(d => d.text).join(''); + const toolCalls = streamRecorder.deltas + .filter(d => d.copilotToolCalls?.length) + .flatMap(d => d.copilotToolCalls!.map(tc => ({ + type: 'tool_call' as const, id: tc.id, name: tc.name, arguments: tc.arguments + }))); + const parts: Array<{ type: string; content?: string; id?: string; name?: string; arguments?: unknown }> = []; + if (responseText) { + parts.push({ type: 'text', content: responseText }); + } + parts.push(...toolCalls); + if (parts.length > 0) { + otelInferenceSpan.setAttribute(GenAiAttr.OUTPUT_MESSAGES, truncateForOTel(JSON.stringify([{ role: 'assistant', parts }]))); + } + } + + // Emit OTel inference details event BEFORE ending the span + // so the log record inherits the active trace context + emitInferenceDetailsEvent( + this._otelService, + { + model: chatEndpoint.model, + temperature: requestOptions?.temperature, + maxTokens: requestOptions?.max_tokens, + }, + result.type === ChatFetchResponseType.Success ? { + id: result.requestId, + model: result.resolvedModel, + finishReasons: ['stop'], + inputTokens: result.usage?.prompt_tokens, + outputTokens: result.usage?.completion_tokens, + } : undefined, + ); + + otelInferenceSpan?.end(); + otelInferenceSpan = undefined; + + // Record OTel time-to-first-token metric + if (timeToFirstToken > 0) { + GenAiMetrics.recordTimeToFirstToken(this._otelService, chatEndpoint.model, timeToFirstToken / 1000); + } + return result; } case FetchResponseKind.Canceled: @@ -380,6 +475,8 @@ export class ChatMLFetcherImpl extends AbstractChatMLFetcher { } } } catch (err) { + // End OTel inference span on error if not already ended + otelInferenceSpan?.end(); const timeToError = Date.now() - baseTelemetry.issuedTime; if (err.fetcherId) { actualFetcher = err.fetcherId; @@ -652,7 +749,7 @@ export class ChatMLFetcherImpl extends AbstractChatMLFetcher { useFetcher?: FetcherId, canRetryOnce?: boolean, requestKindOptions?: IBackgroundRequestOptions | ISubagentRequestOptions, - ): Promise<{ result: ChatResults | ChatRequestFailed | ChatRequestCanceled; fetcher?: FetcherId; bytesReceived?: number; statusCode?: number; suspendEventSeen?: boolean; resumeEventSeen?: boolean }> { + ): Promise<{ result: ChatResults | ChatRequestFailed | ChatRequestCanceled; fetcher?: FetcherId; bytesReceived?: number; statusCode?: number; suspendEventSeen?: boolean; resumeEventSeen?: boolean; otelSpan?: ISpanHandle }> { const isPowerSaveBlockerEnabled = this._configurationService.getExperimentBasedConfig(ConfigKey.TeamInternal.ChatRequestPowerSaveBlocker, this._experimentationService); const blockerHandle = isPowerSaveBlockerEnabled && location !== ChatLocation.Other ? this._powerService.acquirePowerSaveBlocker() : undefined; @@ -725,35 +822,73 @@ export class ChatMLFetcherImpl extends AbstractChatMLFetcher { useFetcher?: FetcherId, canRetryOnce?: boolean, requestKindOptions?: IBackgroundRequestOptions | ISubagentRequestOptions, - ): Promise<{ result: ChatResults | ChatRequestFailed | ChatRequestCanceled; fetcher?: FetcherId; bytesReceived?: number; statusCode?: number }> { + ): Promise<{ result: ChatResults | ChatRequestFailed | ChatRequestCanceled; fetcher?: FetcherId; bytesReceived?: number; statusCode?: number; otelSpan?: ISpanHandle }> { if (cancellationToken.isCancellationRequested) { return { result: { type: FetchResponseKind.Canceled, reason: 'before fetch request' } }; } - this._logService.debug(`modelMaxPromptTokens ${chatEndpointInfo.modelMaxPromptTokens}`); - this._logService.debug(`modelMaxResponseTokens ${request.max_tokens ?? 2048}`); - this._logService.debug(`chat model ${chatEndpointInfo.model}`); + // OTel inference span for this LLM call + const serverAddress = typeof chatEndpointInfo.urlOrRequestMetadata === 'string' + ? (() => { try { return new URL(chatEndpointInfo.urlOrRequestMetadata).hostname; } catch { return undefined; } })() + : undefined; + const otelSpan = this._otelService.startSpan(`chat ${chatEndpointInfo.model}`, { + kind: SpanKind.CLIENT, + attributes: { + [GenAiAttr.OPERATION_NAME]: GenAiOperationName.CHAT, + [GenAiAttr.PROVIDER_NAME]: GenAiProviderName.GITHUB, + [GenAiAttr.REQUEST_MODEL]: chatEndpointInfo.model, + [GenAiAttr.CONVERSATION_ID]: telemetryProperties?.requestId ?? ourRequestId, + [GenAiAttr.REQUEST_MAX_TOKENS]: request.max_tokens ?? request.max_output_tokens ?? request.max_completion_tokens ?? 2048, + ...(request.temperature !== undefined ? { [GenAiAttr.REQUEST_TEMPERATURE]: request.temperature } : {}), + ...(request.top_p !== undefined ? { [GenAiAttr.REQUEST_TOP_P]: request.top_p } : {}), + [CopilotChatAttr.MAX_PROMPT_TOKENS]: chatEndpointInfo.modelMaxPromptTokens, + ...(serverAddress ? { [StdAttr.SERVER_ADDRESS]: serverAddress } : {}), + }, + }); + const otelStartTime = Date.now(); - secretKey ??= copilotToken.token; - if (!secretKey) { - // If no key is set we error - const urlOrRequestMetadata = stringifyUrlOrRequestMetadata(chatEndpointInfo.urlOrRequestMetadata); - this._logService.error(`Failed to send request to ${urlOrRequestMetadata} due to missing key`); - sendCommunicationErrorTelemetry(this._telemetryService, `Failed to send request to ${urlOrRequestMetadata} due to missing key`); - return { - result: { - type: FetchResponseKind.Failed, - modelRequestId: undefined, - failKind: ChatFailKind.TokenExpiredOrInvalid, - reason: 'key is missing' - } - }; - } + try { - // WebSocket path: use persistent WebSocket connection for Responses API endpoints - if (useWebSocket && turnId && conversationId) { - return this._doFetchViaWebSocket( + this._logService.debug(`modelMaxPromptTokens ${chatEndpointInfo.modelMaxPromptTokens}`); + this._logService.debug(`modelMaxResponseTokens ${request.max_tokens ?? 2048}`); + this._logService.debug(`chat model ${chatEndpointInfo.model}`); + + secretKey ??= copilotToken.token; + if (!secretKey) { + // If no key is set we error + const urlOrRequestMetadata = stringifyUrlOrRequestMetadata(chatEndpointInfo.urlOrRequestMetadata); + this._logService.error(`Failed to send request to ${urlOrRequestMetadata} due to missing key`); + sendCommunicationErrorTelemetry(this._telemetryService, `Failed to send request to ${urlOrRequestMetadata} due to missing key`); + return { + result: { + type: FetchResponseKind.Failed, + modelRequestId: undefined, + failKind: ChatFailKind.TokenExpiredOrInvalid, + reason: 'key is missing' + } + }; + } + + // WebSocket path: use persistent WebSocket connection for Responses API endpoints + if (useWebSocket && turnId && conversationId) { + const wsResult = await this._doFetchViaWebSocket( + chatEndpointInfo, + request, + baseTelemetryData, + finishedCb, + secretKey, + location, + ourRequestId, + turnId, + conversationId, + cancellationToken, + telemetryProperties, + ); + return { ...wsResult, otelSpan }; + } + + const httpResult = await this._doFetchViaHttp( chatEndpointInfo, request, baseTelemetryData, @@ -761,29 +896,30 @@ export class ChatMLFetcherImpl extends AbstractChatMLFetcher { secretKey, location, ourRequestId, - turnId, - conversationId, + nChoices, cancellationToken, + userInitiatedRequest, telemetryProperties, + useFetcher, + canRetryOnce, + requestKindOptions, ); - } + return { ...httpResult, otelSpan }; - return this._doFetchViaHttp( - chatEndpointInfo, - request, - baseTelemetryData, - finishedCb, - secretKey, - location, - ourRequestId, - nChoices, - cancellationToken, - userInitiatedRequest, - telemetryProperties, - useFetcher, - canRetryOnce, - requestKindOptions, - ); + } catch (err) { + otelSpan.setStatus(SpanStatusCode.ERROR, err instanceof Error ? err.message : String(err)); + otelSpan.setAttribute(StdAttr.ERROR_TYPE, err instanceof Error ? err.constructor.name : 'Error'); + otelSpan.recordException(err); + throw err; + } finally { + const durationSec = (Date.now() - otelStartTime) / 1000; + GenAiMetrics.recordOperationDuration(this._otelService, durationSec, { + operationName: GenAiOperationName.CHAT, + providerName: GenAiProviderName.GITHUB, + requestModel: chatEndpointInfo.model, + }); + // Span is NOT ended here — caller (fetchMany) will set token attributes and end it + } } /** diff --git a/extensions/copilot/src/extension/prompt/node/codebaseToolCalling.ts b/extensions/copilot/src/extension/prompt/node/codebaseToolCalling.ts index 3bd777863bd..828c4ac108f 100644 --- a/extensions/copilot/src/extension/prompt/node/codebaseToolCalling.ts +++ b/extensions/copilot/src/extension/prompt/node/codebaseToolCalling.ts @@ -12,6 +12,7 @@ import { ISessionTranscriptService } from '../../../platform/chat/common/session import { IConfigurationService } from '../../../platform/configuration/common/configurationService'; import { IEndpointProvider } from '../../../platform/endpoint/common/endpointProvider'; import { ILogService } from '../../../platform/log/common/logService'; +import { IOTelService } from '../../../platform/otel/common/otelService'; import { IRequestLogger } from '../../../platform/requestLogger/node/requestLogger'; import { IExperimentationService } from '../../../platform/telemetry/common/nullExperimentationService'; import { ITelemetryService } from '../../../platform/telemetry/common/telemetry'; @@ -46,8 +47,9 @@ export class CodebaseToolCallingLoop extends ToolCallingLoop { @IToolGroupingService private readonly toolGroupingService: IToolGroupingService, @IChatHookService chatHookService: IChatHookService, @ISessionTranscriptService sessionTranscriptService: ISessionTranscriptService, + @IOTelService otelService: IOTelService, ) { - super(options, instantiationService, endpointProvider, logService, requestLogger, authenticationChatUpgradeService, telemetryService, configurationService, experimentationService, chatHookService, sessionTranscriptService); + super(options, instantiationService, endpointProvider, logService, requestLogger, authenticationChatUpgradeService, telemetryService, configurationService, experimentationService, chatHookService, sessionTranscriptService, otelService); this._register(this.onDidBuildPrompt(({ result, tools, promptTokenLength, toolTokenCount }) => { if (result.metadata.get(SummarizedConversationHistoryMetadata)) { diff --git a/extensions/copilot/src/extension/prompt/node/searchSubagentToolCallingLoop.ts b/extensions/copilot/src/extension/prompt/node/searchSubagentToolCallingLoop.ts index cd873df873a..f13e5393d91 100644 --- a/extensions/copilot/src/extension/prompt/node/searchSubagentToolCallingLoop.ts +++ b/extensions/copilot/src/extension/prompt/node/searchSubagentToolCallingLoop.ts @@ -13,6 +13,7 @@ import { ConfigKey, IConfigurationService } from '../../../platform/configuratio import { ChatEndpointFamily, IEndpointProvider } from '../../../platform/endpoint/common/endpointProvider'; import { ProxyAgenticSearchEndpoint } from '../../../platform/endpoint/node/proxyAgenticSearchEndpoint'; import { ILogService } from '../../../platform/log/common/logService'; +import { IOTelService } from '../../../platform/otel/common/otelService'; import { IRequestLogger } from '../../../platform/requestLogger/node/requestLogger'; import { IExperimentationService } from '../../../platform/telemetry/common/nullExperimentationService'; import { ITelemetryService } from '../../../platform/telemetry/common/telemetry'; @@ -51,8 +52,9 @@ export class SearchSubagentToolCallingLoop extends ToolCallingLoop { [ICAPIClientService, new TestCAPIClientService() as unknown as ICAPIClientService], ]).seal() as unknown as IInstantiationService, new NullChatWebSocketManager(), + new NoopOTelService(resolveOTelConfig({ env: {}, extensionVersion: '0.0.0', sessionId: 'test' })), ); }); diff --git a/extensions/copilot/src/extension/prompt/node/test/chatMLFetcherRetry.spec.ts b/extensions/copilot/src/extension/prompt/node/test/chatMLFetcherRetry.spec.ts index d80fbf04d2f..67012872ea8 100644 --- a/extensions/copilot/src/extension/prompt/node/test/chatMLFetcherRetry.spec.ts +++ b/extensions/copilot/src/extension/prompt/node/test/chatMLFetcherRetry.spec.ts @@ -22,6 +22,8 @@ import { FinishedCallback } from '../../../../platform/networking/common/fetch'; import { IFetcherService, IHeaders, Response } from '../../../../platform/networking/common/fetcherService'; import { IChatEndpoint } from '../../../../platform/networking/common/networking'; import { NullChatWebSocketManager } from '../../../../platform/networking/node/chatWebSocketManager'; +import { NoopOTelService } from '../../../../platform/otel/common/noopOtelService'; +import { resolveOTelConfig } from '../../../../platform/otel/common/otelConfig'; import { NullRequestLogger } from '../../../../platform/requestLogger/node/nullRequestLogger'; import { NullExperimentationService } from '../../../../platform/telemetry/common/nullExperimentationService'; import { NullTelemetryService } from '../../../../platform/telemetry/common/nullTelemetryService'; @@ -78,6 +80,7 @@ describe('ChatMLFetcherImpl retry logic', () => { [ICAPIClientService, new TestCAPIClientService() as unknown as ICAPIClientService], ]).seal() as unknown as IInstantiationService, new NullChatWebSocketManager(), + new NoopOTelService(resolveOTelConfig({ env: {}, extensionVersion: '0.0.0', sessionId: 'test' })), ); // Skip delays in tests for faster execution diff --git a/extensions/copilot/src/extension/tools/vscode-node/toolsService.ts b/extensions/copilot/src/extension/tools/vscode-node/toolsService.ts index 79de466927b..afe0445af4c 100644 --- a/extensions/copilot/src/extension/tools/vscode-node/toolsService.ts +++ b/extensions/copilot/src/extension/tools/vscode-node/toolsService.ts @@ -6,6 +6,8 @@ import * as vscode from 'vscode'; import { ILogService } from '../../../platform/log/common/logService'; import { IChatEndpoint } from '../../../platform/networking/common/networking'; +import { emitToolCallEvent, GenAiAttr, GenAiMetrics, GenAiOperationName, GenAiToolType, StdAttr, truncateForOTel } from '../../../platform/otel/common/index'; +import { IOTelService, SpanKind, SpanStatusCode } from '../../../platform/otel/common/otelService'; import { equals as arraysEqual } from '../../../util/vs/base/common/arrays'; import { Iterable } from '../../../util/vs/base/common/iterator'; import { Lazy } from '../../../util/vs/base/common/lazy'; @@ -81,7 +83,8 @@ export class ToolsService extends BaseToolsService { constructor( @IInstantiationService private readonly _instantiationService: IInstantiationService, - @ILogService logService: ILogService + @ILogService logService: ILogService, + @IOTelService private readonly _otelService: IOTelService, ) { super(logService); this._copilotTools = new Lazy(() => new Map(ToolRegistry.getTools().map(t => [t.toolName, _instantiationService.createInstance(t)] as const))); @@ -115,7 +118,79 @@ export class ToolsService extends BaseToolsService { invokeTool(name: string | ToolName, options: vscode.LanguageModelToolInvocationOptions, token: vscode.CancellationToken): Thenable { this._onWillInvokeTool.fire({ toolName: name }); - return vscode.lm.invokeTool(getContributedToolName(name), options, token); + + const isMcpTool = String(name).includes('mcp_'); + const toolInfo = this.tools.find(t => t.name === String(name)); + const span = this._otelService.startSpan(`execute_tool ${name}`, { + kind: SpanKind.INTERNAL, + attributes: { + [GenAiAttr.OPERATION_NAME]: GenAiOperationName.EXECUTE_TOOL, + [GenAiAttr.TOOL_NAME]: String(name), + [GenAiAttr.TOOL_TYPE]: isMcpTool ? GenAiToolType.EXTENSION : GenAiToolType.FUNCTION, + [GenAiAttr.TOOL_CALL_ID]: (options as { chatStreamToolCallId?: string }).chatStreamToolCallId ?? '', + ...(toolInfo?.description ? { [GenAiAttr.TOOL_DESCRIPTION]: toolInfo.description } : {}), + }, + }); + // Capture tool call arguments when content capture is enabled + if (this._otelService.config.captureContent && options.input !== undefined) { + try { + span.setAttribute(GenAiAttr.TOOL_CALL_ARGUMENTS, truncateForOTel(JSON.stringify(options.input))); + } catch { /* swallow serialization errors */ } + } + + // For runSubagent tool, store the current trace context so the subagent's + // invoke_agent span can be parented to this trace. + // Key by chatRequestId which becomes parentRequestId on the subagent side. + const chatRequestId = (options as { chatRequestId?: string }).chatRequestId; + if (String(name) === 'runSubagent' && chatRequestId) { + const traceCtx = this._otelService.getActiveTraceContext(); + if (traceCtx) { + this._otelService.storeTraceContext(`subagent:${chatRequestId}`, traceCtx); + } + } + + const startTime = Date.now(); + + return vscode.lm.invokeTool(getContributedToolName(name), options, token).then( + result => { + span.setStatus(SpanStatusCode.OK); + // Capture tool result when content capture is enabled + if (this._otelService.config.captureContent) { + try { + const parts: string[] = []; + for (const p of result.content) { + if (p instanceof vscode.LanguageModelTextPart) { + parts.push(p.value); + } else if (p instanceof vscode.LanguageModelPromptTsxPart) { + parts.push(JSON.stringify(p.value)); + } else if (p instanceof vscode.LanguageModelDataPart) { + parts.push(`[${p.mimeType}: ${p.data.byteLength} bytes]`); + } + } + if (parts.length > 0) { + span.setAttribute(GenAiAttr.TOOL_CALL_RESULT, truncateForOTel(parts.join(''))); + } + } catch { /* swallow */ } + } + span.end(); + const durationMs = Date.now() - startTime; + GenAiMetrics.recordToolCallCount(this._otelService, String(name), true); + GenAiMetrics.recordToolCallDuration(this._otelService, String(name), durationMs); + emitToolCallEvent(this._otelService, String(name), durationMs, true); + return result; + }, + err => { + span.setStatus(SpanStatusCode.ERROR, err instanceof Error ? err.message : String(err)); + span.setAttribute(StdAttr.ERROR_TYPE, err instanceof Error ? err.constructor.name : 'Error'); + span.recordException(err); + span.end(); + const durationMs = Date.now() - startTime; + GenAiMetrics.recordToolCallCount(this._otelService, String(name), false); + GenAiMetrics.recordToolCallDuration(this._otelService, String(name), durationMs); + emitToolCallEvent(this._otelService, String(name), durationMs, false, err instanceof Error ? err.constructor.name : 'Error'); + throw err; + }, + ); } override invokeToolWithEndpoint(name: string, options: vscode.LanguageModelToolInvocationOptions, endpoint: IChatEndpoint | undefined, token: vscode.CancellationToken): Thenable { diff --git a/extensions/copilot/src/lib/node/chatLibMain.ts b/extensions/copilot/src/lib/node/chatLibMain.ts index 284b7bfcae2..c70afa94e86 100644 --- a/extensions/copilot/src/lib/node/chatLibMain.ts +++ b/extensions/copilot/src/lib/node/chatLibMain.ts @@ -99,6 +99,9 @@ import { CompletionsFetchService } from '../../platform/nesFetch/node/completion import { FetchOptions, IAbortController, IFetcherService, PaginationOptions } from '../../platform/networking/common/fetcherService'; import { IFetcher } from '../../platform/networking/common/networking'; import { IChatWebSocketManager, NullChatWebSocketManager } from '../../platform/networking/node/chatWebSocketManager'; +import { NoopOTelService } from '../../platform/otel/common/noopOtelService'; +import { resolveOTelConfig } from '../../platform/otel/common/otelConfig'; +import { IOTelService } from '../../platform/otel/common/otelService'; import { IProxyModelsService } from '../../platform/proxyModels/common/proxyModelsService'; import { ProxyModelsService } from '../../platform/proxyModels/node/proxyModelsService'; import { NullRequestLogger } from '../../platform/requestLogger/node/nullRequestLogger'; @@ -371,6 +374,7 @@ function setupServices(options: INESProviderOptions) { builder.define(IPowerService, new SyncDescriptor(NullPowerService)); builder.define(IChatMLFetcher, new SyncDescriptor(ChatMLFetcherImpl)); builder.define(IChatWebSocketManager, new SyncDescriptor(NullChatWebSocketManager)); + builder.define(IOTelService, new NoopOTelService(resolveOTelConfig({ env: {}, extensionVersion: '0.0.0', sessionId: 'chatlib' }))); builder.define(IChatQuotaService, new SyncDescriptor(ChatQuotaService)); builder.define(IInteractionService, new SyncDescriptor(InteractionService)); builder.define(IRequestLogger, new SyncDescriptor(NullRequestLogger)); diff --git a/extensions/copilot/src/platform/configuration/common/configurationService.ts b/extensions/copilot/src/platform/configuration/common/configurationService.ts index 139c9dc5b61..d1403378bec 100644 --- a/extensions/copilot/src/platform/configuration/common/configurationService.ts +++ b/extensions/copilot/src/platform/configuration/common/configurationService.ts @@ -729,6 +729,13 @@ export namespace ConfigKey { /** Simulate GitHub authentication failures for testing. Can't be TeamInternal because we lose these flags as part of testing. */ export const DebugGitHubAuthFailWith = defineSetting<'NotAuthorized' | 'RequestFailed' | 'ParseFailed' | 'HTTP401' | 'RateLimited' | 'GitHubLoginFailed' | null>('chat.debug.githubAuthFailWith', ConfigType.Simple, null); + + // OTel settings + export const OTelEnabled = defineSetting('chat.otel.enabled', ConfigType.Simple, false); + export const OTelExporterType = defineSetting('chat.otel.exporterType', ConfigType.Simple, 'otlp-http'); + export const OTelOtlpEndpoint = defineSetting('chat.otel.otlpEndpoint', ConfigType.Simple, 'http://localhost:4318'); + export const OTelCaptureContent = defineSetting('chat.otel.captureContent', ConfigType.Simple, false); + export const OTelOutfile = defineSetting('chat.otel.outfile', ConfigType.Simple, ''); } /** diff --git a/extensions/copilot/src/platform/embeddings/common/remoteEmbeddingsComputer.ts b/extensions/copilot/src/platform/embeddings/common/remoteEmbeddingsComputer.ts index 3feb8dab50e..b4e251214cc 100644 --- a/extensions/copilot/src/platform/embeddings/common/remoteEmbeddingsComputer.ts +++ b/extensions/copilot/src/platform/embeddings/common/remoteEmbeddingsComputer.ts @@ -16,6 +16,8 @@ import { IEnvService } from '../../env/common/envService'; import { logExecTime } from '../../log/common/logExecTime'; import { ILogService } from '../../log/common/logService'; import { IEmbeddingsEndpoint, postRequest } from '../../networking/common/networking'; +import { GenAiAttr, GenAiOperationName, GenAiProviderName } from '../../otel/common/genAiAttributes'; +import { IOTelService, SpanKind, SpanStatusCode } from '../../otel/common/otelService'; import { ITelemetryService } from '../../telemetry/common/telemetry'; import { ComputeEmbeddingsOptions, Embedding, EmbeddingType, EmbeddingTypeInfo, EmbeddingVector, Embeddings, IEmbeddingsComputer, getWellKnownEmbeddingTypeInfo } from './embeddingsComputer'; @@ -41,6 +43,7 @@ export class RemoteEmbeddingsComputer implements IEmbeddingsComputer { @ITelemetryService private readonly _telemetryService: ITelemetryService, @IEndpointProvider private readonly _endpointProvider: IEndpointProvider, @IInstantiationService private readonly _instantiationService: IInstantiationService, + @IOTelService private readonly _otelService: IOTelService, ) { } public async computeEmbeddings( @@ -50,93 +53,111 @@ export class RemoteEmbeddingsComputer implements IEmbeddingsComputer { telemetryInfo?: TelemetryCorrelationId, cancellationToken?: CancellationToken, ): Promise { - return logExecTime(this._logService, 'RemoteEmbeddingsComputer::computeEmbeddings', async () => { - - // Determine endpoint type: use CAPI for no-auth users, otherwise use GitHub - const copilotToken = await this._authService.getCopilotToken(); - if (copilotToken.isNoAuthUser) { - const embeddings = await this.computeCAPIEmbeddings(inputs, options, cancellationToken); - return embeddings ?? { type: embeddingType, values: [] }; - } - - const token = (await this._authService.getGitHubSession('any', { silent: true }))?.accessToken; - if (!token) { - throw new Error('No authentication token available'); - } - - const embeddingsOut: Embedding[] = []; - for (let i = 0; i < inputs.length; i += this.batchSize) { - const batch = inputs.slice(i, i + this.batchSize); - if (!batch.length) { - break; - } - - const body: { - inputs: readonly string[]; - input_type: 'document' | 'query'; - embedding_model: string; - } = { - inputs: batch, - input_type: options?.inputType ?? 'document', - embedding_model: embeddingType.id, - }; - const response = await this._instantiationService.invokeFunction(postRequest, { - endpointOrUrl: { type: RequestType.DotcomEmbeddings }, - secretKey: token, - intent: 'copilot-panel', - requestId: generateUuid(), - body: body as any, - additionalHeaders: getGithubMetadataHeaders(telemetryInfo?.callTracker ?? new CallTracker(), this._envService), - cancelToken: cancellationToken, - }); - if (!response.ok) { - /* __GDPR__ - "remoteEmbeddingsComputer.computeEmbeddings.error" : { - "owner": "mjbvz", - "comment": "Total time for searchFileChunks to complete", - "source": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Caller" }, - "correlationId": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Correlation id" }, - "embeddingType": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Embedding type" }, - "totalInputLength": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "isMeasurement": true, "comment": "Total length of the input" }, - "batchInputLength": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "isMeasurement": true, "comment": "Total length of the batch" }, - "statusCode": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "isMeasurement": true, "comment": "Status code of the response" } - } - */ - this._telemetryService.sendMSFTTelemetryEvent('remoteEmbeddingsComputer.computeEmbeddings.error', { - source: telemetryInfo?.callTracker.toString(), - correlationId: telemetryInfo?.correlationId, - embeddingType: embeddingType.id, - }, { - totalInputLength: inputs.length, - batchInputLength: batch.length, - statusCode: response.status, - }); - throw new Error(`Error fetching embeddings: ${response.status}`); - } - - type EmbeddingResponse = { - embedding_model: string; - embeddings: Array<{ embedding: number[] }>; - }; - const jsonResponse: EmbeddingResponse = await response.json(); - - const resolvedType = new EmbeddingType(jsonResponse.embedding_model); - if (!resolvedType.equals(embeddingType)) { - throw new Error(`Unexpected embedding model. Got: ${resolvedType}. Expected: ${embeddingType}`); - } - - if (batch.length !== jsonResponse.embeddings.length) { - throw new Error(`Mismatched embedding result count. Expected: ${batch.length}. Got: ${jsonResponse.embeddings.length}`); - } - - embeddingsOut.push(...jsonResponse.embeddings.map(embedding => ({ - type: resolvedType, - value: embedding.embedding, - }))); - } - - return { type: embeddingType, values: embeddingsOut }; + const otelSpan = this._otelService.startSpan(`embeddings ${embeddingType.id}`, { + kind: SpanKind.CLIENT, + attributes: { + [GenAiAttr.OPERATION_NAME]: GenAiOperationName.EMBEDDINGS, + [GenAiAttr.PROVIDER_NAME]: GenAiProviderName.OPENAI, + [GenAiAttr.REQUEST_MODEL]: embeddingType.id, + 'gen_ai.embeddings.input_count': inputs.length, + }, }); + try { + return await logExecTime(this._logService, 'RemoteEmbeddingsComputer::computeEmbeddings', async () => { + + // Determine endpoint type: use CAPI for no-auth users, otherwise use GitHub + const copilotToken = await this._authService.getCopilotToken(); + if (copilotToken.isNoAuthUser) { + const embeddings = await this.computeCAPIEmbeddings(inputs, options, cancellationToken); + return embeddings ?? { type: embeddingType, values: [] }; + } + + const token = (await this._authService.getGitHubSession('any', { silent: true }))?.accessToken; + if (!token) { + throw new Error('No authentication token available'); + } + + const embeddingsOut: Embedding[] = []; + for (let i = 0; i < inputs.length; i += this.batchSize) { + const batch = inputs.slice(i, i + this.batchSize); + if (!batch.length) { + break; + } + + const body: { + inputs: readonly string[]; + input_type: 'document' | 'query'; + embedding_model: string; + } = { + inputs: batch, + input_type: options?.inputType ?? 'document', + embedding_model: embeddingType.id, + }; + const response = await this._instantiationService.invokeFunction(postRequest, { + endpointOrUrl: { type: RequestType.DotcomEmbeddings }, + secretKey: token, + intent: 'copilot-panel', + requestId: generateUuid(), + body: body as any, + additionalHeaders: getGithubMetadataHeaders(telemetryInfo?.callTracker ?? new CallTracker(), this._envService), + cancelToken: cancellationToken, + }); + if (!response.ok) { + /* __GDPR__ + "remoteEmbeddingsComputer.computeEmbeddings.error" : { + "owner": "mjbvz", + "comment": "Total time for searchFileChunks to complete", + "source": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Caller" }, + "correlationId": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Correlation id" }, + "embeddingType": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Embedding type" }, + "totalInputLength": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "isMeasurement": true, "comment": "Total length of the input" }, + "batchInputLength": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "isMeasurement": true, "comment": "Total length of the batch" }, + "statusCode": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "isMeasurement": true, "comment": "Status code of the response" } + } + */ + this._telemetryService.sendMSFTTelemetryEvent('remoteEmbeddingsComputer.computeEmbeddings.error', { + source: telemetryInfo?.callTracker.toString(), + correlationId: telemetryInfo?.correlationId, + embeddingType: embeddingType.id, + }, { + totalInputLength: inputs.length, + batchInputLength: batch.length, + statusCode: response.status, + }); + throw new Error(`Error fetching embeddings: ${response.status}`); + } + + type EmbeddingResponse = { + embedding_model: string; + embeddings: Array<{ embedding: number[] }>; + }; + const jsonResponse: EmbeddingResponse = await response.json(); + + const resolvedType = new EmbeddingType(jsonResponse.embedding_model); + if (!resolvedType.equals(embeddingType)) { + throw new Error(`Unexpected embedding model. Got: ${resolvedType}. Expected: ${embeddingType}`); + } + + if (batch.length !== jsonResponse.embeddings.length) { + throw new Error(`Mismatched embedding result count. Expected: ${batch.length}. Got: ${jsonResponse.embeddings.length}`); + } + + embeddingsOut.push(...jsonResponse.embeddings.map(embedding => ({ + type: resolvedType, + value: embedding.embedding, + }))); + } + + return { type: embeddingType, values: embeddingsOut }; + }); + } catch (err) { + otelSpan.setStatus(SpanStatusCode.ERROR, err instanceof Error ? err.message : String(err)); + otelSpan.setAttribute('error.type', err instanceof Error ? err.constructor.name : 'Error'); + otelSpan.recordException(err); + throw err; + } finally { + otelSpan.end(); + } } private async computeCAPIEmbeddings( diff --git a/extensions/copilot/src/platform/endpoint/vscode-node/extChatEndpoint.ts b/extensions/copilot/src/platform/endpoint/vscode-node/extChatEndpoint.ts index a0e23348d4f..7f77e8202de 100644 --- a/extensions/copilot/src/platform/endpoint/vscode-node/extChatEndpoint.ts +++ b/extensions/copilot/src/platform/endpoint/vscode-node/extChatEndpoint.ts @@ -19,6 +19,7 @@ import { FinishedCallback, OpenAiFunctionTool, OptionalChatRequestParams } from import { Response } from '../../networking/common/fetcherService'; import { IChatEndpoint, ICreateEndpointBodyOptions, IEndpointBody, IMakeChatRequestOptions } from '../../networking/common/networking'; import { ChatCompletion } from '../../networking/common/openai'; +import { IOTelService } from '../../otel/common/otelService'; import { retrieveCapturingTokenByCorrelation, storeCapturingTokenForCorrelation } from '../../requestLogger/node/requestLogger'; import { ITelemetryService } from '../../telemetry/common/telemetry'; import { TelemetryData } from '../../telemetry/common/telemetryData'; @@ -48,6 +49,7 @@ export class ExtensionContributedChatEndpoint implements IChatEndpoint { constructor( private readonly languageModel: vscode.LanguageModelChat, @IInstantiationService private readonly _instantiationService: IInstantiationService, + @IOTelService private readonly _otelService: IOTelService, ) { // Initialize with the model's max tokens this._maxTokens = languageModel.maxInputTokens; @@ -169,25 +171,31 @@ export class ExtensionContributedChatEndpoint implements IChatEndpoint { const vscodeMessages = convertToApiChatMessage(messages); const ourRequestId = generateUuid(); + // Capture active OTel trace context to propagate through IPC to the BYOK provider. + // Each provider creates its own chat span with full usage data: + // - OpenAI-compatible (Azure, OpenAI, etc.): via CopilotLanguageModelWrapper → chatMLFetcher + // - Anthropic: inside AnthropicLMProvider + // - Gemini: inside GeminiNativeBYOKLMProvider + const activeTraceCtx = this._otelService.getActiveTraceContext(); + const vscodeOptions: vscode.LanguageModelChatRequestOptions = { tools: ((requestOptions?.tools ?? []) as OpenAiFunctionTool[]).map(tool => ({ name: tool.function.name, description: tool.function.description, inputSchema: tool.function.parameters, })), - // Pass correlation ID through modelOptions for cross-IPC CapturingToken restoration. - // This allows BYOK providers to associate their requests with the original captureInvocation context. + // Pass correlation ID and OTel trace context through modelOptions for cross-IPC restoration. modelOptions: { - _capturingTokenCorrelationId: ourRequestId + _capturingTokenCorrelationId: ourRequestId, + _otelTraceContext: activeTraceCtx ?? null, } }; // Store current CapturingToken for retrieval by BYOK providers after IPC crossing // - // Note: We intentionally don't log chat requests here for external models (BYOK). - // BYOK providers (Anthropic, Gemini, CopilotLanguageModelWrapper) handle their own - // logging with correct token usage. Logging here would create duplicates with - // incorrect (0) token counts since we don't have access to actual usage stats. + // Note: We intentionally don't create an OTel chat span here for extension-contributed models. + // The BYOK provider (CopilotLanguageModelWrapper) creates the real chat span via chatMLFetcher + // with full token usage, response model, and cache data. Creating a span here would duplicate it. storeCapturingTokenForCorrelation(ourRequestId); const streamRecorder = new FetchStreamRecorder(finishedCb); @@ -202,12 +210,10 @@ export class ExtensionContributedChatEndpoint implements IChatEndpoint { for await (const chunk of response.stream) { if (chunk instanceof vscode.LanguageModelTextPart) { text += chunk.value; - // Call finishedCb with the current chunk of text if (streamRecorder.callback) { await streamRecorder.callback(text, 0, { text: chunk.value }); } } else if (chunk instanceof vscode.LanguageModelToolCallPart) { - // Call finishedCb with updated tool calls if (streamRecorder.callback) { const functionCalls = [chunk].map(tool => ({ name: tool.name ?? '', @@ -226,10 +232,9 @@ export class ExtensionContributedChatEndpoint implements IChatEndpoint { await streamRecorder.callback?.(text, 0, { text: '', contextManagement }); } } else if (chunk instanceof vscode.LanguageModelThinkingPart) { - // Call finishedCb with the current chunk of thinking text with a specific thinking field if (streamRecorder.callback) { await streamRecorder.callback(text, 0, { - text: '', // Use empty text to avoid creating markdown part + text: '', thinking: { text: chunk.value, id: chunk.id || '', @@ -241,7 +246,7 @@ export class ExtensionContributedChatEndpoint implements IChatEndpoint { } if (text || numToolsCalled > 0) { - const response: ChatResponse = { + return { type: ChatFetchResponseType.Success, requestId, serverRequestId: requestId, @@ -249,28 +254,22 @@ export class ExtensionContributedChatEndpoint implements IChatEndpoint { value: text, resolvedModel: this.languageModel.id }; - return response; } else { - const result: ChatResponse = { + return { type: ChatFetchResponseType.Unknown, reason: 'No response from language model', requestId: requestId, serverRequestId: undefined }; - return result; } } catch (e) { - const result: ChatResponse = { + return { type: ChatFetchResponseType.Failed, reason: toErrorMessage(e, true), requestId: generateUuid(), serverRequestId: undefined }; - return result; } finally { - // Clean up correlation map entry to prevent memory leak. - // If the request reached a BYOK provider, they already retrieved and removed this. - // If not (e.g., request failed early or model isn't BYOK), we need to clean it up here. retrieveCapturingTokenByCorrelation(ourRequestId); } } diff --git a/extensions/copilot/src/platform/otel/common/genAiAttributes.ts b/extensions/copilot/src/platform/otel/common/genAiAttributes.ts new file mode 100644 index 00000000000..ad876bf4790 --- /dev/null +++ b/extensions/copilot/src/platform/otel/common/genAiAttributes.ts @@ -0,0 +1,116 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +// gen_ai.operation.name values +export const GenAiOperationName = { + CHAT: 'chat', + INVOKE_AGENT: 'invoke_agent', + EXECUTE_TOOL: 'execute_tool', + EMBEDDINGS: 'embeddings', +} as const; + +// gen_ai.provider.name values +export const GenAiProviderName = { + GITHUB: 'github', + OPENAI: 'openai', + ANTHROPIC: 'anthropic', + AZURE_AI_OPENAI: 'azure.ai.openai', +} as const; + +// gen_ai.token.type values +export const GenAiTokenType = { + INPUT: 'input', + OUTPUT: 'output', +} as const; + +// gen_ai.tool.type values +export const GenAiToolType = { + FUNCTION: 'function', + EXTENSION: 'extension', +} as const; + +/** + * OTel GenAI semantic convention attribute keys. + * @see https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai/gen-ai-spans.md + */ +export const GenAiAttr = { + // Core + OPERATION_NAME: 'gen_ai.operation.name', + PROVIDER_NAME: 'gen_ai.provider.name', + + // Request + REQUEST_MODEL: 'gen_ai.request.model', + REQUEST_TEMPERATURE: 'gen_ai.request.temperature', + REQUEST_MAX_TOKENS: 'gen_ai.request.max_tokens', + REQUEST_TOP_P: 'gen_ai.request.top_p', + REQUEST_FREQUENCY_PENALTY: 'gen_ai.request.frequency_penalty', + REQUEST_PRESENCE_PENALTY: 'gen_ai.request.presence_penalty', + REQUEST_SEED: 'gen_ai.request.seed', + REQUEST_STOP_SEQUENCES: 'gen_ai.request.stop_sequences', + + // Response + RESPONSE_MODEL: 'gen_ai.response.model', + RESPONSE_ID: 'gen_ai.response.id', + RESPONSE_FINISH_REASONS: 'gen_ai.response.finish_reasons', + + // Usage + USAGE_INPUT_TOKENS: 'gen_ai.usage.input_tokens', + USAGE_OUTPUT_TOKENS: 'gen_ai.usage.output_tokens', + USAGE_CACHE_READ_INPUT_TOKENS: 'gen_ai.usage.cache_read.input_tokens', + USAGE_CACHE_CREATION_INPUT_TOKENS: 'gen_ai.usage.cache_creation.input_tokens', + + // Conversation + CONVERSATION_ID: 'gen_ai.conversation.id', + OUTPUT_TYPE: 'gen_ai.output.type', + + // Token type (for metrics) + TOKEN_TYPE: 'gen_ai.token.type', + + // Agent + AGENT_NAME: 'gen_ai.agent.name', + AGENT_ID: 'gen_ai.agent.id', + AGENT_VERSION: 'gen_ai.agent.version', + AGENT_DESCRIPTION: 'gen_ai.agent.description', + + // Tool + TOOL_NAME: 'gen_ai.tool.name', + TOOL_TYPE: 'gen_ai.tool.type', + TOOL_CALL_ID: 'gen_ai.tool.call.id', + TOOL_DESCRIPTION: 'gen_ai.tool.description', + TOOL_CALL_ARGUMENTS: 'gen_ai.tool.call.arguments', + TOOL_CALL_RESULT: 'gen_ai.tool.call.result', + + // Content (opt-in) + INPUT_MESSAGES: 'gen_ai.input.messages', + OUTPUT_MESSAGES: 'gen_ai.output.messages', + SYSTEM_INSTRUCTIONS: 'gen_ai.system_instructions', + TOOL_DEFINITIONS: 'gen_ai.tool.definitions', +} as const; + +/** + * Extension-specific attribute keys (custom namespace). + */ +export const CopilotChatAttr = { + LOCATION: 'copilot_chat.location', + INTENT: 'copilot_chat.intent', + TURN_INDEX: 'copilot_chat.turn.index', + TURN_COUNT: 'copilot_chat.turn_count', + TOOL_CALL_ROUND: 'copilot_chat.tool_call_round', + API_TYPE: 'copilot_chat.api_type', + FETCHER: 'copilot_chat.fetcher', + DEBUG_NAME: 'copilot_chat.debug_name', + ENDPOINT_TYPE: 'copilot_chat.endpoint_type', + MAX_PROMPT_TOKENS: 'copilot_chat.request.max_prompt_tokens', + TIME_TO_FIRST_TOKEN: 'copilot_chat.time_to_first_token', +} as const; + +/** + * Standard OTel attributes used alongside GenAI attributes. + */ +export const StdAttr = { + ERROR_TYPE: 'error.type', + SERVER_ADDRESS: 'server.address', + SERVER_PORT: 'server.port', +} as const; diff --git a/extensions/copilot/src/platform/otel/common/genAiEvents.ts b/extensions/copilot/src/platform/otel/common/genAiEvents.ts new file mode 100644 index 00000000000..0092cf57cb5 --- /dev/null +++ b/extensions/copilot/src/platform/otel/common/genAiEvents.ts @@ -0,0 +1,116 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { GenAiAttr, GenAiOperationName, StdAttr } from './genAiAttributes'; +import { truncateForOTel } from './messageFormatters'; +import type { IOTelService } from './otelService'; + +/** + * Emit OTel GenAI standard events via the IOTelService abstraction. + */ +export function emitInferenceDetailsEvent( + otel: IOTelService, + request: { + model: string; + temperature?: number; + maxTokens?: number; + messages?: unknown; + systemMessage?: unknown; + tools?: unknown; + }, + response: { + id?: string; + model?: string; + finishReasons?: string[]; + inputTokens?: number; + outputTokens?: number; + } | undefined, + error?: { type: string; message: string }, +): void { + const attributes: Record = { + 'event.name': 'gen_ai.client.inference.operation.details', + [GenAiAttr.OPERATION_NAME]: GenAiOperationName.CHAT, + [GenAiAttr.REQUEST_MODEL]: request.model, + }; + + if (response) { + if (response.model) { attributes[GenAiAttr.RESPONSE_MODEL] = response.model; } + if (response.id) { attributes[GenAiAttr.RESPONSE_ID] = response.id; } + if (response.finishReasons) { attributes[GenAiAttr.RESPONSE_FINISH_REASONS] = response.finishReasons; } + if (response.inputTokens !== undefined) { attributes[GenAiAttr.USAGE_INPUT_TOKENS] = response.inputTokens; } + if (response.outputTokens !== undefined) { attributes[GenAiAttr.USAGE_OUTPUT_TOKENS] = response.outputTokens; } + } + + if (request.temperature !== undefined) { attributes[GenAiAttr.REQUEST_TEMPERATURE] = request.temperature; } + if (request.maxTokens !== undefined) { attributes[GenAiAttr.REQUEST_MAX_TOKENS] = request.maxTokens; } + + if (error) { + attributes[StdAttr.ERROR_TYPE] = error.type; + } + + // Full content capture with truncation to prevent OTLP batch failures + if (otel.config.captureContent) { + if (request.messages !== undefined) { + attributes[GenAiAttr.INPUT_MESSAGES] = truncateForOTel(JSON.stringify(request.messages)); + } + if (request.systemMessage !== undefined) { + attributes[GenAiAttr.SYSTEM_INSTRUCTIONS] = truncateForOTel(JSON.stringify(request.systemMessage)); + } + if (request.tools !== undefined) { + attributes[GenAiAttr.TOOL_DEFINITIONS] = truncateForOTel(JSON.stringify(request.tools)); + } + } + + otel.emitLogRecord(`GenAI inference: ${request.model}`, attributes); +} + +/** + * Emit extension-specific events. + */ +export function emitSessionStartEvent( + otel: IOTelService, + sessionId: string, + model: string, + participant: string, +): void { + otel.emitLogRecord('copilot_chat.session.start', { + 'event.name': 'copilot_chat.session.start', + 'session.id': sessionId, + [GenAiAttr.REQUEST_MODEL]: model, + [GenAiAttr.AGENT_NAME]: participant, + }); +} + +export function emitToolCallEvent( + otel: IOTelService, + toolName: string, + durationMs: number, + success: boolean, + error?: string, +): void { + otel.emitLogRecord(`copilot_chat.tool.call: ${toolName}`, { + 'event.name': 'copilot_chat.tool.call', + [GenAiAttr.TOOL_NAME]: toolName, + 'duration_ms': durationMs, + 'success': success, + ...(error ? { [StdAttr.ERROR_TYPE]: error } : {}), + }); +} + +export function emitAgentTurnEvent( + otel: IOTelService, + turnIndex: number, + inputTokens: number, + outputTokens: number, + toolCallCount: number, +): void { + otel.emitLogRecord(`copilot_chat.agent.turn: ${turnIndex}`, { + 'event.name': 'copilot_chat.agent.turn', + 'turn.index': turnIndex, + [GenAiAttr.USAGE_INPUT_TOKENS]: inputTokens, + [GenAiAttr.USAGE_OUTPUT_TOKENS]: outputTokens, + 'tool_call_count': toolCallCount, + }); +} diff --git a/extensions/copilot/src/platform/otel/common/genAiMetrics.ts b/extensions/copilot/src/platform/otel/common/genAiMetrics.ts new file mode 100644 index 00000000000..6685963468e --- /dev/null +++ b/extensions/copilot/src/platform/otel/common/genAiMetrics.ts @@ -0,0 +1,99 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { GenAiAttr, StdAttr } from './genAiAttributes'; +import type { IOTelService } from './otelService'; + +/** + * Pre-configured OTel GenAI metric instruments. + * All methods are static to avoid per-call allocations (aligned with gemini-cli pattern). + */ +export class GenAiMetrics { + + // ── GenAI Convention Metrics ── + + static recordOperationDuration( + otel: IOTelService, + durationSec: number, + attrs: { + operationName: string; + providerName: string; + requestModel: string; + responseModel?: string; + serverAddress?: string; + serverPort?: number; + errorType?: string; + }, + ): void { + otel.recordMetric('gen_ai.client.operation.duration', durationSec, { + [GenAiAttr.OPERATION_NAME]: attrs.operationName, + [GenAiAttr.PROVIDER_NAME]: attrs.providerName, + [GenAiAttr.REQUEST_MODEL]: attrs.requestModel, + ...(attrs.responseModel ? { [GenAiAttr.RESPONSE_MODEL]: attrs.responseModel } : {}), + ...(attrs.serverAddress ? { [StdAttr.SERVER_ADDRESS]: attrs.serverAddress } : {}), + ...(attrs.serverPort ? { [StdAttr.SERVER_PORT]: attrs.serverPort } : {}), + ...(attrs.errorType ? { [StdAttr.ERROR_TYPE]: attrs.errorType } : {}), + }); + } + + static recordTokenUsage( + otel: IOTelService, + tokenCount: number, + tokenType: 'input' | 'output', + attrs: { + operationName: string; + providerName: string; + requestModel: string; + responseModel?: string; + serverAddress?: string; + }, + ): void { + otel.recordMetric('gen_ai.client.token.usage', tokenCount, { + [GenAiAttr.OPERATION_NAME]: attrs.operationName, + [GenAiAttr.PROVIDER_NAME]: attrs.providerName, + [GenAiAttr.TOKEN_TYPE]: tokenType, + [GenAiAttr.REQUEST_MODEL]: attrs.requestModel, + ...(attrs.responseModel ? { [GenAiAttr.RESPONSE_MODEL]: attrs.responseModel } : {}), + ...(attrs.serverAddress ? { [StdAttr.SERVER_ADDRESS]: attrs.serverAddress } : {}), + }); + } + + // ── Extension-Specific Metrics ── + + static recordToolCallCount(otel: IOTelService, toolName: string, success: boolean): void { + otel.incrementCounter('copilot_chat.tool.call.count', 1, { + [GenAiAttr.TOOL_NAME]: toolName, + success, + }); + } + + static recordToolCallDuration(otel: IOTelService, toolName: string, durationMs: number): void { + otel.recordMetric('copilot_chat.tool.call.duration', durationMs, { + [GenAiAttr.TOOL_NAME]: toolName, + }); + } + + static recordAgentDuration(otel: IOTelService, agentName: string, durationSec: number): void { + otel.recordMetric('copilot_chat.agent.invocation.duration', durationSec, { + [GenAiAttr.AGENT_NAME]: agentName, + }); + } + + static recordAgentTurnCount(otel: IOTelService, agentName: string, turnCount: number): void { + otel.recordMetric('copilot_chat.agent.turn.count', turnCount, { + [GenAiAttr.AGENT_NAME]: agentName, + }); + } + + static recordTimeToFirstToken(otel: IOTelService, model: string, ttftSec: number): void { + otel.recordMetric('copilot_chat.time_to_first_token', ttftSec, { + [GenAiAttr.REQUEST_MODEL]: model, + }); + } + + static incrementSessionCount(otel: IOTelService): void { + otel.incrementCounter('copilot_chat.session.count'); + } +} diff --git a/extensions/copilot/src/platform/otel/common/index.ts b/extensions/copilot/src/platform/otel/common/index.ts new file mode 100644 index 00000000000..5b22369bea1 --- /dev/null +++ b/extensions/copilot/src/platform/otel/common/index.ts @@ -0,0 +1,13 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +export { CopilotChatAttr, GenAiAttr, GenAiOperationName, GenAiProviderName, GenAiTokenType, GenAiToolType, StdAttr } from './genAiAttributes'; +export { emitAgentTurnEvent, emitInferenceDetailsEvent, emitSessionStartEvent, emitToolCallEvent } from './genAiEvents'; +export { GenAiMetrics } from './genAiMetrics'; +export { toInputMessages, toOutputMessages, toSystemInstructions, toToolDefinitions, truncateForOTel } from './messageFormatters'; +export { NoopOTelService } from './noopOtelService'; +export { resolveOTelConfig, type OTelConfig, type OTelConfigInput } from './otelConfig'; +export { IOTelService, SpanKind, SpanStatusCode, type ISpanHandle, type OTelModelOptions, type SpanOptions, type TraceContext } from './otelService'; + diff --git a/extensions/copilot/src/platform/otel/common/messageFormatters.ts b/extensions/copilot/src/platform/otel/common/messageFormatters.ts new file mode 100644 index 00000000000..af87ff6b86a --- /dev/null +++ b/extensions/copilot/src/platform/otel/common/messageFormatters.ts @@ -0,0 +1,147 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +/** + * Converts internal message types to OTel GenAI JSON schema format. + * @see https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai/gen-ai-input-messages.json + * @see https://github.com/open-telemetry/semantic-conventions/blob/main/docs/gen-ai/gen-ai-output-messages.json + */ + +/** + * Maximum size (in characters) for a single OTel span/log attribute value. + * Aligned with common backend limits (Jaeger 64KB, Tempo 100KB). + * Matches gemini-cli's approach of capping content to prevent OTLP batch failures. + */ +const MAX_OTEL_ATTRIBUTE_LENGTH = 64_000; + +/** + * Truncate a string to fit within OTel attribute size limits. + * Returns the original string if within bounds, otherwise truncates with a suffix. + */ +export function truncateForOTel(value: string, maxLength: number = MAX_OTEL_ATTRIBUTE_LENGTH): string { + if (value.length <= maxLength) { + return value; + } + const suffix = `...[truncated, original ${value.length} chars]`; + return value.substring(0, maxLength - suffix.length) + suffix; +} + +export interface OTelChatMessage { + role: string | undefined; + parts: OTelMessagePart[]; +} + +export interface OTelOutputMessage extends OTelChatMessage { + finish_reason?: string; +} + +export type OTelMessagePart = + | { type: 'text'; content: string } + | { type: 'tool_call'; id: string; name: string; arguments: unknown } + | { type: 'tool_call_response'; id: string; content: unknown } + | { type: 'reasoning'; content: string }; + +export type OTelSystemInstruction = Array<{ type: 'text'; content: string }>; + +export interface OTelToolDefinition { + type: 'function'; + name: string; + description?: string; + parameters?: unknown; +} + +/** + * Convert an array of internal messages to OTel input message format. + */ +export function toInputMessages(messages: ReadonlyArray<{ role?: string; content?: string; tool_calls?: ReadonlyArray<{ id: string; function: { name: string; arguments: string } }> }>): OTelChatMessage[] { + return messages.map(msg => { + const parts: OTelMessagePart[] = []; + + if (msg.content) { + parts.push({ type: 'text', content: msg.content }); + } + + if (msg.tool_calls) { + for (const tc of msg.tool_calls) { + let args: unknown; + try { args = JSON.parse(tc.function.arguments); } catch { args = tc.function.arguments; } + parts.push({ + type: 'tool_call', + id: tc.id, + name: tc.function.name, + arguments: args, + }); + } + } + + return { role: msg.role, parts }; + }); +} + +/** + * Convert model response choices to OTel output message format. + */ +export function toOutputMessages(choices: ReadonlyArray<{ + message?: { role?: string; content?: string; tool_calls?: ReadonlyArray<{ id: string; function: { name: string; arguments: string } }> }; + finish_reason?: string; +}>): OTelOutputMessage[] { + return choices.map(choice => { + const parts: OTelMessagePart[] = []; + const msg = choice.message; + + if (msg?.content) { + parts.push({ type: 'text', content: msg.content }); + } + + if (msg?.tool_calls) { + for (const tc of msg.tool_calls) { + let args: unknown; + try { args = JSON.parse(tc.function.arguments); } catch { args = tc.function.arguments; } + parts.push({ + type: 'tool_call', + id: tc.id, + name: tc.function.name, + arguments: args, + }); + } + } + + return { + role: msg?.role ?? 'assistant', + parts, + finish_reason: choice.finish_reason, + }; + }); +} + +/** + * Convert system message to OTel system instruction format. + */ +export function toSystemInstructions(systemMessage: string | undefined): OTelSystemInstruction | undefined { + if (!systemMessage) { + return undefined; + } + return [{ type: 'text', content: systemMessage }]; +} + +/** + * Convert tool definitions to OTel tool definition format. + */ +export function toToolDefinitions(tools: ReadonlyArray<{ + type?: string; + function?: { name: string; description?: string; parameters?: unknown }; +}> | undefined): OTelToolDefinition[] | undefined { + if (!tools || tools.length === 0) { + return undefined; + } + return tools + .filter((t): t is typeof t & { function: NonNullable } => !!t.function) + .map(t => ({ + type: 'function' as const, + name: t.function.name, + description: t.function.description, + parameters: t.function.parameters, + })); +} diff --git a/extensions/copilot/src/platform/otel/common/noopOtelService.ts b/extensions/copilot/src/platform/otel/common/noopOtelService.ts new file mode 100644 index 00000000000..686090efb71 --- /dev/null +++ b/extensions/copilot/src/platform/otel/common/noopOtelService.ts @@ -0,0 +1,60 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import type { OTelConfig } from './otelConfig'; +import type { IOTelService, ISpanHandle, SpanOptions, TraceContext } from './otelService'; + +const noopSpan: ISpanHandle = { + setAttribute() { }, + setAttributes() { }, + setStatus() { }, + recordException() { }, + end() { }, +}; + +/** + * No-op implementation of IOTelService. + * All methods are zero-cost when OTel is disabled. + */ +export class NoopOTelService implements IOTelService { + declare readonly _serviceBrand: undefined; + readonly config: OTelConfig; + + constructor(config: OTelConfig) { + this.config = config; + } + + startSpan(_name: string, _options?: SpanOptions): ISpanHandle { + return noopSpan; + } + + startActiveSpan(_name: string, _options: SpanOptions, fn: (span: ISpanHandle) => Promise): Promise { + return fn(noopSpan); + } + + getActiveTraceContext(): TraceContext | undefined { + return undefined; + } + + storeTraceContext(_key: string, _context: TraceContext): void { } + + getStoredTraceContext(_key: string): TraceContext | undefined { + return undefined; + } + + runWithTraceContext(_traceContext: TraceContext, fn: () => Promise): Promise { + return fn(); + } + + recordMetric(_name: string, _value: number, _attributes?: Record): void { } + + incrementCounter(_name: string, _value?: number, _attributes?: Record): void { } + + emitLogRecord(_body: string, _attributes?: Record): void { } + + async flush(): Promise { } + + async shutdown(): Promise { } +} diff --git a/extensions/copilot/src/platform/otel/common/otelConfig.ts b/extensions/copilot/src/platform/otel/common/otelConfig.ts new file mode 100644 index 00000000000..46411e02e87 --- /dev/null +++ b/extensions/copilot/src/platform/otel/common/otelConfig.ts @@ -0,0 +1,180 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +export type OTelExporterType = 'otlp-grpc' | 'otlp-http' | 'console' | 'file'; + +export interface OTelConfig { + readonly enabled: boolean; + readonly exporterType: OTelExporterType; + readonly otlpEndpoint: string; + readonly otlpProtocol: 'grpc' | 'http'; + readonly captureContent: boolean; + readonly fileExporterPath?: string; + readonly logLevel: 'trace' | 'debug' | 'info' | 'warn' | 'error'; + readonly httpInstrumentation: boolean; + readonly serviceName: string; + readonly serviceVersion: string; + readonly sessionId: string; + readonly resourceAttributes: Record; +} + +/** + * Parse `OTEL_RESOURCE_ATTRIBUTES` format: "key1=val1,key2=val2" + */ +function parseResourceAttributes(raw: string | undefined): Record { + if (!raw) { + return {}; + } + const result: Record = {}; + for (const pair of raw.split(',')) { + const eqIdx = pair.indexOf('='); + if (eqIdx > 0) { + const key = pair.substring(0, eqIdx).trim(); + const value = pair.substring(eqIdx + 1).trim(); + if (key) { + result[key] = value; + } + } + } + return result; +} + +/** + * Parse and validate an OTLP endpoint URL. + * For gRPC: returns origin (scheme://host:port). + * For HTTP: returns full href. + */ +function parseOtlpEndpoint(raw: string | undefined, protocol: 'grpc' | 'http'): string | undefined { + if (!raw) { + return undefined; + } + const trimmed = raw.replace(/^["']|["']$/g, ''); + try { + const url = new URL(trimmed); + return protocol === 'grpc' ? url.origin : url.href; + } catch { + return undefined; + } +} + +export interface OTelConfigInput { + env: Record; + settingEnabled?: boolean; + settingExporterType?: OTelExporterType; + settingOtlpEndpoint?: string; + settingCaptureContent?: boolean; + settingOutfile?: string; + extensionVersion: string; + sessionId: string; + vscodeTelemetryLevel?: string; +} + +/** + * Resolve OTel configuration with layered precedence: + * 1. COPILOT_OTEL_* env vars (highest) + * 2. OTEL_EXPORTER_OTLP_* standard env vars + * 3. VS Code settings + * 4. Defaults (lowest) + */ +export function resolveOTelConfig(input: OTelConfigInput): OTelConfig { + const { env } = input; + + // Kill switch: respect VS Code telemetry level + if (input.vscodeTelemetryLevel === 'off') { + return createDisabledConfig(input); + } + + // Determine if enabled: env > setting > default(false) + const enabled = envBool(env['COPILOT_OTEL_ENABLED']) + ?? input.settingEnabled + ?? (!!env['OTEL_EXPORTER_OTLP_ENDPOINT']); + + if (!enabled) { + return createDisabledConfig(input); + } + + // Protocol: env > inferred from exporter type > default + const rawProtocol = env['OTEL_EXPORTER_OTLP_PROTOCOL'] ?? env['COPILOT_OTEL_PROTOCOL']; + const protocol: 'grpc' | 'http' = rawProtocol === 'grpc' ? 'grpc' : 'http'; + + // Endpoint: COPILOT_OTEL env > OTEL env > setting > default + const rawEndpoint = env['COPILOT_OTEL_ENDPOINT'] + ?? env['OTEL_EXPORTER_OTLP_ENDPOINT'] + ?? input.settingOtlpEndpoint + ?? 'http://localhost:4318'; + const otlpEndpoint = parseOtlpEndpoint(rawEndpoint, protocol) ?? 'http://localhost:4318'; + + // File exporter path + const fileExporterPath = env['COPILOT_OTEL_FILE_EXPORTER_PATH'] ?? input.settingOutfile; + + // Exporter type + let exporterType: OTelExporterType; + if (fileExporterPath) { + exporterType = 'file'; + } else if (input.settingExporterType) { + exporterType = input.settingExporterType; + } else { + exporterType = protocol === 'grpc' ? 'otlp-grpc' : 'otlp-http'; + } + + // Content capture + const captureContent = envBool(env['COPILOT_OTEL_CAPTURE_CONTENT']) + ?? input.settingCaptureContent + ?? false; + + // Log level + const validLogLevels = new Set(['trace', 'debug', 'info', 'warn', 'error']); + const rawLogLevel = env['COPILOT_OTEL_LOG_LEVEL']; + const logLevel: OTelConfig['logLevel'] = rawLogLevel && validLogLevels.has(rawLogLevel as OTelConfig['logLevel']) + ? rawLogLevel as OTelConfig['logLevel'] + : 'info'; + + // HTTP instrumentation + const httpInstrumentation = envBool(env['COPILOT_OTEL_HTTP_INSTRUMENTATION']) ?? false; + + // Service name + const serviceName = env['OTEL_SERVICE_NAME'] ?? 'copilot-chat'; + + // Resource attributes + const resourceAttributes = parseResourceAttributes(env['OTEL_RESOURCE_ATTRIBUTES']); + + return Object.freeze({ + enabled: true, + exporterType, + otlpEndpoint, + otlpProtocol: protocol, + captureContent, + fileExporterPath, + logLevel, + httpInstrumentation, + serviceName, + serviceVersion: input.extensionVersion, + sessionId: input.sessionId, + resourceAttributes, + }); +} + +function createDisabledConfig(input: OTelConfigInput): OTelConfig { + return Object.freeze({ + enabled: false, + exporterType: 'otlp-http' as const, + otlpEndpoint: '', + otlpProtocol: 'http' as const, + captureContent: false, + logLevel: 'info' as const, + httpInstrumentation: false, + serviceName: 'copilot-chat', + serviceVersion: input.extensionVersion, + sessionId: input.sessionId, + resourceAttributes: {}, + }); +} + +function envBool(val: string | undefined): boolean | undefined { + if (val === undefined) { + return undefined; + } + return val === 'true' || val === '1'; +} diff --git a/extensions/copilot/src/platform/otel/common/otelService.ts b/extensions/copilot/src/platform/otel/common/otelService.ts new file mode 100644 index 00000000000..41db492178a --- /dev/null +++ b/extensions/copilot/src/platform/otel/common/otelService.ts @@ -0,0 +1,124 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { createServiceIdentifier } from '../../../util/common/services'; +import type { OTelConfig } from './otelConfig'; + +export const IOTelService = createServiceIdentifier('IOTelService'); + +/** + * Serializable trace context for cross-boundary span propagation. + */ +export interface TraceContext { + readonly traceId: string; + readonly spanId: string; +} + +/** + * Abstracts the OpenTelemetry SDK so consumers don't import OTel directly. + * When disabled, all methods are no-ops with zero overhead. + */ +export interface IOTelService { + readonly _serviceBrand: undefined; + readonly config: OTelConfig; + + /** + * Start a new span. Returns a handle to set attributes and end the span. + * If OTel is disabled, returns a no-op handle. + */ + startSpan(name: string, options?: SpanOptions): ISpanHandle; + + /** + * Start a span and set it as active context so child spans are parented. + * Calls `fn` within the active span context. + */ + startActiveSpan(name: string, options: SpanOptions, fn: (span: ISpanHandle) => Promise): Promise; + + /** + * Get the trace context (traceId + spanId) of the currently active span. + * Returns undefined if no span is active or OTel is disabled. + */ + getActiveTraceContext(): TraceContext | undefined; + + /** + * Store a trace context for later retrieval, keyed by a string ID. + * Used to propagate context across async boundaries (e.g., subagent invocations). + */ + storeTraceContext(key: string, context: TraceContext): void; + + /** + * Retrieve and remove a previously stored trace context by key. + */ + getStoredTraceContext(key: string): TraceContext | undefined; + + /** + * Run a function with a remote trace context set as active, without creating a span. + * Child spans created inside `fn` will be parented to the given trace context. + */ + runWithTraceContext(traceContext: TraceContext, fn: () => Promise): Promise; + + /** + * Record a histogram metric value. + */ + recordMetric(name: string, value: number, attributes?: Record): void; + + /** + * Increment a counter metric. + */ + incrementCounter(name: string, value?: number, attributes?: Record): void; + + /** + * Emit an OTel log record / event. + */ + emitLogRecord(body: string, attributes?: Record): void; + + /** + * Force flush all pending telemetry data. + */ + flush(): Promise; + + /** + * Gracefully shut down the OTel SDK. + */ + shutdown(): Promise; +} + +export const enum SpanKind { + INTERNAL = 0, + CLIENT = 2, +} + +export const enum SpanStatusCode { + UNSET = 0, + OK = 1, + ERROR = 2, +} + +export interface SpanOptions { + kind?: SpanKind; + attributes?: Record; + /** When set, the span will be created as a child of this remote trace context. */ + parentTraceContext?: TraceContext; +} + +/** + * Lightweight handle for a span, independent of the OTel SDK types. + */ +export interface ISpanHandle { + setAttribute(key: string, value: string | number | boolean | string[]): void; + setAttributes(attrs: Record): void; + setStatus(code: SpanStatusCode, message?: string): void; + recordException(error: unknown): void; + end(): void; +} + +/** + * Shape of `modelOptions` passed through VS Code IPC for cross-process + * CapturingToken restoration and OTel trace context propagation. + */ +export interface OTelModelOptions { + readonly _capturingTokenCorrelationId?: string; + readonly _otelTraceContext?: TraceContext | null; +} diff --git a/extensions/copilot/src/platform/otel/common/test/agentTraceHierarchy.spec.ts b/extensions/copilot/src/platform/otel/common/test/agentTraceHierarchy.spec.ts new file mode 100644 index 00000000000..fb6cb1579d7 --- /dev/null +++ b/extensions/copilot/src/platform/otel/common/test/agentTraceHierarchy.spec.ts @@ -0,0 +1,226 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { describe, expect, it } from 'vitest'; +import { GenAiAttr, GenAiOperationName, GenAiProviderName } from '../genAiAttributes'; +import { emitAgentTurnEvent, emitSessionStartEvent } from '../genAiEvents'; +import { GenAiMetrics } from '../genAiMetrics'; +import { SpanKind, SpanStatusCode } from '../otelService'; +import { CapturingOTelService } from './capturingOTelService'; + +/** + * Verifies that the OTel instrumentation produces the correct span hierarchy, + * metric recordings, and event emissions for a complete agent interaction. + * + * Span hierarchy (expected): + * invoke_agent copilot [INTERNAL] + * ├── chat gpt-4o [CLIENT] + * ├── execute_tool readFile [INTERNAL] + * └── chat gpt-4o [CLIENT] + * + * Subagent trace propagation (via storeTraceContext/getStoredTraceContext): + * invoke_agent copilot + * ├── execute_tool runSubagent + * │ └── invoke_agent Explore (same traceId via parentTraceContext) + */ +describe('Agent Trace Hierarchy', () => { + it('produces invoke_agent, chat, and execute_tool spans with correct attributes', async () => { + const otel = new CapturingOTelService(); + + // Simulate invoke_agent span + await otel.startActiveSpan('invoke_agent copilot', { + kind: SpanKind.INTERNAL, + attributes: { + [GenAiAttr.OPERATION_NAME]: GenAiOperationName.INVOKE_AGENT, + [GenAiAttr.PROVIDER_NAME]: GenAiProviderName.GITHUB, + [GenAiAttr.AGENT_NAME]: 'copilot', + [GenAiAttr.CONVERSATION_ID]: 'conv-123', + }, + }, async (agentSpan) => { + // Simulate chat span (LLM call) + const chatSpan = otel.startSpan('chat gpt-4o', { + kind: SpanKind.CLIENT, + attributes: { + [GenAiAttr.OPERATION_NAME]: GenAiOperationName.CHAT, + [GenAiAttr.REQUEST_MODEL]: 'gpt-4o', + }, + }); + chatSpan.setAttributes({ + [GenAiAttr.USAGE_INPUT_TOKENS]: 1500, + [GenAiAttr.USAGE_OUTPUT_TOKENS]: 250, + [GenAiAttr.RESPONSE_MODEL]: 'gpt-4o-2024-08-06', + }); + chatSpan.setStatus(SpanStatusCode.OK); + chatSpan.end(); + + // Simulate tool call span + const toolSpan = otel.startSpan('execute_tool readFile', { + kind: SpanKind.INTERNAL, + attributes: { + [GenAiAttr.OPERATION_NAME]: GenAiOperationName.EXECUTE_TOOL, + [GenAiAttr.TOOL_NAME]: 'readFile', + }, + }); + toolSpan.setStatus(SpanStatusCode.OK); + toolSpan.end(); + + // Simulate second chat span + const chat2 = otel.startSpan('chat gpt-4o', { + kind: SpanKind.CLIENT, + attributes: { + [GenAiAttr.OPERATION_NAME]: GenAiOperationName.CHAT, + [GenAiAttr.REQUEST_MODEL]: 'gpt-4o', + }, + }); + chat2.setStatus(SpanStatusCode.OK); + chat2.end(); + + agentSpan.setStatus(SpanStatusCode.OK); + }); + + // Verify all 4 spans created + expect(otel.spans).toHaveLength(4); + + // invoke_agent span + const agentSpan = otel.spans[0]; + expect(agentSpan.name).toBe('invoke_agent copilot'); + expect(agentSpan.kind).toBe(SpanKind.INTERNAL); + expect(agentSpan.attributes[GenAiAttr.OPERATION_NAME]).toBe('invoke_agent'); + expect(agentSpan.attributes[GenAiAttr.AGENT_NAME]).toBe('copilot'); + expect(agentSpan.statusCode).toBe(SpanStatusCode.OK); + expect(agentSpan.ended).toBe(true); + + // First chat span + const chatSpan = otel.spans[1]; + expect(chatSpan.name).toBe('chat gpt-4o'); + expect(chatSpan.kind).toBe(SpanKind.CLIENT); + expect(chatSpan.attributes[GenAiAttr.USAGE_INPUT_TOKENS]).toBe(1500); + expect(chatSpan.attributes[GenAiAttr.RESPONSE_MODEL]).toBe('gpt-4o-2024-08-06'); + + // Tool span + const toolSpan = otel.spans[2]; + expect(toolSpan.name).toBe('execute_tool readFile'); + expect(toolSpan.kind).toBe(SpanKind.INTERNAL); + expect(toolSpan.attributes[GenAiAttr.TOOL_NAME]).toBe('readFile'); + + // Second chat span + expect(otel.spans[3].name).toBe('chat gpt-4o'); + }); + + it('emits session start event and agent metrics', async () => { + const otel = new CapturingOTelService(); + + emitSessionStartEvent(otel, 'sess-abc', 'gpt-4o', 'copilot'); + GenAiMetrics.incrementSessionCount(otel); + GenAiMetrics.recordAgentDuration(otel, 'copilot', 15.2); + GenAiMetrics.recordAgentTurnCount(otel, 'copilot', 4); + emitAgentTurnEvent(otel, 0, 1500, 250, 2); + + // Session event + expect(otel.logRecords).toHaveLength(2); // session.start + agent.turn + expect(otel.logRecords[0].attributes?.['event.name']).toBe('copilot_chat.session.start'); + + // Agent turn event + expect(otel.logRecords[1].attributes?.['event.name']).toBe('copilot_chat.agent.turn'); + expect(otel.logRecords[1].attributes?.['turn.index']).toBe(0); + + // Metrics + expect(otel.counters).toHaveLength(1); + expect(otel.counters[0].name).toBe('copilot_chat.session.count'); + expect(otel.metrics).toHaveLength(2); + expect(otel.metrics[0].name).toBe('copilot_chat.agent.invocation.duration'); + expect(otel.metrics[1].name).toBe('copilot_chat.agent.turn.count'); + }); + + it('propagates trace context for subagent via store/retrieve', () => { + const otel = new CapturingOTelService(); + const parentCtx = { traceId: 'aaaa0000bbbb1111cccc2222dddd3333', spanId: 'eeee4444ffff5555' }; + + // Parent agent stores context when launching subagent + otel.storeTraceContext('subagent:req-123', parentCtx); + + // Subagent retrieves it + const restored = otel.getStoredTraceContext('subagent:req-123'); + expect(restored).toEqual(parentCtx); + + // Create subagent span with parentTraceContext + otel.startSpan('invoke_agent Explore', { + kind: SpanKind.INTERNAL, + attributes: { [GenAiAttr.OPERATION_NAME]: GenAiOperationName.INVOKE_AGENT }, + parentTraceContext: restored, + }); + + const subagentSpan = otel.spans[0]; + expect(subagentSpan.name).toBe('invoke_agent Explore'); + expect(subagentSpan.parentTraceContext).toEqual(parentCtx); + + // Context is consumed (single-use) + expect(otel.getStoredTraceContext('subagent:req-123')).toBeUndefined(); + }); + + it('records error status on failed spans', async () => { + const otel = new CapturingOTelService(); + + await otel.startActiveSpan('chat gpt-4o', { kind: SpanKind.CLIENT, attributes: {} }, async (span) => { + span.setStatus(SpanStatusCode.ERROR, 'timeout'); + span.setAttribute('error.type', 'TimeoutError'); + span.recordException(new Error('Request timed out')); + }); + + const span = otel.spans[0]; + expect(span.statusCode).toBe(SpanStatusCode.ERROR); + expect(span.statusMessage).toBe('timeout'); + expect(span.attributes['error.type']).toBe('TimeoutError'); + expect(span.exceptions).toHaveLength(1); + expect(span.ended).toBe(true); + }); + + it('records tool call metrics and events correctly', () => { + const otel = new CapturingOTelService(); + + // Simulate a successful and failed tool call + GenAiMetrics.recordToolCallCount(otel, 'readFile', true); + GenAiMetrics.recordToolCallDuration(otel, 'readFile', 50); + GenAiMetrics.recordToolCallCount(otel, 'runCommand', false); + GenAiMetrics.recordToolCallDuration(otel, 'runCommand', 5000); + + expect(otel.counters).toHaveLength(2); + expect(otel.counters[0].attributes?.[GenAiAttr.TOOL_NAME]).toBe('readFile'); + expect(otel.counters[0].attributes?.success).toBe(true); + expect(otel.counters[1].attributes?.success).toBe(false); + + expect(otel.metrics).toHaveLength(2); + expect(otel.metrics[0].value).toBe(50); + expect(otel.metrics[1].value).toBe(5000); + }); + + it('records chat operation duration and token usage metrics', () => { + const otel = new CapturingOTelService(); + + GenAiMetrics.recordOperationDuration(otel, 3.5, { + operationName: GenAiOperationName.CHAT, + providerName: GenAiProviderName.GITHUB, + requestModel: 'gpt-4o', + }); + GenAiMetrics.recordTokenUsage(otel, 1500, 'input', { + operationName: GenAiOperationName.CHAT, + providerName: GenAiProviderName.GITHUB, + requestModel: 'gpt-4o', + }); + GenAiMetrics.recordTokenUsage(otel, 250, 'output', { + operationName: GenAiOperationName.CHAT, + providerName: GenAiProviderName.GITHUB, + requestModel: 'gpt-4o', + }); + + expect(otel.metrics).toHaveLength(3); + expect(otel.metrics[0].name).toBe('gen_ai.client.operation.duration'); + expect(otel.metrics[0].value).toBe(3.5); + expect(otel.metrics[1].name).toBe('gen_ai.client.token.usage'); + expect(otel.metrics[1].value).toBe(1500); + expect(otel.metrics[2].name).toBe('gen_ai.client.token.usage'); + expect(otel.metrics[2].value).toBe(250); + }); +}); diff --git a/extensions/copilot/src/platform/otel/common/test/byokProviderSpans.spec.ts b/extensions/copilot/src/platform/otel/common/test/byokProviderSpans.spec.ts new file mode 100644 index 00000000000..273ea9087ff --- /dev/null +++ b/extensions/copilot/src/platform/otel/common/test/byokProviderSpans.spec.ts @@ -0,0 +1,133 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { describe, expect, it } from 'vitest'; +import { GenAiAttr, GenAiOperationName } from '../genAiAttributes'; +import { emitInferenceDetailsEvent } from '../genAiEvents'; +import { SpanKind, SpanStatusCode } from '../otelService'; +import { CapturingOTelService } from './capturingOTelService'; + +/** + * Tests BYOK-style span emission patterns — verifying that chat spans + * are created with correct kind, attributes, status codes, and that + * content capture is properly gated on config.captureContent. + * + * These validate the instrumentation patterns used in anthropicProvider, + * geminiNativeProvider, and chatMLFetcher. + */ +describe('BYOK Provider Span Emission', () => { + it('creates chat span with CLIENT kind and model attributes', () => { + const otel = new CapturingOTelService(); + + const span = otel.startSpan('chat claude-sonnet-4-20250514', { + kind: SpanKind.CLIENT, + attributes: { + [GenAiAttr.OPERATION_NAME]: GenAiOperationName.CHAT, + [GenAiAttr.PROVIDER_NAME]: 'anthropic', + [GenAiAttr.REQUEST_MODEL]: 'claude-sonnet-4-20250514', + [GenAiAttr.AGENT_NAME]: 'AnthropicBYOK', + }, + }); + span.setAttributes({ + [GenAiAttr.USAGE_INPUT_TOKENS]: 2000, + [GenAiAttr.USAGE_OUTPUT_TOKENS]: 500, + [GenAiAttr.RESPONSE_MODEL]: 'claude-sonnet-4-20250514', + [GenAiAttr.RESPONSE_ID]: 'msg_abc123', + }); + span.setStatus(SpanStatusCode.OK); + span.end(); + + expect(otel.spans).toHaveLength(1); + const s = otel.spans[0]; + expect(s.kind).toBe(SpanKind.CLIENT); + expect(s.attributes[GenAiAttr.OPERATION_NAME]).toBe('chat'); + expect(s.attributes[GenAiAttr.PROVIDER_NAME]).toBe('anthropic'); + expect(s.attributes[GenAiAttr.USAGE_INPUT_TOKENS]).toBe(2000); + expect(s.attributes[GenAiAttr.USAGE_OUTPUT_TOKENS]).toBe(500); + expect(s.statusCode).toBe(SpanStatusCode.OK); + expect(s.ended).toBe(true); + }); + + it('sets ERROR status and error.type on failure', () => { + const otel = new CapturingOTelService(); + + const span = otel.startSpan('chat gemini-2.0-flash', { + kind: SpanKind.CLIENT, + attributes: { + [GenAiAttr.OPERATION_NAME]: GenAiOperationName.CHAT, + [GenAiAttr.PROVIDER_NAME]: 'gemini', + [GenAiAttr.REQUEST_MODEL]: 'gemini-2.0-flash', + }, + }); + span.setStatus(SpanStatusCode.ERROR, 'Rate limit exceeded'); + span.setAttribute('error.type', 'RateLimitError'); + span.recordException(new Error('Rate limit exceeded')); + span.end(); + + const s = otel.spans[0]; + expect(s.statusCode).toBe(SpanStatusCode.ERROR); + expect(s.statusMessage).toBe('Rate limit exceeded'); + expect(s.attributes['error.type']).toBe('RateLimitError'); + expect(s.exceptions).toHaveLength(1); + }); + + it('does NOT capture content when captureContent is false', () => { + const otel = new CapturingOTelService({ captureContent: false }); + + // Simulate the input capture gating pattern used in BYOK providers + const span = otel.startSpan('chat gpt-4o', { kind: SpanKind.CLIENT, attributes: {} }); + if (otel.config.captureContent) { + span.setAttribute(GenAiAttr.INPUT_MESSAGES, 'should not appear'); + } + span.end(); + + expect(otel.spans[0].attributes[GenAiAttr.INPUT_MESSAGES]).toBeUndefined(); + }); + + it('captures content when captureContent is true', () => { + const otel = new CapturingOTelService({ captureContent: true }); + + const span = otel.startSpan('chat gpt-4o', { kind: SpanKind.CLIENT, attributes: {} }); + if (otel.config.captureContent) { + span.setAttribute(GenAiAttr.INPUT_MESSAGES, '[{"role":"user","parts":[{"type":"text","content":"hello"}]}]'); + span.setAttribute(GenAiAttr.OUTPUT_MESSAGES, '[{"role":"assistant","parts":[{"type":"text","content":"hi"}]}]'); + } + span.end(); + + expect(otel.spans[0].attributes[GenAiAttr.INPUT_MESSAGES]).toBeDefined(); + expect(otel.spans[0].attributes[GenAiAttr.OUTPUT_MESSAGES]).toBeDefined(); + }); + + it('emits inference details event with request/response data', () => { + const otel = new CapturingOTelService(); + + emitInferenceDetailsEvent( + otel, + { model: 'claude-sonnet-4-20250514', temperature: 0.1, maxTokens: 4096 }, + { id: 'msg_123', model: 'claude-sonnet-4-20250514', finishReasons: ['stop'], inputTokens: 2000, outputTokens: 500 }, + ); + + expect(otel.logRecords).toHaveLength(1); + const attrs = otel.logRecords[0].attributes!; + expect(attrs['event.name']).toBe('gen_ai.client.inference.operation.details'); + expect(attrs[GenAiAttr.REQUEST_MODEL]).toBe('claude-sonnet-4-20250514'); + expect(attrs[GenAiAttr.USAGE_INPUT_TOKENS]).toBe(2000); + expect(attrs[GenAiAttr.USAGE_OUTPUT_TOKENS]).toBe(500); + }); + + it('uses parentTraceContext for CAPI → BYOK trace linking', () => { + const otel = new CapturingOTelService(); + const parentCtx = { traceId: '11112222333344445555666677778888', spanId: 'aabbccddeeff0011' }; + + const span = otel.startSpan('chat gpt-4o', { + kind: SpanKind.CLIENT, + attributes: { [GenAiAttr.OPERATION_NAME]: GenAiOperationName.CHAT }, + parentTraceContext: parentCtx, + }); + span.end(); + + expect(otel.spans[0].parentTraceContext).toEqual(parentCtx); + }); +}); diff --git a/extensions/copilot/src/platform/otel/common/test/capturingOTelService.ts b/extensions/copilot/src/platform/otel/common/test/capturingOTelService.ts new file mode 100644 index 00000000000..1eeca6f70c2 --- /dev/null +++ b/extensions/copilot/src/platform/otel/common/test/capturingOTelService.ts @@ -0,0 +1,160 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { resolveOTelConfig, type OTelConfig } from '../otelConfig'; +import { SpanStatusCode, type IOTelService, type ISpanHandle, type SpanOptions, type TraceContext } from '../otelService'; + +/** + * Captured span record for test assertions. + */ +export interface CapturedSpan { + name: string; + kind?: number; + attributes: Record; + statusCode?: SpanStatusCode; + statusMessage?: string; + exceptions: unknown[]; + ended: boolean; + parentTraceContext?: TraceContext; +} + +/** + * Captured metric record. + */ +export interface CapturedMetric { + name: string; + value: number; + attributes?: Record; +} + +/** + * Captured log record. + */ +export interface CapturedLogRecord { + body: string; + attributes?: Record; +} + +/** + * IOTelService implementation that captures all operations for test verification. + * Unlike NoopOTelService, this records spans, metrics, and logs so tests can + * assert on the OTel output without a real SDK. + */ +export class CapturingOTelService implements IOTelService { + declare readonly _serviceBrand: undefined; + readonly config: OTelConfig; + + readonly spans: CapturedSpan[] = []; + readonly metrics: CapturedMetric[] = []; + readonly counters: CapturedMetric[] = []; + readonly logRecords: CapturedLogRecord[] = []; + private readonly _traceContextStore = new Map(); + + constructor(config?: Partial) { + this.config = { + ...resolveOTelConfig({ env: { 'COPILOT_OTEL_ENABLED': 'true' }, extensionVersion: '1.0.0', sessionId: 'test' }), + ...config, + }; + } + + startSpan(name: string, options?: SpanOptions): ISpanHandle { + const captured: CapturedSpan = { + name, + kind: options?.kind, + attributes: { ...options?.attributes }, + exceptions: [], + ended: false, + parentTraceContext: options?.parentTraceContext, + }; + this.spans.push(captured); + return new CapturingSpanHandle(captured); + } + + async startActiveSpan(name: string, options: SpanOptions, fn: (span: ISpanHandle) => Promise): Promise { + const span = this.startSpan(name, options); + try { + return await fn(span); + } finally { + span.end(); + } + } + + getActiveTraceContext(): TraceContext | undefined { + return undefined; + } + + storeTraceContext(key: string, context: TraceContext): void { + this._traceContextStore.set(key, context); + } + + getStoredTraceContext(key: string): TraceContext | undefined { + const ctx = this._traceContextStore.get(key); + if (ctx) { + this._traceContextStore.delete(key); + } + return ctx; + } + + async runWithTraceContext(_traceContext: TraceContext, fn: () => Promise): Promise { + return fn(); + } + + recordMetric(name: string, value: number, attributes?: Record): void { + this.metrics.push({ name, value, attributes }); + } + + incrementCounter(name: string, value = 1, attributes?: Record): void { + this.counters.push({ name, value, attributes }); + } + + emitLogRecord(body: string, attributes?: Record): void { + this.logRecords.push({ body, attributes }); + } + + async flush(): Promise { } + async shutdown(): Promise { } + + /** Find spans by name prefix. */ + findSpans(namePrefix: string): CapturedSpan[] { + return this.spans.filter(s => s.name.startsWith(namePrefix)); + } + + /** Reset all captured data. */ + reset(): void { + this.spans.length = 0; + this.metrics.length = 0; + this.counters.length = 0; + this.logRecords.length = 0; + } +} + +class CapturingSpanHandle implements ISpanHandle { + constructor(private readonly _captured: CapturedSpan) { } + + setAttribute(key: string, value: string | number | boolean | string[]): void { + this._captured.attributes[key] = value; + } + + setAttributes(attrs: Record): void { + for (const k in attrs) { + if (Object.prototype.hasOwnProperty.call(attrs, k)) { + this._captured.attributes[k] = attrs[k]; + } + } + } + + setStatus(code: SpanStatusCode, message?: string): void { + this._captured.statusCode = code; + this._captured.statusMessage = message; + } + + recordException(error: unknown): void { + this._captured.exceptions.push(error); + } + + end(): void { + this._captured.ended = true; + } +} diff --git a/extensions/copilot/src/platform/otel/common/test/chatMLFetcherSpanLifecycle.spec.ts b/extensions/copilot/src/platform/otel/common/test/chatMLFetcherSpanLifecycle.spec.ts new file mode 100644 index 00000000000..e21d6d84a2f --- /dev/null +++ b/extensions/copilot/src/platform/otel/common/test/chatMLFetcherSpanLifecycle.spec.ts @@ -0,0 +1,112 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { describe, expect, it } from 'vitest'; +import { CopilotChatAttr, GenAiAttr, GenAiOperationName, GenAiProviderName } from '../genAiAttributes'; +import { SpanKind, SpanStatusCode } from '../otelService'; +import { CapturingOTelService } from './capturingOTelService'; + +/** + * Tests the two-phase span lifecycle used in chatMLFetcher: + * 1. _doFetch creates a span, returns it alongside the result + * 2. fetchMany enriches it with token usage and response data, then ends it + * + * This pattern is unique because the span is created in one method + * and ended in another — testing lifecycle correctness. + */ +describe('chatMLFetcher Span Lifecycle', () => { + it('span is created with model and conversation ID in _doFetch phase', () => { + const otel = new CapturingOTelService(); + + // Phase 1: _doFetch creates the span + const span = otel.startSpan('chat gpt-4o', { + kind: SpanKind.CLIENT, + attributes: { + [GenAiAttr.OPERATION_NAME]: GenAiOperationName.CHAT, + [GenAiAttr.PROVIDER_NAME]: GenAiProviderName.GITHUB, + [GenAiAttr.REQUEST_MODEL]: 'gpt-4o', + [GenAiAttr.CONVERSATION_ID]: 'req-abc', + [GenAiAttr.REQUEST_MAX_TOKENS]: 2048, + [CopilotChatAttr.MAX_PROMPT_TOKENS]: 128000, + }, + }); + + const s = otel.spans[0]; + expect(s.name).toBe('chat gpt-4o'); + expect(s.kind).toBe(SpanKind.CLIENT); + expect(s.attributes[GenAiAttr.REQUEST_MODEL]).toBe('gpt-4o'); + expect(s.attributes[GenAiAttr.CONVERSATION_ID]).toBe('req-abc'); + expect(s.ended).toBe(false); + + // Phase 2: fetchMany enriches with response data + span.setAttributes({ + [GenAiAttr.USAGE_INPUT_TOKENS]: 1500, + [GenAiAttr.USAGE_OUTPUT_TOKENS]: 250, + [GenAiAttr.RESPONSE_MODEL]: 'gpt-4o-2024-08-06', + [GenAiAttr.RESPONSE_ID]: 'chatcmpl-xyz', + [GenAiAttr.RESPONSE_FINISH_REASONS]: ['stop'], + [CopilotChatAttr.TIME_TO_FIRST_TOKEN]: 450, + }); + span.setStatus(SpanStatusCode.OK); + span.end(); + + expect(s.attributes[GenAiAttr.USAGE_INPUT_TOKENS]).toBe(1500); + expect(s.attributes[GenAiAttr.RESPONSE_MODEL]).toBe('gpt-4o-2024-08-06'); + expect(s.statusCode).toBe(SpanStatusCode.OK); + expect(s.ended).toBe(true); + }); + + it('span is ended on error path (not leaked)', () => { + const otel = new CapturingOTelService(); + + // Phase 1: span created + const span = otel.startSpan('chat gpt-4o', { + kind: SpanKind.CLIENT, + attributes: { [GenAiAttr.OPERATION_NAME]: GenAiOperationName.CHAT }, + }); + + // Phase 2: error occurs — fetchMany Error path + span.setStatus(SpanStatusCode.ERROR, 'Connection reset'); + span.setAttribute('error.type', 'FetchError'); + span.recordException(new Error('Connection reset')); + span.end(); + + const s = otel.spans[0]; + expect(s.statusCode).toBe(SpanStatusCode.ERROR); + expect(s.ended).toBe(true); + expect(s.exceptions).toHaveLength(1); + }); + + it('operation duration metric is recorded in _doFetch finally block', () => { + const otel = new CapturingOTelService(); + + // Simulate the finally block in _doFetch + const durationSec = 3.5; + otel.recordMetric('gen_ai.client.operation.duration', durationSec, { + [GenAiAttr.OPERATION_NAME]: GenAiOperationName.CHAT, + [GenAiAttr.PROVIDER_NAME]: GenAiProviderName.GITHUB, + [GenAiAttr.REQUEST_MODEL]: 'gpt-4o', + }); + + expect(otel.metrics).toHaveLength(1); + expect(otel.metrics[0].name).toBe('gen_ai.client.operation.duration'); + expect(otel.metrics[0].value).toBe(3.5); + }); + + it('debug name attribute is set after span is returned to fetchMany', () => { + const otel = new CapturingOTelService(); + + const span = otel.startSpan('chat gpt-4o', { + kind: SpanKind.CLIENT, + attributes: { [GenAiAttr.OPERATION_NAME]: GenAiOperationName.CHAT }, + }); + + // fetchMany adds debug name after receiving the span from _doFetch + span.setAttribute(GenAiAttr.AGENT_NAME, 'agentMode'); + span.end(); + + expect(otel.spans[0].attributes[GenAiAttr.AGENT_NAME]).toBe('agentMode'); + }); +}); diff --git a/extensions/copilot/src/platform/otel/common/test/genAiEvents.spec.ts b/extensions/copilot/src/platform/otel/common/test/genAiEvents.spec.ts new file mode 100644 index 00000000000..b3a89c0b2cd --- /dev/null +++ b/extensions/copilot/src/platform/otel/common/test/genAiEvents.spec.ts @@ -0,0 +1,160 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { describe, expect, it, vi } from 'vitest'; +import { GenAiAttr, GenAiOperationName, StdAttr } from '../genAiAttributes'; +import { emitAgentTurnEvent, emitInferenceDetailsEvent, emitSessionStartEvent, emitToolCallEvent } from '../genAiEvents'; +import { resolveOTelConfig } from '../otelConfig'; +import type { IOTelService } from '../otelService'; + +function createMockOTel(captureContent = false): IOTelService & { emitLogRecord: ReturnType } { + const config = resolveOTelConfig({ + env: captureContent ? { 'COPILOT_OTEL_ENABLED': 'true', 'COPILOT_OTEL_CAPTURE_CONTENT': 'true' } : { 'COPILOT_OTEL_ENABLED': 'true' }, + extensionVersion: '1.0.0', + sessionId: 'test', + }); + return { + _serviceBrand: undefined!, + config, + startSpan: vi.fn(), + startActiveSpan: vi.fn(), + getActiveTraceContext: vi.fn(), + storeTraceContext: vi.fn(), + getStoredTraceContext: vi.fn(), + runWithTraceContext: vi.fn((_ctx: any, fn: any) => fn()), + recordMetric: vi.fn(), + incrementCounter: vi.fn(), + emitLogRecord: vi.fn(), + flush: vi.fn(), + shutdown: vi.fn(), + }; +} + +describe('emitInferenceDetailsEvent', () => { + it('emits event with standard attributes', () => { + const otel = createMockOTel(); + emitInferenceDetailsEvent(otel, + { model: 'gpt-4o', temperature: 0.7, maxTokens: 4096 }, + { id: 'resp-1', model: 'gpt-4o', finishReasons: ['stop'], inputTokens: 100, outputTokens: 50 }, + ); + + expect(otel.emitLogRecord).toHaveBeenCalledOnce(); + const [body, attrs] = otel.emitLogRecord.mock.calls[0]; + expect(body).toContain('gpt-4o'); + expect(attrs['event.name']).toBe('gen_ai.client.inference.operation.details'); + expect(attrs[GenAiAttr.OPERATION_NAME]).toBe(GenAiOperationName.CHAT); + expect(attrs[GenAiAttr.REQUEST_MODEL]).toBe('gpt-4o'); + expect(attrs[GenAiAttr.RESPONSE_MODEL]).toBe('gpt-4o'); + expect(attrs[GenAiAttr.RESPONSE_ID]).toBe('resp-1'); + expect(attrs[GenAiAttr.USAGE_INPUT_TOKENS]).toBe(100); + expect(attrs[GenAiAttr.USAGE_OUTPUT_TOKENS]).toBe(50); + expect(attrs[GenAiAttr.REQUEST_TEMPERATURE]).toBe(0.7); + expect(attrs[GenAiAttr.REQUEST_MAX_TOKENS]).toBe(4096); + }); + + it('does not include content attributes when captureContent is false', () => { + const otel = createMockOTel(false); + emitInferenceDetailsEvent(otel, + { model: 'gpt-4o', messages: [{ role: 'user', text: 'secret' }] }, + { id: 'resp-1' }, + ); + + const attrs = otel.emitLogRecord.mock.calls[0][1]; + expect(attrs).not.toHaveProperty(GenAiAttr.INPUT_MESSAGES); + expect(attrs).not.toHaveProperty(GenAiAttr.SYSTEM_INSTRUCTIONS); + expect(attrs).not.toHaveProperty(GenAiAttr.TOOL_DEFINITIONS); + }); + + it('includes content attributes when captureContent is true', () => { + const otel = createMockOTel(true); + const messages = [{ role: 'user', text: 'hello' }]; + const systemMsg = 'You are helpful'; + const tools = [{ name: 'readFile' }]; + + emitInferenceDetailsEvent(otel, + { model: 'gpt-4o', messages, systemMessage: systemMsg, tools }, + undefined, + ); + + const attrs = otel.emitLogRecord.mock.calls[0][1]; + expect(attrs[GenAiAttr.INPUT_MESSAGES]).toBe(JSON.stringify(messages)); + expect(attrs[GenAiAttr.SYSTEM_INSTRUCTIONS]).toBe(JSON.stringify(systemMsg)); + expect(attrs[GenAiAttr.TOOL_DEFINITIONS]).toBe(JSON.stringify(tools)); + }); + + it('includes error.type when error is provided', () => { + const otel = createMockOTel(); + emitInferenceDetailsEvent(otel, + { model: 'gpt-4o' }, + undefined, + { type: 'TimeoutError', message: 'request timed out' }, + ); + + const attrs = otel.emitLogRecord.mock.calls[0][1]; + expect(attrs[StdAttr.ERROR_TYPE]).toBe('TimeoutError'); + }); + + it('handles undefined response', () => { + const otel = createMockOTel(); + emitInferenceDetailsEvent(otel, { model: 'gpt-4o' }, undefined); + + const attrs = otel.emitLogRecord.mock.calls[0][1]; + expect(attrs).not.toHaveProperty(GenAiAttr.RESPONSE_MODEL); + expect(attrs).not.toHaveProperty(GenAiAttr.RESPONSE_ID); + }); +}); + +describe('emitSessionStartEvent', () => { + it('emits session start with required attributes', () => { + const otel = createMockOTel(); + emitSessionStartEvent(otel, 'sess-123', 'gpt-4o', 'copilot'); + + expect(otel.emitLogRecord).toHaveBeenCalledWith('copilot_chat.session.start', { + 'event.name': 'copilot_chat.session.start', + 'session.id': 'sess-123', + [GenAiAttr.REQUEST_MODEL]: 'gpt-4o', + [GenAiAttr.AGENT_NAME]: 'copilot', + }); + }); +}); + +describe('emitToolCallEvent', () => { + it('emits success tool call event', () => { + const otel = createMockOTel(); + emitToolCallEvent(otel, 'readFile', 150, true); + + const [body, attrs] = otel.emitLogRecord.mock.calls[0]; + expect(body).toContain('readFile'); + expect(attrs['event.name']).toBe('copilot_chat.tool.call'); + expect(attrs[GenAiAttr.TOOL_NAME]).toBe('readFile'); + expect(attrs['duration_ms']).toBe(150); + expect(attrs['success']).toBe(true); + expect(attrs).not.toHaveProperty(StdAttr.ERROR_TYPE); + }); + + it('includes error type on failure', () => { + const otel = createMockOTel(); + emitToolCallEvent(otel, 'runCommand', 5000, false, 'TimeoutError'); + + const attrs = otel.emitLogRecord.mock.calls[0][1]; + expect(attrs['success']).toBe(false); + expect(attrs[StdAttr.ERROR_TYPE]).toBe('TimeoutError'); + }); +}); + +describe('emitAgentTurnEvent', () => { + it('emits turn event with all attributes', () => { + const otel = createMockOTel(); + emitAgentTurnEvent(otel, 3, 500, 200, 2); + + const [body, attrs] = otel.emitLogRecord.mock.calls[0]; + expect(body).toContain('3'); + expect(attrs['event.name']).toBe('copilot_chat.agent.turn'); + expect(attrs['turn.index']).toBe(3); + expect(attrs[GenAiAttr.USAGE_INPUT_TOKENS]).toBe(500); + expect(attrs[GenAiAttr.USAGE_OUTPUT_TOKENS]).toBe(200); + expect(attrs['tool_call_count']).toBe(2); + }); +}); diff --git a/extensions/copilot/src/platform/otel/common/test/genAiMetrics.spec.ts b/extensions/copilot/src/platform/otel/common/test/genAiMetrics.spec.ts new file mode 100644 index 00000000000..7904af988b1 --- /dev/null +++ b/extensions/copilot/src/platform/otel/common/test/genAiMetrics.spec.ts @@ -0,0 +1,124 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { describe, expect, it, vi } from 'vitest'; +import { GenAiAttr, GenAiOperationName, GenAiProviderName, GenAiTokenType, StdAttr } from '../genAiAttributes'; +import { GenAiMetrics } from '../genAiMetrics'; +import { resolveOTelConfig } from '../otelConfig'; +import type { IOTelService } from '../otelService'; + +function createMockOTelService(): IOTelService & { recordMetric: ReturnType; incrementCounter: ReturnType } { + const config = resolveOTelConfig({ env: {}, extensionVersion: '1.0.0', sessionId: 'test' }); + return { + _serviceBrand: undefined!, + config, + startSpan: vi.fn(), + startActiveSpan: vi.fn(), + getActiveTraceContext: vi.fn(), + storeTraceContext: vi.fn(), + getStoredTraceContext: vi.fn(), + runWithTraceContext: vi.fn((_ctx: any, fn: any) => fn()), + recordMetric: vi.fn(), + incrementCounter: vi.fn(), + emitLogRecord: vi.fn(), + flush: vi.fn(), + shutdown: vi.fn(), + }; +} + +describe('GenAiMetrics', () => { + it('recordOperationDuration calls recordMetric with correct attributes', () => { + const otel = createMockOTelService(); + + GenAiMetrics.recordOperationDuration(otel, 1.5, { + operationName: GenAiOperationName.CHAT, + providerName: GenAiProviderName.OPENAI, + requestModel: 'gpt-4o', + responseModel: 'gpt-4o-2024-05-13', + serverAddress: 'api.copilot.com', + errorType: 'timeout', + }); + + expect(otel.recordMetric).toHaveBeenCalledWith('gen_ai.client.operation.duration', 1.5, { + [GenAiAttr.OPERATION_NAME]: 'chat', + [GenAiAttr.PROVIDER_NAME]: 'openai', + [GenAiAttr.REQUEST_MODEL]: 'gpt-4o', + [GenAiAttr.RESPONSE_MODEL]: 'gpt-4o-2024-05-13', + [StdAttr.SERVER_ADDRESS]: 'api.copilot.com', + [StdAttr.ERROR_TYPE]: 'timeout', + }); + }); + + it('recordTokenUsage calls recordMetric with token type', () => { + const otel = createMockOTelService(); + + GenAiMetrics.recordTokenUsage(otel, 1000, 'input', { + operationName: GenAiOperationName.CHAT, + providerName: GenAiProviderName.OPENAI, + requestModel: 'gpt-4o', + }); + + expect(otel.recordMetric).toHaveBeenCalledWith('gen_ai.client.token.usage', 1000, { + [GenAiAttr.OPERATION_NAME]: 'chat', + [GenAiAttr.PROVIDER_NAME]: 'openai', + [GenAiAttr.TOKEN_TYPE]: GenAiTokenType.INPUT, + [GenAiAttr.REQUEST_MODEL]: 'gpt-4o', + }); + }); + + it('recordToolCallCount increments counter', () => { + const otel = createMockOTelService(); + + GenAiMetrics.recordToolCallCount(otel, 'readFile', true); + + expect(otel.incrementCounter).toHaveBeenCalledWith('copilot_chat.tool.call.count', 1, { + [GenAiAttr.TOOL_NAME]: 'readFile', + success: true, + }); + }); + + it('recordToolCallDuration records histogram', () => { + const otel = createMockOTelService(); + + GenAiMetrics.recordToolCallDuration(otel, 'runCommand', 500); + + expect(otel.recordMetric).toHaveBeenCalledWith('copilot_chat.tool.call.duration', 500, { + [GenAiAttr.TOOL_NAME]: 'runCommand', + }); + }); + + it('recordAgentDuration records histogram', () => { + const otel = createMockOTelService(); + + GenAiMetrics.recordAgentDuration(otel, 'copilot', 15.2); + + expect(otel.recordMetric).toHaveBeenCalledWith('copilot_chat.agent.invocation.duration', 15.2, { + [GenAiAttr.AGENT_NAME]: 'copilot', + }); + }); + + it('incrementSessionCount increments counter', () => { + const otel = createMockOTelService(); + + GenAiMetrics.incrementSessionCount(otel); + + expect(otel.incrementCounter).toHaveBeenCalledWith('copilot_chat.session.count'); + }); + + it('omits optional attributes when not provided', () => { + const otel = createMockOTelService(); + + GenAiMetrics.recordOperationDuration(otel, 0.5, { + operationName: GenAiOperationName.CHAT, + providerName: GenAiProviderName.OPENAI, + requestModel: 'gpt-4o', + }); + + const attrs = otel.recordMetric.mock.calls[0][2]; + expect(attrs).not.toHaveProperty(GenAiAttr.RESPONSE_MODEL); + expect(attrs).not.toHaveProperty(StdAttr.SERVER_ADDRESS); + expect(attrs).not.toHaveProperty(StdAttr.ERROR_TYPE); + }); +}); diff --git a/extensions/copilot/src/platform/otel/common/test/messageFormatters.spec.ts b/extensions/copilot/src/platform/otel/common/test/messageFormatters.spec.ts new file mode 100644 index 00000000000..572fbd43647 --- /dev/null +++ b/extensions/copilot/src/platform/otel/common/test/messageFormatters.spec.ts @@ -0,0 +1,204 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { describe, expect, it } from 'vitest'; +import { toInputMessages, toOutputMessages, toSystemInstructions, toToolDefinitions, truncateForOTel } from '../messageFormatters'; + +describe('toInputMessages', () => { + it('converts a simple text message', () => { + const result = toInputMessages([{ role: 'user', content: 'Hello' }]); + expect(result).toEqual([{ + role: 'user', + parts: [{ type: 'text', content: 'Hello' }], + }]); + }); + + it('converts messages with tool calls', () => { + const result = toInputMessages([{ + role: 'assistant', + content: '', + tool_calls: [{ + id: 'tc_1', + function: { name: 'readFile', arguments: '{"path":"foo.ts"}' }, + }], + }]); + expect(result).toEqual([{ + role: 'assistant', + parts: [{ + type: 'tool_call', + id: 'tc_1', + name: 'readFile', + arguments: { path: 'foo.ts' }, + }], + }]); + }); + + it('handles invalid JSON in tool call arguments gracefully', () => { + const result = toInputMessages([{ + role: 'assistant', + tool_calls: [{ + id: 'tc_2', + function: { name: 'run', arguments: 'not-json' }, + }], + }]); + expect(result[0].parts[0]).toEqual({ + type: 'tool_call', + id: 'tc_2', + name: 'run', + arguments: 'not-json', + }); + }); + + it('includes both text and tool calls in parts', () => { + const result = toInputMessages([{ + role: 'assistant', + content: 'Here is the result', + tool_calls: [{ id: 'tc', function: { name: 'search', arguments: '{}' } }], + }]); + expect(result[0].parts).toHaveLength(2); + expect(result[0].parts[0]).toEqual({ type: 'text', content: 'Here is the result' }); + expect(result[0].parts[1]).toMatchObject({ type: 'tool_call', name: 'search' }); + }); + + it('handles empty messages array', () => { + expect(toInputMessages([])).toEqual([]); + }); + + it('preserves undefined role', () => { + const result = toInputMessages([{ content: 'no role' }]); + expect(result[0].role).toBeUndefined(); + }); +}); + +describe('toOutputMessages', () => { + it('converts successful text response', () => { + const result = toOutputMessages([{ + message: { role: 'assistant', content: 'Hello!' }, + finish_reason: 'stop', + }]); + expect(result).toEqual([{ + role: 'assistant', + parts: [{ type: 'text', content: 'Hello!' }], + finish_reason: 'stop', + }]); + }); + + it('converts response with tool calls', () => { + const result = toOutputMessages([{ + message: { + content: '', + tool_calls: [{ + id: 'tc_1', + function: { name: 'readFile', arguments: '{"path":"a.ts"}' }, + }], + }, + finish_reason: 'tool_calls', + }]); + expect(result[0].parts).toHaveLength(1); + expect(result[0].parts[0]).toMatchObject({ type: 'tool_call', name: 'readFile' }); + expect(result[0].finish_reason).toBe('tool_calls'); + }); + + it('defaults role to assistant when message has no role', () => { + const result = toOutputMessages([{ message: { content: 'text' }, finish_reason: 'stop' }]); + expect(result[0].role).toBe('assistant'); + }); + + it('defaults role to assistant when message is undefined', () => { + const result = toOutputMessages([{ finish_reason: 'stop' }]); + expect(result[0].role).toBe('assistant'); + expect(result[0].parts).toEqual([]); + }); + + it('handles empty choices array', () => { + expect(toOutputMessages([])).toEqual([]); + }); +}); + +describe('toSystemInstructions', () => { + it('converts a system message string', () => { + expect(toSystemInstructions('You are a helpful assistant')).toEqual([ + { type: 'text', content: 'You are a helpful assistant' }, + ]); + }); + + it('returns undefined for empty string', () => { + expect(toSystemInstructions('')).toBeUndefined(); + }); + + it('returns undefined for undefined input', () => { + expect(toSystemInstructions(undefined)).toBeUndefined(); + }); +}); + +describe('toToolDefinitions', () => { + it('converts tool definitions', () => { + const result = toToolDefinitions([{ + type: 'function', + function: { + name: 'readFile', + description: 'Read a file', + parameters: { type: 'object', properties: { path: { type: 'string' } } }, + }, + }]); + expect(result).toEqual([{ + type: 'function', + name: 'readFile', + description: 'Read a file', + parameters: { type: 'object', properties: { path: { type: 'string' } } }, + }]); + }); + + it('filters out tools without a function property', () => { + const result = toToolDefinitions([ + { type: 'function', function: { name: 'a' } }, + { type: 'function' }, // no function + ]); + expect(result).toHaveLength(1); + expect(result![0].name).toBe('a'); + }); + + it('returns undefined for empty array', () => { + expect(toToolDefinitions([])).toBeUndefined(); + }); + + it('returns undefined for undefined input', () => { + expect(toToolDefinitions(undefined)).toBeUndefined(); + }); +}); + +describe('truncateForOTel', () => { + it('returns short strings unchanged', () => { + expect(truncateForOTel('hello')).toBe('hello'); + }); + + it('returns empty string unchanged', () => { + expect(truncateForOTel('')).toBe(''); + }); + + it('returns string at exact limit unchanged', () => { + const s = 'a'.repeat(100); + expect(truncateForOTel(s, 100)).toBe(s); + }); + + it('truncates strings over the limit with suffix', () => { + const s = 'a'.repeat(200); + const result = truncateForOTel(s, 100); + expect(result.length).toBeLessThanOrEqual(100); + expect(result).toContain('...[truncated, original 200 chars]'); + }); + + it('uses default 64000 limit', () => { + const s = 'x'.repeat(64_001); + const result = truncateForOTel(s); + expect(result.length).toBeLessThanOrEqual(64_000); + expect(result).toContain('...[truncated'); + }); + + it('does not truncate at exactly 64000', () => { + const s = 'x'.repeat(64_000); + expect(truncateForOTel(s)).toBe(s); + }); +}); diff --git a/extensions/copilot/src/platform/otel/common/test/noopOtelService.spec.ts b/extensions/copilot/src/platform/otel/common/test/noopOtelService.spec.ts new file mode 100644 index 00000000000..590df2535b8 --- /dev/null +++ b/extensions/copilot/src/platform/otel/common/test/noopOtelService.spec.ts @@ -0,0 +1,57 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { describe, expect, it } from 'vitest'; +import { NoopOTelService } from '../noopOtelService'; +import { resolveOTelConfig } from '../otelConfig'; +import { SpanStatusCode } from '../otelService'; + +describe('NoopOTelService', () => { + const config = resolveOTelConfig({ env: {}, extensionVersion: '1.0.0', sessionId: 'test' }); + const service = new NoopOTelService(config); + + it('has disabled config', () => { + expect(service.config.enabled).toBe(false); + }); + + it('startSpan returns a noop handle', () => { + const span = service.startSpan('test-span', { attributes: { foo: 'bar' } }); + // All methods should be callable without error + span.setAttribute('key', 'value'); + span.setAttributes({ a: 1, b: 'c' }); + span.setStatus(SpanStatusCode.OK); + span.setStatus(SpanStatusCode.ERROR, 'msg'); + span.recordException(new Error('test')); + span.end(); + }); + + it('startActiveSpan runs the function and returns its result', async () => { + const result = await service.startActiveSpan('test', { attributes: {} }, async (span) => { + span.setAttribute('key', 'val'); + return 42; + }); + expect(result).toBe(42); + }); + + it('recordMetric is a noop', () => { + service.recordMetric('test.metric', 42, { dim: 'val' }); + }); + + it('incrementCounter is a noop', () => { + service.incrementCounter('test.counter', 1, { dim: 'val' }); + }); + + it('emitLogRecord is a noop', () => { + service.emitLogRecord('test body', { key: 'val' }); + }); + + it('flush resolves immediately', async () => { + await service.flush(); + }); + + it('shutdown resolves immediately', async () => { + await service.shutdown(); + }); +}); diff --git a/extensions/copilot/src/platform/otel/common/test/otelConfig.spec.ts b/extensions/copilot/src/platform/otel/common/test/otelConfig.spec.ts new file mode 100644 index 00000000000..2c3a6229e15 --- /dev/null +++ b/extensions/copilot/src/platform/otel/common/test/otelConfig.spec.ts @@ -0,0 +1,164 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { describe, expect, it } from 'vitest'; +import { resolveOTelConfig, type OTelConfigInput } from '../otelConfig'; + +function makeInput(overrides: Partial = {}): OTelConfigInput { + return { + env: {}, + extensionVersion: '1.0.0', + sessionId: 'test-session', + ...overrides, + }; +} + +describe('resolveOTelConfig', () => { + + it('returns disabled config by default', () => { + const config = resolveOTelConfig(makeInput()); + expect(config.enabled).toBe(false); + }); + + it('enables when COPILOT_OTEL_ENABLED=true', () => { + const config = resolveOTelConfig(makeInput({ + env: { 'COPILOT_OTEL_ENABLED': 'true' }, + })); + expect(config.enabled).toBe(true); + }); + + it('enables when OTEL_EXPORTER_OTLP_ENDPOINT is set', () => { + const config = resolveOTelConfig(makeInput({ + env: { 'OTEL_EXPORTER_OTLP_ENDPOINT': 'http://collector:4318' }, + })); + expect(config.enabled).toBe(true); + expect(config.otlpEndpoint).toBe('http://collector:4318/'); + }); + + it('enables via VS Code setting', () => { + const config = resolveOTelConfig(makeInput({ + settingEnabled: true, + })); + expect(config.enabled).toBe(true); + }); + + it('env COPILOT_OTEL_ENABLED overrides VS Code setting', () => { + const config = resolveOTelConfig(makeInput({ + env: { 'COPILOT_OTEL_ENABLED': 'false' }, + settingEnabled: true, + })); + expect(config.enabled).toBe(false); + }); + + it('disables when vscodeTelemetryLevel is off', () => { + const config = resolveOTelConfig(makeInput({ + env: { 'COPILOT_OTEL_ENABLED': 'true' }, + vscodeTelemetryLevel: 'off', + })); + expect(config.enabled).toBe(false); + }); + + it('uses file exporter when COPILOT_OTEL_FILE_EXPORTER_PATH is set', () => { + const config = resolveOTelConfig(makeInput({ + env: { + 'COPILOT_OTEL_ENABLED': 'true', + 'COPILOT_OTEL_FILE_EXPORTER_PATH': '/tmp/otel.jsonl', + }, + })); + expect(config.enabled).toBe(true); + expect(config.exporterType).toBe('file'); + expect(config.fileExporterPath).toBe('/tmp/otel.jsonl'); + }); + + it('uses VS Code setting for exporter type', () => { + const config = resolveOTelConfig(makeInput({ + settingEnabled: true, + settingExporterType: 'console', + })); + expect(config.exporterType).toBe('console'); + }); + + it('resolves COPILOT_OTEL_CAPTURE_CONTENT', () => { + const config = resolveOTelConfig(makeInput({ + env: { + 'COPILOT_OTEL_ENABLED': 'true', + 'COPILOT_OTEL_CAPTURE_CONTENT': 'true', + }, + })); + expect(config.captureContent).toBe(true); + }); + + it('captureContent env overrides VS Code setting', () => { + const config = resolveOTelConfig(makeInput({ + env: { + 'COPILOT_OTEL_ENABLED': 'true', + 'COPILOT_OTEL_CAPTURE_CONTENT': 'false', + }, + settingCaptureContent: true, + })); + expect(config.captureContent).toBe(false); + }); + + it('parses OTEL_RESOURCE_ATTRIBUTES', () => { + const config = resolveOTelConfig(makeInput({ + env: { + 'COPILOT_OTEL_ENABLED': 'true', + 'OTEL_RESOURCE_ATTRIBUTES': 'benchmark.id=test-123,benchmark.name=say_hello', + }, + })); + expect(config.resourceAttributes).toEqual({ + 'benchmark.id': 'test-123', + 'benchmark.name': 'say_hello', + }); + }); + + it('uses grpc protocol when OTEL_EXPORTER_OTLP_PROTOCOL=grpc', () => { + const config = resolveOTelConfig(makeInput({ + env: { + 'COPILOT_OTEL_ENABLED': 'true', + 'OTEL_EXPORTER_OTLP_PROTOCOL': 'grpc', + 'OTEL_EXPORTER_OTLP_ENDPOINT': 'http://collector:4317', + }, + })); + expect(config.otlpProtocol).toBe('grpc'); + expect(config.exporterType).toBe('otlp-grpc'); + // gRPC should use origin (strip path) + expect(config.otlpEndpoint).toBe('http://collector:4317'); + }); + + it('preserves service version and session id', () => { + const config = resolveOTelConfig(makeInput({ + settingEnabled: true, + extensionVersion: '2.5.0', + sessionId: 'abc-123', + })); + expect(config.serviceVersion).toBe('2.5.0'); + expect(config.sessionId).toBe('abc-123'); + }); + + it('defaults service name to copilot-chat', () => { + const config = resolveOTelConfig(makeInput({ + settingEnabled: true, + })); + expect(config.serviceName).toBe('copilot-chat'); + }); + + it('overrides service name from OTEL_SERVICE_NAME', () => { + const config = resolveOTelConfig(makeInput({ + env: { + 'COPILOT_OTEL_ENABLED': 'true', + 'OTEL_SERVICE_NAME': 'my-service', + }, + })); + expect(config.serviceName).toBe('my-service'); + }); + + it('returns frozen config objects', () => { + const enabled = resolveOTelConfig(makeInput({ settingEnabled: true })); + const disabled = resolveOTelConfig(makeInput()); + expect(Object.isFrozen(enabled)).toBe(true); + expect(Object.isFrozen(disabled)).toBe(true); + }); +}); diff --git a/extensions/copilot/src/platform/otel/common/test/serviceRobustness.spec.ts b/extensions/copilot/src/platform/otel/common/test/serviceRobustness.spec.ts new file mode 100644 index 00000000000..1cc3d0e97bb --- /dev/null +++ b/extensions/copilot/src/platform/otel/common/test/serviceRobustness.spec.ts @@ -0,0 +1,104 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { describe, expect, it } from 'vitest'; +import { SpanStatusCode } from '../otelService'; +import { CapturingOTelService } from './capturingOTelService'; + +/** + * Tests service robustness: buffer cap behavior, runWithTraceContext propagation, + * and capturing service correctness. + */ +describe('OTel Service Robustness', () => { + + describe('CapturingOTelService basics', () => { + it('reset clears all captured data', () => { + const otel = new CapturingOTelService(); + + otel.startSpan('test', { attributes: {} }); + otel.recordMetric('m', 1); + otel.incrementCounter('c'); + otel.emitLogRecord('log'); + + otel.reset(); + + expect(otel.spans).toHaveLength(0); + expect(otel.metrics).toHaveLength(0); + expect(otel.counters).toHaveLength(0); + expect(otel.logRecords).toHaveLength(0); + }); + + it('findSpans filters by name prefix', () => { + const otel = new CapturingOTelService(); + + otel.startSpan('chat gpt-4o', { attributes: {} }); + otel.startSpan('chat claude', { attributes: {} }); + otel.startSpan('execute_tool read', { attributes: {} }); + + expect(otel.findSpans('chat')).toHaveLength(2); + expect(otel.findSpans('execute_tool')).toHaveLength(1); + expect(otel.findSpans('invoke_agent')).toHaveLength(0); + }); + }); + + describe('runWithTraceContext', () => { + it('executes the function and returns its result', async () => { + const otel = new CapturingOTelService(); + const ctx = { traceId: 'aaaa0000bbbb1111cccc2222dddd3333', spanId: 'eeee4444ffff5555' }; + + const result = await otel.runWithTraceContext(ctx, async () => { + return 42; + }); + + expect(result).toBe(42); + }); + + it('propagates errors from the wrapped function', async () => { + const otel = new CapturingOTelService(); + const ctx = { traceId: '00000000000000000000000000000000', spanId: '0000000000000000' }; + + await expect(otel.runWithTraceContext(ctx, async () => { + throw new Error('test error'); + })).rejects.toThrow('test error'); + }); + }); + + describe('startActiveSpan lifecycle', () => { + it('ends span even when fn throws', async () => { + const otel = new CapturingOTelService(); + + await expect(otel.startActiveSpan('test', { attributes: {} }, async () => { + throw new Error('boom'); + })).rejects.toThrow('boom'); + + expect(otel.spans[0].ended).toBe(true); + }); + + it('returns fn result on success', async () => { + const otel = new CapturingOTelService(); + + const result = await otel.startActiveSpan('test', { attributes: {} }, async (span) => { + span.setStatus(SpanStatusCode.OK); + return 'hello'; + }); + + expect(result).toBe('hello'); + expect(otel.spans[0].statusCode).toBe(SpanStatusCode.OK); + }); + }); + + describe('storeTraceContext edge cases', () => { + it('overwriting a key replaces the context', () => { + const otel = new CapturingOTelService(); + const ctx1 = { traceId: 'aaaa', spanId: 'bbbb' }; + const ctx2 = { traceId: 'cccc', spanId: 'dddd' }; + + otel.storeTraceContext('key', ctx1); + otel.storeTraceContext('key', ctx2); + + expect(otel.getStoredTraceContext('key')).toEqual(ctx2); + }); + }); +}); diff --git a/extensions/copilot/src/platform/otel/node/fileExporters.ts b/extensions/copilot/src/platform/otel/node/fileExporters.ts new file mode 100644 index 00000000000..d49d61c2a3a --- /dev/null +++ b/extensions/copilot/src/platform/otel/node/fileExporters.ts @@ -0,0 +1,65 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { type ExportResult, ExportResultCode } from '@opentelemetry/core'; +import type { LogRecordExporter, ReadableLogRecord } from '@opentelemetry/sdk-logs'; +import { type PushMetricExporter, type ResourceMetrics, AggregationTemporality } from '@opentelemetry/sdk-metrics'; +import type { ReadableSpan, SpanExporter } from '@opentelemetry/sdk-trace-node'; +import * as fs from 'node:fs'; + +function safeStringify(data: unknown): string { + try { + return JSON.stringify(data); + } catch { + return '{}'; + } +} + +abstract class BaseFileExporter { + protected readonly writeStream: fs.WriteStream; + + constructor(filePath: string) { + this.writeStream = fs.createWriteStream(filePath, { flags: 'a' }); + } + + shutdown(): Promise { + return new Promise(resolve => this.writeStream.end(resolve)); + } + + forceFlush(): Promise { + return Promise.resolve(); + } +} + +export class FileSpanExporter extends BaseFileExporter implements SpanExporter { + export(spans: ReadableSpan[], resultCallback: (result: ExportResult) => void): void { + const data = spans.map(s => safeStringify(s) + '\n').join(''); + this.writeStream.write(data, err => { + resultCallback({ code: err ? ExportResultCode.FAILED : ExportResultCode.SUCCESS, error: err ?? undefined }); + }); + } +} + +export class FileLogExporter extends BaseFileExporter implements LogRecordExporter { + export(logs: ReadableLogRecord[], resultCallback: (result: ExportResult) => void): void { + const data = logs.map(l => safeStringify(l) + '\n').join(''); + this.writeStream.write(data, err => { + resultCallback({ code: err ? ExportResultCode.FAILED : ExportResultCode.SUCCESS, error: err ?? undefined }); + }); + } +} + +export class FileMetricExporter extends BaseFileExporter implements PushMetricExporter { + export(metrics: ResourceMetrics, resultCallback: (result: ExportResult) => void): void { + const data = safeStringify(metrics) + '\n'; + this.writeStream.write(data, err => { + resultCallback({ code: err ? ExportResultCode.FAILED : ExportResultCode.SUCCESS, error: err ?? undefined }); + }); + } + + selectAggregationTemporality(): AggregationTemporality { + return AggregationTemporality.CUMULATIVE; + } +} diff --git a/extensions/copilot/src/platform/otel/node/otelServiceImpl.ts b/extensions/copilot/src/platform/otel/node/otelServiceImpl.ts new file mode 100644 index 00000000000..b52c7e93840 --- /dev/null +++ b/extensions/copilot/src/platform/otel/node/otelServiceImpl.ts @@ -0,0 +1,612 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import type { OTelConfig } from '../common/otelConfig'; +import { type IOTelService, type ISpanHandle, type SpanOptions, type TraceContext, SpanKind, SpanStatusCode } from '../common/otelService'; + +// Type-only imports — erased by esbuild, zero bundle impact +import type { Attributes, Context, Meter, Span, SpanContext, Tracer } from '@opentelemetry/api'; +import type { AnyValueMap, Logger } from '@opentelemetry/api-logs'; +import type { ExportResult } from '@opentelemetry/core'; +import type { BatchLogRecordProcessor, LogRecordExporter } from '@opentelemetry/sdk-logs'; +import type { PeriodicExportingMetricReader, PushMetricExporter } from '@opentelemetry/sdk-metrics'; +import type { BatchSpanProcessor, ReadableSpan, SpanExporter } from '@opentelemetry/sdk-trace-node'; + +interface ExporterSet { + spanExporter: SpanExporter; + logExporter: LogRecordExporter; + metricExporter: PushMetricExporter; +} + +const noopSpanHandle: ISpanHandle = { + setAttribute() { }, + setAttributes() { }, + setStatus() { }, + recordException() { }, + end() { }, +}; + +/** + * Callback for routing OTel service log messages to the extension's output channel. + */ +export type OTelLogFn = (level: 'info' | 'warn' | 'error', message: string) => void; + +/** + * Real OTel service implementation, only instantiated when OTel is enabled. + * Uses dynamic imports so the OTel SDK is not loaded when disabled. + */ +export class NodeOTelService implements IOTelService { + declare readonly _serviceBrand: undefined; + readonly config: OTelConfig; + + private _tracer: Tracer | undefined; + private _meter: Meter | undefined; + private _logger: Logger | undefined; + private _spanProcessor: BatchSpanProcessor | undefined; + private _logProcessor: BatchLogRecordProcessor | undefined; + private _metricReader: PeriodicExportingMetricReader | undefined; + // OTel API reference for context propagation (stored after dynamic import) + private _otelApi: typeof import('@opentelemetry/api') | undefined; + private _initialized = false; + private _initFailed = false; + private static readonly _MAX_BUFFER_SIZE = 1000; + private readonly _log: OTelLogFn; + + // Buffer events until SDK is ready + private readonly _buffer: Array<() => void> = []; + + constructor(config: OTelConfig, logFn?: OTelLogFn) { + this.config = config; + this._log = logFn ?? ((_level, _msg) => { /* silent when no logger wired */ }); + // Start async initialization immediately + void this._initialize(); + } + + private async _initialize(): Promise { + if (this._initialized || !this.config.enabled) { + return; + } + + try { + // Dynamic imports — only loaded when OTel is enabled + const [ + api, + apiLogs, + traceSDK, + logsSDK, + metricsSDK, + resourcesMod, + ] = await Promise.all([ + import('@opentelemetry/api'), + import('@opentelemetry/api-logs'), + import('@opentelemetry/sdk-trace-node'), + import('@opentelemetry/sdk-logs'), + import('@opentelemetry/sdk-metrics'), + import('@opentelemetry/resources'), + ]); + + const BSP = traceSDK.BatchSpanProcessor; + const BLRP = logsSDK.BatchLogRecordProcessor; + const PEMR = metricsSDK.PeriodicExportingMetricReader; + const NodeTracerProvider = traceSDK.NodeTracerProvider; + const MeterProvider = metricsSDK.MeterProvider; + const LoggerProvider = logsSDK.LoggerProvider; + + // Use resourceFromAttributes (available in @opentelemetry/resources v2+) + const resource = resourcesMod.resourceFromAttributes({ + 'service.name': this.config.serviceName, + 'service.version': this.config.serviceVersion, + 'session.id': this.config.sessionId, + ...this.config.resourceAttributes, + }); + + // Create exporters based on config + const { spanExporter, logExporter, metricExporter } = await this._createExporters(); + + // Wrap span exporter with diagnostics to confirm end-to-end connectivity + const diagnosticSpanExporter = new DiagnosticSpanExporter(spanExporter, this.config.exporterType, this._log); + + // Trace provider — pass spanProcessors in constructor (SDK v2 API) + this._spanProcessor = new BSP(diagnosticSpanExporter); + const tracerProvider = new NodeTracerProvider({ + resource, + spanProcessors: [this._spanProcessor], + }); + tracerProvider.register(); + this._tracer = api.trace.getTracer(this.config.serviceName, this.config.serviceVersion); + this._otelApi = api; + + // Log provider — pass processors in constructor (SDK v2 uses 'processors' key) + this._logProcessor = new BLRP(logExporter, { + scheduledDelayMillis: 1000, + maxExportBatchSize: 512, + }); + const loggerProvider = new LoggerProvider({ + resource, + processors: [this._logProcessor], + } as ConstructorParameters[0]); + apiLogs.logs.setGlobalLoggerProvider(loggerProvider); + this._logger = apiLogs.logs.getLogger(this.config.serviceName, this.config.serviceVersion); + + // Metric provider + this._metricReader = new PEMR({ + exporter: metricExporter, + exportIntervalMillis: 10000, + }); + const meterProvider = new MeterProvider({ + resource, + readers: [this._metricReader], + }); + api.metrics.setGlobalMeterProvider(meterProvider); + this._meter = api.metrics.getMeter(this.config.serviceName, this.config.serviceVersion); + + this._initialized = true; + + // Flush buffered events in batches to avoid blocking the event loop + const batch = this._buffer.splice(0); + const BATCH_SIZE = 50; + for (let i = 0; i < batch.length; i += BATCH_SIZE) { + const chunk = batch.slice(i, i + BATCH_SIZE); + for (const fn of chunk) { + try { fn(); } catch { /* swallow */ } + } + if (i + BATCH_SIZE < batch.length) { + // Yield to event loop between batches + await new Promise(resolve => setTimeout(resolve, 0)); + } + } + + } catch (err) { + // OTel init failure should never break the extension + this._initFailed = true; + this._buffer.length = 0; // Discard buffered events on failure + this._log('error', `[OTel] Failed to initialize: ${err}`); + } + } + + private async _createExporters(): Promise { + const { config } = this; + + if (config.exporterType === 'file' && config.fileExporterPath) { + const { FileSpanExporter, FileLogExporter, FileMetricExporter } = await import('./fileExporters'); + return { + spanExporter: new FileSpanExporter(config.fileExporterPath), + logExporter: new FileLogExporter(config.fileExporterPath), + metricExporter: new FileMetricExporter(config.fileExporterPath), + }; + } + + if (config.exporterType === 'console') { + const [traceSDK, logsSDK, metricsSDK] = await Promise.all([ + import('@opentelemetry/sdk-trace-node'), + import('@opentelemetry/sdk-logs'), + import('@opentelemetry/sdk-metrics'), + ]); + return { + spanExporter: new traceSDK.ConsoleSpanExporter(), + logExporter: new logsSDK.ConsoleLogRecordExporter(), + metricExporter: new metricsSDK.ConsoleMetricExporter(), + }; + } + + if (config.exporterType === 'otlp-grpc') { + const [ + { OTLPTraceExporter }, + { OTLPLogExporter }, + { OTLPMetricExporter }, + ] = await Promise.all([ + import('@opentelemetry/exporter-trace-otlp-grpc'), + import('@opentelemetry/exporter-logs-otlp-grpc'), + import('@opentelemetry/exporter-metrics-otlp-grpc'), + ]); + const opts = { url: config.otlpEndpoint }; + return { + spanExporter: new OTLPTraceExporter(opts), + logExporter: new OTLPLogExporter(opts), + metricExporter: new OTLPMetricExporter(opts), + }; + } + + // Default: otlp-http + const [ + { OTLPTraceExporter }, + { OTLPLogExporter }, + { OTLPMetricExporter }, + ] = await Promise.all([ + import('@opentelemetry/exporter-trace-otlp-http'), + import('@opentelemetry/exporter-logs-otlp-http'), + import('@opentelemetry/exporter-metrics-otlp-http'), + ]); + const base = config.otlpEndpoint.replace(/\/$/, ''); + return { + spanExporter: new OTLPTraceExporter({ url: `${base}/v1/traces` }), + logExporter: new OTLPLogExporter({ url: `${base}/v1/logs` }), + metricExporter: new OTLPMetricExporter({ url: `${base}/v1/metrics` }), + }; + } + + // ── Span API ── + + startSpan(name: string, options?: SpanOptions): ISpanHandle { + if (!this._tracer) { + if (this._initFailed || this._buffer.length >= NodeOTelService._MAX_BUFFER_SIZE) { + return noopSpanHandle; + } + const handle = new BufferedSpanHandle(); + this._buffer.push(() => { + const real = this._createSpan(name, options); + handle.replay(real); + }); + return handle; + } + return this._createSpan(name, options); + } + + async startActiveSpan(name: string, options: SpanOptions, fn: (span: ISpanHandle) => Promise): Promise { + if (!this._tracer) { + const handle = this.startSpan(name, options); + try { + return await fn(handle); + } finally { + handle.end(); + } + } + + const spanOpts = { kind: toOTelSpanKind(options?.kind), attributes: options?.attributes as Attributes }; + + // If a parent trace context is provided, create a remote context and start span within it + if (options.parentTraceContext && this._otelApi) { + const parentCtx = this._createRemoteContext(options.parentTraceContext); + return this._tracer.startActiveSpan( + name, + spanOpts, + parentCtx, + async (span: Span) => { + const handle = new RealSpanHandle(span); + try { + return await fn(handle); + } finally { + handle.end(); + } + } + ); + } + + return this._tracer.startActiveSpan( + name, + spanOpts, + async (span: Span) => { + const handle = new RealSpanHandle(span); + try { + return await fn(handle); + } finally { + handle.end(); + } + } + ); + } + + getActiveTraceContext(): TraceContext | undefined { + if (!this._otelApi) { + return undefined; + } + const activeSpan = this._otelApi.trace.getSpan(this._otelApi.context.active()); + if (!activeSpan) { + return undefined; + } + const ctx = activeSpan.spanContext(); + if (!ctx.traceId || !ctx.spanId) { + return undefined; + } + return { traceId: ctx.traceId, spanId: ctx.spanId }; + } + + // ── Trace Context Store ── (for cross-boundary propagation) + + private static readonly _MAX_TRACE_CONTEXT_STORE_SIZE = 100; + private readonly _traceContextStore = new Map(); + private readonly _traceContextTimers = new Map>(); + + storeTraceContext(key: string, context: TraceContext): void { + // Evict oldest entry if at capacity + if (this._traceContextStore.size >= NodeOTelService._MAX_TRACE_CONTEXT_STORE_SIZE) { + const oldestKey = this._traceContextStore.keys().next().value; + if (oldestKey !== undefined) { + this._clearStoredTraceContext(oldestKey); + } + } + this._traceContextStore.set(key, context); + // Auto-cleanup after 5 minutes; tracked for proper disposal + const timer = setTimeout(() => this._clearStoredTraceContext(key), 5 * 60 * 1000); + this._traceContextTimers.set(key, timer); + } + + getStoredTraceContext(key: string): TraceContext | undefined { + const ctx = this._traceContextStore.get(key); + if (ctx) { + this._clearStoredTraceContext(key); + } + return ctx; + } + + private _clearStoredTraceContext(key: string): void { + this._traceContextStore.delete(key); + const timer = this._traceContextTimers.get(key); + if (timer) { + clearTimeout(timer); + this._traceContextTimers.delete(key); + } + } + + /** + * Creates an OTel Context with a remote span context as parent, + * allowing spans created within it to be children of the remote span. + */ + private _createRemoteContext(tc: TraceContext): Context { + const api = this._otelApi!; + const remoteSpanContext: SpanContext = { + traceId: tc.traceId, + spanId: tc.spanId, + traceFlags: 1, // SAMPLED + isRemote: true, + }; + const remoteCtx = api.trace.setSpanContext(api.context.active(), remoteSpanContext); + return remoteCtx; + } + + async runWithTraceContext(traceContext: TraceContext, fn: () => Promise): Promise { + if (!this._otelApi) { + return fn(); + } + const parentCtx = this._createRemoteContext(traceContext); + return this._otelApi.context.with(parentCtx, fn); + } + + private _createSpan(name: string, options?: SpanOptions): ISpanHandle { + const span = this._tracer!.startSpan(name, { + kind: toOTelSpanKind(options?.kind), + attributes: options?.attributes as Attributes, + }); + return new RealSpanHandle(span); + } + + // ── Metric API ── + + private readonly _histograms = new Map>(); + private readonly _counters = new Map>(); + + recordMetric(name: string, value: number, attributes?: Record): void { + if (!this._meter) { + if (!this._initFailed && this._buffer.length < NodeOTelService._MAX_BUFFER_SIZE) { + this._buffer.push(() => this.recordMetric(name, value, attributes)); + } + return; + } + let histogram = this._histograms.get(name); + if (!histogram) { + histogram = this._meter.createHistogram(name); + this._histograms.set(name, histogram); + } + histogram.record(value, attributes); + } + + incrementCounter(name: string, value = 1, attributes?: Record): void { + if (!this._meter) { + if (!this._initFailed && this._buffer.length < NodeOTelService._MAX_BUFFER_SIZE) { + this._buffer.push(() => this.incrementCounter(name, value, attributes)); + } + return; + } + let counter = this._counters.get(name); + if (!counter) { + counter = this._meter.createCounter(name); + this._counters.set(name, counter); + } + counter.add(value, attributes); + } + + // ── Log API ── + + private _logEmitCount = 0; + + emitLogRecord(body: string, attributes?: Record): void { + if (!this._logger) { + if (!this._initFailed && this._buffer.length < NodeOTelService._MAX_BUFFER_SIZE) { + this._buffer.push(() => this.emitLogRecord(body, attributes)); + } + return; + } + // Pass the active context so the log record inherits the trace ID from + // the current span (if any). Without this, logs emitted inside a span + // created via startSpan() (rather than startActiveSpan()) lack trace context. + const ctx = this._otelApi?.context.active(); + this._logger.emit({ body, attributes: attributes as AnyValueMap, ...(ctx ? { context: ctx } : {}) }); + this._logEmitCount++; + if (this._logEmitCount === 1) { + this._log('info', `[OTel] First log record emitted: ${body}`); + } + } + + // ── Lifecycle ── + + async flush(): Promise { + await Promise.all([ + this._spanProcessor?.forceFlush(), + this._logProcessor?.forceFlush(), + this._metricReader?.forceFlush(), + ]); + } + + async shutdown(): Promise { + try { + // Clear all trace context timers + for (const timer of this._traceContextTimers.values()) { + clearTimeout(timer); + } + this._traceContextTimers.clear(); + this._traceContextStore.clear(); + + await this.flush(); + await Promise.all([ + this._spanProcessor?.shutdown(), + this._logProcessor?.shutdown(), + this._metricReader?.shutdown(), + ]); + const api = await import('@opentelemetry/api'); + const apiLogs = await import('@opentelemetry/api-logs'); + api.trace.disable(); + api.metrics.disable(); + apiLogs.logs.disable(); + } catch { + // Swallow shutdown errors + } + } +} + +// ── Span Handle Implementations ── + +class RealSpanHandle implements ISpanHandle { + constructor(private readonly _span: Span) { } + + setAttribute(key: string, value: string | number | boolean | string[]): void { + this._span.setAttribute(key, value); + } + + setAttributes(attrs: Record): void { + for (const k in attrs) { + if (Object.prototype.hasOwnProperty.call(attrs, k)) { + const v = attrs[k]; + if (v !== undefined) { + this._span.setAttribute(k, v); + } + } + } + } + + setStatus(code: SpanStatusCode, message?: string): void { + const otelCode = code === SpanStatusCode.OK ? 1 : code === SpanStatusCode.ERROR ? 2 : 0; + this._span.setStatus({ code: otelCode, message }); + } + + recordException(error: unknown): void { + if (error instanceof Error) { + this._span.recordException(error); + } else { + this._span.recordException(new Error(String(error))); + } + } + + end(): void { + this._span.end(); + } +} + +/** + * Buffers span operations until the SDK is initialized, then replays them. + */ +class BufferedSpanHandle implements ISpanHandle { + private static readonly _MAX_OPS = 200; + private readonly _ops: Array<(span: ISpanHandle) => void> = []; + private _real: ISpanHandle | undefined; + + constructor() { } + + setAttribute(key: string, value: string | number | boolean | string[]): void { + if (this._real) { this._real.setAttribute(key, value); return; } + if (this._ops.length < BufferedSpanHandle._MAX_OPS) { + this._ops.push(s => s.setAttribute(key, value)); + } + } + + setAttributes(attrs: Record): void { + if (this._real) { this._real.setAttributes(attrs); return; } + if (this._ops.length < BufferedSpanHandle._MAX_OPS) { + this._ops.push(s => s.setAttributes(attrs)); + } + } + + setStatus(code: SpanStatusCode, message?: string): void { + if (this._real) { this._real.setStatus(code, message); return; } + if (this._ops.length < BufferedSpanHandle._MAX_OPS) { + this._ops.push(s => s.setStatus(code, message)); + } + } + + recordException(error: unknown): void { + if (this._real) { this._real.recordException(error); return; } + if (this._ops.length < BufferedSpanHandle._MAX_OPS) { + this._ops.push(s => s.recordException(error)); + } + } + + end(): void { + if (this._real) { this._real.end(); return; } + // Always buffer end() regardless of cap — it's critical for span lifecycle + this._ops.push(s => s.end()); + } + + replay(real: ISpanHandle): void { + this._real = real; + for (const op of this._ops) { + op(real); + } + this._ops.length = 0; + } +} + +function toOTelSpanKind(kind: SpanKind | undefined): number { + switch (kind) { + case SpanKind.CLIENT: return 2; // OTel SpanKind.CLIENT + case SpanKind.INTERNAL: return 0; // OTel SpanKind.INTERNAL + default: return 0; // INTERNAL + } +} + +/** + * Wraps a SpanExporter to log diagnostic info about export results. + * Logs once on first successful export (info), and on every failure (warn). + */ +class DiagnosticSpanExporter implements SpanExporter { + private _firstSuccessLogged = false; + private _lastFailureLogTime = 0; + private static readonly _FAILURE_LOG_INTERVAL_MS = 60_000; + private readonly _inner: SpanExporter; + private readonly _exporterType: string; + private readonly _log: OTelLogFn; + + constructor(inner: SpanExporter, exporterType: string, logFn: OTelLogFn) { + this._inner = inner; + this._exporterType = exporterType; + this._log = logFn; + } + + export(spans: ReadableSpan[], resultCallback: (result: ExportResult) => void): void { + this._inner.export(spans, result => { + // ExportResultCode.SUCCESS === 0 + if (result.code === 0) { + if (!this._firstSuccessLogged) { + this._firstSuccessLogged = true; + this._log('info', `[OTel] First span batch exported successfully via ${this._exporterType} (${spans.length} spans)`); + } + } else { + // Rate-limit failure logging to avoid flooding stdout + const now = Date.now(); + if (now - this._lastFailureLogTime >= DiagnosticSpanExporter._FAILURE_LOG_INTERVAL_MS) { + this._lastFailureLogTime = now; + this._log('warn', `[OTel] Span export failed via ${this._exporterType}: ${result.error ?? 'unknown error'}`); + } + } + resultCallback(result); + }); + } + + shutdown(): Promise { + return this._inner.shutdown?.() ?? Promise.resolve(); + } + + forceFlush(): Promise { + return this._inner.forceFlush?.() ?? Promise.resolve(); + } +} diff --git a/extensions/copilot/src/platform/otel/node/test/fileExporters.spec.ts b/extensions/copilot/src/platform/otel/node/test/fileExporters.spec.ts new file mode 100644 index 00000000000..c92aee3dd01 --- /dev/null +++ b/extensions/copilot/src/platform/otel/node/test/fileExporters.spec.ts @@ -0,0 +1,116 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { ExportResultCode } from '@opentelemetry/core'; +import { AggregationTemporality } from '@opentelemetry/sdk-metrics'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { FileLogExporter, FileMetricExporter, FileSpanExporter } from '../fileExporters'; + +describe('FileSpanExporter', () => { + let tmpFile: string; + let exporter: FileSpanExporter; + + beforeEach(() => { + tmpFile = path.join(os.tmpdir(), `otel-test-spans-${Date.now()}.jsonl`); + exporter = new FileSpanExporter(tmpFile); + }); + + afterEach(async () => { + await exporter.shutdown(); + try { fs.unlinkSync(tmpFile); } catch { } + }); + + it('writes span data as JSON lines', async () => { + const fakeSpan = { name: 'test-span', kind: 0, attributes: { a: 1 } }; + await new Promise((resolve, reject) => { + exporter.export([fakeSpan as any], result => { + result.code === ExportResultCode.SUCCESS ? resolve() : reject(result.error); + }); + }); + await exporter.shutdown(); + const content = fs.readFileSync(tmpFile, 'utf-8'); + const parsed = JSON.parse(content.trim()); + expect(parsed.name).toBe('test-span'); + expect(parsed.attributes).toEqual({ a: 1 }); + }); + + it('appends multiple exports', async () => { + for (let i = 0; i < 3; i++) { + await new Promise((resolve, reject) => { + exporter.export([{ name: `span-${i}` } as any], result => { + result.code === ExportResultCode.SUCCESS ? resolve() : reject(result.error); + }); + }); + } + await exporter.shutdown(); + const lines = fs.readFileSync(tmpFile, 'utf-8').trim().split('\n'); + expect(lines).toHaveLength(3); + expect(JSON.parse(lines[0]).name).toBe('span-0'); + expect(JSON.parse(lines[2]).name).toBe('span-2'); + }); +}); + +describe('FileLogExporter', () => { + let tmpFile: string; + let exporter: FileLogExporter; + + beforeEach(() => { + tmpFile = path.join(os.tmpdir(), `otel-test-logs-${Date.now()}.jsonl`); + exporter = new FileLogExporter(tmpFile); + }); + + afterEach(async () => { + await exporter.shutdown(); + try { fs.unlinkSync(tmpFile); } catch { } + }); + + it('writes log records as JSON lines', async () => { + const fakeLog = { body: 'test log', severityText: 'INFO' }; + await new Promise((resolve, reject) => { + exporter.export([fakeLog as any], result => { + result.code === ExportResultCode.SUCCESS ? resolve() : reject(result.error); + }); + }); + await exporter.shutdown(); + const content = fs.readFileSync(tmpFile, 'utf-8'); + const parsed = JSON.parse(content.trim()); + expect(parsed.body).toBe('test log'); + }); +}); + +describe('FileMetricExporter', () => { + let tmpFile: string; + let exporter: FileMetricExporter; + + beforeEach(() => { + tmpFile = path.join(os.tmpdir(), `otel-test-metrics-${Date.now()}.jsonl`); + exporter = new FileMetricExporter(tmpFile); + }); + + afterEach(async () => { + await exporter.shutdown(); + try { fs.unlinkSync(tmpFile); } catch { } + }); + + it('writes metric data as JSON lines', async () => { + const fakeMetrics = { resource: {}, scopeMetrics: [{ metrics: [{ name: 'test' }] }] }; + await new Promise((resolve, reject) => { + exporter.export(fakeMetrics as any, result => { + result.code === ExportResultCode.SUCCESS ? resolve() : reject(result.error); + }); + }); + await exporter.shutdown(); + const content = fs.readFileSync(tmpFile, 'utf-8'); + const parsed = JSON.parse(content.trim()); + expect(parsed.scopeMetrics[0].metrics[0].name).toBe('test'); + }); + + it('returns CUMULATIVE aggregation temporality', () => { + expect(exporter.selectAggregationTemporality()).toBe(AggregationTemporality.CUMULATIVE); + }); +}); diff --git a/extensions/copilot/src/platform/otel/node/test/traceContextPropagation.spec.ts b/extensions/copilot/src/platform/otel/node/test/traceContextPropagation.spec.ts new file mode 100644 index 00000000000..e535811989d --- /dev/null +++ b/extensions/copilot/src/platform/otel/node/test/traceContextPropagation.spec.ts @@ -0,0 +1,144 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { resolveOTelConfig } from '../../common/otelConfig'; +import type { TraceContext } from '../../common/otelService'; +import { NodeOTelService } from '../otelServiceImpl'; + +/** + * Tests for trace context propagation, specifically verifying that + * subagent invoke_agent spans can be linked as children of the parent + * agent's trace via storeTraceContext / getStoredTraceContext / parentTraceContext. + */ +describe('Trace Context Propagation', () => { + let service: NodeOTelService; + + beforeAll(async () => { + const config = resolveOTelConfig({ + env: { + 'COPILOT_OTEL_ENABLED': 'true', + 'COPILOT_OTEL_EXPORTER': 'console', + }, + extensionVersion: '1.0.0', + sessionId: 'test-session', + }); + service = new NodeOTelService(config); + // Wait for async SDK initialization — poll until _initialized + for (let i = 0; i < 50; i++) { + if ((service as any)._initialized) { break; } + await new Promise(r => setTimeout(r, 50)); + } + }); + + afterAll(async () => { + await service.shutdown(); + }); + + describe('storeTraceContext / getStoredTraceContext', () => { + it('round-trips a stored trace context', () => { + const ctx: TraceContext = { traceId: 'aaaa0000bbbb1111cccc2222dddd3333', spanId: 'eeee4444ffff5555' }; + service.storeTraceContext('test-key', ctx); + const retrieved = service.getStoredTraceContext('test-key'); + expect(retrieved).toEqual(ctx); + }); + + it('returns undefined for unknown key', () => { + expect(service.getStoredTraceContext('nonexistent')).toBeUndefined(); + }); + + it('deletes context after retrieval (single-use)', () => { + const ctx: TraceContext = { traceId: 'aaaa0000bbbb1111cccc2222dddd3333', spanId: 'eeee4444ffff5555' }; + service.storeTraceContext('one-shot', ctx); + service.getStoredTraceContext('one-shot'); + expect(service.getStoredTraceContext('one-shot')).toBeUndefined(); + }); + }); + + describe('getActiveTraceContext', () => { + it('returns undefined when no span is active', () => { + expect(service.getActiveTraceContext()).toBeUndefined(); + }); + + it('returns trace context inside startActiveSpan', async () => { + let capturedCtx: TraceContext | undefined; + await service.startActiveSpan('test-parent', { attributes: {} }, async () => { + capturedCtx = service.getActiveTraceContext(); + }); + expect(capturedCtx).toBeDefined(); + expect(capturedCtx!.traceId).toMatch(/^[0-9a-f]{32}$/); + expect(capturedCtx!.spanId).toMatch(/^[0-9a-f]{16}$/); + }); + }); + + describe('parentTraceContext links subagent to parent trace', () => { + it('child span inherits traceId from parent via parentTraceContext', async () => { + // Phase 1: Parent agent creates a span, captures context + let parentCtx: TraceContext | undefined; + await service.startActiveSpan('invoke_agent parent', { attributes: {} }, async () => { + parentCtx = service.getActiveTraceContext(); + }); + expect(parentCtx).toBeDefined(); + + // Phase 2: Subagent uses parentTraceContext (new async context, no active parent) + let childCtx: TraceContext | undefined; + await service.startActiveSpan('invoke_agent subagent', { + attributes: {}, + parentTraceContext: parentCtx, + }, async () => { + childCtx = service.getActiveTraceContext(); + }); + + // Same traceId (same distributed trace), different spanId + expect(childCtx!.traceId).toBe(parentCtx!.traceId); + expect(childCtx!.spanId).not.toBe(parentCtx!.spanId); + }); + + it('without parentTraceContext, spans get independent traceIds', async () => { + let trace1: string | undefined; + let trace2: string | undefined; + + await service.startActiveSpan('agent-1', { attributes: {} }, async () => { + trace1 = service.getActiveTraceContext()!.traceId; + }); + + await service.startActiveSpan('agent-2', { attributes: {} }, async () => { + trace2 = service.getActiveTraceContext()!.traceId; + }); + + expect(trace1).not.toBe(trace2); + }); + + it('full subagent flow: store in tool call, retrieve in subagent', async () => { + let parentTraceId: string | undefined; + let subagentTraceId: string | undefined; + + // Phase 1: Parent agent runs, tool calls runSubagent, stores context + await service.startActiveSpan('invoke_agent main', { attributes: {} }, async () => { + const ctx = service.getActiveTraceContext()!; + parentTraceId = ctx.traceId; + // Simulate execute_tool runSubagent storing the context + service.storeTraceContext('subagent:req-abc', ctx); + }); + + // Phase 2: Subagent request arrives (new async context, no parent span active) + const restoredCtx = service.getStoredTraceContext('subagent:req-abc'); + expect(restoredCtx).toBeDefined(); + + await service.startActiveSpan('invoke_agent subagent', { + attributes: {}, + parentTraceContext: restoredCtx, + }, async () => { + subagentTraceId = service.getActiveTraceContext()!.traceId; + }); + + // Both agents share the same traceId + expect(subagentTraceId).toBe(parentTraceId); + + // The stored context was consumed (single-use) + expect(service.getStoredTraceContext('subagent:req-abc')).toBeUndefined(); + }); + }); +}); diff --git a/extensions/copilot/src/platform/test/node/services.ts b/extensions/copilot/src/platform/test/node/services.ts index 77f5b384d0b..5a7691580ce 100644 --- a/extensions/copilot/src/platform/test/node/services.ts +++ b/extensions/copilot/src/platform/test/node/services.ts @@ -67,6 +67,9 @@ import { HeaderContributors, IHeaderContributors } from '../../networking/common import { NodeFetcherService } from '../../networking/node/test/nodeFetcherService'; import { INotificationService, NullNotificationService } from '../../notification/common/notificationService'; import { IUrlOpener, NullUrlOpener } from '../../open/common/opener'; +import { NoopOTelService } from '../../otel/common/noopOtelService'; +import { resolveOTelConfig } from '../../otel/common/otelConfig'; +import { IOTelService } from '../../otel/common/otelService'; import { IParserService } from '../../parser/node/parserService'; import { ParserServiceImpl } from '../../parser/node/parserServiceImpl'; import { IPromptPathRepresentationService, TestPromptPathRepresentationService } from '../../prompts/common/promptPathRepresentationService'; @@ -221,6 +224,7 @@ export function _createBaselineServices(): TestingServiceCollection { testingServiceCollection.define(IEditSurvivalTrackerService, new SyncDescriptor(NullEditSurvivalTrackerService)); testingServiceCollection.define(IWorkspaceChunkSearchService, new SyncDescriptor(NullWorkspaceChunkSearchService)); testingServiceCollection.define(ICodeSearchAuthenticationService, new SyncDescriptor(BasicCodeSearchAuthenticationService)); + testingServiceCollection.define(IOTelService, new SyncDescriptor(NoopOTelService, [resolveOTelConfig({ env: {}, extensionVersion: '0.0.0', sessionId: 'test' })])); return testingServiceCollection; } diff --git a/extensions/copilot/test/base/cachingEmbeddingsFetcher.ts b/extensions/copilot/test/base/cachingEmbeddingsFetcher.ts index 5246e8a5570..dd142a6e5b2 100644 --- a/extensions/copilot/test/base/cachingEmbeddingsFetcher.ts +++ b/extensions/copilot/test/base/cachingEmbeddingsFetcher.ts @@ -9,6 +9,7 @@ import { RemoteEmbeddingsComputer } from '../../src/platform/embeddings/common/r import { IEndpointProvider } from '../../src/platform/endpoint/common/endpointProvider'; import { IEnvService } from '../../src/platform/env/common/envService'; import { ILogService } from '../../src/platform/log/common/logService'; +import { IOTelService } from '../../src/platform/otel/common/otelService'; import { ITelemetryService } from '../../src/platform/telemetry/common/telemetry'; import { TelemetryCorrelationId } from '../../src/util/common/telemetryCorrelationId'; import { IInstantiationService } from '../../src/util/vs/platform/instantiation/common/instantiation'; @@ -50,6 +51,7 @@ export class CachingEmbeddingsComputer extends RemoteEmbeddingsComputer { @ITelemetryService telemetryService: ITelemetryService, @IEndpointProvider endpointProvider: IEndpointProvider, @IInstantiationService instantiationService: IInstantiationService, + @IOTelService otelService: IOTelService, ) { super( authService, @@ -58,6 +60,7 @@ export class CachingEmbeddingsComputer extends RemoteEmbeddingsComputer { telemetryService, endpointProvider, instantiationService, + otelService, ); }