mirror of
https://github.com/microsoft/vscode.git
synced 2026-09-30 16:27:42 +01:00
sessions: measure and unblock V3 onboarding GitHub personalization (#334696)
The V3 new-session onboarding personalizes its 3 prompt options with GitHub items, falling back to standard options. Telemetry from experiment f716301b showed 61% of impressions timed out with zero GitHub options, and the existing events could not measure partial success: `fallbackReason` was only 'none' when all 3 slots were GitHub, so 2 real issues logged identically to zero. Telemetry (additive; existing field names and values unchanged): - promptStrategy gains gitHubOptionCount, candidatesFound, lookupDurationMs, per-stage durations and outcomes, and timedOutStage. - promptOptionInteraction gains optionIndex and optionKindsShown, so per-option CTR is computable rather than confounded with availability. - Both gain a shared impressionId, replacing lossy DevDeviceId joins. - New 'noComposer' fallback reason splits the infra-bug meaning out of the overloaded 'noCandidate'. - Bounded categories and counts only: no titles, URLs, numbers, repository names or prompt text. Latency, without raising the 10s total budget: - Publish issue candidates before the linkage lookup, which only filters out issues that already have linked pull requests and so must never hide them. - Keep review-thread enrichment off the critical path; it produced the least picked option while being the most expensive stage. - Rebalance sub-budgets to 4000/1500/3000 so the concurrent chains fit. - Stream options into the composer: paint the standard options immediately and replace slots as lookups land, instead of blocking first paint on the lookup. INewSessionPromptOptionsController.resolve gains an optional progress callback that reports whether an update was rendered, so the composer can refuse late updates while the user is acting on the options and telemetry still describes what was actually shown. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
co-authored by
Copilot
parent
aee316035c
commit
d9637b3f2f
@@ -456,6 +456,7 @@ export class NewChatInputWidget extends Disposable implements IHistoryNavigation
|
||||
private _promptOptionsState: NewSessionPromptOptionsState | undefined;
|
||||
private _promptOptionsController: INewSessionPromptOptionsController | undefined;
|
||||
private _promptOptionsDismissed = false;
|
||||
private _promptOptionsSelected = false;
|
||||
|
||||
// Send button
|
||||
private _sendButton: Button | undefined;
|
||||
@@ -677,6 +678,9 @@ export class NewChatInputWidget extends Disposable implements IHistoryNavigation
|
||||
|
||||
this._promptOptionsWidget.value = this.instantiationService.createInstance(NewSessionPromptOptionsWidget, chatInputContainer, {
|
||||
selectOption: async (option, expectedInput, animate) => {
|
||||
// Mark the selection as it begins: the widget only reports it once the prompt
|
||||
// animation completes, and a streamed update in between would clear it.
|
||||
this._promptOptionsSelected = true;
|
||||
this.focus();
|
||||
const inserted = animate
|
||||
? await this.animatePrompt(option.prompt, NEW_SESSION_PROMPT_TYPING_DURATION_MS, option.placeholder, CancellationToken.None, expectedInput)
|
||||
@@ -1762,6 +1766,7 @@ export class NewChatInputWidget extends Disposable implements IHistoryNavigation
|
||||
this._cancelPromptOptionsRefresh(false);
|
||||
this._promptOptionsController = controller;
|
||||
this._promptOptionsDismissed = false;
|
||||
this._promptOptionsSelected = false;
|
||||
}
|
||||
|
||||
preparePromptOptionsRefresh(): boolean {
|
||||
@@ -1769,6 +1774,7 @@ export class NewChatInputWidget extends Disposable implements IHistoryNavigation
|
||||
return false;
|
||||
}
|
||||
this._cancelPromptOptionsRefresh();
|
||||
this._promptOptionsSelected = false;
|
||||
this.showPromptOptions({ kind: 'loading' });
|
||||
return true;
|
||||
}
|
||||
@@ -1809,9 +1815,17 @@ export class NewChatInputWidget extends Disposable implements IHistoryNavigation
|
||||
}
|
||||
const cts = new CancellationTokenSource(token);
|
||||
this._promptOptionsRefresh.value = cts;
|
||||
// Options resolved after a first paint must not replace ones the user is already working with:
|
||||
// re-rendering clears the selection and moves focus out of the options.
|
||||
const canRerender = () => !this._promptOptionsSelected && !this._promptOptionsWidget.value?.hasFocusedOption();
|
||||
let state: NewSessionPromptOptionsState;
|
||||
try {
|
||||
state = await controller.resolve(cts.token);
|
||||
state = await controller.resolve(cts.token, progressState => {
|
||||
if (this._promptOptionsRefresh.value !== cts || cts.token.isCancellationRequested || !canRerender()) {
|
||||
return false;
|
||||
}
|
||||
return this.showPromptOptions(progressState);
|
||||
});
|
||||
} catch (error) {
|
||||
if (this._promptOptionsRefresh.value === cts) {
|
||||
this._promptOptionsRefresh.clear();
|
||||
@@ -1831,6 +1845,9 @@ export class NewChatInputWidget extends Disposable implements IHistoryNavigation
|
||||
return false;
|
||||
}
|
||||
this._promptOptionsRefresh.clear();
|
||||
if (!canRerender()) {
|
||||
return true;
|
||||
}
|
||||
return this.showPromptOptions(state);
|
||||
}
|
||||
|
||||
|
||||
@@ -36,8 +36,23 @@ export const enum NewSessionWorkspacePreselectionSource {
|
||||
Unknown = 'unknown',
|
||||
}
|
||||
|
||||
/**
|
||||
* Reports an intermediate prompt-option state while {@link INewSessionPromptOptionsController.resolve}
|
||||
* is still running, so options can be rendered before the controller has settled on its final set.
|
||||
*
|
||||
* Returns whether the state was rendered. Implementers may refuse a late update, for example while
|
||||
* the user is already acting on the options on screen.
|
||||
*/
|
||||
export type NewSessionPromptOptionsProgress = (state: NewSessionPromptOptionsState) => boolean;
|
||||
|
||||
export interface INewSessionPromptOptionsController {
|
||||
resolve(token: CancellationToken): Promise<NewSessionPromptOptionsState>;
|
||||
/**
|
||||
* Resolves the final prompt options. Single-shot per refresh.
|
||||
*
|
||||
* @param progress Optional sink for intermediate states. Implementers may ignore it; callers that
|
||||
* pass it must tolerate it never being invoked and must apply the returned state regardless.
|
||||
*/
|
||||
resolve(token: CancellationToken, progress?: NewSessionPromptOptionsProgress): Promise<NewSessionPromptOptionsState>;
|
||||
onDidSelectOption(option: INewSessionPromptOption): void;
|
||||
onDidClose(): void;
|
||||
}
|
||||
|
||||
@@ -122,6 +122,11 @@ export class NewSessionPromptOptionsWidget extends Disposable {
|
||||
this._updateButtons();
|
||||
}
|
||||
|
||||
/** Whether focus is inside the rendered options, where re-rendering would move focus to the body. */
|
||||
hasFocusedOption(): boolean {
|
||||
return dom.isAncestorOfActiveElement(this._optionsContainer);
|
||||
}
|
||||
|
||||
shouldClearInputForRefresh(): boolean {
|
||||
const selectedOption = this._buttons.find(candidate => candidate.option.id === this._selectedOptionId)?.option;
|
||||
return this._selecting || this._inputValue.length === 0 || matchesGeneratedPrompt(selectedOption, this._inputValue);
|
||||
|
||||
@@ -13,12 +13,14 @@ import { mock } from '../../../../../base/test/common/mock.js';
|
||||
import { ensureNoDisposablesAreLeakedInTestSuite } from '../../../../../base/test/common/utils.js';
|
||||
import { IHoverService } from '../../../../../platform/hover/browser/hover.js';
|
||||
import { NewChatInputWidget } from '../../browser/newChatInput.js';
|
||||
import { INewSessionPromptOption, INewSessionPromptOptionsController, NewSessionPromptOptionsState } from '../../browser/newSessionComposerService.js';
|
||||
import { INewSessionPromptOption, INewSessionPromptOptionsController, NewSessionPromptOptionsProgress, NewSessionPromptOptionsState } from '../../browser/newSessionComposerService.js';
|
||||
import { NewSessionPromptOptionsWidget } from '../../browser/newSessionPromptOptions.js';
|
||||
|
||||
interface IPromptOptionsRefreshHarness {
|
||||
readonly _promptOptionsRefresh: MutableDisposable<CancellationTokenSource>;
|
||||
readonly _promptOptionsController: INewSessionPromptOptionsController;
|
||||
_promptOptionsSelected: boolean;
|
||||
readonly _promptOptionsWidget: { readonly value: { hasFocusedOption(): boolean } | undefined };
|
||||
preparePromptOptionsRefresh(): boolean;
|
||||
showPromptOptions(state: NewSessionPromptOptionsState | undefined): boolean;
|
||||
}
|
||||
@@ -234,6 +236,8 @@ suite('NewSessionPromptOptionsWidget', () => {
|
||||
const refresh = disposables.add(new MutableDisposable<CancellationTokenSource>());
|
||||
const harness: IPromptOptionsRefreshHarness = {
|
||||
_promptOptionsRefresh: refresh,
|
||||
_promptOptionsSelected: false,
|
||||
_promptOptionsWidget: { value: undefined },
|
||||
_promptOptionsController: {
|
||||
resolve: token => {
|
||||
tokens.push(token);
|
||||
@@ -272,6 +276,79 @@ suite('NewSessionPromptOptionsWidget', () => {
|
||||
});
|
||||
});
|
||||
|
||||
test('starts selectOption before reporting the selection so consumers can guard the in-flight window', async () => {
|
||||
const container = document.createElement('div');
|
||||
const events: string[] = [];
|
||||
const inserting = new DeferredPromise<boolean>();
|
||||
const widget = disposables.add(new NewSessionPromptOptionsWidget(container, {
|
||||
selectOption: async () => {
|
||||
events.push('selectOption');
|
||||
return inserting.p;
|
||||
},
|
||||
onDidSelectOption: () => events.push('onDidSelectOption'),
|
||||
onDidClose: () => undefined,
|
||||
}, new TestHoverService()));
|
||||
|
||||
widget.setState({ kind: 'resolved', options: [option('feature', 'Implement a feature')] });
|
||||
widget.element.querySelector<HTMLElement>('.monaco-button.new-session-prompt-option')?.click();
|
||||
await timeout(0);
|
||||
const duringInsertion = [...events];
|
||||
inserting.complete(true);
|
||||
await timeout(0);
|
||||
|
||||
assert.deepStrictEqual({ duringInsertion, afterInsertion: events }, {
|
||||
duringInsertion: ['selectOption'],
|
||||
afterInsertion: ['selectOption', 'onDidSelectOption'],
|
||||
});
|
||||
});
|
||||
|
||||
test('applies streamed prompt options until the user acts on them', async () => {
|
||||
const result = new DeferredPromise<NewSessionPromptOptionsState>();
|
||||
const states: NewSessionPromptOptionsState[] = [];
|
||||
const refresh = disposables.add(new MutableDisposable<CancellationTokenSource>());
|
||||
let reportProgress: NewSessionPromptOptionsProgress | undefined;
|
||||
const harness: IPromptOptionsRefreshHarness = {
|
||||
_promptOptionsRefresh: refresh,
|
||||
_promptOptionsSelected: false,
|
||||
_promptOptionsWidget: { value: undefined },
|
||||
_promptOptionsController: {
|
||||
resolve: (_token, progress) => {
|
||||
reportProgress = progress;
|
||||
return result.p;
|
||||
},
|
||||
onDidSelectOption: () => undefined,
|
||||
onDidClose: () => undefined,
|
||||
},
|
||||
preparePromptOptionsRefresh: () => {
|
||||
states.push({ kind: 'loading' });
|
||||
return true;
|
||||
},
|
||||
showPromptOptions: state => {
|
||||
if (state) {
|
||||
states.push(state);
|
||||
}
|
||||
return true;
|
||||
},
|
||||
};
|
||||
|
||||
const refreshing = refreshPromptOptions.call(harness);
|
||||
const applied = [
|
||||
reportProgress?.({ kind: 'resolved', options: [option('feature', 'Implement a feature')] }),
|
||||
(harness._promptOptionsSelected = true, reportProgress?.({ kind: 'resolved', options: [option('bug', 'Fix a bug')] })),
|
||||
];
|
||||
result.complete({ kind: 'resolved', options: [option('ci', 'Fix CI')] });
|
||||
|
||||
assert.deepStrictEqual({
|
||||
shown: await refreshing,
|
||||
applied,
|
||||
states: states.map(state => state.kind === 'loading' ? 'loading' : state.options[0].id),
|
||||
}, {
|
||||
shown: true,
|
||||
applied: [true, false],
|
||||
states: ['loading', 'feature'],
|
||||
});
|
||||
});
|
||||
|
||||
test('replaces a generated prompt immediately', () => {
|
||||
let value = 'old prompt';
|
||||
let placeholder: string | undefined;
|
||||
@@ -312,6 +389,8 @@ suite('NewSessionPromptOptionsWidget', () => {
|
||||
const refresh = disposables.add(new MutableDisposable<CancellationTokenSource>());
|
||||
const harness: IPromptOptionsRefreshHarness = {
|
||||
_promptOptionsRefresh: refresh,
|
||||
_promptOptionsSelected: false,
|
||||
_promptOptionsWidget: { value: undefined },
|
||||
_promptOptionsController: {
|
||||
resolve: () => result.p,
|
||||
onDidSelectOption: () => undefined,
|
||||
@@ -346,6 +425,8 @@ suite('NewSessionPromptOptionsWidget', () => {
|
||||
let resolveCount = 0;
|
||||
const harness: IPromptOptionsRefreshHarness = {
|
||||
_promptOptionsRefresh: disposables.add(new MutableDisposable<CancellationTokenSource>()),
|
||||
_promptOptionsSelected: false,
|
||||
_promptOptionsWidget: { value: undefined },
|
||||
_promptOptionsController: {
|
||||
resolve: async () => {
|
||||
resolveCount++;
|
||||
|
||||
@@ -10,6 +10,7 @@ import { CancellationError, isCancellationError } from '../../../../base/common/
|
||||
import { DisposableStore, MutableDisposable } from '../../../../base/common/lifecycle.js';
|
||||
import { autorun } from '../../../../base/common/observable.js';
|
||||
import { format } from '../../../../base/common/strings.js';
|
||||
import { generateUuid } from '../../../../base/common/uuid.js';
|
||||
import { localize } from '../../../../nls.js';
|
||||
import { IConfigurationService } from '../../../../platform/configuration/common/configuration.js';
|
||||
import { IFileService } from '../../../../platform/files/common/files.js';
|
||||
@@ -22,7 +23,7 @@ import { IWorkbenchAssignmentService } from '../../../../workbench/services/assi
|
||||
import { isAgentHostProviderId } from '../../../common/agentHostSessionsProvider.js';
|
||||
import { IActiveSession } from '../../../services/sessions/common/sessionsManagement.js';
|
||||
import { ISessionsService } from '../../../services/sessions/browser/sessionsService.js';
|
||||
import { INewSessionComposer, INewSessionComposerService, INewSessionPromptOption, INewSessionPromptOptionsController, NEW_SESSION_PROMPT_TYPING_DURATION_MS, NewSessionPromptOptionsState } from '../../chat/browser/newSessionComposerService.js';
|
||||
import { INewSessionComposer, INewSessionComposerService, INewSessionPromptOption, INewSessionPromptOptionsController, NEW_SESSION_PROMPT_TYPING_DURATION_MS, NewSessionPromptOptionsProgress, NewSessionPromptOptionsState } from '../../chat/browser/newSessionComposerService.js';
|
||||
import { getGitHubRepositoryFromUri } from '../../github/common/utils.js';
|
||||
import { GitHubAuthenticationError } from '../../github/browser/githubApiClient.js';
|
||||
import { IGitHubRecentIssue, IGitHubRecentPullRequest, IGitHubRecentUserWork } from '../../github/browser/fetchers/githubRecentUserWorkFetcher.js';
|
||||
@@ -31,11 +32,18 @@ import { computeIssueIcon, computePullRequestIcon, GitHubIssueState, GitHubPullR
|
||||
import { resolveGitHubRepositoryFromGitConfig } from './gitHubRepositoryResolver.js';
|
||||
import { NEW_SESSION_VIEW_V3_GITHUB_PROMPT_VARIATION, NEW_SESSION_VIEW_V3_OPTIONS_VARIATION, NEW_SESSION_VIEW_V3_PROMPT_VARIATION, NEW_SESSION_VIEW_V3_TOUR_ID, NEW_SESSION_VIEW_V3_VARIATION_TREATMENT } from './tours/newSessionViewV3Tour.js';
|
||||
|
||||
/**
|
||||
* Per-stage budgets are sized so the longest dependent chain of timed lookups fits inside `totalMs`:
|
||||
* the issue and pull request chains run concurrently, so the worst case is
|
||||
* max(issue 4_000 + 1_500 = 5_500, pull request 4_000 + 3_000 = 7_000) = 7_000ms, leaving ~3_000ms
|
||||
* for repository resolution and rendering. Repository resolution has no budget of its own, and a
|
||||
* workspace change restarts the whole chain, so both are bounded only by the `totalMs` race.
|
||||
*/
|
||||
const DEFAULT_GITHUB_LOOKUP_TIMEOUTS = {
|
||||
totalMs: 10_000,
|
||||
summaryMs: 5_000,
|
||||
linkageMs: 2_500,
|
||||
reviewMs: 4_000,
|
||||
summaryMs: 4_000,
|
||||
linkageMs: 1_500,
|
||||
reviewMs: 3_000,
|
||||
};
|
||||
const LOG_PREFIX = '[NewSessionViewV3Prompt]';
|
||||
const PROMPT_TEMPLATE_TREATMENT = 'onb.newSessionViewV3.promptTemplate';
|
||||
@@ -46,7 +54,25 @@ const PROMPT_OPTION_COUNT = 3;
|
||||
|
||||
export type NewSessionViewV3ConfiguredVariation = 'prompt' | 'githubPrompt' | 'options' | 'unknown';
|
||||
export type NewSessionViewV3EffectiveStrategy = 'prompt' | 'options' | 'githubMergeConflict' | 'githubCiFailure' | 'githubReviewComments' | 'githubIssue';
|
||||
export type NewSessionViewV3FallbackReason = 'none' | 'unsupportedVariation' | 'noRepository' | 'noAuthentication' | 'timeout' | 'requestFailed' | 'noCandidate';
|
||||
export type NewSessionViewV3FallbackReason = 'none' | 'unsupportedVariation' | 'noRepository' | 'noAuthentication' | 'timeout' | 'requestFailed' | 'noCandidate' | 'noComposer';
|
||||
|
||||
/** The dependent GitHub lookup stages that share the total lookup budget. */
|
||||
type NewSessionViewV3LookupStage = 'repository' | 'issueSummary' | 'issueLinkage' | 'prSummary' | 'prReview';
|
||||
|
||||
/** The privacy-safe outcome of a single GitHub lookup stage. */
|
||||
type NewSessionViewV3StageOutcome = 'notRun' | 'skipped' | 'success' | 'cancelled' | 'requestFailed' | 'timeout' | 'noAuthentication';
|
||||
|
||||
/** The stage reported as having consumed the budget when the total lookup times out. */
|
||||
type NewSessionViewV3TimedOutStage = NewSessionViewV3LookupStage | 'none' | 'unknown';
|
||||
|
||||
/**
|
||||
* Stages ordered by how late they run in the lookup. When several stages are still in flight as the
|
||||
* total budget expires, the last pending stage in this order is reported as the one that consumed it.
|
||||
*/
|
||||
const LOOKUP_STAGE_ORDER: readonly NewSessionViewV3LookupStage[] = ['repository', 'issueSummary', 'prSummary', 'issueLinkage', 'prReview'];
|
||||
|
||||
/** Outcome severity, lowest first. Stages that run more than once report their most severe outcome. */
|
||||
const LOOKUP_STAGE_OUTCOME_SEVERITY: readonly NewSessionViewV3StageOutcome[] = ['notRun', 'skipped', 'success', 'cancelled', 'requestFailed', 'timeout', 'noAuthentication'];
|
||||
|
||||
interface INewSessionViewV3PromptPlan {
|
||||
readonly prompt: string;
|
||||
@@ -62,9 +88,48 @@ interface INewSessionViewV3GitHubCandidate {
|
||||
readonly strategy: Exclude<NewSessionViewV3EffectiveStrategy, 'prompt' | 'options'>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Privacy-safe counts, durations and per-stage outcomes describing a single GitHub prompt-option
|
||||
* lookup. Only bounded categories and measurements are captured; never issue, pull request or
|
||||
* repository content.
|
||||
*/
|
||||
interface INewSessionViewV3LookupTelemetry {
|
||||
readonly gitHubOptionCount: number;
|
||||
readonly candidatesFound: number;
|
||||
readonly lookupDurationMs: number;
|
||||
readonly repositoryMs: number;
|
||||
readonly issueSummaryMs: number;
|
||||
readonly issueLinkageMs: number;
|
||||
readonly prSummaryMs: number;
|
||||
readonly prReviewMs: number;
|
||||
readonly issueSummaryOutcome: NewSessionViewV3StageOutcome;
|
||||
readonly issueLinkageOutcome: NewSessionViewV3StageOutcome;
|
||||
readonly prSummaryOutcome: NewSessionViewV3StageOutcome;
|
||||
readonly prReviewOutcome: NewSessionViewV3StageOutcome;
|
||||
readonly timedOutStage: NewSessionViewV3TimedOutStage;
|
||||
}
|
||||
|
||||
/** Reported for variations that never run a GitHub prompt-option lookup. */
|
||||
const NO_LOOKUP_TELEMETRY: INewSessionViewV3LookupTelemetry = {
|
||||
gitHubOptionCount: 0,
|
||||
candidatesFound: 0,
|
||||
lookupDurationMs: 0,
|
||||
repositoryMs: 0,
|
||||
issueSummaryMs: 0,
|
||||
issueLinkageMs: 0,
|
||||
prSummaryMs: 0,
|
||||
prReviewMs: 0,
|
||||
issueSummaryOutcome: 'notRun',
|
||||
issueLinkageOutcome: 'notRun',
|
||||
prSummaryOutcome: 'notRun',
|
||||
prReviewOutcome: 'notRun',
|
||||
timedOutStage: 'none',
|
||||
};
|
||||
|
||||
interface INewSessionViewV3PromptOptionsPlan {
|
||||
readonly options: readonly INewSessionPromptOption[];
|
||||
readonly fallbackReason: NewSessionViewV3FallbackReason;
|
||||
readonly lookup: INewSessionViewV3LookupTelemetry;
|
||||
}
|
||||
|
||||
interface INewSessionViewV3RepositoryContext {
|
||||
@@ -123,9 +188,72 @@ interface IGitHubLookupTimeouts {
|
||||
readonly reviewMs: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Collects per-stage timings and outcomes for one GitHub prompt-option lookup so the strategy
|
||||
* telemetry can report where the shared budget went instead of a single collapsed fallback reason.
|
||||
*/
|
||||
class GitHubLookupDiagnostics {
|
||||
private readonly _startTime = Date.now();
|
||||
private readonly _durations = new Map<NewSessionViewV3LookupStage, number>();
|
||||
private readonly _outcomes = new Map<NewSessionViewV3LookupStage, NewSessionViewV3StageOutcome>();
|
||||
private readonly _pending = new Map<NewSessionViewV3LookupStage, number>();
|
||||
private _timedOutStage: NewSessionViewV3TimedOutStage = 'none';
|
||||
|
||||
/** Marks a stage as in flight so it can be blamed if the total budget expires. */
|
||||
begin(stage: NewSessionViewV3LookupStage): void {
|
||||
this._pending.set(stage, (this._pending.get(stage) ?? 0) + 1);
|
||||
}
|
||||
|
||||
/** Records the outcome of a stage, keeping the longest duration and most severe outcome. */
|
||||
record(stage: NewSessionViewV3LookupStage, outcome: NewSessionViewV3StageOutcome, durationMs: number): void {
|
||||
const pending = (this._pending.get(stage) ?? 1) - 1;
|
||||
if (pending > 0) {
|
||||
this._pending.set(stage, pending);
|
||||
} else {
|
||||
this._pending.delete(stage);
|
||||
}
|
||||
this._durations.set(stage, Math.max(this._durations.get(stage) ?? 0, durationMs));
|
||||
const previous = this._outcomes.get(stage) ?? 'notRun';
|
||||
if (LOOKUP_STAGE_OUTCOME_SEVERITY.indexOf(outcome) > LOOKUP_STAGE_OUTCOME_SEVERITY.indexOf(previous)) {
|
||||
this._outcomes.set(stage, outcome);
|
||||
} else if (!this._outcomes.has(stage)) {
|
||||
this._outcomes.set(stage, previous);
|
||||
}
|
||||
}
|
||||
|
||||
/** Blames the stage that was still running when the total budget expired. */
|
||||
markTimedOut(): void {
|
||||
this._timedOutStage = [...LOOKUP_STAGE_ORDER].reverse().find(stage => this._pending.has(stage)) ?? 'unknown';
|
||||
}
|
||||
|
||||
summarize(candidatesFound: number, gitHubOptionCount: number): INewSessionViewV3LookupTelemetry {
|
||||
return {
|
||||
gitHubOptionCount,
|
||||
candidatesFound,
|
||||
lookupDurationMs: Date.now() - this._startTime,
|
||||
repositoryMs: this._durations.get('repository') ?? 0,
|
||||
issueSummaryMs: this._durations.get('issueSummary') ?? 0,
|
||||
issueLinkageMs: this._durations.get('issueLinkage') ?? 0,
|
||||
prSummaryMs: this._durations.get('prSummary') ?? 0,
|
||||
prReviewMs: this._durations.get('prReview') ?? 0,
|
||||
issueSummaryOutcome: this._outcomes.get('issueSummary') ?? 'notRun',
|
||||
issueLinkageOutcome: this._outcomes.get('issueLinkage') ?? 'notRun',
|
||||
prSummaryOutcome: this._outcomes.get('prSummary') ?? 'notRun',
|
||||
prReviewOutcome: this._outcomes.get('prReview') ?? 'notRun',
|
||||
timedOutStage: this._timedOutStage,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export class NewSessionViewV3PromptRunner {
|
||||
private readonly _gitHubLookupTimeouts: IGitHubLookupTimeouts;
|
||||
|
||||
/** Joins the strategy and interaction events belonging to the same prompt impression. */
|
||||
private _impressionId = generateUuid();
|
||||
|
||||
/** The prompt options most recently handed to the composer, used for interaction telemetry. */
|
||||
private _renderedPromptOptions: readonly INewSessionPromptOption[] = [];
|
||||
|
||||
constructor(
|
||||
private readonly _assignmentService: IWorkbenchAssignmentService,
|
||||
private readonly _configurationService: IConfigurationService,
|
||||
@@ -143,6 +271,8 @@ export class NewSessionViewV3PromptRunner {
|
||||
|
||||
async run(token: CancellationToken): Promise<boolean> {
|
||||
this._logService.info(`${LOG_PREFIX} Starting V3 prompt resolution.`);
|
||||
this._impressionId = generateUuid();
|
||||
this._renderedPromptOptions = [];
|
||||
const configuredVariation = await this._resolveConfiguredVariation();
|
||||
if (token.isCancellationRequested) {
|
||||
this._logService.trace(`${LOG_PREFIX} Prompt resolution was cancelled after resolving the configured variation.`);
|
||||
@@ -207,13 +337,31 @@ export class NewSessionViewV3PromptRunner {
|
||||
const composer = this._getActiveComposer();
|
||||
if (!composer) {
|
||||
this._logService.warn(`${LOG_PREFIX} Skipping prompt options because no active new-session composer is available.`);
|
||||
this._reportStrategy(configuredVariation, 'options', 'noCandidate', false);
|
||||
this._reportStrategy(configuredVariation, 'options', 'noComposer', false);
|
||||
return false;
|
||||
}
|
||||
|
||||
let latestPlan: INewSessionViewV3PromptOptionsPlan | undefined;
|
||||
const resolveOptions = async (refreshToken: CancellationToken): Promise<NewSessionPromptOptionsState> => {
|
||||
latestPlan = await this._resolveGitHubPromptOptionsWithFallback(refreshToken);
|
||||
const resolveOptions = async (refreshToken: CancellationToken, progress?: NewSessionPromptOptionsProgress): Promise<NewSessionPromptOptionsState> => {
|
||||
// The composer resolves again on every refresh, for example when the selected workspace
|
||||
// changes, so the rendered set must be reset per resolve rather than per run.
|
||||
this._renderedPromptOptions = [];
|
||||
const publishOptions = (options: readonly INewSessionPromptOption[]) => {
|
||||
if (!this._hasRenderedPromptOptionsChanged(options)) {
|
||||
return;
|
||||
}
|
||||
// Only record what the composer rendered: it refuses late updates while the user is
|
||||
// already acting on the options on screen, and interaction telemetry must describe
|
||||
// the options they actually saw.
|
||||
if (progress?.({ kind: 'resolved', options }) !== false) {
|
||||
this._renderedPromptOptions = options;
|
||||
}
|
||||
};
|
||||
// Paint the standard options before the GitHub lookup starts so the widget never waits on
|
||||
// it, then stream personalized options into their slots as each lookup stage completes.
|
||||
publishOptions(this._createPromptOptions([]));
|
||||
latestPlan = await this._resolveGitHubPromptOptionsWithFallback(refreshToken, publishOptions);
|
||||
publishOptions(latestPlan.options);
|
||||
return { kind: 'resolved', options: latestPlan.options };
|
||||
};
|
||||
if (composer.setPromptOptionsController && composer.refreshPromptOptions) {
|
||||
@@ -227,13 +375,13 @@ export class NewSessionViewV3PromptRunner {
|
||||
const shown = await composer.refreshPromptOptions(token);
|
||||
const fallbackReason = configuredFallbackReason ?? latestPlan?.fallbackReason ?? (token.isCancellationRequested ? 'requestFailed' : 'noCandidate');
|
||||
this._logService.info(`${LOG_PREFIX} Prompt options completed with shown=${shown} and fallback reason '${fallbackReason}'.`);
|
||||
this._reportStrategy(configuredVariation, 'options', fallbackReason, shown);
|
||||
this._reportStrategy(configuredVariation, 'options', fallbackReason, shown, latestPlan?.lookup);
|
||||
return shown;
|
||||
}
|
||||
|
||||
if (!composer.showPromptOptions({ kind: 'loading' })) {
|
||||
this._logService.warn(`${LOG_PREFIX} Skipping prompt options because the active new-session composer cannot show them.`);
|
||||
this._reportStrategy(configuredVariation, 'options', 'noCandidate', false);
|
||||
this._reportStrategy(configuredVariation, 'options', 'noComposer', false);
|
||||
return false;
|
||||
}
|
||||
this._logService.info(`${LOG_PREFIX} Showing prompt option loading skeletons.`);
|
||||
@@ -241,17 +389,26 @@ export class NewSessionViewV3PromptRunner {
|
||||
if (token.isCancellationRequested || this._newSessionComposerService.activeComposer.get() !== composer || this._sessionsService.activeSession.get()?.isCreated.get()) {
|
||||
composer.showPromptOptions(undefined);
|
||||
this._logService.trace(`${LOG_PREFIX} Prompt option resolution was cancelled or its composer is no longer active.`);
|
||||
this._reportStrategy(configuredVariation, 'options', configuredFallbackReason ?? latestPlan?.fallbackReason ?? 'requestFailed', false);
|
||||
this._reportStrategy(configuredVariation, 'options', configuredFallbackReason ?? latestPlan?.fallbackReason ?? 'requestFailed', false, latestPlan?.lookup);
|
||||
return false;
|
||||
}
|
||||
|
||||
const shown = composer.showPromptOptions(state);
|
||||
const fallbackReason = configuredFallbackReason ?? latestPlan?.fallbackReason ?? 'noCandidate';
|
||||
this._logService.info(`${LOG_PREFIX} Prompt options completed with shown=${shown} and fallback reason '${fallbackReason}'.`);
|
||||
this._reportStrategy(configuredVariation, 'options', fallbackReason, shown);
|
||||
this._reportStrategy(configuredVariation, 'options', fallbackReason, shown, latestPlan?.lookup);
|
||||
return shown;
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the given prompt options differ from the ones last rendered, so unchanged refinements
|
||||
* never re-render the widget.
|
||||
*/
|
||||
private _hasRenderedPromptOptionsChanged(options: readonly INewSessionPromptOption[]): boolean {
|
||||
return this._renderedPromptOptions.length !== options.length
|
||||
|| options.some((option, index) => this._renderedPromptOptions[index]?.id !== option.id);
|
||||
}
|
||||
|
||||
private _getActiveComposer(): INewSessionComposer | undefined {
|
||||
const activeSession = this._sessionsService.activeSession.get();
|
||||
if (activeSession?.isCreated.get()) {
|
||||
@@ -260,23 +417,28 @@ export class NewSessionViewV3PromptRunner {
|
||||
return this._newSessionComposerService.activeComposer.get();
|
||||
}
|
||||
|
||||
private async _resolveGitHubPromptOptionsWithFallback(token: CancellationToken): Promise<INewSessionViewV3PromptOptionsPlan> {
|
||||
private async _resolveGitHubPromptOptionsWithFallback(token: CancellationToken, reportOptions?: (options: readonly INewSessionPromptOption[]) => void): Promise<INewSessionViewV3PromptOptionsPlan> {
|
||||
this._logService.info(`${LOG_PREFIX} Starting GitHub prompt option lookup with a ${this._gitHubLookupTimeouts.totalMs}ms total timeout.`);
|
||||
const operationCts = new CancellationTokenSource(token);
|
||||
const diagnostics = new GitHubLookupDiagnostics();
|
||||
let latestProgress: IGitHubPromptOptionsProgress | undefined;
|
||||
let timedOut = false;
|
||||
const createTimeoutPlan = () => {
|
||||
const candidates = latestProgress && this._isCurrentRepositoryContext(latestProgress.context)
|
||||
? [...latestProgress.issueCandidates, ...latestProgress.pullRequestCandidates]
|
||||
: [];
|
||||
return this._createPromptOptionsPlan(candidates.slice(0, PROMPT_OPTION_COUNT), candidates.length === PROMPT_OPTION_COUNT ? 'none' : 'timeout');
|
||||
return this._createPromptOptionsPlan(candidates, candidates.length === PROMPT_OPTION_COUNT ? 'none' : 'timeout', diagnostics);
|
||||
};
|
||||
try {
|
||||
const result = await raceTimeout(
|
||||
this._resolveGitHubPromptOptions(operationCts.token, progress => latestProgress = progress),
|
||||
this._resolveGitHubPromptOptions(operationCts.token, diagnostics, progress => {
|
||||
latestProgress = progress;
|
||||
reportOptions?.(this._createPromptOptions([...progress.issueCandidates, ...progress.pullRequestCandidates]));
|
||||
}),
|
||||
this._gitHubLookupTimeouts.totalMs,
|
||||
() => {
|
||||
timedOut = true;
|
||||
diagnostics.markTimedOut();
|
||||
this._logService.warn(`${LOG_PREFIX} GitHub prompt option lookup timed out after ${this._gitHubLookupTimeouts.totalMs}ms; filling with standard options.`);
|
||||
operationCts.cancel();
|
||||
},
|
||||
@@ -285,34 +447,37 @@ export class NewSessionViewV3PromptRunner {
|
||||
return createTimeoutPlan();
|
||||
}
|
||||
if (result.kind === 'fallback') {
|
||||
return this._createPromptOptionsPlan([], result.reason);
|
||||
return this._createPromptOptionsPlan([], result.reason, diagnostics);
|
||||
}
|
||||
|
||||
const candidates = [...result.issueCandidates, ...result.pullRequestCandidates].slice(0, PROMPT_OPTION_COUNT);
|
||||
const fallbackReason = candidates.length === PROMPT_OPTION_COUNT ? 'none' : getLookupFallbackReason(result.failures);
|
||||
return this._createPromptOptionsPlan(candidates, fallbackReason);
|
||||
const candidates = [...result.issueCandidates, ...result.pullRequestCandidates];
|
||||
const fallbackReason = candidates.slice(0, PROMPT_OPTION_COUNT).length === PROMPT_OPTION_COUNT ? 'none' : getLookupFallbackReason(result.failures);
|
||||
return this._createPromptOptionsPlan(candidates, fallbackReason, diagnostics);
|
||||
} catch (error) {
|
||||
if (isCancellationError(error) && timedOut) {
|
||||
return createTimeoutPlan();
|
||||
}
|
||||
if (isCancellationError(error) && token.isCancellationRequested) {
|
||||
this._logService.trace(`${LOG_PREFIX} GitHub prompt option lookup was cancelled by the onboarding flow.`);
|
||||
return this._createPromptOptionsPlan([], 'requestFailed');
|
||||
return this._createPromptOptionsPlan([], 'requestFailed', diagnostics);
|
||||
}
|
||||
if (error instanceof GitHubAuthenticationError) {
|
||||
this._logService.warn(`${LOG_PREFIX} No existing GitHub authentication session is available; filling with standard options without requesting sign-in.`);
|
||||
return this._createPromptOptionsPlan([], 'noAuthentication');
|
||||
return this._createPromptOptionsPlan([], 'noAuthentication', diagnostics);
|
||||
}
|
||||
this._logService.error(`${LOG_PREFIX} GitHub prompt option lookup failed; filling with standard options.`, error);
|
||||
return this._createPromptOptionsPlan([], 'requestFailed');
|
||||
return this._createPromptOptionsPlan([], 'requestFailed', diagnostics);
|
||||
} finally {
|
||||
operationCts.dispose();
|
||||
}
|
||||
}
|
||||
|
||||
private async _resolveGitHubPromptOptions(token: CancellationToken, reportProgress: (progress: IGitHubPromptOptionsProgress) => void): Promise<GitHubPromptOptionsResult> {
|
||||
private async _resolveGitHubPromptOptions(token: CancellationToken, diagnostics: GitHubLookupDiagnostics, reportProgress: (progress: IGitHubPromptOptionsProgress) => void): Promise<GitHubPromptOptionsResult> {
|
||||
while (!token.isCancellationRequested) {
|
||||
diagnostics.begin('repository');
|
||||
const repositoryStartTime = Date.now();
|
||||
const context = await this._resolveGitHubRepository(token);
|
||||
diagnostics.record('repository', context ? 'success' : 'skipped', Date.now() - repositoryStartTime);
|
||||
if (!context) {
|
||||
this._logService.warn(`${LOG_PREFIX} Could not resolve a GitHub repository for prompt options.`);
|
||||
return { kind: 'fallback', reason: 'noRepository' };
|
||||
@@ -336,12 +501,15 @@ export class NewSessionViewV3PromptRunner {
|
||||
};
|
||||
publishProgress();
|
||||
const resolveIssues = async () => {
|
||||
issueResult = await this._resolveIssuePromptOptionCandidates(owner, repo, lookupCts.token);
|
||||
issueResult = await this._resolveIssuePromptOptionCandidates(owner, repo, lookupCts.token, diagnostics, candidates => {
|
||||
issueResult = { candidates, failures: [] };
|
||||
publishProgress();
|
||||
});
|
||||
publishProgress();
|
||||
return issueResult;
|
||||
};
|
||||
const resolvePullRequests = async () => {
|
||||
pullRequestResult = await this._resolvePullRequestPromptOptionCandidates(owner, repo, lookupCts.token, candidates => {
|
||||
pullRequestResult = await this._resolvePullRequestPromptOptionCandidates(owner, repo, lookupCts.token, diagnostics, candidates => {
|
||||
pullRequestResult = { candidates, failures: [] };
|
||||
publishProgress();
|
||||
});
|
||||
@@ -369,22 +537,25 @@ export class NewSessionViewV3PromptRunner {
|
||||
return { kind: 'fallback', reason: 'noRepository' };
|
||||
}
|
||||
|
||||
private async _resolveIssuePromptOptionCandidates(owner: string, repo: string, token: CancellationToken): Promise<IGitHubCandidateLookupResult> {
|
||||
const outcome = await this._resolveIssueCandidates(owner, repo, token);
|
||||
private async _resolveIssuePromptOptionCandidates(
|
||||
owner: string,
|
||||
repo: string,
|
||||
token: CancellationToken,
|
||||
diagnostics?: GitHubLookupDiagnostics,
|
||||
reportCandidates: (candidates: readonly INewSessionViewV3GitHubCandidate[]) => void = () => undefined,
|
||||
): Promise<IGitHubCandidateLookupResult> {
|
||||
const outcome = await this._resolveIssueCandidates(owner, repo, token, diagnostics, issues => reportCandidates(toIssuePromptOptionCandidates(issues)));
|
||||
if (outcome.kind === 'failure') {
|
||||
return { candidates: [], failures: [outcome.reason] };
|
||||
}
|
||||
const candidates = [...outcome.value]
|
||||
.sort(compareUpdatedAtDescending)
|
||||
.slice(0, 2)
|
||||
.map(issue => ({ number: issue.number, title: issue.title, url: issue.url, strategy: 'githubIssue' as const }));
|
||||
return { candidates, failures: [] };
|
||||
return { candidates: toIssuePromptOptionCandidates(outcome.value), failures: [] };
|
||||
}
|
||||
|
||||
private async _resolvePullRequestPromptOptionCandidates(
|
||||
owner: string,
|
||||
repo: string,
|
||||
token: CancellationToken,
|
||||
diagnostics?: GitHubLookupDiagnostics,
|
||||
reportCandidates: (candidates: readonly INewSessionViewV3GitHubCandidate[]) => void = () => undefined,
|
||||
): Promise<IGitHubCandidateLookupResult> {
|
||||
const summary = await this._runGitHubLookup(
|
||||
@@ -392,6 +563,8 @@ export class NewSessionViewV3PromptRunner {
|
||||
this._gitHubLookupTimeouts.summaryMs,
|
||||
token,
|
||||
lookupToken => this._gitHubService.getRecentAuthoredPullRequests(owner, repo, lookupToken),
|
||||
'prSummary',
|
||||
diagnostics,
|
||||
);
|
||||
if (summary.kind === 'failure') {
|
||||
return { candidates: [], failures: [summary.reason] };
|
||||
@@ -413,7 +586,10 @@ export class NewSessionViewV3PromptRunner {
|
||||
reportCandidates(stableCandidates);
|
||||
}
|
||||
|
||||
const reviewLookup = await this._resolveReviewCandidates(owner, repo, reviewPullRequests, token);
|
||||
// Review-thread enrichment is a post-render refinement: candidates whose position can no longer
|
||||
// change were already published above, so this stage can only ever delay the `githubReviewComments`
|
||||
// option, never the merge-conflict, CI or issue options that are already known.
|
||||
const reviewLookup = await this._resolveReviewCandidates(owner, repo, reviewPullRequests, token, diagnostics);
|
||||
const candidates = getCandidatesInPullRequestOrder(
|
||||
pullRequests,
|
||||
[...directCandidates.map(entry => entry.candidate), ...reviewLookup.candidates],
|
||||
@@ -551,22 +727,37 @@ export class NewSessionViewV3PromptRunner {
|
||||
return { kind: 'fallback', reason: 'noRepository' };
|
||||
}
|
||||
|
||||
private async _resolveIssueCandidates(owner: string, repo: string, token: CancellationToken): Promise<GitHubLookupOutcome<readonly IGitHubRecentIssue[]>> {
|
||||
private async _resolveIssueCandidates(
|
||||
owner: string,
|
||||
repo: string,
|
||||
token: CancellationToken,
|
||||
diagnostics?: GitHubLookupDiagnostics,
|
||||
reportIssues?: (issues: readonly IGitHubRecentIssue[]) => void,
|
||||
): Promise<GitHubLookupOutcome<readonly IGitHubRecentIssue[]>> {
|
||||
const issues = await this._runGitHubLookup(
|
||||
'assigned issue summaries',
|
||||
this._gitHubLookupTimeouts.summaryMs,
|
||||
token,
|
||||
lookupToken => this._gitHubService.getRecentAssignedIssues(owner, repo, lookupToken),
|
||||
'issueSummary',
|
||||
diagnostics,
|
||||
);
|
||||
if (issues.kind === 'failure' || issues.value.length === 0) {
|
||||
diagnostics?.record('issueLinkage', 'skipped', 0);
|
||||
return issues;
|
||||
}
|
||||
|
||||
// Publish the issues that are already in hand before refining them: linkage only removes issues
|
||||
// that already have a linked pull request, so a slow linkage call must never hide them.
|
||||
reportIssues?.(issues.value);
|
||||
|
||||
const linkedIssues = await this._runGitHubLookup(
|
||||
'issue pull request linkage',
|
||||
this._gitHubLookupTimeouts.linkageMs,
|
||||
token,
|
||||
lookupToken => this._gitHubService.getIssuesWithLinkedPullRequests(owner, repo, issues.value.map(issue => issue.number), lookupToken),
|
||||
'issueLinkage',
|
||||
diagnostics,
|
||||
);
|
||||
if (linkedIssues.kind === 'success') {
|
||||
const unlinkedIssues = issues.value.filter(issue => !linkedIssues.value.has(issue.number));
|
||||
@@ -586,10 +777,12 @@ export class NewSessionViewV3PromptRunner {
|
||||
repo: string,
|
||||
pullRequests: readonly IGitHubRecentPullRequest[],
|
||||
token: CancellationToken,
|
||||
diagnostics?: GitHubLookupDiagnostics,
|
||||
): Promise<IGitHubReviewLookupResult> {
|
||||
const eligiblePullRequests = pullRequests.filter(pullRequest => !!pullRequest.latestCommitAt);
|
||||
if (eligiblePullRequests.length === 0) {
|
||||
this._logService.info(`${LOG_PREFIX} No pull requests have a latest commit timestamp, so review-thread lookup is unnecessary.`);
|
||||
diagnostics?.record('prReview', 'skipped', 0);
|
||||
return { candidates: [], failures: [] };
|
||||
}
|
||||
|
||||
@@ -600,6 +793,8 @@ export class NewSessionViewV3PromptRunner {
|
||||
this._gitHubLookupTimeouts.reviewMs,
|
||||
token,
|
||||
lookupToken => this._gitHubService.getPullRequestReviewThreads(owner, repo, pullRequest.number, lookupToken),
|
||||
'prReview',
|
||||
diagnostics,
|
||||
);
|
||||
if (outcome.kind === 'success') {
|
||||
const completedPullRequest = { ...pullRequest, reviewThreads: outcome.value };
|
||||
@@ -630,11 +825,23 @@ export class NewSessionViewV3PromptRunner {
|
||||
timeoutMs: number,
|
||||
token: CancellationToken,
|
||||
lookup: (token: CancellationToken) => Promise<T>,
|
||||
stage?: NewSessionViewV3LookupStage,
|
||||
diagnostics?: GitHubLookupDiagnostics,
|
||||
): Promise<GitHubLookupOutcome<T>> {
|
||||
const lookupCts = new CancellationTokenSource(token);
|
||||
const startTime = Date.now();
|
||||
let timedOut = false;
|
||||
this._logService.trace(`${LOG_PREFIX} Starting ${label} lookup with a ${timeoutMs}ms timeout.`);
|
||||
const record = (outcome: NewSessionViewV3StageOutcome): number => {
|
||||
const durationMs = Date.now() - startTime;
|
||||
if (stage) {
|
||||
diagnostics?.record(stage, outcome, durationMs);
|
||||
}
|
||||
return durationMs;
|
||||
};
|
||||
if (stage) {
|
||||
diagnostics?.begin(stage);
|
||||
}
|
||||
try {
|
||||
const value = await raceTimeout(
|
||||
lookup(lookupCts.token),
|
||||
@@ -646,23 +853,27 @@ export class NewSessionViewV3PromptRunner {
|
||||
},
|
||||
);
|
||||
if (timedOut || value === undefined) {
|
||||
record('timeout');
|
||||
return { kind: 'failure', reason: 'timeout' };
|
||||
}
|
||||
this._logService.info(`${LOG_PREFIX} ${capitalize(label)} lookup completed in ${Date.now() - startTime}ms.`);
|
||||
this._logService.info(`${LOG_PREFIX} ${capitalize(label)} lookup completed in ${record('success')}ms.`);
|
||||
return { kind: 'success', value };
|
||||
} catch (error) {
|
||||
if (timedOut) {
|
||||
record('timeout');
|
||||
return { kind: 'failure', reason: 'timeout' };
|
||||
}
|
||||
if (error instanceof GitHubAuthenticationError) {
|
||||
this._logService.warn(`${LOG_PREFIX} ${capitalize(label)} lookup could not run because no existing GitHub authentication session is available.`);
|
||||
record('noAuthentication');
|
||||
return { kind: 'failure', reason: 'noAuthentication' };
|
||||
}
|
||||
if (isCancellationError(error) && token.isCancellationRequested) {
|
||||
this._logService.trace(`${LOG_PREFIX} ${capitalize(label)} lookup was cancelled.`);
|
||||
record('cancelled');
|
||||
return { kind: 'failure', reason: 'cancelled' };
|
||||
}
|
||||
this._logService.error(`${LOG_PREFIX} ${capitalize(label)} lookup failed after ${Date.now() - startTime}ms.`, error);
|
||||
this._logService.error(`${LOG_PREFIX} ${capitalize(label)} lookup failed after ${record('requestFailed')}ms.`, error);
|
||||
return { kind: 'failure', reason: 'requestFailed' };
|
||||
} finally {
|
||||
lookupCts.dispose();
|
||||
@@ -834,13 +1045,20 @@ export class NewSessionViewV3PromptRunner {
|
||||
};
|
||||
}
|
||||
|
||||
private _createPromptOptionsPlan(candidates: readonly INewSessionViewV3GitHubCandidate[], fallbackReason: NewSessionViewV3FallbackReason): INewSessionViewV3PromptOptionsPlan {
|
||||
private _createPromptOptionsPlan(candidates: readonly INewSessionViewV3GitHubCandidate[], fallbackReason: NewSessionViewV3FallbackReason, diagnostics: GitHubLookupDiagnostics): INewSessionViewV3PromptOptionsPlan {
|
||||
const options = this._createPromptOptions(candidates);
|
||||
return {
|
||||
options,
|
||||
fallbackReason,
|
||||
lookup: diagnostics.summarize(candidates.length, Math.min(candidates.length, PROMPT_OPTION_COUNT)),
|
||||
};
|
||||
}
|
||||
|
||||
/** Fills the fixed number of prompt slots with the given GitHub candidates, padded with standard options. */
|
||||
private _createPromptOptions(candidates: readonly INewSessionViewV3GitHubCandidate[]): readonly INewSessionPromptOption[] {
|
||||
const gitHubOptions = candidates.slice(0, PROMPT_OPTION_COUNT).map(candidate => this._createGitHubPromptOption(candidate));
|
||||
const standardOptions = this._createStandardPromptOptions();
|
||||
return {
|
||||
options: [...gitHubOptions, ...standardOptions.slice(0, PROMPT_OPTION_COUNT - gitHubOptions.length)],
|
||||
fallbackReason,
|
||||
};
|
||||
return [...gitHubOptions, ...standardOptions.slice(0, PROMPT_OPTION_COUNT - gitHubOptions.length)];
|
||||
}
|
||||
|
||||
private _createGitHubPromptOption(candidate: INewSessionViewV3GitHubCandidate): INewSessionPromptOption {
|
||||
@@ -933,49 +1151,88 @@ export class NewSessionViewV3PromptRunner {
|
||||
return composer.animatePrompt(prompt, NEW_SESSION_PROMPT_TYPING_DURATION_MS, taskPlaceholder, token);
|
||||
}
|
||||
|
||||
private _reportStrategy(configuredVariation: NewSessionViewV3ConfiguredVariation, effectiveStrategy: NewSessionViewV3EffectiveStrategy, fallbackReason: NewSessionViewV3FallbackReason, shown: boolean): void {
|
||||
private _reportStrategy(configuredVariation: NewSessionViewV3ConfiguredVariation, effectiveStrategy: NewSessionViewV3EffectiveStrategy, fallbackReason: NewSessionViewV3FallbackReason, shown: boolean, lookup: INewSessionViewV3LookupTelemetry = NO_LOOKUP_TELEMETRY): void {
|
||||
type OnboardingPromptStrategyEvent = {
|
||||
scenarioId: string;
|
||||
impressionId: string;
|
||||
configuredVariation: string;
|
||||
effectiveStrategy: string;
|
||||
fallbackReason: string;
|
||||
shown: boolean;
|
||||
gitHubOptionCount: number;
|
||||
candidatesFound: number;
|
||||
lookupDurationMs: number;
|
||||
repositoryMs: number;
|
||||
issueSummaryMs: number;
|
||||
issueLinkageMs: number;
|
||||
prSummaryMs: number;
|
||||
prReviewMs: number;
|
||||
issueSummaryOutcome: string;
|
||||
issueLinkageOutcome: string;
|
||||
prSummaryOutcome: string;
|
||||
prReviewOutcome: string;
|
||||
timedOutStage: string;
|
||||
};
|
||||
type OnboardingPromptStrategyClassification = {
|
||||
owner: 'benibenj';
|
||||
comment: 'Reports which prompt experience an onboarding tour selected without collecting prompt or repository content.';
|
||||
comment: 'Reports which prompt experience an onboarding tour selected, and where the GitHub personalization budget went, without collecting prompt or repository content.';
|
||||
scenarioId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The id of the onboarding scenario that ran.' };
|
||||
impressionId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'A random identifier generated for this prompt impression, used only to join this event with the interaction events of the same impression.' };
|
||||
configuredVariation: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The configured prompt experience, reduced to a known category.' };
|
||||
effectiveStrategy: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The effective prompt or prompt-option strategy selected for the tour.' };
|
||||
fallbackReason: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'The categorical reason GitHub personalization fell back to a default prompt or standard options.' };
|
||||
shown: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether the selected prompt or prompt-option widget was shown.' };
|
||||
gitHubOptionCount: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'How many of the rendered prompt-option slots were personalized GitHub items, from zero to the number of slots.' };
|
||||
candidatesFound: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'How many GitHub candidates the lookup found before they were truncated to the number of prompt-option slots.' };
|
||||
lookupDurationMs: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; isMeasurement: true; comment: 'Total time in milliseconds spent in the GitHub prompt-option lookup, including repository resolution.' };
|
||||
repositoryMs: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; isMeasurement: true; comment: 'Time in milliseconds spent resolving the GitHub repository of the selected workspace, or zero when that stage did not run.' };
|
||||
issueSummaryMs: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; isMeasurement: true; comment: 'Time in milliseconds spent fetching assigned issue summaries, or zero when that stage did not run.' };
|
||||
issueLinkageMs: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; isMeasurement: true; comment: 'Time in milliseconds spent checking which assigned issues already have linked pull requests, or zero when that stage did not run.' };
|
||||
prSummaryMs: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; isMeasurement: true; comment: 'Time in milliseconds spent fetching authored pull request summaries, or zero when that stage did not run.' };
|
||||
prReviewMs: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; isMeasurement: true; comment: 'Longest time in milliseconds spent fetching pull request review threads, or zero when that stage did not run.' };
|
||||
issueSummaryOutcome: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'The categorical outcome of the assigned issue summary stage.' };
|
||||
issueLinkageOutcome: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'The categorical outcome of the issue pull request linkage stage.' };
|
||||
prSummaryOutcome: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'The categorical outcome of the authored pull request summary stage.' };
|
||||
prReviewOutcome: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'The most severe categorical outcome of the pull request review-thread stage.' };
|
||||
timedOutStage: { classification: 'SystemMetaData'; purpose: 'PerformanceAndHealth'; comment: 'The categorical lookup stage that was still running when the total lookup budget expired, or none.' };
|
||||
};
|
||||
this._telemetryService.publicLog2<OnboardingPromptStrategyEvent, OnboardingPromptStrategyClassification>('onboarding.promptStrategy', {
|
||||
scenarioId: NEW_SESSION_VIEW_V3_TOUR_ID,
|
||||
impressionId: this._impressionId,
|
||||
configuredVariation,
|
||||
effectiveStrategy,
|
||||
fallbackReason,
|
||||
shown,
|
||||
...lookup,
|
||||
});
|
||||
}
|
||||
|
||||
private _reportPromptOptionInteraction(interaction: 'selected' | 'closed', option?: INewSessionPromptOption): void {
|
||||
type OnboardingPromptOptionInteractionEvent = {
|
||||
scenarioId: string;
|
||||
impressionId: string;
|
||||
interaction: string;
|
||||
option: string;
|
||||
optionIndex: number;
|
||||
optionKindsShown: string;
|
||||
};
|
||||
type OnboardingPromptOptionInteractionClassification = {
|
||||
owner: 'benibenj';
|
||||
comment: 'Reports privacy-safe interactions with V3 onboarding prompt options without collecting prompt or repository content.';
|
||||
scenarioId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The id of the onboarding scenario that showed the prompt options.' };
|
||||
impressionId: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'A random identifier generated for this prompt impression, used only to join this event with the strategy event of the same impression.' };
|
||||
interaction: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'Whether an option was selected or the prompt-option widget was closed.' };
|
||||
option: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The categorical prompt option selected, or none when the widget was closed.' };
|
||||
optionIndex: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; isMeasurement: true; comment: 'The zero-based slot position of the selected option, or -1 when the widget was closed or the option is no longer rendered.' };
|
||||
optionKindsShown: { classification: 'SystemMetaData'; purpose: 'FeatureInsight'; comment: 'The comma separated categorical kinds of the prompt options that were rendered, in slot order.' };
|
||||
};
|
||||
this._telemetryService.publicLog2<OnboardingPromptOptionInteractionEvent, OnboardingPromptOptionInteractionClassification>('onboarding.promptOptionInteraction', {
|
||||
scenarioId: NEW_SESSION_VIEW_V3_TOUR_ID,
|
||||
impressionId: this._impressionId,
|
||||
interaction,
|
||||
option: option ? getPromptOptionTelemetryKind(option) : 'none',
|
||||
optionIndex: option ? this._renderedPromptOptions.findIndex(candidate => candidate.id === option.id) : -1,
|
||||
optionKindsShown: this._renderedPromptOptions.map(getPromptOptionTelemetryKind).join(','),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -1050,6 +1307,13 @@ function toCandidate(pullRequest: IGitHubRecentPullRequest, strategy: 'githubMer
|
||||
return { number: pullRequest.number, title: pullRequest.title, url: pullRequest.url, strategy };
|
||||
}
|
||||
|
||||
function toIssuePromptOptionCandidates(issues: readonly IGitHubRecentIssue[]): INewSessionViewV3GitHubCandidate[] {
|
||||
return [...issues]
|
||||
.sort(compareUpdatedAtDescending)
|
||||
.slice(0, 2)
|
||||
.map(issue => ({ number: issue.number, title: issue.title, url: issue.url, strategy: 'githubIssue' as const }));
|
||||
}
|
||||
|
||||
function getPromptOptionTelemetryKind(option: INewSessionPromptOption): 'implementFeature' | 'fixBug' | 'fixCI' | 'githubIssue' | 'githubPRConflicts' | 'githubPRCI' | 'githubPRComments' | 'unknown' {
|
||||
switch (option.id.split(':', 1)[0]) {
|
||||
case 'standard':
|
||||
|
||||
+361
-25
@@ -39,12 +39,15 @@ class TestAssignmentService extends NullWorkbenchAssignmentService {
|
||||
}
|
||||
}
|
||||
|
||||
type TestTelemetryData = { readonly [key: string]: string | number | boolean | undefined };
|
||||
type TestTelemetryEvent = { readonly name: string; readonly data: TestTelemetryData | undefined };
|
||||
|
||||
class TestTelemetryService extends NullTelemetryServiceShape {
|
||||
readonly events: { readonly name: string; readonly data: object | undefined }[] = [];
|
||||
readonly events: TestTelemetryEvent[] = [];
|
||||
|
||||
override publicLog2(name?: string, data?: object): void {
|
||||
if (name) {
|
||||
this.events.push({ name, data });
|
||||
this.events.push({ name, data: data as TestTelemetryData | undefined });
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -61,6 +64,34 @@ type TestGitHubRequest =
|
||||
| { readonly kind: 'reviews'; readonly owner: string; readonly repo: string; readonly pullRequestNumber: number }
|
||||
| { readonly kind: 'issueLinkage'; readonly owner: string; readonly repo: string; readonly issueNumbers: readonly number[] };
|
||||
|
||||
/** The stage duration fields, which {@link summarizeTelemetry} normalizes because they vary per run. */
|
||||
const LOOKUP_DURATIONS = {
|
||||
lookupDurationMs: '<duration>',
|
||||
repositoryMs: '<duration>',
|
||||
issueSummaryMs: '<duration>',
|
||||
issueLinkageMs: '<duration>',
|
||||
prSummaryMs: '<duration>',
|
||||
prReviewMs: '<duration>',
|
||||
};
|
||||
|
||||
/** Reported by the variations that never run a GitHub prompt-option lookup. */
|
||||
const NO_LOOKUP_TELEMETRY = {
|
||||
...LOOKUP_DURATIONS,
|
||||
gitHubOptionCount: 0,
|
||||
candidatesFound: 0,
|
||||
issueSummaryOutcome: 'notRun',
|
||||
issueLinkageOutcome: 'notRun',
|
||||
prSummaryOutcome: 'notRun',
|
||||
prReviewOutcome: 'notRun',
|
||||
timedOutStage: 'none',
|
||||
};
|
||||
|
||||
const STANDARD_OPTIONS = [
|
||||
{ title: 'Implement a feature', description: 'Describe what you want to build', icon: { id: 'lightbulb-sparkle-autofix', color: undefined } },
|
||||
{ title: 'Fix a bug', description: 'Describe the unexpected behavior', icon: { id: 'bug', color: undefined } },
|
||||
{ title: 'Fix CI', description: 'Describe a failing check or paste a link', icon: { id: 'run-errors', color: undefined } },
|
||||
];
|
||||
|
||||
suite('NewSessionViewV3Prompt', () => {
|
||||
ensureNoDisposablesAreLeakedInTestSuite();
|
||||
|
||||
@@ -122,23 +153,26 @@ suite('NewSessionViewV3Prompt', () => {
|
||||
animation: undefined,
|
||||
states: [
|
||||
{ kind: 'loading' },
|
||||
{
|
||||
kind: 'resolved',
|
||||
options: [
|
||||
{ title: 'Implement a feature', description: 'Describe what you want to build', icon: { id: 'lightbulb-sparkle-autofix', color: undefined } },
|
||||
{ title: 'Fix a bug', description: 'Describe the unexpected behavior', icon: { id: 'bug', color: undefined } },
|
||||
{ title: 'Fix CI', description: 'Describe a failing check or paste a link', icon: { id: 'run-errors', color: undefined } },
|
||||
],
|
||||
},
|
||||
{ kind: 'resolved', options: STANDARD_OPTIONS },
|
||||
{ kind: 'resolved', options: STANDARD_OPTIONS },
|
||||
],
|
||||
telemetry: [{
|
||||
name: 'onboarding.promptStrategy',
|
||||
data: {
|
||||
scenarioId: NEW_SESSION_VIEW_V3_TOUR_ID,
|
||||
impressionId: '<impressionId>',
|
||||
configuredVariation: 'options',
|
||||
effectiveStrategy: 'options',
|
||||
fallbackReason: 'noCandidate',
|
||||
shown: true,
|
||||
...LOOKUP_DURATIONS,
|
||||
gitHubOptionCount: 0,
|
||||
candidatesFound: 0,
|
||||
issueSummaryOutcome: 'success',
|
||||
issueLinkageOutcome: 'skipped',
|
||||
prSummaryOutcome: 'success',
|
||||
prReviewOutcome: 'skipped',
|
||||
timedOutStage: 'none',
|
||||
},
|
||||
}],
|
||||
});
|
||||
@@ -191,10 +225,12 @@ suite('NewSessionViewV3Prompt', () => {
|
||||
name: 'onboarding.promptStrategy',
|
||||
data: {
|
||||
scenarioId: NEW_SESSION_VIEW_V3_TOUR_ID,
|
||||
impressionId: '<impressionId>',
|
||||
configuredVariation: 'githubPrompt',
|
||||
effectiveStrategy: 'githubCiFailure',
|
||||
fallbackReason: 'none',
|
||||
shown: true,
|
||||
...NO_LOOKUP_TELEMETRY,
|
||||
},
|
||||
}],
|
||||
});
|
||||
@@ -224,10 +260,12 @@ suite('NewSessionViewV3Prompt', () => {
|
||||
name: 'onboarding.promptStrategy',
|
||||
data: {
|
||||
scenarioId: NEW_SESSION_VIEW_V3_TOUR_ID,
|
||||
impressionId: '<impressionId>',
|
||||
configuredVariation: 'githubPrompt',
|
||||
effectiveStrategy: 'githubMergeConflict',
|
||||
fallbackReason: 'none',
|
||||
shown: true,
|
||||
...NO_LOOKUP_TELEMETRY,
|
||||
},
|
||||
}],
|
||||
});
|
||||
@@ -251,10 +289,12 @@ suite('NewSessionViewV3Prompt', () => {
|
||||
name: 'onboarding.promptStrategy',
|
||||
data: {
|
||||
scenarioId: NEW_SESSION_VIEW_V3_TOUR_ID,
|
||||
impressionId: '<impressionId>',
|
||||
configuredVariation: 'githubPrompt',
|
||||
effectiveStrategy: 'prompt',
|
||||
fallbackReason: 'noAuthentication',
|
||||
shown: true,
|
||||
...NO_LOOKUP_TELEMETRY,
|
||||
},
|
||||
}],
|
||||
});
|
||||
@@ -286,6 +326,23 @@ suite('NewSessionViewV3Prompt', () => {
|
||||
animation: undefined,
|
||||
states: [
|
||||
{ kind: 'loading' },
|
||||
{ kind: 'resolved', options: STANDARD_OPTIONS },
|
||||
{
|
||||
kind: 'resolved',
|
||||
options: [
|
||||
{ title: 'Tackle issue #14', description: 'Newest assigned issue', icon: { id: 'issue-opened', color: 'charts.green' } },
|
||||
{ title: 'Tackle issue #12', description: 'Older assigned issue', icon: { id: 'issue-opened', color: 'charts.green' } },
|
||||
STANDARD_OPTIONS[0],
|
||||
],
|
||||
},
|
||||
{
|
||||
kind: 'resolved',
|
||||
options: [
|
||||
{ title: 'Tackle issue #14', description: 'Newest assigned issue', icon: { id: 'issue-opened', color: 'charts.green' } },
|
||||
{ title: 'Tackle issue #12', description: 'Older assigned issue', icon: { id: 'issue-opened', color: 'charts.green' } },
|
||||
{ title: 'Resolve conflicts #20', description: 'Conflicted PR', icon: { id: 'git-pull-request-error', color: 'charts.orange' } },
|
||||
],
|
||||
},
|
||||
{
|
||||
kind: 'resolved',
|
||||
options: [
|
||||
@@ -299,10 +356,19 @@ suite('NewSessionViewV3Prompt', () => {
|
||||
name: 'onboarding.promptStrategy',
|
||||
data: {
|
||||
scenarioId: NEW_SESSION_VIEW_V3_TOUR_ID,
|
||||
impressionId: '<impressionId>',
|
||||
configuredVariation: 'options',
|
||||
effectiveStrategy: 'options',
|
||||
fallbackReason: 'none',
|
||||
shown: true,
|
||||
...LOOKUP_DURATIONS,
|
||||
gitHubOptionCount: 3,
|
||||
candidatesFound: 4,
|
||||
issueSummaryOutcome: 'success',
|
||||
issueLinkageOutcome: 'success',
|
||||
prSummaryOutcome: 'success',
|
||||
prReviewOutcome: 'skipped',
|
||||
timedOutStage: 'none',
|
||||
},
|
||||
}],
|
||||
});
|
||||
@@ -327,37 +393,69 @@ suite('NewSessionViewV3Prompt', () => {
|
||||
name: 'onboarding.promptOptionInteraction',
|
||||
data: {
|
||||
scenarioId: NEW_SESSION_VIEW_V3_TOUR_ID,
|
||||
impressionId: '<impressionId>',
|
||||
interaction: 'selected',
|
||||
option: 'githubIssue',
|
||||
optionIndex: 0,
|
||||
optionKindsShown: 'githubIssue,githubPRCI,githubPRComments',
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'onboarding.promptOptionInteraction',
|
||||
data: {
|
||||
scenarioId: NEW_SESSION_VIEW_V3_TOUR_ID,
|
||||
impressionId: '<impressionId>',
|
||||
interaction: 'selected',
|
||||
option: 'githubPRCI',
|
||||
optionIndex: 1,
|
||||
optionKindsShown: 'githubIssue,githubPRCI,githubPRComments',
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'onboarding.promptOptionInteraction',
|
||||
data: {
|
||||
scenarioId: NEW_SESSION_VIEW_V3_TOUR_ID,
|
||||
impressionId: '<impressionId>',
|
||||
interaction: 'selected',
|
||||
option: 'githubPRComments',
|
||||
optionIndex: 2,
|
||||
optionKindsShown: 'githubIssue,githubPRCI,githubPRComments',
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'onboarding.promptOptionInteraction',
|
||||
data: {
|
||||
scenarioId: NEW_SESSION_VIEW_V3_TOUR_ID,
|
||||
impressionId: '<impressionId>',
|
||||
interaction: 'closed',
|
||||
option: 'none',
|
||||
optionIndex: -1,
|
||||
optionKindsShown: 'githubIssue,githubPRCI,githubPRComments',
|
||||
},
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
test('joins the strategy and interaction events of one impression', async () => {
|
||||
const result = await runPrompt(
|
||||
{ 'onb.newSessionViewV3.variation': 'options' },
|
||||
{},
|
||||
{ pullRequests: [], issues: [issue('Ready issue', '2026-08-07T13:00:00Z', 7)] },
|
||||
{ promptOptionInteractions: [0, 'close'] },
|
||||
);
|
||||
const impressionIds = getImpressionIds(result.rawTelemetry);
|
||||
|
||||
assert.deepStrictEqual({
|
||||
eventNames: result.rawTelemetry.map(event => event.name),
|
||||
sharedImpressionIds: new Set(impressionIds).size,
|
||||
isGuid: /^[0-9a-f-]{36}$/.test(impressionIds[0]),
|
||||
}, {
|
||||
eventNames: ['onboarding.promptStrategy', 'onboarding.promptOptionInteraction', 'onboarding.promptOptionInteraction'],
|
||||
sharedImpressionIds: 1,
|
||||
isGuid: true,
|
||||
});
|
||||
});
|
||||
|
||||
test('fills missing prompt options from the fixed standard order after a partial timeout', async () => {
|
||||
const result = await runPrompt(
|
||||
{ 'onb.newSessionViewV3.variation': 'options' },
|
||||
@@ -372,12 +470,21 @@ suite('NewSessionViewV3Prompt', () => {
|
||||
}, {
|
||||
states: [
|
||||
{ kind: 'loading' },
|
||||
{ kind: 'resolved', options: STANDARD_OPTIONS },
|
||||
{
|
||||
kind: 'resolved',
|
||||
options: [
|
||||
{ title: 'Tackle issue #7', description: 'Ready issue', icon: { id: 'issue-opened', color: 'charts.green' } },
|
||||
{ title: 'Implement a feature', description: 'Describe what you want to build', icon: { id: 'lightbulb-sparkle-autofix', color: undefined } },
|
||||
{ title: 'Fix a bug', description: 'Describe the unexpected behavior', icon: { id: 'bug', color: undefined } },
|
||||
STANDARD_OPTIONS[0],
|
||||
STANDARD_OPTIONS[1],
|
||||
],
|
||||
},
|
||||
{
|
||||
kind: 'resolved',
|
||||
options: [
|
||||
{ title: 'Tackle issue #7', description: 'Ready issue', icon: { id: 'issue-opened', color: 'charts.green' } },
|
||||
STANDARD_OPTIONS[0],
|
||||
STANDARD_OPTIONS[1],
|
||||
],
|
||||
},
|
||||
],
|
||||
@@ -385,15 +492,193 @@ suite('NewSessionViewV3Prompt', () => {
|
||||
name: 'onboarding.promptStrategy',
|
||||
data: {
|
||||
scenarioId: NEW_SESSION_VIEW_V3_TOUR_ID,
|
||||
impressionId: '<impressionId>',
|
||||
configuredVariation: 'options',
|
||||
effectiveStrategy: 'options',
|
||||
fallbackReason: 'timeout',
|
||||
shown: true,
|
||||
...LOOKUP_DURATIONS,
|
||||
gitHubOptionCount: 1,
|
||||
candidatesFound: 1,
|
||||
issueSummaryOutcome: 'success',
|
||||
issueLinkageOutcome: 'success',
|
||||
prSummaryOutcome: 'timeout',
|
||||
prReviewOutcome: 'notRun',
|
||||
timedOutStage: 'none',
|
||||
},
|
||||
}],
|
||||
});
|
||||
});
|
||||
|
||||
test('renders standard prompt options before the GitHub lookup and streams partial results into their slots', async () => {
|
||||
const result = await runPrompt(
|
||||
{ 'onb.newSessionViewV3.variation': 'options' },
|
||||
{},
|
||||
{
|
||||
issues: [issue('Assigned issue', '2026-08-07T14:00:00Z', 14)],
|
||||
pullRequests: [
|
||||
pullRequest('CI is failing', '2026-08-07T13:00:00Z', 'FAILURE', undefined, undefined, 21),
|
||||
pullRequest('Awaiting review threads', '2026-08-07T12:00:00Z', undefined, '2026-08-07T09:00:00Z', '2026-08-07T10:00:00Z', 22),
|
||||
],
|
||||
},
|
||||
{ reviewLookupNeverResolves: true },
|
||||
);
|
||||
const issueOption = { title: 'Tackle issue #14', description: 'Assigned issue', icon: { id: 'issue-opened', color: 'charts.green' } };
|
||||
const ciOption = { title: 'Fix CI #21', description: 'CI is failing', icon: { id: 'git-pull-request-error', color: 'charts.orange' } };
|
||||
|
||||
assert.deepStrictEqual({
|
||||
states: summarizePromptOptionStates(result.promptOptionStates),
|
||||
telemetry: result.telemetry,
|
||||
}, {
|
||||
states: [
|
||||
{ kind: 'loading' },
|
||||
{ kind: 'resolved', options: STANDARD_OPTIONS },
|
||||
{ kind: 'resolved', options: [issueOption, STANDARD_OPTIONS[0], STANDARD_OPTIONS[1]] },
|
||||
{ kind: 'resolved', options: [issueOption, ciOption, STANDARD_OPTIONS[0]] },
|
||||
{ kind: 'resolved', options: [issueOption, ciOption, STANDARD_OPTIONS[0]] },
|
||||
],
|
||||
telemetry: [{
|
||||
name: 'onboarding.promptStrategy',
|
||||
data: {
|
||||
scenarioId: NEW_SESSION_VIEW_V3_TOUR_ID,
|
||||
impressionId: '<impressionId>',
|
||||
configuredVariation: 'options',
|
||||
effectiveStrategy: 'options',
|
||||
fallbackReason: 'timeout',
|
||||
shown: true,
|
||||
...LOOKUP_DURATIONS,
|
||||
gitHubOptionCount: 2,
|
||||
candidatesFound: 2,
|
||||
issueSummaryOutcome: 'success',
|
||||
issueLinkageOutcome: 'success',
|
||||
prSummaryOutcome: 'success',
|
||||
prReviewOutcome: 'timeout',
|
||||
timedOutStage: 'none',
|
||||
},
|
||||
}],
|
||||
});
|
||||
});
|
||||
|
||||
test('repaints standard prompt options when the composer resolves again after a workspace change', async () => {
|
||||
const result = await runPrompt(
|
||||
{ 'onb.newSessionViewV3.variation': 'options' },
|
||||
{},
|
||||
{ pullRequests: [], issues: [] },
|
||||
{ refreshPromptOptionsAgain: true },
|
||||
);
|
||||
|
||||
assert.deepStrictEqual(summarizePromptOptionStates(result.promptOptionStates), [
|
||||
{ kind: 'loading' },
|
||||
{ kind: 'resolved', options: STANDARD_OPTIONS },
|
||||
{ kind: 'resolved', options: STANDARD_OPTIONS },
|
||||
{ kind: 'loading' },
|
||||
{ kind: 'resolved', options: STANDARD_OPTIONS },
|
||||
{ kind: 'resolved', options: STANDARD_OPTIONS },
|
||||
]);
|
||||
});
|
||||
|
||||
test('reports the options the composer rendered when it refuses later updates', async () => {
|
||||
const result = await runPrompt(
|
||||
{ 'onb.newSessionViewV3.variation': 'options' },
|
||||
{},
|
||||
{ pullRequests: [], issues: [issue('Assigned issue', '2026-08-07T14:00:00Z', 14)] },
|
||||
{ suppressPromptOptionUpdatesAfter: 1, promptOptionInteractions: [0] },
|
||||
);
|
||||
|
||||
assert.deepStrictEqual({
|
||||
states: summarizePromptOptionStates(result.promptOptionStates),
|
||||
interaction: result.telemetry.filter(event => event.name === 'onboarding.promptOptionInteraction'),
|
||||
}, {
|
||||
states: [
|
||||
{ kind: 'loading' },
|
||||
{ kind: 'resolved', options: STANDARD_OPTIONS },
|
||||
],
|
||||
interaction: [{
|
||||
name: 'onboarding.promptOptionInteraction',
|
||||
data: {
|
||||
scenarioId: NEW_SESSION_VIEW_V3_TOUR_ID,
|
||||
impressionId: '<impressionId>',
|
||||
interaction: 'selected',
|
||||
option: 'implementFeature',
|
||||
optionIndex: 0,
|
||||
optionKindsShown: 'implementFeature,fixBug,fixCI',
|
||||
},
|
||||
}],
|
||||
});
|
||||
});
|
||||
|
||||
test('publishes assigned issues before the linkage lookup can hide them', async () => {
|
||||
const result = await runPrompt(
|
||||
{ 'onb.newSessionViewV3.variation': 'options' },
|
||||
{},
|
||||
{ pullRequests: [], issues: [issue('Unknown linkage', '2026-08-07T13:00:00Z', 9)] },
|
||||
{ issueLinkageLookupNeverResolves: true },
|
||||
);
|
||||
const issueOption = { title: 'Tackle issue #9', description: 'Unknown linkage', icon: { id: 'issue-opened', color: 'charts.green' } };
|
||||
|
||||
assert.deepStrictEqual({
|
||||
states: summarizePromptOptionStates(result.promptOptionStates),
|
||||
telemetry: result.telemetry,
|
||||
}, {
|
||||
states: [
|
||||
{ kind: 'loading' },
|
||||
{ kind: 'resolved', options: STANDARD_OPTIONS },
|
||||
{ kind: 'resolved', options: [issueOption, STANDARD_OPTIONS[0], STANDARD_OPTIONS[1]] },
|
||||
{ kind: 'resolved', options: [issueOption, STANDARD_OPTIONS[0], STANDARD_OPTIONS[1]] },
|
||||
],
|
||||
telemetry: [{
|
||||
name: 'onboarding.promptStrategy',
|
||||
data: {
|
||||
scenarioId: NEW_SESSION_VIEW_V3_TOUR_ID,
|
||||
impressionId: '<impressionId>',
|
||||
configuredVariation: 'options',
|
||||
effectiveStrategy: 'options',
|
||||
fallbackReason: 'noCandidate',
|
||||
shown: true,
|
||||
...LOOKUP_DURATIONS,
|
||||
gitHubOptionCount: 1,
|
||||
candidatesFound: 1,
|
||||
issueSummaryOutcome: 'success',
|
||||
issueLinkageOutcome: 'timeout',
|
||||
prSummaryOutcome: 'success',
|
||||
prReviewOutcome: 'skipped',
|
||||
timedOutStage: 'none',
|
||||
},
|
||||
}],
|
||||
});
|
||||
});
|
||||
|
||||
test('reports the stage that consumed the total lookup budget', async () => {
|
||||
const result = await runPrompt(
|
||||
{ 'onb.newSessionViewV3.variation': 'options' },
|
||||
{},
|
||||
{ pullRequests: [], issues: [issue('Ready issue', '2026-08-07T13:00:00Z', 7)] },
|
||||
{ pullRequestLookupNeverResolves: true, totalMs: 10, summaryMs: 30 },
|
||||
);
|
||||
// Let the pull request lookup that outlived the total budget unwind and dispose its sources.
|
||||
await timeout(80);
|
||||
|
||||
assert.deepStrictEqual(result.telemetry, [{
|
||||
name: 'onboarding.promptStrategy',
|
||||
data: {
|
||||
scenarioId: NEW_SESSION_VIEW_V3_TOUR_ID,
|
||||
impressionId: '<impressionId>',
|
||||
configuredVariation: 'options',
|
||||
effectiveStrategy: 'options',
|
||||
fallbackReason: 'timeout',
|
||||
shown: true,
|
||||
...LOOKUP_DURATIONS,
|
||||
gitHubOptionCount: 1,
|
||||
candidatesFound: 1,
|
||||
issueSummaryOutcome: 'success',
|
||||
issueLinkageOutcome: 'success',
|
||||
prSummaryOutcome: 'notRun',
|
||||
prReviewOutcome: 'notRun',
|
||||
timedOutStage: 'prSummary',
|
||||
},
|
||||
}]);
|
||||
});
|
||||
|
||||
test('uses all standard prompt options when GitHub authentication is unavailable', async () => {
|
||||
const result = await runPrompt(
|
||||
{ 'onb.newSessionViewV3.variation': 'options' },
|
||||
@@ -407,23 +692,26 @@ suite('NewSessionViewV3Prompt', () => {
|
||||
}, {
|
||||
states: [
|
||||
{ kind: 'loading' },
|
||||
{
|
||||
kind: 'resolved',
|
||||
options: [
|
||||
{ title: 'Implement a feature', description: 'Describe what you want to build', icon: { id: 'lightbulb-sparkle-autofix', color: undefined } },
|
||||
{ title: 'Fix a bug', description: 'Describe the unexpected behavior', icon: { id: 'bug', color: undefined } },
|
||||
{ title: 'Fix CI', description: 'Describe a failing check or paste a link', icon: { id: 'run-errors', color: undefined } },
|
||||
],
|
||||
},
|
||||
{ kind: 'resolved', options: STANDARD_OPTIONS },
|
||||
{ kind: 'resolved', options: STANDARD_OPTIONS },
|
||||
],
|
||||
telemetry: [{
|
||||
name: 'onboarding.promptStrategy',
|
||||
data: {
|
||||
scenarioId: NEW_SESSION_VIEW_V3_TOUR_ID,
|
||||
impressionId: '<impressionId>',
|
||||
configuredVariation: 'options',
|
||||
effectiveStrategy: 'options',
|
||||
fallbackReason: 'noAuthentication',
|
||||
shown: true,
|
||||
...LOOKUP_DURATIONS,
|
||||
gitHubOptionCount: 0,
|
||||
candidatesFound: 0,
|
||||
issueSummaryOutcome: 'noAuthentication',
|
||||
issueLinkageOutcome: 'skipped',
|
||||
prSummaryOutcome: 'noAuthentication',
|
||||
prReviewOutcome: 'notRun',
|
||||
timedOutStage: 'none',
|
||||
},
|
||||
}],
|
||||
});
|
||||
@@ -450,10 +738,12 @@ suite('NewSessionViewV3Prompt', () => {
|
||||
name: 'onboarding.promptStrategy',
|
||||
data: {
|
||||
scenarioId: NEW_SESSION_VIEW_V3_TOUR_ID,
|
||||
impressionId: '<impressionId>',
|
||||
configuredVariation: 'githubPrompt',
|
||||
effectiveStrategy: 'githubIssue',
|
||||
fallbackReason: 'none',
|
||||
shown: true,
|
||||
...NO_LOOKUP_TELEMETRY,
|
||||
},
|
||||
}],
|
||||
});
|
||||
@@ -682,17 +972,27 @@ async function runPrompt(
|
||||
readonly enterpriseHost?: string;
|
||||
readonly pullRequestLookupNeverResolves?: boolean;
|
||||
readonly issueLinkageLookupNeverResolves?: boolean;
|
||||
readonly reviewLookupNeverResolves?: boolean;
|
||||
readonly totalMs?: number;
|
||||
readonly summaryMs?: number;
|
||||
readonly suppressPromptOptionUpdatesAfter?: number;
|
||||
readonly refreshPromptOptionsAgain?: boolean;
|
||||
readonly promptOptionInteractions?: readonly (number | 'close')[];
|
||||
} = {},
|
||||
): Promise<{
|
||||
readonly animation: { readonly prompt: string; readonly durationMs: number; readonly placeholder: string } | undefined;
|
||||
readonly promptOptionStates: readonly NewSessionPromptOptionsState[];
|
||||
readonly telemetry: readonly { readonly name: string; readonly data: object | undefined }[];
|
||||
readonly telemetry: readonly TestTelemetryEvent[];
|
||||
readonly rawTelemetry: readonly TestTelemetryEvent[];
|
||||
readonly gitHubRequests: readonly TestGitHubRequest[];
|
||||
}> {
|
||||
let animation: { prompt: string; durationMs: number; placeholder: string } | undefined;
|
||||
const promptOptionStates: NewSessionPromptOptionsState[] = [];
|
||||
let promptOptionsController: INewSessionPromptOptionsController | undefined;
|
||||
let acceptedUpdates = 0;
|
||||
// Mimics the composer refusing late updates once the user acts on the options on screen.
|
||||
const suppressUpdate = () => options.suppressPromptOptionUpdatesAfter !== undefined
|
||||
&& acceptedUpdates >= options.suppressPromptOptionUpdatesAfter;
|
||||
const workspaceUri = options.workspaceUri ?? URI.file('C:\\repo');
|
||||
const workspace = createWorkspace(workspaceUri, 'r', options.includeGitHubInfo !== false);
|
||||
const activeSession = createSession(workspace);
|
||||
@@ -718,8 +1018,17 @@ async function runPrompt(
|
||||
return false;
|
||||
}
|
||||
promptOptionStates.push({ kind: 'loading' });
|
||||
const state = await controller.resolve(token);
|
||||
promptOptionStates.push(state);
|
||||
const state = await controller.resolve(token, progressState => {
|
||||
if (suppressUpdate()) {
|
||||
return false;
|
||||
}
|
||||
acceptedUpdates++;
|
||||
promptOptionStates.push(progressState);
|
||||
return true;
|
||||
});
|
||||
if (!suppressUpdate()) {
|
||||
promptOptionStates.push(state);
|
||||
}
|
||||
return true;
|
||||
},
|
||||
});
|
||||
@@ -746,6 +1055,9 @@ async function runPrompt(
|
||||
}
|
||||
override async getPullRequestReviewThreads(owner: string, repo: string, pullRequestNumber: number) {
|
||||
this.requests.push({ kind: 'reviews', owner, repo, pullRequestNumber });
|
||||
if (options.reviewLookupNeverResolves) {
|
||||
return new Promise<never>(() => { });
|
||||
}
|
||||
if (gitHubResult instanceof Error) {
|
||||
throw gitHubResult;
|
||||
}
|
||||
@@ -787,10 +1099,14 @@ async function runPrompt(
|
||||
gitHubService,
|
||||
telemetryService,
|
||||
new NullLogService(),
|
||||
{ totalMs: 100, summaryMs: 20, linkageMs: 20, reviewMs: 20 },
|
||||
{ totalMs: options.totalMs ?? 100, summaryMs: options.summaryMs ?? 20, linkageMs: 20, reviewMs: 20 },
|
||||
);
|
||||
|
||||
await runner.run(CancellationToken.None);
|
||||
if (options.refreshPromptOptionsAgain) {
|
||||
// The composer resolves again on refresh, for example when the selected workspace changes.
|
||||
await composerService.activeComposer.get().refreshPromptOptions(CancellationToken.None);
|
||||
}
|
||||
if (options.promptOptionInteractions?.length) {
|
||||
const controller = promptOptionsController;
|
||||
const resolvedState = [...promptOptionStates].reverse().find(state => state.kind === 'resolved');
|
||||
@@ -809,7 +1125,7 @@ async function runPrompt(
|
||||
controller.onDidSelectOption(option);
|
||||
}
|
||||
}
|
||||
return { animation, promptOptionStates, telemetry: telemetryService.events, gitHubRequests: gitHubService.requests };
|
||||
return { animation, promptOptionStates, telemetry: summarizeTelemetry(telemetryService.events), rawTelemetry: telemetryService.events, gitHubRequests: gitHubService.requests };
|
||||
}
|
||||
|
||||
function pullRequest(title: string, updatedAt: string, statusCheckRollupState?: string, latestCommitAt?: string, latestCommentAt?: string, number = 1, hasMergeConflicts = false) {
|
||||
@@ -877,3 +1193,23 @@ function summarizePromptOptionStates(states: readonly NewSessionPromptOptionsSta
|
||||
})),
|
||||
});
|
||||
}
|
||||
|
||||
/** Replaces the values that legitimately vary between runs so telemetry can be asserted as a snapshot. */
|
||||
function summarizeTelemetry(events: readonly TestTelemetryEvent[]): TestTelemetryEvent[] {
|
||||
return events.map(event => ({
|
||||
name: event.name,
|
||||
data: event.data && Object.fromEntries(Object.entries(event.data).map(([key, value]) => {
|
||||
if (key === 'impressionId') {
|
||||
return [key, typeof value === 'string' && value.length > 0 ? '<impressionId>' : value];
|
||||
}
|
||||
if (key.endsWith('Ms') && typeof value === 'number') {
|
||||
return [key, '<duration>'];
|
||||
}
|
||||
return [key, value];
|
||||
})),
|
||||
}));
|
||||
}
|
||||
|
||||
function getImpressionIds(events: readonly TestTelemetryEvent[]): string[] {
|
||||
return events.map(event => String(event.data?.impressionId ?? ''));
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user