From d9d6728092f8be8bba69bbbb5cc97beb953ae465 Mon Sep 17 00:00:00 2001 From: Benjamin Steenhoek Date: Thu, 30 Jul 2026 10:57:55 -0500 Subject: [PATCH] Set default NES aggressiveness to medium (#327049) --- .../common/userInteractionMonitor.spec.ts | 42 +++++++++++-- .../xtab/test/node/xtabProvider.spec.ts | 60 ++++++++++++++++++- .../common/configurationService.ts | 2 +- 3 files changed, 96 insertions(+), 8 deletions(-) diff --git a/extensions/copilot/src/extension/inlineEdits/test/common/userInteractionMonitor.spec.ts b/extensions/copilot/src/extension/inlineEdits/test/common/userInteractionMonitor.spec.ts index 60f6990aace..54547474407 100644 --- a/extensions/copilot/src/extension/inlineEdits/test/common/userInteractionMonitor.spec.ts +++ b/extensions/copilot/src/extension/inlineEdits/test/common/userInteractionMonitor.spec.ts @@ -4,10 +4,10 @@ *--------------------------------------------------------------------------------------------*/ import { beforeEach, describe, expect, test } from 'vitest'; -import { ConfigKey } from '../../../../platform/configuration/common/configurationService'; +import { ConfigKey, ExperimentBasedConfig, ExperimentBasedConfigType } from '../../../../platform/configuration/common/configurationService'; import { DefaultsOnlyConfigurationService } from '../../../../platform/configuration/common/defaultsOnlyConfigurationService'; import { InMemoryConfigurationService } from '../../../../platform/configuration/test/common/inMemoryConfigurationService'; -import { AggressivenessLevel, DEFAULT_USER_HAPPINESS_SCORE_CONFIGURATION, UserHappinessScoreConfiguration } from '../../../../platform/inlineEdits/common/dataTypes/xtabPromptOptions'; +import { AggressivenessLevel, AggressivenessSetting, DEFAULT_USER_HAPPINESS_SCORE_CONFIGURATION, UserHappinessScoreConfiguration } from '../../../../platform/inlineEdits/common/dataTypes/xtabPromptOptions'; import { ILogService } from '../../../../platform/log/common/logService'; import { IExperimentationService, NullExperimentationService } from '../../../../platform/telemetry/common/nullExperimentationService'; import { NullTelemetryService } from '../../../../platform/telemetry/common/nullTelemetryService'; @@ -47,9 +47,22 @@ class TestUserInteractionMonitor extends UserInteractionMonitor { * Mock configuration service that allows setting specific config values for testing. */ class MockConfigurationService extends InMemoryConfigurationService { + private _useAdaptiveAggressiveness = false; + constructor() { super(new DefaultsOnlyConfigurationService()); } + + useAdaptiveAggressiveness(): void { + this._useAdaptiveAggressiveness = true; + } + + override getExperimentBasedConfig(key: ExperimentBasedConfig, experimentationService: IExperimentationService): T { + if (this._useAdaptiveAggressiveness && key === ConfigKey.TeamInternal.InlineEditsXtabAggressivenessLevel) { + return undefined as T; + } + return super.getExperimentBasedConfig(key, experimentationService); + } } interface TelemetryCall { @@ -197,13 +210,25 @@ describe('UserInteractionMonitor', () => { }); describe('aggressiveness level calculation', () => { - test('returns neutral aggressiveness with no history', () => { - // With no data, score is 0.5, which is between low and medium thresholds for the default config - const level = monitor.getAggressivenessLevel().aggressivenessLevel; - expect(level).toBe(AggressivenessLevel.Medium); + test('defaults to medium aggressiveness without using adaptive scoring', () => { + expect(monitor.getAggressivenessLevel()).toEqual({ + aggressivenessLevel: AggressivenessLevel.Medium, + userHappinessScore: undefined, + }); + }); + + test('explicit user eagerness takes priority over configured aggressiveness', () => { + configurationService.setConfig(ConfigKey.Advanced.InlineEditsAggressiveness, AggressivenessSetting.High); + configurationService.setConfig(ConfigKey.TeamInternal.InlineEditsXtabAggressivenessLevel, AggressivenessLevel.Low); + + expect(monitor.getAggressivenessLevel()).toEqual({ + aggressivenessLevel: AggressivenessLevel.High, + userHappinessScore: undefined, + }); }); test('returns high aggressiveness after many acceptances', () => { + configurationService.useAdaptiveAggressiveness(); // Fill with 10 acceptances for (let i = 0; i < 10; i++) { monitor.handleAcceptance(); @@ -214,6 +239,7 @@ describe('UserInteractionMonitor', () => { }); test('returns low aggressiveness after many rejections', () => { + configurationService.useAdaptiveAggressiveness(); // Fill with 10 rejections for (let i = 0; i < 10; i++) { monitor.handleRejection(); @@ -239,6 +265,7 @@ describe('UserInteractionMonitor', () => { }); test('recent actions have more weight than older ones', () => { + configurationService.useAdaptiveAggressiveness(); // Start with acceptances, end with rejections for (let i = 0; i < 5; i++) { monitor.handleAcceptance(); @@ -270,6 +297,7 @@ describe('UserInteractionMonitor', () => { describe('ignored action limiting', () => { test('ignored actions are included in aggressiveness calculation', () => { + configurationService.useAdaptiveAggressiveness(); // With custom config that includes ignored actions const customConfig: UserHappinessScoreConfiguration = { ...DEFAULT_USER_HAPPINESS_SCORE_CONFIGURATION, @@ -294,6 +322,7 @@ describe('UserInteractionMonitor', () => { }); test('total ignored limit is respected', () => { + configurationService.useAdaptiveAggressiveness(); const customConfig: UserHappinessScoreConfiguration = { ...DEFAULT_USER_HAPPINESS_SCORE_CONFIGURATION, includeIgnored: true, @@ -325,6 +354,7 @@ describe('UserInteractionMonitor', () => { let mockTelemetryService: MockTelemetryService; beforeEach(() => { + configurationService.useAdaptiveAggressiveness(); mockTelemetryService = new MockTelemetryService(); monitor = new TestUserInteractionMonitor(configurationService, experimentationService, logService, mockTelemetryService); }); diff --git a/extensions/copilot/src/extension/xtab/test/node/xtabProvider.spec.ts b/extensions/copilot/src/extension/xtab/test/node/xtabProvider.spec.ts index b835a18866d..525543dc72d 100644 --- a/extensions/copilot/src/extension/xtab/test/node/xtabProvider.spec.ts +++ b/extensions/copilot/src/extension/xtab/test/node/xtabProvider.spec.ts @@ -14,7 +14,7 @@ import { DocumentId } from '../../../../platform/inlineEdits/common/dataTypes/do import { Edits } from '../../../../platform/inlineEdits/common/dataTypes/edit'; import { ImportChanges } from '../../../../platform/inlineEdits/common/dataTypes/importFilteringOptions'; import { LanguageId } from '../../../../platform/inlineEdits/common/dataTypes/languageId'; -import { DEFAULT_OPTIONS, EarlyDivergenceCancellationMode, LanguageContextLanguages, LintOptionShowCode, LintOptionWarning, ModelConfiguration, PatchModelPrediction, PromptingStrategy, ResponseFormat } from '../../../../platform/inlineEdits/common/dataTypes/xtabPromptOptions'; +import { AggressivenessLevel, DEFAULT_OPTIONS, EarlyDivergenceCancellationMode, LanguageContextLanguages, LintOptionShowCode, LintOptionWarning, ModelConfiguration, PatchModelPrediction, PromptingStrategy, ResponseFormat } from '../../../../platform/inlineEdits/common/dataTypes/xtabPromptOptions'; import { InlineEditRequestLogContext } from '../../../../platform/inlineEdits/common/inlineEditLogContext'; import { IInlineEditsModelService } from '../../../../platform/inlineEdits/common/inlineEditsModelService'; import { NoNextEditReason, StatelessNextEditDocument, StatelessNextEditRequest, StreamedEdit, WithStatelessProviderTelemetry } from '../../../../platform/inlineEdits/common/statelessNextEditProvider'; @@ -1047,6 +1047,40 @@ describe('XtabProvider integration', () => { expect(getMessageText(systemMessage!)).toBe(xtab275SystemPrompt); }); + it('applies configured aggressiveness only to aggressiveness strategies', async () => { + const lines = ['const x = 1;', 'const y = 2;']; + const captureUserPrompt = async (promptingStrategy: PromptingStrategy, aggressivenessLevel: AggressivenessLevel) => { + mockModelService.setSelectedConfig({ promptingStrategy }); + await configService.setConfig(ConfigKey.TeamInternal.InlineEditsXtabAggressivenessLevel, aggressivenessLevel); + streamingFetcher.setStreamingLines(lines); + + const gen = createProvider().provideNextEdit(createRequestWithEdit(lines, { insertionOffset: 3, insertedText: 'a' }), createMockLogger(), createLogContext(), CancellationToken.None); + await AsyncIterUtils.drainUntilReturn(gen); + + const messages = streamingFetcher.capturedOptions.at(-1)?.messages; + const userMessage = messages?.find(message => message.role === Raw.ChatRole.User); + expect(userMessage).toBeDefined(); + return getMessageText(userMessage!); + }; + + const nonAggressiveLow = await captureUserPrompt(PromptingStrategy.Xtab275, AggressivenessLevel.Low); + const nonAggressiveHigh = await captureUserPrompt(PromptingStrategy.Xtab275, AggressivenessLevel.High); + const aggressiveLow = await captureUserPrompt(PromptingStrategy.XtabAggressiveness, AggressivenessLevel.Low); + const aggressiveHigh = await captureUserPrompt(PromptingStrategy.XtabAggressiveness, AggressivenessLevel.High); + + expect({ + nonAggressivePromptsMatch: nonAggressiveLow === nonAggressiveHigh, + nonAggressivePromptHasLevel: nonAggressiveLow.includes('<|aggressive|>'), + aggressiveLowHasLevel: aggressiveLow.includes('<|aggressive|>low<|/aggressive|>'), + aggressiveHighHasLevel: aggressiveHigh.includes('<|aggressive|>high<|/aggressive|>'), + }).toEqual({ + nonAggressivePromptsMatch: true, + nonAggressivePromptHasLevel: false, + aggressiveLowHasLevel: true, + aggressiveHighHasLevel: true, + }); + }); + it('retries with default model after NotFound response', async () => { const provider = createProvider(); @@ -1977,6 +2011,30 @@ describe('XtabProvider integration', () => { // ======================================================================== describe('debounce behavior', () => { + it('does not change timing for a non-aggressiveness strategy when user eagerness is default', async () => { + mockModelService.setSelectedConfig({ promptingStrategy: PromptingStrategy.Xtab275 }); + const setBaseDebounceTime = vi.spyOn(DelaySession.prototype, 'setBaseDebounceTime'); + const setExpectedTotalTime = vi.spyOn(DelaySession.prototype, 'setExpectedTotalTime'); + + try { + const lines = ['const x = 1;', 'const y = 2;']; + streamingFetcher.setStreamingLines(lines); + const gen = createProvider().provideNextEdit(createRequestWithEdit(lines, { insertionOffset: 3, insertedText: 'a' }), createMockLogger(), createLogContext(), CancellationToken.None); + await AsyncIterUtils.drainUntilReturn(gen); + + expect({ + setBaseDebounceTimeCalls: setBaseDebounceTime.mock.calls.length, + setExpectedTotalTimeCalls: setExpectedTotalTime.mock.calls.length, + }).toEqual({ + setBaseDebounceTimeCalls: 0, + setExpectedTotalTimeCalls: 0, + }); + } finally { + setBaseDebounceTime.mockRestore(); + setExpectedTotalTime.mockRestore(); + } + }); + it('debounce is skipped in simulation tests', async () => { // Override the simulation test context to indicate we're in sim tests const testingServiceCollection = createExtensionUnitTestingServices(disposables); diff --git a/extensions/copilot/src/platform/configuration/common/configurationService.ts b/extensions/copilot/src/platform/configuration/common/configurationService.ts index ada9b967f22..9b819f82618 100644 --- a/extensions/copilot/src/platform/configuration/common/configurationService.ts +++ b/extensions/copilot/src/platform/configuration/common/configurationService.ts @@ -907,7 +907,7 @@ export namespace ConfigKey { export const InlineEditsXtabOnlyMergeConflictLines = defineTeamInternalSetting('chat.advanced.inlineEdits.xtabProvider.onlyMergeConflictLines', ConfigType.ExperimentBased, false); export const InlineEditsXtabDuplicateAdditionsMode = defineTeamInternalSetting('chat.advanced.inlineEdits.xtabProvider.diffPatch.duplicateAdditionsMode', ConfigType.ExperimentBased, DuplicateAdditionsMode.Off, DuplicateAdditionsMode.VALIDATOR); export const InlineEditsXtabSplitPatchOnDiff = defineTeamInternalSetting('chat.advanced.inlineEdits.xtabProvider.diffPatch.splitOnDiff', ConfigType.ExperimentBased, false, vBoolean()); - export const InlineEditsXtabAggressivenessLevel = defineTeamInternalSetting('chat.advanced.inlineEdits.xtabProvider.aggressivenessLevel', ConfigType.ExperimentBased, undefined); + export const InlineEditsXtabAggressivenessLevel = defineTeamInternalSetting('chat.advanced.inlineEdits.xtabProvider.aggressivenessLevel', ConfigType.ExperimentBased, xtabPromptOptions.AggressivenessLevel.Medium); export const InlineEditsAggressivenessLowMinResponseTimeMs = defineTeamInternalSetting('chat.advanced.inlineEdits.aggressiveness.lowMinResponseTimeMs', ConfigType.ExperimentBased, 1500); export const InlineEditsAggressivenessMediumMinResponseTimeMs = defineTeamInternalSetting('chat.advanced.inlineEdits.aggressiveness.mediumMinResponseTimeMs', ConfigType.ExperimentBased, 700); export const InlineEditsAggressivenessHighDebounceMs = defineTeamInternalSetting('chat.advanced.inlineEdits.aggressiveness.highDebounceMs', ConfigType.ExperimentBased, 0);