Microsoft/vscode
1
0
Fork 0
mirror of https://github.com/microsoft/vscode.git synced 2026-08-15 02:07:31 +01:00
Code Issues Packages Projects Releases Wiki Activity

Set default NES aggressiveness to medium (#327049)

Browse Source
This commit is contained in:
Benjamin Steenhoek
2026-07-30 15:57:55 +00:00
committed by GitHub
parent abf2ee04c6
commit d9d6728092
3 changed files with 96 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
extensions/copilot/src/extension/inlineEdits/test/common/userInteractionMonitor.spec.ts
+36 -6
View File
@@ -4,10 +4,10 @@
*--------------------------------------------------------------------------------------------*/
import { beforeEach, describe, expect, test } from 'vitest';
import { ConfigKey } from '../../../../platform/configuration/common/configurationService';
import { ConfigKey, ExperimentBasedConfig, ExperimentBasedConfigType } from '../../../../platform/configuration/common/configurationService';
import { DefaultsOnlyConfigurationService } from '../../../../platform/configuration/common/defaultsOnlyConfigurationService';
import { InMemoryConfigurationService } from '../../../../platform/configuration/test/common/inMemoryConfigurationService';
import { AggressivenessLevel, DEFAULT_USER_HAPPINESS_SCORE_CONFIGURATION, UserHappinessScoreConfiguration } from '../../../../platform/inlineEdits/common/dataTypes/xtabPromptOptions';
import { AggressivenessLevel, AggressivenessSetting, DEFAULT_USER_HAPPINESS_SCORE_CONFIGURATION, UserHappinessScoreConfiguration } from '../../../../platform/inlineEdits/common/dataTypes/xtabPromptOptions';
import { ILogService } from '../../../../platform/log/common/logService';
import { IExperimentationService, NullExperimentationService } from '../../../../platform/telemetry/common/nullExperimentationService';
import { NullTelemetryService } from '../../../../platform/telemetry/common/nullTelemetryService';
@@ -47,9 +47,22 @@ class TestUserInteractionMonitor extends UserInteractionMonitor {
* Mock configuration service that allows setting specific config values for testing.
*/
class MockConfigurationService extends InMemoryConfigurationService {
private _useAdaptiveAggressiveness = false;
constructor() {
super(new DefaultsOnlyConfigurationService());
}
useAdaptiveAggressiveness(): void {
this._useAdaptiveAggressiveness = true;
}
override getExperimentBasedConfig<T extends ExperimentBasedConfigType>(key: ExperimentBasedConfig<T>, experimentationService: IExperimentationService): T {
if (this._useAdaptiveAggressiveness && key === ConfigKey.TeamInternal.InlineEditsXtabAggressivenessLevel) {
return undefined as T;
}
return super.getExperimentBasedConfig(key, experimentationService);
}
}
interface TelemetryCall {
@@ -197,13 +210,25 @@ describe('UserInteractionMonitor', () => {
});
describe('aggressiveness level calculation', () => {
test('returns neutral aggressiveness with no history', () => {
// With no data, score is 0.5, which is between low and medium thresholds for the default config
const level = monitor.getAggressivenessLevel().aggressivenessLevel;
expect(level).toBe(AggressivenessLevel.Medium);
test('defaults to medium aggressiveness without using adaptive scoring', () => {
expect(monitor.getAggressivenessLevel()).toEqual({
aggressivenessLevel: AggressivenessLevel.Medium,
userHappinessScore: undefined,
});
});
test('explicit user eagerness takes priority over configured aggressiveness', () => {
configurationService.setConfig(ConfigKey.Advanced.InlineEditsAggressiveness, AggressivenessSetting.High);
configurationService.setConfig(ConfigKey.TeamInternal.InlineEditsXtabAggressivenessLevel, AggressivenessLevel.Low);
expect(monitor.getAggressivenessLevel()).toEqual({
aggressivenessLevel: AggressivenessLevel.High,
userHappinessScore: undefined,
});
});
test('returns high aggressiveness after many acceptances', () => {
configurationService.useAdaptiveAggressiveness();
// Fill with 10 acceptances
for (let i = 0; i < 10; i++) {
monitor.handleAcceptance();
@@ -214,6 +239,7 @@ describe('UserInteractionMonitor', () => {
});
test('returns low aggressiveness after many rejections', () => {
configurationService.useAdaptiveAggressiveness();
// Fill with 10 rejections
for (let i = 0; i < 10; i++) {
monitor.handleRejection();
@@ -239,6 +265,7 @@ describe('UserInteractionMonitor', () => {
});
test('recent actions have more weight than older ones', () => {
configurationService.useAdaptiveAggressiveness();
// Start with acceptances, end with rejections
for (let i = 0; i < 5; i++) {
monitor.handleAcceptance();
@@ -270,6 +297,7 @@ describe('UserInteractionMonitor', () => {
describe('ignored action limiting', () => {
test('ignored actions are included in aggressiveness calculation', () => {
configurationService.useAdaptiveAggressiveness();
// With custom config that includes ignored actions
const customConfig: UserHappinessScoreConfiguration = {
...DEFAULT_USER_HAPPINESS_SCORE_CONFIGURATION,
@@ -294,6 +322,7 @@ describe('UserInteractionMonitor', () => {
});
test('total ignored limit is respected', () => {
configurationService.useAdaptiveAggressiveness();
const customConfig: UserHappinessScoreConfiguration = {
...DEFAULT_USER_HAPPINESS_SCORE_CONFIGURATION,
includeIgnored: true,
@@ -325,6 +354,7 @@ describe('UserInteractionMonitor', () => {
let mockTelemetryService: MockTelemetryService;
beforeEach(() => {
configurationService.useAdaptiveAggressiveness();
mockTelemetryService = new MockTelemetryService();
monitor = new TestUserInteractionMonitor(configurationService, experimentationService, logService, mockTelemetryService);
});
extensions/copilot/src/extension/xtab/test/node/xtabProvider.spec.ts
+59 -1
View File
@@ -14,7 +14,7 @@ import { DocumentId } from '../../../../platform/inlineEdits/common/dataTypes/do
import { Edits } from '../../../../platform/inlineEdits/common/dataTypes/edit';
import { ImportChanges } from '../../../../platform/inlineEdits/common/dataTypes/importFilteringOptions';
import { LanguageId } from '../../../../platform/inlineEdits/common/dataTypes/languageId';
import { DEFAULT_OPTIONS, EarlyDivergenceCancellationMode, LanguageContextLanguages, LintOptionShowCode, LintOptionWarning, ModelConfiguration, PatchModelPrediction, PromptingStrategy, ResponseFormat } from '../../../../platform/inlineEdits/common/dataTypes/xtabPromptOptions';
import { AggressivenessLevel, DEFAULT_OPTIONS, EarlyDivergenceCancellationMode, LanguageContextLanguages, LintOptionShowCode, LintOptionWarning, ModelConfiguration, PatchModelPrediction, PromptingStrategy, ResponseFormat } from '../../../../platform/inlineEdits/common/dataTypes/xtabPromptOptions';
import { InlineEditRequestLogContext } from '../../../../platform/inlineEdits/common/inlineEditLogContext';
import { IInlineEditsModelService } from '../../../../platform/inlineEdits/common/inlineEditsModelService';
import { NoNextEditReason, StatelessNextEditDocument, StatelessNextEditRequest, StreamedEdit, WithStatelessProviderTelemetry } from '../../../../platform/inlineEdits/common/statelessNextEditProvider';
@@ -1047,6 +1047,40 @@ describe('XtabProvider integration', () => {
expect(getMessageText(systemMessage!)).toBe(xtab275SystemPrompt);
});
it('applies configured aggressiveness only to aggressiveness strategies', async () => {
const lines = ['const x = 1;', 'const y = 2;'];
const captureUserPrompt = async (promptingStrategy: PromptingStrategy, aggressivenessLevel: AggressivenessLevel) => {
mockModelService.setSelectedConfig({ promptingStrategy });
await configService.setConfig(ConfigKey.TeamInternal.InlineEditsXtabAggressivenessLevel, aggressivenessLevel);
streamingFetcher.setStreamingLines(lines);
const gen = createProvider().provideNextEdit(createRequestWithEdit(lines, { insertionOffset: 3, insertedText: 'a' }), createMockLogger(), createLogContext(), CancellationToken.None);
await AsyncIterUtils.drainUntilReturn(gen);
const messages = streamingFetcher.capturedOptions.at(-1)?.messages;
const userMessage = messages?.find(message => message.role === Raw.ChatRole.User);
expect(userMessage).toBeDefined();
return getMessageText(userMessage!);
};
const nonAggressiveLow = await captureUserPrompt(PromptingStrategy.Xtab275, AggressivenessLevel.Low);
const nonAggressiveHigh = await captureUserPrompt(PromptingStrategy.Xtab275, AggressivenessLevel.High);
const aggressiveLow = await captureUserPrompt(PromptingStrategy.XtabAggressiveness, AggressivenessLevel.Low);
const aggressiveHigh = await captureUserPrompt(PromptingStrategy.XtabAggressiveness, AggressivenessLevel.High);
expect({
nonAggressivePromptsMatch: nonAggressiveLow === nonAggressiveHigh,
nonAggressivePromptHasLevel: nonAggressiveLow.includes('<|aggressive|>'),
aggressiveLowHasLevel: aggressiveLow.includes('<|aggressive|>low<|/aggressive|>'),
aggressiveHighHasLevel: aggressiveHigh.includes('<|aggressive|>high<|/aggressive|>'),
}).toEqual({
nonAggressivePromptsMatch: true,
nonAggressivePromptHasLevel: false,
aggressiveLowHasLevel: true,
aggressiveHighHasLevel: true,
});
});
it('retries with default model after NotFound response', async () => {
const provider = createProvider();
@@ -1977,6 +2011,30 @@ describe('XtabProvider integration', () => {
// ========================================================================
describe('debounce behavior', () => {
it('does not change timing for a non-aggressiveness strategy when user eagerness is default', async () => {
mockModelService.setSelectedConfig({ promptingStrategy: PromptingStrategy.Xtab275 });
const setBaseDebounceTime = vi.spyOn(DelaySession.prototype, 'setBaseDebounceTime');
const setExpectedTotalTime = vi.spyOn(DelaySession.prototype, 'setExpectedTotalTime');
try {
const lines = ['const x = 1;', 'const y = 2;'];
streamingFetcher.setStreamingLines(lines);
const gen = createProvider().provideNextEdit(createRequestWithEdit(lines, { insertionOffset: 3, insertedText: 'a' }), createMockLogger(), createLogContext(), CancellationToken.None);
await AsyncIterUtils.drainUntilReturn(gen);
expect({
setBaseDebounceTimeCalls: setBaseDebounceTime.mock.calls.length,
setExpectedTotalTimeCalls: setExpectedTotalTime.mock.calls.length,
}).toEqual({
setBaseDebounceTimeCalls: 0,
setExpectedTotalTimeCalls: 0,
});
} finally {
setBaseDebounceTime.mockRestore();
setExpectedTotalTime.mockRestore();
}
});
it('debounce is skipped in simulation tests', async () => {
// Override the simulation test context to indicate we're in sim tests
const testingServiceCollection = createExtensionUnitTestingServices(disposables);
extensions/copilot/src/platform/configuration/common/configurationService.ts
+1 -1
View File
@@ -907,7 +907,7 @@ export namespace ConfigKey {
export const InlineEditsXtabOnlyMergeConflictLines = defineTeamInternalSetting<boolean>('chat.advanced.inlineEdits.xtabProvider.onlyMergeConflictLines', ConfigType.ExperimentBased, false);
export const InlineEditsXtabDuplicateAdditionsMode = defineTeamInternalSetting<DuplicateAdditionsMode>('chat.advanced.inlineEdits.xtabProvider.diffPatch.duplicateAdditionsMode', ConfigType.ExperimentBased, DuplicateAdditionsMode.Off, DuplicateAdditionsMode.VALIDATOR);
export const InlineEditsXtabSplitPatchOnDiff = defineTeamInternalSetting<boolean>('chat.advanced.inlineEdits.xtabProvider.diffPatch.splitOnDiff', ConfigType.ExperimentBased, false, vBoolean());
export const InlineEditsXtabAggressivenessLevel = defineTeamInternalSetting<xtabPromptOptions.AggressivenessLevel | undefined>('chat.advanced.inlineEdits.xtabProvider.aggressivenessLevel', ConfigType.ExperimentBased, undefined);
export const InlineEditsXtabAggressivenessLevel = defineTeamInternalSetting<xtabPromptOptions.AggressivenessLevel | undefined>('chat.advanced.inlineEdits.xtabProvider.aggressivenessLevel', ConfigType.ExperimentBased, xtabPromptOptions.AggressivenessLevel.Medium);
export const InlineEditsAggressivenessLowMinResponseTimeMs = defineTeamInternalSetting<number>('chat.advanced.inlineEdits.aggressiveness.lowMinResponseTimeMs', ConfigType.ExperimentBased, 1500);
export const InlineEditsAggressivenessMediumMinResponseTimeMs = defineTeamInternalSetting<number>('chat.advanced.inlineEdits.aggressiveness.mediumMinResponseTimeMs', ConfigType.ExperimentBased, 700);
export const InlineEditsAggressivenessHighDebounceMs = defineTeamInternalSetting<number>('chat.advanced.inlineEdits.aggressiveness.highDebounceMs', ConfigType.ExperimentBased, 0);