remote: configurable 'reconnection grace time' (#274910)

* reconnection grace period prototype

* plumb through CLI

* polish

* Apply suggestions from code review

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
Josh Spicer
2025-11-12 18:51:28 -08:00
committed by GitHub
parent e3d1e4f115
commit 6cc2564bf9
14 changed files with 137 additions and 18 deletions

View File

@@ -686,6 +686,10 @@ pub struct BaseServerArgs {
/// Set the root path for extensions.
#[clap(long)]
pub extensions_dir: Option<String>,
/// Reconnection grace time in seconds. Defaults to 10800 (3 hours).
#[clap(long)]
pub reconnection_grace_time: Option<u32>,
}
impl BaseServerArgs {
@@ -700,6 +704,10 @@ impl BaseServerArgs {
if let Some(d) = &self.extensions_dir {
csa.extensions_dir = Some(d.clone());
}
if let Some(t) = self.reconnection_grace_time {
csa.reconnection_grace_time = Some(t);
}
}
}

View File

@@ -74,6 +74,8 @@ pub struct CodeServerArgs {
pub connection_token: Option<String>,
pub connection_token_file: Option<String>,
pub without_connection_token: bool,
// reconnection
pub reconnection_grace_time: Option<u32>,
}
impl CodeServerArgs {
@@ -120,6 +122,9 @@ impl CodeServerArgs {
if let Some(i) = self.log {
args.push(format!("--log={i}"));
}
if let Some(t) = self.reconnection_grace_time {
args.push(format!("--reconnection-grace-time={t}"));
}
for extension in &self.install_extensions {
args.push(format!("--install-extension={extension}"));

View File

@@ -14,7 +14,7 @@ import * as performance from '../../../base/common/performance.js';
import { StopWatch } from '../../../base/common/stopwatch.js';
import { generateUuid } from '../../../base/common/uuid.js';
import { IIPCLogger } from '../../../base/parts/ipc/common/ipc.js';
import { Client, ISocket, PersistentProtocol, SocketCloseEventType } from '../../../base/parts/ipc/common/ipc.net.js';
import { Client, ISocket, PersistentProtocol, ProtocolConstants, SocketCloseEventType } from '../../../base/parts/ipc/common/ipc.net.js';
import { ILogService } from '../../log/common/log.js';
import { RemoteAgentConnectionContext } from './remoteAgentEnvironment.js';
import { RemoteAuthorityResolverError, RemoteConnection } from './remoteAuthorityResolver.js';
@@ -563,6 +563,7 @@ export abstract class PersistentConnection extends Disposable {
private _isReconnecting: boolean = false;
private _isDisposed: boolean = false;
private _reconnectionGraceTime: number = ProtocolConstants.ReconnectionGraceTime;
constructor(
private readonly _connectionType: ConnectionType,
@@ -573,6 +574,7 @@ export abstract class PersistentConnection extends Disposable {
) {
super();
this._onDidStateChange.fire(new ConnectionGainEvent(this.reconnectionToken, 0, 0));
this._register(protocol.onSocketClose((e) => {
@@ -611,6 +613,13 @@ export abstract class PersistentConnection extends Disposable {
}
}
public updateGraceTime(graceTime: number): void {
const sanitizedGrace = sanitizeGraceTime(graceTime, ProtocolConstants.ReconnectionGraceTime);
const logPrefix = commonLogPrefix(this._connectionType, this.reconnectionToken, false);
this._options.logService.trace(`${logPrefix} Applying reconnection grace time: ${sanitizedGrace}ms (${Math.floor(sanitizedGrace / 1000)}s)`);
this._reconnectionGraceTime = sanitizedGrace;
}
public override dispose(): void {
super.dispose();
this._isDisposed = true;
@@ -638,6 +647,14 @@ export abstract class PersistentConnection extends Disposable {
this._options.logService.info(`${logPrefix} starting reconnecting loop. You can get more information with the trace log level.`);
this._onDidStateChange.fire(new ConnectionLostEvent(this.reconnectionToken, this.protocol.getMillisSinceLastIncomingData()));
const TIMES = [0, 5, 5, 10, 10, 10, 10, 10, 30];
const graceTime = this._reconnectionGraceTime;
this._options.logService.info(`${logPrefix} starting reconnection with grace time: ${graceTime}ms (${Math.floor(graceTime / 1000)}s)`);
if (graceTime <= 0) {
this._options.logService.error(`${logPrefix} reconnection grace time is set to 0ms, will not attempt to reconnect.`);
this._onReconnectionPermanentFailure(this.protocol.getMillisSinceLastIncomingData(), 0, false);
return;
}
const loopStartTime = Date.now();
let attempt = -1;
do {
attempt++;
@@ -675,9 +692,9 @@ export abstract class PersistentConnection extends Disposable {
this._onReconnectionPermanentFailure(this.protocol.getMillisSinceLastIncomingData(), attempt + 1, false);
break;
}
if (attempt > 360) {
// ReconnectionGraceTime is 3hrs, with 30s between attempts that yields a maximum of 360 attempts
this._options.logService.error(`${logPrefix} An error occurred while reconnecting, but it will be treated as a permanent error because the reconnection grace time has expired! Will give up now! Error:`);
if (Date.now() - loopStartTime >= graceTime) {
const graceSeconds = Math.round(graceTime / 1000);
this._options.logService.error(`${logPrefix} An error occurred while reconnecting, but it will be treated as a permanent error because the reconnection grace time (${graceSeconds}s) has expired! Will give up now! Error:`);
this._options.logService.error(err);
this._onReconnectionPermanentFailure(this.protocol.getMillisSinceLastIncomingData(), attempt + 1, false);
break;
@@ -788,6 +805,16 @@ function getErrorFromMessage(msg: any): Error | null {
return null;
}
function sanitizeGraceTime(candidate: number, fallback: number): number {
if (typeof candidate !== 'number' || !isFinite(candidate) || candidate < 0) {
return fallback;
}
if (candidate > Number.MAX_SAFE_INTEGER) {
return Number.MAX_SAFE_INTEGER;
}
return Math.floor(candidate);
}
function stringRightPad(str: string, len: number): string {
while (str.length < len) {
str += ' ';

View File

@@ -29,6 +29,7 @@ export interface IRemoteAgentEnvironment {
home: URI;
};
isUnsupportedGlibc: boolean;
reconnectionGraceTime?: number;
}
export interface RemoteAgentConnectionContext {

View File

@@ -63,6 +63,9 @@ export async function buildUserEnvironment(startParamsEnv: { [key: string]: stri
env.BROWSER = join(binFolder, 'helpers', isWindows ? 'browser.cmd' : 'browser.sh'); // a command that opens a browser on the local machine
}
env.VSCODE_RECONNECTION_GRACE_TIME = String(environmentService.reconnectionGraceTime);
logService.trace(`[reconnection-grace-time] Setting VSCODE_RECONNECTION_GRACE_TIME env var for extension host: ${environmentService.reconnectionGraceTime}ms (${Math.floor(environmentService.reconnectionGraceTime / 1000)}s)`);
removeNulls(env);
return env;
}

View File

@@ -21,6 +21,7 @@ import { ServerConnectionToken, ServerConnectionTokenType } from './serverConnec
import { IExtensionHostStatusService } from './extensionHostStatusService.js';
import { IUserDataProfilesService } from '../../platform/userDataProfile/common/userDataProfile.js';
import { joinPath } from '../../base/common/resources.js';
import { ILogService } from '../../platform/log/common/log.js';
export class RemoteAgentEnvironmentChannel implements IServerChannel {
@@ -31,6 +32,7 @@ export class RemoteAgentEnvironmentChannel implements IServerChannel {
private readonly _environmentService: IServerEnvironmentService,
private readonly _userDataProfilesService: IUserDataProfilesService,
private readonly _extensionHostStatusService: IExtensionHostStatusService,
private readonly _logService: ILogService,
) {
}
@@ -105,6 +107,7 @@ export class RemoteAgentEnvironmentChannel implements IServerChannel {
const minorVersion = glibcVersion ? parseInt(glibcVersion.split('.')[1]) : 28;
isUnsupportedGlibc = (minorVersion <= 27) || !!process.env['VSCODE_SERVER_CUSTOM_GLIBC_LINKER'];
}
this._logService.trace(`[reconnection-grace-time] Server sending grace time to client: ${this._environmentService.reconnectionGraceTime}ms (${Math.floor(this._environmentService.reconnectionGraceTime / 1000)}s)`);
return {
pid: process.pid,
connectionToken: (this._connectionToken.type !== ServerConnectionTokenType.None ? this._connectionToken.value : ''),
@@ -125,7 +128,8 @@ export class RemoteAgentEnvironmentChannel implements IServerChannel {
home: this._userDataProfilesService.profilesHome,
all: [...this._userDataProfilesService.profiles].map(profile => ({ ...profile }))
},
isUnsupportedGlibc
isUnsupportedGlibc,
reconnectionGraceTime: this._environmentService.reconnectionGraceTime
};
}

View File

@@ -64,6 +64,7 @@ class RemoteExtensionHostAgentServer extends Disposable implements IServerAPI {
private readonly _allReconnectionTokens: Set<string>;
private readonly _webClientServer: WebClientServer | null;
private readonly _webEndpointOriginChecker: WebEndpointOriginChecker;
private readonly _reconnectionGraceTime: number;
private readonly _serverBasePath: string | undefined;
private readonly _serverProductPath: string;
@@ -99,6 +100,7 @@ class RemoteExtensionHostAgentServer extends Disposable implements IServerAPI {
: null
);
this._logService.info(`Extension host agent started.`);
this._reconnectionGraceTime = this._environmentService.reconnectionGraceTime;
this._waitThenShutdown(true);
}
@@ -419,7 +421,7 @@ class RemoteExtensionHostAgentServer extends Disposable implements IServerAPI {
}
protocol.sendControl(VSBuffer.fromString(JSON.stringify({ type: 'ok' })));
const con = new ManagementConnection(this._logService, reconnectionToken, remoteAddress, protocol);
const con = new ManagementConnection(this._logService, reconnectionToken, remoteAddress, protocol, this._reconnectionGraceTime);
this._socketServer.acceptConnection(con.protocol, con.onClose);
this._managementConnections[reconnectionToken] = con;
this._allReconnectionTokens.add(reconnectionToken);

View File

@@ -3,7 +3,7 @@
* Licensed under the MIT License. See License.txt in the project root for license information.
*--------------------------------------------------------------------------------------------*/
import { PersistentProtocol, ProtocolConstants, ISocket } from '../../base/parts/ipc/common/ipc.net.js';
import { PersistentProtocol, ISocket, ProtocolConstants } from '../../base/parts/ipc/common/ipc.net.js';
import { ILogService } from '../../platform/log/common/log.js';
import { Emitter, Event } from '../../base/common/event.js';
import { VSBuffer } from '../../base/common/buffer.js';
@@ -50,10 +50,12 @@ export class ManagementConnection {
private readonly _logService: ILogService,
private readonly _reconnectionToken: string,
remoteAddress: string,
protocol: PersistentProtocol
protocol: PersistentProtocol,
reconnectionGraceTime: number
) {
this._reconnectionGraceTime = ProtocolConstants.ReconnectionGraceTime;
this._reconnectionShortGraceTime = ProtocolConstants.ReconnectionShortGraceTime;
this._reconnectionGraceTime = reconnectionGraceTime;
const defaultShortGrace = ProtocolConstants.ReconnectionShortGraceTime;
this._reconnectionShortGraceTime = reconnectionGraceTime > 0 ? Math.min(defaultShortGrace, reconnectionGraceTime) : 0;
this._remoteAddress = remoteAddress;
this.protocol = protocol;

View File

@@ -13,6 +13,7 @@ import { memoize } from '../../base/common/decorators.js';
import { URI } from '../../base/common/uri.js';
import { joinPath } from '../../base/common/resources.js';
import { join } from '../../base/common/path.js';
import { ProtocolConstants } from '../../base/parts/ipc/common/ipc.net.js';
export const serverOptions: OptionDescriptions<Required<ServerParsedArgs>> = {
@@ -85,6 +86,7 @@ export const serverOptions: OptionDescriptions<Required<ServerParsedArgs>> = {
'use-host-proxy': { type: 'boolean' },
'without-browser-env-var': { type: 'boolean' },
'reconnection-grace-time': { type: 'string', cat: 'o', args: 'seconds', description: nls.localize('reconnection-grace-time', "Override the reconnection grace time window in seconds. Defaults to 10800 (3 hours).") },
/* ----- server cli ----- */
@@ -213,6 +215,7 @@ export interface ServerParsedArgs {
'use-host-proxy'?: boolean;
'without-browser-env-var'?: boolean;
'reconnection-grace-time'?: string;
/* ----- server cli ----- */
help: boolean;
@@ -230,6 +233,7 @@ export interface IServerEnvironmentService extends INativeEnvironmentService {
readonly machineSettingsResource: URI;
readonly mcpResource: URI;
readonly args: ServerParsedArgs;
readonly reconnectionGraceTime: number;
}
export class ServerEnvironmentService extends NativeEnvironmentService implements IServerEnvironmentService {
@@ -240,4 +244,25 @@ export class ServerEnvironmentService extends NativeEnvironmentService implement
@memoize
get mcpResource(): URI { return joinPath(URI.file(join(this.userDataPath, 'User')), 'mcp.json'); }
override get args(): ServerParsedArgs { return super.args as ServerParsedArgs; }
@memoize
get reconnectionGraceTime(): number { return parseGraceTime(this.args['reconnection-grace-time'], ProtocolConstants.ReconnectionGraceTime); }
}
function parseGraceTime(rawValue: string | undefined, fallback: number): number {
if (typeof rawValue !== 'string' || rawValue.trim().length === 0) {
console.log(`[reconnection-grace-time] No CLI argument provided, using default: ${fallback}ms (${Math.floor(fallback / 1000)}s)`);
return fallback;
}
const parsedSeconds = Number(rawValue);
if (!isFinite(parsedSeconds) || parsedSeconds < 0) {
console.log(`[reconnection-grace-time] Invalid value '${rawValue}', using default: ${fallback}ms (${Math.floor(fallback / 1000)}s)`);
return fallback;
}
const millis = Math.floor(parsedSeconds * 1000);
if (!isFinite(millis) || millis > Number.MAX_SAFE_INTEGER) {
console.log(`[reconnection-grace-time] Value too large '${rawValue}', using default: ${fallback}ms (${Math.floor(fallback / 1000)}s)`);
return fallback;
}
console.log(`[reconnection-grace-time] Parsed CLI argument: ${parsedSeconds}s -> ${millis}ms`);
return millis;
}

View File

@@ -216,8 +216,8 @@ export async function setupServerServices(connectionToken: ServerConnectionToken
const ptyHostStarter = instantiationService.createInstance(
NodePtyHostStarter,
{
graceTime: ProtocolConstants.ReconnectionGraceTime,
shortGraceTime: ProtocolConstants.ReconnectionShortGraceTime,
graceTime: environmentService.reconnectionGraceTime,
shortGraceTime: environmentService.reconnectionGraceTime > 0 ? Math.min(ProtocolConstants.ReconnectionShortGraceTime, environmentService.reconnectionGraceTime) : 0,
scrollback: configurationService.getValue<number>(TerminalSettingId.PersistentSessionScrollback) ?? 100
}
);
@@ -235,7 +235,7 @@ export async function setupServerServices(connectionToken: ServerConnectionToken
const extensionsScannerService = accessor.get(IExtensionsScannerService);
const extensionGalleryService = accessor.get(IExtensionGalleryService);
const languagePackService = accessor.get(ILanguagePackService);
const remoteExtensionEnvironmentChannel = new RemoteAgentEnvironmentChannel(connectionToken, environmentService, userDataProfilesService, extensionHostStatusService);
const remoteExtensionEnvironmentChannel = new RemoteAgentEnvironmentChannel(connectionToken, environmentService, userDataProfilesService, extensionHostStatusService, logService);
socketServer.registerChannel('remoteextensionsenvironment', remoteExtensionEnvironmentChannel);
const telemetryChannel = new ServerTelemetryChannel(accessor.get(IServerTelemetryService), oneDsAppender);

View File

@@ -160,6 +160,23 @@ let onTerminate = function (reason: string) {
nativeExit();
};
function readReconnectionValue(envKey: string, fallback: number): number {
const raw = process.env[envKey];
if (typeof raw !== 'string' || raw.trim().length === 0) {
console.log(`[reconnection-grace-time] Extension host: env var ${envKey} not set, using default: ${fallback}ms (${Math.floor(fallback / 1000)}s)`);
return fallback;
}
const parsed = Number(raw);
if (!isFinite(parsed) || parsed < 0) {
console.log(`[reconnection-grace-time] Extension host: env var ${envKey} invalid value '${raw}', using default: ${fallback}ms (${Math.floor(fallback / 1000)}s)`);
return fallback;
}
const millis = Math.floor(parsed);
const result = millis > Number.MAX_SAFE_INTEGER ? Number.MAX_SAFE_INTEGER : millis;
console.log(`[reconnection-grace-time] Extension host: read ${envKey}=${raw}ms (${Math.floor(result / 1000)}s)`);
return result;
}
function _createExtHostProtocol(): Promise<IMessagePassingProtocol> {
const extHostConnection = readExtHostConnection(process.env);
@@ -195,8 +212,8 @@ function _createExtHostProtocol(): Promise<IMessagePassingProtocol> {
onTerminate('VSCODE_EXTHOST_IPC_SOCKET timeout');
}, 60000);
const reconnectionGraceTime = ProtocolConstants.ReconnectionGraceTime;
const reconnectionShortGraceTime = ProtocolConstants.ReconnectionShortGraceTime;
const reconnectionGraceTime = readReconnectionValue('VSCODE_RECONNECTION_GRACE_TIME', ProtocolConstants.ReconnectionGraceTime);
const reconnectionShortGraceTime = reconnectionGraceTime > 0 ? Math.min(ProtocolConstants.ReconnectionShortGraceTime, reconnectionGraceTime) : 0;
const disconnectRunner1 = new ProcessTimeRunOnceScheduler(() => onTerminate('renderer disconnected for too long (1)'), reconnectionGraceTime);
const disconnectRunner2 = new ProcessTimeRunOnceScheduler(() => onTerminate('renderer disconnected for too long (2)'), reconnectionShortGraceTime);

View File

@@ -35,11 +35,11 @@ export abstract class AbstractRemoteAgentService extends Disposable implements I
@IProductService productService: IProductService,
@IRemoteAuthorityResolverService private readonly _remoteAuthorityResolverService: IRemoteAuthorityResolverService,
@ISignService signService: ISignService,
@ILogService logService: ILogService
@ILogService private readonly _logService: ILogService
) {
super();
if (this._environmentService.remoteAuthority) {
this._connection = this._register(new RemoteAgentConnection(this._environmentService.remoteAuthority, productService.commit, productService.quality, this.remoteSocketFactoryService, this._remoteAuthorityResolverService, signService, logService));
this._connection = this._register(new RemoteAgentConnection(this._environmentService.remoteAuthority, productService.commit, productService.quality, this.remoteSocketFactoryService, this._remoteAuthorityResolverService, signService, this._logService));
} else {
this._connection = null;
}
@@ -60,6 +60,12 @@ export abstract class AbstractRemoteAgentService extends Disposable implements I
async (channel, connection) => {
const env = await RemoteExtensionEnvironmentChannelClient.getEnvironmentData(channel, connection.remoteAuthority, this.userDataProfileService.currentProfile.isDefault ? undefined : this.userDataProfileService.currentProfile.id);
this._remoteAuthorityResolverService._setAuthorityConnectionToken(connection.remoteAuthority, env.connectionToken);
if (typeof env.reconnectionGraceTime === 'number') {
this._logService.info(`[reconnection-grace-time] Client received grace time from server: ${env.reconnectionGraceTime}ms (${Math.floor(env.reconnectionGraceTime / 1000)}s)`);
connection.updateGraceTime(env.reconnectionGraceTime);
} else {
this._logService.info(`[reconnection-grace-time] Server did not provide grace time, using default`);
}
return env;
},
null
@@ -149,6 +155,7 @@ class RemoteAgentConnection extends Disposable implements IRemoteAgentConnection
readonly remoteAuthority: string;
private _connection: Promise<Client<RemoteAgentConnectionContext>> | null;
private _managementConnection: ManagementPersistentConnection | null = null;
private _initialConnectionMs: number | undefined;
@@ -192,6 +199,16 @@ class RemoteAgentConnection extends Disposable implements IRemoteAgentConnection
return this._initialConnectionMs!;
}
getManagementConnection(): ManagementPersistentConnection | null {
return this._managementConnection;
}
updateGraceTime(graceTime: number): void {
if (this._managementConnection) {
this._managementConnection.updateGraceTime(graceTime);
}
}
private _getOrCreateConnection(): Promise<Client<RemoteAgentConnectionContext>> {
if (!this._connection) {
this._connection = this._createConnection();
@@ -224,6 +241,7 @@ class RemoteAgentConnection extends Disposable implements IRemoteAgentConnection
const start = Date.now();
try {
connection = this._register(await connectRemoteAgentManagement(options, this.remoteAuthority, `renderer`));
this._managementConnection = connection;
} finally {
this._initialConnectionMs = Date.now() - start;
}

View File

@@ -13,6 +13,7 @@ import { ITelemetryData, TelemetryLevel } from '../../../../platform/telemetry/c
import { IExtensionHostExitInfo } from './remoteAgentService.js';
import { revive } from '../../../../base/common/marshalling.js';
import { IUserDataProfile } from '../../../../platform/userDataProfile/common/userDataProfile.js';
import { ProtocolConstants } from '../../../../base/parts/ipc/common/ipc.net.js';
export interface IGetEnvironmentDataArguments {
remoteAuthority: string;
@@ -45,6 +46,7 @@ export interface IRemoteAgentEnvironmentDTO {
home: UriComponents;
};
isUnsupportedGlibc: boolean;
reconnectionGraceTime?: number;
}
export class RemoteExtensionEnvironmentChannelClient {
@@ -56,6 +58,9 @@ export class RemoteExtensionEnvironmentChannelClient {
};
const data = await channel.call<IRemoteAgentEnvironmentDTO>('getEnvironmentData', args);
const reconnectionGraceTime = (typeof data.reconnectionGraceTime === 'number' && data.reconnectionGraceTime >= 0)
? data.reconnectionGraceTime
: ProtocolConstants.ReconnectionGraceTime;
return {
pid: data.pid,
@@ -74,7 +79,8 @@ export class RemoteExtensionEnvironmentChannelClient {
marks: data.marks,
useHostProxy: data.useHostProxy,
profiles: revive(data.profiles),
isUnsupportedGlibc: data.isUnsupportedGlibc
isUnsupportedGlibc: data.isUnsupportedGlibc,
reconnectionGraceTime
};
}

View File

@@ -65,6 +65,7 @@ export interface IRemoteAgentConnection {
withChannel<T extends IChannel, R>(channelName: string, callback: (channel: T) => Promise<R>): Promise<R>;
registerChannel<T extends IServerChannel<RemoteAgentConnectionContext>>(channelName: string, channel: T): void;
getInitialConnectionTimeMs(): Promise<number>;
updateGraceTime(graceTime: number): void;
}
export interface IRemoteConnectionLatencyMeasurement {