Precompute NLS translation cache identity during packaging (#336109)

nls: precompute translation cache identity during packaging

Stamp the commit and NLS table fingerprint in product metadata and pass it through bootstrap. Reuse cached translations without reading or hashing default tables, while isolating different target tables and preserving legacy product compatibility.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

.github/instructions/buildNext.instructions.md

@@ -153,6 +153,10 @@ Two placeholders that need injection:
 
 **Lesson:** Don't add new output file formats that create parity differences with the old build. The old build is the reference.
 
+### Translated Message Cache Identity
+
+Desktop, server, and server-web packaging compute `nlsMetadataHash` from the commit, ordered NLS keys, and default messages, and stamp it into the product configuration. Native bootstrap and remote language-pack resolution pass this identity to `resolveNLSConfiguration()`, so a cached localized startup does not read or hash the NLS tables. Products without the new field retain the legacy commit-based cache path. Different target tables get separate caches; identical tables at the same commit reuse one cache.
+
 ### 7. Resource Copying: Transpile vs Bundle
 
 **Problem:** The new build used curated, specific resource pattern lists (e.g., `desktopResourcePatterns`) for **both** transpile/dev and production/bundle builds. Team members kept discovering missing resources because every new non-TS file in `src/` required manually adding its pattern.

build/gulpfile.reh.ts

@@ -11,6 +11,7 @@ import { getVersion } from './lib/getVersion.ts';
 import * as task from './lib/gulp/task.ts';
 import * as optimize from './lib/optimize.ts';
 import { inlineMeta } from './lib/inlineMeta.ts';
+import { computeNLSMetadataHash } from './lib/nlsMetadata.ts';
 import product from '../product.json' with { type: 'json' };
 import { getProductionDependencies } from './lib/dependencies.ts';
 import { readISODate } from './lib/date.ts';
@@ -412,6 +413,7 @@ function packageTask(type: string, platform: string, arch: string, sourceFolderN
 		const productJsonStream = gulp.src(['product.json'], { base: '.' })
 			.pipe(jsonEditor((json: Record<string, unknown>) => {
 				json.commit = commit;
+				json.nlsMetadataHash = computeNLSMetadataHash(path.join(REPO_ROOT, sourceFolderName), commit);
 				json.date = readISODate(sourceFolderName);
 				json.version = version;
 				// Stamp agentSdks from the per-platform results file produced

build/gulpfile.vscode.ts

@@ -16,6 +16,7 @@ import * as task from './lib/gulp/task.ts';
 import buildfile from './buildfile.ts';
 import * as optimize from './lib/optimize.ts';
 import { inlineMeta } from './lib/inlineMeta.ts';
+import { computeNLSMetadataHash } from './lib/nlsMetadata.ts';
 import packageJson from '../package.json' with { type: 'json' };
 import product from '../product.json' with { type: 'json' };
 import * as crypto from 'crypto';
@@ -336,6 +337,7 @@ function packageTask(platform: string, arch: string, sourceFolderName: string, d
 		const productJsonStream = gulp.src(['product.json'], { base: '.' })
 			.pipe(jsonEditor((json: Record<string, unknown>) => {
 				json.commit = commit;
+				json.nlsMetadataHash = computeNLSMetadataHash(path.join(import.meta.dirname, '..', out), commit);
 				json.date = readISODate(out);
 				json.checksums = checksums;
 				json.version = version;

build/lib/nlsMetadata.ts

@@ -0,0 +1,18 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { createHash } from 'crypto';
+import { readFileSync } from 'fs';
+import { join } from 'path';
+
+export function computeNLSMetadataHash(nlsMetadataPath: string, commit: string | undefined): string {
+	return createHash('sha256')
+		.update(commit ?? '')
+		.update('\0')
+		.update(readFileSync(join(nlsMetadataPath, 'nls.keys.json')))
+		.update('\0')
+		.update(readFileSync(join(nlsMetadataPath, 'nls.messages.json')))
+		.digest('hex');
+}

build/lib/test/nlsMetadata.test.ts

@@ -0,0 +1,63 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import assert from 'assert';
+import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'fs';
+import { tmpdir } from 'os';
+import { join } from 'path';
+import { afterEach, beforeEach, suite, test } from 'node:test';
+import { computeNLSMetadataHash } from '../nlsMetadata.ts';
+
+suite('NLS metadata identity', () => {
+	let directory: string;
+
+	beforeEach(() => {
+		directory = mkdtempSync(join(tmpdir(), 'vscode-nls-metadata-'));
+	});
+
+	afterEach(() => {
+		rmSync(directory, { recursive: true, force: true });
+	});
+
+	function writeMetadata(name: string, keys: Array<[string, string[]]>, messages: string[]): string {
+		const metadataPath = join(directory, name);
+		mkdirSync(metadataPath);
+		writeFileSync(join(metadataPath, 'nls.keys.json'), JSON.stringify(keys));
+		writeFileSync(join(metadataPath, 'nls.messages.json'), JSON.stringify(messages));
+		return metadataPath;
+	}
+
+	test('identical tables have the same identity regardless of output location', () => {
+		const first = writeMetadata('first', [['module', ['key']]], ['Message']);
+		const second = writeMetadata('second', [['module', ['key']]], ['Message']);
+		const hash = computeNLSMetadataHash(first, 'commit');
+
+		assert.deepStrictEqual({
+			hashLength: hash.length,
+			sameIdentity: hash === computeNLSMetadataHash(second, 'commit')
+		}, {
+			hashLength: 64,
+			sameIdentity: true
+		});
+	});
+
+	test('the identity includes key order, default messages and commit', () => {
+		const original = writeMetadata('original', [['module', ['first', 'second']]], ['Same', 'Same']);
+		const reordered = writeMetadata('reordered', [['module', ['second', 'first']]], ['Same', 'Same']);
+		const changedMessages = writeMetadata('changed', [['module', ['first', 'second']]], ['Changed', 'Same']);
+		const identities = [
+			computeNLSMetadataHash(original, 'commit'),
+			computeNLSMetadataHash(reordered, 'commit'),
+			computeNLSMetadataHash(changedMessages, 'commit'),
+			computeNLSMetadataHash(original, 'another-commit')
+		];
+
+		assert.strictEqual(new Set(identities).size, identities.length);
+	});
+
+	test('missing metadata fails packaging instead of producing a shared fallback identity', () => {
+		assert.throws(() => computeNLSMetadataHash(directory, 'commit'), { code: 'ENOENT' });
+	});
+});

src/cli.ts

@@ -10,7 +10,7 @@ import { resolveNLSConfiguration } from './vs/base/node/nls.js';
 import { product } from './bootstrap-meta.js';
 
 // NLS
-const nlsConfiguration = await resolveNLSConfiguration({ userLocale: 'en', osLocale: 'en', commit: product.commit, userDataPath: '', nlsMetadataPath: import.meta.dirname });
+const nlsConfiguration = await resolveNLSConfiguration({ userLocale: 'en', osLocale: 'en', commit: product.commit, nlsMetadataHash: product.nlsMetadataHash, userDataPath: '', nlsMetadataPath: import.meta.dirname });
 process.env['VSCODE_NLS_CONFIG'] = JSON.stringify(nlsConfiguration); // required for `bootstrap-esm` to pick up NLS messages
 
 // Enable portable support

src/main.ts

@@ -133,6 +133,7 @@ if (userLocale) {
 		userLocale,
 		osLocale,
 		commit: product.commit,
+		nlsMetadataHash: product.nlsMetadataHash,
 		userDataPath,
 		nlsMetadataPath: import.meta.dirname
 	});
@@ -715,6 +716,7 @@ async function resolveNlsConfiguration(): Promise<INLSConfiguration> {
 		userLocale,
 		osLocale,
 		commit: product.commit,
+		nlsMetadataHash: product.nlsMetadataHash,
 		userDataPath,
 		nlsMetadataPath: import.meta.dirname
 	});

src/server-cli.ts

@@ -11,7 +11,7 @@ import { resolveNLSConfiguration } from './vs/base/node/nls.js';
 import { product } from './bootstrap-meta.js';
 
 // NLS
-const nlsConfiguration = await resolveNLSConfiguration({ userLocale: 'en', osLocale: 'en', commit: product.commit, userDataPath: '', nlsMetadataPath: import.meta.dirname });
+const nlsConfiguration = await resolveNLSConfiguration({ userLocale: 'en', osLocale: 'en', commit: product.commit, nlsMetadataHash: product.nlsMetadataHash, userDataPath: '', nlsMetadataPath: import.meta.dirname });
 process.env['VSCODE_NLS_CONFIG'] = JSON.stringify(nlsConfiguration); // required for `bootstrap-esm` to pick up NLS messages
 
 if (process.env['VSCODE_DEV']) {

src/server-main.ts

@@ -43,7 +43,7 @@ const extensionInstallArgs = ['install-extension', 'install-builtin-extension',
 
 const shouldSpawnCli = parsedArgs.help || parsedArgs.version || extensionLookupArgs.some(a => !!parsedArgs[a]) || (extensionInstallArgs.some(a => !!parsedArgs[a]) && !parsedArgs['start-server']);
 
-const nlsConfiguration = await resolveNLSConfiguration({ userLocale: 'en', osLocale: 'en', commit: product.commit, userDataPath: '', nlsMetadataPath: import.meta.dirname });
+const nlsConfiguration = await resolveNLSConfiguration({ userLocale: 'en', osLocale: 'en', commit: product.commit, nlsMetadataHash: product.nlsMetadataHash, userDataPath: '', nlsMetadataPath: import.meta.dirname });
 
 if (shouldSpawnCli) {
 	loadCode(nlsConfiguration).then((mod) => {

src/vs/base/common/product.ts

@@ -131,6 +131,8 @@ export interface IProductConfiguration {
 	readonly webviewContentExternalBaseUrlTemplate?: string;
 	readonly target?: string;
 	readonly nlsCoreBaseUrl?: string;
+	/** Build-time hash of the commit and core NLS tables used to identify translated-message caches. */
+	readonly nlsMetadataHash?: string;
 
 	readonly settingsSearchBuildId?: number;
 	readonly settingsSearchUrl?: string;

src/vs/base/node/nls.ts

@@ -16,6 +16,11 @@ export interface IResolveNLSConfigurationContext {
 	 */
 	readonly nlsMetadataPath: string;
 
+	/**
+	 * Precomputed identity of the commit and NLS tables, supplied by packaged products.
+	 */
+	readonly nlsMetadataHash?: string;
+
 	/**
 	 * Path to the user data directory. Used as a cache for
 	 * language packs converted to the format we need.
@@ -39,7 +44,7 @@ export interface IResolveNLSConfigurationContext {
 	readonly osLocale: string;
 }
 
-export async function resolveNLSConfiguration({ userLocale, osLocale, userDataPath, commit, nlsMetadataPath }: IResolveNLSConfigurationContext): Promise<INLSConfiguration> {
+export async function resolveNLSConfiguration({ userLocale, osLocale, userDataPath, commit, nlsMetadataPath, nlsMetadataHash }: IResolveNLSConfigurationContext): Promise<INLSConfiguration> {
 	mark('code/willGenerateNls');
 
 	if (
@@ -77,7 +82,7 @@ export async function resolveNLSConfiguration({ userLocale, osLocale, userDataPa
 
 		const languagePackId = `${languagePack.hash}.${resolvedLanguage}`;
 		const globalLanguagePackCachePath = join(userDataPath, 'clp', languagePackId);
-		const commitLanguagePackCachePath = join(globalLanguagePackCachePath, commit);
+		const commitLanguagePackCachePath = join(globalLanguagePackCachePath, nlsMetadataHash ?? commit);
 		const languagePackMessagesFile = join(commitLanguagePackCachePath, 'nls.messages.json');
 		const translationsConfigFile = join(globalLanguagePackCachePath, 'tcf.json');
 		const languagePackCorruptMarkerFile = join(globalLanguagePackCachePath, 'corrupted.info');

src/vs/base/test/node/nls.test.ts

@@ -0,0 +1,232 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import assert from 'assert';
+import { createHash } from 'crypto';
+import { promises } from 'fs';
+import { tmpdir } from 'os';
+import { ILanguagePacks } from '../../../nls.js';
+import { join } from '../../common/path.js';
+import { IResolveNLSConfigurationContext, resolveNLSConfiguration } from '../../node/nls.js';
+import { Promises } from '../../node/pfs.js';
+import { ensureNoDisposablesAreLeakedInTestSuite } from '../common/utils.js';
+import { getRandomTestPath } from './testUtils.js';
+
+type NLSMetadata = Pick<IResolveNLSConfigurationContext, 'nlsMetadataPath' | 'nlsMetadataHash'>;
+
+suite('NLS configuration', () => {
+	ensureNoDisposablesAreLeakedInTestSuite();
+
+	const commit = 'test-commit';
+	const languagePackId = 'test-language-pack.de';
+	let testDir: string;
+	let userDataPath: string;
+	let vscodeDev: string | undefined;
+
+	setup(async () => {
+		vscodeDev = process.env['VSCODE_DEV'];
+		delete process.env['VSCODE_DEV'];
+
+		testDir = getRandomTestPath(tmpdir(), 'vsctests', 'nls');
+		userDataPath = join(testDir, 'user');
+		await promises.mkdir(userDataPath, { recursive: true });
+
+		const translationsFile = join(testDir, 'main.i18n.json');
+		const languagePacks: ILanguagePacks = {
+			de: {
+				hash: 'test-language-pack',
+				label: 'Deutsch',
+				extensions: [],
+				translations: { vscode: translationsFile }
+			}
+		};
+		await Promise.all([
+			promises.writeFile(join(userDataPath, 'languagepacks.json'), JSON.stringify(languagePacks)),
+			promises.writeFile(translationsFile, JSON.stringify({
+				contents: {
+					'vs/base/test': { first: 'Erste', second: 'Zweite' },
+					'vs/workbench/api/common/extHostLogService': { remote: 'Entfernt' }
+				}
+			}))
+		]);
+	});
+
+	teardown(async () => {
+		if (vscodeDev === undefined) {
+			delete process.env['VSCODE_DEV'];
+		} else {
+			process.env['VSCODE_DEV'] = vscodeDev;
+		}
+		await Promises.rm(testDir);
+	});
+
+	async function writeMetadata(name: string, keys: Array<[string, string[]]>, messages: string[]): Promise<NLSMetadata> {
+		const nlsMetadataPath = join(testDir, name);
+		await promises.mkdir(nlsMetadataPath, { recursive: true });
+		await Promise.all([
+			promises.writeFile(join(nlsMetadataPath, 'nls.keys.json'), JSON.stringify(keys)),
+			promises.writeFile(join(nlsMetadataPath, 'nls.messages.json'), JSON.stringify(messages))
+		]);
+		return {
+			nlsMetadataPath,
+			nlsMetadataHash: createHash('sha256').update(JSON.stringify({ commit, keys, messages })).digest('hex')
+		};
+	}
+
+	async function resolveMessages(metadata: NLSMetadata) {
+		const configuration = await resolveNLSConfiguration({
+			userLocale: 'de',
+			osLocale: 'de',
+			userDataPath,
+			commit,
+			...metadata
+		});
+		assert.ok(configuration.languagePack);
+		const messages: string[] = JSON.parse(await promises.readFile(configuration.languagePack.messagesFile, 'utf8'));
+		return { ...configuration.languagePack, messages };
+	}
+
+	test('switches between server and server-web tables at the same commit', async () => {
+		const serverMetadata = await writeMetadata('server', [
+			['vs/workbench/api/common/extHostLogService', ['remote']]
+		], ['Remote']);
+		const serverWebMetadata = await writeMetadata('server-web', [
+			['vs/base/test', ['first', 'missing']],
+			['vs/workbench/api/common/extHostLogService', ['remote']]
+		], ['First', 'Fallback', 'Remote']);
+
+		const server = await resolveMessages(serverMetadata);
+		const serverWeb = await resolveMessages(serverWebMetadata);
+		const serverAgain = await resolveMessages(serverMetadata);
+
+		assert.deepStrictEqual({
+			messages: [server.messages, serverWeb.messages, serverAgain.messages],
+			distinctTargetCaches: server.messagesFile !== serverWeb.messagesFile,
+			reusedServerCache: server.messagesFile === serverAgain.messagesFile
+		}, {
+			messages: [['Entfernt'], ['Erste', 'Fallback', 'Entfernt'], ['Entfernt']],
+			distinctTargetCaches: true,
+			reusedServerCache: true
+		});
+	});
+
+	test('distinguishes key order when the default messages are identical', async () => {
+		const firstMetadata = await writeMetadata('first', [['vs/base/test', ['first', 'second']]], ['Same', 'Same']);
+		const secondMetadata = await writeMetadata('second', [['vs/base/test', ['second', 'first']]], ['Same', 'Same']);
+
+		const first = await resolveMessages(firstMetadata);
+		const second = await resolveMessages(secondMetadata);
+
+		assert.deepStrictEqual([first.messages, second.messages], [['Erste', 'Zweite'], ['Zweite', 'Erste']]);
+	});
+
+	test('refreshes fallback messages after rebuilding at the same commit', async () => {
+		const originalMetadata = await writeMetadata('server', [['vs/base/test', ['missing']]], ['Original']);
+		const original = await resolveMessages(originalMetadata);
+		const updatedMetadata = await writeMetadata('server', [['vs/base/test', ['missing']]], ['Updated']);
+		const updated = await resolveMessages(updatedMetadata);
+
+		assert.deepStrictEqual([original.messages, updated.messages], [['Original'], ['Updated']]);
+	});
+
+	test('reuses the cache for identical tables in different locations', async () => {
+		const firstMetadata = await writeMetadata('first', [['vs/base/test', ['first']]], ['First']);
+		const secondMetadata = await writeMetadata('second', [['vs/base/test', ['first']]], ['First']);
+		const first = await resolveMessages(firstMetadata);
+		await promises.writeFile(first.messagesFile, JSON.stringify(['Cached translation']));
+		const second = await resolveMessages(secondMetadata);
+
+		assert.deepStrictEqual({
+			sameCache: first.messagesFile === second.messagesFile,
+			messages: second.messages
+		}, {
+			sameCache: true,
+			messages: ['Cached translation']
+		});
+	});
+
+	test('a cache hit does not read the NLS tables', async () => {
+		const metadata = await writeMetadata('server', [['vs/base/test', ['first']]], ['First']);
+		const first = await resolveMessages(metadata);
+		await Promise.all([
+			promises.unlink(join(metadata.nlsMetadataPath, 'nls.keys.json')),
+			promises.unlink(join(metadata.nlsMetadataPath, 'nls.messages.json'))
+		]);
+		const cached = await resolveMessages(metadata);
+
+		assert.deepStrictEqual({
+			sameCache: first.messagesFile === cached.messagesFile,
+			messages: cached.messages
+		}, {
+			sameCache: true,
+			messages: ['Erste']
+		});
+	});
+
+	test('does not reuse or overwrite a legacy commit-only cache', async () => {
+		const legacyCachePath = join(userDataPath, 'clp', languagePackId, commit);
+		await promises.mkdir(legacyCachePath, { recursive: true });
+		const legacyMessagesFile = join(legacyCachePath, 'nls.messages.json');
+		await promises.writeFile(legacyMessagesFile, JSON.stringify(['Legacy translation']));
+		const metadata = await writeMetadata('server', [['vs/base/test', ['first']]], ['First']);
+		const result = await resolveMessages(metadata);
+
+		assert.deepStrictEqual({
+			messages: result.messages,
+			separateCache: result.messagesFile !== legacyMessagesFile,
+			legacy: JSON.parse(await promises.readFile(legacyMessagesFile, 'utf8'))
+		}, {
+			messages: ['Erste'],
+			separateCache: true,
+			legacy: ['Legacy translation']
+		});
+	});
+
+	test('preserves the legacy cache path for products without an identity', async () => {
+		const metadata = await writeMetadata('server', [['vs/base/test', ['first']]], ['First']);
+		const result = await resolveMessages({ nlsMetadataPath: metadata.nlsMetadataPath });
+
+		assert.deepStrictEqual({
+			messages: result.messages,
+			messagesFile: result.messagesFile
+		}, {
+			messages: ['Erste'],
+			messagesFile: join(userDataPath, 'clp', languagePackId, commit, 'nls.messages.json')
+		});
+	});
+
+	test('regenerates all table caches after a corruption marker', async () => {
+		const firstMetadata = await writeMetadata('first', [['vs/base/test', ['first']]], ['First']);
+		const secondMetadata = await writeMetadata('second', [['vs/base/test', ['second']]], ['Second']);
+		const first = await resolveMessages(firstMetadata);
+		const second = await resolveMessages(secondMetadata);
+		await promises.writeFile(first.corruptMarkerFile, 'corrupt');
+		const regenerated = await resolveMessages(firstMetadata);
+
+		assert.deepStrictEqual({
+			messages: regenerated.messages,
+			secondCacheExists: await Promises.exists(second.messagesFile),
+			translationsConfigExists: await Promises.exists(regenerated.translationsConfigFile)
+		}, {
+			messages: ['Erste'],
+			secondCacheExists: false,
+			translationsConfigExists: true
+		});
+	});
+
+	test('does not require NLS metadata for the default locale', async () => {
+		const nlsMetadataPath = join(testDir, 'missing');
+		const result = await resolveNLSConfiguration({ userLocale: 'en', osLocale: 'en', userDataPath, commit, nlsMetadataPath });
+
+		assert.deepStrictEqual(result, {
+			userLocale: 'en',
+			osLocale: 'en',
+			resolvedLanguage: 'en',
+			defaultMessagesFile: join(nlsMetadataPath, 'nls.messages.json'),
+			locale: 'en',
+			availableLanguages: {}
+		});
+	});
+});

src/vs/server/node/remoteLanguagePacks.ts

@@ -31,7 +31,7 @@ export async function getNLSConfiguration(language: string, userDataPath: string
 	const cacheKey = `${language}||${userDataPath}`;
 	let result = nlsConfigurationCache.get(cacheKey);
 	if (!result) {
-		result = resolveNLSConfiguration({ userLocale: language, osLocale: language, commit: product.commit, userDataPath, nlsMetadataPath });
+		result = resolveNLSConfiguration({ userLocale: language, osLocale: language, commit: product.commit, nlsMetadataHash: product.nlsMetadataHash, userDataPath, nlsMetadataPath });
 		nlsConfigurationCache.set(cacheKey, result);
 	}