From bd0cd43cd7bf3a3c6bd721f729df97fbb7265d58 Mon Sep 17 00:00:00 2001 From: Dmitriy Vasyura Date: Mon, 14 Sep 2026 07:09:44 -0700 Subject: [PATCH] Precompute NLS translation cache identity during packaging (#336109) nls: precompute translation cache identity during packaging Stamp the commit and NLS table fingerprint in product metadata and pass it through bootstrap. Reuse cached translations without reading or hashing default tables, while isolating different target tables and preserving legacy product compatibility. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../instructions/buildNext.instructions.md | 4 + build/gulpfile.reh.ts | 2 + build/gulpfile.vscode.ts | 2 + build/lib/nlsMetadata.ts | 18 ++ build/lib/test/nlsMetadata.test.ts | 63 +++++ src/cli.ts | 2 +- src/main.ts | 2 + src/server-cli.ts | 2 +- src/server-main.ts | 2 +- src/vs/base/common/product.ts | 2 + src/vs/base/node/nls.ts | 9 +- src/vs/base/test/node/nls.test.ts | 232 ++++++++++++++++++ src/vs/server/node/remoteLanguagePacks.ts | 2 +- 13 files changed, 336 insertions(+), 6 deletions(-) create mode 100644 build/lib/nlsMetadata.ts create mode 100644 build/lib/test/nlsMetadata.test.ts create mode 100644 src/vs/base/test/node/nls.test.ts diff --git a/.github/instructions/buildNext.instructions.md b/.github/instructions/buildNext.instructions.md index 9c0d2ebff8c4..bbc91b60ce78 100644 --- a/.github/instructions/buildNext.instructions.md +++ b/.github/instructions/buildNext.instructions.md @@ -153,6 +153,10 @@ Two placeholders that need injection: **Lesson:** Don't add new output file formats that create parity differences with the old build. The old build is the reference. +### Translated Message Cache Identity + +Desktop, server, and server-web packaging compute `nlsMetadataHash` from the commit, ordered NLS keys, and default messages, and stamp it into the product configuration. Native bootstrap and remote language-pack resolution pass this identity to `resolveNLSConfiguration()`, so a cached localized startup does not read or hash the NLS tables. Products without the new field retain the legacy commit-based cache path. Different target tables get separate caches; identical tables at the same commit reuse one cache. + ### 7. Resource Copying: Transpile vs Bundle **Problem:** The new build used curated, specific resource pattern lists (e.g., `desktopResourcePatterns`) for **both** transpile/dev and production/bundle builds. Team members kept discovering missing resources because every new non-TS file in `src/` required manually adding its pattern. diff --git a/build/gulpfile.reh.ts b/build/gulpfile.reh.ts index f0968a51bdc2..aaf9b8a61856 100644 --- a/build/gulpfile.reh.ts +++ b/build/gulpfile.reh.ts @@ -11,6 +11,7 @@ import { getVersion } from './lib/getVersion.ts'; import * as task from './lib/gulp/task.ts'; import * as optimize from './lib/optimize.ts'; import { inlineMeta } from './lib/inlineMeta.ts'; +import { computeNLSMetadataHash } from './lib/nlsMetadata.ts'; import product from '../product.json' with { type: 'json' }; import { getProductionDependencies } from './lib/dependencies.ts'; import { readISODate } from './lib/date.ts'; @@ -412,6 +413,7 @@ function packageTask(type: string, platform: string, arch: string, sourceFolderN const productJsonStream = gulp.src(['product.json'], { base: '.' }) .pipe(jsonEditor((json: Record) => { json.commit = commit; + json.nlsMetadataHash = computeNLSMetadataHash(path.join(REPO_ROOT, sourceFolderName), commit); json.date = readISODate(sourceFolderName); json.version = version; // Stamp agentSdks from the per-platform results file produced diff --git a/build/gulpfile.vscode.ts b/build/gulpfile.vscode.ts index 43cf08bbaa3e..c64792442192 100644 --- a/build/gulpfile.vscode.ts +++ b/build/gulpfile.vscode.ts @@ -16,6 +16,7 @@ import * as task from './lib/gulp/task.ts'; import buildfile from './buildfile.ts'; import * as optimize from './lib/optimize.ts'; import { inlineMeta } from './lib/inlineMeta.ts'; +import { computeNLSMetadataHash } from './lib/nlsMetadata.ts'; import packageJson from '../package.json' with { type: 'json' }; import product from '../product.json' with { type: 'json' }; import * as crypto from 'crypto'; @@ -336,6 +337,7 @@ function packageTask(platform: string, arch: string, sourceFolderName: string, d const productJsonStream = gulp.src(['product.json'], { base: '.' }) .pipe(jsonEditor((json: Record) => { json.commit = commit; + json.nlsMetadataHash = computeNLSMetadataHash(path.join(import.meta.dirname, '..', out), commit); json.date = readISODate(out); json.checksums = checksums; json.version = version; diff --git a/build/lib/nlsMetadata.ts b/build/lib/nlsMetadata.ts new file mode 100644 index 000000000000..0a06b6339e42 --- /dev/null +++ b/build/lib/nlsMetadata.ts @@ -0,0 +1,18 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { createHash } from 'crypto'; +import { readFileSync } from 'fs'; +import { join } from 'path'; + +export function computeNLSMetadataHash(nlsMetadataPath: string, commit: string | undefined): string { + return createHash('sha256') + .update(commit ?? '') + .update('\0') + .update(readFileSync(join(nlsMetadataPath, 'nls.keys.json'))) + .update('\0') + .update(readFileSync(join(nlsMetadataPath, 'nls.messages.json'))) + .digest('hex'); +} diff --git a/build/lib/test/nlsMetadata.test.ts b/build/lib/test/nlsMetadata.test.ts new file mode 100644 index 000000000000..356cd3503bfb --- /dev/null +++ b/build/lib/test/nlsMetadata.test.ts @@ -0,0 +1,63 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; +import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'fs'; +import { tmpdir } from 'os'; +import { join } from 'path'; +import { afterEach, beforeEach, suite, test } from 'node:test'; +import { computeNLSMetadataHash } from '../nlsMetadata.ts'; + +suite('NLS metadata identity', () => { + let directory: string; + + beforeEach(() => { + directory = mkdtempSync(join(tmpdir(), 'vscode-nls-metadata-')); + }); + + afterEach(() => { + rmSync(directory, { recursive: true, force: true }); + }); + + function writeMetadata(name: string, keys: Array<[string, string[]]>, messages: string[]): string { + const metadataPath = join(directory, name); + mkdirSync(metadataPath); + writeFileSync(join(metadataPath, 'nls.keys.json'), JSON.stringify(keys)); + writeFileSync(join(metadataPath, 'nls.messages.json'), JSON.stringify(messages)); + return metadataPath; + } + + test('identical tables have the same identity regardless of output location', () => { + const first = writeMetadata('first', [['module', ['key']]], ['Message']); + const second = writeMetadata('second', [['module', ['key']]], ['Message']); + const hash = computeNLSMetadataHash(first, 'commit'); + + assert.deepStrictEqual({ + hashLength: hash.length, + sameIdentity: hash === computeNLSMetadataHash(second, 'commit') + }, { + hashLength: 64, + sameIdentity: true + }); + }); + + test('the identity includes key order, default messages and commit', () => { + const original = writeMetadata('original', [['module', ['first', 'second']]], ['Same', 'Same']); + const reordered = writeMetadata('reordered', [['module', ['second', 'first']]], ['Same', 'Same']); + const changedMessages = writeMetadata('changed', [['module', ['first', 'second']]], ['Changed', 'Same']); + const identities = [ + computeNLSMetadataHash(original, 'commit'), + computeNLSMetadataHash(reordered, 'commit'), + computeNLSMetadataHash(changedMessages, 'commit'), + computeNLSMetadataHash(original, 'another-commit') + ]; + + assert.strictEqual(new Set(identities).size, identities.length); + }); + + test('missing metadata fails packaging instead of producing a shared fallback identity', () => { + assert.throws(() => computeNLSMetadataHash(directory, 'commit'), { code: 'ENOENT' }); + }); +}); diff --git a/src/cli.ts b/src/cli.ts index b643e346b08e..b86e22fddc84 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -10,7 +10,7 @@ import { resolveNLSConfiguration } from './vs/base/node/nls.js'; import { product } from './bootstrap-meta.js'; // NLS -const nlsConfiguration = await resolveNLSConfiguration({ userLocale: 'en', osLocale: 'en', commit: product.commit, userDataPath: '', nlsMetadataPath: import.meta.dirname }); +const nlsConfiguration = await resolveNLSConfiguration({ userLocale: 'en', osLocale: 'en', commit: product.commit, nlsMetadataHash: product.nlsMetadataHash, userDataPath: '', nlsMetadataPath: import.meta.dirname }); process.env['VSCODE_NLS_CONFIG'] = JSON.stringify(nlsConfiguration); // required for `bootstrap-esm` to pick up NLS messages // Enable portable support diff --git a/src/main.ts b/src/main.ts index 085290ba0d7e..7422ebfbf76d 100644 --- a/src/main.ts +++ b/src/main.ts @@ -133,6 +133,7 @@ if (userLocale) { userLocale, osLocale, commit: product.commit, + nlsMetadataHash: product.nlsMetadataHash, userDataPath, nlsMetadataPath: import.meta.dirname }); @@ -715,6 +716,7 @@ async function resolveNlsConfiguration(): Promise { userLocale, osLocale, commit: product.commit, + nlsMetadataHash: product.nlsMetadataHash, userDataPath, nlsMetadataPath: import.meta.dirname }); diff --git a/src/server-cli.ts b/src/server-cli.ts index e263205cb792..cab86ceef106 100644 --- a/src/server-cli.ts +++ b/src/server-cli.ts @@ -11,7 +11,7 @@ import { resolveNLSConfiguration } from './vs/base/node/nls.js'; import { product } from './bootstrap-meta.js'; // NLS -const nlsConfiguration = await resolveNLSConfiguration({ userLocale: 'en', osLocale: 'en', commit: product.commit, userDataPath: '', nlsMetadataPath: import.meta.dirname }); +const nlsConfiguration = await resolveNLSConfiguration({ userLocale: 'en', osLocale: 'en', commit: product.commit, nlsMetadataHash: product.nlsMetadataHash, userDataPath: '', nlsMetadataPath: import.meta.dirname }); process.env['VSCODE_NLS_CONFIG'] = JSON.stringify(nlsConfiguration); // required for `bootstrap-esm` to pick up NLS messages if (process.env['VSCODE_DEV']) { diff --git a/src/server-main.ts b/src/server-main.ts index 08593e461828..435631a6a73b 100644 --- a/src/server-main.ts +++ b/src/server-main.ts @@ -43,7 +43,7 @@ const extensionInstallArgs = ['install-extension', 'install-builtin-extension', const shouldSpawnCli = parsedArgs.help || parsedArgs.version || extensionLookupArgs.some(a => !!parsedArgs[a]) || (extensionInstallArgs.some(a => !!parsedArgs[a]) && !parsedArgs['start-server']); -const nlsConfiguration = await resolveNLSConfiguration({ userLocale: 'en', osLocale: 'en', commit: product.commit, userDataPath: '', nlsMetadataPath: import.meta.dirname }); +const nlsConfiguration = await resolveNLSConfiguration({ userLocale: 'en', osLocale: 'en', commit: product.commit, nlsMetadataHash: product.nlsMetadataHash, userDataPath: '', nlsMetadataPath: import.meta.dirname }); if (shouldSpawnCli) { loadCode(nlsConfiguration).then((mod) => { diff --git a/src/vs/base/common/product.ts b/src/vs/base/common/product.ts index caa0b4ab95c0..c759062d3f83 100644 --- a/src/vs/base/common/product.ts +++ b/src/vs/base/common/product.ts @@ -131,6 +131,8 @@ export interface IProductConfiguration { readonly webviewContentExternalBaseUrlTemplate?: string; readonly target?: string; readonly nlsCoreBaseUrl?: string; + /** Build-time hash of the commit and core NLS tables used to identify translated-message caches. */ + readonly nlsMetadataHash?: string; readonly settingsSearchBuildId?: number; readonly settingsSearchUrl?: string; diff --git a/src/vs/base/node/nls.ts b/src/vs/base/node/nls.ts index 0bb75a11b81b..2015d2e098ba 100644 --- a/src/vs/base/node/nls.ts +++ b/src/vs/base/node/nls.ts @@ -16,6 +16,11 @@ export interface IResolveNLSConfigurationContext { */ readonly nlsMetadataPath: string; + /** + * Precomputed identity of the commit and NLS tables, supplied by packaged products. + */ + readonly nlsMetadataHash?: string; + /** * Path to the user data directory. Used as a cache for * language packs converted to the format we need. @@ -39,7 +44,7 @@ export interface IResolveNLSConfigurationContext { readonly osLocale: string; } -export async function resolveNLSConfiguration({ userLocale, osLocale, userDataPath, commit, nlsMetadataPath }: IResolveNLSConfigurationContext): Promise { +export async function resolveNLSConfiguration({ userLocale, osLocale, userDataPath, commit, nlsMetadataPath, nlsMetadataHash }: IResolveNLSConfigurationContext): Promise { mark('code/willGenerateNls'); if ( @@ -77,7 +82,7 @@ export async function resolveNLSConfiguration({ userLocale, osLocale, userDataPa const languagePackId = `${languagePack.hash}.${resolvedLanguage}`; const globalLanguagePackCachePath = join(userDataPath, 'clp', languagePackId); - const commitLanguagePackCachePath = join(globalLanguagePackCachePath, commit); + const commitLanguagePackCachePath = join(globalLanguagePackCachePath, nlsMetadataHash ?? commit); const languagePackMessagesFile = join(commitLanguagePackCachePath, 'nls.messages.json'); const translationsConfigFile = join(globalLanguagePackCachePath, 'tcf.json'); const languagePackCorruptMarkerFile = join(globalLanguagePackCachePath, 'corrupted.info'); diff --git a/src/vs/base/test/node/nls.test.ts b/src/vs/base/test/node/nls.test.ts new file mode 100644 index 000000000000..1d84ce845f86 --- /dev/null +++ b/src/vs/base/test/node/nls.test.ts @@ -0,0 +1,232 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; +import { createHash } from 'crypto'; +import { promises } from 'fs'; +import { tmpdir } from 'os'; +import { ILanguagePacks } from '../../../nls.js'; +import { join } from '../../common/path.js'; +import { IResolveNLSConfigurationContext, resolveNLSConfiguration } from '../../node/nls.js'; +import { Promises } from '../../node/pfs.js'; +import { ensureNoDisposablesAreLeakedInTestSuite } from '../common/utils.js'; +import { getRandomTestPath } from './testUtils.js'; + +type NLSMetadata = Pick; + +suite('NLS configuration', () => { + ensureNoDisposablesAreLeakedInTestSuite(); + + const commit = 'test-commit'; + const languagePackId = 'test-language-pack.de'; + let testDir: string; + let userDataPath: string; + let vscodeDev: string | undefined; + + setup(async () => { + vscodeDev = process.env['VSCODE_DEV']; + delete process.env['VSCODE_DEV']; + + testDir = getRandomTestPath(tmpdir(), 'vsctests', 'nls'); + userDataPath = join(testDir, 'user'); + await promises.mkdir(userDataPath, { recursive: true }); + + const translationsFile = join(testDir, 'main.i18n.json'); + const languagePacks: ILanguagePacks = { + de: { + hash: 'test-language-pack', + label: 'Deutsch', + extensions: [], + translations: { vscode: translationsFile } + } + }; + await Promise.all([ + promises.writeFile(join(userDataPath, 'languagepacks.json'), JSON.stringify(languagePacks)), + promises.writeFile(translationsFile, JSON.stringify({ + contents: { + 'vs/base/test': { first: 'Erste', second: 'Zweite' }, + 'vs/workbench/api/common/extHostLogService': { remote: 'Entfernt' } + } + })) + ]); + }); + + teardown(async () => { + if (vscodeDev === undefined) { + delete process.env['VSCODE_DEV']; + } else { + process.env['VSCODE_DEV'] = vscodeDev; + } + await Promises.rm(testDir); + }); + + async function writeMetadata(name: string, keys: Array<[string, string[]]>, messages: string[]): Promise { + const nlsMetadataPath = join(testDir, name); + await promises.mkdir(nlsMetadataPath, { recursive: true }); + await Promise.all([ + promises.writeFile(join(nlsMetadataPath, 'nls.keys.json'), JSON.stringify(keys)), + promises.writeFile(join(nlsMetadataPath, 'nls.messages.json'), JSON.stringify(messages)) + ]); + return { + nlsMetadataPath, + nlsMetadataHash: createHash('sha256').update(JSON.stringify({ commit, keys, messages })).digest('hex') + }; + } + + async function resolveMessages(metadata: NLSMetadata) { + const configuration = await resolveNLSConfiguration({ + userLocale: 'de', + osLocale: 'de', + userDataPath, + commit, + ...metadata + }); + assert.ok(configuration.languagePack); + const messages: string[] = JSON.parse(await promises.readFile(configuration.languagePack.messagesFile, 'utf8')); + return { ...configuration.languagePack, messages }; + } + + test('switches between server and server-web tables at the same commit', async () => { + const serverMetadata = await writeMetadata('server', [ + ['vs/workbench/api/common/extHostLogService', ['remote']] + ], ['Remote']); + const serverWebMetadata = await writeMetadata('server-web', [ + ['vs/base/test', ['first', 'missing']], + ['vs/workbench/api/common/extHostLogService', ['remote']] + ], ['First', 'Fallback', 'Remote']); + + const server = await resolveMessages(serverMetadata); + const serverWeb = await resolveMessages(serverWebMetadata); + const serverAgain = await resolveMessages(serverMetadata); + + assert.deepStrictEqual({ + messages: [server.messages, serverWeb.messages, serverAgain.messages], + distinctTargetCaches: server.messagesFile !== serverWeb.messagesFile, + reusedServerCache: server.messagesFile === serverAgain.messagesFile + }, { + messages: [['Entfernt'], ['Erste', 'Fallback', 'Entfernt'], ['Entfernt']], + distinctTargetCaches: true, + reusedServerCache: true + }); + }); + + test('distinguishes key order when the default messages are identical', async () => { + const firstMetadata = await writeMetadata('first', [['vs/base/test', ['first', 'second']]], ['Same', 'Same']); + const secondMetadata = await writeMetadata('second', [['vs/base/test', ['second', 'first']]], ['Same', 'Same']); + + const first = await resolveMessages(firstMetadata); + const second = await resolveMessages(secondMetadata); + + assert.deepStrictEqual([first.messages, second.messages], [['Erste', 'Zweite'], ['Zweite', 'Erste']]); + }); + + test('refreshes fallback messages after rebuilding at the same commit', async () => { + const originalMetadata = await writeMetadata('server', [['vs/base/test', ['missing']]], ['Original']); + const original = await resolveMessages(originalMetadata); + const updatedMetadata = await writeMetadata('server', [['vs/base/test', ['missing']]], ['Updated']); + const updated = await resolveMessages(updatedMetadata); + + assert.deepStrictEqual([original.messages, updated.messages], [['Original'], ['Updated']]); + }); + + test('reuses the cache for identical tables in different locations', async () => { + const firstMetadata = await writeMetadata('first', [['vs/base/test', ['first']]], ['First']); + const secondMetadata = await writeMetadata('second', [['vs/base/test', ['first']]], ['First']); + const first = await resolveMessages(firstMetadata); + await promises.writeFile(first.messagesFile, JSON.stringify(['Cached translation'])); + const second = await resolveMessages(secondMetadata); + + assert.deepStrictEqual({ + sameCache: first.messagesFile === second.messagesFile, + messages: second.messages + }, { + sameCache: true, + messages: ['Cached translation'] + }); + }); + + test('a cache hit does not read the NLS tables', async () => { + const metadata = await writeMetadata('server', [['vs/base/test', ['first']]], ['First']); + const first = await resolveMessages(metadata); + await Promise.all([ + promises.unlink(join(metadata.nlsMetadataPath, 'nls.keys.json')), + promises.unlink(join(metadata.nlsMetadataPath, 'nls.messages.json')) + ]); + const cached = await resolveMessages(metadata); + + assert.deepStrictEqual({ + sameCache: first.messagesFile === cached.messagesFile, + messages: cached.messages + }, { + sameCache: true, + messages: ['Erste'] + }); + }); + + test('does not reuse or overwrite a legacy commit-only cache', async () => { + const legacyCachePath = join(userDataPath, 'clp', languagePackId, commit); + await promises.mkdir(legacyCachePath, { recursive: true }); + const legacyMessagesFile = join(legacyCachePath, 'nls.messages.json'); + await promises.writeFile(legacyMessagesFile, JSON.stringify(['Legacy translation'])); + const metadata = await writeMetadata('server', [['vs/base/test', ['first']]], ['First']); + const result = await resolveMessages(metadata); + + assert.deepStrictEqual({ + messages: result.messages, + separateCache: result.messagesFile !== legacyMessagesFile, + legacy: JSON.parse(await promises.readFile(legacyMessagesFile, 'utf8')) + }, { + messages: ['Erste'], + separateCache: true, + legacy: ['Legacy translation'] + }); + }); + + test('preserves the legacy cache path for products without an identity', async () => { + const metadata = await writeMetadata('server', [['vs/base/test', ['first']]], ['First']); + const result = await resolveMessages({ nlsMetadataPath: metadata.nlsMetadataPath }); + + assert.deepStrictEqual({ + messages: result.messages, + messagesFile: result.messagesFile + }, { + messages: ['Erste'], + messagesFile: join(userDataPath, 'clp', languagePackId, commit, 'nls.messages.json') + }); + }); + + test('regenerates all table caches after a corruption marker', async () => { + const firstMetadata = await writeMetadata('first', [['vs/base/test', ['first']]], ['First']); + const secondMetadata = await writeMetadata('second', [['vs/base/test', ['second']]], ['Second']); + const first = await resolveMessages(firstMetadata); + const second = await resolveMessages(secondMetadata); + await promises.writeFile(first.corruptMarkerFile, 'corrupt'); + const regenerated = await resolveMessages(firstMetadata); + + assert.deepStrictEqual({ + messages: regenerated.messages, + secondCacheExists: await Promises.exists(second.messagesFile), + translationsConfigExists: await Promises.exists(regenerated.translationsConfigFile) + }, { + messages: ['Erste'], + secondCacheExists: false, + translationsConfigExists: true + }); + }); + + test('does not require NLS metadata for the default locale', async () => { + const nlsMetadataPath = join(testDir, 'missing'); + const result = await resolveNLSConfiguration({ userLocale: 'en', osLocale: 'en', userDataPath, commit, nlsMetadataPath }); + + assert.deepStrictEqual(result, { + userLocale: 'en', + osLocale: 'en', + resolvedLanguage: 'en', + defaultMessagesFile: join(nlsMetadataPath, 'nls.messages.json'), + locale: 'en', + availableLanguages: {} + }); + }); +}); diff --git a/src/vs/server/node/remoteLanguagePacks.ts b/src/vs/server/node/remoteLanguagePacks.ts index 1e6db87fb531..ace2c93861dd 100644 --- a/src/vs/server/node/remoteLanguagePacks.ts +++ b/src/vs/server/node/remoteLanguagePacks.ts @@ -31,7 +31,7 @@ export async function getNLSConfiguration(language: string, userDataPath: string const cacheKey = `${language}||${userDataPath}`; let result = nlsConfigurationCache.get(cacheKey); if (!result) { - result = resolveNLSConfiguration({ userLocale: language, osLocale: language, commit: product.commit, userDataPath, nlsMetadataPath }); + result = resolveNLSConfiguration({ userLocale: language, osLocale: language, commit: product.commit, nlsMetadataHash: product.nlsMetadataHash, userDataPath, nlsMetadataPath }); nlsConfigurationCache.set(cacheKey, result); }