diff --git a/.github/instructions/buildNext.instructions.md b/.github/instructions/buildNext.instructions.md index 9c0d2ebff8c4..bbc91b60ce78 100644 --- a/.github/instructions/buildNext.instructions.md +++ b/.github/instructions/buildNext.instructions.md @@ -153,6 +153,10 @@ Two placeholders that need injection: **Lesson:** Don't add new output file formats that create parity differences with the old build. The old build is the reference. +### Translated Message Cache Identity + +Desktop, server, and server-web packaging compute `nlsMetadataHash` from the commit, ordered NLS keys, and default messages, and stamp it into the product configuration. Native bootstrap and remote language-pack resolution pass this identity to `resolveNLSConfiguration()`, so a cached localized startup does not read or hash the NLS tables. Products without the new field retain the legacy commit-based cache path. Different target tables get separate caches; identical tables at the same commit reuse one cache. + ### 7. Resource Copying: Transpile vs Bundle **Problem:** The new build used curated, specific resource pattern lists (e.g., `desktopResourcePatterns`) for **both** transpile/dev and production/bundle builds. Team members kept discovering missing resources because every new non-TS file in `src/` required manually adding its pattern. diff --git a/build/gulpfile.reh.ts b/build/gulpfile.reh.ts index f0968a51bdc2..aaf9b8a61856 100644 --- a/build/gulpfile.reh.ts +++ b/build/gulpfile.reh.ts @@ -11,6 +11,7 @@ import { getVersion } from './lib/getVersion.ts'; import * as task from './lib/gulp/task.ts'; import * as optimize from './lib/optimize.ts'; import { inlineMeta } from './lib/inlineMeta.ts'; +import { computeNLSMetadataHash } from './lib/nlsMetadata.ts'; import product from '../product.json' with { type: 'json' }; import { getProductionDependencies } from './lib/dependencies.ts'; import { readISODate } from './lib/date.ts'; @@ -412,6 +413,7 @@ function packageTask(type: string, platform: string, arch: string, sourceFolderN const productJsonStream = gulp.src(['product.json'], { base: '.' }) .pipe(jsonEditor((json: Record) => { json.commit = commit; + json.nlsMetadataHash = computeNLSMetadataHash(path.join(REPO_ROOT, sourceFolderName), commit); json.date = readISODate(sourceFolderName); json.version = version; // Stamp agentSdks from the per-platform results file produced diff --git a/build/gulpfile.vscode.ts b/build/gulpfile.vscode.ts index 43cf08bbaa3e..c64792442192 100644 --- a/build/gulpfile.vscode.ts +++ b/build/gulpfile.vscode.ts @@ -16,6 +16,7 @@ import * as task from './lib/gulp/task.ts'; import buildfile from './buildfile.ts'; import * as optimize from './lib/optimize.ts'; import { inlineMeta } from './lib/inlineMeta.ts'; +import { computeNLSMetadataHash } from './lib/nlsMetadata.ts'; import packageJson from '../package.json' with { type: 'json' }; import product from '../product.json' with { type: 'json' }; import * as crypto from 'crypto'; @@ -336,6 +337,7 @@ function packageTask(platform: string, arch: string, sourceFolderName: string, d const productJsonStream = gulp.src(['product.json'], { base: '.' }) .pipe(jsonEditor((json: Record) => { json.commit = commit; + json.nlsMetadataHash = computeNLSMetadataHash(path.join(import.meta.dirname, '..', out), commit); json.date = readISODate(out); json.checksums = checksums; json.version = version; diff --git a/build/lib/nlsMetadata.ts b/build/lib/nlsMetadata.ts new file mode 100644 index 000000000000..0a06b6339e42 --- /dev/null +++ b/build/lib/nlsMetadata.ts @@ -0,0 +1,18 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { createHash } from 'crypto'; +import { readFileSync } from 'fs'; +import { join } from 'path'; + +export function computeNLSMetadataHash(nlsMetadataPath: string, commit: string | undefined): string { + return createHash('sha256') + .update(commit ?? '') + .update('\0') + .update(readFileSync(join(nlsMetadataPath, 'nls.keys.json'))) + .update('\0') + .update(readFileSync(join(nlsMetadataPath, 'nls.messages.json'))) + .digest('hex'); +} diff --git a/build/lib/test/nlsMetadata.test.ts b/build/lib/test/nlsMetadata.test.ts new file mode 100644 index 000000000000..356cd3503bfb --- /dev/null +++ b/build/lib/test/nlsMetadata.test.ts @@ -0,0 +1,63 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; +import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'fs'; +import { tmpdir } from 'os'; +import { join } from 'path'; +import { afterEach, beforeEach, suite, test } from 'node:test'; +import { computeNLSMetadataHash } from '../nlsMetadata.ts'; + +suite('NLS metadata identity', () => { + let directory: string; + + beforeEach(() => { + directory = mkdtempSync(join(tmpdir(), 'vscode-nls-metadata-')); + }); + + afterEach(() => { + rmSync(directory, { recursive: true, force: true }); + }); + + function writeMetadata(name: string, keys: Array<[string, string[]]>, messages: string[]): string { + const metadataPath = join(directory, name); + mkdirSync(metadataPath); + writeFileSync(join(metadataPath, 'nls.keys.json'), JSON.stringify(keys)); + writeFileSync(join(metadataPath, 'nls.messages.json'), JSON.stringify(messages)); + return metadataPath; + } + + test('identical tables have the same identity regardless of output location', () => { + const first = writeMetadata('first', [['module', ['key']]], ['Message']); + const second = writeMetadata('second', [['module', ['key']]], ['Message']); + const hash = computeNLSMetadataHash(first, 'commit'); + + assert.deepStrictEqual({ + hashLength: hash.length, + sameIdentity: hash === computeNLSMetadataHash(second, 'commit') + }, { + hashLength: 64, + sameIdentity: true + }); + }); + + test('the identity includes key order, default messages and commit', () => { + const original = writeMetadata('original', [['module', ['first', 'second']]], ['Same', 'Same']); + const reordered = writeMetadata('reordered', [['module', ['second', 'first']]], ['Same', 'Same']); + const changedMessages = writeMetadata('changed', [['module', ['first', 'second']]], ['Changed', 'Same']); + const identities = [ + computeNLSMetadataHash(original, 'commit'), + computeNLSMetadataHash(reordered, 'commit'), + computeNLSMetadataHash(changedMessages, 'commit'), + computeNLSMetadataHash(original, 'another-commit') + ]; + + assert.strictEqual(new Set(identities).size, identities.length); + }); + + test('missing metadata fails packaging instead of producing a shared fallback identity', () => { + assert.throws(() => computeNLSMetadataHash(directory, 'commit'), { code: 'ENOENT' }); + }); +}); diff --git a/src/cli.ts b/src/cli.ts index b643e346b08e..b86e22fddc84 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -10,7 +10,7 @@ import { resolveNLSConfiguration } from './vs/base/node/nls.js'; import { product } from './bootstrap-meta.js'; // NLS -const nlsConfiguration = await resolveNLSConfiguration({ userLocale: 'en', osLocale: 'en', commit: product.commit, userDataPath: '', nlsMetadataPath: import.meta.dirname }); +const nlsConfiguration = await resolveNLSConfiguration({ userLocale: 'en', osLocale: 'en', commit: product.commit, nlsMetadataHash: product.nlsMetadataHash, userDataPath: '', nlsMetadataPath: import.meta.dirname }); process.env['VSCODE_NLS_CONFIG'] = JSON.stringify(nlsConfiguration); // required for `bootstrap-esm` to pick up NLS messages // Enable portable support diff --git a/src/main.ts b/src/main.ts index 085290ba0d7e..7422ebfbf76d 100644 --- a/src/main.ts +++ b/src/main.ts @@ -133,6 +133,7 @@ if (userLocale) { userLocale, osLocale, commit: product.commit, + nlsMetadataHash: product.nlsMetadataHash, userDataPath, nlsMetadataPath: import.meta.dirname }); @@ -715,6 +716,7 @@ async function resolveNlsConfiguration(): Promise { userLocale, osLocale, commit: product.commit, + nlsMetadataHash: product.nlsMetadataHash, userDataPath, nlsMetadataPath: import.meta.dirname }); diff --git a/src/server-cli.ts b/src/server-cli.ts index e263205cb792..cab86ceef106 100644 --- a/src/server-cli.ts +++ b/src/server-cli.ts @@ -11,7 +11,7 @@ import { resolveNLSConfiguration } from './vs/base/node/nls.js'; import { product } from './bootstrap-meta.js'; // NLS -const nlsConfiguration = await resolveNLSConfiguration({ userLocale: 'en', osLocale: 'en', commit: product.commit, userDataPath: '', nlsMetadataPath: import.meta.dirname }); +const nlsConfiguration = await resolveNLSConfiguration({ userLocale: 'en', osLocale: 'en', commit: product.commit, nlsMetadataHash: product.nlsMetadataHash, userDataPath: '', nlsMetadataPath: import.meta.dirname }); process.env['VSCODE_NLS_CONFIG'] = JSON.stringify(nlsConfiguration); // required for `bootstrap-esm` to pick up NLS messages if (process.env['VSCODE_DEV']) { diff --git a/src/server-main.ts b/src/server-main.ts index 08593e461828..435631a6a73b 100644 --- a/src/server-main.ts +++ b/src/server-main.ts @@ -43,7 +43,7 @@ const extensionInstallArgs = ['install-extension', 'install-builtin-extension', const shouldSpawnCli = parsedArgs.help || parsedArgs.version || extensionLookupArgs.some(a => !!parsedArgs[a]) || (extensionInstallArgs.some(a => !!parsedArgs[a]) && !parsedArgs['start-server']); -const nlsConfiguration = await resolveNLSConfiguration({ userLocale: 'en', osLocale: 'en', commit: product.commit, userDataPath: '', nlsMetadataPath: import.meta.dirname }); +const nlsConfiguration = await resolveNLSConfiguration({ userLocale: 'en', osLocale: 'en', commit: product.commit, nlsMetadataHash: product.nlsMetadataHash, userDataPath: '', nlsMetadataPath: import.meta.dirname }); if (shouldSpawnCli) { loadCode(nlsConfiguration).then((mod) => { diff --git a/src/vs/base/common/product.ts b/src/vs/base/common/product.ts index caa0b4ab95c0..c759062d3f83 100644 --- a/src/vs/base/common/product.ts +++ b/src/vs/base/common/product.ts @@ -131,6 +131,8 @@ export interface IProductConfiguration { readonly webviewContentExternalBaseUrlTemplate?: string; readonly target?: string; readonly nlsCoreBaseUrl?: string; + /** Build-time hash of the commit and core NLS tables used to identify translated-message caches. */ + readonly nlsMetadataHash?: string; readonly settingsSearchBuildId?: number; readonly settingsSearchUrl?: string; diff --git a/src/vs/base/node/nls.ts b/src/vs/base/node/nls.ts index 0bb75a11b81b..2015d2e098ba 100644 --- a/src/vs/base/node/nls.ts +++ b/src/vs/base/node/nls.ts @@ -16,6 +16,11 @@ export interface IResolveNLSConfigurationContext { */ readonly nlsMetadataPath: string; + /** + * Precomputed identity of the commit and NLS tables, supplied by packaged products. + */ + readonly nlsMetadataHash?: string; + /** * Path to the user data directory. Used as a cache for * language packs converted to the format we need. @@ -39,7 +44,7 @@ export interface IResolveNLSConfigurationContext { readonly osLocale: string; } -export async function resolveNLSConfiguration({ userLocale, osLocale, userDataPath, commit, nlsMetadataPath }: IResolveNLSConfigurationContext): Promise { +export async function resolveNLSConfiguration({ userLocale, osLocale, userDataPath, commit, nlsMetadataPath, nlsMetadataHash }: IResolveNLSConfigurationContext): Promise { mark('code/willGenerateNls'); if ( @@ -77,7 +82,7 @@ export async function resolveNLSConfiguration({ userLocale, osLocale, userDataPa const languagePackId = `${languagePack.hash}.${resolvedLanguage}`; const globalLanguagePackCachePath = join(userDataPath, 'clp', languagePackId); - const commitLanguagePackCachePath = join(globalLanguagePackCachePath, commit); + const commitLanguagePackCachePath = join(globalLanguagePackCachePath, nlsMetadataHash ?? commit); const languagePackMessagesFile = join(commitLanguagePackCachePath, 'nls.messages.json'); const translationsConfigFile = join(globalLanguagePackCachePath, 'tcf.json'); const languagePackCorruptMarkerFile = join(globalLanguagePackCachePath, 'corrupted.info'); diff --git a/src/vs/base/test/node/nls.test.ts b/src/vs/base/test/node/nls.test.ts new file mode 100644 index 000000000000..1d84ce845f86 --- /dev/null +++ b/src/vs/base/test/node/nls.test.ts @@ -0,0 +1,232 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import assert from 'assert'; +import { createHash } from 'crypto'; +import { promises } from 'fs'; +import { tmpdir } from 'os'; +import { ILanguagePacks } from '../../../nls.js'; +import { join } from '../../common/path.js'; +import { IResolveNLSConfigurationContext, resolveNLSConfiguration } from '../../node/nls.js'; +import { Promises } from '../../node/pfs.js'; +import { ensureNoDisposablesAreLeakedInTestSuite } from '../common/utils.js'; +import { getRandomTestPath } from './testUtils.js'; + +type NLSMetadata = Pick; + +suite('NLS configuration', () => { + ensureNoDisposablesAreLeakedInTestSuite(); + + const commit = 'test-commit'; + const languagePackId = 'test-language-pack.de'; + let testDir: string; + let userDataPath: string; + let vscodeDev: string | undefined; + + setup(async () => { + vscodeDev = process.env['VSCODE_DEV']; + delete process.env['VSCODE_DEV']; + + testDir = getRandomTestPath(tmpdir(), 'vsctests', 'nls'); + userDataPath = join(testDir, 'user'); + await promises.mkdir(userDataPath, { recursive: true }); + + const translationsFile = join(testDir, 'main.i18n.json'); + const languagePacks: ILanguagePacks = { + de: { + hash: 'test-language-pack', + label: 'Deutsch', + extensions: [], + translations: { vscode: translationsFile } + } + }; + await Promise.all([ + promises.writeFile(join(userDataPath, 'languagepacks.json'), JSON.stringify(languagePacks)), + promises.writeFile(translationsFile, JSON.stringify({ + contents: { + 'vs/base/test': { first: 'Erste', second: 'Zweite' }, + 'vs/workbench/api/common/extHostLogService': { remote: 'Entfernt' } + } + })) + ]); + }); + + teardown(async () => { + if (vscodeDev === undefined) { + delete process.env['VSCODE_DEV']; + } else { + process.env['VSCODE_DEV'] = vscodeDev; + } + await Promises.rm(testDir); + }); + + async function writeMetadata(name: string, keys: Array<[string, string[]]>, messages: string[]): Promise { + const nlsMetadataPath = join(testDir, name); + await promises.mkdir(nlsMetadataPath, { recursive: true }); + await Promise.all([ + promises.writeFile(join(nlsMetadataPath, 'nls.keys.json'), JSON.stringify(keys)), + promises.writeFile(join(nlsMetadataPath, 'nls.messages.json'), JSON.stringify(messages)) + ]); + return { + nlsMetadataPath, + nlsMetadataHash: createHash('sha256').update(JSON.stringify({ commit, keys, messages })).digest('hex') + }; + } + + async function resolveMessages(metadata: NLSMetadata) { + const configuration = await resolveNLSConfiguration({ + userLocale: 'de', + osLocale: 'de', + userDataPath, + commit, + ...metadata + }); + assert.ok(configuration.languagePack); + const messages: string[] = JSON.parse(await promises.readFile(configuration.languagePack.messagesFile, 'utf8')); + return { ...configuration.languagePack, messages }; + } + + test('switches between server and server-web tables at the same commit', async () => { + const serverMetadata = await writeMetadata('server', [ + ['vs/workbench/api/common/extHostLogService', ['remote']] + ], ['Remote']); + const serverWebMetadata = await writeMetadata('server-web', [ + ['vs/base/test', ['first', 'missing']], + ['vs/workbench/api/common/extHostLogService', ['remote']] + ], ['First', 'Fallback', 'Remote']); + + const server = await resolveMessages(serverMetadata); + const serverWeb = await resolveMessages(serverWebMetadata); + const serverAgain = await resolveMessages(serverMetadata); + + assert.deepStrictEqual({ + messages: [server.messages, serverWeb.messages, serverAgain.messages], + distinctTargetCaches: server.messagesFile !== serverWeb.messagesFile, + reusedServerCache: server.messagesFile === serverAgain.messagesFile + }, { + messages: [['Entfernt'], ['Erste', 'Fallback', 'Entfernt'], ['Entfernt']], + distinctTargetCaches: true, + reusedServerCache: true + }); + }); + + test('distinguishes key order when the default messages are identical', async () => { + const firstMetadata = await writeMetadata('first', [['vs/base/test', ['first', 'second']]], ['Same', 'Same']); + const secondMetadata = await writeMetadata('second', [['vs/base/test', ['second', 'first']]], ['Same', 'Same']); + + const first = await resolveMessages(firstMetadata); + const second = await resolveMessages(secondMetadata); + + assert.deepStrictEqual([first.messages, second.messages], [['Erste', 'Zweite'], ['Zweite', 'Erste']]); + }); + + test('refreshes fallback messages after rebuilding at the same commit', async () => { + const originalMetadata = await writeMetadata('server', [['vs/base/test', ['missing']]], ['Original']); + const original = await resolveMessages(originalMetadata); + const updatedMetadata = await writeMetadata('server', [['vs/base/test', ['missing']]], ['Updated']); + const updated = await resolveMessages(updatedMetadata); + + assert.deepStrictEqual([original.messages, updated.messages], [['Original'], ['Updated']]); + }); + + test('reuses the cache for identical tables in different locations', async () => { + const firstMetadata = await writeMetadata('first', [['vs/base/test', ['first']]], ['First']); + const secondMetadata = await writeMetadata('second', [['vs/base/test', ['first']]], ['First']); + const first = await resolveMessages(firstMetadata); + await promises.writeFile(first.messagesFile, JSON.stringify(['Cached translation'])); + const second = await resolveMessages(secondMetadata); + + assert.deepStrictEqual({ + sameCache: first.messagesFile === second.messagesFile, + messages: second.messages + }, { + sameCache: true, + messages: ['Cached translation'] + }); + }); + + test('a cache hit does not read the NLS tables', async () => { + const metadata = await writeMetadata('server', [['vs/base/test', ['first']]], ['First']); + const first = await resolveMessages(metadata); + await Promise.all([ + promises.unlink(join(metadata.nlsMetadataPath, 'nls.keys.json')), + promises.unlink(join(metadata.nlsMetadataPath, 'nls.messages.json')) + ]); + const cached = await resolveMessages(metadata); + + assert.deepStrictEqual({ + sameCache: first.messagesFile === cached.messagesFile, + messages: cached.messages + }, { + sameCache: true, + messages: ['Erste'] + }); + }); + + test('does not reuse or overwrite a legacy commit-only cache', async () => { + const legacyCachePath = join(userDataPath, 'clp', languagePackId, commit); + await promises.mkdir(legacyCachePath, { recursive: true }); + const legacyMessagesFile = join(legacyCachePath, 'nls.messages.json'); + await promises.writeFile(legacyMessagesFile, JSON.stringify(['Legacy translation'])); + const metadata = await writeMetadata('server', [['vs/base/test', ['first']]], ['First']); + const result = await resolveMessages(metadata); + + assert.deepStrictEqual({ + messages: result.messages, + separateCache: result.messagesFile !== legacyMessagesFile, + legacy: JSON.parse(await promises.readFile(legacyMessagesFile, 'utf8')) + }, { + messages: ['Erste'], + separateCache: true, + legacy: ['Legacy translation'] + }); + }); + + test('preserves the legacy cache path for products without an identity', async () => { + const metadata = await writeMetadata('server', [['vs/base/test', ['first']]], ['First']); + const result = await resolveMessages({ nlsMetadataPath: metadata.nlsMetadataPath }); + + assert.deepStrictEqual({ + messages: result.messages, + messagesFile: result.messagesFile + }, { + messages: ['Erste'], + messagesFile: join(userDataPath, 'clp', languagePackId, commit, 'nls.messages.json') + }); + }); + + test('regenerates all table caches after a corruption marker', async () => { + const firstMetadata = await writeMetadata('first', [['vs/base/test', ['first']]], ['First']); + const secondMetadata = await writeMetadata('second', [['vs/base/test', ['second']]], ['Second']); + const first = await resolveMessages(firstMetadata); + const second = await resolveMessages(secondMetadata); + await promises.writeFile(first.corruptMarkerFile, 'corrupt'); + const regenerated = await resolveMessages(firstMetadata); + + assert.deepStrictEqual({ + messages: regenerated.messages, + secondCacheExists: await Promises.exists(second.messagesFile), + translationsConfigExists: await Promises.exists(regenerated.translationsConfigFile) + }, { + messages: ['Erste'], + secondCacheExists: false, + translationsConfigExists: true + }); + }); + + test('does not require NLS metadata for the default locale', async () => { + const nlsMetadataPath = join(testDir, 'missing'); + const result = await resolveNLSConfiguration({ userLocale: 'en', osLocale: 'en', userDataPath, commit, nlsMetadataPath }); + + assert.deepStrictEqual(result, { + userLocale: 'en', + osLocale: 'en', + resolvedLanguage: 'en', + defaultMessagesFile: join(nlsMetadataPath, 'nls.messages.json'), + locale: 'en', + availableLanguages: {} + }); + }); +}); diff --git a/src/vs/server/node/remoteLanguagePacks.ts b/src/vs/server/node/remoteLanguagePacks.ts index 1e6db87fb531..ace2c93861dd 100644 --- a/src/vs/server/node/remoteLanguagePacks.ts +++ b/src/vs/server/node/remoteLanguagePacks.ts @@ -31,7 +31,7 @@ export async function getNLSConfiguration(language: string, userDataPath: string const cacheKey = `${language}||${userDataPath}`; let result = nlsConfigurationCache.get(cacheKey); if (!result) { - result = resolveNLSConfiguration({ userLocale: language, osLocale: language, commit: product.commit, userDataPath, nlsMetadataPath }); + result = resolveNLSConfiguration({ userLocale: language, osLocale: language, commit: product.commit, nlsMetadataHash: product.nlsMetadataHash, userDataPath, nlsMetadataPath }); nlsConfigurationCache.set(cacheKey, result); }