From a094a2ca2b6d8758b4aea24389c35fc4f142b715 Mon Sep 17 00:00:00 2001 From: Fedor Indutny <79877362+indutny-signal@users.noreply.github.com> Date: Mon, 3 Mar 2025 19:10:01 -0800 Subject: [PATCH] Compactify locales even more --- app/locale.ts | 56 ++++++++++++++++-- package.json | 2 +- ts/scripts/generate-compact-locales.ts | 80 +++++++++++++++++++------- 3 files changed, 111 insertions(+), 27 deletions(-) diff --git a/app/locale.ts b/app/locale.ts index 2a959da4d0..a63eef4079 100644 --- a/app/locale.ts +++ b/app/locale.ts @@ -1,8 +1,9 @@ // Copyright 2017 Signal Messenger, LLC // SPDX-License-Identifier: AGPL-3.0-only -import { join } from 'path'; -import { readFileSync } from 'fs'; +import { join } from 'node:path'; +import { readFileSync } from 'node:fs'; +import { app } from 'electron'; import { merge } from 'lodash'; import * as LocaleMatcher from '@formatjs/intl-localematcher'; import { z } from 'zod'; @@ -15,6 +16,9 @@ import type { LocalizerType } from '../ts/types/Util'; import * as Errors from '../ts/types/errors'; import { parseUnknown } from '../ts/util/schemas'; +type CompactLocaleMessagesType = ReadonlyArray; +type CompactLocaleKeysType = ReadonlyArray; + const TextInfoSchema = z.object({ direction: z.enum(['ltr', 'rtl']), }); @@ -25,6 +29,17 @@ function getLocaleMessages(locale: string): LocaleMessagesType { return JSON.parse(readFileSync(targetFile, 'utf-8')); } +function getCompactLocaleKeys(): CompactLocaleKeysType { + const targetFile = join(__dirname, '..', '_locales', 'keys.json'); + return JSON.parse(readFileSync(targetFile, 'utf-8')); +} + +function getCompactLocaleValues(locale: string): CompactLocaleMessagesType { + const targetFile = join(__dirname, '..', '_locales', locale, 'values.json'); + + return JSON.parse(readFileSync(targetFile, 'utf-8')); +} + export type LocaleDisplayNames = Record>; export type CountryDisplayNames = Record>; @@ -139,13 +154,42 @@ export function load({ logger.info(`locale: Matched locale: ${matchedLocale}`); - const matchedLocaleMessages = getLocaleMessages(matchedLocale); - const englishMessages = getLocaleMessages('en'); const localeDisplayNames = getLocaleDisplayNames(); const countryDisplayNames = getCountryDisplayNames(); - // We start with english, then overwrite that with anything present in locale - const finalMessages = merge(englishMessages, matchedLocaleMessages); + let finalMessages: LocaleMessagesType; + if (app.isPackaged) { + const matchedLocaleMessages = getCompactLocaleValues(matchedLocale); + const englishMessages = getCompactLocaleValues('en'); + const keys = getCompactLocaleKeys(); + if (matchedLocaleMessages.length !== keys.length) { + throw new Error( + `Invalid "${matchedLocale}" entry count, ` + + `${matchedLocaleMessages.length} != ${keys.length}` + ); + } + if (englishMessages.length !== keys.length) { + throw new Error( + `Invalid "en" entry count, ${englishMessages.length} != ${keys.length}` + ); + } + + // We start with english, then overwrite that with anything present in locale + finalMessages = Object.create(null); + for (const [i, key] of keys.entries()) { + finalMessages[key] = { + messageformat: + matchedLocaleMessages[i] ?? englishMessages[i] ?? undefined, + }; + } + } else { + const matchedLocaleMessages = getLocaleMessages(matchedLocale); + const englishMessages = getLocaleMessages('en'); + + // We start with english, then overwrite that with anything present in locale + finalMessages = merge(englishMessages, matchedLocaleMessages); + } + const i18n = setupI18n(matchedLocale, finalMessages, { renderEmojify: shouldNeverBeCalled, }); diff --git a/package.json b/package.json index c8f9a18795..f2bd94c9dc 100644 --- a/package.json +++ b/package.json @@ -537,7 +537,7 @@ { "from": "build/compact-locales", "to": "_locales", - "filter": "**/messages.json" + "filter": ["**/values.json", "keys.json"] }, "js/**", "libtextsecure/**", diff --git a/ts/scripts/generate-compact-locales.ts b/ts/scripts/generate-compact-locales.ts index 9a17d638f4..7187d287ac 100644 --- a/ts/scripts/generate-compact-locales.ts +++ b/ts/scripts/generate-compact-locales.ts @@ -3,6 +3,49 @@ import { readdir, mkdir, readFile, writeFile } from 'node:fs/promises'; import { join, dirname } from 'node:path'; +import pMap from 'p-map'; +import { isLocaleMessageType } from '../util/setupI18nMain'; + +async function compact({ + sourceDir, + targetDir, + locale, + keys, +}: { + sourceDir: string; + targetDir: string; + locale: string; + keys: ReadonlyArray; +}): Promise> { + const sourcePath = join(sourceDir, locale, 'messages.json'); + const targetPath = join(targetDir, locale, 'values.json'); + + await mkdir(dirname(targetPath), { recursive: true }); + + const json = JSON.parse(await readFile(sourcePath, 'utf8')); + + const result = new Array(); + for (const key of keys) { + if (json[key] == null) { + // Pull English translation, or leave blank (string was deleted) + result.push(null); + continue; + } + + const value = json[key]; + if (!isLocaleMessageType(value)) { + continue; + } + if (value.messageformat == null) { + continue; + } + result.push(value.messageformat); + } + + await writeFile(targetPath, JSON.stringify(result)); + + return keys; +} async function main(): Promise { const rootDir = join(__dirname, '..', '..'); @@ -11,30 +54,27 @@ async function main(): Promise { const locales = await readdir(sourceDir); - await Promise.all( - locales.map(async locale => { + const allKeys = await pMap( + locales, + async locale => { const sourcePath = join(sourceDir, locale, 'messages.json'); - const targetPath = join(targetDir, locale, 'messages.json'); - - await mkdir(dirname(targetPath), { recursive: true }); - const json = JSON.parse(await readFile(sourcePath, 'utf8')); - for (const value of Object.values(json)) { - const typedValue = value as { description?: string }; - delete typedValue.description; - } - delete json.smartling; + return Object.entries(json) + .filter(([, value]) => isLocaleMessageType(value)) + .map(([key]) => key); + }, + { concurrency: 10 } + ); - const entries = [...Object.entries(json)]; + // Sort keys alphabetically for better incremental updates. + const keys = Array.from(new Set(allKeys.flat())).sort(); + await mkdir(targetDir, { recursive: true }); + await writeFile(join(targetDir, 'keys.json'), JSON.stringify(keys)); - // Sort entries alphabetically for better incremental updates. - entries.sort(([a], [b]) => { - return a < b ? -1 : 1; - }); - - const result = Object.fromEntries(entries); - await writeFile(targetPath, JSON.stringify(result)); - }) + await pMap( + locales, + locale => compact({ sourceDir, targetDir, locale, keys }), + { concurrency: 10 } ); }