Compactify locales even more

app/locale.ts

@@ -1,8 +1,9 @@
 // Copyright 2017 Signal Messenger, LLC
 // SPDX-License-Identifier: AGPL-3.0-only
 
-import { join } from 'path';
-import { readFileSync } from 'fs';
+import { join } from 'node:path';
+import { readFileSync } from 'node:fs';
+import { app } from 'electron';
 import { merge } from 'lodash';
 import * as LocaleMatcher from '@formatjs/intl-localematcher';
 import { z } from 'zod';
@@ -15,6 +16,9 @@ import type { LocalizerType } from '../ts/types/Util';
 import * as Errors from '../ts/types/errors';
 import { parseUnknown } from '../ts/util/schemas';
 
+type CompactLocaleMessagesType = ReadonlyArray<string | null>;
+type CompactLocaleKeysType = ReadonlyArray<string>;
+
 const TextInfoSchema = z.object({
   direction: z.enum(['ltr', 'rtl']),
 });
@@ -25,6 +29,17 @@ function getLocaleMessages(locale: string): LocaleMessagesType {
   return JSON.parse(readFileSync(targetFile, 'utf-8'));
 }
 
+function getCompactLocaleKeys(): CompactLocaleKeysType {
+  const targetFile = join(__dirname, '..', '_locales', 'keys.json');
+  return JSON.parse(readFileSync(targetFile, 'utf-8'));
+}
+
+function getCompactLocaleValues(locale: string): CompactLocaleMessagesType {
+  const targetFile = join(__dirname, '..', '_locales', locale, 'values.json');
+
+  return JSON.parse(readFileSync(targetFile, 'utf-8'));
+}
+
 export type LocaleDisplayNames = Record<string, Record<string, string>>;
 export type CountryDisplayNames = Record<string, Record<string, string>>;
 
@@ -139,13 +154,42 @@ export function load({
 
   logger.info(`locale: Matched locale: ${matchedLocale}`);
 
-  const matchedLocaleMessages = getLocaleMessages(matchedLocale);
-  const englishMessages = getLocaleMessages('en');
   const localeDisplayNames = getLocaleDisplayNames();
   const countryDisplayNames = getCountryDisplayNames();
 
+  let finalMessages: LocaleMessagesType;
+  if (app.isPackaged) {
+    const matchedLocaleMessages = getCompactLocaleValues(matchedLocale);
+    const englishMessages = getCompactLocaleValues('en');
+    const keys = getCompactLocaleKeys();
+    if (matchedLocaleMessages.length !== keys.length) {
+      throw new Error(
+        `Invalid "${matchedLocale}" entry count, ` +
+          `${matchedLocaleMessages.length} != ${keys.length}`
+      );
+    }
+    if (englishMessages.length !== keys.length) {
+      throw new Error(
+        `Invalid "en" entry count, ${englishMessages.length} != ${keys.length}`
+      );
+    }
+
     // We start with english, then overwrite that with anything present in locale
-  const finalMessages = merge(englishMessages, matchedLocaleMessages);
+    finalMessages = Object.create(null);
+    for (const [i, key] of keys.entries()) {
+      finalMessages[key] = {
+        messageformat:
+          matchedLocaleMessages[i] ?? englishMessages[i] ?? undefined,
+      };
+    }
+  } else {
+    const matchedLocaleMessages = getLocaleMessages(matchedLocale);
+    const englishMessages = getLocaleMessages('en');
+
+    // We start with english, then overwrite that with anything present in locale
+    finalMessages = merge(englishMessages, matchedLocaleMessages);
+  }
+
   const i18n = setupI18n(matchedLocale, finalMessages, {
     renderEmojify: shouldNeverBeCalled,
   });

package.json

@@ -537,7 +537,7 @@
       {
         "from": "build/compact-locales",
         "to": "_locales",
-        "filter": "**/messages.json"
+        "filter": ["**/values.json", "keys.json"]
       },
       "js/**",
       "libtextsecure/**",

ts/scripts/generate-compact-locales.ts

@@ -3,6 +3,49 @@
 
 import { readdir, mkdir, readFile, writeFile } from 'node:fs/promises';
 import { join, dirname } from 'node:path';
+import pMap from 'p-map';
+import { isLocaleMessageType } from '../util/setupI18nMain';
+
+async function compact({
+  sourceDir,
+  targetDir,
+  locale,
+  keys,
+}: {
+  sourceDir: string;
+  targetDir: string;
+  locale: string;
+  keys: ReadonlyArray<string>;
+}): Promise<ReadonlyArray<string>> {
+  const sourcePath = join(sourceDir, locale, 'messages.json');
+  const targetPath = join(targetDir, locale, 'values.json');
+
+  await mkdir(dirname(targetPath), { recursive: true });
+
+  const json = JSON.parse(await readFile(sourcePath, 'utf8'));
+
+  const result = new Array<string | null>();
+  for (const key of keys) {
+    if (json[key] == null) {
+      // Pull English translation, or leave blank (string was deleted)
+      result.push(null);
+      continue;
+    }
+
+    const value = json[key];
+    if (!isLocaleMessageType(value)) {
+      continue;
+    }
+    if (value.messageformat == null) {
+      continue;
+    }
+    result.push(value.messageformat);
+  }
+
+  await writeFile(targetPath, JSON.stringify(result));
+
+  return keys;
+}
 
 async function main(): Promise<void> {
   const rootDir = join(__dirname, '..', '..');
@@ -11,30 +54,27 @@ async function main(): Promise<void> {
 
   const locales = await readdir(sourceDir);
 
-  await Promise.all(
-    locales.map(async locale => {
+  const allKeys = await pMap(
+    locales,
+    async locale => {
       const sourcePath = join(sourceDir, locale, 'messages.json');
-      const targetPath = join(targetDir, locale, 'messages.json');
-
-      await mkdir(dirname(targetPath), { recursive: true });
-
       const json = JSON.parse(await readFile(sourcePath, 'utf8'));
-      for (const value of Object.values(json)) {
-        const typedValue = value as { description?: string };
-        delete typedValue.description;
-      }
-      delete json.smartling;
+      return Object.entries(json)
+        .filter(([, value]) => isLocaleMessageType(value))
+        .map(([key]) => key);
+    },
+    { concurrency: 10 }
+  );
 
-      const entries = [...Object.entries(json)];
+  // Sort keys alphabetically for better incremental updates.
+  const keys = Array.from(new Set(allKeys.flat())).sort();
+  await mkdir(targetDir, { recursive: true });
+  await writeFile(join(targetDir, 'keys.json'), JSON.stringify(keys));
 
-      // Sort entries alphabetically for better incremental updates.
-      entries.sort(([a], [b]) => {
-        return a < b ? -1 : 1;
-      });
-
-      const result = Object.fromEntries(entries);
-      await writeFile(targetPath, JSON.stringify(result));
-    })
+  await pMap(
+    locales,
+    locale => compact({ sourceDir, targetDir, locale, keys }),
+    { concurrency: 10 }
   );
 }