Fetch nightly translations during build (#13724)

Co-authored-by: Bram Kragten <mail@bramkragten.nl>

build-scripts/.eslintrc

@@ -1,7 +0,0 @@
-{
-  "rules": {
-    "import/no-extraneous-dependencies": 0,
-    "no-restricted-syntax": 0,
-    "no-console": 0
-  }
-}

build-scripts/.eslintrc.json

@@ -1,7 +1,12 @@
 {
   "extends": "../.eslintrc.json",
   "rules": {
-    "import/no-extraneous-dependencies": 0,
-    "global-require": 0
+    "no-console": "off",
+    "import/no-extraneous-dependencies": "off",
+    "import/extensions": "off",
+    "import/no-dynamic-require": "off",
+    "global-require": "off",
+    "@typescript-eslint/no-var-requires": "off",
+    "prefer-arrow-callback": "off"
   }
 }

build-scripts/babel-plugins/inline-constants-plugin.js

@@ -1,4 +1,3 @@
-/* eslint-disable @typescript-eslint/no-var-requires */
 const path = require("path");
 
 // Currently only supports CommonJS modules, as require is synchronous. `import` would need babel running asynchronous.
@@ -29,7 +28,6 @@ module.exports = function inlineConstants(babel, options, cwd) {
       const absolute = module.startsWith(".")
         ? require.resolve(module, { paths: [cwd] })
         : module;
-      // eslint-disable-next-line import/no-dynamic-require
       return [absolute, require(absolute)];
     })
   );

build-scripts/bundle.js

@@ -1,9 +1,9 @@
-/* eslint-disable @typescript-eslint/no-var-requires */
 const path = require("path");
 const env = require("./env.js");
 const paths = require("./paths.js");
 
 // Files from NPM Packages that should not be imported
+// eslint-disable-next-line unused-imports/no-unused-vars
 module.exports.ignorePackages = ({ latestBuild }) => [
   // Part of yaml.js and only used for !!js functions that we don't use
   require.resolve("esprima"),

build-scripts/env.js

@@ -1,4 +1,3 @@
-/* eslint-disable @typescript-eslint/no-var-requires */
 const fs = require("fs");
 const path = require("path");
 const paths = require("./paths.js");

build-scripts/gulp/entry-html.js

@@ -1,6 +1,4 @@
 // Tasks to generate entry HTML
-/* eslint-disable import/no-dynamic-require */
-/* eslint-disable global-require */
 const gulp = require("gulp");
 const fs = require("fs-extra");
 const path = require("path");
@@ -91,7 +89,9 @@ gulp.task("gen-pages-prod", (done) => {
 });
 
 gulp.task("gen-index-app-dev", (done) => {
-  let latestAppJS, latestCoreJS, latestCustomPanelJS;
+  let latestAppJS;
+  let latestCoreJS;
+  let latestCustomPanelJS;
 
   if (env.useWDS()) {
     latestAppJS = "http://localhost:8000/src/entrypoints/app.ts";

build-scripts/gulp/fetch-nightly_translations.js

@@ -0,0 +1,170 @@
+// Task to download the latest Lokalise translations from the nightly workflow artifacts
+
+const fs = require("fs/promises");
+const path = require("path");
+const process = require("process");
+const del = require("del");
+const gulp = require("gulp");
+const jszip = require("jszip");
+const tar = require("tar");
+const { Octokit } = require("@octokit/rest");
+const { createOAuthDeviceAuth } = require("@octokit/auth-oauth-device");
+
+const MAX_AGE = 24; // hours
+const OWNER = "home-assistant";
+const REPO = "frontend";
+const WORKFLOW_NAME = "nightly.yaml";
+const ARTIFACT_NAME = "translations";
+const CLIENT_ID = "Iv1.3914e28cb27834d1";
+const EXTRACT_DIR = "translations";
+const TOKEN_FILE = path.join(EXTRACT_DIR, "token.json");
+const ARTIFACT_FILE = path.join(EXTRACT_DIR, "artifact.json");
+
+let allowTokenSetup = false;
+gulp.task("allow-setup-fetch-nightly-translations", (done) => {
+  allowTokenSetup = true;
+  done();
+});
+
+gulp.task("fetch-nightly-translations", async function () {
+  // Skip all when environment flag is set (assumes translations are already in place)
+  if (process.env?.SKIP_FETCH_NIGHTLY_TRANSLATIONS) {
+    console.log("Skipping fetch due to environment signal");
+    return;
+  }
+
+  // Read current translations artifact info if it exists,
+  // and stop if they are not old enough
+  let currentArtifact;
+  try {
+    currentArtifact = JSON.parse(await fs.readFile(ARTIFACT_FILE, "utf-8"));
+    const currentAge =
+      (Date.now() - Date.parse(currentArtifact.created_at)) / 3600000;
+    if (currentAge < MAX_AGE) {
+      console.log(
+        "Keeping current translations (only %s hours old)",
+        currentAge.toFixed(1)
+      );
+      return;
+    }
+  } catch {
+    currentArtifact = null;
+  }
+
+  // To store file writing promises
+  const createExtractDir = fs.mkdir(EXTRACT_DIR, { recursive: true });
+  const writings = [];
+
+  // Authenticate to GitHub using GitHub action token if it exists,
+  // otherwise look for a saved user token or generate a new one if none
+  let tokenAuth;
+  if (process.env.GITHUB_TOKEN) {
+    tokenAuth = { token: process.env.GITHUB_TOKEN };
+  } else {
+    try {
+      tokenAuth = JSON.parse(await fs.readFile(TOKEN_FILE, "utf-8"));
+    } catch {
+      if (!allowTokenSetup) {
+        console.log("No token found so  build wil continue with English only");
+        return;
+      }
+      const auth = createOAuthDeviceAuth({
+        clientType: "github-app",
+        clientId: CLIENT_ID,
+        onVerification: (verification) => {
+          console.log(
+            "Task needs to authenticate to GitHub to fetch the translations from nightly workflow\n" +
+              "Please go to %s to authorize this task\n" +
+              "\nEnter user code: %s\n\n" +
+              "This code will expire in %s minutes\n" +
+              "Task will automatically continue after authorization and token will be saved for future use",
+            verification.verification_uri,
+            verification.user_code,
+            (verification.expires_in / 60).toFixed(0)
+          );
+        },
+      });
+      tokenAuth = await auth({ type: "oauth" });
+      writings.push(
+        createExtractDir.then(
+          fs.writeFile(TOKEN_FILE, JSON.stringify(tokenAuth, null, 2))
+        )
+      );
+    }
+  }
+
+  // Authenticate with token and request workflow runs from GitHub
+  console.log("Fetching new translations...");
+  const octokit = new Octokit({
+    userAgent: "Fetch Nightly Translations",
+    auth: tokenAuth.token,
+  });
+
+  const workflowRunsResponse = await octokit.rest.actions.listWorkflowRuns({
+    owner: OWNER,
+    repo: REPO,
+    workflow_id: WORKFLOW_NAME,
+    status: "success",
+    event: "schedule",
+    per_page: 1,
+    exclude_pull_requests: true,
+  });
+  if (workflowRunsResponse.data.total_count === 0) {
+    throw Error("No successful nightly workflow runs found");
+  }
+  const latestNightlyRun = workflowRunsResponse.data.workflow_runs[0];
+
+  // Stop if current is already the latest, otherwise Find the translations artifact
+  if (currentArtifact?.workflow_run.id === latestNightlyRun.id) {
+    console.log("Stopping because current translations are still the latest");
+    return;
+  }
+  const latestArtifact = (
+    await octokit.actions.listWorkflowRunArtifacts({
+      owner: OWNER,
+      repo: REPO,
+      run_id: latestNightlyRun.id,
+    })
+  ).data.artifacts.find((artifact) => artifact.name === ARTIFACT_NAME);
+  if (!latestArtifact) {
+    throw Error("Latest nightly workflow run has no translations artifact");
+  }
+  writings.push(
+    createExtractDir.then(
+      fs.writeFile(ARTIFACT_FILE, JSON.stringify(latestArtifact, null, 2))
+    )
+  );
+
+  // Remove the current translations
+  const deleteCurrent = Promise.all(writings).then(
+    del([`${EXTRACT_DIR}/*`, `!${ARTIFACT_FILE}`, `!${TOKEN_FILE}`])
+  );
+
+  // Get the download URL and follow the redirect to download (stored as ArrayBuffer)
+  const downloadResponse = await octokit.actions.downloadArtifact({
+    owner: OWNER,
+    repo: REPO,
+    artifact_id: latestArtifact.id,
+    archive_format: "zip",
+  });
+  if (downloadResponse.status !== 200) {
+    throw Error("Failure downloading translations artifact");
+  }
+
+  // Artifact is a tarball, but GitHub adds it to a zip file
+  console.log("Unpacking downloaded translations...");
+  const zip = await jszip.loadAsync(downloadResponse.data);
+  await deleteCurrent;
+  const extractStream = zip.file(/.*/)[0].nodeStream().pipe(tar.extract());
+  await new Promise((resolve, reject) => {
+    extractStream.on("close", resolve).on("error", reject);
+  });
+});
+
+gulp.task(
+  "setup-and-fetch-nightly-translations",
+  gulp.series(
+    "allow-setup-fetch-nightly-translations",
+    "fetch-nightly-translations"
+  )
+);

build-scripts/gulp/gallery.js

@@ -1,4 +1,3 @@
-/* eslint-disable */
 // Run demo develop mode
 const gulp = require("gulp");
 const fs = require("fs");
@@ -41,7 +40,7 @@ gulp.task("gather-gallery-pages", async function gatherPages() {
     }
     processed.add(pageId);
 
-    const [category, name] = pageId.split("/", 2);
+    const [category] = pageId.split("/", 2);
 
     const demoFile = path.resolve(pageDir, `${pageId}.ts`);
     const descriptionFile = path.resolve(pageDir, `${pageId}.markdown`);

build-scripts/gulp/locale-data.js

@@ -1,5 +1,3 @@
-/* eslint-disable @typescript-eslint/no-var-requires */
-
 const del = require("del");
 const path = require("path");
 const gulp = require("gulp");

build-scripts/gulp/rollup.js

@@ -5,9 +5,9 @@ const rollup = require("rollup");
 const handler = require("serve-handler");
 const http = require("http");
 const log = require("fancy-log");
+const open = require("open");
 const rollupConfig = require("../rollup");
 const paths = require("../paths");
-const open = require("open");
 
 const bothBuilds = (createConfigFunc, params) =>
   gulp.series(
@@ -30,11 +30,11 @@ const bothBuilds = (createConfigFunc, params) =>
   );
 
 function createServer(serveOptions) {
-  const server = http.createServer((request, response) => {
-    return handler(request, response, {
+  const server = http.createServer((request, response) =>
+    handler(request, response, {
       public: serveOptions.root,
-    });
-  });
+    })
+  );
 
   server.listen(
     serveOptions.port,

build-scripts/gulp/service-worker.js

@@ -1,7 +1,5 @@
 // Generate service worker.
 // Based on manifest, create a file with the content as service_worker.js
-/* eslint-disable import/no-dynamic-require */
-/* eslint-disable global-require */
 const gulp = require("gulp");
 const path = require("path");
 const fs = require("fs-extra");

build-scripts/gulp/translations.js

@@ -1,5 +1,3 @@
-/* eslint-disable @typescript-eslint/no-var-requires */
-
 const crypto = require("crypto");
 const del = require("del");
 const path = require("path");
@@ -15,6 +13,8 @@ const { mapFiles } = require("../util");
 const env = require("../env");
 const paths = require("../paths");
 
+require("./fetch-nightly_translations");
+
 const inFrontendDir = "translations/frontend";
 const inBackendDir = "translations/backend";
 const workDir = "build/translations";
@@ -23,10 +23,13 @@ const coreDir = workDir + "/core";
 const outDir = workDir + "/output";
 let mergeBackend = false;
 
-gulp.task("translations-enable-merge-backend", (done) => {
-  mergeBackend = true;
-  done();
-});
+gulp.task(
+  "translations-enable-merge-backend",
+  gulp.parallel((done) => {
+    mergeBackend = true;
+    done();
+  }, "allow-setup-fetch-nightly-translations")
+);
 
 // Panel translations which should be split from the core translations.
 const TRANSLATION_FRAGMENTS = Object.keys(
@@ -170,17 +173,24 @@ gulp.task("build-master-translation", () => {
     .pipe(transform((data, file) => lokaliseTransform(data, data, file)))
     .pipe(
       merge({
-        fileName: "translationMaster.json",
+        fileName: "en.json",
       })
     )
-    .pipe(gulp.dest(workDir));
+    .pipe(gulp.dest(fullDir));
 });
 
 gulp.task("build-merged-translations", () =>
   gulp
-    .src([inFrontendDir + "/*.json", workDir + "/test.json"], {
-      allowEmpty: true,
-    })
+    .src(
+      [
+        inFrontendDir + "/*.json",
+        "!" + inFrontendDir + "/en.json",
+        workDir + "/test.json",
+      ],
+      {
+        allowEmpty: true,
+      }
+    )
     .pipe(transform((data, file) => lokaliseTransform(data, data, file)))
     .pipe(
       flatmap((stream, file) => {
@@ -193,7 +203,7 @@ gulp.task("build-merged-translations", () =>
         //       than a base translation + region.
         const tr = path.basename(file.history[0], ".json");
         const subtags = tr.split("-");
-        const src = [workDir + "/translationMaster.json"];
+        const src = [fullDir + "/en.json"];
         for (let i = 1; i <= subtags.length; i++) {
           const lang = subtags.slice(0, i).join("-");
           if (lang === "test") {
@@ -378,7 +388,6 @@ gulp.task("build-translation-write-metadata", () =>
           if (value.nativeName) {
             newData[key] = value;
           } else {
-            // eslint-disable-next-line no-console
             console.warn(
               `Skipping language ${key}. Native name was not translated.`
             );
@@ -411,8 +420,10 @@ gulp.task(
 gulp.task(
   "build-translations",
   gulp.series(
-    "clean-translations",
-    "ensure-translations-build-dir",
+    gulp.parallel(
+      "fetch-nightly-translations",
+      gulp.series("clean-translations", "ensure-translations-build-dir")
+    ),
     "create-translations",
     "build-translation-fingerprints",
     "build-translation-write-metadata"
@@ -422,8 +433,10 @@ gulp.task(
 gulp.task(
   "build-supervisor-translations",
   gulp.series(
-    "clean-translations",
-    "ensure-translations-build-dir",
+    gulp.parallel(
+      "fetch-nightly-translations",
+      gulp.series("clean-translations", "ensure-translations-build-dir")
+    ),
     "build-master-translation",
     "build-merged-translations",
     "build-translation-fragment-supervisor",

build-scripts/gulp/webpack.js

@@ -1,4 +1,3 @@
-/* eslint-disable @typescript-eslint/no-var-requires */
 // Tasks to run webpack.
 const fs = require("fs");
 const gulp = require("gulp");
@@ -69,7 +68,6 @@ const doneHandler = (done) => (err, stats) => {
   }
 
   if (stats.hasErrors() || stats.hasWarnings()) {
-    // eslint-disable-next-line no-console
     console.log(stats.toString("minimal"));
   }
 

build-scripts/paths.js

@@ -1,4 +1,3 @@
-/* eslint-disable @typescript-eslint/no-var-requires */
 const path = require("path");
 
 module.exports = {

build-scripts/rollup-plugins/worker-plugin.js

@@ -81,13 +81,13 @@ module.exports = function (opts = {}) {
         opts.workerRegexp.flags
       );
       if (!workerRegexp.test(code)) {
-        return;
+        return undefined;
       }
 
       const ms = new MagicString(code);
       // Reset the regexp
       workerRegexp.lastIndex = 0;
-      while (true) {
+      for (;;) {
         const match = workerRegexp.exec(code);
         if (!match) {
           break;
@@ -98,6 +98,7 @@ module.exports = function (opts = {}) {
         // Parse the optional options object
         if (match[3] && match[3].length > 0) {
           // FIXME: ooooof!
+          // eslint-disable-next-line @typescript-eslint/no-implied-eval
           optionsObject = new Function(`return ${match[3].slice(1)};`)();
         }
         delete optionsObject.type;
@@ -110,12 +111,14 @@ module.exports = function (opts = {}) {
         }
 
         // Find worker file and store it as a chunk with ID prefixed for our loader
+        // eslint-disable-next-line no-await-in-loop
         const resolvedWorkerFile = (await this.resolve(workerFile, id)).id;
         let chunkRefId;
         if (resolvedWorkerFile in refIds) {
           chunkRefId = refIds[resolvedWorkerFile];
         } else {
           this.addWatchFile(resolvedWorkerFile);
+          // eslint-disable-next-line no-await-in-loop
           const source = await getBundledWorker(
             resolvedWorkerFile,
             rollupOptions

build-scripts/rollup.js

@@ -1,4 +1,3 @@
-/* eslint-disable @typescript-eslint/no-var-requires */
 const path = require("path");
 
 const commonjs = require("@rollup/plugin-commonjs");

build-scripts/util.js

@@ -1,4 +1,3 @@
-/* eslint-disable @typescript-eslint/no-var-requires */
 const path = require("path");
 const fs = require("fs");
 

build-scripts/webpack.js

@@ -1,4 +1,3 @@
-/* eslint-disable @typescript-eslint/no-var-requires */
 const webpack = require("webpack");
 const path = require("path");
 const TerserPlugin = require("terser-webpack-plugin");
@@ -103,7 +102,6 @@ const createWebpackConfig = ({
               ? path.resolve(context, resource)
               : require.resolve(resource);
           } catch (err) {
-            // eslint-disable-next-line no-console
             console.error(
               "Error in Home Assistant ignore plugin",
               resource,