chore: import upstream snapshot with attribution

scripts/i18n/sync-ui-keys.mjs

@@ -0,0 +1,496 @@
+#!/usr/bin/env node
+/**
+ * OmniRoute — UI i18n key sync (next-intl message catalogs).
+ *
+ * Source of truth: `src/i18n/messages/en.json`. Every other locale JSON in
+ * `src/i18n/messages/` should mirror the same key tree. This script replicates
+ * any keys that are missing in a target locale, marking them with a
+ * `__MISSING__:<english_value>` sentinel so reviewers (and the optional LLM
+ * pass below) can spot them. It never overwrites an existing translated value.
+ *
+ * Usage (driven by npm scripts in package.json):
+ *   npm run i18n:sync-ui
+ *   npm run i18n:sync-ui -- --locale=pt-BR,zh-CN
+ *   npm run i18n:sync-ui -- --dry-run
+ *   npm run i18n:sync-ui -- --translate-markers
+ *   npm run i18n:sync-ui -- --translate-markers --locale=pt-BR --concurrency=4
+ *
+ * --translate-markers calls the OmniRoute translation backend (same env vars
+ * as `run-translation.mjs`) and replaces every `__MISSING__:<en>` placeholder
+ * with a translated string. Missing env vars cause the script to fail
+ * fast — the markers stay in place for a later run.
+ *
+ * Output examples:
+ *   [i18n-ui-sync] pt-BR: +589 missing keys (589 __MISSING__, 0 translated)
+ *   [i18n-ui-sync] pt-BR: +0 missing keys (already in sync)
+ */
+
+import { promises as fs, existsSync, readFileSync } from "node:fs";
+import path from "node:path";
+import process from "node:process";
+import { fileURLToPath, pathToFileURL } from "node:url";
+
+// ----- .env loader --------------------------------------------------------
+// Loads variables from a local `.env` (gitignored) into process.env without
+// pulling dotenv as a dependency. Already-set env vars take precedence so the
+// shell / CI environment can still override.
+(function loadDotEnv() {
+  const envPath = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..", ".env");
+  if (!existsSync(envPath)) return;
+  try {
+    const raw = readFileSync(envPath, "utf8");
+    for (const rawLine of raw.split(/\r?\n/)) {
+      const line = rawLine.trim();
+      if (!line || line.startsWith("#")) continue;
+      const eq = line.indexOf("=");
+      if (eq <= 0) continue;
+      const key = line.slice(0, eq).trim();
+      if (!key || process.env[key] !== undefined) continue;
+      let value = line.slice(eq + 1);
+      if (
+        (value.startsWith('"') && value.endsWith('"')) ||
+        (value.startsWith("'") && value.endsWith("'"))
+      ) {
+        value = value.slice(1, -1);
+      }
+      process.env[key] = value;
+    }
+  } catch {
+    /* ignore — script will fall back to the requireEnv error path */
+  }
+})();
+
+const SCRIPT_DIR = path.dirname(fileURLToPath(import.meta.url));
+const ROOT = path.resolve(SCRIPT_DIR, "..", "..");
+const CONFIG_PATH = path.join(ROOT, "config", "i18n.json");
+const MESSAGES_DIR = path.join(ROOT, "src", "i18n", "messages");
+const SOURCE_LOCALE = "en";
+const PLACEHOLDER_PREFIX = "__MISSING__:";
+
+// ----- Helpers -------------------------------------------------------------
+
+function logInfo(...parts) {
+  console.log("[i18n-ui-sync]", ...parts);
+}
+function logWarn(...parts) {
+  console.warn("[i18n-ui-sync] WARN", ...parts);
+}
+function logError(...parts) {
+  console.error("[i18n-ui-sync] ERROR", ...parts);
+}
+
+function parseArgs(argv) {
+  const opts = {
+    locales: null,
+    dryRun: false,
+    translateMarkers: false,
+    concurrency: null,
+  };
+  for (const arg of argv.slice(2)) {
+    if (arg === "--dry-run" || arg === "--dryrun") opts.dryRun = true;
+    else if (arg === "--translate-markers") opts.translateMarkers = true;
+    else if (arg.startsWith("--locale=")) {
+      opts.locales = arg
+        .slice(9)
+        .split(",")
+        .map((s) => s.trim())
+        .filter(Boolean);
+    } else if (arg.startsWith("--locales=")) {
+      opts.locales = arg
+        .slice(10)
+        .split(",")
+        .map((s) => s.trim())
+        .filter(Boolean);
+    } else if (arg.startsWith("--concurrency=")) {
+      opts.concurrency = Number(arg.slice(14));
+    } else if (arg === "--help" || arg === "-h") {
+      console.log(
+        [
+          "Usage: node scripts/i18n/sync-ui-keys.mjs [options]",
+          "",
+          "  --locale=<csv>          Target locales (default: all except `en`)",
+          "  --dry-run               Report what would change, write nothing",
+          "  --translate-markers     Call the translation backend to translate every",
+          "                          __MISSING__:<en> placeholder",
+          "  --concurrency=<n>       Parallel translation requests (default: env or 4)",
+        ].join("\n")
+      );
+      process.exit(0);
+    }
+  }
+  return opts;
+}
+
+async function loadConfig() {
+  const raw = await fs.readFile(CONFIG_PATH, "utf8");
+  const cfg = JSON.parse(raw);
+  if (!cfg.default || !Array.isArray(cfg.locales)) {
+    throw new Error("config/i18n.json: invalid shape (need `default` and `locales[]`)");
+  }
+  return cfg;
+}
+
+async function loadJson(filePath) {
+  const raw = await fs.readFile(filePath, "utf8");
+  return JSON.parse(raw);
+}
+
+function isPlainObject(value) {
+  return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+
+// Defensive: reject any key that could traverse into the object prototype
+// chain when we copy/merge values across JSON trees. Our inputs are
+// authored JSON we already control, but excluding these keys is a cheap
+// safety net.
+const FORBIDDEN_KEYS = new Set(["__proto__", "prototype", "constructor"]);
+
+/**
+ * Walks the source tree key-by-key. For each leaf in `source` that is not
+ * present in `target` (or whose corresponding target path is an object when
+ * source is a leaf, etc.), copies the source value into a new merged object,
+ * prefixing scalar values with PLACEHOLDER_PREFIX. Existing translated keys
+ * are preserved verbatim.
+ *
+ * Returns a tuple: { merged, addedPaths } so the caller can report the
+ * additions and (optionally) translate them.
+ */
+function mergeMissing(source, target) {
+  const addedPaths = [];
+
+  function walk(srcNode, tgtNode, prefix) {
+    if (!isPlainObject(srcNode)) {
+      // Source is a leaf. If target is missing or shape-mismatched, insert.
+      if (tgtNode === undefined) {
+        addedPaths.push(prefix);
+        return typeof srcNode === "string" ? `${PLACEHOLDER_PREFIX}${srcNode}` : srcNode;
+      }
+      // Existing value (even if string starts with placeholder) is kept.
+      return tgtNode;
+    }
+
+    // Source is an object — produce a prototype-less object preserving source
+    // key order. Using Object.create(null) guarantees no inherited keys can
+    // leak through later lookups, and we skip any key that resolves to a
+    // built-in prototype property name as a defense in depth.
+    const out = Object.create(null);
+    for (const [key, value] of Object.entries(srcNode)) {
+      if (FORBIDDEN_KEYS.has(key)) continue;
+      const nextPrefix = prefix ? `${prefix}.${key}` : key;
+      let tgtChild;
+      if (isPlainObject(tgtNode) && Object.prototype.hasOwnProperty.call(tgtNode, key)) {
+        // Read the property via Object.entries instead of dynamic bracket
+        // access to keep static analyzers happy.
+        const entry = Object.entries(tgtNode).find(([k]) => k === key);
+        tgtChild = entry ? entry[1] : undefined;
+      }
+      out[key] = walk(value, tgtChild, nextPrefix);
+    }
+    return out;
+  }
+
+  const merged = walk(source, target, "");
+  return { merged, addedPaths };
+}
+
+function countPlaceholders(node) {
+  if (typeof node === "string") return node.startsWith(PLACEHOLDER_PREFIX) ? 1 : 0;
+  if (!isPlainObject(node)) return 0;
+  let total = 0;
+  for (const value of Object.values(node)) total += countPlaceholders(value);
+  return total;
+}
+
+// ----- Translator backend (mirrors run-translation.mjs) --------------------
+
+function requireEnv(name) {
+  const v = process.env[name];
+  if (!v || !v.trim()) {
+    throw new Error(
+      `Missing required env var: ${name}. Set it in .env (see docs/guides/I18N.md → "Translation pipeline").`
+    );
+  }
+  return v.trim();
+}
+
+function backendConfig() {
+  const apiUrl = requireEnv("OMNIROUTE_TRANSLATION_API_URL").replace(/\/$/, "");
+  const apiKey = requireEnv("OMNIROUTE_TRANSLATION_API_KEY");
+  const model = requireEnv("OMNIROUTE_TRANSLATION_MODEL");
+  const timeoutMs = Number(process.env.OMNIROUTE_TRANSLATION_TIMEOUT_MS || 60000);
+  return { apiUrl, apiKey, model, timeoutMs };
+}
+
+async function callChat(messages, { apiUrl, apiKey, model, timeoutMs }, retry = 0) {
+  const ctrl = new AbortController();
+  const timer = setTimeout(() => ctrl.abort(), timeoutMs);
+  try {
+    const res = await fetch(`${apiUrl}/chat/completions`, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${apiKey}`,
+      },
+      body: JSON.stringify({
+        model,
+        messages,
+        temperature: 0.15,
+        stream: false,
+      }),
+      signal: ctrl.signal,
+    });
+    if (!res.ok) {
+      const text = await res.text().catch(() => "");
+      const transient = res.status === 408 || res.status === 429 || res.status >= 500;
+      if (transient && retry < 1) {
+        const wait = 1500 + retry * 1500;
+        logWarn(`upstream ${res.status} — retrying after ${wait}ms`);
+        await new Promise((r) => setTimeout(r, wait));
+        return callChat(messages, { apiUrl, apiKey, model, timeoutMs }, retry + 1);
+      }
+      throw new Error(`upstream ${res.status}: ${text.slice(0, 200)}`);
+    }
+    const json = await res.json();
+    const content = json?.choices?.[0]?.message?.content;
+    if (typeof content !== "string" || !content) {
+      throw new Error("upstream returned empty content");
+    }
+    return content;
+  } catch (err) {
+    if (err?.name === "AbortError") {
+      if (retry < 1) {
+        logWarn(`timeout after ${timeoutMs}ms — retrying`);
+        return callChat(messages, { apiUrl, apiKey, model, timeoutMs }, retry + 1);
+      }
+      throw new Error(`timeout after ${timeoutMs}ms`);
+    }
+    if (
+      retry < 1 &&
+      err instanceof TypeError &&
+      /fetch failed|ECONN|ENOTFOUND|network/i.test(String(err.cause ?? err.message))
+    ) {
+      logWarn(`network error: ${err.message} — retrying`);
+      await new Promise((r) => setTimeout(r, 1500));
+      return callChat(messages, { apiUrl, apiKey, model, timeoutMs }, retry + 1);
+    }
+    throw err;
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+// Simple promise-based semaphore (avoid runtime deps).
+function createLimiter(max) {
+  let active = 0;
+  const queue = [];
+  const next = () => {
+    if (!queue.length || active >= max) return;
+    active++;
+    const { fn, resolve, reject } = queue.shift();
+    fn()
+      .then((v) => {
+        active--;
+        resolve(v);
+        next();
+      })
+      .catch((err) => {
+        active--;
+        reject(err);
+        next();
+      });
+  };
+  return (fn) =>
+    new Promise((resolve, reject) => {
+      queue.push({ fn, resolve, reject });
+      next();
+    });
+}
+
+const TRANSLATION_SYSTEM = (englishName, native) =>
+  [
+    `You are a professional translator for technical software UI strings.`,
+    `Translate the user's English UI string into ${englishName} (native: ${native}).`,
+    `Return ONLY the translated string — no quotes, no commentary, no surrounding markdown.`,
+    `Preserve placeholders such as {name}, {{count}}, %s, %d, and any HTML tags exactly.`,
+    `Do NOT translate command names (npm/git/curl/etc), code identifiers, URLs, or environment variable names.`,
+    `Keep the same casing style (Title Case stays Title Case, sentence case stays sentence case).`,
+    `Keep punctuation and trailing whitespace identical to the source.`,
+  ].join(" ");
+
+async function translateString(englishValue, localeEntry, backend) {
+  const englishName = localeEntry.english ?? localeEntry.name;
+  const native = localeEntry.native ?? localeEntry.name;
+  const messages = [
+    { role: "system", content: TRANSLATION_SYSTEM(englishName, native) },
+    { role: "user", content: englishValue },
+  ];
+  const out = await callChat(messages, backend);
+  return out.trim();
+}
+
+/**
+ * Walks a merged tree, finding every leaf that starts with PLACEHOLDER_PREFIX
+ * and replacing it with the translation produced by the backend.
+ *
+ * Translations happen with bounded concurrency. On failure, the placeholder
+ * is preserved so a later run can retry.
+ */
+async function translatePlaceholders(merged, localeEntry, backend, concurrency) {
+  const tasks = [];
+  function collect(node, parent, key) {
+    if (typeof node === "string") {
+      if (node.startsWith(PLACEHOLDER_PREFIX)) {
+        const englishValue = node.slice(PLACEHOLDER_PREFIX.length);
+        tasks.push({ parent, key, englishValue });
+      }
+      return;
+    }
+    if (!isPlainObject(node)) return;
+    for (const [k, v] of Object.entries(node)) {
+      collect(v, node, k);
+    }
+  }
+  collect(merged, null, null);
+
+  if (tasks.length === 0) return { translated: 0, failed: 0 };
+
+  const limit = createLimiter(concurrency);
+  let translated = 0;
+  let failed = 0;
+  await Promise.all(
+    tasks.map((task) =>
+      limit(async () => {
+        try {
+          const value = await translateString(task.englishValue, localeEntry, backend);
+          task.parent[task.key] = value;
+          translated++;
+        } catch (err) {
+          // Keep the __MISSING__ marker so subsequent runs can retry.
+          failed++;
+          logWarn(`translation failed for ${localeEntry.code}: ${err.message}`);
+        }
+      })
+    )
+  );
+  return { translated, failed };
+}
+
+// ----- Main ----------------------------------------------------------------
+
+async function processLocale(locale, source, config, opts, backend) {
+  const localePath = path.join(MESSAGES_DIR, `${locale}.json`);
+  let target = {};
+  if (existsSync(localePath)) {
+    try {
+      target = await loadJson(localePath);
+    } catch (err) {
+      logWarn(`${locale}: failed to parse existing JSON — starting fresh (${err.message})`);
+      target = {};
+    }
+  } else {
+    logWarn(`${locale}: messages file did not exist — creating it`);
+  }
+
+  const { merged, addedPaths } = mergeMissing(source, target);
+  const placeholderCountBefore = countPlaceholders(merged);
+
+  let translateStats = { translated: 0, failed: 0 };
+  if (opts.translateMarkers && placeholderCountBefore > 0 && backend) {
+    const localeEntry = config.locales.find((l) => l.code === locale);
+    if (!localeEntry) {
+      logWarn(`${locale}: not present in config/i18n.json — skipping translation`);
+    } else {
+      const concurrency =
+        opts.concurrency ?? Number(process.env.OMNIROUTE_TRANSLATION_CONCURRENCY || 4);
+      translateStats = await translatePlaceholders(merged, localeEntry, backend, concurrency);
+    }
+  }
+
+  const placeholderCountAfter = countPlaceholders(merged);
+  const totalMissing = addedPaths.length;
+  const stillPlaceholder = placeholderCountAfter;
+
+  const summary = `${locale}: +${totalMissing} missing keys (${stillPlaceholder} __MISSING__, ${translateStats.translated} translated${translateStats.failed ? `, ${translateStats.failed} failed` : ""})`;
+
+  if (opts.dryRun) {
+    logInfo(`[DRY] ${summary}`);
+    return { addedPaths, translated: translateStats.translated };
+  }
+
+  // Only write when something changed. (json-stable serialization)
+  const before = existsSync(localePath) ? await fs.readFile(localePath, "utf8") : "";
+  const after = JSON.stringify(merged, null, 2) + "\n";
+  if (before === after) {
+    logInfo(`${locale}: already in sync (no changes)`);
+    return { addedPaths, translated: translateStats.translated };
+  }
+  await fs.writeFile(localePath, after, "utf8");
+  logInfo(summary);
+  return { addedPaths, translated: translateStats.translated };
+}
+
+async function main() {
+  const opts = parseArgs(process.argv);
+  const config = await loadConfig();
+
+  const sourcePath = path.join(MESSAGES_DIR, `${SOURCE_LOCALE}.json`);
+  if (!existsSync(sourcePath)) {
+    throw new Error(`Source messages file not found: ${sourcePath}`);
+  }
+  const source = await loadJson(sourcePath);
+
+  // Locales = every code in config except `en`, intersected with locales that
+  // already exist on disk (so we never silently create unknown locale files).
+  const onDisk = new Set(
+    (await fs.readdir(MESSAGES_DIR)).filter((f) => f.endsWith(".json")).map((f) => f.slice(0, -5))
+  );
+
+  let targetLocales = config.locales
+    .map((l) => l.code)
+    .filter((code) => code !== SOURCE_LOCALE && onDisk.has(code));
+
+  if (opts.locales) {
+    const missingFromConfig = opts.locales.filter((c) => !config.locales.some((l) => l.code === c));
+    if (missingFromConfig.length) {
+      logWarn(`--locale contains codes not in config/i18n.json: ${missingFromConfig.join(", ")}`);
+    }
+    targetLocales = targetLocales.filter((code) => opts.locales.includes(code));
+  }
+
+  logInfo(`source: ${path.relative(ROOT, sourcePath)}`);
+  logInfo(`locales: ${targetLocales.length} (${targetLocales.join(", ")})`);
+  logInfo(
+    `dry-run: ${opts.dryRun ? "yes" : "no"}, translate-markers: ${opts.translateMarkers ? "yes" : "no"}`
+  );
+
+  let backend = null;
+  if (opts.translateMarkers && !opts.dryRun) {
+    backend = backendConfig();
+    backend.concurrency =
+      opts.concurrency ?? Number(process.env.OMNIROUTE_TRANSLATION_CONCURRENCY || 4);
+    logInfo(
+      `backend: ${backend.apiUrl} (model=${backend.model}, concurrency=${backend.concurrency}, timeout=${backend.timeoutMs}ms)`
+    );
+  }
+
+  const startMs = Date.now();
+  let totalAdded = 0;
+  let totalTranslated = 0;
+  for (const locale of targetLocales) {
+    const result = await processLocale(locale, source, config, opts, backend);
+    totalAdded += result.addedPaths.length;
+    totalTranslated += result.translated;
+  }
+  const elapsedSec = ((Date.now() - startMs) / 1000).toFixed(1);
+  logInfo(
+    `summary: locales=${targetLocales.length}, added=${totalAdded}, translated=${totalTranslated}, elapsed=${elapsedSec}s`
+  );
+}
+
+const isDirectRun = import.meta.url === pathToFileURL(process.argv[1]).href;
+if (isDirectRun) {
+  main().catch((err) => {
+    logError(err?.stack || err?.message || String(err));
+    process.exit(1);
+  });
+}