wehub-resource-sync
497 lines
18 KiB
JavaScript
Executable File
497 lines
18 KiB
JavaScript
Executable File
#!/usr/bin/env node
|
|
/**
|
|
* OmniRoute — UI i18n key sync (next-intl message catalogs).
|
|
*
|
|
* Source of truth: `src/i18n/messages/en.json`. Every other locale JSON in
|
|
* `src/i18n/messages/` should mirror the same key tree. This script replicates
|
|
* any keys that are missing in a target locale, marking them with a
|
|
* `__MISSING__:<english_value>` sentinel so reviewers (and the optional LLM
|
|
* pass below) can spot them. It never overwrites an existing translated value.
|
|
*
|
|
* Usage (driven by npm scripts in package.json):
|
|
* npm run i18n:sync-ui
|
|
* npm run i18n:sync-ui -- --locale=pt-BR,zh-CN
|
|
* npm run i18n:sync-ui -- --dry-run
|
|
* npm run i18n:sync-ui -- --translate-markers
|
|
* npm run i18n:sync-ui -- --translate-markers --locale=pt-BR --concurrency=4
|
|
*
|
|
* --translate-markers calls the OmniRoute translation backend (same env vars
|
|
* as `run-translation.mjs`) and replaces every `__MISSING__:<en>` placeholder
|
|
* with a translated string. Missing env vars cause the script to fail
|
|
* fast — the markers stay in place for a later run.
|
|
*
|
|
* Output examples:
|
|
* [i18n-ui-sync] pt-BR: +589 missing keys (589 __MISSING__, 0 translated)
|
|
* [i18n-ui-sync] pt-BR: +0 missing keys (already in sync)
|
|
*/
|
|
|
|
import { promises as fs, existsSync, readFileSync } from "node:fs";
|
|
import path from "node:path";
|
|
import process from "node:process";
|
|
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
|
|
// ----- .env loader --------------------------------------------------------
|
|
// Loads variables from a local `.env` (gitignored) into process.env without
|
|
// pulling dotenv as a dependency. Already-set env vars take precedence so the
|
|
// shell / CI environment can still override.
|
|
(function loadDotEnv() {
|
|
const envPath = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", "..", ".env");
|
|
if (!existsSync(envPath)) return;
|
|
try {
|
|
const raw = readFileSync(envPath, "utf8");
|
|
for (const rawLine of raw.split(/\r?\n/)) {
|
|
const line = rawLine.trim();
|
|
if (!line || line.startsWith("#")) continue;
|
|
const eq = line.indexOf("=");
|
|
if (eq <= 0) continue;
|
|
const key = line.slice(0, eq).trim();
|
|
if (!key || process.env[key] !== undefined) continue;
|
|
let value = line.slice(eq + 1);
|
|
if (
|
|
(value.startsWith('"') && value.endsWith('"')) ||
|
|
(value.startsWith("'") && value.endsWith("'"))
|
|
) {
|
|
value = value.slice(1, -1);
|
|
}
|
|
process.env[key] = value;
|
|
}
|
|
} catch {
|
|
/* ignore — script will fall back to the requireEnv error path */
|
|
}
|
|
})();
|
|
|
|
const SCRIPT_DIR = path.dirname(fileURLToPath(import.meta.url));
|
|
const ROOT = path.resolve(SCRIPT_DIR, "..", "..");
|
|
const CONFIG_PATH = path.join(ROOT, "config", "i18n.json");
|
|
const MESSAGES_DIR = path.join(ROOT, "src", "i18n", "messages");
|
|
const SOURCE_LOCALE = "en";
|
|
const PLACEHOLDER_PREFIX = "__MISSING__:";
|
|
|
|
// ----- Helpers -------------------------------------------------------------
|
|
|
|
function logInfo(...parts) {
|
|
console.log("[i18n-ui-sync]", ...parts);
|
|
}
|
|
function logWarn(...parts) {
|
|
console.warn("[i18n-ui-sync] WARN", ...parts);
|
|
}
|
|
function logError(...parts) {
|
|
console.error("[i18n-ui-sync] ERROR", ...parts);
|
|
}
|
|
|
|
function parseArgs(argv) {
|
|
const opts = {
|
|
locales: null,
|
|
dryRun: false,
|
|
translateMarkers: false,
|
|
concurrency: null,
|
|
};
|
|
for (const arg of argv.slice(2)) {
|
|
if (arg === "--dry-run" || arg === "--dryrun") opts.dryRun = true;
|
|
else if (arg === "--translate-markers") opts.translateMarkers = true;
|
|
else if (arg.startsWith("--locale=")) {
|
|
opts.locales = arg
|
|
.slice(9)
|
|
.split(",")
|
|
.map((s) => s.trim())
|
|
.filter(Boolean);
|
|
} else if (arg.startsWith("--locales=")) {
|
|
opts.locales = arg
|
|
.slice(10)
|
|
.split(",")
|
|
.map((s) => s.trim())
|
|
.filter(Boolean);
|
|
} else if (arg.startsWith("--concurrency=")) {
|
|
opts.concurrency = Number(arg.slice(14));
|
|
} else if (arg === "--help" || arg === "-h") {
|
|
console.log(
|
|
[
|
|
"Usage: node scripts/i18n/sync-ui-keys.mjs [options]",
|
|
"",
|
|
" --locale=<csv> Target locales (default: all except `en`)",
|
|
" --dry-run Report what would change, write nothing",
|
|
" --translate-markers Call the translation backend to translate every",
|
|
" __MISSING__:<en> placeholder",
|
|
" --concurrency=<n> Parallel translation requests (default: env or 4)",
|
|
].join("\n")
|
|
);
|
|
process.exit(0);
|
|
}
|
|
}
|
|
return opts;
|
|
}
|
|
|
|
async function loadConfig() {
|
|
const raw = await fs.readFile(CONFIG_PATH, "utf8");
|
|
const cfg = JSON.parse(raw);
|
|
if (!cfg.default || !Array.isArray(cfg.locales)) {
|
|
throw new Error("config/i18n.json: invalid shape (need `default` and `locales[]`)");
|
|
}
|
|
return cfg;
|
|
}
|
|
|
|
async function loadJson(filePath) {
|
|
const raw = await fs.readFile(filePath, "utf8");
|
|
return JSON.parse(raw);
|
|
}
|
|
|
|
function isPlainObject(value) {
|
|
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
}
|
|
|
|
// Defensive: reject any key that could traverse into the object prototype
|
|
// chain when we copy/merge values across JSON trees. Our inputs are
|
|
// authored JSON we already control, but excluding these keys is a cheap
|
|
// safety net.
|
|
const FORBIDDEN_KEYS = new Set(["__proto__", "prototype", "constructor"]);
|
|
|
|
/**
|
|
* Walks the source tree key-by-key. For each leaf in `source` that is not
|
|
* present in `target` (or whose corresponding target path is an object when
|
|
* source is a leaf, etc.), copies the source value into a new merged object,
|
|
* prefixing scalar values with PLACEHOLDER_PREFIX. Existing translated keys
|
|
* are preserved verbatim.
|
|
*
|
|
* Returns a tuple: { merged, addedPaths } so the caller can report the
|
|
* additions and (optionally) translate them.
|
|
*/
|
|
function mergeMissing(source, target) {
|
|
const addedPaths = [];
|
|
|
|
function walk(srcNode, tgtNode, prefix) {
|
|
if (!isPlainObject(srcNode)) {
|
|
// Source is a leaf. If target is missing or shape-mismatched, insert.
|
|
if (tgtNode === undefined) {
|
|
addedPaths.push(prefix);
|
|
return typeof srcNode === "string" ? `${PLACEHOLDER_PREFIX}${srcNode}` : srcNode;
|
|
}
|
|
// Existing value (even if string starts with placeholder) is kept.
|
|
return tgtNode;
|
|
}
|
|
|
|
// Source is an object — produce a prototype-less object preserving source
|
|
// key order. Using Object.create(null) guarantees no inherited keys can
|
|
// leak through later lookups, and we skip any key that resolves to a
|
|
// built-in prototype property name as a defense in depth.
|
|
const out = Object.create(null);
|
|
for (const [key, value] of Object.entries(srcNode)) {
|
|
if (FORBIDDEN_KEYS.has(key)) continue;
|
|
const nextPrefix = prefix ? `${prefix}.${key}` : key;
|
|
let tgtChild;
|
|
if (isPlainObject(tgtNode) && Object.prototype.hasOwnProperty.call(tgtNode, key)) {
|
|
// Read the property via Object.entries instead of dynamic bracket
|
|
// access to keep static analyzers happy.
|
|
const entry = Object.entries(tgtNode).find(([k]) => k === key);
|
|
tgtChild = entry ? entry[1] : undefined;
|
|
}
|
|
out[key] = walk(value, tgtChild, nextPrefix);
|
|
}
|
|
return out;
|
|
}
|
|
|
|
const merged = walk(source, target, "");
|
|
return { merged, addedPaths };
|
|
}
|
|
|
|
function countPlaceholders(node) {
|
|
if (typeof node === "string") return node.startsWith(PLACEHOLDER_PREFIX) ? 1 : 0;
|
|
if (!isPlainObject(node)) return 0;
|
|
let total = 0;
|
|
for (const value of Object.values(node)) total += countPlaceholders(value);
|
|
return total;
|
|
}
|
|
|
|
// ----- Translator backend (mirrors run-translation.mjs) --------------------
|
|
|
|
function requireEnv(name) {
|
|
const v = process.env[name];
|
|
if (!v || !v.trim()) {
|
|
throw new Error(
|
|
`Missing required env var: ${name}. Set it in .env (see docs/guides/I18N.md → "Translation pipeline").`
|
|
);
|
|
}
|
|
return v.trim();
|
|
}
|
|
|
|
function backendConfig() {
|
|
const apiUrl = requireEnv("OMNIROUTE_TRANSLATION_API_URL").replace(/\/$/, "");
|
|
const apiKey = requireEnv("OMNIROUTE_TRANSLATION_API_KEY");
|
|
const model = requireEnv("OMNIROUTE_TRANSLATION_MODEL");
|
|
const timeoutMs = Number(process.env.OMNIROUTE_TRANSLATION_TIMEOUT_MS || 60000);
|
|
return { apiUrl, apiKey, model, timeoutMs };
|
|
}
|
|
|
|
async function callChat(messages, { apiUrl, apiKey, model, timeoutMs }, retry = 0) {
|
|
const ctrl = new AbortController();
|
|
const timer = setTimeout(() => ctrl.abort(), timeoutMs);
|
|
try {
|
|
const res = await fetch(`${apiUrl}/chat/completions`, {
|
|
method: "POST",
|
|
headers: {
|
|
"Content-Type": "application/json",
|
|
Authorization: `Bearer ${apiKey}`,
|
|
},
|
|
body: JSON.stringify({
|
|
model,
|
|
messages,
|
|
temperature: 0.15,
|
|
stream: false,
|
|
}),
|
|
signal: ctrl.signal,
|
|
});
|
|
if (!res.ok) {
|
|
const text = await res.text().catch(() => "");
|
|
const transient = res.status === 408 || res.status === 429 || res.status >= 500;
|
|
if (transient && retry < 1) {
|
|
const wait = 1500 + retry * 1500;
|
|
logWarn(`upstream ${res.status} — retrying after ${wait}ms`);
|
|
await new Promise((r) => setTimeout(r, wait));
|
|
return callChat(messages, { apiUrl, apiKey, model, timeoutMs }, retry + 1);
|
|
}
|
|
throw new Error(`upstream ${res.status}: ${text.slice(0, 200)}`);
|
|
}
|
|
const json = await res.json();
|
|
const content = json?.choices?.[0]?.message?.content;
|
|
if (typeof content !== "string" || !content) {
|
|
throw new Error("upstream returned empty content");
|
|
}
|
|
return content;
|
|
} catch (err) {
|
|
if (err?.name === "AbortError") {
|
|
if (retry < 1) {
|
|
logWarn(`timeout after ${timeoutMs}ms — retrying`);
|
|
return callChat(messages, { apiUrl, apiKey, model, timeoutMs }, retry + 1);
|
|
}
|
|
throw new Error(`timeout after ${timeoutMs}ms`);
|
|
}
|
|
if (
|
|
retry < 1 &&
|
|
err instanceof TypeError &&
|
|
/fetch failed|ECONN|ENOTFOUND|network/i.test(String(err.cause ?? err.message))
|
|
) {
|
|
logWarn(`network error: ${err.message} — retrying`);
|
|
await new Promise((r) => setTimeout(r, 1500));
|
|
return callChat(messages, { apiUrl, apiKey, model, timeoutMs }, retry + 1);
|
|
}
|
|
throw err;
|
|
} finally {
|
|
clearTimeout(timer);
|
|
}
|
|
}
|
|
|
|
// Simple promise-based semaphore (avoid runtime deps).
|
|
function createLimiter(max) {
|
|
let active = 0;
|
|
const queue = [];
|
|
const next = () => {
|
|
if (!queue.length || active >= max) return;
|
|
active++;
|
|
const { fn, resolve, reject } = queue.shift();
|
|
fn()
|
|
.then((v) => {
|
|
active--;
|
|
resolve(v);
|
|
next();
|
|
})
|
|
.catch((err) => {
|
|
active--;
|
|
reject(err);
|
|
next();
|
|
});
|
|
};
|
|
return (fn) =>
|
|
new Promise((resolve, reject) => {
|
|
queue.push({ fn, resolve, reject });
|
|
next();
|
|
});
|
|
}
|
|
|
|
const TRANSLATION_SYSTEM = (englishName, native) =>
|
|
[
|
|
`You are a professional translator for technical software UI strings.`,
|
|
`Translate the user's English UI string into ${englishName} (native: ${native}).`,
|
|
`Return ONLY the translated string — no quotes, no commentary, no surrounding markdown.`,
|
|
`Preserve placeholders such as {name}, {{count}}, %s, %d, and any HTML tags exactly.`,
|
|
`Do NOT translate command names (npm/git/curl/etc), code identifiers, URLs, or environment variable names.`,
|
|
`Keep the same casing style (Title Case stays Title Case, sentence case stays sentence case).`,
|
|
`Keep punctuation and trailing whitespace identical to the source.`,
|
|
].join(" ");
|
|
|
|
async function translateString(englishValue, localeEntry, backend) {
|
|
const englishName = localeEntry.english ?? localeEntry.name;
|
|
const native = localeEntry.native ?? localeEntry.name;
|
|
const messages = [
|
|
{ role: "system", content: TRANSLATION_SYSTEM(englishName, native) },
|
|
{ role: "user", content: englishValue },
|
|
];
|
|
const out = await callChat(messages, backend);
|
|
return out.trim();
|
|
}
|
|
|
|
/**
|
|
* Walks a merged tree, finding every leaf that starts with PLACEHOLDER_PREFIX
|
|
* and replacing it with the translation produced by the backend.
|
|
*
|
|
* Translations happen with bounded concurrency. On failure, the placeholder
|
|
* is preserved so a later run can retry.
|
|
*/
|
|
async function translatePlaceholders(merged, localeEntry, backend, concurrency) {
|
|
const tasks = [];
|
|
function collect(node, parent, key) {
|
|
if (typeof node === "string") {
|
|
if (node.startsWith(PLACEHOLDER_PREFIX)) {
|
|
const englishValue = node.slice(PLACEHOLDER_PREFIX.length);
|
|
tasks.push({ parent, key, englishValue });
|
|
}
|
|
return;
|
|
}
|
|
if (!isPlainObject(node)) return;
|
|
for (const [k, v] of Object.entries(node)) {
|
|
collect(v, node, k);
|
|
}
|
|
}
|
|
collect(merged, null, null);
|
|
|
|
if (tasks.length === 0) return { translated: 0, failed: 0 };
|
|
|
|
const limit = createLimiter(concurrency);
|
|
let translated = 0;
|
|
let failed = 0;
|
|
await Promise.all(
|
|
tasks.map((task) =>
|
|
limit(async () => {
|
|
try {
|
|
const value = await translateString(task.englishValue, localeEntry, backend);
|
|
task.parent[task.key] = value;
|
|
translated++;
|
|
} catch (err) {
|
|
// Keep the __MISSING__ marker so subsequent runs can retry.
|
|
failed++;
|
|
logWarn(`translation failed for ${localeEntry.code}: ${err.message}`);
|
|
}
|
|
})
|
|
)
|
|
);
|
|
return { translated, failed };
|
|
}
|
|
|
|
// ----- Main ----------------------------------------------------------------
|
|
|
|
async function processLocale(locale, source, config, opts, backend) {
|
|
const localePath = path.join(MESSAGES_DIR, `${locale}.json`);
|
|
let target = {};
|
|
if (existsSync(localePath)) {
|
|
try {
|
|
target = await loadJson(localePath);
|
|
} catch (err) {
|
|
logWarn(`${locale}: failed to parse existing JSON — starting fresh (${err.message})`);
|
|
target = {};
|
|
}
|
|
} else {
|
|
logWarn(`${locale}: messages file did not exist — creating it`);
|
|
}
|
|
|
|
const { merged, addedPaths } = mergeMissing(source, target);
|
|
const placeholderCountBefore = countPlaceholders(merged);
|
|
|
|
let translateStats = { translated: 0, failed: 0 };
|
|
if (opts.translateMarkers && placeholderCountBefore > 0 && backend) {
|
|
const localeEntry = config.locales.find((l) => l.code === locale);
|
|
if (!localeEntry) {
|
|
logWarn(`${locale}: not present in config/i18n.json — skipping translation`);
|
|
} else {
|
|
const concurrency =
|
|
opts.concurrency ?? Number(process.env.OMNIROUTE_TRANSLATION_CONCURRENCY || 4);
|
|
translateStats = await translatePlaceholders(merged, localeEntry, backend, concurrency);
|
|
}
|
|
}
|
|
|
|
const placeholderCountAfter = countPlaceholders(merged);
|
|
const totalMissing = addedPaths.length;
|
|
const stillPlaceholder = placeholderCountAfter;
|
|
|
|
const summary = `${locale}: +${totalMissing} missing keys (${stillPlaceholder} __MISSING__, ${translateStats.translated} translated${translateStats.failed ? `, ${translateStats.failed} failed` : ""})`;
|
|
|
|
if (opts.dryRun) {
|
|
logInfo(`[DRY] ${summary}`);
|
|
return { addedPaths, translated: translateStats.translated };
|
|
}
|
|
|
|
// Only write when something changed. (json-stable serialization)
|
|
const before = existsSync(localePath) ? await fs.readFile(localePath, "utf8") : "";
|
|
const after = JSON.stringify(merged, null, 2) + "\n";
|
|
if (before === after) {
|
|
logInfo(`${locale}: already in sync (no changes)`);
|
|
return { addedPaths, translated: translateStats.translated };
|
|
}
|
|
await fs.writeFile(localePath, after, "utf8");
|
|
logInfo(summary);
|
|
return { addedPaths, translated: translateStats.translated };
|
|
}
|
|
|
|
async function main() {
|
|
const opts = parseArgs(process.argv);
|
|
const config = await loadConfig();
|
|
|
|
const sourcePath = path.join(MESSAGES_DIR, `${SOURCE_LOCALE}.json`);
|
|
if (!existsSync(sourcePath)) {
|
|
throw new Error(`Source messages file not found: ${sourcePath}`);
|
|
}
|
|
const source = await loadJson(sourcePath);
|
|
|
|
// Locales = every code in config except `en`, intersected with locales that
|
|
// already exist on disk (so we never silently create unknown locale files).
|
|
const onDisk = new Set(
|
|
(await fs.readdir(MESSAGES_DIR)).filter((f) => f.endsWith(".json")).map((f) => f.slice(0, -5))
|
|
);
|
|
|
|
let targetLocales = config.locales
|
|
.map((l) => l.code)
|
|
.filter((code) => code !== SOURCE_LOCALE && onDisk.has(code));
|
|
|
|
if (opts.locales) {
|
|
const missingFromConfig = opts.locales.filter((c) => !config.locales.some((l) => l.code === c));
|
|
if (missingFromConfig.length) {
|
|
logWarn(`--locale contains codes not in config/i18n.json: ${missingFromConfig.join(", ")}`);
|
|
}
|
|
targetLocales = targetLocales.filter((code) => opts.locales.includes(code));
|
|
}
|
|
|
|
logInfo(`source: ${path.relative(ROOT, sourcePath)}`);
|
|
logInfo(`locales: ${targetLocales.length} (${targetLocales.join(", ")})`);
|
|
logInfo(
|
|
`dry-run: ${opts.dryRun ? "yes" : "no"}, translate-markers: ${opts.translateMarkers ? "yes" : "no"}`
|
|
);
|
|
|
|
let backend = null;
|
|
if (opts.translateMarkers && !opts.dryRun) {
|
|
backend = backendConfig();
|
|
backend.concurrency =
|
|
opts.concurrency ?? Number(process.env.OMNIROUTE_TRANSLATION_CONCURRENCY || 4);
|
|
logInfo(
|
|
`backend: ${backend.apiUrl} (model=${backend.model}, concurrency=${backend.concurrency}, timeout=${backend.timeoutMs}ms)`
|
|
);
|
|
}
|
|
|
|
const startMs = Date.now();
|
|
let totalAdded = 0;
|
|
let totalTranslated = 0;
|
|
for (const locale of targetLocales) {
|
|
const result = await processLocale(locale, source, config, opts, backend);
|
|
totalAdded += result.addedPaths.length;
|
|
totalTranslated += result.translated;
|
|
}
|
|
const elapsedSec = ((Date.now() - startMs) / 1000).toFixed(1);
|
|
logInfo(
|
|
`summary: locales=${targetLocales.length}, added=${totalAdded}, translated=${totalTranslated}, elapsed=${elapsedSec}s`
|
|
);
|
|
}
|
|
|
|
const isDirectRun = import.meta.url === pathToFileURL(process.argv[1]).href;
|
|
if (isDirectRun) {
|
|
main().catch((err) => {
|
|
logError(err?.stack || err?.message || String(err));
|
|
process.exit(1);
|
|
});
|
|
}
|