wehub-resource-sync
172 lines
6.4 KiB
JavaScript
172 lines
6.4 KiB
JavaScript
// Generate the Unicode simple case-mapping tables — the ONE source both
|
|
// runtimes case-map from. Reads UnicodeData.txt (fields 12/13: Simple
|
|
// Uppercase / Simple Lowercase — never SpecialCasing, never locale rules)
|
|
// and emits the same compressed range tables twice:
|
|
//
|
|
// rt/rt.zig the marked GENERATED CASE TABLES region (in place)
|
|
// src/text_tables.ts the whole file (the node-polyfill mirror)
|
|
//
|
|
// Byte-honest contract: simple case mapping is code point -> code point,
|
|
// locale-independent, with no special casing (no ß -> SS, no context
|
|
// forms), so `.toUpperCase()`/`.toLowerCase()` on core bytes produce the
|
|
// same bytes under node (polyfilled from these tables) and native (rt
|
|
// helpers over the same tables) by construction.
|
|
//
|
|
// Compression: consecutive mapped code points sharing one delta collapse
|
|
// into {lo, count, stride, delta} runs (stride 2 covers the alternating
|
|
// upper/lower pairs like U+0100 Ā / U+0101 ā). Unicode 17.0.0 yields ~200
|
|
// ranges per direction at 8 bytes each — ~3.1 KiB total in the binary.
|
|
//
|
|
// Regenerate (a Unicode version bump is the only reason to):
|
|
//
|
|
// node scripts/gen_case_tables.mjs [path/to/UnicodeData.txt]
|
|
//
|
|
// Without a path the script downloads the PINNED version below from
|
|
// unicode.org. Both outputs are committed; CI never runs this.
|
|
|
|
import fs from "node:fs";
|
|
import path from "node:path";
|
|
import { fileURLToPath } from "node:url";
|
|
|
|
const UNICODE_VERSION = "17.0.0";
|
|
const UCD_URL = `https://www.unicode.org/Public/${UNICODE_VERSION}/ucd/UnicodeData.txt`;
|
|
|
|
const pkg = path.dirname(path.dirname(fileURLToPath(import.meta.url)));
|
|
|
|
async function loadUnicodeData() {
|
|
const arg = process.argv[2];
|
|
if (arg) return fs.readFileSync(arg, "utf8");
|
|
console.log(`downloading ${UCD_URL}`);
|
|
const res = await fetch(UCD_URL);
|
|
if (!res.ok) throw new Error(`download failed: ${res.status} ${res.statusText}`);
|
|
return res.text();
|
|
}
|
|
|
|
/// [codePoint, mappedCodePoint] pairs for one UnicodeData field
|
|
/// (12 = Simple_Uppercase_Mapping, 13 = Simple_Lowercase_Mapping).
|
|
function collectMappings(text, field) {
|
|
const out = [];
|
|
for (const line of text.split("\n")) {
|
|
if (!line) continue;
|
|
const fields = line.split(";");
|
|
const mapped = fields[field];
|
|
if (!mapped) continue;
|
|
out.push([parseInt(fields[0], 16), parseInt(mapped, 16)]);
|
|
}
|
|
out.sort((a, b) => a[0] - b[0]);
|
|
return out;
|
|
}
|
|
|
|
/// Collapse sorted [cp, mapped] pairs into {lo, count, stride, delta} runs.
|
|
/// A run holds `count` mapped code points at lo, lo+stride, ...; every one
|
|
/// maps to itself + delta. stride is 1 or 2 (2 = the alternating pairs).
|
|
function compressRanges(entries) {
|
|
const out = [];
|
|
for (const [cp, to] of entries) {
|
|
const delta = to - cp;
|
|
const last = out[out.length - 1];
|
|
if (last && last.delta === delta) {
|
|
const step = cp - (last.lo + (last.count - 1) * last.stride);
|
|
if (last.count === 1 && (step === 1 || step === 2)) {
|
|
last.stride = step;
|
|
last.count += 1;
|
|
continue;
|
|
}
|
|
if (last.count > 1 && step === last.stride) {
|
|
last.count += 1;
|
|
continue;
|
|
}
|
|
}
|
|
out.push({ lo: cp, delta, stride: 1, count: 1 });
|
|
}
|
|
for (const r of out) {
|
|
// The packed layouts below: lo u21, count u16, stride bit, delta i26.
|
|
if (r.lo > 0x10ffff || r.count > 0xffff || Math.abs(r.delta) >= 1 << 25) {
|
|
throw new Error(`range out of packed bounds: ${JSON.stringify(r)}`);
|
|
}
|
|
}
|
|
return out;
|
|
}
|
|
|
|
function hex(cp) {
|
|
return "0x" + cp.toString(16).toUpperCase();
|
|
}
|
|
|
|
function zigRanges(name, ranges) {
|
|
const lines = [`const ${name} = [_]CaseRange{`];
|
|
for (const r of ranges) {
|
|
lines.push(
|
|
` .{ .lo = ${hex(r.lo)}, .count = ${r.count}, .stride2 = ${r.stride === 2}, .delta = ${r.delta} },`,
|
|
);
|
|
}
|
|
lines.push("};");
|
|
return lines.join("\n");
|
|
}
|
|
|
|
function tsRanges(name, ranges) {
|
|
// Flat quadruples [lo, count, stride, delta] — the same runs, same order.
|
|
const parts = ranges.map((r) => `${r.lo}, ${r.count}, ${r.stride}, ${r.delta}`);
|
|
const lines = [`export const ${name}: readonly number[] = [`];
|
|
for (const p of parts) lines.push(` ${p},`);
|
|
lines.push("];");
|
|
return lines.join("\n");
|
|
}
|
|
|
|
const BEGIN = "// ---- BEGIN GENERATED CASE TABLES";
|
|
const END = "// ---- END GENERATED CASE TABLES";
|
|
|
|
const text = await loadUnicodeData();
|
|
const upper = compressRanges(collectMappings(text, 12));
|
|
const lower = compressRanges(collectMappings(text, 13));
|
|
const tableBytes = (upper.length + lower.length) * 8;
|
|
|
|
const header = (comment) =>
|
|
[
|
|
`${comment} Unicode ${UNICODE_VERSION} simple case mappings (UnicodeData.txt fields 12/13).`,
|
|
`${comment} GENERATED by scripts/gen_case_tables.mjs — do not edit by hand; regenerate with:`,
|
|
`${comment} node scripts/gen_case_tables.mjs [path/to/UnicodeData.txt]`,
|
|
`${comment} ${upper.length} + ${lower.length} ranges, 8 bytes each = ${tableBytes} bytes of table data.`,
|
|
].join("\n");
|
|
|
|
// ---------------------------------------------------------------- rt.zig
|
|
|
|
const rtPath = path.join(pkg, "rt", "rt.zig");
|
|
const rt = fs.readFileSync(rtPath, "utf8");
|
|
const begin = rt.indexOf(BEGIN);
|
|
const endMark = rt.indexOf(END);
|
|
if (begin < 0 || endMark < 0) throw new Error(`rt.zig is missing the ${BEGIN} / ${END} markers`);
|
|
const end = rt.indexOf("\n", endMark); // splice through the END marker's whole line
|
|
const zigBlock = [
|
|
`${BEGIN} ----`,
|
|
header("//"),
|
|
"",
|
|
zigRanges("simple_upper_ranges", upper),
|
|
"",
|
|
zigRanges("simple_lower_ranges", lower),
|
|
`${END} ----`,
|
|
].join("\n");
|
|
fs.writeFileSync(rtPath, rt.slice(0, begin) + zigBlock + rt.slice(end));
|
|
console.log(`wrote ${rtPath} (${upper.length} upper + ${lower.length} lower ranges, ${tableBytes} bytes)`);
|
|
|
|
// ---------------------------------------------------------- text_tables.ts
|
|
|
|
const tsPath = path.join(pkg, "src", "text_tables.ts");
|
|
const tsOut = [
|
|
"// The node-side mirror of rt.zig's simple case-mapping tables: the same",
|
|
"// generator emits both from one UnicodeData.txt read, so the devhost",
|
|
"// polyfill (text_polyfill.ts) and the native rt helpers case-map from",
|
|
"// byte-identical data by construction.",
|
|
header("//"),
|
|
"",
|
|
"// Flat quadruples [lo, count, stride, delta]: `count` mapped code points",
|
|
"// at lo, lo+stride, ...; each maps to itself + delta (stride 2 covers the",
|
|
"// alternating upper/lower pairs like U+0100 Ā / U+0101 ā).",
|
|
"",
|
|
tsRanges("SIMPLE_UPPER_RANGES", upper),
|
|
"",
|
|
tsRanges("SIMPLE_LOWER_RANGES", lower),
|
|
"",
|
|
].join("\n");
|
|
fs.writeFileSync(tsPath, tsOut);
|
|
console.log(`wrote ${tsPath}`);
|