// Generate the Unicode simple case-mapping tables — the ONE source both // runtimes case-map from. Reads UnicodeData.txt (fields 12/13: Simple // Uppercase / Simple Lowercase — never SpecialCasing, never locale rules) // and emits the same compressed range tables twice: // // rt/rt.zig the marked GENERATED CASE TABLES region (in place) // src/text_tables.ts the whole file (the node-polyfill mirror) // // Byte-honest contract: simple case mapping is code point -> code point, // locale-independent, with no special casing (no ß -> SS, no context // forms), so `.toUpperCase()`/`.toLowerCase()` on core bytes produce the // same bytes under node (polyfilled from these tables) and native (rt // helpers over the same tables) by construction. // // Compression: consecutive mapped code points sharing one delta collapse // into {lo, count, stride, delta} runs (stride 2 covers the alternating // upper/lower pairs like U+0100 Ā / U+0101 ā). Unicode 17.0.0 yields ~200 // ranges per direction at 8 bytes each — ~3.1 KiB total in the binary. // // Regenerate (a Unicode version bump is the only reason to): // // node scripts/gen_case_tables.mjs [path/to/UnicodeData.txt] // // Without a path the script downloads the PINNED version below from // unicode.org. Both outputs are committed; CI never runs this. import fs from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; const UNICODE_VERSION = "17.0.0"; const UCD_URL = `https://www.unicode.org/Public/${UNICODE_VERSION}/ucd/UnicodeData.txt`; const pkg = path.dirname(path.dirname(fileURLToPath(import.meta.url))); async function loadUnicodeData() { const arg = process.argv[2]; if (arg) return fs.readFileSync(arg, "utf8"); console.log(`downloading ${UCD_URL}`); const res = await fetch(UCD_URL); if (!res.ok) throw new Error(`download failed: ${res.status} ${res.statusText}`); return res.text(); } /// [codePoint, mappedCodePoint] pairs for one UnicodeData field /// (12 = Simple_Uppercase_Mapping, 13 = Simple_Lowercase_Mapping). function collectMappings(text, field) { const out = []; for (const line of text.split("\n")) { if (!line) continue; const fields = line.split(";"); const mapped = fields[field]; if (!mapped) continue; out.push([parseInt(fields[0], 16), parseInt(mapped, 16)]); } out.sort((a, b) => a[0] - b[0]); return out; } /// Collapse sorted [cp, mapped] pairs into {lo, count, stride, delta} runs. /// A run holds `count` mapped code points at lo, lo+stride, ...; every one /// maps to itself + delta. stride is 1 or 2 (2 = the alternating pairs). function compressRanges(entries) { const out = []; for (const [cp, to] of entries) { const delta = to - cp; const last = out[out.length - 1]; if (last && last.delta === delta) { const step = cp - (last.lo + (last.count - 1) * last.stride); if (last.count === 1 && (step === 1 || step === 2)) { last.stride = step; last.count += 1; continue; } if (last.count > 1 && step === last.stride) { last.count += 1; continue; } } out.push({ lo: cp, delta, stride: 1, count: 1 }); } for (const r of out) { // The packed layouts below: lo u21, count u16, stride bit, delta i26. if (r.lo > 0x10ffff || r.count > 0xffff || Math.abs(r.delta) >= 1 << 25) { throw new Error(`range out of packed bounds: ${JSON.stringify(r)}`); } } return out; } function hex(cp) { return "0x" + cp.toString(16).toUpperCase(); } function zigRanges(name, ranges) { const lines = [`const ${name} = [_]CaseRange{`]; for (const r of ranges) { lines.push( ` .{ .lo = ${hex(r.lo)}, .count = ${r.count}, .stride2 = ${r.stride === 2}, .delta = ${r.delta} },`, ); } lines.push("};"); return lines.join("\n"); } function tsRanges(name, ranges) { // Flat quadruples [lo, count, stride, delta] — the same runs, same order. const parts = ranges.map((r) => `${r.lo}, ${r.count}, ${r.stride}, ${r.delta}`); const lines = [`export const ${name}: readonly number[] = [`]; for (const p of parts) lines.push(` ${p},`); lines.push("];"); return lines.join("\n"); } const BEGIN = "// ---- BEGIN GENERATED CASE TABLES"; const END = "// ---- END GENERATED CASE TABLES"; const text = await loadUnicodeData(); const upper = compressRanges(collectMappings(text, 12)); const lower = compressRanges(collectMappings(text, 13)); const tableBytes = (upper.length + lower.length) * 8; const header = (comment) => [ `${comment} Unicode ${UNICODE_VERSION} simple case mappings (UnicodeData.txt fields 12/13).`, `${comment} GENERATED by scripts/gen_case_tables.mjs — do not edit by hand; regenerate with:`, `${comment} node scripts/gen_case_tables.mjs [path/to/UnicodeData.txt]`, `${comment} ${upper.length} + ${lower.length} ranges, 8 bytes each = ${tableBytes} bytes of table data.`, ].join("\n"); // ---------------------------------------------------------------- rt.zig const rtPath = path.join(pkg, "rt", "rt.zig"); const rt = fs.readFileSync(rtPath, "utf8"); const begin = rt.indexOf(BEGIN); const endMark = rt.indexOf(END); if (begin < 0 || endMark < 0) throw new Error(`rt.zig is missing the ${BEGIN} / ${END} markers`); const end = rt.indexOf("\n", endMark); // splice through the END marker's whole line const zigBlock = [ `${BEGIN} ----`, header("//"), "", zigRanges("simple_upper_ranges", upper), "", zigRanges("simple_lower_ranges", lower), `${END} ----`, ].join("\n"); fs.writeFileSync(rtPath, rt.slice(0, begin) + zigBlock + rt.slice(end)); console.log(`wrote ${rtPath} (${upper.length} upper + ${lower.length} lower ranges, ${tableBytes} bytes)`); // ---------------------------------------------------------- text_tables.ts const tsPath = path.join(pkg, "src", "text_tables.ts"); const tsOut = [ "// The node-side mirror of rt.zig's simple case-mapping tables: the same", "// generator emits both from one UnicodeData.txt read, so the devhost", "// polyfill (text_polyfill.ts) and the native rt helpers case-map from", "// byte-identical data by construction.", header("//"), "", "// Flat quadruples [lo, count, stride, delta]: `count` mapped code points", "// at lo, lo+stride, ...; each maps to itself + delta (stride 2 covers the", "// alternating upper/lower pairs like U+0100 Ā / U+0101 ā).", "", tsRanges("SIMPLE_UPPER_RANGES", upper), "", tsRanges("SIMPLE_LOWER_RANGES", lower), "", ].join("\n"); fs.writeFileSync(tsPath, tsOut); console.log(`wrote ${tsPath}`);