chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,90 @@
|
||||
// The transpiler gate: transpile the fixture core, build it against the rt
|
||||
// kernel with the fixture shim, replay the deterministic 1k-message run, and
|
||||
// require the digest of the snapshot+effect log to equal the hand-written
|
||||
// Zig oracle's. Then run the 10k keystroke bench for the perf band.
|
||||
//
|
||||
// Usage: node scripts/gate.mjs [--keep] [--skip-bench]
|
||||
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const pkg = path.dirname(path.dirname(fileURLToPath(import.meta.url)));
|
||||
|
||||
// MD5 of the run1k output (snapshot hex + effect-log hex) produced by the
|
||||
// hand-written oracle core over the fixed SplitMix64 message sequence. The
|
||||
// transpiled core must reproduce it exactly — same model bytes, same effect
|
||||
// stream, for all 1000 dispatches.
|
||||
const ORACLE_RUN1K_MD5 = "4e1140c16ba5569ab73db860894f4eba";
|
||||
|
||||
const keep = process.argv.includes("--keep");
|
||||
const skipBench = process.argv.includes("--skip-bench");
|
||||
|
||||
const work = fs.mkdtempSync(path.join(os.tmpdir(), "native-core-gate-"));
|
||||
const log = (msg) => console.log(`[gate] ${msg}`);
|
||||
|
||||
try {
|
||||
log(`work dir ${work}`);
|
||||
|
||||
// 1. Transpile the fixture core.
|
||||
execFileSync(process.execPath, [
|
||||
path.join(pkg, "src/cli.ts"),
|
||||
path.join(pkg, "test/fixtures/inbox_core_subset.ts"),
|
||||
"-o",
|
||||
path.join(work, "inbox_core.zig"),
|
||||
], { stdio: "inherit" });
|
||||
log("transpiled inbox_core_subset.ts");
|
||||
|
||||
// 2. Assemble the build: rt kernel + shim + harness.
|
||||
for (const [src, dst] of [
|
||||
["rt/rt.zig", "rt.zig"],
|
||||
["test/fixtures/shim.zig", "shim.zig"],
|
||||
["test/fixtures/impl.zig", "impl.zig"],
|
||||
["test/fixtures/bench.zig", "bench.zig"],
|
||||
]) {
|
||||
fs.copyFileSync(path.join(pkg, src), path.join(work, dst));
|
||||
}
|
||||
|
||||
const zig = (args) => execFileSync("zig", args, { cwd: work, stdio: "inherit" });
|
||||
|
||||
// 3. Build ReleaseSafe (the shipping mode: index checks stay on).
|
||||
zig(["build-lib", "-OReleaseSafe", "-femit-bin=libinbox.a", "shim.zig"]);
|
||||
zig([
|
||||
"build-exe", "-OReleaseSafe", "-femit-bin=bench_safe",
|
||||
"--dep", "impl", "-Mroot=bench.zig", "-Mimpl=impl.zig", "libinbox.a", "-lc",
|
||||
]);
|
||||
log("built ReleaseSafe");
|
||||
|
||||
// 4. run1k digest gate.
|
||||
execFileSync(path.join(work, "bench_safe"), ["run1k", "run1k.txt"], { cwd: work, stdio: "inherit" });
|
||||
const digest = createHash("md5").update(fs.readFileSync(path.join(work, "run1k.txt"))).digest("hex");
|
||||
if (digest !== ORACLE_RUN1K_MD5) {
|
||||
console.error(`[gate] FAIL run1k digest ${digest} != oracle ${ORACLE_RUN1K_MD5}`);
|
||||
process.exit(1);
|
||||
}
|
||||
log(`run1k digest matches oracle (${digest})`);
|
||||
|
||||
// 5. Perf band.
|
||||
if (!skipBench) {
|
||||
log("bench10k (ReleaseSafe):");
|
||||
execFileSync(path.join(work, "bench_safe"), ["bench10k"], { cwd: work, stdio: "inherit" });
|
||||
zig(["build-lib", "-OReleaseFast", "-femit-bin=libinbox_fast.a", "shim.zig"]);
|
||||
zig([
|
||||
"build-exe", "-OReleaseFast", "-femit-bin=bench_fast",
|
||||
"--dep", "impl", "-Mroot=bench.zig", "-Mimpl=impl.zig", "libinbox_fast.a", "-lc",
|
||||
]);
|
||||
log("bench10k (ReleaseFast):");
|
||||
execFileSync(path.join(work, "bench_fast"), ["bench10k"], { cwd: work, stdio: "inherit" });
|
||||
}
|
||||
|
||||
log("PASS");
|
||||
} finally {
|
||||
if (keep) {
|
||||
log(`kept ${work}`);
|
||||
} else {
|
||||
fs.rmSync(work, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
// Generate the Unicode simple case-mapping tables — the ONE source both
|
||||
// runtimes case-map from. Reads UnicodeData.txt (fields 12/13: Simple
|
||||
// Uppercase / Simple Lowercase — never SpecialCasing, never locale rules)
|
||||
// and emits the same compressed range tables twice:
|
||||
//
|
||||
// rt/rt.zig the marked GENERATED CASE TABLES region (in place)
|
||||
// src/text_tables.ts the whole file (the node-polyfill mirror)
|
||||
//
|
||||
// Byte-honest contract: simple case mapping is code point -> code point,
|
||||
// locale-independent, with no special casing (no ß -> SS, no context
|
||||
// forms), so `.toUpperCase()`/`.toLowerCase()` on core bytes produce the
|
||||
// same bytes under node (polyfilled from these tables) and native (rt
|
||||
// helpers over the same tables) by construction.
|
||||
//
|
||||
// Compression: consecutive mapped code points sharing one delta collapse
|
||||
// into {lo, count, stride, delta} runs (stride 2 covers the alternating
|
||||
// upper/lower pairs like U+0100 Ā / U+0101 ā). Unicode 17.0.0 yields ~200
|
||||
// ranges per direction at 8 bytes each — ~3.1 KiB total in the binary.
|
||||
//
|
||||
// Regenerate (a Unicode version bump is the only reason to):
|
||||
//
|
||||
// node scripts/gen_case_tables.mjs [path/to/UnicodeData.txt]
|
||||
//
|
||||
// Without a path the script downloads the PINNED version below from
|
||||
// unicode.org. Both outputs are committed; CI never runs this.
|
||||
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const UNICODE_VERSION = "17.0.0";
|
||||
const UCD_URL = `https://www.unicode.org/Public/${UNICODE_VERSION}/ucd/UnicodeData.txt`;
|
||||
|
||||
const pkg = path.dirname(path.dirname(fileURLToPath(import.meta.url)));
|
||||
|
||||
async function loadUnicodeData() {
|
||||
const arg = process.argv[2];
|
||||
if (arg) return fs.readFileSync(arg, "utf8");
|
||||
console.log(`downloading ${UCD_URL}`);
|
||||
const res = await fetch(UCD_URL);
|
||||
if (!res.ok) throw new Error(`download failed: ${res.status} ${res.statusText}`);
|
||||
return res.text();
|
||||
}
|
||||
|
||||
/// [codePoint, mappedCodePoint] pairs for one UnicodeData field
|
||||
/// (12 = Simple_Uppercase_Mapping, 13 = Simple_Lowercase_Mapping).
|
||||
function collectMappings(text, field) {
|
||||
const out = [];
|
||||
for (const line of text.split("\n")) {
|
||||
if (!line) continue;
|
||||
const fields = line.split(";");
|
||||
const mapped = fields[field];
|
||||
if (!mapped) continue;
|
||||
out.push([parseInt(fields[0], 16), parseInt(mapped, 16)]);
|
||||
}
|
||||
out.sort((a, b) => a[0] - b[0]);
|
||||
return out;
|
||||
}
|
||||
|
||||
/// Collapse sorted [cp, mapped] pairs into {lo, count, stride, delta} runs.
|
||||
/// A run holds `count` mapped code points at lo, lo+stride, ...; every one
|
||||
/// maps to itself + delta. stride is 1 or 2 (2 = the alternating pairs).
|
||||
function compressRanges(entries) {
|
||||
const out = [];
|
||||
for (const [cp, to] of entries) {
|
||||
const delta = to - cp;
|
||||
const last = out[out.length - 1];
|
||||
if (last && last.delta === delta) {
|
||||
const step = cp - (last.lo + (last.count - 1) * last.stride);
|
||||
if (last.count === 1 && (step === 1 || step === 2)) {
|
||||
last.stride = step;
|
||||
last.count += 1;
|
||||
continue;
|
||||
}
|
||||
if (last.count > 1 && step === last.stride) {
|
||||
last.count += 1;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
out.push({ lo: cp, delta, stride: 1, count: 1 });
|
||||
}
|
||||
for (const r of out) {
|
||||
// The packed layouts below: lo u21, count u16, stride bit, delta i26.
|
||||
if (r.lo > 0x10ffff || r.count > 0xffff || Math.abs(r.delta) >= 1 << 25) {
|
||||
throw new Error(`range out of packed bounds: ${JSON.stringify(r)}`);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function hex(cp) {
|
||||
return "0x" + cp.toString(16).toUpperCase();
|
||||
}
|
||||
|
||||
function zigRanges(name, ranges) {
|
||||
const lines = [`const ${name} = [_]CaseRange{`];
|
||||
for (const r of ranges) {
|
||||
lines.push(
|
||||
` .{ .lo = ${hex(r.lo)}, .count = ${r.count}, .stride2 = ${r.stride === 2}, .delta = ${r.delta} },`,
|
||||
);
|
||||
}
|
||||
lines.push("};");
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
function tsRanges(name, ranges) {
|
||||
// Flat quadruples [lo, count, stride, delta] — the same runs, same order.
|
||||
const parts = ranges.map((r) => `${r.lo}, ${r.count}, ${r.stride}, ${r.delta}`);
|
||||
const lines = [`export const ${name}: readonly number[] = [`];
|
||||
for (const p of parts) lines.push(` ${p},`);
|
||||
lines.push("];");
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
const BEGIN = "// ---- BEGIN GENERATED CASE TABLES";
|
||||
const END = "// ---- END GENERATED CASE TABLES";
|
||||
|
||||
const text = await loadUnicodeData();
|
||||
const upper = compressRanges(collectMappings(text, 12));
|
||||
const lower = compressRanges(collectMappings(text, 13));
|
||||
const tableBytes = (upper.length + lower.length) * 8;
|
||||
|
||||
const header = (comment) =>
|
||||
[
|
||||
`${comment} Unicode ${UNICODE_VERSION} simple case mappings (UnicodeData.txt fields 12/13).`,
|
||||
`${comment} GENERATED by scripts/gen_case_tables.mjs — do not edit by hand; regenerate with:`,
|
||||
`${comment} node scripts/gen_case_tables.mjs [path/to/UnicodeData.txt]`,
|
||||
`${comment} ${upper.length} + ${lower.length} ranges, 8 bytes each = ${tableBytes} bytes of table data.`,
|
||||
].join("\n");
|
||||
|
||||
// ---------------------------------------------------------------- rt.zig
|
||||
|
||||
const rtPath = path.join(pkg, "rt", "rt.zig");
|
||||
const rt = fs.readFileSync(rtPath, "utf8");
|
||||
const begin = rt.indexOf(BEGIN);
|
||||
const endMark = rt.indexOf(END);
|
||||
if (begin < 0 || endMark < 0) throw new Error(`rt.zig is missing the ${BEGIN} / ${END} markers`);
|
||||
const end = rt.indexOf("\n", endMark); // splice through the END marker's whole line
|
||||
const zigBlock = [
|
||||
`${BEGIN} ----`,
|
||||
header("//"),
|
||||
"",
|
||||
zigRanges("simple_upper_ranges", upper),
|
||||
"",
|
||||
zigRanges("simple_lower_ranges", lower),
|
||||
`${END} ----`,
|
||||
].join("\n");
|
||||
fs.writeFileSync(rtPath, rt.slice(0, begin) + zigBlock + rt.slice(end));
|
||||
console.log(`wrote ${rtPath} (${upper.length} upper + ${lower.length} lower ranges, ${tableBytes} bytes)`);
|
||||
|
||||
// ---------------------------------------------------------- text_tables.ts
|
||||
|
||||
const tsPath = path.join(pkg, "src", "text_tables.ts");
|
||||
const tsOut = [
|
||||
"// The node-side mirror of rt.zig's simple case-mapping tables: the same",
|
||||
"// generator emits both from one UnicodeData.txt read, so the devhost",
|
||||
"// polyfill (text_polyfill.ts) and the native rt helpers case-map from",
|
||||
"// byte-identical data by construction.",
|
||||
header("//"),
|
||||
"",
|
||||
"// Flat quadruples [lo, count, stride, delta]: `count` mapped code points",
|
||||
"// at lo, lo+stride, ...; each maps to itself + delta (stride 2 covers the",
|
||||
"// alternating upper/lower pairs like U+0100 Ā / U+0101 ā).",
|
||||
"",
|
||||
tsRanges("SIMPLE_UPPER_RANGES", upper),
|
||||
"",
|
||||
tsRanges("SIMPLE_LOWER_RANGES", lower),
|
||||
"",
|
||||
].join("\n");
|
||||
fs.writeFileSync(tsPath, tsOut);
|
||||
console.log(`wrote ${tsPath}`);
|
||||
Reference in New Issue
Block a user