wehub-resource-sync
46 lines
1.8 KiB
TypeScript
46 lines
1.8 KiB
TypeScript
import { createHash } from "node:crypto";
|
|
import type { EvalCase, ContentKind } from "./types.ts";
|
|
|
|
const KINDS: ContentKind[] = ["tool-output-json", "logs", "code", "prose", "multi-turn"];
|
|
|
|
/** Best-effort PII markers; captured cases must be anonymized BEFORE ingestion. */
|
|
const PII_PATTERNS: RegExp[] = [
|
|
/[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/, // email
|
|
/\b\d{3}-\d{2}-\d{4}\b/, // US SSN shape
|
|
/\b(?:\d[ -]*?){13,16}\b/, // card-number shape
|
|
];
|
|
|
|
function looksLikePII(text: string): boolean {
|
|
return PII_PATTERNS.some((re) => re.test(text));
|
|
}
|
|
|
|
/**
|
|
* Validate + sanity-check a corpus. Throws on a malformed case so a bad corpus
|
|
* fails loudly rather than silently skewing results. Captured cases (captured === true)
|
|
* are rejected on obvious PII markers (D-D5 anonymization bar); curated seed cases
|
|
* (captured falsy) are trusted as already-vetted.
|
|
*/
|
|
export function loadCorpus(rawCases: EvalCase[]): EvalCase[] {
|
|
return rawCases.map((c) => {
|
|
if (!c.id || !c.context || !c.question) {
|
|
throw new Error(`eval corpus: case "${c.id ?? "?"}" missing id/context/question`);
|
|
}
|
|
if (!KINDS.includes(c.kind)) {
|
|
throw new Error(`eval corpus: case "${c.id}" has unknown kind "${c.kind}"`);
|
|
}
|
|
if (c.captured === true && looksLikePII(c.context)) {
|
|
throw new Error(`eval corpus: captured case "${c.id}" contains an obvious PII marker — anonymize before ingestion`);
|
|
}
|
|
return c;
|
|
});
|
|
}
|
|
|
|
/** Stable, order-independent sha-256 hex over the canonical case payloads (for the report stamp). */
|
|
export function hashCorpus(cases: EvalCase[]): string {
|
|
const canonical = cases
|
|
.map((c) => JSON.stringify([c.id, c.kind, c.context, c.question, c.gold ?? null]))
|
|
.sort()
|
|
.join("\n");
|
|
return createHash("sha256").update(canonical).digest("hex");
|
|
}
|