chore: import upstream snapshot with attribution

worked/example/raw/processor.py

@@ -0,0 +1,71 @@
+"""
+Processor module - transforms validated documents into enriched records
+ready for storage and retrieval.
+"""
+import re
+from storage import load_index, save_processed
+
+
+STOPWORDS = {"the", "a", "an", "and", "or", "but", "in", "on", "at", "to", "for", "of", "with"}
+
+
+def normalize_text(text: str) -> str:
+    """Lowercase, strip extra whitespace, remove control characters."""
+    text = text.lower().strip()
+    text = re.sub(r"\s+", " ", text)
+    text = re.sub(r"[^\x20-\x7e]", "", text)
+    return text
+
+
+def extract_keywords(text: str) -> list:
+    """Pull non-stopword tokens from text, deduplicated."""
+    tokens = re.findall(r"\b[a-z]{3,}\b", normalize_text(text))
+    seen = set()
+    keywords = []
+    for t in tokens:
+        if t not in STOPWORDS and t not in seen:
+            seen.add(t)
+            keywords.append(t)
+    return keywords
+
+
+def enrich_document(doc: dict) -> dict:
+    """Add keyword index and cross-references to a validated document."""
+    text_blob = " ".join([
+        doc.get("title", ""),
+        " ".join(doc.get("sections", [])),
+        " ".join(doc.get("paragraphs", [])),
+    ])
+    doc["keywords"] = extract_keywords(text_blob)
+    doc["cross_refs"] = find_cross_references(doc)
+    return doc
+
+
+def find_cross_references(doc: dict) -> list:
+    """Look up the index and return IDs of related documents by keyword overlap."""
+    index = load_index()
+    keywords = set(doc.get("keywords", []))
+    refs = []
+    for record_id, entry in index.items():
+        other_keywords = set(entry.get("keywords", []))
+        overlap = keywords & other_keywords
+        if len(overlap) >= 3:
+            refs.append({"id": record_id, "shared_keywords": list(overlap)})
+    return refs
+
+
+def process_and_save(doc: dict) -> str:
+    """Enrich a validated document and persist it. Returns the record ID."""
+    enriched = enrich_document(doc)
+    record_id = save_processed(enriched)
+    return record_id
+
+
+def reprocess_all() -> int:
+    """Re-enrich all records in the index. Returns count of records updated."""
+    index = load_index()
+    count = 0
+    for record_id, doc in index.items():
+        process_and_save(doc)
+        count += 1
+    return count