chore: import upstream snapshot with attribution

worked/example/raw/api.py

@@ -0,0 +1,78 @@
+"""
+API module - exposes the document pipeline over HTTP.
+Thin layer over parser, validator, processor, and storage.
+"""
+from parser import batch_parse, parse_file
+from validator import validate_document, ValidationError
+from processor import process_and_save, enrich_document
+from storage import load_record, delete_record, list_records, load_index
+
+
+def handle_upload(paths: list) -> dict:
+    """
+    Accept a list of file paths, run the full pipeline on each,
+    and return a summary of what succeeded and what failed.
+    """
+    results = batch_parse(paths)
+    succeeded = [r for r in results if r["ok"]]
+    failed = [r for r in results if not r["ok"]]
+    return {
+        "uploaded": len(succeeded),
+        "failed": len(failed),
+        "ids": [r["id"] for r in succeeded],
+        "errors": failed,
+    }
+
+
+def handle_get(record_id: str) -> dict:
+    """Fetch a document by ID and return it."""
+    try:
+        return load_record(record_id)
+    except KeyError:
+        return {"error": f"Record {record_id} not found"}
+
+
+def handle_delete(record_id: str) -> dict:
+    """Delete a document by ID."""
+    deleted = delete_record(record_id)
+    if deleted:
+        return {"deleted": record_id}
+    return {"error": f"Record {record_id} not found"}
+
+
+def handle_list() -> dict:
+    """List all document IDs in storage."""
+    return {"records": list_records()}
+
+
+def handle_search(query: str) -> dict:
+    """
+    Simple keyword search over the index.
+    Returns documents whose keyword list overlaps with the query terms.
+    """
+    terms = set(query.lower().split())
+    index = load_index()
+    matches = []
+    for record_id, entry in index.items():
+        keywords = set(entry.get("keywords", []))
+        if terms & keywords:
+            matches.append({
+                "id": record_id,
+                "title": entry.get("title", ""),
+                "matched_keywords": list(terms & keywords),
+            })
+    return {"query": query, "results": matches}
+
+
+def handle_enrich(record_id: str) -> dict:
+    """Re-enrich a document to pick up new cross-references."""
+    try:
+        doc = load_record(record_id)
+    except KeyError:
+        return {"error": f"Record {record_id} not found"}
+    try:
+        validated = validate_document(doc)
+    except ValidationError as e:
+        return {"error": str(e)}
+    enriched_id = process_and_save(validated)
+    return {"enriched": enriched_id}

worked/example/raw/architecture.md

@@ -0,0 +1,37 @@
+# Document Pipeline Architecture
+
+This is a small document ingestion and search system. Files come in, get parsed and validated, keywords get extracted, cross-references get built, and everything ends up queryable via a simple API.
+
+## How data flows
+
+Raw files on disk go through four stages before they are searchable.
+
+**Parsing** reads the file, detects the format (markdown, JSON, plaintext), and converts it into a structured dict. The parser handles each format differently. Markdown gets title, sections, and links extracted. JSON gets loaded directly. Plaintext gets split into paragraphs.
+
+**Validation** checks that the parsed document has the required fields and a known format. It also normalizes text fields (lowercase, trim whitespace, strip control characters) using the processor before the document moves forward.
+
+**Processing** enriches the validated document with a keyword index and cross-references. Cross-references are built by comparing the document's keywords against every other document already in the index. If they share three or more keywords they get linked.
+
+**Storage** persists everything to disk as JSON files and maintains a flat index that maps record IDs to metadata. All other modules read and write through the storage interface so there is one source of truth.
+
+## Module responsibilities
+
+- parser.py: reads files, detects format, calls validate_document and save_parsed
+- validator.py: enforces schema, normalizes fields, calls normalize_text from processor
+- processor.py: extract_keywords, find_cross_references, calls load_index and save_processed
+- storage.py: load_index, save_parsed, save_processed, load_record, delete_record, list_records
+- api.py: HTTP handlers that orchestrate the above modules
+
+## Design decisions
+
+The pipeline is intentionally linear. Each stage has one job and calls the next stage explicitly. There is no event bus or dependency injection. This makes the call graph easy to follow and easy to test.
+
+Storage is intentionally simple. A flat JSON index plus one file per document is enough at small scale. If the corpus grows past a few thousand documents this becomes the bottleneck and should be replaced with SQLite or a proper document store.
+
+Cross-reference detection is intentionally naive. Keyword overlap of three is a reasonable threshold for short documents but will produce too many false positives on long ones. A real system would use TF-IDF or embedding similarity instead.
+
+## Extending the pipeline
+
+To add a new file format, add a branch in parser.py's parse_file function and a new parse_* function. The rest of the pipeline does not need to change.
+
+To add a new enrichment step, add a function in processor.py and call it from enrich_document. Store the result in the document dict and add the field to the index in save_processed if you want it searchable.

worked/example/raw/notes.md

@@ -0,0 +1,39 @@
+# Research Notes
+
+Thoughts and open questions while building the document pipeline. Not polished, just a running log.
+
+## On keyword extraction
+
+The current approach strips stopwords and returns unique tokens. Simple and fast. The problem is it treats all keywords equally. "database" appearing once in a title carries more weight than "database" buried in a paragraph but the code doesn't know that.
+
+TF-IDF would fix this. Term frequency times inverse document frequency gives higher scores to words that are distinctive to a document rather than common across the corpus. Worth switching once the index is big enough for IDF to be meaningful (probably 50+ documents).
+
+Embedding-based similarity is the other option. Run each document through a sentence transformer, store the vector, do nearest-neighbor search at query time. Much better recall but adds a dependency and makes the index opaque. The keyword approach is at least debuggable.
+
+## On cross-reference detection
+
+Three shared keywords is arbitrary. Tuned it by hand on a small test set. On short documents (under 500 words) it produces reasonable results. On long documents everything shares keywords with everything else and the cross-reference graph becomes noise.
+
+A per-document threshold based on document length would be better. Or weight by keyword specificity so rare keywords count more than common ones.
+
+## On storage
+
+Flat files work fine for now. The index fits in memory. Load times are under 10ms for a few hundred documents.
+
+SQLite becomes worth it when you need range queries or you want to update individual fields without rewriting the whole record. The current save_processed rewrites the entire JSON file on every update which is wasteful.
+
+One thing flat files do well: they are easy to inspect. Open the store directory and you can read every document directly. No tooling required. This matters for debugging.
+
+## On the API layer
+
+The API is a thin wrapper. Every handler does one thing: call the right combination of parser, validator, processor, storage. No business logic lives in api.py.
+
+The risk is that this breaks down when you need transactions. Right now parse_and_save in parser.py calls validate_document and save_parsed in sequence. If save_parsed fails after validate_document succeeds you have a partially written record. Not a problem at small scale, becomes a problem under load.
+
+## Open questions
+
+Should validation happen in the parser or as a separate step? Currently it's separate which means the parser can return invalid documents. That feels wrong but keeping them separate makes each module easier to test.
+
+Should cross-references be stored on the document or computed at query time? Storing them is fast to read but goes stale. Computing at query time is always fresh but slow for large indexes.
+
+Is the storage interface the right abstraction? Right now parser, validator, and processor all import from storage directly. A repository pattern would centralize access but adds indirection. Probably not worth it until the storage backend needs to change.

worked/example/raw/parser.py

@@ -0,0 +1,79 @@
+"""
+Parser module - reads raw input documents and converts them into
+a structured format the rest of the pipeline can work with.
+"""
+from validator import validate_document
+from storage import save_parsed
+
+
+SUPPORTED_FORMATS = ["markdown", "plaintext", "json"]
+
+
+def parse_file(path: str) -> dict:
+    """Read a file from disk and return a structured document."""
+    with open(path, "r") as f:
+        raw = f.read()
+
+    ext = path.rsplit(".", 1)[-1].lower()
+    if ext == "md":
+        doc = parse_markdown(raw)
+    elif ext == "json":
+        doc = parse_json(raw)
+    else:
+        doc = parse_plaintext(raw)
+
+    doc["source"] = path
+    return doc
+
+
+def parse_markdown(text: str) -> dict:
+    """Extract title, sections, and links from markdown."""
+    lines = text.splitlines()
+    title = ""
+    sections = []
+    links = []
+
+    for line in lines:
+        if line.startswith("# ") and not title:
+            title = line[2:].strip()
+        elif line.startswith("## "):
+            sections.append(line[3:].strip())
+        elif "](http" in line:
+            start = line.index("](") + 2
+            end = line.index(")", start)
+            links.append(line[start:end])
+
+    return {"title": title, "sections": sections, "links": links, "format": "markdown"}
+
+
+def parse_json(text: str) -> dict:
+    """Parse a JSON document into a structured dict."""
+    import json
+    data = json.loads(text)
+    return {"data": data, "format": "json"}
+
+
+def parse_plaintext(text: str) -> dict:
+    """Split plaintext into paragraphs."""
+    paragraphs = [p.strip() for p in text.split("\n\n") if p.strip()]
+    return {"paragraphs": paragraphs, "format": "plaintext"}
+
+
+def parse_and_save(path: str) -> str:
+    """Full pipeline: parse, validate, save. Returns the saved record ID."""
+    doc = parse_file(path)
+    validated = validate_document(doc)
+    record_id = save_parsed(validated)
+    return record_id
+
+
+def batch_parse(paths: list) -> list:
+    """Parse a list of files and return their record IDs."""
+    results = []
+    for path in paths:
+        try:
+            rid = parse_and_save(path)
+            results.append({"path": path, "id": rid, "ok": True})
+        except Exception as e:
+            results.append({"path": path, "error": str(e), "ok": False})
+    return results

worked/example/raw/processor.py

@@ -0,0 +1,71 @@
+"""
+Processor module - transforms validated documents into enriched records
+ready for storage and retrieval.
+"""
+import re
+from storage import load_index, save_processed
+
+
+STOPWORDS = {"the", "a", "an", "and", "or", "but", "in", "on", "at", "to", "for", "of", "with"}
+
+
+def normalize_text(text: str) -> str:
+    """Lowercase, strip extra whitespace, remove control characters."""
+    text = text.lower().strip()
+    text = re.sub(r"\s+", " ", text)
+    text = re.sub(r"[^\x20-\x7e]", "", text)
+    return text
+
+
+def extract_keywords(text: str) -> list:
+    """Pull non-stopword tokens from text, deduplicated."""
+    tokens = re.findall(r"\b[a-z]{3,}\b", normalize_text(text))
+    seen = set()
+    keywords = []
+    for t in tokens:
+        if t not in STOPWORDS and t not in seen:
+            seen.add(t)
+            keywords.append(t)
+    return keywords
+
+
+def enrich_document(doc: dict) -> dict:
+    """Add keyword index and cross-references to a validated document."""
+    text_blob = " ".join([
+        doc.get("title", ""),
+        " ".join(doc.get("sections", [])),
+        " ".join(doc.get("paragraphs", [])),
+    ])
+    doc["keywords"] = extract_keywords(text_blob)
+    doc["cross_refs"] = find_cross_references(doc)
+    return doc
+
+
+def find_cross_references(doc: dict) -> list:
+    """Look up the index and return IDs of related documents by keyword overlap."""
+    index = load_index()
+    keywords = set(doc.get("keywords", []))
+    refs = []
+    for record_id, entry in index.items():
+        other_keywords = set(entry.get("keywords", []))
+        overlap = keywords & other_keywords
+        if len(overlap) >= 3:
+            refs.append({"id": record_id, "shared_keywords": list(overlap)})
+    return refs
+
+
+def process_and_save(doc: dict) -> str:
+    """Enrich a validated document and persist it. Returns the record ID."""
+    enriched = enrich_document(doc)
+    record_id = save_processed(enriched)
+    return record_id
+
+
+def reprocess_all() -> int:
+    """Re-enrich all records in the index. Returns count of records updated."""
+    index = load_index()
+    count = 0
+    for record_id, doc in index.items():
+        process_and_save(doc)
+        count += 1
+    return count

worked/example/raw/storage.py

@@ -0,0 +1,89 @@
+"""
+Storage module - persists documents to disk and maintains the search index.
+All other modules read and write through this interface.
+"""
+import json
+import uuid
+from pathlib import Path
+
+
+STORAGE_DIR = Path(".graphify_store")
+INDEX_FILE = STORAGE_DIR / "index.json"
+
+
+def _ensure_storage() -> None:
+    STORAGE_DIR.mkdir(exist_ok=True)
+    if not INDEX_FILE.exists():
+        INDEX_FILE.write_text(json.dumps({}))
+
+
+def load_index() -> dict:
+    """Load the full document index from disk."""
+    _ensure_storage()
+    return json.loads(INDEX_FILE.read_text())
+
+
+def save_index(index: dict) -> None:
+    """Persist the index to disk."""
+    _ensure_storage()
+    INDEX_FILE.write_text(json.dumps(index, indent=2))
+
+
+def save_parsed(doc: dict) -> str:
+    """Write a parsed document to storage. Returns the assigned record ID."""
+    _ensure_storage()
+    record_id = str(uuid.uuid4())[:8]
+    path = STORAGE_DIR / f"{record_id}.json"
+    path.write_text(json.dumps(doc, indent=2))
+
+    index = load_index()
+    index[record_id] = {
+        "source": doc.get("source", ""),
+        "format": doc.get("format", ""),
+        "title": doc.get("title", ""),
+    }
+    save_index(index)
+    return record_id
+
+
+def save_processed(doc: dict) -> str:
+    """Write an enriched document to storage, updating the index with keywords."""
+    _ensure_storage()
+    record_id = doc.get("id") or str(uuid.uuid4())[:8]
+    path = STORAGE_DIR / f"{record_id}_processed.json"
+    path.write_text(json.dumps(doc, indent=2))
+
+    index = load_index()
+    if record_id not in index:
+        index[record_id] = {}
+    index[record_id]["keywords"] = doc.get("keywords", [])
+    index[record_id]["cross_refs"] = [r["id"] for r in doc.get("cross_refs", [])]
+    save_index(index)
+    return record_id
+
+
+def load_record(record_id: str) -> dict:
+    """Fetch a single document by ID."""
+    _ensure_storage()
+    path = STORAGE_DIR / f"{record_id}.json"
+    if not path.exists():
+        raise KeyError(f"No record found for ID: {record_id}")
+    return json.loads(path.read_text())
+
+
+def delete_record(record_id: str) -> bool:
+    """Remove a document and its index entry. Returns True if it existed."""
+    _ensure_storage()
+    path = STORAGE_DIR / f"{record_id}.json"
+    if not path.exists():
+        return False
+    path.unlink()
+    index = load_index()
+    index.pop(record_id, None)
+    save_index(index)
+    return True
+
+
+def list_records() -> list:
+    """Return all record IDs currently in storage."""
+    return list(load_index().keys())

worked/example/raw/validator.py

@@ -0,0 +1,61 @@
+"""
+Validator module - checks that parsed documents meet schema requirements
+before they are allowed into storage.
+"""
+from processor import normalize_text
+
+
+REQUIRED_FIELDS = {"source", "format"}
+MAX_TITLE_LENGTH = 200
+ALLOWED_FORMATS = {"markdown", "plaintext", "json"}
+
+
+class ValidationError(Exception):
+    pass
+
+
+def validate_document(doc: dict) -> dict:
+    """Run all validation checks on a parsed document. Raises ValidationError on failure."""
+    check_required_fields(doc)
+    check_format(doc)
+    doc = normalize_fields(doc)
+    return doc
+
+
+def check_required_fields(doc: dict) -> None:
+    """Raise if any required field is missing."""
+    missing = REQUIRED_FIELDS - doc.keys()
+    if missing:
+        raise ValidationError(f"Missing required fields: {missing}")
+
+
+def check_format(doc: dict) -> None:
+    """Raise if the format is not in the allowed list."""
+    fmt = doc.get("format", "")
+    if fmt not in ALLOWED_FORMATS:
+        raise ValidationError(f"Unknown format: {fmt}. Allowed: {ALLOWED_FORMATS}")
+
+
+def normalize_fields(doc: dict) -> dict:
+    """Clean up text fields using the processor."""
+    if "title" in doc:
+        doc["title"] = normalize_text(doc["title"])
+        if len(doc["title"]) > MAX_TITLE_LENGTH:
+            doc["title"] = doc["title"][:MAX_TITLE_LENGTH]
+    if "paragraphs" in doc:
+        doc["paragraphs"] = [normalize_text(p) for p in doc["paragraphs"]]
+    if "sections" in doc:
+        doc["sections"] = [normalize_text(s) for s in doc["sections"]]
+    return doc
+
+
+def validate_batch(docs: list) -> tuple:
+    """Validate a list of documents. Returns (valid_docs, errors)."""
+    valid = []
+    errors = []
+    for doc in docs:
+        try:
+            valid.append(validate_document(doc))
+        except ValidationError as e:
+            errors.append({"doc": doc.get("source", "unknown"), "error": str(e)})
+    return valid, errors