"""Assert doc references into ``src/notebooklm`` stay fresh. Sibling to ``scripts/check_claude_md_freshness.py`` (which guards the ``### Repository Structure`` map in ``docs/architecture.md``). This gate turns the repo's "enforce, don't document" principle onto the *rest* of the docs: after the #1328 refactor promoted flat ``_*.py`` modules into subpackages (``_chat.py`` -> ``_chat/api.py``, ``_runtime_lifecycle.py`` -> ``_runtime/lifecycle.py``, ...), ~25 stale flat references survived across the live docs because a hand audit and a scoped doc-sync PR both missed them. A gate is the only thing that makes that class of drift un-recurrable. Two checks, both read the docs and resolve targets against the repo: **(1) Broken local-link check (strict, no allowlist).** Across ALL ``docs/**/*.md`` + root ``*.md``, every markdown link ``[text](target)`` whose ``target`` is a *relative path into* ``src/notebooklm/`` MUST resolve to an existing file. A broken link into the package is never intentional, even in an ADR or refactor-history doc. **(2) Inline module-ref check (LIVE docs only, allowlisted).** In the *live* docs (``docs/**/*.md`` + root ``*.md`` MINUS the historical-prose docs — ``docs/adr/**``, ``docs/refactor-history.md``, and ``CHANGELOG.md`` — which intentionally name historical modules in prose), every inline code span ```` `` ```` whose ```` matches a ``src/notebooklm`` module shape MUST resolve to ``src/notebooklm/``. The rare intentional historical mention in a live doc is carried in :data:`_ALLOWLIST` (shrink-only). CLAUDE.md is excluded because it is an agent-instruction file, not part of the live docs set. The detector core (:func:`find_violations`) is pure and IO-free — it takes the already-read doc text plus a ``resolver(ref) -> bool`` — so the public test and these CLI self-checks exercise the same logic, exactly like ``tests/_guardrails/test_v080_deprecation_coverage.py``. Usage: python scripts/check_docs_module_refs.py python scripts/check_docs_module_refs.py --repo-root path/to/repo Exit codes: 0 All doc module references are fresh. 1 One or more broken links or dead inline refs were found. 2 Argument error / repo root or docs tree not found. """ from __future__ import annotations import argparse import re import sys from collections.abc import Callable from dataclasses import dataclass from pathlib import Path # The package every reference resolves against. _PACKAGE_RELDIR = "src/notebooklm" # A ``src/notebooklm`` module shape: a lowercase python module (top-level name is # either a private ``_foo`` name or one of the known public module names), with # optional subdirectories, ending in ``.py``. ``test_*.py`` / ``conftest.py`` and # anything under ``tests/`` / ``scripts/`` are excluded by the caller, not here. # # Scope: covers the ``_*`` private modules, the known top-level public modules, # the ``notebooklm_cli`` entry point, and the ``rpc/`` + ``cli/`` subpackages # (broadened per review so an inline ``rpc/types.py`` / ``cli/session_cmd.py`` # ref is resolved, not silently skipped). The test # ``test_test_and_script_refs_are_not_module_shaped`` pins this scope. _MODULE_REF_RE = re.compile( r"^(_[a-z0-9_]+|client|auth|exceptions|config|io|log|migration|paths|research" r"|types|urls|utils|artifacts|notebooklm_cli|rpc|cli)([/][a-z0-9_]+)*\.py$" ) # Inline code spans: ``\`...\```. Non-greedy so adjacent spans on one line are # matched separately. _INLINE_SPAN_RE = re.compile(r"`([^`]+)`") # Markdown links: ``](target)``. The target may carry a ``#anchor`` we strip. _LINK_TARGET_RE = re.compile(r"\]\(([^)]+)\)") # Allowlist for the inline-ref check (check 2): live-doc inline mentions of a # module that no longer exists at that path but is named intentionally (a # historical / deliberately-deleted module, or a placeholder in a how-to). Keyed # by ``":"`` -> reason. This set is SHRINK-ONLY: a test asserts # every entry is still genuinely needed (the doc still mentions it AND the ref # still does not resolve), so a stale allowlist entry fails the gate. Do NOT add # an entry to silence a *real* stale path — fix the path instead. _ALLOWLIST: dict[str, str] = { # `_core.py` was the runtime compatibility shim deleted in v0.5.0. These live # docs intentionally name it to explain why `CORE_LOGGER_NAME` still reads # "notebooklm._core" (a logging compatibility contract), NOT to point at a # live module. See docs/development.md "Logger namespace compatibility". "docs/architecture.md:_core.py": "historical: deleted-in-v0.5.0 compat shim, named to explain CORE_LOGGER_NAME", "docs/configuration.md:_core.py": "historical: deleted-in-v0.5.0 compat shim, named to explain CORE_LOGGER_NAME", "docs/development.md:_core.py": "historical: deleted-in-v0.5.0 compat shim, named to explain CORE_LOGGER_NAME", # `_newfeature.py` is a placeholder in the "adding a new API class" how-to # ("Create `_newfeature.py` ..."), not a real module reference. "docs/development.md:_newfeature.py": "placeholder: example module name in the add-an-API-class how-to", } @dataclass(frozen=True) class Violation: """One dead doc reference, ``kind`` is ``"link"`` or ``"inline"``.""" kind: str doc: str # POSIX-relative path of the doc, for stable messages line: int target: str # the dead link target or inline ref def _is_local_package_link(target: str) -> bool: """True for a *relative* markdown link target into ``src/notebooklm/``. Absolute URLs (``http://...``), in-page anchors (``#...``), and links that do not descend into the package are out of scope — only relative paths whose resolved form lands inside ``src/notebooklm/`` are checked. The substring test is intentional: doc-relative targets reach the package via ``../`` prefixes (``../src/notebooklm/...``, ``../../src/notebooklm/...``). """ if target.startswith(("http://", "https://", "mailto:", "#")): return False return f"{_PACKAGE_RELDIR}/" in target def _is_module_shaped(ref: str) -> bool: """True for an inline ref that looks like an in-package module path. Excludes ``test_*.py`` / ``conftest.py`` and anything under ``tests/`` or ``scripts/`` (those are not ``src/notebooklm`` modules even though they match the ``.py`` shape). """ if "/" in ref: head = ref.split("/", 1)[0] if head in {"tests", "scripts"}: return False leaf = ref.rsplit("/", 1)[-1] if leaf.startswith("test_") or leaf == "conftest.py": return False return bool(_MODULE_REF_RE.match(ref)) def find_violations( doc_relpath: str, text: str, *, resolver: Callable[[str], bool], is_live: bool, allowlist: dict[str, str], ) -> list[Violation]: """Return every dead reference in one doc. Pure: no filesystem access. ``resolver(ref) -> bool`` answers "does ``src/notebooklm/`` exist?" for the inline check, and ``resolver(target) -> bool`` answers "does this doc-relative link target resolve?" for the link check — the CLI passes a filesystem-backed resolver, the tests pass a dict-backed stub. * Link check (always, every doc): a relative link into ``src/notebooklm/`` that does not resolve is a violation. * Inline check (live docs only): a module-shaped inline span that does not resolve to ``src/notebooklm/`` is a violation, unless an ``":"`` allowlist entry covers it. """ violations: list[Violation] = [] for lineno, line in enumerate(text.splitlines(), start=1): for match in _LINK_TARGET_RE.finditer(line): target = match.group(1).split("#", 1)[0].strip() if not target or not _is_local_package_link(target): continue if not resolver(target): violations.append(Violation("link", doc_relpath, lineno, target)) if not is_live: continue for match in _INLINE_SPAN_RE.finditer(line): ref = match.group(1) if not _is_module_shaped(ref): continue if resolver(ref): continue if f"{doc_relpath}:{ref}" in allowlist: continue violations.append(Violation("inline", doc_relpath, lineno, ref)) return violations # --- Filesystem helpers (I/O at the edge) ------------------------------------- def _is_historical_prose(rel: str) -> bool: """True for docs that intentionally name historical/old module paths in prose. These are frozen-or-by-design historical records — ADRs, the refactor history, and the CHANGELOG (whose entries describe edits to modules *as they were named at the time*, e.g. ``cli/note.py`` for a fix that predates the ``_cmd`` rename). The inline module-ref check skips them; the broken-link check still applies (a dead *link* into the package is never intentional, even in history). """ return rel.startswith("docs/adr/") or rel == "docs/refactor-history.md" or rel == "CHANGELOG.md" def _iter_docs(repo_root: Path): """Yield ``(path, relpath, is_live)`` for every doc the gate inspects. Docs = every ``docs/**/*.md`` plus every root-level ``*.md``. CLAUDE.md is excluded because it is an agent-instruction file, not part of the live docs set. ``is_live`` is False for the historical-prose docs (see :func:`_is_historical_prose`) so the inline check skips them while the link check still applies. """ docs_dir = repo_root / "docs" md_paths: list[Path] = [] if docs_dir.is_dir(): md_paths.extend(sorted(docs_dir.rglob("*.md"))) md_paths.extend(sorted(repo_root.glob("*.md"))) for path in md_paths: rel = path.relative_to(repo_root).as_posix() if rel == "CLAUDE.md": continue yield path, rel, not _is_historical_prose(rel) def _make_resolver(repo_root: Path, doc_path: Path) -> Callable[[str], bool]: """Resolver closure for one doc. For a module-shaped inline ref it checks ``src/notebooklm/``; for a doc-relative link target it resolves the target against the doc's directory. Both resolve through the same callable because the link target always contains ``src/notebooklm/`` (so it is never mistaken for a bare ref) and the inline ref never contains a path separator prefix like ``../``. """ package_root = repo_root / _PACKAGE_RELDIR def resolver(ref_or_target: str) -> bool: if _is_local_package_link(ref_or_target): return (doc_path.parent / ref_or_target).resolve().exists() return (package_root / ref_or_target).exists() return resolver def _unused_allowlist_entries( repo_root: Path, allowlist: dict[str, str], *, strict_missing: bool = False ) -> list[str]: """Return allowlist keys that are no longer justified (shrink-only guard). An entry ``":"`` is justified iff the doc still exists, still mentions ```` as a module-shaped inline span, and that ```` still does NOT resolve under ``src/notebooklm/``. The moment any of those stops being true the entry is dead weight and must be removed. ``strict_missing`` controls how a *missing* doc is treated. The default (False) skips an entry whose doc does not exist under ``repo_root`` — this keeps :func:`main` repo-root-agnostic, since the module-level allowlist keys the *real* repo's docs and a caller may point ``--repo-root`` at a synthetic tree. With ``strict_missing=True`` a missing doc IS flagged as stale; the real-repo allowlist test uses this so a renamed/deleted doc can't leave a dangling entry behind. """ package_root = repo_root / _PACKAGE_RELDIR unused: list[str] = [] for key in allowlist: doc_rel, _, ref = key.partition(":") doc_path = repo_root / doc_rel if not doc_path.is_file(): if strict_missing: unused.append(key) continue text = doc_path.read_text(encoding="utf-8") mentioned = any( ref == span and _is_module_shaped(span) for line in text.splitlines() for span in _INLINE_SPAN_RE.findall(line) ) resolves = (package_root / ref).exists() if not mentioned or resolves: unused.append(key) return unused def collect_violations(repo_root: Path) -> list[Violation]: """Read every doc and return all violations (filesystem-backed).""" violations: list[Violation] = [] for path, rel, is_live in _iter_docs(repo_root): text = path.read_text(encoding="utf-8") resolver = _make_resolver(repo_root, path) violations.extend( find_violations( rel, text, resolver=resolver, is_live=is_live, allowlist=_ALLOWLIST, ) ) return violations def main(argv: list[str] | None = None) -> int: ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("--repo-root", default=".") args = ap.parse_args(argv) repo_root = Path(args.repo_root).resolve() if not (repo_root / "docs").is_dir(): print(f"docs/ directory not found under repo root: {repo_root}", file=sys.stderr) return 2 unused = _unused_allowlist_entries(repo_root, _ALLOWLIST) violations = collect_violations(repo_root) if violations or unused: broken_links = [v for v in violations if v.kind == "link"] dead_inline = [v for v in violations if v.kind == "inline"] if broken_links: print("Broken links into src/notebooklm/:", file=sys.stderr) for v in broken_links: print(f" {v.doc}:{v.line} -> {v.target}", file=sys.stderr) if dead_inline: print("Dead inline module refs in live docs:", file=sys.stderr) for v in dead_inline: print(f" {v.doc}:{v.line} -> `{v.target}`", file=sys.stderr) if unused: print("Stale _ALLOWLIST entries (shrink-only; remove them):", file=sys.stderr) for key in unused: print(f" {key}", file=sys.stderr) return 1 print( "OK: all doc links into src/notebooklm resolve; " "all live-doc inline module refs are fresh or allowlisted" ) return 0 if __name__ == "__main__": sys.exit(main())