chore: import upstream snapshot with attribution

scripts/docs/build.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+#
+# Build the v2 documentation site for this checkout into `site/`.
+#
+# Zensical runs no MkDocs plugins or hooks, so the build is three steps:
+# materialise the API reference pages and the concrete config, build the
+# site strictly, then generate llms.txt and the per-page markdown
+# renditions. This script is the single owner of that recipe, dependency
+# sync included — CI (shared.yml, docs-preview.yml) and scripts/build-docs.sh
+# all call it. The toolchain detection in docs-preview.yml and build-docs.sh
+# keys on this file's path and expects the site under site/.
+#
+# Usage:
+#   scripts/docs/build.sh
+#
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# Snippet includes (`--8<--`) resolve against the working directory, which
+# must therefore be the repo root.
+cd "$SCRIPT_DIR/../.."
+
+uv sync --frozen --group docs
+
+# Zensical's incremental cache is unsound: a warm rebuild where only some
+# pages re-render silently drops cross-references to cache-hit pages, and
+# HTML for since-deleted pages lingers in site/. Build cold so the output
+# (and the checks below) are deterministic.
+rm -rf .cache site
+
+uv run --frozen --no-sync python scripts/docs/build_config.py
+uv run --frozen --no-sync zensical build -f mkdocs.gen.yml --strict
+
+# Zensical stays green even under --strict when a cross-reference fails to
+# resolve (rendered as literal bracket text) or an objects.inv inventory
+# fails to download (every link through it silently degrades to plain text);
+# MkDocs strict mode aborted on both. Validate the built site instead.
+uv run --frozen --no-sync python scripts/docs/check_crossrefs.py --site-dir site
+
+uv run --frozen --no-sync python scripts/docs/llms_txt.py --site-dir site

scripts/docs/build_config.py

@@ -0,0 +1,90 @@
+"""Produce the concrete Zensical build config from `mkdocs.yml`.
+
+Zensical builds from `mkdocs.yml` directly, but it has no equivalent of
+mkdocs-literate-nav: the "API Reference" navigation has to be materialised
+as explicit entries. This script regenerates the `docs/api/` tree (via
+gen_ref_pages) and writes `mkdocs.gen.yml` with the real API nav spliced
+in — that generated file is what `zensical build`/`serve` consumes.
+
+Usage:
+    python scripts/docs/build_config.py
+"""
+
+from __future__ import annotations
+
+import posixpath
+import re
+from pathlib import Path
+
+# Both scripts live in this directory, which Python puts on sys.path[0] when
+# `build_config.py` is run directly (its documented invocation).
+import gen_ref_pages
+import yaml
+
+ROOT = Path(__file__).parent.parent.parent
+
+# A scheme-prefixed nav value (https:, mailto:, ...) is an external link, not
+# a page path (same classifier as llms_txt.py; a `://` test would misread
+# scheme-only URIs as pages).
+_EXTERNAL = re.compile(r"[a-zA-Z][a-zA-Z0-9+.-]*:")
+
+
+def _nav_pages(nav: list) -> set[str]:
+    """Collect every local page reference in the nav (external links excluded)."""
+    pages: set[str] = set()
+    for entry in nav:
+        value = next(iter(entry.values())) if isinstance(entry, dict) else entry
+        if isinstance(value, list):
+            pages |= _nav_pages(value)
+        elif not _EXTERNAL.match(value):
+            pages.add(value)
+    return pages
+
+
+def _validate_nav(nav: list, docs_dir: Path) -> None:
+    """Fail on nav/page drift in either direction.
+
+    Zensical (0.0.48) ships a nav entry for a nonexistent page as a broken
+    link without any diagnostic even under --strict, and publishes a page
+    that no nav entry reaches as unreachable orphan HTML; MkDocs aborted the
+    build on both (--strict with `validation.omitted_files: warn`).
+    Validating here keeps those guarantees. The generated `api/` tree is
+    exempt from the orphan check: its nav is spliced in from the same
+    generator that writes the files, so it cannot drift.
+    """
+    pages = _nav_pages(nav)
+    # Containment before existence: `docs_dir / page` would happily resolve
+    # an absolute value or a `../` escape against the wrong root.
+    if escaping := sorted(p for p in pages if p.startswith("/") or posixpath.normpath(p).startswith("..")):
+        raise SystemExit(f"build_config: nav references pages outside docs/: {escaping}")
+    if missing := sorted(page for page in pages if not (docs_dir / page).is_file()):
+        raise SystemExit(f"build_config: nav references pages that don't exist under docs/: {missing}")
+    # Dot-directories (e.g. `.overrides` theme files) are not pages: the site
+    # builder ignores them, so the orphan check must too.
+    relative = (page.relative_to(docs_dir) for page in docs_dir.rglob("*.md"))
+    on_disk = {page.as_posix() for page in relative if not any(part.startswith(".") for part in page.parts)}
+    if orphaned := sorted(page for page in on_disk - pages if not page.startswith("api/")):
+        raise SystemExit(f"build_config: pages under docs/ that no nav entry reaches: {orphaned}")
+
+
+def build_config() -> None:
+    config = yaml.safe_load((ROOT / "mkdocs.yml").read_text(encoding="utf-8"))
+
+    api_nav = gen_ref_pages.generate()
+    if not api_nav:
+        raise SystemExit("build_config: gen_ref_pages produced no API pages — did the src/ layout move?")
+    for entry in config["nav"]:
+        if isinstance(entry, dict) and "API Reference" in entry:
+            entry["API Reference"] = api_nav
+            break
+    else:
+        raise SystemExit("build_config: no 'API Reference' entry found in mkdocs.yml nav")
+
+    _validate_nav(config["nav"], ROOT / "docs")
+
+    output = ROOT / "mkdocs.gen.yml"
+    output.write_text(yaml.safe_dump(config, sort_keys=False, allow_unicode=True), encoding="utf-8")
+
+
+if __name__ == "__main__":
+    build_config()

scripts/docs/check_crossrefs.py

@@ -0,0 +1,170 @@
+"""Fail the docs build when a page's cross-references did not resolve.
+
+Zensical (0.0.48) stays green even under `--strict` on two failure modes
+MkDocs strict mode aborted on:
+
+- An unresolvable `[text][identifier]` cross-reference renders as literal
+  bracket text (mkdocs-autorefs used to warn). The generated API index and
+  the docstring cross-references rely on such references resolving.
+- A failed `objects.inv` inventory download is logged as an ERROR record and
+  otherwise ignored, silently degrading every link through that inventory
+  (thousands of standard-library links alone) to plain text.
+
+Both are caught from the built site itself, so no log-wording change can
+disarm the check: an unresolved reference leaves a tell-tale bracket
+sequence in prose text (code blocks legitimately contain `][`, e.g. dict
+indexing, so only text outside `<pre>`/`<code>` counts), and every inventory
+declared in `mkdocs.yml` must contribute at least one resolved reference —
+an `autorefs-external` anchor, which hand-authored prose links to the same
+host never carry — to the site (an inventory that contributes none is dead
+config and fails too).
+
+Offline contributors can skip the inventory check by setting
+`DOCS_ALLOW_INVENTORY_FAILURE=1`; CI (`CI=true`) never skips it.
+
+Usage:
+    python scripts/docs/check_crossrefs.py --site-dir site
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import re
+import sys
+from html.parser import HTMLParser
+from pathlib import Path
+from urllib.parse import urlsplit
+
+import yaml
+
+ROOT = Path(__file__).parent.parent.parent
+
+# Unresolved cross-reference tell-tales in extracted prose (`\x00` marks a
+# skipped code element, see _ProseTextExtractor): the two-part
+# `[text][identifier]` reconstruction — the identifier part is always plain
+# text, so a code mark inside the second brackets means indexing prose like
+# `data[`x`][`y`]`, not a reference — and the shortcut `[`identifier`]` form,
+# which extracts as `[\x00]` unless a preceding word character or bracket
+# makes it a subscript like `list[`str`]`.
+_UNRESOLVED = re.compile(r"\]\[[^\]\s\x00]*\]|(?<![\w\]\x00])\[\x00\]")
+
+# An inventory-resolved reference renders as an anchor with the
+# `autorefs-external` class; a hand-authored prose link to the same host has
+# no autorefs class and must not satisfy the inventory check.
+_EXTERNAL_REF = re.compile(r"<a\s[^>]*autorefs-external[^>]*>")
+
+
+class _ProseTextExtractor(HTMLParser):
+    """Collect text outside <pre>/<code>/<script>/<style> elements.
+
+    A skipped element leaves a `\\x00` mark. Block-level tag boundaries break
+    the text with a newline, so bracket sequences cannot be synthesized by
+    joining text from unrelated blocks (`x]</td><td>[y`); inline tags break
+    nothing, because their text genuinely flows within one block — a
+    subscript like `**tools**[`0`]` must extract as `tools[\\x00]` so the
+    word-character carve-out in `_UNRESOLVED` still applies. A block-level
+    tag also resets the skip state, so an unclosed inline `<code>` in
+    authored raw HTML cannot hide the rest of the page.
+    """
+
+    _SKIP = frozenset({"pre", "code", "script", "style"})
+    _BLOCK = frozenset(
+        {"article", "blockquote", "caption", "dd", "details", "div", "dl", "dt", "figcaption", "figure"}
+        | {"h1", "h2", "h3", "h4", "h5", "h6", "hr", "li", "ol", "p", "section", "summary"}
+        | {"table", "tbody", "td", "tfoot", "th", "thead", "tr", "ul"}
+    )
+
+    def __init__(self) -> None:
+        super().__init__(convert_charrefs=True)
+        self._skip_depth = 0
+        self.chunks: list[str] = []
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        if tag in self._SKIP:
+            if not self._skip_depth:
+                self.chunks.append("\x00")
+            self._skip_depth += 1
+        elif tag in self._BLOCK:
+            self._skip_depth = 0
+            self.chunks.append("\n")
+        elif tag == "br" and not self._skip_depth:
+            # A line break separates text but implies nothing about open
+            # elements (`<br>` is legal inside `<code>`), so unlike block
+            # tags it must not reset the skip state.
+            self.chunks.append("\n")
+
+    def handle_endtag(self, tag: str) -> None:
+        if tag in self._SKIP:
+            if self._skip_depth:
+                self._skip_depth -= 1
+        elif tag in self._BLOCK and not self._skip_depth:
+            self.chunks.append("\n")
+
+    def handle_data(self, data: str) -> None:
+        if not self._skip_depth:
+            self.chunks.append(data)
+
+
+def unresolved_refs(html: str) -> list[str]:
+    """Return the unresolved cross-reference fragments rendered in `html`."""
+    parser = _ProseTextExtractor()
+    parser.feed(html)
+    return [fragment.replace("\x00", "<code>") for fragment in _UNRESOLVED.findall("".join(parser.chunks))]
+
+
+def _inventory_origins() -> set[str]:
+    """The scheme+host origins of the inventories declared in mkdocs.yml."""
+    config = yaml.safe_load((ROOT / "mkdocs.yml").read_text(encoding="utf-8"))
+    for plugin in config["plugins"]:
+        if isinstance(plugin, dict) and "mkdocstrings" in plugin:
+            inventories = plugin["mkdocstrings"]["handlers"]["python"].get("inventories", [])
+            return {_origin(entry["url"] if isinstance(entry, dict) else entry) for entry in inventories}
+    return set()
+
+
+def _origin(url: str) -> str:
+    parts = urlsplit(url)
+    return f"{parts.scheme}://{parts.netloc}"
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--site-dir", default=str(ROOT / "site"), help="The built site directory to scan.")
+    args = parser.parse_args()
+
+    site_dir = Path(args.site_dir)
+    # rglob on a missing directory yields nothing, which would read as a
+    # clean site (or a bogus inventory failure); fail up front instead.
+    if not site_dir.is_dir():
+        raise SystemExit(f"check_crossrefs: {site_dir} not found (run the build first)")
+
+    unlinked = _inventory_origins()
+    failures: list[str] = []
+    for page in sorted(site_dir.rglob("*.html")):
+        html = page.read_text(encoding="utf-8")
+        if unlinked and "autorefs-external" in html:
+            for tag in _EXTERNAL_REF.finditer(html):
+                unlinked -= {origin for origin in unlinked if f'href="{origin}' in tag.group(0)}
+        # Both tell-tales have a literal signature in the raw HTML ("][" for
+        # the two-part form, "[<code" for the shortcut form); skip the parse
+        # for the majority of pages that contain neither.
+        if "][" in html or "[<code" in html:
+            failures.extend(f"{page}: {fragment}" for fragment in unresolved_refs(html))
+    if failures:
+        print("error: unresolved cross-references rendered as literal text:", file=sys.stderr)
+        print("\n".join(failures[:20]), file=sys.stderr)
+        raise SystemExit(1)
+    offline_ok = os.environ.get("DOCS_ALLOW_INVENTORY_FAILURE") == "1" and os.environ.get("CI") != "true"
+    if unlinked and not offline_ok:
+        print(
+            "error: no page links into these declared inventories (download failed, or dead"
+            " inventory config?): " + ", ".join(sorted(unlinked)),
+            file=sys.stderr,
+        )
+        print("set DOCS_ALLOW_INVENTORY_FAILURE=1 to build offline", file=sys.stderr)
+        raise SystemExit(1)
+
+
+if __name__ == "__main__":
+    main()

scripts/docs/gen_ref_pages.py

@@ -0,0 +1,232 @@
+"""Generate the API reference pages and navigation.
+
+Zensical does not run MkDocs plugins, so the work that `mkdocs-gen-files` and
+`mkdocs-literate-nav` used to do at build time happens here as a plain
+pre-build step: this module writes a mkdocstrings stub (`::: <module>`) for
+every public module under `docs/api/` and returns the matching nested
+navigation, which `scripts/docs/build_config.py` splices into the build config.
+
+Run as a script it just (re)generates `docs/api/`; imported, `generate`
+also returns the nav so the config builder can consume it.
+"""
+
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+
+import griffe
+
+# A MkDocs/Zensical nav is a list of entries, each either `{title: url}` for a
+# page or `{title: [children]}` for a section (a bare `url` string attaches
+# a section index page, courtesy of the `navigation.indexes` feature).
+NavItem = "str | dict[str, str | list[NavItem]]"
+
+ROOT = Path(__file__).parent.parent.parent
+API_DIR = ROOT / "docs" / "api"
+
+# `src/mcp-types` is a distribution directory, not an import package, so each
+# package's dotted module path is taken relative to its own parent: deriving
+# it from `src/` would emit the unimportable `mcp-types.mcp_types.*`.
+PACKAGES = (ROOT / "src" / "mcp", ROOT / "src" / "mcp-types" / "mcp_types")
+
+_KIND_SECTIONS = {
+    griffe.Kind.MODULE: "Modules",
+    griffe.Kind.CLASS: "Classes",
+    griffe.Kind.FUNCTION: "Functions",
+    griffe.Kind.ATTRIBUTE: "Attributes",
+    griffe.Kind.TYPE_ALIAS: "Type aliases",
+}
+
+
+class _Node:
+    """A module (`url`) and/or a package with child modules (`children`)."""
+
+    def __init__(self) -> None:
+        self.url: str | None = None
+        self.children: dict[str, _Node] = {}
+
+    def child(self, name: str) -> _Node:
+        return self.children.setdefault(name, _Node())
+
+    def to_nav(self, title: str) -> NavItem:
+        if not self.children:
+            assert self.url is not None
+            return {title: self.url}
+        items: list[NavItem] = []
+        if self.url is not None:
+            items.append(self.url)
+        items.extend(self.children[name].to_nav(name) for name in sorted(self.children))
+        return {title: items}
+
+
+def _compact_index(module: griffe.Module, documented: set[str]) -> str | None:
+    """Build a compact page body for a module that re-exports from outside its own subtree.
+
+    mkdocstrings renders a re-export whose canonical documentation lives on
+    another page as a full duplicate of it: deterministically for aliases
+    within one top-level package (`mcp.client.auth` re-exporting from
+    `mcp.shared.auth`), and order-dependently across top-level packages
+    (`from mcp_types import y` + `__all__` renders the duplicate only when
+    the other package happens to be loaded already, and silently omits the
+    member when it isn't). Modules whose exports all live in their own
+    subtree (`mcp_types` re-exporting its private `._types` module, or a
+    module whose `__all__` lists only its own definitions) are unaffected
+    and keep the plain `::: module` stub (return `None`): their page is
+    itself the canonical rendering.
+
+    For an affected module, pin the semantics instead of inheriting the
+    accident: every export whose canonical page exists elsewhere under the API
+    reference becomes a link to it, and only exports documented nowhere else
+    (re-exports from private modules) keep their full body here, via an
+    explicit `members:` list.
+    """
+    prefix = f"{module.path}."
+    exports: dict[str, griffe.Object | griffe.Alias] = {}
+    for export in module.exports or ():
+        name = str(export)
+        # Listed exports must be statically documentable: a name provided at
+        # runtime (module `__getattr__`) is a docs error by policy, not a skip.
+        if (member := module.members.get(name)) is None:
+            msg = f"gen_ref_pages: export {module.path}.{name} is not statically visible"
+            raise SystemExit(msg)
+        exports[name] = member
+    if not any(member.is_alias and not member.target_path.startswith(prefix) for member in exports.values()):
+        return None
+
+    # A plain stub also renders the module's own public members that are not
+    # in `__all__` (`show_if_no_docstring: false` hides the docstring-less
+    # ones); keep them, so flipping a page to compact drops nothing.
+    public = dict(exports)
+    for name, member in module.members.items():
+        if (
+            name not in public
+            and not name.startswith("_")
+            and not member.is_alias
+            and member.kind is not griffe.Kind.MODULE
+            and member.has_docstrings
+        ):
+            public[name] = member
+
+    inline: list[str] = []
+    sections: dict[str, list[str]] = {}
+    for name in sorted(public, key=str.lower):
+        member = public[name]
+        try:
+            target = member.final_target if member.is_alias else member
+        except griffe.AliasResolutionError as exc:
+            msg = f"gen_ref_pages: export {module.path}.{name} resolves outside the documented packages"
+            raise SystemExit(msg) from exc
+        # Link to the anchor another page actually renders: the deepest alias
+        # hop whose module is documented (the final target may live in a
+        # private module and only be re-exported by a public one), rendered
+        # there only when docstringed (`show_if_no_docstring: false`).
+        anchor = None
+        hop = member
+        while hop.is_alias:
+            if hop.target_path.rpartition(".")[0] in documented:
+                anchor = hop.target_path
+            hop = hop.target
+        if anchor is not None and member.has_docstrings:
+            link_target = anchor
+        else:
+            inline.append(name)
+            link_target = f"{module.path}.{name}"
+        entry = f"- [`{name}`][{link_target}]"
+        if docstring := target.docstring:
+            summary = " ".join(docstring.value.split("\n\n", 1)[0].split("\n"))
+            entry += f" — {summary}"
+        sections.setdefault(_KIND_SECTIONS[target.kind], []).append(entry)
+
+    # Rendering the stub resolves the cross-package aliases again, in
+    # mkdocstrings' own collection. On a warm incremental rebuild the target
+    # package's pages can all be cache hits, so nothing else loads it and the
+    # resolution crashes (AliasResolutionError); preloading pins it. The
+    # module's own root package needs no pin: rendering the stub loads it.
+    preload = sorted(
+        {member.target_path.split(".")[0] for member in exports.values() if member.is_alias}
+        - {module.path.split(".")[0]}
+    )
+    body = [f"::: {module.path}", "    options:"]
+    if preload:
+        body += ["      preload_modules:", *(f"        - {pkg}" for pkg in preload)]
+    if inline:
+        body += ["      members:", *(f"        - {name}" for name in inline)]
+    else:
+        body += ["      members: false"]
+    body.append("")
+    for title in _KIND_SECTIONS.values():
+        if title in sections:
+            body += [f"## {title}", "", *sections[title], ""]
+    return "\n".join(body)
+
+
+def _stub(title: str, body: str) -> str:
+    """A stub page: explicit title frontmatter plus the page body.
+
+    The explicit title matters: the stubs have no H1 of their own, and a
+    title-less page falls back to "Index"/the filename — which is what
+    pruned nav rows, browser tabs, and search results show.
+    """
+    return f'---\ntitle: "{title}"\n---\n\n{body.rstrip()}\n'
+
+
+def generate() -> list[NavItem]:
+    """Write `docs/api/**.md` stubs and return the API-section navigation."""
+    if API_DIR.exists():
+        shutil.rmtree(API_DIR)
+
+    root = _Node()
+    stubs: dict[Path, str] = {}
+    pages: dict[str, Path] = {}
+    documented: set[str] = set()
+
+    for package in PACKAGES:
+        base = package.parent
+        for path in sorted(package.rglob("*.py")):
+            module_path = path.relative_to(base).with_suffix("")
+            doc_path = path.relative_to(base).with_suffix(".md")
+
+            parts = tuple(module_path.parts)
+            if parts[-1] == "__init__":
+                parts = parts[:-1]
+                doc_path = doc_path.with_name("index.md")
+            # A private component anywhere makes the module private: checking
+            # only the leaf would publish pages for e.g. mcp._vendor.util.
+            if any(part.startswith("_") for part in parts):
+                continue
+
+            ident = ".".join(parts)
+            documented.add(ident)
+            stubs[API_DIR / doc_path] = _stub(parts[-1], f"::: {ident}")
+            pages[ident] = API_DIR / doc_path
+
+            node = root
+            for part in parts:
+                node = node.child(part)
+            node.url = f"api/{doc_path.as_posix()}"
+
+    # Load the root packages before inspecting any module: aliases only
+    # resolve once the module they point at is in the loader's collection.
+    loader = griffe.GriffeLoader(search_paths=[str(package.parent) for package in PACKAGES])
+    for package in PACKAGES:
+        loader.load(package.name)
+    for ident, doc_path in pages.items():
+        try:
+            module = loader.modules_collection[ident]
+        except KeyError as exc:
+            raise SystemExit(f"gen_ref_pages: cannot find {ident} in the loaded packages") from exc
+        if not isinstance(module, griffe.Module):
+            raise SystemExit(f"gen_ref_pages: {ident} is shadowed by a non-module member")
+        if body := _compact_index(module, documented):
+            stubs[doc_path] = _stub(ident.rpartition(".")[2], body)
+
+    for full_doc_path, stub in stubs.items():
+        full_doc_path.parent.mkdir(parents=True, exist_ok=True)
+        full_doc_path.write_text(stub, encoding="utf-8")
+
+    return [root.children[name].to_nav(name) for name in sorted(root.children)]
+
+
+if __name__ == "__main__":
+    generate()

scripts/docs/llms_txt.py

@@ -0,0 +1,355 @@
+"""Generate llms.txt, llms-full.txt, and per-page markdown (https://llmstxt.org/).
+
+Zensical has no equivalent of MkDocs' build hooks, so this runs as a standalone
+post-build step over the source tree (`mkdocs.yml` + `docs/`) and writes
+three kinds of artifact into the built `site/`:
+
+- `llms.txt`: a markdown index of the documentation, one link per page,
+  grouped by nav section.
+- a `.md` rendition of every prose page next to its HTML (e.g.
+  `servers/tools/index.md`), which is what the llms.txt links point at.
+- `llms-full.txt`: every prose page concatenated for single-fetch consumption.
+
+Page markdown is the source markdown with YAML frontmatter stripped, `--8<--`
+snippet includes resolved (so the `docs_src/` code examples appear inline) and
+relative links rewritten to absolute URLs. The API reference pages under
+`api/` are mkdocstrings stubs with no prose source, so they are linked as
+rendered HTML from an Optional section instead of being embedded.
+
+Usage:
+    python scripts/docs/llms_txt.py --site-dir site
+"""
+
+from __future__ import annotations
+
+import argparse
+import posixpath
+import re
+from pathlib import Path, PurePosixPath
+from typing import Any
+
+import yaml
+
+ROOT = Path(__file__).parent.parent.parent
+DOCS = ROOT / "docs"
+
+# Pages with no markdown source, linked as HTML under "## Optional".
+_OPTIONAL_PAGES = [
+    ("api/mcp/index.md", "mcp API reference", "Auto-generated API reference for the mcp package (rendered HTML)"),
+    (
+        "api/mcp_types/index.md",
+        "mcp-types API reference",
+        "Auto-generated API reference for the mcp-types package (rendered HTML)",
+    ),
+]
+
+_SNIPPET_LINE = re.compile(r'^(?P<indent>[ \t]*)--8<-- "(?P<path>[^"\n]+)"$', flags=re.MULTILINE)
+# Every markdown link/image target: `](target#anchor "title")`. Each target is
+# classified in `_rewrite_links` — there is deliberately no shape-based
+# pre-filter here, so no link can dodge validation by its spelling. Zensical's
+# own link validation only covers .md targets (a missing image or
+# directory-style link builds green even under --strict; MkDocs failed the
+# build), so everything else is validated here.
+_LINK = re.compile(r'(\]\([ \t]*)([^)\s]+?)(#[^)\s]*)?( +(?:"[^"]*"|\'[^\']*\'|\([^()]*\)))?([ \t]*\))')
+# CommonMark forms the classifier deliberately rejects rather than models:
+# angle-bracket destinations `](<target>)` and reference-style definitions
+# `[label]: target` (footnote definitions `[^label]:` are a different,
+# supported syntax). Either would otherwise dodge validation by its spelling;
+# failing loud keeps the guarantee without modelling unused syntax.
+_ANGLE_LINK = re.compile(r"\]\([ \t]*<")
+_REF_DEFINITION = re.compile(r"^[ \t]*\[(?!\^)[^\]]+\]:", flags=re.MULTILINE)
+# Block HTML comments are inert in rendered output: python-markdown passes
+# them through verbatim, so commented-out prose must not be validated.
+_HTML_COMMENT = re.compile(r"<!--.*?-->", flags=re.DOTALL)
+# A scheme-prefixed target (https:, mailto:, tel:, ...) is external — the
+# `://` shorthand misses scheme-only URIs like mailto:.
+_EXTERNAL = re.compile(r"[a-zA-Z][a-zA-Z0-9+.-]*:")
+# Fenced code blocks and inline code spans: their content is inert in the
+# rendered HTML, so links inside them are illustrative text, neither validated
+# nor rewritten. Fences are matched line-based in `_code_intervals` (closer at
+# least as long as the opener, unclosed runs to EOF, per CommonMark) and spans
+# only in the text between fences; a span cannot cross a blank line, so a
+# stray unpaired backtick cannot swallow the paragraphs (and links) after it.
+# Known approximations of the renderer's block model: 4-space-indented
+# content is treated as prose, because in this corpus indentation is
+# admonition/list body whose links must stay validated — a link in a true
+# indented code block is over-validated (fails loud or gets rewritten in the
+# rendition), never under-validated; and span pairing is bounded by blank
+# lines rather than full block structure.
+_FENCE = re.compile(r"^[ \t]*(`{3,}|~{3,})")
+_CODE_SPAN = re.compile(r"(?s)(?<!`)(`+)(?!`)((?:(?!\n[ \t]*\n).)+?)(?<!`)\1(?!`)")
+# A leading YAML frontmatter block, as MkDocs/Zensical parse it (mkdocs.utils.meta).
+_FRONTMATTER = re.compile(r"\A---[ \t]*\n(?P<block>.*?)^(?:---|\.\.\.)[ \t]*(?:\n|\Z)", flags=re.MULTILINE | re.DOTALL)
+
+
+class _BuildError(Exception):
+    """A recoverable problem that should fail the docs build with a clear message."""
+
+
+def _dest_md_uri(src_uri: str) -> str:
+    """Map a source page (`servers/tools.md`) to its built rendition (`servers/tools/index.md`)."""
+    path = PurePosixPath(src_uri)
+    directory = path.parent if path.stem == "index" else path.parent / path.stem
+    return "index.md" if directory == PurePosixPath(".") else f"{directory}/index.md"
+
+
+def _page_url(src_uri: str) -> str:
+    """The directory URL of a page relative to the site root (`servers/tools/`, `""` for the home page)."""
+    return _dest_md_uri(src_uri).removesuffix("index.md")
+
+
+def _split_frontmatter(text: str) -> tuple[dict[str, Any], str]:
+    """Split a leading YAML frontmatter block from a page (mirrors mkdocs.utils.meta).
+
+    Hand-rolled deliberately: mkdocs is only a transitive dependency of this
+    toolchain, so the pipeline must not import it. The hook this replaced ran
+    post-frontmatter-extraction, so renditions never contained frontmatter and
+    `meta` fed the page title and llms.txt description. A leading block that
+    isn't a YAML mapping is page content, not frontmatter; an empty block is
+    frontmatter with no meta.
+    """
+    match = _FRONTMATTER.match(text)
+    if match is None:
+        return {}, text
+    try:
+        meta = yaml.safe_load(match["block"])
+    except yaml.YAMLError:
+        return {}, text
+    if meta is not None and not isinstance(meta, dict):
+        return {}, text
+    return meta or {}, text[match.end() :].lstrip("\n")
+
+
+def _collect_pages(items: list, prose: dict[str, str | None]) -> list[str]:
+    """Collect the prose pages under a nav subtree, in nav order.
+
+    Records each page in `prose` (src_uri -> nav title, or `None` to fall
+    back to the page's H1). This is the single owner of the prose-page rule:
+    a page entry counts when it is a local .md path (external URLs render as
+    outbound nav links and are omitted, as the MkDocs pipeline did) and is not
+    part of the generated API reference.
+    """
+    pages: list[str] = []
+    for entry in items:
+        title, value = next(iter(entry.items())) if isinstance(entry, dict) else (None, entry)
+        if isinstance(value, list):
+            pages.extend(_collect_pages(value, prose))
+        elif not _EXTERNAL.match(value) and value.endswith(".md") and not value.startswith("api/"):
+            # Contained values only: an escaping entry would write its
+            # rendition outside the built site.
+            if value.startswith("/") or posixpath.normpath(value).startswith(".."):
+                raise _BuildError(f"llms_txt: nav entry {value!r} escapes docs/")
+            prose[value] = title
+            pages.append(value)
+    return pages
+
+
+def _walk_nav(nav: list, prose: dict[str, str | None], sections: list[tuple[str, list[str]]]) -> list[str]:
+    """Split the nav into a flat list of top-level pages and titled sections.
+
+    Populates `sections` ((title, [src_uri]) in nav order) and returns the
+    top-level page src_uris; page collection itself is `_collect_pages`.
+    """
+    top_level: list[str] = []
+    for entry in nav:
+        title, value = next(iter(entry.items())) if isinstance(entry, dict) else (None, entry)
+        if isinstance(value, list):
+            pages = _collect_pages(value, prose)
+            if pages:
+                assert title is not None
+                sections.append((title, pages))
+        else:
+            top_level.extend(_collect_pages([entry], prose))
+    return top_level
+
+
+def _resolve_snippets(markdown: str, src_uri: str) -> str:
+    def include(match: re.Match[str]) -> str:
+        indent, path = match["indent"], match["path"]
+        # Reject snippet paths that escape the repo root (mirrors the snippets
+        # extension's restrict_base_path).
+        resolved = (ROOT / path).resolve()
+        if not resolved.is_relative_to(ROOT.resolve()):
+            raise _BuildError(f"llms_txt: snippet path {path!r} in {src_uri} escapes the repo root")
+        try:
+            content = resolved.read_text(encoding="utf-8").rstrip("\n")
+        except OSError as exc:
+            raise _BuildError(f"llms_txt: cannot read snippet {path!r} in {src_uri}") from exc
+        if path.endswith(".py"):
+            content = f"# {path}\n{content}"
+        if indent:
+            content = "\n".join(indent + line if line else line for line in content.split("\n"))
+        return content
+
+    resolved, substitutions = _SNIPPET_LINE.subn(include, markdown)
+    if substitutions != sum("--8<--" in line for line in markdown.splitlines()):
+        raise _BuildError(f"llms_txt: unresolved snippet include in {src_uri}")
+    return resolved
+
+
+def _in_code(code: list[tuple[int, int]], position: int) -> bool:
+    """Whether `position` falls inside any code interval."""
+    return any(start <= position < end for start, end in code)
+
+
+def _prose_h1(markdown: str) -> re.Match[str] | None:
+    """The first ATX H1 outside code (at most 3 spaces of indent, per CommonMark).
+
+    Code-awareness matters: every resolved `.py` snippet starts with a
+    `# path` pointer line that must never win over the page's real H1.
+    """
+    code = _code_intervals(markdown)
+    for match in re.finditer(r"^ {0,3}# (.+)$", markdown, flags=re.MULTILINE):
+        if not _in_code(code, match.start()):
+            return match
+    return None
+
+
+def _code_intervals(markdown: str) -> list[tuple[int, int]]:
+    """The character spans of fenced code blocks and inline code spans."""
+    fences: list[tuple[int, int]] = []
+    opener = ""
+    start = offset = 0
+    for line in markdown.splitlines(keepends=True):
+        if not opener:
+            if match := _FENCE.match(line):
+                opener, start = match[1], offset
+        elif (stripped := line.strip()).startswith(opener) and set(stripped) == {opener[0]}:
+            fences.append((start, offset + len(line)))
+            opener = ""
+        offset += len(line)
+    if opener:
+        fences.append((start, len(markdown)))
+
+    intervals = list(fences)
+    previous_end = 0
+    for fence_start, fence_end in [*fences, (len(markdown), len(markdown))]:
+        segment = markdown[previous_end:fence_start]
+        for pattern in (_CODE_SPAN, _HTML_COMMENT):
+            intervals += [(previous_end + m.start(), previous_end + m.end()) for m in pattern.finditer(segment)]
+        previous_end = fence_end
+    return intervals
+
+
+def _rewrite_links(markdown: str, src_uri: str, site_url: str, prose: dict[str, str | None]) -> str:
+    src_dir = posixpath.dirname(src_uri)
+    code = _code_intervals(markdown)
+
+    rejected = ((_ANGLE_LINK, "angle-bracket link destination"), (_REF_DEFINITION, "reference-style link definition"))
+    for pattern, form in rejected:
+        for match in pattern.finditer(markdown):
+            if not _in_code(code, match.start()):
+                raise _BuildError(f"llms_txt: {form} in {src_uri} is not supported here; use a plain inline link")
+
+    def rewrite(match: re.Match[str]) -> str:
+        opening, target, anchor, title, closing = match.groups()
+        if target.startswith("#") or _EXTERNAL.match(target):
+            return match.group(0)  # in-page anchor or external URL (https:, mailto:, ...)
+        if _in_code(code, match.start()):
+            return match.group(0)  # illustrative link inside a code block/span
+        if target.startswith("/"):
+            raise _BuildError(f"llms_txt: absolute link target {target!r} in {src_uri}: link the .md source instead")
+        linked = posixpath.normpath(posixpath.join(src_dir, target))
+        if linked == ".." or linked.startswith("../"):
+            raise _BuildError(f"llms_txt: link target {target!r} in {src_uri} escapes docs/")
+        if (DOCS / linked).is_dir():
+            raise _BuildError(
+                f"llms_txt: directory-style link target {target!r} in {src_uri}: link the page's .md source instead"
+            )
+        if not (DOCS / linked).is_file():
+            raise _BuildError(f"llms_txt: cannot resolve link target {target!r} in {src_uri}")
+        if linked.endswith(".md"):
+            # Pages without a markdown rendition (the api/ stubs) link to their HTML instead.
+            url = _dest_md_uri(linked) if linked in prose else _page_url(linked)
+        else:
+            url = linked  # assets are published at their docs-relative path
+        return f"{opening}{site_url}{url}{anchor or ''}{title or ''}{closing}"
+
+    return _LINK.sub(rewrite, markdown)
+
+
+def _title(src_uri: str, nav_title: str | None, meta: dict[str, Any], body: str) -> str:
+    if nav_title is not None:
+        return nav_title
+    if isinstance(meta_title := meta.get("title"), str):
+        return meta_title
+    if match := _prose_h1(body):
+        return match.group(1).strip()
+    raise _BuildError(f"llms_txt: page {src_uri} has no nav title, no title frontmatter, and no H1")
+
+
+def generate(site_dir: Path) -> None:
+    if not (DOCS / "api").is_dir():
+        raise _BuildError("llms_txt: docs/api not found (run gen_ref_pages first)")
+
+    config = yaml.safe_load((ROOT / "mkdocs.yml").read_text(encoding="utf-8"))
+    site_url = config["site_url"].rstrip("/") + "/"
+
+    prose: dict[str, str | None] = {}
+    sections: list[tuple[str, list[str]]] = []
+    top_level = _walk_nav(config["nav"], prose, sections)
+    ordered: list[tuple[str, list[str]]] = ([("Docs", top_level)] if top_level else []) + sections
+
+    rendered: dict[str, str] = {}
+    metas: dict[str, dict[str, Any]] = {}
+    for src_uri in prose:
+        metas[src_uri], markdown = _split_frontmatter((DOCS / src_uri).read_text(encoding="utf-8"))
+        markdown = _resolve_snippets(markdown, src_uri)
+        rendered[src_uri] = _rewrite_links(markdown, src_uri, site_url, prose)
+
+    index = [f"# {config['site_name']}", "", f"> {config['site_description']}", ""]
+    full: list[str] = []
+    for section_title, pages in ordered:
+        index += [f"## {section_title}", ""]
+        for src_uri in pages:
+            markdown = rendered[src_uri]
+            md_uri = _dest_md_uri(src_uri)
+            (site_dir / md_uri).parent.mkdir(parents=True, exist_ok=True)
+            (site_dir / md_uri).write_text(markdown, encoding="utf-8")
+
+            title = _title(src_uri, prose[src_uri], metas[src_uri], markdown)
+            description = metas[src_uri].get("description")
+            tail = f": {description}" if description else ""
+            index.append(f"- [{title}]({site_url}{md_uri}){tail}")
+
+            # `full` re-titles every page, so drop its first prose H1 (the
+            # same one `_title` falls back to).
+            h1 = _prose_h1(markdown)
+            body = markdown if h1 is None else markdown[: h1.start()] + markdown[h1.end() :]
+            full += [f"# {title}", "", f"Source: {site_url}{_page_url(src_uri)}", "", body.strip(), ""]
+        index.append("")
+
+    index += ["## Optional", ""]
+    # _OPTIONAL_PAGES must match the generated package indexes exactly: a
+    # package added to gen_ref_pages.PACKAGES without an entry here would be
+    # published on the site but silently missing from llms.txt, and a stale
+    # entry would link a page that no longer exists.
+    generated = {f"api/{path.name}/index.md" for path in (DOCS / "api").iterdir() if path.is_dir()}
+    listed = {src_uri for src_uri, _, _ in _OPTIONAL_PAGES}
+    if generated != listed:
+        raise _BuildError(
+            f"llms_txt: _OPTIONAL_PAGES out of sync with docs/api:"
+            f" missing {sorted(generated - listed)}, stale {sorted(listed - generated)}"
+        )
+    for src_uri, title, description in _OPTIONAL_PAGES:
+        index.append(f"- [{title}]({site_url}{_page_url(src_uri)}): {description}")
+    index.append("")
+
+    (site_dir / "llms.txt").write_text("\n".join(index), encoding="utf-8")
+    (site_dir / "llms-full.txt").write_text("\n".join(full), encoding="utf-8")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--site-dir", default=str(ROOT / "site"), help="The built site directory to write into.")
+    args = parser.parse_args()
+    try:
+        generate(Path(args.site_dir))
+    except _BuildError as exc:
+        raise SystemExit(str(exc)) from exc
+    except OSError as exc:
+        raise SystemExit(f"llms_txt: {exc}") from exc
+
+
+if __name__ == "__main__":
+    main()