chore: import upstream snapshot with attribution

doc/source/_ext/api_sidebar.py

@@ -0,0 +1,147 @@
+"""APIs-tab sidebar (Pattern B): a single shared API navigation, loaded client-side.
+
+Ray's global sidebar server-renders the whole toctree into *every* page. The symbol-
+level API nav has ~3k stub pages; rendering those into every page would bloat each one
+(~250 KB of sidebar) and OOM the build. Instead:
+
+  1. Capture the full ``apis/`` toctree once at ``env-updated``, *before*
+     ``sphinx_remove_toctrees`` (priority 500) prunes the stub pages, and render it to
+     a single fragment written to ``_static/api-nav.html`` at ``build-finished`` (the
+     HTML writer isn't ready until then).
+  2. The per-symbol stubs are kept OUT of the global toctree by ``remove_from_toctrees``
+     in conf.py (Ray already does this), so the global ``main-sidebar`` stays small.
+  3. On API pages, swap in a tiny container that loads the shared fragment via
+     ``_static/api-nav-loader.js`` and highlights the current page client-side.
+
+The API reference pages keep their *original* locations (``data/api/``, ``train/api/``,
+``rllib/package_ref/``, ...); they are pulled into the APIs tab purely by
+``apis/index``'s toctree, with no URL change. "Is this an API page?" is therefore
+answered by membership in those known source directories (``API_PATH_PREFIXES``) -- a
+stateless check, so it works under parallel writing (no reliance on cross-process
+state).
+
+So each API page embeds ~no navigation HTML; the nav is one browser-cached file.
+"""
+import os
+import re
+
+import bs4
+from sphinx.environment.adapters.toctree import global_toctree_for_doc
+from sphinx.util import logging as sphinx_logging
+
+# Reuse the in-repo copy rather than importing from ``pydata_sphinx_theme.toctree``:
+# that symbol isn't part of the theme's public API, so importing it directly makes
+# the docs build fragile across theme upgrades. See the docstring on the vendored copy.
+from custom_directives import add_collapse_checkboxes
+
+logger = sphinx_logging.getLogger(__name__)
+
+APIS_PREFIX = "apis"
+NAV_DOCNAME = APIS_PREFIX + "/index"
+NAV_FILENAME = "api-nav.html"
+
+# Source directories whose pages make up the APIs tab. These are aggregated by the
+# toctree in apis/index.md; pages under them get the shared API sidebar. Kept in sync
+# with that toctree. A page's docname starting with one of these (or being the APIs
+# landing itself) marks it as API content. Stateless on purpose -- html-page-context
+# can fire in worker processes under `-j`, where main-process state isn't shared.
+API_PATH_PREFIXES = (
+    "apis/",  # the APIs landing page (apis/index)
+    "data/api/",
+    "train/api/",
+    "tune/api/",
+    "serve/api/",
+    "ray-core/api/",
+    "rllib/package_ref/",
+)
+
+# Captured in the main process at env-updated and consumed in the main process at
+# build-finished (same process), so it is safe under parallel read/write.
+_state = {}
+
+
+def _capture_apis_toctree(app, env):
+    """env-updated @ priority < 500: resolve the full apis/ toctree while the stub
+    pages are still present (sphinx_remove_toctrees prunes them at priority 500)."""
+    try:
+        node = global_toctree_for_doc(
+            env, NAV_DOCNAME, app.builder,
+            collapse=False, maxdepth=6, includehidden=True, titles_only=True,
+        )
+    except Exception as exc:
+        logger.warning("[api_sidebar] could not capture apis toctree: %s", exc)
+        return
+    if node is None:
+        logger.warning("[api_sidebar] apis toctree resolved empty (is %s present?)", NAV_DOCNAME)
+        return
+    _state["node"] = node
+
+
+def _root_relative(html):
+    """global_toctree_for_doc resolves links relative to apis/index, which lives in the
+    apis/ directory (depth 1). The API docs themselves live at their original locations
+    (data/api/, train/api/, ...), so each link is ``../<path>``. Strip the single
+    leading ``../`` to make hrefs root-relative; the loader then resolves them against
+    each page's URL root."""
+    def repl(m):
+        href = m.group(1)
+        if re.match(r"^(https?:|/|#|mailto:)", href):
+            return m.group(0)
+        if href.startswith("../"):
+            href = href[3:]
+        return 'href="%s"' % href
+    return re.sub(r'href="([^"]*)"', repl, html)
+
+
+def _render_and_write(app, exc):
+    """build-finished (main process, writer ready): render the captured global toc,
+    keep ONLY the APIs section, and write it as the shared fragment.
+
+    global_toctree_for_doc returns the whole-site nav (all top-level sections), so we
+    isolate the API subtree here: because the toc was resolved for apis/index, the
+    "APIs" top-level entry is the one marked ``current`` -- we keep just its child
+    list (the libraries and their symbols). Leaving the API section is the top nav's
+    job, so nothing else belongs in this sidebar."""
+    if exc is not None or _state.get("node") is None:
+        return
+    try:
+        html = app.builder.render_partial(_state["node"])["fragment"]
+    except Exception as e:
+        logger.warning("[api_sidebar] could not render apis nav fragment: %s", e)
+        return
+    soup = bs4.BeautifulSoup(html, "html.parser")
+    apis_li = next(
+        (li for li in soup.select("li.toctree-l1") if "current" in (li.get("class") or [])),
+        None,
+    )
+    libs = apis_li.find("ul") if apis_li else None
+    if libs is None:
+        logger.warning("[api_sidebar] could not isolate the APIs section in the toc; "
+                       "fragment not written")
+        return
+    libs["class"] = "nav bd-sidenav"
+    # render_partial emits plain nested <ul>; reuse pydata's helper to add the
+    # collapsible <details>/<summary> structure (closed by default) so the loader can
+    # expand the current path and the theme's CSS styles the chevrons natively.
+    frag_soup = bs4.BeautifulSoup(str(libs), "html.parser")
+    add_collapse_checkboxes(frag_soup)
+    html = _root_relative(str(frag_soup))
+    static_dir = os.path.join(app.outdir, "_static")
+    os.makedirs(static_dir, exist_ok=True)
+    with open(os.path.join(static_dir, NAV_FILENAME), "w", encoding="utf-8") as fh:
+        fh.write(html)
+    logger.info("[api_sidebar] wrote shared apis nav fragment (%d KB, API subtree only) "
+                "to _static/%s", len(html) // 1024, NAV_FILENAME)
+
+
+def _on_html_page_context(app, pagename, templatename, context, doctree):
+    if pagename.startswith(API_PATH_PREFIXES):
+        context["sidebars"] = ["api-sidebar.html"]
+
+
+def setup(app):
+    # priority < 500 so we capture the toctree BEFORE sphinx_remove_toctrees prunes it.
+    app.connect("env-updated", _capture_apis_toctree, priority=1)
+    app.connect("build-finished", _render_and_write)
+    app.connect("html-page-context", _on_html_page_context, priority=900)
+    return {"version": "1.0", "parallel_read_safe": True, "parallel_write_safe": True}

doc/source/_ext/callouts.py

@@ -0,0 +1,199 @@
+from docutils import nodes
+
+from sphinx.util.docutils import SphinxDirective
+from sphinx.transforms import SphinxTransform
+from docutils.nodes import Node
+
+# BASE_NUM = 2775  # black circles, white numbers
+BASE_NUM = 2459  # white circle, black numbers
+
+
+class CalloutIncludePostTransform(SphinxTransform):
+    """Code block post-processor for `literalinclude` blocks used in callouts."""
+
+    default_priority = 400
+
+    def apply(self, **kwargs) -> None:
+        visitor = LiteralIncludeVisitor(self.document)
+        self.document.walkabout(visitor)
+
+
+class LiteralIncludeVisitor(nodes.NodeVisitor):
+    """Change a literal block upon visiting it."""
+
+    def __init__(self, document: nodes.document) -> None:
+        super().__init__(document)
+
+    def unknown_visit(self, node: Node) -> None:
+        pass
+
+    def unknown_departure(self, node: Node) -> None:
+        pass
+
+    def visit_document(self, node: Node) -> None:
+        pass
+
+    def depart_document(self, node: Node) -> None:
+        pass
+
+    def visit_start_of_file(self, node: Node) -> None:
+        pass
+
+    def depart_start_of_file(self, node: Node) -> None:
+        pass
+
+    def visit_literal_block(self, node: nodes.literal_block) -> None:
+        if "<1>" in node.rawsource:
+            source = str(node.rawsource)
+            for i in range(1, 20):
+                source = source.replace(
+                    f"<{i}>", chr(int(f"0x{BASE_NUM + i}", base=16))
+                )
+            node.rawsource = source
+            node[:] = [nodes.Text(source)]
+
+
+class callout(nodes.General, nodes.Element):
+    """Sphinx callout node."""
+
+    pass
+
+
+def visit_callout_node(self, node):
+    """We pass on node visit to prevent the
+    callout being treated as admonition."""
+    pass
+
+
+def depart_callout_node(self, node):
+    """Departing a callout node is a no-op, too."""
+    pass
+
+
+class annotations(nodes.Element):
+    """Sphinx annotations node."""
+
+    pass
+
+
+def _replace_numbers(content: str):
+    """
+    Replaces strings of the form <x> with circled unicode numbers (e.g. ①) as text.
+
+    Args:
+        content: Python str from a callout or annotations directive.
+
+    Returns: The formatted content string.
+    """
+    for i in range(1, 20):
+        content.replace(f"<{i}>", chr(int(f"0x{BASE_NUM + i}", base=16)))
+    return content
+
+
+def _parse_recursively(self, node):
+    """Utility to recursively parse a node from the Sphinx AST."""
+    self.state.nested_parse(self.content, self.content_offset, node)
+
+
+class CalloutDirective(SphinxDirective):
+    """Code callout directive with annotations for Sphinx.
+
+    Use this `callout` directive by wrapping either `code-block` or `literalinclude`
+    directives. Each line that's supposed to be equipped with an annotation should
+    have an inline comment of the form "# <x>" where x is an integer.
+
+    Afterwards use the `annotations` directive to add annotations to the previously
+    defined code labels ("<x>") by using the syntax "<x> my annotation" to produce an
+    annotation "my annotation" for x.
+    Note that annotation lines have to be separated by a new line, i.e.
+
+    .. annotations::
+
+        <1> First comment followed by a newline,
+
+        <2> second comment after the newline.
+
+
+    Usage example:
+    -------------
+
+    .. callout::
+
+        .. code-block:: python
+
+            from ray import tune
+            from ray.tune.search.hyperopt import HyperOptSearch
+            import keras
+
+            def objective(config):  # <1>
+                ...
+
+            search_space = {"activation": tune.choice(["relu", "tanh"])}  # <2>
+            algo = HyperOptSearch()
+
+            tuner = tune.Tuner(  # <3>
+                ...
+            )
+            results = tuner.fit()
+
+        .. annotations::
+
+            <1> Wrap a Keras model in an objective function.
+
+            <2> Define a search space and initialize the search algorithm.
+
+            <3> Start a Tune run that maximizes accuracy.
+    """
+
+    has_content = True
+
+    def run(self):
+        self.assert_has_content()
+
+        content = self.content
+        content = _replace_numbers(content)
+
+        callout_node = callout("\n".join(content))
+        _parse_recursively(self, callout_node)
+
+        return [callout_node]
+
+
+class AnnotationsDirective(SphinxDirective):
+    """Annotations directive, which is only used nested within a Callout directive."""
+
+    has_content = True
+
+    def run(self):
+        content = self.content
+        content = _replace_numbers(content)
+
+        joined_content = "\n".join(content)
+        annotations_node = callout(joined_content)
+        _parse_recursively(self, annotations_node)
+
+        return [annotations_node]
+
+
+def setup(app):
+    # Add new node types
+    app.add_node(
+        callout,
+        html=(visit_callout_node, depart_callout_node),
+        latex=(visit_callout_node, depart_callout_node),
+        text=(visit_callout_node, depart_callout_node),
+    )
+    app.add_node(annotations)
+
+    # Add new directives
+    app.add_directive("callout", CalloutDirective)
+    app.add_directive("annotations", AnnotationsDirective)
+
+    # Add post-processor
+    app.add_post_transform(CalloutIncludePostTransform)
+
+    return {
+        "version": "0.1",
+        "parallel_read_safe": True,
+        "parallel_write_safe": True,
+    }

doc/source/_ext/queryparamrefs.py

@@ -0,0 +1,122 @@
+from docutils import nodes, utils, frontend
+from docutils.parsers.rst import directives, Parser
+from sphinx.util.docutils import SphinxDirective
+
+
+class URLQueryParamRefNode(nodes.General, nodes.Element):
+    """Node type for references that need URL query parameters."""
+
+
+class WrapperNode(nodes.paragraph):
+    """A wrapper node for URL query param references.
+
+    Sphinx will not build if you don't wrap a reference in a paragraph. And a <div>
+    is needed anyway for styling purposes.
+    """
+
+    def visit(self, node):
+        if "class" not in node:
+            raise ValueError(
+                "'class' attribute must be set on the WrapperNode for a "
+                "URL query parameter reference."
+            )
+        self.body.append(self.starttag(node, "div", CLASS=node["class"]))
+
+    def depart(self, node):
+        self.body.append("</div>")
+
+
+class URLQueryParamRefDirective(SphinxDirective):
+    """Sphinx directive to insert a reference with query parameters."""
+
+    required_arguments = 1
+    optional_arguments = 0
+    final_argument_whitespace = True
+    has_content = True
+    option_spec = {
+        "parameters": directives.unchanged_required,
+        "classes": directives.class_option,
+        "ref-type": (
+            lambda choice: directives.choice(choice, ["any", "ref", "doc", "myst"])
+        ),
+    }
+
+    def run(self):
+        ref_type = self.options.get("ref-type", "any")
+        content = self.content.data
+        reftarget = directives.uri(self.arguments[0])
+        return [
+            URLQueryParamRefNode(
+                {
+                    "docname": self.env.docname,
+                    "parameters": self.options.get("parameters", None),
+                    "classes": self.options.get("classes", []),
+                    "reftarget": reftarget,
+                    "refdocname": self.env.docname,
+                    "refdomain": "std" if ref_type in {"ref", "doc"} else "",
+                    "reftype": ref_type,
+                    "refexplicit": content if content else reftarget,
+                    "refwarn": True,
+                }
+            )
+        ]
+
+
+def on_doctree_resolved(app, doctree, docname):
+    """Replace URLQueryParamRefNode instances with real references.
+
+    Any text that lives inside a URLQueryParamRefNode is parsed as usual.
+
+    Args:
+        app: Sphinx application
+        doctree: Doctree which has just been resolved
+        docname: Name of the document containing the reference nodes
+    """
+    parser = Parser()
+    for node in doctree.traverse(URLQueryParamRefNode):
+        tmp_node = utils.new_document(
+            "Content nested under URLQueryParamRefNode",
+            frontend.OptionParser(components=[Parser]).get_default_values(),
+        )
+        text = "\n".join(node.rawsource["refexplicit"])
+
+        # Parse all child RST as usual, then append any parsed nodes to the
+        # reference node.
+        parser.parse(text, tmp_node)
+        ref_node = nodes.reference(
+            rawsource=text,
+            text="",
+        )
+        for child in tmp_node.children:
+            ref_node.append(child)
+
+        # Pass all URLQueryParamRefNode attributes to ref node;
+        # possibly not necessary
+        for key, value in node.rawsource.items():
+            ref_node[key] = value
+
+        # Need to update refuri of ref node to include URL query parameters
+        ref_node["refuri"] = (
+            app.builder.get_relative_uri(
+                docname,
+                node.rawsource["reftarget"],
+            )
+            + node.rawsource["parameters"]
+        )
+
+        # Need to wrap the node in a paragraph for Sphinx to build
+        wrapper = WrapperNode()
+        wrapper["class"] = "query-param-ref-wrapper"
+        wrapper += ref_node
+        node.replace_self([wrapper])
+
+
+def setup(app):
+    app.add_directive("query-param-ref", URLQueryParamRefDirective)
+    app.connect("doctree-resolved", on_doctree_resolved)
+    app.add_node(WrapperNode, html=(WrapperNode.visit, WrapperNode.depart))
+    return {
+        "version": "0.1",
+        "parallel_read_safe": True,
+        "parallel_write_safe": True,
+    }