chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,922 @@
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import sys
|
||||
from datetime import datetime
|
||||
from dataclasses import is_dataclass
|
||||
from importlib import import_module
|
||||
from typing import Any, Dict
|
||||
|
||||
import sphinx
|
||||
from docutils import nodes
|
||||
from jinja2.filters import FILTERS
|
||||
from sphinx.ext.autosummary import generate
|
||||
from sphinx.util.inspect import safe_getattr
|
||||
from sphinx.util.matching import compile_matchers
|
||||
|
||||
DEFAULT_API_GROUP = "Others"
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
sys.path.insert(0, os.path.abspath("."))
|
||||
from custom_directives import ( # noqa
|
||||
DownloadAndPreprocessEcosystemDocs,
|
||||
update_context,
|
||||
LinkcheckSummarizer,
|
||||
parse_navbar_config,
|
||||
setup_context,
|
||||
pregenerate_example_rsts,
|
||||
generate_versions_json,
|
||||
collect_example_orphans,
|
||||
)
|
||||
|
||||
# If extensions (or modules to document with autodoc) are in another directory,
|
||||
# add these directories to sys.path here. If the directory is relative to the
|
||||
# documentation root, use os.path.abspath to make it absolute, like shown here.
|
||||
assert not os.path.exists(
|
||||
"../../python/ray/_raylet.so"
|
||||
), "_raylet.so should not be imported for the purpose for doc build, please rename the file to _raylet.so.bak and try again."
|
||||
sys.path.insert(0, os.path.abspath("../../python/"))
|
||||
|
||||
# -- General configuration ------------------------------------------------
|
||||
|
||||
# This setting controls how single backticks are handled by sphinx. Developers
|
||||
# are used to using single backticks for code, but RST syntax requires that code
|
||||
# code to be denoted with _double_ backticks.
|
||||
# Here we make sphinx treat single backticks as code also, because everyone is
|
||||
# used to using single backticks as is done with markdown; without this setting,
|
||||
# lots of documentation ends up getting committed with single backticks anyway,
|
||||
# so we might as well make it work as developers intend for it to.
|
||||
default_role = "code"
|
||||
|
||||
sys.path.append(os.path.abspath("./_ext"))
|
||||
|
||||
extensions = [
|
||||
"callouts", # custom extension from _ext folder
|
||||
"queryparamrefs",
|
||||
"api_sidebar", # APIs tab: shared client-side API nav (see _ext/api_sidebar.py)
|
||||
"sphinx.ext.autodoc",
|
||||
"sphinx.ext.viewcode",
|
||||
"sphinx.ext.napoleon",
|
||||
"sphinx_click.ext",
|
||||
"sphinx-jsonschema",
|
||||
"sphinxemoji.sphinxemoji",
|
||||
"sphinx_copybutton",
|
||||
"sphinx_sitemap",
|
||||
"myst_nb",
|
||||
"sphinx.ext.doctest",
|
||||
"sphinx.ext.coverage",
|
||||
"sphinx.ext.autosummary",
|
||||
"sphinxcontrib.autodoc_pydantic",
|
||||
"sphinxcontrib.redoc",
|
||||
"sphinx_remove_toctrees",
|
||||
"sphinx_design",
|
||||
"sphinx.ext.intersphinx",
|
||||
"sphinx_docsearch",
|
||||
"sphinx_collections",
|
||||
"sphinx_llms_txt",
|
||||
"sphinxext.opengraph",
|
||||
]
|
||||
|
||||
# -- sphinx-llms-txt: agent-friendly summary and full corpus -----------
|
||||
|
||||
llms_txt_summary = (
|
||||
"Ray is an open-source unified compute framework for scaling AI and "
|
||||
"Python workloads, including data processing, model training, model "
|
||||
"serving, hyperparameter tuning, and reinforcement learning. The full "
|
||||
"documentation lives at https://docs.ray.io/."
|
||||
)
|
||||
|
||||
# Filter low-signal pages from llms-full.txt. Auto-generated API reference
|
||||
# pages (one per public class/method) are excluded because they would
|
||||
# dominate the corpus with autodoc boilerplate. Mirrors the directories in
|
||||
# `remove_from_toctrees` below. Agents needing specific API details can
|
||||
# fetch per-page markdown twins via Read the Docs' Markdown for Agents
|
||||
# content negotiation. Tuning of this list is tracked separately.
|
||||
llms_txt_exclude = [
|
||||
"search",
|
||||
"genindex",
|
||||
"404",
|
||||
"_TableOfContents",
|
||||
"cluster/running-applications/job-submission/doc/*",
|
||||
"ray-observability/reference/doc/*",
|
||||
"ray-core/api/doc/*",
|
||||
"data/api/doc/*",
|
||||
"train/api/doc/*",
|
||||
"tune/api/doc/*",
|
||||
"serve/api/doc/*",
|
||||
"rllib/package_ref/*",
|
||||
]
|
||||
|
||||
# Exclude Jupyter notebooks from llms-full.txt. sphinx-llms-txt reads each
|
||||
# docname's source verbatim from `_sources/`, so for `.ipynb` pages it
|
||||
# appends raw notebook JSON (cells, outputs, embedded base64 images) into
|
||||
# the corpus. `llms_txt_exclude` matches docnames (extension stripped) via
|
||||
# fnmatch, so a `**/*.ipynb` pattern can't work — we enumerate each
|
||||
# notebook's docname instead. Notebooks remain fully rendered in the HTML
|
||||
# build; only the agent corpus drops them.
|
||||
_conf_dir = pathlib.Path(__file__).parent
|
||||
llms_txt_exclude += sorted(
|
||||
p.relative_to(_conf_dir).with_suffix("").as_posix()
|
||||
for p in _conf_dir.rglob("*.ipynb")
|
||||
)
|
||||
|
||||
# -- sphinx-collections: pull external template files at build time -----------
|
||||
# The fetch machinery, template registry, collections config, and _collections/
|
||||
# Sphinx wiring live in template_collections.py so template-publishing changes
|
||||
# stay scoped away from Sphinx config. See that module.
|
||||
from template_collections import (
|
||||
collections,
|
||||
collections_clean,
|
||||
collections_final_clean,
|
||||
)
|
||||
import template_collections
|
||||
|
||||
# The collections config contains a function reference (for the "function" driver)
|
||||
# which Sphinx cannot pickle for caching. This is harmless — suppress the warning
|
||||
# so it doesn't cause a build failure under -W (warnings-as-errors).
|
||||
suppress_warnings = [
|
||||
"config.cache",
|
||||
# sphinxcontrib-redoc (unmaintained, 1.6.0) redundantly copies its bundled
|
||||
# redoc.js asset; Sphinx 8's new copy_overwrite check flags the second copy over
|
||||
# the existing (identical) file. Benign and not fixable upstream.
|
||||
"misc.copy_overwrite",
|
||||
]
|
||||
# Disable autodoc_pydantic features that can produce empty raw directives
|
||||
# (e.g. when schema JSON fails for models with non-serializable fields)
|
||||
autodoc_pydantic_model_show_json = False
|
||||
|
||||
# Configuration for algolia
|
||||
# Note: This API key grants read access to our indexes and is intended to be public.
|
||||
# See https://www.algolia.com/doc/guides/security/api-keys/ for more information.
|
||||
docsearch_app_id = "LBHF0PABBL"
|
||||
docsearch_api_key = "6c42f30d9669d8e42f6fc92f44028596"
|
||||
docsearch_index_name = "docs-ray"
|
||||
|
||||
# Remove the per-symbol autogenerated API reference pages (one page per
|
||||
# class/method) from the rendered toctree via sphinx-remove-toctrees, so the
|
||||
# navigation sidebar isn't swamped by thousands of API stubs. The pages are
|
||||
# still generated and linked from the autosummary tables; this only drops them
|
||||
# from the nav tree. These API-ref directories mirror the API-ref entries in
|
||||
# `llms_txt_exclude` above, which excludes the same pages from the agent corpus.
|
||||
remove_from_toctrees = [
|
||||
"cluster/running-applications/job-submission/doc/*",
|
||||
"ray-observability/reference/doc/*",
|
||||
"ray-core/api/doc/*",
|
||||
"data/api/doc/*",
|
||||
"train/api/doc/*",
|
||||
"tune/api/doc/*",
|
||||
"serve/api/doc/*",
|
||||
"rllib/package_ref/algorithm/*",
|
||||
"rllib/package_ref/policy/*",
|
||||
"rllib/package_ref/models/*",
|
||||
"rllib/package_ref/catalogs/*",
|
||||
"rllib/package_ref/rl_modules/*",
|
||||
"rllib/package_ref/learner/*",
|
||||
"rllib/package_ref/evaluation/*",
|
||||
"rllib/package_ref/replay-buffers/*",
|
||||
"rllib/package_ref/utils/*",
|
||||
]
|
||||
|
||||
myst_enable_extensions = [
|
||||
"dollarmath",
|
||||
"amsmath",
|
||||
"deflist",
|
||||
"html_admonition",
|
||||
"html_image",
|
||||
"colon_fence",
|
||||
"smartquotes",
|
||||
"replacements",
|
||||
]
|
||||
|
||||
myst_heading_anchors = 3
|
||||
|
||||
# Add these for attachment handling
|
||||
nb_render_key_pairs = {
|
||||
"html": [
|
||||
("img", ["src", "alt"]),
|
||||
]
|
||||
}
|
||||
|
||||
nb_output_folder = "_build/jupyter_execute"
|
||||
|
||||
# Make broken internal references into build time errors.
|
||||
# See https://www.sphinx-doc.org/en/master/usage/configuration.html#confval-nitpicky
|
||||
# for more information. :py:class: references are ignored due to false positives
|
||||
# arising from type annotations. See https://github.com/ray-project/ray/pull/46103
|
||||
# for additional context.
|
||||
nitpicky = True
|
||||
nitpick_ignore_regex = [
|
||||
("py:obj", "ray.actor.T"),
|
||||
("py:obj", "ray.data.aggregate.AccumulatorType"),
|
||||
("py:obj", "ray.data.aggregate.SupportsRichComparisonType"),
|
||||
("py:obj", "ray.data.aggregate.AggOutputType"),
|
||||
("py:class", ".*"),
|
||||
# Workaround for https://github.com/sphinx-doc/sphinx/issues/10974
|
||||
("py:obj", "ray\\.data\\.datasource\\.datasink\\.WriteReturnType"),
|
||||
# UnknownPreprocessorError is an internal exception not exported in public API
|
||||
("py:exc", "UnknownPreprocessorError"),
|
||||
("py:exc", "ray\\.data\\.preprocessors\\.version_support\\.UnknownPreprocessorError"),
|
||||
# TypeVar for gRPCInputStream generic type
|
||||
("py:obj", "ray\\.serve\\.grpc_util\\.T"),
|
||||
# autodoc_pydantic generates invalid py:obj refs for pydantic v2 validators
|
||||
# (e.g. "all fields", "_validate_*" references in validator docstrings)
|
||||
("py:obj", r"ray\.serve\.config\.\w+\.all fields"),
|
||||
("py:obj", r"ray\.serve\.config\.GangSchedulingConfig\._validate_runtime_failure_policy"),
|
||||
("py:obj", r"ray\.serve\.schema\.\w+\.all fields"),
|
||||
# autodoc_pydantic also emits invalid field refs for these dashboard job models.
|
||||
("py:obj", r"ray\.dashboard\.modules\.job\.pydantic_models\.(DriverInfo|JobDetails)\.\w+"),
|
||||
]
|
||||
|
||||
# Cache notebook outputs in _build/.jupyter_cache
|
||||
# To prevent notebook execution, set this to "off". To force re-execution, set this to
|
||||
# "force". To cache previous runs, set this to "cache".
|
||||
nb_execution_mode = os.getenv("RUN_NOTEBOOKS", "off")
|
||||
|
||||
# Add a render priority for doctest
|
||||
nb_mime_priority_overrides = [
|
||||
("html", "application/vnd.jupyter.widget-view+json", 10),
|
||||
("html", "application/javascript", 20),
|
||||
("html", "text/html", 30),
|
||||
("html", "image/svg+xml", 40),
|
||||
("html", "image/png", 50),
|
||||
("html", "image/jpeg", 60),
|
||||
("html", "text/markdown", 70),
|
||||
("html", "text/latex", 80),
|
||||
("html", "text/plain", 90),
|
||||
]
|
||||
|
||||
html_extra_path = ["robots.txt"]
|
||||
|
||||
html_baseurl = "https://docs.ray.io/en/latest/"
|
||||
|
||||
# `html_baseurl` already encodes `/en/latest/`, so override sphinx-sitemap's
|
||||
# default `{lang}{version}{link}` scheme to just `{link}`. Otherwise the
|
||||
# extension prepends `en/` again, producing URLs like `en/latesten/<page>`.
|
||||
sitemap_url_scheme = "{link}"
|
||||
|
||||
# sphinxext-opengraph: emit Open Graph metadata per page. Pin `ogp_site_url`
|
||||
# to `html_baseurl` so the `og:url` tag tracks the same canonical URL as
|
||||
# Sphinx's `<link rel="canonical">`. If `ogp_site_url` were left unset, the
|
||||
# extension would fall back to Read the Docs' `READTHEDOCS_CANONICAL_URL`
|
||||
# env var (set by RtD's Addons framework from the project's "Canonical
|
||||
# version" admin setting), which can diverge from `html_baseurl`. Per-page
|
||||
# `:og:description:` and `:og:image:` can still be set in individual files.
|
||||
ogp_site_url = html_baseurl
|
||||
|
||||
# This pattern matches:
|
||||
# - Python Repl prompts (">>> ") and it's continuation ("... ")
|
||||
# - Bash prompts ("$ ")
|
||||
# - IPython prompts ("In []: ", "In [999]: ") and it's continuations
|
||||
# (" ...: ", " : ")
|
||||
copybutton_prompt_text = r">>> |\.\.\. |\$ |In \[\d*\]: | {2,5}\.\.\.: | {5,8}: "
|
||||
copybutton_prompt_is_regexp = True
|
||||
|
||||
# Ignore divs with class="no-copybutton"
|
||||
copybutton_selector = "div:not(.no-copybutton) > div.highlight > pre"
|
||||
|
||||
# By default, tabs can be closed by selecting an open tab. We disable this
|
||||
# functionality with the `sphinx_tabs_disable_tab_closing` option.
|
||||
sphinx_tabs_disable_tab_closing = True
|
||||
|
||||
# Add any paths that contain templates here, relative to this directory.
|
||||
templates_path = ["_templates"]
|
||||
|
||||
# The master toctree document.
|
||||
master_doc = "index"
|
||||
|
||||
# General information about the project.
|
||||
project = "Ray"
|
||||
copyright = str(datetime.now().year) + ", The Ray Team"
|
||||
author = "The Ray Team"
|
||||
|
||||
# The version info for the project you're documenting acts as replacement for
|
||||
# |version| and |release|, and is also used in various other places throughout the
|
||||
# built documents. Retrieve the version using `find_version` rather than importing
|
||||
# directly (from ray import __version__) because initializing ray will prevent
|
||||
# mocking of certain external dependencies.
|
||||
from setup import find_version # noqa
|
||||
|
||||
release = find_version("ray", "_version.py")
|
||||
|
||||
language = "en"
|
||||
|
||||
# autogen files are only used to auto-generate public API documentation.
|
||||
# They are not included in the toctree to avoid warnings such as documents not included
|
||||
# in any toctree.
|
||||
autogen_files = [
|
||||
"data/api/_autogen.rst",
|
||||
]
|
||||
|
||||
# List of patterns, relative to source directory, that match files and
|
||||
# directories to ignore when looking for source files.
|
||||
# Also helps resolve warnings about documents not included in any toctree.
|
||||
exclude_patterns = [
|
||||
"templates/*",
|
||||
"cluster/running-applications/doc/ray.*",
|
||||
"data/api/ray.data.*.rst",
|
||||
# Hide README.md used for display on the console (anyscale templates)
|
||||
"serve/tutorials/**/content/**README.md",
|
||||
"data/examples/**/content/**README.md",
|
||||
"ray-overview/examples/**/content/**README.md",
|
||||
"ray-core/examples/**/content/**README.md",
|
||||
"train/examples/**/content/**README.md",
|
||||
"tune/examples/**/content/**README.md",
|
||||
# Other misc files (overviews, console-only examples, etc)
|
||||
"serve/tutorials/video-analysis/*.ipynb",
|
||||
# Legacy/backward compatibility
|
||||
"ray-overview/examples/**/README.md",
|
||||
"train/examples/**/README.md",
|
||||
] + template_collections.exclude_patterns() + autogen_files
|
||||
|
||||
# If "DOC_LIB" is found, only build that top-level navigation item.
|
||||
build_one_lib = os.getenv("DOC_LIB")
|
||||
|
||||
all_toc_libs = [
|
||||
f.path.strip("./") for f in os.scandir(".") if f.is_dir() and "ray-" in f.path
|
||||
]
|
||||
all_toc_libs += [
|
||||
"cluster",
|
||||
"tune",
|
||||
"data",
|
||||
"train",
|
||||
"rllib",
|
||||
"serve",
|
||||
"llm",
|
||||
"workflows",
|
||||
]
|
||||
if build_one_lib and build_one_lib in all_toc_libs:
|
||||
all_toc_libs.remove(build_one_lib)
|
||||
exclude_patterns += all_toc_libs
|
||||
|
||||
|
||||
# If true, `todo` and `todoList` produce output, else they produce nothing.
|
||||
todo_include_todos = False
|
||||
|
||||
# Do not check anchors for links because it produces many false positives
|
||||
# and is slow (it needs to download the linked website).
|
||||
linkcheck_anchors = False
|
||||
|
||||
if os.environ.get("LINKCHECK_ALL"):
|
||||
# Only check external links, i.e. the ones starting with http:// or https://.
|
||||
linkcheck_ignore = [
|
||||
r"^((?!http).)*$", # exclude links not starting with http
|
||||
"http://ala2017.it.nuigalway.ie/papers/ALA2017_Gupta.pdf", # broken
|
||||
"https://mvnrepository.com/artifact/*", # working but somehow not with linkcheck
|
||||
# This should be fixed -- is temporal the successor of cadence? Do the examples need to be updated?
|
||||
"https://github.com/serverlessworkflow/specification/blob/main/comparisons/comparison-cadence.md",
|
||||
"https://www.oracle.com/java/technologies/javase-jdk15-downloads.html", # forbidden for client
|
||||
"https://speakerdeck.com/*", # forbidden for bots
|
||||
r"https://huggingface.co/*", # seems to be flaky
|
||||
r"https://www.meetup.com/*", # seems to be flaky
|
||||
r"https://www.pettingzoo.ml/*", # seems to be flaky
|
||||
r"http://localhost[:/].*", # Ignore localhost links
|
||||
r"^http:/$", # Ignore incomplete links
|
||||
# 403 Client Error: Forbidden for url.
|
||||
# They ratelimit bots.
|
||||
"https://www.datanami.com/2018/02/01/rays-new-library-targets-high-speed-reinforcement-learning/",
|
||||
# 403 Client Error: Forbidden for url.
|
||||
# They ratelimit bots.
|
||||
"https://www.researchgate.net/publication/222573328_Stochastic_Gradient_Boosting",
|
||||
"https://www.datanami.com/2019/11/05/why-every-python-developer-will-love-ray/",
|
||||
"https://dev.mysql.com/doc/connector-python/en/",
|
||||
# Returning 522s intermittently.
|
||||
"https://lczero.org/",
|
||||
# Returns 406 but remains accessible
|
||||
"https://www.uber.com/blog/elastic-xgboost-ray/",
|
||||
# Aggressive anti-bot checks
|
||||
"https://archive.vn/*",
|
||||
"https://archive.is/*",
|
||||
# 429: Rate limited
|
||||
"https://medium.com/*",
|
||||
"https://towardsdatascience.com/*",
|
||||
]
|
||||
else:
|
||||
# Only check links that point to the ray-project org on github, since those
|
||||
# links are under our control and therefore much more likely to be real
|
||||
# issues that we need to fix if they are broken.
|
||||
linkcheck_ignore = [
|
||||
r"^(?!https://(raw\.githubusercontent|github)\.com/ray-project/).*$"
|
||||
]
|
||||
|
||||
|
||||
# -- Options for HTML output ----------------------------------------------
|
||||
def render_svg_logo(path):
|
||||
with open(pathlib.Path(__file__).parent / path, "r") as f:
|
||||
content = f.read()
|
||||
|
||||
return content
|
||||
|
||||
|
||||
# The theme to use for HTML and HTML Help pages. See the documentation for
|
||||
# a list of builtin themes.
|
||||
html_theme = "pydata_sphinx_theme"
|
||||
|
||||
# Theme options are theme-specific and customize the look and feel of a theme
|
||||
# further. For a list of options available for each theme, see the
|
||||
# documentation.
|
||||
html_theme_options = {
|
||||
"use_edit_page_button": True,
|
||||
"announcement": """Try Ray with $100 credit — <a target="_blank" href="https://console.anyscale.com/register/ha?render_flow=ray&utm_source=ray_docs&utm_medium=docs&utm_campaign=banner">Start now</a><button type="button" id="close-banner" aria-label="Close banner">×</button>""",
|
||||
"logo": {
|
||||
"svg": render_svg_logo("_static/img/ray_logo.svg"),
|
||||
},
|
||||
"navbar_start": ["navbar-ray-logo"],
|
||||
"navbar_end": [
|
||||
"theme-switcher",
|
||||
"version-switcher",
|
||||
"navbar-icon-links",
|
||||
"navbar-anyscale",
|
||||
],
|
||||
"navbar_center": ["navbar-links"],
|
||||
"navbar_align": "left",
|
||||
"secondary_sidebar_items": [
|
||||
"page-toc",
|
||||
"edit-on-github",
|
||||
],
|
||||
"content_footer_items": [
|
||||
"csat",
|
||||
],
|
||||
"navigation_depth": 4,
|
||||
"pygments_light_style": "stata-dark",
|
||||
"pygments_dark_style": "stata-dark",
|
||||
"switcher": {
|
||||
"json_url": "https://docs.ray.io/en/master/_static/versions.json",
|
||||
"version_match": os.getenv("READTHEDOCS_VERSION", "master"),
|
||||
},
|
||||
}
|
||||
|
||||
html_context = {
|
||||
"github_user": "ray-project",
|
||||
"github_repo": "ray",
|
||||
"github_version": "master",
|
||||
"doc_path": "doc/source/",
|
||||
}
|
||||
|
||||
# Pick the sidebar template by build environment: Read the Docs builds
|
||||
# (READTHEDOCS=True) use `main-sidebar-readthedocs`, all other builds use
|
||||
# `main-sidebar`. The `ray-overview/examples` gallery page renders with no
|
||||
# sidebar (empty list).
|
||||
html_sidebars = {
|
||||
"**": [
|
||||
(
|
||||
"main-sidebar-readthedocs"
|
||||
if os.getenv("READTHEDOCS") == "True"
|
||||
else "main-sidebar"
|
||||
)
|
||||
],
|
||||
"ray-overview/examples": [],
|
||||
}
|
||||
|
||||
# The name for this set of Sphinx documents. If None, it defaults to
|
||||
# "<project> v<release> documentation".
|
||||
html_title = f"Ray {release}"
|
||||
|
||||
autodoc_typehints_format = "short"
|
||||
|
||||
# The name of an image file (within the static path) to use as favicon of the
|
||||
# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32
|
||||
# pixels large.
|
||||
html_favicon = "_static/favicon.ico"
|
||||
|
||||
# Add any paths that contain custom static files (such as style sheets) here,
|
||||
# relative to this directory. They are copied after the builtin static files,
|
||||
# so a file named "default.css" will overwrite the builtin "default.css".
|
||||
html_static_path = ["_static"]
|
||||
|
||||
|
||||
# Output file base name for HTML help builder.
|
||||
htmlhelp_basename = "Raydoc"
|
||||
|
||||
# -- Options for LaTeX output ---------------------------------------------
|
||||
|
||||
latex_elements = {
|
||||
# The paper size ('letterpaper' or 'a4paper').
|
||||
# 'papersize': 'letterpaper',
|
||||
# The font size ('10pt', '11pt' or '12pt').
|
||||
# 'pointsize': '10pt',
|
||||
# Additional stuff for the LaTeX preamble.
|
||||
# 'preamble': '',
|
||||
# Latex figure (float) alignment
|
||||
# 'figure_align': 'htbp',
|
||||
}
|
||||
|
||||
latex_documents = [
|
||||
(master_doc, "Ray.tex", "Ray Documentation", author, "manual"),
|
||||
]
|
||||
|
||||
# -- Options for manual page output ---------------------------------------
|
||||
|
||||
man_pages = [(master_doc, "ray", "Ray Documentation", [author], 1)]
|
||||
|
||||
# -- Options for Texinfo output -------------------------------------------
|
||||
texinfo_documents = [
|
||||
(
|
||||
master_doc,
|
||||
"Ray",
|
||||
"Ray Documentation",
|
||||
author,
|
||||
"Ray",
|
||||
"Ray provides a simple, universal API for building distributed applications.",
|
||||
"Miscellaneous",
|
||||
),
|
||||
]
|
||||
|
||||
# Python methods should be presented in source code order
|
||||
autodoc_member_order = "bysource"
|
||||
|
||||
# Better typehint formatting (see custom.css)
|
||||
autodoc_typehints = "signature"
|
||||
|
||||
|
||||
def filter_out_undoc_class_members(member_name, class_name, module_name):
|
||||
module = import_module(module_name)
|
||||
cls = getattr(module, class_name)
|
||||
if getattr(cls, member_name).__doc__:
|
||||
return f"~{class_name}.{member_name}"
|
||||
else:
|
||||
return ""
|
||||
|
||||
|
||||
def has_public_constructor(class_name, module_name):
|
||||
cls = getattr(import_module(module_name), class_name)
|
||||
return _is_public_api(cls)
|
||||
|
||||
|
||||
def get_api_groups(method_names, class_name, module_name):
|
||||
api_groups = set()
|
||||
cls = getattr(import_module(module_name), class_name)
|
||||
for method_name in method_names:
|
||||
method = getattr(cls, method_name)
|
||||
if _is_public_api(method):
|
||||
api_groups.add(
|
||||
safe_getattr(method, "_annotated_api_group", DEFAULT_API_GROUP)
|
||||
)
|
||||
|
||||
return sorted(api_groups)
|
||||
|
||||
|
||||
def select_api_group(method_names, class_name, module_name, api_group):
|
||||
cls = getattr(import_module(module_name), class_name)
|
||||
return [
|
||||
method_name
|
||||
for method_name in method_names
|
||||
if _is_public_api(getattr(cls, method_name))
|
||||
and _is_api_group(getattr(cls, method_name), api_group)
|
||||
]
|
||||
|
||||
|
||||
def _is_public_api(obj):
|
||||
api_type = safe_getattr(obj, "_annotated_type", None)
|
||||
if not api_type:
|
||||
return False
|
||||
return api_type.value == "PublicAPI"
|
||||
|
||||
|
||||
def _is_api_group(obj, group):
|
||||
return safe_getattr(obj, "_annotated_api_group", DEFAULT_API_GROUP) == group
|
||||
|
||||
|
||||
FILTERS["filter_out_undoc_class_members"] = filter_out_undoc_class_members
|
||||
FILTERS["get_api_groups"] = get_api_groups
|
||||
FILTERS["select_api_group"] = select_api_group
|
||||
FILTERS["has_public_constructor"] = has_public_constructor
|
||||
|
||||
|
||||
def add_custom_assets(
|
||||
app: sphinx.application.Sphinx,
|
||||
pagename: str,
|
||||
templatename: str,
|
||||
context: Dict[str, Any],
|
||||
doctree: nodes.Node,
|
||||
):
|
||||
"""Add custom per-page assets.
|
||||
|
||||
See documentation on Sphinx Core Events for more information:
|
||||
https://www.sphinx-doc.org/en/master/extdev/appapi.html#sphinx-core-events
|
||||
"""
|
||||
if pagename == "index":
|
||||
app.add_css_file("css/index.css")
|
||||
app.add_js_file("js/index.js")
|
||||
return "index.html" # Use the special index.html template for this page
|
||||
|
||||
if pagename == "ray-overview/examples":
|
||||
app.add_css_file("css/examples.css")
|
||||
app.add_js_file("js/examples.js")
|
||||
return "ray-overview/examples.html"
|
||||
|
||||
if pagename in [
|
||||
"data/examples",
|
||||
"train/examples",
|
||||
"serve/examples",
|
||||
]:
|
||||
return "examples.html"
|
||||
|
||||
if pagename == "train/train":
|
||||
app.add_css_file("css/ray-train.css")
|
||||
elif pagename == "ray-overview/ray-libraries":
|
||||
app.add_css_file("css/ray-libraries.css")
|
||||
elif pagename == "ray-overview/use-cases":
|
||||
app.add_css_file("css/use_cases.css")
|
||||
|
||||
|
||||
def _autogen_apis(app: sphinx.application.Sphinx):
|
||||
"""
|
||||
Auto-generate public API documentation.
|
||||
"""
|
||||
try:
|
||||
generate.generate_autosummary_docs(
|
||||
[os.path.join(app.srcdir, file) for file in autogen_files],
|
||||
app=app,
|
||||
)
|
||||
except Exception as e:
|
||||
import warnings
|
||||
warnings.warn(f"Skipping autogen due to: {e}")
|
||||
|
||||
|
||||
def process_signature(app, what, name, obj, options, signature, return_annotation):
|
||||
# Sphinx is unable to render dataclass with factory/`field`
|
||||
# https://github.com/sphinx-doc/sphinx/issues/10893
|
||||
if what == "class" and is_dataclass(obj):
|
||||
return signature.replace("<factory>", "..."), return_annotation
|
||||
|
||||
|
||||
def setup(app):
|
||||
# Only generate versions JSON during RTD build
|
||||
if os.getenv("READTHEDOCS") == "True":
|
||||
generate_versions_json()
|
||||
|
||||
pregenerate_example_rsts(app)
|
||||
|
||||
# NOTE: 'MOCK' is a custom option we introduced to illustrate mock outputs. Since
|
||||
# `doctest` doesn't support this flag by default, `sphinx.ext.doctest` raises
|
||||
# warnings when we build the documentation.
|
||||
import doctest
|
||||
|
||||
doctest.register_optionflag("MOCK")
|
||||
app.connect("html-page-context", update_context)
|
||||
|
||||
app.add_config_value("navbar_content_path", "navbar.yml", "env")
|
||||
app.connect("config-inited", parse_navbar_config)
|
||||
app.connect("html-page-context", setup_context)
|
||||
app.connect("html-page-context", add_custom_assets)
|
||||
|
||||
# https://github.com/ines/termynal
|
||||
app.add_js_file("js/termynal.js", defer="defer")
|
||||
app.add_css_file("css/termynal.css")
|
||||
|
||||
app.add_js_file("js/custom.js", defer="defer")
|
||||
app.add_css_file("css/custom.css", priority=800)
|
||||
|
||||
app.add_js_file("js/csat.js", defer="defer")
|
||||
app.add_css_file("css/csat.css")
|
||||
|
||||
app.add_js_file("js/dismissable-banner.js", defer="defer")
|
||||
app.add_css_file("css/dismissable-banner.css")
|
||||
|
||||
base_path = pathlib.Path(__file__).parent
|
||||
github_docs = DownloadAndPreprocessEcosystemDocs(base_path)
|
||||
# Download docs from ecosystem library repos
|
||||
app.connect("builder-inited", github_docs.write_new_docs)
|
||||
# Restore original file content after build
|
||||
app.connect("build-finished", github_docs.write_original_docs)
|
||||
|
||||
# Hook into the logger used by linkcheck to display a summary at the end.
|
||||
linkcheck_summarizer = LinkcheckSummarizer()
|
||||
app.connect("builder-inited", linkcheck_summarizer.add_handler_to_linkcheck)
|
||||
app.connect("build-finished", linkcheck_summarizer.summarize)
|
||||
|
||||
# Hook into the auto generation of public apis
|
||||
app.connect("builder-inited", _autogen_apis)
|
||||
|
||||
app.connect("autodoc-process-signature", process_signature)
|
||||
|
||||
class DuplicateObjectFilter(logging.Filter):
|
||||
def filter(self, record):
|
||||
# Intentionally allow duplicate object description of ray.actor.ActorMethod.bind:
|
||||
# once in Ray Core API and once in Compiled Graph API
|
||||
if "duplicate object description of ray.actor.ActorMethod.bind" in record.getMessage():
|
||||
return False # Don't log this specific warning
|
||||
return True # Log all other warnings
|
||||
|
||||
logging.getLogger("sphinx").addFilter(DuplicateObjectFilter())
|
||||
|
||||
template_collections.register(app)
|
||||
|
||||
# Register hook to mark orphan documents
|
||||
example_orphan_documents = collect_example_orphans(app.confdir, app.srcdir)
|
||||
def mark_orphans(app, docname, _source):
|
||||
if docname in example_orphan_documents:
|
||||
app.env.metadata.setdefault(docname, {})
|
||||
app.env.metadata[docname]["orphan"] = True
|
||||
|
||||
app.connect('source-read', mark_orphans)
|
||||
|
||||
|
||||
app.add_config_value("ipython3_lexer_patterns", [], "env")
|
||||
app.add_config_value("ipython3_lexer_exclude_patterns", [], "env")
|
||||
app.connect("config-inited", _compile_pattern_matchers)
|
||||
app.connect("source-read", apply_ipython3_lexer)
|
||||
|
||||
|
||||
redoc = [
|
||||
{
|
||||
"name": "Ray Jobs API",
|
||||
"page": "cluster/running-applications/job-submission/api",
|
||||
"spec": "cluster/running-applications/job-submission/openapi.yml",
|
||||
"embed": True,
|
||||
},
|
||||
]
|
||||
|
||||
redoc_uri = "https://cdn.redoc.ly/redoc/latest/bundles/redoc.standalone.js"
|
||||
|
||||
# Override the output filenames autosummary generates for these objects.
|
||||
# `ray.serve.deployment` (the decorator) and `ray.serve.Deployment` (the class)
|
||||
# would otherwise write to filenames that collide on case-insensitive
|
||||
# filesystems, so the lowercase decorator is remapped to a distinct name.
|
||||
autosummary_filename_map = {
|
||||
"ray.serve.deployment": "ray.serve.deployment_decorator",
|
||||
"ray.serve.Deployment": "ray.serve.Deployment",
|
||||
}
|
||||
|
||||
# Mock out external dependencies here.
|
||||
|
||||
# Prefer not to mock libraries that are actually installed in the docs build
|
||||
# environment (doc/requirements-doc.lock.txt). Mocking an installed library
|
||||
# shadows the real module: an eager import in a documented class body then hits
|
||||
# the mock and aborts the whole package import as a misleading error. numpy and
|
||||
# pyarrow are installed, so they are not mocked. tensorflow is also installed (a
|
||||
# direct requirements-doc entry), but importing it for real breaks the autodoc
|
||||
# import of ray.rllib.algorithms.algorithm at build time, so it stays mocked.
|
||||
autodoc_mock_imports = [
|
||||
"aiohttp",
|
||||
"async_timeout",
|
||||
"backoff",
|
||||
"cachetools",
|
||||
"comet_ml",
|
||||
"composer",
|
||||
"cupy",
|
||||
"dask",
|
||||
"datasets",
|
||||
"fastapi",
|
||||
"filelock",
|
||||
"fsspec",
|
||||
"google",
|
||||
"grpc",
|
||||
"gymnasium",
|
||||
"horovod",
|
||||
"huggingface",
|
||||
"httpx",
|
||||
"joblib",
|
||||
"lightgbm",
|
||||
"lightgbm_ray",
|
||||
"mlflow",
|
||||
"nevergrad",
|
||||
"pandas",
|
||||
"pytorch_lightning",
|
||||
"scipy",
|
||||
"setproctitle",
|
||||
"skimage",
|
||||
"sklearn",
|
||||
"starlette",
|
||||
"tensorflow",
|
||||
"torch",
|
||||
"torchvision",
|
||||
"transformers",
|
||||
"tree",
|
||||
"typer",
|
||||
"uvicorn",
|
||||
"wandb",
|
||||
"watchfiles",
|
||||
"openai",
|
||||
"xgboost",
|
||||
"xgboost_ray",
|
||||
"psutil",
|
||||
"colorama",
|
||||
"grpc",
|
||||
"vllm",
|
||||
# Internal compiled modules
|
||||
"ray._raylet",
|
||||
"ray.core.generated",
|
||||
"ray.serve.generated",
|
||||
]
|
||||
|
||||
for mock_target in autodoc_mock_imports:
|
||||
if mock_target in sys.modules:
|
||||
logger.info(
|
||||
f"Potentially problematic mock target ({mock_target}) found; "
|
||||
"autodoc_mock_imports cannot mock modules that have already "
|
||||
"been loaded into sys.modules when the sphinx build starts."
|
||||
)
|
||||
|
||||
|
||||
# Other sphinx docs can be linked to if the appropriate URL to the docs
|
||||
# is specified in the `intersphinx_mapping` - for example, types annotations
|
||||
# that are defined in dependencies can link to their respective documentation.
|
||||
#
|
||||
# Each value is (base_url, inventory_url). A None inventory falls back to
|
||||
# <base_url>objects.inv. A few projects (pandas, scipy, tensorflow) pin an
|
||||
# explicit inventory URL because their hosted objects.inv is unreliable; the
|
||||
# ray-project/*/releases/.../object-mirror-* URLs are stable mirrors we control.
|
||||
#
|
||||
# Maintenance note: the build log emits "intersphinx inventory has moved: A -> B"
|
||||
# when A returns a redirect. Only chase it when B is another documentation URL
|
||||
# (the project relocated). Do NOT copy B when it points at a signed, expiring
|
||||
# release-assets.githubusercontent.com URL - that's just GitHub's normal redirect
|
||||
# for a releases/download/ asset, and the github.com/.../releases/download/ URL
|
||||
# is the stable one to keep.
|
||||
intersphinx_mapping = {
|
||||
"aiohttp": ("https://docs.aiohttp.org/en/stable/", None),
|
||||
"composer": ("https://docs.mosaicml.com/en/latest/", None),
|
||||
"dask": ("https://docs.dask.org/en/stable/", None),
|
||||
"datasets": ("https://huggingface.co/docs/datasets/main/en/", None),
|
||||
"distributed": ("https://distributed.dask.org/en/stable/", None),
|
||||
"grpc": ("https://grpc.github.io/grpc/python/", None),
|
||||
"gymnasium": ("https://gymnasium.farama.org/", None),
|
||||
"horovod": ("https://horovod.readthedocs.io/en/stable/", None),
|
||||
"lightgbm": ("https://lightgbm.readthedocs.io/en/latest/", None),
|
||||
"mars": ("https://mars-project.readthedocs.io/en/latest/", None),
|
||||
"modin": ("https://modin.readthedocs.io/en/stable/", None),
|
||||
"nevergrad": ("https://facebookresearch.github.io/nevergrad/", None),
|
||||
"numpy": ("https://numpy.org/doc/stable/", None),
|
||||
"pandas": (
|
||||
"https://pandas.pydata.org/pandas-docs/stable/",
|
||||
"https://github.com/ray-project/pandas/releases/download/object-mirror-0.1.0/objects.inv",
|
||||
),
|
||||
"pyarrow": ("https://arrow.apache.org/docs", None),
|
||||
"pydantic": ("https://pydantic.dev/docs/validation/latest/", None),
|
||||
"pymongoarrow": ("https://mongo-arrow.readthedocs.io/en/latest/", None),
|
||||
"pyspark": ("https://spark.apache.org/docs/latest/api/python/", None),
|
||||
"python": ("https://docs.python.org/3", None),
|
||||
"pytorch_lightning": ("https://lightning.ai/docs/pytorch/stable/", None),
|
||||
"scipy": (
|
||||
"https://docs.scipy.org/doc/scipy/",
|
||||
"https://github.com/ray-project/scipy/releases/download/object-mirror-0.1.0/objects.inv",
|
||||
),
|
||||
"sklearn": ("https://scikit-learn.org/stable/", None),
|
||||
"tensorflow": (
|
||||
"https://www.tensorflow.org/api_docs/python",
|
||||
"https://raw.githubusercontent.com/GPflow/tensorflow-intersphinx/master/tf2_py_objects.inv",
|
||||
),
|
||||
"torch": (
|
||||
"https://docs.pytorch.org/docs/stable/",
|
||||
"https://docs.pytorch.org/docs/2.7/objects.inv",
|
||||
),
|
||||
"torchvision": ("https://docs.pytorch.org/vision/stable/", None),
|
||||
"transformers": ("https://huggingface.co/docs/transformers/main/en/", None),
|
||||
}
|
||||
|
||||
intersphinx_timeout = 15
|
||||
|
||||
# Ray must not be imported in conf.py because third party modules initialized by
|
||||
# `import ray` will no be mocked out correctly. Perform a check here to ensure
|
||||
# ray is not imported by future maintainers.
|
||||
assert (
|
||||
"ray" not in sys.modules
|
||||
), "If ray is already imported, we will not render documentation correctly!"
|
||||
|
||||
os.environ["RAY_DOC_BUILD"] = "1"
|
||||
|
||||
ipython3_lexer_patterns = [
|
||||
*template_collections.IPYTHON3_LEXER_PATTERNS,
|
||||
"ray-overview/examples/**/content/**.ipynb",
|
||||
"serve/tutorials/**/content/**.ipynb",
|
||||
"data/examples/**/content/**.ipynb",
|
||||
"tune/examples/**/content/**.ipynb",
|
||||
]
|
||||
ipython3_lexer_exclude_patterns = []
|
||||
|
||||
|
||||
def _compile_pattern_matchers(app, config):
|
||||
app.ipython3_lexer_patterns = compile_matchers(
|
||||
config.ipython3_lexer_patterns or []
|
||||
)
|
||||
app.ipython3_lexer_exclude_patterns = compile_matchers(
|
||||
config.ipython3_lexer_exclude_patterns or []
|
||||
)
|
||||
|
||||
|
||||
def apply_ipython3_lexer(app, docname, source):
|
||||
"""Force the ipython3 pygments lexer on notebooks matching
|
||||
``ipython3_lexer_patterns`` (minus ``ipython3_lexer_exclude_patterns``).
|
||||
|
||||
Sphinx + myst-nb otherwise default to the python3 lexer, which fails on
|
||||
``!shell`` and ``%magic`` cells and is fatal under Readthedocs ``-W``.
|
||||
"""
|
||||
# Sphinx 8 returns a _StrPath from doc2path; coerce to str so the re-based
|
||||
# matchers (compile_matchers) and .endswith below accept it.
|
||||
doc_source = str(app.env.doc2path(docname, base=False))
|
||||
if not doc_source.endswith(".ipynb"):
|
||||
return
|
||||
if any(m(doc_source) for m in app.ipython3_lexer_exclude_patterns):
|
||||
return
|
||||
if not any(m(doc_source) for m in app.ipython3_lexer_patterns):
|
||||
return
|
||||
|
||||
notebook = json.loads(source[0])
|
||||
lang_info = notebook.setdefault("metadata", {}).setdefault("language_info", {})
|
||||
if lang_info.get("pygments_lexer") != "ipython3":
|
||||
lang_info["pygments_lexer"] = "ipython3"
|
||||
source[0] = json.dumps(notebook, ensure_ascii=False)
|
||||
Reference in New Issue
Block a user