wehub-resource-sync
364 lines
15 KiB
Python
364 lines
15 KiB
Python
"""Build-time fetch and Sphinx wiring for external Anyscale example templates.
|
|
|
|
Everything here exists because the docs build pulls example templates from
|
|
``templates.ci.ray.io`` at build time (via sphinx-collections) and then renders
|
|
the fetched ``_collections/`` content. It is kept out of ``conf.py`` so that
|
|
changes to template fetching/publishing are scoped for CI and ownership, and
|
|
``conf.py`` stays focused on Sphinx behavior.
|
|
|
|
``conf.py`` consumes this module as::
|
|
|
|
from template_collections import (
|
|
collections,
|
|
collections_clean,
|
|
collections_final_clean,
|
|
)
|
|
import template_collections
|
|
|
|
exclude_patterns += template_collections.exclude_patterns()
|
|
ipython3_lexer_patterns = [*template_collections.IPYTHON3_LEXER_PATTERNS, ...]
|
|
|
|
def setup(app):
|
|
template_collections.register(app)
|
|
"""
|
|
import io
|
|
import json
|
|
import logging
|
|
import pathlib
|
|
import random
|
|
import re
|
|
import time
|
|
import zipfile
|
|
from urllib.error import HTTPError, URLError
|
|
from urllib.request import urlopen
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# -- sphinx-collections: pull external template files at build time -----------
|
|
|
|
_TEMPLATES_CI_BASE = "https://templates.ci.ray.io"
|
|
_TEMPLATE_CHANNEL_API = _TEMPLATES_CI_BASE + "/templates/{name}/latest/channel.json"
|
|
|
|
# Hard timeouts on the templates.ci.ray.io HTTP calls. The doc build previously
|
|
# stalled when the templates host was slow or unresponsive (#63112 revert);
|
|
# explicit timeouts let urlopen surface a TimeoutError instead of hanging
|
|
# indefinitely so `_fetch_and_extract_zip` fails fast with a clear error
|
|
# instead of blocking the build.
|
|
_TEMPLATE_CHANNEL_TIMEOUT_S = 30
|
|
_TEMPLATE_DOWNLOAD_TIMEOUT_S = 90
|
|
|
|
# Retry policy for the templates.ci.ray.io HTTP calls. Many doc builds can run
|
|
# concurrently against the same endpoint (PR previews + branch builds); under
|
|
# that load the host returns transient errors (HTTP 5xx/429, connection resets,
|
|
# read timeouts). A single urlopen with no retry would drop the template, and
|
|
# since every template is wired into a toctree that broken ref fails the whole
|
|
# build under `fail_on_warning`. Retry with exponential backoff plus full random
|
|
# jitter so the retry bursts from many concurrent builds don't re-synchronize
|
|
# and hammer the endpoint in lockstep.
|
|
_TEMPLATE_FETCH_ATTEMPTS = 3
|
|
_TEMPLATE_RETRY_BASE_S = 1.0
|
|
|
|
|
|
def _urlopen_read_with_retries(url, timeout):
|
|
"""Fetch `url` and return its body bytes, retrying transient failures.
|
|
|
|
Retries cover both the connection and the body read (a slow `read()` under
|
|
load can itself time out). Only transient conditions are retried — a 4xx
|
|
other than 429 won't fix itself, so it's raised immediately. After the
|
|
final attempt the last exception propagates to the caller, which aborts the
|
|
build with a clear, attributed error.
|
|
"""
|
|
last_exc = None
|
|
for attempt in range(_TEMPLATE_FETCH_ATTEMPTS):
|
|
try:
|
|
with urlopen(url, timeout=timeout) as resp:
|
|
return resp.read()
|
|
except HTTPError as exc:
|
|
last_exc = exc
|
|
# HTTPError is a subclass of URLError, so it must be caught first.
|
|
# Don't retry deterministic client errors (4xx except 429).
|
|
if exc.code != 429 and not (500 <= exc.code < 600):
|
|
raise
|
|
except (URLError, TimeoutError, OSError) as exc:
|
|
# URLError wraps DNS/refused/connection-reset; TimeoutError is the
|
|
# urlopen/read timeout; OSError covers lower-level socket errors.
|
|
last_exc = exc
|
|
if attempt < _TEMPLATE_FETCH_ATTEMPTS - 1:
|
|
base = _TEMPLATE_RETRY_BASE_S * (2 ** attempt)
|
|
delay = base + random.uniform(0, base) # full jitter on the delay
|
|
logger.info(
|
|
"sphinx-collections: retrying %s in %.1fs "
|
|
"(attempt %d/%d) after: %s",
|
|
url,
|
|
delay,
|
|
attempt + 1,
|
|
_TEMPLATE_FETCH_ATTEMPTS,
|
|
last_exc,
|
|
)
|
|
time.sleep(delay)
|
|
raise last_exc
|
|
|
|
_TEMPLATE_COLLECTIONS = {
|
|
"asynchronous_inference": {
|
|
"target": "serve/tutorials/asynchronous-inference",
|
|
},
|
|
"audio-dataset-curation-llm-judge": {
|
|
"target": "ray-overview/examples/e2e-audio",
|
|
},
|
|
"deepspeed_finetune": {
|
|
"target": "train/examples/pytorch/deepspeed_finetune",
|
|
},
|
|
"deployment-serve-llm": {
|
|
"target": "serve/tutorials/deployment-serve-llm",
|
|
},
|
|
"distributing-pytorch": {
|
|
"target": "train/examples/pytorch/distributing-pytorch",
|
|
},
|
|
"e2e-rag-deepdive": {
|
|
"target": "ray-overview/examples/e2e-rag",
|
|
},
|
|
"e2e-timeseries-forecasting": {
|
|
"target": "ray-overview/examples/e2e-timeseries",
|
|
},
|
|
"entity-recognition-with-llms": {
|
|
"target": "ray-overview/examples/entity-recognition-with-llms",
|
|
},
|
|
"image-search-and-classification": {
|
|
"target": "ray-overview/examples/e2e-multimodal-ai-workloads",
|
|
},
|
|
"llm_batch_inference_text": {
|
|
"target": "data/examples/llm_batch_inference_text",
|
|
},
|
|
"llm_batch_inference_vision": {
|
|
"target": "data/examples/llm_batch_inference_vision",
|
|
},
|
|
"langchain-agent-ray-serve": {
|
|
"target": "ray-overview/examples/langchain_agent_ray_serve/content",
|
|
},
|
|
"llm_finetuning": {
|
|
"target": "ray-overview/examples/llamafactory-llm-fine-tune",
|
|
},
|
|
"multi_agent_a2a": {
|
|
"target": "ray-overview/examples/multi_agent_a2a",
|
|
},
|
|
"mcp-ray-serve": {
|
|
"target": "ray-overview/examples/mcp-ray-serve",
|
|
},
|
|
"model-composition-recsys": {
|
|
"target": "serve/tutorials/model-composition-recsys",
|
|
},
|
|
"model-multiplexing": {
|
|
"target": "serve/tutorials/model_multiplexing_forecast",
|
|
},
|
|
"object-detection-video-processing": {
|
|
"target": "ray-overview/examples/object-detection",
|
|
},
|
|
"ray_train_workloads": {
|
|
"target": "train/tutorials",
|
|
},
|
|
"pytorch-fsdp": {
|
|
"target": "train/examples/pytorch/pytorch-fsdp",
|
|
},
|
|
"pytorch-profiling": {
|
|
"target": "train/examples/pytorch/pytorch-profiling",
|
|
},
|
|
"tensor_parallel_autotp": {
|
|
"target": "train/examples/pytorch/tensor_parallel_autotp",
|
|
},
|
|
"tensor_parallel_dtensor": {
|
|
"target": "train/examples/pytorch/tensor_parallel_dtensor",
|
|
},
|
|
"tune_pytorch_asha": {
|
|
"target": "tune/examples/tune_pytorch_asha",
|
|
},
|
|
"unstructured_data_ingestion": {
|
|
"target": "data/examples/unstructured_data_ingestion",
|
|
},
|
|
"xgboost-training-and-serving": {
|
|
"target": "ray-overview/examples/e2e-xgboost",
|
|
},
|
|
}
|
|
|
|
|
|
def _resolve_template_url(name):
|
|
"""Fetch the build zip URL for a template from the channel API."""
|
|
api_url = _TEMPLATE_CHANNEL_API.format(name=name)
|
|
logger.info("sphinx-collections: resolving template URL from %s", api_url)
|
|
data = json.loads(
|
|
_urlopen_read_with_retries(api_url, _TEMPLATE_CHANNEL_TIMEOUT_S)
|
|
)
|
|
url = data["url"]
|
|
# Replace the ascommon:/// protocol with the templates.ci.ray.io base URL.
|
|
url = url.replace("ascommon:///", _TEMPLATES_CI_BASE + "/")
|
|
# Append /build.zip to get the docs build archive.
|
|
url = url.rstrip("/") + "/build.zip"
|
|
logger.info("sphinx-collections: resolved URL %s", url)
|
|
return url
|
|
|
|
|
|
def _fetch_and_extract_zip(config):
|
|
"""Download a zip archive and extract it into the collection target directory.
|
|
|
|
A failure fetching, downloading, or extracting a template aborts the build
|
|
immediately with an error naming the template. Every template is wired into
|
|
a toctree, so a missing one fails the build anyway under `fail_on_warning`,
|
|
but as a "nonexisting document" warning emitted far downstream of the real
|
|
cause. Failing here, at the fetch, points straight at the actual problem
|
|
(the templates host or the archive) instead of a dangling toctree ref.
|
|
Transient host errors are already absorbed by the retry/backoff in
|
|
`_urlopen_read_with_retries`; this fires only once those are exhausted.
|
|
"""
|
|
import shutil
|
|
|
|
name = config["name"]
|
|
target = pathlib.Path(config["target"])
|
|
try:
|
|
url = _resolve_template_url(name)
|
|
if target.is_dir():
|
|
shutil.rmtree(target)
|
|
target.mkdir(parents=True, exist_ok=True)
|
|
logger.info("sphinx-collections: downloading %s -> %s", url, target)
|
|
zip_bytes = io.BytesIO(
|
|
_urlopen_read_with_retries(url, _TEMPLATE_DOWNLOAD_TIMEOUT_S)
|
|
)
|
|
with zipfile.ZipFile(zip_bytes) as zf:
|
|
zf.extractall(target)
|
|
logger.info(
|
|
"sphinx-collections: extracted %d files to %s",
|
|
len(zf.namelist()),
|
|
target,
|
|
)
|
|
except Exception as exc:
|
|
# Don't leave a half-extracted archive in the tree, then abort with a
|
|
# clear, attributed error. sphinx-collections runs in `safe` mode by
|
|
# default, so re-raising here surfaces as a build-halting
|
|
# CollectionsDriverError rather than a swallowed skip.
|
|
if target.is_dir():
|
|
shutil.rmtree(target, ignore_errors=True)
|
|
raise RuntimeError(
|
|
f"sphinx-collections: template {name!r} failed to fetch/extract into "
|
|
f"{target} after {_TEMPLATE_FETCH_ATTEMPTS} attempt(s): {exc}. This is "
|
|
f"a template fetch failure (templates.ci.ray.io or the archive), not a "
|
|
f"broken toctree reference — fix the fetch and rebuild; don't chase the "
|
|
f"downstream 'nonexisting document' warnings it would otherwise cause."
|
|
) from exc
|
|
|
|
|
|
collections = {
|
|
name: {
|
|
"driver": "function",
|
|
"source": _fetch_and_extract_zip,
|
|
"target": coll["target"],
|
|
"clean": False,
|
|
"final_clean": False,
|
|
"write_result": False,
|
|
}
|
|
for name, coll in _TEMPLATE_COLLECTIONS.items()
|
|
}
|
|
|
|
# Don't wipe the target before build — other docs may co-exist in parent dirs.
|
|
collections_clean = True
|
|
# Clean up collected files after build so they don't get committed.
|
|
collections_final_clean = True
|
|
|
|
# External templates fetched by sphinx_collections (see #62179) land here at
|
|
# build time; their notebook JSON has no language_info, so Sphinx defaults to
|
|
# the python3 lexer and chokes on !pip / %magic cells. conf.py splices these
|
|
# into its ipython3_lexer_patterns list alongside the in-tree content notebooks.
|
|
IPYTHON3_LEXER_PATTERNS = [
|
|
"_collections/**/*.ipynb",
|
|
]
|
|
|
|
|
|
def exclude_patterns():
|
|
"""exclude_patterns entries for the fetched ``_collections/`` content."""
|
|
return [
|
|
"_collections/serve/tutorials/deployment-serve-llm/README.*",
|
|
"_collections/serve/tutorials/deployment-serve-llm/*.ipynb",
|
|
"_collections/serve/tutorials/deployment-serve-llm/**/*.ipynb",
|
|
# Each template ships README.md + README.ipynb at the same docname; keep
|
|
# only the .md and exclude the duplicate .ipynb at the template root and
|
|
# in any sub-template directories. The template root README.md is the
|
|
# actual content page that toctrees / examples.yml refer to.
|
|
*[
|
|
pattern
|
|
for coll in _TEMPLATE_COLLECTIONS.values()
|
|
for pattern in (
|
|
f"_collections/{coll['target']}/README.ipynb",
|
|
f"_collections/{coll['target']}/**/README.ipynb",
|
|
)
|
|
],
|
|
# ray_train_workloads bundles sub-folder READMEs that aren't part of any
|
|
# toctree (only the notebooks are). Exclude them to avoid orphan warnings.
|
|
# Keep the root README.* — train.rst's toctree and tutorials button both
|
|
# link to /_collections/train/tutorials/README.
|
|
"_collections/train/tutorials/*/README.*", # one-level sidecars (getting-started, workload-patterns)
|
|
"_collections/train/tutorials/*/**/README.*", # deeper sidecars, if any
|
|
# asynchronous-inference has no docs landing page (nothing links to it), so
|
|
# exclude its README.md to avoid an orphan warning. Do NOT list a template
|
|
# that IS linked from a toctree here: README.ipynb is already globally
|
|
# excluded above, so README.md is that template's canonical page — excluding
|
|
# it deletes the page and breaks the reference (this happened to
|
|
# tune_pytorch_asha when its notebook was renamed to README.ipynb).
|
|
"_collections/serve/tutorials/asynchronous-inference/README.md",
|
|
# llamafactory: master excludes the in-tree paths only, but this branch
|
|
# also pulls a copy via sphinx-collections (see _TEMPLATE_COLLECTIONS).
|
|
# Mirror the in-tree patterns under _collections/ so the fetched copy
|
|
# is suppressed too. The template has no landing page on docs.ray.io.
|
|
"_collections/ray-overview/examples/llamafactory-llm-fine-tune/README.*",
|
|
"_collections/ray-overview/examples/llamafactory-llm-fine-tune/**/*.ipynb",
|
|
]
|
|
|
|
|
|
def register(app):
|
|
"""Connect the Sphinx hooks that handle fetched ``_collections/`` content."""
|
|
|
|
class CollectionsFootnoteFilter(logging.Filter):
|
|
# Example notebooks fetched into _collections (from templates.ci.ray.io) contain
|
|
# prose that myst-parser 5.x parses as reST footnote refs/targets, which docutils
|
|
# then flags as errors. That content is external (not in this repo), so it can't
|
|
# be fixed here -- the fix belongs upstream. Drop these specific errors for
|
|
# _collections paths so the fail_on_warning build is not blocked by fetched content.
|
|
# TODO: fix the offending footnote-like text upstream and remove this filter.
|
|
def filter(self, record):
|
|
# INFO/DEBUG records (the bulk of build output) can't be the
|
|
# footnote/target warnings below, so skip getMessage() for them.
|
|
if record.levelno < logging.WARNING:
|
|
return True
|
|
msg = record.getMessage()
|
|
if (
|
|
"autonumbered footnote references" in msg
|
|
or "Unknown target name" in msg
|
|
):
|
|
location = str(getattr(record, "location", "") or "")
|
|
if "_collections" in location:
|
|
return False
|
|
return True
|
|
|
|
logging.getLogger("sphinx").addFilter(CollectionsFootnoteFilter())
|
|
|
|
# Fix code-block language tags in _collections markdown files.
|
|
# Notebooks converted to markdown tag cells that contain a Jupyter magic or
|
|
# shell escape (e.g. ``!uv pip install ...`` / ``%matplotlib``) as
|
|
# ``python`` code blocks, which Pygments can't lex as Python and which fail
|
|
# the build under ``-W``. Re-tag any such block as ``ipython3`` so the
|
|
# python parts stay highlighted as python and ``!``/``%`` lines render as
|
|
# shell. The magic can appear anywhere in the cell (a cell often runs some
|
|
# Python and then shells out), not only on the first line.
|
|
_PY_CODE_FENCE_RE = re.compile(r"```python\n(.*?)```", re.DOTALL)
|
|
_MAGIC_LINE_RE = re.compile(r"^[ \t]*[!%]\S", re.MULTILINE)
|
|
|
|
def fix_collections_code_blocks(app, docname, source):
|
|
if not docname.startswith("_collections/"):
|
|
return
|
|
|
|
def _retag(match):
|
|
body = match.group(1)
|
|
if _MAGIC_LINE_RE.search(body):
|
|
return "```ipython3\n" + body + "```"
|
|
return match.group(0)
|
|
|
|
source[0] = _PY_CODE_FENCE_RE.sub(_retag, source[0])
|
|
|
|
app.connect('source-read', fix_collections_code_blocks)
|