chore: import upstream snapshot with attribution

docs_new/scripts/gen_redirects.py

@@ -0,0 +1,227 @@
+#!/usr/bin/env python3
+"""Generate Mintlify docs.json redirects from old Sphinx paths to new Mintlify paths."""
+
+from __future__ import annotations
+
+import json
+import os
+from pathlib import Path
+
+REPO = Path(__file__).resolve().parent.parent.parent
+OLD_DOCS = REPO / "docs"
+NEW_DOCS = REPO / "docs_new" / "docs"
+
+# Directory-level renames (old → new, under /docs/ prefix)
+SECTION_RENAMES = {
+    "get_started": "get-started",
+    "platforms": "hardware-platforms",
+    "supported_models": "supported-models",
+    "diffusion": "sglang-diffusion",
+}
+
+# Explicit file-level mappings. Keys are old URL paths (no .html, with leading /).
+# Values are new URL paths (with /docs/ prefix, no extension).
+EXPLICIT = {
+    # get_started → get-started
+    "/get_started/install": "/docs/get-started/installation",
+    # developer_guide rename
+    "/developer_guide/development_jit_kernel_guide": "/docs/developer_guide/JIT_kernels",
+    # platforms → hardware-platforms (with file renames)
+    "/platforms/amd_gpu": "/docs/hardware-platforms/amd-gpus",
+    "/platforms/cpu_server": "/docs/hardware-platforms/cpu-server",
+    "/platforms/tpu": "/docs/hardware-platforms/tpu",
+    "/platforms/xpu": "/docs/hardware-platforms/xpu",
+    # platforms/ascend → hardware-platforms/ascend-npus (flattened, renamed)
+    "/platforms/ascend/ascend_npu": "/docs/hardware-platforms/ascend-npus/SGLang-installation-with-NPUs-support",
+    "/platforms/ascend/ascend_npu_best_practice": "/docs/hardware-platforms/ascend-npus/Best-Practice-on-Ascend-NPU",
+    "/platforms/ascend/ascend_npu_deepseek_example": "/docs/hardware-platforms/ascend-npus/DeepSeek-Examples",
+    "/platforms/ascend/ascend_npu_glm5_examples": "/docs/hardware-platforms/ascend-npus/GLM-5",
+    "/platforms/ascend/ascend_npu_qwen3_examples": "/docs/hardware-platforms/ascend-npus/Qwen3-Examples",
+    "/platforms/ascend/ascend_npu_qwen3_5_examples": "/docs/hardware-platforms/ascend-npus/Qwen3.5",
+    "/platforms/ascend/ascend_npu_support_features": "/docs/hardware-platforms/ascend-npus/Support-Features-on-Ascend-NPU",
+    "/platforms/ascend/ascend_npu_support_models": "/docs/hardware-platforms/ascend-npus/Support-Models-on-Ascend-NPU",
+    # Old pages dropped — redirect to section overview
+    "/platforms/ascend/ascend_contribution_guide": "/docs/hardware-platforms/overview",
+    "/platforms/ascend/ascend_npu_environment_variables": "/docs/hardware-platforms/overview",
+    "/platforms/ascend/ascend_npu_quantization": "/docs/hardware-platforms/overview",
+    "/platforms/ascend/ascend_npu_support": "/docs/hardware-platforms/overview",
+    "/platforms/ascend/mindspore_backend": "/docs/hardware-platforms/overview",
+    "/platforms/ascend_npu_ring_sp_performance": "/docs/hardware-platforms/overview",
+    "/platforms/apple_metal": "/docs/hardware-platforms/overview",
+    "/platforms/mthreads_gpu": "/docs/hardware-platforms/overview",
+    "/platforms/nvidia_jetson": "/docs/hardware-platforms/overview",
+    "/platforms/plugin": "/docs/hardware-platforms/overview",
+    # supported_models → supported-models (flattened, renamed)
+    "/supported_models": "/docs/supported-models",
+    "/supported_models/index": "/docs/supported-models",
+    "/supported_models/extending/mindspore_models": "/docs/supported-models/mindspore-models",
+    "/supported_models/extending/modelscope": "/docs/supported-models/modelscope",
+    "/supported_models/extending/support_new_models": "/docs/supported-models/new-model-support",
+    "/supported_models/extending/transformers_fallback": "/docs/supported-models/transformers-fallback",
+    "/supported_models/extending/index": "/docs/supported-models",
+    "/supported_models/retrieval_ranking/classify_models": "/docs/supported-models/classification-models",
+    "/supported_models/retrieval_ranking/embedding_models": "/docs/supported-models/embedding-models",
+    "/supported_models/retrieval_ranking/rerank_models": "/docs/supported-models/rerank-models",
+    "/supported_models/retrieval_ranking/index": "/docs/supported-models",
+    "/supported_models/specialized/reward_models": "/docs/supported-models/reward-models",
+    "/supported_models/specialized/index": "/docs/supported-models",
+    "/supported_models/text_generation/generative_models": "/docs/supported-models/large-language-models",
+    "/supported_models/text_generation/multimodal_language_models": "/docs/supported-models/vision-language-models",
+    "/supported_models/text_generation/diffusion_language_models": "/docs/supported-models/diffusion-language-models",
+    "/supported_models/text_generation/index": "/docs/supported-models",
+    # diffusion → sglang-diffusion (file renames snake_case → kebab-case)
+    "/diffusion": "/docs/sglang-diffusion/installation",
+    "/diffusion/index": "/docs/sglang-diffusion/installation",
+    "/diffusion/installation": "/docs/sglang-diffusion/installation",
+    "/diffusion/environment_variables": "/docs/sglang-diffusion/environment-variables",
+    "/diffusion/ci_perf": "/docs/sglang-diffusion/ci-performance",
+    "/diffusion/api/cli": "/docs/sglang-diffusion/api/cli",
+    "/diffusion/api/openai_api": "/docs/sglang-diffusion/api/openai-api",
+    "/diffusion/performance/attention_backends": "/docs/sglang-diffusion/attention-backends",
+    "/diffusion/performance/cache/cache_dit": "/docs/sglang-diffusion/cache-dit",
+    "/diffusion/performance/cache/index": "/docs/sglang-diffusion/caching-acceleration",
+    "/diffusion/performance/cache/teacache": "/docs/sglang-diffusion/tea-cache",
+    "/diffusion/performance/index": "/docs/sglang-diffusion/performance-optimization",
+    "/diffusion/performance/profiling": "/docs/sglang-diffusion/profiling",
+    # Diffusion pages dropped
+    "/diffusion/api/post_processing": "/docs/sglang-diffusion/installation",
+    "/diffusion/compatibility_matrix": "/docs/sglang-diffusion/installation",
+    "/diffusion/contributing": "/docs/sglang-diffusion/installation",
+    "/diffusion/development": "/docs/sglang-diffusion/installation",
+    "/diffusion/disaggregation": "/docs/sglang-diffusion/installation",
+    "/diffusion/performance/ring_sp_performance": "/docs/sglang-diffusion/performance-optimization",
+    "/diffusion/quantization": "/docs/sglang-diffusion/installation",
+    "/diffusion/reference": "/docs/sglang-diffusion/installation",
+    "/diffusion/support_new_models": "/docs/sglang-diffusion/installation",
+    "/diffusion/usage": "/docs/sglang-diffusion/installation",
+    # basic_usage pages migrated to cookbook
+    "/basic_usage/kimi_k2_5": "/cookbook/autoregressive/Moonshotai/Kimi-K2.5",
+    "/basic_usage/deepseek_ocr": "/cookbook/autoregressive/DeepSeek/DeepSeek-OCR",
+    "/basic_usage/deepseek_v3": "/cookbook/autoregressive/DeepSeek/DeepSeek-V3",
+    "/basic_usage/deepseek_v32": "/cookbook/autoregressive/DeepSeek/DeepSeek-V3_2",
+    "/basic_usage/glm45": "/cookbook/autoregressive/GLM/GLM-4.5",
+    "/basic_usage/glmv": "/cookbook/autoregressive/GLM/GLM-4.6V",
+    "/basic_usage/gpt_oss": "/cookbook/autoregressive/OpenAI/GPT-OSS",
+    "/basic_usage/llama4": "/cookbook/autoregressive/Llama/Llama4",
+    "/basic_usage/minimax_m2": "/cookbook/autoregressive/MiniMax/MiniMax-M2",
+    "/basic_usage/popular_model_usage": "/cookbook/autoregressive/intro",
+    "/basic_usage/qwen3": "/cookbook/autoregressive/Qwen/Qwen3",
+    "/basic_usage/qwen3_5": "/cookbook/autoregressive/Qwen/Qwen3.5",
+    "/basic_usage/qwen3_vl": "/cookbook/autoregressive/Qwen/Qwen3-VL",
+    # advanced_features dropped pages
+    "/advanced_features/adaptive_speculative_decoding": "/docs/advanced_features/speculative_decoding",
+    "/advanced_features/hisparse_guide": "/docs/advanced_features/overview",
+    # references dropped
+    "/references/learn_more": "/",
+    "/references/release_lookup": "/docs/references/overview",
+    # Root index
+    "/index": "/",
+    "/": "/",
+}
+
+
+def old_url_from_path(rel: Path) -> str | None:
+    """Convert old docs/<rel> to its Sphinx URL path (no .html, leading /)."""
+    parts = list(rel.parts)
+    stem = rel.stem
+    # Skip README, release_lookup/README, top-level non-doc files
+    if stem == "README":
+        return None
+    # Drop the extension → URL path
+    new_parts = parts[:-1] + [stem]
+    return "/" + "/".join(new_parts)
+
+
+def new_url_for(old_url: str, new_files_set: set[str]) -> str | None:
+    """Compute new URL from old URL using section rename + explicit overrides."""
+    if old_url in EXPLICIT:
+        return EXPLICIT[old_url]
+    # Default rule: `/section/path` → `/docs/section/path`, applying section renames
+    parts = old_url.strip("/").split("/")
+    if not parts or not parts[0]:
+        return None
+    section = parts[0]
+    section = SECTION_RENAMES.get(section, section)
+    new_url = "/docs/" + "/".join([section] + parts[1:])
+    # Verify destination exists in new file tree
+    if new_url in new_files_set:
+        return new_url
+    return None  # unmapped
+
+
+def list_new_urls() -> set[str]:
+    urls = set()
+    for p in NEW_DOCS.rglob("*"):
+        if not p.is_file():
+            continue
+        if p.suffix not in (".mdx", ".ipynb", ".md"):
+            continue
+        rel = p.relative_to(NEW_DOCS)
+        # Mintlify routes .mdx / .ipynb as `/docs/<path-without-ext>`
+        url = "/docs/" + str(rel.with_suffix("")).replace(os.sep, "/")
+        urls.add(url)
+    return urls
+
+
+def main():
+    new_urls = list_new_urls()
+    redirects: list[dict] = []
+    seen_sources: set[str] = set()
+    unmapped: list[str] = []
+
+    # Iterate all old files
+    old_files = []
+    for p in sorted(OLD_DOCS.rglob("*")):
+        if not p.is_file():
+            continue
+        if p.suffix not in (".md", ".rst", ".ipynb"):
+            continue
+        rel = p.relative_to(OLD_DOCS)
+        # Skip non-doc dirs
+        if rel.parts and rel.parts[0] in (
+            "_static",
+            "performance_dashboard",
+            "release_lookup",
+        ):
+            continue
+        old_files.append(rel)
+
+    for rel in old_files:
+        old_url = old_url_from_path(rel)
+        if old_url is None:
+            continue
+        # Old Sphinx URLs end in .html
+        source = old_url + ".html"
+        if source in seen_sources:
+            continue
+        new_url = new_url_for(old_url, new_urls)
+        if new_url is None:
+            unmapped.append(source)
+            continue
+        redirects.append({"source": source, "destination": new_url})
+        seen_sources.add(source)
+
+    # Also add explicit entries whose source key wasn't derived from a file (e.g. index variants)
+    for old_key, new_val in EXPLICIT.items():
+        source = old_key + ".html"
+        if source in seen_sources:
+            continue
+        # Only add if old_key corresponds to an actual old page pattern we care about
+        # Skip bare "/" and "/index" (handled by Mintlify default)
+        if old_key in ("/", "/index"):
+            continue
+        redirects.append({"source": source, "destination": new_val})
+        seen_sources.add(source)
+
+    # Output
+    print(f"# Total redirects: {len(redirects)}")
+    print(f"# Unmapped old URLs: {len(unmapped)}")
+    if unmapped:
+        print("# --- UNMAPPED ---")
+        for u in unmapped:
+            print(f"#   {u}")
+    print(json.dumps(redirects, indent=2))
+
+
+if __name__ == "__main__":
+    main()

docs_new/scripts/update_lmsys_sglang_blogs.py

@@ -0,0 +1,286 @@
+#!/usr/bin/env python3
+"""Sync SGLang-related LMSYS blog cards into index.mdx."""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import urllib.request
+from dataclasses import dataclass
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+INDEX_PATH = ROOT / "index.mdx"
+
+START_MARKER = "{/* BEGIN_LMSYS_SGLANG_BLOG_CARDS */}"
+END_MARKER = "{/* END_LMSYS_SGLANG_BLOG_CARDS */}"
+
+LMSYS_BLOG_API_URL = (
+    "https://api.github.com/repos/lm-sys/lm-sys.github.io/contents/blog"
+)
+LMSYS_BLOG_BASE_URL = "https://lmsys.org/blog"
+LMSYS_BASE_URL = "https://lmsys.org"
+DEFAULT_IMAGE_URL = "https://lmsys.org/social.png"
+
+MAX_CARDS = int(os.getenv("LMSYS_SGLANG_MAX_CARDS", "6"))
+KEYWORDS = [
+    "sglang",
+    "sgl-project/sglang",
+    "sgl-kernel",
+    "sglang-jax",
+    "sgl diffusion",
+    "sglang diffusion",
+]
+
+FRONTMATTER_RE = re.compile(r"\A---\s*\n(.*?)\n---\s*\n?", flags=re.DOTALL)
+HTML_IMG_RE = re.compile(r"<img[^>]*\ssrc=[\"']([^\"']+)[\"']", flags=re.IGNORECASE)
+MD_IMG_RE = re.compile(r"!\[[^\]]*]\(([^)]+)\)")
+
+
+@dataclass
+class BlogPost:
+    slug: str
+    title: str
+    url: str
+    image: str
+    date: str
+
+
+def build_headers() -> dict[str, str]:
+    headers = {
+        "Accept": "application/vnd.github+json",
+        "User-Agent": "sgl-docs-lmsys-blog-sync",
+    }
+    token = os.getenv("GITHUB_TOKEN")
+    if token:
+        headers["Authorization"] = f"Bearer {token}"
+    return headers
+
+
+def download_blog_sources() -> list[tuple[str, str]]:
+    # Fetch the directory listing for /blog only — no need to download the whole repo.
+    request = urllib.request.Request(LMSYS_BLOG_API_URL, headers=build_headers())
+    with urllib.request.urlopen(request, timeout=60) as response:
+        items: list[dict] = json.loads(response.read())
+
+    sources: list[tuple[str, str]] = []
+    for item in items:
+        if item.get("type") != "file" or not item.get("name", "").endswith(".md"):
+            continue
+        download_url = item.get("download_url")
+        if not download_url:
+            continue
+        raw_request = urllib.request.Request(download_url, headers=build_headers())
+        with urllib.request.urlopen(raw_request, timeout=30) as raw_response:
+            content = raw_response.read().decode("utf-8", errors="replace")
+        sources.append((item["name"], content))
+
+    return sources
+
+
+def split_frontmatter(content: str) -> tuple[dict[str, str], str]:
+    match = FRONTMATTER_RE.match(content)
+    if not match:
+        return {}, content
+
+    frontmatter: dict[str, str] = {}
+    for raw_line in match.group(1).splitlines():
+        line = raw_line.strip()
+        if not line or ":" not in line:
+            continue
+
+        key, value = line.split(":", 1)
+        cleaned = value.strip()
+        if (
+            (cleaned.startswith('"') and cleaned.endswith('"'))
+            or (cleaned.startswith("'") and cleaned.endswith("'"))
+        ) and len(cleaned) >= 2:
+            cleaned = cleaned[1:-1]
+        frontmatter[key.strip()] = cleaned
+
+    return frontmatter, content[match.end() :]
+
+
+def first_image_from_body(body: str) -> str | None:
+    markdown_match = MD_IMG_RE.search(body)
+    if markdown_match:
+        candidate = markdown_match.group(1).strip()
+        if candidate.startswith("<") and candidate.endswith(">"):
+            candidate = candidate[1:-1]
+        if " " in candidate:
+            candidate = candidate.split(" ", 1)[0]
+        return candidate
+
+    html_match = HTML_IMG_RE.search(body)
+    if html_match:
+        return html_match.group(1).strip()
+
+    return None
+
+
+def to_absolute_url(url_or_path: str | None) -> str:
+    if not url_or_path:
+        return DEFAULT_IMAGE_URL
+
+    value = url_or_path.strip()
+    if value.startswith(("http://", "https://")):
+        return value
+    if value.startswith("//"):
+        return f"https:{value}"
+    return f"{LMSYS_BASE_URL}/{value.lstrip('/')}"
+
+
+def is_relevant(slug: str, title: str, body: str) -> bool:
+    searchable = f"{slug}\n{title}\n{body}".lower()
+    return any(keyword in searchable for keyword in KEYWORDS)
+
+
+def parse_blog_post(filename: str, content: str) -> BlogPost | None:
+    if not filename.endswith(".md"):
+        return None
+
+    slug = filename[:-3]
+    frontmatter, body = split_frontmatter(content)
+
+    title = frontmatter.get("title", "").strip() or slug.replace("-", " ").title()
+    preview_img = frontmatter.get("previewImg") or first_image_from_body(body)
+    image = to_absolute_url(preview_img)
+    url = f"{LMSYS_BLOG_BASE_URL}/{slug}/"
+    date = frontmatter.get("date", "").strip() or slug[:10]
+
+    if not is_relevant(slug=slug, title=title, body=body):
+        return None
+
+    return BlogPost(slug=slug, title=title, url=url, image=image, date=date)
+
+
+def render_cards(posts: list[BlogPost]) -> str:
+    if not posts:
+        return "No relevant LMSYS blog posts matched the current sync keywords."
+
+    lines = [
+        '<div className="not-prose">',
+        "  <div",
+        "    style={{",
+        '      display: "grid",',
+        '      gridTemplateColumns: "repeat(auto-fit, minmax(300px, 1fr))",',
+        '      gap: "1rem",',
+        '      alignItems: "stretch",',
+        "    }}",
+        "  >",
+    ]
+    for post in posts:
+        safe_title = json.dumps(post.title)
+        safe_url = json.dumps(post.url)
+        safe_image = json.dumps(post.image)
+        lines.extend(
+            [
+                "    <a",
+                f"      href={safe_url}",
+                '      target="_blank"',
+                '      rel="noopener noreferrer"',
+                "      style={{",
+                '        display: "block",',
+                '        border: "1px solid rgba(128, 128, 128, 0.3)",',
+                '        borderRadius: "0.75rem",',
+                '        overflow: "hidden",',
+                '        textDecoration: "none",',
+                '        color: "inherit",',
+                '        height: "100%",',
+                "      }}",
+                "    >",
+                "      <div",
+                "        style={{",
+                '          aspectRatio: "16 / 9",',
+                '          overflow: "hidden",',
+                '          background: "rgba(128, 128, 128, 0.15)",',
+                "        }}",
+                "      >",
+                "        <img",
+                f"          src={safe_image}",
+                f"          alt={safe_title}",
+                "          style={{",
+                '            width: "100%",',
+                '            height: "100%",',
+                '            objectFit: "cover",',
+                '            objectPosition: "center",',
+                '            display: "block",',
+                "          }}",
+                "        />",
+                "      </div>",
+                '      <div style={{ padding: "0.9rem 1rem 1rem" }}>',
+                "        <p",
+                "          style={{",
+                "            margin: 0,",
+                "            fontWeight: 600,",
+                "            lineHeight: 1.35,",
+                '            fontSize: "0.98rem",',
+                "          }}",
+                "        >",
+                f"          {{{safe_title}}}",
+                "        </p>",
+                "        <p",
+                "          style={{",
+                '            margin: "0.55rem 0 0",',
+                '            fontSize: "0.85rem",',
+                "            opacity: 0.75,",
+                "          }}",
+                "        >",
+                f"          {{{json.dumps(post.date)}}}",
+                "        </p>",
+                "      </div>",
+                "    </a>",
+            ]
+        )
+    lines.extend(["  </div>", "</div>"])
+    return "\n".join(lines)
+
+
+def replace_generated_block(index_text: str, generated_cards: str) -> str:
+    pattern = re.compile(
+        rf"{re.escape(START_MARKER)}.*?{re.escape(END_MARKER)}",
+        flags=re.DOTALL,
+    )
+    replacement = f"{START_MARKER}\n{generated_cards}\n{END_MARKER}"
+    updated_text, replacements = pattern.subn(
+        lambda _match: replacement, index_text, count=1
+    )
+    if replacements != 1:
+        raise RuntimeError(
+            f"Could not find exactly one marker block in {INDEX_PATH.name}. "
+            f"Expected markers: {START_MARKER} ... {END_MARKER}"
+        )
+    return updated_text
+
+
+def main() -> None:
+    sources = download_blog_sources()
+    relevant_posts: list[BlogPost] = []
+
+    for filename, content in sources:
+        post = parse_blog_post(filename=filename, content=content)
+        if post is not None:
+            relevant_posts.append(post)
+
+    relevant_posts.sort(key=lambda post: post.slug, reverse=True)
+    selected_posts = relevant_posts[:MAX_CARDS]
+
+    generated_cards = render_cards(selected_posts)
+    current_index = INDEX_PATH.read_text(encoding="utf-8")
+    updated_index = replace_generated_block(
+        index_text=current_index, generated_cards=generated_cards
+    )
+
+    if updated_index != current_index:
+        INDEX_PATH.write_text(updated_index, encoding="utf-8")
+
+    print(
+        "Scanned "
+        f"{len(sources)} blog files, matched {len(relevant_posts)} posts, "
+        f"published {len(selected_posts)} cards."
+    )
+
+
+if __name__ == "__main__":
+    main()