#!/usr/bin/env python3 """ Fetch SDK README files from GitHub and write them into docs/documentation/clientsdks/. Run as part of the docs build or manually: python3 scripts/fetch-sdk-docs.py Each SDK gets a markdown file with: - Front matter (description) - The README content with relative paths rewritten to absolute GitHub URLs - An examples table at the bottom (if the repo has an examples directory) """ import json import os import re import sys import urllib.request import urllib.error GITHUB_ORG = "conductor-oss" GITHUB_API = "https://api.github.com" GITHUB_RAW = "https://raw.githubusercontent.com" GITHUB_BLOB = "https://github.com" DOCS_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "docs", "documentation", "clientsdks") # SDK definitions: (repo, output_filename, display_name, examples_dir, description) SDKS = [ ("java-sdk", "java-sdk.md", "Java", "examples", "Build Conductor workers in Java with automated polling, thread management, and Spring Boot integration."), ("python-sdk", "python-sdk.md", "Python", "examples", "Build Conductor workers in Python with decorator-based task definitions, async support, and workflow management."), ("go-sdk", "go-sdk.md", "Go", "examples", "Build Conductor workers in Go with type-safe task definitions and workflow management."), ("javascript-sdk", "js-sdk.md", "JavaScript", "examples", "Build Conductor workers in JavaScript/TypeScript with workflow management and task polling."), ("csharp-sdk", "csharp-sdk.md", "C#", "csharp-examples", "Build Conductor workers in C#/.NET with dependency injection, workflow management, and task polling."), ("ruby-sdk", "ruby-sdk.md", "Ruby", "examples", "Build Conductor workers in Ruby with idiomatic task definitions and workflow management."), ("rust-sdk", "rust-sdk.md", "Rust", "examples", "Build Conductor workers in Rust with type-safe task definitions and async workflow management."), ] def github_get(url): """Fetch a URL from GitHub API or raw content.""" req = urllib.request.Request(url) token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") if not token: # Try gh CLI as fallback try: import subprocess token = subprocess.check_output(["gh", "auth", "token"], stderr=subprocess.DEVNULL).decode().strip() except Exception: pass if token: req.add_header("Authorization", f"token {token}") req.add_header("Accept", "application/vnd.github.v3+json") req.add_header("User-Agent", "conductor-docs-builder") try: with urllib.request.urlopen(req, timeout=30) as resp: return resp.read().decode("utf-8") except urllib.error.HTTPError as e: print(f" WARNING: HTTP {e.code} fetching {url}", file=sys.stderr) return None except Exception as e: print(f" WARNING: {e} fetching {url}", file=sys.stderr) return None def fetch_readme(repo): """Fetch the raw README.md from a GitHub repo.""" url = f"{GITHUB_RAW}/{GITHUB_ORG}/{repo}/main/README.md" content = github_get(url) if content is None: # Try master branch url = f"{GITHUB_RAW}/{GITHUB_ORG}/{repo}/master/README.md" content = github_get(url) return content def rewrite_paths(content, repo): """Rewrite relative paths in markdown to point to GitHub.""" base_raw = f"{GITHUB_RAW}/{GITHUB_ORG}/{repo}/main" base_blob = f"{GITHUB_BLOB}/{GITHUB_ORG}/{repo}/blob/main" # Rewrite image references: ![alt](relative/path.png) # Don't touch absolute URLs (http://, https://) content = re.sub( r'(!\[[^\]]*\]\()(?!https?://|//)([^)]+)(\))', lambda m: f'{m.group(1)}{base_raw}/{m.group(2)}{m.group(3)}', content ) # Rewrite link references: [text](relative/path) but not anchors (#) or absolute URLs content = re.sub( r'(\[[^\]]*\]\()(?!https?://|//|#)([^)]+)(\))', lambda m: f'{m.group(1)}{base_blob}/{m.group(2)}{m.group(3)}', content ) # Rewrite nested badge links: [![...](badge-url)](relative-path) # The main regex can't handle nested brackets, so handle this separately content = re.sub( r'(\]\()(?!https?://|//|#)([^)]+)(\))\s*$', lambda m: f'{m.group(1)}{base_blob}/{m.group(2)}{m.group(3)}', content, flags=re.MULTILINE ) # Rewrite HTML img src attributes content = re.sub( r'(]+src=")(?!https?://|//)([^"]+)(")', lambda m: f'{m.group(1)}{base_raw}/{m.group(2)}{m.group(3)}', content ) return content def strip_title(content): """Remove the first H1 heading (# Title) since we add our own.""" lines = content.split('\n') for i, line in enumerate(lines): stripped = line.strip() if stripped.startswith('# ') and not stripped.startswith('## '): lines.pop(i) # Also remove blank line after title if present if i < len(lines) and lines[i].strip() == '': lines.pop(i) break return '\n'.join(lines) def strip_preamble(content): """Remove badges, intro blurb, and star-request that appear before the first ## heading.""" lines = content.split('\n') first_h2 = None for i, line in enumerate(lines): if line.strip().startswith('## '): first_h2 = i break if first_h2 is None: return content # Keep everything from the first ## heading onward return '\n'.join(lines[first_h2:]) def strip_toc(content): """Remove HTML comment TOC blocks ( ... ) from README.""" content = re.sub(r'.*?', '', content, flags=re.DOTALL) # Also remove markdown-style TOC (indented list of anchor links at the top) # These are blocks of lines that are all " * [text](#anchor)" or "- [text](#anchor)" lines = content.split('\n') result = [] in_toc = False for line in lines: stripped = line.strip() if stripped and re.match(r'^[-*]\s+\[.*\]\(#', stripped): in_toc = True continue if in_toc and stripped == '': in_toc = False continue if in_toc and re.match(r'^[-*]\s+\[.*\]\(#', stripped): continue in_toc = False result.append(line) return '\n'.join(result) def fetch_examples(repo, examples_dir): """Fetch the list of examples from a repo's examples directory.""" if not examples_dir: return [] url = f"{GITHUB_API}/repos/{GITHUB_ORG}/{repo}/contents/{examples_dir}" raw = github_get(url) if raw is None: return [] try: items = json.loads(raw) except json.JSONDecodeError: return [] if not isinstance(items, list): return [] examples = [] for item in sorted(items, key=lambda x: x.get("name", "")): name = item.get("name", "") item_type = item.get("type", "") html_url = item.get("html_url", "") # Skip non-interesting files if name.startswith(".") or name.startswith("__"): continue if name in ("go.mod", "go.sum", "build.gradle", "settings.gradle", "pom.xml", "Cargo.toml", "package.json", "tsconfig.json", "Dockerfile", "DockerfileMacArm", ".gitignore", "csharp-examples.csproj"): continue if name == "Properties": continue # Format display name from filename display = name if item_type == "file": # Remove extension for display display = os.path.splitext(name)[0] # Convert snake_case/kebab-case to readable display = display.replace("_", " ").replace("-", " ").title() if item_type == "dir": display = name.replace("_", " ").replace("-", " ").title() examples.append((display, html_url, item_type)) return examples def build_examples_table(repo, examples_dir): """Build a markdown table of examples.""" examples = fetch_examples(repo, examples_dir) if not examples: return "" base_url = f"{GITHUB_BLOB}/{GITHUB_ORG}/{repo}/tree/main/{examples_dir}" lines = [ "", "## Examples", "", f"Browse all examples on GitHub: [{GITHUB_ORG}/{repo}/{examples_dir}]({base_url})", "", "| Example | Type |", "|---|---|", ] for display, url, item_type in examples: icon = "directory" if item_type == "dir" else "file" lines.append(f"| [{display}]({url}) | {icon} |") lines.append("") return "\n".join(lines) def process_sdk(repo, output_file, display_name, examples_dir, description): """Fetch README, process it, and write the output file.""" print(f"Fetching {display_name} SDK from {GITHUB_ORG}/{repo}...") readme = fetch_readme(repo) if readme is None: print(f" SKIP: Could not fetch README for {repo}", file=sys.stderr) return False # Process content readme = strip_title(readme) readme = strip_toc(readme) readme = strip_preamble(readme) readme = rewrite_paths(readme, repo) # Build examples section examples_section = build_examples_table(repo, examples_dir) # Compose final markdown repo_url = f"{GITHUB_BLOB}/{GITHUB_ORG}/{repo}" output = f"""--- description: "{description}" --- # {display_name} SDK !!! info "Source" GitHub: [{GITHUB_ORG}/{repo}]({repo_url}) | Report issues and contribute on GitHub. {readme} {examples_section}""" # Write output output_path = os.path.join(DOCS_DIR, output_file) with open(output_path, "w") as f: f.write(output) print(f" Wrote {output_path}") return True def main(): os.makedirs(DOCS_DIR, exist_ok=True) success = 0 for repo, output_file, display_name, examples_dir, description in SDKS: if process_sdk(repo, output_file, display_name, examples_dir, description): success += 1 print(f"\nDone: {success}/{len(SDKS)} SDK docs generated.") if success < len(SDKS): print("Some SDKs could not be fetched. Check warnings above.", file=sys.stderr) if __name__ == "__main__": main()