chore: import upstream snapshot with attribution

docs/scripts/build-all.py

@@ -0,0 +1,92 @@
+import os
+import shutil
+import subprocess
+from pathlib import Path
+
+import click
+
+import mlflow
+
+mlflow_version = mlflow.version.VERSION
+
+
+def build_docs(package_manager, version):
+    env = os.environ.copy()
+
+    # ensure it ends with a "/"
+    base_url = env.get("DOCS_BASE_URL", "/docs/").rstrip("/") + "/"
+    api_reference_prefix = (
+        env.get("API_REFERENCE_PREFIX", "https://mlflow.org/docs/").rstrip("/") + "/"
+    )
+
+    output_path = Path(f"_build/{version}")
+    versioned_url = Path(f"{base_url}{version}")
+    build_path = Path("build")
+
+    print(f"Building for `{versioned_url}`...")
+
+    if output_path.exists():
+        shutil.rmtree(output_path)
+
+    if build_path.exists():
+        shutil.rmtree(build_path)
+
+    subprocess.check_call(
+        package_manager + ["build"],
+        env={
+            **env,
+            "DOCS_BASE_URL": str(versioned_url),
+            "API_REFERENCE_PREFIX": f"{api_reference_prefix}{version}",
+            **({"DOCS_NO_INDEX": "true"} if version != "latest" else {}),
+        },
+    )
+    shutil.copytree(build_path, output_path)
+
+
+@click.command()
+@click.option(
+    "--use-npm",
+    "use_npm",
+    is_flag=True,
+    default=False,
+    help="Whether or not to use NPM as a package manager (in case yarn in unavailable)",
+)
+@click.option(
+    "--no-r",
+    "no_r",
+    is_flag=True,
+    default=False,
+    help="Whether or not to skip building R documentation.",
+)
+def main(use_npm, no_r):
+    gtm_id = os.environ.get("GTM_ID")
+
+    assert gtm_id, (
+        "Google Tag Manager ID is missing, please ensure that the GTM_ID environment variable is set"
+    )
+
+    package_manager = ["npm", "run"] if use_npm else ["yarn"]
+    build_command = ["build-api-docs:no-r"] if no_r else ["build-api-docs"]
+
+    subprocess.check_call(package_manager + build_command)
+    subprocess.check_call(package_manager + ["convert-notebooks"])
+
+    output_path = Path("_build")
+    if output_path.exists():
+        shutil.rmtree(output_path)
+    output_path.mkdir(parents=True, exist_ok=True)
+
+    for v in [mlflow_version, "latest"]:
+        build_docs(package_manager, v)
+
+    final_path = Path("build")
+    if final_path.exists():
+        shutil.rmtree(final_path)
+
+    shutil.move(output_path, final_path)
+
+    print("Finished building! Output can be found in the `build` directory")
+
+
+if __name__ == "__main__":
+    main()

docs/scripts/build-api-docs.py

@@ -0,0 +1,43 @@
+import os
+import shutil
+import subprocess
+
+import click
+
+
+@click.command()
+@click.option("--with-r", "with_r", is_flag=True, default=False, help="Build R documentation")
+@click.option(
+    "--with-ts", "with_ts", is_flag=True, default=True, help="Build TypeScript documentation"
+)
+def main(with_r, with_ts):
+    try:
+        # Run "make rsthtml" in "api_reference" subfolder
+        print("Building API reference documentation...")
+        subprocess.run(["make", "clean"], check=True, cwd="api_reference")
+        subprocess.run(["make", "rsthtml"], check=True, cwd="api_reference")
+        subprocess.run(["make", "javadocs"], check=True, cwd="api_reference")
+        if with_r:
+            subprocess.run(["make", "rdocs"], check=True, cwd="api_reference")
+        if with_ts:
+            subprocess.run(["make", "tsdocs"], check=True, cwd="api_reference")
+        print("Build successful.")
+    except subprocess.CalledProcessError as e:
+        print(f"Build failed: {e}")
+        exit(1)
+
+    destination_folder = "static/api_reference"
+    source_folder = "api_reference/build/html"
+
+    # Remove the destination folder if it exists
+    if os.path.exists(destination_folder):
+        shutil.rmtree(destination_folder)
+        print(f"Removed existing static API docs at {destination_folder}.")
+
+    # Copy the contents of "api_reference/build/html" to "static/api_reference"
+    shutil.copytree(source_folder, destination_folder)
+    print(f"Copied files from {source_folder} to {destination_folder}.")
+
+
+if __name__ == "__main__":
+    main()

docs/scripts/check-links.mts

@@ -0,0 +1,81 @@
+import markdownLinkCheck from 'markdown-link-check';
+import { glob } from 'glob';
+import * as fs from 'fs/promises';
+
+async function main() {
+  let encounteredBrokenLinks = false;
+
+  const checkExternalLinks = process.env.CHECK_EXTERNAL_LINKS === 'true';
+  for (const filename of await glob('docs/**/*.mdx')) {
+    console.log('[CHECKING FILE]', filename);
+
+    const content = await fs.readFile(filename, 'utf8');
+    const brokenLinks = await check(content, checkExternalLinks);
+
+    if (brokenLinks.length > 0) {
+      console.log('[BROKEN LINKS]');
+      brokenLinks.forEach((result) => console.log(`${result.link} ${result.statusCode}`));
+      encounteredBrokenLinks = true;
+    } else {
+      console.log('[NO BROKEN LINKS]');
+    }
+  }
+
+  if (encounteredBrokenLinks) {
+    console.error('Found some broken links!');
+    process.exit(1);
+  }
+}
+
+await main();
+
+type Result = {
+  link: string;
+  status: 'alive' | 'dead' | 'ignored';
+  statusCode: number;
+  err: Error | null;
+};
+
+async function check(content: string, checkExternalLinks: boolean): Promise<Result[]> {
+  return new Promise((resolve, reject) => {
+    const config = {
+      httpHeaders: [
+        {
+          urls: ['https://openai.com', 'https://platform.openai.com'],
+          headers: {
+            'User-Agent':
+              'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.5 Safari/605.1.15',
+          },
+        },
+      ],
+      ignorePatterns: checkExternalLinks
+        ? [
+            { pattern: '^(?!http)' }, // relative links
+            { pattern: '^http:\/\/127\.0\.0\.1' }, // local dev
+            { pattern: '^http:\/\/localhost' },
+            { pattern: '^https:\/\/YOUR_DATABRICKS_HOST' },
+          ]
+        : [
+            { pattern: '^(?!https?:\/\/(www\.)?mlflow.org|https:\/\/(www\.)?github\.com\/mlflow\/mlflow)' }, // internal links or mlflow/mlflow repo
+          ],
+    };
+
+    markdownLinkCheck(content, config, function (err: Error | null, results: Result[]) {
+      const brokenLinks = [];
+
+      if (err) {
+        console.error('Error', err);
+        reject(err);
+      } else {
+        results.forEach(function (result: Result) {
+          console.info(`[INFO] ${result.link} is ${result.status} ${result.statusCode}`);
+          if (result.status === 'dead') {
+            brokenLinks.push(result);
+          }
+        });
+
+        resolve(brokenLinks);
+      }
+    });
+  });
+}

docs/scripts/compare-sitemaps.ts

@@ -0,0 +1,78 @@
+import * as fs from 'fs';
+import { XMLParser } from 'fast-xml-parser';
+import fetch from 'node-fetch';
+
+async function readSitemap(input: string): Promise<string> {
+  if (/^https?:\/\//.test(input)) {
+    const res = await fetch(input);
+    if (!res.ok) {
+      const responseBody = await res.text();
+      throw new Error(`Failed to fetch ${input}: ${res.status} ${res.statusText}. Response body: ${responseBody}`);
+    }
+    return await res.text();
+  } else {
+    return await fs.promises.readFile(input, 'utf8');
+  }
+}
+
+function normalizePath(url: string): string {
+  const idx = url.indexOf('/latest/');
+  return idx >= 0 ? url.slice(idx + 'latest/'.length) : url;
+}
+
+async function parseSitemap(input: string): Promise<Map<string, string>> {
+  const xml = await readSitemap(input);
+  const parser = new XMLParser();
+  const parsed = parser.parse(xml);
+  const urlset = parsed.urlset.url as { loc: string }[];
+
+  const urlMap = new Map<string, string>();
+  for (const { loc } of urlset) {
+    const normalized = normalizePath(loc);
+    urlMap.set(normalized, loc);
+  }
+  return urlMap;
+}
+
+function compareSitemaps(mapA: Map<string, string>, mapB: Map<string, string>) {
+  const onlyInA: string[] = [];
+  const onlyInB: string[] = [];
+  const inBoth: string[] = [];
+
+  for (const [url, _] of mapA) {
+    if (!mapB.has(url)) {
+      onlyInA.push(url);
+    } else {
+      inBoth.push(url);
+    }
+  }
+
+  for (const url of mapB.keys()) {
+    if (!mapA.has(url)) {
+      onlyInB.push(url);
+    }
+  }
+
+  return { onlyInA, onlyInB, inBoth };
+}
+
+(async () => {
+  const fileA = process.argv[2];
+  const fileB = process.argv[3];
+
+  if (!fileA || !fileB) {
+    console.error('Usage: tsx compare-sitemaps.ts <fileA|urlA> <fileB|urlB>');
+    process.exit(1);
+  }
+
+  const [mapA, mapB] = await Promise.all([parseSitemap(fileA), parseSitemap(fileB)]);
+
+  const { onlyInA, onlyInB, inBoth } = compareSitemaps(mapA, mapB);
+
+  console.log(`URLs in both: ${inBoth.length}`);
+  console.log(`Only in ${fileA}: ${onlyInA.length}`);
+  onlyInA.forEach((url) => console.log(`  ${url}`));
+
+  console.log(`Only in ${fileB}: ${onlyInB.length}`);
+  onlyInB.forEach((url) => console.log(`  ${url}`));
+})();

docs/scripts/convert-notebooks.py

@@ -0,0 +1,117 @@
+"""
+Converts all .ipynb files from the docs/ folder into .mdx files.
+
+This script uses nbconvert to do the processing.
+"""
+
+import multiprocessing
+import re
+from pathlib import Path
+
+import nbformat
+import yaml
+from nbconvert.exporters import MarkdownExporter
+from nbconvert.preprocessors import Preprocessor
+
+SOURCE_DIR = Path("docs/")
+NOTEBOOK_BASE_EDIT_URL = "https://github.com/mlflow/mlflow/edit/master/docs/"
+NOTEBOOK_BASE_DOWNLOAD_URL = "https://raw.githubusercontent.com/mlflow/mlflow/master/docs/"
+
+
+class EscapeBackticksPreprocessor(Preprocessor):
+    def preprocess_cell(self, cell, resources, cell_index):
+        if cell.cell_type == "code":
+            # escape backticks, as code blocks will be rendered
+            # inside a custom react component like:
+            # <NotebookCellOutput>`{{ content }}`</NotebookCellOutput>
+            # and having the backticks causes issues
+            cell.source = cell.source.replace("`", r"\`")
+
+            if "outputs" in cell:
+                for i, output in enumerate(cell["outputs"]):
+                    if "text" in output:
+                        output["text"] = output["text"].replace("`", r"\`")
+                    elif "data" in output:
+                        for key, value in output["data"].items():
+                            if isinstance(value, str):
+                                output["data"][key] = value.replace("`", r"\`")
+        elif cell.cell_type == "raw":
+            cell.source = cell.source.replace("<br>", "<br />")
+
+        return cell, resources
+
+
+exporter = MarkdownExporter(
+    preprocessors=[EscapeBackticksPreprocessor],
+    template_name="mdx",
+    extra_template_basedirs=["./scripts/nbconvert_templates"],
+)
+
+
+def add_frontmatter(
+    body: str,
+    nb_path: Path,
+) -> str:
+    frontmatter = {
+        "custom_edit_url": NOTEBOOK_BASE_EDIT_URL + str(nb_path),
+        "slug": nb_path.stem,
+    }
+    formatted_frontmatter = yaml.dump(frontmatter)
+
+    return f"""---
+{formatted_frontmatter}
+---
+
+{body}"""
+
+
+def add_download_button(
+    body: str,
+    nb_path: Path,
+) -> str:
+    download_url = NOTEBOOK_BASE_DOWNLOAD_URL + str(nb_path)
+    download_button = f'<NotebookDownloadButton href="{download_url}">Download this notebook</NotebookDownloadButton>'
+
+    # Insert the notebook underneath the first H1 header (assumed to be the title)
+    pattern = r"(^#\s+.+$)"
+    return re.sub(pattern, rf"\1\n\n{download_button}", body, count=1, flags=re.MULTILINE)
+
+
+# add the imports for our custom cell output components
+def add_custom_component_imports(
+    body: str,
+) -> str:
+    return f"""import {{ NotebookCodeCell }} from "@site/src/components/NotebookCodeCell"
+import {{ NotebookCellOutput }} from "@site/src/components/NotebookCellOutput"
+import {{ NotebookHTMLOutput }} from "@site/src/components/NotebookHTMLOutput"
+import {{ NotebookDownloadButton }} from "@site/src/components/NotebookDownloadButton"
+
+{body}
+"""
+
+
+def convert_path(nb_path: Path):
+    mdx_path = nb_path.with_stem(nb_path.stem + "-ipynb").with_suffix(".mdx")
+    with open(nb_path) as f:
+        nb = nbformat.read(f, as_version=4)
+
+        body, _ = exporter.from_notebook_node(nb)
+        body = add_custom_component_imports(body)
+        body = add_frontmatter(body, nb_path)
+        body = add_download_button(body, nb_path)
+
+        with open(mdx_path, "w") as f:
+            f.write(body)
+
+        return mdx_path
+
+
+def main():
+    nb_paths = list(SOURCE_DIR.rglob("*.ipynb"))
+
+    with multiprocessing.Pool() as pool:
+        pool.map(convert_path, nb_paths)
+
+
+if __name__ == "__main__":
+    main()

docs/scripts/nbconvert_templates/mdx/conf.json

@@ -0,0 +1,5 @@
+{
+  "mimetypes": {
+    "text/markdown": true
+  }
+}

docs/scripts/nbconvert_templates/mdx/index.md.j2

@@ -0,0 +1,41 @@
+{% extends 'markdown/index.md.j2' %}
+
+{% block input %}
+<NotebookCodeCell executionCount={ {{ "\" \"" if cell.execution_count is none else cell.execution_count }} }>
+{`{{ cell.source}}`}
+</NotebookCodeCell>{{ "\n" }}
+{% endblock input %}
+
+{%- block traceback_line -%}
+<NotebookCellOutput isStderr>
+{`{{ line.rstrip() | strip_ansi }}`}
+</NotebookCellOutput>{{ "\n" }}
+{%- endblock traceback_line -%}
+
+{%- block stream -%}
+<NotebookCellOutput {%- if output.name == "stderr" -%} {{ " isStderr" }} {%- endif %}>
+{`{{ output.text.rstrip() }}`}
+</NotebookCellOutput>{{ "\n" }}
+{%- endblock stream -%}
+
+{%- block data_text scoped -%}
+<NotebookCellOutput>
+{`{{ output.data['text/plain'].rstrip() }}`}
+</NotebookCellOutput>{{ "\n" }}
+{%- endblock data_text -%}
+
+{%- block data_html scoped -%}
+<NotebookHTMLOutput>
+  <div dangerouslySetInnerHTML={% raw %}{{{% endraw %} __html: `{{ output.data['text/html'] | safe }}`{% raw %}}}{% endraw %} />
+</NotebookHTMLOutput>{{ "\n" }}
+{%- endblock data_html -%}
+
+{%- block data_jpg scoped -%}
+![](data:image/jpg;base64,{{ output.data['image/jpeg'] }})
+
+{%- endblock data_jpg -%}
+
+{%- block data_png scoped -%}
+![](data:image/png;base64,{{ output.data['image/png'] }})
+
+{%- endblock data_png -%}

docs/scripts/update-api-modules.ts

@@ -0,0 +1,27 @@
+import * as fs from 'fs';
+import path, { basename } from 'path';
+
+const PYTHON_API_PATH = 'api_reference/source/python_api/';
+const files = fs.readdirSync(PYTHON_API_PATH, { withFileTypes: false });
+const fileMap = {};
+
+files.forEach((file) => {
+  if (file.startsWith('mlflow') && file.endsWith('.rst')) {
+    const filename = basename(file, '.rst');
+    // the eventual website path
+    fileMap[filename] = 'api_reference/python_api/' + filename + '.html';
+  }
+});
+
+// manual mapping for auth since it's a special case in the docs hierarchy
+fileMap['mlflow.server.auth'] = 'api_reference/auth/python-api.html';
+fileMap['mlflow.server.cli'] = 'api_reference/cli.html';
+fileMap['mlflow.r'] = 'api_reference/R-api.html';
+fileMap['mlflow.java'] = 'api_reference/java_api/index.html';
+fileMap['mlflow.python'] = 'api_reference/python_api/index.html';
+fileMap['mlflow.rest'] = 'api_reference/rest-api.html';
+fileMap['mlflow.typescript'] = 'api_reference/typescript_api/index.html';
+fileMap['mlflow.llms.deployments.api'] = 'api_reference/llms/deployments/api.html';
+
+// write filemap to json file
+fs.writeFileSync('src/api_modules.json', JSON.stringify(fileMap, null, 2));