chore: import upstream snapshot with attribution

packages/transformers/docs/scripts/build_readme.py

@@ -0,0 +1,108 @@
+
+import re
+README_TEMPLATE = """
+
+<p align="center">
+    <br/>
+    <picture> 
+        <source media="(prefers-color-scheme: dark)" srcset="https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/transformersjs-dark.svg" width="500" style="max-width: 100%;">
+        <source media="(prefers-color-scheme: light)" srcset="https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/transformersjs-light.svg" width="500" style="max-width: 100%;">
+        <img alt="transformers.js javascript library logo" src="https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/transformersjs-light.svg" width="500" style="max-width: 100%;">
+    </picture>
+    <br/>
+</p>
+
+<p align="center">
+    <a href="https://www.npmjs.com/package/@huggingface/transformers"><img alt="NPM" src="https://img.shields.io/npm/v/@huggingface/transformers"></a>
+    <a href="https://www.npmjs.com/package/@huggingface/transformers"><img alt="NPM Downloads" src="https://img.shields.io/npm/dw/@huggingface/transformers"></a>
+    <a href="https://www.jsdelivr.com/package/npm/@huggingface/transformers"><img alt="jsDelivr Hits" src="https://img.shields.io/jsdelivr/npm/hw/@huggingface/transformers"></a>
+    <a href="https://github.com/huggingface/transformers.js/blob/main/LICENSE"><img alt="License" src="https://img.shields.io/github/license/huggingface/transformers.js?color=blue"></a>
+    <a href="https://huggingface.co/docs/transformers.js/index"><img alt="Documentation" src="https://img.shields.io/website/http/huggingface.co/docs/transformers.js/index.svg?down_color=red&down_message=offline&up_message=online"></a>
+</p>
+
+{intro}
+
+## Installation
+
+{installation}
+
+## Quick tour
+
+{quick_tour}
+
+## Custom usage
+
+{custom_usage}
+
+## Supported tasks/models
+
+Here is the list of all tasks and architectures currently supported by Transformers.js. If you don't see your task/model listed here or it is not yet supported, feel free to open up a feature request [here](https://github.com/huggingface/transformers.js/issues/new/choose).
+
+To find compatible models on the Hub, select the "transformers.js" library tag in the filter menu (or visit [this link](https://huggingface.co/models?library=transformers.js)). You can refine your search by selecting the task you're interested in (e.g., [text-classification](https://huggingface.co/models?pipeline_tag=text-classification&library=transformers.js)).
+
+{tasks}
+
+{models}
+"""
+
+
+FILES_TO_INCLUDE = dict(
+    intro='./docs/snippets/0_introduction.snippet',
+    quick_tour='./docs/snippets/1_quick-tour.snippet',
+    installation='./docs/snippets/2_installation.snippet',
+    custom_usage='./docs/snippets/3_custom-usage.snippet',
+    tasks='./docs/snippets/4_supported-tasks.snippet',
+    models='./docs/snippets/5_supported-models.snippet',
+)
+
+DOCS_BASE_URL = 'https://huggingface.co/docs/transformers.js'
+
+# Map of custom links to replace, typically used for links to other sections of the README.
+CUSTOM_LINK_MAP = {
+    '/custom_usage#convert-your-models-to-onnx': '#convert-your-models-to-onnx',
+    './api/env': DOCS_BASE_URL + '/api/env',
+    './guides/webgpu': DOCS_BASE_URL + '/guides/webgpu',
+    './guides/dtypes': DOCS_BASE_URL + '/guides/dtypes',
+}
+
+
+def main():
+
+    file_data = {}
+    for key, file_path in FILES_TO_INCLUDE.items():
+        with open(file_path, encoding='utf-8') as f:
+            file_data[key] = f.read()
+
+    # Fix links:
+    # NOTE: This regex does not match all markdown links, but works for the ones we need to replace.
+    LINK_RE = r'(?<=\])\((.+?)\)'
+
+    def replace_fn(match):
+        link = match.group(1)
+
+        if link in CUSTOM_LINK_MAP:
+            link = CUSTOM_LINK_MAP[link]
+
+        elif link.startswith('/'):
+            # Link to docs
+            link = DOCS_BASE_URL + link
+
+        elif link.startswith('./'):
+            # Relative link to file
+            pass
+
+        elif link.startswith('http'):
+            # Link to external site
+            pass
+
+        return f'({link})'
+
+    result = README_TEMPLATE.format(**file_data)
+    result = re.sub(LINK_RE, replace_fn, result, count=0, flags=re.MULTILINE)
+
+    with open('README.md', 'w', encoding='utf-8') as f:
+        f.write(result)
+
+
+if __name__ == '__main__':
+    main()

packages/transformers/docs/scripts/generate.js

@@ -0,0 +1,79 @@
+// Based on [this tutorial](https://github.com/jsdoc2md/jsdoc-to-markdown/wiki/How-to-create-one-output-file-per-class).
+
+import fs from "node:fs";
+import path from "node:path";
+import url from "node:url";
+
+import jsdoc2md from "jsdoc-to-markdown";
+
+const docs = path.dirname(path.dirname(url.fileURLToPath(import.meta.url)));
+const root = path.dirname(docs);
+
+// jsdoc config file
+const conf = path.join(docs, "jsdoc-conf.json");
+
+// input and output paths
+const inputFile = path.join(root, "/src/**/*.js");
+const outputDir = path.join(root, "/docs/source/api/");
+
+// get template data
+const templateData = await jsdoc2md.getTemplateData({
+  files: inputFile,
+  configure: conf,
+  "no-cache": true,
+});
+
+// reduce templateData to an array of module names
+const moduleNames = templateData.reduce((moduleNames, identifier) => {
+  if (identifier.kind === "module") {
+    moduleNames.push(identifier.name);
+  }
+  return moduleNames;
+}, []);
+
+// Clear all existing .md files from output directory (recursively)
+if (fs.existsSync(outputDir)) {
+  const existingFiles = fs.readdirSync(outputDir, { recursive: true });
+  for (const file of existingFiles) {
+    if (file.endsWith(".md")) {
+      fs.unlinkSync(path.join(outputDir, file));
+    }
+  }
+}
+
+// create a documentation file for each module
+for (const moduleName of moduleNames) {
+  const template = `{{#module name="${moduleName}"}}{{>docs}}{{/module}}`;
+  console.log(`rendering ${moduleName}, template: ${template}`);
+  let output = await jsdoc2md.render({
+    data: templateData,
+    template: template,
+    "heading-depth": 1,
+    "no-gfm": true,
+    "name-format": "backticks",
+    "no-cache": true,
+    separators: true,
+    configure: conf,
+  });
+
+  // Post-processing
+  output = output.replace(/(^#+\s.+)/gm, "$1\n"); // Add new line after each header
+
+  // Remove <code> tags from headers
+  output = output.replace(/^#+\s.+$/gm, (match) => match.replace(/<\/?code>/g, ""));
+
+  // Replace all generated marker names with ids (for linking), and add group class
+  output = output.replace(/<a name="(\S+)"><\/a>/g, '<a id="$1" class="group"></a>');
+
+  // Unescape some of the characters which jsdoc2md escapes:
+  // TODO: May need to extend this list
+  output = output.replace(/\\([|_&*])/gm, "$1");
+
+  output = output.replaceAll("new exports.", "new ");
+
+  const outputPath = path.resolve(outputDir, `${moduleName}.md`);
+
+  console.log(`Writing to ${outputPath}`);
+  fs.mkdirSync(path.dirname(outputPath), { recursive: true });
+  fs.writeFileSync(outputPath, output);
+}